W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.35-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC a01ae90d80 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.35-20260428-045924
Source: Original Platform
2026-05-26 11:34:17 +08:00

15707 lines
589 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.52790069580078,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.21088409423828,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.017366409301757812,
"KL/mean": 0.014461815357208252,
"KL/rejected_KL_mean": 0.01155853271484375,
"KL/std": 0.2480686902999878,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.005803465843200684,
"fcm_dpo/q_t": 0.499855101108551,
"grad_norm": 70.91714477539062,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.4816860556602478,
"logits/rejected": -0.4422453045845032,
"logps/chosen": -60.964229583740234,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.66104125976562,
"loss": 1.386,
"margin_dpo/margin_mean": 0.0058036744594573975,
"margin_dpo/margin_std": 0.34888899326324463,
"step": 3
},
{
"KL/chosen_KL_mean": -0.053741455078125,
"KL/mean": 0.010254994034767151,
"KL/rejected_KL_mean": 0.074249267578125,
"KL/std": 0.3106370270252228,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.12799236178398132,
"fcm_dpo/q_t": 0.5031979084014893,
"grad_norm": 72.49424743652344,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.46865934133529663,
"logits/rejected": -0.4410114884376526,
"logps/chosen": -56.82145309448242,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.57286071777344,
"loss": 1.3996,
"margin_dpo/margin_mean": -0.12799254059791565,
"margin_dpo/margin_std": 0.42509347200393677,
"step": 4
},
{
"KL/chosen_KL_mean": 0.016374588012695312,
"KL/mean": 0.007320374250411987,
"KL/rejected_KL_mean": -0.00173187255859375,
"KL/std": 0.25842905044555664,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.018107712268829346,
"fcm_dpo/q_t": 0.4995471239089966,
"grad_norm": 89.32853698730469,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5146687030792236,
"logits/rejected": -0.4709242582321167,
"logps/chosen": -53.84300231933594,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.15091705322266,
"loss": 1.3848,
"margin_dpo/margin_mean": 0.01810815930366516,
"margin_dpo/margin_std": 0.3279362916946411,
"step": 5
},
{
"KL/chosen_KL_mean": -0.033203125,
"KL/mean": -0.029030516743659973,
"KL/rejected_KL_mean": -0.0248565673828125,
"KL/std": 0.2704921364784241,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.008346349000930786,
"fcm_dpo/q_t": 0.5002087354660034,
"grad_norm": 92.05367279052734,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5008316040039062,
"logits/rejected": -0.45828235149383545,
"logps/chosen": -63.040687561035156,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.67020416259766,
"loss": 1.3876,
"margin_dpo/margin_mean": -0.008346527814865112,
"margin_dpo/margin_std": 0.41183364391326904,
"step": 6
},
{
"KL/chosen_KL_mean": -0.0074214935302734375,
"KL/mean": 0.024099931120872498,
"KL/rejected_KL_mean": 0.055622100830078125,
"KL/std": 0.3060908317565918,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06303969025611877,
"fcm_dpo/q_t": 0.5015759468078613,
"grad_norm": 82.6351318359375,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.4966989755630493,
"logits/rejected": -0.4628666639328003,
"logps/chosen": -57.78224182128906,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.86497497558594,
"loss": 1.3931,
"margin_dpo/margin_mean": -0.06303924322128296,
"margin_dpo/margin_std": 0.4346626400947571,
"step": 7
},
{
"KL/chosen_KL_mean": 0.03885078430175781,
"KL/mean": 0.04787926375865936,
"KL/rejected_KL_mean": 0.056911468505859375,
"KL/std": 0.3092794120311737,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.018061310052871704,
"fcm_dpo/q_t": 0.5004498958587646,
"grad_norm": 78.8070068359375,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.4943142533302307,
"logits/rejected": -0.466851145029068,
"logps/chosen": -58.67718505859375,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.2545166015625,
"loss": 1.3885,
"margin_dpo/margin_mean": -0.01806098222732544,
"margin_dpo/margin_std": 0.4174906015396118,
"step": 8
},
{
"KL/chosen_KL_mean": 0.028049468994140625,
"KL/mean": 0.013820052146911621,
"KL/rejected_KL_mean": -0.000408172607421875,
"KL/std": 0.29441195726394653,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.028452813625335693,
"fcm_dpo/q_t": 0.49928951263427734,
"grad_norm": 85.16717529296875,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.5045357346534729,
"logits/rejected": -0.4584883451461792,
"logps/chosen": -69.83879089355469,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.60307312011719,
"loss": 1.384,
"margin_dpo/margin_mean": 0.02845209836959839,
"margin_dpo/margin_std": 0.4499683976173401,
"step": 9
},
{
"KL/chosen_KL_mean": 0.000568389892578125,
"KL/mean": -0.016893252730369568,
"KL/rejected_KL_mean": -0.034351348876953125,
"KL/std": 0.25498026609420776,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.034921616315841675,
"fcm_dpo/q_t": 0.4991276264190674,
"grad_norm": 70.64812469482422,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.48759734630584717,
"logits/rejected": -0.44404521584510803,
"logps/chosen": -48.35711669921875,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.40641784667969,
"loss": 1.3832,
"margin_dpo/margin_mean": 0.034921616315841675,
"margin_dpo/margin_std": 0.374549925327301,
"step": 10
},
{
"KL/chosen_KL_mean": 0.018220901489257812,
"KL/mean": 0.0023308396339416504,
"KL/rejected_KL_mean": -0.01355743408203125,
"KL/std": 0.2806966304779053,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.031774938106536865,
"fcm_dpo/q_t": 0.49920594692230225,
"grad_norm": 68.4999008178711,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4650164842605591,
"logits/rejected": -0.4402990937232971,
"logps/chosen": -52.998634338378906,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.79393005371094,
"loss": 1.3834,
"margin_dpo/margin_mean": 0.03177538514137268,
"margin_dpo/margin_std": 0.34445643424987793,
"step": 11
},
{
"KL/chosen_KL_mean": 0.030973434448242188,
"KL/mean": -0.03026643395423889,
"KL/rejected_KL_mean": -0.09149932861328125,
"KL/std": 0.2564256191253662,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12247461080551147,
"fcm_dpo/q_t": 0.49693846702575684,
"grad_norm": 94.00206756591797,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5384472608566284,
"logits/rejected": -0.5022361278533936,
"logps/chosen": -61.77446365356445,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.94976806640625,
"loss": 1.3744,
"margin_dpo/margin_mean": 0.12247374653816223,
"margin_dpo/margin_std": 0.3362644910812378,
"step": 12
},
{
"KL/chosen_KL_mean": -0.03291130065917969,
"KL/mean": -0.008768260478973389,
"KL/rejected_KL_mean": 0.015377044677734375,
"KL/std": 0.2577245235443115,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04828256368637085,
"fcm_dpo/q_t": 0.50120609998703,
"grad_norm": 79.38003540039062,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.4910910725593567,
"logits/rejected": -0.4638311266899109,
"logps/chosen": -64.29326629638672,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.18770599365234,
"loss": 1.3915,
"margin_dpo/margin_mean": -0.04828298091888428,
"margin_dpo/margin_std": 0.3570724129676819,
"step": 13
},
{
"KL/chosen_KL_mean": -0.013036727905273438,
"KL/mean": 0.009792491793632507,
"KL/rejected_KL_mean": 0.032619476318359375,
"KL/std": 0.2999943494796753,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04565081000328064,
"fcm_dpo/q_t": 0.5011403560638428,
"grad_norm": 85.56097412109375,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.4799395501613617,
"logits/rejected": -0.440725177526474,
"logps/chosen": -58.12324523925781,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.01446533203125,
"loss": 1.3913,
"margin_dpo/margin_mean": -0.045650362968444824,
"margin_dpo/margin_std": 0.404807984828949,
"step": 14
},
{
"KL/chosen_KL_mean": -0.06286811828613281,
"KL/mean": -0.05339600145816803,
"KL/rejected_KL_mean": -0.0439300537109375,
"KL/std": 0.28460076451301575,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01893576979637146,
"fcm_dpo/q_t": 0.5004709959030151,
"grad_norm": 64.50666809082031,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.5097112655639648,
"logits/rejected": -0.49188485741615295,
"logps/chosen": -57.02977752685547,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.85256958007812,
"loss": 1.3887,
"margin_dpo/margin_mean": -0.0189359188079834,
"margin_dpo/margin_std": 0.4248620867729187,
"step": 15
},
{
"KL/chosen_KL_mean": -0.011941909790039062,
"KL/mean": -0.038836658000946045,
"KL/rejected_KL_mean": -0.06572723388671875,
"KL/std": 0.2758028507232666,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0537833571434021,
"fcm_dpo/q_t": 0.49865537881851196,
"grad_norm": 83.70243072509766,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5452868342399597,
"logits/rejected": -0.5062515735626221,
"logps/chosen": -61.751834869384766,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.43519592285156,
"loss": 1.3813,
"margin_dpo/margin_mean": 0.053783416748046875,
"margin_dpo/margin_std": 0.3853508234024048,
"step": 16
},
{
"KL/chosen_KL_mean": -0.00653839111328125,
"KL/mean": -0.028867706656455994,
"KL/rejected_KL_mean": -0.05120086669921875,
"KL/std": 0.2653309106826782,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04466333985328674,
"fcm_dpo/q_t": 0.4988834261894226,
"grad_norm": 79.01133728027344,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.488353431224823,
"logits/rejected": -0.4496167302131653,
"logps/chosen": -67.71687316894531,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.42985534667969,
"loss": 1.3822,
"margin_dpo/margin_mean": 0.04466351866722107,
"margin_dpo/margin_std": 0.36588189005851746,
"step": 17
},
{
"KL/chosen_KL_mean": 0.013227462768554688,
"KL/mean": -0.008445173501968384,
"KL/rejected_KL_mean": -0.030117034912109375,
"KL/std": 0.26379337906837463,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04334592819213867,
"fcm_dpo/q_t": 0.49891626834869385,
"grad_norm": 82.08016967773438,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.5007514357566833,
"logits/rejected": -0.44611847400665283,
"logps/chosen": -47.72626495361328,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.50240325927734,
"loss": 1.3823,
"margin_dpo/margin_mean": 0.04334595799446106,
"margin_dpo/margin_std": 0.3786376118659973,
"step": 18
},
{
"KL/chosen_KL_mean": 0.01825714111328125,
"KL/mean": -0.03181253373622894,
"KL/rejected_KL_mean": -0.081878662109375,
"KL/std": 0.24455955624580383,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1001291573047638,
"fcm_dpo/q_t": 0.49749767780303955,
"grad_norm": 73.14253997802734,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.4779819846153259,
"logits/rejected": -0.42704999446868896,
"logps/chosen": -70.18710327148438,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.83946990966797,
"loss": 1.3766,
"margin_dpo/margin_mean": 0.10012984275817871,
"margin_dpo/margin_std": 0.33631211519241333,
"step": 19
},
{
"KL/chosen_KL_mean": -0.012134552001953125,
"KL/mean": -0.04402177035808563,
"KL/rejected_KL_mean": -0.07590484619140625,
"KL/std": 0.22419436275959015,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06377437710762024,
"fcm_dpo/q_t": 0.4984060525894165,
"grad_norm": 73.54801940917969,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5527380108833313,
"logits/rejected": -0.49656909704208374,
"logps/chosen": -50.81537628173828,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.89925384521484,
"loss": 1.3802,
"margin_dpo/margin_mean": 0.06377455592155457,
"margin_dpo/margin_std": 0.3116491734981537,
"step": 20
},
{
"KL/chosen_KL_mean": 0.03291130065917969,
"KL/mean": -0.04309450834989548,
"KL/rejected_KL_mean": -0.11909866333007812,
"KL/std": 0.277535617351532,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1520112305879593,
"fcm_dpo/q_t": 0.49620136618614197,
"grad_norm": 77.38446044921875,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.5033525824546814,
"logits/rejected": -0.48031163215637207,
"logps/chosen": -50.03010559082031,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.9878921508789,
"loss": 1.3714,
"margin_dpo/margin_mean": 0.1520119160413742,
"margin_dpo/margin_std": 0.33785581588745117,
"step": 21
},
{
"KL/chosen_KL_mean": 0.049625396728515625,
"KL/mean": -0.0646744817495346,
"KL/rejected_KL_mean": -0.17897415161132812,
"KL/std": 0.2905516028404236,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2286020815372467,
"fcm_dpo/q_t": 0.4942886233329773,
"grad_norm": 84.13846588134766,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.47962144017219543,
"logits/rejected": -0.43577441573143005,
"logps/chosen": -59.00801086425781,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.68364715576172,
"loss": 1.364,
"margin_dpo/margin_mean": 0.22860169410705566,
"margin_dpo/margin_std": 0.4178020656108856,
"step": 22
},
{
"KL/chosen_KL_mean": 0.03469657897949219,
"KL/mean": -0.05604839324951172,
"KL/rejected_KL_mean": -0.14679336547851562,
"KL/std": 0.29135066270828247,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1814877986907959,
"fcm_dpo/q_t": 0.4954661726951599,
"grad_norm": 80.80692291259766,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.5033831000328064,
"logits/rejected": -0.48167192935943604,
"logps/chosen": -60.042999267578125,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.28634643554688,
"loss": 1.3686,
"margin_dpo/margin_mean": 0.18148785829544067,
"margin_dpo/margin_std": 0.37989723682403564,
"step": 23
},
{
"KL/chosen_KL_mean": 0.027399063110351562,
"KL/mean": -0.1130581721663475,
"KL/rejected_KL_mean": -0.25351715087890625,
"KL/std": 0.2865855097770691,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2809140086174011,
"fcm_dpo/q_t": 0.4929804801940918,
"grad_norm": 84.92632293701172,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5087974071502686,
"logits/rejected": -0.49225664138793945,
"logps/chosen": -44.263633728027344,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.37873077392578,
"loss": 1.3587,
"margin_dpo/margin_mean": 0.28091397881507874,
"margin_dpo/margin_std": 0.367694616317749,
"step": 24
},
{
"KL/chosen_KL_mean": -0.0003814697265625,
"KL/mean": -0.09033405780792236,
"KL/rejected_KL_mean": -0.1802825927734375,
"KL/std": 0.3605647385120392,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17990106344223022,
"fcm_dpo/q_t": 0.4955053925514221,
"grad_norm": 74.13409423828125,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.5078995823860168,
"logits/rejected": -0.47833961248397827,
"logps/chosen": -52.53743362426758,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.52247619628906,
"loss": 1.3689,
"margin_dpo/margin_mean": 0.17990124225616455,
"margin_dpo/margin_std": 0.45145928859710693,
"step": 25
},
{
"KL/chosen_KL_mean": 0.06395149230957031,
"KL/mean": -0.13807255029678345,
"KL/rejected_KL_mean": -0.3400993347167969,
"KL/std": 0.4370245337486267,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.40405547618865967,
"fcm_dpo/q_t": 0.4899117946624756,
"grad_norm": 86.72003173828125,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5410639047622681,
"logits/rejected": -0.5095586776733398,
"logps/chosen": -53.858856201171875,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.69981384277344,
"loss": 1.3471,
"margin_dpo/margin_mean": 0.4040555953979492,
"margin_dpo/margin_std": 0.5661390423774719,
"step": 26
},
{
"KL/chosen_KL_mean": 0.1539630889892578,
"KL/mean": -0.13156871497631073,
"KL/rejected_KL_mean": -0.4171028137207031,
"KL/std": 0.48169609904289246,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.571062445640564,
"fcm_dpo/q_t": 0.48574382066726685,
"grad_norm": 93.52059936523438,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.526579737663269,
"logits/rejected": -0.49027207493782043,
"logps/chosen": -42.74456787109375,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.14130401611328,
"loss": 1.3308,
"margin_dpo/margin_mean": 0.5710620880126953,
"margin_dpo/margin_std": 0.5639763474464417,
"step": 27
},
{
"KL/chosen_KL_mean": -0.003818511962890625,
"KL/mean": -0.1476629376411438,
"KL/rejected_KL_mean": -0.29151153564453125,
"KL/std": 0.4086608588695526,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.28768980503082275,
"fcm_dpo/q_t": 0.4928103983402252,
"grad_norm": 77.30921936035156,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5286385416984558,
"logits/rejected": -0.4755781292915344,
"logps/chosen": -60.56031799316406,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.692626953125,
"loss": 1.3587,
"margin_dpo/margin_mean": 0.287690132856369,
"margin_dpo/margin_std": 0.6202329993247986,
"step": 28
},
{
"KL/chosen_KL_mean": 0.11627197265625,
"KL/mean": -0.15424633026123047,
"KL/rejected_KL_mean": -0.4247627258300781,
"KL/std": 0.48740673065185547,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5410360097885132,
"fcm_dpo/q_t": 0.4864945113658905,
"grad_norm": 91.11680603027344,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5516057014465332,
"logits/rejected": -0.5051902532577515,
"logps/chosen": -57.69151306152344,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.8191146850586,
"loss": 1.3339,
"margin_dpo/margin_mean": 0.5410366654396057,
"margin_dpo/margin_std": 0.6225091218948364,
"step": 29
},
{
"KL/chosen_KL_mean": 0.14649200439453125,
"KL/mean": -0.21738092601299286,
"KL/rejected_KL_mean": -0.5812568664550781,
"KL/std": 0.6298277378082275,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7277464866638184,
"fcm_dpo/q_t": 0.48183703422546387,
"grad_norm": 87.24919128417969,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.5027915835380554,
"logits/rejected": -0.47236695885658264,
"logps/chosen": -52.430877685546875,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.07046508789062,
"loss": 1.3159,
"margin_dpo/margin_mean": 0.7277467846870422,
"margin_dpo/margin_std": 0.6605062484741211,
"step": 30
},
{
"KL/chosen_KL_mean": 0.08755111694335938,
"KL/mean": -0.1343349814414978,
"KL/rejected_KL_mean": -0.35622406005859375,
"KL/std": 0.5193805694580078,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4437781572341919,
"fcm_dpo/q_t": 0.4889264404773712,
"grad_norm": 68.48614501953125,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.5168442726135254,
"logits/rejected": -0.4734218716621399,
"logps/chosen": -63.71936798095703,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.25022888183594,
"loss": 1.3435,
"margin_dpo/margin_mean": 0.44377774000167847,
"margin_dpo/margin_std": 0.6583547592163086,
"step": 31
},
{
"KL/chosen_KL_mean": 0.18407058715820312,
"KL/mean": -0.19383396208286285,
"KL/rejected_KL_mean": -0.5717430114746094,
"KL/std": 0.7323002815246582,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.755810022354126,
"fcm_dpo/q_t": 0.4811716079711914,
"grad_norm": 83.46903991699219,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5339919328689575,
"logits/rejected": -0.49401533603668213,
"logps/chosen": -62.55545425415039,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.88924407958984,
"loss": 1.3141,
"margin_dpo/margin_mean": 0.7558099627494812,
"margin_dpo/margin_std": 0.8941420316696167,
"step": 32
},
{
"KL/chosen_KL_mean": 0.10718154907226562,
"KL/mean": -0.2103760838508606,
"KL/rejected_KL_mean": -0.5279350280761719,
"KL/std": 0.5602357387542725,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6351122260093689,
"fcm_dpo/q_t": 0.48414498567581177,
"grad_norm": 72.1122817993164,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.489587664604187,
"logits/rejected": -0.46343374252319336,
"logps/chosen": -53.15379333496094,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.41307067871094,
"loss": 1.3248,
"margin_dpo/margin_mean": 0.635112464427948,
"margin_dpo/margin_std": 0.6338217854499817,
"step": 33
},
{
"KL/chosen_KL_mean": 0.08233070373535156,
"KL/mean": -0.30296576023101807,
"KL/rejected_KL_mean": -0.6882667541503906,
"KL/std": 0.7120438814163208,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7705912590026855,
"fcm_dpo/q_t": 0.4807863235473633,
"grad_norm": 78.05076599121094,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.5040176510810852,
"logits/rejected": -0.4865629971027374,
"logps/chosen": -50.7349967956543,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.610107421875,
"loss": 1.3124,
"margin_dpo/margin_mean": 0.7705909013748169,
"margin_dpo/margin_std": 0.8183022141456604,
"step": 34
},
{
"KL/chosen_KL_mean": 0.13801002502441406,
"KL/mean": -0.4717918634414673,
"KL/rejected_KL_mean": -1.0815925598144531,
"KL/std": 1.019068956375122,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2196067571640015,
"fcm_dpo/q_t": 0.46972382068634033,
"grad_norm": 82.06009674072266,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5100936889648438,
"logits/rejected": -0.4733096957206726,
"logps/chosen": -50.88648223876953,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.90602111816406,
"loss": 1.2714,
"margin_dpo/margin_mean": 1.219606637954712,
"margin_dpo/margin_std": 1.1478991508483887,
"step": 35
},
{
"KL/chosen_KL_mean": 0.07729721069335938,
"KL/mean": -0.5002259016036987,
"KL/rejected_KL_mean": -1.0777473449707031,
"KL/std": 1.079554796218872,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1550471782684326,
"fcm_dpo/q_t": 0.47128745913505554,
"grad_norm": 73.2303237915039,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5755394697189331,
"logits/rejected": -0.5400795936584473,
"logps/chosen": -51.91419219970703,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.11837768554688,
"loss": 1.2782,
"margin_dpo/margin_mean": 1.1550464630126953,
"margin_dpo/margin_std": 1.223704218864441,
"step": 36
},
{
"KL/chosen_KL_mean": 0.012973785400390625,
"KL/mean": -0.5021691918373108,
"KL/rejected_KL_mean": -1.0173149108886719,
"KL/std": 1.0568256378173828,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0302867889404297,
"fcm_dpo/q_t": 0.474433034658432,
"grad_norm": 61.57312774658203,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5395572781562805,
"logits/rejected": -0.4981452226638794,
"logps/chosen": -62.79413604736328,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.91238403320312,
"loss": 1.2906,
"margin_dpo/margin_mean": 1.0302867889404297,
"margin_dpo/margin_std": 1.334470510482788,
"step": 37
},
{
"KL/chosen_KL_mean": 0.13834381103515625,
"KL/mean": -0.5442078113555908,
"KL/rejected_KL_mean": -1.2267570495605469,
"KL/std": 1.350521206855774,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3651026487350464,
"fcm_dpo/q_t": 0.4663234353065491,
"grad_norm": 69.9059829711914,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5119627714157104,
"logits/rejected": -0.4792238473892212,
"logps/chosen": -48.252174377441406,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.13919830322266,
"loss": 1.2613,
"margin_dpo/margin_mean": 1.3651025295257568,
"margin_dpo/margin_std": 1.6618046760559082,
"step": 38
},
{
"KL/chosen_KL_mean": 0.09346199035644531,
"KL/mean": -0.7377809882164001,
"KL/rejected_KL_mean": -1.5690231323242188,
"KL/std": 1.284562349319458,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6624829769134521,
"fcm_dpo/q_t": 0.4587351083755493,
"grad_norm": 76.24556732177734,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5892033576965332,
"logits/rejected": -0.5514540076255798,
"logps/chosen": -50.657012939453125,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.13853454589844,
"loss": 1.2315,
"margin_dpo/margin_mean": 1.6624832153320312,
"margin_dpo/margin_std": 1.3248234987258911,
"step": 39
},
{
"KL/chosen_KL_mean": 0.2594871520996094,
"KL/mean": -0.5624086856842041,
"KL/rejected_KL_mean": -1.3843040466308594,
"KL/std": 1.4190328121185303,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6437921524047852,
"fcm_dpo/q_t": 0.4594607949256897,
"grad_norm": 60.083221435546875,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.502472996711731,
"logits/rejected": -0.47134459018707275,
"logps/chosen": -57.72557830810547,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.68437194824219,
"loss": 1.2362,
"margin_dpo/margin_mean": 1.643792986869812,
"margin_dpo/margin_std": 1.6722097396850586,
"step": 40
},
{
"KL/chosen_KL_mean": -0.000926971435546875,
"KL/mean": -0.9511067867279053,
"KL/rejected_KL_mean": -1.9012832641601562,
"KL/std": 1.8827296495437622,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10071444511413574,
"fcm_dpo/delta": 0.0709388256072998,
"fcm_dpo/margin": 1.9003576040267944,
"fcm_dpo/q_t": 0.45327073335647583,
"grad_norm": 68.3230209350586,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5602806806564331,
"logits/rejected": -0.5249433517456055,
"logps/chosen": -62.69674301147461,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.92481231689453,
"loss": 1.2148,
"margin_dpo/margin_mean": 1.9003586769104004,
"margin_dpo/margin_std": 2.0421926975250244,
"step": 41
},
{
"KL/chosen_KL_mean": 0.2127838134765625,
"KL/mean": -1.085558295249939,
"KL/rejected_KL_mean": -2.383892059326172,
"KL/std": 2.2120487689971924,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.10279597342014313,
"fcm_dpo/delta": 0.08528625965118408,
"fcm_dpo/margin": 2.5966811180114746,
"fcm_dpo/q_t": 0.4356800317764282,
"grad_norm": 80.10550689697266,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5423730611801147,
"logits/rejected": -0.4957752823829651,
"logps/chosen": -58.75364303588867,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.29226684570312,
"loss": 1.1536,
"margin_dpo/margin_mean": 2.596680164337158,
"margin_dpo/margin_std": 2.525965690612793,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5627727508544922,
"KL/mean": -0.6873226165771484,
"KL/rejected_KL_mean": -1.9374237060546875,
"KL/std": 1.8489782810211182,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.10443605482578278,
"fcm_dpo/delta": 0.09136471152305603,
"fcm_dpo/margin": 2.5001907348632812,
"fcm_dpo/q_t": 0.4365711510181427,
"grad_norm": 74.09246826171875,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5289781093597412,
"logits/rejected": -0.5033968687057495,
"logps/chosen": -53.593223571777344,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.41761779785156,
"loss": 1.1533,
"margin_dpo/margin_mean": 2.5001907348632812,
"margin_dpo/margin_std": 2.0130953788757324,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2562217712402344,
"KL/mean": -1.1680686473846436,
"KL/rejected_KL_mean": -2.5923614501953125,
"KL/std": 2.291693687438965,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.1057717502117157,
"fcm_dpo/delta": 0.0502425879240036,
"fcm_dpo/margin": 2.8485794067382812,
"fcm_dpo/q_t": 0.42686349153518677,
"grad_norm": 81.92259216308594,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.48028355836868286,
"logits/rejected": -0.46020689606666565,
"logps/chosen": -49.8222770690918,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.3761215209961,
"loss": 1.1213,
"margin_dpo/margin_mean": 2.848578453063965,
"margin_dpo/margin_std": 2.2468016147613525,
"step": 44
},
{
"KL/chosen_KL_mean": 0.1484088897705078,
"KL/mean": -0.985015869140625,
"KL/rejected_KL_mean": -2.118438720703125,
"KL/std": 2.0992965698242188,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.10800284147262573,
"fcm_dpo/delta": 0.10763342678546906,
"fcm_dpo/margin": 2.2668519020080566,
"fcm_dpo/q_t": 0.4414454698562622,
"grad_norm": 65.85893249511719,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.45797061920166016,
"logits/rejected": -0.4448603093624115,
"logps/chosen": -48.266517639160156,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.05487060546875,
"loss": 1.1763,
"margin_dpo/margin_mean": 2.2668511867523193,
"margin_dpo/margin_std": 2.624885082244873,
"step": 45
},
{
"KL/chosen_KL_mean": 0.2292613983154297,
"KL/mean": -1.2564077377319336,
"KL/rejected_KL_mean": -2.742076873779297,
"KL/std": 2.657705783843994,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.1090465784072876,
"fcm_dpo/delta": 0.026768168434500694,
"fcm_dpo/margin": 2.9713401794433594,
"fcm_dpo/q_t": 0.4231029450893402,
"grad_norm": 73.93266296386719,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5324057340621948,
"logits/rejected": -0.48163044452667236,
"logps/chosen": -55.770164489746094,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.39466857910156,
"loss": 1.1181,
"margin_dpo/margin_mean": 2.9713401794433594,
"margin_dpo/margin_std": 3.2657053470611572,
"step": 46
},
{
"KL/chosen_KL_mean": 0.3735790252685547,
"KL/mean": -1.1065313816070557,
"KL/rejected_KL_mean": -2.5866432189941406,
"KL/std": 2.6243019104003906,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.10978727042675018,
"fcm_dpo/delta": 0.02556876838207245,
"fcm_dpo/margin": 2.960222005844116,
"fcm_dpo/q_t": 0.4216909408569336,
"grad_norm": 70.09626007080078,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5489112734794617,
"logits/rejected": -0.49465492367744446,
"logps/chosen": -57.55249786376953,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.26585388183594,
"loss": 1.1101,
"margin_dpo/margin_mean": 2.9602224826812744,
"margin_dpo/margin_std": 2.731231927871704,
"step": 47
},
{
"KL/chosen_KL_mean": 0.04618263244628906,
"KL/mean": -1.5137662887573242,
"KL/rejected_KL_mean": -3.0737152099609375,
"KL/std": 2.6629862785339355,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.1094270646572113,
"fcm_dpo/delta": 0.00876920111477375,
"fcm_dpo/margin": 3.1198978424072266,
"fcm_dpo/q_t": 0.4179936647415161,
"grad_norm": 76.3647689819336,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.6007370948791504,
"logits/rejected": -0.5431762933731079,
"logps/chosen": -57.14189147949219,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.09031677246094,
"loss": 1.1026,
"margin_dpo/margin_mean": 3.1198978424072266,
"margin_dpo/margin_std": 2.9437079429626465,
"step": 48
},
{
"KL/chosen_KL_mean": 0.3652973175048828,
"KL/mean": -1.5648986101150513,
"KL/rejected_KL_mean": -3.4950904846191406,
"KL/std": 3.375471591949463,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.1089581847190857,
"fcm_dpo/delta": -0.07370879501104355,
"fcm_dpo/margin": 3.8603830337524414,
"fcm_dpo/q_t": 0.4010041654109955,
"grad_norm": 74.53569030761719,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5346788763999939,
"logits/rejected": -0.4745950698852539,
"logps/chosen": -61.31997299194336,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.26256561279297,
"loss": 1.0501,
"margin_dpo/margin_mean": 3.8603830337524414,
"margin_dpo/margin_std": 3.772658348083496,
"step": 49
},
{
"KL/chosen_KL_mean": -0.15436363220214844,
"KL/mean": -2.2124156951904297,
"KL/rejected_KL_mean": -4.2704620361328125,
"KL/std": 3.501716136932373,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.10655629634857178,
"fcm_dpo/delta": -0.09280845522880554,
"fcm_dpo/margin": 4.116090297698975,
"fcm_dpo/q_t": 0.3962603211402893,
"grad_norm": 65.80304718017578,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5511945486068726,
"logits/rejected": -0.5155908465385437,
"logps/chosen": -58.878501892089844,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.62860870361328,
"loss": 1.0377,
"margin_dpo/margin_mean": 4.116090297698975,
"margin_dpo/margin_std": 4.020236015319824,
"step": 50
},
{
"KL/chosen_KL_mean": -0.3026409149169922,
"KL/mean": -2.3341808319091797,
"KL/rejected_KL_mean": -4.365726470947266,
"KL/std": 4.1528143882751465,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.10447999835014343,
"fcm_dpo/delta": -0.07857784628868103,
"fcm_dpo/margin": 4.063086032867432,
"fcm_dpo/q_t": 0.40330058336257935,
"grad_norm": 53.87831115722656,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5541812181472778,
"logits/rejected": -0.5227280259132385,
"logps/chosen": -61.676307678222656,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.36772918701172,
"loss": 1.0713,
"margin_dpo/margin_mean": 4.063085556030273,
"margin_dpo/margin_std": 5.101934432983398,
"step": 51
},
{
"KL/chosen_KL_mean": 0.32053184509277344,
"KL/mean": -2.7178292274475098,
"KL/rejected_KL_mean": -5.756187438964844,
"KL/std": 4.713332176208496,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.10033614188432693,
"fcm_dpo/delta": -0.2765745520591736,
"fcm_dpo/margin": 6.076724052429199,
"fcm_dpo/q_t": 0.3581005930900574,
"grad_norm": 59.589393615722656,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5497130751609802,
"logits/rejected": -0.49386975169181824,
"logps/chosen": -52.016822814941406,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.7301025390625,
"loss": 0.9249,
"margin_dpo/margin_mean": 6.076724052429199,
"margin_dpo/margin_std": 5.1023430824279785,
"step": 52
},
{
"KL/chosen_KL_mean": -0.2184162139892578,
"KL/mean": -3.311248302459717,
"KL/rejected_KL_mean": -6.404079437255859,
"KL/std": 5.3616042137146,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.09587851911783218,
"fcm_dpo/delta": -0.2576024532318115,
"fcm_dpo/margin": 6.185666561126709,
"fcm_dpo/q_t": 0.3689725399017334,
"grad_norm": 57.90284729003906,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6244029998779297,
"logits/rejected": -0.6030861139297485,
"logps/chosen": -53.53306579589844,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.18767547607422,
"loss": 0.9643,
"margin_dpo/margin_mean": 6.185666084289551,
"margin_dpo/margin_std": 6.1749587059021,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3918323516845703,
"KL/mean": -3.089035987854004,
"KL/rejected_KL_mean": -5.7862396240234375,
"KL/std": 5.119150161743164,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.09195560961961746,
"fcm_dpo/delta": -0.1534874439239502,
"fcm_dpo/margin": 5.394400119781494,
"fcm_dpo/q_t": 0.3844445049762726,
"grad_norm": 52.45349884033203,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5996332764625549,
"logits/rejected": -0.5465092062950134,
"logps/chosen": -51.08049392700195,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.50163269042969,
"loss": 0.9999,
"margin_dpo/margin_mean": 5.394399642944336,
"margin_dpo/margin_std": 5.113949775695801,
"step": 54
},
{
"KL/chosen_KL_mean": -0.9258518218994141,
"KL/mean": -4.1504225730896,
"KL/rejected_KL_mean": -7.374988555908203,
"KL/std": 6.164496421813965,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.08832551538944244,
"fcm_dpo/delta": -0.23232939839363098,
"fcm_dpo/margin": 6.4491424560546875,
"fcm_dpo/q_t": 0.3728960454463959,
"grad_norm": 50.09782028198242,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6301623582839966,
"logits/rejected": -0.5663325786590576,
"logps/chosen": -63.54108428955078,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.36849212646484,
"loss": 0.991,
"margin_dpo/margin_mean": 6.449141502380371,
"margin_dpo/margin_std": 7.437203407287598,
"step": 55
},
{
"KL/chosen_KL_mean": -0.7239055633544922,
"KL/mean": -3.78607177734375,
"KL/rejected_KL_mean": -6.848243713378906,
"KL/std": 6.047898292541504,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.08475831151008606,
"fcm_dpo/delta": -0.17821934819221497,
"fcm_dpo/margin": 6.124341011047363,
"fcm_dpo/q_t": 0.38335487246513367,
"grad_norm": 44.37767791748047,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.619949460029602,
"logits/rejected": -0.5779317021369934,
"logps/chosen": -58.65663146972656,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -101.0226821899414,
"loss": 1.0199,
"margin_dpo/margin_mean": 6.124341011047363,
"margin_dpo/margin_std": 7.334414482116699,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7689876556396484,
"KL/mean": -4.241398811340332,
"KL/rejected_KL_mean": -7.71380615234375,
"KL/std": 6.030036449432373,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.08145327121019363,
"fcm_dpo/delta": -0.2280295044183731,
"fcm_dpo/margin": 6.944817066192627,
"fcm_dpo/q_t": 0.3689224123954773,
"grad_norm": 48.75211715698242,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5575429797172546,
"logits/rejected": -0.5283694267272949,
"logps/chosen": -71.26427459716797,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.27926635742188,
"loss": 0.968,
"margin_dpo/margin_mean": 6.944817543029785,
"margin_dpo/margin_std": 6.705440044403076,
"step": 57
},
{
"KL/chosen_KL_mean": -1.0128746032714844,
"KL/mean": -4.868136405944824,
"KL/rejected_KL_mean": -8.723396301269531,
"KL/std": 6.835504531860352,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.07740665972232819,
"fcm_dpo/delta": -0.2618388831615448,
"fcm_dpo/margin": 7.710516929626465,
"fcm_dpo/q_t": 0.36622318625450134,
"grad_norm": 50.176239013671875,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5943198204040527,
"logits/rejected": -0.5183029174804688,
"logps/chosen": -63.14581298828125,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.34069061279297,
"loss": 0.9646,
"margin_dpo/margin_mean": 7.710516929626465,
"margin_dpo/margin_std": 8.16922378540039,
"step": 58
},
{
"KL/chosen_KL_mean": -1.5172653198242188,
"KL/mean": -5.735530853271484,
"KL/rejected_KL_mean": -9.95379638671875,
"KL/std": 7.462213516235352,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.07241758704185486,
"fcm_dpo/delta": -0.28130438923835754,
"fcm_dpo/margin": 8.436534881591797,
"fcm_dpo/q_t": 0.3623542785644531,
"grad_norm": 47.03469467163086,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6317316293716431,
"logits/rejected": -0.5917737483978271,
"logps/chosen": -53.449790954589844,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.83899688720703,
"loss": 0.9528,
"margin_dpo/margin_mean": 8.436534881591797,
"margin_dpo/margin_std": 8.641322135925293,
"step": 59
},
{
"KL/chosen_KL_mean": -2.6389827728271484,
"KL/mean": -6.052818298339844,
"KL/rejected_KL_mean": -9.46664810180664,
"KL/std": 6.807157039642334,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.07023543119430542,
"fcm_dpo/delta": -0.13676942884922028,
"fcm_dpo/margin": 6.827664375305176,
"fcm_dpo/q_t": 0.3875412940979004,
"grad_norm": 48.9000244140625,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.5942388772964478,
"logits/rejected": -0.5327954292297363,
"logps/chosen": -63.58116912841797,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.86004638671875,
"loss": 1.0321,
"margin_dpo/margin_mean": 6.827664375305176,
"margin_dpo/margin_std": 7.711092948913574,
"step": 60
},
{
"KL/chosen_KL_mean": -1.5054092407226562,
"KL/mean": -5.698062896728516,
"KL/rejected_KL_mean": -9.890716552734375,
"KL/std": 9.148716926574707,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.06738986074924469,
"fcm_dpo/delta": -0.22911055386066437,
"fcm_dpo/margin": 8.385308265686035,
"fcm_dpo/q_t": 0.37978309392929077,
"grad_norm": 41.77968978881836,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.6104527115821838,
"logits/rejected": -0.5764514207839966,
"logps/chosen": -62.13893127441406,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.74320983886719,
"loss": 1.0155,
"margin_dpo/margin_mean": 8.385307312011719,
"margin_dpo/margin_std": 11.387674331665039,
"step": 61
},
{
"KL/chosen_KL_mean": -1.747161865234375,
"KL/mean": -4.856363773345947,
"KL/rejected_KL_mean": -7.965564727783203,
"KL/std": 6.7253522872924805,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.06641829758882523,
"fcm_dpo/delta": -0.06573815643787384,
"fcm_dpo/margin": 6.218404293060303,
"fcm_dpo/q_t": 0.40589573979377747,
"grad_norm": 40.88990020751953,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.5971473455429077,
"logits/rejected": -0.5622413158416748,
"logps/chosen": -57.89793395996094,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.5317611694336,
"loss": 1.0859,
"margin_dpo/margin_mean": 6.218404769897461,
"margin_dpo/margin_std": 8.258329391479492,
"step": 62
},
{
"KL/chosen_KL_mean": -2.8008480072021484,
"KL/mean": -7.036721229553223,
"KL/rejected_KL_mean": -11.272594451904297,
"KL/std": 8.544384956359863,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.06367187201976776,
"fcm_dpo/delta": -0.20195239782333374,
"fcm_dpo/margin": 8.4717435836792,
"fcm_dpo/q_t": 0.37538081407546997,
"grad_norm": 42.83129119873047,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.583976686000824,
"logits/rejected": -0.5383748412132263,
"logps/chosen": -75.9482421875,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -108.88265991210938,
"loss": 0.9938,
"margin_dpo/margin_mean": 8.471742630004883,
"margin_dpo/margin_std": 8.990781784057617,
"step": 63
},
{
"KL/chosen_KL_mean": -1.124542236328125,
"KL/mean": -6.140851020812988,
"KL/rejected_KL_mean": -11.157161712646484,
"KL/std": 9.27166748046875,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.06026393920183182,
"fcm_dpo/delta": -0.2753208875656128,
"fcm_dpo/margin": 10.03261661529541,
"fcm_dpo/q_t": 0.36537328362464905,
"grad_norm": 38.817970275878906,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.5995528697967529,
"logits/rejected": -0.5680242776870728,
"logps/chosen": -55.12314224243164,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.68736267089844,
"loss": 0.9729,
"margin_dpo/margin_mean": 10.032617568969727,
"margin_dpo/margin_std": 11.401942253112793,
"step": 64
},
{
"KL/chosen_KL_mean": -3.4033946990966797,
"KL/mean": -8.52043342590332,
"KL/rejected_KL_mean": -13.637474060058594,
"KL/std": 10.473337173461914,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.0574415922164917,
"fcm_dpo/delta": -0.2546284794807434,
"fcm_dpo/margin": 10.234081268310547,
"fcm_dpo/q_t": 0.3663010597229004,
"grad_norm": 39.211761474609375,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6607149839401245,
"logits/rejected": -0.6480229496955872,
"logps/chosen": -68.23939514160156,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -123.58393859863281,
"loss": 0.9811,
"margin_dpo/margin_mean": 10.234081268310547,
"margin_dpo/margin_std": 11.489011764526367,
"step": 65
},
{
"KL/chosen_KL_mean": -3.2082653045654297,
"KL/mean": -7.8406548500061035,
"KL/rejected_KL_mean": -12.473045349121094,
"KL/std": 9.421492576599121,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.055195845663547516,
"fcm_dpo/delta": -0.17202530801296234,
"fcm_dpo/margin": 9.264777183532715,
"fcm_dpo/q_t": 0.38581931591033936,
"grad_norm": 34.907039642333984,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6280045509338379,
"logits/rejected": -0.5945169925689697,
"logps/chosen": -54.651790618896484,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -88.10934448242188,
"loss": 1.025,
"margin_dpo/margin_mean": 9.264778137207031,
"margin_dpo/margin_std": 11.395795822143555,
"step": 66
},
{
"KL/chosen_KL_mean": -2.8829917907714844,
"KL/mean": -7.654372215270996,
"KL/rejected_KL_mean": -12.42575454711914,
"KL/std": 9.585004806518555,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.053811218589544296,
"fcm_dpo/delta": -0.1720547378063202,
"fcm_dpo/margin": 9.542757987976074,
"fcm_dpo/q_t": 0.38516417145729065,
"grad_norm": 35.49789047241211,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.599655032157898,
"logits/rejected": -0.5566652417182922,
"logps/chosen": -62.22379684448242,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -85.21304321289062,
"loss": 1.0218,
"margin_dpo/margin_mean": 9.542757987976074,
"margin_dpo/margin_std": 11.59894847869873,
"step": 67
},
{
"KL/chosen_KL_mean": -3.4374046325683594,
"KL/mean": -7.992835521697998,
"KL/rejected_KL_mean": -12.548263549804688,
"KL/std": 8.550180435180664,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.05226890742778778,
"fcm_dpo/delta": -0.13234364986419678,
"fcm_dpo/margin": 9.110858917236328,
"fcm_dpo/q_t": 0.38975322246551514,
"grad_norm": 34.271942138671875,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6388707756996155,
"logits/rejected": -0.5778865814208984,
"logps/chosen": -68.64323425292969,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -89.75550842285156,
"loss": 1.0236,
"margin_dpo/margin_mean": 9.110858917236328,
"margin_dpo/margin_std": 9.605062484741211,
"step": 68
},
{
"KL/chosen_KL_mean": -4.114860534667969,
"KL/mean": -9.939827919006348,
"KL/rejected_KL_mean": -15.764793395996094,
"KL/std": 10.603840827941895,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.04974665865302086,
"fcm_dpo/delta": -0.24509744346141815,
"fcm_dpo/margin": 11.649931907653809,
"fcm_dpo/q_t": 0.3687508702278137,
"grad_norm": 36.500450134277344,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6228272318840027,
"logits/rejected": -0.5997121930122375,
"logps/chosen": -63.93410110473633,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -119.15365600585938,
"loss": 0.9628,
"margin_dpo/margin_mean": 11.649931907653809,
"margin_dpo/margin_std": 11.750423431396484,
"step": 69
},
{
"KL/chosen_KL_mean": -6.090396881103516,
"KL/mean": -12.428565979003906,
"KL/rejected_KL_mean": -18.76673126220703,
"KL/std": 13.280801773071289,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.047557003796100616,
"fcm_dpo/delta": -0.2684404253959656,
"fcm_dpo/margin": 12.676331520080566,
"fcm_dpo/q_t": 0.36635735630989075,
"grad_norm": 36.634315490722656,
"learning_rate": 5e-07,
"logits/chosen": -0.6058599948883057,
"logits/rejected": -0.5707762241363525,
"logps/chosen": -68.02104187011719,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -109.8275146484375,
"loss": 0.9698,
"margin_dpo/margin_mean": 12.676332473754883,
"margin_dpo/margin_std": 14.070359230041504,
"step": 70
},
{
"KL/chosen_KL_mean": -6.061298370361328,
"KL/mean": -13.633516311645508,
"KL/rejected_KL_mean": -21.205734252929688,
"KL/std": 14.106433868408203,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.044265132397413254,
"fcm_dpo/delta": -0.34492480754852295,
"fcm_dpo/margin": 15.14443588256836,
"fcm_dpo/q_t": 0.3534585237503052,
"grad_norm": 34.083553314208984,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6632341146469116,
"logits/rejected": -0.6252255439758301,
"logps/chosen": -67.81163787841797,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -118.5423583984375,
"loss": 0.923,
"margin_dpo/margin_mean": 15.14443588256836,
"margin_dpo/margin_std": 15.41769790649414,
"step": 71
},
{
"KL/chosen_KL_mean": -7.117975234985352,
"KL/mean": -15.00790786743164,
"KL/rejected_KL_mean": -22.897842407226562,
"KL/std": 15.219398498535156,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.04147840291261673,
"fcm_dpo/delta": -0.3267444372177124,
"fcm_dpo/margin": 15.779868125915527,
"fcm_dpo/q_t": 0.35581284761428833,
"grad_norm": 33.533538818359375,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.659099817276001,
"logits/rejected": -0.6226309537887573,
"logps/chosen": -73.17138671875,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -118.18482971191406,
"loss": 0.9517,
"margin_dpo/margin_mean": 15.779868125915527,
"margin_dpo/margin_std": 18.023799896240234,
"step": 72
},
{
"KL/chosen_KL_mean": -9.201299667358398,
"KL/mean": -17.138395309448242,
"KL/rejected_KL_mean": -25.07549285888672,
"KL/std": 19.838104248046875,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.03898513317108154,
"fcm_dpo/delta": -0.28837209939956665,
"fcm_dpo/margin": 15.87419319152832,
"fcm_dpo/q_t": 0.37557166814804077,
"grad_norm": 34.42801284790039,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.690470814704895,
"logits/rejected": -0.6502715349197388,
"logps/chosen": -75.45757293701172,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -115.53163146972656,
"loss": 1.0506,
"margin_dpo/margin_mean": 15.87419319152832,
"margin_dpo/margin_std": 24.97110366821289,
"step": 73
},
{
"KL/chosen_KL_mean": -9.636629104614258,
"KL/mean": -18.45030403137207,
"KL/rejected_KL_mean": -27.263980865478516,
"KL/std": 21.931224822998047,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.036717548966407776,
"fcm_dpo/delta": -0.3187072277069092,
"fcm_dpo/margin": 17.62735366821289,
"fcm_dpo/q_t": 0.3689573109149933,
"grad_norm": 35.52349853515625,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6715037822723389,
"logits/rejected": -0.6643733978271484,
"logps/chosen": -63.0615119934082,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -123.21092224121094,
"loss": 1.007,
"margin_dpo/margin_mean": 17.627355575561523,
"margin_dpo/margin_std": 27.311134338378906,
"step": 74
},
{
"KL/chosen_KL_mean": -9.528879165649414,
"KL/mean": -21.829769134521484,
"KL/rejected_KL_mean": -34.130653381347656,
"KL/std": 22.312917709350586,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.03375118225812912,
"fcm_dpo/delta": -0.5213460922241211,
"fcm_dpo/margin": 24.601776123046875,
"fcm_dpo/q_t": 0.3274177014827728,
"grad_norm": 29.952869415283203,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6653124094009399,
"logits/rejected": -0.6586759090423584,
"logps/chosen": -61.390541076660156,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -145.3846435546875,
"loss": 0.8665,
"margin_dpo/margin_mean": 24.601776123046875,
"margin_dpo/margin_std": 25.416664123535156,
"step": 75
},
{
"KL/chosen_KL_mean": -11.458761215209961,
"KL/mean": -19.180240631103516,
"KL/rejected_KL_mean": -26.901718139648438,
"KL/std": 18.074440002441406,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.031918227672576904,
"fcm_dpo/delta": -0.15023696422576904,
"fcm_dpo/margin": 15.442970275878906,
"fcm_dpo/q_t": 0.3866853415966034,
"grad_norm": 30.53144645690918,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.6922107934951782,
"logits/rejected": -0.6505155563354492,
"logps/chosen": -64.72480010986328,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -105.11834716796875,
"loss": 1.0424,
"margin_dpo/margin_mean": 15.442970275878906,
"margin_dpo/margin_std": 19.84844207763672,
"step": 76
},
{
"KL/chosen_KL_mean": -10.305963516235352,
"KL/mean": -22.921131134033203,
"KL/rejected_KL_mean": -35.53630828857422,
"KL/std": 24.69320297241211,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.029510796070098877,
"fcm_dpo/delta": -0.4299943149089813,
"fcm_dpo/margin": 25.23033905029297,
"fcm_dpo/q_t": 0.3367495834827423,
"grad_norm": 31.526256561279297,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6924519538879395,
"logits/rejected": -0.6561889052391052,
"logps/chosen": -68.40264129638672,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -129.30992126464844,
"loss": 0.913,
"margin_dpo/margin_mean": 25.23033905029297,
"margin_dpo/margin_std": 27.491090774536133,
"step": 77
},
{
"KL/chosen_KL_mean": -10.388359069824219,
"KL/mean": -19.755765914916992,
"KL/rejected_KL_mean": -29.123172760009766,
"KL/std": 19.83435821533203,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.028303634375333786,
"fcm_dpo/delta": -0.19002512097358704,
"fcm_dpo/margin": 18.73480987548828,
"fcm_dpo/q_t": 0.3818398714065552,
"grad_norm": 28.295921325683594,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6519057154655457,
"logits/rejected": -0.628269374370575,
"logps/chosen": -66.00214385986328,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -114.05754089355469,
"loss": 1.0187,
"margin_dpo/margin_mean": 18.73480987548828,
"margin_dpo/margin_std": 22.818925857543945,
"step": 78
},
{
"KL/chosen_KL_mean": -10.710470199584961,
"KL/mean": -20.990741729736328,
"KL/rejected_KL_mean": -31.271018981933594,
"KL/std": 22.39347267150879,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.027051560580730438,
"fcm_dpo/delta": -0.21849405765533447,
"fcm_dpo/margin": 20.56053924560547,
"fcm_dpo/q_t": 0.3806380033493042,
"grad_norm": 25.06480598449707,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6494733095169067,
"logits/rejected": -0.6198798418045044,
"logps/chosen": -66.16095733642578,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -118.9185791015625,
"loss": 1.0248,
"margin_dpo/margin_mean": 20.56053924560547,
"margin_dpo/margin_std": 27.430313110351562,
"step": 79
},
{
"KL/chosen_KL_mean": -13.9652099609375,
"KL/mean": -23.680742263793945,
"KL/rejected_KL_mean": -33.396278381347656,
"KL/std": 23.99298095703125,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.02592482417821884,
"fcm_dpo/delta": -0.16428518295288086,
"fcm_dpo/margin": 19.431066513061523,
"fcm_dpo/q_t": 0.3933182656764984,
"grad_norm": 26.278663635253906,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7399425506591797,
"logits/rejected": -0.7012728452682495,
"logps/chosen": -72.4844970703125,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -120.94377899169922,
"loss": 1.0634,
"margin_dpo/margin_mean": 19.43106460571289,
"margin_dpo/margin_std": 27.870834350585938,
"step": 80
},
{
"KL/chosen_KL_mean": -13.097003936767578,
"KL/mean": -26.419586181640625,
"KL/rejected_KL_mean": -39.74217224121094,
"KL/std": 28.39004898071289,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.02453438937664032,
"fcm_dpo/delta": -0.3274933397769928,
"fcm_dpo/margin": 26.645172119140625,
"fcm_dpo/q_t": 0.36159569025039673,
"grad_norm": 27.374237060546875,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7171902656555176,
"logits/rejected": -0.6900858879089355,
"logps/chosen": -79.54586791992188,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -169.40487670898438,
"loss": 0.9726,
"margin_dpo/margin_mean": 26.645172119140625,
"margin_dpo/margin_std": 33.81873321533203,
"step": 81
},
{
"KL/chosen_KL_mean": -15.472373962402344,
"KL/mean": -26.10114288330078,
"KL/rejected_KL_mean": -36.729915618896484,
"KL/std": 25.81859588623047,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.02389688789844513,
"fcm_dpo/delta": -0.16673710942268372,
"fcm_dpo/margin": 21.257539749145508,
"fcm_dpo/q_t": 0.39267510175704956,
"grad_norm": 29.728790283203125,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6627001166343689,
"logits/rejected": -0.6532809734344482,
"logps/chosen": -67.70475769042969,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -127.4731674194336,
"loss": 1.1015,
"margin_dpo/margin_mean": 21.257539749145508,
"margin_dpo/margin_std": 35.67451095581055,
"step": 82
},
{
"KL/chosen_KL_mean": -15.941610336303711,
"KL/mean": -28.5517635345459,
"KL/rejected_KL_mean": -41.16191482543945,
"KL/std": 26.822208404541016,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.02258511260151863,
"fcm_dpo/delta": -0.23390883207321167,
"fcm_dpo/margin": 25.22031021118164,
"fcm_dpo/q_t": 0.37505269050598145,
"grad_norm": 25.695941925048828,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6607187986373901,
"logits/rejected": -0.6281884908676147,
"logps/chosen": -71.76899719238281,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -144.8778076171875,
"loss": 1.0091,
"margin_dpo/margin_mean": 25.220308303833008,
"margin_dpo/margin_std": 31.849729537963867,
"step": 83
},
{
"KL/chosen_KL_mean": -15.136430740356445,
"KL/mean": -25.702499389648438,
"KL/rejected_KL_mean": -36.26857376098633,
"KL/std": 22.970796585083008,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.021835383027791977,
"fcm_dpo/delta": -0.11882907897233963,
"fcm_dpo/margin": 21.132139205932617,
"fcm_dpo/q_t": 0.3920828700065613,
"grad_norm": 24.43920135498047,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6809099912643433,
"logits/rejected": -0.6443426609039307,
"logps/chosen": -82.3125991821289,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -123.56717681884766,
"loss": 1.0414,
"margin_dpo/margin_mean": 21.132139205932617,
"margin_dpo/margin_std": 23.453916549682617,
"step": 84
},
{
"KL/chosen_KL_mean": -15.309465408325195,
"KL/mean": -26.37848663330078,
"KL/rejected_KL_mean": -37.447505950927734,
"KL/std": 23.634626388549805,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.02158314362168312,
"fcm_dpo/delta": -0.13427422940731049,
"fcm_dpo/margin": 22.138042449951172,
"fcm_dpo/q_t": 0.3926207423210144,
"grad_norm": 24.075742721557617,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6298732757568359,
"logits/rejected": -0.5962429046630859,
"logps/chosen": -73.71607971191406,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -116.0863037109375,
"loss": 1.0553,
"margin_dpo/margin_mean": 22.138044357299805,
"margin_dpo/margin_std": 28.41199493408203,
"step": 85
},
{
"KL/chosen_KL_mean": -19.839567184448242,
"KL/mean": -31.57115364074707,
"KL/rejected_KL_mean": -43.302734375,
"KL/std": 32.33202362060547,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.020907826721668243,
"fcm_dpo/delta": -0.14755286276340485,
"fcm_dpo/margin": 23.463176727294922,
"fcm_dpo/q_t": 0.4030860960483551,
"grad_norm": 28.157058715820312,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6940969824790955,
"logits/rejected": -0.6837495565414429,
"logps/chosen": -75.97703552246094,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -131.42437744140625,
"loss": 1.1321,
"margin_dpo/margin_mean": 23.463176727294922,
"margin_dpo/margin_std": 44.730472564697266,
"step": 86
},
{
"KL/chosen_KL_mean": -19.493261337280273,
"KL/mean": -33.350990295410156,
"KL/rejected_KL_mean": -47.20872116088867,
"KL/std": 32.10251998901367,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.019912412390112877,
"fcm_dpo/delta": -0.2153429239988327,
"fcm_dpo/margin": 27.715457916259766,
"fcm_dpo/q_t": 0.37868428230285645,
"grad_norm": 24.050752639770508,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.7316626310348511,
"logits/rejected": -0.7104564905166626,
"logps/chosen": -75.12935638427734,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -142.67630004882812,
"loss": 1.0449,
"margin_dpo/margin_mean": 27.715457916259766,
"margin_dpo/margin_std": 39.20777893066406,
"step": 87
},
{
"KL/chosen_KL_mean": -24.775909423828125,
"KL/mean": -35.22772216796875,
"KL/rejected_KL_mean": -45.679527282714844,
"KL/std": 33.75319290161133,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.019512329250574112,
"fcm_dpo/delta": -0.061230212450027466,
"fcm_dpo/margin": 20.903621673583984,
"fcm_dpo/q_t": 0.41208845376968384,
"grad_norm": 25.076162338256836,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6718587279319763,
"logits/rejected": -0.6672728061676025,
"logps/chosen": -98.44705963134766,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -152.38803100585938,
"loss": 1.1436,
"margin_dpo/margin_mean": 20.903621673583984,
"margin_dpo/margin_std": 35.89170837402344,
"step": 88
},
{
"KL/chosen_KL_mean": -15.323110580444336,
"KL/mean": -27.0899715423584,
"KL/rejected_KL_mean": -38.856834411621094,
"KL/std": 28.511974334716797,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.01931881159543991,
"fcm_dpo/delta": -0.10952495038509369,
"fcm_dpo/margin": 23.533710479736328,
"fcm_dpo/q_t": 0.39783668518066406,
"grad_norm": 22.88800048828125,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7127549648284912,
"logits/rejected": -0.6814610958099365,
"logps/chosen": -75.94802856445312,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -120.94038391113281,
"loss": 1.0703,
"margin_dpo/margin_mean": 23.533714294433594,
"margin_dpo/margin_std": 32.75722122192383,
"step": 89
},
{
"KL/chosen_KL_mean": -19.520963668823242,
"KL/mean": -35.704227447509766,
"KL/rejected_KL_mean": -51.88749313354492,
"KL/std": 38.83220672607422,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.018566761165857315,
"fcm_dpo/delta": -0.26629799604415894,
"fcm_dpo/margin": 32.36653137207031,
"fcm_dpo/q_t": 0.3788330554962158,
"grad_norm": 23.77947235107422,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.656635046005249,
"logits/rejected": -0.6720539331436157,
"logps/chosen": -72.8062744140625,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -163.4322052001953,
"loss": 1.0449,
"margin_dpo/margin_mean": 32.36653137207031,
"margin_dpo/margin_std": 48.41737365722656,
"step": 90
},
{
"KL/chosen_KL_mean": -20.428979873657227,
"KL/mean": -33.47758483886719,
"KL/rejected_KL_mean": -46.52619171142578,
"KL/std": 30.834447860717773,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.01787046156823635,
"fcm_dpo/delta": -0.12188145518302917,
"fcm_dpo/margin": 26.097206115722656,
"fcm_dpo/q_t": 0.39908722043037415,
"grad_norm": 22.826419830322266,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6783984899520874,
"logits/rejected": -0.646409273147583,
"logps/chosen": -82.23193359375,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -134.400146484375,
"loss": 1.089,
"margin_dpo/margin_mean": 26.09720230102539,
"margin_dpo/margin_std": 40.38761901855469,
"step": 91
},
{
"KL/chosen_KL_mean": -18.27140235900879,
"KL/mean": -31.61359977722168,
"KL/rejected_KL_mean": -44.9557991027832,
"KL/std": 31.74585723876953,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.017471997067332268,
"fcm_dpo/delta": -0.12181757390499115,
"fcm_dpo/margin": 26.68439483642578,
"fcm_dpo/q_t": 0.39748257398605347,
"grad_norm": 21.23597526550293,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.7334482669830322,
"logits/rejected": -0.714074969291687,
"logps/chosen": -69.91217041015625,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -122.83698272705078,
"loss": 1.076,
"margin_dpo/margin_mean": 26.68439483642578,
"margin_dpo/margin_std": 38.410037994384766,
"step": 92
},
{
"KL/chosen_KL_mean": -20.64739227294922,
"KL/mean": -33.98801803588867,
"KL/rejected_KL_mean": -47.32863998413086,
"KL/std": 29.834144592285156,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.017019839957356453,
"fcm_dpo/delta": -0.10895700007677078,
"fcm_dpo/margin": 26.681251525878906,
"fcm_dpo/q_t": 0.3978961706161499,
"grad_norm": 22.11734390258789,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7326809167861938,
"logits/rejected": -0.6875460147857666,
"logps/chosen": -73.1766357421875,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -124.48938751220703,
"loss": 1.0595,
"margin_dpo/margin_mean": 26.681249618530273,
"margin_dpo/margin_std": 34.84459686279297,
"step": 93
},
{
"KL/chosen_KL_mean": -22.205469131469727,
"KL/mean": -37.707279205322266,
"KL/rejected_KL_mean": -53.20908737182617,
"KL/std": 34.52550506591797,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.016398325562477112,
"fcm_dpo/delta": -0.1683567464351654,
"fcm_dpo/margin": 31.003623962402344,
"fcm_dpo/q_t": 0.3850182890892029,
"grad_norm": 21.626747131347656,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.6754692792892456,
"logits/rejected": -0.6637367010116577,
"logps/chosen": -83.42808532714844,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -152.80812072753906,
"loss": 1.0229,
"margin_dpo/margin_mean": 31.00362205505371,
"margin_dpo/margin_std": 36.4665641784668,
"step": 94
},
{
"KL/chosen_KL_mean": -21.489198684692383,
"KL/mean": -33.98937225341797,
"KL/rejected_KL_mean": -46.48954772949219,
"KL/std": 33.66114807128906,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.016146240755915642,
"fcm_dpo/delta": -0.05641428753733635,
"fcm_dpo/margin": 25.000350952148438,
"fcm_dpo/q_t": 0.40661683678627014,
"grad_norm": 20.469913482666016,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.6882494688034058,
"logits/rejected": -0.6622629761695862,
"logps/chosen": -74.01284790039062,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -122.3698959350586,
"loss": 1.0993,
"margin_dpo/margin_mean": 25.000349044799805,
"margin_dpo/margin_std": 35.28263854980469,
"step": 95
},
{
"KL/chosen_KL_mean": -20.916810989379883,
"KL/mean": -37.918636322021484,
"KL/rejected_KL_mean": -54.92045593261719,
"KL/std": 37.27104187011719,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.015592942014336586,
"fcm_dpo/delta": -0.19270357489585876,
"fcm_dpo/margin": 34.0036506652832,
"fcm_dpo/q_t": 0.38242608308792114,
"grad_norm": 20.710969924926758,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.707472562789917,
"logits/rejected": -0.6825254559516907,
"logps/chosen": -83.07378387451172,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -151.5164794921875,
"loss": 1.0147,
"margin_dpo/margin_mean": 34.0036506652832,
"margin_dpo/margin_std": 41.2987060546875,
"step": 96
},
{
"KL/chosen_KL_mean": -22.15590476989746,
"KL/mean": -35.208744049072266,
"KL/rejected_KL_mean": -48.26157760620117,
"KL/std": 31.424772262573242,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.015535632148385048,
"fcm_dpo/delta": -0.05870715156197548,
"fcm_dpo/margin": 26.10567855834961,
"fcm_dpo/q_t": 0.40512728691101074,
"grad_norm": 21.590152740478516,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.5965183973312378,
"logits/rejected": -0.5535776019096375,
"logps/chosen": -76.8022689819336,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -125.22633361816406,
"loss": 1.0831,
"margin_dpo/margin_mean": 26.105682373046875,
"margin_dpo/margin_std": 31.313583374023438,
"step": 97
},
{
"KL/chosen_KL_mean": -27.724029541015625,
"KL/mean": -42.68238830566406,
"KL/rejected_KL_mean": -57.64073944091797,
"KL/std": 36.01885986328125,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.015098905190825462,
"fcm_dpo/delta": -0.10677133500576019,
"fcm_dpo/margin": 29.916717529296875,
"fcm_dpo/q_t": 0.39555469155311584,
"grad_norm": 22.497968673706055,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.6522549390792847,
"logits/rejected": -0.6376872062683105,
"logps/chosen": -92.98265838623047,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -154.16822814941406,
"loss": 1.0859,
"margin_dpo/margin_mean": 29.916717529296875,
"margin_dpo/margin_std": 43.398094177246094,
"step": 98
},
{
"KL/chosen_KL_mean": -21.62339210510254,
"KL/mean": -40.180118560791016,
"KL/rejected_KL_mean": -58.736846923828125,
"KL/std": 38.713218688964844,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.014553403481841087,
"fcm_dpo/delta": -0.20181076228618622,
"fcm_dpo/margin": 37.11345672607422,
"fcm_dpo/q_t": 0.3828354477882385,
"grad_norm": 19.82951545715332,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6461166739463806,
"logits/rejected": -0.6486799716949463,
"logps/chosen": -67.26187896728516,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -145.17477416992188,
"loss": 1.0199,
"margin_dpo/margin_mean": 37.11345672607422,
"margin_dpo/margin_std": 48.379791259765625,
"step": 99
},
{
"KL/chosen_KL_mean": -24.61417579650879,
"KL/mean": -35.05047607421875,
"KL/rejected_KL_mean": -45.48677444458008,
"KL/std": 31.448835372924805,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.01442507840692997,
"fcm_dpo/delta": -0.015769226476550102,
"fcm_dpo/margin": 20.872608184814453,
"fcm_dpo/q_t": 0.43076425790786743,
"grad_norm": 21.535491943359375,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6545113325119019,
"logits/rejected": -0.6143167018890381,
"logps/chosen": -82.20816040039062,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -119.54698181152344,
"loss": 1.1936,
"margin_dpo/margin_mean": 20.872608184814453,
"margin_dpo/margin_std": 40.602088928222656,
"step": 100
},
{
"KL/chosen_KL_mean": -30.125629425048828,
"KL/mean": -43.047637939453125,
"KL/rejected_KL_mean": -55.96964645385742,
"KL/std": 38.88969421386719,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.014283875934779644,
"fcm_dpo/delta": -0.020545855164527893,
"fcm_dpo/margin": 25.844009399414062,
"fcm_dpo/q_t": 0.41966480016708374,
"grad_norm": 21.14192771911621,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6575559377670288,
"logits/rejected": -0.6289372444152832,
"logps/chosen": -91.77448272705078,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -139.15933227539062,
"loss": 1.1538,
"margin_dpo/margin_mean": 25.844009399414062,
"margin_dpo/margin_std": 46.674888610839844,
"step": 101
},
{
"KL/chosen_KL_mean": -32.28032684326172,
"KL/mean": -41.69635009765625,
"KL/rejected_KL_mean": -51.112369537353516,
"KL/std": 36.33782958984375,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.014304671436548233,
"fcm_dpo/delta": -0.026983171701431274,
"fcm_dpo/margin": 18.832042694091797,
"fcm_dpo/q_t": 0.4389786124229431,
"grad_norm": 23.982179641723633,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.6497610211372375,
"logits/rejected": -0.6050263047218323,
"logps/chosen": -96.35920715332031,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -119.29944610595703,
"loss": 1.2298,
"margin_dpo/margin_mean": 18.832042694091797,
"margin_dpo/margin_std": 43.388916015625,
"step": 102
},
{
"KL/chosen_KL_mean": -26.217662811279297,
"KL/mean": -45.513275146484375,
"KL/rejected_KL_mean": -64.80888366699219,
"KL/std": 40.776275634765625,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.013833096250891685,
"fcm_dpo/delta": -0.1946529746055603,
"fcm_dpo/margin": 38.59122085571289,
"fcm_dpo/q_t": 0.38289010524749756,
"grad_norm": 20.647781372070312,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.678255021572113,
"logits/rejected": -0.6474459767341614,
"logps/chosen": -87.51693725585938,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -158.381591796875,
"loss": 1.0221,
"margin_dpo/margin_mean": 38.591217041015625,
"margin_dpo/margin_std": 49.838680267333984,
"step": 103
},
{
"KL/chosen_KL_mean": -28.221818923950195,
"KL/mean": -46.5230712890625,
"KL/rejected_KL_mean": -64.8243179321289,
"KL/std": 44.71335983276367,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.013438082300126553,
"fcm_dpo/delta": -0.1490502506494522,
"fcm_dpo/margin": 36.602508544921875,
"fcm_dpo/q_t": 0.3954700827598572,
"grad_norm": 20.52446937561035,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.6646419763565063,
"logits/rejected": -0.6363682746887207,
"logps/chosen": -82.59458923339844,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -154.3890380859375,
"loss": 1.0727,
"margin_dpo/margin_mean": 36.602508544921875,
"margin_dpo/margin_std": 54.992401123046875,
"step": 104
},
{
"KL/chosen_KL_mean": -26.766700744628906,
"KL/mean": -52.79180908203125,
"KL/rejected_KL_mean": -78.81692504882812,
"KL/std": 43.03718185424805,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.012692511081695557,
"fcm_dpo/delta": -0.3323138356208801,
"fcm_dpo/margin": 52.05022430419922,
"fcm_dpo/q_t": 0.348787784576416,
"grad_norm": 20.9357852935791,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.6385953426361084,
"logits/rejected": -0.6172483563423157,
"logps/chosen": -81.40564727783203,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -176.79043579101562,
"loss": 0.9122,
"margin_dpo/margin_mean": 52.05022430419922,
"margin_dpo/margin_std": 47.93716049194336,
"step": 105
},
{
"KL/chosen_KL_mean": -30.216970443725586,
"KL/mean": -46.443695068359375,
"KL/rejected_KL_mean": -62.67041778564453,
"KL/std": 37.993072509765625,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.012338995933532715,
"fcm_dpo/delta": -0.052742164582014084,
"fcm_dpo/margin": 32.45343780517578,
"fcm_dpo/q_t": 0.40674251317977905,
"grad_norm": 19.48843765258789,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.6057268381118774,
"logits/rejected": -0.5791513919830322,
"logps/chosen": -85.04986572265625,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -147.89503479003906,
"loss": 1.0871,
"margin_dpo/margin_mean": 32.45343780517578,
"margin_dpo/margin_std": 41.20294189453125,
"step": 106
},
{
"KL/chosen_KL_mean": -35.853370666503906,
"KL/mean": -54.165565490722656,
"KL/rejected_KL_mean": -72.4777603149414,
"KL/std": 46.64228439331055,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.01210312731564045,
"fcm_dpo/delta": -0.09832476079463959,
"fcm_dpo/margin": 36.62439727783203,
"fcm_dpo/q_t": 0.3984289765357971,
"grad_norm": 19.72745132446289,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.6482241153717041,
"logits/rejected": -0.6045652627944946,
"logps/chosen": -105.56118774414062,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -167.21726989746094,
"loss": 1.0757,
"margin_dpo/margin_mean": 36.62439727783203,
"margin_dpo/margin_std": 49.4898681640625,
"step": 107
},
{
"KL/chosen_KL_mean": -31.727853775024414,
"KL/mean": -54.42365646362305,
"KL/rejected_KL_mean": -77.11946105957031,
"KL/std": 51.947845458984375,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.011752867139875889,
"fcm_dpo/delta": -0.193751722574234,
"fcm_dpo/margin": 45.3916130065918,
"fcm_dpo/q_t": 0.38766244053840637,
"grad_norm": 19.59799575805664,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.6367114782333374,
"logits/rejected": -0.6192809343338013,
"logps/chosen": -87.73773956298828,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -172.91546630859375,
"loss": 1.0566,
"margin_dpo/margin_mean": 45.39160919189453,
"margin_dpo/margin_std": 67.23835754394531,
"step": 108
},
{
"KL/chosen_KL_mean": -29.637109756469727,
"KL/mean": -53.46610641479492,
"KL/rejected_KL_mean": -77.29510498046875,
"KL/std": 49.2529182434082,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.01125399675220251,
"fcm_dpo/delta": -0.19788390398025513,
"fcm_dpo/margin": 47.657989501953125,
"fcm_dpo/q_t": 0.379015177488327,
"grad_norm": 20.628564834594727,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.5834232568740845,
"logits/rejected": -0.5629913806915283,
"logps/chosen": -92.52259826660156,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -175.9808349609375,
"loss": 1.0206,
"margin_dpo/margin_mean": 47.65799331665039,
"margin_dpo/margin_std": 59.39226531982422,
"step": 109
},
{
"KL/chosen_KL_mean": -31.1754150390625,
"KL/mean": -52.55781555175781,
"KL/rejected_KL_mean": -73.94021606445312,
"KL/std": 49.422637939453125,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.01084812544286251,
"fcm_dpo/delta": -0.12373995780944824,
"fcm_dpo/margin": 42.764793395996094,
"fcm_dpo/q_t": 0.3965584337711334,
"grad_norm": 17.69667625427246,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.5745487213134766,
"logits/rejected": -0.5401524305343628,
"logps/chosen": -89.9291000366211,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -153.6902313232422,
"loss": 1.0805,
"margin_dpo/margin_mean": 42.764793395996094,
"margin_dpo/margin_std": 59.920650482177734,
"step": 110
},
{
"KL/chosen_KL_mean": -35.113006591796875,
"KL/mean": -59.724143981933594,
"KL/rejected_KL_mean": -84.33528137207031,
"KL/std": 56.133522033691406,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.010561220347881317,
"fcm_dpo/delta": -0.18071341514587402,
"fcm_dpo/margin": 49.22227478027344,
"fcm_dpo/q_t": 0.38506948947906494,
"grad_norm": 19.484743118286133,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.6309810876846313,
"logits/rejected": -0.6101324558258057,
"logps/chosen": -103.73710632324219,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -182.76414489746094,
"loss": 1.0576,
"margin_dpo/margin_mean": 49.22227478027344,
"margin_dpo/margin_std": 70.08132934570312,
"step": 111
},
{
"KL/chosen_KL_mean": -32.352989196777344,
"KL/mean": -47.761932373046875,
"KL/rejected_KL_mean": -63.17087936401367,
"KL/std": 38.15313720703125,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.010518467053771019,
"fcm_dpo/delta": 0.02647808939218521,
"fcm_dpo/margin": 30.817890167236328,
"fcm_dpo/q_t": 0.4247877597808838,
"grad_norm": 18.049718856811523,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.5829577445983887,
"logits/rejected": -0.5457116365432739,
"logps/chosen": -82.60263061523438,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -127.9453125,
"loss": 1.1474,
"margin_dpo/margin_mean": 30.817893981933594,
"margin_dpo/margin_std": 46.761627197265625,
"step": 112
},
{
"KL/chosen_KL_mean": -41.24876403808594,
"KL/mean": -57.47443389892578,
"KL/rejected_KL_mean": -73.70010375976562,
"KL/std": 40.94397735595703,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.010588676668703556,
"fcm_dpo/delta": 0.00660739466547966,
"fcm_dpo/margin": 32.45133972167969,
"fcm_dpo/q_t": 0.4201717972755432,
"grad_norm": 18.475440979003906,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.5695180892944336,
"logits/rejected": -0.5281089544296265,
"logps/chosen": -107.96171569824219,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -151.66880798339844,
"loss": 1.126,
"margin_dpo/margin_mean": 32.45133972167969,
"margin_dpo/margin_std": 45.22571563720703,
"step": 113
},
{
"KL/chosen_KL_mean": -37.32588195800781,
"KL/mean": -63.44007873535156,
"KL/rejected_KL_mean": -89.55429077148438,
"KL/std": 60.05897903442383,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.010224482975900173,
"fcm_dpo/delta": -0.19627024233341217,
"fcm_dpo/margin": 52.22840881347656,
"fcm_dpo/q_t": 0.38643473386764526,
"grad_norm": 20.448719024658203,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.5776060819625854,
"logits/rejected": -0.5475910902023315,
"logps/chosen": -95.1109619140625,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -176.6639404296875,
"loss": 1.0341,
"margin_dpo/margin_mean": 52.2284049987793,
"margin_dpo/margin_std": 73.12133026123047,
"step": 114
},
{
"KL/chosen_KL_mean": -48.408294677734375,
"KL/mean": -66.1071548461914,
"KL/rejected_KL_mean": -83.80602264404297,
"KL/std": 57.53717803955078,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.010193654336035252,
"fcm_dpo/delta": -0.011369943618774414,
"fcm_dpo/margin": 35.397727966308594,
"fcm_dpo/q_t": 0.42307719588279724,
"grad_norm": 24.033288955688477,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.6035769581794739,
"logits/rejected": -0.587733268737793,
"logps/chosen": -113.99093627929688,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -182.37155151367188,
"loss": 1.1912,
"margin_dpo/margin_mean": 35.397727966308594,
"margin_dpo/margin_std": 76.34100341796875,
"step": 115
},
{
"KL/chosen_KL_mean": -35.714500427246094,
"KL/mean": -55.78123474121094,
"KL/rejected_KL_mean": -75.84798431396484,
"KL/std": 48.34644317626953,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.01011539064347744,
"fcm_dpo/delta": -0.05847224220633507,
"fcm_dpo/margin": 40.13346862792969,
"fcm_dpo/q_t": 0.4071720540523529,
"grad_norm": 19.61532974243164,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.5785757899284363,
"logits/rejected": -0.5719596147537231,
"logps/chosen": -87.11480712890625,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -156.36984252929688,
"loss": 1.0872,
"margin_dpo/margin_mean": 40.13346862792969,
"margin_dpo/margin_std": 52.50077819824219,
"step": 116
},
{
"KL/chosen_KL_mean": -46.457366943359375,
"KL/mean": -62.5472526550293,
"KL/rejected_KL_mean": -78.63714599609375,
"KL/std": 51.26929473876953,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.010081902146339417,
"fcm_dpo/delta": 0.026419375091791153,
"fcm_dpo/margin": 32.179771423339844,
"fcm_dpo/q_t": 0.4264397621154785,
"grad_norm": 24.782325744628906,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.5525811910629272,
"logits/rejected": -0.5172309875488281,
"logps/chosen": -115.75577545166016,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -145.22113037109375,
"loss": 1.1688,
"margin_dpo/margin_mean": 32.17976760864258,
"margin_dpo/margin_std": 58.62355041503906,
"step": 117
},
{
"KL/chosen_KL_mean": -36.07121276855469,
"KL/mean": -55.96504211425781,
"KL/rejected_KL_mean": -75.85887145996094,
"KL/std": 46.31214904785156,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.010025271214544773,
"fcm_dpo/delta": -0.050886109471321106,
"fcm_dpo/margin": 39.787662506103516,
"fcm_dpo/q_t": 0.40897485613822937,
"grad_norm": 19.208255767822266,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.6033087968826294,
"logits/rejected": -0.5862714052200317,
"logps/chosen": -91.71218872070312,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -151.52792358398438,
"loss": 1.0925,
"margin_dpo/margin_mean": 39.78766632080078,
"margin_dpo/margin_std": 53.461585998535156,
"step": 118
},
{
"KL/chosen_KL_mean": -49.75245666503906,
"KL/mean": -69.50601196289062,
"KL/rejected_KL_mean": -89.25955200195312,
"KL/std": 52.264060974121094,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.009865409694612026,
"fcm_dpo/delta": -0.042472440749406815,
"fcm_dpo/margin": 39.50709533691406,
"fcm_dpo/q_t": 0.413967102766037,
"grad_norm": 20.008079528808594,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.5890421271324158,
"logits/rejected": -0.5784463882446289,
"logps/chosen": -123.26264953613281,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -192.23684692382812,
"loss": 1.1279,
"margin_dpo/margin_mean": 39.50709533691406,
"margin_dpo/margin_std": 64.39222717285156,
"step": 119
},
{
"KL/chosen_KL_mean": -50.61544418334961,
"KL/mean": -77.9512939453125,
"KL/rejected_KL_mean": -105.28715515136719,
"KL/std": 65.06802368164062,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.00962957926094532,
"fcm_dpo/delta": -0.18642207980155945,
"fcm_dpo/margin": 54.67171096801758,
"fcm_dpo/q_t": 0.382351815700531,
"grad_norm": 20.747541427612305,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6031701564788818,
"logits/rejected": -0.575164794921875,
"logps/chosen": -127.39627838134766,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -213.31089782714844,
"loss": 1.0242,
"margin_dpo/margin_mean": 54.67170715332031,
"margin_dpo/margin_std": 68.70195007324219,
"step": 120
},
{
"KL/chosen_KL_mean": -49.010231018066406,
"KL/mean": -79.25602722167969,
"KL/rejected_KL_mean": -109.5018310546875,
"KL/std": 63.33201217651367,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.009193992242217064,
"fcm_dpo/delta": -0.21982212364673615,
"fcm_dpo/margin": 60.49159240722656,
"fcm_dpo/q_t": 0.3774503469467163,
"grad_norm": 23.35782814025879,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.5464009642601013,
"logits/rejected": -0.543734073638916,
"logps/chosen": -110.80012512207031,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -219.49639892578125,
"loss": 1.0224,
"margin_dpo/margin_mean": 60.49159240722656,
"margin_dpo/margin_std": 79.29399108886719,
"step": 121
},
{
"KL/chosen_KL_mean": -44.47397232055664,
"KL/mean": -81.9617919921875,
"KL/rejected_KL_mean": -119.44963073730469,
"KL/std": 71.8316650390625,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.008711527101695538,
"fcm_dpo/delta": -0.32415759563446045,
"fcm_dpo/margin": 74.97564697265625,
"fcm_dpo/q_t": 0.3540286719799042,
"grad_norm": 21.536544799804688,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.535602331161499,
"logits/rejected": -0.5521268844604492,
"logps/chosen": -91.37618255615234,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -226.163818359375,
"loss": 0.9307,
"margin_dpo/margin_mean": 74.97564697265625,
"margin_dpo/margin_std": 76.03938293457031,
"step": 122
},
{
"KL/chosen_KL_mean": -49.342002868652344,
"KL/mean": -73.47360229492188,
"KL/rejected_KL_mean": -97.60520935058594,
"KL/std": 60.79751205444336,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.008437592536211014,
"fcm_dpo/delta": -0.06059284880757332,
"fcm_dpo/margin": 48.26319122314453,
"fcm_dpo/q_t": 0.410540372133255,
"grad_norm": 19.794395446777344,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.5162737369537354,
"logits/rejected": -0.5014143586158752,
"logps/chosen": -110.68064880371094,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -185.38059997558594,
"loss": 1.1306,
"margin_dpo/margin_mean": 48.263187408447266,
"margin_dpo/margin_std": 82.03060913085938,
"step": 123
},
{
"KL/chosen_KL_mean": -57.09165954589844,
"KL/mean": -93.51963806152344,
"KL/rejected_KL_mean": -129.94761657714844,
"KL/std": 84.41590881347656,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.008119284175336361,
"fcm_dpo/delta": -0.2575443387031555,
"fcm_dpo/margin": 72.85594177246094,
"fcm_dpo/q_t": 0.37762200832366943,
"grad_norm": 19.897216796875,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.5154942274093628,
"logits/rejected": -0.5195925235748291,
"logps/chosen": -128.5399932861328,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -247.52818298339844,
"loss": 1.0371,
"margin_dpo/margin_mean": 72.85594177246094,
"margin_dpo/margin_std": 106.19161987304688,
"step": 124
},
{
"KL/chosen_KL_mean": -46.80385208129883,
"KL/mean": -78.55594635009766,
"KL/rejected_KL_mean": -110.30804443359375,
"KL/std": 71.42005157470703,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.007860703393816948,
"fcm_dpo/delta": -0.1568271368741989,
"fcm_dpo/margin": 63.504180908203125,
"fcm_dpo/q_t": 0.3909478783607483,
"grad_norm": 17.979494094848633,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.47995638847351074,
"logits/rejected": -0.48471179604530334,
"logps/chosen": -96.9407958984375,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -194.29666137695312,
"loss": 1.0597,
"margin_dpo/margin_mean": 63.504180908203125,
"margin_dpo/margin_std": 89.72810363769531,
"step": 125
},
{
"KL/chosen_KL_mean": -50.658485412597656,
"KL/mean": -80.66192626953125,
"KL/rejected_KL_mean": -110.66535949707031,
"KL/std": 64.82109832763672,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.007639255840331316,
"fcm_dpo/delta": -0.1139136403799057,
"fcm_dpo/margin": 60.00688552856445,
"fcm_dpo/q_t": 0.39801329374313354,
"grad_norm": 19.024023056030273,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.48899418115615845,
"logits/rejected": -0.4869902729988098,
"logps/chosen": -106.32555389404297,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -208.7951202392578,
"loss": 1.0697,
"margin_dpo/margin_mean": 60.00688552856445,
"margin_dpo/margin_std": 83.00294494628906,
"step": 126
},
{
"KL/chosen_KL_mean": -49.5046272277832,
"KL/mean": -70.91242980957031,
"KL/rejected_KL_mean": -92.32023620605469,
"KL/std": 56.90325164794922,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.007639319635927677,
"fcm_dpo/delta": 0.02368098497390747,
"fcm_dpo/margin": 42.81562042236328,
"fcm_dpo/q_t": 0.42351189255714417,
"grad_norm": 18.42085075378418,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.5007494688034058,
"logits/rejected": -0.48394691944122314,
"logps/chosen": -106.05931091308594,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -169.11602783203125,
"loss": 1.1575,
"margin_dpo/margin_mean": 42.815616607666016,
"margin_dpo/margin_std": 71.940673828125,
"step": 127
},
{
"KL/chosen_KL_mean": -52.58698272705078,
"KL/mean": -75.38080596923828,
"KL/rejected_KL_mean": -98.17463684082031,
"KL/std": 66.4472885131836,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.007676831912249327,
"fcm_dpo/delta": -0.000309869647026062,
"fcm_dpo/margin": 45.58763885498047,
"fcm_dpo/q_t": 0.4211137592792511,
"grad_norm": 22.759822845458984,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.46261486411094666,
"logits/rejected": -0.44473135471343994,
"logps/chosen": -110.70793914794922,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -174.61358642578125,
"loss": 1.1643,
"margin_dpo/margin_mean": 45.587646484375,
"margin_dpo/margin_std": 83.7347183227539,
"step": 128
},
{
"KL/chosen_KL_mean": -63.59666442871094,
"KL/mean": -87.52311706542969,
"KL/rejected_KL_mean": -111.44956970214844,
"KL/std": 74.28958892822266,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.007631917484104633,
"fcm_dpo/delta": -0.015783965587615967,
"fcm_dpo/margin": 47.8529052734375,
"fcm_dpo/q_t": 0.42259424924850464,
"grad_norm": 18.812211990356445,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.4808692932128906,
"logits/rejected": -0.47164222598075867,
"logps/chosen": -130.51303100585938,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -208.09178161621094,
"loss": 1.1733,
"margin_dpo/margin_mean": 47.8529052734375,
"margin_dpo/margin_std": 97.08119201660156,
"step": 129
},
{
"KL/chosen_KL_mean": -46.903900146484375,
"KL/mean": -83.26931762695312,
"KL/rejected_KL_mean": -119.6347427368164,
"KL/std": 74.58981323242188,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.007464664056897163,
"fcm_dpo/delta": -0.20387914776802063,
"fcm_dpo/margin": 72.7308349609375,
"fcm_dpo/q_t": 0.381303608417511,
"grad_norm": 19.38671112060547,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.4573250412940979,
"logits/rejected": -0.45176631212234497,
"logps/chosen": -91.57075500488281,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -202.41639709472656,
"loss": 1.0269,
"margin_dpo/margin_mean": 72.7308349609375,
"margin_dpo/margin_std": 91.81611633300781,
"step": 130
},
{
"KL/chosen_KL_mean": -42.863670349121094,
"KL/mean": -79.78742218017578,
"KL/rejected_KL_mean": -116.711181640625,
"KL/std": 76.12089538574219,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.007108722813427448,
"fcm_dpo/delta": -0.18494439125061035,
"fcm_dpo/margin": 73.84750366210938,
"fcm_dpo/q_t": 0.3789236545562744,
"grad_norm": 25.050138473510742,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.41149088740348816,
"logits/rejected": -0.4173048436641693,
"logps/chosen": -87.78825378417969,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -205.15518188476562,
"loss": 1.0083,
"margin_dpo/margin_mean": 73.84750366210938,
"margin_dpo/margin_std": 82.75117492675781,
"step": 131
},
{
"KL/chosen_KL_mean": -56.301605224609375,
"KL/mean": -86.18394470214844,
"KL/rejected_KL_mean": -116.06629943847656,
"KL/std": 74.9560546875,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.006964902393519878,
"fcm_dpo/delta": -0.06934472173452377,
"fcm_dpo/margin": 59.764686584472656,
"fcm_dpo/q_t": 0.41001617908477783,
"grad_norm": 18.39053726196289,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.4645339548587799,
"logits/rejected": -0.46584564447402954,
"logps/chosen": -115.30268859863281,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -203.9584503173828,
"loss": 1.1137,
"margin_dpo/margin_mean": 59.76469421386719,
"margin_dpo/margin_std": 97.89826965332031,
"step": 132
},
{
"KL/chosen_KL_mean": -66.9070053100586,
"KL/mean": -93.52379608154297,
"KL/rejected_KL_mean": -120.14059448242188,
"KL/std": 67.77723693847656,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.006921333260834217,
"fcm_dpo/delta": -0.01925438642501831,
"fcm_dpo/margin": 53.23359680175781,
"fcm_dpo/q_t": 0.4178709387779236,
"grad_norm": 24.08075523376465,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.4671233296394348,
"logits/rejected": -0.455013632774353,
"logps/chosen": -133.51150512695312,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -216.47415161132812,
"loss": 1.1328,
"margin_dpo/margin_mean": 53.23359680175781,
"margin_dpo/margin_std": 86.62762451171875,
"step": 133
},
{
"KL/chosen_KL_mean": -52.65304946899414,
"KL/mean": -80.64926147460938,
"KL/rejected_KL_mean": -108.64547729492188,
"KL/std": 65.32865142822266,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.0069086989387869835,
"fcm_dpo/delta": -0.03839034587144852,
"fcm_dpo/margin": 55.992435455322266,
"fcm_dpo/q_t": 0.4121500253677368,
"grad_norm": 16.892353057861328,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.432298481464386,
"logits/rejected": -0.4217216372489929,
"logps/chosen": -104.72230529785156,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -196.29998779296875,
"loss": 1.0992,
"margin_dpo/margin_mean": 55.992431640625,
"margin_dpo/margin_std": 76.5130615234375,
"step": 134
},
{
"KL/chosen_KL_mean": -57.416786193847656,
"KL/mean": -98.86827087402344,
"KL/rejected_KL_mean": -140.31976318359375,
"KL/std": 87.58612823486328,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.006657840684056282,
"fcm_dpo/delta": -0.2138073444366455,
"fcm_dpo/margin": 82.90298461914062,
"fcm_dpo/q_t": 0.37990349531173706,
"grad_norm": 19.722549438476562,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.44293975830078125,
"logits/rejected": -0.47638678550720215,
"logps/chosen": -107.77064514160156,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -256.29949951171875,
"loss": 1.0199,
"margin_dpo/margin_mean": 82.90296936035156,
"margin_dpo/margin_std": 106.70145416259766,
"step": 135
},
{
"KL/chosen_KL_mean": -68.58837890625,
"KL/mean": -96.16827392578125,
"KL/rejected_KL_mean": -123.74815368652344,
"KL/std": 80.87139892578125,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.0065402439795434475,
"fcm_dpo/delta": -0.011789493262767792,
"fcm_dpo/margin": 55.159759521484375,
"fcm_dpo/q_t": 0.4252905547618866,
"grad_norm": 18.383068084716797,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.4794670045375824,
"logits/rejected": -0.4755587577819824,
"logps/chosen": -133.660888671875,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -220.0693817138672,
"loss": 1.1626,
"margin_dpo/margin_mean": 55.159759521484375,
"margin_dpo/margin_std": 104.8607177734375,
"step": 136
},
{
"KL/chosen_KL_mean": -64.34596252441406,
"KL/mean": -107.5447998046875,
"KL/rejected_KL_mean": -150.74363708496094,
"KL/std": 105.28067016601562,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.006361949257552624,
"fcm_dpo/delta": -0.21148346364498138,
"fcm_dpo/margin": 86.39767456054688,
"fcm_dpo/q_t": 0.3858060836791992,
"grad_norm": 16.011646270751953,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.39821356534957886,
"logits/rejected": -0.424802303314209,
"logps/chosen": -113.10507202148438,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -264.607421875,
"loss": 1.0481,
"margin_dpo/margin_mean": 86.39767456054688,
"margin_dpo/margin_std": 127.21504211425781,
"step": 137
},
{
"KL/chosen_KL_mean": -67.67312622070312,
"KL/mean": -100.0539321899414,
"KL/rejected_KL_mean": -132.4347381591797,
"KL/std": 79.61257934570312,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.006255497224628925,
"fcm_dpo/delta": -0.05740416795015335,
"fcm_dpo/margin": 64.76160430908203,
"fcm_dpo/q_t": 0.4059930443763733,
"grad_norm": 18.952259063720703,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.41318434476852417,
"logits/rejected": -0.40368473529815674,
"logps/chosen": -128.19277954101562,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -225.63168334960938,
"loss": 1.0821,
"margin_dpo/margin_mean": 64.76160430908203,
"margin_dpo/margin_std": 81.33100891113281,
"step": 138
},
{
"KL/chosen_KL_mean": -56.83777618408203,
"KL/mean": -94.96588897705078,
"KL/rejected_KL_mean": -133.093994140625,
"KL/std": 75.09423065185547,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.006126572377979755,
"fcm_dpo/delta": -0.12276843190193176,
"fcm_dpo/margin": 76.2562255859375,
"fcm_dpo/q_t": 0.39481863379478455,
"grad_norm": 18.048723220825195,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.4068523049354553,
"logits/rejected": -0.399114191532135,
"logps/chosen": -103.72915649414062,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -212.82199096679688,
"loss": 1.0533,
"margin_dpo/margin_mean": 76.25623321533203,
"margin_dpo/margin_std": 98.57319641113281,
"step": 139
},
{
"KL/chosen_KL_mean": -68.70712280273438,
"KL/mean": -102.94752502441406,
"KL/rejected_KL_mean": -137.18792724609375,
"KL/std": 82.87702941894531,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.006015871651470661,
"fcm_dpo/delta": -0.06465409696102142,
"fcm_dpo/margin": 68.48081970214844,
"fcm_dpo/q_t": 0.4066656231880188,
"grad_norm": 19.090078353881836,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.39976561069488525,
"logits/rejected": -0.3883041739463806,
"logps/chosen": -127.68183135986328,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -220.4720458984375,
"loss": 1.0998,
"margin_dpo/margin_mean": 68.4808120727539,
"margin_dpo/margin_std": 100.59109497070312,
"step": 140
},
{
"KL/chosen_KL_mean": -78.09503173828125,
"KL/mean": -110.78746032714844,
"KL/rejected_KL_mean": -143.47988891601562,
"KL/std": 93.90158081054688,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.005964947864413261,
"fcm_dpo/delta": -0.041621994227170944,
"fcm_dpo/margin": 65.38487243652344,
"fcm_dpo/q_t": 0.4113631248474121,
"grad_norm": 23.17949104309082,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.38903534412384033,
"logits/rejected": -0.37867170572280884,
"logps/chosen": -153.17068481445312,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -241.67214965820312,
"loss": 1.1311,
"margin_dpo/margin_mean": 65.3848648071289,
"margin_dpo/margin_std": 109.24861145019531,
"step": 141
},
{
"KL/chosen_KL_mean": -78.31304931640625,
"KL/mean": -115.74881744384766,
"KL/rejected_KL_mean": -153.18460083007812,
"KL/std": 100.52348327636719,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.005887279286980629,
"fcm_dpo/delta": -0.09491311013698578,
"fcm_dpo/margin": 74.87155151367188,
"fcm_dpo/q_t": 0.4024774432182312,
"grad_norm": 22.61551284790039,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.34859800338745117,
"logits/rejected": -0.3484548330307007,
"logps/chosen": -136.34097290039062,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -247.76681518554688,
"loss": 1.108,
"margin_dpo/margin_mean": 74.87155151367188,
"margin_dpo/margin_std": 118.29385375976562,
"step": 142
},
{
"KL/chosen_KL_mean": -82.15674591064453,
"KL/mean": -106.3138427734375,
"KL/rejected_KL_mean": -130.470947265625,
"KL/std": 88.07025146484375,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.005829214118421078,
"fcm_dpo/delta": -0.005923248827457428,
"fcm_dpo/margin": 48.31420135498047,
"fcm_dpo/q_t": 0.4382452368736267,
"grad_norm": 19.985225677490234,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.3621300458908081,
"logits/rejected": -0.35844361782073975,
"logps/chosen": -139.75320434570312,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -209.47052001953125,
"loss": 1.2106,
"margin_dpo/margin_mean": 48.314205169677734,
"margin_dpo/margin_std": 103.77206420898438,
"step": 143
},
{
"KL/chosen_KL_mean": -74.73379516601562,
"KL/mean": -103.55740356445312,
"KL/rejected_KL_mean": -132.38101196289062,
"KL/std": 75.34092712402344,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.005838489159941673,
"fcm_dpo/delta": 0.013897043652832508,
"fcm_dpo/margin": 57.6472053527832,
"fcm_dpo/q_t": 0.4226115942001343,
"grad_norm": 18.863821029663086,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.3513190448284149,
"logits/rejected": -0.3433167338371277,
"logps/chosen": -134.64016723632812,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -214.38125610351562,
"loss": 1.1412,
"margin_dpo/margin_mean": 57.6472053527832,
"margin_dpo/margin_std": 88.47941589355469,
"step": 144
},
{
"KL/chosen_KL_mean": -71.26634979248047,
"KL/mean": -102.69514465332031,
"KL/rejected_KL_mean": -134.12393188476562,
"KL/std": 74.58995056152344,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.005827681161463261,
"fcm_dpo/delta": -0.01694849133491516,
"fcm_dpo/margin": 62.85758972167969,
"fcm_dpo/q_t": 0.4155125617980957,
"grad_norm": 20.758092880249023,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.4076671600341797,
"logits/rejected": -0.398476779460907,
"logps/chosen": -127.86701965332031,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -211.99024963378906,
"loss": 1.1196,
"margin_dpo/margin_mean": 62.85758972167969,
"margin_dpo/margin_std": 91.47295379638672,
"step": 145
},
{
"KL/chosen_KL_mean": -97.09615325927734,
"KL/mean": -121.29571533203125,
"KL/rejected_KL_mean": -145.49526977539062,
"KL/std": 82.2374038696289,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.0058697364293038845,
"fcm_dpo/delta": 0.06787579506635666,
"fcm_dpo/margin": 48.399139404296875,
"fcm_dpo/q_t": 0.43450281023979187,
"grad_norm": 23.965456008911133,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.357689768075943,
"logits/rejected": -0.33603012561798096,
"logps/chosen": -163.0966033935547,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -227.19805908203125,
"loss": 1.2061,
"margin_dpo/margin_mean": 48.39913558959961,
"margin_dpo/margin_std": 99.55833435058594,
"step": 146
},
{
"KL/chosen_KL_mean": -72.37018585205078,
"KL/mean": -105.23790740966797,
"KL/rejected_KL_mean": -138.10562133789062,
"KL/std": 82.96342468261719,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.0058630527928471565,
"fcm_dpo/delta": -0.036948516964912415,
"fcm_dpo/margin": 65.73544311523438,
"fcm_dpo/q_t": 0.41322061419487,
"grad_norm": 19.186176300048828,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.3486284017562866,
"logits/rejected": -0.3298673927783966,
"logps/chosen": -125.77567291259766,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -209.49624633789062,
"loss": 1.1224,
"margin_dpo/margin_mean": 65.73544311523438,
"margin_dpo/margin_std": 103.400634765625,
"step": 147
},
{
"KL/chosen_KL_mean": -71.82711029052734,
"KL/mean": -101.01795959472656,
"KL/rejected_KL_mean": -130.2088165283203,
"KL/std": 84.29672241210938,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.005813123658299446,
"fcm_dpo/delta": -0.08374335616827011,
"fcm_dpo/margin": 58.3817138671875,
"fcm_dpo/q_t": 0.4239063560962677,
"grad_norm": 17.18868064880371,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.45455917716026306,
"logits/rejected": -0.450608491897583,
"logps/chosen": -136.76419067382812,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -233.3026580810547,
"loss": 1.1465,
"margin_dpo/margin_mean": 58.38170623779297,
"margin_dpo/margin_std": 89.03665161132812,
"step": 148
},
{
"KL/chosen_KL_mean": -68.49336242675781,
"KL/mean": -104.46226501464844,
"KL/rejected_KL_mean": -140.43118286132812,
"KL/std": 75.27666473388672,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.005668168887495995,
"fcm_dpo/delta": -0.0618739016354084,
"fcm_dpo/margin": 71.93780517578125,
"fcm_dpo/q_t": 0.4055221676826477,
"grad_norm": 18.368350982666016,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.4379425644874573,
"logits/rejected": -0.42654889822006226,
"logps/chosen": -126.96712493896484,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -239.74591064453125,
"loss": 1.0855,
"margin_dpo/margin_mean": 71.93780517578125,
"margin_dpo/margin_std": 89.95822143554688,
"step": 149
},
{
"KL/chosen_KL_mean": -60.557098388671875,
"KL/mean": -96.39286804199219,
"KL/rejected_KL_mean": -132.2286376953125,
"KL/std": 88.98016357421875,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.005610051564872265,
"fcm_dpo/delta": -0.05549062415957451,
"fcm_dpo/margin": 71.67154693603516,
"fcm_dpo/q_t": 0.41181066632270813,
"grad_norm": 17.013290405273438,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.3922620415687561,
"logits/rejected": -0.3972320556640625,
"logps/chosen": -106.26290893554688,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -215.57623291015625,
"loss": 1.1032,
"margin_dpo/margin_mean": 71.67154693603516,
"margin_dpo/margin_std": 106.37786865234375,
"step": 150
},
{
"KL/chosen_KL_mean": -76.32244873046875,
"KL/mean": -111.68994140625,
"KL/rejected_KL_mean": -147.0574493408203,
"KL/std": 84.279541015625,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.005587285850197077,
"fcm_dpo/delta": -0.04712191969156265,
"fcm_dpo/margin": 70.73500061035156,
"fcm_dpo/q_t": 0.40847840905189514,
"grad_norm": 17.660154342651367,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.46673595905303955,
"logits/rejected": -0.4433782398700714,
"logps/chosen": -146.89328002929688,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -247.52127075195312,
"loss": 1.0921,
"margin_dpo/margin_mean": 70.73500061035156,
"margin_dpo/margin_std": 92.79745483398438,
"step": 151
},
{
"KL/chosen_KL_mean": -66.9847412109375,
"KL/mean": -110.66506958007812,
"KL/rejected_KL_mean": -154.34539794921875,
"KL/std": 86.341552734375,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.005478001199662685,
"fcm_dpo/delta": -0.13483843207359314,
"fcm_dpo/margin": 87.36067199707031,
"fcm_dpo/q_t": 0.39175742864608765,
"grad_norm": 18.47982406616211,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.4129364788532257,
"logits/rejected": -0.4130780100822449,
"logps/chosen": -127.14912414550781,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -260.48583984375,
"loss": 1.0436,
"margin_dpo/margin_mean": 87.36066436767578,
"margin_dpo/margin_std": 106.23178100585938,
"step": 152
},
{
"KL/chosen_KL_mean": -67.83352661132812,
"KL/mean": -104.77737426757812,
"KL/rejected_KL_mean": -141.72122192382812,
"KL/std": 95.0350341796875,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.005398088600486517,
"fcm_dpo/delta": -0.050855688750743866,
"fcm_dpo/margin": 73.8876953125,
"fcm_dpo/q_t": 0.41223540902137756,
"grad_norm": 13.99528694152832,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.4175063967704773,
"logits/rejected": -0.415975421667099,
"logps/chosen": -124.1488037109375,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -227.37704467773438,
"loss": 1.1165,
"margin_dpo/margin_mean": 73.8876953125,
"margin_dpo/margin_std": 117.51593780517578,
"step": 153
},
{
"KL/chosen_KL_mean": -81.009765625,
"KL/mean": -115.98365783691406,
"KL/rejected_KL_mean": -150.95755004882812,
"KL/std": 94.65357208251953,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.005372250452637672,
"fcm_dpo/delta": -0.026948172599077225,
"fcm_dpo/margin": 69.94779205322266,
"fcm_dpo/q_t": 0.41602611541748047,
"grad_norm": 17.572832107543945,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.4119012653827667,
"logits/rejected": -0.4173479378223419,
"logps/chosen": -143.75233459472656,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -255.20175170898438,
"loss": 1.1465,
"margin_dpo/margin_mean": 69.94779205322266,
"margin_dpo/margin_std": 122.20135498046875,
"step": 154
},
{
"KL/chosen_KL_mean": -72.86953735351562,
"KL/mean": -107.74058532714844,
"KL/rejected_KL_mean": -142.6116180419922,
"KL/std": 85.74209594726562,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.005339436233043671,
"fcm_dpo/delta": -0.023303672671318054,
"fcm_dpo/margin": 69.74208068847656,
"fcm_dpo/q_t": 0.41487759351730347,
"grad_norm": 17.113283157348633,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.43295127153396606,
"logits/rejected": -0.41761913895606995,
"logps/chosen": -133.52272033691406,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -220.10382080078125,
"loss": 1.1242,
"margin_dpo/margin_mean": 69.74208068847656,
"margin_dpo/margin_std": 107.39170837402344,
"step": 155
},
{
"KL/chosen_KL_mean": -102.05209350585938,
"KL/mean": -121.50769805908203,
"KL/rejected_KL_mean": -140.9633026123047,
"KL/std": 92.87469482421875,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.005365746095776558,
"fcm_dpo/delta": 0.042066704481840134,
"fcm_dpo/margin": 38.91120910644531,
"fcm_dpo/q_t": 0.45207709074020386,
"grad_norm": 26.29570198059082,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.3665209412574768,
"logits/rejected": -0.3663702607154846,
"logps/chosen": -171.54397583007812,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -218.13259887695312,
"loss": 1.2896,
"margin_dpo/margin_mean": 38.91120910644531,
"margin_dpo/margin_std": 121.42982482910156,
"step": 156
},
{
"KL/chosen_KL_mean": -90.3784408569336,
"KL/mean": -135.53335571289062,
"KL/rejected_KL_mean": -180.68826293945312,
"KL/std": 100.72486877441406,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.005238070152699947,
"fcm_dpo/delta": -0.13022619485855103,
"fcm_dpo/margin": 90.30982208251953,
"fcm_dpo/q_t": 0.3903403580188751,
"grad_norm": 22.826868057250977,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.32157012820243835,
"logits/rejected": -0.32990676164627075,
"logps/chosen": -151.74688720703125,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -288.33465576171875,
"loss": 1.0571,
"margin_dpo/margin_mean": 90.30982971191406,
"margin_dpo/margin_std": 114.6199951171875,
"step": 157
},
{
"KL/chosen_KL_mean": -92.16410827636719,
"KL/mean": -140.3941192626953,
"KL/rejected_KL_mean": -188.62411499023438,
"KL/std": 124.57286834716797,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.005113640800118446,
"fcm_dpo/delta": -0.1506519466638565,
"fcm_dpo/margin": 96.45999908447266,
"fcm_dpo/q_t": 0.3960997462272644,
"grad_norm": 16.963441848754883,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.31055477261543274,
"logits/rejected": -0.3295873701572418,
"logps/chosen": -149.77703857421875,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -302.31878662109375,
"loss": 1.0799,
"margin_dpo/margin_mean": 96.45999908447266,
"margin_dpo/margin_std": 151.36740112304688,
"step": 158
},
{
"KL/chosen_KL_mean": -104.50662231445312,
"KL/mean": -136.48501586914062,
"KL/rejected_KL_mean": -168.46337890625,
"KL/std": 109.98271179199219,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.0051115090027451515,
"fcm_dpo/delta": 0.023419687524437904,
"fcm_dpo/margin": 63.956756591796875,
"fcm_dpo/q_t": 0.42578351497650146,
"grad_norm": 19.77988624572754,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.33859413862228394,
"logits/rejected": -0.3108740448951721,
"logps/chosen": -186.0669708251953,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -257.3620910644531,
"loss": 1.1747,
"margin_dpo/margin_mean": 63.956748962402344,
"margin_dpo/margin_std": 119.0658950805664,
"step": 159
},
{
"KL/chosen_KL_mean": -105.49040985107422,
"KL/mean": -149.2151336669922,
"KL/rejected_KL_mean": -192.93984985351562,
"KL/std": 116.9592514038086,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.005003074184060097,
"fcm_dpo/delta": -0.09241557866334915,
"fcm_dpo/margin": 87.44945526123047,
"fcm_dpo/q_t": 0.4059435725212097,
"grad_norm": 22.12449073791504,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.31422072649002075,
"logits/rejected": -0.3084676265716553,
"logps/chosen": -171.22128295898438,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -290.15765380859375,
"loss": 1.1166,
"margin_dpo/margin_mean": 87.44944763183594,
"margin_dpo/margin_std": 146.5648193359375,
"step": 160
},
{
"KL/chosen_KL_mean": -89.08880615234375,
"KL/mean": -127.54766845703125,
"KL/rejected_KL_mean": -166.0065460205078,
"KL/std": 91.42471313476562,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.004976830445230007,
"fcm_dpo/delta": -0.03411930426955223,
"fcm_dpo/margin": 76.917724609375,
"fcm_dpo/q_t": 0.4135284423828125,
"grad_norm": 19.238542556762695,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.319388210773468,
"logits/rejected": -0.3227710723876953,
"logps/chosen": -141.52528381347656,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -249.4375,
"loss": 1.1164,
"margin_dpo/margin_mean": 76.917724609375,
"margin_dpo/margin_std": 115.2650146484375,
"step": 161
},
{
"KL/chosen_KL_mean": -87.23893737792969,
"KL/mean": -124.27473449707031,
"KL/rejected_KL_mean": -161.310546875,
"KL/std": 101.31791687011719,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.0048839072696864605,
"fcm_dpo/delta": -0.086455799639225,
"fcm_dpo/margin": 74.07159423828125,
"fcm_dpo/q_t": 0.4182526171207428,
"grad_norm": 19.80852508544922,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.3837524354457855,
"logits/rejected": -0.37143242359161377,
"logps/chosen": -149.84951782226562,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -250.70111083984375,
"loss": 1.141,
"margin_dpo/margin_mean": 74.07159423828125,
"margin_dpo/margin_std": 120.83529663085938,
"step": 162
},
{
"KL/chosen_KL_mean": -96.60433959960938,
"KL/mean": -132.7714080810547,
"KL/rejected_KL_mean": -168.9384765625,
"KL/std": 101.78199768066406,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.004882782697677612,
"fcm_dpo/delta": -0.0033093392848968506,
"fcm_dpo/margin": 72.33414459228516,
"fcm_dpo/q_t": 0.4210611581802368,
"grad_norm": 18.05702018737793,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.2983933985233307,
"logits/rejected": -0.28219735622406006,
"logps/chosen": -151.6259765625,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -244.356689453125,
"loss": 1.1471,
"margin_dpo/margin_mean": 72.33415222167969,
"margin_dpo/margin_std": 123.11458587646484,
"step": 163
},
{
"KL/chosen_KL_mean": -88.20658111572266,
"KL/mean": -133.68885803222656,
"KL/rejected_KL_mean": -179.17112731933594,
"KL/std": 99.22830963134766,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.004785895813256502,
"fcm_dpo/delta": -0.09055467694997787,
"fcm_dpo/margin": 90.96453857421875,
"fcm_dpo/q_t": 0.39980244636535645,
"grad_norm": 19.5128231048584,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.3039168119430542,
"logits/rejected": -0.2934049963951111,
"logps/chosen": -143.84725952148438,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -258.83575439453125,
"loss": 1.068,
"margin_dpo/margin_mean": 90.96453857421875,
"margin_dpo/margin_std": 114.03044128417969,
"step": 164
},
{
"KL/chosen_KL_mean": -94.5549087524414,
"KL/mean": -124.16766357421875,
"KL/rejected_KL_mean": -153.7803955078125,
"KL/std": 84.79214477539062,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.004827073775231838,
"fcm_dpo/delta": 0.06602644920349121,
"fcm_dpo/margin": 59.225494384765625,
"fcm_dpo/q_t": 0.4348486065864563,
"grad_norm": 18.897262573242188,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.32065290212631226,
"logits/rejected": -0.2971528172492981,
"logps/chosen": -155.8656005859375,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -227.45101928710938,
"loss": 1.1927,
"margin_dpo/margin_mean": 59.225494384765625,
"margin_dpo/margin_std": 113.44796752929688,
"step": 165
},
{
"KL/chosen_KL_mean": -84.27728271484375,
"KL/mean": -139.05355834960938,
"KL/rejected_KL_mean": -193.82984924316406,
"KL/std": 123.94483947753906,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.004713358357548714,
"fcm_dpo/delta": -0.17637449502944946,
"fcm_dpo/margin": 109.55255889892578,
"fcm_dpo/q_t": 0.39149314165115356,
"grad_norm": 16.227643966674805,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.30493152141571045,
"logits/rejected": -0.31587445735931396,
"logps/chosen": -135.26087951660156,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -291.92498779296875,
"loss": 1.049,
"margin_dpo/margin_mean": 109.55255889892578,
"margin_dpo/margin_std": 156.71432495117188,
"step": 166
},
{
"KL/chosen_KL_mean": -84.61183166503906,
"KL/mean": -142.53759765625,
"KL/rejected_KL_mean": -200.4633331298828,
"KL/std": 112.2884521484375,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.00455870758742094,
"fcm_dpo/delta": -0.18809391558170319,
"fcm_dpo/margin": 115.85150909423828,
"fcm_dpo/q_t": 0.380374550819397,
"grad_norm": 17.7374267578125,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.29031479358673096,
"logits/rejected": -0.2939522862434387,
"logps/chosen": -135.03591918945312,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -296.4937744140625,
"loss": 1.0076,
"margin_dpo/margin_mean": 115.85150909423828,
"margin_dpo/margin_std": 131.866943359375,
"step": 167
},
{
"KL/chosen_KL_mean": -92.86872863769531,
"KL/mean": -134.5798797607422,
"KL/rejected_KL_mean": -176.29103088378906,
"KL/std": 104.45317077636719,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.00448962114751339,
"fcm_dpo/delta": -0.0256113912910223,
"fcm_dpo/margin": 83.42231750488281,
"fcm_dpo/q_t": 0.41462987661361694,
"grad_norm": 19.607961654663086,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.2678576111793518,
"logits/rejected": -0.259553462266922,
"logps/chosen": -142.33154296875,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -251.59959411621094,
"loss": 1.1117,
"margin_dpo/margin_mean": 83.42231750488281,
"margin_dpo/margin_std": 118.4161376953125,
"step": 168
},
{
"KL/chosen_KL_mean": -97.21165466308594,
"KL/mean": -133.64254760742188,
"KL/rejected_KL_mean": -170.0734405517578,
"KL/std": 102.94970703125,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.0044325110502541065,
"fcm_dpo/delta": -0.06697965413331985,
"fcm_dpo/margin": 72.86180114746094,
"fcm_dpo/q_t": 0.4287607669830322,
"grad_norm": 18.7218017578125,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.28774628043174744,
"logits/rejected": -0.2726348638534546,
"logps/chosen": -157.01510620117188,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -253.419189453125,
"loss": 1.1829,
"margin_dpo/margin_mean": 72.86180114746094,
"margin_dpo/margin_std": 142.5083465576172,
"step": 169
},
{
"KL/chosen_KL_mean": -86.51754760742188,
"KL/mean": -129.5677947998047,
"KL/rejected_KL_mean": -172.6180419921875,
"KL/std": 98.91424560546875,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.0043886760249733925,
"fcm_dpo/delta": -0.02974541112780571,
"fcm_dpo/margin": 86.10049438476562,
"fcm_dpo/q_t": 0.41227343678474426,
"grad_norm": 15.836402893066406,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.2773169279098511,
"logits/rejected": -0.26892268657684326,
"logps/chosen": -135.98931884765625,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -248.53538513183594,
"loss": 1.1099,
"margin_dpo/margin_mean": 86.10049438476562,
"margin_dpo/margin_std": 118.6636734008789,
"step": 170
},
{
"KL/chosen_KL_mean": -125.79786682128906,
"KL/mean": -160.0425262451172,
"KL/rejected_KL_mean": -194.28720092773438,
"KL/std": 116.10159301757812,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.004434281028807163,
"fcm_dpo/delta": 0.04777422547340393,
"fcm_dpo/margin": 68.48933410644531,
"fcm_dpo/q_t": 0.4349101185798645,
"grad_norm": 22.20769691467285,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.3094336986541748,
"logits/rejected": -0.2986791431903839,
"logps/chosen": -210.29718017578125,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -303.6693115234375,
"loss": 1.2027,
"margin_dpo/margin_mean": 68.48932647705078,
"margin_dpo/margin_std": 149.46047973632812,
"step": 171
},
{
"KL/chosen_KL_mean": -109.93336486816406,
"KL/mean": -146.34115600585938,
"KL/rejected_KL_mean": -182.74893188476562,
"KL/std": 112.77561950683594,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.0044615124352276325,
"fcm_dpo/delta": 0.025982577353715897,
"fcm_dpo/margin": 72.8155517578125,
"fcm_dpo/q_t": 0.4260951280593872,
"grad_norm": 16.35190773010254,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.33784881234169006,
"logits/rejected": -0.32723069190979004,
"logps/chosen": -178.5872802734375,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -268.18560791015625,
"loss": 1.1606,
"margin_dpo/margin_mean": 72.8155517578125,
"margin_dpo/margin_std": 125.91693115234375,
"step": 172
},
{
"KL/chosen_KL_mean": -99.22401428222656,
"KL/mean": -135.5485076904297,
"KL/rejected_KL_mean": -171.87301635742188,
"KL/std": 104.18135070800781,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.0045021940022706985,
"fcm_dpo/delta": 0.02339586801826954,
"fcm_dpo/margin": 72.64897918701172,
"fcm_dpo/q_t": 0.42472150921821594,
"grad_norm": 18.52365493774414,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.3810299038887024,
"logits/rejected": -0.35811132192611694,
"logps/chosen": -162.27490234375,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -250.5569305419922,
"loss": 1.1417,
"margin_dpo/margin_mean": 72.64897918701172,
"margin_dpo/margin_std": 108.23287963867188,
"step": 173
},
{
"KL/chosen_KL_mean": -92.88302612304688,
"KL/mean": -136.38131713867188,
"KL/rejected_KL_mean": -179.87962341308594,
"KL/std": 106.6904296875,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.004488729871809483,
"fcm_dpo/delta": -0.04252074658870697,
"fcm_dpo/margin": 86.99659729003906,
"fcm_dpo/q_t": 0.41205543279647827,
"grad_norm": 18.965898513793945,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.3001505732536316,
"logits/rejected": -0.318511426448822,
"logps/chosen": -146.24600219726562,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -281.79083251953125,
"loss": 1.1136,
"margin_dpo/margin_mean": 86.99659729003906,
"margin_dpo/margin_std": 129.03590393066406,
"step": 174
},
{
"KL/chosen_KL_mean": -84.55975341796875,
"KL/mean": -143.66534423828125,
"KL/rejected_KL_mean": -202.77093505859375,
"KL/std": 113.07514953613281,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.004336726851761341,
"fcm_dpo/delta": -0.17170114815235138,
"fcm_dpo/margin": 118.21118927001953,
"fcm_dpo/q_t": 0.3807182312011719,
"grad_norm": 26.023164749145508,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.29982995986938477,
"logits/rejected": -0.28770336508750916,
"logps/chosen": -129.97752380371094,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -292.2767333984375,
"loss": 0.9978,
"margin_dpo/margin_mean": 118.21118927001953,
"margin_dpo/margin_std": 118.14212036132812,
"step": 175
},
{
"KL/chosen_KL_mean": -91.14138793945312,
"KL/mean": -141.39892578125,
"KL/rejected_KL_mean": -191.65646362304688,
"KL/std": 113.75053405761719,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.004258063621819019,
"fcm_dpo/delta": -0.08141526579856873,
"fcm_dpo/margin": 100.51506042480469,
"fcm_dpo/q_t": 0.40386536717414856,
"grad_norm": 17.673473358154297,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.2949311137199402,
"logits/rejected": -0.31282657384872437,
"logps/chosen": -141.59423828125,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -287.2154235839844,
"loss": 1.0742,
"margin_dpo/margin_mean": 100.51506042480469,
"margin_dpo/margin_std": 131.3572998046875,
"step": 176
},
{
"KL/chosen_KL_mean": -106.9547119140625,
"KL/mean": -156.6512451171875,
"KL/rejected_KL_mean": -206.34779357910156,
"KL/std": 124.73439025878906,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.004201659932732582,
"fcm_dpo/delta": -0.07051907479763031,
"fcm_dpo/margin": 99.39309692382812,
"fcm_dpo/q_t": 0.40569955110549927,
"grad_norm": 22.143009185791016,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.34562036395072937,
"logits/rejected": -0.34030306339263916,
"logps/chosen": -168.17117309570312,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -302.2415771484375,
"loss": 1.0813,
"margin_dpo/margin_mean": 99.39309692382812,
"margin_dpo/margin_std": 132.520263671875,
"step": 177
},
{
"KL/chosen_KL_mean": -119.19122314453125,
"KL/mean": -181.4816131591797,
"KL/rejected_KL_mean": -243.77198791503906,
"KL/std": 145.41358947753906,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.004061376675963402,
"fcm_dpo/delta": -0.16501104831695557,
"fcm_dpo/margin": 124.58078002929688,
"fcm_dpo/q_t": 0.38797473907470703,
"grad_norm": 22.72697639465332,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.19636118412017822,
"logits/rejected": -0.18638736009597778,
"logps/chosen": -177.4560089111328,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -349.1373291015625,
"loss": 1.0348,
"margin_dpo/margin_mean": 124.58078002929688,
"margin_dpo/margin_std": 159.75628662109375,
"step": 178
},
{
"KL/chosen_KL_mean": -124.12211608886719,
"KL/mean": -167.00396728515625,
"KL/rejected_KL_mean": -209.88583374023438,
"KL/std": 125.32534790039062,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.00402455497533083,
"fcm_dpo/delta": 0.004620734602212906,
"fcm_dpo/margin": 85.76371765136719,
"fcm_dpo/q_t": 0.4227069020271301,
"grad_norm": 29.680816650390625,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.26940906047821045,
"logits/rejected": -0.2804524302482605,
"logps/chosen": -185.18045043945312,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -300.41363525390625,
"loss": 1.1654,
"margin_dpo/margin_mean": 85.76371002197266,
"margin_dpo/margin_std": 157.05172729492188,
"step": 179
},
{
"KL/chosen_KL_mean": -109.26609802246094,
"KL/mean": -166.31593322753906,
"KL/rejected_KL_mean": -223.3657684326172,
"KL/std": 112.9661865234375,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.003975175321102142,
"fcm_dpo/delta": -0.10860335826873779,
"fcm_dpo/margin": 114.09965515136719,
"fcm_dpo/q_t": 0.39550089836120605,
"grad_norm": 17.493207931518555,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.24312031269073486,
"logits/rejected": -0.2381986677646637,
"logps/chosen": -163.60882568359375,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -321.5776062011719,
"loss": 1.0554,
"margin_dpo/margin_mean": 114.09966278076172,
"margin_dpo/margin_std": 141.2296142578125,
"step": 180
},
{
"KL/chosen_KL_mean": -95.30699920654297,
"KL/mean": -130.60113525390625,
"KL/rejected_KL_mean": -165.895263671875,
"KL/std": 105.03799438476562,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.004001600667834282,
"fcm_dpo/delta": 0.06941692531108856,
"fcm_dpo/margin": 70.58827209472656,
"fcm_dpo/q_t": 0.4350418150424957,
"grad_norm": 17.06695556640625,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.2419101893901825,
"logits/rejected": -0.21502138674259186,
"logps/chosen": -150.30746459960938,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -227.55142211914062,
"loss": 1.1912,
"margin_dpo/margin_mean": 70.58827209472656,
"margin_dpo/margin_std": 133.7356719970703,
"step": 181
},
{
"KL/chosen_KL_mean": -91.21590423583984,
"KL/mean": -153.8636016845703,
"KL/rejected_KL_mean": -216.51129150390625,
"KL/std": 126.58869934082031,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.003911532927304506,
"fcm_dpo/delta": -0.14804580807685852,
"fcm_dpo/margin": 125.29539489746094,
"fcm_dpo/q_t": 0.3870447278022766,
"grad_norm": 19.177839279174805,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.24630862474441528,
"logits/rejected": -0.257821261882782,
"logps/chosen": -132.32376098632812,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -306.03289794921875,
"loss": 1.0378,
"margin_dpo/margin_mean": 125.29539489746094,
"margin_dpo/margin_std": 152.4662322998047,
"step": 182
},
{
"KL/chosen_KL_mean": -132.7602996826172,
"KL/mean": -165.40869140625,
"KL/rejected_KL_mean": -198.0570831298828,
"KL/std": 105.83728790283203,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.003861016361042857,
"fcm_dpo/delta": -0.043808713555336,
"fcm_dpo/margin": 65.29679107666016,
"fcm_dpo/q_t": 0.44055789709091187,
"grad_norm": 19.30330467224121,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.26778051257133484,
"logits/rejected": -0.2602764964103699,
"logps/chosen": -190.28485107421875,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -274.0328063964844,
"loss": 1.205,
"margin_dpo/margin_mean": 65.29679107666016,
"margin_dpo/margin_std": 115.09003448486328,
"step": 183
},
{
"KL/chosen_KL_mean": -112.4358139038086,
"KL/mean": -147.748291015625,
"KL/rejected_KL_mean": -183.06076049804688,
"KL/std": 100.15349578857422,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.00385840330272913,
"fcm_dpo/delta": -0.0067726364359259605,
"fcm_dpo/margin": 70.62494659423828,
"fcm_dpo/q_t": 0.4372936487197876,
"grad_norm": 19.006750106811523,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.3127518892288208,
"logits/rejected": -0.30516594648361206,
"logps/chosen": -170.98077392578125,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -259.69482421875,
"loss": 1.1911,
"margin_dpo/margin_mean": 70.62494659423828,
"margin_dpo/margin_std": 126.89500427246094,
"step": 184
},
{
"KL/chosen_KL_mean": -119.8270492553711,
"KL/mean": -147.22073364257812,
"KL/rejected_KL_mean": -174.61444091796875,
"KL/std": 113.41376495361328,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.0038557900115847588,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 54.78740310668945,
"fcm_dpo/q_t": 0.45315247774124146,
"grad_norm": 18.04941749572754,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.27935075759887695,
"logits/rejected": -0.2586449682712555,
"logps/chosen": -181.8529052734375,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -248.37704467773438,
"loss": 1.2524,
"margin_dpo/margin_mean": 54.78739929199219,
"margin_dpo/margin_std": 138.59088134765625,
"step": 185
},
{
"KL/chosen_KL_mean": -109.99482727050781,
"KL/mean": -162.43775939941406,
"KL/rejected_KL_mean": -214.88067626953125,
"KL/std": 113.16603088378906,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.0038217236287891865,
"fcm_dpo/delta": -0.05303904414176941,
"fcm_dpo/margin": 104.88587188720703,
"fcm_dpo/q_t": 0.4073418080806732,
"grad_norm": 22.06662940979004,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.3389773666858673,
"logits/rejected": -0.32196861505508423,
"logps/chosen": -179.34829711914062,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -302.953125,
"loss": 1.0831,
"margin_dpo/margin_mean": 104.8858642578125,
"margin_dpo/margin_std": 130.91024780273438,
"step": 186
},
{
"KL/chosen_KL_mean": -101.02510070800781,
"KL/mean": -145.2807159423828,
"KL/rejected_KL_mean": -189.5363311767578,
"KL/std": 107.92250061035156,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.0038345775101333857,
"fcm_dpo/delta": 0.010683363303542137,
"fcm_dpo/margin": 88.51124572753906,
"fcm_dpo/q_t": 0.4206673204898834,
"grad_norm": 21.087003707885742,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.2874957323074341,
"logits/rejected": -0.281036376953125,
"logps/chosen": -153.78155517578125,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -271.50543212890625,
"loss": 1.124,
"margin_dpo/margin_mean": 88.51124572753906,
"margin_dpo/margin_std": 116.87910461425781,
"step": 187
},
{
"KL/chosen_KL_mean": -93.75711059570312,
"KL/mean": -147.31398010253906,
"KL/rejected_KL_mean": -200.870849609375,
"KL/std": 119.8453598022461,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.0037920954637229443,
"fcm_dpo/delta": -0.05853914096951485,
"fcm_dpo/margin": 107.1137466430664,
"fcm_dpo/q_t": 0.4064713716506958,
"grad_norm": 26.782394409179688,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.288492351770401,
"logits/rejected": -0.29583364725112915,
"logps/chosen": -143.172607421875,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -290.4112854003906,
"loss": 1.0786,
"margin_dpo/margin_mean": 107.1137466430664,
"margin_dpo/margin_std": 131.83758544921875,
"step": 188
},
{
"KL/chosen_KL_mean": -110.248291015625,
"KL/mean": -153.5526885986328,
"KL/rejected_KL_mean": -196.85708618164062,
"KL/std": 120.16261291503906,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.0037943553179502487,
"fcm_dpo/delta": 0.022069651633501053,
"fcm_dpo/margin": 86.60880279541016,
"fcm_dpo/q_t": 0.4262270927429199,
"grad_norm": 19.686960220336914,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.30030959844589233,
"logits/rejected": -0.28199464082717896,
"logps/chosen": -162.64724731445312,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -269.0244445800781,
"loss": 1.1554,
"margin_dpo/margin_mean": 86.60879516601562,
"margin_dpo/margin_std": 148.0962371826172,
"step": 189
},
{
"KL/chosen_KL_mean": -117.99136352539062,
"KL/mean": -169.86740112304688,
"KL/rejected_KL_mean": -221.74343872070312,
"KL/std": 129.9977264404297,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.0037976130843162537,
"fcm_dpo/delta": -0.04660265892744064,
"fcm_dpo/margin": 103.75205993652344,
"fcm_dpo/q_t": 0.41064584255218506,
"grad_norm": 17.065195083618164,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.34390515089035034,
"logits/rejected": -0.3343849182128906,
"logps/chosen": -182.6744384765625,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -324.2939453125,
"loss": 1.1107,
"margin_dpo/margin_mean": 103.75205993652344,
"margin_dpo/margin_std": 149.59950256347656,
"step": 190
},
{
"KL/chosen_KL_mean": -109.85092163085938,
"KL/mean": -184.33343505859375,
"KL/rejected_KL_mean": -258.8159484863281,
"KL/std": 148.56558227539062,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.0036355718038976192,
"fcm_dpo/delta": -0.2035076916217804,
"fcm_dpo/margin": 148.96502685546875,
"fcm_dpo/q_t": 0.3776732087135315,
"grad_norm": 19.175321578979492,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.3079345226287842,
"logits/rejected": -0.2833190858364105,
"logps/chosen": -178.50979614257812,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -368.95562744140625,
"loss": 0.9954,
"margin_dpo/margin_mean": 148.96502685546875,
"margin_dpo/margin_std": 165.2587432861328,
"step": 191
},
{
"KL/chosen_KL_mean": -141.1156463623047,
"KL/mean": -190.08807373046875,
"KL/rejected_KL_mean": -239.06048583984375,
"KL/std": 129.36810302734375,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.003615614492446184,
"fcm_dpo/delta": -0.0043648164719343185,
"fcm_dpo/margin": 97.9448471069336,
"fcm_dpo/q_t": 0.42013078927993774,
"grad_norm": 20.97152328491211,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.274461030960083,
"logits/rejected": -0.2736404240131378,
"logps/chosen": -210.84255981445312,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -342.3818359375,
"loss": 1.1403,
"margin_dpo/margin_mean": 97.9448471069336,
"margin_dpo/margin_std": 160.0266876220703,
"step": 192
},
{
"KL/chosen_KL_mean": -140.22830200195312,
"KL/mean": -168.90858459472656,
"KL/rejected_KL_mean": -197.5888671875,
"KL/std": 121.005859375,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.0036203060299158096,
"fcm_dpo/delta": 0.0363609679043293,
"fcm_dpo/margin": 57.36055374145508,
"fcm_dpo/q_t": 0.4501846432685852,
"grad_norm": 24.104490280151367,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.3034767210483551,
"logits/rejected": -0.29830050468444824,
"logps/chosen": -200.41879272460938,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -273.9964294433594,
"loss": 1.2644,
"margin_dpo/margin_mean": 57.36054992675781,
"margin_dpo/margin_std": 154.25485229492188,
"step": 193
},
{
"KL/chosen_KL_mean": -87.32644653320312,
"KL/mean": -133.92543029785156,
"KL/rejected_KL_mean": -180.52444458007812,
"KL/std": 99.52772521972656,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.0036298225168138742,
"fcm_dpo/delta": 0.011991962790489197,
"fcm_dpo/margin": 93.19798278808594,
"fcm_dpo/q_t": 0.41966894268989563,
"grad_norm": 15.164206504821777,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.19916538894176483,
"logits/rejected": -0.18301814794540405,
"logps/chosen": -125.16681671142578,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -241.20921325683594,
"loss": 1.115,
"margin_dpo/margin_mean": 93.19798278808594,
"margin_dpo/margin_std": 109.33871459960938,
"step": 194
},
{
"KL/chosen_KL_mean": -137.87490844726562,
"KL/mean": -187.89657592773438,
"KL/rejected_KL_mean": -237.91827392578125,
"KL/std": 124.3742904663086,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.0036336728371679783,
"fcm_dpo/delta": -0.014039820060133934,
"fcm_dpo/margin": 100.0433578491211,
"fcm_dpo/q_t": 0.41508179903030396,
"grad_norm": 24.048799514770508,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.23971587419509888,
"logits/rejected": -0.25928401947021484,
"logps/chosen": -192.7664794921875,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -334.689208984375,
"loss": 1.1062,
"margin_dpo/margin_mean": 100.0433578491211,
"margin_dpo/margin_std": 127.86747741699219,
"step": 195
},
{
"KL/chosen_KL_mean": -107.48291015625,
"KL/mean": -164.11669921875,
"KL/rejected_KL_mean": -220.75048828125,
"KL/std": 124.06842041015625,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.0035826051607728004,
"fcm_dpo/delta": -0.05893465504050255,
"fcm_dpo/margin": 113.26756286621094,
"fcm_dpo/q_t": 0.4072193205356598,
"grad_norm": 15.825559616088867,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.28464287519454956,
"logits/rejected": -0.2717984616756439,
"logps/chosen": -160.7281494140625,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -296.8034362792969,
"loss": 1.0916,
"margin_dpo/margin_mean": 113.2675552368164,
"margin_dpo/margin_std": 148.8846435546875,
"step": 196
},
{
"KL/chosen_KL_mean": -110.76636505126953,
"KL/mean": -154.2803955078125,
"KL/rejected_KL_mean": -197.79441833496094,
"KL/std": 108.77426147460938,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.003618548158556223,
"fcm_dpo/delta": 0.03597838059067726,
"fcm_dpo/margin": 87.02806091308594,
"fcm_dpo/q_t": 0.4268850088119507,
"grad_norm": 17.07707977294922,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.2548871636390686,
"logits/rejected": -0.2504875063896179,
"logps/chosen": -171.18670654296875,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -275.0033264160156,
"loss": 1.1446,
"margin_dpo/margin_mean": 87.02806091308594,
"margin_dpo/margin_std": 124.56130981445312,
"step": 197
},
{
"KL/chosen_KL_mean": -123.58761596679688,
"KL/mean": -179.9723663330078,
"KL/rejected_KL_mean": -236.3571319580078,
"KL/std": 136.68484497070312,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.0035852163564413786,
"fcm_dpo/delta": -0.056587982922792435,
"fcm_dpo/margin": 112.76952362060547,
"fcm_dpo/q_t": 0.40880346298217773,
"grad_norm": 18.692174911499023,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.31638431549072266,
"logits/rejected": -0.3171602487564087,
"logps/chosen": -178.62380981445312,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -333.60040283203125,
"loss": 1.0918,
"margin_dpo/margin_mean": 112.76952362060547,
"margin_dpo/margin_std": 154.90292358398438,
"step": 198
},
{
"KL/chosen_KL_mean": -118.904541015625,
"KL/mean": -175.37063598632812,
"KL/rejected_KL_mean": -231.8367156982422,
"KL/std": 124.37785339355469,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.003529233392328024,
"fcm_dpo/delta": -0.051382582634687424,
"fcm_dpo/margin": 112.93217468261719,
"fcm_dpo/q_t": 0.40841788053512573,
"grad_norm": 21.199235916137695,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.28893113136291504,
"logits/rejected": -0.27144867181777954,
"logps/chosen": -175.73336791992188,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -316.48492431640625,
"loss": 1.0975,
"margin_dpo/margin_mean": 112.93217468261719,
"margin_dpo/margin_std": 153.85972595214844,
"step": 199
},
{
"KL/chosen_KL_mean": -116.62962341308594,
"KL/mean": -172.3009033203125,
"KL/rejected_KL_mean": -227.97215270996094,
"KL/std": 133.75917053222656,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.0035058055073022842,
"fcm_dpo/delta": -0.04227417707443237,
"fcm_dpo/margin": 111.34254455566406,
"fcm_dpo/q_t": 0.41175031661987305,
"grad_norm": 18.752197265625,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.24854665994644165,
"logits/rejected": -0.23521234095096588,
"logps/chosen": -169.69668579101562,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -308.5805969238281,
"loss": 1.1174,
"margin_dpo/margin_mean": 111.34254455566406,
"margin_dpo/margin_std": 172.01788330078125,
"step": 200
},
{
"KL/chosen_KL_mean": -122.89008331298828,
"KL/mean": -179.06658935546875,
"KL/rejected_KL_mean": -235.24310302734375,
"KL/std": 143.4940948486328,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.003493840806186199,
"fcm_dpo/delta": -0.04429582878947258,
"fcm_dpo/margin": 112.35299682617188,
"fcm_dpo/q_t": 0.41010570526123047,
"grad_norm": 18.05536651611328,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.3345472812652588,
"logits/rejected": -0.3335646986961365,
"logps/chosen": -198.29229736328125,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -350.05133056640625,
"loss": 1.107,
"margin_dpo/margin_mean": 112.35299682617188,
"margin_dpo/margin_std": 162.2955322265625,
"step": 201
},
{
"KL/chosen_KL_mean": -127.64757537841797,
"KL/mean": -166.51829528808594,
"KL/rejected_KL_mean": -205.38900756835938,
"KL/std": 123.735107421875,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.0035203341394662857,
"fcm_dpo/delta": 0.07842127978801727,
"fcm_dpo/margin": 77.74143981933594,
"fcm_dpo/q_t": 0.4383637309074402,
"grad_norm": 18.04079246520996,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.2855903208255768,
"logits/rejected": -0.3018990755081177,
"logps/chosen": -177.7489013671875,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -292.3740234375,
"loss": 1.207,
"margin_dpo/margin_mean": 77.74143981933594,
"margin_dpo/margin_std": 161.36936950683594,
"step": 202
},
{
"KL/chosen_KL_mean": -126.76716613769531,
"KL/mean": -176.1123504638672,
"KL/rejected_KL_mean": -225.45751953125,
"KL/std": 128.75897216796875,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.0035307754296809435,
"fcm_dpo/delta": 0.0015939096920192242,
"fcm_dpo/margin": 98.69035339355469,
"fcm_dpo/q_t": 0.4193652272224426,
"grad_norm": 20.859642028808594,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.292889267206192,
"logits/rejected": -0.28478172421455383,
"logps/chosen": -187.37686157226562,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -311.353515625,
"loss": 1.1261,
"margin_dpo/margin_mean": 98.69036102294922,
"margin_dpo/margin_std": 141.41165161132812,
"step": 203
},
{
"KL/chosen_KL_mean": -141.1234130859375,
"KL/mean": -186.42263793945312,
"KL/rejected_KL_mean": -231.7218475341797,
"KL/std": 140.6131591796875,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.0035480433143675327,
"fcm_dpo/delta": 0.029507920145988464,
"fcm_dpo/margin": 90.59844970703125,
"fcm_dpo/q_t": 0.4280925393104553,
"grad_norm": 23.841333389282227,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.39130985736846924,
"logits/rejected": -0.3560817539691925,
"logps/chosen": -221.28836059570312,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -319.4177551269531,
"loss": 1.1717,
"margin_dpo/margin_mean": 90.59844970703125,
"margin_dpo/margin_std": 165.41769409179688,
"step": 204
},
{
"KL/chosen_KL_mean": -137.06591796875,
"KL/mean": -196.98837280273438,
"KL/rejected_KL_mean": -256.910888671875,
"KL/std": 140.85963439941406,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.0034892673138529062,
"fcm_dpo/delta": -0.07289686799049377,
"fcm_dpo/margin": 119.84494018554688,
"fcm_dpo/q_t": 0.4026746153831482,
"grad_norm": 19.944299697875977,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.284223735332489,
"logits/rejected": -0.25634074211120605,
"logps/chosen": -196.45065307617188,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -342.0359191894531,
"loss": 1.0855,
"margin_dpo/margin_mean": 119.84494018554688,
"margin_dpo/margin_std": 155.21694946289062,
"step": 205
},
{
"KL/chosen_KL_mean": -124.25179290771484,
"KL/mean": -189.01766967773438,
"KL/rejected_KL_mean": -253.78358459472656,
"KL/std": 130.41976928710938,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.003440842032432556,
"fcm_dpo/delta": -0.10062892735004425,
"fcm_dpo/margin": 129.5317840576172,
"fcm_dpo/q_t": 0.39475929737091064,
"grad_norm": 21.66822624206543,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.2068089246749878,
"logits/rejected": -0.20935988426208496,
"logps/chosen": -171.21629333496094,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -352.737060546875,
"loss": 1.0468,
"margin_dpo/margin_mean": 129.5317840576172,
"margin_dpo/margin_std": 142.85910034179688,
"step": 206
},
{
"KL/chosen_KL_mean": -115.6485595703125,
"KL/mean": -189.80975341796875,
"KL/rejected_KL_mean": -263.9709777832031,
"KL/std": 151.3675537109375,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.0033577182330191135,
"fcm_dpo/delta": -0.15566033124923706,
"fcm_dpo/margin": 148.32240295410156,
"fcm_dpo/q_t": 0.3879104256629944,
"grad_norm": 23.90052604675293,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.3293609023094177,
"logits/rejected": -0.30513134598731995,
"logps/chosen": -171.70481872558594,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -348.41876220703125,
"loss": 1.0267,
"margin_dpo/margin_mean": 148.32241821289062,
"margin_dpo/margin_std": 176.73928833007812,
"step": 207
},
{
"KL/chosen_KL_mean": -176.27822875976562,
"KL/mean": -233.8270263671875,
"KL/rejected_KL_mean": -291.3758544921875,
"KL/std": 145.08538818359375,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.0033049010671675205,
"fcm_dpo/delta": -0.03175726160407066,
"fcm_dpo/margin": 115.09761047363281,
"fcm_dpo/q_t": 0.41257303953170776,
"grad_norm": 28.714868545532227,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.3144323527812958,
"logits/rejected": -0.2976739704608917,
"logps/chosen": -243.34584045410156,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -385.6627502441406,
"loss": 1.1231,
"margin_dpo/margin_mean": 115.09761047363281,
"margin_dpo/margin_std": 178.362548828125,
"step": 208
},
{
"KL/chosen_KL_mean": -148.74696350097656,
"KL/mean": -200.42440795898438,
"KL/rejected_KL_mean": -252.1018524169922,
"KL/std": 131.3517303466797,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.0033130061347037554,
"fcm_dpo/delta": 0.007838414050638676,
"fcm_dpo/margin": 103.35487365722656,
"fcm_dpo/q_t": 0.42111438512802124,
"grad_norm": 22.145084381103516,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.20952649414539337,
"logits/rejected": -0.1976684331893921,
"logps/chosen": -204.92864990234375,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -333.0433654785156,
"loss": 1.1463,
"margin_dpo/margin_mean": 103.35487365722656,
"margin_dpo/margin_std": 165.46578979492188,
"step": 209
},
{
"KL/chosen_KL_mean": -131.93817138671875,
"KL/mean": -192.3253173828125,
"KL/rejected_KL_mean": -252.71246337890625,
"KL/std": 135.9736328125,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.0032999245449900627,
"fcm_dpo/delta": -0.05067028850317001,
"fcm_dpo/margin": 120.77430725097656,
"fcm_dpo/q_t": 0.4087422788143158,
"grad_norm": 21.08250617980957,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.25526225566864014,
"logits/rejected": -0.2489766627550125,
"logps/chosen": -178.30999755859375,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -329.39410400390625,
"loss": 1.0971,
"margin_dpo/margin_mean": 120.7743148803711,
"margin_dpo/margin_std": 164.735595703125,
"step": 210
},
{
"KL/chosen_KL_mean": -187.6256103515625,
"KL/mean": -238.52076721191406,
"KL/rejected_KL_mean": -289.4158935546875,
"KL/std": 153.42758178710938,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.0032891465816646814,
"fcm_dpo/delta": 0.015694769099354744,
"fcm_dpo/margin": 101.79029083251953,
"fcm_dpo/q_t": 0.42646709084510803,
"grad_norm": 28.087804794311523,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.31508252024650574,
"logits/rejected": -0.28154420852661133,
"logps/chosen": -266.5579833984375,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -376.23687744140625,
"loss": 1.1642,
"margin_dpo/margin_mean": 101.79029083251953,
"margin_dpo/margin_std": 181.40985107421875,
"step": 211
},
{
"KL/chosen_KL_mean": -155.69639587402344,
"KL/mean": -228.75802612304688,
"KL/rejected_KL_mean": -301.81964111328125,
"KL/std": 165.00823974609375,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.003207254456356168,
"fcm_dpo/delta": -0.12612880766391754,
"fcm_dpo/margin": 146.12322998046875,
"fcm_dpo/q_t": 0.39321210980415344,
"grad_norm": 25.95430564880371,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.2920724153518677,
"logits/rejected": -0.2858331799507141,
"logps/chosen": -213.89340209960938,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -404.87750244140625,
"loss": 1.0603,
"margin_dpo/margin_mean": 146.12326049804688,
"margin_dpo/margin_std": 185.58700561523438,
"step": 212
},
{
"KL/chosen_KL_mean": -146.40249633789062,
"KL/mean": -214.92503356933594,
"KL/rejected_KL_mean": -283.44757080078125,
"KL/std": 141.89804077148438,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.0031565751414746046,
"fcm_dpo/delta": -0.08675891906023026,
"fcm_dpo/margin": 137.04510498046875,
"fcm_dpo/q_t": 0.39976924657821655,
"grad_norm": 25.628156661987305,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.30385902523994446,
"logits/rejected": -0.2821686863899231,
"logps/chosen": -213.91519165039062,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -377.3623046875,
"loss": 1.067,
"margin_dpo/margin_mean": 137.04510498046875,
"margin_dpo/margin_std": 169.2763671875,
"step": 213
},
{
"KL/chosen_KL_mean": -128.19113159179688,
"KL/mean": -190.6093292236328,
"KL/rejected_KL_mean": -253.02752685546875,
"KL/std": 141.2437744140625,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.0031324918381869793,
"fcm_dpo/delta": -0.04271193593740463,
"fcm_dpo/margin": 124.83644104003906,
"fcm_dpo/q_t": 0.4102705121040344,
"grad_norm": 23.725318908691406,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.21322600543498993,
"logits/rejected": -0.2185087352991104,
"logps/chosen": -169.7960205078125,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -330.5449523925781,
"loss": 1.0966,
"margin_dpo/margin_mean": 124.83644104003906,
"margin_dpo/margin_std": 166.6226806640625,
"step": 214
},
{
"KL/chosen_KL_mean": -150.10049438476562,
"KL/mean": -216.70033264160156,
"KL/rejected_KL_mean": -283.3001708984375,
"KL/std": 148.46713256835938,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.0030872428324073553,
"fcm_dpo/delta": -0.06415253132581711,
"fcm_dpo/margin": 133.19969177246094,
"fcm_dpo/q_t": 0.40401768684387207,
"grad_norm": 25.86069107055664,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.2457331418991089,
"logits/rejected": -0.23719845712184906,
"logps/chosen": -203.3797607421875,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -373.26483154296875,
"loss": 1.0699,
"margin_dpo/margin_mean": 133.19969177246094,
"margin_dpo/margin_std": 153.16122436523438,
"step": 215
},
{
"KL/chosen_KL_mean": -155.20941162109375,
"KL/mean": -220.2621612548828,
"KL/rejected_KL_mean": -285.3149108886719,
"KL/std": 152.76821899414062,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.0030720389913767576,
"fcm_dpo/delta": -0.05181992053985596,
"fcm_dpo/margin": 130.10549926757812,
"fcm_dpo/q_t": 0.40874841809272766,
"grad_norm": 24.124347686767578,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.24108392000198364,
"logits/rejected": -0.22588925063610077,
"logps/chosen": -204.09719848632812,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -362.51385498046875,
"loss": 1.105,
"margin_dpo/margin_mean": 130.1055145263672,
"margin_dpo/margin_std": 188.24560546875,
"step": 216
},
{
"KL/chosen_KL_mean": -166.11288452148438,
"KL/mean": -242.55723571777344,
"KL/rejected_KL_mean": -319.0015869140625,
"KL/std": 159.12432861328125,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.002996720839291811,
"fcm_dpo/delta": -0.1136351153254509,
"fcm_dpo/margin": 152.88873291015625,
"fcm_dpo/q_t": 0.39497053623199463,
"grad_norm": 21.351091384887695,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.20837268233299255,
"logits/rejected": -0.2034785896539688,
"logps/chosen": -215.95819091796875,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -419.0799255371094,
"loss": 1.046,
"margin_dpo/margin_mean": 152.88873291015625,
"margin_dpo/margin_std": 179.6495361328125,
"step": 217
},
{
"KL/chosen_KL_mean": -172.87002563476562,
"KL/mean": -231.294677734375,
"KL/rejected_KL_mean": -289.71929931640625,
"KL/std": 157.75628662109375,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.0029808084946125746,
"fcm_dpo/delta": 0.0017072586342692375,
"fcm_dpo/margin": 116.84925842285156,
"fcm_dpo/q_t": 0.42078712582588196,
"grad_norm": 20.616666793823242,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.23788803815841675,
"logits/rejected": -0.23454323410987854,
"logps/chosen": -231.44671630859375,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -377.5657043457031,
"loss": 1.141,
"margin_dpo/margin_mean": 116.8492660522461,
"margin_dpo/margin_std": 185.65829467773438,
"step": 218
},
{
"KL/chosen_KL_mean": -183.26531982421875,
"KL/mean": -234.03045654296875,
"KL/rejected_KL_mean": -284.79559326171875,
"KL/std": 163.48797607421875,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.002978936769068241,
"fcm_dpo/delta": -0.023041052743792534,
"fcm_dpo/margin": 101.53028869628906,
"fcm_dpo/q_t": 0.43124303221702576,
"grad_norm": 27.521127700805664,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.30178552865982056,
"logits/rejected": -0.2995387315750122,
"logps/chosen": -244.34918212890625,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -370.62603759765625,
"loss": 1.194,
"margin_dpo/margin_mean": 101.53028869628906,
"margin_dpo/margin_std": 203.3063507080078,
"step": 219
},
{
"KL/chosen_KL_mean": -206.34454345703125,
"KL/mean": -247.07632446289062,
"KL/rejected_KL_mean": -287.80810546875,
"KL/std": 141.79627990722656,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.0029919049702584743,
"fcm_dpo/delta": 0.03325273096561432,
"fcm_dpo/margin": 81.4636001586914,
"fcm_dpo/q_t": 0.44203633069992065,
"grad_norm": 26.6117000579834,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.2598545551300049,
"logits/rejected": -0.23767119646072388,
"logps/chosen": -276.3758239746094,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -375.49365234375,
"loss": 1.2032,
"margin_dpo/margin_mean": 81.4636001586914,
"margin_dpo/margin_std": 145.33209228515625,
"step": 220
},
{
"KL/chosen_KL_mean": -169.64871215820312,
"KL/mean": -264.8304748535156,
"KL/rejected_KL_mean": -360.01226806640625,
"KL/std": 177.01751708984375,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.0029050116427242756,
"fcm_dpo/delta": -0.2145058810710907,
"fcm_dpo/margin": 190.363525390625,
"fcm_dpo/q_t": 0.3714696168899536,
"grad_norm": 22.261398315429688,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.27296262979507446,
"logits/rejected": -0.27987274527549744,
"logps/chosen": -221.8034210205078,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -466.47991943359375,
"loss": 0.9744,
"margin_dpo/margin_mean": 190.363525390625,
"margin_dpo/margin_std": 182.53524780273438,
"step": 221
},
{
"KL/chosen_KL_mean": -172.16732788085938,
"KL/mean": -242.41677856445312,
"KL/rejected_KL_mean": -312.6662292480469,
"KL/std": 161.145263671875,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.002854538382962346,
"fcm_dpo/delta": -0.05328700318932533,
"fcm_dpo/margin": 140.4989013671875,
"fcm_dpo/q_t": 0.4074173867702484,
"grad_norm": 18.149749755859375,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.3163185119628906,
"logits/rejected": -0.3211904764175415,
"logps/chosen": -233.138427734375,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -412.6673583984375,
"loss": 1.0889,
"margin_dpo/margin_mean": 140.4989013671875,
"margin_dpo/margin_std": 182.004150390625,
"step": 222
},
{
"KL/chosen_KL_mean": -180.6187286376953,
"KL/mean": -239.54754638671875,
"KL/rejected_KL_mean": -298.4763488769531,
"KL/std": 154.41107177734375,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.002839556895196438,
"fcm_dpo/delta": 0.01586122065782547,
"fcm_dpo/margin": 117.85763549804688,
"fcm_dpo/q_t": 0.4231345057487488,
"grad_norm": 21.138545989990234,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.24469372630119324,
"logits/rejected": -0.24137765169143677,
"logps/chosen": -233.25930786132812,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -381.3013916015625,
"loss": 1.1542,
"margin_dpo/margin_mean": 117.85763549804688,
"margin_dpo/margin_std": 198.8590545654297,
"step": 223
},
{
"KL/chosen_KL_mean": -159.38333129882812,
"KL/mean": -226.15509033203125,
"KL/rejected_KL_mean": -292.9268493652344,
"KL/std": 172.1931610107422,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.0028151879087090492,
"fcm_dpo/delta": -0.02796056494116783,
"fcm_dpo/margin": 133.54351806640625,
"fcm_dpo/q_t": 0.4151793122291565,
"grad_norm": 22.848051071166992,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.3147471845149994,
"logits/rejected": -0.2979544997215271,
"logps/chosen": -207.97874450683594,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -370.0433349609375,
"loss": 1.1202,
"margin_dpo/margin_mean": 133.54351806640625,
"margin_dpo/margin_std": 199.27239990234375,
"step": 224
},
{
"KL/chosen_KL_mean": -179.8327178955078,
"KL/mean": -258.5877990722656,
"KL/rejected_KL_mean": -337.3428955078125,
"KL/std": 164.62838745117188,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.002788371406495571,
"fcm_dpo/delta": -0.09335584938526154,
"fcm_dpo/margin": 157.5101776123047,
"fcm_dpo/q_t": 0.3994441330432892,
"grad_norm": 19.650955200195312,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.3059282898902893,
"logits/rejected": -0.29627132415771484,
"logps/chosen": -237.83319091796875,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -437.2458190917969,
"loss": 1.0678,
"margin_dpo/margin_mean": 157.51019287109375,
"margin_dpo/margin_std": 203.22439575195312,
"step": 225
},
{
"KL/chosen_KL_mean": -156.86215209960938,
"KL/mean": -220.01309204101562,
"KL/rejected_KL_mean": -283.16400146484375,
"KL/std": 162.33700561523438,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.00276209507137537,
"fcm_dpo/delta": 0.0005985237658023834,
"fcm_dpo/margin": 126.30184936523438,
"fcm_dpo/q_t": 0.42073529958724976,
"grad_norm": 28.77495765686035,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.36503708362579346,
"logits/rejected": -0.3280119001865387,
"logps/chosen": -215.76095581054688,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -361.85174560546875,
"loss": 1.1334,
"margin_dpo/margin_mean": 126.30184936523438,
"margin_dpo/margin_std": 188.94320678710938,
"step": 226
},
{
"KL/chosen_KL_mean": -186.6765899658203,
"KL/mean": -266.1453857421875,
"KL/rejected_KL_mean": -345.6142272949219,
"KL/std": 186.829833984375,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.002731041982769966,
"fcm_dpo/delta": -0.08844669163227081,
"fcm_dpo/margin": 158.93765258789062,
"fcm_dpo/q_t": 0.399799644947052,
"grad_norm": 20.401033401489258,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.35669684410095215,
"logits/rejected": -0.34606364369392395,
"logps/chosen": -245.74876403808594,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -445.026611328125,
"loss": 1.0635,
"margin_dpo/margin_mean": 158.93765258789062,
"margin_dpo/margin_std": 194.41900634765625,
"step": 227
},
{
"KL/chosen_KL_mean": -194.19752502441406,
"KL/mean": -249.16688537597656,
"KL/rejected_KL_mean": -304.1362609863281,
"KL/std": 154.22523498535156,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.002752040047198534,
"fcm_dpo/delta": 0.048819925636053085,
"fcm_dpo/margin": 109.93873596191406,
"fcm_dpo/q_t": 0.42963525652885437,
"grad_norm": 22.93690299987793,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.29892176389694214,
"logits/rejected": -0.28179436922073364,
"logps/chosen": -260.08880615234375,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -395.18499755859375,
"loss": 1.1714,
"margin_dpo/margin_mean": 109.93873596191406,
"margin_dpo/margin_std": 189.5914764404297,
"step": 228
},
{
"KL/chosen_KL_mean": -204.4345703125,
"KL/mean": -262.7249755859375,
"KL/rejected_KL_mean": -321.015380859375,
"KL/std": 179.88668823242188,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.0027676185127347708,
"fcm_dpo/delta": 0.028249580413103104,
"fcm_dpo/margin": 116.58082580566406,
"fcm_dpo/q_t": 0.4248887300491333,
"grad_norm": 27.2257137298584,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.41251590847969055,
"logits/rejected": -0.38618093729019165,
"logps/chosen": -275.14093017578125,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -405.54278564453125,
"loss": 1.1518,
"margin_dpo/margin_mean": 116.58082580566406,
"margin_dpo/margin_std": 186.08026123046875,
"step": 229
},
{
"KL/chosen_KL_mean": -156.17324829101562,
"KL/mean": -247.1909942626953,
"KL/rejected_KL_mean": -338.2087707519531,
"KL/std": 167.25390625,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.0027067074552178383,
"fcm_dpo/delta": -0.1502915769815445,
"fcm_dpo/margin": 182.03550720214844,
"fcm_dpo/q_t": 0.38420677185058594,
"grad_norm": 27.937719345092773,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.30553507804870605,
"logits/rejected": -0.3124315142631531,
"logps/chosen": -195.45526123046875,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -423.8306884765625,
"loss": 1.0113,
"margin_dpo/margin_mean": 182.0355224609375,
"margin_dpo/margin_std": 184.9110107421875,
"step": 230
},
{
"KL/chosen_KL_mean": -176.39178466796875,
"KL/mean": -238.8753662109375,
"KL/rejected_KL_mean": -301.35894775390625,
"KL/std": 148.606201171875,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.0026946812868118286,
"fcm_dpo/delta": 0.013711929321289062,
"fcm_dpo/margin": 124.96714782714844,
"fcm_dpo/q_t": 0.42264679074287415,
"grad_norm": 23.682451248168945,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.38201549649238586,
"logits/rejected": -0.35523295402526855,
"logps/chosen": -239.66824340820312,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -375.48284912109375,
"loss": 1.1305,
"margin_dpo/margin_mean": 124.96714782714844,
"margin_dpo/margin_std": 175.9566192626953,
"step": 231
},
{
"KL/chosen_KL_mean": -205.3118133544922,
"KL/mean": -255.06503295898438,
"KL/rejected_KL_mean": -304.8182678222656,
"KL/std": 167.1653289794922,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.002725243801251054,
"fcm_dpo/delta": 0.0810902863740921,
"fcm_dpo/margin": 99.50645446777344,
"fcm_dpo/q_t": 0.43814921379089355,
"grad_norm": 21.643369674682617,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.39978110790252686,
"logits/rejected": -0.3793877959251404,
"logps/chosen": -276.0605773925781,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -388.79534912109375,
"loss": 1.1853,
"margin_dpo/margin_mean": 99.50645446777344,
"margin_dpo/margin_std": 172.71176147460938,
"step": 232
},
{
"KL/chosen_KL_mean": -179.78868103027344,
"KL/mean": -259.36474609375,
"KL/rejected_KL_mean": -338.9408264160156,
"KL/std": 176.54042053222656,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.002710646251216531,
"fcm_dpo/delta": -0.08498845994472504,
"fcm_dpo/margin": 159.15216064453125,
"fcm_dpo/q_t": 0.4036695659160614,
"grad_norm": 24.3404483795166,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.3906532824039459,
"logits/rejected": -0.3928545117378235,
"logps/chosen": -234.67161560058594,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -446.4208984375,
"loss": 1.0915,
"margin_dpo/margin_mean": 159.15216064453125,
"margin_dpo/margin_std": 231.7545166015625,
"step": 233
},
{
"KL/chosen_KL_mean": -166.94924926757812,
"KL/mean": -249.14198303222656,
"KL/rejected_KL_mean": -331.334716796875,
"KL/std": 158.32025146484375,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.0026394538581371307,
"fcm_dpo/delta": -0.08937112987041473,
"fcm_dpo/margin": 164.385498046875,
"fcm_dpo/q_t": 0.3987475037574768,
"grad_norm": 22.889997482299805,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.3914503753185272,
"logits/rejected": -0.41639894247055054,
"logps/chosen": -211.04376220703125,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -431.34136962890625,
"loss": 1.0537,
"margin_dpo/margin_mean": 164.385498046875,
"margin_dpo/margin_std": 177.35736083984375,
"step": 234
},
{
"KL/chosen_KL_mean": -203.8853302001953,
"KL/mean": -256.8182067871094,
"KL/rejected_KL_mean": -309.75103759765625,
"KL/std": 149.91073608398438,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.0026793205179274082,
"fcm_dpo/delta": 0.0680294781923294,
"fcm_dpo/margin": 105.86572265625,
"fcm_dpo/q_t": 0.43403372168540955,
"grad_norm": 19.758920669555664,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.40560123324394226,
"logits/rejected": -0.38443222641944885,
"logps/chosen": -266.12322998046875,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -400.1461181640625,
"loss": 1.1809,
"margin_dpo/margin_mean": 105.86572265625,
"margin_dpo/margin_std": 183.34307861328125,
"step": 235
},
{
"KL/chosen_KL_mean": -155.01666259765625,
"KL/mean": -242.4228515625,
"KL/rejected_KL_mean": -329.82904052734375,
"KL/std": 166.82810974121094,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.0026379311457276344,
"fcm_dpo/delta": -0.11645685136318207,
"fcm_dpo/margin": 174.8123779296875,
"fcm_dpo/q_t": 0.3907100558280945,
"grad_norm": 33.895503997802734,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.3970375657081604,
"logits/rejected": -0.3981223404407501,
"logps/chosen": -204.3580322265625,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -433.3406677246094,
"loss": 1.0198,
"margin_dpo/margin_mean": 174.8123779296875,
"margin_dpo/margin_std": 160.76687622070312,
"step": 236
},
{
"KL/chosen_KL_mean": -225.36773681640625,
"KL/mean": -289.1072998046875,
"KL/rejected_KL_mean": -352.8468017578125,
"KL/std": 175.09231567382812,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.0026289066299796104,
"fcm_dpo/delta": 0.01535837072879076,
"fcm_dpo/margin": 127.47908020019531,
"fcm_dpo/q_t": 0.42304670810699463,
"grad_norm": 28.638195037841797,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.36658668518066406,
"logits/rejected": -0.3595519959926605,
"logps/chosen": -279.5358581542969,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -447.6271667480469,
"loss": 1.1454,
"margin_dpo/margin_mean": 127.47908020019531,
"margin_dpo/margin_std": 202.83706665039062,
"step": 237
},
{
"KL/chosen_KL_mean": -207.52232360839844,
"KL/mean": -273.25469970703125,
"KL/rejected_KL_mean": -338.9870910644531,
"KL/std": 176.7599639892578,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.002619014820083976,
"fcm_dpo/delta": 0.005497526377439499,
"fcm_dpo/margin": 131.4647674560547,
"fcm_dpo/q_t": 0.4229828417301178,
"grad_norm": 27.573131561279297,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.3411535620689392,
"logits/rejected": -0.35550758242607117,
"logps/chosen": -261.4954528808594,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -428.405029296875,
"loss": 1.1376,
"margin_dpo/margin_mean": 131.4647674560547,
"margin_dpo/margin_std": 203.32174682617188,
"step": 238
},
{
"KL/chosen_KL_mean": -223.70838928222656,
"KL/mean": -298.6877746582031,
"KL/rejected_KL_mean": -373.6671447753906,
"KL/std": 167.540771484375,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.0026005804538726807,
"fcm_dpo/delta": -0.0424700528383255,
"fcm_dpo/margin": 149.958740234375,
"fcm_dpo/q_t": 0.4087134003639221,
"grad_norm": 24.893705368041992,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.3514358401298523,
"logits/rejected": -0.3399575352668762,
"logps/chosen": -281.80621337890625,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -467.26007080078125,
"loss": 1.0948,
"margin_dpo/margin_mean": 149.95875549316406,
"margin_dpo/margin_std": 190.29000854492188,
"step": 239
},
{
"KL/chosen_KL_mean": -231.37310791015625,
"KL/mean": -291.67303466796875,
"KL/rejected_KL_mean": -351.9729919433594,
"KL/std": 181.56192016601562,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.0026145458687096834,
"fcm_dpo/delta": 0.0357041209936142,
"fcm_dpo/margin": 120.59989929199219,
"fcm_dpo/q_t": 0.4276321530342102,
"grad_norm": 30.90320587158203,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.38337743282318115,
"logits/rejected": -0.3629865050315857,
"logps/chosen": -291.98760986328125,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -426.0915222167969,
"loss": 1.1746,
"margin_dpo/margin_mean": 120.59988403320312,
"margin_dpo/margin_std": 218.30465698242188,
"step": 240
},
{
"KL/chosen_KL_mean": -191.98458862304688,
"KL/mean": -287.6497497558594,
"KL/rejected_KL_mean": -383.31494140625,
"KL/std": 194.20921325683594,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.002574965823441744,
"fcm_dpo/delta": -0.14985774457454681,
"fcm_dpo/margin": 191.33033752441406,
"fcm_dpo/q_t": 0.3884301781654358,
"grad_norm": 25.951866149902344,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.4079846739768982,
"logits/rejected": -0.38517332077026367,
"logps/chosen": -258.0756530761719,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -471.37579345703125,
"loss": 1.0299,
"margin_dpo/margin_mean": 191.33033752441406,
"margin_dpo/margin_std": 221.56744384765625,
"step": 241
},
{
"KL/chosen_KL_mean": -215.06796264648438,
"KL/mean": -284.0657958984375,
"KL/rejected_KL_mean": -353.0635986328125,
"KL/std": 165.72854614257812,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.0025596697814762592,
"fcm_dpo/delta": -0.003671940416097641,
"fcm_dpo/margin": 137.99563598632812,
"fcm_dpo/q_t": 0.41891148686408997,
"grad_norm": 24.505521774291992,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.3450043797492981,
"logits/rejected": -0.3202177882194519,
"logps/chosen": -282.931884765625,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -436.4239501953125,
"loss": 1.1243,
"margin_dpo/margin_mean": 137.99563598632812,
"margin_dpo/margin_std": 196.5327606201172,
"step": 242
},
{
"KL/chosen_KL_mean": -214.51419067382812,
"KL/mean": -297.41632080078125,
"KL/rejected_KL_mean": -380.3184814453125,
"KL/std": 182.67172241210938,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.002519626170396805,
"fcm_dpo/delta": -0.07080057263374329,
"fcm_dpo/margin": 165.80429077148438,
"fcm_dpo/q_t": 0.40310847759246826,
"grad_norm": 24.171142578125,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.382672518491745,
"logits/rejected": -0.35550010204315186,
"logps/chosen": -277.59844970703125,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -456.65411376953125,
"loss": 1.0728,
"margin_dpo/margin_mean": 165.80429077148438,
"margin_dpo/margin_std": 204.3806610107422,
"step": 243
},
{
"KL/chosen_KL_mean": -183.78140258789062,
"KL/mean": -269.9775695800781,
"KL/rejected_KL_mean": -356.1737060546875,
"KL/std": 174.60687255859375,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.002478396287187934,
"fcm_dpo/delta": -0.08090801537036896,
"fcm_dpo/margin": 172.39230346679688,
"fcm_dpo/q_t": 0.3996942937374115,
"grad_norm": 25.310270309448242,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.4503272771835327,
"logits/rejected": -0.44473206996917725,
"logps/chosen": -244.92210388183594,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -451.0656433105469,
"loss": 1.0502,
"margin_dpo/margin_mean": 172.39230346679688,
"margin_dpo/margin_std": 177.4683837890625,
"step": 244
},
{
"KL/chosen_KL_mean": -209.92994689941406,
"KL/mean": -275.97613525390625,
"KL/rejected_KL_mean": -342.02227783203125,
"KL/std": 168.3325653076172,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.0024427759926766157,
"fcm_dpo/delta": -0.060425255447626114,
"fcm_dpo/margin": 132.0923309326172,
"fcm_dpo/q_t": 0.42357954382896423,
"grad_norm": 22.16962242126465,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.4270048141479492,
"logits/rejected": -0.39348340034484863,
"logps/chosen": -277.1922302246094,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -429.6623840332031,
"loss": 1.1442,
"margin_dpo/margin_mean": 132.0923309326172,
"margin_dpo/margin_std": 188.43490600585938,
"step": 245
},
{
"KL/chosen_KL_mean": -200.66488647460938,
"KL/mean": -268.2365417480469,
"KL/rejected_KL_mean": -335.80816650390625,
"KL/std": 186.197265625,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.0024440872948616743,
"fcm_dpo/delta": 0.02029471844434738,
"fcm_dpo/margin": 135.14329528808594,
"fcm_dpo/q_t": 0.4241391718387604,
"grad_norm": 25.40897560119629,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.4735276699066162,
"logits/rejected": -0.43419474363327026,
"logps/chosen": -267.3618469238281,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -420.154541015625,
"loss": 1.1388,
"margin_dpo/margin_mean": 135.14329528808594,
"margin_dpo/margin_std": 200.08724975585938,
"step": 246
},
{
"KL/chosen_KL_mean": -234.4573516845703,
"KL/mean": -329.88720703125,
"KL/rejected_KL_mean": -425.31707763671875,
"KL/std": 194.06973266601562,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.0024084593169391155,
"fcm_dpo/delta": -0.11511167883872986,
"fcm_dpo/margin": 190.85971069335938,
"fcm_dpo/q_t": 0.3926267623901367,
"grad_norm": 26.054208755493164,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.4317302405834198,
"logits/rejected": -0.4298396706581116,
"logps/chosen": -291.06268310546875,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -531.6103515625,
"loss": 1.0387,
"margin_dpo/margin_mean": 190.85972595214844,
"margin_dpo/margin_std": 208.8062744140625,
"step": 247
},
{
"KL/chosen_KL_mean": -201.06793212890625,
"KL/mean": -285.0498046875,
"KL/rejected_KL_mean": -369.03167724609375,
"KL/std": 158.5778350830078,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.0023695870768278837,
"fcm_dpo/delta": -0.05050865188241005,
"fcm_dpo/margin": 167.96372985839844,
"fcm_dpo/q_t": 0.4044187664985657,
"grad_norm": 25.75300407409668,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.407694935798645,
"logits/rejected": -0.42660677433013916,
"logps/chosen": -245.1111602783203,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -460.8885498046875,
"loss": 1.0603,
"margin_dpo/margin_mean": 167.96371459960938,
"margin_dpo/margin_std": 153.2666015625,
"step": 248
},
{
"KL/chosen_KL_mean": -262.7314147949219,
"KL/mean": -308.6989440917969,
"KL/rejected_KL_mean": -354.66644287109375,
"KL/std": 177.82664489746094,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.002401418052613735,
"fcm_dpo/delta": 0.05994679778814316,
"fcm_dpo/margin": 91.93504333496094,
"fcm_dpo/q_t": 0.44910717010498047,
"grad_norm": 33.85601806640625,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.43958780169487,
"logits/rejected": -0.4103413224220276,
"logps/chosen": -325.17376708984375,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -435.134521484375,
"loss": 1.2408,
"margin_dpo/margin_mean": 91.9350357055664,
"margin_dpo/margin_std": 213.35684204101562,
"step": 249
},
{
"KL/chosen_KL_mean": -231.9749755859375,
"KL/mean": -314.76470947265625,
"KL/rejected_KL_mean": -397.55438232421875,
"KL/std": 174.78016662597656,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.0023939479142427444,
"fcm_dpo/delta": -0.04844534397125244,
"fcm_dpo/margin": 165.57940673828125,
"fcm_dpo/q_t": 0.40629374980926514,
"grad_norm": 41.83970642089844,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.43187060952186584,
"logits/rejected": -0.3728846311569214,
"logps/chosen": -297.6116943359375,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -471.42620849609375,
"loss": 1.0762,
"margin_dpo/margin_mean": 165.57940673828125,
"margin_dpo/margin_std": 183.59869384765625,
"step": 250
},
{
"KL/chosen_KL_mean": -240.02685546875,
"KL/mean": -298.3031005859375,
"KL/rejected_KL_mean": -356.5793762207031,
"KL/std": 184.30174255371094,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.002399963792413473,
"fcm_dpo/delta": 0.07236499339342117,
"fcm_dpo/margin": 116.55254364013672,
"fcm_dpo/q_t": 0.4352172613143921,
"grad_norm": 23.73138427734375,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3889552354812622,
"logits/rejected": -0.35478660464286804,
"logps/chosen": -297.2095642089844,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -434.2427978515625,
"loss": 1.1848,
"margin_dpo/margin_mean": 116.55254364013672,
"margin_dpo/margin_std": 200.21017456054688,
"step": 251
},
{
"KL/chosen_KL_mean": -222.79547119140625,
"KL/mean": -299.47967529296875,
"KL/rejected_KL_mean": -376.16387939453125,
"KL/std": 155.37213134765625,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0024020476266741753,
"fcm_dpo/delta": -0.01919008605182171,
"fcm_dpo/margin": 153.368408203125,
"fcm_dpo/q_t": 0.4108833074569702,
"grad_norm": 22.33046531677246,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.48540571331977844,
"logits/rejected": -0.44611865282058716,
"logps/chosen": -294.48114013671875,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -460.921875,
"loss": 1.086,
"margin_dpo/margin_mean": 153.368408203125,
"margin_dpo/margin_std": 156.39324951171875,
"step": 252
},
{
"KL/chosen_KL_mean": -193.2376708984375,
"KL/mean": -271.1014404296875,
"KL/rejected_KL_mean": -348.9652099609375,
"KL/std": 167.41372680664062,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.002407216466963291,
"fcm_dpo/delta": -0.02628351002931595,
"fcm_dpo/margin": 155.72752380371094,
"fcm_dpo/q_t": 0.4112589955329895,
"grad_norm": 20.50514793395996,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.44385430216789246,
"logits/rejected": -0.43464112281799316,
"logps/chosen": -262.3716125488281,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -447.667724609375,
"loss": 1.0973,
"margin_dpo/margin_mean": 155.72752380371094,
"margin_dpo/margin_std": 186.75709533691406,
"step": 253
},
{
"KL/chosen_KL_mean": -200.95816040039062,
"KL/mean": -266.77008056640625,
"KL/rejected_KL_mean": -332.58203125,
"KL/std": 181.447998046875,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.002402001526206732,
"fcm_dpo/delta": 0.03495839610695839,
"fcm_dpo/margin": 131.62384033203125,
"fcm_dpo/q_t": 0.42862290143966675,
"grad_norm": 22.96527862548828,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.41199079155921936,
"logits/rejected": -0.39535099267959595,
"logps/chosen": -255.1131591796875,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -422.8896484375,
"loss": 1.1643,
"margin_dpo/margin_mean": 131.62384033203125,
"margin_dpo/margin_std": 228.5364532470703,
"step": 254
},
{
"KL/chosen_KL_mean": -213.82733154296875,
"KL/mean": -285.01959228515625,
"KL/rejected_KL_mean": -356.2118225097656,
"KL/std": 163.05636596679688,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.0024028548505157232,
"fcm_dpo/delta": 0.007957128807902336,
"fcm_dpo/margin": 142.38449096679688,
"fcm_dpo/q_t": 0.4198462963104248,
"grad_norm": 21.276325225830078,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.3709820806980133,
"logits/rejected": -0.3580097556114197,
"logps/chosen": -270.968994140625,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -446.42041015625,
"loss": 1.1336,
"margin_dpo/margin_mean": 142.38449096679688,
"margin_dpo/margin_std": 206.68133544921875,
"step": 255
},
{
"KL/chosen_KL_mean": -192.61761474609375,
"KL/mean": -271.03961181640625,
"KL/rejected_KL_mean": -349.4615478515625,
"KL/std": 179.253662109375,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.0024023070000112057,
"fcm_dpo/delta": -0.027830585837364197,
"fcm_dpo/margin": 156.8439483642578,
"fcm_dpo/q_t": 0.41207343339920044,
"grad_norm": 23.000877380371094,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.3232497274875641,
"logits/rejected": -0.3319256007671356,
"logps/chosen": -247.78109741210938,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -442.02447509765625,
"loss": 1.1021,
"margin_dpo/margin_mean": 156.84396362304688,
"margin_dpo/margin_std": 205.78041076660156,
"step": 256
},
{
"KL/chosen_KL_mean": -190.80270385742188,
"KL/mean": -275.1641845703125,
"KL/rejected_KL_mean": -359.5256652832031,
"KL/std": 190.19740295410156,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.0023642941378057003,
"fcm_dpo/delta": -0.052151791751384735,
"fcm_dpo/margin": 168.72293090820312,
"fcm_dpo/q_t": 0.4079574942588806,
"grad_norm": 21.00106430053711,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.3763759434223175,
"logits/rejected": -0.36591070890426636,
"logps/chosen": -240.22640991210938,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -439.0635986328125,
"loss": 1.0929,
"margin_dpo/margin_mean": 168.72293090820312,
"margin_dpo/margin_std": 215.62423706054688,
"step": 257
},
{
"KL/chosen_KL_mean": -259.0428161621094,
"KL/mean": -340.54241943359375,
"KL/rejected_KL_mean": -422.0419921875,
"KL/std": 201.83018493652344,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.002365153282880783,
"fcm_dpo/delta": -0.036950256675481796,
"fcm_dpo/margin": 162.99920654296875,
"fcm_dpo/q_t": 0.4098193347454071,
"grad_norm": 20.96170425415039,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.3399478793144226,
"logits/rejected": -0.3372340202331543,
"logps/chosen": -318.42694091796875,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -518.0321044921875,
"loss": 1.1166,
"margin_dpo/margin_mean": 162.99920654296875,
"margin_dpo/margin_std": 244.92196655273438,
"step": 258
},
{
"KL/chosen_KL_mean": -242.94644165039062,
"KL/mean": -309.83868408203125,
"KL/rejected_KL_mean": -376.73089599609375,
"KL/std": 193.08038330078125,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.00236131576821208,
"fcm_dpo/delta": 0.03512916713953018,
"fcm_dpo/margin": 133.784423828125,
"fcm_dpo/q_t": 0.4273309111595154,
"grad_norm": 20.399890899658203,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.4055570065975189,
"logits/rejected": -0.40263453125953674,
"logps/chosen": -295.7747802734375,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -465.92254638671875,
"loss": 1.1583,
"margin_dpo/margin_mean": 133.784423828125,
"margin_dpo/margin_std": 217.29360961914062,
"step": 259
},
{
"KL/chosen_KL_mean": -235.74888610839844,
"KL/mean": -328.3739929199219,
"KL/rejected_KL_mean": -420.99908447265625,
"KL/std": 199.5541534423828,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.0023506833240389824,
"fcm_dpo/delta": -0.08932534605264664,
"fcm_dpo/margin": 185.25018310546875,
"fcm_dpo/q_t": 0.40073201060295105,
"grad_norm": 22.08340072631836,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.3329894542694092,
"logits/rejected": -0.34919166564941406,
"logps/chosen": -283.16656494140625,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -516.0888671875,
"loss": 1.0557,
"margin_dpo/margin_mean": 185.25018310546875,
"margin_dpo/margin_std": 216.7088623046875,
"step": 260
},
{
"KL/chosen_KL_mean": -244.0491943359375,
"KL/mean": -328.637451171875,
"KL/rejected_KL_mean": -413.2257080078125,
"KL/std": 211.37762451171875,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.002309254603460431,
"fcm_dpo/delta": -0.04260961338877678,
"fcm_dpo/margin": 169.17654418945312,
"fcm_dpo/q_t": 0.41040754318237305,
"grad_norm": 18.893648147583008,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.30380675196647644,
"logits/rejected": -0.30368590354919434,
"logps/chosen": -297.08056640625,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -501.74066162109375,
"loss": 1.1033,
"margin_dpo/margin_mean": 169.17652893066406,
"margin_dpo/margin_std": 237.78912353515625,
"step": 261
},
{
"KL/chosen_KL_mean": -268.54522705078125,
"KL/mean": -337.58929443359375,
"KL/rejected_KL_mean": -406.63336181640625,
"KL/std": 185.41744995117188,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.002329050563275814,
"fcm_dpo/delta": 0.028969408944249153,
"fcm_dpo/margin": 138.08815002441406,
"fcm_dpo/q_t": 0.4243643879890442,
"grad_norm": 25.439929962158203,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.344732403755188,
"logits/rejected": -0.3178238272666931,
"logps/chosen": -328.16534423828125,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -493.0518798828125,
"loss": 1.1353,
"margin_dpo/margin_mean": 138.08815002441406,
"margin_dpo/margin_std": 185.788330078125,
"step": 262
},
{
"KL/chosen_KL_mean": -235.66673278808594,
"KL/mean": -331.74688720703125,
"KL/rejected_KL_mean": -427.82708740234375,
"KL/std": 215.85923767089844,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.0022833282127976418,
"fcm_dpo/delta": -0.09349527209997177,
"fcm_dpo/margin": 192.16030883789062,
"fcm_dpo/q_t": 0.3982081413269043,
"grad_norm": 23.847797393798828,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.36012065410614014,
"logits/rejected": -0.3382565975189209,
"logps/chosen": -295.0876770019531,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -524.6842651367188,
"loss": 1.061,
"margin_dpo/margin_mean": 192.16033935546875,
"margin_dpo/margin_std": 232.20291137695312,
"step": 263
},
{
"KL/chosen_KL_mean": -241.67784118652344,
"KL/mean": -322.71044921875,
"KL/rejected_KL_mean": -403.7430419921875,
"KL/std": 196.90768432617188,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.002260984852910042,
"fcm_dpo/delta": -0.017950933426618576,
"fcm_dpo/margin": 162.06521606445312,
"fcm_dpo/q_t": 0.4153786301612854,
"grad_norm": 21.233423233032227,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.4053404927253723,
"logits/rejected": -0.3976580500602722,
"logps/chosen": -304.3999328613281,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -497.5992431640625,
"loss": 1.1174,
"margin_dpo/margin_mean": 162.06521606445312,
"margin_dpo/margin_std": 225.04757690429688,
"step": 264
},
{
"KL/chosen_KL_mean": -266.4638366699219,
"KL/mean": -342.8912353515625,
"KL/rejected_KL_mean": -419.3185729980469,
"KL/std": 219.68490600585938,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.002274434082210064,
"fcm_dpo/delta": 0.0024239865597337484,
"fcm_dpo/margin": 152.85475158691406,
"fcm_dpo/q_t": 0.4211368262767792,
"grad_norm": 23.371530532836914,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.3490335941314697,
"logits/rejected": -0.3347071707248688,
"logps/chosen": -328.435302734375,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -507.33917236328125,
"loss": 1.1501,
"margin_dpo/margin_mean": 152.85475158691406,
"margin_dpo/margin_std": 261.6484069824219,
"step": 265
},
{
"KL/chosen_KL_mean": -269.20404052734375,
"KL/mean": -327.8632507324219,
"KL/rejected_KL_mean": -386.5224609375,
"KL/std": 181.0565948486328,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0023039982188493013,
"fcm_dpo/delta": 0.08182872086763382,
"fcm_dpo/margin": 117.31837463378906,
"fcm_dpo/q_t": 0.43739640712738037,
"grad_norm": 47.123313903808594,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.3748672306537628,
"logits/rejected": -0.340556800365448,
"logps/chosen": -336.3037109375,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -454.4936828613281,
"loss": 1.1911,
"margin_dpo/margin_mean": 117.31837463378906,
"margin_dpo/margin_std": 212.25018310546875,
"step": 266
},
{
"KL/chosen_KL_mean": -236.6424560546875,
"KL/mean": -310.0815124511719,
"KL/rejected_KL_mean": -383.52056884765625,
"KL/std": 203.6986083984375,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.0023088420275598764,
"fcm_dpo/delta": 0.011129170656204224,
"fcm_dpo/margin": 146.87808227539062,
"fcm_dpo/q_t": 0.4222862124443054,
"grad_norm": 26.301164627075195,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3820560872554779,
"logits/rejected": -0.3540261387825012,
"logps/chosen": -305.61322021484375,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -473.6889953613281,
"loss": 1.1456,
"margin_dpo/margin_mean": 146.87808227539062,
"margin_dpo/margin_std": 237.48892211914062,
"step": 267
},
{
"KL/chosen_KL_mean": -249.36642456054688,
"KL/mean": -328.30133056640625,
"KL/rejected_KL_mean": -407.23626708984375,
"KL/std": 204.91433715820312,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.002304880414158106,
"fcm_dpo/delta": -0.014616616070270538,
"fcm_dpo/margin": 157.86981201171875,
"fcm_dpo/q_t": 0.41689735651016235,
"grad_norm": 24.127378463745117,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.35487100481987,
"logits/rejected": -0.35821008682250977,
"logps/chosen": -305.2667236328125,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -508.8839111328125,
"loss": 1.1305,
"margin_dpo/margin_mean": 157.86981201171875,
"margin_dpo/margin_std": 247.58233642578125,
"step": 268
},
{
"KL/chosen_KL_mean": -278.9156494140625,
"KL/mean": -371.38323974609375,
"KL/rejected_KL_mean": -463.85076904296875,
"KL/std": 207.69430541992188,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.0022870278917253017,
"fcm_dpo/delta": -0.0761084258556366,
"fcm_dpo/margin": 184.93511962890625,
"fcm_dpo/q_t": 0.40323197841644287,
"grad_norm": 24.954824447631836,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.3960835933685303,
"logits/rejected": -0.37948912382125854,
"logps/chosen": -348.9552307128906,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -571.2001953125,
"loss": 1.095,
"margin_dpo/margin_mean": 184.93511962890625,
"margin_dpo/margin_std": 267.55242919921875,
"step": 269
},
{
"KL/chosen_KL_mean": -247.30697631835938,
"KL/mean": -317.04638671875,
"KL/rejected_KL_mean": -386.7857971191406,
"KL/std": 184.7318115234375,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.002292545512318611,
"fcm_dpo/delta": 0.030957801267504692,
"fcm_dpo/margin": 139.4788360595703,
"fcm_dpo/q_t": 0.42589646577835083,
"grad_norm": 21.303037643432617,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.4222424626350403,
"logits/rejected": -0.4178420305252075,
"logps/chosen": -316.8404541015625,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -496.7144470214844,
"loss": 1.1639,
"margin_dpo/margin_mean": 139.4788360595703,
"margin_dpo/margin_std": 239.0890350341797,
"step": 270
},
{
"KL/chosen_KL_mean": -228.56326293945312,
"KL/mean": -315.3665466308594,
"KL/rejected_KL_mean": -402.1697998046875,
"KL/std": 187.98452758789062,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.0022674724459648132,
"fcm_dpo/delta": -0.04571424052119255,
"fcm_dpo/margin": 173.60653686523438,
"fcm_dpo/q_t": 0.4088677167892456,
"grad_norm": 21.11397361755371,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.3680250346660614,
"logits/rejected": -0.3530626893043518,
"logps/chosen": -285.32781982421875,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -494.68365478515625,
"loss": 1.0878,
"margin_dpo/margin_mean": 173.60653686523438,
"margin_dpo/margin_std": 217.1529541015625,
"step": 271
},
{
"KL/chosen_KL_mean": -215.89785766601562,
"KL/mean": -326.992431640625,
"KL/rejected_KL_mean": -438.0870361328125,
"KL/std": 207.18685913085938,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.0022256127558648586,
"fcm_dpo/delta": -0.1517450511455536,
"fcm_dpo/margin": 222.18914794921875,
"fcm_dpo/q_t": 0.38386717438697815,
"grad_norm": 29.405025482177734,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.35582485795021057,
"logits/rejected": -0.36779093742370605,
"logps/chosen": -265.39501953125,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -543.6298217773438,
"loss": 1.007,
"margin_dpo/margin_mean": 222.18914794921875,
"margin_dpo/margin_std": 215.62319946289062,
"step": 272
},
{
"KL/chosen_KL_mean": -264.82843017578125,
"KL/mean": -371.188720703125,
"KL/rejected_KL_mean": -477.54901123046875,
"KL/std": 212.29110717773438,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.0021587321534752846,
"fcm_dpo/delta": -0.11479845643043518,
"fcm_dpo/margin": 212.72059631347656,
"fcm_dpo/q_t": 0.3933224678039551,
"grad_norm": 31.42890739440918,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.3145291805267334,
"logits/rejected": -0.2950317859649658,
"logps/chosen": -327.8038330078125,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -570.047607421875,
"loss": 1.0435,
"margin_dpo/margin_mean": 212.72059631347656,
"margin_dpo/margin_std": 245.05528259277344,
"step": 273
},
{
"KL/chosen_KL_mean": -320.0135498046875,
"KL/mean": -398.0048828125,
"KL/rejected_KL_mean": -475.9962158203125,
"KL/std": 196.2461700439453,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.0021619275212287903,
"fcm_dpo/delta": 0.012957748025655746,
"fcm_dpo/margin": 155.98263549804688,
"fcm_dpo/q_t": 0.42174428701400757,
"grad_norm": 33.96529769897461,
"learning_rate": 3.75e-07,
"logits/chosen": -0.2863592803478241,
"logits/rejected": -0.26870518922805786,
"logps/chosen": -375.6812744140625,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -553.3292846679688,
"loss": 1.1361,
"margin_dpo/margin_mean": 155.98263549804688,
"margin_dpo/margin_std": 228.75283813476562,
"step": 274
},
{
"KL/chosen_KL_mean": -249.88211059570312,
"KL/mean": -336.5286865234375,
"KL/rejected_KL_mean": -423.1752014160156,
"KL/std": 197.85572814941406,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.0021535661071538925,
"fcm_dpo/delta": -0.024154571816325188,
"fcm_dpo/margin": 173.29312133789062,
"fcm_dpo/q_t": 0.4124983847141266,
"grad_norm": 26.547245025634766,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.3534200191497803,
"logits/rejected": -0.3581305146217346,
"logps/chosen": -298.476806640625,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -516.4788818359375,
"loss": 1.1025,
"margin_dpo/margin_mean": 173.29312133789062,
"margin_dpo/margin_std": 222.11753845214844,
"step": 275
},
{
"KL/chosen_KL_mean": -279.96337890625,
"KL/mean": -363.65765380859375,
"KL/rejected_KL_mean": -447.3519287109375,
"KL/std": 210.06130981445312,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.0021390733309090137,
"fcm_dpo/delta": -0.008444213308393955,
"fcm_dpo/margin": 167.38851928710938,
"fcm_dpo/q_t": 0.4185035824775696,
"grad_norm": 25.184022903442383,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.34166771173477173,
"logits/rejected": -0.323291152715683,
"logps/chosen": -336.540771484375,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -517.7175903320312,
"loss": 1.131,
"margin_dpo/margin_mean": 167.38851928710938,
"margin_dpo/margin_std": 257.09930419921875,
"step": 276
},
{
"KL/chosen_KL_mean": -302.4512939453125,
"KL/mean": -391.5897521972656,
"KL/rejected_KL_mean": -480.72821044921875,
"KL/std": 209.8077850341797,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.0021361508406698704,
"fcm_dpo/delta": -0.032061617821455,
"fcm_dpo/margin": 178.2769012451172,
"fcm_dpo/q_t": 0.41165226697921753,
"grad_norm": 24.468589782714844,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.3302989602088928,
"logits/rejected": -0.3095225393772125,
"logps/chosen": -358.7228698730469,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -573.6094970703125,
"loss": 1.1107,
"margin_dpo/margin_mean": 178.2769012451172,
"margin_dpo/margin_std": 252.54180908203125,
"step": 277
},
{
"KL/chosen_KL_mean": -257.0399169921875,
"KL/mean": -361.0743103027344,
"KL/rejected_KL_mean": -465.1087341308594,
"KL/std": 211.69451904296875,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.002095486968755722,
"fcm_dpo/delta": -0.09022445231676102,
"fcm_dpo/margin": 208.06884765625,
"fcm_dpo/q_t": 0.3987731635570526,
"grad_norm": 20.533023834228516,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.3557165265083313,
"logits/rejected": -0.36121219396591187,
"logps/chosen": -309.9818420410156,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -556.3623046875,
"loss": 1.056,
"margin_dpo/margin_mean": 208.06884765625,
"margin_dpo/margin_std": 240.28025817871094,
"step": 278
},
{
"KL/chosen_KL_mean": -297.38983154296875,
"KL/mean": -400.4170227050781,
"KL/rejected_KL_mean": -503.44427490234375,
"KL/std": 230.00518798828125,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.0020560654811561108,
"fcm_dpo/delta": -0.07781803607940674,
"fcm_dpo/margin": 206.0543975830078,
"fcm_dpo/q_t": 0.40380701422691345,
"grad_norm": 25.10165786743164,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.2838994860649109,
"logits/rejected": -0.2843170762062073,
"logps/chosen": -346.0311279296875,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -591.2957153320312,
"loss": 1.0884,
"margin_dpo/margin_mean": 206.05441284179688,
"margin_dpo/margin_std": 287.540771484375,
"step": 279
},
{
"KL/chosen_KL_mean": -289.91473388671875,
"KL/mean": -394.880859375,
"KL/rejected_KL_mean": -499.84698486328125,
"KL/std": 210.3863525390625,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.0020319321192801,
"fcm_dpo/delta": -0.08006896078586578,
"fcm_dpo/margin": 209.9322052001953,
"fcm_dpo/q_t": 0.39913302659988403,
"grad_norm": 28.02513885498047,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.3529083728790283,
"logits/rejected": -0.35713738203048706,
"logps/chosen": -348.71185302734375,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -598.4658203125,
"loss": 1.0566,
"margin_dpo/margin_mean": 209.9322052001953,
"margin_dpo/margin_std": 231.01846313476562,
"step": 280
},
{
"KL/chosen_KL_mean": -261.54742431640625,
"KL/mean": -353.0968933105469,
"KL/rejected_KL_mean": -444.6463623046875,
"KL/std": 201.8822021484375,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0020250456873327494,
"fcm_dpo/delta": -0.02170453406870365,
"fcm_dpo/margin": 183.09893798828125,
"fcm_dpo/q_t": 0.41141772270202637,
"grad_norm": 22.01675796508789,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.38511866331100464,
"logits/rejected": -0.3614617586135864,
"logps/chosen": -317.03594970703125,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -525.5289306640625,
"loss": 1.0996,
"margin_dpo/margin_mean": 183.0989227294922,
"margin_dpo/margin_std": 222.86624145507812,
"step": 281
},
{
"KL/chosen_KL_mean": -272.9205322265625,
"KL/mean": -343.1865234375,
"KL/rejected_KL_mean": -413.4525146484375,
"KL/std": 208.0769500732422,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.0020167033653706312,
"fcm_dpo/delta": 7.871291745686904e-05,
"fcm_dpo/margin": 140.53196716308594,
"fcm_dpo/q_t": 0.43511486053466797,
"grad_norm": 22.090290069580078,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.4389272928237915,
"logits/rejected": -0.41942620277404785,
"logps/chosen": -345.99066162109375,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -508.80352783203125,
"loss": 1.1751,
"margin_dpo/margin_mean": 140.53196716308594,
"margin_dpo/margin_std": 235.73179626464844,
"step": 282
},
{
"KL/chosen_KL_mean": -265.983154296875,
"KL/mean": -372.7215576171875,
"KL/rejected_KL_mean": -479.46002197265625,
"KL/std": 232.66961669921875,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.0020004159305244684,
"fcm_dpo/delta": -0.08047128468751907,
"fcm_dpo/margin": 213.47683715820312,
"fcm_dpo/q_t": 0.3984811007976532,
"grad_norm": 32.03657531738281,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.39184266328811646,
"logits/rejected": -0.3945959210395813,
"logps/chosen": -327.881591796875,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -576.446533203125,
"loss": 1.0528,
"margin_dpo/margin_mean": 213.47683715820312,
"margin_dpo/margin_std": 221.32455444335938,
"step": 283
},
{
"KL/chosen_KL_mean": -246.0831298828125,
"KL/mean": -346.190185546875,
"KL/rejected_KL_mean": -446.2972106933594,
"KL/std": 212.89483642578125,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.0019676508381962776,
"fcm_dpo/delta": -0.04589087888598442,
"fcm_dpo/margin": 200.21408081054688,
"fcm_dpo/q_t": 0.40764474868774414,
"grad_norm": 24.56162452697754,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.452556312084198,
"logits/rejected": -0.43920326232910156,
"logps/chosen": -304.5186767578125,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -539.7664794921875,
"loss": 1.0754,
"margin_dpo/margin_mean": 200.21408081054688,
"margin_dpo/margin_std": 222.98330688476562,
"step": 284
},
{
"KL/chosen_KL_mean": -289.398193359375,
"KL/mean": -374.96954345703125,
"KL/rejected_KL_mean": -460.54095458984375,
"KL/std": 184.91241455078125,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.0019783233292400837,
"fcm_dpo/delta": 0.011391036212444305,
"fcm_dpo/margin": 171.1427764892578,
"fcm_dpo/q_t": 0.41872942447662354,
"grad_norm": 19.697229385375977,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.41778579354286194,
"logits/rejected": -0.4037598967552185,
"logps/chosen": -355.63037109375,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -559.6677856445312,
"loss": 1.1174,
"margin_dpo/margin_mean": 171.14279174804688,
"margin_dpo/margin_std": 203.82138061523438,
"step": 285
},
{
"KL/chosen_KL_mean": -287.46539306640625,
"KL/mean": -386.77532958984375,
"KL/rejected_KL_mean": -486.08526611328125,
"KL/std": 219.8089141845703,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.001961304107680917,
"fcm_dpo/delta": -0.04113783687353134,
"fcm_dpo/margin": 198.61988830566406,
"fcm_dpo/q_t": 0.4081750810146332,
"grad_norm": 24.665103912353516,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.4218488931655884,
"logits/rejected": -0.40640929341316223,
"logps/chosen": -360.4164123535156,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -574.6737060546875,
"loss": 1.0874,
"margin_dpo/margin_mean": 198.619873046875,
"margin_dpo/margin_std": 240.56381225585938,
"step": 286
},
{
"KL/chosen_KL_mean": -281.83392333984375,
"KL/mean": -363.5769958496094,
"KL/rejected_KL_mean": -445.320068359375,
"KL/std": 199.60064697265625,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.001964614959433675,
"fcm_dpo/delta": 0.029743198305368423,
"fcm_dpo/margin": 163.4861297607422,
"fcm_dpo/q_t": 0.42457687854766846,
"grad_norm": 21.264787673950195,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.3639473021030426,
"logits/rejected": -0.33845168352127075,
"logps/chosen": -343.37506103515625,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -523.01611328125,
"loss": 1.1329,
"margin_dpo/margin_mean": 163.4861297607422,
"margin_dpo/margin_std": 215.17059326171875,
"step": 287
},
{
"KL/chosen_KL_mean": -272.1412048339844,
"KL/mean": -371.6726989746094,
"KL/rejected_KL_mean": -471.20416259765625,
"KL/std": 197.98980712890625,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.0019424078054726124,
"fcm_dpo/delta": -0.03923293575644493,
"fcm_dpo/margin": 199.06297302246094,
"fcm_dpo/q_t": 0.40794262290000916,
"grad_norm": 23.609954833984375,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.38256457448005676,
"logits/rejected": -0.3666524887084961,
"logps/chosen": -328.80242919921875,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -558.5399169921875,
"loss": 1.0781,
"margin_dpo/margin_mean": 199.06297302246094,
"margin_dpo/margin_std": 205.2132568359375,
"step": 288
},
{
"KL/chosen_KL_mean": -277.923828125,
"KL/mean": -390.819091796875,
"KL/rejected_KL_mean": -503.71429443359375,
"KL/std": 227.17816162109375,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.001931286184117198,
"fcm_dpo/delta": -0.08991145342588425,
"fcm_dpo/margin": 225.79046630859375,
"fcm_dpo/q_t": 0.3978983163833618,
"grad_norm": 25.743732452392578,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.3490581512451172,
"logits/rejected": -0.35004904866218567,
"logps/chosen": -323.15423583984375,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -591.35693359375,
"loss": 1.05,
"margin_dpo/margin_mean": 225.7904815673828,
"margin_dpo/margin_std": 243.15834045410156,
"step": 289
},
{
"KL/chosen_KL_mean": -311.5221252441406,
"KL/mean": -425.03460693359375,
"KL/rejected_KL_mean": -538.5470581054688,
"KL/std": 240.62857055664062,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.0018900145078077912,
"fcm_dpo/delta": -0.08279038965702057,
"fcm_dpo/margin": 227.02496337890625,
"fcm_dpo/q_t": 0.4012698531150818,
"grad_norm": 22.0297794342041,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.3336731791496277,
"logits/rejected": -0.3561541438102722,
"logps/chosen": -366.9936218261719,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -655.255615234375,
"loss": 1.0686,
"margin_dpo/margin_mean": 227.02493286132812,
"margin_dpo/margin_std": 284.237060546875,
"step": 290
},
{
"KL/chosen_KL_mean": -282.7470703125,
"KL/mean": -385.64276123046875,
"KL/rejected_KL_mean": -488.5384216308594,
"KL/std": 215.71090698242188,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.0018817877862602472,
"fcm_dpo/delta": -0.03895223140716553,
"fcm_dpo/margin": 205.79135131835938,
"fcm_dpo/q_t": 0.4090266227722168,
"grad_norm": 25.234416961669922,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.33878666162490845,
"logits/rejected": -0.3268297016620636,
"logps/chosen": -346.0074462890625,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -577.8355102539062,
"loss": 1.0938,
"margin_dpo/margin_mean": 205.79135131835938,
"margin_dpo/margin_std": 257.6557312011719,
"step": 291
},
{
"KL/chosen_KL_mean": -299.9615173339844,
"KL/mean": -412.6459045410156,
"KL/rejected_KL_mean": -525.330322265625,
"KL/std": 256.40252685546875,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0018554746638983488,
"fcm_dpo/delta": -0.07109744101762772,
"fcm_dpo/margin": 225.3687744140625,
"fcm_dpo/q_t": 0.40268850326538086,
"grad_norm": 20.208969116210938,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.3124640882015228,
"logits/rejected": -0.2998918890953064,
"logps/chosen": -353.88006591796875,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -615.2916870117188,
"loss": 1.0843,
"margin_dpo/margin_mean": 225.3687744140625,
"margin_dpo/margin_std": 301.27008056640625,
"step": 292
},
{
"KL/chosen_KL_mean": -296.6073303222656,
"KL/mean": -382.3493347167969,
"KL/rejected_KL_mean": -468.09130859375,
"KL/std": 245.44065856933594,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.0018549463711678982,
"fcm_dpo/delta": 0.032890163362026215,
"fcm_dpo/margin": 171.4840087890625,
"fcm_dpo/q_t": 0.4270939826965332,
"grad_norm": 38.751129150390625,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.35250991582870483,
"logits/rejected": -0.3470958471298218,
"logps/chosen": -356.98333740234375,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -545.9437255859375,
"loss": 1.1539,
"margin_dpo/margin_mean": 171.4840087890625,
"margin_dpo/margin_std": 275.24749755859375,
"step": 293
},
{
"KL/chosen_KL_mean": -263.4585266113281,
"KL/mean": -359.516845703125,
"KL/rejected_KL_mean": -455.5751953125,
"KL/std": 216.63278198242188,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.0018507996574044228,
"fcm_dpo/delta": -0.005858566612005234,
"fcm_dpo/margin": 192.11663818359375,
"fcm_dpo/q_t": 0.4179803133010864,
"grad_norm": 19.413515090942383,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.34789374470710754,
"logits/rejected": -0.3532540202140808,
"logps/chosen": -311.54608154296875,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -537.47216796875,
"loss": 1.1213,
"margin_dpo/margin_mean": 192.1166534423828,
"margin_dpo/margin_std": 271.4549560546875,
"step": 294
},
{
"KL/chosen_KL_mean": -313.67718505859375,
"KL/mean": -420.85565185546875,
"KL/rejected_KL_mean": -528.0341186523438,
"KL/std": 247.5594482421875,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.0018429165938869119,
"fcm_dpo/delta": -0.046867527067661285,
"fcm_dpo/margin": 214.35690307617188,
"fcm_dpo/q_t": 0.40905821323394775,
"grad_norm": 30.26310157775879,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.3324083089828491,
"logits/rejected": -0.33607107400894165,
"logps/chosen": -363.60186767578125,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -615.4904174804688,
"loss": 1.0985,
"margin_dpo/margin_mean": 214.35690307617188,
"margin_dpo/margin_std": 291.63604736328125,
"step": 295
},
{
"KL/chosen_KL_mean": -384.438232421875,
"KL/mean": -456.2586669921875,
"KL/rejected_KL_mean": -528.0791625976562,
"KL/std": 237.65313720703125,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.0018318177899345756,
"fcm_dpo/delta": -0.012629139237105846,
"fcm_dpo/margin": 143.6409454345703,
"fcm_dpo/q_t": 0.43860769271850586,
"grad_norm": 27.03304672241211,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.38435792922973633,
"logits/rejected": -0.368790864944458,
"logps/chosen": -449.929443359375,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -621.168212890625,
"loss": 1.2127,
"margin_dpo/margin_mean": 143.6409454345703,
"margin_dpo/margin_std": 295.96697998046875,
"step": 296
},
{
"KL/chosen_KL_mean": -326.37908935546875,
"KL/mean": -411.85205078125,
"KL/rejected_KL_mean": -497.32501220703125,
"KL/std": 220.0688934326172,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.0018395561492070556,
"fcm_dpo/delta": 0.03670303151011467,
"fcm_dpo/margin": 170.94590759277344,
"fcm_dpo/q_t": 0.425690233707428,
"grad_norm": 36.579345703125,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.3833608627319336,
"logits/rejected": -0.3880637586116791,
"logps/chosen": -382.8560485839844,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -592.4635620117188,
"loss": 1.1364,
"margin_dpo/margin_mean": 170.94590759277344,
"margin_dpo/margin_std": 221.54315185546875,
"step": 297
},
{
"KL/chosen_KL_mean": -345.55328369140625,
"KL/mean": -456.7684326171875,
"KL/rejected_KL_mean": -567.9835815429688,
"KL/std": 284.334228515625,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.0018349653109908104,
"fcm_dpo/delta": -0.060655344277620316,
"fcm_dpo/margin": 222.43023681640625,
"fcm_dpo/q_t": 0.40840664505958557,
"grad_norm": 25.856088638305664,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.3994889259338379,
"logits/rejected": -0.41677361726760864,
"logps/chosen": -412.87847900390625,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -684.645751953125,
"loss": 1.1025,
"margin_dpo/margin_mean": 222.43026733398438,
"margin_dpo/margin_std": 331.43804931640625,
"step": 298
},
{
"KL/chosen_KL_mean": -279.85418701171875,
"KL/mean": -374.20880126953125,
"KL/rejected_KL_mean": -468.5634460449219,
"KL/std": 216.16893005371094,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.00183395785279572,
"fcm_dpo/delta": 0.0034361593425273895,
"fcm_dpo/margin": 188.70925903320312,
"fcm_dpo/q_t": 0.41983699798583984,
"grad_norm": 25.584741592407227,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.40785565972328186,
"logits/rejected": -0.4132392108440399,
"logps/chosen": -328.8162841796875,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -552.8917236328125,
"loss": 1.1295,
"margin_dpo/margin_mean": 188.7092742919922,
"margin_dpo/margin_std": 268.2356872558594,
"step": 299
},
{
"KL/chosen_KL_mean": -348.40313720703125,
"KL/mean": -467.8739013671875,
"KL/rejected_KL_mean": -587.3447265625,
"KL/std": 258.7288818359375,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.0018063622992485762,
"fcm_dpo/delta": -0.08521745353937149,
"fcm_dpo/margin": 238.9415740966797,
"fcm_dpo/q_t": 0.40012985467910767,
"grad_norm": 30.562604904174805,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.3769553303718567,
"logits/rejected": -0.38268864154815674,
"logps/chosen": -407.4768371582031,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -683.3111572265625,
"loss": 1.0755,
"margin_dpo/margin_mean": 238.9415740966797,
"margin_dpo/margin_std": 310.2802734375,
"step": 300
},
{
"KL/chosen_KL_mean": -289.6461486816406,
"KL/mean": -401.08074951171875,
"KL/rejected_KL_mean": -512.5152587890625,
"KL/std": 234.33538818359375,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.0017770177219063044,
"fcm_dpo/delta": -0.048126157373189926,
"fcm_dpo/margin": 222.86912536621094,
"fcm_dpo/q_t": 0.4084852337837219,
"grad_norm": 24.22814178466797,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.3939129412174225,
"logits/rejected": -0.39491885900497437,
"logps/chosen": -346.8955078125,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -604.8688354492188,
"loss": 1.0851,
"margin_dpo/margin_mean": 222.86912536621094,
"margin_dpo/margin_std": 277.33050537109375,
"step": 301
},
{
"KL/chosen_KL_mean": -257.5218200683594,
"KL/mean": -349.50299072265625,
"KL/rejected_KL_mean": -441.48419189453125,
"KL/std": 211.3658447265625,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.001776274642907083,
"fcm_dpo/delta": 0.02384105697274208,
"fcm_dpo/margin": 183.96234130859375,
"fcm_dpo/q_t": 0.4237297475337982,
"grad_norm": 20.491802215576172,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.4090377688407898,
"logits/rejected": -0.4143017828464508,
"logps/chosen": -308.7198181152344,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -538.7105712890625,
"loss": 1.1344,
"margin_dpo/margin_mean": 183.96234130859375,
"margin_dpo/margin_std": 251.2886962890625,
"step": 302
},
{
"KL/chosen_KL_mean": -268.2716064453125,
"KL/mean": -354.656494140625,
"KL/rejected_KL_mean": -441.0413513183594,
"KL/std": 220.9415740966797,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.0017914820928126574,
"fcm_dpo/delta": 0.04175640642642975,
"fcm_dpo/margin": 172.76974487304688,
"fcm_dpo/q_t": 0.4276770353317261,
"grad_norm": 21.24039649963379,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.39839571714401245,
"logits/rejected": -0.3856062889099121,
"logps/chosen": -334.9855651855469,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -527.98681640625,
"loss": 1.1539,
"margin_dpo/margin_mean": 172.76976013183594,
"margin_dpo/margin_std": 260.4632568359375,
"step": 303
},
{
"KL/chosen_KL_mean": -234.28387451171875,
"KL/mean": -328.23004150390625,
"KL/rejected_KL_mean": -422.1761779785156,
"KL/std": 197.538330078125,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.0018079555593430996,
"fcm_dpo/delta": 0.010449519380927086,
"fcm_dpo/margin": 187.8922882080078,
"fcm_dpo/q_t": 0.41787901520729065,
"grad_norm": 34.292579650878906,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.45170116424560547,
"logits/rejected": -0.44620856642723083,
"logps/chosen": -306.2345886230469,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -512.648193359375,
"loss": 1.0992,
"margin_dpo/margin_mean": 187.89227294921875,
"margin_dpo/margin_std": 169.62152099609375,
"step": 304
},
{
"KL/chosen_KL_mean": -262.1391906738281,
"KL/mean": -348.18524169921875,
"KL/rejected_KL_mean": -434.2312927246094,
"KL/std": 229.55484008789062,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.0018056455301120877,
"fcm_dpo/delta": 0.040238022804260254,
"fcm_dpo/margin": 172.0921173095703,
"fcm_dpo/q_t": 0.4282124638557434,
"grad_norm": 27.539531707763672,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.4220506548881531,
"logits/rejected": -0.4064248204231262,
"logps/chosen": -328.9344177246094,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -526.9859008789062,
"loss": 1.152,
"margin_dpo/margin_mean": 172.09213256835938,
"margin_dpo/margin_std": 254.4590301513672,
"step": 305
},
{
"KL/chosen_KL_mean": -257.07513427734375,
"KL/mean": -341.1536560058594,
"KL/rejected_KL_mean": -425.232177734375,
"KL/std": 204.96353149414062,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.0018320318777114153,
"fcm_dpo/delta": 0.04318378493189812,
"fcm_dpo/margin": 168.15704345703125,
"fcm_dpo/q_t": 0.4273602068424225,
"grad_norm": 24.34828758239746,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.40770423412323,
"logits/rejected": -0.38708335161209106,
"logps/chosen": -326.759033203125,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -510.391357421875,
"loss": 1.1388,
"margin_dpo/margin_mean": 168.15704345703125,
"margin_dpo/margin_std": 215.92709350585938,
"step": 306
},
{
"KL/chosen_KL_mean": -245.8076934814453,
"KL/mean": -336.43817138671875,
"KL/rejected_KL_mean": -427.0686340332031,
"KL/std": 195.40953063964844,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.001836513401940465,
"fcm_dpo/delta": 0.017692940309643745,
"fcm_dpo/margin": 181.26095581054688,
"fcm_dpo/q_t": 0.42179954051971436,
"grad_norm": 24.07006072998047,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.3847889304161072,
"logits/rejected": -0.3667571544647217,
"logps/chosen": -315.9731140136719,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -514.0409545898438,
"loss": 1.1268,
"margin_dpo/margin_mean": 181.26095581054688,
"margin_dpo/margin_std": 240.51271057128906,
"step": 307
},
{
"KL/chosen_KL_mean": -257.4198303222656,
"KL/mean": -357.9421691894531,
"KL/rejected_KL_mean": -458.4644775390625,
"KL/std": 220.29737854003906,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.0018358014058321714,
"fcm_dpo/delta": -0.019804438576102257,
"fcm_dpo/margin": 201.04469299316406,
"fcm_dpo/q_t": 0.4117882549762726,
"grad_norm": 31.097227096557617,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.376716673374176,
"logits/rejected": -0.3686707019805908,
"logps/chosen": -312.6648254394531,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -537.8367919921875,
"loss": 1.0855,
"margin_dpo/margin_mean": 201.04469299316406,
"margin_dpo/margin_std": 205.7581024169922,
"step": 308
},
{
"KL/chosen_KL_mean": -267.55694580078125,
"KL/mean": -369.17938232421875,
"KL/rejected_KL_mean": -470.80181884765625,
"KL/std": 236.59249877929688,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.0018350511090829968,
"fcm_dpo/delta": -0.024183016270399094,
"fcm_dpo/margin": 203.24488830566406,
"fcm_dpo/q_t": 0.4112602770328522,
"grad_norm": 26.018009185791016,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.37056082487106323,
"logits/rejected": -0.36778467893600464,
"logps/chosen": -316.5160217285156,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -553.142578125,
"loss": 1.1,
"margin_dpo/margin_mean": 203.244873046875,
"margin_dpo/margin_std": 249.24404907226562,
"step": 309
},
{
"KL/chosen_KL_mean": -313.85491943359375,
"KL/mean": -406.3193359375,
"KL/rejected_KL_mean": -498.78375244140625,
"KL/std": 212.13653564453125,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.0018333324696868658,
"fcm_dpo/delta": 0.011089498177170753,
"fcm_dpo/margin": 184.9288787841797,
"fcm_dpo/q_t": 0.41921311616897583,
"grad_norm": 34.06706619262695,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.35188862681388855,
"logits/rejected": -0.3397538661956787,
"logps/chosen": -376.5966796875,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -578.7140502929688,
"loss": 1.1142,
"margin_dpo/margin_mean": 184.92886352539062,
"margin_dpo/margin_std": 212.01107788085938,
"step": 310
},
{
"KL/chosen_KL_mean": -326.53228759765625,
"KL/mean": -431.79541015625,
"KL/rejected_KL_mean": -537.0585327148438,
"KL/std": 269.36871337890625,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.0018190003465861082,
"fcm_dpo/delta": -0.034259166568517685,
"fcm_dpo/margin": 210.52621459960938,
"fcm_dpo/q_t": 0.4119686484336853,
"grad_norm": 24.23558807373047,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.3476032316684723,
"logits/rejected": -0.32333052158355713,
"logps/chosen": -379.5602722167969,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -614.4967041015625,
"loss": 1.1021,
"margin_dpo/margin_mean": 210.52622985839844,
"margin_dpo/margin_std": 284.9056091308594,
"step": 311
},
{
"KL/chosen_KL_mean": -306.4524230957031,
"KL/mean": -403.4335021972656,
"KL/rejected_KL_mean": -500.41461181640625,
"KL/std": 241.86181640625,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.0018220641650259495,
"fcm_dpo/delta": -0.0038466081023216248,
"fcm_dpo/margin": 193.96221923828125,
"fcm_dpo/q_t": 0.4196726381778717,
"grad_norm": 23.693248748779297,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.35380294919013977,
"logits/rejected": -0.3449653089046478,
"logps/chosen": -355.8446350097656,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -576.2073974609375,
"loss": 1.1271,
"margin_dpo/margin_mean": 193.96221923828125,
"margin_dpo/margin_std": 287.26824951171875,
"step": 312
},
{
"KL/chosen_KL_mean": -283.44970703125,
"KL/mean": -383.09356689453125,
"KL/rejected_KL_mean": -482.7374267578125,
"KL/std": 249.87867736816406,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.001822406891733408,
"fcm_dpo/delta": -0.014451660215854645,
"fcm_dpo/margin": 199.28773498535156,
"fcm_dpo/q_t": 0.4155638515949249,
"grad_norm": 23.18164825439453,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.4175724983215332,
"logits/rejected": -0.41589704155921936,
"logps/chosen": -333.60247802734375,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -569.1436767578125,
"loss": 1.1203,
"margin_dpo/margin_mean": 199.2877197265625,
"margin_dpo/margin_std": 278.8194580078125,
"step": 313
},
{
"KL/chosen_KL_mean": -298.1259765625,
"KL/mean": -390.4653015136719,
"KL/rejected_KL_mean": -482.80462646484375,
"KL/std": 232.75111389160156,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.0018075141124427319,
"fcm_dpo/delta": 0.016570214182138443,
"fcm_dpo/margin": 184.67868041992188,
"fcm_dpo/q_t": 0.42391207814216614,
"grad_norm": 20.25391960144043,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.40884333848953247,
"logits/rejected": -0.4110562801361084,
"logps/chosen": -355.3635559082031,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -580.4011840820312,
"loss": 1.1494,
"margin_dpo/margin_mean": 184.67868041992188,
"margin_dpo/margin_std": 298.83831787109375,
"step": 314
},
{
"KL/chosen_KL_mean": -263.869873046875,
"KL/mean": -349.53668212890625,
"KL/rejected_KL_mean": -435.2034606933594,
"KL/std": 211.77670288085938,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.001823345897719264,
"fcm_dpo/delta": 0.03882795572280884,
"fcm_dpo/margin": 171.33358764648438,
"fcm_dpo/q_t": 0.42688193917274475,
"grad_norm": 19.43107795715332,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.3321152329444885,
"logits/rejected": -0.33484265208244324,
"logps/chosen": -312.939453125,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -520.8843383789062,
"loss": 1.142,
"margin_dpo/margin_mean": 171.33358764648438,
"margin_dpo/margin_std": 235.25616455078125,
"step": 315
},
{
"KL/chosen_KL_mean": -255.12551879882812,
"KL/mean": -380.3026123046875,
"KL/rejected_KL_mean": -505.4797058105469,
"KL/std": 246.68731689453125,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.0017926250584423542,
"fcm_dpo/delta": -0.10438720881938934,
"fcm_dpo/margin": 250.35418701171875,
"fcm_dpo/q_t": 0.39585646986961365,
"grad_norm": 27.900249481201172,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.37772274017333984,
"logits/rejected": -0.38135725259780884,
"logps/chosen": -309.3862609863281,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -606.7611694335938,
"loss": 1.043,
"margin_dpo/margin_mean": 250.3542022705078,
"margin_dpo/margin_std": 271.3267517089844,
"step": 316
},
{
"KL/chosen_KL_mean": -264.39501953125,
"KL/mean": -394.56658935546875,
"KL/rejected_KL_mean": -524.7380981445312,
"KL/std": 222.24476623535156,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.0017680514138191938,
"fcm_dpo/delta": -0.11548471450805664,
"fcm_dpo/margin": 260.3431091308594,
"fcm_dpo/q_t": 0.39133790135383606,
"grad_norm": 27.655803680419922,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.40937548875808716,
"logits/rejected": -0.4001927971839905,
"logps/chosen": -320.4892578125,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -625.4371337890625,
"loss": 1.0271,
"margin_dpo/margin_mean": 260.3431091308594,
"margin_dpo/margin_std": 250.1036376953125,
"step": 317
},
{
"KL/chosen_KL_mean": -297.4501953125,
"KL/mean": -396.00433349609375,
"KL/rejected_KL_mean": -494.5584716796875,
"KL/std": 234.63751220703125,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.0017554573714733124,
"fcm_dpo/delta": 0.004025392234325409,
"fcm_dpo/margin": 197.1082763671875,
"fcm_dpo/q_t": 0.4192516803741455,
"grad_norm": 23.755502700805664,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.47140195965766907,
"logits/rejected": -0.44852572679519653,
"logps/chosen": -362.09588623046875,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -577.32275390625,
"loss": 1.1185,
"margin_dpo/margin_mean": 197.1082763671875,
"margin_dpo/margin_std": 258.5344543457031,
"step": 318
},
{
"KL/chosen_KL_mean": -272.87457275390625,
"KL/mean": -392.9670104980469,
"KL/rejected_KL_mean": -513.0595092773438,
"KL/std": 259.213134765625,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0017336711753159761,
"fcm_dpo/delta": -0.06930382549762726,
"fcm_dpo/margin": 240.18490600585938,
"fcm_dpo/q_t": 0.4022204279899597,
"grad_norm": 23.22355842590332,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.3927324414253235,
"logits/rejected": -0.4057486057281494,
"logps/chosen": -322.25830078125,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -626.9660034179688,
"loss": 1.0671,
"margin_dpo/margin_mean": 240.18490600585938,
"margin_dpo/margin_std": 278.5548095703125,
"step": 319
},
{
"KL/chosen_KL_mean": -305.7335205078125,
"KL/mean": -437.4515380859375,
"KL/rejected_KL_mean": -569.1694946289062,
"KL/std": 278.39447021484375,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.0016967777628451586,
"fcm_dpo/delta": -0.10202349722385406,
"fcm_dpo/margin": 263.43597412109375,
"fcm_dpo/q_t": 0.39611712098121643,
"grad_norm": 24.74642562866211,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.4070661664009094,
"logits/rejected": -0.41442254185676575,
"logps/chosen": -365.2384033203125,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -666.836669921875,
"loss": 1.0469,
"margin_dpo/margin_mean": 263.43597412109375,
"margin_dpo/margin_std": 295.5460205078125,
"step": 320
},
{
"KL/chosen_KL_mean": -385.0860900878906,
"KL/mean": -497.9861145019531,
"KL/rejected_KL_mean": -610.8861694335938,
"KL/std": 285.7553405761719,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.0016765834297984838,
"fcm_dpo/delta": -0.03041578084230423,
"fcm_dpo/margin": 225.800048828125,
"fcm_dpo/q_t": 0.41292014718055725,
"grad_norm": 27.752023696899414,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.37062233686447144,
"logits/rejected": -0.35946518182754517,
"logps/chosen": -446.634765625,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -702.5272216796875,
"loss": 1.108,
"margin_dpo/margin_mean": 225.80007934570312,
"margin_dpo/margin_std": 308.14105224609375,
"step": 321
},
{
"KL/chosen_KL_mean": -346.7474365234375,
"KL/mean": -466.8512268066406,
"KL/rejected_KL_mean": -586.9550170898438,
"KL/std": 259.3096923828125,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.001664046081714332,
"fcm_dpo/delta": -0.0522020198404789,
"fcm_dpo/margin": 240.2075958251953,
"fcm_dpo/q_t": 0.4059278964996338,
"grad_norm": 28.33850860595703,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.38744407892227173,
"logits/rejected": -0.38698631525039673,
"logps/chosen": -404.0374755859375,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -682.7049560546875,
"loss": 1.0774,
"margin_dpo/margin_mean": 240.20761108398438,
"margin_dpo/margin_std": 275.8708801269531,
"step": 322
},
{
"KL/chosen_KL_mean": -360.1468811035156,
"KL/mean": -461.87078857421875,
"KL/rejected_KL_mean": -563.5946655273438,
"KL/std": 256.2729797363281,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.0016664534341543913,
"fcm_dpo/delta": 0.011339662596583366,
"fcm_dpo/margin": 203.44778442382812,
"fcm_dpo/q_t": 0.4201313257217407,
"grad_norm": 36.73489761352539,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.3776964545249939,
"logits/rejected": -0.37534666061401367,
"logps/chosen": -411.380859375,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -638.6566162109375,
"loss": 1.1293,
"margin_dpo/margin_mean": 203.44778442382812,
"margin_dpo/margin_std": 275.85626220703125,
"step": 323
},
{
"KL/chosen_KL_mean": -409.14288330078125,
"KL/mean": -503.98602294921875,
"KL/rejected_KL_mean": -598.8291625976562,
"KL/std": 269.3505859375,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.0016613883199170232,
"fcm_dpo/delta": -0.046242307871580124,
"fcm_dpo/margin": 189.686279296875,
"fcm_dpo/q_t": 0.42781496047973633,
"grad_norm": 57.406150817871094,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.40174317359924316,
"logits/rejected": -0.385235071182251,
"logps/chosen": -474.2780456542969,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -685.306640625,
"loss": 1.1672,
"margin_dpo/margin_mean": 189.68629455566406,
"margin_dpo/margin_std": 315.1155700683594,
"step": 324
},
{
"KL/chosen_KL_mean": -305.6237487792969,
"KL/mean": -388.1728515625,
"KL/rejected_KL_mean": -470.721923828125,
"KL/std": 234.14627075195312,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.0016727466136217117,
"fcm_dpo/delta": 0.07592117786407471,
"fcm_dpo/margin": 165.09820556640625,
"fcm_dpo/q_t": 0.43507638573646545,
"grad_norm": 27.068408966064453,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.42593660950660706,
"logits/rejected": -0.40495866537094116,
"logps/chosen": -361.83935546875,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -540.807861328125,
"loss": 1.1704,
"margin_dpo/margin_mean": 165.09820556640625,
"margin_dpo/margin_std": 239.03897094726562,
"step": 325
},
{
"KL/chosen_KL_mean": -340.3418273925781,
"KL/mean": -420.40203857421875,
"KL/rejected_KL_mean": -500.4622497558594,
"KL/std": 218.0757293701172,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0016966955736279488,
"fcm_dpo/delta": 0.0805881917476654,
"fcm_dpo/margin": 160.12042236328125,
"fcm_dpo/q_t": 0.43505293130874634,
"grad_norm": 36.45564651489258,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.43507713079452515,
"logits/rejected": -0.40912318229675293,
"logps/chosen": -413.0667724609375,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -580.3090209960938,
"loss": 1.1649,
"margin_dpo/margin_mean": 160.12042236328125,
"margin_dpo/margin_std": 207.34747314453125,
"step": 326
},
{
"KL/chosen_KL_mean": -287.189208984375,
"KL/mean": -408.7220153808594,
"KL/rejected_KL_mean": -530.2548217773438,
"KL/std": 226.6089324951172,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.001686271745711565,
"fcm_dpo/delta": -0.06271137297153473,
"fcm_dpo/margin": 243.0655517578125,
"fcm_dpo/q_t": 0.40232932567596436,
"grad_norm": 45.123172760009766,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.4474729597568512,
"logits/rejected": -0.4362829327583313,
"logps/chosen": -356.32366943359375,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -642.1885986328125,
"loss": 1.0628,
"margin_dpo/margin_mean": 243.0655517578125,
"margin_dpo/margin_std": 256.05047607421875,
"step": 327
},
{
"KL/chosen_KL_mean": -290.7843933105469,
"KL/mean": -401.93585205078125,
"KL/rejected_KL_mean": -513.0873413085938,
"KL/std": 251.8201446533203,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0016804768238216639,
"fcm_dpo/delta": -0.02448493242263794,
"fcm_dpo/margin": 222.30294799804688,
"fcm_dpo/q_t": 0.4131455719470978,
"grad_norm": 26.31197166442871,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.437290757894516,
"logits/rejected": -0.4402696490287781,
"logps/chosen": -350.4715881347656,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -603.9423828125,
"loss": 1.0994,
"margin_dpo/margin_mean": 222.30294799804688,
"margin_dpo/margin_std": 282.1927490234375,
"step": 328
},
{
"KL/chosen_KL_mean": -320.8580017089844,
"KL/mean": -433.61395263671875,
"KL/rejected_KL_mean": -546.369873046875,
"KL/std": 260.9647216796875,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0016629232559353113,
"fcm_dpo/delta": -0.026486670598387718,
"fcm_dpo/margin": 225.51190185546875,
"fcm_dpo/q_t": 0.41196945309638977,
"grad_norm": 24.17496109008789,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.42437106370925903,
"logits/rejected": -0.41386279463768005,
"logps/chosen": -386.10418701171875,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -647.067626953125,
"loss": 1.1029,
"margin_dpo/margin_mean": 225.51190185546875,
"margin_dpo/margin_std": 284.83648681640625,
"step": 329
},
{
"KL/chosen_KL_mean": -273.79058837890625,
"KL/mean": -391.7522888183594,
"KL/rejected_KL_mean": -509.7140197753906,
"KL/std": 246.3189697265625,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.001657246146351099,
"fcm_dpo/delta": -0.04265393316745758,
"fcm_dpo/margin": 235.92340087890625,
"fcm_dpo/q_t": 0.4079107344150543,
"grad_norm": 26.65233039855957,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.38325101137161255,
"logits/rejected": -0.384086012840271,
"logps/chosen": -320.7889404296875,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -596.5908203125,
"loss": 1.0803,
"margin_dpo/margin_mean": 235.92337036132812,
"margin_dpo/margin_std": 269.93707275390625,
"step": 330
},
{
"KL/chosen_KL_mean": -305.1080017089844,
"KL/mean": -425.6927185058594,
"KL/rejected_KL_mean": -546.2774658203125,
"KL/std": 213.3463134765625,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.001642332412302494,
"fcm_dpo/delta": -0.04797166585922241,
"fcm_dpo/margin": 241.16941833496094,
"fcm_dpo/q_t": 0.4048876166343689,
"grad_norm": 22.422067642211914,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.4126804769039154,
"logits/rejected": -0.40818193554878235,
"logps/chosen": -355.63226318359375,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -635.2928466796875,
"loss": 1.0563,
"margin_dpo/margin_mean": 241.16941833496094,
"margin_dpo/margin_std": 207.7781982421875,
"step": 331
},
{
"KL/chosen_KL_mean": -311.51898193359375,
"KL/mean": -397.1397705078125,
"KL/rejected_KL_mean": -482.76055908203125,
"KL/std": 219.59471130371094,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0016566382255405188,
"fcm_dpo/delta": 0.06814493983983994,
"fcm_dpo/margin": 171.24156188964844,
"fcm_dpo/q_t": 0.4337378740310669,
"grad_norm": 18.958757400512695,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.42918598651885986,
"logits/rejected": -0.40887415409088135,
"logps/chosen": -360.69927978515625,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -559.2457275390625,
"loss": 1.1626,
"margin_dpo/margin_mean": 171.24156188964844,
"margin_dpo/margin_std": 248.4321746826172,
"step": 332
},
{
"KL/chosen_KL_mean": -342.23876953125,
"KL/mean": -454.83258056640625,
"KL/rejected_KL_mean": -567.4263916015625,
"KL/std": 263.9354248046875,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.001645524287596345,
"fcm_dpo/delta": -0.022140901535749435,
"fcm_dpo/margin": 225.18756103515625,
"fcm_dpo/q_t": 0.4143007695674896,
"grad_norm": 23.2531795501709,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.43771564960479736,
"logits/rejected": -0.43037185072898865,
"logps/chosen": -405.9945068359375,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -662.470458984375,
"loss": 1.1188,
"margin_dpo/margin_mean": 225.1875762939453,
"margin_dpo/margin_std": 320.94561767578125,
"step": 333
},
{
"KL/chosen_KL_mean": -339.3734130859375,
"KL/mean": -440.17437744140625,
"KL/rejected_KL_mean": -540.975341796875,
"KL/std": 291.3700866699219,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.001655098283663392,
"fcm_dpo/delta": 0.01687953621149063,
"fcm_dpo/margin": 201.60191345214844,
"fcm_dpo/q_t": 0.4232845902442932,
"grad_norm": 21.963340759277344,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.4199391007423401,
"logits/rejected": -0.41611427068710327,
"logps/chosen": -406.3531494140625,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -636.292236328125,
"loss": 1.1508,
"margin_dpo/margin_mean": 201.6019287109375,
"margin_dpo/margin_std": 326.2679443359375,
"step": 334
},
{
"KL/chosen_KL_mean": -366.4427185058594,
"KL/mean": -445.76446533203125,
"KL/rejected_KL_mean": -525.086181640625,
"KL/std": 274.3227844238281,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.0016588786384090781,
"fcm_dpo/delta": 0.001063960837200284,
"fcm_dpo/margin": 158.6435089111328,
"fcm_dpo/q_t": 0.4391080439090729,
"grad_norm": 30.97983741760254,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.4851636588573456,
"logits/rejected": -0.4646086096763611,
"logps/chosen": -428.9851989746094,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -612.7039184570312,
"loss": 1.1978,
"margin_dpo/margin_mean": 158.64349365234375,
"margin_dpo/margin_std": 290.5368957519531,
"step": 335
},
{
"KL/chosen_KL_mean": -371.8771667480469,
"KL/mean": -501.9913330078125,
"KL/rejected_KL_mean": -632.10546875,
"KL/std": 316.42431640625,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0016450767870992422,
"fcm_dpo/delta": -0.08156316727399826,
"fcm_dpo/margin": 260.2283020019531,
"fcm_dpo/q_t": 0.4031341075897217,
"grad_norm": 30.599864959716797,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.4376946687698364,
"logits/rejected": -0.4382895231246948,
"logps/chosen": -426.4083251953125,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -736.5097045898438,
"loss": 1.0942,
"margin_dpo/margin_mean": 260.2283020019531,
"margin_dpo/margin_std": 375.32147216796875,
"step": 336
},
{
"KL/chosen_KL_mean": -335.18377685546875,
"KL/mean": -462.601806640625,
"KL/rejected_KL_mean": -590.0198364257812,
"KL/std": 248.90377807617188,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.0016133894678205252,
"fcm_dpo/delta": -0.06399152427911758,
"fcm_dpo/margin": 254.83612060546875,
"fcm_dpo/q_t": 0.40399160981178284,
"grad_norm": 21.82675552368164,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.46123170852661133,
"logits/rejected": -0.44864845275878906,
"logps/chosen": -400.31243896484375,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -691.746826171875,
"loss": 1.07,
"margin_dpo/margin_mean": 254.83612060546875,
"margin_dpo/margin_std": 295.6842041015625,
"step": 337
},
{
"KL/chosen_KL_mean": -298.3284606933594,
"KL/mean": -423.231201171875,
"KL/rejected_KL_mean": -548.1339111328125,
"KL/std": 241.73153686523438,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0015977486036717892,
"fcm_dpo/delta": -0.05123209208250046,
"fcm_dpo/margin": 249.80540466308594,
"fcm_dpo/q_t": 0.4050595760345459,
"grad_norm": 26.014801025390625,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.4533551335334778,
"logits/rejected": -0.44745928049087524,
"logps/chosen": -356.75115966796875,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -637.202392578125,
"loss": 1.0652,
"margin_dpo/margin_mean": 249.805419921875,
"margin_dpo/margin_std": 252.97518920898438,
"step": 338
},
{
"KL/chosen_KL_mean": -320.6482849121094,
"KL/mean": -444.5311584472656,
"KL/rejected_KL_mean": -568.4140625,
"KL/std": 257.1028137207031,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0015966382343322039,
"fcm_dpo/delta": -0.04843657463788986,
"fcm_dpo/margin": 247.76576232910156,
"fcm_dpo/q_t": 0.4064652621746063,
"grad_norm": 23.043846130371094,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.5105829238891602,
"logits/rejected": -0.49902158975601196,
"logps/chosen": -380.64361572265625,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -672.324951171875,
"loss": 1.0773,
"margin_dpo/margin_mean": 247.7657470703125,
"margin_dpo/margin_std": 259.73455810546875,
"step": 339
},
{
"KL/chosen_KL_mean": -344.79083251953125,
"KL/mean": -446.81793212890625,
"KL/rejected_KL_mean": -548.844970703125,
"KL/std": 248.848876953125,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0015804399736225605,
"fcm_dpo/delta": 0.02824411541223526,
"fcm_dpo/margin": 204.05410766601562,
"fcm_dpo/q_t": 0.4249947667121887,
"grad_norm": 24.647438049316406,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.4062250852584839,
"logits/rejected": -0.38350969552993774,
"logps/chosen": -397.62109375,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -621.9522094726562,
"loss": 1.1452,
"margin_dpo/margin_mean": 204.05410766601562,
"margin_dpo/margin_std": 303.45513916015625,
"step": 340
},
{
"KL/chosen_KL_mean": -344.22686767578125,
"KL/mean": -463.89288330078125,
"KL/rejected_KL_mean": -583.558837890625,
"KL/std": 270.5447082519531,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.001576824113726616,
"fcm_dpo/delta": -0.028752204030752182,
"fcm_dpo/margin": 239.3319549560547,
"fcm_dpo/q_t": 0.41167351603507996,
"grad_norm": 22.612895965576172,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.4313828945159912,
"logits/rejected": -0.42949026823043823,
"logps/chosen": -392.126708984375,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -685.3687133789062,
"loss": 1.0948,
"margin_dpo/margin_mean": 239.33193969726562,
"margin_dpo/margin_std": 287.29522705078125,
"step": 341
},
{
"KL/chosen_KL_mean": -340.2489318847656,
"KL/mean": -441.2088317871094,
"KL/rejected_KL_mean": -542.168701171875,
"KL/std": 256.6587829589844,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.0015923329629004002,
"fcm_dpo/delta": 0.029024792835116386,
"fcm_dpo/margin": 201.91983032226562,
"fcm_dpo/q_t": 0.4247163236141205,
"grad_norm": 23.06468391418457,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.46542230248451233,
"logits/rejected": -0.43691468238830566,
"logps/chosen": -412.2455749511719,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -634.75830078125,
"loss": 1.1482,
"margin_dpo/margin_mean": 201.91983032226562,
"margin_dpo/margin_std": 308.49237060546875,
"step": 342
},
{
"KL/chosen_KL_mean": -311.03472900390625,
"KL/mean": -440.37176513671875,
"KL/rejected_KL_mean": -569.708740234375,
"KL/std": 256.6684875488281,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.0015744275879114866,
"fcm_dpo/delta": -0.05970253422856331,
"fcm_dpo/margin": 258.6740417480469,
"fcm_dpo/q_t": 0.40231946110725403,
"grad_norm": 19.85938262939453,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.4849863052368164,
"logits/rejected": -0.4904642403125763,
"logps/chosen": -365.44036865234375,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -680.7501220703125,
"loss": 1.0525,
"margin_dpo/margin_mean": 258.67401123046875,
"margin_dpo/margin_std": 228.75125122070312,
"step": 343
},
{
"KL/chosen_KL_mean": -311.48944091796875,
"KL/mean": -426.0353698730469,
"KL/rejected_KL_mean": -540.581298828125,
"KL/std": 284.60888671875,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.0015560141764581203,
"fcm_dpo/delta": -0.0887831598520279,
"fcm_dpo/margin": 229.09188842773438,
"fcm_dpo/q_t": 0.41803035140037537,
"grad_norm": 29.873308181762695,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.47299444675445557,
"logits/rejected": -0.4747607707977295,
"logps/chosen": -365.4541015625,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -631.2046508789062,
"loss": 1.1164,
"margin_dpo/margin_mean": 229.09188842773438,
"margin_dpo/margin_std": 292.5248718261719,
"step": 344
},
{
"KL/chosen_KL_mean": -370.3597106933594,
"KL/mean": -495.03125,
"KL/rejected_KL_mean": -619.7027587890625,
"KL/std": 266.9612731933594,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.0015359357930719852,
"fcm_dpo/delta": -0.03427436202764511,
"fcm_dpo/margin": 249.343017578125,
"fcm_dpo/q_t": 0.4097304940223694,
"grad_norm": 23.3608455657959,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.4813809394836426,
"logits/rejected": -0.4792475700378418,
"logps/chosen": -432.04541015625,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -719.193115234375,
"loss": 1.0872,
"margin_dpo/margin_mean": 249.34304809570312,
"margin_dpo/margin_std": 292.41497802734375,
"step": 345
},
{
"KL/chosen_KL_mean": -363.1270751953125,
"KL/mean": -480.3489685058594,
"KL/rejected_KL_mean": -597.5708618164062,
"KL/std": 265.65924072265625,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.0015292272437363863,
"fcm_dpo/delta": -0.008838503621518612,
"fcm_dpo/margin": 234.44375610351562,
"fcm_dpo/q_t": 0.4146372973918915,
"grad_norm": 20.73213005065918,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.44797492027282715,
"logits/rejected": -0.4381539821624756,
"logps/chosen": -418.38336181640625,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -674.9862060546875,
"loss": 1.1038,
"margin_dpo/margin_mean": 234.4437713623047,
"margin_dpo/margin_std": 281.268310546875,
"step": 346
},
{
"KL/chosen_KL_mean": -360.153564453125,
"KL/mean": -475.7332763671875,
"KL/rejected_KL_mean": -591.31298828125,
"KL/std": 275.2700500488281,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.0015283192042261362,
"fcm_dpo/delta": -0.0034055609721690416,
"fcm_dpo/margin": 231.15940856933594,
"fcm_dpo/q_t": 0.417599081993103,
"grad_norm": 22.630468368530273,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.4328559637069702,
"logits/rejected": -0.42671138048171997,
"logps/chosen": -417.71978759765625,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -683.6680908203125,
"loss": 1.1179,
"margin_dpo/margin_mean": 231.15939331054688,
"margin_dpo/margin_std": 312.8233642578125,
"step": 347
},
{
"KL/chosen_KL_mean": -321.64593505859375,
"KL/mean": -419.4115905761719,
"KL/rejected_KL_mean": -517.1773071289062,
"KL/std": 238.86471557617188,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.0015206331154331565,
"fcm_dpo/delta": -0.02389657124876976,
"fcm_dpo/margin": 195.53135681152344,
"fcm_dpo/q_t": 0.4300573468208313,
"grad_norm": 22.362464904785156,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.4757729768753052,
"logits/rejected": -0.47732916474342346,
"logps/chosen": -377.963623046875,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -606.315673828125,
"loss": 1.1552,
"margin_dpo/margin_mean": 195.53135681152344,
"margin_dpo/margin_std": 277.104248046875,
"step": 348
},
{
"KL/chosen_KL_mean": -298.3360595703125,
"KL/mean": -417.1045227050781,
"KL/rejected_KL_mean": -535.8729858398438,
"KL/std": 269.330322265625,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.0015224895905703306,
"fcm_dpo/delta": -0.01223532110452652,
"fcm_dpo/margin": 237.53692626953125,
"fcm_dpo/q_t": 0.4166039228439331,
"grad_norm": 18.555866241455078,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.4357994794845581,
"logits/rejected": -0.43820104002952576,
"logps/chosen": -356.361572265625,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -633.378173828125,
"loss": 1.1159,
"margin_dpo/margin_mean": 237.5369415283203,
"margin_dpo/margin_std": 327.4973449707031,
"step": 349
},
{
"KL/chosen_KL_mean": -326.14739990234375,
"KL/mean": -436.9446716308594,
"KL/rejected_KL_mean": -547.741943359375,
"KL/std": 256.19500732421875,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.001514824340119958,
"fcm_dpo/delta": 0.014613550156354904,
"fcm_dpo/margin": 221.5946044921875,
"fcm_dpo/q_t": 0.4211843013763428,
"grad_norm": 25.532001495361328,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.4588230848312378,
"logits/rejected": -0.4639089107513428,
"logps/chosen": -390.4779052734375,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -637.6136474609375,
"loss": 1.1337,
"margin_dpo/margin_mean": 221.5946044921875,
"margin_dpo/margin_std": 314.7109680175781,
"step": 350
},
{
"KL/chosen_KL_mean": -311.5458679199219,
"KL/mean": -448.833984375,
"KL/rejected_KL_mean": -586.1221313476562,
"KL/std": 308.70892333984375,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0015022414736449718,
"fcm_dpo/delta": -0.0655267983675003,
"fcm_dpo/margin": 274.5762634277344,
"fcm_dpo/q_t": 0.4057248532772064,
"grad_norm": 26.511186599731445,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.47476595640182495,
"logits/rejected": -0.4755515456199646,
"logps/chosen": -372.218017578125,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -687.6875610351562,
"loss": 1.0834,
"margin_dpo/margin_mean": 274.57623291015625,
"margin_dpo/margin_std": 358.6259765625,
"step": 351
},
{
"KL/chosen_KL_mean": -366.9032287597656,
"KL/mean": -447.29217529296875,
"KL/rejected_KL_mean": -527.68115234375,
"KL/std": 277.0421142578125,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.001509011024609208,
"fcm_dpo/delta": 0.024786878377199173,
"fcm_dpo/margin": 160.77786254882812,
"fcm_dpo/q_t": 0.44290652871131897,
"grad_norm": 30.402721405029297,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.4430214464664459,
"logits/rejected": -0.41058680415153503,
"logps/chosen": -437.8466796875,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -604.3231201171875,
"loss": 1.2099,
"margin_dpo/margin_mean": 160.77786254882812,
"margin_dpo/margin_std": 302.5135498046875,
"step": 352
},
{
"KL/chosen_KL_mean": -346.7418212890625,
"KL/mean": -460.20111083984375,
"KL/rejected_KL_mean": -573.660400390625,
"KL/std": 280.99822998046875,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0015030490467324853,
"fcm_dpo/delta": 0.008724931627511978,
"fcm_dpo/margin": 226.9186248779297,
"fcm_dpo/q_t": 0.42118746042251587,
"grad_norm": 23.444734573364258,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.4665010869503021,
"logits/rejected": -0.4548315405845642,
"logps/chosen": -405.1371765136719,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -653.9959716796875,
"loss": 1.1327,
"margin_dpo/margin_mean": 226.9186248779297,
"margin_dpo/margin_std": 323.4075622558594,
"step": 353
},
{
"KL/chosen_KL_mean": -296.152099609375,
"KL/mean": -417.95367431640625,
"KL/rejected_KL_mean": -539.7552490234375,
"KL/std": 264.796630859375,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.0015054075047373772,
"fcm_dpo/delta": -0.01743352971971035,
"fcm_dpo/margin": 243.60313415527344,
"fcm_dpo/q_t": 0.4130624532699585,
"grad_norm": 27.117420196533203,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.44472265243530273,
"logits/rejected": -0.4349956214427948,
"logps/chosen": -355.9551086425781,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -628.5127563476562,
"loss": 1.0915,
"margin_dpo/margin_mean": 243.60311889648438,
"margin_dpo/margin_std": 267.82965087890625,
"step": 354
},
{
"KL/chosen_KL_mean": -299.48431396484375,
"KL/mean": -417.59002685546875,
"KL/rejected_KL_mean": -535.6956787109375,
"KL/std": 261.95001220703125,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.0015026122564449906,
"fcm_dpo/delta": -0.005186091177165508,
"fcm_dpo/margin": 236.21141052246094,
"fcm_dpo/q_t": 0.4154089689254761,
"grad_norm": 33.38981628417969,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.5034153461456299,
"logits/rejected": -0.4952772855758667,
"logps/chosen": -353.61279296875,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -618.101806640625,
"loss": 1.1006,
"margin_dpo/margin_mean": 236.21142578125,
"margin_dpo/margin_std": 263.2491760253906,
"step": 355
},
{
"KL/chosen_KL_mean": -364.2958984375,
"KL/mean": -427.6046142578125,
"KL/rejected_KL_mean": -490.913330078125,
"KL/std": 271.66748046875,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.0015103459591045976,
"fcm_dpo/delta": 0.03656543791294098,
"fcm_dpo/margin": 126.61747741699219,
"fcm_dpo/q_t": 0.45585763454437256,
"grad_norm": 26.874828338623047,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.4800550043582916,
"logits/rejected": -0.4602137804031372,
"logps/chosen": -428.9696960449219,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -566.8126220703125,
"loss": 1.2663,
"margin_dpo/margin_mean": 126.61747741699219,
"margin_dpo/margin_std": 333.15924072265625,
"step": 356
},
{
"KL/chosen_KL_mean": -329.130859375,
"KL/mean": -432.2911376953125,
"KL/rejected_KL_mean": -535.451416015625,
"KL/std": 263.6339111328125,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.0015188403194770217,
"fcm_dpo/delta": 0.03768056631088257,
"fcm_dpo/margin": 206.32057189941406,
"fcm_dpo/q_t": 0.42600899934768677,
"grad_norm": 33.71342468261719,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.4496878385543823,
"logits/rejected": -0.4382253885269165,
"logps/chosen": -381.8566589355469,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -622.2926025390625,
"loss": 1.1443,
"margin_dpo/margin_mean": 206.32058715820312,
"margin_dpo/margin_std": 286.84808349609375,
"step": 357
},
{
"KL/chosen_KL_mean": -299.54290771484375,
"KL/mean": -400.99774169921875,
"KL/rejected_KL_mean": -502.4526062011719,
"KL/std": 268.91558837890625,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0015357965603470802,
"fcm_dpo/delta": 0.03961861506104469,
"fcm_dpo/margin": 202.90966796875,
"fcm_dpo/q_t": 0.42794185876846313,
"grad_norm": 21.35138702392578,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.5003876686096191,
"logits/rejected": -0.484429270029068,
"logps/chosen": -362.74835205078125,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -590.825927734375,
"loss": 1.1493,
"margin_dpo/margin_mean": 202.90966796875,
"margin_dpo/margin_std": 300.5551452636719,
"step": 358
},
{
"KL/chosen_KL_mean": -351.01727294921875,
"KL/mean": -474.3118591308594,
"KL/rejected_KL_mean": -597.6065673828125,
"KL/std": 270.4597473144531,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0015308901201933622,
"fcm_dpo/delta": -0.028654001653194427,
"fcm_dpo/margin": 246.58924865722656,
"fcm_dpo/q_t": 0.41267070174217224,
"grad_norm": 27.014921188354492,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.39821434020996094,
"logits/rejected": -0.38216477632522583,
"logps/chosen": -407.387451171875,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -679.7802734375,
"loss": 1.1054,
"margin_dpo/margin_mean": 246.58926391601562,
"margin_dpo/margin_std": 330.0404052734375,
"step": 359
},
{
"KL/chosen_KL_mean": -346.491943359375,
"KL/mean": -435.66802978515625,
"KL/rejected_KL_mean": -524.8441162109375,
"KL/std": 230.1074676513672,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.0015494239050894976,
"fcm_dpo/delta": 0.07566958665847778,
"fcm_dpo/margin": 178.35223388671875,
"fcm_dpo/q_t": 0.4340076446533203,
"grad_norm": 44.442474365234375,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.4111713767051697,
"logits/rejected": -0.39715811610221863,
"logps/chosen": -397.95233154296875,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -594.6830444335938,
"loss": 1.1574,
"margin_dpo/margin_mean": 178.3522186279297,
"margin_dpo/margin_std": 219.6265869140625,
"step": 360
},
{
"KL/chosen_KL_mean": -368.93402099609375,
"KL/mean": -457.94183349609375,
"KL/rejected_KL_mean": -546.9495849609375,
"KL/std": 270.2475891113281,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0015741336392238736,
"fcm_dpo/delta": 0.07154025882482529,
"fcm_dpo/margin": 178.01559448242188,
"fcm_dpo/q_t": 0.43446093797683716,
"grad_norm": 32.93223190307617,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.4147510528564453,
"logits/rejected": -0.4130870997905731,
"logps/chosen": -422.80352783203125,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -637.7188720703125,
"loss": 1.176,
"margin_dpo/margin_mean": 178.0155792236328,
"margin_dpo/margin_std": 290.41571044921875,
"step": 361
},
{
"KL/chosen_KL_mean": -313.84429931640625,
"KL/mean": -461.6890869140625,
"KL/rejected_KL_mean": -609.533935546875,
"KL/std": 282.345947265625,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.001544571714475751,
"fcm_dpo/delta": -0.11250950396060944,
"fcm_dpo/margin": 295.68963623046875,
"fcm_dpo/q_t": 0.39361345767974854,
"grad_norm": 26.073640823364258,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.44423967599868774,
"logits/rejected": -0.4537912905216217,
"logps/chosen": -372.48333740234375,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -715.1158447265625,
"loss": 1.0313,
"margin_dpo/margin_mean": 295.68963623046875,
"margin_dpo/margin_std": 301.7510986328125,
"step": 362
},
{
"KL/chosen_KL_mean": -284.67132568359375,
"KL/mean": -432.5,
"KL/rejected_KL_mean": -580.3286743164062,
"KL/std": 269.4873962402344,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0015216870233416557,
"fcm_dpo/delta": -0.10447092354297638,
"fcm_dpo/margin": 295.65728759765625,
"fcm_dpo/q_t": 0.3945517838001251,
"grad_norm": 30.12809944152832,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.3808155059814453,
"logits/rejected": -0.37547147274017334,
"logps/chosen": -329.229736328125,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -655.0236206054688,
"loss": 1.0335,
"margin_dpo/margin_mean": 295.65728759765625,
"margin_dpo/margin_std": 295.49774169921875,
"step": 363
},
{
"KL/chosen_KL_mean": -308.2042236328125,
"KL/mean": -420.90313720703125,
"KL/rejected_KL_mean": -533.6021728515625,
"KL/std": 261.20819091796875,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.0015164412325248122,
"fcm_dpo/delta": 0.008228322491049767,
"fcm_dpo/margin": 225.39791870117188,
"fcm_dpo/q_t": 0.41953587532043457,
"grad_norm": 19.665786743164062,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.423043817281723,
"logits/rejected": -0.4345157742500305,
"logps/chosen": -357.09881591796875,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -624.9979248046875,
"loss": 1.1234,
"margin_dpo/margin_mean": 225.39791870117188,
"margin_dpo/margin_std": 298.9825439453125,
"step": 364
},
{
"KL/chosen_KL_mean": -306.31304931640625,
"KL/mean": -421.04754638671875,
"KL/rejected_KL_mean": -535.781982421875,
"KL/std": 272.7528076171875,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.001512871589511633,
"fcm_dpo/delta": 0.002944107400253415,
"fcm_dpo/margin": 229.468994140625,
"fcm_dpo/q_t": 0.41882503032684326,
"grad_norm": 22.943742752075195,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.4167822003364563,
"logits/rejected": -0.4276372194290161,
"logps/chosen": -357.8057861328125,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -628.483642578125,
"loss": 1.1113,
"margin_dpo/margin_mean": 229.46900939941406,
"margin_dpo/margin_std": 278.7212219238281,
"step": 365
},
{
"KL/chosen_KL_mean": -279.43597412109375,
"KL/mean": -397.7274169921875,
"KL/rejected_KL_mean": -516.0189208984375,
"KL/std": 259.2245178222656,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.001505196327343583,
"fcm_dpo/delta": -0.006696049124002457,
"fcm_dpo/margin": 236.58290100097656,
"fcm_dpo/q_t": 0.41680413484573364,
"grad_norm": 20.372705459594727,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.41744059324264526,
"logits/rejected": -0.43434953689575195,
"logps/chosen": -324.15655517578125,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -599.3292846679688,
"loss": 1.1141,
"margin_dpo/margin_mean": 236.5828857421875,
"margin_dpo/margin_std": 308.424072265625,
"step": 366
},
{
"KL/chosen_KL_mean": -296.7589111328125,
"KL/mean": -394.7176818847656,
"KL/rejected_KL_mean": -492.67645263671875,
"KL/std": 231.99639892578125,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0015253680758178234,
"fcm_dpo/delta": 0.05257764458656311,
"fcm_dpo/margin": 195.91758728027344,
"fcm_dpo/q_t": 0.4285646378993988,
"grad_norm": 20.739452362060547,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.4246237277984619,
"logits/rejected": -0.40804946422576904,
"logps/chosen": -355.164306640625,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -569.4277954101562,
"loss": 1.1444,
"margin_dpo/margin_mean": 195.91757202148438,
"margin_dpo/margin_std": 247.79733276367188,
"step": 367
},
{
"KL/chosen_KL_mean": -280.85968017578125,
"KL/mean": -437.2155456542969,
"KL/rejected_KL_mean": -593.5714111328125,
"KL/std": 266.61279296875,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.0014983892906457186,
"fcm_dpo/delta": -0.12442989647388458,
"fcm_dpo/margin": 312.7117919921875,
"fcm_dpo/q_t": 0.38702040910720825,
"grad_norm": 45.461280822753906,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.4298338294029236,
"logits/rejected": -0.43975430727005005,
"logps/chosen": -325.31219482421875,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -692.126708984375,
"loss": 1.0009,
"margin_dpo/margin_mean": 312.7117919921875,
"margin_dpo/margin_std": 233.67335510253906,
"step": 368
},
{
"KL/chosen_KL_mean": -379.5709228515625,
"KL/mean": -460.81072998046875,
"KL/rejected_KL_mean": -542.0505981445312,
"KL/std": 278.4993896484375,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.001493208110332489,
"fcm_dpo/delta": 0.02635270729660988,
"fcm_dpo/margin": 162.47970581054688,
"fcm_dpo/q_t": 0.44229739904403687,
"grad_norm": 22.148353576660156,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.4620640277862549,
"logits/rejected": -0.4419480264186859,
"logps/chosen": -450.9524230957031,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -633.346435546875,
"loss": 1.2114,
"margin_dpo/margin_mean": 162.47970581054688,
"margin_dpo/margin_std": 314.4382019042969,
"step": 369
},
{
"KL/chosen_KL_mean": -407.7558288574219,
"KL/mean": -481.97650146484375,
"KL/rejected_KL_mean": -556.1971435546875,
"KL/std": 289.5532531738281,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.0015031839720904827,
"fcm_dpo/delta": 0.04018635302782059,
"fcm_dpo/margin": 148.44134521484375,
"fcm_dpo/q_t": 0.4480086863040924,
"grad_norm": 38.19746398925781,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.476939857006073,
"logits/rejected": -0.4694327414035797,
"logps/chosen": -479.36334228515625,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -653.4569702148438,
"loss": 1.2328,
"margin_dpo/margin_mean": 148.44134521484375,
"margin_dpo/margin_std": 322.265869140625,
"step": 370
},
{
"KL/chosen_KL_mean": -384.25799560546875,
"KL/mean": -499.07647705078125,
"KL/rejected_KL_mean": -613.8948974609375,
"KL/std": 296.7556457519531,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0015145591460168362,
"fcm_dpo/delta": 0.0020634010434150696,
"fcm_dpo/margin": 229.63694763183594,
"fcm_dpo/q_t": 0.41980546712875366,
"grad_norm": 30.20676612854004,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.4951871335506439,
"logits/rejected": -0.4812811017036438,
"logps/chosen": -453.6724853515625,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -713.067138671875,
"loss": 1.1315,
"margin_dpo/margin_mean": 229.63694763183594,
"margin_dpo/margin_std": 338.1903991699219,
"step": 371
},
{
"KL/chosen_KL_mean": -376.35491943359375,
"KL/mean": -509.65484619140625,
"KL/rejected_KL_mean": -642.9547119140625,
"KL/std": 326.4896545410156,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.0014974682126194239,
"fcm_dpo/delta": -0.05130906403064728,
"fcm_dpo/margin": 266.59979248046875,
"fcm_dpo/q_t": 0.4094997048377991,
"grad_norm": 24.317825317382812,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.46680545806884766,
"logits/rejected": -0.44958722591400146,
"logps/chosen": -438.17291259765625,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -721.4942626953125,
"loss": 1.0968,
"margin_dpo/margin_mean": 266.59979248046875,
"margin_dpo/margin_std": 374.668212890625,
"step": 372
},
{
"KL/chosen_KL_mean": -410.6204833984375,
"KL/mean": -533.13623046875,
"KL/rejected_KL_mean": -655.6519775390625,
"KL/std": 314.3919677734375,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.0014972374774515629,
"fcm_dpo/delta": -0.017848707735538483,
"fcm_dpo/margin": 245.0315399169922,
"fcm_dpo/q_t": 0.4135189950466156,
"grad_norm": 25.24411392211914,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.4661676287651062,
"logits/rejected": -0.44771280884742737,
"logps/chosen": -474.8376159667969,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -741.6116333007812,
"loss": 1.1063,
"margin_dpo/margin_mean": 245.03155517578125,
"margin_dpo/margin_std": 305.9546813964844,
"step": 373
},
{
"KL/chosen_KL_mean": -357.4342346191406,
"KL/mean": -472.685302734375,
"KL/rejected_KL_mean": -587.936279296875,
"KL/std": 340.931396484375,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.001483630738221109,
"fcm_dpo/delta": 0.007859412580728531,
"fcm_dpo/margin": 230.50205993652344,
"fcm_dpo/q_t": 0.4234431982040405,
"grad_norm": 25.0133113861084,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.5473544597625732,
"logits/rejected": -0.5300641059875488,
"logps/chosen": -428.08441162109375,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -681.5764770507812,
"loss": 1.149,
"margin_dpo/margin_mean": 230.50204467773438,
"margin_dpo/margin_std": 383.9598388671875,
"step": 374
},
{
"KL/chosen_KL_mean": -355.9977722167969,
"KL/mean": -470.6821594238281,
"KL/rejected_KL_mean": -585.3665771484375,
"KL/std": 272.306396484375,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.001486303168348968,
"fcm_dpo/delta": 0.008981514722108841,
"fcm_dpo/margin": 229.36874389648438,
"fcm_dpo/q_t": 0.42030322551727295,
"grad_norm": 28.873268127441406,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.5043726563453674,
"logits/rejected": -0.502221941947937,
"logps/chosen": -416.0780029296875,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -674.3048095703125,
"loss": 1.1302,
"margin_dpo/margin_mean": 229.36875915527344,
"margin_dpo/margin_std": 321.7081298828125,
"step": 375
},
{
"KL/chosen_KL_mean": -350.9241943359375,
"KL/mean": -486.59490966796875,
"KL/rejected_KL_mean": -622.265625,
"KL/std": 287.79742431640625,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0014795248862355947,
"fcm_dpo/delta": -0.053773216903209686,
"fcm_dpo/margin": 271.3414306640625,
"fcm_dpo/q_t": 0.40609490871429443,
"grad_norm": 24.37376594543457,
"learning_rate": 2.5e-07,
"logits/chosen": -0.4954999089241028,
"logits/rejected": -0.4836694002151489,
"logps/chosen": -413.58453369140625,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -727.792236328125,
"loss": 1.0827,
"margin_dpo/margin_mean": 271.3414306640625,
"margin_dpo/margin_std": 334.4213562011719,
"step": 376
},
{
"KL/chosen_KL_mean": -335.0908203125,
"KL/mean": -465.4486389160156,
"KL/rejected_KL_mean": -595.8064575195312,
"KL/std": 300.1963195800781,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.0014730466064065695,
"fcm_dpo/delta": -0.03540883585810661,
"fcm_dpo/margin": 260.71563720703125,
"fcm_dpo/q_t": 0.40986379981040955,
"grad_norm": 32.48672866821289,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.46442294120788574,
"logits/rejected": -0.4637567102909088,
"logps/chosen": -389.569580078125,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -694.509765625,
"loss": 1.0863,
"margin_dpo/margin_mean": 260.71563720703125,
"margin_dpo/margin_std": 304.24932861328125,
"step": 377
},
{
"KL/chosen_KL_mean": -324.253173828125,
"KL/mean": -457.3460693359375,
"KL/rejected_KL_mean": -590.43896484375,
"KL/std": 271.17132568359375,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.0014589335769414902,
"fcm_dpo/delta": -0.03989920765161514,
"fcm_dpo/margin": 266.185791015625,
"fcm_dpo/q_t": 0.4074614346027374,
"grad_norm": 30.09598731994629,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.44874757528305054,
"logits/rejected": -0.4621288776397705,
"logps/chosen": -369.273681640625,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -678.4859619140625,
"loss": 1.0774,
"margin_dpo/margin_mean": 266.185791015625,
"margin_dpo/margin_std": 289.00152587890625,
"step": 378
},
{
"KL/chosen_KL_mean": -363.91680908203125,
"KL/mean": -496.1492919921875,
"KL/rejected_KL_mean": -628.3817138671875,
"KL/std": 281.95513916015625,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.001443482586182654,
"fcm_dpo/delta": -0.03341230750083923,
"fcm_dpo/margin": 264.4649658203125,
"fcm_dpo/q_t": 0.41118955612182617,
"grad_norm": 27.302921295166016,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.4446529150009155,
"logits/rejected": -0.462520956993103,
"logps/chosen": -417.09893798828125,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -742.681884765625,
"loss": 1.0973,
"margin_dpo/margin_mean": 264.4649658203125,
"margin_dpo/margin_std": 336.5331115722656,
"step": 379
},
{
"KL/chosen_KL_mean": -347.66009521484375,
"KL/mean": -501.8836669921875,
"KL/rejected_KL_mean": -656.1072998046875,
"KL/std": 324.0223083496094,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.0014270804822444916,
"fcm_dpo/delta": -0.0942816212773323,
"fcm_dpo/margin": 308.44720458984375,
"fcm_dpo/q_t": 0.3965052366256714,
"grad_norm": 23.3795166015625,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.4075812101364136,
"logits/rejected": -0.4323120415210724,
"logps/chosen": -399.01312255859375,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -760.2989501953125,
"loss": 1.0519,
"margin_dpo/margin_mean": 308.44720458984375,
"margin_dpo/margin_std": 349.7032470703125,
"step": 380
},
{
"KL/chosen_KL_mean": -357.15435791015625,
"KL/mean": -460.5580749511719,
"KL/rejected_KL_mean": -563.9618530273438,
"KL/std": 264.73126220703125,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.0014290527906268835,
"fcm_dpo/delta": 0.056157857179641724,
"fcm_dpo/margin": 206.8074951171875,
"fcm_dpo/q_t": 0.43153679370880127,
"grad_norm": 26.060014724731445,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.4189026951789856,
"logits/rejected": -0.3993133306503296,
"logps/chosen": -414.9573974609375,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -643.1812744140625,
"loss": 1.1781,
"margin_dpo/margin_mean": 206.8074951171875,
"margin_dpo/margin_std": 366.61651611328125,
"step": 381
},
{
"KL/chosen_KL_mean": -346.9622497558594,
"KL/mean": -476.4005126953125,
"KL/rejected_KL_mean": -605.8387451171875,
"KL/std": 250.68878173828125,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0014304309152066708,
"fcm_dpo/delta": -0.02109403908252716,
"fcm_dpo/margin": 258.87652587890625,
"fcm_dpo/q_t": 0.41199958324432373,
"grad_norm": 26.346120834350586,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.4333261251449585,
"logits/rejected": -0.43255484104156494,
"logps/chosen": -412.9825439453125,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -716.5488891601562,
"loss": 1.0869,
"margin_dpo/margin_mean": 258.87652587890625,
"margin_dpo/margin_std": 273.289306640625,
"step": 382
},
{
"KL/chosen_KL_mean": -361.38958740234375,
"KL/mean": -491.83441162109375,
"KL/rejected_KL_mean": -622.2792358398438,
"KL/std": 291.9897766113281,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0014227345818653703,
"fcm_dpo/delta": -0.022058716043829918,
"fcm_dpo/margin": 260.88958740234375,
"fcm_dpo/q_t": 0.4132228195667267,
"grad_norm": 29.732643127441406,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.4310753345489502,
"logits/rejected": -0.4417402744293213,
"logps/chosen": -411.78106689453125,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -715.9951171875,
"loss": 1.1175,
"margin_dpo/margin_mean": 260.8896179199219,
"margin_dpo/margin_std": 378.602783203125,
"step": 383
},
{
"KL/chosen_KL_mean": -364.3046569824219,
"KL/mean": -464.1553649902344,
"KL/rejected_KL_mean": -564.006103515625,
"KL/std": 258.0005187988281,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.0014413523022085428,
"fcm_dpo/delta": 0.06357108056545258,
"fcm_dpo/margin": 199.70144653320312,
"fcm_dpo/q_t": 0.43101757764816284,
"grad_norm": 23.503877639770508,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.46157699823379517,
"logits/rejected": -0.452648401260376,
"logps/chosen": -416.35076904296875,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -649.7669677734375,
"loss": 1.1487,
"margin_dpo/margin_mean": 199.70143127441406,
"margin_dpo/margin_std": 238.79812622070312,
"step": 384
},
{
"KL/chosen_KL_mean": -332.13482666015625,
"KL/mean": -453.1705627441406,
"KL/rejected_KL_mean": -574.2062377929688,
"KL/std": 223.9749755859375,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0014425960835069418,
"fcm_dpo/delta": 0.000802924856543541,
"fcm_dpo/margin": 242.0714569091797,
"fcm_dpo/q_t": 0.4163161516189575,
"grad_norm": 27.88534164428711,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.5080987215042114,
"logits/rejected": -0.48287850618362427,
"logps/chosen": -397.6869812011719,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -652.0341796875,
"loss": 1.0991,
"margin_dpo/margin_mean": 242.0714569091797,
"margin_dpo/margin_std": 249.71163940429688,
"step": 385
},
{
"KL/chosen_KL_mean": -357.517578125,
"KL/mean": -492.87701416015625,
"KL/rejected_KL_mean": -628.2364501953125,
"KL/std": 289.87762451171875,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.001433402532711625,
"fcm_dpo/delta": -0.03957575559616089,
"fcm_dpo/margin": 270.7188720703125,
"fcm_dpo/q_t": 0.4095271825790405,
"grad_norm": 19.920806884765625,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.4485896825790405,
"logits/rejected": -0.44217997789382935,
"logps/chosen": -415.7394104003906,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -720.5638427734375,
"loss": 1.0902,
"margin_dpo/margin_mean": 270.7189025878906,
"margin_dpo/margin_std": 338.1651306152344,
"step": 386
},
{
"KL/chosen_KL_mean": -396.12030029296875,
"KL/mean": -507.7943115234375,
"KL/rejected_KL_mean": -619.46826171875,
"KL/std": 272.486328125,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.0014316510641947389,
"fcm_dpo/delta": 0.031094016507267952,
"fcm_dpo/margin": 223.3479461669922,
"fcm_dpo/q_t": 0.42408478260040283,
"grad_norm": 29.61167335510254,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.5005359649658203,
"logits/rejected": -0.4781208038330078,
"logps/chosen": -462.5397644042969,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -711.637451171875,
"loss": 1.1323,
"margin_dpo/margin_mean": 223.3479461669922,
"margin_dpo/margin_std": 280.696044921875,
"step": 387
},
{
"KL/chosen_KL_mean": -375.528076171875,
"KL/mean": -526.5117797851562,
"KL/rejected_KL_mean": -677.4954833984375,
"KL/std": 317.00433349609375,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.0014184208121150732,
"fcm_dpo/delta": -0.08210138231515884,
"fcm_dpo/margin": 301.9674072265625,
"fcm_dpo/q_t": 0.40150347352027893,
"grad_norm": 24.022911071777344,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.41366052627563477,
"logits/rejected": -0.4241475462913513,
"logps/chosen": -425.65753173828125,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -781.9285888671875,
"loss": 1.0641,
"margin_dpo/margin_mean": 301.9674072265625,
"margin_dpo/margin_std": 364.0025634765625,
"step": 388
},
{
"KL/chosen_KL_mean": -425.42889404296875,
"KL/mean": -547.1348876953125,
"KL/rejected_KL_mean": -668.8409423828125,
"KL/std": 310.88848876953125,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.0014143642038106918,
"fcm_dpo/delta": 0.005878929980099201,
"fcm_dpo/margin": 243.41204833984375,
"fcm_dpo/q_t": 0.41971537470817566,
"grad_norm": 36.894657135009766,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.49010196328163147,
"logits/rejected": -0.4835142195224762,
"logps/chosen": -483.33544921875,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -746.7554931640625,
"loss": 1.1298,
"margin_dpo/margin_mean": 243.41204833984375,
"margin_dpo/margin_std": 350.9767761230469,
"step": 389
},
{
"KL/chosen_KL_mean": -416.3863220214844,
"KL/mean": -545.6452026367188,
"KL/rejected_KL_mean": -674.904052734375,
"KL/std": 316.62554931640625,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.0014098630053922534,
"fcm_dpo/delta": -0.01524612121284008,
"fcm_dpo/margin": 258.5177307128906,
"fcm_dpo/q_t": 0.4184497892856598,
"grad_norm": 23.087417602539062,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.44321155548095703,
"logits/rejected": -0.4361841678619385,
"logps/chosen": -465.61224365234375,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -760.4322509765625,
"loss": 1.1294,
"margin_dpo/margin_mean": 258.5177307128906,
"margin_dpo/margin_std": 404.07928466796875,
"step": 390
},
{
"KL/chosen_KL_mean": -422.77203369140625,
"KL/mean": -493.1278076171875,
"KL/rejected_KL_mean": -563.4835205078125,
"KL/std": 304.7773742675781,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0014129449846222997,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 140.71145629882812,
"fcm_dpo/q_t": 0.4527771472930908,
"grad_norm": 47.28102111816406,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.4732820391654968,
"logits/rejected": -0.464574933052063,
"logps/chosen": -487.1016845703125,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -650.2216796875,
"loss": 1.2471,
"margin_dpo/margin_mean": 140.71145629882812,
"margin_dpo/margin_std": 314.5218200683594,
"step": 391
},
{
"KL/chosen_KL_mean": -348.7242431640625,
"KL/mean": -502.31378173828125,
"KL/rejected_KL_mean": -655.9034423828125,
"KL/std": 310.9810791015625,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0014000183437019587,
"fcm_dpo/delta": -0.08359719067811966,
"fcm_dpo/margin": 307.1791076660156,
"fcm_dpo/q_t": 0.39894986152648926,
"grad_norm": 29.815181732177734,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.4261566400527954,
"logits/rejected": -0.4199681282043457,
"logps/chosen": -402.22821044921875,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -758.249267578125,
"loss": 1.0465,
"margin_dpo/margin_mean": 307.17913818359375,
"margin_dpo/margin_std": 307.54779052734375,
"step": 392
},
{
"KL/chosen_KL_mean": -347.76776123046875,
"KL/mean": -462.5784912109375,
"KL/rejected_KL_mean": -577.38916015625,
"KL/std": 302.32501220703125,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0013939584605395794,
"fcm_dpo/delta": 0.03090941533446312,
"fcm_dpo/margin": 229.621337890625,
"fcm_dpo/q_t": 0.4278019666671753,
"grad_norm": 20.847705841064453,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.4470987319946289,
"logits/rejected": -0.44747045636177063,
"logps/chosen": -394.24169921875,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -649.3580322265625,
"loss": 1.1562,
"margin_dpo/margin_mean": 229.62136840820312,
"margin_dpo/margin_std": 383.7559509277344,
"step": 393
},
{
"KL/chosen_KL_mean": -405.95355224609375,
"KL/mean": -526.41748046875,
"KL/rejected_KL_mean": -646.88134765625,
"KL/std": 321.18109130859375,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0013992037856951356,
"fcm_dpo/delta": 0.01332851778715849,
"fcm_dpo/margin": 240.9277801513672,
"fcm_dpo/q_t": 0.4218076467514038,
"grad_norm": 23.301321029663086,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.4254276156425476,
"logits/rejected": -0.42678093910217285,
"logps/chosen": -458.8651123046875,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -737.7039794921875,
"loss": 1.1278,
"margin_dpo/margin_mean": 240.9277801513672,
"margin_dpo/margin_std": 331.58807373046875,
"step": 394
},
{
"KL/chosen_KL_mean": -393.3065185546875,
"KL/mean": -526.68896484375,
"KL/rejected_KL_mean": -660.071533203125,
"KL/std": 314.0694274902344,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0013938779011368752,
"fcm_dpo/delta": -0.02284281887114048,
"fcm_dpo/margin": 266.7650146484375,
"fcm_dpo/q_t": 0.41418999433517456,
"grad_norm": 23.73855972290039,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.49259886145591736,
"logits/rejected": -0.4919501841068268,
"logps/chosen": -455.85260009765625,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -743.8541259765625,
"loss": 1.1106,
"margin_dpo/margin_mean": 266.7650146484375,
"margin_dpo/margin_std": 368.64495849609375,
"step": 395
},
{
"KL/chosen_KL_mean": -391.6252136230469,
"KL/mean": -510.94085693359375,
"KL/rejected_KL_mean": -630.256591796875,
"KL/std": 305.98333740234375,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.0013969738502055407,
"fcm_dpo/delta": 0.017192337661981583,
"fcm_dpo/margin": 238.63131713867188,
"fcm_dpo/q_t": 0.42099273204803467,
"grad_norm": 21.45150375366211,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.5134637355804443,
"logits/rejected": -0.4980122447013855,
"logps/chosen": -460.62115478515625,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -718.9031982421875,
"loss": 1.1162,
"margin_dpo/margin_mean": 238.6313018798828,
"margin_dpo/margin_std": 278.1322021484375,
"step": 396
},
{
"KL/chosen_KL_mean": -366.0447082519531,
"KL/mean": -518.56298828125,
"KL/rejected_KL_mean": -671.0814208984375,
"KL/std": 297.60498046875,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.0013800031738355756,
"fcm_dpo/delta": -0.07457705587148666,
"fcm_dpo/margin": 305.0367126464844,
"fcm_dpo/q_t": 0.4003956913948059,
"grad_norm": 33.268245697021484,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.45172950625419617,
"logits/rejected": -0.45972180366516113,
"logps/chosen": -427.3218688964844,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -774.197509765625,
"loss": 1.0549,
"margin_dpo/margin_mean": 305.03668212890625,
"margin_dpo/margin_std": 317.1044616699219,
"step": 397
},
{
"KL/chosen_KL_mean": -392.1479797363281,
"KL/mean": -530.8260498046875,
"KL/rejected_KL_mean": -669.5042114257812,
"KL/std": 315.48931884765625,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.0013728467747569084,
"fcm_dpo/delta": -0.03198913484811783,
"fcm_dpo/margin": 277.356201171875,
"fcm_dpo/q_t": 0.41205263137817383,
"grad_norm": 21.440746307373047,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.5343962907791138,
"logits/rejected": -0.5324475765228271,
"logps/chosen": -460.2995300292969,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -778.02783203125,
"loss": 1.0983,
"margin_dpo/margin_mean": 277.3562316894531,
"margin_dpo/margin_std": 362.9688720703125,
"step": 398
},
{
"KL/chosen_KL_mean": -341.6549377441406,
"KL/mean": -460.3332214355469,
"KL/rejected_KL_mean": -579.0115356445312,
"KL/std": 272.2782287597656,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0013724948512390256,
"fcm_dpo/delta": 0.024983234703540802,
"fcm_dpo/margin": 237.35659790039062,
"fcm_dpo/q_t": 0.4236915111541748,
"grad_norm": 24.46724510192871,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.4898013174533844,
"logits/rejected": -0.45694947242736816,
"logps/chosen": -402.54473876953125,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -656.9771118164062,
"loss": 1.1392,
"margin_dpo/margin_mean": 237.35659790039062,
"margin_dpo/margin_std": 339.81365966796875,
"step": 399
},
{
"KL/chosen_KL_mean": -335.7412109375,
"KL/mean": -509.16314697265625,
"KL/rejected_KL_mean": -682.585205078125,
"KL/std": 286.89813232421875,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.001349159749224782,
"fcm_dpo/delta": -0.12415604293346405,
"fcm_dpo/margin": 346.8439636230469,
"fcm_dpo/q_t": 0.3882994055747986,
"grad_norm": 20.42949104309082,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.5137765407562256,
"logits/rejected": -0.5164045691490173,
"logps/chosen": -399.3847961425781,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -787.837890625,
"loss": 1.0159,
"margin_dpo/margin_mean": 346.8439636230469,
"margin_dpo/margin_std": 318.0387268066406,
"step": 400
},
{
"KL/chosen_KL_mean": -375.37005615234375,
"KL/mean": -471.6986083984375,
"KL/rejected_KL_mean": -568.027099609375,
"KL/std": 302.75244140625,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0013488534605130553,
"fcm_dpo/delta": 0.018642518669366837,
"fcm_dpo/margin": 192.65704345703125,
"fcm_dpo/q_t": 0.44042983651161194,
"grad_norm": 21.7044677734375,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.4904178977012634,
"logits/rejected": -0.48238423466682434,
"logps/chosen": -432.5331115722656,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -651.819580078125,
"loss": 1.2158,
"margin_dpo/margin_mean": 192.65704345703125,
"margin_dpo/margin_std": 408.41400146484375,
"step": 401
},
{
"KL/chosen_KL_mean": -283.151611328125,
"KL/mean": -461.8321228027344,
"KL/rejected_KL_mean": -640.5125732421875,
"KL/std": 318.71820068359375,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0013233155477792025,
"fcm_dpo/delta": -0.12899458408355713,
"fcm_dpo/margin": 357.3609924316406,
"fcm_dpo/q_t": 0.387925922870636,
"grad_norm": 26.780061721801758,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.5302881002426147,
"logits/rejected": -0.5248584151268005,
"logps/chosen": -333.8919982910156,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -721.5586547851562,
"loss": 1.009,
"margin_dpo/margin_mean": 357.3609924316406,
"margin_dpo/margin_std": 318.2284851074219,
"step": 402
},
{
"KL/chosen_KL_mean": -328.6024169921875,
"KL/mean": -467.64849853515625,
"KL/rejected_KL_mean": -606.694580078125,
"KL/std": 301.4685974121094,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.001311097526922822,
"fcm_dpo/delta": -0.015165827237069607,
"fcm_dpo/margin": 278.0921325683594,
"fcm_dpo/q_t": 0.4141251742839813,
"grad_norm": 29.468088150024414,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.525422990322113,
"logits/rejected": -0.5270965099334717,
"logps/chosen": -375.749755859375,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -683.961181640625,
"loss": 1.0966,
"margin_dpo/margin_mean": 278.0921325683594,
"margin_dpo/margin_std": 317.0299987792969,
"step": 403
},
{
"KL/chosen_KL_mean": -375.371337890625,
"KL/mean": -508.93017578125,
"KL/rejected_KL_mean": -642.4889526367188,
"KL/std": 296.78961181640625,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0013087866827845573,
"fcm_dpo/delta": 0.00037629250437021255,
"fcm_dpo/margin": 267.11761474609375,
"fcm_dpo/q_t": 0.41891252994537354,
"grad_norm": 28.834339141845703,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.5048468708992004,
"logits/rejected": -0.4995875358581543,
"logps/chosen": -423.24664306640625,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -719.6439208984375,
"loss": 1.1319,
"margin_dpo/margin_mean": 267.11761474609375,
"margin_dpo/margin_std": 398.306884765625,
"step": 404
},
{
"KL/chosen_KL_mean": -412.86749267578125,
"KL/mean": -529.7939453125,
"KL/rejected_KL_mean": -646.7203979492188,
"KL/std": 337.2768249511719,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.001304717967286706,
"fcm_dpo/delta": -0.04521414637565613,
"fcm_dpo/margin": 233.85287475585938,
"fcm_dpo/q_t": 0.4310234785079956,
"grad_norm": 30.699373245239258,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.5107825398445129,
"logits/rejected": -0.5024291276931763,
"logps/chosen": -478.0303955078125,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -733.9072265625,
"loss": 1.1835,
"margin_dpo/margin_mean": 233.85287475585938,
"margin_dpo/margin_std": 432.97540283203125,
"step": 405
},
{
"KL/chosen_KL_mean": -389.0184020996094,
"KL/mean": -537.796142578125,
"KL/rejected_KL_mean": -686.5738525390625,
"KL/std": 333.91748046875,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0012974301353096962,
"fcm_dpo/delta": -0.03775372728705406,
"fcm_dpo/margin": 297.55548095703125,
"fcm_dpo/q_t": 0.4103890061378479,
"grad_norm": 29.44150733947754,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.5451512932777405,
"logits/rejected": -0.5550295114517212,
"logps/chosen": -438.75921630859375,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -778.6525268554688,
"loss": 1.0925,
"margin_dpo/margin_mean": 297.55548095703125,
"margin_dpo/margin_std": 369.20068359375,
"step": 406
},
{
"KL/chosen_KL_mean": -414.57012939453125,
"KL/mean": -509.9847106933594,
"KL/rejected_KL_mean": -605.3992919921875,
"KL/std": 269.21441650390625,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.001289546606130898,
"fcm_dpo/delta": 0.003860359312966466,
"fcm_dpo/margin": 190.8292236328125,
"fcm_dpo/q_t": 0.4413040280342102,
"grad_norm": 29.11326789855957,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.5535376071929932,
"logits/rejected": -0.523551344871521,
"logps/chosen": -470.90081787109375,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -682.911376953125,
"loss": 1.2067,
"margin_dpo/margin_mean": 190.82920837402344,
"margin_dpo/margin_std": 349.5740966796875,
"step": 407
},
{
"KL/chosen_KL_mean": -411.55230712890625,
"KL/mean": -515.71240234375,
"KL/rejected_KL_mean": -619.8724365234375,
"KL/std": 258.9462585449219,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.0013068197295069695,
"fcm_dpo/delta": 0.07983996719121933,
"fcm_dpo/margin": 208.32012939453125,
"fcm_dpo/q_t": 0.43545645475387573,
"grad_norm": 23.367450714111328,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.5809944272041321,
"logits/rejected": -0.5597822666168213,
"logps/chosen": -481.34161376953125,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -709.9693603515625,
"loss": 1.1688,
"margin_dpo/margin_mean": 208.32012939453125,
"margin_dpo/margin_std": 296.0139465332031,
"step": 408
},
{
"KL/chosen_KL_mean": -399.9471435546875,
"KL/mean": -502.8631286621094,
"KL/rejected_KL_mean": -605.7791137695312,
"KL/std": 295.7550048828125,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0013286788016557693,
"fcm_dpo/delta": 0.07845626771450043,
"fcm_dpo/margin": 205.8319549560547,
"fcm_dpo/q_t": 0.4356931746006012,
"grad_norm": 44.062828063964844,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.5095489025115967,
"logits/rejected": -0.4916442632675171,
"logps/chosen": -467.26458740234375,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -690.6834106445312,
"loss": 1.1764,
"margin_dpo/margin_mean": 205.8319549560547,
"margin_dpo/margin_std": 322.6492919921875,
"step": 409
},
{
"KL/chosen_KL_mean": -363.33453369140625,
"KL/mean": -491.4117431640625,
"KL/rejected_KL_mean": -619.4889526367188,
"KL/std": 283.58184814453125,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.00133224087767303,
"fcm_dpo/delta": 0.009043154306709766,
"fcm_dpo/margin": 256.1543884277344,
"fcm_dpo/q_t": 0.4186854362487793,
"grad_norm": 23.003204345703125,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.5263328552246094,
"logits/rejected": -0.5296115875244141,
"logps/chosen": -414.7998962402344,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -702.6879272460938,
"loss": 1.1177,
"margin_dpo/margin_mean": 256.1543884277344,
"margin_dpo/margin_std": 320.51104736328125,
"step": 410
},
{
"KL/chosen_KL_mean": -383.42071533203125,
"KL/mean": -508.7550354003906,
"KL/rejected_KL_mean": -634.08935546875,
"KL/std": 312.1067810058594,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0013171627651900053,
"fcm_dpo/delta": -0.06332255154848099,
"fcm_dpo/margin": 250.6686553955078,
"fcm_dpo/q_t": 0.42369675636291504,
"grad_norm": 32.008201599121094,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.5449320077896118,
"logits/rejected": -0.5536935925483704,
"logps/chosen": -435.72796630859375,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -714.7843017578125,
"loss": 1.142,
"margin_dpo/margin_mean": 250.6686553955078,
"margin_dpo/margin_std": 367.11444091796875,
"step": 411
},
{
"KL/chosen_KL_mean": -392.28662109375,
"KL/mean": -521.355712890625,
"KL/rejected_KL_mean": -650.4248046875,
"KL/std": 303.547119140625,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.00132085126824677,
"fcm_dpo/delta": 0.009303496219217777,
"fcm_dpo/margin": 258.1380920410156,
"fcm_dpo/q_t": 0.42002564668655396,
"grad_norm": 34.010955810546875,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.6012539863586426,
"logits/rejected": -0.6298267841339111,
"logps/chosen": -445.4307861328125,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -750.485595703125,
"loss": 1.1274,
"margin_dpo/margin_mean": 258.1380920410156,
"margin_dpo/margin_std": 356.703857421875,
"step": 412
},
{
"KL/chosen_KL_mean": -416.2436218261719,
"KL/mean": -545.9385986328125,
"KL/rejected_KL_mean": -675.6336059570312,
"KL/std": 308.3495178222656,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.001323135569691658,
"fcm_dpo/delta": 0.006968270055949688,
"fcm_dpo/margin": 259.3899841308594,
"fcm_dpo/q_t": 0.41812664270401,
"grad_norm": 36.40060806274414,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.4915761649608612,
"logits/rejected": -0.49571770429611206,
"logps/chosen": -477.8255615234375,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -775.1070556640625,
"loss": 1.12,
"margin_dpo/margin_mean": 259.3900146484375,
"margin_dpo/margin_std": 336.85467529296875,
"step": 413
},
{
"KL/chosen_KL_mean": -375.42376708984375,
"KL/mean": -514.7646484375,
"KL/rejected_KL_mean": -654.1056518554688,
"KL/std": 286.4696960449219,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.0013254533987492323,
"fcm_dpo/delta": -0.02075226604938507,
"fcm_dpo/margin": 278.681884765625,
"fcm_dpo/q_t": 0.41263848543167114,
"grad_norm": 27.233083724975586,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.4962560534477234,
"logits/rejected": -0.4908558428287506,
"logps/chosen": -422.05523681640625,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -741.752197265625,
"loss": 1.0978,
"margin_dpo/margin_mean": 278.681884765625,
"margin_dpo/margin_std": 322.50115966796875,
"step": 414
},
{
"KL/chosen_KL_mean": -410.77630615234375,
"KL/mean": -507.6982421875,
"KL/rejected_KL_mean": -604.6201782226562,
"KL/std": 278.3897399902344,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0013193815248087049,
"fcm_dpo/delta": 0.012895876541733742,
"fcm_dpo/margin": 193.84385681152344,
"fcm_dpo/q_t": 0.43898284435272217,
"grad_norm": 24.22504997253418,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.5504616498947144,
"logits/rejected": -0.5457872152328491,
"logps/chosen": -489.39459228515625,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -705.09765625,
"loss": 1.1892,
"margin_dpo/margin_mean": 193.84385681152344,
"margin_dpo/margin_std": 313.9031982421875,
"step": 415
},
{
"KL/chosen_KL_mean": -355.70068359375,
"KL/mean": -515.9644165039062,
"KL/rejected_KL_mean": -676.2281494140625,
"KL/std": 308.6180419921875,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0013083845842629671,
"fcm_dpo/delta": -0.07235361635684967,
"fcm_dpo/margin": 320.5274658203125,
"fcm_dpo/q_t": 0.4001033306121826,
"grad_norm": 31.607568740844727,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.5469499230384827,
"logits/rejected": -0.5526891946792603,
"logps/chosen": -413.97979736328125,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -766.796875,
"loss": 1.0473,
"margin_dpo/margin_mean": 320.5274658203125,
"margin_dpo/margin_std": 299.2843322753906,
"step": 416
},
{
"KL/chosen_KL_mean": -364.55999755859375,
"KL/mean": -502.7241516113281,
"KL/rejected_KL_mean": -640.8883056640625,
"KL/std": 286.73431396484375,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.0013004662469029427,
"fcm_dpo/delta": -0.009734570048749447,
"fcm_dpo/margin": 276.32830810546875,
"fcm_dpo/q_t": 0.4140380620956421,
"grad_norm": 24.474712371826172,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.5460329651832581,
"logits/rejected": -0.5258671045303345,
"logps/chosen": -414.75872802734375,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -709.0401611328125,
"loss": 1.0922,
"margin_dpo/margin_mean": 276.32830810546875,
"margin_dpo/margin_std": 285.49676513671875,
"step": 417
},
{
"KL/chosen_KL_mean": -389.93212890625,
"KL/mean": -532.0841674804688,
"KL/rejected_KL_mean": -674.2362060546875,
"KL/std": 330.60418701171875,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.001303232740610838,
"fcm_dpo/delta": -0.02225048840045929,
"fcm_dpo/margin": 284.3041076660156,
"fcm_dpo/q_t": 0.41474616527557373,
"grad_norm": 26.265411376953125,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.6291038990020752,
"logits/rejected": -0.6125690937042236,
"logps/chosen": -471.91058349609375,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -778.9276733398438,
"loss": 1.1142,
"margin_dpo/margin_mean": 284.3041076660156,
"margin_dpo/margin_std": 386.27587890625,
"step": 418
},
{
"KL/chosen_KL_mean": -362.5212097167969,
"KL/mean": -517.69580078125,
"KL/rejected_KL_mean": -672.8703002929688,
"KL/std": 289.50018310546875,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.001282334327697754,
"fcm_dpo/delta": -0.049968693405389786,
"fcm_dpo/margin": 310.3490905761719,
"fcm_dpo/q_t": 0.40543508529663086,
"grad_norm": 22.777507781982422,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.5401506423950195,
"logits/rejected": -0.5378715395927429,
"logps/chosen": -415.4698486328125,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -764.453369140625,
"loss": 1.0724,
"margin_dpo/margin_mean": 310.34912109375,
"margin_dpo/margin_std": 338.61309814453125,
"step": 419
},
{
"KL/chosen_KL_mean": -469.8270263671875,
"KL/mean": -561.0233154296875,
"KL/rejected_KL_mean": -652.219482421875,
"KL/std": 309.34521484375,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.0013024189975112677,
"fcm_dpo/delta": 0.11509222537279129,
"fcm_dpo/margin": 182.3925018310547,
"fcm_dpo/q_t": 0.4456251859664917,
"grad_norm": 52.509613037109375,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.5761592388153076,
"logits/rejected": -0.542568564414978,
"logps/chosen": -547.596923828125,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -721.5394287109375,
"loss": 1.2333,
"margin_dpo/margin_mean": 182.3925018310547,
"margin_dpo/margin_std": 416.1588134765625,
"step": 420
},
{
"KL/chosen_KL_mean": -383.7919921875,
"KL/mean": -492.5055236816406,
"KL/rejected_KL_mean": -601.218994140625,
"KL/std": 306.576416015625,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.001326603116467595,
"fcm_dpo/delta": 0.06283024698495865,
"fcm_dpo/margin": 217.42698669433594,
"fcm_dpo/q_t": 0.43084651231765747,
"grad_norm": 25.688739776611328,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.5750212669372559,
"logits/rejected": -0.5611597299575806,
"logps/chosen": -437.557861328125,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -690.5004272460938,
"loss": 1.1646,
"margin_dpo/margin_mean": 217.42698669433594,
"margin_dpo/margin_std": 318.6001892089844,
"step": 421
},
{
"KL/chosen_KL_mean": -440.12738037109375,
"KL/mean": -564.8936767578125,
"KL/rejected_KL_mean": -689.659912109375,
"KL/std": 308.4464111328125,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.0013281775172799826,
"fcm_dpo/delta": 0.019213195890188217,
"fcm_dpo/margin": 249.53256225585938,
"fcm_dpo/q_t": 0.4216609597206116,
"grad_norm": 33.2342414855957,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.5859231352806091,
"logits/rejected": -0.5860297083854675,
"logps/chosen": -508.7611389160156,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -777.5234375,
"loss": 1.1347,
"margin_dpo/margin_mean": 249.53257751464844,
"margin_dpo/margin_std": 353.23614501953125,
"step": 422
},
{
"KL/chosen_KL_mean": -419.96142578125,
"KL/mean": -560.3416748046875,
"KL/rejected_KL_mean": -700.721923828125,
"KL/std": 307.92205810546875,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.001325579360127449,
"fcm_dpo/delta": -0.023041730746626854,
"fcm_dpo/margin": 280.76055908203125,
"fcm_dpo/q_t": 0.412263423204422,
"grad_norm": 29.936748504638672,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.5792564153671265,
"logits/rejected": -0.5877513885498047,
"logps/chosen": -474.95172119140625,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -787.028564453125,
"loss": 1.0937,
"margin_dpo/margin_mean": 280.7605895996094,
"margin_dpo/margin_std": 330.3087158203125,
"step": 423
},
{
"KL/chosen_KL_mean": -394.10980224609375,
"KL/mean": -522.6301879882812,
"KL/rejected_KL_mean": -651.1505126953125,
"KL/std": 303.40478515625,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0013219558168202639,
"fcm_dpo/delta": 0.010341478511691093,
"fcm_dpo/margin": 257.0407409667969,
"fcm_dpo/q_t": 0.42126062512397766,
"grad_norm": 36.76081085205078,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.5852949619293213,
"logits/rejected": -0.5621461868286133,
"logps/chosen": -450.1217041015625,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -717.6295166015625,
"loss": 1.1318,
"margin_dpo/margin_mean": 257.040771484375,
"margin_dpo/margin_std": 368.2303161621094,
"step": 424
},
{
"KL/chosen_KL_mean": -452.0614013671875,
"KL/mean": -553.6272583007812,
"KL/rejected_KL_mean": -655.193115234375,
"KL/std": 295.1512145996094,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.0013455725274980068,
"fcm_dpo/delta": 0.07860462367534637,
"fcm_dpo/margin": 203.1317138671875,
"fcm_dpo/q_t": 0.4350726306438446,
"grad_norm": 33.804561614990234,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.5849053859710693,
"logits/rejected": -0.5874682664871216,
"logps/chosen": -498.9303894042969,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -751.1185302734375,
"loss": 1.1895,
"margin_dpo/margin_mean": 203.1317138671875,
"margin_dpo/margin_std": 363.55322265625,
"step": 425
},
{
"KL/chosen_KL_mean": -423.40118408203125,
"KL/mean": -549.4375,
"KL/rejected_KL_mean": -675.473876953125,
"KL/std": 297.34173583984375,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0013556771446019411,
"fcm_dpo/delta": 0.008176274597644806,
"fcm_dpo/margin": 252.0726776123047,
"fcm_dpo/q_t": 0.41926294565200806,
"grad_norm": 27.19607162475586,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.6059207916259766,
"logits/rejected": -0.5767127871513367,
"logps/chosen": -499.9847412109375,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -756.740478515625,
"loss": 1.1239,
"margin_dpo/margin_mean": 252.0726776123047,
"margin_dpo/margin_std": 330.4275207519531,
"step": 426
},
{
"KL/chosen_KL_mean": -393.76983642578125,
"KL/mean": -488.9925537109375,
"KL/rejected_KL_mean": -584.21533203125,
"KL/std": 265.6219482421875,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.001356072025373578,
"fcm_dpo/delta": 0.02023770846426487,
"fcm_dpo/margin": 190.4454803466797,
"fcm_dpo/q_t": 0.4393557906150818,
"grad_norm": 28.08988380432129,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.6063523292541504,
"logits/rejected": -0.5898270010948181,
"logps/chosen": -458.62371826171875,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -662.7813110351562,
"loss": 1.1898,
"margin_dpo/margin_mean": 190.44546508789062,
"margin_dpo/margin_std": 315.81085205078125,
"step": 427
},
{
"KL/chosen_KL_mean": -493.20880126953125,
"KL/mean": -632.4622802734375,
"KL/rejected_KL_mean": -771.7158203125,
"KL/std": 357.8115234375,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.001346116536296904,
"fcm_dpo/delta": -0.02620844356715679,
"fcm_dpo/margin": 278.5070495605469,
"fcm_dpo/q_t": 0.413205087184906,
"grad_norm": 37.65504455566406,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.5920642614364624,
"logits/rejected": -0.5966956615447998,
"logps/chosen": -555.845458984375,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -874.9976196289062,
"loss": 1.1263,
"margin_dpo/margin_mean": 278.5070495605469,
"margin_dpo/margin_std": 432.5724792480469,
"step": 428
},
{
"KL/chosen_KL_mean": -506.41094970703125,
"KL/mean": -613.6200561523438,
"KL/rejected_KL_mean": -720.8291625976562,
"KL/std": 329.47845458984375,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0013451819540932775,
"fcm_dpo/delta": -0.028253547847270966,
"fcm_dpo/margin": 214.41818237304688,
"fcm_dpo/q_t": 0.43296605348587036,
"grad_norm": 36.45220947265625,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.6370459794998169,
"logits/rejected": -0.6188497543334961,
"logps/chosen": -587.6449584960938,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -812.6240844726562,
"loss": 1.1891,
"margin_dpo/margin_mean": 214.41818237304688,
"margin_dpo/margin_std": 390.85205078125,
"step": 429
},
{
"KL/chosen_KL_mean": -406.6867370605469,
"KL/mean": -513.8056640625,
"KL/rejected_KL_mean": -620.9246215820312,
"KL/std": 296.41180419921875,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.0013558154460042715,
"fcm_dpo/delta": 0.06116756424307823,
"fcm_dpo/margin": 214.23789978027344,
"fcm_dpo/q_t": 0.43244469165802,
"grad_norm": 34.47238540649414,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.5731499791145325,
"logits/rejected": -0.5749542713165283,
"logps/chosen": -467.6070556640625,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -725.347412109375,
"loss": 1.1677,
"margin_dpo/margin_mean": 214.23788452148438,
"margin_dpo/margin_std": 341.7822265625,
"step": 430
},
{
"KL/chosen_KL_mean": -371.7575988769531,
"KL/mean": -523.7132568359375,
"KL/rejected_KL_mean": -675.6690063476562,
"KL/std": 307.078857421875,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0013448914978653193,
"fcm_dpo/delta": -0.06126604229211807,
"fcm_dpo/margin": 303.9114074707031,
"fcm_dpo/q_t": 0.40351831912994385,
"grad_norm": 35.25053405761719,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.5623865723609924,
"logits/rejected": -0.5650370717048645,
"logps/chosen": -429.1063232421875,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -768.50927734375,
"loss": 1.0697,
"margin_dpo/margin_mean": 303.91143798828125,
"margin_dpo/margin_std": 338.7742919921875,
"step": 431
},
{
"KL/chosen_KL_mean": -368.0189514160156,
"KL/mean": -524.6700439453125,
"KL/rejected_KL_mean": -681.3211669921875,
"KL/std": 317.7171936035156,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.001335380831733346,
"fcm_dpo/delta": -0.0716981440782547,
"fcm_dpo/margin": 313.3021545410156,
"fcm_dpo/q_t": 0.4016353487968445,
"grad_norm": 37.07876205444336,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.5415102243423462,
"logits/rejected": -0.5325489044189453,
"logps/chosen": -439.09375,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -779.900634765625,
"loss": 1.0569,
"margin_dpo/margin_mean": 313.3021545410156,
"margin_dpo/margin_std": 315.588134765625,
"step": 432
},
{
"KL/chosen_KL_mean": -469.5299377441406,
"KL/mean": -571.39794921875,
"KL/rejected_KL_mean": -673.2659912109375,
"KL/std": 318.712890625,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0013404275523498654,
"fcm_dpo/delta": 0.07884688675403595,
"fcm_dpo/margin": 203.736083984375,
"fcm_dpo/q_t": 0.4341526925563812,
"grad_norm": 34.616695404052734,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.630656361579895,
"logits/rejected": -0.6229075193405151,
"logps/chosen": -527.8031005859375,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -769.2169189453125,
"loss": 1.1937,
"margin_dpo/margin_mean": 203.73606872558594,
"margin_dpo/margin_std": 376.9918212890625,
"step": 433
},
{
"KL/chosen_KL_mean": -403.37701416015625,
"KL/mean": -527.6131591796875,
"KL/rejected_KL_mean": -651.8493041992188,
"KL/std": 324.9095153808594,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0013430386316031218,
"fcm_dpo/delta": 0.01675334945321083,
"fcm_dpo/margin": 248.47225952148438,
"fcm_dpo/q_t": 0.42638862133026123,
"grad_norm": 28.57368278503418,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.6396076083183289,
"logits/rejected": -0.6434687376022339,
"logps/chosen": -465.3507080078125,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -730.347900390625,
"loss": 1.147,
"margin_dpo/margin_mean": 248.4722900390625,
"margin_dpo/margin_std": 408.96661376953125,
"step": 434
},
{
"KL/chosen_KL_mean": -392.12298583984375,
"KL/mean": -531.1943359375,
"KL/rejected_KL_mean": -670.2657470703125,
"KL/std": 317.26971435546875,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.0013403587508946657,
"fcm_dpo/delta": -0.023820772767066956,
"fcm_dpo/margin": 278.14276123046875,
"fcm_dpo/q_t": 0.41278964281082153,
"grad_norm": 31.200889587402344,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.7370610237121582,
"logits/rejected": -0.731311559677124,
"logps/chosen": -443.6250305175781,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -757.8326416015625,
"loss": 1.1053,
"margin_dpo/margin_mean": 278.1427307128906,
"margin_dpo/margin_std": 360.19976806640625,
"step": 435
},
{
"KL/chosen_KL_mean": -413.7061767578125,
"KL/mean": -527.6519775390625,
"KL/rejected_KL_mean": -641.597900390625,
"KL/std": 283.1363525390625,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.001346162287518382,
"fcm_dpo/delta": 0.04453161358833313,
"fcm_dpo/margin": 227.89169311523438,
"fcm_dpo/q_t": 0.4281315207481384,
"grad_norm": 32.2522087097168,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.6348098516464233,
"logits/rejected": -0.6036412715911865,
"logps/chosen": -485.10986328125,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -724.3256225585938,
"loss": 1.1522,
"margin_dpo/margin_mean": 227.89169311523438,
"margin_dpo/margin_std": 335.2723083496094,
"step": 436
},
{
"KL/chosen_KL_mean": -422.2236328125,
"KL/mean": -543.794189453125,
"KL/rejected_KL_mean": -665.36474609375,
"KL/std": 282.27435302734375,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0013560389634221792,
"fcm_dpo/delta": 0.020979713648557663,
"fcm_dpo/margin": 243.1411590576172,
"fcm_dpo/q_t": 0.4221380054950714,
"grad_norm": 22.420637130737305,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.6972805857658386,
"logits/rejected": -0.67779940366745,
"logps/chosen": -486.9678955078125,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -747.4083251953125,
"loss": 1.1318,
"margin_dpo/margin_mean": 243.14114379882812,
"margin_dpo/margin_std": 330.1854248046875,
"step": 437
},
{
"KL/chosen_KL_mean": -457.51593017578125,
"KL/mean": -587.1756591796875,
"KL/rejected_KL_mean": -716.8353881835938,
"KL/std": 319.5601806640625,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0013543383684009314,
"fcm_dpo/delta": -0.0014728754758834839,
"fcm_dpo/margin": 259.3194274902344,
"fcm_dpo/q_t": 0.41706225275993347,
"grad_norm": 29.479515075683594,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.6999799013137817,
"logits/rejected": -0.6740041971206665,
"logps/chosen": -516.5346069335938,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -799.9122314453125,
"loss": 1.1196,
"margin_dpo/margin_mean": 259.3194274902344,
"margin_dpo/margin_std": 347.9755859375,
"step": 438
},
{
"KL/chosen_KL_mean": -469.77227783203125,
"KL/mean": -582.2373046875,
"KL/rejected_KL_mean": -694.7022705078125,
"KL/std": 336.91986083984375,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.0013382842298597097,
"fcm_dpo/delta": -0.07557918131351471,
"fcm_dpo/margin": 224.93003845214844,
"fcm_dpo/q_t": 0.43068742752075195,
"grad_norm": 33.094661712646484,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.6910462379455566,
"logits/rejected": -0.6919522285461426,
"logps/chosen": -523.5563354492188,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -778.687744140625,
"loss": 1.1801,
"margin_dpo/margin_mean": 224.9300537109375,
"margin_dpo/margin_std": 390.89849853515625,
"step": 439
},
{
"KL/chosen_KL_mean": -492.5137939453125,
"KL/mean": -635.44140625,
"KL/rejected_KL_mean": -778.3690185546875,
"KL/std": 411.0914611816406,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.001335415756329894,
"fcm_dpo/delta": -0.03305999934673309,
"fcm_dpo/margin": 285.8551330566406,
"fcm_dpo/q_t": 0.4155203104019165,
"grad_norm": 49.92783737182617,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.7562768459320068,
"logits/rejected": -0.7523295879364014,
"logps/chosen": -571.08056640625,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -874.86669921875,
"loss": 1.1137,
"margin_dpo/margin_mean": 285.8551025390625,
"margin_dpo/margin_std": 428.9686584472656,
"step": 440
},
{
"KL/chosen_KL_mean": -570.6612548828125,
"KL/mean": -697.5283813476562,
"KL/rejected_KL_mean": -824.3955688476562,
"KL/std": 409.962890625,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0013370553497225046,
"fcm_dpo/delta": 0.01073683425784111,
"fcm_dpo/margin": 253.73431396484375,
"fcm_dpo/q_t": 0.4260130524635315,
"grad_norm": 45.495601654052734,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.7495392560958862,
"logits/rejected": -0.7344924807548523,
"logps/chosen": -631.4856567382812,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -920.8663940429688,
"loss": 1.1806,
"margin_dpo/margin_mean": 253.73431396484375,
"margin_dpo/margin_std": 494.9446105957031,
"step": 441
},
{
"KL/chosen_KL_mean": -478.4798583984375,
"KL/mean": -630.342041015625,
"KL/rejected_KL_mean": -782.2042236328125,
"KL/std": 391.163330078125,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.001329011982306838,
"fcm_dpo/delta": -0.056281156837940216,
"fcm_dpo/margin": 303.7243347167969,
"fcm_dpo/q_t": 0.4097488522529602,
"grad_norm": 31.81182861328125,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.718780517578125,
"logits/rejected": -0.7090387344360352,
"logps/chosen": -525.4910888671875,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -858.7434692382812,
"loss": 1.1065,
"margin_dpo/margin_mean": 303.7243347167969,
"margin_dpo/margin_std": 445.0675964355469,
"step": 442
},
{
"KL/chosen_KL_mean": -530.4336547851562,
"KL/mean": -621.8048706054688,
"KL/rejected_KL_mean": -713.1761474609375,
"KL/std": 391.05328369140625,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0013202856061980128,
"fcm_dpo/delta": 0.0109033128246665,
"fcm_dpo/margin": 182.74246215820312,
"fcm_dpo/q_t": 0.4463666081428528,
"grad_norm": 38.57601547241211,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.724666953086853,
"logits/rejected": -0.701187014579773,
"logps/chosen": -601.7066650390625,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -799.8560791015625,
"loss": 1.2554,
"margin_dpo/margin_mean": 182.74249267578125,
"margin_dpo/margin_std": 479.74188232421875,
"step": 443
},
{
"KL/chosen_KL_mean": -528.6512451171875,
"KL/mean": -687.4662475585938,
"KL/rejected_KL_mean": -846.2813110351562,
"KL/std": 420.274169921875,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0013028013054281473,
"fcm_dpo/delta": -0.06701706349849701,
"fcm_dpo/margin": 317.630126953125,
"fcm_dpo/q_t": 0.40774786472320557,
"grad_norm": 30.262189865112305,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.6718389391899109,
"logits/rejected": -0.6785389184951782,
"logps/chosen": -585.8648681640625,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -943.5361938476562,
"loss": 1.1052,
"margin_dpo/margin_mean": 317.630126953125,
"margin_dpo/margin_std": 487.72283935546875,
"step": 444
},
{
"KL/chosen_KL_mean": -456.7267761230469,
"KL/mean": -602.949951171875,
"KL/rejected_KL_mean": -749.173095703125,
"KL/std": 321.0992431640625,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0012994688004255295,
"fcm_dpo/delta": -0.031238578259944916,
"fcm_dpo/margin": 292.4462890625,
"fcm_dpo/q_t": 0.41294771432876587,
"grad_norm": 30.900972366333008,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.6935943365097046,
"logits/rejected": -0.6810604333877563,
"logps/chosen": -524.026611328125,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -841.855712890625,
"loss": 1.1085,
"margin_dpo/margin_mean": 292.4463195800781,
"margin_dpo/margin_std": 402.79986572265625,
"step": 445
},
{
"KL/chosen_KL_mean": -402.2188720703125,
"KL/mean": -560.820556640625,
"KL/rejected_KL_mean": -719.4222412109375,
"KL/std": 337.19036865234375,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.0012904771137982607,
"fcm_dpo/delta": -0.062246501445770264,
"fcm_dpo/margin": 317.2033386230469,
"fcm_dpo/q_t": 0.4033682346343994,
"grad_norm": 34.864933013916016,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.6820211410522461,
"logits/rejected": -0.6916550397872925,
"logps/chosen": -461.3173522949219,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -820.6864013671875,
"loss": 1.0721,
"margin_dpo/margin_mean": 317.2033386230469,
"margin_dpo/margin_std": 355.0349426269531,
"step": 446
},
{
"KL/chosen_KL_mean": -425.578857421875,
"KL/mean": -594.2220458984375,
"KL/rejected_KL_mean": -762.8651733398438,
"KL/std": 410.32159423828125,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0012605185620486736,
"fcm_dpo/delta": -0.07881560921669006,
"fcm_dpo/margin": 337.28631591796875,
"fcm_dpo/q_t": 0.4043826460838318,
"grad_norm": 26.480072021484375,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.7632917761802673,
"logits/rejected": -0.7825505137443542,
"logps/chosen": -481.6541748046875,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -861.5599365234375,
"loss": 1.079,
"margin_dpo/margin_mean": 337.28631591796875,
"margin_dpo/margin_std": 460.5667419433594,
"step": 447
},
{
"KL/chosen_KL_mean": -467.5102844238281,
"KL/mean": -623.7568359375,
"KL/rejected_KL_mean": -780.00341796875,
"KL/std": 348.59857177734375,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.001254360075108707,
"fcm_dpo/delta": -0.04375584051012993,
"fcm_dpo/margin": 312.4931335449219,
"fcm_dpo/q_t": 0.40883946418762207,
"grad_norm": 32.5056266784668,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.6593608856201172,
"logits/rejected": -0.665467381477356,
"logps/chosen": -527.51416015625,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -882.26806640625,
"loss": 1.0861,
"margin_dpo/margin_mean": 312.4931335449219,
"margin_dpo/margin_std": 379.61871337890625,
"step": 448
},
{
"KL/chosen_KL_mean": -483.6077575683594,
"KL/mean": -639.1844482421875,
"KL/rejected_KL_mean": -794.76123046875,
"KL/std": 414.4624938964844,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.001239268109202385,
"fcm_dpo/delta": -0.03710303083062172,
"fcm_dpo/margin": 311.15350341796875,
"fcm_dpo/q_t": 0.41368383169174194,
"grad_norm": 29.851238250732422,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.7158821821212769,
"logits/rejected": -0.7080788016319275,
"logps/chosen": -551.1343383789062,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -883.358154296875,
"loss": 1.1186,
"margin_dpo/margin_mean": 311.15350341796875,
"margin_dpo/margin_std": 484.8404846191406,
"step": 449
},
{
"KL/chosen_KL_mean": -386.138671875,
"KL/mean": -558.7864990234375,
"KL/rejected_KL_mean": -731.434326171875,
"KL/std": 357.3294982910156,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.001226372318342328,
"fcm_dpo/delta": -0.07663866132497787,
"fcm_dpo/margin": 345.29571533203125,
"fcm_dpo/q_t": 0.3992771804332733,
"grad_norm": 48.51375198364258,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.7433021068572998,
"logits/rejected": -0.7565724849700928,
"logps/chosen": -443.24676513671875,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -834.1893310546875,
"loss": 1.0505,
"margin_dpo/margin_mean": 345.29571533203125,
"margin_dpo/margin_std": 349.22174072265625,
"step": 450
},
{
"KL/chosen_KL_mean": -553.4360961914062,
"KL/mean": -673.3494873046875,
"KL/rejected_KL_mean": -793.2628784179688,
"KL/std": 449.323486328125,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0012323290575295687,
"fcm_dpo/delta": 0.05590929836034775,
"fcm_dpo/margin": 239.82675170898438,
"fcm_dpo/q_t": 0.4289303421974182,
"grad_norm": 36.48598861694336,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.7902913689613342,
"logits/rejected": -0.7614456415176392,
"logps/chosen": -611.9049072265625,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -866.1922607421875,
"loss": 1.2143,
"margin_dpo/margin_mean": 239.8267364501953,
"margin_dpo/margin_std": 536.7329711914062,
"step": 451
},
{
"KL/chosen_KL_mean": -394.72314453125,
"KL/mean": -541.702392578125,
"KL/rejected_KL_mean": -688.681640625,
"KL/std": 334.7508239746094,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0012294357875362039,
"fcm_dpo/delta": -0.01188319455832243,
"fcm_dpo/margin": 293.95849609375,
"fcm_dpo/q_t": 0.4138975441455841,
"grad_norm": 26.67286491394043,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.6999271512031555,
"logits/rejected": -0.7178478240966797,
"logps/chosen": -445.11370849609375,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -786.4530639648438,
"loss": 1.0979,
"margin_dpo/margin_mean": 293.95849609375,
"margin_dpo/margin_std": 333.3941955566406,
"step": 452
},
{
"KL/chosen_KL_mean": -457.27532958984375,
"KL/mean": -610.49267578125,
"KL/rejected_KL_mean": -763.7099609375,
"KL/std": 360.0783386230469,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0012230616994202137,
"fcm_dpo/delta": -0.025895066559314728,
"fcm_dpo/margin": 306.43463134765625,
"fcm_dpo/q_t": 0.41480809450149536,
"grad_norm": 35.29240036010742,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.7407764792442322,
"logits/rejected": -0.7278668880462646,
"logps/chosen": -514.9901733398438,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -845.9173583984375,
"loss": 1.1113,
"margin_dpo/margin_mean": 306.43463134765625,
"margin_dpo/margin_std": 436.13421630859375,
"step": 453
},
{
"KL/chosen_KL_mean": -560.2719116210938,
"KL/mean": -723.659912109375,
"KL/rejected_KL_mean": -887.0479736328125,
"KL/std": 408.50616455078125,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0012127361260354519,
"fcm_dpo/delta": -0.04838482663035393,
"fcm_dpo/margin": 326.7760009765625,
"fcm_dpo/q_t": 0.4091429114341736,
"grad_norm": 25.13304328918457,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.7615550756454468,
"logits/rejected": -0.7526893019676208,
"logps/chosen": -621.217529296875,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -971.9987182617188,
"loss": 1.0997,
"margin_dpo/margin_mean": 326.7760314941406,
"margin_dpo/margin_std": 461.345947265625,
"step": 454
},
{
"KL/chosen_KL_mean": -504.2913818359375,
"KL/mean": -699.232421875,
"KL/rejected_KL_mean": -894.17333984375,
"KL/std": 421.8465576171875,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.001196699682623148,
"fcm_dpo/delta": -0.12219969183206558,
"fcm_dpo/margin": 389.8819580078125,
"fcm_dpo/q_t": 0.39322376251220703,
"grad_norm": 33.46556854248047,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.7414263486862183,
"logits/rejected": -0.7670072317123413,
"logps/chosen": -549.1781005859375,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -1009.4747924804688,
"loss": 1.0514,
"margin_dpo/margin_mean": 389.8819580078125,
"margin_dpo/margin_std": 477.7015380859375,
"step": 455
},
{
"KL/chosen_KL_mean": -507.195556640625,
"KL/mean": -697.402587890625,
"KL/rejected_KL_mean": -887.6096801757812,
"KL/std": 429.1546630859375,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0011668999213725328,
"fcm_dpo/delta": -0.09819206595420837,
"fcm_dpo/margin": 380.4140625,
"fcm_dpo/q_t": 0.39691370725631714,
"grad_norm": 38.52052688598633,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.7470618486404419,
"logits/rejected": -0.7721505165100098,
"logps/chosen": -564.2323608398438,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -992.8275146484375,
"loss": 1.0411,
"margin_dpo/margin_mean": 380.4140625,
"margin_dpo/margin_std": 404.1457214355469,
"step": 456
},
{
"KL/chosen_KL_mean": -478.6456298828125,
"KL/mean": -666.3101806640625,
"KL/rejected_KL_mean": -853.9747314453125,
"KL/std": 396.79046630859375,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.00114365853369236,
"fcm_dpo/delta": -0.0830128863453865,
"fcm_dpo/margin": 375.3291015625,
"fcm_dpo/q_t": 0.3985850214958191,
"grad_norm": 33.69874572753906,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.7634145617485046,
"logits/rejected": -0.7695783376693726,
"logps/chosen": -532.88818359375,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -939.0842895507812,
"loss": 1.0498,
"margin_dpo/margin_mean": 375.3291015625,
"margin_dpo/margin_std": 390.97637939453125,
"step": 457
},
{
"KL/chosen_KL_mean": -554.390869140625,
"KL/mean": -707.5039672851562,
"KL/rejected_KL_mean": -860.6171264648438,
"KL/std": 387.835693359375,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.0011345113161951303,
"fcm_dpo/delta": 0.001986939460039139,
"fcm_dpo/margin": 306.226318359375,
"fcm_dpo/q_t": 0.42130082845687866,
"grad_norm": 35.67972183227539,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.8082045316696167,
"logits/rejected": -0.8004375100135803,
"logps/chosen": -609.7996826171875,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -958.3004150390625,
"loss": 1.1318,
"margin_dpo/margin_mean": 306.226318359375,
"margin_dpo/margin_std": 444.1019287109375,
"step": 458
},
{
"KL/chosen_KL_mean": -591.978515625,
"KL/mean": -779.3623657226562,
"KL/rejected_KL_mean": -966.7462158203125,
"KL/std": 468.7334289550781,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0011250160168856382,
"fcm_dpo/delta": -0.07521242648363113,
"fcm_dpo/margin": 374.7676696777344,
"fcm_dpo/q_t": 0.4042002856731415,
"grad_norm": 38.14002990722656,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.7572959661483765,
"logits/rejected": -0.7787726521492004,
"logps/chosen": -638.5360107421875,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -1052.914794921875,
"loss": 1.0933,
"margin_dpo/margin_mean": 374.7676696777344,
"margin_dpo/margin_std": 536.9511108398438,
"step": 459
},
{
"KL/chosen_KL_mean": -629.0224609375,
"KL/mean": -831.0809936523438,
"KL/rejected_KL_mean": -1033.1395263671875,
"KL/std": 450.43658447265625,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0011082333512604237,
"fcm_dpo/delta": -0.10236979275941849,
"fcm_dpo/margin": 404.1170349121094,
"fcm_dpo/q_t": 0.39786800742149353,
"grad_norm": 28.9811954498291,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.8412591814994812,
"logits/rejected": -0.8749679923057556,
"logps/chosen": -680.6573486328125,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -1137.2587890625,
"loss": 1.055,
"margin_dpo/margin_mean": 404.1170349121094,
"margin_dpo/margin_std": 494.65667724609375,
"step": 460
},
{
"KL/chosen_KL_mean": -658.8389892578125,
"KL/mean": -815.6547241210938,
"KL/rejected_KL_mean": -972.470458984375,
"KL/std": 444.0308837890625,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0011054275091737509,
"fcm_dpo/delta": 0.0032142307609319687,
"fcm_dpo/margin": 313.631591796875,
"fcm_dpo/q_t": 0.42101699113845825,
"grad_norm": 28.833742141723633,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.8508638739585876,
"logits/rejected": -0.8650791049003601,
"logps/chosen": -714.0208740234375,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -1058.9473876953125,
"loss": 1.1359,
"margin_dpo/margin_mean": 313.631591796875,
"margin_dpo/margin_std": 482.70458984375,
"step": 461
},
{
"KL/chosen_KL_mean": -635.347412109375,
"KL/mean": -766.1812744140625,
"KL/rejected_KL_mean": -897.0151977539062,
"KL/std": 428.46905517578125,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.0011110971681773663,
"fcm_dpo/delta": 0.06107814237475395,
"fcm_dpo/margin": 261.667724609375,
"fcm_dpo/q_t": 0.43434569239616394,
"grad_norm": 37.025150299072266,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.8966616988182068,
"logits/rejected": -0.8862354755401611,
"logps/chosen": -705.2755126953125,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -975.8563232421875,
"loss": 1.1858,
"margin_dpo/margin_mean": 261.667724609375,
"margin_dpo/margin_std": 488.41827392578125,
"step": 462
},
{
"KL/chosen_KL_mean": -617.084716796875,
"KL/mean": -794.1759033203125,
"KL/rejected_KL_mean": -971.2671508789062,
"KL/std": 443.2431640625,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.0011113372165709734,
"fcm_dpo/delta": -0.045437999069690704,
"fcm_dpo/margin": 354.18243408203125,
"fcm_dpo/q_t": 0.4097776710987091,
"grad_norm": 36.853878021240234,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.8798764944076538,
"logits/rejected": -0.893752932548523,
"logps/chosen": -672.3590698242188,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -1060.2921142578125,
"loss": 1.101,
"margin_dpo/margin_mean": 354.18243408203125,
"margin_dpo/margin_std": 491.94415283203125,
"step": 463
},
{
"KL/chosen_KL_mean": -604.09423828125,
"KL/mean": -862.4444580078125,
"KL/rejected_KL_mean": -1120.794677734375,
"KL/std": 517.349853515625,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0010644260328263044,
"fcm_dpo/delta": -0.2137536257505417,
"fcm_dpo/margin": 516.7005615234375,
"fcm_dpo/q_t": 0.37335121631622314,
"grad_norm": 34.50212478637695,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.8490620851516724,
"logits/rejected": -0.9159963726997375,
"logps/chosen": -655.0065307617188,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -1223.2841796875,
"loss": 0.9802,
"margin_dpo/margin_mean": 516.7005615234375,
"margin_dpo/margin_std": 523.5531005859375,
"step": 464
},
{
"KL/chosen_KL_mean": -608.3139038085938,
"KL/mean": -818.8243408203125,
"KL/rejected_KL_mean": -1029.334716796875,
"KL/std": 525.769775390625,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0010484305676072836,
"fcm_dpo/delta": -0.09556633979082108,
"fcm_dpo/margin": 421.0207824707031,
"fcm_dpo/q_t": 0.39575302600860596,
"grad_norm": 34.12406539916992,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.8353086709976196,
"logits/rejected": -0.8615255951881409,
"logps/chosen": -668.4307861328125,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -1143.28076171875,
"loss": 1.0727,
"margin_dpo/margin_mean": 421.020751953125,
"margin_dpo/margin_std": 551.701416015625,
"step": 465
},
{
"KL/chosen_KL_mean": -665.450439453125,
"KL/mean": -846.2392578125,
"KL/rejected_KL_mean": -1027.0279541015625,
"KL/std": 486.2896728515625,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0010309141362085938,
"fcm_dpo/delta": -0.02415800467133522,
"fcm_dpo/margin": 361.57757568359375,
"fcm_dpo/q_t": 0.41319042444229126,
"grad_norm": 30.475038528442383,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.8484971523284912,
"logits/rejected": -0.850833535194397,
"logps/chosen": -718.371337890625,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -1117.343505859375,
"loss": 1.1194,
"margin_dpo/margin_mean": 361.57757568359375,
"margin_dpo/margin_std": 526.1229248046875,
"step": 466
},
{
"KL/chosen_KL_mean": -806.384033203125,
"KL/mean": -1001.665283203125,
"KL/rejected_KL_mean": -1196.946533203125,
"KL/std": 618.6810913085938,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0010187213774770498,
"fcm_dpo/delta": -0.05139738321304321,
"fcm_dpo/margin": 390.5625,
"fcm_dpo/q_t": 0.412168025970459,
"grad_norm": 39.5161018371582,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.9231687784194946,
"logits/rejected": -0.9177477359771729,
"logps/chosen": -885.099853515625,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -1299.806640625,
"loss": 1.153,
"margin_dpo/margin_mean": 390.5625,
"margin_dpo/margin_std": 716.0555419921875,
"step": 467
},
{
"KL/chosen_KL_mean": -630.0722045898438,
"KL/mean": -863.9874877929688,
"KL/rejected_KL_mean": -1097.9027099609375,
"KL/std": 546.4240112304688,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.001004476216621697,
"fcm_dpo/delta": -0.12587127089500427,
"fcm_dpo/margin": 467.8305358886719,
"fcm_dpo/q_t": 0.39484673738479614,
"grad_norm": 56.975738525390625,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.8668639659881592,
"logits/rejected": -0.8785967826843262,
"logps/chosen": -700.007568359375,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -1198.9315185546875,
"loss": 1.052,
"margin_dpo/margin_mean": 467.83050537109375,
"margin_dpo/margin_std": 610.6904296875,
"step": 468
},
{
"KL/chosen_KL_mean": -658.3848876953125,
"KL/mean": -892.792724609375,
"KL/rejected_KL_mean": -1127.20068359375,
"KL/std": 478.67694091796875,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.0009764886926859617,
"fcm_dpo/delta": -0.11406978219747543,
"fcm_dpo/margin": 468.81585693359375,
"fcm_dpo/q_t": 0.3934182822704315,
"grad_norm": 53.98129653930664,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.8874689340591431,
"logits/rejected": -0.9010331630706787,
"logps/chosen": -726.509521484375,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -1231.987060546875,
"loss": 1.0452,
"margin_dpo/margin_mean": 468.81585693359375,
"margin_dpo/margin_std": 534.7791748046875,
"step": 469
},
{
"KL/chosen_KL_mean": -561.26513671875,
"KL/mean": -760.3869018554688,
"KL/rejected_KL_mean": -959.5086669921875,
"KL/std": 498.2298278808594,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0009724711999297142,
"fcm_dpo/delta": -0.03880758211016655,
"fcm_dpo/margin": 398.2434387207031,
"fcm_dpo/q_t": 0.41007864475250244,
"grad_norm": 30.534019470214844,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.8203660249710083,
"logits/rejected": -0.8314099311828613,
"logps/chosen": -605.05712890625,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -1042.21142578125,
"loss": 1.1043,
"margin_dpo/margin_mean": 398.2434387207031,
"margin_dpo/margin_std": 553.9703979492188,
"step": 470
},
{
"KL/chosen_KL_mean": -689.858642578125,
"KL/mean": -864.8088989257812,
"KL/rejected_KL_mean": -1039.7591552734375,
"KL/std": 511.3665771484375,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0009652703301981091,
"fcm_dpo/delta": 0.01225661113858223,
"fcm_dpo/margin": 349.9005432128906,
"fcm_dpo/q_t": 0.42378807067871094,
"grad_norm": 45.94660568237305,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.903639554977417,
"logits/rejected": -0.9123305082321167,
"logps/chosen": -753.1981201171875,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -1123.36962890625,
"loss": 1.1473,
"margin_dpo/margin_mean": 349.90057373046875,
"margin_dpo/margin_std": 566.9776000976562,
"step": 471
},
{
"KL/chosen_KL_mean": -701.3533935546875,
"KL/mean": -908.082763671875,
"KL/rejected_KL_mean": -1114.812255859375,
"KL/std": 622.212158203125,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.0009603024809621274,
"fcm_dpo/delta": -0.04937269911170006,
"fcm_dpo/margin": 413.4587707519531,
"fcm_dpo/q_t": 0.4133405089378357,
"grad_norm": 30.506114959716797,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.9473692178726196,
"logits/rejected": -0.9324079751968384,
"logps/chosen": -785.01953125,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -1232.021484375,
"loss": 1.1316,
"margin_dpo/margin_mean": 413.458740234375,
"margin_dpo/margin_std": 694.000732421875,
"step": 472
},
{
"KL/chosen_KL_mean": -834.3516845703125,
"KL/mean": -932.49462890625,
"KL/rejected_KL_mean": -1030.637451171875,
"KL/std": 639.5214233398438,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0009610364213585854,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 196.28573608398438,
"fcm_dpo/q_t": 0.4586009979248047,
"grad_norm": 95.01997375488281,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.9266204833984375,
"logits/rejected": -0.9034590721130371,
"logps/chosen": -897.8486938476562,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -1111.7840576171875,
"loss": 1.3466,
"margin_dpo/margin_mean": 196.28573608398438,
"margin_dpo/margin_std": 829.6607666015625,
"step": 473
},
{
"KL/chosen_KL_mean": -665.4111328125,
"KL/mean": -841.2649536132812,
"KL/rejected_KL_mean": -1017.1187744140625,
"KL/std": 553.4940795898438,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.0009541058680042624,
"fcm_dpo/delta": -0.07264073938131332,
"fcm_dpo/margin": 351.7076416015625,
"fcm_dpo/q_t": 0.4222460687160492,
"grad_norm": 34.34967041015625,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.8338485956192017,
"logits/rejected": -0.8360385894775391,
"logps/chosen": -718.0230712890625,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -1107.19921875,
"loss": 1.1678,
"margin_dpo/margin_mean": 351.7076416015625,
"margin_dpo/margin_std": 621.35986328125,
"step": 474
},
{
"KL/chosen_KL_mean": -480.55462646484375,
"KL/mean": -702.83447265625,
"KL/rejected_KL_mean": -925.1143798828125,
"KL/std": 437.4728088378906,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.0009357599192298949,
"fcm_dpo/delta": -0.06906232237815857,
"fcm_dpo/margin": 444.5597229003906,
"fcm_dpo/q_t": 0.40210431814193726,
"grad_norm": 49.64398956298828,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.8846492767333984,
"logits/rejected": -0.911199688911438,
"logps/chosen": -523.0498046875,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -1015.17724609375,
"loss": 1.0605,
"margin_dpo/margin_mean": 444.5597229003906,
"margin_dpo/margin_std": 480.71630859375,
"step": 475
},
{
"KL/chosen_KL_mean": -625.072509765625,
"KL/mean": -801.80712890625,
"KL/rejected_KL_mean": -978.5418090820312,
"KL/std": 506.44488525390625,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.0009366124868392944,
"fcm_dpo/delta": 0.019588805735111237,
"fcm_dpo/margin": 353.46929931640625,
"fcm_dpo/q_t": 0.4231525659561157,
"grad_norm": 63.98448181152344,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.8908638954162598,
"logits/rejected": -0.9078314304351807,
"logps/chosen": -668.0218505859375,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -1052.2520751953125,
"loss": 1.132,
"margin_dpo/margin_mean": 353.46929931640625,
"margin_dpo/margin_std": 492.0439758300781,
"step": 476
},
{
"KL/chosen_KL_mean": -637.2841186523438,
"KL/mean": -817.2775268554688,
"KL/rejected_KL_mean": -997.2708740234375,
"KL/std": 515.0693359375,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0009418263216502964,
"fcm_dpo/delta": 0.011177122592926025,
"fcm_dpo/margin": 359.98681640625,
"fcm_dpo/q_t": 0.42166703939437866,
"grad_norm": 36.05009460449219,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.8980028629302979,
"logits/rejected": -0.8670873641967773,
"logps/chosen": -708.0567626953125,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -1073.408203125,
"loss": 1.1558,
"margin_dpo/margin_mean": 359.9867858886719,
"margin_dpo/margin_std": 619.1052856445312,
"step": 477
},
{
"KL/chosen_KL_mean": -511.648193359375,
"KL/mean": -720.3424072265625,
"KL/rejected_KL_mean": -929.03662109375,
"KL/std": 461.0159912109375,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.0009357367525808513,
"fcm_dpo/delta": -0.04220425337553024,
"fcm_dpo/margin": 417.388427734375,
"fcm_dpo/q_t": 0.4093322157859802,
"grad_norm": 60.29072189331055,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.7977697849273682,
"logits/rejected": -0.8219680190086365,
"logps/chosen": -553.0887451171875,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -1014.3985595703125,
"loss": 1.0931,
"margin_dpo/margin_mean": 417.388427734375,
"margin_dpo/margin_std": 540.1747436523438,
"step": 478
},
{
"KL/chosen_KL_mean": -685.029541015625,
"KL/mean": -879.80517578125,
"KL/rejected_KL_mean": -1074.580810546875,
"KL/std": 583.0810546875,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.000935424177441746,
"fcm_dpo/delta": -0.015560656785964966,
"fcm_dpo/margin": 389.55133056640625,
"fcm_dpo/q_t": 0.4196696877479553,
"grad_norm": 31.228153228759766,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.8409562706947327,
"logits/rejected": -0.8747595548629761,
"logps/chosen": -738.9374389648438,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -1169.697265625,
"loss": 1.1435,
"margin_dpo/margin_mean": 389.55126953125,
"margin_dpo/margin_std": 670.361328125,
"step": 479
},
{
"KL/chosen_KL_mean": -805.6644897460938,
"KL/mean": -962.97802734375,
"KL/rejected_KL_mean": -1120.29150390625,
"KL/std": 531.8762817382812,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0009248163551092148,
"fcm_dpo/delta": -0.046877436339855194,
"fcm_dpo/margin": 314.626953125,
"fcm_dpo/q_t": 0.43354925513267517,
"grad_norm": 39.37574768066406,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.8963419198989868,
"logits/rejected": -0.898643970489502,
"logps/chosen": -864.34716796875,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -1203.223876953125,
"loss": 1.1908,
"margin_dpo/margin_mean": 314.626953125,
"margin_dpo/margin_std": 573.584228515625,
"step": 480
},
{
"KL/chosen_KL_mean": -680.197509765625,
"KL/mean": -925.4300537109375,
"KL/rejected_KL_mean": -1170.6624755859375,
"KL/std": 552.0270385742188,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.0009076240239664912,
"fcm_dpo/delta": -0.09952923655509949,
"fcm_dpo/margin": 490.46502685546875,
"fcm_dpo/q_t": 0.397752046585083,
"grad_norm": 46.874427795410156,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.9476274251937866,
"logits/rejected": -0.9800101518630981,
"logps/chosen": -735.1617431640625,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -1263.0830078125,
"loss": 1.0567,
"margin_dpo/margin_mean": 490.46502685546875,
"margin_dpo/margin_std": 596.1909790039062,
"step": 481
},
{
"KL/chosen_KL_mean": -753.5008544921875,
"KL/mean": -868.485595703125,
"KL/rejected_KL_mean": -983.4703369140625,
"KL/std": 577.3681640625,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0009085052879527211,
"fcm_dpo/delta": 0.06787097454071045,
"fcm_dpo/margin": 229.9693603515625,
"fcm_dpo/q_t": 0.4502810835838318,
"grad_norm": 55.56328201293945,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.8725358843803406,
"logits/rejected": -0.8671541213989258,
"logps/chosen": -821.0543212890625,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -1071.059814453125,
"loss": 1.2717,
"margin_dpo/margin_mean": 229.9693603515625,
"margin_dpo/margin_std": 657.5565185546875,
"step": 482
},
{
"KL/chosen_KL_mean": -682.5269165039062,
"KL/mean": -933.0989990234375,
"KL/rejected_KL_mean": -1183.6710205078125,
"KL/std": 570.397216796875,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0008986013708636165,
"fcm_dpo/delta": -0.10536154359579086,
"fcm_dpo/margin": 501.1441345214844,
"fcm_dpo/q_t": 0.3984888792037964,
"grad_norm": 29.073741912841797,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.9419503808021545,
"logits/rejected": -0.9568164348602295,
"logps/chosen": -745.856689453125,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -1279.4580078125,
"loss": 1.0585,
"margin_dpo/margin_mean": 501.14410400390625,
"margin_dpo/margin_std": 632.0308227539062,
"step": 483
},
{
"KL/chosen_KL_mean": -616.1393432617188,
"KL/mean": -855.525634765625,
"KL/rejected_KL_mean": -1094.911865234375,
"KL/std": 583.80126953125,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0008886742289178073,
"fcm_dpo/delta": -0.07883388549089432,
"fcm_dpo/margin": 478.7725830078125,
"fcm_dpo/q_t": 0.40235435962677,
"grad_norm": 42.01279067993164,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.9948852062225342,
"logits/rejected": -1.0258457660675049,
"logps/chosen": -675.2774658203125,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -1179.2833251953125,
"loss": 1.0683,
"margin_dpo/margin_mean": 478.7725830078125,
"margin_dpo/margin_std": 582.26611328125,
"step": 484
},
{
"KL/chosen_KL_mean": -638.5044555664062,
"KL/mean": -869.9237060546875,
"KL/rejected_KL_mean": -1101.343017578125,
"KL/std": 587.0706176757812,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0008760282071307302,
"fcm_dpo/delta": -0.05777723342180252,
"fcm_dpo/margin": 462.83856201171875,
"fcm_dpo/q_t": 0.40835797786712646,
"grad_norm": 32.41249084472656,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.9031185507774353,
"logits/rejected": -0.9280800223350525,
"logps/chosen": -697.35400390625,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -1204.70703125,
"loss": 1.0999,
"margin_dpo/margin_mean": 462.83856201171875,
"margin_dpo/margin_std": 665.9918823242188,
"step": 485
},
{
"KL/chosen_KL_mean": -735.9765625,
"KL/mean": -987.707763671875,
"KL/rejected_KL_mean": -1239.43896484375,
"KL/std": 638.144287109375,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.0008581732981838286,
"fcm_dpo/delta": -0.0861148089170456,
"fcm_dpo/margin": 503.46240234375,
"fcm_dpo/q_t": 0.40256860852241516,
"grad_norm": 36.858909606933594,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -1.0510118007659912,
"logits/rejected": -1.0759706497192383,
"logps/chosen": -791.2362060546875,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -1331.578369140625,
"loss": 1.1,
"margin_dpo/margin_mean": 503.4623718261719,
"margin_dpo/margin_std": 770.2353515625,
"step": 486
},
{
"KL/chosen_KL_mean": -800.8622436523438,
"KL/mean": -982.6766357421875,
"KL/rejected_KL_mean": -1164.4910888671875,
"KL/std": 589.8531494140625,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0008646865608170629,
"fcm_dpo/delta": 0.03625814616680145,
"fcm_dpo/margin": 363.62884521484375,
"fcm_dpo/q_t": 0.42635592818260193,
"grad_norm": 31.64628791809082,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -1.0411834716796875,
"logits/rejected": -1.0501723289489746,
"logps/chosen": -853.925537109375,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -1256.909912109375,
"loss": 1.1617,
"margin_dpo/margin_mean": 363.62884521484375,
"margin_dpo/margin_std": 604.9837646484375,
"step": 487
},
{
"KL/chosen_KL_mean": -663.748046875,
"KL/mean": -853.4315185546875,
"KL/rejected_KL_mean": -1043.114990234375,
"KL/std": 536.4071044921875,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0008684194181114435,
"fcm_dpo/delta": 0.020952310413122177,
"fcm_dpo/margin": 379.3669738769531,
"fcm_dpo/q_t": 0.423104465007782,
"grad_norm": 31.610628128051758,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.9978197813034058,
"logits/rejected": -1.009468913078308,
"logps/chosen": -715.9761962890625,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -1127.12158203125,
"loss": 1.1324,
"margin_dpo/margin_mean": 379.36700439453125,
"margin_dpo/margin_std": 518.4456787109375,
"step": 488
},
{
"KL/chosen_KL_mean": -616.62109375,
"KL/mean": -808.0659790039062,
"KL/rejected_KL_mean": -999.5108642578125,
"KL/std": 516.0118408203125,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0008714280556887388,
"fcm_dpo/delta": 0.016632717102766037,
"fcm_dpo/margin": 382.88970947265625,
"fcm_dpo/q_t": 0.4236697554588318,
"grad_norm": 24.073930740356445,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.9905928373336792,
"logits/rejected": -0.991652250289917,
"logps/chosen": -672.6107177734375,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -1078.908935546875,
"loss": 1.1387,
"margin_dpo/margin_mean": 382.8896484375,
"margin_dpo/margin_std": 572.0997314453125,
"step": 489
},
{
"KL/chosen_KL_mean": -702.3668823242188,
"KL/mean": -895.4848022460938,
"KL/rejected_KL_mean": -1088.602783203125,
"KL/std": 633.2512817382812,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.0008734596194699407,
"fcm_dpo/delta": 0.01288134977221489,
"fcm_dpo/margin": 386.2358703613281,
"fcm_dpo/q_t": 0.4240713119506836,
"grad_norm": 39.06137466430664,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -1.0082063674926758,
"logits/rejected": -1.0535714626312256,
"logps/chosen": -754.7332763671875,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -1199.0118408203125,
"loss": 1.1739,
"margin_dpo/margin_mean": 386.2358703613281,
"margin_dpo/margin_std": 737.7401123046875,
"step": 490
},
{
"KL/chosen_KL_mean": -649.2156982421875,
"KL/mean": -784.3421630859375,
"KL/rejected_KL_mean": -919.4685668945312,
"KL/std": 507.2773132324219,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.0008869500597938895,
"fcm_dpo/delta": 0.11293124407529831,
"fcm_dpo/margin": 270.2528991699219,
"fcm_dpo/q_t": 0.44444432854652405,
"grad_norm": 29.092905044555664,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.9913074970245361,
"logits/rejected": -0.9897642135620117,
"logps/chosen": -709.3319702148438,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -992.7413330078125,
"loss": 1.2104,
"margin_dpo/margin_mean": 270.2528991699219,
"margin_dpo/margin_std": 513.86279296875,
"step": 491
},
{
"KL/chosen_KL_mean": -699.1614990234375,
"KL/mean": -827.0299072265625,
"KL/rejected_KL_mean": -954.8983154296875,
"KL/std": 548.1110229492188,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0009072008542716503,
"fcm_dpo/delta": 0.12086275219917297,
"fcm_dpo/margin": 255.7367401123047,
"fcm_dpo/q_t": 0.44698652625083923,
"grad_norm": 44.75581359863281,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -1.0495141744613647,
"logits/rejected": -1.0257760286331177,
"logps/chosen": -753.1557006835938,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -1027.557861328125,
"loss": 1.2431,
"margin_dpo/margin_mean": 255.73672485351562,
"margin_dpo/margin_std": 618.5252685546875,
"step": 492
},
{
"KL/chosen_KL_mean": -726.62841796875,
"KL/mean": -883.1748046875,
"KL/rejected_KL_mean": -1039.721435546875,
"KL/std": 605.7178955078125,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0009097631555050611,
"fcm_dpo/delta": -0.019333893433213234,
"fcm_dpo/margin": 313.09295654296875,
"fcm_dpo/q_t": 0.4324401617050171,
"grad_norm": 32.448036193847656,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -1.0681705474853516,
"logits/rejected": -1.0370638370513916,
"logps/chosen": -802.1256103515625,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -1127.04443359375,
"loss": 1.187,
"margin_dpo/margin_mean": 313.09295654296875,
"margin_dpo/margin_std": 570.6629638671875,
"step": 493
},
{
"KL/chosen_KL_mean": -579.3143310546875,
"KL/mean": -823.8994140625,
"KL/rejected_KL_mean": -1068.484619140625,
"KL/std": 546.38134765625,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.0008967167232185602,
"fcm_dpo/delta": -0.09278169274330139,
"fcm_dpo/margin": 489.1702575683594,
"fcm_dpo/q_t": 0.3974645435810089,
"grad_norm": 38.209651947021484,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.9156688451766968,
"logits/rejected": -0.9544728994369507,
"logps/chosen": -620.673583984375,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -1154.575927734375,
"loss": 1.0522,
"margin_dpo/margin_mean": 489.1702880859375,
"margin_dpo/margin_std": 552.1724853515625,
"step": 494
},
{
"KL/chosen_KL_mean": -666.56298828125,
"KL/mean": -872.1007080078125,
"KL/rejected_KL_mean": -1077.638427734375,
"KL/std": 544.0244140625,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0008921578410081565,
"fcm_dpo/delta": -0.017406316474080086,
"fcm_dpo/margin": 411.0754699707031,
"fcm_dpo/q_t": 0.4172486662864685,
"grad_norm": 37.34906768798828,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -1.006931185722351,
"logits/rejected": -1.0146992206573486,
"logps/chosen": -730.0980224609375,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -1169.06298828125,
"loss": 1.1308,
"margin_dpo/margin_mean": 411.07550048828125,
"margin_dpo/margin_std": 657.4510498046875,
"step": 495
},
{
"KL/chosen_KL_mean": -814.9395751953125,
"KL/mean": -919.1443481445312,
"KL/rejected_KL_mean": -1023.34912109375,
"KL/std": 457.9229736328125,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0008899334352463484,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 208.4095458984375,
"fcm_dpo/q_t": 0.45584726333618164,
"grad_norm": 70.3835678100586,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.9732378721237183,
"logits/rejected": -0.9386388063430786,
"logps/chosen": -887.531494140625,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -1107.678466796875,
"loss": 1.2632,
"margin_dpo/margin_mean": 208.4095458984375,
"margin_dpo/margin_std": 522.48583984375,
"step": 496
},
{
"KL/chosen_KL_mean": -753.662353515625,
"KL/mean": -864.582763671875,
"KL/rejected_KL_mean": -975.5030517578125,
"KL/std": 551.1240844726562,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0008933351491577923,
"fcm_dpo/delta": 0.01907578855752945,
"fcm_dpo/margin": 221.8406524658203,
"fcm_dpo/q_t": 0.453231543302536,
"grad_norm": 33.05242919921875,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.9515343904495239,
"logits/rejected": -0.9357314109802246,
"logps/chosen": -812.25634765625,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -1051.7913818359375,
"loss": 1.2493,
"margin_dpo/margin_mean": 221.84066772460938,
"margin_dpo/margin_std": 499.05657958984375,
"step": 497
},
{
"KL/chosen_KL_mean": -736.4669189453125,
"KL/mean": -919.92578125,
"KL/rejected_KL_mean": -1103.384521484375,
"KL/std": 573.03173828125,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0008958862163126469,
"fcm_dpo/delta": 0.02201257273554802,
"fcm_dpo/margin": 366.917724609375,
"fcm_dpo/q_t": 0.4242166578769684,
"grad_norm": 28.690711975097656,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.9741387367248535,
"logits/rejected": -0.9623622894287109,
"logps/chosen": -807.672607421875,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -1187.3426513671875,
"loss": 1.1655,
"margin_dpo/margin_mean": 366.917724609375,
"margin_dpo/margin_std": 657.1267700195312,
"step": 498
},
{
"KL/chosen_KL_mean": -616.2876586914062,
"KL/mean": -847.274169921875,
"KL/rejected_KL_mean": -1078.2607421875,
"KL/std": 576.01513671875,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0008876968640834093,
"fcm_dpo/delta": -0.06278754025697708,
"fcm_dpo/margin": 461.9729309082031,
"fcm_dpo/q_t": 0.4059438109397888,
"grad_norm": 41.73751449584961,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.9784067869186401,
"logits/rejected": -1.0155872106552124,
"logps/chosen": -667.5428466796875,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -1179.33935546875,
"loss": 1.0909,
"margin_dpo/margin_mean": 461.972900390625,
"margin_dpo/margin_std": 637.4395751953125,
"step": 499
},
{
"KL/chosen_KL_mean": -716.6522216796875,
"KL/mean": -908.4356689453125,
"KL/rejected_KL_mean": -1100.218994140625,
"KL/std": 504.1438293457031,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0008866304415278137,
"fcm_dpo/delta": 0.010259722359478474,
"fcm_dpo/margin": 383.56671142578125,
"fcm_dpo/q_t": 0.42117249965667725,
"grad_norm": 27.376264572143555,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.9571424722671509,
"logits/rejected": -0.9902607202529907,
"logps/chosen": -773.6796875,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -1194.1531982421875,
"loss": 1.1487,
"margin_dpo/margin_mean": 383.56671142578125,
"margin_dpo/margin_std": 624.0594482421875,
"step": 500
},
{
"KL/chosen_KL_mean": -627.3432006835938,
"KL/mean": -835.041259765625,
"KL/rejected_KL_mean": -1042.7393798828125,
"KL/std": 518.466796875,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.0008866213611327112,
"fcm_dpo/delta": -0.019011791795492172,
"fcm_dpo/margin": 415.3961486816406,
"fcm_dpo/q_t": 0.415674090385437,
"grad_norm": 28.41216278076172,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.8981431722640991,
"logits/rejected": -0.88893723487854,
"logps/chosen": -681.7027587890625,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -1122.89599609375,
"loss": 1.1279,
"margin_dpo/margin_mean": 415.3961486816406,
"margin_dpo/margin_std": 643.92919921875,
"step": 501
},
{
"KL/chosen_KL_mean": -574.186279296875,
"KL/mean": -764.1837158203125,
"KL/rejected_KL_mean": -954.1812133789062,
"KL/std": 467.6396484375,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0008872783510014415,
"fcm_dpo/delta": 0.013246024027466774,
"fcm_dpo/margin": 379.9948425292969,
"fcm_dpo/q_t": 0.4216458797454834,
"grad_norm": 25.857147216796875,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.854095995426178,
"logits/rejected": -0.8424793481826782,
"logps/chosen": -641.786865234375,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -1037.1300048828125,
"loss": 1.1233,
"margin_dpo/margin_mean": 379.9948425292969,
"margin_dpo/margin_std": 498.8470458984375,
"step": 502
},
{
"KL/chosen_KL_mean": -585.6944580078125,
"KL/mean": -766.4293212890625,
"KL/rejected_KL_mean": -947.1641845703125,
"KL/std": 479.78033447265625,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.0008887014118954539,
"fcm_dpo/delta": 0.029671432450413704,
"fcm_dpo/margin": 361.46978759765625,
"fcm_dpo/q_t": 0.42668020725250244,
"grad_norm": 25.81717872619629,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.8250192403793335,
"logits/rejected": -0.8123034834861755,
"logps/chosen": -640.7728271484375,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -1029.669677734375,
"loss": 1.1462,
"margin_dpo/margin_mean": 361.46978759765625,
"margin_dpo/margin_std": 550.151611328125,
"step": 503
},
{
"KL/chosen_KL_mean": -624.8487548828125,
"KL/mean": -847.6356201171875,
"KL/rejected_KL_mean": -1070.4224853515625,
"KL/std": 563.498779296875,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.0008875064086169004,
"fcm_dpo/delta": -0.047302231192588806,
"fcm_dpo/margin": 445.57373046875,
"fcm_dpo/q_t": 0.409305602312088,
"grad_norm": 32.672576904296875,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.8826575875282288,
"logits/rejected": -0.9081119298934937,
"logps/chosen": -684.8145141601562,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -1174.1845703125,
"loss": 1.1008,
"margin_dpo/margin_mean": 445.57373046875,
"margin_dpo/margin_std": 609.791015625,
"step": 504
},
{
"KL/chosen_KL_mean": -699.5025024414062,
"KL/mean": -810.5694580078125,
"KL/rejected_KL_mean": -921.6364135742188,
"KL/std": 508.30511474609375,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0008896557847037911,
"fcm_dpo/delta": 0.06633193790912628,
"fcm_dpo/margin": 222.13388061523438,
"fcm_dpo/q_t": 0.45340317487716675,
"grad_norm": 50.9668083190918,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.9198693037033081,
"logits/rejected": -0.8950393199920654,
"logps/chosen": -775.6574096679688,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -1010.2218017578125,
"loss": 1.2443,
"margin_dpo/margin_mean": 222.13388061523438,
"margin_dpo/margin_std": 481.71832275390625,
"step": 505
},
{
"KL/chosen_KL_mean": -559.4972534179688,
"KL/mean": -758.4755859375,
"KL/rejected_KL_mean": -957.4539794921875,
"KL/std": 527.7215576171875,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0008918484672904015,
"fcm_dpo/delta": -0.00534551776945591,
"fcm_dpo/margin": 397.9566650390625,
"fcm_dpo/q_t": 0.4175879955291748,
"grad_norm": 34.8582878112793,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.8557130098342896,
"logits/rejected": -0.8738311529159546,
"logps/chosen": -608.457763671875,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -1035.8690185546875,
"loss": 1.1269,
"margin_dpo/margin_mean": 397.9566650390625,
"margin_dpo/margin_std": 581.9273681640625,
"step": 506
},
{
"KL/chosen_KL_mean": -641.53173828125,
"KL/mean": -815.253662109375,
"KL/rejected_KL_mean": -988.9755859375,
"KL/std": 610.8743896484375,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0008992149960249662,
"fcm_dpo/delta": 0.03880162909626961,
"fcm_dpo/margin": 347.44384765625,
"fcm_dpo/q_t": 0.4313592314720154,
"grad_norm": 27.87244987487793,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.8712353706359863,
"logits/rejected": -0.8783326148986816,
"logps/chosen": -708.333251953125,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -1084.3485107421875,
"loss": 1.181,
"margin_dpo/margin_mean": 347.44384765625,
"margin_dpo/margin_std": 672.9422607421875,
"step": 507
},
{
"KL/chosen_KL_mean": -666.12744140625,
"KL/mean": -816.7818603515625,
"KL/rejected_KL_mean": -967.436279296875,
"KL/std": 569.1419677734375,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0009134809952229261,
"fcm_dpo/delta": 0.0766909047961235,
"fcm_dpo/margin": 301.30889892578125,
"fcm_dpo/q_t": 0.43825656175613403,
"grad_norm": 29.6184139251709,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.8354727029800415,
"logits/rejected": -0.8378840684890747,
"logps/chosen": -737.430908203125,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -1063.0638427734375,
"loss": 1.2145,
"margin_dpo/margin_mean": 301.308837890625,
"margin_dpo/margin_std": 656.0669555664062,
"step": 508
},
{
"KL/chosen_KL_mean": -506.78466796875,
"KL/mean": -663.2677001953125,
"KL/rejected_KL_mean": -819.750732421875,
"KL/std": 414.9226989746094,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0009152308921329677,
"fcm_dpo/delta": -0.002988607855513692,
"fcm_dpo/margin": 312.9660949707031,
"fcm_dpo/q_t": 0.43361881375312805,
"grad_norm": 33.5871467590332,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.765209972858429,
"logits/rejected": -0.762215256690979,
"logps/chosen": -570.6036376953125,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -903.0072021484375,
"loss": 1.1739,
"margin_dpo/margin_mean": 312.966064453125,
"margin_dpo/margin_std": 512.6011962890625,
"step": 509
},
{
"KL/chosen_KL_mean": -638.8026123046875,
"KL/mean": -840.986328125,
"KL/rejected_KL_mean": -1043.169921875,
"KL/std": 484.48175048828125,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0009141661575995386,
"fcm_dpo/delta": -0.02046576887369156,
"fcm_dpo/margin": 404.3673095703125,
"fcm_dpo/q_t": 0.4124307632446289,
"grad_norm": 39.50413513183594,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.783023476600647,
"logits/rejected": -0.8079743385314941,
"logps/chosen": -690.6810913085938,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -1145.93505859375,
"loss": 1.095,
"margin_dpo/margin_mean": 404.3673095703125,
"margin_dpo/margin_std": 470.9833984375,
"step": 510
},
{
"KL/chosen_KL_mean": -606.0228271484375,
"KL/mean": -764.6362915039062,
"KL/rejected_KL_mean": -923.249755859375,
"KL/std": 534.836669921875,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.0009196955943480134,
"fcm_dpo/delta": 0.05997824668884277,
"fcm_dpo/margin": 317.2269287109375,
"fcm_dpo/q_t": 0.433167427778244,
"grad_norm": 34.168601989746094,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.8136961460113525,
"logits/rejected": -0.8219088315963745,
"logps/chosen": -666.260986328125,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -1016.1065063476562,
"loss": 1.1989,
"margin_dpo/margin_mean": 317.2269287109375,
"margin_dpo/margin_std": 654.623046875,
"step": 511
},
{
"KL/chosen_KL_mean": -481.8533935546875,
"KL/mean": -713.4747314453125,
"KL/rejected_KL_mean": -945.095947265625,
"KL/std": 487.7607116699219,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.0009103405755013227,
"fcm_dpo/delta": -0.07506463676691055,
"fcm_dpo/margin": 463.2425537109375,
"fcm_dpo/q_t": 0.4011862874031067,
"grad_norm": 54.72198486328125,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.812955915927887,
"logits/rejected": -0.8373688459396362,
"logps/chosen": -536.7589111328125,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -1026.9718017578125,
"loss": 1.0568,
"margin_dpo/margin_mean": 463.2425537109375,
"margin_dpo/margin_std": 502.77996826171875,
"step": 512
},
{
"KL/chosen_KL_mean": -666.6439819335938,
"KL/mean": -828.3573608398438,
"KL/rejected_KL_mean": -990.07080078125,
"KL/std": 492.90380859375,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.0009010280482470989,
"fcm_dpo/delta": -0.041153233498334885,
"fcm_dpo/margin": 323.4268493652344,
"fcm_dpo/q_t": 0.43294602632522583,
"grad_norm": 58.28951644897461,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.8997926712036133,
"logits/rejected": -0.8882004022598267,
"logps/chosen": -731.5604248046875,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -1066.13330078125,
"loss": 1.1844,
"margin_dpo/margin_mean": 323.4268493652344,
"margin_dpo/margin_std": 577.8350830078125,
"step": 513
},
{
"KL/chosen_KL_mean": -695.1303100585938,
"KL/mean": -851.9722900390625,
"KL/rejected_KL_mean": -1008.8141479492188,
"KL/std": 538.831298828125,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0009115079883486032,
"fcm_dpo/delta": 0.06578437238931656,
"fcm_dpo/margin": 313.683837890625,
"fcm_dpo/q_t": 0.4355640113353729,
"grad_norm": 39.681461334228516,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.8688774108886719,
"logits/rejected": -0.8621048927307129,
"logps/chosen": -769.35986328125,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -1087.759765625,
"loss": 1.1896,
"margin_dpo/margin_mean": 313.683837890625,
"margin_dpo/margin_std": 597.1491088867188,
"step": 514
},
{
"KL/chosen_KL_mean": -576.4310302734375,
"KL/mean": -700.958251953125,
"KL/rejected_KL_mean": -825.4854736328125,
"KL/std": 449.62347412109375,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0009299020748585463,
"fcm_dpo/delta": 0.12117768824100494,
"fcm_dpo/margin": 249.0543975830078,
"fcm_dpo/q_t": 0.444853812456131,
"grad_norm": 43.30710983276367,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.8548616170883179,
"logits/rejected": -0.8692120313644409,
"logps/chosen": -626.8326416015625,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -912.5831909179688,
"loss": 1.2272,
"margin_dpo/margin_mean": 249.05441284179688,
"margin_dpo/margin_std": 538.2990112304688,
"step": 515
},
{
"KL/chosen_KL_mean": -610.23974609375,
"KL/mean": -794.4168090820312,
"KL/rejected_KL_mean": -978.5938110351562,
"KL/std": 516.6522216796875,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.000938748475164175,
"fcm_dpo/delta": 0.0041639544069767,
"fcm_dpo/margin": 368.35406494140625,
"fcm_dpo/q_t": 0.41812512278556824,
"grad_norm": 38.28722381591797,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.9362125396728516,
"logits/rejected": -0.9661528468132019,
"logps/chosen": -674.8893432617188,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -1090.316162109375,
"loss": 1.128,
"margin_dpo/margin_mean": 368.3540344238281,
"margin_dpo/margin_std": 521.5465698242188,
"step": 516
},
{
"KL/chosen_KL_mean": -608.512451171875,
"KL/mean": -828.2478637695312,
"KL/rejected_KL_mean": -1047.983154296875,
"KL/std": 547.919677734375,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0009262575185857713,
"fcm_dpo/delta": -0.05964816361665726,
"fcm_dpo/margin": 439.4707946777344,
"fcm_dpo/q_t": 0.4080393314361572,
"grad_norm": 26.922121047973633,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.8890372514724731,
"logits/rejected": -0.8907492160797119,
"logps/chosen": -669.426025390625,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -1137.0662841796875,
"loss": 1.0943,
"margin_dpo/margin_mean": 439.4707946777344,
"margin_dpo/margin_std": 619.3283081054688,
"step": 517
},
{
"KL/chosen_KL_mean": -538.5008544921875,
"KL/mean": -726.7237548828125,
"KL/rejected_KL_mean": -914.9466552734375,
"KL/std": 435.29779052734375,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.0009226626716554165,
"fcm_dpo/delta": 0.00259285606443882,
"fcm_dpo/margin": 376.4457702636719,
"fcm_dpo/q_t": 0.4179810881614685,
"grad_norm": 46.04099655151367,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.8912504315376282,
"logits/rejected": -0.9100457429885864,
"logps/chosen": -595.956787109375,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -1000.2593383789062,
"loss": 1.1172,
"margin_dpo/margin_mean": 376.4457702636719,
"margin_dpo/margin_std": 483.531005859375,
"step": 518
},
{
"KL/chosen_KL_mean": -560.754150390625,
"KL/mean": -688.62060546875,
"KL/rejected_KL_mean": -816.4871215820312,
"KL/std": 400.0428466796875,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0009278228390030563,
"fcm_dpo/delta": 0.02487240917980671,
"fcm_dpo/margin": 255.73304748535156,
"fcm_dpo/q_t": 0.44359180331230164,
"grad_norm": 25.208648681640625,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.8528093695640564,
"logits/rejected": -0.8358422517776489,
"logps/chosen": -634.8174438476562,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -920.9312744140625,
"loss": 1.2158,
"margin_dpo/margin_mean": 255.7330322265625,
"margin_dpo/margin_std": 497.2356872558594,
"step": 519
},
{
"KL/chosen_KL_mean": -596.6747436523438,
"KL/mean": -771.2025146484375,
"KL/rejected_KL_mean": -945.730224609375,
"KL/std": 479.77325439453125,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.0009306604042649269,
"fcm_dpo/delta": 0.025856276974081993,
"fcm_dpo/margin": 349.0555114746094,
"fcm_dpo/q_t": 0.42620331048965454,
"grad_norm": 33.09168243408203,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.8668011426925659,
"logits/rejected": -0.8748083114624023,
"logps/chosen": -666.974609375,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -1045.7115478515625,
"loss": 1.151,
"margin_dpo/margin_mean": 349.05548095703125,
"margin_dpo/margin_std": 541.3212890625,
"step": 520
},
{
"KL/chosen_KL_mean": -511.3445739746094,
"KL/mean": -700.4781494140625,
"KL/rejected_KL_mean": -889.61181640625,
"KL/std": 478.0359802246094,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.0009390891063958406,
"fcm_dpo/delta": -0.0059568993747234344,
"fcm_dpo/margin": 378.2672119140625,
"fcm_dpo/q_t": 0.41709330677986145,
"grad_norm": 33.98050308227539,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.8779432773590088,
"logits/rejected": -0.8840290904045105,
"logps/chosen": -569.4874877929688,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -972.8924560546875,
"loss": 1.1127,
"margin_dpo/margin_mean": 378.2672119140625,
"margin_dpo/margin_std": 479.72076416015625,
"step": 521
},
{
"KL/chosen_KL_mean": -599.886962890625,
"KL/mean": -763.5291748046875,
"KL/rejected_KL_mean": -927.1713256835938,
"KL/std": 497.3731689453125,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.0009402185096405447,
"fcm_dpo/delta": 0.043608807027339935,
"fcm_dpo/margin": 327.2843017578125,
"fcm_dpo/q_t": 0.42860668897628784,
"grad_norm": 30.430320739746094,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.8880289793014526,
"logits/rejected": -0.8938380479812622,
"logps/chosen": -646.4346923828125,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -993.1851806640625,
"loss": 1.1638,
"margin_dpo/margin_mean": 327.2843017578125,
"margin_dpo/margin_std": 539.9135131835938,
"step": 522
},
{
"KL/chosen_KL_mean": -660.646240234375,
"KL/mean": -847.5511474609375,
"KL/rejected_KL_mean": -1034.4560546875,
"KL/std": 619.6279296875,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0009420792339369655,
"fcm_dpo/delta": -0.002239507157355547,
"fcm_dpo/margin": 373.80975341796875,
"fcm_dpo/q_t": 0.4196031093597412,
"grad_norm": 37.910400390625,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.9370777606964111,
"logits/rejected": -0.9447219371795654,
"logps/chosen": -722.4158935546875,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -1118.217529296875,
"loss": 1.1449,
"margin_dpo/margin_mean": 373.80975341796875,
"margin_dpo/margin_std": 621.3948364257812,
"step": 523
},
{
"KL/chosen_KL_mean": -649.7862548828125,
"KL/mean": -849.8914794921875,
"KL/rejected_KL_mean": -1049.99658203125,
"KL/std": 567.035888671875,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0009336816146969795,
"fcm_dpo/delta": -0.02515629678964615,
"fcm_dpo/margin": 400.2103271484375,
"fcm_dpo/q_t": 0.4117085337638855,
"grad_norm": 40.177085876464844,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.9887799024581909,
"logits/rejected": -0.9843355417251587,
"logps/chosen": -727.8583374023438,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -1131.298583984375,
"loss": 1.1197,
"margin_dpo/margin_mean": 400.2103576660156,
"margin_dpo/margin_std": 576.9923095703125,
"step": 524
},
{
"KL/chosen_KL_mean": -653.83642578125,
"KL/mean": -902.3298950195312,
"KL/rejected_KL_mean": -1150.8232421875,
"KL/std": 586.9584350585938,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0009176610037684441,
"fcm_dpo/delta": -0.11192645132541656,
"fcm_dpo/margin": 496.9869384765625,
"fcm_dpo/q_t": 0.3969133496284485,
"grad_norm": 28.977869033813477,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.9288259744644165,
"logits/rejected": -0.9718555212020874,
"logps/chosen": -704.664306640625,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1250.876220703125,
"loss": 1.0598,
"margin_dpo/margin_mean": 496.9869384765625,
"margin_dpo/margin_std": 639.2304077148438,
"step": 525
},
{
"KL/chosen_KL_mean": -707.6033935546875,
"KL/mean": -946.774169921875,
"KL/rejected_KL_mean": -1185.945068359375,
"KL/std": 587.451171875,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.0009070096421055496,
"fcm_dpo/delta": -0.08758525550365448,
"fcm_dpo/margin": 478.34161376953125,
"fcm_dpo/q_t": 0.4001006484031677,
"grad_norm": 27.308345794677734,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.9423408508300781,
"logits/rejected": -0.943749189376831,
"logps/chosen": -770.7706298828125,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1272.25439453125,
"loss": 1.0858,
"margin_dpo/margin_mean": 478.34161376953125,
"margin_dpo/margin_std": 668.8135986328125,
"step": 526
},
{
"KL/chosen_KL_mean": -757.494384765625,
"KL/mean": -920.9222412109375,
"KL/rejected_KL_mean": -1084.3499755859375,
"KL/std": 633.6776123046875,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.0009061114978976548,
"fcm_dpo/delta": 0.05550983548164368,
"fcm_dpo/margin": 326.85565185546875,
"fcm_dpo/q_t": 0.43042802810668945,
"grad_norm": 39.842193603515625,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -1.1526522636413574,
"logits/rejected": -1.125427007675171,
"logps/chosen": -828.623046875,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -1162.692626953125,
"loss": 1.1639,
"margin_dpo/margin_mean": 326.85565185546875,
"margin_dpo/margin_std": 512.5347900390625,
"step": 527
},
{
"KL/chosen_KL_mean": -766.772705078125,
"KL/mean": -964.36474609375,
"KL/rejected_KL_mean": -1161.956787109375,
"KL/std": 593.4571533203125,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0009081118041649461,
"fcm_dpo/delta": -0.00923129916191101,
"fcm_dpo/margin": 395.18408203125,
"fcm_dpo/q_t": 0.42053911089897156,
"grad_norm": 42.25053024291992,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -1.0484631061553955,
"logits/rejected": -1.0378153324127197,
"logps/chosen": -834.8621826171875,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1255.866943359375,
"loss": 1.1581,
"margin_dpo/margin_mean": 395.18408203125,
"margin_dpo/margin_std": 718.8801879882812,
"step": 528
},
{
"KL/chosen_KL_mean": -914.0716552734375,
"KL/mean": -1060.9349365234375,
"KL/rejected_KL_mean": -1207.7982177734375,
"KL/std": 576.7805786132812,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.0009093580883927643,
"fcm_dpo/delta": 0.004190761595964432,
"fcm_dpo/margin": 293.7265319824219,
"fcm_dpo/q_t": 0.43876832723617554,
"grad_norm": 51.59579086303711,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -1.104278326034546,
"logits/rejected": -1.1218066215515137,
"logps/chosen": -969.6466064453125,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1297.00732421875,
"loss": 1.245,
"margin_dpo/margin_mean": 293.7265319824219,
"margin_dpo/margin_std": 738.239990234375,
"step": 529
},
{
"KL/chosen_KL_mean": -789.41162109375,
"KL/mean": -1014.2277221679688,
"KL/rejected_KL_mean": -1239.0438232421875,
"KL/std": 690.6798095703125,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.000902860308997333,
"fcm_dpo/delta": -0.058281224220991135,
"fcm_dpo/margin": 449.6322021484375,
"fcm_dpo/q_t": 0.41204962134361267,
"grad_norm": 49.076507568359375,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.9682959318161011,
"logits/rejected": -0.9685269594192505,
"logps/chosen": -837.0130615234375,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1326.328369140625,
"loss": 1.1613,
"margin_dpo/margin_mean": 449.6322021484375,
"margin_dpo/margin_std": 891.6737060546875,
"step": 530
},
{
"KL/chosen_KL_mean": -859.981201171875,
"KL/mean": -1053.0469970703125,
"KL/rejected_KL_mean": -1246.1129150390625,
"KL/std": 698.8087158203125,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.000901082472410053,
"fcm_dpo/delta": 0.0020340122282505035,
"fcm_dpo/margin": 386.13177490234375,
"fcm_dpo/q_t": 0.42149466276168823,
"grad_norm": 49.02323913574219,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -1.0975751876831055,
"logits/rejected": -1.0849671363830566,
"logps/chosen": -915.2272338867188,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1316.718994140625,
"loss": 1.1536,
"margin_dpo/margin_mean": 386.1317443847656,
"margin_dpo/margin_std": 673.9454956054688,
"step": 531
},
{
"KL/chosen_KL_mean": -877.0020751953125,
"KL/mean": -1081.432373046875,
"KL/rejected_KL_mean": -1285.862548828125,
"KL/std": 670.0946044921875,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0008910692995414138,
"fcm_dpo/delta": -0.01589033380150795,
"fcm_dpo/margin": 408.8604431152344,
"fcm_dpo/q_t": 0.4186711609363556,
"grad_norm": 81.9896469116211,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -1.0430657863616943,
"logits/rejected": -1.0236797332763672,
"logps/chosen": -947.2880859375,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1372.453857421875,
"loss": 1.1835,
"margin_dpo/margin_mean": 408.8604736328125,
"margin_dpo/margin_std": 825.3141479492188,
"step": 532
},
{
"KL/chosen_KL_mean": -810.05078125,
"KL/mean": -956.464599609375,
"KL/rejected_KL_mean": -1102.878173828125,
"KL/std": 583.2452392578125,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0008952566422522068,
"fcm_dpo/delta": -0.012017594650387764,
"fcm_dpo/margin": 292.827392578125,
"fcm_dpo/q_t": 0.4408678412437439,
"grad_norm": 35.946956634521484,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -1.065936803817749,
"logits/rejected": -1.0475103855133057,
"logps/chosen": -874.9058837890625,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -1179.46630859375,
"loss": 1.232,
"margin_dpo/margin_mean": 292.827392578125,
"margin_dpo/margin_std": 690.3145141601562,
"step": 533
},
{
"KL/chosen_KL_mean": -825.17578125,
"KL/mean": -1033.3275146484375,
"KL/rejected_KL_mean": -1241.479248046875,
"KL/std": 642.9903564453125,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0008851932943798602,
"fcm_dpo/delta": -0.020383629947900772,
"fcm_dpo/margin": 416.30340576171875,
"fcm_dpo/q_t": 0.41717272996902466,
"grad_norm": 40.804595947265625,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -1.0429396629333496,
"logits/rejected": -1.0457737445831299,
"logps/chosen": -885.295166015625,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1320.022705078125,
"loss": 1.1406,
"margin_dpo/margin_mean": 416.30340576171875,
"margin_dpo/margin_std": 685.5845947265625,
"step": 534
},
{
"KL/chosen_KL_mean": -707.3360595703125,
"KL/mean": -944.3709716796875,
"KL/rejected_KL_mean": -1181.406005859375,
"KL/std": 619.294189453125,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.000885907094925642,
"fcm_dpo/delta": -0.07328834384679794,
"fcm_dpo/margin": 474.0699462890625,
"fcm_dpo/q_t": 0.40550702810287476,
"grad_norm": 24.909685134887695,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.9549378752708435,
"logits/rejected": -0.9660934209823608,
"logps/chosen": -761.666259765625,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -1277.713623046875,
"loss": 1.0916,
"margin_dpo/margin_mean": 474.0699462890625,
"margin_dpo/margin_std": 659.700927734375,
"step": 535
},
{
"KL/chosen_KL_mean": -628.810791015625,
"KL/mean": -908.8687744140625,
"KL/rejected_KL_mean": -1188.9267578125,
"KL/std": 691.511962890625,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.0008606475312262774,
"fcm_dpo/delta": -0.13862332701683044,
"fcm_dpo/margin": 560.1160278320312,
"fcm_dpo/q_t": 0.39692431688308716,
"grad_norm": 28.304698944091797,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.9188047051429749,
"logits/rejected": -0.9584248065948486,
"logps/chosen": -675.8912963867188,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1278.024658203125,
"loss": 1.0688,
"margin_dpo/margin_mean": 560.115966796875,
"margin_dpo/margin_std": 843.6351318359375,
"step": 536
},
{
"KL/chosen_KL_mean": -652.699462890625,
"KL/mean": -839.2890625,
"KL/rejected_KL_mean": -1025.878662109375,
"KL/std": 556.4027099609375,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.0008501795819029212,
"fcm_dpo/delta": -0.04009390249848366,
"fcm_dpo/margin": 373.17919921875,
"fcm_dpo/q_t": 0.42610064148902893,
"grad_norm": 43.84196090698242,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -1.0005278587341309,
"logits/rejected": -0.9875226020812988,
"logps/chosen": -710.4468994140625,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -1096.3170166015625,
"loss": 1.1769,
"margin_dpo/margin_mean": 373.17919921875,
"margin_dpo/margin_std": 682.537841796875,
"step": 537
},
{
"KL/chosen_KL_mean": -809.7417602539062,
"KL/mean": -1029.7225341796875,
"KL/rejected_KL_mean": -1249.703369140625,
"KL/std": 651.2928466796875,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.0008414477342739701,
"fcm_dpo/delta": -0.02124924398958683,
"fcm_dpo/margin": 439.9615478515625,
"fcm_dpo/q_t": 0.4152023196220398,
"grad_norm": 31.94780158996582,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -1.0291862487792969,
"logits/rejected": -1.023388147354126,
"logps/chosen": -876.15771484375,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1333.931396484375,
"loss": 1.1207,
"margin_dpo/margin_mean": 439.9615478515625,
"margin_dpo/margin_std": 652.1756591796875,
"step": 538
},
{
"KL/chosen_KL_mean": -676.092529296875,
"KL/mean": -877.2006225585938,
"KL/rejected_KL_mean": -1078.3087158203125,
"KL/std": 510.26898193359375,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.000846359645947814,
"fcm_dpo/delta": 0.009787611663341522,
"fcm_dpo/margin": 402.2161865234375,
"fcm_dpo/q_t": 0.42187222838401794,
"grad_norm": 27.017549514770508,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -1.0366239547729492,
"logits/rejected": -1.0354104042053223,
"logps/chosen": -734.5853881835938,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -1170.16259765625,
"loss": 1.133,
"margin_dpo/margin_mean": 402.2161865234375,
"margin_dpo/margin_std": 587.4945068359375,
"step": 539
},
{
"KL/chosen_KL_mean": -674.0075073242188,
"KL/mean": -940.6689453125,
"KL/rejected_KL_mean": -1207.330322265625,
"KL/std": 633.3556518554688,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.0008355865720659494,
"fcm_dpo/delta": -0.1000160276889801,
"fcm_dpo/margin": 533.3228149414062,
"fcm_dpo/q_t": 0.40245524048805237,
"grad_norm": 33.840572357177734,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.9341703653335571,
"logits/rejected": -0.9644461870193481,
"logps/chosen": -737.489990234375,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1323.7603759765625,
"loss": 1.0855,
"margin_dpo/margin_mean": 533.3228149414062,
"margin_dpo/margin_std": 773.7269287109375,
"step": 540
},
{
"KL/chosen_KL_mean": -790.6764526367188,
"KL/mean": -917.277099609375,
"KL/rejected_KL_mean": -1043.877685546875,
"KL/std": 539.9120483398438,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0008294563158415258,
"fcm_dpo/delta": 0.008179640397429466,
"fcm_dpo/margin": 253.2011260986328,
"fcm_dpo/q_t": 0.4516167938709259,
"grad_norm": 48.75702667236328,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -1.0939087867736816,
"logits/rejected": -1.086462378501892,
"logps/chosen": -868.956787109375,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -1147.1513671875,
"loss": 1.2617,
"margin_dpo/margin_mean": 253.2011260986328,
"margin_dpo/margin_std": 658.490478515625,
"step": 541
},
{
"KL/chosen_KL_mean": -658.6339721679688,
"KL/mean": -913.677978515625,
"KL/rejected_KL_mean": -1168.721923828125,
"KL/std": 608.7216186523438,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.0008145465981215239,
"fcm_dpo/delta": -0.07008374482393265,
"fcm_dpo/margin": 510.0878601074219,
"fcm_dpo/q_t": 0.4053560495376587,
"grad_norm": 48.96900939941406,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -1.0721611976623535,
"logits/rejected": -1.1146684885025024,
"logps/chosen": -716.1189575195312,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -1265.197021484375,
"loss": 1.0974,
"margin_dpo/margin_mean": 510.08782958984375,
"margin_dpo/margin_std": 724.8299560546875,
"step": 542
},
{
"KL/chosen_KL_mean": -741.8212280273438,
"KL/mean": -929.9910888671875,
"KL/rejected_KL_mean": -1118.1610107421875,
"KL/std": 717.8450317382812,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0008209783118218184,
"fcm_dpo/delta": 0.0423150472342968,
"fcm_dpo/margin": 376.3399353027344,
"fcm_dpo/q_t": 0.4371965527534485,
"grad_norm": 36.86077117919922,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.9811362028121948,
"logits/rejected": -1.0160373449325562,
"logps/chosen": -802.438720703125,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1223.760009765625,
"loss": 1.2106,
"margin_dpo/margin_mean": 376.33990478515625,
"margin_dpo/margin_std": 861.6043701171875,
"step": 543
},
{
"KL/chosen_KL_mean": -747.0059204101562,
"KL/mean": -959.01513671875,
"KL/rejected_KL_mean": -1171.0242919921875,
"KL/std": 579.2896728515625,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0008243329357355833,
"fcm_dpo/delta": 0.0004746345803141594,
"fcm_dpo/margin": 424.0183410644531,
"fcm_dpo/q_t": 0.41962099075317383,
"grad_norm": 35.00315475463867,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -1.0107485055923462,
"logits/rejected": -1.0369402170181274,
"logps/chosen": -806.648193359375,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -1271.97900390625,
"loss": 1.1376,
"margin_dpo/margin_mean": 424.01837158203125,
"margin_dpo/margin_std": 667.390380859375,
"step": 544
},
{
"KL/chosen_KL_mean": -678.4371948242188,
"KL/mean": -930.0258178710938,
"KL/rejected_KL_mean": -1181.614501953125,
"KL/std": 583.45068359375,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0008170028449967504,
"fcm_dpo/delta": -0.06372885406017303,
"fcm_dpo/margin": 503.17724609375,
"fcm_dpo/q_t": 0.4060240387916565,
"grad_norm": 38.147010803222656,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.9224939346313477,
"logits/rejected": -0.9622963666915894,
"logps/chosen": -746.0858154296875,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1277.5224609375,
"loss": 1.0858,
"margin_dpo/margin_mean": 503.17724609375,
"margin_dpo/margin_std": 667.4920043945312,
"step": 545
},
{
"KL/chosen_KL_mean": -660.3955688476562,
"KL/mean": -818.3583984375,
"KL/rejected_KL_mean": -976.3211669921875,
"KL/std": 512.590087890625,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0008259355090558529,
"fcm_dpo/delta": 0.09142404794692993,
"fcm_dpo/margin": 315.9256591796875,
"fcm_dpo/q_t": 0.43779951333999634,
"grad_norm": 28.25826644897461,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.990828275680542,
"logits/rejected": -0.9813590049743652,
"logps/chosen": -711.1397705078125,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -1058.1875,
"loss": 1.1939,
"margin_dpo/margin_mean": 315.9256591796875,
"margin_dpo/margin_std": 555.3873291015625,
"step": 546
},
{
"KL/chosen_KL_mean": -655.533935546875,
"KL/mean": -874.7945556640625,
"KL/rejected_KL_mean": -1094.05517578125,
"KL/std": 580.2192993164062,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.0008292993879877031,
"fcm_dpo/delta": -0.014224422164261341,
"fcm_dpo/margin": 438.5212707519531,
"fcm_dpo/q_t": 0.4169241786003113,
"grad_norm": 42.50648498535156,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.9036662578582764,
"logits/rejected": -0.9190746545791626,
"logps/chosen": -729.2216796875,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -1184.816650390625,
"loss": 1.1249,
"margin_dpo/margin_mean": 438.5212707519531,
"margin_dpo/margin_std": 658.936279296875,
"step": 547
},
{
"KL/chosen_KL_mean": -698.337890625,
"KL/mean": -896.851806640625,
"KL/rejected_KL_mean": -1095.3656005859375,
"KL/std": 622.6420288085938,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0008297248277813196,
"fcm_dpo/delta": 0.02127872034907341,
"fcm_dpo/margin": 397.02783203125,
"fcm_dpo/q_t": 0.427689790725708,
"grad_norm": 23.98332977294922,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -1.0400688648223877,
"logits/rejected": -1.0478882789611816,
"logps/chosen": -763.584228515625,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -1189.483642578125,
"loss": 1.1561,
"margin_dpo/margin_mean": 397.0278625488281,
"margin_dpo/margin_std": 693.2183837890625,
"step": 548
},
{
"KL/chosen_KL_mean": -738.9791870117188,
"KL/mean": -889.6734008789062,
"KL/rejected_KL_mean": -1040.36767578125,
"KL/std": 521.7401733398438,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0008309698896482587,
"fcm_dpo/delta": 0.0033192974515259266,
"fcm_dpo/margin": 301.388427734375,
"fcm_dpo/q_t": 0.4423815608024597,
"grad_norm": 49.45306396484375,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -1.0476170778274536,
"logits/rejected": -1.0386322736740112,
"logps/chosen": -788.1915283203125,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -1114.2779541015625,
"loss": 1.2187,
"margin_dpo/margin_mean": 301.388427734375,
"margin_dpo/margin_std": 629.03125,
"step": 549
},
{
"KL/chosen_KL_mean": -725.575927734375,
"KL/mean": -906.271240234375,
"KL/rejected_KL_mean": -1086.966552734375,
"KL/std": 568.6636962890625,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0008367022383026779,
"fcm_dpo/delta": 0.04913487285375595,
"fcm_dpo/margin": 361.39056396484375,
"fcm_dpo/q_t": 0.42964881658554077,
"grad_norm": 34.70016098022461,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.9834055304527283,
"logits/rejected": -1.0090551376342773,
"logps/chosen": -782.3829345703125,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -1182.09228515625,
"loss": 1.164,
"margin_dpo/margin_mean": 361.3905334472656,
"margin_dpo/margin_std": 592.5504150390625,
"step": 550
},
{
"KL/chosen_KL_mean": -631.9072265625,
"KL/mean": -930.6796875,
"KL/rejected_KL_mean": -1229.4521484375,
"KL/std": 619.4073486328125,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.0008201512973755598,
"fcm_dpo/delta": -0.14753000438213348,
"fcm_dpo/margin": 597.544921875,
"fcm_dpo/q_t": 0.38655346632003784,
"grad_norm": 45.9125862121582,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.9045864343643188,
"logits/rejected": -0.9689292907714844,
"logps/chosen": -691.0135498046875,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1341.1248779296875,
"loss": 1.0182,
"margin_dpo/margin_mean": 597.544921875,
"margin_dpo/margin_std": 639.778564453125,
"step": 551
},
{
"KL/chosen_KL_mean": -622.3704223632812,
"KL/mean": -966.5551147460938,
"KL/rejected_KL_mean": -1310.73974609375,
"KL/std": 730.783203125,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.0007834373391233385,
"fcm_dpo/delta": -0.20374326407909393,
"fcm_dpo/margin": 688.369384765625,
"fcm_dpo/q_t": 0.3815649747848511,
"grad_norm": 56.078468322753906,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -1.027387022972107,
"logits/rejected": -1.0806533098220825,
"logps/chosen": -684.7250366210938,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1415.302001953125,
"loss": 1.016,
"margin_dpo/margin_mean": 688.369384765625,
"margin_dpo/margin_std": 843.5326538085938,
"step": 552
},
{
"KL/chosen_KL_mean": -718.2479248046875,
"KL/mean": -891.719970703125,
"KL/rejected_KL_mean": -1065.19189453125,
"KL/std": 602.6245727539062,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0007897895993664861,
"fcm_dpo/delta": 0.07819828391075134,
"fcm_dpo/margin": 346.944091796875,
"fcm_dpo/q_t": 0.4390897750854492,
"grad_norm": 29.118030548095703,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.9896628856658936,
"logits/rejected": -0.9859640598297119,
"logps/chosen": -786.5067138671875,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -1163.2890625,
"loss": 1.1954,
"margin_dpo/margin_mean": 346.944091796875,
"margin_dpo/margin_std": 672.8452758789062,
"step": 553
},
{
"KL/chosen_KL_mean": -761.680419921875,
"KL/mean": -982.8804931640625,
"KL/rejected_KL_mean": -1204.08056640625,
"KL/std": 681.6918334960938,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.000800039735622704,
"fcm_dpo/delta": -0.00511806458234787,
"fcm_dpo/margin": 442.40008544921875,
"fcm_dpo/q_t": 0.41944724321365356,
"grad_norm": 130.93975830078125,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.9642994403839111,
"logits/rejected": -0.9368813037872314,
"logps/chosen": -829.6281127929688,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1293.86328125,
"loss": 1.1836,
"margin_dpo/margin_mean": 442.40008544921875,
"margin_dpo/margin_std": 893.775634765625,
"step": 554
},
{
"KL/chosen_KL_mean": -751.6488037109375,
"KL/mean": -1048.9979248046875,
"KL/rejected_KL_mean": -1346.3470458984375,
"KL/std": 696.0157470703125,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.0007814068230800331,
"fcm_dpo/delta": -0.120139941573143,
"fcm_dpo/margin": 594.6982421875,
"fcm_dpo/q_t": 0.3991040587425232,
"grad_norm": 35.379878997802734,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -1.0353598594665527,
"logits/rejected": -1.1041009426116943,
"logps/chosen": -804.979248046875,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1454.826416015625,
"loss": 1.0837,
"margin_dpo/margin_mean": 594.6982421875,
"margin_dpo/margin_std": 901.941162109375,
"step": 555
},
{
"KL/chosen_KL_mean": -742.638916015625,
"KL/mean": -909.952880859375,
"KL/rejected_KL_mean": -1077.2667236328125,
"KL/std": 527.5067138671875,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.0007855216972529888,
"fcm_dpo/delta": 0.0895049199461937,
"fcm_dpo/margin": 334.6279296875,
"fcm_dpo/q_t": 0.43837928771972656,
"grad_norm": 30.330717086791992,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -1.0571699142456055,
"logits/rejected": -1.0535497665405273,
"logps/chosen": -801.2833862304688,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -1178.607177734375,
"loss": 1.1816,
"margin_dpo/margin_mean": 334.6279296875,
"margin_dpo/margin_std": 526.4218139648438,
"step": 556
},
{
"KL/chosen_KL_mean": -823.6038818359375,
"KL/mean": -1078.040283203125,
"KL/rejected_KL_mean": -1332.476806640625,
"KL/std": 743.6300048828125,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.000781252165324986,
"fcm_dpo/delta": -0.049805186688899994,
"fcm_dpo/margin": 508.87274169921875,
"fcm_dpo/q_t": 0.4107533097267151,
"grad_norm": 44.83199691772461,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -1.0895347595214844,
"logits/rejected": -1.1284986734390259,
"logps/chosen": -891.444580078125,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1442.4163818359375,
"loss": 1.1255,
"margin_dpo/margin_mean": 508.8727722167969,
"margin_dpo/margin_std": 828.09326171875,
"step": 557
},
{
"KL/chosen_KL_mean": -763.7918090820312,
"KL/mean": -1106.19775390625,
"KL/rejected_KL_mean": -1448.603759765625,
"KL/std": 703.3863525390625,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0007613534107804298,
"fcm_dpo/delta": -0.18066798150539398,
"fcm_dpo/margin": 684.81201171875,
"fcm_dpo/q_t": 0.37840861082077026,
"grad_norm": 27.58951187133789,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -1.1353588104248047,
"logits/rejected": -1.158677577972412,
"logps/chosen": -826.1600341796875,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1550.7647705078125,
"loss": 1.0166,
"margin_dpo/margin_mean": 684.81201171875,
"margin_dpo/margin_std": 793.9622802734375,
"step": 558
},
{
"KL/chosen_KL_mean": -867.53369140625,
"KL/mean": -1147.8270263671875,
"KL/rejected_KL_mean": -1428.120361328125,
"KL/std": 725.3985595703125,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.0007472336874343455,
"fcm_dpo/delta": -0.07184131443500519,
"fcm_dpo/margin": 560.5865478515625,
"fcm_dpo/q_t": 0.4061052203178406,
"grad_norm": 31.313798904418945,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -1.2728649377822876,
"logits/rejected": -1.2823593616485596,
"logps/chosen": -928.2860717773438,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1521.5626220703125,
"loss": 1.0946,
"margin_dpo/margin_mean": 560.5865478515625,
"margin_dpo/margin_std": 811.6783447265625,
"step": 559
},
{
"KL/chosen_KL_mean": -794.177734375,
"KL/mean": -987.329833984375,
"KL/rejected_KL_mean": -1180.4820556640625,
"KL/std": 631.4752197265625,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.000741536554414779,
"fcm_dpo/delta": -0.008997570723295212,
"fcm_dpo/margin": 386.3043212890625,
"fcm_dpo/q_t": 0.4339349865913391,
"grad_norm": 36.68315505981445,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.9954744577407837,
"logits/rejected": -0.9756410121917725,
"logps/chosen": -852.2815551757812,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -1260.4732666015625,
"loss": 1.1833,
"margin_dpo/margin_mean": 386.3043212890625,
"margin_dpo/margin_std": 689.2628173828125,
"step": 560
},
{
"KL/chosen_KL_mean": -918.6915283203125,
"KL/mean": -1091.2904052734375,
"KL/rejected_KL_mean": -1263.8892822265625,
"KL/std": 642.873291015625,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0007538319332525134,
"fcm_dpo/delta": 0.09177450835704803,
"fcm_dpo/margin": 345.19769287109375,
"fcm_dpo/q_t": 0.43946483731269836,
"grad_norm": 40.61189651489258,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -1.1234174966812134,
"logits/rejected": -1.147684097290039,
"logps/chosen": -969.6044311523438,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1362.957763671875,
"loss": 1.2324,
"margin_dpo/margin_mean": 345.19769287109375,
"margin_dpo/margin_std": 824.038330078125,
"step": 561
},
{
"KL/chosen_KL_mean": -841.947509765625,
"KL/mean": -1096.912841796875,
"KL/rejected_KL_mean": -1351.8782958984375,
"KL/std": 714.989990234375,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0007471415447071195,
"fcm_dpo/delta": -0.03312261030077934,
"fcm_dpo/margin": 509.93084716796875,
"fcm_dpo/q_t": 0.4142550826072693,
"grad_norm": 37.721954345703125,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -1.1088937520980835,
"logits/rejected": -1.1244970560073853,
"logps/chosen": -901.411865234375,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1448.4208984375,
"loss": 1.1201,
"margin_dpo/margin_mean": 509.93084716796875,
"margin_dpo/margin_std": 781.1636962890625,
"step": 562
},
{
"KL/chosen_KL_mean": -965.8650512695312,
"KL/mean": -1130.153564453125,
"KL/rejected_KL_mean": -1294.44189453125,
"KL/std": 774.4320068359375,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.00074770423816517,
"fcm_dpo/delta": -0.0345328263938427,
"fcm_dpo/margin": 328.5767822265625,
"fcm_dpo/q_t": 0.43725377321243286,
"grad_norm": 60.61211013793945,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -1.101536512374878,
"logits/rejected": -1.1053555011749268,
"logps/chosen": -1026.4732666015625,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -1389.009521484375,
"loss": 1.2732,
"margin_dpo/margin_mean": 328.5767822265625,
"margin_dpo/margin_std": 918.204345703125,
"step": 563
},
{
"KL/chosen_KL_mean": -839.5714111328125,
"KL/mean": -1072.73291015625,
"KL/rejected_KL_mean": -1305.894287109375,
"KL/std": 678.1080322265625,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0007467868272215128,
"fcm_dpo/delta": 0.0017370246350765228,
"fcm_dpo/margin": 466.32293701171875,
"fcm_dpo/q_t": 0.42049744725227356,
"grad_norm": 39.77863693237305,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -1.1387296915054321,
"logits/rejected": -1.1342732906341553,
"logps/chosen": -913.8397827148438,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1399.132568359375,
"loss": 1.1292,
"margin_dpo/margin_mean": 466.32293701171875,
"margin_dpo/margin_std": 674.2352294921875,
"step": 564
},
{
"KL/chosen_KL_mean": -903.0402221679688,
"KL/mean": -1139.46142578125,
"KL/rejected_KL_mean": -1375.882568359375,
"KL/std": 793.594482421875,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0007455106824636459,
"fcm_dpo/delta": -0.002605642657727003,
"fcm_dpo/margin": 472.84234619140625,
"fcm_dpo/q_t": 0.424316942691803,
"grad_norm": 58.753501892089844,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -1.083191156387329,
"logits/rejected": -1.084733486175537,
"logps/chosen": -972.0601806640625,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1461.6614990234375,
"loss": 1.1859,
"margin_dpo/margin_mean": 472.84234619140625,
"margin_dpo/margin_std": 984.09130859375,
"step": 565
},
{
"KL/chosen_KL_mean": -851.157958984375,
"KL/mean": -1113.0625,
"KL/rejected_KL_mean": -1374.967041015625,
"KL/std": 766.4264526367188,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0007446268573403358,
"fcm_dpo/delta": -0.04210010543465614,
"fcm_dpo/margin": 523.8090209960938,
"fcm_dpo/q_t": 0.41205883026123047,
"grad_norm": 30.277292251586914,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -1.0920714139938354,
"logits/rejected": -1.1179513931274414,
"logps/chosen": -917.7032470703125,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1478.8363037109375,
"loss": 1.1391,
"margin_dpo/margin_mean": 523.8089599609375,
"margin_dpo/margin_std": 888.4176025390625,
"step": 566
},
{
"KL/chosen_KL_mean": -797.819091796875,
"KL/mean": -1001.6353759765625,
"KL/rejected_KL_mean": -1205.4515380859375,
"KL/std": 539.5685424804688,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.0007430259138345718,
"fcm_dpo/delta": 0.04861754924058914,
"fcm_dpo/margin": 407.6325378417969,
"fcm_dpo/q_t": 0.42830806970596313,
"grad_norm": 25.824920654296875,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -1.213765263557434,
"logits/rejected": -1.21918523311615,
"logps/chosen": -850.6773681640625,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1290.822509765625,
"loss": 1.1507,
"margin_dpo/margin_mean": 407.632568359375,
"margin_dpo/margin_std": 579.9044189453125,
"step": 567
},
{
"KL/chosen_KL_mean": -774.791748046875,
"KL/mean": -1049.3804931640625,
"KL/rejected_KL_mean": -1323.96923828125,
"KL/std": 630.48486328125,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0007391650578938425,
"fcm_dpo/delta": -0.05833129584789276,
"fcm_dpo/margin": 549.1774291992188,
"fcm_dpo/q_t": 0.40518689155578613,
"grad_norm": 27.453920364379883,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -1.0664093494415283,
"logits/rejected": -1.1100949048995972,
"logps/chosen": -819.984130859375,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1413.0615234375,
"loss": 1.0805,
"margin_dpo/margin_mean": 549.1774291992188,
"margin_dpo/margin_std": 684.4533081054688,
"step": 568
},
{
"KL/chosen_KL_mean": -870.124267578125,
"KL/mean": -1090.1845703125,
"KL/rejected_KL_mean": -1310.244873046875,
"KL/std": 802.5746459960938,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0007340764277614653,
"fcm_dpo/delta": -0.04408378154039383,
"fcm_dpo/margin": 440.1204833984375,
"fcm_dpo/q_t": 0.42503905296325684,
"grad_norm": 42.93728256225586,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -1.109398365020752,
"logits/rejected": -1.1187589168548584,
"logps/chosen": -933.8448486328125,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1389.3480224609375,
"loss": 1.1831,
"margin_dpo/margin_mean": 440.1204833984375,
"margin_dpo/margin_std": 843.69482421875,
"step": 569
},
{
"KL/chosen_KL_mean": -786.7838134765625,
"KL/mean": -990.749755859375,
"KL/rejected_KL_mean": -1194.715576171875,
"KL/std": 626.913818359375,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0007354548433795571,
"fcm_dpo/delta": 0.05156288668513298,
"fcm_dpo/margin": 407.93182373046875,
"fcm_dpo/q_t": 0.43151092529296875,
"grad_norm": 29.369874954223633,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -1.1785709857940674,
"logits/rejected": -1.1703860759735107,
"logps/chosen": -848.3983764648438,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1276.8575439453125,
"loss": 1.1709,
"margin_dpo/margin_mean": 407.9317932128906,
"margin_dpo/margin_std": 689.3609619140625,
"step": 570
},
{
"KL/chosen_KL_mean": -815.0283813476562,
"KL/mean": -1055.030029296875,
"KL/rejected_KL_mean": -1295.03173828125,
"KL/std": 642.4834594726562,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0007371066603809595,
"fcm_dpo/delta": -0.0039865849539637566,
"fcm_dpo/margin": 480.00323486328125,
"fcm_dpo/q_t": 0.41918835043907166,
"grad_norm": 41.169490814208984,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -1.0355522632598877,
"logits/rejected": -1.0509165525436401,
"logps/chosen": -868.0824584960938,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1386.3685302734375,
"loss": 1.1453,
"margin_dpo/margin_mean": 480.0032653808594,
"margin_dpo/margin_std": 803.4883422851562,
"step": 571
},
{
"KL/chosen_KL_mean": -836.864013671875,
"KL/mean": -1090.23388671875,
"KL/rejected_KL_mean": -1343.603759765625,
"KL/std": 737.3343505859375,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.0007338828290812671,
"fcm_dpo/delta": -0.022855721414089203,
"fcm_dpo/margin": 506.73992919921875,
"fcm_dpo/q_t": 0.41840416193008423,
"grad_norm": 32.44576644897461,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -1.0318219661712646,
"logits/rejected": -1.06544828414917,
"logps/chosen": -885.3232421875,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1427.160888671875,
"loss": 1.1474,
"margin_dpo/margin_mean": 506.73992919921875,
"margin_dpo/margin_std": 905.3984985351562,
"step": 572
},
{
"KL/chosen_KL_mean": -788.7066040039062,
"KL/mean": -1008.2884521484375,
"KL/rejected_KL_mean": -1227.870361328125,
"KL/std": 634.485107421875,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0007377726724371314,
"fcm_dpo/delta": 0.02684948407113552,
"fcm_dpo/margin": 439.1636962890625,
"fcm_dpo/q_t": 0.42565691471099854,
"grad_norm": 25.155153274536133,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -1.1160049438476562,
"logits/rejected": -1.1299877166748047,
"logps/chosen": -840.3292236328125,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1313.1953125,
"loss": 1.1386,
"margin_dpo/margin_mean": 439.1636657714844,
"margin_dpo/margin_std": 636.15185546875,
"step": 573
},
{
"KL/chosen_KL_mean": -715.6622314453125,
"KL/mean": -959.97314453125,
"KL/rejected_KL_mean": -1204.2841796875,
"KL/std": 584.0929565429688,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0007384730270132422,
"fcm_dpo/delta": -0.011282573454082012,
"fcm_dpo/margin": 488.62188720703125,
"fcm_dpo/q_t": 0.416146457195282,
"grad_norm": 30.116424560546875,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -1.0976953506469727,
"logits/rejected": -1.1268978118896484,
"logps/chosen": -766.7066650390625,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -1297.090576171875,
"loss": 1.1183,
"margin_dpo/margin_mean": 488.62188720703125,
"margin_dpo/margin_std": 683.5946044921875,
"step": 574
},
{
"KL/chosen_KL_mean": -779.8887939453125,
"KL/mean": -997.6763916015625,
"KL/rejected_KL_mean": -1215.464111328125,
"KL/std": 598.38037109375,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0007295336690731347,
"fcm_dpo/delta": -0.049235548824071884,
"fcm_dpo/margin": 435.5752258300781,
"fcm_dpo/q_t": 0.4261569082736969,
"grad_norm": 35.20523452758789,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.9492102861404419,
"logits/rejected": -0.9579809904098511,
"logps/chosen": -851.678955078125,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1310.850341796875,
"loss": 1.1713,
"margin_dpo/margin_mean": 435.57525634765625,
"margin_dpo/margin_std": 770.2555541992188,
"step": 575
},
{
"KL/chosen_KL_mean": -756.765869140625,
"KL/mean": -921.8978271484375,
"KL/rejected_KL_mean": -1087.0296630859375,
"KL/std": 545.06005859375,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.0007330116932280362,
"fcm_dpo/delta": 0.023780547082424164,
"fcm_dpo/margin": 330.26385498046875,
"fcm_dpo/q_t": 0.4433874785900116,
"grad_norm": 32.09224319458008,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -1.070650339126587,
"logits/rejected": -1.1040256023406982,
"logps/chosen": -811.02880859375,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -1187.783935546875,
"loss": 1.211,
"margin_dpo/margin_mean": 330.2638854980469,
"margin_dpo/margin_std": 628.616943359375,
"step": 576
},
{
"KL/chosen_KL_mean": -669.3681640625,
"KL/mean": -892.837890625,
"KL/rejected_KL_mean": -1116.3076171875,
"KL/std": 621.8170776367188,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0007347659557126462,
"fcm_dpo/delta": 0.022340623661875725,
"fcm_dpo/margin": 446.939453125,
"fcm_dpo/q_t": 0.424075722694397,
"grad_norm": 25.360074996948242,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -1.0318996906280518,
"logits/rejected": -1.0215978622436523,
"logps/chosen": -731.2778930664062,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1200.3782958984375,
"loss": 1.1501,
"margin_dpo/margin_mean": 446.93951416015625,
"margin_dpo/margin_std": 677.6488037109375,
"step": 577
},
{
"KL/chosen_KL_mean": -641.9940185546875,
"KL/mean": -880.4281005859375,
"KL/rejected_KL_mean": -1118.8621826171875,
"KL/std": 590.3482055664062,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.000733315828256309,
"fcm_dpo/delta": -1.9393861293792725e-05,
"fcm_dpo/margin": 476.8681335449219,
"fcm_dpo/q_t": 0.41781163215637207,
"grad_norm": 46.754920959472656,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.9348374605178833,
"logits/rejected": -0.9725053310394287,
"logps/chosen": -691.2577514648438,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1202.2984619140625,
"loss": 1.1143,
"margin_dpo/margin_mean": 476.8681335449219,
"margin_dpo/margin_std": 598.9371337890625,
"step": 578
},
{
"KL/chosen_KL_mean": -752.2142333984375,
"KL/mean": -893.55517578125,
"KL/rejected_KL_mean": -1034.8961181640625,
"KL/std": 593.69775390625,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0007440147455781698,
"fcm_dpo/delta": 0.052174538373947144,
"fcm_dpo/margin": 282.68194580078125,
"fcm_dpo/q_t": 0.4504472315311432,
"grad_norm": 50.663330078125,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.9605880975723267,
"logits/rejected": -0.964146077632904,
"logps/chosen": -811.10009765625,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -1129.683837890625,
"loss": 1.2467,
"margin_dpo/margin_mean": 282.68194580078125,
"margin_dpo/margin_std": 665.921142578125,
"step": 579
},
{
"KL/chosen_KL_mean": -525.0825805664062,
"KL/mean": -796.9617919921875,
"KL/rejected_KL_mean": -1068.841064453125,
"KL/std": 658.41455078125,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0007361972820945084,
"fcm_dpo/delta": -0.05270449444651604,
"fcm_dpo/margin": 543.7584228515625,
"fcm_dpo/q_t": 0.4098510146141052,
"grad_norm": 40.2070426940918,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.8550982475280762,
"logits/rejected": -0.8935449719429016,
"logps/chosen": -573.7894287109375,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1150.5994873046875,
"loss": 1.0934,
"margin_dpo/margin_mean": 543.7584228515625,
"margin_dpo/margin_std": 754.7635498046875,
"step": 580
},
{
"KL/chosen_KL_mean": -729.9277954101562,
"KL/mean": -922.0654296875,
"KL/rejected_KL_mean": -1114.202880859375,
"KL/std": 613.97216796875,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0007442636415362358,
"fcm_dpo/delta": 0.06579051911830902,
"fcm_dpo/margin": 384.27520751953125,
"fcm_dpo/q_t": 0.434751033782959,
"grad_norm": 25.00486946105957,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -1.0764214992523193,
"logits/rejected": -1.0914995670318604,
"logps/chosen": -792.150146484375,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1205.938720703125,
"loss": 1.1788,
"margin_dpo/margin_mean": 384.27520751953125,
"margin_dpo/margin_std": 675.5390625,
"step": 581
},
{
"KL/chosen_KL_mean": -657.9926147460938,
"KL/mean": -858.8969116210938,
"KL/rejected_KL_mean": -1059.80126953125,
"KL/std": 526.9097290039062,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0007423992501571774,
"fcm_dpo/delta": -0.04802374914288521,
"fcm_dpo/margin": 401.8086242675781,
"fcm_dpo/q_t": 0.4290406405925751,
"grad_norm": 27.108118057250977,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -0.9977607727050781,
"logits/rejected": -1.0202760696411133,
"logps/chosen": -716.2213134765625,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -1169.870849609375,
"loss": 1.1596,
"margin_dpo/margin_mean": 401.80865478515625,
"margin_dpo/margin_std": 577.4631958007812,
"step": 582
},
{
"KL/chosen_KL_mean": -748.9545288085938,
"KL/mean": -883.51416015625,
"KL/rejected_KL_mean": -1018.0738525390625,
"KL/std": 600.503662109375,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0007386098150163889,
"fcm_dpo/delta": -0.0015174677828326821,
"fcm_dpo/margin": 269.1192626953125,
"fcm_dpo/q_t": 0.45400410890579224,
"grad_norm": 39.618011474609375,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.953858494758606,
"logits/rejected": -0.9514139890670776,
"logps/chosen": -805.8173828125,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -1106.477783203125,
"loss": 1.2716,
"margin_dpo/margin_mean": 269.1192626953125,
"margin_dpo/margin_std": 726.809814453125,
"step": 583
},
{
"KL/chosen_KL_mean": -650.4002075195312,
"KL/mean": -920.8975830078125,
"KL/rejected_KL_mean": -1191.39501953125,
"KL/std": 627.6343383789062,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0007337032584473491,
"fcm_dpo/delta": -0.0488472580909729,
"fcm_dpo/margin": 540.9948120117188,
"fcm_dpo/q_t": 0.40848594903945923,
"grad_norm": 33.2059326171875,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -0.9727611541748047,
"logits/rejected": -0.9878000020980835,
"logps/chosen": -707.3009033203125,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1289.0311279296875,
"loss": 1.0841,
"margin_dpo/margin_mean": 540.9948120117188,
"margin_dpo/margin_std": 661.6787109375,
"step": 584
},
{
"KL/chosen_KL_mean": -755.4912109375,
"KL/mean": -1001.9600219726562,
"KL/rejected_KL_mean": -1248.4288330078125,
"KL/std": 691.0889892578125,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0007283834274858236,
"fcm_dpo/delta": -0.009564774110913277,
"fcm_dpo/margin": 492.93756103515625,
"fcm_dpo/q_t": 0.4175519645214081,
"grad_norm": 28.804832458496094,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -1.0524985790252686,
"logits/rejected": -1.0806963443756104,
"logps/chosen": -816.4652099609375,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1332.598388671875,
"loss": 1.1445,
"margin_dpo/margin_mean": 492.93756103515625,
"margin_dpo/margin_std": 832.9927978515625,
"step": 585
},
{
"KL/chosen_KL_mean": -810.9488525390625,
"KL/mean": -974.180419921875,
"KL/rejected_KL_mean": -1137.412109375,
"KL/std": 624.2877197265625,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0007359388982877135,
"fcm_dpo/delta": 0.04029836505651474,
"fcm_dpo/margin": 326.46331787109375,
"fcm_dpo/q_t": 0.4438827335834503,
"grad_norm": 31.14897918701172,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -1.0293405055999756,
"logits/rejected": -1.0565171241760254,
"logps/chosen": -867.804443359375,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -1229.2147216796875,
"loss": 1.2212,
"margin_dpo/margin_mean": 326.46331787109375,
"margin_dpo/margin_std": 682.7779541015625,
"step": 586
},
{
"KL/chosen_KL_mean": -576.955810546875,
"KL/mean": -799.3982543945312,
"KL/rejected_KL_mean": -1021.8407592773438,
"KL/std": 647.3914794921875,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0007399095338769257,
"fcm_dpo/delta": 0.02138981781899929,
"fcm_dpo/margin": 444.885009765625,
"fcm_dpo/q_t": 0.42297637462615967,
"grad_norm": 35.817604064941406,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -1.0655786991119385,
"logits/rejected": -1.0954036712646484,
"logps/chosen": -621.6473388671875,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -1104.464599609375,
"loss": 1.1398,
"margin_dpo/margin_mean": 444.885009765625,
"margin_dpo/margin_std": 656.8631591796875,
"step": 587
},
{
"KL/chosen_KL_mean": -763.5947265625,
"KL/mean": -951.0715942382812,
"KL/rejected_KL_mean": -1138.5484619140625,
"KL/std": 554.4877319335938,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0007451215060427785,
"fcm_dpo/delta": 0.07268988341093063,
"fcm_dpo/margin": 374.9537353515625,
"fcm_dpo/q_t": 0.4341343939304352,
"grad_norm": 28.395036697387695,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -1.0229034423828125,
"logits/rejected": -1.052525281906128,
"logps/chosen": -813.8897094726562,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1245.9183349609375,
"loss": 1.1757,
"margin_dpo/margin_mean": 374.9537353515625,
"margin_dpo/margin_std": 601.4207763671875,
"step": 588
},
{
"KL/chosen_KL_mean": -765.1441650390625,
"KL/mean": -1015.72314453125,
"KL/rejected_KL_mean": -1266.30224609375,
"KL/std": 625.617431640625,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.000747905985917896,
"fcm_dpo/delta": -0.025782965123653412,
"fcm_dpo/margin": 501.15802001953125,
"fcm_dpo/q_t": 0.4127042889595032,
"grad_norm": 36.19021224975586,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -1.0197863578796387,
"logits/rejected": -1.0424385070800781,
"logps/chosen": -825.0740356445312,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1377.95751953125,
"loss": 1.1029,
"margin_dpo/margin_mean": 501.1579895019531,
"margin_dpo/margin_std": 657.3384399414062,
"step": 589
},
{
"KL/chosen_KL_mean": -667.9539794921875,
"KL/mean": -920.4093627929688,
"KL/rejected_KL_mean": -1172.86474609375,
"KL/std": 630.0244140625,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.000742416363209486,
"fcm_dpo/delta": -0.02587101422250271,
"fcm_dpo/margin": 504.91082763671875,
"fcm_dpo/q_t": 0.41454923152923584,
"grad_norm": 20.466495513916016,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -1.0065345764160156,
"logits/rejected": -1.039345145225525,
"logps/chosen": -723.7637939453125,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1278.927734375,
"loss": 1.109,
"margin_dpo/margin_mean": 504.91082763671875,
"margin_dpo/margin_std": 708.3488159179688,
"step": 590
},
{
"KL/chosen_KL_mean": -719.654296875,
"KL/mean": -956.586181640625,
"KL/rejected_KL_mean": -1193.51806640625,
"KL/std": 603.654052734375,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0007421453483402729,
"fcm_dpo/delta": -0.0017371654976159334,
"fcm_dpo/margin": 473.86370849609375,
"fcm_dpo/q_t": 0.41778093576431274,
"grad_norm": 29.069992065429688,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -1.1044466495513916,
"logits/rejected": -1.1089308261871338,
"logps/chosen": -775.8948974609375,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1277.3043212890625,
"loss": 1.1284,
"margin_dpo/margin_mean": 473.86370849609375,
"margin_dpo/margin_std": 696.78857421875,
"step": 591
},
{
"KL/chosen_KL_mean": -779.455078125,
"KL/mean": -986.9091186523438,
"KL/rejected_KL_mean": -1194.3631591796875,
"KL/std": 615.1773681640625,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0007462240173481405,
"fcm_dpo/delta": 0.04169132187962532,
"fcm_dpo/margin": 414.90814208984375,
"fcm_dpo/q_t": 0.4289587736129761,
"grad_norm": 39.19664764404297,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -1.0807688236236572,
"logits/rejected": -1.090897560119629,
"logps/chosen": -827.3953247070312,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1270.0960693359375,
"loss": 1.1963,
"margin_dpo/margin_mean": 414.90814208984375,
"margin_dpo/margin_std": 837.1720581054688,
"step": 592
},
{
"KL/chosen_KL_mean": -763.120849609375,
"KL/mean": -918.5281982421875,
"KL/rejected_KL_mean": -1073.9356689453125,
"KL/std": 664.6796264648438,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0007523976382799447,
"fcm_dpo/delta": 0.028210198506712914,
"fcm_dpo/margin": 310.814697265625,
"fcm_dpo/q_t": 0.4463445246219635,
"grad_norm": 45.363460540771484,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.9572064280509949,
"logits/rejected": -0.9502384662628174,
"logps/chosen": -811.8116455078125,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -1134.8436279296875,
"loss": 1.239,
"margin_dpo/margin_mean": 310.814697265625,
"margin_dpo/margin_std": 729.053955078125,
"step": 593
},
{
"KL/chosen_KL_mean": -731.2477416992188,
"KL/mean": -935.2242431640625,
"KL/rejected_KL_mean": -1139.2008056640625,
"KL/std": 653.4246215820312,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0007495468016713858,
"fcm_dpo/delta": -0.03803475201129913,
"fcm_dpo/margin": 407.953125,
"fcm_dpo/q_t": 0.429071843624115,
"grad_norm": 32.1541633605957,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -1.05826735496521,
"logits/rejected": -1.0492217540740967,
"logps/chosen": -786.1826171875,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -1225.300537109375,
"loss": 1.1734,
"margin_dpo/margin_mean": 407.953125,
"margin_dpo/margin_std": 699.8004150390625,
"step": 594
},
{
"KL/chosen_KL_mean": -674.6976928710938,
"KL/mean": -915.5961303710938,
"KL/rejected_KL_mean": -1156.4945068359375,
"KL/std": 617.6531982421875,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0007444015936926007,
"fcm_dpo/delta": -0.00904869381338358,
"fcm_dpo/margin": 481.7968444824219,
"fcm_dpo/q_t": 0.4190996289253235,
"grad_norm": 51.79603958129883,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -1.0517390966415405,
"logits/rejected": -1.0796115398406982,
"logps/chosen": -724.1181030273438,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1237.121826171875,
"loss": 1.1206,
"margin_dpo/margin_mean": 481.7968444824219,
"margin_dpo/margin_std": 694.1858520507812,
"step": 595
},
{
"KL/chosen_KL_mean": -815.898193359375,
"KL/mean": -957.6881103515625,
"KL/rejected_KL_mean": -1099.47802734375,
"KL/std": 608.5431518554688,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.0007497658370994031,
"fcm_dpo/delta": 0.05895322561264038,
"fcm_dpo/margin": 283.5798645019531,
"fcm_dpo/q_t": 0.4523196518421173,
"grad_norm": 73.0402603149414,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -1.0392305850982666,
"logits/rejected": -1.003946304321289,
"logps/chosen": -875.6898803710938,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -1179.569091796875,
"loss": 1.2571,
"margin_dpo/margin_mean": 283.5798645019531,
"margin_dpo/margin_std": 738.8785400390625,
"step": 596
},
{
"KL/chosen_KL_mean": -754.0137939453125,
"KL/mean": -1037.7945556640625,
"KL/rejected_KL_mean": -1321.5755615234375,
"KL/std": 707.1693725585938,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.000738929957151413,
"fcm_dpo/delta": -0.0748884379863739,
"fcm_dpo/margin": 567.561767578125,
"fcm_dpo/q_t": 0.405043363571167,
"grad_norm": 26.4957218170166,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.9764111042022705,
"logits/rejected": -1.030094861984253,
"logps/chosen": -811.2745361328125,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1422.269287109375,
"loss": 1.0919,
"margin_dpo/margin_mean": 567.561767578125,
"margin_dpo/margin_std": 766.98095703125,
"step": 597
},
{
"KL/chosen_KL_mean": -748.9894409179688,
"KL/mean": -973.43359375,
"KL/rejected_KL_mean": -1197.8779296875,
"KL/std": 626.2220458984375,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0007343441247940063,
"fcm_dpo/delta": -0.058417148888111115,
"fcm_dpo/margin": 448.888427734375,
"fcm_dpo/q_t": 0.42271342873573303,
"grad_norm": 28.324201583862305,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -1.1111347675323486,
"logits/rejected": -1.1283093690872192,
"logps/chosen": -801.5079345703125,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1287.32177734375,
"loss": 1.1504,
"margin_dpo/margin_mean": 448.888427734375,
"margin_dpo/margin_std": 702.4248657226562,
"step": 598
},
{
"KL/chosen_KL_mean": -776.1297607421875,
"KL/mean": -960.819580078125,
"KL/rejected_KL_mean": -1145.50927734375,
"KL/std": 598.0007934570312,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0007289372151717544,
"fcm_dpo/delta": -0.03695056587457657,
"fcm_dpo/margin": 369.37945556640625,
"fcm_dpo/q_t": 0.43398356437683105,
"grad_norm": 35.090877532958984,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -1.0137587785720825,
"logits/rejected": -1.0232412815093994,
"logps/chosen": -825.9324951171875,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -1228.48779296875,
"loss": 1.1817,
"margin_dpo/margin_mean": 369.3795166015625,
"margin_dpo/margin_std": 569.4313354492188,
"step": 599
},
{
"KL/chosen_KL_mean": -844.0216064453125,
"KL/mean": -1028.517578125,
"KL/rejected_KL_mean": -1213.0135498046875,
"KL/std": 647.4127197265625,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0007366039790213108,
"fcm_dpo/delta": 0.08046019077301025,
"fcm_dpo/margin": 368.991943359375,
"fcm_dpo/q_t": 0.4391680657863617,
"grad_norm": 28.268569946289062,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -1.1482144594192505,
"logits/rejected": -1.1489068269729614,
"logps/chosen": -910.4564208984375,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1298.469970703125,
"loss": 1.2098,
"margin_dpo/margin_mean": 368.991943359375,
"margin_dpo/margin_std": 782.9544677734375,
"step": 600
},
{
"KL/chosen_KL_mean": -827.043701171875,
"KL/mean": -1054.9534912109375,
"KL/rejected_KL_mean": -1282.863037109375,
"KL/std": 646.4471435546875,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.0007413103012368083,
"fcm_dpo/delta": 0.012507637962698936,
"fcm_dpo/margin": 455.8194274902344,
"fcm_dpo/q_t": 0.42060860991477966,
"grad_norm": 33.074012756347656,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -1.059302568435669,
"logits/rejected": -1.0803344249725342,
"logps/chosen": -886.1773681640625,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1377.5540771484375,
"loss": 1.1316,
"margin_dpo/margin_mean": 455.81939697265625,
"margin_dpo/margin_std": 648.2555541992188,
"step": 601
},
{
"KL/chosen_KL_mean": -575.5924072265625,
"KL/mean": -873.7586669921875,
"KL/rejected_KL_mean": -1171.925048828125,
"KL/std": 629.0206298828125,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0007364454213529825,
"fcm_dpo/delta": -0.09336760640144348,
"fcm_dpo/margin": 596.3326416015625,
"fcm_dpo/q_t": 0.3994067311286926,
"grad_norm": 68.0593032836914,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -1.0809543132781982,
"logits/rejected": -1.1263089179992676,
"logps/chosen": -624.1859130859375,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -1259.593505859375,
"loss": 1.0572,
"margin_dpo/margin_mean": 596.3326416015625,
"margin_dpo/margin_std": 703.1005859375,
"step": 602
},
{
"KL/chosen_KL_mean": -738.450927734375,
"KL/mean": -995.5992431640625,
"KL/rejected_KL_mean": -1252.74755859375,
"KL/std": 682.3411865234375,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0007258434779942036,
"fcm_dpo/delta": -0.024222414940595627,
"fcm_dpo/margin": 514.296630859375,
"fcm_dpo/q_t": 0.41643890738487244,
"grad_norm": 37.625732421875,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -1.057875156402588,
"logits/rejected": -1.049300193786621,
"logps/chosen": -808.8655395507812,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -1353.0731201171875,
"loss": 1.1244,
"margin_dpo/margin_mean": 514.2965698242188,
"margin_dpo/margin_std": 806.4513549804688,
"step": 603
},
{
"KL/chosen_KL_mean": -721.03076171875,
"KL/mean": -1012.0377197265625,
"KL/rejected_KL_mean": -1303.0445556640625,
"KL/std": 684.166015625,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0007178003434091806,
"fcm_dpo/delta": -0.0707206279039383,
"fcm_dpo/margin": 582.013916015625,
"fcm_dpo/q_t": 0.406802773475647,
"grad_norm": 36.53494644165039,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -1.0587637424468994,
"logits/rejected": -1.1084504127502441,
"logps/chosen": -767.4888305664062,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1394.8990478515625,
"loss": 1.098,
"margin_dpo/margin_mean": 582.0138549804688,
"margin_dpo/margin_std": 858.50927734375,
"step": 604
},
{
"KL/chosen_KL_mean": -756.2188720703125,
"KL/mean": -961.7646484375,
"KL/rejected_KL_mean": -1167.310546875,
"KL/std": 611.2559814453125,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0007192428456619382,
"fcm_dpo/delta": 0.055984023958444595,
"fcm_dpo/margin": 411.09173583984375,
"fcm_dpo/q_t": 0.43355774879455566,
"grad_norm": 26.453216552734375,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -1.0269505977630615,
"logits/rejected": -1.0451889038085938,
"logps/chosen": -822.4681396484375,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1269.615478515625,
"loss": 1.1786,
"margin_dpo/margin_mean": 411.09173583984375,
"margin_dpo/margin_std": 747.6804809570312,
"step": 605
},
{
"KL/chosen_KL_mean": -741.6259765625,
"KL/mean": -970.2193603515625,
"KL/rejected_KL_mean": -1198.8126220703125,
"KL/std": 623.1279296875,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0007271652575582266,
"fcm_dpo/delta": 0.017948858439922333,
"fcm_dpo/margin": 457.1866760253906,
"fcm_dpo/q_t": 0.4236467778682709,
"grad_norm": 27.061796188354492,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -1.0122807025909424,
"logits/rejected": -1.0320955514907837,
"logps/chosen": -796.445068359375,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -1297.18408203125,
"loss": 1.1365,
"margin_dpo/margin_mean": 457.1866760253906,
"margin_dpo/margin_std": 670.32568359375,
"step": 606
},
{
"KL/chosen_KL_mean": -794.986083984375,
"KL/mean": -985.2572021484375,
"KL/rejected_KL_mean": -1175.5283203125,
"KL/std": 660.7117919921875,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0007356663700193167,
"fcm_dpo/delta": 0.0717119500041008,
"fcm_dpo/margin": 380.5422668457031,
"fcm_dpo/q_t": 0.43587759137153625,
"grad_norm": 24.40001106262207,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -1.0124876499176025,
"logits/rejected": -1.0030550956726074,
"logps/chosen": -853.070068359375,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -1255.305419921875,
"loss": 1.191,
"margin_dpo/margin_mean": 380.542236328125,
"margin_dpo/margin_std": 708.0584106445312,
"step": 607
},
{
"KL/chosen_KL_mean": -712.60595703125,
"KL/mean": -935.021484375,
"KL/rejected_KL_mean": -1157.4371337890625,
"KL/std": 576.3223876953125,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.0007404502248391509,
"fcm_dpo/delta": 0.02115270122885704,
"fcm_dpo/margin": 444.8311767578125,
"fcm_dpo/q_t": 0.4227282702922821,
"grad_norm": 26.049560546875,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -1.084040880203247,
"logits/rejected": -1.1021060943603516,
"logps/chosen": -770.0567626953125,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1252.21044921875,
"loss": 1.133,
"margin_dpo/margin_mean": 444.8311767578125,
"margin_dpo/margin_std": 611.6470947265625,
"step": 608
},
{
"KL/chosen_KL_mean": -711.9459228515625,
"KL/mean": -1001.8514404296875,
"KL/rejected_KL_mean": -1291.7567138671875,
"KL/std": 775.0626220703125,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.0007308817002922297,
"fcm_dpo/delta": -0.0770505964756012,
"fcm_dpo/margin": 579.810791015625,
"fcm_dpo/q_t": 0.408769428730011,
"grad_norm": 27.387683868408203,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -1.1206148862838745,
"logits/rejected": -1.1497302055358887,
"logps/chosen": -770.7513427734375,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1380.57275390625,
"loss": 1.104,
"margin_dpo/margin_mean": 579.810791015625,
"margin_dpo/margin_std": 940.2999877929688,
"step": 609
},
{
"KL/chosen_KL_mean": -716.533447265625,
"KL/mean": -895.5557861328125,
"KL/rejected_KL_mean": -1074.578125,
"KL/std": 590.9163818359375,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0007388823432847857,
"fcm_dpo/delta": 0.08757883310317993,
"fcm_dpo/margin": 358.044677734375,
"fcm_dpo/q_t": 0.43797656893730164,
"grad_norm": 45.32019805908203,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.9885549545288086,
"logits/rejected": -0.9654494524002075,
"logps/chosen": -782.228515625,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -1157.9835205078125,
"loss": 1.2023,
"margin_dpo/margin_mean": 358.044677734375,
"margin_dpo/margin_std": 698.3663330078125,
"step": 610
},
{
"KL/chosen_KL_mean": -749.7844848632812,
"KL/mean": -1059.1370849609375,
"KL/rejected_KL_mean": -1368.4896240234375,
"KL/std": 790.021728515625,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0007288827328011394,
"fcm_dpo/delta": -0.10581991076469421,
"fcm_dpo/margin": 618.7051391601562,
"fcm_dpo/q_t": 0.4007767140865326,
"grad_norm": 31.054868698120117,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -1.0566998720169067,
"logits/rejected": -1.1077455282211304,
"logps/chosen": -802.3839111328125,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1454.820556640625,
"loss": 1.0823,
"margin_dpo/margin_mean": 618.7051391601562,
"margin_dpo/margin_std": 904.1781616210938,
"step": 611
},
{
"KL/chosen_KL_mean": -812.09033203125,
"KL/mean": -1034.2083740234375,
"KL/rejected_KL_mean": -1256.326416015625,
"KL/std": 622.982177734375,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0007307927589863539,
"fcm_dpo/delta": 0.02587536908686161,
"fcm_dpo/margin": 444.236083984375,
"fcm_dpo/q_t": 0.42509013414382935,
"grad_norm": 33.46781539916992,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -1.0426099300384521,
"logits/rejected": -1.0452769994735718,
"logps/chosen": -871.4140625,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1344.638916015625,
"loss": 1.1473,
"margin_dpo/margin_mean": 444.23602294921875,
"margin_dpo/margin_std": 683.746337890625,
"step": 612
},
{
"KL/chosen_KL_mean": -720.933837890625,
"KL/mean": -993.418212890625,
"KL/rejected_KL_mean": -1265.902587890625,
"KL/std": 689.9868774414062,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0007219946710392833,
"fcm_dpo/delta": -0.045596893876791,
"fcm_dpo/margin": 544.9688110351562,
"fcm_dpo/q_t": 0.40862947702407837,
"grad_norm": 33.90407180786133,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -1.073573112487793,
"logits/rejected": -1.12931227684021,
"logps/chosen": -780.663818359375,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1371.01025390625,
"loss": 1.0944,
"margin_dpo/margin_mean": 544.9688110351562,
"margin_dpo/margin_std": 720.446044921875,
"step": 613
},
{
"KL/chosen_KL_mean": -778.2044677734375,
"KL/mean": -1059.3028564453125,
"KL/rejected_KL_mean": -1340.4013671875,
"KL/std": 686.830078125,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0007176370127126575,
"fcm_dpo/delta": -0.05566265806555748,
"fcm_dpo/margin": 562.1969604492188,
"fcm_dpo/q_t": 0.40819481015205383,
"grad_norm": 38.53693771362305,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -1.0261731147766113,
"logits/rejected": -1.1077706813812256,
"logps/chosen": -831.1434326171875,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1445.080810546875,
"loss": 1.1008,
"margin_dpo/margin_mean": 562.1968994140625,
"margin_dpo/margin_std": 811.7864379882812,
"step": 614
},
{
"KL/chosen_KL_mean": -728.3233642578125,
"KL/mean": -960.8916015625,
"KL/rejected_KL_mean": -1193.4598388671875,
"KL/std": 708.7926025390625,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0007183545967563987,
"fcm_dpo/delta": 0.016210701316595078,
"fcm_dpo/margin": 465.13641357421875,
"fcm_dpo/q_t": 0.4231938123703003,
"grad_norm": 28.220327377319336,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -1.0408499240875244,
"logits/rejected": -1.0693387985229492,
"logps/chosen": -794.1406860351562,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1288.6373291015625,
"loss": 1.15,
"margin_dpo/margin_mean": 465.13641357421875,
"margin_dpo/margin_std": 769.0079956054688,
"step": 615
},
{
"KL/chosen_KL_mean": -863.922119140625,
"KL/mean": -1049.33056640625,
"KL/rejected_KL_mean": -1234.73876953125,
"KL/std": 591.33251953125,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0007266043685376644,
"fcm_dpo/delta": 0.08268511295318604,
"fcm_dpo/margin": 370.8167419433594,
"fcm_dpo/q_t": 0.4374980330467224,
"grad_norm": 35.82476043701172,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -1.0865511894226074,
"logits/rejected": -1.0711218118667603,
"logps/chosen": -929.0549926757812,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1309.439453125,
"loss": 1.1912,
"margin_dpo/margin_mean": 370.8167724609375,
"margin_dpo/margin_std": 663.240234375,
"step": 616
},
{
"KL/chosen_KL_mean": -840.3502197265625,
"KL/mean": -974.4063720703125,
"KL/rejected_KL_mean": -1108.4625244140625,
"KL/std": 581.7191772460938,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0007288898341357708,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 268.1123046875,
"fcm_dpo/q_t": 0.4547814726829529,
"grad_norm": 50.21003723144531,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -1.0201683044433594,
"logits/rejected": -0.9916302561759949,
"logps/chosen": -903.3557739257812,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -1172.6966552734375,
"loss": 1.2642,
"margin_dpo/margin_mean": 268.1123046875,
"margin_dpo/margin_std": 707.6526489257812,
"step": 617
},
{
"KL/chosen_KL_mean": -859.969482421875,
"KL/mean": -1152.5023193359375,
"KL/rejected_KL_mean": -1445.03515625,
"KL/std": 750.560791015625,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0007248672191053629,
"fcm_dpo/delta": -0.07764283567667007,
"fcm_dpo/margin": 585.065673828125,
"fcm_dpo/q_t": 0.4056781530380249,
"grad_norm": 38.26071548461914,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -1.059650182723999,
"logits/rejected": -1.103996753692627,
"logps/chosen": -927.0708618164062,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1537.1884765625,
"loss": 1.1236,
"margin_dpo/margin_mean": 585.065673828125,
"margin_dpo/margin_std": 982.0968017578125,
"step": 618
},
{
"KL/chosen_KL_mean": -856.5992431640625,
"KL/mean": -1078.2330322265625,
"KL/rejected_KL_mean": -1299.86669921875,
"KL/std": 716.7399291992188,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0007212937925942242,
"fcm_dpo/delta": 0.03125857189297676,
"fcm_dpo/margin": 443.2675476074219,
"fcm_dpo/q_t": 0.4321790337562561,
"grad_norm": 46.859371185302734,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -1.0055409669876099,
"logits/rejected": -1.019460916519165,
"logps/chosen": -912.5774536132812,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1393.0521240234375,
"loss": 1.2044,
"margin_dpo/margin_mean": 443.2675476074219,
"margin_dpo/margin_std": 974.4349365234375,
"step": 619
},
{
"KL/chosen_KL_mean": -782.436279296875,
"KL/mean": -988.9633178710938,
"KL/rejected_KL_mean": -1195.490478515625,
"KL/std": 634.9549560546875,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0007255082600750029,
"fcm_dpo/delta": 0.051843997091054916,
"fcm_dpo/margin": 413.05419921875,
"fcm_dpo/q_t": 0.4310418963432312,
"grad_norm": 33.33245086669922,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -1.0578522682189941,
"logits/rejected": -1.0688400268554688,
"logps/chosen": -842.2337646484375,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1273.9012451171875,
"loss": 1.1738,
"margin_dpo/margin_mean": 413.05419921875,
"margin_dpo/margin_std": 718.75439453125,
"step": 620
},
{
"KL/chosen_KL_mean": -774.94873046875,
"KL/mean": -1094.044677734375,
"KL/rejected_KL_mean": -1413.1407470703125,
"KL/std": 769.3406982421875,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.0007223983411677182,
"fcm_dpo/delta": -0.11662234365940094,
"fcm_dpo/margin": 638.1920166015625,
"fcm_dpo/q_t": 0.3942115306854248,
"grad_norm": 33.19838333129883,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -1.1112885475158691,
"logits/rejected": -1.1674792766571045,
"logps/chosen": -828.8825073242188,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1501.51025390625,
"loss": 1.0704,
"margin_dpo/margin_mean": 638.1920166015625,
"margin_dpo/margin_std": 859.8441772460938,
"step": 621
},
{
"KL/chosen_KL_mean": -750.149169921875,
"KL/mean": -967.1200561523438,
"KL/rejected_KL_mean": -1184.0909423828125,
"KL/std": 593.5496826171875,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0007141625974327326,
"fcm_dpo/delta": 0.04117283225059509,
"fcm_dpo/margin": 433.9416809082031,
"fcm_dpo/q_t": 0.4296455979347229,
"grad_norm": 34.48857116699219,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.9647907018661499,
"logits/rejected": -0.9519217014312744,
"logps/chosen": -810.43505859375,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1269.609619140625,
"loss": 1.1662,
"margin_dpo/margin_mean": 433.9416809082031,
"margin_dpo/margin_std": 747.2349853515625,
"step": 622
},
{
"KL/chosen_KL_mean": -820.90869140625,
"KL/mean": -982.7512817382812,
"KL/rejected_KL_mean": -1144.5938720703125,
"KL/std": 598.2750244140625,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0007311557419598103,
"fcm_dpo/delta": 0.11611227691173553,
"fcm_dpo/margin": 323.68511962890625,
"fcm_dpo/q_t": 0.446946382522583,
"grad_norm": 42.85483932495117,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -1.119500756263733,
"logits/rejected": -1.1187694072723389,
"logps/chosen": -885.065673828125,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1229.6768798828125,
"loss": 1.2308,
"margin_dpo/margin_mean": 323.68511962890625,
"margin_dpo/margin_std": 731.9136962890625,
"step": 623
},
{
"KL/chosen_KL_mean": -821.5892944335938,
"KL/mean": -1092.430419921875,
"KL/rejected_KL_mean": -1363.271484375,
"KL/std": 638.8692016601562,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0007304972968995571,
"fcm_dpo/delta": -0.047577153891325,
"fcm_dpo/margin": 541.6821899414062,
"fcm_dpo/q_t": 0.4065777659416199,
"grad_norm": 38.01239776611328,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -1.1294810771942139,
"logits/rejected": -1.137779951095581,
"logps/chosen": -893.5079345703125,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1460.403564453125,
"loss": 1.0894,
"margin_dpo/margin_mean": 541.6821899414062,
"margin_dpo/margin_std": 687.8084716796875,
"step": 624
},
{
"KL/chosen_KL_mean": -741.5074462890625,
"KL/mean": -1046.6978759765625,
"KL/rejected_KL_mean": -1351.88818359375,
"KL/std": 684.56396484375,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0007204815046861768,
"fcm_dpo/delta": -0.09379212558269501,
"fcm_dpo/margin": 610.380859375,
"fcm_dpo/q_t": 0.39741039276123047,
"grad_norm": 39.902103424072266,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -1.004154920578003,
"logits/rejected": -1.0372995138168335,
"logps/chosen": -799.8494873046875,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1437.978515625,
"loss": 1.0431,
"margin_dpo/margin_mean": 610.3807983398438,
"margin_dpo/margin_std": 639.5909423828125,
"step": 625
},
{
"KL/chosen_KL_mean": -940.815185546875,
"KL/mean": -1116.69189453125,
"KL/rejected_KL_mean": -1292.5687255859375,
"KL/std": 766.899169921875,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0007251551141962409,
"fcm_dpo/delta": 0.09746433794498444,
"fcm_dpo/margin": 351.75341796875,
"fcm_dpo/q_t": 0.439464807510376,
"grad_norm": 44.423484802246094,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.9951568841934204,
"logits/rejected": -0.9892057776451111,
"logps/chosen": -1015.9278564453125,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1391.7574462890625,
"loss": 1.2212,
"margin_dpo/margin_mean": 351.75341796875,
"margin_dpo/margin_std": 743.8319702148438,
"step": 626
},
{
"KL/chosen_KL_mean": -655.391357421875,
"KL/mean": -983.7683715820312,
"KL/rejected_KL_mean": -1312.145263671875,
"KL/std": 831.3623046875,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0007200604304671288,
"fcm_dpo/delta": -0.12902742624282837,
"fcm_dpo/margin": 656.75390625,
"fcm_dpo/q_t": 0.40084049105644226,
"grad_norm": 29.196271896362305,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.9449664354324341,
"logits/rejected": -1.0268559455871582,
"logps/chosen": -703.134521484375,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1418.8997802734375,
"loss": 1.0784,
"margin_dpo/margin_mean": 656.7538452148438,
"margin_dpo/margin_std": 977.4828491210938,
"step": 627
},
{
"KL/chosen_KL_mean": -853.753662109375,
"KL/mean": -1100.697265625,
"KL/rejected_KL_mean": -1347.640869140625,
"KL/std": 691.8834228515625,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0007095912005752325,
"fcm_dpo/delta": -0.0005148975178599358,
"fcm_dpo/margin": 493.88702392578125,
"fcm_dpo/q_t": 0.4203591048717499,
"grad_norm": 30.30500602722168,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -1.106518268585205,
"logits/rejected": -1.131546139717102,
"logps/chosen": -913.9365844726562,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1449.1954345703125,
"loss": 1.1425,
"margin_dpo/margin_mean": 493.88702392578125,
"margin_dpo/margin_std": 802.3726196289062,
"step": 628
},
{
"KL/chosen_KL_mean": -840.9974365234375,
"KL/mean": -1075.9169921875,
"KL/rejected_KL_mean": -1310.8365478515625,
"KL/std": 669.786376953125,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0007125267293304205,
"fcm_dpo/delta": 0.015744555741548538,
"fcm_dpo/margin": 469.839111328125,
"fcm_dpo/q_t": 0.42130935192108154,
"grad_norm": 34.061126708984375,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -1.0905547142028809,
"logits/rejected": -1.110518455505371,
"logps/chosen": -905.2109375,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1402.490234375,
"loss": 1.1358,
"margin_dpo/margin_mean": 469.839111328125,
"margin_dpo/margin_std": 684.5360107421875,
"step": 629
},
{
"KL/chosen_KL_mean": -732.7456665039062,
"KL/mean": -860.0938720703125,
"KL/rejected_KL_mean": -987.442138671875,
"KL/std": 659.4423828125,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.000715771981049329,
"fcm_dpo/delta": 0.04146602377295494,
"fcm_dpo/margin": 254.6964874267578,
"fcm_dpo/q_t": 0.46173208951950073,
"grad_norm": 44.93278503417969,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -1.049222469329834,
"logits/rejected": -1.0251140594482422,
"logps/chosen": -792.0366821289062,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -1071.04052734375,
"loss": 1.2901,
"margin_dpo/margin_mean": 254.6964874267578,
"margin_dpo/margin_std": 802.978271484375,
"step": 630
},
{
"KL/chosen_KL_mean": -897.2791748046875,
"KL/mean": -1110.93359375,
"KL/rejected_KL_mean": -1324.5877685546875,
"KL/std": 662.3657836914062,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.0007242325227707624,
"fcm_dpo/delta": 0.041726578027009964,
"fcm_dpo/margin": 427.30865478515625,
"fcm_dpo/q_t": 0.4278268814086914,
"grad_norm": 31.293115615844727,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -1.1521632671356201,
"logits/rejected": -1.1659929752349854,
"logps/chosen": -956.7327880859375,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1405.539306640625,
"loss": 1.174,
"margin_dpo/margin_mean": 427.3086242675781,
"margin_dpo/margin_std": 762.1250610351562,
"step": 631
},
{
"KL/chosen_KL_mean": -787.10400390625,
"KL/mean": -995.85400390625,
"KL/rejected_KL_mean": -1204.6038818359375,
"KL/std": 610.8785400390625,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0007272514048963785,
"fcm_dpo/delta": 0.04772178828716278,
"fcm_dpo/margin": 417.49993896484375,
"fcm_dpo/q_t": 0.4296777546405792,
"grad_norm": 37.82929229736328,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -1.1057119369506836,
"logits/rejected": -1.1097162961959839,
"logps/chosen": -848.45556640625,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1290.7640380859375,
"loss": 1.1786,
"margin_dpo/margin_mean": 417.49993896484375,
"margin_dpo/margin_std": 755.839111328125,
"step": 632
},
{
"KL/chosen_KL_mean": -875.8489379882812,
"KL/mean": -1027.587890625,
"KL/rejected_KL_mean": -1179.326904296875,
"KL/std": 598.5611572265625,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.0007348891813308001,
"fcm_dpo/delta": 0.043210797011852264,
"fcm_dpo/margin": 303.47796630859375,
"fcm_dpo/q_t": 0.44790202379226685,
"grad_norm": 37.137454986572266,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -1.1341545581817627,
"logits/rejected": -1.1472094058990479,
"logps/chosen": -933.1270751953125,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1270.910888671875,
"loss": 1.2351,
"margin_dpo/margin_mean": 303.4779968261719,
"margin_dpo/margin_std": 675.3363037109375,
"step": 633
},
{
"KL/chosen_KL_mean": -960.07275390625,
"KL/mean": -1097.663330078125,
"KL/rejected_KL_mean": -1235.254150390625,
"KL/std": 704.655517578125,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0007380647002719343,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 275.1812744140625,
"fcm_dpo/q_t": 0.4533958435058594,
"grad_norm": 27.810382843017578,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -1.0175862312316895,
"logits/rejected": -1.0139384269714355,
"logps/chosen": -1026.6917724609375,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1342.379638671875,
"loss": 1.2542,
"margin_dpo/margin_mean": 275.1812744140625,
"margin_dpo/margin_std": 683.193115234375,
"step": 634
},
{
"KL/chosen_KL_mean": -855.4681396484375,
"KL/mean": -1050.290771484375,
"KL/rejected_KL_mean": -1245.11328125,
"KL/std": 766.2486572265625,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0007452410645782948,
"fcm_dpo/delta": 0.06136660277843475,
"fcm_dpo/margin": 389.6451721191406,
"fcm_dpo/q_t": 0.4358566403388977,
"grad_norm": 46.93721008300781,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -1.081568717956543,
"logits/rejected": -1.064734697341919,
"logps/chosen": -929.4236450195312,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1327.61376953125,
"loss": 1.1961,
"margin_dpo/margin_mean": 389.64520263671875,
"margin_dpo/margin_std": 794.4766235351562,
"step": 635
},
{
"KL/chosen_KL_mean": -784.0025634765625,
"KL/mean": -1020.8980712890625,
"KL/rejected_KL_mean": -1257.793701171875,
"KL/std": 716.466064453125,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0007509600836783648,
"fcm_dpo/delta": -0.00671178475022316,
"fcm_dpo/margin": 473.79107666015625,
"fcm_dpo/q_t": 0.42060697078704834,
"grad_norm": 27.554264068603516,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.9968219995498657,
"logits/rejected": -0.9979686141014099,
"logps/chosen": -843.6314697265625,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1339.7724609375,
"loss": 1.1335,
"margin_dpo/margin_mean": 473.79107666015625,
"margin_dpo/margin_std": 754.580810546875,
"step": 636
},
{
"KL/chosen_KL_mean": -779.475341796875,
"KL/mean": -1054.064453125,
"KL/rejected_KL_mean": -1328.653564453125,
"KL/std": 737.46728515625,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0007367645157501101,
"fcm_dpo/delta": -0.05750354379415512,
"fcm_dpo/margin": 549.1782836914062,
"fcm_dpo/q_t": 0.4083985686302185,
"grad_norm": 25.79489517211914,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -1.1218140125274658,
"logits/rejected": -1.1801137924194336,
"logps/chosen": -829.1279907226562,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1427.0587158203125,
"loss": 1.0945,
"margin_dpo/margin_mean": 549.1782836914062,
"margin_dpo/margin_std": 762.412841796875,
"step": 637
},
{
"KL/chosen_KL_mean": -761.9863891601562,
"KL/mean": -976.8961791992188,
"KL/rejected_KL_mean": -1191.8060302734375,
"KL/std": 677.3199462890625,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0007319189608097076,
"fcm_dpo/delta": -0.07822597026824951,
"fcm_dpo/margin": 429.8195495605469,
"fcm_dpo/q_t": 0.4225819408893585,
"grad_norm": 34.85166931152344,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.981365978717804,
"logits/rejected": -0.9689379334449768,
"logps/chosen": -820.14306640625,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1271.107421875,
"loss": 1.1841,
"margin_dpo/margin_mean": 429.8195495605469,
"margin_dpo/margin_std": 802.29248046875,
"step": 638
},
{
"KL/chosen_KL_mean": -1073.5750732421875,
"KL/mean": -1151.7091064453125,
"KL/rejected_KL_mean": -1229.84326171875,
"KL/std": 669.1240844726562,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0007261935388669372,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 156.26809692382812,
"fcm_dpo/q_t": 0.47418278455734253,
"grad_norm": 120.29761505126953,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.9551519155502319,
"logits/rejected": -0.8928138017654419,
"logps/chosen": -1145.8983154296875,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1304.1181640625,
"loss": 1.3672,
"margin_dpo/margin_mean": 156.26809692382812,
"margin_dpo/margin_std": 847.606201171875,
"step": 639
},
{
"KL/chosen_KL_mean": -795.9225463867188,
"KL/mean": -1108.8028564453125,
"KL/rejected_KL_mean": -1421.683349609375,
"KL/std": 764.7296752929688,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0007176444632932544,
"fcm_dpo/delta": -0.10362177342176437,
"fcm_dpo/margin": 625.7607421875,
"fcm_dpo/q_t": 0.40179121494293213,
"grad_norm": 35.442649841308594,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.9908477663993835,
"logits/rejected": -1.0309135913848877,
"logps/chosen": -852.056884765625,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1530.283447265625,
"loss": 1.0819,
"margin_dpo/margin_mean": 625.7606201171875,
"margin_dpo/margin_std": 917.645751953125,
"step": 640
},
{
"KL/chosen_KL_mean": -917.562744140625,
"KL/mean": -1132.0595703125,
"KL/rejected_KL_mean": -1346.556396484375,
"KL/std": 655.5465087890625,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.000714754918590188,
"fcm_dpo/delta": 0.04476023092865944,
"fcm_dpo/margin": 428.99371337890625,
"fcm_dpo/q_t": 0.43139177560806274,
"grad_norm": 29.970199584960938,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -1.057366132736206,
"logits/rejected": -1.0727591514587402,
"logps/chosen": -982.5596313476562,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1433.548828125,
"loss": 1.1919,
"margin_dpo/margin_mean": 428.99371337890625,
"margin_dpo/margin_std": 868.6700439453125,
"step": 641
},
{
"KL/chosen_KL_mean": -880.2049560546875,
"KL/mean": -1136.517333984375,
"KL/rejected_KL_mean": -1392.829833984375,
"KL/std": 870.1756591796875,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0007182994158938527,
"fcm_dpo/delta": -0.01912054233253002,
"fcm_dpo/margin": 512.6249389648438,
"fcm_dpo/q_t": 0.42391371726989746,
"grad_norm": 37.84080505371094,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.9898715019226074,
"logits/rejected": -1.0340471267700195,
"logps/chosen": -945.8941650390625,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1503.0718994140625,
"loss": 1.1715,
"margin_dpo/margin_mean": 512.625,
"margin_dpo/margin_std": 1061.2357177734375,
"step": 642
},
{
"KL/chosen_KL_mean": -796.40869140625,
"KL/mean": -983.58837890625,
"KL/rejected_KL_mean": -1170.7681884765625,
"KL/std": 615.07666015625,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0007112710736691952,
"fcm_dpo/delta": -0.025832796469330788,
"fcm_dpo/margin": 374.35943603515625,
"fcm_dpo/q_t": 0.43726229667663574,
"grad_norm": 34.36274719238281,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -1.1126079559326172,
"logits/rejected": -1.1123908758163452,
"logps/chosen": -848.3587036132812,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1258.236572265625,
"loss": 1.1891,
"margin_dpo/margin_mean": 374.3594665527344,
"margin_dpo/margin_std": 623.792724609375,
"step": 643
},
{
"KL/chosen_KL_mean": -790.0933227539062,
"KL/mean": -986.8052978515625,
"KL/rejected_KL_mean": -1183.517333984375,
"KL/std": 741.110107421875,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0007180780521593988,
"fcm_dpo/delta": 0.0694977343082428,
"fcm_dpo/margin": 393.4239807128906,
"fcm_dpo/q_t": 0.43679407238960266,
"grad_norm": 47.92481231689453,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -1.0378338098526,
"logits/rejected": -1.0484693050384521,
"logps/chosen": -849.1109619140625,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1270.654052734375,
"loss": 1.2172,
"margin_dpo/margin_mean": 393.4239501953125,
"margin_dpo/margin_std": 875.6864013671875,
"step": 644
},
{
"KL/chosen_KL_mean": -823.27685546875,
"KL/mean": -912.3017578125,
"KL/rejected_KL_mean": -1001.3265380859375,
"KL/std": 572.473876953125,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.0007212264463305473,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.04962158203125,
"fcm_dpo/q_t": 0.47089576721191406,
"grad_norm": 86.09916687011719,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -1.0229761600494385,
"logits/rejected": -1.0112075805664062,
"logps/chosen": -879.1529541015625,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -1099.1072998046875,
"loss": 1.3404,
"margin_dpo/margin_mean": 178.04962158203125,
"margin_dpo/margin_std": 762.8621826171875,
"step": 645
},
{
"KL/chosen_KL_mean": -781.0083618164062,
"KL/mean": -941.683837890625,
"KL/rejected_KL_mean": -1102.3594970703125,
"KL/std": 565.9122924804688,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0007259917329065502,
"fcm_dpo/delta": 0.032927513122558594,
"fcm_dpo/margin": 321.35107421875,
"fcm_dpo/q_t": 0.4461080729961395,
"grad_norm": 55.66950225830078,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -1.0636550188064575,
"logits/rejected": -1.0476542711257935,
"logps/chosen": -842.2841796875,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -1179.865234375,
"loss": 1.2275,
"margin_dpo/margin_mean": 321.3511047363281,
"margin_dpo/margin_std": 680.1424560546875,
"step": 646
},
{
"KL/chosen_KL_mean": -680.39453125,
"KL/mean": -889.831787109375,
"KL/rejected_KL_mean": -1099.26904296875,
"KL/std": 623.091796875,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0007311520748771727,
"fcm_dpo/delta": 0.04510268568992615,
"fcm_dpo/margin": 418.8743896484375,
"fcm_dpo/q_t": 0.4280046820640564,
"grad_norm": 27.29217529296875,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.9219083786010742,
"logits/rejected": -0.95496666431427,
"logps/chosen": -735.2469482421875,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1192.788330078125,
"loss": 1.1548,
"margin_dpo/margin_mean": 418.8743896484375,
"margin_dpo/margin_std": 625.203857421875,
"step": 647
},
{
"KL/chosen_KL_mean": -736.6182861328125,
"KL/mean": -1046.520263671875,
"KL/rejected_KL_mean": -1356.42236328125,
"KL/std": 752.701904296875,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.0007215134100988507,
"fcm_dpo/delta": -0.10175478458404541,
"fcm_dpo/margin": 619.80419921875,
"fcm_dpo/q_t": 0.4019627571105957,
"grad_norm": 25.925161361694336,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -1.08555269241333,
"logits/rejected": -1.1510529518127441,
"logps/chosen": -790.7897338867188,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1455.13525390625,
"loss": 1.0815,
"margin_dpo/margin_mean": 619.80419921875,
"margin_dpo/margin_std": 915.1447143554688,
"step": 648
},
{
"KL/chosen_KL_mean": -784.3709106445312,
"KL/mean": -931.5806884765625,
"KL/rejected_KL_mean": -1078.79052734375,
"KL/std": 599.5159301757812,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0007213630015030503,
"fcm_dpo/delta": 0.04922113195061684,
"fcm_dpo/margin": 294.419677734375,
"fcm_dpo/q_t": 0.4535311460494995,
"grad_norm": 27.51808738708496,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -1.1117441654205322,
"logits/rejected": -1.1074135303497314,
"logps/chosen": -846.8512573242188,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -1158.86767578125,
"loss": 1.2587,
"margin_dpo/margin_mean": 294.4196472167969,
"margin_dpo/margin_std": 746.974853515625,
"step": 649
},
{
"KL/chosen_KL_mean": -838.1929931640625,
"KL/mean": -1064.818359375,
"KL/rejected_KL_mean": -1291.44384765625,
"KL/std": 749.1234130859375,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.0007260285783559084,
"fcm_dpo/delta": 0.021617352962493896,
"fcm_dpo/margin": 453.2508544921875,
"fcm_dpo/q_t": 0.4258856177330017,
"grad_norm": 36.681617736816406,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -1.0278213024139404,
"logits/rejected": -1.050445795059204,
"logps/chosen": -894.2857666015625,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1389.708740234375,
"loss": 1.175,
"margin_dpo/margin_mean": 453.2508544921875,
"margin_dpo/margin_std": 861.7562255859375,
"step": 650
},
{
"KL/chosen_KL_mean": -549.6793212890625,
"KL/mean": -854.8587036132812,
"KL/rejected_KL_mean": -1160.0380859375,
"KL/std": 682.4044799804688,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0007174712372943759,
"fcm_dpo/delta": -0.09204652160406113,
"fcm_dpo/margin": 610.3588256835938,
"fcm_dpo/q_t": 0.39922603964805603,
"grad_norm": 39.13344955444336,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -1.0335227251052856,
"logits/rejected": -1.0952156782150269,
"logps/chosen": -593.104736328125,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1259.99609375,
"loss": 1.055,
"margin_dpo/margin_mean": 610.3587646484375,
"margin_dpo/margin_std": 714.5980224609375,
"step": 651
},
{
"KL/chosen_KL_mean": -708.1835327148438,
"KL/mean": -933.1541137695312,
"KL/rejected_KL_mean": -1158.1246337890625,
"KL/std": 705.0360107421875,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0007186533184722066,
"fcm_dpo/delta": 0.0274675115942955,
"fcm_dpo/margin": 449.94110107421875,
"fcm_dpo/q_t": 0.42639029026031494,
"grad_norm": 38.71635818481445,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -1.0511879920959473,
"logits/rejected": -1.0809803009033203,
"logps/chosen": -770.7603759765625,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1269.8924560546875,
"loss": 1.1698,
"margin_dpo/margin_mean": 449.941162109375,
"margin_dpo/margin_std": 828.396240234375,
"step": 652
},
{
"KL/chosen_KL_mean": -871.048583984375,
"KL/mean": -1087.985107421875,
"KL/rejected_KL_mean": -1304.921630859375,
"KL/std": 736.3079833984375,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0007236155215650797,
"fcm_dpo/delta": 0.03713195025920868,
"fcm_dpo/margin": 433.8730773925781,
"fcm_dpo/q_t": 0.4294109344482422,
"grad_norm": 38.670223236083984,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -1.1091747283935547,
"logits/rejected": -1.138980746269226,
"logps/chosen": -932.16162109375,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1408.17138671875,
"loss": 1.1733,
"margin_dpo/margin_mean": 433.8730773925781,
"margin_dpo/margin_std": 794.478271484375,
"step": 653
},
{
"KL/chosen_KL_mean": -779.94384765625,
"KL/mean": -982.3485107421875,
"KL/rejected_KL_mean": -1184.753173828125,
"KL/std": 596.6171875,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.0007300974684767425,
"fcm_dpo/delta": 0.05611284077167511,
"fcm_dpo/margin": 404.8092041015625,
"fcm_dpo/q_t": 0.4334927797317505,
"grad_norm": 37.371498107910156,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -1.0734694004058838,
"logits/rejected": -1.092237949371338,
"logps/chosen": -841.6719970703125,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1283.5269775390625,
"loss": 1.1781,
"margin_dpo/margin_mean": 404.8092346191406,
"margin_dpo/margin_std": 726.0567626953125,
"step": 654
},
{
"KL/chosen_KL_mean": -724.4075317382812,
"KL/mean": -935.876220703125,
"KL/rejected_KL_mean": -1147.344970703125,
"KL/std": 602.5899658203125,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0007367508951574564,
"fcm_dpo/delta": 0.03962551802396774,
"fcm_dpo/margin": 422.9373779296875,
"fcm_dpo/q_t": 0.4289171099662781,
"grad_norm": 30.014909744262695,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -1.055437684059143,
"logits/rejected": -1.1015443801879883,
"logps/chosen": -773.984375,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1245.63671875,
"loss": 1.166,
"margin_dpo/margin_mean": 422.9373779296875,
"margin_dpo/margin_std": 725.2291259765625,
"step": 655
},
{
"KL/chosen_KL_mean": -779.2088623046875,
"KL/mean": -1150.2603759765625,
"KL/rejected_KL_mean": -1521.31201171875,
"KL/std": 814.0103759765625,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0007195958169177175,
"fcm_dpo/delta": -0.1939731240272522,
"fcm_dpo/margin": 742.10302734375,
"fcm_dpo/q_t": 0.3803486227989197,
"grad_norm": 68.15776824951172,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.9589822888374329,
"logits/rejected": -1.0362895727157593,
"logps/chosen": -831.75830078125,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1634.986572265625,
"loss": 1.0025,
"margin_dpo/margin_mean": 742.10302734375,
"margin_dpo/margin_std": 839.2947998046875,
"step": 656
},
{
"KL/chosen_KL_mean": -735.045654296875,
"KL/mean": -1036.506591796875,
"KL/rejected_KL_mean": -1337.9677734375,
"KL/std": 803.0293579101562,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.00070092489477247,
"fcm_dpo/delta": -0.07597053050994873,
"fcm_dpo/margin": 602.922119140625,
"fcm_dpo/q_t": 0.4036826491355896,
"grad_norm": 41.26325225830078,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -1.0972576141357422,
"logits/rejected": -1.155198335647583,
"logps/chosen": -781.7461547851562,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1435.882568359375,
"loss": 1.0879,
"margin_dpo/margin_mean": 602.922119140625,
"margin_dpo/margin_std": 850.9063720703125,
"step": 657
},
{
"KL/chosen_KL_mean": -768.9866943359375,
"KL/mean": -999.5380249023438,
"KL/rejected_KL_mean": -1230.08935546875,
"KL/std": 592.4909057617188,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0006991151021793485,
"fcm_dpo/delta": 0.02827742137014866,
"fcm_dpo/margin": 461.10272216796875,
"fcm_dpo/q_t": 0.42443007230758667,
"grad_norm": 35.22886276245117,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -1.0550436973571777,
"logits/rejected": -1.0875539779663086,
"logps/chosen": -829.9449462890625,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1326.02880859375,
"loss": 1.1446,
"margin_dpo/margin_mean": 461.10272216796875,
"margin_dpo/margin_std": 672.0286865234375,
"step": 658
},
{
"KL/chosen_KL_mean": -714.39892578125,
"KL/mean": -941.6336059570312,
"KL/rejected_KL_mean": -1168.868408203125,
"KL/std": 622.14306640625,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0007049725973047316,
"fcm_dpo/delta": 0.0305656585842371,
"fcm_dpo/margin": 454.4694519042969,
"fcm_dpo/q_t": 0.4260675311088562,
"grad_norm": 35.09855270385742,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.985196053981781,
"logits/rejected": -0.9553881883621216,
"logps/chosen": -791.1419067382812,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1256.33935546875,
"loss": 1.1443,
"margin_dpo/margin_mean": 454.469482421875,
"margin_dpo/margin_std": 673.2275390625,
"step": 659
},
{
"KL/chosen_KL_mean": -742.4683837890625,
"KL/mean": -1035.2835693359375,
"KL/rejected_KL_mean": -1328.0987548828125,
"KL/std": 711.6279296875,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0007018651231192052,
"fcm_dpo/delta": -0.0636008009314537,
"fcm_dpo/margin": 585.63037109375,
"fcm_dpo/q_t": 0.4052135944366455,
"grad_norm": 43.428226470947266,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -1.0797405242919922,
"logits/rejected": -1.1055940389633179,
"logps/chosen": -801.5162963867188,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1404.058837890625,
"loss": 1.0847,
"margin_dpo/margin_mean": 585.63037109375,
"margin_dpo/margin_std": 770.469970703125,
"step": 660
},
{
"KL/chosen_KL_mean": -663.244140625,
"KL/mean": -936.0105590820312,
"KL/rejected_KL_mean": -1208.77685546875,
"KL/std": 813.6663818359375,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.0006977942539379001,
"fcm_dpo/delta": -0.03203122317790985,
"fcm_dpo/margin": 545.5327758789062,
"fcm_dpo/q_t": 0.4161611795425415,
"grad_norm": 36.02547836303711,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -1.0755589008331299,
"logits/rejected": -1.100312352180481,
"logps/chosen": -713.9181518554688,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -1294.7825927734375,
"loss": 1.112,
"margin_dpo/margin_mean": 545.5327758789062,
"margin_dpo/margin_std": 835.273681640625,
"step": 661
},
{
"KL/chosen_KL_mean": -777.9444580078125,
"KL/mean": -982.4157104492188,
"KL/rejected_KL_mean": -1186.886962890625,
"KL/std": 655.0043334960938,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0007006356026977301,
"fcm_dpo/delta": 0.06537885963916779,
"fcm_dpo/margin": 408.9425048828125,
"fcm_dpo/q_t": 0.4345260560512543,
"grad_norm": 26.648513793945312,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -1.0024073123931885,
"logits/rejected": -1.0070304870605469,
"logps/chosen": -847.20556640625,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1275.94287109375,
"loss": 1.1959,
"margin_dpo/margin_mean": 408.9425048828125,
"margin_dpo/margin_std": 810.3031005859375,
"step": 662
},
{
"KL/chosen_KL_mean": -734.29833984375,
"KL/mean": -990.4869995117188,
"KL/rejected_KL_mean": -1246.6756591796875,
"KL/std": 746.890625,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.000701136770658195,
"fcm_dpo/delta": -0.009660286828875542,
"fcm_dpo/margin": 512.3773193359375,
"fcm_dpo/q_t": 0.42002010345458984,
"grad_norm": 25.236740112304688,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -1.055614709854126,
"logits/rejected": -1.0850775241851807,
"logps/chosen": -799.17724609375,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1360.60107421875,
"loss": 1.1439,
"margin_dpo/margin_mean": 512.3772583007812,
"margin_dpo/margin_std": 886.00927734375,
"step": 663
},
{
"KL/chosen_KL_mean": -807.125732421875,
"KL/mean": -1088.34619140625,
"KL/rejected_KL_mean": -1369.56689453125,
"KL/std": 771.4961547851562,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.0006953716510906816,
"fcm_dpo/delta": -0.04308585077524185,
"fcm_dpo/margin": 562.4410400390625,
"fcm_dpo/q_t": 0.41181105375289917,
"grad_norm": 27.388290405273438,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -1.0791889429092407,
"logits/rejected": -1.1189603805541992,
"logps/chosen": -868.0142211914062,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1475.088623046875,
"loss": 1.1093,
"margin_dpo/margin_mean": 562.4410400390625,
"margin_dpo/margin_std": 830.4803466796875,
"step": 664
},
{
"KL/chosen_KL_mean": -685.2548828125,
"KL/mean": -923.9151000976562,
"KL/rejected_KL_mean": -1162.5753173828125,
"KL/std": 609.5552368164062,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0006976306904107332,
"fcm_dpo/delta": 0.017594607546925545,
"fcm_dpo/margin": 477.3204345703125,
"fcm_dpo/q_t": 0.42177683115005493,
"grad_norm": 50.124088287353516,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.1326605081558228,
"logits/rejected": -1.160711646080017,
"logps/chosen": -745.8190307617188,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1247.384033203125,
"loss": 1.1244,
"margin_dpo/margin_mean": 477.3204345703125,
"margin_dpo/margin_std": 607.7765502929688,
"step": 665
},
{
"KL/chosen_KL_mean": -722.7728271484375,
"KL/mean": -959.1403198242188,
"KL/rejected_KL_mean": -1195.5078125,
"KL/std": 615.6036376953125,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0006999197648838162,
"fcm_dpo/delta": 0.019781440496444702,
"fcm_dpo/margin": 472.7351379394531,
"fcm_dpo/q_t": 0.4240819215774536,
"grad_norm": 30.830541610717773,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.9478716254234314,
"logits/rejected": -0.9615623950958252,
"logps/chosen": -787.1927490234375,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1291.3995361328125,
"loss": 1.1399,
"margin_dpo/margin_mean": 472.73516845703125,
"margin_dpo/margin_std": 689.4872436523438,
"step": 666
},
{
"KL/chosen_KL_mean": -859.1533203125,
"KL/mean": -1136.722900390625,
"KL/rejected_KL_mean": -1414.29248046875,
"KL/std": 699.344482421875,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.0006951624527573586,
"fcm_dpo/delta": -0.037639595568180084,
"fcm_dpo/margin": 555.1390380859375,
"fcm_dpo/q_t": 0.4112476706504822,
"grad_norm": 36.62522506713867,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.9919800758361816,
"logits/rejected": -0.995948076248169,
"logps/chosen": -928.4303588867188,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1502.1279296875,
"loss": 1.1057,
"margin_dpo/margin_mean": 555.1390380859375,
"margin_dpo/margin_std": 778.8447875976562,
"step": 667
},
{
"KL/chosen_KL_mean": -890.34130859375,
"KL/mean": -1082.375732421875,
"KL/rejected_KL_mean": -1274.4102783203125,
"KL/std": 727.134765625,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0006961192702874541,
"fcm_dpo/delta": 0.003592910710722208,
"fcm_dpo/margin": 384.0690002441406,
"fcm_dpo/q_t": 0.4404904246330261,
"grad_norm": 48.40626525878906,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -1.0494754314422607,
"logits/rejected": -1.0595271587371826,
"logps/chosen": -962.9453125,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1378.1494140625,
"loss": 1.2243,
"margin_dpo/margin_mean": 384.0690002441406,
"margin_dpo/margin_std": 874.9485473632812,
"step": 668
},
{
"KL/chosen_KL_mean": -679.6414184570312,
"KL/mean": -960.998291015625,
"KL/rejected_KL_mean": -1242.3551025390625,
"KL/std": 686.39013671875,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.000693183159455657,
"fcm_dpo/delta": -0.04167580232024193,
"fcm_dpo/margin": 562.7136840820312,
"fcm_dpo/q_t": 0.4098045825958252,
"grad_norm": 26.838964462280273,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -1.047389268875122,
"logits/rejected": -1.0622926950454712,
"logps/chosen": -725.7578125,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1320.279541015625,
"loss": 1.101,
"margin_dpo/margin_mean": 562.7137451171875,
"margin_dpo/margin_std": 774.2056884765625,
"step": 669
},
{
"KL/chosen_KL_mean": -649.9046630859375,
"KL/mean": -927.82958984375,
"KL/rejected_KL_mean": -1205.754638671875,
"KL/std": 644.1104736328125,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.0006895088008604944,
"fcm_dpo/delta": -0.03472786396741867,
"fcm_dpo/margin": 555.849853515625,
"fcm_dpo/q_t": 0.41115692257881165,
"grad_norm": 25.038528442382812,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.946910560131073,
"logits/rejected": -0.9696442484855652,
"logps/chosen": -712.2504272460938,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1302.695068359375,
"loss": 1.1056,
"margin_dpo/margin_mean": 555.849853515625,
"margin_dpo/margin_std": 760.0970458984375,
"step": 670
},
{
"KL/chosen_KL_mean": -844.815673828125,
"KL/mean": -1066.053955078125,
"KL/rejected_KL_mean": -1287.292236328125,
"KL/std": 671.5584716796875,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0006929665105417371,
"fcm_dpo/delta": 0.044294700026512146,
"fcm_dpo/margin": 442.4765625,
"fcm_dpo/q_t": 0.4280014634132385,
"grad_norm": 30.275554656982422,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -1.0639939308166504,
"logits/rejected": -1.0980405807495117,
"logps/chosen": -892.8157958984375,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1371.111572265625,
"loss": 1.1821,
"margin_dpo/margin_mean": 442.4765625,
"margin_dpo/margin_std": 831.6336669921875,
"step": 671
},
{
"KL/chosen_KL_mean": -921.38525390625,
"KL/mean": -1149.651123046875,
"KL/rejected_KL_mean": -1377.9171142578125,
"KL/std": 785.9630737304688,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0006937608122825623,
"fcm_dpo/delta": 0.0343257300555706,
"fcm_dpo/margin": 456.5320129394531,
"fcm_dpo/q_t": 0.4287392497062683,
"grad_norm": 48.789405822753906,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.1506716012954712,
"logits/rejected": -1.1607093811035156,
"logps/chosen": -979.968505859375,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1471.057373046875,
"loss": 1.1798,
"margin_dpo/margin_mean": 456.5320129394531,
"margin_dpo/margin_std": 864.0927124023438,
"step": 672
},
{
"KL/chosen_KL_mean": -810.012939453125,
"KL/mean": -1019.599365234375,
"KL/rejected_KL_mean": -1229.185791015625,
"KL/std": 655.4266357421875,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0007023122161626816,
"fcm_dpo/delta": 0.05732039362192154,
"fcm_dpo/margin": 419.1728210449219,
"fcm_dpo/q_t": 0.4328360855579376,
"grad_norm": 23.517154693603516,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -1.093796968460083,
"logits/rejected": -1.101298451423645,
"logps/chosen": -856.7362060546875,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1314.4820556640625,
"loss": 1.1744,
"margin_dpo/margin_mean": 419.1728515625,
"margin_dpo/margin_std": 737.906982421875,
"step": 673
},
{
"KL/chosen_KL_mean": -668.804443359375,
"KL/mean": -940.5889892578125,
"KL/rejected_KL_mean": -1212.37353515625,
"KL/std": 636.3111572265625,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0007032062858343124,
"fcm_dpo/delta": -0.033593274652957916,
"fcm_dpo/margin": 543.5691528320312,
"fcm_dpo/q_t": 0.4136511981487274,
"grad_norm": 31.608436584472656,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.9837385416030884,
"logits/rejected": -1.0068650245666504,
"logps/chosen": -714.25,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1282.41943359375,
"loss": 1.1079,
"margin_dpo/margin_mean": 543.5691528320312,
"margin_dpo/margin_std": 769.9119873046875,
"step": 674
},
{
"KL/chosen_KL_mean": -757.3980712890625,
"KL/mean": -1031.643798828125,
"KL/rejected_KL_mean": -1305.8896484375,
"KL/std": 698.0523681640625,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.0006924683111719787,
"fcm_dpo/delta": -0.0319318063557148,
"fcm_dpo/margin": 548.4915771484375,
"fcm_dpo/q_t": 0.4143555462360382,
"grad_norm": 27.629240036010742,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -1.0433604717254639,
"logits/rejected": -1.0594494342803955,
"logps/chosen": -801.5743408203125,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1379.981689453125,
"loss": 1.1111,
"margin_dpo/margin_mean": 548.4916381835938,
"margin_dpo/margin_std": 764.034912109375,
"step": 675
},
{
"KL/chosen_KL_mean": -832.3480834960938,
"KL/mean": -1099.799560546875,
"KL/rejected_KL_mean": -1367.2510986328125,
"KL/std": 692.36865234375,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0006945295608602464,
"fcm_dpo/delta": -0.02236580289900303,
"fcm_dpo/margin": 534.9030151367188,
"fcm_dpo/q_t": 0.4144228398799896,
"grad_norm": 33.126529693603516,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -1.004886269569397,
"logits/rejected": -1.0242345333099365,
"logps/chosen": -903.74658203125,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1455.60986328125,
"loss": 1.109,
"margin_dpo/margin_mean": 534.9030151367188,
"margin_dpo/margin_std": 720.2886352539062,
"step": 676
},
{
"KL/chosen_KL_mean": -826.7559814453125,
"KL/mean": -1088.8828125,
"KL/rejected_KL_mean": -1351.010009765625,
"KL/std": 718.5059204101562,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0006902614841237664,
"fcm_dpo/delta": -0.012368248775601387,
"fcm_dpo/margin": 524.25390625,
"fcm_dpo/q_t": 0.4213380217552185,
"grad_norm": 35.2880859375,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -1.128098487854004,
"logits/rejected": -1.131150722503662,
"logps/chosen": -883.2833862304688,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1429.2364501953125,
"loss": 1.1378,
"margin_dpo/margin_mean": 524.25390625,
"margin_dpo/margin_std": 882.9517822265625,
"step": 677
},
{
"KL/chosen_KL_mean": -681.46044921875,
"KL/mean": -999.7894287109375,
"KL/rejected_KL_mean": -1318.118408203125,
"KL/std": 790.3182983398438,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.0006847955519333482,
"fcm_dpo/delta": -0.08993227779865265,
"fcm_dpo/margin": 636.6580810546875,
"fcm_dpo/q_t": 0.40213915705680847,
"grad_norm": 32.831016540527344,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -1.0311408042907715,
"logits/rejected": -1.0793263912200928,
"logps/chosen": -727.5949096679688,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1398.72314453125,
"loss": 1.0853,
"margin_dpo/margin_mean": 636.6580200195312,
"margin_dpo/margin_std": 912.1826171875,
"step": 678
},
{
"KL/chosen_KL_mean": -798.2120361328125,
"KL/mean": -1003.3867797851562,
"KL/rejected_KL_mean": -1208.5615234375,
"KL/std": 617.0496826171875,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.000684911385178566,
"fcm_dpo/delta": 0.07099346816539764,
"fcm_dpo/margin": 410.34954833984375,
"fcm_dpo/q_t": 0.43571871519088745,
"grad_norm": 32.56108093261719,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -1.0531508922576904,
"logits/rejected": -1.048056960105896,
"logps/chosen": -848.5069580078125,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1285.15966796875,
"loss": 1.1832,
"margin_dpo/margin_mean": 410.3495178222656,
"margin_dpo/margin_std": 727.0447387695312,
"step": 679
},
{
"KL/chosen_KL_mean": -812.5574951171875,
"KL/mean": -1099.232177734375,
"KL/rejected_KL_mean": -1385.906982421875,
"KL/std": 805.0675659179688,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0006806158926337957,
"fcm_dpo/delta": -0.04249640554189682,
"fcm_dpo/margin": 573.349365234375,
"fcm_dpo/q_t": 0.4099177122116089,
"grad_norm": 40.93464660644531,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -1.0480668544769287,
"logits/rejected": -1.0822138786315918,
"logps/chosen": -889.4732055664062,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1498.291748046875,
"loss": 1.1149,
"margin_dpo/margin_mean": 573.349365234375,
"margin_dpo/margin_std": 864.7239990234375,
"step": 680
},
{
"KL/chosen_KL_mean": -819.076416015625,
"KL/mean": -1035.499755859375,
"KL/rejected_KL_mean": -1251.92333984375,
"KL/std": 668.314697265625,
"epoch": 1.0,
"fcm_dpo/beta": 0.0006794101791456342,
"fcm_dpo/delta": -0.042327266186475754,
"fcm_dpo/margin": 432.8468933105469,
"fcm_dpo/q_t": 0.4326745867729187,
"grad_norm": 40.8109130859375,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -1.1136659383773804,
"logits/rejected": -1.1350388526916504,
"logps/chosen": -880.03369140625,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1340.481201171875,
"loss": 1.1847,
"margin_dpo/margin_mean": 432.84686279296875,
"margin_dpo/margin_std": 766.4469604492188,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.1324475014612363,
"train_runtime": 1725.4575,
"train_samples_per_second": 25.268,
"train_steps_per_second": 0.395
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink