W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.4
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
af6050114561fc2071d105180a340aa0c799d04a
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC af60501145 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.4
Source: Original Platform
2026-05-18 06:10:14 +08:00

15707 lines
584 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.52816009521484,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.20118713378906,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.009592056274414062,
"KL/mean": -0.00013332068920135498,
"KL/rejected_KL_mean": -0.009857177734375,
"KL/std": 0.27916020154953003,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.019449442625045776,
"fcm_dpo/q_t": 0.4995156228542328,
"grad_norm": 70.91877746582031,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.4816860258579254,
"logits/rejected": -0.4422576129436493,
"logps/chosen": -60.97200393676758,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.68244934082031,
"loss": 1.3848,
"margin_dpo/margin_mean": 0.019450068473815918,
"margin_dpo/margin_std": 0.41192615032196045,
"step": 3
},
{
"KL/chosen_KL_mean": 0.01334381103515625,
"KL/mean": 0.011412113904953003,
"KL/rejected_KL_mean": 0.009479522705078125,
"KL/std": 0.27838367223739624,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.003867626190185547,
"fcm_dpo/q_t": 0.4999042749404907,
"grad_norm": 72.21875762939453,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.4687785506248474,
"logits/rejected": -0.4412214756011963,
"logps/chosen": -56.75436782836914,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.63763427734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 0.0038670599460601807,
"margin_dpo/margin_std": 0.4134891629219055,
"step": 4
},
{
"KL/chosen_KL_mean": -0.00311279296875,
"KL/mean": 0.004465892910957336,
"KL/rejected_KL_mean": 0.012050628662109375,
"KL/std": 0.24947434663772583,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.015161305665969849,
"fcm_dpo/q_t": 0.5003792643547058,
"grad_norm": 90.08293151855469,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5145992040634155,
"logits/rejected": -0.4708704352378845,
"logps/chosen": -53.86248779296875,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.13713073730469,
"loss": 1.3881,
"margin_dpo/margin_mean": -0.0151614248752594,
"margin_dpo/margin_std": 0.32647037506103516,
"step": 5
},
{
"KL/chosen_KL_mean": -0.0019550323486328125,
"KL/mean": -0.008544057607650757,
"KL/rejected_KL_mean": -0.015132904052734375,
"KL/std": 0.27398955821990967,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0131760835647583,
"fcm_dpo/q_t": 0.49967002868652344,
"grad_norm": 92.1026840209961,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5129680633544922,
"logits/rejected": -0.4713151454925537,
"logps/chosen": -63.009437561035156,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.66047668457031,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.013176411390304565,
"margin_dpo/margin_std": 0.4235680103302002,
"step": 6
},
{
"KL/chosen_KL_mean": 0.002979278564453125,
"KL/mean": 0.014112532138824463,
"KL/rejected_KL_mean": 0.025249481201171875,
"KL/std": 0.30075156688690186,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.022269725799560547,
"fcm_dpo/q_t": 0.5005556344985962,
"grad_norm": 82.2314682006836,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.4966992735862732,
"logits/rejected": -0.46284064650535583,
"logps/chosen": -57.77184295654297,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.89533996582031,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02226969599723816,
"margin_dpo/margin_std": 0.41930609941482544,
"step": 7
},
{
"KL/chosen_KL_mean": 0.03544044494628906,
"KL/mean": 0.03716662526130676,
"KL/rejected_KL_mean": 0.0388946533203125,
"KL/std": 0.318942129611969,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0034527182579040527,
"fcm_dpo/q_t": 0.5000861883163452,
"grad_norm": 78.34169006347656,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.5110943913459778,
"logits/rejected": -0.48551392555236816,
"logps/chosen": -58.68059539794922,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.27252960205078,
"loss": 1.387,
"margin_dpo/margin_mean": -0.003452509641647339,
"margin_dpo/margin_std": 0.3985304534435272,
"step": 8
},
{
"KL/chosen_KL_mean": 0.007297515869140625,
"KL/mean": -0.008765771985054016,
"KL/rejected_KL_mean": -0.0248260498046875,
"KL/std": 0.2620963752269745,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03212156891822815,
"fcm_dpo/q_t": 0.49919742345809937,
"grad_norm": 84.95182037353516,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.4905078411102295,
"logits/rejected": -0.4435485601425171,
"logps/chosen": -69.85954284667969,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.62748718261719,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.03212141990661621,
"margin_dpo/margin_std": 0.38075071573257446,
"step": 9
},
{
"KL/chosen_KL_mean": 0.015682220458984375,
"KL/mean": 0.008813418447971344,
"KL/rejected_KL_mean": 0.00194549560546875,
"KL/std": 0.24493810534477234,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01373349130153656,
"fcm_dpo/q_t": 0.4996568560600281,
"grad_norm": 70.63052368164062,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.47287142276763916,
"logits/rejected": -0.4286770224571228,
"logps/chosen": -48.342002868652344,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.3701171875,
"loss": 1.3852,
"margin_dpo/margin_mean": 0.01373395323753357,
"margin_dpo/margin_std": 0.3393193483352661,
"step": 10
},
{
"KL/chosen_KL_mean": -0.027435302734375,
"KL/mean": -0.032692402601242065,
"KL/rejected_KL_mean": -0.0379486083984375,
"KL/std": 0.2535749673843384,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.010510385036468506,
"fcm_dpo/q_t": 0.49973729252815247,
"grad_norm": 68.74665069580078,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4513794481754303,
"logits/rejected": -0.42600423097610474,
"logps/chosen": -53.044288635253906,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.81832885742188,
"loss": 1.3856,
"margin_dpo/margin_mean": 0.010511070489883423,
"margin_dpo/margin_std": 0.3717014193534851,
"step": 11
},
{
"KL/chosen_KL_mean": -0.03303718566894531,
"KL/mean": -0.05988152325153351,
"KL/rejected_KL_mean": -0.08672332763671875,
"KL/std": 0.31390875577926636,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05368778109550476,
"fcm_dpo/q_t": 0.4986579418182373,
"grad_norm": 89.02582550048828,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5268337726593018,
"logits/rejected": -0.4902796745300293,
"logps/chosen": -61.83847427368164,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.94499206542969,
"loss": 1.3813,
"margin_dpo/margin_mean": 0.05368664860725403,
"margin_dpo/margin_std": 0.38752928376197815,
"step": 12
},
{
"KL/chosen_KL_mean": -0.001468658447265625,
"KL/mean": -0.00235767662525177,
"KL/rejected_KL_mean": -0.003246307373046875,
"KL/std": 0.2623201906681061,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0017842352390289307,
"fcm_dpo/q_t": 0.49995577335357666,
"grad_norm": 79.24825286865234,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.4681378901004791,
"logits/rejected": -0.43913543224334717,
"logps/chosen": -64.26182556152344,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.20632934570312,
"loss": 1.3865,
"margin_dpo/margin_mean": 0.0017848014831542969,
"margin_dpo/margin_std": 0.3650243878364563,
"step": 13
},
{
"KL/chosen_KL_mean": 0.018545150756835938,
"KL/mean": -0.004325702786445618,
"KL/rejected_KL_mean": -0.02719879150390625,
"KL/std": 0.2847668528556824,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04574933648109436,
"fcm_dpo/q_t": 0.4988548755645752,
"grad_norm": 85.14924621582031,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.4632902145385742,
"logits/rejected": -0.4233039319515228,
"logps/chosen": -58.09166717529297,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.07427978515625,
"loss": 1.3823,
"margin_dpo/margin_mean": 0.04574891924858093,
"margin_dpo/margin_std": 0.4520788788795471,
"step": 14
},
{
"KL/chosen_KL_mean": 0.0231170654296875,
"KL/mean": -0.0037177205085754395,
"KL/rejected_KL_mean": -0.03055572509765625,
"KL/std": 0.24491383135318756,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05367732048034668,
"fcm_dpo/q_t": 0.4986582100391388,
"grad_norm": 64.11997985839844,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.4985060691833496,
"logits/rejected": -0.4801892936229706,
"logps/chosen": -56.94379425048828,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.83920288085938,
"loss": 1.3813,
"margin_dpo/margin_mean": 0.05367732048034668,
"margin_dpo/margin_std": 0.3837876319885254,
"step": 15
},
{
"KL/chosen_KL_mean": 0.0002803802490234375,
"KL/mean": -0.022383689880371094,
"KL/rejected_KL_mean": -0.0450439453125,
"KL/std": 0.26514291763305664,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.045323848724365234,
"fcm_dpo/q_t": 0.4988669157028198,
"grad_norm": 83.59847259521484,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5230243802070618,
"logits/rejected": -0.4819951355457306,
"logps/chosen": -61.7396125793457,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.41452026367188,
"loss": 1.3821,
"margin_dpo/margin_mean": 0.04532337188720703,
"margin_dpo/margin_std": 0.37663698196411133,
"step": 16
},
{
"KL/chosen_KL_mean": 0.04068756103515625,
"KL/mean": -0.002854481339454651,
"KL/rejected_KL_mean": -0.046398162841796875,
"KL/std": 0.2759934067726135,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08708342909812927,
"fcm_dpo/q_t": 0.4978230595588684,
"grad_norm": 78.46449279785156,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.5030894875526428,
"logits/rejected": -0.466357946395874,
"logps/chosen": -67.66964721679688,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.425048828125,
"loss": 1.378,
"margin_dpo/margin_mean": 0.08708390593528748,
"margin_dpo/margin_std": 0.3927149772644043,
"step": 17
},
{
"KL/chosen_KL_mean": 0.0130767822265625,
"KL/mean": -0.0022923946380615234,
"KL/rejected_KL_mean": -0.017665863037109375,
"KL/std": 0.22370792925357819,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.030742883682250977,
"fcm_dpo/q_t": 0.49923163652420044,
"grad_norm": 82.09223937988281,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.5195785164833069,
"logits/rejected": -0.4671386778354645,
"logps/chosen": -47.72641372680664,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.48995208740234,
"loss": 1.3834,
"margin_dpo/margin_mean": 0.030742555856704712,
"margin_dpo/margin_std": 0.27723872661590576,
"step": 18
},
{
"KL/chosen_KL_mean": 0.00811004638671875,
"KL/mean": -0.019911006093025208,
"KL/rejected_KL_mean": -0.047924041748046875,
"KL/std": 0.21907614171504974,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05602636933326721,
"fcm_dpo/q_t": 0.4985996186733246,
"grad_norm": 73.40470886230469,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.5081275701522827,
"logits/rejected": -0.4595046043395996,
"logps/chosen": -70.19725036621094,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.80551147460938,
"loss": 1.3809,
"margin_dpo/margin_mean": 0.056026577949523926,
"margin_dpo/margin_std": 0.2629699409008026,
"step": 19
},
{
"KL/chosen_KL_mean": -0.0014286041259765625,
"KL/mean": -0.05187162756919861,
"KL/rejected_KL_mean": -0.1023101806640625,
"KL/std": 0.2537362277507782,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10088193416595459,
"fcm_dpo/q_t": 0.4974784851074219,
"grad_norm": 73.6449203491211,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5645604133605957,
"logits/rejected": -0.5095418691635132,
"logps/chosen": -50.80467224121094,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.9256591796875,
"loss": 1.3765,
"margin_dpo/margin_mean": 0.10088211297988892,
"margin_dpo/margin_std": 0.2900768518447876,
"step": 20
},
{
"KL/chosen_KL_mean": -0.03330421447753906,
"KL/mean": -0.08112373948097229,
"KL/rejected_KL_mean": -0.12894439697265625,
"KL/std": 0.2684672474861145,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09563732147216797,
"fcm_dpo/q_t": 0.49761033058166504,
"grad_norm": 77.29470825195312,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.5119160413742065,
"logits/rejected": -0.48931699991226196,
"logps/chosen": -50.09632110595703,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.99774169921875,
"loss": 1.3771,
"margin_dpo/margin_mean": 0.09563717246055603,
"margin_dpo/margin_std": 0.37452447414398193,
"step": 21
},
{
"KL/chosen_KL_mean": 0.038906097412109375,
"KL/mean": -0.027102157473564148,
"KL/rejected_KL_mean": -0.093109130859375,
"KL/std": 0.253788024187088,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1320144236087799,
"fcm_dpo/q_t": 0.49670103192329407,
"grad_norm": 84.75688934326172,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.5193873643875122,
"logits/rejected": -0.4773576855659485,
"logps/chosen": -59.01873016357422,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.5977783203125,
"loss": 1.3734,
"margin_dpo/margin_mean": 0.13201433420181274,
"margin_dpo/margin_std": 0.3331354558467865,
"step": 22
},
{
"KL/chosen_KL_mean": 0.08184623718261719,
"KL/mean": -0.05801086127758026,
"KL/rejected_KL_mean": -0.19786643981933594,
"KL/std": 0.3483527898788452,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2797093689441681,
"fcm_dpo/q_t": 0.49301281571388245,
"grad_norm": 80.32367706298828,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.48532232642173767,
"logits/rejected": -0.4625855088233948,
"logps/chosen": -59.995849609375,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.33741760253906,
"loss": 1.359,
"margin_dpo/margin_mean": 0.2797098159790039,
"margin_dpo/margin_std": 0.42796939611434937,
"step": 23
},
{
"KL/chosen_KL_mean": 0.01682281494140625,
"KL/mean": -0.07509848475456238,
"KL/rejected_KL_mean": -0.167022705078125,
"KL/std": 0.2538023591041565,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1838463544845581,
"fcm_dpo/q_t": 0.49540600180625916,
"grad_norm": 83.12644958496094,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.4901927709579468,
"logits/rejected": -0.4739952087402344,
"logps/chosen": -44.27421188354492,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.292236328125,
"loss": 1.3683,
"margin_dpo/margin_mean": 0.1838468313217163,
"margin_dpo/margin_std": 0.3329934775829315,
"step": 24
},
{
"KL/chosen_KL_mean": 0.04872703552246094,
"KL/mean": -0.11213195323944092,
"KL/rejected_KL_mean": -0.27298736572265625,
"KL/std": 0.3789520263671875,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3217122554779053,
"fcm_dpo/q_t": 0.49196386337280273,
"grad_norm": 73.51007080078125,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.5064820051193237,
"logits/rejected": -0.4769260883331299,
"logps/chosen": -52.48832702636719,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.61517333984375,
"loss": 1.355,
"margin_dpo/margin_mean": 0.3217119574546814,
"margin_dpo/margin_std": 0.491582453250885,
"step": 25
},
{
"KL/chosen_KL_mean": 0.11263847351074219,
"KL/mean": -0.10163936018943787,
"KL/rejected_KL_mean": -0.31591796875,
"KL/std": 0.4192284941673279,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4285600185394287,
"fcm_dpo/q_t": 0.48929810523986816,
"grad_norm": 87.42533874511719,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5297478437423706,
"logits/rejected": -0.49793004989624023,
"logps/chosen": -53.8101692199707,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.67562866210938,
"loss": 1.3446,
"margin_dpo/margin_mean": 0.42856070399284363,
"margin_dpo/margin_std": 0.5423732399940491,
"step": 26
},
{
"KL/chosen_KL_mean": 0.13956642150878906,
"KL/mean": -0.14148437976837158,
"KL/rejected_KL_mean": -0.4225349426269531,
"KL/std": 0.4998953342437744,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5621016025543213,
"fcm_dpo/q_t": 0.48596882820129395,
"grad_norm": 93.27300262451172,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5154296159744263,
"logits/rejected": -0.47898948192596436,
"logps/chosen": -42.75896453857422,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.14673614501953,
"loss": 1.3318,
"margin_dpo/margin_mean": 0.5621014833450317,
"margin_dpo/margin_std": 0.6122180223464966,
"step": 27
},
{
"KL/chosen_KL_mean": 0.04203224182128906,
"KL/mean": -0.12256798148155212,
"KL/rejected_KL_mean": -0.28717041015625,
"KL/std": 0.3935839533805847,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3292032480239868,
"fcm_dpo/q_t": 0.4917774200439453,
"grad_norm": 75.40629577636719,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5347428917884827,
"logits/rejected": -0.48221421241760254,
"logps/chosen": -60.514469146728516,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.68828582763672,
"loss": 1.3544,
"margin_dpo/margin_mean": 0.3292037844657898,
"margin_dpo/margin_std": 0.5351283550262451,
"step": 28
},
{
"KL/chosen_KL_mean": 0.10456466674804688,
"KL/mean": -0.1523105800151825,
"KL/rejected_KL_mean": -0.4091835021972656,
"KL/std": 0.4780128300189972,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5137494802474976,
"fcm_dpo/q_t": 0.4871741533279419,
"grad_norm": 91.09429931640625,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5602052211761475,
"logits/rejected": -0.5143990516662598,
"logps/chosen": -57.70322036743164,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.80353546142578,
"loss": 1.3365,
"margin_dpo/margin_mean": 0.5137503147125244,
"margin_dpo/margin_std": 0.600021243095398,
"step": 29
},
{
"KL/chosen_KL_mean": 0.15185928344726562,
"KL/mean": -0.17665454745292664,
"KL/rejected_KL_mean": -0.5051727294921875,
"KL/std": 0.569696307182312,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6570307016372681,
"fcm_dpo/q_t": 0.4835994839668274,
"grad_norm": 87.53655242919922,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.48063158988952637,
"logits/rejected": -0.4493868350982666,
"logps/chosen": -52.425514221191406,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -98.99437713623047,
"loss": 1.3227,
"margin_dpo/margin_mean": 0.6570307016372681,
"margin_dpo/margin_std": 0.6358605623245239,
"step": 30
},
{
"KL/chosen_KL_mean": 0.12508392333984375,
"KL/mean": -0.11352300643920898,
"KL/rejected_KL_mean": -0.3521308898925781,
"KL/std": 0.5105932950973511,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.47721391916275024,
"fcm_dpo/q_t": 0.4880916476249695,
"grad_norm": 68.69351196289062,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.5206561088562012,
"logits/rejected": -0.47763532400131226,
"logps/chosen": -63.68183898925781,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.24613952636719,
"loss": 1.3401,
"margin_dpo/margin_mean": 0.4772138297557831,
"margin_dpo/margin_std": 0.6101304888725281,
"step": 31
},
{
"KL/chosen_KL_mean": 0.19174575805664062,
"KL/mean": -0.19748108088970184,
"KL/rejected_KL_mean": -0.5867080688476562,
"KL/std": 0.7203556299209595,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7784509658813477,
"fcm_dpo/q_t": 0.48061084747314453,
"grad_norm": 82.00362396240234,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5107109546661377,
"logits/rejected": -0.4691959619522095,
"logps/chosen": -62.54777908325195,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.90420532226562,
"loss": 1.312,
"margin_dpo/margin_mean": 0.7784509658813477,
"margin_dpo/margin_std": 0.9079021215438843,
"step": 32
},
{
"KL/chosen_KL_mean": 0.12497520446777344,
"KL/mean": -0.18040965497493744,
"KL/rejected_KL_mean": -0.4857940673828125,
"KL/std": 0.5566126108169556,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6107670068740845,
"fcm_dpo/q_t": 0.48475056886672974,
"grad_norm": 72.61372375488281,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.4933505058288574,
"logits/rejected": -0.46733930706977844,
"logps/chosen": -53.1359977722168,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.37092590332031,
"loss": 1.3271,
"margin_dpo/margin_mean": 0.6107674241065979,
"margin_dpo/margin_std": 0.6001892685890198,
"step": 33
},
{
"KL/chosen_KL_mean": 0.0882568359375,
"KL/mean": -0.30000877380371094,
"KL/rejected_KL_mean": -0.6882743835449219,
"KL/std": 0.7007868885993958,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7765321731567383,
"fcm_dpo/q_t": 0.48063427209854126,
"grad_norm": 78.93215942382812,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.4917735457420349,
"logits/rejected": -0.47427982091903687,
"logps/chosen": -50.72907257080078,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.61012268066406,
"loss": 1.3118,
"margin_dpo/margin_mean": 0.7765311002731323,
"margin_dpo/margin_std": 0.8206242918968201,
"step": 34
},
{
"KL/chosen_KL_mean": 0.14589881896972656,
"KL/mean": -0.4312659204006195,
"KL/rejected_KL_mean": -1.0084266662597656,
"KL/std": 0.9634122848510742,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1543281078338623,
"fcm_dpo/q_t": 0.471314013004303,
"grad_norm": 83.42659759521484,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5353403687477112,
"logits/rejected": -0.4983571767807007,
"logps/chosen": -50.87859344482422,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.83285522460938,
"loss": 1.2774,
"margin_dpo/margin_mean": 1.1543282270431519,
"margin_dpo/margin_std": 1.1236690282821655,
"step": 35
},
{
"KL/chosen_KL_mean": 0.009319305419921875,
"KL/mean": -0.544201135635376,
"KL/rejected_KL_mean": -1.0977210998535156,
"KL/std": 1.0783436298370361,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1070445775985718,
"fcm_dpo/q_t": 0.47246915102005005,
"grad_norm": 72.56640625,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5723892450332642,
"logits/rejected": -0.5367264747619629,
"logps/chosen": -51.98217010498047,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.13835144042969,
"loss": 1.2826,
"margin_dpo/margin_mean": 1.107043981552124,
"margin_dpo/margin_std": 1.2233386039733887,
"step": 36
},
{
"KL/chosen_KL_mean": 0.03542518615722656,
"KL/mean": -0.48510926961898804,
"KL/rejected_KL_mean": -1.0056495666503906,
"KL/std": 1.0858180522918701,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0410770177841187,
"fcm_dpo/q_t": 0.4742150902748108,
"grad_norm": 61.9765739440918,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5248156785964966,
"logits/rejected": -0.48221951723098755,
"logps/chosen": -62.77168273925781,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.90072631835938,
"loss": 1.29,
"margin_dpo/margin_mean": 1.0410770177841187,
"margin_dpo/margin_std": 1.3847689628601074,
"step": 37
},
{
"KL/chosen_KL_mean": 0.14590072631835938,
"KL/mean": -0.5701018571853638,
"KL/rejected_KL_mean": -1.2861061096191406,
"KL/std": 1.3780711889266968,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4320064783096313,
"fcm_dpo/q_t": 0.46469372510910034,
"grad_norm": 69.0830307006836,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5364372730255127,
"logits/rejected": -0.5037224292755127,
"logps/chosen": -48.24462127685547,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.19854736328125,
"loss": 1.2553,
"margin_dpo/margin_mean": 1.4320067167282104,
"margin_dpo/margin_std": 1.680180549621582,
"step": 38
},
{
"KL/chosen_KL_mean": 0.06763076782226562,
"KL/mean": -0.74673992395401,
"KL/rejected_KL_mean": -1.5611076354980469,
"KL/std": 1.2961926460266113,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6287341117858887,
"fcm_dpo/q_t": 0.4595925509929657,
"grad_norm": 73.7347412109375,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.537227988243103,
"logits/rejected": -0.49611860513687134,
"logps/chosen": -50.68284225463867,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.130615234375,
"loss": 1.2351,
"margin_dpo/margin_mean": 1.6287344694137573,
"margin_dpo/margin_std": 1.415596604347229,
"step": 39
},
{
"KL/chosen_KL_mean": 0.19949913024902344,
"KL/mean": -0.5562039613723755,
"KL/rejected_KL_mean": -1.3119163513183594,
"KL/std": 1.4023091793060303,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5114145278930664,
"fcm_dpo/q_t": 0.4627044200897217,
"grad_norm": 59.9863395690918,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5215098857879639,
"logits/rejected": -0.4915487766265869,
"logps/chosen": -57.78556823730469,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.61198425292969,
"loss": 1.2484,
"margin_dpo/margin_mean": 1.5114144086837769,
"margin_dpo/margin_std": 1.707202672958374,
"step": 40
},
{
"KL/chosen_KL_mean": 0.02559661865234375,
"KL/mean": -0.9471765756607056,
"KL/rejected_KL_mean": -1.9199447631835938,
"KL/std": 1.8845467567443848,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9455442428588867,
"fcm_dpo/q_t": 0.45214998722076416,
"grad_norm": 67.88764190673828,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5397985577583313,
"logits/rejected": -0.5032455325126648,
"logps/chosen": -62.67021942138672,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.94347381591797,
"loss": 1.2116,
"margin_dpo/margin_mean": 1.9455441236495972,
"margin_dpo/margin_std": 2.008014678955078,
"step": 41
},
{
"KL/chosen_KL_mean": 0.22910499572753906,
"KL/mean": -1.0767121315002441,
"KL/rejected_KL_mean": -2.382526397705078,
"KL/std": 2.184157609939575,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.611631393432617,
"fcm_dpo/q_t": 0.4363771677017212,
"grad_norm": 78.85198974609375,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5462131500244141,
"logits/rejected": -0.49966076016426086,
"logps/chosen": -58.73731994628906,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.29090118408203,
"loss": 1.1569,
"margin_dpo/margin_mean": 2.611631393432617,
"margin_dpo/margin_std": 2.447122097015381,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5043964385986328,
"KL/mean": -0.7495235204696655,
"KL/rejected_KL_mean": -2.003448486328125,
"KL/std": 1.8596460819244385,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.5078420639038086,
"fcm_dpo/q_t": 0.4384229779243469,
"grad_norm": 71.84793090820312,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.529945433139801,
"logits/rejected": -0.5044955015182495,
"logps/chosen": -53.6515998840332,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.483642578125,
"loss": 1.1612,
"margin_dpo/margin_mean": 2.5078420639038086,
"margin_dpo/margin_std": 2.0080738067626953,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2628173828125,
"KL/mean": -1.1443766355514526,
"KL/rejected_KL_mean": -2.5515708923339844,
"KL/std": 2.2642464637756348,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.814387321472168,
"fcm_dpo/q_t": 0.4311271011829376,
"grad_norm": 78.9500503540039,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4813832640647888,
"logits/rejected": -0.461247056722641,
"logps/chosen": -49.81568145751953,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.3353271484375,
"loss": 1.1366,
"margin_dpo/margin_mean": 2.8143868446350098,
"margin_dpo/margin_std": 2.230133295059204,
"step": 44
},
{
"KL/chosen_KL_mean": 0.1612529754638672,
"KL/mean": -0.9844990968704224,
"KL/rejected_KL_mean": -2.1302528381347656,
"KL/std": 2.070618152618408,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.291508674621582,
"fcm_dpo/q_t": 0.4442916810512543,
"grad_norm": 62.898319244384766,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.47336873412132263,
"logits/rejected": -0.4605395793914795,
"logps/chosen": -48.25367736816406,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.06668090820312,
"loss": 1.1865,
"margin_dpo/margin_mean": 2.291508197784424,
"margin_dpo/margin_std": 2.556962013244629,
"step": 45
},
{
"KL/chosen_KL_mean": 0.16820144653320312,
"KL/mean": -1.308139443397522,
"KL/rejected_KL_mean": -2.78448486328125,
"KL/std": 2.6418280601501465,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.9526872634887695,
"fcm_dpo/q_t": 0.4291154146194458,
"grad_norm": 69.75140380859375,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5498015284538269,
"logits/rejected": -0.4997956156730652,
"logps/chosen": -55.83122253417969,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.43707275390625,
"loss": 1.1372,
"margin_dpo/margin_mean": 2.9526872634887695,
"margin_dpo/margin_std": 3.2058982849121094,
"step": 46
},
{
"KL/chosen_KL_mean": 0.37381935119628906,
"KL/mean": -1.1091471910476685,
"KL/rejected_KL_mean": -2.592113494873047,
"KL/std": 2.6175854206085205,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.9659318923950195,
"fcm_dpo/q_t": 0.4280517101287842,
"grad_norm": 65.22331237792969,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5681500434875488,
"logits/rejected": -0.5152395963668823,
"logps/chosen": -57.55226135253906,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.27131652832031,
"loss": 1.1305,
"margin_dpo/margin_mean": 2.965932846069336,
"margin_dpo/margin_std": 2.734755039215088,
"step": 47
},
{
"KL/chosen_KL_mean": 0.06793022155761719,
"KL/mean": -1.4929344654083252,
"KL/rejected_KL_mean": -3.053802490234375,
"KL/std": 2.6948609352111816,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.121734857559204,
"fcm_dpo/q_t": 0.4245325028896332,
"grad_norm": 73.7392349243164,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.6097103357315063,
"logits/rejected": -0.5529145002365112,
"logps/chosen": -57.12014389038086,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.07040405273438,
"loss": 1.1239,
"margin_dpo/margin_mean": 3.1217353343963623,
"margin_dpo/margin_std": 2.990403652191162,
"step": 48
},
{
"KL/chosen_KL_mean": 0.31961822509765625,
"KL/mean": -1.590240478515625,
"KL/rejected_KL_mean": -3.5000991821289062,
"KL/std": 3.3908257484436035,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.09978070855140686,
"fcm_dpo/delta": -0.021977046504616737,
"fcm_dpo/margin": 3.8197124004364014,
"fcm_dpo/q_t": 0.41010648012161255,
"grad_norm": 60.99309158325195,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.547630786895752,
"logits/rejected": -0.48894059658050537,
"logps/chosen": -61.36565399169922,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.267578125,
"loss": 1.0779,
"margin_dpo/margin_mean": 3.8197121620178223,
"margin_dpo/margin_std": 3.9132919311523438,
"step": 49
},
{
"KL/chosen_KL_mean": -0.1897144317626953,
"KL/mean": -2.2449421882629395,
"KL/rejected_KL_mean": -4.300167083740234,
"KL/std": 3.5077741146087646,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.09924671053886414,
"fcm_dpo/delta": -0.015830103307962418,
"fcm_dpo/margin": 4.11044454574585,
"fcm_dpo/q_t": 0.40354403853416443,
"grad_norm": 62.01202392578125,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5537872314453125,
"logits/rejected": -0.5180896520614624,
"logps/chosen": -58.91385269165039,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.65830993652344,
"loss": 1.057,
"margin_dpo/margin_mean": 4.11044454574585,
"margin_dpo/margin_std": 4.035107612609863,
"step": 50
},
{
"KL/chosen_KL_mean": -0.27092552185058594,
"KL/mean": -2.297037124633789,
"KL/rejected_KL_mean": -4.323150634765625,
"KL/std": 4.201503276824951,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.09835103154182434,
"fcm_dpo/delta": -0.04532879590988159,
"fcm_dpo/margin": 4.052219390869141,
"fcm_dpo/q_t": 0.40886732935905457,
"grad_norm": 51.64212417602539,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5401347875595093,
"logits/rejected": -0.5079134702682495,
"logps/chosen": -61.64459228515625,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.32514953613281,
"loss": 1.0861,
"margin_dpo/margin_mean": 4.052219867706299,
"margin_dpo/margin_std": 5.1329803466796875,
"step": 51
},
{
"KL/chosen_KL_mean": 0.35683631896972656,
"KL/mean": -2.742495536804199,
"KL/rejected_KL_mean": -5.841827392578125,
"KL/std": 4.826806545257568,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.09542756527662277,
"fcm_dpo/delta": -0.20356540381908417,
"fcm_dpo/margin": 6.198663234710693,
"fcm_dpo/q_t": 0.3626432716846466,
"grad_norm": 57.56940841674805,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.550842821598053,
"logits/rejected": -0.495021253824234,
"logps/chosen": -51.98052215576172,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.81574249267578,
"loss": 0.9351,
"margin_dpo/margin_mean": 6.198663711547852,
"margin_dpo/margin_std": 5.227818489074707,
"step": 52
},
{
"KL/chosen_KL_mean": -0.19112777709960938,
"KL/mean": -3.3118577003479004,
"KL/rejected_KL_mean": -6.432586669921875,
"KL/std": 5.454026699066162,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.09252598136663437,
"fcm_dpo/delta": -0.1881176084280014,
"fcm_dpo/margin": 6.241460800170898,
"fcm_dpo/q_t": 0.3727007806301117,
"grad_norm": 56.1482048034668,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6339062452316284,
"logits/rejected": -0.6129721403121948,
"logps/chosen": -53.50577926635742,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.2161865234375,
"loss": 0.9727,
"margin_dpo/margin_mean": 6.241460800170898,
"margin_dpo/margin_std": 6.26479434967041,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3959083557128906,
"KL/mean": -3.1197850704193115,
"KL/rejected_KL_mean": -5.843662261962891,
"KL/std": 5.133924961090088,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.08979058265686035,
"fcm_dpo/delta": -0.09366016089916229,
"fcm_dpo/margin": 5.44775390625,
"fcm_dpo/q_t": 0.3860328197479248,
"grad_norm": 67.31330108642578,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.590550422668457,
"logits/rejected": -0.5370299816131592,
"logps/chosen": -51.084571838378906,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.55905151367188,
"loss": 1.0022,
"margin_dpo/margin_mean": 5.447753429412842,
"margin_dpo/margin_std": 5.062417030334473,
"step": 54
},
{
"KL/chosen_KL_mean": -0.9079132080078125,
"KL/mean": -4.178384304046631,
"KL/rejected_KL_mean": -7.4488525390625,
"KL/std": 6.216054439544678,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.08719541132450104,
"fcm_dpo/delta": -0.18026816844940186,
"fcm_dpo/margin": 6.5409345626831055,
"fcm_dpo/q_t": 0.3730924427509308,
"grad_norm": 49.59605026245117,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6388111114501953,
"logits/rejected": -0.575678825378418,
"logps/chosen": -63.52314758300781,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.44235229492188,
"loss": 0.9889,
"margin_dpo/margin_mean": 6.540935516357422,
"margin_dpo/margin_std": 7.489261627197266,
"step": 55
},
{
"KL/chosen_KL_mean": -0.7456607818603516,
"KL/mean": -3.813748836517334,
"KL/rejected_KL_mean": -6.8818359375,
"KL/std": 6.068477630615234,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.08457927405834198,
"fcm_dpo/delta": -0.12541212141513824,
"fcm_dpo/margin": 6.136178016662598,
"fcm_dpo/q_t": 0.38377007842063904,
"grad_norm": 44.31554412841797,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.5918300151824951,
"logits/rejected": -0.548652708530426,
"logps/chosen": -58.67839050292969,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -101.0562744140625,
"loss": 1.0201,
"margin_dpo/margin_mean": 6.136178016662598,
"margin_dpo/margin_std": 7.338691711425781,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7408714294433594,
"KL/mean": -4.242293357849121,
"KL/rejected_KL_mean": -7.743717193603516,
"KL/std": 5.987005233764648,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.08201777935028076,
"fcm_dpo/delta": -0.18457511067390442,
"fcm_dpo/margin": 7.0028462409973145,
"fcm_dpo/q_t": 0.3672080636024475,
"grad_norm": 48.90700149536133,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5889043807983398,
"logits/rejected": -0.5612311363220215,
"logps/chosen": -71.23616027832031,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.30917358398438,
"loss": 0.9601,
"margin_dpo/margin_mean": 7.002847671508789,
"margin_dpo/margin_std": 6.586859703063965,
"step": 57
},
{
"KL/chosen_KL_mean": -0.9988975524902344,
"KL/mean": -4.867576599121094,
"KL/rejected_KL_mean": -8.736251831054688,
"KL/std": 6.84279727935791,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.078616663813591,
"fcm_dpo/delta": -0.22122687101364136,
"fcm_dpo/margin": 7.737354278564453,
"fcm_dpo/q_t": 0.36434701085090637,
"grad_norm": 50.516578674316406,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5851288437843323,
"logits/rejected": -0.5081213712692261,
"logps/chosen": -63.1318359375,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.35354614257812,
"loss": 0.9582,
"margin_dpo/margin_mean": 7.737354278564453,
"margin_dpo/margin_std": 8.133914947509766,
"step": 58
},
{
"KL/chosen_KL_mean": -1.5034770965576172,
"KL/mean": -5.772602081298828,
"KL/rejected_KL_mean": -10.041728973388672,
"KL/std": 7.538321495056152,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.07399021834135056,
"fcm_dpo/delta": -0.25073447823524475,
"fcm_dpo/margin": 8.53825569152832,
"fcm_dpo/q_t": 0.35876306891441345,
"grad_norm": 47.74569320678711,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.637381911277771,
"logits/rejected": -0.5979089140892029,
"logps/chosen": -53.436004638671875,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.92692565917969,
"loss": 0.9422,
"margin_dpo/margin_mean": 8.538256645202637,
"margin_dpo/margin_std": 8.657896041870117,
"step": 59
},
{
"KL/chosen_KL_mean": -2.7102012634277344,
"KL/mean": -6.095211982727051,
"KL/rejected_KL_mean": -9.480220794677734,
"KL/std": 6.847818374633789,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.07247201353311539,
"fcm_dpo/delta": -0.07915282994508743,
"fcm_dpo/margin": 6.77001953125,
"fcm_dpo/q_t": 0.3859708905220032,
"grad_norm": 50.09484100341797,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6102770566940308,
"logits/rejected": -0.5501413941383362,
"logps/chosen": -63.65238952636719,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.87361907958984,
"loss": 1.029,
"margin_dpo/margin_mean": 6.770020008087158,
"margin_dpo/margin_std": 7.763751029968262,
"step": 60
},
{
"KL/chosen_KL_mean": -1.4959545135498047,
"KL/mean": -5.729641914367676,
"KL/rejected_KL_mean": -9.963333129882812,
"KL/std": 9.229528427124023,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.07002333551645279,
"fcm_dpo/delta": -0.20614930987358093,
"fcm_dpo/margin": 8.467378616333008,
"fcm_dpo/q_t": 0.375395804643631,
"grad_norm": 42.78828811645508,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.5964009761810303,
"logits/rejected": -0.5620393753051758,
"logps/chosen": -62.129478454589844,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.81582641601562,
"loss": 1.002,
"margin_dpo/margin_mean": 8.467377662658691,
"margin_dpo/margin_std": 11.42286491394043,
"step": 61
},
{
"KL/chosen_KL_mean": -1.7143287658691406,
"KL/mean": -4.863595008850098,
"KL/rejected_KL_mean": -8.012863159179688,
"KL/std": 6.748926162719727,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.06915012747049332,
"fcm_dpo/delta": -0.0570698156952858,
"fcm_dpo/margin": 6.2985358238220215,
"fcm_dpo/q_t": 0.40163713693618774,
"grad_norm": 41.4459228515625,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.6113085746765137,
"logits/rejected": -0.5773282647132874,
"logps/chosen": -57.8651008605957,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.57905578613281,
"loss": 1.0737,
"margin_dpo/margin_mean": 6.29853630065918,
"margin_dpo/margin_std": 8.275651931762695,
"step": 62
},
{
"KL/chosen_KL_mean": -2.811429977416992,
"KL/mean": -7.000448226928711,
"KL/rejected_KL_mean": -11.189468383789062,
"KL/std": 8.535360336303711,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.06666617840528488,
"fcm_dpo/delta": -0.16953536868095398,
"fcm_dpo/margin": 8.378037452697754,
"fcm_dpo/q_t": 0.3721930980682373,
"grad_norm": 44.12747573852539,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.5932517051696777,
"logits/rejected": -0.5480700731277466,
"logps/chosen": -75.95882415771484,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -108.79953002929688,
"loss": 0.9855,
"margin_dpo/margin_mean": 8.378036499023438,
"margin_dpo/margin_std": 8.932016372680664,
"step": 63
},
{
"KL/chosen_KL_mean": -1.1035079956054688,
"KL/mean": -6.1234307289123535,
"KL/rejected_KL_mean": -11.143352508544922,
"KL/std": 9.246552467346191,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.06336793303489685,
"fcm_dpo/delta": -0.2566346526145935,
"fcm_dpo/margin": 10.039838790893555,
"fcm_dpo/q_t": 0.3602312207221985,
"grad_norm": 39.74605941772461,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.5810163021087646,
"logits/rejected": -0.5485190749168396,
"logps/chosen": -55.102108001708984,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.67355346679688,
"loss": 0.9588,
"margin_dpo/margin_mean": 10.039839744567871,
"margin_dpo/margin_std": 11.341650009155273,
"step": 64
},
{
"KL/chosen_KL_mean": -3.4348411560058594,
"KL/mean": -8.460206985473633,
"KL/rejected_KL_mean": -13.485572814941406,
"KL/std": 10.407320022583008,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.06073646619915962,
"fcm_dpo/delta": -0.22589105367660522,
"fcm_dpo/margin": 10.050735473632812,
"fcm_dpo/q_t": 0.3629263639450073,
"grad_norm": 40.756656646728516,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6741570830345154,
"logits/rejected": -0.6617323756217957,
"logps/chosen": -68.27084350585938,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -123.43203735351562,
"loss": 0.9743,
"margin_dpo/margin_mean": 10.050735473632812,
"margin_dpo/margin_std": 11.43245792388916,
"step": 65
},
{
"KL/chosen_KL_mean": -3.1920318603515625,
"KL/mean": -7.833024501800537,
"KL/rejected_KL_mean": -12.474014282226562,
"KL/std": 9.44174861907959,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.058656755834817886,
"fcm_dpo/delta": -0.14220839738845825,
"fcm_dpo/margin": 9.28198528289795,
"fcm_dpo/q_t": 0.3805831968784332,
"grad_norm": 36.21018981933594,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6206960678100586,
"logits/rejected": -0.586733341217041,
"logps/chosen": -54.63555908203125,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -88.11031341552734,
"loss": 1.0098,
"margin_dpo/margin_mean": 9.28198528289795,
"margin_dpo/margin_std": 11.416315078735352,
"step": 66
},
{
"KL/chosen_KL_mean": -2.8563461303710938,
"KL/mean": -7.659085273742676,
"KL/rejected_KL_mean": -12.461822509765625,
"KL/std": 9.597047805786133,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.05742337554693222,
"fcm_dpo/delta": -0.16015625,
"fcm_dpo/margin": 9.605472564697266,
"fcm_dpo/q_t": 0.3784703016281128,
"grad_norm": 37.310569763183594,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.605553150177002,
"logits/rejected": -0.563081681728363,
"logps/chosen": -62.19715118408203,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -85.24910736083984,
"loss": 1.0033,
"margin_dpo/margin_mean": 9.605473518371582,
"margin_dpo/margin_std": 11.608689308166504,
"step": 67
},
{
"KL/chosen_KL_mean": -3.41162109375,
"KL/mean": -7.9624128341674805,
"KL/rejected_KL_mean": -12.513202667236328,
"KL/std": 8.548230171203613,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.05597271770238876,
"fcm_dpo/delta": -0.11515870690345764,
"fcm_dpo/margin": 9.101577758789062,
"fcm_dpo/q_t": 0.3834843039512634,
"grad_norm": 35.94131088256836,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6418497562408447,
"logits/rejected": -0.5813031196594238,
"logps/chosen": -68.6174545288086,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -89.72045135498047,
"loss": 1.0057,
"margin_dpo/margin_mean": 9.101577758789062,
"margin_dpo/margin_std": 9.605649948120117,
"step": 68
},
{
"KL/chosen_KL_mean": -4.020790100097656,
"KL/mean": -9.76811408996582,
"KL/rejected_KL_mean": -15.51544189453125,
"KL/std": 10.38883113861084,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.053416017442941666,
"fcm_dpo/delta": -0.2294478863477707,
"fcm_dpo/margin": 11.494646072387695,
"fcm_dpo/q_t": 0.36246609687805176,
"grad_norm": 38.16155242919922,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.5999042987823486,
"logits/rejected": -0.5770286321640015,
"logps/chosen": -63.840030670166016,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -118.90430450439453,
"loss": 0.945,
"margin_dpo/margin_mean": 11.494646072387695,
"margin_dpo/margin_std": 11.5130615234375,
"step": 69
},
{
"KL/chosen_KL_mean": -5.912725448608398,
"KL/mean": -12.12542724609375,
"KL/rejected_KL_mean": -18.338130950927734,
"KL/std": 12.911046028137207,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.05124839395284653,
"fcm_dpo/delta": -0.2522848844528198,
"fcm_dpo/margin": 12.425400733947754,
"fcm_dpo/q_t": 0.36073604226112366,
"grad_norm": 38.75048065185547,
"learning_rate": 5e-07,
"logits/chosen": -0.606624960899353,
"logits/rejected": -0.5714987516403198,
"logps/chosen": -67.84336853027344,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -109.39891052246094,
"loss": 0.9549,
"margin_dpo/margin_mean": 12.425400733947754,
"margin_dpo/margin_std": 13.759818077087402,
"step": 70
},
{
"KL/chosen_KL_mean": -5.871559143066406,
"KL/mean": -13.31947135925293,
"KL/rejected_KL_mean": -20.76738739013672,
"KL/std": 13.823541641235352,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.047790415585041046,
"fcm_dpo/delta": -0.3372606635093689,
"fcm_dpo/margin": 14.895828247070312,
"fcm_dpo/q_t": 0.34655076265335083,
"grad_norm": 35.317962646484375,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6532018184661865,
"logits/rejected": -0.6144231557846069,
"logps/chosen": -67.62189483642578,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -118.10400390625,
"loss": 0.9032,
"margin_dpo/margin_mean": 14.895828247070312,
"margin_dpo/margin_std": 15.058696746826172,
"step": 71
},
{
"KL/chosen_KL_mean": -6.858076095581055,
"KL/mean": -14.622879028320312,
"KL/rejected_KL_mean": -22.387680053710938,
"KL/std": 14.882482528686523,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.044840212911367416,
"fcm_dpo/delta": -0.31945478916168213,
"fcm_dpo/margin": 15.529606819152832,
"fcm_dpo/q_t": 0.3486083149909973,
"grad_norm": 35.28294372558594,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6572631597518921,
"logits/rejected": -0.6205873489379883,
"logps/chosen": -72.91148376464844,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -117.67466735839844,
"loss": 0.9327,
"margin_dpo/margin_mean": 15.529606819152832,
"margin_dpo/margin_std": 17.4892520904541,
"step": 72
},
{
"KL/chosen_KL_mean": -8.918149948120117,
"KL/mean": -16.7022705078125,
"KL/rejected_KL_mean": -24.48639678955078,
"KL/std": 19.29153060913086,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.04222744703292847,
"fcm_dpo/delta": -0.2773255407810211,
"fcm_dpo/margin": 15.568246841430664,
"fcm_dpo/q_t": 0.3702467083930969,
"grad_norm": 36.51060104370117,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.7001978158950806,
"logits/rejected": -0.6606754064559937,
"logps/chosen": -75.17442321777344,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -114.94253540039062,
"loss": 1.0394,
"margin_dpo/margin_mean": 15.568246841430664,
"margin_dpo/margin_std": 24.206798553466797,
"step": 73
},
{
"KL/chosen_KL_mean": -9.382577896118164,
"KL/mean": -17.968414306640625,
"KL/rejected_KL_mean": -26.554244995117188,
"KL/std": 21.26626205444336,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.039852894842624664,
"fcm_dpo/delta": -0.3063187003135681,
"fcm_dpo/margin": 17.171672821044922,
"fcm_dpo/q_t": 0.36331337690353394,
"grad_norm": 35.83381652832031,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6333750486373901,
"logits/rejected": -0.6257501244544983,
"logps/chosen": -62.80746078491211,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -122.50118255615234,
"loss": 0.9965,
"margin_dpo/margin_mean": 17.171672821044922,
"margin_dpo/margin_std": 26.445632934570312,
"step": 74
},
{
"KL/chosen_KL_mean": -9.15802001953125,
"KL/mean": -21.245464324951172,
"KL/rejected_KL_mean": -33.33289337158203,
"KL/std": 21.812652587890625,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.03664487600326538,
"fcm_dpo/delta": -0.530461847782135,
"fcm_dpo/margin": 24.17488670349121,
"fcm_dpo/q_t": 0.31928879022598267,
"grad_norm": 31.040130615234375,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6646023988723755,
"logits/rejected": -0.6575799584388733,
"logps/chosen": -61.019683837890625,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -144.58688354492188,
"loss": 0.8454,
"margin_dpo/margin_mean": 24.174884796142578,
"margin_dpo/margin_std": 24.763885498046875,
"step": 75
},
{
"KL/chosen_KL_mean": -11.104362487792969,
"KL/mean": -18.701168060302734,
"KL/rejected_KL_mean": -26.297969818115234,
"KL/std": 17.725784301757812,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.03470379486680031,
"fcm_dpo/delta": -0.13428181409835815,
"fcm_dpo/margin": 15.193618774414062,
"fcm_dpo/q_t": 0.38014188408851624,
"grad_norm": 32.03413009643555,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.690595269203186,
"logits/rejected": -0.6483590006828308,
"logps/chosen": -64.37039947509766,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -104.51460266113281,
"loss": 1.0267,
"margin_dpo/margin_mean": 15.193617820739746,
"margin_dpo/margin_std": 19.406496047973633,
"step": 76
},
{
"KL/chosen_KL_mean": -9.964458465576172,
"KL/mean": -22.325725555419922,
"KL/rejected_KL_mean": -34.686988830566406,
"KL/std": 24.06493377685547,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.032092347741127014,
"fcm_dpo/delta": -0.4313170313835144,
"fcm_dpo/margin": 24.722532272338867,
"fcm_dpo/q_t": 0.32912200689315796,
"grad_norm": 33.1654052734375,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6741265058517456,
"logits/rejected": -0.6357508897781372,
"logps/chosen": -68.06114196777344,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -128.46060180664062,
"loss": 0.8956,
"margin_dpo/margin_mean": 24.7225341796875,
"margin_dpo/margin_std": 26.815866470336914,
"step": 77
},
{
"KL/chosen_KL_mean": -9.932151794433594,
"KL/mean": -19.229045867919922,
"KL/rejected_KL_mean": -28.525943756103516,
"KL/std": 19.422195434570312,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.030831564217805862,
"fcm_dpo/delta": -0.18346719443798065,
"fcm_dpo/margin": 18.593788146972656,
"fcm_dpo/q_t": 0.3741081953048706,
"grad_norm": 30.061494827270508,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6306928396224976,
"logits/rejected": -0.6057921648025513,
"logps/chosen": -65.54593658447266,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -113.46031188964844,
"loss": 0.9976,
"margin_dpo/margin_mean": 18.593788146972656,
"margin_dpo/margin_std": 22.29560089111328,
"step": 78
},
{
"KL/chosen_KL_mean": -10.282918930053711,
"KL/mean": -20.42000961303711,
"KL/rejected_KL_mean": -30.55709457397461,
"KL/std": 21.94013214111328,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.02950279787182808,
"fcm_dpo/delta": -0.21087872982025146,
"fcm_dpo/margin": 20.274171829223633,
"fcm_dpo/q_t": 0.3741893768310547,
"grad_norm": 26.36469078063965,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6447913646697998,
"logits/rejected": -0.6144734025001526,
"logps/chosen": -65.73340606689453,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -118.20465850830078,
"loss": 1.0083,
"margin_dpo/margin_mean": 20.274171829223633,
"margin_dpo/margin_std": 26.865041732788086,
"step": 79
},
{
"KL/chosen_KL_mean": -13.432994842529297,
"KL/mean": -22.9481201171875,
"KL/rejected_KL_mean": -32.46324157714844,
"KL/std": 23.311492919921875,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.028320040553808212,
"fcm_dpo/delta": -0.1528511494398117,
"fcm_dpo/margin": 19.03024673461914,
"fcm_dpo/q_t": 0.3878602981567383,
"grad_norm": 27.577178955078125,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7253065705299377,
"logits/rejected": -0.6845601201057434,
"logps/chosen": -71.95228576660156,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -120.0107421875,
"loss": 1.0491,
"margin_dpo/margin_mean": 19.030248641967773,
"margin_dpo/margin_std": 27.02452850341797,
"step": 80
},
{
"KL/chosen_KL_mean": -12.46990966796875,
"KL/mean": -25.47945213317871,
"KL/rejected_KL_mean": -38.48899841308594,
"KL/std": 27.457473754882812,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.02683407999575138,
"fcm_dpo/delta": -0.32329061627388,
"fcm_dpo/margin": 26.019084930419922,
"fcm_dpo/q_t": 0.35463476181030273,
"grad_norm": 28.8442325592041,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.6985138654708862,
"logits/rejected": -0.6699087619781494,
"logps/chosen": -78.91877746582031,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -168.15170288085938,
"loss": 0.9548,
"margin_dpo/margin_mean": 26.019084930419922,
"margin_dpo/margin_std": 32.673709869384766,
"step": 81
},
{
"KL/chosen_KL_mean": -14.771663665771484,
"KL/mean": -25.181900024414062,
"KL/rejected_KL_mean": -35.592132568359375,
"KL/std": 25.033601760864258,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.026078231632709503,
"fcm_dpo/delta": -0.17972172796726227,
"fcm_dpo/margin": 20.82046890258789,
"fcm_dpo/q_t": 0.38705772161483765,
"grad_norm": 31.77438735961914,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6351274251937866,
"logits/rejected": -0.6246193647384644,
"logps/chosen": -67.0040512084961,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -126.33538818359375,
"loss": 1.0902,
"margin_dpo/margin_mean": 20.82046890258789,
"margin_dpo/margin_std": 34.549964904785156,
"step": 82
},
{
"KL/chosen_KL_mean": -15.13831901550293,
"KL/mean": -27.50519371032715,
"KL/rejected_KL_mean": -39.872066497802734,
"KL/std": 26.05057716369629,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.024679036810994148,
"fcm_dpo/delta": -0.22516962885856628,
"fcm_dpo/margin": 24.733749389648438,
"fcm_dpo/q_t": 0.36837995052337646,
"grad_norm": 26.978626251220703,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6816978454589844,
"logits/rejected": -0.6491335034370422,
"logps/chosen": -70.96570587158203,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -143.5879669189453,
"loss": 0.9914,
"margin_dpo/margin_mean": 24.733749389648438,
"margin_dpo/margin_std": 30.88388442993164,
"step": 83
},
{
"KL/chosen_KL_mean": -14.203386306762695,
"KL/mean": -24.5653076171875,
"KL/rejected_KL_mean": -34.9272346496582,
"KL/std": 22.138011932373047,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.02386614680290222,
"fcm_dpo/delta": -0.12736207246780396,
"fcm_dpo/margin": 20.723844528198242,
"fcm_dpo/q_t": 0.38502997159957886,
"grad_norm": 25.739469528198242,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6767555475234985,
"logits/rejected": -0.6396181583404541,
"logps/chosen": -81.37955474853516,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -122.22583770751953,
"loss": 1.0217,
"margin_dpo/margin_mean": 20.723844528198242,
"margin_dpo/margin_std": 22.54238510131836,
"step": 84
},
{
"KL/chosen_KL_mean": -14.595979690551758,
"KL/mean": -25.3236083984375,
"KL/rejected_KL_mean": -36.051231384277344,
"KL/std": 22.786161422729492,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.023556005209684372,
"fcm_dpo/delta": -0.13166888058185577,
"fcm_dpo/margin": 21.45526123046875,
"fcm_dpo/q_t": 0.3877854645252228,
"grad_norm": 25.446636199951172,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6357869505882263,
"logits/rejected": -0.6025545001029968,
"logps/chosen": -73.00259399414062,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -114.69003295898438,
"loss": 1.0431,
"margin_dpo/margin_mean": 21.45526123046875,
"margin_dpo/margin_std": 27.411766052246094,
"step": 85
},
{
"KL/chosen_KL_mean": -18.884050369262695,
"KL/mean": -30.216419219970703,
"KL/rejected_KL_mean": -41.54878616333008,
"KL/std": 30.91357421875,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.02299971505999565,
"fcm_dpo/delta": -0.1070268452167511,
"fcm_dpo/margin": 22.664737701416016,
"fcm_dpo/q_t": 0.3989836275577545,
"grad_norm": 39.153846740722656,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6874663829803467,
"logits/rejected": -0.6762598156929016,
"logps/chosen": -75.02151489257812,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -129.67044067382812,
"loss": 1.1246,
"margin_dpo/margin_mean": 22.664737701416016,
"margin_dpo/margin_std": 42.79364776611328,
"step": 86
},
{
"KL/chosen_KL_mean": -18.476444244384766,
"KL/mean": -31.937850952148438,
"KL/rejected_KL_mean": -45.39925765991211,
"KL/std": 30.784648895263672,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.021954335272312164,
"fcm_dpo/delta": -0.20476816594600677,
"fcm_dpo/margin": 26.92281723022461,
"fcm_dpo/q_t": 0.3721076250076294,
"grad_norm": 25.699562072753906,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.7187229990959167,
"logits/rejected": -0.6957570910453796,
"logps/chosen": -74.11254119873047,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -140.86683654785156,
"loss": 1.0289,
"margin_dpo/margin_mean": 26.92281723022461,
"margin_dpo/margin_std": 37.453208923339844,
"step": 87
},
{
"KL/chosen_KL_mean": -23.373714447021484,
"KL/mean": -33.52701187133789,
"KL/rejected_KL_mean": -43.68030548095703,
"KL/std": 32.281776428222656,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.02150532230734825,
"fcm_dpo/delta": -0.07733839750289917,
"fcm_dpo/margin": 20.306591033935547,
"fcm_dpo/q_t": 0.4068375527858734,
"grad_norm": 26.6627140045166,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6914920806884766,
"logits/rejected": -0.6863746643066406,
"logps/chosen": -97.04486083984375,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -150.38880920410156,
"loss": 1.1317,
"margin_dpo/margin_mean": 20.306591033935547,
"margin_dpo/margin_std": 34.122745513916016,
"step": 88
},
{
"KL/chosen_KL_mean": -14.407289505004883,
"KL/mean": -25.77096176147461,
"KL/rejected_KL_mean": -37.1346321105957,
"KL/std": 27.240581512451172,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.02131272479891777,
"fcm_dpo/delta": -0.09037049859762192,
"fcm_dpo/margin": 22.727336883544922,
"fcm_dpo/q_t": 0.3924558758735657,
"grad_norm": 24.11750030517578,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7111062407493591,
"logits/rejected": -0.6786768436431885,
"logps/chosen": -75.0322036743164,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -119.21818542480469,
"loss": 1.0546,
"margin_dpo/margin_mean": 22.727336883544922,
"margin_dpo/margin_std": 31.17041778564453,
"step": 89
},
{
"KL/chosen_KL_mean": -18.334720611572266,
"KL/mean": -33.93196105957031,
"KL/rejected_KL_mean": -49.52920150756836,
"KL/std": 36.926090240478516,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.02063620463013649,
"fcm_dpo/delta": -0.23454345762729645,
"fcm_dpo/margin": 31.19448471069336,
"fcm_dpo/q_t": 0.37270212173461914,
"grad_norm": 25.166067123413086,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.640715479850769,
"logits/rejected": -0.655485987663269,
"logps/chosen": -71.62003326416016,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -161.07391357421875,
"loss": 1.0296,
"margin_dpo/margin_mean": 31.194486618041992,
"margin_dpo/margin_std": 45.882774353027344,
"step": 90
},
{
"KL/chosen_KL_mean": -19.304616928100586,
"KL/mean": -32.036643981933594,
"KL/rejected_KL_mean": -44.76866912841797,
"KL/std": 29.606094360351562,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.019963329657912254,
"fcm_dpo/delta": -0.09465456753969193,
"fcm_dpo/margin": 25.464061737060547,
"fcm_dpo/q_t": 0.3919849693775177,
"grad_norm": 24.47185516357422,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6711220741271973,
"logits/rejected": -0.6374910473823547,
"logps/chosen": -81.10757446289062,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -132.6426239013672,
"loss": 1.0706,
"margin_dpo/margin_mean": 25.464059829711914,
"margin_dpo/margin_std": 38.521697998046875,
"step": 91
},
{
"KL/chosen_KL_mean": -17.32529640197754,
"KL/mean": -30.216405868530273,
"KL/rejected_KL_mean": -43.10751724243164,
"KL/std": 30.38365936279297,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.01955636963248253,
"fcm_dpo/delta": -0.1114797294139862,
"fcm_dpo/margin": 25.7822208404541,
"fcm_dpo/q_t": 0.39096271991729736,
"grad_norm": 22.75739860534668,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.7390532493591309,
"logits/rejected": -0.7194732427597046,
"logps/chosen": -68.966064453125,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -120.98870086669922,
"loss": 1.0599,
"margin_dpo/margin_mean": 25.782222747802734,
"margin_dpo/margin_std": 36.745933532714844,
"step": 92
},
{
"KL/chosen_KL_mean": -19.497028350830078,
"KL/mean": -32.41360855102539,
"KL/rejected_KL_mean": -45.33018112182617,
"KL/std": 28.625198364257812,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.019083872437477112,
"fcm_dpo/delta": -0.09772805124521255,
"fcm_dpo/margin": 25.83315658569336,
"fcm_dpo/q_t": 0.390662282705307,
"grad_norm": 23.420873641967773,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7268893718719482,
"logits/rejected": -0.6788473129272461,
"logps/chosen": -72.0262680053711,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -122.49092864990234,
"loss": 1.0385,
"margin_dpo/margin_mean": 25.833154678344727,
"margin_dpo/margin_std": 33.194129943847656,
"step": 93
},
{
"KL/chosen_KL_mean": -20.87965965270996,
"KL/mean": -35.868045806884766,
"KL/rejected_KL_mean": -50.8564338684082,
"KL/std": 32.95391845703125,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.01840830221772194,
"fcm_dpo/delta": -0.1622575968503952,
"fcm_dpo/margin": 29.97677993774414,
"fcm_dpo/q_t": 0.37686464190483093,
"grad_norm": 23.023033142089844,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.69089674949646,
"logits/rejected": -0.6784852147102356,
"logps/chosen": -82.1022720336914,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -150.45547485351562,
"loss": 0.9994,
"margin_dpo/margin_mean": 29.97677993774414,
"margin_dpo/margin_std": 34.474708557128906,
"step": 94
},
{
"KL/chosen_KL_mean": -20.05255699157715,
"KL/mean": -32.12548065185547,
"KL/rejected_KL_mean": -44.198402404785156,
"KL/std": 31.86922836303711,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.018146492540836334,
"fcm_dpo/delta": -0.05808256193995476,
"fcm_dpo/margin": 24.145843505859375,
"fcm_dpo/q_t": 0.39963364601135254,
"grad_norm": 21.70364761352539,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.6787654161453247,
"logits/rejected": -0.6507630348205566,
"logps/chosen": -72.57620239257812,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -120.07875061035156,
"loss": 1.0803,
"margin_dpo/margin_mean": 24.145843505859375,
"margin_dpo/margin_std": 33.2855339050293,
"step": 95
},
{
"KL/chosen_KL_mean": -19.471036911010742,
"KL/mean": -35.86725616455078,
"KL/rejected_KL_mean": -52.26348114013672,
"KL/std": 35.56018829345703,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.01755816489458084,
"fcm_dpo/delta": -0.16479085385799408,
"fcm_dpo/margin": 32.792442321777344,
"fcm_dpo/q_t": 0.37453415989875793,
"grad_norm": 22.211780548095703,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.7198976278305054,
"logits/rejected": -0.69416344165802,
"logps/chosen": -81.62800598144531,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -148.8594970703125,
"loss": 0.9915,
"margin_dpo/margin_mean": 32.792442321777344,
"margin_dpo/margin_std": 39.19865417480469,
"step": 96
},
{
"KL/chosen_KL_mean": -20.839025497436523,
"KL/mean": -33.39268493652344,
"KL/rejected_KL_mean": -45.94635009765625,
"KL/std": 29.86338996887207,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.017377909272909164,
"fcm_dpo/delta": -0.10373032093048096,
"fcm_dpo/margin": 25.10732078552246,
"fcm_dpo/q_t": 0.39860257506370544,
"grad_norm": 22.988815307617188,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6391160488128662,
"logits/rejected": -0.5976995229721069,
"logps/chosen": -75.48538970947266,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -122.91110229492188,
"loss": 1.066,
"margin_dpo/margin_mean": 25.10732078552246,
"margin_dpo/margin_std": 29.785751342773438,
"step": 97
},
{
"KL/chosen_KL_mean": -25.700241088867188,
"KL/mean": -40.22949981689453,
"KL/rejected_KL_mean": -54.758758544921875,
"KL/std": 34.170616149902344,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.01692095398902893,
"fcm_dpo/delta": -0.09668763726949692,
"fcm_dpo/margin": 29.058513641357422,
"fcm_dpo/q_t": 0.3878336548805237,
"grad_norm": 24.090316772460938,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.6487174034118652,
"logits/rejected": -0.6329121589660645,
"logps/chosen": -90.95886993408203,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -151.2862548828125,
"loss": 1.0659,
"margin_dpo/margin_mean": 29.058513641357422,
"margin_dpo/margin_std": 41.047019958496094,
"step": 98
},
{
"KL/chosen_KL_mean": -20.114553451538086,
"KL/mean": -37.91062545776367,
"KL/rejected_KL_mean": -55.706695556640625,
"KL/std": 36.737327575683594,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.016340188682079315,
"fcm_dpo/delta": -0.19358737766742706,
"fcm_dpo/margin": 35.59214782714844,
"fcm_dpo/q_t": 0.3759217858314514,
"grad_norm": 20.969350814819336,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6549317240715027,
"logits/rejected": -0.6567766666412354,
"logps/chosen": -65.75303649902344,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -142.14462280273438,
"loss": 0.9999,
"margin_dpo/margin_mean": 35.59214401245117,
"margin_dpo/margin_std": 45.7916145324707,
"step": 99
},
{
"KL/chosen_KL_mean": -22.74945640563965,
"KL/mean": -32.91205596923828,
"KL/rejected_KL_mean": -43.07464599609375,
"KL/std": 29.567760467529297,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.01622767373919487,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.325191497802734,
"fcm_dpo/q_t": 0.425083190202713,
"grad_norm": 23.01412582397461,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6575253009796143,
"logits/rejected": -0.615835428237915,
"logps/chosen": -80.34343719482422,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -117.13485717773438,
"loss": 1.1781,
"margin_dpo/margin_mean": 20.325193405151367,
"margin_dpo/margin_std": 38.162071228027344,
"step": 100
},
{
"KL/chosen_KL_mean": -27.887283325195312,
"KL/mean": -40.41047668457031,
"KL/rejected_KL_mean": -52.93367385864258,
"KL/std": 36.532798767089844,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.016045931726694107,
"fcm_dpo/delta": -0.05631338432431221,
"fcm_dpo/margin": 25.046384811401367,
"fcm_dpo/q_t": 0.4135172367095947,
"grad_norm": 22.6212215423584,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6584815979003906,
"logits/rejected": -0.6281433701515198,
"logps/chosen": -89.5361328125,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -136.12335205078125,
"loss": 1.1374,
"margin_dpo/margin_mean": 25.04638671875,
"margin_dpo/margin_std": 43.94956970214844,
"step": 101
},
{
"KL/chosen_KL_mean": -29.988710403442383,
"KL/mean": -39.130943298339844,
"KL/rejected_KL_mean": -48.27317428588867,
"KL/std": 34.25249099731445,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.016045931726694107,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.284465789794922,
"fcm_dpo/q_t": 0.43444907665252686,
"grad_norm": 25.962202072143555,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.6689609289169312,
"logits/rejected": -0.6241482496261597,
"logps/chosen": -94.06758880615234,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -116.46025085449219,
"loss": 1.2176,
"margin_dpo/margin_mean": 18.284465789794922,
"margin_dpo/margin_std": 40.5817985534668,
"step": 102
},
{
"KL/chosen_KL_mean": -23.946800231933594,
"KL/mean": -42.415340423583984,
"KL/rejected_KL_mean": -60.88387680053711,
"KL/std": 38.02168273925781,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.015578062273561954,
"fcm_dpo/delta": -0.18654988706111908,
"fcm_dpo/margin": 36.937076568603516,
"fcm_dpo/q_t": 0.3750496506690979,
"grad_norm": 22.1878719329834,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6738394498825073,
"logits/rejected": -0.6411100029945374,
"logps/chosen": -85.24607849121094,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -154.4565887451172,
"loss": 0.9984,
"margin_dpo/margin_mean": 36.937076568603516,
"margin_dpo/margin_std": 46.08087158203125,
"step": 103
},
{
"KL/chosen_KL_mean": -25.76188087463379,
"KL/mean": -43.38178634643555,
"KL/rejected_KL_mean": -61.001686096191406,
"KL/std": 41.89414978027344,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.01516028307378292,
"fcm_dpo/delta": -0.14165204763412476,
"fcm_dpo/margin": 35.23980712890625,
"fcm_dpo/q_t": 0.3880753517150879,
"grad_norm": 21.539640426635742,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.6753038167953491,
"logits/rejected": -0.6453461647033691,
"logps/chosen": -80.13465881347656,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -150.56640625,
"loss": 1.0513,
"margin_dpo/margin_mean": 35.23980712890625,
"margin_dpo/margin_std": 51.31709671020508,
"step": 104
},
{
"KL/chosen_KL_mean": -24.360563278198242,
"KL/mean": -49.227760314941406,
"KL/rejected_KL_mean": -74.09496307373047,
"KL/std": 40.60966873168945,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.014314282685518265,
"fcm_dpo/delta": -0.33550798892974854,
"fcm_dpo/margin": 49.734397888183594,
"fcm_dpo/q_t": 0.33919352293014526,
"grad_norm": 22.338714599609375,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.6681811809539795,
"logits/rejected": -0.6469371914863586,
"logps/chosen": -78.99951171875,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -172.0684814453125,
"loss": 0.8853,
"margin_dpo/margin_mean": 49.734397888183594,
"margin_dpo/margin_std": 45.006996154785156,
"step": 105
},
{
"KL/chosen_KL_mean": -27.47538185119629,
"KL/mean": -42.99120330810547,
"KL/rejected_KL_mean": -58.50702667236328,
"KL/std": 35.57778549194336,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.013949505984783173,
"fcm_dpo/delta": -0.03634671866893768,
"fcm_dpo/margin": 31.031639099121094,
"fcm_dpo/q_t": 0.4003314971923828,
"grad_norm": 21.185850143432617,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.6308009624481201,
"logits/rejected": -0.602495014667511,
"logps/chosen": -82.30827331542969,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -143.7316436767578,
"loss": 1.0686,
"margin_dpo/margin_mean": 31.031639099121094,
"margin_dpo/margin_std": 38.763301849365234,
"step": 106
},
{
"KL/chosen_KL_mean": -32.56428909301758,
"KL/mean": -50.00350570678711,
"KL/rejected_KL_mean": -67.4427261352539,
"KL/std": 43.262245178222656,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.013720358721911907,
"fcm_dpo/delta": -0.08281677961349487,
"fcm_dpo/margin": 34.87843704223633,
"fcm_dpo/q_t": 0.39147496223449707,
"grad_norm": 20.624536514282227,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.6635041832923889,
"logits/rejected": -0.6185137629508972,
"logps/chosen": -102.27210235595703,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -162.18223571777344,
"loss": 1.0561,
"margin_dpo/margin_mean": 34.87843704223633,
"margin_dpo/margin_std": 45.82408905029297,
"step": 107
},
{
"KL/chosen_KL_mean": -29.11627769470215,
"KL/mean": -50.483489990234375,
"KL/rejected_KL_mean": -71.85070037841797,
"KL/std": 48.153316497802734,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.013360362499952316,
"fcm_dpo/delta": -0.18124811351299286,
"fcm_dpo/margin": 42.73442459106445,
"fcm_dpo/q_t": 0.3816313147544861,
"grad_norm": 20.98419189453125,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.6446486711502075,
"logits/rejected": -0.6245523691177368,
"logps/chosen": -85.12615966796875,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -167.64671325683594,
"loss": 1.0411,
"margin_dpo/margin_mean": 42.73442840576172,
"margin_dpo/margin_std": 62.17555236816406,
"step": 108
},
{
"KL/chosen_KL_mean": -27.09517478942871,
"KL/mean": -49.685211181640625,
"KL/rejected_KL_mean": -72.27523803710938,
"KL/std": 45.911842346191406,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.012815423309803009,
"fcm_dpo/delta": -0.19097067415714264,
"fcm_dpo/margin": 45.18007278442383,
"fcm_dpo/q_t": 0.37120842933654785,
"grad_norm": 21.367916107177734,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.5958420634269714,
"logits/rejected": -0.5749087333679199,
"logps/chosen": -89.98066711425781,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -170.96096801757812,
"loss": 0.9976,
"margin_dpo/margin_mean": 45.18007278442383,
"margin_dpo/margin_std": 54.63614273071289,
"step": 109
},
{
"KL/chosen_KL_mean": -28.124114990234375,
"KL/mean": -48.460716247558594,
"KL/rejected_KL_mean": -68.79730987548828,
"KL/std": 45.917694091796875,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.012292098253965378,
"fcm_dpo/delta": -0.1774018108844757,
"fcm_dpo/margin": 40.673194885253906,
"fcm_dpo/q_t": 0.38942158222198486,
"grad_norm": 18.53866958618164,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.5888636708259583,
"logits/rejected": -0.5525768995285034,
"logps/chosen": -86.87779998779297,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -148.54733276367188,
"loss": 1.0605,
"margin_dpo/margin_mean": 40.673194885253906,
"margin_dpo/margin_std": 55.483089447021484,
"step": 110
},
{
"KL/chosen_KL_mean": -31.884164810180664,
"KL/mean": -55.051116943359375,
"KL/rejected_KL_mean": -78.21807861328125,
"KL/std": 51.810791015625,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.011932818219065666,
"fcm_dpo/delta": -0.16332484781742096,
"fcm_dpo/margin": 46.33391571044922,
"fcm_dpo/q_t": 0.3792319893836975,
"grad_norm": 20.914552688598633,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.6331917643547058,
"logits/rejected": -0.6107040643692017,
"logps/chosen": -100.50827026367188,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -176.64694213867188,
"loss": 1.0404,
"margin_dpo/margin_mean": 46.33391571044922,
"margin_dpo/margin_std": 64.15996551513672,
"step": 111
},
{
"KL/chosen_KL_mean": -29.431987762451172,
"KL/mean": -44.088584899902344,
"KL/rejected_KL_mean": -58.74518966674805,
"KL/std": 35.674102783203125,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.011897064745426178,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 29.31319808959961,
"fcm_dpo/q_t": 0.4195147752761841,
"grad_norm": 18.906843185424805,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.6224333047866821,
"logits/rejected": -0.5856015086174011,
"logps/chosen": -79.68162536621094,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -123.51962280273438,
"loss": 1.132,
"margin_dpo/margin_mean": 29.313201904296875,
"margin_dpo/margin_std": 43.370330810546875,
"step": 112
},
{
"KL/chosen_KL_mean": -37.25122833251953,
"KL/mean": -52.803138732910156,
"KL/rejected_KL_mean": -68.35505676269531,
"KL/std": 37.59297180175781,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.011897064745426178,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 31.103832244873047,
"fcm_dpo/q_t": 0.4143812954425812,
"grad_norm": 19.883005142211914,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.5839675664901733,
"logits/rejected": -0.5408110022544861,
"logps/chosen": -103.96417999267578,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -146.32376098632812,
"loss": 1.108,
"margin_dpo/margin_mean": 31.103832244873047,
"margin_dpo/margin_std": 41.73796844482422,
"step": 113
},
{
"KL/chosen_KL_mean": -34.05321502685547,
"KL/mean": -59.04277420043945,
"KL/rejected_KL_mean": -84.03233337402344,
"KL/std": 56.13905334472656,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.011499330401420593,
"fcm_dpo/delta": -0.18732763826847076,
"fcm_dpo/margin": 49.9791145324707,
"fcm_dpo/q_t": 0.3790869116783142,
"grad_norm": 20.808910369873047,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.6015588045120239,
"logits/rejected": -0.5708324313163757,
"logps/chosen": -91.83829498291016,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -171.14199829101562,
"loss": 1.0125,
"margin_dpo/margin_mean": 49.97911834716797,
"margin_dpo/margin_std": 68.06692504882812,
"step": 114
},
{
"KL/chosen_KL_mean": -44.401611328125,
"KL/mean": -61.324501037597656,
"KL/rejected_KL_mean": -78.24739074707031,
"KL/std": 53.49150085449219,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.011459581553936005,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 33.84577941894531,
"fcm_dpo/q_t": 0.41818803548812866,
"grad_norm": 27.005939483642578,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.641567587852478,
"logits/rejected": -0.624779224395752,
"logps/chosen": -109.9842529296875,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -176.8129119873047,
"loss": 1.1781,
"margin_dpo/margin_mean": 33.84577560424805,
"margin_dpo/margin_std": 70.62551879882812,
"step": 115
},
{
"KL/chosen_KL_mean": -33.02043914794922,
"KL/mean": -52.104915618896484,
"KL/rejected_KL_mean": -71.18939971923828,
"KL/std": 45.22285461425781,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.011387938633561134,
"fcm_dpo/delta": -0.06291196495294571,
"fcm_dpo/margin": 38.16896057128906,
"fcm_dpo/q_t": 0.4015217423439026,
"grad_norm": 20.68140983581543,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.5794106721878052,
"logits/rejected": -0.5696778297424316,
"logps/chosen": -84.42074584960938,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -151.71124267578125,
"loss": 1.0702,
"margin_dpo/margin_mean": 38.16896057128906,
"margin_dpo/margin_std": 49.083984375,
"step": 116
},
{
"KL/chosen_KL_mean": -42.41381072998047,
"KL/mean": -57.908756256103516,
"KL/rejected_KL_mean": -73.40370178222656,
"KL/std": 47.984107971191406,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.011316295713186264,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 30.98989486694336,
"fcm_dpo/q_t": 0.42085617780685425,
"grad_norm": 26.12360954284668,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.5579597353935242,
"logits/rejected": -0.5199344158172607,
"logps/chosen": -111.71221923828125,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -139.98770141601562,
"loss": 1.1531,
"margin_dpo/margin_mean": 30.98989486694336,
"margin_dpo/margin_std": 54.445655822753906,
"step": 117
},
{
"KL/chosen_KL_mean": -33.375709533691406,
"KL/mean": -52.365203857421875,
"KL/rejected_KL_mean": -71.35469818115234,
"KL/std": 43.37816619873047,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.011291439644992352,
"fcm_dpo/delta": -0.0220131054520607,
"fcm_dpo/margin": 37.979000091552734,
"fcm_dpo/q_t": 0.4032064974308014,
"grad_norm": 20.045106887817383,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.6095850467681885,
"logits/rejected": -0.5907869338989258,
"logps/chosen": -89.01669311523438,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -147.0237579345703,
"loss": 1.0744,
"margin_dpo/margin_mean": 37.979000091552734,
"margin_dpo/margin_std": 49.81169891357422,
"step": 118
},
{
"KL/chosen_KL_mean": -45.96607208251953,
"KL/mean": -64.73681640625,
"KL/rejected_KL_mean": -83.50755310058594,
"KL/std": 48.845481872558594,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.011094784364104271,
"fcm_dpo/delta": -0.07682996988296509,
"fcm_dpo/margin": 37.541481018066406,
"fcm_dpo/q_t": 0.4086949825286865,
"grad_norm": 22.310869216918945,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.6086139678955078,
"logits/rejected": -0.5971540808677673,
"logps/chosen": -119.47627258300781,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -186.48483276367188,
"loss": 1.1131,
"margin_dpo/margin_mean": 37.541481018066406,
"margin_dpo/margin_std": 59.72161102294922,
"step": 119
},
{
"KL/chosen_KL_mean": -47.09067916870117,
"KL/mean": -73.13763427734375,
"KL/rejected_KL_mean": -99.18460083007812,
"KL/std": 61.26211166381836,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.010806088335812092,
"fcm_dpo/delta": -0.17278769612312317,
"fcm_dpo/margin": 52.09393310546875,
"fcm_dpo/q_t": 0.375888466835022,
"grad_norm": 21.352399826049805,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6157863140106201,
"logits/rejected": -0.5857179164886475,
"logps/chosen": -123.87151336669922,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -207.20834350585938,
"loss": 1.0065,
"margin_dpo/margin_mean": 52.09393310546875,
"margin_dpo/margin_std": 64.51720428466797,
"step": 120
},
{
"KL/chosen_KL_mean": -45.46478271484375,
"KL/mean": -74.26813507080078,
"KL/rejected_KL_mean": -103.07148742675781,
"KL/std": 59.699256896972656,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.010338496416807175,
"fcm_dpo/delta": -0.20949624478816986,
"fcm_dpo/margin": 57.6067008972168,
"fcm_dpo/q_t": 0.3712344169616699,
"grad_norm": 24.244958877563477,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.5617018342018127,
"logits/rejected": -0.5581203699111938,
"logps/chosen": -107.25468444824219,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -213.06605529785156,
"loss": 1.0057,
"margin_dpo/margin_mean": 57.60670471191406,
"margin_dpo/margin_std": 74.79032135009766,
"step": 121
},
{
"KL/chosen_KL_mean": -41.58914566040039,
"KL/mean": -76.96797943115234,
"KL/rejected_KL_mean": -112.3468017578125,
"KL/std": 67.52272033691406,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.009813392534852028,
"fcm_dpo/delta": -0.31619852781295776,
"fcm_dpo/margin": 70.75765991210938,
"fcm_dpo/q_t": 0.3468356132507324,
"grad_norm": 22.89113998413086,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.5372201204299927,
"logits/rejected": -0.5534934997558594,
"logps/chosen": -88.4913558959961,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -219.06100463867188,
"loss": 0.9104,
"margin_dpo/margin_mean": 70.75765991210938,
"margin_dpo/margin_std": 70.87075805664062,
"step": 122
},
{
"KL/chosen_KL_mean": -45.56751251220703,
"KL/mean": -68.76374053955078,
"KL/rejected_KL_mean": -91.95997619628906,
"KL/std": 57.52820587158203,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.009498516097664833,
"fcm_dpo/delta": -0.07821456342935562,
"fcm_dpo/margin": 46.39246368408203,
"fcm_dpo/q_t": 0.40447816252708435,
"grad_norm": 20.638565063476562,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.5457350015640259,
"logits/rejected": -0.5298566222190857,
"logps/chosen": -106.9061508178711,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -179.73536682128906,
"loss": 1.1155,
"margin_dpo/margin_mean": 46.392459869384766,
"margin_dpo/margin_std": 77.1932144165039,
"step": 123
},
{
"KL/chosen_KL_mean": -52.27129364013672,
"KL/mean": -86.85035705566406,
"KL/rejected_KL_mean": -121.42942810058594,
"KL/std": 79.01206970214844,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.009129964746534824,
"fcm_dpo/delta": -0.2478792667388916,
"fcm_dpo/margin": 69.15812683105469,
"fcm_dpo/q_t": 0.371742308139801,
"grad_norm": 24.180330276489258,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.5349669456481934,
"logits/rejected": -0.5380312204360962,
"logps/chosen": -123.7196273803711,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -239.00999450683594,
"loss": 1.0206,
"margin_dpo/margin_mean": 69.15812683105469,
"margin_dpo/margin_std": 99.04975891113281,
"step": 124
},
{
"KL/chosen_KL_mean": -43.38472366333008,
"KL/mean": -73.71883392333984,
"KL/rejected_KL_mean": -104.05294799804688,
"KL/std": 66.72522735595703,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.008861461654305458,
"fcm_dpo/delta": -0.14523999392986298,
"fcm_dpo/margin": 60.668209075927734,
"fcm_dpo/q_t": 0.3841526210308075,
"grad_norm": 18.966266632080078,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.4806230664253235,
"logits/rejected": -0.4828706979751587,
"logps/chosen": -93.52166748046875,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -188.04156494140625,
"loss": 1.0392,
"margin_dpo/margin_mean": 60.668209075927734,
"margin_dpo/margin_std": 83.17915344238281,
"step": 125
},
{
"KL/chosen_KL_mean": -46.75824737548828,
"KL/mean": -75.15432739257812,
"KL/rejected_KL_mean": -103.55040740966797,
"KL/std": 60.45394515991211,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.008667214773595333,
"fcm_dpo/delta": -0.06482094526290894,
"fcm_dpo/margin": 56.79215621948242,
"fcm_dpo/q_t": 0.3921211361885071,
"grad_norm": 20.3935546875,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.5126634836196899,
"logits/rejected": -0.5087581872940063,
"logps/chosen": -102.42532348632812,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -201.68017578125,
"loss": 1.0527,
"margin_dpo/margin_mean": 56.792152404785156,
"margin_dpo/margin_std": 77.17770385742188,
"step": 126
},
{
"KL/chosen_KL_mean": -45.504390716552734,
"KL/mean": -66.04163360595703,
"KL/rejected_KL_mean": -86.57887268066406,
"KL/std": 52.939697265625,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.008667214773595333,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.07448196411133,
"fcm_dpo/q_t": 0.417339950799942,
"grad_norm": 20.634532928466797,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.5034447908401489,
"logits/rejected": -0.48421603441238403,
"logps/chosen": -102.05906677246094,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -163.37464904785156,
"loss": 1.1409,
"margin_dpo/margin_mean": 41.07448196411133,
"margin_dpo/margin_std": 66.85935974121094,
"step": 127
},
{
"KL/chosen_KL_mean": -48.32811737060547,
"KL/mean": -70.08799743652344,
"KL/rejected_KL_mean": -91.84788513183594,
"KL/std": 61.827518463134766,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.008642185479402542,
"fcm_dpo/delta": -0.028961315751075745,
"fcm_dpo/margin": 43.519771575927734,
"fcm_dpo/q_t": 0.41590309143066406,
"grad_norm": 26.325315475463867,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.5076520442962646,
"logits/rejected": -0.4892594516277313,
"logps/chosen": -106.4490737915039,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -168.28684997558594,
"loss": 1.1502,
"margin_dpo/margin_mean": 43.519771575927734,
"margin_dpo/margin_std": 77.54118347167969,
"step": 128
},
{
"KL/chosen_KL_mean": -57.91839599609375,
"KL/mean": -80.90028381347656,
"KL/rejected_KL_mean": -103.88216400146484,
"KL/std": 68.20243835449219,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.008617157116532326,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 45.963768005371094,
"fcm_dpo/q_t": 0.41664600372314453,
"grad_norm": 19.90182113647461,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.5122944116592407,
"logits/rejected": -0.5017467737197876,
"logps/chosen": -124.83477020263672,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -200.52438354492188,
"loss": 1.1552,
"margin_dpo/margin_mean": 45.963768005371094,
"margin_dpo/margin_std": 88.79536437988281,
"step": 129
},
{
"KL/chosen_KL_mean": -42.74785614013672,
"KL/mean": -77.1688232421875,
"KL/rejected_KL_mean": -111.58978271484375,
"KL/std": 69.51666259765625,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.008449437096714973,
"fcm_dpo/delta": -0.19852416217327118,
"fcm_dpo/margin": 68.84193420410156,
"fcm_dpo/q_t": 0.3743343651294708,
"grad_norm": 19.98097801208496,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.4600975513458252,
"logits/rejected": -0.45375195145606995,
"logps/chosen": -87.41470336914062,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -194.37142944335938,
"loss": 1.0076,
"margin_dpo/margin_mean": 68.84193420410156,
"margin_dpo/margin_std": 84.98545837402344,
"step": 130
},
{
"KL/chosen_KL_mean": -38.25885772705078,
"KL/mean": -72.83998107910156,
"KL/rejected_KL_mean": -107.42111206054688,
"KL/std": 69.28550720214844,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.008066684007644653,
"fcm_dpo/delta": -0.16752536594867706,
"fcm_dpo/margin": 69.16224670410156,
"fcm_dpo/q_t": 0.3724893033504486,
"grad_norm": 28.86225700378418,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.48446083068847656,
"logits/rejected": -0.4910423159599304,
"logps/chosen": -83.18344116210938,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -195.8651123046875,
"loss": 0.9888,
"margin_dpo/margin_mean": 69.16224670410156,
"margin_dpo/margin_std": 75.45245361328125,
"step": 131
},
{
"KL/chosen_KL_mean": -50.65435791015625,
"KL/mean": -78.87272644042969,
"KL/rejected_KL_mean": -107.09109497070312,
"KL/std": 68.87550354003906,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.00793751422315836,
"fcm_dpo/delta": -0.044726189225912094,
"fcm_dpo/margin": 56.436744689941406,
"fcm_dpo/q_t": 0.404166042804718,
"grad_norm": 19.247264862060547,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.5044941902160645,
"logits/rejected": -0.5029531717300415,
"logps/chosen": -109.65544128417969,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -194.98324584960938,
"loss": 1.0959,
"margin_dpo/margin_mean": 56.436744689941406,
"margin_dpo/margin_std": 89.56974792480469,
"step": 132
},
{
"KL/chosen_KL_mean": -59.93231964111328,
"KL/mean": -84.84762573242188,
"KL/rejected_KL_mean": -109.76292419433594,
"KL/std": 62.10484313964844,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.00793751422315836,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 49.83061981201172,
"fcm_dpo/q_t": 0.41286903619766235,
"grad_norm": 26.079952239990234,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.5067698955535889,
"logits/rejected": -0.4933437705039978,
"logps/chosen": -126.53681945800781,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -206.09649658203125,
"loss": 1.1177,
"margin_dpo/margin_mean": 49.830623626708984,
"margin_dpo/margin_std": 79.16134643554688,
"step": 133
},
{
"KL/chosen_KL_mean": -47.43791198730469,
"KL/mean": -74.1932601928711,
"KL/rejected_KL_mean": -100.9486083984375,
"KL/std": 60.470733642578125,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.007902203127741814,
"fcm_dpo/delta": -0.04468559846282005,
"fcm_dpo/margin": 53.51069641113281,
"fcm_dpo/q_t": 0.4046560823917389,
"grad_norm": 18.731077194213867,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.4694488048553467,
"logits/rejected": -0.4564399719238281,
"logps/chosen": -99.50717163085938,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -188.60311889648438,
"loss": 1.0759,
"margin_dpo/margin_mean": 53.51069641113281,
"margin_dpo/margin_std": 70.26188659667969,
"step": 134
},
{
"KL/chosen_KL_mean": -52.651573181152344,
"KL/mean": -91.39772033691406,
"KL/rejected_KL_mean": -130.1438446044922,
"KL/std": 81.02012634277344,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.007630997337400913,
"fcm_dpo/delta": -0.2034302055835724,
"fcm_dpo/margin": 77.4922866821289,
"fcm_dpo/q_t": 0.37322452664375305,
"grad_norm": 21.743507385253906,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.4417390823364258,
"logits/rejected": -0.47695955634117126,
"logps/chosen": -103.00543212890625,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -246.12359619140625,
"loss": 1.0012,
"margin_dpo/margin_mean": 77.4922866821289,
"margin_dpo/margin_std": 98.09565734863281,
"step": 135
},
{
"KL/chosen_KL_mean": -62.86431884765625,
"KL/mean": -88.71379089355469,
"KL/rejected_KL_mean": -114.56326293945312,
"KL/std": 73.90873718261719,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.007553243078291416,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 51.698944091796875,
"fcm_dpo/q_t": 0.4202331304550171,
"grad_norm": 20.616310119628906,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.4841381907463074,
"logits/rejected": -0.47736936807632446,
"logps/chosen": -127.93682861328125,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -210.88449096679688,
"loss": 1.1472,
"margin_dpo/margin_mean": 51.698944091796875,
"margin_dpo/margin_std": 95.19708251953125,
"step": 136
},
{
"KL/chosen_KL_mean": -60.16282272338867,
"KL/mean": -100.787841796875,
"KL/rejected_KL_mean": -141.41287231445312,
"KL/std": 97.6104736328125,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.00731696980074048,
"fcm_dpo/delta": -0.20704714953899384,
"fcm_dpo/margin": 81.25003051757812,
"fcm_dpo/q_t": 0.3785414397716522,
"grad_norm": 19.7189884185791,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.4397382140159607,
"logits/rejected": -0.4667167663574219,
"logps/chosen": -108.92193603515625,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -255.27664184570312,
"loss": 1.0296,
"margin_dpo/margin_mean": 81.25003051757812,
"margin_dpo/margin_std": 117.20115661621094,
"step": 137
},
{
"KL/chosen_KL_mean": -62.80693435668945,
"KL/mean": -93.51780700683594,
"KL/rejected_KL_mean": -124.22868347167969,
"KL/std": 74.23326110839844,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.007216614671051502,
"fcm_dpo/delta": -0.041904207319021225,
"fcm_dpo/margin": 61.4217414855957,
"fcm_dpo/q_t": 0.39843374490737915,
"grad_norm": 20.12140655517578,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.43256235122680664,
"logits/rejected": -0.4217318296432495,
"logps/chosen": -123.32658386230469,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -217.42562866210938,
"loss": 1.0607,
"margin_dpo/margin_mean": 61.42173767089844,
"margin_dpo/margin_std": 75.93879699707031,
"step": 138
},
{
"KL/chosen_KL_mean": -52.439796447753906,
"KL/mean": -88.96459197998047,
"KL/rejected_KL_mean": -125.48937225341797,
"KL/std": 70.41232299804688,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.00707232067361474,
"fcm_dpo/delta": -0.12281207740306854,
"fcm_dpo/margin": 73.04957580566406,
"fcm_dpo/q_t": 0.38536033034324646,
"grad_norm": 18.63878059387207,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.44589370489120483,
"logits/rejected": -0.43810388445854187,
"logps/chosen": -99.3311767578125,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -205.21734619140625,
"loss": 1.0259,
"margin_dpo/margin_mean": 73.04957580566406,
"margin_dpo/margin_std": 91.57994079589844,
"step": 139
},
{
"KL/chosen_KL_mean": -63.55427551269531,
"KL/mean": -96.54176330566406,
"KL/rejected_KL_mean": -129.5292510986328,
"KL/std": 77.9601058959961,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.006944697350263596,
"fcm_dpo/delta": -0.06101107597351074,
"fcm_dpo/margin": 65.9749755859375,
"fcm_dpo/q_t": 0.39790695905685425,
"grad_norm": 20.669536590576172,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.41300731897354126,
"logits/rejected": -0.3993472456932068,
"logps/chosen": -122.52899169921875,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -212.8133544921875,
"loss": 1.0752,
"margin_dpo/margin_mean": 65.97498321533203,
"margin_dpo/margin_std": 93.88746643066406,
"step": 140
},
{
"KL/chosen_KL_mean": -71.32264709472656,
"KL/mean": -102.6048583984375,
"KL/rejected_KL_mean": -133.88706970214844,
"KL/std": 86.29281616210938,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.006908806972205639,
"fcm_dpo/delta": -0.026286346837878227,
"fcm_dpo/margin": 62.564414978027344,
"fcm_dpo/q_t": 0.40315961837768555,
"grad_norm": 24.427490234375,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.42676639556884766,
"logits/rejected": -0.41550058126449585,
"logps/chosen": -146.3983154296875,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -232.07933044433594,
"loss": 1.1076,
"margin_dpo/margin_mean": 62.56441879272461,
"margin_dpo/margin_std": 99.85594177246094,
"step": 141
},
{
"KL/chosen_KL_mean": -72.5107421875,
"KL/mean": -108.30811309814453,
"KL/rejected_KL_mean": -144.10546875,
"KL/std": 93.80851745605469,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.0068291304633021355,
"fcm_dpo/delta": -0.09008042514324188,
"fcm_dpo/margin": 71.59473419189453,
"fcm_dpo/q_t": 0.3943261504173279,
"grad_norm": 26.210159301757812,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.3901626169681549,
"logits/rejected": -0.3892650604248047,
"logps/chosen": -130.53866577148438,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -238.6876983642578,
"loss": 1.0881,
"margin_dpo/margin_mean": 71.5947265625,
"margin_dpo/margin_std": 110.5821533203125,
"step": 142
},
{
"KL/chosen_KL_mean": -76.32169342041016,
"KL/mean": -99.2608871459961,
"KL/rejected_KL_mean": -122.2000732421875,
"KL/std": 82.2864990234375,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.006767615210264921,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 45.878387451171875,
"fcm_dpo/q_t": 0.4329405725002289,
"grad_norm": 22.109413146972656,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.37516605854034424,
"logits/rejected": -0.3687092661857605,
"logps/chosen": -133.91815185546875,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -201.19964599609375,
"loss": 1.1982,
"margin_dpo/margin_mean": 45.878387451171875,
"margin_dpo/margin_std": 96.17039489746094,
"step": 143
},
{
"KL/chosen_KL_mean": -68.8632583618164,
"KL/mean": -96.65010070800781,
"KL/rejected_KL_mean": -124.43695068359375,
"KL/std": 69.90775299072266,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.006767615210264921,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 55.573699951171875,
"fcm_dpo/q_t": 0.41452693939208984,
"grad_norm": 20.535696029663086,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.3825682997703552,
"logits/rejected": -0.37160438299179077,
"logps/chosen": -128.76962280273438,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -206.4372100830078,
"loss": 1.118,
"margin_dpo/margin_mean": 55.57369613647461,
"margin_dpo/margin_std": 82.33287048339844,
"step": 144
},
{
"KL/chosen_KL_mean": -65.35738372802734,
"KL/mean": -95.54832458496094,
"KL/rejected_KL_mean": -125.73927307128906,
"KL/std": 69.90843963623047,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.006767615210264921,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 60.381893157958984,
"fcm_dpo/q_t": 0.4071277678012848,
"grad_norm": 23.468578338623047,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.426013708114624,
"logits/rejected": -0.4119391441345215,
"logps/chosen": -121.95805358886719,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -203.6055908203125,
"loss": 1.0948,
"margin_dpo/margin_mean": 60.38189697265625,
"margin_dpo/margin_std": 84.77351379394531,
"step": 145
},
{
"KL/chosen_KL_mean": -89.39501953125,
"KL/mean": -112.8216323852539,
"KL/rejected_KL_mean": -136.2482452392578,
"KL/std": 76.4080810546875,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.006767615210264921,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 46.85322570800781,
"fcm_dpo/q_t": 0.42726507782936096,
"grad_norm": 25.80341339111328,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.39261966943740845,
"logits/rejected": -0.36783578991889954,
"logps/chosen": -155.39547729492188,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -217.95103454589844,
"loss": 1.1871,
"margin_dpo/margin_mean": 46.85322570800781,
"margin_dpo/margin_std": 91.74784851074219,
"step": 146
},
{
"KL/chosen_KL_mean": -65.36270141601562,
"KL/mean": -96.96876525878906,
"KL/rejected_KL_mean": -128.5748291015625,
"KL/std": 76.57128143310547,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.006728795822709799,
"fcm_dpo/delta": -0.028762944042682648,
"fcm_dpo/margin": 63.212127685546875,
"fcm_dpo/q_t": 0.40543532371520996,
"grad_norm": 19.389053344726562,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.4179861545562744,
"logits/rejected": -0.3990488648414612,
"logps/chosen": -118.7681884765625,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -199.9654541015625,
"loss": 1.0988,
"margin_dpo/margin_mean": 63.212127685546875,
"margin_dpo/margin_std": 95.10125732421875,
"step": 147
},
{
"KL/chosen_KL_mean": -63.96573257446289,
"KL/mean": -91.10762786865234,
"KL/rejected_KL_mean": -118.24954223632812,
"KL/std": 76.30313873291016,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.006679433397948742,
"fcm_dpo/delta": -0.07390298694372177,
"fcm_dpo/margin": 54.2838020324707,
"fcm_dpo/q_t": 0.4192584753036499,
"grad_norm": 18.27964210510254,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.47013741731643677,
"logits/rejected": -0.463634192943573,
"logps/chosen": -128.90281677246094,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -221.3433837890625,
"loss": 1.1319,
"margin_dpo/margin_mean": 54.2838020324707,
"margin_dpo/margin_std": 80.29562377929688,
"step": 148
},
{
"KL/chosen_KL_mean": -61.568382263183594,
"KL/mean": -95.15155029296875,
"KL/rejected_KL_mean": -128.73471069335938,
"KL/std": 68.74234771728516,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.006500914692878723,
"fcm_dpo/delta": -0.0983634814620018,
"fcm_dpo/margin": 67.16634368896484,
"fcm_dpo/q_t": 0.3992913067340851,
"grad_norm": 17.962369918823242,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.4860546588897705,
"logits/rejected": -0.4728388786315918,
"logps/chosen": -120.04214477539062,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -228.04946899414062,
"loss": 1.0664,
"margin_dpo/margin_mean": 67.16634368896484,
"margin_dpo/margin_std": 81.42725372314453,
"step": 149
},
{
"KL/chosen_KL_mean": -54.187557220458984,
"KL/mean": -87.24864196777344,
"KL/rejected_KL_mean": -120.30972290039062,
"KL/std": 80.83369445800781,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.006397702731192112,
"fcm_dpo/delta": -0.08001944422721863,
"fcm_dpo/margin": 66.1221694946289,
"fcm_dpo/q_t": 0.407664030790329,
"grad_norm": 17.84654998779297,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.42700618505477905,
"logits/rejected": -0.4307962954044342,
"logps/chosen": -99.89337158203125,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -203.65731811523438,
"loss": 1.0908,
"margin_dpo/margin_mean": 66.1221694946289,
"margin_dpo/margin_std": 96.51167297363281,
"step": 150
},
{
"KL/chosen_KL_mean": -67.81385803222656,
"KL/mean": -101.16264343261719,
"KL/rejected_KL_mean": -134.51144409179688,
"KL/std": 76.48846435546875,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.006397702731192112,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 66.69757080078125,
"fcm_dpo/q_t": 0.4020676910877228,
"grad_norm": 18.534128189086914,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.5162950754165649,
"logits/rejected": -0.4906059503555298,
"logps/chosen": -138.3846893310547,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -234.97525024414062,
"loss": 1.0707,
"margin_dpo/margin_mean": 66.69757080078125,
"margin_dpo/margin_std": 83.75027465820312,
"step": 151
},
{
"KL/chosen_KL_mean": -59.39875030517578,
"KL/mean": -99.93858337402344,
"KL/rejected_KL_mean": -140.47842407226562,
"KL/std": 78.93248748779297,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.006302128545939922,
"fcm_dpo/delta": -0.116790771484375,
"fcm_dpo/margin": 81.07966613769531,
"fcm_dpo/q_t": 0.38571709394454956,
"grad_norm": 20.45277214050293,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.48105257749557495,
"logits/rejected": -0.48062583804130554,
"logps/chosen": -119.56314086914062,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -246.61886596679688,
"loss": 1.0258,
"margin_dpo/margin_mean": 81.07966613769531,
"margin_dpo/margin_std": 96.61679077148438,
"step": 152
},
{
"KL/chosen_KL_mean": -60.735313415527344,
"KL/mean": -95.41152954101562,
"KL/rejected_KL_mean": -130.08775329589844,
"KL/std": 86.6027603149414,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.006227484904229641,
"fcm_dpo/delta": -0.036148011684417725,
"fcm_dpo/margin": 69.35243225097656,
"fcm_dpo/q_t": 0.4058881402015686,
"grad_norm": 15.226004600524902,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.450962096452713,
"logits/rejected": -0.44640836119651794,
"logps/chosen": -117.05059051513672,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -215.74359130859375,
"loss": 1.0967,
"margin_dpo/margin_mean": 69.35243225097656,
"margin_dpo/margin_std": 106.32671356201172,
"step": 153
},
{
"KL/chosen_KL_mean": -72.78416442871094,
"KL/mean": -105.78938293457031,
"KL/rejected_KL_mean": -138.7946014404297,
"KL/std": 86.74234008789062,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.006178042385727167,
"fcm_dpo/delta": -0.04359261319041252,
"fcm_dpo/margin": 66.01043701171875,
"fcm_dpo/q_t": 0.4098336100578308,
"grad_norm": 18.44878578186035,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.4380859136581421,
"logits/rejected": -0.44127899408340454,
"logps/chosen": -135.5267333984375,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -243.03880310058594,
"loss": 1.129,
"margin_dpo/margin_mean": 66.01044464111328,
"margin_dpo/margin_std": 111.0909423828125,
"step": 154
},
{
"KL/chosen_KL_mean": -66.35116577148438,
"KL/mean": -99.84371948242188,
"KL/rejected_KL_mean": -133.33627319335938,
"KL/std": 80.27971649169922,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.006131485104560852,
"fcm_dpo/delta": -0.03200829401612282,
"fcm_dpo/margin": 66.985107421875,
"fcm_dpo/q_t": 0.4072626829147339,
"grad_norm": 18.364580154418945,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.4624621868133545,
"logits/rejected": -0.44429630041122437,
"logps/chosen": -127.00434875488281,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -210.82847595214844,
"loss": 1.1023,
"margin_dpo/margin_mean": 66.985107421875,
"margin_dpo/margin_std": 99.73149108886719,
"step": 155
},
{
"KL/chosen_KL_mean": -93.23342895507812,
"KL/mean": -112.24707794189453,
"KL/rejected_KL_mean": -131.26072692871094,
"KL/std": 85.90792846679688,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.00611185934394598,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 38.027313232421875,
"fcm_dpo/q_t": 0.4467718005180359,
"grad_norm": 27.262943267822266,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.4021250605583191,
"logits/rejected": -0.3974895477294922,
"logps/chosen": -162.72531127929688,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -208.43002319335938,
"loss": 1.2791,
"margin_dpo/margin_mean": 38.02731704711914,
"margin_dpo/margin_std": 112.55517578125,
"step": 156
},
{
"KL/chosen_KL_mean": -83.86450958251953,
"KL/mean": -126.67929077148438,
"KL/rejected_KL_mean": -169.49407958984375,
"KL/std": 93.74600219726562,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.005980256479233503,
"fcm_dpo/delta": -0.10883772373199463,
"fcm_dpo/margin": 85.62956237792969,
"fcm_dpo/q_t": 0.38287657499313354,
"grad_norm": 23.121597290039062,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.37844234704971313,
"logits/rejected": -0.3867532014846802,
"logps/chosen": -145.23294067382812,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -277.14044189453125,
"loss": 1.0374,
"margin_dpo/margin_mean": 85.62956237792969,
"margin_dpo/margin_std": 106.631591796875,
"step": 157
},
{
"KL/chosen_KL_mean": -84.5057144165039,
"KL/mean": -130.03994750976562,
"KL/rejected_KL_mean": -175.57415771484375,
"KL/std": 115.93599700927734,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.005857234820723534,
"fcm_dpo/delta": -0.14090833067893982,
"fcm_dpo/margin": 91.06845092773438,
"fcm_dpo/q_t": 0.38985028862953186,
"grad_norm": 18.224163055419922,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.3638108968734741,
"logits/rejected": -0.38175833225250244,
"logps/chosen": -142.11863708496094,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -289.26885986328125,
"loss": 1.0635,
"margin_dpo/margin_mean": 91.06845092773438,
"margin_dpo/margin_std": 140.60455322265625,
"step": 158
},
{
"KL/chosen_KL_mean": -95.19334411621094,
"KL/mean": -126.32515716552734,
"KL/rejected_KL_mean": -157.45697021484375,
"KL/std": 102.09504699707031,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.005800278857350349,
"fcm_dpo/delta": -0.02378561533987522,
"fcm_dpo/margin": 62.26362991333008,
"fcm_dpo/q_t": 0.41876220703125,
"grad_norm": 20.06854820251465,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.3674774169921875,
"logits/rejected": -0.3357643485069275,
"logps/chosen": -176.75369262695312,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -246.35568237304688,
"loss": 1.1549,
"margin_dpo/margin_mean": 62.263633728027344,
"margin_dpo/margin_std": 110.5584487915039,
"step": 159
},
{
"KL/chosen_KL_mean": -97.48786163330078,
"KL/mean": -139.24900817871094,
"KL/rejected_KL_mean": -181.01016235351562,
"KL/std": 109.07857513427734,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.00568758137524128,
"fcm_dpo/delta": -0.08619771152734756,
"fcm_dpo/margin": 83.52229309082031,
"fcm_dpo/q_t": 0.39961978793144226,
"grad_norm": 22.184825897216797,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.3567469120025635,
"logits/rejected": -0.3495233654975891,
"logps/chosen": -163.21875,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -278.22796630859375,
"loss": 1.0994,
"margin_dpo/margin_mean": 83.52229309082031,
"margin_dpo/margin_std": 136.4792938232422,
"step": 160
},
{
"KL/chosen_KL_mean": -82.90977478027344,
"KL/mean": -119.42204284667969,
"KL/rejected_KL_mean": -155.93431091308594,
"KL/std": 85.58193969726562,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.00568758137524128,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 73.02452087402344,
"fcm_dpo/q_t": 0.40774428844451904,
"grad_norm": 20.76175308227539,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.35023656487464905,
"logits/rejected": -0.350460946559906,
"logps/chosen": -135.34625244140625,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -239.36526489257812,
"loss": 1.1007,
"margin_dpo/margin_mean": 73.02452087402344,
"margin_dpo/margin_std": 108.37924194335938,
"step": 161
},
{
"KL/chosen_KL_mean": -80.26211547851562,
"KL/mean": -116.17857360839844,
"KL/rejected_KL_mean": -152.09503173828125,
"KL/std": 94.54278564453125,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.005591993685811758,
"fcm_dpo/delta": -0.08474621176719666,
"fcm_dpo/margin": 71.83290100097656,
"fcm_dpo/q_t": 0.4103257656097412,
"grad_norm": 21.48539161682129,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.40853458642959595,
"logits/rejected": -0.3933084309101105,
"logps/chosen": -142.87269592285156,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -241.485595703125,
"loss": 1.1179,
"margin_dpo/margin_mean": 71.83290100097656,
"margin_dpo/margin_std": 112.16954040527344,
"step": 162
},
{
"KL/chosen_KL_mean": -90.45162200927734,
"KL/mean": -125.58489990234375,
"KL/rejected_KL_mean": -160.71817016601562,
"KL/std": 95.8338623046875,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.005591993685811758,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 70.26655578613281,
"fcm_dpo/q_t": 0.4135151505470276,
"grad_norm": 19.8275146484375,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.33946388959884644,
"logits/rejected": -0.3220199942588806,
"logps/chosen": -145.47325134277344,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -236.1363983154297,
"loss": 1.126,
"margin_dpo/margin_mean": 70.26655578613281,
"margin_dpo/margin_std": 115.48532104492188,
"step": 163
},
{
"KL/chosen_KL_mean": -82.01603698730469,
"KL/mean": -125.43939208984375,
"KL/rejected_KL_mean": -168.86273193359375,
"KL/std": 93.03357696533203,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.0054749310947954655,
"fcm_dpo/delta": -0.10578101128339767,
"fcm_dpo/margin": 86.84669494628906,
"fcm_dpo/q_t": 0.3918527662754059,
"grad_norm": 19.96858787536621,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.3466936945915222,
"logits/rejected": -0.3345338702201843,
"logps/chosen": -137.65670776367188,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -248.52735900878906,
"loss": 1.0454,
"margin_dpo/margin_mean": 86.84669494628906,
"margin_dpo/margin_std": 106.67892456054688,
"step": 164
},
{
"KL/chosen_KL_mean": -88.45890045166016,
"KL/mean": -116.44608306884766,
"KL/rejected_KL_mean": -144.4332733154297,
"KL/std": 78.84156799316406,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.0054749310947954655,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 55.974361419677734,
"fcm_dpo/q_t": 0.43027693033218384,
"grad_norm": 21.361570358276367,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.3570517301559448,
"logits/rejected": -0.3310251235961914,
"logps/chosen": -149.76959228515625,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -218.1038818359375,
"loss": 1.1816,
"margin_dpo/margin_mean": 55.974365234375,
"margin_dpo/margin_std": 105.02626037597656,
"step": 165
},
{
"KL/chosen_KL_mean": -78.31771850585938,
"KL/mean": -130.20628356933594,
"KL/rejected_KL_mean": -182.0948486328125,
"KL/std": 114.64451599121094,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.005322150886058807,
"fcm_dpo/delta": -0.16226670145988464,
"fcm_dpo/margin": 103.7771224975586,
"fcm_dpo/q_t": 0.3852335214614868,
"grad_norm": 17.009685516357422,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.34249216318130493,
"logits/rejected": -0.35319170355796814,
"logps/chosen": -129.3013153076172,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -280.1899719238281,
"loss": 1.0298,
"margin_dpo/margin_mean": 103.77713012695312,
"margin_dpo/margin_std": 144.28770446777344,
"step": 166
},
{
"KL/chosen_KL_mean": -78.3003158569336,
"KL/mean": -133.010009765625,
"KL/rejected_KL_mean": -187.7197265625,
"KL/std": 104.11771392822266,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.0051628341898322105,
"fcm_dpo/delta": -0.17484354972839355,
"fcm_dpo/margin": 109.41941833496094,
"fcm_dpo/q_t": 0.3730447590351105,
"grad_norm": 19.03751564025879,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.34518417716026306,
"logits/rejected": -0.348605215549469,
"logps/chosen": -128.7244110107422,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -283.7501525878906,
"loss": 0.9857,
"margin_dpo/margin_mean": 109.41941833496094,
"margin_dpo/margin_std": 120.54164123535156,
"step": 167
},
{
"KL/chosen_KL_mean": -85.6524658203125,
"KL/mean": -125.55575561523438,
"KL/rejected_KL_mean": -165.4590606689453,
"KL/std": 97.6776123046875,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.005095277912914753,
"fcm_dpo/delta": -0.0222168006002903,
"fcm_dpo/margin": 79.80659484863281,
"fcm_dpo/q_t": 0.408231645822525,
"grad_norm": 20.391569137573242,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.3299752473831177,
"logits/rejected": -0.32102906703948975,
"logps/chosen": -135.11529541015625,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -240.76760864257812,
"loss": 1.0914,
"margin_dpo/margin_mean": 79.80659484863281,
"margin_dpo/margin_std": 109.659912109375,
"step": 168
},
{
"KL/chosen_KL_mean": -88.48287200927734,
"KL/mean": -123.318603515625,
"KL/rejected_KL_mean": -158.15432739257812,
"KL/std": 96.05093383789062,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.005034225527197123,
"fcm_dpo/delta": -0.06027255952358246,
"fcm_dpo/margin": 69.67146301269531,
"fcm_dpo/q_t": 0.42353829741477966,
"grad_norm": 19.06806755065918,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.34940391778945923,
"logits/rejected": -0.3333583474159241,
"logps/chosen": -148.28631591796875,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -241.5000762939453,
"loss": 1.169,
"margin_dpo/margin_mean": 69.67145538330078,
"margin_dpo/margin_std": 132.4425048828125,
"step": 169
},
{
"KL/chosen_KL_mean": -78.66925048828125,
"KL/mean": -119.93560791015625,
"KL/rejected_KL_mean": -161.20196533203125,
"KL/std": 92.10433959960938,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.004970300942659378,
"fcm_dpo/delta": -0.06389647722244263,
"fcm_dpo/margin": 82.53272247314453,
"fcm_dpo/q_t": 0.4052248001098633,
"grad_norm": 17.099119186401367,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.30527323484420776,
"logits/rejected": -0.29494979977607727,
"logps/chosen": -128.14102172851562,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -237.11929321289062,
"loss": 1.0902,
"margin_dpo/margin_mean": 82.53272247314453,
"margin_dpo/margin_std": 110.43910217285156,
"step": 170
},
{
"KL/chosen_KL_mean": -114.73477172851562,
"KL/mean": -147.73040771484375,
"KL/rejected_KL_mean": -180.72601318359375,
"KL/std": 106.39347839355469,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.004970300942659378,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 65.99124908447266,
"fcm_dpo/q_t": 0.42960548400878906,
"grad_norm": 33.073822021484375,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.3449512720108032,
"logits/rejected": -0.3299598693847656,
"logps/chosen": -199.2340850830078,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -290.10809326171875,
"loss": 1.1864,
"margin_dpo/margin_mean": 65.99124908447266,
"margin_dpo/margin_std": 136.73788452148438,
"step": 171
},
{
"KL/chosen_KL_mean": -99.86283874511719,
"KL/mean": -134.49224853515625,
"KL/rejected_KL_mean": -169.12164306640625,
"KL/std": 102.2164535522461,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.004970300942659378,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.2587890625,
"fcm_dpo/q_t": 0.4216139614582062,
"grad_norm": 18.176748275756836,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.39472442865371704,
"logits/rejected": -0.3820643723011017,
"logps/chosen": -168.51675415039062,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -254.55831909179688,
"loss": 1.1471,
"margin_dpo/margin_mean": 69.2587890625,
"margin_dpo/margin_std": 115.08087158203125,
"step": 172
},
{
"KL/chosen_KL_mean": -90.9837875366211,
"KL/mean": -126.10589599609375,
"KL/rejected_KL_mean": -161.22801208496094,
"KL/std": 96.37551879882812,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.004970300942659378,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 70.24421691894531,
"fcm_dpo/q_t": 0.4198143780231476,
"grad_norm": 19.0076961517334,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.38060057163238525,
"logits/rejected": -0.35156911611557007,
"logps/chosen": -154.03466796875,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -239.91192626953125,
"loss": 1.1262,
"margin_dpo/margin_mean": 70.24421691894531,
"margin_dpo/margin_std": 101.06126403808594,
"step": 173
},
{
"KL/chosen_KL_mean": -87.39854431152344,
"KL/mean": -129.3033447265625,
"KL/rejected_KL_mean": -171.20811462402344,
"KL/std": 101.4627685546875,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.004937914200127125,
"fcm_dpo/delta": -0.06558923423290253,
"fcm_dpo/margin": 83.8095703125,
"fcm_dpo/q_t": 0.4074831008911133,
"grad_norm": 26.417648315429688,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.3339517116546631,
"logits/rejected": -0.35172325372695923,
"logps/chosen": -140.7615203857422,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -273.11932373046875,
"loss": 1.1007,
"margin_dpo/margin_mean": 83.80956268310547,
"margin_dpo/margin_std": 121.98419189453125,
"step": 174
},
{
"KL/chosen_KL_mean": -79.93464660644531,
"KL/mean": -137.0467529296875,
"KL/rejected_KL_mean": -194.15887451171875,
"KL/std": 108.39655303955078,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.004787640646100044,
"fcm_dpo/delta": -0.15558278560638428,
"fcm_dpo/margin": 114.22422790527344,
"fcm_dpo/q_t": 0.37422001361846924,
"grad_norm": 24.844676971435547,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.30949610471725464,
"logits/rejected": -0.2958728075027466,
"logps/chosen": -125.35240936279297,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -283.6646728515625,
"loss": 0.9785,
"margin_dpo/margin_mean": 114.22422790527344,
"margin_dpo/margin_std": 113.23709106445312,
"step": 175
},
{
"KL/chosen_KL_mean": -87.88569641113281,
"KL/mean": -136.27401733398438,
"KL/rejected_KL_mean": -184.66232299804688,
"KL/std": 109.21138763427734,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.004732812754809856,
"fcm_dpo/delta": -0.047373853623867035,
"fcm_dpo/margin": 96.77662658691406,
"fcm_dpo/q_t": 0.3987988531589508,
"grad_norm": 18.611600875854492,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.34232744574546814,
"logits/rejected": -0.35991567373275757,
"logps/chosen": -138.33853149414062,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -280.2212829589844,
"loss": 1.06,
"margin_dpo/margin_mean": 96.77662658691406,
"margin_dpo/margin_std": 126.63328552246094,
"step": 176
},
{
"KL/chosen_KL_mean": -104.66439819335938,
"KL/mean": -152.41554260253906,
"KL/rejected_KL_mean": -200.16668701171875,
"KL/std": 119.41682434082031,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.004684681072831154,
"fcm_dpo/delta": -0.05488370358943939,
"fcm_dpo/margin": 95.50228881835938,
"fcm_dpo/q_t": 0.4001327157020569,
"grad_norm": 25.9334774017334,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.34294700622558594,
"logits/rejected": -0.3354364037513733,
"logps/chosen": -165.880859375,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -296.06048583984375,
"loss": 1.0655,
"margin_dpo/margin_mean": 95.50228881835938,
"margin_dpo/margin_std": 125.90945434570312,
"step": 177
},
{
"KL/chosen_KL_mean": -115.59068298339844,
"KL/mean": -175.99154663085938,
"KL/rejected_KL_mean": -236.3924102783203,
"KL/std": 140.442626953125,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.004535360727459192,
"fcm_dpo/delta": -0.15726302564144135,
"fcm_dpo/margin": 120.80174255371094,
"fcm_dpo/q_t": 0.3810286521911621,
"grad_norm": 25.16571807861328,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.2117362767457962,
"logits/rejected": -0.20060741901397705,
"logps/chosen": -173.85546875,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -341.75775146484375,
"loss": 1.0165,
"margin_dpo/margin_mean": 120.8017349243164,
"margin_dpo/margin_std": 154.52880859375,
"step": 178
},
{
"KL/chosen_KL_mean": -121.65875244140625,
"KL/mean": -163.83865356445312,
"KL/rejected_KL_mean": -206.0185546875,
"KL/std": 121.72943115234375,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.004478609189391136,
"fcm_dpo/delta": -0.040146298706531525,
"fcm_dpo/margin": 84.35980987548828,
"fcm_dpo/q_t": 0.41631701588630676,
"grad_norm": 35.56280517578125,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.2706071734428406,
"logits/rejected": -0.280755877494812,
"logps/chosen": -182.71707153320312,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -296.54638671875,
"loss": 1.1507,
"margin_dpo/margin_mean": 84.35980987548828,
"margin_dpo/margin_std": 150.8819122314453,
"step": 179
},
{
"KL/chosen_KL_mean": -104.479248046875,
"KL/mean": -159.38909912109375,
"KL/rejected_KL_mean": -214.29896545410156,
"KL/std": 107.94509887695312,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.004415547009557486,
"fcm_dpo/delta": -0.0893382877111435,
"fcm_dpo/margin": 109.81971740722656,
"fcm_dpo/q_t": 0.3893117904663086,
"grad_norm": 19.452550888061523,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.28269416093826294,
"logits/rejected": -0.2765018045902252,
"logps/chosen": -158.82196044921875,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -312.51080322265625,
"loss": 1.0378,
"margin_dpo/margin_mean": 109.81971740722656,
"margin_dpo/margin_std": 133.3249053955078,
"step": 180
},
{
"KL/chosen_KL_mean": -90.27413177490234,
"KL/mean": -124.33741760253906,
"KL/rejected_KL_mean": -158.4007110595703,
"KL/std": 99.86650085449219,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.004399297758936882,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 68.1265869140625,
"fcm_dpo/q_t": 0.43104779720306396,
"grad_norm": 18.65866470336914,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.2932761311531067,
"logits/rejected": -0.26991933584213257,
"logps/chosen": -145.27459716796875,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -220.056884765625,
"loss": 1.1809,
"margin_dpo/margin_mean": 68.1265869140625,
"margin_dpo/margin_std": 125.74884033203125,
"step": 181
},
{
"KL/chosen_KL_mean": -84.55804443359375,
"KL/mean": -144.46820068359375,
"KL/rejected_KL_mean": -204.37832641601562,
"KL/std": 119.23335266113281,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.004303830675780773,
"fcm_dpo/delta": -0.12271112948656082,
"fcm_dpo/margin": 119.82029724121094,
"fcm_dpo/q_t": 0.3822292983531952,
"grad_norm": 19.109874725341797,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.2857114374637604,
"logits/rejected": -0.29627925157546997,
"logps/chosen": -125.6658935546875,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -293.89990234375,
"loss": 1.0243,
"margin_dpo/margin_mean": 119.82029724121094,
"margin_dpo/margin_std": 143.70375061035156,
"step": 182
},
{
"KL/chosen_KL_mean": -124.24063110351562,
"KL/mean": -155.76622009277344,
"KL/rejected_KL_mean": -187.29180908203125,
"KL/std": 99.46566772460938,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.004267096519470215,
"fcm_dpo/delta": -0.029845165088772774,
"fcm_dpo/margin": 63.05116653442383,
"fcm_dpo/q_t": 0.4370817542076111,
"grad_norm": 19.169570922851562,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.2434646189212799,
"logits/rejected": -0.2298406958580017,
"logps/chosen": -181.76519775390625,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -263.26751708984375,
"loss": 1.1949,
"margin_dpo/margin_mean": 63.05116271972656,
"margin_dpo/margin_std": 108.97421264648438,
"step": 183
},
{
"KL/chosen_KL_mean": -104.53349304199219,
"KL/mean": -138.28854370117188,
"KL/rejected_KL_mean": -172.0435791015625,
"KL/std": 93.82521057128906,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.004267096519470215,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.51007080078125,
"fcm_dpo/q_t": 0.4340338706970215,
"grad_norm": 17.413782119750977,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.31900352239608765,
"logits/rejected": -0.31066498160362244,
"logps/chosen": -163.07846069335938,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -248.67764282226562,
"loss": 1.1807,
"margin_dpo/margin_mean": 67.51007080078125,
"margin_dpo/margin_std": 118.04556274414062,
"step": 184
},
{
"KL/chosen_KL_mean": -112.33355712890625,
"KL/mean": -138.57923889160156,
"KL/rejected_KL_mean": -164.82492065429688,
"KL/std": 107.27217864990234,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.004267096519470215,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 52.49135208129883,
"fcm_dpo/q_t": 0.45065242052078247,
"grad_norm": 18.927852630615234,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.28930944204330444,
"logits/rejected": -0.2664622366428375,
"logps/chosen": -174.35940551757812,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -238.58750915527344,
"loss": 1.2459,
"margin_dpo/margin_mean": 52.49134826660156,
"margin_dpo/margin_std": 130.1099853515625,
"step": 185
},
{
"KL/chosen_KL_mean": -102.53080749511719,
"KL/mean": -152.23342895507812,
"KL/rejected_KL_mean": -201.93603515625,
"KL/std": 106.55513000488281,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.004267096519470215,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.40524291992188,
"fcm_dpo/q_t": 0.4030308723449707,
"grad_norm": 27.395605087280273,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.3761294484138489,
"logits/rejected": -0.36072081327438354,
"logps/chosen": -171.88427734375,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -290.00848388671875,
"loss": 1.0696,
"margin_dpo/margin_mean": 99.40524291992188,
"margin_dpo/margin_std": 122.63768005371094,
"step": 186
},
{
"KL/chosen_KL_mean": -94.88285827636719,
"KL/mean": -137.2830047607422,
"KL/rejected_KL_mean": -179.6831512451172,
"KL/std": 102.76565551757812,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.004259498789906502,
"fcm_dpo/delta": -0.01783652789890766,
"fcm_dpo/margin": 84.80029296875,
"fcm_dpo/q_t": 0.4160510301589966,
"grad_norm": 23.19233512878418,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.3276948034763336,
"logits/rejected": -0.32193610072135925,
"logps/chosen": -147.63931274414062,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -261.6522521972656,
"loss": 1.1113,
"margin_dpo/margin_mean": 84.80029296875,
"margin_dpo/margin_std": 111.11736297607422,
"step": 187
},
{
"KL/chosen_KL_mean": -88.96598815917969,
"KL/mean": -139.68350219726562,
"KL/rejected_KL_mean": -190.40103149414062,
"KL/std": 113.15321350097656,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.004251901526004076,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.43505096435547,
"fcm_dpo/q_t": 0.40194350481033325,
"grad_norm": 29.918031692504883,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.2987292408943176,
"logits/rejected": -0.3055376410484314,
"logps/chosen": -138.3814697265625,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -279.94146728515625,
"loss": 1.0642,
"margin_dpo/margin_mean": 101.43505096435547,
"margin_dpo/margin_std": 123.50503540039062,
"step": 188
},
{
"KL/chosen_KL_mean": -104.55441284179688,
"KL/mean": -146.60751342773438,
"KL/rejected_KL_mean": -188.66064453125,
"KL/std": 115.1569595336914,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.004251901526004076,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 84.10621643066406,
"fcm_dpo/q_t": 0.4205946922302246,
"grad_norm": 22.207258224487305,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.32266396284103394,
"logits/rejected": -0.30563491582870483,
"logps/chosen": -156.953369140625,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -260.8280029296875,
"loss": 1.14,
"margin_dpo/margin_mean": 84.10621643066406,
"margin_dpo/margin_std": 141.17153930664062,
"step": 189
},
{
"KL/chosen_KL_mean": -111.84954833984375,
"KL/mean": -161.93978881835938,
"KL/rejected_KL_mean": -212.030029296875,
"KL/std": 124.21896362304688,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.004212085623294115,
"fcm_dpo/delta": -0.09453053772449493,
"fcm_dpo/margin": 100.18048858642578,
"fcm_dpo/q_t": 0.40519779920578003,
"grad_norm": 17.787500381469727,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.3172937035560608,
"logits/rejected": -0.30383723974227905,
"logps/chosen": -176.53260803222656,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -314.58056640625,
"loss": 1.0978,
"margin_dpo/margin_mean": 100.18048858642578,
"margin_dpo/margin_std": 143.24652099609375,
"step": 190
},
{
"KL/chosen_KL_mean": -104.78083801269531,
"KL/mean": -175.7665252685547,
"KL/rejected_KL_mean": -246.75222778320312,
"KL/std": 141.02308654785156,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.00404349435120821,
"fcm_dpo/delta": -0.1856611967086792,
"fcm_dpo/margin": 141.97137451171875,
"fcm_dpo/q_t": 0.37194889783859253,
"grad_norm": 18.88121795654297,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.35467493534088135,
"logits/rejected": -0.3345186114311218,
"logps/chosen": -173.43971252441406,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -356.89190673828125,
"loss": 0.979,
"margin_dpo/margin_mean": 141.97137451171875,
"margin_dpo/margin_std": 156.3507537841797,
"step": 191
},
{
"KL/chosen_KL_mean": -134.30404663085938,
"KL/mean": -181.43359375,
"KL/rejected_KL_mean": -228.56317138671875,
"KL/std": 122.91511535644531,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.004020185209810734,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.25910949707031,
"fcm_dpo/q_t": 0.41552644968032837,
"grad_norm": 22.49018669128418,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.28340697288513184,
"logits/rejected": -0.2827571630477905,
"logps/chosen": -204.03097534179688,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -331.884521484375,
"loss": 1.1286,
"margin_dpo/margin_mean": 94.25910949707031,
"margin_dpo/margin_std": 152.60736083984375,
"step": 192
},
{
"KL/chosen_KL_mean": -134.03817749023438,
"KL/mean": -161.839599609375,
"KL/rejected_KL_mean": -189.6409912109375,
"KL/std": 115.02464294433594,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.004020185209810734,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 55.60279083251953,
"fcm_dpo/q_t": 0.4464014768600464,
"grad_norm": 26.761646270751953,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.30809807777404785,
"logits/rejected": -0.3014001250267029,
"logps/chosen": -194.2286834716797,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -266.0485534667969,
"loss": 1.2585,
"margin_dpo/margin_mean": 55.60279083251953,
"margin_dpo/margin_std": 146.70774841308594,
"step": 193
},
{
"KL/chosen_KL_mean": -83.75235748291016,
"KL/mean": -128.62440490722656,
"KL/rejected_KL_mean": -173.4964599609375,
"KL/std": 95.19931030273438,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.004020185209810734,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.74411010742188,
"fcm_dpo/q_t": 0.4147985279560089,
"grad_norm": 15.454571723937988,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.20827534794807434,
"logits/rejected": -0.19151608645915985,
"logps/chosen": -121.59272766113281,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -234.18124389648438,
"loss": 1.1011,
"margin_dpo/margin_mean": 89.74411010742188,
"margin_dpo/margin_std": 105.07684326171875,
"step": 194
},
{
"KL/chosen_KL_mean": -132.5264129638672,
"KL/mean": -180.77197265625,
"KL/rejected_KL_mean": -229.01751708984375,
"KL/std": 118.66985321044922,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.004020185209810734,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 96.49112701416016,
"fcm_dpo/q_t": 0.4101010859012604,
"grad_norm": 24.827335357666016,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.22600211203098297,
"logits/rejected": -0.24551793932914734,
"logps/chosen": -187.41798400878906,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -325.7884826660156,
"loss": 1.0917,
"margin_dpo/margin_mean": 96.49112701416016,
"margin_dpo/margin_std": 122.10552978515625,
"step": 195
},
{
"KL/chosen_KL_mean": -101.63624572753906,
"KL/mean": -156.90122985839844,
"KL/rejected_KL_mean": -212.1662139892578,
"KL/std": 120.26493835449219,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.003964636474847794,
"fcm_dpo/delta": -0.06956931948661804,
"fcm_dpo/margin": 110.52996063232422,
"fcm_dpo/q_t": 0.4009883403778076,
"grad_norm": 16.279319763183594,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.29676854610443115,
"logits/rejected": -0.2831350266933441,
"logps/chosen": -154.88150024414062,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -288.21917724609375,
"loss": 1.0747,
"margin_dpo/margin_mean": 110.52996063232422,
"margin_dpo/margin_std": 143.91986083984375,
"step": 196
},
{
"KL/chosen_KL_mean": -104.97657012939453,
"KL/mean": -147.35794067382812,
"KL/rejected_KL_mean": -189.73931884765625,
"KL/std": 104.29344177246094,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.003964636474847794,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 84.76274871826172,
"fcm_dpo/q_t": 0.4220603108406067,
"grad_norm": 18.666196823120117,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.26582375168800354,
"logits/rejected": -0.26124078035354614,
"logps/chosen": -165.39691162109375,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -266.9482421875,
"loss": 1.13,
"margin_dpo/margin_mean": 84.76274871826172,
"margin_dpo/margin_std": 117.65675354003906,
"step": 197
},
{
"KL/chosen_KL_mean": -118.09703063964844,
"KL/mean": -172.27029418945312,
"KL/rejected_KL_mean": -226.4435577392578,
"KL/std": 130.2771759033203,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.003964636474847794,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.34652709960938,
"fcm_dpo/q_t": 0.4042630195617676,
"grad_norm": 20.83625030517578,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.2953726649284363,
"logits/rejected": -0.29592087864875793,
"logps/chosen": -173.13320922851562,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -323.68682861328125,
"loss": 1.0773,
"margin_dpo/margin_mean": 108.34652709960938,
"margin_dpo/margin_std": 146.12347412109375,
"step": 198
},
{
"KL/chosen_KL_mean": -111.96086120605469,
"KL/mean": -165.84420776367188,
"KL/rejected_KL_mean": -219.72755432128906,
"KL/std": 117.84519958496094,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.00391243165358901,
"fcm_dpo/delta": -0.06627540290355682,
"fcm_dpo/margin": 107.7667007446289,
"fcm_dpo/q_t": 0.4036487638950348,
"grad_norm": 21.15707015991211,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.3135220408439636,
"logits/rejected": -0.2968115210533142,
"logps/chosen": -168.78968811035156,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -304.3757629394531,
"loss": 1.0843,
"margin_dpo/margin_mean": 107.7667007446289,
"margin_dpo/margin_std": 144.26821899414062,
"step": 199
},
{
"KL/chosen_KL_mean": -109.51300048828125,
"KL/mean": -163.57774353027344,
"KL/rejected_KL_mean": -217.64247131347656,
"KL/std": 127.20454406738281,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.0038815850857645273,
"fcm_dpo/delta": -0.0395774245262146,
"fcm_dpo/margin": 108.12948608398438,
"fcm_dpo/q_t": 0.4061018228530884,
"grad_norm": 18.211580276489258,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.2737102806568146,
"logits/rejected": -0.25955069065093994,
"logps/chosen": -162.58006286621094,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -298.25091552734375,
"loss": 1.1009,
"margin_dpo/margin_mean": 108.12948608398438,
"margin_dpo/margin_std": 162.80935668945312,
"step": 200
},
{
"KL/chosen_KL_mean": -115.50804138183594,
"KL/mean": -169.43679809570312,
"KL/rejected_KL_mean": -223.36557006835938,
"KL/std": 135.0660400390625,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.00386931118555367,
"fcm_dpo/delta": -0.03172111138701439,
"fcm_dpo/margin": 107.85752868652344,
"fcm_dpo/q_t": 0.40528595447540283,
"grad_norm": 18.747127532958984,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.3500000834465027,
"logits/rejected": -0.34820225834846497,
"logps/chosen": -190.91026306152344,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -338.17376708984375,
"loss": 1.0919,
"margin_dpo/margin_mean": 107.85752868652344,
"margin_dpo/margin_std": 151.97970581054688,
"step": 201
},
{
"KL/chosen_KL_mean": -120.22370910644531,
"KL/mean": -157.83718872070312,
"KL/rejected_KL_mean": -195.4506378173828,
"KL/std": 117.38187408447266,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.0038570372853428125,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 75.22691345214844,
"fcm_dpo/q_t": 0.43460530042648315,
"grad_norm": 19.376781463623047,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.28714412450790405,
"logits/rejected": -0.3028670847415924,
"logps/chosen": -170.3250274658203,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -282.4356689453125,
"loss": 1.1981,
"margin_dpo/margin_mean": 75.22691345214844,
"margin_dpo/margin_std": 152.60952758789062,
"step": 202
},
{
"KL/chosen_KL_mean": -118.58921813964844,
"KL/mean": -165.52542114257812,
"KL/rejected_KL_mean": -212.46160888671875,
"KL/std": 120.00934600830078,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.0038570372853428125,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 93.87237548828125,
"fcm_dpo/q_t": 0.4165031909942627,
"grad_norm": 20.891557693481445,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.3353193402290344,
"logits/rejected": -0.3288806676864624,
"logps/chosen": -179.19891357421875,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -298.3575744628906,
"loss": 1.117,
"margin_dpo/margin_mean": 93.87239074707031,
"margin_dpo/margin_std": 132.0635528564453,
"step": 203
},
{
"KL/chosen_KL_mean": -132.33493041992188,
"KL/mean": -175.73989868164062,
"KL/rejected_KL_mean": -219.1448974609375,
"KL/std": 130.3503875732422,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.0038570372853428125,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 86.80996704101562,
"fcm_dpo/q_t": 0.4250984191894531,
"grad_norm": 22.802963256835938,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.4051492214202881,
"logits/rejected": -0.37047427892684937,
"logps/chosen": -212.4998779296875,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -306.8408203125,
"loss": 1.1606,
"margin_dpo/margin_mean": 86.80996704101562,
"margin_dpo/margin_std": 152.51388549804688,
"step": 204
},
{
"KL/chosen_KL_mean": -129.84996032714844,
"KL/mean": -187.99693298339844,
"KL/rejected_KL_mean": -246.14390563964844,
"KL/std": 135.32766723632812,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.00378048256970942,
"fcm_dpo/delta": -0.10023847967386246,
"fcm_dpo/margin": 116.29393005371094,
"fcm_dpo/q_t": 0.39798709750175476,
"grad_norm": 19.018651962280273,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.2791271209716797,
"logits/rejected": -0.24926945567131042,
"logps/chosen": -189.2346954345703,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -331.2689514160156,
"loss": 1.0719,
"margin_dpo/margin_mean": 116.29393005371094,
"margin_dpo/margin_std": 147.2044219970703,
"step": 205
},
{
"KL/chosen_KL_mean": -118.81938171386719,
"KL/mean": -181.1719512939453,
"KL/rejected_KL_mean": -243.52452087402344,
"KL/std": 125.76171875,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.0037328554317355156,
"fcm_dpo/delta": -0.06339111924171448,
"fcm_dpo/margin": 124.70513153076172,
"fcm_dpo/q_t": 0.39114850759506226,
"grad_norm": 23.02773666381836,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.23025387525558472,
"logits/rejected": -0.233394593000412,
"logps/chosen": -165.78387451171875,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -342.47802734375,
"loss": 1.0355,
"margin_dpo/margin_mean": 124.70513153076172,
"margin_dpo/margin_std": 136.64340209960938,
"step": 206
},
{
"KL/chosen_KL_mean": -110.89537048339844,
"KL/mean": -183.31405639648438,
"KL/rejected_KL_mean": -255.7327117919922,
"KL/std": 147.05844116210938,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.003661469090729952,
"fcm_dpo/delta": -0.13748954236507416,
"fcm_dpo/margin": 144.83734130859375,
"fcm_dpo/q_t": 0.3823302388191223,
"grad_norm": 22.23126983642578,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.33647215366363525,
"logits/rejected": -0.3128640651702881,
"logps/chosen": -166.95162963867188,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -340.1805114746094,
"loss": 1.0102,
"margin_dpo/margin_mean": 144.83734130859375,
"margin_dpo/margin_std": 171.68792724609375,
"step": 207
},
{
"KL/chosen_KL_mean": -171.9359130859375,
"KL/mean": -227.58004760742188,
"KL/rejected_KL_mean": -283.22418212890625,
"KL/std": 141.04026794433594,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.0036316080950200558,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 111.28825378417969,
"fcm_dpo/q_t": 0.4080774188041687,
"grad_norm": 24.357837677001953,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.314586877822876,
"logits/rejected": -0.2982891798019409,
"logps/chosen": -239.00352478027344,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -377.51104736328125,
"loss": 1.1139,
"margin_dpo/margin_mean": 111.28825378417969,
"margin_dpo/margin_std": 172.75132751464844,
"step": 208
},
{
"KL/chosen_KL_mean": -144.75634765625,
"KL/mean": -193.95156860351562,
"KL/rejected_KL_mean": -243.14678955078125,
"KL/std": 125.75456237792969,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.0036316080950200558,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.39041137695312,
"fcm_dpo/q_t": 0.4181509017944336,
"grad_norm": 24.6387996673584,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.21858316659927368,
"logits/rejected": -0.20688988268375397,
"logps/chosen": -200.93804931640625,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -324.08831787109375,
"loss": 1.1421,
"margin_dpo/margin_mean": 98.39041900634766,
"margin_dpo/margin_std": 159.32110595703125,
"step": 209
},
{
"KL/chosen_KL_mean": -128.8101806640625,
"KL/mean": -186.57366943359375,
"KL/rejected_KL_mean": -244.337158203125,
"KL/std": 130.371337890625,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.003612897824496031,
"fcm_dpo/delta": -0.05178801342844963,
"fcm_dpo/margin": 115.52698516845703,
"fcm_dpo/q_t": 0.40541771054267883,
"grad_norm": 23.523893356323242,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.2616923749446869,
"logits/rejected": -0.2551494240760803,
"logps/chosen": -175.18199157714844,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -321.018798828125,
"loss": 1.089,
"margin_dpo/margin_mean": 115.52698516845703,
"margin_dpo/margin_std": 158.4886474609375,
"step": 210
},
{
"KL/chosen_KL_mean": -182.32998657226562,
"KL/mean": -231.79617309570312,
"KL/rejected_KL_mean": -281.26239013671875,
"KL/std": 148.31312561035156,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.003594187553972006,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.93236541748047,
"fcm_dpo/q_t": 0.422549843788147,
"grad_norm": 32.76312255859375,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.2987380921840668,
"logits/rejected": -0.261538565158844,
"logps/chosen": -261.2623291015625,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -368.0833740234375,
"loss": 1.1547,
"margin_dpo/margin_mean": 98.93236541748047,
"margin_dpo/margin_std": 174.6763916015625,
"step": 211
},
{
"KL/chosen_KL_mean": -151.54200744628906,
"KL/mean": -221.18905639648438,
"KL/rejected_KL_mean": -290.8360900878906,
"KL/std": 158.43215942382812,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.0035146158188581467,
"fcm_dpo/delta": -0.11193866282701492,
"fcm_dpo/margin": 139.29408264160156,
"fcm_dpo/q_t": 0.3895455300807953,
"grad_norm": 22.986331939697266,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.29403156042099,
"logits/rejected": -0.2868325114250183,
"logps/chosen": -209.739013671875,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -393.8939208984375,
"loss": 1.0528,
"margin_dpo/margin_mean": 139.29409790039062,
"margin_dpo/margin_std": 178.13589477539062,
"step": 212
},
{
"KL/chosen_KL_mean": -139.25265502929688,
"KL/mean": -205.28900146484375,
"KL/rejected_KL_mean": -271.3253479003906,
"KL/std": 135.00584411621094,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.0034705563448369503,
"fcm_dpo/delta": -0.0630766823887825,
"fcm_dpo/margin": 132.0726776123047,
"fcm_dpo/q_t": 0.3947675824165344,
"grad_norm": 29.329484939575195,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.3089921474456787,
"logits/rejected": -0.28634005784988403,
"logps/chosen": -206.765380859375,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -365.24005126953125,
"loss": 1.0522,
"margin_dpo/margin_mean": 132.0726776123047,
"margin_dpo/margin_std": 160.37890625,
"step": 213
},
{
"KL/chosen_KL_mean": -118.66336059570312,
"KL/mean": -177.6033935546875,
"KL/rejected_KL_mean": -236.54342651367188,
"KL/std": 133.26171875,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.0034705563448369503,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.88006591796875,
"fcm_dpo/q_t": 0.4071076512336731,
"grad_norm": 22.763870239257812,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.2438046634197235,
"logits/rejected": -0.24939894676208496,
"logps/chosen": -160.26824951171875,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -314.06085205078125,
"loss": 1.0871,
"margin_dpo/margin_mean": 117.88006591796875,
"margin_dpo/margin_std": 156.95803833007812,
"step": 214
},
{
"KL/chosen_KL_mean": -137.9038543701172,
"KL/mean": -200.85073852539062,
"KL/rejected_KL_mean": -263.797607421875,
"KL/std": 138.6534423828125,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.003440548898652196,
"fcm_dpo/delta": -0.04341936483979225,
"fcm_dpo/margin": 125.89375305175781,
"fcm_dpo/q_t": 0.3995819091796875,
"grad_norm": 24.996137619018555,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.2863304018974304,
"logits/rejected": -0.2791479825973511,
"logps/chosen": -191.18312072753906,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -353.76226806640625,
"loss": 1.0566,
"margin_dpo/margin_mean": 125.89375305175781,
"margin_dpo/margin_std": 142.8882293701172,
"step": 215
},
{
"KL/chosen_KL_mean": -142.95309448242188,
"KL/mean": -204.64840698242188,
"KL/rejected_KL_mean": -266.3437194824219,
"KL/std": 143.56790161132812,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.0034245119895786047,
"fcm_dpo/delta": -0.0468301959335804,
"fcm_dpo/margin": 123.39063262939453,
"fcm_dpo/q_t": 0.404532790184021,
"grad_norm": 28.86602210998535,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.26124703884124756,
"logits/rejected": -0.24592363834381104,
"logps/chosen": -191.84088134765625,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -343.54266357421875,
"loss": 1.0941,
"margin_dpo/margin_mean": 123.390625,
"margin_dpo/margin_std": 177.62725830078125,
"step": 216
},
{
"KL/chosen_KL_mean": -147.24102783203125,
"KL/mean": -219.417236328125,
"KL/rejected_KL_mean": -291.59344482421875,
"KL/std": 148.2445068359375,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.003357392270117998,
"fcm_dpo/delta": -0.08919186890125275,
"fcm_dpo/margin": 144.35240173339844,
"fcm_dpo/q_t": 0.3901098668575287,
"grad_norm": 20.184051513671875,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.24297907948493958,
"logits/rejected": -0.2382565438747406,
"logps/chosen": -197.08633422851562,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -391.6717529296875,
"loss": 1.0309,
"margin_dpo/margin_mean": 144.35240173339844,
"margin_dpo/margin_std": 167.79852294921875,
"step": 217
},
{
"KL/chosen_KL_mean": -154.8172607421875,
"KL/mean": -210.88455200195312,
"KL/rejected_KL_mean": -266.95184326171875,
"KL/std": 147.3013458251953,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.0033482126891613007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 112.13459777832031,
"fcm_dpo/q_t": 0.41538113355636597,
"grad_norm": 20.418493270874023,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.2989402413368225,
"logits/rejected": -0.2960107922554016,
"logps/chosen": -213.39393615722656,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -354.7982482910156,
"loss": 1.1249,
"margin_dpo/margin_mean": 112.13459777832031,
"margin_dpo/margin_std": 173.35887145996094,
"step": 218
},
{
"KL/chosen_KL_mean": -163.69097900390625,
"KL/mean": -214.0511474609375,
"KL/rejected_KL_mean": -264.4113464355469,
"KL/std": 152.53524780273438,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.0033482126891613007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 100.72035217285156,
"fcm_dpo/q_t": 0.4240277409553528,
"grad_norm": 25.4362850189209,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.28101325035095215,
"logits/rejected": -0.2753964066505432,
"logps/chosen": -224.77484130859375,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -350.24176025390625,
"loss": 1.1707,
"margin_dpo/margin_mean": 100.7203598022461,
"margin_dpo/margin_std": 188.10244750976562,
"step": 219
},
{
"KL/chosen_KL_mean": -189.14373779296875,
"KL/mean": -229.32162475585938,
"KL/rejected_KL_mean": -269.49951171875,
"KL/std": 133.23536682128906,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.0033482126891613007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 80.35578918457031,
"fcm_dpo/q_t": 0.43612149357795715,
"grad_norm": 23.351408004760742,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.276694655418396,
"logits/rejected": -0.25455957651138306,
"logps/chosen": -259.175048828125,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -357.18505859375,
"loss": 1.1856,
"margin_dpo/margin_mean": 80.35578918457031,
"margin_dpo/margin_std": 136.25027465820312,
"step": 220
},
{
"KL/chosen_KL_mean": -161.2969970703125,
"KL/mean": -251.148193359375,
"KL/rejected_KL_mean": -340.9993896484375,
"KL/std": 167.5245819091797,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.003260483266785741,
"fcm_dpo/delta": -0.19711089134216309,
"fcm_dpo/margin": 179.70236206054688,
"fcm_dpo/q_t": 0.3655005693435669,
"grad_norm": 21.982431411743164,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.26910918951034546,
"logits/rejected": -0.27597469091415405,
"logps/chosen": -213.45172119140625,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -447.4670715332031,
"loss": 0.9583,
"margin_dpo/margin_mean": 179.70236206054688,
"margin_dpo/margin_std": 173.05661010742188,
"step": 221
},
{
"KL/chosen_KL_mean": -167.68533325195312,
"KL/mean": -235.232177734375,
"KL/rejected_KL_mean": -302.77899169921875,
"KL/std": 153.12326049804688,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.003201256040483713,
"fcm_dpo/delta": -0.05476263165473938,
"fcm_dpo/margin": 135.09365844726562,
"fcm_dpo/q_t": 0.4009937047958374,
"grad_norm": 19.333269119262695,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.32539641857147217,
"logits/rejected": -0.33047088980674744,
"logps/chosen": -228.65643310546875,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -402.7801513671875,
"loss": 1.0713,
"margin_dpo/margin_mean": 135.09365844726562,
"margin_dpo/margin_std": 172.55303955078125,
"step": 222
},
{
"KL/chosen_KL_mean": -176.12669372558594,
"KL/mean": -233.26063537597656,
"KL/rejected_KL_mean": -290.39459228515625,
"KL/std": 149.42013549804688,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.0031837252900004387,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.26789855957031,
"fcm_dpo/q_t": 0.41713905334472656,
"grad_norm": 22.279924392700195,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.24122165143489838,
"logits/rejected": -0.23779387772083282,
"logps/chosen": -228.76727294921875,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -373.2196044921875,
"loss": 1.1392,
"margin_dpo/margin_mean": 114.26790618896484,
"margin_dpo/margin_std": 188.8822021484375,
"step": 223
},
{
"KL/chosen_KL_mean": -155.34686279296875,
"KL/mean": -219.0513153076172,
"KL/rejected_KL_mean": -282.75579833984375,
"KL/std": 164.88922119140625,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.0031389761716127396,
"fcm_dpo/delta": -0.07077645510435104,
"fcm_dpo/margin": 127.40892028808594,
"fcm_dpo/q_t": 0.4104171395301819,
"grad_norm": 22.986743927001953,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.28872960805892944,
"logits/rejected": -0.271827757358551,
"logps/chosen": -203.9422607421875,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -359.87225341796875,
"loss": 1.1095,
"margin_dpo/margin_mean": 127.40892028808594,
"margin_dpo/margin_std": 189.6688690185547,
"step": 224
},
{
"KL/chosen_KL_mean": -172.8418731689453,
"KL/mean": -249.9188232421875,
"KL/rejected_KL_mean": -326.9957580566406,
"KL/std": 159.6756591796875,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.003098821034654975,
"fcm_dpo/delta": -0.0816635936498642,
"fcm_dpo/margin": 154.15386962890625,
"fcm_dpo/q_t": 0.3921276330947876,
"grad_norm": 20.74527931213379,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.2806037664413452,
"logits/rejected": -0.2695569694042206,
"logps/chosen": -230.84234619140625,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -426.898681640625,
"loss": 1.0479,
"margin_dpo/margin_mean": 154.1538848876953,
"margin_dpo/margin_std": 196.17489624023438,
"step": 225
},
{
"KL/chosen_KL_mean": -144.2257843017578,
"KL/mean": -204.41741943359375,
"KL/rejected_KL_mean": -264.6090393066406,
"KL/std": 153.60317993164062,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.0030607180669903755,
"fcm_dpo/delta": -0.0445721298456192,
"fcm_dpo/margin": 120.38326263427734,
"fcm_dpo/q_t": 0.4166339933872223,
"grad_norm": 27.664154052734375,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.36745959520339966,
"logits/rejected": -0.33029118180274963,
"logps/chosen": -203.1245880126953,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -343.29681396484375,
"loss": 1.1233,
"margin_dpo/margin_mean": 120.38326263427734,
"margin_dpo/margin_std": 179.03895568847656,
"step": 226
},
{
"KL/chosen_KL_mean": -170.3883514404297,
"KL/mean": -246.78062438964844,
"KL/rejected_KL_mean": -323.1728820800781,
"KL/std": 175.88458251953125,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.0030199983157217503,
"fcm_dpo/delta": -0.06696625053882599,
"fcm_dpo/margin": 152.78453063964844,
"fcm_dpo/q_t": 0.39447590708732605,
"grad_norm": 20.940410614013672,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.37393718957901,
"logits/rejected": -0.36207836866378784,
"logps/chosen": -229.46054077148438,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -422.58526611328125,
"loss": 1.0474,
"margin_dpo/margin_mean": 152.78453063964844,
"margin_dpo/margin_std": 183.7152099609375,
"step": 227
},
{
"KL/chosen_KL_mean": -176.73321533203125,
"KL/mean": -230.6518096923828,
"KL/rejected_KL_mean": -284.57037353515625,
"KL/std": 145.71096801757812,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.0030199983157217503,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.83721160888672,
"fcm_dpo/q_t": 0.4245913624763489,
"grad_norm": 22.438199996948242,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.33213791251182556,
"logits/rejected": -0.3147745132446289,
"logps/chosen": -242.62449645996094,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -375.619140625,
"loss": 1.1566,
"margin_dpo/margin_mean": 107.83721160888672,
"margin_dpo/margin_std": 179.61813354492188,
"step": 228
},
{
"KL/chosen_KL_mean": -183.93844604492188,
"KL/mean": -240.5111083984375,
"KL/rejected_KL_mean": -297.08380126953125,
"KL/std": 168.53048706054688,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.0030199983157217503,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 113.14534759521484,
"fcm_dpo/q_t": 0.4204818904399872,
"grad_norm": 26.519044876098633,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.4333471655845642,
"logits/rejected": -0.40338772535324097,
"logps/chosen": -254.6448211669922,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -381.6112060546875,
"loss": 1.1392,
"margin_dpo/margin_mean": 113.14534759521484,
"margin_dpo/margin_std": 175.4351348876953,
"step": 229
},
{
"KL/chosen_KL_mean": -138.13897705078125,
"KL/mean": -226.32997131347656,
"KL/rejected_KL_mean": -314.52099609375,
"KL/std": 157.35964965820312,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.0029599140398204327,
"fcm_dpo/delta": -0.12893977761268616,
"fcm_dpo/margin": 176.38198852539062,
"fcm_dpo/q_t": 0.3783706724643707,
"grad_norm": 25.281761169433594,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.3073171377182007,
"logits/rejected": -0.31276828050613403,
"logps/chosen": -177.42098999023438,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -400.14288330078125,
"loss": 0.9923,
"margin_dpo/margin_mean": 176.38198852539062,
"margin_dpo/margin_std": 174.53048706054688,
"step": 230
},
{
"KL/chosen_KL_mean": -159.91831970214844,
"KL/mean": -220.16099548339844,
"KL/rejected_KL_mean": -280.4036865234375,
"KL/std": 140.28118896484375,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.0029431143775582314,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 120.48533630371094,
"fcm_dpo/q_t": 0.4189344346523285,
"grad_norm": 22.254228591918945,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.390174925327301,
"logits/rejected": -0.3631623685359955,
"logps/chosen": -223.19476318359375,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -354.527587890625,
"loss": 1.1191,
"margin_dpo/margin_mean": 120.48533630371094,
"margin_dpo/margin_std": 167.16416931152344,
"step": 231
},
{
"KL/chosen_KL_mean": -196.7489776611328,
"KL/mean": -244.6427001953125,
"KL/rejected_KL_mean": -292.536376953125,
"KL/std": 162.15066528320312,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.0029431143775582314,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.78742980957031,
"fcm_dpo/q_t": 0.43585118651390076,
"grad_norm": 21.261966705322266,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.38710182905197144,
"logits/rejected": -0.3654683232307434,
"logps/chosen": -267.49774169921875,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -376.5134582519531,
"loss": 1.1816,
"margin_dpo/margin_mean": 95.78742218017578,
"margin_dpo/margin_std": 168.59783935546875,
"step": 232
},
{
"KL/chosen_KL_mean": -180.105224609375,
"KL/mean": -257.1478576660156,
"KL/rejected_KL_mean": -334.1905212402344,
"KL/std": 175.86370849609375,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.0029314416460692883,
"fcm_dpo/delta": -0.03981967642903328,
"fcm_dpo/margin": 154.08529663085938,
"fcm_dpo/q_t": 0.40144720673561096,
"grad_norm": 25.009721755981445,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.3826596736907959,
"logits/rejected": -0.38623514771461487,
"logps/chosen": -234.9881591796875,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -441.67059326171875,
"loss": 1.0906,
"margin_dpo/margin_mean": 154.08529663085938,
"margin_dpo/margin_std": 233.36492919921875,
"step": 233
},
{
"KL/chosen_KL_mean": -169.5860595703125,
"KL/mean": -250.97674560546875,
"KL/rejected_KL_mean": -332.367431640625,
"KL/std": 157.31170654296875,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.0028566650580614805,
"fcm_dpo/delta": -0.10924752801656723,
"fcm_dpo/margin": 162.78138732910156,
"fcm_dpo/q_t": 0.39263916015625,
"grad_norm": 25.143383026123047,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.3675558865070343,
"logits/rejected": -0.3938511610031128,
"logps/chosen": -213.68057250976562,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -432.37408447265625,
"loss": 1.0386,
"margin_dpo/margin_mean": 162.7813720703125,
"margin_dpo/margin_std": 178.6966552734375,
"step": 234
},
{
"KL/chosen_KL_mean": -198.6753387451172,
"KL/mean": -249.81863403320312,
"KL/rejected_KL_mean": -300.9619140625,
"KL/std": 150.10362243652344,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.0028566650580614805,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.28659057617188,
"fcm_dpo/q_t": 0.43217870593070984,
"grad_norm": 22.54652214050293,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.41138529777526855,
"logits/rejected": -0.3917854428291321,
"logps/chosen": -260.91326904296875,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -391.35699462890625,
"loss": 1.1836,
"margin_dpo/margin_mean": 102.28659057617188,
"margin_dpo/margin_std": 185.93734741210938,
"step": 235
},
{
"KL/chosen_KL_mean": -141.9903106689453,
"KL/mean": -226.0331573486328,
"KL/rejected_KL_mean": -310.0760192871094,
"KL/std": 158.168701171875,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.002825359348207712,
"fcm_dpo/delta": -0.07859396189451218,
"fcm_dpo/margin": 168.08572387695312,
"fcm_dpo/q_t": 0.3882787823677063,
"grad_norm": 43.635643005371094,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.36709725856781006,
"logits/rejected": -0.36788949370384216,
"logps/chosen": -191.3316650390625,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -413.587646484375,
"loss": 1.0116,
"margin_dpo/margin_mean": 168.08570861816406,
"margin_dpo/margin_std": 155.02297973632812,
"step": 236
},
{
"KL/chosen_KL_mean": -197.8418426513672,
"KL/mean": -259.5992431640625,
"KL/rejected_KL_mean": -321.35662841796875,
"KL/std": 161.23373413085938,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.002812112681567669,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 123.5147705078125,
"fcm_dpo/q_t": 0.4200955033302307,
"grad_norm": 25.65804672241211,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.39958909153938293,
"logits/rejected": -0.3944680690765381,
"logps/chosen": -252.0099639892578,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -416.136962890625,
"loss": 1.1343,
"margin_dpo/margin_mean": 123.5147705078125,
"margin_dpo/margin_std": 188.49781799316406,
"step": 237
},
{
"KL/chosen_KL_mean": -180.33523559570312,
"KL/mean": -243.35289001464844,
"KL/rejected_KL_mean": -306.37054443359375,
"KL/std": 161.61019897460938,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.002812112681567669,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.03531646728516,
"fcm_dpo/q_t": 0.42012059688568115,
"grad_norm": 20.620817184448242,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.3614025413990021,
"logits/rejected": -0.37703627347946167,
"logps/chosen": -234.308349609375,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -395.7884826660156,
"loss": 1.1248,
"margin_dpo/margin_mean": 126.03531646728516,
"margin_dpo/margin_std": 182.96377563476562,
"step": 238
},
{
"KL/chosen_KL_mean": -196.66397094726562,
"KL/mean": -265.8099365234375,
"KL/rejected_KL_mean": -334.95587158203125,
"KL/std": 151.65145874023438,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.0027831587940454483,
"fcm_dpo/delta": -0.0517473965883255,
"fcm_dpo/margin": 138.29190063476562,
"fcm_dpo/q_t": 0.4096493124961853,
"grad_norm": 25.863664627075195,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.3546360731124878,
"logits/rejected": -0.3427826464176178,
"logps/chosen": -254.7617950439453,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -428.548828125,
"loss": 1.096,
"margin_dpo/margin_mean": 138.29190063476562,
"margin_dpo/margin_std": 172.67640686035156,
"step": 239
},
{
"KL/chosen_KL_mean": -210.21449279785156,
"KL/mean": -268.498291015625,
"KL/rejected_KL_mean": -326.7821044921875,
"KL/std": 167.20452880859375,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.0027831587940454483,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.56759643554688,
"fcm_dpo/q_t": 0.425168514251709,
"grad_norm": 30.908700942993164,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.4022897183895111,
"logits/rejected": -0.38126668334007263,
"logps/chosen": -270.8289794921875,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -400.900634765625,
"loss": 1.1631,
"margin_dpo/margin_mean": 116.56758880615234,
"margin_dpo/margin_std": 200.62228393554688,
"step": 240
},
{
"KL/chosen_KL_mean": -172.1416778564453,
"KL/mean": -263.7938537597656,
"KL/rejected_KL_mean": -355.446044921875,
"KL/std": 180.43846130371094,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0027444493025541306,
"fcm_dpo/delta": -0.10839249193668365,
"fcm_dpo/margin": 183.3043670654297,
"fcm_dpo/q_t": 0.3863786458969116,
"grad_norm": 19.921833038330078,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.43084996938705444,
"logits/rejected": -0.40479138493537903,
"logps/chosen": -238.23275756835938,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -443.5069274902344,
"loss": 1.0213,
"margin_dpo/margin_mean": 183.3043670654297,
"margin_dpo/margin_std": 207.9857635498047,
"step": 241
},
{
"KL/chosen_KL_mean": -193.84503173828125,
"KL/mean": -259.82501220703125,
"KL/rejected_KL_mean": -325.80499267578125,
"KL/std": 155.22760009765625,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.00272347335703671,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 131.9599609375,
"fcm_dpo/q_t": 0.41780009865760803,
"grad_norm": 27.682950973510742,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.3546012043952942,
"logits/rejected": -0.32627618312835693,
"logps/chosen": -261.708984375,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -409.16534423828125,
"loss": 1.1181,
"margin_dpo/margin_mean": 131.9599609375,
"margin_dpo/margin_std": 183.73941040039062,
"step": 242
},
{
"KL/chosen_KL_mean": -194.37255859375,
"KL/mean": -274.17266845703125,
"KL/rejected_KL_mean": -353.9727783203125,
"KL/std": 172.1506805419922,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.002707144943997264,
"fcm_dpo/delta": -0.03006729856133461,
"fcm_dpo/margin": 159.60023498535156,
"fcm_dpo/q_t": 0.40053194761276245,
"grad_norm": 21.327482223510742,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.39147722721099854,
"logits/rejected": -0.3596557080745697,
"logps/chosen": -257.456787109375,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -430.30841064453125,
"loss": 1.0628,
"margin_dpo/margin_mean": 159.6002197265625,
"margin_dpo/margin_std": 193.20114135742188,
"step": 243
},
{
"KL/chosen_KL_mean": -172.62156677246094,
"KL/mean": -254.97146606445312,
"KL/rejected_KL_mean": -337.3213806152344,
"KL/std": 166.24649047851562,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.0026877904310822487,
"fcm_dpo/delta": -0.035875409841537476,
"fcm_dpo/margin": 164.69979858398438,
"fcm_dpo/q_t": 0.39697980880737305,
"grad_norm": 31.131752014160156,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.45547521114349365,
"logits/rejected": -0.44788095355033875,
"logps/chosen": -233.76226806640625,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -432.21331787109375,
"loss": 1.0421,
"margin_dpo/margin_mean": 164.69979858398438,
"margin_dpo/margin_std": 171.30508422851562,
"step": 244
},
{
"KL/chosen_KL_mean": -195.41578674316406,
"KL/mean": -259.2579040527344,
"KL/rejected_KL_mean": -323.1000061035156,
"KL/std": 162.6224365234375,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.0026638784911483526,
"fcm_dpo/delta": -0.04468155652284622,
"fcm_dpo/margin": 127.68421936035156,
"fcm_dpo/q_t": 0.4205586314201355,
"grad_norm": 25.96481704711914,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.41297852993011475,
"logits/rejected": -0.3774294853210449,
"logps/chosen": -262.6780700683594,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -410.7401123046875,
"loss": 1.1345,
"margin_dpo/margin_mean": 127.6842269897461,
"margin_dpo/margin_std": 181.09228515625,
"step": 245
},
{
"KL/chosen_KL_mean": -190.72445678710938,
"KL/mean": -256.1783447265625,
"KL/rejected_KL_mean": -321.6322326660156,
"KL/std": 178.58486938476562,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.0026638784911483526,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.90777587890625,
"fcm_dpo/q_t": 0.42047402262687683,
"grad_norm": 23.14719581604004,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.413362979888916,
"logits/rejected": -0.3701326251029968,
"logps/chosen": -257.4214172363281,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -405.97857666015625,
"loss": 1.1292,
"margin_dpo/margin_mean": 130.9077911376953,
"margin_dpo/margin_std": 194.133056640625,
"step": 246
},
{
"KL/chosen_KL_mean": -225.75881958007812,
"KL/mean": -318.2025146484375,
"KL/rejected_KL_mean": -410.646240234375,
"KL/std": 188.978759765625,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.0026259967125952244,
"fcm_dpo/delta": -0.0899805873632431,
"fcm_dpo/margin": 184.88742065429688,
"fcm_dpo/q_t": 0.38787591457366943,
"grad_norm": 25.931663513183594,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.38560110330581665,
"logits/rejected": -0.38438546657562256,
"logps/chosen": -282.3641662597656,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -516.9395141601562,
"loss": 1.027,
"margin_dpo/margin_mean": 184.88742065429688,
"margin_dpo/margin_std": 205.1929931640625,
"step": 247
},
{
"KL/chosen_KL_mean": -197.12680053710938,
"KL/mean": -277.889892578125,
"KL/rejected_KL_mean": -358.6529541015625,
"KL/std": 154.3875732421875,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.002593907294794917,
"fcm_dpo/delta": -0.043108776211738586,
"fcm_dpo/margin": 161.52613830566406,
"fcm_dpo/q_t": 0.40013599395751953,
"grad_norm": 23.251401901245117,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.3690452575683594,
"logits/rejected": -0.3877558708190918,
"logps/chosen": -241.17001342773438,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -450.50982666015625,
"loss": 1.0481,
"margin_dpo/margin_mean": 161.52615356445312,
"margin_dpo/margin_std": 151.2039337158203,
"step": 248
},
{
"KL/chosen_KL_mean": -256.06048583984375,
"KL/mean": -301.0152587890625,
"KL/rejected_KL_mean": -345.97003173828125,
"KL/std": 172.55154418945312,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.002593907294794917,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.90953063964844,
"fcm_dpo/q_t": 0.4462352693080902,
"grad_norm": 38.79088592529297,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.36389392614364624,
"logits/rejected": -0.3354836106300354,
"logps/chosen": -318.5028381347656,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -426.4381103515625,
"loss": 1.2342,
"margin_dpo/margin_mean": 89.90953063964844,
"margin_dpo/margin_std": 204.7283935546875,
"step": 249
},
{
"KL/chosen_KL_mean": -224.89846801757812,
"KL/mean": -307.3691711425781,
"KL/rejected_KL_mean": -389.83990478515625,
"KL/std": 172.57041931152344,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.002579119522124529,
"fcm_dpo/delta": -0.05733676999807358,
"fcm_dpo/margin": 164.94143676757812,
"fcm_dpo/q_t": 0.4000697433948517,
"grad_norm": 38.31913375854492,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.4156474471092224,
"logits/rejected": -0.36385685205459595,
"logps/chosen": -290.53515625,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -463.71173095703125,
"loss": 1.0555,
"margin_dpo/margin_mean": 164.94143676757812,
"margin_dpo/margin_std": 175.64585876464844,
"step": 250
},
{
"KL/chosen_KL_mean": -236.84722900390625,
"KL/mean": -293.100830078125,
"KL/rejected_KL_mean": -349.3543701171875,
"KL/std": 183.2646026611328,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.0025643319822847843,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 112.50713348388672,
"fcm_dpo/q_t": 0.4335523843765259,
"grad_norm": 30.411144256591797,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3748970031738281,
"logits/rejected": -0.3472249507904053,
"logps/chosen": -294.02996826171875,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -427.017822265625,
"loss": 1.1849,
"margin_dpo/margin_mean": 112.50714111328125,
"margin_dpo/margin_std": 199.13619995117188,
"step": 251
},
{
"KL/chosen_KL_mean": -229.87887573242188,
"KL/mean": -305.1651916503906,
"KL/rejected_KL_mean": -380.4515075683594,
"KL/std": 155.89443969726562,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0025595955085009336,
"fcm_dpo/delta": -0.009243616834282875,
"fcm_dpo/margin": 150.5726318359375,
"fcm_dpo/q_t": 0.40730518102645874,
"grad_norm": 21.492137908935547,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.4269337058067322,
"logits/rejected": -0.38683801889419556,
"logps/chosen": -301.56451416015625,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -465.2095031738281,
"loss": 1.0784,
"margin_dpo/margin_mean": 150.5726318359375,
"margin_dpo/margin_std": 159.85842895507812,
"step": 252
},
{
"KL/chosen_KL_mean": -193.25228881835938,
"KL/mean": -267.5342102050781,
"KL/rejected_KL_mean": -341.81610107421875,
"KL/std": 163.1719970703125,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.00254888366907835,
"fcm_dpo/delta": -0.04202554374933243,
"fcm_dpo/margin": 148.56381225585938,
"fcm_dpo/q_t": 0.4106384813785553,
"grad_norm": 20.642892837524414,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.42303064465522766,
"logits/rejected": -0.4157634973526001,
"logps/chosen": -262.38623046875,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -440.5186462402344,
"loss": 1.0971,
"margin_dpo/margin_mean": 148.56381225585938,
"margin_dpo/margin_std": 182.68304443359375,
"step": 253
},
{
"KL/chosen_KL_mean": -186.4593048095703,
"KL/mean": -248.51510620117188,
"KL/rejected_KL_mean": -310.5709228515625,
"KL/std": 170.81390380859375,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.00253817206248641,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.11161041259766,
"fcm_dpo/q_t": 0.4287106394767761,
"grad_norm": 20.37973403930664,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.40077197551727295,
"logits/rejected": -0.38768458366394043,
"logps/chosen": -240.61428833007812,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -400.8785400390625,
"loss": 1.165,
"margin_dpo/margin_mean": 124.11161041259766,
"margin_dpo/margin_std": 215.81329345703125,
"step": 254
},
{
"KL/chosen_KL_mean": -191.34371948242188,
"KL/mean": -258.4233093261719,
"KL/rejected_KL_mean": -325.5028991699219,
"KL/std": 152.7626953125,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.00253817206248641,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 134.15916442871094,
"fcm_dpo/q_t": 0.42020100355148315,
"grad_norm": 18.49299430847168,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.33083224296569824,
"logits/rejected": -0.3193257451057434,
"logps/chosen": -248.4853973388672,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -415.71148681640625,
"loss": 1.1331,
"margin_dpo/margin_mean": 134.1591796875,
"margin_dpo/margin_std": 192.447265625,
"step": 255
},
{
"KL/chosen_KL_mean": -165.99559020996094,
"KL/mean": -238.594482421875,
"KL/rejected_KL_mean": -311.193359375,
"KL/std": 164.0802459716797,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.00253817206248641,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.19778442382812,
"fcm_dpo/q_t": 0.41385918855667114,
"grad_norm": 21.11256217956543,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.2933967709541321,
"logits/rejected": -0.30158305168151855,
"logps/chosen": -221.15908813476562,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -403.75628662109375,
"loss": 1.106,
"margin_dpo/margin_mean": 145.19778442382812,
"margin_dpo/margin_std": 189.60250854492188,
"step": 256
},
{
"KL/chosen_KL_mean": -161.5221405029297,
"KL/mean": -238.52484130859375,
"KL/rejected_KL_mean": -315.5274963378906,
"KL/std": 170.44703674316406,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.002513015875592828,
"fcm_dpo/delta": -0.04980289936065674,
"fcm_dpo/margin": 154.00537109375,
"fcm_dpo/q_t": 0.41046983003616333,
"grad_norm": 19.582237243652344,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.33271628618240356,
"logits/rejected": -0.3201237618923187,
"logps/chosen": -210.94583129882812,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -395.0654296875,
"loss": 1.0972,
"margin_dpo/margin_mean": 154.00535583496094,
"margin_dpo/margin_std": 195.84552001953125,
"step": 257
},
{
"KL/chosen_KL_mean": -220.34742736816406,
"KL/mean": -294.8319091796875,
"KL/rejected_KL_mean": -369.3164367675781,
"KL/std": 181.22947692871094,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.002513015875592828,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.96900939941406,
"fcm_dpo/q_t": 0.41232049465179443,
"grad_norm": 20.54438591003418,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.30997389554977417,
"logits/rejected": -0.310375452041626,
"logps/chosen": -279.7315673828125,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -465.3065490722656,
"loss": 1.118,
"margin_dpo/margin_mean": 148.96900939941406,
"margin_dpo/margin_std": 218.79287719726562,
"step": 258
},
{
"KL/chosen_KL_mean": -211.32296752929688,
"KL/mean": -275.07977294921875,
"KL/rejected_KL_mean": -338.8366394042969,
"KL/std": 175.24095153808594,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.002513015875592828,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 127.51367950439453,
"fcm_dpo/q_t": 0.4257126450538635,
"grad_norm": 21.773229598999023,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.3975661098957062,
"logits/rejected": -0.39768803119659424,
"logps/chosen": -264.15130615234375,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -428.0282897949219,
"loss": 1.1505,
"margin_dpo/margin_mean": 127.51367950439453,
"margin_dpo/margin_std": 197.5504150390625,
"step": 259
},
{
"KL/chosen_KL_mean": -218.90866088867188,
"KL/mean": -306.67657470703125,
"KL/rejected_KL_mean": -394.44451904296875,
"KL/std": 183.72035217285156,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.002498188056051731,
"fcm_dpo/delta": -0.05935557931661606,
"fcm_dpo/margin": 175.53582763671875,
"fcm_dpo/q_t": 0.3994418978691101,
"grad_norm": 25.106760025024414,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.32701295614242554,
"logits/rejected": -0.3442246615886688,
"logps/chosen": -266.3263244628906,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -489.5343017578125,
"loss": 1.0499,
"margin_dpo/margin_mean": 175.5358123779297,
"margin_dpo/margin_std": 196.42477416992188,
"step": 260
},
{
"KL/chosen_KL_mean": -242.05313110351562,
"KL/mean": -322.48565673828125,
"KL/rejected_KL_mean": -402.918212890625,
"KL/std": 203.7381591796875,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.0024833600036799908,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 160.8650665283203,
"fcm_dpo/q_t": 0.40943995118141174,
"grad_norm": 20.951894760131836,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.2890021800994873,
"logits/rejected": -0.2918785512447357,
"logps/chosen": -295.08453369140625,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -491.43316650390625,
"loss": 1.1019,
"margin_dpo/margin_mean": 160.8650665283203,
"margin_dpo/margin_std": 231.27334594726562,
"step": 261
},
{
"KL/chosen_KL_mean": -284.0287170410156,
"KL/mean": -350.4783935546875,
"KL/rejected_KL_mean": -416.92803955078125,
"KL/std": 187.28726196289062,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.0024833600036799908,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 132.89935302734375,
"fcm_dpo/q_t": 0.42301496863365173,
"grad_norm": 28.10672950744629,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.3489972949028015,
"logits/rejected": -0.3238402009010315,
"logps/chosen": -343.64886474609375,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -503.3465881347656,
"loss": 1.1355,
"margin_dpo/margin_mean": 132.89935302734375,
"margin_dpo/margin_std": 188.70492553710938,
"step": 262
},
{
"KL/chosen_KL_mean": -261.15643310546875,
"KL/mean": -355.87921142578125,
"KL/rejected_KL_mean": -450.60198974609375,
"KL/std": 221.2732696533203,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.0024432563222944736,
"fcm_dpo/delta": -0.0814036875963211,
"fcm_dpo/margin": 189.44557189941406,
"fcm_dpo/q_t": 0.39440131187438965,
"grad_norm": 25.79364585876465,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.34519362449645996,
"logits/rejected": -0.3233689069747925,
"logps/chosen": -320.5773620605469,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -547.4591674804688,
"loss": 1.0557,
"margin_dpo/margin_mean": 189.445556640625,
"margin_dpo/margin_std": 238.8328857421875,
"step": 263
},
{
"KL/chosen_KL_mean": -260.68182373046875,
"KL/mean": -341.7437744140625,
"KL/rejected_KL_mean": -422.80572509765625,
"KL/std": 199.98663330078125,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.0024124952033162117,
"fcm_dpo/delta": -0.06335047632455826,
"fcm_dpo/margin": 162.12388610839844,
"fcm_dpo/q_t": 0.41067957878112793,
"grad_norm": 21.558048248291016,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.3743303418159485,
"logits/rejected": -0.36552029848098755,
"logps/chosen": -323.4039306640625,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -516.6619262695312,
"loss": 1.1084,
"margin_dpo/margin_mean": 162.12387084960938,
"margin_dpo/margin_std": 230.56280517578125,
"step": 264
},
{
"KL/chosen_KL_mean": -281.0964660644531,
"KL/mean": -358.7066650390625,
"KL/rejected_KL_mean": -436.31683349609375,
"KL/std": 221.1083984375,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.0024124952033162117,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.22036743164062,
"fcm_dpo/q_t": 0.41601699590682983,
"grad_norm": 27.02475357055664,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.3154297471046448,
"logits/rejected": -0.2991935908794403,
"logps/chosen": -343.06793212890625,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -524.33740234375,
"loss": 1.1413,
"margin_dpo/margin_mean": 155.2203826904297,
"margin_dpo/margin_std": 266.7834167480469,
"step": 265
},
{
"KL/chosen_KL_mean": -279.3717041015625,
"KL/mean": -336.39202880859375,
"KL/rejected_KL_mean": -393.41229248046875,
"KL/std": 181.82192993164062,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0024124952033162117,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.04059600830078,
"fcm_dpo/q_t": 0.43650779128074646,
"grad_norm": 46.004493713378906,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.37979966402053833,
"logits/rejected": -0.34439635276794434,
"logps/chosen": -346.47137451171875,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -461.383544921875,
"loss": 1.1933,
"margin_dpo/margin_mean": 114.04059600830078,
"margin_dpo/margin_std": 214.42086791992188,
"step": 266
},
{
"KL/chosen_KL_mean": -240.74594116210938,
"KL/mean": -311.76678466796875,
"KL/rejected_KL_mean": -382.7876892089844,
"KL/std": 195.85140991210938,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.0024124952033162117,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 142.041748046875,
"fcm_dpo/q_t": 0.42108699679374695,
"grad_norm": 30.3566837310791,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3702971339225769,
"logits/rejected": -0.34150320291519165,
"logps/chosen": -309.7166748046875,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -472.95611572265625,
"loss": 1.1432,
"margin_dpo/margin_mean": 142.041748046875,
"margin_dpo/margin_std": 227.41090393066406,
"step": 267
},
{
"KL/chosen_KL_mean": -239.63525390625,
"KL/mean": -315.9992980957031,
"KL/rejected_KL_mean": -392.36334228515625,
"KL/std": 190.10870361328125,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.0024124952033162117,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.72805786132812,
"fcm_dpo/q_t": 0.41565245389938354,
"grad_norm": 22.044809341430664,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.3633720874786377,
"logits/rejected": -0.36941152811050415,
"logps/chosen": -295.53558349609375,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -494.010986328125,
"loss": 1.1219,
"margin_dpo/margin_mean": 152.72805786132812,
"margin_dpo/margin_std": 229.2215576171875,
"step": 268
},
{
"KL/chosen_KL_mean": -255.48077392578125,
"KL/mean": -346.63116455078125,
"KL/rejected_KL_mean": -437.78155517578125,
"KL/std": 193.64755249023438,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.00240440689958632,
"fcm_dpo/delta": -0.033639829605817795,
"fcm_dpo/margin": 182.30081176757812,
"fcm_dpo/q_t": 0.40017130970954895,
"grad_norm": 20.53118896484375,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.40448078513145447,
"logits/rejected": -0.38746941089630127,
"logps/chosen": -325.52032470703125,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -545.1309814453125,
"loss": 1.0795,
"margin_dpo/margin_mean": 182.30081176757812,
"margin_dpo/margin_std": 250.51087951660156,
"step": 269
},
{
"KL/chosen_KL_mean": -224.14105224609375,
"KL/mean": -290.2872314453125,
"KL/rejected_KL_mean": -356.43341064453125,
"KL/std": 168.980224609375,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.002396318595856428,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 132.29232788085938,
"fcm_dpo/q_t": 0.4260002374649048,
"grad_norm": 24.173391342163086,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.4147574007511139,
"logits/rejected": -0.4084188640117645,
"logps/chosen": -293.6745300292969,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -466.362060546875,
"loss": 1.1591,
"margin_dpo/margin_mean": 132.29232788085938,
"margin_dpo/margin_std": 217.41592407226562,
"step": 270
},
{
"KL/chosen_KL_mean": -206.2494659423828,
"KL/mean": -287.3037109375,
"KL/rejected_KL_mean": -368.3579406738281,
"KL/std": 166.89303588867188,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.002387921093031764,
"fcm_dpo/delta": -0.01755235157907009,
"fcm_dpo/margin": 162.10848999023438,
"fcm_dpo/q_t": 0.4096933901309967,
"grad_norm": 18.710674285888672,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.3881264925003052,
"logits/rejected": -0.3716167211532593,
"logps/chosen": -263.0140380859375,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -460.87176513671875,
"loss": 1.0844,
"margin_dpo/margin_mean": 162.10848999023438,
"margin_dpo/margin_std": 189.31494140625,
"step": 271
},
{
"KL/chosen_KL_mean": -187.37637329101562,
"KL/mean": -287.665283203125,
"KL/rejected_KL_mean": -387.9542541503906,
"KL/std": 180.75425720214844,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.0023701349273324013,
"fcm_dpo/delta": -0.07504396140575409,
"fcm_dpo/margin": 200.577880859375,
"fcm_dpo/q_t": 0.3880541920661926,
"grad_norm": 31.674179077148438,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.33635494112968445,
"logits/rejected": -0.3487662672996521,
"logps/chosen": -236.87351989746094,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -493.4970397949219,
"loss": 1.0159,
"margin_dpo/margin_mean": 200.577880859375,
"margin_dpo/margin_std": 190.57537841796875,
"step": 272
},
{
"KL/chosen_KL_mean": -226.82720947265625,
"KL/mean": -325.7548828125,
"KL/rejected_KL_mean": -424.68255615234375,
"KL/std": 184.29931640625,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.002323293825611472,
"fcm_dpo/delta": -0.06214258447289467,
"fcm_dpo/margin": 197.85537719726562,
"fcm_dpo/q_t": 0.3923366069793701,
"grad_norm": 25.173011779785156,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.3470395803451538,
"logits/rejected": -0.33116915822029114,
"logps/chosen": -289.8026123046875,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -517.18115234375,
"loss": 1.0317,
"margin_dpo/margin_mean": 197.85537719726562,
"margin_dpo/margin_std": 203.93701171875,
"step": 273
},
{
"KL/chosen_KL_mean": -281.4830017089844,
"KL/mean": -354.01580810546875,
"KL/rejected_KL_mean": -426.54864501953125,
"KL/std": 169.19039916992188,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.002323293825611472,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.06565856933594,
"fcm_dpo/q_t": 0.4209598898887634,
"grad_norm": 30.572553634643555,
"learning_rate": 3.75e-07,
"logits/chosen": -0.2957637906074524,
"logits/rejected": -0.281542032957077,
"logps/chosen": -337.15069580078125,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -503.8817138671875,
"loss": 1.1268,
"margin_dpo/margin_mean": 145.06565856933594,
"margin_dpo/margin_std": 194.85675048828125,
"step": 274
},
{
"KL/chosen_KL_mean": -227.81744384765625,
"KL/mean": -306.25030517578125,
"KL/rejected_KL_mean": -384.68316650390625,
"KL/std": 173.9310302734375,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.002323293825611472,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 156.86573791503906,
"fcm_dpo/q_t": 0.4142495393753052,
"grad_norm": 31.266094207763672,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.3424875736236572,
"logits/rejected": -0.350277841091156,
"logps/chosen": -276.4121398925781,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -477.98687744140625,
"loss": 1.1034,
"margin_dpo/margin_mean": 156.86572265625,
"margin_dpo/margin_std": 193.51849365234375,
"step": 275
},
{
"KL/chosen_KL_mean": -254.9441680908203,
"KL/mean": -334.92266845703125,
"KL/rejected_KL_mean": -414.9011535644531,
"KL/std": 187.92633056640625,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.002323293825611472,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 159.95697021484375,
"fcm_dpo/q_t": 0.41468003392219543,
"grad_norm": 23.98878288269043,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.3111796975135803,
"logits/rejected": -0.2913016676902771,
"logps/chosen": -311.5215759277344,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -485.2668151855469,
"loss": 1.1146,
"margin_dpo/margin_mean": 159.95697021484375,
"margin_dpo/margin_std": 226.91453552246094,
"step": 276
},
{
"KL/chosen_KL_mean": -278.7911071777344,
"KL/mean": -364.60546875,
"KL/rejected_KL_mean": -450.41986083984375,
"KL/std": 194.7156524658203,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.0023189065977931023,
"fcm_dpo/delta": -0.018919235095381737,
"fcm_dpo/margin": 171.62875366210938,
"fcm_dpo/q_t": 0.4078858494758606,
"grad_norm": 28.692832946777344,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.3414786756038666,
"logits/rejected": -0.3268578350543976,
"logps/chosen": -335.06268310546875,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -543.3011474609375,
"loss": 1.0971,
"margin_dpo/margin_mean": 171.62875366210938,
"margin_dpo/margin_std": 233.7840576171875,
"step": 277
},
{
"KL/chosen_KL_mean": -250.5606689453125,
"KL/mean": -350.6166076660156,
"KL/rejected_KL_mean": -450.6725158691406,
"KL/std": 205.93980407714844,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.0022912982385605574,
"fcm_dpo/delta": -0.05041739344596863,
"fcm_dpo/margin": 200.11187744140625,
"fcm_dpo/q_t": 0.3950769305229187,
"grad_norm": 23.307388305664062,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.33211779594421387,
"logits/rejected": -0.3396395444869995,
"logps/chosen": -303.50262451171875,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -541.9260864257812,
"loss": 1.0454,
"margin_dpo/margin_mean": 200.11187744140625,
"margin_dpo/margin_std": 233.86526489257812,
"step": 278
},
{
"KL/chosen_KL_mean": -291.84429931640625,
"KL/mean": -390.2003479003906,
"KL/rejected_KL_mean": -488.55645751953125,
"KL/std": 218.87045288085938,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.002256458392366767,
"fcm_dpo/delta": -0.076610267162323,
"fcm_dpo/margin": 196.71217346191406,
"fcm_dpo/q_t": 0.40036946535110474,
"grad_norm": 24.625885009765625,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.2654384970664978,
"logits/rejected": -0.2664095461368561,
"logps/chosen": -340.485595703125,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -576.4078979492188,
"loss": 1.0794,
"margin_dpo/margin_mean": 196.712158203125,
"margin_dpo/margin_std": 274.7304992675781,
"step": 279
},
{
"KL/chosen_KL_mean": -287.09918212890625,
"KL/mean": -388.5111083984375,
"KL/rejected_KL_mean": -489.9229736328125,
"KL/std": 199.4989013671875,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.0022379662841558456,
"fcm_dpo/delta": -0.05646730586886406,
"fcm_dpo/margin": 202.82379150390625,
"fcm_dpo/q_t": 0.3937091827392578,
"grad_norm": 25.541810989379883,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.29510489106178284,
"logits/rejected": -0.29825708270072937,
"logps/chosen": -345.89630126953125,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -588.5418701171875,
"loss": 1.0388,
"margin_dpo/margin_mean": 202.82379150390625,
"margin_dpo/margin_std": 217.46322631835938,
"step": 280
},
{
"KL/chosen_KL_mean": -250.59390258789062,
"KL/mean": -339.3375244140625,
"KL/rejected_KL_mean": -428.0810852050781,
"KL/std": 193.9304656982422,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0022240430116653442,
"fcm_dpo/delta": -0.031813472509384155,
"fcm_dpo/margin": 177.48715209960938,
"fcm_dpo/q_t": 0.40596914291381836,
"grad_norm": 20.3207950592041,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.35418635606765747,
"logits/rejected": -0.3321439325809479,
"logps/chosen": -306.0824279785156,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -508.96368408203125,
"loss": 1.0843,
"margin_dpo/margin_mean": 177.48715209960938,
"margin_dpo/margin_std": 211.81997680664062,
"step": 281
},
{
"KL/chosen_KL_mean": -276.3832092285156,
"KL/mean": -345.36126708984375,
"KL/rejected_KL_mean": -414.33935546875,
"KL/std": 201.35035705566406,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.0022169677540659904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.95614624023438,
"fcm_dpo/q_t": 0.43155795335769653,
"grad_norm": 23.50833511352539,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.39513063430786133,
"logits/rejected": -0.37628278136253357,
"logps/chosen": -349.453369140625,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -509.69036865234375,
"loss": 1.1661,
"margin_dpo/margin_mean": 137.95614624023438,
"margin_dpo/margin_std": 238.62408447265625,
"step": 282
},
{
"KL/chosen_KL_mean": -274.4816589355469,
"KL/mean": -375.57525634765625,
"KL/rejected_KL_mean": -476.66888427734375,
"KL/std": 229.27352905273438,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.0022034652065485716,
"fcm_dpo/delta": -0.06127964332699776,
"fcm_dpo/margin": 202.1872100830078,
"fcm_dpo/q_t": 0.39549362659454346,
"grad_norm": 34.2918815612793,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.37188607454299927,
"logits/rejected": -0.3768133819103241,
"logps/chosen": -336.3800964355469,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -573.6553955078125,
"loss": 1.0463,
"margin_dpo/margin_mean": 202.18719482421875,
"margin_dpo/margin_std": 219.53720092773438,
"step": 283
},
{
"KL/chosen_KL_mean": -246.93585205078125,
"KL/mean": -343.9705505371094,
"KL/rejected_KL_mean": -441.0052490234375,
"KL/std": 210.19834899902344,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.002178533934056759,
"fcm_dpo/delta": -0.026161545887589455,
"fcm_dpo/margin": 194.06942749023438,
"fcm_dpo/q_t": 0.40190061926841736,
"grad_norm": 26.171096801757812,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.40229523181915283,
"logits/rejected": -0.38924217224121094,
"logps/chosen": -305.37139892578125,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -534.4744873046875,
"loss": 1.0582,
"margin_dpo/margin_mean": 194.06942749023438,
"margin_dpo/margin_std": 215.73760986328125,
"step": 284
},
{
"KL/chosen_KL_mean": -277.75640869140625,
"KL/mean": -361.48651123046875,
"KL/rejected_KL_mean": -445.216552734375,
"KL/std": 180.0686798095703,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.0021733520552515984,
"fcm_dpo/delta": -0.023842982947826385,
"fcm_dpo/margin": 167.46017456054688,
"fcm_dpo/q_t": 0.41295209527015686,
"grad_norm": 26.861412048339844,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.3674265146255493,
"logits/rejected": -0.35316190123558044,
"logps/chosen": -343.98858642578125,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -544.3433837890625,
"loss": 1.1025,
"margin_dpo/margin_mean": 167.46017456054688,
"margin_dpo/margin_std": 200.82656860351562,
"step": 285
},
{
"KL/chosen_KL_mean": -270.828125,
"KL/mean": -364.634765625,
"KL/rejected_KL_mean": -458.4414367675781,
"KL/std": 212.66998291015625,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.0021647391840815544,
"fcm_dpo/delta": -0.01585019752383232,
"fcm_dpo/margin": 187.61329650878906,
"fcm_dpo/q_t": 0.4055512249469757,
"grad_norm": 26.513324737548828,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.36591431498527527,
"logits/rejected": -0.3500216603279114,
"logps/chosen": -343.77911376953125,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -547.0299072265625,
"loss": 1.0822,
"margin_dpo/margin_mean": 187.61331176757812,
"margin_dpo/margin_std": 235.45333862304688,
"step": 286
},
{
"KL/chosen_KL_mean": -253.10614013671875,
"KL/mean": -328.2662048339844,
"KL/rejected_KL_mean": -403.42626953125,
"KL/std": 183.10687255859375,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.0021613079588860273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 150.32009887695312,
"fcm_dpo/q_t": 0.42383337020874023,
"grad_norm": 24.7573184967041,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.32295361161231995,
"logits/rejected": -0.2973632216453552,
"logps/chosen": -314.6473083496094,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -481.1223449707031,
"loss": 1.1348,
"margin_dpo/margin_mean": 150.32009887695312,
"margin_dpo/margin_std": 206.97796630859375,
"step": 287
},
{
"KL/chosen_KL_mean": -244.66409301757812,
"KL/mean": -341.7845458984375,
"KL/rejected_KL_mean": -438.90496826171875,
"KL/std": 186.95571899414062,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.0021289377473294735,
"fcm_dpo/delta": -0.07545191794633865,
"fcm_dpo/margin": 194.24093627929688,
"fcm_dpo/q_t": 0.4019187092781067,
"grad_norm": 22.394235610961914,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.3601091504096985,
"logits/rejected": -0.34544798731803894,
"logps/chosen": -301.3253173828125,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -526.24072265625,
"loss": 1.0607,
"margin_dpo/margin_mean": 194.2409210205078,
"margin_dpo/margin_std": 198.42041015625,
"step": 288
},
{
"KL/chosen_KL_mean": -245.93588256835938,
"KL/mean": -353.0520935058594,
"KL/rejected_KL_mean": -460.1683349609375,
"KL/std": 213.45416259765625,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.002117036608979106,
"fcm_dpo/delta": -0.05621682479977608,
"fcm_dpo/margin": 214.232421875,
"fcm_dpo/q_t": 0.39465266466140747,
"grad_norm": 25.998571395874023,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.31153666973114014,
"logits/rejected": -0.3129374086856842,
"logps/chosen": -291.166259765625,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -547.81103515625,
"loss": 1.0397,
"margin_dpo/margin_mean": 214.232421875,
"margin_dpo/margin_std": 230.45819091796875,
"step": 289
},
{
"KL/chosen_KL_mean": -280.966552734375,
"KL/mean": -389.29156494140625,
"KL/rejected_KL_mean": -497.6165771484375,
"KL/std": 227.33908081054688,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.0020852871239185333,
"fcm_dpo/delta": -0.047366589307785034,
"fcm_dpo/margin": 216.65000915527344,
"fcm_dpo/q_t": 0.3967745900154114,
"grad_norm": 22.709102630615234,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.3108745515346527,
"logits/rejected": -0.331379234790802,
"logps/chosen": -336.43804931640625,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -614.3251342773438,
"loss": 1.0563,
"margin_dpo/margin_mean": 216.64999389648438,
"margin_dpo/margin_std": 267.9898376464844,
"step": 290
},
{
"KL/chosen_KL_mean": -246.94216918945312,
"KL/mean": -346.29931640625,
"KL/rejected_KL_mean": -445.6565246582031,
"KL/std": 199.556884765625,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.002075555734336376,
"fcm_dpo/delta": -0.04688554257154465,
"fcm_dpo/margin": 198.71432495117188,
"fcm_dpo/q_t": 0.4034580588340759,
"grad_norm": 22.80071449279785,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.30826008319854736,
"logits/rejected": -0.29707035422325134,
"logps/chosen": -310.2025451660156,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -534.95361328125,
"loss": 1.0765,
"margin_dpo/margin_mean": 198.71434020996094,
"margin_dpo/margin_std": 241.03353881835938,
"step": 291
},
{
"KL/chosen_KL_mean": -286.8546142578125,
"KL/mean": -396.198486328125,
"KL/rejected_KL_mean": -505.5423583984375,
"KL/std": 243.14376831054688,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.002054115291684866,
"fcm_dpo/delta": -0.057004090398550034,
"fcm_dpo/margin": 218.687744140625,
"fcm_dpo/q_t": 0.3968570828437805,
"grad_norm": 27.79972267150879,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.31336697936058044,
"logits/rejected": -0.3037183880805969,
"logps/chosen": -340.77313232421875,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -595.5037841796875,
"loss": 1.0678,
"margin_dpo/margin_mean": 218.68775939941406,
"margin_dpo/margin_std": 288.130126953125,
"step": 292
},
{
"KL/chosen_KL_mean": -290.5531311035156,
"KL/mean": -378.1788330078125,
"KL/rejected_KL_mean": -465.8045654296875,
"KL/std": 239.843994140625,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.002042406238615513,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 175.25146484375,
"fcm_dpo/q_t": 0.4187690317630768,
"grad_norm": 38.308929443359375,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.2938636541366577,
"logits/rejected": -0.28838860988616943,
"logps/chosen": -350.92913818359375,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -543.656982421875,
"loss": 1.1302,
"margin_dpo/margin_mean": 175.25146484375,
"margin_dpo/margin_std": 270.41204833984375,
"step": 293
},
{
"KL/chosen_KL_mean": -270.2442932128906,
"KL/mean": -364.52728271484375,
"KL/rejected_KL_mean": -458.810302734375,
"KL/std": 215.75311279296875,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.002042406238615513,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 188.5659942626953,
"fcm_dpo/q_t": 0.4126633405685425,
"grad_norm": 21.470027923583984,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.32128533720970154,
"logits/rejected": -0.3278936743736267,
"logps/chosen": -318.33184814453125,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -540.707275390625,
"loss": 1.1073,
"margin_dpo/margin_mean": 188.5659942626953,
"margin_dpo/margin_std": 268.078125,
"step": 294
},
{
"KL/chosen_KL_mean": -317.76177978515625,
"KL/mean": -423.4981689453125,
"KL/rejected_KL_mean": -529.2345581054688,
"KL/std": 245.9403076171875,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.002035372657701373,
"fcm_dpo/delta": -0.03455709293484688,
"fcm_dpo/margin": 211.4727325439453,
"fcm_dpo/q_t": 0.402416467666626,
"grad_norm": 26.90194320678711,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.32361164689064026,
"logits/rejected": -0.3285978138446808,
"logps/chosen": -367.68646240234375,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -616.6908569335938,
"loss": 1.0834,
"margin_dpo/margin_mean": 211.47274780273438,
"margin_dpo/margin_std": 289.3609313964844,
"step": 295
},
{
"KL/chosen_KL_mean": -381.29339599609375,
"KL/mean": -451.5999755859375,
"KL/rejected_KL_mean": -521.9066162109375,
"KL/std": 226.91986083984375,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.0020283390767872334,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 140.61317443847656,
"fcm_dpo/q_t": 0.43434810638427734,
"grad_norm": 37.26921844482422,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.3297673165798187,
"logits/rejected": -0.31439995765686035,
"logps/chosen": -446.7846374511719,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -614.9956665039062,
"loss": 1.205,
"margin_dpo/margin_mean": 140.61317443847656,
"margin_dpo/margin_std": 288.5725402832031,
"step": 296
},
{
"KL/chosen_KL_mean": -315.0804443359375,
"KL/mean": -400.39654541015625,
"KL/rejected_KL_mean": -485.7126159667969,
"KL/std": 217.16357421875,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.0020283390767872334,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.63214111328125,
"fcm_dpo/q_t": 0.41893017292022705,
"grad_norm": 26.733783721923828,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.32497072219848633,
"logits/rejected": -0.3295114040374756,
"logps/chosen": -371.55743408203125,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -580.8511962890625,
"loss": 1.1184,
"margin_dpo/margin_mean": 170.6321563720703,
"margin_dpo/margin_std": 222.90133666992188,
"step": 297
},
{
"KL/chosen_KL_mean": -330.19439697265625,
"KL/mean": -439.9351806640625,
"KL/rejected_KL_mean": -549.6759643554688,
"KL/std": 282.5112609863281,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.002016341080889106,
"fcm_dpo/delta": -0.05950422212481499,
"fcm_dpo/margin": 219.48158264160156,
"fcm_dpo/q_t": 0.4024728238582611,
"grad_norm": 25.506681442260742,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.36218610405921936,
"logits/rejected": -0.3801892399787903,
"logps/chosen": -397.51959228515625,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -666.338134765625,
"loss": 1.0872,
"margin_dpo/margin_mean": 219.4815673828125,
"margin_dpo/margin_std": 327.8146057128906,
"step": 298
},
{
"KL/chosen_KL_mean": -258.4277038574219,
"KL/mean": -353.2135925292969,
"KL/rejected_KL_mean": -447.99951171875,
"KL/std": 213.69512939453125,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.001993919722735882,
"fcm_dpo/delta": -0.052276819944381714,
"fcm_dpo/margin": 189.57174682617188,
"fcm_dpo/q_t": 0.4135133624076843,
"grad_norm": 22.83670997619629,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.35608577728271484,
"logits/rejected": -0.36062926054000854,
"logps/chosen": -307.3898010253906,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -532.3277587890625,
"loss": 1.1126,
"margin_dpo/margin_mean": 189.57174682617188,
"margin_dpo/margin_std": 266.7342224121094,
"step": 299
},
{
"KL/chosen_KL_mean": -330.7674255371094,
"KL/mean": -447.5982360839844,
"KL/rejected_KL_mean": -564.4290161132812,
"KL/std": 253.9108123779297,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.0019719761330634356,
"fcm_dpo/delta": -0.05841895192861557,
"fcm_dpo/margin": 233.66162109375,
"fcm_dpo/q_t": 0.3947865962982178,
"grad_norm": 22.9200382232666,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.3574295938014984,
"logits/rejected": -0.36054882407188416,
"logps/chosen": -389.84112548828125,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -660.3955078125,
"loss": 1.0641,
"margin_dpo/margin_mean": 233.66162109375,
"margin_dpo/margin_std": 307.6568908691406,
"step": 300
},
{
"KL/chosen_KL_mean": -282.0446472167969,
"KL/mean": -391.8178405761719,
"KL/rejected_KL_mean": -501.5910339355469,
"KL/std": 232.38865661621094,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.0019472182029858232,
"fcm_dpo/delta": -0.033876933157444,
"fcm_dpo/margin": 219.54641723632812,
"fcm_dpo/q_t": 0.40307682752609253,
"grad_norm": 26.896240234375,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.3596671223640442,
"logits/rejected": -0.35958829522132874,
"logps/chosen": -339.29400634765625,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -593.944580078125,
"loss": 1.0718,
"margin_dpo/margin_mean": 219.54641723632812,
"margin_dpo/margin_std": 281.4098815917969,
"step": 301
},
{
"KL/chosen_KL_mean": -246.8507080078125,
"KL/mean": -341.59490966796875,
"KL/rejected_KL_mean": -436.339111328125,
"KL/std": 210.02322387695312,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.0019371429225429893,
"fcm_dpo/delta": -0.025938192382454872,
"fcm_dpo/margin": 189.48838806152344,
"fcm_dpo/q_t": 0.41519904136657715,
"grad_norm": 18.704797744750977,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.33896106481552124,
"logits/rejected": -0.3427407741546631,
"logps/chosen": -298.0487060546875,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -533.5654907226562,
"loss": 1.1105,
"margin_dpo/margin_mean": 189.4884033203125,
"margin_dpo/margin_std": 254.37966918945312,
"step": 302
},
{
"KL/chosen_KL_mean": -267.103515625,
"KL/mean": -351.097412109375,
"KL/rejected_KL_mean": -435.0912780761719,
"KL/std": 222.8134765625,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.0019371429225429893,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 167.98776245117188,
"fcm_dpo/q_t": 0.4245578348636627,
"grad_norm": 21.095529556274414,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.3702804148197174,
"logits/rejected": -0.35734331607818604,
"logps/chosen": -333.8174743652344,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -522.0367431640625,
"loss": 1.1509,
"margin_dpo/margin_mean": 167.98776245117188,
"margin_dpo/margin_std": 264.559814453125,
"step": 303
},
{
"KL/chosen_KL_mean": -238.75244140625,
"KL/mean": -331.8809509277344,
"KL/rejected_KL_mean": -425.00946044921875,
"KL/std": 199.30747985839844,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.0019338021520525217,
"fcm_dpo/delta": -0.01727566123008728,
"fcm_dpo/margin": 186.2570343017578,
"fcm_dpo/q_t": 0.41349148750305176,
"grad_norm": 37.11244583129883,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.401491641998291,
"logits/rejected": -0.3919214606285095,
"logps/chosen": -310.703125,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -515.4815063476562,
"loss": 1.0899,
"margin_dpo/margin_mean": 186.2570343017578,
"margin_dpo/margin_std": 182.33255004882812,
"step": 304
},
{
"KL/chosen_KL_mean": -261.7723083496094,
"KL/mean": -347.92303466796875,
"KL/rejected_KL_mean": -434.0737609863281,
"KL/std": 228.82867431640625,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.0019304613815620542,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 172.3014373779297,
"fcm_dpo/q_t": 0.42360255122184753,
"grad_norm": 20.761266708374023,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.40634918212890625,
"logits/rejected": -0.39003273844718933,
"logps/chosen": -328.56756591796875,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -526.828369140625,
"loss": 1.1393,
"margin_dpo/margin_mean": 172.3014373779297,
"margin_dpo/margin_std": 253.9287872314453,
"step": 305
},
{
"KL/chosen_KL_mean": -256.45379638671875,
"KL/mean": -338.452880859375,
"KL/rejected_KL_mean": -420.4520263671875,
"KL/std": 199.2619171142578,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.0019304613815620542,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 163.99819946289062,
"fcm_dpo/q_t": 0.4252777099609375,
"grad_norm": 23.1071834564209,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.387315571308136,
"logits/rejected": -0.3650142550468445,
"logps/chosen": -326.1376953125,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -505.6112060546875,
"loss": 1.1352,
"margin_dpo/margin_mean": 163.99819946289062,
"margin_dpo/margin_std": 214.847412109375,
"step": 306
},
{
"KL/chosen_KL_mean": -229.98654174804688,
"KL/mean": -315.85711669921875,
"KL/rejected_KL_mean": -401.7276611328125,
"KL/std": 180.1286163330078,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.0019304613815620542,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 171.74110412597656,
"fcm_dpo/q_t": 0.4218043088912964,
"grad_norm": 25.3315372467041,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.3864956200122833,
"logits/rejected": -0.3664168119430542,
"logps/chosen": -300.1519775390625,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -488.699951171875,
"loss": 1.1255,
"margin_dpo/margin_mean": 171.74111938476562,
"margin_dpo/margin_std": 222.21095275878906,
"step": 307
},
{
"KL/chosen_KL_mean": -238.41258239746094,
"KL/mean": -333.81622314453125,
"KL/rejected_KL_mean": -429.21990966796875,
"KL/std": 205.3813934326172,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.0019304613815620542,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 190.80731201171875,
"fcm_dpo/q_t": 0.4120708405971527,
"grad_norm": 28.616819381713867,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.36001917719841003,
"logits/rejected": -0.34961068630218506,
"logps/chosen": -293.6575927734375,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -508.5921630859375,
"loss": 1.086,
"margin_dpo/margin_mean": 190.80731201171875,
"margin_dpo/margin_std": 194.4600067138672,
"step": 308
},
{
"KL/chosen_KL_mean": -246.6319580078125,
"KL/mean": -343.8309020996094,
"KL/rejected_KL_mean": -441.02984619140625,
"KL/std": 216.66287231445312,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.0019234230276197195,
"fcm_dpo/delta": -0.036592997610569,
"fcm_dpo/margin": 194.39788818359375,
"fcm_dpo/q_t": 0.4111284017562866,
"grad_norm": 21.753137588500977,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.33818137645721436,
"logits/rejected": -0.33194500207901,
"logps/chosen": -295.5910339355469,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -523.37060546875,
"loss": 1.0974,
"margin_dpo/margin_mean": 194.3979034423828,
"margin_dpo/margin_std": 233.65219116210938,
"step": 309
},
{
"KL/chosen_KL_mean": -290.893798828125,
"KL/mean": -381.14703369140625,
"KL/rejected_KL_mean": -471.4002380371094,
"KL/std": 195.173828125,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.0019163846736773849,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 180.50640869140625,
"fcm_dpo/q_t": 0.4173274040222168,
"grad_norm": 29.18208122253418,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.4003097414970398,
"logits/rejected": -0.39030611515045166,
"logps/chosen": -353.6356201171875,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -551.33056640625,
"loss": 1.103,
"margin_dpo/margin_mean": 180.50640869140625,
"margin_dpo/margin_std": 187.60948181152344,
"step": 310
},
{
"KL/chosen_KL_mean": -302.8953857421875,
"KL/mean": -404.6163330078125,
"KL/rejected_KL_mean": -506.3373107910156,
"KL/std": 253.53402709960938,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.0019163846736773849,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 203.44192504882812,
"fcm_dpo/q_t": 0.4103236794471741,
"grad_norm": 23.260826110839844,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.3798481822013855,
"logits/rejected": -0.3574289083480835,
"logps/chosen": -355.92333984375,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -583.7755126953125,
"loss": 1.0942,
"margin_dpo/margin_mean": 203.44192504882812,
"margin_dpo/margin_std": 265.4727783203125,
"step": 311
},
{
"KL/chosen_KL_mean": -281.3587341308594,
"KL/mean": -376.16253662109375,
"KL/rejected_KL_mean": -470.96636962890625,
"KL/std": 222.47909545898438,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.0019128790590912104,
"fcm_dpo/delta": -0.018326791003346443,
"fcm_dpo/margin": 189.607666015625,
"fcm_dpo/q_t": 0.41681230068206787,
"grad_norm": 20.335750579833984,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.35430508852005005,
"logits/rejected": -0.345356285572052,
"logps/chosen": -330.7509460449219,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -546.7591552734375,
"loss": 1.1149,
"margin_dpo/margin_mean": 189.607666015625,
"margin_dpo/margin_std": 263.2210693359375,
"step": 312
},
{
"KL/chosen_KL_mean": -257.8659973144531,
"KL/mean": -354.26409912109375,
"KL/rejected_KL_mean": -450.66229248046875,
"KL/std": 233.85096740722656,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.001899800612591207,
"fcm_dpo/delta": -0.050388313829898834,
"fcm_dpo/margin": 192.7962646484375,
"fcm_dpo/q_t": 0.4142175018787384,
"grad_norm": 21.600605010986328,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.4061209559440613,
"logits/rejected": -0.4058893322944641,
"logps/chosen": -308.01873779296875,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -537.0684814453125,
"loss": 1.1116,
"margin_dpo/margin_mean": 192.79627990722656,
"margin_dpo/margin_std": 252.84710693359375,
"step": 313
},
{
"KL/chosen_KL_mean": -273.6401672363281,
"KL/mean": -364.00518798828125,
"KL/rejected_KL_mean": -454.3702392578125,
"KL/std": 219.98101806640625,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.0018902278970927,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 180.7301025390625,
"fcm_dpo/q_t": 0.42193371057510376,
"grad_norm": 21.16115379333496,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.3945729732513428,
"logits/rejected": -0.3945373296737671,
"logps/chosen": -330.87774658203125,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -551.966796875,
"loss": 1.1404,
"margin_dpo/margin_mean": 180.7301025390625,
"margin_dpo/margin_std": 281.34197998046875,
"step": 314
},
{
"KL/chosen_KL_mean": -255.35317993164062,
"KL/mean": -339.78680419921875,
"KL/rejected_KL_mean": -424.220458984375,
"KL/std": 200.34494018554688,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.0018902278970927,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 168.86724853515625,
"fcm_dpo/q_t": 0.4248756170272827,
"grad_norm": 19.619430541992188,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.3391643762588501,
"logits/rejected": -0.3408266305923462,
"logps/chosen": -304.42279052734375,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -509.9013366699219,
"loss": 1.1346,
"margin_dpo/margin_mean": 168.86724853515625,
"margin_dpo/margin_std": 222.66644287109375,
"step": 315
},
{
"KL/chosen_KL_mean": -263.96673583984375,
"KL/mean": -385.8088073730469,
"KL/rejected_KL_mean": -507.65087890625,
"KL/std": 244.05743408203125,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.0018620459595695138,
"fcm_dpo/delta": -0.0751076489686966,
"fcm_dpo/margin": 243.68417358398438,
"fcm_dpo/q_t": 0.39560264348983765,
"grad_norm": 24.86118507385254,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.37902936339378357,
"logits/rejected": -0.38191086053848267,
"logps/chosen": -318.22747802734375,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -608.932373046875,
"loss": 1.0429,
"margin_dpo/margin_mean": 243.6841583251953,
"margin_dpo/margin_std": 270.5803527832031,
"step": 316
},
{
"KL/chosen_KL_mean": -278.44287109375,
"KL/mean": -410.1659240722656,
"KL/rejected_KL_mean": -541.888916015625,
"KL/std": 223.99581909179688,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.0018459368729963899,
"fcm_dpo/delta": -0.08726991713047028,
"fcm_dpo/margin": 263.4460754394531,
"fcm_dpo/q_t": 0.38609233498573303,
"grad_norm": 25.92868995666504,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.38822823762893677,
"logits/rejected": -0.380374550819397,
"logps/chosen": -334.537109375,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -642.5880126953125,
"loss": 1.0104,
"margin_dpo/margin_mean": 263.4460754394531,
"margin_dpo/margin_std": 248.41903686523438,
"step": 317
},
{
"KL/chosen_KL_mean": -324.0246276855469,
"KL/mean": -418.319580078125,
"KL/rejected_KL_mean": -512.6144409179688,
"KL/std": 236.26177978515625,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.001829827786423266,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 188.58981323242188,
"fcm_dpo/q_t": 0.4197791814804077,
"grad_norm": 24.93431854248047,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.4239439368247986,
"logits/rejected": -0.3966226577758789,
"logps/chosen": -388.67034912109375,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -595.3787231445312,
"loss": 1.1248,
"margin_dpo/margin_mean": 188.58982849121094,
"margin_dpo/margin_std": 262.1299133300781,
"step": 318
},
{
"KL/chosen_KL_mean": -292.07940673828125,
"KL/mean": -410.94036865234375,
"KL/rejected_KL_mean": -529.8013305664062,
"KL/std": 264.96746826171875,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0018263421952724457,
"fcm_dpo/delta": -0.01908506453037262,
"fcm_dpo/margin": 237.72190856933594,
"fcm_dpo/q_t": 0.3996313810348511,
"grad_norm": 24.182884216308594,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.38866111636161804,
"logits/rejected": -0.40275949239730835,
"logps/chosen": -341.4631652832031,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -643.7078247070312,
"loss": 1.0623,
"margin_dpo/margin_mean": 237.72190856933594,
"margin_dpo/margin_std": 287.574951171875,
"step": 319
},
{
"KL/chosen_KL_mean": -308.5746765136719,
"KL/mean": -437.24224853515625,
"KL/rejected_KL_mean": -565.9098510742188,
"KL/std": 273.13323974609375,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.0017987672472372651,
"fcm_dpo/delta": -0.06651625782251358,
"fcm_dpo/margin": 257.33514404296875,
"fcm_dpo/q_t": 0.39359739422798157,
"grad_norm": 25.218090057373047,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.4375894367694855,
"logits/rejected": -0.444126695394516,
"logps/chosen": -368.0795593261719,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -663.5770263671875,
"loss": 1.0414,
"margin_dpo/margin_mean": 257.33514404296875,
"margin_dpo/margin_std": 295.08465576171875,
"step": 320
},
{
"KL/chosen_KL_mean": -382.15399169921875,
"KL/mean": -493.6779479980469,
"KL/rejected_KL_mean": -605.201904296875,
"KL/std": 284.0462646484375,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.0017797566251829267,
"fcm_dpo/delta": -0.053124669939279556,
"fcm_dpo/margin": 223.04794311523438,
"fcm_dpo/q_t": 0.4093397259712219,
"grad_norm": 23.08179473876953,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.34922099113464355,
"logits/rejected": -0.3383622467517853,
"logps/chosen": -443.70263671875,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -696.8428955078125,
"loss": 1.0986,
"margin_dpo/margin_mean": 223.04794311523438,
"margin_dpo/margin_std": 303.15924072265625,
"step": 321
},
{
"KL/chosen_KL_mean": -333.64361572265625,
"KL/mean": -453.05535888671875,
"KL/rejected_KL_mean": -572.467041015625,
"KL/std": 251.202392578125,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.001764493528753519,
"fcm_dpo/delta": -0.04306449741125107,
"fcm_dpo/margin": 238.8234100341797,
"fcm_dpo/q_t": 0.4015629291534424,
"grad_norm": 24.869430541992188,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.3516596555709839,
"logits/rejected": -0.3475235104560852,
"logps/chosen": -390.93365478515625,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -668.2169799804688,
"loss": 1.0646,
"margin_dpo/margin_mean": 238.82342529296875,
"margin_dpo/margin_std": 272.7815246582031,
"step": 322
},
{
"KL/chosen_KL_mean": -347.6518249511719,
"KL/mean": -450.37103271484375,
"KL/rejected_KL_mean": -553.0902099609375,
"KL/std": 251.44210815429688,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.001764493528753519,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 205.43838500976562,
"fcm_dpo/q_t": 0.4151211678981781,
"grad_norm": 26.678951263427734,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.39251190423965454,
"logits/rejected": -0.39162832498550415,
"logps/chosen": -398.88580322265625,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -628.152099609375,
"loss": 1.1155,
"margin_dpo/margin_mean": 205.43838500976562,
"margin_dpo/margin_std": 275.69659423828125,
"step": 323
},
{
"KL/chosen_KL_mean": -400.1598205566406,
"KL/mean": -495.57952880859375,
"KL/rejected_KL_mean": -590.9992065429688,
"KL/std": 267.0489501953125,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.0017571839271113276,
"fcm_dpo/delta": -0.04159846156835556,
"fcm_dpo/margin": 190.83935546875,
"fcm_dpo/q_t": 0.42405766248703003,
"grad_norm": 43.70282745361328,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.41459232568740845,
"logits/rejected": -0.3984828293323517,
"logps/chosen": -465.29498291015625,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -677.4766845703125,
"loss": 1.1581,
"margin_dpo/margin_mean": 190.83935546875,
"margin_dpo/margin_std": 312.6383056640625,
"step": 324
},
{
"KL/chosen_KL_mean": -312.2501525878906,
"KL/mean": -392.975341796875,
"KL/rejected_KL_mean": -473.7005615234375,
"KL/std": 232.5697021484375,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.0017498743254691362,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 161.45037841796875,
"fcm_dpo/q_t": 0.43325573205947876,
"grad_norm": 29.16065216064453,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.43759685754776,
"logits/rejected": -0.4164847731590271,
"logps/chosen": -368.46575927734375,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -543.7864990234375,
"loss": 1.1674,
"margin_dpo/margin_mean": 161.45037841796875,
"margin_dpo/margin_std": 235.22097778320312,
"step": 325
},
{
"KL/chosen_KL_mean": -339.22479248046875,
"KL/mean": -419.08111572265625,
"KL/rejected_KL_mean": -498.9373779296875,
"KL/std": 221.22219848632812,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0017498743254691362,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 159.71255493164062,
"fcm_dpo/q_t": 0.4328235685825348,
"grad_norm": 41.599281311035156,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.4632147550582886,
"logits/rejected": -0.44045084714889526,
"logps/chosen": -411.94976806640625,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -578.7841796875,
"loss": 1.16,
"margin_dpo/margin_mean": 159.71255493164062,
"margin_dpo/margin_std": 209.68670654296875,
"step": 326
},
{
"KL/chosen_KL_mean": -285.94122314453125,
"KL/mean": -405.94384765625,
"KL/rejected_KL_mean": -525.9464111328125,
"KL/std": 225.80906677246094,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.0017384198727086186,
"fcm_dpo/delta": -0.032836832106113434,
"fcm_dpo/margin": 240.0051727294922,
"fcm_dpo/q_t": 0.40102726221084595,
"grad_norm": 37.66844177246094,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.4289267659187317,
"logits/rejected": -0.4173303246498108,
"logps/chosen": -355.07568359375,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -637.8801879882812,
"loss": 1.0585,
"margin_dpo/margin_mean": 240.0051727294922,
"margin_dpo/margin_std": 252.79605102539062,
"step": 327
},
{
"KL/chosen_KL_mean": -292.0697021484375,
"KL/mean": -397.9315185546875,
"KL/rejected_KL_mean": -503.7932434082031,
"KL/std": 248.23992919921875,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0017384198727086186,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 211.72357177734375,
"fcm_dpo/q_t": 0.41479289531707764,
"grad_norm": 32.22370147705078,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.39838314056396484,
"logits/rejected": -0.4005652070045471,
"logps/chosen": -351.75689697265625,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -594.6482543945312,
"loss": 1.1064,
"margin_dpo/margin_mean": 211.7235870361328,
"margin_dpo/margin_std": 279.32781982421875,
"step": 328
},
{
"KL/chosen_KL_mean": -312.3409118652344,
"KL/mean": -422.5838623046875,
"KL/rejected_KL_mean": -532.8267822265625,
"KL/std": 251.46182250976562,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0017300716135650873,
"fcm_dpo/delta": -0.024068888276815414,
"fcm_dpo/margin": 220.48590087890625,
"fcm_dpo/q_t": 0.4105890095233917,
"grad_norm": 22.827281951904297,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.4284207224845886,
"logits/rejected": -0.42305588722229004,
"logps/chosen": -377.58709716796875,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -633.5245361328125,
"loss": 1.0963,
"margin_dpo/margin_mean": 220.4858856201172,
"margin_dpo/margin_std": 273.05218505859375,
"step": 329
},
{
"KL/chosen_KL_mean": -261.9295654296875,
"KL/mean": -373.082763671875,
"KL/rejected_KL_mean": -484.2359619140625,
"KL/std": 231.3890380859375,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.0017300716135650873,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 222.30641174316406,
"fcm_dpo/q_t": 0.40928471088409424,
"grad_norm": 28.69952964782715,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.39853858947753906,
"logits/rejected": -0.40001553297042847,
"logps/chosen": -308.92791748046875,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -571.11279296875,
"loss": 1.0831,
"margin_dpo/margin_mean": 222.30641174316406,
"margin_dpo/margin_std": 250.98162841796875,
"step": 330
},
{
"KL/chosen_KL_mean": -287.3751220703125,
"KL/mean": -403.163330078125,
"KL/rejected_KL_mean": -518.9515380859375,
"KL/std": 203.0115203857422,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.0017300716135650873,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 231.57643127441406,
"fcm_dpo/q_t": 0.40439319610595703,
"grad_norm": 26.881025314331055,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.4044077396392822,
"logits/rejected": -0.403422474861145,
"logps/chosen": -337.89935302734375,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -607.9669799804688,
"loss": 1.0529,
"margin_dpo/margin_mean": 231.57643127441406,
"margin_dpo/margin_std": 198.66049194335938,
"step": 331
},
{
"KL/chosen_KL_mean": -285.37774658203125,
"KL/mean": -369.9641418457031,
"KL/rejected_KL_mean": -454.550537109375,
"KL/std": 204.74688720703125,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0017300716135650873,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 169.17282104492188,
"fcm_dpo/q_t": 0.43071043491363525,
"grad_norm": 19.305227279663086,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.41311419010162354,
"logits/rejected": -0.3951377272605896,
"logps/chosen": -334.5580139160156,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -531.03564453125,
"loss": 1.1515,
"margin_dpo/margin_mean": 169.17282104492188,
"margin_dpo/margin_std": 226.8921356201172,
"step": 332
},
{
"KL/chosen_KL_mean": -330.1800537109375,
"KL/mean": -439.9482116699219,
"KL/rejected_KL_mean": -549.7164306640625,
"KL/std": 246.501708984375,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.0017133665969595313,
"fcm_dpo/delta": -0.048513028770685196,
"fcm_dpo/margin": 219.5363311767578,
"fcm_dpo/q_t": 0.4124882221221924,
"grad_norm": 21.121355056762695,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.4535391628742218,
"logits/rejected": -0.45026063919067383,
"logps/chosen": -393.935791015625,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -644.760498046875,
"loss": 1.1113,
"margin_dpo/margin_mean": 219.53634643554688,
"margin_dpo/margin_std": 301.273193359375,
"step": 333
},
{
"KL/chosen_KL_mean": -326.6143798828125,
"KL/mean": -427.30743408203125,
"KL/rejected_KL_mean": -528.00048828125,
"KL/std": 280.7286071777344,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.0017133665969595313,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 201.3861541748047,
"fcm_dpo/q_t": 0.42041999101638794,
"grad_norm": 20.316781997680664,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.44451385736465454,
"logits/rejected": -0.44493532180786133,
"logps/chosen": -393.5941162109375,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -623.3174438476562,
"loss": 1.1404,
"margin_dpo/margin_mean": 201.38613891601562,
"margin_dpo/margin_std": 315.01531982421875,
"step": 334
},
{
"KL/chosen_KL_mean": -355.42633056640625,
"KL/mean": -433.4613037109375,
"KL/rejected_KL_mean": -511.4962158203125,
"KL/std": 259.311279296875,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.0017133665969595313,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 156.06985473632812,
"fcm_dpo/q_t": 0.4383538067340851,
"grad_norm": 35.47575378417969,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.5016331672668457,
"logits/rejected": -0.4807831048965454,
"logps/chosen": -417.96881103515625,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -599.1138916015625,
"loss": 1.1951,
"margin_dpo/margin_mean": 156.06985473632812,
"margin_dpo/margin_std": 283.9935302734375,
"step": 335
},
{
"KL/chosen_KL_mean": -360.68878173828125,
"KL/mean": -491.305419921875,
"KL/rejected_KL_mean": -621.9219970703125,
"KL/std": 304.59197998046875,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0017038836376741529,
"fcm_dpo/delta": -0.05565565079450607,
"fcm_dpo/margin": 261.233154296875,
"fcm_dpo/q_t": 0.3995817005634308,
"grad_norm": 31.521181106567383,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.437822163105011,
"logits/rejected": -0.4387591481208801,
"logps/chosen": -415.2199401855469,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -726.3262329101562,
"loss": 1.0782,
"margin_dpo/margin_mean": 261.233154296875,
"margin_dpo/margin_std": 359.9222412109375,
"step": 336
},
{
"KL/chosen_KL_mean": -337.1014709472656,
"KL/mean": -464.4716796875,
"KL/rejected_KL_mean": -591.8418579101562,
"KL/std": 251.95245361328125,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.0016836783615872264,
"fcm_dpo/delta": -0.03174104169011116,
"fcm_dpo/margin": 254.7404327392578,
"fcm_dpo/q_t": 0.40087810158729553,
"grad_norm": 24.891990661621094,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.47642776370048523,
"logits/rejected": -0.4633823037147522,
"logps/chosen": -402.23016357421875,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -693.56884765625,
"loss": 1.0609,
"margin_dpo/margin_mean": 254.7404327392578,
"margin_dpo/margin_std": 297.6883544921875,
"step": 337
},
{
"KL/chosen_KL_mean": -304.122802734375,
"KL/mean": -430.45965576171875,
"KL/rejected_KL_mean": -556.7965087890625,
"KL/std": 246.09283447265625,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0016771751688793302,
"fcm_dpo/delta": -0.0193497184664011,
"fcm_dpo/margin": 252.67369079589844,
"fcm_dpo/q_t": 0.4003807306289673,
"grad_norm": 23.349788665771484,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.46150559186935425,
"logits/rejected": -0.4557410180568695,
"logps/chosen": -362.5455322265625,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -645.8651123046875,
"loss": 1.0515,
"margin_dpo/margin_mean": 252.6737060546875,
"margin_dpo/margin_std": 260.1355895996094,
"step": 338
},
{
"KL/chosen_KL_mean": -338.9353942871094,
"KL/mean": -461.8382263183594,
"KL/rejected_KL_mean": -584.7410888671875,
"KL/std": 261.80877685546875,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0016616201028227806,
"fcm_dpo/delta": -0.09361618757247925,
"fcm_dpo/margin": 245.80567932128906,
"fcm_dpo/q_t": 0.40382570028305054,
"grad_norm": 25.955692291259766,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.5029301047325134,
"logits/rejected": -0.49164479970932007,
"logps/chosen": -398.93072509765625,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -688.6519775390625,
"loss": 1.0729,
"margin_dpo/margin_mean": 245.8056640625,
"margin_dpo/margin_std": 265.2763366699219,
"step": 339
},
{
"KL/chosen_KL_mean": -354.7925720214844,
"KL/mean": -459.96978759765625,
"KL/rejected_KL_mean": -565.14697265625,
"KL/std": 254.24560546875,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0016460651531815529,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 210.3543701171875,
"fcm_dpo/q_t": 0.4198751151561737,
"grad_norm": 25.57111358642578,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.43487322330474854,
"logits/rejected": -0.41735124588012695,
"logps/chosen": -407.622802734375,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -638.2542114257812,
"loss": 1.1323,
"margin_dpo/margin_mean": 210.3543701171875,
"margin_dpo/margin_std": 311.27935791015625,
"step": 340
},
{
"KL/chosen_KL_mean": -356.2671203613281,
"KL/mean": -481.8756103515625,
"KL/rejected_KL_mean": -607.4840087890625,
"KL/std": 277.663330078125,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.0016328769270330667,
"fcm_dpo/delta": -0.04022127762436867,
"fcm_dpo/margin": 251.21697998046875,
"fcm_dpo/q_t": 0.4051659405231476,
"grad_norm": 23.709854125976562,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.44025927782058716,
"logits/rejected": -0.4373493194580078,
"logps/chosen": -404.16693115234375,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -709.2939453125,
"loss": 1.0766,
"margin_dpo/margin_mean": 251.2169647216797,
"margin_dpo/margin_std": 301.60986328125,
"step": 341
},
{
"KL/chosen_KL_mean": -355.93658447265625,
"KL/mean": -460.60601806640625,
"KL/rejected_KL_mean": -565.2755126953125,
"KL/std": 265.940673828125,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.0016328769270330667,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 209.3389129638672,
"fcm_dpo/q_t": 0.42056310176849365,
"grad_norm": 25.819406509399414,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.46601247787475586,
"logits/rejected": -0.4378005266189575,
"logps/chosen": -427.9332275390625,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -657.8651123046875,
"loss": 1.1384,
"margin_dpo/margin_mean": 209.3389129638672,
"margin_dpo/margin_std": 320.1954345703125,
"step": 342
},
{
"KL/chosen_KL_mean": -332.80279541015625,
"KL/mean": -464.02789306640625,
"KL/rejected_KL_mean": -595.2529907226562,
"KL/std": 269.5361633300781,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.001626888057217002,
"fcm_dpo/delta": -0.018372224643826485,
"fcm_dpo/margin": 262.4502258300781,
"fcm_dpo/q_t": 0.39894723892211914,
"grad_norm": 25.481830596923828,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.4878358244895935,
"logits/rejected": -0.4912916421890259,
"logps/chosen": -387.20843505859375,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -706.29443359375,
"loss": 1.0437,
"margin_dpo/margin_mean": 262.4501953125,
"margin_dpo/margin_std": 245.56637573242188,
"step": 343
},
{
"KL/chosen_KL_mean": -328.3038024902344,
"KL/mean": -443.7353210449219,
"KL/rejected_KL_mean": -559.1668701171875,
"KL/std": 301.187744140625,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.0016134674660861492,
"fcm_dpo/delta": -0.0831800252199173,
"fcm_dpo/margin": 230.86305236816406,
"fcm_dpo/q_t": 0.4162444472312927,
"grad_norm": 28.6507625579834,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.45678555965423584,
"logits/rejected": -0.4595603346824646,
"logps/chosen": -382.2684631347656,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -649.7902221679688,
"loss": 1.1177,
"margin_dpo/margin_mean": 230.86305236816406,
"margin_dpo/margin_std": 319.19000244140625,
"step": 344
},
{
"KL/chosen_KL_mean": -378.3681945800781,
"KL/mean": -508.10675048828125,
"KL/rejected_KL_mean": -637.84521484375,
"KL/std": 275.95458984375,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.0015956300776451826,
"fcm_dpo/delta": -0.027681613340973854,
"fcm_dpo/margin": 259.47705078125,
"fcm_dpo/q_t": 0.4036673307418823,
"grad_norm": 21.58487319946289,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.48699015378952026,
"logits/rejected": -0.48181208968162537,
"logps/chosen": -440.05389404296875,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -737.335693359375,
"loss": 1.0708,
"margin_dpo/margin_mean": 259.47705078125,
"margin_dpo/margin_std": 307.22149658203125,
"step": 345
},
{
"KL/chosen_KL_mean": -373.90875244140625,
"KL/mean": -488.475830078125,
"KL/rejected_KL_mean": -603.04296875,
"KL/std": 266.25341796875,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.001591213047504425,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 229.13418579101562,
"fcm_dpo/q_t": 0.4135410785675049,
"grad_norm": 20.24309730529785,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.4244663417339325,
"logits/rejected": -0.4114094376564026,
"logps/chosen": -429.16497802734375,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -680.458251953125,
"loss": 1.1037,
"margin_dpo/margin_mean": 229.13418579101562,
"margin_dpo/margin_std": 285.3810729980469,
"step": 346
},
{
"KL/chosen_KL_mean": -370.2218017578125,
"KL/mean": -484.5078430175781,
"KL/rejected_KL_mean": -598.7938842773438,
"KL/std": 279.05316162109375,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.001591213047504425,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 228.57208251953125,
"fcm_dpo/q_t": 0.41585496068000793,
"grad_norm": 24.884403228759766,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.42082369327545166,
"logits/rejected": -0.4123762845993042,
"logps/chosen": -427.7880554199219,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -691.1489868164062,
"loss": 1.1164,
"margin_dpo/margin_mean": 228.5720672607422,
"margin_dpo/margin_std": 320.4414367675781,
"step": 347
},
{
"KL/chosen_KL_mean": -320.3187561035156,
"KL/mean": -418.7101135253906,
"KL/rejected_KL_mean": -517.1014404296875,
"KL/std": 240.5898895263672,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.001591213047504425,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 196.78271484375,
"fcm_dpo/q_t": 0.42725011706352234,
"grad_norm": 21.602191925048828,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.43160480260849,
"logits/rejected": -0.4293760657310486,
"logps/chosen": -376.636474609375,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -606.2398071289062,
"loss": 1.1474,
"margin_dpo/margin_mean": 196.78271484375,
"margin_dpo/margin_std": 281.14556884765625,
"step": 348
},
{
"KL/chosen_KL_mean": -299.2859802246094,
"KL/mean": -415.08184814453125,
"KL/rejected_KL_mean": -530.877685546875,
"KL/std": 265.25457763671875,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.001591213047504425,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 231.59170532226562,
"fcm_dpo/q_t": 0.41541868448257446,
"grad_norm": 19.77528190612793,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.44090381264686584,
"logits/rejected": -0.44044074416160583,
"logps/chosen": -357.3114929199219,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -628.3828125,
"loss": 1.115,
"margin_dpo/margin_mean": 231.59170532226562,
"margin_dpo/margin_std": 324.92840576171875,
"step": 349
},
{
"KL/chosen_KL_mean": -323.4007873535156,
"KL/mean": -432.3330078125,
"KL/rejected_KL_mean": -541.2652587890625,
"KL/std": 254.46958923339844,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.001591213047504425,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 217.86448669433594,
"fcm_dpo/q_t": 0.41904622316360474,
"grad_norm": 28.25132179260254,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.46974977850914,
"logits/rejected": -0.47410547733306885,
"logps/chosen": -387.73126220703125,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -631.1369018554688,
"loss": 1.1313,
"margin_dpo/margin_mean": 217.86448669433594,
"margin_dpo/margin_std": 318.74139404296875,
"step": 350
},
{
"KL/chosen_KL_mean": -299.21258544921875,
"KL/mean": -435.21917724609375,
"KL/rejected_KL_mean": -571.2257080078125,
"KL/std": 299.67193603515625,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0015760548412799835,
"fcm_dpo/delta": -0.04785942658782005,
"fcm_dpo/margin": 272.0131530761719,
"fcm_dpo/q_t": 0.4027807116508484,
"grad_norm": 22.712406158447266,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.46151453256607056,
"logits/rejected": -0.4617936313152313,
"logps/chosen": -359.88470458984375,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -672.7911376953125,
"loss": 1.0746,
"margin_dpo/margin_mean": 272.0131530761719,
"margin_dpo/margin_std": 353.1802673339844,
"step": 351
},
{
"KL/chosen_KL_mean": -352.48883056640625,
"KL/mean": -430.4334716796875,
"KL/rejected_KL_mean": -508.3780822753906,
"KL/std": 270.8203125,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.0015760548412799835,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.88925170898438,
"fcm_dpo/q_t": 0.4424957036972046,
"grad_norm": 27.769054412841797,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.43828731775283813,
"logits/rejected": -0.40562915802001953,
"logps/chosen": -423.43231201171875,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -585.02001953125,
"loss": 1.2111,
"margin_dpo/margin_mean": 155.88925170898438,
"margin_dpo/margin_std": 301.49072265625,
"step": 352
},
{
"KL/chosen_KL_mean": -329.95526123046875,
"KL/mean": -440.26611328125,
"KL/rejected_KL_mean": -550.5770263671875,
"KL/std": 270.30609130859375,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": -0.02639908902347088,
"fcm_dpo/margin": 220.6217498779297,
"fcm_dpo/q_t": 0.41987377405166626,
"grad_norm": 23.161598205566406,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.46952152252197266,
"logits/rejected": -0.4589802622795105,
"logps/chosen": -388.3505859375,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -630.9125366210938,
"loss": 1.1279,
"margin_dpo/margin_mean": 220.6217498779297,
"margin_dpo/margin_std": 308.440673828125,
"step": 353
},
{
"KL/chosen_KL_mean": -278.52569580078125,
"KL/mean": -397.5231628417969,
"KL/rejected_KL_mean": -516.5206298828125,
"KL/std": 254.86386108398438,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 237.9949493408203,
"fcm_dpo/q_t": 0.4116242527961731,
"grad_norm": 19.947385787963867,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.4535951018333435,
"logits/rejected": -0.44199591875076294,
"logps/chosen": -338.32867431640625,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -605.2781372070312,
"loss": 1.0851,
"margin_dpo/margin_mean": 237.99493408203125,
"margin_dpo/margin_std": 252.84063720703125,
"step": 354
},
{
"KL/chosen_KL_mean": -279.12908935546875,
"KL/mean": -396.08306884765625,
"KL/rejected_KL_mean": -513.037109375,
"KL/std": 246.86309814453125,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 233.90802001953125,
"fcm_dpo/q_t": 0.41276803612709045,
"grad_norm": 27.744285583496094,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.5026720762252808,
"logits/rejected": -0.4931907057762146,
"logps/chosen": -333.257568359375,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -595.4431762695312,
"loss": 1.0913,
"margin_dpo/margin_mean": 233.90802001953125,
"margin_dpo/margin_std": 254.10516357421875,
"step": 355
},
{
"KL/chosen_KL_mean": -336.5333251953125,
"KL/mean": -401.6588134765625,
"KL/rejected_KL_mean": -466.7843322753906,
"KL/std": 260.546142578125,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.2509765625,
"fcm_dpo/q_t": 0.45279616117477417,
"grad_norm": 25.384258270263672,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.44052648544311523,
"logits/rejected": -0.41894644498825073,
"logps/chosen": -401.2071533203125,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -542.68359375,
"loss": 1.254,
"margin_dpo/margin_mean": 130.2509765625,
"margin_dpo/margin_std": 317.67974853515625,
"step": 356
},
{
"KL/chosen_KL_mean": -309.42535400390625,
"KL/mean": -412.43572998046875,
"KL/rejected_KL_mean": -515.4461059570312,
"KL/std": 250.17401123046875,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 206.020751953125,
"fcm_dpo/q_t": 0.42332154512405396,
"grad_norm": 22.215511322021484,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.46077555418014526,
"logits/rejected": -0.44984954595565796,
"logps/chosen": -362.1511535644531,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -602.2872314453125,
"loss": 1.1306,
"margin_dpo/margin_mean": 206.020751953125,
"margin_dpo/margin_std": 262.2787170410156,
"step": 357
},
{
"KL/chosen_KL_mean": -280.0639953613281,
"KL/mean": -377.8428955078125,
"KL/rejected_KL_mean": -475.62176513671875,
"KL/std": 249.71923828125,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 195.55775451660156,
"fcm_dpo/q_t": 0.428341269493103,
"grad_norm": 23.287120819091797,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.46829819679260254,
"logits/rejected": -0.4512297511100769,
"logps/chosen": -343.2694396972656,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -563.9950561523438,
"loss": 1.1479,
"margin_dpo/margin_mean": 195.55775451660156,
"margin_dpo/margin_std": 277.779296875,
"step": 358
},
{
"KL/chosen_KL_mean": -324.8550109863281,
"KL/mean": -441.377685546875,
"KL/rejected_KL_mean": -557.900390625,
"KL/std": 250.80963134765625,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 233.04534912109375,
"fcm_dpo/q_t": 0.41488760709762573,
"grad_norm": 28.12652587890625,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.4374353289604187,
"logits/rejected": -0.4322439730167389,
"logps/chosen": -381.2252197265625,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -640.0740966796875,
"loss": 1.1074,
"margin_dpo/margin_mean": 233.04534912109375,
"margin_dpo/margin_std": 298.8861083984375,
"step": 359
},
{
"KL/chosen_KL_mean": -324.079345703125,
"KL/mean": -412.91864013671875,
"KL/rejected_KL_mean": -501.7579345703125,
"KL/std": 216.36245727539062,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 177.67855834960938,
"fcm_dpo/q_t": 0.432937353849411,
"grad_norm": 41.02315139770508,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.3901089131832123,
"logits/rejected": -0.3776787519454956,
"logps/chosen": -375.53973388671875,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -571.5968017578125,
"loss": 1.1544,
"margin_dpo/margin_mean": 177.67855834960938,
"margin_dpo/margin_std": 213.44635009765625,
"step": 360
},
{
"KL/chosen_KL_mean": -345.0618591308594,
"KL/mean": -432.0201110839844,
"KL/rejected_KL_mean": -518.9783935546875,
"KL/std": 251.68954467773438,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0015677554765716195,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 173.91651916503906,
"fcm_dpo/q_t": 0.4354216456413269,
"grad_norm": 32.919921875,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.407523512840271,
"logits/rejected": -0.4052034020423889,
"logps/chosen": -398.9313659667969,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -609.7476806640625,
"loss": 1.1766,
"margin_dpo/margin_mean": 173.91650390625,
"margin_dpo/margin_std": 272.8636474609375,
"step": 361
},
{
"KL/chosen_KL_mean": -295.00146484375,
"KL/mean": -441.1478271484375,
"KL/rejected_KL_mean": -587.294189453125,
"KL/std": 273.6327819824219,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.0015489872312173247,
"fcm_dpo/delta": -0.06021817773580551,
"fcm_dpo/margin": 292.292724609375,
"fcm_dpo/q_t": 0.3948058485984802,
"grad_norm": 20.981021881103516,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.4132572114467621,
"logits/rejected": -0.4234897494316101,
"logps/chosen": -353.6405334472656,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -692.8761596679688,
"loss": 1.0325,
"margin_dpo/margin_mean": 292.2927551269531,
"margin_dpo/margin_std": 293.92108154296875,
"step": 362
},
{
"KL/chosen_KL_mean": -274.0687255859375,
"KL/mean": -418.25189208984375,
"KL/rejected_KL_mean": -562.43505859375,
"KL/std": 264.7567138671875,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0015410305932164192,
"fcm_dpo/delta": -0.05163221433758736,
"fcm_dpo/margin": 288.3663024902344,
"fcm_dpo/q_t": 0.39612919092178345,
"grad_norm": 21.660552978515625,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.37580353021621704,
"logits/rejected": -0.3681963384151459,
"logps/chosen": -318.62713623046875,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -637.1300048828125,
"loss": 1.0369,
"margin_dpo/margin_mean": 288.3663024902344,
"margin_dpo/margin_std": 287.9776916503906,
"step": 363
},
{
"KL/chosen_KL_mean": -291.2659606933594,
"KL/mean": -400.46966552734375,
"KL/rejected_KL_mean": -509.6733703613281,
"KL/std": 254.54420471191406,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.0015330739552155137,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 218.40740966796875,
"fcm_dpo/q_t": 0.42098337411880493,
"grad_norm": 20.794410705566406,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.4048645496368408,
"logits/rejected": -0.41446197032928467,
"logps/chosen": -340.16058349609375,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -601.069091796875,
"loss": 1.128,
"margin_dpo/margin_mean": 218.4074249267578,
"margin_dpo/margin_std": 291.99188232421875,
"step": 364
},
{
"KL/chosen_KL_mean": -289.51519775390625,
"KL/mean": -402.83343505859375,
"KL/rejected_KL_mean": -516.1517333984375,
"KL/std": 266.38848876953125,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0015330739552155137,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 226.6365203857422,
"fcm_dpo/q_t": 0.418827623128891,
"grad_norm": 20.097055435180664,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.41409730911254883,
"logits/rejected": -0.4254269003868103,
"logps/chosen": -341.0079345703125,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -608.8533935546875,
"loss": 1.1112,
"margin_dpo/margin_mean": 226.6365203857422,
"margin_dpo/margin_std": 275.52093505859375,
"step": 365
},
{
"KL/chosen_KL_mean": -264.1698303222656,
"KL/mean": -380.6763000488281,
"KL/rejected_KL_mean": -497.1827392578125,
"KL/std": 253.22549438476562,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.0015279599465429783,
"fcm_dpo/delta": -0.016706913709640503,
"fcm_dpo/margin": 233.01290893554688,
"fcm_dpo/q_t": 0.4166252017021179,
"grad_norm": 18.73899269104004,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.39603766798973083,
"logits/rejected": -0.4109054207801819,
"logps/chosen": -308.8904113769531,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -580.4931640625,
"loss": 1.1126,
"margin_dpo/margin_mean": 233.01290893554688,
"margin_dpo/margin_std": 300.14300537109375,
"step": 366
},
{
"KL/chosen_KL_mean": -282.4344482421875,
"KL/mean": -374.6944885253906,
"KL/rejected_KL_mean": -466.95452880859375,
"KL/std": 224.01869201660156,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0015279599465429783,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 184.52005004882812,
"fcm_dpo/q_t": 0.43212103843688965,
"grad_norm": 19.241165161132812,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.40832167863845825,
"logits/rejected": -0.3911445736885071,
"logps/chosen": -340.8398742675781,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -543.705810546875,
"loss": 1.1573,
"margin_dpo/margin_mean": 184.52005004882812,
"margin_dpo/margin_std": 241.02235412597656,
"step": 367
},
{
"KL/chosen_KL_mean": -264.31964111328125,
"KL/mean": -416.4783630371094,
"KL/rejected_KL_mean": -568.6371459960938,
"KL/std": 257.91253662109375,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.0015139949973672628,
"fcm_dpo/delta": -0.0636521428823471,
"fcm_dpo/margin": 304.3175048828125,
"fcm_dpo/q_t": 0.38951247930526733,
"grad_norm": 34.35197830200195,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.40232932567596436,
"logits/rejected": -0.41085970401763916,
"logps/chosen": -308.77215576171875,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -667.1923828125,
"loss": 1.0069,
"margin_dpo/margin_mean": 304.3175048828125,
"margin_dpo/margin_std": 230.89181518554688,
"step": 368
},
{
"KL/chosen_KL_mean": -361.3604736328125,
"KL/mean": -439.23187255859375,
"KL/rejected_KL_mean": -517.103271484375,
"KL/std": 264.01806640625,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.0015086316270753741,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.74282836914062,
"fcm_dpo/q_t": 0.44402188062667847,
"grad_norm": 23.49022674560547,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.43115919828414917,
"logits/rejected": -0.4122559428215027,
"logps/chosen": -432.741943359375,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -608.3991088867188,
"loss": 1.2154,
"margin_dpo/margin_mean": 155.74282836914062,
"margin_dpo/margin_std": 302.1126403808594,
"step": 369
},
{
"KL/chosen_KL_mean": -383.2568664550781,
"KL/mean": -458.2979736328125,
"KL/rejected_KL_mean": -533.339111328125,
"KL/std": 276.94122314453125,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.0015086316270753741,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 150.08221435546875,
"fcm_dpo/q_t": 0.4468126595020294,
"grad_norm": 24.77077865600586,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.451901376247406,
"logits/rejected": -0.4463951885700226,
"logps/chosen": -454.8643798828125,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -630.598876953125,
"loss": 1.227,
"margin_dpo/margin_mean": 150.0822296142578,
"margin_dpo/margin_std": 312.60546875,
"step": 370
},
{
"KL/chosen_KL_mean": -370.62091064453125,
"KL/mean": -483.0035400390625,
"KL/rejected_KL_mean": -595.3861083984375,
"KL/std": 281.2415466308594,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0015086316270753741,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 224.7652130126953,
"fcm_dpo/q_t": 0.4212295413017273,
"grad_norm": 24.414459228515625,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.49121659994125366,
"logits/rejected": -0.4816587269306183,
"logps/chosen": -440.035400390625,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -694.558349609375,
"loss": 1.1338,
"margin_dpo/margin_mean": 224.76519775390625,
"margin_dpo/margin_std": 325.4281005859375,
"step": 371
},
{
"KL/chosen_KL_mean": -364.821533203125,
"KL/mean": -496.4064636230469,
"KL/rejected_KL_mean": -627.991455078125,
"KL/std": 315.2689208984375,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.0015086316270753741,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 263.16986083984375,
"fcm_dpo/q_t": 0.40971818566322327,
"grad_norm": 20.666433334350586,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.4030493497848511,
"logits/rejected": -0.3827578127384186,
"logps/chosen": -426.6395263671875,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -706.5308837890625,
"loss": 1.0942,
"margin_dpo/margin_mean": 263.16986083984375,
"margin_dpo/margin_std": 357.79962158203125,
"step": 372
},
{
"KL/chosen_KL_mean": -407.0643310546875,
"KL/mean": -529.6451416015625,
"KL/rejected_KL_mean": -652.2260131835938,
"KL/std": 310.29119873046875,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.0015036920085549355,
"fcm_dpo/delta": -0.032850027084350586,
"fcm_dpo/margin": 245.16168212890625,
"fcm_dpo/q_t": 0.4128532111644745,
"grad_norm": 24.507394790649414,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.4359899163246155,
"logits/rejected": -0.4177612066268921,
"logps/chosen": -471.2814636230469,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -738.1856689453125,
"loss": 1.1025,
"margin_dpo/margin_mean": 245.16168212890625,
"margin_dpo/margin_std": 297.9869079589844,
"step": 373
},
{
"KL/chosen_KL_mean": -354.3941955566406,
"KL/mean": -471.44720458984375,
"KL/rejected_KL_mean": -588.500244140625,
"KL/std": 333.9794006347656,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.0014987523900344968,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 234.10601806640625,
"fcm_dpo/q_t": 0.4215013086795807,
"grad_norm": 20.826824188232422,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.4955642819404602,
"logits/rejected": -0.4787040948867798,
"logps/chosen": -425.04437255859375,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -682.140380859375,
"loss": 1.1402,
"margin_dpo/margin_mean": 234.10601806640625,
"margin_dpo/margin_std": 377.1463623046875,
"step": 374
},
{
"KL/chosen_KL_mean": -363.83551025390625,
"KL/mean": -474.2115173339844,
"KL/rejected_KL_mean": -584.5875244140625,
"KL/std": 268.56109619140625,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.0014939869288355112,
"fcm_dpo/delta": -0.015923552215099335,
"fcm_dpo/margin": 220.7519989013672,
"fcm_dpo/q_t": 0.4224032163619995,
"grad_norm": 23.4717960357666,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.4513305425643921,
"logits/rejected": -0.44845885038375854,
"logps/chosen": -423.9157409667969,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -673.5258178710938,
"loss": 1.1395,
"margin_dpo/margin_mean": 220.75198364257812,
"margin_dpo/margin_std": 318.7281494140625,
"step": 375
},
{
"KL/chosen_KL_mean": -352.83441162109375,
"KL/mean": -490.5364990234375,
"KL/rejected_KL_mean": -628.2386474609375,
"KL/std": 288.8502197265625,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0014939869288355112,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 275.4041748046875,
"fcm_dpo/q_t": 0.40431779623031616,
"grad_norm": 19.61762046813965,
"learning_rate": 2.5e-07,
"logits/chosen": -0.4351574778556824,
"logits/rejected": -0.4265507459640503,
"logps/chosen": -415.4947509765625,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -733.7651977539062,
"loss": 1.0763,
"margin_dpo/margin_mean": 275.4041748046875,
"margin_dpo/margin_std": 336.59124755859375,
"step": 376
},
{
"KL/chosen_KL_mean": -350.9661865234375,
"KL/mean": -482.7196960449219,
"KL/rejected_KL_mean": -614.4732666015625,
"KL/std": 300.648193359375,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.001491023926064372,
"fcm_dpo/delta": -0.019872425124049187,
"fcm_dpo/margin": 263.50701904296875,
"fcm_dpo/q_t": 0.4078996479511261,
"grad_norm": 24.318864822387695,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.4018658399581909,
"logits/rejected": -0.40109121799468994,
"logps/chosen": -405.4449157714844,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -713.1765747070312,
"loss": 1.0815,
"margin_dpo/margin_mean": 263.50701904296875,
"margin_dpo/margin_std": 306.6456298828125,
"step": 377
},
{
"KL/chosen_KL_mean": -330.1966552734375,
"KL/mean": -464.34942626953125,
"KL/rejected_KL_mean": -598.502197265625,
"KL/std": 271.9918212890625,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.001488060923293233,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 268.3055419921875,
"fcm_dpo/q_t": 0.4054878354072571,
"grad_norm": 25.818546295166016,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.4009881019592285,
"logits/rejected": -0.4145526885986328,
"logps/chosen": -375.2171630859375,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -686.5491943359375,
"loss": 1.0712,
"margin_dpo/margin_mean": 268.3055419921875,
"margin_dpo/margin_std": 292.1213684082031,
"step": 378
},
{
"KL/chosen_KL_mean": -372.35675048828125,
"KL/mean": -506.56787109375,
"KL/rejected_KL_mean": -640.779052734375,
"KL/std": 279.959228515625,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.0014759085606783628,
"fcm_dpo/delta": -0.0410006120800972,
"fcm_dpo/margin": 268.42230224609375,
"fcm_dpo/q_t": 0.40835413336753845,
"grad_norm": 25.43598175048828,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.3865029811859131,
"logits/rejected": -0.40706324577331543,
"logps/chosen": -425.538818359375,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -755.0792236328125,
"loss": 1.0911,
"margin_dpo/margin_mean": 268.42230224609375,
"margin_dpo/margin_std": 345.518798828125,
"step": 379
},
{
"KL/chosen_KL_mean": -355.4010314941406,
"KL/mean": -506.8250732421875,
"KL/rejected_KL_mean": -658.2490844726562,
"KL/std": 315.39031982421875,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.0014712773263454437,
"fcm_dpo/delta": -0.03147752955555916,
"fcm_dpo/margin": 302.84808349609375,
"fcm_dpo/q_t": 0.39569777250289917,
"grad_norm": 25.094398498535156,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.4250390827655792,
"logits/rejected": -0.4504019021987915,
"logps/chosen": -406.7540588378906,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -762.4407958984375,
"loss": 1.0505,
"margin_dpo/margin_mean": 302.84808349609375,
"margin_dpo/margin_std": 347.689453125,
"step": 380
},
{
"KL/chosen_KL_mean": -372.30035400390625,
"KL/mean": -475.10699462890625,
"KL/rejected_KL_mean": -577.91357421875,
"KL/std": 262.00640869140625,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 205.61322021484375,
"fcm_dpo/q_t": 0.4295688271522522,
"grad_norm": 22.824941635131836,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.4074001610279083,
"logits/rejected": -0.3904430866241455,
"logps/chosen": -430.1034240722656,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -657.1329956054688,
"loss": 1.1742,
"margin_dpo/margin_mean": 205.61322021484375,
"margin_dpo/margin_std": 360.230224609375,
"step": 381
},
{
"KL/chosen_KL_mean": -361.166015625,
"KL/mean": -490.76263427734375,
"KL/rejected_KL_mean": -620.3592529296875,
"KL/std": 250.95346069335938,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 259.1932067871094,
"fcm_dpo/q_t": 0.41025757789611816,
"grad_norm": 23.912845611572266,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.4556125998497009,
"logits/rejected": -0.46023356914520264,
"logps/chosen": -427.18634033203125,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -731.0693969726562,
"loss": 1.0818,
"margin_dpo/margin_mean": 259.1932067871094,
"margin_dpo/margin_std": 278.27044677734375,
"step": 382
},
{
"KL/chosen_KL_mean": -376.48089599609375,
"KL/mean": -501.99945068359375,
"KL/rejected_KL_mean": -627.5180053710938,
"KL/std": 281.06005859375,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 251.037109375,
"fcm_dpo/q_t": 0.41382551193237305,
"grad_norm": 27.376548767089844,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.42573854327201843,
"logits/rejected": -0.43815886974334717,
"logps/chosen": -426.87237548828125,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -721.23388671875,
"loss": 1.1202,
"margin_dpo/margin_mean": 251.037109375,
"margin_dpo/margin_std": 367.67462158203125,
"step": 383
},
{
"KL/chosen_KL_mean": -368.24322509765625,
"KL/mean": -472.5242004394531,
"KL/rejected_KL_mean": -576.8052368164062,
"KL/std": 268.2153015136719,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 208.56201171875,
"fcm_dpo/q_t": 0.4268280863761902,
"grad_norm": 20.0333309173584,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.4748450815677643,
"logits/rejected": -0.4683492183685303,
"logps/chosen": -420.289306640625,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -662.566162109375,
"loss": 1.1363,
"margin_dpo/margin_mean": 208.56201171875,
"margin_dpo/margin_std": 250.0782470703125,
"step": 384
},
{
"KL/chosen_KL_mean": -347.3446044921875,
"KL/mean": -467.85711669921875,
"KL/rejected_KL_mean": -588.3695678710938,
"KL/std": 225.39920043945312,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 241.02496337890625,
"fcm_dpo/q_t": 0.4153004288673401,
"grad_norm": 24.630815505981445,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.4682900905609131,
"logits/rejected": -0.4416571855545044,
"logps/chosen": -412.89678955078125,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -666.197509765625,
"loss": 1.0957,
"margin_dpo/margin_mean": 241.02496337890625,
"margin_dpo/margin_std": 247.9585418701172,
"step": 385
},
{
"KL/chosen_KL_mean": -364.7396240234375,
"KL/mean": -497.43408203125,
"KL/rejected_KL_mean": -630.1285400390625,
"KL/std": 291.9441833496094,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 265.3889465332031,
"fcm_dpo/q_t": 0.4097556471824646,
"grad_norm": 26.513629913330078,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.4592726230621338,
"logits/rejected": -0.4541982114315033,
"logps/chosen": -422.96148681640625,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -722.4559936523438,
"loss": 1.0928,
"margin_dpo/margin_mean": 265.38897705078125,
"margin_dpo/margin_std": 341.11126708984375,
"step": 386
},
{
"KL/chosen_KL_mean": -377.314453125,
"KL/mean": -486.5088195800781,
"KL/rejected_KL_mean": -595.703125,
"KL/std": 265.19122314453125,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.0014666462084278464,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 218.38868713378906,
"fcm_dpo/q_t": 0.4235340654850006,
"grad_norm": 28.172088623046875,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.4886634945869446,
"logits/rejected": -0.46841585636138916,
"logps/chosen": -443.7339172363281,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -687.872314453125,
"loss": 1.1308,
"margin_dpo/margin_mean": 218.388671875,
"margin_dpo/margin_std": 269.91070556640625,
"step": 387
},
{
"KL/chosen_KL_mean": -362.4182434082031,
"KL/mean": -510.37042236328125,
"KL/rejected_KL_mean": -658.3226318359375,
"KL/std": 306.993896484375,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.0014555505476891994,
"fcm_dpo/delta": -0.037970419973134995,
"fcm_dpo/margin": 295.90435791015625,
"fcm_dpo/q_t": 0.4009212851524353,
"grad_norm": 24.37848663330078,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.45971643924713135,
"logits/rejected": -0.4707677662372589,
"logps/chosen": -412.5476989746094,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -762.7556762695312,
"loss": 1.0598,
"margin_dpo/margin_mean": 295.90435791015625,
"margin_dpo/margin_std": 346.856201171875,
"step": 388
},
{
"KL/chosen_KL_mean": -401.6351318359375,
"KL/mean": -524.97998046875,
"KL/rejected_KL_mean": -648.3248291015625,
"KL/std": 304.2716064453125,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.0014555505476891994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 246.6896514892578,
"fcm_dpo/q_t": 0.416399747133255,
"grad_norm": 29.7141170501709,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.4501149654388428,
"logits/rejected": -0.4422289729118347,
"logps/chosen": -459.541748046875,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -726.2393798828125,
"loss": 1.1164,
"margin_dpo/margin_mean": 246.68966674804688,
"margin_dpo/margin_std": 338.0137939453125,
"step": 389
},
{
"KL/chosen_KL_mean": -388.4798278808594,
"KL/mean": -514.3911743164062,
"KL/rejected_KL_mean": -640.302490234375,
"KL/std": 296.8943786621094,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.0014555505476891994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 251.8226776123047,
"fcm_dpo/q_t": 0.4174543619155884,
"grad_norm": 22.752588272094727,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.42283350229263306,
"logits/rejected": -0.4163636565208435,
"logps/chosen": -437.70574951171875,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -725.8306884765625,
"loss": 1.1234,
"margin_dpo/margin_mean": 251.8226776123047,
"margin_dpo/margin_std": 379.67108154296875,
"step": 390
},
{
"KL/chosen_KL_mean": -398.8809814453125,
"KL/mean": -467.99664306640625,
"KL/rejected_KL_mean": -537.1123657226562,
"KL/std": 290.8458251953125,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0014555505476891994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 138.23133850097656,
"fcm_dpo/q_t": 0.45229679346084595,
"grad_norm": 38.93089294433594,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.44258245825767517,
"logits/rejected": -0.4322088956832886,
"logps/chosen": -463.21063232421875,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -623.8505859375,
"loss": 1.2447,
"margin_dpo/margin_mean": 138.23133850097656,
"margin_dpo/margin_std": 306.81280517578125,
"step": 391
},
{
"KL/chosen_KL_mean": -327.77587890625,
"KL/mean": -475.56134033203125,
"KL/rejected_KL_mean": -623.3468017578125,
"KL/std": 294.4037780761719,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0014487283769994974,
"fcm_dpo/delta": -0.047090742737054825,
"fcm_dpo/margin": 295.57098388671875,
"fcm_dpo/q_t": 0.39948129653930664,
"grad_norm": 26.698942184448242,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.44229134917259216,
"logits/rejected": -0.4364994168281555,
"logps/chosen": -381.27984619140625,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -725.692626953125,
"loss": 1.0466,
"margin_dpo/margin_mean": 295.57098388671875,
"margin_dpo/margin_std": 292.6898498535156,
"step": 392
},
{
"KL/chosen_KL_mean": -326.3807067871094,
"KL/mean": -437.4690246582031,
"KL/rejected_KL_mean": -548.557373046875,
"KL/std": 287.9085388183594,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0014419063227251172,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 222.1766357421875,
"fcm_dpo/q_t": 0.4272076487541199,
"grad_norm": 22.053794860839844,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.3985256254673004,
"logits/rejected": -0.395794153213501,
"logps/chosen": -372.8546142578125,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -620.5261840820312,
"loss": 1.1533,
"margin_dpo/margin_mean": 222.17662048339844,
"margin_dpo/margin_std": 362.2658996582031,
"step": 393
},
{
"KL/chosen_KL_mean": -371.293212890625,
"KL/mean": -487.92718505859375,
"KL/rejected_KL_mean": -604.5611572265625,
"KL/std": 307.582275390625,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0014419063227251172,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 233.26791381835938,
"fcm_dpo/q_t": 0.42159971594810486,
"grad_norm": 20.481840133666992,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.4049949645996094,
"logits/rejected": -0.40527600049972534,
"logps/chosen": -424.20477294921875,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -695.3837890625,
"loss": 1.1278,
"margin_dpo/margin_mean": 233.26791381835938,
"margin_dpo/margin_std": 318.6451416015625,
"step": 394
},
{
"KL/chosen_KL_mean": -358.0785827636719,
"KL/mean": -485.55322265625,
"KL/rejected_KL_mean": -613.0279541015625,
"KL/std": 292.2204284667969,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0014419063227251172,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 254.94932556152344,
"fcm_dpo/q_t": 0.4145504832267761,
"grad_norm": 21.921274185180664,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.4451723098754883,
"logits/rejected": -0.44253167510032654,
"logps/chosen": -420.62469482421875,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -696.810546875,
"loss": 1.1073,
"margin_dpo/margin_mean": 254.9493408203125,
"margin_dpo/margin_std": 334.0831298828125,
"step": 395
},
{
"KL/chosen_KL_mean": -358.3993835449219,
"KL/mean": -474.07427978515625,
"KL/rejected_KL_mean": -589.7491455078125,
"KL/std": 286.84027099609375,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.0014419063227251172,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 231.34983825683594,
"fcm_dpo/q_t": 0.42063820362091064,
"grad_norm": 19.1959285736084,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.4682362973690033,
"logits/rejected": -0.4516993761062622,
"logps/chosen": -427.39532470703125,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -678.3958740234375,
"loss": 1.1148,
"margin_dpo/margin_mean": 231.34982299804688,
"margin_dpo/margin_std": 264.4104919433594,
"step": 396
},
{
"KL/chosen_KL_mean": -338.35693359375,
"KL/mean": -482.5387268066406,
"KL/rejected_KL_mean": -626.7205810546875,
"KL/std": 283.781494140625,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.001432217308320105,
"fcm_dpo/delta": -0.03371122106909752,
"fcm_dpo/margin": 288.3636169433594,
"fcm_dpo/q_t": 0.40260809659957886,
"grad_norm": 33.34209060668945,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.388605535030365,
"logits/rejected": -0.3912005126476288,
"logps/chosen": -399.63409423828125,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -729.836669921875,
"loss": 1.0596,
"margin_dpo/margin_mean": 288.3636169433594,
"margin_dpo/margin_std": 301.2374572753906,
"step": 397
},
{
"KL/chosen_KL_mean": -358.54351806640625,
"KL/mean": -494.4063415527344,
"KL/rejected_KL_mean": -630.2691650390625,
"KL/std": 298.5025634765625,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.001432217308320105,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 271.7256164550781,
"fcm_dpo/q_t": 0.4102500081062317,
"grad_norm": 19.53269386291504,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.4787027835845947,
"logits/rejected": -0.47340676188468933,
"logps/chosen": -426.695068359375,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -738.792724609375,
"loss": 1.0924,
"margin_dpo/margin_mean": 271.7255859375,
"margin_dpo/margin_std": 350.1674499511719,
"step": 398
},
{
"KL/chosen_KL_mean": -310.72174072265625,
"KL/mean": -426.0609436035156,
"KL/rejected_KL_mean": -541.400146484375,
"KL/std": 257.1795349121094,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.001432217308320105,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 230.67837524414062,
"fcm_dpo/q_t": 0.42226526141166687,
"grad_norm": 22.05406379699707,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.43538278341293335,
"logits/rejected": -0.4008658528327942,
"logps/chosen": -371.611572265625,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -619.36572265625,
"loss": 1.1347,
"margin_dpo/margin_mean": 230.67835998535156,
"margin_dpo/margin_std": 322.523193359375,
"step": 399
},
{
"KL/chosen_KL_mean": -292.40625,
"KL/mean": -462.7926025390625,
"KL/rejected_KL_mean": -633.178955078125,
"KL/std": 276.2427062988281,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.0014116978272795677,
"fcm_dpo/delta": -0.0853746086359024,
"fcm_dpo/margin": 340.772705078125,
"fcm_dpo/q_t": 0.38592517375946045,
"grad_norm": 21.224273681640625,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.3986595571041107,
"logits/rejected": -0.39339679479599,
"logps/chosen": -356.0498046875,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -738.431640625,
"loss": 1.0063,
"margin_dpo/margin_mean": 340.772705078125,
"margin_dpo/margin_std": 307.47308349609375,
"step": 400
},
{
"KL/chosen_KL_mean": -346.3468322753906,
"KL/mean": -440.0452880859375,
"KL/rejected_KL_mean": -533.7437744140625,
"KL/std": 287.55645751953125,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0014079699758440256,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 187.39691162109375,
"fcm_dpo/q_t": 0.4391755163669586,
"grad_norm": 19.834428787231445,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.4189956784248352,
"logits/rejected": -0.41130340099334717,
"logps/chosen": -403.5098876953125,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -617.5362548828125,
"loss": 1.2105,
"margin_dpo/margin_mean": 187.39694213867188,
"margin_dpo/margin_std": 384.96673583984375,
"step": 401
},
{
"KL/chosen_KL_mean": -254.55259704589844,
"KL/mean": -429.1209716796875,
"KL/rejected_KL_mean": -603.6893310546875,
"KL/std": 311.1974182128906,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0013890372356399894,
"fcm_dpo/delta": -0.08931900560855865,
"fcm_dpo/margin": 349.13671875,
"fcm_dpo/q_t": 0.38594573736190796,
"grad_norm": 18.52750015258789,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.38897186517715454,
"logits/rejected": -0.3770354986190796,
"logps/chosen": -305.29296875,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -684.7353515625,
"loss": 1.0045,
"margin_dpo/margin_mean": 349.13671875,
"margin_dpo/margin_std": 321.05426025390625,
"step": 402
},
{
"KL/chosen_KL_mean": -277.5030822753906,
"KL/mean": -412.81787109375,
"KL/rejected_KL_mean": -548.1326293945312,
"KL/std": 296.6151123046875,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0013830416137352586,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 270.62957763671875,
"fcm_dpo/q_t": 0.4117102026939392,
"grad_norm": 22.302377700805664,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.37483614683151245,
"logits/rejected": -0.36954933404922485,
"logps/chosen": -324.650390625,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -625.3992919921875,
"loss": 1.0885,
"margin_dpo/margin_mean": 270.62957763671875,
"margin_dpo/margin_std": 300.69293212890625,
"step": 403
},
{
"KL/chosen_KL_mean": -307.9491882324219,
"KL/mean": -442.93231201171875,
"KL/rejected_KL_mean": -577.9154663085938,
"KL/std": 279.23492431640625,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0013830416137352586,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 269.96630859375,
"fcm_dpo/q_t": 0.412688672542572,
"grad_norm": 18.491226196289062,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.35618770122528076,
"logits/rejected": -0.3467029333114624,
"logps/chosen": -355.824462890625,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -655.0704345703125,
"loss": 1.1043,
"margin_dpo/margin_mean": 269.96630859375,
"margin_dpo/margin_std": 352.7061462402344,
"step": 404
},
{
"KL/chosen_KL_mean": -365.7634582519531,
"KL/mean": -475.2872619628906,
"KL/rejected_KL_mean": -584.81103515625,
"KL/std": 317.9582824707031,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.0013830416137352586,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 219.04754638671875,
"fcm_dpo/q_t": 0.43119341135025024,
"grad_norm": 26.632732391357422,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.34994158148765564,
"logits/rejected": -0.34071725606918335,
"logps/chosen": -430.9263610839844,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -671.997802734375,
"loss": 1.1815,
"margin_dpo/margin_mean": 219.04754638671875,
"margin_dpo/margin_std": 398.8152770996094,
"step": 405
},
{
"KL/chosen_KL_mean": -336.44659423828125,
"KL/mean": -477.87347412109375,
"KL/rejected_KL_mean": -619.3004150390625,
"KL/std": 311.3196716308594,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0013787212083116174,
"fcm_dpo/delta": -0.031336523592472076,
"fcm_dpo/margin": 282.85382080078125,
"fcm_dpo/q_t": 0.4096255302429199,
"grad_norm": 21.68229866027832,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.3854570984840393,
"logits/rejected": -0.389883816242218,
"logps/chosen": -386.1874084472656,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -711.3790283203125,
"loss": 1.0887,
"margin_dpo/margin_mean": 282.85382080078125,
"margin_dpo/margin_std": 349.2008361816406,
"step": 406
},
{
"KL/chosen_KL_mean": -359.400390625,
"KL/mean": -452.02783203125,
"KL/rejected_KL_mean": -544.6552734375,
"KL/std": 254.18994140625,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0013744008028879762,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 185.25485229492188,
"fcm_dpo/q_t": 0.439433753490448,
"grad_norm": 27.686962127685547,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.3911685347557068,
"logits/rejected": -0.3647308945655823,
"logps/chosen": -415.7311096191406,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -622.1673583984375,
"loss": 1.2016,
"margin_dpo/margin_mean": 185.25485229492188,
"margin_dpo/margin_std": 333.753662109375,
"step": 407
},
{
"KL/chosen_KL_mean": -372.42059326171875,
"KL/mean": -469.0271911621094,
"KL/rejected_KL_mean": -565.6337890625,
"KL/std": 247.71975708007812,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.0013744008028879762,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 193.213134765625,
"fcm_dpo/q_t": 0.43635261058807373,
"grad_norm": 23.493730545043945,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.40377092361450195,
"logits/rejected": -0.37954843044281006,
"logps/chosen": -442.20989990234375,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -655.730712890625,
"loss": 1.1722,
"margin_dpo/margin_mean": 193.21316528320312,
"margin_dpo/margin_std": 271.8493347167969,
"step": 408
},
{
"KL/chosen_KL_mean": -360.97454833984375,
"KL/mean": -456.0081787109375,
"KL/rejected_KL_mean": -551.041748046875,
"KL/std": 256.92303466796875,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0013744008028879762,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 190.06727600097656,
"fcm_dpo/q_t": 0.43794065713882446,
"grad_norm": 24.473705291748047,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.3902217745780945,
"logits/rejected": -0.3765072524547577,
"logps/chosen": -428.2919921875,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -635.946044921875,
"loss": 1.185,
"margin_dpo/margin_mean": 190.06727600097656,
"margin_dpo/margin_std": 304.51153564453125,
"step": 409
},
{
"KL/chosen_KL_mean": -328.8135986328125,
"KL/mean": -451.9912109375,
"KL/rejected_KL_mean": -575.1687622070312,
"KL/std": 273.52239990234375,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.0013744008028879762,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 246.3551025390625,
"fcm_dpo/q_t": 0.4192972481250763,
"grad_norm": 23.798898696899414,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.3959043622016907,
"logits/rejected": -0.394477903842926,
"logps/chosen": -380.27899169921875,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -658.3677368164062,
"loss": 1.1192,
"margin_dpo/margin_mean": 246.3551025390625,
"margin_dpo/margin_std": 308.258544921875,
"step": 410
},
{
"KL/chosen_KL_mean": -354.54156494140625,
"KL/mean": -472.2734680175781,
"KL/rejected_KL_mean": -590.00537109375,
"KL/std": 299.3707580566406,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0013673783978447318,
"fcm_dpo/delta": -0.025612497702240944,
"fcm_dpo/margin": 235.46383666992188,
"fcm_dpo/q_t": 0.42583271861076355,
"grad_norm": 23.410425186157227,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.3593885600566864,
"logits/rejected": -0.3671804666519165,
"logps/chosen": -406.84881591796875,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -670.7003173828125,
"loss": 1.1471,
"margin_dpo/margin_mean": 235.46383666992188,
"margin_dpo/margin_std": 350.5140686035156,
"step": 411
},
{
"KL/chosen_KL_mean": -356.0343017578125,
"KL/mean": -475.4249267578125,
"KL/rejected_KL_mean": -594.8155517578125,
"KL/std": 302.0730285644531,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0013673783978447318,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 238.78121948242188,
"fcm_dpo/q_t": 0.4237388074398041,
"grad_norm": 26.060949325561523,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.4340513348579407,
"logits/rejected": -0.4596172571182251,
"logps/chosen": -409.1784362792969,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -694.8763427734375,
"loss": 1.143,
"margin_dpo/margin_mean": 238.78121948242188,
"margin_dpo/margin_std": 358.81866455078125,
"step": 412
},
{
"KL/chosen_KL_mean": -380.462158203125,
"KL/mean": -503.0280456542969,
"KL/rejected_KL_mean": -625.5939331054688,
"KL/std": 281.5146484375,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.0013673783978447318,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 245.13180541992188,
"fcm_dpo/q_t": 0.4199550151824951,
"grad_norm": 21.529010772705078,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.4233962595462799,
"logits/rejected": -0.42641475796699524,
"logps/chosen": -442.04412841796875,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -725.0673828125,
"loss": 1.1257,
"margin_dpo/margin_mean": 245.13180541992188,
"margin_dpo/margin_std": 321.50811767578125,
"step": 413
},
{
"KL/chosen_KL_mean": -334.89654541015625,
"KL/mean": -471.7811279296875,
"KL/rejected_KL_mean": -608.6657104492188,
"KL/std": 274.99615478515625,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.001360746449790895,
"fcm_dpo/delta": -0.048737846314907074,
"fcm_dpo/margin": 273.7691650390625,
"fcm_dpo/q_t": 0.411694198846817,
"grad_norm": 25.56825065612793,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.3538694977760315,
"logits/rejected": -0.34508466720581055,
"logps/chosen": -381.52801513671875,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -696.312255859375,
"loss": 1.0954,
"margin_dpo/margin_mean": 273.7691650390625,
"margin_dpo/margin_std": 315.960205078125,
"step": 414
},
{
"KL/chosen_KL_mean": -381.809814453125,
"KL/mean": -484.69952392578125,
"KL/rejected_KL_mean": -587.5892333984375,
"KL/std": 283.8314208984375,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0013541145017370582,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 205.77943420410156,
"fcm_dpo/q_t": 0.4336971342563629,
"grad_norm": 24.777685165405273,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.4540392756462097,
"logits/rejected": -0.4499126672744751,
"logps/chosen": -460.4281005859375,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -688.0667724609375,
"loss": 1.1731,
"margin_dpo/margin_mean": 205.77943420410156,
"margin_dpo/margin_std": 321.399658203125,
"step": 415
},
{
"KL/chosen_KL_mean": -339.66680908203125,
"KL/mean": -497.25067138671875,
"KL/rejected_KL_mean": -654.83447265625,
"KL/std": 297.5770263671875,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0013482770882546902,
"fcm_dpo/delta": -0.04329550638794899,
"fcm_dpo/margin": 315.16766357421875,
"fcm_dpo/q_t": 0.39873379468917847,
"grad_norm": 23.55436897277832,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.47019827365875244,
"logits/rejected": -0.4745001792907715,
"logps/chosen": -397.9459533691406,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -745.4031982421875,
"loss": 1.0405,
"margin_dpo/margin_mean": 315.16766357421875,
"margin_dpo/margin_std": 279.1081237792969,
"step": 416
},
{
"KL/chosen_KL_mean": -337.95068359375,
"KL/mean": -468.36810302734375,
"KL/rejected_KL_mean": -598.7855834960938,
"KL/std": 279.1325378417969,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.0013424396747723222,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 260.8348388671875,
"fcm_dpo/q_t": 0.41633397340774536,
"grad_norm": 25.135456085205078,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.43549424409866333,
"logits/rejected": -0.4124263525009155,
"logps/chosen": -388.1494140625,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -666.9373779296875,
"loss": 1.0988,
"margin_dpo/margin_mean": 260.8348693847656,
"margin_dpo/margin_std": 273.3210144042969,
"step": 417
},
{
"KL/chosen_KL_mean": -365.82684326171875,
"KL/mean": -508.16168212890625,
"KL/rejected_KL_mean": -650.49658203125,
"KL/std": 315.8503112792969,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.0013350698864087462,
"fcm_dpo/delta": -0.05520197004079819,
"fcm_dpo/margin": 284.669677734375,
"fcm_dpo/q_t": 0.4123033881187439,
"grad_norm": 18.67411994934082,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.5395079851150513,
"logits/rejected": -0.5210288763046265,
"logps/chosen": -447.8052978515625,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -755.18798828125,
"loss": 1.1026,
"margin_dpo/margin_mean": 284.669677734375,
"margin_dpo/margin_std": 370.0165100097656,
"step": 418
},
{
"KL/chosen_KL_mean": -346.25262451171875,
"KL/mean": -497.578369140625,
"KL/rejected_KL_mean": -648.904052734375,
"KL/std": 279.85736083984375,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 302.6514587402344,
"fcm_dpo/q_t": 0.40497830510139465,
"grad_norm": 21.298076629638672,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.4583272933959961,
"logits/rejected": -0.4552687406539917,
"logps/chosen": -399.2012634277344,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -740.4871826171875,
"loss": 1.068,
"margin_dpo/margin_mean": 302.6514892578125,
"margin_dpo/margin_std": 322.56439208984375,
"step": 419
},
{
"KL/chosen_KL_mean": -457.07464599609375,
"KL/mean": -541.6314697265625,
"KL/rejected_KL_mean": -626.1882934570312,
"KL/std": 307.27313232421875,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 169.1136474609375,
"fcm_dpo/q_t": 0.4484458863735199,
"grad_norm": 47.015316009521484,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.49456986784935,
"logits/rejected": -0.45519959926605225,
"logps/chosen": -534.8446044921875,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -695.5081787109375,
"loss": 1.2491,
"margin_dpo/margin_mean": 169.11366271972656,
"margin_dpo/margin_std": 422.0369873046875,
"step": 420
},
{
"KL/chosen_KL_mean": -364.79150390625,
"KL/mean": -468.5289306640625,
"KL/rejected_KL_mean": -572.2662963867188,
"KL/std": 288.0106201171875,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 207.47483825683594,
"fcm_dpo/q_t": 0.43372586369514465,
"grad_norm": 20.53417205810547,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.5256436467170715,
"logits/rejected": -0.5119495391845703,
"logps/chosen": -418.557373046875,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -661.5477294921875,
"loss": 1.1738,
"margin_dpo/margin_mean": 207.47483825683594,
"margin_dpo/margin_std": 313.608642578125,
"step": 421
},
{
"KL/chosen_KL_mean": -423.76123046875,
"KL/mean": -540.0737915039062,
"KL/rejected_KL_mean": -656.3863525390625,
"KL/std": 297.48114013671875,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 232.6251220703125,
"fcm_dpo/q_t": 0.4267650246620178,
"grad_norm": 33.49336242675781,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.5396426320075989,
"logits/rejected": -0.538498044013977,
"logps/chosen": -492.39501953125,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -744.2498779296875,
"loss": 1.1543,
"margin_dpo/margin_mean": 232.6251220703125,
"margin_dpo/margin_std": 355.2101135253906,
"step": 422
},
{
"KL/chosen_KL_mean": -390.7974853515625,
"KL/mean": -527.0419311523438,
"KL/rejected_KL_mean": -663.286376953125,
"KL/std": 293.29345703125,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 272.4888916015625,
"fcm_dpo/q_t": 0.41438615322113037,
"grad_norm": 27.167430877685547,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.5402317047119141,
"logits/rejected": -0.5469857454299927,
"logps/chosen": -445.78778076171875,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -749.5929565429688,
"loss": 1.0977,
"margin_dpo/margin_mean": 272.4888916015625,
"margin_dpo/margin_std": 312.23480224609375,
"step": 423
},
{
"KL/chosen_KL_mean": -360.069580078125,
"KL/mean": -478.40814208984375,
"KL/rejected_KL_mean": -596.7467041015625,
"KL/std": 285.7010192871094,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 236.6771697998047,
"fcm_dpo/q_t": 0.42635974287986755,
"grad_norm": 26.857175827026367,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.5228101015090942,
"logits/rejected": -0.49360746145248413,
"logps/chosen": -416.08148193359375,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -663.2257080078125,
"loss": 1.1475,
"margin_dpo/margin_mean": 236.6771697998047,
"margin_dpo/margin_std": 346.0706481933594,
"step": 424
},
{
"KL/chosen_KL_mean": -377.1744079589844,
"KL/mean": -482.8753662109375,
"KL/rejected_KL_mean": -588.5763549804688,
"KL/std": 269.3935241699219,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 211.40194702148438,
"fcm_dpo/q_t": 0.4325962960720062,
"grad_norm": 26.28618812561035,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.565961480140686,
"logits/rejected": -0.5686656832695007,
"logps/chosen": -424.04339599609375,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -684.5018310546875,
"loss": 1.1705,
"margin_dpo/margin_mean": 211.40194702148438,
"margin_dpo/margin_std": 325.16778564453125,
"step": 425
},
{
"KL/chosen_KL_mean": -362.37982177734375,
"KL/mean": -483.747802734375,
"KL/rejected_KL_mean": -605.11572265625,
"KL/std": 271.43121337890625,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 242.7359161376953,
"fcm_dpo/q_t": 0.42305266857147217,
"grad_norm": 23.226341247558594,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.5489311814308167,
"logits/rejected": -0.5112883448600769,
"logps/chosen": -438.96337890625,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -686.38232421875,
"loss": 1.1309,
"margin_dpo/margin_mean": 242.7359161376953,
"margin_dpo/margin_std": 302.89013671875,
"step": 426
},
{
"KL/chosen_KL_mean": -354.0020446777344,
"KL/mean": -441.369140625,
"KL/rejected_KL_mean": -528.736328125,
"KL/std": 247.91751098632812,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 174.73423767089844,
"fcm_dpo/q_t": 0.4447828233242035,
"grad_norm": 20.944759368896484,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.5702467560768127,
"logits/rejected": -0.5489529967308044,
"logps/chosen": -418.85589599609375,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -607.3023071289062,
"loss": 1.2039,
"margin_dpo/margin_mean": 174.73423767089844,
"margin_dpo/margin_std": 289.51605224609375,
"step": 427
},
{
"KL/chosen_KL_mean": -424.8520202636719,
"KL/mean": -555.02294921875,
"KL/rejected_KL_mean": -685.19384765625,
"KL/std": 331.234130859375,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 260.3418273925781,
"fcm_dpo/q_t": 0.41999179124832153,
"grad_norm": 27.35421371459961,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.557364284992218,
"logits/rejected": -0.5577110648155212,
"logps/chosen": -487.48870849609375,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -788.4757080078125,
"loss": 1.137,
"margin_dpo/margin_mean": 260.3418273925781,
"margin_dpo/margin_std": 400.5129699707031,
"step": 428
},
{
"KL/chosen_KL_mean": -433.4362487792969,
"KL/mean": -528.7349853515625,
"KL/rejected_KL_mean": -624.03369140625,
"KL/std": 286.8631286621094,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 190.597412109375,
"fcm_dpo/q_t": 0.44049394130706787,
"grad_norm": 28.649442672729492,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.5428134202957153,
"logits/rejected": -0.5149238705635071,
"logps/chosen": -514.6702880859375,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -715.82861328125,
"loss": 1.2062,
"margin_dpo/margin_mean": 190.597412109375,
"margin_dpo/margin_std": 351.88348388671875,
"step": 429
},
{
"KL/chosen_KL_mean": -341.23309326171875,
"KL/mean": -457.59912109375,
"KL/rejected_KL_mean": -573.965087890625,
"KL/std": 271.2836608886719,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 232.73202514648438,
"fcm_dpo/q_t": 0.4272611141204834,
"grad_norm": 21.780847549438477,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.5611949563026428,
"logits/rejected": -0.5625407695770264,
"logps/chosen": -402.1534423828125,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -678.387939453125,
"loss": 1.14,
"margin_dpo/margin_mean": 232.73202514648438,
"margin_dpo/margin_std": 307.2655334472656,
"step": 430
},
{
"KL/chosen_KL_mean": -343.27020263671875,
"KL/mean": -497.19476318359375,
"KL/rejected_KL_mean": -651.1193237304688,
"KL/std": 300.940185546875,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0013277000980451703,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 307.84918212890625,
"fcm_dpo/q_t": 0.4039904475212097,
"grad_norm": 20.312416076660156,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.49830469489097595,
"logits/rejected": -0.4969218075275421,
"logps/chosen": -400.61895751953125,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -743.9595947265625,
"loss": 1.0662,
"margin_dpo/margin_mean": 307.8491516113281,
"margin_dpo/margin_std": 330.0069885253906,
"step": 431
},
{
"KL/chosen_KL_mean": -338.7581787109375,
"KL/mean": -483.9551086425781,
"KL/rejected_KL_mean": -629.152099609375,
"KL/std": 284.55670166015625,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.0013210558099672198,
"fcm_dpo/delta": -0.05029592290520668,
"fcm_dpo/margin": 290.3939208984375,
"fcm_dpo/q_t": 0.40896129608154297,
"grad_norm": 28.03597068786621,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.5391901135444641,
"logits/rejected": -0.5290787220001221,
"logps/chosen": -409.8329772949219,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -727.7316284179688,
"loss": 1.0785,
"margin_dpo/margin_mean": 290.3939208984375,
"margin_dpo/margin_std": 291.03668212890625,
"step": 432
},
{
"KL/chosen_KL_mean": -428.12890625,
"KL/mean": -531.45703125,
"KL/rejected_KL_mean": -634.7850341796875,
"KL/std": 282.63128662109375,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 206.6561737060547,
"fcm_dpo/q_t": 0.4347584843635559,
"grad_norm": 22.199871063232422,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.5286056995391846,
"logits/rejected": -0.514957845211029,
"logps/chosen": -486.402099609375,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -730.7359619140625,
"loss": 1.1798,
"margin_dpo/margin_mean": 206.65615844726562,
"margin_dpo/margin_std": 333.2256774902344,
"step": 433
},
{
"KL/chosen_KL_mean": -372.42156982421875,
"KL/mean": -489.6736145019531,
"KL/rejected_KL_mean": -606.9257202148438,
"KL/std": 296.7366027832031,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 234.50413513183594,
"fcm_dpo/q_t": 0.430846631526947,
"grad_norm": 20.755958557128906,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.5101474523544312,
"logits/rejected": -0.5049848556518555,
"logps/chosen": -434.395263671875,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -685.42431640625,
"loss": 1.1581,
"margin_dpo/margin_mean": 234.50413513183594,
"margin_dpo/margin_std": 380.45330810546875,
"step": 434
},
{
"KL/chosen_KL_mean": -333.2384033203125,
"KL/mean": -460.36224365234375,
"KL/rejected_KL_mean": -587.486083984375,
"KL/std": 281.7438659667969,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 254.24761962890625,
"fcm_dpo/q_t": 0.42064914107322693,
"grad_norm": 23.37683868408203,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.5346446633338928,
"logits/rejected": -0.5243590474128723,
"logps/chosen": -384.740478515625,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -675.052978515625,
"loss": 1.1226,
"margin_dpo/margin_mean": 254.24765014648438,
"margin_dpo/margin_std": 314.2124328613281,
"step": 435
},
{
"KL/chosen_KL_mean": -353.6714782714844,
"KL/mean": -460.28839111328125,
"KL/rejected_KL_mean": -566.9053344726562,
"KL/std": 247.24136352539062,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 213.23385620117188,
"fcm_dpo/q_t": 0.43298617005348206,
"grad_norm": 21.458030700683594,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.5430168509483337,
"logits/rejected": -0.5158405303955078,
"logps/chosen": -425.0751953125,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -649.633056640625,
"loss": 1.1615,
"margin_dpo/margin_mean": 213.23385620117188,
"margin_dpo/margin_std": 293.1329345703125,
"step": 436
},
{
"KL/chosen_KL_mean": -359.79595947265625,
"KL/mean": -469.1462707519531,
"KL/rejected_KL_mean": -578.49658203125,
"KL/std": 235.53416442871094,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 218.7006072998047,
"fcm_dpo/q_t": 0.431079626083374,
"grad_norm": 21.566225051879883,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.5344359874725342,
"logits/rejected": -0.5146011114120483,
"logps/chosen": -424.54022216796875,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -660.5401611328125,
"loss": 1.155,
"margin_dpo/margin_mean": 218.70062255859375,
"margin_dpo/margin_std": 291.8827209472656,
"step": 437
},
{
"KL/chosen_KL_mean": -366.44976806640625,
"KL/mean": -488.88330078125,
"KL/rejected_KL_mean": -611.3168334960938,
"KL/std": 269.77825927734375,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 244.86703491210938,
"fcm_dpo/q_t": 0.42326781153678894,
"grad_norm": 24.300569534301758,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.500368595123291,
"logits/rejected": -0.48205146193504333,
"logps/chosen": -425.46844482421875,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -694.3936767578125,
"loss": 1.1274,
"margin_dpo/margin_mean": 244.86703491210938,
"margin_dpo/margin_std": 296.9903564453125,
"step": 438
},
{
"KL/chosen_KL_mean": -368.8246765136719,
"KL/mean": -475.66510009765625,
"KL/rejected_KL_mean": -582.5054931640625,
"KL/std": 270.74139404296875,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 213.68080139160156,
"fcm_dpo/q_t": 0.43394535779953003,
"grad_norm": 23.145774841308594,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.516400933265686,
"logits/rejected": -0.5202809572219849,
"logps/chosen": -422.6087646484375,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -666.490966796875,
"loss": 1.1686,
"margin_dpo/margin_mean": 213.6807861328125,
"margin_dpo/margin_std": 307.72296142578125,
"step": 439
},
{
"KL/chosen_KL_mean": -393.8909912109375,
"KL/mean": -513.2696533203125,
"KL/rejected_KL_mean": -632.6483764648438,
"KL/std": 314.48468017578125,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 238.75735473632812,
"fcm_dpo/q_t": 0.4269280433654785,
"grad_norm": 29.134511947631836,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.5639290809631348,
"logits/rejected": -0.5608119964599609,
"logps/chosen": -472.4577331542969,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -729.1461181640625,
"loss": 1.1428,
"margin_dpo/margin_mean": 238.75735473632812,
"margin_dpo/margin_std": 335.62493896484375,
"step": 440
},
{
"KL/chosen_KL_mean": -423.23974609375,
"KL/mean": -543.760986328125,
"KL/rejected_KL_mean": -664.2822265625,
"KL/std": 322.83819580078125,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0013144115218892694,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 241.04251098632812,
"fcm_dpo/q_t": 0.4266929030418396,
"grad_norm": 26.923349380493164,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.5657366514205933,
"logits/rejected": -0.5528466701507568,
"logps/chosen": -484.064208984375,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -760.7530517578125,
"loss": 1.159,
"margin_dpo/margin_mean": 241.04249572753906,
"margin_dpo/margin_std": 389.2134704589844,
"step": 441
},
{
"KL/chosen_KL_mean": -364.3332824707031,
"KL/mean": -503.16485595703125,
"KL/rejected_KL_mean": -641.9964599609375,
"KL/std": 306.99761962890625,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.0013098118361085653,
"fcm_dpo/delta": -0.03511664643883705,
"fcm_dpo/margin": 277.6631164550781,
"fcm_dpo/q_t": 0.41515475511550903,
"grad_norm": 21.374515533447266,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.5080809593200684,
"logits/rejected": -0.49998366832733154,
"logps/chosen": -411.344482421875,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -718.53564453125,
"loss": 1.1045,
"margin_dpo/margin_mean": 277.6631164550781,
"margin_dpo/margin_std": 339.3021240234375,
"step": 442
},
{
"KL/chosen_KL_mean": -419.7547302246094,
"KL/mean": -508.86871337890625,
"KL/rejected_KL_mean": -597.982666015625,
"KL/std": 292.1375427246094,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0013052122667431831,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.22793579101562,
"fcm_dpo/q_t": 0.4458271265029907,
"grad_norm": 28.088706970214844,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.5053350329399109,
"logits/rejected": -0.4879445731639862,
"logps/chosen": -491.02777099609375,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -684.66259765625,
"loss": 1.2247,
"margin_dpo/margin_mean": 178.22792053222656,
"margin_dpo/margin_std": 371.9207458496094,
"step": 443
},
{
"KL/chosen_KL_mean": -403.64312744140625,
"KL/mean": -544.897705078125,
"KL/rejected_KL_mean": -686.1524047851562,
"KL/std": 324.3928527832031,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0013052122667431831,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 282.50927734375,
"fcm_dpo/q_t": 0.4141772389411926,
"grad_norm": 25.328685760498047,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.4659857749938965,
"logits/rejected": -0.46927428245544434,
"logps/chosen": -460.8568115234375,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -783.4072875976562,
"loss": 1.1037,
"margin_dpo/margin_mean": 282.5092468261719,
"margin_dpo/margin_std": 361.67974853515625,
"step": 444
},
{
"KL/chosen_KL_mean": -367.4541015625,
"KL/mean": -502.7689208984375,
"KL/rejected_KL_mean": -638.083740234375,
"KL/std": 279.38037109375,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0013052122667431831,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 270.629638671875,
"fcm_dpo/q_t": 0.4174720048904419,
"grad_norm": 22.808685302734375,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.48408472537994385,
"logits/rejected": -0.47305405139923096,
"logps/chosen": -434.75390625,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -730.7664794921875,
"loss": 1.1139,
"margin_dpo/margin_mean": 270.6296691894531,
"margin_dpo/margin_std": 345.13861083984375,
"step": 445
},
{
"KL/chosen_KL_mean": -335.86492919921875,
"KL/mean": -480.3028869628906,
"KL/rejected_KL_mean": -624.7408447265625,
"KL/std": 285.0141296386719,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.0013007477391511202,
"fcm_dpo/delta": -0.03432300314307213,
"fcm_dpo/margin": 288.8758544921875,
"fcm_dpo/q_t": 0.4105881452560425,
"grad_norm": 23.083541870117188,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.5223360061645508,
"logits/rejected": -0.5316995978355408,
"logps/chosen": -394.96343994140625,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -726.0050048828125,
"loss": 1.0862,
"margin_dpo/margin_mean": 288.8758544921875,
"margin_dpo/margin_std": 308.46868896484375,
"step": 446
},
{
"KL/chosen_KL_mean": -355.49981689453125,
"KL/mean": -508.11871337890625,
"KL/rejected_KL_mean": -660.7376098632812,
"KL/std": 347.4691467285156,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.001289354171603918,
"fcm_dpo/delta": -0.026798272505402565,
"fcm_dpo/margin": 305.23779296875,
"fcm_dpo/q_t": 0.41011327505111694,
"grad_norm": 29.463502883911133,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.5696920156478882,
"logits/rejected": -0.5802756547927856,
"logps/chosen": -411.57513427734375,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -759.432373046875,
"loss": 1.0888,
"margin_dpo/margin_mean": 305.23779296875,
"margin_dpo/margin_std": 392.8905029296875,
"step": 447
},
{
"KL/chosen_KL_mean": -386.01611328125,
"KL/mean": -539.498046875,
"KL/rejected_KL_mean": -692.97998046875,
"KL/std": 296.15045166015625,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.0012851132778450847,
"fcm_dpo/delta": -0.033000439405441284,
"fcm_dpo/margin": 306.9638671875,
"fcm_dpo/q_t": 0.4075472354888916,
"grad_norm": 39.277767181396484,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.4916926622390747,
"logits/rejected": -0.4980274438858032,
"logps/chosen": -446.01995849609375,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -795.24462890625,
"loss": 1.0759,
"margin_dpo/margin_mean": 306.9638671875,
"margin_dpo/margin_std": 335.7113037109375,
"step": 448
},
{
"KL/chosen_KL_mean": -411.36688232421875,
"KL/mean": -551.0133666992188,
"KL/rejected_KL_mean": -690.6597900390625,
"KL/std": 341.72772216796875,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.0012808723840862513,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 279.2928466796875,
"fcm_dpo/q_t": 0.4179277718067169,
"grad_norm": 20.590986251831055,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.5620814561843872,
"logits/rejected": -0.5571545362472534,
"logps/chosen": -478.8935241699219,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -779.2567138671875,
"loss": 1.125,
"margin_dpo/margin_mean": 279.2928466796875,
"margin_dpo/margin_std": 413.36419677734375,
"step": 449
},
{
"KL/chosen_KL_mean": -333.99755859375,
"KL/mean": -498.18817138671875,
"KL/rejected_KL_mean": -662.3787231445312,
"KL/std": 331.12115478515625,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.0012786297593265772,
"fcm_dpo/delta": -0.017539218068122864,
"fcm_dpo/margin": 328.38116455078125,
"fcm_dpo/q_t": 0.40046894550323486,
"grad_norm": 31.005373001098633,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.5352098941802979,
"logits/rejected": -0.5448019504547119,
"logps/chosen": -391.1056823730469,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -765.1336669921875,
"loss": 1.0514,
"margin_dpo/margin_mean": 328.38116455078125,
"margin_dpo/margin_std": 326.8597717285156,
"step": 450
},
{
"KL/chosen_KL_mean": -435.90960693359375,
"KL/mean": -556.161376953125,
"KL/rejected_KL_mean": -676.4130859375,
"KL/std": 351.29571533203125,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0012763871345669031,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 240.50343322753906,
"fcm_dpo/q_t": 0.4259311258792877,
"grad_norm": 27.074934005737305,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.5356581211090088,
"logits/rejected": -0.514275074005127,
"logps/chosen": -494.3784484863281,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -749.3424682617188,
"loss": 1.1687,
"margin_dpo/margin_mean": 240.50343322753906,
"margin_dpo/margin_std": 412.58929443359375,
"step": 451
},
{
"KL/chosen_KL_mean": -310.4342346191406,
"KL/mean": -462.1864013671875,
"KL/rejected_KL_mean": -613.9385986328125,
"KL/std": 299.0002746582031,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0012763871345669031,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 303.5043640136719,
"fcm_dpo/q_t": 0.40798717737197876,
"grad_norm": 17.62459945678711,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.4971578121185303,
"logits/rejected": -0.5155096054077148,
"logps/chosen": -360.82476806640625,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -711.7100219726562,
"loss": 1.0732,
"margin_dpo/margin_mean": 303.5043640136719,
"margin_dpo/margin_std": 309.4674377441406,
"step": 452
},
{
"KL/chosen_KL_mean": -380.0153503417969,
"KL/mean": -510.1275939941406,
"KL/rejected_KL_mean": -640.2398681640625,
"KL/std": 297.6603088378906,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0012763871345669031,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 260.2244873046875,
"fcm_dpo/q_t": 0.42284131050109863,
"grad_norm": 34.658294677734375,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.5617681741714478,
"logits/rejected": -0.5431563258171082,
"logps/chosen": -437.730224609375,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -722.447265625,
"loss": 1.1294,
"margin_dpo/margin_mean": 260.2244873046875,
"margin_dpo/margin_std": 350.82586669921875,
"step": 453
},
{
"KL/chosen_KL_mean": -447.6456298828125,
"KL/mean": -597.4365234375,
"KL/rejected_KL_mean": -747.2275390625,
"KL/std": 339.4739990234375,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0012763871345669031,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 299.58184814453125,
"fcm_dpo/q_t": 0.41170454025268555,
"grad_norm": 25.235965728759766,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.5426455736160278,
"logits/rejected": -0.5316953659057617,
"logps/chosen": -508.5912780761719,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -832.1783447265625,
"loss": 1.0988,
"margin_dpo/margin_mean": 299.5818786621094,
"margin_dpo/margin_std": 396.16448974609375,
"step": 454
},
{
"KL/chosen_KL_mean": -390.29376220703125,
"KL/mean": -577.4393920898438,
"KL/rejected_KL_mean": -764.5850219726562,
"KL/std": 369.1748046875,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.001265165745280683,
"fcm_dpo/delta": -0.08869747817516327,
"fcm_dpo/margin": 374.29132080078125,
"fcm_dpo/q_t": 0.3919771909713745,
"grad_norm": 29.067773818969727,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.5430868268013,
"logits/rejected": -0.5612744092941284,
"logps/chosen": -435.1804504394531,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -879.886474609375,
"loss": 1.0373,
"margin_dpo/margin_mean": 374.29132080078125,
"margin_dpo/margin_std": 424.240234375,
"step": 455
},
{
"KL/chosen_KL_mean": -419.44073486328125,
"KL/mean": -587.567626953125,
"KL/rejected_KL_mean": -755.6944580078125,
"KL/std": 357.8977355957031,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0012515599373728037,
"fcm_dpo/delta": -0.019051332026720047,
"fcm_dpo/margin": 336.253662109375,
"fcm_dpo/q_t": 0.40199345350265503,
"grad_norm": 25.685274124145508,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.5657025575637817,
"logits/rejected": -0.5827906131744385,
"logps/chosen": -476.4775390625,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -860.9122924804688,
"loss": 1.0551,
"margin_dpo/margin_mean": 336.25372314453125,
"margin_dpo/margin_std": 357.0649719238281,
"step": 456
},
{
"KL/chosen_KL_mean": -389.86431884765625,
"KL/mean": -559.647705078125,
"KL/rejected_KL_mean": -729.43115234375,
"KL/std": 333.1719970703125,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.0012405143352225423,
"fcm_dpo/delta": -0.03478863835334778,
"fcm_dpo/margin": 339.56689453125,
"fcm_dpo/q_t": 0.39991387724876404,
"grad_norm": 31.572132110595703,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.5767790079116821,
"logits/rejected": -0.5724951028823853,
"logps/chosen": -444.10687255859375,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -814.540771484375,
"loss": 1.0492,
"margin_dpo/margin_mean": 339.56689453125,
"margin_dpo/margin_std": 329.47491455078125,
"step": 457
},
{
"KL/chosen_KL_mean": -409.2054443359375,
"KL/mean": -559.6752319335938,
"KL/rejected_KL_mean": -710.14501953125,
"KL/std": 302.4131774902344,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.0012405143352225423,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 300.9395751953125,
"fcm_dpo/q_t": 0.4121519923210144,
"grad_norm": 21.94940185546875,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.5830236673355103,
"logits/rejected": -0.5776429772377014,
"logps/chosen": -464.61431884765625,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -807.8282470703125,
"loss": 1.0911,
"margin_dpo/margin_mean": 300.9395751953125,
"margin_dpo/margin_std": 344.2265625,
"step": 458
},
{
"KL/chosen_KL_mean": -434.1375427246094,
"KL/mean": -599.1243896484375,
"KL/rejected_KL_mean": -764.1112060546875,
"KL/std": 358.45794677734375,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.001232771435752511,
"fcm_dpo/delta": -0.0313061960041523,
"fcm_dpo/margin": 329.97369384765625,
"fcm_dpo/q_t": 0.40537169575691223,
"grad_norm": 33.05942916870117,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.589569091796875,
"logits/rejected": -0.6022425889968872,
"logps/chosen": -480.69500732421875,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -850.27978515625,
"loss": 1.081,
"margin_dpo/margin_mean": 329.9737243652344,
"margin_dpo/margin_std": 406.204345703125,
"step": 459
},
{
"KL/chosen_KL_mean": -496.5813903808594,
"KL/mean": -666.2108154296875,
"KL/rejected_KL_mean": -835.840087890625,
"KL/std": 349.3248291015625,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0012299625668674707,
"fcm_dpo/delta": -0.022837545722723007,
"fcm_dpo/margin": 339.2586975097656,
"fcm_dpo/q_t": 0.40289586782455444,
"grad_norm": 32.447635650634766,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.6300777792930603,
"logits/rejected": -0.6566332578659058,
"logps/chosen": -548.21630859375,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -939.95947265625,
"loss": 1.0673,
"margin_dpo/margin_mean": 339.2586975097656,
"margin_dpo/margin_std": 395.57269287109375,
"step": 460
},
{
"KL/chosen_KL_mean": -494.4696044921875,
"KL/mean": -633.8778076171875,
"KL/rejected_KL_mean": -773.2859497070312,
"KL/std": 341.07598876953125,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0012271535815671086,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 278.81634521484375,
"fcm_dpo/q_t": 0.4200149476528168,
"grad_norm": 27.538618087768555,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.6771355867385864,
"logits/rejected": -0.6876776814460754,
"logps/chosen": -549.6515502929688,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -859.7628173828125,
"loss": 1.1277,
"margin_dpo/margin_mean": 278.81634521484375,
"margin_dpo/margin_std": 390.0162658691406,
"step": 461
},
{
"KL/chosen_KL_mean": -510.42669677734375,
"KL/mean": -613.5990600585938,
"KL/rejected_KL_mean": -716.7714233398438,
"KL/std": 337.86505126953125,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.0012271535815671086,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 206.34474182128906,
"fcm_dpo/q_t": 0.4412042796611786,
"grad_norm": 38.32763671875,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.6980989575386047,
"logits/rejected": -0.6892492771148682,
"logps/chosen": -580.354736328125,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -795.612548828125,
"loss": 1.2047,
"margin_dpo/margin_mean": 206.3447265625,
"margin_dpo/margin_std": 392.1902160644531,
"step": 462
},
{
"KL/chosen_KL_mean": -485.9715881347656,
"KL/mean": -636.2039794921875,
"KL/rejected_KL_mean": -786.4364013671875,
"KL/std": 374.5822448730469,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.0012250933796167374,
"fcm_dpo/delta": -0.01681620068848133,
"fcm_dpo/margin": 300.4647521972656,
"fcm_dpo/q_t": 0.41564083099365234,
"grad_norm": 36.514583587646484,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.6740679740905762,
"logits/rejected": -0.681418776512146,
"logps/chosen": -541.2459716796875,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -875.4613647460938,
"loss": 1.1198,
"margin_dpo/margin_mean": 300.4647521972656,
"margin_dpo/margin_std": 436.19610595703125,
"step": 463
},
{
"KL/chosen_KL_mean": -479.87335205078125,
"KL/mean": -703.5326538085938,
"KL/rejected_KL_mean": -927.1920166015625,
"KL/std": 418.5787353515625,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0011922982521355152,
"fcm_dpo/delta": -0.14186176657676697,
"fcm_dpo/margin": 447.31866455078125,
"fcm_dpo/q_t": 0.37638723850250244,
"grad_norm": 34.769657135009766,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.6487765312194824,
"logits/rejected": -0.6982420682907104,
"logps/chosen": -530.78564453125,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -1029.681396484375,
"loss": 0.9827,
"margin_dpo/margin_mean": 447.3186340332031,
"margin_dpo/margin_std": 431.365478515625,
"step": 464
},
{
"KL/chosen_KL_mean": -455.03009033203125,
"KL/mean": -633.11962890625,
"KL/rejected_KL_mean": -811.2090454101562,
"KL/std": 406.3192138671875,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0011822583619505167,
"fcm_dpo/delta": -0.05550694093108177,
"fcm_dpo/margin": 356.178955078125,
"fcm_dpo/q_t": 0.401016503572464,
"grad_norm": 38.73068618774414,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.6660338640213013,
"logits/rejected": -0.6831067800521851,
"logps/chosen": -515.14697265625,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -925.1550903320312,
"loss": 1.0704,
"margin_dpo/margin_mean": 356.1789855957031,
"margin_dpo/margin_std": 426.8403015136719,
"step": 465
},
{
"KL/chosen_KL_mean": -490.3927917480469,
"KL/mean": -640.9849853515625,
"KL/rejected_KL_mean": -791.5771484375,
"KL/std": 363.56646728515625,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0011756960302591324,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 301.18438720703125,
"fcm_dpo/q_t": 0.41663411259651184,
"grad_norm": 36.27015686035156,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.6633919477462769,
"logits/rejected": -0.6672701835632324,
"logps/chosen": -543.313720703125,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -881.892578125,
"loss": 1.1181,
"margin_dpo/margin_mean": 301.18438720703125,
"margin_dpo/margin_std": 405.44305419921875,
"step": 466
},
{
"KL/chosen_KL_mean": -629.504638671875,
"KL/mean": -789.4716186523438,
"KL/rejected_KL_mean": -949.4385986328125,
"KL/std": 462.40643310546875,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0011614387622103095,
"fcm_dpo/delta": -0.061003945767879486,
"fcm_dpo/margin": 319.9339294433594,
"fcm_dpo/q_t": 0.4168221950531006,
"grad_norm": 45.457401275634766,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.7396456003189087,
"logits/rejected": -0.7342954277992249,
"logps/chosen": -708.220458984375,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -1052.298828125,
"loss": 1.1552,
"margin_dpo/margin_mean": 319.9339599609375,
"margin_dpo/margin_std": 570.8200073242188,
"step": 467
},
{
"KL/chosen_KL_mean": -484.61407470703125,
"KL/mean": -672.2987060546875,
"KL/rejected_KL_mean": -859.9832763671875,
"KL/std": 430.0896911621094,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0011516888625919819,
"fcm_dpo/delta": -0.04215071350336075,
"fcm_dpo/margin": 375.36920166015625,
"fcm_dpo/q_t": 0.40216344594955444,
"grad_norm": 38.76249694824219,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.726195752620697,
"logits/rejected": -0.7313699126243591,
"logps/chosen": -554.5494384765625,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -961.0120849609375,
"loss": 1.0731,
"margin_dpo/margin_mean": 375.36920166015625,
"margin_dpo/margin_std": 493.8682556152344,
"step": 468
},
{
"KL/chosen_KL_mean": -523.5865478515625,
"KL/mean": -722.1326904296875,
"KL/rejected_KL_mean": -920.6788330078125,
"KL/std": 390.2972717285156,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.00113427615724504,
"fcm_dpo/delta": -0.07617395371198654,
"fcm_dpo/margin": 397.0923156738281,
"fcm_dpo/q_t": 0.39496946334838867,
"grad_norm": 52.0592155456543,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.7656177282333374,
"logits/rejected": -0.7790836691856384,
"logps/chosen": -591.711181640625,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -1025.46533203125,
"loss": 1.044,
"margin_dpo/margin_mean": 397.09234619140625,
"margin_dpo/margin_std": 434.4091491699219,
"step": 469
},
{
"KL/chosen_KL_mean": -428.40618896484375,
"KL/mean": -600.690673828125,
"KL/rejected_KL_mean": -772.9750366210938,
"KL/std": 402.35675048828125,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.00113427615724504,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 344.56884765625,
"fcm_dpo/q_t": 0.40878430008888245,
"grad_norm": 27.16141128540039,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.7324019074440002,
"logits/rejected": -0.7460435032844543,
"logps/chosen": -472.1981201171875,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -855.6778564453125,
"loss": 1.0924,
"margin_dpo/margin_mean": 344.56884765625,
"margin_dpo/margin_std": 443.0701904296875,
"step": 470
},
{
"KL/chosen_KL_mean": -538.134521484375,
"KL/mean": -695.206787109375,
"KL/rejected_KL_mean": -852.2789916992188,
"KL/std": 426.8754577636719,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0011279982281848788,
"fcm_dpo/delta": -0.027750782668590546,
"fcm_dpo/margin": 314.14447021484375,
"fcm_dpo/q_t": 0.4184933304786682,
"grad_norm": 29.48638916015625,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.8245488405227661,
"logits/rejected": -0.8365659713745117,
"logps/chosen": -601.4740600585938,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -935.8894653320312,
"loss": 1.1244,
"margin_dpo/margin_mean": 314.14447021484375,
"margin_dpo/margin_std": 448.8436279296875,
"step": 471
},
{
"KL/chosen_KL_mean": -556.806640625,
"KL/mean": -734.4447021484375,
"KL/rejected_KL_mean": -912.0826416015625,
"KL/std": 493.10186767578125,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.0011279982281848788,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 355.27606201171875,
"fcm_dpo/q_t": 0.4122043251991272,
"grad_norm": 31.485414505004883,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.8135469555854797,
"logits/rejected": -0.7923085689544678,
"logps/chosen": -640.4727783203125,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -1029.2918701171875,
"loss": 1.1123,
"margin_dpo/margin_mean": 355.27606201171875,
"margin_dpo/margin_std": 546.5753784179688,
"step": 472
},
{
"KL/chosen_KL_mean": -683.7619018554688,
"KL/mean": -787.9921264648438,
"KL/rejected_KL_mean": -892.2222900390625,
"KL/std": 545.3720092773438,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0011279982281848788,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 208.46038818359375,
"fcm_dpo/q_t": 0.44825372099876404,
"grad_norm": 76.29390716552734,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.841565728187561,
"logits/rejected": -0.8225346207618713,
"logps/chosen": -747.2589111328125,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -973.3688354492188,
"loss": 1.3034,
"margin_dpo/margin_mean": 208.46038818359375,
"margin_dpo/margin_std": 709.546875,
"step": 473
},
{
"KL/chosen_KL_mean": -571.6044921875,
"KL/mean": -715.5053100585938,
"KL/rejected_KL_mean": -859.4061279296875,
"KL/std": 460.3998718261719,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.0011236746795475483,
"fcm_dpo/delta": -0.038477614521980286,
"fcm_dpo/margin": 287.80169677734375,
"fcm_dpo/q_t": 0.4247364401817322,
"grad_norm": 35.654964447021484,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.7764319181442261,
"logits/rejected": -0.7763692736625671,
"logps/chosen": -624.2164306640625,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -949.486572265625,
"loss": 1.1789,
"margin_dpo/margin_mean": 287.80169677734375,
"margin_dpo/margin_std": 533.5931396484375,
"step": 474
},
{
"KL/chosen_KL_mean": -415.87945556640625,
"KL/mean": -611.8514404296875,
"KL/rejected_KL_mean": -807.8234252929688,
"KL/std": 382.92364501953125,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.0011138062691316009,
"fcm_dpo/delta": -0.024829309433698654,
"fcm_dpo/margin": 391.9439697265625,
"fcm_dpo/q_t": 0.3984021544456482,
"grad_norm": 41.1740608215332,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.8495703935623169,
"logits/rejected": -0.8721433281898499,
"logps/chosen": -458.3746337890625,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -897.8863525390625,
"loss": 1.0471,
"margin_dpo/margin_mean": 391.9439697265625,
"margin_dpo/margin_std": 420.0411376953125,
"step": 475
},
{
"KL/chosen_KL_mean": -541.1224975585938,
"KL/mean": -693.9903564453125,
"KL/rejected_KL_mean": -846.858154296875,
"KL/std": 429.5723876953125,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.0011138062691316009,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 305.7356262207031,
"fcm_dpo/q_t": 0.421493262052536,
"grad_norm": 45.96709442138672,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.8673366904258728,
"logits/rejected": -0.880070149898529,
"logps/chosen": -584.0718994140625,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -920.568359375,
"loss": 1.1285,
"margin_dpo/margin_mean": 305.73565673828125,
"margin_dpo/margin_std": 432.8980712890625,
"step": 476
},
{
"KL/chosen_KL_mean": -573.6900634765625,
"KL/mean": -727.4942626953125,
"KL/rejected_KL_mean": -881.2985229492188,
"KL/std": 450.3263244628906,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0011138062691316009,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 307.6085205078125,
"fcm_dpo/q_t": 0.4209038019180298,
"grad_norm": 32.88685989379883,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.8998314142227173,
"logits/rejected": -0.8742116689682007,
"logps/chosen": -644.462646484375,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -957.4359130859375,
"loss": 1.1596,
"margin_dpo/margin_mean": 307.6085205078125,
"margin_dpo/margin_std": 546.6884155273438,
"step": 477
},
{
"KL/chosen_KL_mean": -472.537841796875,
"KL/mean": -655.8150024414062,
"KL/rejected_KL_mean": -839.0921630859375,
"KL/std": 408.32635498046875,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.0011108508333563805,
"fcm_dpo/delta": -0.026605598628520966,
"fcm_dpo/margin": 366.5543518066406,
"fcm_dpo/q_t": 0.4059169888496399,
"grad_norm": 34.70343017578125,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.8026444911956787,
"logits/rejected": -0.8240790367126465,
"logps/chosen": -513.9783935546875,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -924.4541625976562,
"loss": 1.0865,
"margin_dpo/margin_mean": 366.5543518066406,
"margin_dpo/margin_std": 478.48858642578125,
"step": 478
},
{
"KL/chosen_KL_mean": -608.5978393554688,
"KL/mean": -790.894775390625,
"KL/rejected_KL_mean": -973.1917724609375,
"KL/std": 528.50146484375,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.0011017767246812582,
"fcm_dpo/delta": -0.05553380772471428,
"fcm_dpo/margin": 364.5938720703125,
"fcm_dpo/q_t": 0.41255372762680054,
"grad_norm": 35.93061065673828,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.8539482355117798,
"logits/rejected": -0.887352705001831,
"logps/chosen": -662.5057373046875,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -1068.30810546875,
"loss": 1.125,
"margin_dpo/margin_mean": 364.5938720703125,
"margin_dpo/margin_std": 614.5701904296875,
"step": 479
},
{
"KL/chosen_KL_mean": -745.3245849609375,
"KL/mean": -879.6640625,
"KL/rejected_KL_mean": -1014.0034790039062,
"KL/std": 472.1075439453125,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0010918761836364865,
"fcm_dpo/delta": -0.034638624638319016,
"fcm_dpo/margin": 268.6788635253906,
"fcm_dpo/q_t": 0.43371596932411194,
"grad_norm": 47.8585090637207,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.888252854347229,
"logits/rejected": -0.887417197227478,
"logps/chosen": -804.00732421875,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -1096.93603515625,
"loss": 1.2006,
"margin_dpo/margin_mean": 268.6788330078125,
"margin_dpo/margin_std": 522.060302734375,
"step": 480
},
{
"KL/chosen_KL_mean": -616.970458984375,
"KL/mean": -833.5260620117188,
"KL/rejected_KL_mean": -1050.0816650390625,
"KL/std": 495.3128662109375,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.001077498309314251,
"fcm_dpo/delta": -0.06990566849708557,
"fcm_dpo/margin": 433.1112060546875,
"fcm_dpo/q_t": 0.39440447092056274,
"grad_norm": 38.01708221435547,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.9300037026405334,
"logits/rejected": -0.9615186452865601,
"logps/chosen": -671.9346923828125,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -1142.502197265625,
"loss": 1.049,
"margin_dpo/margin_mean": 433.1112060546875,
"margin_dpo/margin_std": 537.92578125,
"step": 481
},
{
"KL/chosen_KL_mean": -663.9671630859375,
"KL/mean": -759.5072631835938,
"KL/rejected_KL_mean": -855.0474243164062,
"KL/std": 483.645263671875,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0010729870991781354,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 191.0802459716797,
"fcm_dpo/q_t": 0.4497986435890198,
"grad_norm": 47.595367431640625,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.8404616117477417,
"logits/rejected": -0.8292778730392456,
"logps/chosen": -731.5206298828125,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -942.636962890625,
"loss": 1.2789,
"margin_dpo/margin_mean": 191.0802459716797,
"margin_dpo/margin_std": 566.9451293945312,
"step": 482
},
{
"KL/chosen_KL_mean": -605.1905517578125,
"KL/mean": -833.1956787109375,
"KL/rejected_KL_mean": -1061.200927734375,
"KL/std": 506.8423767089844,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0010574627667665482,
"fcm_dpo/delta": -0.0865851491689682,
"fcm_dpo/margin": 456.01025390625,
"fcm_dpo/q_t": 0.39256346225738525,
"grad_norm": 33.234920501708984,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.8963823318481445,
"logits/rejected": -0.9069424271583557,
"logps/chosen": -668.5203857421875,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -1156.98779296875,
"loss": 1.0479,
"margin_dpo/margin_mean": 456.01025390625,
"margin_dpo/margin_std": 591.51123046875,
"step": 483
},
{
"KL/chosen_KL_mean": -507.6318359375,
"KL/mean": -704.4056396484375,
"KL/rejected_KL_mean": -901.179443359375,
"KL/std": 471.9982604980469,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0010493712034076452,
"fcm_dpo/delta": -0.049505699425935745,
"fcm_dpo/margin": 393.54754638671875,
"fcm_dpo/q_t": 0.4051154851913452,
"grad_norm": 46.96840286254883,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.9186742901802063,
"logits/rejected": -0.9407310485839844,
"logps/chosen": -566.77001953125,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -985.5509033203125,
"loss": 1.0752,
"margin_dpo/margin_mean": 393.54754638671875,
"margin_dpo/margin_std": 481.69659423828125,
"step": 484
},
{
"KL/chosen_KL_mean": -556.6889038085938,
"KL/mean": -737.6572265625,
"KL/rejected_KL_mean": -918.6255493164062,
"KL/std": 474.49932861328125,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.00104417628608644,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 361.93670654296875,
"fcm_dpo/q_t": 0.41450387239456177,
"grad_norm": 32.620994567871094,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.8470867872238159,
"logits/rejected": -0.8637259006500244,
"logps/chosen": -615.5384521484375,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -1021.9896240234375,
"loss": 1.1206,
"margin_dpo/margin_mean": 361.93670654296875,
"margin_dpo/margin_std": 549.8648071289062,
"step": 485
},
{
"KL/chosen_KL_mean": -620.675048828125,
"KL/mean": -835.8392944335938,
"KL/rejected_KL_mean": -1051.0035400390625,
"KL/std": 538.46240234375,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.0010384198976680636,
"fcm_dpo/delta": -0.027640754356980324,
"fcm_dpo/margin": 430.3284606933594,
"fcm_dpo/q_t": 0.40111875534057617,
"grad_norm": 29.77266502380371,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -0.9278169870376587,
"logits/rejected": -0.9417098760604858,
"logps/chosen": -675.9347534179688,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -1143.142822265625,
"loss": 1.0937,
"margin_dpo/margin_mean": 430.32843017578125,
"margin_dpo/margin_std": 660.73291015625,
"step": 486
},
{
"KL/chosen_KL_mean": -627.740478515625,
"KL/mean": -792.3194580078125,
"KL/rejected_KL_mean": -956.8984985351562,
"KL/std": 484.5578308105469,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0010384198976680636,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 329.1581115722656,
"fcm_dpo/q_t": 0.41984665393829346,
"grad_norm": 34.58036804199219,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -0.9389762878417969,
"logits/rejected": -0.9452526569366455,
"logps/chosen": -680.8037719726562,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -1049.3173828125,
"loss": 1.1402,
"margin_dpo/margin_mean": 329.1580810546875,
"margin_dpo/margin_std": 512.0164794921875,
"step": 487
},
{
"KL/chosen_KL_mean": -516.5718994140625,
"KL/mean": -692.9715576171875,
"KL/rejected_KL_mean": -869.3712158203125,
"KL/std": 444.61041259765625,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0010365236084908247,
"fcm_dpo/delta": -0.01829504780471325,
"fcm_dpo/margin": 352.79931640625,
"fcm_dpo/q_t": 0.4146237373352051,
"grad_norm": 29.500595092773438,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.8787708282470703,
"logits/rejected": -0.8868429064750671,
"logps/chosen": -568.800048828125,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -953.3777465820312,
"loss": 1.1012,
"margin_dpo/margin_mean": 352.79931640625,
"margin_dpo/margin_std": 430.9080810546875,
"step": 488
},
{
"KL/chosen_KL_mean": -498.3140869140625,
"KL/mean": -664.83203125,
"KL/rejected_KL_mean": -831.3499755859375,
"KL/std": 436.31561279296875,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.001034627202898264,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 333.0358581542969,
"fcm_dpo/q_t": 0.4215896427631378,
"grad_norm": 28.19427490234375,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.8919167518615723,
"logits/rejected": -0.8911116719245911,
"logps/chosen": -554.3037109375,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -910.748046875,
"loss": 1.1309,
"margin_dpo/margin_mean": 333.0358581542969,
"margin_dpo/margin_std": 491.57147216796875,
"step": 489
},
{
"KL/chosen_KL_mean": -567.6314697265625,
"KL/mean": -747.6055908203125,
"KL/rejected_KL_mean": -927.5797729492188,
"KL/std": 524.840576171875,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.001034627202898264,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 359.94830322265625,
"fcm_dpo/q_t": 0.4163244962692261,
"grad_norm": 44.42933654785156,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.9381183981895447,
"logits/rejected": -0.9800983667373657,
"logps/chosen": -619.9978637695312,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -1037.98876953125,
"loss": 1.1393,
"margin_dpo/margin_mean": 359.94830322265625,
"margin_dpo/margin_std": 605.6275634765625,
"step": 490
},
{
"KL/chosen_KL_mean": -538.706787109375,
"KL/mean": -658.9903564453125,
"KL/rejected_KL_mean": -779.27392578125,
"KL/std": 446.2900390625,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.001034627202898264,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 240.56715393066406,
"fcm_dpo/q_t": 0.4419635236263275,
"grad_norm": 29.538942337036133,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.8918236494064331,
"logits/rejected": -0.887330949306488,
"logps/chosen": -598.822998046875,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -852.5467529296875,
"loss": 1.2013,
"margin_dpo/margin_mean": 240.56716918945312,
"margin_dpo/margin_std": 437.52325439453125,
"step": 491
},
{
"KL/chosen_KL_mean": -576.6124267578125,
"KL/mean": -691.875732421875,
"KL/rejected_KL_mean": -807.13916015625,
"KL/std": 466.2126770019531,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.001034627202898264,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 230.52671813964844,
"fcm_dpo/q_t": 0.44509798288345337,
"grad_norm": 23.890384674072266,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -0.9451920986175537,
"logits/rejected": -0.9290696382522583,
"logps/chosen": -630.6065673828125,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -879.7987670898438,
"loss": 1.2337,
"margin_dpo/margin_mean": 230.52671813964844,
"margin_dpo/margin_std": 528.3690795898438,
"step": 492
},
{
"KL/chosen_KL_mean": -610.3974609375,
"KL/mean": -739.4407348632812,
"KL/rejected_KL_mean": -868.4840698242188,
"KL/std": 505.65179443359375,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.001034627202898264,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 258.0867004394531,
"fcm_dpo/q_t": 0.4370383620262146,
"grad_norm": 40.53925323486328,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -0.9742394685745239,
"logits/rejected": -0.9517089128494263,
"logps/chosen": -685.8946533203125,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -955.80712890625,
"loss": 1.203,
"margin_dpo/margin_mean": 258.0867004394531,
"margin_dpo/margin_std": 504.8338928222656,
"step": 493
},
{
"KL/chosen_KL_mean": -448.5633544921875,
"KL/mean": -657.5833129882812,
"KL/rejected_KL_mean": -866.603271484375,
"KL/std": 430.7680969238281,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.0010329412762075663,
"fcm_dpo/delta": -0.01632118411362171,
"fcm_dpo/margin": 418.0399169921875,
"fcm_dpo/q_t": 0.3988221287727356,
"grad_norm": 39.62709426879883,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.795988917350769,
"logits/rejected": -0.8313146829605103,
"logps/chosen": -489.922607421875,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -952.6946411132812,
"loss": 1.0487,
"margin_dpo/margin_mean": 418.0399169921875,
"margin_dpo/margin_std": 434.9868469238281,
"step": 494
},
{
"KL/chosen_KL_mean": -520.1987915039062,
"KL/mean": -700.3590087890625,
"KL/rejected_KL_mean": -880.519287109375,
"KL/std": 453.6920471191406,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 360.3204345703125,
"fcm_dpo/q_t": 0.4160408079624176,
"grad_norm": 31.332609176635742,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -0.9381812810897827,
"logits/rejected": -0.947554349899292,
"logps/chosen": -583.73388671875,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -971.9437255859375,
"loss": 1.1239,
"margin_dpo/margin_mean": 360.3204345703125,
"margin_dpo/margin_std": 561.566650390625,
"step": 495
},
{
"KL/chosen_KL_mean": -635.0003662109375,
"KL/mean": -740.7531127929688,
"KL/rejected_KL_mean": -846.505859375,
"KL/std": 361.4820556640625,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 211.50555419921875,
"fcm_dpo/q_t": 0.44787222146987915,
"grad_norm": 42.50273513793945,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.8592913150787354,
"logits/rejected": -0.8332956433296204,
"logps/chosen": -707.59228515625,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -930.835205078125,
"loss": 1.2259,
"margin_dpo/margin_mean": 211.50555419921875,
"margin_dpo/margin_std": 415.5306091308594,
"step": 496
},
{
"KL/chosen_KL_mean": -589.6499633789062,
"KL/mean": -693.0792236328125,
"KL/rejected_KL_mean": -796.5084838867188,
"KL/std": 438.46783447265625,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 206.85845947265625,
"fcm_dpo/q_t": 0.4494214653968811,
"grad_norm": 27.05514144897461,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.8634764552116394,
"logits/rejected": -0.8541151285171509,
"logps/chosen": -648.2439575195312,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -872.796875,
"loss": 1.2327,
"margin_dpo/margin_mean": 206.8584747314453,
"margin_dpo/margin_std": 418.838623046875,
"step": 497
},
{
"KL/chosen_KL_mean": -564.5078125,
"KL/mean": -724.235107421875,
"KL/rejected_KL_mean": -883.96240234375,
"KL/std": 477.89898681640625,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 319.4546203613281,
"fcm_dpo/q_t": 0.4236466884613037,
"grad_norm": 31.779359817504883,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.8575003147125244,
"logits/rejected": -0.8462520837783813,
"logps/chosen": -635.7134399414062,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -967.9204711914062,
"loss": 1.1577,
"margin_dpo/margin_mean": 319.45465087890625,
"margin_dpo/margin_std": 545.30078125,
"step": 498
},
{
"KL/chosen_KL_mean": -495.37310791015625,
"KL/mean": -694.611083984375,
"KL/rejected_KL_mean": -893.8490600585938,
"KL/std": 456.4473571777344,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 398.4759826660156,
"fcm_dpo/q_t": 0.4053412973880768,
"grad_norm": 37.97807693481445,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.849279522895813,
"logits/rejected": -0.8824639916419983,
"logps/chosen": -546.6282958984375,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -994.927734375,
"loss": 1.0835,
"margin_dpo/margin_mean": 398.4759826660156,
"margin_dpo/margin_std": 521.2337036132812,
"step": 499
},
{
"KL/chosen_KL_mean": -614.5982666015625,
"KL/mean": -786.6547241210938,
"KL/rejected_KL_mean": -958.7110595703125,
"KL/std": 431.516357421875,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 344.1127624511719,
"fcm_dpo/q_t": 0.4179650545120239,
"grad_norm": 38.53366470336914,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.8466501235961914,
"logits/rejected": -0.8797069787979126,
"logps/chosen": -671.625732421875,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -1052.645263671875,
"loss": 1.1331,
"margin_dpo/margin_mean": 344.11279296875,
"margin_dpo/margin_std": 528.118896484375,
"step": 500
},
{
"KL/chosen_KL_mean": -541.839599609375,
"KL/mean": -719.80517578125,
"KL/rejected_KL_mean": -897.7708129882812,
"KL/std": 457.7550048828125,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 355.93121337890625,
"fcm_dpo/q_t": 0.4169032573699951,
"grad_norm": 30.2011661529541,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.8327955007553101,
"logits/rejected": -0.8246780633926392,
"logps/chosen": -596.1990966796875,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -977.927490234375,
"loss": 1.1334,
"margin_dpo/margin_mean": 355.93121337890625,
"margin_dpo/margin_std": 572.0584716796875,
"step": 501
},
{
"KL/chosen_KL_mean": -501.7233581542969,
"KL/mean": -686.9359130859375,
"KL/rejected_KL_mean": -872.1483154296875,
"KL/std": 432.0093078613281,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 370.42498779296875,
"fcm_dpo/q_t": 0.41214871406555176,
"grad_norm": 28.975494384765625,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.7686693668365479,
"logits/rejected": -0.7646470069885254,
"logps/chosen": -569.3238525390625,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -955.0971069335938,
"loss": 1.0911,
"margin_dpo/margin_mean": 370.42498779296875,
"margin_dpo/margin_std": 456.77459716796875,
"step": 502
},
{
"KL/chosen_KL_mean": -531.9078369140625,
"KL/mean": -705.465576171875,
"KL/rejected_KL_mean": -879.0233154296875,
"KL/std": 428.1451416015625,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 347.11553955078125,
"fcm_dpo/q_t": 0.41783469915390015,
"grad_norm": 29.5872859954834,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.7954933643341064,
"logits/rejected": -0.7909719944000244,
"logps/chosen": -586.9862060546875,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -961.5287475585938,
"loss": 1.1139,
"margin_dpo/margin_mean": 347.11553955078125,
"margin_dpo/margin_std": 466.2398376464844,
"step": 503
},
{
"KL/chosen_KL_mean": -569.9815673828125,
"KL/mean": -776.477783203125,
"KL/rejected_KL_mean": -982.9739990234375,
"KL/std": 500.4878845214844,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 412.99249267578125,
"fcm_dpo/q_t": 0.40421169996261597,
"grad_norm": 30.236486434936523,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.8175668716430664,
"logits/rejected": -0.8474366664886475,
"logps/chosen": -629.9473266601562,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -1086.736083984375,
"loss": 1.0912,
"margin_dpo/margin_mean": 412.99249267578125,
"margin_dpo/margin_std": 581.5658569335938,
"step": 504
},
{
"KL/chosen_KL_mean": -668.5108642578125,
"KL/mean": -772.6334228515625,
"KL/rejected_KL_mean": -876.7559814453125,
"KL/std": 486.93994140625,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0010312554659321904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 208.24505615234375,
"fcm_dpo/q_t": 0.4494059085845947,
"grad_norm": 49.470855712890625,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.8806732296943665,
"logits/rejected": -0.8661482334136963,
"logps/chosen": -744.665771484375,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -965.34130859375,
"loss": 1.2396,
"margin_dpo/margin_mean": 208.24505615234375,
"margin_dpo/margin_std": 461.67218017578125,
"step": 505
},
{
"KL/chosen_KL_mean": -527.1854248046875,
"KL/mean": -717.6492919921875,
"KL/rejected_KL_mean": -908.11328125,
"KL/std": 489.8760986328125,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0010227421298623085,
"fcm_dpo/delta": -0.041447706520557404,
"fcm_dpo/margin": 380.9278564453125,
"fcm_dpo/q_t": 0.41084128618240356,
"grad_norm": 33.7549934387207,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.8268887400627136,
"logits/rejected": -0.8512546420097351,
"logps/chosen": -576.1458740234375,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -986.5283203125,
"loss": 1.109,
"margin_dpo/margin_mean": 380.9278869628906,
"margin_dpo/margin_std": 550.8912353515625,
"step": 506
},
{
"KL/chosen_KL_mean": -606.8805541992188,
"KL/mean": -778.807861328125,
"KL/rejected_KL_mean": -950.7351684570312,
"KL/std": 575.7430419921875,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0010227421298623085,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 343.8546142578125,
"fcm_dpo/q_t": 0.4232868254184723,
"grad_norm": 32.357879638671875,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.8557635545730591,
"logits/rejected": -0.876491129398346,
"logps/chosen": -673.6820678710938,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -1046.1080322265625,
"loss": 1.1556,
"margin_dpo/margin_mean": 343.8546142578125,
"margin_dpo/margin_std": 620.67529296875,
"step": 507
},
{
"KL/chosen_KL_mean": -669.8018188476562,
"KL/mean": -812.485107421875,
"KL/rejected_KL_mean": -955.1682739257812,
"KL/std": 556.7152099609375,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0010227421298623085,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 285.36639404296875,
"fcm_dpo/q_t": 0.43599405884742737,
"grad_norm": 31.804035186767578,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.8378009796142578,
"logits/rejected": -0.8496159315109253,
"logps/chosen": -741.1052856445312,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -1050.7958984375,
"loss": 1.2193,
"margin_dpo/margin_mean": 285.36639404296875,
"margin_dpo/margin_std": 657.32666015625,
"step": 508
},
{
"KL/chosen_KL_mean": -510.67974853515625,
"KL/mean": -655.689453125,
"KL/rejected_KL_mean": -800.69921875,
"KL/std": 391.1563720703125,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0010227421298623085,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 290.0194091796875,
"fcm_dpo/q_t": 0.4315534234046936,
"grad_norm": 37.032257080078125,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.7452073693275452,
"logits/rejected": -0.7534062266349792,
"logps/chosen": -574.4987182617188,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -883.9556884765625,
"loss": 1.175,
"margin_dpo/margin_mean": 290.0194091796875,
"margin_dpo/margin_std": 502.0699462890625,
"step": 509
},
{
"KL/chosen_KL_mean": -629.8031005859375,
"KL/mean": -827.0128173828125,
"KL/rejected_KL_mean": -1024.2225341796875,
"KL/std": 466.19384765625,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0010203055571764708,
"fcm_dpo/delta": -0.023880695924162865,
"fcm_dpo/margin": 394.41937255859375,
"fcm_dpo/q_t": 0.40587687492370605,
"grad_norm": 34.109798431396484,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.7773102521896362,
"logits/rejected": -0.8000172972679138,
"logps/chosen": -681.6815795898438,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -1126.9876708984375,
"loss": 1.0775,
"margin_dpo/margin_mean": 394.4193420410156,
"margin_dpo/margin_std": 466.3671569824219,
"step": 510
},
{
"KL/chosen_KL_mean": -586.57666015625,
"KL/mean": -742.509033203125,
"KL/rejected_KL_mean": -898.44140625,
"KL/std": 519.578125,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.001017868984490633,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 311.86468505859375,
"fcm_dpo/q_t": 0.4275307357311249,
"grad_norm": 43.292728424072266,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.8213226199150085,
"logits/rejected": -0.8411952257156372,
"logps/chosen": -646.8148193359375,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -991.2981567382812,
"loss": 1.1908,
"margin_dpo/margin_mean": 311.86468505859375,
"margin_dpo/margin_std": 644.8040771484375,
"step": 511
},
{
"KL/chosen_KL_mean": -477.79278564453125,
"KL/mean": -694.4921264648438,
"KL/rejected_KL_mean": -911.1914672851562,
"KL/std": 460.89337158203125,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.0010110008297488093,
"fcm_dpo/delta": -0.03385215252637863,
"fcm_dpo/margin": 433.3987121582031,
"fcm_dpo/q_t": 0.39783334732055664,
"grad_norm": 42.38991165161133,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.7866002321243286,
"logits/rejected": -0.8186514377593994,
"logps/chosen": -532.6982421875,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -993.0673828125,
"loss": 1.0468,
"margin_dpo/margin_mean": 433.3987121582031,
"margin_dpo/margin_std": 465.8355407714844,
"step": 512
},
{
"KL/chosen_KL_mean": -600.0064697265625,
"KL/mean": -747.2848510742188,
"KL/rejected_KL_mean": -894.563232421875,
"KL/std": 432.29364013671875,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.0010110008297488093,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 294.5567626953125,
"fcm_dpo/q_t": 0.43137532472610474,
"grad_norm": 51.88436508178711,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.8510720729827881,
"logits/rejected": -0.8437062501907349,
"logps/chosen": -664.9228515625,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -970.6256713867188,
"loss": 1.1772,
"margin_dpo/margin_mean": 294.5567626953125,
"margin_dpo/margin_std": 507.2457275390625,
"step": 513
},
{
"KL/chosen_KL_mean": -626.586181640625,
"KL/mean": -769.6492919921875,
"KL/rejected_KL_mean": -912.7125244140625,
"KL/std": 469.3660583496094,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0010110008297488093,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 286.12640380859375,
"fcm_dpo/q_t": 0.4342566430568695,
"grad_norm": 38.388465881347656,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.7949620485305786,
"logits/rejected": -0.7888473868370056,
"logps/chosen": -700.815673828125,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -991.6580810546875,
"loss": 1.1817,
"margin_dpo/margin_mean": 286.12640380859375,
"margin_dpo/margin_std": 518.3568725585938,
"step": 514
},
{
"KL/chosen_KL_mean": -515.0665893554688,
"KL/mean": -635.9742431640625,
"KL/rejected_KL_mean": -756.8817138671875,
"KL/std": 402.12554931640625,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0010110008297488093,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 241.81515502929688,
"fcm_dpo/q_t": 0.44154810905456543,
"grad_norm": 42.603919982910156,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.7839655876159668,
"logits/rejected": -0.7985705137252808,
"logps/chosen": -565.4682006835938,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -843.9794921875,
"loss": 1.211,
"margin_dpo/margin_mean": 241.81515502929688,
"margin_dpo/margin_std": 474.5227966308594,
"step": 515
},
{
"KL/chosen_KL_mean": -560.7139892578125,
"KL/mean": -737.4444580078125,
"KL/rejected_KL_mean": -914.175048828125,
"KL/std": 449.7982177734375,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.0010110008297488093,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 353.46099853515625,
"fcm_dpo/q_t": 0.4154972732067108,
"grad_norm": 34.534523010253906,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.8315219283103943,
"logits/rejected": -0.8588598966598511,
"logps/chosen": -625.3635864257812,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -1025.8974609375,
"loss": 1.11,
"margin_dpo/margin_mean": 353.46099853515625,
"margin_dpo/margin_std": 447.361572265625,
"step": 516
},
{
"KL/chosen_KL_mean": -585.670166015625,
"KL/mean": -785.64111328125,
"KL/rejected_KL_mean": -985.612060546875,
"KL/std": 497.2357482910156,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.001006539911031723,
"fcm_dpo/delta": -0.022110939025878906,
"fcm_dpo/margin": 399.94189453125,
"fcm_dpo/q_t": 0.40923649072647095,
"grad_norm": 30.278491973876953,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.7785888910293579,
"logits/rejected": -0.772436261177063,
"logps/chosen": -646.583740234375,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -1074.6951904296875,
"loss": 1.0955,
"margin_dpo/margin_mean": 399.94189453125,
"margin_dpo/margin_std": 557.2489013671875,
"step": 517
},
{
"KL/chosen_KL_mean": -527.8907470703125,
"KL/mean": -694.472412109375,
"KL/rejected_KL_mean": -861.0540771484375,
"KL/std": 399.5091552734375,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.001006539911031723,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 333.16326904296875,
"fcm_dpo/q_t": 0.42090073227882385,
"grad_norm": 35.90395736694336,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.8241711854934692,
"logits/rejected": -0.8401827216148376,
"logps/chosen": -585.3466186523438,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -946.36669921875,
"loss": 1.128,
"margin_dpo/margin_mean": 333.16326904296875,
"margin_dpo/margin_std": 447.3388366699219,
"step": 518
},
{
"KL/chosen_KL_mean": -524.0,
"KL/mean": -653.171875,
"KL/rejected_KL_mean": -782.34375,
"KL/std": 372.1302490234375,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.001006539911031723,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 258.34375,
"fcm_dpo/q_t": 0.438649445772171,
"grad_norm": 21.516462326049805,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.769424319267273,
"logits/rejected": -0.7528333067893982,
"logps/chosen": -598.0633544921875,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -886.7879028320312,
"loss": 1.1966,
"margin_dpo/margin_mean": 258.34375,
"margin_dpo/margin_std": 464.10809326171875,
"step": 519
},
{
"KL/chosen_KL_mean": -573.615234375,
"KL/mean": -723.7618408203125,
"KL/rejected_KL_mean": -873.908447265625,
"KL/std": 434.37841796875,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.001006539911031723,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 300.2933044433594,
"fcm_dpo/q_t": 0.43047094345092773,
"grad_norm": 40.110504150390625,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.7728183269500732,
"logits/rejected": -0.7749868631362915,
"logps/chosen": -643.9150390625,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -973.8897705078125,
"loss": 1.1648,
"margin_dpo/margin_mean": 300.29327392578125,
"margin_dpo/margin_std": 477.5467529296875,
"step": 520
},
{
"KL/chosen_KL_mean": -506.3398132324219,
"KL/mean": -672.2125244140625,
"KL/rejected_KL_mean": -838.0853271484375,
"KL/std": 427.4813232421875,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.0010050538694486022,
"fcm_dpo/delta": -0.014785980805754662,
"fcm_dpo/margin": 331.74554443359375,
"fcm_dpo/q_t": 0.4215025305747986,
"grad_norm": 33.34300231933594,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.8194113969802856,
"logits/rejected": -0.8199684619903564,
"logps/chosen": -564.4827270507812,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -921.365966796875,
"loss": 1.1274,
"margin_dpo/margin_mean": 331.74554443359375,
"margin_dpo/margin_std": 433.67059326171875,
"step": 521
},
{
"KL/chosen_KL_mean": -563.2149658203125,
"KL/mean": -721.0223388671875,
"KL/rejected_KL_mean": -878.829833984375,
"KL/std": 471.86846923828125,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.0010035678278654814,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 315.6148681640625,
"fcm_dpo/q_t": 0.42625993490219116,
"grad_norm": 27.813331604003906,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.8177077770233154,
"logits/rejected": -0.8223693370819092,
"logps/chosen": -609.7625732421875,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -944.8436889648438,
"loss": 1.1588,
"margin_dpo/margin_mean": 315.6148681640625,
"margin_dpo/margin_std": 516.5402221679688,
"step": 522
},
{
"KL/chosen_KL_mean": -598.5844116210938,
"KL/mean": -762.6793212890625,
"KL/rejected_KL_mean": -926.7742919921875,
"KL/std": 517.547607421875,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0010035678278654814,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 328.1898498535156,
"fcm_dpo/q_t": 0.4245453476905823,
"grad_norm": 35.869773864746094,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.8979830741882324,
"logits/rejected": -0.8981304168701172,
"logps/chosen": -660.35400390625,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -1010.5357055664062,
"loss": 1.158,
"margin_dpo/margin_mean": 328.1898498535156,
"margin_dpo/margin_std": 560.0676879882812,
"step": 523
},
{
"KL/chosen_KL_mean": -606.493408203125,
"KL/mean": -787.1712646484375,
"KL/rejected_KL_mean": -967.84912109375,
"KL/std": 502.6514892578125,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.000997315626591444,
"fcm_dpo/delta": -0.03124724142253399,
"fcm_dpo/margin": 361.355712890625,
"fcm_dpo/q_t": 0.41488218307495117,
"grad_norm": 33.932228088378906,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.915719747543335,
"logits/rejected": -0.9063006639480591,
"logps/chosen": -684.5654296875,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -1049.151123046875,
"loss": 1.1214,
"margin_dpo/margin_mean": 361.355712890625,
"margin_dpo/margin_std": 500.954345703125,
"step": 524
},
{
"KL/chosen_KL_mean": -579.07470703125,
"KL/mean": -802.6356811523438,
"KL/rejected_KL_mean": -1026.1966552734375,
"KL/std": 518.89892578125,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0009856013348326087,
"fcm_dpo/delta": -0.059076473116874695,
"fcm_dpo/margin": 447.1219482421875,
"fcm_dpo/q_t": 0.4000527560710907,
"grad_norm": 29.54397201538086,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.8346696496009827,
"logits/rejected": -0.8713185787200928,
"logps/chosen": -629.902587890625,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1126.24951171875,
"loss": 1.0653,
"margin_dpo/margin_mean": 447.1219482421875,
"margin_dpo/margin_std": 567.528564453125,
"step": 525
},
{
"KL/chosen_KL_mean": -608.8138427734375,
"KL/mean": -826.2957763671875,
"KL/rejected_KL_mean": -1043.777587890625,
"KL/std": 492.2466735839844,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.0009817786049097776,
"fcm_dpo/delta": -0.03893730789422989,
"fcm_dpo/margin": 434.9637451171875,
"fcm_dpo/q_t": 0.40048468112945557,
"grad_norm": 30.214094161987305,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.8993504047393799,
"logits/rejected": -0.9024904370307922,
"logps/chosen": -671.9810791015625,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1130.0869140625,
"loss": 1.0727,
"margin_dpo/margin_mean": 434.9637451171875,
"margin_dpo/margin_std": 545.2774047851562,
"step": 526
},
{
"KL/chosen_KL_mean": -662.787109375,
"KL/mean": -804.6998291015625,
"KL/rejected_KL_mean": -946.6126098632812,
"KL/std": 530.4249267578125,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.0009779557585716248,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 283.82550048828125,
"fcm_dpo/q_t": 0.4336469769477844,
"grad_norm": 32.90876007080078,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -1.0440315008163452,
"logits/rejected": -1.018221378326416,
"logps/chosen": -733.915771484375,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -1024.9552001953125,
"loss": 1.1731,
"margin_dpo/margin_mean": 283.8254699707031,
"margin_dpo/margin_std": 440.089111328125,
"step": 527
},
{
"KL/chosen_KL_mean": -678.576416015625,
"KL/mean": -860.773193359375,
"KL/rejected_KL_mean": -1042.969970703125,
"KL/std": 518.66552734375,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0009779557585716248,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 364.39361572265625,
"fcm_dpo/q_t": 0.42012155055999756,
"grad_norm": 39.71454620361328,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -0.9921354055404663,
"logits/rejected": -0.9825552701950073,
"logps/chosen": -746.6658935546875,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1136.8800048828125,
"loss": 1.1484,
"margin_dpo/margin_mean": 364.3935852050781,
"margin_dpo/margin_std": 623.328857421875,
"step": 528
},
{
"KL/chosen_KL_mean": -818.5634765625,
"KL/mean": -943.8115234375,
"KL/rejected_KL_mean": -1069.059814453125,
"KL/std": 499.3103942871094,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.0009779557585716248,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 250.49615478515625,
"fcm_dpo/q_t": 0.4428832530975342,
"grad_norm": 39.008636474609375,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -1.0032634735107422,
"logits/rejected": -1.0176358222961426,
"logps/chosen": -874.138427734375,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1158.268798828125,
"loss": 1.2489,
"margin_dpo/margin_mean": 250.4961700439453,
"margin_dpo/margin_std": 632.3921508789062,
"step": 529
},
{
"KL/chosen_KL_mean": -697.18505859375,
"KL/mean": -897.6633911132812,
"KL/rejected_KL_mean": -1098.1417236328125,
"KL/std": 595.1827392578125,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.0009779557585716248,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 400.9566345214844,
"fcm_dpo/q_t": 0.4137534499168396,
"grad_norm": 41.25276184082031,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.9190390110015869,
"logits/rejected": -0.9196174144744873,
"logps/chosen": -744.7864990234375,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1185.42626953125,
"loss": 1.1537,
"margin_dpo/margin_mean": 400.9566345214844,
"margin_dpo/margin_std": 755.96826171875,
"step": 530
},
{
"KL/chosen_KL_mean": -761.462158203125,
"KL/mean": -938.3872680664062,
"KL/rejected_KL_mean": -1115.3123779296875,
"KL/std": 620.985595703125,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.0009779557585716248,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 353.85015869140625,
"fcm_dpo/q_t": 0.4216981530189514,
"grad_norm": 41.99516296386719,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -1.0249850749969482,
"logits/rejected": -1.0165561437606812,
"logps/chosen": -816.708251953125,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1185.91845703125,
"loss": 1.1491,
"margin_dpo/margin_mean": 353.8501892089844,
"margin_dpo/margin_std": 595.3245849609375,
"step": 531
},
{
"KL/chosen_KL_mean": -767.3759155273438,
"KL/mean": -957.3938598632812,
"KL/rejected_KL_mean": -1147.411865234375,
"KL/std": 608.4393920898438,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0009684361866675317,
"fcm_dpo/delta": -0.048909228295087814,
"fcm_dpo/margin": 380.03594970703125,
"fcm_dpo/q_t": 0.4171864986419678,
"grad_norm": 66.07853698730469,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -0.9798325896263123,
"logits/rejected": -0.9659659266471863,
"logps/chosen": -837.6619262695312,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1234.003173828125,
"loss": 1.1672,
"margin_dpo/margin_mean": 380.03594970703125,
"margin_dpo/margin_std": 719.28515625,
"step": 532
},
{
"KL/chosen_KL_mean": -700.1473388671875,
"KL/mean": -833.2701416015625,
"KL/rejected_KL_mean": -966.3930053710938,
"KL/std": 496.19708251953125,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0009684361866675317,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 266.2456359863281,
"fcm_dpo/q_t": 0.44103071093559265,
"grad_norm": 40.109352111816406,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -0.9989339113235474,
"logits/rejected": -0.9852375388145447,
"logps/chosen": -765.00244140625,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -1042.9810791015625,
"loss": 1.2276,
"margin_dpo/margin_mean": 266.2456359863281,
"margin_dpo/margin_std": 604.99951171875,
"step": 533
},
{
"KL/chosen_KL_mean": -748.6889038085938,
"KL/mean": -934.053955078125,
"KL/rejected_KL_mean": -1119.4190673828125,
"KL/std": 555.2821044921875,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0009583088685758412,
"fcm_dpo/delta": -0.052562322467565536,
"fcm_dpo/margin": 370.7301940917969,
"fcm_dpo/q_t": 0.4191279709339142,
"grad_norm": 30.985227584838867,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -0.9798387885093689,
"logits/rejected": -0.9826042056083679,
"logps/chosen": -808.8082885742188,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1197.9625244140625,
"loss": 1.142,
"margin_dpo/margin_mean": 370.7301940917969,
"margin_dpo/margin_std": 594.4391479492188,
"step": 534
},
{
"KL/chosen_KL_mean": -624.0946044921875,
"KL/mean": -831.3848266601562,
"KL/rejected_KL_mean": -1038.6749267578125,
"KL/std": 534.6995239257812,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.0009523722692392766,
"fcm_dpo/delta": -0.062335532158613205,
"fcm_dpo/margin": 414.58038330078125,
"fcm_dpo/q_t": 0.4099598526954651,
"grad_norm": 27.550270080566406,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.8934037685394287,
"logits/rejected": -0.9010223150253296,
"logps/chosen": -678.4248657226562,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -1134.982666015625,
"loss": 1.1031,
"margin_dpo/margin_mean": 414.58038330078125,
"margin_dpo/margin_std": 572.0284423828125,
"step": 535
},
{
"KL/chosen_KL_mean": -564.6681518554688,
"KL/mean": -820.0997314453125,
"KL/rejected_KL_mean": -1075.5313720703125,
"KL/std": 627.7283325195312,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.000939212623052299,
"fcm_dpo/delta": -0.03830568864941597,
"fcm_dpo/margin": 510.8631591796875,
"fcm_dpo/q_t": 0.39753982424736023,
"grad_norm": 33.21775436401367,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.8998743295669556,
"logits/rejected": -0.9397362470626831,
"logps/chosen": -611.7486572265625,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1164.629150390625,
"loss": 1.0702,
"margin_dpo/margin_mean": 510.8631591796875,
"margin_dpo/margin_std": 767.861572265625,
"step": 536
},
{
"KL/chosen_KL_mean": -593.1109008789062,
"KL/mean": -760.2655029296875,
"KL/rejected_KL_mean": -927.4201049804688,
"KL/std": 485.60101318359375,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.000939212623052299,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 334.30914306640625,
"fcm_dpo/q_t": 0.4269586205482483,
"grad_norm": 37.82878494262695,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.943962812423706,
"logits/rejected": -0.9385641813278198,
"logps/chosen": -650.8583984375,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -997.8585205078125,
"loss": 1.1749,
"margin_dpo/margin_mean": 334.30914306640625,
"margin_dpo/margin_std": 601.835693359375,
"step": 537
},
{
"KL/chosen_KL_mean": -740.7052001953125,
"KL/mean": -931.8037109375,
"KL/rejected_KL_mean": -1122.9022216796875,
"KL/std": 551.2314453125,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.000939212623052299,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 382.19696044921875,
"fcm_dpo/q_t": 0.4175069332122803,
"grad_norm": 34.995201110839844,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.9600582122802734,
"logits/rejected": -0.954196572303772,
"logps/chosen": -807.12109375,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1207.13037109375,
"loss": 1.1259,
"margin_dpo/margin_mean": 382.19696044921875,
"margin_dpo/margin_std": 565.8170166015625,
"step": 538
},
{
"KL/chosen_KL_mean": -622.4293212890625,
"KL/mean": -803.8492431640625,
"KL/rejected_KL_mean": -985.2692260742188,
"KL/std": 459.92645263671875,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.000939212623052299,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 362.83990478515625,
"fcm_dpo/q_t": 0.42134806513786316,
"grad_norm": 27.91701316833496,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.956368088722229,
"logits/rejected": -0.9539651870727539,
"logps/chosen": -680.9221801757812,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -1077.1231689453125,
"loss": 1.133,
"margin_dpo/margin_mean": 362.83990478515625,
"margin_dpo/margin_std": 528.1322631835938,
"step": 539
},
{
"KL/chosen_KL_mean": -613.2940673828125,
"KL/mean": -858.5700073242188,
"KL/rejected_KL_mean": -1103.845947265625,
"KL/std": 557.71875,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.0009325757855549455,
"fcm_dpo/delta": -0.07116752117872238,
"fcm_dpo/margin": 490.5519104003906,
"fcm_dpo/q_t": 0.3990428149700165,
"grad_norm": 35.79396438598633,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.9001051187515259,
"logits/rejected": -0.93080735206604,
"logps/chosen": -676.7765502929688,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1220.27587890625,
"loss": 1.0722,
"margin_dpo/margin_mean": 490.5518798828125,
"margin_dpo/margin_std": 673.9207763671875,
"step": 540
},
{
"KL/chosen_KL_mean": -723.417724609375,
"KL/mean": -848.6341552734375,
"KL/rejected_KL_mean": -973.8507080078125,
"KL/std": 496.656494140625,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0009259390062652528,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 250.43295288085938,
"fcm_dpo/q_t": 0.44649481773376465,
"grad_norm": 42.431304931640625,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -1.0044571161270142,
"logits/rejected": -1.0006784200668335,
"logps/chosen": -801.6980590820312,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -1077.124267578125,
"loss": 1.2451,
"margin_dpo/margin_mean": 250.43295288085938,
"margin_dpo/margin_std": 604.05517578125,
"step": 541
},
{
"KL/chosen_KL_mean": -614.3228759765625,
"KL/mean": -852.7803955078125,
"KL/rejected_KL_mean": -1091.23779296875,
"KL/std": 551.623046875,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": -0.09556810557842255,
"fcm_dpo/margin": 476.91497802734375,
"fcm_dpo/q_t": 0.4020240902900696,
"grad_norm": 39.426239013671875,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -0.9757965803146362,
"logits/rejected": -1.0126338005065918,
"logps/chosen": -671.807861328125,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -1187.712890625,
"loss": 1.0837,
"margin_dpo/margin_mean": 476.91497802734375,
"margin_dpo/margin_std": 653.942138671875,
"step": 542
},
{
"KL/chosen_KL_mean": -700.5947265625,
"KL/mean": -872.3914794921875,
"KL/rejected_KL_mean": -1044.188232421875,
"KL/std": 673.9176025390625,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 343.59356689453125,
"fcm_dpo/q_t": 0.4374322295188904,
"grad_norm": 31.346834182739258,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.9188766479492188,
"logits/rejected": -0.9484801888465881,
"logps/chosen": -761.2122192382812,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1149.7872314453125,
"loss": 1.2094,
"margin_dpo/margin_mean": 343.59356689453125,
"margin_dpo/margin_std": 792.45556640625,
"step": 543
},
{
"KL/chosen_KL_mean": -696.1849975585938,
"KL/mean": -888.3853759765625,
"KL/rejected_KL_mean": -1080.5858154296875,
"KL/std": 521.3961181640625,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 384.40069580078125,
"fcm_dpo/q_t": 0.41937947273254395,
"grad_norm": 30.976642608642578,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.9292371273040771,
"logits/rejected": -0.949985921382904,
"logps/chosen": -755.8272705078125,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -1181.54052734375,
"loss": 1.1344,
"margin_dpo/margin_mean": 384.40069580078125,
"margin_dpo/margin_std": 590.4559326171875,
"step": 544
},
{
"KL/chosen_KL_mean": -651.0039672851562,
"KL/mean": -878.35302734375,
"KL/rejected_KL_mean": -1105.7021484375,
"KL/std": 524.6614990234375,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 454.6982421875,
"fcm_dpo/q_t": 0.4059000313282013,
"grad_norm": 29.362411499023438,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.8762187957763672,
"logits/rejected": -0.9087934494018555,
"logps/chosen": -718.652587890625,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1201.610107421875,
"loss": 1.0835,
"margin_dpo/margin_mean": 454.6982421875,
"margin_dpo/margin_std": 596.3817138671875,
"step": 545
},
{
"KL/chosen_KL_mean": -614.3394775390625,
"KL/mean": -765.3737182617188,
"KL/rejected_KL_mean": -916.407958984375,
"KL/std": 452.42413330078125,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 302.068603515625,
"fcm_dpo/q_t": 0.4340311288833618,
"grad_norm": 33.5017204284668,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.8841739296913147,
"logits/rejected": -0.8689842224121094,
"logps/chosen": -665.0836791992188,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -998.2742309570312,
"loss": 1.1769,
"margin_dpo/margin_mean": 302.068603515625,
"margin_dpo/margin_std": 477.5527038574219,
"step": 546
},
{
"KL/chosen_KL_mean": -617.981689453125,
"KL/mean": -808.706787109375,
"KL/rejected_KL_mean": -999.431884765625,
"KL/std": 510.5972595214844,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 381.4501953125,
"fcm_dpo/q_t": 0.4202958643436432,
"grad_norm": 41.74578094482422,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.8707060813903809,
"logits/rejected": -0.8808646202087402,
"logps/chosen": -691.66943359375,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -1090.1932373046875,
"loss": 1.1313,
"margin_dpo/margin_mean": 381.4501953125,
"margin_dpo/margin_std": 563.5973510742188,
"step": 547
},
{
"KL/chosen_KL_mean": -636.4512939453125,
"KL/mean": -818.338134765625,
"KL/rejected_KL_mean": -1000.2249755859375,
"KL/std": 539.5311279296875,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 363.773681640625,
"fcm_dpo/q_t": 0.4263709485530853,
"grad_norm": 24.599536895751953,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -0.9489472508430481,
"logits/rejected": -0.9488046169281006,
"logps/chosen": -701.6976318359375,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -1094.343017578125,
"loss": 1.1491,
"margin_dpo/margin_mean": 363.773681640625,
"margin_dpo/margin_std": 603.0093994140625,
"step": 548
},
{
"KL/chosen_KL_mean": -664.5079345703125,
"KL/mean": -797.922119140625,
"KL/rejected_KL_mean": -931.3363037109375,
"KL/std": 454.3035888671875,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 266.828369140625,
"fcm_dpo/q_t": 0.4441815912723541,
"grad_norm": 32.23136901855469,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -0.939894437789917,
"logits/rejected": -0.9260774850845337,
"logps/chosen": -713.7202758789062,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -1005.24658203125,
"loss": 1.2198,
"margin_dpo/margin_mean": 266.828369140625,
"margin_dpo/margin_std": 551.3993530273438,
"step": 549
},
{
"KL/chosen_KL_mean": -666.9960327148438,
"KL/mean": -836.1296997070312,
"KL/rejected_KL_mean": -1005.263427734375,
"KL/std": 507.13153076171875,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0009084090124815702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 338.26739501953125,
"fcm_dpo/q_t": 0.4283323884010315,
"grad_norm": 34.777339935302734,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.8918087482452393,
"logits/rejected": -0.9147313237190247,
"logps/chosen": -723.802978515625,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -1100.38916015625,
"loss": 1.1613,
"margin_dpo/margin_mean": 338.26739501953125,
"margin_dpo/margin_std": 551.6748046875,
"step": 550
},
{
"KL/chosen_KL_mean": -579.2154541015625,
"KL/mean": -854.2412719726562,
"KL/rejected_KL_mean": -1129.26708984375,
"KL/std": 542.586181640625,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.000893935386557132,
"fcm_dpo/delta": -0.09667488187551498,
"fcm_dpo/margin": 550.0517578125,
"fcm_dpo/q_t": 0.38580551743507385,
"grad_norm": 45.99151611328125,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.8883758187294006,
"logits/rejected": -0.944542407989502,
"logps/chosen": -638.32177734375,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1240.93994140625,
"loss": 1.0109,
"margin_dpo/margin_mean": 550.0516357421875,
"margin_dpo/margin_std": 557.3201293945312,
"step": 551
},
{
"KL/chosen_KL_mean": -577.863037109375,
"KL/mean": -876.5994873046875,
"KL/rejected_KL_mean": -1175.335693359375,
"KL/std": 629.6008911132812,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.000866841000970453,
"fcm_dpo/delta": -0.13752080500125885,
"fcm_dpo/margin": 597.4727783203125,
"fcm_dpo/q_t": 0.385326087474823,
"grad_norm": 36.25246047973633,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.968271017074585,
"logits/rejected": -1.0044474601745605,
"logps/chosen": -640.2176513671875,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1279.89794921875,
"loss": 1.0223,
"margin_dpo/margin_mean": 597.4727783203125,
"margin_dpo/margin_std": 725.0616455078125,
"step": 552
},
{
"KL/chosen_KL_mean": -655.1487426757812,
"KL/mean": -812.9437255859375,
"KL/rejected_KL_mean": -970.7386474609375,
"KL/std": 523.8294067382812,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.000866841000970453,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 315.5899353027344,
"fcm_dpo/q_t": 0.4384046494960785,
"grad_norm": 26.22563362121582,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.9064982533454895,
"logits/rejected": -0.9010224938392639,
"logps/chosen": -723.4075927734375,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -1068.8358154296875,
"loss": 1.1923,
"margin_dpo/margin_mean": 315.5899353027344,
"margin_dpo/margin_std": 594.8109130859375,
"step": 553
},
{
"KL/chosen_KL_mean": -673.444580078125,
"KL/mean": -887.5985717773438,
"KL/rejected_KL_mean": -1101.752685546875,
"KL/std": 560.7201538085938,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.000863053894136101,
"fcm_dpo/delta": -0.04388042911887169,
"fcm_dpo/margin": 428.30804443359375,
"fcm_dpo/q_t": 0.41699787974357605,
"grad_norm": 47.83314514160156,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.8999230861663818,
"logits/rejected": -0.8767585158348083,
"logps/chosen": -741.3922119140625,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1191.535400390625,
"loss": 1.1412,
"margin_dpo/margin_mean": 428.30804443359375,
"margin_dpo/margin_std": 716.1023559570312,
"step": 554
},
{
"KL/chosen_KL_mean": -677.0280151367188,
"KL/mean": -928.472412109375,
"KL/rejected_KL_mean": -1179.916748046875,
"KL/std": 603.3981323242188,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.000859266787301749,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 502.8887023925781,
"fcm_dpo/q_t": 0.4060073494911194,
"grad_norm": 28.9240665435791,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -0.9636756181716919,
"logits/rejected": -1.018855333328247,
"logps/chosen": -730.3585205078125,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1288.3961181640625,
"loss": 1.1002,
"margin_dpo/margin_mean": 502.8887023925781,
"margin_dpo/margin_std": 782.5892944335938,
"step": 555
},
{
"KL/chosen_KL_mean": -670.974853515625,
"KL/mean": -819.40380859375,
"KL/rejected_KL_mean": -967.832763671875,
"KL/std": 458.6606140136719,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.000859266787301749,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 296.85791015625,
"fcm_dpo/q_t": 0.4399312734603882,
"grad_norm": 25.964445114135742,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -0.9628030061721802,
"logits/rejected": -0.9593526124954224,
"logps/chosen": -729.6193237304688,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -1069.1732177734375,
"loss": 1.1873,
"margin_dpo/margin_mean": 296.85791015625,
"margin_dpo/margin_std": 475.78826904296875,
"step": 556
},
{
"KL/chosen_KL_mean": -718.9598388671875,
"KL/mean": -942.1116943359375,
"KL/rejected_KL_mean": -1165.263671875,
"KL/std": 635.51220703125,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.000859266787301749,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 446.30389404296875,
"fcm_dpo/q_t": 0.41387179493904114,
"grad_norm": 57.082557678222656,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -0.9800699949264526,
"logits/rejected": -1.0134789943695068,
"logps/chosen": -786.8004150390625,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1275.203369140625,
"loss": 1.1283,
"margin_dpo/margin_mean": 446.30389404296875,
"margin_dpo/margin_std": 714.6304321289062,
"step": 557
},
{
"KL/chosen_KL_mean": -653.600830078125,
"KL/mean": -958.6536865234375,
"KL/rejected_KL_mean": -1263.70654296875,
"KL/std": 602.1583251953125,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0008436896023340523,
"fcm_dpo/delta": -0.12100745737552643,
"fcm_dpo/margin": 610.1058349609375,
"fcm_dpo/q_t": 0.3799861669540405,
"grad_norm": 30.4708194732666,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -1.0119829177856445,
"logits/rejected": -1.0322072505950928,
"logps/chosen": -715.9690551757812,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1365.86767578125,
"loss": 1.0123,
"margin_dpo/margin_mean": 610.1058349609375,
"margin_dpo/margin_std": 676.6812133789062,
"step": 558
},
{
"KL/chosen_KL_mean": -751.7446899414062,
"KL/mean": -994.5537109375,
"KL/rejected_KL_mean": -1237.362548828125,
"KL/std": 611.50537109375,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.000838720821775496,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 485.61785888671875,
"fcm_dpo/q_t": 0.40765488147735596,
"grad_norm": 27.252429962158203,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -1.1454588174819946,
"logits/rejected": -1.1528816223144531,
"logps/chosen": -812.4970703125,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1330.804931640625,
"loss": 1.0922,
"margin_dpo/margin_mean": 485.617919921875,
"margin_dpo/margin_std": 667.4962158203125,
"step": 559
},
{
"KL/chosen_KL_mean": -696.612548828125,
"KL/mean": -864.5140991210938,
"KL/rejected_KL_mean": -1032.4156494140625,
"KL/std": 541.3946533203125,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.000838720821775496,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 335.80303955078125,
"fcm_dpo/q_t": 0.43424075841903687,
"grad_norm": 36.40821075439453,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.8833180069923401,
"logits/rejected": -0.8601206541061401,
"logps/chosen": -754.7164306640625,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -1112.4068603515625,
"loss": 1.1824,
"margin_dpo/margin_mean": 335.8030090332031,
"margin_dpo/margin_std": 580.6434326171875,
"step": 560
},
{
"KL/chosen_KL_mean": -802.1702880859375,
"KL/mean": -957.013427734375,
"KL/rejected_KL_mean": -1111.8564453125,
"KL/std": 551.462890625,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.000838720821775496,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 309.6861572265625,
"fcm_dpo/q_t": 0.43889930844306946,
"grad_norm": 37.50949478149414,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -1.0387603044509888,
"logits/rejected": -1.0596671104431152,
"logps/chosen": -853.0831298828125,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1210.925048828125,
"loss": 1.2241,
"margin_dpo/margin_mean": 309.6861877441406,
"margin_dpo/margin_std": 698.983642578125,
"step": 561
},
{
"KL/chosen_KL_mean": -746.2704467773438,
"KL/mean": -969.857177734375,
"KL/rejected_KL_mean": -1193.44384765625,
"KL/std": 613.4844970703125,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0008317689062096179,
"fcm_dpo/delta": -0.04161640629172325,
"fcm_dpo/margin": 447.1734924316406,
"fcm_dpo/q_t": 0.4157896041870117,
"grad_norm": 28.799089431762695,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -1.0045117139816284,
"logits/rejected": -1.021229863166809,
"logps/chosen": -805.73486328125,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1289.986572265625,
"loss": 1.1226,
"margin_dpo/margin_mean": 447.1734924316406,
"margin_dpo/margin_std": 680.6741333007812,
"step": 562
},
{
"KL/chosen_KL_mean": -833.0841064453125,
"KL/mean": -988.2342529296875,
"KL/rejected_KL_mean": -1143.384521484375,
"KL/std": 653.4237060546875,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.0008317689062096179,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 310.3004150390625,
"fcm_dpo/q_t": 0.4358706772327423,
"grad_norm": 54.44693374633789,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -0.9969061613082886,
"logits/rejected": -1.0025964975357056,
"logps/chosen": -893.6923217773438,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -1237.9521484375,
"loss": 1.25,
"margin_dpo/margin_mean": 310.3004150390625,
"margin_dpo/margin_std": 777.999267578125,
"step": 563
},
{
"KL/chosen_KL_mean": -749.1875,
"KL/mean": -958.049072265625,
"KL/rejected_KL_mean": -1166.91064453125,
"KL/std": 577.4024658203125,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0008317689062096179,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 417.72308349609375,
"fcm_dpo/q_t": 0.41977375745773315,
"grad_norm": 37.301246643066406,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -1.0899568796157837,
"logits/rejected": -1.0908780097961426,
"logps/chosen": -823.4559326171875,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1260.1488037109375,
"loss": 1.1222,
"margin_dpo/margin_mean": 417.72308349609375,
"margin_dpo/margin_std": 562.4461669921875,
"step": 564
},
{
"KL/chosen_KL_mean": -788.7421264648438,
"KL/mean": -1010.16650390625,
"KL/rejected_KL_mean": -1231.5908203125,
"KL/std": 673.0230712890625,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0008317689062096179,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 442.8487548828125,
"fcm_dpo/q_t": 0.4197535514831543,
"grad_norm": 43.9473991394043,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -0.9752411842346191,
"logits/rejected": -0.9796169996261597,
"logps/chosen": -857.7620849609375,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1317.369873046875,
"loss": 1.1576,
"margin_dpo/margin_mean": 442.8487548828125,
"margin_dpo/margin_std": 822.1761474609375,
"step": 565
},
{
"KL/chosen_KL_mean": -741.95458984375,
"KL/mean": -990.75732421875,
"KL/rejected_KL_mean": -1239.56005859375,
"KL/std": 691.6851196289062,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0008263904601335526,
"fcm_dpo/delta": -0.06508417427539825,
"fcm_dpo/margin": 497.6054382324219,
"fcm_dpo/q_t": 0.40809524059295654,
"grad_norm": 30.059511184692383,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -1.0234966278076172,
"logits/rejected": -1.0528680086135864,
"logps/chosen": -808.4998779296875,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1343.4293212890625,
"loss": 1.1205,
"margin_dpo/margin_mean": 497.6054382324219,
"margin_dpo/margin_std": 798.7295532226562,
"step": 566
},
{
"KL/chosen_KL_mean": -725.8890380859375,
"KL/mean": -903.4608154296875,
"KL/rejected_KL_mean": -1081.032470703125,
"KL/std": 483.6868896484375,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.0008210120722651482,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 355.1435546875,
"fcm_dpo/q_t": 0.43069371581077576,
"grad_norm": 33.07154846191406,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -1.095928430557251,
"logits/rejected": -1.1078492403030396,
"logps/chosen": -778.747314453125,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1166.403564453125,
"loss": 1.1605,
"margin_dpo/margin_mean": 355.1435546875,
"margin_dpo/margin_std": 527.5757446289062,
"step": 567
},
{
"KL/chosen_KL_mean": -705.9122314453125,
"KL/mean": -956.9530029296875,
"KL/rejected_KL_mean": -1207.99365234375,
"KL/std": 561.6522216796875,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0008210120722651482,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 502.0815124511719,
"fcm_dpo/q_t": 0.4041532874107361,
"grad_norm": 27.16654396057129,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -1.0139139890670776,
"logits/rejected": -1.053145170211792,
"logps/chosen": -751.1046142578125,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1297.0860595703125,
"loss": 1.0734,
"margin_dpo/margin_mean": 502.08154296875,
"margin_dpo/margin_std": 605.7379150390625,
"step": 568
},
{
"KL/chosen_KL_mean": -794.1281127929688,
"KL/mean": -997.6332397460938,
"KL/rejected_KL_mean": -1201.138427734375,
"KL/std": 705.3419189453125,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0008191708475351334,
"fcm_dpo/delta": -0.022476255893707275,
"fcm_dpo/margin": 407.01025390625,
"fcm_dpo/q_t": 0.42279374599456787,
"grad_norm": 37.293025970458984,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -1.0435158014297485,
"logits/rejected": -1.0531883239746094,
"logps/chosen": -857.8486938476562,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1280.2415771484375,
"loss": 1.1715,
"margin_dpo/margin_mean": 407.01025390625,
"margin_dpo/margin_std": 746.8839111328125,
"step": 569
},
{
"KL/chosen_KL_mean": -751.165771484375,
"KL/mean": -937.5892333984375,
"KL/rejected_KL_mean": -1124.0125732421875,
"KL/std": 564.7623901367188,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0008173296810127795,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 372.8468017578125,
"fcm_dpo/q_t": 0.43035823106765747,
"grad_norm": 28.89210319519043,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -1.108044147491455,
"logits/rejected": -1.1026582717895508,
"logps/chosen": -812.7803955078125,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1206.154541015625,
"loss": 1.1683,
"margin_dpo/margin_mean": 372.8468017578125,
"margin_dpo/margin_std": 630.748291015625,
"step": 570
},
{
"KL/chosen_KL_mean": -763.8045654296875,
"KL/mean": -990.728515625,
"KL/rejected_KL_mean": -1217.65234375,
"KL/std": 579.8861083984375,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0008173296810127795,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 453.8477783203125,
"fcm_dpo/q_t": 0.41571101546287537,
"grad_norm": 36.950618743896484,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -1.000840663909912,
"logits/rejected": -1.015519142150879,
"logps/chosen": -816.858642578125,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1308.9891357421875,
"loss": 1.1287,
"margin_dpo/margin_mean": 453.8477783203125,
"margin_dpo/margin_std": 714.1256103515625,
"step": 571
},
{
"KL/chosen_KL_mean": -789.1917114257812,
"KL/mean": -1021.33642578125,
"KL/rejected_KL_mean": -1253.481201171875,
"KL/std": 666.8356323242188,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.0008173296810127795,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 464.2894287109375,
"fcm_dpo/q_t": 0.417039155960083,
"grad_norm": 28.30565643310547,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -1.017333984375,
"logits/rejected": -1.0511081218719482,
"logps/chosen": -837.6510009765625,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1337.0382080078125,
"loss": 1.1434,
"margin_dpo/margin_mean": 464.2894592285156,
"margin_dpo/margin_std": 822.282958984375,
"step": 572
},
{
"KL/chosen_KL_mean": -799.183349609375,
"KL/mean": -1003.74462890625,
"KL/rejected_KL_mean": -1208.3060302734375,
"KL/std": 615.51416015625,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0008173296810127795,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 409.12261962890625,
"fcm_dpo/q_t": 0.4239146113395691,
"grad_norm": 31.040990829467773,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -1.0863670110702515,
"logits/rejected": -1.096688985824585,
"logps/chosen": -850.8059692382812,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1293.6309814453125,
"loss": 1.1352,
"margin_dpo/margin_mean": 409.12261962890625,
"margin_dpo/margin_std": 608.7803344726562,
"step": 573
},
{
"KL/chosen_KL_mean": -725.0342407226562,
"KL/mean": -954.8162231445312,
"KL/rejected_KL_mean": -1184.5982666015625,
"KL/std": 558.27392578125,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0008173296810127795,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 459.56402587890625,
"fcm_dpo/q_t": 0.4135124981403351,
"grad_norm": 31.780391693115234,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -1.0929412841796875,
"logits/rejected": -1.1233415603637695,
"logps/chosen": -776.0787353515625,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -1277.4046630859375,
"loss": 1.1118,
"margin_dpo/margin_mean": 459.5640563964844,
"margin_dpo/margin_std": 647.52099609375,
"step": 574
},
{
"KL/chosen_KL_mean": -799.796875,
"KL/mean": -1004.4361572265625,
"KL/rejected_KL_mean": -1209.075439453125,
"KL/std": 569.8582153320312,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0008112110663205385,
"fcm_dpo/delta": -0.03757143393158913,
"fcm_dpo/margin": 409.27850341796875,
"fcm_dpo/q_t": 0.42343461513519287,
"grad_norm": 33.69365692138672,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.9379447102546692,
"logits/rejected": -0.9466258883476257,
"logps/chosen": -871.5870361328125,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1304.461669921875,
"loss": 1.1679,
"margin_dpo/margin_mean": 409.27850341796875,
"margin_dpo/margin_std": 728.100341796875,
"step": 575
},
{
"KL/chosen_KL_mean": -759.4315185546875,
"KL/mean": -915.8280029296875,
"KL/rejected_KL_mean": -1072.2244873046875,
"KL/std": 518.5361328125,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.0008112110663205385,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 312.79302978515625,
"fcm_dpo/q_t": 0.4410494565963745,
"grad_norm": 25.922910690307617,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -1.0505657196044922,
"logits/rejected": -1.0803673267364502,
"logps/chosen": -813.6944580078125,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -1172.978759765625,
"loss": 1.2076,
"margin_dpo/margin_mean": 312.79302978515625,
"margin_dpo/margin_std": 605.8926391601562,
"step": 576
},
{
"KL/chosen_KL_mean": -682.141357421875,
"KL/mean": -892.861328125,
"KL/rejected_KL_mean": -1103.581298828125,
"KL/std": 589.8372802734375,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0008112110663205385,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 421.4400329589844,
"fcm_dpo/q_t": 0.4217475652694702,
"grad_norm": 27.589473724365234,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -1.0263875722885132,
"logits/rejected": -1.0151700973510742,
"logps/chosen": -744.051025390625,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1187.652099609375,
"loss": 1.1455,
"margin_dpo/margin_mean": 421.44000244140625,
"margin_dpo/margin_std": 642.5458984375,
"step": 577
},
{
"KL/chosen_KL_mean": -675.6703491210938,
"KL/mean": -901.8224487304688,
"KL/rejected_KL_mean": -1127.9744873046875,
"KL/std": 575.355712890625,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.0008064049761742353,
"fcm_dpo/delta": -0.02971097081899643,
"fcm_dpo/margin": 452.30413818359375,
"fcm_dpo/q_t": 0.41487276554107666,
"grad_norm": 40.21394729614258,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.9069193005561829,
"logits/rejected": -0.9442964196205139,
"logps/chosen": -724.93408203125,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1211.4107666015625,
"loss": 1.1069,
"margin_dpo/margin_mean": 452.30413818359375,
"margin_dpo/margin_std": 572.793701171875,
"step": 578
},
{
"KL/chosen_KL_mean": -781.8958129882812,
"KL/mean": -914.868896484375,
"KL/rejected_KL_mean": -1047.8419189453125,
"KL/std": 587.2415161132812,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0008064049761742353,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 265.9461669921875,
"fcm_dpo/q_t": 0.44937944412231445,
"grad_norm": 66.76544189453125,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.9628750085830688,
"logits/rejected": -0.9616245031356812,
"logps/chosen": -840.7816162109375,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -1142.629638671875,
"loss": 1.2513,
"margin_dpo/margin_mean": 265.9461669921875,
"margin_dpo/margin_std": 656.545654296875,
"step": 579
},
{
"KL/chosen_KL_mean": -554.861083984375,
"KL/mean": -820.6229248046875,
"KL/rejected_KL_mean": -1086.384765625,
"KL/std": 653.8794555664062,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0007996794884093106,
"fcm_dpo/delta": -0.04187547415494919,
"fcm_dpo/margin": 531.5237426757812,
"fcm_dpo/q_t": 0.40619370341300964,
"grad_norm": 29.74520492553711,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.8521532416343689,
"logits/rejected": -0.8880908489227295,
"logps/chosen": -603.56787109375,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1168.143310546875,
"loss": 1.0849,
"margin_dpo/margin_mean": 531.5238037109375,
"margin_dpo/margin_std": 759.3482666015625,
"step": 580
},
{
"KL/chosen_KL_mean": -741.0847778320312,
"KL/mean": -923.89697265625,
"KL/rejected_KL_mean": -1106.7091064453125,
"KL/std": 567.8367919921875,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0007996794884093106,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 365.624267578125,
"fcm_dpo/q_t": 0.43274974822998047,
"grad_norm": 26.411640167236328,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -1.067054033279419,
"logits/rejected": -1.0800729990005493,
"logps/chosen": -803.30712890625,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1198.44482421875,
"loss": 1.177,
"margin_dpo/margin_mean": 365.624267578125,
"margin_dpo/margin_std": 645.589111328125,
"step": 581
},
{
"KL/chosen_KL_mean": -692.6724853515625,
"KL/mean": -903.4718017578125,
"KL/rejected_KL_mean": -1114.271240234375,
"KL/std": 547.7958984375,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0007957734633237123,
"fcm_dpo/delta": -0.04908556863665581,
"fcm_dpo/margin": 421.5987548828125,
"fcm_dpo/q_t": 0.42105481028556824,
"grad_norm": 29.055042266845703,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -1.0077698230743408,
"logits/rejected": -1.030693769454956,
"logps/chosen": -750.901123046875,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -1224.3408203125,
"loss": 1.1379,
"margin_dpo/margin_mean": 421.5987548828125,
"margin_dpo/margin_std": 591.310791015625,
"step": 582
},
{
"KL/chosen_KL_mean": -767.380859375,
"KL/mean": -899.6068115234375,
"KL/rejected_KL_mean": -1031.832763671875,
"KL/std": 589.245361328125,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 264.45184326171875,
"fcm_dpo/q_t": 0.452210009098053,
"grad_norm": 37.549983978271484,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.940156102180481,
"logits/rejected": -0.9366773962974548,
"logps/chosen": -824.2437744140625,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -1120.2366943359375,
"loss": 1.2701,
"margin_dpo/margin_mean": 264.45184326171875,
"margin_dpo/margin_std": 711.6224975585938,
"step": 583
},
{
"KL/chosen_KL_mean": -665.3277587890625,
"KL/mean": -932.5770874023438,
"KL/rejected_KL_mean": -1199.826416015625,
"KL/std": 615.3153686523438,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 534.498779296875,
"fcm_dpo/q_t": 0.4037357568740845,
"grad_norm": 26.586563110351562,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -1.0058352947235107,
"logits/rejected": -1.0271427631378174,
"logps/chosen": -722.2283935546875,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1297.4625244140625,
"loss": 1.0674,
"margin_dpo/margin_mean": 534.498779296875,
"margin_dpo/margin_std": 645.0361328125,
"step": 584
},
{
"KL/chosen_KL_mean": -809.52587890625,
"KL/mean": -1037.6292724609375,
"KL/rejected_KL_mean": -1265.732666015625,
"KL/std": 684.6345825195312,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 456.20672607421875,
"fcm_dpo/q_t": 0.4182851314544678,
"grad_norm": 28.249164581298828,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -1.0811936855316162,
"logits/rejected": -1.1092218160629272,
"logps/chosen": -870.4998168945312,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1349.902099609375,
"loss": 1.155,
"margin_dpo/margin_mean": 456.2066955566406,
"margin_dpo/margin_std": 826.325439453125,
"step": 585
},
{
"KL/chosen_KL_mean": -812.726318359375,
"KL/mean": -970.64892578125,
"KL/rejected_KL_mean": -1128.571533203125,
"KL/std": 599.4317626953125,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 315.8452453613281,
"fcm_dpo/q_t": 0.44157183170318604,
"grad_norm": 30.333675384521484,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -1.034977674484253,
"logits/rejected": -1.0590405464172363,
"logps/chosen": -869.5819091796875,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -1220.374267578125,
"loss": 1.2157,
"margin_dpo/margin_mean": 315.8452453613281,
"margin_dpo/margin_std": 651.1234130859375,
"step": 586
},
{
"KL/chosen_KL_mean": -565.1972045898438,
"KL/mean": -782.4954833984375,
"KL/rejected_KL_mean": -999.7936401367188,
"KL/std": 601.401123046875,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 434.596435546875,
"fcm_dpo/q_t": 0.4192296862602234,
"grad_norm": 25.787561416625977,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -1.0264978408813477,
"logits/rejected": -1.0559778213500977,
"logps/chosen": -609.8887939453125,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -1082.41748046875,
"loss": 1.1286,
"margin_dpo/margin_mean": 434.596435546875,
"margin_dpo/margin_std": 620.5191040039062,
"step": 587
},
{
"KL/chosen_KL_mean": -749.8773193359375,
"KL/mean": -935.8150024414062,
"KL/rejected_KL_mean": -1121.752685546875,
"KL/std": 530.2380981445312,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 371.8753662109375,
"fcm_dpo/q_t": 0.43039530515670776,
"grad_norm": 32.1988410949707,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -1.0299625396728516,
"logits/rejected": -1.055286169052124,
"logps/chosen": -800.1722412109375,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1229.12255859375,
"loss": 1.1617,
"margin_dpo/margin_mean": 371.8753662109375,
"margin_dpo/margin_std": 559.5547485351562,
"step": 588
},
{
"KL/chosen_KL_mean": -759.330810546875,
"KL/mean": -999.5764770507812,
"KL/rejected_KL_mean": -1239.8221435546875,
"KL/std": 591.2251586914062,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 480.4913024902344,
"fcm_dpo/q_t": 0.4115689992904663,
"grad_norm": 29.426036834716797,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -0.9973963499069214,
"logits/rejected": -1.0138342380523682,
"logps/chosen": -819.2607421875,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1351.4775390625,
"loss": 1.098,
"margin_dpo/margin_mean": 480.4913024902344,
"margin_dpo/margin_std": 619.790771484375,
"step": 589
},
{
"KL/chosen_KL_mean": -664.0067749023438,
"KL/mean": -904.7217407226562,
"KL/rejected_KL_mean": -1145.436767578125,
"KL/std": 613.1650390625,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 481.42999267578125,
"fcm_dpo/q_t": 0.41361021995544434,
"grad_norm": 24.00367546081543,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -1.0117418766021729,
"logits/rejected": -1.039717435836792,
"logps/chosen": -719.8165283203125,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1251.49951171875,
"loss": 1.1077,
"margin_dpo/margin_mean": 481.42999267578125,
"margin_dpo/margin_std": 685.4844970703125,
"step": 590
},
{
"KL/chosen_KL_mean": -728.047119140625,
"KL/mean": -937.2919311523438,
"KL/rejected_KL_mean": -1146.5367431640625,
"KL/std": 563.9609375,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 418.4896240234375,
"fcm_dpo/q_t": 0.4223765730857849,
"grad_norm": 30.11712074279785,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -1.0658780336380005,
"logits/rejected": -1.0678385496139526,
"logps/chosen": -784.2877197265625,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1230.322998046875,
"loss": 1.1451,
"margin_dpo/margin_mean": 418.48968505859375,
"margin_dpo/margin_std": 651.3126220703125,
"step": 591
},
{
"KL/chosen_KL_mean": -759.2822875976562,
"KL/mean": -946.3499755859375,
"KL/rejected_KL_mean": -1133.417724609375,
"KL/std": 568.3065185546875,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 374.1354064941406,
"fcm_dpo/q_t": 0.431357741355896,
"grad_norm": 27.800474166870117,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -1.065044641494751,
"logits/rejected": -1.0687838792800903,
"logps/chosen": -807.2225341796875,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1209.150634765625,
"loss": 1.2046,
"margin_dpo/margin_mean": 374.13543701171875,
"margin_dpo/margin_std": 777.451416015625,
"step": 592
},
{
"KL/chosen_KL_mean": -711.5416259765625,
"KL/mean": -860.16455078125,
"KL/rejected_KL_mean": -1008.7875366210938,
"KL/std": 603.8402099609375,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0007918673800304532,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 297.2458801269531,
"fcm_dpo/q_t": 0.44561654329299927,
"grad_norm": 39.802852630615234,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.9177390336990356,
"logits/rejected": -0.9115041494369507,
"logps/chosen": -760.232421875,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -1069.695556640625,
"loss": 1.2348,
"margin_dpo/margin_mean": 297.2458801269531,
"margin_dpo/margin_std": 679.549560546875,
"step": 593
},
{
"KL/chosen_KL_mean": -699.04541015625,
"KL/mean": -888.05126953125,
"KL/rejected_KL_mean": -1077.05712890625,
"KL/std": 604.8076782226562,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0007907521212473512,
"fcm_dpo/delta": -0.014104235917329788,
"fcm_dpo/margin": 378.01171875,
"fcm_dpo/q_t": 0.43051910400390625,
"grad_norm": 28.592931747436523,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -1.0373632907867432,
"logits/rejected": -1.0325448513031006,
"logps/chosen": -753.9803466796875,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -1163.1568603515625,
"loss": 1.1764,
"margin_dpo/margin_mean": 378.01171875,
"margin_dpo/margin_std": 645.0611572265625,
"step": 594
},
{
"KL/chosen_KL_mean": -658.1973876953125,
"KL/mean": -873.8628540039062,
"KL/rejected_KL_mean": -1089.5283203125,
"KL/std": 567.3275146484375,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0007896368042565882,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 431.3309326171875,
"fcm_dpo/q_t": 0.42307084798812866,
"grad_norm": 42.018104553222656,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -0.970179557800293,
"logits/rejected": -0.9921086430549622,
"logps/chosen": -707.6177978515625,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1170.1556396484375,
"loss": 1.1329,
"margin_dpo/margin_mean": 431.3309631347656,
"margin_dpo/margin_std": 637.8607788085938,
"step": 595
},
{
"KL/chosen_KL_mean": -758.0401611328125,
"KL/mean": -909.93603515625,
"KL/rejected_KL_mean": -1061.83203125,
"KL/std": 575.744873046875,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.0007896368042565882,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 303.7917785644531,
"fcm_dpo/q_t": 0.44556793570518494,
"grad_norm": 48.209266662597656,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -0.9976510405540466,
"logits/rejected": -0.9678058624267578,
"logps/chosen": -817.8319091796875,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -1141.923095703125,
"loss": 1.2296,
"margin_dpo/margin_mean": 303.791748046875,
"margin_dpo/margin_std": 685.4385986328125,
"step": 596
},
{
"KL/chosen_KL_mean": -714.3123779296875,
"KL/mean": -975.7779541015625,
"KL/rejected_KL_mean": -1237.2435302734375,
"KL/std": 652.4100341796875,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.0007754022371955216,
"fcm_dpo/delta": -0.09095588326454163,
"fcm_dpo/margin": 522.9312133789062,
"fcm_dpo/q_t": 0.407283753156662,
"grad_norm": 21.951868057250977,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.9390630722045898,
"logits/rejected": -0.9882240295410156,
"logps/chosen": -771.5731201171875,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1337.937255859375,
"loss": 1.0969,
"margin_dpo/margin_mean": 522.9312133789062,
"margin_dpo/margin_std": 705.5875244140625,
"step": 597
},
{
"KL/chosen_KL_mean": -718.0220947265625,
"KL/mean": -918.1834106445312,
"KL/rejected_KL_mean": -1118.3447265625,
"KL/std": 569.5988159179688,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0007724062888883054,
"fcm_dpo/delta": -0.019356058910489082,
"fcm_dpo/margin": 400.3226623535156,
"fcm_dpo/q_t": 0.42755886912345886,
"grad_norm": 36.96177291870117,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -1.0392732620239258,
"logits/rejected": -1.0519664287567139,
"logps/chosen": -770.5406494140625,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1207.78857421875,
"loss": 1.1634,
"margin_dpo/margin_mean": 400.3226623535156,
"margin_dpo/margin_std": 644.6939697265625,
"step": 598
},
{
"KL/chosen_KL_mean": -722.3475952148438,
"KL/mean": -891.9060668945312,
"KL/rejected_KL_mean": -1061.4644775390625,
"KL/std": 531.6864013671875,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0007724062888883054,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 339.116943359375,
"fcm_dpo/q_t": 0.4360012114048004,
"grad_norm": 26.386310577392578,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.9559237957000732,
"logits/rejected": -0.9652050733566284,
"logps/chosen": -772.1502685546875,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -1144.443115234375,
"loss": 1.1851,
"margin_dpo/margin_mean": 339.116943359375,
"margin_dpo/margin_std": 516.4267578125,
"step": 599
},
{
"KL/chosen_KL_mean": -781.0634155273438,
"KL/mean": -944.1868896484375,
"KL/rejected_KL_mean": -1107.310302734375,
"KL/std": 575.037841796875,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0007724062888883054,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 326.24700927734375,
"fcm_dpo/q_t": 0.4429008364677429,
"grad_norm": 27.988651275634766,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -1.0766232013702393,
"logits/rejected": -1.078524112701416,
"logps/chosen": -847.498291015625,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1192.766845703125,
"loss": 1.2171,
"margin_dpo/margin_mean": 326.2469787597656,
"margin_dpo/margin_std": 697.498779296875,
"step": 600
},
{
"KL/chosen_KL_mean": -774.169677734375,
"KL/mean": -979.185302734375,
"KL/rejected_KL_mean": -1184.200927734375,
"KL/std": 580.6097412109375,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.0007724062888883054,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 410.0313720703125,
"fcm_dpo/q_t": 0.4246867299079895,
"grad_norm": 25.131288528442383,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -1.0171586275100708,
"logits/rejected": -1.038601279258728,
"logps/chosen": -833.30322265625,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1278.891845703125,
"loss": 1.1409,
"margin_dpo/margin_mean": 410.0313720703125,
"margin_dpo/margin_std": 568.695556640625,
"step": 601
},
{
"KL/chosen_KL_mean": -545.2393798828125,
"KL/mean": -812.69677734375,
"KL/rejected_KL_mean": -1080.1541748046875,
"KL/std": 542.9259033203125,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0007678022375330329,
"fcm_dpo/delta": -0.0599651113152504,
"fcm_dpo/margin": 534.914794921875,
"fcm_dpo/q_t": 0.4043405055999756,
"grad_norm": 52.572025299072266,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -1.060288906097412,
"logits/rejected": -1.098497986793518,
"logps/chosen": -593.8328857421875,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -1167.82275390625,
"loss": 1.0672,
"margin_dpo/margin_mean": 534.9148559570312,
"margin_dpo/margin_std": 592.32177734375,
"step": 602
},
{
"KL/chosen_KL_mean": -685.7906494140625,
"KL/mean": -919.4617919921875,
"KL/rejected_KL_mean": -1153.133056640625,
"KL/std": 605.2296142578125,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 467.34228515625,
"fcm_dpo/q_t": 0.41905176639556885,
"grad_norm": 32.92748260498047,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -1.00080406665802,
"logits/rejected": -0.9961497783660889,
"logps/chosen": -756.205322265625,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -1253.45849609375,
"loss": 1.1298,
"margin_dpo/margin_mean": 467.34228515625,
"margin_dpo/margin_std": 717.1328125,
"step": 603
},
{
"KL/chosen_KL_mean": -669.0608520507812,
"KL/mean": -923.9398193359375,
"KL/rejected_KL_mean": -1178.81884765625,
"KL/std": 580.1771850585938,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 509.75799560546875,
"fcm_dpo/q_t": 0.41114941239356995,
"grad_norm": 32.59992218017578,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -1.008441686630249,
"logits/rejected": -1.044859528541565,
"logps/chosen": -715.5189208984375,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1270.67333984375,
"loss": 1.1041,
"margin_dpo/margin_mean": 509.7580261230469,
"margin_dpo/margin_std": 713.919677734375,
"step": 604
},
{
"KL/chosen_KL_mean": -706.5936279296875,
"KL/mean": -900.7664184570312,
"KL/rejected_KL_mean": -1094.939208984375,
"KL/std": 555.1638793945312,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 388.345703125,
"fcm_dpo/q_t": 0.43225014209747314,
"grad_norm": 25.63776397705078,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -0.9656909108161926,
"logits/rejected": -0.9828107357025146,
"logps/chosen": -772.8428955078125,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1197.2442626953125,
"loss": 1.1707,
"margin_dpo/margin_mean": 388.34576416015625,
"margin_dpo/margin_std": 662.9732055664062,
"step": 605
},
{
"KL/chosen_KL_mean": -704.7627563476562,
"KL/mean": -911.3067626953125,
"KL/rejected_KL_mean": -1117.8507080078125,
"KL/std": 570.53369140625,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 413.08795166015625,
"fcm_dpo/q_t": 0.42691662907600403,
"grad_norm": 29.93854331970215,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -0.980074942111969,
"logits/rejected": -0.9964933395385742,
"logps/chosen": -759.5819091796875,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -1216.22216796875,
"loss": 1.1487,
"margin_dpo/margin_mean": 413.087890625,
"margin_dpo/margin_std": 620.9015502929688,
"step": 606
},
{
"KL/chosen_KL_mean": -739.030029296875,
"KL/mean": -908.6751708984375,
"KL/rejected_KL_mean": -1078.3203125,
"KL/std": 575.9705810546875,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 339.290283203125,
"fcm_dpo/q_t": 0.4394988417625427,
"grad_norm": 23.68671226501465,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.9747462868690491,
"logits/rejected": -0.9673136472702026,
"logps/chosen": -797.114013671875,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -1158.097412109375,
"loss": 1.1971,
"margin_dpo/margin_mean": 339.290283203125,
"margin_dpo/margin_std": 617.9051513671875,
"step": 607
},
{
"KL/chosen_KL_mean": -661.8232421875,
"KL/mean": -859.321044921875,
"KL/rejected_KL_mean": -1056.81884765625,
"KL/std": 510.9901123046875,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 394.99566650390625,
"fcm_dpo/q_t": 0.42847907543182373,
"grad_norm": 27.059940338134766,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -1.0220744609832764,
"logits/rejected": -1.0379165410995483,
"logps/chosen": -719.2740478515625,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1151.59228515625,
"loss": 1.1471,
"margin_dpo/margin_mean": 394.99566650390625,
"margin_dpo/margin_std": 532.2371826171875,
"step": 608
},
{
"KL/chosen_KL_mean": -683.0916748046875,
"KL/mean": -953.2722778320312,
"KL/rejected_KL_mean": -1223.452880859375,
"KL/std": 723.2427368164062,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 540.3612060546875,
"fcm_dpo/q_t": 0.4110172986984253,
"grad_norm": 34.754119873046875,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -1.0745786428451538,
"logits/rejected": -1.1043390035629272,
"logps/chosen": -741.89697265625,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1312.268798828125,
"loss": 1.1054,
"margin_dpo/margin_mean": 540.3612060546875,
"margin_dpo/margin_std": 860.9444580078125,
"step": 609
},
{
"KL/chosen_KL_mean": -675.1845703125,
"KL/mean": -831.5191650390625,
"KL/rejected_KL_mean": -987.8536987304688,
"KL/std": 541.0578002929688,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 312.6691589355469,
"fcm_dpo/q_t": 0.44272029399871826,
"grad_norm": 35.40081787109375,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.96883225440979,
"logits/rejected": -0.9453009366989136,
"logps/chosen": -740.879638671875,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -1071.2591552734375,
"loss": 1.2153,
"margin_dpo/margin_mean": 312.669189453125,
"margin_dpo/margin_std": 619.25634765625,
"step": 610
},
{
"KL/chosen_KL_mean": -716.7320556640625,
"KL/mean": -999.7886962890625,
"KL/rejected_KL_mean": -1282.84521484375,
"KL/std": 708.412841796875,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 566.11328125,
"fcm_dpo/q_t": 0.4043336808681488,
"grad_norm": 31.667987823486328,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -1.024104118347168,
"logits/rejected": -1.0761734247207642,
"logps/chosen": -769.3314819335938,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1369.17626953125,
"loss": 1.0855,
"margin_dpo/margin_mean": 566.11328125,
"margin_dpo/margin_std": 806.8145751953125,
"step": 611
},
{
"KL/chosen_KL_mean": -774.6546630859375,
"KL/mean": -993.3455810546875,
"KL/rejected_KL_mean": -1212.03662109375,
"KL/std": 588.0178833007812,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 437.3819885253906,
"fcm_dpo/q_t": 0.42229294776916504,
"grad_norm": 27.343502044677734,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -1.014435887336731,
"logits/rejected": -1.028846263885498,
"logps/chosen": -833.9783935546875,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1300.3489990234375,
"loss": 1.1351,
"margin_dpo/margin_mean": 437.3819885253906,
"margin_dpo/margin_std": 627.1123046875,
"step": 612
},
{
"KL/chosen_KL_mean": -668.7769775390625,
"KL/mean": -925.552490234375,
"KL/rejected_KL_mean": -1182.3280029296875,
"KL/std": 624.155517578125,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 513.551025390625,
"fcm_dpo/q_t": 0.40929657220840454,
"grad_norm": 32.1468505859375,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -0.9937447309494019,
"logits/rejected": -1.0488264560699463,
"logps/chosen": -728.5069580078125,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1287.435546875,
"loss": 1.0907,
"margin_dpo/margin_mean": 513.5510864257812,
"margin_dpo/margin_std": 658.1903076171875,
"step": 613
},
{
"KL/chosen_KL_mean": -755.2948608398438,
"KL/mean": -1007.0299072265625,
"KL/rejected_KL_mean": -1258.764892578125,
"KL/std": 626.5299072265625,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 503.47003173828125,
"fcm_dpo/q_t": 0.41245073080062866,
"grad_norm": 35.746726989746094,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -0.9777568578720093,
"logits/rejected": -1.0548135042190552,
"logps/chosen": -808.2338256835938,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1363.4443359375,
"loss": 1.1156,
"margin_dpo/margin_mean": 503.47003173828125,
"margin_dpo/margin_std": 753.4693603515625,
"step": 614
},
{
"KL/chosen_KL_mean": -689.6256103515625,
"KL/mean": -910.9967041015625,
"KL/rejected_KL_mean": -1132.367919921875,
"KL/std": 654.9644165039062,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 442.74224853515625,
"fcm_dpo/q_t": 0.4224264621734619,
"grad_norm": 23.212697982788086,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -1.0044828653335571,
"logits/rejected": -1.031577706336975,
"logps/chosen": -755.44287109375,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1227.54541015625,
"loss": 1.1484,
"margin_dpo/margin_mean": 442.74224853515625,
"margin_dpo/margin_std": 733.4412231445312,
"step": 615
},
{
"KL/chosen_KL_mean": -838.1975708007812,
"KL/mean": -1010.1671142578125,
"KL/rejected_KL_mean": -1182.1365966796875,
"KL/std": 552.5067138671875,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 343.93902587890625,
"fcm_dpo/q_t": 0.43850862979888916,
"grad_norm": 26.084741592407227,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -1.1023294925689697,
"logits/rejected": -1.0941028594970703,
"logps/chosen": -903.3304443359375,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1256.837158203125,
"loss": 1.1956,
"margin_dpo/margin_mean": 343.93902587890625,
"margin_dpo/margin_std": 621.4561157226562,
"step": 616
},
{
"KL/chosen_KL_mean": -798.08203125,
"KL/mean": -914.913818359375,
"KL/rejected_KL_mean": -1031.74560546875,
"KL/std": 524.7188720703125,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0007631981279700994,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 233.6636505126953,
"fcm_dpo/q_t": 0.4582458734512329,
"grad_norm": 49.886898040771484,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.9841203093528748,
"logits/rejected": -0.9550663232803345,
"logps/chosen": -861.0875244140625,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -1095.979736328125,
"loss": 1.2706,
"margin_dpo/margin_mean": 233.6636505126953,
"margin_dpo/margin_std": 622.0703125,
"step": 617
},
{
"KL/chosen_KL_mean": -829.4715576171875,
"KL/mean": -1094.8660888671875,
"KL/rejected_KL_mean": -1360.2607421875,
"KL/std": 697.8109130859375,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0007587473373860121,
"fcm_dpo/delta": -0.05866000056266785,
"fcm_dpo/margin": 530.7892456054688,
"fcm_dpo/q_t": 0.40983498096466064,
"grad_norm": 34.46674346923828,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -1.030335545539856,
"logits/rejected": -1.073843240737915,
"logps/chosen": -896.5728759765625,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1452.4140625,
"loss": 1.1343,
"margin_dpo/margin_mean": 530.7892456054688,
"margin_dpo/margin_std": 913.232421875,
"step": 618
},
{
"KL/chosen_KL_mean": -814.1534423828125,
"KL/mean": -1020.8068237304688,
"KL/rejected_KL_mean": -1227.460205078125,
"KL/std": 658.5218505859375,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0007542966050095856,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 413.30682373046875,
"fcm_dpo/q_t": 0.43316370248794556,
"grad_norm": 42.285247802734375,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -1.0271763801574707,
"logits/rejected": -1.0437747240066528,
"logps/chosen": -870.131591796875,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1320.6456298828125,
"loss": 1.1994,
"margin_dpo/margin_mean": 413.30682373046875,
"margin_dpo/margin_std": 883.5606079101562,
"step": 619
},
{
"KL/chosen_KL_mean": -766.2904052734375,
"KL/mean": -959.615966796875,
"KL/rejected_KL_mean": -1152.94140625,
"KL/std": 587.8455810546875,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0007542966050095856,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 386.6510009765625,
"fcm_dpo/q_t": 0.43245673179626465,
"grad_norm": 27.1217041015625,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -1.046451210975647,
"logits/rejected": -1.0556658506393433,
"logps/chosen": -826.087890625,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1231.3521728515625,
"loss": 1.1788,
"margin_dpo/margin_mean": 386.6510314941406,
"margin_dpo/margin_std": 680.88525390625,
"step": 620
},
{
"KL/chosen_KL_mean": -724.4033203125,
"KL/mean": -1034.7967529296875,
"KL/rejected_KL_mean": -1345.1900634765625,
"KL/std": 727.23095703125,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.000746812263969332,
"fcm_dpo/delta": -0.10022050142288208,
"fcm_dpo/margin": 620.78662109375,
"fcm_dpo/q_t": 0.3940914273262024,
"grad_norm": 38.11147689819336,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -1.0878169536590576,
"logits/rejected": -1.14664888381958,
"logps/chosen": -778.337158203125,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1433.5595703125,
"loss": 1.0629,
"margin_dpo/margin_mean": 620.78662109375,
"margin_dpo/margin_std": 812.3016967773438,
"step": 621
},
{
"KL/chosen_KL_mean": -727.606689453125,
"KL/mean": -923.2297973632812,
"KL/rejected_KL_mean": -1118.852783203125,
"KL/std": 539.9659423828125,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0007393279229290783,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 391.2462463378906,
"fcm_dpo/q_t": 0.4339316189289093,
"grad_norm": 23.3463134765625,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.9580384492874146,
"logits/rejected": -0.9486066102981567,
"logps/chosen": -787.8924560546875,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1204.37158203125,
"loss": 1.1786,
"margin_dpo/margin_mean": 391.24627685546875,
"margin_dpo/margin_std": 691.7193603515625,
"step": 622
},
{
"KL/chosen_KL_mean": -779.237548828125,
"KL/mean": -930.7427368164062,
"KL/rejected_KL_mean": -1082.247802734375,
"KL/std": 557.14306640625,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0007393279229290783,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 303.01031494140625,
"fcm_dpo/q_t": 0.4486698806285858,
"grad_norm": 29.320268630981445,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -1.0816329717636108,
"logits/rejected": -1.0815224647521973,
"logps/chosen": -843.39453125,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1167.330810546875,
"loss": 1.2321,
"margin_dpo/margin_mean": 303.01031494140625,
"margin_dpo/margin_std": 666.27685546875,
"step": 623
},
{
"KL/chosen_KL_mean": -772.2291259765625,
"KL/mean": -1019.4240112304688,
"KL/rejected_KL_mean": -1266.61865234375,
"KL/std": 589.1283569335938,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0007393279229290783,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 494.3896484375,
"fcm_dpo/q_t": 0.4135315716266632,
"grad_norm": 35.011497497558594,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -1.0676138401031494,
"logits/rejected": -1.0803065299987793,
"logps/chosen": -844.1478271484375,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1363.750732421875,
"loss": 1.1081,
"margin_dpo/margin_mean": 494.3896484375,
"margin_dpo/margin_std": 643.760498046875,
"step": 624
},
{
"KL/chosen_KL_mean": -739.02734375,
"KL/mean": -1006.8427124023438,
"KL/rejected_KL_mean": -1274.657958984375,
"KL/std": 642.2593994140625,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0007375460118055344,
"fcm_dpo/delta": -0.024159858003258705,
"fcm_dpo/margin": 535.6307373046875,
"fcm_dpo/q_t": 0.4078494906425476,
"grad_norm": 42.963836669921875,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -1.011317491531372,
"logits/rejected": -1.04207181930542,
"logps/chosen": -797.369384765625,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1360.7484130859375,
"loss": 1.0744,
"margin_dpo/margin_mean": 535.6307373046875,
"margin_dpo/margin_std": 594.4256591796875,
"step": 625
},
{
"KL/chosen_KL_mean": -903.0684204101562,
"KL/mean": -1061.467041015625,
"KL/rejected_KL_mean": -1219.86572265625,
"KL/std": 698.789794921875,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0007357641006819904,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 316.7972717285156,
"fcm_dpo/q_t": 0.444003701210022,
"grad_norm": 30.258146286010742,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.9826500415802002,
"logits/rejected": -0.9770318865776062,
"logps/chosen": -978.1810302734375,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1319.054443359375,
"loss": 1.234,
"margin_dpo/margin_mean": 316.7972717285156,
"margin_dpo/margin_std": 695.1666259765625,
"step": 626
},
{
"KL/chosen_KL_mean": -622.5956420898438,
"KL/mean": -943.0328369140625,
"KL/rejected_KL_mean": -1263.469970703125,
"KL/std": 787.0336303710938,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0007278695702552795,
"fcm_dpo/delta": -0.10846550762653351,
"fcm_dpo/margin": 640.8743896484375,
"fcm_dpo/q_t": 0.40123480558395386,
"grad_norm": 27.875293731689453,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.9350000619888306,
"logits/rejected": -1.0127406120300293,
"logps/chosen": -670.3388061523438,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1370.224609375,
"loss": 1.0751,
"margin_dpo/margin_mean": 640.8743896484375,
"margin_dpo/margin_std": 925.1202392578125,
"step": 627
},
{
"KL/chosen_KL_mean": -809.4762573242188,
"KL/mean": -1047.6484375,
"KL/rejected_KL_mean": -1285.82080078125,
"KL/std": 634.0107421875,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 476.3446044921875,
"fcm_dpo/q_t": 0.42106395959854126,
"grad_norm": 28.623077392578125,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -1.049999713897705,
"logits/rejected": -1.072727084159851,
"logps/chosen": -869.6591796875,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1387.37548828125,
"loss": 1.1354,
"margin_dpo/margin_mean": 476.3446044921875,
"margin_dpo/margin_std": 717.890380859375,
"step": 628
},
{
"KL/chosen_KL_mean": -812.48046875,
"KL/mean": -1029.31640625,
"KL/rejected_KL_mean": -1246.15234375,
"KL/std": 613.6525268554688,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 433.6718444824219,
"fcm_dpo/q_t": 0.4258197546005249,
"grad_norm": 36.2744140625,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -1.0528905391693115,
"logits/rejected": -1.0721955299377441,
"logps/chosen": -876.6940307617188,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1337.805908203125,
"loss": 1.1441,
"margin_dpo/margin_mean": 433.6718444824219,
"margin_dpo/margin_std": 607.2192993164062,
"step": 629
},
{
"KL/chosen_KL_mean": -723.0999755859375,
"KL/mean": -842.289306640625,
"KL/rejected_KL_mean": -961.4786987304688,
"KL/std": 608.8505249023438,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 238.37872314453125,
"fcm_dpo/q_t": 0.4632091224193573,
"grad_norm": 34.26353454589844,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -1.034097671508789,
"logits/rejected": -1.011200189590454,
"logps/chosen": -782.3909912109375,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -1045.0770263671875,
"loss": 1.2898,
"margin_dpo/margin_mean": 238.37872314453125,
"margin_dpo/margin_std": 737.2360229492188,
"step": 630
},
{
"KL/chosen_KL_mean": -853.849609375,
"KL/mean": -1046.713623046875,
"KL/rejected_KL_mean": -1239.577392578125,
"KL/std": 606.48828125,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 385.7279052734375,
"fcm_dpo/q_t": 0.43468064069747925,
"grad_norm": 24.484596252441406,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -1.1066150665283203,
"logits/rejected": -1.1154310703277588,
"logps/chosen": -913.30322265625,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1320.529052734375,
"loss": 1.1894,
"margin_dpo/margin_mean": 385.7279052734375,
"margin_dpo/margin_std": 702.0499877929688,
"step": 631
},
{
"KL/chosen_KL_mean": -769.9806518554688,
"KL/mean": -960.3197021484375,
"KL/rejected_KL_mean": -1150.65869140625,
"KL/std": 546.0325317382812,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 380.67803955078125,
"fcm_dpo/q_t": 0.43564653396606445,
"grad_norm": 42.972713470458984,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -1.0557451248168945,
"logits/rejected": -1.0606086254119873,
"logps/chosen": -831.3322143554688,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1236.81884765625,
"loss": 1.192,
"margin_dpo/margin_mean": 380.67803955078125,
"margin_dpo/margin_std": 693.4515380859375,
"step": 632
},
{
"KL/chosen_KL_mean": -814.6301879882812,
"KL/mean": -957.8812255859375,
"KL/rejected_KL_mean": -1101.13232421875,
"KL/std": 546.3907470703125,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 286.50201416015625,
"fcm_dpo/q_t": 0.45074015855789185,
"grad_norm": 33.58137130737305,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -1.1074434518814087,
"logits/rejected": -1.1195733547210693,
"logps/chosen": -871.9083251953125,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1192.71630859375,
"loss": 1.2413,
"margin_dpo/margin_mean": 286.5020446777344,
"margin_dpo/margin_std": 632.80322265625,
"step": 633
},
{
"KL/chosen_KL_mean": -923.3136596679688,
"KL/mean": -1051.839599609375,
"KL/rejected_KL_mean": -1180.3656005859375,
"KL/std": 654.6397705078125,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 257.0519714355469,
"fcm_dpo/q_t": 0.456850528717041,
"grad_norm": 22.398880004882812,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.9894505739212036,
"logits/rejected": -0.9836434721946716,
"logps/chosen": -989.9326171875,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1287.4912109375,
"loss": 1.2595,
"margin_dpo/margin_mean": 257.05194091796875,
"margin_dpo/margin_std": 630.111572265625,
"step": 634
},
{
"KL/chosen_KL_mean": -822.2686767578125,
"KL/mean": -1000.0654296875,
"KL/rejected_KL_mean": -1177.862060546875,
"KL/std": 712.7052001953125,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 355.593505859375,
"fcm_dpo/q_t": 0.4414767026901245,
"grad_norm": 33.33140182495117,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -1.037980079650879,
"logits/rejected": -1.0179985761642456,
"logps/chosen": -896.2242431640625,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1260.362548828125,
"loss": 1.209,
"margin_dpo/margin_mean": 355.593505859375,
"margin_dpo/margin_std": 720.908447265625,
"step": 635
},
{
"KL/chosen_KL_mean": -765.69140625,
"KL/mean": -987.3884887695312,
"KL/rejected_KL_mean": -1209.085693359375,
"KL/std": 667.54638671875,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0007199749816209078,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 443.39422607421875,
"fcm_dpo/q_t": 0.42743438482284546,
"grad_norm": 24.014665603637695,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.9858390092849731,
"logits/rejected": -0.9905164241790771,
"logps/chosen": -825.3203125,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1291.064453125,
"loss": 1.1495,
"margin_dpo/margin_mean": 443.39422607421875,
"margin_dpo/margin_std": 695.56005859375,
"step": 636
},
{
"KL/chosen_KL_mean": -779.787841796875,
"KL/mean": -1034.97900390625,
"KL/rejected_KL_mean": -1290.170166015625,
"KL/std": 696.4139404296875,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0007171693141572177,
"fcm_dpo/delta": -0.01952262967824936,
"fcm_dpo/margin": 510.3822937011719,
"fcm_dpo/q_t": 0.41653138399124146,
"grad_norm": 25.295166015625,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -1.0853636264801025,
"logits/rejected": -1.139721393585205,
"logps/chosen": -829.4405517578125,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1388.5751953125,
"loss": 1.1162,
"margin_dpo/margin_mean": 510.3822937011719,
"margin_dpo/margin_std": 726.550537109375,
"step": 637
},
{
"KL/chosen_KL_mean": -763.474609375,
"KL/mean": -958.1444091796875,
"KL/rejected_KL_mean": -1152.814208984375,
"KL/std": 652.8692626953125,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0007153811166062951,
"fcm_dpo/delta": -0.024996092543005943,
"fcm_dpo/margin": 389.33966064453125,
"fcm_dpo/q_t": 0.4316685199737549,
"grad_norm": 34.41019821166992,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.9672867655754089,
"logits/rejected": -0.9581058025360107,
"logps/chosen": -821.6312255859375,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1232.11572265625,
"loss": 1.2066,
"margin_dpo/margin_mean": 389.33966064453125,
"margin_dpo/margin_std": 788.7205810546875,
"step": 638
},
{
"KL/chosen_KL_mean": -1031.480224609375,
"KL/mean": -1104.28857421875,
"KL/rejected_KL_mean": -1177.0970458984375,
"KL/std": 619.0423583984375,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0007135929772630334,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.61685180664062,
"fcm_dpo/q_t": 0.4758584499359131,
"grad_norm": 83.87274169921875,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.9700315594673157,
"logits/rejected": -0.910193920135498,
"logps/chosen": -1103.803466796875,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1251.3719482421875,
"loss": 1.3523,
"margin_dpo/margin_mean": 145.61685180664062,
"margin_dpo/margin_std": 737.2562866210938,
"step": 639
},
{
"KL/chosen_KL_mean": -792.1627197265625,
"KL/mean": -1085.340576171875,
"KL/rejected_KL_mean": -1378.5186767578125,
"KL/std": 706.311767578125,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0007101770024746656,
"fcm_dpo/delta": -0.048100586980581284,
"fcm_dpo/margin": 586.3558959960938,
"fcm_dpo/q_t": 0.4069620370864868,
"grad_norm": 31.302490234375,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.959877073764801,
"logits/rejected": -1.0006804466247559,
"logps/chosen": -848.2970581054688,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1487.1187744140625,
"loss": 1.0927,
"margin_dpo/margin_mean": 586.3558959960938,
"margin_dpo/margin_std": 834.8697509765625,
"step": 640
},
{
"KL/chosen_KL_mean": -899.358642578125,
"KL/mean": -1098.58837890625,
"KL/rejected_KL_mean": -1297.818115234375,
"KL/std": 614.996337890625,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 398.45941162109375,
"fcm_dpo/q_t": 0.43619343638420105,
"grad_norm": 32.60629653930664,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -1.0186073780059814,
"logits/rejected": -1.039933443069458,
"logps/chosen": -964.3555908203125,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1384.810546875,
"loss": 1.2018,
"margin_dpo/margin_mean": 398.45941162109375,
"margin_dpo/margin_std": 817.0283203125,
"step": 641
},
{
"KL/chosen_KL_mean": -874.2130126953125,
"KL/mean": -1114.15283203125,
"KL/rejected_KL_mean": -1354.0927734375,
"KL/std": 816.8272705078125,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 479.8797912597656,
"fcm_dpo/q_t": 0.4284675419330597,
"grad_norm": 34.0765380859375,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.9778116941452026,
"logits/rejected": -1.0190857648849487,
"logps/chosen": -939.9022216796875,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1464.3349609375,
"loss": 1.1809,
"margin_dpo/margin_mean": 479.87982177734375,
"margin_dpo/margin_std": 996.5196533203125,
"step": 642
},
{
"KL/chosen_KL_mean": -800.347900390625,
"KL/mean": -969.1351318359375,
"KL/rejected_KL_mean": -1137.92236328125,
"KL/std": 588.9737548828125,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 337.57440185546875,
"fcm_dpo/q_t": 0.4438466429710388,
"grad_norm": 34.371337890625,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -1.1373050212860107,
"logits/rejected": -1.135626196861267,
"logps/chosen": -852.2979125976562,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1225.390625,
"loss": 1.2082,
"margin_dpo/margin_mean": 337.57440185546875,
"margin_dpo/margin_std": 598.9072265625,
"step": 643
},
{
"KL/chosen_KL_mean": -768.4283447265625,
"KL/mean": -946.823974609375,
"KL/rejected_KL_mean": -1125.2196044921875,
"KL/std": 693.099609375,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 356.79119873046875,
"fcm_dpo/q_t": 0.44256871938705444,
"grad_norm": 40.267459869384766,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -1.0229220390319824,
"logits/rejected": -1.031947374343872,
"logps/chosen": -827.446044921875,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1212.356201171875,
"loss": 1.2272,
"margin_dpo/margin_mean": 356.79119873046875,
"margin_dpo/margin_std": 801.4711303710938,
"step": 644
},
{
"KL/chosen_KL_mean": -819.4630737304688,
"KL/mean": -900.1171875,
"KL/rejected_KL_mean": -980.771240234375,
"KL/std": 550.095703125,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 161.30816650390625,
"fcm_dpo/q_t": 0.4736112058162689,
"grad_norm": 67.07617950439453,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.9778566956520081,
"logits/rejected": -0.9598023891448975,
"logps/chosen": -875.339111328125,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -1078.552001953125,
"loss": 1.3464,
"margin_dpo/margin_mean": 161.3081817626953,
"margin_dpo/margin_std": 739.6782836914062,
"step": 645
},
{
"KL/chosen_KL_mean": -777.6273193359375,
"KL/mean": -928.5137939453125,
"KL/rejected_KL_mean": -1079.4002685546875,
"KL/std": 526.18310546875,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 301.7729797363281,
"fcm_dpo/q_t": 0.44995421171188354,
"grad_norm": 36.729408264160156,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -1.0355250835418701,
"logits/rejected": -1.019197940826416,
"logps/chosen": -838.903076171875,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -1156.906005859375,
"loss": 1.2347,
"margin_dpo/margin_mean": 301.7729797363281,
"margin_dpo/margin_std": 637.1296997070312,
"step": 646
},
{
"KL/chosen_KL_mean": -677.3515625,
"KL/mean": -878.120361328125,
"KL/rejected_KL_mean": -1078.88916015625,
"KL/std": 601.9970703125,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 401.5375671386719,
"fcm_dpo/q_t": 0.4323872923851013,
"grad_norm": 28.29328155517578,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.9465760588645935,
"logits/rejected": -0.9807703495025635,
"logps/chosen": -732.2039794921875,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1172.408447265625,
"loss": 1.1631,
"margin_dpo/margin_mean": 401.53753662109375,
"margin_dpo/margin_std": 573.2171020507812,
"step": 647
},
{
"KL/chosen_KL_mean": -734.8883056640625,
"KL/mean": -1029.216064453125,
"KL/rejected_KL_mean": -1323.543701171875,
"KL/std": 709.6268310546875,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 588.655517578125,
"fcm_dpo/q_t": 0.4075320065021515,
"grad_norm": 26.580156326293945,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -1.0718775987625122,
"logits/rejected": -1.1320219039916992,
"logps/chosen": -789.0597534179688,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1422.256591796875,
"loss": 1.0914,
"margin_dpo/margin_mean": 588.6554565429688,
"margin_dpo/margin_std": 848.3179931640625,
"step": 648
},
{
"KL/chosen_KL_mean": -751.6907958984375,
"KL/mean": -895.4127197265625,
"KL/rejected_KL_mean": -1039.134765625,
"KL/std": 570.4094848632812,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 287.4439697265625,
"fcm_dpo/q_t": 0.4549896717071533,
"grad_norm": 25.27387237548828,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -1.1144146919250488,
"logits/rejected": -1.115525722503662,
"logps/chosen": -814.171142578125,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -1119.2119140625,
"loss": 1.2578,
"margin_dpo/margin_mean": 287.4439697265625,
"margin_dpo/margin_std": 714.5806274414062,
"step": 649
},
{
"KL/chosen_KL_mean": -815.21728515625,
"KL/mean": -1038.905517578125,
"KL/rejected_KL_mean": -1262.5936279296875,
"KL/std": 688.6070556640625,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 447.37640380859375,
"fcm_dpo/q_t": 0.4277136027812958,
"grad_norm": 28.098812103271484,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -1.0027992725372314,
"logits/rejected": -1.0247873067855835,
"logps/chosen": -871.31005859375,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1360.8585205078125,
"loss": 1.1696,
"margin_dpo/margin_mean": 447.3763427734375,
"margin_dpo/margin_std": 797.6195068359375,
"step": 650
},
{
"KL/chosen_KL_mean": -560.206787109375,
"KL/mean": -845.2562255859375,
"KL/rejected_KL_mean": -1130.3057861328125,
"KL/std": 642.8323974609375,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 570.0989990234375,
"fcm_dpo/q_t": 0.4065355062484741,
"grad_norm": 31.23227310180664,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -1.0280869007110596,
"logits/rejected": -1.0860235691070557,
"logps/chosen": -603.6322021484375,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1230.263671875,
"loss": 1.0737,
"margin_dpo/margin_mean": 570.0989990234375,
"margin_dpo/margin_std": 660.94873046875,
"step": 651
},
{
"KL/chosen_KL_mean": -692.7413940429688,
"KL/mean": -903.6035766601562,
"KL/rejected_KL_mean": -1114.4656982421875,
"KL/std": 650.1327514648438,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 421.72430419921875,
"fcm_dpo/q_t": 0.4312537908554077,
"grad_norm": 29.79759979248047,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -1.0472981929779053,
"logits/rejected": -1.076317548751831,
"logps/chosen": -755.3182373046875,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1226.2335205078125,
"loss": 1.176,
"margin_dpo/margin_mean": 421.7243347167969,
"margin_dpo/margin_std": 753.9987182617188,
"step": 652
},
{
"KL/chosen_KL_mean": -857.264892578125,
"KL/mean": -1053.7757568359375,
"KL/rejected_KL_mean": -1250.28662109375,
"KL/std": 699.4908447265625,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 393.0216979980469,
"fcm_dpo/q_t": 0.4362325668334961,
"grad_norm": 27.61276626586914,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -1.0763492584228516,
"logits/rejected": -1.103668451309204,
"logps/chosen": -918.3778686523438,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1353.5361328125,
"loss": 1.1908,
"margin_dpo/margin_mean": 393.021728515625,
"margin_dpo/margin_std": 730.29736328125,
"step": 653
},
{
"KL/chosen_KL_mean": -755.6949462890625,
"KL/mean": -948.0558471679688,
"KL/rejected_KL_mean": -1140.416748046875,
"KL/std": 557.1722412109375,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 384.72174072265625,
"fcm_dpo/q_t": 0.4374183714389801,
"grad_norm": 26.79352378845215,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -1.0465385913848877,
"logits/rejected": -1.0679962635040283,
"logps/chosen": -817.423095703125,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1239.1905517578125,
"loss": 1.1862,
"margin_dpo/margin_mean": 384.72174072265625,
"margin_dpo/margin_std": 672.5704345703125,
"step": 654
},
{
"KL/chosen_KL_mean": -700.713134765625,
"KL/mean": -898.3726806640625,
"KL/rejected_KL_mean": -1096.0322265625,
"KL/std": 572.2005615234375,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0007067610276862979,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 395.3191223144531,
"fcm_dpo/q_t": 0.4351748824119568,
"grad_norm": 37.17662048339844,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -1.0191048383712769,
"logits/rejected": -1.0588314533233643,
"logps/chosen": -750.2899169921875,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1194.3240966796875,
"loss": 1.1804,
"margin_dpo/margin_mean": 395.3191223144531,
"margin_dpo/margin_std": 675.3500366210938,
"step": 655
},
{
"KL/chosen_KL_mean": -771.8296508789062,
"KL/mean": -1122.322265625,
"KL/rejected_KL_mean": -1472.814697265625,
"KL/std": 778.0101318359375,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0007016347954049706,
"fcm_dpo/delta": -0.07306308299303055,
"fcm_dpo/margin": 700.985107421875,
"fcm_dpo/q_t": 0.38852617144584656,
"grad_norm": 64.08390045166016,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.9458581209182739,
"logits/rejected": -1.018227219581604,
"logps/chosen": -824.3790893554688,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1586.489501953125,
"loss": 1.0211,
"margin_dpo/margin_mean": 700.985107421875,
"margin_dpo/margin_std": 782.2880859375,
"step": 656
},
{
"KL/chosen_KL_mean": -732.1683959960938,
"KL/mean": -1004.7838745117188,
"KL/rejected_KL_mean": -1277.3992919921875,
"KL/std": 734.6845703125,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 545.23095703125,
"fcm_dpo/q_t": 0.4126076102256775,
"grad_norm": 33.66898727416992,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -1.0927517414093018,
"logits/rejected": -1.140802025794983,
"logps/chosen": -778.868896484375,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1375.314208984375,
"loss": 1.1085,
"margin_dpo/margin_mean": 545.23095703125,
"margin_dpo/margin_std": 772.8052978515625,
"step": 657
},
{
"KL/chosen_KL_mean": -780.8671264648438,
"KL/mean": -991.93359375,
"KL/rejected_KL_mean": -1203.0001220703125,
"KL/std": 568.8956909179688,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 422.13299560546875,
"fcm_dpo/q_t": 0.43089789152145386,
"grad_norm": 30.343259811401367,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -1.0211938619613647,
"logits/rejected": -1.0497910976409912,
"logps/chosen": -841.8253173828125,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1298.9395751953125,
"loss": 1.1636,
"margin_dpo/margin_mean": 422.13299560546875,
"margin_dpo/margin_std": 647.49755859375,
"step": 658
},
{
"KL/chosen_KL_mean": -705.3890991210938,
"KL/mean": -921.2344970703125,
"KL/rejected_KL_mean": -1137.079833984375,
"KL/std": 596.189208984375,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 431.69073486328125,
"fcm_dpo/q_t": 0.4299265444278717,
"grad_norm": 36.30439758300781,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.9837784767150879,
"logits/rejected": -0.964771568775177,
"logps/chosen": -782.132080078125,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1224.55078125,
"loss": 1.1541,
"margin_dpo/margin_mean": 431.690673828125,
"margin_dpo/margin_std": 629.0660400390625,
"step": 659
},
{
"KL/chosen_KL_mean": -754.4503173828125,
"KL/mean": -1027.2886962890625,
"KL/rejected_KL_mean": -1300.127197265625,
"KL/std": 671.2972412109375,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 545.6766967773438,
"fcm_dpo/q_t": 0.4120747745037079,
"grad_norm": 35.72873306274414,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -1.084972620010376,
"logits/rejected": -1.1093543767929077,
"logps/chosen": -813.4982299804688,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1376.087158203125,
"loss": 1.1026,
"margin_dpo/margin_mean": 545.6766967773438,
"margin_dpo/margin_std": 731.752685546875,
"step": 660
},
{
"KL/chosen_KL_mean": -672.3802490234375,
"KL/mean": -915.9332275390625,
"KL/rejected_KL_mean": -1159.486083984375,
"KL/std": 750.931640625,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 487.10589599609375,
"fcm_dpo/q_t": 0.42373281717300415,
"grad_norm": 40.01762008666992,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -1.0510165691375732,
"logits/rejected": -1.0702614784240723,
"logps/chosen": -723.05419921875,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -1245.4918212890625,
"loss": 1.1354,
"margin_dpo/margin_mean": 487.10589599609375,
"margin_dpo/margin_std": 751.4103393554688,
"step": 661
},
{
"KL/chosen_KL_mean": -763.1075439453125,
"KL/mean": -949.8360595703125,
"KL/rejected_KL_mean": -1136.5645751953125,
"KL/std": 620.0942993164062,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 373.45697021484375,
"fcm_dpo/q_t": 0.4395660161972046,
"grad_norm": 23.511491775512695,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -0.9717986583709717,
"logits/rejected": -0.9751859903335571,
"logps/chosen": -832.3685913085938,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1225.6204833984375,
"loss": 1.2118,
"margin_dpo/margin_mean": 373.45697021484375,
"margin_dpo/margin_std": 774.0943603515625,
"step": 662
},
{
"KL/chosen_KL_mean": -730.2545166015625,
"KL/mean": -961.2086181640625,
"KL/rejected_KL_mean": -1192.162841796875,
"KL/std": 697.215087890625,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 461.908203125,
"fcm_dpo/q_t": 0.42762649059295654,
"grad_norm": 23.27935791015625,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -1.0581246614456177,
"logits/rejected": -1.0871996879577637,
"logps/chosen": -795.1334228515625,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1306.088134765625,
"loss": 1.1632,
"margin_dpo/margin_mean": 461.908203125,
"margin_dpo/margin_std": 824.1527099609375,
"step": 663
},
{
"KL/chosen_KL_mean": -802.4429931640625,
"KL/mean": -1063.125244140625,
"KL/rejected_KL_mean": -1323.807373046875,
"KL/std": 709.4827270507812,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 521.3643798828125,
"fcm_dpo/q_t": 0.4172210693359375,
"grad_norm": 22.374834060668945,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -1.033679485321045,
"logits/rejected": -1.0704450607299805,
"logps/chosen": -863.3314208984375,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1429.3291015625,
"loss": 1.1229,
"margin_dpo/margin_mean": 521.3643798828125,
"margin_dpo/margin_std": 771.2794189453125,
"step": 664
},
{
"KL/chosen_KL_mean": -692.1058959960938,
"KL/mean": -907.7689208984375,
"KL/rejected_KL_mean": -1123.4320068359375,
"KL/std": 584.74462890625,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 431.32611083984375,
"fcm_dpo/q_t": 0.42915087938308716,
"grad_norm": 37.84297561645508,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.1375402212142944,
"logits/rejected": -1.1711466312408447,
"logps/chosen": -752.6700439453125,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1208.24072265625,
"loss": 1.1491,
"margin_dpo/margin_mean": 431.32611083984375,
"margin_dpo/margin_std": 589.2435913085938,
"step": 665
},
{
"KL/chosen_KL_mean": -727.8260498046875,
"KL/mean": -953.4937744140625,
"KL/rejected_KL_mean": -1179.161376953125,
"KL/std": 599.885986328125,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 451.33538818359375,
"fcm_dpo/q_t": 0.4272992014884949,
"grad_norm": 32.65074920654297,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.9382809400558472,
"logits/rejected": -0.9508320689201355,
"logps/chosen": -792.2459716796875,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1275.052978515625,
"loss": 1.1502,
"margin_dpo/margin_mean": 451.33538818359375,
"margin_dpo/margin_std": 672.0390625,
"step": 666
},
{
"KL/chosen_KL_mean": -848.9732666015625,
"KL/mean": -1107.0611572265625,
"KL/rejected_KL_mean": -1365.1490478515625,
"KL/std": 657.8707275390625,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 516.17578125,
"fcm_dpo/q_t": 0.4165239930152893,
"grad_norm": 35.096256256103516,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.9879491925239563,
"logits/rejected": -0.9914569854736328,
"logps/chosen": -918.250244140625,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1452.9844970703125,
"loss": 1.1182,
"margin_dpo/margin_mean": 516.17578125,
"margin_dpo/margin_std": 720.5601196289062,
"step": 667
},
{
"KL/chosen_KL_mean": -890.6829223632812,
"KL/mean": -1073.6181640625,
"KL/rejected_KL_mean": -1256.55322265625,
"KL/std": 695.8572998046875,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 365.870361328125,
"fcm_dpo/q_t": 0.4426293969154358,
"grad_norm": 37.71943283081055,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -1.043158769607544,
"logits/rejected": -1.0555496215820312,
"logps/chosen": -963.2869262695312,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1360.292236328125,
"loss": 1.2257,
"margin_dpo/margin_mean": 365.870361328125,
"margin_dpo/margin_std": 822.745361328125,
"step": 668
},
{
"KL/chosen_KL_mean": -685.7474975585938,
"KL/mean": -940.034423828125,
"KL/rejected_KL_mean": -1194.3214111328125,
"KL/std": 647.0894775390625,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0006965085049159825,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 508.57391357421875,
"fcm_dpo/q_t": 0.41749686002731323,
"grad_norm": 23.442853927612305,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -1.006242036819458,
"logits/rejected": -1.0165035724639893,
"logps/chosen": -731.8638916015625,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1272.245849609375,
"loss": 1.1237,
"margin_dpo/margin_mean": 508.57391357421875,
"margin_dpo/margin_std": 728.7122192382812,
"step": 669
},
{
"KL/chosen_KL_mean": -646.3446044921875,
"KL/mean": -914.180419921875,
"KL/rejected_KL_mean": -1182.01611328125,
"KL/std": 602.09423828125,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.0006949029630050063,
"fcm_dpo/delta": -0.023104503750801086,
"fcm_dpo/margin": 535.6715087890625,
"fcm_dpo/q_t": 0.4133542776107788,
"grad_norm": 20.55622673034668,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.9502823948860168,
"logits/rejected": -0.9757121801376343,
"logps/chosen": -708.6904296875,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1278.9566650390625,
"loss": 1.1106,
"margin_dpo/margin_mean": 535.6715087890625,
"margin_dpo/margin_std": 729.982177734375,
"step": 670
},
{
"KL/chosen_KL_mean": -812.72900390625,
"KL/mean": -1021.0582275390625,
"KL/rejected_KL_mean": -1229.387451171875,
"KL/std": 600.5374145507812,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0006932974210940301,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 416.6583251953125,
"fcm_dpo/q_t": 0.4308261275291443,
"grad_norm": 28.0316162109375,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -1.0668823719024658,
"logits/rejected": -1.100010633468628,
"logps/chosen": -860.7291259765625,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1313.206787109375,
"loss": 1.1802,
"margin_dpo/margin_mean": 416.65838623046875,
"margin_dpo/margin_std": 731.703857421875,
"step": 671
},
{
"KL/chosen_KL_mean": -910.474853515625,
"KL/mean": -1116.897705078125,
"KL/rejected_KL_mean": -1323.320556640625,
"KL/std": 720.9313354492188,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0006932974210940301,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 412.8456115722656,
"fcm_dpo/q_t": 0.4346805214881897,
"grad_norm": 42.60050964355469,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.1313029527664185,
"logits/rejected": -1.1380281448364258,
"logps/chosen": -969.0581665039062,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1416.460693359375,
"loss": 1.1939,
"margin_dpo/margin_mean": 412.84564208984375,
"margin_dpo/margin_std": 798.4884033203125,
"step": 672
},
{
"KL/chosen_KL_mean": -806.967529296875,
"KL/mean": -1001.2117919921875,
"KL/rejected_KL_mean": -1195.4560546875,
"KL/std": 610.96337890625,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0006932974210940301,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 388.4884033203125,
"fcm_dpo/q_t": 0.43748611211776733,
"grad_norm": 24.804006576538086,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -1.0505502223968506,
"logits/rejected": -1.054823637008667,
"logps/chosen": -853.6907958984375,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1280.752197265625,
"loss": 1.1877,
"margin_dpo/margin_mean": 388.4884033203125,
"margin_dpo/margin_std": 689.441650390625,
"step": 673
},
{
"KL/chosen_KL_mean": -668.0368041992188,
"KL/mean": -928.5050048828125,
"KL/rejected_KL_mean": -1188.9732666015625,
"KL/std": 601.396240234375,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0006932974210940301,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 520.9365234375,
"fcm_dpo/q_t": 0.41753798723220825,
"grad_norm": 24.077198028564453,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.9701206684112549,
"logits/rejected": -0.9928334951400757,
"logps/chosen": -713.4822998046875,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1259.019287109375,
"loss": 1.1158,
"margin_dpo/margin_mean": 520.9364624023438,
"margin_dpo/margin_std": 724.0059814453125,
"step": 674
},
{
"KL/chosen_KL_mean": -739.5490112304688,
"KL/mean": -1002.5335083007812,
"KL/rejected_KL_mean": -1265.51806640625,
"KL/std": 662.862060546875,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.0006878894637338817,
"fcm_dpo/delta": -0.039154816418886185,
"fcm_dpo/margin": 525.9688720703125,
"fcm_dpo/q_t": 0.4174017012119293,
"grad_norm": 23.40828514099121,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -1.038738489151001,
"logits/rejected": -1.0554476976394653,
"logps/chosen": -783.725341796875,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1339.6099853515625,
"loss": 1.1166,
"margin_dpo/margin_mean": 525.9689331054688,
"margin_dpo/margin_std": 709.8507080078125,
"step": 675
},
{
"KL/chosen_KL_mean": -795.6382446289062,
"KL/mean": -1048.945068359375,
"KL/rejected_KL_mean": -1302.251953125,
"KL/std": 633.6385498046875,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0006878894637338817,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 506.61358642578125,
"fcm_dpo/q_t": 0.4184497594833374,
"grad_norm": 25.839481353759766,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -0.9737539291381836,
"logits/rejected": -0.9969291687011719,
"logps/chosen": -867.0367431640625,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1390.610595703125,
"loss": 1.1168,
"margin_dpo/margin_mean": 506.6136169433594,
"margin_dpo/margin_std": 650.7418823242188,
"step": 676
},
{
"KL/chosen_KL_mean": -806.3810424804688,
"KL/mean": -1042.6671142578125,
"KL/rejected_KL_mean": -1278.953125,
"KL/std": 644.4155883789062,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0006878894637338817,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 472.57208251953125,
"fcm_dpo/q_t": 0.4279462397098541,
"grad_norm": 28.72382164001465,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -1.048870325088501,
"logits/rejected": -1.046755075454712,
"logps/chosen": -862.908447265625,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1357.1796875,
"loss": 1.1547,
"margin_dpo/margin_mean": 472.57208251953125,
"margin_dpo/margin_std": 792.822998046875,
"step": 677
},
{
"KL/chosen_KL_mean": -672.162841796875,
"KL/mean": -966.9105834960938,
"KL/rejected_KL_mean": -1261.658447265625,
"KL/std": 726.8078002929688,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.0006846521864645183,
"fcm_dpo/delta": -0.04728377237915993,
"fcm_dpo/margin": 589.4955444335938,
"fcm_dpo/q_t": 0.40849733352661133,
"grad_norm": 29.151065826416016,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -0.9713025093078613,
"logits/rejected": -1.014235496520996,
"logps/chosen": -718.2973022460938,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1342.262939453125,
"loss": 1.0989,
"margin_dpo/margin_mean": 589.4955444335938,
"margin_dpo/margin_std": 841.677001953125,
"step": 678
},
{
"KL/chosen_KL_mean": -776.6614990234375,
"KL/mean": -964.9478149414062,
"KL/rejected_KL_mean": -1153.234130859375,
"KL/std": 569.8460693359375,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0006814149091951549,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 376.57269287109375,
"fcm_dpo/q_t": 0.44045108556747437,
"grad_norm": 29.10932731628418,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -1.0045051574707031,
"logits/rejected": -0.9941396117210388,
"logps/chosen": -826.9564208984375,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1229.832275390625,
"loss": 1.1964,
"margin_dpo/margin_mean": 376.57269287109375,
"margin_dpo/margin_std": 676.597412109375,
"step": 679
},
{
"KL/chosen_KL_mean": -779.4834594726562,
"KL/mean": -1044.25634765625,
"KL/rejected_KL_mean": -1309.029296875,
"KL/std": 738.444091796875,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0006786106387153268,
"fcm_dpo/delta": -0.020619181916117668,
"fcm_dpo/margin": 529.5457763671875,
"fcm_dpo/q_t": 0.4161868691444397,
"grad_norm": 33.955841064453125,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -0.9959018230438232,
"logits/rejected": -1.0296885967254639,
"logps/chosen": -856.399169921875,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1421.4140625,
"loss": 1.1302,
"margin_dpo/margin_mean": 529.5457763671875,
"margin_dpo/margin_std": 809.1583251953125,
"step": 680
},
{
"KL/chosen_KL_mean": -775.05322265625,
"KL/mean": -983.98583984375,
"KL/rejected_KL_mean": -1192.91845703125,
"KL/std": 620.6334838867188,
"epoch": 1.0,
"fcm_dpo/beta": 0.0006786106387153268,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 417.8653564453125,
"fcm_dpo/q_t": 0.4342602491378784,
"grad_norm": 24.113101959228516,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -1.0600388050079346,
"logits/rejected": -1.078291654586792,
"logps/chosen": -836.010498046875,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1281.4765625,
"loss": 1.1804,
"margin_dpo/margin_mean": 417.8653259277344,
"margin_dpo/margin_std": 694.8834228515625,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.127045486467,
"train_runtime": 1737.1181,
"train_samples_per_second": 25.098,
"train_steps_per_second": 0.392
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink