{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.00527191162109375, "KL/mean": 0.016706019639968872, "KL/rejected_KL_mean": 0.028141021728515625, "KL/std": 0.272699236869812, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02287006378173828, "fcm_dpo/q_t": 0.5005706548690796, "grad_norm": 83.52816009521484, "learning_rate": 0.0, "logits/chosen": -0.4974287748336792, "logits/rejected": -0.43299180269241333, "logps/chosen": -50.1435661315918, "logps/ref_chosen": -50.14883804321289, "logps/ref_rejected": -74.1280517578125, "logps/rejected": -74.09991455078125, "loss": 1.389, "margin_dpo/margin_mean": -0.02287048101425171, "margin_dpo/margin_std": 0.41920793056488037, "step": 1 }, { "KL/chosen_KL_mean": -0.03498649597167969, "KL/mean": -0.00212840735912323, "KL/rejected_KL_mean": 0.030735015869140625, "KL/std": 0.24797174334526062, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06572261452674866, "fcm_dpo/q_t": 0.5016425848007202, "grad_norm": 72.20118713378906, "learning_rate": 7.246376811594203e-09, "logits/chosen": -0.49536412954330444, "logits/rejected": -0.4594460427761078, "logps/chosen": -52.65568923950195, "logps/ref_chosen": -52.620704650878906, "logps/ref_rejected": -75.30413818359375, "logps/rejected": -75.27340698242188, "loss": 1.3932, "margin_dpo/margin_mean": -0.06572240591049194, "margin_dpo/margin_std": 0.35048407316207886, "step": 2 }, { "KL/chosen_KL_mean": 0.009592056274414062, "KL/mean": -0.00013332068920135498, "KL/rejected_KL_mean": -0.009857177734375, "KL/std": 0.27916020154953003, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.019449442625045776, "fcm_dpo/q_t": 0.4995156228542328, "grad_norm": 70.91877746582031, "learning_rate": 1.4492753623188406e-08, "logits/chosen": -0.4816860258579254, "logits/rejected": -0.4422576129436493, "logps/chosen": -60.97200393676758, "logps/ref_chosen": -60.981597900390625, "logps/ref_rejected": -68.67259216308594, "logps/rejected": -68.68244934082031, "loss": 1.3848, "margin_dpo/margin_mean": 0.019450068473815918, "margin_dpo/margin_std": 0.41192615032196045, "step": 3 }, { "KL/chosen_KL_mean": 0.01334381103515625, "KL/mean": 0.011412113904953003, "KL/rejected_KL_mean": 0.009479522705078125, "KL/std": 0.27838367223739624, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003867626190185547, "fcm_dpo/q_t": 0.4999042749404907, "grad_norm": 72.21875762939453, "learning_rate": 2.1739130434782606e-08, "logits/chosen": -0.4687785506248474, "logits/rejected": -0.4412214756011963, "logps/chosen": -56.75436782836914, "logps/ref_chosen": -56.7677116394043, "logps/ref_rejected": -86.64710998535156, "logps/rejected": -86.63763427734375, "loss": 1.3863, "margin_dpo/margin_mean": 0.0038670599460601807, "margin_dpo/margin_std": 0.4134891629219055, "step": 4 }, { "KL/chosen_KL_mean": -0.00311279296875, "KL/mean": 0.004465892910957336, "KL/rejected_KL_mean": 0.012050628662109375, "KL/std": 0.24947434663772583, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.015161305665969849, "fcm_dpo/q_t": 0.5003792643547058, "grad_norm": 90.08293151855469, "learning_rate": 2.898550724637681e-08, "logits/chosen": -0.5145992040634155, "logits/rejected": -0.4708704352378845, "logps/chosen": -53.86248779296875, "logps/ref_chosen": -53.859375, "logps/ref_rejected": -84.14918518066406, "logps/rejected": -84.13713073730469, "loss": 1.3881, "margin_dpo/margin_mean": -0.0151614248752594, "margin_dpo/margin_std": 0.32647037506103516, "step": 5 }, { "KL/chosen_KL_mean": -0.0019550323486328125, "KL/mean": -0.008544057607650757, "KL/rejected_KL_mean": -0.015132904052734375, "KL/std": 0.27398955821990967, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0131760835647583, "fcm_dpo/q_t": 0.49967002868652344, "grad_norm": 92.1026840209961, "learning_rate": 3.6231884057971014e-08, "logits/chosen": -0.5129680633544922, "logits/rejected": -0.4713151454925537, "logps/chosen": -63.009437561035156, "logps/ref_chosen": -63.007484436035156, "logps/ref_rejected": -92.64534759521484, "logps/rejected": -92.66047668457031, "loss": 1.3854, "margin_dpo/margin_mean": 0.013176411390304565, "margin_dpo/margin_std": 0.4235680103302002, "step": 6 }, { "KL/chosen_KL_mean": 0.002979278564453125, "KL/mean": 0.014112532138824463, "KL/rejected_KL_mean": 0.025249481201171875, "KL/std": 0.30075156688690186, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.022269725799560547, "fcm_dpo/q_t": 0.5005556344985962, "grad_norm": 82.2314682006836, "learning_rate": 4.347826086956521e-08, "logits/chosen": -0.4966992735862732, "logits/rejected": -0.46284064650535583, "logps/chosen": -57.77184295654297, "logps/ref_chosen": -57.774818420410156, "logps/ref_rejected": -103.92059326171875, "logps/rejected": -103.89533996582031, "loss": 1.389, "margin_dpo/margin_mean": -0.02226969599723816, "margin_dpo/margin_std": 0.41930609941482544, "step": 7 }, { "KL/chosen_KL_mean": 0.03544044494628906, "KL/mean": 0.03716662526130676, "KL/rejected_KL_mean": 0.0388946533203125, "KL/std": 0.318942129611969, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0034527182579040527, "fcm_dpo/q_t": 0.5000861883163452, "grad_norm": 78.34169006347656, "learning_rate": 5.0724637681159424e-08, "logits/chosen": -0.5110943913459778, "logits/rejected": -0.48551392555236816, "logps/chosen": -58.68059539794922, "logps/ref_chosen": -58.716033935546875, "logps/ref_rejected": -79.3114242553711, "logps/rejected": -79.27252960205078, "loss": 1.387, "margin_dpo/margin_mean": -0.003452509641647339, "margin_dpo/margin_std": 0.3985304534435272, "step": 8 }, { "KL/chosen_KL_mean": 0.007297515869140625, "KL/mean": -0.008765771985054016, "KL/rejected_KL_mean": -0.0248260498046875, "KL/std": 0.2620963752269745, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03212156891822815, "fcm_dpo/q_t": 0.49919742345809937, "grad_norm": 84.95182037353516, "learning_rate": 5.797101449275362e-08, "logits/chosen": -0.4905078411102295, "logits/rejected": -0.4435485601425171, "logps/chosen": -69.85954284667969, "logps/ref_chosen": -69.8668441772461, "logps/ref_rejected": -99.6026611328125, "logps/rejected": -99.62748718261719, "loss": 1.3835, "margin_dpo/margin_mean": 0.03212141990661621, "margin_dpo/margin_std": 0.38075071573257446, "step": 9 }, { "KL/chosen_KL_mean": 0.015682220458984375, "KL/mean": 0.008813418447971344, "KL/rejected_KL_mean": 0.00194549560546875, "KL/std": 0.24493810534477234, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01373349130153656, "fcm_dpo/q_t": 0.4996568560600281, "grad_norm": 70.63052368164062, "learning_rate": 6.521739130434782e-08, "logits/chosen": -0.47287142276763916, "logits/rejected": -0.4286770224571228, "logps/chosen": -48.342002868652344, "logps/ref_chosen": -48.35768508911133, "logps/ref_rejected": -80.37206268310547, "logps/rejected": -80.3701171875, "loss": 1.3852, "margin_dpo/margin_mean": 0.01373395323753357, "margin_dpo/margin_std": 0.3393193483352661, "step": 10 }, { "KL/chosen_KL_mean": -0.027435302734375, "KL/mean": -0.032692402601242065, "KL/rejected_KL_mean": -0.0379486083984375, "KL/std": 0.2535749673843384, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.010510385036468506, "fcm_dpo/q_t": 0.49973729252815247, "grad_norm": 68.74665069580078, "learning_rate": 7.246376811594203e-08, "logits/chosen": -0.4513794481754303, "logits/rejected": -0.42600423097610474, "logps/chosen": -53.044288635253906, "logps/ref_chosen": -53.01685333251953, "logps/ref_rejected": -87.78038024902344, "logps/rejected": -87.81832885742188, "loss": 1.3856, "margin_dpo/margin_mean": 0.010511070489883423, "margin_dpo/margin_std": 0.3717014193534851, "step": 11 }, { "KL/chosen_KL_mean": -0.03303718566894531, "KL/mean": -0.05988152325153351, "KL/rejected_KL_mean": -0.08672332763671875, "KL/std": 0.31390875577926636, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05368778109550476, "fcm_dpo/q_t": 0.4986579418182373, "grad_norm": 89.02582550048828, "learning_rate": 7.971014492753623e-08, "logits/chosen": -0.5268337726593018, "logits/rejected": -0.4902796745300293, "logps/chosen": -61.83847427368164, "logps/ref_chosen": -61.80543518066406, "logps/ref_rejected": -104.8582763671875, "logps/rejected": -104.94499206542969, "loss": 1.3813, "margin_dpo/margin_mean": 0.05368664860725403, "margin_dpo/margin_std": 0.38752928376197815, "step": 12 }, { "KL/chosen_KL_mean": -0.001468658447265625, "KL/mean": -0.00235767662525177, "KL/rejected_KL_mean": -0.003246307373046875, "KL/std": 0.2623201906681061, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0017842352390289307, "fcm_dpo/q_t": 0.49995577335357666, "grad_norm": 79.24825286865234, "learning_rate": 8.695652173913042e-08, "logits/chosen": -0.4681378901004791, "logits/rejected": -0.43913543224334717, "logps/chosen": -64.26182556152344, "logps/ref_chosen": -64.2603530883789, "logps/ref_rejected": -87.20307922363281, "logps/rejected": -87.20632934570312, "loss": 1.3865, "margin_dpo/margin_mean": 0.0017848014831542969, "margin_dpo/margin_std": 0.3650243878364563, "step": 13 }, { "KL/chosen_KL_mean": 0.018545150756835938, "KL/mean": -0.004325702786445618, "KL/rejected_KL_mean": -0.02719879150390625, "KL/std": 0.2847668528556824, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04574933648109436, "fcm_dpo/q_t": 0.4988548755645752, "grad_norm": 85.14924621582031, "learning_rate": 9.420289855072464e-08, "logits/chosen": -0.4632902145385742, "logits/rejected": -0.4233039319515228, "logps/chosen": -58.09166717529297, "logps/ref_chosen": -58.11021041870117, "logps/ref_rejected": -104.04708099365234, "logps/rejected": -104.07427978515625, "loss": 1.3823, "margin_dpo/margin_mean": 0.04574891924858093, "margin_dpo/margin_std": 0.4520788788795471, "step": 14 }, { "KL/chosen_KL_mean": 0.0231170654296875, "KL/mean": -0.0037177205085754395, "KL/rejected_KL_mean": -0.03055572509765625, "KL/std": 0.24491383135318756, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05367732048034668, "fcm_dpo/q_t": 0.4986582100391388, "grad_norm": 64.11997985839844, "learning_rate": 1.0144927536231885e-07, "logits/chosen": -0.4985060691833496, "logits/rejected": -0.4801892936229706, "logps/chosen": -56.94379425048828, "logps/ref_chosen": -56.96691131591797, "logps/ref_rejected": -80.80863952636719, "logps/rejected": -80.83920288085938, "loss": 1.3813, "margin_dpo/margin_mean": 0.05367732048034668, "margin_dpo/margin_std": 0.3837876319885254, "step": 15 }, { "KL/chosen_KL_mean": 0.0002803802490234375, "KL/mean": -0.022383689880371094, "KL/rejected_KL_mean": -0.0450439453125, "KL/std": 0.26514291763305664, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.045323848724365234, "fcm_dpo/q_t": 0.4988669157028198, "grad_norm": 83.59847259521484, "learning_rate": 1.0869565217391303e-07, "logits/chosen": -0.5230243802070618, "logits/rejected": -0.4819951355457306, "logps/chosen": -61.7396125793457, "logps/ref_chosen": -61.739891052246094, "logps/ref_rejected": -84.36947631835938, "logps/rejected": -84.41452026367188, "loss": 1.3821, "margin_dpo/margin_mean": 0.04532337188720703, "margin_dpo/margin_std": 0.37663698196411133, "step": 16 }, { "KL/chosen_KL_mean": 0.04068756103515625, "KL/mean": -0.002854481339454651, "KL/rejected_KL_mean": -0.046398162841796875, "KL/std": 0.2759934067726135, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08708342909812927, "fcm_dpo/q_t": 0.4978230595588684, "grad_norm": 78.46449279785156, "learning_rate": 1.1594202898550725e-07, "logits/chosen": -0.5030894875526428, "logits/rejected": -0.466357946395874, "logps/chosen": -67.66964721679688, "logps/ref_chosen": -67.71033477783203, "logps/ref_rejected": -85.37865447998047, "logps/rejected": -85.425048828125, "loss": 1.378, "margin_dpo/margin_mean": 0.08708390593528748, "margin_dpo/margin_std": 0.3927149772644043, "step": 17 }, { "KL/chosen_KL_mean": 0.0130767822265625, "KL/mean": -0.0022923946380615234, "KL/rejected_KL_mean": -0.017665863037109375, "KL/std": 0.22370792925357819, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.030742883682250977, "fcm_dpo/q_t": 0.49923163652420044, "grad_norm": 82.09223937988281, "learning_rate": 1.2318840579710146e-07, "logits/chosen": -0.5195785164833069, "logits/rejected": -0.4671386778354645, "logps/chosen": -47.72641372680664, "logps/ref_chosen": -47.7394905090332, "logps/ref_rejected": -75.4722900390625, "logps/rejected": -75.48995208740234, "loss": 1.3834, "margin_dpo/margin_mean": 0.030742555856704712, "margin_dpo/margin_std": 0.27723872661590576, "step": 18 }, { "KL/chosen_KL_mean": 0.00811004638671875, "KL/mean": -0.019911006093025208, "KL/rejected_KL_mean": -0.047924041748046875, "KL/std": 0.21907614171504974, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05602636933326721, "fcm_dpo/q_t": 0.4985996186733246, "grad_norm": 73.40470886230469, "learning_rate": 1.3043478260869563e-07, "logits/chosen": -0.5081275701522827, "logits/rejected": -0.4595046043395996, "logps/chosen": -70.19725036621094, "logps/ref_chosen": -70.20536041259766, "logps/ref_rejected": -89.7575912475586, "logps/rejected": -89.80551147460938, "loss": 1.3809, "margin_dpo/margin_mean": 0.056026577949523926, "margin_dpo/margin_std": 0.2629699409008026, "step": 19 }, { "KL/chosen_KL_mean": -0.0014286041259765625, "KL/mean": -0.05187162756919861, "KL/rejected_KL_mean": -0.1023101806640625, "KL/std": 0.2537362277507782, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10088193416595459, "fcm_dpo/q_t": 0.4974784851074219, "grad_norm": 73.6449203491211, "learning_rate": 1.3768115942028986e-07, "logits/chosen": -0.5645604133605957, "logits/rejected": -0.5095418691635132, "logps/chosen": -50.80467224121094, "logps/ref_chosen": -50.80324172973633, "logps/ref_rejected": -78.82334899902344, "logps/rejected": -78.9256591796875, "loss": 1.3765, "margin_dpo/margin_mean": 0.10088211297988892, "margin_dpo/margin_std": 0.2900768518447876, "step": 20 }, { "KL/chosen_KL_mean": -0.03330421447753906, "KL/mean": -0.08112373948097229, "KL/rejected_KL_mean": -0.12894439697265625, "KL/std": 0.2684672474861145, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09563732147216797, "fcm_dpo/q_t": 0.49761033058166504, "grad_norm": 77.29470825195312, "learning_rate": 1.4492753623188405e-07, "logits/chosen": -0.5119160413742065, "logits/rejected": -0.48931699991226196, "logps/chosen": -50.09632110595703, "logps/ref_chosen": -50.063018798828125, "logps/ref_rejected": -77.86878967285156, "logps/rejected": -77.99774169921875, "loss": 1.3771, "margin_dpo/margin_mean": 0.09563717246055603, "margin_dpo/margin_std": 0.37452447414398193, "step": 21 }, { "KL/chosen_KL_mean": 0.038906097412109375, "KL/mean": -0.027102157473564148, "KL/rejected_KL_mean": -0.093109130859375, "KL/std": 0.253788024187088, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1320144236087799, "fcm_dpo/q_t": 0.49670103192329407, "grad_norm": 84.75688934326172, "learning_rate": 1.5217391304347825e-07, "logits/chosen": -0.5193873643875122, "logits/rejected": -0.4773576855659485, "logps/chosen": -59.01873016357422, "logps/ref_chosen": -59.05763626098633, "logps/ref_rejected": -97.50466918945312, "logps/rejected": -97.5977783203125, "loss": 1.3734, "margin_dpo/margin_mean": 0.13201433420181274, "margin_dpo/margin_std": 0.3331354558467865, "step": 22 }, { "KL/chosen_KL_mean": 0.08184623718261719, "KL/mean": -0.05801086127758026, "KL/rejected_KL_mean": -0.19786643981933594, "KL/std": 0.3483527898788452, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2797093689441681, "fcm_dpo/q_t": 0.49301281571388245, "grad_norm": 80.32367706298828, "learning_rate": 1.5942028985507245e-07, "logits/chosen": -0.48532232642173767, "logits/rejected": -0.4625855088233948, "logps/chosen": -59.995849609375, "logps/ref_chosen": -60.07769775390625, "logps/ref_rejected": -81.13955688476562, "logps/rejected": -81.33741760253906, "loss": 1.359, "margin_dpo/margin_mean": 0.2797098159790039, "margin_dpo/margin_std": 0.42796939611434937, "step": 23 }, { "KL/chosen_KL_mean": 0.01682281494140625, "KL/mean": -0.07509848475456238, "KL/rejected_KL_mean": -0.167022705078125, "KL/std": 0.2538023591041565, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1838463544845581, "fcm_dpo/q_t": 0.49540600180625916, "grad_norm": 83.12644958496094, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.4901927709579468, "logits/rejected": -0.4739952087402344, "logps/chosen": -44.27421188354492, "logps/ref_chosen": -44.29103469848633, "logps/ref_rejected": -99.12521362304688, "logps/rejected": -99.292236328125, "loss": 1.3683, "margin_dpo/margin_mean": 0.1838468313217163, "margin_dpo/margin_std": 0.3329934775829315, "step": 24 }, { "KL/chosen_KL_mean": 0.04872703552246094, "KL/mean": -0.11213195323944092, "KL/rejected_KL_mean": -0.27298736572265625, "KL/std": 0.3789520263671875, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3217122554779053, "fcm_dpo/q_t": 0.49196386337280273, "grad_norm": 73.51007080078125, "learning_rate": 1.7391304347826085e-07, "logits/chosen": -0.5064820051193237, "logits/rejected": -0.4769260883331299, "logps/chosen": -52.48832702636719, "logps/ref_chosen": -52.537052154541016, "logps/ref_rejected": -89.34219360351562, "logps/rejected": -89.61517333984375, "loss": 1.355, "margin_dpo/margin_mean": 0.3217119574546814, "margin_dpo/margin_std": 0.491582453250885, "step": 25 }, { "KL/chosen_KL_mean": 0.11263847351074219, "KL/mean": -0.10163936018943787, "KL/rejected_KL_mean": -0.31591796875, "KL/std": 0.4192284941673279, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4285600185394287, "fcm_dpo/q_t": 0.48929810523986816, "grad_norm": 87.42533874511719, "learning_rate": 1.8115942028985507e-07, "logits/chosen": -0.5297478437423706, "logits/rejected": -0.49793004989624023, "logps/chosen": -53.8101692199707, "logps/ref_chosen": -53.92280578613281, "logps/ref_rejected": -103.35971069335938, "logps/rejected": -103.67562866210938, "loss": 1.3446, "margin_dpo/margin_mean": 0.42856070399284363, "margin_dpo/margin_std": 0.5423732399940491, "step": 26 }, { "KL/chosen_KL_mean": 0.13956642150878906, "KL/mean": -0.14148437976837158, "KL/rejected_KL_mean": -0.4225349426269531, "KL/std": 0.4998953342437744, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5621016025543213, "fcm_dpo/q_t": 0.48596882820129395, "grad_norm": 93.27300262451172, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.5154296159744263, "logits/rejected": -0.47898948192596436, "logps/chosen": -42.75896453857422, "logps/ref_chosen": -42.898529052734375, "logps/ref_rejected": -98.72419738769531, "logps/rejected": -99.14673614501953, "loss": 1.3318, "margin_dpo/margin_mean": 0.5621014833450317, "margin_dpo/margin_std": 0.6122180223464966, "step": 27 }, { "KL/chosen_KL_mean": 0.04203224182128906, "KL/mean": -0.12256798148155212, "KL/rejected_KL_mean": -0.28717041015625, "KL/std": 0.3935839533805847, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3292032480239868, "fcm_dpo/q_t": 0.4917774200439453, "grad_norm": 75.40629577636719, "learning_rate": 1.9565217391304347e-07, "logits/chosen": -0.5347428917884827, "logits/rejected": -0.48221421241760254, "logps/chosen": -60.514469146728516, "logps/ref_chosen": -60.55650329589844, "logps/ref_rejected": -91.40111541748047, "logps/rejected": -91.68828582763672, "loss": 1.3544, "margin_dpo/margin_mean": 0.3292037844657898, "margin_dpo/margin_std": 0.5351283550262451, "step": 28 }, { "KL/chosen_KL_mean": 0.10456466674804688, "KL/mean": -0.1523105800151825, "KL/rejected_KL_mean": -0.4091835021972656, "KL/std": 0.4780128300189972, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5137494802474976, "fcm_dpo/q_t": 0.4871741533279419, "grad_norm": 91.09429931640625, "learning_rate": 2.028985507246377e-07, "logits/chosen": -0.5602052211761475, "logits/rejected": -0.5143990516662598, "logps/chosen": -57.70322036743164, "logps/ref_chosen": -57.80778503417969, "logps/ref_rejected": -97.39434814453125, "logps/rejected": -97.80353546142578, "loss": 1.3365, "margin_dpo/margin_mean": 0.5137503147125244, "margin_dpo/margin_std": 0.600021243095398, "step": 29 }, { "KL/chosen_KL_mean": 0.15185928344726562, "KL/mean": -0.17665454745292664, "KL/rejected_KL_mean": -0.5051727294921875, "KL/std": 0.569696307182312, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6570307016372681, "fcm_dpo/q_t": 0.4835994839668274, "grad_norm": 87.53655242919922, "learning_rate": 2.1014492753623187e-07, "logits/chosen": -0.48063158988952637, "logits/rejected": -0.4493868350982666, "logps/chosen": -52.425514221191406, "logps/ref_chosen": -52.577369689941406, "logps/ref_rejected": -98.48920440673828, "logps/rejected": -98.99437713623047, "loss": 1.3227, "margin_dpo/margin_mean": 0.6570307016372681, "margin_dpo/margin_std": 0.6358605623245239, "step": 30 }, { "KL/chosen_KL_mean": 0.12508392333984375, "KL/mean": -0.11352300643920898, "KL/rejected_KL_mean": -0.3521308898925781, "KL/std": 0.5105932950973511, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.47721391916275024, "fcm_dpo/q_t": 0.4880916476249695, "grad_norm": 68.69351196289062, "learning_rate": 2.1739130434782607e-07, "logits/chosen": -0.5206561088562012, "logits/rejected": -0.47763532400131226, "logps/chosen": -63.68183898925781, "logps/ref_chosen": -63.806922912597656, "logps/ref_rejected": -72.89400482177734, "logps/rejected": -73.24613952636719, "loss": 1.3401, "margin_dpo/margin_mean": 0.4772138297557831, "margin_dpo/margin_std": 0.6101304888725281, "step": 31 }, { "KL/chosen_KL_mean": 0.19174575805664062, "KL/mean": -0.19748108088970184, "KL/rejected_KL_mean": -0.5867080688476562, "KL/std": 0.7203556299209595, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7784509658813477, "fcm_dpo/q_t": 0.48061084747314453, "grad_norm": 82.00362396240234, "learning_rate": 2.2463768115942027e-07, "logits/chosen": -0.5107109546661377, "logits/rejected": -0.4691959619522095, "logps/chosen": -62.54777908325195, "logps/ref_chosen": -62.739524841308594, "logps/ref_rejected": -89.3175048828125, "logps/rejected": -89.90420532226562, "loss": 1.312, "margin_dpo/margin_mean": 0.7784509658813477, "margin_dpo/margin_std": 0.9079021215438843, "step": 32 }, { "KL/chosen_KL_mean": 0.12497520446777344, "KL/mean": -0.18040965497493744, "KL/rejected_KL_mean": -0.4857940673828125, "KL/std": 0.5566126108169556, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6107670068740845, "fcm_dpo/q_t": 0.48475056886672974, "grad_norm": 72.61372375488281, "learning_rate": 2.318840579710145e-07, "logits/chosen": -0.4933505058288574, "logits/rejected": -0.46733930706977844, "logps/chosen": -53.1359977722168, "logps/ref_chosen": -53.26097106933594, "logps/ref_rejected": -87.8851318359375, "logps/rejected": -88.37092590332031, "loss": 1.3271, "margin_dpo/margin_mean": 0.6107674241065979, "margin_dpo/margin_std": 0.6001892685890198, "step": 33 }, { "KL/chosen_KL_mean": 0.0882568359375, "KL/mean": -0.30000877380371094, "KL/rejected_KL_mean": -0.6882743835449219, "KL/std": 0.7007868885993958, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7765321731567383, "fcm_dpo/q_t": 0.48063427209854126, "grad_norm": 78.93215942382812, "learning_rate": 2.391304347826087e-07, "logits/chosen": -0.4917735457420349, "logits/rejected": -0.47427982091903687, "logps/chosen": -50.72907257080078, "logps/ref_chosen": -50.81732940673828, "logps/ref_rejected": -101.92184448242188, "logps/rejected": -102.61012268066406, "loss": 1.3118, "margin_dpo/margin_mean": 0.7765311002731323, "margin_dpo/margin_std": 0.8206242918968201, "step": 34 }, { "KL/chosen_KL_mean": 0.14589881896972656, "KL/mean": -0.4312659204006195, "KL/rejected_KL_mean": -1.0084266662597656, "KL/std": 0.9634122848510742, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1543281078338623, "fcm_dpo/q_t": 0.471314013004303, "grad_norm": 83.42659759521484, "learning_rate": 2.463768115942029e-07, "logits/chosen": -0.5353403687477112, "logits/rejected": -0.4983571767807007, "logps/chosen": -50.87859344482422, "logps/ref_chosen": -51.02449035644531, "logps/ref_rejected": -106.82443237304688, "logps/rejected": -107.83285522460938, "loss": 1.2774, "margin_dpo/margin_mean": 1.1543282270431519, "margin_dpo/margin_std": 1.1236690282821655, "step": 35 }, { "KL/chosen_KL_mean": 0.009319305419921875, "KL/mean": -0.544201135635376, "KL/rejected_KL_mean": -1.0977210998535156, "KL/std": 1.0783436298370361, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1070445775985718, "fcm_dpo/q_t": 0.47246915102005005, "grad_norm": 72.56640625, "learning_rate": 2.536231884057971e-07, "logits/chosen": -0.5723892450332642, "logits/rejected": -0.5367264747619629, "logps/chosen": -51.98217010498047, "logps/ref_chosen": -51.991493225097656, "logps/ref_rejected": -86.0406265258789, "logps/rejected": -87.13835144042969, "loss": 1.2826, "margin_dpo/margin_mean": 1.107043981552124, "margin_dpo/margin_std": 1.2233386039733887, "step": 36 }, { "KL/chosen_KL_mean": 0.03542518615722656, "KL/mean": -0.48510926961898804, "KL/rejected_KL_mean": -1.0056495666503906, "KL/std": 1.0858180522918701, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0410770177841187, "fcm_dpo/q_t": 0.4742150902748108, "grad_norm": 61.9765739440918, "learning_rate": 2.6086956521739126e-07, "logits/chosen": -0.5248156785964966, "logits/rejected": -0.48221951723098755, "logps/chosen": -62.77168273925781, "logps/ref_chosen": -62.807106018066406, "logps/ref_rejected": -77.89507293701172, "logps/rejected": -78.90072631835938, "loss": 1.29, "margin_dpo/margin_mean": 1.0410770177841187, "margin_dpo/margin_std": 1.3847689628601074, "step": 37 }, { "KL/chosen_KL_mean": 0.14590072631835938, "KL/mean": -0.5701018571853638, "KL/rejected_KL_mean": -1.2861061096191406, "KL/std": 1.3780711889266968, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4320064783096313, "fcm_dpo/q_t": 0.46469372510910034, "grad_norm": 69.0830307006836, "learning_rate": 2.681159420289855e-07, "logits/chosen": -0.5364372730255127, "logits/rejected": -0.5037224292755127, "logps/chosen": -48.24462127685547, "logps/ref_chosen": -48.39051818847656, "logps/ref_rejected": -97.91244506835938, "logps/rejected": -99.19854736328125, "loss": 1.2553, "margin_dpo/margin_mean": 1.4320067167282104, "margin_dpo/margin_std": 1.680180549621582, "step": 38 }, { "KL/chosen_KL_mean": 0.06763076782226562, "KL/mean": -0.74673992395401, "KL/rejected_KL_mean": -1.5611076354980469, "KL/std": 1.2961926460266113, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6287341117858887, "fcm_dpo/q_t": 0.4595925509929657, "grad_norm": 73.7347412109375, "learning_rate": 2.753623188405797e-07, "logits/chosen": -0.537227988243103, "logits/rejected": -0.49611860513687134, "logps/chosen": -50.68284225463867, "logps/ref_chosen": -50.75047302246094, "logps/ref_rejected": -78.56951141357422, "logps/rejected": -80.130615234375, "loss": 1.2351, "margin_dpo/margin_mean": 1.6287344694137573, "margin_dpo/margin_std": 1.415596604347229, "step": 39 }, { "KL/chosen_KL_mean": 0.19949913024902344, "KL/mean": -0.5562039613723755, "KL/rejected_KL_mean": -1.3119163513183594, "KL/std": 1.4023091793060303, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5114145278930664, "fcm_dpo/q_t": 0.4627044200897217, "grad_norm": 59.9863395690918, "learning_rate": 2.8260869565217386e-07, "logits/chosen": -0.5215098857879639, "logits/rejected": -0.4915487766265869, "logps/chosen": -57.78556823730469, "logps/ref_chosen": -57.985069274902344, "logps/ref_rejected": -74.3000717163086, "logps/rejected": -75.61198425292969, "loss": 1.2484, "margin_dpo/margin_mean": 1.5114144086837769, "margin_dpo/margin_std": 1.707202672958374, "step": 40 }, { "KL/chosen_KL_mean": 0.02559661865234375, "KL/mean": -0.9471765756607056, "KL/rejected_KL_mean": -1.9199447631835938, "KL/std": 1.8845467567443848, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9455442428588867, "fcm_dpo/q_t": 0.45214998722076416, "grad_norm": 67.88764190673828, "learning_rate": 2.898550724637681e-07, "logits/chosen": -0.5397985577583313, "logits/rejected": -0.5032455325126648, "logps/chosen": -62.67021942138672, "logps/ref_chosen": -62.69581604003906, "logps/ref_rejected": -97.02352905273438, "logps/rejected": -98.94347381591797, "loss": 1.2116, "margin_dpo/margin_mean": 1.9455441236495972, "margin_dpo/margin_std": 2.008014678955078, "step": 41 }, { "KL/chosen_KL_mean": 0.22910499572753906, "KL/mean": -1.0767121315002441, "KL/rejected_KL_mean": -2.382526397705078, "KL/std": 2.184157609939575, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.611631393432617, "fcm_dpo/q_t": 0.4363771677017212, "grad_norm": 78.85198974609375, "learning_rate": 2.971014492753623e-07, "logits/chosen": -0.5462131500244141, "logits/rejected": -0.49966076016426086, "logps/chosen": -58.73731994628906, "logps/ref_chosen": -58.966426849365234, "logps/ref_rejected": -109.90837097167969, "logps/rejected": -112.29090118408203, "loss": 1.1569, "margin_dpo/margin_mean": 2.611631393432617, "margin_dpo/margin_std": 2.447122097015381, "step": 42 }, { "KL/chosen_KL_mean": 0.5043964385986328, "KL/mean": -0.7495235204696655, "KL/rejected_KL_mean": -2.003448486328125, "KL/std": 1.8596460819244385, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.5078420639038086, "fcm_dpo/q_t": 0.4384229779243469, "grad_norm": 71.84793090820312, "learning_rate": 3.043478260869565e-07, "logits/chosen": -0.529945433139801, "logits/rejected": -0.5044955015182495, "logps/chosen": -53.6515998840332, "logps/ref_chosen": -54.15599822998047, "logps/ref_rejected": -96.48019409179688, "logps/rejected": -98.483642578125, "loss": 1.1612, "margin_dpo/margin_mean": 2.5078420639038086, "margin_dpo/margin_std": 2.0080738067626953, "step": 43 }, { "KL/chosen_KL_mean": 0.2628173828125, "KL/mean": -1.1443766355514526, "KL/rejected_KL_mean": -2.5515708923339844, "KL/std": 2.2642464637756348, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.814387321472168, "fcm_dpo/q_t": 0.4311271011829376, "grad_norm": 78.9500503540039, "learning_rate": 3.115942028985507e-07, "logits/chosen": -0.4813832640647888, "logits/rejected": -0.461247056722641, "logps/chosen": -49.81568145751953, "logps/ref_chosen": -50.07849884033203, "logps/ref_rejected": -108.78376007080078, "logps/rejected": -111.3353271484375, "loss": 1.1366, "margin_dpo/margin_mean": 2.8143868446350098, "margin_dpo/margin_std": 2.230133295059204, "step": 44 }, { "KL/chosen_KL_mean": 0.1612529754638672, "KL/mean": -0.9844990968704224, "KL/rejected_KL_mean": -2.1302528381347656, "KL/std": 2.070618152618408, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.291508674621582, "fcm_dpo/q_t": 0.4442916810512543, "grad_norm": 62.898319244384766, "learning_rate": 3.188405797101449e-07, "logits/chosen": -0.47336873412132263, "logits/rejected": -0.4605395793914795, "logps/chosen": -48.25367736816406, "logps/ref_chosen": -48.4149284362793, "logps/ref_rejected": -77.93643188476562, "logps/rejected": -80.06668090820312, "loss": 1.1865, "margin_dpo/margin_mean": 2.291508197784424, "margin_dpo/margin_std": 2.556962013244629, "step": 45 }, { "KL/chosen_KL_mean": 0.16820144653320312, "KL/mean": -1.308139443397522, "KL/rejected_KL_mean": -2.78448486328125, "KL/std": 2.6418280601501465, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9526872634887695, "fcm_dpo/q_t": 0.4291154146194458, "grad_norm": 69.75140380859375, "learning_rate": 3.260869565217391e-07, "logits/chosen": -0.5498015284538269, "logits/rejected": -0.4997956156730652, "logps/chosen": -55.83122253417969, "logps/ref_chosen": -55.999427795410156, "logps/ref_rejected": -95.652587890625, "logps/rejected": -98.43707275390625, "loss": 1.1372, "margin_dpo/margin_mean": 2.9526872634887695, "margin_dpo/margin_std": 3.2058982849121094, "step": 46 }, { "KL/chosen_KL_mean": 0.37381935119628906, "KL/mean": -1.1091471910476685, "KL/rejected_KL_mean": -2.592113494873047, "KL/std": 2.6175854206085205, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9659318923950195, "fcm_dpo/q_t": 0.4280517101287842, "grad_norm": 65.22331237792969, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.5681500434875488, "logits/rejected": -0.5152395963668823, "logps/chosen": -57.55226135253906, "logps/ref_chosen": -57.92607879638672, "logps/ref_rejected": -94.67920684814453, "logps/rejected": -97.27131652832031, "loss": 1.1305, "margin_dpo/margin_mean": 2.965932846069336, "margin_dpo/margin_std": 2.734755039215088, "step": 47 }, { "KL/chosen_KL_mean": 0.06793022155761719, "KL/mean": -1.4929344654083252, "KL/rejected_KL_mean": -3.053802490234375, "KL/std": 2.6948609352111816, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.121734857559204, "fcm_dpo/q_t": 0.4245325028896332, "grad_norm": 73.7392349243164, "learning_rate": 3.4057971014492755e-07, "logits/chosen": -0.6097103357315063, "logits/rejected": -0.5529145002365112, "logps/chosen": -57.12014389038086, "logps/ref_chosen": -57.188072204589844, "logps/ref_rejected": -88.0166015625, "logps/rejected": -91.07040405273438, "loss": 1.1239, "margin_dpo/margin_mean": 3.1217353343963623, "margin_dpo/margin_std": 2.990403652191162, "step": 48 }, { "KL/chosen_KL_mean": 0.31961822509765625, "KL/mean": -1.590240478515625, "KL/rejected_KL_mean": -3.5000991821289062, "KL/std": 3.3908257484436035, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.09978070855140686, "fcm_dpo/delta": -0.021977046504616737, "fcm_dpo/margin": 3.8197124004364014, "fcm_dpo/q_t": 0.41010648012161255, "grad_norm": 60.99309158325195, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.547630786895752, "logits/rejected": -0.48894059658050537, "logps/chosen": -61.36565399169922, "logps/ref_chosen": -61.685272216796875, "logps/ref_rejected": -83.76747131347656, "logps/rejected": -87.267578125, "loss": 1.0779, "margin_dpo/margin_mean": 3.8197121620178223, "margin_dpo/margin_std": 3.9132919311523438, "step": 49 }, { "KL/chosen_KL_mean": -0.1897144317626953, "KL/mean": -2.2449421882629395, "KL/rejected_KL_mean": -4.300167083740234, "KL/std": 3.5077741146087646, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.09924671053886414, "fcm_dpo/delta": -0.015830103307962418, "fcm_dpo/margin": 4.11044454574585, "fcm_dpo/q_t": 0.40354403853416443, "grad_norm": 62.01202392578125, "learning_rate": 3.5507246376811595e-07, "logits/chosen": -0.5537872314453125, "logits/rejected": -0.5180896520614624, "logps/chosen": -58.91385269165039, "logps/ref_chosen": -58.72413635253906, "logps/ref_rejected": -96.35814666748047, "logps/rejected": -100.65830993652344, "loss": 1.057, "margin_dpo/margin_mean": 4.11044454574585, "margin_dpo/margin_std": 4.035107612609863, "step": 50 }, { "KL/chosen_KL_mean": -0.27092552185058594, "KL/mean": -2.297037124633789, "KL/rejected_KL_mean": -4.323150634765625, "KL/std": 4.201503276824951, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.09835103154182434, "fcm_dpo/delta": -0.04532879590988159, "fcm_dpo/margin": 4.052219390869141, "fcm_dpo/q_t": 0.40886732935905457, "grad_norm": 51.64212417602539, "learning_rate": 3.6231884057971015e-07, "logits/chosen": -0.5401347875595093, "logits/rejected": -0.5079134702682495, "logps/chosen": -61.64459228515625, "logps/ref_chosen": -61.3736686706543, "logps/ref_rejected": -76.00199890136719, "logps/rejected": -80.32514953613281, "loss": 1.0861, "margin_dpo/margin_mean": 4.052219867706299, "margin_dpo/margin_std": 5.1329803466796875, "step": 51 }, { "KL/chosen_KL_mean": 0.35683631896972656, "KL/mean": -2.742495536804199, "KL/rejected_KL_mean": -5.841827392578125, "KL/std": 4.826806545257568, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.09542756527662277, "fcm_dpo/delta": -0.20356540381908417, "fcm_dpo/margin": 6.198663234710693, "fcm_dpo/q_t": 0.3626432716846466, "grad_norm": 57.56940841674805, "learning_rate": 3.695652173913043e-07, "logits/chosen": -0.550842821598053, "logits/rejected": -0.495021253824234, "logps/chosen": -51.98052215576172, "logps/ref_chosen": -52.33735656738281, "logps/ref_rejected": -79.97391510009766, "logps/rejected": -85.81574249267578, "loss": 0.9351, "margin_dpo/margin_mean": 6.198663711547852, "margin_dpo/margin_std": 5.227818489074707, "step": 52 }, { "KL/chosen_KL_mean": -0.19112777709960938, "KL/mean": -3.3118577003479004, "KL/rejected_KL_mean": -6.432586669921875, "KL/std": 5.454026699066162, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.09252598136663437, "fcm_dpo/delta": -0.1881176084280014, "fcm_dpo/margin": 6.241460800170898, "fcm_dpo/q_t": 0.3727007806301117, "grad_norm": 56.1482048034668, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.6339062452316284, "logits/rejected": -0.6129721403121948, "logps/chosen": -53.50577926635742, "logps/ref_chosen": -53.31465148925781, "logps/ref_rejected": -91.78359985351562, "logps/rejected": -98.2161865234375, "loss": 0.9727, "margin_dpo/margin_mean": 6.241460800170898, "margin_dpo/margin_std": 6.26479434967041, "step": 53 }, { "KL/chosen_KL_mean": -0.3959083557128906, "KL/mean": -3.1197850704193115, "KL/rejected_KL_mean": -5.843662261962891, "KL/std": 5.133924961090088, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.08979058265686035, "fcm_dpo/delta": -0.09366016089916229, "fcm_dpo/margin": 5.44775390625, "fcm_dpo/q_t": 0.3860328197479248, "grad_norm": 67.31330108642578, "learning_rate": 3.8405797101449274e-07, "logits/chosen": -0.590550422668457, "logits/rejected": -0.5370299816131592, "logps/chosen": -51.084571838378906, "logps/ref_chosen": -50.68865966796875, "logps/ref_rejected": -91.71539306640625, "logps/rejected": -97.55905151367188, "loss": 1.0022, "margin_dpo/margin_mean": 5.447753429412842, "margin_dpo/margin_std": 5.062417030334473, "step": 54 }, { "KL/chosen_KL_mean": -0.9079132080078125, "KL/mean": -4.178384304046631, "KL/rejected_KL_mean": -7.4488525390625, "KL/std": 6.216054439544678, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.08719541132450104, "fcm_dpo/delta": -0.18026816844940186, "fcm_dpo/margin": 6.5409345626831055, "fcm_dpo/q_t": 0.3730924427509308, "grad_norm": 49.59605026245117, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.6388111114501953, "logits/rejected": -0.575678825378418, "logps/chosen": -63.52314758300781, "logps/ref_chosen": -62.615234375, "logps/ref_rejected": -88.99349975585938, "logps/rejected": -96.44235229492188, "loss": 0.9889, "margin_dpo/margin_mean": 6.540935516357422, "margin_dpo/margin_std": 7.489261627197266, "step": 55 }, { "KL/chosen_KL_mean": -0.7456607818603516, "KL/mean": -3.813748836517334, "KL/rejected_KL_mean": -6.8818359375, "KL/std": 6.068477630615234, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.08457927405834198, "fcm_dpo/delta": -0.12541212141513824, "fcm_dpo/margin": 6.136178016662598, "fcm_dpo/q_t": 0.38377007842063904, "grad_norm": 44.31554412841797, "learning_rate": 3.9855072463768114e-07, "logits/chosen": -0.5918300151824951, "logits/rejected": -0.548652708530426, "logps/chosen": -58.67839050292969, "logps/ref_chosen": -57.9327278137207, "logps/ref_rejected": -94.1744384765625, "logps/rejected": -101.0562744140625, "loss": 1.0201, "margin_dpo/margin_mean": 6.136178016662598, "margin_dpo/margin_std": 7.338691711425781, "step": 56 }, { "KL/chosen_KL_mean": -0.7408714294433594, "KL/mean": -4.242293357849121, "KL/rejected_KL_mean": -7.743717193603516, "KL/std": 5.987005233764648, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.08201777935028076, "fcm_dpo/delta": -0.18457511067390442, "fcm_dpo/margin": 7.0028462409973145, "fcm_dpo/q_t": 0.3672080636024475, "grad_norm": 48.90700149536133, "learning_rate": 4.057971014492754e-07, "logits/chosen": -0.5889043807983398, "logits/rejected": -0.5612311363220215, "logps/chosen": -71.23616027832031, "logps/ref_chosen": -70.49528503417969, "logps/ref_rejected": -95.56546020507812, "logps/rejected": -103.30917358398438, "loss": 0.9601, "margin_dpo/margin_mean": 7.002847671508789, "margin_dpo/margin_std": 6.586859703063965, "step": 57 }, { "KL/chosen_KL_mean": -0.9988975524902344, "KL/mean": -4.867576599121094, "KL/rejected_KL_mean": -8.736251831054688, "KL/std": 6.84279727935791, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.078616663813591, "fcm_dpo/delta": -0.22122687101364136, "fcm_dpo/margin": 7.737354278564453, "fcm_dpo/q_t": 0.36434701085090637, "grad_norm": 50.516578674316406, "learning_rate": 4.1304347826086954e-07, "logits/chosen": -0.5851288437843323, "logits/rejected": -0.5081213712692261, "logps/chosen": -63.1318359375, "logps/ref_chosen": -62.13294219970703, "logps/ref_rejected": -84.61729431152344, "logps/rejected": -93.35354614257812, "loss": 0.9582, "margin_dpo/margin_mean": 7.737354278564453, "margin_dpo/margin_std": 8.133914947509766, "step": 58 }, { "KL/chosen_KL_mean": -1.5034770965576172, "KL/mean": -5.772602081298828, "KL/rejected_KL_mean": -10.041728973388672, "KL/std": 7.538321495056152, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.07399021834135056, "fcm_dpo/delta": -0.25073447823524475, "fcm_dpo/margin": 8.53825569152832, "fcm_dpo/q_t": 0.35876306891441345, "grad_norm": 47.74569320678711, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.637381911277771, "logits/rejected": -0.5979089140892029, "logps/chosen": -53.436004638671875, "logps/ref_chosen": -51.932525634765625, "logps/ref_rejected": -88.88520050048828, "logps/rejected": -98.92692565917969, "loss": 0.9422, "margin_dpo/margin_mean": 8.538256645202637, "margin_dpo/margin_std": 8.657896041870117, "step": 59 }, { "KL/chosen_KL_mean": -2.7102012634277344, "KL/mean": -6.095211982727051, "KL/rejected_KL_mean": -9.480220794677734, "KL/std": 6.847818374633789, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.07247201353311539, "fcm_dpo/delta": -0.07915282994508743, "fcm_dpo/margin": 6.77001953125, "fcm_dpo/q_t": 0.3859708905220032, "grad_norm": 50.09484100341797, "learning_rate": 4.2753623188405794e-07, "logits/chosen": -0.6102770566940308, "logits/rejected": -0.5501413941383362, "logps/chosen": -63.65238952636719, "logps/ref_chosen": -60.94218826293945, "logps/ref_rejected": -85.39340209960938, "logps/rejected": -94.87361907958984, "loss": 1.029, "margin_dpo/margin_mean": 6.770020008087158, "margin_dpo/margin_std": 7.763751029968262, "step": 60 }, { "KL/chosen_KL_mean": -1.4959545135498047, "KL/mean": -5.729641914367676, "KL/rejected_KL_mean": -9.963333129882812, "KL/std": 9.229528427124023, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.07002333551645279, "fcm_dpo/delta": -0.20614930987358093, "fcm_dpo/margin": 8.467378616333008, "fcm_dpo/q_t": 0.375395804643631, "grad_norm": 42.78828811645508, "learning_rate": 4.3478260869565214e-07, "logits/chosen": -0.5964009761810303, "logits/rejected": -0.5620393753051758, "logps/chosen": -62.129478454589844, "logps/ref_chosen": -60.633522033691406, "logps/ref_rejected": -89.85249328613281, "logps/rejected": -99.81582641601562, "loss": 1.002, "margin_dpo/margin_mean": 8.467377662658691, "margin_dpo/margin_std": 11.42286491394043, "step": 61 }, { "KL/chosen_KL_mean": -1.7143287658691406, "KL/mean": -4.863595008850098, "KL/rejected_KL_mean": -8.012863159179688, "KL/std": 6.748926162719727, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.06915012747049332, "fcm_dpo/delta": -0.0570698156952858, "fcm_dpo/margin": 6.2985358238220215, "fcm_dpo/q_t": 0.40163713693618774, "grad_norm": 41.4459228515625, "learning_rate": 4.420289855072464e-07, "logits/chosen": -0.6113085746765137, "logits/rejected": -0.5773282647132874, "logps/chosen": -57.8651008605957, "logps/ref_chosen": -56.15077209472656, "logps/ref_rejected": -75.56619262695312, "logps/rejected": -83.57905578613281, "loss": 1.0737, "margin_dpo/margin_mean": 6.29853630065918, "margin_dpo/margin_std": 8.275651931762695, "step": 62 }, { "KL/chosen_KL_mean": -2.811429977416992, "KL/mean": -7.000448226928711, "KL/rejected_KL_mean": -11.189468383789062, "KL/std": 8.535360336303711, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.06666617840528488, "fcm_dpo/delta": -0.16953536868095398, "fcm_dpo/margin": 8.378037452697754, "fcm_dpo/q_t": 0.3721930980682373, "grad_norm": 44.12747573852539, "learning_rate": 4.4927536231884053e-07, "logits/chosen": -0.5932517051696777, "logits/rejected": -0.5480700731277466, "logps/chosen": -75.95882415771484, "logps/ref_chosen": -73.14739227294922, "logps/ref_rejected": -97.61006164550781, "logps/rejected": -108.79953002929688, "loss": 0.9855, "margin_dpo/margin_mean": 8.378036499023438, "margin_dpo/margin_std": 8.932016372680664, "step": 63 }, { "KL/chosen_KL_mean": -1.1035079956054688, "KL/mean": -6.1234307289123535, "KL/rejected_KL_mean": -11.143352508544922, "KL/std": 9.246552467346191, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.06336793303489685, "fcm_dpo/delta": -0.2566346526145935, "fcm_dpo/margin": 10.039838790893555, "fcm_dpo/q_t": 0.3602312207221985, "grad_norm": 39.74605941772461, "learning_rate": 4.5652173913043473e-07, "logits/chosen": -0.5810163021087646, "logits/rejected": -0.5485190749168396, "logps/chosen": -55.102108001708984, "logps/ref_chosen": -53.998600006103516, "logps/ref_rejected": -93.53019714355469, "logps/rejected": -104.67355346679688, "loss": 0.9588, "margin_dpo/margin_mean": 10.039839744567871, "margin_dpo/margin_std": 11.341650009155273, "step": 64 }, { "KL/chosen_KL_mean": -3.4348411560058594, "KL/mean": -8.460206985473633, "KL/rejected_KL_mean": -13.485572814941406, "KL/std": 10.407320022583008, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.06073646619915962, "fcm_dpo/delta": -0.22589105367660522, "fcm_dpo/margin": 10.050735473632812, "fcm_dpo/q_t": 0.3629263639450073, "grad_norm": 40.756656646728516, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.6741570830345154, "logits/rejected": -0.6617323756217957, "logps/chosen": -68.27084350585938, "logps/ref_chosen": -64.83599853515625, "logps/ref_rejected": -109.94645690917969, "logps/rejected": -123.43203735351562, "loss": 0.9743, "margin_dpo/margin_mean": 10.050735473632812, "margin_dpo/margin_std": 11.43245792388916, "step": 65 }, { "KL/chosen_KL_mean": -3.1920318603515625, "KL/mean": -7.833024501800537, "KL/rejected_KL_mean": -12.474014282226562, "KL/std": 9.44174861907959, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.058656755834817886, "fcm_dpo/delta": -0.14220839738845825, "fcm_dpo/margin": 9.28198528289795, "fcm_dpo/q_t": 0.3805831968784332, "grad_norm": 36.21018981933594, "learning_rate": 4.7101449275362313e-07, "logits/chosen": -0.6206960678100586, "logits/rejected": -0.586733341217041, "logps/chosen": -54.63555908203125, "logps/ref_chosen": -51.44352722167969, "logps/ref_rejected": -75.63629913330078, "logps/rejected": -88.11031341552734, "loss": 1.0098, "margin_dpo/margin_mean": 9.28198528289795, "margin_dpo/margin_std": 11.416315078735352, "step": 66 }, { "KL/chosen_KL_mean": -2.8563461303710938, "KL/mean": -7.659085273742676, "KL/rejected_KL_mean": -12.461822509765625, "KL/std": 9.597047805786133, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.05742337554693222, "fcm_dpo/delta": -0.16015625, "fcm_dpo/margin": 9.605472564697266, "fcm_dpo/q_t": 0.3784703016281128, "grad_norm": 37.310569763183594, "learning_rate": 4.782608695652174e-07, "logits/chosen": -0.605553150177002, "logits/rejected": -0.563081681728363, "logps/chosen": -62.19715118408203, "logps/ref_chosen": -59.34080505371094, "logps/ref_rejected": -72.78728485107422, "logps/rejected": -85.24910736083984, "loss": 1.0033, "margin_dpo/margin_mean": 9.605473518371582, "margin_dpo/margin_std": 11.608689308166504, "step": 67 }, { "KL/chosen_KL_mean": -3.41162109375, "KL/mean": -7.9624128341674805, "KL/rejected_KL_mean": -12.513202667236328, "KL/std": 8.548230171203613, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.05597271770238876, "fcm_dpo/delta": -0.11515870690345764, "fcm_dpo/margin": 9.101577758789062, "fcm_dpo/q_t": 0.3834843039512634, "grad_norm": 35.94131088256836, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.6418497562408447, "logits/rejected": -0.5813031196594238, "logps/chosen": -68.6174545288086, "logps/ref_chosen": -65.2058334350586, "logps/ref_rejected": -77.20724487304688, "logps/rejected": -89.72045135498047, "loss": 1.0057, "margin_dpo/margin_mean": 9.101577758789062, "margin_dpo/margin_std": 9.605649948120117, "step": 68 }, { "KL/chosen_KL_mean": -4.020790100097656, "KL/mean": -9.76811408996582, "KL/rejected_KL_mean": -15.51544189453125, "KL/std": 10.38883113861084, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.053416017442941666, "fcm_dpo/delta": -0.2294478863477707, "fcm_dpo/margin": 11.494646072387695, "fcm_dpo/q_t": 0.36246609687805176, "grad_norm": 38.16155242919922, "learning_rate": 4.927536231884058e-07, "logits/chosen": -0.5999042987823486, "logits/rejected": -0.5770286321640015, "logps/chosen": -63.840030670166016, "logps/ref_chosen": -59.81924057006836, "logps/ref_rejected": -103.38886260986328, "logps/rejected": -118.90430450439453, "loss": 0.945, "margin_dpo/margin_mean": 11.494646072387695, "margin_dpo/margin_std": 11.5130615234375, "step": 69 }, { "KL/chosen_KL_mean": -5.912725448608398, "KL/mean": -12.12542724609375, "KL/rejected_KL_mean": -18.338130950927734, "KL/std": 12.911046028137207, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.05124839395284653, "fcm_dpo/delta": -0.2522848844528198, "fcm_dpo/margin": 12.425400733947754, "fcm_dpo/q_t": 0.36073604226112366, "grad_norm": 38.75048065185547, "learning_rate": 5e-07, "logits/chosen": -0.606624960899353, "logits/rejected": -0.5714987516403198, "logps/chosen": -67.84336853027344, "logps/ref_chosen": -61.930641174316406, "logps/ref_rejected": -91.06078338623047, "logps/rejected": -109.39891052246094, "loss": 0.9549, "margin_dpo/margin_mean": 12.425400733947754, "margin_dpo/margin_std": 13.759818077087402, "step": 70 }, { "KL/chosen_KL_mean": -5.871559143066406, "KL/mean": -13.31947135925293, "KL/rejected_KL_mean": -20.76738739013672, "KL/std": 13.823541641235352, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.047790415585041046, "fcm_dpo/delta": -0.3372606635093689, "fcm_dpo/margin": 14.895828247070312, "fcm_dpo/q_t": 0.34655076265335083, "grad_norm": 35.317962646484375, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.6532018184661865, "logits/rejected": -0.6144231557846069, "logps/chosen": -67.62189483642578, "logps/ref_chosen": -61.750335693359375, "logps/ref_rejected": -97.33662414550781, "logps/rejected": -118.10400390625, "loss": 0.9032, "margin_dpo/margin_mean": 14.895828247070312, "margin_dpo/margin_std": 15.058696746826172, "step": 71 }, { "KL/chosen_KL_mean": -6.858076095581055, "KL/mean": -14.622879028320312, "KL/rejected_KL_mean": -22.387680053710938, "KL/std": 14.882482528686523, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.044840212911367416, "fcm_dpo/delta": -0.31945478916168213, "fcm_dpo/margin": 15.529606819152832, "fcm_dpo/q_t": 0.3486083149909973, "grad_norm": 35.28294372558594, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.6572631597518921, "logits/rejected": -0.6205873489379883, "logps/chosen": -72.91148376464844, "logps/ref_chosen": -66.05341339111328, "logps/ref_rejected": -95.2869873046875, "logps/rejected": -117.67466735839844, "loss": 0.9327, "margin_dpo/margin_mean": 15.529606819152832, "margin_dpo/margin_std": 17.4892520904541, "step": 72 }, { "KL/chosen_KL_mean": -8.918149948120117, "KL/mean": -16.7022705078125, "KL/rejected_KL_mean": -24.48639678955078, "KL/std": 19.29153060913086, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.04222744703292847, "fcm_dpo/delta": -0.2773255407810211, "fcm_dpo/margin": 15.568246841430664, "fcm_dpo/q_t": 0.3702467083930969, "grad_norm": 36.51060104370117, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.7001978158950806, "logits/rejected": -0.6606754064559937, "logps/chosen": -75.17442321777344, "logps/ref_chosen": -66.25627136230469, "logps/ref_rejected": -90.45613098144531, "logps/rejected": -114.94253540039062, "loss": 1.0394, "margin_dpo/margin_mean": 15.568246841430664, "margin_dpo/margin_std": 24.206798553466797, "step": 73 }, { "KL/chosen_KL_mean": -9.382577896118164, "KL/mean": -17.968414306640625, "KL/rejected_KL_mean": -26.554244995117188, "KL/std": 21.26626205444336, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.039852894842624664, "fcm_dpo/delta": -0.3063187003135681, "fcm_dpo/margin": 17.171672821044922, "fcm_dpo/q_t": 0.36331337690353394, "grad_norm": 35.83381652832031, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.6333750486373901, "logits/rejected": -0.6257501244544983, "logps/chosen": -62.80746078491211, "logps/ref_chosen": -53.42488098144531, "logps/ref_rejected": -95.94693756103516, "logps/rejected": -122.50118255615234, "loss": 0.9965, "margin_dpo/margin_mean": 17.171672821044922, "margin_dpo/margin_std": 26.445632934570312, "step": 74 }, { "KL/chosen_KL_mean": -9.15802001953125, "KL/mean": -21.245464324951172, "KL/rejected_KL_mean": -33.33289337158203, "KL/std": 21.812652587890625, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.03664487600326538, "fcm_dpo/delta": -0.530461847782135, "fcm_dpo/margin": 24.17488670349121, "fcm_dpo/q_t": 0.31928879022598267, "grad_norm": 31.040130615234375, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.6646023988723755, "logits/rejected": -0.6575799584388733, "logps/chosen": -61.019683837890625, "logps/ref_chosen": -51.861663818359375, "logps/ref_rejected": -111.25398254394531, "logps/rejected": -144.58688354492188, "loss": 0.8454, "margin_dpo/margin_mean": 24.174884796142578, "margin_dpo/margin_std": 24.763885498046875, "step": 75 }, { "KL/chosen_KL_mean": -11.104362487792969, "KL/mean": -18.701168060302734, "KL/rejected_KL_mean": -26.297969818115234, "KL/std": 17.725784301757812, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.03470379486680031, "fcm_dpo/delta": -0.13428181409835815, "fcm_dpo/margin": 15.193618774414062, "fcm_dpo/q_t": 0.38014188408851624, "grad_norm": 32.03413009643555, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.690595269203186, "logits/rejected": -0.6483590006828308, "logps/chosen": -64.37039947509766, "logps/ref_chosen": -53.26603698730469, "logps/ref_rejected": -78.21662902832031, "logps/rejected": -104.51460266113281, "loss": 1.0267, "margin_dpo/margin_mean": 15.193617820739746, "margin_dpo/margin_std": 19.406496047973633, "step": 76 }, { "KL/chosen_KL_mean": -9.964458465576172, "KL/mean": -22.325725555419922, "KL/rejected_KL_mean": -34.686988830566406, "KL/std": 24.06493377685547, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.032092347741127014, "fcm_dpo/delta": -0.4313170313835144, "fcm_dpo/margin": 24.722532272338867, "fcm_dpo/q_t": 0.32912200689315796, "grad_norm": 33.1654052734375, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.6741265058517456, "logits/rejected": -0.6357508897781372, "logps/chosen": -68.06114196777344, "logps/ref_chosen": -58.0966796875, "logps/ref_rejected": -93.77361297607422, "logps/rejected": -128.46060180664062, "loss": 0.8956, "margin_dpo/margin_mean": 24.7225341796875, "margin_dpo/margin_std": 26.815866470336914, "step": 77 }, { "KL/chosen_KL_mean": -9.932151794433594, "KL/mean": -19.229045867919922, "KL/rejected_KL_mean": -28.525943756103516, "KL/std": 19.422195434570312, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.030831564217805862, "fcm_dpo/delta": -0.18346719443798065, "fcm_dpo/margin": 18.593788146972656, "fcm_dpo/q_t": 0.3741081953048706, "grad_norm": 30.061494827270508, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.6306928396224976, "logits/rejected": -0.6057921648025513, "logps/chosen": -65.54593658447266, "logps/ref_chosen": -55.61378479003906, "logps/ref_rejected": -84.93436431884766, "logps/rejected": -113.46031188964844, "loss": 0.9976, "margin_dpo/margin_mean": 18.593788146972656, "margin_dpo/margin_std": 22.29560089111328, "step": 78 }, { "KL/chosen_KL_mean": -10.282918930053711, "KL/mean": -20.42000961303711, "KL/rejected_KL_mean": -30.55709457397461, "KL/std": 21.94013214111328, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.02950279787182808, "fcm_dpo/delta": -0.21087872982025146, "fcm_dpo/margin": 20.274171829223633, "fcm_dpo/q_t": 0.3741893768310547, "grad_norm": 26.36469078063965, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6447913646697998, "logits/rejected": -0.6144734025001526, "logps/chosen": -65.73340606689453, "logps/ref_chosen": -55.45048522949219, "logps/ref_rejected": -87.64756774902344, "logps/rejected": -118.20465850830078, "loss": 1.0083, "margin_dpo/margin_mean": 20.274171829223633, "margin_dpo/margin_std": 26.865041732788086, "step": 79 }, { "KL/chosen_KL_mean": -13.432994842529297, "KL/mean": -22.9481201171875, "KL/rejected_KL_mean": -32.46324157714844, "KL/std": 23.311492919921875, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.028320040553808212, "fcm_dpo/delta": -0.1528511494398117, "fcm_dpo/margin": 19.03024673461914, "fcm_dpo/q_t": 0.3878602981567383, "grad_norm": 27.577178955078125, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7253065705299377, "logits/rejected": -0.6845601201057434, "logps/chosen": -71.95228576660156, "logps/ref_chosen": -58.519290924072266, "logps/ref_rejected": -87.54750061035156, "logps/rejected": -120.0107421875, "loss": 1.0491, "margin_dpo/margin_mean": 19.030248641967773, "margin_dpo/margin_std": 27.02452850341797, "step": 80 }, { "KL/chosen_KL_mean": -12.46990966796875, "KL/mean": -25.47945213317871, "KL/rejected_KL_mean": -38.48899841308594, "KL/std": 27.457473754882812, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.02683407999575138, "fcm_dpo/delta": -0.32329061627388, "fcm_dpo/margin": 26.019084930419922, "fcm_dpo/q_t": 0.35463476181030273, "grad_norm": 28.8442325592041, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6985138654708862, "logits/rejected": -0.6699087619781494, "logps/chosen": -78.91877746582031, "logps/ref_chosen": -66.44886779785156, "logps/ref_rejected": -129.66270446777344, "logps/rejected": -168.15170288085938, "loss": 0.9548, "margin_dpo/margin_mean": 26.019084930419922, "margin_dpo/margin_std": 32.673709869384766, "step": 81 }, { "KL/chosen_KL_mean": -14.771663665771484, "KL/mean": -25.181900024414062, "KL/rejected_KL_mean": -35.592132568359375, "KL/std": 25.033601760864258, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.026078231632709503, "fcm_dpo/delta": -0.17972172796726227, "fcm_dpo/margin": 20.82046890258789, "fcm_dpo/q_t": 0.38705772161483765, "grad_norm": 31.77438735961914, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.6351274251937866, "logits/rejected": -0.6246193647384644, "logps/chosen": -67.0040512084961, "logps/ref_chosen": -52.232383728027344, "logps/ref_rejected": -90.74325561523438, "logps/rejected": -126.33538818359375, "loss": 1.0902, "margin_dpo/margin_mean": 20.82046890258789, "margin_dpo/margin_std": 34.549964904785156, "step": 82 }, { "KL/chosen_KL_mean": -15.13831901550293, "KL/mean": -27.50519371032715, "KL/rejected_KL_mean": -39.872066497802734, "KL/std": 26.05057716369629, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.024679036810994148, "fcm_dpo/delta": -0.22516962885856628, "fcm_dpo/margin": 24.733749389648438, "fcm_dpo/q_t": 0.36837995052337646, "grad_norm": 26.978626251220703, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6816978454589844, "logits/rejected": -0.6491335034370422, "logps/chosen": -70.96570587158203, "logps/ref_chosen": -55.82738494873047, "logps/ref_rejected": -103.71589660644531, "logps/rejected": -143.5879669189453, "loss": 0.9914, "margin_dpo/margin_mean": 24.733749389648438, "margin_dpo/margin_std": 30.88388442993164, "step": 83 }, { "KL/chosen_KL_mean": -14.203386306762695, "KL/mean": -24.5653076171875, "KL/rejected_KL_mean": -34.9272346496582, "KL/std": 22.138011932373047, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.02386614680290222, "fcm_dpo/delta": -0.12736207246780396, "fcm_dpo/margin": 20.723844528198242, "fcm_dpo/q_t": 0.38502997159957886, "grad_norm": 25.739469528198242, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.6767555475234985, "logits/rejected": -0.6396181583404541, "logps/chosen": -81.37955474853516, "logps/ref_chosen": -67.1761703491211, "logps/ref_rejected": -87.29859924316406, "logps/rejected": -122.22583770751953, "loss": 1.0217, "margin_dpo/margin_mean": 20.723844528198242, "margin_dpo/margin_std": 22.54238510131836, "step": 84 }, { "KL/chosen_KL_mean": -14.595979690551758, "KL/mean": -25.3236083984375, "KL/rejected_KL_mean": -36.051231384277344, "KL/std": 22.786161422729492, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.023556005209684372, "fcm_dpo/delta": -0.13166888058185577, "fcm_dpo/margin": 21.45526123046875, "fcm_dpo/q_t": 0.3877854645252228, "grad_norm": 25.446636199951172, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.6357869505882263, "logits/rejected": -0.6025545001029968, "logps/chosen": -73.00259399414062, "logps/ref_chosen": -58.4066162109375, "logps/ref_rejected": -78.63880157470703, "logps/rejected": -114.69003295898438, "loss": 1.0431, "margin_dpo/margin_mean": 21.45526123046875, "margin_dpo/margin_std": 27.411766052246094, "step": 85 }, { "KL/chosen_KL_mean": -18.884050369262695, "KL/mean": -30.216419219970703, "KL/rejected_KL_mean": -41.54878616333008, "KL/std": 30.91357421875, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.02299971505999565, "fcm_dpo/delta": -0.1070268452167511, "fcm_dpo/margin": 22.664737701416016, "fcm_dpo/q_t": 0.3989836275577545, "grad_norm": 39.153846740722656, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6874663829803467, "logits/rejected": -0.6762598156929016, "logps/chosen": -75.02151489257812, "logps/ref_chosen": -56.13746643066406, "logps/ref_rejected": -88.12165069580078, "logps/rejected": -129.67044067382812, "loss": 1.1246, "margin_dpo/margin_mean": 22.664737701416016, "margin_dpo/margin_std": 42.79364776611328, "step": 86 }, { "KL/chosen_KL_mean": -18.476444244384766, "KL/mean": -31.937850952148438, "KL/rejected_KL_mean": -45.39925765991211, "KL/std": 30.784648895263672, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.021954335272312164, "fcm_dpo/delta": -0.20476816594600677, "fcm_dpo/margin": 26.92281723022461, "fcm_dpo/q_t": 0.3721076250076294, "grad_norm": 25.699562072753906, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.7187229990959167, "logits/rejected": -0.6957570910453796, "logps/chosen": -74.11254119873047, "logps/ref_chosen": -55.63609313964844, "logps/ref_rejected": -95.46757507324219, "logps/rejected": -140.86683654785156, "loss": 1.0289, "margin_dpo/margin_mean": 26.92281723022461, "margin_dpo/margin_std": 37.453208923339844, "step": 87 }, { "KL/chosen_KL_mean": -23.373714447021484, "KL/mean": -33.52701187133789, "KL/rejected_KL_mean": -43.68030548095703, "KL/std": 32.281776428222656, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.02150532230734825, "fcm_dpo/delta": -0.07733839750289917, "fcm_dpo/margin": 20.306591033935547, "fcm_dpo/q_t": 0.4068375527858734, "grad_norm": 26.6627140045166, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.6914920806884766, "logits/rejected": -0.6863746643066406, "logps/chosen": -97.04486083984375, "logps/ref_chosen": -73.67115020751953, "logps/ref_rejected": -106.70849609375, "logps/rejected": -150.38880920410156, "loss": 1.1317, "margin_dpo/margin_mean": 20.306591033935547, "margin_dpo/margin_std": 34.122745513916016, "step": 88 }, { "KL/chosen_KL_mean": -14.407289505004883, "KL/mean": -25.77096176147461, "KL/rejected_KL_mean": -37.1346321105957, "KL/std": 27.240581512451172, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.02131272479891777, "fcm_dpo/delta": -0.09037049859762192, "fcm_dpo/margin": 22.727336883544922, "fcm_dpo/q_t": 0.3924558758735657, "grad_norm": 24.11750030517578, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7111062407493591, "logits/rejected": -0.6786768436431885, "logps/chosen": -75.0322036743164, "logps/ref_chosen": -60.624916076660156, "logps/ref_rejected": -82.08354949951172, "logps/rejected": -119.21818542480469, "loss": 1.0546, "margin_dpo/margin_mean": 22.727336883544922, "margin_dpo/margin_std": 31.17041778564453, "step": 89 }, { "KL/chosen_KL_mean": -18.334720611572266, "KL/mean": -33.93196105957031, "KL/rejected_KL_mean": -49.52920150756836, "KL/std": 36.926090240478516, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.02063620463013649, "fcm_dpo/delta": -0.23454345762729645, "fcm_dpo/margin": 31.19448471069336, "fcm_dpo/q_t": 0.37270212173461914, "grad_norm": 25.166067123413086, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.640715479850769, "logits/rejected": -0.655485987663269, "logps/chosen": -71.62003326416016, "logps/ref_chosen": -53.285308837890625, "logps/ref_rejected": -111.54470825195312, "logps/rejected": -161.07391357421875, "loss": 1.0296, "margin_dpo/margin_mean": 31.194486618041992, "margin_dpo/margin_std": 45.882774353027344, "step": 90 }, { "KL/chosen_KL_mean": -19.304616928100586, "KL/mean": -32.036643981933594, "KL/rejected_KL_mean": -44.76866912841797, "KL/std": 29.606094360351562, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.019963329657912254, "fcm_dpo/delta": -0.09465456753969193, "fcm_dpo/margin": 25.464061737060547, "fcm_dpo/q_t": 0.3919849693775177, "grad_norm": 24.47185516357422, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6711220741271973, "logits/rejected": -0.6374910473823547, "logps/chosen": -81.10757446289062, "logps/ref_chosen": -61.802955627441406, "logps/ref_rejected": -87.87395477294922, "logps/rejected": -132.6426239013672, "loss": 1.0706, "margin_dpo/margin_mean": 25.464059829711914, "margin_dpo/margin_std": 38.521697998046875, "step": 91 }, { "KL/chosen_KL_mean": -17.32529640197754, "KL/mean": -30.216405868530273, "KL/rejected_KL_mean": -43.10751724243164, "KL/std": 30.38365936279297, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.01955636963248253, "fcm_dpo/delta": -0.1114797294139862, "fcm_dpo/margin": 25.7822208404541, "fcm_dpo/q_t": 0.39096271991729736, "grad_norm": 22.75739860534668, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.7390532493591309, "logits/rejected": -0.7194732427597046, "logps/chosen": -68.966064453125, "logps/ref_chosen": -51.640769958496094, "logps/ref_rejected": -77.88117980957031, "logps/rejected": -120.98870086669922, "loss": 1.0599, "margin_dpo/margin_mean": 25.782222747802734, "margin_dpo/margin_std": 36.745933532714844, "step": 92 }, { "KL/chosen_KL_mean": -19.497028350830078, "KL/mean": -32.41360855102539, "KL/rejected_KL_mean": -45.33018112182617, "KL/std": 28.625198364257812, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.019083872437477112, "fcm_dpo/delta": -0.09772805124521255, "fcm_dpo/margin": 25.83315658569336, "fcm_dpo/q_t": 0.390662282705307, "grad_norm": 23.420873641967773, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.7268893718719482, "logits/rejected": -0.6788473129272461, "logps/chosen": -72.0262680053711, "logps/ref_chosen": -52.529239654541016, "logps/ref_rejected": -77.16075134277344, "logps/rejected": -122.49092864990234, "loss": 1.0385, "margin_dpo/margin_mean": 25.833154678344727, "margin_dpo/margin_std": 33.194129943847656, "step": 93 }, { "KL/chosen_KL_mean": -20.87965965270996, "KL/mean": -35.868045806884766, "KL/rejected_KL_mean": -50.8564338684082, "KL/std": 32.95391845703125, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.01840830221772194, "fcm_dpo/delta": -0.1622575968503952, "fcm_dpo/margin": 29.97677993774414, "fcm_dpo/q_t": 0.37686464190483093, "grad_norm": 23.023033142089844, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.69089674949646, "logits/rejected": -0.6784852147102356, "logps/chosen": -82.1022720336914, "logps/ref_chosen": -61.22261047363281, "logps/ref_rejected": -99.59902954101562, "logps/rejected": -150.45547485351562, "loss": 0.9994, "margin_dpo/margin_mean": 29.97677993774414, "margin_dpo/margin_std": 34.474708557128906, "step": 94 }, { "KL/chosen_KL_mean": -20.05255699157715, "KL/mean": -32.12548065185547, "KL/rejected_KL_mean": -44.198402404785156, "KL/std": 31.86922836303711, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.018146492540836334, "fcm_dpo/delta": -0.05808256193995476, "fcm_dpo/margin": 24.145843505859375, "fcm_dpo/q_t": 0.39963364601135254, "grad_norm": 21.70364761352539, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.6787654161453247, "logits/rejected": -0.6507630348205566, "logps/chosen": -72.57620239257812, "logps/ref_chosen": -52.523643493652344, "logps/ref_rejected": -75.8803482055664, "logps/rejected": -120.07875061035156, "loss": 1.0803, "margin_dpo/margin_mean": 24.145843505859375, "margin_dpo/margin_std": 33.2855339050293, "step": 95 }, { "KL/chosen_KL_mean": -19.471036911010742, "KL/mean": -35.86725616455078, "KL/rejected_KL_mean": -52.26348114013672, "KL/std": 35.56018829345703, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.01755816489458084, "fcm_dpo/delta": -0.16479085385799408, "fcm_dpo/margin": 32.792442321777344, "fcm_dpo/q_t": 0.37453415989875793, "grad_norm": 22.211780548095703, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7198976278305054, "logits/rejected": -0.69416344165802, "logps/chosen": -81.62800598144531, "logps/ref_chosen": -62.15697479248047, "logps/ref_rejected": -96.59601593017578, "logps/rejected": -148.8594970703125, "loss": 0.9915, "margin_dpo/margin_mean": 32.792442321777344, "margin_dpo/margin_std": 39.19865417480469, "step": 96 }, { "KL/chosen_KL_mean": -20.839025497436523, "KL/mean": -33.39268493652344, "KL/rejected_KL_mean": -45.94635009765625, "KL/std": 29.86338996887207, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.017377909272909164, "fcm_dpo/delta": -0.10373032093048096, "fcm_dpo/margin": 25.10732078552246, "fcm_dpo/q_t": 0.39860257506370544, "grad_norm": 22.988815307617188, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.6391160488128662, "logits/rejected": -0.5976995229721069, "logps/chosen": -75.48538970947266, "logps/ref_chosen": -54.646366119384766, "logps/ref_rejected": -76.96475219726562, "logps/rejected": -122.91110229492188, "loss": 1.066, "margin_dpo/margin_mean": 25.10732078552246, "margin_dpo/margin_std": 29.785751342773438, "step": 97 }, { "KL/chosen_KL_mean": -25.700241088867188, "KL/mean": -40.22949981689453, "KL/rejected_KL_mean": -54.758758544921875, "KL/std": 34.170616149902344, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.01692095398902893, "fcm_dpo/delta": -0.09668763726949692, "fcm_dpo/margin": 29.058513641357422, "fcm_dpo/q_t": 0.3878336548805237, "grad_norm": 24.090316772460938, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6487174034118652, "logits/rejected": -0.6329121589660645, "logps/chosen": -90.95886993408203, "logps/ref_chosen": -65.25862884521484, "logps/ref_rejected": -96.5274887084961, "logps/rejected": -151.2862548828125, "loss": 1.0659, "margin_dpo/margin_mean": 29.058513641357422, "margin_dpo/margin_std": 41.047019958496094, "step": 98 }, { "KL/chosen_KL_mean": -20.114553451538086, "KL/mean": -37.91062545776367, "KL/rejected_KL_mean": -55.706695556640625, "KL/std": 36.737327575683594, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.016340188682079315, "fcm_dpo/delta": -0.19358737766742706, "fcm_dpo/margin": 35.59214782714844, "fcm_dpo/q_t": 0.3759217858314514, "grad_norm": 20.969350814819336, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.6549317240715027, "logits/rejected": -0.6567766666412354, "logps/chosen": -65.75303649902344, "logps/ref_chosen": -45.638484954833984, "logps/ref_rejected": -86.43793487548828, "logps/rejected": -142.14462280273438, "loss": 0.9999, "margin_dpo/margin_mean": 35.59214401245117, "margin_dpo/margin_std": 45.7916145324707, "step": 99 }, { "KL/chosen_KL_mean": -22.74945640563965, "KL/mean": -32.91205596923828, "KL/rejected_KL_mean": -43.07464599609375, "KL/std": 29.567760467529297, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.01622767373919487, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.325191497802734, "fcm_dpo/q_t": 0.425083190202713, "grad_norm": 23.01412582397461, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6575253009796143, "logits/rejected": -0.615835428237915, "logps/chosen": -80.34343719482422, "logps/ref_chosen": -57.59397888183594, "logps/ref_rejected": -74.06021118164062, "logps/rejected": -117.13485717773438, "loss": 1.1781, "margin_dpo/margin_mean": 20.325193405151367, "margin_dpo/margin_std": 38.162071228027344, "step": 100 }, { "KL/chosen_KL_mean": -27.887283325195312, "KL/mean": -40.41047668457031, "KL/rejected_KL_mean": -52.93367385864258, "KL/std": 36.532798767089844, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.016045931726694107, "fcm_dpo/delta": -0.05631338432431221, "fcm_dpo/margin": 25.046384811401367, "fcm_dpo/q_t": 0.4135172367095947, "grad_norm": 22.6212215423584, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.6584815979003906, "logits/rejected": -0.6281433701515198, "logps/chosen": -89.5361328125, "logps/ref_chosen": -61.64885330200195, "logps/ref_rejected": -83.18968200683594, "logps/rejected": -136.12335205078125, "loss": 1.1374, "margin_dpo/margin_mean": 25.04638671875, "margin_dpo/margin_std": 43.94956970214844, "step": 101 }, { "KL/chosen_KL_mean": -29.988710403442383, "KL/mean": -39.130943298339844, "KL/rejected_KL_mean": -48.27317428588867, "KL/std": 34.25249099731445, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.016045931726694107, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.284465789794922, "fcm_dpo/q_t": 0.43444907665252686, "grad_norm": 25.962202072143555, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.6689609289169312, "logits/rejected": -0.6241482496261597, "logps/chosen": -94.06758880615234, "logps/ref_chosen": -64.0788803100586, "logps/ref_rejected": -68.18707275390625, "logps/rejected": -116.46025085449219, "loss": 1.2176, "margin_dpo/margin_mean": 18.284465789794922, "margin_dpo/margin_std": 40.5817985534668, "step": 102 }, { "KL/chosen_KL_mean": -23.946800231933594, "KL/mean": -42.415340423583984, "KL/rejected_KL_mean": -60.88387680053711, "KL/std": 38.02168273925781, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.015578062273561954, "fcm_dpo/delta": -0.18654988706111908, "fcm_dpo/margin": 36.937076568603516, "fcm_dpo/q_t": 0.3750496506690979, "grad_norm": 22.1878719329834, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6738394498825073, "logits/rejected": -0.6411100029945374, "logps/chosen": -85.24607849121094, "logps/ref_chosen": -61.299278259277344, "logps/ref_rejected": -93.57270812988281, "logps/rejected": -154.4565887451172, "loss": 0.9984, "margin_dpo/margin_mean": 36.937076568603516, "margin_dpo/margin_std": 46.08087158203125, "step": 103 }, { "KL/chosen_KL_mean": -25.76188087463379, "KL/mean": -43.38178634643555, "KL/rejected_KL_mean": -61.001686096191406, "KL/std": 41.89414978027344, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.01516028307378292, "fcm_dpo/delta": -0.14165204763412476, "fcm_dpo/margin": 35.23980712890625, "fcm_dpo/q_t": 0.3880753517150879, "grad_norm": 21.539640426635742, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.6753038167953491, "logits/rejected": -0.6453461647033691, "logps/chosen": -80.13465881347656, "logps/ref_chosen": -54.372772216796875, "logps/ref_rejected": -89.5647201538086, "logps/rejected": -150.56640625, "loss": 1.0513, "margin_dpo/margin_mean": 35.23980712890625, "margin_dpo/margin_std": 51.31709671020508, "step": 104 }, { "KL/chosen_KL_mean": -24.360563278198242, "KL/mean": -49.227760314941406, "KL/rejected_KL_mean": -74.09496307373047, "KL/std": 40.60966873168945, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.014314282685518265, "fcm_dpo/delta": -0.33550798892974854, "fcm_dpo/margin": 49.734397888183594, "fcm_dpo/q_t": 0.33919352293014526, "grad_norm": 22.338714599609375, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.6681811809539795, "logits/rejected": -0.6469371914863586, "logps/chosen": -78.99951171875, "logps/ref_chosen": -54.638946533203125, "logps/ref_rejected": -97.97351837158203, "logps/rejected": -172.0684814453125, "loss": 0.8853, "margin_dpo/margin_mean": 49.734397888183594, "margin_dpo/margin_std": 45.006996154785156, "step": 105 }, { "KL/chosen_KL_mean": -27.47538185119629, "KL/mean": -42.99120330810547, "KL/rejected_KL_mean": -58.50702667236328, "KL/std": 35.57778549194336, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.013949505984783173, "fcm_dpo/delta": -0.03634671866893768, "fcm_dpo/margin": 31.031639099121094, "fcm_dpo/q_t": 0.4003314971923828, "grad_norm": 21.185850143432617, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.6308009624481201, "logits/rejected": -0.602495014667511, "logps/chosen": -82.30827331542969, "logps/ref_chosen": -54.83289337158203, "logps/ref_rejected": -85.22461700439453, "logps/rejected": -143.7316436767578, "loss": 1.0686, "margin_dpo/margin_mean": 31.031639099121094, "margin_dpo/margin_std": 38.763301849365234, "step": 106 }, { "KL/chosen_KL_mean": -32.56428909301758, "KL/mean": -50.00350570678711, "KL/rejected_KL_mean": -67.4427261352539, "KL/std": 43.262245178222656, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.013720358721911907, "fcm_dpo/delta": -0.08281677961349487, "fcm_dpo/margin": 34.87843704223633, "fcm_dpo/q_t": 0.39147496223449707, "grad_norm": 20.624536514282227, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.6635041832923889, "logits/rejected": -0.6185137629508972, "logps/chosen": -102.27210235595703, "logps/ref_chosen": -69.70780944824219, "logps/ref_rejected": -94.73950958251953, "logps/rejected": -162.18223571777344, "loss": 1.0561, "margin_dpo/margin_mean": 34.87843704223633, "margin_dpo/margin_std": 45.82408905029297, "step": 107 }, { "KL/chosen_KL_mean": -29.11627769470215, "KL/mean": -50.483489990234375, "KL/rejected_KL_mean": -71.85070037841797, "KL/std": 48.153316497802734, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.013360362499952316, "fcm_dpo/delta": -0.18124811351299286, "fcm_dpo/margin": 42.73442459106445, "fcm_dpo/q_t": 0.3816313147544861, "grad_norm": 20.98419189453125, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.6446486711502075, "logits/rejected": -0.6245523691177368, "logps/chosen": -85.12615966796875, "logps/ref_chosen": -56.0098876953125, "logps/ref_rejected": -95.79601287841797, "logps/rejected": -167.64671325683594, "loss": 1.0411, "margin_dpo/margin_mean": 42.73442840576172, "margin_dpo/margin_std": 62.17555236816406, "step": 108 }, { "KL/chosen_KL_mean": -27.09517478942871, "KL/mean": -49.685211181640625, "KL/rejected_KL_mean": -72.27523803710938, "KL/std": 45.911842346191406, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.012815423309803009, "fcm_dpo/delta": -0.19097067415714264, "fcm_dpo/margin": 45.18007278442383, "fcm_dpo/q_t": 0.37120842933654785, "grad_norm": 21.367916107177734, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5958420634269714, "logits/rejected": -0.5749087333679199, "logps/chosen": -89.98066711425781, "logps/ref_chosen": -62.88549041748047, "logps/ref_rejected": -98.68573760986328, "logps/rejected": -170.96096801757812, "loss": 0.9976, "margin_dpo/margin_mean": 45.18007278442383, "margin_dpo/margin_std": 54.63614273071289, "step": 109 }, { "KL/chosen_KL_mean": -28.124114990234375, "KL/mean": -48.460716247558594, "KL/rejected_KL_mean": -68.79730987548828, "KL/std": 45.917694091796875, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.012292098253965378, "fcm_dpo/delta": -0.1774018108844757, "fcm_dpo/margin": 40.673194885253906, "fcm_dpo/q_t": 0.38942158222198486, "grad_norm": 18.53866958618164, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.5888636708259583, "logits/rejected": -0.5525768995285034, "logps/chosen": -86.87779998779297, "logps/ref_chosen": -58.753684997558594, "logps/ref_rejected": -79.75001525878906, "logps/rejected": -148.54733276367188, "loss": 1.0605, "margin_dpo/margin_mean": 40.673194885253906, "margin_dpo/margin_std": 55.483089447021484, "step": 110 }, { "KL/chosen_KL_mean": -31.884164810180664, "KL/mean": -55.051116943359375, "KL/rejected_KL_mean": -78.21807861328125, "KL/std": 51.810791015625, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.011932818219065666, "fcm_dpo/delta": -0.16332484781742096, "fcm_dpo/margin": 46.33391571044922, "fcm_dpo/q_t": 0.3792319893836975, "grad_norm": 20.914552688598633, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.6331917643547058, "logits/rejected": -0.6107040643692017, "logps/chosen": -100.50827026367188, "logps/ref_chosen": -68.62410736083984, "logps/ref_rejected": -98.42886352539062, "logps/rejected": -176.64694213867188, "loss": 1.0404, "margin_dpo/margin_mean": 46.33391571044922, "margin_dpo/margin_std": 64.15996551513672, "step": 111 }, { "KL/chosen_KL_mean": -29.431987762451172, "KL/mean": -44.088584899902344, "KL/rejected_KL_mean": -58.74518966674805, "KL/std": 35.674102783203125, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.011897064745426178, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.31319808959961, "fcm_dpo/q_t": 0.4195147752761841, "grad_norm": 18.906843185424805, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.6224333047866821, "logits/rejected": -0.5856015086174011, "logps/chosen": -79.68162536621094, "logps/ref_chosen": -50.24964141845703, "logps/ref_rejected": -64.77442932128906, "logps/rejected": -123.51962280273438, "loss": 1.132, "margin_dpo/margin_mean": 29.313201904296875, "margin_dpo/margin_std": 43.370330810546875, "step": 112 }, { "KL/chosen_KL_mean": -37.25122833251953, "KL/mean": -52.803138732910156, "KL/rejected_KL_mean": -68.35505676269531, "KL/std": 37.59297180175781, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.011897064745426178, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.103832244873047, "fcm_dpo/q_t": 0.4143812954425812, "grad_norm": 19.883005142211914, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.5839675664901733, "logits/rejected": -0.5408110022544861, "logps/chosen": -103.96417999267578, "logps/ref_chosen": -66.71295166015625, "logps/ref_rejected": -77.96870422363281, "logps/rejected": -146.32376098632812, "loss": 1.108, "margin_dpo/margin_mean": 31.103832244873047, "margin_dpo/margin_std": 41.73796844482422, "step": 113 }, { "KL/chosen_KL_mean": -34.05321502685547, "KL/mean": -59.04277420043945, "KL/rejected_KL_mean": -84.03233337402344, "KL/std": 56.13905334472656, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.011499330401420593, "fcm_dpo/delta": -0.18732763826847076, "fcm_dpo/margin": 49.9791145324707, "fcm_dpo/q_t": 0.3790869116783142, "grad_norm": 20.808910369873047, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.6015588045120239, "logits/rejected": -0.5708324313163757, "logps/chosen": -91.83829498291016, "logps/ref_chosen": -57.78507995605469, "logps/ref_rejected": -87.10966491699219, "logps/rejected": -171.14199829101562, "loss": 1.0125, "margin_dpo/margin_mean": 49.97911834716797, "margin_dpo/margin_std": 68.06692504882812, "step": 114 }, { "KL/chosen_KL_mean": -44.401611328125, "KL/mean": -61.324501037597656, "KL/rejected_KL_mean": -78.24739074707031, "KL/std": 53.49150085449219, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.011459581553936005, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.84577941894531, "fcm_dpo/q_t": 0.41818803548812866, "grad_norm": 27.005939483642578, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.641567587852478, "logits/rejected": -0.624779224395752, "logps/chosen": -109.9842529296875, "logps/ref_chosen": -65.5826416015625, "logps/ref_rejected": -98.56552124023438, "logps/rejected": -176.8129119873047, "loss": 1.1781, "margin_dpo/margin_mean": 33.84577560424805, "margin_dpo/margin_std": 70.62551879882812, "step": 115 }, { "KL/chosen_KL_mean": -33.02043914794922, "KL/mean": -52.104915618896484, "KL/rejected_KL_mean": -71.18939971923828, "KL/std": 45.22285461425781, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.011387938633561134, "fcm_dpo/delta": -0.06291196495294571, "fcm_dpo/margin": 38.16896057128906, "fcm_dpo/q_t": 0.4015217423439026, "grad_norm": 20.68140983581543, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.5794106721878052, "logits/rejected": -0.5696778297424316, "logps/chosen": -84.42074584960938, "logps/ref_chosen": -51.40031433105469, "logps/ref_rejected": -80.5218505859375, "logps/rejected": -151.71124267578125, "loss": 1.0702, "margin_dpo/margin_mean": 38.16896057128906, "margin_dpo/margin_std": 49.083984375, "step": 116 }, { "KL/chosen_KL_mean": -42.41381072998047, "KL/mean": -57.908756256103516, "KL/rejected_KL_mean": -73.40370178222656, "KL/std": 47.984107971191406, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.011316295713186264, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.98989486694336, "fcm_dpo/q_t": 0.42085617780685425, "grad_norm": 26.12360954284668, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.5579597353935242, "logits/rejected": -0.5199344158172607, "logps/chosen": -111.71221923828125, "logps/ref_chosen": -69.29840850830078, "logps/ref_rejected": -66.583984375, "logps/rejected": -139.98770141601562, "loss": 1.1531, "margin_dpo/margin_mean": 30.98989486694336, "margin_dpo/margin_std": 54.445655822753906, "step": 117 }, { "KL/chosen_KL_mean": -33.375709533691406, "KL/mean": -52.365203857421875, "KL/rejected_KL_mean": -71.35469818115234, "KL/std": 43.37816619873047, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.011291439644992352, "fcm_dpo/delta": -0.0220131054520607, "fcm_dpo/margin": 37.979000091552734, "fcm_dpo/q_t": 0.4032064974308014, "grad_norm": 20.045106887817383, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.6095850467681885, "logits/rejected": -0.5907869338989258, "logps/chosen": -89.01669311523438, "logps/ref_chosen": -55.6409797668457, "logps/ref_rejected": -75.66905975341797, "logps/rejected": -147.0237579345703, "loss": 1.0744, "margin_dpo/margin_mean": 37.979000091552734, "margin_dpo/margin_std": 49.81169891357422, "step": 118 }, { "KL/chosen_KL_mean": -45.96607208251953, "KL/mean": -64.73681640625, "KL/rejected_KL_mean": -83.50755310058594, "KL/std": 48.845481872558594, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.011094784364104271, "fcm_dpo/delta": -0.07682996988296509, "fcm_dpo/margin": 37.541481018066406, "fcm_dpo/q_t": 0.4086949825286865, "grad_norm": 22.310869216918945, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.6086139678955078, "logits/rejected": -0.5971540808677673, "logps/chosen": -119.47627258300781, "logps/ref_chosen": -73.51019287109375, "logps/ref_rejected": -102.977294921875, "logps/rejected": -186.48483276367188, "loss": 1.1131, "margin_dpo/margin_mean": 37.541481018066406, "margin_dpo/margin_std": 59.72161102294922, "step": 119 }, { "KL/chosen_KL_mean": -47.09067916870117, "KL/mean": -73.13763427734375, "KL/rejected_KL_mean": -99.18460083007812, "KL/std": 61.26211166381836, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.010806088335812092, "fcm_dpo/delta": -0.17278769612312317, "fcm_dpo/margin": 52.09393310546875, "fcm_dpo/q_t": 0.375888466835022, "grad_norm": 21.352399826049805, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.6157863140106201, "logits/rejected": -0.5857179164886475, "logps/chosen": -123.87151336669922, "logps/ref_chosen": -76.78083801269531, "logps/ref_rejected": -108.02374267578125, "logps/rejected": -207.20834350585938, "loss": 1.0065, "margin_dpo/margin_mean": 52.09393310546875, "margin_dpo/margin_std": 64.51720428466797, "step": 120 }, { "KL/chosen_KL_mean": -45.46478271484375, "KL/mean": -74.26813507080078, "KL/rejected_KL_mean": -103.07148742675781, "KL/std": 59.699256896972656, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.010338496416807175, "fcm_dpo/delta": -0.20949624478816986, "fcm_dpo/margin": 57.6067008972168, "fcm_dpo/q_t": 0.3712344169616699, "grad_norm": 24.244958877563477, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.5617018342018127, "logits/rejected": -0.5581203699111938, "logps/chosen": -107.25468444824219, "logps/ref_chosen": -61.789894104003906, "logps/ref_rejected": -109.99456787109375, "logps/rejected": -213.06605529785156, "loss": 1.0057, "margin_dpo/margin_mean": 57.60670471191406, "margin_dpo/margin_std": 74.79032135009766, "step": 121 }, { "KL/chosen_KL_mean": -41.58914566040039, "KL/mean": -76.96797943115234, "KL/rejected_KL_mean": -112.3468017578125, "KL/std": 67.52272033691406, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.009813392534852028, "fcm_dpo/delta": -0.31619852781295776, "fcm_dpo/margin": 70.75765991210938, "fcm_dpo/q_t": 0.3468356132507324, "grad_norm": 22.89113998413086, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.5372201204299927, "logits/rejected": -0.5534934997558594, "logps/chosen": -88.4913558959961, "logps/ref_chosen": -46.9022102355957, "logps/ref_rejected": -106.71418762207031, "logps/rejected": -219.06100463867188, "loss": 0.9104, "margin_dpo/margin_mean": 70.75765991210938, "margin_dpo/margin_std": 70.87075805664062, "step": 122 }, { "KL/chosen_KL_mean": -45.56751251220703, "KL/mean": -68.76374053955078, "KL/rejected_KL_mean": -91.95997619628906, "KL/std": 57.52820587158203, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.009498516097664833, "fcm_dpo/delta": -0.07821456342935562, "fcm_dpo/margin": 46.39246368408203, "fcm_dpo/q_t": 0.40447816252708435, "grad_norm": 20.638565063476562, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.5457350015640259, "logits/rejected": -0.5298566222190857, "logps/chosen": -106.9061508178711, "logps/ref_chosen": -61.33863830566406, "logps/ref_rejected": -87.775390625, "logps/rejected": -179.73536682128906, "loss": 1.1155, "margin_dpo/margin_mean": 46.392459869384766, "margin_dpo/margin_std": 77.1932144165039, "step": 123 }, { "KL/chosen_KL_mean": -52.27129364013672, "KL/mean": -86.85035705566406, "KL/rejected_KL_mean": -121.42942810058594, "KL/std": 79.01206970214844, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.009129964746534824, "fcm_dpo/delta": -0.2478792667388916, "fcm_dpo/margin": 69.15812683105469, "fcm_dpo/q_t": 0.371742308139801, "grad_norm": 24.180330276489258, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.5349669456481934, "logits/rejected": -0.5380312204360962, "logps/chosen": -123.7196273803711, "logps/ref_chosen": -71.44833374023438, "logps/ref_rejected": -117.58056640625, "logps/rejected": -239.00999450683594, "loss": 1.0206, "margin_dpo/margin_mean": 69.15812683105469, "margin_dpo/margin_std": 99.04975891113281, "step": 124 }, { "KL/chosen_KL_mean": -43.38472366333008, "KL/mean": -73.71883392333984, "KL/rejected_KL_mean": -104.05294799804688, "KL/std": 66.72522735595703, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.008861461654305458, "fcm_dpo/delta": -0.14523999392986298, "fcm_dpo/margin": 60.668209075927734, "fcm_dpo/q_t": 0.3841526210308075, "grad_norm": 18.966266632080078, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.4806230664253235, "logits/rejected": -0.4828706979751587, "logps/chosen": -93.52166748046875, "logps/ref_chosen": -50.136940002441406, "logps/ref_rejected": -83.98861694335938, "logps/rejected": -188.04156494140625, "loss": 1.0392, "margin_dpo/margin_mean": 60.668209075927734, "margin_dpo/margin_std": 83.17915344238281, "step": 125 }, { "KL/chosen_KL_mean": -46.75824737548828, "KL/mean": -75.15432739257812, "KL/rejected_KL_mean": -103.55040740966797, "KL/std": 60.45394515991211, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.008667214773595333, "fcm_dpo/delta": -0.06482094526290894, "fcm_dpo/margin": 56.79215621948242, "fcm_dpo/q_t": 0.3921211361885071, "grad_norm": 20.3935546875, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.5126634836196899, "logits/rejected": -0.5087581872940063, "logps/chosen": -102.42532348632812, "logps/ref_chosen": -55.66706848144531, "logps/ref_rejected": -98.1297607421875, "logps/rejected": -201.68017578125, "loss": 1.0527, "margin_dpo/margin_mean": 56.792152404785156, "margin_dpo/margin_std": 77.17770385742188, "step": 126 }, { "KL/chosen_KL_mean": -45.504390716552734, "KL/mean": -66.04163360595703, "KL/rejected_KL_mean": -86.57887268066406, "KL/std": 52.939697265625, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.008667214773595333, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.07448196411133, "fcm_dpo/q_t": 0.417339950799942, "grad_norm": 20.634532928466797, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.5034447908401489, "logits/rejected": -0.48421603441238403, "logps/chosen": -102.05906677246094, "logps/ref_chosen": -56.55467987060547, "logps/ref_rejected": -76.7957763671875, "logps/rejected": -163.37464904785156, "loss": 1.1409, "margin_dpo/margin_mean": 41.07448196411133, "margin_dpo/margin_std": 66.85935974121094, "step": 127 }, { "KL/chosen_KL_mean": -48.32811737060547, "KL/mean": -70.08799743652344, "KL/rejected_KL_mean": -91.84788513183594, "KL/std": 61.827518463134766, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.008642185479402542, "fcm_dpo/delta": -0.028961315751075745, "fcm_dpo/margin": 43.519771575927734, "fcm_dpo/q_t": 0.41590309143066406, "grad_norm": 26.325315475463867, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.5076520442962646, "logits/rejected": -0.4892594516277313, "logps/chosen": -106.4490737915039, "logps/ref_chosen": -58.12095642089844, "logps/ref_rejected": -76.43896484375, "logps/rejected": -168.28684997558594, "loss": 1.1502, "margin_dpo/margin_mean": 43.519771575927734, "margin_dpo/margin_std": 77.54118347167969, "step": 128 }, { "KL/chosen_KL_mean": -57.91839599609375, "KL/mean": -80.90028381347656, "KL/rejected_KL_mean": -103.88216400146484, "KL/std": 68.20243835449219, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.008617157116532326, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.963768005371094, "fcm_dpo/q_t": 0.41664600372314453, "grad_norm": 19.90182113647461, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.5122944116592407, "logits/rejected": -0.5017467737197876, "logps/chosen": -124.83477020263672, "logps/ref_chosen": -66.91637420654297, "logps/ref_rejected": -96.6422119140625, "logps/rejected": -200.52438354492188, "loss": 1.1552, "margin_dpo/margin_mean": 45.963768005371094, "margin_dpo/margin_std": 88.79536437988281, "step": 129 }, { "KL/chosen_KL_mean": -42.74785614013672, "KL/mean": -77.1688232421875, "KL/rejected_KL_mean": -111.58978271484375, "KL/std": 69.51666259765625, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.008449437096714973, "fcm_dpo/delta": -0.19852416217327118, "fcm_dpo/margin": 68.84193420410156, "fcm_dpo/q_t": 0.3743343651294708, "grad_norm": 19.98097801208496, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.4600975513458252, "logits/rejected": -0.45375195145606995, "logps/chosen": -87.41470336914062, "logps/ref_chosen": -44.66685104370117, "logps/ref_rejected": -82.78165435791016, "logps/rejected": -194.37142944335938, "loss": 1.0076, "margin_dpo/margin_mean": 68.84193420410156, "margin_dpo/margin_std": 84.98545837402344, "step": 130 }, { "KL/chosen_KL_mean": -38.25885772705078, "KL/mean": -72.83998107910156, "KL/rejected_KL_mean": -107.42111206054688, "KL/std": 69.28550720214844, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.008066684007644653, "fcm_dpo/delta": -0.16752536594867706, "fcm_dpo/margin": 69.16224670410156, "fcm_dpo/q_t": 0.3724893033504486, "grad_norm": 28.86225700378418, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.48446083068847656, "logits/rejected": -0.4910423159599304, "logps/chosen": -83.18344116210938, "logps/ref_chosen": -44.924591064453125, "logps/ref_rejected": -88.44401550292969, "logps/rejected": -195.8651123046875, "loss": 0.9888, "margin_dpo/margin_mean": 69.16224670410156, "margin_dpo/margin_std": 75.45245361328125, "step": 131 }, { "KL/chosen_KL_mean": -50.65435791015625, "KL/mean": -78.87272644042969, "KL/rejected_KL_mean": -107.09109497070312, "KL/std": 68.87550354003906, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.00793751422315836, "fcm_dpo/delta": -0.044726189225912094, "fcm_dpo/margin": 56.436744689941406, "fcm_dpo/q_t": 0.404166042804718, "grad_norm": 19.247264862060547, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.5044941902160645, "logits/rejected": -0.5029531717300415, "logps/chosen": -109.65544128417969, "logps/ref_chosen": -59.00108337402344, "logps/ref_rejected": -87.89215087890625, "logps/rejected": -194.98324584960938, "loss": 1.0959, "margin_dpo/margin_mean": 56.436744689941406, "margin_dpo/margin_std": 89.56974792480469, "step": 132 }, { "KL/chosen_KL_mean": -59.93231964111328, "KL/mean": -84.84762573242188, "KL/rejected_KL_mean": -109.76292419433594, "KL/std": 62.10484313964844, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.00793751422315836, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 49.83061981201172, "fcm_dpo/q_t": 0.41286903619766235, "grad_norm": 26.079952239990234, "learning_rate": 4.870401618977415e-07, "logits/chosen": -0.5067698955535889, "logits/rejected": -0.4933437705039978, "logps/chosen": -126.53681945800781, "logps/ref_chosen": -66.60449981689453, "logps/ref_rejected": -96.33355712890625, "logps/rejected": -206.09649658203125, "loss": 1.1177, "margin_dpo/margin_mean": 49.830623626708984, "margin_dpo/margin_std": 79.16134643554688, "step": 133 }, { "KL/chosen_KL_mean": -47.43791198730469, "KL/mean": -74.1932601928711, "KL/rejected_KL_mean": -100.9486083984375, "KL/std": 60.470733642578125, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.007902203127741814, "fcm_dpo/delta": -0.04468559846282005, "fcm_dpo/margin": 53.51069641113281, "fcm_dpo/q_t": 0.4046560823917389, "grad_norm": 18.731077194213867, "learning_rate": 4.866292092063986e-07, "logits/chosen": -0.4694488048553467, "logits/rejected": -0.4564399719238281, "logps/chosen": -99.50717163085938, "logps/ref_chosen": -52.06925582885742, "logps/ref_rejected": -87.6545181274414, "logps/rejected": -188.60311889648438, "loss": 1.0759, "margin_dpo/margin_mean": 53.51069641113281, "margin_dpo/margin_std": 70.26188659667969, "step": 134 }, { "KL/chosen_KL_mean": -52.651573181152344, "KL/mean": -91.39772033691406, "KL/rejected_KL_mean": -130.1438446044922, "KL/std": 81.02012634277344, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.007630997337400913, "fcm_dpo/delta": -0.2034302055835724, "fcm_dpo/margin": 77.4922866821289, "fcm_dpo/q_t": 0.37322452664375305, "grad_norm": 21.743507385253906, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.4417390823364258, "logits/rejected": -0.47695955634117126, "logps/chosen": -103.00543212890625, "logps/ref_chosen": -50.353858947753906, "logps/ref_rejected": -115.97975158691406, "logps/rejected": -246.12359619140625, "loss": 1.0012, "margin_dpo/margin_mean": 77.4922866821289, "margin_dpo/margin_std": 98.09565734863281, "step": 135 }, { "KL/chosen_KL_mean": -62.86431884765625, "KL/mean": -88.71379089355469, "KL/rejected_KL_mean": -114.56326293945312, "KL/std": 73.90873718261719, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.007553243078291416, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.698944091796875, "fcm_dpo/q_t": 0.4202331304550171, "grad_norm": 20.616310119628906, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.4841381907463074, "logits/rejected": -0.47736936807632446, "logps/chosen": -127.93682861328125, "logps/ref_chosen": -65.072509765625, "logps/ref_rejected": -96.32122802734375, "logps/rejected": -210.88449096679688, "loss": 1.1472, "margin_dpo/margin_mean": 51.698944091796875, "margin_dpo/margin_std": 95.19708251953125, "step": 136 }, { "KL/chosen_KL_mean": -60.16282272338867, "KL/mean": -100.787841796875, "KL/rejected_KL_mean": -141.41287231445312, "KL/std": 97.6104736328125, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.00731696980074048, "fcm_dpo/delta": -0.20704714953899384, "fcm_dpo/margin": 81.25003051757812, "fcm_dpo/q_t": 0.3785414397716522, "grad_norm": 19.7189884185791, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.4397382140159607, "logits/rejected": -0.4667167663574219, "logps/chosen": -108.92193603515625, "logps/ref_chosen": -48.759117126464844, "logps/ref_rejected": -113.86376953125, "logps/rejected": -255.27664184570312, "loss": 1.0296, "margin_dpo/margin_mean": 81.25003051757812, "margin_dpo/margin_std": 117.20115661621094, "step": 137 }, { "KL/chosen_KL_mean": -62.80693435668945, "KL/mean": -93.51780700683594, "KL/rejected_KL_mean": -124.22868347167969, "KL/std": 74.23326110839844, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.007216614671051502, "fcm_dpo/delta": -0.041904207319021225, "fcm_dpo/margin": 61.4217414855957, "fcm_dpo/q_t": 0.39843374490737915, "grad_norm": 20.12140655517578, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.43256235122680664, "logits/rejected": -0.4217318296432495, "logps/chosen": -123.32658386230469, "logps/ref_chosen": -60.519649505615234, "logps/ref_rejected": -93.19694519042969, "logps/rejected": -217.42562866210938, "loss": 1.0607, "margin_dpo/margin_mean": 61.42173767089844, "margin_dpo/margin_std": 75.93879699707031, "step": 138 }, { "KL/chosen_KL_mean": -52.439796447753906, "KL/mean": -88.96459197998047, "KL/rejected_KL_mean": -125.48937225341797, "KL/std": 70.41232299804688, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.00707232067361474, "fcm_dpo/delta": -0.12281207740306854, "fcm_dpo/margin": 73.04957580566406, "fcm_dpo/q_t": 0.38536033034324646, "grad_norm": 18.63878059387207, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.44589370489120483, "logits/rejected": -0.43810388445854187, "logps/chosen": -99.3311767578125, "logps/ref_chosen": -46.89138412475586, "logps/ref_rejected": -79.72798156738281, "logps/rejected": -205.21734619140625, "loss": 1.0259, "margin_dpo/margin_mean": 73.04957580566406, "margin_dpo/margin_std": 91.57994079589844, "step": 139 }, { "KL/chosen_KL_mean": -63.55427551269531, "KL/mean": -96.54176330566406, "KL/rejected_KL_mean": -129.5292510986328, "KL/std": 77.9601058959961, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.006944697350263596, "fcm_dpo/delta": -0.06101107597351074, "fcm_dpo/margin": 65.9749755859375, "fcm_dpo/q_t": 0.39790695905685425, "grad_norm": 20.669536590576172, "learning_rate": 4.840329401637809e-07, "logits/chosen": -0.41300731897354126, "logits/rejected": -0.3993472456932068, "logps/chosen": -122.52899169921875, "logps/ref_chosen": -58.97471618652344, "logps/ref_rejected": -83.28410339355469, "logps/rejected": -212.8133544921875, "loss": 1.0752, "margin_dpo/margin_mean": 65.97498321533203, "margin_dpo/margin_std": 93.88746643066406, "step": 140 }, { "KL/chosen_KL_mean": -71.32264709472656, "KL/mean": -102.6048583984375, "KL/rejected_KL_mean": -133.88706970214844, "KL/std": 86.29281616210938, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.006908806972205639, "fcm_dpo/delta": -0.026286346837878227, "fcm_dpo/margin": 62.564414978027344, "fcm_dpo/q_t": 0.40315961837768555, "grad_norm": 24.427490234375, "learning_rate": 4.83578576263792e-07, "logits/chosen": -0.42676639556884766, "logits/rejected": -0.41550058126449585, "logps/chosen": -146.3983154296875, "logps/ref_chosen": -75.07566833496094, "logps/ref_rejected": -98.1922607421875, "logps/rejected": -232.07933044433594, "loss": 1.1076, "margin_dpo/margin_mean": 62.56441879272461, "margin_dpo/margin_std": 99.85594177246094, "step": 141 }, { "KL/chosen_KL_mean": -72.5107421875, "KL/mean": -108.30811309814453, "KL/rejected_KL_mean": -144.10546875, "KL/std": 93.80851745605469, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.0068291304633021355, "fcm_dpo/delta": -0.09008042514324188, "fcm_dpo/margin": 71.59473419189453, "fcm_dpo/q_t": 0.3943261504173279, "grad_norm": 26.210159301757812, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.3901626169681549, "logits/rejected": -0.3892650604248047, "logps/chosen": -130.53866577148438, "logps/ref_chosen": -58.027931213378906, "logps/ref_rejected": -94.58222961425781, "logps/rejected": -238.6876983642578, "loss": 1.0881, "margin_dpo/margin_mean": 71.5947265625, "margin_dpo/margin_std": 110.5821533203125, "step": 142 }, { "KL/chosen_KL_mean": -76.32169342041016, "KL/mean": -99.2608871459961, "KL/rejected_KL_mean": -122.2000732421875, "KL/std": 82.2864990234375, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.006767615210264921, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.878387451171875, "fcm_dpo/q_t": 0.4329405725002289, "grad_norm": 22.109413146972656, "learning_rate": 4.826513955607734e-07, "logits/chosen": -0.37516605854034424, "logits/rejected": -0.3687092661857605, "logps/chosen": -133.91815185546875, "logps/ref_chosen": -57.59645080566406, "logps/ref_rejected": -78.99957275390625, "logps/rejected": -201.19964599609375, "loss": 1.1982, "margin_dpo/margin_mean": 45.878387451171875, "margin_dpo/margin_std": 96.17039489746094, "step": 143 }, { "KL/chosen_KL_mean": -68.8632583618164, "KL/mean": -96.65010070800781, "KL/rejected_KL_mean": -124.43695068359375, "KL/std": 69.90775299072266, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.006767615210264921, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.573699951171875, "fcm_dpo/q_t": 0.41452693939208984, "grad_norm": 20.535696029663086, "learning_rate": 4.821786031898176e-07, "logits/chosen": -0.3825682997703552, "logits/rejected": -0.37160438299179077, "logps/chosen": -128.76962280273438, "logps/ref_chosen": -59.90636444091797, "logps/ref_rejected": -82.00025939941406, "logps/rejected": -206.4372100830078, "loss": 1.118, "margin_dpo/margin_mean": 55.57369613647461, "margin_dpo/margin_std": 82.33287048339844, "step": 144 }, { "KL/chosen_KL_mean": -65.35738372802734, "KL/mean": -95.54832458496094, "KL/rejected_KL_mean": -125.73927307128906, "KL/std": 69.90843963623047, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.006767615210264921, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.381893157958984, "fcm_dpo/q_t": 0.4071277678012848, "grad_norm": 23.468578338623047, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.426013708114624, "logits/rejected": -0.4119391441345215, "logps/chosen": -121.95805358886719, "logps/ref_chosen": -56.60066604614258, "logps/ref_rejected": -77.86631774902344, "logps/rejected": -203.6055908203125, "loss": 1.0948, "margin_dpo/margin_mean": 60.38189697265625, "margin_dpo/margin_std": 84.77351379394531, "step": 145 }, { "KL/chosen_KL_mean": -89.39501953125, "KL/mean": -112.8216323852539, "KL/rejected_KL_mean": -136.2482452392578, "KL/std": 76.4080810546875, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.006767615210264921, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 46.85322570800781, "fcm_dpo/q_t": 0.42726507782936096, "grad_norm": 25.80341339111328, "learning_rate": 4.812146767012779e-07, "logits/chosen": -0.39261966943740845, "logits/rejected": -0.36783578991889954, "logps/chosen": -155.39547729492188, "logps/ref_chosen": -66.00045013427734, "logps/ref_rejected": -81.70278930664062, "logps/rejected": -217.95103454589844, "loss": 1.1871, "margin_dpo/margin_mean": 46.85322570800781, "margin_dpo/margin_std": 91.74784851074219, "step": 146 }, { "KL/chosen_KL_mean": -65.36270141601562, "KL/mean": -96.96876525878906, "KL/rejected_KL_mean": -128.5748291015625, "KL/std": 76.57128143310547, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.006728795822709799, "fcm_dpo/delta": -0.028762944042682648, "fcm_dpo/margin": 63.212127685546875, "fcm_dpo/q_t": 0.40543532371520996, "grad_norm": 19.389053344726562, "learning_rate": 4.807235679840536e-07, "logits/chosen": -0.4179861545562744, "logits/rejected": -0.3990488648414612, "logps/chosen": -118.7681884765625, "logps/ref_chosen": -53.405487060546875, "logps/ref_rejected": -71.39060974121094, "logps/rejected": -199.9654541015625, "loss": 1.0988, "margin_dpo/margin_mean": 63.212127685546875, "margin_dpo/margin_std": 95.10125732421875, "step": 147 }, { "KL/chosen_KL_mean": -63.96573257446289, "KL/mean": -91.10762786865234, "KL/rejected_KL_mean": -118.24954223632812, "KL/std": 76.30313873291016, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.006679433397948742, "fcm_dpo/delta": -0.07390298694372177, "fcm_dpo/margin": 54.2838020324707, "fcm_dpo/q_t": 0.4192584753036499, "grad_norm": 18.27964210510254, "learning_rate": 4.802263794862384e-07, "logits/chosen": -0.47013741731643677, "logits/rejected": -0.463634192943573, "logps/chosen": -128.90281677246094, "logps/ref_chosen": -64.93708038330078, "logps/ref_rejected": -103.09384155273438, "logps/rejected": -221.3433837890625, "loss": 1.1319, "margin_dpo/margin_mean": 54.2838020324707, "margin_dpo/margin_std": 80.29562377929688, "step": 148 }, { "KL/chosen_KL_mean": -61.568382263183594, "KL/mean": -95.15155029296875, "KL/rejected_KL_mean": -128.73471069335938, "KL/std": 68.74234771728516, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.006500914692878723, "fcm_dpo/delta": -0.0983634814620018, "fcm_dpo/margin": 67.16634368896484, "fcm_dpo/q_t": 0.3992913067340851, "grad_norm": 17.962369918823242, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.4860546588897705, "logits/rejected": -0.4728388786315918, "logps/chosen": -120.04214477539062, "logps/ref_chosen": -58.47376251220703, "logps/ref_rejected": -99.31474304199219, "logps/rejected": -228.04946899414062, "loss": 1.0664, "margin_dpo/margin_mean": 67.16634368896484, "margin_dpo/margin_std": 81.42725372314453, "step": 149 }, { "KL/chosen_KL_mean": -54.187557220458984, "KL/mean": -87.24864196777344, "KL/rejected_KL_mean": -120.30972290039062, "KL/std": 80.83369445800781, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.006397702731192112, "fcm_dpo/delta": -0.08001944422721863, "fcm_dpo/margin": 66.1221694946289, "fcm_dpo/q_t": 0.407664030790329, "grad_norm": 17.84654998779297, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.42700618505477905, "logits/rejected": -0.4307962954044342, "logps/chosen": -99.89337158203125, "logps/ref_chosen": -45.705810546875, "logps/ref_rejected": -83.34759521484375, "logps/rejected": -203.65731811523438, "loss": 1.0908, "margin_dpo/margin_mean": 66.1221694946289, "margin_dpo/margin_std": 96.51167297363281, "step": 150 }, { "KL/chosen_KL_mean": -67.81385803222656, "KL/mean": -101.16264343261719, "KL/rejected_KL_mean": -134.51144409179688, "KL/std": 76.48846435546875, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.006397702731192112, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.69757080078125, "fcm_dpo/q_t": 0.4020676910877228, "grad_norm": 18.534128189086914, "learning_rate": 4.786984671220053e-07, "logits/chosen": -0.5162950754165649, "logits/rejected": -0.4906059503555298, "logps/chosen": -138.3846893310547, "logps/ref_chosen": -70.57083129882812, "logps/ref_rejected": -100.46382141113281, "logps/rejected": -234.97525024414062, "loss": 1.0707, "margin_dpo/margin_mean": 66.69757080078125, "margin_dpo/margin_std": 83.75027465820312, "step": 151 }, { "KL/chosen_KL_mean": -59.39875030517578, "KL/mean": -99.93858337402344, "KL/rejected_KL_mean": -140.47842407226562, "KL/std": 78.93248748779297, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.006302128545939922, "fcm_dpo/delta": -0.116790771484375, "fcm_dpo/margin": 81.07966613769531, "fcm_dpo/q_t": 0.38571709394454956, "grad_norm": 20.45277214050293, "learning_rate": 4.78177092112495e-07, "logits/chosen": -0.48105257749557495, "logits/rejected": -0.48062583804130554, "logps/chosen": -119.56314086914062, "logps/ref_chosen": -60.16438674926758, "logps/ref_rejected": -106.14045715332031, "logps/rejected": -246.61886596679688, "loss": 1.0258, "margin_dpo/margin_mean": 81.07966613769531, "margin_dpo/margin_std": 96.61679077148438, "step": 152 }, { "KL/chosen_KL_mean": -60.735313415527344, "KL/mean": -95.41152954101562, "KL/rejected_KL_mean": -130.08775329589844, "KL/std": 86.6027603149414, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.006227484904229641, "fcm_dpo/delta": -0.036148011684417725, "fcm_dpo/margin": 69.35243225097656, "fcm_dpo/q_t": 0.4058881402015686, "grad_norm": 15.226004600524902, "learning_rate": 4.776497044244016e-07, "logits/chosen": -0.450962096452713, "logits/rejected": -0.44640836119651794, "logps/chosen": -117.05059051513672, "logps/ref_chosen": -56.315277099609375, "logps/ref_rejected": -85.65583801269531, "logps/rejected": -215.74359130859375, "loss": 1.0967, "margin_dpo/margin_mean": 69.35243225097656, "margin_dpo/margin_std": 106.32671356201172, "step": 153 }, { "KL/chosen_KL_mean": -72.78416442871094, "KL/mean": -105.78938293457031, "KL/rejected_KL_mean": -138.7946014404297, "KL/std": 86.74234008789062, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.006178042385727167, "fcm_dpo/delta": -0.04359261319041252, "fcm_dpo/margin": 66.01043701171875, "fcm_dpo/q_t": 0.4098336100578308, "grad_norm": 18.44878578186035, "learning_rate": 4.771163179548808e-07, "logits/chosen": -0.4380859136581421, "logits/rejected": -0.44127899408340454, "logps/chosen": -135.5267333984375, "logps/ref_chosen": -62.74256896972656, "logps/ref_rejected": -104.24420166015625, "logps/rejected": -243.03880310058594, "loss": 1.129, "margin_dpo/margin_mean": 66.01044464111328, "margin_dpo/margin_std": 111.0909423828125, "step": 154 }, { "KL/chosen_KL_mean": -66.35116577148438, "KL/mean": -99.84371948242188, "KL/rejected_KL_mean": -133.33627319335938, "KL/std": 80.27971649169922, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.006131485104560852, "fcm_dpo/delta": -0.03200829401612282, "fcm_dpo/margin": 66.985107421875, "fcm_dpo/q_t": 0.4072626829147339, "grad_norm": 18.364580154418945, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.4624621868133545, "logits/rejected": -0.44429630041122437, "logps/chosen": -127.00434875488281, "logps/ref_chosen": -60.65318298339844, "logps/ref_rejected": -77.49220275878906, "logps/rejected": -210.82847595214844, "loss": 1.1023, "margin_dpo/margin_mean": 66.985107421875, "margin_dpo/margin_std": 99.73149108886719, "step": 155 }, { "KL/chosen_KL_mean": -93.23342895507812, "KL/mean": -112.24707794189453, "KL/rejected_KL_mean": -131.26072692871094, "KL/std": 85.90792846679688, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.00611185934394598, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.027313232421875, "fcm_dpo/q_t": 0.4467718005180359, "grad_norm": 27.262943267822266, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -0.4021250605583191, "logits/rejected": -0.3974895477294922, "logps/chosen": -162.72531127929688, "logps/ref_chosen": -69.49188232421875, "logps/ref_rejected": -77.16929626464844, "logps/rejected": -208.43002319335938, "loss": 1.2791, "margin_dpo/margin_mean": 38.02731704711914, "margin_dpo/margin_std": 112.55517578125, "step": 156 }, { "KL/chosen_KL_mean": -83.86450958251953, "KL/mean": -126.67929077148438, "KL/rejected_KL_mean": -169.49407958984375, "KL/std": 93.74600219726562, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.005980256479233503, "fcm_dpo/delta": -0.10883772373199463, "fcm_dpo/margin": 85.62956237792969, "fcm_dpo/q_t": 0.38287657499313354, "grad_norm": 23.121597290039062, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -0.37844234704971313, "logits/rejected": -0.3867532014846802, "logps/chosen": -145.23294067382812, "logps/ref_chosen": -61.368438720703125, "logps/ref_rejected": -107.64636993408203, "logps/rejected": -277.14044189453125, "loss": 1.0374, "margin_dpo/margin_mean": 85.62956237792969, "margin_dpo/margin_std": 106.631591796875, "step": 157 }, { "KL/chosen_KL_mean": -84.5057144165039, "KL/mean": -130.03994750976562, "KL/rejected_KL_mean": -175.57415771484375, "KL/std": 115.93599700927734, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.005857234820723534, "fcm_dpo/delta": -0.14090833067893982, "fcm_dpo/margin": 91.06845092773438, "fcm_dpo/q_t": 0.38985028862953186, "grad_norm": 18.224163055419922, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -0.3638108968734741, "logits/rejected": -0.38175833225250244, "logps/chosen": -142.11863708496094, "logps/ref_chosen": -57.612918853759766, "logps/ref_rejected": -113.6946792602539, "logps/rejected": -289.26885986328125, "loss": 1.0635, "margin_dpo/margin_mean": 91.06845092773438, "margin_dpo/margin_std": 140.60455322265625, "step": 158 }, { "KL/chosen_KL_mean": -95.19334411621094, "KL/mean": -126.32515716552734, "KL/rejected_KL_mean": -157.45697021484375, "KL/std": 102.09504699707031, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.005800278857350349, "fcm_dpo/delta": -0.02378561533987522, "fcm_dpo/margin": 62.26362991333008, "fcm_dpo/q_t": 0.41876220703125, "grad_norm": 20.06854820251465, "learning_rate": 4.743599013306165e-07, "logits/chosen": -0.3674774169921875, "logits/rejected": -0.3357643485069275, "logps/chosen": -176.75369262695312, "logps/ref_chosen": -81.56034851074219, "logps/ref_rejected": -88.89871215820312, "logps/rejected": -246.35568237304688, "loss": 1.1549, "margin_dpo/margin_mean": 62.263633728027344, "margin_dpo/margin_std": 110.5584487915039, "step": 159 }, { "KL/chosen_KL_mean": -97.48786163330078, "KL/mean": -139.24900817871094, "KL/rejected_KL_mean": -181.01016235351562, "KL/std": 109.07857513427734, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.00568758137524128, "fcm_dpo/delta": -0.08619771152734756, "fcm_dpo/margin": 83.52229309082031, "fcm_dpo/q_t": 0.39961978793144226, "grad_norm": 22.184825897216797, "learning_rate": 4.737908228387656e-07, "logits/chosen": -0.3567469120025635, "logits/rejected": -0.3495233654975891, "logps/chosen": -163.21875, "logps/ref_chosen": -65.73088073730469, "logps/ref_rejected": -97.21781921386719, "logps/rejected": -278.22796630859375, "loss": 1.0994, "margin_dpo/margin_mean": 83.52229309082031, "margin_dpo/margin_std": 136.4792938232422, "step": 160 }, { "KL/chosen_KL_mean": -82.90977478027344, "KL/mean": -119.42204284667969, "KL/rejected_KL_mean": -155.93431091308594, "KL/std": 85.58193969726562, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.00568758137524128, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.02452087402344, "fcm_dpo/q_t": 0.40774428844451904, "grad_norm": 20.76175308227539, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -0.35023656487464905, "logits/rejected": -0.350460946559906, "logps/chosen": -135.34625244140625, "logps/ref_chosen": -52.43647003173828, "logps/ref_rejected": -83.43095397949219, "logps/rejected": -239.36526489257812, "loss": 1.1007, "margin_dpo/margin_mean": 73.02452087402344, "margin_dpo/margin_std": 108.37924194335938, "step": 161 }, { "KL/chosen_KL_mean": -80.26211547851562, "KL/mean": -116.17857360839844, "KL/rejected_KL_mean": -152.09503173828125, "KL/std": 94.54278564453125, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.005591993685811758, "fcm_dpo/delta": -0.08474621176719666, "fcm_dpo/margin": 71.83290100097656, "fcm_dpo/q_t": 0.4103257656097412, "grad_norm": 21.48539161682129, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.40853458642959595, "logits/rejected": -0.3933084309101105, "logps/chosen": -142.87269592285156, "logps/ref_chosen": -62.6105842590332, "logps/ref_rejected": -89.39057922363281, "logps/rejected": -241.485595703125, "loss": 1.1179, "margin_dpo/margin_mean": 71.83290100097656, "margin_dpo/margin_std": 112.16954040527344, "step": 162 }, { "KL/chosen_KL_mean": -90.45162200927734, "KL/mean": -125.58489990234375, "KL/rejected_KL_mean": -160.71817016601562, "KL/std": 95.8338623046875, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.005591993685811758, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.26655578613281, "fcm_dpo/q_t": 0.4135151505470276, "grad_norm": 19.8275146484375, "learning_rate": 4.720482655449212e-07, "logits/chosen": -0.33946388959884644, "logits/rejected": -0.3220199942588806, "logps/chosen": -145.47325134277344, "logps/ref_chosen": -55.021629333496094, "logps/ref_rejected": -75.418212890625, "logps/rejected": -236.1363983154297, "loss": 1.126, "margin_dpo/margin_mean": 70.26655578613281, "margin_dpo/margin_std": 115.48532104492188, "step": 163 }, { "KL/chosen_KL_mean": -82.01603698730469, "KL/mean": -125.43939208984375, "KL/rejected_KL_mean": -168.86273193359375, "KL/std": 93.03357696533203, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.0054749310947954655, "fcm_dpo/delta": -0.10578101128339767, "fcm_dpo/margin": 86.84669494628906, "fcm_dpo/q_t": 0.3918527662754059, "grad_norm": 19.96858787536621, "learning_rate": 4.714556901942599e-07, "logits/chosen": -0.3466936945915222, "logits/rejected": -0.3345338702201843, "logps/chosen": -137.65670776367188, "logps/ref_chosen": -55.64066696166992, "logps/ref_rejected": -79.66463470458984, "logps/rejected": -248.52735900878906, "loss": 1.0454, "margin_dpo/margin_mean": 86.84669494628906, "margin_dpo/margin_std": 106.67892456054688, "step": 164 }, { "KL/chosen_KL_mean": -88.45890045166016, "KL/mean": -116.44608306884766, "KL/rejected_KL_mean": -144.4332733154297, "KL/std": 78.84156799316406, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.0054749310947954655, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.974361419677734, "fcm_dpo/q_t": 0.43027693033218384, "grad_norm": 21.361570358276367, "learning_rate": 4.708572792802069e-07, "logits/chosen": -0.3570517301559448, "logits/rejected": -0.3310251235961914, "logps/chosen": -149.76959228515625, "logps/ref_chosen": -61.310691833496094, "logps/ref_rejected": -73.67060852050781, "logps/rejected": -218.1038818359375, "loss": 1.1816, "margin_dpo/margin_mean": 55.974365234375, "margin_dpo/margin_std": 105.02626037597656, "step": 165 }, { "KL/chosen_KL_mean": -78.31771850585938, "KL/mean": -130.20628356933594, "KL/rejected_KL_mean": -182.0948486328125, "KL/std": 114.64451599121094, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.005322150886058807, "fcm_dpo/delta": -0.16226670145988464, "fcm_dpo/margin": 103.7771224975586, "fcm_dpo/q_t": 0.3852335214614868, "grad_norm": 17.009685516357422, "learning_rate": 4.702530485714461e-07, "logits/chosen": -0.34249216318130493, "logits/rejected": -0.35319170355796814, "logps/chosen": -129.3013153076172, "logps/ref_chosen": -50.98360061645508, "logps/ref_rejected": -98.09512329101562, "logps/rejected": -280.1899719238281, "loss": 1.0298, "margin_dpo/margin_mean": 103.77713012695312, "margin_dpo/margin_std": 144.28770446777344, "step": 166 }, { "KL/chosen_KL_mean": -78.3003158569336, "KL/mean": -133.010009765625, "KL/rejected_KL_mean": -187.7197265625, "KL/std": 104.11771392822266, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.0051628341898322105, "fcm_dpo/delta": -0.17484354972839355, "fcm_dpo/margin": 109.41941833496094, "fcm_dpo/q_t": 0.3730447590351105, "grad_norm": 19.03751564025879, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.34518417716026306, "logits/rejected": -0.348605215549469, "logps/chosen": -128.7244110107422, "logps/ref_chosen": -50.424095153808594, "logps/ref_rejected": -96.03042602539062, "logps/rejected": -283.7501525878906, "loss": 0.9857, "margin_dpo/margin_mean": 109.41941833496094, "margin_dpo/margin_std": 120.54164123535156, "step": 167 }, { "KL/chosen_KL_mean": -85.6524658203125, "KL/mean": -125.55575561523438, "KL/rejected_KL_mean": -165.4590606689453, "KL/std": 97.6776123046875, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.005095277912914753, "fcm_dpo/delta": -0.0222168006002903, "fcm_dpo/margin": 79.80659484863281, "fcm_dpo/q_t": 0.408231645822525, "grad_norm": 20.391569137573242, "learning_rate": 4.690271916109034e-07, "logits/chosen": -0.3299752473831177, "logits/rejected": -0.32102906703948975, "logps/chosen": -135.11529541015625, "logps/ref_chosen": -49.462825775146484, "logps/ref_rejected": -75.30855560302734, "logps/rejected": -240.76760864257812, "loss": 1.0914, "margin_dpo/margin_mean": 79.80659484863281, "margin_dpo/margin_std": 109.659912109375, "step": 168 }, { "KL/chosen_KL_mean": -88.48287200927734, "KL/mean": -123.318603515625, "KL/rejected_KL_mean": -158.15432739257812, "KL/std": 96.05093383789062, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.005034225527197123, "fcm_dpo/delta": -0.06027255952358246, "fcm_dpo/margin": 69.67146301269531, "fcm_dpo/q_t": 0.42353829741477966, "grad_norm": 19.06806755065918, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -0.34940391778945923, "logits/rejected": -0.3333583474159241, "logps/chosen": -148.28631591796875, "logps/ref_chosen": -59.803443908691406, "logps/ref_rejected": -83.34574890136719, "logps/rejected": -241.5000762939453, "loss": 1.169, "margin_dpo/margin_mean": 69.67145538330078, "margin_dpo/margin_std": 132.4425048828125, "step": 169 }, { "KL/chosen_KL_mean": -78.66925048828125, "KL/mean": -119.93560791015625, "KL/rejected_KL_mean": -161.20196533203125, "KL/std": 92.10433959960938, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.004970300942659378, "fcm_dpo/delta": -0.06389647722244263, "fcm_dpo/margin": 82.53272247314453, "fcm_dpo/q_t": 0.4052248001098633, "grad_norm": 17.099119186401367, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -0.30527323484420776, "logits/rejected": -0.29494979977607727, "logps/chosen": -128.14102172851562, "logps/ref_chosen": -49.471771240234375, "logps/ref_rejected": -75.91734313964844, "logps/rejected": -237.11929321289062, "loss": 1.0902, "margin_dpo/margin_mean": 82.53272247314453, "margin_dpo/margin_std": 110.43910217285156, "step": 170 }, { "KL/chosen_KL_mean": -114.73477172851562, "KL/mean": -147.73040771484375, "KL/rejected_KL_mean": -180.72601318359375, "KL/std": 106.39347839355469, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.004970300942659378, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.99124908447266, "fcm_dpo/q_t": 0.42960548400878906, "grad_norm": 33.073822021484375, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -0.3449512720108032, "logits/rejected": -0.3299598693847656, "logps/chosen": -199.2340850830078, "logps/ref_chosen": -84.49931335449219, "logps/ref_rejected": -109.38209533691406, "logps/rejected": -290.10809326171875, "loss": 1.1864, "margin_dpo/margin_mean": 65.99124908447266, "margin_dpo/margin_std": 136.73788452148438, "step": 171 }, { "KL/chosen_KL_mean": -99.86283874511719, "KL/mean": -134.49224853515625, "KL/rejected_KL_mean": -169.12164306640625, "KL/std": 102.2164535522461, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.004970300942659378, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.2587890625, "fcm_dpo/q_t": 0.4216139614582062, "grad_norm": 18.176748275756836, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -0.39472442865371704, "logits/rejected": -0.3820643723011017, "logps/chosen": -168.51675415039062, "logps/ref_chosen": -68.65391540527344, "logps/ref_rejected": -85.43667602539062, "logps/rejected": -254.55831909179688, "loss": 1.1471, "margin_dpo/margin_mean": 69.2587890625, "margin_dpo/margin_std": 115.08087158203125, "step": 172 }, { "KL/chosen_KL_mean": -90.9837875366211, "KL/mean": -126.10589599609375, "KL/rejected_KL_mean": -161.22801208496094, "KL/std": 96.37551879882812, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.004970300942659378, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.24421691894531, "fcm_dpo/q_t": 0.4198143780231476, "grad_norm": 19.0076961517334, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -0.38060057163238525, "logits/rejected": -0.35156911611557007, "logps/chosen": -154.03466796875, "logps/ref_chosen": -63.050880432128906, "logps/ref_rejected": -78.68392181396484, "logps/rejected": -239.91192626953125, "loss": 1.1262, "margin_dpo/margin_mean": 70.24421691894531, "margin_dpo/margin_std": 101.06126403808594, "step": 173 }, { "KL/chosen_KL_mean": -87.39854431152344, "KL/mean": -129.3033447265625, "KL/rejected_KL_mean": -171.20811462402344, "KL/std": 101.4627685546875, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.004937914200127125, "fcm_dpo/delta": -0.06558923423290253, "fcm_dpo/margin": 83.8095703125, "fcm_dpo/q_t": 0.4074831008911133, "grad_norm": 26.417648315429688, "learning_rate": 4.652116329460919e-07, "logits/chosen": -0.3339517116546631, "logits/rejected": -0.35172325372695923, "logps/chosen": -140.7615203857422, "logps/ref_chosen": -53.36296844482422, "logps/ref_rejected": -101.91120910644531, "logps/rejected": -273.11932373046875, "loss": 1.1007, "margin_dpo/margin_mean": 83.80956268310547, "margin_dpo/margin_std": 121.98419189453125, "step": 174 }, { "KL/chosen_KL_mean": -79.93464660644531, "KL/mean": -137.0467529296875, "KL/rejected_KL_mean": -194.15887451171875, "KL/std": 108.39655303955078, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.004787640646100044, "fcm_dpo/delta": -0.15558278560638428, "fcm_dpo/margin": 114.22422790527344, "fcm_dpo/q_t": 0.37422001361846924, "grad_norm": 24.844676971435547, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.30949610471725464, "logits/rejected": -0.2958728075027466, "logps/chosen": -125.35240936279297, "logps/ref_chosen": -45.417762756347656, "logps/ref_rejected": -89.50579833984375, "logps/rejected": -283.6646728515625, "loss": 0.9785, "margin_dpo/margin_mean": 114.22422790527344, "margin_dpo/margin_std": 113.23709106445312, "step": 175 }, { "KL/chosen_KL_mean": -87.88569641113281, "KL/mean": -136.27401733398438, "KL/rejected_KL_mean": -184.66232299804688, "KL/std": 109.21138763427734, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.004732812754809856, "fcm_dpo/delta": -0.047373853623867035, "fcm_dpo/margin": 96.77662658691406, "fcm_dpo/q_t": 0.3987988531589508, "grad_norm": 18.611600875854492, "learning_rate": 4.638942309888058e-07, "logits/chosen": -0.34232744574546814, "logits/rejected": -0.35991567373275757, "logps/chosen": -138.33853149414062, "logps/ref_chosen": -50.452842712402344, "logps/ref_rejected": -95.5589599609375, "logps/rejected": -280.2212829589844, "loss": 1.06, "margin_dpo/margin_mean": 96.77662658691406, "margin_dpo/margin_std": 126.63328552246094, "step": 176 }, { "KL/chosen_KL_mean": -104.66439819335938, "KL/mean": -152.41554260253906, "KL/rejected_KL_mean": -200.16668701171875, "KL/std": 119.41682434082031, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.004684681072831154, "fcm_dpo/delta": -0.05488370358943939, "fcm_dpo/margin": 95.50228881835938, "fcm_dpo/q_t": 0.4001327157020569, "grad_norm": 25.9334774017334, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -0.34294700622558594, "logits/rejected": -0.3354364037513733, "logps/chosen": -165.880859375, "logps/ref_chosen": -61.216468811035156, "logps/ref_rejected": -95.89378356933594, "logps/rejected": -296.06048583984375, "loss": 1.0655, "margin_dpo/margin_mean": 95.50228881835938, "margin_dpo/margin_std": 125.90945434570312, "step": 177 }, { "KL/chosen_KL_mean": -115.59068298339844, "KL/mean": -175.99154663085938, "KL/rejected_KL_mean": -236.3924102783203, "KL/std": 140.442626953125, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.004535360727459192, "fcm_dpo/delta": -0.15726302564144135, "fcm_dpo/margin": 120.80174255371094, "fcm_dpo/q_t": 0.3810286521911621, "grad_norm": 25.16571807861328, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -0.2117362767457962, "logits/rejected": -0.20060741901397705, "logps/chosen": -173.85546875, "logps/ref_chosen": -58.26478958129883, "logps/ref_rejected": -105.3653335571289, "logps/rejected": -341.75775146484375, "loss": 1.0165, "margin_dpo/margin_mean": 120.8017349243164, "margin_dpo/margin_std": 154.52880859375, "step": 178 }, { "KL/chosen_KL_mean": -121.65875244140625, "KL/mean": -163.83865356445312, "KL/rejected_KL_mean": -206.0185546875, "KL/std": 121.72943115234375, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.004478609189391136, "fcm_dpo/delta": -0.040146298706531525, "fcm_dpo/margin": 84.35980987548828, "fcm_dpo/q_t": 0.41631701588630676, "grad_norm": 35.56280517578125, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -0.2706071734428406, "logits/rejected": -0.280755877494812, "logps/chosen": -182.71707153320312, "logps/ref_chosen": -61.05832290649414, "logps/ref_rejected": -90.52782440185547, "logps/rejected": -296.54638671875, "loss": 1.1507, "margin_dpo/margin_mean": 84.35980987548828, "margin_dpo/margin_std": 150.8819122314453, "step": 179 }, { "KL/chosen_KL_mean": -104.479248046875, "KL/mean": -159.38909912109375, "KL/rejected_KL_mean": -214.29896545410156, "KL/std": 107.94509887695312, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.004415547009557486, "fcm_dpo/delta": -0.0893382877111435, "fcm_dpo/margin": 109.81971740722656, "fcm_dpo/q_t": 0.3893117904663086, "grad_norm": 19.452550888061523, "learning_rate": 4.611919330113591e-07, "logits/chosen": -0.28269416093826294, "logits/rejected": -0.2765018045902252, "logps/chosen": -158.82196044921875, "logps/ref_chosen": -54.34272003173828, "logps/ref_rejected": -98.21183776855469, "logps/rejected": -312.51080322265625, "loss": 1.0378, "margin_dpo/margin_mean": 109.81971740722656, "margin_dpo/margin_std": 133.3249053955078, "step": 180 }, { "KL/chosen_KL_mean": -90.27413177490234, "KL/mean": -124.33741760253906, "KL/rejected_KL_mean": -158.4007110595703, "KL/std": 99.86650085449219, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.004399297758936882, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.1265869140625, "fcm_dpo/q_t": 0.43104779720306396, "grad_norm": 18.65866470336914, "learning_rate": 4.605024008834863e-07, "logits/chosen": -0.2932761311531067, "logits/rejected": -0.26991933584213257, "logps/chosen": -145.27459716796875, "logps/ref_chosen": -55.000457763671875, "logps/ref_rejected": -61.656166076660156, "logps/rejected": -220.056884765625, "loss": 1.1809, "margin_dpo/margin_mean": 68.1265869140625, "margin_dpo/margin_std": 125.74884033203125, "step": 181 }, { "KL/chosen_KL_mean": -84.55804443359375, "KL/mean": -144.46820068359375, "KL/rejected_KL_mean": -204.37832641601562, "KL/std": 119.23335266113281, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.004303830675780773, "fcm_dpo/delta": -0.12271112948656082, "fcm_dpo/margin": 119.82029724121094, "fcm_dpo/q_t": 0.3822292983531952, "grad_norm": 19.109874725341797, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.2857114374637604, "logits/rejected": -0.29627925157546997, "logps/chosen": -125.6658935546875, "logps/ref_chosen": -41.107852935791016, "logps/ref_rejected": -89.5215835571289, "logps/rejected": -293.89990234375, "loss": 1.0243, "margin_dpo/margin_mean": 119.82029724121094, "margin_dpo/margin_std": 143.70375061035156, "step": 182 }, { "KL/chosen_KL_mean": -124.24063110351562, "KL/mean": -155.76622009277344, "KL/rejected_KL_mean": -187.29180908203125, "KL/std": 99.46566772460938, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.004267096519470215, "fcm_dpo/delta": -0.029845165088772774, "fcm_dpo/margin": 63.05116653442383, "fcm_dpo/q_t": 0.4370817542076111, "grad_norm": 19.169570922851562, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -0.2434646189212799, "logits/rejected": -0.2298406958580017, "logps/chosen": -181.76519775390625, "logps/ref_chosen": -57.52456283569336, "logps/ref_rejected": -75.97572326660156, "logps/rejected": -263.26751708984375, "loss": 1.1949, "margin_dpo/margin_mean": 63.05116271972656, "margin_dpo/margin_std": 108.97421264648438, "step": 183 }, { "KL/chosen_KL_mean": -104.53349304199219, "KL/mean": -138.28854370117188, "KL/rejected_KL_mean": -172.0435791015625, "KL/std": 93.82521057128906, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.004267096519470215, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.51007080078125, "fcm_dpo/q_t": 0.4340338706970215, "grad_norm": 17.413782119750977, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -0.31900352239608765, "logits/rejected": -0.31066498160362244, "logps/chosen": -163.07846069335938, "logps/ref_chosen": -58.544952392578125, "logps/ref_rejected": -76.63406372070312, "logps/rejected": -248.67764282226562, "loss": 1.1807, "margin_dpo/margin_mean": 67.51007080078125, "margin_dpo/margin_std": 118.04556274414062, "step": 184 }, { "KL/chosen_KL_mean": -112.33355712890625, "KL/mean": -138.57923889160156, "KL/rejected_KL_mean": -164.82492065429688, "KL/std": 107.27217864990234, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.004267096519470215, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.49135208129883, "fcm_dpo/q_t": 0.45065242052078247, "grad_norm": 18.927852630615234, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -0.28930944204330444, "logits/rejected": -0.2664622366428375, "logps/chosen": -174.35940551757812, "logps/ref_chosen": -62.025848388671875, "logps/ref_rejected": -73.7625961303711, "logps/rejected": -238.58750915527344, "loss": 1.2459, "margin_dpo/margin_mean": 52.49134826660156, "margin_dpo/margin_std": 130.1099853515625, "step": 185 }, { "KL/chosen_KL_mean": -102.53080749511719, "KL/mean": -152.23342895507812, "KL/rejected_KL_mean": -201.93603515625, "KL/std": 106.55513000488281, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.004267096519470215, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.40524291992188, "fcm_dpo/q_t": 0.4030308723449707, "grad_norm": 27.395605087280273, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -0.3761294484138489, "logits/rejected": -0.36072081327438354, "logps/chosen": -171.88427734375, "logps/ref_chosen": -69.35346984863281, "logps/ref_rejected": -88.07244873046875, "logps/rejected": -290.00848388671875, "loss": 1.0696, "margin_dpo/margin_mean": 99.40524291992188, "margin_dpo/margin_std": 122.63768005371094, "step": 186 }, { "KL/chosen_KL_mean": -94.88285827636719, "KL/mean": -137.2830047607422, "KL/rejected_KL_mean": -179.6831512451172, "KL/std": 102.76565551757812, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.004259498789906502, "fcm_dpo/delta": -0.01783652789890766, "fcm_dpo/margin": 84.80029296875, "fcm_dpo/q_t": 0.4160510301589966, "grad_norm": 23.19233512878418, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -0.3276948034763336, "logits/rejected": -0.32193610072135925, "logps/chosen": -147.63931274414062, "logps/ref_chosen": -52.7564582824707, "logps/ref_rejected": -81.96910095214844, "logps/rejected": -261.6522521972656, "loss": 1.1113, "margin_dpo/margin_mean": 84.80029296875, "margin_dpo/margin_std": 111.11736297607422, "step": 187 }, { "KL/chosen_KL_mean": -88.96598815917969, "KL/mean": -139.68350219726562, "KL/rejected_KL_mean": -190.40103149414062, "KL/std": 113.15321350097656, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.004251901526004076, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.43505096435547, "fcm_dpo/q_t": 0.40194350481033325, "grad_norm": 29.918031692504883, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -0.2987292408943176, "logits/rejected": -0.3055376410484314, "logps/chosen": -138.3814697265625, "logps/ref_chosen": -49.415489196777344, "logps/ref_rejected": -89.54043579101562, "logps/rejected": -279.94146728515625, "loss": 1.0642, "margin_dpo/margin_mean": 101.43505096435547, "margin_dpo/margin_std": 123.50503540039062, "step": 188 }, { "KL/chosen_KL_mean": -104.55441284179688, "KL/mean": -146.60751342773438, "KL/rejected_KL_mean": -188.66064453125, "KL/std": 115.1569595336914, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.004251901526004076, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.10621643066406, "fcm_dpo/q_t": 0.4205946922302246, "grad_norm": 22.207258224487305, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -0.32266396284103394, "logits/rejected": -0.30563491582870483, "logps/chosen": -156.953369140625, "logps/ref_chosen": -52.39896011352539, "logps/ref_rejected": -72.16735076904297, "logps/rejected": -260.8280029296875, "loss": 1.14, "margin_dpo/margin_mean": 84.10621643066406, "margin_dpo/margin_std": 141.17153930664062, "step": 189 }, { "KL/chosen_KL_mean": -111.84954833984375, "KL/mean": -161.93978881835938, "KL/rejected_KL_mean": -212.030029296875, "KL/std": 124.21896362304688, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.004212085623294115, "fcm_dpo/delta": -0.09453053772449493, "fcm_dpo/margin": 100.18048858642578, "fcm_dpo/q_t": 0.40519779920578003, "grad_norm": 17.787500381469727, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -0.3172937035560608, "logits/rejected": -0.30383723974227905, "logps/chosen": -176.53260803222656, "logps/ref_chosen": -64.68305969238281, "logps/ref_rejected": -102.55052185058594, "logps/rejected": -314.58056640625, "loss": 1.0978, "margin_dpo/margin_mean": 100.18048858642578, "margin_dpo/margin_std": 143.24652099609375, "step": 190 }, { "KL/chosen_KL_mean": -104.78083801269531, "KL/mean": -175.7665252685547, "KL/rejected_KL_mean": -246.75222778320312, "KL/std": 141.02308654785156, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.00404349435120821, "fcm_dpo/delta": -0.1856611967086792, "fcm_dpo/margin": 141.97137451171875, "fcm_dpo/q_t": 0.37194889783859253, "grad_norm": 18.88121795654297, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -0.35467493534088135, "logits/rejected": -0.3345186114311218, "logps/chosen": -173.43971252441406, "logps/ref_chosen": -68.65887451171875, "logps/ref_rejected": -110.1396713256836, "logps/rejected": -356.89190673828125, "loss": 0.979, "margin_dpo/margin_mean": 141.97137451171875, "margin_dpo/margin_std": 156.3507537841797, "step": 191 }, { "KL/chosen_KL_mean": -134.30404663085938, "KL/mean": -181.43359375, "KL/rejected_KL_mean": -228.56317138671875, "KL/std": 122.91511535644531, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.004020185209810734, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.25910949707031, "fcm_dpo/q_t": 0.41552644968032837, "grad_norm": 22.49018669128418, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -0.28340697288513184, "logits/rejected": -0.2827571630477905, "logps/chosen": -204.03097534179688, "logps/ref_chosen": -69.72691345214844, "logps/ref_rejected": -103.32135009765625, "logps/rejected": -331.884521484375, "loss": 1.1286, "margin_dpo/margin_mean": 94.25910949707031, "margin_dpo/margin_std": 152.60736083984375, "step": 192 }, { "KL/chosen_KL_mean": -134.03817749023438, "KL/mean": -161.839599609375, "KL/rejected_KL_mean": -189.6409912109375, "KL/std": 115.02464294433594, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.004020185209810734, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.60279083251953, "fcm_dpo/q_t": 0.4464014768600464, "grad_norm": 26.761646270751953, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -0.30809807777404785, "logits/rejected": -0.3014001250267029, "logps/chosen": -194.2286834716797, "logps/ref_chosen": -60.19049835205078, "logps/ref_rejected": -76.40755462646484, "logps/rejected": -266.0485534667969, "loss": 1.2585, "margin_dpo/margin_mean": 55.60279083251953, "margin_dpo/margin_std": 146.70774841308594, "step": 193 }, { "KL/chosen_KL_mean": -83.75235748291016, "KL/mean": -128.62440490722656, "KL/rejected_KL_mean": -173.4964599609375, "KL/std": 95.19931030273438, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.004020185209810734, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.74411010742188, "fcm_dpo/q_t": 0.4147985279560089, "grad_norm": 15.454571723937988, "learning_rate": 4.510405240853854e-07, "logits/chosen": -0.20827534794807434, "logits/rejected": -0.19151608645915985, "logps/chosen": -121.59272766113281, "logps/ref_chosen": -37.84037399291992, "logps/ref_rejected": -60.684783935546875, "logps/rejected": -234.18124389648438, "loss": 1.1011, "margin_dpo/margin_mean": 89.74411010742188, "margin_dpo/margin_std": 105.07684326171875, "step": 194 }, { "KL/chosen_KL_mean": -132.5264129638672, "KL/mean": -180.77197265625, "KL/rejected_KL_mean": -229.01751708984375, "KL/std": 118.66985321044922, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.004020185209810734, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.49112701416016, "fcm_dpo/q_t": 0.4101010859012604, "grad_norm": 24.827335357666016, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -0.22600211203098297, "logits/rejected": -0.24551793932914734, "logps/chosen": -187.41798400878906, "logps/ref_chosen": -54.891571044921875, "logps/ref_rejected": -96.77095794677734, "logps/rejected": -325.7884826660156, "loss": 1.0917, "margin_dpo/margin_mean": 96.49112701416016, "margin_dpo/margin_std": 122.10552978515625, "step": 195 }, { "KL/chosen_KL_mean": -101.63624572753906, "KL/mean": -156.90122985839844, "KL/rejected_KL_mean": -212.1662139892578, "KL/std": 120.26493835449219, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.003964636474847794, "fcm_dpo/delta": -0.06956931948661804, "fcm_dpo/margin": 110.52996063232422, "fcm_dpo/q_t": 0.4009883403778076, "grad_norm": 16.279319763183594, "learning_rate": 4.495043068200599e-07, "logits/chosen": -0.29676854610443115, "logits/rejected": -0.2831350266933441, "logps/chosen": -154.88150024414062, "logps/ref_chosen": -53.245243072509766, "logps/ref_rejected": -76.05294799804688, "logps/rejected": -288.21917724609375, "loss": 1.0747, "margin_dpo/margin_mean": 110.52996063232422, "margin_dpo/margin_std": 143.91986083984375, "step": 196 }, { "KL/chosen_KL_mean": -104.97657012939453, "KL/mean": -147.35794067382812, "KL/rejected_KL_mean": -189.73931884765625, "KL/std": 104.29344177246094, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.003964636474847794, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.76274871826172, "fcm_dpo/q_t": 0.4220603108406067, "grad_norm": 18.666196823120117, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -0.26582375168800354, "logits/rejected": -0.26124078035354614, "logps/chosen": -165.39691162109375, "logps/ref_chosen": -60.42033386230469, "logps/ref_rejected": -77.20890808105469, "logps/rejected": -266.9482421875, "loss": 1.13, "margin_dpo/margin_mean": 84.76274871826172, "margin_dpo/margin_std": 117.65675354003906, "step": 197 }, { "KL/chosen_KL_mean": -118.09703063964844, "KL/mean": -172.27029418945312, "KL/rejected_KL_mean": -226.4435577392578, "KL/std": 130.2771759033203, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.003964636474847794, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.34652709960938, "fcm_dpo/q_t": 0.4042630195617676, "grad_norm": 20.83625030517578, "learning_rate": 4.479470611971645e-07, "logits/chosen": -0.2953726649284363, "logits/rejected": -0.29592087864875793, "logps/chosen": -173.13320922851562, "logps/ref_chosen": -55.03618621826172, "logps/ref_rejected": -97.24325561523438, "logps/rejected": -323.68682861328125, "loss": 1.0773, "margin_dpo/margin_mean": 108.34652709960938, "margin_dpo/margin_std": 146.12347412109375, "step": 198 }, { "KL/chosen_KL_mean": -111.96086120605469, "KL/mean": -165.84420776367188, "KL/rejected_KL_mean": -219.72755432128906, "KL/std": 117.84519958496094, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.00391243165358901, "fcm_dpo/delta": -0.06627540290355682, "fcm_dpo/margin": 107.7667007446289, "fcm_dpo/q_t": 0.4036487638950348, "grad_norm": 21.15707015991211, "learning_rate": 4.471606039587695e-07, "logits/chosen": -0.3135220408439636, "logits/rejected": -0.2968115210533142, "logps/chosen": -168.78968811035156, "logps/ref_chosen": -56.828826904296875, "logps/ref_rejected": -84.64820861816406, "logps/rejected": -304.3757629394531, "loss": 1.0843, "margin_dpo/margin_mean": 107.7667007446289, "margin_dpo/margin_std": 144.26821899414062, "step": 199 }, { "KL/chosen_KL_mean": -109.51300048828125, "KL/mean": -163.57774353027344, "KL/rejected_KL_mean": -217.64247131347656, "KL/std": 127.20454406738281, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.0038815850857645273, "fcm_dpo/delta": -0.0395774245262146, "fcm_dpo/margin": 108.12948608398438, "fcm_dpo/q_t": 0.4061018228530884, "grad_norm": 18.211580276489258, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -0.2737102806568146, "logits/rejected": -0.25955069065093994, "logps/chosen": -162.58006286621094, "logps/ref_chosen": -53.06706237792969, "logps/ref_rejected": -80.60843658447266, "logps/rejected": -298.25091552734375, "loss": 1.1009, "margin_dpo/margin_mean": 108.12948608398438, "margin_dpo/margin_std": 162.80935668945312, "step": 200 }, { "KL/chosen_KL_mean": -115.50804138183594, "KL/mean": -169.43679809570312, "KL/rejected_KL_mean": -223.36557006835938, "KL/std": 135.0660400390625, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.00386931118555367, "fcm_dpo/delta": -0.03172111138701439, "fcm_dpo/margin": 107.85752868652344, "fcm_dpo/q_t": 0.40528595447540283, "grad_norm": 18.747127532958984, "learning_rate": 4.455721242469372e-07, "logits/chosen": -0.3500000834465027, "logits/rejected": -0.34820225834846497, "logps/chosen": -190.91026306152344, "logps/ref_chosen": -75.4022216796875, "logps/ref_rejected": -114.80821990966797, "logps/rejected": -338.17376708984375, "loss": 1.0919, "margin_dpo/margin_mean": 107.85752868652344, "margin_dpo/margin_std": 151.97970581054688, "step": 201 }, { "KL/chosen_KL_mean": -120.22370910644531, "KL/mean": -157.83718872070312, "KL/rejected_KL_mean": -195.4506378173828, "KL/std": 117.38187408447266, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.0038570372853428125, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.22691345214844, "fcm_dpo/q_t": 0.43460530042648315, "grad_norm": 19.376781463623047, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -0.28714412450790405, "logits/rejected": -0.3028670847415924, "logps/chosen": -170.3250274658203, "logps/ref_chosen": -50.101318359375, "logps/ref_rejected": -86.98503112792969, "logps/rejected": -282.4356689453125, "loss": 1.1981, "margin_dpo/margin_mean": 75.22691345214844, "margin_dpo/margin_std": 152.60952758789062, "step": 202 }, { "KL/chosen_KL_mean": -118.58921813964844, "KL/mean": -165.52542114257812, "KL/rejected_KL_mean": -212.46160888671875, "KL/std": 120.00934600830078, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.0038570372853428125, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.87237548828125, "fcm_dpo/q_t": 0.4165031909942627, "grad_norm": 20.891557693481445, "learning_rate": 4.439630306414758e-07, "logits/chosen": -0.3353193402290344, "logits/rejected": -0.3288806676864624, "logps/chosen": -179.19891357421875, "logps/ref_chosen": -60.60969543457031, "logps/ref_rejected": -85.89596557617188, "logps/rejected": -298.3575744628906, "loss": 1.117, "margin_dpo/margin_mean": 93.87239074707031, "margin_dpo/margin_std": 132.0635528564453, "step": 203 }, { "KL/chosen_KL_mean": -132.33493041992188, "KL/mean": -175.73989868164062, "KL/rejected_KL_mean": -219.1448974609375, "KL/std": 130.3503875732422, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.0038570372853428125, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.80996704101562, "fcm_dpo/q_t": 0.4250984191894531, "grad_norm": 22.802963256835938, "learning_rate": 4.431508065452897e-07, "logits/chosen": -0.4051492214202881, "logits/rejected": -0.37047427892684937, "logps/chosen": -212.4998779296875, "logps/ref_chosen": -80.16496276855469, "logps/ref_rejected": -87.69590759277344, "logps/rejected": -306.8408203125, "loss": 1.1606, "margin_dpo/margin_mean": 86.80996704101562, "margin_dpo/margin_std": 152.51388549804688, "step": 204 }, { "KL/chosen_KL_mean": -129.84996032714844, "KL/mean": -187.99693298339844, "KL/rejected_KL_mean": -246.14390563964844, "KL/std": 135.32766723632812, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.00378048256970942, "fcm_dpo/delta": -0.10023847967386246, "fcm_dpo/margin": 116.29393005371094, "fcm_dpo/q_t": 0.39798709750175476, "grad_norm": 19.018651962280273, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -0.2791271209716797, "logits/rejected": -0.24926945567131042, "logps/chosen": -189.2346954345703, "logps/ref_chosen": -59.384735107421875, "logps/ref_rejected": -85.12505340576172, "logps/rejected": -331.2689514160156, "loss": 1.0719, "margin_dpo/margin_mean": 116.29393005371094, "margin_dpo/margin_std": 147.2044219970703, "step": 205 }, { "KL/chosen_KL_mean": -118.81938171386719, "KL/mean": -181.1719512939453, "KL/rejected_KL_mean": -243.52452087402344, "KL/std": 125.76171875, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.0037328554317355156, "fcm_dpo/delta": -0.06339111924171448, "fcm_dpo/margin": 124.70513153076172, "fcm_dpo/q_t": 0.39114850759506226, "grad_norm": 23.02773666381836, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.23025387525558472, "logits/rejected": -0.233394593000412, "logps/chosen": -165.78387451171875, "logps/ref_chosen": -46.964500427246094, "logps/ref_rejected": -98.9534912109375, "logps/rejected": -342.47802734375, "loss": 1.0355, "margin_dpo/margin_mean": 124.70513153076172, "margin_dpo/margin_std": 136.64340209960938, "step": 206 }, { "KL/chosen_KL_mean": -110.89537048339844, "KL/mean": -183.31405639648438, "KL/rejected_KL_mean": -255.7327117919922, "KL/std": 147.05844116210938, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.003661469090729952, "fcm_dpo/delta": -0.13748954236507416, "fcm_dpo/margin": 144.83734130859375, "fcm_dpo/q_t": 0.3823302388191223, "grad_norm": 22.23126983642578, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -0.33647215366363525, "logits/rejected": -0.3128640651702881, "logps/chosen": -166.95162963867188, "logps/ref_chosen": -56.05625915527344, "logps/ref_rejected": -84.44779968261719, "logps/rejected": -340.1805114746094, "loss": 1.0102, "margin_dpo/margin_mean": 144.83734130859375, "margin_dpo/margin_std": 171.68792724609375, "step": 207 }, { "KL/chosen_KL_mean": -171.9359130859375, "KL/mean": -227.58004760742188, "KL/rejected_KL_mean": -283.22418212890625, "KL/std": 141.04026794433594, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.0036316080950200558, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.28825378417969, "fcm_dpo/q_t": 0.4080774188041687, "grad_norm": 24.357837677001953, "learning_rate": 4.398512291636768e-07, "logits/chosen": -0.314586877822876, "logits/rejected": -0.2982891798019409, "logps/chosen": -239.00352478027344, "logps/ref_chosen": -67.06761169433594, "logps/ref_rejected": -94.28689575195312, "logps/rejected": -377.51104736328125, "loss": 1.1139, "margin_dpo/margin_mean": 111.28825378417969, "margin_dpo/margin_std": 172.75132751464844, "step": 208 }, { "KL/chosen_KL_mean": -144.75634765625, "KL/mean": -193.95156860351562, "KL/rejected_KL_mean": -243.14678955078125, "KL/std": 125.75456237792969, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.0036316080950200558, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.39041137695312, "fcm_dpo/q_t": 0.4181509017944336, "grad_norm": 24.6387996673584, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -0.21858316659927368, "logits/rejected": -0.20688988268375397, "logps/chosen": -200.93804931640625, "logps/ref_chosen": -56.18169403076172, "logps/ref_rejected": -80.94152069091797, "logps/rejected": -324.08831787109375, "loss": 1.1421, "margin_dpo/margin_mean": 98.39041900634766, "margin_dpo/margin_std": 159.32110595703125, "step": 209 }, { "KL/chosen_KL_mean": -128.8101806640625, "KL/mean": -186.57366943359375, "KL/rejected_KL_mean": -244.337158203125, "KL/std": 130.371337890625, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.003612897824496031, "fcm_dpo/delta": -0.05178801342844963, "fcm_dpo/margin": 115.52698516845703, "fcm_dpo/q_t": 0.40541771054267883, "grad_norm": 23.523893356323242, "learning_rate": 4.381713366536311e-07, "logits/chosen": -0.2616923749446869, "logits/rejected": -0.2551494240760803, "logps/chosen": -175.18199157714844, "logps/ref_chosen": -46.371822357177734, "logps/ref_rejected": -76.68162536621094, "logps/rejected": -321.018798828125, "loss": 1.089, "margin_dpo/margin_mean": 115.52698516845703, "margin_dpo/margin_std": 158.4886474609375, "step": 210 }, { "KL/chosen_KL_mean": -182.32998657226562, "KL/mean": -231.79617309570312, "KL/rejected_KL_mean": -281.26239013671875, "KL/std": 148.31312561035156, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.003594187553972006, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.93236541748047, "fcm_dpo/q_t": 0.422549843788147, "grad_norm": 32.76312255859375, "learning_rate": 4.373239415645323e-07, "logits/chosen": -0.2987380921840668, "logits/rejected": -0.261538565158844, "logps/chosen": -261.2623291015625, "logps/ref_chosen": -78.93235778808594, "logps/ref_rejected": -86.82098388671875, "logps/rejected": -368.0833740234375, "loss": 1.1547, "margin_dpo/margin_mean": 98.93236541748047, "margin_dpo/margin_std": 174.6763916015625, "step": 211 }, { "KL/chosen_KL_mean": -151.54200744628906, "KL/mean": -221.18905639648438, "KL/rejected_KL_mean": -290.8360900878906, "KL/std": 158.43215942382812, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.0035146158188581467, "fcm_dpo/delta": -0.11193866282701492, "fcm_dpo/margin": 139.29408264160156, "fcm_dpo/q_t": 0.3895455300807953, "grad_norm": 22.986331939697266, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -0.29403156042099, "logits/rejected": -0.2868325114250183, "logps/chosen": -209.739013671875, "logps/ref_chosen": -58.19701385498047, "logps/ref_rejected": -103.05785369873047, "logps/rejected": -393.8939208984375, "loss": 1.0528, "margin_dpo/margin_mean": 139.29409790039062, "margin_dpo/margin_std": 178.13589477539062, "step": 212 }, { "KL/chosen_KL_mean": -139.25265502929688, "KL/mean": -205.28900146484375, "KL/rejected_KL_mean": -271.3253479003906, "KL/std": 135.00584411621094, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.0034705563448369503, "fcm_dpo/delta": -0.0630766823887825, "fcm_dpo/margin": 132.0726776123047, "fcm_dpo/q_t": 0.3947675824165344, "grad_norm": 29.329484939575195, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -0.3089921474456787, "logits/rejected": -0.28634005784988403, "logps/chosen": -206.765380859375, "logps/ref_chosen": -67.51271057128906, "logps/ref_rejected": -93.91471862792969, "logps/rejected": -365.24005126953125, "loss": 1.0522, "margin_dpo/margin_mean": 132.0726776123047, "margin_dpo/margin_std": 160.37890625, "step": 213 }, { "KL/chosen_KL_mean": -118.66336059570312, "KL/mean": -177.6033935546875, "KL/rejected_KL_mean": -236.54342651367188, "KL/std": 133.26171875, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.0034705563448369503, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.88006591796875, "fcm_dpo/q_t": 0.4071076512336731, "grad_norm": 22.763870239257812, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -0.2438046634197235, "logits/rejected": -0.24939894676208496, "logps/chosen": -160.26824951171875, "logps/ref_chosen": -41.604888916015625, "logps/ref_rejected": -77.51741027832031, "logps/rejected": -314.06085205078125, "loss": 1.0871, "margin_dpo/margin_mean": 117.88006591796875, "margin_dpo/margin_std": 156.95803833007812, "step": 214 }, { "KL/chosen_KL_mean": -137.9038543701172, "KL/mean": -200.85073852539062, "KL/rejected_KL_mean": -263.797607421875, "KL/std": 138.6534423828125, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.003440548898652196, "fcm_dpo/delta": -0.04341936483979225, "fcm_dpo/margin": 125.89375305175781, "fcm_dpo/q_t": 0.3995819091796875, "grad_norm": 24.996137619018555, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -0.2863304018974304, "logits/rejected": -0.2791479825973511, "logps/chosen": -191.18312072753906, "logps/ref_chosen": -53.279266357421875, "logps/ref_rejected": -89.96464538574219, "logps/rejected": -353.76226806640625, "loss": 1.0566, "margin_dpo/margin_mean": 125.89375305175781, "margin_dpo/margin_std": 142.8882293701172, "step": 215 }, { "KL/chosen_KL_mean": -142.95309448242188, "KL/mean": -204.64840698242188, "KL/rejected_KL_mean": -266.3437194824219, "KL/std": 143.56790161132812, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.0034245119895786047, "fcm_dpo/delta": -0.0468301959335804, "fcm_dpo/margin": 123.39063262939453, "fcm_dpo/q_t": 0.404532790184021, "grad_norm": 28.86602210998535, "learning_rate": 4.330133748510036e-07, "logits/chosen": -0.26124703884124756, "logits/rejected": -0.24592363834381104, "logps/chosen": -191.84088134765625, "logps/ref_chosen": -48.887794494628906, "logps/ref_rejected": -77.19892883300781, "logps/rejected": -343.54266357421875, "loss": 1.0941, "margin_dpo/margin_mean": 123.390625, "margin_dpo/margin_std": 177.62725830078125, "step": 216 }, { "KL/chosen_KL_mean": -147.24102783203125, "KL/mean": -219.417236328125, "KL/rejected_KL_mean": -291.59344482421875, "KL/std": 148.2445068359375, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.003357392270117998, "fcm_dpo/delta": -0.08919186890125275, "fcm_dpo/margin": 144.35240173339844, "fcm_dpo/q_t": 0.3901098668575287, "grad_norm": 20.184051513671875, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -0.24297907948493958, "logits/rejected": -0.2382565438747406, "logps/chosen": -197.08633422851562, "logps/ref_chosen": -49.845306396484375, "logps/ref_rejected": -100.07832336425781, "logps/rejected": -391.6717529296875, "loss": 1.0309, "margin_dpo/margin_mean": 144.35240173339844, "margin_dpo/margin_std": 167.79852294921875, "step": 217 }, { "KL/chosen_KL_mean": -154.8172607421875, "KL/mean": -210.88455200195312, "KL/rejected_KL_mean": -266.95184326171875, "KL/std": 147.3013458251953, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.0033482126891613007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.13459777832031, "fcm_dpo/q_t": 0.41538113355636597, "grad_norm": 20.418493270874023, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -0.2989402413368225, "logits/rejected": -0.2960107922554016, "logps/chosen": -213.39393615722656, "logps/ref_chosen": -58.576683044433594, "logps/ref_rejected": -87.84639739990234, "logps/rejected": -354.7982482910156, "loss": 1.1249, "margin_dpo/margin_mean": 112.13459777832031, "margin_dpo/margin_std": 173.35887145996094, "step": 218 }, { "KL/chosen_KL_mean": -163.69097900390625, "KL/mean": -214.0511474609375, "KL/rejected_KL_mean": -264.4113464355469, "KL/std": 152.53524780273438, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.0033482126891613007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.72035217285156, "fcm_dpo/q_t": 0.4240277409553528, "grad_norm": 25.4362850189209, "learning_rate": 4.303689819449636e-07, "logits/chosen": -0.28101325035095215, "logits/rejected": -0.2753964066505432, "logps/chosen": -224.77484130859375, "logps/ref_chosen": -61.083858489990234, "logps/ref_rejected": -85.83042907714844, "logps/rejected": -350.24176025390625, "loss": 1.1707, "margin_dpo/margin_mean": 100.7203598022461, "margin_dpo/margin_std": 188.10244750976562, "step": 219 }, { "KL/chosen_KL_mean": -189.14373779296875, "KL/mean": -229.32162475585938, "KL/rejected_KL_mean": -269.49951171875, "KL/std": 133.23536682128906, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.0033482126891613007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.35578918457031, "fcm_dpo/q_t": 0.43612149357795715, "grad_norm": 23.351408004760742, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -0.276694655418396, "logits/rejected": -0.25455957651138306, "logps/chosen": -259.175048828125, "logps/ref_chosen": -70.03128051757812, "logps/ref_rejected": -87.68551635742188, "logps/rejected": -357.18505859375, "loss": 1.1856, "margin_dpo/margin_mean": 80.35578918457031, "margin_dpo/margin_std": 136.25027465820312, "step": 220 }, { "KL/chosen_KL_mean": -161.2969970703125, "KL/mean": -251.148193359375, "KL/rejected_KL_mean": -340.9993896484375, "KL/std": 167.5245819091797, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.003260483266785741, "fcm_dpo/delta": -0.19711089134216309, "fcm_dpo/margin": 179.70236206054688, "fcm_dpo/q_t": 0.3655005693435669, "grad_norm": 21.982431411743164, "learning_rate": 4.285822501755485e-07, "logits/chosen": -0.26910918951034546, "logits/rejected": -0.27597469091415405, "logps/chosen": -213.45172119140625, "logps/ref_chosen": -52.15470886230469, "logps/ref_rejected": -106.46768188476562, "logps/rejected": -447.4670715332031, "loss": 0.9583, "margin_dpo/margin_mean": 179.70236206054688, "margin_dpo/margin_std": 173.05661010742188, "step": 221 }, { "KL/chosen_KL_mean": -167.68533325195312, "KL/mean": -235.232177734375, "KL/rejected_KL_mean": -302.77899169921875, "KL/std": 153.12326049804688, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.003201256040483713, "fcm_dpo/delta": -0.05476263165473938, "fcm_dpo/margin": 135.09365844726562, "fcm_dpo/q_t": 0.4009937047958374, "grad_norm": 19.333269119262695, "learning_rate": 4.276818137766118e-07, "logits/chosen": -0.32539641857147217, "logits/rejected": -0.33047088980674744, "logps/chosen": -228.65643310546875, "logps/ref_chosen": -60.971099853515625, "logps/ref_rejected": -100.00115203857422, "logps/rejected": -402.7801513671875, "loss": 1.0713, "margin_dpo/margin_mean": 135.09365844726562, "margin_dpo/margin_std": 172.55303955078125, "step": 222 }, { "KL/chosen_KL_mean": -176.12669372558594, "KL/mean": -233.26063537597656, "KL/rejected_KL_mean": -290.39459228515625, "KL/std": 149.42013549804688, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.0031837252900004387, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.26789855957031, "fcm_dpo/q_t": 0.41713905334472656, "grad_norm": 22.279924392700195, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -0.24122165143489838, "logits/rejected": -0.23779387772083282, "logps/chosen": -228.76727294921875, "logps/ref_chosen": -52.64057540893555, "logps/ref_rejected": -82.82502746582031, "logps/rejected": -373.2196044921875, "loss": 1.1392, "margin_dpo/margin_mean": 114.26790618896484, "margin_dpo/margin_std": 188.8822021484375, "step": 223 }, { "KL/chosen_KL_mean": -155.34686279296875, "KL/mean": -219.0513153076172, "KL/rejected_KL_mean": -282.75579833984375, "KL/std": 164.88922119140625, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.0031389761716127396, "fcm_dpo/delta": -0.07077645510435104, "fcm_dpo/margin": 127.40892028808594, "fcm_dpo/q_t": 0.4104171395301819, "grad_norm": 22.986743927001953, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -0.28872960805892944, "logits/rejected": -0.271827757358551, "logps/chosen": -203.9422607421875, "logps/ref_chosen": -48.59541320800781, "logps/ref_rejected": -77.11648559570312, "logps/rejected": -359.87225341796875, "loss": 1.1095, "margin_dpo/margin_mean": 127.40892028808594, "margin_dpo/margin_std": 189.6688690185547, "step": 224 }, { "KL/chosen_KL_mean": -172.8418731689453, "KL/mean": -249.9188232421875, "KL/rejected_KL_mean": -326.9957580566406, "KL/std": 159.6756591796875, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.003098821034654975, "fcm_dpo/delta": -0.0816635936498642, "fcm_dpo/margin": 154.15386962890625, "fcm_dpo/q_t": 0.3921276330947876, "grad_norm": 20.74527931213379, "learning_rate": 4.249525076191759e-07, "logits/chosen": -0.2806037664413452, "logits/rejected": -0.2695569694042206, "logps/chosen": -230.84234619140625, "logps/ref_chosen": -58.000465393066406, "logps/ref_rejected": -99.90291595458984, "logps/rejected": -426.898681640625, "loss": 1.0479, "margin_dpo/margin_mean": 154.1538848876953, "margin_dpo/margin_std": 196.17489624023438, "step": 225 }, { "KL/chosen_KL_mean": -144.2257843017578, "KL/mean": -204.41741943359375, "KL/rejected_KL_mean": -264.6090393066406, "KL/std": 153.60317993164062, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.0030607180669903755, "fcm_dpo/delta": -0.0445721298456192, "fcm_dpo/margin": 120.38326263427734, "fcm_dpo/q_t": 0.4166339933872223, "grad_norm": 27.664154052734375, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -0.36745959520339966, "logits/rejected": -0.33029118180274963, "logps/chosen": -203.1245880126953, "logps/ref_chosen": -58.898799896240234, "logps/ref_rejected": -78.68775939941406, "logps/rejected": -343.29681396484375, "loss": 1.1233, "margin_dpo/margin_mean": 120.38326263427734, "margin_dpo/margin_std": 179.03895568847656, "step": 226 }, { "KL/chosen_KL_mean": -170.3883514404297, "KL/mean": -246.78062438964844, "KL/rejected_KL_mean": -323.1728820800781, "KL/std": 175.88458251953125, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.0030199983157217503, "fcm_dpo/delta": -0.06696625053882599, "fcm_dpo/margin": 152.78453063964844, "fcm_dpo/q_t": 0.39447590708732605, "grad_norm": 20.940410614013672, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -0.37393718957901, "logits/rejected": -0.36207836866378784, "logps/chosen": -229.46054077148438, "logps/ref_chosen": -59.072181701660156, "logps/ref_rejected": -99.41236877441406, "logps/rejected": -422.58526611328125, "loss": 1.0474, "margin_dpo/margin_mean": 152.78453063964844, "margin_dpo/margin_std": 183.7152099609375, "step": 227 }, { "KL/chosen_KL_mean": -176.73321533203125, "KL/mean": -230.6518096923828, "KL/rejected_KL_mean": -284.57037353515625, "KL/std": 145.71096801757812, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.0030199983157217503, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.83721160888672, "fcm_dpo/q_t": 0.4245913624763489, "grad_norm": 22.438199996948242, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -0.33213791251182556, "logits/rejected": -0.3147745132446289, "logps/chosen": -242.62449645996094, "logps/ref_chosen": -65.89128875732422, "logps/ref_rejected": -91.04875183105469, "logps/rejected": -375.619140625, "loss": 1.1566, "margin_dpo/margin_mean": 107.83721160888672, "margin_dpo/margin_std": 179.61813354492188, "step": 228 }, { "KL/chosen_KL_mean": -183.93844604492188, "KL/mean": -240.5111083984375, "KL/rejected_KL_mean": -297.08380126953125, "KL/std": 168.53048706054688, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.0030199983157217503, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.14534759521484, "fcm_dpo/q_t": 0.4204818904399872, "grad_norm": 26.519044876098633, "learning_rate": 4.212490049118951e-07, "logits/chosen": -0.4333471655845642, "logits/rejected": -0.40338772535324097, "logps/chosen": -254.6448211669922, "logps/ref_chosen": -70.70637512207031, "logps/ref_rejected": -84.52741241455078, "logps/rejected": -381.6112060546875, "loss": 1.1392, "margin_dpo/margin_mean": 113.14534759521484, "margin_dpo/margin_std": 175.4351348876953, "step": 229 }, { "KL/chosen_KL_mean": -138.13897705078125, "KL/mean": -226.32997131347656, "KL/rejected_KL_mean": -314.52099609375, "KL/std": 157.35964965820312, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.0029599140398204327, "fcm_dpo/delta": -0.12893977761268616, "fcm_dpo/margin": 176.38198852539062, "fcm_dpo/q_t": 0.3783706724643707, "grad_norm": 25.281761169433594, "learning_rate": 4.203117865141635e-07, "logits/chosen": -0.3073171377182007, "logits/rejected": -0.31276828050613403, "logps/chosen": -177.42098999023438, "logps/ref_chosen": -39.282005310058594, "logps/ref_rejected": -85.62191009521484, "logps/rejected": -400.14288330078125, "loss": 0.9923, "margin_dpo/margin_mean": 176.38198852539062, "margin_dpo/margin_std": 174.53048706054688, "step": 230 }, { "KL/chosen_KL_mean": -159.91831970214844, "KL/mean": -220.16099548339844, "KL/rejected_KL_mean": -280.4036865234375, "KL/std": 140.28118896484375, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.0029431143775582314, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.48533630371094, "fcm_dpo/q_t": 0.4189344346523285, "grad_norm": 22.254228591918945, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -0.390174925327301, "logits/rejected": -0.3631623685359955, "logps/chosen": -223.19476318359375, "logps/ref_chosen": -63.27644348144531, "logps/ref_rejected": -74.1239013671875, "logps/rejected": -354.527587890625, "loss": 1.1191, "margin_dpo/margin_mean": 120.48533630371094, "margin_dpo/margin_std": 167.16416931152344, "step": 231 }, { "KL/chosen_KL_mean": -196.7489776611328, "KL/mean": -244.6427001953125, "KL/rejected_KL_mean": -292.536376953125, "KL/std": 162.15066528320312, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.0029431143775582314, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.78742980957031, "fcm_dpo/q_t": 0.43585118651390076, "grad_norm": 21.261966705322266, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -0.38710182905197144, "logits/rejected": -0.3654683232307434, "logps/chosen": -267.49774169921875, "logps/ref_chosen": -70.74876403808594, "logps/ref_rejected": -83.97706604003906, "logps/rejected": -376.5134582519531, "loss": 1.1816, "margin_dpo/margin_mean": 95.78742218017578, "margin_dpo/margin_std": 168.59783935546875, "step": 232 }, { "KL/chosen_KL_mean": -180.105224609375, "KL/mean": -257.1478576660156, "KL/rejected_KL_mean": -334.1905212402344, "KL/std": 175.86370849609375, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.0029314416460692883, "fcm_dpo/delta": -0.03981967642903328, "fcm_dpo/margin": 154.08529663085938, "fcm_dpo/q_t": 0.40144720673561096, "grad_norm": 25.009721755981445, "learning_rate": 4.174733034541245e-07, "logits/chosen": -0.3826596736907959, "logits/rejected": -0.38623514771461487, "logps/chosen": -234.9881591796875, "logps/ref_chosen": -54.8829345703125, "logps/ref_rejected": -107.4800796508789, "logps/rejected": -441.67059326171875, "loss": 1.0906, "margin_dpo/margin_mean": 154.08529663085938, "margin_dpo/margin_std": 233.36492919921875, "step": 233 }, { "KL/chosen_KL_mean": -169.5860595703125, "KL/mean": -250.97674560546875, "KL/rejected_KL_mean": -332.367431640625, "KL/std": 157.31170654296875, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.0028566650580614805, "fcm_dpo/delta": -0.10924752801656723, "fcm_dpo/margin": 162.78138732910156, "fcm_dpo/q_t": 0.39263916015625, "grad_norm": 25.143383026123047, "learning_rate": 4.165182829193126e-07, "logits/chosen": -0.3675558865070343, "logits/rejected": -0.3938511610031128, "logps/chosen": -213.68057250976562, "logps/ref_chosen": -44.094520568847656, "logps/ref_rejected": -100.00663757324219, "logps/rejected": -432.37408447265625, "loss": 1.0386, "margin_dpo/margin_mean": 162.7813720703125, "margin_dpo/margin_std": 178.6966552734375, "step": 234 }, { "KL/chosen_KL_mean": -198.6753387451172, "KL/mean": -249.81863403320312, "KL/rejected_KL_mean": -300.9619140625, "KL/std": 150.10362243652344, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.0028566650580614805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.28659057617188, "fcm_dpo/q_t": 0.43217870593070984, "grad_norm": 22.54652214050293, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -0.41138529777526855, "logits/rejected": -0.3917854428291321, "logps/chosen": -260.91326904296875, "logps/ref_chosen": -62.237911224365234, "logps/ref_rejected": -90.39506530761719, "logps/rejected": -391.35699462890625, "loss": 1.1836, "margin_dpo/margin_mean": 102.28659057617188, "margin_dpo/margin_std": 185.93734741210938, "step": 235 }, { "KL/chosen_KL_mean": -141.9903106689453, "KL/mean": -226.0331573486328, "KL/rejected_KL_mean": -310.0760192871094, "KL/std": 158.168701171875, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.002825359348207712, "fcm_dpo/delta": -0.07859396189451218, "fcm_dpo/margin": 168.08572387695312, "fcm_dpo/q_t": 0.3882787823677063, "grad_norm": 43.635643005371094, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -0.36709725856781006, "logits/rejected": -0.36788949370384216, "logps/chosen": -191.3316650390625, "logps/ref_chosen": -49.34136199951172, "logps/ref_rejected": -103.51162719726562, "logps/rejected": -413.587646484375, "loss": 1.0116, "margin_dpo/margin_mean": 168.08570861816406, "margin_dpo/margin_std": 155.02297973632812, "step": 236 }, { "KL/chosen_KL_mean": -197.8418426513672, "KL/mean": -259.5992431640625, "KL/rejected_KL_mean": -321.35662841796875, "KL/std": 161.23373413085938, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.002812112681567669, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.5147705078125, "fcm_dpo/q_t": 0.4200955033302307, "grad_norm": 25.65804672241211, "learning_rate": 4.136269950853473e-07, "logits/chosen": -0.39958909153938293, "logits/rejected": -0.3944680690765381, "logps/chosen": -252.0099639892578, "logps/ref_chosen": -54.168121337890625, "logps/ref_rejected": -94.78036499023438, "logps/rejected": -416.136962890625, "loss": 1.1343, "margin_dpo/margin_mean": 123.5147705078125, "margin_dpo/margin_std": 188.49781799316406, "step": 237 }, { "KL/chosen_KL_mean": -180.33523559570312, "KL/mean": -243.35289001464844, "KL/rejected_KL_mean": -306.37054443359375, "KL/std": 161.61019897460938, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.002812112681567669, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.03531646728516, "fcm_dpo/q_t": 0.42012059688568115, "grad_norm": 20.620817184448242, "learning_rate": 4.126545750510605e-07, "logits/chosen": -0.3614025413990021, "logits/rejected": -0.37703627347946167, "logps/chosen": -234.308349609375, "logps/ref_chosen": -53.973121643066406, "logps/ref_rejected": -89.41795349121094, "logps/rejected": -395.7884826660156, "loss": 1.1248, "margin_dpo/margin_mean": 126.03531646728516, "margin_dpo/margin_std": 182.96377563476562, "step": 238 }, { "KL/chosen_KL_mean": -196.66397094726562, "KL/mean": -265.8099365234375, "KL/rejected_KL_mean": -334.95587158203125, "KL/std": 151.65145874023438, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.0027831587940454483, "fcm_dpo/delta": -0.0517473965883255, "fcm_dpo/margin": 138.29190063476562, "fcm_dpo/q_t": 0.4096493124961853, "grad_norm": 25.863664627075195, "learning_rate": 4.116778689174514e-07, "logits/chosen": -0.3546360731124878, "logits/rejected": -0.3427826464176178, "logps/chosen": -254.7617950439453, "logps/ref_chosen": -58.09782409667969, "logps/ref_rejected": -93.59294128417969, "logps/rejected": -428.548828125, "loss": 1.096, "margin_dpo/margin_mean": 138.29190063476562, "margin_dpo/margin_std": 172.67640686035156, "step": 239 }, { "KL/chosen_KL_mean": -210.21449279785156, "KL/mean": -268.498291015625, "KL/rejected_KL_mean": -326.7821044921875, "KL/std": 167.20452880859375, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.0027831587940454483, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.56759643554688, "fcm_dpo/q_t": 0.425168514251709, "grad_norm": 30.908700942993164, "learning_rate": 4.106969024216348e-07, "logits/chosen": -0.4022897183895111, "logits/rejected": -0.38126668334007263, "logps/chosen": -270.8289794921875, "logps/ref_chosen": -60.6144905090332, "logps/ref_rejected": -74.1185302734375, "logps/rejected": -400.900634765625, "loss": 1.1631, "margin_dpo/margin_mean": 116.56758880615234, "margin_dpo/margin_std": 200.62228393554688, "step": 240 }, { "KL/chosen_KL_mean": -172.1416778564453, "KL/mean": -263.7938537597656, "KL/rejected_KL_mean": -355.446044921875, "KL/std": 180.43846130371094, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.0027444493025541306, "fcm_dpo/delta": -0.10839249193668365, "fcm_dpo/margin": 183.3043670654297, "fcm_dpo/q_t": 0.3863786458969116, "grad_norm": 19.921833038330078, "learning_rate": 4.097117014129903e-07, "logits/chosen": -0.43084996938705444, "logits/rejected": -0.40479138493537903, "logps/chosen": -238.23275756835938, "logps/ref_chosen": -66.091064453125, "logps/ref_rejected": -88.06088256835938, "logps/rejected": -443.5069274902344, "loss": 1.0213, "margin_dpo/margin_mean": 183.3043670654297, "margin_dpo/margin_std": 207.9857635498047, "step": 241 }, { "KL/chosen_KL_mean": -193.84503173828125, "KL/mean": -259.82501220703125, "KL/rejected_KL_mean": -325.80499267578125, "KL/std": 155.22760009765625, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.00272347335703671, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.9599609375, "fcm_dpo/q_t": 0.41780009865760803, "grad_norm": 27.682950973510742, "learning_rate": 4.087222918524807e-07, "logits/chosen": -0.3546012043952942, "logits/rejected": -0.32627618312835693, "logps/chosen": -261.708984375, "logps/ref_chosen": -67.86392974853516, "logps/ref_rejected": -83.36033630371094, "logps/rejected": -409.16534423828125, "loss": 1.1181, "margin_dpo/margin_mean": 131.9599609375, "margin_dpo/margin_std": 183.73941040039062, "step": 242 }, { "KL/chosen_KL_mean": -194.37255859375, "KL/mean": -274.17266845703125, "KL/rejected_KL_mean": -353.9727783203125, "KL/std": 172.1506805419922, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.002707144943997264, "fcm_dpo/delta": -0.03006729856133461, "fcm_dpo/margin": 159.60023498535156, "fcm_dpo/q_t": 0.40053194761276245, "grad_norm": 21.327482223510742, "learning_rate": 4.07728699811968e-07, "logits/chosen": -0.39147722721099854, "logits/rejected": -0.3596557080745697, "logps/chosen": -257.456787109375, "logps/ref_chosen": -63.0842399597168, "logps/ref_rejected": -76.33563232421875, "logps/rejected": -430.30841064453125, "loss": 1.0628, "margin_dpo/margin_mean": 159.6002197265625, "margin_dpo/margin_std": 193.20114135742188, "step": 243 }, { "KL/chosen_KL_mean": -172.62156677246094, "KL/mean": -254.97146606445312, "KL/rejected_KL_mean": -337.3213806152344, "KL/std": 166.24649047851562, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.0026877904310822487, "fcm_dpo/delta": -0.035875409841537476, "fcm_dpo/margin": 164.69979858398438, "fcm_dpo/q_t": 0.39697980880737305, "grad_norm": 31.131752014160156, "learning_rate": 4.067309514735267e-07, "logits/chosen": -0.45547521114349365, "logits/rejected": -0.44788095355033875, "logps/chosen": -233.76226806640625, "logps/ref_chosen": -61.140689849853516, "logps/ref_rejected": -94.89193725585938, "logps/rejected": -432.21331787109375, "loss": 1.0421, "margin_dpo/margin_mean": 164.69979858398438, "margin_dpo/margin_std": 171.30508422851562, "step": 244 }, { "KL/chosen_KL_mean": -195.41578674316406, "KL/mean": -259.2579040527344, "KL/rejected_KL_mean": -323.1000061035156, "KL/std": 162.6224365234375, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.0026638784911483526, "fcm_dpo/delta": -0.04468155652284622, "fcm_dpo/margin": 127.68421936035156, "fcm_dpo/q_t": 0.4205586314201355, "grad_norm": 25.96481704711914, "learning_rate": 4.057290731287531e-07, "logits/chosen": -0.41297852993011475, "logits/rejected": -0.3774294853210449, "logps/chosen": -262.6780700683594, "logps/ref_chosen": -67.26228332519531, "logps/ref_rejected": -87.64010620117188, "logps/rejected": -410.7401123046875, "loss": 1.1345, "margin_dpo/margin_mean": 127.6842269897461, "margin_dpo/margin_std": 181.09228515625, "step": 245 }, { "KL/chosen_KL_mean": -190.72445678710938, "KL/mean": -256.1783447265625, "KL/rejected_KL_mean": -321.6322326660156, "KL/std": 178.58486938476562, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.0026638784911483526, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.90777587890625, "fcm_dpo/q_t": 0.42047402262687683, "grad_norm": 23.14719581604004, "learning_rate": 4.047230911780736e-07, "logits/chosen": -0.413362979888916, "logits/rejected": -0.3701326251029968, "logps/chosen": -257.4214172363281, "logps/ref_chosen": -66.69696807861328, "logps/ref_rejected": -84.34634399414062, "logps/rejected": -405.97857666015625, "loss": 1.1292, "margin_dpo/margin_mean": 130.9077911376953, "margin_dpo/margin_std": 194.133056640625, "step": 246 }, { "KL/chosen_KL_mean": -225.75881958007812, "KL/mean": -318.2025146484375, "KL/rejected_KL_mean": -410.646240234375, "KL/std": 188.978759765625, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.0026259967125952244, "fcm_dpo/delta": -0.0899805873632431, "fcm_dpo/margin": 184.88742065429688, "fcm_dpo/q_t": 0.38787591457366943, "grad_norm": 25.931663513183594, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -0.38560110330581665, "logits/rejected": -0.38438546657562256, "logps/chosen": -282.3641662597656, "logps/ref_chosen": -56.6053466796875, "logps/ref_rejected": -106.29326629638672, "logps/rejected": -516.9395141601562, "loss": 1.027, "margin_dpo/margin_mean": 184.88742065429688, "margin_dpo/margin_std": 205.1929931640625, "step": 247 }, { "KL/chosen_KL_mean": -197.12680053710938, "KL/mean": -277.889892578125, "KL/rejected_KL_mean": -358.6529541015625, "KL/std": 154.3875732421875, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.002593907294794917, "fcm_dpo/delta": -0.043108776211738586, "fcm_dpo/margin": 161.52613830566406, "fcm_dpo/q_t": 0.40013599395751953, "grad_norm": 23.251401901245117, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -0.3690452575683594, "logits/rejected": -0.3877558708190918, "logps/chosen": -241.17001342773438, "logps/ref_chosen": -44.043216705322266, "logps/ref_rejected": -91.85687255859375, "logps/rejected": -450.50982666015625, "loss": 1.0481, "margin_dpo/margin_mean": 161.52615356445312, "margin_dpo/margin_std": 151.2039337158203, "step": 248 }, { "KL/chosen_KL_mean": -256.06048583984375, "KL/mean": -301.0152587890625, "KL/rejected_KL_mean": -345.97003173828125, "KL/std": 172.55154418945312, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.002593907294794917, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.90953063964844, "fcm_dpo/q_t": 0.4462352693080902, "grad_norm": 38.79088592529297, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -0.36389392614364624, "logits/rejected": -0.3354836106300354, "logps/chosen": -318.5028381347656, "logps/ref_chosen": -62.442352294921875, "logps/ref_rejected": -80.46806335449219, "logps/rejected": -426.4381103515625, "loss": 1.2342, "margin_dpo/margin_mean": 89.90953063964844, "margin_dpo/margin_std": 204.7283935546875, "step": 249 }, { "KL/chosen_KL_mean": -224.89846801757812, "KL/mean": -307.3691711425781, "KL/rejected_KL_mean": -389.83990478515625, "KL/std": 172.57041931152344, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.002579119522124529, "fcm_dpo/delta": -0.05733676999807358, "fcm_dpo/margin": 164.94143676757812, "fcm_dpo/q_t": 0.4000697433948517, "grad_norm": 38.31913375854492, "learning_rate": 4.006586590948141e-07, "logits/chosen": -0.4156474471092224, "logits/rejected": -0.36385685205459595, "logps/chosen": -290.53515625, "logps/ref_chosen": -65.63668823242188, "logps/ref_rejected": -73.87184143066406, "logps/rejected": -463.71173095703125, "loss": 1.0555, "margin_dpo/margin_mean": 164.94143676757812, "margin_dpo/margin_std": 175.64585876464844, "step": 250 }, { "KL/chosen_KL_mean": -236.84722900390625, "KL/mean": -293.100830078125, "KL/rejected_KL_mean": -349.3543701171875, "KL/std": 183.2646026611328, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.0025643319822847843, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.50713348388672, "fcm_dpo/q_t": 0.4335523843765259, "grad_norm": 30.411144256591797, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -0.3748970031738281, "logits/rejected": -0.3472249507904053, "logps/chosen": -294.02996826171875, "logps/ref_chosen": -57.182716369628906, "logps/ref_rejected": -77.66343688964844, "logps/rejected": -427.017822265625, "loss": 1.1849, "margin_dpo/margin_mean": 112.50714111328125, "margin_dpo/margin_std": 199.13619995117188, "step": 251 }, { "KL/chosen_KL_mean": -229.87887573242188, "KL/mean": -305.1651916503906, "KL/rejected_KL_mean": -380.4515075683594, "KL/std": 155.89443969726562, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.0025595955085009336, "fcm_dpo/delta": -0.009243616834282875, "fcm_dpo/margin": 150.5726318359375, "fcm_dpo/q_t": 0.40730518102645874, "grad_norm": 21.492137908935547, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -0.4269337058067322, "logits/rejected": -0.38683801889419556, "logps/chosen": -301.56451416015625, "logps/ref_chosen": -71.68563842773438, "logps/ref_rejected": -84.75799560546875, "logps/rejected": -465.2095031738281, "loss": 1.0784, "margin_dpo/margin_mean": 150.5726318359375, "margin_dpo/margin_std": 159.85842895507812, "step": 252 }, { "KL/chosen_KL_mean": -193.25228881835938, "KL/mean": -267.5342102050781, "KL/rejected_KL_mean": -341.81610107421875, "KL/std": 163.1719970703125, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.00254888366907835, "fcm_dpo/delta": -0.04202554374933243, "fcm_dpo/margin": 148.56381225585938, "fcm_dpo/q_t": 0.4106384813785553, "grad_norm": 20.642892837524414, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -0.42303064465522766, "logits/rejected": -0.4157634973526001, "logps/chosen": -262.38623046875, "logps/ref_chosen": -69.1339340209961, "logps/ref_rejected": -98.70252990722656, "logps/rejected": -440.5186462402344, "loss": 1.0971, "margin_dpo/margin_mean": 148.56381225585938, "margin_dpo/margin_std": 182.68304443359375, "step": 253 }, { "KL/chosen_KL_mean": -186.4593048095703, "KL/mean": -248.51510620117188, "KL/rejected_KL_mean": -310.5709228515625, "KL/std": 170.81390380859375, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.00253817206248641, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.11161041259766, "fcm_dpo/q_t": 0.4287106394767761, "grad_norm": 20.37973403930664, "learning_rate": 3.965307091713037e-07, "logits/chosen": -0.40077197551727295, "logits/rejected": -0.38768458366394043, "logps/chosen": -240.61428833007812, "logps/ref_chosen": -54.154998779296875, "logps/ref_rejected": -90.30764770507812, "logps/rejected": -400.8785400390625, "loss": 1.165, "margin_dpo/margin_mean": 124.11161041259766, "margin_dpo/margin_std": 215.81329345703125, "step": 254 }, { "KL/chosen_KL_mean": -191.34371948242188, "KL/mean": -258.4233093261719, "KL/rejected_KL_mean": -325.5028991699219, "KL/std": 152.7626953125, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.00253817206248641, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.15916442871094, "fcm_dpo/q_t": 0.42020100355148315, "grad_norm": 18.49299430847168, "learning_rate": 3.954890003969163e-07, "logits/chosen": -0.33083224296569824, "logits/rejected": -0.3193257451057434, "logps/chosen": -248.4853973388672, "logps/ref_chosen": -57.14167022705078, "logps/ref_rejected": -90.2085952758789, "logps/rejected": -415.71148681640625, "loss": 1.1331, "margin_dpo/margin_mean": 134.1591796875, "margin_dpo/margin_std": 192.447265625, "step": 255 }, { "KL/chosen_KL_mean": -165.99559020996094, "KL/mean": -238.594482421875, "KL/rejected_KL_mean": -311.193359375, "KL/std": 164.0802459716797, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.00253817206248641, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.19778442382812, "fcm_dpo/q_t": 0.41385918855667114, "grad_norm": 21.11256217956543, "learning_rate": 3.944434578520628e-07, "logits/chosen": -0.2933967709541321, "logits/rejected": -0.30158305168151855, "logps/chosen": -221.15908813476562, "logps/ref_chosen": -55.163490295410156, "logps/ref_rejected": -92.56291961669922, "logps/rejected": -403.75628662109375, "loss": 1.106, "margin_dpo/margin_mean": 145.19778442382812, "margin_dpo/margin_std": 189.60250854492188, "step": 256 }, { "KL/chosen_KL_mean": -161.5221405029297, "KL/mean": -238.52484130859375, "KL/rejected_KL_mean": -315.5274963378906, "KL/std": 170.44703674316406, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.002513015875592828, "fcm_dpo/delta": -0.04980289936065674, "fcm_dpo/margin": 154.00537109375, "fcm_dpo/q_t": 0.41046983003616333, "grad_norm": 19.582237243652344, "learning_rate": 3.933941090877615e-07, "logits/chosen": -0.33271628618240356, "logits/rejected": -0.3201237618923187, "logps/chosen": -210.94583129882812, "logps/ref_chosen": -49.42369842529297, "logps/ref_rejected": -79.53791809082031, "logps/rejected": -395.0654296875, "loss": 1.0972, "margin_dpo/margin_mean": 154.00535583496094, "margin_dpo/margin_std": 195.84552001953125, "step": 257 }, { "KL/chosen_KL_mean": -220.34742736816406, "KL/mean": -294.8319091796875, "KL/rejected_KL_mean": -369.3164367675781, "KL/std": 181.22947692871094, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.002513015875592828, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.96900939941406, "fcm_dpo/q_t": 0.41232049465179443, "grad_norm": 20.54438591003418, "learning_rate": 3.923409817553284e-07, "logits/chosen": -0.30997389554977417, "logits/rejected": -0.310375452041626, "logps/chosen": -279.7315673828125, "logps/ref_chosen": -59.384124755859375, "logps/ref_rejected": -95.99010467529297, "logps/rejected": -465.3065490722656, "loss": 1.118, "margin_dpo/margin_mean": 148.96900939941406, "margin_dpo/margin_std": 218.79287719726562, "step": 258 }, { "KL/chosen_KL_mean": -211.32296752929688, "KL/mean": -275.07977294921875, "KL/rejected_KL_mean": -338.8366394042969, "KL/std": 175.24095153808594, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.002513015875592828, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.51367950439453, "fcm_dpo/q_t": 0.4257126450538635, "grad_norm": 21.773229598999023, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -0.3975661098957062, "logits/rejected": -0.39768803119659424, "logps/chosen": -264.15130615234375, "logps/ref_chosen": -52.828346252441406, "logps/ref_rejected": -89.191650390625, "logps/rejected": -428.0282897949219, "loss": 1.1505, "margin_dpo/margin_mean": 127.51367950439453, "margin_dpo/margin_std": 197.5504150390625, "step": 259 }, { "KL/chosen_KL_mean": -218.90866088867188, "KL/mean": -306.67657470703125, "KL/rejected_KL_mean": -394.44451904296875, "KL/std": 183.72035217285156, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.002498188056051731, "fcm_dpo/delta": -0.05935557931661606, "fcm_dpo/margin": 175.53582763671875, "fcm_dpo/q_t": 0.3994418978691101, "grad_norm": 25.106760025024414, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -0.32701295614242554, "logits/rejected": -0.3442246615886688, "logps/chosen": -266.3263244628906, "logps/ref_chosen": -47.41767501831055, "logps/ref_rejected": -95.08978271484375, "logps/rejected": -489.5343017578125, "loss": 1.0499, "margin_dpo/margin_mean": 175.5358123779297, "margin_dpo/margin_std": 196.42477416992188, "step": 260 }, { "KL/chosen_KL_mean": -242.05313110351562, "KL/mean": -322.48565673828125, "KL/rejected_KL_mean": -402.918212890625, "KL/std": 203.7381591796875, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.0024833600036799908, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 160.8650665283203, "fcm_dpo/q_t": 0.40943995118141174, "grad_norm": 20.951894760131836, "learning_rate": 3.891592063515376e-07, "logits/chosen": -0.2890021800994873, "logits/rejected": -0.2918785512447357, "logps/chosen": -295.08453369140625, "logps/ref_chosen": -53.03137969970703, "logps/ref_rejected": -88.51494598388672, "logps/rejected": -491.43316650390625, "loss": 1.1019, "margin_dpo/margin_mean": 160.8650665283203, "margin_dpo/margin_std": 231.27334594726562, "step": 261 }, { "KL/chosen_KL_mean": -284.0287170410156, "KL/mean": -350.4783935546875, "KL/rejected_KL_mean": -416.92803955078125, "KL/std": 187.28726196289062, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.0024833600036799908, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.89935302734375, "fcm_dpo/q_t": 0.42301496863365173, "grad_norm": 28.10672950744629, "learning_rate": 3.880912432401264e-07, "logits/chosen": -0.3489972949028015, "logits/rejected": -0.3238402009010315, "logps/chosen": -343.64886474609375, "logps/ref_chosen": -59.620140075683594, "logps/ref_rejected": -86.41853332519531, "logps/rejected": -503.3465881347656, "loss": 1.1355, "margin_dpo/margin_mean": 132.89935302734375, "margin_dpo/margin_std": 188.70492553710938, "step": 262 }, { "KL/chosen_KL_mean": -261.15643310546875, "KL/mean": -355.87921142578125, "KL/rejected_KL_mean": -450.60198974609375, "KL/std": 221.2732696533203, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.0024432563222944736, "fcm_dpo/delta": -0.0814036875963211, "fcm_dpo/margin": 189.44557189941406, "fcm_dpo/q_t": 0.39440131187438965, "grad_norm": 25.79364585876465, "learning_rate": 3.870196412960302e-07, "logits/chosen": -0.34519362449645996, "logits/rejected": -0.3233689069747925, "logps/chosen": -320.5773620605469, "logps/ref_chosen": -59.42094421386719, "logps/ref_rejected": -96.85720825195312, "logps/rejected": -547.4591674804688, "loss": 1.0557, "margin_dpo/margin_mean": 189.445556640625, "margin_dpo/margin_std": 238.8328857421875, "step": 263 }, { "KL/chosen_KL_mean": -260.68182373046875, "KL/mean": -341.7437744140625, "KL/rejected_KL_mean": -422.80572509765625, "KL/std": 199.98663330078125, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.0024124952033162117, "fcm_dpo/delta": -0.06335047632455826, "fcm_dpo/margin": 162.12388610839844, "fcm_dpo/q_t": 0.41067957878112793, "grad_norm": 21.558048248291016, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -0.3743303418159485, "logits/rejected": -0.36552029848098755, "logps/chosen": -323.4039306640625, "logps/ref_chosen": -62.722084045410156, "logps/ref_rejected": -93.85620880126953, "logps/rejected": -516.6619262695312, "loss": 1.1084, "margin_dpo/margin_mean": 162.12387084960938, "margin_dpo/margin_std": 230.56280517578125, "step": 264 }, { "KL/chosen_KL_mean": -281.0964660644531, "KL/mean": -358.7066650390625, "KL/rejected_KL_mean": -436.31683349609375, "KL/std": 221.1083984375, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0024124952033162117, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 155.22036743164062, "fcm_dpo/q_t": 0.41601699590682983, "grad_norm": 27.02475357055664, "learning_rate": 3.848656339557562e-07, "logits/chosen": -0.3154297471046448, "logits/rejected": -0.2991935908794403, "logps/chosen": -343.06793212890625, "logps/ref_chosen": -61.971466064453125, "logps/ref_rejected": -88.02059936523438, "logps/rejected": -524.33740234375, "loss": 1.1413, "margin_dpo/margin_mean": 155.2203826904297, "margin_dpo/margin_std": 266.7834167480469, "step": 265 }, { "KL/chosen_KL_mean": -279.3717041015625, "KL/mean": -336.39202880859375, "KL/rejected_KL_mean": -393.41229248046875, "KL/std": 181.82192993164062, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0024124952033162117, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.04059600830078, "fcm_dpo/q_t": 0.43650779128074646, "grad_norm": 46.004493713378906, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -0.37979966402053833, "logits/rejected": -0.34439635276794434, "logps/chosen": -346.47137451171875, "logps/ref_chosen": -67.09967041015625, "logps/ref_rejected": -67.97122192382812, "logps/rejected": -461.383544921875, "loss": 1.1933, "margin_dpo/margin_mean": 114.04059600830078, "margin_dpo/margin_std": 214.42086791992188, "step": 266 }, { "KL/chosen_KL_mean": -240.74594116210938, "KL/mean": -311.76678466796875, "KL/rejected_KL_mean": -382.7876892089844, "KL/std": 195.85140991210938, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.0024124952033162117, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 142.041748046875, "fcm_dpo/q_t": 0.42108699679374695, "grad_norm": 30.3566837310791, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -0.3702971339225769, "logits/rejected": -0.34150320291519165, "logps/chosen": -309.7166748046875, "logps/ref_chosen": -68.97075653076172, "logps/ref_rejected": -90.16844940185547, "logps/rejected": -472.95611572265625, "loss": 1.1432, "margin_dpo/margin_mean": 142.041748046875, "margin_dpo/margin_std": 227.41090393066406, "step": 267 }, { "KL/chosen_KL_mean": -239.63525390625, "KL/mean": -315.9992980957031, "KL/rejected_KL_mean": -392.36334228515625, "KL/std": 190.10870361328125, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0024124952033162117, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.72805786132812, "fcm_dpo/q_t": 0.41565245389938354, "grad_norm": 22.044809341430664, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -0.3633720874786377, "logits/rejected": -0.36941152811050415, "logps/chosen": -295.53558349609375, "logps/ref_chosen": -55.90031051635742, "logps/ref_rejected": -101.64763641357422, "logps/rejected": -494.010986328125, "loss": 1.1219, "margin_dpo/margin_mean": 152.72805786132812, "margin_dpo/margin_std": 229.2215576171875, "step": 268 }, { "KL/chosen_KL_mean": -255.48077392578125, "KL/mean": -346.63116455078125, "KL/rejected_KL_mean": -437.78155517578125, "KL/std": 193.64755249023438, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.00240440689958632, "fcm_dpo/delta": -0.033639829605817795, "fcm_dpo/margin": 182.30081176757812, "fcm_dpo/q_t": 0.40017130970954895, "grad_norm": 20.53118896484375, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -0.40448078513145447, "logits/rejected": -0.38746941089630127, "logps/chosen": -325.52032470703125, "logps/ref_chosen": -70.03955841064453, "logps/ref_rejected": -107.34937286376953, "logps/rejected": -545.1309814453125, "loss": 1.0795, "margin_dpo/margin_mean": 182.30081176757812, "margin_dpo/margin_std": 250.51087951660156, "step": 269 }, { "KL/chosen_KL_mean": -224.14105224609375, "KL/mean": -290.2872314453125, "KL/rejected_KL_mean": -356.43341064453125, "KL/std": 168.980224609375, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.002396318595856428, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.29232788085938, "fcm_dpo/q_t": 0.4260002374649048, "grad_norm": 24.173391342163086, "learning_rate": 3.794189242333106e-07, "logits/chosen": -0.4147574007511139, "logits/rejected": -0.4084188640117645, "logps/chosen": -293.6745300292969, "logps/ref_chosen": -69.53347778320312, "logps/ref_rejected": -109.92864990234375, "logps/rejected": -466.362060546875, "loss": 1.1591, "margin_dpo/margin_mean": 132.29232788085938, "margin_dpo/margin_std": 217.41592407226562, "step": 270 }, { "KL/chosen_KL_mean": -206.2494659423828, "KL/mean": -287.3037109375, "KL/rejected_KL_mean": -368.3579406738281, "KL/std": 166.89303588867188, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.002387921093031764, "fcm_dpo/delta": -0.01755235157907009, "fcm_dpo/margin": 162.10848999023438, "fcm_dpo/q_t": 0.4096933901309967, "grad_norm": 18.710674285888672, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -0.3881264925003052, "logits/rejected": -0.3716167211532593, "logps/chosen": -263.0140380859375, "logps/ref_chosen": -56.76456832885742, "logps/ref_rejected": -92.51383972167969, "logps/rejected": -460.87176513671875, "loss": 1.0844, "margin_dpo/margin_mean": 162.10848999023438, "margin_dpo/margin_std": 189.31494140625, "step": 271 }, { "KL/chosen_KL_mean": -187.37637329101562, "KL/mean": -287.665283203125, "KL/rejected_KL_mean": -387.9542541503906, "KL/std": 180.75425720214844, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.0023701349273324013, "fcm_dpo/delta": -0.07504396140575409, "fcm_dpo/margin": 200.577880859375, "fcm_dpo/q_t": 0.3880541920661926, "grad_norm": 31.674179077148438, "learning_rate": 3.772161666010912e-07, "logits/chosen": -0.33635494112968445, "logits/rejected": -0.3487662672996521, "logps/chosen": -236.87351989746094, "logps/ref_chosen": -49.497154235839844, "logps/ref_rejected": -105.54279327392578, "logps/rejected": -493.4970397949219, "loss": 1.0159, "margin_dpo/margin_mean": 200.577880859375, "margin_dpo/margin_std": 190.57537841796875, "step": 272 }, { "KL/chosen_KL_mean": -226.82720947265625, "KL/mean": -325.7548828125, "KL/rejected_KL_mean": -424.68255615234375, "KL/std": 184.29931640625, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.002323293825611472, "fcm_dpo/delta": -0.06214258447289467, "fcm_dpo/margin": 197.85537719726562, "fcm_dpo/q_t": 0.3923366069793701, "grad_norm": 25.173011779785156, "learning_rate": 3.761097448550755e-07, "logits/chosen": -0.3470395803451538, "logits/rejected": -0.33116915822029114, "logps/chosen": -289.8026123046875, "logps/ref_chosen": -62.97539520263672, "logps/ref_rejected": -92.49858093261719, "logps/rejected": -517.18115234375, "loss": 1.0317, "margin_dpo/margin_mean": 197.85537719726562, "margin_dpo/margin_std": 203.93701171875, "step": 273 }, { "KL/chosen_KL_mean": -281.4830017089844, "KL/mean": -354.01580810546875, "KL/rejected_KL_mean": -426.54864501953125, "KL/std": 169.19039916992188, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.002323293825611472, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.06565856933594, "fcm_dpo/q_t": 0.4209598898887634, "grad_norm": 30.572553634643555, "learning_rate": 3.75e-07, "logits/chosen": -0.2957637906074524, "logits/rejected": -0.281542032957077, "logps/chosen": -337.15069580078125, "logps/ref_chosen": -55.66770935058594, "logps/ref_rejected": -77.33308410644531, "logps/rejected": -503.8817138671875, "loss": 1.1268, "margin_dpo/margin_mean": 145.06565856933594, "margin_dpo/margin_std": 194.85675048828125, "step": 274 }, { "KL/chosen_KL_mean": -227.81744384765625, "KL/mean": -306.25030517578125, "KL/rejected_KL_mean": -384.68316650390625, "KL/std": 173.9310302734375, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.002323293825611472, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.86573791503906, "fcm_dpo/q_t": 0.4142495393753052, "grad_norm": 31.266094207763672, "learning_rate": 3.738869612786737e-07, "logits/chosen": -0.3424875736236572, "logits/rejected": -0.350277841091156, "logps/chosen": -276.4121398925781, "logps/ref_chosen": -48.594703674316406, "logps/ref_rejected": -93.30369567871094, "logps/rejected": -477.98687744140625, "loss": 1.1034, "margin_dpo/margin_mean": 156.86572265625, "margin_dpo/margin_std": 193.51849365234375, "step": 275 }, { "KL/chosen_KL_mean": -254.9441680908203, "KL/mean": -334.92266845703125, "KL/rejected_KL_mean": -414.9011535644531, "KL/std": 187.92633056640625, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.002323293825611472, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 159.95697021484375, "fcm_dpo/q_t": 0.41468003392219543, "grad_norm": 23.98878288269043, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -0.3111796975135803, "logits/rejected": -0.2913016676902771, "logps/chosen": -311.5215759277344, "logps/ref_chosen": -56.57740783691406, "logps/ref_rejected": -70.36566925048828, "logps/rejected": -485.2668151855469, "loss": 1.1146, "margin_dpo/margin_mean": 159.95697021484375, "margin_dpo/margin_std": 226.91453552246094, "step": 276 }, { "KL/chosen_KL_mean": -278.7911071777344, "KL/mean": -364.60546875, "KL/rejected_KL_mean": -450.41986083984375, "KL/std": 194.7156524658203, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.0023189065977931023, "fcm_dpo/delta": -0.018919235095381737, "fcm_dpo/margin": 171.62875366210938, "fcm_dpo/q_t": 0.4078858494758606, "grad_norm": 28.692832946777344, "learning_rate": 3.71651119641714e-07, "logits/chosen": -0.3414786756038666, "logits/rejected": -0.3268578350543976, "logps/chosen": -335.06268310546875, "logps/ref_chosen": -56.27156066894531, "logps/ref_rejected": -92.88127136230469, "logps/rejected": -543.3011474609375, "loss": 1.0971, "margin_dpo/margin_mean": 171.62875366210938, "margin_dpo/margin_std": 233.7840576171875, "step": 277 }, { "KL/chosen_KL_mean": -250.5606689453125, "KL/mean": -350.6166076660156, "KL/rejected_KL_mean": -450.6725158691406, "KL/std": 205.93980407714844, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.0022912982385605574, "fcm_dpo/delta": -0.05041739344596863, "fcm_dpo/margin": 200.11187744140625, "fcm_dpo/q_t": 0.3950769305229187, "grad_norm": 23.307388305664062, "learning_rate": 3.705283756425872e-07, "logits/chosen": -0.33211779594421387, "logits/rejected": -0.3396395444869995, "logps/chosen": -303.50262451171875, "logps/ref_chosen": -52.94194030761719, "logps/ref_rejected": -91.25357818603516, "logps/rejected": -541.9260864257812, "loss": 1.0454, "margin_dpo/margin_mean": 200.11187744140625, "margin_dpo/margin_std": 233.86526489257812, "step": 278 }, { "KL/chosen_KL_mean": -291.84429931640625, "KL/mean": -390.2003479003906, "KL/rejected_KL_mean": -488.55645751953125, "KL/std": 218.87045288085938, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.002256458392366767, "fcm_dpo/delta": -0.076610267162323, "fcm_dpo/margin": 196.71217346191406, "fcm_dpo/q_t": 0.40036946535110474, "grad_norm": 24.625885009765625, "learning_rate": 3.6940245560867e-07, "logits/chosen": -0.2654384970664978, "logits/rejected": -0.2664095461368561, "logps/chosen": -340.485595703125, "logps/ref_chosen": -48.641319274902344, "logps/ref_rejected": -87.8514404296875, "logps/rejected": -576.4078979492188, "loss": 1.0794, "margin_dpo/margin_mean": 196.712158203125, "margin_dpo/margin_std": 274.7304992675781, "step": 279 }, { "KL/chosen_KL_mean": -287.09918212890625, "KL/mean": -388.5111083984375, "KL/rejected_KL_mean": -489.9229736328125, "KL/std": 199.4989013671875, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.0022379662841558456, "fcm_dpo/delta": -0.05646730586886406, "fcm_dpo/margin": 202.82379150390625, "fcm_dpo/q_t": 0.3937091827392578, "grad_norm": 25.541810989379883, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -0.29510489106178284, "logits/rejected": -0.29825708270072937, "logps/chosen": -345.89630126953125, "logps/ref_chosen": -58.797122955322266, "logps/ref_rejected": -98.61885070800781, "logps/rejected": -588.5418701171875, "loss": 1.0388, "margin_dpo/margin_mean": 202.82379150390625, "margin_dpo/margin_std": 217.46322631835938, "step": 280 }, { "KL/chosen_KL_mean": -250.59390258789062, "KL/mean": -339.3375244140625, "KL/rejected_KL_mean": -428.0810852050781, "KL/std": 193.9304656982422, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0022240430116653442, "fcm_dpo/delta": -0.031813472509384155, "fcm_dpo/margin": 177.48715209960938, "fcm_dpo/q_t": 0.40596914291381836, "grad_norm": 20.3207950592041, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -0.35418635606765747, "logits/rejected": -0.3321439325809479, "logps/chosen": -306.0824279785156, "logps/ref_chosen": -55.488521575927734, "logps/ref_rejected": -80.88258361816406, "logps/rejected": -508.96368408203125, "loss": 1.0843, "margin_dpo/margin_mean": 177.48715209960938, "margin_dpo/margin_std": 211.81997680664062, "step": 281 }, { "KL/chosen_KL_mean": -276.3832092285156, "KL/mean": -345.36126708984375, "KL/rejected_KL_mean": -414.33935546875, "KL/std": 201.35035705566406, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0022169677540659904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.95614624023438, "fcm_dpo/q_t": 0.43155795335769653, "grad_norm": 23.50833511352539, "learning_rate": 3.660059364023408e-07, "logits/chosen": -0.39513063430786133, "logits/rejected": -0.37628278136253357, "logps/chosen": -349.453369140625, "logps/ref_chosen": -73.07014465332031, "logps/ref_rejected": -95.35098266601562, "logps/rejected": -509.69036865234375, "loss": 1.1661, "margin_dpo/margin_mean": 137.95614624023438, "margin_dpo/margin_std": 238.62408447265625, "step": 282 }, { "KL/chosen_KL_mean": -274.4816589355469, "KL/mean": -375.57525634765625, "KL/rejected_KL_mean": -476.66888427734375, "KL/std": 229.27352905273438, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0022034652065485716, "fcm_dpo/delta": -0.06127964332699776, "fcm_dpo/margin": 202.1872100830078, "fcm_dpo/q_t": 0.39549362659454346, "grad_norm": 34.2918815612793, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -0.37188607454299927, "logits/rejected": -0.3768133819103241, "logps/chosen": -336.3800964355469, "logps/ref_chosen": -61.89844512939453, "logps/ref_rejected": -96.98655700683594, "logps/rejected": -573.6553955078125, "loss": 1.0463, "margin_dpo/margin_mean": 202.18719482421875, "margin_dpo/margin_std": 219.53720092773438, "step": 283 }, { "KL/chosen_KL_mean": -246.93585205078125, "KL/mean": -343.9705505371094, "KL/rejected_KL_mean": -441.0052490234375, "KL/std": 210.19834899902344, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.002178533934056759, "fcm_dpo/delta": -0.026161545887589455, "fcm_dpo/margin": 194.06942749023438, "fcm_dpo/q_t": 0.40190061926841736, "grad_norm": 26.171096801757812, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -0.40229523181915283, "logits/rejected": -0.38924217224121094, "logps/chosen": -305.37139892578125, "logps/ref_chosen": -58.4355354309082, "logps/ref_rejected": -93.46926879882812, "logps/rejected": -534.4744873046875, "loss": 1.0582, "margin_dpo/margin_mean": 194.06942749023438, "margin_dpo/margin_std": 215.73760986328125, "step": 284 }, { "KL/chosen_KL_mean": -277.75640869140625, "KL/mean": -361.48651123046875, "KL/rejected_KL_mean": -445.216552734375, "KL/std": 180.0686798095703, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.0021733520552515984, "fcm_dpo/delta": -0.023842982947826385, "fcm_dpo/margin": 167.46017456054688, "fcm_dpo/q_t": 0.41295209527015686, "grad_norm": 26.861412048339844, "learning_rate": 3.625819059005228e-07, "logits/chosen": -0.3674265146255493, "logits/rejected": -0.35316190123558044, "logps/chosen": -343.98858642578125, "logps/ref_chosen": -66.23219299316406, "logps/ref_rejected": -99.1268310546875, "logps/rejected": -544.3433837890625, "loss": 1.1025, "margin_dpo/margin_mean": 167.46017456054688, "margin_dpo/margin_std": 200.82656860351562, "step": 285 }, { "KL/chosen_KL_mean": -270.828125, "KL/mean": -364.634765625, "KL/rejected_KL_mean": -458.4414367675781, "KL/std": 212.66998291015625, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.0021647391840815544, "fcm_dpo/delta": -0.01585019752383232, "fcm_dpo/margin": 187.61329650878906, "fcm_dpo/q_t": 0.4055512249469757, "grad_norm": 26.513324737548828, "learning_rate": 3.614345889441346e-07, "logits/chosen": -0.36591431498527527, "logits/rejected": -0.3500216603279114, "logps/chosen": -343.77911376953125, "logps/ref_chosen": -72.95100402832031, "logps/ref_rejected": -88.58845520019531, "logps/rejected": -547.0299072265625, "loss": 1.0822, "margin_dpo/margin_mean": 187.61331176757812, "margin_dpo/margin_std": 235.45333862304688, "step": 286 }, { "KL/chosen_KL_mean": -253.10614013671875, "KL/mean": -328.2662048339844, "KL/rejected_KL_mean": -403.42626953125, "KL/std": 183.10687255859375, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.0021613079588860273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 150.32009887695312, "fcm_dpo/q_t": 0.42383337020874023, "grad_norm": 24.7573184967041, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -0.32295361161231995, "logits/rejected": -0.2973632216453552, "logps/chosen": -314.6473083496094, "logps/ref_chosen": -61.54115295410156, "logps/ref_rejected": -77.69607543945312, "logps/rejected": -481.1223449707031, "loss": 1.1348, "margin_dpo/margin_mean": 150.32009887695312, "margin_dpo/margin_std": 206.97796630859375, "step": 287 }, { "KL/chosen_KL_mean": -244.66409301757812, "KL/mean": -341.7845458984375, "KL/rejected_KL_mean": -438.90496826171875, "KL/std": 186.95571899414062, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.0021289377473294735, "fcm_dpo/delta": -0.07545191794633865, "fcm_dpo/margin": 194.24093627929688, "fcm_dpo/q_t": 0.4019187092781067, "grad_norm": 22.394235610961914, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -0.3601091504096985, "logits/rejected": -0.34544798731803894, "logps/chosen": -301.3253173828125, "logps/ref_chosen": -56.661224365234375, "logps/ref_rejected": -87.33570098876953, "logps/rejected": -526.24072265625, "loss": 1.0607, "margin_dpo/margin_mean": 194.2409210205078, "margin_dpo/margin_std": 198.42041015625, "step": 288 }, { "KL/chosen_KL_mean": -245.93588256835938, "KL/mean": -353.0520935058594, "KL/rejected_KL_mean": -460.1683349609375, "KL/std": 213.45416259765625, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.002117036608979106, "fcm_dpo/delta": -0.05621682479977608, "fcm_dpo/margin": 214.232421875, "fcm_dpo/q_t": 0.39465266466140747, "grad_norm": 25.998571395874023, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -0.31153666973114014, "logits/rejected": -0.3129374086856842, "logps/chosen": -291.166259765625, "logps/ref_chosen": -45.23039245605469, "logps/ref_rejected": -87.64266967773438, "logps/rejected": -547.81103515625, "loss": 1.0397, "margin_dpo/margin_mean": 214.232421875, "margin_dpo/margin_std": 230.45819091796875, "step": 289 }, { "KL/chosen_KL_mean": -280.966552734375, "KL/mean": -389.29156494140625, "KL/rejected_KL_mean": -497.6165771484375, "KL/std": 227.33908081054688, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.0020852871239185333, "fcm_dpo/delta": -0.047366589307785034, "fcm_dpo/margin": 216.65000915527344, "fcm_dpo/q_t": 0.3967745900154114, "grad_norm": 22.709102630615234, "learning_rate": 3.568162605525952e-07, "logits/chosen": -0.3108745515346527, "logits/rejected": -0.331379234790802, "logps/chosen": -336.43804931640625, "logps/ref_chosen": -55.47149658203125, "logps/ref_rejected": -116.70857238769531, "logps/rejected": -614.3251342773438, "loss": 1.0563, "margin_dpo/margin_mean": 216.64999389648438, "margin_dpo/margin_std": 267.9898376464844, "step": 290 }, { "KL/chosen_KL_mean": -246.94216918945312, "KL/mean": -346.29931640625, "KL/rejected_KL_mean": -445.6565246582031, "KL/std": 199.556884765625, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.002075555734336376, "fcm_dpo/delta": -0.04688554257154465, "fcm_dpo/margin": 198.71432495117188, "fcm_dpo/q_t": 0.4034580588340759, "grad_norm": 22.80071449279785, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -0.30826008319854736, "logits/rejected": -0.29707035422325134, "logps/chosen": -310.2025451660156, "logps/ref_chosen": -63.26036834716797, "logps/ref_rejected": -89.29708862304688, "logps/rejected": -534.95361328125, "loss": 1.0765, "margin_dpo/margin_mean": 198.71434020996094, "margin_dpo/margin_std": 241.03353881835938, "step": 291 }, { "KL/chosen_KL_mean": -286.8546142578125, "KL/mean": -396.198486328125, "KL/rejected_KL_mean": -505.5423583984375, "KL/std": 243.14376831054688, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.002054115291684866, "fcm_dpo/delta": -0.057004090398550034, "fcm_dpo/margin": 218.687744140625, "fcm_dpo/q_t": 0.3968570828437805, "grad_norm": 27.79972267150879, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -0.31336697936058044, "logits/rejected": -0.3037183880805969, "logps/chosen": -340.77313232421875, "logps/ref_chosen": -53.91852951049805, "logps/ref_rejected": -89.96138000488281, "logps/rejected": -595.5037841796875, "loss": 1.0678, "margin_dpo/margin_mean": 218.68775939941406, "margin_dpo/margin_std": 288.130126953125, "step": 292 }, { "KL/chosen_KL_mean": -290.5531311035156, "KL/mean": -378.1788330078125, "KL/rejected_KL_mean": -465.8045654296875, "KL/std": 239.843994140625, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.002042406238615513, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 175.25146484375, "fcm_dpo/q_t": 0.4187690317630768, "grad_norm": 38.308929443359375, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -0.2938636541366577, "logits/rejected": -0.28838860988616943, "logps/chosen": -350.92913818359375, "logps/ref_chosen": -60.376033782958984, "logps/ref_rejected": -77.85244750976562, "logps/rejected": -543.656982421875, "loss": 1.1302, "margin_dpo/margin_mean": 175.25146484375, "margin_dpo/margin_std": 270.41204833984375, "step": 293 }, { "KL/chosen_KL_mean": -270.2442932128906, "KL/mean": -364.52728271484375, "KL/rejected_KL_mean": -458.810302734375, "KL/std": 215.75311279296875, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.002042406238615513, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 188.5659942626953, "fcm_dpo/q_t": 0.4126633405685425, "grad_norm": 21.470027923583984, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -0.32128533720970154, "logits/rejected": -0.3278936743736267, "logps/chosen": -318.33184814453125, "logps/ref_chosen": -48.0875358581543, "logps/ref_rejected": -81.89698791503906, "logps/rejected": -540.707275390625, "loss": 1.1073, "margin_dpo/margin_mean": 188.5659942626953, "margin_dpo/margin_std": 268.078125, "step": 294 }, { "KL/chosen_KL_mean": -317.76177978515625, "KL/mean": -423.4981689453125, "KL/rejected_KL_mean": -529.2345581054688, "KL/std": 245.9403076171875, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.002035372657701373, "fcm_dpo/delta": -0.03455709293484688, "fcm_dpo/margin": 211.4727325439453, "fcm_dpo/q_t": 0.402416467666626, "grad_norm": 26.90194320678711, "learning_rate": 3.509802512179737e-07, "logits/chosen": -0.32361164689064026, "logits/rejected": -0.3285978138446808, "logps/chosen": -367.68646240234375, "logps/ref_chosen": -49.92467498779297, "logps/ref_rejected": -87.45632934570312, "logps/rejected": -616.6908569335938, "loss": 1.0834, "margin_dpo/margin_mean": 211.47274780273438, "margin_dpo/margin_std": 289.3609313964844, "step": 295 }, { "KL/chosen_KL_mean": -381.29339599609375, "KL/mean": -451.5999755859375, "KL/rejected_KL_mean": -521.9066162109375, "KL/std": 226.91986083984375, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.0020283390767872334, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 140.61317443847656, "fcm_dpo/q_t": 0.43434810638427734, "grad_norm": 37.26921844482422, "learning_rate": 3.498049431928577e-07, "logits/chosen": -0.3297673165798187, "logits/rejected": -0.31439995765686035, "logps/chosen": -446.7846374511719, "logps/ref_chosen": -65.49124145507812, "logps/ref_rejected": -93.08908081054688, "logps/rejected": -614.9956665039062, "loss": 1.205, "margin_dpo/margin_mean": 140.61317443847656, "margin_dpo/margin_std": 288.5725402832031, "step": 296 }, { "KL/chosen_KL_mean": -315.0804443359375, "KL/mean": -400.39654541015625, "KL/rejected_KL_mean": -485.7126159667969, "KL/std": 217.16357421875, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.0020283390767872334, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 170.63214111328125, "fcm_dpo/q_t": 0.41893017292022705, "grad_norm": 26.733783721923828, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.32497072219848633, "logits/rejected": -0.3295114040374756, "logps/chosen": -371.55743408203125, "logps/ref_chosen": -56.476951599121094, "logps/ref_rejected": -95.1385498046875, "logps/rejected": -580.8511962890625, "loss": 1.1184, "margin_dpo/margin_mean": 170.6321563720703, "margin_dpo/margin_std": 222.90133666992188, "step": 297 }, { "KL/chosen_KL_mean": -330.19439697265625, "KL/mean": -439.9351806640625, "KL/rejected_KL_mean": -549.6759643554688, "KL/std": 282.5112609863281, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.002016341080889106, "fcm_dpo/delta": -0.05950422212481499, "fcm_dpo/margin": 219.48158264160156, "fcm_dpo/q_t": 0.4024728238582611, "grad_norm": 25.506681442260742, "learning_rate": 3.474464683231698e-07, "logits/chosen": -0.36218610405921936, "logits/rejected": -0.3801892399787903, "logps/chosen": -397.51959228515625, "logps/ref_chosen": -67.32516479492188, "logps/ref_rejected": -116.66217041015625, "logps/rejected": -666.338134765625, "loss": 1.0872, "margin_dpo/margin_mean": 219.4815673828125, "margin_dpo/margin_std": 327.8146057128906, "step": 298 }, { "KL/chosen_KL_mean": -258.4277038574219, "KL/mean": -353.2135925292969, "KL/rejected_KL_mean": -447.99951171875, "KL/std": 213.69512939453125, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.001993919722735882, "fcm_dpo/delta": -0.052276819944381714, "fcm_dpo/margin": 189.57174682617188, "fcm_dpo/q_t": 0.4135133624076843, "grad_norm": 22.83670997619629, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.35608577728271484, "logits/rejected": -0.36062926054000854, "logps/chosen": -307.3898010253906, "logps/ref_chosen": -48.96209716796875, "logps/ref_rejected": -84.32823944091797, "logps/rejected": -532.3277587890625, "loss": 1.1126, "margin_dpo/margin_mean": 189.57174682617188, "margin_dpo/margin_std": 266.7342224121094, "step": 299 }, { "KL/chosen_KL_mean": -330.7674255371094, "KL/mean": -447.5982360839844, "KL/rejected_KL_mean": -564.4290161132812, "KL/std": 253.9108123779297, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.0019719761330634356, "fcm_dpo/delta": -0.05841895192861557, "fcm_dpo/margin": 233.66162109375, "fcm_dpo/q_t": 0.3947865962982178, "grad_norm": 22.9200382232666, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -0.3574295938014984, "logits/rejected": -0.36054882407188416, "logps/chosen": -389.84112548828125, "logps/ref_chosen": -59.07371139526367, "logps/ref_rejected": -95.9664535522461, "logps/rejected": -660.3955078125, "loss": 1.0641, "margin_dpo/margin_mean": 233.66162109375, "margin_dpo/margin_std": 307.6568908691406, "step": 300 }, { "KL/chosen_KL_mean": -282.0446472167969, "KL/mean": -391.8178405761719, "KL/rejected_KL_mean": -501.5910339355469, "KL/std": 232.38865661621094, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.0019472182029858232, "fcm_dpo/delta": -0.033876933157444, "fcm_dpo/margin": 219.54641723632812, "fcm_dpo/q_t": 0.40307682752609253, "grad_norm": 26.896240234375, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -0.3596671223640442, "logits/rejected": -0.35958829522132874, "logps/chosen": -339.29400634765625, "logps/ref_chosen": -57.249366760253906, "logps/ref_rejected": -92.35354614257812, "logps/rejected": -593.944580078125, "loss": 1.0718, "margin_dpo/margin_mean": 219.54641723632812, "margin_dpo/margin_std": 281.4098815917969, "step": 301 }, { "KL/chosen_KL_mean": -246.8507080078125, "KL/mean": -341.59490966796875, "KL/rejected_KL_mean": -436.339111328125, "KL/std": 210.02322387695312, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0019371429225429893, "fcm_dpo/delta": -0.025938192382454872, "fcm_dpo/margin": 189.48838806152344, "fcm_dpo/q_t": 0.41519904136657715, "grad_norm": 18.704797744750977, "learning_rate": 3.426989547989902e-07, "logits/chosen": -0.33896106481552124, "logits/rejected": -0.3427407741546631, "logps/chosen": -298.0487060546875, "logps/ref_chosen": -51.197994232177734, "logps/ref_rejected": -97.22636413574219, "logps/rejected": -533.5654907226562, "loss": 1.1105, "margin_dpo/margin_mean": 189.4884033203125, "margin_dpo/margin_std": 254.37966918945312, "step": 302 }, { "KL/chosen_KL_mean": -267.103515625, "KL/mean": -351.097412109375, "KL/rejected_KL_mean": -435.0912780761719, "KL/std": 222.8134765625, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.0019371429225429893, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 167.98776245117188, "fcm_dpo/q_t": 0.4245578348636627, "grad_norm": 21.095529556274414, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -0.3702804148197174, "logits/rejected": -0.35734331607818604, "logps/chosen": -333.8174743652344, "logps/ref_chosen": -66.71394348144531, "logps/ref_rejected": -86.94542694091797, "logps/rejected": -522.0367431640625, "loss": 1.1509, "margin_dpo/margin_mean": 167.98776245117188, "margin_dpo/margin_std": 264.559814453125, "step": 303 }, { "KL/chosen_KL_mean": -238.75244140625, "KL/mean": -331.8809509277344, "KL/rejected_KL_mean": -425.00946044921875, "KL/std": 199.30747985839844, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0019338021520525217, "fcm_dpo/delta": -0.01727566123008728, "fcm_dpo/margin": 186.2570343017578, "fcm_dpo/q_t": 0.41349148750305176, "grad_norm": 37.11244583129883, "learning_rate": 3.403104165467883e-07, "logits/chosen": -0.401491641998291, "logits/rejected": -0.3919214606285095, "logps/chosen": -310.703125, "logps/ref_chosen": -71.95069885253906, "logps/ref_rejected": -90.47203063964844, "logps/rejected": -515.4815063476562, "loss": 1.0899, "margin_dpo/margin_mean": 186.2570343017578, "margin_dpo/margin_std": 182.33255004882812, "step": 304 }, { "KL/chosen_KL_mean": -261.7723083496094, "KL/mean": -347.92303466796875, "KL/rejected_KL_mean": -434.0737609863281, "KL/std": 228.82867431640625, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.0019304613815620542, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 172.3014373779297, "fcm_dpo/q_t": 0.42360255122184753, "grad_norm": 20.761266708374023, "learning_rate": 3.391125620245535e-07, "logits/chosen": -0.40634918212890625, "logits/rejected": -0.39003273844718933, "logps/chosen": -328.56756591796875, "logps/ref_chosen": -66.79523468017578, "logps/ref_rejected": -92.75459289550781, "logps/rejected": -526.828369140625, "loss": 1.1393, "margin_dpo/margin_mean": 172.3014373779297, "margin_dpo/margin_std": 253.9287872314453, "step": 305 }, { "KL/chosen_KL_mean": -256.45379638671875, "KL/mean": -338.452880859375, "KL/rejected_KL_mean": -420.4520263671875, "KL/std": 199.2619171142578, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.0019304613815620542, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 163.99819946289062, "fcm_dpo/q_t": 0.4252777099609375, "grad_norm": 23.1071834564209, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -0.387315571308136, "logits/rejected": -0.3650142550468445, "logps/chosen": -326.1376953125, "logps/ref_chosen": -69.68389892578125, "logps/ref_rejected": -85.15919494628906, "logps/rejected": -505.6112060546875, "loss": 1.1352, "margin_dpo/margin_mean": 163.99819946289062, "margin_dpo/margin_std": 214.847412109375, "step": 306 }, { "KL/chosen_KL_mean": -229.98654174804688, "KL/mean": -315.85711669921875, "KL/rejected_KL_mean": -401.7276611328125, "KL/std": 180.1286163330078, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.0019304613815620542, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 171.74110412597656, "fcm_dpo/q_t": 0.4218043088912964, "grad_norm": 25.3315372467041, "learning_rate": 3.367098400098881e-07, "logits/chosen": -0.3864956200122833, "logits/rejected": -0.3664168119430542, "logps/chosen": -300.1519775390625, "logps/ref_chosen": -70.16542053222656, "logps/ref_rejected": -86.97230529785156, "logps/rejected": -488.699951171875, "loss": 1.1255, "margin_dpo/margin_mean": 171.74111938476562, "margin_dpo/margin_std": 222.21095275878906, "step": 307 }, { "KL/chosen_KL_mean": -238.41258239746094, "KL/mean": -333.81622314453125, "KL/rejected_KL_mean": -429.21990966796875, "KL/std": 205.3813934326172, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.0019304613815620542, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 190.80731201171875, "fcm_dpo/q_t": 0.4120708405971527, "grad_norm": 28.616819381713867, "learning_rate": 3.355050358314172e-07, "logits/chosen": -0.36001917719841003, "logits/rejected": -0.34961068630218506, "logps/chosen": -293.6575927734375, "logps/ref_chosen": -55.2449951171875, "logps/ref_rejected": -79.37226104736328, "logps/rejected": -508.5921630859375, "loss": 1.086, "margin_dpo/margin_mean": 190.80731201171875, "margin_dpo/margin_std": 194.4600067138672, "step": 308 }, { "KL/chosen_KL_mean": -246.6319580078125, "KL/mean": -343.8309020996094, "KL/rejected_KL_mean": -441.02984619140625, "KL/std": 216.66287231445312, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0019234230276197195, "fcm_dpo/delta": -0.036592997610569, "fcm_dpo/margin": 194.39788818359375, "fcm_dpo/q_t": 0.4111284017562866, "grad_norm": 21.753137588500977, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -0.33818137645721436, "logits/rejected": -0.33194500207901, "logps/chosen": -295.5910339355469, "logps/ref_chosen": -48.959083557128906, "logps/ref_rejected": -82.34072875976562, "logps/rejected": -523.37060546875, "loss": 1.0974, "margin_dpo/margin_mean": 194.3979034423828, "margin_dpo/margin_std": 233.65219116210938, "step": 309 }, { "KL/chosen_KL_mean": -290.893798828125, "KL/mean": -381.14703369140625, "KL/rejected_KL_mean": -471.4002380371094, "KL/std": 195.173828125, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.0019163846736773849, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 180.50640869140625, "fcm_dpo/q_t": 0.4173274040222168, "grad_norm": 29.18208122253418, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -0.4003097414970398, "logits/rejected": -0.39030611515045166, "logps/chosen": -353.6356201171875, "logps/ref_chosen": -62.74177932739258, "logps/ref_rejected": -79.9302978515625, "logps/rejected": -551.33056640625, "loss": 1.103, "margin_dpo/margin_mean": 180.50640869140625, "margin_dpo/margin_std": 187.60948181152344, "step": 310 }, { "KL/chosen_KL_mean": -302.8953857421875, "KL/mean": -404.6163330078125, "KL/rejected_KL_mean": -506.3373107910156, "KL/std": 253.53402709960938, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.0019163846736773849, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 203.44192504882812, "fcm_dpo/q_t": 0.4103236794471741, "grad_norm": 23.260826110839844, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -0.3798481822013855, "logits/rejected": -0.3574289083480835, "logps/chosen": -355.92333984375, "logps/ref_chosen": -53.02798080444336, "logps/ref_rejected": -77.43820190429688, "logps/rejected": -583.7755126953125, "loss": 1.0942, "margin_dpo/margin_mean": 203.44192504882812, "margin_dpo/margin_std": 265.4727783203125, "step": 311 }, { "KL/chosen_KL_mean": -281.3587341308594, "KL/mean": -376.16253662109375, "KL/rejected_KL_mean": -470.96636962890625, "KL/std": 222.47909545898438, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.0019128790590912104, "fcm_dpo/delta": -0.018326791003346443, "fcm_dpo/margin": 189.607666015625, "fcm_dpo/q_t": 0.41681230068206787, "grad_norm": 20.335750579833984, "learning_rate": 3.306636061080487e-07, "logits/chosen": -0.35430508852005005, "logits/rejected": -0.345356285572052, "logps/chosen": -330.7509460449219, "logps/ref_chosen": -49.39221954345703, "logps/ref_rejected": -75.79280853271484, "logps/rejected": -546.7591552734375, "loss": 1.1149, "margin_dpo/margin_mean": 189.607666015625, "margin_dpo/margin_std": 263.2210693359375, "step": 312 }, { "KL/chosen_KL_mean": -257.8659973144531, "KL/mean": -354.26409912109375, "KL/rejected_KL_mean": -450.66229248046875, "KL/std": 233.85096740722656, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.001899800612591207, "fcm_dpo/delta": -0.050388313829898834, "fcm_dpo/margin": 192.7962646484375, "fcm_dpo/q_t": 0.4142175018787384, "grad_norm": 21.600605010986328, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -0.4061209559440613, "logits/rejected": -0.4058893322944641, "logps/chosen": -308.01873779296875, "logps/ref_chosen": -50.152740478515625, "logps/ref_rejected": -86.40620422363281, "logps/rejected": -537.0684814453125, "loss": 1.1116, "margin_dpo/margin_mean": 192.79627990722656, "margin_dpo/margin_std": 252.84710693359375, "step": 313 }, { "KL/chosen_KL_mean": -273.6401672363281, "KL/mean": -364.00518798828125, "KL/rejected_KL_mean": -454.3702392578125, "KL/std": 219.98101806640625, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.0018902278970927, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 180.7301025390625, "fcm_dpo/q_t": 0.42193371057510376, "grad_norm": 21.16115379333496, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -0.3945729732513428, "logits/rejected": -0.3945373296737671, "logps/chosen": -330.87774658203125, "logps/ref_chosen": -57.237579345703125, "logps/ref_rejected": -97.5965347290039, "logps/rejected": -551.966796875, "loss": 1.1404, "margin_dpo/margin_mean": 180.7301025390625, "margin_dpo/margin_std": 281.34197998046875, "step": 314 }, { "KL/chosen_KL_mean": -255.35317993164062, "KL/mean": -339.78680419921875, "KL/rejected_KL_mean": -424.220458984375, "KL/std": 200.34494018554688, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.0018902278970927, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 168.86724853515625, "fcm_dpo/q_t": 0.4248756170272827, "grad_norm": 19.619430541992188, "learning_rate": 3.270101039870797e-07, "logits/chosen": -0.3391643762588501, "logits/rejected": -0.3408266305923462, "logps/chosen": -304.42279052734375, "logps/ref_chosen": -49.06958770751953, "logps/ref_rejected": -85.68087768554688, "logps/rejected": -509.9013366699219, "loss": 1.1346, "margin_dpo/margin_mean": 168.86724853515625, "margin_dpo/margin_std": 222.66644287109375, "step": 315 }, { "KL/chosen_KL_mean": -263.96673583984375, "KL/mean": -385.8088073730469, "KL/rejected_KL_mean": -507.65087890625, "KL/std": 244.05743408203125, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.0018620459595695138, "fcm_dpo/delta": -0.0751076489686966, "fcm_dpo/margin": 243.68417358398438, "fcm_dpo/q_t": 0.39560264348983765, "grad_norm": 24.86118507385254, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -0.37902936339378357, "logits/rejected": -0.38191086053848267, "logps/chosen": -318.22747802734375, "logps/ref_chosen": -54.26074981689453, "logps/ref_rejected": -101.2814712524414, "logps/rejected": -608.932373046875, "loss": 1.0429, "margin_dpo/margin_mean": 243.6841583251953, "margin_dpo/margin_std": 270.5803527832031, "step": 316 }, { "KL/chosen_KL_mean": -278.44287109375, "KL/mean": -410.1659240722656, "KL/rejected_KL_mean": -541.888916015625, "KL/std": 223.99581909179688, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0018459368729963899, "fcm_dpo/delta": -0.08726991713047028, "fcm_dpo/margin": 263.4460754394531, "fcm_dpo/q_t": 0.38609233498573303, "grad_norm": 25.92868995666504, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -0.38822823762893677, "logits/rejected": -0.380374550819397, "logps/chosen": -334.537109375, "logps/ref_chosen": -56.094207763671875, "logps/ref_rejected": -100.69905090332031, "logps/rejected": -642.5880126953125, "loss": 1.0104, "margin_dpo/margin_mean": 263.4460754394531, "margin_dpo/margin_std": 248.41903686523438, "step": 317 }, { "KL/chosen_KL_mean": -324.0246276855469, "KL/mean": -418.319580078125, "KL/rejected_KL_mean": -512.6144409179688, "KL/std": 236.26177978515625, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.001829827786423266, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 188.58981323242188, "fcm_dpo/q_t": 0.4197791814804077, "grad_norm": 24.93431854248047, "learning_rate": 3.233383385962115e-07, "logits/chosen": -0.4239439368247986, "logits/rejected": -0.3966226577758789, "logps/chosen": -388.67034912109375, "logps/ref_chosen": -64.64569854736328, "logps/ref_rejected": -82.76425170898438, "logps/rejected": -595.3787231445312, "loss": 1.1248, "margin_dpo/margin_mean": 188.58982849121094, "margin_dpo/margin_std": 262.1299133300781, "step": 318 }, { "KL/chosen_KL_mean": -292.07940673828125, "KL/mean": -410.94036865234375, "KL/rejected_KL_mean": -529.8013305664062, "KL/std": 264.96746826171875, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.0018263421952724457, "fcm_dpo/delta": -0.01908506453037262, "fcm_dpo/margin": 237.72190856933594, "fcm_dpo/q_t": 0.3996313810348511, "grad_norm": 24.182884216308594, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -0.38866111636161804, "logits/rejected": -0.40275949239730835, "logps/chosen": -341.4631652832031, "logps/ref_chosen": -49.383758544921875, "logps/ref_rejected": -113.90650939941406, "logps/rejected": -643.7078247070312, "loss": 1.0623, "margin_dpo/margin_mean": 237.72190856933594, "margin_dpo/margin_std": 287.574951171875, "step": 319 }, { "KL/chosen_KL_mean": -308.5746765136719, "KL/mean": -437.24224853515625, "KL/rejected_KL_mean": -565.9098510742188, "KL/std": 273.13323974609375, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0017987672472372651, "fcm_dpo/delta": -0.06651625782251358, "fcm_dpo/margin": 257.33514404296875, "fcm_dpo/q_t": 0.39359739422798157, "grad_norm": 25.218090057373047, "learning_rate": 3.208807785813777e-07, "logits/chosen": -0.4375894367694855, "logits/rejected": -0.444126695394516, "logps/chosen": -368.0795593261719, "logps/ref_chosen": -59.50489044189453, "logps/ref_rejected": -97.66717529296875, "logps/rejected": -663.5770263671875, "loss": 1.0414, "margin_dpo/margin_mean": 257.33514404296875, "margin_dpo/margin_std": 295.08465576171875, "step": 320 }, { "KL/chosen_KL_mean": -382.15399169921875, "KL/mean": -493.6779479980469, "KL/rejected_KL_mean": -605.201904296875, "KL/std": 284.0462646484375, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.0017797566251829267, "fcm_dpo/delta": -0.053124669939279556, "fcm_dpo/margin": 223.04794311523438, "fcm_dpo/q_t": 0.4093397259712219, "grad_norm": 23.08179473876953, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -0.34922099113464355, "logits/rejected": -0.3383622467517853, "logps/chosen": -443.70263671875, "logps/ref_chosen": -61.548683166503906, "logps/ref_rejected": -91.64103698730469, "logps/rejected": -696.8428955078125, "loss": 1.0986, "margin_dpo/margin_mean": 223.04794311523438, "margin_dpo/margin_std": 303.15924072265625, "step": 321 }, { "KL/chosen_KL_mean": -333.64361572265625, "KL/mean": -453.05535888671875, "KL/rejected_KL_mean": -572.467041015625, "KL/std": 251.202392578125, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.001764493528753519, "fcm_dpo/delta": -0.04306449741125107, "fcm_dpo/margin": 238.8234100341797, "fcm_dpo/q_t": 0.4015629291534424, "grad_norm": 24.869430541992188, "learning_rate": 3.184157475180207e-07, "logits/chosen": -0.3516596555709839, "logits/rejected": -0.3475235104560852, "logps/chosen": -390.93365478515625, "logps/ref_chosen": -57.29003143310547, "logps/ref_rejected": -95.74992370605469, "logps/rejected": -668.2169799804688, "loss": 1.0646, "margin_dpo/margin_mean": 238.82342529296875, "margin_dpo/margin_std": 272.7815246582031, "step": 322 }, { "KL/chosen_KL_mean": -347.6518249511719, "KL/mean": -450.37103271484375, "KL/rejected_KL_mean": -553.0902099609375, "KL/std": 251.44210815429688, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.001764493528753519, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 205.43838500976562, "fcm_dpo/q_t": 0.4151211678981781, "grad_norm": 26.678951263427734, "learning_rate": 3.171805115074251e-07, "logits/chosen": -0.39251190423965454, "logits/rejected": -0.39162832498550415, "logps/chosen": -398.88580322265625, "logps/ref_chosen": -51.23395919799805, "logps/ref_rejected": -75.06192016601562, "logps/rejected": -628.152099609375, "loss": 1.1155, "margin_dpo/margin_mean": 205.43838500976562, "margin_dpo/margin_std": 275.69659423828125, "step": 323 }, { "KL/chosen_KL_mean": -400.1598205566406, "KL/mean": -495.57952880859375, "KL/rejected_KL_mean": -590.9992065429688, "KL/std": 267.0489501953125, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.0017571839271113276, "fcm_dpo/delta": -0.04159846156835556, "fcm_dpo/margin": 190.83935546875, "fcm_dpo/q_t": 0.42405766248703003, "grad_norm": 43.70282745361328, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -0.41459232568740845, "logits/rejected": -0.3984828293323517, "logps/chosen": -465.29498291015625, "logps/ref_chosen": -65.13516998291016, "logps/ref_rejected": -86.47750854492188, "logps/rejected": -677.4766845703125, "loss": 1.1581, "margin_dpo/margin_mean": 190.83935546875, "margin_dpo/margin_std": 312.6383056640625, "step": 324 }, { "KL/chosen_KL_mean": -312.2501525878906, "KL/mean": -392.975341796875, "KL/rejected_KL_mean": -473.7005615234375, "KL/std": 232.5697021484375, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.0017498743254691362, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.45037841796875, "fcm_dpo/q_t": 0.43325573205947876, "grad_norm": 29.16065216064453, "learning_rate": 3.147047612756302e-07, "logits/chosen": -0.43759685754776, "logits/rejected": -0.4164847731590271, "logps/chosen": -368.46575927734375, "logps/ref_chosen": -56.215599060058594, "logps/ref_rejected": -70.08592987060547, "logps/rejected": -543.7864990234375, "loss": 1.1674, "margin_dpo/margin_mean": 161.45037841796875, "margin_dpo/margin_std": 235.22097778320312, "step": 325 }, { "KL/chosen_KL_mean": -339.22479248046875, "KL/mean": -419.08111572265625, "KL/rejected_KL_mean": -498.9373779296875, "KL/std": 221.22219848632812, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0017498743254691362, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 159.71255493164062, "fcm_dpo/q_t": 0.4328235685825348, "grad_norm": 41.599281311035156, "learning_rate": 3.134643122927519e-07, "logits/chosen": -0.4632147550582886, "logits/rejected": -0.44045084714889526, "logps/chosen": -411.94976806640625, "logps/ref_chosen": -72.72496032714844, "logps/ref_rejected": -79.8467788696289, "logps/rejected": -578.7841796875, "loss": 1.16, "margin_dpo/margin_mean": 159.71255493164062, "margin_dpo/margin_std": 209.68670654296875, "step": 326 }, { "KL/chosen_KL_mean": -285.94122314453125, "KL/mean": -405.94384765625, "KL/rejected_KL_mean": -525.9464111328125, "KL/std": 225.80906677246094, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.0017384198727086186, "fcm_dpo/delta": -0.032836832106113434, "fcm_dpo/margin": 240.0051727294922, "fcm_dpo/q_t": 0.40102726221084595, "grad_norm": 37.66844177246094, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -0.4289267659187317, "logits/rejected": -0.4173303246498108, "logps/chosen": -355.07568359375, "logps/ref_chosen": -69.13441467285156, "logps/ref_rejected": -111.93377685546875, "logps/rejected": -637.8801879882812, "loss": 1.0585, "margin_dpo/margin_mean": 240.0051727294922, "margin_dpo/margin_std": 252.79605102539062, "step": 327 }, { "KL/chosen_KL_mean": -292.0697021484375, "KL/mean": -397.9315185546875, "KL/rejected_KL_mean": -503.7932434082031, "KL/std": 248.23992919921875, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0017384198727086186, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 211.72357177734375, "fcm_dpo/q_t": 0.41479289531707764, "grad_norm": 32.22370147705078, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -0.39838314056396484, "logits/rejected": -0.4005652070045471, "logps/chosen": -351.75689697265625, "logps/ref_chosen": -59.68719482421875, "logps/ref_rejected": -90.85499572753906, "logps/rejected": -594.6482543945312, "loss": 1.1064, "margin_dpo/margin_mean": 211.7235870361328, "margin_dpo/margin_std": 279.32781982421875, "step": 328 }, { "KL/chosen_KL_mean": -312.3409118652344, "KL/mean": -422.5838623046875, "KL/rejected_KL_mean": -532.8267822265625, "KL/std": 251.46182250976562, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0017300716135650873, "fcm_dpo/delta": -0.024068888276815414, "fcm_dpo/margin": 220.48590087890625, "fcm_dpo/q_t": 0.4105890095233917, "grad_norm": 22.827281951904297, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -0.4284207224845886, "logits/rejected": -0.42305588722229004, "logps/chosen": -377.58709716796875, "logps/ref_chosen": -65.2461929321289, "logps/ref_rejected": -100.69770812988281, "logps/rejected": -633.5245361328125, "loss": 1.0963, "margin_dpo/margin_mean": 220.4858856201172, "margin_dpo/margin_std": 273.05218505859375, "step": 329 }, { "KL/chosen_KL_mean": -261.9295654296875, "KL/mean": -373.082763671875, "KL/rejected_KL_mean": -484.2359619140625, "KL/std": 231.3890380859375, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.0017300716135650873, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 222.30641174316406, "fcm_dpo/q_t": 0.40928471088409424, "grad_norm": 28.69952964782715, "learning_rate": 3.084861204504122e-07, "logits/chosen": -0.39853858947753906, "logits/rejected": -0.40001553297042847, "logps/chosen": -308.92791748046875, "logps/ref_chosen": -46.998348236083984, "logps/ref_rejected": -86.87684631347656, "logps/rejected": -571.11279296875, "loss": 1.0831, "margin_dpo/margin_mean": 222.30641174316406, "margin_dpo/margin_std": 250.98162841796875, "step": 330 }, { "KL/chosen_KL_mean": -287.3751220703125, "KL/mean": -403.163330078125, "KL/rejected_KL_mean": -518.9515380859375, "KL/std": 203.0115203857422, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0017300716135650873, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 231.57643127441406, "fcm_dpo/q_t": 0.40439319610595703, "grad_norm": 26.881025314331055, "learning_rate": 3.072376374875335e-07, "logits/chosen": -0.4044077396392822, "logits/rejected": -0.403422474861145, "logps/chosen": -337.89935302734375, "logps/ref_chosen": -50.52424621582031, "logps/ref_rejected": -89.01544189453125, "logps/rejected": -607.9669799804688, "loss": 1.0529, "margin_dpo/margin_mean": 231.57643127441406, "margin_dpo/margin_std": 198.66049194335938, "step": 331 }, { "KL/chosen_KL_mean": -285.37774658203125, "KL/mean": -369.9641418457031, "KL/rejected_KL_mean": -454.550537109375, "KL/std": 204.74688720703125, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.0017300716135650873, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 169.17282104492188, "fcm_dpo/q_t": 0.43071043491363525, "grad_norm": 19.305227279663086, "learning_rate": 3.059876462596758e-07, "logits/chosen": -0.41311419010162354, "logits/rejected": -0.3951377272605896, "logps/chosen": -334.5580139160156, "logps/ref_chosen": -49.18028259277344, "logps/ref_rejected": -76.48515319824219, "logps/rejected": -531.03564453125, "loss": 1.1515, "margin_dpo/margin_mean": 169.17282104492188, "margin_dpo/margin_std": 226.8921356201172, "step": 332 }, { "KL/chosen_KL_mean": -330.1800537109375, "KL/mean": -439.9482116699219, "KL/rejected_KL_mean": -549.7164306640625, "KL/std": 246.501708984375, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0017133665969595313, "fcm_dpo/delta": -0.048513028770685196, "fcm_dpo/margin": 219.5363311767578, "fcm_dpo/q_t": 0.4124882221221924, "grad_norm": 21.121355056762695, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -0.4535391628742218, "logits/rejected": -0.45026063919067383, "logps/chosen": -393.935791015625, "logps/ref_chosen": -63.75574493408203, "logps/ref_rejected": -95.04411315917969, "logps/rejected": -644.760498046875, "loss": 1.1113, "margin_dpo/margin_mean": 219.53634643554688, "margin_dpo/margin_std": 301.273193359375, "step": 333 }, { "KL/chosen_KL_mean": -326.6143798828125, "KL/mean": -427.30743408203125, "KL/rejected_KL_mean": -528.00048828125, "KL/std": 280.7286071777344, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0017133665969595313, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 201.3861541748047, "fcm_dpo/q_t": 0.42041999101638794, "grad_norm": 20.316781997680664, "learning_rate": 3.034832708016243e-07, "logits/chosen": -0.44451385736465454, "logits/rejected": -0.44493532180786133, "logps/chosen": -393.5941162109375, "logps/ref_chosen": -66.97975158691406, "logps/ref_rejected": -95.31692504882812, "logps/rejected": -623.3174438476562, "loss": 1.1404, "margin_dpo/margin_mean": 201.38613891601562, "margin_dpo/margin_std": 315.01531982421875, "step": 334 }, { "KL/chosen_KL_mean": -355.42633056640625, "KL/mean": -433.4613037109375, "KL/rejected_KL_mean": -511.4962158203125, "KL/std": 259.311279296875, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.0017133665969595313, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.06985473632812, "fcm_dpo/q_t": 0.4383538067340851, "grad_norm": 35.47575378417969, "learning_rate": 3.022289525640531e-07, "logits/chosen": -0.5016331672668457, "logits/rejected": -0.4807831048965454, "logps/chosen": -417.96881103515625, "logps/ref_chosen": -62.54248046875, "logps/ref_rejected": -87.61770629882812, "logps/rejected": -599.1138916015625, "loss": 1.1951, "margin_dpo/margin_mean": 156.06985473632812, "margin_dpo/margin_std": 283.9935302734375, "step": 335 }, { "KL/chosen_KL_mean": -360.68878173828125, "KL/mean": -491.305419921875, "KL/rejected_KL_mean": -621.9219970703125, "KL/std": 304.59197998046875, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.0017038836376741529, "fcm_dpo/delta": -0.05565565079450607, "fcm_dpo/margin": 261.233154296875, "fcm_dpo/q_t": 0.3995817005634308, "grad_norm": 31.521181106567383, "learning_rate": 3.009732580450086e-07, "logits/chosen": -0.437822163105011, "logits/rejected": -0.4387591481208801, "logps/chosen": -415.2199401855469, "logps/ref_chosen": -54.53115463256836, "logps/ref_rejected": -104.40424346923828, "logps/rejected": -726.3262329101562, "loss": 1.0782, "margin_dpo/margin_mean": 261.233154296875, "margin_dpo/margin_std": 359.9222412109375, "step": 336 }, { "KL/chosen_KL_mean": -337.1014709472656, "KL/mean": -464.4716796875, "KL/rejected_KL_mean": -591.8418579101562, "KL/std": 251.95245361328125, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.0016836783615872264, "fcm_dpo/delta": -0.03174104169011116, "fcm_dpo/margin": 254.7404327392578, "fcm_dpo/q_t": 0.40087810158729553, "grad_norm": 24.891990661621094, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -0.47642776370048523, "logits/rejected": -0.4633823037147522, "logps/chosen": -402.23016357421875, "logps/ref_chosen": -65.12869262695312, "logps/ref_rejected": -101.72701263427734, "logps/rejected": -693.56884765625, "loss": 1.0609, "margin_dpo/margin_mean": 254.7404327392578, "margin_dpo/margin_std": 297.6883544921875, "step": 337 }, { "KL/chosen_KL_mean": -304.122802734375, "KL/mean": -430.45965576171875, "KL/rejected_KL_mean": -556.7965087890625, "KL/std": 246.09283447265625, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.0016771751688793302, "fcm_dpo/delta": -0.0193497184664011, "fcm_dpo/margin": 252.67369079589844, "fcm_dpo/q_t": 0.4003807306289673, "grad_norm": 23.349788665771484, "learning_rate": 2.984578725527675e-07, "logits/chosen": -0.46150559186935425, "logits/rejected": -0.4557410180568695, "logps/chosen": -362.5455322265625, "logps/ref_chosen": -58.422706604003906, "logps/ref_rejected": -89.06854248046875, "logps/rejected": -645.8651123046875, "loss": 1.0515, "margin_dpo/margin_mean": 252.6737060546875, "margin_dpo/margin_std": 260.1355895996094, "step": 338 }, { "KL/chosen_KL_mean": -338.9353942871094, "KL/mean": -461.8382263183594, "KL/rejected_KL_mean": -584.7410888671875, "KL/std": 261.80877685546875, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.0016616201028227806, "fcm_dpo/delta": -0.09361618757247925, "fcm_dpo/margin": 245.80567932128906, "fcm_dpo/q_t": 0.40382570028305054, "grad_norm": 25.955692291259766, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -0.5029301047325134, "logits/rejected": -0.49164479970932007, "logps/chosen": -398.93072509765625, "logps/ref_chosen": -59.99531555175781, "logps/ref_rejected": -103.9109115600586, "logps/rejected": -688.6519775390625, "loss": 1.0729, "margin_dpo/margin_mean": 245.8056640625, "margin_dpo/margin_std": 265.2763366699219, "step": 339 }, { "KL/chosen_KL_mean": -354.7925720214844, "KL/mean": -459.96978759765625, "KL/rejected_KL_mean": -565.14697265625, "KL/std": 254.24560546875, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0016460651531815529, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 210.3543701171875, "fcm_dpo/q_t": 0.4198751151561737, "grad_norm": 25.57111358642578, "learning_rate": 2.959373794541426e-07, "logits/chosen": -0.43487322330474854, "logits/rejected": -0.41735124588012695, "logps/chosen": -407.622802734375, "logps/ref_chosen": -52.83022689819336, "logps/ref_rejected": -73.10723114013672, "logps/rejected": -638.2542114257812, "loss": 1.1323, "margin_dpo/margin_mean": 210.3543701171875, "margin_dpo/margin_std": 311.27935791015625, "step": 340 }, { "KL/chosen_KL_mean": -356.2671203613281, "KL/mean": -481.8756103515625, "KL/rejected_KL_mean": -607.4840087890625, "KL/std": 277.663330078125, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.0016328769270330667, "fcm_dpo/delta": -0.04022127762436867, "fcm_dpo/margin": 251.21697998046875, "fcm_dpo/q_t": 0.4051659405231476, "grad_norm": 23.709854125976562, "learning_rate": 2.946753005532965e-07, "logits/chosen": -0.44025927782058716, "logits/rejected": -0.4373493194580078, "logps/chosen": -404.16693115234375, "logps/ref_chosen": -47.899803161621094, "logps/ref_rejected": -101.80987548828125, "logps/rejected": -709.2939453125, "loss": 1.0766, "margin_dpo/margin_mean": 251.2169647216797, "margin_dpo/margin_std": 301.60986328125, "step": 341 }, { "KL/chosen_KL_mean": -355.93658447265625, "KL/mean": -460.60601806640625, "KL/rejected_KL_mean": -565.2755126953125, "KL/std": 265.940673828125, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.0016328769270330667, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 209.3389129638672, "fcm_dpo/q_t": 0.42056310176849365, "grad_norm": 25.819406509399414, "learning_rate": 2.934120444167326e-07, "logits/chosen": -0.46601247787475586, "logits/rejected": -0.4378005266189575, "logps/chosen": -427.9332275390625, "logps/ref_chosen": -71.99664306640625, "logps/ref_rejected": -92.58959197998047, "logps/rejected": -657.8651123046875, "loss": 1.1384, "margin_dpo/margin_mean": 209.3389129638672, "margin_dpo/margin_std": 320.1954345703125, "step": 342 }, { "KL/chosen_KL_mean": -332.80279541015625, "KL/mean": -464.02789306640625, "KL/rejected_KL_mean": -595.2529907226562, "KL/std": 269.5361633300781, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.001626888057217002, "fcm_dpo/delta": -0.018372224643826485, "fcm_dpo/margin": 262.4502258300781, "fcm_dpo/q_t": 0.39894723892211914, "grad_norm": 25.481830596923828, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -0.4878358244895935, "logits/rejected": -0.4912916421890259, "logps/chosen": -387.20843505859375, "logps/ref_chosen": -54.405616760253906, "logps/ref_rejected": -111.04142761230469, "logps/rejected": -706.29443359375, "loss": 1.0437, "margin_dpo/margin_mean": 262.4501953125, "margin_dpo/margin_std": 245.56637573242188, "step": 343 }, { "KL/chosen_KL_mean": -328.3038024902344, "KL/mean": -443.7353210449219, "KL/rejected_KL_mean": -559.1668701171875, "KL/std": 301.187744140625, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.0016134674660861492, "fcm_dpo/delta": -0.0831800252199173, "fcm_dpo/margin": 230.86305236816406, "fcm_dpo/q_t": 0.4162444472312927, "grad_norm": 28.6507625579834, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -0.45678555965423584, "logits/rejected": -0.4595603346824646, "logps/chosen": -382.2684631347656, "logps/ref_chosen": -53.96466827392578, "logps/ref_rejected": -90.62336730957031, "logps/rejected": -649.7902221679688, "loss": 1.1177, "margin_dpo/margin_mean": 230.86305236816406, "margin_dpo/margin_std": 319.19000244140625, "step": 344 }, { "KL/chosen_KL_mean": -378.3681945800781, "KL/mean": -508.10675048828125, "KL/rejected_KL_mean": -637.84521484375, "KL/std": 275.95458984375, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.0015956300776451826, "fcm_dpo/delta": -0.027681613340973854, "fcm_dpo/margin": 259.47705078125, "fcm_dpo/q_t": 0.4036673307418823, "grad_norm": 21.58487319946289, "learning_rate": 2.896155456223163e-07, "logits/chosen": -0.48699015378952026, "logits/rejected": -0.48181208968162537, "logps/chosen": -440.05389404296875, "logps/ref_chosen": -61.685699462890625, "logps/ref_rejected": -99.49041748046875, "logps/rejected": -737.335693359375, "loss": 1.0708, "margin_dpo/margin_mean": 259.47705078125, "margin_dpo/margin_std": 307.22149658203125, "step": 345 }, { "KL/chosen_KL_mean": -373.90875244140625, "KL/mean": -488.475830078125, "KL/rejected_KL_mean": -603.04296875, "KL/std": 266.25341796875, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.001591213047504425, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 229.13418579101562, "fcm_dpo/q_t": 0.4135410785675049, "grad_norm": 20.24309730529785, "learning_rate": 2.883479137196714e-07, "logits/chosen": -0.4244663417339325, "logits/rejected": -0.4114094376564026, "logps/chosen": -429.16497802734375, "logps/ref_chosen": -55.256263732910156, "logps/ref_rejected": -77.41532135009766, "logps/rejected": -680.458251953125, "loss": 1.1037, "margin_dpo/margin_mean": 229.13418579101562, "margin_dpo/margin_std": 285.3810729980469, "step": 346 }, { "KL/chosen_KL_mean": -370.2218017578125, "KL/mean": -484.5078430175781, "KL/rejected_KL_mean": -598.7938842773438, "KL/std": 279.05316162109375, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.001591213047504425, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 228.57208251953125, "fcm_dpo/q_t": 0.41585496068000793, "grad_norm": 24.884403228759766, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -0.42082369327545166, "logits/rejected": -0.4123762845993042, "logps/chosen": -427.7880554199219, "logps/ref_chosen": -57.56623840332031, "logps/ref_rejected": -92.35509490966797, "logps/rejected": -691.1489868164062, "loss": 1.1164, "margin_dpo/margin_mean": 228.5720672607422, "margin_dpo/margin_std": 320.4414367675781, "step": 347 }, { "KL/chosen_KL_mean": -320.3187561035156, "KL/mean": -418.7101135253906, "KL/rejected_KL_mean": -517.1014404296875, "KL/std": 240.5898895263672, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.001591213047504425, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 196.78271484375, "fcm_dpo/q_t": 0.42725011706352234, "grad_norm": 21.602191925048828, "learning_rate": 2.858096518347179e-07, "logits/chosen": -0.43160480260849, "logits/rejected": -0.4293760657310486, "logps/chosen": -376.636474609375, "logps/ref_chosen": -56.31770324707031, "logps/ref_rejected": -89.13836669921875, "logps/rejected": -606.2398071289062, "loss": 1.1474, "margin_dpo/margin_mean": 196.78271484375, "margin_dpo/margin_std": 281.14556884765625, "step": 348 }, { "KL/chosen_KL_mean": -299.2859802246094, "KL/mean": -415.08184814453125, "KL/rejected_KL_mean": -530.877685546875, "KL/std": 265.25457763671875, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.001591213047504425, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 231.59170532226562, "fcm_dpo/q_t": 0.41541868448257446, "grad_norm": 19.77528190612793, "learning_rate": 2.845390887379706e-07, "logits/chosen": -0.44090381264686584, "logits/rejected": -0.44044074416160583, "logps/chosen": -357.3114929199219, "logps/ref_chosen": -58.025516510009766, "logps/ref_rejected": -97.50515747070312, "logps/rejected": -628.3828125, "loss": 1.115, "margin_dpo/margin_mean": 231.59170532226562, "margin_dpo/margin_std": 324.92840576171875, "step": 349 }, { "KL/chosen_KL_mean": -323.4007873535156, "KL/mean": -432.3330078125, "KL/rejected_KL_mean": -541.2652587890625, "KL/std": 254.46958923339844, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.001591213047504425, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 217.86448669433594, "fcm_dpo/q_t": 0.41904622316360474, "grad_norm": 28.25132179260254, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -0.46974977850914, "logits/rejected": -0.47410547733306885, "logps/chosen": -387.73126220703125, "logps/ref_chosen": -64.33049011230469, "logps/ref_rejected": -89.87164306640625, "logps/rejected": -631.1369018554688, "loss": 1.1313, "margin_dpo/margin_mean": 217.86448669433594, "margin_dpo/margin_std": 318.74139404296875, "step": 350 }, { "KL/chosen_KL_mean": -299.21258544921875, "KL/mean": -435.21917724609375, "KL/rejected_KL_mean": -571.2257080078125, "KL/std": 299.67193603515625, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0015760548412799835, "fcm_dpo/delta": -0.04785942658782005, "fcm_dpo/margin": 272.0131530761719, "fcm_dpo/q_t": 0.4027807116508484, "grad_norm": 22.712406158447266, "learning_rate": 2.819952656376487e-07, "logits/chosen": -0.46151453256607056, "logits/rejected": -0.4617936313152313, "logps/chosen": -359.88470458984375, "logps/ref_chosen": -60.6721305847168, "logps/ref_rejected": -101.5654296875, "logps/rejected": -672.7911376953125, "loss": 1.0746, "margin_dpo/margin_mean": 272.0131530761719, "margin_dpo/margin_std": 353.1802673339844, "step": 351 }, { "KL/chosen_KL_mean": -352.48883056640625, "KL/mean": -430.4334716796875, "KL/rejected_KL_mean": -508.3780822753906, "KL/std": 270.8203125, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.0015760548412799835, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 155.88925170898438, "fcm_dpo/q_t": 0.4424957036972046, "grad_norm": 27.769054412841797, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -0.43828731775283813, "logits/rejected": -0.40562915802001953, "logps/chosen": -423.43231201171875, "logps/ref_chosen": -70.9434585571289, "logps/ref_rejected": -76.6419677734375, "logps/rejected": -585.02001953125, "loss": 1.2111, "margin_dpo/margin_mean": 155.88925170898438, "margin_dpo/margin_std": 301.49072265625, "step": 352 }, { "KL/chosen_KL_mean": -329.95526123046875, "KL/mean": -440.26611328125, "KL/rejected_KL_mean": -550.5770263671875, "KL/std": 270.30609130859375, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": -0.02639908902347088, "fcm_dpo/margin": 220.6217498779297, "fcm_dpo/q_t": 0.41987377405166626, "grad_norm": 23.161598205566406, "learning_rate": 2.794480701395219e-07, "logits/chosen": -0.46952152252197266, "logits/rejected": -0.4589802622795105, "logps/chosen": -388.3505859375, "logps/ref_chosen": -58.39533996582031, "logps/ref_rejected": -80.33553314208984, "logps/rejected": -630.9125366210938, "loss": 1.1279, "margin_dpo/margin_mean": 220.6217498779297, "margin_dpo/margin_std": 308.440673828125, "step": 353 }, { "KL/chosen_KL_mean": -278.52569580078125, "KL/mean": -397.5231628417969, "KL/rejected_KL_mean": -516.5206298828125, "KL/std": 254.86386108398438, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 237.9949493408203, "fcm_dpo/q_t": 0.4116242527961731, "grad_norm": 19.947385787963867, "learning_rate": 2.781732916288303e-07, "logits/chosen": -0.4535951018333435, "logits/rejected": -0.44199591875076294, "logps/chosen": -338.32867431640625, "logps/ref_chosen": -59.80299377441406, "logps/ref_rejected": -88.75750732421875, "logps/rejected": -605.2781372070312, "loss": 1.0851, "margin_dpo/margin_mean": 237.99493408203125, "margin_dpo/margin_std": 252.84063720703125, "step": 354 }, { "KL/chosen_KL_mean": -279.12908935546875, "KL/mean": -396.08306884765625, "KL/rejected_KL_mean": -513.037109375, "KL/std": 246.86309814453125, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.90802001953125, "fcm_dpo/q_t": 0.41276803612709045, "grad_norm": 27.744285583496094, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -0.5026720762252808, "logits/rejected": -0.4931907057762146, "logps/chosen": -333.257568359375, "logps/ref_chosen": -54.12849807739258, "logps/ref_rejected": -82.40606689453125, "logps/rejected": -595.4431762695312, "loss": 1.0913, "margin_dpo/margin_mean": 233.90802001953125, "margin_dpo/margin_std": 254.10516357421875, "step": 355 }, { "KL/chosen_KL_mean": -336.5333251953125, "KL/mean": -401.6588134765625, "KL/rejected_KL_mean": -466.7843322753906, "KL/std": 260.546142578125, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.2509765625, "fcm_dpo/q_t": 0.45279616117477417, "grad_norm": 25.384258270263672, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -0.44052648544311523, "logits/rejected": -0.41894644498825073, "logps/chosen": -401.2071533203125, "logps/ref_chosen": -64.6738052368164, "logps/ref_rejected": -75.89926147460938, "logps/rejected": -542.68359375, "loss": 1.254, "margin_dpo/margin_mean": 130.2509765625, "margin_dpo/margin_std": 317.67974853515625, "step": 356 }, { "KL/chosen_KL_mean": -309.42535400390625, "KL/mean": -412.43572998046875, "KL/rejected_KL_mean": -515.4461059570312, "KL/std": 250.17401123046875, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 206.020751953125, "fcm_dpo/q_t": 0.42332154512405396, "grad_norm": 22.215511322021484, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -0.46077555418014526, "logits/rejected": -0.44984954595565796, "logps/chosen": -362.1511535644531, "logps/ref_chosen": -52.725799560546875, "logps/ref_rejected": -86.84115600585938, "logps/rejected": -602.2872314453125, "loss": 1.1306, "margin_dpo/margin_mean": 206.020751953125, "margin_dpo/margin_std": 262.2787170410156, "step": 357 }, { "KL/chosen_KL_mean": -280.0639953613281, "KL/mean": -377.8428955078125, "KL/rejected_KL_mean": -475.62176513671875, "KL/std": 249.71923828125, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 195.55775451660156, "fcm_dpo/q_t": 0.428341269493103, "grad_norm": 23.287120819091797, "learning_rate": 2.730670898658255e-07, "logits/chosen": -0.46829819679260254, "logits/rejected": -0.4512297511100769, "logps/chosen": -343.2694396972656, "logps/ref_chosen": -63.20543670654297, "logps/ref_rejected": -88.373291015625, "logps/rejected": -563.9950561523438, "loss": 1.1479, "margin_dpo/margin_mean": 195.55775451660156, "margin_dpo/margin_std": 277.779296875, "step": 358 }, { "KL/chosen_KL_mean": -324.8550109863281, "KL/mean": -441.377685546875, "KL/rejected_KL_mean": -557.900390625, "KL/std": 250.80963134765625, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.04534912109375, "fcm_dpo/q_t": 0.41488760709762573, "grad_norm": 28.12652587890625, "learning_rate": 2.717889356869146e-07, "logits/chosen": -0.4374353289604187, "logits/rejected": -0.4322439730167389, "logps/chosen": -381.2252197265625, "logps/ref_chosen": -56.370216369628906, "logps/ref_rejected": -82.17375183105469, "logps/rejected": -640.0740966796875, "loss": 1.1074, "margin_dpo/margin_mean": 233.04534912109375, "margin_dpo/margin_std": 298.8861083984375, "step": 359 }, { "KL/chosen_KL_mean": -324.079345703125, "KL/mean": -412.91864013671875, "KL/rejected_KL_mean": -501.7579345703125, "KL/std": 216.36245727539062, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 177.67855834960938, "fcm_dpo/q_t": 0.432937353849411, "grad_norm": 41.02315139770508, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -0.3901089131832123, "logits/rejected": -0.3776787519454956, "logps/chosen": -375.53973388671875, "logps/ref_chosen": -51.460384368896484, "logps/ref_rejected": -69.83892059326172, "logps/rejected": -571.5968017578125, "loss": 1.1544, "margin_dpo/margin_mean": 177.67855834960938, "margin_dpo/margin_std": 213.44635009765625, "step": 360 }, { "KL/chosen_KL_mean": -345.0618591308594, "KL/mean": -432.0201110839844, "KL/rejected_KL_mean": -518.9783935546875, "KL/std": 251.68954467773438, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.0015677554765716195, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 173.91651916503906, "fcm_dpo/q_t": 0.4354216456413269, "grad_norm": 32.919921875, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -0.407523512840271, "logits/rejected": -0.4052034020423889, "logps/chosen": -398.9313659667969, "logps/ref_chosen": -53.86951446533203, "logps/ref_rejected": -90.7692642211914, "logps/rejected": -609.7476806640625, "loss": 1.1766, "margin_dpo/margin_mean": 173.91650390625, "margin_dpo/margin_std": 272.8636474609375, "step": 361 }, { "KL/chosen_KL_mean": -295.00146484375, "KL/mean": -441.1478271484375, "KL/rejected_KL_mean": -587.294189453125, "KL/std": 273.6327819824219, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0015489872312173247, "fcm_dpo/delta": -0.06021817773580551, "fcm_dpo/margin": 292.292724609375, "fcm_dpo/q_t": 0.3948058485984802, "grad_norm": 20.981021881103516, "learning_rate": 2.679511629948319e-07, "logits/chosen": -0.4132572114467621, "logits/rejected": -0.4234897494316101, "logps/chosen": -353.6405334472656, "logps/ref_chosen": -58.639060974121094, "logps/ref_rejected": -105.58195495605469, "logps/rejected": -692.8761596679688, "loss": 1.0325, "margin_dpo/margin_mean": 292.2927551269531, "margin_dpo/margin_std": 293.92108154296875, "step": 362 }, { "KL/chosen_KL_mean": -274.0687255859375, "KL/mean": -418.25189208984375, "KL/rejected_KL_mean": -562.43505859375, "KL/std": 264.7567138671875, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.0015410305932164192, "fcm_dpo/delta": -0.05163221433758736, "fcm_dpo/margin": 288.3663024902344, "fcm_dpo/q_t": 0.39612919092178345, "grad_norm": 21.660552978515625, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -0.37580353021621704, "logits/rejected": -0.3681963384151459, "logps/chosen": -318.62713623046875, "logps/ref_chosen": -44.558380126953125, "logps/ref_rejected": -74.69496154785156, "logps/rejected": -637.1300048828125, "loss": 1.0369, "margin_dpo/margin_mean": 288.3663024902344, "margin_dpo/margin_std": 287.9776916503906, "step": 363 }, { "KL/chosen_KL_mean": -291.2659606933594, "KL/mean": -400.46966552734375, "KL/rejected_KL_mean": -509.6733703613281, "KL/std": 254.54420471191406, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0015330739552155137, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 218.40740966796875, "fcm_dpo/q_t": 0.42098337411880493, "grad_norm": 20.794410705566406, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -0.4048645496368408, "logits/rejected": -0.41446197032928467, "logps/chosen": -340.16058349609375, "logps/ref_chosen": -48.894622802734375, "logps/ref_rejected": -91.395751953125, "logps/rejected": -601.069091796875, "loss": 1.128, "margin_dpo/margin_mean": 218.4074249267578, "margin_dpo/margin_std": 291.99188232421875, "step": 364 }, { "KL/chosen_KL_mean": -289.51519775390625, "KL/mean": -402.83343505859375, "KL/rejected_KL_mean": -516.1517333984375, "KL/std": 266.38848876953125, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0015330739552155137, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 226.6365203857422, "fcm_dpo/q_t": 0.418827623128891, "grad_norm": 20.097055435180664, "learning_rate": 2.641091331089811e-07, "logits/chosen": -0.41409730911254883, "logits/rejected": -0.4254269003868103, "logps/chosen": -341.0079345703125, "logps/ref_chosen": -51.49274444580078, "logps/ref_rejected": -92.70166778564453, "logps/rejected": -608.8533935546875, "loss": 1.1112, "margin_dpo/margin_mean": 226.6365203857422, "margin_dpo/margin_std": 275.52093505859375, "step": 365 }, { "KL/chosen_KL_mean": -264.1698303222656, "KL/mean": -380.6763000488281, "KL/rejected_KL_mean": -497.1827392578125, "KL/std": 253.22549438476562, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.0015279599465429783, "fcm_dpo/delta": -0.016706913709640503, "fcm_dpo/margin": 233.01290893554688, "fcm_dpo/q_t": 0.4166252017021179, "grad_norm": 18.73899269104004, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -0.39603766798973083, "logits/rejected": -0.4109054207801819, "logps/chosen": -308.8904113769531, "logps/ref_chosen": -44.7205696105957, "logps/ref_rejected": -83.31040954589844, "logps/rejected": -580.4931640625, "loss": 1.1126, "margin_dpo/margin_mean": 233.01290893554688, "margin_dpo/margin_std": 300.14300537109375, "step": 366 }, { "KL/chosen_KL_mean": -282.4344482421875, "KL/mean": -374.6944885253906, "KL/rejected_KL_mean": -466.95452880859375, "KL/std": 224.01869201660156, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0015279599465429783, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 184.52005004882812, "fcm_dpo/q_t": 0.43212103843688965, "grad_norm": 19.241165161132812, "learning_rate": 2.615458646614349e-07, "logits/chosen": -0.40832167863845825, "logits/rejected": -0.3911445736885071, "logps/chosen": -340.8398742675781, "logps/ref_chosen": -58.405418395996094, "logps/ref_rejected": -76.75132751464844, "logps/rejected": -543.705810546875, "loss": 1.1573, "margin_dpo/margin_mean": 184.52005004882812, "margin_dpo/margin_std": 241.02235412597656, "step": 367 }, { "KL/chosen_KL_mean": -264.31964111328125, "KL/mean": -416.4783630371094, "KL/rejected_KL_mean": -568.6371459960938, "KL/std": 257.91253662109375, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0015139949973672628, "fcm_dpo/delta": -0.0636521428823471, "fcm_dpo/margin": 304.3175048828125, "fcm_dpo/q_t": 0.38951247930526733, "grad_norm": 34.35197830200195, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -0.40232932567596436, "logits/rejected": -0.41085970401763916, "logps/chosen": -308.77215576171875, "logps/ref_chosen": -44.452518463134766, "logps/ref_rejected": -98.55526733398438, "logps/rejected": -667.1923828125, "loss": 1.0069, "margin_dpo/margin_mean": 304.3175048828125, "margin_dpo/margin_std": 230.89181518554688, "step": 368 }, { "KL/chosen_KL_mean": -361.3604736328125, "KL/mean": -439.23187255859375, "KL/rejected_KL_mean": -517.103271484375, "KL/std": 264.01806640625, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0015086316270753741, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 155.74282836914062, "fcm_dpo/q_t": 0.44402188062667847, "grad_norm": 23.49022674560547, "learning_rate": 2.589813792448196e-07, "logits/chosen": -0.43115919828414917, "logits/rejected": -0.4122559428215027, "logps/chosen": -432.741943359375, "logps/ref_chosen": -71.38150024414062, "logps/ref_rejected": -91.29582214355469, "logps/rejected": -608.3991088867188, "loss": 1.2154, "margin_dpo/margin_mean": 155.74282836914062, "margin_dpo/margin_std": 302.1126403808594, "step": 369 }, { "KL/chosen_KL_mean": -383.2568664550781, "KL/mean": -458.2979736328125, "KL/rejected_KL_mean": -533.339111328125, "KL/std": 276.94122314453125, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.0015086316270753741, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 150.08221435546875, "fcm_dpo/q_t": 0.4468126595020294, "grad_norm": 24.77077865600586, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -0.451901376247406, "logits/rejected": -0.4463951885700226, "logps/chosen": -454.8643798828125, "logps/ref_chosen": -71.60749816894531, "logps/ref_rejected": -97.25978088378906, "logps/rejected": -630.598876953125, "loss": 1.227, "margin_dpo/margin_mean": 150.0822296142578, "margin_dpo/margin_std": 312.60546875, "step": 370 }, { "KL/chosen_KL_mean": -370.62091064453125, "KL/mean": -483.0035400390625, "KL/rejected_KL_mean": -595.3861083984375, "KL/std": 281.2415466308594, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.0015086316270753741, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 224.7652130126953, "fcm_dpo/q_t": 0.4212295413017273, "grad_norm": 24.414459228515625, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -0.49121659994125366, "logits/rejected": -0.4816587269306183, "logps/chosen": -440.035400390625, "logps/ref_chosen": -69.41448974609375, "logps/ref_rejected": -99.17217254638672, "logps/rejected": -694.558349609375, "loss": 1.1338, "margin_dpo/margin_mean": 224.76519775390625, "margin_dpo/margin_std": 325.4281005859375, "step": 371 }, { "KL/chosen_KL_mean": -364.821533203125, "KL/mean": -496.4064636230469, "KL/rejected_KL_mean": -627.991455078125, "KL/std": 315.2689208984375, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.0015086316270753741, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 263.16986083984375, "fcm_dpo/q_t": 0.40971818566322327, "grad_norm": 20.666433334350586, "learning_rate": 2.551329606220976e-07, "logits/chosen": -0.4030493497848511, "logits/rejected": -0.3827578127384186, "logps/chosen": -426.6395263671875, "logps/ref_chosen": -61.8179931640625, "logps/ref_rejected": -78.53948974609375, "logps/rejected": -706.5308837890625, "loss": 1.0942, "margin_dpo/margin_mean": 263.16986083984375, "margin_dpo/margin_std": 357.79962158203125, "step": 372 }, { "KL/chosen_KL_mean": -407.0643310546875, "KL/mean": -529.6451416015625, "KL/rejected_KL_mean": -652.2260131835938, "KL/std": 310.29119873046875, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.0015036920085549355, "fcm_dpo/delta": -0.032850027084350586, "fcm_dpo/margin": 245.16168212890625, "fcm_dpo/q_t": 0.4128532111644745, "grad_norm": 24.507394790649414, "learning_rate": 2.538498388222517e-07, "logits/chosen": -0.4359899163246155, "logits/rejected": -0.4177612066268921, "logps/chosen": -471.2814636230469, "logps/ref_chosen": -64.21713256835938, "logps/ref_rejected": -85.95960998535156, "logps/rejected": -738.1856689453125, "loss": 1.1025, "margin_dpo/margin_mean": 245.16168212890625, "margin_dpo/margin_std": 297.9869079589844, "step": 373 }, { "KL/chosen_KL_mean": -354.3941955566406, "KL/mean": -471.44720458984375, "KL/rejected_KL_mean": -588.500244140625, "KL/std": 333.9794006347656, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.0014987523900344968, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 234.10601806640625, "fcm_dpo/q_t": 0.4215013086795807, "grad_norm": 20.826824188232422, "learning_rate": 2.525666155755725e-07, "logits/chosen": -0.4955642819404602, "logits/rejected": -0.4787040948867798, "logps/chosen": -425.04437255859375, "logps/ref_chosen": -70.65018463134766, "logps/ref_rejected": -93.64016723632812, "logps/rejected": -682.140380859375, "loss": 1.1402, "margin_dpo/margin_mean": 234.10601806640625, "margin_dpo/margin_std": 377.1463623046875, "step": 374 }, { "KL/chosen_KL_mean": -363.83551025390625, "KL/mean": -474.2115173339844, "KL/rejected_KL_mean": -584.5875244140625, "KL/std": 268.56109619140625, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0014939869288355112, "fcm_dpo/delta": -0.015923552215099335, "fcm_dpo/margin": 220.7519989013672, "fcm_dpo/q_t": 0.4224032163619995, "grad_norm": 23.4717960357666, "learning_rate": 2.512833246961859e-07, "logits/chosen": -0.4513305425643921, "logits/rejected": -0.44845885038375854, "logps/chosen": -423.9157409667969, "logps/ref_chosen": -60.080223083496094, "logps/ref_rejected": -88.93830871582031, "logps/rejected": -673.5258178710938, "loss": 1.1395, "margin_dpo/margin_mean": 220.75198364257812, "margin_dpo/margin_std": 318.7281494140625, "step": 375 }, { "KL/chosen_KL_mean": -352.83441162109375, "KL/mean": -490.5364990234375, "KL/rejected_KL_mean": -628.2386474609375, "KL/std": 288.8502197265625, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0014939869288355112, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 275.4041748046875, "fcm_dpo/q_t": 0.40431779623031616, "grad_norm": 19.61762046813965, "learning_rate": 2.5e-07, "logits/chosen": -0.4351574778556824, "logits/rejected": -0.4265507459640503, "logps/chosen": -415.4947509765625, "logps/ref_chosen": -62.660308837890625, "logps/ref_rejected": -105.52660369873047, "logps/rejected": -733.7651977539062, "loss": 1.0763, "margin_dpo/margin_mean": 275.4041748046875, "margin_dpo/margin_std": 336.59124755859375, "step": 376 }, { "KL/chosen_KL_mean": -350.9661865234375, "KL/mean": -482.7196960449219, "KL/rejected_KL_mean": -614.4732666015625, "KL/std": 300.648193359375, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.001491023926064372, "fcm_dpo/delta": -0.019872425124049187, "fcm_dpo/margin": 263.50701904296875, "fcm_dpo/q_t": 0.4078996479511261, "grad_norm": 24.318864822387695, "learning_rate": 2.487166753038141e-07, "logits/chosen": -0.4018658399581909, "logits/rejected": -0.40109121799468994, "logps/chosen": -405.4449157714844, "logps/ref_chosen": -54.478736877441406, "logps/ref_rejected": -98.70335388183594, "logps/rejected": -713.1765747070312, "loss": 1.0815, "margin_dpo/margin_mean": 263.50701904296875, "margin_dpo/margin_std": 306.6456298828125, "step": 377 }, { "KL/chosen_KL_mean": -330.1966552734375, "KL/mean": -464.34942626953125, "KL/rejected_KL_mean": -598.502197265625, "KL/std": 271.9918212890625, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.001488060923293233, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 268.3055419921875, "fcm_dpo/q_t": 0.4054878354072571, "grad_norm": 25.818546295166016, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -0.4009881019592285, "logits/rejected": -0.4145526885986328, "logps/chosen": -375.2171630859375, "logps/ref_chosen": -45.02053451538086, "logps/ref_rejected": -88.0469741821289, "logps/rejected": -686.5491943359375, "loss": 1.0712, "margin_dpo/margin_mean": 268.3055419921875, "margin_dpo/margin_std": 292.1213684082031, "step": 378 }, { "KL/chosen_KL_mean": -372.35675048828125, "KL/mean": -506.56787109375, "KL/rejected_KL_mean": -640.779052734375, "KL/std": 279.959228515625, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.0014759085606783628, "fcm_dpo/delta": -0.0410006120800972, "fcm_dpo/margin": 268.42230224609375, "fcm_dpo/q_t": 0.40835413336753845, "grad_norm": 25.43598175048828, "learning_rate": 2.461501611777483e-07, "logits/chosen": -0.3865029811859131, "logits/rejected": -0.40706324577331543, "logps/chosen": -425.538818359375, "logps/ref_chosen": -53.182098388671875, "logps/ref_rejected": -114.3001708984375, "logps/rejected": -755.0792236328125, "loss": 1.0911, "margin_dpo/margin_mean": 268.42230224609375, "margin_dpo/margin_std": 345.518798828125, "step": 379 }, { "KL/chosen_KL_mean": -355.4010314941406, "KL/mean": -506.8250732421875, "KL/rejected_KL_mean": -658.2490844726562, "KL/std": 315.39031982421875, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.0014712773263454437, "fcm_dpo/delta": -0.03147752955555916, "fcm_dpo/margin": 302.84808349609375, "fcm_dpo/q_t": 0.39569777250289917, "grad_norm": 25.094398498535156, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -0.4250390827655792, "logits/rejected": -0.4504019021987915, "logps/chosen": -406.7540588378906, "logps/ref_chosen": -51.3530387878418, "logps/ref_rejected": -104.19169616699219, "logps/rejected": -762.4407958984375, "loss": 1.0505, "margin_dpo/margin_mean": 302.84808349609375, "margin_dpo/margin_std": 347.689453125, "step": 380 }, { "KL/chosen_KL_mean": -372.30035400390625, "KL/mean": -475.10699462890625, "KL/rejected_KL_mean": -577.91357421875, "KL/std": 262.00640869140625, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 205.61322021484375, "fcm_dpo/q_t": 0.4295688271522522, "grad_norm": 22.824941635131836, "learning_rate": 2.435840528363426e-07, "logits/chosen": -0.4074001610279083, "logits/rejected": -0.3904430866241455, "logps/chosen": -430.1034240722656, "logps/ref_chosen": -57.80306625366211, "logps/ref_rejected": -79.21940612792969, "logps/rejected": -657.1329956054688, "loss": 1.1742, "margin_dpo/margin_mean": 205.61322021484375, "margin_dpo/margin_std": 360.230224609375, "step": 381 }, { "KL/chosen_KL_mean": -361.166015625, "KL/mean": -490.76263427734375, "KL/rejected_KL_mean": -620.3592529296875, "KL/std": 250.95346069335938, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 259.1932067871094, "fcm_dpo/q_t": 0.41025757789611816, "grad_norm": 23.912845611572266, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -0.4556125998497009, "logits/rejected": -0.46023356914520264, "logps/chosen": -427.18634033203125, "logps/ref_chosen": -66.02030181884766, "logps/ref_rejected": -110.71016693115234, "logps/rejected": -731.0693969726562, "loss": 1.0818, "margin_dpo/margin_mean": 259.1932067871094, "margin_dpo/margin_std": 278.27044677734375, "step": 382 }, { "KL/chosen_KL_mean": -376.48089599609375, "KL/mean": -501.99945068359375, "KL/rejected_KL_mean": -627.5180053710938, "KL/std": 281.06005859375, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 251.037109375, "fcm_dpo/q_t": 0.41382551193237305, "grad_norm": 27.376548767089844, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -0.42573854327201843, "logits/rejected": -0.43815886974334717, "logps/chosen": -426.87237548828125, "logps/ref_chosen": -50.39148712158203, "logps/ref_rejected": -93.71589660644531, "logps/rejected": -721.23388671875, "loss": 1.1202, "margin_dpo/margin_mean": 251.037109375, "margin_dpo/margin_std": 367.67462158203125, "step": 383 }, { "KL/chosen_KL_mean": -368.24322509765625, "KL/mean": -472.5242004394531, "KL/rejected_KL_mean": -576.8052368164062, "KL/std": 268.2153015136719, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 208.56201171875, "fcm_dpo/q_t": 0.4268280863761902, "grad_norm": 20.0333309173584, "learning_rate": 2.397362428170992e-07, "logits/chosen": -0.4748450815677643, "logits/rejected": -0.4683492183685303, "logps/chosen": -420.289306640625, "logps/ref_chosen": -52.046104431152344, "logps/ref_rejected": -85.76089477539062, "logps/rejected": -662.566162109375, "loss": 1.1363, "margin_dpo/margin_mean": 208.56201171875, "margin_dpo/margin_std": 250.0782470703125, "step": 384 }, { "KL/chosen_KL_mean": -347.3446044921875, "KL/mean": -467.85711669921875, "KL/rejected_KL_mean": -588.3695678710938, "KL/std": 225.39920043945312, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 241.02496337890625, "fcm_dpo/q_t": 0.4153004288673401, "grad_norm": 24.630815505981445, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -0.4682900905609131, "logits/rejected": -0.4416571855545044, "logps/chosen": -412.89678955078125, "logps/ref_chosen": -65.55215454101562, "logps/ref_rejected": -77.82792663574219, "logps/rejected": -666.197509765625, "loss": 1.0957, "margin_dpo/margin_mean": 241.02496337890625, "margin_dpo/margin_std": 247.9585418701172, "step": 385 }, { "KL/chosen_KL_mean": -364.7396240234375, "KL/mean": -497.43408203125, "KL/rejected_KL_mean": -630.1285400390625, "KL/std": 291.9441833496094, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 265.3889465332031, "fcm_dpo/q_t": 0.4097556471824646, "grad_norm": 26.513629913330078, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -0.4592726230621338, "logits/rejected": -0.4541982114315033, "logps/chosen": -422.96148681640625, "logps/ref_chosen": -58.22185516357422, "logps/ref_rejected": -92.32742309570312, "logps/rejected": -722.4559936523438, "loss": 1.0928, "margin_dpo/margin_mean": 265.38897705078125, "margin_dpo/margin_std": 341.11126708984375, "step": 386 }, { "KL/chosen_KL_mean": -377.314453125, "KL/mean": -486.5088195800781, "KL/rejected_KL_mean": -595.703125, "KL/std": 265.19122314453125, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0014666462084278464, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 218.38868713378906, "fcm_dpo/q_t": 0.4235340654850006, "grad_norm": 28.172088623046875, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -0.4886634945869446, "logits/rejected": -0.46841585636138916, "logps/chosen": -443.7339172363281, "logps/ref_chosen": -66.41944885253906, "logps/ref_rejected": -92.16915893554688, "logps/rejected": -687.872314453125, "loss": 1.1308, "margin_dpo/margin_mean": 218.388671875, "margin_dpo/margin_std": 269.91070556640625, "step": 387 }, { "KL/chosen_KL_mean": -362.4182434082031, "KL/mean": -510.37042236328125, "KL/rejected_KL_mean": -658.3226318359375, "KL/std": 306.993896484375, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.0014555505476891994, "fcm_dpo/delta": -0.037970419973134995, "fcm_dpo/margin": 295.90435791015625, "fcm_dpo/q_t": 0.4009212851524353, "grad_norm": 24.37848663330078, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -0.45971643924713135, "logits/rejected": -0.4707677662372589, "logps/chosen": -412.5476989746094, "logps/ref_chosen": -50.129459381103516, "logps/ref_rejected": -104.43305969238281, "logps/rejected": -762.7556762695312, "loss": 1.0598, "margin_dpo/margin_mean": 295.90435791015625, "margin_dpo/margin_std": 346.856201171875, "step": 388 }, { "KL/chosen_KL_mean": -401.6351318359375, "KL/mean": -524.97998046875, "KL/rejected_KL_mean": -648.3248291015625, "KL/std": 304.2716064453125, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.0014555505476891994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 246.6896514892578, "fcm_dpo/q_t": 0.416399747133255, "grad_norm": 29.7141170501709, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -0.4501149654388428, "logits/rejected": -0.4422289729118347, "logps/chosen": -459.541748046875, "logps/ref_chosen": -57.906593322753906, "logps/ref_rejected": -77.91454315185547, "logps/rejected": -726.2393798828125, "loss": 1.1164, "margin_dpo/margin_mean": 246.68966674804688, "margin_dpo/margin_std": 338.0137939453125, "step": 389 }, { "KL/chosen_KL_mean": -388.4798278808594, "KL/mean": -514.3911743164062, "KL/rejected_KL_mean": -640.302490234375, "KL/std": 296.8943786621094, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.0014555505476891994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 251.8226776123047, "fcm_dpo/q_t": 0.4174543619155884, "grad_norm": 22.752588272094727, "learning_rate": 2.320488370051681e-07, "logits/chosen": -0.42283350229263306, "logits/rejected": -0.4163636565208435, "logps/chosen": -437.70574951171875, "logps/ref_chosen": -49.22591781616211, "logps/ref_rejected": -85.5281982421875, "logps/rejected": -725.8306884765625, "loss": 1.1234, "margin_dpo/margin_mean": 251.8226776123047, "margin_dpo/margin_std": 379.67108154296875, "step": 390 }, { "KL/chosen_KL_mean": -398.8809814453125, "KL/mean": -467.99664306640625, "KL/rejected_KL_mean": -537.1123657226562, "KL/std": 290.8458251953125, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.0014555505476891994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.23133850097656, "fcm_dpo/q_t": 0.45229679346084595, "grad_norm": 38.93089294433594, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -0.44258245825767517, "logits/rejected": -0.4322088956832886, "logps/chosen": -463.21063232421875, "logps/ref_chosen": -64.32965087890625, "logps/ref_rejected": -86.73820495605469, "logps/rejected": -623.8505859375, "loss": 1.2447, "margin_dpo/margin_mean": 138.23133850097656, "margin_dpo/margin_std": 306.81280517578125, "step": 391 }, { "KL/chosen_KL_mean": -327.77587890625, "KL/mean": -475.56134033203125, "KL/rejected_KL_mean": -623.3468017578125, "KL/std": 294.4037780761719, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0014487283769994974, "fcm_dpo/delta": -0.047090742737054825, "fcm_dpo/margin": 295.57098388671875, "fcm_dpo/q_t": 0.39948129653930664, "grad_norm": 26.698942184448242, "learning_rate": 2.294897926507156e-07, "logits/chosen": -0.44229134917259216, "logits/rejected": -0.4364994168281555, "logps/chosen": -381.27984619140625, "logps/ref_chosen": -53.50397872924805, "logps/ref_rejected": -102.34584045410156, "logps/rejected": -725.692626953125, "loss": 1.0466, "margin_dpo/margin_mean": 295.57098388671875, "margin_dpo/margin_std": 292.6898498535156, "step": 392 }, { "KL/chosen_KL_mean": -326.3807067871094, "KL/mean": -437.4690246582031, "KL/rejected_KL_mean": -548.557373046875, "KL/std": 287.9085388183594, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0014419063227251172, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 222.1766357421875, "fcm_dpo/q_t": 0.4272076487541199, "grad_norm": 22.053794860839844, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -0.3985256254673004, "logits/rejected": -0.395794153213501, "logps/chosen": -372.8546142578125, "logps/ref_chosen": -46.473915100097656, "logps/ref_rejected": -71.96885681152344, "logps/rejected": -620.5261840820312, "loss": 1.1533, "margin_dpo/margin_mean": 222.17662048339844, "margin_dpo/margin_std": 362.2658996582031, "step": 393 }, { "KL/chosen_KL_mean": -371.293212890625, "KL/mean": -487.92718505859375, "KL/rejected_KL_mean": -604.5611572265625, "KL/std": 307.582275390625, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.0014419063227251172, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.26791381835938, "fcm_dpo/q_t": 0.42159971594810486, "grad_norm": 20.481840133666992, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -0.4049949645996094, "logits/rejected": -0.40527600049972534, "logps/chosen": -424.20477294921875, "logps/ref_chosen": -52.91154861450195, "logps/ref_rejected": -90.8226318359375, "logps/rejected": -695.3837890625, "loss": 1.1278, "margin_dpo/margin_mean": 233.26791381835938, "margin_dpo/margin_std": 318.6451416015625, "step": 394 }, { "KL/chosen_KL_mean": -358.0785827636719, "KL/mean": -485.55322265625, "KL/rejected_KL_mean": -613.0279541015625, "KL/std": 292.2204284667969, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.0014419063227251172, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 254.94932556152344, "fcm_dpo/q_t": 0.4145504832267761, "grad_norm": 21.921274185180664, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -0.4451723098754883, "logits/rejected": -0.44253167510032654, "logps/chosen": -420.62469482421875, "logps/ref_chosen": -62.546112060546875, "logps/ref_rejected": -83.78262329101562, "logps/rejected": -696.810546875, "loss": 1.1073, "margin_dpo/margin_mean": 254.9493408203125, "margin_dpo/margin_std": 334.0831298828125, "step": 395 }, { "KL/chosen_KL_mean": -358.3993835449219, "KL/mean": -474.07427978515625, "KL/rejected_KL_mean": -589.7491455078125, "KL/std": 286.84027099609375, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.0014419063227251172, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 231.34983825683594, "fcm_dpo/q_t": 0.42063820362091064, "grad_norm": 19.1959285736084, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -0.4682362973690033, "logits/rejected": -0.4516993761062622, "logps/chosen": -427.39532470703125, "logps/ref_chosen": -68.99594116210938, "logps/ref_rejected": -88.64665985107422, "logps/rejected": -678.3958740234375, "loss": 1.1148, "margin_dpo/margin_mean": 231.34982299804688, "margin_dpo/margin_std": 264.4104919433594, "step": 396 }, { "KL/chosen_KL_mean": -338.35693359375, "KL/mean": -482.5387268066406, "KL/rejected_KL_mean": -626.7205810546875, "KL/std": 283.781494140625, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.001432217308320105, "fcm_dpo/delta": -0.03371122106909752, "fcm_dpo/margin": 288.3636169433594, "fcm_dpo/q_t": 0.40260809659957886, "grad_norm": 33.34209060668945, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -0.388605535030365, "logits/rejected": -0.3912005126476288, "logps/chosen": -399.63409423828125, "logps/ref_chosen": -61.27716827392578, "logps/ref_rejected": -103.11612701416016, "logps/rejected": -729.836669921875, "loss": 1.0596, "margin_dpo/margin_mean": 288.3636169433594, "margin_dpo/margin_std": 301.2374572753906, "step": 397 }, { "KL/chosen_KL_mean": -358.54351806640625, "KL/mean": -494.4063415527344, "KL/rejected_KL_mean": -630.2691650390625, "KL/std": 298.5025634765625, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.001432217308320105, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 271.7256164550781, "fcm_dpo/q_t": 0.4102500081062317, "grad_norm": 19.53269386291504, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -0.4787027835845947, "logits/rejected": -0.47340676188468933, "logps/chosen": -426.695068359375, "logps/ref_chosen": -68.15155029296875, "logps/ref_rejected": -108.52360534667969, "logps/rejected": -738.792724609375, "loss": 1.0924, "margin_dpo/margin_mean": 271.7255859375, "margin_dpo/margin_std": 350.1674499511719, "step": 398 }, { "KL/chosen_KL_mean": -310.72174072265625, "KL/mean": -426.0609436035156, "KL/rejected_KL_mean": -541.400146484375, "KL/std": 257.1795349121094, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.001432217308320105, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 230.67837524414062, "fcm_dpo/q_t": 0.42226526141166687, "grad_norm": 22.05406379699707, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -0.43538278341293335, "logits/rejected": -0.4008658528327942, "logps/chosen": -371.611572265625, "logps/ref_chosen": -60.889801025390625, "logps/ref_rejected": -77.965576171875, "logps/rejected": -619.36572265625, "loss": 1.1347, "margin_dpo/margin_mean": 230.67835998535156, "margin_dpo/margin_std": 322.523193359375, "step": 399 }, { "KL/chosen_KL_mean": -292.40625, "KL/mean": -462.7926025390625, "KL/rejected_KL_mean": -633.178955078125, "KL/std": 276.2427062988281, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0014116978272795677, "fcm_dpo/delta": -0.0853746086359024, "fcm_dpo/margin": 340.772705078125, "fcm_dpo/q_t": 0.38592517375946045, "grad_norm": 21.224273681640625, "learning_rate": 2.192779273338215e-07, "logits/chosen": -0.3986595571041107, "logits/rejected": -0.39339679479599, "logps/chosen": -356.0498046875, "logps/ref_chosen": -63.64359664916992, "logps/ref_rejected": -105.252685546875, "logps/rejected": -738.431640625, "loss": 1.0063, "margin_dpo/margin_mean": 340.772705078125, "margin_dpo/margin_std": 307.47308349609375, "step": 400 }, { "KL/chosen_KL_mean": -346.3468322753906, "KL/mean": -440.0452880859375, "KL/rejected_KL_mean": -533.7437744140625, "KL/std": 287.55645751953125, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0014079699758440256, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 187.39691162109375, "fcm_dpo/q_t": 0.4391755163669586, "grad_norm": 19.834428787231445, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -0.4189956784248352, "logits/rejected": -0.41130340099334717, "logps/chosen": -403.5098876953125, "logps/ref_chosen": -57.16303253173828, "logps/ref_rejected": -83.79249572753906, "logps/rejected": -617.5362548828125, "loss": 1.2105, "margin_dpo/margin_mean": 187.39694213867188, "margin_dpo/margin_std": 384.96673583984375, "step": 401 }, { "KL/chosen_KL_mean": -254.55259704589844, "KL/mean": -429.1209716796875, "KL/rejected_KL_mean": -603.6893310546875, "KL/std": 311.1974182128906, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0013890372356399894, "fcm_dpo/delta": -0.08931900560855865, "fcm_dpo/margin": 349.13671875, "fcm_dpo/q_t": 0.38594573736190796, "grad_norm": 18.52750015258789, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -0.38897186517715454, "logits/rejected": -0.3770354986190796, "logps/chosen": -305.29296875, "logps/ref_chosen": -50.74037170410156, "logps/ref_rejected": -81.0460433959961, "logps/rejected": -684.7353515625, "loss": 1.0045, "margin_dpo/margin_mean": 349.13671875, "margin_dpo/margin_std": 321.05426025390625, "step": 402 }, { "KL/chosen_KL_mean": -277.5030822753906, "KL/mean": -412.81787109375, "KL/rejected_KL_mean": -548.1326293945312, "KL/std": 296.6151123046875, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0013830416137352586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 270.62957763671875, "fcm_dpo/q_t": 0.4117102026939392, "grad_norm": 22.302377700805664, "learning_rate": 2.154609112620295e-07, "logits/chosen": -0.37483614683151245, "logits/rejected": -0.36954933404922485, "logps/chosen": -324.650390625, "logps/ref_chosen": -47.14731216430664, "logps/ref_rejected": -77.2666015625, "logps/rejected": -625.3992919921875, "loss": 1.0885, "margin_dpo/margin_mean": 270.62957763671875, "margin_dpo/margin_std": 300.69293212890625, "step": 403 }, { "KL/chosen_KL_mean": -307.9491882324219, "KL/mean": -442.93231201171875, "KL/rejected_KL_mean": -577.9154663085938, "KL/std": 279.23492431640625, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0013830416137352586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 269.96630859375, "fcm_dpo/q_t": 0.412688672542572, "grad_norm": 18.491226196289062, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -0.35618770122528076, "logits/rejected": -0.3467029333114624, "logps/chosen": -355.824462890625, "logps/ref_chosen": -47.875274658203125, "logps/ref_rejected": -77.15499877929688, "logps/rejected": -655.0704345703125, "loss": 1.1043, "margin_dpo/margin_mean": 269.96630859375, "margin_dpo/margin_std": 352.7061462402344, "step": 404 }, { "KL/chosen_KL_mean": -365.7634582519531, "KL/mean": -475.2872619628906, "KL/rejected_KL_mean": -584.81103515625, "KL/std": 317.9582824707031, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0013830416137352586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 219.04754638671875, "fcm_dpo/q_t": 0.43119341135025024, "grad_norm": 26.632732391357422, "learning_rate": 2.129207286861638e-07, "logits/chosen": -0.34994158148765564, "logits/rejected": -0.34071725606918335, "logps/chosen": -430.9263610839844, "logps/ref_chosen": -65.16290283203125, "logps/ref_rejected": -87.18678283691406, "logps/rejected": -671.997802734375, "loss": 1.1815, "margin_dpo/margin_mean": 219.04754638671875, "margin_dpo/margin_std": 398.8152770996094, "step": 405 }, { "KL/chosen_KL_mean": -336.44659423828125, "KL/mean": -477.87347412109375, "KL/rejected_KL_mean": -619.3004150390625, "KL/std": 311.3196716308594, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0013787212083116174, "fcm_dpo/delta": -0.031336523592472076, "fcm_dpo/margin": 282.85382080078125, "fcm_dpo/q_t": 0.4096255302429199, "grad_norm": 21.68229866027832, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -0.3854570984840393, "logits/rejected": -0.389883816242218, "logps/chosen": -386.1874084472656, "logps/ref_chosen": -49.740814208984375, "logps/ref_rejected": -92.07862854003906, "logps/rejected": -711.3790283203125, "loss": 1.0887, "margin_dpo/margin_mean": 282.85382080078125, "margin_dpo/margin_std": 349.2008361816406, "step": 406 }, { "KL/chosen_KL_mean": -359.400390625, "KL/mean": -452.02783203125, "KL/rejected_KL_mean": -544.6552734375, "KL/std": 254.18994140625, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0013744008028879762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 185.25485229492188, "fcm_dpo/q_t": 0.439433753490448, "grad_norm": 27.686962127685547, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -0.3911685347557068, "logits/rejected": -0.3647308945655823, "logps/chosen": -415.7311096191406, "logps/ref_chosen": -56.33069610595703, "logps/ref_rejected": -77.51209259033203, "logps/rejected": -622.1673583984375, "loss": 1.2016, "margin_dpo/margin_mean": 185.25485229492188, "margin_dpo/margin_std": 333.753662109375, "step": 407 }, { "KL/chosen_KL_mean": -372.42059326171875, "KL/mean": -469.0271911621094, "KL/rejected_KL_mean": -565.6337890625, "KL/std": 247.71975708007812, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0013744008028879762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 193.213134765625, "fcm_dpo/q_t": 0.43635261058807373, "grad_norm": 23.493730545043945, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -0.40377092361450195, "logits/rejected": -0.37954843044281006, "logps/chosen": -442.20989990234375, "logps/ref_chosen": -69.789306640625, "logps/ref_rejected": -90.09693908691406, "logps/rejected": -655.730712890625, "loss": 1.1722, "margin_dpo/margin_mean": 193.21316528320312, "margin_dpo/margin_std": 271.8493347167969, "step": 408 }, { "KL/chosen_KL_mean": -360.97454833984375, "KL/mean": -456.0081787109375, "KL/rejected_KL_mean": -551.041748046875, "KL/std": 256.92303466796875, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0013744008028879762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 190.06727600097656, "fcm_dpo/q_t": 0.43794065713882446, "grad_norm": 24.473705291748047, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -0.3902217745780945, "logits/rejected": -0.3765072524547577, "logps/chosen": -428.2919921875, "logps/ref_chosen": -67.31744384765625, "logps/ref_rejected": -84.904296875, "logps/rejected": -635.946044921875, "loss": 1.185, "margin_dpo/margin_mean": 190.06727600097656, "margin_dpo/margin_std": 304.51153564453125, "step": 409 }, { "KL/chosen_KL_mean": -328.8135986328125, "KL/mean": -451.9912109375, "KL/rejected_KL_mean": -575.1687622070312, "KL/std": 273.52239990234375, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0013744008028879762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 246.3551025390625, "fcm_dpo/q_t": 0.4192972481250763, "grad_norm": 23.798898696899414, "learning_rate": 2.065879555832674e-07, "logits/chosen": -0.3959043622016907, "logits/rejected": -0.394477903842926, "logps/chosen": -380.27899169921875, "logps/ref_chosen": -51.465354919433594, "logps/ref_rejected": -83.198974609375, "logps/rejected": -658.3677368164062, "loss": 1.1192, "margin_dpo/margin_mean": 246.3551025390625, "margin_dpo/margin_std": 308.258544921875, "step": 410 }, { "KL/chosen_KL_mean": -354.54156494140625, "KL/mean": -472.2734680175781, "KL/rejected_KL_mean": -590.00537109375, "KL/std": 299.3707580566406, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0013673783978447318, "fcm_dpo/delta": -0.025612497702240944, "fcm_dpo/margin": 235.46383666992188, "fcm_dpo/q_t": 0.42583271861076355, "grad_norm": 23.410425186157227, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -0.3593885600566864, "logits/rejected": -0.3671804666519165, "logps/chosen": -406.84881591796875, "logps/ref_chosen": -52.30727005004883, "logps/ref_rejected": -80.69495391845703, "logps/rejected": -670.7003173828125, "loss": 1.1471, "margin_dpo/margin_mean": 235.46383666992188, "margin_dpo/margin_std": 350.5140686035156, "step": 411 }, { "KL/chosen_KL_mean": -356.0343017578125, "KL/mean": -475.4249267578125, "KL/rejected_KL_mean": -594.8155517578125, "KL/std": 302.0730285644531, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0013673783978447318, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 238.78121948242188, "fcm_dpo/q_t": 0.4237388074398041, "grad_norm": 26.060949325561523, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -0.4340513348579407, "logits/rejected": -0.4596172571182251, "logps/chosen": -409.1784362792969, "logps/ref_chosen": -53.144126892089844, "logps/ref_rejected": -100.0608139038086, "logps/rejected": -694.8763427734375, "loss": 1.143, "margin_dpo/margin_mean": 238.78121948242188, "margin_dpo/margin_std": 358.81866455078125, "step": 412 }, { "KL/chosen_KL_mean": -380.462158203125, "KL/mean": -503.0280456542969, "KL/rejected_KL_mean": -625.5939331054688, "KL/std": 281.5146484375, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0013673783978447318, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 245.13180541992188, "fcm_dpo/q_t": 0.4199550151824951, "grad_norm": 21.529010772705078, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -0.4233962595462799, "logits/rejected": -0.42641475796699524, "logps/chosen": -442.04412841796875, "logps/ref_chosen": -61.58196258544922, "logps/ref_rejected": -99.47340393066406, "logps/rejected": -725.0673828125, "loss": 1.1257, "margin_dpo/margin_mean": 245.13180541992188, "margin_dpo/margin_std": 321.50811767578125, "step": 413 }, { "KL/chosen_KL_mean": -334.89654541015625, "KL/mean": -471.7811279296875, "KL/rejected_KL_mean": -608.6657104492188, "KL/std": 274.99615478515625, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.001360746449790895, "fcm_dpo/delta": -0.048737846314907074, "fcm_dpo/margin": 273.7691650390625, "fcm_dpo/q_t": 0.411694198846817, "grad_norm": 25.56825065612793, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -0.3538694977760315, "logits/rejected": -0.34508466720581055, "logps/chosen": -381.52801513671875, "logps/ref_chosen": -46.63148498535156, "logps/ref_rejected": -87.64653015136719, "logps/rejected": -696.312255859375, "loss": 1.0954, "margin_dpo/margin_mean": 273.7691650390625, "margin_dpo/margin_std": 315.960205078125, "step": 414 }, { "KL/chosen_KL_mean": -381.809814453125, "KL/mean": -484.69952392578125, "KL/rejected_KL_mean": -587.5892333984375, "KL/std": 283.8314208984375, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0013541145017370582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 205.77943420410156, "fcm_dpo/q_t": 0.4336971342563629, "grad_norm": 24.777685165405273, "learning_rate": 2.002837796667909e-07, "logits/chosen": -0.4540392756462097, "logits/rejected": -0.4499126672744751, "logps/chosen": -460.4281005859375, "logps/ref_chosen": -78.6182861328125, "logps/ref_rejected": -100.47752380371094, "logps/rejected": -688.0667724609375, "loss": 1.1731, "margin_dpo/margin_mean": 205.77943420410156, "margin_dpo/margin_std": 321.399658203125, "step": 415 }, { "KL/chosen_KL_mean": -339.66680908203125, "KL/mean": -497.25067138671875, "KL/rejected_KL_mean": -654.83447265625, "KL/std": 297.5770263671875, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0013482770882546902, "fcm_dpo/delta": -0.04329550638794899, "fcm_dpo/margin": 315.16766357421875, "fcm_dpo/q_t": 0.39873379468917847, "grad_norm": 23.55436897277832, "learning_rate": 1.990267419549914e-07, "logits/chosen": -0.47019827365875244, "logits/rejected": -0.4745001792907715, "logps/chosen": -397.9459533691406, "logps/ref_chosen": -58.27912521362305, "logps/ref_rejected": -90.56871795654297, "logps/rejected": -745.4031982421875, "loss": 1.0405, "margin_dpo/margin_mean": 315.16766357421875, "margin_dpo/margin_std": 279.1081237792969, "step": 416 }, { "KL/chosen_KL_mean": -337.95068359375, "KL/mean": -468.36810302734375, "KL/rejected_KL_mean": -598.7855834960938, "KL/std": 279.1325378417969, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0013424396747723222, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 260.8348388671875, "fcm_dpo/q_t": 0.41633397340774536, "grad_norm": 25.135456085205078, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -0.43549424409866333, "logits/rejected": -0.4124263525009155, "logps/chosen": -388.1494140625, "logps/ref_chosen": -50.1987190246582, "logps/ref_rejected": -68.15184020996094, "logps/rejected": -666.9373779296875, "loss": 1.0988, "margin_dpo/margin_mean": 260.8348693847656, "margin_dpo/margin_std": 273.3210144042969, "step": 417 }, { "KL/chosen_KL_mean": -365.82684326171875, "KL/mean": -508.16168212890625, "KL/rejected_KL_mean": -650.49658203125, "KL/std": 315.8503112792969, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0013350698864087462, "fcm_dpo/delta": -0.05520197004079819, "fcm_dpo/margin": 284.669677734375, "fcm_dpo/q_t": 0.4123033881187439, "grad_norm": 18.67411994934082, "learning_rate": 1.965167291983757e-07, "logits/chosen": -0.5395079851150513, "logits/rejected": -0.5210288763046265, "logps/chosen": -447.8052978515625, "logps/ref_chosen": -81.97846984863281, "logps/ref_rejected": -104.69148254394531, "logps/rejected": -755.18798828125, "loss": 1.1026, "margin_dpo/margin_mean": 284.669677734375, "margin_dpo/margin_std": 370.0165100097656, "step": 418 }, { "KL/chosen_KL_mean": -346.25262451171875, "KL/mean": -497.578369140625, "KL/rejected_KL_mean": -648.904052734375, "KL/std": 279.85736083984375, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 302.6514587402344, "fcm_dpo/q_t": 0.40497830510139465, "grad_norm": 21.298076629638672, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -0.4583272933959961, "logits/rejected": -0.4552687406539917, "logps/chosen": -399.2012634277344, "logps/ref_chosen": -52.948646545410156, "logps/ref_rejected": -91.58309936523438, "logps/rejected": -740.4871826171875, "loss": 1.068, "margin_dpo/margin_mean": 302.6514892578125, "margin_dpo/margin_std": 322.56439208984375, "step": 419 }, { "KL/chosen_KL_mean": -457.07464599609375, "KL/mean": -541.6314697265625, "KL/rejected_KL_mean": -626.1882934570312, "KL/std": 307.27313232421875, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 169.1136474609375, "fcm_dpo/q_t": 0.4484458863735199, "grad_norm": 47.015316009521484, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -0.49456986784935, "logits/rejected": -0.45519959926605225, "logps/chosen": -534.8446044921875, "logps/ref_chosen": -77.7699203491211, "logps/ref_rejected": -69.31985473632812, "logps/rejected": -695.5081787109375, "loss": 1.2491, "margin_dpo/margin_mean": 169.11366271972656, "margin_dpo/margin_std": 422.0369873046875, "step": 420 }, { "KL/chosen_KL_mean": -364.79150390625, "KL/mean": -468.5289306640625, "KL/rejected_KL_mean": -572.2662963867188, "KL/std": 288.0106201171875, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 207.47483825683594, "fcm_dpo/q_t": 0.43372586369514465, "grad_norm": 20.53417205810547, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -0.5256436467170715, "logits/rejected": -0.5119495391845703, "logps/chosen": -418.557373046875, "logps/ref_chosen": -53.765865325927734, "logps/ref_rejected": -89.28144836425781, "logps/rejected": -661.5477294921875, "loss": 1.1738, "margin_dpo/margin_mean": 207.47483825683594, "margin_dpo/margin_std": 313.608642578125, "step": 421 }, { "KL/chosen_KL_mean": -423.76123046875, "KL/mean": -540.0737915039062, "KL/rejected_KL_mean": -656.3863525390625, "KL/std": 297.48114013671875, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 232.6251220703125, "fcm_dpo/q_t": 0.4267650246620178, "grad_norm": 33.49336242675781, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -0.5396426320075989, "logits/rejected": -0.538498044013977, "logps/chosen": -492.39501953125, "logps/ref_chosen": -68.6337661743164, "logps/ref_rejected": -87.86351013183594, "logps/rejected": -744.2498779296875, "loss": 1.1543, "margin_dpo/margin_mean": 232.6251220703125, "margin_dpo/margin_std": 355.2101135253906, "step": 422 }, { "KL/chosen_KL_mean": -390.7974853515625, "KL/mean": -527.0419311523438, "KL/rejected_KL_mean": -663.286376953125, "KL/std": 293.29345703125, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 272.4888916015625, "fcm_dpo/q_t": 0.41438615322113037, "grad_norm": 27.167430877685547, "learning_rate": 1.902669377503756e-07, "logits/chosen": -0.5402317047119141, "logits/rejected": -0.5469857454299927, "logps/chosen": -445.78778076171875, "logps/ref_chosen": -54.99030303955078, "logps/ref_rejected": -86.30654907226562, "logps/rejected": -749.5929565429688, "loss": 1.0977, "margin_dpo/margin_mean": 272.4888916015625, "margin_dpo/margin_std": 312.23480224609375, "step": 423 }, { "KL/chosen_KL_mean": -360.069580078125, "KL/mean": -478.40814208984375, "KL/rejected_KL_mean": -596.7467041015625, "KL/std": 285.7010192871094, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 236.6771697998047, "fcm_dpo/q_t": 0.42635974287986755, "grad_norm": 26.857175827026367, "learning_rate": 1.890215699729057e-07, "logits/chosen": -0.5228101015090942, "logits/rejected": -0.49360746145248413, "logps/chosen": -416.08148193359375, "logps/ref_chosen": -56.01192092895508, "logps/ref_rejected": -66.47896575927734, "logps/rejected": -663.2257080078125, "loss": 1.1475, "margin_dpo/margin_mean": 236.6771697998047, "margin_dpo/margin_std": 346.0706481933594, "step": 424 }, { "KL/chosen_KL_mean": -377.1744079589844, "KL/mean": -482.8753662109375, "KL/rejected_KL_mean": -588.5763549804688, "KL/std": 269.3935241699219, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 211.40194702148438, "fcm_dpo/q_t": 0.4325962960720062, "grad_norm": 26.28618812561035, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -0.565961480140686, "logits/rejected": -0.5686656832695007, "logps/chosen": -424.04339599609375, "logps/ref_chosen": -46.86899948120117, "logps/ref_rejected": -95.92545318603516, "logps/rejected": -684.5018310546875, "loss": 1.1705, "margin_dpo/margin_mean": 211.40194702148438, "margin_dpo/margin_std": 325.16778564453125, "step": 425 }, { "KL/chosen_KL_mean": -362.37982177734375, "KL/mean": -483.747802734375, "KL/rejected_KL_mean": -605.11572265625, "KL/std": 271.43121337890625, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 242.7359161376953, "fcm_dpo/q_t": 0.42305266857147217, "grad_norm": 23.226341247558594, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -0.5489311814308167, "logits/rejected": -0.5112883448600769, "logps/chosen": -438.96337890625, "logps/ref_chosen": -76.58354187011719, "logps/ref_rejected": -81.26658630371094, "logps/rejected": -686.38232421875, "loss": 1.1309, "margin_dpo/margin_mean": 242.7359161376953, "margin_dpo/margin_std": 302.89013671875, "step": 426 }, { "KL/chosen_KL_mean": -354.0020446777344, "KL/mean": -441.369140625, "KL/rejected_KL_mean": -528.736328125, "KL/std": 247.91751098632812, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 174.73423767089844, "fcm_dpo/q_t": 0.4447828233242035, "grad_norm": 20.944759368896484, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -0.5702467560768127, "logits/rejected": -0.5489529967308044, "logps/chosen": -418.85589599609375, "logps/ref_chosen": -64.8538818359375, "logps/ref_rejected": -78.5660171508789, "logps/rejected": -607.3023071289062, "loss": 1.2039, "margin_dpo/margin_mean": 174.73423767089844, "margin_dpo/margin_std": 289.51605224609375, "step": 427 }, { "KL/chosen_KL_mean": -424.8520202636719, "KL/mean": -555.02294921875, "KL/rejected_KL_mean": -685.19384765625, "KL/std": 331.234130859375, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 260.3418273925781, "fcm_dpo/q_t": 0.41999179124832153, "grad_norm": 27.35421371459961, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -0.557364284992218, "logits/rejected": -0.5577110648155212, "logps/chosen": -487.48870849609375, "logps/ref_chosen": -62.63666534423828, "logps/ref_rejected": -103.28181457519531, "logps/rejected": -788.4757080078125, "loss": 1.137, "margin_dpo/margin_mean": 260.3418273925781, "margin_dpo/margin_std": 400.5129699707031, "step": 428 }, { "KL/chosen_KL_mean": -433.4362487792969, "KL/mean": -528.7349853515625, "KL/rejected_KL_mean": -624.03369140625, "KL/std": 286.8631286621094, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 190.597412109375, "fcm_dpo/q_t": 0.44049394130706787, "grad_norm": 28.649442672729492, "learning_rate": 1.828194884925749e-07, "logits/chosen": -0.5428134202957153, "logits/rejected": -0.5149238705635071, "logps/chosen": -514.6702880859375, "logps/ref_chosen": -81.23401641845703, "logps/ref_rejected": -91.79493713378906, "logps/rejected": -715.82861328125, "loss": 1.2062, "margin_dpo/margin_mean": 190.597412109375, "margin_dpo/margin_std": 351.88348388671875, "step": 429 }, { "KL/chosen_KL_mean": -341.23309326171875, "KL/mean": -457.59912109375, "KL/rejected_KL_mean": -573.965087890625, "KL/std": 271.2836608886719, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 232.73202514648438, "fcm_dpo/q_t": 0.4272611141204834, "grad_norm": 21.780847549438477, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -0.5611949563026428, "logits/rejected": -0.5625407695770264, "logps/chosen": -402.1534423828125, "logps/ref_chosen": -60.920326232910156, "logps/ref_rejected": -104.42280578613281, "logps/rejected": -678.387939453125, "loss": 1.14, "margin_dpo/margin_mean": 232.73202514648438, "margin_dpo/margin_std": 307.2655334472656, "step": 430 }, { "KL/chosen_KL_mean": -343.27020263671875, "KL/mean": -497.19476318359375, "KL/rejected_KL_mean": -651.1193237304688, "KL/std": 300.940185546875, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0013277000980451703, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 307.84918212890625, "fcm_dpo/q_t": 0.4039904475212097, "grad_norm": 20.312416076660156, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -0.49830469489097595, "logits/rejected": -0.4969218075275421, "logps/chosen": -400.61895751953125, "logps/ref_chosen": -57.34874725341797, "logps/ref_rejected": -92.84022521972656, "logps/rejected": -743.9595947265625, "loss": 1.0662, "margin_dpo/margin_mean": 307.8491516113281, "margin_dpo/margin_std": 330.0069885253906, "step": 431 }, { "KL/chosen_KL_mean": -338.7581787109375, "KL/mean": -483.9551086425781, "KL/rejected_KL_mean": -629.152099609375, "KL/std": 284.55670166015625, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0013210558099672198, "fcm_dpo/delta": -0.05029592290520668, "fcm_dpo/margin": 290.3939208984375, "fcm_dpo/q_t": 0.40896129608154297, "grad_norm": 28.03597068786621, "learning_rate": 1.791192214186223e-07, "logits/chosen": -0.5391901135444641, "logits/rejected": -0.5290787220001221, "logps/chosen": -409.8329772949219, "logps/ref_chosen": -71.07479095458984, "logps/ref_rejected": -98.57952880859375, "logps/rejected": -727.7316284179688, "loss": 1.0785, "margin_dpo/margin_mean": 290.3939208984375, "margin_dpo/margin_std": 291.03668212890625, "step": 432 }, { "KL/chosen_KL_mean": -428.12890625, "KL/mean": -531.45703125, "KL/rejected_KL_mean": -634.7850341796875, "KL/std": 282.63128662109375, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 206.6561737060547, "fcm_dpo/q_t": 0.4347584843635559, "grad_norm": 22.199871063232422, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -0.5286056995391846, "logits/rejected": -0.514957845211029, "logps/chosen": -486.402099609375, "logps/ref_chosen": -58.273193359375, "logps/ref_rejected": -95.95089721679688, "logps/rejected": -730.7359619140625, "loss": 1.1798, "margin_dpo/margin_mean": 206.65615844726562, "margin_dpo/margin_std": 333.2256774902344, "step": 433 }, { "KL/chosen_KL_mean": -372.42156982421875, "KL/mean": -489.6736145019531, "KL/rejected_KL_mean": -606.9257202148438, "KL/std": 296.7366027832031, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 234.50413513183594, "fcm_dpo/q_t": 0.430846631526947, "grad_norm": 20.755958557128906, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -0.5101474523544312, "logits/rejected": -0.5049848556518555, "logps/chosen": -434.395263671875, "logps/ref_chosen": -61.97370147705078, "logps/ref_rejected": -78.49861145019531, "logps/rejected": -685.42431640625, "loss": 1.1581, "margin_dpo/margin_mean": 234.50413513183594, "margin_dpo/margin_std": 380.45330810546875, "step": 434 }, { "KL/chosen_KL_mean": -333.2384033203125, "KL/mean": -460.36224365234375, "KL/rejected_KL_mean": -587.486083984375, "KL/std": 281.7438659667969, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 254.24761962890625, "fcm_dpo/q_t": 0.42064914107322693, "grad_norm": 23.37683868408203, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -0.5346446633338928, "logits/rejected": -0.5243590474128723, "logps/chosen": -384.740478515625, "logps/ref_chosen": -51.502052307128906, "logps/ref_rejected": -87.56689453125, "logps/rejected": -675.052978515625, "loss": 1.1226, "margin_dpo/margin_mean": 254.24765014648438, "margin_dpo/margin_std": 314.2124328613281, "step": 435 }, { "KL/chosen_KL_mean": -353.6714782714844, "KL/mean": -460.28839111328125, "KL/rejected_KL_mean": -566.9053344726562, "KL/std": 247.24136352539062, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 213.23385620117188, "fcm_dpo/q_t": 0.43298617005348206, "grad_norm": 21.458030700683594, "learning_rate": 1.742118314717391e-07, "logits/chosen": -0.5430168509483337, "logits/rejected": -0.5158405303955078, "logps/chosen": -425.0751953125, "logps/ref_chosen": -71.40371704101562, "logps/ref_rejected": -82.72775268554688, "logps/rejected": -649.633056640625, "loss": 1.1615, "margin_dpo/margin_mean": 213.23385620117188, "margin_dpo/margin_std": 293.1329345703125, "step": 436 }, { "KL/chosen_KL_mean": -359.79595947265625, "KL/mean": -469.1462707519531, "KL/rejected_KL_mean": -578.49658203125, "KL/std": 235.53416442871094, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 218.7006072998047, "fcm_dpo/q_t": 0.431079626083374, "grad_norm": 21.566225051879883, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -0.5344359874725342, "logits/rejected": -0.5146011114120483, "logps/chosen": -424.54022216796875, "logps/ref_chosen": -64.7442626953125, "logps/ref_rejected": -82.04356384277344, "logps/rejected": -660.5401611328125, "loss": 1.155, "margin_dpo/margin_mean": 218.70062255859375, "margin_dpo/margin_std": 291.8827209472656, "step": 437 }, { "KL/chosen_KL_mean": -366.44976806640625, "KL/mean": -488.88330078125, "KL/rejected_KL_mean": -611.3168334960938, "KL/std": 269.77825927734375, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 244.86703491210938, "fcm_dpo/q_t": 0.42326781153678894, "grad_norm": 24.300569534301758, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -0.500368595123291, "logits/rejected": -0.48205146193504333, "logps/chosen": -425.46844482421875, "logps/ref_chosen": -59.0186653137207, "logps/ref_rejected": -83.07682800292969, "logps/rejected": -694.3936767578125, "loss": 1.1274, "margin_dpo/margin_mean": 244.86703491210938, "margin_dpo/margin_std": 296.9903564453125, "step": 438 }, { "KL/chosen_KL_mean": -368.8246765136719, "KL/mean": -475.66510009765625, "KL/rejected_KL_mean": -582.5054931640625, "KL/std": 270.74139404296875, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 213.68080139160156, "fcm_dpo/q_t": 0.43394535779953003, "grad_norm": 23.145774841308594, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -0.516400933265686, "logits/rejected": -0.5202809572219849, "logps/chosen": -422.6087646484375, "logps/ref_chosen": -53.78407669067383, "logps/ref_rejected": -83.98545837402344, "logps/rejected": -666.490966796875, "loss": 1.1686, "margin_dpo/margin_mean": 213.6807861328125, "margin_dpo/margin_std": 307.72296142578125, "step": 439 }, { "KL/chosen_KL_mean": -393.8909912109375, "KL/mean": -513.2696533203125, "KL/rejected_KL_mean": -632.6483764648438, "KL/std": 314.48468017578125, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 238.75735473632812, "fcm_dpo/q_t": 0.4269280433654785, "grad_norm": 29.134511947631836, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -0.5639290809631348, "logits/rejected": -0.5608119964599609, "logps/chosen": -472.4577331542969, "logps/ref_chosen": -78.56671905517578, "logps/ref_rejected": -96.49775695800781, "logps/rejected": -729.1461181640625, "loss": 1.1428, "margin_dpo/margin_mean": 238.75735473632812, "margin_dpo/margin_std": 335.62493896484375, "step": 440 }, { "KL/chosen_KL_mean": -423.23974609375, "KL/mean": -543.760986328125, "KL/rejected_KL_mean": -664.2822265625, "KL/std": 322.83819580078125, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0013144115218892694, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 241.04251098632812, "fcm_dpo/q_t": 0.4266929030418396, "grad_norm": 26.923349380493164, "learning_rate": 1.681227682404166e-07, "logits/chosen": -0.5657366514205933, "logits/rejected": -0.5528466701507568, "logps/chosen": -484.064208984375, "logps/ref_chosen": -60.824440002441406, "logps/ref_rejected": -96.47080993652344, "logps/rejected": -760.7530517578125, "loss": 1.159, "margin_dpo/margin_mean": 241.04249572753906, "margin_dpo/margin_std": 389.2134704589844, "step": 441 }, { "KL/chosen_KL_mean": -364.3332824707031, "KL/mean": -503.16485595703125, "KL/rejected_KL_mean": -641.9964599609375, "KL/std": 306.99761962890625, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0013098118361085653, "fcm_dpo/delta": -0.03511664643883705, "fcm_dpo/margin": 277.6631164550781, "fcm_dpo/q_t": 0.41515475511550903, "grad_norm": 21.374515533447266, "learning_rate": 1.669113001300851e-07, "logits/chosen": -0.5080809593200684, "logits/rejected": -0.49998366832733154, "logps/chosen": -411.344482421875, "logps/ref_chosen": -47.01121520996094, "logps/ref_rejected": -76.53926086425781, "logps/rejected": -718.53564453125, "loss": 1.1045, "margin_dpo/margin_mean": 277.6631164550781, "margin_dpo/margin_std": 339.3021240234375, "step": 442 }, { "KL/chosen_KL_mean": -419.7547302246094, "KL/mean": -508.86871337890625, "KL/rejected_KL_mean": -597.982666015625, "KL/std": 292.1375427246094, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0013052122667431831, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 178.22793579101562, "fcm_dpo/q_t": 0.4458271265029907, "grad_norm": 28.088706970214844, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -0.5053350329399109, "logits/rejected": -0.4879445731639862, "logps/chosen": -491.02777099609375, "logps/ref_chosen": -71.27301788330078, "logps/ref_rejected": -86.679931640625, "logps/rejected": -684.66259765625, "loss": 1.2247, "margin_dpo/margin_mean": 178.22792053222656, "margin_dpo/margin_std": 371.9207458496094, "step": 443 }, { "KL/chosen_KL_mean": -403.64312744140625, "KL/mean": -544.897705078125, "KL/rejected_KL_mean": -686.1524047851562, "KL/std": 324.3928527832031, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0013052122667431831, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 282.50927734375, "fcm_dpo/q_t": 0.4141772389411926, "grad_norm": 25.328685760498047, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -0.4659857749938965, "logits/rejected": -0.46927428245544434, "logps/chosen": -460.8568115234375, "logps/ref_chosen": -57.213706970214844, "logps/ref_rejected": -97.25489807128906, "logps/rejected": -783.4072875976562, "loss": 1.1037, "margin_dpo/margin_mean": 282.5092468261719, "margin_dpo/margin_std": 361.67974853515625, "step": 444 }, { "KL/chosen_KL_mean": -367.4541015625, "KL/mean": -502.7689208984375, "KL/rejected_KL_mean": -638.083740234375, "KL/std": 279.38037109375, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0013052122667431831, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 270.629638671875, "fcm_dpo/q_t": 0.4174720048904419, "grad_norm": 22.808685302734375, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -0.48408472537994385, "logits/rejected": -0.47305405139923096, "logps/chosen": -434.75390625, "logps/ref_chosen": -67.29979705810547, "logps/ref_rejected": -92.68267059326172, "logps/rejected": -730.7664794921875, "loss": 1.1139, "margin_dpo/margin_mean": 270.6296691894531, "margin_dpo/margin_std": 345.13861083984375, "step": 445 }, { "KL/chosen_KL_mean": -335.86492919921875, "KL/mean": -480.3028869628906, "KL/rejected_KL_mean": -624.7408447265625, "KL/std": 285.0141296386719, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0013007477391511202, "fcm_dpo/delta": -0.03432300314307213, "fcm_dpo/margin": 288.8758544921875, "fcm_dpo/q_t": 0.4105881452560425, "grad_norm": 23.083541870117188, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -0.5223360061645508, "logits/rejected": -0.5316995978355408, "logps/chosen": -394.96343994140625, "logps/ref_chosen": -59.098487854003906, "logps/ref_rejected": -101.26419067382812, "logps/rejected": -726.0050048828125, "loss": 1.0862, "margin_dpo/margin_mean": 288.8758544921875, "margin_dpo/margin_std": 308.46868896484375, "step": 446 }, { "KL/chosen_KL_mean": -355.49981689453125, "KL/mean": -508.11871337890625, "KL/rejected_KL_mean": -660.7376098632812, "KL/std": 347.4691467285156, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.001289354171603918, "fcm_dpo/delta": -0.026798272505402565, "fcm_dpo/margin": 305.23779296875, "fcm_dpo/q_t": 0.41011327505111694, "grad_norm": 29.463502883911133, "learning_rate": 1.608874379754465e-07, "logits/chosen": -0.5696920156478882, "logits/rejected": -0.5802756547927856, "logps/chosen": -411.57513427734375, "logps/ref_chosen": -56.07533264160156, "logps/ref_rejected": -98.69475555419922, "logps/rejected": -759.432373046875, "loss": 1.0888, "margin_dpo/margin_mean": 305.23779296875, "margin_dpo/margin_std": 392.8905029296875, "step": 447 }, { "KL/chosen_KL_mean": -386.01611328125, "KL/mean": -539.498046875, "KL/rejected_KL_mean": -692.97998046875, "KL/std": 296.15045166015625, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0012851132778450847, "fcm_dpo/delta": -0.033000439405441284, "fcm_dpo/margin": 306.9638671875, "fcm_dpo/q_t": 0.4075472354888916, "grad_norm": 39.277767181396484, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -0.4916926622390747, "logits/rejected": -0.4980274438858032, "logps/chosen": -446.01995849609375, "logps/ref_chosen": -60.00384521484375, "logps/ref_rejected": -102.26465606689453, "logps/rejected": -795.24462890625, "loss": 1.0759, "margin_dpo/margin_mean": 306.9638671875, "margin_dpo/margin_std": 335.7113037109375, "step": 448 }, { "KL/chosen_KL_mean": -411.36688232421875, "KL/mean": -551.0133666992188, "KL/rejected_KL_mean": -690.6597900390625, "KL/std": 341.72772216796875, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0012808723840862513, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 279.2928466796875, "fcm_dpo/q_t": 0.4179277718067169, "grad_norm": 20.590986251831055, "learning_rate": 1.584941086944423e-07, "logits/chosen": -0.5620814561843872, "logits/rejected": -0.5571545362472534, "logps/chosen": -478.8935241699219, "logps/ref_chosen": -67.52661895751953, "logps/ref_rejected": -88.59690856933594, "logps/rejected": -779.2567138671875, "loss": 1.125, "margin_dpo/margin_mean": 279.2928466796875, "margin_dpo/margin_std": 413.36419677734375, "step": 449 }, { "KL/chosen_KL_mean": -333.99755859375, "KL/mean": -498.18817138671875, "KL/rejected_KL_mean": -662.3787231445312, "KL/std": 331.12115478515625, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0012786297593265772, "fcm_dpo/delta": -0.017539218068122864, "fcm_dpo/margin": 328.38116455078125, "fcm_dpo/q_t": 0.40046894550323486, "grad_norm": 31.005373001098633, "learning_rate": 1.573010452010098e-07, "logits/chosen": -0.5352098941802979, "logits/rejected": -0.5448019504547119, "logps/chosen": -391.1056823730469, "logps/ref_chosen": -57.10811996459961, "logps/ref_rejected": -102.75494384765625, "logps/rejected": -765.1336669921875, "loss": 1.0514, "margin_dpo/margin_mean": 328.38116455078125, "margin_dpo/margin_std": 326.8597717285156, "step": 450 }, { "KL/chosen_KL_mean": -435.90960693359375, "KL/mean": -556.161376953125, "KL/rejected_KL_mean": -676.4130859375, "KL/std": 351.29571533203125, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0012763871345669031, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 240.50343322753906, "fcm_dpo/q_t": 0.4259311258792877, "grad_norm": 27.074934005737305, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -0.5356581211090088, "logits/rejected": -0.514275074005127, "logps/chosen": -494.3784484863281, "logps/ref_chosen": -58.46883010864258, "logps/ref_rejected": -72.92941284179688, "logps/rejected": -749.3424682617188, "loss": 1.1687, "margin_dpo/margin_mean": 240.50343322753906, "margin_dpo/margin_std": 412.58929443359375, "step": 451 }, { "KL/chosen_KL_mean": -310.4342346191406, "KL/mean": -462.1864013671875, "KL/rejected_KL_mean": -613.9385986328125, "KL/std": 299.0002746582031, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0012763871345669031, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 303.5043640136719, "fcm_dpo/q_t": 0.40798717737197876, "grad_norm": 17.62459945678711, "learning_rate": 1.549222776991186e-07, "logits/chosen": -0.4971578121185303, "logits/rejected": -0.5155096054077148, "logps/chosen": -360.82476806640625, "logps/ref_chosen": -50.39055252075195, "logps/ref_rejected": -97.77142333984375, "logps/rejected": -711.7100219726562, "loss": 1.0732, "margin_dpo/margin_mean": 303.5043640136719, "margin_dpo/margin_std": 309.4674377441406, "step": 452 }, { "KL/chosen_KL_mean": -380.0153503417969, "KL/mean": -510.1275939941406, "KL/rejected_KL_mean": -640.2398681640625, "KL/std": 297.6603088378906, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0012763871345669031, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 260.2244873046875, "fcm_dpo/q_t": 0.42284131050109863, "grad_norm": 34.658294677734375, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -0.5617681741714478, "logits/rejected": -0.5431563258171082, "logps/chosen": -437.730224609375, "logps/ref_chosen": -57.71485137939453, "logps/ref_rejected": -82.20741271972656, "logps/rejected": -722.447265625, "loss": 1.1294, "margin_dpo/margin_mean": 260.2244873046875, "margin_dpo/margin_std": 350.82586669921875, "step": 453 }, { "KL/chosen_KL_mean": -447.6456298828125, "KL/mean": -597.4365234375, "KL/rejected_KL_mean": -747.2275390625, "KL/std": 339.4739990234375, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0012763871345669031, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 299.58184814453125, "fcm_dpo/q_t": 0.41170454025268555, "grad_norm": 25.235965728759766, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -0.5426455736160278, "logits/rejected": -0.5316953659057617, "logps/chosen": -508.5912780761719, "logps/ref_chosen": -60.945648193359375, "logps/ref_rejected": -84.95079040527344, "logps/rejected": -832.1783447265625, "loss": 1.0988, "margin_dpo/margin_mean": 299.5818786621094, "margin_dpo/margin_std": 396.16448974609375, "step": 454 }, { "KL/chosen_KL_mean": -390.29376220703125, "KL/mean": -577.4393920898438, "KL/rejected_KL_mean": -764.5850219726562, "KL/std": 369.1748046875, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.001265165745280683, "fcm_dpo/delta": -0.08869747817516327, "fcm_dpo/margin": 374.29132080078125, "fcm_dpo/q_t": 0.3919771909713745, "grad_norm": 29.067773818969727, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -0.5430868268013, "logits/rejected": -0.5612744092941284, "logps/chosen": -435.1804504394531, "logps/ref_chosen": -44.88671112060547, "logps/ref_rejected": -115.30147552490234, "logps/rejected": -879.886474609375, "loss": 1.0373, "margin_dpo/margin_mean": 374.29132080078125, "margin_dpo/margin_std": 424.240234375, "step": 455 }, { "KL/chosen_KL_mean": -419.44073486328125, "KL/mean": -587.567626953125, "KL/rejected_KL_mean": -755.6944580078125, "KL/std": 357.8977355957031, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0012515599373728037, "fcm_dpo/delta": -0.019051332026720047, "fcm_dpo/margin": 336.253662109375, "fcm_dpo/q_t": 0.40199345350265503, "grad_norm": 25.685274124145508, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -0.5657025575637817, "logits/rejected": -0.5827906131744385, "logps/chosen": -476.4775390625, "logps/ref_chosen": -57.036781311035156, "logps/ref_rejected": -105.21784210205078, "logps/rejected": -860.9122924804688, "loss": 1.0551, "margin_dpo/margin_mean": 336.25372314453125, "margin_dpo/margin_std": 357.0649719238281, "step": 456 }, { "KL/chosen_KL_mean": -389.86431884765625, "KL/mean": -559.647705078125, "KL/rejected_KL_mean": -729.43115234375, "KL/std": 333.1719970703125, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0012405143352225423, "fcm_dpo/delta": -0.03478863835334778, "fcm_dpo/margin": 339.56689453125, "fcm_dpo/q_t": 0.39991387724876404, "grad_norm": 31.572132110595703, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -0.5767790079116821, "logits/rejected": -0.5724951028823853, "logps/chosen": -444.10687255859375, "logps/ref_chosen": -54.24253845214844, "logps/ref_rejected": -85.10956573486328, "logps/rejected": -814.540771484375, "loss": 1.0492, "margin_dpo/margin_mean": 339.56689453125, "margin_dpo/margin_std": 329.47491455078125, "step": 457 }, { "KL/chosen_KL_mean": -409.2054443359375, "KL/mean": -559.6752319335938, "KL/rejected_KL_mean": -710.14501953125, "KL/std": 302.4131774902344, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.0012405143352225423, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 300.9395751953125, "fcm_dpo/q_t": 0.4121519923210144, "grad_norm": 21.94940185546875, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -0.5830236673355103, "logits/rejected": -0.5776429772377014, "logps/chosen": -464.61431884765625, "logps/ref_chosen": -55.40888214111328, "logps/ref_rejected": -97.68325805664062, "logps/rejected": -807.8282470703125, "loss": 1.0911, "margin_dpo/margin_mean": 300.9395751953125, "margin_dpo/margin_std": 344.2265625, "step": 458 }, { "KL/chosen_KL_mean": -434.1375427246094, "KL/mean": -599.1243896484375, "KL/rejected_KL_mean": -764.1112060546875, "KL/std": 358.45794677734375, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.001232771435752511, "fcm_dpo/delta": -0.0313061960041523, "fcm_dpo/margin": 329.97369384765625, "fcm_dpo/q_t": 0.40537169575691223, "grad_norm": 33.05942916870117, "learning_rate": 1.466771464027316e-07, "logits/chosen": -0.589569091796875, "logits/rejected": -0.6022425889968872, "logps/chosen": -480.69500732421875, "logps/ref_chosen": -46.55748748779297, "logps/ref_rejected": -86.16854095458984, "logps/rejected": -850.27978515625, "loss": 1.081, "margin_dpo/margin_mean": 329.9737243652344, "margin_dpo/margin_std": 406.204345703125, "step": 459 }, { "KL/chosen_KL_mean": -496.5813903808594, "KL/mean": -666.2108154296875, "KL/rejected_KL_mean": -835.840087890625, "KL/std": 349.3248291015625, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0012299625668674707, "fcm_dpo/delta": -0.022837545722723007, "fcm_dpo/margin": 339.2586975097656, "fcm_dpo/q_t": 0.40289586782455444, "grad_norm": 32.447635650634766, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -0.6300777792930603, "logits/rejected": -0.6566332578659058, "logps/chosen": -548.21630859375, "logps/ref_chosen": -51.63489532470703, "logps/ref_rejected": -104.11935424804688, "logps/rejected": -939.95947265625, "loss": 1.0673, "margin_dpo/margin_mean": 339.2586975097656, "margin_dpo/margin_std": 395.57269287109375, "step": 460 }, { "KL/chosen_KL_mean": -494.4696044921875, "KL/mean": -633.8778076171875, "KL/rejected_KL_mean": -773.2859497070312, "KL/std": 341.07598876953125, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0012271535815671086, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 278.81634521484375, "fcm_dpo/q_t": 0.4200149476528168, "grad_norm": 27.538618087768555, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -0.6771355867385864, "logits/rejected": -0.6876776814460754, "logps/chosen": -549.6515502929688, "logps/ref_chosen": -55.18195724487305, "logps/ref_rejected": -86.47689819335938, "logps/rejected": -859.7628173828125, "loss": 1.1277, "margin_dpo/margin_mean": 278.81634521484375, "margin_dpo/margin_std": 390.0162658691406, "step": 461 }, { "KL/chosen_KL_mean": -510.42669677734375, "KL/mean": -613.5990600585938, "KL/rejected_KL_mean": -716.7714233398438, "KL/std": 337.86505126953125, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.0012271535815671086, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 206.34474182128906, "fcm_dpo/q_t": 0.4412042796611786, "grad_norm": 38.32763671875, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -0.6980989575386047, "logits/rejected": -0.6892492771148682, "logps/chosen": -580.354736328125, "logps/ref_chosen": -69.92803192138672, "logps/ref_rejected": -78.84111022949219, "logps/rejected": -795.612548828125, "loss": 1.2047, "margin_dpo/margin_mean": 206.3447265625, "margin_dpo/margin_std": 392.1902160644531, "step": 462 }, { "KL/chosen_KL_mean": -485.9715881347656, "KL/mean": -636.2039794921875, "KL/rejected_KL_mean": -786.4364013671875, "KL/std": 374.5822448730469, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0012250933796167374, "fcm_dpo/delta": -0.01681620068848133, "fcm_dpo/margin": 300.4647521972656, "fcm_dpo/q_t": 0.41564083099365234, "grad_norm": 36.514583587646484, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -0.6740679740905762, "logits/rejected": -0.681418776512146, "logps/chosen": -541.2459716796875, "logps/ref_chosen": -55.27437210083008, "logps/ref_rejected": -89.02497863769531, "logps/rejected": -875.4613647460938, "loss": 1.1198, "margin_dpo/margin_mean": 300.4647521972656, "margin_dpo/margin_std": 436.19610595703125, "step": 463 }, { "KL/chosen_KL_mean": -479.87335205078125, "KL/mean": -703.5326538085938, "KL/rejected_KL_mean": -927.1920166015625, "KL/std": 418.5787353515625, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0011922982521355152, "fcm_dpo/delta": -0.14186176657676697, "fcm_dpo/margin": 447.31866455078125, "fcm_dpo/q_t": 0.37638723850250244, "grad_norm": 34.769657135009766, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -0.6487765312194824, "logits/rejected": -0.6982420682907104, "logps/chosen": -530.78564453125, "logps/ref_chosen": -50.91230010986328, "logps/ref_rejected": -102.4893798828125, "logps/rejected": -1029.681396484375, "loss": 0.9827, "margin_dpo/margin_mean": 447.3186340332031, "margin_dpo/margin_std": 431.365478515625, "step": 464 }, { "KL/chosen_KL_mean": -455.03009033203125, "KL/mean": -633.11962890625, "KL/rejected_KL_mean": -811.2090454101562, "KL/std": 406.3192138671875, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0011822583619505167, "fcm_dpo/delta": -0.05550694093108177, "fcm_dpo/margin": 356.178955078125, "fcm_dpo/q_t": 0.401016503572464, "grad_norm": 38.73068618774414, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -0.6660338640213013, "logits/rejected": -0.6831067800521851, "logps/chosen": -515.14697265625, "logps/ref_chosen": -60.116851806640625, "logps/ref_rejected": -113.94602966308594, "logps/rejected": -925.1550903320312, "loss": 1.0704, "margin_dpo/margin_mean": 356.1789855957031, "margin_dpo/margin_std": 426.8403015136719, "step": 465 }, { "KL/chosen_KL_mean": -490.3927917480469, "KL/mean": -640.9849853515625, "KL/rejected_KL_mean": -791.5771484375, "KL/std": 363.56646728515625, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0011756960302591324, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 301.18438720703125, "fcm_dpo/q_t": 0.41663411259651184, "grad_norm": 36.27015686035156, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -0.6633919477462769, "logits/rejected": -0.6672701835632324, "logps/chosen": -543.313720703125, "logps/ref_chosen": -52.920921325683594, "logps/ref_rejected": -90.3154296875, "logps/rejected": -881.892578125, "loss": 1.1181, "margin_dpo/margin_mean": 301.18438720703125, "margin_dpo/margin_std": 405.44305419921875, "step": 466 }, { "KL/chosen_KL_mean": -629.504638671875, "KL/mean": -789.4716186523438, "KL/rejected_KL_mean": -949.4385986328125, "KL/std": 462.40643310546875, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0011614387622103095, "fcm_dpo/delta": -0.061003945767879486, "fcm_dpo/margin": 319.9339294433594, "fcm_dpo/q_t": 0.4168221950531006, "grad_norm": 45.457401275634766, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -0.7396456003189087, "logits/rejected": -0.7342954277992249, "logps/chosen": -708.220458984375, "logps/ref_chosen": -78.7158203125, "logps/ref_rejected": -102.86019897460938, "logps/rejected": -1052.298828125, "loss": 1.1552, "margin_dpo/margin_mean": 319.9339599609375, "margin_dpo/margin_std": 570.8200073242188, "step": 467 }, { "KL/chosen_KL_mean": -484.61407470703125, "KL/mean": -672.2987060546875, "KL/rejected_KL_mean": -859.9832763671875, "KL/std": 430.0896911621094, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0011516888625919819, "fcm_dpo/delta": -0.04215071350336075, "fcm_dpo/margin": 375.36920166015625, "fcm_dpo/q_t": 0.40216344594955444, "grad_norm": 38.76249694824219, "learning_rate": 1.362737437810114e-07, "logits/chosen": -0.726195752620697, "logits/rejected": -0.7313699126243591, "logps/chosen": -554.5494384765625, "logps/ref_chosen": -69.93536376953125, "logps/ref_rejected": -101.02880859375, "logps/rejected": -961.0120849609375, "loss": 1.0731, "margin_dpo/margin_mean": 375.36920166015625, "margin_dpo/margin_std": 493.8682556152344, "step": 468 }, { "KL/chosen_KL_mean": -523.5865478515625, "KL/mean": -722.1326904296875, "KL/rejected_KL_mean": -920.6788330078125, "KL/std": 390.2972717285156, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.00113427615724504, "fcm_dpo/delta": -0.07617395371198654, "fcm_dpo/margin": 397.0923156738281, "fcm_dpo/q_t": 0.39496946334838867, "grad_norm": 52.0592155456543, "learning_rate": 1.351323902551631e-07, "logits/chosen": -0.7656177282333374, "logits/rejected": -0.7790836691856384, "logps/chosen": -591.711181640625, "logps/ref_chosen": -68.12469482421875, "logps/ref_rejected": -104.78640747070312, "logps/rejected": -1025.46533203125, "loss": 1.044, "margin_dpo/margin_mean": 397.09234619140625, "margin_dpo/margin_std": 434.4091491699219, "step": 469 }, { "KL/chosen_KL_mean": -428.40618896484375, "KL/mean": -600.690673828125, "KL/rejected_KL_mean": -772.9750366210938, "KL/std": 402.35675048828125, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.00113427615724504, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 344.56884765625, "fcm_dpo/q_t": 0.40878430008888245, "grad_norm": 27.16141128540039, "learning_rate": 1.339940635976592e-07, "logits/chosen": -0.7324019074440002, "logits/rejected": -0.7460435032844543, "logps/chosen": -472.1981201171875, "logps/ref_chosen": -43.791927337646484, "logps/ref_rejected": -82.70285034179688, "logps/rejected": -855.6778564453125, "loss": 1.0924, "margin_dpo/margin_mean": 344.56884765625, "margin_dpo/margin_std": 443.0701904296875, "step": 470 }, { "KL/chosen_KL_mean": -538.134521484375, "KL/mean": -695.206787109375, "KL/rejected_KL_mean": -852.2789916992188, "KL/std": 426.8754577636719, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0011279982281848788, "fcm_dpo/delta": -0.027750782668590546, "fcm_dpo/margin": 314.14447021484375, "fcm_dpo/q_t": 0.4184933304786682, "grad_norm": 29.48638916015625, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -0.8245488405227661, "logits/rejected": -0.8365659713745117, "logps/chosen": -601.4740600585938, "logps/ref_chosen": -63.33952331542969, "logps/ref_rejected": -83.61048126220703, "logps/rejected": -935.8894653320312, "loss": 1.1244, "margin_dpo/margin_mean": 314.14447021484375, "margin_dpo/margin_std": 448.8436279296875, "step": 471 }, { "KL/chosen_KL_mean": -556.806640625, "KL/mean": -734.4447021484375, "KL/rejected_KL_mean": -912.0826416015625, "KL/std": 493.10186767578125, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0011279982281848788, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 355.27606201171875, "fcm_dpo/q_t": 0.4122043251991272, "grad_norm": 31.485414505004883, "learning_rate": 1.317266107909975e-07, "logits/chosen": -0.8135469555854797, "logits/rejected": -0.7923085689544678, "logps/chosen": -640.4727783203125, "logps/ref_chosen": -83.66610717773438, "logps/ref_rejected": -117.20919799804688, "logps/rejected": -1029.2918701171875, "loss": 1.1123, "margin_dpo/margin_mean": 355.27606201171875, "margin_dpo/margin_std": 546.5753784179688, "step": 472 }, { "KL/chosen_KL_mean": -683.7619018554688, "KL/mean": -787.9921264648438, "KL/rejected_KL_mean": -892.2222900390625, "KL/std": 545.3720092773438, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0011279982281848788, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 208.46038818359375, "fcm_dpo/q_t": 0.44825372099876404, "grad_norm": 76.29390716552734, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -0.841565728187561, "logits/rejected": -0.8225346207618713, "logps/chosen": -747.2589111328125, "logps/ref_chosen": -63.49696731567383, "logps/ref_rejected": -81.14657592773438, "logps/rejected": -973.3688354492188, "loss": 1.3034, "margin_dpo/margin_mean": 208.46038818359375, "margin_dpo/margin_std": 709.546875, "step": 473 }, { "KL/chosen_KL_mean": -571.6044921875, "KL/mean": -715.5053100585938, "KL/rejected_KL_mean": -859.4061279296875, "KL/std": 460.3998718261719, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0011236746795475483, "fcm_dpo/delta": -0.038477614521980286, "fcm_dpo/margin": 287.80169677734375, "fcm_dpo/q_t": 0.4247364401817322, "grad_norm": 35.654964447021484, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -0.7764319181442261, "logits/rejected": -0.7763692736625671, "logps/chosen": -624.2164306640625, "logps/ref_chosen": -52.6119384765625, "logps/ref_rejected": -90.08041381835938, "logps/rejected": -949.486572265625, "loss": 1.1789, "margin_dpo/margin_mean": 287.80169677734375, "margin_dpo/margin_std": 533.5931396484375, "step": 474 }, { "KL/chosen_KL_mean": -415.87945556640625, "KL/mean": -611.8514404296875, "KL/rejected_KL_mean": -807.8234252929688, "KL/std": 382.92364501953125, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0011138062691316009, "fcm_dpo/delta": -0.024829309433698654, "fcm_dpo/margin": 391.9439697265625, "fcm_dpo/q_t": 0.3984021544456482, "grad_norm": 41.1740608215332, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -0.8495703935623169, "logits/rejected": -0.8721433281898499, "logps/chosen": -458.3746337890625, "logps/ref_chosen": -42.49519348144531, "logps/ref_rejected": -90.06294250488281, "logps/rejected": -897.8863525390625, "loss": 1.0471, "margin_dpo/margin_mean": 391.9439697265625, "margin_dpo/margin_std": 420.0411376953125, "step": 475 }, { "KL/chosen_KL_mean": -541.1224975585938, "KL/mean": -693.9903564453125, "KL/rejected_KL_mean": -846.858154296875, "KL/std": 429.5723876953125, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.0011138062691316009, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 305.7356262207031, "fcm_dpo/q_t": 0.421493262052536, "grad_norm": 45.96709442138672, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -0.8673366904258728, "logits/rejected": -0.880070149898529, "logps/chosen": -584.0718994140625, "logps/ref_chosen": -42.94938278198242, "logps/ref_rejected": -73.71023559570312, "logps/rejected": -920.568359375, "loss": 1.1285, "margin_dpo/margin_mean": 305.73565673828125, "margin_dpo/margin_std": 432.8980712890625, "step": 476 }, { "KL/chosen_KL_mean": -573.6900634765625, "KL/mean": -727.4942626953125, "KL/rejected_KL_mean": -881.2985229492188, "KL/std": 450.3263244628906, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0011138062691316009, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 307.6085205078125, "fcm_dpo/q_t": 0.4209038019180298, "grad_norm": 32.88685989379883, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -0.8998314142227173, "logits/rejected": -0.8742116689682007, "logps/chosen": -644.462646484375, "logps/ref_chosen": -70.77261352539062, "logps/ref_rejected": -76.13737487792969, "logps/rejected": -957.4359130859375, "loss": 1.1596, "margin_dpo/margin_mean": 307.6085205078125, "margin_dpo/margin_std": 546.6884155273438, "step": 477 }, { "KL/chosen_KL_mean": -472.537841796875, "KL/mean": -655.8150024414062, "KL/rejected_KL_mean": -839.0921630859375, "KL/std": 408.32635498046875, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.0011108508333563805, "fcm_dpo/delta": -0.026605598628520966, "fcm_dpo/margin": 366.5543518066406, "fcm_dpo/q_t": 0.4059169888496399, "grad_norm": 34.70343017578125, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.8026444911956787, "logits/rejected": -0.8240790367126465, "logps/chosen": -513.9783935546875, "logps/ref_chosen": -41.440513610839844, "logps/ref_rejected": -85.36196899414062, "logps/rejected": -924.4541625976562, "loss": 1.0865, "margin_dpo/margin_mean": 366.5543518066406, "margin_dpo/margin_std": 478.48858642578125, "step": 478 }, { "KL/chosen_KL_mean": -608.5978393554688, "KL/mean": -790.894775390625, "KL/rejected_KL_mean": -973.1917724609375, "KL/std": 528.50146484375, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.0011017767246812582, "fcm_dpo/delta": -0.05553380772471428, "fcm_dpo/margin": 364.5938720703125, "fcm_dpo/q_t": 0.41255372762680054, "grad_norm": 35.93061065673828, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -0.8539482355117798, "logits/rejected": -0.887352705001831, "logps/chosen": -662.5057373046875, "logps/ref_chosen": -53.907920837402344, "logps/ref_rejected": -95.1163330078125, "logps/rejected": -1068.30810546875, "loss": 1.125, "margin_dpo/margin_mean": 364.5938720703125, "margin_dpo/margin_std": 614.5701904296875, "step": 479 }, { "KL/chosen_KL_mean": -745.3245849609375, "KL/mean": -879.6640625, "KL/rejected_KL_mean": -1014.0034790039062, "KL/std": 472.1075439453125, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0010918761836364865, "fcm_dpo/delta": -0.034638624638319016, "fcm_dpo/margin": 268.6788635253906, "fcm_dpo/q_t": 0.43371596932411194, "grad_norm": 47.8585090637207, "learning_rate": 1.227838333989088e-07, "logits/chosen": -0.888252854347229, "logits/rejected": -0.887417197227478, "logps/chosen": -804.00732421875, "logps/ref_chosen": -58.682701110839844, "logps/ref_rejected": -82.93248748779297, "logps/rejected": -1096.93603515625, "loss": 1.2006, "margin_dpo/margin_mean": 268.6788330078125, "margin_dpo/margin_std": 522.060302734375, "step": 480 }, { "KL/chosen_KL_mean": -616.970458984375, "KL/mean": -833.5260620117188, "KL/rejected_KL_mean": -1050.0816650390625, "KL/std": 495.3128662109375, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.001077498309314251, "fcm_dpo/delta": -0.06990566849708557, "fcm_dpo/margin": 433.1112060546875, "fcm_dpo/q_t": 0.39440447092056274, "grad_norm": 38.01708221435547, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -0.9300037026405334, "logits/rejected": -0.9615186452865601, "logps/chosen": -671.9346923828125, "logps/ref_chosen": -54.964271545410156, "logps/ref_rejected": -92.42044067382812, "logps/rejected": -1142.502197265625, "loss": 1.049, "margin_dpo/margin_mean": 433.1112060546875, "margin_dpo/margin_std": 537.92578125, "step": 481 }, { "KL/chosen_KL_mean": -663.9671630859375, "KL/mean": -759.5072631835938, "KL/rejected_KL_mean": -855.0474243164062, "KL/std": 483.645263671875, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0010729870991781354, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 191.0802459716797, "fcm_dpo/q_t": 0.4497986435890198, "grad_norm": 47.595367431640625, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -0.8404616117477417, "logits/rejected": -0.8292778730392456, "logps/chosen": -731.5206298828125, "logps/ref_chosen": -67.553466796875, "logps/ref_rejected": -87.58953857421875, "logps/rejected": -942.636962890625, "loss": 1.2789, "margin_dpo/margin_mean": 191.0802459716797, "margin_dpo/margin_std": 566.9451293945312, "step": 482 }, { "KL/chosen_KL_mean": -605.1905517578125, "KL/mean": -833.1956787109375, "KL/rejected_KL_mean": -1061.200927734375, "KL/std": 506.8423767089844, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0010574627667665482, "fcm_dpo/delta": -0.0865851491689682, "fcm_dpo/margin": 456.01025390625, "fcm_dpo/q_t": 0.39256346225738525, "grad_norm": 33.234920501708984, "learning_rate": 1.194847979251979e-07, "logits/chosen": -0.8963823318481445, "logits/rejected": -0.9069424271583557, "logps/chosen": -668.5203857421875, "logps/ref_chosen": -63.32981872558594, "logps/ref_rejected": -95.78697204589844, "logps/rejected": -1156.98779296875, "loss": 1.0479, "margin_dpo/margin_mean": 456.01025390625, "margin_dpo/margin_std": 591.51123046875, "step": 483 }, { "KL/chosen_KL_mean": -507.6318359375, "KL/mean": -704.4056396484375, "KL/rejected_KL_mean": -901.179443359375, "KL/std": 471.9982604980469, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0010493712034076452, "fcm_dpo/delta": -0.049505699425935745, "fcm_dpo/margin": 393.54754638671875, "fcm_dpo/q_t": 0.4051154851913452, "grad_norm": 46.96840286254883, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -0.9186742901802063, "logits/rejected": -0.9407310485839844, "logps/chosen": -566.77001953125, "logps/ref_chosen": -59.13812255859375, "logps/ref_rejected": -84.37144470214844, "logps/rejected": -985.5509033203125, "loss": 1.0752, "margin_dpo/margin_mean": 393.54754638671875, "margin_dpo/margin_std": 481.69659423828125, "step": 484 }, { "KL/chosen_KL_mean": -556.6889038085938, "KL/mean": -737.6572265625, "KL/rejected_KL_mean": -918.6255493164062, "KL/std": 474.49932861328125, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.00104417628608644, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 361.93670654296875, "fcm_dpo/q_t": 0.41450387239456177, "grad_norm": 32.620994567871094, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -0.8470867872238159, "logits/rejected": -0.8637259006500244, "logps/chosen": -615.5384521484375, "logps/ref_chosen": -58.849571228027344, "logps/ref_rejected": -103.36408233642578, "logps/rejected": -1021.9896240234375, "loss": 1.1206, "margin_dpo/margin_mean": 361.93670654296875, "margin_dpo/margin_std": 549.8648071289062, "step": 485 }, { "KL/chosen_KL_mean": -620.675048828125, "KL/mean": -835.8392944335938, "KL/rejected_KL_mean": -1051.0035400390625, "KL/std": 538.46240234375, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0010384198976680636, "fcm_dpo/delta": -0.027640754356980324, "fcm_dpo/margin": 430.3284606933594, "fcm_dpo/q_t": 0.40111875534057617, "grad_norm": 29.77266502380371, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -0.9278169870376587, "logits/rejected": -0.9417098760604858, "logps/chosen": -675.9347534179688, "logps/ref_chosen": -55.25966262817383, "logps/ref_rejected": -92.13936614990234, "logps/rejected": -1143.142822265625, "loss": 1.0937, "margin_dpo/margin_mean": 430.32843017578125, "margin_dpo/margin_std": 660.73291015625, "step": 486 }, { "KL/chosen_KL_mean": -627.740478515625, "KL/mean": -792.3194580078125, "KL/rejected_KL_mean": -956.8984985351562, "KL/std": 484.5578308105469, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0010384198976680636, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 329.1581115722656, "fcm_dpo/q_t": 0.41984665393829346, "grad_norm": 34.58036804199219, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -0.9389762878417969, "logits/rejected": -0.9452526569366455, "logps/chosen": -680.8037719726562, "logps/ref_chosen": -53.06330871582031, "logps/ref_rejected": -92.41883087158203, "logps/rejected": -1049.3173828125, "loss": 1.1402, "margin_dpo/margin_mean": 329.1580810546875, "margin_dpo/margin_std": 512.0164794921875, "step": 487 }, { "KL/chosen_KL_mean": -516.5718994140625, "KL/mean": -692.9715576171875, "KL/rejected_KL_mean": -869.3712158203125, "KL/std": 444.61041259765625, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0010365236084908247, "fcm_dpo/delta": -0.01829504780471325, "fcm_dpo/margin": 352.79931640625, "fcm_dpo/q_t": 0.4146237373352051, "grad_norm": 29.500595092773438, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -0.8787708282470703, "logits/rejected": -0.8868429064750671, "logps/chosen": -568.800048828125, "logps/ref_chosen": -52.22815704345703, "logps/ref_rejected": -84.00656127929688, "logps/rejected": -953.3777465820312, "loss": 1.1012, "margin_dpo/margin_mean": 352.79931640625, "margin_dpo/margin_std": 430.9080810546875, "step": 488 }, { "KL/chosen_KL_mean": -498.3140869140625, "KL/mean": -664.83203125, "KL/rejected_KL_mean": -831.3499755859375, "KL/std": 436.31561279296875, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.001034627202898264, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 333.0358581542969, "fcm_dpo/q_t": 0.4215896427631378, "grad_norm": 28.19427490234375, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -0.8919167518615723, "logits/rejected": -0.8911116719245911, "logps/chosen": -554.3037109375, "logps/ref_chosen": -55.989627838134766, "logps/ref_rejected": -79.39812469482422, "logps/rejected": -910.748046875, "loss": 1.1309, "margin_dpo/margin_mean": 333.0358581542969, "margin_dpo/margin_std": 491.57147216796875, "step": 489 }, { "KL/chosen_KL_mean": -567.6314697265625, "KL/mean": -747.6055908203125, "KL/rejected_KL_mean": -927.5797729492188, "KL/std": 524.840576171875, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.001034627202898264, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 359.94830322265625, "fcm_dpo/q_t": 0.4163244962692261, "grad_norm": 44.42933654785156, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -0.9381183981895447, "logits/rejected": -0.9800983667373657, "logps/chosen": -619.9978637695312, "logps/ref_chosen": -52.36639404296875, "logps/ref_rejected": -110.4090576171875, "logps/rejected": -1037.98876953125, "loss": 1.1393, "margin_dpo/margin_mean": 359.94830322265625, "margin_dpo/margin_std": 605.6275634765625, "step": 490 }, { "KL/chosen_KL_mean": -538.706787109375, "KL/mean": -658.9903564453125, "KL/rejected_KL_mean": -779.27392578125, "KL/std": 446.2900390625, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.001034627202898264, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 240.56715393066406, "fcm_dpo/q_t": 0.4419635236263275, "grad_norm": 29.538942337036133, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -0.8918236494064331, "logits/rejected": -0.887330949306488, "logps/chosen": -598.822998046875, "logps/ref_chosen": -60.11626434326172, "logps/ref_rejected": -73.27278900146484, "logps/rejected": -852.5467529296875, "loss": 1.2013, "margin_dpo/margin_mean": 240.56716918945312, "margin_dpo/margin_std": 437.52325439453125, "step": 491 }, { "KL/chosen_KL_mean": -576.6124267578125, "KL/mean": -691.875732421875, "KL/rejected_KL_mean": -807.13916015625, "KL/std": 466.2126770019531, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.001034627202898264, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 230.52671813964844, "fcm_dpo/q_t": 0.44509798288345337, "grad_norm": 23.890384674072266, "learning_rate": 1.097764975115576e-07, "logits/chosen": -0.9451920986175537, "logits/rejected": -0.9290696382522583, "logps/chosen": -630.6065673828125, "logps/ref_chosen": -53.994178771972656, "logps/ref_rejected": -72.65962219238281, "logps/rejected": -879.7987670898438, "loss": 1.2337, "margin_dpo/margin_mean": 230.52671813964844, "margin_dpo/margin_std": 528.3690795898438, "step": 492 }, { "KL/chosen_KL_mean": -610.3974609375, "KL/mean": -739.4407348632812, "KL/rejected_KL_mean": -868.4840698242188, "KL/std": 505.65179443359375, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.001034627202898264, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 258.0867004394531, "fcm_dpo/q_t": 0.4370383620262146, "grad_norm": 40.53925323486328, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -0.9742394685745239, "logits/rejected": -0.9517089128494263, "logps/chosen": -685.8946533203125, "logps/ref_chosen": -75.49723815917969, "logps/ref_rejected": -87.32301330566406, "logps/rejected": -955.80712890625, "loss": 1.203, "margin_dpo/margin_mean": 258.0867004394531, "margin_dpo/margin_std": 504.8338928222656, "step": 493 }, { "KL/chosen_KL_mean": -448.5633544921875, "KL/mean": -657.5833129882812, "KL/rejected_KL_mean": -866.603271484375, "KL/std": 430.7680969238281, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.0010329412762075663, "fcm_dpo/delta": -0.01632118411362171, "fcm_dpo/margin": 418.0399169921875, "fcm_dpo/q_t": 0.3988221287727356, "grad_norm": 39.62709426879883, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -0.795988917350769, "logits/rejected": -0.8313146829605103, "logps/chosen": -489.922607421875, "logps/ref_chosen": -41.35926818847656, "logps/ref_rejected": -86.09136962890625, "logps/rejected": -952.6946411132812, "loss": 1.0487, "margin_dpo/margin_mean": 418.0399169921875, "margin_dpo/margin_std": 434.9868469238281, "step": 494 }, { "KL/chosen_KL_mean": -520.1987915039062, "KL/mean": -700.3590087890625, "KL/rejected_KL_mean": -880.519287109375, "KL/std": 453.6920471191406, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 360.3204345703125, "fcm_dpo/q_t": 0.4160408079624176, "grad_norm": 31.332609176635742, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -0.9381812810897827, "logits/rejected": -0.947554349899292, "logps/chosen": -583.73388671875, "logps/ref_chosen": -63.53507995605469, "logps/ref_rejected": -91.42443084716797, "logps/rejected": -971.9437255859375, "loss": 1.1239, "margin_dpo/margin_mean": 360.3204345703125, "margin_dpo/margin_std": 561.566650390625, "step": 495 }, { "KL/chosen_KL_mean": -635.0003662109375, "KL/mean": -740.7531127929688, "KL/rejected_KL_mean": -846.505859375, "KL/std": 361.4820556640625, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 211.50555419921875, "fcm_dpo/q_t": 0.44787222146987915, "grad_norm": 42.50273513793945, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -0.8592913150787354, "logits/rejected": -0.8332956433296204, "logps/chosen": -707.59228515625, "logps/ref_chosen": -72.5919189453125, "logps/ref_rejected": -84.32933807373047, "logps/rejected": -930.835205078125, "loss": 1.2259, "margin_dpo/margin_mean": 211.50555419921875, "margin_dpo/margin_std": 415.5306091308594, "step": 496 }, { "KL/chosen_KL_mean": -589.6499633789062, "KL/mean": -693.0792236328125, "KL/rejected_KL_mean": -796.5084838867188, "KL/std": 438.46783447265625, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 206.85845947265625, "fcm_dpo/q_t": 0.4494214653968811, "grad_norm": 27.05514144897461, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -0.8634764552116394, "logits/rejected": -0.8541151285171509, "logps/chosen": -648.2439575195312, "logps/ref_chosen": -58.59397506713867, "logps/ref_rejected": -76.28836822509766, "logps/rejected": -872.796875, "loss": 1.2327, "margin_dpo/margin_mean": 206.8584747314453, "margin_dpo/margin_std": 418.838623046875, "step": 497 }, { "KL/chosen_KL_mean": -564.5078125, "KL/mean": -724.235107421875, "KL/rejected_KL_mean": -883.96240234375, "KL/std": 477.89898681640625, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 319.4546203613281, "fcm_dpo/q_t": 0.4236466884613037, "grad_norm": 31.779359817504883, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -0.8575003147125244, "logits/rejected": -0.8462520837783813, "logps/chosen": -635.7134399414062, "logps/ref_chosen": -71.20565795898438, "logps/ref_rejected": -83.95803833007812, "logps/rejected": -967.9204711914062, "loss": 1.1577, "margin_dpo/margin_mean": 319.45465087890625, "margin_dpo/margin_std": 545.30078125, "step": 498 }, { "KL/chosen_KL_mean": -495.37310791015625, "KL/mean": -694.611083984375, "KL/rejected_KL_mean": -893.8490600585938, "KL/std": 456.4473571777344, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 398.4759826660156, "fcm_dpo/q_t": 0.4053412973880768, "grad_norm": 37.97807693481445, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -0.849279522895813, "logits/rejected": -0.8824639916419983, "logps/chosen": -546.6282958984375, "logps/ref_chosen": -51.25519561767578, "logps/ref_rejected": -101.07870483398438, "logps/rejected": -994.927734375, "loss": 1.0835, "margin_dpo/margin_mean": 398.4759826660156, "margin_dpo/margin_std": 521.2337036132812, "step": 499 }, { "KL/chosen_KL_mean": -614.5982666015625, "KL/mean": -786.6547241210938, "KL/rejected_KL_mean": -958.7110595703125, "KL/std": 431.516357421875, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 344.1127624511719, "fcm_dpo/q_t": 0.4179650545120239, "grad_norm": 38.53366470336914, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -0.8466501235961914, "logits/rejected": -0.8797069787979126, "logps/chosen": -671.625732421875, "logps/ref_chosen": -57.027442932128906, "logps/ref_rejected": -93.93421173095703, "logps/rejected": -1052.645263671875, "loss": 1.1331, "margin_dpo/margin_mean": 344.11279296875, "margin_dpo/margin_std": 528.118896484375, "step": 500 }, { "KL/chosen_KL_mean": -541.839599609375, "KL/mean": -719.80517578125, "KL/rejected_KL_mean": -897.7708129882812, "KL/std": 457.7550048828125, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 355.93121337890625, "fcm_dpo/q_t": 0.4169032573699951, "grad_norm": 30.2011661529541, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -0.8327955007553101, "logits/rejected": -0.8246780633926392, "logps/chosen": -596.1990966796875, "logps/ref_chosen": -54.359527587890625, "logps/ref_rejected": -80.15670013427734, "logps/rejected": -977.927490234375, "loss": 1.1334, "margin_dpo/margin_mean": 355.93121337890625, "margin_dpo/margin_std": 572.0584716796875, "step": 501 }, { "KL/chosen_KL_mean": -501.7233581542969, "KL/mean": -686.9359130859375, "KL/rejected_KL_mean": -872.1483154296875, "KL/std": 432.0093078613281, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 370.42498779296875, "fcm_dpo/q_t": 0.41214871406555176, "grad_norm": 28.975494384765625, "learning_rate": 9.934134090518592e-08, "logits/chosen": -0.7686693668365479, "logits/rejected": -0.7646470069885254, "logps/chosen": -569.3238525390625, "logps/ref_chosen": -67.60050964355469, "logps/ref_rejected": -82.94876098632812, "logps/rejected": -955.0971069335938, "loss": 1.0911, "margin_dpo/margin_mean": 370.42498779296875, "margin_dpo/margin_std": 456.77459716796875, "step": 502 }, { "KL/chosen_KL_mean": -531.9078369140625, "KL/mean": -705.465576171875, "KL/rejected_KL_mean": -879.0233154296875, "KL/std": 428.1451416015625, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 347.11553955078125, "fcm_dpo/q_t": 0.41783469915390015, "grad_norm": 29.5872859954834, "learning_rate": 9.831921068732571e-08, "logits/chosen": -0.7954933643341064, "logits/rejected": -0.7909719944000244, "logps/chosen": -586.9862060546875, "logps/ref_chosen": -55.078407287597656, "logps/ref_rejected": -82.50544738769531, "logps/rejected": -961.5287475585938, "loss": 1.1139, "margin_dpo/margin_mean": 347.11553955078125, "margin_dpo/margin_std": 466.2398376464844, "step": 503 }, { "KL/chosen_KL_mean": -569.9815673828125, "KL/mean": -776.477783203125, "KL/rejected_KL_mean": -982.9739990234375, "KL/std": 500.4878845214844, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 412.99249267578125, "fcm_dpo/q_t": 0.40421169996261597, "grad_norm": 30.236486434936523, "learning_rate": 9.730107739932805e-08, "logits/chosen": -0.8175668716430664, "logits/rejected": -0.8474366664886475, "logps/chosen": -629.9473266601562, "logps/ref_chosen": -59.96575164794922, "logps/ref_rejected": -103.76212310791016, "logps/rejected": -1086.736083984375, "loss": 1.0912, "margin_dpo/margin_mean": 412.99249267578125, "margin_dpo/margin_std": 581.5658569335938, "step": 504 }, { "KL/chosen_KL_mean": -668.5108642578125, "KL/mean": -772.6334228515625, "KL/rejected_KL_mean": -876.7559814453125, "KL/std": 486.93994140625, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0010312554659321904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 208.24505615234375, "fcm_dpo/q_t": 0.4494059085845947, "grad_norm": 49.470855712890625, "learning_rate": 9.628696786995188e-08, "logits/chosen": -0.8806732296943665, "logits/rejected": -0.8661482334136963, "logps/chosen": -744.665771484375, "logps/ref_chosen": -76.1549072265625, "logps/ref_rejected": -88.58537292480469, "logps/rejected": -965.34130859375, "loss": 1.2396, "margin_dpo/margin_mean": 208.24505615234375, "margin_dpo/margin_std": 461.67218017578125, "step": 505 }, { "KL/chosen_KL_mean": -527.1854248046875, "KL/mean": -717.6492919921875, "KL/rejected_KL_mean": -908.11328125, "KL/std": 489.8760986328125, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0010227421298623085, "fcm_dpo/delta": -0.041447706520557404, "fcm_dpo/margin": 380.9278564453125, "fcm_dpo/q_t": 0.41084128618240356, "grad_norm": 33.7549934387207, "learning_rate": 9.527690882192635e-08, "logits/chosen": -0.8268887400627136, "logits/rejected": -0.8512546420097351, "logps/chosen": -576.1458740234375, "logps/ref_chosen": -48.96050262451172, "logps/ref_rejected": -78.41505432128906, "logps/rejected": -986.5283203125, "loss": 1.109, "margin_dpo/margin_mean": 380.9278869628906, "margin_dpo/margin_std": 550.8912353515625, "step": 506 }, { "KL/chosen_KL_mean": -606.8805541992188, "KL/mean": -778.807861328125, "KL/rejected_KL_mean": -950.7351684570312, "KL/std": 575.7430419921875, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0010227421298623085, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 343.8546142578125, "fcm_dpo/q_t": 0.4232868254184723, "grad_norm": 32.357879638671875, "learning_rate": 9.427092687124691e-08, "logits/chosen": -0.8557635545730591, "logits/rejected": -0.876491129398346, "logps/chosen": -673.6820678710938, "logps/ref_chosen": -66.80149841308594, "logps/ref_rejected": -95.37289428710938, "logps/rejected": -1046.1080322265625, "loss": 1.1556, "margin_dpo/margin_mean": 343.8546142578125, "margin_dpo/margin_std": 620.67529296875, "step": 507 }, { "KL/chosen_KL_mean": -669.8018188476562, "KL/mean": -812.485107421875, "KL/rejected_KL_mean": -955.1682739257812, "KL/std": 556.7152099609375, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0010227421298623085, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 285.36639404296875, "fcm_dpo/q_t": 0.43599405884742737, "grad_norm": 31.804035186767578, "learning_rate": 9.326904852647344e-08, "logits/chosen": -0.8378009796142578, "logits/rejected": -0.8496159315109253, "logps/chosen": -741.1052856445312, "logps/ref_chosen": -71.303466796875, "logps/ref_rejected": -95.6275405883789, "logps/rejected": -1050.7958984375, "loss": 1.2193, "margin_dpo/margin_mean": 285.36639404296875, "margin_dpo/margin_std": 657.32666015625, "step": 508 }, { "KL/chosen_KL_mean": -510.67974853515625, "KL/mean": -655.689453125, "KL/rejected_KL_mean": -800.69921875, "KL/std": 391.1563720703125, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0010227421298623085, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 290.0194091796875, "fcm_dpo/q_t": 0.4315534234046936, "grad_norm": 37.032257080078125, "learning_rate": 9.227130018803195e-08, "logits/chosen": -0.7452073693275452, "logits/rejected": -0.7534062266349792, "logps/chosen": -574.4987182617188, "logps/ref_chosen": -63.81895065307617, "logps/ref_rejected": -83.25643920898438, "logps/rejected": -883.9556884765625, "loss": 1.175, "margin_dpo/margin_mean": 290.0194091796875, "margin_dpo/margin_std": 502.0699462890625, "step": 509 }, { "KL/chosen_KL_mean": -629.8031005859375, "KL/mean": -827.0128173828125, "KL/rejected_KL_mean": -1024.2225341796875, "KL/std": 466.19384765625, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0010203055571764708, "fcm_dpo/delta": -0.023880695924162865, "fcm_dpo/margin": 394.41937255859375, "fcm_dpo/q_t": 0.40587687492370605, "grad_norm": 34.109798431396484, "learning_rate": 9.127770814751932e-08, "logits/chosen": -0.7773102521896362, "logits/rejected": -0.8000172972679138, "logps/chosen": -681.6815795898438, "logps/ref_chosen": -51.878448486328125, "logps/ref_rejected": -102.7651596069336, "logps/rejected": -1126.9876708984375, "loss": 1.0775, "margin_dpo/margin_mean": 394.4193420410156, "margin_dpo/margin_std": 466.3671569824219, "step": 510 }, { "KL/chosen_KL_mean": -586.57666015625, "KL/mean": -742.509033203125, "KL/rejected_KL_mean": -898.44140625, "KL/std": 519.578125, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.001017868984490633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 311.86468505859375, "fcm_dpo/q_t": 0.4275307357311249, "grad_norm": 43.292728424072266, "learning_rate": 9.028829858700973e-08, "logits/chosen": -0.8213226199150085, "logits/rejected": -0.8411952257156372, "logps/chosen": -646.8148193359375, "logps/ref_chosen": -60.23811721801758, "logps/ref_rejected": -92.85676574707031, "logps/rejected": -991.2981567382812, "loss": 1.1908, "margin_dpo/margin_mean": 311.86468505859375, "margin_dpo/margin_std": 644.8040771484375, "step": 511 }, { "KL/chosen_KL_mean": -477.79278564453125, "KL/mean": -694.4921264648438, "KL/rejected_KL_mean": -911.1914672851562, "KL/std": 460.89337158203125, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0010110008297488093, "fcm_dpo/delta": -0.03385215252637863, "fcm_dpo/margin": 433.3987121582031, "fcm_dpo/q_t": 0.39783334732055664, "grad_norm": 42.38991165161133, "learning_rate": 8.930309757836516e-08, "logits/chosen": -0.7866002321243286, "logits/rejected": -0.8186514377593994, "logps/chosen": -532.6982421875, "logps/ref_chosen": -54.905494689941406, "logps/ref_rejected": -81.87586975097656, "logps/rejected": -993.0673828125, "loss": 1.0468, "margin_dpo/margin_mean": 433.3987121582031, "margin_dpo/margin_std": 465.8355407714844, "step": 512 }, { "KL/chosen_KL_mean": -600.0064697265625, "KL/mean": -747.2848510742188, "KL/rejected_KL_mean": -894.563232421875, "KL/std": 432.29364013671875, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0010110008297488093, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 294.5567626953125, "fcm_dpo/q_t": 0.43137532472610474, "grad_norm": 51.88436508178711, "learning_rate": 8.832213108254863e-08, "logits/chosen": -0.8510720729827881, "logits/rejected": -0.8437062501907349, "logps/chosen": -664.9228515625, "logps/ref_chosen": -64.91644287109375, "logps/ref_rejected": -76.06245422363281, "logps/rejected": -970.6256713867188, "loss": 1.1772, "margin_dpo/margin_mean": 294.5567626953125, "margin_dpo/margin_std": 507.2457275390625, "step": 513 }, { "KL/chosen_KL_mean": -626.586181640625, "KL/mean": -769.6492919921875, "KL/rejected_KL_mean": -912.7125244140625, "KL/std": 469.3660583496094, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.0010110008297488093, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 286.12640380859375, "fcm_dpo/q_t": 0.4342566430568695, "grad_norm": 38.388465881347656, "learning_rate": 8.734542494893954e-08, "logits/chosen": -0.7949620485305786, "logits/rejected": -0.7888473868370056, "logps/chosen": -700.815673828125, "logps/ref_chosen": -74.22957611083984, "logps/ref_rejected": -78.945556640625, "logps/rejected": -991.6580810546875, "loss": 1.1817, "margin_dpo/margin_mean": 286.12640380859375, "margin_dpo/margin_std": 518.3568725585938, "step": 514 }, { "KL/chosen_KL_mean": -515.0665893554688, "KL/mean": -635.9742431640625, "KL/rejected_KL_mean": -756.8817138671875, "KL/std": 402.12554931640625, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0010110008297488093, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 241.81515502929688, "fcm_dpo/q_t": 0.44154810905456543, "grad_norm": 42.603919982910156, "learning_rate": 8.637300491465272e-08, "logits/chosen": -0.7839655876159668, "logits/rejected": -0.7985705137252808, "logps/chosen": -565.4682006835938, "logps/ref_chosen": -50.40156555175781, "logps/ref_rejected": -87.09774780273438, "logps/rejected": -843.9794921875, "loss": 1.211, "margin_dpo/margin_mean": 241.81515502929688, "margin_dpo/margin_std": 474.5227966308594, "step": 515 }, { "KL/chosen_KL_mean": -560.7139892578125, "KL/mean": -737.4444580078125, "KL/rejected_KL_mean": -914.175048828125, "KL/std": 449.7982177734375, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0010110008297488093, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 353.46099853515625, "fcm_dpo/q_t": 0.4154972732067108, "grad_norm": 34.534523010253906, "learning_rate": 8.540489660386064e-08, "logits/chosen": -0.8315219283103943, "logits/rejected": -0.8588598966598511, "logps/chosen": -625.3635864257812, "logps/ref_chosen": -64.64956665039062, "logps/ref_rejected": -111.72237396240234, "logps/rejected": -1025.8974609375, "loss": 1.11, "margin_dpo/margin_mean": 353.46099853515625, "margin_dpo/margin_std": 447.361572265625, "step": 516 }, { "KL/chosen_KL_mean": -585.670166015625, "KL/mean": -785.64111328125, "KL/rejected_KL_mean": -985.612060546875, "KL/std": 497.2357482910156, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.001006539911031723, "fcm_dpo/delta": -0.022110939025878906, "fcm_dpo/margin": 399.94189453125, "fcm_dpo/q_t": 0.40923649072647095, "grad_norm": 30.278491973876953, "learning_rate": 8.444112552711752e-08, "logits/chosen": -0.7785888910293579, "logits/rejected": -0.772436261177063, "logps/chosen": -646.583740234375, "logps/ref_chosen": -60.913551330566406, "logps/ref_rejected": -89.08308410644531, "logps/rejected": -1074.6951904296875, "loss": 1.0955, "margin_dpo/margin_mean": 399.94189453125, "margin_dpo/margin_std": 557.2489013671875, "step": 517 }, { "KL/chosen_KL_mean": -527.8907470703125, "KL/mean": -694.472412109375, "KL/rejected_KL_mean": -861.0540771484375, "KL/std": 399.5091552734375, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.001006539911031723, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 333.16326904296875, "fcm_dpo/q_t": 0.42090073227882385, "grad_norm": 35.90395736694336, "learning_rate": 8.348171708068747e-08, "logits/chosen": -0.8241711854934692, "logits/rejected": -0.8401827216148376, "logps/chosen": -585.3466186523438, "logps/ref_chosen": -57.45589065551758, "logps/ref_rejected": -85.31269836425781, "logps/rejected": -946.36669921875, "loss": 1.128, "margin_dpo/margin_mean": 333.16326904296875, "margin_dpo/margin_std": 447.3388366699219, "step": 518 }, { "KL/chosen_KL_mean": -524.0, "KL/mean": -653.171875, "KL/rejected_KL_mean": -782.34375, "KL/std": 372.1302490234375, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.001006539911031723, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 258.34375, "fcm_dpo/q_t": 0.438649445772171, "grad_norm": 21.516462326049805, "learning_rate": 8.25266965458755e-08, "logits/chosen": -0.769424319267273, "logits/rejected": -0.7528333067893982, "logps/chosen": -598.0633544921875, "logps/ref_chosen": -74.06331634521484, "logps/ref_rejected": -104.44416809082031, "logps/rejected": -886.7879028320312, "loss": 1.1966, "margin_dpo/margin_mean": 258.34375, "margin_dpo/margin_std": 464.10809326171875, "step": 519 }, { "KL/chosen_KL_mean": -573.615234375, "KL/mean": -723.7618408203125, "KL/rejected_KL_mean": -873.908447265625, "KL/std": 434.37841796875, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.001006539911031723, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 300.2933044433594, "fcm_dpo/q_t": 0.43047094345092773, "grad_norm": 40.110504150390625, "learning_rate": 8.15760890883607e-08, "logits/chosen": -0.7728183269500732, "logits/rejected": -0.7749868631362915, "logps/chosen": -643.9150390625, "logps/ref_chosen": -70.2998275756836, "logps/ref_rejected": -99.98133850097656, "logps/rejected": -973.8897705078125, "loss": 1.1648, "margin_dpo/margin_mean": 300.29327392578125, "margin_dpo/margin_std": 477.5467529296875, "step": 520 }, { "KL/chosen_KL_mean": -506.3398132324219, "KL/mean": -672.2125244140625, "KL/rejected_KL_mean": -838.0853271484375, "KL/std": 427.4813232421875, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.0010050538694486022, "fcm_dpo/delta": -0.014785980805754662, "fcm_dpo/margin": 331.74554443359375, "fcm_dpo/q_t": 0.4215025305747986, "grad_norm": 33.34300231933594, "learning_rate": 8.062991975753378e-08, "logits/chosen": -0.8194113969802856, "logits/rejected": -0.8199684619903564, "logps/chosen": -564.4827270507812, "logps/ref_chosen": -58.14292526245117, "logps/ref_rejected": -83.28060913085938, "logps/rejected": -921.365966796875, "loss": 1.1274, "margin_dpo/margin_mean": 331.74554443359375, "margin_dpo/margin_std": 433.67059326171875, "step": 521 }, { "KL/chosen_KL_mean": -563.2149658203125, "KL/mean": -721.0223388671875, "KL/rejected_KL_mean": -878.829833984375, "KL/std": 471.86846923828125, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0010035678278654814, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 315.6148681640625, "fcm_dpo/q_t": 0.42625993490219116, "grad_norm": 27.813331604003906, "learning_rate": 7.968821348583643e-08, "logits/chosen": -0.8177077770233154, "logits/rejected": -0.8223693370819092, "logps/chosen": -609.7625732421875, "logps/ref_chosen": -46.54766845703125, "logps/ref_rejected": -66.01388549804688, "logps/rejected": -944.8436889648438, "loss": 1.1588, "margin_dpo/margin_mean": 315.6148681640625, "margin_dpo/margin_std": 516.5402221679688, "step": 522 }, { "KL/chosen_KL_mean": -598.5844116210938, "KL/mean": -762.6793212890625, "KL/rejected_KL_mean": -926.7742919921875, "KL/std": 517.547607421875, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0010035678278654814, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 328.1898498535156, "fcm_dpo/q_t": 0.4245453476905823, "grad_norm": 35.869773864746094, "learning_rate": 7.875099508810484e-08, "logits/chosen": -0.8979830741882324, "logits/rejected": -0.8981304168701172, "logps/chosen": -660.35400390625, "logps/ref_chosen": -61.76960372924805, "logps/ref_rejected": -83.76141357421875, "logps/rejected": -1010.5357055664062, "loss": 1.158, "margin_dpo/margin_mean": 328.1898498535156, "margin_dpo/margin_std": 560.0676879882812, "step": 523 }, { "KL/chosen_KL_mean": -606.493408203125, "KL/mean": -787.1712646484375, "KL/rejected_KL_mean": -967.84912109375, "KL/std": 502.6514892578125, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.000997315626591444, "fcm_dpo/delta": -0.03124724142253399, "fcm_dpo/margin": 361.355712890625, "fcm_dpo/q_t": 0.41488218307495117, "grad_norm": 33.932228088378906, "learning_rate": 7.781828926091535e-08, "logits/chosen": -0.915719747543335, "logits/rejected": -0.9063006639480591, "logps/chosen": -684.5654296875, "logps/ref_chosen": -78.0720443725586, "logps/ref_rejected": -81.30198669433594, "logps/rejected": -1049.151123046875, "loss": 1.1214, "margin_dpo/margin_mean": 361.355712890625, "margin_dpo/margin_std": 500.954345703125, "step": 524 }, { "KL/chosen_KL_mean": -579.07470703125, "KL/mean": -802.6356811523438, "KL/rejected_KL_mean": -1026.1966552734375, "KL/std": 518.89892578125, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0009856013348326087, "fcm_dpo/delta": -0.059076473116874695, "fcm_dpo/margin": 447.1219482421875, "fcm_dpo/q_t": 0.4000527560710907, "grad_norm": 29.54397201538086, "learning_rate": 7.689012058193384e-08, "logits/chosen": -0.8346696496009827, "logits/rejected": -0.8713185787200928, "logps/chosen": -629.902587890625, "logps/ref_chosen": -50.827857971191406, "logps/ref_rejected": -100.05294036865234, "logps/rejected": -1126.24951171875, "loss": 1.0653, "margin_dpo/margin_mean": 447.1219482421875, "margin_dpo/margin_std": 567.528564453125, "step": 525 }, { "KL/chosen_KL_mean": -608.8138427734375, "KL/mean": -826.2957763671875, "KL/rejected_KL_mean": -1043.777587890625, "KL/std": 492.2466735839844, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0009817786049097776, "fcm_dpo/delta": -0.03893730789422989, "fcm_dpo/margin": 434.9637451171875, "fcm_dpo/q_t": 0.40048468112945557, "grad_norm": 30.214094161987305, "learning_rate": 7.596651350926836e-08, "logits/chosen": -0.8993504047393799, "logits/rejected": -0.9024904370307922, "logps/chosen": -671.9810791015625, "logps/ref_chosen": -63.167236328125, "logps/ref_rejected": -86.30934143066406, "logps/rejected": -1130.0869140625, "loss": 1.0727, "margin_dpo/margin_mean": 434.9637451171875, "margin_dpo/margin_std": 545.2774047851562, "step": 526 }, { "KL/chosen_KL_mean": -662.787109375, "KL/mean": -804.6998291015625, "KL/rejected_KL_mean": -946.6126098632812, "KL/std": 530.4249267578125, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0009779557585716248, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 283.82550048828125, "fcm_dpo/q_t": 0.4336469769477844, "grad_norm": 32.90876007080078, "learning_rate": 7.504749238082414e-08, "logits/chosen": -1.0440315008163452, "logits/rejected": -1.018221378326416, "logps/chosen": -733.915771484375, "logps/ref_chosen": -71.12867736816406, "logps/ref_rejected": -78.3425521850586, "logps/rejected": -1024.9552001953125, "loss": 1.1731, "margin_dpo/margin_mean": 283.8254699707031, "margin_dpo/margin_std": 440.089111328125, "step": 527 }, { "KL/chosen_KL_mean": -678.576416015625, "KL/mean": -860.773193359375, "KL/rejected_KL_mean": -1042.969970703125, "KL/std": 518.66552734375, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0009779557585716248, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 364.39361572265625, "fcm_dpo/q_t": 0.42012155055999756, "grad_norm": 39.71454620361328, "learning_rate": 7.413308141366254e-08, "logits/chosen": -0.9921354055404663, "logits/rejected": -0.9825552701950073, "logps/chosen": -746.6658935546875, "logps/ref_chosen": -68.0894546508789, "logps/ref_rejected": -93.91006469726562, "logps/rejected": -1136.8800048828125, "loss": 1.1484, "margin_dpo/margin_mean": 364.3935852050781, "margin_dpo/margin_std": 623.328857421875, "step": 528 }, { "KL/chosen_KL_mean": -818.5634765625, "KL/mean": -943.8115234375, "KL/rejected_KL_mean": -1069.059814453125, "KL/std": 499.3103942871094, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0009779557585716248, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 250.49615478515625, "fcm_dpo/q_t": 0.4428832530975342, "grad_norm": 39.008636474609375, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.0032634735107422, "logits/rejected": -1.0176358222961426, "logps/chosen": -874.138427734375, "logps/ref_chosen": -55.57495880126953, "logps/ref_rejected": -89.20909118652344, "logps/rejected": -1158.268798828125, "loss": 1.2489, "margin_dpo/margin_mean": 250.4961700439453, "margin_dpo/margin_std": 632.3921508789062, "step": 529 }, { "KL/chosen_KL_mean": -697.18505859375, "KL/mean": -897.6633911132812, "KL/rejected_KL_mean": -1098.1417236328125, "KL/std": 595.1827392578125, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0009779557585716248, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 400.9566345214844, "fcm_dpo/q_t": 0.4137534499168396, "grad_norm": 41.25276184082031, "learning_rate": 7.231818622338822e-08, "logits/chosen": -0.9190390110015869, "logits/rejected": -0.9196174144744873, "logps/chosen": -744.7864990234375, "logps/ref_chosen": -47.601417541503906, "logps/ref_rejected": -87.2845230102539, "logps/rejected": -1185.42626953125, "loss": 1.1537, "margin_dpo/margin_mean": 400.9566345214844, "margin_dpo/margin_std": 755.96826171875, "step": 530 }, { "KL/chosen_KL_mean": -761.462158203125, "KL/mean": -938.3872680664062, "KL/rejected_KL_mean": -1115.3123779296875, "KL/std": 620.985595703125, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.0009779557585716248, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 353.85015869140625, "fcm_dpo/q_t": 0.4216981530189514, "grad_norm": 41.99516296386719, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.0249850749969482, "logits/rejected": -1.0165561437606812, "logps/chosen": -816.708251953125, "logps/ref_chosen": -55.246063232421875, "logps/ref_rejected": -70.60598754882812, "logps/rejected": -1185.91845703125, "loss": 1.1491, "margin_dpo/margin_mean": 353.8501892089844, "margin_dpo/margin_std": 595.3245849609375, "step": 531 }, { "KL/chosen_KL_mean": -767.3759155273438, "KL/mean": -957.3938598632812, "KL/rejected_KL_mean": -1147.411865234375, "KL/std": 608.4393920898438, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0009684361866675317, "fcm_dpo/delta": -0.048909228295087814, "fcm_dpo/margin": 380.03594970703125, "fcm_dpo/q_t": 0.4171864986419678, "grad_norm": 66.07853698730469, "learning_rate": 7.052201923388953e-08, "logits/chosen": -0.9798325896263123, "logits/rejected": -0.9659659266471863, "logps/chosen": -837.6619262695312, "logps/ref_chosen": -70.28601837158203, "logps/ref_rejected": -86.5913314819336, "logps/rejected": -1234.003173828125, "loss": 1.1672, "margin_dpo/margin_mean": 380.03594970703125, "margin_dpo/margin_std": 719.28515625, "step": 532 }, { "KL/chosen_KL_mean": -700.1473388671875, "KL/mean": -833.2701416015625, "KL/rejected_KL_mean": -966.3930053710938, "KL/std": 496.19708251953125, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0009684361866675317, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 266.2456359863281, "fcm_dpo/q_t": 0.44103071093559265, "grad_norm": 40.109352111816406, "learning_rate": 6.963101805503646e-08, "logits/chosen": -0.9989339113235474, "logits/rejected": -0.9852375388145447, "logps/chosen": -765.00244140625, "logps/ref_chosen": -64.8551025390625, "logps/ref_rejected": -76.58805847167969, "logps/rejected": -1042.9810791015625, "loss": 1.2276, "margin_dpo/margin_mean": 266.2456359863281, "margin_dpo/margin_std": 604.99951171875, "step": 533 }, { "KL/chosen_KL_mean": -748.6889038085938, "KL/mean": -934.053955078125, "KL/rejected_KL_mean": -1119.4190673828125, "KL/std": 555.2821044921875, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0009583088685758412, "fcm_dpo/delta": -0.052562322467565536, "fcm_dpo/margin": 370.7301940917969, "fcm_dpo/q_t": 0.4191279709339142, "grad_norm": 30.985227584838867, "learning_rate": 6.874476976660184e-08, "logits/chosen": -0.9798387885093689, "logits/rejected": -0.9826042056083679, "logps/chosen": -808.8082885742188, "logps/ref_chosen": -60.119388580322266, "logps/ref_rejected": -78.54347229003906, "logps/rejected": -1197.9625244140625, "loss": 1.142, "margin_dpo/margin_mean": 370.7301940917969, "margin_dpo/margin_std": 594.4391479492188, "step": 534 }, { "KL/chosen_KL_mean": -624.0946044921875, "KL/mean": -831.3848266601562, "KL/rejected_KL_mean": -1038.6749267578125, "KL/std": 534.6995239257812, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0009523722692392766, "fcm_dpo/delta": -0.062335532158613205, "fcm_dpo/margin": 414.58038330078125, "fcm_dpo/q_t": 0.4099598526954651, "grad_norm": 27.550270080566406, "learning_rate": 6.786329772205246e-08, "logits/chosen": -0.8934037685394287, "logits/rejected": -0.9010223150253296, "logps/chosen": -678.4248657226562, "logps/ref_chosen": -54.330238342285156, "logps/ref_rejected": -96.30763244628906, "logps/rejected": -1134.982666015625, "loss": 1.1031, "margin_dpo/margin_mean": 414.58038330078125, "margin_dpo/margin_std": 572.0284423828125, "step": 535 }, { "KL/chosen_KL_mean": -564.6681518554688, "KL/mean": -820.0997314453125, "KL/rejected_KL_mean": -1075.5313720703125, "KL/std": 627.7283325195312, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.000939212623052299, "fcm_dpo/delta": -0.03830568864941597, "fcm_dpo/margin": 510.8631591796875, "fcm_dpo/q_t": 0.39753982424736023, "grad_norm": 33.21775436401367, "learning_rate": 6.698662514899638e-08, "logits/chosen": -0.8998743295669556, "logits/rejected": -0.9397362470626831, "logps/chosen": -611.7486572265625, "logps/ref_chosen": -47.08053207397461, "logps/ref_rejected": -89.09783935546875, "logps/rejected": -1164.629150390625, "loss": 1.0702, "margin_dpo/margin_mean": 510.8631591796875, "margin_dpo/margin_std": 767.861572265625, "step": 536 }, { "KL/chosen_KL_mean": -593.1109008789062, "KL/mean": -760.2655029296875, "KL/rejected_KL_mean": -927.4201049804688, "KL/std": 485.60101318359375, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.000939212623052299, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 334.30914306640625, "fcm_dpo/q_t": 0.4269586205482483, "grad_norm": 37.82878494262695, "learning_rate": 6.611477514857114e-08, "logits/chosen": -0.943962812423706, "logits/rejected": -0.9385641813278198, "logps/chosen": -650.8583984375, "logps/ref_chosen": -57.747467041015625, "logps/ref_rejected": -70.43838500976562, "logps/rejected": -997.8585205078125, "loss": 1.1749, "margin_dpo/margin_mean": 334.30914306640625, "margin_dpo/margin_std": 601.835693359375, "step": 537 }, { "KL/chosen_KL_mean": -740.7052001953125, "KL/mean": -931.8037109375, "KL/rejected_KL_mean": -1122.9022216796875, "KL/std": 551.2314453125, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.000939212623052299, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 382.19696044921875, "fcm_dpo/q_t": 0.4175069332122803, "grad_norm": 34.995201110839844, "learning_rate": 6.524777069483525e-08, "logits/chosen": -0.9600582122802734, "logits/rejected": -0.954196572303772, "logps/chosen": -807.12109375, "logps/ref_chosen": -66.41594696044922, "logps/ref_rejected": -84.22808837890625, "logps/rejected": -1207.13037109375, "loss": 1.1259, "margin_dpo/margin_mean": 382.19696044921875, "margin_dpo/margin_std": 565.8170166015625, "step": 538 }, { "KL/chosen_KL_mean": -622.4293212890625, "KL/mean": -803.8492431640625, "KL/rejected_KL_mean": -985.2692260742188, "KL/std": 459.92645263671875, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.000939212623052299, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 362.83990478515625, "fcm_dpo/q_t": 0.42134806513786316, "grad_norm": 27.91701316833496, "learning_rate": 6.438563463416221e-08, "logits/chosen": -0.956368088722229, "logits/rejected": -0.9539651870727539, "logps/chosen": -680.9221801757812, "logps/ref_chosen": -58.492855072021484, "logps/ref_rejected": -91.85395050048828, "logps/rejected": -1077.1231689453125, "loss": 1.133, "margin_dpo/margin_mean": 362.83990478515625, "margin_dpo/margin_std": 528.1322631835938, "step": 539 }, { "KL/chosen_KL_mean": -613.2940673828125, "KL/mean": -858.5700073242188, "KL/rejected_KL_mean": -1103.845947265625, "KL/std": 557.71875, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0009325757855549455, "fcm_dpo/delta": -0.07116752117872238, "fcm_dpo/margin": 490.5519104003906, "fcm_dpo/q_t": 0.3990428149700165, "grad_norm": 35.79396438598633, "learning_rate": 6.352838968463919e-08, "logits/chosen": -0.9001051187515259, "logits/rejected": -0.93080735206604, "logps/chosen": -676.7765502929688, "logps/ref_chosen": -63.482513427734375, "logps/ref_rejected": -116.42999267578125, "logps/rejected": -1220.27587890625, "loss": 1.0722, "margin_dpo/margin_mean": 490.5518798828125, "margin_dpo/margin_std": 673.9207763671875, "step": 540 }, { "KL/chosen_KL_mean": -723.417724609375, "KL/mean": -848.6341552734375, "KL/rejected_KL_mean": -973.8507080078125, "KL/std": 496.656494140625, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0009259390062652528, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 250.43295288085938, "fcm_dpo/q_t": 0.44649481773376465, "grad_norm": 42.431304931640625, "learning_rate": 6.267605843546767e-08, "logits/chosen": -1.0044571161270142, "logits/rejected": -1.0006784200668335, "logps/chosen": -801.6980590820312, "logps/ref_chosen": -78.28036499023438, "logps/ref_rejected": -103.273681640625, "logps/rejected": -1077.124267578125, "loss": 1.2451, "margin_dpo/margin_mean": 250.43295288085938, "margin_dpo/margin_std": 604.05517578125, "step": 541 }, { "KL/chosen_KL_mean": -614.3228759765625, "KL/mean": -852.7803955078125, "KL/rejected_KL_mean": -1091.23779296875, "KL/std": 551.623046875, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": -0.09556810557842255, "fcm_dpo/margin": 476.91497802734375, "fcm_dpo/q_t": 0.4020240902900696, "grad_norm": 39.426239013671875, "learning_rate": 6.182866334636888e-08, "logits/chosen": -0.9757965803146362, "logits/rejected": -1.0126338005065918, "logps/chosen": -671.807861328125, "logps/ref_chosen": -57.48497009277344, "logps/ref_rejected": -96.47506713867188, "logps/rejected": -1187.712890625, "loss": 1.0837, "margin_dpo/margin_mean": 476.91497802734375, "margin_dpo/margin_std": 653.942138671875, "step": 542 }, { "KL/chosen_KL_mean": -700.5947265625, "KL/mean": -872.3914794921875, "KL/rejected_KL_mean": -1044.188232421875, "KL/std": 673.9176025390625, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 343.59356689453125, "fcm_dpo/q_t": 0.4374322295188904, "grad_norm": 31.346834182739258, "learning_rate": 6.098622674699147e-08, "logits/chosen": -0.9188766479492188, "logits/rejected": -0.9484801888465881, "logps/chosen": -761.2122192382812, "logps/ref_chosen": -60.61750793457031, "logps/ref_rejected": -105.59896850585938, "logps/rejected": -1149.7872314453125, "loss": 1.2094, "margin_dpo/margin_mean": 343.59356689453125, "margin_dpo/margin_std": 792.45556640625, "step": 543 }, { "KL/chosen_KL_mean": -696.1849975585938, "KL/mean": -888.3853759765625, "KL/rejected_KL_mean": -1080.5858154296875, "KL/std": 521.3961181640625, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 384.40069580078125, "fcm_dpo/q_t": 0.41937947273254395, "grad_norm": 30.976642608642578, "learning_rate": 6.01487708363232e-08, "logits/chosen": -0.9292371273040771, "logits/rejected": -0.949985921382904, "logps/chosen": -755.8272705078125, "logps/ref_chosen": -59.642303466796875, "logps/ref_rejected": -100.95469665527344, "logps/rejected": -1181.54052734375, "loss": 1.1344, "margin_dpo/margin_mean": 384.40069580078125, "margin_dpo/margin_std": 590.4559326171875, "step": 544 }, { "KL/chosen_KL_mean": -651.0039672851562, "KL/mean": -878.35302734375, "KL/rejected_KL_mean": -1105.7021484375, "KL/std": 524.6614990234375, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 454.6982421875, "fcm_dpo/q_t": 0.4059000313282013, "grad_norm": 29.362411499023438, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -0.8762187957763672, "logits/rejected": -0.9087934494018555, "logps/chosen": -718.652587890625, "logps/ref_chosen": -67.64859771728516, "logps/ref_rejected": -95.90800476074219, "logps/rejected": -1201.610107421875, "loss": 1.0835, "margin_dpo/margin_mean": 454.6982421875, "margin_dpo/margin_std": 596.3817138671875, "step": 545 }, { "KL/chosen_KL_mean": -614.3394775390625, "KL/mean": -765.3737182617188, "KL/rejected_KL_mean": -916.407958984375, "KL/std": 452.42413330078125, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 302.068603515625, "fcm_dpo/q_t": 0.4340311288833618, "grad_norm": 33.5017204284668, "learning_rate": 5.848888922025552e-08, "logits/chosen": -0.8841739296913147, "logits/rejected": -0.8689842224121094, "logps/chosen": -665.0836791992188, "logps/ref_chosen": -50.744232177734375, "logps/ref_rejected": -81.86622619628906, "logps/rejected": -998.2742309570312, "loss": 1.1769, "margin_dpo/margin_mean": 302.068603515625, "margin_dpo/margin_std": 477.5527038574219, "step": 546 }, { "KL/chosen_KL_mean": -617.981689453125, "KL/mean": -808.706787109375, "KL/rejected_KL_mean": -999.431884765625, "KL/std": 510.5972595214844, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 381.4501953125, "fcm_dpo/q_t": 0.4202958643436432, "grad_norm": 41.74578094482422, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -0.8707060813903809, "logits/rejected": -0.8808646202087402, "logps/chosen": -691.66943359375, "logps/ref_chosen": -73.6877212524414, "logps/ref_rejected": -90.76136779785156, "logps/rejected": -1090.1932373046875, "loss": 1.1313, "margin_dpo/margin_mean": 381.4501953125, "margin_dpo/margin_std": 563.5973510742188, "step": 547 }, { "KL/chosen_KL_mean": -636.4512939453125, "KL/mean": -818.338134765625, "KL/rejected_KL_mean": -1000.2249755859375, "KL/std": 539.5311279296875, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 363.773681640625, "fcm_dpo/q_t": 0.4263709485530853, "grad_norm": 24.599536895751953, "learning_rate": 5.684919345471029e-08, "logits/chosen": -0.9489472508430481, "logits/rejected": -0.9488046169281006, "logps/chosen": -701.6976318359375, "logps/ref_chosen": -65.24634552001953, "logps/ref_rejected": -94.11807250976562, "logps/rejected": -1094.343017578125, "loss": 1.1491, "margin_dpo/margin_mean": 363.773681640625, "margin_dpo/margin_std": 603.0093994140625, "step": 548 }, { "KL/chosen_KL_mean": -664.5079345703125, "KL/mean": -797.922119140625, "KL/rejected_KL_mean": -931.3363037109375, "KL/std": 454.3035888671875, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 266.828369140625, "fcm_dpo/q_t": 0.4441815912723541, "grad_norm": 32.23136901855469, "learning_rate": 5.603696935852426e-08, "logits/chosen": -0.939894437789917, "logits/rejected": -0.9260774850845337, "logps/chosen": -713.7202758789062, "logps/ref_chosen": -49.21235656738281, "logps/ref_rejected": -73.91031646728516, "logps/rejected": -1005.24658203125, "loss": 1.2198, "margin_dpo/margin_mean": 266.828369140625, "margin_dpo/margin_std": 551.3993530273438, "step": 549 }, { "KL/chosen_KL_mean": -666.9960327148438, "KL/mean": -836.1296997070312, "KL/rejected_KL_mean": -1005.263427734375, "KL/std": 507.13153076171875, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0009084090124815702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 338.26739501953125, "fcm_dpo/q_t": 0.4283323884010315, "grad_norm": 34.777339935302734, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -0.8918087482452393, "logits/rejected": -0.9147313237190247, "logps/chosen": -723.802978515625, "logps/ref_chosen": -56.80695343017578, "logps/ref_rejected": -95.12580871582031, "logps/rejected": -1100.38916015625, "loss": 1.1613, "margin_dpo/margin_mean": 338.26739501953125, "margin_dpo/margin_std": 551.6748046875, "step": 550 }, { "KL/chosen_KL_mean": -579.2154541015625, "KL/mean": -854.2412719726562, "KL/rejected_KL_mean": -1129.26708984375, "KL/std": 542.586181640625, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.000893935386557132, "fcm_dpo/delta": -0.09667488187551498, "fcm_dpo/margin": 550.0517578125, "fcm_dpo/q_t": 0.38580551743507385, "grad_norm": 45.99151611328125, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -0.8883758187294006, "logits/rejected": -0.944542407989502, "logps/chosen": -638.32177734375, "logps/ref_chosen": -59.10633087158203, "logps/ref_rejected": -111.67280578613281, "logps/rejected": -1240.93994140625, "loss": 1.0109, "margin_dpo/margin_mean": 550.0516357421875, "margin_dpo/margin_std": 557.3201293945312, "step": 551 }, { "KL/chosen_KL_mean": -577.863037109375, "KL/mean": -876.5994873046875, "KL/rejected_KL_mean": -1175.335693359375, "KL/std": 629.6008911132812, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.000866841000970453, "fcm_dpo/delta": -0.13752080500125885, "fcm_dpo/margin": 597.4727783203125, "fcm_dpo/q_t": 0.385326087474823, "grad_norm": 36.25246047973633, "learning_rate": 5.363104864490034e-08, "logits/chosen": -0.968271017074585, "logits/rejected": -1.0044474601745605, "logps/chosen": -640.2176513671875, "logps/ref_chosen": -62.35459899902344, "logps/ref_rejected": -104.56210327148438, "logps/rejected": -1279.89794921875, "loss": 1.0223, "margin_dpo/margin_mean": 597.4727783203125, "margin_dpo/margin_std": 725.0616455078125, "step": 552 }, { "KL/chosen_KL_mean": -655.1487426757812, "KL/mean": -812.9437255859375, "KL/rejected_KL_mean": -970.7386474609375, "KL/std": 523.8294067382812, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.000866841000970453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 315.5899353027344, "fcm_dpo/q_t": 0.4384046494960785, "grad_norm": 26.22563362121582, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -0.9064982533454895, "logits/rejected": -0.9010224938392639, "logps/chosen": -723.4075927734375, "logps/ref_chosen": -68.25881958007812, "logps/ref_rejected": -98.0971450805664, "logps/rejected": -1068.8358154296875, "loss": 1.1923, "margin_dpo/margin_mean": 315.5899353027344, "margin_dpo/margin_std": 594.8109130859375, "step": 553 }, { "KL/chosen_KL_mean": -673.444580078125, "KL/mean": -887.5985717773438, "KL/rejected_KL_mean": -1101.752685546875, "KL/std": 560.7201538085938, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.000863053894136101, "fcm_dpo/delta": -0.04388042911887169, "fcm_dpo/margin": 428.30804443359375, "fcm_dpo/q_t": 0.41699787974357605, "grad_norm": 47.83314514160156, "learning_rate": 5.205293880283551e-08, "logits/chosen": -0.8999230861663818, "logits/rejected": -0.8767585158348083, "logps/chosen": -741.3922119140625, "logps/ref_chosen": -67.94767761230469, "logps/ref_rejected": -89.78272247314453, "logps/rejected": -1191.535400390625, "loss": 1.1412, "margin_dpo/margin_mean": 428.30804443359375, "margin_dpo/margin_std": 716.1023559570312, "step": 554 }, { "KL/chosen_KL_mean": -677.0280151367188, "KL/mean": -928.472412109375, "KL/rejected_KL_mean": -1179.916748046875, "KL/std": 603.3981323242188, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.000859266787301749, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 502.8887023925781, "fcm_dpo/q_t": 0.4060073494911194, "grad_norm": 28.9240665435791, "learning_rate": 5.127169765359515e-08, "logits/chosen": -0.9636756181716919, "logits/rejected": -1.018855333328247, "logps/chosen": -730.3585205078125, "logps/ref_chosen": -53.33049011230469, "logps/ref_rejected": -108.47937774658203, "logps/rejected": -1288.3961181640625, "loss": 1.1002, "margin_dpo/margin_mean": 502.8887023925781, "margin_dpo/margin_std": 782.5892944335938, "step": 555 }, { "KL/chosen_KL_mean": -670.974853515625, "KL/mean": -819.40380859375, "KL/rejected_KL_mean": -967.832763671875, "KL/std": 458.6606140136719, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.000859266787301749, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 296.85791015625, "fcm_dpo/q_t": 0.4399312734603882, "grad_norm": 25.964445114135742, "learning_rate": 5.049569317994012e-08, "logits/chosen": -0.9628030061721802, "logits/rejected": -0.9593526124954224, "logps/chosen": -729.6193237304688, "logps/ref_chosen": -58.64447021484375, "logps/ref_rejected": -101.34040832519531, "logps/rejected": -1069.1732177734375, "loss": 1.1873, "margin_dpo/margin_mean": 296.85791015625, "margin_dpo/margin_std": 475.78826904296875, "step": 556 }, { "KL/chosen_KL_mean": -718.9598388671875, "KL/mean": -942.1116943359375, "KL/rejected_KL_mean": -1165.263671875, "KL/std": 635.51220703125, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.000859266787301749, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 446.30389404296875, "fcm_dpo/q_t": 0.41387179493904114, "grad_norm": 57.082557678222656, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -0.9800699949264526, "logits/rejected": -1.0134789943695068, "logps/chosen": -786.8004150390625, "logps/ref_chosen": -67.84066009521484, "logps/ref_rejected": -109.93965911865234, "logps/rejected": -1275.203369140625, "loss": 1.1283, "margin_dpo/margin_mean": 446.30389404296875, "margin_dpo/margin_std": 714.6304321289062, "step": 557 }, { "KL/chosen_KL_mean": -653.600830078125, "KL/mean": -958.6536865234375, "KL/rejected_KL_mean": -1263.70654296875, "KL/std": 602.1583251953125, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0008436896023340523, "fcm_dpo/delta": -0.12100745737552643, "fcm_dpo/margin": 610.1058349609375, "fcm_dpo/q_t": 0.3799861669540405, "grad_norm": 30.4708194732666, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -1.0119829177856445, "logits/rejected": -1.0322072505950928, "logps/chosen": -715.9690551757812, "logps/ref_chosen": -62.36824035644531, "logps/ref_rejected": -102.16102600097656, "logps/rejected": -1365.86767578125, "loss": 1.0123, "margin_dpo/margin_mean": 610.1058349609375, "margin_dpo/margin_std": 676.6812133789062, "step": 558 }, { "KL/chosen_KL_mean": -751.7446899414062, "KL/mean": -994.5537109375, "KL/rejected_KL_mean": -1237.362548828125, "KL/std": 611.50537109375, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.000838720821775496, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 485.61785888671875, "fcm_dpo/q_t": 0.40765488147735596, "grad_norm": 27.252429962158203, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -1.1454588174819946, "logits/rejected": -1.1528816223144531, "logps/chosen": -812.4970703125, "logps/ref_chosen": -60.752323150634766, "logps/ref_rejected": -93.44229125976562, "logps/rejected": -1330.804931640625, "loss": 1.0922, "margin_dpo/margin_mean": 485.617919921875, "margin_dpo/margin_std": 667.4962158203125, "step": 559 }, { "KL/chosen_KL_mean": -696.612548828125, "KL/mean": -864.5140991210938, "KL/rejected_KL_mean": -1032.4156494140625, "KL/std": 541.3946533203125, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.000838720821775496, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 335.80303955078125, "fcm_dpo/q_t": 0.43424075841903687, "grad_norm": 36.40821075439453, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -0.8833180069923401, "logits/rejected": -0.8601206541061401, "logps/chosen": -754.7164306640625, "logps/ref_chosen": -58.10382080078125, "logps/ref_rejected": -79.99122619628906, "logps/rejected": -1112.4068603515625, "loss": 1.1824, "margin_dpo/margin_mean": 335.8030090332031, "margin_dpo/margin_std": 580.6434326171875, "step": 560 }, { "KL/chosen_KL_mean": -802.1702880859375, "KL/mean": -957.013427734375, "KL/rejected_KL_mean": -1111.8564453125, "KL/std": 551.462890625, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.000838720821775496, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 309.6861572265625, "fcm_dpo/q_t": 0.43889930844306946, "grad_norm": 37.50949478149414, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.0387603044509888, "logits/rejected": -1.0596671104431152, "logps/chosen": -853.0831298828125, "logps/ref_chosen": -50.912879943847656, "logps/ref_rejected": -99.06856536865234, "logps/rejected": -1210.925048828125, "loss": 1.2241, "margin_dpo/margin_mean": 309.6861877441406, "margin_dpo/margin_std": 698.983642578125, "step": 561 }, { "KL/chosen_KL_mean": -746.2704467773438, "KL/mean": -969.857177734375, "KL/rejected_KL_mean": -1193.44384765625, "KL/std": 613.4844970703125, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0008317689062096179, "fcm_dpo/delta": -0.04161640629172325, "fcm_dpo/margin": 447.1734924316406, "fcm_dpo/q_t": 0.4157896041870117, "grad_norm": 28.799089431762695, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -1.0045117139816284, "logits/rejected": -1.021229863166809, "logps/chosen": -805.73486328125, "logps/ref_chosen": -59.46440124511719, "logps/ref_rejected": -96.54266357421875, "logps/rejected": -1289.986572265625, "loss": 1.1226, "margin_dpo/margin_mean": 447.1734924316406, "margin_dpo/margin_std": 680.6741333007812, "step": 562 }, { "KL/chosen_KL_mean": -833.0841064453125, "KL/mean": -988.2342529296875, "KL/rejected_KL_mean": -1143.384521484375, "KL/std": 653.4237060546875, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0008317689062096179, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 310.3004150390625, "fcm_dpo/q_t": 0.4358706772327423, "grad_norm": 54.44693374633789, "learning_rate": 4.521198892775202e-08, "logits/chosen": -0.9969061613082886, "logits/rejected": -1.0025964975357056, "logps/chosen": -893.6923217773438, "logps/ref_chosen": -60.60819625854492, "logps/ref_rejected": -94.56770324707031, "logps/rejected": -1237.9521484375, "loss": 1.25, "margin_dpo/margin_mean": 310.3004150390625, "margin_dpo/margin_std": 777.999267578125, "step": 563 }, { "KL/chosen_KL_mean": -749.1875, "KL/mean": -958.049072265625, "KL/rejected_KL_mean": -1166.91064453125, "KL/std": 577.4024658203125, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0008317689062096179, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 417.72308349609375, "fcm_dpo/q_t": 0.41977375745773315, "grad_norm": 37.301246643066406, "learning_rate": 4.447860229910544e-08, "logits/chosen": -1.0899568796157837, "logits/rejected": -1.0908780097961426, "logps/chosen": -823.4559326171875, "logps/ref_chosen": -74.26837921142578, "logps/ref_rejected": -93.23818969726562, "logps/rejected": -1260.1488037109375, "loss": 1.1222, "margin_dpo/margin_mean": 417.72308349609375, "margin_dpo/margin_std": 562.4461669921875, "step": 564 }, { "KL/chosen_KL_mean": -788.7421264648438, "KL/mean": -1010.16650390625, "KL/rejected_KL_mean": -1231.5908203125, "KL/std": 673.0230712890625, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0008317689062096179, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 442.8487548828125, "fcm_dpo/q_t": 0.4197535514831543, "grad_norm": 43.9473991394043, "learning_rate": 4.375063135042445e-08, "logits/chosen": -0.9752411842346191, "logits/rejected": -0.9796169996261597, "logps/chosen": -857.7620849609375, "logps/ref_chosen": -69.0199203491211, "logps/ref_rejected": -85.7789306640625, "logps/rejected": -1317.369873046875, "loss": 1.1576, "margin_dpo/margin_mean": 442.8487548828125, "margin_dpo/margin_std": 822.1761474609375, "step": 565 }, { "KL/chosen_KL_mean": -741.95458984375, "KL/mean": -990.75732421875, "KL/rejected_KL_mean": -1239.56005859375, "KL/std": 691.6851196289062, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0008263904601335526, "fcm_dpo/delta": -0.06508417427539825, "fcm_dpo/margin": 497.6054382324219, "fcm_dpo/q_t": 0.40809524059295654, "grad_norm": 30.059511184692383, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.0234966278076172, "logits/rejected": -1.0528680086135864, "logps/chosen": -808.4998779296875, "logps/ref_chosen": -66.5453109741211, "logps/ref_rejected": -103.86932373046875, "logps/rejected": -1343.4293212890625, "loss": 1.1205, "margin_dpo/margin_mean": 497.6054382324219, "margin_dpo/margin_std": 798.7295532226562, "step": 566 }, { "KL/chosen_KL_mean": -725.8890380859375, "KL/mean": -903.4608154296875, "KL/rejected_KL_mean": -1081.032470703125, "KL/std": 483.6868896484375, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.0008210120722651482, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 355.1435546875, "fcm_dpo/q_t": 0.43069371581077576, "grad_norm": 33.07154846191406, "learning_rate": 4.231101308059165e-08, "logits/chosen": -1.095928430557251, "logits/rejected": -1.1078492403030396, "logps/chosen": -778.747314453125, "logps/ref_chosen": -52.85829544067383, "logps/ref_rejected": -85.37095642089844, "logps/rejected": -1166.403564453125, "loss": 1.1605, "margin_dpo/margin_mean": 355.1435546875, "margin_dpo/margin_std": 527.5757446289062, "step": 567 }, { "KL/chosen_KL_mean": -705.9122314453125, "KL/mean": -956.9530029296875, "KL/rejected_KL_mean": -1207.99365234375, "KL/std": 561.6522216796875, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0008210120722651482, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 502.0815124511719, "fcm_dpo/q_t": 0.4041532874107361, "grad_norm": 27.16654396057129, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.0139139890670776, "logits/rejected": -1.053145170211792, "logps/chosen": -751.1046142578125, "logps/ref_chosen": -45.1923828125, "logps/ref_rejected": -89.09236907958984, "logps/rejected": -1297.0860595703125, "loss": 1.0734, "margin_dpo/margin_mean": 502.08154296875, "margin_dpo/margin_std": 605.7379150390625, "step": 568 }, { "KL/chosen_KL_mean": -794.1281127929688, "KL/mean": -997.6332397460938, "KL/rejected_KL_mean": -1201.138427734375, "KL/std": 705.3419189453125, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0008191708475351334, "fcm_dpo/delta": -0.022476255893707275, "fcm_dpo/margin": 407.01025390625, "fcm_dpo/q_t": 0.42279374599456787, "grad_norm": 37.293025970458984, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.0435158014297485, "logits/rejected": -1.0531883239746094, "logps/chosen": -857.8486938476562, "logps/ref_chosen": -63.72056198120117, "logps/ref_rejected": -79.10325622558594, "logps/rejected": -1280.2415771484375, "loss": 1.1715, "margin_dpo/margin_mean": 407.01025390625, "margin_dpo/margin_std": 746.8839111328125, "step": 569 }, { "KL/chosen_KL_mean": -751.165771484375, "KL/mean": -937.5892333984375, "KL/rejected_KL_mean": -1124.0125732421875, "KL/std": 564.7623901367188, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0008173296810127795, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 372.8468017578125, "fcm_dpo/q_t": 0.43035823106765747, "grad_norm": 28.89210319519043, "learning_rate": 4.019267817841834e-08, "logits/chosen": -1.108044147491455, "logits/rejected": -1.1026582717895508, "logps/chosen": -812.7803955078125, "logps/ref_chosen": -61.61454391479492, "logps/ref_rejected": -82.14186096191406, "logps/rejected": -1206.154541015625, "loss": 1.1683, "margin_dpo/margin_mean": 372.8468017578125, "margin_dpo/margin_std": 630.748291015625, "step": 570 }, { "KL/chosen_KL_mean": -763.8045654296875, "KL/mean": -990.728515625, "KL/rejected_KL_mean": -1217.65234375, "KL/std": 579.8861083984375, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.0008173296810127795, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 453.8477783203125, "fcm_dpo/q_t": 0.41571101546287537, "grad_norm": 36.950618743896484, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.000840663909912, "logits/rejected": -1.015519142150879, "logps/chosen": -816.858642578125, "logps/ref_chosen": -53.05406188964844, "logps/ref_rejected": -91.33682250976562, "logps/rejected": -1308.9891357421875, "loss": 1.1287, "margin_dpo/margin_mean": 453.8477783203125, "margin_dpo/margin_std": 714.1256103515625, "step": 571 }, { "KL/chosen_KL_mean": -789.1917114257812, "KL/mean": -1021.33642578125, "KL/rejected_KL_mean": -1253.481201171875, "KL/std": 666.8356323242188, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0008173296810127795, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 464.2894287109375, "fcm_dpo/q_t": 0.417039155960083, "grad_norm": 28.30565643310547, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.017333984375, "logits/rejected": -1.0511081218719482, "logps/chosen": -837.6510009765625, "logps/ref_chosen": -48.45928955078125, "logps/ref_rejected": -83.55703735351562, "logps/rejected": -1337.0382080078125, "loss": 1.1434, "margin_dpo/margin_mean": 464.2894592285156, "margin_dpo/margin_std": 822.282958984375, "step": 572 }, { "KL/chosen_KL_mean": -799.183349609375, "KL/mean": -1003.74462890625, "KL/rejected_KL_mean": -1208.3060302734375, "KL/std": 615.51416015625, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0008173296810127795, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 409.12261962890625, "fcm_dpo/q_t": 0.4239146113395691, "grad_norm": 31.040990829467773, "learning_rate": 3.812409996461275e-08, "logits/chosen": -1.0863670110702515, "logits/rejected": -1.096688985824585, "logps/chosen": -850.8059692382812, "logps/ref_chosen": -51.62262725830078, "logps/ref_rejected": -85.32499694824219, "logps/rejected": -1293.6309814453125, "loss": 1.1352, "margin_dpo/margin_mean": 409.12261962890625, "margin_dpo/margin_std": 608.7803344726562, "step": 573 }, { "KL/chosen_KL_mean": -725.0342407226562, "KL/mean": -954.8162231445312, "KL/rejected_KL_mean": -1184.5982666015625, "KL/std": 558.27392578125, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0008173296810127795, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 459.56402587890625, "fcm_dpo/q_t": 0.4135124981403351, "grad_norm": 31.780391693115234, "learning_rate": 3.74457160675965e-08, "logits/chosen": -1.0929412841796875, "logits/rejected": -1.1233415603637695, "logps/chosen": -776.0787353515625, "logps/ref_chosen": -51.04446029663086, "logps/ref_rejected": -92.80640411376953, "logps/rejected": -1277.4046630859375, "loss": 1.1118, "margin_dpo/margin_mean": 459.5640563964844, "margin_dpo/margin_std": 647.52099609375, "step": 574 }, { "KL/chosen_KL_mean": -799.796875, "KL/mean": -1004.4361572265625, "KL/rejected_KL_mean": -1209.075439453125, "KL/std": 569.8582153320312, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0008112110663205385, "fcm_dpo/delta": -0.03757143393158913, "fcm_dpo/margin": 409.27850341796875, "fcm_dpo/q_t": 0.42343461513519287, "grad_norm": 33.69365692138672, "learning_rate": 3.677293317363864e-08, "logits/chosen": -0.9379447102546692, "logits/rejected": -0.9466258883476257, "logps/chosen": -871.5870361328125, "logps/ref_chosen": -71.7901382446289, "logps/ref_rejected": -95.38619995117188, "logps/rejected": -1304.461669921875, "loss": 1.1679, "margin_dpo/margin_mean": 409.27850341796875, "margin_dpo/margin_std": 728.100341796875, "step": 575 }, { "KL/chosen_KL_mean": -759.4315185546875, "KL/mean": -915.8280029296875, "KL/rejected_KL_mean": -1072.2244873046875, "KL/std": 518.5361328125, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.0008112110663205385, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 312.79302978515625, "fcm_dpo/q_t": 0.4410494565963745, "grad_norm": 25.922910690307617, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.0505657196044922, "logits/rejected": -1.0803673267364502, "logps/chosen": -813.6944580078125, "logps/ref_chosen": -54.262962341308594, "logps/ref_rejected": -100.75428009033203, "logps/rejected": -1172.978759765625, "loss": 1.2076, "margin_dpo/margin_mean": 312.79302978515625, "margin_dpo/margin_std": 605.8926391601562, "step": 576 }, { "KL/chosen_KL_mean": -682.141357421875, "KL/mean": -892.861328125, "KL/rejected_KL_mean": -1103.581298828125, "KL/std": 589.8372802734375, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0008112110663205385, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 421.4400329589844, "fcm_dpo/q_t": 0.4217475652694702, "grad_norm": 27.589473724365234, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.0263875722885132, "logits/rejected": -1.0151700973510742, "logps/chosen": -744.051025390625, "logps/ref_chosen": -61.909706115722656, "logps/ref_rejected": -84.07069396972656, "logps/rejected": -1187.652099609375, "loss": 1.1455, "margin_dpo/margin_mean": 421.44000244140625, "margin_dpo/margin_std": 642.5458984375, "step": 577 }, { "KL/chosen_KL_mean": -675.6703491210938, "KL/mean": -901.8224487304688, "KL/rejected_KL_mean": -1127.9744873046875, "KL/std": 575.355712890625, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.0008064049761742353, "fcm_dpo/delta": -0.02971097081899643, "fcm_dpo/margin": 452.30413818359375, "fcm_dpo/q_t": 0.41487276554107666, "grad_norm": 40.21394729614258, "learning_rate": 3.478836705390808e-08, "logits/chosen": -0.9069193005561829, "logits/rejected": -0.9442964196205139, "logps/chosen": -724.93408203125, "logps/ref_chosen": -49.26368713378906, "logps/ref_rejected": -83.4362564086914, "logps/rejected": -1211.4107666015625, "loss": 1.1069, "margin_dpo/margin_mean": 452.30413818359375, "margin_dpo/margin_std": 572.793701171875, "step": 578 }, { "KL/chosen_KL_mean": -781.8958129882812, "KL/mean": -914.868896484375, "KL/rejected_KL_mean": -1047.8419189453125, "KL/std": 587.2415161132812, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0008064049761742353, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 265.9461669921875, "fcm_dpo/q_t": 0.44937944412231445, "grad_norm": 66.76544189453125, "learning_rate": 3.41381639738331e-08, "logits/chosen": -0.9628750085830688, "logits/rejected": -0.9616245031356812, "logps/chosen": -840.7816162109375, "logps/ref_chosen": -58.88581848144531, "logps/ref_rejected": -94.78762817382812, "logps/rejected": -1142.629638671875, "loss": 1.2513, "margin_dpo/margin_mean": 265.9461669921875, "margin_dpo/margin_std": 656.545654296875, "step": 579 }, { "KL/chosen_KL_mean": -554.861083984375, "KL/mean": -820.6229248046875, "KL/rejected_KL_mean": -1086.384765625, "KL/std": 653.8794555664062, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0007996794884093106, "fcm_dpo/delta": -0.04187547415494919, "fcm_dpo/margin": 531.5237426757812, "fcm_dpo/q_t": 0.40619370341300964, "grad_norm": 29.74520492553711, "learning_rate": 3.349364905389032e-08, "logits/chosen": -0.8521532416343689, "logits/rejected": -0.8880908489227295, "logps/chosen": -603.56787109375, "logps/ref_chosen": -48.70683670043945, "logps/ref_rejected": -81.7583999633789, "logps/rejected": -1168.143310546875, "loss": 1.0849, "margin_dpo/margin_mean": 531.5238037109375, "margin_dpo/margin_std": 759.3482666015625, "step": 580 }, { "KL/chosen_KL_mean": -741.0847778320312, "KL/mean": -923.89697265625, "KL/rejected_KL_mean": -1106.7091064453125, "KL/std": 567.8367919921875, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0007996794884093106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 365.624267578125, "fcm_dpo/q_t": 0.43274974822998047, "grad_norm": 26.411640167236328, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.067054033279419, "logits/rejected": -1.0800729990005493, "logps/chosen": -803.30712890625, "logps/ref_chosen": -62.22235107421875, "logps/ref_rejected": -91.73568725585938, "logps/rejected": -1198.44482421875, "loss": 1.177, "margin_dpo/margin_mean": 365.624267578125, "margin_dpo/margin_std": 645.589111328125, "step": 581 }, { "KL/chosen_KL_mean": -692.6724853515625, "KL/mean": -903.4718017578125, "KL/rejected_KL_mean": -1114.271240234375, "KL/std": 547.7958984375, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0007957734633237123, "fcm_dpo/delta": -0.04908556863665581, "fcm_dpo/margin": 421.5987548828125, "fcm_dpo/q_t": 0.42105481028556824, "grad_norm": 29.055042266845703, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.0077698230743408, "logits/rejected": -1.030693769454956, "logps/chosen": -750.901123046875, "logps/ref_chosen": -58.228660583496094, "logps/ref_rejected": -110.06959533691406, "logps/rejected": -1224.3408203125, "loss": 1.1379, "margin_dpo/margin_mean": 421.5987548828125, "margin_dpo/margin_std": 591.310791015625, "step": 582 }, { "KL/chosen_KL_mean": -767.380859375, "KL/mean": -899.6068115234375, "KL/rejected_KL_mean": -1031.832763671875, "KL/std": 589.245361328125, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 264.45184326171875, "fcm_dpo/q_t": 0.452210009098053, "grad_norm": 37.549983978271484, "learning_rate": 3.159440233840763e-08, "logits/chosen": -0.940156102180481, "logits/rejected": -0.9366773962974548, "logps/chosen": -824.2437744140625, "logps/ref_chosen": -56.86286163330078, "logps/ref_rejected": -88.4039306640625, "logps/rejected": -1120.2366943359375, "loss": 1.2701, "margin_dpo/margin_mean": 264.45184326171875, "margin_dpo/margin_std": 711.6224975585938, "step": 583 }, { "KL/chosen_KL_mean": -665.3277587890625, "KL/mean": -932.5770874023438, "KL/rejected_KL_mean": -1199.826416015625, "KL/std": 615.3153686523438, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 534.498779296875, "fcm_dpo/q_t": 0.4037357568740845, "grad_norm": 26.586563110351562, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.0058352947235107, "logits/rejected": -1.0271427631378174, "logps/chosen": -722.2283935546875, "logps/ref_chosen": -56.90068054199219, "logps/ref_rejected": -97.63606262207031, "logps/rejected": -1297.4625244140625, "loss": 1.0674, "margin_dpo/margin_mean": 534.498779296875, "margin_dpo/margin_std": 645.0361328125, "step": 584 }, { "KL/chosen_KL_mean": -809.52587890625, "KL/mean": -1037.6292724609375, "KL/rejected_KL_mean": -1265.732666015625, "KL/std": 684.6345825195312, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 456.20672607421875, "fcm_dpo/q_t": 0.4182851314544678, "grad_norm": 28.249164581298828, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.0811936855316162, "logits/rejected": -1.1092218160629272, "logps/chosen": -870.4998168945312, "logps/ref_chosen": -60.973968505859375, "logps/ref_rejected": -84.16952514648438, "logps/rejected": -1349.902099609375, "loss": 1.155, "margin_dpo/margin_mean": 456.2066955566406, "margin_dpo/margin_std": 826.325439453125, "step": 585 }, { "KL/chosen_KL_mean": -812.726318359375, "KL/mean": -970.64892578125, "KL/rejected_KL_mean": -1128.571533203125, "KL/std": 599.4317626953125, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 315.8452453613281, "fcm_dpo/q_t": 0.44157183170318604, "grad_norm": 30.333675384521484, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.034977674484253, "logits/rejected": -1.0590405464172363, "logps/chosen": -869.5819091796875, "logps/ref_chosen": -56.85559844970703, "logps/ref_rejected": -91.80261993408203, "logps/rejected": -1220.374267578125, "loss": 1.2157, "margin_dpo/margin_mean": 315.8452453613281, "margin_dpo/margin_std": 651.1234130859375, "step": 586 }, { "KL/chosen_KL_mean": -565.1972045898438, "KL/mean": -782.4954833984375, "KL/rejected_KL_mean": -999.7936401367188, "KL/std": 601.401123046875, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 434.596435546875, "fcm_dpo/q_t": 0.4192296862602234, "grad_norm": 25.787561416625977, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -1.0264978408813477, "logits/rejected": -1.0559778213500977, "logps/chosen": -609.8887939453125, "logps/ref_chosen": -44.69159698486328, "logps/ref_rejected": -82.62385559082031, "logps/rejected": -1082.41748046875, "loss": 1.1286, "margin_dpo/margin_mean": 434.596435546875, "margin_dpo/margin_std": 620.5191040039062, "step": 587 }, { "KL/chosen_KL_mean": -749.8773193359375, "KL/mean": -935.8150024414062, "KL/rejected_KL_mean": -1121.752685546875, "KL/std": 530.2380981445312, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 371.8753662109375, "fcm_dpo/q_t": 0.43039530515670776, "grad_norm": 32.1988410949707, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.0299625396728516, "logits/rejected": -1.055286169052124, "logps/chosen": -800.1722412109375, "logps/ref_chosen": -50.29494857788086, "logps/ref_rejected": -107.36988067626953, "logps/rejected": -1229.12255859375, "loss": 1.1617, "margin_dpo/margin_mean": 371.8753662109375, "margin_dpo/margin_std": 559.5547485351562, "step": 588 }, { "KL/chosen_KL_mean": -759.330810546875, "KL/mean": -999.5764770507812, "KL/rejected_KL_mean": -1239.8221435546875, "KL/std": 591.2251586914062, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 480.4913024902344, "fcm_dpo/q_t": 0.4115689992904663, "grad_norm": 29.426036834716797, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -0.9973963499069214, "logits/rejected": -1.0138342380523682, "logps/chosen": -819.2607421875, "logps/ref_chosen": -59.929908752441406, "logps/ref_rejected": -111.65534973144531, "logps/rejected": -1351.4775390625, "loss": 1.098, "margin_dpo/margin_mean": 480.4913024902344, "margin_dpo/margin_std": 619.790771484375, "step": 589 }, { "KL/chosen_KL_mean": -664.0067749023438, "KL/mean": -904.7217407226562, "KL/rejected_KL_mean": -1145.436767578125, "KL/std": 613.1650390625, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 481.42999267578125, "fcm_dpo/q_t": 0.41361021995544434, "grad_norm": 24.00367546081543, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.0117418766021729, "logits/rejected": -1.039717435836792, "logps/chosen": -719.8165283203125, "logps/ref_chosen": -55.80979537963867, "logps/ref_rejected": -106.06282043457031, "logps/rejected": -1251.49951171875, "loss": 1.1077, "margin_dpo/margin_mean": 481.42999267578125, "margin_dpo/margin_std": 685.4844970703125, "step": 590 }, { "KL/chosen_KL_mean": -728.047119140625, "KL/mean": -937.2919311523438, "KL/rejected_KL_mean": -1146.5367431640625, "KL/std": 563.9609375, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 418.4896240234375, "fcm_dpo/q_t": 0.4223765730857849, "grad_norm": 30.11712074279785, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.0658780336380005, "logits/rejected": -1.0678385496139526, "logps/chosen": -784.2877197265625, "logps/ref_chosen": -56.24061965942383, "logps/ref_rejected": -83.78629302978516, "logps/rejected": -1230.322998046875, "loss": 1.1451, "margin_dpo/margin_mean": 418.48968505859375, "margin_dpo/margin_std": 651.3126220703125, "step": 591 }, { "KL/chosen_KL_mean": -759.2822875976562, "KL/mean": -946.3499755859375, "KL/rejected_KL_mean": -1133.417724609375, "KL/std": 568.3065185546875, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 374.1354064941406, "fcm_dpo/q_t": 0.431357741355896, "grad_norm": 27.800474166870117, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.065044641494751, "logits/rejected": -1.0687838792800903, "logps/chosen": -807.2225341796875, "logps/ref_chosen": -47.94025421142578, "logps/ref_rejected": -75.73287963867188, "logps/rejected": -1209.150634765625, "loss": 1.2046, "margin_dpo/margin_mean": 374.13543701171875, "margin_dpo/margin_std": 777.451416015625, "step": 592 }, { "KL/chosen_KL_mean": -711.5416259765625, "KL/mean": -860.16455078125, "KL/rejected_KL_mean": -1008.7875366210938, "KL/std": 603.8402099609375, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.0007918673800304532, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 297.2458801269531, "fcm_dpo/q_t": 0.44561654329299927, "grad_norm": 39.802852630615234, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.9177390336990356, "logits/rejected": -0.9115041494369507, "logps/chosen": -760.232421875, "logps/ref_chosen": -48.690757751464844, "logps/ref_rejected": -60.90800094604492, "logps/rejected": -1069.695556640625, "loss": 1.2348, "margin_dpo/margin_mean": 297.2458801269531, "margin_dpo/margin_std": 679.549560546875, "step": 593 }, { "KL/chosen_KL_mean": -699.04541015625, "KL/mean": -888.05126953125, "KL/rejected_KL_mean": -1077.05712890625, "KL/std": 604.8076782226562, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0007907521212473512, "fcm_dpo/delta": -0.014104235917329788, "fcm_dpo/margin": 378.01171875, "fcm_dpo/q_t": 0.43051910400390625, "grad_norm": 28.592931747436523, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.0373632907867432, "logits/rejected": -1.0325448513031006, "logps/chosen": -753.9803466796875, "logps/ref_chosen": -54.93488693237305, "logps/ref_rejected": -86.09967803955078, "logps/rejected": -1163.1568603515625, "loss": 1.1764, "margin_dpo/margin_mean": 378.01171875, "margin_dpo/margin_std": 645.0611572265625, "step": 594 }, { "KL/chosen_KL_mean": -658.1973876953125, "KL/mean": -873.8628540039062, "KL/rejected_KL_mean": -1089.5283203125, "KL/std": 567.3275146484375, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0007896368042565882, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 431.3309326171875, "fcm_dpo/q_t": 0.42307084798812866, "grad_norm": 42.018104553222656, "learning_rate": 2.451969280180849e-08, "logits/chosen": -0.970179557800293, "logits/rejected": -0.9921086430549622, "logps/chosen": -707.6177978515625, "logps/ref_chosen": -49.4204216003418, "logps/ref_rejected": -80.62731170654297, "logps/rejected": -1170.1556396484375, "loss": 1.1329, "margin_dpo/margin_mean": 431.3309631347656, "margin_dpo/margin_std": 637.8607788085938, "step": 595 }, { "KL/chosen_KL_mean": -758.0401611328125, "KL/mean": -909.93603515625, "KL/rejected_KL_mean": -1061.83203125, "KL/std": 575.744873046875, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0007896368042565882, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 303.7917785644531, "fcm_dpo/q_t": 0.44556793570518494, "grad_norm": 48.209266662597656, "learning_rate": 2.396839494982103e-08, "logits/chosen": -0.9976510405540466, "logits/rejected": -0.9678058624267578, "logps/chosen": -817.8319091796875, "logps/ref_chosen": -59.791683197021484, "logps/ref_rejected": -80.09111785888672, "logps/rejected": -1141.923095703125, "loss": 1.2296, "margin_dpo/margin_mean": 303.791748046875, "margin_dpo/margin_std": 685.4385986328125, "step": 596 }, { "KL/chosen_KL_mean": -714.3123779296875, "KL/mean": -975.7779541015625, "KL/rejected_KL_mean": -1237.2435302734375, "KL/std": 652.4100341796875, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0007754022371955216, "fcm_dpo/delta": -0.09095588326454163, "fcm_dpo/margin": 522.9312133789062, "fcm_dpo/q_t": 0.407283753156662, "grad_norm": 21.951868057250977, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -0.9390630722045898, "logits/rejected": -0.9882240295410156, "logps/chosen": -771.5731201171875, "logps/ref_chosen": -57.26078796386719, "logps/ref_rejected": -100.6937255859375, "logps/rejected": -1337.937255859375, "loss": 1.0969, "margin_dpo/margin_mean": 522.9312133789062, "margin_dpo/margin_std": 705.5875244140625, "step": 597 }, { "KL/chosen_KL_mean": -718.0220947265625, "KL/mean": -918.1834106445312, "KL/rejected_KL_mean": -1118.3447265625, "KL/std": 569.5988159179688, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0007724062888883054, "fcm_dpo/delta": -0.019356058910489082, "fcm_dpo/margin": 400.3226623535156, "fcm_dpo/q_t": 0.42755886912345886, "grad_norm": 36.96177291870117, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.0392732620239258, "logits/rejected": -1.0519664287567139, "logps/chosen": -770.5406494140625, "logps/ref_chosen": -52.51850509643555, "logps/ref_rejected": -89.44385528564453, "logps/rejected": -1207.78857421875, "loss": 1.1634, "margin_dpo/margin_mean": 400.3226623535156, "margin_dpo/margin_std": 644.6939697265625, "step": 598 }, { "KL/chosen_KL_mean": -722.3475952148438, "KL/mean": -891.9060668945312, "KL/rejected_KL_mean": -1061.4644775390625, "KL/std": 531.6864013671875, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0007724062888883054, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 339.116943359375, "fcm_dpo/q_t": 0.4360012114048004, "grad_norm": 26.386310577392578, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -0.9559237957000732, "logits/rejected": -0.9652050733566284, "logps/chosen": -772.1502685546875, "logps/ref_chosen": -49.802677154541016, "logps/ref_rejected": -82.978515625, "logps/rejected": -1144.443115234375, "loss": 1.1851, "margin_dpo/margin_mean": 339.116943359375, "margin_dpo/margin_std": 516.4267578125, "step": 599 }, { "KL/chosen_KL_mean": -781.0634155273438, "KL/mean": -944.1868896484375, "KL/rejected_KL_mean": -1107.310302734375, "KL/std": 575.037841796875, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.0007724062888883054, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 326.24700927734375, "fcm_dpo/q_t": 0.4429008364677429, "grad_norm": 27.988651275634766, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -1.0766232013702393, "logits/rejected": -1.078524112701416, "logps/chosen": -847.498291015625, "logps/ref_chosen": -66.43487548828125, "logps/ref_rejected": -85.45649719238281, "logps/rejected": -1192.766845703125, "loss": 1.2171, "margin_dpo/margin_mean": 326.2469787597656, "margin_dpo/margin_std": 697.498779296875, "step": 600 }, { "KL/chosen_KL_mean": -774.169677734375, "KL/mean": -979.185302734375, "KL/rejected_KL_mean": -1184.200927734375, "KL/std": 580.6097412109375, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0007724062888883054, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 410.0313720703125, "fcm_dpo/q_t": 0.4246867299079895, "grad_norm": 25.131288528442383, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.0171586275100708, "logits/rejected": -1.038601279258728, "logps/chosen": -833.30322265625, "logps/ref_chosen": -59.13361358642578, "logps/ref_rejected": -94.69093322753906, "logps/rejected": -1278.891845703125, "loss": 1.1409, "margin_dpo/margin_mean": 410.0313720703125, "margin_dpo/margin_std": 568.695556640625, "step": 601 }, { "KL/chosen_KL_mean": -545.2393798828125, "KL/mean": -812.69677734375, "KL/rejected_KL_mean": -1080.1541748046875, "KL/std": 542.9259033203125, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0007678022375330329, "fcm_dpo/delta": -0.0599651113152504, "fcm_dpo/margin": 534.914794921875, "fcm_dpo/q_t": 0.4043405055999756, "grad_norm": 52.572025299072266, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -1.060288906097412, "logits/rejected": -1.098497986793518, "logps/chosen": -593.8328857421875, "logps/ref_chosen": -48.59352111816406, "logps/ref_rejected": -87.6685562133789, "logps/rejected": -1167.82275390625, "loss": 1.0672, "margin_dpo/margin_mean": 534.9148559570312, "margin_dpo/margin_std": 592.32177734375, "step": 602 }, { "KL/chosen_KL_mean": -685.7906494140625, "KL/mean": -919.4617919921875, "KL/rejected_KL_mean": -1153.133056640625, "KL/std": 605.2296142578125, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 467.34228515625, "fcm_dpo/q_t": 0.41905176639556885, "grad_norm": 32.92748260498047, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.00080406665802, "logits/rejected": -0.9961497783660889, "logps/chosen": -756.205322265625, "logps/ref_chosen": -70.41461944580078, "logps/ref_rejected": -100.32559967041016, "logps/rejected": -1253.45849609375, "loss": 1.1298, "margin_dpo/margin_mean": 467.34228515625, "margin_dpo/margin_std": 717.1328125, "step": 603 }, { "KL/chosen_KL_mean": -669.0608520507812, "KL/mean": -923.9398193359375, "KL/rejected_KL_mean": -1178.81884765625, "KL/std": 580.1771850585938, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 509.75799560546875, "fcm_dpo/q_t": 0.41114941239356995, "grad_norm": 32.59992218017578, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.008441686630249, "logits/rejected": -1.044859528541565, "logps/chosen": -715.5189208984375, "logps/ref_chosen": -46.45808029174805, "logps/ref_rejected": -91.8544921875, "logps/rejected": -1270.67333984375, "loss": 1.1041, "margin_dpo/margin_mean": 509.7580261230469, "margin_dpo/margin_std": 713.919677734375, "step": 604 }, { "KL/chosen_KL_mean": -706.5936279296875, "KL/mean": -900.7664184570312, "KL/rejected_KL_mean": -1094.939208984375, "KL/std": 555.1638793945312, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 388.345703125, "fcm_dpo/q_t": 0.43225014209747314, "grad_norm": 25.63776397705078, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -0.9656909108161926, "logits/rejected": -0.9828107357025146, "logps/chosen": -772.8428955078125, "logps/ref_chosen": -66.24933624267578, "logps/ref_rejected": -102.30496978759766, "logps/rejected": -1197.2442626953125, "loss": 1.1707, "margin_dpo/margin_mean": 388.34576416015625, "margin_dpo/margin_std": 662.9732055664062, "step": 605 }, { "KL/chosen_KL_mean": -704.7627563476562, "KL/mean": -911.3067626953125, "KL/rejected_KL_mean": -1117.8507080078125, "KL/std": 570.53369140625, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 413.08795166015625, "fcm_dpo/q_t": 0.42691662907600403, "grad_norm": 29.93854331970215, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -0.980074942111969, "logits/rejected": -0.9964933395385742, "logps/chosen": -759.5819091796875, "logps/ref_chosen": -54.819122314453125, "logps/ref_rejected": -98.37146759033203, "logps/rejected": -1216.22216796875, "loss": 1.1487, "margin_dpo/margin_mean": 413.087890625, "margin_dpo/margin_std": 620.9015502929688, "step": 606 }, { "KL/chosen_KL_mean": -739.030029296875, "KL/mean": -908.6751708984375, "KL/rejected_KL_mean": -1078.3203125, "KL/std": 575.9705810546875, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 339.290283203125, "fcm_dpo/q_t": 0.4394988417625427, "grad_norm": 23.68671226501465, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -0.9747462868690491, "logits/rejected": -0.9673136472702026, "logps/chosen": -797.114013671875, "logps/ref_chosen": -58.08403778076172, "logps/ref_rejected": -79.777099609375, "logps/rejected": -1158.097412109375, "loss": 1.1971, "margin_dpo/margin_mean": 339.290283203125, "margin_dpo/margin_std": 617.9051513671875, "step": 607 }, { "KL/chosen_KL_mean": -661.8232421875, "KL/mean": -859.321044921875, "KL/rejected_KL_mean": -1056.81884765625, "KL/std": 510.9901123046875, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 394.99566650390625, "fcm_dpo/q_t": 0.42847907543182373, "grad_norm": 27.059940338134766, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.0220744609832764, "logits/rejected": -1.0379165410995483, "logps/chosen": -719.2740478515625, "logps/ref_chosen": -57.450836181640625, "logps/ref_rejected": -94.77339172363281, "logps/rejected": -1151.59228515625, "loss": 1.1471, "margin_dpo/margin_mean": 394.99566650390625, "margin_dpo/margin_std": 532.2371826171875, "step": 608 }, { "KL/chosen_KL_mean": -683.0916748046875, "KL/mean": -953.2722778320312, "KL/rejected_KL_mean": -1223.452880859375, "KL/std": 723.2427368164062, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 540.3612060546875, "fcm_dpo/q_t": 0.4110172986984253, "grad_norm": 34.754119873046875, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -1.0745786428451538, "logits/rejected": -1.1043390035629272, "logps/chosen": -741.89697265625, "logps/ref_chosen": -58.805355072021484, "logps/ref_rejected": -88.81600952148438, "logps/rejected": -1312.268798828125, "loss": 1.1054, "margin_dpo/margin_mean": 540.3612060546875, "margin_dpo/margin_std": 860.9444580078125, "step": 609 }, { "KL/chosen_KL_mean": -675.1845703125, "KL/mean": -831.5191650390625, "KL/rejected_KL_mean": -987.8536987304688, "KL/std": 541.0578002929688, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 312.6691589355469, "fcm_dpo/q_t": 0.44272029399871826, "grad_norm": 35.40081787109375, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -0.96883225440979, "logits/rejected": -0.9453009366989136, "logps/chosen": -740.879638671875, "logps/ref_chosen": -65.69503784179688, "logps/ref_rejected": -83.40538787841797, "logps/rejected": -1071.2591552734375, "loss": 1.2153, "margin_dpo/margin_mean": 312.669189453125, "margin_dpo/margin_std": 619.25634765625, "step": 610 }, { "KL/chosen_KL_mean": -716.7320556640625, "KL/mean": -999.7886962890625, "KL/rejected_KL_mean": -1282.84521484375, "KL/std": 708.412841796875, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 566.11328125, "fcm_dpo/q_t": 0.4043336808681488, "grad_norm": 31.667987823486328, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.024104118347168, "logits/rejected": -1.0761734247207642, "logps/chosen": -769.3314819335938, "logps/ref_chosen": -52.59946823120117, "logps/ref_rejected": -86.33099365234375, "logps/rejected": -1369.17626953125, "loss": 1.0855, "margin_dpo/margin_mean": 566.11328125, "margin_dpo/margin_std": 806.8145751953125, "step": 611 }, { "KL/chosen_KL_mean": -774.6546630859375, "KL/mean": -993.3455810546875, "KL/rejected_KL_mean": -1212.03662109375, "KL/std": 588.0178833007812, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 437.3819885253906, "fcm_dpo/q_t": 0.42229294776916504, "grad_norm": 27.343502044677734, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.014435887336731, "logits/rejected": -1.028846263885498, "logps/chosen": -833.9783935546875, "logps/ref_chosen": -59.32372283935547, "logps/ref_rejected": -88.31239318847656, "logps/rejected": -1300.3489990234375, "loss": 1.1351, "margin_dpo/margin_mean": 437.3819885253906, "margin_dpo/margin_std": 627.1123046875, "step": 612 }, { "KL/chosen_KL_mean": -668.7769775390625, "KL/mean": -925.552490234375, "KL/rejected_KL_mean": -1182.3280029296875, "KL/std": 624.155517578125, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 513.551025390625, "fcm_dpo/q_t": 0.40929657220840454, "grad_norm": 32.1468505859375, "learning_rate": 1.551886292185553e-08, "logits/chosen": -0.9937447309494019, "logits/rejected": -1.0488264560699463, "logps/chosen": -728.5069580078125, "logps/ref_chosen": -59.72996520996094, "logps/ref_rejected": -105.10752868652344, "logps/rejected": -1287.435546875, "loss": 1.0907, "margin_dpo/margin_mean": 513.5510864257812, "margin_dpo/margin_std": 658.1903076171875, "step": 613 }, { "KL/chosen_KL_mean": -755.2948608398438, "KL/mean": -1007.0299072265625, "KL/rejected_KL_mean": -1258.764892578125, "KL/std": 626.5299072265625, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 503.47003173828125, "fcm_dpo/q_t": 0.41245073080062866, "grad_norm": 35.746726989746094, "learning_rate": 1.507684480352292e-08, "logits/chosen": -0.9777568578720093, "logits/rejected": -1.0548135042190552, "logps/chosen": -808.2338256835938, "logps/ref_chosen": -52.93898010253906, "logps/ref_rejected": -104.67938232421875, "logps/rejected": -1363.4443359375, "loss": 1.1156, "margin_dpo/margin_mean": 503.47003173828125, "margin_dpo/margin_std": 753.4693603515625, "step": 614 }, { "KL/chosen_KL_mean": -689.6256103515625, "KL/mean": -910.9967041015625, "KL/rejected_KL_mean": -1132.367919921875, "KL/std": 654.9644165039062, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 442.74224853515625, "fcm_dpo/q_t": 0.4224264621734619, "grad_norm": 23.212697982788086, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.0044828653335571, "logits/rejected": -1.031577706336975, "logps/chosen": -755.44287109375, "logps/ref_chosen": -65.81727600097656, "logps/ref_rejected": -95.17749786376953, "logps/rejected": -1227.54541015625, "loss": 1.1484, "margin_dpo/margin_mean": 442.74224853515625, "margin_dpo/margin_std": 733.4412231445312, "step": 615 }, { "KL/chosen_KL_mean": -838.1975708007812, "KL/mean": -1010.1671142578125, "KL/rejected_KL_mean": -1182.1365966796875, "KL/std": 552.5067138671875, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 343.93902587890625, "fcm_dpo/q_t": 0.43850862979888916, "grad_norm": 26.084741592407227, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.1023294925689697, "logits/rejected": -1.0941028594970703, "logps/chosen": -903.3304443359375, "logps/ref_chosen": -65.13285827636719, "logps/ref_rejected": -74.70050048828125, "logps/rejected": -1256.837158203125, "loss": 1.1956, "margin_dpo/margin_mean": 343.93902587890625, "margin_dpo/margin_std": 621.4561157226562, "step": 616 }, { "KL/chosen_KL_mean": -798.08203125, "KL/mean": -914.913818359375, "KL/rejected_KL_mean": -1031.74560546875, "KL/std": 524.7188720703125, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0007631981279700994, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.6636505126953, "fcm_dpo/q_t": 0.4582458734512329, "grad_norm": 49.886898040771484, "learning_rate": 1.378797888467345e-08, "logits/chosen": -0.9841203093528748, "logits/rejected": -0.9550663232803345, "logps/chosen": -861.0875244140625, "logps/ref_chosen": -63.005550384521484, "logps/ref_rejected": -64.234130859375, "logps/rejected": -1095.979736328125, "loss": 1.2706, "margin_dpo/margin_mean": 233.6636505126953, "margin_dpo/margin_std": 622.0703125, "step": 617 }, { "KL/chosen_KL_mean": -829.4715576171875, "KL/mean": -1094.8660888671875, "KL/rejected_KL_mean": -1360.2607421875, "KL/std": 697.8109130859375, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0007587473373860121, "fcm_dpo/delta": -0.05866000056266785, "fcm_dpo/margin": 530.7892456054688, "fcm_dpo/q_t": 0.40983498096466064, "grad_norm": 34.46674346923828, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.030335545539856, "logits/rejected": -1.073843240737915, "logps/chosen": -896.5728759765625, "logps/ref_chosen": -67.10134887695312, "logps/ref_rejected": -92.15340423583984, "logps/rejected": -1452.4140625, "loss": 1.1343, "margin_dpo/margin_mean": 530.7892456054688, "margin_dpo/margin_std": 913.232421875, "step": 618 }, { "KL/chosen_KL_mean": -814.1534423828125, "KL/mean": -1020.8068237304688, "KL/rejected_KL_mean": -1227.460205078125, "KL/std": 658.5218505859375, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0007542966050095856, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 413.30682373046875, "fcm_dpo/q_t": 0.43316370248794556, "grad_norm": 42.285247802734375, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.0271763801574707, "logits/rejected": -1.0437747240066528, "logps/chosen": -870.131591796875, "logps/ref_chosen": -55.978233337402344, "logps/ref_rejected": -93.1854019165039, "logps/rejected": -1320.6456298828125, "loss": 1.1994, "margin_dpo/margin_mean": 413.30682373046875, "margin_dpo/margin_std": 883.5606079101562, "step": 619 }, { "KL/chosen_KL_mean": -766.2904052734375, "KL/mean": -959.615966796875, "KL/rejected_KL_mean": -1152.94140625, "KL/std": 587.8455810546875, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0007542966050095856, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 386.6510009765625, "fcm_dpo/q_t": 0.43245673179626465, "grad_norm": 27.1217041015625, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -1.046451210975647, "logits/rejected": -1.0556658506393433, "logps/chosen": -826.087890625, "logps/ref_chosen": -59.79750061035156, "logps/ref_rejected": -78.41075134277344, "logps/rejected": -1231.3521728515625, "loss": 1.1788, "margin_dpo/margin_mean": 386.6510314941406, "margin_dpo/margin_std": 680.88525390625, "step": 620 }, { "KL/chosen_KL_mean": -724.4033203125, "KL/mean": -1034.7967529296875, "KL/rejected_KL_mean": -1345.1900634765625, "KL/std": 727.23095703125, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.000746812263969332, "fcm_dpo/delta": -0.10022050142288208, "fcm_dpo/margin": 620.78662109375, "fcm_dpo/q_t": 0.3940914273262024, "grad_norm": 38.11147689819336, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.0878169536590576, "logits/rejected": -1.14664888381958, "logps/chosen": -778.337158203125, "logps/ref_chosen": -53.93375778198242, "logps/ref_rejected": -88.36951446533203, "logps/rejected": -1433.5595703125, "loss": 1.0629, "margin_dpo/margin_mean": 620.78662109375, "margin_dpo/margin_std": 812.3016967773438, "step": 621 }, { "KL/chosen_KL_mean": -727.606689453125, "KL/mean": -923.2297973632812, "KL/rejected_KL_mean": -1118.852783203125, "KL/std": 539.9659423828125, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0007393279229290783, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 391.2462463378906, "fcm_dpo/q_t": 0.4339316189289093, "grad_norm": 23.3463134765625, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -0.9580384492874146, "logits/rejected": -0.9486066102981567, "logps/chosen": -787.8924560546875, "logps/ref_chosen": -60.28582000732422, "logps/ref_rejected": -85.51873779296875, "logps/rejected": -1204.37158203125, "loss": 1.1786, "margin_dpo/margin_mean": 391.24627685546875, "margin_dpo/margin_std": 691.7193603515625, "step": 622 }, { "KL/chosen_KL_mean": -779.237548828125, "KL/mean": -930.7427368164062, "KL/rejected_KL_mean": -1082.247802734375, "KL/std": 557.14306640625, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0007393279229290783, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 303.01031494140625, "fcm_dpo/q_t": 0.4486698806285858, "grad_norm": 29.320268630981445, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -1.0816329717636108, "logits/rejected": -1.0815224647521973, "logps/chosen": -843.39453125, "logps/ref_chosen": -64.1569595336914, "logps/ref_rejected": -85.08304595947266, "logps/rejected": -1167.330810546875, "loss": 1.2321, "margin_dpo/margin_mean": 303.01031494140625, "margin_dpo/margin_std": 666.27685546875, "step": 623 }, { "KL/chosen_KL_mean": -772.2291259765625, "KL/mean": -1019.4240112304688, "KL/rejected_KL_mean": -1266.61865234375, "KL/std": 589.1283569335938, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0007393279229290783, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 494.3896484375, "fcm_dpo/q_t": 0.4135315716266632, "grad_norm": 35.011497497558594, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -1.0676138401031494, "logits/rejected": -1.0803065299987793, "logps/chosen": -844.1478271484375, "logps/ref_chosen": -71.91862487792969, "logps/ref_rejected": -97.13203430175781, "logps/rejected": -1363.750732421875, "loss": 1.1081, "margin_dpo/margin_mean": 494.3896484375, "margin_dpo/margin_std": 643.760498046875, "step": 624 }, { "KL/chosen_KL_mean": -739.02734375, "KL/mean": -1006.8427124023438, "KL/rejected_KL_mean": -1274.657958984375, "KL/std": 642.2593994140625, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0007375460118055344, "fcm_dpo/delta": -0.024159858003258705, "fcm_dpo/margin": 535.6307373046875, "fcm_dpo/q_t": 0.4078494906425476, "grad_norm": 42.963836669921875, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.011317491531372, "logits/rejected": -1.04207181930542, "logps/chosen": -797.369384765625, "logps/ref_chosen": -58.342071533203125, "logps/ref_rejected": -86.09038543701172, "logps/rejected": -1360.7484130859375, "loss": 1.0744, "margin_dpo/margin_mean": 535.6307373046875, "margin_dpo/margin_std": 594.4256591796875, "step": 625 }, { "KL/chosen_KL_mean": -903.0684204101562, "KL/mean": -1061.467041015625, "KL/rejected_KL_mean": -1219.86572265625, "KL/std": 698.789794921875, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0007357641006819904, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 316.7972717285156, "fcm_dpo/q_t": 0.444003701210022, "grad_norm": 30.258146286010742, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -0.9826500415802002, "logits/rejected": -0.9770318865776062, "logps/chosen": -978.1810302734375, "logps/ref_chosen": -75.11260986328125, "logps/ref_rejected": -99.188720703125, "logps/rejected": -1319.054443359375, "loss": 1.234, "margin_dpo/margin_mean": 316.7972717285156, "margin_dpo/margin_std": 695.1666259765625, "step": 626 }, { "KL/chosen_KL_mean": -622.5956420898438, "KL/mean": -943.0328369140625, "KL/rejected_KL_mean": -1263.469970703125, "KL/std": 787.0336303710938, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0007278695702552795, "fcm_dpo/delta": -0.10846550762653351, "fcm_dpo/margin": 640.8743896484375, "fcm_dpo/q_t": 0.40123480558395386, "grad_norm": 27.875293731689453, "learning_rate": 9.897955805412e-09, "logits/chosen": -0.9350000619888306, "logits/rejected": -1.0127406120300293, "logps/chosen": -670.3388061523438, "logps/ref_chosen": -47.74314880371094, "logps/ref_rejected": -106.75448608398438, "logps/rejected": -1370.224609375, "loss": 1.0751, "margin_dpo/margin_mean": 640.8743896484375, "margin_dpo/margin_std": 925.1202392578125, "step": 627 }, { "KL/chosen_KL_mean": -809.4762573242188, "KL/mean": -1047.6484375, "KL/rejected_KL_mean": -1285.82080078125, "KL/std": 634.0107421875, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 476.3446044921875, "fcm_dpo/q_t": 0.42106395959854126, "grad_norm": 28.623077392578125, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.049999713897705, "logits/rejected": -1.072727084159851, "logps/chosen": -869.6591796875, "logps/ref_chosen": -60.182945251464844, "logps/ref_rejected": -101.55467224121094, "logps/rejected": -1387.37548828125, "loss": 1.1354, "margin_dpo/margin_mean": 476.3446044921875, "margin_dpo/margin_std": 717.890380859375, "step": 628 }, { "KL/chosen_KL_mean": -812.48046875, "KL/mean": -1029.31640625, "KL/rejected_KL_mean": -1246.15234375, "KL/std": 613.6525268554688, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 433.6718444824219, "fcm_dpo/q_t": 0.4258197546005249, "grad_norm": 36.2744140625, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.0528905391693115, "logits/rejected": -1.0721955299377441, "logps/chosen": -876.6940307617188, "logps/ref_chosen": -64.21354675292969, "logps/ref_rejected": -91.65367126464844, "logps/rejected": -1337.805908203125, "loss": 1.1441, "margin_dpo/margin_mean": 433.6718444824219, "margin_dpo/margin_std": 607.2192993164062, "step": 629 }, { "KL/chosen_KL_mean": -723.0999755859375, "KL/mean": -842.289306640625, "KL/rejected_KL_mean": -961.4786987304688, "KL/std": 608.8505249023438, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 238.37872314453125, "fcm_dpo/q_t": 0.4632091224193573, "grad_norm": 34.26353454589844, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.034097671508789, "logits/rejected": -1.011200189590454, "logps/chosen": -782.3909912109375, "logps/ref_chosen": -59.29100036621094, "logps/ref_rejected": -83.59829711914062, "logps/rejected": -1045.0770263671875, "loss": 1.2898, "margin_dpo/margin_mean": 238.37872314453125, "margin_dpo/margin_std": 737.2360229492188, "step": 630 }, { "KL/chosen_KL_mean": -853.849609375, "KL/mean": -1046.713623046875, "KL/rejected_KL_mean": -1239.577392578125, "KL/std": 606.48828125, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 385.7279052734375, "fcm_dpo/q_t": 0.43468064069747925, "grad_norm": 24.484596252441406, "learning_rate": 8.518543427732949e-09, "logits/chosen": -1.1066150665283203, "logits/rejected": -1.1154310703277588, "logps/chosen": -913.30322265625, "logps/ref_chosen": -59.45360565185547, "logps/ref_rejected": -80.95156860351562, "logps/rejected": -1320.529052734375, "loss": 1.1894, "margin_dpo/margin_mean": 385.7279052734375, "margin_dpo/margin_std": 702.0499877929688, "step": 631 }, { "KL/chosen_KL_mean": -769.9806518554688, "KL/mean": -960.3197021484375, "KL/rejected_KL_mean": -1150.65869140625, "KL/std": 546.0325317382812, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 380.67803955078125, "fcm_dpo/q_t": 0.43564653396606445, "grad_norm": 42.972713470458984, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.0557451248168945, "logits/rejected": -1.0606086254119873, "logps/chosen": -831.3322143554688, "logps/ref_chosen": -61.35155487060547, "logps/ref_rejected": -86.16017150878906, "logps/rejected": -1236.81884765625, "loss": 1.192, "margin_dpo/margin_mean": 380.67803955078125, "margin_dpo/margin_std": 693.4515380859375, "step": 632 }, { "KL/chosen_KL_mean": -814.6301879882812, "KL/mean": -957.8812255859375, "KL/rejected_KL_mean": -1101.13232421875, "KL/std": 546.3907470703125, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 286.50201416015625, "fcm_dpo/q_t": 0.45074015855789185, "grad_norm": 33.58137130737305, "learning_rate": 7.866980873399015e-09, "logits/chosen": -1.1074434518814087, "logits/rejected": -1.1195733547210693, "logps/chosen": -871.9083251953125, "logps/ref_chosen": -57.27816390991211, "logps/ref_rejected": -91.58395385742188, "logps/rejected": -1192.71630859375, "loss": 1.2413, "margin_dpo/margin_mean": 286.5020446777344, "margin_dpo/margin_std": 632.80322265625, "step": 633 }, { "KL/chosen_KL_mean": -923.3136596679688, "KL/mean": -1051.839599609375, "KL/rejected_KL_mean": -1180.3656005859375, "KL/std": 654.6397705078125, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 257.0519714355469, "fcm_dpo/q_t": 0.456850528717041, "grad_norm": 22.398880004882812, "learning_rate": 7.550765991247654e-09, "logits/chosen": -0.9894505739212036, "logits/rejected": -0.9836434721946716, "logps/chosen": -989.9326171875, "logps/ref_chosen": -66.61896514892578, "logps/ref_rejected": -107.12564849853516, "logps/rejected": -1287.4912109375, "loss": 1.2595, "margin_dpo/margin_mean": 257.05194091796875, "margin_dpo/margin_std": 630.111572265625, "step": 634 }, { "KL/chosen_KL_mean": -822.2686767578125, "KL/mean": -1000.0654296875, "KL/rejected_KL_mean": -1177.862060546875, "KL/std": 712.7052001953125, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 355.593505859375, "fcm_dpo/q_t": 0.4414767026901245, "grad_norm": 33.33140182495117, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.037980079650879, "logits/rejected": -1.0179985761642456, "logps/chosen": -896.2242431640625, "logps/ref_chosen": -73.95551300048828, "logps/ref_rejected": -82.50045776367188, "logps/rejected": -1260.362548828125, "loss": 1.209, "margin_dpo/margin_mean": 355.593505859375, "margin_dpo/margin_std": 720.908447265625, "step": 635 }, { "KL/chosen_KL_mean": -765.69140625, "KL/mean": -987.3884887695312, "KL/rejected_KL_mean": -1209.085693359375, "KL/std": 667.54638671875, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0007199749816209078, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 443.39422607421875, "fcm_dpo/q_t": 0.42743438482284546, "grad_norm": 24.014665603637695, "learning_rate": 6.937510679537628e-09, "logits/chosen": -0.9858390092849731, "logits/rejected": -0.9905164241790771, "logps/chosen": -825.3203125, "logps/ref_chosen": -59.628910064697266, "logps/ref_rejected": -81.97883605957031, "logps/rejected": -1291.064453125, "loss": 1.1495, "margin_dpo/margin_mean": 443.39422607421875, "margin_dpo/margin_std": 695.56005859375, "step": 636 }, { "KL/chosen_KL_mean": -779.787841796875, "KL/mean": -1034.97900390625, "KL/rejected_KL_mean": -1290.170166015625, "KL/std": 696.4139404296875, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0007171693141572177, "fcm_dpo/delta": -0.01952262967824936, "fcm_dpo/margin": 510.3822937011719, "fcm_dpo/q_t": 0.41653138399124146, "grad_norm": 25.295166015625, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.0853636264801025, "logits/rejected": -1.139721393585205, "logps/chosen": -829.4405517578125, "logps/ref_chosen": -49.652687072753906, "logps/ref_rejected": -98.40513610839844, "logps/rejected": -1388.5751953125, "loss": 1.1162, "margin_dpo/margin_mean": 510.3822937011719, "margin_dpo/margin_std": 726.550537109375, "step": 637 }, { "KL/chosen_KL_mean": -763.474609375, "KL/mean": -958.1444091796875, "KL/rejected_KL_mean": -1152.814208984375, "KL/std": 652.8692626953125, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0007153811166062951, "fcm_dpo/delta": -0.024996092543005943, "fcm_dpo/margin": 389.33966064453125, "fcm_dpo/q_t": 0.4316685199737549, "grad_norm": 34.41019821166992, "learning_rate": 6.349874889624962e-09, "logits/chosen": -0.9672867655754089, "logits/rejected": -0.9581058025360107, "logps/chosen": -821.6312255859375, "logps/ref_chosen": -58.156639099121094, "logps/ref_rejected": -79.3014907836914, "logps/rejected": -1232.11572265625, "loss": 1.2066, "margin_dpo/margin_mean": 389.33966064453125, "margin_dpo/margin_std": 788.7205810546875, "step": 638 }, { "KL/chosen_KL_mean": -1031.480224609375, "KL/mean": -1104.28857421875, "KL/rejected_KL_mean": -1177.0970458984375, "KL/std": 619.0423583984375, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0007135929772630334, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.61685180664062, "fcm_dpo/q_t": 0.4758584499359131, "grad_norm": 83.87274169921875, "learning_rate": 6.065683776815933e-09, "logits/chosen": -0.9700315594673157, "logits/rejected": -0.910193920135498, "logps/chosen": -1103.803466796875, "logps/ref_chosen": -72.32319641113281, "logps/ref_rejected": -74.2749252319336, "logps/rejected": -1251.3719482421875, "loss": 1.3523, "margin_dpo/margin_mean": 145.61685180664062, "margin_dpo/margin_std": 737.2562866210938, "step": 639 }, { "KL/chosen_KL_mean": -792.1627197265625, "KL/mean": -1085.340576171875, "KL/rejected_KL_mean": -1378.5186767578125, "KL/std": 706.311767578125, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0007101770024746656, "fcm_dpo/delta": -0.048100586980581284, "fcm_dpo/margin": 586.3558959960938, "fcm_dpo/q_t": 0.4069620370864868, "grad_norm": 31.302490234375, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -0.959877073764801, "logits/rejected": -1.0006804466247559, "logps/chosen": -848.2970581054688, "logps/ref_chosen": -56.13436508178711, "logps/ref_rejected": -108.60014343261719, "logps/rejected": -1487.1187744140625, "loss": 1.0927, "margin_dpo/margin_mean": 586.3558959960938, "margin_dpo/margin_std": 834.8697509765625, "step": 640 }, { "KL/chosen_KL_mean": -899.358642578125, "KL/mean": -1098.58837890625, "KL/rejected_KL_mean": -1297.818115234375, "KL/std": 614.996337890625, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 398.45941162109375, "fcm_dpo/q_t": 0.43619343638420105, "grad_norm": 32.60629653930664, "learning_rate": 5.516592558795746e-09, "logits/chosen": -1.0186073780059814, "logits/rejected": -1.039933443069458, "logps/chosen": -964.3555908203125, "logps/ref_chosen": -64.99689483642578, "logps/ref_rejected": -86.99232482910156, "logps/rejected": -1384.810546875, "loss": 1.2018, "margin_dpo/margin_mean": 398.45941162109375, "margin_dpo/margin_std": 817.0283203125, "step": 641 }, { "KL/chosen_KL_mean": -874.2130126953125, "KL/mean": -1114.15283203125, "KL/rejected_KL_mean": -1354.0927734375, "KL/std": 816.8272705078125, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 479.8797912597656, "fcm_dpo/q_t": 0.4284675419330597, "grad_norm": 34.0765380859375, "learning_rate": 5.251706922648868e-09, "logits/chosen": -0.9778116941452026, "logits/rejected": -1.0190857648849487, "logps/chosen": -939.9022216796875, "logps/ref_chosen": -65.68924713134766, "logps/ref_rejected": -110.24205017089844, "logps/rejected": -1464.3349609375, "loss": 1.1809, "margin_dpo/margin_mean": 479.87982177734375, "margin_dpo/margin_std": 996.5196533203125, "step": 642 }, { "KL/chosen_KL_mean": -800.347900390625, "KL/mean": -969.1351318359375, "KL/rejected_KL_mean": -1137.92236328125, "KL/std": 588.9737548828125, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 337.57440185546875, "fcm_dpo/q_t": 0.4438466429710388, "grad_norm": 34.371337890625, "learning_rate": 4.993270631642038e-09, "logits/chosen": -1.1373050212860107, "logits/rejected": -1.135626196861267, "logps/chosen": -852.2979125976562, "logps/ref_chosen": -51.94999694824219, "logps/ref_rejected": -87.46833801269531, "logps/rejected": -1225.390625, "loss": 1.2082, "margin_dpo/margin_mean": 337.57440185546875, "margin_dpo/margin_std": 598.9072265625, "step": 643 }, { "KL/chosen_KL_mean": -768.4283447265625, "KL/mean": -946.823974609375, "KL/rejected_KL_mean": -1125.2196044921875, "KL/std": 693.099609375, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 356.79119873046875, "fcm_dpo/q_t": 0.44256871938705444, "grad_norm": 40.267459869384766, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.0229220390319824, "logits/rejected": -1.031947374343872, "logps/chosen": -827.446044921875, "logps/ref_chosen": -59.017662048339844, "logps/ref_rejected": -87.13668823242188, "logps/rejected": -1212.356201171875, "loss": 1.2272, "margin_dpo/margin_mean": 356.79119873046875, "margin_dpo/margin_std": 801.4711303710938, "step": 644 }, { "KL/chosen_KL_mean": -819.4630737304688, "KL/mean": -900.1171875, "KL/rejected_KL_mean": -980.771240234375, "KL/std": 550.095703125, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.30816650390625, "fcm_dpo/q_t": 0.4736112058162689, "grad_norm": 67.07617950439453, "learning_rate": 4.495773155069299e-09, "logits/chosen": -0.9778566956520081, "logits/rejected": -0.9598023891448975, "logps/chosen": -875.339111328125, "logps/ref_chosen": -55.87602233886719, "logps/ref_rejected": -97.78080749511719, "logps/rejected": -1078.552001953125, "loss": 1.3464, "margin_dpo/margin_mean": 161.3081817626953, "margin_dpo/margin_std": 739.6782836914062, "step": 645 }, { "KL/chosen_KL_mean": -777.6273193359375, "KL/mean": -928.5137939453125, "KL/rejected_KL_mean": -1079.4002685546875, "KL/std": 526.18310546875, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 301.7729797363281, "fcm_dpo/q_t": 0.44995421171188354, "grad_norm": 36.729408264160156, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.0355250835418701, "logits/rejected": -1.019197940826416, "logps/chosen": -838.903076171875, "logps/ref_chosen": -61.275787353515625, "logps/ref_rejected": -77.50580596923828, "logps/rejected": -1156.906005859375, "loss": 1.2347, "margin_dpo/margin_mean": 301.7729797363281, "margin_dpo/margin_std": 637.1296997070312, "step": 646 }, { "KL/chosen_KL_mean": -677.3515625, "KL/mean": -878.120361328125, "KL/rejected_KL_mean": -1078.88916015625, "KL/std": 601.9970703125, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 401.5375671386719, "fcm_dpo/q_t": 0.4323872923851013, "grad_norm": 28.29328155517578, "learning_rate": 4.024152566816791e-09, "logits/chosen": -0.9465760588645935, "logits/rejected": -0.9807703495025635, "logps/chosen": -732.2039794921875, "logps/ref_chosen": -54.8524169921875, "logps/ref_rejected": -93.5194091796875, "logps/rejected": -1172.408447265625, "loss": 1.1631, "margin_dpo/margin_mean": 401.53753662109375, "margin_dpo/margin_std": 573.2171020507812, "step": 647 }, { "KL/chosen_KL_mean": -734.8883056640625, "KL/mean": -1029.216064453125, "KL/rejected_KL_mean": -1323.543701171875, "KL/std": 709.6268310546875, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 588.655517578125, "fcm_dpo/q_t": 0.4075320065021515, "grad_norm": 26.580156326293945, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.0718775987625122, "logits/rejected": -1.1320219039916992, "logps/chosen": -789.0597534179688, "logps/ref_chosen": -54.17146682739258, "logps/ref_rejected": -98.7127914428711, "logps/rejected": -1422.256591796875, "loss": 1.0914, "margin_dpo/margin_mean": 588.6554565429688, "margin_dpo/margin_std": 848.3179931640625, "step": 648 }, { "KL/chosen_KL_mean": -751.6907958984375, "KL/mean": -895.4127197265625, "KL/rejected_KL_mean": -1039.134765625, "KL/std": 570.4094848632812, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 287.4439697265625, "fcm_dpo/q_t": 0.4549896717071533, "grad_norm": 25.27387237548828, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -1.1144146919250488, "logits/rejected": -1.115525722503662, "logps/chosen": -814.171142578125, "logps/ref_chosen": -62.480350494384766, "logps/ref_rejected": -80.07717895507812, "logps/rejected": -1119.2119140625, "loss": 1.2578, "margin_dpo/margin_mean": 287.4439697265625, "margin_dpo/margin_std": 714.5806274414062, "step": 649 }, { "KL/chosen_KL_mean": -815.21728515625, "KL/mean": -1038.905517578125, "KL/rejected_KL_mean": -1262.5936279296875, "KL/std": 688.6070556640625, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 447.37640380859375, "fcm_dpo/q_t": 0.4277136027812958, "grad_norm": 28.098812103271484, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.0027992725372314, "logits/rejected": -1.0247873067855835, "logps/chosen": -871.31005859375, "logps/ref_chosen": -56.09281921386719, "logps/ref_rejected": -98.26483917236328, "logps/rejected": -1360.8585205078125, "loss": 1.1696, "margin_dpo/margin_mean": 447.3763427734375, "margin_dpo/margin_std": 797.6195068359375, "step": 650 }, { "KL/chosen_KL_mean": -560.206787109375, "KL/mean": -845.2562255859375, "KL/rejected_KL_mean": -1130.3057861328125, "KL/std": 642.8323974609375, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 570.0989990234375, "fcm_dpo/q_t": 0.4065355062484741, "grad_norm": 31.23227310180664, "learning_rate": 3.158738163478475e-09, "logits/chosen": -1.0280869007110596, "logits/rejected": -1.0860235691070557, "logps/chosen": -603.6322021484375, "logps/ref_chosen": -43.42544937133789, "logps/ref_rejected": -99.95791625976562, "logps/rejected": -1230.263671875, "loss": 1.0737, "margin_dpo/margin_mean": 570.0989990234375, "margin_dpo/margin_std": 660.94873046875, "step": 651 }, { "KL/chosen_KL_mean": -692.7413940429688, "KL/mean": -903.6035766601562, "KL/rejected_KL_mean": -1114.4656982421875, "KL/std": 650.1327514648438, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 421.72430419921875, "fcm_dpo/q_t": 0.4312537908554077, "grad_norm": 29.79759979248047, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -1.0472981929779053, "logits/rejected": -1.076317548751831, "logps/chosen": -755.3182373046875, "logps/ref_chosen": -62.57680892944336, "logps/ref_rejected": -111.76779174804688, "logps/rejected": -1226.2335205078125, "loss": 1.176, "margin_dpo/margin_mean": 421.7243347167969, "margin_dpo/margin_std": 753.9987182617188, "step": 652 }, { "KL/chosen_KL_mean": -857.264892578125, "KL/mean": -1053.7757568359375, "KL/rejected_KL_mean": -1250.28662109375, "KL/std": 699.4908447265625, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 393.0216979980469, "fcm_dpo/q_t": 0.4362325668334961, "grad_norm": 27.61276626586914, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -1.0763492584228516, "logits/rejected": -1.103668451309204, "logps/chosen": -918.3778686523438, "logps/ref_chosen": -61.11295700073242, "logps/ref_rejected": -103.24960327148438, "logps/rejected": -1353.5361328125, "loss": 1.1908, "margin_dpo/margin_mean": 393.021728515625, "margin_dpo/margin_std": 730.29736328125, "step": 653 }, { "KL/chosen_KL_mean": -755.6949462890625, "KL/mean": -948.0558471679688, "KL/rejected_KL_mean": -1140.416748046875, "KL/std": 557.1722412109375, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 384.72174072265625, "fcm_dpo/q_t": 0.4374183714389801, "grad_norm": 26.79352378845215, "learning_rate": 2.577954022936174e-09, "logits/chosen": -1.0465385913848877, "logits/rejected": -1.0679962635040283, "logps/chosen": -817.423095703125, "logps/ref_chosen": -61.7281379699707, "logps/ref_rejected": -98.7738037109375, "logps/rejected": -1239.1905517578125, "loss": 1.1862, "margin_dpo/margin_mean": 384.72174072265625, "margin_dpo/margin_std": 672.5704345703125, "step": 654 }, { "KL/chosen_KL_mean": -700.713134765625, "KL/mean": -898.3726806640625, "KL/rejected_KL_mean": -1096.0322265625, "KL/std": 572.2005615234375, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0007067610276862979, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 395.3191223144531, "fcm_dpo/q_t": 0.4351748824119568, "grad_norm": 37.17662048339844, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.0191048383712769, "logits/rejected": -1.0588314533233643, "logps/chosen": -750.2899169921875, "logps/ref_chosen": -49.576812744140625, "logps/ref_rejected": -98.29183197021484, "logps/rejected": -1194.3240966796875, "loss": 1.1804, "margin_dpo/margin_mean": 395.3191223144531, "margin_dpo/margin_std": 675.3500366210938, "step": 655 }, { "KL/chosen_KL_mean": -771.8296508789062, "KL/mean": -1122.322265625, "KL/rejected_KL_mean": -1472.814697265625, "KL/std": 778.0101318359375, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0007016347954049706, "fcm_dpo/delta": -0.07306308299303055, "fcm_dpo/margin": 700.985107421875, "fcm_dpo/q_t": 0.38852617144584656, "grad_norm": 64.08390045166016, "learning_rate": 2.223355098446622e-09, "logits/chosen": -0.9458581209182739, "logits/rejected": -1.018227219581604, "logps/chosen": -824.3790893554688, "logps/ref_chosen": -52.54943084716797, "logps/ref_rejected": -113.67464447021484, "logps/rejected": -1586.489501953125, "loss": 1.0211, "margin_dpo/margin_mean": 700.985107421875, "margin_dpo/margin_std": 782.2880859375, "step": 656 }, { "KL/chosen_KL_mean": -732.1683959960938, "KL/mean": -1004.7838745117188, "KL/rejected_KL_mean": -1277.3992919921875, "KL/std": 734.6845703125, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 545.23095703125, "fcm_dpo/q_t": 0.4126076102256775, "grad_norm": 33.66898727416992, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.0927517414093018, "logits/rejected": -1.140802025794983, "logps/chosen": -778.868896484375, "logps/ref_chosen": -46.700538635253906, "logps/ref_rejected": -97.91487121582031, "logps/rejected": -1375.314208984375, "loss": 1.1085, "margin_dpo/margin_mean": 545.23095703125, "margin_dpo/margin_std": 772.8052978515625, "step": 657 }, { "KL/chosen_KL_mean": -780.8671264648438, "KL/mean": -991.93359375, "KL/rejected_KL_mean": -1203.0001220703125, "KL/std": 568.8956909179688, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 422.13299560546875, "fcm_dpo/q_t": 0.43089789152145386, "grad_norm": 30.343259811401367, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.0211938619613647, "logits/rejected": -1.0497910976409912, "logps/chosen": -841.8253173828125, "logps/ref_chosen": -60.95820999145508, "logps/ref_rejected": -95.93949127197266, "logps/rejected": -1298.9395751953125, "loss": 1.1636, "margin_dpo/margin_mean": 422.13299560546875, "margin_dpo/margin_std": 647.49755859375, "step": 658 }, { "KL/chosen_KL_mean": -705.3890991210938, "KL/mean": -921.2344970703125, "KL/rejected_KL_mean": -1137.079833984375, "KL/std": 596.189208984375, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 431.69073486328125, "fcm_dpo/q_t": 0.4299265444278717, "grad_norm": 36.30439758300781, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -0.9837784767150879, "logits/rejected": -0.964771568775177, "logps/chosen": -782.132080078125, "logps/ref_chosen": -76.74298095703125, "logps/ref_rejected": -87.4709701538086, "logps/rejected": -1224.55078125, "loss": 1.1541, "margin_dpo/margin_mean": 431.690673828125, "margin_dpo/margin_std": 629.0660400390625, "step": 659 }, { "KL/chosen_KL_mean": -754.4503173828125, "KL/mean": -1027.2886962890625, "KL/rejected_KL_mean": -1300.127197265625, "KL/std": 671.2972412109375, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 545.6766967773438, "fcm_dpo/q_t": 0.4120747745037079, "grad_norm": 35.72873306274414, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.084972620010376, "logits/rejected": -1.1093543767929077, "logps/chosen": -813.4982299804688, "logps/ref_chosen": -59.04788589477539, "logps/ref_rejected": -75.96005249023438, "logps/rejected": -1376.087158203125, "loss": 1.1026, "margin_dpo/margin_mean": 545.6766967773438, "margin_dpo/margin_std": 731.752685546875, "step": 660 }, { "KL/chosen_KL_mean": -672.3802490234375, "KL/mean": -915.9332275390625, "KL/rejected_KL_mean": -1159.486083984375, "KL/std": 750.931640625, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 487.10589599609375, "fcm_dpo/q_t": 0.42373281717300415, "grad_norm": 40.01762008666992, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.0510165691375732, "logits/rejected": -1.0702614784240723, "logps/chosen": -723.05419921875, "logps/ref_chosen": -50.673973083496094, "logps/ref_rejected": -86.00569152832031, "logps/rejected": -1245.4918212890625, "loss": 1.1354, "margin_dpo/margin_mean": 487.10589599609375, "margin_dpo/margin_std": 751.4103393554688, "step": 661 }, { "KL/chosen_KL_mean": -763.1075439453125, "KL/mean": -949.8360595703125, "KL/rejected_KL_mean": -1136.5645751953125, "KL/std": 620.0942993164062, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 373.45697021484375, "fcm_dpo/q_t": 0.4395660161972046, "grad_norm": 23.511491775512695, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -0.9717986583709717, "logits/rejected": -0.9751859903335571, "logps/chosen": -832.3685913085938, "logps/ref_chosen": -69.26106262207031, "logps/ref_rejected": -89.05593872070312, "logps/rejected": -1225.6204833984375, "loss": 1.2118, "margin_dpo/margin_mean": 373.45697021484375, "margin_dpo/margin_std": 774.0943603515625, "step": 662 }, { "KL/chosen_KL_mean": -730.2545166015625, "KL/mean": -961.2086181640625, "KL/rejected_KL_mean": -1192.162841796875, "KL/std": 697.215087890625, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 461.908203125, "fcm_dpo/q_t": 0.42762649059295654, "grad_norm": 23.27935791015625, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -1.0581246614456177, "logits/rejected": -1.0871996879577637, "logps/chosen": -795.1334228515625, "logps/ref_chosen": -64.87890625, "logps/ref_rejected": -113.92536926269531, "logps/rejected": -1306.088134765625, "loss": 1.1632, "margin_dpo/margin_mean": 461.908203125, "margin_dpo/margin_std": 824.1527099609375, "step": 663 }, { "KL/chosen_KL_mean": -802.4429931640625, "KL/mean": -1063.125244140625, "KL/rejected_KL_mean": -1323.807373046875, "KL/std": 709.4827270507812, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 521.3643798828125, "fcm_dpo/q_t": 0.4172210693359375, "grad_norm": 22.374834060668945, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.033679485321045, "logits/rejected": -1.0704450607299805, "logps/chosen": -863.3314208984375, "logps/ref_chosen": -60.88847351074219, "logps/ref_rejected": -105.521728515625, "logps/rejected": -1429.3291015625, "loss": 1.1229, "margin_dpo/margin_mean": 521.3643798828125, "margin_dpo/margin_std": 771.2794189453125, "step": 664 }, { "KL/chosen_KL_mean": -692.1058959960938, "KL/mean": -907.7689208984375, "KL/rejected_KL_mean": -1123.4320068359375, "KL/std": 584.74462890625, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 431.32611083984375, "fcm_dpo/q_t": 0.42915087938308716, "grad_norm": 37.84297561645508, "learning_rate": 9.513254770636137e-10, "logits/chosen": -1.1375402212142944, "logits/rejected": -1.1711466312408447, "logps/chosen": -752.6700439453125, "logps/ref_chosen": -60.56413269042969, "logps/ref_rejected": -84.80882263183594, "logps/rejected": -1208.24072265625, "loss": 1.1491, "margin_dpo/margin_mean": 431.32611083984375, "margin_dpo/margin_std": 589.2435913085938, "step": 665 }, { "KL/chosen_KL_mean": -727.8260498046875, "KL/mean": -953.4937744140625, "KL/rejected_KL_mean": -1179.161376953125, "KL/std": 599.885986328125, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 451.33538818359375, "fcm_dpo/q_t": 0.4272992014884949, "grad_norm": 32.65074920654297, "learning_rate": 8.427576920763956e-10, "logits/chosen": -0.9382809400558472, "logits/rejected": -0.9508320689201355, "logps/chosen": -792.2459716796875, "logps/ref_chosen": -64.41996002197266, "logps/ref_rejected": -95.8916244506836, "logps/rejected": -1275.052978515625, "loss": 1.1502, "margin_dpo/margin_mean": 451.33538818359375, "margin_dpo/margin_std": 672.0390625, "step": 666 }, { "KL/chosen_KL_mean": -848.9732666015625, "KL/mean": -1107.0611572265625, "KL/rejected_KL_mean": -1365.1490478515625, "KL/std": 657.8707275390625, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 516.17578125, "fcm_dpo/q_t": 0.4165239930152893, "grad_norm": 35.096256256103516, "learning_rate": 7.407554321417764e-10, "logits/chosen": -0.9879491925239563, "logits/rejected": -0.9914569854736328, "logps/chosen": -918.250244140625, "logps/ref_chosen": -69.27702331542969, "logps/ref_rejected": -87.83549499511719, "logps/rejected": -1452.9844970703125, "loss": 1.1182, "margin_dpo/margin_mean": 516.17578125, "margin_dpo/margin_std": 720.5601196289062, "step": 667 }, { "KL/chosen_KL_mean": -890.6829223632812, "KL/mean": -1073.6181640625, "KL/rejected_KL_mean": -1256.55322265625, "KL/std": 695.8572998046875, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 365.870361328125, "fcm_dpo/q_t": 0.4426293969154358, "grad_norm": 37.71943283081055, "learning_rate": 6.453213851142225e-10, "logits/chosen": -1.043158769607544, "logits/rejected": -1.0555496215820312, "logps/chosen": -963.2869262695312, "logps/ref_chosen": -72.60400390625, "logps/ref_rejected": -103.73905944824219, "logps/rejected": -1360.292236328125, "loss": 1.2257, "margin_dpo/margin_mean": 365.870361328125, "margin_dpo/margin_std": 822.745361328125, "step": 668 }, { "KL/chosen_KL_mean": -685.7474975585938, "KL/mean": -940.034423828125, "KL/rejected_KL_mean": -1194.3214111328125, "KL/std": 647.0894775390625, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0006965085049159825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 508.57391357421875, "fcm_dpo/q_t": 0.41749686002731323, "grad_norm": 23.442853927612305, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.006242036819458, "logits/rejected": -1.0165035724639893, "logps/chosen": -731.8638916015625, "logps/ref_chosen": -46.116416931152344, "logps/ref_rejected": -77.92434692382812, "logps/rejected": -1272.245849609375, "loss": 1.1237, "margin_dpo/margin_mean": 508.57391357421875, "margin_dpo/margin_std": 728.7122192382812, "step": 669 }, { "KL/chosen_KL_mean": -646.3446044921875, "KL/mean": -914.180419921875, "KL/rejected_KL_mean": -1182.01611328125, "KL/std": 602.09423828125, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0006949029630050063, "fcm_dpo/delta": -0.023104503750801086, "fcm_dpo/margin": 535.6715087890625, "fcm_dpo/q_t": 0.4133542776107788, "grad_norm": 20.55622673034668, "learning_rate": 4.741678157389739e-10, "logits/chosen": -0.9502823948860168, "logits/rejected": -0.9757121801376343, "logps/chosen": -708.6904296875, "logps/ref_chosen": -62.34575271606445, "logps/ref_rejected": -96.9405517578125, "logps/rejected": -1278.9566650390625, "loss": 1.1106, "margin_dpo/margin_mean": 535.6715087890625, "margin_dpo/margin_std": 729.982177734375, "step": 670 }, { "KL/chosen_KL_mean": -812.72900390625, "KL/mean": -1021.0582275390625, "KL/rejected_KL_mean": -1229.387451171875, "KL/std": 600.5374145507812, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0006932974210940301, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 416.6583251953125, "fcm_dpo/q_t": 0.4308261275291443, "grad_norm": 28.0316162109375, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.0668823719024658, "logits/rejected": -1.100010633468628, "logps/chosen": -860.7291259765625, "logps/ref_chosen": -48.00010681152344, "logps/ref_rejected": -83.81932067871094, "logps/rejected": -1313.206787109375, "loss": 1.1802, "margin_dpo/margin_mean": 416.65838623046875, "margin_dpo/margin_std": 731.703857421875, "step": 671 }, { "KL/chosen_KL_mean": -910.474853515625, "KL/mean": -1116.897705078125, "KL/rejected_KL_mean": -1323.320556640625, "KL/std": 720.9313354492188, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0006932974210940301, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 412.8456115722656, "fcm_dpo/q_t": 0.4346805214881897, "grad_norm": 42.60050964355469, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.1313029527664185, "logits/rejected": -1.1380281448364258, "logps/chosen": -969.0581665039062, "logps/ref_chosen": -58.58328628540039, "logps/ref_rejected": -93.14015197753906, "logps/rejected": -1416.460693359375, "loss": 1.1939, "margin_dpo/margin_mean": 412.84564208984375, "margin_dpo/margin_std": 798.4884033203125, "step": 672 }, { "KL/chosen_KL_mean": -806.967529296875, "KL/mean": -1001.2117919921875, "KL/rejected_KL_mean": -1195.4560546875, "KL/std": 610.96337890625, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0006932974210940301, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 388.4884033203125, "fcm_dpo/q_t": 0.43748611211776733, "grad_norm": 24.804006576538086, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.0505502223968506, "logits/rejected": -1.054823637008667, "logps/chosen": -853.6907958984375, "logps/ref_chosen": -46.72320556640625, "logps/ref_rejected": -85.29623413085938, "logps/rejected": -1280.752197265625, "loss": 1.1877, "margin_dpo/margin_mean": 388.4884033203125, "margin_dpo/margin_std": 689.441650390625, "step": 673 }, { "KL/chosen_KL_mean": -668.0368041992188, "KL/mean": -928.5050048828125, "KL/rejected_KL_mean": -1188.9732666015625, "KL/std": 601.396240234375, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0006932974210940301, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 520.9365234375, "fcm_dpo/q_t": 0.41753798723220825, "grad_norm": 24.077198028564453, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -0.9701206684112549, "logits/rejected": -0.9928334951400757, "logps/chosen": -713.4822998046875, "logps/ref_chosen": -45.445526123046875, "logps/ref_rejected": -70.04593658447266, "logps/rejected": -1259.019287109375, "loss": 1.1158, "margin_dpo/margin_mean": 520.9364624023438, "margin_dpo/margin_std": 724.0059814453125, "step": 674 }, { "KL/chosen_KL_mean": -739.5490112304688, "KL/mean": -1002.5335083007812, "KL/rejected_KL_mean": -1265.51806640625, "KL/std": 662.862060546875, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0006878894637338817, "fcm_dpo/delta": -0.039154816418886185, "fcm_dpo/margin": 525.9688720703125, "fcm_dpo/q_t": 0.4174017012119293, "grad_norm": 23.40828514099121, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.038738489151001, "logits/rejected": -1.0554476976394653, "logps/chosen": -783.725341796875, "logps/ref_chosen": -44.17628479003906, "logps/ref_rejected": -74.09197998046875, "logps/rejected": -1339.6099853515625, "loss": 1.1166, "margin_dpo/margin_mean": 525.9689331054688, "margin_dpo/margin_std": 709.8507080078125, "step": 675 }, { "KL/chosen_KL_mean": -795.6382446289062, "KL/mean": -1048.945068359375, "KL/rejected_KL_mean": -1302.251953125, "KL/std": 633.6385498046875, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0006878894637338817, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 506.61358642578125, "fcm_dpo/q_t": 0.4184497594833374, "grad_norm": 25.839481353759766, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -0.9737539291381836, "logits/rejected": -0.9969291687011719, "logps/chosen": -867.0367431640625, "logps/ref_chosen": -71.39852905273438, "logps/ref_rejected": -88.3587646484375, "logps/rejected": -1390.610595703125, "loss": 1.1168, "margin_dpo/margin_mean": 506.6136169433594, "margin_dpo/margin_std": 650.7418823242188, "step": 676 }, { "KL/chosen_KL_mean": -806.3810424804688, "KL/mean": -1042.6671142578125, "KL/rejected_KL_mean": -1278.953125, "KL/std": 644.4155883789062, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0006878894637338817, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 472.57208251953125, "fcm_dpo/q_t": 0.4279462397098541, "grad_norm": 28.72382164001465, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.048870325088501, "logits/rejected": -1.046755075454712, "logps/chosen": -862.908447265625, "logps/ref_chosen": -56.527435302734375, "logps/ref_rejected": -78.22654724121094, "logps/rejected": -1357.1796875, "loss": 1.1547, "margin_dpo/margin_mean": 472.57208251953125, "margin_dpo/margin_std": 792.822998046875, "step": 677 }, { "KL/chosen_KL_mean": -672.162841796875, "KL/mean": -966.9105834960938, "KL/rejected_KL_mean": -1261.658447265625, "KL/std": 726.8078002929688, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0006846521864645183, "fcm_dpo/delta": -0.04728377237915993, "fcm_dpo/margin": 589.4955444335938, "fcm_dpo/q_t": 0.40849733352661133, "grad_norm": 29.151065826416016, "learning_rate": 5.270012410216185e-11, "logits/chosen": -0.9713025093078613, "logits/rejected": -1.014235496520996, "logps/chosen": -718.2973022460938, "logps/ref_chosen": -46.13447570800781, "logps/ref_rejected": -80.60462951660156, "logps/rejected": -1342.262939453125, "loss": 1.0989, "margin_dpo/margin_mean": 589.4955444335938, "margin_dpo/margin_std": 841.677001953125, "step": 678 }, { "KL/chosen_KL_mean": -776.6614990234375, "KL/mean": -964.9478149414062, "KL/rejected_KL_mean": -1153.234130859375, "KL/std": 569.8460693359375, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0006814149091951549, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 376.57269287109375, "fcm_dpo/q_t": 0.44045108556747437, "grad_norm": 29.10932731628418, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -1.0045051574707031, "logits/rejected": -0.9941396117210388, "logps/chosen": -826.9564208984375, "logps/ref_chosen": -50.294921875, "logps/ref_rejected": -76.59813690185547, "logps/rejected": -1229.832275390625, "loss": 1.1964, "margin_dpo/margin_mean": 376.57269287109375, "margin_dpo/margin_std": 676.597412109375, "step": 679 }, { "KL/chosen_KL_mean": -779.4834594726562, "KL/mean": -1044.25634765625, "KL/rejected_KL_mean": -1309.029296875, "KL/std": 738.444091796875, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0006786106387153268, "fcm_dpo/delta": -0.020619181916117668, "fcm_dpo/margin": 529.5457763671875, "fcm_dpo/q_t": 0.4161868691444397, "grad_norm": 33.955841064453125, "learning_rate": 1.31753782067201e-11, "logits/chosen": -0.9959018230438232, "logits/rejected": -1.0296885967254639, "logps/chosen": -856.399169921875, "logps/ref_chosen": -76.91569519042969, "logps/ref_rejected": -112.384765625, "logps/rejected": -1421.4140625, "loss": 1.1302, "margin_dpo/margin_mean": 529.5457763671875, "margin_dpo/margin_std": 809.1583251953125, "step": 680 }, { "KL/chosen_KL_mean": -775.05322265625, "KL/mean": -983.98583984375, "KL/rejected_KL_mean": -1192.91845703125, "KL/std": 620.6334838867188, "epoch": 1.0, "fcm_dpo/beta": 0.0006786106387153268, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 417.8653564453125, "fcm_dpo/q_t": 0.4342602491378784, "grad_norm": 24.113101959228516, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -1.0600388050079346, "logits/rejected": -1.078291654586792, "logps/chosen": -836.010498046875, "logps/ref_chosen": -60.957279205322266, "logps/ref_rejected": -88.55797576904297, "logps/rejected": -1281.4765625, "loss": 1.1804, "margin_dpo/margin_mean": 417.8653259277344, "margin_dpo/margin_std": 694.8834228515625, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.127045486467, "train_runtime": 1737.1181, "train_samples_per_second": 25.098, "train_steps_per_second": 0.392 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }