{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.00527191162109375, "KL/mean": 0.016706019639968872, "KL/rejected_KL_mean": 0.028141021728515625, "KL/std": 0.272699236869812, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02287006378173828, "fcm_dpo/q_t": 0.5005706548690796, "grad_norm": 83.51216888427734, "learning_rate": 0.0, "logits/chosen": -0.4974287748336792, "logits/rejected": -0.43299180269241333, "logps/chosen": -50.1435661315918, "logps/ref_chosen": -50.14883804321289, "logps/ref_rejected": -74.1280517578125, "logps/rejected": -74.09991455078125, "loss": 1.389, "margin_dpo/margin_mean": -0.02287048101425171, "margin_dpo/margin_std": 0.41920793056488037, "step": 1 }, { "KL/chosen_KL_mean": -0.03498649597167969, "KL/mean": -0.00212840735912323, "KL/rejected_KL_mean": 0.030735015869140625, "KL/std": 0.24797174334526062, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06572261452674866, "fcm_dpo/q_t": 0.5016425848007202, "grad_norm": 72.19176483154297, "learning_rate": 7.246376811594203e-09, "logits/chosen": -0.49536412954330444, "logits/rejected": -0.4594460427761078, "logps/chosen": -52.65568923950195, "logps/ref_chosen": -52.620704650878906, "logps/ref_rejected": -75.30413818359375, "logps/rejected": -75.27340698242188, "loss": 1.3932, "margin_dpo/margin_mean": -0.06572240591049194, "margin_dpo/margin_std": 0.35048407316207886, "step": 2 }, { "KL/chosen_KL_mean": 0.011138916015625, "KL/mean": 0.01624371111392975, "KL/rejected_KL_mean": 0.021350860595703125, "KL/std": 0.2411436140537262, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.010211020708084106, "fcm_dpo/q_t": 0.5002557039260864, "grad_norm": 71.00299072265625, "learning_rate": 1.4492753623188406e-08, "logits/chosen": -0.48165208101272583, "logits/rejected": -0.44220495223999023, "logps/chosen": -60.970458984375, "logps/ref_chosen": -60.981597900390625, "logps/ref_rejected": -68.67259216308594, "logps/rejected": -68.6512451171875, "loss": 1.3877, "margin_dpo/margin_mean": -0.010210752487182617, "margin_dpo/margin_std": 0.36640018224716187, "step": 3 }, { "KL/chosen_KL_mean": 0.000141143798828125, "KL/mean": 0.032934680581092834, "KL/rejected_KL_mean": 0.06572723388671875, "KL/std": 0.26030904054641724, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06558361649513245, "fcm_dpo/q_t": 0.501638650894165, "grad_norm": 72.05967712402344, "learning_rate": 2.1739130434782606e-08, "logits/chosen": -0.46887367963790894, "logits/rejected": -0.4411189556121826, "logps/chosen": -56.76757049560547, "logps/ref_chosen": -56.7677116394043, "logps/ref_rejected": -86.64710998535156, "logps/rejected": -86.58138275146484, "loss": 1.3932, "margin_dpo/margin_mean": -0.06558331847190857, "margin_dpo/margin_std": 0.38221287727355957, "step": 4 }, { "KL/chosen_KL_mean": -0.008298873901367188, "KL/mean": 0.015575245022773743, "KL/rejected_KL_mean": 0.03945159912109375, "KL/std": 0.26084285974502563, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04775431752204895, "fcm_dpo/q_t": 0.5011929273605347, "grad_norm": 89.86566925048828, "learning_rate": 2.898550724637681e-08, "logits/chosen": -0.5145574808120728, "logits/rejected": -0.47081369161605835, "logps/chosen": -53.86767578125, "logps/ref_chosen": -53.859375, "logps/ref_rejected": -84.14918518066406, "logps/rejected": -84.10973358154297, "loss": 1.3914, "margin_dpo/margin_mean": -0.04775422811508179, "margin_dpo/margin_std": 0.37420204281806946, "step": 5 }, { "KL/chosen_KL_mean": -0.02581787109375, "KL/mean": 0.0013223886489868164, "KL/rejected_KL_mean": 0.02846527099609375, "KL/std": 0.254623144865036, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05428045988082886, "fcm_dpo/q_t": 0.5013556480407715, "grad_norm": 91.9102554321289, "learning_rate": 3.6231884057971014e-08, "logits/chosen": -0.5099787712097168, "logits/rejected": -0.4680352807044983, "logps/chosen": -63.033302307128906, "logps/ref_chosen": -63.007484436035156, "logps/ref_rejected": -92.64534759521484, "logps/rejected": -92.61688232421875, "loss": 1.3921, "margin_dpo/margin_mean": -0.05428001284599304, "margin_dpo/margin_std": 0.3631099462509155, "step": 6 }, { "KL/chosen_KL_mean": 0.014619827270507812, "KL/mean": 0.007828325033187866, "KL/rejected_KL_mean": 0.00103759765625, "KL/std": 0.28590360283851624, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013581156730651855, "fcm_dpo/q_t": 0.4996616840362549, "grad_norm": 82.45801544189453, "learning_rate": 4.347826086956521e-08, "logits/chosen": -0.5054452419281006, "logits/rejected": -0.47154390811920166, "logps/chosen": -57.76020050048828, "logps/ref_chosen": -57.774818420410156, "logps/ref_rejected": -103.92059326171875, "logps/rejected": -103.9195556640625, "loss": 1.3854, "margin_dpo/margin_mean": 0.01358148455619812, "margin_dpo/margin_std": 0.4422546327114105, "step": 7 }, { "KL/chosen_KL_mean": 0.01812744140625, "KL/mean": 0.019948050379753113, "KL/rejected_KL_mean": 0.021778106689453125, "KL/std": 0.27578145265579224, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.003651231527328491, "fcm_dpo/q_t": 0.5000911951065063, "grad_norm": 79.03565216064453, "learning_rate": 5.0724637681159424e-08, "logits/chosen": -0.49302494525909424, "logits/rejected": -0.46536964178085327, "logps/chosen": -58.697906494140625, "logps/ref_chosen": -58.716033935546875, "logps/ref_rejected": -79.3114242553711, "logps/rejected": -79.28964233398438, "loss": 1.387, "margin_dpo/margin_mean": -0.0036510825157165527, "margin_dpo/margin_std": 0.3407641053199768, "step": 8 }, { "KL/chosen_KL_mean": 0.00136566162109375, "KL/mean": 0.0017981678247451782, "KL/rejected_KL_mean": 0.002231597900390625, "KL/std": 0.2728290259838104, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.000867694616317749, "fcm_dpo/q_t": 0.5000215768814087, "grad_norm": 85.38078308105469, "learning_rate": 5.797101449275362e-08, "logits/chosen": -0.48713457584381104, "logits/rejected": -0.4398614168167114, "logps/chosen": -69.865478515625, "logps/ref_chosen": -69.8668441772461, "logps/ref_rejected": -99.6026611328125, "logps/rejected": -99.60043334960938, "loss": 1.3868, "margin_dpo/margin_mean": -0.0008689463138580322, "margin_dpo/margin_std": 0.41773098707199097, "step": 9 }, { "KL/chosen_KL_mean": -0.0075225830078125, "KL/mean": -0.013112656772136688, "KL/rejected_KL_mean": -0.018695831298828125, "KL/std": 0.253761887550354, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01117675006389618, "fcm_dpo/q_t": 0.4997211694717407, "grad_norm": 70.45758819580078, "learning_rate": 6.521739130434782e-08, "logits/chosen": -0.498451828956604, "logits/rejected": -0.4553406834602356, "logps/chosen": -48.36520767211914, "logps/ref_chosen": -48.35768508911133, "logps/ref_rejected": -80.37206268310547, "logps/rejected": -80.39076232910156, "loss": 1.3855, "margin_dpo/margin_mean": 0.011177659034729004, "margin_dpo/margin_std": 0.3267936706542969, "step": 10 }, { "KL/chosen_KL_mean": 0.0040302276611328125, "KL/mean": -0.007809385657310486, "KL/rejected_KL_mean": -0.019649505615234375, "KL/std": 0.23007726669311523, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.023678749799728394, "fcm_dpo/q_t": 0.49940839409828186, "grad_norm": 68.75594329833984, "learning_rate": 7.246376811594203e-08, "logits/chosen": -0.45761120319366455, "logits/rejected": -0.4324725270271301, "logps/chosen": -53.01282501220703, "logps/ref_chosen": -53.01685333251953, "logps/ref_rejected": -87.78038024902344, "logps/rejected": -87.8000259399414, "loss": 1.3842, "margin_dpo/margin_mean": 0.02367931604385376, "margin_dpo/margin_std": 0.3247877359390259, "step": 11 }, { "KL/chosen_KL_mean": -0.022077560424804688, "KL/mean": -0.030066579580307007, "KL/rejected_KL_mean": -0.038055419921875, "KL/std": 0.2843082547187805, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.015976309776306152, "fcm_dpo/q_t": 0.4996001422405243, "grad_norm": 89.8901596069336, "learning_rate": 7.971014492753623e-08, "logits/chosen": -0.5207371711730957, "logits/rejected": -0.48408442735671997, "logps/chosen": -61.8275146484375, "logps/ref_chosen": -61.80543518066406, "logps/ref_rejected": -104.8582763671875, "logps/rejected": -104.89633178710938, "loss": 1.3851, "margin_dpo/margin_mean": 0.015975654125213623, "margin_dpo/margin_std": 0.39000314474105835, "step": 12 }, { "KL/chosen_KL_mean": 0.018079757690429688, "KL/mean": 0.03311806917190552, "KL/rejected_KL_mean": 0.04815673828125, "KL/std": 0.299828439950943, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.030076265335083008, "fcm_dpo/q_t": 0.5007518529891968, "grad_norm": 79.48653411865234, "learning_rate": 8.695652173913042e-08, "logits/chosen": -0.46800822019577026, "logits/rejected": -0.4389190673828125, "logps/chosen": -64.24227905273438, "logps/ref_chosen": -64.2603530883789, "logps/ref_rejected": -87.20307922363281, "logps/rejected": -87.15492248535156, "loss": 1.3898, "margin_dpo/margin_mean": -0.030076295137405396, "margin_dpo/margin_std": 0.40763989090919495, "step": 13 }, { "KL/chosen_KL_mean": -0.027566909790039062, "KL/mean": -0.022734537720680237, "KL/rejected_KL_mean": -0.01790618896484375, "KL/std": 0.27561646699905396, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.009662479162216187, "fcm_dpo/q_t": 0.5002409219741821, "grad_norm": 85.24372863769531, "learning_rate": 9.420289855072464e-08, "logits/chosen": -0.46077555418014526, "logits/rejected": -0.420840322971344, "logps/chosen": -58.137779235839844, "logps/ref_chosen": -58.11021041870117, "logps/ref_rejected": -104.04708099365234, "logps/rejected": -104.06498718261719, "loss": 1.3878, "margin_dpo/margin_mean": -0.009661853313446045, "margin_dpo/margin_std": 0.4318341016769409, "step": 14 }, { "KL/chosen_KL_mean": -0.0386199951171875, "KL/mean": -0.045700281858444214, "KL/rejected_KL_mean": -0.052783966064453125, "KL/std": 0.23048102855682373, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014169454574584961, "fcm_dpo/q_t": 0.49964454770088196, "grad_norm": 64.31354522705078, "learning_rate": 1.0144927536231885e-07, "logits/chosen": -0.5125108361244202, "logits/rejected": -0.49486109614372253, "logps/chosen": -57.005531311035156, "logps/ref_chosen": -56.96691131591797, "logps/ref_rejected": -80.80863952636719, "logps/rejected": -80.8614273071289, "loss": 1.3852, "margin_dpo/margin_mean": 0.014169484376907349, "margin_dpo/margin_std": 0.3576519191265106, "step": 15 }, { "KL/chosen_KL_mean": 0.019407272338867188, "KL/mean": -0.03246238827705383, "KL/rejected_KL_mean": -0.08432769775390625, "KL/std": 0.27561452984809875, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10373848676681519, "fcm_dpo/q_t": 0.49740785360336304, "grad_norm": 83.85110473632812, "learning_rate": 1.0869565217391303e-07, "logits/chosen": -0.4948147237300873, "logits/rejected": -0.45085063576698303, "logps/chosen": -61.72048568725586, "logps/ref_chosen": -61.739891052246094, "logps/ref_rejected": -84.36947631835938, "logps/rejected": -84.45379638671875, "loss": 1.3763, "margin_dpo/margin_mean": 0.10373827815055847, "margin_dpo/margin_std": 0.38368722796440125, "step": 16 }, { "KL/chosen_KL_mean": 0.032100677490234375, "KL/mean": -0.02624189853668213, "KL/rejected_KL_mean": -0.08458328247070312, "KL/std": 0.2859901189804077, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11668473482131958, "fcm_dpo/q_t": 0.497084379196167, "grad_norm": 78.15997314453125, "learning_rate": 1.1594202898550725e-07, "logits/chosen": -0.511766791343689, "logits/rejected": -0.475982666015625, "logps/chosen": -67.67823791503906, "logps/ref_chosen": -67.71033477783203, "logps/ref_rejected": -85.37865447998047, "logps/rejected": -85.46324157714844, "loss": 1.3751, "margin_dpo/margin_mean": 0.11668440699577332, "margin_dpo/margin_std": 0.39858338236808777, "step": 17 }, { "KL/chosen_KL_mean": 0.030401229858398438, "KL/mean": -0.01864929497241974, "KL/rejected_KL_mean": -0.06769943237304688, "KL/std": 0.24296830594539642, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09810277819633484, "fcm_dpo/q_t": 0.4975478947162628, "grad_norm": 81.9465560913086, "learning_rate": 1.2318840579710146e-07, "logits/chosen": -0.5076760053634644, "logits/rejected": -0.4538722038269043, "logps/chosen": -47.70909118652344, "logps/ref_chosen": -47.7394905090332, "logps/ref_rejected": -75.4722900390625, "logps/rejected": -75.53998565673828, "loss": 1.3768, "margin_dpo/margin_mean": 0.09810280799865723, "margin_dpo/margin_std": 0.3186327815055847, "step": 18 }, { "KL/chosen_KL_mean": 0.016094207763671875, "KL/mean": -0.009688735008239746, "KL/rejected_KL_mean": -0.035465240478515625, "KL/std": 0.24112674593925476, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05155247449874878, "fcm_dpo/q_t": 0.49871158599853516, "grad_norm": 73.13494110107422, "learning_rate": 1.3043478260869563e-07, "logits/chosen": -0.5065875053405762, "logits/rejected": -0.4579053223133087, "logps/chosen": -70.18927001953125, "logps/ref_chosen": -70.20536041259766, "logps/ref_rejected": -89.7575912475586, "logps/rejected": -89.79306030273438, "loss": 1.3814, "margin_dpo/margin_mean": 0.05155327916145325, "margin_dpo/margin_std": 0.3446183204650879, "step": 19 }, { "KL/chosen_KL_mean": -0.03119659423828125, "KL/mean": -0.04489029943943024, "KL/rejected_KL_mean": -0.0585784912109375, "KL/std": 0.27683231234550476, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.027387112379074097, "fcm_dpo/q_t": 0.4993155002593994, "grad_norm": 73.76658630371094, "learning_rate": 1.3768115942028986e-07, "logits/chosen": -0.5554596185684204, "logits/rejected": -0.4994776248931885, "logps/chosen": -50.83443832397461, "logps/ref_chosen": -50.80324172973633, "logps/ref_rejected": -78.82334899902344, "logps/rejected": -78.88192749023438, "loss": 1.3839, "margin_dpo/margin_mean": 0.027386903762817383, "margin_dpo/margin_std": 0.36115431785583496, "step": 20 }, { "KL/chosen_KL_mean": 0.008577346801757812, "KL/mean": -0.03955095261335373, "KL/rejected_KL_mean": -0.087677001953125, "KL/std": 0.25458982586860657, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0962568074464798, "fcm_dpo/q_t": 0.49759435653686523, "grad_norm": 77.66507720947266, "learning_rate": 1.4492753623188405e-07, "logits/chosen": -0.4872417449951172, "logits/rejected": -0.46275877952575684, "logps/chosen": -50.054439544677734, "logps/ref_chosen": -50.063018798828125, "logps/ref_rejected": -77.86878967285156, "logps/rejected": -77.95646667480469, "loss": 1.3769, "margin_dpo/margin_mean": 0.09625642001628876, "margin_dpo/margin_std": 0.3011889159679413, "step": 21 }, { "KL/chosen_KL_mean": 0.05830574035644531, "KL/mean": -0.05371597409248352, "KL/rejected_KL_mean": -0.16573715209960938, "KL/std": 0.32429108023643494, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22404277324676514, "fcm_dpo/q_t": 0.494402676820755, "grad_norm": 84.32189178466797, "learning_rate": 1.5217391304347825e-07, "logits/chosen": -0.502210259437561, "logits/rejected": -0.459368497133255, "logps/chosen": -58.99932861328125, "logps/ref_chosen": -59.05763626098633, "logps/ref_rejected": -97.50466918945312, "logps/rejected": -97.67041015625, "loss": 1.3646, "margin_dpo/margin_mean": 0.22404247522354126, "margin_dpo/margin_std": 0.4611518979072571, "step": 22 }, { "KL/chosen_KL_mean": 0.014989852905273438, "KL/mean": -0.061570554971694946, "KL/rejected_KL_mean": -0.13812637329101562, "KL/std": 0.32376909255981445, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1531180739402771, "fcm_dpo/q_t": 0.49617335200309753, "grad_norm": 80.89447021484375, "learning_rate": 1.5942028985507245e-07, "logits/chosen": -0.4944395422935486, "logits/rejected": -0.4721882939338684, "logps/chosen": -60.062705993652344, "logps/ref_chosen": -60.07769775390625, "logps/ref_rejected": -81.13955688476562, "logps/rejected": -81.27767944335938, "loss": 1.3716, "margin_dpo/margin_mean": 0.1531185805797577, "margin_dpo/margin_std": 0.4450337886810303, "step": 23 }, { "KL/chosen_KL_mean": 0.009403228759765625, "KL/mean": -0.08805283904075623, "KL/rejected_KL_mean": -0.18550872802734375, "KL/std": 0.26540276408195496, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19491511583328247, "fcm_dpo/q_t": 0.4951290786266327, "grad_norm": 82.0745849609375, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.49009591341018677, "logits/rejected": -0.47387370467185974, "logps/chosen": -44.28163146972656, "logps/ref_chosen": -44.29103469848633, "logps/ref_rejected": -99.12521362304688, "logps/rejected": -99.31072235107422, "loss": 1.3672, "margin_dpo/margin_mean": 0.19491589069366455, "margin_dpo/margin_std": 0.3483674228191376, "step": 24 }, { "KL/chosen_KL_mean": 0.0049686431884765625, "KL/mean": -0.09683172404766083, "KL/rejected_KL_mean": -0.19862747192382812, "KL/std": 0.31956106424331665, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20359697937965393, "fcm_dpo/q_t": 0.4949125647544861, "grad_norm": 73.71135711669922, "learning_rate": 1.7391304347826085e-07, "logits/chosen": -0.48128461837768555, "logits/rejected": -0.45070409774780273, "logps/chosen": -52.532081604003906, "logps/ref_chosen": -52.537052154541016, "logps/ref_rejected": -89.34219360351562, "logps/rejected": -89.54081726074219, "loss": 1.3664, "margin_dpo/margin_mean": 0.20359739661216736, "margin_dpo/margin_std": 0.36989957094192505, "step": 25 }, { "KL/chosen_KL_mean": 0.07361602783203125, "KL/mean": -0.10151402652263641, "KL/rejected_KL_mean": -0.2766456604003906, "KL/std": 0.41175055503845215, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3502626121044159, "fcm_dpo/q_t": 0.49125543236732483, "grad_norm": 87.7682876586914, "learning_rate": 1.8115942028985507e-07, "logits/chosen": -0.5210403800010681, "logits/rejected": -0.48897308111190796, "logps/chosen": -53.84918975830078, "logps/ref_chosen": -53.92280578613281, "logps/ref_rejected": -103.35971069335938, "logps/rejected": -103.63636016845703, "loss": 1.3523, "margin_dpo/margin_mean": 0.3502630591392517, "margin_dpo/margin_std": 0.549583911895752, "step": 26 }, { "KL/chosen_KL_mean": 0.12030792236328125, "KL/mean": -0.14729268848896027, "KL/rejected_KL_mean": -0.4149017333984375, "KL/std": 0.4725148677825928, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5352023839950562, "fcm_dpo/q_t": 0.4866352677345276, "grad_norm": 93.7070541381836, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.4835624098777771, "logits/rejected": -0.4467238783836365, "logps/chosen": -42.778221130371094, "logps/ref_chosen": -42.898529052734375, "logps/ref_rejected": -98.72419738769531, "logps/rejected": -99.13909912109375, "loss": 1.3343, "margin_dpo/margin_mean": 0.535202145576477, "margin_dpo/margin_std": 0.5412513017654419, "step": 27 }, { "KL/chosen_KL_mean": -0.009973526000976562, "KL/mean": -0.14437878131866455, "KL/rejected_KL_mean": -0.27878570556640625, "KL/std": 0.4356634020805359, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26881152391433716, "fcm_dpo/q_t": 0.4932839870452881, "grad_norm": 75.4567642211914, "learning_rate": 1.9565217391304347e-07, "logits/chosen": -0.5053049921989441, "logits/rejected": -0.4499465525150299, "logps/chosen": -60.56647491455078, "logps/ref_chosen": -60.55650329589844, "logps/ref_rejected": -91.40111541748047, "logps/rejected": -91.67990112304688, "loss": 1.3606, "margin_dpo/margin_mean": 0.26881229877471924, "margin_dpo/margin_std": 0.6206564903259277, "step": 28 }, { "KL/chosen_KL_mean": 0.102294921875, "KL/mean": -0.17882275581359863, "KL/rejected_KL_mean": -0.4599418640136719, "KL/std": 0.5050041079521179, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5622385144233704, "fcm_dpo/q_t": 0.4859647750854492, "grad_norm": 90.32099151611328, "learning_rate": 2.028985507246377e-07, "logits/chosen": -0.5919088125228882, "logits/rejected": -0.5483555793762207, "logps/chosen": -57.70549011230469, "logps/ref_chosen": -57.80778503417969, "logps/ref_rejected": -97.39434814453125, "logps/rejected": -97.85429382324219, "loss": 1.3318, "margin_dpo/margin_mean": 0.562239408493042, "margin_dpo/margin_std": 0.5976601839065552, "step": 29 }, { "KL/chosen_KL_mean": 0.1593170166015625, "KL/mean": -0.19309324026107788, "KL/rejected_KL_mean": -0.5455055236816406, "KL/std": 0.6001869440078735, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7048233151435852, "fcm_dpo/q_t": 0.48240745067596436, "grad_norm": 87.2791519165039, "learning_rate": 2.1014492753623187e-07, "logits/chosen": -0.4931980073451996, "logits/rejected": -0.4624643921852112, "logps/chosen": -52.418052673339844, "logps/ref_chosen": -52.577369689941406, "logps/ref_rejected": -98.48920440673828, "logps/rejected": -99.03471374511719, "loss": 1.3181, "margin_dpo/margin_mean": 0.7048235535621643, "margin_dpo/margin_std": 0.6384134292602539, "step": 30 }, { "KL/chosen_KL_mean": 0.028833389282226562, "KL/mean": -0.1849493831396103, "KL/rejected_KL_mean": -0.39873504638671875, "KL/std": 0.5692216157913208, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.42756929993629456, "fcm_dpo/q_t": 0.489329993724823, "grad_norm": 67.7244873046875, "learning_rate": 2.1739130434782607e-07, "logits/chosen": -0.5044924020767212, "logits/rejected": -0.4595579504966736, "logps/chosen": -63.7780876159668, "logps/ref_chosen": -63.806922912597656, "logps/ref_rejected": -72.89400482177734, "logps/rejected": -73.29273986816406, "loss": 1.3454, "margin_dpo/margin_mean": 0.42756909132003784, "margin_dpo/margin_std": 0.738195538520813, "step": 31 }, { "KL/chosen_KL_mean": 0.25072669982910156, "KL/mean": -0.14465753734111786, "KL/rejected_KL_mean": -0.5400428771972656, "KL/std": 0.7525659799575806, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7907640933990479, "fcm_dpo/q_t": 0.4803106188774109, "grad_norm": 83.765625, "learning_rate": 2.2463768115942027e-07, "logits/chosen": -0.5086926221847534, "logits/rejected": -0.46712368726730347, "logps/chosen": -62.488800048828125, "logps/ref_chosen": -62.739524841308594, "logps/ref_rejected": -89.3175048828125, "logps/rejected": -89.8575439453125, "loss": 1.311, "margin_dpo/margin_mean": 0.790764331817627, "margin_dpo/margin_std": 0.9426593780517578, "step": 32 }, { "KL/chosen_KL_mean": 0.07011795043945312, "KL/mean": -0.21620376408100128, "KL/rejected_KL_mean": -0.5025215148925781, "KL/std": 0.5353639125823975, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.572637677192688, "fcm_dpo/q_t": 0.4857015609741211, "grad_norm": 72.43000030517578, "learning_rate": 2.318840579710145e-07, "logits/chosen": -0.46579599380493164, "logits/rejected": -0.4388483166694641, "logps/chosen": -53.19085693359375, "logps/ref_chosen": -53.26097106933594, "logps/ref_rejected": -87.8851318359375, "logps/rejected": -88.38765716552734, "loss": 1.3308, "margin_dpo/margin_mean": 0.5726377964019775, "margin_dpo/margin_std": 0.5959737300872803, "step": 33 }, { "KL/chosen_KL_mean": 0.08471107482910156, "KL/mean": -0.3118361830711365, "KL/rejected_KL_mean": -0.7083892822265625, "KL/std": 0.7339533567428589, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7930940389633179, "fcm_dpo/q_t": 0.48022860288619995, "grad_norm": 77.76030731201172, "learning_rate": 2.391304347826087e-07, "logits/chosen": -0.45824792981147766, "logits/rejected": -0.4408394694328308, "logps/chosen": -50.73261642456055, "logps/ref_chosen": -50.81732940673828, "logps/ref_rejected": -101.92184448242188, "logps/rejected": -102.63023376464844, "loss": 1.3103, "margin_dpo/margin_mean": 0.7930936813354492, "margin_dpo/margin_std": 0.8268696069717407, "step": 34 }, { "KL/chosen_KL_mean": 0.15523910522460938, "KL/mean": -0.4550577700138092, "KL/rejected_KL_mean": -1.0653495788574219, "KL/std": 1.036551594734192, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2205886840820312, "fcm_dpo/q_t": 0.46972745656967163, "grad_norm": 82.48926544189453, "learning_rate": 2.463768115942029e-07, "logits/chosen": -0.5093280076980591, "logits/rejected": -0.47247546911239624, "logps/chosen": -50.8692512512207, "logps/ref_chosen": -51.02449035644531, "logps/ref_rejected": -106.82443237304688, "logps/rejected": -107.88977813720703, "loss": 1.2718, "margin_dpo/margin_mean": 1.2205885648727417, "margin_dpo/margin_std": 1.2264792919158936, "step": 35 }, { "KL/chosen_KL_mean": 0.03851318359375, "KL/mean": -0.5172837972640991, "KL/rejected_KL_mean": -1.0730781555175781, "KL/std": 1.0524473190307617, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1115931272506714, "fcm_dpo/q_t": 0.47235026955604553, "grad_norm": 73.26398468017578, "learning_rate": 2.536231884057971e-07, "logits/chosen": -0.5541160106658936, "logits/rejected": -0.5175681114196777, "logps/chosen": -51.952980041503906, "logps/ref_chosen": -51.991493225097656, "logps/ref_rejected": -86.0406265258789, "logps/rejected": -87.11370849609375, "loss": 1.282, "margin_dpo/margin_mean": 1.1115922927856445, "margin_dpo/margin_std": 1.172994613647461, "step": 36 }, { "KL/chosen_KL_mean": 0.0073757171630859375, "KL/mean": -0.5105187892913818, "KL/rejected_KL_mean": -1.0284156799316406, "KL/std": 1.0426149368286133, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0357892513275146, "fcm_dpo/q_t": 0.4743063151836395, "grad_norm": 61.77984619140625, "learning_rate": 2.6086956521739126e-07, "logits/chosen": -0.511062502861023, "logits/rejected": -0.4673815965652466, "logps/chosen": -62.79973220825195, "logps/ref_chosen": -62.807106018066406, "logps/ref_rejected": -77.89507293701172, "logps/rejected": -78.92349243164062, "loss": 1.2901, "margin_dpo/margin_mean": 1.0357893705368042, "margin_dpo/margin_std": 1.3421692848205566, "step": 37 }, { "KL/chosen_KL_mean": 0.18421363830566406, "KL/mean": -0.5241457223892212, "KL/rejected_KL_mean": -1.2325019836425781, "KL/std": 1.3683629035949707, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4167163372039795, "fcm_dpo/q_t": 0.4650591313838959, "grad_norm": 69.80412292480469, "learning_rate": 2.681159420289855e-07, "logits/chosen": -0.5317132472991943, "logits/rejected": -0.4989948868751526, "logps/chosen": -48.20630645751953, "logps/ref_chosen": -48.39051818847656, "logps/ref_rejected": -97.91244506835938, "logps/rejected": -99.14494323730469, "loss": 1.2566, "margin_dpo/margin_mean": 1.4167169332504272, "margin_dpo/margin_std": 1.6653265953063965, "step": 38 }, { "KL/chosen_KL_mean": 0.10546493530273438, "KL/mean": -0.7446496486663818, "KL/rejected_KL_mean": -1.5947608947753906, "KL/std": 1.3329132795333862, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7002233266830444, "fcm_dpo/q_t": 0.45783674716949463, "grad_norm": 76.12843322753906, "learning_rate": 2.753623188405797e-07, "logits/chosen": -0.5526484847068787, "logits/rejected": -0.5124742984771729, "logps/chosen": -50.6450080871582, "logps/ref_chosen": -50.75047302246094, "logps/ref_rejected": -78.56951141357422, "logps/rejected": -80.16427612304688, "loss": 1.2289, "margin_dpo/margin_mean": 1.7002232074737549, "margin_dpo/margin_std": 1.4578522443771362, "step": 39 }, { "KL/chosen_KL_mean": 0.2315082550048828, "KL/mean": -0.5488898754119873, "KL/rejected_KL_mean": -1.3292922973632812, "KL/std": 1.390199065208435, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5607997179031372, "fcm_dpo/q_t": 0.4614887237548828, "grad_norm": 59.96656036376953, "learning_rate": 2.8260869565217386e-07, "logits/chosen": -0.5169125199317932, "logits/rejected": -0.48675721883773804, "logps/chosen": -57.75355911254883, "logps/ref_chosen": -57.985069274902344, "logps/ref_rejected": -74.3000717163086, "logps/rejected": -75.62936401367188, "loss": 1.2437, "margin_dpo/margin_mean": 1.560800313949585, "margin_dpo/margin_std": 1.693495750427246, "step": 40 }, { "KL/chosen_KL_mean": 0.028055191040039062, "KL/mean": -0.925218403339386, "KL/rejected_KL_mean": -1.8784866333007812, "KL/std": 1.867018699645996, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9065409898757935, "fcm_dpo/q_t": 0.4531286358833313, "grad_norm": 67.948974609375, "learning_rate": 2.898550724637681e-07, "logits/chosen": -0.5261883735656738, "logits/rejected": -0.4887946844100952, "logps/chosen": -62.667762756347656, "logps/ref_chosen": -62.69581604003906, "logps/ref_rejected": -97.02352905273438, "logps/rejected": -98.90201568603516, "loss": 1.2148, "margin_dpo/margin_mean": 1.9065414667129517, "margin_dpo/margin_std": 1.9881553649902344, "step": 41 }, { "KL/chosen_KL_mean": 0.22861289978027344, "KL/mean": -1.08327054977417, "KL/rejected_KL_mean": -2.3951492309570312, "KL/std": 2.1879308223724365, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.11069603264331818, "fcm_dpo/delta": 0.12672916054725647, "fcm_dpo/margin": 2.623763084411621, "fcm_dpo/q_t": 0.43353042006492615, "grad_norm": 85.34500122070312, "learning_rate": 2.971014492753623e-07, "logits/chosen": -0.5527105331420898, "logits/rejected": -0.5064513683319092, "logps/chosen": -58.737815856933594, "logps/ref_chosen": -58.966426849365234, "logps/ref_rejected": -109.90837097167969, "logps/rejected": -112.30352783203125, "loss": 1.134, "margin_dpo/margin_mean": 2.623763084411621, "margin_dpo/margin_std": 2.4292526245117188, "step": 42 }, { "KL/chosen_KL_mean": 0.5411643981933594, "KL/mean": -0.7341057062149048, "KL/rejected_KL_mean": -2.0093841552734375, "KL/std": 1.8598041534423828, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.12222427129745483, "fcm_dpo/delta": 0.10482510924339294, "fcm_dpo/margin": 2.550539255142212, "fcm_dpo/q_t": 0.4277779459953308, "grad_norm": 83.47020721435547, "learning_rate": 3.043478260869565e-07, "logits/chosen": -0.5464547276496887, "logits/rejected": -0.5218390226364136, "logps/chosen": -53.614830017089844, "logps/ref_chosen": -54.15599822998047, "logps/ref_rejected": -96.48019409179688, "logps/rejected": -98.48957824707031, "loss": 1.1131, "margin_dpo/margin_mean": 2.550539255142212, "margin_dpo/margin_std": 1.9405533075332642, "step": 43 }, { "KL/chosen_KL_mean": 0.24042892456054688, "KL/mean": -1.1693687438964844, "KL/rejected_KL_mean": -2.5791664123535156, "KL/std": 2.276883602142334, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.12953037023544312, "fcm_dpo/delta": 0.042644016444683075, "fcm_dpo/margin": 2.819594383239746, "fcm_dpo/q_t": 0.41355597972869873, "grad_norm": 95.25187683105469, "learning_rate": 3.115942028985507e-07, "logits/chosen": -0.4547986388206482, "logits/rejected": -0.4352615475654602, "logps/chosen": -49.838069915771484, "logps/ref_chosen": -50.07849884033203, "logps/ref_rejected": -108.78376007080078, "logps/rejected": -111.36293029785156, "loss": 1.0737, "margin_dpo/margin_mean": 2.819593906402588, "margin_dpo/margin_std": 2.199711322784424, "step": 44 }, { "KL/chosen_KL_mean": 0.1290454864501953, "KL/mean": -1.0186073780059814, "KL/rejected_KL_mean": -2.166259765625, "KL/std": 2.0731301307678223, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.1417458951473236, "fcm_dpo/delta": 0.08769898861646652, "fcm_dpo/margin": 2.295309066772461, "fcm_dpo/q_t": 0.42574048042297363, "grad_norm": 79.44340515136719, "learning_rate": 3.188405797101449e-07, "logits/chosen": -0.4766472280025482, "logits/rejected": -0.4639527201652527, "logps/chosen": -48.28588104248047, "logps/ref_chosen": -48.4149284362793, "logps/ref_rejected": -77.93643188476562, "logps/rejected": -80.10269165039062, "loss": 1.1186, "margin_dpo/margin_mean": 2.2953085899353027, "margin_dpo/margin_std": 2.5507259368896484, "step": 45 }, { "KL/chosen_KL_mean": 0.21740150451660156, "KL/mean": -1.253394365310669, "KL/rejected_KL_mean": -2.724193572998047, "KL/std": 2.6467161178588867, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.14241422712802887, "fcm_dpo/delta": -0.024083826690912247, "fcm_dpo/margin": 2.941596031188965, "fcm_dpo/q_t": 0.4018367528915405, "grad_norm": 87.27326965332031, "learning_rate": 3.260869565217391e-07, "logits/chosen": -0.5110729932785034, "logits/rejected": -0.4591823220252991, "logps/chosen": -55.78202438354492, "logps/ref_chosen": -55.999427795410156, "logps/ref_rejected": -95.652587890625, "logps/rejected": -98.37678527832031, "loss": 1.0582, "margin_dpo/margin_mean": 2.9415955543518066, "margin_dpo/margin_std": 3.2013213634490967, "step": 46 }, { "KL/chosen_KL_mean": 0.37536048889160156, "KL/mean": -1.1281816959381104, "KL/rejected_KL_mean": -2.6317214965820312, "KL/std": 2.644698143005371, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.14004012942314148, "fcm_dpo/delta": -0.02779523655772209, "fcm_dpo/margin": 3.007082462310791, "fcm_dpo/q_t": 0.39907705783843994, "grad_norm": 82.21805572509766, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.5815363526344299, "logits/rejected": -0.5293323397636414, "logps/chosen": -57.55072021484375, "logps/ref_chosen": -57.92607879638672, "logps/ref_rejected": -94.67920684814453, "logps/rejected": -97.31092834472656, "loss": 1.046, "margin_dpo/margin_mean": 3.007082462310791, "margin_dpo/margin_std": 2.746891975402832, "step": 47 }, { "KL/chosen_KL_mean": 0.05398750305175781, "KL/mean": -1.5238816738128662, "KL/rejected_KL_mean": -3.1017494201660156, "KL/std": 2.673588514328003, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.1313745081424713, "fcm_dpo/delta": -0.02106369659304619, "fcm_dpo/margin": 3.1557374000549316, "fcm_dpo/q_t": 0.4008999466896057, "grad_norm": 87.41127014160156, "learning_rate": 3.4057971014492755e-07, "logits/chosen": -0.5923665761947632, "logits/rejected": -0.5343451499938965, "logps/chosen": -57.13408660888672, "logps/ref_chosen": -57.188072204589844, "logps/ref_rejected": -88.0166015625, "logps/rejected": -91.11834716796875, "loss": 1.0545, "margin_dpo/margin_mean": 3.15573787689209, "margin_dpo/margin_std": 2.916724681854248, "step": 48 }, { "KL/chosen_KL_mean": 0.3004131317138672, "KL/mean": -1.5658769607543945, "KL/rejected_KL_mean": -3.4321632385253906, "KL/std": 3.3690695762634277, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.12684862315654755, "fcm_dpo/delta": -0.09592577815055847, "fcm_dpo/margin": 3.7325761318206787, "fcm_dpo/q_t": 0.38731497526168823, "grad_norm": 71.87007141113281, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.5684317350387573, "logits/rejected": -0.5121915936470032, "logps/chosen": -61.384857177734375, "logps/ref_chosen": -61.685272216796875, "logps/ref_rejected": -83.76747131347656, "logps/rejected": -87.19963836669922, "loss": 1.025, "margin_dpo/margin_mean": 3.7325758934020996, "margin_dpo/margin_std": 3.874316930770874, "step": 49 }, { "KL/chosen_KL_mean": -0.1456451416015625, "KL/mean": -2.1883692741394043, "KL/rejected_KL_mean": -4.231086730957031, "KL/std": 3.4617319107055664, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.11314891278743744, "fcm_dpo/delta": -0.08234786987304688, "fcm_dpo/margin": 4.085442066192627, "fcm_dpo/q_t": 0.3882601261138916, "grad_norm": 68.02591705322266, "learning_rate": 3.5507246376811595e-07, "logits/chosen": -0.5450817346572876, "logits/rejected": -0.5092841982841492, "logps/chosen": -58.869781494140625, "logps/ref_chosen": -58.72413635253906, "logps/ref_rejected": -96.35814666748047, "logps/rejected": -100.5892333984375, "loss": 1.0227, "margin_dpo/margin_mean": 4.085441589355469, "margin_dpo/margin_std": 3.960975408554077, "step": 50 }, { "KL/chosen_KL_mean": -0.2403240203857422, "KL/mean": -2.285604953765869, "KL/rejected_KL_mean": -4.330890655517578, "KL/std": 4.1982645988464355, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10526888072490692, "fcm_dpo/delta": -0.04377248138189316, "fcm_dpo/margin": 4.090564250946045, "fcm_dpo/q_t": 0.4009581208229065, "grad_norm": 53.3188591003418, "learning_rate": 3.6231884057971015e-07, "logits/chosen": -0.5137035250663757, "logits/rejected": -0.4802321493625641, "logps/chosen": -61.613990783691406, "logps/ref_chosen": -61.3736686706543, "logps/ref_rejected": -76.00199890136719, "logps/rejected": -80.33289337158203, "loss": 1.0685, "margin_dpo/margin_mean": 4.090563774108887, "margin_dpo/margin_std": 5.139671325683594, "step": 51 }, { "KL/chosen_KL_mean": 0.37200355529785156, "KL/mean": -2.7163591384887695, "KL/rejected_KL_mean": -5.804725646972656, "KL/std": 4.783724784851074, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.0902351513504982, "fcm_dpo/delta": -0.22192168235778809, "fcm_dpo/margin": 6.176729202270508, "fcm_dpo/q_t": 0.35875505208969116, "grad_norm": 56.78388595581055, "learning_rate": 3.695652173913043e-07, "logits/chosen": -0.5432096719741821, "logits/rejected": -0.48703598976135254, "logps/chosen": -51.965354919433594, "logps/ref_chosen": -52.33735656738281, "logps/ref_rejected": -79.97391510009766, "logps/rejected": -85.77864074707031, "loss": 0.9533, "margin_dpo/margin_mean": 6.176729202270508, "margin_dpo/margin_std": 5.141578197479248, "step": 52 }, { "KL/chosen_KL_mean": -0.20578384399414062, "KL/mean": -3.3196845054626465, "KL/rejected_KL_mean": -6.433582305908203, "KL/std": 5.414214611053467, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.08034509420394897, "fcm_dpo/delta": -0.13391053676605225, "fcm_dpo/margin": 6.227805137634277, "fcm_dpo/q_t": 0.38188230991363525, "grad_norm": 52.141883850097656, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.6204248666763306, "logits/rejected": -0.5989758968353271, "logps/chosen": -53.52043533325195, "logps/ref_chosen": -53.31465148925781, "logps/ref_rejected": -91.78359985351562, "logps/rejected": -98.21717834472656, "loss": 1.0126, "margin_dpo/margin_mean": 6.227805137634277, "margin_dpo/margin_std": 6.215023517608643, "step": 53 }, { "KL/chosen_KL_mean": -0.3681602478027344, "KL/mean": -3.0784969329833984, "KL/rejected_KL_mean": -5.788829803466797, "KL/std": 5.0799665451049805, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.07495428621768951, "fcm_dpo/delta": -0.008042382076382637, "fcm_dpo/margin": 5.420670032501221, "fcm_dpo/q_t": 0.4037601351737976, "grad_norm": 46.64132308959961, "learning_rate": 3.8405797101449274e-07, "logits/chosen": -0.5916078090667725, "logits/rejected": -0.5381488800048828, "logps/chosen": -51.05682373046875, "logps/ref_chosen": -50.68865966796875, "logps/ref_rejected": -91.71539306640625, "logps/rejected": -97.50422668457031, "loss": 1.0548, "margin_dpo/margin_mean": 5.420670032501221, "margin_dpo/margin_std": 5.109761714935303, "step": 54 }, { "KL/chosen_KL_mean": -0.9354953765869141, "KL/mean": -4.157511234283447, "KL/rejected_KL_mean": -7.379520416259766, "KL/std": 6.1912150382995605, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.07125985622406006, "fcm_dpo/delta": -0.07659684866666794, "fcm_dpo/margin": 6.444026470184326, "fcm_dpo/q_t": 0.39212772250175476, "grad_norm": 44.093040466308594, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.6305919885635376, "logits/rejected": -0.5667720437049866, "logps/chosen": -63.55072784423828, "logps/ref_chosen": -62.615234375, "logps/ref_rejected": -88.99349975585938, "logps/rejected": -96.3730239868164, "loss": 1.0454, "margin_dpo/margin_mean": 6.44402551651001, "margin_dpo/margin_std": 7.479229927062988, "step": 55 }, { "KL/chosen_KL_mean": -0.7067909240722656, "KL/mean": -3.802842140197754, "KL/rejected_KL_mean": -6.898895263671875, "KL/std": 6.042604446411133, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.0678362250328064, "fcm_dpo/delta": -0.02534756436944008, "fcm_dpo/margin": 6.192107677459717, "fcm_dpo/q_t": 0.40231359004974365, "grad_norm": 39.32019805908203, "learning_rate": 3.9855072463768114e-07, "logits/chosen": -0.6268018484115601, "logits/rejected": -0.5851395130157471, "logps/chosen": -58.63951873779297, "logps/ref_chosen": -57.9327278137207, "logps/ref_rejected": -94.1744384765625, "logps/rejected": -101.07333374023438, "loss": 1.0686, "margin_dpo/margin_mean": 6.192108154296875, "margin_dpo/margin_std": 7.3723554611206055, "step": 56 }, { "KL/chosen_KL_mean": -0.7017936706542969, "KL/mean": -4.175287246704102, "KL/rejected_KL_mean": -7.648784637451172, "KL/std": 5.941527366638184, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.06503595411777496, "fcm_dpo/delta": -0.06703046709299088, "fcm_dpo/margin": 6.946985244750977, "fcm_dpo/q_t": 0.3904920220375061, "grad_norm": 43.75485610961914, "learning_rate": 4.057971014492754e-07, "logits/chosen": -0.588129997253418, "logits/rejected": -0.5603782534599304, "logps/chosen": -71.19708251953125, "logps/ref_chosen": -70.49528503417969, "logps/ref_rejected": -95.56546020507812, "logps/rejected": -103.21424102783203, "loss": 1.0297, "margin_dpo/margin_mean": 6.946985721588135, "margin_dpo/margin_std": 6.547435283660889, "step": 57 }, { "KL/chosen_KL_mean": -1.027395248413086, "KL/mean": -4.887474060058594, "KL/rejected_KL_mean": -8.747550964355469, "KL/std": 6.817049980163574, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.06007317081093788, "fcm_dpo/delta": -0.08280885219573975, "fcm_dpo/margin": 7.720156192779541, "fcm_dpo/q_t": 0.3894719183444977, "grad_norm": 42.88956069946289, "learning_rate": 4.1304347826086954e-07, "logits/chosen": -0.5993902683258057, "logits/rejected": -0.5237987041473389, "logps/chosen": -63.160335540771484, "logps/ref_chosen": -62.13294219970703, "logps/ref_rejected": -84.61729431152344, "logps/rejected": -93.3648452758789, "loss": 1.0325, "margin_dpo/margin_mean": 7.720156669616699, "margin_dpo/margin_std": 8.126496315002441, "step": 58 }, { "KL/chosen_KL_mean": -1.5261516571044922, "KL/mean": -5.766140460968018, "KL/rejected_KL_mean": -10.00613021850586, "KL/std": 7.495608806610107, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.053062424063682556, "fcm_dpo/delta": -0.07194283604621887, "fcm_dpo/margin": 8.47997760772705, "fcm_dpo/q_t": 0.39142686128616333, "grad_norm": 40.191532135009766, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.6348840594291687, "logits/rejected": -0.595291793346405, "logps/chosen": -53.45867919921875, "logps/ref_chosen": -51.932525634765625, "logps/ref_rejected": -88.88520050048828, "logps/rejected": -98.89132690429688, "loss": 1.0373, "margin_dpo/margin_mean": 8.479978561401367, "margin_dpo/margin_std": 8.641672134399414, "step": 59 }, { "KL/chosen_KL_mean": -2.7480087280273438, "KL/mean": -6.107456684112549, "KL/rejected_KL_mean": -9.466899871826172, "KL/std": 6.831831455230713, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.054739855229854584, "fcm_dpo/delta": 0.039379529654979706, "fcm_dpo/margin": 6.718890190124512, "fcm_dpo/q_t": 0.4139224886894226, "grad_norm": 41.76387023925781, "learning_rate": 4.2753623188405794e-07, "logits/chosen": -0.6037277579307556, "logits/rejected": -0.5432413816452026, "logps/chosen": -63.6901969909668, "logps/ref_chosen": -60.94218826293945, "logps/ref_rejected": -85.39340209960938, "logps/rejected": -94.86029815673828, "loss": 1.096, "margin_dpo/margin_mean": 6.718890190124512, "margin_dpo/margin_std": 7.675610065460205, "step": 60 }, { "KL/chosen_KL_mean": -1.6316070556640625, "KL/mean": -5.852381706237793, "KL/rejected_KL_mean": -10.073162078857422, "KL/std": 9.248079299926758, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.05262649431824684, "fcm_dpo/delta": -0.059413738548755646, "fcm_dpo/margin": 8.441549301147461, "fcm_dpo/q_t": 0.3997190594673157, "grad_norm": 35.92430877685547, "learning_rate": 4.3478260869565214e-07, "logits/chosen": -0.597582995891571, "logits/rejected": -0.5633736848831177, "logps/chosen": -62.26512908935547, "logps/ref_chosen": -60.633522033691406, "logps/ref_rejected": -89.85249328613281, "logps/rejected": -99.9256591796875, "loss": 1.0687, "margin_dpo/margin_mean": 8.441547393798828, "margin_dpo/margin_std": 11.492226600646973, "step": 61 }, { "KL/chosen_KL_mean": -1.7598438262939453, "KL/mean": -4.921261787414551, "KL/rejected_KL_mean": -8.082679748535156, "KL/std": 6.796797275543213, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.05579136312007904, "fcm_dpo/delta": 0.055100131779909134, "fcm_dpo/margin": 6.3228349685668945, "fcm_dpo/q_t": 0.4199411869049072, "grad_norm": 35.19557189941406, "learning_rate": 4.420289855072464e-07, "logits/chosen": -0.611149251461029, "logits/rejected": -0.5770503282546997, "logps/chosen": -57.910614013671875, "logps/ref_chosen": -56.15077209472656, "logps/ref_rejected": -75.56619262695312, "logps/rejected": -83.64888000488281, "loss": 1.1166, "margin_dpo/margin_mean": 6.322835922241211, "margin_dpo/margin_std": 8.347599029541016, "step": 62 }, { "KL/chosen_KL_mean": -2.888803482055664, "KL/mean": -7.189858436584473, "KL/rejected_KL_mean": -11.490913391113281, "KL/std": 8.740306854248047, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.051797665655612946, "fcm_dpo/delta": -0.06353449821472168, "fcm_dpo/margin": 8.602106094360352, "fcm_dpo/q_t": 0.3924814760684967, "grad_norm": 37.61563491821289, "learning_rate": 4.4927536231884053e-07, "logits/chosen": -0.6047888398170471, "logits/rejected": -0.5606403350830078, "logps/chosen": -76.03619384765625, "logps/ref_chosen": -73.14739227294922, "logps/ref_rejected": -97.61006164550781, "logps/rejected": -109.1009750366211, "loss": 1.0442, "margin_dpo/margin_mean": 8.602106094360352, "margin_dpo/margin_std": 9.156990051269531, "step": 63 }, { "KL/chosen_KL_mean": -1.1866321563720703, "KL/mean": -6.277315139770508, "KL/rejected_KL_mean": -11.368000030517578, "KL/std": 9.430252075195312, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.0463373102247715, "fcm_dpo/delta": -0.10717649757862091, "fcm_dpo/margin": 10.181361198425293, "fcm_dpo/q_t": 0.38632842898368835, "grad_norm": 33.78566360473633, "learning_rate": 4.5652173913043473e-07, "logits/chosen": -0.5959369540214539, "logits/rejected": -0.5642282366752625, "logps/chosen": -55.18523406982422, "logps/ref_chosen": -53.998600006103516, "logps/ref_rejected": -93.53019714355469, "logps/rejected": -104.89820098876953, "loss": 1.0364, "margin_dpo/margin_mean": 10.18136215209961, "margin_dpo/margin_std": 11.540030479431152, "step": 64 }, { "KL/chosen_KL_mean": -3.5648746490478516, "KL/mean": -8.770621299743652, "KL/rejected_KL_mean": -13.976367950439453, "KL/std": 10.70554256439209, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.043465256690979004, "fcm_dpo/delta": -0.07250865548849106, "fcm_dpo/margin": 10.411500930786133, "fcm_dpo/q_t": 0.39109233021736145, "grad_norm": 33.85374450683594, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.6580019593238831, "logits/rejected": -0.6455116271972656, "logps/chosen": -68.40087890625, "logps/ref_chosen": -64.83599853515625, "logps/ref_rejected": -109.94645690917969, "logps/rejected": -123.9228286743164, "loss": 1.0464, "margin_dpo/margin_mean": 10.411500930786133, "margin_dpo/margin_std": 11.7676362991333, "step": 65 }, { "KL/chosen_KL_mean": -3.3290576934814453, "KL/mean": -8.054281234741211, "KL/rejected_KL_mean": -12.77950668334961, "KL/std": 9.668466567993164, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.042388349771499634, "fcm_dpo/delta": -0.004084587097167969, "fcm_dpo/margin": 9.450448989868164, "fcm_dpo/q_t": 0.4077484607696533, "grad_norm": 29.53615951538086, "learning_rate": 4.7101449275362313e-07, "logits/chosen": -0.6404643058776855, "logits/rejected": -0.6079626083374023, "logps/chosen": -54.7725830078125, "logps/ref_chosen": -51.44352722167969, "logps/ref_rejected": -75.63629913330078, "logps/rejected": -88.41580200195312, "loss": 1.0842, "margin_dpo/margin_mean": 9.450448989868164, "margin_dpo/margin_std": 11.738363265991211, "step": 66 }, { "KL/chosen_KL_mean": -3.056062698364258, "KL/mean": -7.95568323135376, "KL/rejected_KL_mean": -12.855304718017578, "KL/std": 9.883098602294922, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.043049998581409454, "fcm_dpo/delta": -0.02787696197628975, "fcm_dpo/margin": 9.79924201965332, "fcm_dpo/q_t": 0.4025185704231262, "grad_norm": 30.74718475341797, "learning_rate": 4.782608695652174e-07, "logits/chosen": -0.5991535186767578, "logits/rejected": -0.5557255744934082, "logps/chosen": -62.39686584472656, "logps/ref_chosen": -59.34080505371094, "logps/ref_rejected": -72.78728485107422, "logps/rejected": -85.64259338378906, "loss": 1.0678, "margin_dpo/margin_mean": 9.799242973327637, "margin_dpo/margin_std": 11.892528533935547, "step": 67 }, { "KL/chosen_KL_mean": -3.8343505859375, "KL/mean": -8.516582489013672, "KL/rejected_KL_mean": -13.198814392089844, "KL/std": 8.895482063293457, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.04273660480976105, "fcm_dpo/delta": -0.000813731923699379, "fcm_dpo/margin": 9.364463806152344, "fcm_dpo/q_t": 0.40617847442626953, "grad_norm": 30.00288963317871, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.6549866199493408, "logits/rejected": -0.5948320627212524, "logps/chosen": -69.0401840209961, "logps/ref_chosen": -65.2058334350586, "logps/ref_rejected": -77.20724487304688, "logps/rejected": -90.40606689453125, "loss": 1.0692, "margin_dpo/margin_mean": 9.364463806152344, "margin_dpo/margin_std": 10.025825500488281, "step": 68 }, { "KL/chosen_KL_mean": -4.661190032958984, "KL/mean": -10.855348587036133, "KL/rejected_KL_mean": -17.04950714111328, "KL/std": 11.315301895141602, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.03835180401802063, "fcm_dpo/delta": -0.10325458645820618, "fcm_dpo/margin": 12.388309478759766, "fcm_dpo/q_t": 0.3859746754169464, "grad_norm": 31.52351188659668, "learning_rate": 4.927536231884058e-07, "logits/chosen": -0.6151754856109619, "logits/rejected": -0.5926914215087891, "logps/chosen": -64.48043060302734, "logps/ref_chosen": -59.81924057006836, "logps/ref_rejected": -103.38886260986328, "logps/rejected": -120.43836975097656, "loss": 1.0203, "margin_dpo/margin_mean": 12.388309478759766, "margin_dpo/margin_std": 12.643640518188477, "step": 69 }, { "KL/chosen_KL_mean": -7.054567337036133, "KL/mean": -13.696563720703125, "KL/rejected_KL_mean": -20.338558197021484, "KL/std": 14.29193115234375, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.03588715195655823, "fcm_dpo/delta": -0.10018091648817062, "fcm_dpo/margin": 13.283990859985352, "fcm_dpo/q_t": 0.38754719495773315, "grad_norm": 31.2943115234375, "learning_rate": 5e-07, "logits/chosen": -0.614654004573822, "logits/rejected": -0.5801568627357483, "logps/chosen": -68.9852066040039, "logps/ref_chosen": -61.930641174316406, "logps/ref_rejected": -91.06078338623047, "logps/rejected": -111.39933776855469, "loss": 1.0364, "margin_dpo/margin_mean": 13.283990859985352, "margin_dpo/margin_std": 15.29912281036377, "step": 70 }, { "KL/chosen_KL_mean": -7.13560676574707, "KL/mean": -15.221653938293457, "KL/rejected_KL_mean": -23.307708740234375, "KL/std": 15.503862380981445, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.030674291774630547, "fcm_dpo/delta": -0.13285793364048004, "fcm_dpo/margin": 16.172096252441406, "fcm_dpo/q_t": 0.3823116719722748, "grad_norm": 28.419921875, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.649775505065918, "logits/rejected": -0.6120765805244446, "logps/chosen": -68.88594055175781, "logps/ref_chosen": -61.750335693359375, "logps/ref_rejected": -97.33662414550781, "logps/rejected": -120.64433288574219, "loss": 1.0137, "margin_dpo/margin_mean": 16.172096252441406, "margin_dpo/margin_std": 17.033647537231445, "step": 71 }, { "KL/chosen_KL_mean": -8.378385543823242, "KL/mean": -16.67862892150879, "KL/rejected_KL_mean": -24.978866577148438, "KL/std": 16.624849319458008, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.028072141110897064, "fcm_dpo/delta": -0.08738085627555847, "fcm_dpo/margin": 16.600479125976562, "fcm_dpo/q_t": 0.38960447907447815, "grad_norm": 30.47901153564453, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.6570615172386169, "logits/rejected": -0.6202565431594849, "logps/chosen": -74.43179321289062, "logps/ref_chosen": -66.05341339111328, "logps/ref_rejected": -95.2869873046875, "logps/rejected": -120.26585388183594, "loss": 1.0429, "margin_dpo/margin_mean": 16.600479125976562, "margin_dpo/margin_std": 19.797344207763672, "step": 72 }, { "KL/chosen_KL_mean": -10.47215461730957, "KL/mean": -18.878131866455078, "KL/rejected_KL_mean": -27.284114837646484, "KL/std": 21.38900375366211, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.026148542761802673, "fcm_dpo/delta": -0.0531650111079216, "fcm_dpo/margin": 16.811960220336914, "fcm_dpo/q_t": 0.40286535024642944, "grad_norm": 26.106237411499023, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.7131662368774414, "logits/rejected": -0.6752426624298096, "logps/chosen": -76.72842407226562, "logps/ref_chosen": -66.25627136230469, "logps/ref_rejected": -90.45613098144531, "logps/rejected": -117.74024963378906, "loss": 1.1056, "margin_dpo/margin_mean": 16.811960220336914, "margin_dpo/margin_std": 27.132373809814453, "step": 73 }, { "KL/chosen_KL_mean": -10.981948852539062, "KL/mean": -20.437519073486328, "KL/rejected_KL_mean": -29.893081665039062, "KL/std": 23.818477630615234, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.024446729570627213, "fcm_dpo/delta": -0.08252619206905365, "fcm_dpo/margin": 18.911136627197266, "fcm_dpo/q_t": 0.39837193489074707, "grad_norm": 26.160118103027344, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.6098172664642334, "logits/rejected": -0.6038111448287964, "logps/chosen": -64.40682983398438, "logps/ref_chosen": -53.42488098144531, "logps/ref_rejected": -95.94693756103516, "logps/rejected": -125.84001922607422, "loss": 1.0783, "margin_dpo/margin_mean": 18.911136627197266, "margin_dpo/margin_std": 29.8692569732666, "step": 74 }, { "KL/chosen_KL_mean": -10.932317733764648, "KL/mean": -24.136571884155273, "KL/rejected_KL_mean": -37.3408203125, "KL/std": 24.335262298583984, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.021133817732334137, "fcm_dpo/delta": -0.21684792637825012, "fcm_dpo/margin": 26.408512115478516, "fcm_dpo/q_t": 0.3661307096481323, "grad_norm": 24.379812240600586, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.6909558773040771, "logits/rejected": -0.6851760149002075, "logps/chosen": -62.79397964477539, "logps/ref_chosen": -51.861663818359375, "logps/ref_rejected": -111.25398254394531, "logps/rejected": -148.5948028564453, "loss": 0.9864, "margin_dpo/margin_mean": 26.408510208129883, "margin_dpo/margin_std": 27.935894012451172, "step": 75 }, { "KL/chosen_KL_mean": -13.196050643920898, "KL/mean": -21.37349510192871, "KL/rejected_KL_mean": -29.55093765258789, "KL/std": 19.751428604125977, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.02066803351044655, "fcm_dpo/delta": 0.07377178966999054, "fcm_dpo/margin": 16.354894638061523, "fcm_dpo/q_t": 0.4222240149974823, "grad_norm": 23.001667022705078, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.6944067478179932, "logits/rejected": -0.6538985967636108, "logps/chosen": -66.46208953857422, "logps/ref_chosen": -53.26603698730469, "logps/ref_rejected": -78.21662902832031, "logps/rejected": -107.76756286621094, "loss": 1.1248, "margin_dpo/margin_mean": 16.354894638061523, "margin_dpo/margin_std": 21.762468338012695, "step": 76 }, { "KL/chosen_KL_mean": -12.05851936340332, "KL/mean": -25.64178466796875, "KL/rejected_KL_mean": -39.22505187988281, "KL/std": 27.021289825439453, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.018379777669906616, "fcm_dpo/delta": -0.14052104949951172, "fcm_dpo/margin": 27.166534423828125, "fcm_dpo/q_t": 0.3777894973754883, "grad_norm": 23.42577362060547, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.7054200172424316, "logits/rejected": -0.6716604232788086, "logps/chosen": -70.15519714355469, "logps/ref_chosen": -58.0966796875, "logps/ref_rejected": -93.77361297607422, "logps/rejected": -132.9986572265625, "loss": 1.0196, "margin_dpo/margin_mean": 27.166534423828125, "margin_dpo/margin_std": 30.322921752929688, "step": 77 }, { "KL/chosen_KL_mean": -12.313678741455078, "KL/mean": -22.357601165771484, "KL/rejected_KL_mean": -32.401519775390625, "KL/std": 22.272029876708984, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.01876748539507389, "fcm_dpo/delta": 0.02741411328315735, "fcm_dpo/margin": 20.087839126586914, "fcm_dpo/q_t": 0.41380858421325684, "grad_norm": 21.392480850219727, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.6116062998771667, "logits/rejected": -0.587343156337738, "logps/chosen": -67.92745971679688, "logps/ref_chosen": -55.61378479003906, "logps/ref_rejected": -84.93436431884766, "logps/rejected": -117.33588409423828, "loss": 1.1009, "margin_dpo/margin_mean": 20.087839126586914, "margin_dpo/margin_std": 25.943618774414062, "step": 78 }, { "KL/chosen_KL_mean": -12.862058639526367, "KL/mean": -23.823688507080078, "KL/rejected_KL_mean": -34.78531265258789, "KL/std": 24.85503578186035, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.018557265400886536, "fcm_dpo/delta": -0.008593140169978142, "fcm_dpo/margin": 21.92325210571289, "fcm_dpo/q_t": 0.40805089473724365, "grad_norm": 20.17791748046875, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6797605752944946, "logits/rejected": -0.6534685492515564, "logps/chosen": -68.31254577636719, "logps/ref_chosen": -55.45048522949219, "logps/ref_rejected": -87.64756774902344, "logps/rejected": -122.43287658691406, "loss": 1.0936, "margin_dpo/margin_mean": 21.92325210571289, "margin_dpo/margin_std": 30.41646385192871, "step": 79 }, { "KL/chosen_KL_mean": -16.53986358642578, "KL/mean": -27.252384185791016, "KL/rejected_KL_mean": -37.964908599853516, "KL/std": 27.36221694946289, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.01815933734178543, "fcm_dpo/delta": 0.010311007499694824, "fcm_dpo/margin": 21.425039291381836, "fcm_dpo/q_t": 0.4146670699119568, "grad_norm": 22.17724609375, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7165290117263794, "logits/rejected": -0.6780725717544556, "logps/chosen": -75.05915832519531, "logps/ref_chosen": -58.519290924072266, "logps/ref_rejected": -87.54750061035156, "logps/rejected": -125.51240539550781, "loss": 1.1159, "margin_dpo/margin_mean": 21.42504119873047, "margin_dpo/margin_std": 31.943340301513672, "step": 80 }, { "KL/chosen_KL_mean": -15.941177368164062, "KL/mean": -30.589588165283203, "KL/rejected_KL_mean": -45.237998962402344, "KL/std": 32.461692810058594, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.016676858067512512, "fcm_dpo/delta": -0.1239388957619667, "fcm_dpo/margin": 29.296829223632812, "fcm_dpo/q_t": 0.38681286573410034, "grad_norm": 21.989290237426758, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.7118068337440491, "logits/rejected": -0.68656325340271, "logps/chosen": -82.39004516601562, "logps/ref_chosen": -66.44886779785156, "logps/ref_rejected": -129.66270446777344, "logps/rejected": -174.90069580078125, "loss": 1.0461, "margin_dpo/margin_mean": 29.296829223632812, "margin_dpo/margin_std": 38.70458984375, "step": 81 }, { "KL/chosen_KL_mean": -18.573457717895508, "KL/mean": -30.226879119873047, "KL/rejected_KL_mean": -41.88030242919922, "KL/std": 29.723350524902344, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.017191056162118912, "fcm_dpo/delta": -0.009201280772686005, "fcm_dpo/margin": 23.30685043334961, "fcm_dpo/q_t": 0.41180986166000366, "grad_norm": 23.152570724487305, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.6191596984863281, "logits/rejected": -0.6112022995948792, "logps/chosen": -70.80584716796875, "logps/ref_chosen": -52.232383728027344, "logps/ref_rejected": -90.74325561523438, "logps/rejected": -132.62355041503906, "loss": 1.1455, "margin_dpo/margin_mean": 23.30685043334961, "margin_dpo/margin_std": 41.297733306884766, "step": 82 }, { "KL/chosen_KL_mean": -19.169687271118164, "KL/mean": -33.314022064208984, "KL/rejected_KL_mean": -47.4583625793457, "KL/std": 30.860076904296875, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.01552935317158699, "fcm_dpo/delta": -0.05240233987569809, "fcm_dpo/margin": 28.28866958618164, "fcm_dpo/q_t": 0.3983423411846161, "grad_norm": 21.990673065185547, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6577206254005432, "logits/rejected": -0.6294866800308228, "logps/chosen": -74.9970703125, "logps/ref_chosen": -55.82738494873047, "logps/ref_rejected": -103.71589660644531, "logps/rejected": -151.17425537109375, "loss": 1.0706, "margin_dpo/margin_mean": 28.288667678833008, "margin_dpo/margin_std": 36.98453903198242, "step": 83 }, { "KL/chosen_KL_mean": -18.952524185180664, "KL/mean": -30.40399169921875, "KL/rejected_KL_mean": -41.855464935302734, "KL/std": 26.464885711669922, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.01563824899494648, "fcm_dpo/delta": 0.048310477286577225, "fcm_dpo/margin": 22.902938842773438, "fcm_dpo/q_t": 0.4166381359100342, "grad_norm": 20.835840225219727, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.6432810425758362, "logits/rejected": -0.6044536828994751, "logps/chosen": -86.12869262695312, "logps/ref_chosen": -67.1761703491211, "logps/ref_rejected": -87.29859924316406, "logps/rejected": -129.15406799316406, "loss": 1.1085, "margin_dpo/margin_mean": 22.902938842773438, "margin_dpo/margin_std": 26.787670135498047, "step": 84 }, { "KL/chosen_KL_mean": -19.004091262817383, "KL/mean": -31.023822784423828, "KL/rejected_KL_mean": -43.043556213378906, "KL/std": 27.054758071899414, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.016782496124505997, "fcm_dpo/delta": -0.009143978357315063, "fcm_dpo/margin": 24.039470672607422, "fcm_dpo/q_t": 0.40713611245155334, "grad_norm": 21.613723754882812, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.6534522175788879, "logits/rejected": -0.622204065322876, "logps/chosen": -77.41070556640625, "logps/ref_chosen": -58.4066162109375, "logps/ref_rejected": -78.63880157470703, "logps/rejected": -121.68235778808594, "loss": 1.0991, "margin_dpo/margin_mean": 24.039470672607422, "margin_dpo/margin_std": 32.703369140625, "step": 85 }, { "KL/chosen_KL_mean": -24.901784896850586, "KL/mean": -37.69927978515625, "KL/rejected_KL_mean": -50.49677276611328, "KL/std": 38.232906341552734, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.01626906730234623, "fcm_dpo/delta": -0.02235548198223114, "fcm_dpo/margin": 25.594993591308594, "fcm_dpo/q_t": 0.41634446382522583, "grad_norm": 25.438549041748047, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6633529663085938, "logits/rejected": -0.6544677019119263, "logps/chosen": -81.03924560546875, "logps/ref_chosen": -56.13746643066406, "logps/ref_rejected": -88.12165069580078, "logps/rejected": -138.61842346191406, "loss": 1.1718, "margin_dpo/margin_mean": 25.594993591308594, "margin_dpo/margin_std": 53.408973693847656, "step": 86 }, { "KL/chosen_KL_mean": -24.417339324951172, "KL/mean": -40.079750061035156, "KL/rejected_KL_mean": -55.74216079711914, "KL/std": 38.12931823730469, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.014551367610692978, "fcm_dpo/delta": -0.07786225527524948, "fcm_dpo/margin": 31.324817657470703, "fcm_dpo/q_t": 0.39422881603240967, "grad_norm": 21.019412994384766, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.6794058084487915, "logits/rejected": -0.6601496934890747, "logps/chosen": -80.05343627929688, "logps/ref_chosen": -55.63609313964844, "logps/ref_rejected": -95.46757507324219, "logps/rejected": -151.20974731445312, "loss": 1.0915, "margin_dpo/margin_mean": 31.324819564819336, "margin_dpo/margin_std": 47.0085334777832, "step": 87 }, { "KL/chosen_KL_mean": -30.980731964111328, "KL/mean": -42.74275207519531, "KL/rejected_KL_mean": -54.50476837158203, "KL/std": 40.58386993408203, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.014943553134799004, "fcm_dpo/delta": 0.057977911084890366, "fcm_dpo/margin": 23.52403450012207, "fcm_dpo/q_t": 0.42546147108078003, "grad_norm": 21.812320709228516, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.6842135787010193, "logits/rejected": -0.6828039288520813, "logps/chosen": -104.65188598632812, "logps/ref_chosen": -73.67115020751953, "logps/ref_rejected": -106.70849609375, "logps/rejected": -161.21327209472656, "loss": 1.1737, "margin_dpo/margin_mean": 23.52403450012207, "margin_dpo/margin_std": 43.78386306762695, "step": 88 }, { "KL/chosen_KL_mean": -19.886613845825195, "KL/mean": -33.26313781738281, "KL/rejected_KL_mean": -46.63965606689453, "KL/std": 34.684234619140625, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.015584435313940048, "fcm_dpo/delta": -0.02295827493071556, "fcm_dpo/margin": 26.753049850463867, "fcm_dpo/q_t": 0.40534842014312744, "grad_norm": 21.010114669799805, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7116603851318359, "logits/rejected": -0.6837852001190186, "logps/chosen": -80.51152801513672, "logps/ref_chosen": -60.624916076660156, "logps/ref_rejected": -82.08354949951172, "logps/rejected": -128.72320556640625, "loss": 1.1014, "margin_dpo/margin_mean": 26.753049850463867, "margin_dpo/margin_std": 40.44371032714844, "step": 89 }, { "KL/chosen_KL_mean": -24.155946731567383, "KL/mean": -43.333587646484375, "KL/rejected_KL_mean": -62.511234283447266, "KL/std": 47.100379943847656, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.013927513733506203, "fcm_dpo/delta": -0.18348340690135956, "fcm_dpo/margin": 38.355281829833984, "fcm_dpo/q_t": 0.38429707288742065, "grad_norm": 21.59499740600586, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6261708736419678, "logits/rejected": -0.646406888961792, "logps/chosen": -77.44125366210938, "logps/ref_chosen": -53.285308837890625, "logps/ref_rejected": -111.54470825195312, "logps/rejected": -174.05593872070312, "loss": 1.0798, "margin_dpo/margin_mean": 38.35528564453125, "margin_dpo/margin_std": 59.089813232421875, "step": 90 }, { "KL/chosen_KL_mean": -25.931039810180664, "KL/mean": -40.36906814575195, "KL/rejected_KL_mean": -54.807098388671875, "KL/std": 36.47618103027344, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.013058323413133621, "fcm_dpo/delta": 0.026791641488671303, "fcm_dpo/margin": 28.876054763793945, "fcm_dpo/q_t": 0.4173550009727478, "grad_norm": 19.14020347595215, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6626990437507629, "logits/rejected": -0.6330728530883789, "logps/chosen": -87.73399353027344, "logps/ref_chosen": -61.802955627441406, "logps/ref_rejected": -87.87395477294922, "logps/rejected": -142.68106079101562, "loss": 1.1368, "margin_dpo/margin_mean": 28.876056671142578, "margin_dpo/margin_std": 48.72987365722656, "step": 91 }, { "KL/chosen_KL_mean": -22.967710494995117, "KL/mean": -38.062904357910156, "KL/rejected_KL_mean": -53.15810012817383, "KL/std": 38.03881072998047, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.013261953368782997, "fcm_dpo/delta": -0.0033196136355400085, "fcm_dpo/margin": 30.190397262573242, "fcm_dpo/q_t": 0.4104510247707367, "grad_norm": 18.81719398498535, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.6928216218948364, "logits/rejected": -0.6727030277252197, "logps/chosen": -74.60847473144531, "logps/ref_chosen": -51.640769958496094, "logps/ref_rejected": -77.88117980957031, "logps/rejected": -131.03927612304688, "loss": 1.1136, "margin_dpo/margin_mean": 30.190399169921875, "margin_dpo/margin_std": 46.062583923339844, "step": 92 }, { "KL/chosen_KL_mean": -26.070533752441406, "KL/mean": -40.96013641357422, "KL/rejected_KL_mean": -55.8497314453125, "KL/std": 34.986263275146484, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.013152128085494041, "fcm_dpo/delta": 0.009675238281488419, "fcm_dpo/margin": 29.77920150756836, "fcm_dpo/q_t": 0.4116442799568176, "grad_norm": 19.744487762451172, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.7134292125701904, "logits/rejected": -0.6725969910621643, "logps/chosen": -78.59977722167969, "logps/ref_chosen": -52.529239654541016, "logps/ref_rejected": -77.16075134277344, "logps/rejected": -133.01048278808594, "loss": 1.0986, "margin_dpo/margin_mean": 29.77920150756836, "margin_dpo/margin_std": 41.22077941894531, "step": 93 }, { "KL/chosen_KL_mean": -27.902204513549805, "KL/mean": -45.52301025390625, "KL/rejected_KL_mean": -63.14381790161133, "KL/std": 41.25383758544922, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.012278085574507713, "fcm_dpo/delta": -0.0462297722697258, "fcm_dpo/margin": 35.24161148071289, "fcm_dpo/q_t": 0.39923009276390076, "grad_norm": 18.810970306396484, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.645513653755188, "logits/rejected": -0.637039303779602, "logps/chosen": -89.12481689453125, "logps/ref_chosen": -61.22261047363281, "logps/ref_rejected": -99.59902954101562, "logps/rejected": -162.74285888671875, "loss": 1.0683, "margin_dpo/margin_mean": 35.24161148071289, "margin_dpo/margin_std": 43.92166519165039, "step": 94 }, { "KL/chosen_KL_mean": -27.263856887817383, "KL/mean": -41.50673294067383, "KL/rejected_KL_mean": -55.749610900878906, "KL/std": 41.475608825683594, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.012666614726185799, "fcm_dpo/delta": 0.047285325825214386, "fcm_dpo/margin": 28.485750198364258, "fcm_dpo/q_t": 0.41799068450927734, "grad_norm": 18.525175094604492, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.6651968359947205, "logits/rejected": -0.6414941549301147, "logps/chosen": -79.78750610351562, "logps/ref_chosen": -52.523643493652344, "logps/ref_rejected": -75.8803482055664, "logps/rejected": -131.6299591064453, "loss": 1.131, "margin_dpo/margin_mean": 28.48575210571289, "margin_dpo/margin_std": 43.52128601074219, "step": 95 }, { "KL/chosen_KL_mean": -27.1774845123291, "KL/mean": -46.510169982910156, "KL/rejected_KL_mean": -65.84285736083984, "KL/std": 44.56120300292969, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.011798870749771595, "fcm_dpo/delta": -0.08383983373641968, "fcm_dpo/margin": 38.665374755859375, "fcm_dpo/q_t": 0.39412879943847656, "grad_norm": 18.564252853393555, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.6842184066772461, "logits/rejected": -0.662356972694397, "logps/chosen": -89.33445739746094, "logps/ref_chosen": -62.15697479248047, "logps/ref_rejected": -96.59601593017578, "logps/rejected": -162.43887329101562, "loss": 1.0612, "margin_dpo/margin_mean": 38.665374755859375, "margin_dpo/margin_std": 50.07075500488281, "step": 96 }, { "KL/chosen_KL_mean": -28.180774688720703, "KL/mean": -42.7745361328125, "KL/rejected_KL_mean": -57.36829376220703, "KL/std": 38.20796203613281, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.011734258383512497, "fcm_dpo/delta": -0.04553954675793648, "fcm_dpo/margin": 29.187524795532227, "fcm_dpo/q_t": 0.4178887903690338, "grad_norm": 19.07831573486328, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.6238841414451599, "logits/rejected": -0.588768720626831, "logps/chosen": -82.82714080810547, "logps/ref_chosen": -54.646366119384766, "logps/ref_rejected": -76.96475219726562, "logps/rejected": -134.33303833007812, "loss": 1.1271, "margin_dpo/margin_mean": 29.187524795532227, "margin_dpo/margin_std": 38.123573303222656, "step": 97 }, { "KL/chosen_KL_mean": -35.863746643066406, "KL/mean": -52.72106170654297, "KL/rejected_KL_mean": -69.57838439941406, "KL/std": 44.223392486572266, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.011502566747367382, "fcm_dpo/delta": 0.014971842989325523, "fcm_dpo/margin": 33.714637756347656, "fcm_dpo/q_t": 0.41005343198776245, "grad_norm": 20.55078887939453, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6732558012008667, "logits/rejected": -0.6637331247329712, "logps/chosen": -101.12237548828125, "logps/ref_chosen": -65.25862884521484, "logps/ref_rejected": -96.5274887084961, "logps/rejected": -166.10586547851562, "loss": 1.1266, "margin_dpo/margin_mean": 33.714637756347656, "margin_dpo/margin_std": 53.72822570800781, "step": 98 }, { "KL/chosen_KL_mean": -27.312252044677734, "KL/mean": -48.77324676513672, "KL/rejected_KL_mean": -70.2342300415039, "KL/std": 46.53744125366211, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.010750599205493927, "fcm_dpo/delta": -0.08328957110643387, "fcm_dpo/margin": 42.921993255615234, "fcm_dpo/q_t": 0.39549580216407776, "grad_norm": 17.073204040527344, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.6471402049064636, "logits/rejected": -0.651877224445343, "logps/chosen": -72.95073699951172, "logps/ref_chosen": -45.638484954833984, "logps/ref_rejected": -86.43793487548828, "logps/rejected": -156.6721649169922, "loss": 1.0633, "margin_dpo/margin_mean": 42.921993255615234, "margin_dpo/margin_std": 58.77742004394531, "step": 99 }, { "KL/chosen_KL_mean": -31.377222061157227, "KL/mean": -42.55781555175781, "KL/rejected_KL_mean": -53.738407135009766, "KL/std": 38.19342803955078, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.010938813909888268, "fcm_dpo/delta": 0.041864871978759766, "fcm_dpo/margin": 22.361183166503906, "fcm_dpo/q_t": 0.4438517093658447, "grad_norm": 18.95781898498535, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6145943403244019, "logits/rejected": -0.5761110782623291, "logps/chosen": -88.97120666503906, "logps/ref_chosen": -57.59397888183594, "logps/ref_rejected": -74.06021118164062, "logps/rejected": -127.79861450195312, "loss": 1.2277, "margin_dpo/margin_mean": 22.361183166503906, "margin_dpo/margin_std": 49.1837158203125, "step": 100 }, { "KL/chosen_KL_mean": -38.442962646484375, "KL/mean": -52.74586486816406, "KL/rejected_KL_mean": -67.04876708984375, "KL/std": 47.44648742675781, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.011103025637567043, "fcm_dpo/delta": -0.0058130607940256596, "fcm_dpo/margin": 28.60578727722168, "fcm_dpo/q_t": 0.4297153949737549, "grad_norm": 19.153141021728516, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.6148165464401245, "logits/rejected": -0.5886383056640625, "logps/chosen": -100.09181213378906, "logps/ref_chosen": -61.64885330200195, "logps/ref_rejected": -83.18968200683594, "logps/rejected": -150.2384490966797, "loss": 1.1865, "margin_dpo/margin_mean": 28.605789184570312, "margin_dpo/margin_std": 56.565982818603516, "step": 101 }, { "KL/chosen_KL_mean": -40.61670684814453, "KL/mean": -50.855247497558594, "KL/rejected_KL_mean": -61.093788146972656, "KL/std": 44.045799255371094, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.011240271851420403, "fcm_dpo/delta": 0.024421557784080505, "fcm_dpo/margin": 20.477081298828125, "fcm_dpo/q_t": 0.44768211245536804, "grad_norm": 22.304677963256836, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.673102080821991, "logits/rejected": -0.6373000144958496, "logps/chosen": -104.69558715820312, "logps/ref_chosen": -64.0788803100586, "logps/ref_rejected": -68.18707275390625, "logps/rejected": -129.28086853027344, "loss": 1.2568, "margin_dpo/margin_mean": 20.477081298828125, "margin_dpo/margin_std": 53.06024169921875, "step": 102 }, { "KL/chosen_KL_mean": -34.13248825073242, "KL/mean": -55.830631256103516, "KL/rejected_KL_mean": -77.52876281738281, "KL/std": 49.93252182006836, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.010603921487927437, "fcm_dpo/delta": -0.07997814565896988, "fcm_dpo/margin": 43.39628601074219, "fcm_dpo/q_t": 0.3960493803024292, "grad_norm": 18.789276123046875, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6219948530197144, "logits/rejected": -0.59108966588974, "logps/chosen": -95.4317626953125, "logps/ref_chosen": -61.299278259277344, "logps/ref_rejected": -93.57270812988281, "logps/rejected": -171.10147094726562, "loss": 1.0721, "margin_dpo/margin_mean": 43.39628601074219, "margin_dpo/margin_std": 62.30549621582031, "step": 103 }, { "KL/chosen_KL_mean": -35.155120849609375, "KL/mean": -55.93231964111328, "KL/rejected_KL_mean": -76.70951080322266, "KL/std": 53.614723205566406, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.010277830064296722, "fcm_dpo/delta": -0.03438165411353111, "fcm_dpo/margin": 41.55438995361328, "fcm_dpo/q_t": 0.40735965967178345, "grad_norm": 18.074642181396484, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.6393266916275024, "logits/rejected": -0.6158906817436218, "logps/chosen": -89.52789306640625, "logps/ref_chosen": -54.372772216796875, "logps/ref_rejected": -89.5647201538086, "logps/rejected": -166.27423095703125, "loss": 1.1084, "margin_dpo/margin_mean": 41.55438995361328, "margin_dpo/margin_std": 66.71173095703125, "step": 104 }, { "KL/chosen_KL_mean": -33.67083740234375, "KL/mean": -63.1674919128418, "KL/rejected_KL_mean": -92.66414642333984, "KL/std": 50.66936492919922, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.008944441564381123, "fcm_dpo/delta": -0.17402513325214386, "fcm_dpo/margin": 58.99330139160156, "fcm_dpo/q_t": 0.36932459473609924, "grad_norm": 18.142919540405273, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.5875688791275024, "logits/rejected": -0.5673741102218628, "logps/chosen": -88.30978393554688, "logps/ref_chosen": -54.638946533203125, "logps/ref_rejected": -97.97351837158203, "logps/rejected": -190.63766479492188, "loss": 0.9869, "margin_dpo/margin_mean": 58.99330139160156, "margin_dpo/margin_std": 57.22583770751953, "step": 105 }, { "KL/chosen_KL_mean": -38.18315124511719, "KL/mean": -56.45439910888672, "KL/rejected_KL_mean": -74.72565460205078, "KL/std": 45.731204986572266, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.009024987928569317, "fcm_dpo/delta": 0.08437706530094147, "fcm_dpo/margin": 36.542503356933594, "fcm_dpo/q_t": 0.42528021335601807, "grad_norm": 16.702926635742188, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.535358190536499, "logits/rejected": -0.5128501653671265, "logps/chosen": -93.01604461669922, "logps/ref_chosen": -54.83289337158203, "logps/ref_rejected": -85.22461700439453, "logps/rejected": -159.9502716064453, "loss": 1.1339, "margin_dpo/margin_mean": 36.542503356933594, "margin_dpo/margin_std": 49.62269592285156, "step": 106 }, { "KL/chosen_KL_mean": -45.14348220825195, "KL/mean": -65.80903625488281, "KL/rejected_KL_mean": -86.47457885742188, "KL/std": 56.287105560302734, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.009254291653633118, "fcm_dpo/delta": 0.020608151331543922, "fcm_dpo/margin": 41.33110427856445, "fcm_dpo/q_t": 0.4126184582710266, "grad_norm": 17.756500244140625, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.6197066307067871, "logits/rejected": -0.580697774887085, "logps/chosen": -114.8512954711914, "logps/ref_chosen": -69.70780944824219, "logps/ref_rejected": -94.73950958251953, "logps/rejected": -181.21408081054688, "loss": 1.115, "margin_dpo/margin_mean": 41.33110046386719, "margin_dpo/margin_std": 59.82461166381836, "step": 107 }, { "KL/chosen_KL_mean": -39.11261749267578, "KL/mean": -65.50827026367188, "KL/rejected_KL_mean": -91.90391540527344, "KL/std": 62.87705993652344, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.008883420377969742, "fcm_dpo/delta": -0.09007138013839722, "fcm_dpo/margin": 52.791290283203125, "fcm_dpo/q_t": 0.39717066287994385, "grad_norm": 17.294050216674805, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.5953981280326843, "logits/rejected": -0.5820811986923218, "logps/chosen": -95.12250518798828, "logps/ref_chosen": -56.0098876953125, "logps/ref_rejected": -95.79601287841797, "logps/rejected": -187.69992065429688, "loss": 1.0879, "margin_dpo/margin_mean": 52.791290283203125, "margin_dpo/margin_std": 81.55433654785156, "step": 108 }, { "KL/chosen_KL_mean": -37.30243682861328, "KL/mean": -64.34707641601562, "KL/rejected_KL_mean": -91.39170837402344, "KL/std": 59.183998107910156, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.008149518631398678, "fcm_dpo/delta": -0.05536172538995743, "fcm_dpo/margin": 54.08927536010742, "fcm_dpo/q_t": 0.3969169855117798, "grad_norm": 18.013111114501953, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5478425025939941, "logits/rejected": -0.528139054775238, "logps/chosen": -100.18792724609375, "logps/ref_chosen": -62.88549041748047, "logps/ref_rejected": -98.68573760986328, "logps/rejected": -190.07745361328125, "loss": 1.0772, "margin_dpo/margin_mean": 54.08927536010742, "margin_dpo/margin_std": 73.88963317871094, "step": 109 }, { "KL/chosen_KL_mean": -39.2655029296875, "KL/mean": -63.571868896484375, "KL/rejected_KL_mean": -87.87823486328125, "KL/std": 59.547828674316406, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.007406959775835276, "fcm_dpo/delta": -0.10038627684116364, "fcm_dpo/margin": 48.61272430419922, "fcm_dpo/q_t": 0.4130057692527771, "grad_norm": 13.945056915283203, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.5412209033966064, "logits/rejected": -0.5109491348266602, "logps/chosen": -98.0191879272461, "logps/ref_chosen": -58.753684997558594, "logps/ref_rejected": -79.75001525878906, "logps/rejected": -167.6282501220703, "loss": 1.1332, "margin_dpo/margin_mean": 48.61272430419922, "margin_dpo/margin_std": 72.85327911376953, "step": 110 }, { "KL/chosen_KL_mean": -43.91405487060547, "KL/mean": -72.29114532470703, "KL/rejected_KL_mean": -100.66822052001953, "KL/std": 67.76360321044922, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.0071860793977975845, "fcm_dpo/delta": -0.012220162898302078, "fcm_dpo/margin": 56.754173278808594, "fcm_dpo/q_t": 0.4068235754966736, "grad_norm": 15.937190055847168, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.5931087732315063, "logits/rejected": -0.5733714699745178, "logps/chosen": -112.53816223144531, "logps/ref_chosen": -68.62410736083984, "logps/ref_rejected": -98.42886352539062, "logps/rejected": -199.09707641601562, "loss": 1.1115, "margin_dpo/margin_mean": 56.754173278808594, "margin_dpo/margin_std": 86.54757690429688, "step": 111 }, { "KL/chosen_KL_mean": -40.607540130615234, "KL/mean": -57.865150451660156, "KL/rejected_KL_mean": -75.12275695800781, "KL/std": 45.083778381347656, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.007618125528097153, "fcm_dpo/delta": 0.04033050686120987, "fcm_dpo/margin": 34.51521682739258, "fcm_dpo/q_t": 0.43914544582366943, "grad_norm": 15.417643547058105, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.5523747801780701, "logits/rejected": -0.5217946767807007, "logps/chosen": -90.857177734375, "logps/ref_chosen": -50.24964141845703, "logps/ref_rejected": -64.77442932128906, "logps/rejected": -139.89718627929688, "loss": 1.1866, "margin_dpo/margin_mean": 34.51521682739258, "margin_dpo/margin_std": 56.524749755859375, "step": 112 }, { "KL/chosen_KL_mean": -53.0630989074707, "KL/mean": -70.5762939453125, "KL/rejected_KL_mean": -88.08948516845703, "KL/std": 50.8385009765625, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.008391564711928368, "fcm_dpo/delta": 0.12340584397315979, "fcm_dpo/margin": 35.0263786315918, "fcm_dpo/q_t": 0.4351421892642975, "grad_norm": 17.364025115966797, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.5283902883529663, "logits/rejected": -0.4908086061477661, "logps/chosen": -119.77604675292969, "logps/ref_chosen": -66.71295166015625, "logps/ref_rejected": -77.96870422363281, "logps/rejected": -166.05819702148438, "loss": 1.1661, "margin_dpo/margin_mean": 35.0263786315918, "margin_dpo/margin_std": 56.08738327026367, "step": 113 }, { "KL/chosen_KL_mean": -47.82563018798828, "KL/mean": -77.25653076171875, "KL/rejected_KL_mean": -106.68740844726562, "KL/std": 72.02836608886719, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.007831193506717682, "fcm_dpo/delta": -0.08718626201152802, "fcm_dpo/margin": 58.861785888671875, "fcm_dpo/q_t": 0.39837566018104553, "grad_norm": 19.08915138244629, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.5314050912857056, "logits/rejected": -0.5059875845909119, "logps/chosen": -105.6107177734375, "logps/ref_chosen": -57.78507995605469, "logps/ref_rejected": -87.10966491699219, "logps/rejected": -193.7970733642578, "loss": 1.0774, "margin_dpo/margin_mean": 58.861785888671875, "margin_dpo/margin_std": 88.37593078613281, "step": 114 }, { "KL/chosen_KL_mean": -60.48192596435547, "KL/mean": -80.31732177734375, "KL/rejected_KL_mean": -100.15271759033203, "KL/std": 69.82859802246094, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.00848847534507513, "fcm_dpo/delta": 0.0731273964047432, "fcm_dpo/margin": 39.67078399658203, "fcm_dpo/q_t": 0.43038254976272583, "grad_norm": 25.075674057006836, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.5371870398521423, "logits/rejected": -0.5258318185806274, "logps/chosen": -126.06456756591797, "logps/ref_chosen": -65.5826416015625, "logps/ref_rejected": -98.56552124023438, "logps/rejected": -198.71823120117188, "loss": 1.2184, "margin_dpo/margin_mean": 39.67078399658203, "margin_dpo/margin_std": 93.87147521972656, "step": 115 }, { "KL/chosen_KL_mean": -43.80780792236328, "KL/mean": -66.62762451171875, "KL/rejected_KL_mean": -89.44744110107422, "KL/std": 57.25769805908203, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.008733630180358887, "fcm_dpo/delta": -0.0017695911228656769, "fcm_dpo/margin": 45.639625549316406, "fcm_dpo/q_t": 0.4090723991394043, "grad_norm": 21.11087417602539, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.5265697836875916, "logits/rejected": -0.5241330862045288, "logps/chosen": -95.20811462402344, "logps/ref_chosen": -51.40031433105469, "logps/ref_rejected": -80.5218505859375, "logps/rejected": -169.96929931640625, "loss": 1.1005, "margin_dpo/margin_mean": 45.639625549316406, "margin_dpo/margin_std": 63.12974548339844, "step": 116 }, { "KL/chosen_KL_mean": -56.89840316772461, "KL/mean": -74.12403869628906, "KL/rejected_KL_mean": -91.34967041015625, "KL/std": 59.87590789794922, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.009167262353003025, "fcm_dpo/delta": 0.09936673939228058, "fcm_dpo/margin": 34.45126724243164, "fcm_dpo/q_t": 0.43182051181793213, "grad_norm": 28.09205436706543, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.5017322301864624, "logits/rejected": -0.4683611989021301, "logps/chosen": -126.19680786132812, "logps/ref_chosen": -69.29840850830078, "logps/ref_rejected": -66.583984375, "logps/rejected": -157.93365478515625, "loss": 1.1893, "margin_dpo/margin_mean": 34.451271057128906, "margin_dpo/margin_std": 69.84674072265625, "step": 117 }, { "KL/chosen_KL_mean": -43.14284133911133, "KL/mean": -65.05967712402344, "KL/rejected_KL_mean": -86.97651672363281, "KL/std": 53.93627166748047, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.009347407147288322, "fcm_dpo/delta": -0.012348111718893051, "fcm_dpo/margin": 43.83367919921875, "fcm_dpo/q_t": 0.4077704846858978, "grad_norm": 19.32469367980957, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.5494426488876343, "logits/rejected": -0.5325976014137268, "logps/chosen": -98.78382110595703, "logps/ref_chosen": -55.6409797668457, "logps/ref_rejected": -75.66905975341797, "logps/rejected": -162.64556884765625, "loss": 1.0957, "margin_dpo/margin_mean": 43.83367919921875, "margin_dpo/margin_std": 62.52387237548828, "step": 118 }, { "KL/chosen_KL_mean": -59.46000289916992, "KL/mean": -80.9298095703125, "KL/rejected_KL_mean": -102.39959716796875, "KL/std": 60.10303497314453, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.008993204683065414, "fcm_dpo/delta": 0.011231735348701477, "fcm_dpo/margin": 42.93959426879883, "fcm_dpo/q_t": 0.4158446788787842, "grad_norm": 21.934490203857422, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.5536962151527405, "logits/rejected": -0.5457164645195007, "logps/chosen": -132.97019958496094, "logps/ref_chosen": -73.51019287109375, "logps/ref_rejected": -102.977294921875, "logps/rejected": -205.37689208984375, "loss": 1.1473, "margin_dpo/margin_mean": 42.939598083496094, "margin_dpo/margin_std": 75.34599304199219, "step": 119 }, { "KL/chosen_KL_mean": -58.99882888793945, "KL/mean": -89.11383056640625, "KL/rejected_KL_mean": -119.22883605957031, "KL/std": 73.892822265625, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.008354677818715572, "fcm_dpo/delta": -0.1406194120645523, "fcm_dpo/margin": 60.230003356933594, "fcm_dpo/q_t": 0.3826143443584442, "grad_norm": 22.082054138183594, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.567842960357666, "logits/rejected": -0.5426411032676697, "logps/chosen": -135.7796630859375, "logps/ref_chosen": -76.78083801269531, "logps/ref_rejected": -108.02374267578125, "logps/rejected": -227.25257873535156, "loss": 1.0416, "margin_dpo/margin_mean": 60.230003356933594, "margin_dpo/margin_std": 79.12581634521484, "step": 120 }, { "KL/chosen_KL_mean": -55.65784454345703, "KL/mean": -89.1383056640625, "KL/rejected_KL_mean": -122.61875915527344, "KL/std": 71.20858764648438, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.007244884967803955, "fcm_dpo/delta": -0.11996462941169739, "fcm_dpo/margin": 66.9609375, "fcm_dpo/q_t": 0.3862529993057251, "grad_norm": 18.99169158935547, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.5176138877868652, "logits/rejected": -0.5163988471031189, "logps/chosen": -117.44773864746094, "logps/ref_chosen": -61.789894104003906, "logps/ref_rejected": -109.99456787109375, "logps/rejected": -232.61334228515625, "loss": 1.0532, "margin_dpo/margin_mean": 66.9609375, "margin_dpo/margin_std": 89.08659362792969, "step": 121 }, { "KL/chosen_KL_mean": -50.142337799072266, "KL/mean": -91.1891098022461, "KL/rejected_KL_mean": -132.2358856201172, "KL/std": 79.52822875976562, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.0063782669603824615, "fcm_dpo/delta": -0.16817903518676758, "fcm_dpo/margin": 82.09354400634766, "fcm_dpo/q_t": 0.3728944957256317, "grad_norm": 18.489049911499023, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.46613478660583496, "logits/rejected": -0.48410728573799133, "logps/chosen": -97.04454803466797, "logps/ref_chosen": -46.9022102355957, "logps/ref_rejected": -106.71418762207031, "logps/rejected": -238.9500732421875, "loss": 0.9968, "margin_dpo/margin_mean": 82.09354400634766, "margin_dpo/margin_std": 84.7960205078125, "step": 122 }, { "KL/chosen_KL_mean": -57.29137420654297, "KL/mean": -83.6397476196289, "KL/rejected_KL_mean": -109.98812866210938, "KL/std": 68.29986572265625, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.006330414209514856, "fcm_dpo/delta": 0.07942930608987808, "fcm_dpo/margin": 52.696746826171875, "fcm_dpo/q_t": 0.4271467626094818, "grad_norm": 17.179462432861328, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.4523668885231018, "logits/rejected": -0.43804460763931274, "logps/chosen": -118.63002014160156, "logps/ref_chosen": -61.33863830566406, "logps/ref_rejected": -87.775390625, "logps/rejected": -197.76351928710938, "loss": 1.1663, "margin_dpo/margin_mean": 52.69674301147461, "margin_dpo/margin_std": 93.03097534179688, "step": 123 }, { "KL/chosen_KL_mean": -66.95594787597656, "KL/mean": -107.28865051269531, "KL/rejected_KL_mean": -147.62136840820312, "KL/std": 95.69377899169922, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.005985944531857967, "fcm_dpo/delta": -0.11251755058765411, "fcm_dpo/margin": 80.66542053222656, "fcm_dpo/q_t": 0.39224308729171753, "grad_norm": 17.14952278137207, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.46528786420822144, "logits/rejected": -0.47065889835357666, "logps/chosen": -138.40428161621094, "logps/ref_chosen": -71.44833374023438, "logps/ref_rejected": -117.58056640625, "logps/rejected": -265.2019348144531, "loss": 1.0778, "margin_dpo/margin_mean": 80.66542053222656, "margin_dpo/margin_std": 120.44160461425781, "step": 124 }, { "KL/chosen_KL_mean": -54.70975112915039, "KL/mean": -89.0744857788086, "KL/rejected_KL_mean": -123.43922424316406, "KL/std": 80.83454895019531, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.0059068650007247925, "fcm_dpo/delta": -0.007670434191823006, "fcm_dpo/margin": 68.72946166992188, "fcm_dpo/q_t": 0.40873342752456665, "grad_norm": 14.995677947998047, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.36938369274139404, "logits/rejected": -0.37332677841186523, "logps/chosen": -104.84669494628906, "logps/ref_chosen": -50.136940002441406, "logps/ref_rejected": -83.98861694335938, "logps/rejected": -207.42782592773438, "loss": 1.1061, "margin_dpo/margin_mean": 68.72946166992188, "margin_dpo/margin_std": 102.3861083984375, "step": 125 }, { "KL/chosen_KL_mean": -59.12432861328125, "KL/mean": -92.44718933105469, "KL/rejected_KL_mean": -125.77005767822266, "KL/std": 74.66473388671875, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.005859220400452614, "fcm_dpo/delta": 0.011660443618893623, "fcm_dpo/margin": 66.6457290649414, "fcm_dpo/q_t": 0.4124431610107422, "grad_norm": 16.17157554626465, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.44136887788772583, "logits/rejected": -0.441866010427475, "logps/chosen": -114.79139709472656, "logps/ref_chosen": -55.66706848144531, "logps/ref_rejected": -98.1297607421875, "logps/rejected": -223.89981079101562, "loss": 1.1071, "margin_dpo/margin_mean": 66.64572143554688, "margin_dpo/margin_std": 96.18585968017578, "step": 126 }, { "KL/chosen_KL_mean": -58.726707458496094, "KL/mean": -81.67306518554688, "KL/rejected_KL_mean": -104.61940002441406, "KL/std": 65.82872009277344, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.006503104232251644, "fcm_dpo/delta": 0.11829523742198944, "fcm_dpo/margin": 45.892696380615234, "fcm_dpo/q_t": 0.43302494287490845, "grad_norm": 19.481531143188477, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.459053099155426, "logits/rejected": -0.44523656368255615, "logps/chosen": -115.28138732910156, "logps/ref_chosen": -56.55467987060547, "logps/ref_rejected": -76.7957763671875, "logps/rejected": -181.41519165039062, "loss": 1.1793, "margin_dpo/margin_mean": 45.8927001953125, "margin_dpo/margin_std": 83.20150756835938, "step": 127 }, { "KL/chosen_KL_mean": -61.33363342285156, "KL/mean": -86.33102416992188, "KL/rejected_KL_mean": -111.32841491699219, "KL/std": 76.6578598022461, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.007103352341800928, "fcm_dpo/delta": 0.04982195049524307, "fcm_dpo/margin": 49.994781494140625, "fcm_dpo/q_t": 0.42171311378479004, "grad_norm": 28.149051666259766, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.4325607419013977, "logits/rejected": -0.42009297013282776, "logps/chosen": -119.45458984375, "logps/ref_chosen": -58.12095642089844, "logps/ref_rejected": -76.43896484375, "logps/rejected": -187.7673797607422, "loss": 1.1761, "margin_dpo/margin_mean": 49.994781494140625, "margin_dpo/margin_std": 97.5225830078125, "step": 128 }, { "KL/chosen_KL_mean": -75.2124252319336, "KL/mean": -101.20868682861328, "KL/rejected_KL_mean": -127.2049560546875, "KL/std": 87.85305786132812, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.007200577296316624, "fcm_dpo/delta": 0.03126451000571251, "fcm_dpo/margin": 51.99251174926758, "fcm_dpo/q_t": 0.4244125485420227, "grad_norm": 21.352294921875, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.4686553478240967, "logits/rejected": -0.46320652961730957, "logps/chosen": -142.12879943847656, "logps/ref_chosen": -66.91637420654297, "logps/ref_rejected": -96.6422119140625, "logps/rejected": -223.84716796875, "loss": 1.1947, "margin_dpo/margin_mean": 51.992515563964844, "margin_dpo/margin_std": 116.50946044921875, "step": 129 }, { "KL/chosen_KL_mean": -54.154781341552734, "KL/mean": -93.47927856445312, "KL/rejected_KL_mean": -132.8037567138672, "KL/std": 83.20738983154297, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.006708543747663498, "fcm_dpo/delta": -0.17494350671768188, "fcm_dpo/margin": 78.64897918701172, "fcm_dpo/q_t": 0.379627525806427, "grad_norm": 17.628036499023438, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.44320395588874817, "logits/rejected": -0.4385804831981659, "logps/chosen": -98.8216323852539, "logps/ref_chosen": -44.66685104370117, "logps/ref_rejected": -82.78165435791016, "logps/rejected": -215.58541870117188, "loss": 1.0494, "margin_dpo/margin_mean": 78.64897918701172, "margin_dpo/margin_std": 104.21408081054688, "step": 130 }, { "KL/chosen_KL_mean": -50.51225280761719, "KL/mean": -91.10379791259766, "KL/rejected_KL_mean": -131.69534301757812, "KL/std": 87.86686706542969, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.005672046914696693, "fcm_dpo/delta": -0.07984572649002075, "fcm_dpo/margin": 81.18309020996094, "fcm_dpo/q_t": 0.38995444774627686, "grad_norm": 26.029661178588867, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.4229122996330261, "logits/rejected": -0.4288443326950073, "logps/chosen": -95.43684387207031, "logps/ref_chosen": -44.924591064453125, "logps/ref_rejected": -88.44401550292969, "logps/rejected": -220.1393585205078, "loss": 1.0467, "margin_dpo/margin_mean": 81.18309020996094, "margin_dpo/margin_std": 95.4244384765625, "step": 131 }, { "KL/chosen_KL_mean": -65.28816986083984, "KL/mean": -97.25325012207031, "KL/rejected_KL_mean": -129.2183380126953, "KL/std": 84.85006713867188, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.0057223797775805, "fcm_dpo/delta": 0.04182172939181328, "fcm_dpo/margin": 63.93017578125, "fcm_dpo/q_t": 0.4215155839920044, "grad_norm": 17.443683624267578, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.4490656852722168, "logits/rejected": -0.45410823822021484, "logps/chosen": -124.28925323486328, "logps/ref_chosen": -59.00108337402344, "logps/ref_rejected": -87.89215087890625, "logps/rejected": -217.11048889160156, "loss": 1.1448, "margin_dpo/margin_mean": 63.93017578125, "margin_dpo/margin_std": 110.97628784179688, "step": 132 }, { "KL/chosen_KL_mean": -76.72837829589844, "KL/mean": -104.55119323730469, "KL/rejected_KL_mean": -132.37400817871094, "KL/std": 75.45304107666016, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.006116401869803667, "fcm_dpo/delta": 0.07216479629278183, "fcm_dpo/margin": 55.64562225341797, "fcm_dpo/q_t": 0.4264053702354431, "grad_norm": 22.552947998046875, "learning_rate": 4.870401618977415e-07, "logits/chosen": -0.42806553840637207, "logits/rejected": -0.4164883494377136, "logps/chosen": -143.3328857421875, "logps/ref_chosen": -66.60449981689453, "logps/ref_rejected": -96.33355712890625, "logps/rejected": -228.7075653076172, "loss": 1.1563, "margin_dpo/margin_mean": 55.64562225341797, "margin_dpo/margin_std": 97.3055191040039, "step": 133 }, { "KL/chosen_KL_mean": -58.90048599243164, "KL/mean": -88.99266815185547, "KL/rejected_KL_mean": -119.08485412597656, "KL/std": 71.10643005371094, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.006456311792135239, "fcm_dpo/delta": 0.012345774099230766, "fcm_dpo/margin": 60.18437194824219, "fcm_dpo/q_t": 0.4125699996948242, "grad_norm": 18.19769287109375, "learning_rate": 4.866292092063986e-07, "logits/chosen": -0.4092817008495331, "logits/rejected": -0.3999384641647339, "logps/chosen": -110.96974182128906, "logps/ref_chosen": -52.06925582885742, "logps/ref_rejected": -87.6545181274414, "logps/rejected": -206.7393798828125, "loss": 1.1018, "margin_dpo/margin_mean": 60.18437194824219, "margin_dpo/margin_std": 84.06219482421875, "step": 134 }, { "KL/chosen_KL_mean": -64.12809753417969, "KL/mean": -109.45974731445312, "KL/rejected_KL_mean": -154.7913818359375, "KL/std": 97.96502685546875, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.005646052770316601, "fcm_dpo/delta": -0.15241768956184387, "fcm_dpo/margin": 90.66329193115234, "fcm_dpo/q_t": 0.38228270411491394, "grad_norm": 22.45749855041504, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.4356979727745056, "logits/rejected": -0.46872130036354065, "logps/chosen": -114.48196411132812, "logps/ref_chosen": -50.353858947753906, "logps/ref_rejected": -115.97975158691406, "logps/rejected": -270.7711181640625, "loss": 1.0405, "margin_dpo/margin_mean": 90.66329193115234, "margin_dpo/margin_std": 120.07051086425781, "step": 135 }, { "KL/chosen_KL_mean": -76.92674255371094, "KL/mean": -106.96867370605469, "KL/rejected_KL_mean": -137.01058959960938, "KL/std": 91.17539978027344, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.005636686459183693, "fcm_dpo/delta": 0.07331253588199615, "fcm_dpo/margin": 60.0838623046875, "fcm_dpo/q_t": 0.4317668080329895, "grad_norm": 19.86505699157715, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.42861732840538025, "logits/rejected": -0.42438289523124695, "logps/chosen": -141.99925231933594, "logps/ref_chosen": -65.072509765625, "logps/ref_rejected": -96.32122802734375, "logps/rejected": -233.33181762695312, "loss": 1.1772, "margin_dpo/margin_mean": 60.083858489990234, "margin_dpo/margin_std": 119.13693237304688, "step": 136 }, { "KL/chosen_KL_mean": -70.731201171875, "KL/mean": -117.82597351074219, "KL/rejected_KL_mean": -164.92074584960938, "KL/std": 116.82910919189453, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.005281469784677029, "fcm_dpo/delta": -0.1319933831691742, "fcm_dpo/margin": 94.18954467773438, "fcm_dpo/q_t": 0.3907054662704468, "grad_norm": 16.0644588470459, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.38837531208992004, "logits/rejected": -0.4134828746318817, "logps/chosen": -119.49031829833984, "logps/ref_chosen": -48.759117126464844, "logps/ref_rejected": -113.86376953125, "logps/rejected": -278.7845153808594, "loss": 1.0696, "margin_dpo/margin_mean": 94.1895523071289, "margin_dpo/margin_std": 142.3530731201172, "step": 137 }, { "KL/chosen_KL_mean": -75.57490539550781, "KL/mean": -110.52081298828125, "KL/rejected_KL_mean": -145.46673583984375, "KL/std": 88.41027069091797, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.005329763051122427, "fcm_dpo/delta": 0.032860539853572845, "fcm_dpo/margin": 69.89181518554688, "fcm_dpo/q_t": 0.41438084840774536, "grad_norm": 17.347095489501953, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.4172437787055969, "logits/rejected": -0.40940582752227783, "logps/chosen": -136.09454345703125, "logps/ref_chosen": -60.519649505615234, "logps/ref_rejected": -93.19694519042969, "logps/rejected": -238.66368103027344, "loss": 1.1021, "margin_dpo/margin_mean": 69.89181518554688, "margin_dpo/margin_std": 89.81456756591797, "step": 138 }, { "KL/chosen_KL_mean": -63.819305419921875, "KL/mean": -104.35844421386719, "KL/rejected_KL_mean": -144.8975830078125, "KL/std": 82.37762451171875, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.005249645560979843, "fcm_dpo/delta": -0.032708242535591125, "fcm_dpo/margin": 81.07829284667969, "fcm_dpo/q_t": 0.402879536151886, "grad_norm": 16.59463882446289, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.35705071687698364, "logits/rejected": -0.3492419719696045, "logps/chosen": -110.710693359375, "logps/ref_chosen": -46.89138412475586, "logps/ref_rejected": -79.72798156738281, "logps/rejected": -224.62557983398438, "loss": 1.0796, "margin_dpo/margin_mean": 81.07829284667969, "margin_dpo/margin_std": 109.54811096191406, "step": 139 }, { "KL/chosen_KL_mean": -76.06951904296875, "KL/mean": -112.28123474121094, "KL/rejected_KL_mean": -148.49295043945312, "KL/std": 90.1976318359375, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.0052453503012657166, "fcm_dpo/delta": 0.02480418235063553, "fcm_dpo/margin": 72.42343139648438, "fcm_dpo/q_t": 0.41421765089035034, "grad_norm": 17.3253116607666, "learning_rate": 4.840329401637809e-07, "logits/chosen": -0.35899558663368225, "logits/rejected": -0.34745436906814575, "logps/chosen": -135.0442352294922, "logps/ref_chosen": -58.97471618652344, "logps/ref_rejected": -83.28410339355469, "logps/rejected": -231.77706909179688, "loss": 1.1193, "margin_dpo/margin_mean": 72.42343139648438, "margin_dpo/margin_std": 110.29766845703125, "step": 140 }, { "KL/chosen_KL_mean": -86.08993530273438, "KL/mean": -120.1361312866211, "KL/rejected_KL_mean": -154.1823272705078, "KL/std": 103.17254638671875, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.0054466212168335915, "fcm_dpo/delta": 0.03537699952721596, "fcm_dpo/margin": 68.09239196777344, "fcm_dpo/q_t": 0.4169376492500305, "grad_norm": 22.393936157226562, "learning_rate": 4.83578576263792e-07, "logits/chosen": -0.36837026476860046, "logits/rejected": -0.35952502489089966, "logps/chosen": -161.16558837890625, "logps/ref_chosen": -75.07566833496094, "logps/ref_rejected": -98.1922607421875, "logps/rejected": -252.3745880126953, "loss": 1.148, "margin_dpo/margin_mean": 68.09239196777344, "margin_dpo/margin_std": 120.2028579711914, "step": 141 }, { "KL/chosen_KL_mean": -84.57060241699219, "KL/mean": -123.71681213378906, "KL/rejected_KL_mean": -162.86300659179688, "KL/std": 108.09979248046875, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.005447980482131243, "fcm_dpo/delta": -0.03504451364278793, "fcm_dpo/margin": 78.29240417480469, "fcm_dpo/q_t": 0.40442773699760437, "grad_norm": 23.142438888549805, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.34042251110076904, "logits/rejected": -0.341153621673584, "logps/chosen": -142.59854125976562, "logps/ref_chosen": -58.027931213378906, "logps/ref_rejected": -94.58222961425781, "logps/rejected": -257.44525146484375, "loss": 1.1208, "margin_dpo/margin_mean": 78.29240417480469, "margin_dpo/margin_std": 126.79087829589844, "step": 142 }, { "KL/chosen_KL_mean": -86.95822143554688, "KL/mean": -112.62377166748047, "KL/rejected_KL_mean": -138.28932189941406, "KL/std": 93.756591796875, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.005426798947155476, "fcm_dpo/delta": 0.027737241238355637, "fcm_dpo/margin": 51.33110046386719, "fcm_dpo/q_t": 0.4396059513092041, "grad_norm": 20.347646713256836, "learning_rate": 4.826513955607734e-07, "logits/chosen": -0.3267577290534973, "logits/rejected": -0.3233267068862915, "logps/chosen": -144.55467224121094, "logps/ref_chosen": -57.59645080566406, "logps/ref_rejected": -78.99957275390625, "logps/rejected": -217.2888946533203, "loss": 1.2111, "margin_dpo/margin_mean": 51.33110046386719, "margin_dpo/margin_std": 109.97335815429688, "step": 143 }, { "KL/chosen_KL_mean": -79.65751647949219, "KL/mean": -109.87149810791016, "KL/rejected_KL_mean": -140.08547973632812, "KL/std": 80.81317138671875, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.005699482746422291, "fcm_dpo/delta": 0.06714989244937897, "fcm_dpo/margin": 60.42796325683594, "fcm_dpo/q_t": 0.42339879274368286, "grad_norm": 19.846458435058594, "learning_rate": 4.821786031898176e-07, "logits/chosen": -0.3399592638015747, "logits/rejected": -0.3357587456703186, "logps/chosen": -139.56387329101562, "logps/ref_chosen": -59.90636444091797, "logps/ref_rejected": -82.00025939941406, "logps/rejected": -222.0857391357422, "loss": 1.1396, "margin_dpo/margin_mean": 60.427955627441406, "margin_dpo/margin_std": 94.39094543457031, "step": 144 }, { "KL/chosen_KL_mean": -76.95576477050781, "KL/mean": -109.23151397705078, "KL/rejected_KL_mean": -141.50726318359375, "KL/std": 79.54987335205078, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.0058981748297810555, "fcm_dpo/delta": 0.02381444163620472, "fcm_dpo/margin": 64.55150604248047, "fcm_dpo/q_t": 0.4141959547996521, "grad_norm": 23.242799758911133, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.37550991773605347, "logits/rejected": -0.3661358058452606, "logps/chosen": -133.55642700195312, "logps/ref_chosen": -56.60066604614258, "logps/ref_rejected": -77.86631774902344, "logps/rejected": -219.3735809326172, "loss": 1.1192, "margin_dpo/margin_mean": 64.55149841308594, "margin_dpo/margin_std": 98.04264831542969, "step": 145 }, { "KL/chosen_KL_mean": -103.22034454345703, "KL/mean": -128.1830596923828, "KL/rejected_KL_mean": -153.14578247070312, "KL/std": 87.43064880371094, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.006305292714387178, "fcm_dpo/delta": 0.10162100195884705, "fcm_dpo/margin": 49.925445556640625, "fcm_dpo/q_t": 0.43169236183166504, "grad_norm": 39.6645622253418, "learning_rate": 4.812146767012779e-07, "logits/chosen": -0.32967615127563477, "logits/rejected": -0.30988651514053345, "logps/chosen": -169.22079467773438, "logps/ref_chosen": -66.00045013427734, "logps/ref_rejected": -81.70278930664062, "logps/rejected": -234.84857177734375, "loss": 1.206, "margin_dpo/margin_mean": 49.925445556640625, "margin_dpo/margin_std": 104.19798278808594, "step": 146 }, { "KL/chosen_KL_mean": -75.02632904052734, "KL/mean": -108.63874816894531, "KL/rejected_KL_mean": -142.25115966796875, "KL/std": 86.88672637939453, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.0063508497551083565, "fcm_dpo/delta": -0.035061098635196686, "fcm_dpo/margin": 67.22482299804688, "fcm_dpo/q_t": 0.4050912857055664, "grad_norm": 20.40033721923828, "learning_rate": 4.807235679840536e-07, "logits/chosen": -0.3828004002571106, "logits/rejected": -0.3709457516670227, "logps/chosen": -128.43182373046875, "logps/ref_chosen": -53.405487060546875, "logps/ref_rejected": -71.39060974121094, "logps/rejected": -213.6417694091797, "loss": 1.1127, "margin_dpo/margin_mean": 67.22482299804688, "margin_dpo/margin_std": 109.10481262207031, "step": 147 }, { "KL/chosen_KL_mean": -73.86956787109375, "KL/mean": -103.73741149902344, "KL/rejected_KL_mean": -133.60525512695312, "KL/std": 89.116455078125, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.0060739642940461636, "fcm_dpo/delta": -0.0918317511677742, "fcm_dpo/margin": 59.735694885253906, "fcm_dpo/q_t": 0.41868463158607483, "grad_norm": 18.132606506347656, "learning_rate": 4.802263794862384e-07, "logits/chosen": -0.41148072481155396, "logits/rejected": -0.4058622121810913, "logps/chosen": -138.806640625, "logps/ref_chosen": -64.93708038330078, "logps/ref_rejected": -103.09384155273438, "logps/rejected": -236.6990966796875, "loss": 1.1443, "margin_dpo/margin_mean": 59.735694885253906, "margin_dpo/margin_std": 94.41209411621094, "step": 148 }, { "KL/chosen_KL_mean": -69.35594940185547, "KL/mean": -106.29493713378906, "KL/rejected_KL_mean": -143.23391723632812, "KL/std": 77.87938690185547, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.005501337815076113, "fcm_dpo/delta": -0.0162881538271904, "fcm_dpo/margin": 73.87796020507812, "fcm_dpo/q_t": 0.40519750118255615, "grad_norm": 16.045988082885742, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.428414911031723, "logits/rejected": -0.4166084825992584, "logps/chosen": -127.8297119140625, "logps/ref_chosen": -58.47376251220703, "logps/ref_rejected": -99.31474304199219, "logps/rejected": -242.5486602783203, "loss": 1.0887, "margin_dpo/margin_mean": 73.87796020507812, "margin_dpo/margin_std": 93.21217346191406, "step": 149 }, { "KL/chosen_KL_mean": -60.34211349487305, "KL/mean": -97.03437042236328, "KL/rejected_KL_mean": -133.72662353515625, "KL/std": 91.03880310058594, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.005468531046062708, "fcm_dpo/delta": -0.007947839796543121, "fcm_dpo/margin": 73.38451385498047, "fcm_dpo/q_t": 0.411196231842041, "grad_norm": 18.763927459716797, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.4295479655265808, "logits/rejected": -0.4361698627471924, "logps/chosen": -106.04792785644531, "logps/ref_chosen": -45.705810546875, "logps/ref_rejected": -83.34759521484375, "logps/rejected": -217.07421875, "loss": 1.1058, "margin_dpo/margin_mean": 73.38451385498047, "margin_dpo/margin_std": 108.1720199584961, "step": 150 }, { "KL/chosen_KL_mean": -77.08334350585938, "KL/mean": -112.62428283691406, "KL/rejected_KL_mean": -148.16522216796875, "KL/std": 86.95265197753906, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.005633922293782234, "fcm_dpo/delta": -0.0006460570730268955, "fcm_dpo/margin": 71.08187866210938, "fcm_dpo/q_t": 0.40812182426452637, "grad_norm": 22.76422882080078, "learning_rate": 4.786984671220053e-07, "logits/chosen": -0.47929656505584717, "logits/rejected": -0.4552541971206665, "logps/chosen": -147.6541748046875, "logps/ref_chosen": -70.57083129882812, "logps/ref_rejected": -100.46382141113281, "logps/rejected": -248.62904357910156, "loss": 1.0926, "margin_dpo/margin_mean": 71.0818862915039, "margin_dpo/margin_std": 95.64910888671875, "step": 151 }, { "KL/chosen_KL_mean": -66.67381286621094, "KL/mean": -110.26457214355469, "KL/rejected_KL_mean": -153.8553466796875, "KL/std": 87.53227233886719, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.0053495545871555805, "fcm_dpo/delta": -0.08620619773864746, "fcm_dpo/margin": 87.18153381347656, "fcm_dpo/q_t": 0.39188152551651, "grad_norm": 20.685396194458008, "learning_rate": 4.78177092112495e-07, "logits/chosen": -0.44135403633117676, "logits/rejected": -0.4416983723640442, "logps/chosen": -126.83819580078125, "logps/ref_chosen": -60.16438674926758, "logps/ref_rejected": -106.14045715332031, "logps/rejected": -259.99578857421875, "loss": 1.0522, "margin_dpo/margin_mean": 87.18153381347656, "margin_dpo/margin_std": 108.27259826660156, "step": 152 }, { "KL/chosen_KL_mean": -65.79019927978516, "KL/mean": -103.00405883789062, "KL/rejected_KL_mean": -140.2178955078125, "KL/std": 96.10725402832031, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.005271708592772484, "fcm_dpo/delta": 0.008975501172244549, "fcm_dpo/margin": 74.42771911621094, "fcm_dpo/q_t": 0.4141613245010376, "grad_norm": 15.699677467346191, "learning_rate": 4.776497044244016e-07, "logits/chosen": -0.4330843687057495, "logits/rejected": -0.4293479323387146, "logps/chosen": -122.10546875, "logps/ref_chosen": -56.315277099609375, "logps/ref_rejected": -85.65583801269531, "logps/rejected": -225.87374877929688, "loss": 1.1194, "margin_dpo/margin_mean": 74.42771911621094, "margin_dpo/margin_std": 118.5946273803711, "step": 153 }, { "KL/chosen_KL_mean": -79.26509094238281, "KL/mean": -114.19869995117188, "KL/rejected_KL_mean": -149.13230895996094, "KL/std": 93.6673812866211, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.005427438300102949, "fcm_dpo/delta": 0.023521043360233307, "fcm_dpo/margin": 69.86721801757812, "fcm_dpo/q_t": 0.41615161299705505, "grad_norm": 17.67850112915039, "learning_rate": 4.771163179548808e-07, "logits/chosen": -0.43425455689430237, "logits/rejected": -0.4390127658843994, "logps/chosen": -142.00765991210938, "logps/ref_chosen": -62.74256896972656, "logps/ref_rejected": -104.24420166015625, "logps/rejected": -253.3765106201172, "loss": 1.1462, "margin_dpo/margin_mean": 69.86721801757812, "margin_dpo/margin_std": 122.60856628417969, "step": 154 }, { "KL/chosen_KL_mean": -72.60445404052734, "KL/mean": -106.42167663574219, "KL/rejected_KL_mean": -140.2388916015625, "KL/std": 85.55455017089844, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.005552366841584444, "fcm_dpo/delta": 0.028694026172161102, "fcm_dpo/margin": 67.63446044921875, "fcm_dpo/q_t": 0.41562047600746155, "grad_norm": 19.240365982055664, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.44728028774261475, "logits/rejected": -0.4302216172218323, "logps/chosen": -133.25762939453125, "logps/ref_chosen": -60.65318298339844, "logps/ref_rejected": -77.49220275878906, "logps/rejected": -217.73110961914062, "loss": 1.13, "margin_dpo/margin_mean": 67.63445281982422, "margin_dpo/margin_std": 108.91281127929688, "step": 155 }, { "KL/chosen_KL_mean": -99.16265869140625, "KL/mean": -118.9976806640625, "KL/rejected_KL_mean": -138.83270263671875, "KL/std": 91.43487548828125, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.005871580448001623, "fcm_dpo/delta": 0.06297972053289413, "fcm_dpo/margin": 39.67005157470703, "fcm_dpo/q_t": 0.44856178760528564, "grad_norm": 27.07191276550293, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -0.3714250922203064, "logits/rejected": -0.3661670684814453, "logps/chosen": -168.654541015625, "logps/ref_chosen": -69.49188232421875, "logps/ref_rejected": -77.16929626464844, "logps/rejected": -216.0019989013672, "loss": 1.2847, "margin_dpo/margin_mean": 39.67005157470703, "margin_dpo/margin_std": 120.68568420410156, "step": 156 }, { "KL/chosen_KL_mean": -87.81330871582031, "KL/mean": -132.32769775390625, "KL/rejected_KL_mean": -176.84207153320312, "KL/std": 99.45307159423828, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.005287863779813051, "fcm_dpo/delta": -0.10068926960229874, "fcm_dpo/margin": 89.02876281738281, "fcm_dpo/q_t": 0.3861895203590393, "grad_norm": 22.06880760192871, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -0.36405149102211, "logits/rejected": -0.3723771572113037, "logps/chosen": -149.18174743652344, "logps/ref_chosen": -61.368438720703125, "logps/ref_rejected": -107.64636993408203, "logps/rejected": -284.4884338378906, "loss": 1.0573, "margin_dpo/margin_mean": 89.02876281738281, "margin_dpo/margin_std": 111.94908142089844, "step": 157 }, { "KL/chosen_KL_mean": -89.31040954589844, "KL/mean": -137.76190185546875, "KL/rejected_KL_mean": -186.21340942382812, "KL/std": 123.51625061035156, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.004927870351821184, "fcm_dpo/delta": -0.10203598439693451, "fcm_dpo/margin": 96.90299224853516, "fcm_dpo/q_t": 0.3954607844352722, "grad_norm": 16.532785415649414, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -0.3246377408504486, "logits/rejected": -0.34248343110084534, "logps/chosen": -146.92332458496094, "logps/ref_chosen": -57.612918853759766, "logps/ref_rejected": -113.6946792602539, "logps/rejected": -299.9080810546875, "loss": 1.0826, "margin_dpo/margin_mean": 96.90299224853516, "margin_dpo/margin_std": 150.00479125976562, "step": 158 }, { "KL/chosen_KL_mean": -102.0267333984375, "KL/mean": -133.82208251953125, "KL/rejected_KL_mean": -165.6173858642578, "KL/std": 108.39896392822266, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.005198903381824493, "fcm_dpo/delta": 0.07878997921943665, "fcm_dpo/margin": 63.59064483642578, "fcm_dpo/q_t": 0.42666539549827576, "grad_norm": 25.30782127380371, "learning_rate": 4.743599013306165e-07, "logits/chosen": -0.35725462436676025, "logits/rejected": -0.32486408948898315, "logps/chosen": -183.5870819091797, "logps/ref_chosen": -81.56034851074219, "logps/ref_rejected": -88.89871215820312, "logps/rejected": -254.51609802246094, "loss": 1.1718, "margin_dpo/margin_mean": 63.59064865112305, "margin_dpo/margin_std": 117.3126220703125, "step": 159 }, { "KL/chosen_KL_mean": -103.35143280029297, "KL/mean": -147.1392059326172, "KL/rejected_KL_mean": -190.927001953125, "KL/std": 114.80732727050781, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.0049042413011193275, "fcm_dpo/delta": -0.04174552857875824, "fcm_dpo/margin": 87.57557678222656, "fcm_dpo/q_t": 0.40538012981414795, "grad_norm": 21.559865951538086, "learning_rate": 4.737908228387656e-07, "logits/chosen": -0.3620595932006836, "logits/rejected": -0.3560444116592407, "logps/chosen": -169.08230590820312, "logps/ref_chosen": -65.73088073730469, "logps/ref_rejected": -97.21781921386719, "logps/rejected": -288.14483642578125, "loss": 1.1145, "margin_dpo/margin_mean": 87.57556915283203, "margin_dpo/margin_std": 143.64688110351562, "step": 160 }, { "KL/chosen_KL_mean": -87.88373565673828, "KL/mean": -126.63963317871094, "KL/rejected_KL_mean": -165.39553833007812, "KL/std": 91.78167724609375, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.0050220671109855175, "fcm_dpo/delta": 0.013326739892363548, "fcm_dpo/margin": 77.51181030273438, "fcm_dpo/q_t": 0.4130176603794098, "grad_norm": 18.22276496887207, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -0.362446665763855, "logits/rejected": -0.3627144396305084, "logps/chosen": -140.32020568847656, "logps/ref_chosen": -52.43647003173828, "logps/ref_rejected": -83.43095397949219, "logps/rejected": -248.82647705078125, "loss": 1.1129, "margin_dpo/margin_mean": 77.51181030273438, "margin_dpo/margin_std": 115.73544311523438, "step": 161 }, { "KL/chosen_KL_mean": -86.13906860351562, "KL/mean": -123.78965759277344, "KL/rejected_KL_mean": -161.44024658203125, "KL/std": 101.90530395507812, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.004698555450886488, "fcm_dpo/delta": -0.07051464915275574, "fcm_dpo/margin": 75.30118560791016, "fcm_dpo/q_t": 0.4173450469970703, "grad_norm": 18.306982040405273, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.4190049171447754, "logits/rejected": -0.4046899676322937, "logps/chosen": -148.74964904785156, "logps/ref_chosen": -62.6105842590332, "logps/ref_rejected": -89.39057922363281, "logps/rejected": -250.83084106445312, "loss": 1.1434, "margin_dpo/margin_mean": 75.30118560791016, "margin_dpo/margin_std": 122.22592163085938, "step": 162 }, { "KL/chosen_KL_mean": -95.6829833984375, "KL/mean": -131.91665649414062, "KL/rejected_KL_mean": -168.15032958984375, "KL/std": 101.21453857421875, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.004903091117739677, "fcm_dpo/delta": 0.054114192724227905, "fcm_dpo/margin": 72.46734619140625, "fcm_dpo/q_t": 0.4222571849822998, "grad_norm": 17.79399871826172, "learning_rate": 4.720482655449212e-07, "logits/chosen": -0.3502505421638489, "logits/rejected": -0.3333583474159241, "logps/chosen": -150.70462036132812, "logps/ref_chosen": -55.021629333496094, "logps/ref_rejected": -75.418212890625, "logps/rejected": -243.5685577392578, "loss": 1.1443, "margin_dpo/margin_mean": 72.46734619140625, "margin_dpo/margin_std": 121.69917297363281, "step": 163 }, { "KL/chosen_KL_mean": -88.13164520263672, "KL/mean": -133.67013549804688, "KL/rejected_KL_mean": -179.20860290527344, "KL/std": 99.38763427734375, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.00465630367398262, "fcm_dpo/delta": -0.03786290064454079, "fcm_dpo/margin": 91.07695770263672, "fcm_dpo/q_t": 0.4004104733467102, "grad_norm": 21.530731201171875, "learning_rate": 4.714556901942599e-07, "logits/chosen": -0.34161853790283203, "logits/rejected": -0.3289684057235718, "logps/chosen": -143.77230834960938, "logps/ref_chosen": -55.64066696166992, "logps/ref_rejected": -79.66463470458984, "logps/rejected": -258.87322998046875, "loss": 1.0758, "margin_dpo/margin_mean": 91.07695770263672, "margin_dpo/margin_std": 114.45172119140625, "step": 164 }, { "KL/chosen_KL_mean": -95.0910873413086, "KL/mean": -124.63126373291016, "KL/rejected_KL_mean": -154.17144775390625, "KL/std": 86.06841278076172, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.005135393235832453, "fcm_dpo/delta": 0.1146107017993927, "fcm_dpo/margin": 59.080352783203125, "fcm_dpo/q_t": 0.434864342212677, "grad_norm": 25.962055206298828, "learning_rate": 4.708572792802069e-07, "logits/chosen": -0.36488398909568787, "logits/rejected": -0.34004294872283936, "logps/chosen": -156.4017791748047, "logps/ref_chosen": -61.310691833496094, "logps/ref_rejected": -73.67060852050781, "logps/rejected": -227.84205627441406, "loss": 1.192, "margin_dpo/margin_mean": 59.080360412597656, "margin_dpo/margin_std": 115.70834350585938, "step": 165 }, { "KL/chosen_KL_mean": -85.34329223632812, "KL/mean": -141.81805419921875, "KL/rejected_KL_mean": -198.2928009033203, "KL/std": 127.079345703125, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.004592292942106724, "fcm_dpo/delta": -0.1705796718597412, "fcm_dpo/margin": 112.94950866699219, "fcm_dpo/q_t": 0.385653555393219, "grad_norm": 15.64688777923584, "learning_rate": 4.702530485714461e-07, "logits/chosen": -0.29077380895614624, "logits/rejected": -0.30136969685554504, "logps/chosen": -136.326904296875, "logps/ref_chosen": -50.98360061645508, "logps/ref_rejected": -98.09512329101562, "logps/rejected": -296.387939453125, "loss": 1.0467, "margin_dpo/margin_mean": 112.94950866699219, "margin_dpo/margin_std": 160.54930114746094, "step": 166 }, { "KL/chosen_KL_mean": -87.4961166381836, "KL/mean": -146.18600463867188, "KL/rejected_KL_mean": -204.87586975097656, "KL/std": 114.61824035644531, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.004137314390391111, "fcm_dpo/delta": -0.11398793756961823, "fcm_dpo/margin": 117.37975311279297, "fcm_dpo/q_t": 0.3846902847290039, "grad_norm": 18.566125869750977, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.31079137325286865, "logits/rejected": -0.31497058272361755, "logps/chosen": -137.9202117919922, "logps/ref_chosen": -50.424095153808594, "logps/ref_rejected": -96.03042602539062, "logps/rejected": -300.90631103515625, "loss": 1.0292, "margin_dpo/margin_mean": 117.3797607421875, "margin_dpo/margin_std": 134.6339874267578, "step": 167 }, { "KL/chosen_KL_mean": -97.19728088378906, "KL/mean": -139.75576782226562, "KL/rejected_KL_mean": -182.31427001953125, "KL/std": 108.02088928222656, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.0041666412726044655, "fcm_dpo/delta": 0.0552922822535038, "fcm_dpo/margin": 85.11698913574219, "fcm_dpo/q_t": 0.42072856426239014, "grad_norm": 19.070110321044922, "learning_rate": 4.690271916109034e-07, "logits/chosen": -0.292841374874115, "logits/rejected": -0.28525102138519287, "logps/chosen": -146.6601104736328, "logps/ref_chosen": -49.462825775146484, "logps/ref_rejected": -75.30855560302734, "logps/rejected": -257.6228332519531, "loss": 1.1239, "margin_dpo/margin_mean": 85.11698913574219, "margin_dpo/margin_std": 122.18766021728516, "step": 168 }, { "KL/chosen_KL_mean": -102.30801391601562, "KL/mean": -139.7085418701172, "KL/rejected_KL_mean": -177.10906982421875, "KL/std": 107.6697998046875, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.004094083793461323, "fcm_dpo/delta": -0.03929879516363144, "fcm_dpo/margin": 74.80106353759766, "fcm_dpo/q_t": 0.43097662925720215, "grad_norm": 22.41761016845703, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -0.3144596219062805, "logits/rejected": -0.3004973530769348, "logps/chosen": -162.1114501953125, "logps/ref_chosen": -59.803443908691406, "logps/ref_rejected": -83.34574890136719, "logps/rejected": -260.454833984375, "loss": 1.1915, "margin_dpo/margin_mean": 74.80106353759766, "margin_dpo/margin_std": 148.64874267578125, "step": 169 }, { "KL/chosen_KL_mean": -92.78736877441406, "KL/mean": -136.38568115234375, "KL/rejected_KL_mean": -179.98399353027344, "KL/std": 105.04856872558594, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.00412098178640008, "fcm_dpo/delta": 0.04728086292743683, "fcm_dpo/margin": 87.19662475585938, "fcm_dpo/q_t": 0.4182136654853821, "grad_norm": 17.567453384399414, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -0.2699648141860962, "logits/rejected": -0.2612574100494385, "logps/chosen": -142.25914001464844, "logps/ref_chosen": -49.471771240234375, "logps/ref_rejected": -75.91734313964844, "logps/rejected": -255.90133666992188, "loss": 1.1277, "margin_dpo/margin_mean": 87.19662475585938, "margin_dpo/margin_std": 125.52247619628906, "step": 170 }, { "KL/chosen_KL_mean": -132.91552734375, "KL/mean": -168.7076416015625, "KL/rejected_KL_mean": -204.499755859375, "KL/std": 122.58496856689453, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.004573632497340441, "fcm_dpo/delta": 0.08680512011051178, "fcm_dpo/margin": 71.584228515625, "fcm_dpo/q_t": 0.4330134391784668, "grad_norm": 25.66452980041504, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -0.2910054922103882, "logits/rejected": -0.27732402086257935, "logps/chosen": -217.41482543945312, "logps/ref_chosen": -84.49931335449219, "logps/ref_rejected": -109.38209533691406, "logps/rejected": -313.8818359375, "loss": 1.2, "margin_dpo/margin_mean": 71.584228515625, "margin_dpo/margin_std": 157.88197326660156, "step": 171 }, { "KL/chosen_KL_mean": -115.34175109863281, "KL/mean": -152.7630615234375, "KL/rejected_KL_mean": -190.18435668945312, "KL/std": 117.2271957397461, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.004836169071495533, "fcm_dpo/delta": 0.046509332954883575, "fcm_dpo/margin": 74.84261322021484, "fcm_dpo/q_t": 0.42060211300849915, "grad_norm": 21.91267967224121, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -0.34988853335380554, "logits/rejected": -0.33963823318481445, "logps/chosen": -183.99566650390625, "logps/ref_chosen": -68.65391540527344, "logps/ref_rejected": -85.43667602539062, "logps/rejected": -275.62103271484375, "loss": 1.149, "margin_dpo/margin_mean": 74.84262084960938, "margin_dpo/margin_std": 131.61460876464844, "step": 172 }, { "KL/chosen_KL_mean": -105.72806549072266, "KL/mean": -143.4074249267578, "KL/rejected_KL_mean": -181.08677673339844, "KL/std": 112.28417205810547, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.005077026784420013, "fcm_dpo/delta": 0.019231334328651428, "fcm_dpo/margin": 75.35870361328125, "fcm_dpo/q_t": 0.4149942994117737, "grad_norm": 22.99903678894043, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -0.34687381982803345, "logits/rejected": -0.32024502754211426, "logps/chosen": -168.77894592285156, "logps/ref_chosen": -63.050880432128906, "logps/ref_rejected": -78.68392181396484, "logps/rejected": -259.77069091796875, "loss": 1.1198, "margin_dpo/margin_mean": 75.35870361328125, "margin_dpo/margin_std": 116.21147155761719, "step": 173 }, { "KL/chosen_KL_mean": -97.80389404296875, "KL/mean": -142.25250244140625, "KL/rejected_KL_mean": -186.7010955810547, "KL/std": 110.95504760742188, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.004968566354364157, "fcm_dpo/delta": -0.05666285380721092, "fcm_dpo/margin": 88.89718627929688, "fcm_dpo/q_t": 0.4014216363430023, "grad_norm": 25.833757400512695, "learning_rate": 4.652116329460919e-07, "logits/chosen": -0.28936031460762024, "logits/rejected": -0.30766457319259644, "logps/chosen": -151.1668701171875, "logps/ref_chosen": -53.36296844482422, "logps/ref_rejected": -101.91120910644531, "logps/rejected": -288.6123046875, "loss": 1.0988, "margin_dpo/margin_mean": 88.8971939086914, "margin_dpo/margin_std": 134.5579833984375, "step": 174 }, { "KL/chosen_KL_mean": -90.04649353027344, "KL/mean": -149.2164306640625, "KL/rejected_KL_mean": -208.3863525390625, "KL/std": 116.98501586914062, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.004224025644361973, "fcm_dpo/delta": -0.13619616627693176, "fcm_dpo/margin": 118.33984375, "fcm_dpo/q_t": 0.37867677211761475, "grad_norm": 29.216033935546875, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.2830924093723297, "logits/rejected": -0.2700659930706024, "logps/chosen": -135.46426391601562, "logps/ref_chosen": -45.417762756347656, "logps/ref_rejected": -89.50579833984375, "logps/rejected": -297.89215087890625, "loss": 1.0097, "margin_dpo/margin_mean": 118.33984375, "margin_dpo/margin_std": 123.62623596191406, "step": 175 }, { "KL/chosen_KL_mean": -93.11599731445312, "KL/mean": -144.95571899414062, "KL/rejected_KL_mean": -196.79547119140625, "KL/std": 118.29825592041016, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.004063988104462624, "fcm_dpo/delta": -0.027029920369386673, "fcm_dpo/margin": 103.6794662475586, "fcm_dpo/q_t": 0.40472695231437683, "grad_norm": 17.027528762817383, "learning_rate": 4.638942309888058e-07, "logits/chosen": -0.287243127822876, "logits/rejected": -0.30266469717025757, "logps/chosen": -143.56883239746094, "logps/ref_chosen": -50.452842712402344, "logps/ref_rejected": -95.5589599609375, "logps/rejected": -292.35443115234375, "loss": 1.0783, "margin_dpo/margin_mean": 103.67945861816406, "margin_dpo/margin_std": 136.32940673828125, "step": 176 }, { "KL/chosen_KL_mean": -108.14009094238281, "KL/mean": -158.50189208984375, "KL/rejected_KL_mean": -208.86370849609375, "KL/std": 128.51873779296875, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.004049934446811676, "fcm_dpo/delta": -0.011070270091295242, "fcm_dpo/margin": 100.7236099243164, "fcm_dpo/q_t": 0.4076334238052368, "grad_norm": 22.69172477722168, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -0.3215820789337158, "logits/rejected": -0.3144031763076782, "logps/chosen": -169.35655212402344, "logps/ref_chosen": -61.216468811035156, "logps/ref_rejected": -95.89378356933594, "logps/rejected": -304.75750732421875, "loss": 1.0863, "margin_dpo/margin_mean": 100.72361755371094, "margin_dpo/margin_std": 134.19070434570312, "step": 177 }, { "KL/chosen_KL_mean": -118.08624267578125, "KL/mean": -180.44430541992188, "KL/rejected_KL_mean": -242.80238342285156, "KL/std": 147.27687072753906, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.003680055495351553, "fcm_dpo/delta": -0.0795358195900917, "fcm_dpo/margin": 124.71614074707031, "fcm_dpo/q_t": 0.3944624662399292, "grad_norm": 22.354063034057617, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -0.2147635966539383, "logits/rejected": -0.20747539401054382, "logps/chosen": -176.35104370117188, "logps/ref_chosen": -58.26478958129883, "logps/ref_rejected": -105.3653335571289, "logps/rejected": -348.167724609375, "loss": 1.0579, "margin_dpo/margin_mean": 124.71614074707031, "margin_dpo/margin_std": 161.7968292236328, "step": 178 }, { "KL/chosen_KL_mean": -123.17328643798828, "KL/mean": -166.720458984375, "KL/rejected_KL_mean": -210.26763916015625, "KL/std": 128.99139404296875, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.0038236351683735847, "fcm_dpo/delta": 0.08045655488967896, "fcm_dpo/margin": 87.09436798095703, "fcm_dpo/q_t": 0.4275627136230469, "grad_norm": 26.390382766723633, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -0.2743037939071655, "logits/rejected": -0.2854520082473755, "logps/chosen": -184.23159790039062, "logps/ref_chosen": -61.05832290649414, "logps/ref_rejected": -90.52782440185547, "logps/rejected": -300.79547119140625, "loss": 1.1747, "margin_dpo/margin_mean": 87.09436798095703, "margin_dpo/margin_std": 162.98179626464844, "step": 179 }, { "KL/chosen_KL_mean": -106.45706176757812, "KL/mean": -164.29107666015625, "KL/rejected_KL_mean": -222.1250762939453, "KL/std": 113.41049194335938, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.003801894374191761, "fcm_dpo/delta": -0.05152449384331703, "fcm_dpo/margin": 115.66802978515625, "fcm_dpo/q_t": 0.39676570892333984, "grad_norm": 18.645387649536133, "learning_rate": 4.611919330113591e-07, "logits/chosen": -0.25623831152915955, "logits/rejected": -0.25502607226371765, "logps/chosen": -160.79977416992188, "logps/ref_chosen": -54.34272003173828, "logps/ref_rejected": -98.21183776855469, "logps/rejected": -320.3369140625, "loss": 1.0622, "margin_dpo/margin_mean": 115.66802215576172, "margin_dpo/margin_std": 143.18829345703125, "step": 180 }, { "KL/chosen_KL_mean": -91.30926513671875, "KL/mean": -126.50997924804688, "KL/rejected_KL_mean": -161.710693359375, "KL/std": 105.37017822265625, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.004171198233962059, "fcm_dpo/delta": 0.12288638204336166, "fcm_dpo/margin": 70.40142059326172, "fcm_dpo/q_t": 0.4360322952270508, "grad_norm": 18.242246627807617, "learning_rate": 4.605024008834863e-07, "logits/chosen": -0.2822926938533783, "logits/rejected": -0.25778597593307495, "logps/chosen": -146.30972290039062, "logps/ref_chosen": -55.000457763671875, "logps/ref_rejected": -61.656166076660156, "logps/rejected": -223.36685180664062, "loss": 1.1884, "margin_dpo/margin_mean": 70.40142059326172, "margin_dpo/margin_std": 135.00912475585938, "step": 181 }, { "KL/chosen_KL_mean": -84.9980239868164, "KL/mean": -147.97802734375, "KL/rejected_KL_mean": -210.95803833007812, "KL/std": 125.26875305175781, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.0038350564427673817, "fcm_dpo/delta": -0.11499916762113571, "fcm_dpo/margin": 125.96003723144531, "fcm_dpo/q_t": 0.3841468095779419, "grad_norm": 14.498435974121094, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.27837133407592773, "logits/rejected": -0.28961560130119324, "logps/chosen": -126.10587310791016, "logps/ref_chosen": -41.107852935791016, "logps/ref_rejected": -89.5215835571289, "logps/rejected": -300.4796142578125, "loss": 1.0402, "margin_dpo/margin_mean": 125.96002960205078, "margin_dpo/margin_std": 152.85032653808594, "step": 182 }, { "KL/chosen_KL_mean": -127.10066223144531, "KL/mean": -160.03028869628906, "KL/rejected_KL_mean": -192.95993041992188, "KL/std": 105.72233581542969, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.0037473866250365973, "fcm_dpo/delta": -0.014634971506893635, "fcm_dpo/margin": 65.85924530029297, "fcm_dpo/q_t": 0.4414650797843933, "grad_norm": 16.12941551208496, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -0.2845485210418701, "logits/rejected": -0.27767112851142883, "logps/chosen": -184.62522888183594, "logps/ref_chosen": -57.52456283569336, "logps/ref_rejected": -75.97572326660156, "logps/rejected": -268.9356689453125, "loss": 1.2074, "margin_dpo/margin_mean": 65.85924530029297, "margin_dpo/margin_std": 116.302734375, "step": 183 }, { "KL/chosen_KL_mean": -108.40987396240234, "KL/mean": -143.2193603515625, "KL/rejected_KL_mean": -178.02880859375, "KL/std": 99.74555206298828, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.0037929057143628597, "fcm_dpo/delta": 0.024003401398658752, "fcm_dpo/margin": 69.61894226074219, "fcm_dpo/q_t": 0.4396555423736572, "grad_norm": 15.583113670349121, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -0.3264748156070709, "logits/rejected": -0.31739190220832825, "logps/chosen": -166.954833984375, "logps/ref_chosen": -58.544952392578125, "logps/ref_rejected": -76.63406372070312, "logps/rejected": -254.66287231445312, "loss": 1.1946, "margin_dpo/margin_mean": 69.61894226074219, "margin_dpo/margin_std": 125.76559448242188, "step": 184 }, { "KL/chosen_KL_mean": -119.91249084472656, "KL/mean": -147.1962890625, "KL/rejected_KL_mean": -174.48007202148438, "KL/std": 115.40476989746094, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.003838424803689122, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.567588806152344, "fcm_dpo/q_t": 0.45364660024642944, "grad_norm": 19.94534683227539, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -0.2822297513484955, "logits/rejected": -0.2613981068134308, "logps/chosen": -181.93833923339844, "logps/ref_chosen": -62.025848388671875, "logps/ref_rejected": -73.7625961303711, "logps/rejected": -248.24266052246094, "loss": 1.2533, "margin_dpo/margin_mean": 54.567588806152344, "margin_dpo/margin_std": 138.52407836914062, "step": 185 }, { "KL/chosen_KL_mean": -114.25096893310547, "KL/mean": -166.89956665039062, "KL/rejected_KL_mean": -219.54815673828125, "KL/std": 117.91287231445312, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.003811160335317254, "fcm_dpo/delta": -0.0017963321879506111, "fcm_dpo/margin": 105.29721069335938, "fcm_dpo/q_t": 0.40835633873939514, "grad_norm": 19.27111053466797, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -0.31773799657821655, "logits/rejected": -0.3034238815307617, "logps/chosen": -183.60443115234375, "logps/ref_chosen": -69.35346984863281, "logps/ref_rejected": -88.07244873046875, "logps/rejected": -307.62060546875, "loss": 1.0898, "margin_dpo/margin_mean": 105.29721069335938, "margin_dpo/margin_std": 139.1805419921875, "step": 186 }, { "KL/chosen_KL_mean": -102.67378234863281, "KL/mean": -148.59390258789062, "KL/rejected_KL_mean": -194.5139923095703, "KL/std": 112.72882080078125, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.004015129059553146, "fcm_dpo/delta": 0.03561420738697052, "fcm_dpo/margin": 91.84022521972656, "fcm_dpo/q_t": 0.41552722454071045, "grad_norm": 19.90249252319336, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -0.27692389488220215, "logits/rejected": -0.27118462324142456, "logps/chosen": -155.43023681640625, "logps/ref_chosen": -52.7564582824707, "logps/ref_rejected": -81.96910095214844, "logps/rejected": -276.48309326171875, "loss": 1.1093, "margin_dpo/margin_mean": 91.84022521972656, "margin_dpo/margin_std": 122.15457916259766, "step": 187 }, { "KL/chosen_KL_mean": -98.16572570800781, "KL/mean": -153.54725646972656, "KL/rejected_KL_mean": -208.92877197265625, "KL/std": 125.24346923828125, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.003873111680150032, "fcm_dpo/delta": -0.036867331713438034, "fcm_dpo/margin": 110.76304626464844, "fcm_dpo/q_t": 0.4013431966304779, "grad_norm": 21.790515899658203, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -0.2571265697479248, "logits/rejected": -0.26449519395828247, "logps/chosen": -147.58120727539062, "logps/ref_chosen": -49.415489196777344, "logps/ref_rejected": -89.54043579101562, "logps/rejected": -298.4692077636719, "loss": 1.0701, "margin_dpo/margin_mean": 110.76305389404297, "margin_dpo/margin_std": 140.52169799804688, "step": 188 }, { "KL/chosen_KL_mean": -120.33238220214844, "KL/mean": -165.42770385742188, "KL/rejected_KL_mean": -210.52301025390625, "KL/std": 131.658203125, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.003984817303717136, "fcm_dpo/delta": 0.04916512593626976, "fcm_dpo/margin": 90.19061279296875, "fcm_dpo/q_t": 0.42296701669692993, "grad_norm": 27.90737533569336, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -0.2483985424041748, "logits/rejected": -0.23404812812805176, "logps/chosen": -172.73133850097656, "logps/ref_chosen": -52.39896011352539, "logps/ref_rejected": -72.16735076904297, "logps/rejected": -282.69036865234375, "loss": 1.1524, "margin_dpo/margin_mean": 90.19062042236328, "margin_dpo/margin_std": 164.16030883789062, "step": 189 }, { "KL/chosen_KL_mean": -128.05345153808594, "KL/mean": -182.98655700683594, "KL/rejected_KL_mean": -237.91966247558594, "KL/std": 141.72225952148438, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.004035481717437506, "fcm_dpo/delta": -0.06281507015228271, "fcm_dpo/margin": 109.86622619628906, "fcm_dpo/q_t": 0.40009400248527527, "grad_norm": 21.0288143157959, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -0.29694175720214844, "logits/rejected": -0.2908221483230591, "logps/chosen": -192.73651123046875, "logps/ref_chosen": -64.68305969238281, "logps/ref_rejected": -102.55052185058594, "logps/rejected": -340.47021484375, "loss": 1.1029, "margin_dpo/margin_mean": 109.86621856689453, "margin_dpo/margin_std": 163.8136444091797, "step": 190 }, { "KL/chosen_KL_mean": -120.60160827636719, "KL/mean": -199.1689910888672, "KL/rejected_KL_mean": -277.73638916015625, "KL/std": 161.70599365234375, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.0032641002908349037, "fcm_dpo/delta": -0.16109062731266022, "fcm_dpo/margin": 157.13475036621094, "fcm_dpo/q_t": 0.377569317817688, "grad_norm": 17.576215744018555, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -0.2872377634048462, "logits/rejected": -0.26138466596603394, "logps/chosen": -189.26048278808594, "logps/ref_chosen": -68.65887451171875, "logps/ref_rejected": -110.1396713256836, "logps/rejected": -387.8760681152344, "loss": 1.0153, "margin_dpo/margin_mean": 157.134765625, "margin_dpo/margin_std": 182.57261657714844, "step": 191 }, { "KL/chosen_KL_mean": -150.72152709960938, "KL/mean": -201.56427001953125, "KL/rejected_KL_mean": -252.40701293945312, "KL/std": 139.52310180664062, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.0034080445766448975, "fcm_dpo/delta": 0.06307677179574966, "fcm_dpo/margin": 101.68548583984375, "fcm_dpo/q_t": 0.4241996109485626, "grad_norm": 20.135229110717773, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -0.22673380374908447, "logits/rejected": -0.2247309386730194, "logps/chosen": -220.44842529296875, "logps/ref_chosen": -69.72691345214844, "logps/ref_rejected": -103.32135009765625, "logps/rejected": -355.7283630371094, "loss": 1.1507, "margin_dpo/margin_mean": 101.68548583984375, "margin_dpo/margin_std": 175.06744384765625, "step": 192 }, { "KL/chosen_KL_mean": -148.80224609375, "KL/mean": -178.2123260498047, "KL/rejected_KL_mean": -207.62240600585938, "KL/std": 128.90753173828125, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.0035294066183269024, "fcm_dpo/delta": 0.06064280495047569, "fcm_dpo/margin": 58.82013702392578, "fcm_dpo/q_t": 0.45174944400787354, "grad_norm": 27.31705665588379, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -0.24892589449882507, "logits/rejected": -0.23975345492362976, "logps/chosen": -208.99273681640625, "logps/ref_chosen": -60.19049835205078, "logps/ref_rejected": -76.40755462646484, "logps/rejected": -284.02996826171875, "loss": 1.2674, "margin_dpo/margin_mean": 58.82013702392578, "margin_dpo/margin_std": 162.4898681640625, "step": 193 }, { "KL/chosen_KL_mean": -91.62898254394531, "KL/mean": -139.33377075195312, "KL/rejected_KL_mean": -187.03854370117188, "KL/std": 104.66587829589844, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.003725461196154356, "fcm_dpo/delta": 0.053960170596838, "fcm_dpo/margin": 95.4095687866211, "fcm_dpo/q_t": 0.4181906580924988, "grad_norm": 16.410791397094727, "learning_rate": 4.510405240853854e-07, "logits/chosen": -0.19228887557983398, "logits/rejected": -0.1776125431060791, "logps/chosen": -129.4693603515625, "logps/ref_chosen": -37.84037399291992, "logps/ref_rejected": -60.684783935546875, "logps/rejected": -247.72332763671875, "loss": 1.1084, "margin_dpo/margin_mean": 95.40956115722656, "margin_dpo/margin_std": 115.09669494628906, "step": 194 }, { "KL/chosen_KL_mean": -141.21636962890625, "KL/mean": -192.36294555664062, "KL/rejected_KL_mean": -243.50949096679688, "KL/std": 128.15513610839844, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0038534309715032578, "fcm_dpo/delta": 0.007251395843923092, "fcm_dpo/margin": 102.29312133789062, "fcm_dpo/q_t": 0.40964868664741516, "grad_norm": 27.93120765686035, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -0.26708340644836426, "logits/rejected": -0.285901814699173, "logps/chosen": -196.10794067382812, "logps/ref_chosen": -54.891571044921875, "logps/ref_rejected": -96.77095794677734, "logps/rejected": -340.28045654296875, "loss": 1.0919, "margin_dpo/margin_mean": 102.29312133789062, "margin_dpo/margin_std": 133.5623779296875, "step": 195 }, { "KL/chosen_KL_mean": -111.13381958007812, "KL/mean": -168.23362731933594, "KL/rejected_KL_mean": -225.33343505859375, "KL/std": 128.486083984375, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.0036738794296979904, "fcm_dpo/delta": -0.02965060994029045, "fcm_dpo/margin": 114.1996078491211, "fcm_dpo/q_t": 0.40418341755867004, "grad_norm": 19.077123641967773, "learning_rate": 4.495043068200599e-07, "logits/chosen": -0.2980375289916992, "logits/rejected": -0.2835182845592499, "logps/chosen": -164.37905883789062, "logps/ref_chosen": -53.245243072509766, "logps/ref_rejected": -76.05294799804688, "logps/rejected": -301.3863830566406, "loss": 1.0895, "margin_dpo/margin_mean": 114.19961547851562, "margin_dpo/margin_std": 154.56077575683594, "step": 196 }, { "KL/chosen_KL_mean": -110.27510833740234, "KL/mean": -154.23858642578125, "KL/rejected_KL_mean": -198.20204162597656, "KL/std": 109.70982360839844, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.003983784466981888, "fcm_dpo/delta": 0.05786813423037529, "fcm_dpo/margin": 87.92694091796875, "fcm_dpo/q_t": 0.4213848114013672, "grad_norm": 21.03490447998047, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -0.28392934799194336, "logits/rejected": -0.27791300415992737, "logps/chosen": -170.6954345703125, "logps/ref_chosen": -60.42033386230469, "logps/ref_rejected": -77.20890808105469, "logps/rejected": -275.41094970703125, "loss": 1.125, "margin_dpo/margin_mean": 87.92694091796875, "margin_dpo/margin_std": 125.18641662597656, "step": 197 }, { "KL/chosen_KL_mean": -122.45361328125, "KL/mean": -180.43850708007812, "KL/rejected_KL_mean": -238.42340087890625, "KL/std": 138.5358428955078, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.0038145678117871284, "fcm_dpo/delta": -0.054569028317928314, "fcm_dpo/margin": 115.96977996826172, "fcm_dpo/q_t": 0.399750292301178, "grad_norm": 20.83743667602539, "learning_rate": 4.479470611971645e-07, "logits/chosen": -0.32312026619911194, "logits/rejected": -0.322989821434021, "logps/chosen": -177.48980712890625, "logps/ref_chosen": -55.03618621826172, "logps/ref_rejected": -97.24325561523438, "logps/rejected": -335.6666564941406, "loss": 1.0714, "margin_dpo/margin_mean": 115.96978759765625, "margin_dpo/margin_std": 157.2657470703125, "step": 198 }, { "KL/chosen_KL_mean": -116.48646545410156, "KL/mean": -172.57876586914062, "KL/rejected_KL_mean": -228.67108154296875, "KL/std": 122.16108703613281, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.0036303927190601826, "fcm_dpo/delta": -0.012894738465547562, "fcm_dpo/margin": 112.18461608886719, "fcm_dpo/q_t": 0.4059009552001953, "grad_norm": 20.86189079284668, "learning_rate": 4.471606039587695e-07, "logits/chosen": -0.3357727527618408, "logits/rejected": -0.3183862566947937, "logps/chosen": -173.31529235839844, "logps/ref_chosen": -56.828826904296875, "logps/ref_rejected": -84.64820861816406, "logps/rejected": -313.31927490234375, "loss": 1.094, "margin_dpo/margin_mean": 112.18460845947266, "margin_dpo/margin_std": 152.09129333496094, "step": 199 }, { "KL/chosen_KL_mean": -113.27946472167969, "KL/mean": -171.24513244628906, "KL/rejected_KL_mean": -229.2107696533203, "KL/std": 133.42398071289062, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.0035990295000374317, "fcm_dpo/delta": -0.02324402891099453, "fcm_dpo/margin": 115.93130493164062, "fcm_dpo/q_t": 0.40560150146484375, "grad_norm": 18.164777755737305, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -0.2667478621006012, "logits/rejected": -0.24850307404994965, "logps/chosen": -166.34652709960938, "logps/ref_chosen": -53.06706237792969, "logps/ref_rejected": -80.60843658447266, "logps/rejected": -309.8192138671875, "loss": 1.0995, "margin_dpo/margin_mean": 115.93130493164062, "margin_dpo/margin_std": 171.93081665039062, "step": 200 }, { "KL/chosen_KL_mean": -123.16534423828125, "KL/mean": -179.07032775878906, "KL/rejected_KL_mean": -234.97531127929688, "KL/std": 142.31027221679688, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.003661088179796934, "fcm_dpo/delta": -0.012851493433117867, "fcm_dpo/margin": 111.80996704101562, "fcm_dpo/q_t": 0.40664464235305786, "grad_norm": 20.639188766479492, "learning_rate": 4.455721242469372e-07, "logits/chosen": -0.37277939915657043, "logits/rejected": -0.3691471219062805, "logps/chosen": -198.56756591796875, "logps/ref_chosen": -75.4022216796875, "logps/ref_rejected": -114.80821990966797, "logps/rejected": -349.78350830078125, "loss": 1.1012, "margin_dpo/margin_mean": 111.80996704101562, "margin_dpo/margin_std": 161.63238525390625, "step": 201 }, { "KL/chosen_KL_mean": -127.29255676269531, "KL/mean": -166.71469116210938, "KL/rejected_KL_mean": -206.13681030273438, "KL/std": 124.88301849365234, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.003907524980604649, "fcm_dpo/delta": 0.10762214660644531, "fcm_dpo/margin": 78.84428405761719, "fcm_dpo/q_t": 0.4342569410800934, "grad_norm": 23.870006561279297, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -0.28092265129089355, "logits/rejected": -0.29571932554244995, "logps/chosen": -177.3938751220703, "logps/ref_chosen": -50.101318359375, "logps/ref_rejected": -86.98503112792969, "logps/rejected": -293.1218566894531, "loss": 1.1957, "margin_dpo/margin_mean": 78.84428405761719, "margin_dpo/margin_std": 163.82876586914062, "step": 202 }, { "KL/chosen_KL_mean": -127.91600036621094, "KL/mean": -176.91217041015625, "KL/rejected_KL_mean": -225.90835571289062, "KL/std": 129.85752868652344, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.003999835811555386, "fcm_dpo/delta": 0.009929049760103226, "fcm_dpo/margin": 97.99237060546875, "fcm_dpo/q_t": 0.4120560884475708, "grad_norm": 20.524028778076172, "learning_rate": 4.439630306414758e-07, "logits/chosen": -0.31167030334472656, "logits/rejected": -0.3005950450897217, "logps/chosen": -188.52569580078125, "logps/ref_chosen": -60.60969543457031, "logps/ref_rejected": -85.89596557617188, "logps/rejected": -311.8043212890625, "loss": 1.1109, "margin_dpo/margin_mean": 97.99237060546875, "margin_dpo/margin_std": 146.369140625, "step": 203 }, { "KL/chosen_KL_mean": -140.44757080078125, "KL/mean": -185.04940795898438, "KL/rejected_KL_mean": -229.6512451171875, "KL/std": 140.68167114257812, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.0041489386931061745, "fcm_dpo/delta": 0.03660349175333977, "fcm_dpo/margin": 89.20367431640625, "fcm_dpo/q_t": 0.4208757281303406, "grad_norm": 35.41379928588867, "learning_rate": 4.431508065452897e-07, "logits/chosen": -0.3780588209629059, "logits/rejected": -0.33705025911331177, "logps/chosen": -220.61251831054688, "logps/ref_chosen": -80.16496276855469, "logps/ref_rejected": -87.69590759277344, "logps/rejected": -317.34716796875, "loss": 1.1626, "margin_dpo/margin_mean": 89.20367431640625, "margin_dpo/margin_std": 168.600341796875, "step": 204 }, { "KL/chosen_KL_mean": -134.366455078125, "KL/mean": -193.7669677734375, "KL/rejected_KL_mean": -253.16749572753906, "KL/std": 141.5926513671875, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.0037632733583450317, "fcm_dpo/delta": -0.07647587358951569, "fcm_dpo/margin": 118.80104064941406, "fcm_dpo/q_t": 0.3929186463356018, "grad_norm": 22.13498878479004, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -0.3193209767341614, "logits/rejected": -0.2915324568748474, "logps/chosen": -193.75119018554688, "logps/ref_chosen": -59.384735107421875, "logps/ref_rejected": -85.12505340576172, "logps/rejected": -338.29254150390625, "loss": 1.0786, "margin_dpo/margin_mean": 118.80104064941406, "margin_dpo/margin_std": 156.5557861328125, "step": 205 }, { "KL/chosen_KL_mean": -120.28572082519531, "KL/mean": -186.2382354736328, "KL/rejected_KL_mean": -252.1907501220703, "KL/std": 128.577880859375, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.003537021577358246, "fcm_dpo/delta": -0.0920427143573761, "fcm_dpo/margin": 131.90501403808594, "fcm_dpo/q_t": 0.3862435817718506, "grad_norm": 23.741201400756836, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.2517659068107605, "logits/rejected": -0.2520918548107147, "logps/chosen": -167.25021362304688, "logps/ref_chosen": -46.964500427246094, "logps/ref_rejected": -98.9534912109375, "logps/rejected": -351.14422607421875, "loss": 1.0313, "margin_dpo/margin_mean": 131.90501403808594, "margin_dpo/margin_std": 139.59078979492188, "step": 206 }, { "KL/chosen_KL_mean": -114.99893188476562, "KL/mean": -187.25506591796875, "KL/rejected_KL_mean": -259.5111999511719, "KL/std": 148.06365966796875, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.003286686260253191, "fcm_dpo/delta": -0.09856441617012024, "fcm_dpo/margin": 144.51226806640625, "fcm_dpo/q_t": 0.38960766792297363, "grad_norm": 21.21698760986328, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -0.35248827934265137, "logits/rejected": -0.329440712928772, "logps/chosen": -171.05519104003906, "logps/ref_chosen": -56.05625915527344, "logps/ref_rejected": -84.44779968261719, "logps/rejected": -343.958984375, "loss": 1.0401, "margin_dpo/margin_mean": 144.51226806640625, "margin_dpo/margin_std": 174.37086486816406, "step": 207 }, { "KL/chosen_KL_mean": -171.49325561523438, "KL/mean": -230.3494110107422, "KL/rejected_KL_mean": -289.20556640625, "KL/std": 144.81240844726562, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.0032218745909631252, "fcm_dpo/delta": 0.02522367238998413, "fcm_dpo/margin": 117.71229553222656, "fcm_dpo/q_t": 0.41357824206352234, "grad_norm": 22.431468963623047, "learning_rate": 4.398512291636768e-07, "logits/chosen": -0.3535154461860657, "logits/rejected": -0.3377763032913208, "logps/chosen": -238.5608673095703, "logps/ref_chosen": -67.06761169433594, "logps/ref_rejected": -94.28689575195312, "logps/rejected": -383.4924621582031, "loss": 1.1181, "margin_dpo/margin_mean": 117.71229553222656, "margin_dpo/margin_std": 175.20639038085938, "step": 208 }, { "KL/chosen_KL_mean": -144.38653564453125, "KL/mean": -197.31893920898438, "KL/rejected_KL_mean": -250.25132751464844, "KL/std": 131.18588256835938, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.003419704968109727, "fcm_dpo/delta": 0.04580961540341377, "fcm_dpo/margin": 105.86478424072266, "fcm_dpo/q_t": 0.41855117678642273, "grad_norm": 19.670215606689453, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -0.231040820479393, "logits/rejected": -0.2151014506816864, "logps/chosen": -200.5682373046875, "logps/ref_chosen": -56.18169403076172, "logps/ref_rejected": -80.94152069091797, "logps/rejected": -331.1928405761719, "loss": 1.1333, "margin_dpo/margin_mean": 105.86478424072266, "margin_dpo/margin_std": 164.6919708251953, "step": 209 }, { "KL/chosen_KL_mean": -127.3232192993164, "KL/mean": -187.63894653320312, "KL/rejected_KL_mean": -247.95468139648438, "KL/std": 131.5801544189453, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.0034523247741162777, "fcm_dpo/delta": -0.0221034474670887, "fcm_dpo/margin": 120.6314697265625, "fcm_dpo/q_t": 0.4037366807460785, "grad_norm": 20.2919979095459, "learning_rate": 4.381713366536311e-07, "logits/chosen": -0.28856372833251953, "logits/rejected": -0.2808060348033905, "logps/chosen": -173.69503784179688, "logps/ref_chosen": -46.371822357177734, "logps/ref_rejected": -76.68162536621094, "logps/rejected": -324.63629150390625, "loss": 1.0825, "margin_dpo/margin_mean": 120.6314697265625, "margin_dpo/margin_std": 156.716552734375, "step": 210 }, { "KL/chosen_KL_mean": -186.89505004882812, "KL/mean": -239.9453125, "KL/rejected_KL_mean": -292.99560546875, "KL/std": 155.1609649658203, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.003472290001809597, "fcm_dpo/delta": 0.038492538034915924, "fcm_dpo/margin": 106.10052490234375, "fcm_dpo/q_t": 0.42122966051101685, "grad_norm": 28.37523078918457, "learning_rate": 4.373239415645323e-07, "logits/chosen": -0.2967555522918701, "logits/rejected": -0.2570844888687134, "logps/chosen": -265.827392578125, "logps/ref_chosen": -78.93235778808594, "logps/ref_rejected": -86.82098388671875, "logps/rejected": -379.81658935546875, "loss": 1.1436, "margin_dpo/margin_mean": 106.10053253173828, "margin_dpo/margin_std": 179.58560180664062, "step": 211 }, { "KL/chosen_KL_mean": -154.07533264160156, "KL/mean": -228.19821166992188, "KL/rejected_KL_mean": -302.32110595703125, "KL/std": 162.68435668945312, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.0031211567111313343, "fcm_dpo/delta": -0.09513364732265472, "fcm_dpo/margin": 148.24575805664062, "fcm_dpo/q_t": 0.3897473216056824, "grad_norm": 25.847610473632812, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -0.2827294170856476, "logits/rejected": -0.2756040692329407, "logps/chosen": -212.2723388671875, "logps/ref_chosen": -58.19701385498047, "logps/ref_rejected": -103.05785369873047, "logps/rejected": -405.3789367675781, "loss": 1.0549, "margin_dpo/margin_mean": 148.24575805664062, "margin_dpo/margin_std": 177.65496826171875, "step": 212 }, { "KL/chosen_KL_mean": -155.3287811279297, "KL/mean": -228.24905395507812, "KL/rejected_KL_mean": -301.16937255859375, "KL/std": 149.9835205078125, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.0030004819855093956, "fcm_dpo/delta": -0.05112471804022789, "fcm_dpo/margin": 145.84056091308594, "fcm_dpo/q_t": 0.3968704044818878, "grad_norm": 19.53304672241211, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -0.25978952646255493, "logits/rejected": -0.23081305623054504, "logps/chosen": -222.84149169921875, "logps/ref_chosen": -67.51271057128906, "logps/ref_rejected": -93.91471862792969, "logps/rejected": -395.08404541015625, "loss": 1.061, "margin_dpo/margin_mean": 145.84056091308594, "margin_dpo/margin_std": 175.0486602783203, "step": 213 }, { "KL/chosen_KL_mean": -141.36532592773438, "KL/mean": -206.33975219726562, "KL/rejected_KL_mean": -271.31414794921875, "KL/std": 151.19796752929688, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.0030389183666557074, "fcm_dpo/delta": 0.006323062814772129, "fcm_dpo/margin": 129.9488525390625, "fcm_dpo/q_t": 0.41051051020622253, "grad_norm": 20.851654052734375, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -0.2020268589258194, "logits/rejected": -0.20587500929832458, "logps/chosen": -182.97021484375, "logps/ref_chosen": -41.604888916015625, "logps/ref_rejected": -77.51741027832031, "logps/rejected": -348.83160400390625, "loss": 1.0964, "margin_dpo/margin_mean": 129.94886779785156, "margin_dpo/margin_std": 176.67694091796875, "step": 214 }, { "KL/chosen_KL_mean": -167.13980102539062, "KL/mean": -235.6959228515625, "KL/rejected_KL_mean": -304.2520446777344, "KL/std": 154.10137939453125, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.0029814334120601416, "fcm_dpo/delta": -0.011681612581014633, "fcm_dpo/margin": 137.1122283935547, "fcm_dpo/q_t": 0.4044470191001892, "grad_norm": 21.97011947631836, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -0.2605951428413391, "logits/rejected": -0.25532659888267517, "logps/chosen": -220.4190673828125, "logps/ref_chosen": -53.279266357421875, "logps/ref_rejected": -89.96464538574219, "logps/rejected": -394.2166748046875, "loss": 1.0713, "margin_dpo/margin_mean": 137.11224365234375, "margin_dpo/margin_std": 158.41067504882812, "step": 215 }, { "KL/chosen_KL_mean": -169.673583984375, "KL/mean": -237.0709228515625, "KL/rejected_KL_mean": -304.46826171875, "KL/std": 161.4061737060547, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.003034801222383976, "fcm_dpo/delta": -0.012860134243965149, "fcm_dpo/margin": 134.794677734375, "fcm_dpo/q_t": 0.4068460166454315, "grad_norm": 28.930782318115234, "learning_rate": 4.330133748510036e-07, "logits/chosen": -0.2749112844467163, "logits/rejected": -0.260329008102417, "logps/chosen": -218.56137084960938, "logps/ref_chosen": -48.887794494628906, "logps/ref_rejected": -77.19892883300781, "logps/rejected": -381.66717529296875, "loss": 1.1026, "margin_dpo/margin_mean": 134.794677734375, "margin_dpo/margin_std": 196.2810821533203, "step": 216 }, { "KL/chosen_KL_mean": -173.36903381347656, "KL/mean": -254.24615478515625, "KL/rejected_KL_mean": -335.123291015625, "KL/std": 167.5054168701172, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.0027936077676713467, "fcm_dpo/delta": -0.06825002282857895, "fcm_dpo/margin": 161.75425720214844, "fcm_dpo/q_t": 0.39398378133773804, "grad_norm": 19.45440673828125, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -0.2623336613178253, "logits/rejected": -0.2586092948913574, "logps/chosen": -223.21434020996094, "logps/ref_chosen": -49.845306396484375, "logps/ref_rejected": -100.07832336425781, "logps/rejected": -435.20159912109375, "loss": 1.0481, "margin_dpo/margin_mean": 161.75424194335938, "margin_dpo/margin_std": 188.6632080078125, "step": 217 }, { "KL/chosen_KL_mean": -177.95050048828125, "KL/mean": -239.9977264404297, "KL/rejected_KL_mean": -302.0449523925781, "KL/std": 164.52496337890625, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.0028708684258162975, "fcm_dpo/delta": 0.05333707481622696, "fcm_dpo/margin": 124.09444427490234, "fcm_dpo/q_t": 0.42141902446746826, "grad_norm": 22.139909744262695, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -0.296860933303833, "logits/rejected": -0.29232728481292725, "logps/chosen": -236.52719116210938, "logps/ref_chosen": -58.576683044433594, "logps/ref_rejected": -87.84639739990234, "logps/rejected": -389.891357421875, "loss": 1.1373, "margin_dpo/margin_mean": 124.09444427490234, "margin_dpo/margin_std": 196.5043487548828, "step": 218 }, { "KL/chosen_KL_mean": -184.52737426757812, "KL/mean": -234.72857666015625, "KL/rejected_KL_mean": -284.92974853515625, "KL/std": 165.06085205078125, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.002932458883151412, "fcm_dpo/delta": 0.0077366819605231285, "fcm_dpo/margin": 100.40234375, "fcm_dpo/q_t": 0.4327548146247864, "grad_norm": 33.82648468017578, "learning_rate": 4.303689819449636e-07, "logits/chosen": -0.30966275930404663, "logits/rejected": -0.3071323335170746, "logps/chosen": -245.61123657226562, "logps/ref_chosen": -61.083858489990234, "logps/ref_rejected": -85.83042907714844, "logps/rejected": -370.76019287109375, "loss": 1.1977, "margin_dpo/margin_mean": 100.40234375, "margin_dpo/margin_std": 202.93426513671875, "step": 219 }, { "KL/chosen_KL_mean": -190.68328857421875, "KL/mean": -234.95462036132812, "KL/rejected_KL_mean": -279.2259216308594, "KL/std": 134.0245361328125, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.00323110050521791, "fcm_dpo/delta": 0.1334986686706543, "fcm_dpo/margin": 88.54261779785156, "fcm_dpo/q_t": 0.4363420009613037, "grad_norm": 33.53060531616211, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -0.3398175835609436, "logits/rejected": -0.31802213191986084, "logps/chosen": -260.714599609375, "logps/ref_chosen": -70.03128051757812, "logps/ref_rejected": -87.68551635742188, "logps/rejected": -366.91143798828125, "loss": 1.1692, "margin_dpo/margin_mean": 88.54261779785156, "margin_dpo/margin_std": 137.38961791992188, "step": 220 }, { "KL/chosen_KL_mean": -137.20208740234375, "KL/mean": -223.91976928710938, "KL/rejected_KL_mean": -310.637451171875, "KL/std": 157.56552124023438, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.003003270598128438, "fcm_dpo/delta": -0.16315576434135437, "fcm_dpo/margin": 173.43536376953125, "fcm_dpo/q_t": 0.37127232551574707, "grad_norm": 135.19117736816406, "learning_rate": 4.285822501755485e-07, "logits/chosen": -0.33175361156463623, "logits/rejected": -0.3400239944458008, "logps/chosen": -189.3568115234375, "logps/ref_chosen": -52.15470886230469, "logps/ref_rejected": -106.46768188476562, "logps/rejected": -417.10516357421875, "loss": 0.987, "margin_dpo/margin_mean": 173.43536376953125, "margin_dpo/margin_std": 158.20767211914062, "step": 221 }, { "KL/chosen_KL_mean": -147.8143768310547, "KL/mean": -214.55850219726562, "KL/rejected_KL_mean": -281.3026123046875, "KL/std": 146.41705322265625, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.0029258220456540585, "fcm_dpo/delta": 0.01042582280933857, "fcm_dpo/margin": 133.4882354736328, "fcm_dpo/q_t": 0.40958648920059204, "grad_norm": 62.3775520324707, "learning_rate": 4.276818137766118e-07, "logits/chosen": -0.38752639293670654, "logits/rejected": -0.3927309811115265, "logps/chosen": -208.7854766845703, "logps/ref_chosen": -60.971099853515625, "logps/ref_rejected": -100.00115203857422, "logps/rejected": -381.30377197265625, "loss": 1.0883, "margin_dpo/margin_mean": 133.4882354736328, "margin_dpo/margin_std": 164.23587036132812, "step": 222 }, { "KL/chosen_KL_mean": -165.07833862304688, "KL/mean": -222.50390625, "KL/rejected_KL_mean": -279.92950439453125, "KL/std": 146.90487670898438, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.0030144588090479374, "fcm_dpo/delta": 0.06525313854217529, "fcm_dpo/margin": 114.85115051269531, "fcm_dpo/q_t": 0.4231482148170471, "grad_norm": 22.007301330566406, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -0.2665255069732666, "logits/rejected": -0.26349782943725586, "logps/chosen": -217.71890258789062, "logps/ref_chosen": -52.64057540893555, "logps/ref_rejected": -82.82502746582031, "logps/rejected": -362.7545166015625, "loss": 1.1441, "margin_dpo/margin_mean": 114.85115051269531, "margin_dpo/margin_std": 187.14393615722656, "step": 223 }, { "KL/chosen_KL_mean": -144.54835510253906, "KL/mean": -207.91268920898438, "KL/rejected_KL_mean": -271.27703857421875, "KL/std": 162.03009033203125, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.003018029499799013, "fcm_dpo/delta": 0.017022643238306046, "fcm_dpo/margin": 126.72867584228516, "fcm_dpo/q_t": 0.41384878754615784, "grad_norm": 22.61432456970215, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -0.29570257663726807, "logits/rejected": -0.28287479281425476, "logps/chosen": -193.14376831054688, "logps/ref_chosen": -48.59541320800781, "logps/ref_rejected": -77.11648559570312, "logps/rejected": -348.39349365234375, "loss": 1.1178, "margin_dpo/margin_mean": 126.72867584228516, "margin_dpo/margin_std": 185.35171508789062, "step": 224 }, { "KL/chosen_KL_mean": -163.48281860351562, "KL/mean": -239.61199951171875, "KL/rejected_KL_mean": -315.74114990234375, "KL/std": 156.39309692382812, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.002961981575936079, "fcm_dpo/delta": -0.066794753074646, "fcm_dpo/margin": 152.25836181640625, "fcm_dpo/q_t": 0.3953250050544739, "grad_norm": 24.43362045288086, "learning_rate": 4.249525076191759e-07, "logits/chosen": -0.27601003646850586, "logits/rejected": -0.26785707473754883, "logps/chosen": -221.4832763671875, "logps/ref_chosen": -58.000465393066406, "logps/ref_rejected": -99.90291595458984, "logps/rejected": -415.64410400390625, "loss": 1.0628, "margin_dpo/margin_mean": 152.25836181640625, "margin_dpo/margin_std": 197.67453002929688, "step": 225 }, { "KL/chosen_KL_mean": -141.99343872070312, "KL/mean": -200.15957641601562, "KL/rejected_KL_mean": -258.32568359375, "KL/std": 151.08013916015625, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.002992121037095785, "fcm_dpo/delta": 0.06153475120663643, "fcm_dpo/margin": 116.332275390625, "fcm_dpo/q_t": 0.4230756461620331, "grad_norm": 27.1488037109375, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -0.33648887276649475, "logits/rejected": -0.3032319247722626, "logps/chosen": -200.89224243164062, "logps/ref_chosen": -58.898799896240234, "logps/ref_rejected": -78.68775939941406, "logps/rejected": -337.0134582519531, "loss": 1.1404, "margin_dpo/margin_mean": 116.332275390625, "margin_dpo/margin_std": 179.59532165527344, "step": 226 }, { "KL/chosen_KL_mean": -164.3446807861328, "KL/mean": -242.88470458984375, "KL/rejected_KL_mean": -321.42474365234375, "KL/std": 180.00967407226562, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.002881108084693551, "fcm_dpo/delta": -0.07316943258047104, "fcm_dpo/margin": 157.080078125, "fcm_dpo/q_t": 0.3937509059906006, "grad_norm": 23.208093643188477, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -0.31838715076446533, "logits/rejected": -0.3055545687675476, "logps/chosen": -223.41685485839844, "logps/ref_chosen": -59.072181701660156, "logps/ref_rejected": -99.41236877441406, "logps/rejected": -420.8371276855469, "loss": 1.0542, "margin_dpo/margin_mean": 157.08009338378906, "margin_dpo/margin_std": 193.092529296875, "step": 227 }, { "KL/chosen_KL_mean": -175.99685668945312, "KL/mean": -229.3104248046875, "KL/rejected_KL_mean": -282.62396240234375, "KL/std": 151.4019775390625, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.003110603429377079, "fcm_dpo/delta": 0.08030124008655548, "fcm_dpo/margin": 106.62710571289062, "fcm_dpo/q_t": 0.42634642124176025, "grad_norm": 26.213809967041016, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -0.29573142528533936, "logits/rejected": -0.28110069036483765, "logps/chosen": -241.88815307617188, "logps/ref_chosen": -65.89128875732422, "logps/ref_rejected": -91.04875183105469, "logps/rejected": -373.6727294921875, "loss": 1.1646, "margin_dpo/margin_mean": 106.62710571289062, "margin_dpo/margin_std": 190.84280395507812, "step": 228 }, { "KL/chosen_KL_mean": -185.06878662109375, "KL/mean": -240.96780395507812, "KL/rejected_KL_mean": -296.8668212890625, "KL/std": 172.71731567382812, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.003258654149249196, "fcm_dpo/delta": 0.043283235281705856, "fcm_dpo/margin": 111.79805755615234, "fcm_dpo/q_t": 0.41782456636428833, "grad_norm": 42.24066925048828, "learning_rate": 4.212490049118951e-07, "logits/chosen": -0.40128546953201294, "logits/rejected": -0.3743479251861572, "logps/chosen": -255.775146484375, "logps/ref_chosen": -70.70637512207031, "logps/ref_rejected": -84.52741241455078, "logps/rejected": -381.39422607421875, "loss": 1.1386, "margin_dpo/margin_mean": 111.79806518554688, "margin_dpo/margin_std": 184.9072265625, "step": 229 }, { "KL/chosen_KL_mean": -130.72586059570312, "KL/mean": -217.06655883789062, "KL/rejected_KL_mean": -303.4072265625, "KL/std": 151.964111328125, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.0029333364218473434, "fcm_dpo/delta": -0.14437267184257507, "fcm_dpo/margin": 172.68138122558594, "fcm_dpo/q_t": 0.37497806549072266, "grad_norm": 67.53182983398438, "learning_rate": 4.203117865141635e-07, "logits/chosen": -0.2871912717819214, "logits/rejected": -0.2930827736854553, "logps/chosen": -170.0078582763672, "logps/ref_chosen": -39.282005310058594, "logps/ref_rejected": -85.62191009521484, "logps/rejected": -389.0291442871094, "loss": 0.9974, "margin_dpo/margin_mean": 172.68138122558594, "margin_dpo/margin_std": 165.72486877441406, "step": 230 }, { "KL/chosen_KL_mean": -154.43719482421875, "KL/mean": -212.76937866210938, "KL/rejected_KL_mean": -271.1015319824219, "KL/std": 134.67486572265625, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.002986223204061389, "fcm_dpo/delta": 0.06204049289226532, "fcm_dpo/margin": 116.66436767578125, "fcm_dpo/q_t": 0.42223888635635376, "grad_norm": 29.774654388427734, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -0.36699360609054565, "logits/rejected": -0.34106624126434326, "logps/chosen": -217.713623046875, "logps/ref_chosen": -63.27644348144531, "logps/ref_rejected": -74.1239013671875, "logps/rejected": -345.2254638671875, "loss": 1.1221, "margin_dpo/margin_mean": 116.66436004638672, "margin_dpo/margin_std": 160.6752471923828, "step": 231 }, { "KL/chosen_KL_mean": -198.68173217773438, "KL/mean": -247.72219848632812, "KL/rejected_KL_mean": -296.7626953125, "KL/std": 164.42752075195312, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.003230078611522913, "fcm_dpo/delta": 0.09927871823310852, "fcm_dpo/margin": 98.08092498779297, "fcm_dpo/q_t": 0.43197596073150635, "grad_norm": 22.91642189025879, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -0.3694447875022888, "logits/rejected": -0.35070955753326416, "logps/chosen": -269.43048095703125, "logps/ref_chosen": -70.74876403808594, "logps/ref_rejected": -83.97706604003906, "logps/rejected": -380.73974609375, "loss": 1.1624, "margin_dpo/margin_mean": 98.08092498779297, "margin_dpo/margin_std": 167.9994354248047, "step": 232 }, { "KL/chosen_KL_mean": -184.214599609375, "KL/mean": -265.7254638671875, "KL/rejected_KL_mean": -347.236328125, "KL/std": 183.4271240234375, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.0030570742674171925, "fcm_dpo/delta": -0.13046759366989136, "fcm_dpo/margin": 163.021728515625, "fcm_dpo/q_t": 0.38850700855255127, "grad_norm": 26.11095428466797, "learning_rate": 4.174733034541245e-07, "logits/chosen": -0.33819979429244995, "logits/rejected": -0.3396344780921936, "logps/chosen": -239.0975341796875, "logps/ref_chosen": -54.8829345703125, "logps/ref_rejected": -107.4800796508789, "logps/rejected": -454.7164001464844, "loss": 1.0684, "margin_dpo/margin_mean": 163.021728515625, "margin_dpo/margin_std": 239.56597900390625, "step": 233 }, { "KL/chosen_KL_mean": -182.37313842773438, "KL/mean": -264.9031982421875, "KL/rejected_KL_mean": -347.43328857421875, "KL/std": 161.71844482421875, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.0026757342275232077, "fcm_dpo/delta": -0.06500078737735748, "fcm_dpo/margin": 165.0601348876953, "fcm_dpo/q_t": 0.3940809965133667, "grad_norm": 23.45154571533203, "learning_rate": 4.165182829193126e-07, "logits/chosen": -0.3571465015411377, "logits/rejected": -0.3816158175468445, "logps/chosen": -226.4676513671875, "logps/ref_chosen": -44.094520568847656, "logps/ref_rejected": -100.00663757324219, "logps/rejected": -447.4399108886719, "loss": 1.0517, "margin_dpo/margin_mean": 165.0601348876953, "margin_dpo/margin_std": 179.3123779296875, "step": 234 }, { "KL/chosen_KL_mean": -224.09396362304688, "KL/mean": -279.3703308105469, "KL/rejected_KL_mean": -334.64666748046875, "KL/std": 163.57676696777344, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.0029698603320866823, "fcm_dpo/delta": 0.08258074522018433, "fcm_dpo/margin": 110.55270385742188, "fcm_dpo/q_t": 0.42682477831840515, "grad_norm": 27.62843894958496, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -0.34939903020858765, "logits/rejected": -0.33593958616256714, "logps/chosen": -286.3318786621094, "logps/ref_chosen": -62.237911224365234, "logps/ref_rejected": -90.39506530761719, "logps/rejected": -425.041748046875, "loss": 1.1655, "margin_dpo/margin_mean": 110.55270385742188, "margin_dpo/margin_std": 197.34738159179688, "step": 235 }, { "KL/chosen_KL_mean": -168.29141235351562, "KL/mean": -257.59796142578125, "KL/rejected_KL_mean": -346.9044494628906, "KL/std": 175.53897094726562, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.002734276931732893, "fcm_dpo/delta": -0.11726871132850647, "fcm_dpo/margin": 178.61306762695312, "fcm_dpo/q_t": 0.3805997967720032, "grad_norm": 28.633899688720703, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -0.33454635739326477, "logits/rejected": -0.337698370218277, "logps/chosen": -217.6327667236328, "logps/ref_chosen": -49.34136199951172, "logps/ref_rejected": -103.51162719726562, "logps/rejected": -450.41607666015625, "loss": 1.0055, "margin_dpo/margin_mean": 178.61306762695312, "margin_dpo/margin_std": 166.25515747070312, "step": 236 }, { "KL/chosen_KL_mean": -238.2225341796875, "KL/mean": -303.3404541015625, "KL/rejected_KL_mean": -368.45843505859375, "KL/std": 182.74667358398438, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.0027634776197373867, "fcm_dpo/delta": 0.04837137833237648, "fcm_dpo/margin": 130.23587036132812, "fcm_dpo/q_t": 0.4194791316986084, "grad_norm": 28.578073501586914, "learning_rate": 4.136269950853473e-07, "logits/chosen": -0.3175830841064453, "logits/rejected": -0.31506621837615967, "logps/chosen": -292.3906555175781, "logps/ref_chosen": -54.168121337890625, "logps/ref_rejected": -94.78036499023438, "logps/rejected": -463.23876953125, "loss": 1.1321, "margin_dpo/margin_mean": 130.23587036132812, "margin_dpo/margin_std": 204.54656982421875, "step": 237 }, { "KL/chosen_KL_mean": -218.0069580078125, "KL/mean": -284.595458984375, "KL/rejected_KL_mean": -351.18402099609375, "KL/std": 181.19268798828125, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.002782113617286086, "fcm_dpo/delta": 0.034211453050374985, "fcm_dpo/margin": 133.17703247070312, "fcm_dpo/q_t": 0.41864013671875, "grad_norm": 27.10376739501953, "learning_rate": 4.126545750510605e-07, "logits/chosen": -0.2829251289367676, "logits/rejected": -0.29289430379867554, "logps/chosen": -271.9801025390625, "logps/ref_chosen": -53.973121643066406, "logps/ref_rejected": -89.41795349121094, "logps/rejected": -440.6019592285156, "loss": 1.1244, "margin_dpo/margin_mean": 133.17703247070312, "margin_dpo/margin_std": 202.1953582763672, "step": 238 }, { "KL/chosen_KL_mean": -232.07142639160156, "KL/mean": -307.0456848144531, "KL/rejected_KL_mean": -382.0199279785156, "KL/std": 173.7324981689453, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.002740685362368822, "fcm_dpo/delta": -0.020037677139043808, "fcm_dpo/margin": 149.948486328125, "fcm_dpo/q_t": 0.4032841622829437, "grad_norm": 26.712921142578125, "learning_rate": 4.116778689174514e-07, "logits/chosen": -0.308989942073822, "logits/rejected": -0.3054299056529999, "logps/chosen": -290.16925048828125, "logps/ref_chosen": -58.09782409667969, "logps/ref_rejected": -93.59294128417969, "logps/rejected": -475.61285400390625, "loss": 1.0861, "margin_dpo/margin_mean": 149.948486328125, "margin_dpo/margin_std": 189.98800659179688, "step": 239 }, { "KL/chosen_KL_mean": -229.48007202148438, "KL/mean": -289.2586669921875, "KL/rejected_KL_mean": -349.0372314453125, "KL/std": 180.47412109375, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.0029091420583426952, "fcm_dpo/delta": 0.06307174265384674, "fcm_dpo/margin": 119.55715942382812, "fcm_dpo/q_t": 0.4230736494064331, "grad_norm": 33.61320114135742, "learning_rate": 4.106969024216348e-07, "logits/chosen": -0.33556127548217773, "logits/rejected": -0.3180205523967743, "logps/chosen": -290.09454345703125, "logps/ref_chosen": -60.6144905090332, "logps/ref_rejected": -74.1185302734375, "logps/rejected": -423.15576171875, "loss": 1.164, "margin_dpo/margin_mean": 119.55715942382812, "margin_dpo/margin_std": 214.35040283203125, "step": 240 }, { "KL/chosen_KL_mean": -187.94955444335938, "KL/mean": -281.572998046875, "KL/rejected_KL_mean": -375.19647216796875, "KL/std": 189.39794921875, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.0027145131025463343, "fcm_dpo/delta": -0.1450301706790924, "fcm_dpo/margin": 187.2469024658203, "fcm_dpo/q_t": 0.37965553998947144, "grad_norm": 25.904090881347656, "learning_rate": 4.097117014129903e-07, "logits/chosen": -0.3710273504257202, "logits/rejected": -0.35753804445266724, "logps/chosen": -254.04063415527344, "logps/ref_chosen": -66.091064453125, "logps/ref_rejected": -88.06088256835938, "logps/rejected": -463.25738525390625, "loss": 1.0206, "margin_dpo/margin_mean": 187.24691772460938, "margin_dpo/margin_std": 215.96273803710938, "step": 241 }, { "KL/chosen_KL_mean": -197.42364501953125, "KL/mean": -263.04095458984375, "KL/rejected_KL_mean": -328.6582336425781, "KL/std": 158.09832763671875, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.002751118503510952, "fcm_dpo/delta": 0.04367631673812866, "fcm_dpo/margin": 131.2345428466797, "fcm_dpo/q_t": 0.418499618768692, "grad_norm": 23.890216827392578, "learning_rate": 4.087222918524807e-07, "logits/chosen": -0.3082933723926544, "logits/rejected": -0.2922123968601227, "logps/chosen": -265.2875671386719, "logps/ref_chosen": -67.86392974853516, "logps/ref_rejected": -83.36033630371094, "logps/rejected": -412.0185546875, "loss": 1.1207, "margin_dpo/margin_mean": 131.2345428466797, "margin_dpo/margin_std": 186.8716278076172, "step": 242 }, { "KL/chosen_KL_mean": -195.4427032470703, "KL/mean": -275.4404296875, "KL/rejected_KL_mean": -355.43817138671875, "KL/std": 174.02914428710938, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.0026427144184708595, "fcm_dpo/delta": -0.029283707961440086, "fcm_dpo/margin": 159.99542236328125, "fcm_dpo/q_t": 0.4009614586830139, "grad_norm": 22.617460250854492, "learning_rate": 4.07728699811968e-07, "logits/chosen": -0.3353385329246521, "logits/rejected": -0.31745070219039917, "logps/chosen": -258.5269470214844, "logps/ref_chosen": -63.0842399597168, "logps/ref_rejected": -76.33563232421875, "logps/rejected": -431.7738037109375, "loss": 1.0667, "margin_dpo/margin_mean": 159.99542236328125, "margin_dpo/margin_std": 191.55514526367188, "step": 243 }, { "KL/chosen_KL_mean": -166.66122436523438, "KL/mean": -250.8197021484375, "KL/rejected_KL_mean": -334.9781799316406, "KL/std": 169.25035095214844, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.002540828660130501, "fcm_dpo/delta": -0.0361553393304348, "fcm_dpo/margin": 168.3169403076172, "fcm_dpo/q_t": 0.3987560272216797, "grad_norm": 25.22274398803711, "learning_rate": 4.067309514735267e-07, "logits/chosen": -0.409039705991745, "logits/rejected": -0.4093121290206909, "logps/chosen": -227.80191040039062, "logps/ref_chosen": -61.140689849853516, "logps/ref_rejected": -94.89193725585938, "logps/rejected": -429.8701171875, "loss": 1.0496, "margin_dpo/margin_mean": 168.3169403076172, "margin_dpo/margin_std": 171.68531799316406, "step": 244 }, { "KL/chosen_KL_mean": -197.50634765625, "KL/mean": -263.1775817871094, "KL/rejected_KL_mean": -328.84881591796875, "KL/std": 168.23464965820312, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.0024484535679221153, "fcm_dpo/delta": -0.03805915266275406, "fcm_dpo/margin": 131.34243774414062, "fcm_dpo/q_t": 0.4226040244102478, "grad_norm": 26.150102615356445, "learning_rate": 4.057290731287531e-07, "logits/chosen": -0.36390721797943115, "logits/rejected": -0.351279079914093, "logps/chosen": -264.76861572265625, "logps/ref_chosen": -67.26228332519531, "logps/ref_rejected": -87.64010620117188, "logps/rejected": -416.4888916015625, "loss": 1.1443, "margin_dpo/margin_mean": 131.3424530029297, "margin_dpo/margin_std": 187.38204956054688, "step": 245 }, { "KL/chosen_KL_mean": -202.00189208984375, "KL/mean": -270.38726806640625, "KL/rejected_KL_mean": -338.77264404296875, "KL/std": 191.9010009765625, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.00253550847992301, "fcm_dpo/delta": 0.0644979327917099, "fcm_dpo/margin": 136.77073669433594, "fcm_dpo/q_t": 0.42329055070877075, "grad_norm": 24.134305953979492, "learning_rate": 4.047230911780736e-07, "logits/chosen": -0.394417405128479, "logits/rejected": -0.3600098490715027, "logps/chosen": -268.6988525390625, "logps/ref_chosen": -66.69696807861328, "logps/ref_rejected": -84.34634399414062, "logps/rejected": -423.1189880371094, "loss": 1.1347, "margin_dpo/margin_mean": 136.770751953125, "margin_dpo/margin_std": 207.49807739257812, "step": 246 }, { "KL/chosen_KL_mean": -252.14999389648438, "KL/mean": -353.76324462890625, "KL/rejected_KL_mean": -455.37652587890625, "KL/std": 205.67657470703125, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.002377366181463003, "fcm_dpo/delta": -0.11207044124603271, "fcm_dpo/margin": 203.22650146484375, "fcm_dpo/q_t": 0.38493090867996216, "grad_norm": 25.502174377441406, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -0.320733904838562, "logits/rejected": -0.31779003143310547, "logps/chosen": -308.7553405761719, "logps/ref_chosen": -56.6053466796875, "logps/ref_rejected": -106.29326629638672, "logps/rejected": -561.6697998046875, "loss": 1.0303, "margin_dpo/margin_mean": 203.22650146484375, "margin_dpo/margin_std": 228.47640991210938, "step": 247 }, { "KL/chosen_KL_mean": -224.57611083984375, "KL/mean": -309.53057861328125, "KL/rejected_KL_mean": -394.4850158691406, "KL/std": 169.4248046875, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.0023004484828561544, "fcm_dpo/delta": 0.009408872574567795, "fcm_dpo/margin": 169.908935546875, "fcm_dpo/q_t": 0.4071027934551239, "grad_norm": 27.232206344604492, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -0.2981037199497223, "logits/rejected": -0.3177593946456909, "logps/chosen": -268.61932373046875, "logps/ref_chosen": -44.043216705322266, "logps/ref_rejected": -91.85687255859375, "logps/rejected": -486.3419189453125, "loss": 1.0721, "margin_dpo/margin_mean": 169.908935546875, "margin_dpo/margin_std": 169.20985412597656, "step": 248 }, { "KL/chosen_KL_mean": -287.9716796875, "KL/mean": -332.658203125, "KL/rejected_KL_mean": -377.3447570800781, "KL/std": 193.0209197998047, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.0025631722528487444, "fcm_dpo/delta": 0.08395271003246307, "fcm_dpo/margin": 89.3730697631836, "fcm_dpo/q_t": 0.45033693313598633, "grad_norm": 45.21783447265625, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -0.35276949405670166, "logits/rejected": -0.33473479747772217, "logps/chosen": -350.4140625, "logps/ref_chosen": -62.442352294921875, "logps/ref_rejected": -80.46806335449219, "logps/rejected": -457.81280517578125, "loss": 1.2555, "margin_dpo/margin_mean": 89.37306213378906, "margin_dpo/margin_std": 233.79147338867188, "step": 249 }, { "KL/chosen_KL_mean": -257.8434143066406, "KL/mean": -340.539794921875, "KL/rejected_KL_mean": -423.2361755371094, "KL/std": 185.04319763183594, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.0025648935697972775, "fcm_dpo/delta": -0.032477494329214096, "fcm_dpo/margin": 165.39276123046875, "fcm_dpo/q_t": 0.40042954683303833, "grad_norm": 49.331077575683594, "learning_rate": 4.006586590948141e-07, "logits/chosen": -0.35356414318084717, "logits/rejected": -0.31186503171920776, "logps/chosen": -323.4801025390625, "logps/ref_chosen": -65.63668823242188, "logps/ref_rejected": -73.87184143066406, "logps/rejected": -497.1080322265625, "loss": 1.0711, "margin_dpo/margin_mean": 165.39276123046875, "margin_dpo/margin_std": 199.77915954589844, "step": 250 }, { "KL/chosen_KL_mean": -263.6090393066406, "KL/mean": -316.90277099609375, "KL/rejected_KL_mean": -370.1965026855469, "KL/std": 189.91702270507812, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.002717065392062068, "fcm_dpo/delta": 0.12958815693855286, "fcm_dpo/margin": 106.58744812011719, "fcm_dpo/q_t": 0.43745461106300354, "grad_norm": 40.08333969116211, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -0.32426249980926514, "logits/rejected": -0.2984544336795807, "logps/chosen": -320.791748046875, "logps/ref_chosen": -57.182716369628906, "logps/ref_rejected": -77.66343688964844, "logps/rejected": -447.85992431640625, "loss": 1.1986, "margin_dpo/margin_mean": 106.58744812011719, "margin_dpo/margin_std": 211.80274963378906, "step": 251 }, { "KL/chosen_KL_mean": -247.48324584960938, "KL/mean": -321.72723388671875, "KL/rejected_KL_mean": -395.97125244140625, "KL/std": 152.3659210205078, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.0027652715798467398, "fcm_dpo/delta": -0.014062562957406044, "fcm_dpo/margin": 148.48800659179688, "fcm_dpo/q_t": 0.4011853337287903, "grad_norm": 32.023529052734375, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -0.4131581485271454, "logits/rejected": -0.37648361921310425, "logps/chosen": -319.16888427734375, "logps/ref_chosen": -71.68563842773438, "logps/ref_rejected": -84.75799560546875, "logps/rejected": -480.729248046875, "loss": 1.06, "margin_dpo/margin_mean": 148.48800659179688, "margin_dpo/margin_std": 151.195068359375, "step": 252 }, { "KL/chosen_KL_mean": -203.74603271484375, "KL/mean": -282.53802490234375, "KL/rejected_KL_mean": -361.3300476074219, "KL/std": 168.71078491210938, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.002772426698356867, "fcm_dpo/delta": -0.049864716827869415, "fcm_dpo/margin": 157.58399963378906, "fcm_dpo/q_t": 0.3958263397216797, "grad_norm": 24.330591201782227, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -0.42228442430496216, "logits/rejected": -0.4147264063358307, "logps/chosen": -272.8799743652344, "logps/ref_chosen": -69.1339340209961, "logps/ref_rejected": -98.70252990722656, "logps/rejected": -460.0325927734375, "loss": 1.0635, "margin_dpo/margin_mean": 157.58401489257812, "margin_dpo/margin_std": 181.4701690673828, "step": 253 }, { "KL/chosen_KL_mean": -186.63661193847656, "KL/mean": -252.297119140625, "KL/rejected_KL_mean": -317.9576416015625, "KL/std": 167.77099609375, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.0027375142090022564, "fcm_dpo/delta": 0.049500077962875366, "fcm_dpo/margin": 131.32101440429688, "fcm_dpo/q_t": 0.4207710027694702, "grad_norm": 25.899538040161133, "learning_rate": 3.965307091713037e-07, "logits/chosen": -0.38440778851509094, "logits/rejected": -0.37307465076446533, "logps/chosen": -240.79161071777344, "logps/ref_chosen": -54.154998779296875, "logps/ref_rejected": -90.30764770507812, "logps/rejected": -408.2652587890625, "loss": 1.1333, "margin_dpo/margin_mean": 131.32101440429688, "margin_dpo/margin_std": 208.37350463867188, "step": 254 }, { "KL/chosen_KL_mean": -187.1531982421875, "KL/mean": -254.55125427246094, "KL/rejected_KL_mean": -321.9493103027344, "KL/std": 147.3501739501953, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.002789452439174056, "fcm_dpo/delta": 0.02846667543053627, "fcm_dpo/margin": 134.7960968017578, "fcm_dpo/q_t": 0.41337624192237854, "grad_norm": 26.342166900634766, "learning_rate": 3.954890003969163e-07, "logits/chosen": -0.3332477807998657, "logits/rejected": -0.3225042521953583, "logps/chosen": -244.29486083984375, "logps/ref_chosen": -57.14167022705078, "logps/ref_rejected": -90.2085952758789, "logps/rejected": -412.15789794921875, "loss": 1.1121, "margin_dpo/margin_mean": 134.7960968017578, "margin_dpo/margin_std": 186.9753875732422, "step": 255 }, { "KL/chosen_KL_mean": -162.30020141601562, "KL/mean": -233.31613159179688, "KL/rejected_KL_mean": -304.33203125, "KL/std": 154.13583374023438, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.0028459273744374514, "fcm_dpo/delta": -0.0053253816440701485, "fcm_dpo/margin": 142.03182983398438, "fcm_dpo/q_t": 0.40578973293304443, "grad_norm": 36.771114349365234, "learning_rate": 3.944434578520628e-07, "logits/chosen": -0.30460691452026367, "logits/rejected": -0.3128143548965454, "logps/chosen": -217.4636993408203, "logps/ref_chosen": -55.163490295410156, "logps/ref_rejected": -92.56291961669922, "logps/rejected": -396.89495849609375, "loss": 1.0809, "margin_dpo/margin_mean": 142.03182983398438, "margin_dpo/margin_std": 175.64019775390625, "step": 256 }, { "KL/chosen_KL_mean": -163.07763671875, "KL/mean": -238.74008178710938, "KL/rejected_KL_mean": -314.40252685546875, "KL/std": 163.1609649658203, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.002718387870118022, "fcm_dpo/delta": -0.019889619201421738, "fcm_dpo/margin": 151.32489013671875, "fcm_dpo/q_t": 0.4035347104072571, "grad_norm": 23.515918731689453, "learning_rate": 3.933941090877615e-07, "logits/chosen": -0.35459989309310913, "logits/rejected": -0.3430092930793762, "logps/chosen": -212.5013427734375, "logps/ref_chosen": -49.42369842529297, "logps/ref_rejected": -79.53791809082031, "logps/rejected": -393.9404296875, "loss": 1.0789, "margin_dpo/margin_mean": 151.32489013671875, "margin_dpo/margin_std": 181.58273315429688, "step": 257 }, { "KL/chosen_KL_mean": -215.13340759277344, "KL/mean": -289.7934875488281, "KL/rejected_KL_mean": -364.45355224609375, "KL/std": 176.51971435546875, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.002779233269393444, "fcm_dpo/delta": -0.019095513969659805, "fcm_dpo/margin": 149.32015991210938, "fcm_dpo/q_t": 0.40277743339538574, "grad_norm": 23.90601348876953, "learning_rate": 3.923409817553284e-07, "logits/chosen": -0.3055630028247833, "logits/rejected": -0.3058106601238251, "logps/chosen": -274.51751708984375, "logps/ref_chosen": -59.384124755859375, "logps/ref_rejected": -95.99010467529297, "logps/rejected": -460.44366455078125, "loss": 1.0945, "margin_dpo/margin_mean": 149.32015991210938, "margin_dpo/margin_std": 211.73117065429688, "step": 258 }, { "KL/chosen_KL_mean": -202.0823974609375, "KL/mean": -263.96234130859375, "KL/rejected_KL_mean": -325.84234619140625, "KL/std": 168.348876953125, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.002821682021021843, "fcm_dpo/delta": 0.06138894706964493, "fcm_dpo/margin": 123.75993347167969, "fcm_dpo/q_t": 0.4222329556941986, "grad_norm": 20.800395965576172, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -0.35386550426483154, "logits/rejected": -0.35250523686408997, "logps/chosen": -254.91073608398438, "logps/ref_chosen": -52.828346252441406, "logps/ref_rejected": -89.191650390625, "logps/rejected": -415.03399658203125, "loss": 1.1393, "margin_dpo/margin_mean": 123.75993347167969, "margin_dpo/margin_std": 192.9962921142578, "step": 259 }, { "KL/chosen_KL_mean": -198.77923583984375, "KL/mean": -283.6119079589844, "KL/rejected_KL_mean": -368.444580078125, "KL/std": 172.5734405517578, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.0027726658154278994, "fcm_dpo/delta": -0.09174390137195587, "fcm_dpo/margin": 169.66534423828125, "fcm_dpo/q_t": 0.3891771137714386, "grad_norm": 29.0617733001709, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -0.33367860317230225, "logits/rejected": -0.3492773771286011, "logps/chosen": -246.1968994140625, "logps/ref_chosen": -47.41767501831055, "logps/ref_rejected": -95.08978271484375, "logps/rejected": -463.53436279296875, "loss": 1.0302, "margin_dpo/margin_mean": 169.66534423828125, "margin_dpo/margin_std": 185.7799530029297, "step": 260 }, { "KL/chosen_KL_mean": -203.46487426757812, "KL/mean": -282.5809326171875, "KL/rejected_KL_mean": -361.697021484375, "KL/std": 188.90347290039062, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.0025986204855144024, "fcm_dpo/delta": -0.014911869540810585, "fcm_dpo/margin": 158.2321319580078, "fcm_dpo/q_t": 0.4053801894187927, "grad_norm": 20.706195831298828, "learning_rate": 3.891592063515376e-07, "logits/chosen": -0.30493831634521484, "logits/rejected": -0.3092959225177765, "logps/chosen": -256.4962463378906, "logps/ref_chosen": -53.03137969970703, "logps/ref_rejected": -88.51494598388672, "logps/rejected": -450.21197509765625, "loss": 1.0849, "margin_dpo/margin_mean": 158.23214721679688, "margin_dpo/margin_std": 208.61734008789062, "step": 261 }, { "KL/chosen_KL_mean": -237.848876953125, "KL/mean": -303.23284912109375, "KL/rejected_KL_mean": -368.6168212890625, "KL/std": 171.96548461914062, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.0027585539501160383, "fcm_dpo/delta": 0.045463643968105316, "fcm_dpo/margin": 130.7679443359375, "fcm_dpo/q_t": 0.4169616997241974, "grad_norm": 26.5394344329834, "learning_rate": 3.880912432401264e-07, "logits/chosen": -0.2885762155056, "logits/rejected": -0.26217561960220337, "logps/chosen": -297.4690246582031, "logps/ref_chosen": -59.620140075683594, "logps/ref_rejected": -86.41853332519531, "logps/rejected": -455.03533935546875, "loss": 1.1097, "margin_dpo/margin_mean": 130.7679443359375, "margin_dpo/margin_std": 169.8227081298828, "step": 262 }, { "KL/chosen_KL_mean": -214.3430938720703, "KL/mean": -308.7781982421875, "KL/rejected_KL_mean": -403.21331787109375, "KL/std": 202.47735595703125, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.002491295337677002, "fcm_dpo/delta": -0.09768804162740707, "fcm_dpo/margin": 188.87020874023438, "fcm_dpo/q_t": 0.38718199729919434, "grad_norm": 18.903581619262695, "learning_rate": 3.870196412960302e-07, "logits/chosen": -0.31606876850128174, "logits/rejected": -0.2949512004852295, "logps/chosen": -273.7640380859375, "logps/ref_chosen": -59.42094421386719, "logps/ref_rejected": -96.85720825195312, "logps/rejected": -500.0705261230469, "loss": 1.0366, "margin_dpo/margin_mean": 188.87022399902344, "margin_dpo/margin_std": 214.49734497070312, "step": 263 }, { "KL/chosen_KL_mean": -232.32882690429688, "KL/mean": -313.197998046875, "KL/rejected_KL_mean": -394.0671691894531, "KL/std": 192.44293212890625, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.0024121187161654234, "fcm_dpo/delta": 0.006785430014133453, "fcm_dpo/margin": 161.7383270263672, "fcm_dpo/q_t": 0.41011467576026917, "grad_norm": 23.93086814880371, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -0.35104095935821533, "logits/rejected": -0.3437178134918213, "logps/chosen": -295.0509033203125, "logps/ref_chosen": -62.722084045410156, "logps/ref_rejected": -93.85620880126953, "logps/rejected": -487.92340087890625, "loss": 1.1065, "margin_dpo/margin_mean": 161.7383270263672, "margin_dpo/margin_std": 220.1507110595703, "step": 264 }, { "KL/chosen_KL_mean": -254.60040283203125, "KL/mean": -330.61474609375, "KL/rejected_KL_mean": -406.629150390625, "KL/std": 214.48471069335938, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0025378770660609007, "fcm_dpo/delta": 0.017527392134070396, "fcm_dpo/margin": 152.0287322998047, "fcm_dpo/q_t": 0.4144975543022156, "grad_norm": 22.09156036376953, "learning_rate": 3.848656339557562e-07, "logits/chosen": -0.2829780578613281, "logits/rejected": -0.2669457197189331, "logps/chosen": -316.5718688964844, "logps/ref_chosen": -61.971466064453125, "logps/ref_rejected": -88.02059936523438, "logps/rejected": -494.6497497558594, "loss": 1.1316, "margin_dpo/margin_mean": 152.0287322998047, "margin_dpo/margin_std": 254.72515869140625, "step": 265 }, { "KL/chosen_KL_mean": -259.6724853515625, "KL/mean": -320.30584716796875, "KL/rejected_KL_mean": -380.9391784667969, "KL/std": 179.23141479492188, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0027112066745758057, "fcm_dpo/delta": 0.08521057665348053, "fcm_dpo/margin": 121.26667022705078, "fcm_dpo/q_t": 0.4275960922241211, "grad_norm": 47.17190933227539, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -0.31882771849632263, "logits/rejected": -0.2803471088409424, "logps/chosen": -326.77215576171875, "logps/ref_chosen": -67.09967041015625, "logps/ref_rejected": -67.97122192382812, "logps/rejected": -448.910400390625, "loss": 1.1572, "margin_dpo/margin_mean": 121.26667785644531, "margin_dpo/margin_std": 205.98829650878906, "step": 266 }, { "KL/chosen_KL_mean": -229.09750366210938, "KL/mean": -301.92169189453125, "KL/rejected_KL_mean": -374.7459411621094, "KL/std": 188.1318359375, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.002730957930907607, "fcm_dpo/delta": 0.0015622973442077637, "fcm_dpo/margin": 145.6484375, "fcm_dpo/q_t": 0.4101436734199524, "grad_norm": 41.06880187988281, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -0.3704749643802643, "logits/rejected": -0.3451113700866699, "logps/chosen": -298.0682373046875, "logps/ref_chosen": -68.97075653076172, "logps/ref_rejected": -90.16844940185547, "logps/rejected": -464.91436767578125, "loss": 1.1069, "margin_dpo/margin_mean": 145.64842224121094, "margin_dpo/margin_std": 215.05789184570312, "step": 267 }, { "KL/chosen_KL_mean": -242.15602111816406, "KL/mean": -318.61297607421875, "KL/rejected_KL_mean": -395.07000732421875, "KL/std": 188.01754760742188, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0026982720009982586, "fcm_dpo/delta": -0.017409201711416245, "fcm_dpo/margin": 152.9139404296875, "fcm_dpo/q_t": 0.40618669986724854, "grad_norm": 27.902847290039062, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -0.31142598390579224, "logits/rejected": -0.31501471996307373, "logps/chosen": -298.05633544921875, "logps/ref_chosen": -55.90031051635742, "logps/ref_rejected": -101.64763641357422, "logps/rejected": -496.7176208496094, "loss": 1.1023, "margin_dpo/margin_mean": 152.9139404296875, "margin_dpo/margin_std": 228.34695434570312, "step": 268 }, { "KL/chosen_KL_mean": -274.3188171386719, "KL/mean": -364.34295654296875, "KL/rejected_KL_mean": -454.36712646484375, "KL/std": 197.2436981201172, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.0025738244876265526, "fcm_dpo/delta": -0.0825297087430954, "fcm_dpo/margin": 180.04827880859375, "fcm_dpo/q_t": 0.39260607957839966, "grad_norm": 29.512990951538086, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -0.3739697337150574, "logits/rejected": -0.3583334684371948, "logps/chosen": -344.3583984375, "logps/ref_chosen": -70.03955841064453, "logps/ref_rejected": -107.34937286376953, "logps/rejected": -561.7164916992188, "loss": 1.0732, "margin_dpo/margin_mean": 180.04827880859375, "margin_dpo/margin_std": 252.80072021484375, "step": 269 }, { "KL/chosen_KL_mean": -230.86578369140625, "KL/mean": -299.2672119140625, "KL/rejected_KL_mean": -367.66864013671875, "KL/std": 178.17739868164062, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.0026433742605149746, "fcm_dpo/delta": 0.0446162149310112, "fcm_dpo/margin": 136.8028564453125, "fcm_dpo/q_t": 0.41896626353263855, "grad_norm": 47.853179931640625, "learning_rate": 3.794189242333106e-07, "logits/chosen": -0.4406346380710602, "logits/rejected": -0.4400424361228943, "logps/chosen": -300.3992614746094, "logps/ref_chosen": -69.53347778320312, "logps/ref_rejected": -109.92864990234375, "logps/rejected": -477.5972900390625, "loss": 1.1416, "margin_dpo/margin_mean": 136.8028564453125, "margin_dpo/margin_std": 227.7311248779297, "step": 270 }, { "KL/chosen_KL_mean": -227.30824279785156, "KL/mean": -316.3374938964844, "KL/rejected_KL_mean": -405.3667297363281, "KL/std": 184.22903442382812, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.00247220229357481, "fcm_dpo/delta": -0.054896239191293716, "fcm_dpo/margin": 178.0584716796875, "fcm_dpo/q_t": 0.3969544470310211, "grad_norm": 26.639240264892578, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -0.3195192515850067, "logits/rejected": -0.3056868612766266, "logps/chosen": -284.07281494140625, "logps/ref_chosen": -56.76456832885742, "logps/ref_rejected": -92.51383972167969, "logps/rejected": -497.88055419921875, "loss": 1.0576, "margin_dpo/margin_mean": 178.0584716796875, "margin_dpo/margin_std": 211.52894592285156, "step": 271 }, { "KL/chosen_KL_mean": -217.37164306640625, "KL/mean": -324.1234130859375, "KL/rejected_KL_mean": -430.87518310546875, "KL/std": 205.4072723388672, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.002309414092451334, "fcm_dpo/delta": -0.12269287556409836, "fcm_dpo/margin": 213.5035400390625, "fcm_dpo/q_t": 0.3797604441642761, "grad_norm": 40.22795486450195, "learning_rate": 3.772161666010912e-07, "logits/chosen": -0.2815614342689514, "logits/rejected": -0.29494649171829224, "logps/chosen": -266.8688049316406, "logps/ref_chosen": -49.497154235839844, "logps/ref_rejected": -105.54279327392578, "logps/rejected": -536.41796875, "loss": 1.0089, "margin_dpo/margin_mean": 213.5035400390625, "margin_dpo/margin_std": 207.53280639648438, "step": 272 }, { "KL/chosen_KL_mean": -249.883056640625, "KL/mean": -348.0146789550781, "KL/rejected_KL_mean": -446.14630126953125, "KL/std": 197.6843719482422, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.002137008123099804, "fcm_dpo/delta": -0.02540270984172821, "fcm_dpo/margin": 196.2632293701172, "fcm_dpo/q_t": 0.4012957215309143, "grad_norm": 24.651897430419922, "learning_rate": 3.761097448550755e-07, "logits/chosen": -0.30211585760116577, "logits/rejected": -0.28442516922950745, "logps/chosen": -312.85845947265625, "logps/ref_chosen": -62.97539520263672, "logps/ref_rejected": -92.49858093261719, "logps/rejected": -538.6448974609375, "loss": 1.0672, "margin_dpo/margin_mean": 196.2632293701172, "margin_dpo/margin_std": 230.588134765625, "step": 273 }, { "KL/chosen_KL_mean": -293.39508056640625, "KL/mean": -369.06951904296875, "KL/rejected_KL_mean": -444.743896484375, "KL/std": 191.70803833007812, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.002289203926920891, "fcm_dpo/delta": 0.061743948608636856, "fcm_dpo/margin": 151.3488311767578, "fcm_dpo/q_t": 0.42134177684783936, "grad_norm": 22.297304153442383, "learning_rate": 3.75e-07, "logits/chosen": -0.2628266215324402, "logits/rejected": -0.24505800008773804, "logps/chosen": -349.06280517578125, "logps/ref_chosen": -55.66770935058594, "logps/ref_rejected": -77.33308410644531, "logps/rejected": -522.0770263671875, "loss": 1.1298, "margin_dpo/margin_mean": 151.3488311767578, "margin_dpo/margin_std": 218.46170043945312, "step": 274 }, { "KL/chosen_KL_mean": -226.7274169921875, "KL/mean": -308.6673889160156, "KL/rejected_KL_mean": -390.60736083984375, "KL/std": 187.83544921875, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.0023457312490791082, "fcm_dpo/delta": 0.01810782589018345, "fcm_dpo/margin": 163.87994384765625, "fcm_dpo/q_t": 0.4110303521156311, "grad_norm": 29.85940933227539, "learning_rate": 3.738869612786737e-07, "logits/chosen": -0.31675028800964355, "logits/rejected": -0.32569241523742676, "logps/chosen": -275.3221130371094, "logps/ref_chosen": -48.594703674316406, "logps/ref_rejected": -93.30369567871094, "logps/rejected": -483.91107177734375, "loss": 1.096, "margin_dpo/margin_mean": 163.87994384765625, "margin_dpo/margin_std": 209.12115478515625, "step": 275 }, { "KL/chosen_KL_mean": -254.89431762695312, "KL/mean": -338.76141357421875, "KL/rejected_KL_mean": -422.6285400390625, "KL/std": 204.18536376953125, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.0023221762385219336, "fcm_dpo/delta": 0.012708023190498352, "fcm_dpo/margin": 167.73422241210938, "fcm_dpo/q_t": 0.4127948582172394, "grad_norm": 25.3048038482666, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -0.2963874936103821, "logits/rejected": -0.2773512601852417, "logps/chosen": -311.47174072265625, "logps/ref_chosen": -56.57740783691406, "logps/ref_rejected": -70.36566925048828, "logps/rejected": -492.99420166015625, "loss": 1.1126, "margin_dpo/margin_mean": 167.73422241210938, "margin_dpo/margin_std": 250.09458923339844, "step": 276 }, { "KL/chosen_KL_mean": -274.00732421875, "KL/mean": -361.88043212890625, "KL/rejected_KL_mean": -449.7535400390625, "KL/std": 200.5372772216797, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.0023581404238939285, "fcm_dpo/delta": -0.018989086151123047, "fcm_dpo/margin": 175.74618530273438, "fcm_dpo/q_t": 0.4043876826763153, "grad_norm": 22.430313110351562, "learning_rate": 3.71651119641714e-07, "logits/chosen": -0.31193846464157104, "logits/rejected": -0.2933298349380493, "logps/chosen": -330.27886962890625, "logps/ref_chosen": -56.27156066894531, "logps/ref_rejected": -92.88127136230469, "logps/rejected": -542.634765625, "loss": 1.091, "margin_dpo/margin_mean": 175.74620056152344, "margin_dpo/margin_std": 240.4161376953125, "step": 277 }, { "KL/chosen_KL_mean": -238.85679626464844, "KL/mean": -339.3966064453125, "KL/rejected_KL_mean": -439.93634033203125, "KL/std": 207.98587036132812, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.00218441104516387, "fcm_dpo/delta": -0.052646711468696594, "fcm_dpo/margin": 201.0795440673828, "fcm_dpo/q_t": 0.3967970907688141, "grad_norm": 22.02371597290039, "learning_rate": 3.705283756425872e-07, "logits/chosen": -0.30684125423431396, "logits/rejected": -0.31390851736068726, "logps/chosen": -291.7987365722656, "logps/ref_chosen": -52.94194030761719, "logps/ref_rejected": -91.25357818603516, "logps/rejected": -531.18994140625, "loss": 1.0559, "margin_dpo/margin_mean": 201.0795440673828, "margin_dpo/margin_std": 235.0767822265625, "step": 278 }, { "KL/chosen_KL_mean": -266.6314697265625, "KL/mean": -369.5718994140625, "KL/rejected_KL_mean": -472.5123291015625, "KL/std": 229.59307861328125, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.0020728304516524076, "fcm_dpo/delta": -0.038570303469896317, "fcm_dpo/margin": 205.880859375, "fcm_dpo/q_t": 0.4020066261291504, "grad_norm": 23.30048179626465, "learning_rate": 3.6940245560867e-07, "logits/chosen": -0.25095367431640625, "logits/rejected": -0.2509291172027588, "logps/chosen": -315.2727966308594, "logps/ref_chosen": -48.641319274902344, "logps/ref_rejected": -87.8514404296875, "logps/rejected": -560.36376953125, "loss": 1.0833, "margin_dpo/margin_mean": 205.880859375, "margin_dpo/margin_std": 281.98443603515625, "step": 279 }, { "KL/chosen_KL_mean": -237.64895629882812, "KL/mean": -337.5757751464844, "KL/rejected_KL_mean": -437.50262451171875, "KL/std": 190.99032592773438, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.0020760963670909405, "fcm_dpo/delta": -0.018862612545490265, "fcm_dpo/margin": 199.85365295410156, "fcm_dpo/q_t": 0.40116894245147705, "grad_norm": 22.5253963470459, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -0.3350566625595093, "logits/rejected": -0.33742278814315796, "logps/chosen": -296.4460754394531, "logps/ref_chosen": -58.797122955322266, "logps/ref_rejected": -98.61885070800781, "logps/rejected": -536.1214599609375, "loss": 1.0562, "margin_dpo/margin_mean": 199.8536376953125, "margin_dpo/margin_std": 203.86276245117188, "step": 280 }, { "KL/chosen_KL_mean": -224.87203979492188, "KL/mean": -309.09222412109375, "KL/rejected_KL_mean": -393.31243896484375, "KL/std": 193.19955444335938, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0021694411989301443, "fcm_dpo/delta": 0.04002968966960907, "fcm_dpo/margin": 168.44039916992188, "fcm_dpo/q_t": 0.4139762818813324, "grad_norm": 26.64156150817871, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -0.3864981532096863, "logits/rejected": -0.3639860153198242, "logps/chosen": -280.3605651855469, "logps/ref_chosen": -55.488521575927734, "logps/ref_rejected": -80.88258361816406, "logps/rejected": -474.19500732421875, "loss": 1.1042, "margin_dpo/margin_mean": 168.4403839111328, "margin_dpo/margin_std": 207.57632446289062, "step": 281 }, { "KL/chosen_KL_mean": -260.92059326171875, "KL/mean": -328.2933349609375, "KL/rejected_KL_mean": -395.66607666015625, "KL/std": 199.9541015625, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.002180553739890456, "fcm_dpo/delta": 0.025225864723324776, "fcm_dpo/margin": 134.74557495117188, "fcm_dpo/q_t": 0.43476957082748413, "grad_norm": 23.243419647216797, "learning_rate": 3.660059364023408e-07, "logits/chosen": -0.39929962158203125, "logits/rejected": -0.3811987638473511, "logps/chosen": -333.99072265625, "logps/ref_chosen": -73.07014465332031, "logps/ref_rejected": -95.35098266601562, "logps/rejected": -491.01708984375, "loss": 1.1715, "margin_dpo/margin_mean": 134.7455596923828, "margin_dpo/margin_std": 234.26336669921875, "step": 282 }, { "KL/chosen_KL_mean": -273.17254638671875, "KL/mean": -379.52557373046875, "KL/rejected_KL_mean": -485.87860107421875, "KL/std": 243.8074951171875, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0021401268895715475, "fcm_dpo/delta": -0.07182439416646957, "fcm_dpo/margin": 212.7060546875, "fcm_dpo/q_t": 0.3907451927661896, "grad_norm": 35.38416290283203, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -0.3343729078769684, "logits/rejected": -0.3363478183746338, "logps/chosen": -335.07098388671875, "logps/ref_chosen": -61.89844512939453, "logps/ref_rejected": -96.98655700683594, "logps/rejected": -582.8651123046875, "loss": 1.0446, "margin_dpo/margin_mean": 212.7060546875, "margin_dpo/margin_std": 232.63815307617188, "step": 283 }, { "KL/chosen_KL_mean": -261.57415771484375, "KL/mean": -360.19207763671875, "KL/rejected_KL_mean": -458.80999755859375, "KL/std": 222.4651336669922, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.0020390492863953114, "fcm_dpo/delta": -0.0028852270916104317, "fcm_dpo/margin": 197.23583984375, "fcm_dpo/q_t": 0.4071544408798218, "grad_norm": 24.07853126525879, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -0.37924909591674805, "logits/rejected": -0.3620849549770355, "logps/chosen": -320.00970458984375, "logps/ref_chosen": -58.4355354309082, "logps/ref_rejected": -93.46926879882812, "logps/rejected": -552.279296875, "loss": 1.0777, "margin_dpo/margin_mean": 197.23580932617188, "margin_dpo/margin_std": 236.99972534179688, "step": 284 }, { "KL/chosen_KL_mean": -298.85516357421875, "KL/mean": -381.50390625, "KL/rejected_KL_mean": -464.1526184082031, "KL/std": 190.44850158691406, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.0021772286854684353, "fcm_dpo/delta": 0.044019557535648346, "fcm_dpo/margin": 165.29745483398438, "fcm_dpo/q_t": 0.4151372015476227, "grad_norm": 23.67416763305664, "learning_rate": 3.625819059005228e-07, "logits/chosen": -0.3634873628616333, "logits/rejected": -0.348185658454895, "logps/chosen": -365.08734130859375, "logps/ref_chosen": -66.23219299316406, "logps/ref_rejected": -99.1268310546875, "logps/rejected": -563.2794189453125, "loss": 1.1137, "margin_dpo/margin_mean": 165.29745483398438, "margin_dpo/margin_std": 213.57778930664062, "step": 285 }, { "KL/chosen_KL_mean": -292.51617431640625, "KL/mean": -385.5911560058594, "KL/rejected_KL_mean": -478.6661682128906, "KL/std": 223.45005798339844, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.002157814335078001, "fcm_dpo/delta": -0.0024737920612096786, "fcm_dpo/margin": 186.14996337890625, "fcm_dpo/q_t": 0.40657156705856323, "grad_norm": 23.773475646972656, "learning_rate": 3.614345889441346e-07, "logits/chosen": -0.3397786319255829, "logits/rejected": -0.32064247131347656, "logps/chosen": -365.4671936035156, "logps/ref_chosen": -72.95100402832031, "logps/ref_rejected": -88.58845520019531, "logps/rejected": -567.254638671875, "loss": 1.0899, "margin_dpo/margin_mean": 186.14996337890625, "margin_dpo/margin_std": 243.01104736328125, "step": 286 }, { "KL/chosen_KL_mean": -262.6454162597656, "KL/mean": -341.39129638671875, "KL/rejected_KL_mean": -420.13714599609375, "KL/std": 194.3848114013672, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.0022398338187485933, "fcm_dpo/delta": 0.05684395506978035, "fcm_dpo/margin": 157.49172973632812, "fcm_dpo/q_t": 0.4202299118041992, "grad_norm": 20.517465591430664, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -0.30254948139190674, "logits/rejected": -0.27677813172340393, "logps/chosen": -324.18658447265625, "logps/ref_chosen": -61.54115295410156, "logps/ref_rejected": -77.69607543945312, "logps/rejected": -497.833251953125, "loss": 1.1189, "margin_dpo/margin_mean": 157.49172973632812, "margin_dpo/margin_std": 216.00836181640625, "step": 287 }, { "KL/chosen_KL_mean": -242.85244750976562, "KL/mean": -340.43890380859375, "KL/rejected_KL_mean": -438.025390625, "KL/std": 184.95721435546875, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.002138757612556219, "fcm_dpo/delta": -0.028658173978328705, "fcm_dpo/margin": 195.1729278564453, "fcm_dpo/q_t": 0.3991842269897461, "grad_norm": 20.475711822509766, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -0.35998380184173584, "logits/rejected": -0.34525611996650696, "logps/chosen": -299.513671875, "logps/ref_chosen": -56.661224365234375, "logps/ref_rejected": -87.33570098876953, "logps/rejected": -525.361083984375, "loss": 1.0586, "margin_dpo/margin_mean": 195.17294311523438, "margin_dpo/margin_std": 196.0745391845703, "step": 288 }, { "KL/chosen_KL_mean": -236.07003784179688, "KL/mean": -339.13861083984375, "KL/rejected_KL_mean": -442.2071533203125, "KL/std": 204.4755859375, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.002140854485332966, "fcm_dpo/delta": -0.05301383137702942, "fcm_dpo/margin": 206.13711547851562, "fcm_dpo/q_t": 0.3952919840812683, "grad_norm": 35.90175247192383, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -0.28917020559310913, "logits/rejected": -0.29095658659935, "logps/chosen": -281.3004455566406, "logps/ref_chosen": -45.23039245605469, "logps/ref_rejected": -87.64266967773438, "logps/rejected": -529.849853515625, "loss": 1.0471, "margin_dpo/margin_mean": 206.13711547851562, "margin_dpo/margin_std": 221.66661071777344, "step": 289 }, { "KL/chosen_KL_mean": -259.9752502441406, "KL/mean": -367.92950439453125, "KL/rejected_KL_mean": -475.8837890625, "KL/std": 215.7952880859375, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.0020013581961393356, "fcm_dpo/delta": -0.04185499623417854, "fcm_dpo/margin": 215.90850830078125, "fcm_dpo/q_t": 0.39859092235565186, "grad_norm": 23.12173843383789, "learning_rate": 3.568162605525952e-07, "logits/chosen": -0.29490506649017334, "logits/rejected": -0.3174816370010376, "logps/chosen": -315.4467468261719, "logps/ref_chosen": -55.47149658203125, "logps/ref_rejected": -116.70857238769531, "logps/rejected": -592.5923461914062, "loss": 1.0603, "margin_dpo/margin_mean": 215.9084930419922, "margin_dpo/margin_std": 255.39891052246094, "step": 290 }, { "KL/chosen_KL_mean": -236.2718505859375, "KL/mean": -337.82806396484375, "KL/rejected_KL_mean": -439.3843078613281, "KL/std": 192.71876525878906, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.0020202212035655975, "fcm_dpo/delta": -0.014722079038619995, "fcm_dpo/margin": 203.11245727539062, "fcm_dpo/q_t": 0.4027057886123657, "grad_norm": 20.603652954101562, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -0.3010188043117523, "logits/rejected": -0.29001563787460327, "logps/chosen": -299.5322265625, "logps/ref_chosen": -63.26036834716797, "logps/ref_rejected": -89.29708862304688, "logps/rejected": -528.681396484375, "loss": 1.0707, "margin_dpo/margin_mean": 203.11245727539062, "margin_dpo/margin_std": 228.27931213378906, "step": 291 }, { "KL/chosen_KL_mean": -271.1993408203125, "KL/mean": -373.78936767578125, "KL/rejected_KL_mean": -476.37933349609375, "KL/std": 224.70799255371094, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0019872216507792473, "fcm_dpo/delta": -0.010810526087880135, "fcm_dpo/margin": 205.17999267578125, "fcm_dpo/q_t": 0.40419653058052063, "grad_norm": 20.72088050842285, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -0.3157344460487366, "logits/rejected": -0.3041796088218689, "logps/chosen": -325.1178894042969, "logps/ref_chosen": -53.91852951049805, "logps/ref_rejected": -89.96138000488281, "logps/rejected": -566.3406982421875, "loss": 1.0851, "margin_dpo/margin_mean": 205.17999267578125, "margin_dpo/margin_std": 263.08953857421875, "step": 292 }, { "KL/chosen_KL_mean": -287.68072509765625, "KL/mean": -370.7479248046875, "KL/rejected_KL_mean": -453.81512451171875, "KL/std": 234.54473876953125, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.0020656294655054808, "fcm_dpo/delta": 0.06754275411367416, "fcm_dpo/margin": 166.13436889648438, "fcm_dpo/q_t": 0.42421412467956543, "grad_norm": 27.3696346282959, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -0.332236647605896, "logits/rejected": -0.3292074203491211, "logps/chosen": -348.0567626953125, "logps/ref_chosen": -60.376033782958984, "logps/ref_rejected": -77.85244750976562, "logps/rejected": -531.6675415039062, "loss": 1.144, "margin_dpo/margin_mean": 166.13436889648438, "margin_dpo/margin_std": 270.0863037109375, "step": 293 }, { "KL/chosen_KL_mean": -266.4648742675781, "KL/mean": -360.0727844238281, "KL/rejected_KL_mean": -453.68072509765625, "KL/std": 205.4690704345703, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.002078534569591284, "fcm_dpo/delta": 0.012987792491912842, "fcm_dpo/margin": 187.21585083007812, "fcm_dpo/q_t": 0.41207996010780334, "grad_norm": 24.852489471435547, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -0.3020135164260864, "logits/rejected": -0.30931025743484497, "logps/chosen": -314.5523986816406, "logps/ref_chosen": -48.0875358581543, "logps/ref_rejected": -81.89698791503906, "logps/rejected": -535.5776977539062, "loss": 1.102, "margin_dpo/margin_mean": 187.21585083007812, "margin_dpo/margin_std": 258.83837890625, "step": 294 }, { "KL/chosen_KL_mean": -315.635009765625, "KL/mean": -417.57843017578125, "KL/rejected_KL_mean": -519.52197265625, "KL/std": 232.22276306152344, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.0020754141733050346, "fcm_dpo/delta": -0.029316715896129608, "fcm_dpo/margin": 203.88693237304688, "fcm_dpo/q_t": 0.40314337611198425, "grad_norm": 19.910776138305664, "learning_rate": 3.509802512179737e-07, "logits/chosen": -0.31562861800193787, "logits/rejected": -0.3209300637245178, "logps/chosen": -365.5596618652344, "logps/ref_chosen": -49.92467498779297, "logps/ref_rejected": -87.45632934570312, "logps/rejected": -606.978271484375, "loss": 1.0858, "margin_dpo/margin_mean": 203.88693237304688, "margin_dpo/margin_std": 279.4998779296875, "step": 295 }, { "KL/chosen_KL_mean": -389.09619140625, "KL/mean": -459.28369140625, "KL/rejected_KL_mean": -529.4712524414062, "KL/std": 224.94338989257812, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.00205283984541893, "fcm_dpo/delta": 0.0012772375484928489, "fcm_dpo/margin": 140.37503051757812, "fcm_dpo/q_t": 0.4331241846084595, "grad_norm": 30.800323486328125, "learning_rate": 3.498049431928577e-07, "logits/chosen": -0.3613106906414032, "logits/rejected": -0.3486577272415161, "logps/chosen": -454.5874328613281, "logps/ref_chosen": -65.49124145507812, "logps/ref_rejected": -93.08908081054688, "logps/rejected": -622.560302734375, "loss": 1.2004, "margin_dpo/margin_mean": 140.37501525878906, "margin_dpo/margin_std": 281.86309814453125, "step": 296 }, { "KL/chosen_KL_mean": -337.59783935546875, "KL/mean": -417.08062744140625, "KL/rejected_KL_mean": -496.5633850097656, "KL/std": 210.40191650390625, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.002149302512407303, "fcm_dpo/delta": 0.07061058282852173, "fcm_dpo/margin": 158.96554565429688, "fcm_dpo/q_t": 0.4227258563041687, "grad_norm": 33.55876541137695, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.368080735206604, "logits/rejected": -0.3742336928844452, "logps/chosen": -394.0747985839844, "logps/ref_chosen": -56.476951599121094, "logps/ref_rejected": -95.1385498046875, "logps/rejected": -591.701904296875, "loss": 1.1254, "margin_dpo/margin_mean": 158.96554565429688, "margin_dpo/margin_std": 216.52734375, "step": 297 }, { "KL/chosen_KL_mean": -349.87359619140625, "KL/mean": -458.6307373046875, "KL/rejected_KL_mean": -567.387939453125, "KL/std": 274.2647399902344, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.002113178838044405, "fcm_dpo/delta": -0.0772165060043335, "fcm_dpo/margin": 217.51431274414062, "fcm_dpo/q_t": 0.3965296745300293, "grad_norm": 24.389272689819336, "learning_rate": 3.474464683231698e-07, "logits/chosen": -0.401616632938385, "logits/rejected": -0.4193766713142395, "logps/chosen": -417.19873046875, "logps/ref_chosen": -67.32516479492188, "logps/ref_rejected": -116.66217041015625, "logps/rejected": -684.050048828125, "loss": 1.0813, "margin_dpo/margin_mean": 217.51431274414062, "margin_dpo/margin_std": 321.95660400390625, "step": 298 }, { "KL/chosen_KL_mean": -289.93463134765625, "KL/mean": -381.6058044433594, "KL/rejected_KL_mean": -473.2769470214844, "KL/std": 202.38690185546875, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.002111277775838971, "fcm_dpo/delta": 0.012855540961027145, "fcm_dpo/margin": 183.3423614501953, "fcm_dpo/q_t": 0.4118254780769348, "grad_norm": 35.68212890625, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.35824018716812134, "logits/rejected": -0.36311280727386475, "logps/chosen": -338.896728515625, "logps/ref_chosen": -48.96209716796875, "logps/ref_rejected": -84.32823944091797, "logps/rejected": -557.605224609375, "loss": 1.104, "margin_dpo/margin_mean": 183.34234619140625, "margin_dpo/margin_std": 251.83197021484375, "step": 299 }, { "KL/chosen_KL_mean": -350.1387939453125, "KL/mean": -458.59375, "KL/rejected_KL_mean": -567.0487060546875, "KL/std": 237.21231079101562, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.0020199865102767944, "fcm_dpo/delta": -0.048955146223306656, "fcm_dpo/margin": 216.90985107421875, "fcm_dpo/q_t": 0.39751458168029785, "grad_norm": 26.843948364257812, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -0.3743543028831482, "logits/rejected": -0.3817921280860901, "logps/chosen": -409.2125244140625, "logps/ref_chosen": -59.07371139526367, "logps/ref_rejected": -95.9664535522461, "logps/rejected": -663.01513671875, "loss": 1.0789, "margin_dpo/margin_mean": 216.90985107421875, "margin_dpo/margin_std": 290.77459716796875, "step": 300 }, { "KL/chosen_KL_mean": -294.7914733886719, "KL/mean": -403.30487060546875, "KL/rejected_KL_mean": -511.8182373046875, "KL/std": 218.48614501953125, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.0019110521534457803, "fcm_dpo/delta": -0.019859077408909798, "fcm_dpo/margin": 217.02674865722656, "fcm_dpo/q_t": 0.40373334288597107, "grad_norm": 25.57866096496582, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -0.3827190399169922, "logits/rejected": -0.38337594270706177, "logps/chosen": -352.04083251953125, "logps/ref_chosen": -57.249366760253906, "logps/ref_rejected": -92.35354614257812, "logps/rejected": -604.1717529296875, "loss": 1.0724, "margin_dpo/margin_mean": 217.02676391601562, "margin_dpo/margin_std": 262.17144775390625, "step": 301 }, { "KL/chosen_KL_mean": -263.9697570800781, "KL/mean": -353.9146728515625, "KL/rejected_KL_mean": -443.85955810546875, "KL/std": 204.00518798828125, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0019708811305463314, "fcm_dpo/delta": 0.054747726768255234, "fcm_dpo/margin": 179.88980102539062, "fcm_dpo/q_t": 0.4198175072669983, "grad_norm": 19.74114227294922, "learning_rate": 3.426989547989902e-07, "logits/chosen": -0.3587857484817505, "logits/rejected": -0.36537155508995056, "logps/chosen": -315.167724609375, "logps/ref_chosen": -51.197994232177734, "logps/ref_rejected": -97.22636413574219, "logps/rejected": -541.0859375, "loss": 1.1197, "margin_dpo/margin_mean": 179.88980102539062, "margin_dpo/margin_std": 243.09542846679688, "step": 302 }, { "KL/chosen_KL_mean": -275.6668701171875, "KL/mean": -358.65966796875, "KL/rejected_KL_mean": -441.6524658203125, "KL/std": 205.9825439453125, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.002106656786054373, "fcm_dpo/delta": 0.061119839549064636, "fcm_dpo/margin": 165.985595703125, "fcm_dpo/q_t": 0.42138999700546265, "grad_norm": 28.86655044555664, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -0.3849058747291565, "logits/rejected": -0.3747110366821289, "logps/chosen": -342.38079833984375, "logps/ref_chosen": -66.71394348144531, "logps/ref_rejected": -86.94542694091797, "logps/rejected": -528.597900390625, "loss": 1.1338, "margin_dpo/margin_mean": 165.985595703125, "margin_dpo/margin_std": 249.7038116455078, "step": 303 }, { "KL/chosen_KL_mean": -257.2261962890625, "KL/mean": -351.8838806152344, "KL/rejected_KL_mean": -446.54156494140625, "KL/std": 196.91989135742188, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0021874920930713415, "fcm_dpo/delta": -0.019974112510681152, "fcm_dpo/margin": 189.31533813476562, "fcm_dpo/q_t": 0.40017253160476685, "grad_norm": 45.90519332885742, "learning_rate": 3.403104165467883e-07, "logits/chosen": -0.41976654529571533, "logits/rejected": -0.4154810905456543, "logps/chosen": -329.1769104003906, "logps/ref_chosen": -71.95069885253906, "logps/ref_rejected": -90.47203063964844, "logps/rejected": -537.0135498046875, "loss": 1.0521, "margin_dpo/margin_mean": 189.31533813476562, "margin_dpo/margin_std": 171.4813232421875, "step": 304 }, { "KL/chosen_KL_mean": -267.2148132324219, "KL/mean": -351.92645263671875, "KL/rejected_KL_mean": -436.63812255859375, "KL/std": 221.58193969726562, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.0021368793677538633, "fcm_dpo/delta": 0.04446440190076828, "fcm_dpo/margin": 169.42330932617188, "fcm_dpo/q_t": 0.4194003641605377, "grad_norm": 24.6429500579834, "learning_rate": 3.391125620245535e-07, "logits/chosen": -0.431082159280777, "logits/rejected": -0.417194128036499, "logps/chosen": -334.0100402832031, "logps/ref_chosen": -66.79523468017578, "logps/ref_rejected": -92.75459289550781, "logps/rejected": -529.3927001953125, "loss": 1.1257, "margin_dpo/margin_mean": 169.42330932617188, "margin_dpo/margin_std": 248.18634033203125, "step": 305 }, { "KL/chosen_KL_mean": -270.4483947753906, "KL/mean": -353.85845947265625, "KL/rejected_KL_mean": -437.2685241699219, "KL/std": 196.28871154785156, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.002273739315569401, "fcm_dpo/delta": 0.025212926790118217, "fcm_dpo/margin": 166.82009887695312, "fcm_dpo/q_t": 0.41293442249298096, "grad_norm": 20.455917358398438, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -0.4016483426094055, "logits/rejected": -0.37884002923965454, "logps/chosen": -340.1322937011719, "logps/ref_chosen": -69.68389892578125, "logps/ref_rejected": -85.15919494628906, "logps/rejected": -522.427734375, "loss": 1.0949, "margin_dpo/margin_mean": 166.82009887695312, "margin_dpo/margin_std": 209.34255981445312, "step": 306 }, { "KL/chosen_KL_mean": -245.0524139404297, "KL/mean": -331.04681396484375, "KL/rejected_KL_mean": -417.041259765625, "KL/std": 183.830810546875, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.0022681187838315964, "fcm_dpo/delta": 0.011811528354883194, "fcm_dpo/margin": 171.98887634277344, "fcm_dpo/q_t": 0.4100651741027832, "grad_norm": 28.9613037109375, "learning_rate": 3.367098400098881e-07, "logits/chosen": -0.4207409620285034, "logits/rejected": -0.40590375661849976, "logps/chosen": -315.21783447265625, "logps/ref_chosen": -70.16542053222656, "logps/ref_rejected": -86.97230529785156, "logps/rejected": -504.0135803222656, "loss": 1.096, "margin_dpo/margin_mean": 171.98886108398438, "margin_dpo/margin_std": 226.1441650390625, "step": 307 }, { "KL/chosen_KL_mean": -252.00428771972656, "KL/mean": -349.07568359375, "KL/rejected_KL_mean": -446.1470642089844, "KL/std": 207.59048461914062, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.002229331061244011, "fcm_dpo/delta": -0.04198146238923073, "fcm_dpo/margin": 194.14276123046875, "fcm_dpo/q_t": 0.3967881202697754, "grad_norm": 28.756671905517578, "learning_rate": 3.355050358314172e-07, "logits/chosen": -0.4297657012939453, "logits/rejected": -0.4254001975059509, "logps/chosen": -307.249267578125, "logps/ref_chosen": -55.2449951171875, "logps/ref_rejected": -79.37226104736328, "logps/rejected": -525.5193481445312, "loss": 1.0511, "margin_dpo/margin_mean": 194.14276123046875, "margin_dpo/margin_std": 207.66583251953125, "step": 308 }, { "KL/chosen_KL_mean": -257.0223388671875, "KL/mean": -352.66180419921875, "KL/rejected_KL_mean": -448.30126953125, "KL/std": 222.42701721191406, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0022140173241496086, "fcm_dpo/delta": -0.03186354413628578, "fcm_dpo/margin": 191.2789306640625, "fcm_dpo/q_t": 0.3998178541660309, "grad_norm": 29.72759437561035, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -0.3393629789352417, "logits/rejected": -0.33330753445625305, "logps/chosen": -305.98138427734375, "logps/ref_chosen": -48.959083557128906, "logps/ref_rejected": -82.34072875976562, "logps/rejected": -530.6419677734375, "loss": 1.0754, "margin_dpo/margin_mean": 191.2789306640625, "margin_dpo/margin_std": 235.2989959716797, "step": 309 }, { "KL/chosen_KL_mean": -304.2635498046875, "KL/mean": -393.31304931640625, "KL/rejected_KL_mean": -482.362548828125, "KL/std": 200.93919372558594, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.0021948455832898617, "fcm_dpo/delta": 0.009528759866952896, "fcm_dpo/margin": 178.0990447998047, "fcm_dpo/q_t": 0.4077993631362915, "grad_norm": 36.597747802734375, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -0.4227510690689087, "logits/rejected": -0.41301751136779785, "logps/chosen": -367.00531005859375, "logps/ref_chosen": -62.74177932739258, "logps/ref_rejected": -79.9302978515625, "logps/rejected": -562.2928466796875, "loss": 1.0787, "margin_dpo/margin_mean": 178.09906005859375, "margin_dpo/margin_std": 195.08804321289062, "step": 310 }, { "KL/chosen_KL_mean": -307.46826171875, "KL/mean": -412.06182861328125, "KL/rejected_KL_mean": -516.6553955078125, "KL/std": 252.58099365234375, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.0020815101452171803, "fcm_dpo/delta": -0.045531198382377625, "fcm_dpo/margin": 209.18707275390625, "fcm_dpo/q_t": 0.39924418926239014, "grad_norm": 25.931556701660156, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -0.4020995497703552, "logits/rejected": -0.3801601529121399, "logps/chosen": -360.4962463378906, "logps/ref_chosen": -53.02798080444336, "logps/ref_rejected": -77.43820190429688, "logps/rejected": -594.0935668945312, "loss": 1.0693, "margin_dpo/margin_mean": 209.18707275390625, "margin_dpo/margin_std": 267.8011474609375, "step": 311 }, { "KL/chosen_KL_mean": -301.262451171875, "KL/mean": -398.7344970703125, "KL/rejected_KL_mean": -496.20654296875, "KL/std": 227.84207153320312, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.002090971916913986, "fcm_dpo/delta": -0.011386025696992874, "fcm_dpo/margin": 194.94412231445312, "fcm_dpo/q_t": 0.40784695744514465, "grad_norm": 25.662092208862305, "learning_rate": 3.306636061080487e-07, "logits/chosen": -0.40964388847351074, "logits/rejected": -0.40318238735198975, "logps/chosen": -350.6546630859375, "logps/ref_chosen": -49.39221954345703, "logps/ref_rejected": -75.79280853271484, "logps/rejected": -571.9993896484375, "loss": 1.0933, "margin_dpo/margin_mean": 194.94412231445312, "margin_dpo/margin_std": 271.61602783203125, "step": 312 }, { "KL/chosen_KL_mean": -282.75341796875, "KL/mean": -380.6422119140625, "KL/rejected_KL_mean": -478.531005859375, "KL/std": 236.21067810058594, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.002096590120345354, "fcm_dpo/delta": -0.019848383963108063, "fcm_dpo/margin": 195.77755737304688, "fcm_dpo/q_t": 0.4035242199897766, "grad_norm": 27.727643966674805, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -0.4416351914405823, "logits/rejected": -0.43936672806739807, "logps/chosen": -332.90618896484375, "logps/ref_chosen": -50.152740478515625, "logps/ref_rejected": -86.40620422363281, "logps/rejected": -564.9371948242188, "loss": 1.0933, "margin_dpo/margin_mean": 195.77755737304688, "margin_dpo/margin_std": 253.7050018310547, "step": 313 }, { "KL/chosen_KL_mean": -304.5816345214844, "KL/mean": -394.99072265625, "KL/rejected_KL_mean": -485.39984130859375, "KL/std": 222.5576934814453, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.0020326506346464157, "fcm_dpo/delta": 0.03956056758761406, "fcm_dpo/margin": 180.8181915283203, "fcm_dpo/q_t": 0.41847917437553406, "grad_norm": 20.639629364013672, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -0.476603627204895, "logits/rejected": -0.4820059835910797, "logps/chosen": -361.8192138671875, "logps/ref_chosen": -57.237579345703125, "logps/ref_rejected": -97.5965347290039, "logps/rejected": -582.996337890625, "loss": 1.1336, "margin_dpo/margin_mean": 180.81817626953125, "margin_dpo/margin_std": 289.74945068359375, "step": 314 }, { "KL/chosen_KL_mean": -279.29681396484375, "KL/mean": -364.7213134765625, "KL/rejected_KL_mean": -450.1457824707031, "KL/std": 205.07464599609375, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.00213609728962183, "fcm_dpo/delta": 0.04297637194395065, "fcm_dpo/margin": 170.8489990234375, "fcm_dpo/q_t": 0.41723307967185974, "grad_norm": 21.063323974609375, "learning_rate": 3.270101039870797e-07, "logits/chosen": -0.3772663474082947, "logits/rejected": -0.3812904357910156, "logps/chosen": -328.36639404296875, "logps/ref_chosen": -49.06958770751953, "logps/ref_rejected": -85.68087768554688, "logps/rejected": -535.82666015625, "loss": 1.1108, "margin_dpo/margin_mean": 170.8489990234375, "margin_dpo/margin_std": 229.18209838867188, "step": 315 }, { "KL/chosen_KL_mean": -275.72784423828125, "KL/mean": -397.86114501953125, "KL/rejected_KL_mean": -519.994384765625, "KL/std": 247.28073120117188, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.001942518400028348, "fcm_dpo/delta": -0.10697119683027267, "fcm_dpo/margin": 244.26651000976562, "fcm_dpo/q_t": 0.3864254951477051, "grad_norm": 23.46164321899414, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -0.42778435349464417, "logits/rejected": -0.43224072456359863, "logps/chosen": -329.9886169433594, "logps/ref_chosen": -54.26074981689453, "logps/ref_rejected": -101.2814712524414, "logps/rejected": -621.27587890625, "loss": 1.0309, "margin_dpo/margin_mean": 244.2665252685547, "margin_dpo/margin_std": 268.9966125488281, "step": 316 }, { "KL/chosen_KL_mean": -286.0876159667969, "KL/mean": -423.0874938964844, "KL/rejected_KL_mean": -560.0873413085938, "KL/std": 227.30641174316406, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0018125809729099274, "fcm_dpo/delta": -0.1276942938566208, "fcm_dpo/margin": 273.9997253417969, "fcm_dpo/q_t": 0.3785746097564697, "grad_norm": 27.784400939941406, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -0.4147995114326477, "logits/rejected": -0.4096861481666565, "logps/chosen": -342.18182373046875, "logps/ref_chosen": -56.094207763671875, "logps/ref_rejected": -100.69905090332031, "logps/rejected": -660.786376953125, "loss": 1.0049, "margin_dpo/margin_mean": 273.99969482421875, "margin_dpo/margin_std": 257.4659423828125, "step": 317 }, { "KL/chosen_KL_mean": -317.5200500488281, "KL/mean": -415.6494140625, "KL/rejected_KL_mean": -513.7787475585938, "KL/std": 229.84683227539062, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.0018144804053008556, "fcm_dpo/delta": 0.05106057599186897, "fcm_dpo/margin": 196.25872802734375, "fcm_dpo/q_t": 0.41813182830810547, "grad_norm": 26.109601974487305, "learning_rate": 3.233383385962115e-07, "logits/chosen": -0.4953124225139618, "logits/rejected": -0.46893250942230225, "logps/chosen": -382.1657409667969, "logps/ref_chosen": -64.64569854736328, "logps/ref_rejected": -82.76425170898438, "logps/rejected": -596.5430297851562, "loss": 1.1096, "margin_dpo/margin_mean": 196.25872802734375, "margin_dpo/margin_std": 247.5173797607422, "step": 318 }, { "KL/chosen_KL_mean": -285.78082275390625, "KL/mean": -403.2270812988281, "KL/rejected_KL_mean": -520.67333984375, "KL/std": 257.381591796875, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.0017810399876907468, "fcm_dpo/delta": -0.023179180920124054, "fcm_dpo/margin": 234.89251708984375, "fcm_dpo/q_t": 0.40157684683799744, "grad_norm": 22.635221481323242, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -0.38388240337371826, "logits/rejected": -0.396739661693573, "logps/chosen": -335.16455078125, "logps/ref_chosen": -49.383758544921875, "logps/ref_rejected": -113.90650939941406, "logps/rejected": -634.579833984375, "loss": 1.0691, "margin_dpo/margin_mean": 234.8925323486328, "margin_dpo/margin_std": 279.86474609375, "step": 319 }, { "KL/chosen_KL_mean": -290.6177978515625, "KL/mean": -416.1560974121094, "KL/rejected_KL_mean": -541.6944580078125, "KL/std": 259.626220703125, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0016947748372331262, "fcm_dpo/delta": -0.03422039747238159, "fcm_dpo/margin": 251.07666015625, "fcm_dpo/q_t": 0.39976027607917786, "grad_norm": 21.635658264160156, "learning_rate": 3.208807785813777e-07, "logits/chosen": -0.44464847445487976, "logits/rejected": -0.4517730474472046, "logps/chosen": -350.1226806640625, "logps/ref_chosen": -59.50489044189453, "logps/ref_rejected": -97.66717529296875, "logps/rejected": -639.361572265625, "loss": 1.0582, "margin_dpo/margin_mean": 251.07666015625, "margin_dpo/margin_std": 280.9095458984375, "step": 320 }, { "KL/chosen_KL_mean": -359.953369140625, "KL/mean": -468.767333984375, "KL/rejected_KL_mean": -577.5813598632812, "KL/std": 275.1668701171875, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.0017093562055379152, "fcm_dpo/delta": 0.03284335136413574, "fcm_dpo/margin": 217.62799072265625, "fcm_dpo/q_t": 0.41511040925979614, "grad_norm": 19.775318145751953, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -0.3967721462249756, "logits/rejected": -0.38455623388290405, "logps/chosen": -421.5020446777344, "logps/ref_chosen": -61.548683166503906, "logps/ref_rejected": -91.64103698730469, "logps/rejected": -669.222412109375, "loss": 1.1103, "margin_dpo/margin_mean": 217.62796020507812, "margin_dpo/margin_std": 292.2899169921875, "step": 321 }, { "KL/chosen_KL_mean": -302.359130859375, "KL/mean": -423.87506103515625, "KL/rejected_KL_mean": -545.3909912109375, "KL/std": 239.97723388671875, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.001710197888314724, "fcm_dpo/delta": -0.02115067094564438, "fcm_dpo/margin": 243.0318603515625, "fcm_dpo/q_t": 0.4011186957359314, "grad_norm": 20.75707244873047, "learning_rate": 3.184157475180207e-07, "logits/chosen": -0.381791889667511, "logits/rejected": -0.3810097575187683, "logps/chosen": -359.649169921875, "logps/ref_chosen": -57.29003143310547, "logps/ref_rejected": -95.74992370605469, "logps/rejected": -641.1409912109375, "loss": 1.0584, "margin_dpo/margin_mean": 243.0318603515625, "margin_dpo/margin_std": 252.65171813964844, "step": 322 }, { "KL/chosen_KL_mean": -329.60552978515625, "KL/mean": -433.67523193359375, "KL/rejected_KL_mean": -537.7449340820312, "KL/std": 240.89913940429688, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.0017628297209739685, "fcm_dpo/delta": 0.04052872955799103, "fcm_dpo/margin": 208.13937377929688, "fcm_dpo/q_t": 0.4153671860694885, "grad_norm": 26.74566078186035, "learning_rate": 3.171805115074251e-07, "logits/chosen": -0.4394986629486084, "logits/rejected": -0.4391285479068756, "logps/chosen": -380.8395080566406, "logps/ref_chosen": -51.23395919799805, "logps/ref_rejected": -75.06192016601562, "logps/rejected": -612.806884765625, "loss": 1.1044, "margin_dpo/margin_mean": 208.13937377929688, "margin_dpo/margin_std": 257.1087951660156, "step": 323 }, { "KL/chosen_KL_mean": -379.00439453125, "KL/mean": -478.7786865234375, "KL/rejected_KL_mean": -578.552978515625, "KL/std": 264.73980712890625, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.0018952081445604563, "fcm_dpo/delta": 0.019777312874794006, "fcm_dpo/margin": 199.5486297607422, "fcm_dpo/q_t": 0.415118932723999, "grad_norm": 40.34514236450195, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -0.446133553981781, "logits/rejected": -0.43301481008529663, "logps/chosen": -444.1395568847656, "logps/ref_chosen": -65.13516998291016, "logps/ref_rejected": -86.47750854492188, "logps/rejected": -665.030517578125, "loss": 1.1304, "margin_dpo/margin_mean": 199.5486297607422, "margin_dpo/margin_std": 308.2591247558594, "step": 324 }, { "KL/chosen_KL_mean": -287.88055419921875, "KL/mean": -370.6061096191406, "KL/rejected_KL_mean": -453.3316650390625, "KL/std": 218.6322021484375, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.001969131175428629, "fcm_dpo/delta": 0.08780458569526672, "fcm_dpo/margin": 165.45111083984375, "fcm_dpo/q_t": 0.4262668192386627, "grad_norm": 25.783239364624023, "learning_rate": 3.147047612756302e-07, "logits/chosen": -0.46015477180480957, "logits/rejected": -0.4445416331291199, "logps/chosen": -344.09613037109375, "logps/ref_chosen": -56.215599060058594, "logps/ref_rejected": -70.08592987060547, "logps/rejected": -523.4176025390625, "loss": 1.1354, "margin_dpo/margin_mean": 165.45111083984375, "margin_dpo/margin_std": 224.27264404296875, "step": 325 }, { "KL/chosen_KL_mean": -307.934814453125, "KL/mean": -389.87255859375, "KL/rejected_KL_mean": -471.8103332519531, "KL/std": 201.8898162841797, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0020997067913413048, "fcm_dpo/delta": 0.06763522326946259, "fcm_dpo/margin": 163.87550354003906, "fcm_dpo/q_t": 0.4207178056240082, "grad_norm": 62.081993103027344, "learning_rate": 3.134643122927519e-07, "logits/chosen": -0.4982990622520447, "logits/rejected": -0.4790222644805908, "logps/chosen": -380.6597595214844, "logps/ref_chosen": -72.72496032714844, "logps/ref_rejected": -79.8467788696289, "logps/rejected": -551.6571044921875, "loss": 1.1107, "margin_dpo/margin_mean": 163.87550354003906, "margin_dpo/margin_std": 188.82302856445312, "step": 326 }, { "KL/chosen_KL_mean": -277.514404296875, "KL/mean": -394.7227783203125, "KL/rejected_KL_mean": -511.93115234375, "KL/std": 218.5492706298828, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.0019716641400009394, "fcm_dpo/delta": -0.08498930931091309, "fcm_dpo/margin": 234.416748046875, "fcm_dpo/q_t": 0.38751423358917236, "grad_norm": 51.83660888671875, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -0.45873162150382996, "logits/rejected": -0.4484935998916626, "logps/chosen": -346.6488037109375, "logps/ref_chosen": -69.13441467285156, "logps/ref_rejected": -111.93377685546875, "logps/rejected": -623.8649291992188, "loss": 1.0338, "margin_dpo/margin_mean": 234.41673278808594, "margin_dpo/margin_std": 248.63563537597656, "step": 327 }, { "KL/chosen_KL_mean": -291.4725341796875, "KL/mean": -397.5861511230469, "KL/rejected_KL_mean": -503.69976806640625, "KL/std": 236.2239990234375, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0019511812133714557, "fcm_dpo/delta": -0.01778026856482029, "fcm_dpo/margin": 212.22723388671875, "fcm_dpo/q_t": 0.4037235677242279, "grad_norm": 34.96001052856445, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -0.4395790100097656, "logits/rejected": -0.4473843276500702, "logps/chosen": -351.15972900390625, "logps/ref_chosen": -59.68719482421875, "logps/ref_rejected": -90.85499572753906, "logps/rejected": -594.5548095703125, "loss": 1.0754, "margin_dpo/margin_mean": 212.22726440429688, "margin_dpo/margin_std": 262.13189697265625, "step": 328 }, { "KL/chosen_KL_mean": -334.9793395996094, "KL/mean": -451.97674560546875, "KL/rejected_KL_mean": -568.97412109375, "KL/std": 270.3482666015625, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0018274824833497405, "fcm_dpo/delta": -0.03972601890563965, "fcm_dpo/margin": 233.9947509765625, "fcm_dpo/q_t": 0.3984408974647522, "grad_norm": 25.084033966064453, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -0.4626386761665344, "logits/rejected": -0.46101057529449463, "logps/chosen": -400.22552490234375, "logps/ref_chosen": -65.2461929321289, "logps/ref_rejected": -100.69770812988281, "logps/rejected": -669.671875, "loss": 1.0719, "margin_dpo/margin_mean": 233.9947509765625, "margin_dpo/margin_std": 285.23626708984375, "step": 329 }, { "KL/chosen_KL_mean": -292.37579345703125, "KL/mean": -412.8188171386719, "KL/rejected_KL_mean": -533.2619018554688, "KL/std": 249.09326171875, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.0017980177653953433, "fcm_dpo/delta": -0.04239234700798988, "fcm_dpo/margin": 240.88613891601562, "fcm_dpo/q_t": 0.3976569175720215, "grad_norm": 20.89375877380371, "learning_rate": 3.084861204504122e-07, "logits/chosen": -0.4702164828777313, "logits/rejected": -0.46967270970344543, "logps/chosen": -339.3741149902344, "logps/ref_chosen": -46.998348236083984, "logps/ref_rejected": -86.87684631347656, "logps/rejected": -620.1387329101562, "loss": 1.0555, "margin_dpo/margin_mean": 240.88613891601562, "margin_dpo/margin_std": 273.5047607421875, "step": 330 }, { "KL/chosen_KL_mean": -330.68896484375, "KL/mean": -452.95233154296875, "KL/rejected_KL_mean": -575.2156982421875, "KL/std": 218.610107421875, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0017413150053471327, "fcm_dpo/delta": -0.032744720578193665, "fcm_dpo/margin": 244.52676391601562, "fcm_dpo/q_t": 0.3974621891975403, "grad_norm": 26.476423263549805, "learning_rate": 3.072376374875335e-07, "logits/chosen": -0.49178826808929443, "logits/rejected": -0.4904711842536926, "logps/chosen": -381.21319580078125, "logps/ref_chosen": -50.52424621582031, "logps/ref_rejected": -89.01544189453125, "logps/rejected": -664.231201171875, "loss": 1.0385, "margin_dpo/margin_mean": 244.52676391601562, "margin_dpo/margin_std": 216.18482971191406, "step": 331 }, { "KL/chosen_KL_mean": -348.40509033203125, "KL/mean": -431.3126220703125, "KL/rejected_KL_mean": -514.2201538085938, "KL/std": 222.2275390625, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.0018814262002706528, "fcm_dpo/delta": 0.10219782590866089, "fcm_dpo/margin": 165.81512451171875, "fcm_dpo/q_t": 0.43078944087028503, "grad_norm": 26.434301376342773, "learning_rate": 3.059876462596758e-07, "logits/chosen": -0.5475494861602783, "logits/rejected": -0.5299344062805176, "logps/chosen": -397.5853576660156, "logps/ref_chosen": -49.18028259277344, "logps/ref_rejected": -76.48515319824219, "logps/rejected": -590.705322265625, "loss": 1.152, "margin_dpo/margin_mean": 165.81512451171875, "margin_dpo/margin_std": 254.50332641601562, "step": 332 }, { "KL/chosen_KL_mean": -377.22314453125, "KL/mean": -493.54986572265625, "KL/rejected_KL_mean": -609.8765869140625, "KL/std": 277.18450927734375, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0017979848198592663, "fcm_dpo/delta": -0.030853308737277985, "fcm_dpo/margin": 232.65338134765625, "fcm_dpo/q_t": 0.40362921357154846, "grad_norm": 32.03265380859375, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -0.5083990097045898, "logits/rejected": -0.5068196654319763, "logps/chosen": -440.9789123535156, "logps/ref_chosen": -63.75574493408203, "logps/ref_rejected": -95.04411315917969, "logps/rejected": -704.920654296875, "loss": 1.1067, "margin_dpo/margin_mean": 232.6533660888672, "margin_dpo/margin_std": 344.5853271484375, "step": 333 }, { "KL/chosen_KL_mean": -350.3871765136719, "KL/mean": -450.59454345703125, "KL/rejected_KL_mean": -550.802001953125, "KL/std": 298.42266845703125, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0018730323063209653, "fcm_dpo/delta": 0.030153676867485046, "fcm_dpo/margin": 200.414794921875, "fcm_dpo/q_t": 0.41732466220855713, "grad_norm": 27.673667907714844, "learning_rate": 3.034832708016243e-07, "logits/chosen": -0.4616066813468933, "logits/rejected": -0.45988792181015015, "logps/chosen": -417.366943359375, "logps/ref_chosen": -66.97975158691406, "logps/ref_rejected": -95.31692504882812, "logps/rejected": -646.118896484375, "loss": 1.1399, "margin_dpo/margin_mean": 200.41477966308594, "margin_dpo/margin_std": 338.0478515625, "step": 334 }, { "KL/chosen_KL_mean": -360.7254333496094, "KL/mean": -446.093017578125, "KL/rejected_KL_mean": -531.4605712890625, "KL/std": 273.3331298828125, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.00204592477530241, "fcm_dpo/delta": 0.05596485733985901, "fcm_dpo/margin": 170.73513793945312, "fcm_dpo/q_t": 0.42263150215148926, "grad_norm": 35.19724655151367, "learning_rate": 3.022289525640531e-07, "logits/chosen": -0.569267749786377, "logits/rejected": -0.5530425310134888, "logps/chosen": -423.2679138183594, "logps/ref_chosen": -62.54248046875, "logps/ref_rejected": -87.61770629882812, "logps/rejected": -619.0782470703125, "loss": 1.15, "margin_dpo/margin_mean": 170.73513793945312, "margin_dpo/margin_std": 286.39031982421875, "step": 335 }, { "KL/chosen_KL_mean": -362.0695495605469, "KL/mean": -486.9052429199219, "KL/rejected_KL_mean": -611.740966796875, "KL/std": 306.3258972167969, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.0019088383996859193, "fcm_dpo/delta": -0.10008023679256439, "fcm_dpo/margin": 249.67137145996094, "fcm_dpo/q_t": 0.3910997807979584, "grad_norm": 27.048595428466797, "learning_rate": 3.009732580450086e-07, "logits/chosen": -0.4783538281917572, "logits/rejected": -0.4791324734687805, "logps/chosen": -416.6007080078125, "logps/ref_chosen": -54.53115463256836, "logps/ref_rejected": -104.40424346923828, "logps/rejected": -716.1451416015625, "loss": 1.078, "margin_dpo/margin_mean": 249.67135620117188, "margin_dpo/margin_std": 367.23291015625, "step": 336 }, { "KL/chosen_KL_mean": -319.1661376953125, "KL/mean": -445.8056335449219, "KL/rejected_KL_mean": -572.445068359375, "KL/std": 243.7745361328125, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.0017321293707937002, "fcm_dpo/delta": -0.051446449011564255, "fcm_dpo/margin": 253.27894592285156, "fcm_dpo/q_t": 0.39632317423820496, "grad_norm": 19.951826095581055, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -0.5001170635223389, "logits/rejected": -0.4860590696334839, "logps/chosen": -384.29486083984375, "logps/ref_chosen": -65.12869262695312, "logps/ref_rejected": -101.72701263427734, "logps/rejected": -674.172119140625, "loss": 1.0528, "margin_dpo/margin_mean": 253.27896118164062, "margin_dpo/margin_std": 288.4793701171875, "step": 337 }, { "KL/chosen_KL_mean": -283.015869140625, "KL/mean": -405.06072998046875, "KL/rejected_KL_mean": -527.1055908203125, "KL/std": 232.23532104492188, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.0016998077044263482, "fcm_dpo/delta": -0.01908147893846035, "fcm_dpo/margin": 244.0897216796875, "fcm_dpo/q_t": 0.40122056007385254, "grad_norm": 33.17329025268555, "learning_rate": 2.984578725527675e-07, "logits/chosen": -0.5003715753555298, "logits/rejected": -0.49724671244621277, "logps/chosen": -341.4385986328125, "logps/ref_chosen": -58.422706604003906, "logps/ref_rejected": -89.06854248046875, "logps/rejected": -616.1741333007812, "loss": 1.0533, "margin_dpo/margin_mean": 244.0897216796875, "margin_dpo/margin_std": 240.3944854736328, "step": 338 }, { "KL/chosen_KL_mean": -306.18310546875, "KL/mean": -420.02911376953125, "KL/rejected_KL_mean": -533.8751220703125, "KL/std": 236.42514038085938, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.0017625135369598866, "fcm_dpo/delta": -0.008848987519741058, "fcm_dpo/margin": 227.69198608398438, "fcm_dpo/q_t": 0.40451204776763916, "grad_norm": 25.832143783569336, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -0.5664355754852295, "logits/rejected": -0.5560377836227417, "logps/chosen": -366.17840576171875, "logps/ref_chosen": -59.99531555175781, "logps/ref_rejected": -103.9109115600586, "logps/rejected": -637.7860107421875, "loss": 1.077, "margin_dpo/margin_mean": 227.69198608398438, "margin_dpo/margin_std": 242.7012481689453, "step": 339 }, { "KL/chosen_KL_mean": -306.2630920410156, "KL/mean": -406.93267822265625, "KL/rejected_KL_mean": -507.6022644042969, "KL/std": 228.15411376953125, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.001739348517730832, "fcm_dpo/delta": 0.06040128692984581, "fcm_dpo/margin": 201.33917236328125, "fcm_dpo/q_t": 0.4204002022743225, "grad_norm": 19.13730812072754, "learning_rate": 2.959373794541426e-07, "logits/chosen": -0.46875977516174316, "logits/rejected": -0.4506962299346924, "logps/chosen": -359.09332275390625, "logps/ref_chosen": -52.83022689819336, "logps/ref_rejected": -73.10723114013672, "logps/rejected": -580.70947265625, "loss": 1.1223, "margin_dpo/margin_mean": 201.33917236328125, "margin_dpo/margin_std": 275.57354736328125, "step": 340 }, { "KL/chosen_KL_mean": -299.728515625, "KL/mean": -421.93115234375, "KL/rejected_KL_mean": -544.1337890625, "KL/std": 255.9670867919922, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.001722757937386632, "fcm_dpo/delta": -0.028678692877292633, "fcm_dpo/margin": 244.40525817871094, "fcm_dpo/q_t": 0.40045589208602905, "grad_norm": 22.329763412475586, "learning_rate": 2.946753005532965e-07, "logits/chosen": -0.4424135684967041, "logits/rejected": -0.4432716369628906, "logps/chosen": -347.6282958984375, "logps/ref_chosen": -47.899803161621094, "logps/ref_rejected": -101.80987548828125, "logps/rejected": -645.9436645507812, "loss": 1.0604, "margin_dpo/margin_mean": 244.40525817871094, "margin_dpo/margin_std": 266.0626525878906, "step": 341 }, { "KL/chosen_KL_mean": -297.13800048828125, "KL/mean": -400.4307861328125, "KL/rejected_KL_mean": -503.7236022949219, "KL/std": 236.86354064941406, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.0018168420065194368, "fcm_dpo/delta": 0.027337031438946724, "fcm_dpo/margin": 206.58563232421875, "fcm_dpo/q_t": 0.41394251585006714, "grad_norm": 22.068641662597656, "learning_rate": 2.934120444167326e-07, "logits/chosen": -0.49012503027915955, "logits/rejected": -0.4645516276359558, "logps/chosen": -369.1346435546875, "logps/ref_chosen": -71.99664306640625, "logps/ref_rejected": -92.58959197998047, "logps/rejected": -596.313232421875, "loss": 1.109, "margin_dpo/margin_mean": 206.5856170654297, "margin_dpo/margin_std": 283.00140380859375, "step": 342 }, { "KL/chosen_KL_mean": -280.635986328125, "KL/mean": -405.8623046875, "KL/rejected_KL_mean": -531.088623046875, "KL/std": 239.86712646484375, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.001725044334307313, "fcm_dpo/delta": -0.0413195863366127, "fcm_dpo/margin": 250.45265197753906, "fcm_dpo/q_t": 0.39542150497436523, "grad_norm": 20.26138687133789, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -0.5200803875923157, "logits/rejected": -0.5258131623268127, "logps/chosen": -335.0416259765625, "logps/ref_chosen": -54.405616760253906, "logps/ref_rejected": -111.04142761230469, "logps/rejected": -642.1300048828125, "loss": 1.0338, "margin_dpo/margin_mean": 250.45265197753906, "margin_dpo/margin_std": 214.74595642089844, "step": 343 }, { "KL/chosen_KL_mean": -282.39642333984375, "KL/mean": -391.85748291015625, "KL/rejected_KL_mean": -501.31854248046875, "KL/std": 262.4002685546875, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.0017953403294086456, "fcm_dpo/delta": -0.00011947751045227051, "fcm_dpo/margin": 218.922119140625, "fcm_dpo/q_t": 0.40945079922676086, "grad_norm": 18.697933197021484, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -0.5233709216117859, "logits/rejected": -0.5257829427719116, "logps/chosen": -336.361083984375, "logps/ref_chosen": -53.96466827392578, "logps/ref_rejected": -90.62336730957031, "logps/rejected": -591.94189453125, "loss": 1.0912, "margin_dpo/margin_mean": 218.922119140625, "margin_dpo/margin_std": 262.0262756347656, "step": 344 }, { "KL/chosen_KL_mean": -341.17315673828125, "KL/mean": -463.88018798828125, "KL/rejected_KL_mean": -586.5872802734375, "KL/std": 246.993408203125, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.0017087084706872702, "fcm_dpo/delta": -0.024968912824988365, "fcm_dpo/margin": 245.41409301757812, "fcm_dpo/q_t": 0.4007132053375244, "grad_norm": 23.478904724121094, "learning_rate": 2.896155456223163e-07, "logits/chosen": -0.5109938383102417, "logits/rejected": -0.5099775791168213, "logps/chosen": -402.85882568359375, "logps/ref_chosen": -61.685699462890625, "logps/ref_rejected": -99.49041748046875, "logps/rejected": -686.0776977539062, "loss": 1.0622, "margin_dpo/margin_mean": 245.41409301757812, "margin_dpo/margin_std": 272.0403137207031, "step": 345 }, { "KL/chosen_KL_mean": -347.7568359375, "KL/mean": -463.9387512207031, "KL/rejected_KL_mean": -580.12060546875, "KL/std": 252.9781494140625, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.001682568108662963, "fcm_dpo/delta": 0.011223375797271729, "fcm_dpo/margin": 232.36383056640625, "fcm_dpo/q_t": 0.40780335664749146, "grad_norm": 23.162200927734375, "learning_rate": 2.883479137196714e-07, "logits/chosen": -0.49171721935272217, "logits/rejected": -0.4840758442878723, "logps/chosen": -403.0130920410156, "logps/ref_chosen": -55.256263732910156, "logps/ref_rejected": -77.41532135009766, "logps/rejected": -657.5359497070312, "loss": 1.0805, "margin_dpo/margin_mean": 232.36383056640625, "margin_dpo/margin_std": 264.86688232421875, "step": 346 }, { "KL/chosen_KL_mean": -359.0545654296875, "KL/mean": -472.8544616699219, "KL/rejected_KL_mean": -586.6543579101562, "KL/std": 266.8034973144531, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.001710890093818307, "fcm_dpo/delta": 0.013139501214027405, "fcm_dpo/margin": 227.59982299804688, "fcm_dpo/q_t": 0.4103218615055084, "grad_norm": 22.08174705505371, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -0.5109871625900269, "logits/rejected": -0.5056133270263672, "logps/chosen": -416.62078857421875, "logps/ref_chosen": -57.56623840332031, "logps/ref_rejected": -92.35509490966797, "logps/rejected": -679.0094604492188, "loss": 1.095, "margin_dpo/margin_mean": 227.59979248046875, "margin_dpo/margin_std": 297.329833984375, "step": 347 }, { "KL/chosen_KL_mean": -317.605712890625, "KL/mean": -419.5076904296875, "KL/rejected_KL_mean": -521.40966796875, "KL/std": 239.0012969970703, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.0017384930979460478, "fcm_dpo/delta": 0.0538782924413681, "fcm_dpo/margin": 203.80392456054688, "fcm_dpo/q_t": 0.41967394948005676, "grad_norm": 22.87740707397461, "learning_rate": 2.858096518347179e-07, "logits/chosen": -0.5410622358322144, "logits/rejected": -0.5414994359016418, "logps/chosen": -373.9234313964844, "logps/ref_chosen": -56.31770324707031, "logps/ref_rejected": -89.13836669921875, "logps/rejected": -610.5479736328125, "loss": 1.122, "margin_dpo/margin_mean": 203.80392456054688, "margin_dpo/margin_std": 276.4021301269531, "step": 348 }, { "KL/chosen_KL_mean": -315.38214111328125, "KL/mean": -426.52508544921875, "KL/rejected_KL_mean": -537.6680297851562, "KL/std": 259.76336669921875, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.0018351702019572258, "fcm_dpo/delta": -0.011926159262657166, "fcm_dpo/margin": 222.285888671875, "fcm_dpo/q_t": 0.407881498336792, "grad_norm": 21.936599731445312, "learning_rate": 2.845390887379706e-07, "logits/chosen": -0.5372369289398193, "logits/rejected": -0.5374791622161865, "logps/chosen": -373.40765380859375, "logps/ref_chosen": -58.025516510009766, "logps/ref_rejected": -97.50515747070312, "logps/rejected": -635.1732177734375, "loss": 1.1019, "margin_dpo/margin_mean": 222.28587341308594, "margin_dpo/margin_std": 322.0589904785156, "step": 349 }, { "KL/chosen_KL_mean": -336.73089599609375, "KL/mean": -447.26806640625, "KL/rejected_KL_mean": -557.8052978515625, "KL/std": 256.64642333984375, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.001765610184520483, "fcm_dpo/delta": 0.010335829108953476, "fcm_dpo/margin": 221.074462890625, "fcm_dpo/q_t": 0.40952080488204956, "grad_norm": 30.984949111938477, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -0.5725632905960083, "logits/rejected": -0.5793622136116028, "logps/chosen": -401.0613708496094, "logps/ref_chosen": -64.33049011230469, "logps/ref_rejected": -89.87164306640625, "logps/rejected": -647.677001953125, "loss": 1.1081, "margin_dpo/margin_mean": 221.074462890625, "margin_dpo/margin_std": 314.63995361328125, "step": 350 }, { "KL/chosen_KL_mean": -301.34967041015625, "KL/mean": -436.94683837890625, "KL/rejected_KL_mean": -572.5440673828125, "KL/std": 292.54278564453125, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0016822493635118008, "fcm_dpo/delta": -0.07549858838319778, "fcm_dpo/margin": 271.1943664550781, "fcm_dpo/q_t": 0.3937299847602844, "grad_norm": 28.201416015625, "learning_rate": 2.819952656376487e-07, "logits/chosen": -0.5415765047073364, "logits/rejected": -0.5444436073303223, "logps/chosen": -362.02178955078125, "logps/ref_chosen": -60.6721305847168, "logps/ref_rejected": -101.5654296875, "logps/rejected": -674.1094970703125, "loss": 1.0575, "margin_dpo/margin_mean": 271.19439697265625, "margin_dpo/margin_std": 342.4002685546875, "step": 351 }, { "KL/chosen_KL_mean": -353.643798828125, "KL/mean": -438.07720947265625, "KL/rejected_KL_mean": -522.5106811523438, "KL/std": 269.5029602050781, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.0018008626066148281, "fcm_dpo/delta": 0.11381582170724869, "fcm_dpo/margin": 168.86691284179688, "fcm_dpo/q_t": 0.4331480860710144, "grad_norm": 61.400474548339844, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -0.5702152252197266, "logits/rejected": -0.5352950096130371, "logps/chosen": -424.58721923828125, "logps/ref_chosen": -70.9434585571289, "logps/ref_rejected": -76.6419677734375, "logps/rejected": -599.1526489257812, "loss": 1.1724, "margin_dpo/margin_mean": 168.86691284179688, "margin_dpo/margin_std": 291.0610656738281, "step": 352 }, { "KL/chosen_KL_mean": -321.31304931640625, "KL/mean": -428.07708740234375, "KL/rejected_KL_mean": -534.8411865234375, "KL/std": 256.8054504394531, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.0018330267630517483, "fcm_dpo/delta": 0.006989631801843643, "fcm_dpo/margin": 213.528076171875, "fcm_dpo/q_t": 0.4106243848800659, "grad_norm": 29.105669021606445, "learning_rate": 2.794480701395219e-07, "logits/chosen": -0.6194638609886169, "logits/rejected": -0.604763925075531, "logps/chosen": -379.7083740234375, "logps/ref_chosen": -58.39533996582031, "logps/ref_rejected": -80.33553314208984, "logps/rejected": -615.1766967773438, "loss": 1.104, "margin_dpo/margin_mean": 213.528076171875, "margin_dpo/margin_std": 291.9771728515625, "step": 353 }, { "KL/chosen_KL_mean": -276.51214599609375, "KL/mean": -391.6829833984375, "KL/rejected_KL_mean": -506.8538513183594, "KL/std": 234.94125366210938, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.0018385781440883875, "fcm_dpo/delta": -0.030174316838383675, "fcm_dpo/margin": 230.3416748046875, "fcm_dpo/q_t": 0.39935770630836487, "grad_norm": 32.30131149291992, "learning_rate": 2.781732916288303e-07, "logits/chosen": -0.6025009155273438, "logits/rejected": -0.5897366404533386, "logps/chosen": -336.3151550292969, "logps/ref_chosen": -59.80299377441406, "logps/ref_rejected": -88.75750732421875, "logps/rejected": -595.611328125, "loss": 1.0512, "margin_dpo/margin_mean": 230.3416748046875, "margin_dpo/margin_std": 236.0069580078125, "step": 354 }, { "KL/chosen_KL_mean": -267.94769287109375, "KL/mean": -392.4739685058594, "KL/rejected_KL_mean": -517.000244140625, "KL/std": 249.90625, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0017431924352422357, "fcm_dpo/delta": -0.045911140739917755, "fcm_dpo/margin": 249.0526123046875, "fcm_dpo/q_t": 0.3952483534812927, "grad_norm": 41.20962905883789, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -0.6518046855926514, "logits/rejected": -0.6433999538421631, "logps/chosen": -322.076171875, "logps/ref_chosen": -54.12849807739258, "logps/ref_rejected": -82.40606689453125, "logps/rejected": -599.4063110351562, "loss": 1.0418, "margin_dpo/margin_mean": 249.05262756347656, "margin_dpo/margin_std": 242.5153045654297, "step": 355 }, { "KL/chosen_KL_mean": -348.6526184082031, "KL/mean": -410.7809753417969, "KL/rejected_KL_mean": -472.9093322753906, "KL/std": 255.02890014648438, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.0017826826078817248, "fcm_dpo/delta": 0.03428832069039345, "fcm_dpo/margin": 124.2567367553711, "fcm_dpo/q_t": 0.45051053166389465, "grad_norm": 29.759109497070312, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -0.5975973010063171, "logits/rejected": -0.57161945104599, "logps/chosen": -413.326416015625, "logps/ref_chosen": -64.6738052368164, "logps/ref_rejected": -75.89926147460938, "logps/rejected": -548.80859375, "loss": 1.2561, "margin_dpo/margin_mean": 124.25672912597656, "margin_dpo/margin_std": 320.7979736328125, "step": 356 }, { "KL/chosen_KL_mean": -309.07611083984375, "KL/mean": -413.7547607421875, "KL/rejected_KL_mean": -518.4334106445312, "KL/std": 256.1674499511719, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0018109192606061697, "fcm_dpo/delta": 0.02454628050327301, "fcm_dpo/margin": 209.35728454589844, "fcm_dpo/q_t": 0.41207998991012573, "grad_norm": 25.09831428527832, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -0.5878846645355225, "logits/rejected": -0.5737650990486145, "logps/chosen": -361.80194091796875, "logps/ref_chosen": -52.725799560546875, "logps/ref_rejected": -86.84115600585938, "logps/rejected": -605.2745361328125, "loss": 1.0991, "margin_dpo/margin_mean": 209.3572998046875, "margin_dpo/margin_std": 263.3780517578125, "step": 357 }, { "KL/chosen_KL_mean": -287.3702697753906, "KL/mean": -386.8647155761719, "KL/rejected_KL_mean": -486.3591613769531, "KL/std": 241.16976928710938, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0018941482994705439, "fcm_dpo/delta": 0.0284799225628376, "fcm_dpo/margin": 198.98886108398438, "fcm_dpo/q_t": 0.4145725965499878, "grad_norm": 26.890031814575195, "learning_rate": 2.730670898658255e-07, "logits/chosen": -0.5883272886276245, "logits/rejected": -0.5693084001541138, "logps/chosen": -350.5757141113281, "logps/ref_chosen": -63.20543670654297, "logps/ref_rejected": -88.373291015625, "logps/rejected": -574.732421875, "loss": 1.1049, "margin_dpo/margin_mean": 198.98886108398438, "margin_dpo/margin_std": 268.44134521484375, "step": 358 }, { "KL/chosen_KL_mean": -328.99090576171875, "KL/mean": -444.7538146972656, "KL/rejected_KL_mean": -560.5167236328125, "KL/std": 247.1792755126953, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0018442978616803885, "fcm_dpo/delta": -0.0352255180478096, "fcm_dpo/margin": 231.5258331298828, "fcm_dpo/q_t": 0.4013112187385559, "grad_norm": 33.82530212402344, "learning_rate": 2.717889356869146e-07, "logits/chosen": -0.5302337408065796, "logits/rejected": -0.5221052765846252, "logps/chosen": -385.3611145019531, "logps/ref_chosen": -56.370216369628906, "logps/ref_rejected": -82.17375183105469, "logps/rejected": -642.6904907226562, "loss": 1.0792, "margin_dpo/margin_mean": 231.5258331298828, "margin_dpo/margin_std": 305.66802978515625, "step": 359 }, { "KL/chosen_KL_mean": -329.69427490234375, "KL/mean": -418.23699951171875, "KL/rejected_KL_mean": -506.77972412109375, "KL/std": 206.728271484375, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.0019492888823151588, "fcm_dpo/delta": 0.0655849277973175, "fcm_dpo/margin": 177.08547973632812, "fcm_dpo/q_t": 0.420721173286438, "grad_norm": 44.412681579589844, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -0.5013865828514099, "logits/rejected": -0.48618659377098083, "logps/chosen": -381.1546630859375, "logps/ref_chosen": -51.460384368896484, "logps/ref_rejected": -69.83892059326172, "logps/rejected": -576.61865234375, "loss": 1.1116, "margin_dpo/margin_mean": 177.08547973632812, "margin_dpo/margin_std": 211.73977661132812, "step": 360 }, { "KL/chosen_KL_mean": -342.9272766113281, "KL/mean": -429.4185791015625, "KL/rejected_KL_mean": -515.9098510742188, "KL/std": 240.47056579589844, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.0020752213895320892, "fcm_dpo/delta": 0.047938957810401917, "fcm_dpo/margin": 172.98260498046875, "fcm_dpo/q_t": 0.41855740547180176, "grad_norm": 35.09357833862305, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -0.5015807747840881, "logits/rejected": -0.49445807933807373, "logps/chosen": -396.79681396484375, "logps/ref_chosen": -53.86951446533203, "logps/ref_rejected": -90.7692642211914, "logps/rejected": -606.6791381835938, "loss": 1.1306, "margin_dpo/margin_mean": 172.9825897216797, "margin_dpo/margin_std": 263.9880676269531, "step": 361 }, { "KL/chosen_KL_mean": -294.02984619140625, "KL/mean": -431.71832275390625, "KL/rejected_KL_mean": -569.40673828125, "KL/std": 248.303955078125, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0018198469188064337, "fcm_dpo/delta": -0.14192384481430054, "fcm_dpo/margin": 275.376953125, "fcm_dpo/q_t": 0.37714123725891113, "grad_norm": 30.883852005004883, "learning_rate": 2.679511629948319e-07, "logits/chosen": -0.5016046166419983, "logits/rejected": -0.5126606225967407, "logps/chosen": -352.66888427734375, "logps/ref_chosen": -58.639060974121094, "logps/ref_rejected": -105.58195495605469, "logps/rejected": -674.9887084960938, "loss": 0.9979, "margin_dpo/margin_mean": 275.376953125, "margin_dpo/margin_std": 260.5057373046875, "step": 362 }, { "KL/chosen_KL_mean": -272.70916748046875, "KL/mean": -416.3315124511719, "KL/rejected_KL_mean": -559.953857421875, "KL/std": 247.9210205078125, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.0016754900570958853, "fcm_dpo/delta": -0.10649609565734863, "fcm_dpo/margin": 287.24468994140625, "fcm_dpo/q_t": 0.3827672302722931, "grad_norm": 27.67137908935547, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -0.4572371542453766, "logits/rejected": -0.4535822868347168, "logps/chosen": -317.26751708984375, "logps/ref_chosen": -44.558380126953125, "logps/ref_rejected": -74.69496154785156, "logps/rejected": -634.6488037109375, "loss": 1.0097, "margin_dpo/margin_mean": 287.24468994140625, "margin_dpo/margin_std": 268.51934814453125, "step": 363 }, { "KL/chosen_KL_mean": -304.93585205078125, "KL/mean": -410.62762451171875, "KL/rejected_KL_mean": -516.3193969726562, "KL/std": 243.23239135742188, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0017072297632694244, "fcm_dpo/delta": 0.043868713080883026, "fcm_dpo/margin": 211.38351440429688, "fcm_dpo/q_t": 0.4163602590560913, "grad_norm": 26.54780387878418, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -0.4880487322807312, "logits/rejected": -0.49908387660980225, "logps/chosen": -353.8304443359375, "logps/ref_chosen": -48.894622802734375, "logps/ref_rejected": -91.395751953125, "logps/rejected": -607.7151489257812, "loss": 1.1142, "margin_dpo/margin_mean": 211.38351440429688, "margin_dpo/margin_std": 280.0267028808594, "step": 364 }, { "KL/chosen_KL_mean": -315.26373291015625, "KL/mean": -426.0964050292969, "KL/rejected_KL_mean": -536.9290771484375, "KL/std": 257.34033203125, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0017189306672662497, "fcm_dpo/delta": 0.0233314111828804, "fcm_dpo/margin": 221.66537475585938, "fcm_dpo/q_t": 0.41273248195648193, "grad_norm": 22.404428482055664, "learning_rate": 2.641091331089811e-07, "logits/chosen": -0.5078902244567871, "logits/rejected": -0.5193514227867126, "logps/chosen": -366.7564697265625, "logps/ref_chosen": -51.49274444580078, "logps/ref_rejected": -92.70166778564453, "logps/rejected": -629.6307373046875, "loss": 1.0905, "margin_dpo/margin_mean": 221.66537475585938, "margin_dpo/margin_std": 264.7396240234375, "step": 365 }, { "KL/chosen_KL_mean": -298.6568603515625, "KL/mean": -417.4523010253906, "KL/rejected_KL_mean": -536.2477416992188, "KL/std": 257.2602233886719, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.0016834220150485635, "fcm_dpo/delta": -0.002375733107328415, "fcm_dpo/margin": 237.59083557128906, "fcm_dpo/q_t": 0.4071364402770996, "grad_norm": 26.219554901123047, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -0.49354439973831177, "logits/rejected": -0.507148027420044, "logps/chosen": -343.37744140625, "logps/ref_chosen": -44.7205696105957, "logps/ref_rejected": -83.31040954589844, "logps/rejected": -619.55810546875, "loss": 1.0866, "margin_dpo/margin_mean": 237.59083557128906, "margin_dpo/margin_std": 297.007080078125, "step": 366 }, { "KL/chosen_KL_mean": -325.56622314453125, "KL/mean": -423.55853271484375, "KL/rejected_KL_mean": -521.5508422851562, "KL/std": 235.90292358398438, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.001811149064451456, "fcm_dpo/delta": 0.053424857556819916, "fcm_dpo/margin": 195.98464965820312, "fcm_dpo/q_t": 0.41807931661605835, "grad_norm": 25.911218643188477, "learning_rate": 2.615458646614349e-07, "logits/chosen": -0.5182558298110962, "logits/rejected": -0.5040343999862671, "logps/chosen": -383.97161865234375, "logps/ref_chosen": -58.405418395996094, "logps/ref_rejected": -76.75132751464844, "logps/rejected": -598.3021850585938, "loss": 1.1129, "margin_dpo/margin_mean": 195.98464965820312, "margin_dpo/margin_std": 253.58770751953125, "step": 367 }, { "KL/chosen_KL_mean": -320.83441162109375, "KL/mean": -478.84674072265625, "KL/rejected_KL_mean": -636.8590087890625, "KL/std": 274.4027099609375, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.001620815135538578, "fcm_dpo/delta": -0.1518523395061493, "fcm_dpo/margin": 316.0246276855469, "fcm_dpo/q_t": 0.3704509735107422, "grad_norm": 30.158796310424805, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -0.5701684951782227, "logits/rejected": -0.5831023454666138, "logps/chosen": -365.28692626953125, "logps/ref_chosen": -44.452518463134766, "logps/ref_rejected": -98.55526733398438, "logps/rejected": -735.414306640625, "loss": 0.9731, "margin_dpo/margin_mean": 316.02459716796875, "margin_dpo/margin_std": 235.96087646484375, "step": 368 }, { "KL/chosen_KL_mean": -417.2987976074219, "KL/mean": -500.9288330078125, "KL/rejected_KL_mean": -584.5588989257812, "KL/std": 269.81658935546875, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0015922598540782928, "fcm_dpo/delta": 0.041051216423511505, "fcm_dpo/margin": 167.26007080078125, "fcm_dpo/q_t": 0.4384276270866394, "grad_norm": 24.34749984741211, "learning_rate": 2.589813792448196e-07, "logits/chosen": -0.5948071479797363, "logits/rejected": -0.575736403465271, "logps/chosen": -488.6802978515625, "logps/ref_chosen": -71.38150024414062, "logps/ref_rejected": -91.29582214355469, "logps/rejected": -675.854736328125, "loss": 1.1934, "margin_dpo/margin_mean": 167.26007080078125, "margin_dpo/margin_std": 302.8238525390625, "step": 369 }, { "KL/chosen_KL_mean": -449.61785888671875, "KL/mean": -520.163330078125, "KL/rejected_KL_mean": -590.7088623046875, "KL/std": 294.0730895996094, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.0016860844334587455, "fcm_dpo/delta": 0.07256323844194412, "fcm_dpo/margin": 141.0910186767578, "fcm_dpo/q_t": 0.4467887282371521, "grad_norm": 34.77388000488281, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -0.6763471364974976, "logits/rejected": -0.6677573323249817, "logps/chosen": -521.225341796875, "logps/ref_chosen": -71.60749816894531, "logps/ref_rejected": -97.25978088378906, "logps/rejected": -687.9686279296875, "loss": 1.2396, "margin_dpo/margin_mean": 141.0910186767578, "margin_dpo/margin_std": 338.9228515625, "step": 370 }, { "KL/chosen_KL_mean": -435.55035400390625, "KL/mean": -551.7735595703125, "KL/rejected_KL_mean": -667.9967041015625, "KL/std": 304.8518371582031, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.0017521032132208347, "fcm_dpo/delta": -0.009448290802538395, "fcm_dpo/margin": 232.4463348388672, "fcm_dpo/q_t": 0.40719670057296753, "grad_norm": 27.79642105102539, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -0.7335249185562134, "logits/rejected": -0.720642626285553, "logps/chosen": -504.96484375, "logps/ref_chosen": -69.41448974609375, "logps/ref_rejected": -99.17217254638672, "logps/rejected": -767.1689453125, "loss": 1.099, "margin_dpo/margin_mean": 232.4463348388672, "margin_dpo/margin_std": 332.423095703125, "step": 371 }, { "KL/chosen_KL_mean": -432.0025939941406, "KL/mean": -568.457763671875, "KL/rejected_KL_mean": -704.9129638671875, "KL/std": 349.4569396972656, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.001637202687561512, "fcm_dpo/delta": -0.061564333736896515, "fcm_dpo/margin": 272.91033935546875, "fcm_dpo/q_t": 0.3999387323856354, "grad_norm": 26.669132232666016, "learning_rate": 2.551329606220976e-07, "logits/chosen": -0.7301241755485535, "logits/rejected": -0.7251428365707397, "logps/chosen": -493.8205871582031, "logps/ref_chosen": -61.8179931640625, "logps/ref_rejected": -78.53948974609375, "logps/rejected": -783.452392578125, "loss": 1.0802, "margin_dpo/margin_mean": 272.9103088378906, "margin_dpo/margin_std": 398.3546142578125, "step": 372 }, { "KL/chosen_KL_mean": -448.3118896484375, "KL/mean": -581.019775390625, "KL/rejected_KL_mean": -713.7276000976562, "KL/std": 327.57122802734375, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.0016204612329602242, "fcm_dpo/delta": -0.039967313408851624, "fcm_dpo/margin": 265.41571044921875, "fcm_dpo/q_t": 0.39898842573165894, "grad_norm": 32.85871505737305, "learning_rate": 2.538498388222517e-07, "logits/chosen": -0.7037230730056763, "logits/rejected": -0.6890777349472046, "logps/chosen": -512.529052734375, "logps/ref_chosen": -64.21713256835938, "logps/ref_rejected": -85.95960998535156, "logps/rejected": -799.687255859375, "loss": 1.0727, "margin_dpo/margin_mean": 265.41571044921875, "margin_dpo/margin_std": 323.6532897949219, "step": 373 }, { "KL/chosen_KL_mean": -407.01507568359375, "KL/mean": -524.6700439453125, "KL/rejected_KL_mean": -642.324951171875, "KL/std": 347.67022705078125, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.0015636862954124808, "fcm_dpo/delta": 0.03692319244146347, "fcm_dpo/margin": 235.3098602294922, "fcm_dpo/q_t": 0.42001497745513916, "grad_norm": 27.749187469482422, "learning_rate": 2.525666155755725e-07, "logits/chosen": -0.7614662647247314, "logits/rejected": -0.7477431297302246, "logps/chosen": -477.6652526855469, "logps/ref_chosen": -70.65018463134766, "logps/ref_rejected": -93.64016723632812, "logps/rejected": -735.965087890625, "loss": 1.1424, "margin_dpo/margin_mean": 235.3098602294922, "margin_dpo/margin_std": 396.4349060058594, "step": 374 }, { "KL/chosen_KL_mean": -424.40948486328125, "KL/mean": -535.830078125, "KL/rejected_KL_mean": -647.2506713867188, "KL/std": 291.63531494140625, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0016217189840972424, "fcm_dpo/delta": 0.04367804527282715, "fcm_dpo/margin": 222.84117126464844, "fcm_dpo/q_t": 0.418157160282135, "grad_norm": 41.795082092285156, "learning_rate": 2.512833246961859e-07, "logits/chosen": -0.6727270483970642, "logits/rejected": -0.6715774536132812, "logps/chosen": -484.4897155761719, "logps/ref_chosen": -60.080223083496094, "logps/ref_rejected": -88.93830871582031, "logps/rejected": -736.18896484375, "loss": 1.1357, "margin_dpo/margin_mean": 222.84115600585938, "margin_dpo/margin_std": 338.76904296875, "step": 375 }, { "KL/chosen_KL_mean": -414.8541259765625, "KL/mean": -552.9400634765625, "KL/rejected_KL_mean": -691.0260009765625, "KL/std": 303.83843994140625, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0016104718670248985, "fcm_dpo/delta": -0.058481134474277496, "fcm_dpo/margin": 276.171875, "fcm_dpo/q_t": 0.39490610361099243, "grad_norm": 31.177749633789062, "learning_rate": 2.5e-07, "logits/chosen": -0.7422023415565491, "logits/rejected": -0.7344397306442261, "logps/chosen": -477.5144348144531, "logps/ref_chosen": -62.660308837890625, "logps/ref_rejected": -105.52660369873047, "logps/rejected": -796.5526123046875, "loss": 1.0616, "margin_dpo/margin_mean": 276.171875, "margin_dpo/margin_std": 342.929443359375, "step": 376 }, { "KL/chosen_KL_mean": -424.4516906738281, "KL/mean": -556.843994140625, "KL/rejected_KL_mean": -689.2363891601562, "KL/std": 344.01544189453125, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.0015855635283514857, "fcm_dpo/delta": -0.02529209665954113, "fcm_dpo/margin": 264.78466796875, "fcm_dpo/q_t": 0.40445077419281006, "grad_norm": 25.540964126586914, "learning_rate": 2.487166753038141e-07, "logits/chosen": -0.6894611120223999, "logits/rejected": -0.6936970949172974, "logps/chosen": -478.930419921875, "logps/ref_chosen": -54.478736877441406, "logps/ref_rejected": -98.70335388183594, "logps/rejected": -787.939697265625, "loss": 1.0868, "margin_dpo/margin_mean": 264.78466796875, "margin_dpo/margin_std": 363.934326171875, "step": 377 }, { "KL/chosen_KL_mean": -386.81268310546875, "KL/mean": -535.3828735351562, "KL/rejected_KL_mean": -683.9530029296875, "KL/std": 294.50201416015625, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.001496739685535431, "fcm_dpo/delta": -0.057502150535583496, "fcm_dpo/margin": 297.14031982421875, "fcm_dpo/q_t": 0.3941471576690674, "grad_norm": 38.976097106933594, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -0.7334337830543518, "logits/rejected": -0.7559584379196167, "logps/chosen": -431.8332214355469, "logps/ref_chosen": -45.02053451538086, "logps/ref_rejected": -88.0469741821289, "logps/rejected": -772.0, "loss": 1.0487, "margin_dpo/margin_mean": 297.14031982421875, "margin_dpo/margin_std": 334.1382141113281, "step": 378 }, { "KL/chosen_KL_mean": -417.34716796875, "KL/mean": -559.3892211914062, "KL/rejected_KL_mean": -701.4312744140625, "KL/std": 313.11883544921875, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.001429541502147913, "fcm_dpo/delta": -0.010058283805847168, "fcm_dpo/margin": 284.0840759277344, "fcm_dpo/q_t": 0.4058372974395752, "grad_norm": 29.72431182861328, "learning_rate": 2.461501611777483e-07, "logits/chosen": -0.6593658328056335, "logits/rejected": -0.6784616112709045, "logps/chosen": -470.52923583984375, "logps/ref_chosen": -53.182098388671875, "logps/ref_rejected": -114.3001708984375, "logps/rejected": -815.7314453125, "loss": 1.0836, "margin_dpo/margin_mean": 284.0840759277344, "margin_dpo/margin_std": 356.0497131347656, "step": 379 }, { "KL/chosen_KL_mean": -403.3964538574219, "KL/mean": -557.42529296875, "KL/rejected_KL_mean": -711.4541625976562, "KL/std": 354.645751953125, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.0014169125352054834, "fcm_dpo/delta": -0.04655863344669342, "fcm_dpo/margin": 308.05767822265625, "fcm_dpo/q_t": 0.39721158146858215, "grad_norm": 28.100101470947266, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -0.632209300994873, "logits/rejected": -0.664146900177002, "logps/chosen": -454.7494812011719, "logps/ref_chosen": -51.3530387878418, "logps/ref_rejected": -104.19169616699219, "logps/rejected": -815.6458740234375, "loss": 1.0674, "margin_dpo/margin_mean": 308.05767822265625, "margin_dpo/margin_std": 388.6807861328125, "step": 380 }, { "KL/chosen_KL_mean": -407.2333068847656, "KL/mean": -519.652587890625, "KL/rejected_KL_mean": -632.0718994140625, "KL/std": 297.33251953125, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.0014676698483526707, "fcm_dpo/delta": 0.08423132449388504, "fcm_dpo/margin": 224.83859252929688, "fcm_dpo/q_t": 0.4268224537372589, "grad_norm": 25.78670883178711, "learning_rate": 2.435840528363426e-07, "logits/chosen": -0.6134881973266602, "logits/rejected": -0.5925121307373047, "logps/chosen": -465.036376953125, "logps/ref_chosen": -57.80306625366211, "logps/ref_rejected": -79.21940612792969, "logps/rejected": -711.291259765625, "loss": 1.1633, "margin_dpo/margin_mean": 224.83859252929688, "margin_dpo/margin_std": 391.60101318359375, "step": 381 }, { "KL/chosen_KL_mean": -392.236572265625, "KL/mean": -526.9730834960938, "KL/rejected_KL_mean": -661.7095947265625, "KL/std": 286.292724609375, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0015065961051732302, "fcm_dpo/delta": -0.007509663701057434, "fcm_dpo/margin": 269.4730529785156, "fcm_dpo/q_t": 0.40533730387687683, "grad_norm": 28.11603546142578, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -0.6188876628875732, "logits/rejected": -0.6265856027603149, "logps/chosen": -458.2568359375, "logps/ref_chosen": -66.02030181884766, "logps/ref_rejected": -110.71016693115234, "logps/rejected": -772.4197387695312, "loss": 1.0722, "margin_dpo/margin_mean": 269.4730224609375, "margin_dpo/margin_std": 310.5892333984375, "step": 382 }, { "KL/chosen_KL_mean": -413.4278869628906, "KL/mean": -549.6930541992188, "KL/rejected_KL_mean": -685.958251953125, "KL/std": 322.25799560546875, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0014835065230727196, "fcm_dpo/delta": -0.005721503868699074, "fcm_dpo/margin": 272.53033447265625, "fcm_dpo/q_t": 0.4068153500556946, "grad_norm": 22.861156463623047, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -0.5609561204910278, "logits/rejected": -0.5730615854263306, "logps/chosen": -463.81939697265625, "logps/ref_chosen": -50.39148712158203, "logps/ref_rejected": -93.71589660644531, "logps/rejected": -779.674072265625, "loss": 1.1028, "margin_dpo/margin_mean": 272.53033447265625, "margin_dpo/margin_std": 395.47784423828125, "step": 383 }, { "KL/chosen_KL_mean": -410.1112060546875, "KL/mean": -516.384033203125, "KL/rejected_KL_mean": -622.6568603515625, "KL/std": 280.2078857421875, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.001605054596439004, "fcm_dpo/delta": 0.06686155498027802, "fcm_dpo/margin": 212.545654296875, "fcm_dpo/q_t": 0.42119306325912476, "grad_norm": 24.116987228393555, "learning_rate": 2.397362428170992e-07, "logits/chosen": -0.5835964679718018, "logits/rejected": -0.5749599933624268, "logps/chosen": -462.15728759765625, "logps/ref_chosen": -52.046104431152344, "logps/ref_rejected": -85.76089477539062, "logps/rejected": -708.417724609375, "loss": 1.1212, "margin_dpo/margin_mean": 212.54566955566406, "margin_dpo/margin_std": 272.0644226074219, "step": 384 }, { "KL/chosen_KL_mean": -378.491455078125, "KL/mean": -509.3629150390625, "KL/rejected_KL_mean": -640.234375, "KL/std": 248.0293426513672, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.0015787326265126467, "fcm_dpo/delta": -0.016653146594762802, "fcm_dpo/margin": 261.7428894042969, "fcm_dpo/q_t": 0.4017704725265503, "grad_norm": 38.663368225097656, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -0.6341022253036499, "logits/rejected": -0.6096962690353394, "logps/chosen": -444.04364013671875, "logps/ref_chosen": -65.55215454101562, "logps/ref_rejected": -77.82792663574219, "logps/rejected": -718.062255859375, "loss": 1.0631, "margin_dpo/margin_mean": 261.7428894042969, "margin_dpo/margin_std": 283.84356689453125, "step": 385 }, { "KL/chosen_KL_mean": -397.4541015625, "KL/mean": -538.3778686523438, "KL/rejected_KL_mean": -679.3016357421875, "KL/std": 300.32208251953125, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0015297075733542442, "fcm_dpo/delta": -0.039631377905607224, "fcm_dpo/margin": 281.8475341796875, "fcm_dpo/q_t": 0.39953434467315674, "grad_norm": 25.86907196044922, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -0.5929363965988159, "logits/rejected": -0.5882759094238281, "logps/chosen": -455.67596435546875, "logps/ref_chosen": -58.22185516357422, "logps/ref_rejected": -92.32742309570312, "logps/rejected": -771.6290283203125, "loss": 1.0695, "margin_dpo/margin_mean": 281.8475341796875, "margin_dpo/margin_std": 355.7461242675781, "step": 386 }, { "KL/chosen_KL_mean": -450.69427490234375, "KL/mean": -571.585693359375, "KL/rejected_KL_mean": -692.4771728515625, "KL/std": 309.3702392578125, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0015234483871608973, "fcm_dpo/delta": 0.03800199180841446, "fcm_dpo/margin": 241.78292846679688, "fcm_dpo/q_t": 0.4150587320327759, "grad_norm": 32.03988265991211, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -0.6444403529167175, "logits/rejected": -0.6319328546524048, "logps/chosen": -517.1137084960938, "logps/ref_chosen": -66.41944885253906, "logps/ref_rejected": -92.16915893554688, "logps/rejected": -784.6463623046875, "loss": 1.1086, "margin_dpo/margin_mean": 241.78292846679688, "margin_dpo/margin_std": 314.6268310546875, "step": 387 }, { "KL/chosen_KL_mean": -483.2527770996094, "KL/mean": -631.25, "KL/rejected_KL_mean": -779.247314453125, "KL/std": 369.490234375, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.001472370233386755, "fcm_dpo/delta": -0.05027584731578827, "fcm_dpo/margin": 295.9945068359375, "fcm_dpo/q_t": 0.40026527643203735, "grad_norm": 68.39986419677734, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -0.6584256887435913, "logits/rejected": -0.6686611175537109, "logps/chosen": -533.3822021484375, "logps/ref_chosen": -50.129459381103516, "logps/ref_rejected": -104.43305969238281, "logps/rejected": -883.6803588867188, "loss": 1.0768, "margin_dpo/margin_mean": 295.9945068359375, "margin_dpo/margin_std": 401.4984130859375, "step": 388 }, { "KL/chosen_KL_mean": -505.9647521972656, "KL/mean": -641.0068359375, "KL/rejected_KL_mean": -776.0489501953125, "KL/std": 349.13677978515625, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.001461312174797058, "fcm_dpo/delta": 0.00432373583316803, "fcm_dpo/margin": 270.084228515625, "fcm_dpo/q_t": 0.4113960266113281, "grad_norm": 51.183631896972656, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -0.6953232288360596, "logits/rejected": -0.696498453617096, "logps/chosen": -563.871337890625, "logps/ref_chosen": -57.906593322753906, "logps/ref_rejected": -77.91454315185547, "logps/rejected": -853.9635009765625, "loss": 1.1163, "margin_dpo/margin_mean": 270.084228515625, "margin_dpo/margin_std": 410.4221496582031, "step": 389 }, { "KL/chosen_KL_mean": -545.4517211914062, "KL/mean": -687.8853149414062, "KL/rejected_KL_mean": -830.3189086914062, "KL/std": 405.5350646972656, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.0014416101621463895, "fcm_dpo/delta": -0.01683669537305832, "fcm_dpo/margin": 284.8671875, "fcm_dpo/q_t": 0.4116922616958618, "grad_norm": 30.454208374023438, "learning_rate": 2.320488370051681e-07, "logits/chosen": -0.614460825920105, "logits/rejected": -0.6173591017723083, "logps/chosen": -594.6776123046875, "logps/ref_chosen": -49.22591781616211, "logps/ref_rejected": -85.5281982421875, "logps/rejected": -915.8471069335938, "loss": 1.127, "margin_dpo/margin_mean": 284.8671875, "margin_dpo/margin_std": 498.7476806640625, "step": 390 }, { "KL/chosen_KL_mean": -546.4203491210938, "KL/mean": -634.4364013671875, "KL/rejected_KL_mean": -722.452392578125, "KL/std": 375.4483337402344, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.0016221825499087572, "fcm_dpo/delta": 0.13364286720752716, "fcm_dpo/margin": 176.031982421875, "fcm_dpo/q_t": 0.4394233822822571, "grad_norm": 55.282936096191406, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -0.6284788846969604, "logits/rejected": -0.6224967241287231, "logps/chosen": -610.75, "logps/ref_chosen": -64.32965087890625, "logps/ref_rejected": -86.73820495605469, "logps/rejected": -809.1905517578125, "loss": 1.2094, "margin_dpo/margin_mean": 176.03196716308594, "margin_dpo/margin_std": 371.1688537597656, "step": 391 }, { "KL/chosen_KL_mean": -451.90289306640625, "KL/mean": -614.0924072265625, "KL/rejected_KL_mean": -776.2818603515625, "KL/std": 364.5159912109375, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.001549735781736672, "fcm_dpo/delta": -0.13615985214710236, "fcm_dpo/margin": 324.3789367675781, "fcm_dpo/q_t": 0.3801257610321045, "grad_norm": 34.18647766113281, "learning_rate": 2.294897926507156e-07, "logits/chosen": -0.6204630136489868, "logits/rejected": -0.6229850053787231, "logps/chosen": -505.4068908691406, "logps/ref_chosen": -53.50397872924805, "logps/ref_rejected": -102.34584045410156, "logps/rejected": -878.627685546875, "loss": 1.0167, "margin_dpo/margin_mean": 324.3789367675781, "margin_dpo/margin_std": 355.14715576171875, "step": 392 }, { "KL/chosen_KL_mean": -472.4049072265625, "KL/mean": -620.7923583984375, "KL/rejected_KL_mean": -769.1798095703125, "KL/std": 418.1456604003906, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0014102270361036062, "fcm_dpo/delta": -0.026089845225214958, "fcm_dpo/margin": 296.77490234375, "fcm_dpo/q_t": 0.4100596308708191, "grad_norm": 26.125350952148438, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -0.6747171878814697, "logits/rejected": -0.6873846054077148, "logps/chosen": -518.8788452148438, "logps/ref_chosen": -46.473915100097656, "logps/ref_rejected": -71.96885681152344, "logps/rejected": -841.148681640625, "loss": 1.1204, "margin_dpo/margin_mean": 296.77490234375, "margin_dpo/margin_std": 503.2057189941406, "step": 393 }, { "KL/chosen_KL_mean": -548.73779296875, "KL/mean": -690.5399169921875, "KL/rejected_KL_mean": -832.342041015625, "KL/std": 433.2088623046875, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.0014103709254413843, "fcm_dpo/delta": -0.0008256174623966217, "fcm_dpo/margin": 283.604248046875, "fcm_dpo/q_t": 0.4093853831291199, "grad_norm": 22.54509162902832, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -0.64982008934021, "logits/rejected": -0.654583215713501, "logps/chosen": -601.6493530273438, "logps/ref_chosen": -52.91154861450195, "logps/ref_rejected": -90.8226318359375, "logps/rejected": -923.1646728515625, "loss": 1.1035, "margin_dpo/margin_mean": 283.604248046875, "margin_dpo/margin_std": 410.2900695800781, "step": 394 }, { "KL/chosen_KL_mean": -520.23828125, "KL/mean": -669.6795654296875, "KL/rejected_KL_mean": -819.120849609375, "KL/std": 406.344482421875, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.0013870133552700281, "fcm_dpo/delta": -0.019791439175605774, "fcm_dpo/margin": 298.882568359375, "fcm_dpo/q_t": 0.40929269790649414, "grad_norm": 33.52304458618164, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -0.7437783479690552, "logits/rejected": -0.7509876489639282, "logps/chosen": -582.784423828125, "logps/ref_chosen": -62.546112060546875, "logps/ref_rejected": -83.78262329101562, "logps/rejected": -902.9034423828125, "loss": 1.1252, "margin_dpo/margin_mean": 298.882568359375, "margin_dpo/margin_std": 504.57586669921875, "step": 395 }, { "KL/chosen_KL_mean": -525.498779296875, "KL/mean": -663.1034545898438, "KL/rejected_KL_mean": -800.7081298828125, "KL/std": 373.2579650878906, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.0014105092268437147, "fcm_dpo/delta": 0.014646814204752445, "fcm_dpo/margin": 275.20941162109375, "fcm_dpo/q_t": 0.4090290665626526, "grad_norm": 26.613161087036133, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -0.7049363851547241, "logits/rejected": -0.6887962818145752, "logps/chosen": -594.4947509765625, "logps/ref_chosen": -68.99594116210938, "logps/ref_rejected": -88.64665985107422, "logps/rejected": -889.3547973632812, "loss": 1.0943, "margin_dpo/margin_mean": 275.20941162109375, "margin_dpo/margin_std": 351.0611572265625, "step": 396 }, { "KL/chosen_KL_mean": -484.30328369140625, "KL/mean": -674.1336669921875, "KL/rejected_KL_mean": -863.964111328125, "KL/std": 388.2706298828125, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.0012838148977607489, "fcm_dpo/delta": -0.11897562444210052, "fcm_dpo/margin": 379.66082763671875, "fcm_dpo/q_t": 0.3814941942691803, "grad_norm": 36.56934356689453, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -0.7270249128341675, "logits/rejected": -0.7500828504562378, "logps/chosen": -545.5804443359375, "logps/ref_chosen": -61.27716827392578, "logps/ref_rejected": -103.11612701416016, "logps/rejected": -967.0802001953125, "loss": 1.0163, "margin_dpo/margin_mean": 379.66082763671875, "margin_dpo/margin_std": 395.0962829589844, "step": 397 }, { "KL/chosen_KL_mean": -559.9149169921875, "KL/mean": -713.6265869140625, "KL/rejected_KL_mean": -867.3382568359375, "KL/std": 434.58392333984375, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.0012684231624007225, "fcm_dpo/delta": 0.012464390136301517, "fcm_dpo/margin": 307.4233093261719, "fcm_dpo/q_t": 0.41516533493995667, "grad_norm": 29.09773063659668, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -0.7229080200195312, "logits/rejected": -0.7291736006736755, "logps/chosen": -628.06640625, "logps/ref_chosen": -68.15155029296875, "logps/ref_rejected": -108.52360534667969, "logps/rejected": -975.86181640625, "loss": 1.1261, "margin_dpo/margin_mean": 307.42333984375, "margin_dpo/margin_std": 505.4172058105469, "step": 398 }, { "KL/chosen_KL_mean": -482.8316650390625, "KL/mean": -642.2623901367188, "KL/rejected_KL_mean": -801.6930541992188, "KL/std": 400.37786865234375, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0012626744573935866, "fcm_dpo/delta": -0.003726685419678688, "fcm_dpo/margin": 318.8613586425781, "fcm_dpo/q_t": 0.4093470573425293, "grad_norm": 36.91798400878906, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -0.7428172826766968, "logits/rejected": -0.7134918570518494, "logps/chosen": -543.7214965820312, "logps/ref_chosen": -60.889801025390625, "logps/ref_rejected": -77.965576171875, "logps/rejected": -879.6586303710938, "loss": 1.1144, "margin_dpo/margin_mean": 318.8613586425781, "margin_dpo/margin_std": 496.3259582519531, "step": 399 }, { "KL/chosen_KL_mean": -457.9895324707031, "KL/mean": -660.6600341796875, "KL/rejected_KL_mean": -863.33056640625, "KL/std": 368.4857177734375, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0011767219984903932, "fcm_dpo/delta": -0.10189603269100189, "fcm_dpo/margin": 405.341064453125, "fcm_dpo/q_t": 0.38432997465133667, "grad_norm": 36.20450973510742, "learning_rate": 2.192779273338215e-07, "logits/chosen": -0.7186334133148193, "logits/rejected": -0.7250533103942871, "logps/chosen": -521.6331176757812, "logps/ref_chosen": -63.64359664916992, "logps/ref_rejected": -105.252685546875, "logps/rejected": -968.583251953125, "loss": 1.0222, "margin_dpo/margin_mean": 405.341064453125, "margin_dpo/margin_std": 422.13092041015625, "step": 400 }, { "KL/chosen_KL_mean": -539.432861328125, "KL/mean": -647.3896484375, "KL/rejected_KL_mean": -755.346435546875, "KL/std": 385.900146484375, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0012087201466783881, "fcm_dpo/delta": 0.04972143471240997, "fcm_dpo/margin": 215.91360473632812, "fcm_dpo/q_t": 0.44173485040664673, "grad_norm": 31.359392166137695, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -0.7117223739624023, "logits/rejected": -0.7047185301780701, "logps/chosen": -596.5958862304688, "logps/ref_chosen": -57.16303253173828, "logps/ref_rejected": -83.79249572753906, "logps/rejected": -839.138916015625, "loss": 1.2256, "margin_dpo/margin_mean": 215.9136199951172, "margin_dpo/margin_std": 492.71820068359375, "step": 401 }, { "KL/chosen_KL_mean": -420.78167724609375, "KL/mean": -644.1975708007812, "KL/rejected_KL_mean": -867.613525390625, "KL/std": 450.4141845703125, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0010973564349114895, "fcm_dpo/delta": -0.12137363106012344, "fcm_dpo/margin": 446.831787109375, "fcm_dpo/q_t": 0.3830028772354126, "grad_norm": 26.827964782714844, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -0.7724575996398926, "logits/rejected": -0.7748699188232422, "logps/chosen": -471.52203369140625, "logps/ref_chosen": -50.74037170410156, "logps/ref_rejected": -81.0460433959961, "logps/rejected": -948.6595458984375, "loss": 1.0181, "margin_dpo/margin_mean": 446.831787109375, "margin_dpo/margin_std": 491.77197265625, "step": 402 }, { "KL/chosen_KL_mean": -427.5725402832031, "KL/mean": -592.4838256835938, "KL/rejected_KL_mean": -757.39501953125, "KL/std": 362.35382080078125, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0011073750210925937, "fcm_dpo/delta": 0.04235755652189255, "fcm_dpo/margin": 329.82257080078125, "fcm_dpo/q_t": 0.41572099924087524, "grad_norm": 28.50166893005371, "learning_rate": 2.154609112620295e-07, "logits/chosen": -0.7852243185043335, "logits/rejected": -0.7927581071853638, "logps/chosen": -474.7198486328125, "logps/ref_chosen": -47.14731216430664, "logps/ref_rejected": -77.2666015625, "logps/rejected": -834.6616821289062, "loss": 1.1015, "margin_dpo/margin_mean": 329.82257080078125, "margin_dpo/margin_std": 395.7626037597656, "step": 403 }, { "KL/chosen_KL_mean": -516.7240600585938, "KL/mean": -690.0592041015625, "KL/rejected_KL_mean": -863.3944091796875, "KL/std": 423.0172119140625, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0011222802568227053, "fcm_dpo/delta": 0.01353571005165577, "fcm_dpo/margin": 346.67041015625, "fcm_dpo/q_t": 0.4125150740146637, "grad_norm": 25.302902221679688, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -0.7894694805145264, "logits/rejected": -0.7903016805648804, "logps/chosen": -564.5993041992188, "logps/ref_chosen": -47.875274658203125, "logps/ref_rejected": -77.15499877929688, "logps/rejected": -940.5494384765625, "loss": 1.1134, "margin_dpo/margin_mean": 346.67041015625, "margin_dpo/margin_std": 523.8775024414062, "step": 404 }, { "KL/chosen_KL_mean": -592.7296142578125, "KL/mean": -713.805419921875, "KL/rejected_KL_mean": -834.8812255859375, "KL/std": 422.7801513671875, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.001129148993641138, "fcm_dpo/delta": -0.005241929553449154, "fcm_dpo/margin": 242.15170288085938, "fcm_dpo/q_t": 0.4377569258213043, "grad_norm": 39.06339645385742, "learning_rate": 2.129207286861638e-07, "logits/chosen": -0.783086895942688, "logits/rejected": -0.7755570411682129, "logps/chosen": -657.8924560546875, "logps/ref_chosen": -65.16290283203125, "logps/ref_rejected": -87.18678283691406, "logps/rejected": -922.0680541992188, "loss": 1.2157, "margin_dpo/margin_mean": 242.15170288085938, "margin_dpo/margin_std": 514.3509521484375, "step": 405 }, { "KL/chosen_KL_mean": -550.7774658203125, "KL/mean": -743.4031982421875, "KL/rejected_KL_mean": -936.0289306640625, "KL/std": 466.4752197265625, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0011199053842574358, "fcm_dpo/delta": -0.04166682809591293, "fcm_dpo/margin": 385.25140380859375, "fcm_dpo/q_t": 0.40454065799713135, "grad_norm": 29.93971061706543, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -0.8605983257293701, "logits/rejected": -0.8947657346725464, "logps/chosen": -600.518310546875, "logps/ref_chosen": -49.740814208984375, "logps/ref_rejected": -92.07862854003906, "logps/rejected": -1028.1075439453125, "loss": 1.0915, "margin_dpo/margin_mean": 385.25140380859375, "margin_dpo/margin_std": 562.6111450195312, "step": 406 }, { "KL/chosen_KL_mean": -557.0032348632812, "KL/mean": -658.51025390625, "KL/rejected_KL_mean": -760.0172729492188, "KL/std": 408.6435852050781, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0011084176367148757, "fcm_dpo/delta": 0.05087389424443245, "fcm_dpo/margin": 203.0139923095703, "fcm_dpo/q_t": 0.44716814160346985, "grad_norm": 29.988346099853516, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -0.9323893785476685, "logits/rejected": -0.8998297452926636, "logps/chosen": -613.333984375, "logps/ref_chosen": -56.33069610595703, "logps/ref_rejected": -77.51209259033203, "logps/rejected": -837.5293579101562, "loss": 1.2531, "margin_dpo/margin_mean": 203.01400756835938, "margin_dpo/margin_std": 529.3558959960938, "step": 407 }, { "KL/chosen_KL_mean": -562.3118896484375, "KL/mean": -679.2642822265625, "KL/rejected_KL_mean": -796.2166748046875, "KL/std": 361.14996337890625, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0012600787449628115, "fcm_dpo/delta": 0.12069714069366455, "fcm_dpo/margin": 233.90478515625, "fcm_dpo/q_t": 0.43428733944892883, "grad_norm": 34.71678161621094, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -0.9175188541412354, "logits/rejected": -0.8953307867050171, "logps/chosen": -632.1011962890625, "logps/ref_chosen": -69.789306640625, "logps/ref_rejected": -90.09693908691406, "logps/rejected": -886.3135986328125, "loss": 1.1787, "margin_dpo/margin_mean": 233.90478515625, "margin_dpo/margin_std": 410.2579345703125, "step": 408 }, { "KL/chosen_KL_mean": -531.5665893554688, "KL/mean": -659.5404663085938, "KL/rejected_KL_mean": -787.514404296875, "KL/std": 434.8625183105469, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0013779137516394258, "fcm_dpo/delta": 0.05021357536315918, "fcm_dpo/margin": 255.94775390625, "fcm_dpo/q_t": 0.41811615228652954, "grad_norm": 53.07970428466797, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -0.801446795463562, "logits/rejected": -0.78272545337677, "logps/chosen": -598.884033203125, "logps/ref_chosen": -67.31744384765625, "logps/ref_rejected": -84.904296875, "logps/rejected": -872.4186401367188, "loss": 1.1736, "margin_dpo/margin_mean": 255.94773864746094, "margin_dpo/margin_std": 472.74395751953125, "step": 409 }, { "KL/chosen_KL_mean": -491.54156494140625, "KL/mean": -645.4672241210938, "KL/rejected_KL_mean": -799.392822265625, "KL/std": 392.8507385253906, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0013317305129021406, "fcm_dpo/delta": -0.012640236876904964, "fcm_dpo/margin": 307.85125732421875, "fcm_dpo/q_t": 0.40540868043899536, "grad_norm": 28.094219207763672, "learning_rate": 2.065879555832674e-07, "logits/chosen": -0.8844398260116577, "logits/rejected": -0.8990808725357056, "logps/chosen": -543.0069580078125, "logps/ref_chosen": -51.465354919433594, "logps/ref_rejected": -83.198974609375, "logps/rejected": -882.591796875, "loss": 1.0982, "margin_dpo/margin_mean": 307.85125732421875, "margin_dpo/margin_std": 440.26861572265625, "step": 410 }, { "KL/chosen_KL_mean": -492.3099670410156, "KL/mean": -623.0086059570312, "KL/rejected_KL_mean": -753.7071533203125, "KL/std": 405.5526123046875, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0012510113883763552, "fcm_dpo/delta": -0.062365103513002396, "fcm_dpo/margin": 261.3972473144531, "fcm_dpo/q_t": 0.42639321088790894, "grad_norm": 39.17562484741211, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -0.9218274354934692, "logits/rejected": -0.9327045679092407, "logps/chosen": -544.6172485351562, "logps/ref_chosen": -52.30727005004883, "logps/ref_rejected": -80.69495391845703, "logps/rejected": -834.4021606445312, "loss": 1.1733, "margin_dpo/margin_mean": 261.397216796875, "margin_dpo/margin_std": 473.8624267578125, "step": 411 }, { "KL/chosen_KL_mean": -472.92578125, "KL/mean": -619.5107421875, "KL/rejected_KL_mean": -766.095703125, "KL/std": 371.1781005859375, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0012766936561092734, "fcm_dpo/delta": 0.031684231013059616, "fcm_dpo/margin": 293.1698913574219, "fcm_dpo/q_t": 0.41487789154052734, "grad_norm": 37.587581634521484, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -0.9481087327003479, "logits/rejected": -0.9869192838668823, "logps/chosen": -526.0699462890625, "logps/ref_chosen": -53.144126892089844, "logps/ref_rejected": -100.0608139038086, "logps/rejected": -866.156494140625, "loss": 1.1127, "margin_dpo/margin_mean": 293.1698913574219, "margin_dpo/margin_std": 415.78509521484375, "step": 412 }, { "KL/chosen_KL_mean": -546.9744873046875, "KL/mean": -695.7308349609375, "KL/rejected_KL_mean": -844.4873046875, "KL/std": 423.4039306640625, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0013015582226216793, "fcm_dpo/delta": 0.0158353503793478, "fcm_dpo/margin": 297.5128173828125, "fcm_dpo/q_t": 0.4121004343032837, "grad_norm": 36.7993049621582, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -0.8297325968742371, "logits/rejected": -0.8475293517112732, "logps/chosen": -608.556396484375, "logps/ref_chosen": -61.58196258544922, "logps/ref_rejected": -99.47340393066406, "logps/rejected": -943.960693359375, "loss": 1.1252, "margin_dpo/margin_mean": 297.5128173828125, "margin_dpo/margin_std": 477.887451171875, "step": 413 }, { "KL/chosen_KL_mean": -513.6051025390625, "KL/mean": -674.848876953125, "KL/rejected_KL_mean": -836.0927124023438, "KL/std": 387.4964294433594, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0013245465233922005, "fcm_dpo/delta": -0.03918169438838959, "fcm_dpo/margin": 322.4875793457031, "fcm_dpo/q_t": 0.3994929790496826, "grad_norm": 43.984615325927734, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -0.7619544267654419, "logits/rejected": -0.7554280757904053, "logps/chosen": -560.236572265625, "logps/ref_chosen": -46.63148498535156, "logps/ref_rejected": -87.64653015136719, "logps/rejected": -923.7392578125, "loss": 1.096, "margin_dpo/margin_mean": 322.4875793457031, "margin_dpo/margin_std": 455.6772766113281, "step": 414 }, { "KL/chosen_KL_mean": -519.7416381835938, "KL/mean": -640.4639892578125, "KL/rejected_KL_mean": -761.186279296875, "KL/std": 358.64996337890625, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.001341789960861206, "fcm_dpo/delta": 0.09107168018817902, "fcm_dpo/margin": 241.4446258544922, "fcm_dpo/q_t": 0.42783957719802856, "grad_norm": 31.809572219848633, "learning_rate": 2.002837796667909e-07, "logits/chosen": -0.7860684990882874, "logits/rejected": -0.7803636789321899, "logps/chosen": -598.3599853515625, "logps/ref_chosen": -78.6182861328125, "logps/ref_rejected": -100.47752380371094, "logps/rejected": -861.663818359375, "loss": 1.1536, "margin_dpo/margin_mean": 241.44464111328125, "margin_dpo/margin_std": 387.5479431152344, "step": 415 }, { "KL/chosen_KL_mean": -462.289306640625, "KL/mean": -652.3590087890625, "KL/rejected_KL_mean": -842.4287109375, "KL/std": 418.4091796875, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0012784367427229881, "fcm_dpo/delta": -0.11329878866672516, "fcm_dpo/margin": 380.139404296875, "fcm_dpo/q_t": 0.38521865010261536, "grad_norm": 30.71705436706543, "learning_rate": 1.990267419549914e-07, "logits/chosen": -0.7888989448547363, "logits/rejected": -0.80275559425354, "logps/chosen": -520.5684204101562, "logps/ref_chosen": -58.27912521362305, "logps/ref_rejected": -90.56871795654297, "logps/rejected": -932.9974365234375, "loss": 1.0321, "margin_dpo/margin_mean": 380.1394348144531, "margin_dpo/margin_std": 444.5953369140625, "step": 416 }, { "KL/chosen_KL_mean": -447.36212158203125, "KL/mean": -610.0339965820312, "KL/rejected_KL_mean": -772.7059326171875, "KL/std": 384.17840576171875, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0012431307695806026, "fcm_dpo/delta": -0.006137235090136528, "fcm_dpo/margin": 325.34375, "fcm_dpo/q_t": 0.4053758382797241, "grad_norm": 39.485050201416016, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -0.7083637714385986, "logits/rejected": -0.6805850863456726, "logps/chosen": -497.56085205078125, "logps/ref_chosen": -50.1987190246582, "logps/ref_rejected": -68.15184020996094, "logps/rejected": -840.8577880859375, "loss": 1.0748, "margin_dpo/margin_mean": 325.3437805175781, "margin_dpo/margin_std": 378.35235595703125, "step": 417 }, { "KL/chosen_KL_mean": -498.91168212890625, "KL/mean": -664.226318359375, "KL/rejected_KL_mean": -829.5408935546875, "KL/std": 416.49688720703125, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0012598626781255007, "fcm_dpo/delta": -0.02940238267183304, "fcm_dpo/margin": 330.62921142578125, "fcm_dpo/q_t": 0.4053995609283447, "grad_norm": 24.680252075195312, "learning_rate": 1.965167291983757e-07, "logits/chosen": -0.796850323677063, "logits/rejected": -0.7809560894966125, "logps/chosen": -580.89013671875, "logps/ref_chosen": -81.97846984863281, "logps/ref_rejected": -104.69148254394531, "logps/rejected": -934.2323608398438, "loss": 1.1034, "margin_dpo/margin_mean": 330.62921142578125, "margin_dpo/margin_std": 468.53973388671875, "step": 418 }, { "KL/chosen_KL_mean": -468.39617919921875, "KL/mean": -656.1996459960938, "KL/rejected_KL_mean": -844.0031127929688, "KL/std": 375.4820556640625, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0011481782421469688, "fcm_dpo/delta": -0.040392979979515076, "fcm_dpo/margin": 375.6069641113281, "fcm_dpo/q_t": 0.39766836166381836, "grad_norm": 35.54838562011719, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -0.6798853278160095, "logits/rejected": -0.6791231632232666, "logps/chosen": -521.3448486328125, "logps/ref_chosen": -52.948646545410156, "logps/ref_rejected": -91.58309936523438, "logps/rejected": -935.586181640625, "loss": 1.0594, "margin_dpo/margin_mean": 375.60693359375, "margin_dpo/margin_std": 432.56622314453125, "step": 419 }, { "KL/chosen_KL_mean": -585.70361328125, "KL/mean": -711.439208984375, "KL/rejected_KL_mean": -837.1748657226562, "KL/std": 429.8891296386719, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0012487138155847788, "fcm_dpo/delta": 0.09940655529499054, "fcm_dpo/margin": 251.47128295898438, "fcm_dpo/q_t": 0.4328416585922241, "grad_norm": 44.155242919921875, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -0.810115396976471, "logits/rejected": -0.7681174278259277, "logps/chosen": -663.4735107421875, "logps/ref_chosen": -77.7699203491211, "logps/ref_rejected": -69.31985473632812, "logps/rejected": -906.4947509765625, "loss": 1.1992, "margin_dpo/margin_mean": 251.47128295898438, "margin_dpo/margin_std": 534.525390625, "step": 420 }, { "KL/chosen_KL_mean": -498.656005859375, "KL/mean": -636.3494873046875, "KL/rejected_KL_mean": -774.04296875, "KL/std": 402.7722473144531, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0013234158977866173, "fcm_dpo/delta": 0.041565168648958206, "fcm_dpo/margin": 275.386962890625, "fcm_dpo/q_t": 0.41690126061439514, "grad_norm": 27.106782913208008, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -0.7847892045974731, "logits/rejected": -0.7707439661026001, "logps/chosen": -552.421875, "logps/ref_chosen": -53.765865325927734, "logps/ref_rejected": -89.28144836425781, "logps/rejected": -863.324462890625, "loss": 1.1328, "margin_dpo/margin_mean": 275.3869934082031, "margin_dpo/margin_std": 421.31298828125, "step": 421 }, { "KL/chosen_KL_mean": -588.316162109375, "KL/mean": -753.931640625, "KL/rejected_KL_mean": -919.5469970703125, "KL/std": 450.30499267578125, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0012808962492272258, "fcm_dpo/delta": -0.031088268384337425, "fcm_dpo/margin": 331.2308349609375, "fcm_dpo/q_t": 0.4033207297325134, "grad_norm": 26.31610107421875, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -0.8320428133010864, "logits/rejected": -0.8418641090393066, "logps/chosen": -656.949951171875, "logps/ref_chosen": -68.6337661743164, "logps/ref_rejected": -87.86351013183594, "logps/rejected": -1007.4105224609375, "loss": 1.1125, "margin_dpo/margin_mean": 331.2308349609375, "margin_dpo/margin_std": 535.4093627929688, "step": 422 }, { "KL/chosen_KL_mean": -558.1864624023438, "KL/mean": -731.43212890625, "KL/rejected_KL_mean": -904.677734375, "KL/std": 429.63372802734375, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0012374825309962034, "fcm_dpo/delta": -0.0368797592818737, "fcm_dpo/margin": 346.49127197265625, "fcm_dpo/q_t": 0.40145501494407654, "grad_norm": 33.875404357910156, "learning_rate": 1.902669377503756e-07, "logits/chosen": -0.8104654550552368, "logits/rejected": -0.827370822429657, "logps/chosen": -613.1767578125, "logps/ref_chosen": -54.99030303955078, "logps/ref_rejected": -86.30654907226562, "logps/rejected": -990.9842529296875, "loss": 1.0925, "margin_dpo/margin_mean": 346.4912414550781, "margin_dpo/margin_std": 503.84454345703125, "step": 423 }, { "KL/chosen_KL_mean": -510.28045654296875, "KL/mean": -682.5518188476562, "KL/rejected_KL_mean": -854.8231201171875, "KL/std": 413.15972900390625, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.001186060020700097, "fcm_dpo/delta": -0.01438332349061966, "fcm_dpo/margin": 344.54266357421875, "fcm_dpo/q_t": 0.40733110904693604, "grad_norm": 37.09081268310547, "learning_rate": 1.890215699729057e-07, "logits/chosen": -0.7975935935974121, "logits/rejected": -0.7788383960723877, "logps/chosen": -566.2923583984375, "logps/ref_chosen": -56.01192092895508, "logps/ref_rejected": -66.47896575927734, "logps/rejected": -921.3020629882812, "loss": 1.0954, "margin_dpo/margin_mean": 344.54266357421875, "margin_dpo/margin_std": 480.84130859375, "step": 424 }, { "KL/chosen_KL_mean": -609.7572021484375, "KL/mean": -721.0950927734375, "KL/rejected_KL_mean": -832.43310546875, "KL/std": 407.82891845703125, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0012294601183384657, "fcm_dpo/delta": 0.027459844946861267, "fcm_dpo/margin": 222.6759490966797, "fcm_dpo/q_t": 0.4368723928928375, "grad_norm": 42.740379333496094, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -0.7805401086807251, "logits/rejected": -0.7881333827972412, "logps/chosen": -656.626220703125, "logps/ref_chosen": -46.86899948120117, "logps/ref_rejected": -95.92545318603516, "logps/rejected": -928.3585815429688, "loss": 1.2167, "margin_dpo/margin_mean": 222.67593383789062, "margin_dpo/margin_std": 486.15093994140625, "step": 425 }, { "KL/chosen_KL_mean": -552.8094482421875, "KL/mean": -687.5247192382812, "KL/rejected_KL_mean": -822.239990234375, "KL/std": 365.3626403808594, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0013338683638721704, "fcm_dpo/delta": 0.04180013760924339, "fcm_dpo/margin": 269.4305419921875, "fcm_dpo/q_t": 0.4174906015396118, "grad_norm": 31.232389450073242, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -0.7621093988418579, "logits/rejected": -0.7303955554962158, "logps/chosen": -629.3930053710938, "logps/ref_chosen": -76.58354187011719, "logps/ref_rejected": -81.26658630371094, "logps/rejected": -903.506591796875, "loss": 1.1272, "margin_dpo/margin_mean": 269.4305419921875, "margin_dpo/margin_std": 378.654052734375, "step": 426 }, { "KL/chosen_KL_mean": -467.79608154296875, "KL/mean": -581.63623046875, "KL/rejected_KL_mean": -695.476318359375, "KL/std": 333.85369873046875, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.001394979190081358, "fcm_dpo/delta": 0.09802726656198502, "fcm_dpo/margin": 227.68026733398438, "fcm_dpo/q_t": 0.4301951825618744, "grad_norm": 31.7608642578125, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -0.8063141703605652, "logits/rejected": -0.78669673204422, "logps/chosen": -532.6499633789062, "logps/ref_chosen": -64.8538818359375, "logps/ref_rejected": -78.5660171508789, "logps/rejected": -774.0423583984375, "loss": 1.1608, "margin_dpo/margin_mean": 227.68026733398438, "margin_dpo/margin_std": 382.787353515625, "step": 427 }, { "KL/chosen_KL_mean": -608.790771484375, "KL/mean": -786.853515625, "KL/rejected_KL_mean": -964.916259765625, "KL/std": 468.04583740234375, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0013052403228357434, "fcm_dpo/delta": -0.090778648853302, "fcm_dpo/margin": 356.1253967285156, "fcm_dpo/q_t": 0.3941190838813782, "grad_norm": 28.149211883544922, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -0.7752005457878113, "logits/rejected": -0.7893061637878418, "logps/chosen": -671.427490234375, "logps/ref_chosen": -62.63666534423828, "logps/ref_rejected": -103.28181457519531, "logps/rejected": -1068.197998046875, "loss": 1.088, "margin_dpo/margin_mean": 356.1253967285156, "margin_dpo/margin_std": 543.524169921875, "step": 428 }, { "KL/chosen_KL_mean": -629.3082275390625, "KL/mean": -756.3204345703125, "KL/rejected_KL_mean": -883.3326416015625, "KL/std": 410.94122314453125, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0012956894934177399, "fcm_dpo/delta": -0.020538704469799995, "fcm_dpo/margin": 254.02444458007812, "fcm_dpo/q_t": 0.4243999719619751, "grad_norm": 43.50310516357422, "learning_rate": 1.828194884925749e-07, "logits/chosen": -0.7827075719833374, "logits/rejected": -0.7630304098129272, "logps/chosen": -710.542236328125, "logps/ref_chosen": -81.23401641845703, "logps/ref_rejected": -91.79493713378906, "logps/rejected": -975.1275634765625, "loss": 1.1856, "margin_dpo/margin_mean": 254.02444458007812, "margin_dpo/margin_std": 494.28680419921875, "step": 429 }, { "KL/chosen_KL_mean": -546.2984619140625, "KL/mean": -644.2332763671875, "KL/rejected_KL_mean": -742.1680297851562, "KL/std": 393.195556640625, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0013126166304573417, "fcm_dpo/delta": 0.04607279598712921, "fcm_dpo/margin": 195.86962890625, "fcm_dpo/q_t": 0.44172602891921997, "grad_norm": 39.36235427856445, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -0.7814037799835205, "logits/rejected": -0.7843170166015625, "logps/chosen": -607.21875, "logps/ref_chosen": -60.920326232910156, "logps/ref_rejected": -104.42280578613281, "logps/rejected": -846.5908203125, "loss": 1.2239, "margin_dpo/margin_mean": 195.86962890625, "margin_dpo/margin_std": 441.08013916015625, "step": 430 }, { "KL/chosen_KL_mean": -474.2396240234375, "KL/mean": -651.661376953125, "KL/rejected_KL_mean": -829.0831298828125, "KL/std": 401.1722412109375, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0012775419745594263, "fcm_dpo/delta": -0.06899101287126541, "fcm_dpo/margin": 354.843505859375, "fcm_dpo/q_t": 0.39390021562576294, "grad_norm": 31.863265991210938, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -0.7518160343170166, "logits/rejected": -0.7592271566390991, "logps/chosen": -531.58837890625, "logps/ref_chosen": -57.34874725341797, "logps/ref_rejected": -92.84022521972656, "logps/rejected": -921.9234008789062, "loss": 1.067, "margin_dpo/margin_mean": 354.8435363769531, "margin_dpo/margin_std": 455.1669921875, "step": 431 }, { "KL/chosen_KL_mean": -427.3382263183594, "KL/mean": -617.7256469726562, "KL/rejected_KL_mean": -808.113037109375, "KL/std": 403.57916259765625, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.001215646741911769, "fcm_dpo/delta": -0.08303441107273102, "fcm_dpo/margin": 380.77484130859375, "fcm_dpo/q_t": 0.39162373542785645, "grad_norm": 47.37468719482422, "learning_rate": 1.791192214186223e-07, "logits/chosen": -0.6925067901611328, "logits/rejected": -0.685772180557251, "logps/chosen": -498.41302490234375, "logps/ref_chosen": -71.07479095458984, "logps/ref_rejected": -98.57952880859375, "logps/rejected": -906.692626953125, "loss": 1.0385, "margin_dpo/margin_mean": 380.77484130859375, "margin_dpo/margin_std": 422.8524169921875, "step": 432 }, { "KL/chosen_KL_mean": -561.0573120117188, "KL/mean": -678.9639282226562, "KL/rejected_KL_mean": -796.8705444335938, "KL/std": 433.2787780761719, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0012703599641099572, "fcm_dpo/delta": 0.11619932949542999, "fcm_dpo/margin": 235.8131561279297, "fcm_dpo/q_t": 0.4302235245704651, "grad_norm": 37.460628509521484, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -0.7900607585906982, "logits/rejected": -0.7896082401275635, "logps/chosen": -619.3305053710938, "logps/ref_chosen": -58.273193359375, "logps/ref_rejected": -95.95089721679688, "logps/rejected": -892.8214111328125, "loss": 1.2116, "margin_dpo/margin_mean": 235.81317138671875, "margin_dpo/margin_std": 522.4442749023438, "step": 433 }, { "KL/chosen_KL_mean": -496.65863037109375, "KL/mean": -640.20361328125, "KL/rejected_KL_mean": -783.7486572265625, "KL/std": 404.4720458984375, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0013023980427533388, "fcm_dpo/delta": 0.0312434583902359, "fcm_dpo/margin": 287.09002685546875, "fcm_dpo/q_t": 0.4213806688785553, "grad_norm": 25.82297706604004, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -0.8235654830932617, "logits/rejected": -0.8260775208473206, "logps/chosen": -558.63232421875, "logps/ref_chosen": -61.97370147705078, "logps/ref_rejected": -78.49861145019531, "logps/rejected": -862.2472534179688, "loss": 1.1396, "margin_dpo/margin_mean": 287.09002685546875, "margin_dpo/margin_std": 501.02880859375, "step": 434 }, { "KL/chosen_KL_mean": -480.15570068359375, "KL/mean": -635.9765625, "KL/rejected_KL_mean": -791.79736328125, "KL/std": 393.2747802734375, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0013056218158453703, "fcm_dpo/delta": -0.01002051867544651, "fcm_dpo/margin": 311.6417236328125, "fcm_dpo/q_t": 0.4069361090660095, "grad_norm": 45.785640716552734, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -0.8830646276473999, "logits/rejected": -0.8846416473388672, "logps/chosen": -531.65771484375, "logps/ref_chosen": -51.502052307128906, "logps/ref_rejected": -87.56689453125, "logps/rejected": -879.3642578125, "loss": 1.0991, "margin_dpo/margin_mean": 311.6416931152344, "margin_dpo/margin_std": 440.7518615722656, "step": 435 }, { "KL/chosen_KL_mean": -499.36517333984375, "KL/mean": -633.2975463867188, "KL/rejected_KL_mean": -767.22998046875, "KL/std": 352.81036376953125, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0013422095216810703, "fcm_dpo/delta": 0.048316676169633865, "fcm_dpo/margin": 267.86474609375, "fcm_dpo/q_t": 0.4203526973724365, "grad_norm": 30.017845153808594, "learning_rate": 1.742118314717391e-07, "logits/chosen": -0.732751727104187, "logits/rejected": -0.6973142623901367, "logps/chosen": -570.7689208984375, "logps/ref_chosen": -71.40371704101562, "logps/ref_rejected": -82.72775268554688, "logps/rejected": -849.9577026367188, "loss": 1.1375, "margin_dpo/margin_mean": 267.86474609375, "margin_dpo/margin_std": 426.91168212890625, "step": 436 }, { "KL/chosen_KL_mean": -500.2353515625, "KL/mean": -638.3201293945312, "KL/rejected_KL_mean": -776.4049072265625, "KL/std": 347.3167724609375, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0014004592085257173, "fcm_dpo/delta": 0.01640990376472473, "fcm_dpo/margin": 276.16949462890625, "fcm_dpo/q_t": 0.4110352396965027, "grad_norm": 28.889192581176758, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -0.8302721977233887, "logits/rejected": -0.814739465713501, "logps/chosen": -564.9796142578125, "logps/ref_chosen": -64.7442626953125, "logps/ref_rejected": -82.04356384277344, "logps/rejected": -858.4484252929688, "loss": 1.1092, "margin_dpo/margin_mean": 276.16949462890625, "margin_dpo/margin_std": 396.4515686035156, "step": 437 }, { "KL/chosen_KL_mean": -522.9442749023438, "KL/mean": -677.5450439453125, "KL/rejected_KL_mean": -832.145751953125, "KL/std": 379.323974609375, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.001349923899397254, "fcm_dpo/delta": -0.025827843695878983, "fcm_dpo/margin": 309.201416015625, "fcm_dpo/q_t": 0.40191444754600525, "grad_norm": 42.13845443725586, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -0.8437032699584961, "logits/rejected": -0.8374447226524353, "logps/chosen": -581.9629516601562, "logps/ref_chosen": -59.0186653137207, "logps/ref_rejected": -83.07682800292969, "logps/rejected": -915.2225341796875, "loss": 1.0822, "margin_dpo/margin_mean": 309.2014465332031, "margin_dpo/margin_std": 397.62255859375, "step": 438 }, { "KL/chosen_KL_mean": -549.39306640625, "KL/mean": -681.3974609375, "KL/rejected_KL_mean": -813.4017944335938, "KL/std": 387.31488037109375, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.001252303714863956, "fcm_dpo/delta": -0.09410447627305984, "fcm_dpo/margin": 264.0087890625, "fcm_dpo/q_t": 0.4208075702190399, "grad_norm": 36.43398666381836, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -0.8629584908485413, "logits/rejected": -0.8729925155639648, "logps/chosen": -603.1771240234375, "logps/ref_chosen": -53.78407669067383, "logps/ref_rejected": -83.98545837402344, "logps/rejected": -897.3872680664062, "loss": 1.1705, "margin_dpo/margin_mean": 264.0087890625, "margin_dpo/margin_std": 464.46630859375, "step": 439 }, { "KL/chosen_KL_mean": -529.5724487304688, "KL/mean": -701.4462890625, "KL/rejected_KL_mean": -873.320068359375, "KL/std": 479.51513671875, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0012315923813730478, "fcm_dpo/delta": -0.029587693512439728, "fcm_dpo/margin": 343.74761962890625, "fcm_dpo/q_t": 0.4066435694694519, "grad_norm": 53.8433837890625, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -0.8866897821426392, "logits/rejected": -0.8979896903038025, "logps/chosen": -608.13916015625, "logps/ref_chosen": -78.56671905517578, "logps/ref_rejected": -96.49775695800781, "logps/rejected": -969.81787109375, "loss": 1.085, "margin_dpo/margin_mean": 343.74761962890625, "margin_dpo/margin_std": 490.02239990234375, "step": 440 }, { "KL/chosen_KL_mean": -619.644775390625, "KL/mean": -767.532470703125, "KL/rejected_KL_mean": -915.4201049804688, "KL/std": 493.6292419433594, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0012763803824782372, "fcm_dpo/delta": 0.022389892488718033, "fcm_dpo/margin": 295.7752685546875, "fcm_dpo/q_t": 0.4217098355293274, "grad_norm": 33.3763313293457, "learning_rate": 1.681227682404166e-07, "logits/chosen": -0.8680696487426758, "logits/rejected": -0.8612392544746399, "logps/chosen": -680.46923828125, "logps/ref_chosen": -60.824440002441406, "logps/ref_rejected": -96.47080993652344, "logps/rejected": -1011.890869140625, "loss": 1.1805, "margin_dpo/margin_mean": 295.7752990722656, "margin_dpo/margin_std": 602.29541015625, "step": 441 }, { "KL/chosen_KL_mean": -522.046875, "KL/mean": -706.1878662109375, "KL/rejected_KL_mean": -890.3289184570312, "KL/std": 459.5678405761719, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.001231772475875914, "fcm_dpo/delta": -0.07431311160326004, "fcm_dpo/margin": 368.2820129394531, "fcm_dpo/q_t": 0.3975549638271332, "grad_norm": 34.67390823364258, "learning_rate": 1.669113001300851e-07, "logits/chosen": -0.8087929487228394, "logits/rejected": -0.8081426620483398, "logps/chosen": -569.05810546875, "logps/ref_chosen": -47.01121520996094, "logps/ref_rejected": -76.53926086425781, "logps/rejected": -966.8681640625, "loss": 1.089, "margin_dpo/margin_mean": 368.28204345703125, "margin_dpo/margin_std": 529.7601928710938, "step": 442 }, { "KL/chosen_KL_mean": -569.135009765625, "KL/mean": -666.7360229492188, "KL/rejected_KL_mean": -764.3370361328125, "KL/std": 426.0760498046875, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0012039494467899203, "fcm_dpo/delta": 0.04213346913456917, "fcm_dpo/margin": 195.20201110839844, "fcm_dpo/q_t": 0.44704896211624146, "grad_norm": 43.334136962890625, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -0.8095992803573608, "logits/rejected": -0.7834180593490601, "logps/chosen": -640.4080810546875, "logps/ref_chosen": -71.27301788330078, "logps/ref_rejected": -86.679931640625, "logps/rejected": -851.0169677734375, "loss": 1.2562, "margin_dpo/margin_mean": 195.20199584960938, "margin_dpo/margin_std": 502.8472900390625, "step": 443 }, { "KL/chosen_KL_mean": -557.4791259765625, "KL/mean": -734.4202270507812, "KL/rejected_KL_mean": -911.361328125, "KL/std": 455.6096496582031, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.001166836591437459, "fcm_dpo/delta": -0.01813267171382904, "fcm_dpo/margin": 353.8822021484375, "fcm_dpo/q_t": 0.4072955846786499, "grad_norm": 32.24861145019531, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -0.7987985610961914, "logits/rejected": -0.8144040703773499, "logps/chosen": -614.6928100585938, "logps/ref_chosen": -57.213706970214844, "logps/ref_rejected": -97.25489807128906, "logps/rejected": -1008.6162109375, "loss": 1.099, "margin_dpo/margin_mean": 353.8822021484375, "margin_dpo/margin_std": 520.219970703125, "step": 444 }, { "KL/chosen_KL_mean": -508.13671875, "KL/mean": -659.2096557617188, "KL/rejected_KL_mean": -810.2826538085938, "KL/std": 361.33935546875, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0012313367333263159, "fcm_dpo/delta": 0.032493121922016144, "fcm_dpo/margin": 302.14593505859375, "fcm_dpo/q_t": 0.4164279103279114, "grad_norm": 38.02067565917969, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -0.7978518009185791, "logits/rejected": -0.7957039475440979, "logps/chosen": -575.4365234375, "logps/ref_chosen": -67.29979705810547, "logps/ref_rejected": -92.68267059326172, "logps/rejected": -902.96533203125, "loss": 1.1248, "margin_dpo/margin_mean": 302.14593505859375, "margin_dpo/margin_std": 454.81597900390625, "step": 445 }, { "KL/chosen_KL_mean": -450.0630798339844, "KL/mean": -626.6373291015625, "KL/rejected_KL_mean": -803.211669921875, "KL/std": 392.5597229003906, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.001226876862347126, "fcm_dpo/delta": -0.046956416219472885, "fcm_dpo/margin": 353.14862060546875, "fcm_dpo/q_t": 0.3958050608634949, "grad_norm": 34.05545425415039, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -0.8068540692329407, "logits/rejected": -0.8256067037582397, "logps/chosen": -509.16156005859375, "logps/ref_chosen": -59.098487854003906, "logps/ref_rejected": -101.26419067382812, "logps/rejected": -904.475830078125, "loss": 1.0592, "margin_dpo/margin_mean": 353.14862060546875, "margin_dpo/margin_std": 380.9633483886719, "step": 446 }, { "KL/chosen_KL_mean": -495.2170715332031, "KL/mean": -681.0345458984375, "KL/rejected_KL_mean": -866.8519897460938, "KL/std": 464.1243591308594, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0011278802994638681, "fcm_dpo/delta": -0.025529345497488976, "fcm_dpo/margin": 371.6348876953125, "fcm_dpo/q_t": 0.4053482413291931, "grad_norm": 30.737730026245117, "learning_rate": 1.608874379754465e-07, "logits/chosen": -0.9250874519348145, "logits/rejected": -0.9507585763931274, "logps/chosen": -551.2924194335938, "logps/ref_chosen": -56.07533264160156, "logps/ref_rejected": -98.69475555419922, "logps/rejected": -965.5467529296875, "loss": 1.086, "margin_dpo/margin_mean": 371.6348876953125, "margin_dpo/margin_std": 520.7388916015625, "step": 447 }, { "KL/chosen_KL_mean": -539.2561645507812, "KL/mean": -716.0037841796875, "KL/rejected_KL_mean": -892.7515258789062, "KL/std": 413.3590087890625, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0011455777566879988, "fcm_dpo/delta": -0.006920462474226952, "fcm_dpo/margin": 353.4953308105469, "fcm_dpo/q_t": 0.4064434766769409, "grad_norm": 37.22543716430664, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -0.8032995462417603, "logits/rejected": -0.8209420442581177, "logps/chosen": -599.260009765625, "logps/ref_chosen": -60.00384521484375, "logps/ref_rejected": -102.26465606689453, "logps/rejected": -995.0161743164062, "loss": 1.083, "margin_dpo/margin_mean": 353.4953308105469, "margin_dpo/margin_std": 442.79266357421875, "step": 448 }, { "KL/chosen_KL_mean": -577.392333984375, "KL/mean": -767.4705810546875, "KL/rejected_KL_mean": -957.548828125, "KL/std": 516.8084716796875, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.001104456139728427, "fcm_dpo/delta": -0.025262048467993736, "fcm_dpo/margin": 380.15655517578125, "fcm_dpo/q_t": 0.40797027945518494, "grad_norm": 27.65226173400879, "learning_rate": 1.584941086944423e-07, "logits/chosen": -0.8589591979980469, "logits/rejected": -0.8559813499450684, "logps/chosen": -644.9189453125, "logps/ref_chosen": -67.52661895751953, "logps/ref_rejected": -88.59690856933594, "logps/rejected": -1046.145751953125, "loss": 1.1076, "margin_dpo/margin_mean": 380.15655517578125, "margin_dpo/margin_std": 612.1381225585938, "step": 449 }, { "KL/chosen_KL_mean": -496.8646240234375, "KL/mean": -713.1530151367188, "KL/rejected_KL_mean": -929.4413452148438, "KL/std": 456.3707275390625, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.00104955630376935, "fcm_dpo/delta": -0.06971128284931183, "fcm_dpo/margin": 432.5767822265625, "fcm_dpo/q_t": 0.3907149136066437, "grad_norm": 37.34757614135742, "learning_rate": 1.573010452010098e-07, "logits/chosen": -0.9782512187957764, "logits/rejected": -1.0043165683746338, "logps/chosen": -553.9727172851562, "logps/ref_chosen": -57.10811996459961, "logps/ref_rejected": -102.75494384765625, "logps/rejected": -1032.1962890625, "loss": 1.0356, "margin_dpo/margin_mean": 432.5767822265625, "margin_dpo/margin_std": 448.83770751953125, "step": 450 }, { "KL/chosen_KL_mean": -742.9453125, "KL/mean": -891.0797119140625, "KL/rejected_KL_mean": -1039.214111328125, "KL/std": 587.5035400390625, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0011060864198952913, "fcm_dpo/delta": 0.08447873592376709, "fcm_dpo/margin": 296.268798828125, "fcm_dpo/q_t": 0.42436882853507996, "grad_norm": 39.329833984375, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -1.0949430465698242, "logits/rejected": -1.0765621662139893, "logps/chosen": -801.4141845703125, "logps/ref_chosen": -58.46883010864258, "logps/ref_rejected": -72.92941284179688, "logps/rejected": -1112.1435546875, "loss": 1.2346, "margin_dpo/margin_mean": 296.268798828125, "margin_dpo/margin_std": 738.4291381835938, "step": 451 }, { "KL/chosen_KL_mean": -594.1732177734375, "KL/mean": -763.3720703125, "KL/rejected_KL_mean": -932.5709228515625, "KL/std": 460.3730773925781, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0011289503891021013, "fcm_dpo/delta": 0.02220895327627659, "fcm_dpo/margin": 338.39764404296875, "fcm_dpo/q_t": 0.41150838136672974, "grad_norm": 33.76683044433594, "learning_rate": 1.549222776991186e-07, "logits/chosen": -0.9474809169769287, "logits/rejected": -0.9722226858139038, "logps/chosen": -644.5637817382812, "logps/ref_chosen": -50.39055252075195, "logps/ref_rejected": -97.77142333984375, "logps/rejected": -1030.34228515625, "loss": 1.1039, "margin_dpo/margin_mean": 338.39764404296875, "margin_dpo/margin_std": 456.95947265625, "step": 452 }, { "KL/chosen_KL_mean": -649.19287109375, "KL/mean": -851.3839721679688, "KL/rejected_KL_mean": -1053.5750732421875, "KL/std": 513.1878051757812, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0010724079329520464, "fcm_dpo/delta": -0.04788634181022644, "fcm_dpo/margin": 404.3822021484375, "fcm_dpo/q_t": 0.40426748991012573, "grad_norm": 26.855628967285156, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.0186734199523926, "logits/rejected": -1.0134930610656738, "logps/chosen": -706.90771484375, "logps/ref_chosen": -57.71485137939453, "logps/ref_rejected": -82.20741271972656, "logps/rejected": -1135.782470703125, "loss": 1.1022, "margin_dpo/margin_mean": 404.3821716308594, "margin_dpo/margin_std": 631.357421875, "step": 453 }, { "KL/chosen_KL_mean": -824.8762817382812, "KL/mean": -1020.529052734375, "KL/rejected_KL_mean": -1216.181640625, "KL/std": 584.076171875, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0010508534032851458, "fcm_dpo/delta": -0.01683557778596878, "fcm_dpo/margin": 391.305419921875, "fcm_dpo/q_t": 0.4103407859802246, "grad_norm": 36.570125579833984, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.0660200119018555, "logits/rejected": -1.0628576278686523, "logps/chosen": -885.8218994140625, "logps/ref_chosen": -60.945648193359375, "logps/ref_rejected": -84.95079040527344, "logps/rejected": -1301.1324462890625, "loss": 1.1279, "margin_dpo/margin_mean": 391.305419921875, "margin_dpo/margin_std": 684.9031982421875, "step": 454 }, { "KL/chosen_KL_mean": -688.5186157226562, "KL/mean": -917.4297485351562, "KL/rejected_KL_mean": -1146.3408203125, "KL/std": 531.5499877929688, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0010322749149054289, "fcm_dpo/delta": -0.09659485518932343, "fcm_dpo/margin": 457.82232666015625, "fcm_dpo/q_t": 0.38912808895111084, "grad_norm": 33.90567398071289, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -0.9941831827163696, "logits/rejected": -1.0336955785751343, "logps/chosen": -733.4053344726562, "logps/ref_chosen": -44.88671112060547, "logps/ref_rejected": -115.30147552490234, "logps/rejected": -1261.642333984375, "loss": 1.063, "margin_dpo/margin_mean": 457.8222961425781, "margin_dpo/margin_std": 598.6575927734375, "step": 455 }, { "KL/chosen_KL_mean": -691.5862426757812, "KL/mean": -909.7722778320312, "KL/rejected_KL_mean": -1127.958251953125, "KL/std": 566.892578125, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0009644461097195745, "fcm_dpo/delta": -0.02674005925655365, "fcm_dpo/margin": 436.37200927734375, "fcm_dpo/q_t": 0.40306729078292847, "grad_norm": 33.307159423828125, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -0.9692044258117676, "logits/rejected": -0.9966904520988464, "logps/chosen": -748.623046875, "logps/ref_chosen": -57.036781311035156, "logps/ref_rejected": -105.21784210205078, "logps/rejected": -1233.1761474609375, "loss": 1.0658, "margin_dpo/margin_mean": 436.37200927734375, "margin_dpo/margin_std": 519.3140258789062, "step": 456 }, { "KL/chosen_KL_mean": -584.7559204101562, "KL/mean": -828.9456787109375, "KL/rejected_KL_mean": -1073.135498046875, "KL/std": 520.8341064453125, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0009004294406622648, "fcm_dpo/delta": -0.0521702915430069, "fcm_dpo/margin": 488.3794860839844, "fcm_dpo/q_t": 0.39493173360824585, "grad_norm": 42.788997650146484, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -0.982511043548584, "logits/rejected": -1.0036410093307495, "logps/chosen": -638.9984130859375, "logps/ref_chosen": -54.24253845214844, "logps/ref_rejected": -85.10956573486328, "logps/rejected": -1158.2449951171875, "loss": 1.0442, "margin_dpo/margin_mean": 488.3795166015625, "margin_dpo/margin_std": 512.117919921875, "step": 457 }, { "KL/chosen_KL_mean": -680.8214721679688, "KL/mean": -844.1943359375, "KL/rejected_KL_mean": -1007.5672607421875, "KL/std": 468.6507568359375, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.0009023591410368681, "fcm_dpo/delta": 0.005208446644246578, "fcm_dpo/margin": 326.74578857421875, "fcm_dpo/q_t": 0.4328933656215668, "grad_norm": 29.520099639892578, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -0.9825581312179565, "logits/rejected": -0.9830326437950134, "logps/chosen": -736.2303466796875, "logps/ref_chosen": -55.40888214111328, "logps/ref_rejected": -97.68325805664062, "logps/rejected": -1105.25048828125, "loss": 1.1728, "margin_dpo/margin_mean": 326.74578857421875, "margin_dpo/margin_std": 542.132080078125, "step": 458 }, { "KL/chosen_KL_mean": -676.8546752929688, "KL/mean": -901.5927734375, "KL/rejected_KL_mean": -1126.330810546875, "KL/std": 550.5717163085938, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0008877612417563796, "fcm_dpo/delta": -1.3709068298339844e-06, "fcm_dpo/margin": 449.47607421875, "fcm_dpo/q_t": 0.40948015451431274, "grad_norm": 22.407264709472656, "learning_rate": 1.466771464027316e-07, "logits/chosen": -0.9684402942657471, "logits/rejected": -1.0020538568496704, "logps/chosen": -723.4121704101562, "logps/ref_chosen": -46.55748748779297, "logps/ref_rejected": -86.16854095458984, "logps/rejected": -1212.4993896484375, "loss": 1.1013, "margin_dpo/margin_mean": 449.47607421875, "margin_dpo/margin_std": 635.2473754882812, "step": 459 }, { "KL/chosen_KL_mean": -719.9354248046875, "KL/mean": -945.6378173828125, "KL/rejected_KL_mean": -1171.3402099609375, "KL/std": 503.3649597167969, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.00090052280575037, "fcm_dpo/delta": -0.008194062858819962, "fcm_dpo/margin": 451.40478515625, "fcm_dpo/q_t": 0.40608513355255127, "grad_norm": 31.070011138916016, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -0.9713107943534851, "logits/rejected": -1.0081372261047363, "logps/chosen": -771.5703125, "logps/ref_chosen": -51.63489532470703, "logps/ref_rejected": -104.11935424804688, "logps/rejected": -1275.45947265625, "loss": 1.0757, "margin_dpo/margin_mean": 451.40478515625, "margin_dpo/margin_std": 542.3765869140625, "step": 460 }, { "KL/chosen_KL_mean": -740.08251953125, "KL/mean": -919.7174072265625, "KL/rejected_KL_mean": -1099.352294921875, "KL/std": 511.79473876953125, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0009528920636512339, "fcm_dpo/delta": 0.06786449253559113, "fcm_dpo/margin": 359.2697448730469, "fcm_dpo/q_t": 0.4245432913303375, "grad_norm": 25.247751235961914, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.029995083808899, "logits/rejected": -1.0514973402023315, "logps/chosen": -795.2644653320312, "logps/ref_chosen": -55.18195724487305, "logps/ref_rejected": -86.47689819335938, "logps/rejected": -1185.8291015625, "loss": 1.1423, "margin_dpo/margin_mean": 359.269775390625, "margin_dpo/margin_std": 578.6834716796875, "step": 461 }, { "KL/chosen_KL_mean": -744.9185791015625, "KL/mean": -907.572265625, "KL/rejected_KL_mean": -1070.2259521484375, "KL/std": 544.3043212890625, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.0010181250981986523, "fcm_dpo/delta": 0.0823717713356018, "fcm_dpo/margin": 325.307373046875, "fcm_dpo/q_t": 0.4292236864566803, "grad_norm": 35.24434280395508, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -1.0601680278778076, "logits/rejected": -1.0431756973266602, "logps/chosen": -814.8466796875, "logps/ref_chosen": -69.92803192138672, "logps/ref_rejected": -78.84111022949219, "logps/rejected": -1149.067138671875, "loss": 1.1657, "margin_dpo/margin_mean": 325.307373046875, "margin_dpo/margin_std": 593.1353759765625, "step": 462 }, { "KL/chosen_KL_mean": -733.4375, "KL/mean": -934.1990966796875, "KL/rejected_KL_mean": -1134.960693359375, "KL/std": 516.4095458984375, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.001036065979860723, "fcm_dpo/delta": -0.020512185990810394, "fcm_dpo/margin": 401.523193359375, "fcm_dpo/q_t": 0.4055131673812866, "grad_norm": 23.178422927856445, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -0.9877623319625854, "logits/rejected": -1.0045299530029297, "logps/chosen": -788.7119140625, "logps/ref_chosen": -55.27437210083008, "logps/ref_rejected": -89.02497863769531, "logps/rejected": -1223.9857177734375, "loss": 1.0915, "margin_dpo/margin_mean": 401.523193359375, "margin_dpo/margin_std": 563.6845092773438, "step": 463 }, { "KL/chosen_KL_mean": -697.1048583984375, "KL/mean": -991.504150390625, "KL/rejected_KL_mean": -1285.9033203125, "KL/std": 571.4072265625, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0008614182588644326, "fcm_dpo/delta": -0.16394121944904327, "fcm_dpo/margin": 588.7985229492188, "fcm_dpo/q_t": 0.37307262420654297, "grad_norm": 46.0228157043457, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -0.9791674017906189, "logits/rejected": -1.0607080459594727, "logps/chosen": -748.0171508789062, "logps/ref_chosen": -50.91230010986328, "logps/ref_rejected": -102.4893798828125, "logps/rejected": -1388.3927001953125, "loss": 1.002, "margin_dpo/margin_mean": 588.7985229492188, "margin_dpo/margin_std": 583.0584716796875, "step": 464 }, { "KL/chosen_KL_mean": -694.8887939453125, "KL/mean": -952.684814453125, "KL/rejected_KL_mean": -1210.4808349609375, "KL/std": 611.1224365234375, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0008492013439536095, "fcm_dpo/delta": -0.048867009580135345, "fcm_dpo/margin": 515.5921020507812, "fcm_dpo/q_t": 0.39614564180374146, "grad_norm": 30.500961303710938, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -0.9008214473724365, "logits/rejected": -0.9379734396934509, "logps/chosen": -755.005615234375, "logps/ref_chosen": -60.116851806640625, "logps/ref_rejected": -113.94602966308594, "logps/rejected": -1324.4268798828125, "loss": 1.0666, "margin_dpo/margin_mean": 515.5921020507812, "margin_dpo/margin_std": 642.5521240234375, "step": 465 }, { "KL/chosen_KL_mean": -780.0445556640625, "KL/mean": -987.330322265625, "KL/rejected_KL_mean": -1194.6162109375, "KL/std": 571.94091796875, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0008426806889474392, "fcm_dpo/delta": 0.06057006120681763, "fcm_dpo/margin": 414.57159423828125, "fcm_dpo/q_t": 0.4206228256225586, "grad_norm": 33.371273040771484, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.0010290145874023, "logits/rejected": -1.007284164428711, "logps/chosen": -832.9654541015625, "logps/ref_chosen": -52.920921325683594, "logps/ref_rejected": -90.3154296875, "logps/rejected": -1284.931640625, "loss": 1.1329, "margin_dpo/margin_mean": 414.57159423828125, "margin_dpo/margin_std": 603.785400390625, "step": 466 }, { "KL/chosen_KL_mean": -1015.46484375, "KL/mean": -1223.3994140625, "KL/rejected_KL_mean": -1431.3341064453125, "KL/std": 710.5194091796875, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0008342203218489885, "fcm_dpo/delta": -0.047805603593587875, "fcm_dpo/margin": 415.86932373046875, "fcm_dpo/q_t": 0.42023301124572754, "grad_norm": 45.32796859741211, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -1.1425104141235352, "logits/rejected": -1.14357328414917, "logps/chosen": -1094.1806640625, "logps/ref_chosen": -78.7158203125, "logps/ref_rejected": -102.86019897460938, "logps/rejected": -1534.1943359375, "loss": 1.1881, "margin_dpo/margin_mean": 415.86932373046875, "margin_dpo/margin_std": 850.1127319335938, "step": 467 }, { "KL/chosen_KL_mean": -783.6282348632812, "KL/mean": -1058.208984375, "KL/rejected_KL_mean": -1332.789794921875, "KL/std": 650.537109375, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0008028654847294092, "fcm_dpo/delta": -0.052505575120449066, "fcm_dpo/margin": 549.16162109375, "fcm_dpo/q_t": 0.40023478865623474, "grad_norm": 24.539749145507812, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.0484925508499146, "logits/rejected": -1.0707437992095947, "logps/chosen": -853.5635986328125, "logps/ref_chosen": -69.93536376953125, "logps/ref_rejected": -101.02880859375, "logps/rejected": -1433.818603515625, "loss": 1.074, "margin_dpo/margin_mean": 549.16162109375, "margin_dpo/margin_std": 754.311279296875, "step": 468 }, { "KL/chosen_KL_mean": -840.949951171875, "KL/mean": -1125.788330078125, "KL/rejected_KL_mean": -1410.626953125, "KL/std": 636.927001953125, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0007483696099370718, "fcm_dpo/delta": -0.038286350667476654, "fcm_dpo/margin": 569.6768798828125, "fcm_dpo/q_t": 0.3994740843772888, "grad_norm": 37.96442413330078, "learning_rate": 1.351323902551631e-07, "logits/chosen": -1.0542805194854736, "logits/rejected": -1.0895564556121826, "logps/chosen": -909.0746459960938, "logps/ref_chosen": -68.12469482421875, "logps/ref_rejected": -104.78640747070312, "logps/rejected": -1515.413330078125, "loss": 1.0689, "margin_dpo/margin_mean": 569.6769409179688, "margin_dpo/margin_std": 686.1154174804688, "step": 469 }, { "KL/chosen_KL_mean": -708.578125, "KL/mean": -937.5423583984375, "KL/rejected_KL_mean": -1166.5064697265625, "KL/std": 602.3682861328125, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0007999057415872812, "fcm_dpo/delta": 0.038524776697158813, "fcm_dpo/margin": 457.9284362792969, "fcm_dpo/q_t": 0.41636377573013306, "grad_norm": 24.361658096313477, "learning_rate": 1.339940635976592e-07, "logits/chosen": -0.9929207563400269, "logits/rejected": -1.0160140991210938, "logps/chosen": -752.3699951171875, "logps/ref_chosen": -43.791927337646484, "logps/ref_rejected": -82.70285034179688, "logps/rejected": -1249.2093505859375, "loss": 1.1274, "margin_dpo/margin_mean": 457.9284362792969, "margin_dpo/margin_std": 697.048095703125, "step": 470 }, { "KL/chosen_KL_mean": -843.971923828125, "KL/mean": -1043.7379150390625, "KL/rejected_KL_mean": -1243.5037841796875, "KL/std": 644.9224853515625, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0008243155316449702, "fcm_dpo/delta": 0.0843021422624588, "fcm_dpo/margin": 399.5318603515625, "fcm_dpo/q_t": 0.42968451976776123, "grad_norm": 39.733333587646484, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.1035377979278564, "logits/rejected": -1.1254429817199707, "logps/chosen": -907.3114624023438, "logps/ref_chosen": -63.33952331542969, "logps/ref_rejected": -83.61048126220703, "logps/rejected": -1327.1142578125, "loss": 1.1649, "margin_dpo/margin_mean": 399.5318603515625, "margin_dpo/margin_std": 707.281982421875, "step": 471 }, { "KL/chosen_KL_mean": -862.677734375, "KL/mean": -1102.48583984375, "KL/rejected_KL_mean": -1342.2938232421875, "KL/std": 753.7301025390625, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0008416681084781885, "fcm_dpo/delta": -0.006307002156972885, "fcm_dpo/margin": 479.61614990234375, "fcm_dpo/q_t": 0.411409854888916, "grad_norm": 30.461442947387695, "learning_rate": 1.317266107909975e-07, "logits/chosen": -1.062950849533081, "logits/rejected": -1.0488746166229248, "logps/chosen": -946.3438110351562, "logps/ref_chosen": -83.66610717773438, "logps/ref_rejected": -117.20919799804688, "logps/rejected": -1459.5030517578125, "loss": 1.1356, "margin_dpo/margin_mean": 479.61614990234375, "margin_dpo/margin_std": 826.3499755859375, "step": 472 }, { "KL/chosen_KL_mean": -1003.8850708007812, "KL/mean": -1114.1983642578125, "KL/rejected_KL_mean": -1224.5115966796875, "KL/std": 738.931640625, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0008559616981074214, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 220.62660217285156, "fcm_dpo/q_t": 0.4583190083503723, "grad_norm": 101.26480102539062, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.0762083530426025, "logits/rejected": -1.0582425594329834, "logps/chosen": -1067.382080078125, "logps/ref_chosen": -63.49696731567383, "logps/ref_rejected": -81.14657592773438, "logps/rejected": -1305.658203125, "loss": 1.3516, "margin_dpo/margin_mean": 220.62660217285156, "margin_dpo/margin_std": 942.107421875, "step": 473 }, { "KL/chosen_KL_mean": -769.4073486328125, "KL/mean": -986.923583984375, "KL/rejected_KL_mean": -1204.43994140625, "KL/std": 693.4749755859375, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.00081443547969684, "fcm_dpo/delta": -0.10206399112939835, "fcm_dpo/margin": 435.03271484375, "fcm_dpo/q_t": 0.4185516834259033, "grad_norm": 32.762020111083984, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.048344612121582, "logits/rejected": -1.0651113986968994, "logps/chosen": -822.019287109375, "logps/ref_chosen": -52.6119384765625, "logps/ref_rejected": -90.08041381835938, "logps/rejected": -1294.5203857421875, "loss": 1.1724, "margin_dpo/margin_mean": 435.03271484375, "margin_dpo/margin_std": 791.048583984375, "step": 474 }, { "KL/chosen_KL_mean": -539.2615356445312, "KL/mean": -790.7316284179688, "KL/rejected_KL_mean": -1042.20166015625, "KL/std": 493.97027587890625, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0007748671341687441, "fcm_dpo/delta": 0.01266252901405096, "fcm_dpo/margin": 502.940185546875, "fcm_dpo/q_t": 0.40863102674484253, "grad_norm": 52.324066162109375, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -1.0229177474975586, "logits/rejected": -1.0607411861419678, "logps/chosen": -581.7567138671875, "logps/ref_chosen": -42.49519348144531, "logps/ref_rejected": -90.06294250488281, "logps/rejected": -1132.2646484375, "loss": 1.0766, "margin_dpo/margin_mean": 502.940185546875, "margin_dpo/margin_std": 544.89208984375, "step": 475 }, { "KL/chosen_KL_mean": -634.4459228515625, "KL/mean": -844.801025390625, "KL/rejected_KL_mean": -1055.1561279296875, "KL/std": 564.7354736328125, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.0008217949070967734, "fcm_dpo/delta": 0.06553815305233002, "fcm_dpo/margin": 420.71014404296875, "fcm_dpo/q_t": 0.4223518669605255, "grad_norm": 62.65557098388672, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -0.9976848363876343, "logits/rejected": -1.0245535373687744, "logps/chosen": -677.3953247070312, "logps/ref_chosen": -42.94938278198242, "logps/ref_rejected": -73.71023559570312, "logps/rejected": -1128.866455078125, "loss": 1.1221, "margin_dpo/margin_mean": 420.71014404296875, "margin_dpo/margin_std": 576.537353515625, "step": 476 }, { "KL/chosen_KL_mean": -674.2581787109375, "KL/mean": -865.6505126953125, "KL/rejected_KL_mean": -1057.0428466796875, "KL/std": 557.9041748046875, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0008936204831115901, "fcm_dpo/delta": 0.06722961366176605, "fcm_dpo/margin": 382.7846984863281, "fcm_dpo/q_t": 0.42294949293136597, "grad_norm": 28.50010108947754, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -1.0419853925704956, "logits/rejected": -1.0101861953735352, "logps/chosen": -745.03076171875, "logps/ref_chosen": -70.77261352539062, "logps/ref_rejected": -76.13737487792969, "logps/rejected": -1133.18017578125, "loss": 1.1533, "margin_dpo/margin_mean": 382.78466796875, "margin_dpo/margin_std": 655.9279174804688, "step": 477 }, { "KL/chosen_KL_mean": -557.695068359375, "KL/mean": -778.8765869140625, "KL/rejected_KL_mean": -1000.0580444335938, "KL/std": 482.09033203125, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.0009058309369720519, "fcm_dpo/delta": -0.0017963144928216934, "fcm_dpo/margin": 442.363037109375, "fcm_dpo/q_t": 0.407336950302124, "grad_norm": 43.02243423461914, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.946250319480896, "logits/rejected": -0.9788520336151123, "logps/chosen": -599.1356201171875, "logps/ref_chosen": -41.440513610839844, "logps/ref_rejected": -85.36196899414062, "logps/rejected": -1085.4200439453125, "loss": 1.0866, "margin_dpo/margin_mean": 442.363037109375, "margin_dpo/margin_std": 562.7466430664062, "step": 478 }, { "KL/chosen_KL_mean": -741.406982421875, "KL/mean": -950.5042114257812, "KL/rejected_KL_mean": -1159.601318359375, "KL/std": 631.4393920898438, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.0009333643829450011, "fcm_dpo/delta": 0.004260525107383728, "fcm_dpo/margin": 418.19439697265625, "fcm_dpo/q_t": 0.4147453308105469, "grad_norm": 30.091028213500977, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -0.9998948574066162, "logits/rejected": -1.0395790338516235, "logps/chosen": -795.31494140625, "logps/ref_chosen": -53.907920837402344, "logps/ref_rejected": -95.1163330078125, "logps/rejected": -1254.7177734375, "loss": 1.1436, "margin_dpo/margin_mean": 418.19439697265625, "margin_dpo/margin_std": 732.4306640625, "step": 479 }, { "KL/chosen_KL_mean": -865.813720703125, "KL/mean": -1064.1876220703125, "KL/rejected_KL_mean": -1262.5615234375, "KL/std": 623.685791015625, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0009484676993452013, "fcm_dpo/delta": 0.020787112414836884, "fcm_dpo/margin": 396.74774169921875, "fcm_dpo/q_t": 0.41742902994155884, "grad_norm": 50.18581008911133, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.0877306461334229, "logits/rejected": -1.0981130599975586, "logps/chosen": -924.4964599609375, "logps/ref_chosen": -58.682701110839844, "logps/ref_rejected": -82.93248748779297, "logps/rejected": -1345.493896484375, "loss": 1.1513, "margin_dpo/margin_mean": 396.74774169921875, "margin_dpo/margin_std": 687.515869140625, "step": 480 }, { "KL/chosen_KL_mean": -735.0712890625, "KL/mean": -1004.9774169921875, "KL/rejected_KL_mean": -1274.883544921875, "KL/std": 640.9407958984375, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.000862010580021888, "fcm_dpo/delta": -0.08503904193639755, "fcm_dpo/margin": 539.812255859375, "fcm_dpo/q_t": 0.391613632440567, "grad_norm": 32.593658447265625, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.1026833057403564, "logits/rejected": -1.142751693725586, "logps/chosen": -790.0355224609375, "logps/ref_chosen": -54.964271545410156, "logps/ref_rejected": -92.42044067382812, "logps/rejected": -1367.303955078125, "loss": 1.0549, "margin_dpo/margin_mean": 539.812255859375, "margin_dpo/margin_std": 687.2750854492188, "step": 481 }, { "KL/chosen_KL_mean": -830.9714965820312, "KL/mean": -957.885009765625, "KL/rejected_KL_mean": -1084.7984619140625, "KL/std": 625.652099609375, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0008747933316044509, "fcm_dpo/delta": 0.0788145363330841, "fcm_dpo/margin": 253.82705688476562, "fcm_dpo/q_t": 0.4496998190879822, "grad_norm": 46.61983871459961, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -0.9891968369483948, "logits/rejected": -0.9823375940322876, "logps/chosen": -898.5249633789062, "logps/ref_chosen": -67.553466796875, "logps/ref_rejected": -87.58953857421875, "logps/rejected": -1172.387939453125, "loss": 1.2716, "margin_dpo/margin_mean": 253.82705688476562, "margin_dpo/margin_std": 734.9395751953125, "step": 482 }, { "KL/chosen_KL_mean": -761.9283447265625, "KL/mean": -1030.1768798828125, "KL/rejected_KL_mean": -1298.425537109375, "KL/std": 638.6688232421875, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0008516391972079873, "fcm_dpo/delta": -0.07522018998861313, "fcm_dpo/margin": 536.4971923828125, "fcm_dpo/q_t": 0.3947756588459015, "grad_norm": 29.71031379699707, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.0698552131652832, "logits/rejected": -1.0944595336914062, "logps/chosen": -825.2581787109375, "logps/ref_chosen": -63.32981872558594, "logps/ref_rejected": -95.78697204589844, "logps/rejected": -1394.21240234375, "loss": 1.0606, "margin_dpo/margin_mean": 536.4971313476562, "margin_dpo/margin_std": 694.40087890625, "step": 483 }, { "KL/chosen_KL_mean": -655.5699462890625, "KL/mean": -913.00341796875, "KL/rejected_KL_mean": -1170.43701171875, "KL/std": 608.8349609375, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.000842659268528223, "fcm_dpo/delta": -0.0462777316570282, "fcm_dpo/margin": 514.8670043945312, "fcm_dpo/q_t": 0.3998936414718628, "grad_norm": 36.89812469482422, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -1.0860092639923096, "logits/rejected": -1.119147539138794, "logps/chosen": -714.7080688476562, "logps/ref_chosen": -59.13812255859375, "logps/ref_rejected": -84.37144470214844, "logps/rejected": -1254.808349609375, "loss": 1.0775, "margin_dpo/margin_mean": 514.8670043945312, "margin_dpo/margin_std": 665.43310546875, "step": 484 }, { "KL/chosen_KL_mean": -662.37353515625, "KL/mean": -922.7737426757812, "KL/rejected_KL_mean": -1183.174072265625, "KL/std": 650.9013671875, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0007992424070835114, "fcm_dpo/delta": -0.020606661215424538, "fcm_dpo/margin": 520.8004760742188, "fcm_dpo/q_t": 0.4069255590438843, "grad_norm": 32.7896842956543, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -0.9968384504318237, "logits/rejected": -1.0324972867965698, "logps/chosen": -721.2230834960938, "logps/ref_chosen": -58.849571228027344, "logps/ref_rejected": -103.36408233642578, "logps/rejected": -1286.5380859375, "loss": 1.0967, "margin_dpo/margin_mean": 520.8004760742188, "margin_dpo/margin_std": 764.5574951171875, "step": 485 }, { "KL/chosen_KL_mean": -828.6250610351562, "KL/mean": -1101.12109375, "KL/rejected_KL_mean": -1373.6173095703125, "KL/std": 724.00244140625, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0007718969136476517, "fcm_dpo/delta": -0.02629101648926735, "fcm_dpo/margin": 544.9921875, "fcm_dpo/q_t": 0.4044193923473358, "grad_norm": 39.17950439453125, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.1160706281661987, "logits/rejected": -1.1396567821502686, "logps/chosen": -883.884765625, "logps/ref_chosen": -55.25966262817383, "logps/ref_rejected": -92.13936614990234, "logps/rejected": -1465.756591796875, "loss": 1.1214, "margin_dpo/margin_mean": 544.9921875, "margin_dpo/margin_std": 907.802001953125, "step": 486 }, { "KL/chosen_KL_mean": -896.8312377929688, "KL/mean": -1101.290771484375, "KL/rejected_KL_mean": -1305.750244140625, "KL/std": 672.8538818359375, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0008255624561570585, "fcm_dpo/delta": 0.07200143486261368, "fcm_dpo/margin": 408.9190673828125, "fcm_dpo/q_t": 0.42319005727767944, "grad_norm": 29.267309188842773, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -1.1223185062408447, "logits/rejected": -1.135122537612915, "logps/chosen": -949.89453125, "logps/ref_chosen": -53.06330871582031, "logps/ref_rejected": -92.41883087158203, "logps/rejected": -1398.1690673828125, "loss": 1.1616, "margin_dpo/margin_mean": 408.9190673828125, "margin_dpo/margin_std": 730.7452392578125, "step": 487 }, { "KL/chosen_KL_mean": -746.6181640625, "KL/mean": -975.8692626953125, "KL/rejected_KL_mean": -1205.120361328125, "KL/std": 613.75439453125, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0008516233647242188, "fcm_dpo/delta": 0.0090005062520504, "fcm_dpo/margin": 458.5020751953125, "fcm_dpo/q_t": 0.41139358282089233, "grad_norm": 33.18574523925781, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.0628390312194824, "logits/rejected": -1.0790064334869385, "logps/chosen": -798.8463745117188, "logps/ref_chosen": -52.22815704345703, "logps/ref_rejected": -84.00656127929688, "logps/rejected": -1289.1268310546875, "loss": 1.1022, "margin_dpo/margin_mean": 458.5020751953125, "margin_dpo/margin_std": 630.7826538085938, "step": 488 }, { "KL/chosen_KL_mean": -688.994873046875, "KL/mean": -898.1849365234375, "KL/rejected_KL_mean": -1107.375, "KL/std": 607.5086669921875, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.0008741497294977307, "fcm_dpo/delta": 0.03951010853052139, "fcm_dpo/margin": 418.3800964355469, "fcm_dpo/q_t": 0.41885343194007874, "grad_norm": 23.560508728027344, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.0819485187530518, "logits/rejected": -1.084855318069458, "logps/chosen": -744.9844970703125, "logps/ref_chosen": -55.989627838134766, "logps/ref_rejected": -79.39812469482422, "logps/rejected": -1186.7730712890625, "loss": 1.1243, "margin_dpo/margin_mean": 418.380126953125, "margin_dpo/margin_std": 633.3609619140625, "step": 489 }, { "KL/chosen_KL_mean": -841.709716796875, "KL/mean": -1069.041015625, "KL/rejected_KL_mean": -1296.372314453125, "KL/std": 748.8676147460938, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0008761119097471237, "fcm_dpo/delta": 0.0016551017761230469, "fcm_dpo/margin": 454.66265869140625, "fcm_dpo/q_t": 0.41499996185302734, "grad_norm": 53.128875732421875, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -1.144291639328003, "logits/rejected": -1.196203589439392, "logps/chosen": -894.0761108398438, "logps/ref_chosen": -52.36639404296875, "logps/ref_rejected": -110.4090576171875, "logps/rejected": -1406.7813720703125, "loss": 1.1641, "margin_dpo/margin_mean": 454.6626281738281, "margin_dpo/margin_std": 890.287353515625, "step": 490 }, { "KL/chosen_KL_mean": -801.8388061523438, "KL/mean": -956.7105102539062, "KL/rejected_KL_mean": -1111.5821533203125, "KL/std": 636.75390625, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0009558784076943994, "fcm_dpo/delta": 0.12077458947896957, "fcm_dpo/margin": 309.7433776855469, "fcm_dpo/q_t": 0.4361428916454315, "grad_norm": 39.206363677978516, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.1435798406600952, "logits/rejected": -1.1361366510391235, "logps/chosen": -861.955078125, "logps/ref_chosen": -60.11626434326172, "logps/ref_rejected": -73.27278900146484, "logps/rejected": -1184.85498046875, "loss": 1.1942, "margin_dpo/margin_mean": 309.7433776855469, "margin_dpo/margin_std": 618.770751953125, "step": 491 }, { "KL/chosen_KL_mean": -781.2764892578125, "KL/mean": -925.2930908203125, "KL/rejected_KL_mean": -1069.3095703125, "KL/std": 593.3115844726562, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0010589244775474072, "fcm_dpo/delta": 0.11252547055482864, "fcm_dpo/margin": 288.0330810546875, "fcm_dpo/q_t": 0.43613219261169434, "grad_norm": 65.94483184814453, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.2232658863067627, "logits/rejected": -1.1998727321624756, "logps/chosen": -835.270751953125, "logps/ref_chosen": -53.994178771972656, "logps/ref_rejected": -72.65962219238281, "logps/rejected": -1141.96923828125, "loss": 1.2344, "margin_dpo/margin_mean": 288.0330810546875, "margin_dpo/margin_std": 703.6218872070312, "step": 492 }, { "KL/chosen_KL_mean": -823.78759765625, "KL/mean": -995.8330078125, "KL/rejected_KL_mean": -1167.87841796875, "KL/std": 663.9137573242188, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.001078893314115703, "fcm_dpo/delta": 0.029794402420520782, "fcm_dpo/margin": 344.0907897949219, "fcm_dpo/q_t": 0.41683828830718994, "grad_norm": 46.201839447021484, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -1.2597391605377197, "logits/rejected": -1.2192950248718262, "logps/chosen": -899.2848510742188, "logps/ref_chosen": -75.49723815917969, "logps/ref_rejected": -87.32301330566406, "logps/rejected": -1255.201416015625, "loss": 1.1584, "margin_dpo/margin_mean": 344.0908203125, "margin_dpo/margin_std": 617.4366455078125, "step": 493 }, { "KL/chosen_KL_mean": -648.2301025390625, "KL/mean": -912.5694580078125, "KL/rejected_KL_mean": -1176.9088134765625, "KL/std": 621.4219360351562, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.000986246857792139, "fcm_dpo/delta": -0.16815879940986633, "fcm_dpo/margin": 528.6787109375, "fcm_dpo/q_t": 0.37546783685684204, "grad_norm": 32.91933822631836, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.0481430292129517, "logits/rejected": -1.0907659530639648, "logps/chosen": -689.58935546875, "logps/ref_chosen": -41.35926818847656, "logps/ref_rejected": -86.09136962890625, "logps/rejected": -1263.000244140625, "loss": 1.0206, "margin_dpo/margin_mean": 528.6787109375, "margin_dpo/margin_std": 635.6686401367188, "step": 494 }, { "KL/chosen_KL_mean": -706.6998291015625, "KL/mean": -920.7889404296875, "KL/rejected_KL_mean": -1134.8779296875, "KL/std": 593.68115234375, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0009533504489809275, "fcm_dpo/delta": -0.010413305833935738, "fcm_dpo/margin": 428.17816162109375, "fcm_dpo/q_t": 0.4118821620941162, "grad_norm": 38.663597106933594, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -1.2045929431915283, "logits/rejected": -1.22074294090271, "logps/chosen": -770.23486328125, "logps/ref_chosen": -63.53507995605469, "logps/ref_rejected": -91.42443084716797, "logps/rejected": -1226.302490234375, "loss": 1.1255, "margin_dpo/margin_mean": 428.17816162109375, "margin_dpo/margin_std": 734.76904296875, "step": 495 }, { "KL/chosen_KL_mean": -847.32470703125, "KL/mean": -956.1488037109375, "KL/rejected_KL_mean": -1064.972900390625, "KL/std": 484.2099609375, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0009861316066235304, "fcm_dpo/delta": 0.08399142324924469, "fcm_dpo/margin": 217.6483154296875, "fcm_dpo/q_t": 0.45106980204582214, "grad_norm": 73.58963012695312, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -1.1229890584945679, "logits/rejected": -1.0876094102859497, "logps/chosen": -919.9166259765625, "logps/ref_chosen": -72.5919189453125, "logps/ref_rejected": -84.32933807373047, "logps/rejected": -1149.30224609375, "loss": 1.2516, "margin_dpo/margin_mean": 217.6483154296875, "margin_dpo/margin_std": 544.780517578125, "step": 496 }, { "KL/chosen_KL_mean": -713.8504028320312, "KL/mean": -832.6170654296875, "KL/rejected_KL_mean": -951.3837890625, "KL/std": 529.62255859375, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0010455565061420202, "fcm_dpo/delta": 0.017400572076439857, "fcm_dpo/margin": 237.53341674804688, "fcm_dpo/q_t": 0.44449126720428467, "grad_norm": 23.886857986450195, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.10499107837677, "logits/rejected": -1.0940041542053223, "logps/chosen": -772.4443359375, "logps/ref_chosen": -58.59397506713867, "logps/ref_rejected": -76.28836822509766, "logps/rejected": -1027.672119140625, "loss": 1.2264, "margin_dpo/margin_mean": 237.53341674804688, "margin_dpo/margin_std": 521.065673828125, "step": 497 }, { "KL/chosen_KL_mean": -697.0528564453125, "KL/mean": -902.389404296875, "KL/rejected_KL_mean": -1107.725830078125, "KL/std": 593.9395751953125, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0010199977550655603, "fcm_dpo/delta": -0.024301331490278244, "fcm_dpo/margin": 410.67303466796875, "fcm_dpo/q_t": 0.40479618310928345, "grad_norm": 30.96971321105957, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.1298916339874268, "logits/rejected": -1.1229459047317505, "logps/chosen": -768.258544921875, "logps/ref_chosen": -71.20565795898438, "logps/ref_rejected": -83.95803833007812, "logps/rejected": -1191.6839599609375, "loss": 1.1151, "margin_dpo/margin_mean": 410.67303466796875, "margin_dpo/margin_std": 663.222900390625, "step": 498 }, { "KL/chosen_KL_mean": -577.2625732421875, "KL/mean": -813.7861328125, "KL/rejected_KL_mean": -1050.3096923828125, "KL/std": 576.3260498046875, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0009620316559448838, "fcm_dpo/delta": -0.0720800906419754, "fcm_dpo/margin": 473.047119140625, "fcm_dpo/q_t": 0.3953206241130829, "grad_norm": 36.52189254760742, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.118801474571228, "logits/rejected": -1.1649246215820312, "logps/chosen": -628.5177001953125, "logps/ref_chosen": -51.25519561767578, "logps/ref_rejected": -101.07870483398438, "logps/rejected": -1151.388427734375, "loss": 1.0697, "margin_dpo/margin_mean": 473.047119140625, "margin_dpo/margin_std": 653.6705322265625, "step": 499 }, { "KL/chosen_KL_mean": -667.104248046875, "KL/mean": -863.0795288085938, "KL/rejected_KL_mean": -1059.054931640625, "KL/std": 504.5117492675781, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.000964233884587884, "fcm_dpo/delta": 0.027243074029684067, "fcm_dpo/margin": 391.9505615234375, "fcm_dpo/q_t": 0.4149992763996124, "grad_norm": 31.563304901123047, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.1256649494171143, "logits/rejected": -1.1642621755599976, "logps/chosen": -724.1317138671875, "logps/ref_chosen": -57.027442932128906, "logps/ref_rejected": -93.93421173095703, "logps/rejected": -1152.989013671875, "loss": 1.1294, "margin_dpo/margin_mean": 391.9505615234375, "margin_dpo/margin_std": 622.8431396484375, "step": 500 }, { "KL/chosen_KL_mean": -576.75732421875, "KL/mean": -784.8577880859375, "KL/rejected_KL_mean": -992.958251953125, "KL/std": 533.4993286132812, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.0009827818721532822, "fcm_dpo/delta": -0.012303851544857025, "fcm_dpo/margin": 416.20098876953125, "fcm_dpo/q_t": 0.40838170051574707, "grad_norm": 32.86457061767578, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.0381156206130981, "logits/rejected": -1.0312669277191162, "logps/chosen": -631.1168212890625, "logps/ref_chosen": -54.359527587890625, "logps/ref_rejected": -80.15670013427734, "logps/rejected": -1073.114990234375, "loss": 1.1167, "margin_dpo/margin_mean": 416.2009582519531, "margin_dpo/margin_std": 665.16796875, "step": 501 }, { "KL/chosen_KL_mean": -536.04345703125, "KL/mean": -745.6707763671875, "KL/rejected_KL_mean": -955.2979736328125, "KL/std": 505.134765625, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0009646775433793664, "fcm_dpo/delta": -0.00565625075250864, "fcm_dpo/margin": 419.25445556640625, "fcm_dpo/q_t": 0.40891292691230774, "grad_norm": 21.100860595703125, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.0066035985946655, "logits/rejected": -0.9983298182487488, "logps/chosen": -603.64404296875, "logps/ref_chosen": -67.60050964355469, "logps/ref_rejected": -82.94876098632812, "logps/rejected": -1038.246826171875, "loss": 1.0891, "margin_dpo/margin_mean": 419.25445556640625, "margin_dpo/margin_std": 562.9284057617188, "step": 502 }, { "KL/chosen_KL_mean": -536.14404296875, "KL/mean": -733.31298828125, "KL/rejected_KL_mean": -930.4820556640625, "KL/std": 481.45635986328125, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.0009646597900427878, "fcm_dpo/delta": 0.023929888382554054, "fcm_dpo/margin": 394.33795166015625, "fcm_dpo/q_t": 0.4147661328315735, "grad_norm": 21.984729766845703, "learning_rate": 9.831921068732571e-08, "logits/chosen": -0.999343752861023, "logits/rejected": -0.9944263696670532, "logps/chosen": -591.222412109375, "logps/ref_chosen": -55.078407287597656, "logps/ref_rejected": -82.50544738769531, "logps/rejected": -1012.9874877929688, "loss": 1.1043, "margin_dpo/margin_mean": 394.33795166015625, "margin_dpo/margin_std": 542.9268798828125, "step": 503 }, { "KL/chosen_KL_mean": -596.7254028320312, "KL/mean": -843.4456787109375, "KL/rejected_KL_mean": -1090.1658935546875, "KL/std": 561.0550537109375, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0009342943667434156, "fcm_dpo/delta": -0.07896675169467926, "fcm_dpo/margin": 493.44061279296875, "fcm_dpo/q_t": 0.39245522022247314, "grad_norm": 32.341758728027344, "learning_rate": 9.730107739932805e-08, "logits/chosen": -1.0672714710235596, "logits/rejected": -1.1046226024627686, "logps/chosen": -656.691162109375, "logps/ref_chosen": -59.96575164794922, "logps/ref_rejected": -103.76212310791016, "logps/rejected": -1193.927978515625, "loss": 1.058, "margin_dpo/margin_mean": 493.4405517578125, "margin_dpo/margin_std": 614.1143188476562, "step": 504 }, { "KL/chosen_KL_mean": -736.76806640625, "KL/mean": -838.422607421875, "KL/rejected_KL_mean": -940.0771484375, "KL/std": 535.1500244140625, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0009070243104360998, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 203.3091583251953, "fcm_dpo/q_t": 0.45628899335861206, "grad_norm": 33.11906814575195, "learning_rate": 9.628696786995188e-08, "logits/chosen": -1.0850073099136353, "logits/rejected": -1.0549143552780151, "logps/chosen": -812.9229736328125, "logps/ref_chosen": -76.1549072265625, "logps/ref_rejected": -88.58537292480469, "logps/rejected": -1028.6624755859375, "loss": 1.2666, "margin_dpo/margin_mean": 203.30914306640625, "margin_dpo/margin_std": 520.629638671875, "step": 505 }, { "KL/chosen_KL_mean": -579.7349853515625, "KL/mean": -793.5402221679688, "KL/rejected_KL_mean": -1007.345458984375, "KL/std": 549.4849853515625, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0009010919602587819, "fcm_dpo/delta": 0.016882240772247314, "fcm_dpo/margin": 427.61041259765625, "fcm_dpo/q_t": 0.4115670919418335, "grad_norm": 39.05805587768555, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.088112235069275, "logits/rejected": -1.1115524768829346, "logps/chosen": -628.695556640625, "logps/ref_chosen": -48.96050262451172, "logps/ref_rejected": -78.41505432128906, "logps/rejected": -1085.760498046875, "loss": 1.1101, "margin_dpo/margin_mean": 427.61041259765625, "margin_dpo/margin_std": 606.4409790039062, "step": 506 }, { "KL/chosen_KL_mean": -689.96875, "KL/mean": -874.892578125, "KL/rejected_KL_mean": -1059.81640625, "KL/std": 653.38037109375, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0009622873039916158, "fcm_dpo/delta": 0.053409378975629807, "fcm_dpo/margin": 369.84759521484375, "fcm_dpo/q_t": 0.42470353841781616, "grad_norm": 33.041019439697266, "learning_rate": 9.427092687124691e-08, "logits/chosen": -1.1047194004058838, "logits/rejected": -1.1137211322784424, "logps/chosen": -756.770263671875, "logps/ref_chosen": -66.80149841308594, "logps/ref_rejected": -95.37289428710938, "logps/rejected": -1155.1893310546875, "loss": 1.1655, "margin_dpo/margin_mean": 369.84759521484375, "margin_dpo/margin_std": 710.9828491210938, "step": 507 }, { "KL/chosen_KL_mean": -782.4303588867188, "KL/mean": -958.6895751953125, "KL/rejected_KL_mean": -1134.94873046875, "KL/std": 643.3538818359375, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.001016983762383461, "fcm_dpo/delta": 0.04988085478544235, "fcm_dpo/margin": 352.5183410644531, "fcm_dpo/q_t": 0.4246112108230591, "grad_norm": 35.355621337890625, "learning_rate": 9.326904852647344e-08, "logits/chosen": -1.1216602325439453, "logits/rejected": -1.1300599575042725, "logps/chosen": -853.7338256835938, "logps/ref_chosen": -71.303466796875, "logps/ref_rejected": -95.6275405883789, "logps/rejected": -1230.576171875, "loss": 1.1984, "margin_dpo/margin_mean": 352.51837158203125, "margin_dpo/margin_std": 768.9783935546875, "step": 508 }, { "KL/chosen_KL_mean": -592.2100830078125, "KL/mean": -759.6229248046875, "KL/rejected_KL_mean": -927.03564453125, "KL/std": 474.44573974609375, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0010790289379656315, "fcm_dpo/delta": 0.04428262263536453, "fcm_dpo/margin": 334.8254699707031, "fcm_dpo/q_t": 0.4207102656364441, "grad_norm": 41.111671447753906, "learning_rate": 9.227130018803195e-08, "logits/chosen": -0.9959691166877747, "logits/rejected": -0.9935662150382996, "logps/chosen": -656.029052734375, "logps/ref_chosen": -63.81895065307617, "logps/ref_rejected": -83.25643920898438, "logps/rejected": -1010.2921142578125, "loss": 1.1492, "margin_dpo/margin_mean": 334.8254699707031, "margin_dpo/margin_std": 584.1224975585938, "step": 509 }, { "KL/chosen_KL_mean": -746.1397705078125, "KL/mean": -961.1923828125, "KL/rejected_KL_mean": -1176.244873046875, "KL/std": 542.093017578125, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0010361545719206333, "fcm_dpo/delta": -0.05862744525074959, "fcm_dpo/margin": 430.10516357421875, "fcm_dpo/q_t": 0.394826740026474, "grad_norm": 30.757688522338867, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.0323694944381714, "logits/rejected": -1.0630911588668823, "logps/chosen": -798.0181884765625, "logps/ref_chosen": -51.878448486328125, "logps/ref_rejected": -102.7651596069336, "logps/rejected": -1279.0101318359375, "loss": 1.0551, "margin_dpo/margin_mean": 430.10516357421875, "margin_dpo/margin_std": 506.75775146484375, "step": 510 }, { "KL/chosen_KL_mean": -670.8756103515625, "KL/mean": -856.5685424804688, "KL/rejected_KL_mean": -1042.261474609375, "KL/std": 570.9805908203125, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.001025435863994062, "fcm_dpo/delta": 0.02368216961622238, "fcm_dpo/margin": 371.38592529296875, "fcm_dpo/q_t": 0.415951669216156, "grad_norm": 40.323726654052734, "learning_rate": 9.028829858700973e-08, "logits/chosen": -1.109389305114746, "logits/rejected": -1.1236944198608398, "logps/chosen": -731.1136474609375, "logps/ref_chosen": -60.23811721801758, "logps/ref_rejected": -92.85676574707031, "logps/rejected": -1135.1181640625, "loss": 1.155, "margin_dpo/margin_mean": 371.38592529296875, "margin_dpo/margin_std": 696.55078125, "step": 511 }, { "KL/chosen_KL_mean": -533.606201171875, "KL/mean": -791.3221435546875, "KL/rejected_KL_mean": -1049.0382080078125, "KL/std": 548.933349609375, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.000933335511945188, "fcm_dpo/delta": -0.11121343821287155, "fcm_dpo/margin": 515.4320068359375, "fcm_dpo/q_t": 0.385158896446228, "grad_norm": 59.88717269897461, "learning_rate": 8.930309757836516e-08, "logits/chosen": -0.9930525422096252, "logits/rejected": -1.025518536567688, "logps/chosen": -588.51171875, "logps/ref_chosen": -54.905494689941406, "logps/ref_rejected": -81.87586975097656, "logps/rejected": -1130.9140625, "loss": 1.0262, "margin_dpo/margin_mean": 515.4320068359375, "margin_dpo/margin_std": 572.5833740234375, "step": 512 }, { "KL/chosen_KL_mean": -714.98828125, "KL/mean": -890.2576904296875, "KL/rejected_KL_mean": -1065.5272216796875, "KL/std": 549.4346923828125, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0008974907686933875, "fcm_dpo/delta": -0.028862863779067993, "fcm_dpo/margin": 350.5388488769531, "fcm_dpo/q_t": 0.4272831082344055, "grad_norm": 52.60966491699219, "learning_rate": 8.832213108254863e-08, "logits/chosen": -1.1262786388397217, "logits/rejected": -1.1136434078216553, "logps/chosen": -779.90478515625, "logps/ref_chosen": -64.91644287109375, "logps/ref_rejected": -76.06245422363281, "logps/rejected": -1141.589599609375, "loss": 1.1783, "margin_dpo/margin_mean": 350.5388488769531, "margin_dpo/margin_std": 643.74267578125, "step": 513 }, { "KL/chosen_KL_mean": -711.4935302734375, "KL/mean": -886.0343017578125, "KL/rejected_KL_mean": -1060.5751953125, "KL/std": 575.5819091796875, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.0009602649370208383, "fcm_dpo/delta": 0.07626126706600189, "fcm_dpo/margin": 349.08160400390625, "fcm_dpo/q_t": 0.4296727478504181, "grad_norm": 45.58797073364258, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.0165659189224243, "logits/rejected": -1.0113214254379272, "logps/chosen": -785.72314453125, "logps/ref_chosen": -74.22957611083984, "logps/ref_rejected": -78.945556640625, "logps/rejected": -1139.520751953125, "loss": 1.1706, "margin_dpo/margin_mean": 349.08160400390625, "margin_dpo/margin_std": 668.0552978515625, "step": 514 }, { "KL/chosen_KL_mean": -587.929443359375, "KL/mean": -739.9951171875, "KL/rejected_KL_mean": -892.060791015625, "KL/std": 471.86102294921875, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0010444549843668938, "fcm_dpo/delta": 0.09740975499153137, "fcm_dpo/margin": 304.1312255859375, "fcm_dpo/q_t": 0.42857980728149414, "grad_norm": 32.755496978759766, "learning_rate": 8.637300491465272e-08, "logits/chosen": -0.9877028465270996, "logits/rejected": -1.01090669631958, "logps/chosen": -638.3310546875, "logps/ref_chosen": -50.40156555175781, "logps/ref_rejected": -87.09774780273438, "logps/rejected": -979.1585083007812, "loss": 1.1743, "margin_dpo/margin_mean": 304.1312561035156, "margin_dpo/margin_std": 567.49853515625, "step": 515 }, { "KL/chosen_KL_mean": -636.4197998046875, "KL/mean": -818.3901977539062, "KL/rejected_KL_mean": -1000.3605346679688, "KL/std": 521.9057006835938, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.001094970153644681, "fcm_dpo/delta": -0.001060720533132553, "fcm_dpo/margin": 363.9407958984375, "fcm_dpo/q_t": 0.40709108114242554, "grad_norm": 48.295448303222656, "learning_rate": 8.540489660386064e-08, "logits/chosen": -1.0736280679702759, "logits/rejected": -1.1098659038543701, "logps/chosen": -701.0693359375, "logps/ref_chosen": -64.64956665039062, "logps/ref_rejected": -111.72237396240234, "logps/rejected": -1112.0830078125, "loss": 1.1144, "margin_dpo/margin_mean": 363.9407958984375, "margin_dpo/margin_std": 547.8302001953125, "step": 516 }, { "KL/chosen_KL_mean": -603.1934814453125, "KL/mean": -841.0816650390625, "KL/rejected_KL_mean": -1078.9698486328125, "KL/std": 555.549560546875, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0009808378526940942, "fcm_dpo/delta": -0.0903298482298851, "fcm_dpo/margin": 475.7763671875, "fcm_dpo/q_t": 0.39226794242858887, "grad_norm": 38.17343521118164, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.0043656826019287, "logits/rejected": -1.0225131511688232, "logps/chosen": -664.1070556640625, "logps/ref_chosen": -60.913551330566406, "logps/ref_rejected": -89.08308410644531, "logps/rejected": -1168.052978515625, "loss": 1.0563, "margin_dpo/margin_mean": 475.7763671875, "margin_dpo/margin_std": 629.1284790039062, "step": 517 }, { "KL/chosen_KL_mean": -587.3173828125, "KL/mean": -774.2802734375, "KL/rejected_KL_mean": -961.2432250976562, "KL/std": 448.082275390625, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.000986230792477727, "fcm_dpo/delta": 0.03777404874563217, "fcm_dpo/margin": 373.92578125, "fcm_dpo/q_t": 0.4149107336997986, "grad_norm": 56.80478286743164, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.0099992752075195, "logits/rejected": -1.0327041149139404, "logps/chosen": -644.7733154296875, "logps/ref_chosen": -57.45589065551758, "logps/ref_rejected": -85.31269836425781, "logps/rejected": -1046.555908203125, "loss": 1.111, "margin_dpo/margin_mean": 373.92578125, "margin_dpo/margin_std": 499.00146484375, "step": 518 }, { "KL/chosen_KL_mean": -576.0409545898438, "KL/mean": -715.2539672851562, "KL/rejected_KL_mean": -854.4669189453125, "KL/std": 430.7130432128906, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0011134599335491657, "fcm_dpo/delta": 0.10244397819042206, "fcm_dpo/margin": 278.426025390625, "fcm_dpo/q_t": 0.4303070902824402, "grad_norm": 32.800045013427734, "learning_rate": 8.25266965458755e-08, "logits/chosen": -0.9758340716362, "logits/rejected": -0.9646449089050293, "logps/chosen": -650.104248046875, "logps/ref_chosen": -74.06331634521484, "logps/ref_rejected": -104.44416809082031, "logps/rejected": -958.9111328125, "loss": 1.1866, "margin_dpo/margin_mean": 278.426025390625, "margin_dpo/margin_std": 533.133544921875, "step": 519 }, { "KL/chosen_KL_mean": -643.1387939453125, "KL/mean": -824.5906372070312, "KL/rejected_KL_mean": -1006.0426025390625, "KL/std": 498.93865966796875, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0011077395174652338, "fcm_dpo/delta": -0.002978809177875519, "fcm_dpo/margin": 362.90374755859375, "fcm_dpo/q_t": 0.4119797348976135, "grad_norm": 35.526668548583984, "learning_rate": 8.15760890883607e-08, "logits/chosen": -0.9478990435600281, "logits/rejected": -0.9614932537078857, "logps/chosen": -713.4385986328125, "logps/ref_chosen": -70.2998275756836, "logps/ref_rejected": -99.98133850097656, "logps/rejected": -1106.02392578125, "loss": 1.1183, "margin_dpo/margin_mean": 362.90374755859375, "margin_dpo/margin_std": 579.5814208984375, "step": 520 }, { "KL/chosen_KL_mean": -554.781005859375, "KL/mean": -760.6822509765625, "KL/rejected_KL_mean": -966.5836181640625, "KL/std": 510.9144287109375, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.001102518173865974, "fcm_dpo/delta": -0.07349302619695663, "fcm_dpo/margin": 411.80267333984375, "fcm_dpo/q_t": 0.394964337348938, "grad_norm": 34.72807312011719, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.0138747692108154, "logits/rejected": -1.0302083492279053, "logps/chosen": -612.9238891601562, "logps/ref_chosen": -58.14292526245117, "logps/ref_rejected": -83.28060913085938, "logps/rejected": -1049.8642578125, "loss": 1.0735, "margin_dpo/margin_mean": 411.80267333984375, "margin_dpo/margin_std": 541.2650146484375, "step": 521 }, { "KL/chosen_KL_mean": -658.7055053710938, "KL/mean": -834.223388671875, "KL/rejected_KL_mean": -1009.7413330078125, "KL/std": 540.191162109375, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0010687486501410604, "fcm_dpo/delta": 0.030154742300510406, "fcm_dpo/margin": 351.0357666015625, "fcm_dpo/q_t": 0.4162691831588745, "grad_norm": 32.86531448364258, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.0368986129760742, "logits/rejected": -1.046702265739441, "logps/chosen": -705.253173828125, "logps/ref_chosen": -46.54766845703125, "logps/ref_rejected": -66.01388549804688, "logps/rejected": -1075.755126953125, "loss": 1.1409, "margin_dpo/margin_mean": 351.0357666015625, "margin_dpo/margin_std": 594.4010009765625, "step": 522 }, { "KL/chosen_KL_mean": -706.0521240234375, "KL/mean": -906.0491333007812, "KL/rejected_KL_mean": -1106.046142578125, "KL/std": 648.95263671875, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0010503309313207865, "fcm_dpo/delta": -0.025494202971458435, "fcm_dpo/margin": 399.9941101074219, "fcm_dpo/q_t": 0.4063454568386078, "grad_norm": 37.535797119140625, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.0674784183502197, "logits/rejected": -1.0807902812957764, "logps/chosen": -767.8216552734375, "logps/ref_chosen": -61.76960372924805, "logps/ref_rejected": -83.76141357421875, "logps/rejected": -1189.8076171875, "loss": 1.1229, "margin_dpo/margin_mean": 399.9941101074219, "margin_dpo/margin_std": 670.1151733398438, "step": 523 }, { "KL/chosen_KL_mean": -687.168212890625, "KL/mean": -907.5570068359375, "KL/rejected_KL_mean": -1127.94580078125, "KL/std": 601.67529296875, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0009792076889425516, "fcm_dpo/delta": -0.04597056284546852, "fcm_dpo/margin": 440.77752685546875, "fcm_dpo/q_t": 0.3970939517021179, "grad_norm": 35.97694396972656, "learning_rate": 7.781828926091535e-08, "logits/chosen": -1.1234776973724365, "logits/rejected": -1.125186562538147, "logps/chosen": -765.240234375, "logps/ref_chosen": -78.0720443725586, "logps/ref_rejected": -81.30198669433594, "logps/rejected": -1209.247802734375, "loss": 1.0894, "margin_dpo/margin_mean": 440.77752685546875, "margin_dpo/margin_std": 615.4475708007812, "step": 524 }, { "KL/chosen_KL_mean": -658.93115234375, "KL/mean": -915.503173828125, "KL/rejected_KL_mean": -1172.0750732421875, "KL/std": 587.5389404296875, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0009053426329046488, "fcm_dpo/delta": -0.09118733555078506, "fcm_dpo/margin": 513.14404296875, "fcm_dpo/q_t": 0.3911314010620117, "grad_norm": 35.581817626953125, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.0382120609283447, "logits/rejected": -1.0887622833251953, "logps/chosen": -709.7589721679688, "logps/ref_chosen": -50.827857971191406, "logps/ref_rejected": -100.05294036865234, "logps/rejected": -1272.1280517578125, "loss": 1.0522, "margin_dpo/margin_mean": 513.1439819335938, "margin_dpo/margin_std": 641.33203125, "step": 525 }, { "KL/chosen_KL_mean": -707.2711791992188, "KL/mean": -966.6031494140625, "KL/rejected_KL_mean": -1225.93505859375, "KL/std": 583.5233154296875, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0008811709121800959, "fcm_dpo/delta": -0.07432842254638672, "fcm_dpo/margin": 518.663818359375, "fcm_dpo/q_t": 0.3920350670814514, "grad_norm": 27.579317092895508, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.0754790306091309, "logits/rejected": -1.084367036819458, "logps/chosen": -770.4384155273438, "logps/ref_chosen": -63.167236328125, "logps/ref_rejected": -86.30934143066406, "logps/rejected": -1312.244384765625, "loss": 1.0666, "margin_dpo/margin_mean": 518.663818359375, "margin_dpo/margin_std": 673.5228271484375, "step": 526 }, { "KL/chosen_KL_mean": -777.7960815429688, "KL/mean": -947.6534423828125, "KL/rejected_KL_mean": -1117.5108642578125, "KL/std": 600.279296875, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.000911904382519424, "fcm_dpo/delta": 0.10690906643867493, "fcm_dpo/margin": 339.7147521972656, "fcm_dpo/q_t": 0.4307979345321655, "grad_norm": 33.39142990112305, "learning_rate": 7.504749238082414e-08, "logits/chosen": -1.33701753616333, "logits/rejected": -1.3142709732055664, "logps/chosen": -848.9248046875, "logps/ref_chosen": -71.12867736816406, "logps/ref_rejected": -78.3425521850586, "logps/rejected": -1195.853515625, "loss": 1.154, "margin_dpo/margin_mean": 339.71478271484375, "margin_dpo/margin_std": 517.4578857421875, "step": 527 }, { "KL/chosen_KL_mean": -758.856689453125, "KL/mean": -977.1895751953125, "KL/rejected_KL_mean": -1195.5224609375, "KL/std": 594.4635620117188, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0009343413985334337, "fcm_dpo/delta": -0.010038571432232857, "fcm_dpo/margin": 436.66583251953125, "fcm_dpo/q_t": 0.4094654321670532, "grad_norm": 30.87571144104004, "learning_rate": 7.413308141366254e-08, "logits/chosen": -1.206683874130249, "logits/rejected": -1.1996240615844727, "logps/chosen": -826.9461059570312, "logps/ref_chosen": -68.0894546508789, "logps/ref_rejected": -93.91006469726562, "logps/rejected": -1289.4324951171875, "loss": 1.1199, "margin_dpo/margin_mean": 436.66583251953125, "margin_dpo/margin_std": 710.944580078125, "step": 528 }, { "KL/chosen_KL_mean": -954.2504272460938, "KL/mean": -1106.777099609375, "KL/rejected_KL_mean": -1259.303955078125, "KL/std": 612.4547119140625, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0009363851859234273, "fcm_dpo/delta": 0.0063527897000312805, "fcm_dpo/margin": 305.05340576171875, "fcm_dpo/q_t": 0.43478527665138245, "grad_norm": 38.029884338378906, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.2603811025619507, "logits/rejected": -1.278627872467041, "logps/chosen": -1009.825439453125, "logps/ref_chosen": -55.57495880126953, "logps/ref_rejected": -89.20909118652344, "logps/rejected": -1348.512939453125, "loss": 1.2423, "margin_dpo/margin_mean": 305.0534362792969, "margin_dpo/margin_std": 756.2225341796875, "step": 529 }, { "KL/chosen_KL_mean": -787.18115234375, "KL/mean": -1030.659912109375, "KL/rejected_KL_mean": -1274.138671875, "KL/std": 670.89990234375, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.00090701476437971, "fcm_dpo/delta": -0.053333625197410583, "fcm_dpo/margin": 486.95751953125, "fcm_dpo/q_t": 0.40362924337387085, "grad_norm": 35.42485427856445, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.1458381414413452, "logits/rejected": -1.1499226093292236, "logps/chosen": -834.7825927734375, "logps/ref_chosen": -47.601417541503906, "logps/ref_rejected": -87.2845230102539, "logps/rejected": -1361.4232177734375, "loss": 1.1316, "margin_dpo/margin_mean": 486.95745849609375, "margin_dpo/margin_std": 884.4109497070312, "step": 530 }, { "KL/chosen_KL_mean": -894.1881713867188, "KL/mean": -1081.9078369140625, "KL/rejected_KL_mean": -1269.6273193359375, "KL/std": 714.610595703125, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.0009350181790068746, "fcm_dpo/delta": 0.05843368172645569, "fcm_dpo/margin": 375.4392395019531, "fcm_dpo/q_t": 0.42438310384750366, "grad_norm": 46.751792907714844, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.278576374053955, "logits/rejected": -1.2574585676193237, "logps/chosen": -949.4342041015625, "logps/ref_chosen": -55.246063232421875, "logps/ref_rejected": -70.60598754882812, "logps/rejected": -1340.2333984375, "loss": 1.1589, "margin_dpo/margin_mean": 375.439208984375, "margin_dpo/margin_std": 681.22314453125, "step": 531 }, { "KL/chosen_KL_mean": -914.6450805664062, "KL/mean": -1124.872314453125, "KL/rejected_KL_mean": -1335.0997314453125, "KL/std": 674.3599853515625, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0009111958206631243, "fcm_dpo/delta": 0.013179205358028412, "fcm_dpo/margin": 420.45458984375, "fcm_dpo/q_t": 0.4144301414489746, "grad_norm": 74.0493392944336, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.2184380292892456, "logits/rejected": -1.1921205520629883, "logps/chosen": -984.93115234375, "logps/ref_chosen": -70.28601837158203, "logps/ref_rejected": -86.5913314819336, "logps/rejected": -1421.6910400390625, "loss": 1.184, "margin_dpo/margin_mean": 420.4546203613281, "margin_dpo/margin_std": 841.721435546875, "step": 532 }, { "KL/chosen_KL_mean": -811.30615234375, "KL/mean": -966.7421875, "KL/rejected_KL_mean": -1122.1781005859375, "KL/std": 577.7342529296875, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0009565654909238219, "fcm_dpo/delta": 0.0060767000541090965, "fcm_dpo/margin": 310.87200927734375, "fcm_dpo/q_t": 0.4336027204990387, "grad_norm": 33.39649200439453, "learning_rate": 6.963101805503646e-08, "logits/chosen": -1.299651861190796, "logits/rejected": -1.2833778858184814, "logps/chosen": -876.1612548828125, "logps/ref_chosen": -64.8551025390625, "logps/ref_rejected": -76.58805847167969, "logps/rejected": -1198.76611328125, "loss": 1.2121, "margin_dpo/margin_mean": 310.87200927734375, "margin_dpo/margin_std": 690.0610961914062, "step": 533 }, { "KL/chosen_KL_mean": -830.42724609375, "KL/mean": -1037.4508056640625, "KL/rejected_KL_mean": -1244.474365234375, "KL/std": 619.809814453125, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0009335308568552136, "fcm_dpo/delta": 0.01120779663324356, "fcm_dpo/margin": 414.0472412109375, "fcm_dpo/q_t": 0.41367307305336, "grad_norm": 32.04544448852539, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.2206509113311768, "logits/rejected": -1.2227022647857666, "logps/chosen": -890.546630859375, "logps/ref_chosen": -60.119388580322266, "logps/ref_rejected": -78.54347229003906, "logps/rejected": -1323.017822265625, "loss": 1.1307, "margin_dpo/margin_mean": 414.0472412109375, "margin_dpo/margin_std": 661.77587890625, "step": 534 }, { "KL/chosen_KL_mean": -699.33740234375, "KL/mean": -953.7452392578125, "KL/rejected_KL_mean": -1208.1529541015625, "KL/std": 633.1478271484375, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0009318743832409382, "fcm_dpo/delta": -0.09808388352394104, "fcm_dpo/margin": 508.8156433105469, "fcm_dpo/q_t": 0.390717089176178, "grad_norm": 24.964298248291016, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.1217422485351562, "logits/rejected": -1.1360340118408203, "logps/chosen": -753.6676635742188, "logps/ref_chosen": -54.330238342285156, "logps/ref_rejected": -96.30763244628906, "logps/rejected": -1304.460693359375, "loss": 1.0588, "margin_dpo/margin_mean": 508.81561279296875, "margin_dpo/margin_std": 653.384521484375, "step": 535 }, { "KL/chosen_KL_mean": -622.41259765625, "KL/mean": -909.5772705078125, "KL/rejected_KL_mean": -1196.741943359375, "KL/std": 661.69384765625, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0008232737891376019, "fcm_dpo/delta": -0.09523215144872665, "fcm_dpo/margin": 574.329345703125, "fcm_dpo/q_t": 0.3940427899360657, "grad_norm": 29.166486740112305, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.0844101905822754, "logits/rejected": -1.1293668746948242, "logps/chosen": -669.4931030273438, "logps/ref_chosen": -47.08053207397461, "logps/ref_rejected": -89.09783935546875, "logps/rejected": -1285.83984375, "loss": 1.0587, "margin_dpo/margin_mean": 574.329345703125, "margin_dpo/margin_std": 803.4403076171875, "step": 536 }, { "KL/chosen_KL_mean": -635.1380615234375, "KL/mean": -826.725830078125, "KL/rejected_KL_mean": -1018.3135375976562, "KL/std": 565.7158203125, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0007962306845001876, "fcm_dpo/delta": -0.008790962398052216, "fcm_dpo/margin": 383.1754150390625, "fcm_dpo/q_t": 0.42851394414901733, "grad_norm": 35.423980712890625, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.1369528770446777, "logits/rejected": -1.1194478273391724, "logps/chosen": -692.8856201171875, "logps/ref_chosen": -57.747467041015625, "logps/ref_rejected": -70.43838500976562, "logps/rejected": -1088.751953125, "loss": 1.1758, "margin_dpo/margin_mean": 383.1754150390625, "margin_dpo/margin_std": 672.041015625, "step": 537 }, { "KL/chosen_KL_mean": -784.5943603515625, "KL/mean": -1004.52880859375, "KL/rejected_KL_mean": -1224.46337890625, "KL/std": 607.7210693359375, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0008164436439983547, "fcm_dpo/delta": 0.049948450177907944, "fcm_dpo/margin": 439.8689270019531, "fcm_dpo/q_t": 0.4188556671142578, "grad_norm": 27.3602237701416, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.1543678045272827, "logits/rejected": -1.1441106796264648, "logps/chosen": -851.0103149414062, "logps/ref_chosen": -66.41594696044922, "logps/ref_rejected": -84.22808837890625, "logps/rejected": -1308.69140625, "loss": 1.1229, "margin_dpo/margin_mean": 439.86895751953125, "margin_dpo/margin_std": 635.7586669921875, "step": 538 }, { "KL/chosen_KL_mean": -652.3861083984375, "KL/mean": -859.1839599609375, "KL/rejected_KL_mean": -1065.981689453125, "KL/std": 471.9359130859375, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.0008725939551368356, "fcm_dpo/delta": 0.04645565152168274, "fcm_dpo/margin": 413.5956115722656, "fcm_dpo/q_t": 0.417635977268219, "grad_norm": 29.32832145690918, "learning_rate": 6.438563463416221e-08, "logits/chosen": -1.157194972038269, "logits/rejected": -1.1582703590393066, "logps/chosen": -710.8789672851562, "logps/ref_chosen": -58.492855072021484, "logps/ref_rejected": -91.85395050048828, "logps/rejected": -1157.835693359375, "loss": 1.1091, "margin_dpo/margin_mean": 413.5955810546875, "margin_dpo/margin_std": 535.5076904296875, "step": 539 }, { "KL/chosen_KL_mean": -629.462890625, "KL/mean": -898.05859375, "KL/rejected_KL_mean": -1166.654296875, "KL/std": 595.0738525390625, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0008491129847243428, "fcm_dpo/delta": -0.07360121607780457, "fcm_dpo/margin": 537.1914672851562, "fcm_dpo/q_t": 0.396462619304657, "grad_norm": 36.619747161865234, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.031808614730835, "logits/rejected": -1.0707483291625977, "logps/chosen": -692.9454345703125, "logps/ref_chosen": -63.482513427734375, "logps/ref_rejected": -116.42999267578125, "logps/rejected": -1283.0843505859375, "loss": 1.0689, "margin_dpo/margin_mean": 537.1914672851562, "margin_dpo/margin_std": 714.2697143554688, "step": 540 }, { "KL/chosen_KL_mean": -754.865966796875, "KL/mean": -900.2635498046875, "KL/rejected_KL_mean": -1045.6612548828125, "KL/std": 539.58740234375, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.000835648737847805, "fcm_dpo/delta": 0.02992171235382557, "fcm_dpo/margin": 290.7952880859375, "fcm_dpo/q_t": 0.44497525691986084, "grad_norm": 49.2709846496582, "learning_rate": 6.267605843546767e-08, "logits/chosen": -1.191582202911377, "logits/rejected": -1.1900990009307861, "logps/chosen": -833.1463012695312, "logps/ref_chosen": -78.28036499023438, "logps/ref_rejected": -103.273681640625, "logps/rejected": -1148.934814453125, "loss": 1.2274, "margin_dpo/margin_mean": 290.79522705078125, "margin_dpo/margin_std": 635.66015625, "step": 541 }, { "KL/chosen_KL_mean": -641.0167236328125, "KL/mean": -899.6398315429688, "KL/rejected_KL_mean": -1158.262939453125, "KL/std": 611.9912109375, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0007914283778518438, "fcm_dpo/delta": -0.019995450973510742, "fcm_dpo/margin": 517.24609375, "fcm_dpo/q_t": 0.40515321493148804, "grad_norm": 45.66024398803711, "learning_rate": 6.182866334636888e-08, "logits/chosen": -1.1752982139587402, "logits/rejected": -1.219357967376709, "logps/chosen": -698.501708984375, "logps/ref_chosen": -57.48497009277344, "logps/ref_rejected": -96.47506713867188, "logps/rejected": -1254.738037109375, "loss": 1.0996, "margin_dpo/margin_mean": 517.24609375, "margin_dpo/margin_std": 717.5678100585938, "step": 542 }, { "KL/chosen_KL_mean": -710.8384399414062, "KL/mean": -893.802001953125, "KL/rejected_KL_mean": -1076.7655029296875, "KL/std": 660.750244140625, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0008738016476854682, "fcm_dpo/delta": 0.09592510759830475, "fcm_dpo/margin": 365.9271240234375, "fcm_dpo/q_t": 0.4370731711387634, "grad_norm": 29.643529891967773, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.101868748664856, "logits/rejected": -1.1417419910430908, "logps/chosen": -771.4559326171875, "logps/ref_chosen": -60.61750793457031, "logps/ref_rejected": -105.59896850585938, "logps/rejected": -1182.364501953125, "loss": 1.2076, "margin_dpo/margin_mean": 365.9271545410156, "margin_dpo/margin_std": 836.8877563476562, "step": 543 }, { "KL/chosen_KL_mean": -727.2119140625, "KL/mean": -949.6517333984375, "KL/rejected_KL_mean": -1172.091552734375, "KL/std": 569.668701171875, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0008942891145125031, "fcm_dpo/delta": 0.0022725816816091537, "fcm_dpo/margin": 444.879638671875, "fcm_dpo/q_t": 0.4095289707183838, "grad_norm": 34.229854583740234, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.1131432056427002, "logits/rejected": -1.14817476272583, "logps/chosen": -786.8541870117188, "logps/ref_chosen": -59.642303466796875, "logps/ref_rejected": -100.95469665527344, "logps/rejected": -1273.046142578125, "loss": 1.1092, "margin_dpo/margin_mean": 444.879638671875, "margin_dpo/margin_std": 663.5440673828125, "step": 544 }, { "KL/chosen_KL_mean": -659.4088745117188, "KL/mean": -919.2174072265625, "KL/rejected_KL_mean": -1179.02587890625, "KL/std": 598.9300537109375, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.000862646906170994, "fcm_dpo/delta": -0.06230539083480835, "fcm_dpo/margin": 519.6170043945312, "fcm_dpo/q_t": 0.3970434069633484, "grad_norm": 32.03169631958008, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -0.9890373349189758, "logits/rejected": -1.033557653427124, "logps/chosen": -727.0574951171875, "logps/ref_chosen": -67.64859771728516, "logps/ref_rejected": -95.90800476074219, "logps/rejected": -1274.933837890625, "loss": 1.0686, "margin_dpo/margin_mean": 519.6170654296875, "margin_dpo/margin_std": 691.0440673828125, "step": 545 }, { "KL/chosen_KL_mean": -640.8040771484375, "KL/mean": -804.4512939453125, "KL/rejected_KL_mean": -968.0985717773438, "KL/std": 497.0710144042969, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0009246976114809513, "fcm_dpo/delta": 0.11454042792320251, "fcm_dpo/margin": 327.29449462890625, "fcm_dpo/q_t": 0.4316098690032959, "grad_norm": 30.14081573486328, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.0944793224334717, "logits/rejected": -1.0822169780731201, "logps/chosen": -691.5482788085938, "logps/ref_chosen": -50.744232177734375, "logps/ref_rejected": -81.86622619628906, "logps/rejected": -1049.96484375, "loss": 1.1688, "margin_dpo/margin_mean": 327.2945251464844, "margin_dpo/margin_std": 540.7611694335938, "step": 546 }, { "KL/chosen_KL_mean": -660.4383544921875, "KL/mean": -884.519287109375, "KL/rejected_KL_mean": -1108.60009765625, "KL/std": 564.5079345703125, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0009412175277248025, "fcm_dpo/delta": -0.02773449756205082, "fcm_dpo/margin": 448.1617431640625, "fcm_dpo/q_t": 0.4049096703529358, "grad_norm": 41.60638427734375, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.058458685874939, "logits/rejected": -1.0802452564239502, "logps/chosen": -734.1260986328125, "logps/ref_chosen": -73.6877212524414, "logps/ref_rejected": -90.76136779785156, "logps/rejected": -1199.3614501953125, "loss": 1.0946, "margin_dpo/margin_mean": 448.1617431640625, "margin_dpo/margin_std": 651.8272094726562, "step": 547 }, { "KL/chosen_KL_mean": -706.6986694335938, "KL/mean": -914.0684814453125, "KL/rejected_KL_mean": -1121.4383544921875, "KL/std": 620.1270141601562, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0009347201557829976, "fcm_dpo/delta": 0.015313264913856983, "fcm_dpo/margin": 414.7396240234375, "fcm_dpo/q_t": 0.41696617007255554, "grad_norm": 34.767852783203125, "learning_rate": 5.684919345471029e-08, "logits/chosen": -1.1285653114318848, "logits/rejected": -1.1398406028747559, "logps/chosen": -771.9450073242188, "logps/ref_chosen": -65.24634552001953, "logps/ref_rejected": -94.11807250976562, "logps/rejected": -1215.556396484375, "loss": 1.1259, "margin_dpo/margin_mean": 414.7396240234375, "margin_dpo/margin_std": 696.6866455078125, "step": 548 }, { "KL/chosen_KL_mean": -716.9879760742188, "KL/mean": -874.5885009765625, "KL/rejected_KL_mean": -1032.1890869140625, "KL/std": 489.6221008300781, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0009415035601705313, "fcm_dpo/delta": 0.00250604678876698, "fcm_dpo/margin": 315.2010803222656, "fcm_dpo/q_t": 0.4317595362663269, "grad_norm": 44.2504768371582, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.176891803741455, "logits/rejected": -1.1708965301513672, "logps/chosen": -766.2003173828125, "logps/ref_chosen": -49.21235656738281, "logps/ref_rejected": -73.91031646728516, "logps/rejected": -1106.099365234375, "loss": 1.188, "margin_dpo/margin_mean": 315.2010803222656, "margin_dpo/margin_std": 595.5227661132812, "step": 549 }, { "KL/chosen_KL_mean": -721.7799682617188, "KL/mean": -916.480224609375, "KL/rejected_KL_mean": -1111.1806640625, "KL/std": 571.6375732421875, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.000960228149779141, "fcm_dpo/delta": 0.03213486075401306, "fcm_dpo/margin": 389.4005126953125, "fcm_dpo/q_t": 0.4163832664489746, "grad_norm": 41.49302673339844, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.0517723560333252, "logits/rejected": -1.0827746391296387, "logps/chosen": -778.5869140625, "logps/ref_chosen": -56.80695343017578, "logps/ref_rejected": -95.12580871582031, "logps/rejected": -1206.306396484375, "loss": 1.1212, "margin_dpo/margin_mean": 389.4005126953125, "margin_dpo/margin_std": 595.0445556640625, "step": 550 }, { "KL/chosen_KL_mean": -613.4744262695312, "KL/mean": -925.0634765625, "KL/rejected_KL_mean": -1236.652587890625, "KL/std": 624.64697265625, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.0008410347509197891, "fcm_dpo/delta": -0.17439556121826172, "fcm_dpo/margin": 623.1781005859375, "fcm_dpo/q_t": 0.3711840510368347, "grad_norm": 51.76773452758789, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.0433658361434937, "logits/rejected": -1.1170668601989746, "logps/chosen": -672.5807495117188, "logps/ref_chosen": -59.10633087158203, "logps/ref_rejected": -111.67280578613281, "logps/rejected": -1348.3253173828125, "loss": 0.9923, "margin_dpo/margin_mean": 623.1781005859375, "margin_dpo/margin_std": 630.8492431640625, "step": 551 }, { "KL/chosen_KL_mean": -602.475341796875, "KL/mean": -958.9727783203125, "KL/rejected_KL_mean": -1315.47021484375, "KL/std": 719.78564453125, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0006975846481509507, "fcm_dpo/delta": -0.14975111186504364, "fcm_dpo/margin": 712.994873046875, "fcm_dpo/q_t": 0.3810081481933594, "grad_norm": 42.0149040222168, "learning_rate": 5.363104864490034e-08, "logits/chosen": -1.1741893291473389, "logits/rejected": -1.2385377883911133, "logps/chosen": -664.8298950195312, "logps/ref_chosen": -62.35459899902344, "logps/ref_rejected": -104.56210327148438, "logps/rejected": -1420.0322265625, "loss": 1.0282, "margin_dpo/margin_mean": 712.994873046875, "margin_dpo/margin_std": 833.3380737304688, "step": 552 }, { "KL/chosen_KL_mean": -699.2514038085938, "KL/mean": -884.60302734375, "KL/rejected_KL_mean": -1069.95458984375, "KL/std": 591.6678466796875, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0007735902327112854, "fcm_dpo/delta": 0.1325617879629135, "fcm_dpo/margin": 370.7031555175781, "fcm_dpo/q_t": 0.43872201442718506, "grad_norm": 23.199739456176758, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.1220920085906982, "logits/rejected": -1.1197218894958496, "logps/chosen": -767.51025390625, "logps/ref_chosen": -68.25881958007812, "logps/ref_rejected": -98.0971450805664, "logps/rejected": -1168.0517578125, "loss": 1.1789, "margin_dpo/margin_mean": 370.70318603515625, "margin_dpo/margin_std": 653.1337890625, "step": 553 }, { "KL/chosen_KL_mean": -751.51806640625, "KL/mean": -983.1180419921875, "KL/rejected_KL_mean": -1214.7178955078125, "KL/std": 659.7369995117188, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0008373720338568091, "fcm_dpo/delta": 0.009692121297121048, "fcm_dpo/margin": 463.19989013671875, "fcm_dpo/q_t": 0.4117772877216339, "grad_norm": 118.59552001953125, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.087932825088501, "logits/rejected": -1.0606003999710083, "logps/chosen": -819.4658203125, "logps/ref_chosen": -67.94767761230469, "logps/ref_rejected": -89.78272247314453, "logps/rejected": -1304.5006103515625, "loss": 1.1619, "margin_dpo/margin_mean": 463.1998596191406, "margin_dpo/margin_std": 863.3409423828125, "step": 554 }, { "KL/chosen_KL_mean": -739.2120971679688, "KL/mean": -1043.806396484375, "KL/rejected_KL_mean": -1348.400634765625, "KL/std": 680.260009765625, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.0007673618383705616, "fcm_dpo/delta": -0.08767017722129822, "fcm_dpo/margin": 609.1884155273438, "fcm_dpo/q_t": 0.3946698307991028, "grad_norm": 30.83856773376465, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.1824922561645508, "logits/rejected": -1.2582507133483887, "logps/chosen": -792.5426025390625, "logps/ref_chosen": -53.33049011230469, "logps/ref_rejected": -108.47937774658203, "logps/rejected": -1456.8798828125, "loss": 1.0729, "margin_dpo/margin_mean": 609.1884765625, "margin_dpo/margin_std": 872.76953125, "step": 555 }, { "KL/chosen_KL_mean": -732.7947998046875, "KL/mean": -903.1065673828125, "KL/rejected_KL_mean": -1073.418212890625, "KL/std": 530.9589233398438, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.0008233854314312339, "fcm_dpo/delta": 0.13905900716781616, "fcm_dpo/margin": 340.62347412109375, "fcm_dpo/q_t": 0.4381856620311737, "grad_norm": 29.910762786865234, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.185271143913269, "logits/rejected": -1.1860347986221313, "logps/chosen": -791.4393310546875, "logps/ref_chosen": -58.64447021484375, "logps/ref_rejected": -101.34040832519531, "logps/rejected": -1174.7587890625, "loss": 1.1731, "margin_dpo/margin_mean": 340.6235046386719, "margin_dpo/margin_std": 539.8701171875, "step": 556 }, { "KL/chosen_KL_mean": -786.4280395507812, "KL/mean": -1055.865234375, "KL/rejected_KL_mean": -1325.302490234375, "KL/std": 711.47607421875, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.0008084324654191732, "fcm_dpo/delta": -0.04845455288887024, "fcm_dpo/margin": 538.87451171875, "fcm_dpo/q_t": 0.4022066295146942, "grad_norm": 46.43569564819336, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.2151281833648682, "logits/rejected": -1.2664202451705933, "logps/chosen": -854.2686767578125, "logps/ref_chosen": -67.84066009521484, "logps/ref_rejected": -109.93965911865234, "logps/rejected": -1435.2421875, "loss": 1.0994, "margin_dpo/margin_mean": 538.8744506835938, "margin_dpo/margin_std": 822.3883056640625, "step": 557 }, { "KL/chosen_KL_mean": -752.0896606445312, "KL/mean": -1117.599853515625, "KL/rejected_KL_mean": -1483.10986328125, "KL/std": 693.58544921875, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0007167826406657696, "fcm_dpo/delta": -0.16944512724876404, "fcm_dpo/margin": 731.0201416015625, "fcm_dpo/q_t": 0.3706546425819397, "grad_norm": 24.203054428100586, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -1.274876594543457, "logits/rejected": -1.3060567378997803, "logps/chosen": -814.4578857421875, "logps/ref_chosen": -62.36824035644531, "logps/ref_rejected": -102.16102600097656, "logps/rejected": -1585.2708740234375, "loss": 1.0044, "margin_dpo/margin_mean": 731.0202026367188, "margin_dpo/margin_std": 783.4100341796875, "step": 558 }, { "KL/chosen_KL_mean": -840.5665893554688, "KL/mean": -1157.062744140625, "KL/rejected_KL_mean": -1473.558837890625, "KL/std": 744.0479736328125, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0006758394883945584, "fcm_dpo/delta": -0.03540560603141785, "fcm_dpo/margin": 632.9921264648438, "fcm_dpo/q_t": 0.401652991771698, "grad_norm": 23.325172424316406, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -1.4109092950820923, "logits/rejected": -1.432593822479248, "logps/chosen": -901.3189697265625, "logps/ref_chosen": -60.752323150634766, "logps/ref_rejected": -93.44229125976562, "logps/rejected": -1567.0009765625, "loss": 1.0774, "margin_dpo/margin_mean": 632.9921875, "margin_dpo/margin_std": 839.2745361328125, "step": 559 }, { "KL/chosen_KL_mean": -801.013916015625, "KL/mean": -1016.2203369140625, "KL/rejected_KL_mean": -1231.4267578125, "KL/std": 670.739990234375, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0006765261059626937, "fcm_dpo/delta": 0.019914967939257622, "fcm_dpo/margin": 430.41290283203125, "fcm_dpo/q_t": 0.43432098627090454, "grad_norm": 40.84651184082031, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.0937294960021973, "logits/rejected": -1.0813250541687012, "logps/chosen": -859.11767578125, "logps/ref_chosen": -58.10382080078125, "logps/ref_rejected": -79.99122619628906, "logps/rejected": -1311.41796875, "loss": 1.1813, "margin_dpo/margin_mean": 430.41290283203125, "margin_dpo/margin_std": 772.7431640625, "step": 560 }, { "KL/chosen_KL_mean": -897.7193603515625, "KL/mean": -1091.67431640625, "KL/rejected_KL_mean": -1285.6295166015625, "KL/std": 642.45654296875, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0007529509020969272, "fcm_dpo/delta": 0.1230059415102005, "fcm_dpo/margin": 387.91015625, "fcm_dpo/q_t": 0.435150682926178, "grad_norm": 30.059293746948242, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.2934693098068237, "logits/rejected": -1.3272886276245117, "logps/chosen": -948.6322021484375, "logps/ref_chosen": -50.912879943847656, "logps/ref_rejected": -99.06856536865234, "logps/rejected": -1384.6981201171875, "loss": 1.1972, "margin_dpo/margin_mean": 387.91015625, "margin_dpo/margin_std": 791.5865478515625, "step": 561 }, { "KL/chosen_KL_mean": -864.7420654296875, "KL/mean": -1108.170654296875, "KL/rejected_KL_mean": -1351.5992431640625, "KL/std": 697.1679077148438, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0007649194449186325, "fcm_dpo/delta": 0.03210524842143059, "fcm_dpo/margin": 486.857177734375, "fcm_dpo/q_t": 0.416918009519577, "grad_norm": 28.984006881713867, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -1.2221224308013916, "logits/rejected": -1.2382316589355469, "logps/chosen": -924.2064819335938, "logps/ref_chosen": -59.46440124511719, "logps/ref_rejected": -96.54266357421875, "logps/rejected": -1448.141845703125, "loss": 1.1243, "margin_dpo/margin_mean": 486.857177734375, "margin_dpo/margin_std": 740.4078369140625, "step": 562 }, { "KL/chosen_KL_mean": -973.072021484375, "KL/mean": -1153.509033203125, "KL/rejected_KL_mean": -1333.9459228515625, "KL/std": 739.7679443359375, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0007791572716087103, "fcm_dpo/delta": -0.02793397195637226, "fcm_dpo/margin": 360.8739013671875, "fcm_dpo/q_t": 0.4309625029563904, "grad_norm": 40.373382568359375, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.2764613628387451, "logits/rejected": -1.2854523658752441, "logps/chosen": -1033.68017578125, "logps/ref_chosen": -60.60819625854492, "logps/ref_rejected": -94.56770324707031, "logps/rejected": -1428.513671875, "loss": 1.2446, "margin_dpo/margin_mean": 360.8739013671875, "margin_dpo/margin_std": 878.1210327148438, "step": 563 }, { "KL/chosen_KL_mean": -857.3223266601562, "KL/mean": -1105.55712890625, "KL/rejected_KL_mean": -1353.7919921875, "KL/std": 703.2030029296875, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0007832963019609451, "fcm_dpo/delta": 0.013243109919130802, "fcm_dpo/margin": 496.4696960449219, "fcm_dpo/q_t": 0.4130350947380066, "grad_norm": 32.131954193115234, "learning_rate": 4.447860229910544e-08, "logits/chosen": -1.2859337329864502, "logits/rejected": -1.2894582748413086, "logps/chosen": -931.5906982421875, "logps/ref_chosen": -74.26837921142578, "logps/ref_rejected": -93.23818969726562, "logps/rejected": -1447.0301513671875, "loss": 1.1066, "margin_dpo/margin_mean": 496.4696960449219, "margin_dpo/margin_std": 708.5772094726562, "step": 564 }, { "KL/chosen_KL_mean": -940.7181396484375, "KL/mean": -1180.5489501953125, "KL/rejected_KL_mean": -1420.3797607421875, "KL/std": 811.2318115234375, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0007922237273305655, "fcm_dpo/delta": 0.024702370166778564, "fcm_dpo/margin": 479.6616516113281, "fcm_dpo/q_t": 0.4200623631477356, "grad_norm": 52.22495651245117, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.1902871131896973, "logits/rejected": -1.1874666213989258, "logps/chosen": -1009.738037109375, "logps/ref_chosen": -69.0199203491211, "logps/ref_rejected": -85.7789306640625, "logps/rejected": -1506.15869140625, "loss": 1.1803, "margin_dpo/margin_mean": 479.6616516113281, "margin_dpo/margin_std": 994.3869018554688, "step": 565 }, { "KL/chosen_KL_mean": -863.586669921875, "KL/mean": -1147.6942138671875, "KL/rejected_KL_mean": -1431.8017578125, "KL/std": 777.2615966796875, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0007855127332732081, "fcm_dpo/delta": -0.061380840837955475, "fcm_dpo/margin": 568.2152099609375, "fcm_dpo/q_t": 0.3984350264072418, "grad_norm": 28.156330108642578, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.2587931156158447, "logits/rejected": -1.2928824424743652, "logps/chosen": -930.1319580078125, "logps/ref_chosen": -66.5453109741211, "logps/ref_rejected": -103.86932373046875, "logps/rejected": -1535.671142578125, "loss": 1.1124, "margin_dpo/margin_mean": 568.2152099609375, "margin_dpo/margin_std": 901.3643188476562, "step": 566 }, { "KL/chosen_KL_mean": -786.615478515625, "KL/mean": -1003.1062622070312, "KL/rejected_KL_mean": -1219.5970458984375, "KL/std": 541.6770629882812, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.000789005309343338, "fcm_dpo/delta": 0.07048732787370682, "fcm_dpo/margin": 432.98162841796875, "fcm_dpo/q_t": 0.4219757318496704, "grad_norm": 25.511581420898438, "learning_rate": 4.231101308059165e-08, "logits/chosen": -1.3670188188552856, "logits/rejected": -1.3844826221466064, "logps/chosen": -839.4737548828125, "logps/ref_chosen": -52.85829544067383, "logps/ref_rejected": -85.37095642089844, "logps/rejected": -1304.968017578125, "loss": 1.1217, "margin_dpo/margin_mean": 432.98162841796875, "margin_dpo/margin_std": 570.1590576171875, "step": 567 }, { "KL/chosen_KL_mean": -747.6436767578125, "KL/mean": -1037.877197265625, "KL/rejected_KL_mean": -1328.11083984375, "KL/std": 627.2274780273438, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0007679754635319114, "fcm_dpo/delta": -0.05931420996785164, "fcm_dpo/margin": 580.4671630859375, "fcm_dpo/q_t": 0.39421334862709045, "grad_norm": 34.71445846557617, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.1856874227523804, "logits/rejected": -1.243179202079773, "logps/chosen": -792.8360595703125, "logps/ref_chosen": -45.1923828125, "logps/ref_rejected": -89.09236907958984, "logps/rejected": -1417.2032470703125, "loss": 1.0511, "margin_dpo/margin_mean": 580.4671630859375, "margin_dpo/margin_std": 675.10009765625, "step": 568 }, { "KL/chosen_KL_mean": -887.8179321289062, "KL/mean": -1117.02783203125, "KL/rejected_KL_mean": -1346.2376708984375, "KL/std": 795.90283203125, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0008052626508288085, "fcm_dpo/delta": 0.030735157430171967, "fcm_dpo/margin": 458.41961669921875, "fcm_dpo/q_t": 0.41605180501937866, "grad_norm": 35.24816131591797, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.2379553318023682, "logits/rejected": -1.2469182014465332, "logps/chosen": -951.5385131835938, "logps/ref_chosen": -63.72056198120117, "logps/ref_rejected": -79.10325622558594, "logps/rejected": -1425.3408203125, "loss": 1.1615, "margin_dpo/margin_mean": 458.41961669921875, "margin_dpo/margin_std": 826.8089599609375, "step": 569 }, { "KL/chosen_KL_mean": -801.3278198242188, "KL/mean": -1011.711669921875, "KL/rejected_KL_mean": -1222.095458984375, "KL/std": 619.7181396484375, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0008181704906746745, "fcm_dpo/delta": 0.06755281984806061, "fcm_dpo/margin": 420.76763916015625, "fcm_dpo/q_t": 0.42422983050346375, "grad_norm": 28.688831329345703, "learning_rate": 4.019267817841834e-08, "logits/chosen": -1.3244519233703613, "logits/rejected": -1.3194363117218018, "logps/chosen": -862.9423828125, "logps/ref_chosen": -61.61454391479492, "logps/ref_rejected": -82.14186096191406, "logps/rejected": -1304.2373046875, "loss": 1.1451, "margin_dpo/margin_mean": 420.76763916015625, "margin_dpo/margin_std": 679.84033203125, "step": 570 }, { "KL/chosen_KL_mean": -844.9622802734375, "KL/mean": -1082.0240478515625, "KL/rejected_KL_mean": -1319.085693359375, "KL/std": 643.911865234375, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.0008347969851456583, "fcm_dpo/delta": 0.005073328502476215, "fcm_dpo/margin": 474.1234130859375, "fcm_dpo/q_t": 0.412005752325058, "grad_norm": 48.486839294433594, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.1971943378448486, "logits/rejected": -1.2182611227035522, "logps/chosen": -898.016357421875, "logps/ref_chosen": -53.05406188964844, "logps/ref_rejected": -91.33682250976562, "logps/rejected": -1410.422607421875, "loss": 1.1393, "margin_dpo/margin_mean": 474.1234436035156, "margin_dpo/margin_std": 829.1712646484375, "step": 571 }, { "KL/chosen_KL_mean": -877.3575439453125, "KL/mean": -1135.824951171875, "KL/rejected_KL_mean": -1394.29248046875, "KL/std": 772.8182373046875, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.000814650789834559, "fcm_dpo/delta": -0.027804527431726456, "fcm_dpo/margin": 516.9349365234375, "fcm_dpo/q_t": 0.40952882170677185, "grad_norm": 26.61905288696289, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.2630120515823364, "logits/rejected": -1.3064543008804321, "logps/chosen": -925.8168334960938, "logps/ref_chosen": -48.45928955078125, "logps/ref_rejected": -83.55703735351562, "logps/rejected": -1477.849609375, "loss": 1.1352, "margin_dpo/margin_mean": 516.9349365234375, "margin_dpo/margin_std": 933.8126220703125, "step": 572 }, { "KL/chosen_KL_mean": -831.7431640625, "KL/mean": -1071.42333984375, "KL/rejected_KL_mean": -1311.103515625, "KL/std": 665.356201171875, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0008302472997456789, "fcm_dpo/delta": 0.0017357263714075089, "fcm_dpo/margin": 479.3603515625, "fcm_dpo/q_t": 0.4101864695549011, "grad_norm": 33.60465621948242, "learning_rate": 3.812409996461275e-08, "logits/chosen": -1.2730963230133057, "logits/rejected": -1.296579360961914, "logps/chosen": -883.36572265625, "logps/ref_chosen": -51.62262725830078, "logps/ref_rejected": -85.32499694824219, "logps/rejected": -1396.428466796875, "loss": 1.0996, "margin_dpo/margin_mean": 479.3603515625, "margin_dpo/margin_std": 673.96435546875, "step": 573 }, { "KL/chosen_KL_mean": -761.8274536132812, "KL/mean": -1024.19091796875, "KL/rejected_KL_mean": -1286.554443359375, "KL/std": 611.08935546875, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0008096909150481224, "fcm_dpo/delta": -0.03171788156032562, "fcm_dpo/margin": 524.7269287109375, "fcm_dpo/q_t": 0.40161073207855225, "grad_norm": 34.691165924072266, "learning_rate": 3.74457160675965e-08, "logits/chosen": -1.2997612953186035, "logits/rejected": -1.3460713624954224, "logps/chosen": -812.8719482421875, "logps/ref_chosen": -51.04446029663086, "logps/ref_rejected": -92.80640411376953, "logps/rejected": -1379.36083984375, "loss": 1.0834, "margin_dpo/margin_mean": 524.7269287109375, "margin_dpo/margin_std": 706.32373046875, "step": 574 }, { "KL/chosen_KL_mean": -815.347412109375, "KL/mean": -1056.3719482421875, "KL/rejected_KL_mean": -1297.396484375, "KL/std": 656.7642211914062, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0007790890522301197, "fcm_dpo/delta": 0.024538405239582062, "fcm_dpo/margin": 482.0490417480469, "fcm_dpo/q_t": 0.4154299199581146, "grad_norm": 46.73399353027344, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.071021318435669, "logits/rejected": -1.0894410610198975, "logps/chosen": -887.1375732421875, "logps/ref_chosen": -71.7901382446289, "logps/ref_rejected": -95.38619995117188, "logps/rejected": -1392.78271484375, "loss": 1.1529, "margin_dpo/margin_mean": 482.049072265625, "margin_dpo/margin_std": 828.638671875, "step": 575 }, { "KL/chosen_KL_mean": -783.9774780273438, "KL/mean": -965.3140869140625, "KL/rejected_KL_mean": -1146.650634765625, "KL/std": 556.8798828125, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.000867416150867939, "fcm_dpo/delta": 0.10194016993045807, "fcm_dpo/margin": 362.67315673828125, "fcm_dpo/q_t": 0.4307815134525299, "grad_norm": 33.000823974609375, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.2250196933746338, "logits/rejected": -1.2645316123962402, "logps/chosen": -838.240478515625, "logps/ref_chosen": -54.262962341308594, "logps/ref_rejected": -100.75428009033203, "logps/rejected": -1247.405029296875, "loss": 1.1724, "margin_dpo/margin_mean": 362.67315673828125, "margin_dpo/margin_std": 646.2176513671875, "step": 576 }, { "KL/chosen_KL_mean": -681.5286254882812, "KL/mean": -917.073974609375, "KL/rejected_KL_mean": -1152.619384765625, "KL/std": 618.7384033203125, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0008785222889855504, "fcm_dpo/delta": -0.01841025985777378, "fcm_dpo/margin": 471.0908203125, "fcm_dpo/q_t": 0.4073464274406433, "grad_norm": 27.403722763061523, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.1902132034301758, "logits/rejected": -1.181257724761963, "logps/chosen": -743.4383544921875, "logps/ref_chosen": -61.909706115722656, "logps/ref_rejected": -84.07069396972656, "logps/rejected": -1236.6900634765625, "loss": 1.1109, "margin_dpo/margin_mean": 471.0908203125, "margin_dpo/margin_std": 701.18603515625, "step": 577 }, { "KL/chosen_KL_mean": -655.4379272460938, "KL/mean": -906.4620361328125, "KL/rejected_KL_mean": -1157.486083984375, "KL/std": 596.8307495117188, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.000839193002320826, "fcm_dpo/delta": -0.03209435194730759, "fcm_dpo/margin": 502.04815673828125, "fcm_dpo/q_t": 0.4018974304199219, "grad_norm": 33.629188537597656, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.0385322570800781, "logits/rejected": -1.082366943359375, "logps/chosen": -704.7015991210938, "logps/ref_chosen": -49.26368713378906, "logps/ref_rejected": -83.4362564086914, "logps/rejected": -1240.92236328125, "loss": 1.0797, "margin_dpo/margin_mean": 502.04815673828125, "margin_dpo/margin_std": 644.6083984375, "step": 578 }, { "KL/chosen_KL_mean": -802.5963134765625, "KL/mean": -953.2222900390625, "KL/rejected_KL_mean": -1103.8482666015625, "KL/std": 631.9645385742188, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0009417862165719271, "fcm_dpo/delta": 0.13616657257080078, "fcm_dpo/margin": 301.25201416015625, "fcm_dpo/q_t": 0.43976956605911255, "grad_norm": 40.763694763183594, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.1537394523620605, "logits/rejected": -1.164426326751709, "logps/chosen": -861.482177734375, "logps/ref_chosen": -58.88581848144531, "logps/ref_rejected": -94.78762817382812, "logps/rejected": -1198.635986328125, "loss": 1.2241, "margin_dpo/margin_mean": 301.25201416015625, "margin_dpo/margin_std": 703.5513916015625, "step": 579 }, { "KL/chosen_KL_mean": -557.6210327148438, "KL/mean": -839.1734619140625, "KL/rejected_KL_mean": -1120.725830078125, "KL/std": 694.3538208007812, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0008712954586371779, "fcm_dpo/delta": -0.1274590790271759, "fcm_dpo/margin": 563.1048583984375, "fcm_dpo/q_t": 0.38948333263397217, "grad_norm": 29.91592788696289, "learning_rate": 3.349364905389032e-08, "logits/chosen": -0.9779269695281982, "logits/rejected": -1.0172150135040283, "logps/chosen": -606.327880859375, "logps/ref_chosen": -48.70683670043945, "logps/ref_rejected": -81.7583999633789, "logps/rejected": -1202.4842529296875, "loss": 1.0591, "margin_dpo/margin_mean": 563.1048583984375, "margin_dpo/margin_std": 814.8396606445312, "step": 580 }, { "KL/chosen_KL_mean": -770.3370361328125, "KL/mean": -978.4288330078125, "KL/rejected_KL_mean": -1186.520751953125, "KL/std": 666.4249267578125, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0008886848227120936, "fcm_dpo/delta": 0.03695109486579895, "fcm_dpo/margin": 416.1837158203125, "fcm_dpo/q_t": 0.41858458518981934, "grad_norm": 36.696754455566406, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.2336806058883667, "logits/rejected": -1.262026071548462, "logps/chosen": -832.559326171875, "logps/ref_chosen": -62.22235107421875, "logps/ref_rejected": -91.73568725585938, "logps/rejected": -1278.25634765625, "loss": 1.1399, "margin_dpo/margin_mean": 416.1837158203125, "margin_dpo/margin_std": 706.4039306640625, "step": 581 }, { "KL/chosen_KL_mean": -691.3490600585938, "KL/mean": -915.9049072265625, "KL/rejected_KL_mean": -1140.460693359375, "KL/std": 554.6382446289062, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.000924235675483942, "fcm_dpo/delta": -0.028372012078762054, "fcm_dpo/margin": 449.111572265625, "fcm_dpo/q_t": 0.40326303243637085, "grad_norm": 34.956790924072266, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.1378693580627441, "logits/rejected": -1.1712710857391357, "logps/chosen": -749.5777587890625, "logps/ref_chosen": -58.228660583496094, "logps/ref_rejected": -110.06959533691406, "logps/rejected": -1250.5302734375, "loss": 1.1015, "margin_dpo/margin_mean": 449.111572265625, "margin_dpo/margin_std": 610.8257446289062, "step": 582 }, { "KL/chosen_KL_mean": -759.1519775390625, "KL/mean": -915.6939697265625, "KL/rejected_KL_mean": -1072.2359619140625, "KL/std": 620.579833984375, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0008432507747784257, "fcm_dpo/delta": -0.03345673531293869, "fcm_dpo/margin": 313.083984375, "fcm_dpo/q_t": 0.43994593620300293, "grad_norm": 49.73270034790039, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.1442230939865112, "logits/rejected": -1.1517397165298462, "logps/chosen": -816.0148315429688, "logps/ref_chosen": -56.86286163330078, "logps/ref_rejected": -88.4039306640625, "logps/rejected": -1160.639892578125, "loss": 1.2397, "margin_dpo/margin_mean": 313.083984375, "margin_dpo/margin_std": 745.907470703125, "step": 583 }, { "KL/chosen_KL_mean": -676.06298828125, "KL/mean": -953.474853515625, "KL/rejected_KL_mean": -1230.88671875, "KL/std": 634.57861328125, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0008079511462710798, "fcm_dpo/delta": -0.06212177127599716, "fcm_dpo/margin": 554.8237915039062, "fcm_dpo/q_t": 0.3955593705177307, "grad_norm": 27.940731048583984, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.0924582481384277, "logits/rejected": -1.1197701692581177, "logps/chosen": -732.9636840820312, "logps/ref_chosen": -56.90068054199219, "logps/ref_rejected": -97.63606262207031, "logps/rejected": -1328.5228271484375, "loss": 1.0545, "margin_dpo/margin_mean": 554.8238525390625, "margin_dpo/margin_std": 655.2168579101562, "step": 584 }, { "KL/chosen_KL_mean": -754.1314697265625, "KL/mean": -1025.2532958984375, "KL/rejected_KL_mean": -1296.3751220703125, "KL/std": 712.186279296875, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0007676214445382357, "fcm_dpo/delta": -0.022018061950802803, "fcm_dpo/margin": 542.24365234375, "fcm_dpo/q_t": 0.40494245290756226, "grad_norm": 26.670785903930664, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.1742076873779297, "logits/rejected": -1.2190337181091309, "logps/chosen": -815.10546875, "logps/ref_chosen": -60.973968505859375, "logps/ref_rejected": -84.16952514648438, "logps/rejected": -1380.544677734375, "loss": 1.1091, "margin_dpo/margin_mean": 542.24365234375, "margin_dpo/margin_std": 845.0147094726562, "step": 585 }, { "KL/chosen_KL_mean": -803.51416015625, "KL/mean": -982.947021484375, "KL/rejected_KL_mean": -1162.3798828125, "KL/std": 643.473876953125, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0008414636831730604, "fcm_dpo/delta": 0.11576494574546814, "fcm_dpo/margin": 358.8658447265625, "fcm_dpo/q_t": 0.43414974212646484, "grad_norm": 35.62641525268555, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.1895267963409424, "logits/rejected": -1.2259807586669922, "logps/chosen": -860.3697509765625, "logps/ref_chosen": -56.85559844970703, "logps/ref_rejected": -91.80261993408203, "logps/rejected": -1254.1826171875, "loss": 1.1841, "margin_dpo/margin_mean": 358.8658447265625, "margin_dpo/margin_std": 677.7850952148438, "step": 586 }, { "KL/chosen_KL_mean": -560.0857543945312, "KL/mean": -783.292724609375, "KL/rejected_KL_mean": -1006.4996337890625, "KL/std": 627.01611328125, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0008853634353727102, "fcm_dpo/delta": 0.005062144249677658, "fcm_dpo/margin": 446.41387939453125, "fcm_dpo/q_t": 0.4096928834915161, "grad_norm": 31.3030948638916, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -1.2153189182281494, "logits/rejected": -1.2506670951843262, "logps/chosen": -604.77734375, "logps/ref_chosen": -44.69159698486328, "logps/ref_rejected": -82.62385559082031, "logps/rejected": -1089.12353515625, "loss": 1.1015, "margin_dpo/margin_mean": 446.41387939453125, "margin_dpo/margin_std": 625.9844970703125, "step": 587 }, { "KL/chosen_KL_mean": -758.8787841796875, "KL/mean": -942.8692626953125, "KL/rejected_KL_mean": -1126.859619140625, "KL/std": 569.2108154296875, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0009099560556933284, "fcm_dpo/delta": 0.07821156829595566, "fcm_dpo/margin": 367.9808349609375, "fcm_dpo/q_t": 0.42506054043769836, "grad_norm": 27.26927947998047, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.142134666442871, "logits/rejected": -1.1726654767990112, "logps/chosen": -809.1737670898438, "logps/ref_chosen": -50.29494857788086, "logps/ref_rejected": -107.36988067626953, "logps/rejected": -1234.2294921875, "loss": 1.1514, "margin_dpo/margin_mean": 367.9808349609375, "margin_dpo/margin_std": 589.927734375, "step": 588 }, { "KL/chosen_KL_mean": -769.4779663085938, "KL/mean": -1022.2039794921875, "KL/rejected_KL_mean": -1274.929931640625, "KL/std": 642.8394775390625, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0008996998658403754, "fcm_dpo/delta": -0.07083002477884293, "fcm_dpo/margin": 505.4520263671875, "fcm_dpo/q_t": 0.39429712295532227, "grad_norm": 30.192138671875, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.128014087677002, "logits/rejected": -1.1533284187316895, "logps/chosen": -829.4078369140625, "logps/ref_chosen": -59.929908752441406, "logps/ref_rejected": -111.65534973144531, "logps/rejected": -1386.5853271484375, "loss": 1.0682, "margin_dpo/margin_mean": 505.4520263671875, "margin_dpo/margin_std": 678.6458740234375, "step": 589 }, { "KL/chosen_KL_mean": -634.3226928710938, "KL/mean": -899.4765625, "KL/rejected_KL_mean": -1164.63037109375, "KL/std": 646.88037109375, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.000832444173283875, "fcm_dpo/delta": -0.05546820908784866, "fcm_dpo/margin": 530.3076782226562, "fcm_dpo/q_t": 0.3984396457672119, "grad_norm": 26.239267349243164, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.1118061542510986, "logits/rejected": -1.1564977169036865, "logps/chosen": -690.1324462890625, "logps/ref_chosen": -55.80979537963867, "logps/ref_rejected": -106.06282043457031, "logps/rejected": -1270.693115234375, "loss": 1.0645, "margin_dpo/margin_mean": 530.3077392578125, "margin_dpo/margin_std": 680.8991088867188, "step": 590 }, { "KL/chosen_KL_mean": -681.802490234375, "KL/mean": -927.5146484375, "KL/rejected_KL_mean": -1173.226806640625, "KL/std": 607.2724609375, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0008302570786327124, "fcm_dpo/delta": -0.010073492303490639, "fcm_dpo/margin": 491.42431640625, "fcm_dpo/q_t": 0.40566641092300415, "grad_norm": 36.62338638305664, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.1887190341949463, "logits/rejected": -1.203287124633789, "logps/chosen": -738.0430908203125, "logps/ref_chosen": -56.24061965942383, "logps/ref_rejected": -83.78629302978516, "logps/rejected": -1257.012939453125, "loss": 1.0977, "margin_dpo/margin_mean": 491.42431640625, "margin_dpo/margin_std": 697.486572265625, "step": 591 }, { "KL/chosen_KL_mean": -740.6405639648438, "KL/mean": -975.1524658203125, "KL/rejected_KL_mean": -1209.6644287109375, "KL/std": 639.179931640625, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0008298831526190042, "fcm_dpo/delta": 0.013343624770641327, "fcm_dpo/margin": 469.0238037109375, "fcm_dpo/q_t": 0.413235068321228, "grad_norm": 32.85460662841797, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.1930391788482666, "logits/rejected": -1.2116713523864746, "logps/chosen": -788.580810546875, "logps/ref_chosen": -47.94025421142578, "logps/ref_rejected": -75.73287963867188, "logps/rejected": -1285.397216796875, "loss": 1.152, "margin_dpo/margin_mean": 469.0238342285156, "margin_dpo/margin_std": 840.769775390625, "step": 592 }, { "KL/chosen_KL_mean": -722.489013671875, "KL/mean": -891.135986328125, "KL/rejected_KL_mean": -1059.782958984375, "KL/std": 666.1737060546875, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.0008565220050513744, "fcm_dpo/delta": 0.02374492585659027, "fcm_dpo/margin": 337.2939147949219, "fcm_dpo/q_t": 0.4361189305782318, "grad_norm": 36.292320251464844, "learning_rate": 2.564009866938349e-08, "logits/chosen": -1.0026201009750366, "logits/rejected": -1.000308871269226, "logps/chosen": -771.1798095703125, "logps/ref_chosen": -48.690757751464844, "logps/ref_rejected": -60.90800094604492, "logps/rejected": -1120.69091796875, "loss": 1.2122, "margin_dpo/margin_mean": 337.2939147949219, "margin_dpo/margin_std": 735.4708862304688, "step": 593 }, { "KL/chosen_KL_mean": -694.244384765625, "KL/mean": -903.6416015625, "KL/rejected_KL_mean": -1113.038818359375, "KL/std": 638.9234619140625, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.000834466889500618, "fcm_dpo/delta": -0.05287259444594383, "fcm_dpo/margin": 418.79449462890625, "fcm_dpo/q_t": 0.41835910081863403, "grad_norm": 43.15362548828125, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.1837817430496216, "logits/rejected": -1.1828114986419678, "logps/chosen": -749.1792602539062, "logps/ref_chosen": -54.93488693237305, "logps/ref_rejected": -86.09967803955078, "logps/rejected": -1199.1385498046875, "loss": 1.1504, "margin_dpo/margin_mean": 418.7945556640625, "margin_dpo/margin_std": 674.802001953125, "step": 594 }, { "KL/chosen_KL_mean": -639.7455444335938, "KL/mean": -877.6624755859375, "KL/rejected_KL_mean": -1115.579345703125, "KL/std": 593.7979736328125, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0008135140524245799, "fcm_dpo/delta": 0.015700072050094604, "fcm_dpo/margin": 475.833740234375, "fcm_dpo/q_t": 0.4137133061885834, "grad_norm": 40.35921859741211, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.1239649057388306, "logits/rejected": -1.1565492153167725, "logps/chosen": -689.1659545898438, "logps/ref_chosen": -49.4204216003418, "logps/ref_rejected": -80.62731170654297, "logps/rejected": -1196.20654296875, "loss": 1.1059, "margin_dpo/margin_mean": 475.83380126953125, "margin_dpo/margin_std": 678.7783203125, "step": 595 }, { "KL/chosen_KL_mean": -760.5462646484375, "KL/mean": -910.779052734375, "KL/rejected_KL_mean": -1061.011962890625, "KL/std": 600.0560913085938, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0009213665034621954, "fcm_dpo/delta": 0.14197216928005219, "fcm_dpo/margin": 300.4656982421875, "fcm_dpo/q_t": 0.4435434341430664, "grad_norm": 95.65324401855469, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.1321377754211426, "logits/rejected": -1.1026864051818848, "logps/chosen": -820.3379516601562, "logps/ref_chosen": -59.791683197021484, "logps/ref_rejected": -80.09111785888672, "logps/rejected": -1141.10302734375, "loss": 1.2344, "margin_dpo/margin_mean": 300.4656677246094, "margin_dpo/margin_std": 739.6461181640625, "step": 596 }, { "KL/chosen_KL_mean": -720.8272705078125, "KL/mean": -1009.4996337890625, "KL/rejected_KL_mean": -1298.1719970703125, "KL/std": 710.2843017578125, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0008209669031202793, "fcm_dpo/delta": -0.12117333710193634, "fcm_dpo/margin": 577.3447265625, "fcm_dpo/q_t": 0.389828622341156, "grad_norm": 26.715089797973633, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.0798090696334839, "logits/rejected": -1.139901876449585, "logps/chosen": -778.0880126953125, "logps/ref_chosen": -57.26078796386719, "logps/ref_rejected": -100.6937255859375, "logps/rejected": -1398.86572265625, "loss": 1.0726, "margin_dpo/margin_mean": 577.3447265625, "margin_dpo/margin_std": 803.3443603515625, "step": 597 }, { "KL/chosen_KL_mean": -685.9241943359375, "KL/mean": -916.6578369140625, "KL/rejected_KL_mean": -1147.391357421875, "KL/std": 597.9251098632812, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0008235409623011947, "fcm_dpo/delta": 0.01904139667749405, "fcm_dpo/margin": 461.46710205078125, "fcm_dpo/q_t": 0.4120750427246094, "grad_norm": 30.65525245666504, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.1381169557571411, "logits/rejected": -1.1627564430236816, "logps/chosen": -738.4427490234375, "logps/ref_chosen": -52.51850509643555, "logps/ref_rejected": -89.44385528564453, "logps/rejected": -1236.835205078125, "loss": 1.1215, "margin_dpo/margin_mean": 461.4671630859375, "margin_dpo/margin_std": 674.5496826171875, "step": 598 }, { "KL/chosen_KL_mean": -730.5025634765625, "KL/mean": -930.897216796875, "KL/rejected_KL_mean": -1131.2918701171875, "KL/std": 586.238037109375, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0008123376173898578, "fcm_dpo/delta": -0.05563164874911308, "fcm_dpo/margin": 400.7892761230469, "fcm_dpo/q_t": 0.4195353388786316, "grad_norm": 32.0465202331543, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.0951640605926514, "logits/rejected": -1.108811616897583, "logps/chosen": -780.3052978515625, "logps/ref_chosen": -49.802677154541016, "logps/ref_rejected": -82.978515625, "logps/rejected": -1214.2703857421875, "loss": 1.1353, "margin_dpo/margin_mean": 400.789306640625, "margin_dpo/margin_std": 522.7140502929688, "step": 599 }, { "KL/chosen_KL_mean": -804.6961669921875, "KL/mean": -995.6710205078125, "KL/rejected_KL_mean": -1186.645751953125, "KL/std": 597.6373291015625, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.0008582416921854019, "fcm_dpo/delta": 0.08676525950431824, "fcm_dpo/margin": 381.94964599609375, "fcm_dpo/q_t": 0.4284188151359558, "grad_norm": 34.12265396118164, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -1.236138939857483, "logits/rejected": -1.2366291284561157, "logps/chosen": -871.131103515625, "logps/ref_chosen": -66.43487548828125, "logps/ref_rejected": -85.45649719238281, "logps/rejected": -1272.102294921875, "loss": 1.1788, "margin_dpo/margin_mean": 381.94964599609375, "margin_dpo/margin_std": 733.8538818359375, "step": 600 }, { "KL/chosen_KL_mean": -790.10302734375, "KL/mean": -1049.4549560546875, "KL/rejected_KL_mean": -1308.806884765625, "KL/std": 661.8978271484375, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.000848105177283287, "fcm_dpo/delta": -0.05170179903507233, "fcm_dpo/margin": 518.703857421875, "fcm_dpo/q_t": 0.39643436670303345, "grad_norm": 31.908023834228516, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.1642279624938965, "logits/rejected": -1.2025851011276245, "logps/chosen": -849.2366943359375, "logps/ref_chosen": -59.13361358642578, "logps/ref_rejected": -94.69093322753906, "logps/rejected": -1403.497802734375, "loss": 1.0642, "margin_dpo/margin_mean": 518.703857421875, "margin_dpo/margin_std": 650.58935546875, "step": 601 }, { "KL/chosen_KL_mean": -551.074951171875, "KL/mean": -840.6611938476562, "KL/rejected_KL_mean": -1130.24755859375, "KL/std": 586.427490234375, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0008106931345537305, "fcm_dpo/delta": -0.0917593240737915, "fcm_dpo/margin": 579.1724853515625, "fcm_dpo/q_t": 0.38940227031707764, "grad_norm": 74.04801177978516, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -1.1794953346252441, "logits/rejected": -1.2283287048339844, "logps/chosen": -599.66845703125, "logps/ref_chosen": -48.59352111816406, "logps/ref_rejected": -87.6685562133789, "logps/rejected": -1217.916015625, "loss": 1.0376, "margin_dpo/margin_mean": 579.1725463867188, "margin_dpo/margin_std": 647.2698974609375, "step": 602 }, { "KL/chosen_KL_mean": -703.7813720703125, "KL/mean": -975.3844604492188, "KL/rejected_KL_mean": -1246.987548828125, "KL/std": 681.866943359375, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0007574381306767464, "fcm_dpo/delta": -0.014486655592918396, "fcm_dpo/margin": 543.206298828125, "fcm_dpo/q_t": 0.4078108072280884, "grad_norm": 36.255306243896484, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.1725047826766968, "logits/rejected": -1.1789551973342896, "logps/chosen": -774.1959838867188, "logps/ref_chosen": -70.41461944580078, "logps/ref_rejected": -100.32559967041016, "logps/rejected": -1347.313232421875, "loss": 1.0949, "margin_dpo/margin_mean": 543.206298828125, "margin_dpo/margin_std": 783.991943359375, "step": 603 }, { "KL/chosen_KL_mean": -677.0938110351562, "KL/mean": -970.4219360351562, "KL/rejected_KL_mean": -1263.75, "KL/std": 666.1785888671875, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0007347924984060228, "fcm_dpo/delta": -0.03963564336299896, "fcm_dpo/margin": 586.65625, "fcm_dpo/q_t": 0.40320873260498047, "grad_norm": 34.325931549072266, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.1817028522491455, "logits/rejected": -1.2359271049499512, "logps/chosen": -723.5518798828125, "logps/ref_chosen": -46.45808029174805, "logps/ref_rejected": -91.8544921875, "logps/rejected": -1355.6044921875, "loss": 1.0899, "margin_dpo/margin_mean": 586.65625, "margin_dpo/margin_std": 854.323974609375, "step": 604 }, { "KL/chosen_KL_mean": -748.796875, "KL/mean": -951.0224609375, "KL/rejected_KL_mean": -1153.248046875, "KL/std": 612.2020263671875, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.000774614978581667, "fcm_dpo/delta": 0.10340875387191772, "fcm_dpo/margin": 404.4511413574219, "fcm_dpo/q_t": 0.4324561655521393, "grad_norm": 47.217437744140625, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.118881344795227, "logits/rejected": -1.1395517587661743, "logps/chosen": -815.0462646484375, "logps/ref_chosen": -66.24933624267578, "logps/ref_rejected": -102.30496978759766, "logps/rejected": -1255.552978515625, "loss": 1.176, "margin_dpo/margin_mean": 404.451171875, "margin_dpo/margin_std": 743.8991088867188, "step": 605 }, { "KL/chosen_KL_mean": -717.6261596679688, "KL/mean": -960.13623046875, "KL/rejected_KL_mean": -1202.646240234375, "KL/std": 601.002685546875, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0008218929287977517, "fcm_dpo/delta": 0.00010516494512557983, "fcm_dpo/margin": 485.0201721191406, "fcm_dpo/q_t": 0.40895628929138184, "grad_norm": 25.46479606628418, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.141870141029358, "logits/rejected": -1.163278579711914, "logps/chosen": -772.4452514648438, "logps/ref_chosen": -54.819122314453125, "logps/ref_rejected": -98.37146759033203, "logps/rejected": -1301.017822265625, "loss": 1.0914, "margin_dpo/margin_mean": 485.02020263671875, "margin_dpo/margin_std": 640.8707275390625, "step": 606 }, { "KL/chosen_KL_mean": -767.1453247070312, "KL/mean": -966.7313232421875, "KL/rejected_KL_mean": -1166.3173828125, "KL/std": 642.6415405273438, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0008614829275757074, "fcm_dpo/delta": 0.06497818231582642, "fcm_dpo/margin": 399.1719970703125, "fcm_dpo/q_t": 0.42393672466278076, "grad_norm": 33.80935287475586, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.1142241954803467, "logits/rejected": -1.104911208152771, "logps/chosen": -825.2293701171875, "logps/ref_chosen": -58.08403778076172, "logps/ref_rejected": -79.777099609375, "logps/rejected": -1246.094482421875, "loss": 1.159, "margin_dpo/margin_mean": 399.1720275878906, "margin_dpo/margin_std": 708.9419555664062, "step": 607 }, { "KL/chosen_KL_mean": -686.8588256835938, "KL/mean": -919.4632568359375, "KL/rejected_KL_mean": -1152.0677490234375, "KL/std": 572.4481811523438, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0008695480646565557, "fcm_dpo/delta": -0.006582053378224373, "fcm_dpo/margin": 465.2089538574219, "fcm_dpo/q_t": 0.40590620040893555, "grad_norm": 34.56772232055664, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.1690213680267334, "logits/rejected": -1.1923552751541138, "logps/chosen": -744.3096923828125, "logps/ref_chosen": -57.450836181640625, "logps/ref_rejected": -94.77339172363281, "logps/rejected": -1246.841064453125, "loss": 1.0818, "margin_dpo/margin_mean": 465.208984375, "margin_dpo/margin_std": 576.3578491210938, "step": 608 }, { "KL/chosen_KL_mean": -712.329345703125, "KL/mean": -1004.8988037109375, "KL/rejected_KL_mean": -1297.46826171875, "KL/std": 753.2168579101562, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.000799752539023757, "fcm_dpo/delta": -0.0890444964170456, "fcm_dpo/margin": 585.1388549804688, "fcm_dpo/q_t": 0.39761674404144287, "grad_norm": 28.98295783996582, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -1.2440345287322998, "logits/rejected": -1.281120777130127, "logps/chosen": -771.134765625, "logps/ref_chosen": -58.805355072021484, "logps/ref_rejected": -88.81600952148438, "logps/rejected": -1386.2841796875, "loss": 1.082, "margin_dpo/margin_mean": 585.1388549804688, "margin_dpo/margin_std": 920.4691162109375, "step": 609 }, { "KL/chosen_KL_mean": -670.919189453125, "KL/mean": -864.196044921875, "KL/rejected_KL_mean": -1057.472900390625, "KL/std": 553.8067626953125, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0008423338877037168, "fcm_dpo/delta": 0.08745961636304855, "fcm_dpo/margin": 386.55364990234375, "fcm_dpo/q_t": 0.42681533098220825, "grad_norm": 36.83287811279297, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.1055896282196045, "logits/rejected": -1.0923852920532227, "logps/chosen": -736.6142578125, "logps/ref_chosen": -65.69503784179688, "logps/ref_rejected": -83.40538787841797, "logps/rejected": -1140.878173828125, "loss": 1.1577, "margin_dpo/margin_mean": 386.55364990234375, "margin_dpo/margin_std": 650.5970458984375, "step": 610 }, { "KL/chosen_KL_mean": -696.1116943359375, "KL/mean": -999.7044677734375, "KL/rejected_KL_mean": -1303.29736328125, "KL/std": 736.2969970703125, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0007899197516962886, "fcm_dpo/delta": -0.10505129396915436, "fcm_dpo/margin": 607.1857299804688, "fcm_dpo/q_t": 0.39069780707359314, "grad_norm": 37.02203369140625, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.1951907873153687, "logits/rejected": -1.256514310836792, "logps/chosen": -748.7111206054688, "logps/ref_chosen": -52.59946823120117, "logps/ref_rejected": -86.33099365234375, "logps/rejected": -1389.62841796875, "loss": 1.0611, "margin_dpo/margin_mean": 607.1856689453125, "margin_dpo/margin_std": 833.6317138671875, "step": 611 }, { "KL/chosen_KL_mean": -774.832275390625, "KL/mean": -1000.640625, "KL/rejected_KL_mean": -1226.448974609375, "KL/std": 606.4058837890625, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0008069322793744504, "fcm_dpo/delta": 0.04110585153102875, "fcm_dpo/margin": 451.61676025390625, "fcm_dpo/q_t": 0.417686402797699, "grad_norm": 29.89786148071289, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.1885337829589844, "logits/rejected": -1.1959373950958252, "logps/chosen": -834.156005859375, "logps/ref_chosen": -59.32372283935547, "logps/ref_rejected": -88.31239318847656, "logps/rejected": -1314.761474609375, "loss": 1.1247, "margin_dpo/margin_mean": 451.61676025390625, "margin_dpo/margin_std": 676.6371459960938, "step": 612 }, { "KL/chosen_KL_mean": -677.6046142578125, "KL/mean": -974.4893188476562, "KL/rejected_KL_mean": -1271.3740234375, "KL/std": 672.0018310546875, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0007531556766480207, "fcm_dpo/delta": -0.0629609227180481, "fcm_dpo/margin": 593.7694702148438, "fcm_dpo/q_t": 0.39546507596969604, "grad_norm": 26.967117309570312, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.1772477626800537, "logits/rejected": -1.2423419952392578, "logps/chosen": -737.3345336914062, "logps/ref_chosen": -59.72996520996094, "logps/ref_rejected": -105.10752868652344, "logps/rejected": -1376.481689453125, "loss": 1.0554, "margin_dpo/margin_mean": 593.7694702148438, "margin_dpo/margin_std": 724.92333984375, "step": 613 }, { "KL/chosen_KL_mean": -767.2695922851562, "KL/mean": -1050.36181640625, "KL/rejected_KL_mean": -1333.453857421875, "KL/std": 694.145751953125, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0007397818844765425, "fcm_dpo/delta": -0.0238279327750206, "fcm_dpo/margin": 566.184326171875, "fcm_dpo/q_t": 0.4034821391105652, "grad_norm": 31.130779266357422, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.1655349731445312, "logits/rejected": -1.2575234174728394, "logps/chosen": -820.2085571289062, "logps/ref_chosen": -52.93898010253906, "logps/ref_rejected": -104.67938232421875, "logps/rejected": -1438.13330078125, "loss": 1.0995, "margin_dpo/margin_mean": 566.184326171875, "margin_dpo/margin_std": 844.2413330078125, "step": 614 }, { "KL/chosen_KL_mean": -710.9827270507812, "KL/mean": -932.9111328125, "KL/rejected_KL_mean": -1154.839599609375, "KL/std": 673.3746337890625, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0007818863959982991, "fcm_dpo/delta": 0.06144233047962189, "fcm_dpo/margin": 443.85687255859375, "fcm_dpo/q_t": 0.4220801591873169, "grad_norm": 32.169227600097656, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.1130198240280151, "logits/rejected": -1.1419811248779297, "logps/chosen": -776.800048828125, "logps/ref_chosen": -65.81727600097656, "logps/ref_rejected": -95.17749786376953, "logps/rejected": -1250.01708984375, "loss": 1.1437, "margin_dpo/margin_mean": 443.85687255859375, "margin_dpo/margin_std": 725.8150634765625, "step": 615 }, { "KL/chosen_KL_mean": -881.0988159179688, "KL/mean": -1073.1583251953125, "KL/rejected_KL_mean": -1265.2177734375, "KL/std": 594.451171875, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0008393003372475505, "fcm_dpo/delta": 0.09190573543310165, "fcm_dpo/margin": 384.11895751953125, "fcm_dpo/q_t": 0.428872287273407, "grad_norm": 44.38339614868164, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.2580994367599487, "logits/rejected": -1.2420811653137207, "logps/chosen": -946.231689453125, "logps/ref_chosen": -65.13285827636719, "logps/ref_rejected": -74.70050048828125, "logps/rejected": -1339.9183349609375, "loss": 1.1633, "margin_dpo/margin_mean": 384.11895751953125, "margin_dpo/margin_std": 659.7879638671875, "step": 616 }, { "KL/chosen_KL_mean": -825.0336303710938, "KL/mean": -983.9627685546875, "KL/rejected_KL_mean": -1142.891845703125, "KL/std": 582.34765625, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0009453271050006151, "fcm_dpo/delta": 0.11414283514022827, "fcm_dpo/margin": 317.8581848144531, "fcm_dpo/q_t": 0.4370740056037903, "grad_norm": 73.6228256225586, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.1125876903533936, "logits/rejected": -1.0867927074432373, "logps/chosen": -888.0391845703125, "logps/ref_chosen": -63.005550384521484, "logps/ref_rejected": -64.234130859375, "logps/rejected": -1207.1259765625, "loss": 1.2115, "margin_dpo/margin_mean": 317.85821533203125, "margin_dpo/margin_std": 713.1170654296875, "step": 617 }, { "KL/chosen_KL_mean": -846.281494140625, "KL/mean": -1157.50244140625, "KL/rejected_KL_mean": -1468.7235107421875, "KL/std": 774.007080078125, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0008609164506196976, "fcm_dpo/delta": -0.18371938169002533, "fcm_dpo/margin": 622.4420776367188, "fcm_dpo/q_t": 0.38035058975219727, "grad_norm": 32.81733322143555, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.1643829345703125, "logits/rejected": -1.221153736114502, "logps/chosen": -913.3828125, "logps/ref_chosen": -67.10134887695312, "logps/ref_rejected": -92.15340423583984, "logps/rejected": -1560.876953125, "loss": 1.087, "margin_dpo/margin_mean": 622.4420776367188, "margin_dpo/margin_std": 1005.3324584960938, "step": 618 }, { "KL/chosen_KL_mean": -804.8983154296875, "KL/mean": -1043.05419921875, "KL/rejected_KL_mean": -1281.2099609375, "KL/std": 693.5120239257812, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.000811479170806706, "fcm_dpo/delta": 0.0165516696870327, "fcm_dpo/margin": 476.311767578125, "fcm_dpo/q_t": 0.4198111295700073, "grad_norm": 43.509002685546875, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.1278049945831299, "logits/rejected": -1.1508402824401855, "logps/chosen": -860.87646484375, "logps/ref_chosen": -55.978233337402344, "logps/ref_rejected": -93.1854019165039, "logps/rejected": -1374.3953857421875, "loss": 1.1626, "margin_dpo/margin_mean": 476.311767578125, "margin_dpo/margin_std": 932.0376586914062, "step": 619 }, { "KL/chosen_KL_mean": -741.0186157226562, "KL/mean": -960.3197021484375, "KL/rejected_KL_mean": -1179.6207275390625, "KL/std": 604.758544921875, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.000822564004920423, "fcm_dpo/delta": 0.046854112297296524, "fcm_dpo/margin": 438.6020812988281, "fcm_dpo/q_t": 0.4194413125514984, "grad_norm": 36.871070861816406, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -1.2204086780548096, "logits/rejected": -1.2356070280075073, "logps/chosen": -800.8161010742188, "logps/ref_chosen": -59.79750061035156, "logps/ref_rejected": -78.41075134277344, "logps/rejected": -1258.031494140625, "loss": 1.131, "margin_dpo/margin_mean": 438.6020812988281, "margin_dpo/margin_std": 668.6788940429688, "step": 620 }, { "KL/chosen_KL_mean": -726.4075317382812, "KL/mean": -1026.935546875, "KL/rejected_KL_mean": -1327.463623046875, "KL/std": 694.7574462890625, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0008126673055812716, "fcm_dpo/delta": -0.11893101036548615, "fcm_dpo/margin": 601.0560302734375, "fcm_dpo/q_t": 0.38432037830352783, "grad_norm": 41.782691955566406, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.2633055448532104, "logits/rejected": -1.3201026916503906, "logps/chosen": -780.34130859375, "logps/ref_chosen": -53.93375778198242, "logps/ref_rejected": -88.36951446533203, "logps/rejected": -1415.8330078125, "loss": 1.0593, "margin_dpo/margin_mean": 601.0560302734375, "margin_dpo/margin_std": 780.2889404296875, "step": 621 }, { "KL/chosen_KL_mean": -730.9711303710938, "KL/mean": -956.7696533203125, "KL/rejected_KL_mean": -1182.568115234375, "KL/std": 568.827392578125, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0007760204607620835, "fcm_dpo/delta": 0.0598348043859005, "fcm_dpo/margin": 451.59698486328125, "fcm_dpo/q_t": 0.4228161573410034, "grad_norm": 30.10220718383789, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.0861146450042725, "logits/rejected": -1.0753684043884277, "logps/chosen": -791.2569580078125, "logps/ref_chosen": -60.28582000732422, "logps/ref_rejected": -85.51873779296875, "logps/rejected": -1268.0869140625, "loss": 1.1376, "margin_dpo/margin_mean": 451.5970153808594, "margin_dpo/margin_std": 714.3831787109375, "step": 622 }, { "KL/chosen_KL_mean": -812.854736328125, "KL/mean": -974.1890258789062, "KL/rejected_KL_mean": -1135.523193359375, "KL/std": 589.9398193359375, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.000887159607373178, "fcm_dpo/delta": 0.13209515810012817, "fcm_dpo/margin": 322.6684875488281, "fcm_dpo/q_t": 0.440715491771698, "grad_norm": 47.28063201904297, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -1.2864371538162231, "logits/rejected": -1.2883734703063965, "logps/chosen": -877.01171875, "logps/ref_chosen": -64.1569595336914, "logps/ref_rejected": -85.08304595947266, "logps/rejected": -1220.606201171875, "loss": 1.2157, "margin_dpo/margin_mean": 322.6684875488281, "margin_dpo/margin_std": 727.4967041015625, "step": 623 }, { "KL/chosen_KL_mean": -803.52783203125, "KL/mean": -1085.2625732421875, "KL/rejected_KL_mean": -1366.997314453125, "KL/std": 636.1299438476562, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0008517538080923259, "fcm_dpo/delta": -0.10489484667778015, "fcm_dpo/margin": 563.469482421875, "fcm_dpo/q_t": 0.3851096034049988, "grad_norm": 33.36922836303711, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -1.2476253509521484, "logits/rejected": -1.259102463722229, "logps/chosen": -875.446533203125, "logps/ref_chosen": -71.91862487792969, "logps/ref_rejected": -97.13203430175781, "logps/rejected": -1464.12939453125, "loss": 1.0463, "margin_dpo/margin_mean": 563.469482421875, "margin_dpo/margin_std": 700.0657958984375, "step": 624 }, { "KL/chosen_KL_mean": -754.678466796875, "KL/mean": -1054.7698974609375, "KL/rejected_KL_mean": -1354.8614501953125, "KL/std": 678.5252685546875, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.000783138326369226, "fcm_dpo/delta": -0.09115005284547806, "fcm_dpo/margin": 600.1829833984375, "fcm_dpo/q_t": 0.38854146003723145, "grad_norm": 52.66489791870117, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.1888365745544434, "logits/rejected": -1.228683352470398, "logps/chosen": -813.0205078125, "logps/ref_chosen": -58.342071533203125, "logps/ref_rejected": -86.09038543701172, "logps/rejected": -1440.9517822265625, "loss": 1.0299, "margin_dpo/margin_mean": 600.1829833984375, "margin_dpo/margin_std": 645.32763671875, "step": 625 }, { "KL/chosen_KL_mean": -938.5303344726562, "KL/mean": -1111.8682861328125, "KL/rejected_KL_mean": -1285.206298828125, "KL/std": 750.4013671875, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0008260097820311785, "fcm_dpo/delta": 0.13311442732810974, "fcm_dpo/margin": 346.6759033203125, "fcm_dpo/q_t": 0.4367343485355377, "grad_norm": 28.725139617919922, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.1261563301086426, "logits/rejected": -1.1264156103134155, "logps/chosen": -1013.6429443359375, "logps/ref_chosen": -75.11260986328125, "logps/ref_rejected": -99.188720703125, "logps/rejected": -1384.39501953125, "loss": 1.2138, "margin_dpo/margin_mean": 346.6759033203125, "margin_dpo/margin_std": 732.3070068359375, "step": 626 }, { "KL/chosen_KL_mean": -613.0308837890625, "KL/mean": -944.4686279296875, "KL/rejected_KL_mean": -1275.90625, "KL/std": 782.295166015625, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.000785567332059145, "fcm_dpo/delta": -0.16194023191928864, "fcm_dpo/margin": 662.8754272460938, "fcm_dpo/q_t": 0.38459348678588867, "grad_norm": 26.760295867919922, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.0580464601516724, "logits/rejected": -1.1465545892715454, "logps/chosen": -660.7740478515625, "logps/ref_chosen": -47.74314880371094, "logps/ref_rejected": -106.75448608398438, "logps/rejected": -1382.6607666015625, "loss": 1.0457, "margin_dpo/margin_mean": 662.8753662109375, "margin_dpo/margin_std": 917.1488647460938, "step": 627 }, { "KL/chosen_KL_mean": -848.233642578125, "KL/mean": -1102.293212890625, "KL/rejected_KL_mean": -1356.35302734375, "KL/std": 702.3494873046875, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0007409874233417213, "fcm_dpo/delta": 0.02891882136464119, "fcm_dpo/margin": 508.11920166015625, "fcm_dpo/q_t": 0.41614317893981934, "grad_norm": 28.3284854888916, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.2321488857269287, "logits/rejected": -1.258644700050354, "logps/chosen": -908.4166259765625, "logps/ref_chosen": -60.182945251464844, "logps/ref_rejected": -101.55467224121094, "logps/rejected": -1457.90771484375, "loss": 1.1272, "margin_dpo/margin_mean": 508.1192626953125, "margin_dpo/margin_std": 800.9361572265625, "step": 628 }, { "KL/chosen_KL_mean": -841.8361206054688, "KL/mean": -1071.118408203125, "KL/rejected_KL_mean": -1300.4007568359375, "KL/std": 656.1351318359375, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0007858018507249653, "fcm_dpo/delta": 0.04754742980003357, "fcm_dpo/margin": 458.56463623046875, "fcm_dpo/q_t": 0.4175514578819275, "grad_norm": 37.84564208984375, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.2372636795043945, "logits/rejected": -1.2568615674972534, "logps/chosen": -906.0496826171875, "logps/ref_chosen": -64.21354675292969, "logps/ref_rejected": -91.65367126464844, "logps/rejected": -1392.054443359375, "loss": 1.1255, "margin_dpo/margin_mean": 458.56463623046875, "margin_dpo/margin_std": 679.4099731445312, "step": 629 }, { "KL/chosen_KL_mean": -726.173583984375, "KL/mean": -865.2925415039062, "KL/rejected_KL_mean": -1004.4114990234375, "KL/std": 653.0287475585938, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0008041888941079378, "fcm_dpo/delta": 0.038379114121198654, "fcm_dpo/margin": 278.2379455566406, "fcm_dpo/q_t": 0.45435744524002075, "grad_norm": 49.139530181884766, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.1940525770187378, "logits/rejected": -1.1697330474853516, "logps/chosen": -785.464599609375, "logps/ref_chosen": -59.29100036621094, "logps/ref_rejected": -83.59829711914062, "logps/rejected": -1088.009765625, "loss": 1.2713, "margin_dpo/margin_mean": 278.2379455566406, "margin_dpo/margin_std": 808.236572265625, "step": 630 }, { "KL/chosen_KL_mean": -864.2023315429688, "KL/mean": -1085.380126953125, "KL/rejected_KL_mean": -1306.557861328125, "KL/std": 646.6876831054688, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.000846027978695929, "fcm_dpo/delta": 0.031014172360301018, "fcm_dpo/margin": 442.35552978515625, "fcm_dpo/q_t": 0.4147540330886841, "grad_norm": 34.697113037109375, "learning_rate": 8.518543427732949e-09, "logits/chosen": -1.3160864114761353, "logits/rejected": -1.3359469175338745, "logps/chosen": -923.6559448242188, "logps/ref_chosen": -59.45360565185547, "logps/ref_rejected": -80.95156860351562, "logps/rejected": -1387.509521484375, "loss": 1.1439, "margin_dpo/margin_mean": 442.35552978515625, "margin_dpo/margin_std": 760.668212890625, "step": 631 }, { "KL/chosen_KL_mean": -795.5738525390625, "KL/mean": -1007.8973388671875, "KL/rejected_KL_mean": -1220.220947265625, "KL/std": 619.8748779296875, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0008544891024939716, "fcm_dpo/delta": 0.044195935130119324, "fcm_dpo/margin": 424.64697265625, "fcm_dpo/q_t": 0.4180447459220886, "grad_norm": 51.36729049682617, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.279064416885376, "logits/rejected": -1.281667947769165, "logps/chosen": -856.9254150390625, "logps/ref_chosen": -61.35155487060547, "logps/ref_rejected": -86.16017150878906, "logps/rejected": -1306.381103515625, "loss": 1.1631, "margin_dpo/margin_mean": 424.64697265625, "margin_dpo/margin_std": 784.182373046875, "step": 632 }, { "KL/chosen_KL_mean": -906.9592895507812, "KL/mean": -1059.207275390625, "KL/rejected_KL_mean": -1211.455322265625, "KL/std": 601.04248046875, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.000900345272384584, "fcm_dpo/delta": 0.03711273893713951, "fcm_dpo/margin": 304.4959716796875, "fcm_dpo/q_t": 0.4383799433708191, "grad_norm": 50.936614990234375, "learning_rate": 7.866980873399015e-09, "logits/chosen": -1.30977463722229, "logits/rejected": -1.322211503982544, "logps/chosen": -964.2374267578125, "logps/ref_chosen": -57.27816390991211, "logps/ref_rejected": -91.58395385742188, "logps/rejected": -1303.039306640625, "loss": 1.2248, "margin_dpo/margin_mean": 304.4960021972656, "margin_dpo/margin_std": 702.047119140625, "step": 633 }, { "KL/chosen_KL_mean": -957.939208984375, "KL/mean": -1103.0045166015625, "KL/rejected_KL_mean": -1248.06982421875, "KL/std": 700.0567626953125, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0010044092778116465, "fcm_dpo/delta": 0.12742158770561218, "fcm_dpo/margin": 290.1304931640625, "fcm_dpo/q_t": 0.4410281479358673, "grad_norm": 48.39108657836914, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.1118695735931396, "logits/rejected": -1.1093440055847168, "logps/chosen": -1024.5582275390625, "logps/ref_chosen": -66.61896514892578, "logps/ref_rejected": -107.12564849853516, "logps/rejected": -1355.1954345703125, "loss": 1.2305, "margin_dpo/margin_mean": 290.1304931640625, "margin_dpo/margin_std": 708.994873046875, "step": 634 }, { "KL/chosen_KL_mean": -850.6639404296875, "KL/mean": -1053.29736328125, "KL/rejected_KL_mean": -1255.930908203125, "KL/std": 765.2185668945312, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.001036031637340784, "fcm_dpo/delta": -0.02556215599179268, "fcm_dpo/margin": 405.26690673828125, "fcm_dpo/q_t": 0.41270193457603455, "grad_norm": 61.66108703613281, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.2386977672576904, "logits/rejected": -1.2290127277374268, "logps/chosen": -924.6194458007812, "logps/ref_chosen": -73.95551300048828, "logps/ref_rejected": -82.50045776367188, "logps/rejected": -1338.431396484375, "loss": 1.1466, "margin_dpo/margin_mean": 405.2669372558594, "margin_dpo/margin_std": 769.2952880859375, "step": 635 }, { "KL/chosen_KL_mean": -771.644775390625, "KL/mean": -1004.4685668945312, "KL/rejected_KL_mean": -1237.29248046875, "KL/std": 704.637451171875, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0009967228397727013, "fcm_dpo/delta": -0.08795134723186493, "fcm_dpo/margin": 465.6476135253906, "fcm_dpo/q_t": 0.39764294028282166, "grad_norm": 32.073509216308594, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.1573783159255981, "logits/rejected": -1.1642378568649292, "logps/chosen": -831.273681640625, "logps/ref_chosen": -59.628910064697266, "logps/ref_rejected": -81.97883605957031, "logps/rejected": -1319.271240234375, "loss": 1.1023, "margin_dpo/margin_mean": 465.6475830078125, "margin_dpo/margin_std": 754.404541015625, "step": 636 }, { "KL/chosen_KL_mean": -774.3438110351562, "KL/mean": -1051.230712890625, "KL/rejected_KL_mean": -1328.117431640625, "KL/std": 712.5130615234375, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0008392821764573455, "fcm_dpo/delta": -0.09347308427095413, "fcm_dpo/margin": 553.773681640625, "fcm_dpo/q_t": 0.39158618450164795, "grad_norm": 31.033771514892578, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.2771704196929932, "logits/rejected": -1.3423259258270264, "logps/chosen": -823.9964599609375, "logps/ref_chosen": -49.652687072753906, "logps/ref_rejected": -98.40513610839844, "logps/rejected": -1426.5225830078125, "loss": 1.0572, "margin_dpo/margin_mean": 553.773681640625, "margin_dpo/margin_std": 714.4674072265625, "step": 637 }, { "KL/chosen_KL_mean": -740.0574951171875, "KL/mean": -960.1834716796875, "KL/rejected_KL_mean": -1180.3095703125, "KL/std": 647.9360961914062, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0008902228437364101, "fcm_dpo/delta": -0.0007760748267173767, "fcm_dpo/margin": 440.2519836425781, "fcm_dpo/q_t": 0.40659117698669434, "grad_norm": 46.73960494995117, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.0999634265899658, "logits/rejected": -1.0819281339645386, "logps/chosen": -798.2141723632812, "logps/ref_chosen": -58.156639099121094, "logps/ref_rejected": -79.3014907836914, "logps/rejected": -1259.6109619140625, "loss": 1.1511, "margin_dpo/margin_mean": 440.2519836425781, "margin_dpo/margin_std": 770.6236572265625, "step": 638 }, { "KL/chosen_KL_mean": -1026.832275390625, "KL/mean": -1113.654052734375, "KL/rejected_KL_mean": -1200.4757080078125, "KL/std": 620.957275390625, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0008462556288577616, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 173.64344787597656, "fcm_dpo/q_t": 0.46698197722435, "grad_norm": 103.9713363647461, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.081902265548706, "logits/rejected": -1.0169289112091064, "logps/chosen": -1099.155517578125, "logps/ref_chosen": -72.32319641113281, "logps/ref_rejected": -74.2749252319336, "logps/rejected": -1274.7506103515625, "loss": 1.3501, "margin_dpo/margin_mean": 173.64346313476562, "margin_dpo/margin_std": 787.2015380859375, "step": 639 }, { "KL/chosen_KL_mean": -761.8798828125, "KL/mean": -1064.907958984375, "KL/rejected_KL_mean": -1367.93603515625, "KL/std": 741.346923828125, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.000793866696767509, "fcm_dpo/delta": -0.10620252788066864, "fcm_dpo/margin": 606.05615234375, "fcm_dpo/q_t": 0.3925554156303406, "grad_norm": 52.245452880859375, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.0945394039154053, "logits/rejected": -1.133887767791748, "logps/chosen": -818.0142822265625, "logps/ref_chosen": -56.13436508178711, "logps/ref_rejected": -108.60014343261719, "logps/rejected": -1476.5361328125, "loss": 1.0622, "margin_dpo/margin_mean": 606.05615234375, "margin_dpo/margin_std": 859.9293212890625, "step": 640 }, { "KL/chosen_KL_mean": -864.1361083984375, "KL/mean": -1076.710693359375, "KL/rejected_KL_mean": -1289.285400390625, "KL/std": 632.7630615234375, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0008008034783415496, "fcm_dpo/delta": 0.0718461126089096, "fcm_dpo/margin": 425.14935302734375, "fcm_dpo/q_t": 0.4279997944831848, "grad_norm": 27.010753631591797, "learning_rate": 5.516592558795746e-09, "logits/chosen": -1.1861107349395752, "logits/rejected": -1.2011582851409912, "logps/chosen": -929.1329345703125, "logps/ref_chosen": -64.99689483642578, "logps/ref_rejected": -86.99232482910156, "logps/rejected": -1376.2777099609375, "loss": 1.1776, "margin_dpo/margin_mean": 425.14935302734375, "margin_dpo/margin_std": 845.0499877929688, "step": 641 }, { "KL/chosen_KL_mean": -837.578125, "KL/mean": -1093.5718994140625, "KL/rejected_KL_mean": -1349.565673828125, "KL/std": 812.8326416015625, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0008224358316510916, "fcm_dpo/delta": -0.02903025597333908, "fcm_dpo/margin": 511.987548828125, "fcm_dpo/q_t": 0.412765234708786, "grad_norm": 36.307071685791016, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.1126117706298828, "logits/rejected": -1.1542222499847412, "logps/chosen": -903.267333984375, "logps/ref_chosen": -65.68924713134766, "logps/ref_rejected": -110.24205017089844, "logps/rejected": -1459.8076171875, "loss": 1.152, "margin_dpo/margin_mean": 511.987548828125, "margin_dpo/margin_std": 1003.8565673828125, "step": 642 }, { "KL/chosen_KL_mean": -778.371826171875, "KL/mean": -960.838623046875, "KL/rejected_KL_mean": -1143.305419921875, "KL/std": 605.3773193359375, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0007829120149835944, "fcm_dpo/delta": -0.014414433389902115, "fcm_dpo/margin": 364.93359375, "fcm_dpo/q_t": 0.43285301327705383, "grad_norm": 33.376853942871094, "learning_rate": 4.993270631642038e-09, "logits/chosen": -1.2616610527038574, "logits/rejected": -1.2592954635620117, "logps/chosen": -830.32177734375, "logps/ref_chosen": -51.94999694824219, "logps/ref_rejected": -87.46833801269531, "logps/rejected": -1230.7738037109375, "loss": 1.1815, "margin_dpo/margin_mean": 364.93359375, "margin_dpo/margin_std": 620.4666748046875, "step": 643 }, { "KL/chosen_KL_mean": -716.4739990234375, "KL/mean": -920.228515625, "KL/rejected_KL_mean": -1123.983154296875, "KL/std": 688.679931640625, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0008282763883471489, "fcm_dpo/delta": 0.07501886785030365, "fcm_dpo/margin": 407.50909423828125, "fcm_dpo/q_t": 0.4266040325164795, "grad_norm": 32.15571594238281, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.1580393314361572, "logits/rejected": -1.1756101846694946, "logps/chosen": -775.49169921875, "logps/ref_chosen": -59.017662048339844, "logps/ref_rejected": -87.13668823242188, "logps/rejected": -1211.1197509765625, "loss": 1.1794, "margin_dpo/margin_mean": 407.50909423828125, "margin_dpo/margin_std": 801.0850830078125, "step": 644 }, { "KL/chosen_KL_mean": -772.6549072265625, "KL/mean": -870.29248046875, "KL/rejected_KL_mean": -967.9300537109375, "KL/std": 544.7769165039062, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0009023309103213251, "fcm_dpo/delta": 0.06694209575653076, "fcm_dpo/margin": 195.2750701904297, "fcm_dpo/q_t": 0.4632098078727722, "grad_norm": 85.04663848876953, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.126030683517456, "logits/rejected": -1.115586280822754, "logps/chosen": -828.5309448242188, "logps/ref_chosen": -55.87602233886719, "logps/ref_rejected": -97.78080749511719, "logps/rejected": -1065.7108154296875, "loss": 1.3318, "margin_dpo/margin_mean": 195.2751007080078, "margin_dpo/margin_std": 740.9451904296875, "step": 645 }, { "KL/chosen_KL_mean": -735.0755615234375, "KL/mean": -908.2972412109375, "KL/rejected_KL_mean": -1081.5189208984375, "KL/std": 522.7762451171875, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0009436524705961347, "fcm_dpo/delta": 0.08735965192317963, "fcm_dpo/margin": 346.443359375, "fcm_dpo/q_t": 0.4290798604488373, "grad_norm": 62.62091064453125, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.1851344108581543, "logits/rejected": -1.1642956733703613, "logps/chosen": -796.351318359375, "logps/ref_chosen": -61.275787353515625, "logps/ref_rejected": -77.50580596923828, "logps/rejected": -1159.024658203125, "loss": 1.1762, "margin_dpo/margin_mean": 346.4433288574219, "margin_dpo/margin_std": 640.25537109375, "step": 646 }, { "KL/chosen_KL_mean": -630.638671875, "KL/mean": -835.8479614257812, "KL/rejected_KL_mean": -1041.0572509765625, "KL/std": 575.6964111328125, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0009774916106835008, "fcm_dpo/delta": -0.0016585299745202065, "fcm_dpo/margin": 410.41851806640625, "fcm_dpo/q_t": 0.40817171335220337, "grad_norm": 27.86429786682129, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.0603611469268799, "logits/rejected": -1.0982961654663086, "logps/chosen": -685.4910888671875, "logps/ref_chosen": -54.8524169921875, "logps/ref_rejected": -93.5194091796875, "logps/rejected": -1134.57666015625, "loss": 1.1114, "margin_dpo/margin_mean": 410.41851806640625, "margin_dpo/margin_std": 619.114990234375, "step": 647 }, { "KL/chosen_KL_mean": -673.4736328125, "KL/mean": -996.9141845703125, "KL/rejected_KL_mean": -1320.354736328125, "KL/std": 748.5928955078125, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0008425582782365382, "fcm_dpo/delta": -0.20228338241577148, "fcm_dpo/margin": 646.8811645507812, "fcm_dpo/q_t": 0.3778739869594574, "grad_norm": 32.02235412597656, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.1987550258636475, "logits/rejected": -1.2712297439575195, "logps/chosen": -727.6451416015625, "logps/ref_chosen": -54.17146682739258, "logps/ref_rejected": -98.7127914428711, "logps/rejected": -1419.067626953125, "loss": 1.0347, "margin_dpo/margin_mean": 646.8811645507812, "margin_dpo/margin_std": 918.9876708984375, "step": 648 }, { "KL/chosen_KL_mean": -728.5607299804688, "KL/mean": -868.1865234375, "KL/rejected_KL_mean": -1007.812255859375, "KL/std": 557.9706420898438, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0008311655838042498, "fcm_dpo/delta": 0.06771388649940491, "fcm_dpo/margin": 279.2515869140625, "fcm_dpo/q_t": 0.4502527713775635, "grad_norm": 32.22618865966797, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -1.224090814590454, "logits/rejected": -1.2157777547836304, "logps/chosen": -791.0410766601562, "logps/ref_chosen": -62.480350494384766, "logps/ref_rejected": -80.07717895507812, "logps/rejected": -1087.8895263671875, "loss": 1.2478, "margin_dpo/margin_mean": 279.2515869140625, "margin_dpo/margin_std": 673.363525390625, "step": 649 }, { "KL/chosen_KL_mean": -760.45458984375, "KL/mean": -985.4943237304688, "KL/rejected_KL_mean": -1210.533935546875, "KL/std": 681.9244995117188, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0008620332810096443, "fcm_dpo/delta": 0.014763827435672283, "fcm_dpo/margin": 450.07940673828125, "fcm_dpo/q_t": 0.4147789478302002, "grad_norm": 35.338233947753906, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.1790560483932495, "logits/rejected": -1.2069194316864014, "logps/chosen": -816.5474243164062, "logps/ref_chosen": -56.09281921386719, "logps/ref_rejected": -98.26483917236328, "logps/rejected": -1308.798828125, "loss": 1.1394, "margin_dpo/margin_mean": 450.0793762207031, "margin_dpo/margin_std": 782.4627685546875, "step": 650 }, { "KL/chosen_KL_mean": -495.2056884765625, "KL/mean": -796.1655883789062, "KL/rejected_KL_mean": -1097.12548828125, "KL/std": 634.3084716796875, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0007995986961759627, "fcm_dpo/delta": -0.10846766829490662, "fcm_dpo/margin": 601.9198608398438, "fcm_dpo/q_t": 0.3852821886539459, "grad_norm": 45.97235107421875, "learning_rate": 3.158738163478475e-09, "logits/chosen": -1.143014669418335, "logits/rejected": -1.2076648473739624, "logps/chosen": -538.631103515625, "logps/ref_chosen": -43.42544937133789, "logps/ref_rejected": -99.95791625976562, "logps/rejected": -1197.08349609375, "loss": 1.023, "margin_dpo/margin_mean": 601.9197998046875, "margin_dpo/margin_std": 651.2605590820312, "step": 651 }, { "KL/chosen_KL_mean": -655.0028076171875, "KL/mean": -886.6578979492188, "KL/rejected_KL_mean": -1118.31298828125, "KL/std": 625.3679809570312, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0008092697244137526, "fcm_dpo/delta": 0.029820626601576805, "fcm_dpo/margin": 463.31011962890625, "fcm_dpo/q_t": 0.41610437631607056, "grad_norm": 35.829437255859375, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -1.1463913917541504, "logits/rejected": -1.1780339479446411, "logps/chosen": -717.5796508789062, "logps/ref_chosen": -62.57680892944336, "logps/ref_rejected": -111.76779174804688, "logps/rejected": -1230.080810546875, "loss": 1.1258, "margin_dpo/margin_mean": 463.31011962890625, "margin_dpo/margin_std": 727.44580078125, "step": 652 }, { "KL/chosen_KL_mean": -785.6455078125, "KL/mean": -1017.7234497070312, "KL/rejected_KL_mean": -1249.8013916015625, "KL/std": 702.5745849609375, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0008226313511840999, "fcm_dpo/delta": 0.022288989275693893, "fcm_dpo/margin": 464.1558837890625, "fcm_dpo/q_t": 0.41595858335494995, "grad_norm": 36.48124313354492, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -1.2419447898864746, "logits/rejected": -1.2747020721435547, "logps/chosen": -846.7584838867188, "logps/ref_chosen": -61.11295700073242, "logps/ref_rejected": -103.24960327148438, "logps/rejected": -1353.051025390625, "loss": 1.123, "margin_dpo/margin_mean": 464.1558837890625, "margin_dpo/margin_std": 734.2294311523438, "step": 653 }, { "KL/chosen_KL_mean": -720.1655883789062, "KL/mean": -930.6776733398438, "KL/rejected_KL_mean": -1141.189697265625, "KL/std": 579.1441650390625, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0008555342210456729, "fcm_dpo/delta": 0.04842513054609299, "fcm_dpo/margin": 421.0242004394531, "fcm_dpo/q_t": 0.4207690358161926, "grad_norm": 29.795970916748047, "learning_rate": 2.577954022936174e-09, "logits/chosen": -1.1985931396484375, "logits/rejected": -1.2237248420715332, "logps/chosen": -781.8937377929688, "logps/ref_chosen": -61.7281379699707, "logps/ref_rejected": -98.7738037109375, "logps/rejected": -1239.963623046875, "loss": 1.1322, "margin_dpo/margin_mean": 421.0242004394531, "margin_dpo/margin_std": 659.9177856445312, "step": 654 }, { "KL/chosen_KL_mean": -680.0551147460938, "KL/mean": -888.6227416992188, "KL/rejected_KL_mean": -1097.1903076171875, "KL/std": 573.6949462890625, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.000887269270606339, "fcm_dpo/delta": 0.036719754338264465, "fcm_dpo/margin": 417.13525390625, "fcm_dpo/q_t": 0.4181985855102539, "grad_norm": 32.07929229736328, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.17026948928833, "logits/rejected": -1.2217252254486084, "logps/chosen": -729.6319580078125, "logps/ref_chosen": -49.576812744140625, "logps/ref_rejected": -98.29183197021484, "logps/rejected": -1195.482177734375, "loss": 1.1344, "margin_dpo/margin_mean": 417.1352233886719, "margin_dpo/margin_std": 680.7271728515625, "step": 655 }, { "KL/chosen_KL_mean": -690.5496826171875, "KL/mean": -1050.8857421875, "KL/rejected_KL_mean": -1411.2216796875, "KL/std": 771.2440185546875, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0007734264945611358, "fcm_dpo/delta": -0.2174438238143921, "fcm_dpo/margin": 720.6719970703125, "fcm_dpo/q_t": 0.3665843605995178, "grad_norm": 77.32756805419922, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.065166711807251, "logits/rejected": -1.151442289352417, "logps/chosen": -743.09912109375, "logps/ref_chosen": -52.54943084716797, "logps/ref_rejected": -113.67464447021484, "logps/rejected": -1524.896484375, "loss": 0.9841, "margin_dpo/margin_mean": 720.6719970703125, "margin_dpo/margin_std": 759.9878540039062, "step": 656 }, { "KL/chosen_KL_mean": -651.868896484375, "KL/mean": -958.4510498046875, "KL/rejected_KL_mean": -1265.033203125, "KL/std": 730.9705810546875, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0006971274269744754, "fcm_dpo/delta": -0.03567207232117653, "fcm_dpo/margin": 613.1644287109375, "fcm_dpo/q_t": 0.4013122618198395, "grad_norm": 41.946922302246094, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.2377593517303467, "logits/rejected": -1.309966802597046, "logps/chosen": -698.5694580078125, "logps/ref_chosen": -46.700538635253906, "logps/ref_rejected": -97.91487121582031, "logps/rejected": -1362.947998046875, "loss": 1.0675, "margin_dpo/margin_mean": 613.1643676757812, "margin_dpo/margin_std": 760.7774658203125, "step": 657 }, { "KL/chosen_KL_mean": -706.0740966796875, "KL/mean": -946.1854248046875, "KL/rejected_KL_mean": -1186.2967529296875, "KL/std": 566.0189208984375, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0007183150737546384, "fcm_dpo/delta": 0.06623679399490356, "fcm_dpo/margin": 480.22265625, "fcm_dpo/q_t": 0.42133376002311707, "grad_norm": 38.043453216552734, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.1505177021026611, "logits/rejected": -1.187692642211914, "logps/chosen": -767.0323486328125, "logps/ref_chosen": -60.95820999145508, "logps/ref_rejected": -95.93949127197266, "logps/rejected": -1282.2362060546875, "loss": 1.1244, "margin_dpo/margin_mean": 480.2226257324219, "margin_dpo/margin_std": 638.424072265625, "step": 658 }, { "KL/chosen_KL_mean": -624.73291015625, "KL/mean": -855.0606689453125, "KL/rejected_KL_mean": -1085.388427734375, "KL/std": 586.4114990234375, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0007668976904824376, "fcm_dpo/delta": 0.0567876435816288, "fcm_dpo/margin": 460.6553649902344, "fcm_dpo/q_t": 0.4216195344924927, "grad_norm": 27.48058319091797, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -1.0954115390777588, "logits/rejected": -1.0741057395935059, "logps/chosen": -701.4759521484375, "logps/ref_chosen": -76.74298095703125, "logps/ref_rejected": -87.4709701538086, "logps/rejected": -1172.859375, "loss": 1.1233, "margin_dpo/margin_mean": 460.6553955078125, "margin_dpo/margin_std": 662.3754272460938, "step": 659 }, { "KL/chosen_KL_mean": -714.209228515625, "KL/mean": -1003.320068359375, "KL/rejected_KL_mean": -1292.430908203125, "KL/std": 690.353515625, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0007632029009982944, "fcm_dpo/delta": -0.052839018404483795, "fcm_dpo/margin": 578.2216796875, "fcm_dpo/q_t": 0.3979361057281494, "grad_norm": 33.15445327758789, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.1898115873336792, "logits/rejected": -1.206876277923584, "logps/chosen": -773.257080078125, "logps/ref_chosen": -59.04788589477539, "logps/ref_rejected": -75.96005249023438, "logps/rejected": -1368.3909912109375, "loss": 1.0682, "margin_dpo/margin_mean": 578.2216796875, "margin_dpo/margin_std": 753.8338623046875, "step": 660 }, { "KL/chosen_KL_mean": -610.59130859375, "KL/mean": -878.7642211914062, "KL/rejected_KL_mean": -1146.9371337890625, "KL/std": 757.8616943359375, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0007565034320577979, "fcm_dpo/delta": -0.0085027776658535, "fcm_dpo/margin": 536.3458251953125, "fcm_dpo/q_t": 0.40905699133872986, "grad_norm": 47.93645477294922, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.2252672910690308, "logits/rejected": -1.252467393875122, "logps/chosen": -661.2652587890625, "logps/ref_chosen": -50.673973083496094, "logps/ref_rejected": -86.00569152832031, "logps/rejected": -1232.94287109375, "loss": 1.0877, "margin_dpo/margin_mean": 536.3458251953125, "margin_dpo/margin_std": 738.3463134765625, "step": 661 }, { "KL/chosen_KL_mean": -732.677490234375, "KL/mean": -931.5089721679688, "KL/rejected_KL_mean": -1130.3404541015625, "KL/std": 597.563232421875, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0007978413486853242, "fcm_dpo/delta": 0.09812071919441223, "fcm_dpo/margin": 397.6629333496094, "fcm_dpo/q_t": 0.4306020140647888, "grad_norm": 29.834253311157227, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.1183321475982666, "logits/rejected": -1.1220099925994873, "logps/chosen": -801.9385986328125, "logps/ref_chosen": -69.26106262207031, "logps/ref_rejected": -89.05593872070312, "logps/rejected": -1219.3963623046875, "loss": 1.178, "margin_dpo/margin_mean": 397.66290283203125, "margin_dpo/margin_std": 750.21630859375, "step": 662 }, { "KL/chosen_KL_mean": -667.935546875, "KL/mean": -934.6746826171875, "KL/rejected_KL_mean": -1201.4139404296875, "KL/std": 711.580810546875, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.0007925699464976788, "fcm_dpo/delta": -0.02965104579925537, "fcm_dpo/margin": 533.4783935546875, "fcm_dpo/q_t": 0.40630367398262024, "grad_norm": 32.280399322509766, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -1.1441211700439453, "logits/rejected": -1.1800665855407715, "logps/chosen": -732.814453125, "logps/ref_chosen": -64.87890625, "logps/ref_rejected": -113.92536926269531, "logps/rejected": -1315.33935546875, "loss": 1.1028, "margin_dpo/margin_mean": 533.4784545898438, "margin_dpo/margin_std": 843.329345703125, "step": 663 }, { "KL/chosen_KL_mean": -713.7249145507812, "KL/mean": -985.4025268554688, "KL/rejected_KL_mean": -1257.0802001953125, "KL/std": 668.9918823242188, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0007669959450140595, "fcm_dpo/delta": -0.022985609248280525, "fcm_dpo/margin": 543.3553466796875, "fcm_dpo/q_t": 0.40422388911247253, "grad_norm": 36.076908111572266, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.1553318500518799, "logits/rejected": -1.198824167251587, "logps/chosen": -774.6134033203125, "logps/ref_chosen": -60.88847351074219, "logps/ref_rejected": -105.521728515625, "logps/rejected": -1362.6019287109375, "loss": 1.0837, "margin_dpo/margin_mean": 543.3553466796875, "margin_dpo/margin_std": 723.924072265625, "step": 664 }, { "KL/chosen_KL_mean": -651.1171875, "KL/mean": -885.782470703125, "KL/rejected_KL_mean": -1120.4478759765625, "KL/std": 580.0274658203125, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0007947899866849184, "fcm_dpo/delta": 0.033155716955661774, "fcm_dpo/margin": 469.3306579589844, "fcm_dpo/q_t": 0.4146922826766968, "grad_norm": 52.002296447753906, "learning_rate": 9.513254770636137e-10, "logits/chosen": -1.2848122119903564, "logits/rejected": -1.3209171295166016, "logps/chosen": -711.6812744140625, "logps/ref_chosen": -60.56413269042969, "logps/ref_rejected": -84.80882263183594, "logps/rejected": -1205.256591796875, "loss": 1.1028, "margin_dpo/margin_mean": 469.33062744140625, "margin_dpo/margin_std": 604.48974609375, "step": 665 }, { "KL/chosen_KL_mean": -704.703125, "KL/mean": -937.7362060546875, "KL/rejected_KL_mean": -1170.769287109375, "KL/std": 603.8134765625, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0008114855154417455, "fcm_dpo/delta": 0.026824505999684334, "fcm_dpo/margin": 466.06622314453125, "fcm_dpo/q_t": 0.4145530164241791, "grad_norm": 36.94413375854492, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.0528377294540405, "logits/rejected": -1.0693092346191406, "logps/chosen": -769.123046875, "logps/ref_chosen": -64.41996002197266, "logps/ref_rejected": -95.8916244506836, "logps/rejected": -1266.660888671875, "loss": 1.1138, "margin_dpo/margin_mean": 466.06622314453125, "margin_dpo/margin_std": 662.6096801757812, "step": 666 }, { "KL/chosen_KL_mean": -802.0358276367188, "KL/mean": -1084.480712890625, "KL/rejected_KL_mean": -1366.925537109375, "KL/std": 679.0506591796875, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0007717009866610169, "fcm_dpo/delta": -0.05106900632381439, "fcm_dpo/margin": 564.8895874023438, "fcm_dpo/q_t": 0.39843225479125977, "grad_norm": 29.158369064331055, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.1201040744781494, "logits/rejected": -1.1252251863479614, "logps/chosen": -871.3128662109375, "logps/ref_chosen": -69.27702331542969, "logps/ref_rejected": -87.83549499511719, "logps/rejected": -1454.760986328125, "loss": 1.0729, "margin_dpo/margin_mean": 564.8895874023438, "margin_dpo/margin_std": 735.8818359375, "step": 667 }, { "KL/chosen_KL_mean": -843.2930297851562, "KL/mean": -1036.6741943359375, "KL/rejected_KL_mean": -1230.05517578125, "KL/std": 693.7163696289062, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0008507584570907056, "fcm_dpo/delta": 0.08011265844106674, "fcm_dpo/margin": 386.7622375488281, "fcm_dpo/q_t": 0.43072831630706787, "grad_norm": 44.3238410949707, "learning_rate": 6.453213851142225e-10, "logits/chosen": -1.2108023166656494, "logits/rejected": -1.2172951698303223, "logps/chosen": -915.8970336914062, "logps/ref_chosen": -72.60400390625, "logps/ref_rejected": -103.73905944824219, "logps/rejected": -1333.7943115234375, "loss": 1.1988, "margin_dpo/margin_mean": 386.7622375488281, "margin_dpo/margin_std": 833.273681640625, "step": 668 }, { "KL/chosen_KL_mean": -618.487060546875, "KL/mean": -896.7724609375, "KL/rejected_KL_mean": -1175.057861328125, "KL/std": 641.3170166015625, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0008110044291242957, "fcm_dpo/delta": -0.06620398163795471, "fcm_dpo/margin": 556.57080078125, "fcm_dpo/q_t": 0.39469441771507263, "grad_norm": 24.252483367919922, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.144885778427124, "logits/rejected": -1.1587302684783936, "logps/chosen": -664.603515625, "logps/ref_chosen": -46.116416931152344, "logps/ref_rejected": -77.92434692382812, "logps/rejected": -1252.9822998046875, "loss": 1.0677, "margin_dpo/margin_mean": 556.57080078125, "margin_dpo/margin_std": 739.805419921875, "step": 669 }, { "KL/chosen_KL_mean": -602.510009765625, "KL/mean": -883.6866455078125, "KL/rejected_KL_mean": -1164.86328125, "KL/std": 612.798583984375, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0007813957636244595, "fcm_dpo/delta": -0.051431987434625626, "fcm_dpo/margin": 562.3533935546875, "fcm_dpo/q_t": 0.3977850079536438, "grad_norm": 31.681093215942383, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.0778193473815918, "logits/rejected": -1.1026027202606201, "logps/chosen": -664.8557739257812, "logps/ref_chosen": -62.34575271606445, "logps/ref_rejected": -96.9405517578125, "logps/rejected": -1261.803955078125, "loss": 1.0741, "margin_dpo/margin_mean": 562.3533935546875, "margin_dpo/margin_std": 736.040771484375, "step": 670 }, { "KL/chosen_KL_mean": -771.7396240234375, "KL/mean": -993.390380859375, "KL/rejected_KL_mean": -1215.041015625, "KL/std": 630.7474365234375, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0008008808945305645, "fcm_dpo/delta": 0.051010433584451675, "fcm_dpo/margin": 443.301513671875, "fcm_dpo/q_t": 0.4195772409439087, "grad_norm": 30.4383602142334, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.2151209115982056, "logits/rejected": -1.2516989707946777, "logps/chosen": -819.73974609375, "logps/ref_chosen": -48.00010681152344, "logps/ref_rejected": -83.81932067871094, "logps/rejected": -1298.8603515625, "loss": 1.1545, "margin_dpo/margin_mean": 443.301513671875, "margin_dpo/margin_std": 778.079833984375, "step": 671 }, { "KL/chosen_KL_mean": -849.791015625, "KL/mean": -1072.2750244140625, "KL/rejected_KL_mean": -1294.759033203125, "KL/std": 755.86083984375, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0008099371916614473, "fcm_dpo/delta": 0.04807715862989426, "fcm_dpo/margin": 444.9681396484375, "fcm_dpo/q_t": 0.42253392934799194, "grad_norm": 44.19379425048828, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.274925947189331, "logits/rejected": -1.2912979125976562, "logps/chosen": -908.374267578125, "logps/ref_chosen": -58.58328628540039, "logps/ref_rejected": -93.14015197753906, "logps/rejected": -1387.899169921875, "loss": 1.1696, "margin_dpo/margin_mean": 444.9681091308594, "margin_dpo/margin_std": 849.6900634765625, "step": 672 }, { "KL/chosen_KL_mean": -751.1322631835938, "KL/mean": -956.3372802734375, "KL/rejected_KL_mean": -1161.542236328125, "KL/std": 581.4315795898438, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0008629739750176668, "fcm_dpo/delta": 0.05584951490163803, "fcm_dpo/margin": 410.40997314453125, "fcm_dpo/q_t": 0.4219851493835449, "grad_norm": 32.456729888916016, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.208233118057251, "logits/rejected": -1.214905023574829, "logps/chosen": -797.85546875, "logps/ref_chosen": -46.72320556640625, "logps/ref_rejected": -85.29623413085938, "logps/rejected": -1246.83837890625, "loss": 1.1363, "margin_dpo/margin_mean": 410.40997314453125, "margin_dpo/margin_std": 664.936279296875, "step": 673 }, { "KL/chosen_KL_mean": -606.1369018554688, "KL/mean": -891.3228759765625, "KL/rejected_KL_mean": -1176.5089111328125, "KL/std": 615.0496826171875, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0008298626053147018, "fcm_dpo/delta": -0.09556920826435089, "fcm_dpo/margin": 570.3719482421875, "fcm_dpo/q_t": 0.39160943031311035, "grad_norm": 39.19919204711914, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.1228229999542236, "logits/rejected": -1.1515860557556152, "logps/chosen": -651.5823974609375, "logps/ref_chosen": -45.445526123046875, "logps/ref_rejected": -70.04593658447266, "logps/rejected": -1246.5548095703125, "loss": 1.0523, "margin_dpo/margin_mean": 570.3719482421875, "margin_dpo/margin_std": 732.308349609375, "step": 674 }, { "KL/chosen_KL_mean": -702.86669921875, "KL/mean": -980.7730712890625, "KL/rejected_KL_mean": -1258.679443359375, "KL/std": 685.6355590820312, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0007509513525292277, "fcm_dpo/delta": -0.030970744788646698, "fcm_dpo/margin": 555.8126220703125, "fcm_dpo/q_t": 0.404343843460083, "grad_norm": 24.052825927734375, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.1530423164367676, "logits/rejected": -1.1733019351959229, "logps/chosen": -747.04296875, "logps/ref_chosen": -44.17628479003906, "logps/ref_rejected": -74.09197998046875, "logps/rejected": -1332.7713623046875, "loss": 1.0862, "margin_dpo/margin_mean": 555.8126831054688, "margin_dpo/margin_std": 725.0802001953125, "step": 675 }, { "KL/chosen_KL_mean": -762.6793823242188, "KL/mean": -1022.1181030273438, "KL/rejected_KL_mean": -1281.556884765625, "KL/std": 637.0001220703125, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0007771970704197884, "fcm_dpo/delta": -0.0041960906237363815, "fcm_dpo/margin": 518.8775024414062, "fcm_dpo/q_t": 0.40673425793647766, "grad_norm": 22.74205780029297, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.0892168283462524, "logits/rejected": -1.10811448097229, "logps/chosen": -834.077880859375, "logps/ref_chosen": -71.39852905273438, "logps/ref_rejected": -88.3587646484375, "logps/rejected": -1369.9156494140625, "loss": 1.0849, "margin_dpo/margin_mean": 518.8775024414062, "margin_dpo/margin_std": 657.755859375, "step": 676 }, { "KL/chosen_KL_mean": -713.2413940429688, "KL/mean": -988.1437377929688, "KL/rejected_KL_mean": -1263.046142578125, "KL/std": 681.3233032226562, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0007561305537819862, "fcm_dpo/delta": -0.020175673067569733, "fcm_dpo/margin": 549.8046875, "fcm_dpo/q_t": 0.40893790125846863, "grad_norm": 24.729129791259766, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.1915206909179688, "logits/rejected": -1.1915239095687866, "logps/chosen": -769.768798828125, "logps/ref_chosen": -56.527435302734375, "logps/ref_rejected": -78.22654724121094, "logps/rejected": -1341.2725830078125, "loss": 1.0947, "margin_dpo/margin_mean": 549.8046875, "margin_dpo/margin_std": 822.973876953125, "step": 677 }, { "KL/chosen_KL_mean": -606.6762084960938, "KL/mean": -909.5905151367188, "KL/rejected_KL_mean": -1212.5048828125, "KL/std": 725.3612060546875, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.000741387193556875, "fcm_dpo/delta": -0.06460094451904297, "fcm_dpo/margin": 605.8285522460938, "fcm_dpo/q_t": 0.396203875541687, "grad_norm": 33.879825592041016, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.1325280666351318, "logits/rejected": -1.1821608543395996, "logps/chosen": -652.8106689453125, "logps/ref_chosen": -46.13447570800781, "logps/ref_rejected": -80.60462951660156, "logps/rejected": -1293.109375, "loss": 1.0706, "margin_dpo/margin_mean": 605.82861328125, "margin_dpo/margin_std": 814.54296875, "step": 678 }, { "KL/chosen_KL_mean": -715.3004150390625, "KL/mean": -929.665771484375, "KL/rejected_KL_mean": -1144.03125, "KL/std": 562.1583251953125, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0007575766649097204, "fcm_dpo/delta": 0.0897865891456604, "fcm_dpo/margin": 428.7308349609375, "fcm_dpo/q_t": 0.4284992218017578, "grad_norm": 30.023422241210938, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -1.1566033363342285, "logits/rejected": -1.156817078590393, "logps/chosen": -765.5953369140625, "logps/ref_chosen": -50.294921875, "logps/ref_rejected": -76.59813690185547, "logps/rejected": -1220.62939453125, "loss": 1.1495, "margin_dpo/margin_mean": 428.7308349609375, "margin_dpo/margin_std": 679.2525634765625, "step": 679 }, { "KL/chosen_KL_mean": -714.9935302734375, "KL/mean": -986.1622314453125, "KL/rejected_KL_mean": -1257.3310546875, "KL/std": 710.556884765625, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0007529498543590307, "fcm_dpo/delta": -0.012800686061382294, "fcm_dpo/margin": 542.3373413085938, "fcm_dpo/q_t": 0.4060339331626892, "grad_norm": 35.72113800048828, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.134705901145935, "logits/rejected": -1.1759812831878662, "logps/chosen": -791.9093017578125, "logps/ref_chosen": -76.91569519042969, "logps/ref_rejected": -112.384765625, "logps/rejected": -1369.7158203125, "loss": 1.1036, "margin_dpo/margin_mean": 542.3373413085938, "margin_dpo/margin_std": 799.07666015625, "step": 680 }, { "KL/chosen_KL_mean": -731.4549560546875, "KL/mean": -950.778076171875, "KL/rejected_KL_mean": -1170.10107421875, "KL/std": 619.8388671875, "epoch": 1.0, "fcm_dpo/beta": 0.000749021302908659, "fcm_dpo/delta": -0.04601984843611717, "fcm_dpo/margin": 438.6461181640625, "fcm_dpo/q_t": 0.4235824942588806, "grad_norm": 26.493305206298828, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -1.2273634672164917, "logits/rejected": -1.2576552629470825, "logps/chosen": -792.4122314453125, "logps/ref_chosen": -60.957279205322266, "logps/ref_rejected": -88.55797576904297, "logps/rejected": -1258.6590576171875, "loss": 1.1554, "margin_dpo/margin_mean": 438.6461486816406, "margin_dpo/margin_std": 686.915771484375, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1259927408285604, "train_runtime": 1736.5167, "train_samples_per_second": 25.107, "train_steps_per_second": 0.392 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }