{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.999244142101285, "eval_steps": 200, "global_step": 661, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.02867889404296875, "KL/mean": 0.029354453086853027, "KL/rejected_KL_mean": 0.030029296875, "KL/std": 0.2071000635623932, "epoch": 0.0015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0013532638549804688, "fcm_dpo/q_t": 0.5000336766242981, "grad_norm": 28.21991729736328, "learning_rate": 0.0, "logits/chosen": 0.13337239623069763, "logits/rejected": 0.12492949515581131, "logps/chosen": -64.5841293334961, "logps/ref_chosen": -64.61280822753906, "logps/ref_rejected": -64.17195129394531, "logps/rejected": -64.14192199707031, "loss": 1.3866, "margin_dpo/margin_mean": -0.0013527870178222656, "margin_dpo/margin_std": 0.2561596930027008, "step": 1 }, { "KL/chosen_KL_mean": -0.00289154052734375, "KL/mean": -0.021616414189338684, "KL/rejected_KL_mean": -0.04033660888671875, "KL/std": 0.19624735414981842, "epoch": 0.0030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.037450045347213745, "fcm_dpo/q_t": 0.4990639090538025, "grad_norm": 27.826953887939453, "learning_rate": 7.462686567164179e-09, "logits/chosen": 0.09414851665496826, "logits/rejected": 0.07363267242908478, "logps/chosen": -56.101890563964844, "logps/ref_chosen": -56.0989990234375, "logps/ref_rejected": -66.59971618652344, "logps/rejected": -66.64006042480469, "loss": 1.3828, "margin_dpo/margin_mean": 0.03744968771934509, "margin_dpo/margin_std": 0.27811938524246216, "step": 2 }, { "KL/chosen_KL_mean": 0.04581451416015625, "KL/mean": 0.04929549992084503, "KL/rejected_KL_mean": 0.052776336669921875, "KL/std": 0.24392041563987732, "epoch": 0.0045351473922902496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.006956905126571655, "fcm_dpo/q_t": 0.5001732110977173, "grad_norm": 31.296030044555664, "learning_rate": 1.4925373134328357e-08, "logits/chosen": 0.09936262667179108, "logits/rejected": 0.061408448964357376, "logps/chosen": -65.41144561767578, "logps/ref_chosen": -65.45726013183594, "logps/ref_rejected": -90.82853698730469, "logps/rejected": -90.7757568359375, "loss": 1.3872, "margin_dpo/margin_mean": -0.006957113742828369, "margin_dpo/margin_std": 0.30191972851753235, "step": 3 }, { "KL/chosen_KL_mean": -0.03472900390625, "KL/mean": -0.02750520408153534, "KL/rejected_KL_mean": -0.02028656005859375, "KL/std": 0.22726918756961823, "epoch": 0.006046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014443963766098022, "fcm_dpo/q_t": 0.5003613233566284, "grad_norm": 34.52410125732422, "learning_rate": 2.2388059701492534e-08, "logits/chosen": 0.11272424459457397, "logits/rejected": 0.09639942646026611, "logps/chosen": -76.89491271972656, "logps/ref_chosen": -76.86018371582031, "logps/ref_rejected": -79.91523742675781, "logps/rejected": -79.9355239868164, "loss": 1.388, "margin_dpo/margin_mean": -0.014444082975387573, "margin_dpo/margin_std": 0.33552950620651245, "step": 4 }, { "KL/chosen_KL_mean": -0.0019626617431640625, "KL/mean": 0.013008832931518555, "KL/rejected_KL_mean": 0.027984619140625, "KL/std": 0.20796799659729004, "epoch": 0.007558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.029944956302642822, "fcm_dpo/q_t": 0.500747561454773, "grad_norm": 29.703157424926758, "learning_rate": 2.9850746268656714e-08, "logits/chosen": 0.06775817275047302, "logits/rejected": 0.029598888009786606, "logps/chosen": -62.973304748535156, "logps/ref_chosen": -62.97134017944336, "logps/ref_rejected": -79.9192123413086, "logps/rejected": -79.89122772216797, "loss": 1.3896, "margin_dpo/margin_mean": -0.02994471788406372, "margin_dpo/margin_std": 0.3299190402030945, "step": 5 }, { "KL/chosen_KL_mean": -0.018129348754882812, "KL/mean": 0.0032161325216293335, "KL/rejected_KL_mean": 0.024560928344726562, "KL/std": 0.22389458119869232, "epoch": 0.009070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.042690426111221313, "fcm_dpo/q_t": 0.5010668039321899, "grad_norm": 29.70970344543457, "learning_rate": 3.731343283582089e-08, "logits/chosen": 0.14717215299606323, "logits/rejected": 0.10750843584537506, "logps/chosen": -51.32549285888672, "logps/ref_chosen": -51.30736541748047, "logps/ref_rejected": -82.77239227294922, "logps/rejected": -82.74783325195312, "loss": 1.3908, "margin_dpo/margin_mean": -0.042690128087997437, "margin_dpo/margin_std": 0.3145572245121002, "step": 6 }, { "KL/chosen_KL_mean": 0.0044879913330078125, "KL/mean": -0.0019683837890625, "KL/rejected_KL_mean": -0.008426666259765625, "KL/std": 0.21890875697135925, "epoch": 0.010582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012911856174468994, "fcm_dpo/q_t": 0.4996773600578308, "grad_norm": 27.1543025970459, "learning_rate": 4.477611940298507e-08, "logits/chosen": 0.018971581012010574, "logits/rejected": -0.02494441345334053, "logps/chosen": -51.454925537109375, "logps/ref_chosen": -51.45941162109375, "logps/ref_rejected": -66.3828125, "logps/rejected": -66.3912353515625, "loss": 1.3852, "margin_dpo/margin_mean": 0.012911945581436157, "margin_dpo/margin_std": 0.26080113649368286, "step": 7 }, { "KL/chosen_KL_mean": -0.011938095092773438, "KL/mean": -0.01912282407283783, "KL/rejected_KL_mean": -0.026304244995117188, "KL/std": 0.23343831300735474, "epoch": 0.012093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014365166425704956, "fcm_dpo/q_t": 0.4996413290500641, "grad_norm": 28.60932731628418, "learning_rate": 5.223880597014925e-08, "logits/chosen": 0.0810195729136467, "logits/rejected": 0.05867453292012215, "logps/chosen": -62.2094841003418, "logps/ref_chosen": -62.197547912597656, "logps/ref_rejected": -74.66180419921875, "logps/rejected": -74.6881103515625, "loss": 1.3851, "margin_dpo/margin_mean": 0.014364898204803467, "margin_dpo/margin_std": 0.3226226270198822, "step": 8 }, { "KL/chosen_KL_mean": -0.01309967041015625, "KL/mean": -0.027955979108810425, "KL/rejected_KL_mean": -0.042816162109375, "KL/std": 0.23443172872066498, "epoch": 0.013605442176870748, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02971470355987549, "fcm_dpo/q_t": 0.4992576837539673, "grad_norm": 31.4260311126709, "learning_rate": 5.970149253731343e-08, "logits/chosen": 0.17611047625541687, "logits/rejected": 0.11633238196372986, "logps/chosen": -55.642822265625, "logps/ref_chosen": -55.629722595214844, "logps/ref_rejected": -86.21221923828125, "logps/rejected": -86.25503540039062, "loss": 1.3836, "margin_dpo/margin_mean": 0.029714584350585938, "margin_dpo/margin_std": 0.33428698778152466, "step": 9 }, { "KL/chosen_KL_mean": 0.0005950927734375, "KL/mean": -0.003879353404045105, "KL/rejected_KL_mean": -0.00835418701171875, "KL/std": 0.23718583583831787, "epoch": 0.015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.008948296308517456, "fcm_dpo/q_t": 0.49977636337280273, "grad_norm": 29.694154739379883, "learning_rate": 6.71641791044776e-08, "logits/chosen": 0.13127511739730835, "logits/rejected": 0.10026749223470688, "logps/chosen": -62.690006256103516, "logps/ref_chosen": -62.69060134887695, "logps/ref_rejected": -90.610107421875, "logps/rejected": -90.61846923828125, "loss": 1.3857, "margin_dpo/margin_mean": 0.008948534727096558, "margin_dpo/margin_std": 0.3706110715866089, "step": 10 }, { "KL/chosen_KL_mean": 0.015949249267578125, "KL/mean": 0.014684125781059265, "KL/rejected_KL_mean": 0.013416290283203125, "KL/std": 0.1973499059677124, "epoch": 0.016628873771730914, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0025348961353302, "fcm_dpo/q_t": 0.49993592500686646, "grad_norm": 29.386581420898438, "learning_rate": 7.462686567164178e-08, "logits/chosen": 0.11886583268642426, "logits/rejected": 0.11185969412326813, "logps/chosen": -65.75116729736328, "logps/ref_chosen": -65.76712036132812, "logps/ref_rejected": -72.4764633178711, "logps/rejected": -72.46304321289062, "loss": 1.3863, "margin_dpo/margin_mean": 0.002535104751586914, "margin_dpo/margin_std": 0.299337238073349, "step": 11 }, { "KL/chosen_KL_mean": -0.029123306274414062, "KL/mean": -0.021389245986938477, "KL/rejected_KL_mean": -0.0136566162109375, "KL/std": 0.21308918297290802, "epoch": 0.018140589569160998, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01546633243560791, "fcm_dpo/q_t": 0.5003858804702759, "grad_norm": 28.376405715942383, "learning_rate": 8.208955223880596e-08, "logits/chosen": 0.02483578771352768, "logits/rejected": 0.008677439764142036, "logps/chosen": -60.73401641845703, "logps/ref_chosen": -60.704891204833984, "logps/ref_rejected": -69.41564178466797, "logps/rejected": -69.4292984008789, "loss": 1.3881, "margin_dpo/margin_mean": -0.015465915203094482, "margin_dpo/margin_std": 0.32422810792922974, "step": 12 }, { "KL/chosen_KL_mean": 0.00815582275390625, "KL/mean": 0.008319318294525146, "KL/rejected_KL_mean": 0.00849151611328125, "KL/std": 0.22867777943611145, "epoch": 0.019652305366591082, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.00033211708068847656, "fcm_dpo/q_t": 0.5000076293945312, "grad_norm": 29.0564022064209, "learning_rate": 8.955223880597014e-08, "logits/chosen": 0.1282486915588379, "logits/rejected": 0.06378625333309174, "logps/chosen": -49.901100158691406, "logps/ref_chosen": -49.90925598144531, "logps/ref_rejected": -92.37818145751953, "logps/rejected": -92.36968994140625, "loss": 1.3866, "margin_dpo/margin_mean": -0.00033277273178100586, "margin_dpo/margin_std": 0.3163772225379944, "step": 13 }, { "KL/chosen_KL_mean": 0.020704269409179688, "KL/mean": -0.00048294663429260254, "KL/rejected_KL_mean": -0.021671295166015625, "KL/std": 0.22362124919891357, "epoch": 0.021164021164021163, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04237312078475952, "fcm_dpo/q_t": 0.4989417791366577, "grad_norm": 29.4510498046875, "learning_rate": 9.701492537313432e-08, "logits/chosen": 0.08631706982851028, "logits/rejected": 0.06857284903526306, "logps/chosen": -60.59809112548828, "logps/ref_chosen": -60.61879348754883, "logps/ref_rejected": -71.79306030273438, "logps/rejected": -71.81472778320312, "loss": 1.3824, "margin_dpo/margin_mean": 0.04237350821495056, "margin_dpo/margin_std": 0.33905673027038574, "step": 14 }, { "KL/chosen_KL_mean": -0.028501510620117188, "KL/mean": -0.025412842631340027, "KL/rejected_KL_mean": -0.0223236083984375, "KL/std": 0.21838335692882538, "epoch": 0.022675736961451247, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0061851441860198975, "fcm_dpo/q_t": 0.5001543760299683, "grad_norm": 33.28445816040039, "learning_rate": 1.044776119402985e-07, "logits/chosen": 0.08525186032056808, "logits/rejected": 0.04112376272678375, "logps/chosen": -63.49803924560547, "logps/ref_chosen": -63.46953582763672, "logps/ref_rejected": -88.88951110839844, "logps/rejected": -88.91183471679688, "loss": 1.3871, "margin_dpo/margin_mean": -0.006184697151184082, "margin_dpo/margin_std": 0.2978639602661133, "step": 15 }, { "KL/chosen_KL_mean": -0.00461578369140625, "KL/mean": -0.003109276294708252, "KL/rejected_KL_mean": -0.001605987548828125, "KL/std": 0.21290913224220276, "epoch": 0.02418745275888133, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.003010869026184082, "fcm_dpo/q_t": 0.500075101852417, "grad_norm": 26.85666847229004, "learning_rate": 1.1194029850746268e-07, "logits/chosen": 0.11474336683750153, "logits/rejected": 0.07771545648574829, "logps/chosen": -46.53691482543945, "logps/ref_chosen": -46.53229904174805, "logps/ref_rejected": -74.27533721923828, "logps/rejected": -74.27694702148438, "loss": 1.3868, "margin_dpo/margin_mean": -0.0030111074447631836, "margin_dpo/margin_std": 0.2670174837112427, "step": 16 }, { "KL/chosen_KL_mean": 0.010129928588867188, "KL/mean": -0.019616693258285522, "KL/rejected_KL_mean": -0.0493621826171875, "KL/std": 0.2056957632303238, "epoch": 0.025699168556311415, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.059494197368621826, "fcm_dpo/q_t": 0.49851351976394653, "grad_norm": 32.781803131103516, "learning_rate": 1.1940298507462686e-07, "logits/chosen": 0.06784220784902573, "logits/rejected": 0.04859541356563568, "logps/chosen": -64.06770324707031, "logps/ref_chosen": -64.07783508300781, "logps/ref_rejected": -86.40876770019531, "logps/rejected": -86.4581298828125, "loss": 1.3806, "margin_dpo/margin_mean": 0.0594935417175293, "margin_dpo/margin_std": 0.29295259714126587, "step": 17 }, { "KL/chosen_KL_mean": 0.03248405456542969, "KL/mean": 0.021163254976272583, "KL/rejected_KL_mean": 0.009840011596679688, "KL/std": 0.181782528758049, "epoch": 0.027210884353741496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02264273166656494, "fcm_dpo/q_t": 0.4994339942932129, "grad_norm": 27.759876251220703, "learning_rate": 1.2686567164179106e-07, "logits/chosen": 0.06629162281751633, "logits/rejected": 0.02278873696923256, "logps/chosen": -44.84185028076172, "logps/ref_chosen": -44.87433624267578, "logps/ref_rejected": -70.97604370117188, "logps/rejected": -70.9662094116211, "loss": 1.3842, "margin_dpo/margin_mean": 0.022643178701400757, "margin_dpo/margin_std": 0.2225417196750641, "step": 18 }, { "KL/chosen_KL_mean": 0.041255950927734375, "KL/mean": 0.006726168096065521, "KL/rejected_KL_mean": -0.027805328369140625, "KL/std": 0.25561243295669556, "epoch": 0.02872260015117158, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.069069042801857, "fcm_dpo/q_t": 0.4982735514640808, "grad_norm": 30.727636337280273, "learning_rate": 1.343283582089552e-07, "logits/chosen": 0.07066254317760468, "logits/rejected": 0.057106949388980865, "logps/chosen": -68.1185531616211, "logps/ref_chosen": -68.1598129272461, "logps/ref_rejected": -81.17138671875, "logps/rejected": -81.1991958618164, "loss": 1.3796, "margin_dpo/margin_mean": 0.0690688043832779, "margin_dpo/margin_std": 0.28775882720947266, "step": 19 }, { "KL/chosen_KL_mean": -0.025074005126953125, "KL/mean": 0.0022208094596862793, "KL/rejected_KL_mean": 0.02951812744140625, "KL/std": 0.21140390634536743, "epoch": 0.030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.054593682289123535, "fcm_dpo/q_t": 0.5013641119003296, "grad_norm": 29.338727951049805, "learning_rate": 1.4179104477611938e-07, "logits/chosen": 0.13627377152442932, "logits/rejected": 0.11312912404537201, "logps/chosen": -53.703636169433594, "logps/ref_chosen": -53.67856216430664, "logps/ref_rejected": -74.16911315917969, "logps/rejected": -74.13960266113281, "loss": 1.392, "margin_dpo/margin_mean": -0.05459386110305786, "margin_dpo/margin_std": 0.3187302052974701, "step": 20 }, { "KL/chosen_KL_mean": 0.007659912109375, "KL/mean": 0.021049678325653076, "KL/rejected_KL_mean": 0.0344390869140625, "KL/std": 0.23415425419807434, "epoch": 0.031746031746031744, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.026774346828460693, "fcm_dpo/q_t": 0.500669538974762, "grad_norm": 29.406789779663086, "learning_rate": 1.4925373134328355e-07, "logits/chosen": 0.11505385488271713, "logits/rejected": 0.08910010010004044, "logps/chosen": -64.69389343261719, "logps/ref_chosen": -64.70155334472656, "logps/ref_rejected": -81.02095031738281, "logps/rejected": -80.98651123046875, "loss": 1.3893, "margin_dpo/margin_mean": -0.02677476406097412, "margin_dpo/margin_std": 0.33221936225891113, "step": 21 }, { "KL/chosen_KL_mean": -0.019384384155273438, "KL/mean": -0.028041943907737732, "KL/rejected_KL_mean": -0.0366973876953125, "KL/std": 0.20282882452011108, "epoch": 0.03325774754346183, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.017315417528152466, "fcm_dpo/q_t": 0.4995673894882202, "grad_norm": 28.919084548950195, "learning_rate": 1.5671641791044775e-07, "logits/chosen": 0.002440758515149355, "logits/rejected": -0.01844123937189579, "logps/chosen": -58.05537414550781, "logps/ref_chosen": -58.03599166870117, "logps/ref_rejected": -80.72721862792969, "logps/rejected": -80.763916015625, "loss": 1.3848, "margin_dpo/margin_mean": 0.01731511950492859, "margin_dpo/margin_std": 0.29331278800964355, "step": 22 }, { "KL/chosen_KL_mean": -0.01165771484375, "KL/mean": -0.003425806760787964, "KL/rejected_KL_mean": 0.0048065185546875, "KL/std": 0.2561618685722351, "epoch": 0.03476946334089191, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.016467154026031494, "fcm_dpo/q_t": 0.5004115104675293, "grad_norm": 33.099586486816406, "learning_rate": 1.6417910447761193e-07, "logits/chosen": 0.13895675539970398, "logits/rejected": 0.11350340396165848, "logps/chosen": -66.36774444580078, "logps/ref_chosen": -66.35608673095703, "logps/ref_rejected": -93.02769470214844, "logps/rejected": -93.02288818359375, "loss": 1.3882, "margin_dpo/margin_mean": -0.01646745204925537, "margin_dpo/margin_std": 0.3485786020755768, "step": 23 }, { "KL/chosen_KL_mean": -0.023984909057617188, "KL/mean": 0.010361678898334503, "KL/rejected_KL_mean": 0.04471015930175781, "KL/std": 0.2105405628681183, "epoch": 0.036281179138321996, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06869538128376007, "fcm_dpo/q_t": 0.5017166137695312, "grad_norm": 26.39677619934082, "learning_rate": 1.716417910447761e-07, "logits/chosen": 0.12875144183635712, "logits/rejected": 0.0960879772901535, "logps/chosen": -54.48522186279297, "logps/ref_chosen": -54.461238861083984, "logps/ref_rejected": -68.33817291259766, "logps/rejected": -68.29346466064453, "loss": 1.3934, "margin_dpo/margin_mean": -0.06869524717330933, "margin_dpo/margin_std": 0.27617546916007996, "step": 24 }, { "KL/chosen_KL_mean": -0.023723602294921875, "KL/mean": -0.036136969923973083, "KL/rejected_KL_mean": -0.048553466796875, "KL/std": 0.2357097566127777, "epoch": 0.03779289493575208, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.024826496839523315, "fcm_dpo/q_t": 0.49938008189201355, "grad_norm": 29.66562271118164, "learning_rate": 1.7910447761194027e-07, "logits/chosen": 0.12693685293197632, "logits/rejected": 0.07386107742786407, "logps/chosen": -60.02792739868164, "logps/ref_chosen": -60.00420379638672, "logps/ref_rejected": -90.47376251220703, "logps/rejected": -90.5223159790039, "loss": 1.3841, "margin_dpo/margin_mean": 0.02482694387435913, "margin_dpo/margin_std": 0.3424012064933777, "step": 25 }, { "KL/chosen_KL_mean": -0.027313232421875, "KL/mean": -0.026278428733348846, "KL/rejected_KL_mean": -0.02524566650390625, "KL/std": 0.23390698432922363, "epoch": 0.039304610733182165, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0020656436681747437, "fcm_dpo/q_t": 0.500051736831665, "grad_norm": 29.946218490600586, "learning_rate": 1.8656716417910447e-07, "logits/chosen": 0.10996918380260468, "logits/rejected": 0.09148485213518143, "logps/chosen": -56.846466064453125, "logps/ref_chosen": -56.81915283203125, "logps/ref_rejected": -77.84333038330078, "logps/rejected": -77.86857604980469, "loss": 1.3867, "margin_dpo/margin_mean": -0.0020662248134613037, "margin_dpo/margin_std": 0.3143938183784485, "step": 26 }, { "KL/chosen_KL_mean": -0.01342010498046875, "KL/mean": -0.03130936622619629, "KL/rejected_KL_mean": -0.049198150634765625, "KL/std": 0.2537550628185272, "epoch": 0.04081632653061224, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03577548265457153, "fcm_dpo/q_t": 0.49910616874694824, "grad_norm": 28.97505760192871, "learning_rate": 1.9402985074626865e-07, "logits/chosen": 0.10381777584552765, "logits/rejected": 0.07897449284791946, "logps/chosen": -62.890445709228516, "logps/ref_chosen": -62.87702560424805, "logps/ref_rejected": -71.34437561035156, "logps/rejected": -71.39356994628906, "loss": 1.383, "margin_dpo/margin_mean": 0.035775572061538696, "margin_dpo/margin_std": 0.3350522220134735, "step": 27 }, { "KL/chosen_KL_mean": -0.023956298828125, "KL/mean": -0.030458927154541016, "KL/rejected_KL_mean": -0.03696250915527344, "KL/std": 0.21456307172775269, "epoch": 0.042328042328042326, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013008713722229004, "fcm_dpo/q_t": 0.499674916267395, "grad_norm": 27.615636825561523, "learning_rate": 2.0149253731343282e-07, "logits/chosen": 0.05342138558626175, "logits/rejected": 0.044701073318719864, "logps/chosen": -59.857330322265625, "logps/ref_chosen": -59.8333740234375, "logps/ref_rejected": -70.39804077148438, "logps/rejected": -70.43499755859375, "loss": 1.3852, "margin_dpo/margin_mean": 0.013009041547775269, "margin_dpo/margin_std": 0.2985638976097107, "step": 28 }, { "KL/chosen_KL_mean": -0.0227813720703125, "KL/mean": -0.03702476620674133, "KL/rejected_KL_mean": -0.05126190185546875, "KL/std": 0.20070354640483856, "epoch": 0.04383975812547241, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028487086296081543, "fcm_dpo/q_t": 0.4992881417274475, "grad_norm": 32.572021484375, "learning_rate": 2.08955223880597e-07, "logits/chosen": 0.1341443955898285, "logits/rejected": 0.11656548082828522, "logps/chosen": -74.14298248291016, "logps/ref_chosen": -74.12020111083984, "logps/ref_rejected": -83.33099365234375, "logps/rejected": -83.38224792480469, "loss": 1.3837, "margin_dpo/margin_mean": 0.028487682342529297, "margin_dpo/margin_std": 0.29014480113983154, "step": 29 }, { "KL/chosen_KL_mean": -0.020137786865234375, "KL/mean": -0.029683440923690796, "KL/rejected_KL_mean": -0.039226531982421875, "KL/std": 0.2251281887292862, "epoch": 0.045351473922902494, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.019087255001068115, "fcm_dpo/q_t": 0.49952298402786255, "grad_norm": 30.433258056640625, "learning_rate": 2.1641791044776117e-07, "logits/chosen": 0.12191575765609741, "logits/rejected": 0.06771814823150635, "logps/chosen": -50.771427154541016, "logps/ref_chosen": -50.75128936767578, "logps/ref_rejected": -89.29063415527344, "logps/rejected": -89.32986450195312, "loss": 1.3846, "margin_dpo/margin_mean": 0.01908770203590393, "margin_dpo/margin_std": 0.3208455443382263, "step": 30 }, { "KL/chosen_KL_mean": -0.019695281982421875, "KL/mean": -0.04813128709793091, "KL/rejected_KL_mean": -0.07656478881835938, "KL/std": 0.2744419574737549, "epoch": 0.04686318972033258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.056871771812438965, "fcm_dpo/q_t": 0.49857938289642334, "grad_norm": 34.18708801269531, "learning_rate": 2.2388059701492537e-07, "logits/chosen": 0.08505978435277939, "logits/rejected": 0.040289558470249176, "logps/chosen": -65.3564453125, "logps/ref_chosen": -65.33675384521484, "logps/ref_rejected": -100.76666259765625, "logps/rejected": -100.84323120117188, "loss": 1.381, "margin_dpo/margin_mean": 0.05687209963798523, "margin_dpo/margin_std": 0.36973148584365845, "step": 31 }, { "KL/chosen_KL_mean": -0.0136260986328125, "KL/mean": 0.004444405436515808, "KL/rejected_KL_mean": 0.02251434326171875, "KL/std": 0.2495591938495636, "epoch": 0.04837490551776266, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.036137908697128296, "fcm_dpo/q_t": 0.500903308391571, "grad_norm": 30.191768646240234, "learning_rate": 2.3134328358208954e-07, "logits/chosen": 0.0718865916132927, "logits/rejected": 0.06413871794939041, "logps/chosen": -67.19696044921875, "logps/ref_chosen": -67.18333435058594, "logps/ref_rejected": -82.80763244628906, "logps/rejected": -82.78512573242188, "loss": 1.3902, "margin_dpo/margin_mean": -0.03613784909248352, "margin_dpo/margin_std": 0.31982362270355225, "step": 32 }, { "KL/chosen_KL_mean": -0.05531883239746094, "KL/mean": -0.05424413084983826, "KL/rejected_KL_mean": -0.053165435791015625, "KL/std": 0.22134271264076233, "epoch": 0.049886621315192746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.002153933048248291, "fcm_dpo/q_t": 0.5000538229942322, "grad_norm": 31.012001037597656, "learning_rate": 2.388059701492537e-07, "logits/chosen": 0.03484850376844406, "logits/rejected": 0.008806673809885979, "logps/chosen": -64.09480285644531, "logps/ref_chosen": -64.03948211669922, "logps/ref_rejected": -75.68357849121094, "logps/rejected": -75.73674774169922, "loss": 1.3867, "margin_dpo/margin_mean": -0.0021534860134124756, "margin_dpo/margin_std": 0.2684057652950287, "step": 33 }, { "KL/chosen_KL_mean": -0.04139518737792969, "KL/mean": -0.06535254418849945, "KL/rejected_KL_mean": -0.0893096923828125, "KL/std": 0.17330102622509003, "epoch": 0.05139833711262283, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04792061448097229, "fcm_dpo/q_t": 0.4988023340702057, "grad_norm": 28.53345489501953, "learning_rate": 2.4626865671641786e-07, "logits/chosen": 0.11218427121639252, "logits/rejected": 0.08181029558181763, "logps/chosen": -53.7056884765625, "logps/ref_chosen": -53.6642951965332, "logps/ref_rejected": -65.77989959716797, "logps/rejected": -65.86920928955078, "loss": 1.3816, "margin_dpo/margin_mean": 0.047920435667037964, "margin_dpo/margin_std": 0.22822073101997375, "step": 34 }, { "KL/chosen_KL_mean": -0.10145187377929688, "KL/mean": -0.10271087288856506, "KL/rejected_KL_mean": -0.10396957397460938, "KL/std": 0.251731276512146, "epoch": 0.05291005291005291, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.002517104148864746, "fcm_dpo/q_t": 0.4999368190765381, "grad_norm": 27.890995025634766, "learning_rate": 2.537313432835821e-07, "logits/chosen": 0.04327782988548279, "logits/rejected": 0.02118547447025776, "logps/chosen": -61.118316650390625, "logps/ref_chosen": -61.01686096191406, "logps/ref_rejected": -72.78598022460938, "logps/rejected": -72.88995361328125, "loss": 1.3864, "margin_dpo/margin_mean": 0.0025173723697662354, "margin_dpo/margin_std": 0.36896008253097534, "step": 35 }, { "KL/chosen_KL_mean": -0.06800651550292969, "KL/mean": -0.07391357421875, "KL/rejected_KL_mean": -0.07982254028320312, "KL/std": 0.23057693243026733, "epoch": 0.05442176870748299, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011810481548309326, "fcm_dpo/q_t": 0.49970513582229614, "grad_norm": 28.75467872619629, "learning_rate": 2.611940298507462e-07, "logits/chosen": 0.09926285594701767, "logits/rejected": 0.046571869403123856, "logps/chosen": -50.605369567871094, "logps/ref_chosen": -50.53736114501953, "logps/ref_rejected": -78.11678314208984, "logps/rejected": -78.19660949707031, "loss": 1.3854, "margin_dpo/margin_mean": 0.011810898780822754, "margin_dpo/margin_std": 0.35508257150650024, "step": 36 }, { "KL/chosen_KL_mean": -0.033443450927734375, "KL/mean": -0.09048253297805786, "KL/rejected_KL_mean": -0.14752197265625, "KL/std": 0.2617317736148834, "epoch": 0.055933484504913075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11407029628753662, "fcm_dpo/q_t": 0.49715059995651245, "grad_norm": 37.495147705078125, "learning_rate": 2.686567164179104e-07, "logits/chosen": 0.1185469850897789, "logits/rejected": 0.03703591972589493, "logps/chosen": -59.5873908996582, "logps/ref_chosen": -59.55394744873047, "logps/ref_rejected": -108.27702331542969, "logps/rejected": -108.42454528808594, "loss": 1.3753, "margin_dpo/margin_mean": 0.11406975984573364, "margin_dpo/margin_std": 0.3664522171020508, "step": 37 }, { "KL/chosen_KL_mean": -0.0837249755859375, "KL/mean": -0.09539508819580078, "KL/rejected_KL_mean": -0.10706520080566406, "KL/std": 0.2474774271249771, "epoch": 0.05744520030234316, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02333807945251465, "fcm_dpo/q_t": 0.499416708946228, "grad_norm": 29.70965576171875, "learning_rate": 2.761194029850746e-07, "logits/chosen": 0.08251446485519409, "logits/rejected": 0.06839097291231155, "logps/chosen": -65.87208557128906, "logps/ref_chosen": -65.78836059570312, "logps/ref_rejected": -76.1619873046875, "logps/rejected": -76.26905059814453, "loss": 1.3843, "margin_dpo/margin_mean": 0.023338019847869873, "margin_dpo/margin_std": 0.35831284523010254, "step": 38 }, { "KL/chosen_KL_mean": -0.0574493408203125, "KL/mean": -0.08865617215633392, "KL/rejected_KL_mean": -0.11986541748046875, "KL/std": 0.25045037269592285, "epoch": 0.05895691609977324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06241282820701599, "fcm_dpo/q_t": 0.49844029545783997, "grad_norm": 29.232280731201172, "learning_rate": 2.8358208955223876e-07, "logits/chosen": 0.16253948211669922, "logits/rejected": 0.1356792151927948, "logps/chosen": -57.23426055908203, "logps/ref_chosen": -57.17681121826172, "logps/ref_rejected": -79.486328125, "logps/rejected": -79.606201171875, "loss": 1.3804, "margin_dpo/margin_mean": 0.062412917613983154, "margin_dpo/margin_std": 0.3511146306991577, "step": 39 }, { "KL/chosen_KL_mean": -0.12008476257324219, "KL/mean": -0.1282183676958084, "KL/rejected_KL_mean": -0.1363525390625, "KL/std": 0.25607573986053467, "epoch": 0.06046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.016262918710708618, "fcm_dpo/q_t": 0.4995928108692169, "grad_norm": 31.482582092285156, "learning_rate": 2.9104477611940296e-07, "logits/chosen": 0.1266281008720398, "logits/rejected": 0.07608100771903992, "logps/chosen": -61.454254150390625, "logps/ref_chosen": -61.33416748046875, "logps/ref_rejected": -79.10697174072266, "logps/rejected": -79.24332427978516, "loss": 1.385, "margin_dpo/margin_mean": 0.016263514757156372, "margin_dpo/margin_std": 0.35121607780456543, "step": 40 }, { "KL/chosen_KL_mean": -0.07088088989257812, "KL/mean": -0.14460709691047668, "KL/rejected_KL_mean": -0.21832656860351562, "KL/std": 0.3124772906303406, "epoch": 0.06198034769463341, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14744830131530762, "fcm_dpo/q_t": 0.49631595611572266, "grad_norm": 30.304224014282227, "learning_rate": 2.985074626865671e-07, "logits/chosen": 0.028469331562519073, "logits/rejected": 0.009036978706717491, "logps/chosen": -67.61761474609375, "logps/ref_chosen": -67.5467300415039, "logps/ref_rejected": -83.87788391113281, "logps/rejected": -84.0962142944336, "loss": 1.3721, "margin_dpo/margin_mean": 0.14744865894317627, "margin_dpo/margin_std": 0.4309389591217041, "step": 41 }, { "KL/chosen_KL_mean": -0.10775375366210938, "KL/mean": -0.13053923845291138, "KL/rejected_KL_mean": -0.15332794189453125, "KL/std": 0.2780609726905823, "epoch": 0.06349206349206349, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04558022320270538, "fcm_dpo/q_t": 0.4988611340522766, "grad_norm": 29.170860290527344, "learning_rate": 3.059701492537313e-07, "logits/chosen": 0.03401852399110794, "logits/rejected": 0.0130385123193264, "logps/chosen": -61.37261199951172, "logps/ref_chosen": -61.26485824584961, "logps/ref_rejected": -76.3629150390625, "logps/rejected": -76.51624298095703, "loss": 1.3821, "margin_dpo/margin_mean": 0.04557971656322479, "margin_dpo/margin_std": 0.39606839418411255, "step": 42 }, { "KL/chosen_KL_mean": -0.10474014282226562, "KL/mean": -0.1389007568359375, "KL/rejected_KL_mean": -0.17305946350097656, "KL/std": 0.30590346455574036, "epoch": 0.06500377928949358, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06832152605056763, "fcm_dpo/q_t": 0.49829238653182983, "grad_norm": 34.56472396850586, "learning_rate": 3.134328358208955e-07, "logits/chosen": 0.09799328446388245, "logits/rejected": 0.08677279949188232, "logps/chosen": -71.91377258300781, "logps/ref_chosen": -71.80902862548828, "logps/ref_rejected": -81.12464141845703, "logps/rejected": -81.29769897460938, "loss": 1.38, "margin_dpo/margin_mean": 0.06832125782966614, "margin_dpo/margin_std": 0.44653332233428955, "step": 43 }, { "KL/chosen_KL_mean": -0.14278602600097656, "KL/mean": -0.14451465010643005, "KL/rejected_KL_mean": -0.14624786376953125, "KL/std": 0.32325345277786255, "epoch": 0.06651549508692366, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0034592151641845703, "fcm_dpo/q_t": 0.4999151825904846, "grad_norm": 32.58658218383789, "learning_rate": 3.2089552238805965e-07, "logits/chosen": 0.0625898465514183, "logits/rejected": 0.031519316136837006, "logps/chosen": -66.6932144165039, "logps/ref_chosen": -66.55043029785156, "logps/ref_rejected": -85.06198120117188, "logps/rejected": -85.2082290649414, "loss": 1.3865, "margin_dpo/margin_mean": 0.003458946943283081, "margin_dpo/margin_std": 0.4451693296432495, "step": 44 }, { "KL/chosen_KL_mean": -0.10950851440429688, "KL/mean": -0.1634192317724228, "KL/rejected_KL_mean": -0.21733474731445312, "KL/std": 0.2610400915145874, "epoch": 0.06802721088435375, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10782626271247864, "fcm_dpo/q_t": 0.4973050355911255, "grad_norm": 31.791259765625, "learning_rate": 3.2835820895522385e-07, "logits/chosen": 0.11043379455804825, "logits/rejected": 0.05831673741340637, "logps/chosen": -62.353363037109375, "logps/ref_chosen": -62.24385452270508, "logps/ref_rejected": -92.96665954589844, "logps/rejected": -93.18399810791016, "loss": 1.3759, "margin_dpo/margin_mean": 0.10782578587532043, "margin_dpo/margin_std": 0.3529035449028015, "step": 45 }, { "KL/chosen_KL_mean": -0.10295486450195312, "KL/mean": -0.18219387531280518, "KL/rejected_KL_mean": -0.26143646240234375, "KL/std": 0.3079463541507721, "epoch": 0.06953892668178382, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15847516059875488, "fcm_dpo/q_t": 0.49603956937789917, "grad_norm": 31.26382827758789, "learning_rate": 3.3582089552238805e-07, "logits/chosen": 0.12944480776786804, "logits/rejected": 0.08334116637706757, "logps/chosen": -61.60186004638672, "logps/ref_chosen": -61.498905181884766, "logps/ref_rejected": -78.91172790527344, "logps/rejected": -79.17316436767578, "loss": 1.3709, "margin_dpo/margin_mean": 0.1584753394126892, "margin_dpo/margin_std": 0.4019874036312103, "step": 46 }, { "KL/chosen_KL_mean": -0.13724899291992188, "KL/mean": -0.19762343168258667, "KL/rejected_KL_mean": -0.2579994201660156, "KL/std": 0.29829901456832886, "epoch": 0.0710506424792139, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12074863910675049, "fcm_dpo/q_t": 0.49698197841644287, "grad_norm": 28.516429901123047, "learning_rate": 3.432835820895522e-07, "logits/chosen": 0.040763009339571, "logits/rejected": -0.0024642946664243937, "logps/chosen": -51.71559524536133, "logps/ref_chosen": -51.578346252441406, "logps/ref_rejected": -68.2215576171875, "logps/rejected": -68.47955322265625, "loss": 1.3748, "margin_dpo/margin_mean": 0.1207488477230072, "margin_dpo/margin_std": 0.4426974654197693, "step": 47 }, { "KL/chosen_KL_mean": -0.23521041870117188, "KL/mean": -0.26210901141166687, "KL/rejected_KL_mean": -0.28900718688964844, "KL/std": 0.29804930090904236, "epoch": 0.07256235827664399, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05379486083984375, "fcm_dpo/q_t": 0.49865633249282837, "grad_norm": 26.82840347290039, "learning_rate": 3.507462686567164e-07, "logits/chosen": 0.15432414412498474, "logits/rejected": 0.12441211938858032, "logps/chosen": -52.02886199951172, "logps/ref_chosen": -51.79365158081055, "logps/ref_rejected": -64.22503662109375, "logps/rejected": -64.51404571533203, "loss": 1.3813, "margin_dpo/margin_mean": 0.05379548668861389, "margin_dpo/margin_std": 0.4050604999065399, "step": 48 }, { "KL/chosen_KL_mean": -0.2191181182861328, "KL/mean": -0.2685723900794983, "KL/rejected_KL_mean": -0.318023681640625, "KL/std": 0.3182642459869385, "epoch": 0.07407407407407407, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09890323877334595, "fcm_dpo/q_t": 0.49752840399742126, "grad_norm": 27.29543113708496, "learning_rate": 3.5820895522388055e-07, "logits/chosen": 0.02532416582107544, "logits/rejected": 0.004013156518340111, "logps/chosen": -58.353721618652344, "logps/ref_chosen": -58.13460159301758, "logps/ref_rejected": -64.63206481933594, "logps/rejected": -64.95008850097656, "loss": 1.3769, "margin_dpo/margin_mean": 0.09890377521514893, "margin_dpo/margin_std": 0.45572179555892944, "step": 49 }, { "KL/chosen_KL_mean": -0.2713756561279297, "KL/mean": -0.33710426092147827, "KL/rejected_KL_mean": -0.40283203125, "KL/std": 0.3075346350669861, "epoch": 0.07558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13145622611045837, "fcm_dpo/q_t": 0.49671512842178345, "grad_norm": 27.953882217407227, "learning_rate": 3.6567164179104475e-07, "logits/chosen": 0.11536800861358643, "logits/rejected": 0.08536919206380844, "logps/chosen": -53.12781524658203, "logps/ref_chosen": -52.85643768310547, "logps/ref_rejected": -72.17460632324219, "logps/rejected": -72.57743835449219, "loss": 1.3735, "margin_dpo/margin_mean": 0.13145634531974792, "margin_dpo/margin_std": 0.3600861430168152, "step": 50 }, { "KL/chosen_KL_mean": -0.31192779541015625, "KL/mean": -0.411691814661026, "KL/rejected_KL_mean": -0.5114593505859375, "KL/std": 0.4130152761936188, "epoch": 0.07709750566893424, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19952988624572754, "fcm_dpo/q_t": 0.495017945766449, "grad_norm": 30.98239517211914, "learning_rate": 3.7313432835820895e-07, "logits/chosen": 0.09327873587608337, "logits/rejected": 0.06517933309078217, "logps/chosen": -63.96836853027344, "logps/ref_chosen": -63.65644073486328, "logps/ref_rejected": -86.13229370117188, "logps/rejected": -86.64375305175781, "loss": 1.3671, "margin_dpo/margin_mean": 0.19953015446662903, "margin_dpo/margin_std": 0.5144891738891602, "step": 51 }, { "KL/chosen_KL_mean": -0.3197002410888672, "KL/mean": -0.42734378576278687, "KL/rejected_KL_mean": -0.5349884033203125, "KL/std": 0.4443569481372833, "epoch": 0.07860922146636433, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2152886539697647, "fcm_dpo/q_t": 0.494623064994812, "grad_norm": 32.54359436035156, "learning_rate": 3.805970149253731e-07, "logits/chosen": 0.07348201423883438, "logits/rejected": 0.025112273171544075, "logps/chosen": -68.159912109375, "logps/ref_chosen": -67.8402099609375, "logps/ref_rejected": -96.97090911865234, "logps/rejected": -97.50589752197266, "loss": 1.3657, "margin_dpo/margin_mean": 0.21528802812099457, "margin_dpo/margin_std": 0.5663281679153442, "step": 52 }, { "KL/chosen_KL_mean": -0.3238105773925781, "KL/mean": -0.4126301109790802, "KL/rejected_KL_mean": -0.5014514923095703, "KL/std": 0.3923019766807556, "epoch": 0.0801209372637944, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17764323949813843, "fcm_dpo/q_t": 0.4955608546733856, "grad_norm": 27.431011199951172, "learning_rate": 3.880597014925373e-07, "logits/chosen": 0.09138220548629761, "logits/rejected": 0.08053071051836014, "logps/chosen": -57.20195007324219, "logps/ref_chosen": -56.87813949584961, "logps/ref_rejected": -60.75569152832031, "logps/rejected": -61.257144927978516, "loss": 1.3692, "margin_dpo/margin_mean": 0.17764326930046082, "margin_dpo/margin_std": 0.4564116597175598, "step": 53 }, { "KL/chosen_KL_mean": -0.3661918640136719, "KL/mean": -0.45817917585372925, "KL/rejected_KL_mean": -0.5501670837402344, "KL/std": 0.39255958795547485, "epoch": 0.08163265306122448, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18397372961044312, "fcm_dpo/q_t": 0.495402991771698, "grad_norm": 26.49314308166504, "learning_rate": 3.9552238805970144e-07, "logits/chosen": 0.061346061527729034, "logits/rejected": 0.045975200831890106, "logps/chosen": -47.63311004638672, "logps/ref_chosen": -47.26692199707031, "logps/ref_rejected": -62.19426727294922, "logps/rejected": -62.74443054199219, "loss": 1.3686, "margin_dpo/margin_mean": 0.1839737594127655, "margin_dpo/margin_std": 0.4734251797199249, "step": 54 }, { "KL/chosen_KL_mean": -0.3494873046875, "KL/mean": -0.4886450469493866, "KL/rejected_KL_mean": -0.6278076171875, "KL/std": 0.4888648986816406, "epoch": 0.08314436885865457, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.27831727266311646, "fcm_dpo/q_t": 0.4930543303489685, "grad_norm": 31.363264083862305, "learning_rate": 4.0298507462686564e-07, "logits/chosen": 0.02471158653497696, "logits/rejected": -0.050151094794273376, "logps/chosen": -50.67567825317383, "logps/ref_chosen": -50.32619094848633, "logps/ref_rejected": -92.44389343261719, "logps/rejected": -93.07170104980469, "loss": 1.3597, "margin_dpo/margin_mean": 0.2783171534538269, "margin_dpo/margin_std": 0.6392854452133179, "step": 55 }, { "KL/chosen_KL_mean": -0.38335418701171875, "KL/mean": -0.465893417596817, "KL/rejected_KL_mean": -0.5484275817871094, "KL/std": 0.49349987506866455, "epoch": 0.08465608465608465, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16507351398468018, "fcm_dpo/q_t": 0.49587583541870117, "grad_norm": 27.5733642578125, "learning_rate": 4.1044776119402984e-07, "logits/chosen": 0.1194012314081192, "logits/rejected": 0.09707341343164444, "logps/chosen": -57.150325775146484, "logps/ref_chosen": -56.766971588134766, "logps/ref_rejected": -66.30504608154297, "logps/rejected": -66.85346984863281, "loss": 1.371, "margin_dpo/margin_mean": 0.16507357358932495, "margin_dpo/margin_std": 0.6623785495758057, "step": 56 }, { "KL/chosen_KL_mean": -0.5016689300537109, "KL/mean": -0.6514174938201904, "KL/rejected_KL_mean": -0.8011627197265625, "KL/std": 0.5826833844184875, "epoch": 0.08616780045351474, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2994988262653351, "fcm_dpo/q_t": 0.49252432584762573, "grad_norm": 30.63138771057129, "learning_rate": 4.17910447761194e-07, "logits/chosen": 0.08919486403465271, "logits/rejected": 0.025800924748182297, "logps/chosen": -58.269412994384766, "logps/ref_chosen": -57.76774597167969, "logps/ref_rejected": -82.75698852539062, "logps/rejected": -83.55815124511719, "loss": 1.3578, "margin_dpo/margin_mean": 0.29949846863746643, "margin_dpo/margin_std": 0.6790944337844849, "step": 57 }, { "KL/chosen_KL_mean": -0.5987739562988281, "KL/mean": -0.6989187598228455, "KL/rejected_KL_mean": -0.7990646362304688, "KL/std": 0.6531132459640503, "epoch": 0.08767951625094482, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20029249787330627, "fcm_dpo/q_t": 0.4950439929962158, "grad_norm": 30.253463745117188, "learning_rate": 4.253731343283582e-07, "logits/chosen": 0.06011844053864479, "logits/rejected": 0.04454775154590607, "logps/chosen": -73.36286163330078, "logps/ref_chosen": -72.76408386230469, "logps/ref_rejected": -84.49275207519531, "logps/rejected": -85.29181671142578, "loss": 1.3688, "margin_dpo/margin_mean": 0.20029214024543762, "margin_dpo/margin_std": 0.9416682720184326, "step": 58 }, { "KL/chosen_KL_mean": -0.5233745574951172, "KL/mean": -0.6699809432029724, "KL/rejected_KL_mean": -0.8165874481201172, "KL/std": 0.6026298403739929, "epoch": 0.08919123204837491, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.29320916533470154, "fcm_dpo/q_t": 0.49269628524780273, "grad_norm": 26.622543334960938, "learning_rate": 4.3283582089552234e-07, "logits/chosen": 0.10866847634315491, "logits/rejected": 0.04470418766140938, "logps/chosen": -50.34415054321289, "logps/ref_chosen": -49.820777893066406, "logps/ref_rejected": -77.14368438720703, "logps/rejected": -77.96027374267578, "loss": 1.359, "margin_dpo/margin_mean": 0.2932102382183075, "margin_dpo/margin_std": 0.7765747308731079, "step": 59 }, { "KL/chosen_KL_mean": -0.6319961547851562, "KL/mean": -0.6532078981399536, "KL/rejected_KL_mean": -0.6744232177734375, "KL/std": 0.6227794885635376, "epoch": 0.09070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04242426156997681, "fcm_dpo/q_t": 0.498939573764801, "grad_norm": 29.584983825683594, "learning_rate": 4.4029850746268654e-07, "logits/chosen": 0.13120165467262268, "logits/rejected": 0.12976253032684326, "logps/chosen": -63.85676956176758, "logps/ref_chosen": -63.22477340698242, "logps/ref_rejected": -61.360477447509766, "logps/rejected": -62.0349006652832, "loss": 1.3836, "margin_dpo/margin_mean": 0.04242423176765442, "margin_dpo/margin_std": 0.7847387790679932, "step": 60 }, { "KL/chosen_KL_mean": -0.7138442993164062, "KL/mean": -0.7507110834121704, "KL/rejected_KL_mean": -0.7875823974609375, "KL/std": 0.709343433380127, "epoch": 0.09221466364323508, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07373163104057312, "fcm_dpo/q_t": 0.4981583058834076, "grad_norm": 27.950082778930664, "learning_rate": 4.4776119402985074e-07, "logits/chosen": 0.15023085474967957, "logits/rejected": 0.11694887280464172, "logps/chosen": -49.73064422607422, "logps/ref_chosen": -49.01679992675781, "logps/ref_rejected": -74.90817260742188, "logps/rejected": -75.69575500488281, "loss": 1.3814, "margin_dpo/margin_mean": 0.073731929063797, "margin_dpo/margin_std": 0.9730736017227173, "step": 61 }, { "KL/chosen_KL_mean": -0.7303638458251953, "KL/mean": -0.8440359830856323, "KL/rejected_KL_mean": -0.9577064514160156, "KL/std": 0.6483147144317627, "epoch": 0.09372637944066516, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22734490036964417, "fcm_dpo/q_t": 0.49432116746902466, "grad_norm": 28.671464920043945, "learning_rate": 4.552238805970149e-07, "logits/chosen": 0.09102478623390198, "logits/rejected": 0.05276797339320183, "logps/chosen": -63.48223114013672, "logps/ref_chosen": -62.751869201660156, "logps/ref_rejected": -78.93360900878906, "logps/rejected": -79.89131164550781, "loss": 1.3658, "margin_dpo/margin_mean": 0.22734498977661133, "margin_dpo/margin_std": 0.9036822319030762, "step": 62 }, { "KL/chosen_KL_mean": -0.5533409118652344, "KL/mean": -0.8310414552688599, "KL/rejected_KL_mean": -1.1087455749511719, "KL/std": 0.6519574522972107, "epoch": 0.09523809523809523, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5554065704345703, "fcm_dpo/q_t": 0.48614123463630676, "grad_norm": 31.55145263671875, "learning_rate": 4.626865671641791e-07, "logits/chosen": 0.20062537491321564, "logits/rejected": 0.17529505491256714, "logps/chosen": -61.06858825683594, "logps/ref_chosen": -60.51525115966797, "logps/ref_rejected": -85.11021423339844, "logps/rejected": -86.21896362304688, "loss": 1.3332, "margin_dpo/margin_mean": 0.5554060935974121, "margin_dpo/margin_std": 0.8089257478713989, "step": 63 }, { "KL/chosen_KL_mean": -0.8106708526611328, "KL/mean": -0.8460452556610107, "KL/rejected_KL_mean": -0.8814239501953125, "KL/std": 0.7306973338127136, "epoch": 0.09674981103552532, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0707455575466156, "fcm_dpo/q_t": 0.49822551012039185, "grad_norm": 26.139772415161133, "learning_rate": 4.701492537313433e-07, "logits/chosen": 0.11065845191478729, "logits/rejected": 0.0851249247789383, "logps/chosen": -52.01751708984375, "logps/ref_chosen": -51.20684814453125, "logps/ref_rejected": -66.93081665039062, "logps/rejected": -67.81224060058594, "loss": 1.3812, "margin_dpo/margin_mean": 0.07074528932571411, "margin_dpo/margin_std": 0.8756983876228333, "step": 64 }, { "KL/chosen_KL_mean": -0.8438053131103516, "KL/mean": -1.088397741317749, "KL/rejected_KL_mean": -1.3329925537109375, "KL/std": 0.892356276512146, "epoch": 0.0982615268329554, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4891848862171173, "fcm_dpo/q_t": 0.48782041668891907, "grad_norm": 30.766826629638672, "learning_rate": 4.776119402985074e-07, "logits/chosen": 0.17410069704055786, "logits/rejected": 0.14507675170898438, "logps/chosen": -68.13249969482422, "logps/ref_chosen": -67.2886962890625, "logps/ref_rejected": -74.44281005859375, "logps/rejected": -75.77580261230469, "loss": 1.3414, "margin_dpo/margin_mean": 0.48918458819389343, "margin_dpo/margin_std": 1.1623709201812744, "step": 65 }, { "KL/chosen_KL_mean": -0.920684814453125, "KL/mean": -1.086906909942627, "KL/rejected_KL_mean": -1.2531318664550781, "KL/std": 0.839600682258606, "epoch": 0.09977324263038549, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.33244842290878296, "fcm_dpo/q_t": 0.4917258024215698, "grad_norm": 29.311410903930664, "learning_rate": 4.850746268656717e-07, "logits/chosen": 0.11044877022504807, "logits/rejected": 0.08624343574047089, "logps/chosen": -71.66409301757812, "logps/ref_chosen": -70.743408203125, "logps/ref_rejected": -77.26499938964844, "logps/rejected": -78.51813507080078, "loss": 1.3569, "margin_dpo/margin_mean": 0.332448273897171, "margin_dpo/margin_std": 1.181469202041626, "step": 66 }, { "KL/chosen_KL_mean": -0.7882022857666016, "KL/mean": -0.9426577091217041, "KL/rejected_KL_mean": -1.0971145629882812, "KL/std": 0.7659956812858582, "epoch": 0.10128495842781557, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3089101016521454, "fcm_dpo/q_t": 0.4923003911972046, "grad_norm": 29.101470947265625, "learning_rate": 4.925373134328357e-07, "logits/chosen": 0.07350900769233704, "logits/rejected": 0.018813468515872955, "logps/chosen": -61.390804290771484, "logps/ref_chosen": -60.60260009765625, "logps/ref_rejected": -75.22235870361328, "logps/rejected": -76.31947326660156, "loss": 1.358, "margin_dpo/margin_mean": 0.3089095652103424, "margin_dpo/margin_std": 0.9819750189781189, "step": 67 }, { "KL/chosen_KL_mean": -1.2221183776855469, "KL/mean": -1.4071699380874634, "KL/rejected_KL_mean": -1.5922203063964844, "KL/std": 0.9413119554519653, "epoch": 0.10279667422524566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.37010324001312256, "fcm_dpo/q_t": 0.4907914996147156, "grad_norm": 31.142738342285156, "learning_rate": 5e-07, "logits/chosen": 0.07065162807703018, "logits/rejected": 0.04052513837814331, "logps/chosen": -78.75048828125, "logps/ref_chosen": -77.52836608886719, "logps/ref_rejected": -93.17778015136719, "logps/rejected": -94.7699966430664, "loss": 1.3535, "margin_dpo/margin_mean": 0.3701035976409912, "margin_dpo/margin_std": 1.222214698791504, "step": 68 }, { "KL/chosen_KL_mean": -1.1215229034423828, "KL/mean": -1.4538966417312622, "KL/rejected_KL_mean": -1.7862663269042969, "KL/std": 1.0045855045318604, "epoch": 0.10430839002267574, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6647469997406006, "fcm_dpo/q_t": 0.4834825098514557, "grad_norm": 30.769756317138672, "learning_rate": 4.999965034812934e-07, "logits/chosen": 0.11138215661048889, "logits/rejected": 0.0678541287779808, "logps/chosen": -67.0645751953125, "logps/ref_chosen": -65.94305419921875, "logps/ref_rejected": -89.7735595703125, "logps/rejected": -91.55982971191406, "loss": 1.3248, "margin_dpo/margin_mean": 0.6647467613220215, "margin_dpo/margin_std": 1.2122726440429688, "step": 69 }, { "KL/chosen_KL_mean": -1.1970710754394531, "KL/mean": -1.3843632936477661, "KL/rejected_KL_mean": -1.5716571807861328, "KL/std": 0.9146664142608643, "epoch": 0.10582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3745828866958618, "fcm_dpo/q_t": 0.4906681180000305, "grad_norm": 28.702402114868164, "learning_rate": 4.999860140229787e-07, "logits/chosen": 0.15375293791294098, "logits/rejected": 0.13041463494300842, "logps/chosen": -63.15498352050781, "logps/ref_chosen": -61.95791244506836, "logps/ref_rejected": -75.80945587158203, "logps/rejected": -77.38111877441406, "loss": 1.353, "margin_dpo/margin_mean": 0.3745833933353424, "margin_dpo/margin_std": 1.2272173166275024, "step": 70 }, { "KL/chosen_KL_mean": -1.423257827758789, "KL/mean": -1.5462465286254883, "KL/rejected_KL_mean": -1.669240951538086, "KL/std": 1.0030606985092163, "epoch": 0.1073318216175359, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24598197638988495, "fcm_dpo/q_t": 0.4938420355319977, "grad_norm": 28.510639190673828, "learning_rate": 4.999685319184688e-07, "logits/chosen": 0.10399076342582703, "logits/rejected": 0.0884547010064125, "logps/chosen": -64.77083587646484, "logps/ref_chosen": -63.34757995605469, "logps/ref_rejected": -67.49658203125, "logps/rejected": -69.16581726074219, "loss": 1.367, "margin_dpo/margin_mean": 0.24598146975040436, "margin_dpo/margin_std": 1.4251511096954346, "step": 71 }, { "KL/chosen_KL_mean": -1.1918869018554688, "KL/mean": -1.5472865104675293, "KL/rejected_KL_mean": -1.9026832580566406, "KL/std": 1.048713207244873, "epoch": 0.10884353741496598, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7107953429222107, "fcm_dpo/q_t": 0.4823157489299774, "grad_norm": 30.407228469848633, "learning_rate": 4.999440576567755e-07, "logits/chosen": 0.11062829196453094, "logits/rejected": 0.04809199273586273, "logps/chosen": -57.051185607910156, "logps/ref_chosen": -55.85929870605469, "logps/ref_rejected": -68.45423889160156, "logps/rejected": -70.35692596435547, "loss": 1.3207, "margin_dpo/margin_mean": 0.7107953429222107, "margin_dpo/margin_std": 1.270460605621338, "step": 72 }, { "KL/chosen_KL_mean": -1.8556289672851562, "KL/mean": -1.8964017629623413, "KL/rejected_KL_mean": -1.9371776580810547, "KL/std": 1.2240742444992065, "epoch": 0.11035525321239607, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08155509829521179, "fcm_dpo/q_t": 0.4981040060520172, "grad_norm": 32.02695846557617, "learning_rate": 4.999125919224965e-07, "logits/chosen": 0.10665452480316162, "logits/rejected": 0.09263806790113449, "logps/chosen": -70.99443054199219, "logps/ref_chosen": -69.13880920410156, "logps/ref_rejected": -79.04586791992188, "logps/rejected": -80.98304748535156, "loss": 1.3849, "margin_dpo/margin_mean": 0.08155572414398193, "margin_dpo/margin_std": 1.5625742673873901, "step": 73 }, { "KL/chosen_KL_mean": -1.274038314819336, "KL/mean": -1.612091064453125, "KL/rejected_KL_mean": -1.9501457214355469, "KL/std": 1.0038673877716064, "epoch": 0.11186696900982615, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6761044859886169, "fcm_dpo/q_t": 0.4832031726837158, "grad_norm": 27.87704086303711, "learning_rate": 4.998741355957963e-07, "logits/chosen": 0.16232308745384216, "logits/rejected": 0.10681352019309998, "logps/chosen": -51.197776794433594, "logps/ref_chosen": -49.923736572265625, "logps/ref_rejected": -81.73213958740234, "logps/rejected": -83.68228149414062, "loss": 1.3242, "margin_dpo/margin_mean": 0.6761043071746826, "margin_dpo/margin_std": 1.2969073057174683, "step": 74 }, { "KL/chosen_KL_mean": -1.4171257019042969, "KL/mean": -1.7749662399291992, "KL/rejected_KL_mean": -2.132802963256836, "KL/std": 1.1096692085266113, "epoch": 0.11337868480725624, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7156823873519897, "fcm_dpo/q_t": 0.48222440481185913, "grad_norm": 26.229402542114258, "learning_rate": 4.998286897523808e-07, "logits/chosen": 0.13490338623523712, "logits/rejected": 0.10130466520786285, "logps/chosen": -47.485877990722656, "logps/ref_chosen": -46.06875228881836, "logps/ref_rejected": -66.1181411743164, "logps/rejected": -68.25094604492188, "loss": 1.3206, "margin_dpo/margin_mean": 0.7156822681427002, "margin_dpo/margin_std": 1.358628749847412, "step": 75 }, { "KL/chosen_KL_mean": -1.6028976440429688, "KL/mean": -1.7485299110412598, "KL/rejected_KL_mean": -1.8941612243652344, "KL/std": 1.1897375583648682, "epoch": 0.11489040060468632, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.29126521944999695, "fcm_dpo/q_t": 0.4927474558353424, "grad_norm": 29.370731353759766, "learning_rate": 4.997762556634679e-07, "logits/chosen": 0.13600164651870728, "logits/rejected": 0.09152773022651672, "logps/chosen": -55.6656494140625, "logps/ref_chosen": -54.06275177001953, "logps/ref_rejected": -74.87464141845703, "logps/rejected": -76.76880645751953, "loss": 1.3633, "margin_dpo/margin_mean": 0.2912653684616089, "margin_dpo/margin_std": 1.5319194793701172, "step": 76 }, { "KL/chosen_KL_mean": -1.63665771484375, "KL/mean": -2.04000186920166, "KL/rejected_KL_mean": -2.4433441162109375, "KL/std": 1.240219235420227, "epoch": 0.1164021164021164, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8066871166229248, "fcm_dpo/q_t": 0.4799628257751465, "grad_norm": 29.07001495361328, "learning_rate": 4.99716834795752e-07, "logits/chosen": 0.14318928122520447, "logits/rejected": 0.10253836214542389, "logps/chosen": -54.71274948120117, "logps/ref_chosen": -53.07609176635742, "logps/ref_rejected": -74.45601654052734, "logps/rejected": -76.89936065673828, "loss": 1.3128, "margin_dpo/margin_mean": 0.8066874742507935, "margin_dpo/margin_std": 1.4628499746322632, "step": 77 }, { "KL/chosen_KL_mean": -1.8630790710449219, "KL/mean": -2.0672872066497803, "KL/rejected_KL_mean": -2.271495819091797, "KL/std": 1.271705150604248, "epoch": 0.11791383219954649, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40840888023376465, "fcm_dpo/q_t": 0.4898289144039154, "grad_norm": 29.537355422973633, "learning_rate": 4.996504288113623e-07, "logits/chosen": 0.13026684522628784, "logits/rejected": 0.10955053567886353, "logps/chosen": -69.5885009765625, "logps/ref_chosen": -67.72541809082031, "logps/ref_rejected": -79.03926849365234, "logps/rejected": -81.31076049804688, "loss": 1.351, "margin_dpo/margin_mean": 0.4084087610244751, "margin_dpo/margin_std": 1.4295289516448975, "step": 78 }, { "KL/chosen_KL_mean": -1.7345352172851562, "KL/mean": -2.2997708320617676, "KL/rejected_KL_mean": -2.8650054931640625, "KL/std": 1.5332820415496826, "epoch": 0.11942554799697656, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1304694414138794, "fcm_dpo/q_t": 0.4722587466239929, "grad_norm": 30.759552001953125, "learning_rate": 4.995770395678171e-07, "logits/chosen": 0.15435688197612762, "logits/rejected": 0.09562011808156967, "logps/chosen": -53.895179748535156, "logps/ref_chosen": -52.16064453125, "logps/ref_rejected": -83.31062316894531, "logps/rejected": -86.17562866210938, "loss": 1.2866, "margin_dpo/margin_mean": 1.1304690837860107, "margin_dpo/margin_std": 2.0189428329467773, "step": 79 }, { "KL/chosen_KL_mean": -2.062671661376953, "KL/mean": -2.3099541664123535, "KL/rejected_KL_mean": -2.5572357177734375, "KL/std": 1.3525840044021606, "epoch": 0.12093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4945679008960724, "fcm_dpo/q_t": 0.4877622127532959, "grad_norm": 28.568126678466797, "learning_rate": 4.994966691179711e-07, "logits/chosen": 0.1576176881790161, "logits/rejected": 0.09683984518051147, "logps/chosen": -63.47323226928711, "logps/ref_chosen": -61.410560607910156, "logps/ref_rejected": -78.66004943847656, "logps/rejected": -81.21728515625, "loss": 1.3467, "margin_dpo/margin_mean": 0.4945681691169739, "margin_dpo/margin_std": 1.8858641386032104, "step": 80 }, { "KL/chosen_KL_mean": -2.1507835388183594, "KL/mean": -2.569912910461426, "KL/rejected_KL_mean": -2.9890403747558594, "KL/std": 1.4435718059539795, "epoch": 0.12244897959183673, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8382568955421448, "fcm_dpo/q_t": 0.47932901978492737, "grad_norm": 29.092945098876953, "learning_rate": 4.994093197099587e-07, "logits/chosen": 0.12218126654624939, "logits/rejected": 0.08859767019748688, "logps/chosen": -65.95515441894531, "logps/ref_chosen": -63.80437088012695, "logps/ref_rejected": -79.3484115600586, "logps/rejected": -82.33744812011719, "loss": 1.3118, "margin_dpo/margin_mean": 0.8382560610771179, "margin_dpo/margin_std": 1.6820642948150635, "step": 81 }, { "KL/chosen_KL_mean": -1.7358322143554688, "KL/mean": -2.2893571853637695, "KL/rejected_KL_mean": -2.842884063720703, "KL/std": 1.3926236629486084, "epoch": 0.12396069538926682, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1070516109466553, "fcm_dpo/q_t": 0.4725184440612793, "grad_norm": 26.486122131347656, "learning_rate": 4.993149937871306e-07, "logits/chosen": 0.12228363752365112, "logits/rejected": 0.058549731969833374, "logps/chosen": -50.55372619628906, "logps/ref_chosen": -48.817893981933594, "logps/ref_rejected": -70.31497955322266, "logps/rejected": -73.15786743164062, "loss": 1.2846, "margin_dpo/margin_mean": 1.107051134109497, "margin_dpo/margin_std": 1.5328946113586426, "step": 82 }, { "KL/chosen_KL_mean": -2.0348987579345703, "KL/mean": -2.634671211242676, "KL/rejected_KL_mean": -3.2344436645507812, "KL/std": 1.4608440399169922, "epoch": 0.1254724111866969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.199547529220581, "fcm_dpo/q_t": 0.4703766703605652, "grad_norm": 29.12187385559082, "learning_rate": 4.992136939879856e-07, "logits/chosen": 0.18589824438095093, "logits/rejected": 0.13650354743003845, "logps/chosen": -59.185672760009766, "logps/ref_chosen": -57.15077209472656, "logps/ref_rejected": -75.1710205078125, "logps/rejected": -78.40547180175781, "loss": 1.278, "margin_dpo/margin_mean": 1.1995470523834229, "margin_dpo/margin_std": 1.78159499168396, "step": 83 }, { "KL/chosen_KL_mean": -2.5271358489990234, "KL/mean": -2.9374001026153564, "KL/rejected_KL_mean": -3.347667694091797, "KL/std": 1.8282861709594727, "epoch": 0.12698412698412698, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8205302953720093, "fcm_dpo/q_t": 0.47971442341804504, "grad_norm": 30.673738479614258, "learning_rate": 4.991054231460969e-07, "logits/chosen": 0.13898839056491852, "logits/rejected": 0.09928076714277267, "logps/chosen": -67.30443572998047, "logps/ref_chosen": -64.77729797363281, "logps/ref_rejected": -84.71949768066406, "logps/rejected": -88.06716918945312, "loss": 1.3149, "margin_dpo/margin_mean": 0.820530116558075, "margin_dpo/margin_std": 1.8985121250152588, "step": 84 }, { "KL/chosen_KL_mean": -2.2237377166748047, "KL/mean": -2.8801236152648926, "KL/rejected_KL_mean": -3.536510467529297, "KL/std": 1.7785743474960327, "epoch": 0.12849584278155707, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3127713203430176, "fcm_dpo/q_t": 0.46756821870803833, "grad_norm": 26.659639358520508, "learning_rate": 4.989901842900325e-07, "logits/chosen": 0.1752845197916031, "logits/rejected": 0.13079001009464264, "logps/chosen": -52.475433349609375, "logps/ref_chosen": -50.25169372558594, "logps/ref_rejected": -66.55439758300781, "logps/rejected": -70.09090423583984, "loss": 1.2692, "margin_dpo/margin_mean": 1.3127710819244385, "margin_dpo/margin_std": 1.9418845176696777, "step": 85 }, { "KL/chosen_KL_mean": -2.8079910278320312, "KL/mean": -3.3052587509155273, "KL/rejected_KL_mean": -3.8025245666503906, "KL/std": 1.901952862739563, "epoch": 0.13000755857898716, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9945327639579773, "fcm_dpo/q_t": 0.4754181504249573, "grad_norm": 27.24299430847168, "learning_rate": 4.988679806432711e-07, "logits/chosen": 0.20016081631183624, "logits/rejected": 0.18127304315567017, "logps/chosen": -63.53717041015625, "logps/ref_chosen": -60.72917938232422, "logps/ref_rejected": -72.30961608886719, "logps/rejected": -76.11213684082031, "loss": 1.2994, "margin_dpo/margin_mean": 0.9945329427719116, "margin_dpo/margin_std": 2.011247396469116, "step": 86 }, { "KL/chosen_KL_mean": -3.0212860107421875, "KL/mean": -3.5553457736968994, "KL/rejected_KL_mean": -4.0894012451171875, "KL/std": 2.0620341300964355, "epoch": 0.13151927437641722, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.068122148513794, "fcm_dpo/q_t": 0.4736815392971039, "grad_norm": 30.50346565246582, "learning_rate": 4.987388156241114e-07, "logits/chosen": 0.1732024997472763, "logits/rejected": 0.1137414500117302, "logps/chosen": -68.77925109863281, "logps/ref_chosen": -65.75796508789062, "logps/ref_rejected": -84.81159973144531, "logps/rejected": -88.9010009765625, "loss": 1.2969, "margin_dpo/margin_mean": 1.0681220293045044, "margin_dpo/margin_std": 2.4039626121520996, "step": 87 }, { "KL/chosen_KL_mean": -3.031696319580078, "KL/mean": -3.444826126098633, "KL/rejected_KL_mean": -3.857952117919922, "KL/std": 2.2947521209716797, "epoch": 0.1330309901738473, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8262563347816467, "fcm_dpo/q_t": 0.47979599237442017, "grad_norm": 30.31769561767578, "learning_rate": 4.986026928455767e-07, "logits/chosen": 0.20679256319999695, "logits/rejected": 0.1803770810365677, "logps/chosen": -65.85572814941406, "logps/ref_chosen": -62.82402801513672, "logps/ref_rejected": -74.9607162475586, "logps/rejected": -78.81866455078125, "loss": 1.3276, "margin_dpo/margin_mean": 0.8262564539909363, "margin_dpo/margin_std": 2.897235870361328, "step": 88 }, { "KL/chosen_KL_mean": -2.560823440551758, "KL/mean": -3.250973701477051, "KL/rejected_KL_mean": -3.9411239624023438, "KL/std": 2.1574809551239014, "epoch": 0.1345427059712774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3802971839904785, "fcm_dpo/q_t": 0.46636801958084106, "grad_norm": 29.29635238647461, "learning_rate": 4.984596161153135e-07, "logits/chosen": 0.2754148244857788, "logits/rejected": 0.1902625858783722, "logps/chosen": -43.75225830078125, "logps/ref_chosen": -41.191436767578125, "logps/ref_rejected": -85.44769287109375, "logps/rejected": -89.38882446289062, "loss": 1.2696, "margin_dpo/margin_mean": 1.3802977800369263, "margin_dpo/margin_std": 2.5125932693481445, "step": 89 }, { "KL/chosen_KL_mean": -3.1273326873779297, "KL/mean": -3.7636611461639404, "KL/rejected_KL_mean": -4.399990081787109, "KL/std": 2.468470335006714, "epoch": 0.1360544217687075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.27265465259552, "fcm_dpo/q_t": 0.4687526822090149, "grad_norm": 29.193382263183594, "learning_rate": 4.983095894354857e-07, "logits/chosen": 0.14260095357894897, "logits/rejected": 0.08857794851064682, "logps/chosen": -59.71124267578125, "logps/ref_chosen": -56.58390808105469, "logps/ref_rejected": -86.86978149414062, "logps/rejected": -91.269775390625, "loss": 1.2791, "margin_dpo/margin_mean": 1.272654414176941, "margin_dpo/margin_std": 2.4919843673706055, "step": 90 }, { "KL/chosen_KL_mean": -3.0993270874023438, "KL/mean": -3.6860151290893555, "KL/rejected_KL_mean": -4.272705078125, "KL/std": 2.293879985809326, "epoch": 0.13756613756613756, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.173375129699707, "fcm_dpo/q_t": 0.4715588688850403, "grad_norm": 25.667625427246094, "learning_rate": 4.98152617002662e-07, "logits/chosen": 0.15594482421875, "logits/rejected": 0.11233173310756683, "logps/chosen": -55.48167037963867, "logps/ref_chosen": -52.38234329223633, "logps/ref_rejected": -72.17642211914062, "logps/rejected": -76.44912719726562, "loss": 1.2915, "margin_dpo/margin_mean": 1.173375129699707, "margin_dpo/margin_std": 2.77371883392334, "step": 91 }, { "KL/chosen_KL_mean": -3.2288036346435547, "KL/mean": -4.0197906494140625, "KL/rejected_KL_mean": -4.810783386230469, "KL/std": 2.6627650260925293, "epoch": 0.13907785336356765, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5819768905639648, "fcm_dpo/q_t": 0.46180659532546997, "grad_norm": 26.956077575683594, "learning_rate": 4.979887032076988e-07, "logits/chosen": 0.19724005460739136, "logits/rejected": 0.15849006175994873, "logps/chosen": -56.23750305175781, "logps/ref_chosen": -53.00870132446289, "logps/ref_rejected": -79.77812957763672, "logps/rejected": -84.58891296386719, "loss": 1.2584, "margin_dpo/margin_mean": 1.5819770097732544, "margin_dpo/margin_std": 3.115145206451416, "step": 92 }, { "KL/chosen_KL_mean": -3.5571727752685547, "KL/mean": -4.073748588562012, "KL/rejected_KL_mean": -4.590320587158203, "KL/std": 2.6155309677124023, "epoch": 0.14058956916099774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0331499576568604, "fcm_dpo/q_t": 0.4753725230693817, "grad_norm": 24.509965896606445, "learning_rate": 4.978178526356172e-07, "logits/chosen": 0.19289353489875793, "logits/rejected": 0.16430367529392242, "logps/chosen": -48.46422576904297, "logps/ref_chosen": -44.90705108642578, "logps/ref_rejected": -58.7879524230957, "logps/rejected": -63.378273010253906, "loss": 1.3113, "margin_dpo/margin_mean": 1.03315007686615, "margin_dpo/margin_std": 3.2502970695495605, "step": 93 }, { "KL/chosen_KL_mean": -3.304903030395508, "KL/mean": -4.201479434967041, "KL/rejected_KL_mean": -5.098056793212891, "KL/std": 3.1756069660186768, "epoch": 0.1421012849584278, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.793152928352356, "fcm_dpo/q_t": 0.4567800760269165, "grad_norm": 27.731033325195312, "learning_rate": 4.976400700654751e-07, "logits/chosen": 0.26267024874687195, "logits/rejected": 0.2218886911869049, "logps/chosen": -63.24267578125, "logps/ref_chosen": -59.93777084350586, "logps/ref_rejected": -79.3138427734375, "logps/rejected": -84.41189575195312, "loss": 1.2533, "margin_dpo/margin_mean": 1.7931523323059082, "margin_dpo/margin_std": 3.9416329860687256, "step": 94 }, { "KL/chosen_KL_mean": -4.260015487670898, "KL/mean": -5.063937187194824, "KL/rejected_KL_mean": -5.86785888671875, "KL/std": 2.690230131149292, "epoch": 0.1436130007558579, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.607844591140747, "fcm_dpo/q_t": 0.46108755469322205, "grad_norm": 29.20717430114746, "learning_rate": 4.974553604702332e-07, "logits/chosen": 0.14258426427841187, "logits/rejected": 0.0792468786239624, "logps/chosen": -64.42850494384766, "logps/ref_chosen": -60.168487548828125, "logps/ref_rejected": -90.73665618896484, "logps/rejected": -96.6045150756836, "loss": 1.2598, "margin_dpo/margin_mean": 1.6078438758850098, "margin_dpo/margin_std": 3.364819049835205, "step": 95 }, { "KL/chosen_KL_mean": -4.149753570556641, "KL/mean": -5.083588123321533, "KL/rejected_KL_mean": -6.017417907714844, "KL/std": 3.411038398742676, "epoch": 0.14512471655328799, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8676685094833374, "fcm_dpo/q_t": 0.45662426948547363, "grad_norm": 27.411863327026367, "learning_rate": 4.972637290166157e-07, "logits/chosen": 0.18811967968940735, "logits/rejected": 0.1457143872976303, "logps/chosen": -64.81852722167969, "logps/ref_chosen": -60.66877746582031, "logps/ref_rejected": -88.30673217773438, "logps/rejected": -94.32415008544922, "loss": 1.2587, "margin_dpo/margin_mean": 1.8676682710647583, "margin_dpo/margin_std": 4.52335786819458, "step": 96 }, { "KL/chosen_KL_mean": -5.019113540649414, "KL/mean": -5.533176422119141, "KL/rejected_KL_mean": -6.0472412109375, "KL/std": 3.6687517166137695, "epoch": 0.14663643235071808, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0281330347061157, "fcm_dpo/q_t": 0.475546658039093, "grad_norm": 36.294029235839844, "learning_rate": 4.970651810649666e-07, "logits/chosen": 0.15061883628368378, "logits/rejected": 0.10639595985412598, "logps/chosen": -70.063232421875, "logps/ref_chosen": -65.04412078857422, "logps/ref_rejected": -78.42092895507812, "logps/rejected": -84.46817016601562, "loss": 1.347, "margin_dpo/margin_mean": 1.0281330347061157, "margin_dpo/margin_std": 5.024996280670166, "step": 97 }, { "KL/chosen_KL_mean": -4.630523681640625, "KL/mean": -4.976384162902832, "KL/rejected_KL_mean": -5.322246551513672, "KL/std": 3.221841812133789, "epoch": 0.14814814814814814, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6917204856872559, "fcm_dpo/q_t": 0.482790470123291, "grad_norm": 30.246551513671875, "learning_rate": 4.968597221690985e-07, "logits/chosen": 0.2132185399532318, "logits/rejected": 0.186471626162529, "logps/chosen": -60.13375473022461, "logps/ref_chosen": -55.503231048583984, "logps/ref_rejected": -72.81553649902344, "logps/rejected": -78.13778686523438, "loss": 1.3441, "margin_dpo/margin_mean": 0.6917206048965454, "margin_dpo/margin_std": 3.1939637660980225, "step": 98 }, { "KL/chosen_KL_mean": -4.680410385131836, "KL/mean": -5.174110412597656, "KL/rejected_KL_mean": -5.667812347412109, "KL/std": 3.500779151916504, "epoch": 0.14965986394557823, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9873988628387451, "fcm_dpo/q_t": 0.4767622947692871, "grad_norm": 32.56043243408203, "learning_rate": 4.966473580761389e-07, "logits/chosen": 0.24544592201709747, "logits/rejected": 0.2058737725019455, "logps/chosen": -63.25605010986328, "logps/ref_chosen": -58.57563781738281, "logps/ref_rejected": -78.693603515625, "logps/rejected": -84.36141967773438, "loss": 1.3409, "margin_dpo/margin_mean": 0.9873991012573242, "margin_dpo/margin_std": 4.55812931060791, "step": 99 }, { "KL/chosen_KL_mean": -5.287479400634766, "KL/mean": -6.03157901763916, "KL/rejected_KL_mean": -6.775676727294922, "KL/std": 4.144209384918213, "epoch": 0.15117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4881947040557861, "fcm_dpo/q_t": 0.4669296145439148, "grad_norm": 33.02346420288086, "learning_rate": 4.964280947263676e-07, "logits/chosen": 0.2219209372997284, "logits/rejected": 0.21426787972450256, "logps/chosen": -84.87091064453125, "logps/ref_chosen": -79.58343505859375, "logps/ref_rejected": -92.152587890625, "logps/rejected": -98.92826080322266, "loss": 1.3292, "margin_dpo/margin_mean": 1.4881953001022339, "margin_dpo/margin_std": 5.861654281616211, "step": 100 }, { "KL/chosen_KL_mean": -4.301380157470703, "KL/mean": -5.327387809753418, "KL/rejected_KL_mean": -6.353397369384766, "KL/std": 3.664494514465332, "epoch": 0.15268329554043839, "fcm_dpo/beta": 0.10289128869771957, "fcm_dpo/delta": 0.28107818961143494, "fcm_dpo/margin": 2.0520172119140625, "fcm_dpo/q_t": 0.45036324858665466, "grad_norm": 26.742868423461914, "learning_rate": 4.96201938253052e-07, "logits/chosen": 0.25061866641044617, "logits/rejected": 0.21055373549461365, "logps/chosen": -56.6341667175293, "logps/ref_chosen": -52.332786560058594, "logps/ref_rejected": -69.55589294433594, "logps/rejected": -75.90929412841797, "loss": 1.2325, "margin_dpo/margin_mean": 2.0520172119140625, "margin_dpo/margin_std": 4.156118869781494, "step": 101 }, { "KL/chosen_KL_mean": -5.317193984985352, "KL/mean": -5.750248908996582, "KL/rejected_KL_mean": -6.183307647705078, "KL/std": 3.721498966217041, "epoch": 0.15419501133786848, "fcm_dpo/beta": 0.10578257590532303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8661136627197266, "fcm_dpo/q_t": 0.4816773533821106, "grad_norm": 34.89421081542969, "learning_rate": 4.959688949822748e-07, "logits/chosen": 0.1457417905330658, "logits/rejected": 0.1073441207408905, "logps/chosen": -70.06067657470703, "logps/ref_chosen": -64.74348449707031, "logps/ref_rejected": -69.06132507324219, "logps/rejected": -75.24463653564453, "loss": 1.375, "margin_dpo/margin_mean": 0.8661136627197266, "margin_dpo/margin_std": 5.487614154815674, "step": 102 }, { "KL/chosen_KL_mean": -4.824554443359375, "KL/mean": -5.601681709289551, "KL/rejected_KL_mean": -6.378810882568359, "KL/std": 3.8675785064697266, "epoch": 0.15570672713529857, "fcm_dpo/beta": 0.10896602272987366, "fcm_dpo/delta": 0.2922336161136627, "fcm_dpo/margin": 1.5542576313018799, "fcm_dpo/q_t": 0.46070000529289246, "grad_norm": 31.94975471496582, "learning_rate": 4.957289714327572e-07, "logits/chosen": 0.2437642365694046, "logits/rejected": 0.21064837276935577, "logps/chosen": -68.66119384765625, "logps/ref_chosen": -63.83664321899414, "logps/ref_rejected": -79.32362365722656, "logps/rejected": -85.70243835449219, "loss": 1.2614, "margin_dpo/margin_mean": 1.5542573928833008, "margin_dpo/margin_std": 3.6246559619903564, "step": 103 }, { "KL/chosen_KL_mean": -4.671621322631836, "KL/mean": -5.832412242889404, "KL/rejected_KL_mean": -6.9932098388671875, "KL/std": 4.947868347167969, "epoch": 0.15721844293272866, "fcm_dpo/beta": 0.11496137082576752, "fcm_dpo/delta": 0.24464420974254608, "fcm_dpo/margin": 2.3215885162353516, "fcm_dpo/q_t": 0.445722758769989, "grad_norm": 35.56681442260742, "learning_rate": 4.954821743156767e-07, "logits/chosen": 0.2890929579734802, "logits/rejected": 0.19809794425964355, "logps/chosen": -65.67082977294922, "logps/ref_chosen": -60.99920654296875, "logps/ref_rejected": -98.84645080566406, "logps/rejected": -105.83966064453125, "loss": 1.2572, "margin_dpo/margin_mean": 2.3215882778167725, "margin_dpo/margin_std": 6.310924530029297, "step": 104 }, { "KL/chosen_KL_mean": -5.358615875244141, "KL/mean": -6.24442195892334, "KL/rejected_KL_mean": -7.130226135253906, "KL/std": 3.9761853218078613, "epoch": 0.15873015873015872, "fcm_dpo/beta": 0.11777327209711075, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7716116905212402, "fcm_dpo/q_t": 0.45355188846588135, "grad_norm": 36.21678161621094, "learning_rate": 4.952285105344791e-07, "logits/chosen": 0.20373253524303436, "logits/rejected": 0.14855940639972687, "logps/chosen": -76.30888366699219, "logps/ref_chosen": -70.95027160644531, "logps/ref_rejected": -87.88340759277344, "logps/rejected": -95.01364135742188, "loss": 1.2679, "margin_dpo/margin_mean": 1.7716121673583984, "margin_dpo/margin_std": 4.9041643142700195, "step": 105 }, { "KL/chosen_KL_mean": -5.010364532470703, "KL/mean": -5.787876129150391, "KL/rejected_KL_mean": -6.5653839111328125, "KL/std": 3.9396629333496094, "epoch": 0.1602418745275888, "fcm_dpo/beta": 0.11777327209711075, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5550204515457153, "fcm_dpo/q_t": 0.4568718373775482, "grad_norm": 37.559017181396484, "learning_rate": 4.949679871846857e-07, "logits/chosen": 0.22271032631397247, "logits/rejected": 0.20895954966545105, "logps/chosen": -67.46969604492188, "logps/ref_chosen": -62.45933151245117, "logps/ref_rejected": -67.00595092773438, "logps/rejected": -73.57133483886719, "loss": 1.3045, "margin_dpo/margin_mean": 1.5550203323364258, "margin_dpo/margin_std": 5.320964813232422, "step": 106 }, { "KL/chosen_KL_mean": -6.474697113037109, "KL/mean": -6.880711555480957, "KL/rejected_KL_mean": -7.2867279052734375, "KL/std": 4.923356056213379, "epoch": 0.1617535903250189, "fcm_dpo/beta": 0.12500976026058197, "fcm_dpo/delta": 0.2981524169445038, "fcm_dpo/margin": 0.8120315074920654, "fcm_dpo/q_t": 0.47507673501968384, "grad_norm": 55.240623474121094, "learning_rate": 4.947006115536947e-07, "logits/chosen": 0.183563694357872, "logits/rejected": 0.1593797206878662, "logps/chosen": -82.31266784667969, "logps/ref_chosen": -75.83796691894531, "logps/ref_rejected": -87.74038696289062, "logps/rejected": -95.02711486816406, "loss": 1.4266, "margin_dpo/margin_mean": 0.8120315670967102, "margin_dpo/margin_std": 6.172867774963379, "step": 107 }, { "KL/chosen_KL_mean": -5.433837890625, "KL/mean": -6.352394104003906, "KL/rejected_KL_mean": -7.2709503173828125, "KL/std": 4.04887580871582, "epoch": 0.16326530612244897, "fcm_dpo/beta": 0.1361444890499115, "fcm_dpo/delta": 0.5635470151901245, "fcm_dpo/margin": 1.8371151685714722, "fcm_dpo/q_t": 0.4484684467315674, "grad_norm": 40.03304672241211, "learning_rate": 4.944263911205772e-07, "logits/chosen": 0.1733136624097824, "logits/rejected": 0.1424228549003601, "logps/chosen": -73.82707214355469, "logps/ref_chosen": -68.39323425292969, "logps/ref_rejected": -83.24267578125, "logps/rejected": -90.51362609863281, "loss": 1.274, "margin_dpo/margin_mean": 1.8371152877807617, "margin_dpo/margin_std": 5.285173416137695, "step": 108 }, { "KL/chosen_KL_mean": -4.783725738525391, "KL/mean": -6.253157615661621, "KL/rejected_KL_mean": -7.722591400146484, "KL/std": 4.904244899749756, "epoch": 0.16477702191987906, "fcm_dpo/beta": 0.14831490814685822, "fcm_dpo/delta": 0.3801053464412689, "fcm_dpo/margin": 2.9388718605041504, "fcm_dpo/q_t": 0.4147114157676697, "grad_norm": 40.31388854980469, "learning_rate": 4.941453335558681e-07, "logits/chosen": 0.17615270614624023, "logits/rejected": 0.1229727491736412, "logps/chosen": -60.31120681762695, "logps/ref_chosen": -55.52748107910156, "logps/ref_rejected": -83.55218505859375, "logps/rejected": -91.2747802734375, "loss": 1.1218, "margin_dpo/margin_mean": 2.938871383666992, "margin_dpo/margin_std": 5.313412666320801, "step": 109 }, { "KL/chosen_KL_mean": -6.120460510253906, "KL/mean": -6.251190185546875, "KL/rejected_KL_mean": -6.381919860839844, "KL/std": 3.969235420227051, "epoch": 0.16628873771730915, "fcm_dpo/beta": 0.15097622573375702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26145556569099426, "fcm_dpo/q_t": 0.48575273156166077, "grad_norm": 72.10308074951172, "learning_rate": 4.938574467213517e-07, "logits/chosen": 0.1714879870414734, "logits/rejected": 0.18068574368953705, "logps/chosen": -87.27920532226562, "logps/ref_chosen": -81.15874481201172, "logps/ref_rejected": -72.56021118164062, "logps/rejected": -78.94213104248047, "loss": 1.4907, "margin_dpo/margin_mean": 0.26145583391189575, "margin_dpo/margin_std": 5.336217403411865, "step": 110 }, { "KL/chosen_KL_mean": -5.464059829711914, "KL/mean": -6.4398345947265625, "KL/rejected_KL_mean": -7.415611267089844, "KL/std": 4.08194637298584, "epoch": 0.16780045351473924, "fcm_dpo/beta": 0.16302645206451416, "fcm_dpo/delta": 0.4963175654411316, "fcm_dpo/margin": 1.951550006866455, "fcm_dpo/q_t": 0.43752434849739075, "grad_norm": 42.85625457763672, "learning_rate": 4.935627386698418e-07, "logits/chosen": 0.3038405776023865, "logits/rejected": 0.26507824659347534, "logps/chosen": -57.82304382324219, "logps/ref_chosen": -52.358985900878906, "logps/ref_rejected": -77.06150817871094, "logps/rejected": -84.47711181640625, "loss": 1.2376, "margin_dpo/margin_mean": 1.9515496492385864, "margin_dpo/margin_std": 4.929745674133301, "step": 111 }, { "KL/chosen_KL_mean": -5.119453430175781, "KL/mean": -6.2895002365112305, "KL/rejected_KL_mean": -7.4595489501953125, "KL/std": 4.037227630615234, "epoch": 0.1693121693121693, "fcm_dpo/beta": 0.17741267383098602, "fcm_dpo/delta": 0.4004848003387451, "fcm_dpo/margin": 2.3400979042053223, "fcm_dpo/q_t": 0.41141277551651, "grad_norm": 55.50019454956055, "learning_rate": 4.932612176449559e-07, "logits/chosen": 0.21610409021377563, "logits/rejected": 0.1481962949037552, "logps/chosen": -68.13951873779297, "logps/ref_chosen": -63.02006530761719, "logps/ref_rejected": -111.36941528320312, "logps/rejected": -118.82896423339844, "loss": 1.1581, "margin_dpo/margin_mean": 2.340097665786743, "margin_dpo/margin_std": 4.509957790374756, "step": 112 }, { "KL/chosen_KL_mean": -5.140375137329102, "KL/mean": -5.796399116516113, "KL/rejected_KL_mean": -6.452423095703125, "KL/std": 3.3658576011657715, "epoch": 0.1708238851095994, "fcm_dpo/beta": 0.18488053977489471, "fcm_dpo/delta": 0.22963446378707886, "fcm_dpo/margin": 1.3120431900024414, "fcm_dpo/q_t": 0.445895791053772, "grad_norm": 69.64769744873047, "learning_rate": 4.929528920808854e-07, "logits/chosen": 0.21564623713493347, "logits/rejected": 0.1755000799894333, "logps/chosen": -60.948036193847656, "logps/ref_chosen": -55.80766296386719, "logps/ref_rejected": -69.84014129638672, "logps/rejected": -76.29256439208984, "loss": 1.3029, "margin_dpo/margin_mean": 1.3120434284210205, "margin_dpo/margin_std": 4.295475006103516, "step": 113 }, { "KL/chosen_KL_mean": -4.754701614379883, "KL/mean": -5.911993026733398, "KL/rejected_KL_mean": -7.069286346435547, "KL/std": 3.651322841644287, "epoch": 0.17233560090702948, "fcm_dpo/beta": 0.19917583465576172, "fcm_dpo/delta": 0.3558206558227539, "fcm_dpo/margin": 2.3145804405212402, "fcm_dpo/q_t": 0.4093940258026123, "grad_norm": 57.998111724853516, "learning_rate": 4.92637770602159e-07, "logits/chosen": 0.2491624653339386, "logits/rejected": 0.18856176733970642, "logps/chosen": -71.08747863769531, "logps/ref_chosen": -66.33277130126953, "logps/ref_rejected": -71.61489868164062, "logps/rejected": -78.6841812133789, "loss": 1.1409, "margin_dpo/margin_mean": 2.3145809173583984, "margin_dpo/margin_std": 4.402614593505859, "step": 114 }, { "KL/chosen_KL_mean": -5.139909744262695, "KL/mean": -6.020167350769043, "KL/rejected_KL_mean": -6.900426864624023, "KL/std": 3.308725595474243, "epoch": 0.17384731670445955, "fcm_dpo/beta": 0.21640095114707947, "fcm_dpo/delta": 0.43631792068481445, "fcm_dpo/margin": 1.7605162858963013, "fcm_dpo/q_t": 0.4244588017463684, "grad_norm": 63.72927474975586, "learning_rate": 4.923158620234019e-07, "logits/chosen": 0.24715614318847656, "logits/rejected": 0.18282869458198547, "logps/chosen": -60.888946533203125, "logps/ref_chosen": -55.74903869628906, "logps/ref_rejected": -79.59849548339844, "logps/rejected": -86.4989242553711, "loss": 1.2101, "margin_dpo/margin_mean": 1.7605164051055908, "margin_dpo/margin_std": 4.01201057434082, "step": 115 }, { "KL/chosen_KL_mean": -4.754339218139648, "KL/mean": -6.064699172973633, "KL/rejected_KL_mean": -7.37506103515625, "KL/std": 3.3698506355285645, "epoch": 0.17535903250188964, "fcm_dpo/beta": 0.2299938201904297, "fcm_dpo/delta": 0.2077406346797943, "fcm_dpo/margin": 2.6207261085510254, "fcm_dpo/q_t": 0.37513041496276855, "grad_norm": 48.991607666015625, "learning_rate": 4.91987175349089e-07, "logits/chosen": 0.21752528846263885, "logits/rejected": 0.14967337250709534, "logps/chosen": -54.1195068359375, "logps/ref_chosen": -49.36516571044922, "logps/ref_rejected": -72.84671020507812, "logps/rejected": -80.22177124023438, "loss": 1.0273, "margin_dpo/margin_mean": 2.6207261085510254, "margin_dpo/margin_std": 3.627825975418091, "step": 116 }, { "KL/chosen_KL_mean": -4.6624298095703125, "KL/mean": -5.545193195343018, "KL/rejected_KL_mean": -6.427959442138672, "KL/std": 3.1073155403137207, "epoch": 0.17687074829931973, "fcm_dpo/beta": 0.23274856805801392, "fcm_dpo/delta": 0.037866029888391495, "fcm_dpo/margin": 1.7655225992202759, "fcm_dpo/q_t": 0.4122212529182434, "grad_norm": 68.00582122802734, "learning_rate": 4.916517197732933e-07, "logits/chosen": 0.23936522006988525, "logits/rejected": 0.20207154750823975, "logps/chosen": -62.373329162597656, "logps/ref_chosen": -57.710899353027344, "logps/ref_rejected": -69.77253723144531, "logps/rejected": -76.20050048828125, "loss": 1.2094, "margin_dpo/margin_mean": 1.7655227184295654, "margin_dpo/margin_std": 3.8186731338500977, "step": 117 }, { "KL/chosen_KL_mean": -4.468959808349609, "KL/mean": -5.641507148742676, "KL/rejected_KL_mean": -6.814048767089844, "KL/std": 3.3355770111083984, "epoch": 0.17838246409674982, "fcm_dpo/beta": 0.2407892644405365, "fcm_dpo/delta": 0.2500095069408417, "fcm_dpo/margin": 2.345097541809082, "fcm_dpo/q_t": 0.3831072747707367, "grad_norm": 61.70059585571289, "learning_rate": 4.913095046794281e-07, "logits/chosen": 0.30035635828971863, "logits/rejected": 0.2581939697265625, "logps/chosen": -56.94886016845703, "logps/ref_chosen": -52.479896545410156, "logps/ref_rejected": -81.359130859375, "logps/rejected": -88.17317199707031, "loss": 1.052, "margin_dpo/margin_mean": 2.345097064971924, "margin_dpo/margin_std": 3.4296135902404785, "step": 118 }, { "KL/chosen_KL_mean": -5.600677490234375, "KL/mean": -6.450741291046143, "KL/rejected_KL_mean": -7.300800323486328, "KL/std": 3.7528910636901855, "epoch": 0.17989417989417988, "fcm_dpo/beta": 0.25984010100364685, "fcm_dpo/delta": 0.3726928234100342, "fcm_dpo/margin": 1.7001276016235352, "fcm_dpo/q_t": 0.4158112704753876, "grad_norm": 81.90481567382812, "learning_rate": 4.909605396399855e-07, "logits/chosen": 0.18883086740970612, "logits/rejected": 0.15015271306037903, "logps/chosen": -66.9583511352539, "logps/ref_chosen": -61.35767364501953, "logps/ref_rejected": -75.71510314941406, "logps/rejected": -83.01589965820312, "loss": 1.283, "margin_dpo/margin_mean": 1.7001276016235352, "margin_dpo/margin_std": 4.581892967224121, "step": 119 }, { "KL/chosen_KL_mean": -4.532203674316406, "KL/mean": -5.765423774719238, "KL/rejected_KL_mean": -6.9986419677734375, "KL/std": 3.1511075496673584, "epoch": 0.18140589569160998, "fcm_dpo/beta": 0.2700057625770569, "fcm_dpo/delta": 0.1430681198835373, "fcm_dpo/margin": 2.4664409160614014, "fcm_dpo/q_t": 0.365251362323761, "grad_norm": 63.3851203918457, "learning_rate": 4.906048344162676e-07, "logits/chosen": 0.18862515687942505, "logits/rejected": 0.12877434492111206, "logps/chosen": -64.43977355957031, "logps/ref_chosen": -59.907569885253906, "logps/ref_rejected": -79.6910629272461, "logps/rejected": -86.68970489501953, "loss": 0.9804, "margin_dpo/margin_mean": 2.4664411544799805, "margin_dpo/margin_std": 3.162449836730957, "step": 120 }, { "KL/chosen_KL_mean": -4.892913818359375, "KL/mean": -6.147130966186523, "KL/rejected_KL_mean": -7.4013519287109375, "KL/std": 3.5860328674316406, "epoch": 0.18291761148904007, "fcm_dpo/beta": 0.27552086114883423, "fcm_dpo/delta": 0.11710186302661896, "fcm_dpo/margin": 2.5084357261657715, "fcm_dpo/q_t": 0.3749258816242218, "grad_norm": 61.17416000366211, "learning_rate": 4.902423989581143e-07, "logits/chosen": 0.30166101455688477, "logits/rejected": 0.21480508148670197, "logps/chosen": -60.558956146240234, "logps/ref_chosen": -55.66604232788086, "logps/ref_rejected": -101.56233978271484, "logps/rejected": -108.96369171142578, "loss": 1.014, "margin_dpo/margin_mean": 2.5084352493286133, "margin_dpo/margin_std": 3.9617276191711426, "step": 121 }, { "KL/chosen_KL_mean": -5.208187103271484, "KL/mean": -6.4071879386901855, "KL/rejected_KL_mean": -7.606189727783203, "KL/std": 3.7734618186950684, "epoch": 0.18442932728647016, "fcm_dpo/beta": 0.284409761428833, "fcm_dpo/delta": 0.1253754049539566, "fcm_dpo/margin": 2.397998571395874, "fcm_dpo/q_t": 0.38381946086883545, "grad_norm": 89.1806640625, "learning_rate": 4.898732434036243e-07, "logits/chosen": 0.19963660836219788, "logits/rejected": 0.16456535458564758, "logps/chosen": -68.54255676269531, "logps/ref_chosen": -63.334373474121094, "logps/ref_rejected": -73.67523193359375, "logps/rejected": -81.28142547607422, "loss": 1.1012, "margin_dpo/margin_mean": 2.397998332977295, "margin_dpo/margin_std": 4.460177421569824, "step": 122 }, { "KL/chosen_KL_mean": -4.837169647216797, "KL/mean": -5.956876754760742, "KL/rejected_KL_mean": -7.0765838623046875, "KL/std": 3.257021188735962, "epoch": 0.18594104308390022, "fcm_dpo/beta": 0.29277366399765015, "fcm_dpo/delta": 0.15317855775356293, "fcm_dpo/margin": 2.239412784576416, "fcm_dpo/q_t": 0.3722848892211914, "grad_norm": 71.29094696044922, "learning_rate": 4.894973780788722e-07, "logits/chosen": 0.22779148817062378, "logits/rejected": 0.18621155619621277, "logps/chosen": -61.73591232299805, "logps/ref_chosen": -56.89874267578125, "logps/ref_rejected": -78.97028350830078, "logps/rejected": -86.04686737060547, "loss": 1.0543, "margin_dpo/margin_mean": 2.239412307739258, "margin_dpo/margin_std": 3.5291574001312256, "step": 123 }, { "KL/chosen_KL_mean": -5.083835601806641, "KL/mean": -6.601924896240234, "KL/rejected_KL_mean": -8.120014190673828, "KL/std": 3.4692869186401367, "epoch": 0.1874527588813303, "fcm_dpo/beta": 0.2885604500770569, "fcm_dpo/delta": -0.08434109389781952, "fcm_dpo/margin": 3.036177396774292, "fcm_dpo/q_t": 0.32999253273010254, "grad_norm": 60.00226974487305, "learning_rate": 4.89114813497619e-07, "logits/chosen": 0.25246724486351013, "logits/rejected": 0.19060075283050537, "logps/chosen": -62.199920654296875, "logps/ref_chosen": -57.116085052490234, "logps/ref_rejected": -87.93074035644531, "logps/rejected": -96.0507583618164, "loss": 0.8918, "margin_dpo/margin_mean": 3.036177396774292, "margin_dpo/margin_std": 3.5483925342559814, "step": 124 }, { "KL/chosen_KL_mean": -5.379281997680664, "KL/mean": -6.626376152038574, "KL/rejected_KL_mean": -7.873466491699219, "KL/std": 3.7205638885498047, "epoch": 0.1889644746787604, "fcm_dpo/beta": 0.2898848354816437, "fcm_dpo/delta": 0.0822625532746315, "fcm_dpo/margin": 2.4941811561584473, "fcm_dpo/q_t": 0.3657096028327942, "grad_norm": 73.03401947021484, "learning_rate": 4.887255603610184e-07, "logits/chosen": 0.2572287619113922, "logits/rejected": 0.19734513759613037, "logps/chosen": -71.08546447753906, "logps/ref_chosen": -65.7061767578125, "logps/ref_rejected": -91.72711944580078, "logps/rejected": -99.6005859375, "loss": 1.0256, "margin_dpo/margin_mean": 2.4941816329956055, "margin_dpo/margin_std": 3.7722997665405273, "step": 125 }, { "KL/chosen_KL_mean": -4.964244842529297, "KL/mean": -5.7615766525268555, "KL/rejected_KL_mean": -6.558906555175781, "KL/std": 3.618135690689087, "epoch": 0.19047619047619047, "fcm_dpo/beta": 0.308197557926178, "fcm_dpo/delta": 0.3242906332015991, "fcm_dpo/margin": 1.5946621894836426, "fcm_dpo/q_t": 0.40892714262008667, "grad_norm": 82.392333984375, "learning_rate": 4.883296295573176e-07, "logits/chosen": 0.09359246492385864, "logits/rejected": 0.08766859024763107, "logps/chosen": -73.14033508300781, "logps/ref_chosen": -68.17608642578125, "logps/ref_rejected": -65.1175537109375, "logps/rejected": -71.67646026611328, "loss": 1.2336, "margin_dpo/margin_mean": 1.5946623086929321, "margin_dpo/margin_std": 3.873288631439209, "step": 126 }, { "KL/chosen_KL_mean": -5.131870269775391, "KL/mean": -6.483763217926025, "KL/rejected_KL_mean": -7.835659027099609, "KL/std": 3.119049549102783, "epoch": 0.19198790627362056, "fcm_dpo/beta": 0.30798083543777466, "fcm_dpo/delta": -0.038629740476608276, "fcm_dpo/margin": 2.703786849975586, "fcm_dpo/q_t": 0.33612674474716187, "grad_norm": 72.35118103027344, "learning_rate": 4.87927032161552e-07, "logits/chosen": 0.19651305675506592, "logits/rejected": 0.165663942694664, "logps/chosen": -67.01210021972656, "logps/ref_chosen": -61.88023376464844, "logps/ref_rejected": -68.46012878417969, "logps/rejected": -76.29578399658203, "loss": 0.9255, "margin_dpo/margin_mean": 2.703786849975586, "margin_dpo/margin_std": 3.216360092163086, "step": 127 }, { "KL/chosen_KL_mean": -5.400476455688477, "KL/mean": -6.705332279205322, "KL/rejected_KL_mean": -8.010189056396484, "KL/std": 3.893587350845337, "epoch": 0.19349962207105065, "fcm_dpo/beta": 0.30849719047546387, "fcm_dpo/delta": -0.006407309323549271, "fcm_dpo/margin": 2.6097097396850586, "fcm_dpo/q_t": 0.36017996072769165, "grad_norm": 74.5383071899414, "learning_rate": 4.875177794352363e-07, "logits/chosen": 0.22811782360076904, "logits/rejected": 0.169705331325531, "logps/chosen": -72.10945892333984, "logps/ref_chosen": -66.708984375, "logps/ref_rejected": -94.97969055175781, "logps/rejected": -102.98988342285156, "loss": 1.1106, "margin_dpo/margin_mean": 2.6097097396850586, "margin_dpo/margin_std": 4.57863187789917, "step": 128 }, { "KL/chosen_KL_mean": -5.905820846557617, "KL/mean": -6.7846760749816895, "KL/rejected_KL_mean": -7.663532257080078, "KL/std": 3.419311046600342, "epoch": 0.19501133786848074, "fcm_dpo/beta": 0.3223820924758911, "fcm_dpo/delta": 0.2470059096813202, "fcm_dpo/margin": 1.757707953453064, "fcm_dpo/q_t": 0.39814797043800354, "grad_norm": 109.02452850341797, "learning_rate": 4.871018828260491e-07, "logits/chosen": 0.1916099488735199, "logits/rejected": 0.18546706438064575, "logps/chosen": -71.24464416503906, "logps/ref_chosen": -65.33882904052734, "logps/ref_rejected": -68.06109619140625, "logps/rejected": -75.72462463378906, "loss": 1.2241, "margin_dpo/margin_mean": 1.7577086687088013, "margin_dpo/margin_std": 3.991182565689087, "step": 129 }, { "KL/chosen_KL_mean": -5.464776992797852, "KL/mean": -6.652037620544434, "KL/rejected_KL_mean": -7.839298248291016, "KL/std": 3.3182621002197266, "epoch": 0.1965230536659108, "fcm_dpo/beta": 0.322995662689209, "fcm_dpo/delta": 0.0319809764623642, "fcm_dpo/margin": 2.3745198249816895, "fcm_dpo/q_t": 0.35143962502479553, "grad_norm": 92.80889892578125, "learning_rate": 4.866793539675126e-07, "logits/chosen": 0.15645794570446014, "logits/rejected": 0.10603205859661102, "logps/chosen": -64.12551879882812, "logps/ref_chosen": -58.660743713378906, "logps/ref_rejected": -79.24510192871094, "logps/rejected": -87.08439636230469, "loss": 0.9934, "margin_dpo/margin_mean": 2.3745198249816895, "margin_dpo/margin_std": 3.211923599243164, "step": 130 }, { "KL/chosen_KL_mean": -5.23967170715332, "KL/mean": -6.682486057281494, "KL/rejected_KL_mean": -8.125301361083984, "KL/std": 3.897421360015869, "epoch": 0.1980347694633409, "fcm_dpo/beta": 0.32098785042762756, "fcm_dpo/delta": -0.13940098881721497, "fcm_dpo/margin": 2.8856329917907715, "fcm_dpo/q_t": 0.3405371308326721, "grad_norm": 77.46083068847656, "learning_rate": 4.86250204678667e-07, "logits/chosen": 0.17860546708106995, "logits/rejected": 0.1137775182723999, "logps/chosen": -57.754207611083984, "logps/ref_chosen": -52.51453399658203, "logps/ref_rejected": -85.18299865722656, "logps/rejected": -93.30830383300781, "loss": 1.0287, "margin_dpo/margin_mean": 2.8856334686279297, "margin_dpo/margin_std": 4.351919651031494, "step": 131 }, { "KL/chosen_KL_mean": -5.565944671630859, "KL/mean": -6.807995796203613, "KL/rejected_KL_mean": -8.050046920776367, "KL/std": 3.650963544845581, "epoch": 0.19954648526077098, "fcm_dpo/beta": 0.31917959451675415, "fcm_dpo/delta": 0.007726715877652168, "fcm_dpo/margin": 2.484100818634033, "fcm_dpo/q_t": 0.3431278467178345, "grad_norm": 81.25666809082031, "learning_rate": 4.858144469637408e-07, "logits/chosen": 0.24190431833267212, "logits/rejected": 0.20946651697158813, "logps/chosen": -71.2510757446289, "logps/ref_chosen": -65.68513488769531, "logps/ref_rejected": -69.54120635986328, "logps/rejected": -77.59125518798828, "loss": 1.0342, "margin_dpo/margin_mean": 2.484100580215454, "margin_dpo/margin_std": 3.7712955474853516, "step": 132 }, { "KL/chosen_KL_mean": -5.565555572509766, "KL/mean": -6.669124603271484, "KL/rejected_KL_mean": -7.7726898193359375, "KL/std": 3.5532050132751465, "epoch": 0.20105820105820105, "fcm_dpo/beta": 0.3241734504699707, "fcm_dpo/delta": 0.09094859659671783, "fcm_dpo/margin": 2.207136631011963, "fcm_dpo/q_t": 0.3639262616634369, "grad_norm": 92.71206665039062, "learning_rate": 4.853720930118138e-07, "logits/chosen": 0.15483415126800537, "logits/rejected": 0.14521589875221252, "logps/chosen": -69.16366577148438, "logps/ref_chosen": -63.598114013671875, "logps/ref_rejected": -73.72798156738281, "logps/rejected": -81.50067138671875, "loss": 1.0788, "margin_dpo/margin_mean": 2.207136631011963, "margin_dpo/margin_std": 3.691882610321045, "step": 133 }, { "KL/chosen_KL_mean": -5.148525238037109, "KL/mean": -6.78424072265625, "KL/rejected_KL_mean": -8.41995620727539, "KL/std": 4.025635719299316, "epoch": 0.20256991685563114, "fcm_dpo/beta": 0.31296706199645996, "fcm_dpo/delta": -0.2501199245452881, "fcm_dpo/margin": 3.2714293003082275, "fcm_dpo/q_t": 0.31163257360458374, "grad_norm": 70.16565704345703, "learning_rate": 4.849231551964771e-07, "logits/chosen": 0.2684386968612671, "logits/rejected": 0.21187162399291992, "logps/chosen": -58.94309616088867, "logps/ref_chosen": -53.79457092285156, "logps/ref_rejected": -74.16741943359375, "logps/rejected": -82.58737182617188, "loss": 0.8797, "margin_dpo/margin_mean": 3.271430253982544, "margin_dpo/margin_std": 3.986935615539551, "step": 134 }, { "KL/chosen_KL_mean": -5.128990173339844, "KL/mean": -6.169395446777344, "KL/rejected_KL_mean": -7.209804534912109, "KL/std": 3.7049777507781982, "epoch": 0.20408163265306123, "fcm_dpo/beta": 0.31707170605659485, "fcm_dpo/delta": 0.1498485505580902, "fcm_dpo/margin": 2.080817699432373, "fcm_dpo/q_t": 0.3767089545726776, "grad_norm": 72.5808334350586, "learning_rate": 4.844676460754862e-07, "logits/chosen": 0.19408661127090454, "logits/rejected": 0.1618993878364563, "logps/chosen": -54.570068359375, "logps/ref_chosen": -49.441078186035156, "logps/ref_rejected": -65.96878051757812, "logps/rejected": -73.1785888671875, "loss": 1.1257, "margin_dpo/margin_mean": 2.080817222595215, "margin_dpo/margin_std": 3.858844518661499, "step": 135 }, { "KL/chosen_KL_mean": -6.007057189941406, "KL/mean": -7.334796905517578, "KL/rejected_KL_mean": -8.66253662109375, "KL/std": 4.03962516784668, "epoch": 0.20559334845049132, "fcm_dpo/beta": 0.3157135248184204, "fcm_dpo/delta": -0.04242422431707382, "fcm_dpo/margin": 2.655482530593872, "fcm_dpo/q_t": 0.36486750841140747, "grad_norm": 97.05379486083984, "learning_rate": 4.840055783904106e-07, "logits/chosen": 0.17138057947158813, "logits/rejected": 0.10115008801221848, "logps/chosen": -72.76631927490234, "logps/ref_chosen": -66.75926208496094, "logps/ref_rejected": -94.61787414550781, "logps/rejected": -103.28041076660156, "loss": 1.1929, "margin_dpo/margin_mean": 2.655484199523926, "margin_dpo/margin_std": 5.235541343688965, "step": 136 }, { "KL/chosen_KL_mean": -5.561779022216797, "KL/mean": -6.875943183898926, "KL/rejected_KL_mean": -8.190105438232422, "KL/std": 3.721090316772461, "epoch": 0.20710506424792138, "fcm_dpo/beta": 0.31635814905166626, "fcm_dpo/delta": -0.03473253175616264, "fcm_dpo/margin": 2.628328323364258, "fcm_dpo/q_t": 0.3503134548664093, "grad_norm": 74.95806884765625, "learning_rate": 4.835369650662767e-07, "logits/chosen": 0.21699392795562744, "logits/rejected": 0.1880975365638733, "logps/chosen": -62.345577239990234, "logps/ref_chosen": -56.78379821777344, "logps/ref_rejected": -69.89952087402344, "logps/rejected": -78.08963012695312, "loss": 1.0336, "margin_dpo/margin_mean": 2.628328561782837, "margin_dpo/margin_std": 4.104913711547852, "step": 137 }, { "KL/chosen_KL_mean": -6.465946197509766, "KL/mean": -7.493434906005859, "KL/rejected_KL_mean": -8.520923614501953, "KL/std": 3.798523187637329, "epoch": 0.20861678004535147, "fcm_dpo/beta": 0.32131582498550415, "fcm_dpo/delta": 0.14952073991298676, "fcm_dpo/margin": 2.0549755096435547, "fcm_dpo/q_t": 0.376254677772522, "grad_norm": 85.68193054199219, "learning_rate": 4.830618192112065e-07, "logits/chosen": 0.2118818461894989, "logits/rejected": 0.1755654513835907, "logps/chosen": -65.2319564819336, "logps/ref_chosen": -58.766014099121094, "logps/ref_rejected": -68.12371826171875, "logps/rejected": -76.64463806152344, "loss": 1.1774, "margin_dpo/margin_mean": 2.0549752712249756, "margin_dpo/margin_std": 4.0768890380859375, "step": 138 }, { "KL/chosen_KL_mean": -6.410148620605469, "KL/mean": -7.718695640563965, "KL/rejected_KL_mean": -9.027240753173828, "KL/std": 3.6766586303710938, "epoch": 0.21012849584278157, "fcm_dpo/beta": 0.3235969543457031, "fcm_dpo/delta": -0.05197485536336899, "fcm_dpo/margin": 2.6170876026153564, "fcm_dpo/q_t": 0.34167400002479553, "grad_norm": 83.7813720703125, "learning_rate": 4.825801541160509e-07, "logits/chosen": 0.15254321694374084, "logits/rejected": 0.12392173707485199, "logps/chosen": -77.6357421875, "logps/ref_chosen": -71.2255859375, "logps/ref_rejected": -82.1834716796875, "logps/rejected": -91.21070861816406, "loss": 0.9695, "margin_dpo/margin_mean": 2.6170878410339355, "margin_dpo/margin_std": 3.5879359245300293, "step": 139 }, { "KL/chosen_KL_mean": -5.829267501831055, "KL/mean": -7.498285293579102, "KL/rejected_KL_mean": -9.167301177978516, "KL/std": 4.7677483558654785, "epoch": 0.21164021164021163, "fcm_dpo/beta": 0.30738502740859985, "fcm_dpo/delta": -0.25325673818588257, "fcm_dpo/margin": 3.33803653717041, "fcm_dpo/q_t": 0.3198111951351166, "grad_norm": 91.1104965209961, "learning_rate": 4.820919832540181e-07, "logits/chosen": 0.18167796730995178, "logits/rejected": 0.1350601315498352, "logps/chosen": -69.10693359375, "logps/ref_chosen": -63.27766418457031, "logps/ref_rejected": -83.30647277832031, "logps/rejected": -92.47377014160156, "loss": 1.0517, "margin_dpo/margin_mean": 3.3380370140075684, "margin_dpo/margin_std": 5.130913734436035, "step": 140 }, { "KL/chosen_KL_mean": -5.997409820556641, "KL/mean": -7.6452717781066895, "KL/rejected_KL_mean": -9.293132781982422, "KL/std": 3.922736167907715, "epoch": 0.21315192743764172, "fcm_dpo/beta": 0.2922168970108032, "fcm_dpo/delta": -0.1869136393070221, "fcm_dpo/margin": 3.295717716217041, "fcm_dpo/q_t": 0.3368871808052063, "grad_norm": 77.85089111328125, "learning_rate": 4.815973202802966e-07, "logits/chosen": 0.20691722631454468, "logits/rejected": 0.16274985671043396, "logps/chosen": -67.76417541503906, "logps/ref_chosen": -61.76676940917969, "logps/ref_rejected": -88.60601806640625, "logps/rejected": -97.89915466308594, "loss": 1.0053, "margin_dpo/margin_mean": 3.295717716217041, "margin_dpo/margin_std": 4.843393325805664, "step": 141 }, { "KL/chosen_KL_mean": -5.920963287353516, "KL/mean": -7.147356033325195, "KL/rejected_KL_mean": -8.37375259399414, "KL/std": 3.805548667907715, "epoch": 0.2146636432350718, "fcm_dpo/beta": 0.2973337769508362, "fcm_dpo/delta": 0.07598335295915604, "fcm_dpo/margin": 2.452789783477783, "fcm_dpo/q_t": 0.36309534311294556, "grad_norm": 72.36812591552734, "learning_rate": 4.810961790316729e-07, "logits/chosen": 0.1847228705883026, "logits/rejected": 0.16021020710468292, "logps/chosen": -71.19573974609375, "logps/ref_chosen": -65.2747802734375, "logps/ref_rejected": -81.1378173828125, "logps/rejected": -89.51156616210938, "loss": 1.0487, "margin_dpo/margin_mean": 2.452789545059204, "margin_dpo/margin_std": 3.8934617042541504, "step": 142 }, { "KL/chosen_KL_mean": -6.140836715698242, "KL/mean": -7.2427263259887695, "KL/rejected_KL_mean": -8.344612121582031, "KL/std": 4.039183616638184, "epoch": 0.2161753590325019, "fcm_dpo/beta": 0.30352532863616943, "fcm_dpo/delta": 0.14069342613220215, "fcm_dpo/margin": 2.203770637512207, "fcm_dpo/q_t": 0.3680863380432129, "grad_norm": 99.17074584960938, "learning_rate": 4.805885735261454e-07, "logits/chosen": 0.19867965579032898, "logits/rejected": 0.1820322573184967, "logps/chosen": -68.7586669921875, "logps/ref_chosen": -62.617828369140625, "logps/ref_rejected": -70.39239501953125, "logps/rejected": -78.73699951171875, "loss": 1.2289, "margin_dpo/margin_mean": 2.203770637512207, "margin_dpo/margin_std": 4.742335319519043, "step": 143 }, { "KL/chosen_KL_mean": -6.31688117980957, "KL/mean": -7.679841995239258, "KL/rejected_KL_mean": -9.042800903320312, "KL/std": 4.1917219161987305, "epoch": 0.21768707482993196, "fcm_dpo/beta": 0.303509384393692, "fcm_dpo/delta": -0.030424147844314575, "fcm_dpo/margin": 2.7259182929992676, "fcm_dpo/q_t": 0.35583314299583435, "grad_norm": 83.09550476074219, "learning_rate": 4.800745179625307e-07, "logits/chosen": 0.17651958763599396, "logits/rejected": 0.14690782129764557, "logps/chosen": -67.11956787109375, "logps/ref_chosen": -60.80268859863281, "logps/ref_rejected": -79.07284545898438, "logps/rejected": -88.11564636230469, "loss": 1.0936, "margin_dpo/margin_mean": 2.725918769836426, "margin_dpo/margin_std": 4.664007663726807, "step": 144 }, { "KL/chosen_KL_mean": -5.749847412109375, "KL/mean": -7.1203932762146, "KL/rejected_KL_mean": -8.490936279296875, "KL/std": 4.098171234130859, "epoch": 0.21919879062736206, "fcm_dpo/beta": 0.30279701948165894, "fcm_dpo/delta": -0.03274145722389221, "fcm_dpo/margin": 2.7410945892333984, "fcm_dpo/q_t": 0.3517426550388336, "grad_norm": 89.16951751708984, "learning_rate": 4.795540267200686e-07, "logits/chosen": 0.1450120210647583, "logits/rejected": 0.16164004802703857, "logps/chosen": -80.36131286621094, "logps/ref_chosen": -74.61146545410156, "logps/ref_rejected": -83.24461364746094, "logps/rejected": -91.73554992675781, "loss": 1.0838, "margin_dpo/margin_mean": 2.7410945892333984, "margin_dpo/margin_std": 4.671388626098633, "step": 145 }, { "KL/chosen_KL_mean": -5.396156311035156, "KL/mean": -6.788448810577393, "KL/rejected_KL_mean": -8.180740356445312, "KL/std": 3.8954482078552246, "epoch": 0.22071050642479215, "fcm_dpo/beta": 0.3020232021808624, "fcm_dpo/delta": -0.04509184509515762, "fcm_dpo/margin": 2.7845849990844727, "fcm_dpo/q_t": 0.3457740843296051, "grad_norm": 79.14019775390625, "learning_rate": 4.790271143580173e-07, "logits/chosen": 0.13101473450660706, "logits/rejected": 0.11654573678970337, "logps/chosen": -63.23714065551758, "logps/ref_chosen": -57.84098434448242, "logps/ref_rejected": -67.47422790527344, "logps/rejected": -75.65496826171875, "loss": 1.0088, "margin_dpo/margin_mean": 2.7845849990844727, "margin_dpo/margin_std": 4.094869136810303, "step": 146 }, { "KL/chosen_KL_mean": -6.266658782958984, "KL/mean": -7.423336982727051, "KL/rejected_KL_mean": -8.580020904541016, "KL/std": 4.117965221405029, "epoch": 0.2222222222222222, "fcm_dpo/beta": 0.30491483211517334, "fcm_dpo/delta": 0.10152038931846619, "fcm_dpo/margin": 2.313361167907715, "fcm_dpo/q_t": 0.37097328901290894, "grad_norm": 102.72885131835938, "learning_rate": 4.784937956152489e-07, "logits/chosen": 0.17838065326213837, "logits/rejected": 0.1357332020998001, "logps/chosen": -73.08013153076172, "logps/ref_chosen": -66.81346893310547, "logps/ref_rejected": -81.1796875, "logps/rejected": -89.75971221923828, "loss": 1.1535, "margin_dpo/margin_mean": 2.3133602142333984, "margin_dpo/margin_std": 4.482905387878418, "step": 147 }, { "KL/chosen_KL_mean": -5.813823699951172, "KL/mean": -7.228766441345215, "KL/rejected_KL_mean": -8.643707275390625, "KL/std": 3.8219246864318848, "epoch": 0.2237339380196523, "fcm_dpo/beta": 0.3052898645401001, "fcm_dpo/delta": -0.07084967195987701, "fcm_dpo/margin": 2.829887628555298, "fcm_dpo/q_t": 0.34750601649284363, "grad_norm": 67.65303802490234, "learning_rate": 4.779540854098347e-07, "logits/chosen": 0.28564712405204773, "logits/rejected": 0.2142116129398346, "logps/chosen": -54.50157928466797, "logps/ref_chosen": -48.6877555847168, "logps/ref_rejected": -67.50503540039062, "logps/rejected": -76.14874267578125, "loss": 1.0565, "margin_dpo/margin_mean": 2.8298869132995605, "margin_dpo/margin_std": 4.450958728790283, "step": 148 }, { "KL/chosen_KL_mean": -7.005121231079102, "KL/mean": -8.661083221435547, "KL/rejected_KL_mean": -10.31704330444336, "KL/std": 4.533745765686035, "epoch": 0.2252456538170824, "fcm_dpo/beta": 0.29332873225212097, "fcm_dpo/delta": -0.1899138242006302, "fcm_dpo/margin": 3.311924457550049, "fcm_dpo/q_t": 0.3203445076942444, "grad_norm": 66.86536407470703, "learning_rate": 4.774079988386296e-07, "logits/chosen": 0.15289664268493652, "logits/rejected": 0.10625985264778137, "logps/chosen": -62.14889907836914, "logps/ref_chosen": -55.143775939941406, "logps/ref_rejected": -64.79888916015625, "logps/rejected": -75.11593627929688, "loss": 0.9195, "margin_dpo/margin_mean": 3.311924457550049, "margin_dpo/margin_std": 4.24875545501709, "step": 149 }, { "KL/chosen_KL_mean": -5.336969375610352, "KL/mean": -7.341297149658203, "KL/rejected_KL_mean": -9.345619201660156, "KL/std": 4.29542350769043, "epoch": 0.22675736961451248, "fcm_dpo/beta": 0.2759903073310852, "fcm_dpo/delta": -0.34455248713493347, "fcm_dpo/margin": 4.008656024932861, "fcm_dpo/q_t": 0.2917425036430359, "grad_norm": 61.21348190307617, "learning_rate": 4.768555511768486e-07, "logits/chosen": 0.1695353239774704, "logits/rejected": 0.12736281752586365, "logps/chosen": -72.8077163696289, "logps/ref_chosen": -67.47074890136719, "logps/ref_rejected": -89.21170806884766, "logps/rejected": -98.55732727050781, "loss": 0.8057, "margin_dpo/margin_mean": 4.0086565017700195, "margin_dpo/margin_std": 4.115660667419434, "step": 150 }, { "KL/chosen_KL_mean": -5.009391784667969, "KL/mean": -7.053393840789795, "KL/rejected_KL_mean": -9.097404479980469, "KL/std": 4.357527256011963, "epoch": 0.22826908541194255, "fcm_dpo/beta": 0.26016658544540405, "fcm_dpo/delta": -0.2943726181983948, "fcm_dpo/margin": 4.088005065917969, "fcm_dpo/q_t": 0.30344998836517334, "grad_norm": 55.01530456542969, "learning_rate": 4.762967578776406e-07, "logits/chosen": 0.16540372371673584, "logits/rejected": 0.11411911994218826, "logps/chosen": -57.46893310546875, "logps/ref_chosen": -52.45954132080078, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -88.16041564941406, "loss": 0.8328, "margin_dpo/margin_mean": 4.088005542755127, "margin_dpo/margin_std": 4.563044548034668, "step": 151 }, { "KL/chosen_KL_mean": -6.312803268432617, "KL/mean": -7.799456596374512, "KL/rejected_KL_mean": -9.286109924316406, "KL/std": 4.282715320587158, "epoch": 0.22978080120937264, "fcm_dpo/beta": 0.2594345211982727, "fcm_dpo/delta": 0.028224565088748932, "fcm_dpo/margin": 2.973308563232422, "fcm_dpo/q_t": 0.3617689311504364, "grad_norm": 69.3165054321289, "learning_rate": 4.757316345716553e-07, "logits/chosen": 0.24428924918174744, "logits/rejected": 0.1937820166349411, "logps/chosen": -62.86663818359375, "logps/ref_chosen": -56.5538330078125, "logps/ref_rejected": -76.55074310302734, "logps/rejected": -85.83685302734375, "loss": 1.0719, "margin_dpo/margin_mean": 2.9733083248138428, "margin_dpo/margin_std": 4.82022762298584, "step": 152 }, { "KL/chosen_KL_mean": -5.811321258544922, "KL/mean": -7.454206943511963, "KL/rejected_KL_mean": -9.097091674804688, "KL/std": 4.515659332275391, "epoch": 0.23129251700680273, "fcm_dpo/beta": 0.2544195055961609, "fcm_dpo/delta": -0.03977034240961075, "fcm_dpo/margin": 3.2857680320739746, "fcm_dpo/q_t": 0.33868566155433655, "grad_norm": 61.1727409362793, "learning_rate": 4.751601970666064e-07, "logits/chosen": 0.1626579612493515, "logits/rejected": 0.12607043981552124, "logps/chosen": -73.8182144165039, "logps/ref_chosen": -68.00689697265625, "logps/ref_rejected": -74.83482360839844, "logps/rejected": -83.93191528320312, "loss": 0.9371, "margin_dpo/margin_mean": 3.2857680320739746, "margin_dpo/margin_std": 4.14332914352417, "step": 153 }, { "KL/chosen_KL_mean": -6.67750358581543, "KL/mean": -7.8356099128723145, "KL/rejected_KL_mean": -8.993721008300781, "KL/std": 4.44697904586792, "epoch": 0.2328042328042328, "fcm_dpo/beta": 0.26349014043807983, "fcm_dpo/delta": 0.20121973752975464, "fcm_dpo/margin": 2.316216468811035, "fcm_dpo/q_t": 0.3814409077167511, "grad_norm": 69.11931610107422, "learning_rate": 4.745824613468292e-07, "logits/chosen": 0.23633871972560883, "logits/rejected": 0.2331632822751999, "logps/chosen": -65.90003967285156, "logps/ref_chosen": -59.222537994384766, "logps/ref_rejected": -64.19131469726562, "logps/rejected": -73.18504333496094, "loss": 1.1743, "margin_dpo/margin_mean": 2.3162169456481934, "margin_dpo/margin_std": 4.630753517150879, "step": 154 }, { "KL/chosen_KL_mean": -6.549232482910156, "KL/mean": -8.094888687133789, "KL/rejected_KL_mean": -9.64053726196289, "KL/std": 4.321755886077881, "epoch": 0.23431594860166288, "fcm_dpo/beta": 0.2623800039291382, "fcm_dpo/delta": -0.014081403613090515, "fcm_dpo/margin": 3.09130597114563, "fcm_dpo/q_t": 0.3595314919948578, "grad_norm": 71.75285339355469, "learning_rate": 4.7399844357283393e-07, "logits/chosen": 0.22018414735794067, "logits/rejected": 0.20080286264419556, "logps/chosen": -75.0039291381836, "logps/ref_chosen": -68.45469665527344, "logps/ref_rejected": -77.91763305664062, "logps/rejected": -87.55817413330078, "loss": 1.1156, "margin_dpo/margin_mean": 3.091305732727051, "margin_dpo/margin_std": 5.317093849182129, "step": 155 }, { "KL/chosen_KL_mean": -6.38591194152832, "KL/mean": -8.252660751342773, "KL/rejected_KL_mean": -10.119407653808594, "KL/std": 4.605101585388184, "epoch": 0.23582766439909297, "fcm_dpo/beta": 0.25796937942504883, "fcm_dpo/delta": -0.18036875128746033, "fcm_dpo/margin": 3.7334985733032227, "fcm_dpo/q_t": 0.3195089101791382, "grad_norm": 62.9564323425293, "learning_rate": 4.7340816008085305e-07, "logits/chosen": 0.17611095309257507, "logits/rejected": 0.13236644864082336, "logps/chosen": -73.65550994873047, "logps/ref_chosen": -67.26959991455078, "logps/ref_rejected": -86.95914459228516, "logps/rejected": -97.07855224609375, "loss": 0.8812, "margin_dpo/margin_mean": 3.7334985733032227, "margin_dpo/margin_std": 4.432024955749512, "step": 156 }, { "KL/chosen_KL_mean": -5.90778923034668, "KL/mean": -7.382824420928955, "KL/rejected_KL_mean": -8.857852935791016, "KL/std": 4.559696197509766, "epoch": 0.23733938019652306, "fcm_dpo/beta": 0.2535499334335327, "fcm_dpo/delta": 0.052449166774749756, "fcm_dpo/margin": 2.9500629901885986, "fcm_dpo/q_t": 0.35793808102607727, "grad_norm": 63.09500503540039, "learning_rate": 4.728116273823847e-07, "logits/chosen": 0.1454748958349228, "logits/rejected": 0.12664461135864258, "logps/chosen": -60.6806640625, "logps/ref_chosen": -54.77287292480469, "logps/ref_rejected": -63.87866973876953, "logps/rejected": -72.73652648925781, "loss": 1.0299, "margin_dpo/margin_mean": 2.9500629901885986, "margin_dpo/margin_std": 4.351293563842773, "step": 157 }, { "KL/chosen_KL_mean": -6.305976867675781, "KL/mean": -7.767054080963135, "KL/rejected_KL_mean": -9.228130340576172, "KL/std": 4.4387922286987305, "epoch": 0.23885109599395313, "fcm_dpo/beta": 0.25931790471076965, "fcm_dpo/delta": 0.04571561515331268, "fcm_dpo/margin": 2.922152280807495, "fcm_dpo/q_t": 0.35533732175827026, "grad_norm": 68.03280639648438, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 0.18727868795394897, "logits/rejected": 0.15256363153457642, "logps/chosen": -71.22869873046875, "logps/ref_chosen": -64.92271423339844, "logps/ref_rejected": -82.23789978027344, "logps/rejected": -91.46602630615234, "loss": 1.0286, "margin_dpo/margin_mean": 2.922152042388916, "margin_dpo/margin_std": 4.38568115234375, "step": 158 }, { "KL/chosen_KL_mean": -6.437795639038086, "KL/mean": -8.105405807495117, "KL/rejected_KL_mean": -9.773014068603516, "KL/std": 4.696426868438721, "epoch": 0.24036281179138322, "fcm_dpo/beta": 0.2608144283294678, "fcm_dpo/delta": -0.07918387651443481, "fcm_dpo/margin": 3.3352150917053223, "fcm_dpo/q_t": 0.3356255888938904, "grad_norm": 76.94505310058594, "learning_rate": 4.715998812855304e-07, "logits/chosen": 0.21540075540542603, "logits/rejected": 0.17887872457504272, "logps/chosen": -63.48478698730469, "logps/ref_chosen": -57.046993255615234, "logps/ref_rejected": -73.32441711425781, "logps/rejected": -83.09742736816406, "loss": 1.0131, "margin_dpo/margin_mean": 3.3352150917053223, "margin_dpo/margin_std": 4.913866996765137, "step": 159 }, { "KL/chosen_KL_mean": -7.160207748413086, "KL/mean": -8.652528762817383, "KL/rejected_KL_mean": -10.144851684570312, "KL/std": 4.411318778991699, "epoch": 0.2418745275888133, "fcm_dpo/beta": 0.25836193561553955, "fcm_dpo/delta": 0.030919140204787254, "fcm_dpo/margin": 2.9846417903900146, "fcm_dpo/q_t": 0.36456459760665894, "grad_norm": 64.14701843261719, "learning_rate": 4.7098470178228755e-07, "logits/chosen": 0.0936279147863388, "logits/rejected": 0.05139011889696121, "logps/chosen": -56.96712112426758, "logps/ref_chosen": -49.806915283203125, "logps/ref_rejected": -68.3370132446289, "logps/rejected": -78.48186492919922, "loss": 1.0898, "margin_dpo/margin_mean": 2.9846420288085938, "margin_dpo/margin_std": 5.151812553405762, "step": 160 }, { "KL/chosen_KL_mean": -6.983335494995117, "KL/mean": -8.564802169799805, "KL/rejected_KL_mean": -10.146270751953125, "KL/std": 4.450295925140381, "epoch": 0.24338624338624337, "fcm_dpo/beta": 0.2577332854270935, "fcm_dpo/delta": -0.016592275351285934, "fcm_dpo/margin": 3.1629390716552734, "fcm_dpo/q_t": 0.35136890411376953, "grad_norm": 60.320228576660156, "learning_rate": 4.703633408618955e-07, "logits/chosen": 0.21055185794830322, "logits/rejected": 0.17318174242973328, "logps/chosen": -59.48382568359375, "logps/ref_chosen": -52.50048828125, "logps/ref_rejected": -66.04540252685547, "logps/rejected": -76.1916732788086, "loss": 1.0397, "margin_dpo/margin_mean": 3.1629395484924316, "margin_dpo/margin_std": 4.972609519958496, "step": 161 }, { "KL/chosen_KL_mean": -7.363391876220703, "KL/mean": -9.55433177947998, "KL/rejected_KL_mean": -11.74527359008789, "KL/std": 4.978542804718018, "epoch": 0.24489795918367346, "fcm_dpo/beta": 0.24483485519886017, "fcm_dpo/delta": -0.3069154620170593, "fcm_dpo/margin": 4.381883144378662, "fcm_dpo/q_t": 0.2978348135948181, "grad_norm": 55.664127349853516, "learning_rate": 4.697358159051549e-07, "logits/chosen": 0.25998812913894653, "logits/rejected": 0.21173785626888275, "logps/chosen": -76.83258056640625, "logps/ref_chosen": -69.46919250488281, "logps/ref_rejected": -92.00952911376953, "logps/rejected": -103.75480651855469, "loss": 0.8312, "margin_dpo/margin_mean": 4.3818840980529785, "margin_dpo/margin_std": 4.7379302978515625, "step": 162 }, { "KL/chosen_KL_mean": -6.585916519165039, "KL/mean": -8.661686897277832, "KL/rejected_KL_mean": -10.73746109008789, "KL/std": 4.580445289611816, "epoch": 0.24640967498110355, "fcm_dpo/beta": 0.23843413591384888, "fcm_dpo/delta": -0.2134178876876831, "fcm_dpo/margin": 4.151541709899902, "fcm_dpo/q_t": 0.3177376091480255, "grad_norm": 58.10890579223633, "learning_rate": 4.691021444652876e-07, "logits/chosen": 0.18294395506381989, "logits/rejected": 0.13766877353191376, "logps/chosen": -57.19975280761719, "logps/ref_chosen": -50.613834381103516, "logps/ref_rejected": -74.62033081054688, "logps/rejected": -85.3577880859375, "loss": 0.9309, "margin_dpo/margin_mean": 4.1515421867370605, "margin_dpo/margin_std": 5.052495002746582, "step": 163 }, { "KL/chosen_KL_mean": -7.12010383605957, "KL/mean": -9.122835159301758, "KL/rejected_KL_mean": -11.125564575195312, "KL/std": 4.873750686645508, "epoch": 0.24792139077853365, "fcm_dpo/beta": 0.22845444083213806, "fcm_dpo/delta": -0.12654295563697815, "fcm_dpo/margin": 4.005465030670166, "fcm_dpo/q_t": 0.33134955167770386, "grad_norm": 53.08348846435547, "learning_rate": 4.6846234426744624e-07, "logits/chosen": 0.17225149273872375, "logits/rejected": 0.11162863671779633, "logps/chosen": -61.96821594238281, "logps/ref_chosen": -54.848114013671875, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -90.1885757446289, "loss": 0.9499, "margin_dpo/margin_mean": 4.005465030670166, "margin_dpo/margin_std": 5.20572566986084, "step": 164 }, { "KL/chosen_KL_mean": -7.681882858276367, "KL/mean": -9.503345489501953, "KL/rejected_KL_mean": -11.324806213378906, "KL/std": 4.7919440269470215, "epoch": 0.2494331065759637, "fcm_dpo/beta": 0.22671417891979218, "fcm_dpo/delta": -0.029426731169223785, "fcm_dpo/margin": 3.6429243087768555, "fcm_dpo/q_t": 0.33616942167282104, "grad_norm": 53.045433044433594, "learning_rate": 4.678164332082175e-07, "logits/chosen": 0.2291611135005951, "logits/rejected": 0.17568400502204895, "logps/chosen": -58.771095275878906, "logps/ref_chosen": -51.089210510253906, "logps/ref_rejected": -71.23370361328125, "logps/rejected": -82.55850219726562, "loss": 0.9491, "margin_dpo/margin_mean": 3.6429238319396973, "margin_dpo/margin_std": 4.521003246307373, "step": 165 }, { "KL/chosen_KL_mean": -7.4115447998046875, "KL/mean": -8.709308624267578, "KL/rejected_KL_mean": -10.007076263427734, "KL/std": 4.388735294342041, "epoch": 0.2509448223733938, "fcm_dpo/beta": 0.23121148347854614, "fcm_dpo/delta": 0.2130114734172821, "fcm_dpo/margin": 2.5955326557159424, "fcm_dpo/q_t": 0.38784292340278625, "grad_norm": 65.73187255859375, "learning_rate": 4.6716442935512214e-07, "logits/chosen": 0.2012665569782257, "logits/rejected": 0.11512681841850281, "logps/chosen": -70.60235595703125, "logps/ref_chosen": -63.19081115722656, "logps/ref_rejected": -93.8402099609375, "logps/rejected": -103.8472900390625, "loss": 1.0979, "margin_dpo/margin_mean": 2.5955324172973633, "margin_dpo/margin_std": 4.49823522567749, "step": 166 }, { "KL/chosen_KL_mean": -6.62812614440918, "KL/mean": -8.661577224731445, "KL/rejected_KL_mean": -10.695026397705078, "KL/std": 4.661866188049316, "epoch": 0.25245653817082386, "fcm_dpo/beta": 0.22737373411655426, "fcm_dpo/delta": -0.14043092727661133, "fcm_dpo/margin": 4.066901206970215, "fcm_dpo/q_t": 0.3204006850719452, "grad_norm": 48.181392669677734, "learning_rate": 4.6650635094610966e-07, "logits/chosen": 0.1581970453262329, "logits/rejected": 0.12355121970176697, "logps/chosen": -65.55239868164062, "logps/ref_chosen": -58.92427062988281, "logps/ref_rejected": -72.97377014160156, "logps/rejected": -83.66879272460938, "loss": 0.8813, "margin_dpo/margin_mean": 4.066901206970215, "margin_dpo/margin_std": 4.630178451538086, "step": 167 }, { "KL/chosen_KL_mean": -8.006879806518555, "KL/mean": -9.33842658996582, "KL/rejected_KL_mean": -10.66998291015625, "KL/std": 4.7535505294799805, "epoch": 0.25396825396825395, "fcm_dpo/beta": 0.23451144993305206, "fcm_dpo/delta": 0.18685543537139893, "fcm_dpo/margin": 2.6631035804748535, "fcm_dpo/q_t": 0.3788146674633026, "grad_norm": 65.586669921875, "learning_rate": 4.6584221638904767e-07, "logits/chosen": 0.17673316597938538, "logits/rejected": 0.14384135603904724, "logps/chosen": -73.65826416015625, "logps/ref_chosen": -65.65138244628906, "logps/ref_rejected": -79.71418762207031, "logps/rejected": -90.38417053222656, "loss": 1.0702, "margin_dpo/margin_mean": 2.6631031036376953, "margin_dpo/margin_std": 4.263519287109375, "step": 168 }, { "KL/chosen_KL_mean": -6.801374435424805, "KL/mean": -8.660758972167969, "KL/rejected_KL_mean": -10.5201416015625, "KL/std": 4.897111892700195, "epoch": 0.25547996976568405, "fcm_dpo/beta": 0.2340339571237564, "fcm_dpo/delta": -0.07695929706096649, "fcm_dpo/margin": 3.7187600135803223, "fcm_dpo/q_t": 0.3465001881122589, "grad_norm": 60.35015869140625, "learning_rate": 4.651720442612075e-07, "logits/chosen": 0.2228689342737198, "logits/rejected": 0.19062112271785736, "logps/chosen": -68.22724151611328, "logps/ref_chosen": -61.425865173339844, "logps/ref_rejected": -76.09590148925781, "logps/rejected": -86.61604309082031, "loss": 0.9808, "margin_dpo/margin_mean": 3.7187604904174805, "margin_dpo/margin_std": 5.548947811126709, "step": 169 }, { "KL/chosen_KL_mean": -6.976308822631836, "KL/mean": -8.671058654785156, "KL/rejected_KL_mean": -10.365806579589844, "KL/std": 4.641627788543701, "epoch": 0.25699168556311414, "fcm_dpo/beta": 0.23378986120224, "fcm_dpo/delta": 0.008020089939236641, "fcm_dpo/margin": 3.3894968032836914, "fcm_dpo/q_t": 0.3479105830192566, "grad_norm": 51.56340408325195, "learning_rate": 4.6449585330874425e-07, "logits/chosen": 0.2002856433391571, "logits/rejected": 0.19725605845451355, "logps/chosen": -63.62950134277344, "logps/ref_chosen": -56.65319061279297, "logps/ref_rejected": -63.45965576171875, "logps/rejected": -73.8254623413086, "loss": 1.0364, "margin_dpo/margin_mean": 3.3894965648651123, "margin_dpo/margin_std": 5.16367244720459, "step": 170 }, { "KL/chosen_KL_mean": -7.357227325439453, "KL/mean": -9.279874801635742, "KL/rejected_KL_mean": -11.202518463134766, "KL/std": 5.292311668395996, "epoch": 0.2585034013605442, "fcm_dpo/beta": 0.22743916511535645, "fcm_dpo/delta": -0.08598235249519348, "fcm_dpo/margin": 3.8452939987182617, "fcm_dpo/q_t": 0.33920013904571533, "grad_norm": 63.153053283691406, "learning_rate": 4.6381366244617224e-07, "logits/chosen": 0.23392340540885925, "logits/rejected": 0.18576276302337646, "logps/chosen": -71.09198760986328, "logps/ref_chosen": -63.73476028442383, "logps/ref_rejected": -78.50328063964844, "logps/rejected": -89.70579528808594, "loss": 1.0377, "margin_dpo/margin_mean": 3.845294237136841, "margin_dpo/margin_std": 5.816746711730957, "step": 171 }, { "KL/chosen_KL_mean": -7.918674468994141, "KL/mean": -9.873100280761719, "KL/rejected_KL_mean": -11.827526092529297, "KL/std": 5.193835258483887, "epoch": 0.2600151171579743, "fcm_dpo/beta": 0.22867974638938904, "fcm_dpo/delta": -0.10525374859571457, "fcm_dpo/margin": 3.9088492393493652, "fcm_dpo/q_t": 0.326849102973938, "grad_norm": 50.447669982910156, "learning_rate": 4.631254907558365e-07, "logits/chosen": 0.2409161925315857, "logits/rejected": 0.1868460476398468, "logps/chosen": -60.12043380737305, "logps/ref_chosen": -52.201759338378906, "logps/ref_rejected": -82.85285949707031, "logps/rejected": -94.68038940429688, "loss": 0.9546, "margin_dpo/margin_mean": 3.908848762512207, "margin_dpo/margin_std": 5.0718889236450195, "step": 172 }, { "KL/chosen_KL_mean": -7.773256301879883, "KL/mean": -9.733634948730469, "KL/rejected_KL_mean": -11.694019317626953, "KL/std": 5.298603057861328, "epoch": 0.2615268329554044, "fcm_dpo/beta": 0.21797175705432892, "fcm_dpo/delta": -0.06880420446395874, "fcm_dpo/margin": 3.92075777053833, "fcm_dpo/q_t": 0.353862464427948, "grad_norm": 52.90761184692383, "learning_rate": 4.624313574873786e-07, "logits/chosen": 0.2440553605556488, "logits/rejected": 0.15911364555358887, "logps/chosen": -63.20798110961914, "logps/ref_chosen": -55.434722900390625, "logps/ref_rejected": -77.81967163085938, "logps/rejected": -89.5136947631836, "loss": 1.1063, "margin_dpo/margin_mean": 3.920758008956909, "margin_dpo/margin_std": 6.446454048156738, "step": 173 }, { "KL/chosen_KL_mean": -8.353595733642578, "KL/mean": -10.422002792358398, "KL/rejected_KL_mean": -12.490409851074219, "KL/std": 5.196247100830078, "epoch": 0.26303854875283444, "fcm_dpo/beta": 0.2176172137260437, "fcm_dpo/delta": -0.11027532070875168, "fcm_dpo/margin": 4.136814594268799, "fcm_dpo/q_t": 0.3365994691848755, "grad_norm": 58.16306686401367, "learning_rate": 4.61731282057198e-07, "logits/chosen": 0.22505061328411102, "logits/rejected": 0.16084152460098267, "logps/chosen": -65.52555084228516, "logps/ref_chosen": -57.17195129394531, "logps/ref_rejected": -85.47578430175781, "logps/rejected": -97.9661865234375, "loss": 1.0065, "margin_dpo/margin_mean": 4.136814594268799, "margin_dpo/margin_std": 6.124249458312988, "step": 174 }, { "KL/chosen_KL_mean": -8.06877326965332, "KL/mean": -10.374343872070312, "KL/rejected_KL_mean": -12.679920196533203, "KL/std": 5.566990852355957, "epoch": 0.26455026455026454, "fcm_dpo/beta": 0.21098440885543823, "fcm_dpo/delta": -0.19120723009109497, "fcm_dpo/margin": 4.61114501953125, "fcm_dpo/q_t": 0.3338923156261444, "grad_norm": 56.7152099609375, "learning_rate": 4.6102528404790965e-07, "logits/chosen": 0.2686522901058197, "logits/rejected": 0.2384568750858307, "logps/chosen": -75.73440551757812, "logps/ref_chosen": -67.6656265258789, "logps/ref_rejected": -84.36766815185547, "logps/rejected": -97.04759216308594, "loss": 0.9772, "margin_dpo/margin_mean": 4.611144065856934, "margin_dpo/margin_std": 6.451984405517578, "step": 175 }, { "KL/chosen_KL_mean": -9.094661712646484, "KL/mean": -10.573260307312012, "KL/rejected_KL_mean": -12.051864624023438, "KL/std": 5.919432640075684, "epoch": 0.2660619803476946, "fcm_dpo/beta": 0.21295952796936035, "fcm_dpo/delta": 0.18204456567764282, "fcm_dpo/margin": 2.957207202911377, "fcm_dpo/q_t": 0.39002811908721924, "grad_norm": 72.97171783447266, "learning_rate": 4.603133832077953e-07, "logits/chosen": 0.1966366022825241, "logits/rejected": 0.17095749080181122, "logps/chosen": -86.95342254638672, "logps/ref_chosen": -77.8587646484375, "logps/ref_rejected": -81.08732604980469, "logps/rejected": -93.13919067382812, "loss": 1.2114, "margin_dpo/margin_mean": 2.9572067260742188, "margin_dpo/margin_std": 6.396285533905029, "step": 176 }, { "KL/chosen_KL_mean": -7.932289123535156, "KL/mean": -10.874521255493164, "KL/rejected_KL_mean": -13.816753387451172, "KL/std": 6.001509666442871, "epoch": 0.2675736961451247, "fcm_dpo/beta": 0.2011113166809082, "fcm_dpo/delta": -0.43956103920936584, "fcm_dpo/margin": 5.884462833404541, "fcm_dpo/q_t": 0.279480516910553, "grad_norm": 52.27606201171875, "learning_rate": 4.5959559945025183e-07, "logits/chosen": 0.3216002583503723, "logits/rejected": 0.22915410995483398, "logps/chosen": -63.152687072753906, "logps/ref_chosen": -55.22039794921875, "logps/ref_rejected": -92.54973602294922, "logps/rejected": -106.36648559570312, "loss": 0.7694, "margin_dpo/margin_mean": 5.884462356567383, "margin_dpo/margin_std": 5.901601314544678, "step": 177 }, { "KL/chosen_KL_mean": -8.448629379272461, "KL/mean": -10.219297409057617, "KL/rejected_KL_mean": -11.98996353149414, "KL/std": 5.255385875701904, "epoch": 0.2690854119425548, "fcm_dpo/beta": 0.1983993649482727, "fcm_dpo/delta": 0.1025046855211258, "fcm_dpo/margin": 3.541327476501465, "fcm_dpo/q_t": 0.36179715394973755, "grad_norm": 53.7444953918457, "learning_rate": 4.588719528532341e-07, "logits/chosen": 0.18969491124153137, "logits/rejected": 0.14258652925491333, "logps/chosen": -69.25912475585938, "logps/ref_chosen": -60.81049346923828, "logps/ref_rejected": -81.12973022460938, "logps/rejected": -93.11968994140625, "loss": 1.027, "margin_dpo/margin_mean": 3.541327476501465, "margin_dpo/margin_std": 5.07969331741333, "step": 178 }, { "KL/chosen_KL_mean": -9.205842971801758, "KL/mean": -11.216886520385742, "KL/rejected_KL_mean": -13.227928161621094, "KL/std": 5.487806797027588, "epoch": 0.2705971277399849, "fcm_dpo/beta": 0.20230549573898315, "fcm_dpo/delta": -0.015091773122549057, "fcm_dpo/margin": 4.022085189819336, "fcm_dpo/q_t": 0.3512629568576813, "grad_norm": 55.53172302246094, "learning_rate": 4.581424636586928e-07, "logits/chosen": 0.22318163514137268, "logits/rejected": 0.2072812020778656, "logps/chosen": -74.87755584716797, "logps/ref_chosen": -65.67171478271484, "logps/ref_rejected": -75.32586669921875, "logps/rejected": -88.55379486083984, "loss": 1.0564, "margin_dpo/margin_mean": 4.022086143493652, "margin_dpo/margin_std": 6.479291915893555, "step": 179 }, { "KL/chosen_KL_mean": -7.661951065063477, "KL/mean": -9.359560012817383, "KL/rejected_KL_mean": -11.057170867919922, "KL/std": 5.326980113983154, "epoch": 0.272108843537415, "fcm_dpo/beta": 0.20592835545539856, "fcm_dpo/delta": 0.10698837041854858, "fcm_dpo/margin": 3.3952207565307617, "fcm_dpo/q_t": 0.3744320273399353, "grad_norm": 57.96144104003906, "learning_rate": 4.5740715227200897e-07, "logits/chosen": 0.09134892374277115, "logits/rejected": 0.07213237881660461, "logps/chosen": -64.34475708007812, "logps/ref_chosen": -56.68280792236328, "logps/ref_rejected": -64.94414520263672, "logps/rejected": -76.0013198852539, "loss": 1.1279, "margin_dpo/margin_mean": 3.3952207565307617, "margin_dpo/margin_std": 6.209173679351807, "step": 180 }, { "KL/chosen_KL_mean": -7.21190071105957, "KL/mean": -9.833938598632812, "KL/rejected_KL_mean": -12.455978393554688, "KL/std": 5.954924583435059, "epoch": 0.273620559334845, "fcm_dpo/beta": 0.19779911637306213, "fcm_dpo/delta": -0.26495885848999023, "fcm_dpo/margin": 5.244076728820801, "fcm_dpo/q_t": 0.29986950755119324, "grad_norm": 46.34823226928711, "learning_rate": 4.566660392614228e-07, "logits/chosen": 0.25384521484375, "logits/rejected": 0.21548154950141907, "logps/chosen": -67.98794555664062, "logps/ref_chosen": -60.77604675292969, "logps/ref_rejected": -83.98361206054688, "logps/rejected": -96.43959045410156, "loss": 0.8039, "margin_dpo/margin_mean": 5.244076728820801, "margin_dpo/margin_std": 5.329998016357422, "step": 181 }, { "KL/chosen_KL_mean": -8.529552459716797, "KL/mean": -11.048151016235352, "KL/rejected_KL_mean": -13.566749572753906, "KL/std": 6.021827697753906, "epoch": 0.2751322751322751, "fcm_dpo/beta": 0.18954861164093018, "fcm_dpo/delta": -0.17215055227279663, "fcm_dpo/margin": 5.037204265594482, "fcm_dpo/q_t": 0.3292804956436157, "grad_norm": 52.639610290527344, "learning_rate": 4.5591914535745817e-07, "logits/chosen": 0.21993078291416168, "logits/rejected": 0.14366433024406433, "logps/chosen": -68.78334045410156, "logps/ref_chosen": -60.2537841796875, "logps/ref_rejected": -89.7706298828125, "logps/rejected": -103.3373794555664, "loss": 0.9768, "margin_dpo/margin_mean": 5.037204265594482, "margin_dpo/margin_std": 6.95863151550293, "step": 182 }, { "KL/chosen_KL_mean": -10.168844223022461, "KL/mean": -11.266523361206055, "KL/rejected_KL_mean": -12.364204406738281, "KL/std": 6.000180721282959, "epoch": 0.2766439909297052, "fcm_dpo/beta": 0.1989203244447708, "fcm_dpo/delta": 0.3810041546821594, "fcm_dpo/margin": 2.195363998413086, "fcm_dpo/q_t": 0.41308414936065674, "grad_norm": 58.42243194580078, "learning_rate": 4.551664914523433e-07, "logits/chosen": 0.21723271906375885, "logits/rejected": 0.19659434258937836, "logps/chosen": -71.93026733398438, "logps/ref_chosen": -61.76142120361328, "logps/ref_rejected": -72.54627990722656, "logps/rejected": -84.91049194335938, "loss": 1.2629, "margin_dpo/margin_mean": 2.195363759994507, "margin_dpo/margin_std": 5.551734924316406, "step": 183 }, { "KL/chosen_KL_mean": -7.7139739990234375, "KL/mean": -9.649314880371094, "KL/rejected_KL_mean": -11.584651947021484, "KL/std": 5.03806209564209, "epoch": 0.2781557067271353, "fcm_dpo/beta": 0.2014952301979065, "fcm_dpo/delta": 0.018574021756649017, "fcm_dpo/margin": 3.8706769943237305, "fcm_dpo/q_t": 0.3508874773979187, "grad_norm": 43.059913635253906, "learning_rate": 4.544080985994258e-07, "logits/chosen": 0.2943282425403595, "logits/rejected": 0.23392927646636963, "logps/chosen": -54.55469512939453, "logps/ref_chosen": -46.840721130371094, "logps/ref_rejected": -69.3609390258789, "logps/rejected": -80.94558715820312, "loss": 0.9506, "margin_dpo/margin_mean": 3.8706767559051514, "margin_dpo/margin_std": 4.824084281921387, "step": 184 }, { "KL/chosen_KL_mean": -8.752460479736328, "KL/mean": -10.798396110534668, "KL/rejected_KL_mean": -12.84432601928711, "KL/std": 5.784908294677734, "epoch": 0.2796674225245654, "fcm_dpo/beta": 0.20158517360687256, "fcm_dpo/delta": -0.029050692915916443, "fcm_dpo/margin": 4.091870307922363, "fcm_dpo/q_t": 0.3572160005569458, "grad_norm": 51.14906311035156, "learning_rate": 4.5364398801258394e-07, "logits/chosen": 0.2673199474811554, "logits/rejected": 0.22023963928222656, "logps/chosen": -61.07360076904297, "logps/ref_chosen": -52.32114028930664, "logps/ref_rejected": -68.3885726928711, "logps/rejected": -81.23289489746094, "loss": 1.1251, "margin_dpo/margin_mean": 4.091870307922363, "margin_dpo/margin_std": 7.132951736450195, "step": 185 }, { "KL/chosen_KL_mean": -8.121000289916992, "KL/mean": -10.394832611083984, "KL/rejected_KL_mean": -12.66866683959961, "KL/std": 5.741436958312988, "epoch": 0.2811791383219955, "fcm_dpo/beta": 0.19917798042297363, "fcm_dpo/delta": -0.1167006641626358, "fcm_dpo/margin": 4.547662734985352, "fcm_dpo/q_t": 0.33835506439208984, "grad_norm": 63.45903778076172, "learning_rate": 4.5287418106563354e-07, "logits/chosen": 0.15486830472946167, "logits/rejected": 0.11560823023319244, "logps/chosen": -75.54113006591797, "logps/ref_chosen": -67.42012786865234, "logps/ref_rejected": -82.50968933105469, "logps/rejected": -95.17835998535156, "loss": 1.0224, "margin_dpo/margin_mean": 4.547662734985352, "margin_dpo/margin_std": 6.847630023956299, "step": 186 }, { "KL/chosen_KL_mean": -9.027420043945312, "KL/mean": -11.191137313842773, "KL/rejected_KL_mean": -13.354854583740234, "KL/std": 5.968443393707275, "epoch": 0.28269085411942557, "fcm_dpo/beta": 0.19584915041923523, "fcm_dpo/delta": -0.05276474729180336, "fcm_dpo/margin": 4.327441692352295, "fcm_dpo/q_t": 0.34239810705184937, "grad_norm": 62.50242233276367, "learning_rate": 4.520986992917297e-07, "logits/chosen": 0.20313122868537903, "logits/rejected": 0.15035201609134674, "logps/chosen": -84.55291748046875, "logps/ref_chosen": -75.52549743652344, "logps/ref_rejected": -94.76289367675781, "logps/rejected": -108.11775207519531, "loss": 1.0374, "margin_dpo/margin_mean": 4.327441215515137, "margin_dpo/margin_std": 6.5522308349609375, "step": 187 }, { "KL/chosen_KL_mean": -8.111988067626953, "KL/mean": -10.224084854125977, "KL/rejected_KL_mean": -12.336181640625, "KL/std": 5.906611442565918, "epoch": 0.2842025699168556, "fcm_dpo/beta": 0.195459246635437, "fcm_dpo/delta": -0.02797466143965721, "fcm_dpo/margin": 4.224184036254883, "fcm_dpo/q_t": 0.34629908204078674, "grad_norm": 58.858394622802734, "learning_rate": 4.5131756438276466e-07, "logits/chosen": 0.21499189734458923, "logits/rejected": 0.17359653115272522, "logps/chosen": -79.63531494140625, "logps/ref_chosen": -71.52333068847656, "logps/ref_rejected": -78.29949951171875, "logps/rejected": -90.63568115234375, "loss": 1.0415, "margin_dpo/margin_mean": 4.224184036254883, "margin_dpo/margin_std": 6.645949363708496, "step": 188 }, { "KL/chosen_KL_mean": -7.738534927368164, "KL/mean": -10.014970779418945, "KL/rejected_KL_mean": -12.291412353515625, "KL/std": 5.990107536315918, "epoch": 0.2857142857142857, "fcm_dpo/beta": 0.1896684467792511, "fcm_dpo/delta": -0.07736320793628693, "fcm_dpo/margin": 4.552879333496094, "fcm_dpo/q_t": 0.33414921164512634, "grad_norm": 52.09397888183594, "learning_rate": 4.5053079818876096e-07, "logits/chosen": 0.22613701224327087, "logits/rejected": 0.23754778504371643, "logps/chosen": -79.91480255126953, "logps/ref_chosen": -72.17626953125, "logps/ref_rejected": -75.26313781738281, "logps/rejected": -87.55455017089844, "loss": 0.975, "margin_dpo/margin_mean": 4.55288028717041, "margin_dpo/margin_std": 5.972061634063721, "step": 189 }, { "KL/chosen_KL_mean": -7.69354248046875, "KL/mean": -10.335224151611328, "KL/rejected_KL_mean": -12.97690200805664, "KL/std": 5.906277656555176, "epoch": 0.2872260015117158, "fcm_dpo/beta": 0.1865214705467224, "fcm_dpo/delta": -0.20571252703666687, "fcm_dpo/margin": 5.283355712890625, "fcm_dpo/q_t": 0.3164942264556885, "grad_norm": 51.77168273925781, "learning_rate": 4.4973842271726024e-07, "logits/chosen": 0.28338149189949036, "logits/rejected": 0.1495637595653534, "logps/chosen": -62.317813873291016, "logps/ref_chosen": -54.624271392822266, "logps/ref_rejected": -101.47068786621094, "logps/rejected": -114.44758605957031, "loss": 0.8768, "margin_dpo/margin_mean": 5.283356189727783, "margin_dpo/margin_std": 6.187994480133057, "step": 190 }, { "KL/chosen_KL_mean": -9.158432006835938, "KL/mean": -11.346714973449707, "KL/rejected_KL_mean": -13.53499984741211, "KL/std": 6.011017799377441, "epoch": 0.2887377173091459, "fcm_dpo/beta": 0.1845850646495819, "fcm_dpo/delta": -0.008593167178332806, "fcm_dpo/margin": 4.376575469970703, "fcm_dpo/q_t": 0.3456087112426758, "grad_norm": 58.76239776611328, "learning_rate": 4.48940460132708e-07, "logits/chosen": 0.2690032124519348, "logits/rejected": 0.24326932430267334, "logps/chosen": -82.0909423828125, "logps/ref_chosen": -72.93251037597656, "logps/ref_rejected": -89.95103454589844, "logps/rejected": -103.48603820800781, "loss": 1.0117, "margin_dpo/margin_mean": 4.376575469970703, "margin_dpo/margin_std": 6.37799072265625, "step": 191 }, { "KL/chosen_KL_mean": -9.196332931518555, "KL/mean": -10.66668701171875, "KL/rejected_KL_mean": -12.137035369873047, "KL/std": 6.126029014587402, "epoch": 0.29024943310657597, "fcm_dpo/beta": 0.19202987849712372, "fcm_dpo/delta": 0.2476315051317215, "fcm_dpo/margin": 2.940706968307495, "fcm_dpo/q_t": 0.3896513879299164, "grad_norm": 46.325660705566406, "learning_rate": 4.481369327558329e-07, "logits/chosen": 0.2842397689819336, "logits/rejected": 0.2590170204639435, "logps/chosen": -63.197452545166016, "logps/ref_chosen": -54.001121520996094, "logps/ref_rejected": -63.531551361083984, "logps/rejected": -75.66858673095703, "loss": 1.145, "margin_dpo/margin_mean": 2.940706729888916, "margin_dpo/margin_std": 5.678344249725342, "step": 192 }, { "KL/chosen_KL_mean": -8.372961044311523, "KL/mean": -10.790081024169922, "KL/rejected_KL_mean": -13.207202911376953, "KL/std": 6.010296821594238, "epoch": 0.29176114890400606, "fcm_dpo/beta": 0.1883402019739151, "fcm_dpo/delta": -0.12426133453845978, "fcm_dpo/margin": 4.8342437744140625, "fcm_dpo/q_t": 0.3302828073501587, "grad_norm": 45.016563415527344, "learning_rate": 4.47327863063023e-07, "logits/chosen": 0.19597403705120087, "logits/rejected": 0.17333577573299408, "logps/chosen": -65.12223815917969, "logps/ref_chosen": -56.74927520751953, "logps/ref_rejected": -58.80629348754883, "logps/rejected": -72.01349639892578, "loss": 0.9163, "margin_dpo/margin_mean": 4.834243297576904, "margin_dpo/margin_std": 6.111197471618652, "step": 193 }, { "KL/chosen_KL_mean": -7.8690032958984375, "KL/mean": -9.918621063232422, "KL/rejected_KL_mean": -11.968246459960938, "KL/std": 5.435059070587158, "epoch": 0.29327286470143615, "fcm_dpo/beta": 0.18736335635185242, "fcm_dpo/delta": 0.03185667097568512, "fcm_dpo/margin": 4.099242210388184, "fcm_dpo/q_t": 0.3544465899467468, "grad_norm": 43.90078353881836, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 0.2552631199359894, "logits/rejected": 0.2272903025150299, "logps/chosen": -64.51844787597656, "logps/ref_chosen": -56.64944076538086, "logps/ref_rejected": -69.98954772949219, "logps/rejected": -81.95779418945312, "loss": 1.0746, "margin_dpo/margin_mean": 4.099242210388184, "margin_dpo/margin_std": 6.507394313812256, "step": 194 }, { "KL/chosen_KL_mean": -9.42181396484375, "KL/mean": -11.70624828338623, "KL/rejected_KL_mean": -13.990684509277344, "KL/std": 6.544086456298828, "epoch": 0.2947845804988662, "fcm_dpo/beta": 0.1866236925125122, "fcm_dpo/delta": -0.059793997555971146, "fcm_dpo/margin": 4.568871974945068, "fcm_dpo/q_t": 0.338507741689682, "grad_norm": 51.90018081665039, "learning_rate": 4.4569318740967043e-07, "logits/chosen": 0.14530959725379944, "logits/rejected": 0.14585906267166138, "logps/chosen": -79.83158874511719, "logps/ref_chosen": -70.40977478027344, "logps/ref_rejected": -74.39448547363281, "logps/rejected": -88.38516998291016, "loss": 0.9728, "margin_dpo/margin_mean": 4.568871974945068, "margin_dpo/margin_std": 6.183128356933594, "step": 195 }, { "KL/chosen_KL_mean": -8.454660415649414, "KL/mean": -10.585471153259277, "KL/rejected_KL_mean": -12.71627426147461, "KL/std": 6.18372917175293, "epoch": 0.2962962962962963, "fcm_dpo/beta": 0.18937581777572632, "fcm_dpo/delta": -0.009729728102684021, "fcm_dpo/margin": 4.261617183685303, "fcm_dpo/q_t": 0.33870434761047363, "grad_norm": 41.523197174072266, "learning_rate": 4.448676271745197e-07, "logits/chosen": 0.25778093934059143, "logits/rejected": 0.21833831071853638, "logps/chosen": -67.68223571777344, "logps/ref_chosen": -59.227577209472656, "logps/ref_rejected": -83.54757690429688, "logps/rejected": -96.26384735107422, "loss": 0.9843, "margin_dpo/margin_mean": 4.261617183685303, "margin_dpo/margin_std": 5.713452339172363, "step": 196 }, { "KL/chosen_KL_mean": -8.017414093017578, "KL/mean": -10.4995698928833, "KL/rejected_KL_mean": -12.981727600097656, "KL/std": 6.454028606414795, "epoch": 0.29780801209372637, "fcm_dpo/beta": 0.18606367707252502, "fcm_dpo/delta": -0.13813000917434692, "fcm_dpo/margin": 4.9643073081970215, "fcm_dpo/q_t": 0.3351513743400574, "grad_norm": 48.245933532714844, "learning_rate": 4.440366160729392e-07, "logits/chosen": 0.3265201151371002, "logits/rejected": 0.2801608443260193, "logps/chosen": -59.546546936035156, "logps/ref_chosen": -51.52912902832031, "logps/ref_rejected": -73.70631408691406, "logps/rejected": -86.68803405761719, "loss": 1.0636, "margin_dpo/margin_mean": 4.964306831359863, "margin_dpo/margin_std": 7.59263801574707, "step": 197 }, { "KL/chosen_KL_mean": -8.19921875, "KL/mean": -10.83227252960205, "KL/rejected_KL_mean": -13.465335845947266, "KL/std": 5.659462928771973, "epoch": 0.29931972789115646, "fcm_dpo/beta": 0.17989234626293182, "fcm_dpo/delta": -0.1634722501039505, "fcm_dpo/margin": 5.266110420227051, "fcm_dpo/q_t": 0.31562328338623047, "grad_norm": 43.244449615478516, "learning_rate": 4.432001773500957e-07, "logits/chosen": 0.30426979064941406, "logits/rejected": 0.26571154594421387, "logps/chosen": -67.98190307617188, "logps/ref_chosen": -59.78268051147461, "logps/ref_rejected": -72.24533081054688, "logps/rejected": -85.71066284179688, "loss": 0.8725, "margin_dpo/margin_mean": 5.266110420227051, "margin_dpo/margin_std": 5.919608116149902, "step": 198 }, { "KL/chosen_KL_mean": -8.999025344848633, "KL/mean": -11.057474136352539, "KL/rejected_KL_mean": -13.115921020507812, "KL/std": 5.817185878753662, "epoch": 0.30083144368858655, "fcm_dpo/beta": 0.1805601716041565, "fcm_dpo/delta": 0.05563047528266907, "fcm_dpo/margin": 4.116896629333496, "fcm_dpo/q_t": 0.35699188709259033, "grad_norm": 49.35909652709961, "learning_rate": 4.4235833440297856e-07, "logits/chosen": 0.23536258935928345, "logits/rejected": 0.15354374051094055, "logps/chosen": -65.38579559326172, "logps/ref_chosen": -56.38677215576172, "logps/ref_rejected": -74.56779479980469, "logps/rejected": -87.6837158203125, "loss": 1.1103, "margin_dpo/margin_mean": 4.116896152496338, "margin_dpo/margin_std": 6.822243690490723, "step": 199 }, { "KL/chosen_KL_mean": -7.210149765014648, "KL/mean": -10.162313461303711, "KL/rejected_KL_mean": -13.114479064941406, "KL/std": 6.341808795928955, "epoch": 0.30234315948601664, "fcm_dpo/beta": 0.17075824737548828, "fcm_dpo/delta": -0.23531164228916168, "fcm_dpo/margin": 5.9043288230896, "fcm_dpo/q_t": 0.31821250915527344, "grad_norm": 43.43456268310547, "learning_rate": 4.415111107797445e-07, "logits/chosen": 0.29511919617652893, "logits/rejected": 0.22751715779304504, "logps/chosen": -65.03447723388672, "logps/ref_chosen": -57.82432556152344, "logps/ref_rejected": -89.28246307373047, "logps/rejected": -102.39694213867188, "loss": 0.9253, "margin_dpo/margin_mean": 5.904328346252441, "margin_dpo/margin_std": 7.588106155395508, "step": 200 }, { "KL/chosen_KL_mean": -9.282886505126953, "KL/mean": -11.540687561035156, "KL/rejected_KL_mean": -13.798492431640625, "KL/std": 6.509415626525879, "epoch": 0.30385487528344673, "fcm_dpo/beta": 0.17012134194374084, "fcm_dpo/delta": 0.034042563289403915, "fcm_dpo/margin": 4.515607833862305, "fcm_dpo/q_t": 0.35858190059661865, "grad_norm": 52.089141845703125, "learning_rate": 4.4065853017905953e-07, "logits/chosen": 0.31442439556121826, "logits/rejected": 0.27165716886520386, "logps/chosen": -68.28264617919922, "logps/ref_chosen": -58.999759674072266, "logps/ref_rejected": -84.67575073242188, "logps/rejected": -98.4742431640625, "loss": 1.0222, "margin_dpo/margin_mean": 4.515607833862305, "margin_dpo/margin_std": 6.753261566162109, "step": 201 }, { "KL/chosen_KL_mean": -7.988605499267578, "KL/mean": -10.506321907043457, "KL/rejected_KL_mean": -13.024032592773438, "KL/std": 6.282382011413574, "epoch": 0.30536659108087677, "fcm_dpo/beta": 0.1711854636669159, "fcm_dpo/delta": -0.06901469826698303, "fcm_dpo/margin": 5.035429000854492, "fcm_dpo/q_t": 0.3341776430606842, "grad_norm": 40.801334381103516, "learning_rate": 4.3980061644943575e-07, "logits/chosen": 0.19821295142173767, "logits/rejected": 0.1341780424118042, "logps/chosen": -55.649253845214844, "logps/ref_chosen": -47.660648345947266, "logps/ref_rejected": -73.63249969482422, "logps/rejected": -86.65653228759766, "loss": 0.9692, "margin_dpo/margin_mean": 5.035429000854492, "margin_dpo/margin_std": 6.7977294921875, "step": 202 }, { "KL/chosen_KL_mean": -8.52560043334961, "KL/mean": -10.693042755126953, "KL/rejected_KL_mean": -12.860485076904297, "KL/std": 6.2385053634643555, "epoch": 0.30687830687830686, "fcm_dpo/beta": 0.17108216881752014, "fcm_dpo/delta": 0.06265536695718765, "fcm_dpo/margin": 4.334885597229004, "fcm_dpo/q_t": 0.3605703115463257, "grad_norm": 49.72651290893555, "learning_rate": 4.3893739358856455e-07, "logits/chosen": 0.29747486114501953, "logits/rejected": 0.22976483404636383, "logps/chosen": -70.85113525390625, "logps/ref_chosen": -62.32553482055664, "logps/ref_rejected": -99.37226104736328, "logps/rejected": -112.23274230957031, "loss": 1.0333, "margin_dpo/margin_mean": 4.334885597229004, "margin_dpo/margin_std": 6.700814723968506, "step": 203 }, { "KL/chosen_KL_mean": -8.656208038330078, "KL/mean": -11.099645614624023, "KL/rejected_KL_mean": -13.543079376220703, "KL/std": 6.6452507972717285, "epoch": 0.30839002267573695, "fcm_dpo/beta": 0.16719385981559753, "fcm_dpo/delta": -0.026872709393501282, "fcm_dpo/margin": 4.886868953704834, "fcm_dpo/q_t": 0.3485722541809082, "grad_norm": 41.345577239990234, "learning_rate": 4.380688857426449e-07, "logits/chosen": 0.22623693943023682, "logits/rejected": 0.16254332661628723, "logps/chosen": -59.2855224609375, "logps/ref_chosen": -50.62931823730469, "logps/ref_rejected": -66.60475158691406, "logps/rejected": -80.1478271484375, "loss": 1.0118, "margin_dpo/margin_mean": 4.886869430541992, "margin_dpo/margin_std": 6.901353359222412, "step": 204 }, { "KL/chosen_KL_mean": -9.14675521850586, "KL/mean": -11.405117988586426, "KL/rejected_KL_mean": -13.66347885131836, "KL/std": 6.326295852661133, "epoch": 0.30990173847316704, "fcm_dpo/beta": 0.16965240240097046, "fcm_dpo/delta": 0.03534369170665741, "fcm_dpo/margin": 4.516724109649658, "fcm_dpo/q_t": 0.3603339195251465, "grad_norm": 61.0053596496582, "learning_rate": 4.3719511720570814e-07, "logits/chosen": 0.2996768355369568, "logits/rejected": 0.24160867929458618, "logps/chosen": -79.5029296875, "logps/ref_chosen": -70.3561782836914, "logps/ref_rejected": -93.39848327636719, "logps/rejected": -107.06195831298828, "loss": 1.1121, "margin_dpo/margin_mean": 4.5167236328125, "margin_dpo/margin_std": 7.951142311096191, "step": 205 }, { "KL/chosen_KL_mean": -9.403274536132812, "KL/mean": -11.332551956176758, "KL/rejected_KL_mean": -13.261837005615234, "KL/std": 6.498051166534424, "epoch": 0.31141345427059713, "fcm_dpo/beta": 0.17620447278022766, "fcm_dpo/delta": 0.12605662643909454, "fcm_dpo/margin": 3.85856032371521, "fcm_dpo/q_t": 0.37659770250320435, "grad_norm": 51.65834426879883, "learning_rate": 4.363161124189387e-07, "logits/chosen": 0.30237025022506714, "logits/rejected": 0.28653794527053833, "logps/chosen": -77.04875183105469, "logps/ref_chosen": -67.64547729492188, "logps/ref_rejected": -79.89584350585938, "logps/rejected": -93.15768432617188, "loss": 1.1709, "margin_dpo/margin_mean": 3.858560562133789, "margin_dpo/margin_std": 7.424094200134277, "step": 206 }, { "KL/chosen_KL_mean": -10.225160598754883, "KL/mean": -12.855207443237305, "KL/rejected_KL_mean": -15.48525619506836, "KL/std": 6.979071140289307, "epoch": 0.3129251700680272, "fcm_dpo/beta": 0.17152279615402222, "fcm_dpo/delta": -0.11443052440881729, "fcm_dpo/margin": 5.260091781616211, "fcm_dpo/q_t": 0.3384243845939636, "grad_norm": 42.885921478271484, "learning_rate": 4.3543189596998986e-07, "logits/chosen": 0.22064357995986938, "logits/rejected": 0.1584734171628952, "logps/chosen": -77.88935852050781, "logps/ref_chosen": -67.66419219970703, "logps/ref_rejected": -85.10249328613281, "logps/rejected": -100.5877456665039, "loss": 0.9697, "margin_dpo/margin_mean": 5.260092735290527, "margin_dpo/margin_std": 7.317288398742676, "step": 207 }, { "KL/chosen_KL_mean": -8.486841201782227, "KL/mean": -10.045097351074219, "KL/rejected_KL_mean": -11.603355407714844, "KL/std": 6.03558349609375, "epoch": 0.3144368858654573, "fcm_dpo/beta": 0.17885954678058624, "fcm_dpo/delta": 0.2562446892261505, "fcm_dpo/margin": 3.1165127754211426, "fcm_dpo/q_t": 0.3973674178123474, "grad_norm": 51.5666389465332, "learning_rate": 4.3454249259229664e-07, "logits/chosen": 0.23146170377731323, "logits/rejected": 0.20724132657051086, "logps/chosen": -66.21855163574219, "logps/ref_chosen": -57.731712341308594, "logps/ref_rejected": -74.19276428222656, "logps/rejected": -85.7961196899414, "loss": 1.2168, "margin_dpo/margin_mean": 3.1165130138397217, "margin_dpo/margin_std": 6.801269054412842, "step": 208 }, { "KL/chosen_KL_mean": -8.83671760559082, "KL/mean": -11.54526138305664, "KL/rejected_KL_mean": -14.25381088256836, "KL/std": 6.972295761108398, "epoch": 0.31594860166288735, "fcm_dpo/beta": 0.17789439857006073, "fcm_dpo/delta": -0.18146634101867676, "fcm_dpo/margin": 5.417089462280273, "fcm_dpo/q_t": 0.3251495361328125, "grad_norm": 53.9567756652832, "learning_rate": 4.336479271643833e-07, "logits/chosen": 0.20470848679542542, "logits/rejected": 0.15327095985412598, "logps/chosen": -77.38679504394531, "logps/ref_chosen": -68.55007934570312, "logps/ref_rejected": -87.90541076660156, "logps/rejected": -102.15922546386719, "loss": 1.001, "margin_dpo/margin_mean": 5.417089939117432, "margin_dpo/margin_std": 7.826272010803223, "step": 209 }, { "KL/chosen_KL_mean": -8.892902374267578, "KL/mean": -11.775569915771484, "KL/rejected_KL_mean": -14.65823745727539, "KL/std": 7.52336311340332, "epoch": 0.31746031746031744, "fcm_dpo/beta": 0.16959837079048157, "fcm_dpo/delta": -0.19686026871204376, "fcm_dpo/margin": 5.765329360961914, "fcm_dpo/q_t": 0.33063024282455444, "grad_norm": 41.87021255493164, "learning_rate": 4.327482247091679e-07, "logits/chosen": 0.2841176688671112, "logits/rejected": 0.1984454244375229, "logps/chosen": -66.16117858886719, "logps/ref_chosen": -57.268272399902344, "logps/ref_rejected": -85.72807312011719, "logps/rejected": -100.38631439208984, "loss": 0.9738, "margin_dpo/margin_mean": 5.765328407287598, "margin_dpo/margin_std": 8.068181991577148, "step": 210 }, { "KL/chosen_KL_mean": -8.13759994506836, "KL/mean": -10.518904685974121, "KL/rejected_KL_mean": -12.90020751953125, "KL/std": 6.33951997756958, "epoch": 0.31897203325774753, "fcm_dpo/beta": 0.16893050074577332, "fcm_dpo/delta": -0.006408661603927612, "fcm_dpo/margin": 4.7626051902771, "fcm_dpo/q_t": 0.34970852732658386, "grad_norm": 51.645408630371094, "learning_rate": 4.3184341039326217e-07, "logits/chosen": 0.33264777064323425, "logits/rejected": 0.24676403403282166, "logps/chosen": -61.77831268310547, "logps/ref_chosen": -53.640708923339844, "logps/ref_rejected": -93.0387954711914, "logps/rejected": -105.93900299072266, "loss": 0.978, "margin_dpo/margin_mean": 4.762604236602783, "margin_dpo/margin_std": 6.6331682205200195, "step": 211 }, { "KL/chosen_KL_mean": -9.244085311889648, "KL/mean": -11.903303146362305, "KL/rejected_KL_mean": -14.562515258789062, "KL/std": 6.864345550537109, "epoch": 0.3204837490551776, "fcm_dpo/beta": 0.16524195671081543, "fcm_dpo/delta": -0.08646350353956223, "fcm_dpo/margin": 5.318434238433838, "fcm_dpo/q_t": 0.3367248773574829, "grad_norm": 43.68009948730469, "learning_rate": 4.309335095262675e-07, "logits/chosen": 0.2854962944984436, "logits/rejected": 0.2196652889251709, "logps/chosen": -66.61082458496094, "logps/ref_chosen": -57.36674499511719, "logps/ref_rejected": -79.89643096923828, "logps/rejected": -94.45894622802734, "loss": 0.9843, "margin_dpo/margin_mean": 5.3184332847595215, "margin_dpo/margin_std": 7.531092643737793, "step": 212 }, { "KL/chosen_KL_mean": -7.1361083984375, "KL/mean": -10.112091064453125, "KL/rejected_KL_mean": -13.088077545166016, "KL/std": 6.956264495849609, "epoch": 0.3219954648526077, "fcm_dpo/beta": 0.15949341654777527, "fcm_dpo/delta": -0.1670059859752655, "fcm_dpo/margin": 5.951968193054199, "fcm_dpo/q_t": 0.3250270485877991, "grad_norm": 41.68220520019531, "learning_rate": 4.3001854756006724e-07, "logits/chosen": 0.29428863525390625, "logits/rejected": 0.2708255648612976, "logps/chosen": -72.35722351074219, "logps/ref_chosen": -65.22111511230469, "logps/ref_rejected": -80.1810302734375, "logps/rejected": -93.26910400390625, "loss": 0.9682, "margin_dpo/margin_mean": 5.951968193054199, "margin_dpo/margin_std": 8.158895492553711, "step": 213 }, { "KL/chosen_KL_mean": -8.329187393188477, "KL/mean": -11.186603546142578, "KL/rejected_KL_mean": -14.044015884399414, "KL/std": 6.9872565269470215, "epoch": 0.3235071806500378, "fcm_dpo/beta": 0.15824972093105316, "fcm_dpo/delta": -0.11610302329063416, "fcm_dpo/margin": 5.714832305908203, "fcm_dpo/q_t": 0.3369448184967041, "grad_norm": 47.012935638427734, "learning_rate": 4.290985500881143e-07, "logits/chosen": 0.1531170755624771, "logits/rejected": 0.1307399570941925, "logps/chosen": -69.62152099609375, "logps/ref_chosen": -61.292327880859375, "logps/ref_rejected": -67.69841003417969, "logps/rejected": -81.74242401123047, "loss": 0.9658, "margin_dpo/margin_mean": 5.714832305908203, "margin_dpo/margin_std": 7.725873947143555, "step": 214 }, { "KL/chosen_KL_mean": -9.156429290771484, "KL/mean": -12.22635555267334, "KL/rejected_KL_mean": -15.296276092529297, "KL/std": 7.057165145874023, "epoch": 0.3250188964474679, "fcm_dpo/beta": 0.15315330028533936, "fcm_dpo/delta": -0.15492382645606995, "fcm_dpo/margin": 6.139848709106445, "fcm_dpo/q_t": 0.32713302969932556, "grad_norm": 42.17342758178711, "learning_rate": 4.281735428447157e-07, "logits/chosen": 0.1951332986354828, "logits/rejected": 0.10021235793828964, "logps/chosen": -73.02556610107422, "logps/ref_chosen": -63.869136810302734, "logps/ref_rejected": -98.7657241821289, "logps/rejected": -114.06199645996094, "loss": 0.9299, "margin_dpo/margin_mean": 6.139849662780762, "margin_dpo/margin_std": 7.7090067863464355, "step": 215 }, { "KL/chosen_KL_mean": -8.983440399169922, "KL/mean": -12.1254301071167, "KL/rejected_KL_mean": -15.267414093017578, "KL/std": 7.741682052612305, "epoch": 0.32653061224489793, "fcm_dpo/beta": 0.1480444371700287, "fcm_dpo/delta": -0.14343297481536865, "fcm_dpo/margin": 6.283971786499023, "fcm_dpo/q_t": 0.3363327980041504, "grad_norm": 37.66606903076172, "learning_rate": 4.2724355170431247e-07, "logits/chosen": 0.33266764879226685, "logits/rejected": 0.2524465322494507, "logps/chosen": -76.80839538574219, "logps/ref_chosen": -67.824951171875, "logps/ref_rejected": -96.40231323242188, "logps/rejected": -111.66973114013672, "loss": 0.9613, "margin_dpo/margin_mean": 6.283972263336182, "margin_dpo/margin_std": 9.071916580200195, "step": 216 }, { "KL/chosen_KL_mean": -9.532360076904297, "KL/mean": -12.935258865356445, "KL/rejected_KL_mean": -16.33816146850586, "KL/std": 7.542253017425537, "epoch": 0.328042328042328, "fcm_dpo/beta": 0.14228859543800354, "fcm_dpo/delta": -0.18680410087108612, "fcm_dpo/margin": 6.8057990074157715, "fcm_dpo/q_t": 0.3188336491584778, "grad_norm": 36.55585861206055, "learning_rate": 4.26308602680756e-07, "logits/chosen": 0.24256332218647003, "logits/rejected": 0.1458836793899536, "logps/chosen": -70.037353515625, "logps/ref_chosen": -60.5049934387207, "logps/ref_rejected": -84.26618194580078, "logps/rejected": -100.60433959960938, "loss": 0.865, "margin_dpo/margin_mean": 6.805798530578613, "margin_dpo/margin_std": 7.959721565246582, "step": 217 }, { "KL/chosen_KL_mean": -9.821697235107422, "KL/mean": -11.902130126953125, "KL/rejected_KL_mean": -13.982563018798828, "KL/std": 6.682396411895752, "epoch": 0.3295540438397581, "fcm_dpo/beta": 0.14719802141189575, "fcm_dpo/delta": 0.19345054030418396, "fcm_dpo/margin": 4.160863876342773, "fcm_dpo/q_t": 0.3849286139011383, "grad_norm": 44.22843551635742, "learning_rate": 4.253687219265803e-07, "logits/chosen": 0.16697943210601807, "logits/rejected": 0.1599193960428238, "logps/chosen": -80.416015625, "logps/ref_chosen": -70.59431457519531, "logps/ref_rejected": -73.89038848876953, "logps/rejected": -87.87295532226562, "loss": 1.2018, "margin_dpo/margin_mean": 4.160863876342773, "margin_dpo/margin_std": 8.526304244995117, "step": 218 }, { "KL/chosen_KL_mean": -9.010517120361328, "KL/mean": -10.984228134155273, "KL/rejected_KL_mean": -12.957944869995117, "KL/std": 7.001385688781738, "epoch": 0.3310657596371882, "fcm_dpo/beta": 0.1508154571056366, "fcm_dpo/delta": 0.2181931436061859, "fcm_dpo/margin": 3.9474315643310547, "fcm_dpo/q_t": 0.38393646478652954, "grad_norm": 43.46605682373047, "learning_rate": 4.2442393573227043e-07, "logits/chosen": 0.21026837825775146, "logits/rejected": 0.17155671119689941, "logps/chosen": -69.50146484375, "logps/ref_chosen": -60.490943908691406, "logps/ref_rejected": -75.85001373291016, "logps/rejected": -88.80795288085938, "loss": 1.097, "margin_dpo/margin_mean": 3.9474315643310547, "margin_dpo/margin_std": 6.710477352142334, "step": 219 }, { "KL/chosen_KL_mean": -8.571613311767578, "KL/mean": -11.141824722290039, "KL/rejected_KL_mean": -13.7120361328125, "KL/std": 7.094627380371094, "epoch": 0.3325774754346183, "fcm_dpo/beta": 0.1525796800851822, "fcm_dpo/delta": 0.016179494559764862, "fcm_dpo/margin": 5.140424728393555, "fcm_dpo/q_t": 0.3578713536262512, "grad_norm": 36.82176208496094, "learning_rate": 4.234742705255272e-07, "logits/chosen": 0.30773574113845825, "logits/rejected": 0.2503643333911896, "logps/chosen": -53.58501434326172, "logps/ref_chosen": -45.013397216796875, "logps/ref_rejected": -70.49369812011719, "logps/rejected": -84.20573425292969, "loss": 1.0369, "margin_dpo/margin_mean": 5.140424728393555, "margin_dpo/margin_std": 7.912476539611816, "step": 220 }, { "KL/chosen_KL_mean": -8.029104232788086, "KL/mean": -10.780109405517578, "KL/rejected_KL_mean": -13.531108856201172, "KL/std": 6.923809051513672, "epoch": 0.3340891912320484, "fcm_dpo/beta": 0.15349453687667847, "fcm_dpo/delta": -0.04903030022978783, "fcm_dpo/margin": 5.502012252807617, "fcm_dpo/q_t": 0.3516319692134857, "grad_norm": 39.72418975830078, "learning_rate": 4.22519752870528e-07, "logits/chosen": 0.2654990255832672, "logits/rejected": 0.20310282707214355, "logps/chosen": -67.12493896484375, "logps/ref_chosen": -59.09584045410156, "logps/ref_rejected": -88.64388275146484, "logps/rejected": -102.17498779296875, "loss": 1.0059, "margin_dpo/margin_mean": 5.502011775970459, "margin_dpo/margin_std": 8.320975303649902, "step": 221 }, { "KL/chosen_KL_mean": -8.494781494140625, "KL/mean": -12.023693084716797, "KL/rejected_KL_mean": -15.552600860595703, "KL/std": 8.078587532043457, "epoch": 0.3356009070294785, "fcm_dpo/beta": 0.14657087624073029, "fcm_dpo/delta": -0.26139819622039795, "fcm_dpo/margin": 7.057823181152344, "fcm_dpo/q_t": 0.3049485385417938, "grad_norm": 38.150306701660156, "learning_rate": 4.2156040946718343e-07, "logits/chosen": 0.3249172568321228, "logits/rejected": 0.24639061093330383, "logps/chosen": -64.49247741699219, "logps/ref_chosen": -55.9976921081543, "logps/ref_rejected": -111.94727325439453, "logps/rejected": -127.4998779296875, "loss": 0.8544, "margin_dpo/margin_mean": 7.057823181152344, "margin_dpo/margin_std": 7.885249137878418, "step": 222 }, { "KL/chosen_KL_mean": -8.748092651367188, "KL/mean": -12.026680946350098, "KL/rejected_KL_mean": -15.305274963378906, "KL/std": 7.710515975952148, "epoch": 0.3371126228269085, "fcm_dpo/beta": 0.1403246521949768, "fcm_dpo/delta": -0.13532818853855133, "fcm_dpo/margin": 6.557183742523193, "fcm_dpo/q_t": 0.3230859637260437, "grad_norm": 33.81869125366211, "learning_rate": 4.2059626715039065e-07, "logits/chosen": 0.285654753446579, "logits/rejected": 0.23378512263298035, "logps/chosen": -68.63951110839844, "logps/ref_chosen": -59.891422271728516, "logps/ref_rejected": -86.28954315185547, "logps/rejected": -101.59481811523438, "loss": 0.8742, "margin_dpo/margin_mean": 6.557183265686035, "margin_dpo/margin_std": 7.321591377258301, "step": 223 }, { "KL/chosen_KL_mean": -9.40049934387207, "KL/mean": -11.442483901977539, "KL/rejected_KL_mean": -13.484466552734375, "KL/std": 7.180830001831055, "epoch": 0.3386243386243386, "fcm_dpo/beta": 0.1448686271905899, "fcm_dpo/delta": 0.22197052836418152, "fcm_dpo/margin": 4.0839691162109375, "fcm_dpo/q_t": 0.38782450556755066, "grad_norm": 42.53331756591797, "learning_rate": 4.1962735288928304e-07, "logits/chosen": 0.3183774948120117, "logits/rejected": 0.2993980050086975, "logps/chosen": -73.44513702392578, "logps/ref_chosen": -64.04463195800781, "logps/ref_rejected": -75.05450439453125, "logps/rejected": -88.53897094726562, "loss": 1.081, "margin_dpo/margin_mean": 4.0839691162109375, "margin_dpo/margin_std": 6.880286693572998, "step": 224 }, { "KL/chosen_KL_mean": -9.292135238647461, "KL/mean": -12.396595001220703, "KL/rejected_KL_mean": -15.501056671142578, "KL/std": 8.57298469543457, "epoch": 0.3401360544217687, "fcm_dpo/beta": 0.143642395734787, "fcm_dpo/delta": -0.10299123823642731, "fcm_dpo/margin": 6.20892333984375, "fcm_dpo/q_t": 0.33599036931991577, "grad_norm": 49.038352966308594, "learning_rate": 4.186536937864752e-07, "logits/chosen": 0.31163620948791504, "logits/rejected": 0.20546850562095642, "logps/chosen": -75.38795471191406, "logps/ref_chosen": -66.0958251953125, "logps/ref_rejected": -97.68675231933594, "logps/rejected": -113.18780517578125, "loss": 0.9474, "margin_dpo/margin_mean": 6.20892333984375, "margin_dpo/margin_std": 8.138298988342285, "step": 225 }, { "KL/chosen_KL_mean": -8.449029922485352, "KL/mean": -11.095566749572754, "KL/rejected_KL_mean": -13.742103576660156, "KL/std": 7.645484447479248, "epoch": 0.3416477702191988, "fcm_dpo/beta": 0.144595205783844, "fcm_dpo/delta": 0.03752833604812622, "fcm_dpo/margin": 5.293073654174805, "fcm_dpo/q_t": 0.35609138011932373, "grad_norm": 34.39674377441406, "learning_rate": 4.176753170773052e-07, "logits/chosen": 0.3167535364627838, "logits/rejected": 0.27241265773773193, "logps/chosen": -59.865901947021484, "logps/ref_chosen": -51.4168701171875, "logps/ref_rejected": -66.30068969726562, "logps/rejected": -80.04278564453125, "loss": 1.043, "margin_dpo/margin_mean": 5.293073654174805, "margin_dpo/margin_std": 8.19736099243164, "step": 226 }, { "KL/chosen_KL_mean": -8.788469314575195, "KL/mean": -11.457413673400879, "KL/rejected_KL_mean": -14.126358032226562, "KL/std": 8.207555770874023, "epoch": 0.3431594860166289, "fcm_dpo/beta": 0.1451629102230072, "fcm_dpo/delta": 0.027083417400717735, "fcm_dpo/margin": 5.337895393371582, "fcm_dpo/q_t": 0.36309635639190674, "grad_norm": 46.18406295776367, "learning_rate": 4.166922501290729e-07, "logits/chosen": 0.3289709687232971, "logits/rejected": 0.29214709997177124, "logps/chosen": -66.77824401855469, "logps/ref_chosen": -57.989776611328125, "logps/ref_rejected": -75.05464172363281, "logps/rejected": -89.18099975585938, "loss": 1.0797, "margin_dpo/margin_mean": 5.337895393371582, "margin_dpo/margin_std": 8.865333557128906, "step": 227 }, { "KL/chosen_KL_mean": -9.585563659667969, "KL/mean": -12.280923843383789, "KL/rejected_KL_mean": -14.97628402709961, "KL/std": 7.654868125915527, "epoch": 0.34467120181405897, "fcm_dpo/beta": 0.14699329435825348, "fcm_dpo/delta": 0.007324397563934326, "fcm_dpo/margin": 5.390713214874268, "fcm_dpo/q_t": 0.3541639447212219, "grad_norm": 37.86101150512695, "learning_rate": 4.1570452044027405e-07, "logits/chosen": 0.3056115508079529, "logits/rejected": 0.23478630185127258, "logps/chosen": -65.14492797851562, "logps/ref_chosen": -55.55936813354492, "logps/ref_rejected": -77.02364349365234, "logps/rejected": -91.99992370605469, "loss": 1.0464, "margin_dpo/margin_mean": 5.390713691711426, "margin_dpo/margin_std": 8.415397644042969, "step": 228 }, { "KL/chosen_KL_mean": -8.10175895690918, "KL/mean": -10.556568145751953, "KL/rejected_KL_mean": -13.011375427246094, "KL/std": 7.001654148101807, "epoch": 0.34618291761148906, "fcm_dpo/beta": 0.14782226085662842, "fcm_dpo/delta": 0.08006762713193893, "fcm_dpo/margin": 4.909615516662598, "fcm_dpo/q_t": 0.35694581270217896, "grad_norm": 102.48412322998047, "learning_rate": 4.147121556398312e-07, "logits/chosen": 0.38529714941978455, "logits/rejected": 0.32762086391448975, "logps/chosen": -58.89642333984375, "logps/ref_chosen": -50.79466247558594, "logps/ref_rejected": -78.4474105834961, "logps/rejected": -91.45878601074219, "loss": 1.0866, "margin_dpo/margin_mean": 4.909615516662598, "margin_dpo/margin_std": 8.180277824401855, "step": 229 }, { "KL/chosen_KL_mean": -8.855535507202148, "KL/mean": -11.712057113647461, "KL/rejected_KL_mean": -14.568580627441406, "KL/std": 7.761747360229492, "epoch": 0.3476946334089191, "fcm_dpo/beta": 0.14939934015274048, "fcm_dpo/delta": -0.0625389814376831, "fcm_dpo/margin": 5.713042259216309, "fcm_dpo/q_t": 0.3402790129184723, "grad_norm": 41.512351989746094, "learning_rate": 4.137151834863213e-07, "logits/chosen": 0.27887076139450073, "logits/rejected": 0.2750973701477051, "logps/chosen": -65.58476257324219, "logps/ref_chosen": -56.729225158691406, "logps/ref_rejected": -62.99180603027344, "logps/rejected": -77.56038665771484, "loss": 0.9828, "margin_dpo/margin_mean": 5.713043212890625, "margin_dpo/margin_std": 7.821670055389404, "step": 230 }, { "KL/chosen_KL_mean": -9.283416748046875, "KL/mean": -13.417741775512695, "KL/rejected_KL_mean": -17.552074432373047, "KL/std": 7.7223920822143555, "epoch": 0.3492063492063492, "fcm_dpo/beta": 0.13743643462657928, "fcm_dpo/delta": -0.384117990732193, "fcm_dpo/margin": 8.268657684326172, "fcm_dpo/q_t": 0.27989715337753296, "grad_norm": 35.174171447753906, "learning_rate": 4.1271363186719835e-07, "logits/chosen": 0.22936806082725525, "logits/rejected": 0.21267296373844147, "logps/chosen": -81.8805160522461, "logps/ref_chosen": -72.59709930419922, "logps/ref_rejected": -86.2322998046875, "logps/rejected": -103.78437805175781, "loss": 0.7754, "margin_dpo/margin_mean": 8.268656730651855, "margin_dpo/margin_std": 7.923949241638184, "step": 231 }, { "KL/chosen_KL_mean": -9.224300384521484, "KL/mean": -12.099236488342285, "KL/rejected_KL_mean": -14.974166870117188, "KL/std": 8.310755729675293, "epoch": 0.3507180650037793, "fcm_dpo/beta": 0.1356590837240219, "fcm_dpo/delta": 0.021425606682896614, "fcm_dpo/margin": 5.749870300292969, "fcm_dpo/q_t": 0.3614855706691742, "grad_norm": 41.390968322753906, "learning_rate": 4.1170752879801436e-07, "logits/chosen": 0.24424928426742554, "logits/rejected": 0.21669036149978638, "logps/chosen": -77.34283447265625, "logps/ref_chosen": -68.1185302734375, "logps/ref_rejected": -83.79415893554688, "logps/rejected": -98.76832580566406, "loss": 1.0644, "margin_dpo/margin_mean": 5.749871253967285, "margin_dpo/margin_std": 9.516833305358887, "step": 232 }, { "KL/chosen_KL_mean": -10.805501937866211, "KL/mean": -13.204916954040527, "KL/rejected_KL_mean": -15.604331970214844, "KL/std": 8.392253875732422, "epoch": 0.35222978080120937, "fcm_dpo/beta": 0.1332673728466034, "fcm_dpo/delta": -0.11425915360450745, "fcm_dpo/margin": 4.798825740814209, "fcm_dpo/q_t": 0.37997913360595703, "grad_norm": 36.0398063659668, "learning_rate": 4.106969024216348e-07, "logits/chosen": 0.28187963366508484, "logits/rejected": 0.2293119728565216, "logps/chosen": -65.87565612792969, "logps/ref_chosen": -55.070152282714844, "logps/ref_rejected": -66.61845397949219, "logps/rejected": -82.2227783203125, "loss": 1.0971, "margin_dpo/margin_mean": 4.798825740814209, "margin_dpo/margin_std": 7.930860996246338, "step": 233 }, { "KL/chosen_KL_mean": -9.701313018798828, "KL/mean": -11.782073974609375, "KL/rejected_KL_mean": -13.862836837768555, "KL/std": 7.602666854858398, "epoch": 0.35374149659863946, "fcm_dpo/beta": 0.13307172060012817, "fcm_dpo/delta": -0.014703216031193733, "fcm_dpo/margin": 4.161526679992676, "fcm_dpo/q_t": 0.3957321345806122, "grad_norm": 38.414398193359375, "learning_rate": 4.09681781007452e-07, "logits/chosen": 0.22231775522232056, "logits/rejected": 0.20927371084690094, "logps/chosen": -65.62721252441406, "logps/ref_chosen": -55.92589569091797, "logps/ref_rejected": -51.11608123779297, "logps/rejected": -64.97891998291016, "loss": 1.1773, "margin_dpo/margin_mean": 4.161526679992676, "margin_dpo/margin_std": 8.233041763305664, "step": 234 }, { "KL/chosen_KL_mean": -8.079273223876953, "KL/mean": -11.814519882202148, "KL/rejected_KL_mean": -15.54977035522461, "KL/std": 8.269691467285156, "epoch": 0.35525321239606955, "fcm_dpo/beta": 0.12929514050483704, "fcm_dpo/delta": -0.18368813395500183, "fcm_dpo/margin": 7.470497131347656, "fcm_dpo/q_t": 0.3049233853816986, "grad_norm": 32.90544509887695, "learning_rate": 4.08662192950594e-07, "logits/chosen": 0.31978702545166016, "logits/rejected": 0.3014755845069885, "logps/chosen": -72.6189956665039, "logps/ref_chosen": -64.53972625732422, "logps/ref_rejected": -77.69151306152344, "logps/rejected": -93.24127960205078, "loss": 0.8075, "margin_dpo/margin_mean": 7.470496654510498, "margin_dpo/margin_std": 7.160727500915527, "step": 235 }, { "KL/chosen_KL_mean": -11.760652542114258, "KL/mean": -14.860483169555664, "KL/rejected_KL_mean": -17.960315704345703, "KL/std": 8.125932693481445, "epoch": 0.35676492819349964, "fcm_dpo/beta": 0.12746058404445648, "fcm_dpo/delta": 0.009847693145275116, "fcm_dpo/margin": 6.19966459274292, "fcm_dpo/q_t": 0.3519732356071472, "grad_norm": 36.77629470825195, "learning_rate": 4.076381667711306e-07, "logits/chosen": 0.2771919369697571, "logits/rejected": 0.2631545066833496, "logps/chosen": -82.91539001464844, "logps/ref_chosen": -71.15473937988281, "logps/ref_rejected": -84.88541412353516, "logps/rejected": -102.84573364257812, "loss": 1.0304, "margin_dpo/margin_mean": 6.199664115905762, "margin_dpo/margin_std": 9.351386070251465, "step": 236 }, { "KL/chosen_KL_mean": -9.832389831542969, "KL/mean": -12.970930099487305, "KL/rejected_KL_mean": -16.10947036743164, "KL/std": 7.85981559753418, "epoch": 0.35827664399092973, "fcm_dpo/beta": 0.12796080112457275, "fcm_dpo/delta": -0.003629550337791443, "fcm_dpo/margin": 6.27708101272583, "fcm_dpo/q_t": 0.34321698546409607, "grad_norm": 39.69746398925781, "learning_rate": 4.066097311132753e-07, "logits/chosen": 0.3301270008087158, "logits/rejected": 0.3169901371002197, "logps/chosen": -85.97440338134766, "logps/ref_chosen": -76.14201354980469, "logps/ref_rejected": -80.88479614257812, "logps/rejected": -96.9942626953125, "loss": 1.0223, "margin_dpo/margin_mean": 6.277080535888672, "margin_dpo/margin_std": 9.050657272338867, "step": 237 }, { "KL/chosen_KL_mean": -9.327678680419922, "KL/mean": -12.499244689941406, "KL/rejected_KL_mean": -15.670822143554688, "KL/std": 8.36613655090332, "epoch": 0.35978835978835977, "fcm_dpo/beta": 0.1276305615901947, "fcm_dpo/delta": -0.010671408846974373, "fcm_dpo/margin": 6.343140125274658, "fcm_dpo/q_t": 0.3476073741912842, "grad_norm": 53.07609939575195, "learning_rate": 4.0557691474458414e-07, "logits/chosen": 0.26444271206855774, "logits/rejected": 0.24774247407913208, "logps/chosen": -78.2125244140625, "logps/ref_chosen": -68.88484954833984, "logps/ref_rejected": -75.8946304321289, "logps/rejected": -91.5654525756836, "loss": 0.9909, "margin_dpo/margin_mean": 6.343140602111816, "margin_dpo/margin_std": 8.846379280090332, "step": 238 }, { "KL/chosen_KL_mean": -11.43686294555664, "KL/mean": -14.855867385864258, "KL/rejected_KL_mean": -18.27486801147461, "KL/std": 9.134950637817383, "epoch": 0.36130007558578986, "fcm_dpo/beta": 0.1278074085712433, "fcm_dpo/delta": -0.08258944004774094, "fcm_dpo/margin": 6.838010311126709, "fcm_dpo/q_t": 0.3380742073059082, "grad_norm": 36.243316650390625, "learning_rate": 4.045397465551513e-07, "logits/chosen": 0.3806946277618408, "logits/rejected": 0.2709374725818634, "logps/chosen": -68.20869445800781, "logps/ref_chosen": -56.771827697753906, "logps/ref_rejected": -116.23050689697266, "logps/rejected": -134.50537109375, "loss": 0.9782, "margin_dpo/margin_mean": 6.838012218475342, "margin_dpo/margin_std": 9.391035079956055, "step": 239 }, { "KL/chosen_KL_mean": -10.77096939086914, "KL/mean": -14.825616836547852, "KL/rejected_KL_mean": -18.880260467529297, "KL/std": 9.48426628112793, "epoch": 0.36281179138321995, "fcm_dpo/beta": 0.12276466935873032, "fcm_dpo/delta": -0.21665102243423462, "fcm_dpo/margin": 8.109292984008789, "fcm_dpo/q_t": 0.30741313099861145, "grad_norm": 27.501493453979492, "learning_rate": 4.0349825555680045e-07, "logits/chosen": 0.2991008460521698, "logits/rejected": 0.21405178308486938, "logps/chosen": -64.12507629394531, "logps/ref_chosen": -53.35411071777344, "logps/ref_rejected": -80.12019348144531, "logps/rejected": -99.00045776367188, "loss": 0.8584, "margin_dpo/margin_mean": 8.109292984008789, "margin_dpo/margin_std": 8.866523742675781, "step": 240 }, { "KL/chosen_KL_mean": -10.55604362487793, "KL/mean": -12.819741249084473, "KL/rejected_KL_mean": -15.083438873291016, "KL/std": 7.995132923126221, "epoch": 0.36432350718065004, "fcm_dpo/beta": 0.12550224363803864, "fcm_dpo/delta": 0.24444246292114258, "fcm_dpo/margin": 4.5273942947387695, "fcm_dpo/q_t": 0.39284324645996094, "grad_norm": 40.22063064575195, "learning_rate": 4.0245247088227377e-07, "logits/chosen": 0.24775752425193787, "logits/rejected": 0.2146153748035431, "logps/chosen": -82.45145416259766, "logps/ref_chosen": -71.89541625976562, "logps/ref_rejected": -83.03492736816406, "logps/rejected": -98.11836242675781, "loss": 1.1212, "margin_dpo/margin_mean": 4.527395248413086, "margin_dpo/margin_std": 8.242197036743164, "step": 241 }, { "KL/chosen_KL_mean": -9.93331527709961, "KL/mean": -13.826549530029297, "KL/rejected_KL_mean": -17.71977996826172, "KL/std": 9.657007217407227, "epoch": 0.36583522297808013, "fcm_dpo/beta": 0.12183021008968353, "fcm_dpo/delta": -0.17005649209022522, "fcm_dpo/margin": 7.786465644836426, "fcm_dpo/q_t": 0.32561737298965454, "grad_norm": 29.045923233032227, "learning_rate": 4.0140242178441665e-07, "logits/chosen": 0.26129892468452454, "logits/rejected": 0.23901167511940002, "logps/chosen": -67.86074829101562, "logps/ref_chosen": -57.927433013916016, "logps/ref_rejected": -67.838623046875, "logps/rejected": -85.55839538574219, "loss": 0.9366, "margin_dpo/margin_mean": 7.786465167999268, "margin_dpo/margin_std": 10.202951431274414, "step": 242 }, { "KL/chosen_KL_mean": -10.123703002929688, "KL/mean": -13.311759948730469, "KL/rejected_KL_mean": -16.49981689453125, "KL/std": 8.933574676513672, "epoch": 0.3673469387755102, "fcm_dpo/beta": 0.12265647202730179, "fcm_dpo/delta": 0.01945311576128006, "fcm_dpo/margin": 6.376116752624512, "fcm_dpo/q_t": 0.3523695170879364, "grad_norm": 36.763248443603516, "learning_rate": 4.003481376353596e-07, "logits/chosen": 0.2812059819698334, "logits/rejected": 0.2792655825614929, "logps/chosen": -84.40037536621094, "logps/ref_chosen": -74.27667236328125, "logps/ref_rejected": -73.24340057373047, "logps/rejected": -89.74321746826172, "loss": 1.0002, "margin_dpo/margin_mean": 6.376116752624512, "margin_dpo/margin_std": 9.081279754638672, "step": 243 }, { "KL/chosen_KL_mean": -10.219358444213867, "KL/mean": -14.52243423461914, "KL/rejected_KL_mean": -18.825511932373047, "KL/std": 8.50840950012207, "epoch": 0.3688586545729403, "fcm_dpo/beta": 0.11894027888774872, "fcm_dpo/delta": -0.2484353482723236, "fcm_dpo/margin": 8.606154441833496, "fcm_dpo/q_t": 0.2950640320777893, "grad_norm": 25.763652801513672, "learning_rate": 3.9928964792569654e-07, "logits/chosen": 0.3132631182670593, "logits/rejected": 0.23436595499515533, "logps/chosen": -63.5832633972168, "logps/ref_chosen": -53.36390686035156, "logps/ref_rejected": -71.10276794433594, "logps/rejected": -89.92828369140625, "loss": 0.7863, "margin_dpo/margin_mean": 8.60615348815918, "margin_dpo/margin_std": 8.056436538696289, "step": 244 }, { "KL/chosen_KL_mean": -10.508058547973633, "KL/mean": -15.217109680175781, "KL/rejected_KL_mean": -19.92615509033203, "KL/std": 9.513158798217773, "epoch": 0.37037037037037035, "fcm_dpo/beta": 0.11189082264900208, "fcm_dpo/delta": -0.28425049781799316, "fcm_dpo/margin": 9.418100357055664, "fcm_dpo/q_t": 0.29473158717155457, "grad_norm": 77.52033996582031, "learning_rate": 3.982269822636601e-07, "logits/chosen": 0.33938145637512207, "logits/rejected": 0.3116254210472107, "logps/chosen": -81.70316314697266, "logps/ref_chosen": -71.19510650634766, "logps/ref_rejected": -80.76235961914062, "logps/rejected": -100.68851470947266, "loss": 0.8046, "margin_dpo/margin_mean": 9.41810131072998, "margin_dpo/margin_std": 9.551580429077148, "step": 245 }, { "KL/chosen_KL_mean": -12.594314575195312, "KL/mean": -16.568037033081055, "KL/rejected_KL_mean": -20.541751861572266, "KL/std": 9.444729804992676, "epoch": 0.37188208616780044, "fcm_dpo/beta": 0.10943619906902313, "fcm_dpo/delta": -0.07631853222846985, "fcm_dpo/margin": 7.9474382400512695, "fcm_dpo/q_t": 0.33763816952705383, "grad_norm": 35.117340087890625, "learning_rate": 3.971601703742932e-07, "logits/chosen": 0.33962783217430115, "logits/rejected": 0.2832232415676117, "logps/chosen": -84.21536254882812, "logps/ref_chosen": -71.62104797363281, "logps/ref_rejected": -94.03392028808594, "logps/rejected": -114.57567596435547, "loss": 0.981, "margin_dpo/margin_mean": 7.947439193725586, "margin_dpo/margin_std": 11.065793991088867, "step": 246 }, { "KL/chosen_KL_mean": -13.099170684814453, "KL/mean": -15.340646743774414, "KL/rejected_KL_mean": -17.582115173339844, "KL/std": 8.530660629272461, "epoch": 0.37339380196523053, "fcm_dpo/beta": 0.11499364674091339, "fcm_dpo/delta": 0.29317426681518555, "fcm_dpo/margin": 4.482947826385498, "fcm_dpo/q_t": 0.39475196599960327, "grad_norm": 39.144874572753906, "learning_rate": 3.960892420986177e-07, "logits/chosen": 0.35208380222320557, "logits/rejected": 0.3406439423561096, "logps/chosen": -93.12171936035156, "logps/ref_chosen": -80.02254486083984, "logps/ref_rejected": -89.22705841064453, "logps/rejected": -106.80917358398438, "loss": 1.1837, "margin_dpo/margin_mean": 4.48294734954834, "margin_dpo/margin_std": 9.121728897094727, "step": 247 }, { "KL/chosen_KL_mean": -11.430730819702148, "KL/mean": -15.32693862915039, "KL/rejected_KL_mean": -19.223148345947266, "KL/std": 10.37348461151123, "epoch": 0.3749055177626606, "fcm_dpo/beta": 0.11418326944112778, "fcm_dpo/delta": -0.09853696823120117, "fcm_dpo/margin": 7.792417526245117, "fcm_dpo/q_t": 0.343853235244751, "grad_norm": 34.979244232177734, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 0.31661760807037354, "logits/rejected": 0.3491419553756714, "logps/chosen": -76.80868530273438, "logps/ref_chosen": -65.37796020507812, "logps/ref_rejected": -61.365787506103516, "logps/rejected": -80.58893585205078, "loss": 1.0005, "margin_dpo/margin_mean": 7.792417526245117, "margin_dpo/margin_std": 11.333248138427734, "step": 248 }, { "KL/chosen_KL_mean": -14.106277465820312, "KL/mean": -15.700241088867188, "KL/rejected_KL_mean": -17.294208526611328, "KL/std": 9.214258193969727, "epoch": 0.3764172335600907, "fcm_dpo/beta": 0.11448922753334045, "fcm_dpo/delta": 0.12567032873630524, "fcm_dpo/margin": 3.1879303455352783, "fcm_dpo/q_t": 0.42352503538131714, "grad_norm": 45.407470703125, "learning_rate": 3.9393515632731094e-07, "logits/chosen": 0.30869317054748535, "logits/rejected": 0.33915185928344727, "logps/chosen": -88.70773315429688, "logps/ref_chosen": -74.60145568847656, "logps/ref_rejected": -63.79338455200195, "logps/rejected": -81.08759307861328, "loss": 1.3556, "margin_dpo/margin_mean": 3.1879305839538574, "margin_dpo/margin_std": 10.071321487426758, "step": 249 }, { "KL/chosen_KL_mean": -11.981225967407227, "KL/mean": -16.120920181274414, "KL/rejected_KL_mean": -20.2606201171875, "KL/std": 10.126691818237305, "epoch": 0.3779289493575208, "fcm_dpo/beta": 0.11403284966945648, "fcm_dpo/delta": -0.15904656052589417, "fcm_dpo/margin": 8.279390335083008, "fcm_dpo/q_t": 0.32035911083221436, "grad_norm": 31.749574661254883, "learning_rate": 3.9285205908608934e-07, "logits/chosen": 0.3941000699996948, "logits/rejected": 0.3523021936416626, "logps/chosen": -73.91943359375, "logps/ref_chosen": -61.938209533691406, "logps/ref_rejected": -72.21602630615234, "logps/rejected": -92.47664642333984, "loss": 0.9328, "margin_dpo/margin_mean": 8.279389381408691, "margin_dpo/margin_std": 10.527669906616211, "step": 250 }, { "KL/chosen_KL_mean": -12.728778839111328, "KL/mean": -15.447917938232422, "KL/rejected_KL_mean": -18.16705322265625, "KL/std": 9.414742469787598, "epoch": 0.3794406651549509, "fcm_dpo/beta": 0.11536653339862823, "fcm_dpo/delta": 0.1842348873615265, "fcm_dpo/margin": 5.4382710456848145, "fcm_dpo/q_t": 0.382129967212677, "grad_norm": 40.527164459228516, "learning_rate": 3.9176496596569265e-07, "logits/chosen": 0.38748690485954285, "logits/rejected": 0.34921795129776, "logps/chosen": -79.58572387695312, "logps/ref_chosen": -66.85694885253906, "logps/ref_rejected": -84.83396911621094, "logps/rejected": -103.00102233886719, "loss": 1.1419, "margin_dpo/margin_mean": 5.438270568847656, "margin_dpo/margin_std": 10.37491512298584, "step": 251 }, { "KL/chosen_KL_mean": -12.064062118530273, "KL/mean": -14.76373291015625, "KL/rejected_KL_mean": -17.463409423828125, "KL/std": 10.329211235046387, "epoch": 0.38095238095238093, "fcm_dpo/beta": 0.12140388786792755, "fcm_dpo/delta": 0.1437467634677887, "fcm_dpo/margin": 5.399339199066162, "fcm_dpo/q_t": 0.37345564365386963, "grad_norm": 39.94149398803711, "learning_rate": 3.9067390737445254e-07, "logits/chosen": 0.26361334323883057, "logits/rejected": 0.21315331757068634, "logps/chosen": -68.28799438476562, "logps/ref_chosen": -56.22393035888672, "logps/ref_rejected": -77.1136245727539, "logps/rejected": -94.57703399658203, "loss": 1.2079, "margin_dpo/margin_mean": 5.399339199066162, "margin_dpo/margin_std": 10.947092056274414, "step": 252 }, { "KL/chosen_KL_mean": -12.338104248046875, "KL/mean": -15.188192367553711, "KL/rejected_KL_mean": -18.03827667236328, "KL/std": 9.178722381591797, "epoch": 0.382464096749811, "fcm_dpo/beta": 0.12051292508840561, "fcm_dpo/delta": 0.11948665976524353, "fcm_dpo/margin": 5.700176239013672, "fcm_dpo/q_t": 0.37205415964126587, "grad_norm": 34.4255256652832, "learning_rate": 3.8957891383162304e-07, "logits/chosen": 0.36925697326660156, "logits/rejected": 0.32966768741607666, "logps/chosen": -64.5481185913086, "logps/ref_chosen": -52.21001434326172, "logps/ref_rejected": -58.75764846801758, "logps/rejected": -76.79592895507812, "loss": 1.0874, "margin_dpo/margin_mean": 5.700176239013672, "margin_dpo/margin_std": 9.250205039978027, "step": 253 }, { "KL/chosen_KL_mean": -12.473712921142578, "KL/mean": -15.742783546447754, "KL/rejected_KL_mean": -19.01184844970703, "KL/std": 10.278692245483398, "epoch": 0.3839758125472411, "fcm_dpo/beta": 0.12202567607164383, "fcm_dpo/delta": 0.0014530569314956665, "fcm_dpo/margin": 6.5381340980529785, "fcm_dpo/q_t": 0.3580353856086731, "grad_norm": 39.03754425048828, "learning_rate": 3.884800159665276e-07, "logits/chosen": 0.29129236936569214, "logits/rejected": 0.24309565126895905, "logps/chosen": -78.11003875732422, "logps/ref_chosen": -65.63632202148438, "logps/ref_rejected": -82.34425354003906, "logps/rejected": -101.35609436035156, "loss": 1.0392, "margin_dpo/margin_mean": 6.5381340980529785, "margin_dpo/margin_std": 10.117410659790039, "step": 254 }, { "KL/chosen_KL_mean": -10.72909164428711, "KL/mean": -14.877052307128906, "KL/rejected_KL_mean": -19.025009155273438, "KL/std": 10.288159370422363, "epoch": 0.3854875283446712, "fcm_dpo/beta": 0.1184849888086319, "fcm_dpo/delta": -0.20485825836658478, "fcm_dpo/margin": 8.295927047729492, "fcm_dpo/q_t": 0.32767289876937866, "grad_norm": 32.1718635559082, "learning_rate": 3.873772445177015e-07, "logits/chosen": 0.2984389662742615, "logits/rejected": 0.26697784662246704, "logps/chosen": -78.64018249511719, "logps/ref_chosen": -67.91108703613281, "logps/ref_rejected": -83.89114379882812, "logps/rejected": -102.91615295410156, "loss": 0.9405, "margin_dpo/margin_mean": 8.295927047729492, "margin_dpo/margin_std": 11.091558456420898, "step": 255 }, { "KL/chosen_KL_mean": -12.830934524536133, "KL/mean": -16.304931640625, "KL/rejected_KL_mean": -19.778934478759766, "KL/std": 9.357809066772461, "epoch": 0.3869992441421013, "fcm_dpo/beta": 0.11697204411029816, "fcm_dpo/delta": -0.014655433595180511, "fcm_dpo/margin": 6.947994709014893, "fcm_dpo/q_t": 0.3568224012851715, "grad_norm": 38.28756332397461, "learning_rate": 3.862706303320329e-07, "logits/chosen": 0.30309706926345825, "logits/rejected": 0.24692294001579285, "logps/chosen": -76.33091735839844, "logps/ref_chosen": -63.49998474121094, "logps/ref_rejected": -90.77104187011719, "logps/rejected": -110.54997253417969, "loss": 1.0638, "margin_dpo/margin_mean": 6.947993755340576, "margin_dpo/margin_std": 11.271989822387695, "step": 256 }, { "KL/chosen_KL_mean": -13.398094177246094, "KL/mean": -18.00149917602539, "KL/rejected_KL_mean": -22.604900360107422, "KL/std": 11.196935653686523, "epoch": 0.3885109599395314, "fcm_dpo/beta": 0.11260446161031723, "fcm_dpo/delta": -0.2655514180660248, "fcm_dpo/margin": 9.206808090209961, "fcm_dpo/q_t": 0.3245314955711365, "grad_norm": 37.96772766113281, "learning_rate": 3.851602043638994e-07, "logits/chosen": 0.28178295493125916, "logits/rejected": 0.22410434484481812, "logps/chosen": -83.99874877929688, "logps/ref_chosen": -70.60064697265625, "logps/ref_rejected": -108.58313751220703, "logps/rejected": -131.1880340576172, "loss": 0.9396, "margin_dpo/margin_mean": 9.206808090209961, "margin_dpo/margin_std": 12.629020690917969, "step": 257 }, { "KL/chosen_KL_mean": -13.584089279174805, "KL/mean": -17.121944427490234, "KL/rejected_KL_mean": -20.659801483154297, "KL/std": 9.215842247009277, "epoch": 0.3900226757369615, "fcm_dpo/beta": 0.11255334317684174, "fcm_dpo/delta": 0.0021555423736572266, "fcm_dpo/margin": 7.075709819793701, "fcm_dpo/q_t": 0.3390353322029114, "grad_norm": 34.61642074584961, "learning_rate": 3.840459976743023e-07, "logits/chosen": 0.3270367980003357, "logits/rejected": 0.28003180027008057, "logps/chosen": -72.8382568359375, "logps/ref_chosen": -59.25416564941406, "logps/ref_rejected": -85.58709716796875, "logps/rejected": -106.24689483642578, "loss": 0.9257, "margin_dpo/margin_mean": 7.075709342956543, "margin_dpo/margin_std": 8.351495742797852, "step": 258 }, { "KL/chosen_KL_mean": -10.683589935302734, "KL/mean": -16.045759201049805, "KL/rejected_KL_mean": -21.407928466796875, "KL/std": 10.769359588623047, "epoch": 0.3915343915343915, "fcm_dpo/beta": 0.10471875965595245, "fcm_dpo/delta": -0.36887070536613464, "fcm_dpo/margin": 10.724337577819824, "fcm_dpo/q_t": 0.288051962852478, "grad_norm": 28.6914119720459, "learning_rate": 3.8292804142999796e-07, "logits/chosen": 0.2580224275588989, "logits/rejected": 0.16532495617866516, "logps/chosen": -76.11846923828125, "logps/ref_chosen": -65.43487548828125, "logps/ref_rejected": -95.41731262207031, "logps/rejected": -116.82524108886719, "loss": 0.8081, "margin_dpo/margin_mean": 10.72433853149414, "margin_dpo/margin_std": 11.031912803649902, "step": 259 }, { "KL/chosen_KL_mean": -11.84138298034668, "KL/mean": -15.952880859375, "KL/rejected_KL_mean": -20.064380645751953, "KL/std": 9.865896224975586, "epoch": 0.3930461073318216, "fcm_dpo/beta": 0.10231013596057892, "fcm_dpo/delta": -0.046053703874349594, "fcm_dpo/margin": 8.22299861907959, "fcm_dpo/q_t": 0.3451218008995056, "grad_norm": 29.589677810668945, "learning_rate": 3.818063669026256e-07, "logits/chosen": 0.2971701920032501, "logits/rejected": 0.21861127018928528, "logps/chosen": -60.930965423583984, "logps/ref_chosen": -49.08958435058594, "logps/ref_rejected": -79.01708221435547, "logps/rejected": -99.08146667480469, "loss": 0.9952, "margin_dpo/margin_mean": 8.22299861907959, "margin_dpo/margin_std": 11.613203048706055, "step": 260 }, { "KL/chosen_KL_mean": -11.726337432861328, "KL/mean": -15.044000625610352, "KL/rejected_KL_mean": -18.361663818359375, "KL/std": 9.771665573120117, "epoch": 0.3945578231292517, "fcm_dpo/beta": 0.10444238036870956, "fcm_dpo/delta": 0.11493270099163055, "fcm_dpo/margin": 6.635324001312256, "fcm_dpo/q_t": 0.3719925880432129, "grad_norm": 37.57694625854492, "learning_rate": 3.806810054678331e-07, "logits/chosen": 0.1974962055683136, "logits/rejected": 0.21522969007492065, "logps/chosen": -82.5987319946289, "logps/ref_chosen": -70.87239074707031, "logps/ref_rejected": -65.01522064208984, "logps/rejected": -83.37688446044922, "loss": 1.0653, "margin_dpo/margin_mean": 6.635324478149414, "margin_dpo/margin_std": 10.821256637573242, "step": 261 }, { "KL/chosen_KL_mean": -12.111078262329102, "KL/mean": -15.972871780395508, "KL/rejected_KL_mean": -19.834667205810547, "KL/std": 9.611106872558594, "epoch": 0.3960695389266818, "fcm_dpo/beta": 0.1053507924079895, "fcm_dpo/delta": -0.01538146659731865, "fcm_dpo/margin": 7.723592758178711, "fcm_dpo/q_t": 0.345390260219574, "grad_norm": 32.13874053955078, "learning_rate": 3.7955198860439887e-07, "logits/chosen": 0.3609021306037903, "logits/rejected": 0.3037142753601074, "logps/chosen": -79.98170471191406, "logps/ref_chosen": -67.8706283569336, "logps/ref_rejected": -88.7205810546875, "logps/rejected": -108.55525207519531, "loss": 0.9575, "margin_dpo/margin_mean": 7.723593235015869, "margin_dpo/margin_std": 10.076356887817383, "step": 262 }, { "KL/chosen_KL_mean": -11.720417022705078, "KL/mean": -14.867897033691406, "KL/rejected_KL_mean": -18.015369415283203, "KL/std": 9.436576843261719, "epoch": 0.3975812547241119, "fcm_dpo/beta": 0.10749039798974991, "fcm_dpo/delta": 0.13047100603580475, "fcm_dpo/margin": 6.294952392578125, "fcm_dpo/q_t": 0.36887580156326294, "grad_norm": 31.042261123657227, "learning_rate": 3.784193478933516e-07, "logits/chosen": 0.30105894804000854, "logits/rejected": 0.20555788278579712, "logps/chosen": -66.91500091552734, "logps/ref_chosen": -55.194583892822266, "logps/ref_rejected": -80.54048156738281, "logps/rejected": -98.55585479736328, "loss": 1.0718, "margin_dpo/margin_mean": 6.294952392578125, "margin_dpo/margin_std": 10.169485092163086, "step": 263 }, { "KL/chosen_KL_mean": -11.863348007202148, "KL/mean": -15.760964393615723, "KL/rejected_KL_mean": -19.658584594726562, "KL/std": 10.724000930786133, "epoch": 0.39909297052154197, "fcm_dpo/beta": 0.10693207383155823, "fcm_dpo/delta": -0.03660469502210617, "fcm_dpo/margin": 7.795228958129883, "fcm_dpo/q_t": 0.3510364890098572, "grad_norm": 36.74233627319336, "learning_rate": 3.7728311501708674e-07, "logits/chosen": 0.23294666409492493, "logits/rejected": 0.18829098343849182, "logps/chosen": -95.0340347290039, "logps/ref_chosen": -83.17068481445312, "logps/ref_rejected": -88.33625793457031, "logps/rejected": -107.99484252929688, "loss": 1.0338, "margin_dpo/margin_mean": 7.795228958129883, "margin_dpo/margin_std": 12.017763137817383, "step": 264 }, { "KL/chosen_KL_mean": -11.657770156860352, "KL/mean": -15.780095100402832, "KL/rejected_KL_mean": -19.902423858642578, "KL/std": 10.775840759277344, "epoch": 0.40060468631897206, "fcm_dpo/beta": 0.10615849494934082, "fcm_dpo/delta": -0.0829143300652504, "fcm_dpo/margin": 8.244649887084961, "fcm_dpo/q_t": 0.34060657024383545, "grad_norm": 37.233978271484375, "learning_rate": 3.7614332175848027e-07, "logits/chosen": 0.39187127351760864, "logits/rejected": 0.3272865414619446, "logps/chosen": -63.320621490478516, "logps/ref_chosen": -51.66284942626953, "logps/ref_rejected": -67.1720962524414, "logps/rejected": -87.07452392578125, "loss": 1.064, "margin_dpo/margin_mean": 8.244649887084961, "margin_dpo/margin_std": 12.866454124450684, "step": 265 }, { "KL/chosen_KL_mean": -10.886266708374023, "KL/mean": -14.718881607055664, "KL/rejected_KL_mean": -18.551498413085938, "KL/std": 10.924835205078125, "epoch": 0.4021164021164021, "fcm_dpo/beta": 0.10467538237571716, "fcm_dpo/delta": -0.002675635740160942, "fcm_dpo/margin": 7.665229797363281, "fcm_dpo/q_t": 0.3502148389816284, "grad_norm": 33.46331787109375, "learning_rate": 3.75e-07, "logits/chosen": 0.31710296869277954, "logits/rejected": 0.24927489459514618, "logps/chosen": -68.33676147460938, "logps/ref_chosen": -57.45049285888672, "logps/ref_rejected": -77.60826110839844, "logps/rejected": -96.15975952148438, "loss": 1.0042, "margin_dpo/margin_mean": 7.665229797363281, "margin_dpo/margin_std": 11.188663482666016, "step": 266 }, { "KL/chosen_KL_mean": -9.593021392822266, "KL/mean": -12.639689445495605, "KL/rejected_KL_mean": -15.686355590820312, "KL/std": 9.48430347442627, "epoch": 0.4036281179138322, "fcm_dpo/beta": 0.10875646024942398, "fcm_dpo/delta": 0.14000800251960754, "fcm_dpo/margin": 6.093325614929199, "fcm_dpo/q_t": 0.37472468614578247, "grad_norm": 31.011960983276367, "learning_rate": 3.738531817228131e-07, "logits/chosen": 0.31285083293914795, "logits/rejected": 0.2954583168029785, "logps/chosen": -64.62837219238281, "logps/ref_chosen": -55.03535079956055, "logps/ref_rejected": -66.0953369140625, "logps/rejected": -81.78169250488281, "loss": 1.1384, "margin_dpo/margin_mean": 6.093325614929199, "margin_dpo/margin_std": 11.069232940673828, "step": 267 }, { "KL/chosen_KL_mean": -9.607749938964844, "KL/mean": -13.186819076538086, "KL/rejected_KL_mean": -16.765892028808594, "KL/std": 10.487470626831055, "epoch": 0.4051398337112623, "fcm_dpo/beta": 0.1089181900024414, "fcm_dpo/delta": 0.021044310182332993, "fcm_dpo/margin": 7.158140659332275, "fcm_dpo/q_t": 0.3529961109161377, "grad_norm": 29.234092712402344, "learning_rate": 3.7270289900589204e-07, "logits/chosen": 0.21093052625656128, "logits/rejected": 0.1905559003353119, "logps/chosen": -74.67948913574219, "logps/ref_chosen": -65.07174682617188, "logps/ref_rejected": -71.42485809326172, "logps/rejected": -88.19075012207031, "loss": 0.9735, "margin_dpo/margin_mean": 7.158140182495117, "margin_dpo/margin_std": 9.659717559814453, "step": 268 }, { "KL/chosen_KL_mean": -11.00994873046875, "KL/mean": -15.063467025756836, "KL/rejected_KL_mean": -19.116985321044922, "KL/std": 10.769641876220703, "epoch": 0.40665154950869237, "fcm_dpo/beta": 0.10864944756031036, "fcm_dpo/delta": -0.09277642518281937, "fcm_dpo/margin": 8.107038497924805, "fcm_dpo/q_t": 0.33289867639541626, "grad_norm": 29.763572692871094, "learning_rate": 3.7154918402511714e-07, "logits/chosen": 0.44051095843315125, "logits/rejected": 0.3936043679714203, "logps/chosen": -78.14615631103516, "logps/ref_chosen": -67.1362075805664, "logps/ref_rejected": -82.55778503417969, "logps/rejected": -101.67477416992188, "loss": 0.9513, "margin_dpo/margin_mean": 8.107038497924805, "margin_dpo/margin_std": 10.162799835205078, "step": 269 }, { "KL/chosen_KL_mean": -10.442401885986328, "KL/mean": -13.696825981140137, "KL/rejected_KL_mean": -16.951248168945312, "KL/std": 10.145946502685547, "epoch": 0.40816326530612246, "fcm_dpo/beta": 0.107734814286232, "fcm_dpo/delta": 0.10635486245155334, "fcm_dpo/margin": 6.508844375610352, "fcm_dpo/q_t": 0.35935863852500916, "grad_norm": 34.854164123535156, "learning_rate": 3.7039206905237656e-07, "logits/chosen": 0.3360714316368103, "logits/rejected": 0.26730990409851074, "logps/chosen": -77.131103515625, "logps/ref_chosen": -66.6886978149414, "logps/ref_rejected": -85.16129302978516, "logps/rejected": -102.11254119873047, "loss": 1.037, "margin_dpo/margin_mean": 6.508844375610352, "margin_dpo/margin_std": 9.949800491333008, "step": 270 }, { "KL/chosen_KL_mean": -9.98006820678711, "KL/mean": -12.884943008422852, "KL/rejected_KL_mean": -15.789817810058594, "KL/std": 10.585667610168457, "epoch": 0.40967498110355255, "fcm_dpo/beta": 0.11065304279327393, "fcm_dpo/delta": 0.16814447939395905, "fcm_dpo/margin": 5.809749126434326, "fcm_dpo/q_t": 0.4004424214363098, "grad_norm": 39.59679412841797, "learning_rate": 3.692315864546635e-07, "logits/chosen": 0.33203697204589844, "logits/rejected": 0.2761046290397644, "logps/chosen": -82.38761901855469, "logps/ref_chosen": -72.40754699707031, "logps/ref_rejected": -92.06311798095703, "logps/rejected": -107.85293579101562, "loss": 1.2269, "margin_dpo/margin_mean": 5.809748649597168, "margin_dpo/margin_std": 12.749189376831055, "step": 271 }, { "KL/chosen_KL_mean": -8.188089370727539, "KL/mean": -13.272232055664062, "KL/rejected_KL_mean": -18.35637664794922, "KL/std": 10.037698745727539, "epoch": 0.41118669690098264, "fcm_dpo/beta": 0.10786165297031403, "fcm_dpo/delta": -0.332021564245224, "fcm_dpo/margin": 10.168288230895996, "fcm_dpo/q_t": 0.2953736186027527, "grad_norm": 29.516803741455078, "learning_rate": 3.6806776869317067e-07, "logits/chosen": 0.33715561032295227, "logits/rejected": 0.34234732389450073, "logps/chosen": -74.78948974609375, "logps/ref_chosen": -66.60140228271484, "logps/ref_rejected": -67.74340057373047, "logps/rejected": -86.09977722167969, "loss": 0.7911, "margin_dpo/margin_mean": 10.168288230895996, "margin_dpo/margin_std": 10.265481948852539, "step": 272 }, { "KL/chosen_KL_mean": -11.362102508544922, "KL/mean": -15.370185852050781, "KL/rejected_KL_mean": -19.37826156616211, "KL/std": 10.579803466796875, "epoch": 0.4126984126984127, "fcm_dpo/beta": 0.10470890998840332, "fcm_dpo/delta": -0.04298901930451393, "fcm_dpo/margin": 8.016157150268555, "fcm_dpo/q_t": 0.3473791182041168, "grad_norm": 33.51838302612305, "learning_rate": 3.669006483223828e-07, "logits/chosen": 0.339900940656662, "logits/rejected": 0.27993035316467285, "logps/chosen": -68.71697998046875, "logps/ref_chosen": -57.35487747192383, "logps/ref_rejected": -84.17168426513672, "logps/rejected": -103.54994201660156, "loss": 1.0409, "margin_dpo/margin_mean": 8.016157150268555, "margin_dpo/margin_std": 12.47188663482666, "step": 273 }, { "KL/chosen_KL_mean": -10.67816162109375, "KL/mean": -14.641273498535156, "KL/rejected_KL_mean": -18.604381561279297, "KL/std": 10.35463809967041, "epoch": 0.41421012849584277, "fcm_dpo/beta": 0.10404108464717865, "fcm_dpo/delta": -0.026914719492197037, "fcm_dpo/margin": 7.9262237548828125, "fcm_dpo/q_t": 0.34564095735549927, "grad_norm": 28.05985450744629, "learning_rate": 3.657302579891656e-07, "logits/chosen": 0.20250210165977478, "logits/rejected": 0.17951546609401703, "logps/chosen": -70.31965637207031, "logps/ref_chosen": -59.64149475097656, "logps/ref_rejected": -68.29348754882812, "logps/rejected": -86.89786529541016, "loss": 1.0126, "margin_dpo/margin_mean": 7.9262237548828125, "margin_dpo/margin_std": 11.62763786315918, "step": 274 }, { "KL/chosen_KL_mean": -10.336505889892578, "KL/mean": -14.75816535949707, "KL/rejected_KL_mean": -19.179826736450195, "KL/std": 10.844429016113281, "epoch": 0.41572184429327286, "fcm_dpo/beta": 0.1024307906627655, "fcm_dpo/delta": -0.11632996052503586, "fcm_dpo/margin": 8.843318939208984, "fcm_dpo/q_t": 0.32907673716545105, "grad_norm": 28.298322677612305, "learning_rate": 3.645566304318526e-07, "logits/chosen": 0.33164167404174805, "logits/rejected": 0.25221920013427734, "logps/chosen": -63.6031494140625, "logps/ref_chosen": -53.26664352416992, "logps/ref_rejected": -73.84062194824219, "logps/rejected": -93.02044677734375, "loss": 0.9391, "margin_dpo/margin_mean": 8.843318939208984, "margin_dpo/margin_std": 11.451114654541016, "step": 275 }, { "KL/chosen_KL_mean": -8.990222930908203, "KL/mean": -13.361787796020508, "KL/rejected_KL_mean": -17.733346939086914, "KL/std": 9.985158920288086, "epoch": 0.41723356009070295, "fcm_dpo/beta": 0.09962226450443268, "fcm_dpo/delta": -0.07865756750106812, "fcm_dpo/margin": 8.743131637573242, "fcm_dpo/q_t": 0.325851172208786, "grad_norm": 27.074356079101562, "learning_rate": 3.633797984793294e-07, "logits/chosen": 0.26929759979248047, "logits/rejected": 0.23500406742095947, "logps/chosen": -62.011016845703125, "logps/ref_chosen": -53.02079772949219, "logps/ref_rejected": -61.56678771972656, "logps/rejected": -79.30014038085938, "loss": 0.8889, "margin_dpo/margin_mean": 8.743131637573242, "margin_dpo/margin_std": 9.81624984741211, "step": 276 }, { "KL/chosen_KL_mean": -11.36018180847168, "KL/mean": -13.965312004089355, "KL/rejected_KL_mean": -16.57044219970703, "KL/std": 10.15247631072998, "epoch": 0.41874527588813304, "fcm_dpo/beta": 0.10431164503097534, "fcm_dpo/delta": 0.26928621530532837, "fcm_dpo/margin": 5.210268020629883, "fcm_dpo/q_t": 0.3985578715801239, "grad_norm": 35.48118209838867, "learning_rate": 3.6219979505011555e-07, "logits/chosen": 0.35306310653686523, "logits/rejected": 0.37153375148773193, "logps/chosen": -82.79316711425781, "logps/ref_chosen": -71.43299102783203, "logps/ref_rejected": -67.65852355957031, "logps/rejected": -84.22897338867188, "loss": 1.1703, "margin_dpo/margin_mean": 5.210267066955566, "margin_dpo/margin_std": 10.70171070098877, "step": 277 }, { "KL/chosen_KL_mean": -11.002376556396484, "KL/mean": -15.127434730529785, "KL/rejected_KL_mean": -19.252490997314453, "KL/std": 11.776235580444336, "epoch": 0.42025699168556313, "fcm_dpo/beta": 0.1055298000574112, "fcm_dpo/delta": -0.08074446022510529, "fcm_dpo/margin": 8.25011157989502, "fcm_dpo/q_t": 0.33413276076316833, "grad_norm": 35.327354431152344, "learning_rate": 3.6101665315144353e-07, "logits/chosen": 0.23632648587226868, "logits/rejected": 0.1905459463596344, "logps/chosen": -78.11314392089844, "logps/ref_chosen": -67.11076354980469, "logps/ref_rejected": -88.74851989746094, "logps/rejected": -108.00101470947266, "loss": 0.9829, "margin_dpo/margin_mean": 8.25011157989502, "margin_dpo/margin_std": 11.206602096557617, "step": 278 }, { "KL/chosen_KL_mean": -7.801475524902344, "KL/mean": -12.950601577758789, "KL/rejected_KL_mean": -18.09972381591797, "KL/std": 10.632495880126953, "epoch": 0.4217687074829932, "fcm_dpo/beta": 0.1002463549375534, "fcm_dpo/delta": -0.2583245038986206, "fcm_dpo/margin": 10.29825210571289, "fcm_dpo/q_t": 0.3018980622291565, "grad_norm": 25.455596923828125, "learning_rate": 3.5983040587833563e-07, "logits/chosen": 0.2795836925506592, "logits/rejected": 0.23831382393836975, "logps/chosen": -62.2989616394043, "logps/ref_chosen": -54.49748611450195, "logps/ref_rejected": -70.42373657226562, "logps/rejected": -88.52345275878906, "loss": 0.8223, "margin_dpo/margin_mean": 10.29825210571289, "margin_dpo/margin_std": 10.669867515563965, "step": 279 }, { "KL/chosen_KL_mean": -8.367753982543945, "KL/mean": -13.79076862335205, "KL/rejected_KL_mean": -19.213787078857422, "KL/std": 11.13604736328125, "epoch": 0.42328042328042326, "fcm_dpo/beta": 0.09465308487415314, "fcm_dpo/delta": -0.2527472972869873, "fcm_dpo/margin": 10.846031188964844, "fcm_dpo/q_t": 0.2982625365257263, "grad_norm": 23.867778778076172, "learning_rate": 3.586410864126781e-07, "logits/chosen": 0.32292693853378296, "logits/rejected": 0.2808569371700287, "logps/chosen": -68.80056762695312, "logps/ref_chosen": -60.43281173706055, "logps/ref_rejected": -78.39051818847656, "logps/rejected": -97.60430908203125, "loss": 0.7738, "margin_dpo/margin_mean": 10.846031188964844, "margin_dpo/margin_std": 10.204471588134766, "step": 280 }, { "KL/chosen_KL_mean": -9.530132293701172, "KL/mean": -14.347930908203125, "KL/rejected_KL_mean": -19.165735244750977, "KL/std": 11.033517837524414, "epoch": 0.42479213907785335, "fcm_dpo/beta": 0.09185758978128433, "fcm_dpo/delta": -0.09395193308591843, "fcm_dpo/margin": 9.635604858398438, "fcm_dpo/q_t": 0.32688847184181213, "grad_norm": 24.12879753112793, "learning_rate": 3.574487280222929e-07, "logits/chosen": 0.29109102487564087, "logits/rejected": 0.29816192388534546, "logps/chosen": -69.81222534179688, "logps/ref_chosen": -60.2820930480957, "logps/ref_rejected": -62.04009246826172, "logps/rejected": -81.20582580566406, "loss": 0.9115, "margin_dpo/margin_mean": 9.635605812072754, "margin_dpo/margin_std": 11.505289077758789, "step": 281 }, { "KL/chosen_KL_mean": -11.271791458129883, "KL/mean": -15.761783599853516, "KL/rejected_KL_mean": -20.25177764892578, "KL/std": 11.111000061035156, "epoch": 0.42630385487528344, "fcm_dpo/beta": 0.09275418519973755, "fcm_dpo/delta": -0.04129917919635773, "fcm_dpo/margin": 8.979986190795898, "fcm_dpo/q_t": 0.34636229276657104, "grad_norm": 29.4520206451416, "learning_rate": 3.562533640600075e-07, "logits/chosen": 0.2616288959980011, "logits/rejected": 0.21098898351192474, "logps/chosen": -71.89571380615234, "logps/ref_chosen": -60.623924255371094, "logps/ref_rejected": -68.67400360107422, "logps/rejected": -88.92578125, "loss": 0.9787, "margin_dpo/margin_mean": 8.979985237121582, "margin_dpo/margin_std": 11.980823516845703, "step": 282 }, { "KL/chosen_KL_mean": -12.055740356445312, "KL/mean": -15.898777961730957, "KL/rejected_KL_mean": -19.741825103759766, "KL/std": 11.119794845581055, "epoch": 0.42781557067271353, "fcm_dpo/beta": 0.0929352194070816, "fcm_dpo/delta": 0.09024599939584732, "fcm_dpo/margin": 7.6860809326171875, "fcm_dpo/q_t": 0.3683665096759796, "grad_norm": 34.17851638793945, "learning_rate": 3.550550279627215e-07, "logits/chosen": 0.29102981090545654, "logits/rejected": 0.20298334956169128, "logps/chosen": -79.7034912109375, "logps/ref_chosen": -67.64775085449219, "logps/ref_rejected": -99.96835327148438, "logps/rejected": -119.7101821899414, "loss": 1.0697, "margin_dpo/margin_mean": 7.686081886291504, "margin_dpo/margin_std": 12.399127960205078, "step": 283 }, { "KL/chosen_KL_mean": -11.307464599609375, "KL/mean": -15.802129745483398, "KL/rejected_KL_mean": -20.296794891357422, "KL/std": 11.915395736694336, "epoch": 0.4293272864701436, "fcm_dpo/beta": 0.0916890799999237, "fcm_dpo/delta": -0.027363691478967667, "fcm_dpo/margin": 8.989330291748047, "fcm_dpo/q_t": 0.33987635374069214, "grad_norm": 25.31052589416504, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 0.3455355763435364, "logits/rejected": 0.285012423992157, "logps/chosen": -68.27488708496094, "logps/ref_chosen": -56.96742630004883, "logps/ref_rejected": -86.36236572265625, "logps/rejected": -106.65916442871094, "loss": 0.9337, "margin_dpo/margin_mean": 8.989330291748047, "margin_dpo/margin_std": 11.496484756469727, "step": 284 }, { "KL/chosen_KL_mean": -14.233577728271484, "KL/mean": -18.005645751953125, "KL/rejected_KL_mean": -21.777708053588867, "KL/std": 10.855697631835938, "epoch": 0.4308390022675737, "fcm_dpo/beta": 0.09401977807283401, "fcm_dpo/delta": 0.09626507759094238, "fcm_dpo/margin": 7.544127464294434, "fcm_dpo/q_t": 0.36788657307624817, "grad_norm": 32.50141525268555, "learning_rate": 3.5264957352549375e-07, "logits/chosen": 0.3715507984161377, "logits/rejected": 0.34357473254203796, "logps/chosen": -85.88968658447266, "logps/ref_chosen": -71.65611267089844, "logps/ref_rejected": -81.63829803466797, "logps/rejected": -103.41600799560547, "loss": 1.0366, "margin_dpo/margin_mean": 7.544128894805908, "margin_dpo/margin_std": 11.49722671508789, "step": 285 }, { "KL/chosen_KL_mean": -12.889640808105469, "KL/mean": -18.95220947265625, "KL/rejected_KL_mean": -25.01477813720703, "KL/std": 12.186092376708984, "epoch": 0.4323507180650038, "fcm_dpo/beta": 0.08927808701992035, "fcm_dpo/delta": -0.31835198402404785, "fcm_dpo/margin": 12.125144958496094, "fcm_dpo/q_t": 0.29214009642601013, "grad_norm": 25.466272354125977, "learning_rate": 3.514425224712835e-07, "logits/chosen": 0.2750386595726013, "logits/rejected": 0.18463309109210968, "logps/chosen": -73.96916198730469, "logps/ref_chosen": -61.07952117919922, "logps/ref_rejected": -91.28128051757812, "logps/rejected": -116.29605102539062, "loss": 0.7992, "margin_dpo/margin_mean": 12.125143051147461, "margin_dpo/margin_std": 12.32309341430664, "step": 286 }, { "KL/chosen_KL_mean": -11.025419235229492, "KL/mean": -16.85832977294922, "KL/rejected_KL_mean": -22.691240310668945, "KL/std": 13.327524185180664, "epoch": 0.43386243386243384, "fcm_dpo/beta": 0.08564598858356476, "fcm_dpo/delta": -0.2206934094429016, "fcm_dpo/margin": 11.665821075439453, "fcm_dpo/q_t": 0.3125588297843933, "grad_norm": 24.484149932861328, "learning_rate": 3.502326338516534e-07, "logits/chosen": 0.3468093276023865, "logits/rejected": 0.30625537037849426, "logps/chosen": -57.06121063232422, "logps/ref_chosen": -46.035789489746094, "logps/ref_rejected": -59.95293426513672, "logps/rejected": -82.64418029785156, "loss": 0.853, "margin_dpo/margin_mean": 11.665821075439453, "margin_dpo/margin_std": 13.080723762512207, "step": 287 }, { "KL/chosen_KL_mean": -14.967060089111328, "KL/mean": -19.00350570678711, "KL/rejected_KL_mean": -23.03994369506836, "KL/std": 12.503591537475586, "epoch": 0.43537414965986393, "fcm_dpo/beta": 0.08595605194568634, "fcm_dpo/delta": 0.11360116302967072, "fcm_dpo/margin": 8.072890281677246, "fcm_dpo/q_t": 0.36748266220092773, "grad_norm": 31.567214965820312, "learning_rate": 3.490199415097892e-07, "logits/chosen": 0.21245820820331573, "logits/rejected": 0.1599254459142685, "logps/chosen": -80.35791015625, "logps/ref_chosen": -65.3908462524414, "logps/ref_rejected": -88.53607940673828, "logps/rejected": -111.57601928710938, "loss": 1.0563, "margin_dpo/margin_mean": 8.072890281677246, "margin_dpo/margin_std": 12.816825866699219, "step": 288 }, { "KL/chosen_KL_mean": -15.324104309082031, "KL/mean": -19.525135040283203, "KL/rejected_KL_mean": -23.72616195678711, "KL/std": 11.964265823364258, "epoch": 0.436885865457294, "fcm_dpo/beta": 0.08751742541790009, "fcm_dpo/delta": 0.06912409514188766, "fcm_dpo/margin": 8.402060508728027, "fcm_dpo/q_t": 0.3677643835544586, "grad_norm": 27.802019119262695, "learning_rate": 3.4780447936730247e-07, "logits/chosen": 0.40906429290771484, "logits/rejected": 0.3720993399620056, "logps/chosen": -69.91773223876953, "logps/ref_chosen": -54.5936279296875, "logps/ref_rejected": -67.20855712890625, "logps/rejected": -90.9347152709961, "loss": 1.07, "margin_dpo/margin_mean": 8.402060508728027, "margin_dpo/margin_std": 13.60258674621582, "step": 289 }, { "KL/chosen_KL_mean": -17.48955535888672, "KL/mean": -22.728565216064453, "KL/rejected_KL_mean": -27.967571258544922, "KL/std": 12.214452743530273, "epoch": 0.4383975812547241, "fcm_dpo/beta": 0.08549906313419342, "fcm_dpo/delta": -0.10699837654829025, "fcm_dpo/margin": 10.478013038635254, "fcm_dpo/q_t": 0.33341556787490845, "grad_norm": 32.633052825927734, "learning_rate": 3.465862814232821e-07, "logits/chosen": 0.4238796830177307, "logits/rejected": 0.35533106327056885, "logps/chosen": -78.87413024902344, "logps/ref_chosen": -61.38457489013672, "logps/ref_rejected": -91.92778015136719, "logps/rejected": -119.89534759521484, "loss": 0.9527, "margin_dpo/margin_mean": 10.478012084960938, "margin_dpo/margin_std": 13.828596115112305, "step": 290 }, { "KL/chosen_KL_mean": -16.39398765563965, "KL/mean": -21.614397048950195, "KL/rejected_KL_mean": -26.834808349609375, "KL/std": 12.491828918457031, "epoch": 0.4399092970521542, "fcm_dpo/beta": 0.08563029021024704, "fcm_dpo/delta": -0.10686700791120529, "fcm_dpo/margin": 10.44081974029541, "fcm_dpo/q_t": 0.33649247884750366, "grad_norm": 30.15843963623047, "learning_rate": 3.4536538175334343e-07, "logits/chosen": 0.4574052095413208, "logits/rejected": 0.3944157361984253, "logps/chosen": -67.25701904296875, "logps/ref_chosen": -50.863037109375, "logps/ref_rejected": -82.20868682861328, "logps/rejected": -109.04349517822266, "loss": 0.9609, "margin_dpo/margin_mean": 10.440818786621094, "margin_dpo/margin_std": 13.68259048461914, "step": 291 }, { "KL/chosen_KL_mean": -15.907407760620117, "KL/mean": -20.485567092895508, "KL/rejected_KL_mean": -25.063724517822266, "KL/std": 12.832338333129883, "epoch": 0.4414210128495843, "fcm_dpo/beta": 0.08440513163805008, "fcm_dpo/delta": 0.02916642650961876, "fcm_dpo/margin": 9.156310081481934, "fcm_dpo/q_t": 0.3514280319213867, "grad_norm": 32.388160705566406, "learning_rate": 3.4414181450867465e-07, "logits/chosen": 0.35157227516174316, "logits/rejected": 0.3027421832084656, "logps/chosen": -80.25629425048828, "logps/ref_chosen": -64.34888458251953, "logps/ref_rejected": -72.86434173583984, "logps/rejected": -97.92807006835938, "loss": 1.0183, "margin_dpo/margin_mean": 9.156310081481934, "margin_dpo/margin_std": 13.515422821044922, "step": 292 }, { "KL/chosen_KL_mean": -17.491422653198242, "KL/mean": -23.502586364746094, "KL/rejected_KL_mean": -29.513744354248047, "KL/std": 13.59200668334961, "epoch": 0.4429327286470144, "fcm_dpo/beta": 0.08238425850868225, "fcm_dpo/delta": -0.21106353402137756, "fcm_dpo/margin": 12.022321701049805, "fcm_dpo/q_t": 0.3155782222747803, "grad_norm": 22.351655960083008, "learning_rate": 3.4291561391508185e-07, "logits/chosen": 0.4406129717826843, "logits/rejected": 0.3572370707988739, "logps/chosen": -72.36088562011719, "logps/ref_chosen": -54.869468688964844, "logps/ref_rejected": -81.858642578125, "logps/rejected": -111.37239074707031, "loss": 0.93, "margin_dpo/margin_mean": 12.022321701049805, "margin_dpo/margin_std": 15.044371604919434, "step": 293 }, { "KL/chosen_KL_mean": -16.280914306640625, "KL/mean": -21.383319854736328, "KL/rejected_KL_mean": -26.485719680786133, "KL/std": 14.02008056640625, "epoch": 0.4444444444444444, "fcm_dpo/beta": 0.07986710965633392, "fcm_dpo/delta": -0.01819680631160736, "fcm_dpo/margin": 10.204802513122559, "fcm_dpo/q_t": 0.3480180501937866, "grad_norm": 23.034154891967773, "learning_rate": 3.4168681427203153e-07, "logits/chosen": 0.362077921628952, "logits/rejected": 0.31854838132858276, "logps/chosen": -72.95181274414062, "logps/ref_chosen": -56.670902252197266, "logps/ref_rejected": -70.32819366455078, "logps/rejected": -96.81391143798828, "loss": 0.952, "margin_dpo/margin_mean": 10.204801559448242, "margin_dpo/margin_std": 13.412900924682617, "step": 294 }, { "KL/chosen_KL_mean": -18.015230178833008, "KL/mean": -22.486682891845703, "KL/rejected_KL_mean": -26.958141326904297, "KL/std": 13.785063743591309, "epoch": 0.4459561602418745, "fcm_dpo/beta": 0.08139024674892426, "fcm_dpo/delta": 0.07783761620521545, "fcm_dpo/margin": 8.942903518676758, "fcm_dpo/q_t": 0.3671254515647888, "grad_norm": 30.077556610107422, "learning_rate": 3.4045544995169125e-07, "logits/chosen": 0.3807584345340729, "logits/rejected": 0.2897271513938904, "logps/chosen": -68.41612243652344, "logps/ref_chosen": -50.40088653564453, "logps/ref_rejected": -83.43521881103516, "logps/rejected": -110.39335632324219, "loss": 1.0405, "margin_dpo/margin_mean": 8.942903518676758, "margin_dpo/margin_std": 13.93426513671875, "step": 295 }, { "KL/chosen_KL_mean": -18.11003875732422, "KL/mean": -23.50696563720703, "KL/rejected_KL_mean": -28.903888702392578, "KL/std": 14.025440216064453, "epoch": 0.4474678760393046, "fcm_dpo/beta": 0.08038632571697235, "fcm_dpo/delta": -0.07529260218143463, "fcm_dpo/margin": 10.793851852416992, "fcm_dpo/q_t": 0.33729755878448486, "grad_norm": 28.054275512695312, "learning_rate": 3.392215553979679e-07, "logits/chosen": 0.3302781879901886, "logits/rejected": 0.28872621059417725, "logps/chosen": -87.26039123535156, "logps/ref_chosen": -69.15034484863281, "logps/ref_rejected": -89.60166931152344, "logps/rejected": -118.50555419921875, "loss": 0.9484, "margin_dpo/margin_mean": 10.793851852416992, "margin_dpo/margin_std": 14.17785930633545, "step": 296 }, { "KL/chosen_KL_mean": -19.260316848754883, "KL/mean": -24.803028106689453, "KL/rejected_KL_mean": -30.34573745727539, "KL/std": 12.583738327026367, "epoch": 0.4489795918367347, "fcm_dpo/beta": 0.08010244369506836, "fcm_dpo/delta": -0.09729193150997162, "fcm_dpo/margin": 11.085417747497559, "fcm_dpo/q_t": 0.32611143589019775, "grad_norm": 24.371068954467773, "learning_rate": 3.3798516512554485e-07, "logits/chosen": 0.3475460112094879, "logits/rejected": 0.28787142038345337, "logps/chosen": -77.27662658691406, "logps/ref_chosen": -58.01630401611328, "logps/ref_rejected": -69.95780944824219, "logps/rejected": -100.30354309082031, "loss": 0.8852, "margin_dpo/margin_mean": 11.085416793823242, "margin_dpo/margin_std": 12.328832626342773, "step": 297 }, { "KL/chosen_KL_mean": -18.95709991455078, "KL/mean": -23.594552993774414, "KL/rejected_KL_mean": -28.232009887695312, "KL/std": 13.951750755310059, "epoch": 0.4504913076341648, "fcm_dpo/beta": 0.07963920384645462, "fcm_dpo/delta": 0.06629949063062668, "fcm_dpo/margin": 9.274908065795898, "fcm_dpo/q_t": 0.3650180995464325, "grad_norm": 27.34107208251953, "learning_rate": 3.367463137189156e-07, "logits/chosen": 0.4564887285232544, "logits/rejected": 0.39622586965560913, "logps/chosen": -75.12641143798828, "logps/ref_chosen": -56.1693115234375, "logps/ref_rejected": -68.55052185058594, "logps/rejected": -96.78253173828125, "loss": 1.0754, "margin_dpo/margin_mean": 9.274909973144531, "margin_dpo/margin_std": 15.322766304016113, "step": 298 }, { "KL/chosen_KL_mean": -18.38543128967285, "KL/mean": -22.410602569580078, "KL/rejected_KL_mean": -26.435768127441406, "KL/std": 13.180704116821289, "epoch": 0.4520030234315949, "fcm_dpo/beta": 0.08154933154582977, "fcm_dpo/delta": 0.15386077761650085, "fcm_dpo/margin": 8.050336837768555, "fcm_dpo/q_t": 0.382235050201416, "grad_norm": 29.40821075439453, "learning_rate": 3.355050358314172e-07, "logits/chosen": 0.27602773904800415, "logits/rejected": 0.24822445213794708, "logps/chosen": -80.70323181152344, "logps/ref_chosen": -62.31780242919922, "logps/ref_rejected": -72.60028839111328, "logps/rejected": -99.03605651855469, "loss": 1.1539, "margin_dpo/margin_mean": 8.050336837768555, "margin_dpo/margin_std": 15.153702735900879, "step": 299 }, { "KL/chosen_KL_mean": -19.27455711364746, "KL/mean": -24.168970108032227, "KL/rejected_KL_mean": -29.06338119506836, "KL/std": 13.379597663879395, "epoch": 0.45351473922902497, "fcm_dpo/beta": 0.08266250789165497, "fcm_dpo/delta": -0.010115906596183777, "fcm_dpo/margin": 9.788824081420898, "fcm_dpo/q_t": 0.34636732935905457, "grad_norm": 28.359783172607422, "learning_rate": 3.3426136618426043e-07, "logits/chosen": 0.3955562710762024, "logits/rejected": 0.3320777416229248, "logps/chosen": -79.65613555908203, "logps/ref_chosen": -60.38157653808594, "logps/ref_rejected": -75.45442199707031, "logps/rejected": -104.51780700683594, "loss": 1.0271, "margin_dpo/margin_mean": 9.788824081420898, "margin_dpo/margin_std": 14.559015274047852, "step": 300 }, { "KL/chosen_KL_mean": -18.379371643066406, "KL/mean": -22.911306381225586, "KL/rejected_KL_mean": -27.443241119384766, "KL/std": 13.371219635009766, "epoch": 0.455026455026455, "fcm_dpo/beta": 0.08241571485996246, "fcm_dpo/delta": 0.05638513341546059, "fcm_dpo/margin": 9.063875198364258, "fcm_dpo/q_t": 0.3640829026699066, "grad_norm": 25.278783798217773, "learning_rate": 3.3301533956555885e-07, "logits/chosen": 0.42995303869247437, "logits/rejected": 0.39872127771377563, "logps/chosen": -71.23026275634766, "logps/ref_chosen": -52.85089111328125, "logps/ref_rejected": -69.97584533691406, "logps/rejected": -97.4190902709961, "loss": 1.0742, "margin_dpo/margin_mean": 9.063875198364258, "margin_dpo/margin_std": 14.9423828125, "step": 301 }, { "KL/chosen_KL_mean": -20.356815338134766, "KL/mean": -23.603713989257812, "KL/rejected_KL_mean": -26.850624084472656, "KL/std": 12.972681045532227, "epoch": 0.4565381708238851, "fcm_dpo/beta": 0.08642345666885376, "fcm_dpo/delta": 0.2532394230365753, "fcm_dpo/margin": 6.493806838989258, "fcm_dpo/q_t": 0.39695829153060913, "grad_norm": 33.59251403808594, "learning_rate": 3.317669908293554e-07, "logits/chosen": 0.26218295097351074, "logits/rejected": 0.2100965827703476, "logps/chosen": -87.32332611083984, "logps/ref_chosen": -66.96650695800781, "logps/ref_rejected": -88.09510803222656, "logps/rejected": -114.94573974609375, "loss": 1.1935, "margin_dpo/margin_mean": 6.493806838989258, "margin_dpo/margin_std": 13.955561637878418, "step": 302 }, { "KL/chosen_KL_mean": -17.155065536499023, "KL/mean": -23.425048828125, "KL/rejected_KL_mean": -29.695026397705078, "KL/std": 13.855140686035156, "epoch": 0.4580498866213152, "fcm_dpo/beta": 0.08415515720844269, "fcm_dpo/delta": -0.2851126790046692, "fcm_dpo/margin": 12.539966583251953, "fcm_dpo/q_t": 0.31551820039749146, "grad_norm": 24.16552734375, "learning_rate": 3.3051635489464793e-07, "logits/chosen": 0.37548404932022095, "logits/rejected": 0.3113349378108978, "logps/chosen": -79.27659606933594, "logps/ref_chosen": -62.12152862548828, "logps/ref_rejected": -90.31204223632812, "logps/rejected": -120.00707244873047, "loss": 0.9423, "margin_dpo/margin_mean": 12.539965629577637, "margin_dpo/margin_std": 16.383039474487305, "step": 303 }, { "KL/chosen_KL_mean": -15.978334426879883, "KL/mean": -22.127933502197266, "KL/rejected_KL_mean": -28.277530670166016, "KL/std": 14.6383638381958, "epoch": 0.4595616024187453, "fcm_dpo/beta": 0.08040931820869446, "fcm_dpo/delta": -0.20942339301109314, "fcm_dpo/margin": 12.299189567565918, "fcm_dpo/q_t": 0.3040643036365509, "grad_norm": 24.452234268188477, "learning_rate": 3.292634667444117e-07, "logits/chosen": 0.35874032974243164, "logits/rejected": 0.3055311143398285, "logps/chosen": -76.67342376708984, "logps/ref_chosen": -60.695091247558594, "logps/ref_rejected": -78.2525405883789, "logps/rejected": -106.53007507324219, "loss": 0.827, "margin_dpo/margin_mean": 12.299190521240234, "margin_dpo/margin_std": 12.52480697631836, "step": 304 }, { "KL/chosen_KL_mean": -19.23602294921875, "KL/mean": -24.019302368164062, "KL/rejected_KL_mean": -28.80258560180664, "KL/std": 13.73517894744873, "epoch": 0.46107331821617537, "fcm_dpo/beta": 0.07919107377529144, "fcm_dpo/delta": 0.0445355623960495, "fcm_dpo/margin": 9.566566467285156, "fcm_dpo/q_t": 0.35818392038345337, "grad_norm": 30.306407928466797, "learning_rate": 3.280083614246217e-07, "logits/chosen": 0.3176822066307068, "logits/rejected": 0.33436745405197144, "logps/chosen": -91.93516540527344, "logps/ref_chosen": -72.69914245605469, "logps/ref_rejected": -65.65670776367188, "logps/rejected": -94.45929718017578, "loss": 1.0632, "margin_dpo/margin_mean": 9.566566467285156, "margin_dpo/margin_std": 15.191731452941895, "step": 305 }, { "KL/chosen_KL_mean": -17.9437198638916, "KL/mean": -22.24250602722168, "KL/rejected_KL_mean": -26.541290283203125, "KL/std": 13.069320678710938, "epoch": 0.46258503401360546, "fcm_dpo/beta": 0.08063946664333344, "fcm_dpo/delta": 0.11361439526081085, "fcm_dpo/margin": 8.597566604614258, "fcm_dpo/q_t": 0.3632877469062805, "grad_norm": 24.9428653717041, "learning_rate": 3.267510740432719e-07, "logits/chosen": 0.41871410608291626, "logits/rejected": 0.3094066083431244, "logps/chosen": -71.91424560546875, "logps/ref_chosen": -53.97052764892578, "logps/ref_rejected": -71.02423095703125, "logps/rejected": -97.56552124023438, "loss": 1.0196, "margin_dpo/margin_mean": 8.597567558288574, "margin_dpo/margin_std": 12.287683486938477, "step": 306 }, { "KL/chosen_KL_mean": -15.186479568481445, "KL/mean": -18.630168914794922, "KL/rejected_KL_mean": -22.073871612548828, "KL/std": 13.487642288208008, "epoch": 0.46409674981103555, "fcm_dpo/beta": 0.08535341918468475, "fcm_dpo/delta": 0.22243700921535492, "fcm_dpo/margin": 6.887389659881592, "fcm_dpo/q_t": 0.40226835012435913, "grad_norm": 32.934783935546875, "learning_rate": 3.2549163976939285e-07, "logits/chosen": 0.39580583572387695, "logits/rejected": 0.35165637731552124, "logps/chosen": -72.5995864868164, "logps/ref_chosen": -57.413108825683594, "logps/ref_rejected": -68.68010711669922, "logps/rejected": -90.75398254394531, "loss": 1.2401, "margin_dpo/margin_mean": 6.887389183044434, "margin_dpo/margin_std": 15.407466888427734, "step": 307 }, { "KL/chosen_KL_mean": -16.288928985595703, "KL/mean": -20.996000289916992, "KL/rejected_KL_mean": -25.703067779541016, "KL/std": 13.363460540771484, "epoch": 0.4656084656084656, "fcm_dpo/beta": 0.08504834771156311, "fcm_dpo/delta": -0.0017690509557724, "fcm_dpo/margin": 9.414133071899414, "fcm_dpo/q_t": 0.3468315005302429, "grad_norm": 26.06618309020996, "learning_rate": 3.2423009383206874e-07, "logits/chosen": 0.36595430970191956, "logits/rejected": 0.34875592589378357, "logps/chosen": -82.88772583007812, "logps/ref_chosen": -66.59879302978516, "logps/ref_rejected": -74.337158203125, "logps/rejected": -100.04022216796875, "loss": 1.024, "margin_dpo/margin_mean": 9.414133071899414, "margin_dpo/margin_std": 13.817611694335938, "step": 308 }, { "KL/chosen_KL_mean": -19.097213745117188, "KL/mean": -23.660863876342773, "KL/rejected_KL_mean": -28.224517822265625, "KL/std": 12.788629531860352, "epoch": 0.4671201814058957, "fcm_dpo/beta": 0.08512066304683685, "fcm_dpo/delta": 0.023319870233535767, "fcm_dpo/margin": 9.12730598449707, "fcm_dpo/q_t": 0.3563210070133209, "grad_norm": 34.374366760253906, "learning_rate": 3.229664715194511e-07, "logits/chosen": 0.4130929410457611, "logits/rejected": 0.3572024703025818, "logps/chosen": -84.49195861816406, "logps/ref_chosen": -65.39474487304688, "logps/ref_rejected": -75.70930480957031, "logps/rejected": -103.93382263183594, "loss": 1.0346, "margin_dpo/margin_mean": 9.12730598449707, "margin_dpo/margin_std": 13.821783065795898, "step": 309 }, { "KL/chosen_KL_mean": -19.24032974243164, "KL/mean": -22.127826690673828, "KL/rejected_KL_mean": -25.015323638916016, "KL/std": 11.900168418884277, "epoch": 0.46863189720332576, "fcm_dpo/beta": 0.08966143429279327, "fcm_dpo/delta": 0.2983810007572174, "fcm_dpo/margin": 5.774989128112793, "fcm_dpo/q_t": 0.41219472885131836, "grad_norm": 35.78599548339844, "learning_rate": 3.2170080817777257e-07, "logits/chosen": 0.4417685568332672, "logits/rejected": 0.429890513420105, "logps/chosen": -93.90859985351562, "logps/ref_chosen": -74.66827392578125, "logps/ref_rejected": -80.5689697265625, "logps/rejected": -105.58428955078125, "loss": 1.2452, "margin_dpo/margin_mean": 5.774989128112793, "margin_dpo/margin_std": 13.688153266906738, "step": 310 }, { "KL/chosen_KL_mean": -15.007062911987305, "KL/mean": -19.197357177734375, "KL/rejected_KL_mean": -23.387653350830078, "KL/std": 14.218526840209961, "epoch": 0.47014361300075586, "fcm_dpo/beta": 0.09115281701087952, "fcm_dpo/delta": 0.03813778609037399, "fcm_dpo/margin": 8.380594253540039, "fcm_dpo/q_t": 0.3628461956977844, "grad_norm": 31.881589889526367, "learning_rate": 3.204331392103574e-07, "logits/chosen": 0.31646156311035156, "logits/rejected": 0.1944255530834198, "logps/chosen": -74.7450942993164, "logps/ref_chosen": -59.738033294677734, "logps/ref_rejected": -93.60757446289062, "logps/rejected": -116.99522399902344, "loss": 1.0809, "margin_dpo/margin_mean": 8.380593299865723, "margin_dpo/margin_std": 14.251326560974121, "step": 311 }, { "KL/chosen_KL_mean": -15.454090118408203, "KL/mean": -21.195587158203125, "KL/rejected_KL_mean": -26.937084197998047, "KL/std": 13.556852340698242, "epoch": 0.47165532879818595, "fcm_dpo/beta": 0.08916601538658142, "fcm_dpo/delta": -0.24872109293937683, "fcm_dpo/margin": 11.482992172241211, "fcm_dpo/q_t": 0.30444252490997314, "grad_norm": 24.69047737121582, "learning_rate": 3.1916350007663176e-07, "logits/chosen": 0.3721472918987274, "logits/rejected": 0.27872756123542786, "logps/chosen": -69.27052307128906, "logps/ref_chosen": -53.816436767578125, "logps/ref_rejected": -68.6575698852539, "logps/rejected": -95.59465026855469, "loss": 0.8523, "margin_dpo/margin_mean": 11.482992172241211, "margin_dpo/margin_std": 12.813817977905273, "step": 312 }, { "KL/chosen_KL_mean": -15.105289459228516, "KL/mean": -18.892580032348633, "KL/rejected_KL_mean": -22.679874420166016, "KL/std": 13.558688163757324, "epoch": 0.47316704459561604, "fcm_dpo/beta": 0.08916016668081284, "fcm_dpo/delta": 0.13385319709777832, "fcm_dpo/margin": 7.574581146240234, "fcm_dpo/q_t": 0.3830859661102295, "grad_norm": 30.752466201782227, "learning_rate": 3.178919262911314e-07, "logits/chosen": 0.4537992477416992, "logits/rejected": 0.431113064289093, "logps/chosen": -75.06265258789062, "logps/ref_chosen": -59.957359313964844, "logps/ref_rejected": -69.31729888916016, "logps/rejected": -91.99717712402344, "loss": 1.1541, "margin_dpo/margin_mean": 7.574580669403076, "margin_dpo/margin_std": 14.770940780639648, "step": 313 }, { "KL/chosen_KL_mean": -14.3424072265625, "KL/mean": -20.40030288696289, "KL/rejected_KL_mean": -26.458194732666016, "KL/std": 13.566604614257812, "epoch": 0.47467876039304613, "fcm_dpo/beta": 0.0859234556555748, "fcm_dpo/delta": -0.27131664752960205, "fcm_dpo/margin": 12.1157808303833, "fcm_dpo/q_t": 0.30542221665382385, "grad_norm": 24.523052215576172, "learning_rate": 3.166184534225087e-07, "logits/chosen": 0.37127453088760376, "logits/rejected": 0.38916367292404175, "logps/chosen": -84.61056518554688, "logps/ref_chosen": -70.26815795898438, "logps/ref_rejected": -69.23971557617188, "logps/rejected": -95.69790649414062, "loss": 0.8307, "margin_dpo/margin_mean": 12.1157808303833, "margin_dpo/margin_std": 13.054606437683105, "step": 314 }, { "KL/chosen_KL_mean": -15.060813903808594, "KL/mean": -19.923267364501953, "KL/rejected_KL_mean": -24.785724639892578, "KL/std": 13.563258171081543, "epoch": 0.47619047619047616, "fcm_dpo/beta": 0.08582372963428497, "fcm_dpo/delta": -0.040948450565338135, "fcm_dpo/margin": 9.724916458129883, "fcm_dpo/q_t": 0.3378736972808838, "grad_norm": 25.47266960144043, "learning_rate": 3.1534311709253723e-07, "logits/chosen": 0.3101431727409363, "logits/rejected": 0.2680996358394623, "logps/chosen": -82.85550689697266, "logps/ref_chosen": -67.79469299316406, "logps/ref_rejected": -74.55148315429688, "logps/rejected": -99.33720397949219, "loss": 0.9679, "margin_dpo/margin_mean": 9.724916458129883, "margin_dpo/margin_std": 12.58622932434082, "step": 315 }, { "KL/chosen_KL_mean": -14.215065002441406, "KL/mean": -20.012006759643555, "KL/rejected_KL_mean": -25.80895233154297, "KL/std": 13.516332626342773, "epoch": 0.47770219198790626, "fcm_dpo/beta": 0.08367159217596054, "fcm_dpo/delta": -0.19175595045089722, "fcm_dpo/margin": 11.593890190124512, "fcm_dpo/q_t": 0.31763750314712524, "grad_norm": 27.956092834472656, "learning_rate": 3.1406595297511564e-07, "logits/chosen": 0.2943209409713745, "logits/rejected": 0.17805573344230652, "logps/chosen": -69.50354766845703, "logps/ref_chosen": -55.288482666015625, "logps/ref_rejected": -96.15723419189453, "logps/rejected": -121.9661865234375, "loss": 0.8898, "margin_dpo/margin_mean": 11.593889236450195, "margin_dpo/margin_std": 12.889469146728516, "step": 316 }, { "KL/chosen_KL_mean": -12.102411270141602, "KL/mean": -18.74686050415039, "KL/rejected_KL_mean": -25.391311645507812, "KL/std": 13.107463836669922, "epoch": 0.47921390778533635, "fcm_dpo/beta": 0.07773043215274811, "fcm_dpo/delta": -0.26238229870796204, "fcm_dpo/margin": 13.288894653320312, "fcm_dpo/q_t": 0.2983187437057495, "grad_norm": 23.87213134765625, "learning_rate": 3.1278699679526975e-07, "logits/chosen": 0.44091135263442993, "logits/rejected": 0.39091646671295166, "logps/chosen": -66.68379211425781, "logps/ref_chosen": -54.58137512207031, "logps/ref_rejected": -72.77232360839844, "logps/rejected": -98.16363525390625, "loss": 0.8, "margin_dpo/margin_mean": 13.288894653320312, "margin_dpo/margin_std": 13.080042839050293, "step": 317 }, { "KL/chosen_KL_mean": -14.92811393737793, "KL/mean": -19.01508331298828, "KL/rejected_KL_mean": -23.10205841064453, "KL/std": 13.509125709533691, "epoch": 0.48072562358276644, "fcm_dpo/beta": 0.07906337827444077, "fcm_dpo/delta": 0.16437119245529175, "fcm_dpo/margin": 8.173942565917969, "fcm_dpo/q_t": 0.38638216257095337, "grad_norm": 29.862834930419922, "learning_rate": 3.1150628432815336e-07, "logits/chosen": 0.4298110008239746, "logits/rejected": 0.36635473370552063, "logps/chosen": -67.81634521484375, "logps/ref_chosen": -52.88822937011719, "logps/ref_rejected": -80.63988494873047, "logps/rejected": -103.741943359375, "loss": 1.223, "margin_dpo/margin_mean": 8.173942565917969, "margin_dpo/margin_std": 17.332202911376953, "step": 318 }, { "KL/chosen_KL_mean": -12.821718215942383, "KL/mean": -18.56632423400879, "KL/rejected_KL_mean": -24.310924530029297, "KL/std": 13.857823371887207, "epoch": 0.48223733938019653, "fcm_dpo/beta": 0.07831594347953796, "fcm_dpo/delta": -0.10991809517145157, "fcm_dpo/margin": 11.489205360412598, "fcm_dpo/q_t": 0.3347237706184387, "grad_norm": 24.75864601135254, "learning_rate": 3.1022385139804707e-07, "logits/chosen": 0.35719725489616394, "logits/rejected": 0.33642852306365967, "logps/chosen": -77.18505096435547, "logps/ref_chosen": -64.36333465576172, "logps/ref_rejected": -79.47296142578125, "logps/rejected": -103.78388214111328, "loss": 0.9462, "margin_dpo/margin_mean": 11.489205360412598, "margin_dpo/margin_std": 15.297908782958984, "step": 319 }, { "KL/chosen_KL_mean": -12.259525299072266, "KL/mean": -16.699268341064453, "KL/rejected_KL_mean": -21.13901138305664, "KL/std": 14.101001739501953, "epoch": 0.4837490551776266, "fcm_dpo/beta": 0.08037566393613815, "fcm_dpo/delta": 0.08531537652015686, "fcm_dpo/margin": 8.879487991333008, "fcm_dpo/q_t": 0.3749288320541382, "grad_norm": 27.196882247924805, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 0.2817011773586273, "logits/rejected": 0.23932918906211853, "logps/chosen": -61.81827163696289, "logps/ref_chosen": -49.558746337890625, "logps/ref_rejected": -71.23444366455078, "logps/rejected": -92.37345886230469, "loss": 1.0895, "margin_dpo/margin_mean": 8.879487037658691, "margin_dpo/margin_std": 14.906489372253418, "step": 320 }, { "KL/chosen_KL_mean": -13.375057220458984, "KL/mean": -18.363353729248047, "KL/rejected_KL_mean": -23.351652145385742, "KL/std": 13.216703414916992, "epoch": 0.4852607709750567, "fcm_dpo/beta": 0.07867099344730377, "fcm_dpo/delta": 0.014172382652759552, "fcm_dpo/margin": 9.976592063903809, "fcm_dpo/q_t": 0.34726887941360474, "grad_norm": 24.233501434326172, "learning_rate": 3.0765396768561004e-07, "logits/chosen": 0.3476206064224243, "logits/rejected": 0.3286382555961609, "logps/chosen": -65.4603271484375, "logps/ref_chosen": -52.08526611328125, "logps/ref_rejected": -55.58674621582031, "logps/rejected": -78.93840026855469, "loss": 1.0234, "margin_dpo/margin_mean": 9.976592063903809, "margin_dpo/margin_std": 14.353278160095215, "step": 321 }, { "KL/chosen_KL_mean": -14.237586975097656, "KL/mean": -20.744802474975586, "KL/rejected_KL_mean": -27.252017974853516, "KL/std": 13.349346160888672, "epoch": 0.48677248677248675, "fcm_dpo/beta": 0.07750476151704788, "fcm_dpo/delta": -0.23154690861701965, "fcm_dpo/margin": 13.014429092407227, "fcm_dpo/q_t": 0.2995946407318115, "grad_norm": 22.012561798095703, "learning_rate": 3.063665887884511e-07, "logits/chosen": 0.4243776202201843, "logits/rejected": 0.34563034772872925, "logps/chosen": -61.64169692993164, "logps/ref_chosen": -47.404109954833984, "logps/ref_rejected": -73.4260025024414, "logps/rejected": -100.67802429199219, "loss": 0.8164, "margin_dpo/margin_mean": 13.014429092407227, "margin_dpo/margin_std": 12.93503189086914, "step": 322 }, { "KL/chosen_KL_mean": -14.6673583984375, "KL/mean": -19.150596618652344, "KL/rejected_KL_mean": -23.633834838867188, "KL/std": 14.315862655639648, "epoch": 0.48828420256991684, "fcm_dpo/beta": 0.07800118625164032, "fcm_dpo/delta": 0.10459380596876144, "fcm_dpo/margin": 8.966480255126953, "fcm_dpo/q_t": 0.38355547189712524, "grad_norm": 29.142436981201172, "learning_rate": 3.0507763319663517e-07, "logits/chosen": 0.33733895421028137, "logits/rejected": 0.2629064917564392, "logps/chosen": -84.67366027832031, "logps/ref_chosen": -70.00630187988281, "logps/ref_rejected": -86.96690368652344, "logps/rejected": -110.60073852539062, "loss": 1.1505, "margin_dpo/margin_mean": 8.966480255126953, "margin_dpo/margin_std": 17.100955963134766, "step": 323 }, { "KL/chosen_KL_mean": -11.98857307434082, "KL/mean": -17.415842056274414, "KL/rejected_KL_mean": -22.84311294555664, "KL/std": 14.033093452453613, "epoch": 0.4897959183673469, "fcm_dpo/beta": 0.0767996609210968, "fcm_dpo/delta": -0.03729076683521271, "fcm_dpo/margin": 10.854536056518555, "fcm_dpo/q_t": 0.33308106660842896, "grad_norm": 26.14889907836914, "learning_rate": 3.0378713696502097e-07, "logits/chosen": 0.4245554208755493, "logits/rejected": 0.366682767868042, "logps/chosen": -67.87739562988281, "logps/ref_chosen": -55.88882064819336, "logps/ref_rejected": -75.23088073730469, "logps/rejected": -98.07398986816406, "loss": 0.8972, "margin_dpo/margin_mean": 10.854536056518555, "margin_dpo/margin_std": 12.291630744934082, "step": 324 }, { "KL/chosen_KL_mean": -15.521261215209961, "KL/mean": -20.76903533935547, "KL/rejected_KL_mean": -26.016807556152344, "KL/std": 13.765624046325684, "epoch": 0.491307634164777, "fcm_dpo/beta": 0.07600394636392593, "fcm_dpo/delta": -0.001028984785079956, "fcm_dpo/margin": 10.495555877685547, "fcm_dpo/q_t": 0.34639471769332886, "grad_norm": 24.565717697143555, "learning_rate": 3.0249513619156206e-07, "logits/chosen": 0.36406660079956055, "logits/rejected": 0.3028401732444763, "logps/chosen": -79.66827392578125, "logps/ref_chosen": -64.14701843261719, "logps/ref_rejected": -79.91143798828125, "logps/rejected": -105.92823791503906, "loss": 0.9933, "margin_dpo/margin_mean": 10.495555877685547, "margin_dpo/margin_std": 14.726129531860352, "step": 325 }, { "KL/chosen_KL_mean": -18.79259490966797, "KL/mean": -21.617759704589844, "KL/rejected_KL_mean": -24.44293212890625, "KL/std": 13.5921630859375, "epoch": 0.4928193499622071, "fcm_dpo/beta": 0.08110679686069489, "fcm_dpo/delta": 0.3585972487926483, "fcm_dpo/margin": 5.650336265563965, "fcm_dpo/q_t": 0.4131473898887634, "grad_norm": 33.04014205932617, "learning_rate": 3.012016670162977e-07, "logits/chosen": 0.31773847341537476, "logits/rejected": 0.3185911178588867, "logps/chosen": -94.32390594482422, "logps/ref_chosen": -75.53131103515625, "logps/ref_rejected": -76.5898666381836, "logps/rejected": -101.03279876708984, "loss": 1.2653, "margin_dpo/margin_mean": 5.650336265563965, "margin_dpo/margin_std": 14.059412956237793, "step": 326 }, { "KL/chosen_KL_mean": -16.67436981201172, "KL/mean": -21.07805633544922, "KL/rejected_KL_mean": -25.481735229492188, "KL/std": 14.634458541870117, "epoch": 0.4943310657596372, "fcm_dpo/beta": 0.08383779972791672, "fcm_dpo/delta": 0.06597737222909927, "fcm_dpo/margin": 8.807355880737305, "fcm_dpo/q_t": 0.37003931403160095, "grad_norm": 30.968721389770508, "learning_rate": 2.99906765620341e-07, "logits/chosen": 0.2856214940547943, "logits/rejected": 0.24533884227275848, "logps/chosen": -86.01153564453125, "logps/ref_chosen": -69.33717346191406, "logps/ref_rejected": -73.37751770019531, "logps/rejected": -98.8592529296875, "loss": 1.1417, "margin_dpo/margin_mean": 8.807354927062988, "margin_dpo/margin_std": 16.436473846435547, "step": 327 }, { "KL/chosen_KL_mean": -13.426244735717773, "KL/mean": -18.46278953552246, "KL/rejected_KL_mean": -23.49933624267578, "KL/std": 13.932609558105469, "epoch": 0.4958427815570673, "fcm_dpo/beta": 0.08340919762849808, "fcm_dpo/delta": -0.043860193341970444, "fcm_dpo/margin": 10.073086738586426, "fcm_dpo/q_t": 0.342735230922699, "grad_norm": 26.51942253112793, "learning_rate": 2.9861046822486766e-07, "logits/chosen": 0.3326423764228821, "logits/rejected": 0.3011217713356018, "logps/chosen": -75.13247680664062, "logps/ref_chosen": -61.70623016357422, "logps/ref_rejected": -83.73808288574219, "logps/rejected": -107.23741149902344, "loss": 0.9377, "margin_dpo/margin_mean": 10.073087692260742, "margin_dpo/margin_std": 13.004785537719727, "step": 328 }, { "KL/chosen_KL_mean": -16.34695816040039, "KL/mean": -21.78904151916504, "KL/rejected_KL_mean": -27.231124877929688, "KL/std": 14.933101654052734, "epoch": 0.4973544973544973, "fcm_dpo/beta": 0.08136263489723206, "fcm_dpo/delta": -0.09547744691371918, "fcm_dpo/margin": 10.884174346923828, "fcm_dpo/q_t": 0.33907365798950195, "grad_norm": 28.799869537353516, "learning_rate": 2.9731281109010253e-07, "logits/chosen": 0.40021979808807373, "logits/rejected": 0.346312016248703, "logps/chosen": -80.84536743164062, "logps/ref_chosen": -64.4984130859375, "logps/ref_rejected": -83.6591796875, "logps/rejected": -110.89030456542969, "loss": 0.9483, "margin_dpo/margin_mean": 10.884174346923828, "margin_dpo/margin_std": 14.5554780960083, "step": 329 }, { "KL/chosen_KL_mean": -13.03089714050293, "KL/mean": -18.402721405029297, "KL/rejected_KL_mean": -23.77454376220703, "KL/std": 13.772836685180664, "epoch": 0.4988662131519274, "fcm_dpo/beta": 0.0811014473438263, "fcm_dpo/delta": -0.07852261513471603, "fcm_dpo/margin": 10.743646621704102, "fcm_dpo/q_t": 0.33982962369918823, "grad_norm": 25.62102699279785, "learning_rate": 2.9601383051430505e-07, "logits/chosen": 0.3539420962333679, "logits/rejected": 0.2868247628211975, "logps/chosen": -67.83554077148438, "logps/ref_chosen": -54.80464172363281, "logps/ref_rejected": -75.3194351196289, "logps/rejected": -99.09397888183594, "loss": 1.0445, "margin_dpo/margin_mean": 10.743644714355469, "margin_dpo/margin_std": 16.401023864746094, "step": 330 }, { "KL/chosen_KL_mean": -14.137678146362305, "KL/mean": -21.697162628173828, "KL/rejected_KL_mean": -29.256637573242188, "KL/std": 15.32611083984375, "epoch": 0.5003779289493575, "fcm_dpo/beta": 0.07569363713264465, "fcm_dpo/delta": -0.3894200921058655, "fcm_dpo/margin": 15.118959426879883, "fcm_dpo/q_t": 0.29206815361976624, "grad_norm": 22.627124786376953, "learning_rate": 2.947135628327544e-07, "logits/chosen": 0.4677159786224365, "logits/rejected": 0.4367586076259613, "logps/chosen": -73.38026428222656, "logps/ref_chosen": -59.242584228515625, "logps/ref_rejected": -69.87483215332031, "logps/rejected": -99.1314697265625, "loss": 0.8435, "margin_dpo/margin_mean": 15.118961334228516, "margin_dpo/margin_std": 17.141651153564453, "step": 331 }, { "KL/chosen_KL_mean": -15.791358947753906, "KL/mean": -21.50833511352539, "KL/rejected_KL_mean": -27.22531509399414, "KL/std": 14.802399635314941, "epoch": 0.5018896447467877, "fcm_dpo/beta": 0.07454757392406464, "fcm_dpo/delta": -0.060058847069740295, "fcm_dpo/margin": 11.433955192565918, "fcm_dpo/q_t": 0.342519074678421, "grad_norm": 24.499897003173828, "learning_rate": 2.934120444167326e-07, "logits/chosen": 0.3224112391471863, "logits/rejected": 0.2759397625923157, "logps/chosen": -82.90111541748047, "logps/ref_chosen": -67.10975646972656, "logps/ref_rejected": -77.11839294433594, "logps/rejected": -104.34370422363281, "loss": 0.9353, "margin_dpo/margin_mean": 11.433955192565918, "margin_dpo/margin_std": 14.521880149841309, "step": 332 }, { "KL/chosen_KL_mean": -16.0902156829834, "KL/mean": -23.16205596923828, "KL/rejected_KL_mean": -30.23389434814453, "KL/std": 15.021678924560547, "epoch": 0.5034013605442177, "fcm_dpo/beta": 0.07109569013118744, "fcm_dpo/delta": -0.22809955477714539, "fcm_dpo/margin": 14.1436767578125, "fcm_dpo/q_t": 0.3082965314388275, "grad_norm": 21.058923721313477, "learning_rate": 2.921093116725076e-07, "logits/chosen": 0.42060232162475586, "logits/rejected": 0.3509610593318939, "logps/chosen": -74.47134399414062, "logps/ref_chosen": -58.381134033203125, "logps/ref_rejected": -85.02839660644531, "logps/rejected": -115.26228332519531, "loss": 0.8292, "margin_dpo/margin_mean": 14.1436767578125, "margin_dpo/margin_std": 15.049093246459961, "step": 333 }, { "KL/chosen_KL_mean": -16.76015853881836, "KL/mean": -21.550758361816406, "KL/rejected_KL_mean": -26.341354370117188, "KL/std": 15.843827247619629, "epoch": 0.5049130763416477, "fcm_dpo/beta": 0.0714268833398819, "fcm_dpo/delta": 0.1240236908197403, "fcm_dpo/margin": 9.581196784973145, "fcm_dpo/q_t": 0.37355661392211914, "grad_norm": 28.009719848632812, "learning_rate": 2.9080540104031484e-07, "logits/chosen": 0.4134102761745453, "logits/rejected": 0.3671502470970154, "logps/chosen": -83.65216064453125, "logps/ref_chosen": -66.89199829101562, "logps/ref_rejected": -91.83695220947266, "logps/rejected": -118.17830657958984, "loss": 1.1305, "margin_dpo/margin_mean": 9.581195831298828, "margin_dpo/margin_std": 17.586097717285156, "step": 334 }, { "KL/chosen_KL_mean": -16.416725158691406, "KL/mean": -22.06592559814453, "KL/rejected_KL_mean": -27.715133666992188, "KL/std": 15.565815925598145, "epoch": 0.5064247921390779, "fcm_dpo/beta": 0.07200112193822861, "fcm_dpo/delta": -0.01507379487156868, "fcm_dpo/margin": 11.298408508300781, "fcm_dpo/q_t": 0.3517000675201416, "grad_norm": 25.294702529907227, "learning_rate": 2.895003489933375e-07, "logits/chosen": 0.395161509513855, "logits/rejected": 0.3578339219093323, "logps/chosen": -77.93118286132812, "logps/ref_chosen": -61.51445770263672, "logps/ref_rejected": -75.68916320800781, "logps/rejected": -103.404296875, "loss": 1.0078, "margin_dpo/margin_mean": 11.298408508300781, "margin_dpo/margin_std": 16.401405334472656, "step": 335 }, { "KL/chosen_KL_mean": -21.026308059692383, "KL/mean": -26.18271255493164, "KL/rejected_KL_mean": -31.33911895751953, "KL/std": 15.847661972045898, "epoch": 0.5079365079365079, "fcm_dpo/beta": 0.07149013131856918, "fcm_dpo/delta": 0.0657254010438919, "fcm_dpo/margin": 10.312803268432617, "fcm_dpo/q_t": 0.37415605783462524, "grad_norm": 27.998376846313477, "learning_rate": 2.8819419203668675e-07, "logits/chosen": 0.32004469633102417, "logits/rejected": 0.29973897337913513, "logps/chosen": -89.87637329101562, "logps/ref_chosen": -68.85006713867188, "logps/ref_rejected": -92.99603271484375, "logps/rejected": -124.33514404296875, "loss": 1.0709, "margin_dpo/margin_mean": 10.312803268432617, "margin_dpo/margin_std": 17.34792709350586, "step": 336 }, { "KL/chosen_KL_mean": -20.296260833740234, "KL/mean": -24.938064575195312, "KL/rejected_KL_mean": -29.57986831665039, "KL/std": 15.817164421081543, "epoch": 0.509448223733938, "fcm_dpo/beta": 0.07388242334127426, "fcm_dpo/delta": 0.12251827120780945, "fcm_dpo/margin": 9.283602714538574, "fcm_dpo/q_t": 0.3708915710449219, "grad_norm": 26.191547393798828, "learning_rate": 2.8688696670638053e-07, "logits/chosen": 0.2764531373977661, "logits/rejected": 0.24225616455078125, "logps/chosen": -93.48410034179688, "logps/ref_chosen": -73.18783569335938, "logps/ref_rejected": -86.89118957519531, "logps/rejected": -116.47105407714844, "loss": 1.0657, "margin_dpo/margin_mean": 9.283601760864258, "margin_dpo/margin_std": 15.305516242980957, "step": 337 }, { "KL/chosen_KL_mean": -20.28105926513672, "KL/mean": -25.161582946777344, "KL/rejected_KL_mean": -30.042102813720703, "KL/std": 15.790666580200195, "epoch": 0.5109599395313681, "fcm_dpo/beta": 0.07535530626773834, "fcm_dpo/delta": 0.06915910542011261, "fcm_dpo/margin": 9.761046409606934, "fcm_dpo/q_t": 0.36161673069000244, "grad_norm": 27.036073684692383, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 0.3518291711807251, "logits/rejected": 0.3244253396987915, "logps/chosen": -84.22067260742188, "logps/ref_chosen": -63.939613342285156, "logps/ref_rejected": -75.34243774414062, "logps/rejected": -105.3845443725586, "loss": 1.0544, "margin_dpo/margin_mean": 9.761046409606934, "margin_dpo/margin_std": 15.557717323303223, "step": 338 }, { "KL/chosen_KL_mean": -16.798437118530273, "KL/mean": -22.42432975769043, "KL/rejected_KL_mean": -28.050220489501953, "KL/std": 14.957649230957031, "epoch": 0.5124716553287982, "fcm_dpo/beta": 0.07454653829336166, "fcm_dpo/delta": -0.04305371642112732, "fcm_dpo/margin": 11.251781463623047, "fcm_dpo/q_t": 0.34308749437332153, "grad_norm": 24.66077423095703, "learning_rate": 2.842694572172736e-07, "logits/chosen": 0.49047064781188965, "logits/rejected": 0.40806323289871216, "logps/chosen": -62.34756851196289, "logps/ref_chosen": -45.54913330078125, "logps/ref_rejected": -67.0482177734375, "logps/rejected": -95.09843444824219, "loss": 0.9566, "margin_dpo/margin_mean": 11.25178337097168, "margin_dpo/margin_std": 14.857077598571777, "step": 339 }, { "KL/chosen_KL_mean": -19.570297241210938, "KL/mean": -25.74864959716797, "KL/rejected_KL_mean": -31.9270076751709, "KL/std": 17.383949279785156, "epoch": 0.5139833711262283, "fcm_dpo/beta": 0.07418019324541092, "fcm_dpo/delta": -0.1296280026435852, "fcm_dpo/margin": 12.356712341308594, "fcm_dpo/q_t": 0.34674328565597534, "grad_norm": 25.045578002929688, "learning_rate": 2.8295924627584004e-07, "logits/chosen": 0.381099671125412, "logits/rejected": 0.35601145029067993, "logps/chosen": -73.57594299316406, "logps/ref_chosen": -54.00564956665039, "logps/ref_rejected": -61.314430236816406, "logps/rejected": -93.24143981933594, "loss": 1.0518, "margin_dpo/margin_mean": 12.356712341308594, "margin_dpo/margin_std": 19.074668884277344, "step": 340 }, { "KL/chosen_KL_mean": -18.521739959716797, "KL/mean": -24.781999588012695, "KL/rejected_KL_mean": -31.042255401611328, "KL/std": 15.980314254760742, "epoch": 0.5154950869236583, "fcm_dpo/beta": 0.07009994983673096, "fcm_dpo/delta": -0.09934549033641815, "fcm_dpo/margin": 12.520519256591797, "fcm_dpo/q_t": 0.3412610590457916, "grad_norm": 26.827455520629883, "learning_rate": 2.816481133934373e-07, "logits/chosen": 0.4074775278568268, "logits/rejected": 0.3589476943016052, "logps/chosen": -81.91683959960938, "logps/ref_chosen": -63.39509582519531, "logps/ref_rejected": -76.20973205566406, "logps/rejected": -107.25199127197266, "loss": 1.0172, "margin_dpo/margin_mean": 12.520517349243164, "margin_dpo/margin_std": 17.281801223754883, "step": 341 }, { "KL/chosen_KL_mean": -18.645898818969727, "KL/mean": -25.731111526489258, "KL/rejected_KL_mean": -32.81632995605469, "KL/std": 17.429080963134766, "epoch": 0.5170068027210885, "fcm_dpo/beta": 0.06906507164239883, "fcm_dpo/delta": -0.19917050004005432, "fcm_dpo/margin": 14.170430183410645, "fcm_dpo/q_t": 0.3246821165084839, "grad_norm": 20.034122467041016, "learning_rate": 2.8033609524527046e-07, "logits/chosen": 0.3918822407722473, "logits/rejected": 0.3495151698589325, "logps/chosen": -71.69371032714844, "logps/ref_chosen": -53.047813415527344, "logps/ref_rejected": -68.2854232788086, "logps/rejected": -101.10175323486328, "loss": 0.8962, "margin_dpo/margin_mean": 14.170430183410645, "margin_dpo/margin_std": 17.488643646240234, "step": 342 }, { "KL/chosen_KL_mean": -17.64687728881836, "KL/mean": -21.812679290771484, "KL/rejected_KL_mean": -25.97848129272461, "KL/std": 16.159303665161133, "epoch": 0.5185185185185185, "fcm_dpo/beta": 0.06995899975299835, "fcm_dpo/delta": 0.2293698936700821, "fcm_dpo/margin": 8.33160400390625, "fcm_dpo/q_t": 0.38500741124153137, "grad_norm": 24.00664710998535, "learning_rate": 2.7902322853130753e-07, "logits/chosen": 0.2996237874031067, "logits/rejected": 0.2916584610939026, "logps/chosen": -88.22540283203125, "logps/ref_chosen": -70.57852935791016, "logps/ref_rejected": -84.73873901367188, "logps/rejected": -110.71722412109375, "loss": 1.1325, "margin_dpo/margin_mean": 8.33160400390625, "margin_dpo/margin_std": 14.704826354980469, "step": 343 }, { "KL/chosen_KL_mean": -21.23578453063965, "KL/mean": -27.21549415588379, "KL/rejected_KL_mean": -33.19519805908203, "KL/std": 16.221303939819336, "epoch": 0.5200302343159486, "fcm_dpo/beta": 0.07086661458015442, "fcm_dpo/delta": -0.052916139364242554, "fcm_dpo/margin": 11.959415435791016, "fcm_dpo/q_t": 0.3419157862663269, "grad_norm": 26.274948120117188, "learning_rate": 2.7770954997525274e-07, "logits/chosen": 0.39441317319869995, "logits/rejected": 0.3343184292316437, "logps/chosen": -77.04679107666016, "logps/ref_chosen": -55.811004638671875, "logps/ref_rejected": -84.77637481689453, "logps/rejected": -117.97157287597656, "loss": 0.9865, "margin_dpo/margin_mean": 11.959415435791016, "margin_dpo/margin_std": 16.728431701660156, "step": 344 }, { "KL/chosen_KL_mean": -15.331901550292969, "KL/mean": -21.359270095825195, "KL/rejected_KL_mean": -27.386638641357422, "KL/std": 15.091092109680176, "epoch": 0.5215419501133787, "fcm_dpo/beta": 0.0703570544719696, "fcm_dpo/delta": -0.05284047871828079, "fcm_dpo/margin": 12.054736137390137, "fcm_dpo/q_t": 0.34237420558929443, "grad_norm": 22.506139755249023, "learning_rate": 2.7639509632351927e-07, "logits/chosen": 0.4815177321434021, "logits/rejected": 0.4361530840396881, "logps/chosen": -73.11799621582031, "logps/ref_chosen": -57.78609848022461, "logps/ref_rejected": -78.91847229003906, "logps/rejected": -106.30511474609375, "loss": 0.9907, "margin_dpo/margin_mean": 12.054737091064453, "margin_dpo/margin_std": 17.231456756591797, "step": 345 }, { "KL/chosen_KL_mean": -17.275279998779297, "KL/mean": -24.348899841308594, "KL/rejected_KL_mean": -31.422523498535156, "KL/std": 17.06797218322754, "epoch": 0.5230536659108088, "fcm_dpo/beta": 0.0688161551952362, "fcm_dpo/delta": -0.191911518573761, "fcm_dpo/margin": 14.147241592407227, "fcm_dpo/q_t": 0.32509636878967285, "grad_norm": 24.509672164916992, "learning_rate": 2.7507990434420123e-07, "logits/chosen": 0.4331340193748474, "logits/rejected": 0.36041998863220215, "logps/chosen": -73.56040954589844, "logps/ref_chosen": -56.285125732421875, "logps/ref_rejected": -91.15303039550781, "logps/rejected": -122.5755615234375, "loss": 0.9438, "margin_dpo/margin_mean": 14.147241592407227, "margin_dpo/margin_std": 18.288009643554688, "step": 346 }, { "KL/chosen_KL_mean": -20.0422306060791, "KL/mean": -25.513660430908203, "KL/rejected_KL_mean": -30.98508071899414, "KL/std": 17.32723617553711, "epoch": 0.5245653817082389, "fcm_dpo/beta": 0.06831781566143036, "fcm_dpo/delta": 0.05656781792640686, "fcm_dpo/margin": 10.942852973937988, "fcm_dpo/q_t": 0.36600062251091003, "grad_norm": 24.8229923248291, "learning_rate": 2.737640108260456e-07, "logits/chosen": 0.5241929292678833, "logits/rejected": 0.47482144832611084, "logps/chosen": -73.5417709350586, "logps/ref_chosen": -53.499542236328125, "logps/ref_rejected": -72.52565002441406, "logps/rejected": -103.51073455810547, "loss": 1.047, "margin_dpo/margin_mean": 10.942852973937988, "margin_dpo/margin_std": 17.520071029663086, "step": 347 }, { "KL/chosen_KL_mean": -16.621702194213867, "KL/mean": -23.973649978637695, "KL/rejected_KL_mean": -31.325599670410156, "KL/std": 17.199548721313477, "epoch": 0.5260770975056689, "fcm_dpo/beta": 0.06601608544588089, "fcm_dpo/delta": -0.1918950080871582, "fcm_dpo/margin": 14.703897476196289, "fcm_dpo/q_t": 0.3412366807460785, "grad_norm": 21.577510833740234, "learning_rate": 2.724474525774229e-07, "logits/chosen": 0.5218849182128906, "logits/rejected": 0.4909231662750244, "logps/chosen": -67.40855407714844, "logps/ref_chosen": -50.78684997558594, "logps/ref_rejected": -68.63732147216797, "logps/rejected": -99.96292114257812, "loss": 0.9651, "margin_dpo/margin_mean": 14.703897476196289, "margin_dpo/margin_std": 20.968624114990234, "step": 348 }, { "KL/chosen_KL_mean": -17.335826873779297, "KL/mean": -24.095809936523438, "KL/rejected_KL_mean": -30.855792999267578, "KL/std": 17.796833038330078, "epoch": 0.527588813303099, "fcm_dpo/beta": 0.06530401110649109, "fcm_dpo/delta": -0.09097161144018173, "fcm_dpo/margin": 13.519969940185547, "fcm_dpo/q_t": 0.3345542550086975, "grad_norm": 23.182514190673828, "learning_rate": 2.711302664252973e-07, "logits/chosen": 0.43678218126296997, "logits/rejected": 0.3523603677749634, "logps/chosen": -70.66083526611328, "logps/ref_chosen": -53.325008392333984, "logps/ref_rejected": -83.21236419677734, "logps/rejected": -114.06816101074219, "loss": 0.9442, "margin_dpo/margin_mean": 13.519969940185547, "margin_dpo/margin_std": 17.63075065612793, "step": 349 }, { "KL/chosen_KL_mean": -19.102840423583984, "KL/mean": -27.495264053344727, "KL/rejected_KL_mean": -35.88768768310547, "KL/std": 18.53017807006836, "epoch": 0.5291005291005291, "fcm_dpo/beta": 0.06193181127309799, "fcm_dpo/delta": -0.2687161862850189, "fcm_dpo/margin": 16.784847259521484, "fcm_dpo/q_t": 0.3046181797981262, "grad_norm": 22.8939208984375, "learning_rate": 2.698124892141971e-07, "logits/chosen": 0.3871225416660309, "logits/rejected": 0.31201279163360596, "logps/chosen": -80.72860717773438, "logps/ref_chosen": -61.625770568847656, "logps/ref_rejected": -87.63627624511719, "logps/rejected": -123.52395629882812, "loss": 0.8411, "margin_dpo/margin_mean": 16.784847259521484, "margin_dpo/margin_std": 18.485410690307617, "step": 350 }, { "KL/chosen_KL_mean": -17.614654541015625, "KL/mean": -24.530780792236328, "KL/rejected_KL_mean": -31.4469051361084, "KL/std": 16.754365921020508, "epoch": 0.5306122448979592, "fcm_dpo/beta": 0.06079188734292984, "fcm_dpo/delta": -0.04478435963392258, "fcm_dpo/margin": 13.832258224487305, "fcm_dpo/q_t": 0.33148348331451416, "grad_norm": 19.812429428100586, "learning_rate": 2.6849415780518357e-07, "logits/chosen": 0.36037567257881165, "logits/rejected": 0.28710806369781494, "logps/chosen": -73.87098693847656, "logps/ref_chosen": -56.2563362121582, "logps/ref_rejected": -79.11589813232422, "logps/rejected": -110.56280517578125, "loss": 1.0035, "margin_dpo/margin_mean": 13.832258224487305, "margin_dpo/margin_std": 19.36257553100586, "step": 351 }, { "KL/chosen_KL_mean": -17.411983489990234, "KL/mean": -23.69668960571289, "KL/rejected_KL_mean": -29.98139190673828, "KL/std": 16.408573150634766, "epoch": 0.5321239606953893, "fcm_dpo/beta": 0.0607098788022995, "fcm_dpo/delta": 0.039311863481998444, "fcm_dpo/margin": 12.569415092468262, "fcm_dpo/q_t": 0.3572866916656494, "grad_norm": 21.984020233154297, "learning_rate": 2.6717530907482024e-07, "logits/chosen": 0.4072480797767639, "logits/rejected": 0.35473206639289856, "logps/chosen": -80.46393585205078, "logps/ref_chosen": -63.05195236206055, "logps/ref_rejected": -85.52035522460938, "logps/rejected": -115.50174713134766, "loss": 0.9818, "margin_dpo/margin_mean": 12.569416046142578, "margin_dpo/margin_std": 17.520357131958008, "step": 352 }, { "KL/chosen_KL_mean": -15.802751541137695, "KL/mean": -22.807369232177734, "KL/rejected_KL_mean": -29.811988830566406, "KL/std": 17.35757064819336, "epoch": 0.5336356764928194, "fcm_dpo/beta": 0.061134468764066696, "fcm_dpo/delta": -0.06244935095310211, "fcm_dpo/margin": 14.009233474731445, "fcm_dpo/q_t": 0.3378174304962158, "grad_norm": 21.547151565551758, "learning_rate": 2.658559799141411e-07, "logits/chosen": 0.42149317264556885, "logits/rejected": 0.42055612802505493, "logps/chosen": -84.81193542480469, "logps/ref_chosen": -69.00918579101562, "logps/ref_rejected": -72.65840148925781, "logps/rejected": -102.47039031982422, "loss": 0.9734, "margin_dpo/margin_mean": 14.009233474731445, "margin_dpo/margin_std": 19.090660095214844, "step": 353 }, { "KL/chosen_KL_mean": -16.668289184570312, "KL/mean": -24.297048568725586, "KL/rejected_KL_mean": -31.925806045532227, "KL/std": 16.85162353515625, "epoch": 0.5351473922902494, "fcm_dpo/beta": 0.05897783115506172, "fcm_dpo/delta": -0.11220181733369827, "fcm_dpo/margin": 15.25751781463623, "fcm_dpo/q_t": 0.3294384777545929, "grad_norm": 23.117361068725586, "learning_rate": 2.6453620722761895e-07, "logits/chosen": 0.47396335005760193, "logits/rejected": 0.3462182283401489, "logps/chosen": -56.456626892089844, "logps/ref_chosen": -39.78833770751953, "logps/ref_rejected": -69.56885528564453, "logps/rejected": -101.49465942382812, "loss": 0.9378, "margin_dpo/margin_mean": 15.25751781463623, "margin_dpo/margin_std": 19.276477813720703, "step": 354 }, { "KL/chosen_KL_mean": -19.132593154907227, "KL/mean": -26.75240707397461, "KL/rejected_KL_mean": -34.372222900390625, "KL/std": 19.179889678955078, "epoch": 0.5366591080876795, "fcm_dpo/beta": 0.0584687814116478, "fcm_dpo/delta": -0.09985160082578659, "fcm_dpo/margin": 15.239628791809082, "fcm_dpo/q_t": 0.33161279559135437, "grad_norm": 22.381999969482422, "learning_rate": 2.632160279321328e-07, "logits/chosen": 0.4477384388446808, "logits/rejected": 0.32458215951919556, "logps/chosen": -65.38796997070312, "logps/ref_chosen": -46.25537872314453, "logps/ref_rejected": -78.20236206054688, "logps/rejected": -112.5745849609375, "loss": 0.9671, "margin_dpo/margin_mean": 15.239627838134766, "margin_dpo/margin_std": 20.278135299682617, "step": 355 }, { "KL/chosen_KL_mean": -17.08836555480957, "KL/mean": -23.33837127685547, "KL/rejected_KL_mean": -29.588375091552734, "KL/std": 18.691139221191406, "epoch": 0.5381708238851096, "fcm_dpo/beta": 0.058297883719205856, "fcm_dpo/delta": 0.076647087931633, "fcm_dpo/margin": 12.500007629394531, "fcm_dpo/q_t": 0.3649879992008209, "grad_norm": 22.381298065185547, "learning_rate": 2.618954789559356e-07, "logits/chosen": 0.4272801876068115, "logits/rejected": 0.35394901037216187, "logps/chosen": -64.99452209472656, "logps/ref_chosen": -47.906158447265625, "logps/ref_rejected": -74.29397583007812, "logps/rejected": -103.8823471069336, "loss": 1.1716, "margin_dpo/margin_mean": 12.500007629394531, "margin_dpo/margin_std": 23.757095336914062, "step": 356 }, { "KL/chosen_KL_mean": -20.82411766052246, "KL/mean": -26.578861236572266, "KL/rejected_KL_mean": -32.33361053466797, "KL/std": 18.383647918701172, "epoch": 0.5396825396825397, "fcm_dpo/beta": 0.05899185687303543, "fcm_dpo/delta": 0.12616032361984253, "fcm_dpo/margin": 11.509489059448242, "fcm_dpo/q_t": 0.37107419967651367, "grad_norm": 23.38809585571289, "learning_rate": 2.6057459723762076e-07, "logits/chosen": 0.43112170696258545, "logits/rejected": 0.40187525749206543, "logps/chosen": -83.45912170410156, "logps/ref_chosen": -62.63500213623047, "logps/ref_rejected": -65.11399841308594, "logps/rejected": -97.4476089477539, "loss": 1.0963, "margin_dpo/margin_mean": 11.509489059448242, "margin_dpo/margin_std": 18.848796844482422, "step": 357 }, { "KL/chosen_KL_mean": -18.612560272216797, "KL/mean": -26.671905517578125, "KL/rejected_KL_mean": -34.73124694824219, "KL/std": 18.668180465698242, "epoch": 0.5411942554799698, "fcm_dpo/beta": 0.059323687106370926, "fcm_dpo/delta": -0.1726943552494049, "fcm_dpo/margin": 16.118690490722656, "fcm_dpo/q_t": 0.32745832204818726, "grad_norm": 23.334945678710938, "learning_rate": 2.5925341972508954e-07, "logits/chosen": 0.3674103021621704, "logits/rejected": 0.3703601658344269, "logps/chosen": -85.82217407226562, "logps/ref_chosen": -67.20960998535156, "logps/ref_rejected": -69.34715270996094, "logps/rejected": -104.07839965820312, "loss": 0.9313, "margin_dpo/margin_mean": 16.118690490722656, "margin_dpo/margin_std": 20.610336303710938, "step": 358 }, { "KL/chosen_KL_mean": -20.406143188476562, "KL/mean": -25.056005477905273, "KL/rejected_KL_mean": -29.705869674682617, "KL/std": 18.56830596923828, "epoch": 0.5427059712773998, "fcm_dpo/beta": 0.06133756786584854, "fcm_dpo/delta": 0.23471681773662567, "fcm_dpo/margin": 9.29973030090332, "fcm_dpo/q_t": 0.39298108220100403, "grad_norm": 25.49129867553711, "learning_rate": 2.579319833745169e-07, "logits/chosen": 0.3720915913581848, "logits/rejected": 0.3445711135864258, "logps/chosen": -82.93193054199219, "logps/ref_chosen": -62.52578353881836, "logps/ref_rejected": -76.63114929199219, "logps/rejected": -106.33702087402344, "loss": 1.1465, "margin_dpo/margin_mean": 9.29973030090332, "margin_dpo/margin_std": 17.169992446899414, "step": 359 }, { "KL/chosen_KL_mean": -20.363914489746094, "KL/mean": -26.988189697265625, "KL/rejected_KL_mean": -33.612464904785156, "KL/std": 18.771831512451172, "epoch": 0.54421768707483, "fcm_dpo/beta": 0.060898374766111374, "fcm_dpo/delta": -0.007558280602097511, "fcm_dpo/margin": 13.248543739318848, "fcm_dpo/q_t": 0.3549214005470276, "grad_norm": 23.553117752075195, "learning_rate": 2.5661032514931834e-07, "logits/chosen": 0.33502113819122314, "logits/rejected": 0.2443276047706604, "logps/chosen": -83.85163879394531, "logps/ref_chosen": -63.48772048950195, "logps/ref_rejected": -90.6891098022461, "logps/rejected": -124.30157470703125, "loss": 0.9896, "margin_dpo/margin_mean": 13.248543739318848, "margin_dpo/margin_std": 19.227291107177734, "step": 360 }, { "KL/chosen_KL_mean": -19.366771697998047, "KL/mean": -27.24359893798828, "KL/rejected_KL_mean": -35.12042999267578, "KL/std": 19.667043685913086, "epoch": 0.54572940287226, "fcm_dpo/beta": 0.059625256806612015, "fcm_dpo/delta": -0.15384510159492493, "fcm_dpo/margin": 15.753656387329102, "fcm_dpo/q_t": 0.32015830278396606, "grad_norm": 23.828569412231445, "learning_rate": 2.552884820191154e-07, "logits/chosen": 0.44018304347991943, "logits/rejected": 0.38934749364852905, "logps/chosen": -77.28392028808594, "logps/ref_chosen": -57.917144775390625, "logps/ref_rejected": -72.39089965820312, "logps/rejected": -107.5113296508789, "loss": 0.8998, "margin_dpo/margin_mean": 15.753657341003418, "margin_dpo/margin_std": 19.081100463867188, "step": 361 }, { "KL/chosen_KL_mean": -18.872962951660156, "KL/mean": -26.762571334838867, "KL/rejected_KL_mean": -34.65217590332031, "KL/std": 18.649995803833008, "epoch": 0.54724111866969, "fcm_dpo/beta": 0.058759547770023346, "fcm_dpo/delta": -0.14217320084571838, "fcm_dpo/margin": 15.779216766357422, "fcm_dpo/q_t": 0.32848185300827026, "grad_norm": 21.277912139892578, "learning_rate": 2.53966490958702e-07, "logits/chosen": 0.49138182401657104, "logits/rejected": 0.38976341485977173, "logps/chosen": -82.31643676757812, "logps/ref_chosen": -63.4434700012207, "logps/ref_rejected": -103.45516967773438, "logps/rejected": -138.1073455810547, "loss": 0.9292, "margin_dpo/margin_mean": 15.779217720031738, "margin_dpo/margin_std": 19.549943923950195, "step": 362 }, { "KL/chosen_KL_mean": -22.18973159790039, "KL/mean": -30.150291442871094, "KL/rejected_KL_mean": -38.11085510253906, "KL/std": 19.200645446777344, "epoch": 0.5487528344671202, "fcm_dpo/beta": 0.05654866620898247, "fcm_dpo/delta": -0.11033031344413757, "fcm_dpo/margin": 15.921126365661621, "fcm_dpo/q_t": 0.3285680413246155, "grad_norm": 20.080604553222656, "learning_rate": 2.526443889470099e-07, "logits/chosen": 0.45604056119918823, "logits/rejected": 0.3346443176269531, "logps/chosen": -70.84156036376953, "logps/ref_chosen": -48.65182876586914, "logps/ref_rejected": -88.65904235839844, "logps/rejected": -126.7698974609375, "loss": 0.9166, "margin_dpo/margin_mean": 15.921126365661621, "margin_dpo/margin_std": 19.751773834228516, "step": 363 }, { "KL/chosen_KL_mean": -18.949443817138672, "KL/mean": -27.227487564086914, "KL/rejected_KL_mean": -35.505531311035156, "KL/std": 20.297557830810547, "epoch": 0.5502645502645502, "fcm_dpo/beta": 0.055115051567554474, "fcm_dpo/delta": -0.12417322397232056, "fcm_dpo/margin": 16.55609130859375, "fcm_dpo/q_t": 0.33811190724372864, "grad_norm": 18.80527114868164, "learning_rate": 2.513222129660744e-07, "logits/chosen": 0.3324137330055237, "logits/rejected": 0.23981907963752747, "logps/chosen": -76.82051849365234, "logps/ref_chosen": -57.87107467651367, "logps/ref_rejected": -80.95503234863281, "logps/rejected": -116.46056365966797, "loss": 1.0054, "margin_dpo/margin_mean": 16.55609130859375, "margin_dpo/margin_std": 24.270000457763672, "step": 364 }, { "KL/chosen_KL_mean": -16.02554702758789, "KL/mean": -23.708436965942383, "KL/rejected_KL_mean": -31.391326904296875, "KL/std": 20.104900360107422, "epoch": 0.5517762660619804, "fcm_dpo/beta": 0.054510001093149185, "fcm_dpo/delta": -0.04106505215167999, "fcm_dpo/margin": 15.365779876708984, "fcm_dpo/q_t": 0.33312904834747314, "grad_norm": 18.545867919921875, "learning_rate": 2.5e-07, "logits/chosen": 0.44149670004844666, "logits/rejected": 0.4310864508152008, "logps/chosen": -80.96772766113281, "logps/ref_chosen": -64.94217681884766, "logps/ref_rejected": -74.8599853515625, "logps/rejected": -106.25131225585938, "loss": 0.9047, "margin_dpo/margin_mean": 15.365779876708984, "margin_dpo/margin_std": 17.665653228759766, "step": 365 }, { "KL/chosen_KL_mean": -14.832332611083984, "KL/mean": -21.632671356201172, "KL/rejected_KL_mean": -28.43301010131836, "KL/std": 17.586444854736328, "epoch": 0.5532879818594104, "fcm_dpo/beta": 0.05539874732494354, "fcm_dpo/delta": 0.047521963715553284, "fcm_dpo/margin": 13.600683212280273, "fcm_dpo/q_t": 0.3618229031562805, "grad_norm": 21.13431739807129, "learning_rate": 2.486777870339255e-07, "logits/chosen": 0.37482506036758423, "logits/rejected": 0.35465550422668457, "logps/chosen": -69.9983139038086, "logps/ref_chosen": -55.16598129272461, "logps/ref_rejected": -65.26121520996094, "logps/rejected": -93.69422149658203, "loss": 1.0611, "margin_dpo/margin_mean": 13.600683212280273, "margin_dpo/margin_std": 21.480377197265625, "step": 366 }, { "KL/chosen_KL_mean": -19.378307342529297, "KL/mean": -25.97715187072754, "KL/rejected_KL_mean": -32.575992584228516, "KL/std": 19.300640106201172, "epoch": 0.5547996976568406, "fcm_dpo/beta": 0.055359289050102234, "fcm_dpo/delta": 0.07484433799982071, "fcm_dpo/margin": 13.197684288024902, "fcm_dpo/q_t": 0.35306018590927124, "grad_norm": 21.098054885864258, "learning_rate": 2.4735561105299014e-07, "logits/chosen": 0.3957292139530182, "logits/rejected": 0.29518669843673706, "logps/chosen": -75.3887710571289, "logps/ref_chosen": -56.01046371459961, "logps/ref_rejected": -77.31010437011719, "logps/rejected": -109.88610076904297, "loss": 1.0177, "margin_dpo/margin_mean": 13.197683334350586, "margin_dpo/margin_std": 19.19463539123535, "step": 367 }, { "KL/chosen_KL_mean": -21.07410430908203, "KL/mean": -27.410612106323242, "KL/rejected_KL_mean": -33.74711990356445, "KL/std": 18.693862915039062, "epoch": 0.5563114134542706, "fcm_dpo/beta": 0.056563593447208405, "fcm_dpo/delta": 0.0895220935344696, "fcm_dpo/margin": 12.673014640808105, "fcm_dpo/q_t": 0.36096638441085815, "grad_norm": 22.17424964904785, "learning_rate": 2.46033509041298e-07, "logits/chosen": 0.2558997869491577, "logits/rejected": 0.24736079573631287, "logps/chosen": -95.90338134765625, "logps/ref_chosen": -74.82927703857422, "logps/ref_rejected": -76.11680603027344, "logps/rejected": -109.86392211914062, "loss": 1.0518, "margin_dpo/margin_mean": 12.673015594482422, "margin_dpo/margin_std": 19.6419620513916, "step": 368 }, { "KL/chosen_KL_mean": -20.30316162109375, "KL/mean": -26.592472076416016, "KL/rejected_KL_mean": -32.88178253173828, "KL/std": 19.905651092529297, "epoch": 0.5578231292517006, "fcm_dpo/beta": 0.057878073304891586, "fcm_dpo/delta": 0.07577980309724808, "fcm_dpo/margin": 12.578624725341797, "fcm_dpo/q_t": 0.36213284730911255, "grad_norm": 21.1649112701416, "learning_rate": 2.447115179808846e-07, "logits/chosen": 0.4168482720851898, "logits/rejected": 0.3676788806915283, "logps/chosen": -78.62937927246094, "logps/ref_chosen": -58.32621765136719, "logps/ref_rejected": -80.92183685302734, "logps/rejected": -113.80361938476562, "loss": 1.052, "margin_dpo/margin_mean": 12.578624725341797, "margin_dpo/margin_std": 19.676862716674805, "step": 369 }, { "KL/chosen_KL_mean": -19.603811264038086, "KL/mean": -27.947715759277344, "KL/rejected_KL_mean": -36.29161834716797, "KL/std": 21.021804809570312, "epoch": 0.5593348450491308, "fcm_dpo/beta": 0.05625259131193161, "fcm_dpo/delta": -0.15349817276000977, "fcm_dpo/margin": 16.687808990478516, "fcm_dpo/q_t": 0.3311444818973541, "grad_norm": 21.677335739135742, "learning_rate": 2.4338967485068164e-07, "logits/chosen": 0.49989569187164307, "logits/rejected": 0.4341312050819397, "logps/chosen": -72.48753356933594, "logps/ref_chosen": -52.88372039794922, "logps/ref_rejected": -79.43692016601562, "logps/rejected": -115.7285385131836, "loss": 1.0079, "margin_dpo/margin_mean": 16.687808990478516, "margin_dpo/margin_std": 24.34197235107422, "step": 370 }, { "KL/chosen_KL_mean": -19.531940460205078, "KL/mean": -27.286758422851562, "KL/rejected_KL_mean": -35.04158020019531, "KL/std": 20.940555572509766, "epoch": 0.5608465608465608, "fcm_dpo/beta": 0.055904123932123184, "fcm_dpo/delta": -0.07506079971790314, "fcm_dpo/margin": 15.509639739990234, "fcm_dpo/q_t": 0.34340009093284607, "grad_norm": 21.58955192565918, "learning_rate": 2.420680166254831e-07, "logits/chosen": 0.5270491242408752, "logits/rejected": 0.49259671568870544, "logps/chosen": -68.75614929199219, "logps/ref_chosen": -49.224212646484375, "logps/ref_rejected": -63.348472595214844, "logps/rejected": -98.39005279541016, "loss": 0.986, "margin_dpo/margin_mean": 15.509639739990234, "margin_dpo/margin_std": 21.508487701416016, "step": 371 }, { "KL/chosen_KL_mean": -21.663251876831055, "KL/mean": -26.70006561279297, "KL/rejected_KL_mean": -31.736879348754883, "KL/std": 19.95144271850586, "epoch": 0.562358276643991, "fcm_dpo/beta": 0.057659346610307693, "fcm_dpo/delta": 0.22692981362342834, "fcm_dpo/margin": 10.073625564575195, "fcm_dpo/q_t": 0.3950161337852478, "grad_norm": 28.074542999267578, "learning_rate": 2.4074658027491044e-07, "logits/chosen": 0.4539080858230591, "logits/rejected": 0.3669462203979492, "logps/chosen": -73.93280029296875, "logps/ref_chosen": -52.269554138183594, "logps/ref_rejected": -72.99522399902344, "logps/rejected": -104.73210144042969, "loss": 1.3033, "margin_dpo/margin_mean": 10.073625564575195, "margin_dpo/margin_std": 24.8012752532959, "step": 372 }, { "KL/chosen_KL_mean": -24.116884231567383, "KL/mean": -30.572702407836914, "KL/rejected_KL_mean": -37.02852249145508, "KL/std": 20.55548858642578, "epoch": 0.563869992441421, "fcm_dpo/beta": 0.058202799409627914, "fcm_dpo/delta": 0.052256401628255844, "fcm_dpo/margin": 12.911628723144531, "fcm_dpo/q_t": 0.37231653928756714, "grad_norm": 31.09714126586914, "learning_rate": 2.394254027623792e-07, "logits/chosen": 0.4642970860004425, "logits/rejected": 0.3941759765148163, "logps/chosen": -85.22988891601562, "logps/ref_chosen": -61.112998962402344, "logps/ref_rejected": -76.24851989746094, "logps/rejected": -113.27703857421875, "loss": 1.1794, "margin_dpo/margin_mean": 12.911629676818848, "margin_dpo/margin_std": 25.102832794189453, "step": 373 }, { "KL/chosen_KL_mean": -20.470600128173828, "KL/mean": -30.454559326171875, "KL/rejected_KL_mean": -40.43851852416992, "KL/std": 20.557178497314453, "epoch": 0.5653817082388511, "fcm_dpo/beta": 0.05505363643169403, "fcm_dpo/delta": -0.3393453061580658, "fcm_dpo/margin": 19.967914581298828, "fcm_dpo/q_t": 0.2875816822052002, "grad_norm": 24.58673858642578, "learning_rate": 2.381045210440644e-07, "logits/chosen": 0.33565571904182434, "logits/rejected": 0.32568567991256714, "logps/chosen": -93.13980102539062, "logps/ref_chosen": -72.66920471191406, "logps/ref_rejected": -76.83158874511719, "logps/rejected": -117.27011108398438, "loss": 0.787, "margin_dpo/margin_mean": 19.967914581298828, "margin_dpo/margin_std": 19.655311584472656, "step": 374 }, { "KL/chosen_KL_mean": -19.648855209350586, "KL/mean": -25.847253799438477, "KL/rejected_KL_mean": -32.04564666748047, "KL/std": 20.406822204589844, "epoch": 0.5668934240362812, "fcm_dpo/beta": 0.055172353982925415, "fcm_dpo/delta": 0.12467709928750992, "fcm_dpo/margin": 12.396795272827148, "fcm_dpo/q_t": 0.3717153072357178, "grad_norm": 24.862714767456055, "learning_rate": 2.3678397206786715e-07, "logits/chosen": 0.44463562965393066, "logits/rejected": 0.3905963599681854, "logps/chosen": -77.33216094970703, "logps/ref_chosen": -57.68330383300781, "logps/ref_rejected": -79.34097290039062, "logps/rejected": -111.3866195678711, "loss": 1.0856, "margin_dpo/margin_mean": 12.396795272827148, "margin_dpo/margin_std": 20.809398651123047, "step": 375 }, { "KL/chosen_KL_mean": -21.62276268005371, "KL/mean": -31.52696418762207, "KL/rejected_KL_mean": -41.43116760253906, "KL/std": 21.306678771972656, "epoch": 0.5684051398337112, "fcm_dpo/beta": 0.053182363510131836, "fcm_dpo/delta": -0.285739004611969, "fcm_dpo/margin": 19.808406829833984, "fcm_dpo/q_t": 0.3129886984825134, "grad_norm": 21.40363883972168, "learning_rate": 2.3546379277238103e-07, "logits/chosen": 0.47032493352890015, "logits/rejected": 0.40646880865097046, "logps/chosen": -73.29682922363281, "logps/ref_chosen": -51.674072265625, "logps/ref_rejected": -75.69713592529297, "logps/rejected": -117.12830352783203, "loss": 0.9135, "margin_dpo/margin_mean": 19.808406829833984, "margin_dpo/margin_std": 24.855318069458008, "step": 376 }, { "KL/chosen_KL_mean": -22.585697174072266, "KL/mean": -29.564029693603516, "KL/rejected_KL_mean": -36.54236602783203, "KL/std": 20.325942993164062, "epoch": 0.5699168556311414, "fcm_dpo/beta": 0.053586531430482864, "fcm_dpo/delta": 0.054271575063467026, "fcm_dpo/margin": 13.956663131713867, "fcm_dpo/q_t": 0.35530292987823486, "grad_norm": 20.946025848388672, "learning_rate": 2.3414402008585886e-07, "logits/chosen": 0.41215571761131287, "logits/rejected": 0.3891144394874573, "logps/chosen": -68.76422882080078, "logps/ref_chosen": -46.17853546142578, "logps/ref_rejected": -57.756500244140625, "logps/rejected": -94.29886627197266, "loss": 1.0308, "margin_dpo/margin_mean": 13.956663131713867, "margin_dpo/margin_std": 20.671688079833984, "step": 377 }, { "KL/chosen_KL_mean": -21.790258407592773, "KL/mean": -27.98929214477539, "KL/rejected_KL_mean": -34.188323974609375, "KL/std": 21.171764373779297, "epoch": 0.5714285714285714, "fcm_dpo/beta": 0.054639630019664764, "fcm_dpo/delta": 0.12991301715373993, "fcm_dpo/margin": 12.398064613342285, "fcm_dpo/q_t": 0.3740558624267578, "grad_norm": 23.512828826904297, "learning_rate": 2.3282469092517977e-07, "logits/chosen": 0.46384197473526, "logits/rejected": 0.41567403078079224, "logps/chosen": -81.00914001464844, "logps/ref_chosen": -59.21887969970703, "logps/ref_rejected": -71.24818420410156, "logps/rejected": -105.43650817871094, "loss": 1.1043, "margin_dpo/margin_mean": 12.398064613342285, "margin_dpo/margin_std": 21.77179718017578, "step": 378 }, { "KL/chosen_KL_mean": -19.731494903564453, "KL/mean": -28.210784912109375, "KL/rejected_KL_mean": -36.690086364746094, "KL/std": 20.615100860595703, "epoch": 0.5729402872260015, "fcm_dpo/beta": 0.05373363196849823, "fcm_dpo/delta": -0.12232452630996704, "fcm_dpo/margin": 16.95858383178711, "fcm_dpo/q_t": 0.332636296749115, "grad_norm": 23.33953285217285, "learning_rate": 2.3150584219481643e-07, "logits/chosen": 0.46484893560409546, "logits/rejected": 0.39884310960769653, "logps/chosen": -96.04808044433594, "logps/ref_chosen": -76.31658935546875, "logps/ref_rejected": -104.26200103759766, "logps/rejected": -140.95208740234375, "loss": 0.9318, "margin_dpo/margin_mean": 16.95858383178711, "margin_dpo/margin_std": 21.930681228637695, "step": 379 }, { "KL/chosen_KL_mean": -18.223163604736328, "KL/mean": -27.74345588684082, "KL/rejected_KL_mean": -37.26375198364258, "KL/std": 19.516929626464844, "epoch": 0.5744520030234316, "fcm_dpo/beta": 0.05176670104265213, "fcm_dpo/delta": -0.20594152808189392, "fcm_dpo/margin": 19.04058837890625, "fcm_dpo/q_t": 0.3121461868286133, "grad_norm": 20.619007110595703, "learning_rate": 2.3018751078580283e-07, "logits/chosen": 0.454281210899353, "logits/rejected": 0.4146312475204468, "logps/chosen": -79.50633239746094, "logps/ref_chosen": -61.283164978027344, "logps/ref_rejected": -72.38892364501953, "logps/rejected": -109.65267944335938, "loss": 0.9093, "margin_dpo/margin_mean": 19.04058837890625, "margin_dpo/margin_std": 23.003185272216797, "step": 380 }, { "KL/chosen_KL_mean": -23.842905044555664, "KL/mean": -27.962167739868164, "KL/rejected_KL_mean": -32.081424713134766, "KL/std": 20.596736907958984, "epoch": 0.5759637188208617, "fcm_dpo/beta": 0.051557689905166626, "fcm_dpo/delta": 0.06958503276109695, "fcm_dpo/margin": 8.238516807556152, "fcm_dpo/q_t": 0.4186873435974121, "grad_norm": 24.15900421142578, "learning_rate": 2.288697335747027e-07, "logits/chosen": 0.41163092851638794, "logits/rejected": 0.385709285736084, "logps/chosen": -82.05690002441406, "logps/ref_chosen": -58.2139892578125, "logps/ref_rejected": -60.78669357299805, "logps/rejected": -92.86811828613281, "loss": 1.2613, "margin_dpo/margin_mean": 8.238516807556152, "margin_dpo/margin_std": 20.93825912475586, "step": 381 }, { "KL/chosen_KL_mean": -24.551029205322266, "KL/mean": -31.066709518432617, "KL/rejected_KL_mean": -37.58238983154297, "KL/std": 21.11056137084961, "epoch": 0.5774754346182918, "fcm_dpo/beta": 0.05340856313705444, "fcm_dpo/delta": 0.1072634756565094, "fcm_dpo/margin": 13.031362533569336, "fcm_dpo/q_t": 0.3651576638221741, "grad_norm": 22.464122772216797, "learning_rate": 2.2755254742257706e-07, "logits/chosen": 0.4638483226299286, "logits/rejected": 0.4158809781074524, "logps/chosen": -86.37635803222656, "logps/ref_chosen": -61.82532501220703, "logps/ref_rejected": -83.0452880859375, "logps/rejected": -120.62767028808594, "loss": 1.0163, "margin_dpo/margin_mean": 13.031362533569336, "margin_dpo/margin_std": 18.759021759033203, "step": 382 }, { "KL/chosen_KL_mean": -22.348957061767578, "KL/mean": -30.717674255371094, "KL/rejected_KL_mean": -39.086395263671875, "KL/std": 21.498252868652344, "epoch": 0.5789871504157218, "fcm_dpo/beta": 0.05201186239719391, "fcm_dpo/delta": -0.07921411097049713, "fcm_dpo/margin": 16.737438201904297, "fcm_dpo/q_t": 0.34389805793762207, "grad_norm": 23.397573471069336, "learning_rate": 2.2623598917395436e-07, "logits/chosen": 0.33132052421569824, "logits/rejected": 0.3495492935180664, "logps/chosen": -102.91221618652344, "logps/ref_chosen": -80.56326293945312, "logps/ref_rejected": -74.62922668457031, "logps/rejected": -113.71562194824219, "loss": 1.0053, "margin_dpo/margin_mean": 16.737438201904297, "margin_dpo/margin_std": 24.382492065429688, "step": 383 }, { "KL/chosen_KL_mean": -23.70047950744629, "KL/mean": -31.56975555419922, "KL/rejected_KL_mean": -39.43903350830078, "KL/std": 21.286376953125, "epoch": 0.5804988662131519, "fcm_dpo/beta": 0.05222079157829285, "fcm_dpo/delta": -0.02404698356986046, "fcm_dpo/margin": 15.738565444946289, "fcm_dpo/q_t": 0.34051215648651123, "grad_norm": 21.495080947875977, "learning_rate": 2.2492009565579875e-07, "logits/chosen": 0.44171950221061707, "logits/rejected": 0.39739423990249634, "logps/chosen": -89.17562866210938, "logps/ref_chosen": -65.47514343261719, "logps/ref_rejected": -79.67378234863281, "logps/rejected": -119.11282348632812, "loss": 0.9563, "margin_dpo/margin_mean": 15.738564491271973, "margin_dpo/margin_std": 20.684423446655273, "step": 384 }, { "KL/chosen_KL_mean": -22.995277404785156, "KL/mean": -31.722030639648438, "KL/rejected_KL_mean": -40.44878005981445, "KL/std": 20.493146896362305, "epoch": 0.582010582010582, "fcm_dpo/beta": 0.05136442184448242, "fcm_dpo/delta": -0.10590949654579163, "fcm_dpo/margin": 17.45350456237793, "fcm_dpo/q_t": 0.3273235857486725, "grad_norm": 21.761558532714844, "learning_rate": 2.2360490367648084e-07, "logits/chosen": 0.382984459400177, "logits/rejected": 0.3436170220375061, "logps/chosen": -89.05178833007812, "logps/ref_chosen": -66.0565185546875, "logps/ref_rejected": -86.68023681640625, "logps/rejected": -127.12901306152344, "loss": 0.9035, "margin_dpo/margin_mean": 17.453502655029297, "margin_dpo/margin_std": 20.69217300415039, "step": 385 }, { "KL/chosen_KL_mean": -26.571060180664062, "KL/mean": -32.992767333984375, "KL/rejected_KL_mean": -39.41448211669922, "KL/std": 21.929290771484375, "epoch": 0.5835222978080121, "fcm_dpo/beta": 0.05175807327032089, "fcm_dpo/delta": 0.14512205123901367, "fcm_dpo/margin": 12.843421936035156, "fcm_dpo/q_t": 0.36995601654052734, "grad_norm": 23.099040985107422, "learning_rate": 2.2229045002474724e-07, "logits/chosen": 0.38640153408050537, "logits/rejected": 0.32749783992767334, "logps/chosen": -102.19471740722656, "logps/ref_chosen": -75.6236572265625, "logps/ref_rejected": -92.62330627441406, "logps/rejected": -132.0377960205078, "loss": 1.0665, "margin_dpo/margin_mean": 12.843421936035156, "margin_dpo/margin_std": 20.646764755249023, "step": 386 }, { "KL/chosen_KL_mean": -23.750308990478516, "KL/mean": -32.67223358154297, "KL/rejected_KL_mean": -41.59415817260742, "KL/std": 20.869110107421875, "epoch": 0.5850340136054422, "fcm_dpo/beta": 0.0514523983001709, "fcm_dpo/delta": -0.12989971041679382, "fcm_dpo/margin": 17.843847274780273, "fcm_dpo/q_t": 0.3237932324409485, "grad_norm": 20.114234924316406, "learning_rate": 2.209767714686924e-07, "logits/chosen": 0.4548864960670471, "logits/rejected": 0.3586340546607971, "logps/chosen": -70.97201538085938, "logps/ref_chosen": -47.22170639038086, "logps/ref_rejected": -87.338134765625, "logps/rejected": -128.9322967529297, "loss": 0.8648, "margin_dpo/margin_mean": 17.84384536743164, "margin_dpo/margin_std": 20.037647247314453, "step": 387 }, { "KL/chosen_KL_mean": -23.332077026367188, "KL/mean": -29.797340393066406, "KL/rejected_KL_mean": -36.26259994506836, "KL/std": 20.760181427001953, "epoch": 0.5865457294028723, "fcm_dpo/beta": 0.051743343472480774, "fcm_dpo/delta": 0.1403004229068756, "fcm_dpo/margin": 12.930517196655273, "fcm_dpo/q_t": 0.3804495930671692, "grad_norm": 22.458322525024414, "learning_rate": 2.1966390475472954e-07, "logits/chosen": 0.46087580919265747, "logits/rejected": 0.4473402202129364, "logps/chosen": -97.91154479980469, "logps/ref_chosen": -74.5794677734375, "logps/ref_rejected": -79.92558288574219, "logps/rejected": -116.18818664550781, "loss": 1.1276, "margin_dpo/margin_mean": 12.93051815032959, "margin_dpo/margin_std": 23.506805419921875, "step": 388 }, { "KL/chosen_KL_mean": -25.07407569885254, "KL/mean": -33.77997970581055, "KL/rejected_KL_mean": -42.48588943481445, "KL/std": 21.998149871826172, "epoch": 0.5880574452003023, "fcm_dpo/beta": 0.051715172827243805, "fcm_dpo/delta": -0.11027075350284576, "fcm_dpo/margin": 17.41181182861328, "fcm_dpo/q_t": 0.33060115575790405, "grad_norm": 39.576087951660156, "learning_rate": 2.1835188660656265e-07, "logits/chosen": 0.435343861579895, "logits/rejected": 0.398581862449646, "logps/chosen": -86.69844055175781, "logps/ref_chosen": -61.624366760253906, "logps/ref_rejected": -76.50978088378906, "logps/rejected": -118.99566650390625, "loss": 0.9666, "margin_dpo/margin_mean": 17.41181182861328, "margin_dpo/margin_std": 23.3957576751709, "step": 389 }, { "KL/chosen_KL_mean": -22.420507431030273, "KL/mean": -30.25091552734375, "KL/rejected_KL_mean": -38.081329345703125, "KL/std": 21.462738037109375, "epoch": 0.5895691609977324, "fcm_dpo/beta": 0.051210470497608185, "fcm_dpo/delta": -0.002262681722640991, "fcm_dpo/margin": 15.660820960998535, "fcm_dpo/q_t": 0.34443798661231995, "grad_norm": 20.065223693847656, "learning_rate": 2.170407537241599e-07, "logits/chosen": 0.4916721284389496, "logits/rejected": 0.42790883779525757, "logps/chosen": -68.29237365722656, "logps/ref_chosen": -45.871864318847656, "logps/ref_rejected": -61.305999755859375, "logps/rejected": -99.3873291015625, "loss": 0.9572, "margin_dpo/margin_mean": 15.660821914672852, "margin_dpo/margin_std": 20.54749298095703, "step": 390 }, { "KL/chosen_KL_mean": -23.46485710144043, "KL/mean": -32.447296142578125, "KL/rejected_KL_mean": -41.42973709106445, "KL/std": 22.090721130371094, "epoch": 0.5910808767951625, "fcm_dpo/beta": 0.0502469539642334, "fcm_dpo/delta": -0.11369304358959198, "fcm_dpo/margin": 17.964881896972656, "fcm_dpo/q_t": 0.3362467288970947, "grad_norm": 21.207117080688477, "learning_rate": 2.1573054278272636e-07, "logits/chosen": 0.4358006715774536, "logits/rejected": 0.3706232011318207, "logps/chosen": -81.65187072753906, "logps/ref_chosen": -58.18701171875, "logps/ref_rejected": -83.63442993164062, "logps/rejected": -125.06417083740234, "loss": 1.0037, "margin_dpo/margin_mean": 17.964881896972656, "margin_dpo/margin_std": 26.00237274169922, "step": 391 }, { "KL/chosen_KL_mean": -19.514892578125, "KL/mean": -29.6951904296875, "KL/rejected_KL_mean": -39.87548065185547, "KL/std": 22.76531219482422, "epoch": 0.5925925925925926, "fcm_dpo/beta": 0.04908373951911926, "fcm_dpo/delta": -0.22158034145832062, "fcm_dpo/margin": 20.360595703125, "fcm_dpo/q_t": 0.3249374032020569, "grad_norm": 21.660350799560547, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 0.5273096561431885, "logits/rejected": 0.4682093858718872, "logps/chosen": -89.2594223022461, "logps/ref_chosen": -69.7445297241211, "logps/ref_rejected": -94.05877685546875, "logps/rejected": -133.93426513671875, "loss": 0.9433, "margin_dpo/margin_mean": 20.360595703125, "margin_dpo/margin_std": 27.020687103271484, "step": 392 }, { "KL/chosen_KL_mean": -25.08833122253418, "KL/mean": -34.988651275634766, "KL/rejected_KL_mean": -44.888973236083984, "KL/std": 22.98666763305664, "epoch": 0.5941043083900227, "fcm_dpo/beta": 0.046630434691905975, "fcm_dpo/delta": -0.13836631178855896, "fcm_dpo/margin": 19.800640106201172, "fcm_dpo/q_t": 0.3229396343231201, "grad_norm": 18.535551071166992, "learning_rate": 2.131130332936195e-07, "logits/chosen": 0.48956263065338135, "logits/rejected": 0.44990265369415283, "logps/chosen": -77.42323303222656, "logps/ref_chosen": -52.33489990234375, "logps/ref_rejected": -74.33809661865234, "logps/rejected": -119.22706604003906, "loss": 0.8709, "margin_dpo/margin_mean": 19.800640106201172, "margin_dpo/margin_std": 22.033645629882812, "step": 393 }, { "KL/chosen_KL_mean": -23.047691345214844, "KL/mean": -30.93512725830078, "KL/rejected_KL_mean": -38.82255935668945, "KL/std": 19.23456573486328, "epoch": 0.5956160241874527, "fcm_dpo/beta": 0.04730773717164993, "fcm_dpo/delta": 0.0572335422039032, "fcm_dpo/margin": 15.77486801147461, "fcm_dpo/q_t": 0.34974604845046997, "grad_norm": 24.01777458190918, "learning_rate": 2.1180580796331323e-07, "logits/chosen": 0.5058435201644897, "logits/rejected": 0.4730924963951111, "logps/chosen": -83.72383117675781, "logps/ref_chosen": -60.6761360168457, "logps/ref_rejected": -71.36074829101562, "logps/rejected": -110.18331146240234, "loss": 0.956, "margin_dpo/margin_mean": 15.77486801147461, "margin_dpo/margin_std": 19.874174118041992, "step": 394 }, { "KL/chosen_KL_mean": -24.339027404785156, "KL/mean": -31.597896575927734, "KL/rejected_KL_mean": -38.85675811767578, "KL/std": 21.322864532470703, "epoch": 0.5971277399848829, "fcm_dpo/beta": 0.04836595058441162, "fcm_dpo/delta": 0.10261309146881104, "fcm_dpo/margin": 14.517733573913574, "fcm_dpo/q_t": 0.3664752244949341, "grad_norm": 23.358461380004883, "learning_rate": 2.104996510066625e-07, "logits/chosen": 0.46007978916168213, "logits/rejected": 0.3733363747596741, "logps/chosen": -74.94335174560547, "logps/ref_chosen": -50.60432434082031, "logps/ref_rejected": -77.08731079101562, "logps/rejected": -115.94407653808594, "loss": 1.0173, "margin_dpo/margin_mean": 14.517733573913574, "margin_dpo/margin_std": 21.095766067504883, "step": 395 }, { "KL/chosen_KL_mean": -23.455059051513672, "KL/mean": -32.36639404296875, "KL/rejected_KL_mean": -41.2777214050293, "KL/std": 24.375507354736328, "epoch": 0.5986394557823129, "fcm_dpo/beta": 0.046926774084568024, "fcm_dpo/delta": -0.048550158739089966, "fcm_dpo/margin": 17.822664260864258, "fcm_dpo/q_t": 0.3385160267353058, "grad_norm": 20.81109619140625, "learning_rate": 2.0919459895968517e-07, "logits/chosen": 0.4549301564693451, "logits/rejected": 0.36139392852783203, "logps/chosen": -74.81466674804688, "logps/ref_chosen": -51.35961151123047, "logps/ref_rejected": -79.89360046386719, "logps/rejected": -121.17131805419922, "loss": 0.9418, "margin_dpo/margin_mean": 17.822662353515625, "margin_dpo/margin_std": 21.45787811279297, "step": 396 }, { "KL/chosen_KL_mean": -28.059925079345703, "KL/mean": -32.58983612060547, "KL/rejected_KL_mean": -37.11973190307617, "KL/std": 23.16779899597168, "epoch": 0.600151171579743, "fcm_dpo/beta": 0.050632819533348083, "fcm_dpo/delta": 0.3545893132686615, "fcm_dpo/margin": 9.059812545776367, "fcm_dpo/q_t": 0.4069092869758606, "grad_norm": 33.780357360839844, "learning_rate": 2.078906883274924e-07, "logits/chosen": 0.3834320902824402, "logits/rejected": 0.33937060832977295, "logps/chosen": -94.51615142822266, "logps/ref_chosen": -66.45622253417969, "logps/ref_rejected": -85.74736785888672, "logps/rejected": -122.86709594726562, "loss": 1.3316, "margin_dpo/margin_mean": 9.059812545776367, "margin_dpo/margin_std": 25.34778594970703, "step": 397 }, { "KL/chosen_KL_mean": -23.686159133911133, "KL/mean": -33.28020095825195, "KL/rejected_KL_mean": -42.874244689941406, "KL/std": 23.769163131713867, "epoch": 0.6016628873771731, "fcm_dpo/beta": 0.0496966615319252, "fcm_dpo/delta": -0.1711999624967575, "fcm_dpo/margin": 19.18807601928711, "fcm_dpo/q_t": 0.3260038495063782, "grad_norm": 21.69061851501465, "learning_rate": 2.065879555832674e-07, "logits/chosen": 0.4456123113632202, "logits/rejected": 0.3802449703216553, "logps/chosen": -72.93040466308594, "logps/ref_chosen": -49.244239807128906, "logps/ref_rejected": -75.18949127197266, "logps/rejected": -118.06373596191406, "loss": 0.8871, "margin_dpo/margin_mean": 19.18807601928711, "margin_dpo/margin_std": 23.326528549194336, "step": 398 }, { "KL/chosen_KL_mean": -26.581912994384766, "KL/mean": -37.71871566772461, "KL/rejected_KL_mean": -48.85551452636719, "KL/std": 23.6285457611084, "epoch": 0.6031746031746031, "fcm_dpo/beta": 0.047361068427562714, "fcm_dpo/delta": -0.2857217788696289, "fcm_dpo/margin": 22.273609161376953, "fcm_dpo/q_t": 0.30849629640579224, "grad_norm": 20.774028778076172, "learning_rate": 2.052864371672457e-07, "logits/chosen": 0.39033693075180054, "logits/rejected": 0.2585032880306244, "logps/chosen": -94.88870239257812, "logps/ref_chosen": -68.30679321289062, "logps/ref_rejected": -113.2708511352539, "logps/rejected": -162.12637329101562, "loss": 0.8394, "margin_dpo/margin_mean": 22.273609161376953, "margin_dpo/margin_std": 25.607967376708984, "step": 399 }, { "KL/chosen_KL_mean": -31.611953735351562, "KL/mean": -38.24412536621094, "KL/rejected_KL_mean": -44.876312255859375, "KL/std": 23.857276916503906, "epoch": 0.6046863189720333, "fcm_dpo/beta": 0.04748568683862686, "fcm_dpo/delta": 0.1808999478816986, "fcm_dpo/margin": 13.264352798461914, "fcm_dpo/q_t": 0.38564884662628174, "grad_norm": 29.400917053222656, "learning_rate": 2.0398616948569493e-07, "logits/chosen": 0.4609530568122864, "logits/rejected": 0.4057099223136902, "logps/chosen": -103.23844909667969, "logps/ref_chosen": -71.62649536132812, "logps/ref_rejected": -90.98765563964844, "logps/rejected": -135.8639678955078, "loss": 1.1172, "margin_dpo/margin_mean": 13.264352798461914, "margin_dpo/margin_std": 23.702287673950195, "step": 400 }, { "KL/chosen_KL_mean": -22.307126998901367, "KL/mean": -33.3577766418457, "KL/rejected_KL_mean": -44.408424377441406, "KL/std": 25.831157684326172, "epoch": 0.6061980347694633, "fcm_dpo/beta": 0.04660568758845329, "fcm_dpo/delta": -0.25692829489707947, "fcm_dpo/margin": 22.101306915283203, "fcm_dpo/q_t": 0.30949389934539795, "grad_norm": 15.676539421081543, "learning_rate": 2.0268718890989752e-07, "logits/chosen": 0.4863562285900116, "logits/rejected": 0.3901751935482025, "logps/chosen": -76.0320816040039, "logps/ref_chosen": -53.72495651245117, "logps/ref_rejected": -75.06304931640625, "logps/rejected": -119.47147369384766, "loss": 0.8297, "margin_dpo/margin_mean": 22.101306915283203, "margin_dpo/margin_std": 24.05353546142578, "step": 401 }, { "KL/chosen_KL_mean": -25.090410232543945, "KL/mean": -33.568870544433594, "KL/rejected_KL_mean": -42.04733657836914, "KL/std": 23.15046501159668, "epoch": 0.6077097505668935, "fcm_dpo/beta": 0.04576147347688675, "fcm_dpo/delta": 0.02423737943172455, "fcm_dpo/margin": 16.956928253173828, "fcm_dpo/q_t": 0.35193073749542236, "grad_norm": 20.219722747802734, "learning_rate": 2.013895317751323e-07, "logits/chosen": 0.4610044062137604, "logits/rejected": 0.42455703020095825, "logps/chosen": -86.96434020996094, "logps/ref_chosen": -61.873931884765625, "logps/ref_rejected": -66.15198516845703, "logps/rejected": -108.19932556152344, "loss": 1.0099, "margin_dpo/margin_mean": 16.956926345825195, "margin_dpo/margin_std": 24.181861877441406, "step": 402 }, { "KL/chosen_KL_mean": -26.66484260559082, "KL/mean": -37.880027770996094, "KL/rejected_KL_mean": -49.0952262878418, "KL/std": 26.00891876220703, "epoch": 0.6092214663643235, "fcm_dpo/beta": 0.04488415643572807, "fcm_dpo/delta": -0.2293957620859146, "fcm_dpo/margin": 22.430383682250977, "fcm_dpo/q_t": 0.3173214793205261, "grad_norm": 19.93121910095215, "learning_rate": 2.0009323437965898e-07, "logits/chosen": 0.5491383075714111, "logits/rejected": 0.4719066619873047, "logps/chosen": -77.98634338378906, "logps/ref_chosen": -51.321502685546875, "logps/ref_rejected": -86.54010772705078, "logps/rejected": -135.6353302001953, "loss": 0.8867, "margin_dpo/margin_mean": 22.430381774902344, "margin_dpo/margin_std": 26.761947631835938, "step": 403 }, { "KL/chosen_KL_mean": -24.51099967956543, "KL/mean": -34.40475082397461, "KL/rejected_KL_mean": -44.298500061035156, "KL/std": 23.84885025024414, "epoch": 0.6107331821617535, "fcm_dpo/beta": 0.04325392469763756, "fcm_dpo/delta": -0.06493037939071655, "fcm_dpo/margin": 19.787498474121094, "fcm_dpo/q_t": 0.33856746554374695, "grad_norm": 20.945728302001953, "learning_rate": 1.9879833298370237e-07, "logits/chosen": 0.4318276643753052, "logits/rejected": 0.34396785497665405, "logps/chosen": -86.77388763427734, "logps/ref_chosen": -62.26288604736328, "logps/ref_rejected": -95.19029998779297, "logps/rejected": -139.48880004882812, "loss": 0.9584, "margin_dpo/margin_mean": 19.787500381469727, "margin_dpo/margin_std": 25.836761474609375, "step": 404 }, { "KL/chosen_KL_mean": -26.306303024291992, "KL/mean": -34.85593795776367, "KL/rejected_KL_mean": -43.405582427978516, "KL/std": 23.573284149169922, "epoch": 0.6122448979591837, "fcm_dpo/beta": 0.043290168046951294, "fcm_dpo/delta": 0.05967658758163452, "fcm_dpo/margin": 17.099279403686523, "fcm_dpo/q_t": 0.358862042427063, "grad_norm": 17.597557067871094, "learning_rate": 1.975048638084379e-07, "logits/chosen": 0.5193575620651245, "logits/rejected": 0.4726630747318268, "logps/chosen": -76.89064025878906, "logps/ref_chosen": -50.5843391418457, "logps/ref_rejected": -65.43156433105469, "logps/rejected": -108.83714294433594, "loss": 0.9932, "margin_dpo/margin_mean": 17.099279403686523, "margin_dpo/margin_std": 22.427947998046875, "step": 405 }, { "KL/chosen_KL_mean": -25.730045318603516, "KL/mean": -36.04084777832031, "KL/rejected_KL_mean": -46.35163879394531, "KL/std": 24.698978424072266, "epoch": 0.6137566137566137, "fcm_dpo/beta": 0.04326792061328888, "fcm_dpo/delta": -0.10235971957445145, "fcm_dpo/margin": 20.621601104736328, "fcm_dpo/q_t": 0.33060896396636963, "grad_norm": 18.64668846130371, "learning_rate": 1.9621286303497914e-07, "logits/chosen": 0.5034130811691284, "logits/rejected": 0.35505062341690063, "logps/chosen": -74.72564697265625, "logps/ref_chosen": -48.99560546875, "logps/ref_rejected": -92.47774505615234, "logps/rejected": -138.82937622070312, "loss": 0.9493, "margin_dpo/margin_mean": 20.62160301208496, "margin_dpo/margin_std": 26.697372436523438, "step": 406 }, { "KL/chosen_KL_mean": -32.66217803955078, "KL/mean": -40.6044921875, "KL/rejected_KL_mean": -48.546810150146484, "KL/std": 26.861190795898438, "epoch": 0.6152683295540439, "fcm_dpo/beta": 0.04399215802550316, "fcm_dpo/delta": 0.10871283710002899, "fcm_dpo/margin": 15.884638786315918, "fcm_dpo/q_t": 0.36650407314300537, "grad_norm": 22.695158004760742, "learning_rate": 1.9492236680336483e-07, "logits/chosen": 0.3715978264808655, "logits/rejected": 0.29767975211143494, "logps/chosen": -122.062744140625, "logps/ref_chosen": -89.40056610107422, "logps/ref_rejected": -99.28775024414062, "logps/rejected": -147.83456420898438, "loss": 1.0453, "margin_dpo/margin_mean": 15.884637832641602, "margin_dpo/margin_std": 24.79953384399414, "step": 407 }, { "KL/chosen_KL_mean": -23.589235305786133, "KL/mean": -35.32756805419922, "KL/rejected_KL_mean": -47.06589126586914, "KL/std": 24.08920669555664, "epoch": 0.6167800453514739, "fcm_dpo/beta": 0.043174490332603455, "fcm_dpo/delta": -0.23764048516750336, "fcm_dpo/margin": 23.47665023803711, "fcm_dpo/q_t": 0.30589696764945984, "grad_norm": 16.45615577697754, "learning_rate": 1.9363341121154895e-07, "logits/chosen": 0.46635252237319946, "logits/rejected": 0.387617826461792, "logps/chosen": -78.29315185546875, "logps/ref_chosen": -54.70391845703125, "logps/ref_rejected": -73.98648834228516, "logps/rejected": -121.05238342285156, "loss": 0.8415, "margin_dpo/margin_mean": 23.47665023803711, "margin_dpo/margin_std": 24.754615783691406, "step": 408 }, { "KL/chosen_KL_mean": -29.324310302734375, "KL/mean": -36.13206481933594, "KL/rejected_KL_mean": -42.93982696533203, "KL/std": 22.537269592285156, "epoch": 0.618291761148904, "fcm_dpo/beta": 0.04347747936844826, "fcm_dpo/delta": 0.22149579226970673, "fcm_dpo/margin": 13.61551284790039, "fcm_dpo/q_t": 0.3939046263694763, "grad_norm": 21.601776123046875, "learning_rate": 1.9234603231438994e-07, "logits/chosen": 0.4784550368785858, "logits/rejected": 0.4812003970146179, "logps/chosen": -91.44253540039062, "logps/ref_chosen": -62.11822509765625, "logps/ref_rejected": -61.933509826660156, "logps/rejected": -104.87333679199219, "loss": 1.1333, "margin_dpo/margin_mean": 13.615513801574707, "margin_dpo/margin_std": 25.808116912841797, "step": 409 }, { "KL/chosen_KL_mean": -27.31926727294922, "KL/mean": -36.74012756347656, "KL/rejected_KL_mean": -46.16097640991211, "KL/std": 23.640226364135742, "epoch": 0.6198034769463341, "fcm_dpo/beta": 0.043405093252658844, "fcm_dpo/delta": -0.02259385585784912, "fcm_dpo/margin": 18.84171485900879, "fcm_dpo/q_t": 0.333656370639801, "grad_norm": 19.895095825195312, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 0.47311586141586304, "logits/rejected": 0.44732558727264404, "logps/chosen": -89.12193298339844, "logps/ref_chosen": -61.80266189575195, "logps/ref_rejected": -76.60002136230469, "logps/rejected": -122.76099395751953, "loss": 0.8784, "margin_dpo/margin_mean": 18.84171485900879, "margin_dpo/margin_std": 19.36935806274414, "step": 410 }, { "KL/chosen_KL_mean": -32.32759475708008, "KL/mean": -42.412288665771484, "KL/rejected_KL_mean": -52.496978759765625, "KL/std": 28.044313430786133, "epoch": 0.6213151927437641, "fcm_dpo/beta": 0.04318157210946083, "fcm_dpo/delta": -0.07840226590633392, "fcm_dpo/margin": 20.169395446777344, "fcm_dpo/q_t": 0.3437616527080536, "grad_norm": 24.5039119720459, "learning_rate": 1.8977614860195296e-07, "logits/chosen": 0.47114986181259155, "logits/rejected": 0.4128589630126953, "logps/chosen": -86.77298736572266, "logps/ref_chosen": -54.44539260864258, "logps/ref_rejected": -74.5650863647461, "logps/rejected": -127.06206512451172, "loss": 1.0059, "margin_dpo/margin_mean": 20.169395446777344, "margin_dpo/margin_std": 29.603900909423828, "step": 411 }, { "KL/chosen_KL_mean": -34.439205169677734, "KL/mean": -44.22332763671875, "KL/rejected_KL_mean": -54.00743865966797, "KL/std": 24.429527282714844, "epoch": 0.6228269085411943, "fcm_dpo/beta": 0.042693983763456345, "fcm_dpo/delta": -0.0395471453666687, "fcm_dpo/margin": 19.5682373046875, "fcm_dpo/q_t": 0.34421515464782715, "grad_norm": 19.786897659301758, "learning_rate": 1.8849371567184662e-07, "logits/chosen": 0.4648180603981018, "logits/rejected": 0.39858123660087585, "logps/chosen": -89.68728637695312, "logps/ref_chosen": -55.248085021972656, "logps/ref_rejected": -68.96623229980469, "logps/rejected": -122.97366333007812, "loss": 0.9329, "margin_dpo/margin_mean": 19.5682373046875, "margin_dpo/margin_std": 24.99486541748047, "step": 412 }, { "KL/chosen_KL_mean": -35.61238098144531, "KL/mean": -44.424903869628906, "KL/rejected_KL_mean": -53.237422943115234, "KL/std": 24.641502380371094, "epoch": 0.6243386243386243, "fcm_dpo/beta": 0.04312215372920036, "fcm_dpo/delta": 0.04325508326292038, "fcm_dpo/margin": 17.625036239624023, "fcm_dpo/q_t": 0.35896366834640503, "grad_norm": 25.902362823486328, "learning_rate": 1.872130032047302e-07, "logits/chosen": 0.3176858425140381, "logits/rejected": 0.284635990858078, "logps/chosen": -104.3331298828125, "logps/ref_chosen": -68.72074890136719, "logps/ref_rejected": -78.76539611816406, "logps/rejected": -132.00282287597656, "loss": 1.0748, "margin_dpo/margin_mean": 17.62503433227539, "margin_dpo/margin_std": 29.25439453125, "step": 413 }, { "KL/chosen_KL_mean": -32.86102294921875, "KL/mean": -43.88529968261719, "KL/rejected_KL_mean": -54.909568786621094, "KL/std": 29.22039222717285, "epoch": 0.6258503401360545, "fcm_dpo/beta": 0.04218984767794609, "fcm_dpo/delta": -0.14414259791374207, "fcm_dpo/margin": 22.048547744750977, "fcm_dpo/q_t": 0.324825644493103, "grad_norm": 18.57297706604004, "learning_rate": 1.8593404702488436e-07, "logits/chosen": 0.4745393991470337, "logits/rejected": 0.4136067032814026, "logps/chosen": -86.99923706054688, "logps/ref_chosen": -54.138214111328125, "logps/ref_rejected": -74.65741729736328, "logps/rejected": -129.56698608398438, "loss": 0.91, "margin_dpo/margin_mean": 22.04854965209961, "margin_dpo/margin_std": 27.189411163330078, "step": 414 }, { "KL/chosen_KL_mean": -32.201171875, "KL/mean": -42.07051467895508, "KL/rejected_KL_mean": -51.939857482910156, "KL/std": 28.312644958496094, "epoch": 0.6273620559334845, "fcm_dpo/beta": 0.04169802740216255, "fcm_dpo/delta": -0.025423936545848846, "fcm_dpo/margin": 19.738685607910156, "fcm_dpo/q_t": 0.3469679355621338, "grad_norm": 21.86366844177246, "learning_rate": 1.846568829074628e-07, "logits/chosen": 0.47773563861846924, "logits/rejected": 0.45856568217277527, "logps/chosen": -88.11973571777344, "logps/ref_chosen": -55.91856002807617, "logps/ref_rejected": -61.747703552246094, "logps/rejected": -113.68756103515625, "loss": 1.0134, "margin_dpo/margin_mean": 19.738685607910156, "margin_dpo/margin_std": 29.190603256225586, "step": 415 }, { "KL/chosen_KL_mean": -35.82605743408203, "KL/mean": -43.756290435791016, "KL/rejected_KL_mean": -51.686519622802734, "KL/std": 29.21381378173828, "epoch": 0.6288737717309146, "fcm_dpo/beta": 0.04106555134057999, "fcm_dpo/delta": -0.1727903038263321, "fcm_dpo/margin": 15.860455513000488, "fcm_dpo/q_t": 0.37967050075531006, "grad_norm": 23.822437286376953, "learning_rate": 1.8338154657749128e-07, "logits/chosen": 0.4701812267303467, "logits/rejected": 0.419062077999115, "logps/chosen": -90.54914093017578, "logps/ref_chosen": -54.72308349609375, "logps/ref_rejected": -69.17388916015625, "logps/rejected": -120.86041259765625, "loss": 1.1331, "margin_dpo/margin_mean": 15.860454559326172, "margin_dpo/margin_std": 27.254606246948242, "step": 416 }, { "KL/chosen_KL_mean": -38.493438720703125, "KL/mean": -49.2604866027832, "KL/rejected_KL_mean": -60.02753448486328, "KL/std": 26.403274536132812, "epoch": 0.6303854875283447, "fcm_dpo/beta": 0.040380626916885376, "fcm_dpo/delta": -0.07827968895435333, "fcm_dpo/margin": 21.53409194946289, "fcm_dpo/q_t": 0.3280333876609802, "grad_norm": 22.19675064086914, "learning_rate": 1.8210807370886849e-07, "logits/chosen": 0.5746136903762817, "logits/rejected": 0.5063204169273376, "logps/chosen": -95.28469848632812, "logps/ref_chosen": -56.791259765625, "logps/ref_rejected": -68.7791748046875, "logps/rejected": -128.80670166015625, "loss": 0.967, "margin_dpo/margin_mean": 21.534093856811523, "margin_dpo/margin_std": 28.17495346069336, "step": 417 }, { "KL/chosen_KL_mean": -41.96894454956055, "KL/mean": -49.41185760498047, "KL/rejected_KL_mean": -56.854766845703125, "KL/std": 28.295509338378906, "epoch": 0.6318972033257747, "fcm_dpo/beta": 0.040416378527879715, "fcm_dpo/delta": 0.20983785390853882, "fcm_dpo/margin": 14.885822296142578, "fcm_dpo/q_t": 0.3924134373664856, "grad_norm": 26.25454330444336, "learning_rate": 1.8083649992336825e-07, "logits/chosen": 0.5161227583885193, "logits/rejected": 0.522834062576294, "logps/chosen": -111.07693481445312, "logps/ref_chosen": -69.10798645019531, "logps/ref_rejected": -75.09132385253906, "logps/rejected": -131.9460906982422, "loss": 1.1556, "margin_dpo/margin_mean": 14.885823249816895, "margin_dpo/margin_std": 28.37472915649414, "step": 418 }, { "KL/chosen_KL_mean": -30.667163848876953, "KL/mean": -42.797237396240234, "KL/rejected_KL_mean": -54.92730712890625, "KL/std": 27.933753967285156, "epoch": 0.6334089191232048, "fcm_dpo/beta": 0.040009621530771255, "fcm_dpo/delta": -0.19122833013534546, "fcm_dpo/margin": 24.260149002075195, "fcm_dpo/q_t": 0.32048365473747253, "grad_norm": 19.607967376708984, "learning_rate": 1.7956686078964255e-07, "logits/chosen": 0.3947601318359375, "logits/rejected": 0.3380334973335266, "logps/chosen": -88.83892822265625, "logps/ref_chosen": -58.1717643737793, "logps/ref_rejected": -71.67066955566406, "logps/rejected": -126.59797668457031, "loss": 0.8978, "margin_dpo/margin_mean": 24.260147094726562, "margin_dpo/margin_std": 29.379589080810547, "step": 419 }, { "KL/chosen_KL_mean": -42.81511688232422, "KL/mean": -49.22902297973633, "KL/rejected_KL_mean": -55.642921447753906, "KL/std": 28.587371826171875, "epoch": 0.6349206349206349, "fcm_dpo/beta": 0.041276998817920685, "fcm_dpo/delta": 0.2865866422653198, "fcm_dpo/margin": 12.827811241149902, "fcm_dpo/q_t": 0.4050375819206238, "grad_norm": 25.256057739257812, "learning_rate": 1.782991918222275e-07, "logits/chosen": 0.4635976552963257, "logits/rejected": 0.4160757064819336, "logps/chosen": -99.86862182617188, "logps/ref_chosen": -57.05351257324219, "logps/ref_rejected": -62.670982360839844, "logps/rejected": -118.31390380859375, "loss": 1.2747, "margin_dpo/margin_mean": 12.827811241149902, "margin_dpo/margin_std": 31.175682067871094, "step": 420 }, { "KL/chosen_KL_mean": -40.56446075439453, "KL/mean": -49.28892135620117, "KL/rejected_KL_mean": -58.01338577270508, "KL/std": 28.834896087646484, "epoch": 0.636432350718065, "fcm_dpo/beta": 0.04242285341024399, "fcm_dpo/delta": 0.06434623152017593, "fcm_dpo/margin": 17.448923110961914, "fcm_dpo/q_t": 0.3707146644592285, "grad_norm": 26.102148056030273, "learning_rate": 1.7703352848054887e-07, "logits/chosen": 0.43699386715888977, "logits/rejected": 0.3786683678627014, "logps/chosen": -97.88771057128906, "logps/ref_chosen": -57.32324981689453, "logps/ref_rejected": -75.33782958984375, "logps/rejected": -133.35121154785156, "loss": 1.1907, "margin_dpo/margin_mean": 17.448925018310547, "margin_dpo/margin_std": 34.14581298828125, "step": 421 }, { "KL/chosen_KL_mean": -33.99824142456055, "KL/mean": -44.69304275512695, "KL/rejected_KL_mean": -55.387847900390625, "KL/std": 26.90380859375, "epoch": 0.6379440665154951, "fcm_dpo/beta": 0.042237088084220886, "fcm_dpo/delta": -0.11359019577503204, "fcm_dpo/margin": 21.389610290527344, "fcm_dpo/q_t": 0.3315642476081848, "grad_norm": 22.26106834411621, "learning_rate": 1.7576990616793137e-07, "logits/chosen": 0.4836987257003784, "logits/rejected": 0.4552809000015259, "logps/chosen": -101.05581665039062, "logps/ref_chosen": -67.05757141113281, "logps/ref_rejected": -72.12803649902344, "logps/rejected": -127.51588439941406, "loss": 0.93, "margin_dpo/margin_mean": 21.389610290527344, "margin_dpo/margin_std": 27.35793685913086, "step": 422 }, { "KL/chosen_KL_mean": -36.689964294433594, "KL/mean": -48.771995544433594, "KL/rejected_KL_mean": -60.854026794433594, "KL/std": 30.184703826904297, "epoch": 0.6394557823129252, "fcm_dpo/beta": 0.04042931646108627, "fcm_dpo/delta": -0.19745704531669617, "fcm_dpo/margin": 24.1640625, "fcm_dpo/q_t": 0.32379740476608276, "grad_norm": 20.41207504272461, "learning_rate": 1.745083602306071e-07, "logits/chosen": 0.5219099521636963, "logits/rejected": 0.45282691717147827, "logps/chosen": -90.75163269042969, "logps/ref_chosen": -54.06167221069336, "logps/ref_rejected": -76.64092254638672, "logps/rejected": -137.4949493408203, "loss": 0.9104, "margin_dpo/margin_mean": 24.1640625, "margin_dpo/margin_std": 30.572715759277344, "step": 423 }, { "KL/chosen_KL_mean": -36.551551818847656, "KL/mean": -49.043670654296875, "KL/rejected_KL_mean": -61.53579330444336, "KL/std": 27.892314910888672, "epoch": 0.6409674981103552, "fcm_dpo/beta": 0.03874318301677704, "fcm_dpo/delta": -0.1893288791179657, "fcm_dpo/margin": 24.984237670898438, "fcm_dpo/q_t": 0.3210356831550598, "grad_norm": 19.895740509033203, "learning_rate": 1.7324892595672804e-07, "logits/chosen": 0.41898566484451294, "logits/rejected": 0.38174790143966675, "logps/chosen": -90.16041564941406, "logps/ref_chosen": -53.60887145996094, "logps/ref_rejected": -79.2139892578125, "logps/rejected": -140.74978637695312, "loss": 0.8929, "margin_dpo/margin_mean": 24.984237670898438, "margin_dpo/margin_std": 29.843223571777344, "step": 424 }, { "KL/chosen_KL_mean": -36.94670867919922, "KL/mean": -46.86540985107422, "KL/rejected_KL_mean": -56.78410339355469, "KL/std": 28.853939056396484, "epoch": 0.6424792139077853, "fcm_dpo/beta": 0.038760796189308167, "fcm_dpo/delta": 0.033539630472660065, "fcm_dpo/margin": 19.837398529052734, "fcm_dpo/q_t": 0.35705694556236267, "grad_norm": 19.53350830078125, "learning_rate": 1.7199163857537824e-07, "logits/chosen": 0.5291003584861755, "logits/rejected": 0.4982011318206787, "logps/chosen": -95.36138916015625, "logps/ref_chosen": -58.41468048095703, "logps/ref_rejected": -66.59054565429688, "logps/rejected": -123.37464904785156, "loss": 1.0102, "margin_dpo/margin_mean": 19.837398529052734, "margin_dpo/margin_std": 29.011436462402344, "step": 425 }, { "KL/chosen_KL_mean": -46.007965087890625, "KL/mean": -52.275394439697266, "KL/rejected_KL_mean": -58.54281997680664, "KL/std": 27.631093978881836, "epoch": 0.6439909297052154, "fcm_dpo/beta": 0.040759261697530746, "fcm_dpo/delta": 0.3047676682472229, "fcm_dpo/margin": 12.534847259521484, "fcm_dpo/q_t": 0.40703439712524414, "grad_norm": 30.417728424072266, "learning_rate": 1.7073653325558828e-07, "logits/chosen": 0.4371623992919922, "logits/rejected": 0.4418582022190094, "logps/chosen": -117.7161865234375, "logps/ref_chosen": -71.70822143554688, "logps/ref_rejected": -73.57725524902344, "logps/rejected": -132.12008666992188, "loss": 1.2719, "margin_dpo/margin_mean": 12.534847259521484, "margin_dpo/margin_std": 31.50692367553711, "step": 426 }, { "KL/chosen_KL_mean": -42.995643615722656, "KL/mean": -52.96397018432617, "KL/rejected_KL_mean": -62.93229675292969, "KL/std": 30.741945266723633, "epoch": 0.6455026455026455, "fcm_dpo/beta": 0.041296087205410004, "fcm_dpo/delta": -0.02539144642651081, "fcm_dpo/margin": 19.936656951904297, "fcm_dpo/q_t": 0.3556680679321289, "grad_norm": 25.805469512939453, "learning_rate": 1.6948364510535218e-07, "logits/chosen": 0.5277816653251648, "logits/rejected": 0.47356265783309937, "logps/chosen": -101.63841247558594, "logps/ref_chosen": -58.64276885986328, "logps/ref_rejected": -86.25437927246094, "logps/rejected": -149.18667602539062, "loss": 1.0654, "margin_dpo/margin_mean": 19.936656951904297, "margin_dpo/margin_std": 32.75372314453125, "step": 427 }, { "KL/chosen_KL_mean": -40.44948959350586, "KL/mean": -52.776580810546875, "KL/rejected_KL_mean": -65.10366821289062, "KL/std": 31.763904571533203, "epoch": 0.6470143613000756, "fcm_dpo/beta": 0.040205471217632294, "fcm_dpo/delta": -0.2117908000946045, "fcm_dpo/margin": 24.6541748046875, "fcm_dpo/q_t": 0.32707393169403076, "grad_norm": 22.02398109436035, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 0.4488622546195984, "logits/rejected": 0.39951950311660767, "logps/chosen": -107.0455322265625, "logps/ref_chosen": -66.5960464477539, "logps/ref_rejected": -82.3941650390625, "logps/rejected": -147.49783325195312, "loss": 0.9509, "margin_dpo/margin_mean": 24.6541748046875, "margin_dpo/margin_std": 33.260841369628906, "step": 428 }, { "KL/chosen_KL_mean": -43.406707763671875, "KL/mean": -52.90144348144531, "KL/rejected_KL_mean": -62.39617919921875, "KL/std": 29.375505447387695, "epoch": 0.6485260770975056, "fcm_dpo/beta": 0.040120624005794525, "fcm_dpo/delta": 0.038818396627902985, "fcm_dpo/margin": 18.98946762084961, "fcm_dpo/q_t": 0.3599487543106079, "grad_norm": 23.097915649414062, "learning_rate": 1.669846604344412e-07, "logits/chosen": 0.4337460398674011, "logits/rejected": 0.44470861554145813, "logps/chosen": -100.41641235351562, "logps/ref_chosen": -57.00970458984375, "logps/ref_rejected": -59.86549377441406, "logps/rejected": -122.26167297363281, "loss": 1.0651, "margin_dpo/margin_mean": 18.98946762084961, "margin_dpo/margin_std": 30.113819122314453, "step": 429 }, { "KL/chosen_KL_mean": -39.49730682373047, "KL/mean": -52.07994842529297, "KL/rejected_KL_mean": -64.66258239746094, "KL/std": 29.181060791015625, "epoch": 0.6500377928949358, "fcm_dpo/beta": 0.0386689230799675, "fcm_dpo/delta": -0.1919030398130417, "fcm_dpo/margin": 25.165279388427734, "fcm_dpo/q_t": 0.317873477935791, "grad_norm": 19.91645622253418, "learning_rate": 1.6573863381573954e-07, "logits/chosen": 0.40141820907592773, "logits/rejected": 0.3949132263660431, "logps/chosen": -99.06050109863281, "logps/ref_chosen": -59.563194274902344, "logps/ref_rejected": -70.52289581298828, "logps/rejected": -135.18548583984375, "loss": 0.8937, "margin_dpo/margin_mean": 25.165279388427734, "margin_dpo/margin_std": 30.14092254638672, "step": 430 }, { "KL/chosen_KL_mean": -35.84678649902344, "KL/mean": -45.385589599609375, "KL/rejected_KL_mean": -54.92439270019531, "KL/std": 29.223169326782227, "epoch": 0.6515495086923658, "fcm_dpo/beta": 0.03864824026823044, "fcm_dpo/delta": 0.06772233545780182, "fcm_dpo/margin": 19.077606201171875, "fcm_dpo/q_t": 0.35975968837738037, "grad_norm": 20.980796813964844, "learning_rate": 1.6449496416858282e-07, "logits/chosen": 0.46574872732162476, "logits/rejected": 0.41989612579345703, "logps/chosen": -86.04711151123047, "logps/ref_chosen": -50.20032501220703, "logps/ref_rejected": -77.81680297851562, "logps/rejected": -132.74119567871094, "loss": 1.0254, "margin_dpo/margin_mean": 19.077606201171875, "margin_dpo/margin_std": 28.919309616088867, "step": 431 }, { "KL/chosen_KL_mean": -37.896148681640625, "KL/mean": -47.94352722167969, "KL/rejected_KL_mean": -57.990901947021484, "KL/std": 27.537246704101562, "epoch": 0.6530612244897959, "fcm_dpo/beta": 0.038857653737068176, "fcm_dpo/delta": 0.020730186253786087, "fcm_dpo/margin": 20.09475326538086, "fcm_dpo/q_t": 0.35490578413009644, "grad_norm": 21.475805282592773, "learning_rate": 1.632536862810844e-07, "logits/chosen": 0.49139007925987244, "logits/rejected": 0.44433116912841797, "logps/chosen": -99.55890655517578, "logps/ref_chosen": -61.662757873535156, "logps/ref_rejected": -83.94496154785156, "logps/rejected": -141.93585205078125, "loss": 1.0392, "margin_dpo/margin_mean": 20.09475326538086, "margin_dpo/margin_std": 31.069698333740234, "step": 432 }, { "KL/chosen_KL_mean": -37.729713439941406, "KL/mean": -50.83671188354492, "KL/rejected_KL_mean": -63.9437141418457, "KL/std": 30.168743133544922, "epoch": 0.654572940287226, "fcm_dpo/beta": 0.038006819784641266, "fcm_dpo/delta": -0.21750611066818237, "fcm_dpo/margin": 26.213993072509766, "fcm_dpo/q_t": 0.31744182109832764, "grad_norm": 18.48863983154297, "learning_rate": 1.6201483487445515e-07, "logits/chosen": 0.5908124446868896, "logits/rejected": 0.5816738605499268, "logps/chosen": -101.45889282226562, "logps/ref_chosen": -63.72917938232422, "logps/ref_rejected": -65.8391342163086, "logps/rejected": -129.7828369140625, "loss": 0.9122, "margin_dpo/margin_mean": 26.213993072509766, "margin_dpo/margin_std": 32.15654754638672, "step": 433 }, { "KL/chosen_KL_mean": -33.527801513671875, "KL/mean": -45.83747100830078, "KL/rejected_KL_mean": -58.14714813232422, "KL/std": 28.78369903564453, "epoch": 0.656084656084656, "fcm_dpo/beta": 0.03632465749979019, "fcm_dpo/delta": -0.10747957974672318, "fcm_dpo/margin": 24.619342803955078, "fcm_dpo/q_t": 0.32917672395706177, "grad_norm": 18.348094940185547, "learning_rate": 1.6077844460203204e-07, "logits/chosen": 0.5969550609588623, "logits/rejected": 0.5314214825630188, "logps/chosen": -81.50111389160156, "logps/ref_chosen": -47.97331619262695, "logps/ref_rejected": -72.51132202148438, "logps/rejected": -130.65847778320312, "loss": 0.9834, "margin_dpo/margin_mean": 24.619342803955078, "margin_dpo/margin_std": 33.49932098388672, "step": 434 }, { "KL/chosen_KL_mean": -39.075477600097656, "KL/mean": -49.97040557861328, "KL/rejected_KL_mean": -60.865325927734375, "KL/std": 29.0299015045166, "epoch": 0.6575963718820862, "fcm_dpo/beta": 0.03686121478676796, "fcm_dpo/delta": -0.005386091768741608, "fcm_dpo/margin": 21.789846420288086, "fcm_dpo/q_t": 0.3473183214664459, "grad_norm": 20.230436325073242, "learning_rate": 1.5954455004830878e-07, "logits/chosen": 0.6023925542831421, "logits/rejected": 0.5621955990791321, "logps/chosen": -96.13572692871094, "logps/ref_chosen": -57.06024932861328, "logps/ref_rejected": -71.69146728515625, "logps/rejected": -132.55679321289062, "loss": 0.9939, "margin_dpo/margin_mean": 21.789846420288086, "margin_dpo/margin_std": 30.15741729736328, "step": 435 }, { "KL/chosen_KL_mean": -37.826961517333984, "KL/mean": -47.93717575073242, "KL/rejected_KL_mean": -58.04738998413086, "KL/std": 29.83072280883789, "epoch": 0.6591080876795162, "fcm_dpo/beta": 0.0369018018245697, "fcm_dpo/delta": 0.05788743868470192, "fcm_dpo/margin": 20.220436096191406, "fcm_dpo/q_t": 0.3603705167770386, "grad_norm": 19.659523010253906, "learning_rate": 1.5831318572796847e-07, "logits/chosen": 0.4917636513710022, "logits/rejected": 0.4353067874908447, "logps/chosen": -93.98501586914062, "logps/ref_chosen": -56.158050537109375, "logps/ref_rejected": -67.63787841796875, "logps/rejected": -125.68527221679688, "loss": 1.0435, "margin_dpo/margin_mean": 20.220436096191406, "margin_dpo/margin_std": 31.14856719970703, "step": 436 }, { "KL/chosen_KL_mean": -41.31220245361328, "KL/mean": -51.047908782958984, "KL/rejected_KL_mean": -60.78361892700195, "KL/std": 29.29422950744629, "epoch": 0.6606198034769464, "fcm_dpo/beta": 0.03676586598157883, "fcm_dpo/delta": 0.08598443865776062, "fcm_dpo/margin": 19.471410751342773, "fcm_dpo/q_t": 0.37089112401008606, "grad_norm": 21.479209899902344, "learning_rate": 1.5708438608491815e-07, "logits/chosen": 0.4810078740119934, "logits/rejected": 0.35320615768432617, "logps/chosen": -98.29798889160156, "logps/ref_chosen": -56.98578643798828, "logps/ref_rejected": -85.61524963378906, "logps/rejected": -146.39886474609375, "loss": 1.1268, "margin_dpo/margin_mean": 19.471410751342773, "margin_dpo/margin_std": 33.92582702636719, "step": 437 }, { "KL/chosen_KL_mean": -36.553382873535156, "KL/mean": -49.979061126708984, "KL/rejected_KL_mean": -63.40473175048828, "KL/std": 32.274356842041016, "epoch": 0.6621315192743764, "fcm_dpo/beta": 0.036649350076913834, "fcm_dpo/delta": -0.20369064807891846, "fcm_dpo/margin": 26.85135841369629, "fcm_dpo/q_t": 0.32231590151786804, "grad_norm": 20.170085906982422, "learning_rate": 1.558581854913253e-07, "logits/chosen": 0.5478121042251587, "logits/rejected": 0.48098286986351013, "logps/chosen": -77.83116149902344, "logps/ref_chosen": -41.27777862548828, "logps/ref_rejected": -65.33840942382812, "logps/rejected": -128.74314880371094, "loss": 0.907, "margin_dpo/margin_mean": 26.851356506347656, "margin_dpo/margin_std": 33.0677490234375, "step": 438 }, { "KL/chosen_KL_mean": -37.44815444946289, "KL/mean": -48.96977996826172, "KL/rejected_KL_mean": -60.49140930175781, "KL/std": 30.81436538696289, "epoch": 0.6636432350718064, "fcm_dpo/beta": 0.036244139075279236, "fcm_dpo/delta": -0.040471963584423065, "fcm_dpo/margin": 23.043258666992188, "fcm_dpo/q_t": 0.33878690004348755, "grad_norm": 19.450157165527344, "learning_rate": 1.5463461824665658e-07, "logits/chosen": 0.43274006247520447, "logits/rejected": 0.3965134620666504, "logps/chosen": -118.86580657958984, "logps/ref_chosen": -81.41764831542969, "logps/ref_rejected": -94.72309875488281, "logps/rejected": -155.21450805664062, "loss": 0.9298, "margin_dpo/margin_mean": 23.043258666992188, "margin_dpo/margin_std": 28.11056137084961, "step": 439 }, { "KL/chosen_KL_mean": -35.14704132080078, "KL/mean": -47.27345657348633, "KL/rejected_KL_mean": -59.399871826171875, "KL/std": 28.511383056640625, "epoch": 0.6651549508692366, "fcm_dpo/beta": 0.03566228598356247, "fcm_dpo/delta": -0.07170647382736206, "fcm_dpo/margin": 24.252822875976562, "fcm_dpo/q_t": 0.33973562717437744, "grad_norm": 26.302644729614258, "learning_rate": 1.534137185767178e-07, "logits/chosen": 0.4645977020263672, "logits/rejected": 0.3686174154281616, "logps/chosen": -77.68522644042969, "logps/ref_chosen": -42.538185119628906, "logps/ref_rejected": -69.78813934326172, "logps/rejected": -129.18801879882812, "loss": 0.9728, "margin_dpo/margin_mean": 24.252822875976562, "margin_dpo/margin_std": 32.987892150878906, "step": 440 }, { "KL/chosen_KL_mean": -32.36697006225586, "KL/mean": -45.41593551635742, "KL/rejected_KL_mean": -58.46489715576172, "KL/std": 29.915069580078125, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0345030352473259, "fcm_dpo/delta": -0.11155462265014648, "fcm_dpo/margin": 26.097930908203125, "fcm_dpo/q_t": 0.320126473903656, "grad_norm": 17.972639083862305, "learning_rate": 1.521955206326976e-07, "logits/chosen": 0.457683265209198, "logits/rejected": 0.36754998564720154, "logps/chosen": -89.96018981933594, "logps/ref_chosen": -57.593223571777344, "logps/ref_rejected": -84.82878875732422, "logps/rejected": -143.29368591308594, "loss": 0.8487, "margin_dpo/margin_mean": 26.097930908203125, "margin_dpo/margin_std": 26.978858947753906, "step": 441 }, { "KL/chosen_KL_mean": -42.26836395263672, "KL/mean": -54.38083267211914, "KL/rejected_KL_mean": -66.49330139160156, "KL/std": 29.323583602905273, "epoch": 0.6681783824640968, "fcm_dpo/beta": 0.034237202256917953, "fcm_dpo/delta": -0.03224414587020874, "fcm_dpo/margin": 24.224939346313477, "fcm_dpo/q_t": 0.34052807092666626, "grad_norm": 19.867328643798828, "learning_rate": 1.5098005849021078e-07, "logits/chosen": 0.4998372793197632, "logits/rejected": 0.45130839943885803, "logps/chosen": -109.72958374023438, "logps/ref_chosen": -67.46121978759766, "logps/ref_rejected": -89.0693588256836, "logps/rejected": -155.56265258789062, "loss": 0.9555, "margin_dpo/margin_mean": 24.224939346313477, "margin_dpo/margin_std": 31.437952041625977, "step": 442 }, { "KL/chosen_KL_mean": -34.84177780151367, "KL/mean": -49.455894470214844, "KL/rejected_KL_mean": -64.07000732421875, "KL/std": 32.635169982910156, "epoch": 0.6696900982615268, "fcm_dpo/beta": 0.033378299325704575, "fcm_dpo/delta": -0.19421806931495667, "fcm_dpo/margin": 29.228227615356445, "fcm_dpo/q_t": 0.31669682264328003, "grad_norm": 19.16834259033203, "learning_rate": 1.4976736614834662e-07, "logits/chosen": 0.5136935710906982, "logits/rejected": 0.44100815057754517, "logps/chosen": -89.63787841796875, "logps/ref_chosen": -54.79610061645508, "logps/ref_rejected": -77.80781555175781, "logps/rejected": -141.87783813476562, "loss": 0.9044, "margin_dpo/margin_mean": 29.228227615356445, "margin_dpo/margin_std": 35.64894485473633, "step": 443 }, { "KL/chosen_KL_mean": -44.902381896972656, "KL/mean": -50.10022735595703, "KL/rejected_KL_mean": -55.29807662963867, "KL/std": 30.544994354248047, "epoch": 0.671201814058957, "fcm_dpo/beta": 0.033911701291799545, "fcm_dpo/delta": 0.14310048520565033, "fcm_dpo/margin": 10.39570426940918, "fcm_dpo/q_t": 0.4301198720932007, "grad_norm": 23.90824317932129, "learning_rate": 1.4855747752871654e-07, "logits/chosen": 0.5034670829772949, "logits/rejected": 0.4187578558921814, "logps/chosen": -103.65144348144531, "logps/ref_chosen": -58.749061584472656, "logps/ref_rejected": -86.87396240234375, "logps/rejected": -142.1720428466797, "loss": 1.3012, "margin_dpo/margin_mean": 10.39570426940918, "margin_dpo/margin_std": 29.98647689819336, "step": 444 }, { "KL/chosen_KL_mean": -43.239959716796875, "KL/mean": -55.58454132080078, "KL/rejected_KL_mean": -67.92911529541016, "KL/std": 31.177345275878906, "epoch": 0.672713529856387, "fcm_dpo/beta": 0.033827681094408035, "fcm_dpo/delta": -0.038472820073366165, "fcm_dpo/margin": 24.68915557861328, "fcm_dpo/q_t": 0.3407127261161804, "grad_norm": 19.97569465637207, "learning_rate": 1.473504264745062e-07, "logits/chosen": 0.48346036672592163, "logits/rejected": 0.466796875, "logps/chosen": -104.15739440917969, "logps/ref_chosen": -60.91743850708008, "logps/ref_rejected": -71.5637435913086, "logps/rejected": -139.49285888671875, "loss": 0.956, "margin_dpo/margin_mean": 24.68915557861328, "margin_dpo/margin_std": 32.249595642089844, "step": 445 }, { "KL/chosen_KL_mean": -35.729644775390625, "KL/mean": -51.252044677734375, "KL/rejected_KL_mean": -66.77444458007812, "KL/std": 31.810598373413086, "epoch": 0.674225245653817, "fcm_dpo/beta": 0.03296538069844246, "fcm_dpo/delta": -0.2501997649669647, "fcm_dpo/margin": 31.0447998046875, "fcm_dpo/q_t": 0.2978588044643402, "grad_norm": 14.429303169250488, "learning_rate": 1.461462467495284e-07, "logits/chosen": 0.5177422761917114, "logits/rejected": 0.4415864944458008, "logps/chosen": -84.52889251708984, "logps/ref_chosen": -48.79924774169922, "logps/ref_rejected": -71.8719482421875, "logps/rejected": -138.64639282226562, "loss": 0.7828, "margin_dpo/margin_mean": 31.0447998046875, "margin_dpo/margin_std": 27.450092315673828, "step": 446 }, { "KL/chosen_KL_mean": -34.26579284667969, "KL/mean": -49.93895721435547, "KL/rejected_KL_mean": -65.61212158203125, "KL/std": 31.172748565673828, "epoch": 0.6757369614512472, "fcm_dpo/beta": 0.031016748398542404, "fcm_dpo/delta": -0.19166170060634613, "fcm_dpo/margin": 31.346324920654297, "fcm_dpo/q_t": 0.3062456548213959, "grad_norm": 18.319395065307617, "learning_rate": 1.4494497203727843e-07, "logits/chosen": 0.43708741664886475, "logits/rejected": 0.3397822082042694, "logps/chosen": -87.9485092163086, "logps/ref_chosen": -53.682716369628906, "logps/ref_rejected": -88.17315673828125, "logps/rejected": -153.7852783203125, "loss": 0.8674, "margin_dpo/margin_mean": 31.346324920654297, "margin_dpo/margin_std": 34.39900207519531, "step": 447 }, { "KL/chosen_KL_mean": -38.89901351928711, "KL/mean": -51.21916198730469, "KL/rejected_KL_mean": -63.53931427001953, "KL/std": 31.395763397216797, "epoch": 0.6772486772486772, "fcm_dpo/beta": 0.030965670943260193, "fcm_dpo/delta": 0.04007108882069588, "fcm_dpo/margin": 24.640295028686523, "fcm_dpo/q_t": 0.3454161286354065, "grad_norm": 15.666644096374512, "learning_rate": 1.4374663593999256e-07, "logits/chosen": 0.5203387141227722, "logits/rejected": 0.46833568811416626, "logps/chosen": -92.6502685546875, "logps/ref_chosen": -53.75125503540039, "logps/ref_rejected": -77.17623901367188, "logps/rejected": -140.71554565429688, "loss": 0.9618, "margin_dpo/margin_mean": 24.64029312133789, "margin_dpo/margin_std": 31.69839859008789, "step": 448 }, { "KL/chosen_KL_mean": -48.00468063354492, "KL/mean": -54.875057220458984, "KL/rejected_KL_mean": -61.74543380737305, "KL/std": 31.669912338256836, "epoch": 0.6787603930461074, "fcm_dpo/beta": 0.03255101665854454, "fcm_dpo/delta": 0.37090978026390076, "fcm_dpo/margin": 13.740757942199707, "fcm_dpo/q_t": 0.41214174032211304, "grad_norm": 24.92876625061035, "learning_rate": 1.4255127197770707e-07, "logits/chosen": 0.36804431676864624, "logits/rejected": 0.35851621627807617, "logps/chosen": -123.83206176757812, "logps/ref_chosen": -75.82737731933594, "logps/ref_rejected": -82.20687866210938, "logps/rejected": -143.95230102539062, "loss": 1.1675, "margin_dpo/margin_mean": 13.740757942199707, "margin_dpo/margin_std": 27.162803649902344, "step": 449 }, { "KL/chosen_KL_mean": -37.56608581542969, "KL/mean": -46.34649658203125, "KL/rejected_KL_mean": -55.12691116333008, "KL/std": 29.79147720336914, "epoch": 0.6802721088435374, "fcm_dpo/beta": 0.03442070633172989, "fcm_dpo/delta": 0.20848983526229858, "fcm_dpo/margin": 17.560827255249023, "fcm_dpo/q_t": 0.3890036344528198, "grad_norm": 21.428207397460938, "learning_rate": 1.4135891358732205e-07, "logits/chosen": 0.5896279215812683, "logits/rejected": 0.4792223274707794, "logps/chosen": -84.68180847167969, "logps/ref_chosen": -47.11572265625, "logps/ref_rejected": -78.7546615600586, "logps/rejected": -133.88157653808594, "loss": 1.114, "margin_dpo/margin_mean": 17.56082534790039, "margin_dpo/margin_std": 31.81939697265625, "step": 450 }, { "KL/chosen_KL_mean": -38.557586669921875, "KL/mean": -48.040714263916016, "KL/rejected_KL_mean": -57.52384567260742, "KL/std": 30.844379425048828, "epoch": 0.6817838246409675, "fcm_dpo/beta": 0.03564320504665375, "fcm_dpo/delta": 0.13255426287651062, "fcm_dpo/margin": 18.966249465942383, "fcm_dpo/q_t": 0.3727077841758728, "grad_norm": 20.818710327148438, "learning_rate": 1.4016959412166437e-07, "logits/chosen": 0.46854376792907715, "logits/rejected": 0.4182296395301819, "logps/chosen": -101.90803527832031, "logps/ref_chosen": -63.350440979003906, "logps/ref_rejected": -76.28530883789062, "logps/rejected": -133.80914306640625, "loss": 1.0651, "margin_dpo/margin_mean": 18.966251373291016, "margin_dpo/margin_std": 30.914825439453125, "step": 451 }, { "KL/chosen_KL_mean": -38.73877716064453, "KL/mean": -49.515777587890625, "KL/rejected_KL_mean": -60.29277801513672, "KL/std": 30.167449951171875, "epoch": 0.6832955404383976, "fcm_dpo/beta": 0.035997722297906876, "fcm_dpo/delta": 0.02604661136865616, "fcm_dpo/margin": 21.553997039794922, "fcm_dpo/q_t": 0.3517000377178192, "grad_norm": 20.391658782958984, "learning_rate": 1.3898334684855645e-07, "logits/chosen": 0.47314220666885376, "logits/rejected": 0.3956415057182312, "logps/chosen": -94.32461547851562, "logps/ref_chosen": -55.58583450317383, "logps/ref_rejected": -77.68738555908203, "logps/rejected": -137.98016357421875, "loss": 1.0526, "margin_dpo/margin_mean": 21.553997039794922, "margin_dpo/margin_std": 33.468746185302734, "step": 452 }, { "KL/chosen_KL_mean": -36.91946029663086, "KL/mean": -48.99592590332031, "KL/rejected_KL_mean": -61.0723876953125, "KL/std": 30.896400451660156, "epoch": 0.6848072562358276, "fcm_dpo/beta": 0.03558473661541939, "fcm_dpo/delta": -0.06524658203125, "fcm_dpo/margin": 24.152931213378906, "fcm_dpo/q_t": 0.3470444679260254, "grad_norm": 26.403024673461914, "learning_rate": 1.3780020494988445e-07, "logits/chosen": 0.46417856216430664, "logits/rejected": 0.43202823400497437, "logps/chosen": -98.69766235351562, "logps/ref_chosen": -61.778202056884766, "logps/ref_rejected": -71.51403045654297, "logps/rejected": -132.58642578125, "loss": 1.0065, "margin_dpo/margin_mean": 24.152931213378906, "margin_dpo/margin_std": 35.1126708984375, "step": 453 }, { "KL/chosen_KL_mean": -34.68466567993164, "KL/mean": -47.10640335083008, "KL/rejected_KL_mean": -59.52813720703125, "KL/std": 32.204864501953125, "epoch": 0.6863189720332578, "fcm_dpo/beta": 0.035109348595142365, "fcm_dpo/delta": -0.07914970070123672, "fcm_dpo/margin": 24.843469619750977, "fcm_dpo/q_t": 0.3328770399093628, "grad_norm": 18.879844665527344, "learning_rate": 1.366202015206706e-07, "logits/chosen": 0.5144060850143433, "logits/rejected": 0.47107866406440735, "logps/chosen": -86.27981567382812, "logps/ref_chosen": -51.59515380859375, "logps/ref_rejected": -63.96732711791992, "logps/rejected": -123.49546813964844, "loss": 0.9862, "margin_dpo/margin_mean": 24.843469619750977, "margin_dpo/margin_std": 34.712310791015625, "step": 454 }, { "KL/chosen_KL_mean": -40.27165985107422, "KL/mean": -52.858604431152344, "KL/rejected_KL_mean": -65.44554901123047, "KL/std": 31.356714248657227, "epoch": 0.6878306878306878, "fcm_dpo/beta": 0.034394294023513794, "fcm_dpo/delta": -0.07318489253520966, "fcm_dpo/margin": 25.173885345458984, "fcm_dpo/q_t": 0.33834701776504517, "grad_norm": 19.06486701965332, "learning_rate": 1.354433695681474e-07, "logits/chosen": 0.36979854106903076, "logits/rejected": 0.3327193856239319, "logps/chosen": -110.92337036132812, "logps/ref_chosen": -70.65170288085938, "logps/ref_rejected": -77.44276428222656, "logps/rejected": -142.8883056640625, "loss": 0.93, "margin_dpo/margin_mean": 25.173885345458984, "margin_dpo/margin_std": 31.99268913269043, "step": 455 }, { "KL/chosen_KL_mean": -39.89326477050781, "KL/mean": -50.87599182128906, "KL/rejected_KL_mean": -61.858734130859375, "KL/std": 31.083784103393555, "epoch": 0.6893424036281179, "fcm_dpo/beta": 0.03456852585077286, "fcm_dpo/delta": 0.043941106647253036, "fcm_dpo/margin": 21.965469360351562, "fcm_dpo/q_t": 0.35616612434387207, "grad_norm": 22.54862403869629, "learning_rate": 1.3426974201083439e-07, "logits/chosen": 0.41186925768852234, "logits/rejected": 0.3542647957801819, "logps/chosen": -96.29154968261719, "logps/ref_chosen": -56.398284912109375, "logps/ref_rejected": -82.61642456054688, "logps/rejected": -144.47515869140625, "loss": 1.0011, "margin_dpo/margin_mean": 21.96546745300293, "margin_dpo/margin_std": 31.476985931396484, "step": 456 }, { "KL/chosen_KL_mean": -41.337158203125, "KL/mean": -52.13238525390625, "KL/rejected_KL_mean": -62.927616119384766, "KL/std": 32.70546340942383, "epoch": 0.690854119425548, "fcm_dpo/beta": 0.03470756113529205, "fcm_dpo/delta": 0.053345732390880585, "fcm_dpo/margin": 21.590456008911133, "fcm_dpo/q_t": 0.3532480001449585, "grad_norm": 17.304533004760742, "learning_rate": 1.3309935167761717e-07, "logits/chosen": 0.5655590295791626, "logits/rejected": 0.493374228477478, "logps/chosen": -86.0577392578125, "logps/ref_chosen": -44.72057342529297, "logps/ref_rejected": -68.1158676147461, "logps/rejected": -131.04348754882812, "loss": 0.9677, "margin_dpo/margin_mean": 21.590456008911133, "margin_dpo/margin_std": 27.787086486816406, "step": 457 }, { "KL/chosen_KL_mean": -36.466941833496094, "KL/mean": -48.640602111816406, "KL/rejected_KL_mean": -60.81425476074219, "KL/std": 33.46665954589844, "epoch": 0.6923658352229781, "fcm_dpo/beta": 0.034741275012493134, "fcm_dpo/delta": -0.05059751495718956, "fcm_dpo/margin": 24.34731674194336, "fcm_dpo/q_t": 0.3377965986728668, "grad_norm": 18.70404815673828, "learning_rate": 1.3193223130682936e-07, "logits/chosen": 0.4883077144622803, "logits/rejected": 0.37608468532562256, "logps/chosen": -86.4726333618164, "logps/ref_chosen": -50.00569152832031, "logps/ref_rejected": -87.50015258789062, "logps/rejected": -148.3144073486328, "loss": 0.9674, "margin_dpo/margin_mean": 24.347320556640625, "margin_dpo/margin_std": 32.958587646484375, "step": 458 }, { "KL/chosen_KL_mean": -35.83440017700195, "KL/mean": -50.071266174316406, "KL/rejected_KL_mean": -64.30813598632812, "KL/std": 33.272953033447266, "epoch": 0.6938775510204082, "fcm_dpo/beta": 0.034444473683834076, "fcm_dpo/delta": -0.20470258593559265, "fcm_dpo/margin": 28.473735809326172, "fcm_dpo/q_t": 0.3152352571487427, "grad_norm": 18.777652740478516, "learning_rate": 1.3076841354533658e-07, "logits/chosen": 0.5320541858673096, "logits/rejected": 0.4936879873275757, "logps/chosen": -101.21234893798828, "logps/ref_chosen": -65.37794494628906, "logps/ref_rejected": -88.19244384765625, "logps/rejected": -152.50057983398438, "loss": 0.8944, "margin_dpo/margin_mean": 28.473735809326172, "margin_dpo/margin_std": 31.597108840942383, "step": 459 }, { "KL/chosen_KL_mean": -39.52485656738281, "KL/mean": -54.53339385986328, "KL/rejected_KL_mean": -69.54193115234375, "KL/std": 32.83936309814453, "epoch": 0.6953892668178382, "fcm_dpo/beta": 0.032258301973342896, "fcm_dpo/delta": -0.18851391971111298, "fcm_dpo/margin": 30.017070770263672, "fcm_dpo/q_t": 0.3154947757720947, "grad_norm": 17.20231056213379, "learning_rate": 1.2960793094762345e-07, "logits/chosen": 0.5572216510772705, "logits/rejected": 0.4307131767272949, "logps/chosen": -104.0865249633789, "logps/ref_chosen": -64.5616683959961, "logps/ref_rejected": -88.67890167236328, "logps/rejected": -158.2208251953125, "loss": 0.8574, "margin_dpo/margin_mean": 30.017070770263672, "margin_dpo/margin_std": 33.085060119628906, "step": 460 }, { "KL/chosen_KL_mean": -32.57419204711914, "KL/mean": -46.81694030761719, "KL/rejected_KL_mean": -61.0596923828125, "KL/std": 30.406360626220703, "epoch": 0.6969009826152683, "fcm_dpo/beta": 0.031209370121359825, "fcm_dpo/delta": -0.10361548513174057, "fcm_dpo/margin": 28.48550033569336, "fcm_dpo/q_t": 0.3270339369773865, "grad_norm": 16.9658203125, "learning_rate": 1.2845081597488286e-07, "logits/chosen": 0.6307059526443481, "logits/rejected": 0.5390516519546509, "logps/chosen": -82.05210876464844, "logps/ref_chosen": -49.4779167175293, "logps/ref_rejected": -72.65262603759766, "logps/rejected": -133.71231079101562, "loss": 0.8926, "margin_dpo/margin_mean": 28.48550033569336, "margin_dpo/margin_std": 31.559621810913086, "step": 461 }, { "KL/chosen_KL_mean": -36.72931671142578, "KL/mean": -52.78752899169922, "KL/rejected_KL_mean": -68.84573364257812, "KL/std": 33.4709587097168, "epoch": 0.6984126984126984, "fcm_dpo/beta": 0.030348652973771095, "fcm_dpo/delta": -0.19649741053581238, "fcm_dpo/margin": 32.11642074584961, "fcm_dpo/q_t": 0.3061380684375763, "grad_norm": 16.422710418701172, "learning_rate": 1.27297100994108e-07, "logits/chosen": 0.49276840686798096, "logits/rejected": 0.43394631147384644, "logps/chosen": -97.22443389892578, "logps/ref_chosen": -60.4951171875, "logps/ref_rejected": -74.82136535644531, "logps/rejected": -143.6671142578125, "loss": 0.8381, "margin_dpo/margin_mean": 32.11642074584961, "margin_dpo/margin_std": 33.13414001464844, "step": 462 }, { "KL/chosen_KL_mean": -41.92523956298828, "KL/mean": -52.19264602661133, "KL/rejected_KL_mean": -62.46005630493164, "KL/std": 30.781005859375, "epoch": 0.6999244142101285, "fcm_dpo/beta": 0.030946975573897362, "fcm_dpo/delta": 0.1760467141866684, "fcm_dpo/margin": 20.53483009338379, "fcm_dpo/q_t": 0.3690449595451355, "grad_norm": 20.61341667175293, "learning_rate": 1.2614681827718695e-07, "logits/chosen": 0.4970882833003998, "logits/rejected": 0.4902383089065552, "logps/chosen": -109.6103515625, "logps/ref_chosen": -67.68511962890625, "logps/ref_rejected": -71.32196044921875, "logps/rejected": -133.78201293945312, "loss": 1.025, "margin_dpo/margin_mean": 20.53483009338379, "margin_dpo/margin_std": 28.81637954711914, "step": 463 }, { "KL/chosen_KL_mean": -41.53732681274414, "KL/mean": -54.971805572509766, "KL/rejected_KL_mean": -68.40628051757812, "KL/std": 33.168521881103516, "epoch": 0.7014361300075586, "fcm_dpo/beta": 0.031339820474386215, "fcm_dpo/delta": -0.0464196652173996, "fcm_dpo/margin": 26.86896514892578, "fcm_dpo/q_t": 0.3392544388771057, "grad_norm": 20.619489669799805, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 0.5063756704330444, "logits/rejected": 0.4772282838821411, "logps/chosen": -100.70297241210938, "logps/ref_chosen": -59.16564178466797, "logps/ref_rejected": -69.56146240234375, "logps/rejected": -137.96774291992188, "loss": 0.9753, "margin_dpo/margin_mean": 26.86896514892578, "margin_dpo/margin_std": 35.64836883544922, "step": 464 }, { "KL/chosen_KL_mean": -44.17216110229492, "KL/mean": -55.57133102416992, "KL/rejected_KL_mean": -66.97048950195312, "KL/std": 32.37845230102539, "epoch": 0.7029478458049887, "fcm_dpo/beta": 0.031462736427783966, "fcm_dpo/delta": 0.08915092796087265, "fcm_dpo/margin": 22.798336029052734, "fcm_dpo/q_t": 0.3652403652667999, "grad_norm": 22.905973434448242, "learning_rate": 1.238566782415197e-07, "logits/chosen": 0.5705190896987915, "logits/rejected": 0.507891833782196, "logps/chosen": -102.68583679199219, "logps/ref_chosen": -58.513671875, "logps/ref_rejected": -84.31745910644531, "logps/rejected": -151.28794860839844, "loss": 1.0512, "margin_dpo/margin_mean": 22.7983341217041, "margin_dpo/margin_std": 35.545631408691406, "step": 465 }, { "KL/chosen_KL_mean": -45.907772064208984, "KL/mean": -53.66548156738281, "KL/rejected_KL_mean": -61.42319869995117, "KL/std": 33.30298614501953, "epoch": 0.7044595616024187, "fcm_dpo/beta": 0.03294553607702255, "fcm_dpo/delta": 0.3054059147834778, "fcm_dpo/margin": 15.515429496765137, "fcm_dpo/q_t": 0.4019228518009186, "grad_norm": 25.403379440307617, "learning_rate": 1.2271688498291334e-07, "logits/chosen": 0.5198469161987305, "logits/rejected": 0.5233859419822693, "logps/chosen": -119.173583984375, "logps/ref_chosen": -73.26580810546875, "logps/ref_rejected": -74.83621215820312, "logps/rejected": -136.25941467285156, "loss": 1.1349, "margin_dpo/margin_mean": 15.51543140411377, "margin_dpo/margin_std": 29.112449645996094, "step": 466 }, { "KL/chosen_KL_mean": -43.1234130859375, "KL/mean": -54.358455657958984, "KL/rejected_KL_mean": -65.593505859375, "KL/std": 34.307823181152344, "epoch": 0.7059712773998488, "fcm_dpo/beta": 0.0338030569255352, "fcm_dpo/delta": 0.04378930479288101, "fcm_dpo/margin": 22.47007942199707, "fcm_dpo/q_t": 0.3505485951900482, "grad_norm": 18.757200241088867, "learning_rate": 1.2158065210664848e-07, "logits/chosen": 0.5469769835472107, "logits/rejected": 0.3987196385860443, "logps/chosen": -90.702880859375, "logps/ref_chosen": -47.57947540283203, "logps/ref_rejected": -78.68522644042969, "logps/rejected": -144.27871704101562, "loss": 0.9565, "margin_dpo/margin_mean": 22.470081329345703, "margin_dpo/margin_std": 29.331863403320312, "step": 467 }, { "KL/chosen_KL_mean": -36.90409851074219, "KL/mean": -53.89661407470703, "KL/rejected_KL_mean": -70.88912963867188, "KL/std": 35.14104080200195, "epoch": 0.7074829931972789, "fcm_dpo/beta": 0.032390519976615906, "fcm_dpo/delta": -0.33713847398757935, "fcm_dpo/margin": 33.98503494262695, "fcm_dpo/q_t": 0.29361268877983093, "grad_norm": 18.30913734436035, "learning_rate": 1.204480113956011e-07, "logits/chosen": 0.4965260624885559, "logits/rejected": 0.48857080936431885, "logps/chosen": -100.8318862915039, "logps/ref_chosen": -63.92778778076172, "logps/ref_rejected": -76.51626586914062, "logps/rejected": -147.4053955078125, "loss": 0.8048, "margin_dpo/margin_mean": 33.98503112792969, "margin_dpo/margin_std": 35.08293914794922, "step": 468 }, { "KL/chosen_KL_mean": -38.55724334716797, "KL/mean": -52.420753479003906, "KL/rejected_KL_mean": -66.28427124023438, "KL/std": 32.45834732055664, "epoch": 0.708994708994709, "fcm_dpo/beta": 0.030920889228582382, "fcm_dpo/delta": -0.06855427473783493, "fcm_dpo/margin": 27.72703742980957, "fcm_dpo/q_t": 0.33362048864364624, "grad_norm": 19.86760902404785, "learning_rate": 1.1931899453216697e-07, "logits/chosen": 0.5974500179290771, "logits/rejected": 0.5843087434768677, "logps/chosen": -97.61542510986328, "logps/ref_chosen": -59.05818176269531, "logps/ref_rejected": -75.67672729492188, "logps/rejected": -141.96099853515625, "loss": 0.9071, "margin_dpo/margin_mean": 27.727035522460938, "margin_dpo/margin_std": 31.478986740112305, "step": 469 }, { "KL/chosen_KL_mean": -40.4164924621582, "KL/mean": -53.52194595336914, "KL/rejected_KL_mean": -66.62739562988281, "KL/std": 32.51350402832031, "epoch": 0.7105064247921391, "fcm_dpo/beta": 0.03135482966899872, "fcm_dpo/delta": -0.02440682053565979, "fcm_dpo/margin": 26.21091079711914, "fcm_dpo/q_t": 0.3376317322254181, "grad_norm": 16.981752395629883, "learning_rate": 1.1819363309737438e-07, "logits/chosen": 0.5298024415969849, "logits/rejected": 0.460299551486969, "logps/chosen": -88.283935546875, "logps/ref_chosen": -47.86743927001953, "logps/ref_rejected": -65.96859741210938, "logps/rejected": -132.5959930419922, "loss": 0.9647, "margin_dpo/margin_mean": 26.21091079711914, "margin_dpo/margin_std": 34.34257507324219, "step": 470 }, { "KL/chosen_KL_mean": -35.6857795715332, "KL/mean": -51.856781005859375, "KL/rejected_KL_mean": -68.02778625488281, "KL/std": 33.62290573120117, "epoch": 0.7120181405895691, "fcm_dpo/beta": 0.030277177691459656, "fcm_dpo/delta": -0.19842389225959778, "fcm_dpo/margin": 32.342002868652344, "fcm_dpo/q_t": 0.3140113055706024, "grad_norm": 17.59916877746582, "learning_rate": 1.1707195857000215e-07, "logits/chosen": 0.5253442525863647, "logits/rejected": 0.4637981951236725, "logps/chosen": -93.46363830566406, "logps/ref_chosen": -57.777854919433594, "logps/ref_rejected": -73.81172180175781, "logps/rejected": -141.83950805664062, "loss": 0.8804, "margin_dpo/margin_mean": 32.342002868652344, "margin_dpo/margin_std": 37.22600555419922, "step": 471 }, { "KL/chosen_KL_mean": -37.23526382446289, "KL/mean": -50.47922897338867, "KL/rejected_KL_mean": -63.72319030761719, "KL/std": 33.24669647216797, "epoch": 0.7135298563869993, "fcm_dpo/beta": 0.03014422580599785, "fcm_dpo/delta": 0.00048603862524032593, "fcm_dpo/margin": 26.48792266845703, "fcm_dpo/q_t": 0.35318523645401, "grad_norm": 19.13536834716797, "learning_rate": 1.1595400232569768e-07, "logits/chosen": 0.5265265703201294, "logits/rejected": 0.47909194231033325, "logps/chosen": -93.14393615722656, "logps/ref_chosen": -55.908668518066406, "logps/ref_rejected": -74.70294189453125, "logps/rejected": -138.42613220214844, "loss": 1.019, "margin_dpo/margin_mean": 26.48792266845703, "margin_dpo/margin_std": 38.939327239990234, "step": 472 }, { "KL/chosen_KL_mean": -37.54869842529297, "KL/mean": -50.015480041503906, "KL/rejected_KL_mean": -62.48226547241211, "KL/std": 35.00571823120117, "epoch": 0.7150415721844293, "fcm_dpo/beta": 0.030113372951745987, "fcm_dpo/delta": 0.0531911626458168, "fcm_dpo/margin": 24.933568954467773, "fcm_dpo/q_t": 0.3640349209308624, "grad_norm": 21.23849105834961, "learning_rate": 1.1483979563610069e-07, "logits/chosen": 0.612478494644165, "logits/rejected": 0.5012402534484863, "logps/chosen": -91.70957946777344, "logps/ref_chosen": -54.16088104248047, "logps/ref_rejected": -92.76789855957031, "logps/rejected": -155.25015258789062, "loss": 1.0737, "margin_dpo/margin_mean": 24.933568954467773, "margin_dpo/margin_std": 40.785545349121094, "step": 473 }, { "KL/chosen_KL_mean": -43.27687072753906, "KL/mean": -54.43352508544922, "KL/rejected_KL_mean": -65.59016418457031, "KL/std": 34.479217529296875, "epoch": 0.7165532879818595, "fcm_dpo/beta": 0.030809249728918076, "fcm_dpo/delta": 0.12073996663093567, "fcm_dpo/margin": 22.31329345703125, "fcm_dpo/q_t": 0.3735297918319702, "grad_norm": 23.4538631439209, "learning_rate": 1.1372936966796709e-07, "logits/chosen": 0.627903401851654, "logits/rejected": 0.5501044988632202, "logps/chosen": -89.96257781982422, "logps/ref_chosen": -46.685707092285156, "logps/ref_rejected": -71.44731903076172, "logps/rejected": -137.0374755859375, "loss": 1.09, "margin_dpo/margin_mean": 22.31329345703125, "margin_dpo/margin_std": 37.7652587890625, "step": 474 }, { "KL/chosen_KL_mean": -40.59794616699219, "KL/mean": -58.311309814453125, "KL/rejected_KL_mean": -76.02467346191406, "KL/std": 35.56901550292969, "epoch": 0.7180650037792895, "fcm_dpo/beta": 0.029465150088071823, "fcm_dpo/delta": -0.2766759693622589, "fcm_dpo/margin": 35.42672348022461, "fcm_dpo/q_t": 0.2970644235610962, "grad_norm": 15.178966522216797, "learning_rate": 1.126227554822985e-07, "logits/chosen": 0.5457562208175659, "logits/rejected": 0.4907382130622864, "logps/chosen": -99.08525085449219, "logps/ref_chosen": -58.4873046875, "logps/ref_rejected": -87.00187683105469, "logps/rejected": -163.02655029296875, "loss": 0.7885, "margin_dpo/margin_mean": 35.42672348022461, "margin_dpo/margin_std": 33.91914367675781, "step": 475 }, { "KL/chosen_KL_mean": -48.61393356323242, "KL/mean": -60.248931884765625, "KL/rejected_KL_mean": -71.8839340209961, "KL/std": 35.561439514160156, "epoch": 0.7195767195767195, "fcm_dpo/beta": 0.029918601736426353, "fcm_dpo/delta": 0.11073946952819824, "fcm_dpo/margin": 23.26999855041504, "fcm_dpo/q_t": 0.3694406747817993, "grad_norm": 21.122638702392578, "learning_rate": 1.1151998403347243e-07, "logits/chosen": 0.4469614028930664, "logits/rejected": 0.43910348415374756, "logps/chosen": -123.99555969238281, "logps/ref_chosen": -75.38162231445312, "logps/ref_rejected": -76.99822235107422, "logps/rejected": -148.8821563720703, "loss": 1.0497, "margin_dpo/margin_mean": 23.270000457763672, "margin_dpo/margin_std": 36.53971481323242, "step": 476 }, { "KL/chosen_KL_mean": -49.38127899169922, "KL/mean": -61.15679931640625, "KL/rejected_KL_mean": -72.93232727050781, "KL/std": 35.085174560546875, "epoch": 0.7210884353741497, "fcm_dpo/beta": 0.03036920353770256, "fcm_dpo/delta": 0.09131810069084167, "fcm_dpo/margin": 23.55103874206543, "fcm_dpo/q_t": 0.36580678820610046, "grad_norm": 24.693286895751953, "learning_rate": 1.1042108616837692e-07, "logits/chosen": 0.5518933534622192, "logits/rejected": 0.5037527084350586, "logps/chosen": -110.45466613769531, "logps/ref_chosen": -61.073387145996094, "logps/ref_rejected": -81.34375, "logps/rejected": -154.27606201171875, "loss": 1.1101, "margin_dpo/margin_mean": 23.551036834716797, "margin_dpo/margin_std": 40.80988311767578, "step": 477 }, { "KL/chosen_KL_mean": -41.85948181152344, "KL/mean": -52.576988220214844, "KL/rejected_KL_mean": -63.29448699951172, "KL/std": 33.8995475769043, "epoch": 0.7226001511715797, "fcm_dpo/beta": 0.031108807772397995, "fcm_dpo/delta": 0.14290378987789154, "fcm_dpo/margin": 21.435009002685547, "fcm_dpo/q_t": 0.380104124546051, "grad_norm": 25.761295318603516, "learning_rate": 1.0932609262554746e-07, "logits/chosen": 0.49377888441085815, "logits/rejected": 0.4970715641975403, "logps/chosen": -99.02679443359375, "logps/ref_chosen": -57.16731643676758, "logps/ref_rejected": -53.30917739868164, "logps/rejected": -116.60366821289062, "loss": 1.1518, "margin_dpo/margin_mean": 21.435009002685547, "margin_dpo/margin_std": 40.712486267089844, "step": 478 }, { "KL/chosen_KL_mean": -44.9219856262207, "KL/mean": -54.529151916503906, "KL/rejected_KL_mean": -64.13630676269531, "KL/std": 31.78256607055664, "epoch": 0.7241118669690099, "fcm_dpo/beta": 0.032088786363601685, "fcm_dpo/delta": 0.19578224420547485, "fcm_dpo/margin": 19.21432876586914, "fcm_dpo/q_t": 0.3787573575973511, "grad_norm": 22.831226348876953, "learning_rate": 1.0823503403430734e-07, "logits/chosen": 0.45101386308670044, "logits/rejected": 0.39903637766838074, "logps/chosen": -103.83529663085938, "logps/ref_chosen": -58.91331481933594, "logps/ref_rejected": -63.7403450012207, "logps/rejected": -127.87665557861328, "loss": 1.123, "margin_dpo/margin_mean": 19.21432876586914, "margin_dpo/margin_std": 34.718929290771484, "step": 479 }, { "KL/chosen_KL_mean": -43.14649200439453, "KL/mean": -56.787742614746094, "KL/rejected_KL_mean": -70.42898559570312, "KL/std": 32.69535827636719, "epoch": 0.7256235827664399, "fcm_dpo/beta": 0.03261556476354599, "fcm_dpo/delta": -0.10105873644351959, "fcm_dpo/margin": 27.282489776611328, "fcm_dpo/q_t": 0.3315708041191101, "grad_norm": 24.251625061035156, "learning_rate": 1.0714794091391072e-07, "logits/chosen": 0.4233095645904541, "logits/rejected": 0.41327613592147827, "logps/chosen": -105.94709777832031, "logps/ref_chosen": -62.80061340332031, "logps/ref_rejected": -67.58859252929688, "logps/rejected": -138.017578125, "loss": 1.0316, "margin_dpo/margin_mean": 27.282489776611328, "margin_dpo/margin_std": 39.7613525390625, "step": 480 }, { "KL/chosen_KL_mean": -42.957916259765625, "KL/mean": -57.020362854003906, "KL/rejected_KL_mean": -71.08280944824219, "KL/std": 36.26377868652344, "epoch": 0.72713529856387, "fcm_dpo/beta": 0.03143627941608429, "fcm_dpo/delta": -0.09304574131965637, "fcm_dpo/margin": 28.12488555908203, "fcm_dpo/q_t": 0.3399431109428406, "grad_norm": 21.250574111938477, "learning_rate": 1.0606484367268906e-07, "logits/chosen": 0.4657706618309021, "logits/rejected": 0.4617427885532379, "logps/chosen": -108.2444076538086, "logps/ref_chosen": -65.28649139404297, "logps/ref_rejected": -70.78668212890625, "logps/rejected": -141.86949157714844, "loss": 0.9748, "margin_dpo/margin_mean": 28.12488555908203, "margin_dpo/margin_std": 38.75299072265625, "step": 481 }, { "KL/chosen_KL_mean": -48.609527587890625, "KL/mean": -60.3476676940918, "KL/rejected_KL_mean": -72.08580780029297, "KL/std": 34.90166473388672, "epoch": 0.7286470143613001, "fcm_dpo/beta": 0.031867966055870056, "fcm_dpo/delta": 0.054899971932172775, "fcm_dpo/margin": 23.47627830505371, "fcm_dpo/q_t": 0.36690202355384827, "grad_norm": 25.70149040222168, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 0.4665817618370056, "logits/rejected": 0.3242020606994629, "logps/chosen": -109.51571655273438, "logps/ref_chosen": -60.906185150146484, "logps/ref_rejected": -103.44656372070312, "logps/rejected": -175.53237915039062, "loss": 1.1398, "margin_dpo/margin_mean": 23.47627830505371, "margin_dpo/margin_std": 42.882415771484375, "step": 482 }, { "KL/chosen_KL_mean": -41.326690673828125, "KL/mean": -55.978431701660156, "KL/rejected_KL_mean": -70.63018035888672, "KL/std": 33.76634979248047, "epoch": 0.7301587301587301, "fcm_dpo/beta": 0.031191810965538025, "fcm_dpo/delta": -0.12544870376586914, "fcm_dpo/margin": 29.303483963012695, "fcm_dpo/q_t": 0.32259464263916016, "grad_norm": 20.508411407470703, "learning_rate": 1.0391075790138232e-07, "logits/chosen": 0.5879669189453125, "logits/rejected": 0.4857994616031647, "logps/chosen": -94.51870727539062, "logps/ref_chosen": -53.192012786865234, "logps/ref_rejected": -81.83927154541016, "logps/rejected": -152.46945190429688, "loss": 0.919, "margin_dpo/margin_mean": 29.303482055664062, "margin_dpo/margin_std": 36.168243408203125, "step": 483 }, { "KL/chosen_KL_mean": -41.7369499206543, "KL/mean": -54.1743049621582, "KL/rejected_KL_mean": -66.61164855957031, "KL/std": 32.98217010498047, "epoch": 0.7316704459561603, "fcm_dpo/beta": 0.031263504177331924, "fcm_dpo/delta": 0.023358818143606186, "fcm_dpo/margin": 24.87470245361328, "fcm_dpo/q_t": 0.344391405582428, "grad_norm": 21.53194808959961, "learning_rate": 1.0283982962570681e-07, "logits/chosen": 0.5854651927947998, "logits/rejected": 0.5462943911552429, "logps/chosen": -99.50640869140625, "logps/ref_chosen": -57.76945877075195, "logps/ref_rejected": -71.6829833984375, "logps/rejected": -138.29464721679688, "loss": 0.9359, "margin_dpo/margin_mean": 24.87470245361328, "margin_dpo/margin_std": 30.0977783203125, "step": 484 }, { "KL/chosen_KL_mean": -41.71012878417969, "KL/mean": -54.20851516723633, "KL/rejected_KL_mean": -66.70690155029297, "KL/std": 35.231590270996094, "epoch": 0.7331821617535903, "fcm_dpo/beta": 0.030793113633990288, "fcm_dpo/delta": 0.02861456573009491, "fcm_dpo/margin": 24.996782302856445, "fcm_dpo/q_t": 0.3545028567314148, "grad_norm": 19.133703231811523, "learning_rate": 1.0177301773633992e-07, "logits/chosen": 0.5398764610290527, "logits/rejected": 0.5143481492996216, "logps/chosen": -98.34597778320312, "logps/ref_chosen": -56.63584899902344, "logps/ref_rejected": -70.85614013671875, "logps/rejected": -137.56304931640625, "loss": 1.004, "margin_dpo/margin_mean": 24.996782302856445, "margin_dpo/margin_std": 35.169715881347656, "step": 485 }, { "KL/chosen_KL_mean": -52.103946685791016, "KL/mean": -62.98194885253906, "KL/rejected_KL_mean": -73.85993957519531, "KL/std": 37.7567138671875, "epoch": 0.7346938775510204, "fcm_dpo/beta": 0.03189978003501892, "fcm_dpo/delta": 0.113531693816185, "fcm_dpo/margin": 21.755998611450195, "fcm_dpo/q_t": 0.3762109875679016, "grad_norm": 22.65691375732422, "learning_rate": 1.007103520743035e-07, "logits/chosen": 0.5532824993133545, "logits/rejected": 0.4355977177619934, "logps/chosen": -108.45097351074219, "logps/ref_chosen": -56.347023010253906, "logps/ref_rejected": -85.97221374511719, "logps/rejected": -159.8321533203125, "loss": 1.11, "margin_dpo/margin_mean": 21.756000518798828, "margin_dpo/margin_std": 39.52480697631836, "step": 486 }, { "KL/chosen_KL_mean": -46.250057220458984, "KL/mean": -58.29290771484375, "KL/rejected_KL_mean": -70.33575439453125, "KL/std": 31.51739501953125, "epoch": 0.7362055933484505, "fcm_dpo/beta": 0.03211412951350212, "fcm_dpo/delta": 0.028735807165503502, "fcm_dpo/margin": 24.085702896118164, "fcm_dpo/q_t": 0.3533576428890228, "grad_norm": 25.153072357177734, "learning_rate": 9.965186236464046e-08, "logits/chosen": 0.6023333668708801, "logits/rejected": 0.5408201217651367, "logps/chosen": -106.86727905273438, "logps/ref_chosen": -60.617218017578125, "logps/ref_rejected": -82.50975036621094, "logps/rejected": -152.8455047607422, "loss": 1.0122, "margin_dpo/margin_mean": 24.085702896118164, "margin_dpo/margin_std": 35.66450500488281, "step": 487 }, { "KL/chosen_KL_mean": -41.968387603759766, "KL/mean": -56.97295379638672, "KL/rejected_KL_mean": -71.97752380371094, "KL/std": 33.929840087890625, "epoch": 0.7377173091458806, "fcm_dpo/beta": 0.031626880168914795, "fcm_dpo/delta": -0.16451723873615265, "fcm_dpo/margin": 30.00912857055664, "fcm_dpo/q_t": 0.3240189850330353, "grad_norm": 22.58987045288086, "learning_rate": 9.859757821558337e-08, "logits/chosen": 0.5339430570602417, "logits/rejected": 0.46293365955352783, "logps/chosen": -105.07743835449219, "logps/ref_chosen": -63.10905075073242, "logps/ref_rejected": -82.49348449707031, "logps/rejected": -154.47100830078125, "loss": 0.888, "margin_dpo/margin_mean": 30.009132385253906, "margin_dpo/margin_std": 35.635520935058594, "step": 488 }, { "KL/chosen_KL_mean": -48.408485412597656, "KL/mean": -56.583763122558594, "KL/rejected_KL_mean": -64.75904846191406, "KL/std": 34.571693420410156, "epoch": 0.7392290249433107, "fcm_dpo/beta": 0.03243595361709595, "fcm_dpo/delta": 0.2853114604949951, "fcm_dpo/margin": 16.350563049316406, "fcm_dpo/q_t": 0.40409570932388306, "grad_norm": 25.71051025390625, "learning_rate": 9.754752911772615e-08, "logits/chosen": 0.49547716975212097, "logits/rejected": 0.4507462978363037, "logps/chosen": -113.39744567871094, "logps/ref_chosen": -64.98896026611328, "logps/ref_rejected": -84.39607238769531, "logps/rejected": -149.15512084960938, "loss": 1.2059, "margin_dpo/margin_mean": 16.350563049316406, "margin_dpo/margin_std": 35.75886154174805, "step": 489 }, { "KL/chosen_KL_mean": -42.95598602294922, "KL/mean": -53.87427520751953, "KL/rejected_KL_mean": -64.79255676269531, "KL/std": 36.25221633911133, "epoch": 0.7407407407407407, "fcm_dpo/beta": 0.03329470753669739, "fcm_dpo/delta": 0.07874666899442673, "fcm_dpo/margin": 21.836578369140625, "fcm_dpo/q_t": 0.3706052899360657, "grad_norm": 23.770259857177734, "learning_rate": 9.650174444319956e-08, "logits/chosen": 0.5940006375312805, "logits/rejected": 0.567367434501648, "logps/chosen": -104.86473083496094, "logps/ref_chosen": -61.90874481201172, "logps/ref_rejected": -70.58566284179688, "logps/rejected": -135.3782196044922, "loss": 1.2193, "margin_dpo/margin_mean": 21.836578369140625, "margin_dpo/margin_std": 44.323036193847656, "step": 490 }, { "KL/chosen_KL_mean": -42.657981872558594, "KL/mean": -54.71141052246094, "KL/rejected_KL_mean": -66.76484680175781, "KL/std": 33.00274658203125, "epoch": 0.7422524565381708, "fcm_dpo/beta": 0.03311682492494583, "fcm_dpo/delta": -0.0010885968804359436, "fcm_dpo/margin": 24.10685920715332, "fcm_dpo/q_t": 0.3481886386871338, "grad_norm": 19.895963668823242, "learning_rate": 9.546025344484868e-08, "logits/chosen": 0.4401285648345947, "logits/rejected": 0.38554367423057556, "logps/chosen": -98.13368225097656, "logps/ref_chosen": -55.47570037841797, "logps/ref_rejected": -78.70318603515625, "logps/rejected": -145.468017578125, "loss": 0.9984, "margin_dpo/margin_mean": 24.10685920715332, "margin_dpo/margin_std": 33.24077606201172, "step": 491 }, { "KL/chosen_KL_mean": -53.54511260986328, "KL/mean": -63.428951263427734, "KL/rejected_KL_mean": -73.31278991699219, "KL/std": 37.33125305175781, "epoch": 0.7437641723356009, "fcm_dpo/beta": 0.03454633057117462, "fcm_dpo/delta": 0.12079239636659622, "fcm_dpo/margin": 19.767681121826172, "fcm_dpo/q_t": 0.38081228733062744, "grad_norm": 26.302976608276367, "learning_rate": 9.442308525541589e-08, "logits/chosen": 0.49925363063812256, "logits/rejected": 0.42450058460235596, "logps/chosen": -120.83149719238281, "logps/ref_chosen": -67.28638458251953, "logps/ref_rejected": -82.78628540039062, "logps/rejected": -156.09909057617188, "loss": 1.1787, "margin_dpo/margin_mean": 19.767677307128906, "margin_dpo/margin_std": 38.34576416015625, "step": 492 }, { "KL/chosen_KL_mean": -41.301639556884766, "KL/mean": -55.703651428222656, "KL/rejected_KL_mean": -70.10565948486328, "KL/std": 35.31755447387695, "epoch": 0.745275888133031, "fcm_dpo/beta": 0.03364936634898186, "fcm_dpo/delta": -0.18719664216041565, "fcm_dpo/margin": 28.804014205932617, "fcm_dpo/q_t": 0.3174014687538147, "grad_norm": 20.67091178894043, "learning_rate": 9.339026888672468e-08, "logits/chosen": 0.48015329241752625, "logits/rejected": 0.3986372947692871, "logps/chosen": -97.22914123535156, "logps/ref_chosen": -55.92750549316406, "logps/ref_rejected": -79.12149810791016, "logps/rejected": -149.22715759277344, "loss": 0.9349, "margin_dpo/margin_mean": 28.80401611328125, "margin_dpo/margin_std": 37.06830596923828, "step": 493 }, { "KL/chosen_KL_mean": -43.42155838012695, "KL/mean": -53.814109802246094, "KL/rejected_KL_mean": -64.2066650390625, "KL/std": 34.440528869628906, "epoch": 0.7467876039304611, "fcm_dpo/beta": 0.033497944474220276, "fcm_dpo/delta": 0.1116424947977066, "fcm_dpo/margin": 20.78510284423828, "fcm_dpo/q_t": 0.37312787771224976, "grad_norm": 27.04566764831543, "learning_rate": 9.236183322886945e-08, "logits/chosen": 0.4167022109031677, "logits/rejected": 0.36079585552215576, "logps/chosen": -111.37567138671875, "logps/ref_chosen": -67.95410919189453, "logps/ref_rejected": -90.50865173339844, "logps/rejected": -154.71531677246094, "loss": 1.1999, "margin_dpo/margin_mean": 20.78510284423828, "margin_dpo/margin_std": 42.49012756347656, "step": 494 }, { "KL/chosen_KL_mean": -41.72477722167969, "KL/mean": -51.252037048339844, "KL/rejected_KL_mean": -60.779300689697266, "KL/std": 33.50157165527344, "epoch": 0.7482993197278912, "fcm_dpo/beta": 0.03483235090970993, "fcm_dpo/delta": 0.14320899546146393, "fcm_dpo/margin": 19.054513931274414, "fcm_dpo/q_t": 0.3821936249732971, "grad_norm": 25.66999626159668, "learning_rate": 9.133780704940594e-08, "logits/chosen": 0.6148525476455688, "logits/rejected": 0.5430424213409424, "logps/chosen": -94.35023498535156, "logps/ref_chosen": -52.62546157836914, "logps/ref_rejected": -72.06781005859375, "logps/rejected": -132.84710693359375, "loss": 1.1185, "margin_dpo/margin_mean": 19.054515838623047, "margin_dpo/margin_std": 34.809410095214844, "step": 495 }, { "KL/chosen_KL_mean": -46.68023681640625, "KL/mean": -58.58380889892578, "KL/rejected_KL_mean": -70.48738098144531, "KL/std": 34.729984283447266, "epoch": 0.7498110355253212, "fcm_dpo/beta": 0.034354642033576965, "fcm_dpo/delta": -0.021552205085754395, "fcm_dpo/margin": 23.80715560913086, "fcm_dpo/q_t": 0.3606174886226654, "grad_norm": 22.564855575561523, "learning_rate": 9.031821899254797e-08, "logits/chosen": 0.5376992225646973, "logits/rejected": 0.42464008927345276, "logps/chosen": -104.27755737304688, "logps/ref_chosen": -57.597320556640625, "logps/ref_rejected": -94.36127471923828, "logps/rejected": -164.84866333007812, "loss": 1.0669, "margin_dpo/margin_mean": 23.80715560913086, "margin_dpo/margin_std": 39.395347595214844, "step": 496 }, { "KL/chosen_KL_mean": -46.8216552734375, "KL/mean": -61.867698669433594, "KL/rejected_KL_mean": -76.91375732421875, "KL/std": 36.08230972290039, "epoch": 0.7513227513227513, "fcm_dpo/beta": 0.03335585445165634, "fcm_dpo/delta": -0.22783339023590088, "fcm_dpo/margin": 30.09210205078125, "fcm_dpo/q_t": 0.31446051597595215, "grad_norm": 19.67770004272461, "learning_rate": 8.930309757836516e-08, "logits/chosen": 0.5464165210723877, "logits/rejected": 0.5131070613861084, "logps/chosen": -119.61160278320312, "logps/ref_chosen": -72.78994750976562, "logps/ref_rejected": -89.48483276367188, "logps/rejected": -166.39859008789062, "loss": 0.9049, "margin_dpo/margin_mean": 30.09210205078125, "margin_dpo/margin_std": 37.14356231689453, "step": 497 }, { "KL/chosen_KL_mean": -42.5738525390625, "KL/mean": -56.43513870239258, "KL/rejected_KL_mean": -70.29643249511719, "KL/std": 35.45441436767578, "epoch": 0.7528344671201814, "fcm_dpo/beta": 0.03254364803433418, "fcm_dpo/delta": -0.11236497759819031, "fcm_dpo/margin": 27.72256851196289, "fcm_dpo/q_t": 0.3303517699241638, "grad_norm": 22.639673233032227, "learning_rate": 8.829247120198563e-08, "logits/chosen": 0.5322725176811218, "logits/rejected": 0.49616390466690063, "logps/chosen": -110.9395751953125, "logps/ref_chosen": -68.36572265625, "logps/ref_rejected": -71.28846740722656, "logps/rejected": -141.58489990234375, "loss": 0.9052, "margin_dpo/margin_mean": 27.722570419311523, "margin_dpo/margin_std": 33.93904113769531, "step": 498 }, { "KL/chosen_KL_mean": -40.05500411987305, "KL/mean": -53.02268981933594, "KL/rejected_KL_mean": -65.9903793334961, "KL/std": 34.889076232910156, "epoch": 0.7543461829176115, "fcm_dpo/beta": 0.03228309750556946, "fcm_dpo/delta": -0.040800243616104126, "fcm_dpo/margin": 25.935379028320312, "fcm_dpo/q_t": 0.3481753468513489, "grad_norm": 23.10430145263672, "learning_rate": 8.728636813280163e-08, "logits/chosen": 0.5184221267700195, "logits/rejected": 0.4499463438987732, "logps/chosen": -101.96383666992188, "logps/ref_chosen": -61.90882873535156, "logps/ref_rejected": -91.9411392211914, "logps/rejected": -157.9315185546875, "loss": 1.0679, "margin_dpo/margin_mean": 25.93537712097168, "margin_dpo/margin_std": 41.14997100830078, "step": 499 }, { "KL/chosen_KL_mean": -43.87847137451172, "KL/mean": -56.561927795410156, "KL/rejected_KL_mean": -69.24537658691406, "KL/std": 33.54810333251953, "epoch": 0.7558578987150416, "fcm_dpo/beta": 0.03199248015880585, "fcm_dpo/delta": -0.012633852660655975, "fcm_dpo/margin": 25.366901397705078, "fcm_dpo/q_t": 0.34740620851516724, "grad_norm": 24.847909927368164, "learning_rate": 8.628481651367875e-08, "logits/chosen": 0.4715893566608429, "logits/rejected": 0.46398016810417175, "logps/chosen": -114.10430145263672, "logps/ref_chosen": -70.225830078125, "logps/ref_rejected": -71.72203063964844, "logps/rejected": -140.9674072265625, "loss": 1.0277, "margin_dpo/margin_mean": 25.366901397705078, "margin_dpo/margin_std": 37.98773956298828, "step": 500 }, { "KL/chosen_KL_mean": -43.072383880615234, "KL/mean": -54.34223175048828, "KL/rejected_KL_mean": -65.61207580566406, "KL/std": 34.32850646972656, "epoch": 0.7573696145124716, "fcm_dpo/beta": 0.032621338963508606, "fcm_dpo/delta": 0.06773628294467926, "fcm_dpo/margin": 22.539688110351562, "fcm_dpo/q_t": 0.35141244530677795, "grad_norm": 17.035171508789062, "learning_rate": 8.528784436016878e-08, "logits/chosen": 0.5009163618087769, "logits/rejected": 0.494549036026001, "logps/chosen": -107.67118835449219, "logps/ref_chosen": -64.59880828857422, "logps/ref_rejected": -70.59329223632812, "logps/rejected": -136.20535278320312, "loss": 0.9471, "margin_dpo/margin_mean": 22.539688110351562, "margin_dpo/margin_std": 26.915557861328125, "step": 501 }, { "KL/chosen_KL_mean": -41.572818756103516, "KL/mean": -53.240657806396484, "KL/rejected_KL_mean": -64.90849304199219, "KL/std": 33.89706039428711, "epoch": 0.7588813303099018, "fcm_dpo/beta": 0.03277207911014557, "fcm_dpo/delta": 0.03764596953988075, "fcm_dpo/margin": 23.335674285888672, "fcm_dpo/q_t": 0.35313090682029724, "grad_norm": 23.165653228759766, "learning_rate": 8.4295479559726e-08, "logits/chosen": 0.5166738033294678, "logits/rejected": 0.46954381465911865, "logps/chosen": -107.0394515991211, "logps/ref_chosen": -65.46662902832031, "logps/ref_rejected": -90.22233581542969, "logps/rejected": -155.13082885742188, "loss": 0.9621, "margin_dpo/margin_mean": 23.335674285888672, "margin_dpo/margin_std": 31.047378540039062, "step": 502 }, { "KL/chosen_KL_mean": -38.187461853027344, "KL/mean": -50.40232467651367, "KL/rejected_KL_mean": -62.617183685302734, "KL/std": 34.579429626464844, "epoch": 0.7603930461073318, "fcm_dpo/beta": 0.032632194459438324, "fcm_dpo/delta": 0.0029222238808870316, "fcm_dpo/margin": 24.42971420288086, "fcm_dpo/q_t": 0.3498132824897766, "grad_norm": 19.567941665649414, "learning_rate": 8.330774987092712e-08, "logits/chosen": 0.5233071446418762, "logits/rejected": 0.5128970146179199, "logps/chosen": -90.02223205566406, "logps/ref_chosen": -51.83476257324219, "logps/ref_rejected": -57.62522506713867, "logps/rejected": -120.2424087524414, "loss": 1.0475, "margin_dpo/margin_mean": 24.42971420288086, "margin_dpo/margin_std": 37.409400939941406, "step": 503 }, { "KL/chosen_KL_mean": -39.26737976074219, "KL/mean": -55.71110534667969, "KL/rejected_KL_mean": -72.15483093261719, "KL/std": 33.35061264038086, "epoch": 0.7619047619047619, "fcm_dpo/beta": 0.03167734667658806, "fcm_dpo/delta": -0.2690557837486267, "fcm_dpo/margin": 32.88744354248047, "fcm_dpo/q_t": 0.29890376329421997, "grad_norm": 23.10883140563965, "learning_rate": 8.232468292269479e-08, "logits/chosen": 0.4726182222366333, "logits/rejected": 0.44822973012924194, "logps/chosen": -107.9185791015625, "logps/ref_chosen": -68.65119934082031, "logps/ref_rejected": -77.91394805908203, "logps/rejected": -150.06878662109375, "loss": 0.7965, "margin_dpo/margin_mean": 32.88744354248047, "margin_dpo/margin_std": 32.372901916503906, "step": 504 }, { "KL/chosen_KL_mean": -41.26458740234375, "KL/mean": -51.810516357421875, "KL/rejected_KL_mean": -62.35645294189453, "KL/std": 36.557350158691406, "epoch": 0.763416477702192, "fcm_dpo/beta": 0.03131503239274025, "fcm_dpo/delta": 0.14808888733386993, "fcm_dpo/margin": 21.09187126159668, "fcm_dpo/q_t": 0.3847939372062683, "grad_norm": 23.190839767456055, "learning_rate": 8.134630621352483e-08, "logits/chosen": 0.5082249641418457, "logits/rejected": 0.4687017798423767, "logps/chosen": -101.26343536376953, "logps/ref_chosen": -59.99884796142578, "logps/ref_rejected": -76.88048553466797, "logps/rejected": -139.2369384765625, "loss": 1.1594, "margin_dpo/margin_mean": 21.091869354248047, "margin_dpo/margin_std": 40.211585998535156, "step": 505 }, { "KL/chosen_KL_mean": -38.800819396972656, "KL/mean": -49.737892150878906, "KL/rejected_KL_mean": -60.674964904785156, "KL/std": 32.124107360839844, "epoch": 0.764928193499622, "fcm_dpo/beta": 0.03223157674074173, "fcm_dpo/delta": 0.10202737152576447, "fcm_dpo/margin": 21.874143600463867, "fcm_dpo/q_t": 0.3677794635295868, "grad_norm": 22.604772567749023, "learning_rate": 8.037264711071698e-08, "logits/chosen": 0.5274264812469482, "logits/rejected": 0.5046772956848145, "logps/chosen": -108.87211608886719, "logps/ref_chosen": -70.07130432128906, "logps/ref_rejected": -82.03775024414062, "logps/rejected": -142.71270751953125, "loss": 1.1034, "margin_dpo/margin_mean": 21.874143600463867, "margin_dpo/margin_std": 38.15148162841797, "step": 506 }, { "KL/chosen_KL_mean": -43.51123046875, "KL/mean": -56.59437942504883, "KL/rejected_KL_mean": -69.67752075195312, "KL/std": 34.81589126586914, "epoch": 0.7664399092970522, "fcm_dpo/beta": 0.032264385372400284, "fcm_dpo/delta": -0.04865808039903641, "fcm_dpo/margin": 26.166301727294922, "fcm_dpo/q_t": 0.35244131088256836, "grad_norm": 20.77465057373047, "learning_rate": 7.940373284960933e-08, "logits/chosen": 0.5034874081611633, "logits/rejected": 0.45192790031433105, "logps/chosen": -115.51826477050781, "logps/ref_chosen": -72.00703430175781, "logps/ref_rejected": -93.94987487792969, "logps/rejected": -163.6273956298828, "loss": 1.0071, "margin_dpo/margin_mean": 26.166301727294922, "margin_dpo/margin_std": 38.84321594238281, "step": 507 }, { "KL/chosen_KL_mean": -39.335166931152344, "KL/mean": -53.35478591918945, "KL/rejected_KL_mean": -67.37440490722656, "KL/std": 35.52327346801758, "epoch": 0.7679516250944822, "fcm_dpo/beta": 0.032064348459243774, "fcm_dpo/delta": -0.10965946316719055, "fcm_dpo/margin": 28.039234161376953, "fcm_dpo/q_t": 0.33421868085861206, "grad_norm": 20.46658706665039, "learning_rate": 7.843959053281663e-08, "logits/chosen": 0.4993664622306824, "logits/rejected": 0.372380793094635, "logps/chosen": -99.55509185791016, "logps/ref_chosen": -60.21992492675781, "logps/ref_rejected": -95.9200668334961, "logps/rejected": -163.29446411132812, "loss": 0.9376, "margin_dpo/margin_mean": 28.039234161376953, "margin_dpo/margin_std": 35.876731872558594, "step": 508 }, { "KL/chosen_KL_mean": -42.130619049072266, "KL/mean": -54.088356018066406, "KL/rejected_KL_mean": -66.04609680175781, "KL/std": 34.5006217956543, "epoch": 0.7694633408919124, "fcm_dpo/beta": 0.03165077418088913, "fcm_dpo/delta": 0.04634883999824524, "fcm_dpo/margin": 23.91547966003418, "fcm_dpo/q_t": 0.35058048367500305, "grad_norm": 21.03251838684082, "learning_rate": 7.748024712947204e-08, "logits/chosen": 0.46801474690437317, "logits/rejected": 0.44395408034324646, "logps/chosen": -108.40078735351562, "logps/ref_chosen": -66.27017211914062, "logps/ref_rejected": -71.73065185546875, "logps/rejected": -137.77674865722656, "loss": 0.9953, "margin_dpo/margin_mean": 23.915477752685547, "margin_dpo/margin_std": 33.073978424072266, "step": 509 }, { "KL/chosen_KL_mean": -41.646202087402344, "KL/mean": -55.19512176513672, "KL/rejected_KL_mean": -68.74403381347656, "KL/std": 33.61712646484375, "epoch": 0.7709750566893424, "fcm_dpo/beta": 0.03142780438065529, "fcm_dpo/delta": -0.05741361528635025, "fcm_dpo/margin": 27.09783172607422, "fcm_dpo/q_t": 0.34271568059921265, "grad_norm": 19.332889556884766, "learning_rate": 7.652572947447272e-08, "logits/chosen": 0.6219863891601562, "logits/rejected": 0.5252499580383301, "logps/chosen": -95.19107818603516, "logps/ref_chosen": -53.54487609863281, "logps/ref_rejected": -91.36648559570312, "logps/rejected": -160.11053466796875, "loss": 0.9856, "margin_dpo/margin_mean": 27.09783172607422, "margin_dpo/margin_std": 38.01484680175781, "step": 510 }, { "KL/chosen_KL_mean": -42.845680236816406, "KL/mean": -58.7375373840332, "KL/rejected_KL_mean": -74.62939453125, "KL/std": 35.452720642089844, "epoch": 0.7724867724867724, "fcm_dpo/beta": 0.030704926699399948, "fcm_dpo/delta": -0.19456720352172852, "fcm_dpo/margin": 31.78371238708496, "fcm_dpo/q_t": 0.3209506869316101, "grad_norm": 19.716928482055664, "learning_rate": 7.557606426772961e-08, "logits/chosen": 0.5421375036239624, "logits/rejected": 0.48624932765960693, "logps/chosen": -98.6900634765625, "logps/ref_chosen": -55.844383239746094, "logps/ref_rejected": -86.49819946289062, "logps/rejected": -161.12759399414062, "loss": 0.9377, "margin_dpo/margin_mean": 31.783714294433594, "margin_dpo/margin_std": 40.96788787841797, "step": 511 }, { "KL/chosen_KL_mean": -40.811737060546875, "KL/mean": -52.81915283203125, "KL/rejected_KL_mean": -64.82656860351562, "KL/std": 32.51373291015625, "epoch": 0.7739984882842026, "fcm_dpo/beta": 0.030653230845928192, "fcm_dpo/delta": 0.06883647292852402, "fcm_dpo/margin": 24.014835357666016, "fcm_dpo/q_t": 0.3677193522453308, "grad_norm": 19.665395736694336, "learning_rate": 7.463127807341966e-08, "logits/chosen": 0.4046107530593872, "logits/rejected": 0.3887116312980652, "logps/chosen": -102.46477508544922, "logps/ref_chosen": -61.653038024902344, "logps/ref_rejected": -72.83148193359375, "logps/rejected": -137.65805053710938, "loss": 1.0805, "margin_dpo/margin_mean": 24.014835357666016, "margin_dpo/margin_std": 40.261390686035156, "step": 512 }, { "KL/chosen_KL_mean": -37.0535888671875, "KL/mean": -50.049591064453125, "KL/rejected_KL_mean": -63.045597076416016, "KL/std": 34.30278015136719, "epoch": 0.7755102040816326, "fcm_dpo/beta": 0.030527032911777496, "fcm_dpo/delta": 0.005987279117107391, "fcm_dpo/margin": 25.99200439453125, "fcm_dpo/q_t": 0.3471200466156006, "grad_norm": 17.770233154296875, "learning_rate": 7.369139731924401e-08, "logits/chosen": 0.6464536786079407, "logits/rejected": 0.5946289300918579, "logps/chosen": -87.90615844726562, "logps/ref_chosen": -50.85256576538086, "logps/ref_rejected": -69.21754455566406, "logps/rejected": -132.2631378173828, "loss": 0.9392, "margin_dpo/margin_mean": 25.99200439453125, "margin_dpo/margin_std": 33.05658721923828, "step": 513 }, { "KL/chosen_KL_mean": -41.696224212646484, "KL/mean": -56.08380889892578, "KL/rejected_KL_mean": -70.47138977050781, "KL/std": 33.583072662353516, "epoch": 0.7770219198790628, "fcm_dpo/beta": 0.03049997240304947, "fcm_dpo/delta": -0.08509883284568787, "fcm_dpo/margin": 28.775178909301758, "fcm_dpo/q_t": 0.32821977138519287, "grad_norm": 18.91166877746582, "learning_rate": 7.275644829568747e-08, "logits/chosen": 0.5680727362632751, "logits/rejected": 0.5272949934005737, "logps/chosen": -111.08116149902344, "logps/ref_chosen": -69.38493347167969, "logps/ref_rejected": -83.32447814941406, "logps/rejected": -153.79586791992188, "loss": 0.922, "margin_dpo/margin_mean": 28.77518081665039, "margin_dpo/margin_std": 35.041664123535156, "step": 514 }, { "KL/chosen_KL_mean": -47.496299743652344, "KL/mean": -60.2870979309082, "KL/rejected_KL_mean": -73.07791137695312, "KL/std": 33.58501434326172, "epoch": 0.7785336356764928, "fcm_dpo/beta": 0.030235309153795242, "fcm_dpo/delta": 0.028453219681978226, "fcm_dpo/margin": 25.581607818603516, "fcm_dpo/q_t": 0.35126420855522156, "grad_norm": 21.126333236694336, "learning_rate": 7.182645715528435e-08, "logits/chosen": 0.5368679761886597, "logits/rejected": 0.4550856649875641, "logps/chosen": -101.18333435058594, "logps/ref_chosen": -53.687034606933594, "logps/ref_rejected": -83.59614562988281, "logps/rejected": -156.67404174804688, "loss": 0.9922, "margin_dpo/margin_mean": 25.581605911254883, "margin_dpo/margin_std": 35.4997444152832, "step": 515 }, { "KL/chosen_KL_mean": -39.445045471191406, "KL/mean": -50.16015625, "KL/rejected_KL_mean": -60.875267028808594, "KL/std": 32.9947395324707, "epoch": 0.780045351473923, "fcm_dpo/beta": 0.031198721379041672, "fcm_dpo/delta": 0.139729306101799, "fcm_dpo/margin": 21.430225372314453, "fcm_dpo/q_t": 0.36643847823143005, "grad_norm": 22.579246520996094, "learning_rate": 7.090144991188568e-08, "logits/chosen": 0.48671144247055054, "logits/rejected": 0.43809783458709717, "logps/chosen": -96.34677124023438, "logps/ref_chosen": -56.9017219543457, "logps/ref_rejected": -67.83477783203125, "logps/rejected": -128.71005249023438, "loss": 1.0848, "margin_dpo/margin_mean": 21.430227279663086, "margin_dpo/margin_std": 35.83974838256836, "step": 516 }, { "KL/chosen_KL_mean": -48.5676155090332, "KL/mean": -58.601890563964844, "KL/rejected_KL_mean": -68.63616943359375, "KL/std": 33.302223205566406, "epoch": 0.781557067271353, "fcm_dpo/beta": 0.03218144178390503, "fcm_dpo/delta": 0.16378606855869293, "fcm_dpo/margin": 20.068546295166016, "fcm_dpo/q_t": 0.3800460696220398, "grad_norm": 23.016653060913086, "learning_rate": 6.998145243993284e-08, "logits/chosen": 0.5580442547798157, "logits/rejected": 0.5505795478820801, "logps/chosen": -110.34275817871094, "logps/ref_chosen": -61.775142669677734, "logps/ref_rejected": -62.88270950317383, "logps/rejected": -131.51889038085938, "loss": 1.0889, "margin_dpo/margin_mean": 20.068546295166016, "margin_dpo/margin_std": 34.71467590332031, "step": 517 }, { "KL/chosen_KL_mean": -38.425636291503906, "KL/mean": -49.03041458129883, "KL/rejected_KL_mean": -59.635189056396484, "KL/std": 33.62690353393555, "epoch": 0.783068783068783, "fcm_dpo/beta": 0.032912157475948334, "fcm_dpo/delta": 0.10931538045406342, "fcm_dpo/margin": 21.209548950195312, "fcm_dpo/q_t": 0.3719986081123352, "grad_norm": 20.80961036682129, "learning_rate": 6.906649047373245e-08, "logits/chosen": 0.5091816186904907, "logits/rejected": 0.4563264846801758, "logps/chosen": -100.45086669921875, "logps/ref_chosen": -62.02523422241211, "logps/ref_rejected": -79.06085205078125, "logps/rejected": -138.696044921875, "loss": 1.0609, "margin_dpo/margin_mean": 21.209548950195312, "margin_dpo/margin_std": 34.63611602783203, "step": 518 }, { "KL/chosen_KL_mean": -47.89688491821289, "KL/mean": -56.045166015625, "KL/rejected_KL_mean": -64.19346618652344, "KL/std": 33.93360900878906, "epoch": 0.7845804988662132, "fcm_dpo/beta": 0.03449524939060211, "fcm_dpo/delta": 0.24971872568130493, "fcm_dpo/margin": 16.296571731567383, "fcm_dpo/q_t": 0.4011024832725525, "grad_norm": 32.703617095947266, "learning_rate": 6.815658960673781e-08, "logits/chosen": 0.5393502712249756, "logits/rejected": 0.4814758896827698, "logps/chosen": -109.50325012207031, "logps/ref_chosen": -61.60636901855469, "logps/ref_rejected": -74.50727844238281, "logps/rejected": -138.70074462890625, "loss": 1.3142, "margin_dpo/margin_mean": 16.29656982421875, "margin_dpo/margin_std": 41.276824951171875, "step": 519 }, { "KL/chosen_KL_mean": -42.39877700805664, "KL/mean": -52.96178436279297, "KL/rejected_KL_mean": -63.5247917175293, "KL/std": 33.05720138549805, "epoch": 0.7860922146636432, "fcm_dpo/beta": 0.03477100655436516, "fcm_dpo/delta": 0.06951478123664856, "fcm_dpo/margin": 21.126018524169922, "fcm_dpo/q_t": 0.3603675365447998, "grad_norm": 26.019474029541016, "learning_rate": 6.725177529083209e-08, "logits/chosen": 0.5982969999313354, "logits/rejected": 0.5375903248786926, "logps/chosen": -105.27220916748047, "logps/ref_chosen": -62.87343215942383, "logps/ref_rejected": -76.505615234375, "logps/rejected": -140.0303955078125, "loss": 1.036, "margin_dpo/margin_mean": 21.126018524169922, "margin_dpo/margin_std": 31.89874839782715, "step": 520 }, { "KL/chosen_KL_mean": -43.5118522644043, "KL/mean": -56.493614196777344, "KL/rejected_KL_mean": -69.47537994384766, "KL/std": 32.319976806640625, "epoch": 0.7876039304610734, "fcm_dpo/beta": 0.03475981950759888, "fcm_dpo/delta": -0.11252060532569885, "fcm_dpo/margin": 25.96352767944336, "fcm_dpo/q_t": 0.3285518288612366, "grad_norm": 26.865089416503906, "learning_rate": 6.63520728356167e-08, "logits/chosen": 0.42334693670272827, "logits/rejected": 0.3446962237358093, "logps/chosen": -107.7185287475586, "logps/ref_chosen": -64.20668029785156, "logps/ref_rejected": -92.28083038330078, "logps/rejected": -161.75621032714844, "loss": 0.9379, "margin_dpo/margin_mean": 25.96352767944336, "margin_dpo/margin_std": 33.507266998291016, "step": 521 }, { "KL/chosen_KL_mean": -41.547828674316406, "KL/mean": -52.55769348144531, "KL/rejected_KL_mean": -63.56755828857422, "KL/std": 35.37586975097656, "epoch": 0.7891156462585034, "fcm_dpo/beta": 0.03454647958278656, "fcm_dpo/delta": 0.04247668385505676, "fcm_dpo/margin": 22.01972770690918, "fcm_dpo/q_t": 0.366945743560791, "grad_norm": 27.949600219726562, "learning_rate": 6.545750740770336e-08, "logits/chosen": 0.5129572749137878, "logits/rejected": 0.5026385188102722, "logps/chosen": -99.91755676269531, "logps/ref_chosen": -58.369720458984375, "logps/ref_rejected": -68.79248046875, "logps/rejected": -132.36004638671875, "loss": 1.1764, "margin_dpo/margin_mean": 22.019729614257812, "margin_dpo/margin_std": 42.28102111816406, "step": 522 }, { "KL/chosen_KL_mean": -45.6678581237793, "KL/mean": -58.220497131347656, "KL/rejected_KL_mean": -70.77313232421875, "KL/std": 33.746395111083984, "epoch": 0.7906273620559335, "fcm_dpo/beta": 0.03418704867362976, "fcm_dpo/delta": -0.0648900717496872, "fcm_dpo/margin": 25.105276107788086, "fcm_dpo/q_t": 0.3337825834751129, "grad_norm": 30.4615478515625, "learning_rate": 6.456810403001012e-08, "logits/chosen": 0.5436546802520752, "logits/rejected": 0.42364761233329773, "logps/chosen": -111.38109588623047, "logps/ref_chosen": -65.71324157714844, "logps/ref_rejected": -91.98896789550781, "logps/rejected": -162.76211547851562, "loss": 1.0339, "margin_dpo/margin_mean": 25.105276107788086, "margin_dpo/margin_std": 37.20182800292969, "step": 523 }, { "KL/chosen_KL_mean": -36.952911376953125, "KL/mean": -47.42578887939453, "KL/rejected_KL_mean": -57.89865493774414, "KL/std": 30.665462493896484, "epoch": 0.7921390778533636, "fcm_dpo/beta": 0.03484393283724785, "fcm_dpo/delta": 0.07469457387924194, "fcm_dpo/margin": 20.945743560791016, "fcm_dpo/q_t": 0.35503047704696655, "grad_norm": 23.227760314941406, "learning_rate": 6.368388758106134e-08, "logits/chosen": 0.4536820650100708, "logits/rejected": 0.4314419627189636, "logps/chosen": -113.30416107177734, "logps/ref_chosen": -76.35124969482422, "logps/ref_rejected": -89.96072387695312, "logps/rejected": -147.859375, "loss": 1.0179, "margin_dpo/margin_mean": 20.945743560791016, "margin_dpo/margin_std": 30.672534942626953, "step": 524 }, { "KL/chosen_KL_mean": -41.9482307434082, "KL/mean": -50.95294189453125, "KL/rejected_KL_mean": -59.95765686035156, "KL/std": 30.69168472290039, "epoch": 0.7936507936507936, "fcm_dpo/beta": 0.035696301609277725, "fcm_dpo/delta": 0.16779160499572754, "fcm_dpo/margin": 18.009424209594727, "fcm_dpo/q_t": 0.3807103633880615, "grad_norm": 26.58245277404785, "learning_rate": 6.280488279429185e-08, "logits/chosen": 0.35608696937561035, "logits/rejected": 0.34583795070648193, "logps/chosen": -117.44401550292969, "logps/ref_chosen": -75.49578857421875, "logps/ref_rejected": -84.04852294921875, "logps/rejected": -144.0061798095703, "loss": 1.1024, "margin_dpo/margin_mean": 18.00942611694336, "margin_dpo/margin_std": 31.897258758544922, "step": 525 }, { "KL/chosen_KL_mean": -44.732521057128906, "KL/mean": -54.399269104003906, "KL/rejected_KL_mean": -64.06602478027344, "KL/std": 31.663698196411133, "epoch": 0.7951625094482238, "fcm_dpo/beta": 0.03591318428516388, "fcm_dpo/delta": 0.10863810777664185, "fcm_dpo/margin": 19.333499908447266, "fcm_dpo/q_t": 0.3647363781929016, "grad_norm": 24.61683464050293, "learning_rate": 6.193111425735515e-08, "logits/chosen": 0.5265854597091675, "logits/rejected": 0.4591507315635681, "logps/chosen": -106.02494049072266, "logps/ref_chosen": -61.29241943359375, "logps/ref_rejected": -82.47763061523438, "logps/rejected": -146.54364013671875, "loss": 1.0761, "margin_dpo/margin_mean": 19.3335018157959, "margin_dpo/margin_std": 30.015331268310547, "step": 526 }, { "KL/chosen_KL_mean": -46.88569641113281, "KL/mean": -54.68010711669922, "KL/rejected_KL_mean": -62.474517822265625, "KL/std": 31.084400177001953, "epoch": 0.7966742252456538, "fcm_dpo/beta": 0.03820966184139252, "fcm_dpo/delta": 0.21504317224025726, "fcm_dpo/margin": 15.588825225830078, "fcm_dpo/q_t": 0.3955618739128113, "grad_norm": 30.495433807373047, "learning_rate": 6.106260641143546e-08, "logits/chosen": 0.6129889488220215, "logits/rejected": 0.5357133746147156, "logps/chosen": -108.35832214355469, "logps/ref_chosen": -61.472625732421875, "logps/ref_rejected": -90.52831268310547, "logps/rejected": -153.00283813476562, "loss": 1.2257, "margin_dpo/margin_mean": 15.588825225830078, "margin_dpo/margin_std": 34.68670654296875, "step": 527 }, { "KL/chosen_KL_mean": -43.2332763671875, "KL/mean": -51.08110046386719, "KL/rejected_KL_mean": -58.92891311645508, "KL/std": 31.543315887451172, "epoch": 0.7981859410430839, "fcm_dpo/beta": 0.0391681045293808, "fcm_dpo/delta": 0.19740989804267883, "fcm_dpo/margin": 15.695642471313477, "fcm_dpo/q_t": 0.3858627676963806, "grad_norm": 34.73171615600586, "learning_rate": 6.019938355056422e-08, "logits/chosen": 0.43448758125305176, "logits/rejected": 0.35089200735092163, "logps/chosen": -102.0252914428711, "logps/ref_chosen": -58.792015075683594, "logps/ref_rejected": -71.82516479492188, "logps/rejected": -130.7540740966797, "loss": 1.2577, "margin_dpo/margin_mean": 15.695640563964844, "margin_dpo/margin_std": 35.330623626708984, "step": 528 }, { "KL/chosen_KL_mean": -40.24607849121094, "KL/mean": -58.42034912109375, "KL/rejected_KL_mean": -76.59461975097656, "KL/std": 35.38734436035156, "epoch": 0.799697656840514, "fcm_dpo/beta": 0.03599797189235687, "fcm_dpo/delta": -0.5999960899353027, "fcm_dpo/margin": 36.348541259765625, "fcm_dpo/q_t": 0.2665252983570099, "grad_norm": 20.099990844726562, "learning_rate": 5.934146982094049e-08, "logits/chosen": 0.4503468871116638, "logits/rejected": 0.39226827025413513, "logps/chosen": -95.3170394897461, "logps/ref_chosen": -55.070960998535156, "logps/ref_rejected": -75.44007873535156, "logps/rejected": -152.03469848632812, "loss": 0.7776, "margin_dpo/margin_mean": 36.348541259765625, "margin_dpo/margin_std": 36.78180694580078, "step": 529 }, { "KL/chosen_KL_mean": -41.45820999145508, "KL/mean": -52.497615814208984, "KL/rejected_KL_mean": -63.537017822265625, "KL/std": 33.97320556640625, "epoch": 0.8012093726379441, "fcm_dpo/beta": 0.03562724217772484, "fcm_dpo/delta": 0.014175062999129295, "fcm_dpo/margin": 22.078805923461914, "fcm_dpo/q_t": 0.3531653881072998, "grad_norm": 22.5502986907959, "learning_rate": 5.848888922025552e-08, "logits/chosen": 0.5277613401412964, "logits/rejected": 0.485904723405838, "logps/chosen": -98.2020263671875, "logps/ref_chosen": -56.743812561035156, "logps/ref_rejected": -76.6692123413086, "logps/rejected": -140.20623779296875, "loss": 0.9715, "margin_dpo/margin_mean": 22.078805923461914, "margin_dpo/margin_std": 29.655553817749023, "step": 530 }, { "KL/chosen_KL_mean": -42.525909423828125, "KL/mean": -52.817569732666016, "KL/rejected_KL_mean": -63.109230041503906, "KL/std": 33.2888069152832, "epoch": 0.8027210884353742, "fcm_dpo/beta": 0.03640486299991608, "fcm_dpo/delta": 0.04753082990646362, "fcm_dpo/margin": 20.583328247070312, "fcm_dpo/q_t": 0.3651937246322632, "grad_norm": 27.369985580444336, "learning_rate": 5.7641665597021435e-08, "logits/chosen": 0.5008730888366699, "logits/rejected": 0.4344239830970764, "logps/chosen": -93.64236450195312, "logps/ref_chosen": -51.116455078125, "logps/ref_rejected": -79.52884674072266, "logps/rejected": -142.63807678222656, "loss": 1.1001, "margin_dpo/margin_mean": 20.58332633972168, "margin_dpo/margin_std": 34.31003189086914, "step": 531 }, { "KL/chosen_KL_mean": -48.05036926269531, "KL/mean": -60.802268981933594, "KL/rejected_KL_mean": -73.55416107177734, "KL/std": 31.71292495727539, "epoch": 0.8042328042328042, "fcm_dpo/beta": 0.03519435226917267, "fcm_dpo/delta": -0.10766403377056122, "fcm_dpo/margin": 25.503793716430664, "fcm_dpo/q_t": 0.33816105127334595, "grad_norm": 22.831806182861328, "learning_rate": 5.679982264990424e-08, "logits/chosen": 0.45231497287750244, "logits/rejected": 0.3992037773132324, "logps/chosen": -106.330322265625, "logps/ref_chosen": -58.279945373535156, "logps/ref_rejected": -78.05426788330078, "logps/rejected": -151.60842895507812, "loss": 1.0079, "margin_dpo/margin_mean": 25.503793716430664, "margin_dpo/margin_std": 37.307273864746094, "step": 532 }, { "KL/chosen_KL_mean": -34.005062103271484, "KL/mean": -45.2939567565918, "KL/rejected_KL_mean": -56.582855224609375, "KL/std": 33.391746520996094, "epoch": 0.8057445200302343, "fcm_dpo/beta": 0.03524235635995865, "fcm_dpo/delta": 0.0043890513479709625, "fcm_dpo/margin": 22.57779884338379, "fcm_dpo/q_t": 0.3489721715450287, "grad_norm": 18.338743209838867, "learning_rate": 5.596338392706076e-08, "logits/chosen": 0.6012002229690552, "logits/rejected": 0.5376021265983582, "logps/chosen": -90.4230728149414, "logps/ref_chosen": -56.41801071166992, "logps/ref_rejected": -73.89324951171875, "logps/rejected": -130.47610473632812, "loss": 1.0184, "margin_dpo/margin_mean": 22.577800750732422, "margin_dpo/margin_std": 33.613582611083984, "step": 533 }, { "KL/chosen_KL_mean": -38.655921936035156, "KL/mean": -51.250999450683594, "KL/rejected_KL_mean": -63.84608840942383, "KL/std": 32.82971954345703, "epoch": 0.8072562358276644, "fcm_dpo/beta": 0.03448425233364105, "fcm_dpo/delta": -0.07676675170660019, "fcm_dpo/margin": 25.190170288085938, "fcm_dpo/q_t": 0.3423423171043396, "grad_norm": 20.127193450927734, "learning_rate": 5.513237282548033e-08, "logits/chosen": 0.5142478942871094, "logits/rejected": 0.4719163179397583, "logps/chosen": -99.40460968017578, "logps/ref_chosen": -60.748687744140625, "logps/ref_rejected": -73.8623046875, "logps/rejected": -137.70838928222656, "loss": 1.0206, "margin_dpo/margin_mean": 25.190170288085938, "margin_dpo/margin_std": 37.62242126464844, "step": 534 }, { "KL/chosen_KL_mean": -46.664085388183594, "KL/mean": -57.11603546142578, "KL/rejected_KL_mean": -67.56797790527344, "KL/std": 35.46537780761719, "epoch": 0.8087679516250945, "fcm_dpo/beta": 0.035117894411087036, "fcm_dpo/delta": 0.07030545920133591, "fcm_dpo/margin": 20.903888702392578, "fcm_dpo/q_t": 0.3673914074897766, "grad_norm": 25.055700302124023, "learning_rate": 5.430681259032957e-08, "logits/chosen": 0.4137476086616516, "logits/rejected": 0.3489595055580139, "logps/chosen": -108.30149841308594, "logps/ref_chosen": -61.637413024902344, "logps/ref_rejected": -80.93138885498047, "logps/rejected": -148.49935913085938, "loss": 1.1134, "margin_dpo/margin_mean": 20.90389060974121, "margin_dpo/margin_std": 37.192806243896484, "step": 535 }, { "KL/chosen_KL_mean": -39.921539306640625, "KL/mean": -54.335689544677734, "KL/rejected_KL_mean": -68.74983978271484, "KL/std": 34.82649612426758, "epoch": 0.8102796674225246, "fcm_dpo/beta": 0.033950429409742355, "fcm_dpo/delta": -0.19902583956718445, "fcm_dpo/margin": 28.82830047607422, "fcm_dpo/q_t": 0.31275475025177, "grad_norm": 17.977373123168945, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 0.5412774682044983, "logits/rejected": 0.45443296432495117, "logps/chosen": -91.81051635742188, "logps/ref_chosen": -51.88897705078125, "logps/ref_rejected": -73.34864044189453, "logps/rejected": -142.09848022460938, "loss": 0.8262, "margin_dpo/margin_mean": 28.828296661376953, "margin_dpo/margin_std": 30.391464233398438, "step": 536 }, { "KL/chosen_KL_mean": -45.69548416137695, "KL/mean": -57.746681213378906, "KL/rejected_KL_mean": -69.79786682128906, "KL/std": 36.305625915527344, "epoch": 0.8117913832199547, "fcm_dpo/beta": 0.03342650458216667, "fcm_dpo/delta": -0.007336266338825226, "fcm_dpo/margin": 24.102388381958008, "fcm_dpo/q_t": 0.35287484526634216, "grad_norm": 20.5838680267334, "learning_rate": 5.267213693697695e-08, "logits/chosen": 0.6057338714599609, "logits/rejected": 0.5138819217681885, "logps/chosen": -99.94410705566406, "logps/ref_chosen": -54.248619079589844, "logps/ref_rejected": -94.94343566894531, "logps/rejected": -164.74130249023438, "loss": 1.0201, "margin_dpo/margin_mean": 24.102388381958008, "margin_dpo/margin_std": 35.75040817260742, "step": 537 }, { "KL/chosen_KL_mean": -43.729248046875, "KL/mean": -57.41365051269531, "KL/rejected_KL_mean": -71.09806060791016, "KL/std": 34.0536003112793, "epoch": 0.8133030990173847, "fcm_dpo/beta": 0.033294469118118286, "fcm_dpo/delta": -0.12256969511508942, "fcm_dpo/margin": 27.368816375732422, "fcm_dpo/q_t": 0.3270268440246582, "grad_norm": 21.047393798828125, "learning_rate": 5.1863067244167144e-08, "logits/chosen": 0.5291279554367065, "logits/rejected": 0.49264344573020935, "logps/chosen": -113.82278442382812, "logps/ref_chosen": -70.09353637695312, "logps/ref_rejected": -79.49833679199219, "logps/rejected": -150.59640502929688, "loss": 0.9168, "margin_dpo/margin_mean": 27.368816375732422, "margin_dpo/margin_std": 33.95469284057617, "step": 538 }, { "KL/chosen_KL_mean": -44.2559700012207, "KL/mean": -55.49917221069336, "KL/rejected_KL_mean": -66.74237060546875, "KL/std": 33.12880325317383, "epoch": 0.8148148148148148, "fcm_dpo/beta": 0.032977961003780365, "fcm_dpo/delta": 0.06287629902362823, "fcm_dpo/margin": 22.486400604248047, "fcm_dpo/q_t": 0.3576567769050598, "grad_norm": 21.476024627685547, "learning_rate": 5.105953986729195e-08, "logits/chosen": 0.4770214557647705, "logits/rejected": 0.3988742530345917, "logps/chosen": -106.18766784667969, "logps/ref_chosen": -61.93169403076172, "logps/ref_rejected": -84.08946228027344, "logps/rejected": -150.8318328857422, "loss": 0.9873, "margin_dpo/margin_mean": 22.486400604248047, "margin_dpo/margin_std": 31.093643188476562, "step": 539 }, { "KL/chosen_KL_mean": -41.77684783935547, "KL/mean": -56.266788482666016, "KL/rejected_KL_mean": -70.75672149658203, "KL/std": 35.936214447021484, "epoch": 0.8163265306122449, "fcm_dpo/beta": 0.03290818631649017, "fcm_dpo/delta": -0.1717756688594818, "fcm_dpo/margin": 28.979869842529297, "fcm_dpo/q_t": 0.3199213445186615, "grad_norm": 20.107343673706055, "learning_rate": 5.026157728273966e-08, "logits/chosen": 0.5539939403533936, "logits/rejected": 0.45686525106430054, "logps/chosen": -104.4811019897461, "logps/ref_chosen": -62.704254150390625, "logps/ref_rejected": -95.63597106933594, "logps/rejected": -166.3927001953125, "loss": 0.8828, "margin_dpo/margin_mean": 28.979869842529297, "margin_dpo/margin_std": 32.712791442871094, "step": 540 }, { "KL/chosen_KL_mean": -39.33304214477539, "KL/mean": -53.905635833740234, "KL/rejected_KL_mean": -68.47822570800781, "KL/std": 35.508766174316406, "epoch": 0.817838246409675, "fcm_dpo/beta": 0.031236249953508377, "fcm_dpo/delta": -0.12479140609502792, "fcm_dpo/margin": 29.145185470581055, "fcm_dpo/q_t": 0.3244413137435913, "grad_norm": 19.948898315429688, "learning_rate": 4.9469201811239035e-08, "logits/chosen": 0.5305638909339905, "logits/rejected": 0.5417298078536987, "logps/chosen": -101.81388854980469, "logps/ref_chosen": -62.48084259033203, "logps/ref_rejected": -57.55541229248047, "logps/rejected": -126.03364562988281, "loss": 0.8897, "margin_dpo/margin_mean": 29.145185470581055, "margin_dpo/margin_std": 33.119834899902344, "step": 541 }, { "KL/chosen_KL_mean": -34.64030838012695, "KL/mean": -49.422584533691406, "KL/rejected_KL_mean": -64.20486450195312, "KL/std": 33.06791687011719, "epoch": 0.8193499622071051, "fcm_dpo/beta": 0.030691038817167282, "fcm_dpo/delta": -0.11876482516527176, "fcm_dpo/margin": 29.564552307128906, "fcm_dpo/q_t": 0.32889920473098755, "grad_norm": 17.267976760864258, "learning_rate": 4.868243561723534e-08, "logits/chosen": 0.5545158982276917, "logits/rejected": 0.5000336170196533, "logps/chosen": -84.09519958496094, "logps/ref_chosen": -49.454891204833984, "logps/ref_rejected": -65.33275604248047, "logps/rejected": -129.53762817382812, "loss": 0.9402, "margin_dpo/margin_mean": 29.564552307128906, "margin_dpo/margin_std": 38.110191345214844, "step": 542 }, { "KL/chosen_KL_mean": -39.595008850097656, "KL/mean": -52.76807403564453, "KL/rejected_KL_mean": -65.94114685058594, "KL/std": 33.115394592285156, "epoch": 0.8208616780045351, "fcm_dpo/beta": 0.03072110377252102, "fcm_dpo/delta": -0.01072387769818306, "fcm_dpo/margin": 26.34613800048828, "fcm_dpo/q_t": 0.3363792300224304, "grad_norm": 15.949999809265137, "learning_rate": 4.790130070827028e-08, "logits/chosen": 0.527794599533081, "logits/rejected": 0.4371853172779083, "logps/chosen": -90.69586181640625, "logps/ref_chosen": -51.100860595703125, "logps/ref_rejected": -76.06130981445312, "logps/rejected": -142.00245666503906, "loss": 0.9566, "margin_dpo/margin_mean": 26.34613800048828, "margin_dpo/margin_std": 34.06919860839844, "step": 543 }, { "KL/chosen_KL_mean": -41.76414108276367, "KL/mean": -57.745872497558594, "KL/rejected_KL_mean": -73.72760009765625, "KL/std": 35.78254699707031, "epoch": 0.8223733938019653, "fcm_dpo/beta": 0.029825232923030853, "fcm_dpo/delta": -0.16930848360061646, "fcm_dpo/margin": 31.963459014892578, "fcm_dpo/q_t": 0.32053062319755554, "grad_norm": 16.513900756835938, "learning_rate": 4.7125818934366454e-08, "logits/chosen": 0.5192776322364807, "logits/rejected": 0.4438924193382263, "logps/chosen": -102.04137420654297, "logps/ref_chosen": -60.2772331237793, "logps/ref_rejected": -88.40553283691406, "logps/rejected": -162.13314819335938, "loss": 0.8972, "margin_dpo/margin_mean": 31.96346092224121, "margin_dpo/margin_std": 38.4505729675293, "step": 544 }, { "KL/chosen_KL_mean": -44.36931610107422, "KL/mean": -55.0029296875, "KL/rejected_KL_mean": -65.63653564453125, "KL/std": 35.26418685913086, "epoch": 0.8238851095993953, "fcm_dpo/beta": 0.03026110678911209, "fcm_dpo/delta": 0.16723273694515228, "fcm_dpo/margin": 21.267223358154297, "fcm_dpo/q_t": 0.374667763710022, "grad_norm": 18.300811767578125, "learning_rate": 4.635601198741607e-08, "logits/chosen": 0.4809110462665558, "logits/rejected": 0.42077383399009705, "logps/chosen": -105.98455810546875, "logps/ref_chosen": -61.61524963378906, "logps/ref_rejected": -78.71266174316406, "logps/rejected": -144.3491973876953, "loss": 1.0727, "margin_dpo/margin_mean": 21.267223358154297, "margin_dpo/margin_std": 34.66839599609375, "step": 545 }, { "KL/chosen_KL_mean": -41.47944641113281, "KL/mean": -53.87554931640625, "KL/rejected_KL_mean": -66.27165222167969, "KL/std": 31.500242233276367, "epoch": 0.8253968253968254, "fcm_dpo/beta": 0.030640142038464546, "fcm_dpo/delta": 0.043633583933115005, "fcm_dpo/margin": 24.79220962524414, "fcm_dpo/q_t": 0.34807056188583374, "grad_norm": 21.26259994506836, "learning_rate": 4.559190140057428e-08, "logits/chosen": 0.5719273090362549, "logits/rejected": 0.5586786270141602, "logps/chosen": -100.79270935058594, "logps/ref_chosen": -59.313262939453125, "logps/ref_rejected": -64.73631286621094, "logps/rejected": -131.00796508789062, "loss": 0.9787, "margin_dpo/margin_mean": 24.79220962524414, "margin_dpo/margin_std": 32.89813232421875, "step": 546 }, { "KL/chosen_KL_mean": -37.5168342590332, "KL/mean": -51.37678527832031, "KL/rejected_KL_mean": -65.23674011230469, "KL/std": 32.473785400390625, "epoch": 0.8269085411942555, "fcm_dpo/beta": 0.030482735484838486, "fcm_dpo/delta": -0.049464091658592224, "fcm_dpo/margin": 27.719913482666016, "fcm_dpo/q_t": 0.32883119583129883, "grad_norm": 19.5365047454834, "learning_rate": 4.483350854765672e-08, "logits/chosen": 0.45498591661453247, "logits/rejected": 0.3874082565307617, "logps/chosen": -92.49357604980469, "logps/ref_chosen": -54.97674560546875, "logps/ref_rejected": -75.35922241210938, "logps/rejected": -140.59596252441406, "loss": 0.9384, "margin_dpo/margin_mean": 27.719913482666016, "margin_dpo/margin_std": 34.43286895751953, "step": 547 }, { "KL/chosen_KL_mean": -44.438194274902344, "KL/mean": -54.13751220703125, "KL/rejected_KL_mean": -63.836830139160156, "KL/std": 34.09819030761719, "epoch": 0.8284202569916855, "fcm_dpo/beta": 0.03167761117219925, "fcm_dpo/delta": 0.19408327341079712, "fcm_dpo/margin": 19.398635864257812, "fcm_dpo/q_t": 0.3884551227092743, "grad_norm": 22.586746215820312, "learning_rate": 4.4080854642541826e-08, "logits/chosen": 0.44461339712142944, "logits/rejected": 0.37233829498291016, "logps/chosen": -107.64886474609375, "logps/ref_chosen": -63.21067428588867, "logps/ref_rejected": -81.23347473144531, "logps/rejected": -145.0703125, "loss": 1.1137, "margin_dpo/margin_mean": 19.398635864257812, "margin_dpo/margin_std": 34.943870544433594, "step": 548 }, { "KL/chosen_KL_mean": -42.7745475769043, "KL/mean": -53.077823638916016, "KL/rejected_KL_mean": -63.381103515625, "KL/std": 33.2050666809082, "epoch": 0.8299319727891157, "fcm_dpo/beta": 0.03237488865852356, "fcm_dpo/delta": 0.1423811912536621, "fcm_dpo/margin": 20.60655975341797, "fcm_dpo/q_t": 0.3701988160610199, "grad_norm": 21.782363891601562, "learning_rate": 4.333396073857723e-08, "logits/chosen": 0.5990781784057617, "logits/rejected": 0.5306206345558167, "logps/chosen": -107.04806518554688, "logps/ref_chosen": -64.27351379394531, "logps/ref_rejected": -92.31663513183594, "logps/rejected": -155.69773864746094, "loss": 1.0858, "margin_dpo/margin_mean": 20.60655975341797, "margin_dpo/margin_std": 34.06036376953125, "step": 549 }, { "KL/chosen_KL_mean": -47.24556350708008, "KL/mean": -55.89653778076172, "KL/rejected_KL_mean": -64.54751586914062, "KL/std": 33.23480987548828, "epoch": 0.8314436885865457, "fcm_dpo/beta": 0.03347238153219223, "fcm_dpo/delta": 0.23484791815280914, "fcm_dpo/margin": 17.301950454711914, "fcm_dpo/q_t": 0.3901774287223816, "grad_norm": 22.462038040161133, "learning_rate": 4.259284772799099e-08, "logits/chosen": 0.5305013060569763, "logits/rejected": 0.4978870749473572, "logps/chosen": -103.47599792480469, "logps/ref_chosen": -56.230438232421875, "logps/ref_rejected": -62.59788513183594, "logps/rejected": -127.14540100097656, "loss": 1.1365, "margin_dpo/margin_mean": 17.301950454711914, "margin_dpo/margin_std": 31.971088409423828, "step": 550 }, { "KL/chosen_KL_mean": -43.95768737792969, "KL/mean": -54.62355422973633, "KL/rejected_KL_mean": -65.2894287109375, "KL/std": 32.458740234375, "epoch": 0.8329554043839759, "fcm_dpo/beta": 0.03470988571643829, "fcm_dpo/delta": 0.0632665827870369, "fcm_dpo/margin": 21.33173179626465, "fcm_dpo/q_t": 0.360112726688385, "grad_norm": 21.560270309448242, "learning_rate": 4.1857536341307176e-08, "logits/chosen": 0.5501875877380371, "logits/rejected": 0.5143569707870483, "logps/chosen": -111.70489501953125, "logps/ref_chosen": -67.74720764160156, "logps/ref_rejected": -87.04285430908203, "logps/rejected": -152.332275390625, "loss": 0.9942, "margin_dpo/margin_mean": 21.331729888916016, "margin_dpo/margin_std": 30.471147537231445, "step": 551 }, { "KL/chosen_KL_mean": -41.49971008300781, "KL/mean": -54.09961700439453, "KL/rejected_KL_mean": -66.69952392578125, "KL/std": 33.441932678222656, "epoch": 0.8344671201814059, "fcm_dpo/beta": 0.034507181495428085, "fcm_dpo/delta": -0.07692129164934158, "fcm_dpo/margin": 25.199825286865234, "fcm_dpo/q_t": 0.3218590021133423, "grad_norm": 26.227811813354492, "learning_rate": 4.112804714676593e-08, "logits/chosen": 0.4940846860408783, "logits/rejected": 0.4388379454612732, "logps/chosen": -104.42596435546875, "logps/ref_chosen": -62.92625427246094, "logps/ref_rejected": -82.98365783691406, "logps/rejected": -149.6831817626953, "loss": 0.9485, "margin_dpo/margin_mean": 25.199825286865234, "margin_dpo/margin_std": 31.810752868652344, "step": 552 }, { "KL/chosen_KL_mean": -47.06233215332031, "KL/mean": -58.25977325439453, "KL/rejected_KL_mean": -69.45719909667969, "KL/std": 34.09125518798828, "epoch": 0.8359788359788359, "fcm_dpo/beta": 0.034190136939287186, "fcm_dpo/delta": 0.03711305931210518, "fcm_dpo/margin": 22.39487075805664, "fcm_dpo/q_t": 0.37020933628082275, "grad_norm": 24.34133529663086, "learning_rate": 4.0404400549748144e-08, "logits/chosen": 0.49757149815559387, "logits/rejected": 0.3874713182449341, "logps/chosen": -103.100830078125, "logps/ref_chosen": -56.038490295410156, "logps/ref_rejected": -84.48454284667969, "logps/rejected": -153.94174194335938, "loss": 1.1121, "margin_dpo/margin_mean": 22.39487075805664, "margin_dpo/margin_std": 40.02998352050781, "step": 553 }, { "KL/chosen_KL_mean": -43.54690933227539, "KL/mean": -55.97633361816406, "KL/rejected_KL_mean": -68.40576934814453, "KL/std": 33.090389251708984, "epoch": 0.8374905517762661, "fcm_dpo/beta": 0.03405106067657471, "fcm_dpo/delta": -0.05085303634405136, "fcm_dpo/margin": 24.858858108520508, "fcm_dpo/q_t": 0.34414488077163696, "grad_norm": 25.071014404296875, "learning_rate": 3.968661679220467e-08, "logits/chosen": 0.4864945411682129, "logits/rejected": 0.46010738611221313, "logps/chosen": -108.07749938964844, "logps/ref_chosen": -64.53059387207031, "logps/ref_rejected": -71.2155990600586, "logps/rejected": -139.62136840820312, "loss": 0.9963, "margin_dpo/margin_mean": 24.858858108520508, "margin_dpo/margin_std": 36.17631530761719, "step": 554 }, { "KL/chosen_KL_mean": -48.42340850830078, "KL/mean": -60.23335647583008, "KL/rejected_KL_mean": -72.04330444335938, "KL/std": 32.82422637939453, "epoch": 0.8390022675736961, "fcm_dpo/beta": 0.03450681269168854, "fcm_dpo/delta": -0.022731244564056396, "fcm_dpo/margin": 23.619890213012695, "fcm_dpo/q_t": 0.3422093391418457, "grad_norm": 25.595046997070312, "learning_rate": 3.89747159520904e-08, "logits/chosen": 0.5255084037780762, "logits/rejected": 0.4931866526603699, "logps/chosen": -115.07533264160156, "logps/ref_chosen": -66.65191650390625, "logps/ref_rejected": -68.6667251586914, "logps/rejected": -140.71002197265625, "loss": 1.0571, "margin_dpo/margin_mean": 23.619890213012695, "margin_dpo/margin_std": 35.09877014160156, "step": 555 }, { "KL/chosen_KL_mean": -48.8804931640625, "KL/mean": -59.77089309692383, "KL/rejected_KL_mean": -70.66129302978516, "KL/std": 36.48758316040039, "epoch": 0.8405139833711263, "fcm_dpo/beta": 0.034058939665555954, "fcm_dpo/delta": 0.06282925605773926, "fcm_dpo/margin": 21.78080177307129, "fcm_dpo/q_t": 0.373182475566864, "grad_norm": 22.0106201171875, "learning_rate": 3.826871794280192e-08, "logits/chosen": 0.5408731698989868, "logits/rejected": 0.486266165971756, "logps/chosen": -101.71286010742188, "logps/ref_chosen": -52.832366943359375, "logps/ref_rejected": -64.49044036865234, "logps/rejected": -135.1517333984375, "loss": 1.1129, "margin_dpo/margin_mean": 21.780799865722656, "margin_dpo/margin_std": 38.57257843017578, "step": 556 }, { "KL/chosen_KL_mean": -47.91828536987305, "KL/mean": -62.190040588378906, "KL/rejected_KL_mean": -76.4617919921875, "KL/std": 34.56977081298828, "epoch": 0.8420256991685563, "fcm_dpo/beta": 0.03313036635518074, "fcm_dpo/delta": -0.16441971063613892, "fcm_dpo/margin": 28.54350471496582, "fcm_dpo/q_t": 0.3250656723976135, "grad_norm": 22.4103946685791, "learning_rate": 3.756864251262143e-08, "logits/chosen": 0.626792848110199, "logits/rejected": 0.5437628626823425, "logps/chosen": -102.95426940917969, "logps/ref_chosen": -55.03598403930664, "logps/ref_rejected": -75.80644989013672, "logps/rejected": -152.26824951171875, "loss": 0.8931, "margin_dpo/margin_mean": 28.543506622314453, "margin_dpo/margin_std": 33.947593688964844, "step": 557 }, { "KL/chosen_KL_mean": -40.828495025634766, "KL/mean": -55.58545684814453, "KL/rejected_KL_mean": -70.34242248535156, "KL/std": 35.937992095947266, "epoch": 0.8435374149659864, "fcm_dpo/beta": 0.03220053017139435, "fcm_dpo/delta": -0.16744528710842133, "fcm_dpo/margin": 29.5139217376709, "fcm_dpo/q_t": 0.3242967128753662, "grad_norm": 18.342342376708984, "learning_rate": 3.687450924416341e-08, "logits/chosen": 0.6076118350028992, "logits/rejected": 0.5475571751594543, "logps/chosen": -104.05484008789062, "logps/ref_chosen": -63.226348876953125, "logps/ref_rejected": -91.46881866455078, "logps/rejected": -161.81124877929688, "loss": 0.8853, "margin_dpo/margin_mean": 29.5139217376709, "margin_dpo/margin_std": 34.866798400878906, "step": 558 }, { "KL/chosen_KL_mean": -45.766387939453125, "KL/mean": -59.00743103027344, "KL/rejected_KL_mean": -72.24846649169922, "KL/std": 34.96254348754883, "epoch": 0.8450491307634165, "fcm_dpo/beta": 0.031507570296525955, "fcm_dpo/delta": -0.040891341865062714, "fcm_dpo/margin": 26.482074737548828, "fcm_dpo/q_t": 0.34787797927856445, "grad_norm": 19.380739212036133, "learning_rate": 3.6186337553827743e-08, "logits/chosen": 0.4963953197002411, "logits/rejected": 0.4245949387550354, "logps/chosen": -107.28803253173828, "logps/ref_chosen": -61.521644592285156, "logps/ref_rejected": -82.83859252929688, "logps/rejected": -155.08706665039062, "loss": 0.9999, "margin_dpo/margin_mean": 26.482074737548828, "margin_dpo/margin_std": 37.74245834350586, "step": 559 }, { "KL/chosen_KL_mean": -46.50340270996094, "KL/mean": -58.90251159667969, "KL/rejected_KL_mean": -71.3016128540039, "KL/std": 33.09571075439453, "epoch": 0.8465608465608465, "fcm_dpo/beta": 0.032199226319789886, "fcm_dpo/delta": -0.0014753341674804688, "fcm_dpo/margin": 24.79821014404297, "fcm_dpo/q_t": 0.3513341546058655, "grad_norm": 21.060949325561523, "learning_rate": 3.550414669125573e-08, "logits/chosen": 0.5204602479934692, "logits/rejected": 0.4805315136909485, "logps/chosen": -107.14462280273438, "logps/ref_chosen": -60.64122009277344, "logps/ref_rejected": -78.75474548339844, "logps/rejected": -150.05636596679688, "loss": 0.9854, "margin_dpo/margin_mean": 24.798206329345703, "margin_dpo/margin_std": 33.596336364746094, "step": 560 }, { "KL/chosen_KL_mean": -40.62968444824219, "KL/mean": -54.02823257446289, "KL/rejected_KL_mean": -67.42678833007812, "KL/std": 33.786094665527344, "epoch": 0.8480725623582767, "fcm_dpo/beta": 0.03154933452606201, "fcm_dpo/delta": -0.04966564476490021, "fcm_dpo/margin": 26.797100067138672, "fcm_dpo/q_t": 0.34592384099960327, "grad_norm": 17.998380661010742, "learning_rate": 3.482795573879241e-08, "logits/chosen": 0.510515570640564, "logits/rejected": 0.4774494171142578, "logps/chosen": -103.12828063964844, "logps/ref_chosen": -62.49859619140625, "logps/ref_rejected": -78.72064208984375, "logps/rejected": -146.14743041992188, "loss": 0.9637, "margin_dpo/margin_mean": 26.797100067138672, "margin_dpo/margin_std": 36.19969177246094, "step": 561 }, { "KL/chosen_KL_mean": -45.762332916259766, "KL/mean": -60.15574264526367, "KL/rejected_KL_mean": -74.54913330078125, "KL/std": 35.69789123535156, "epoch": 0.8495842781557067, "fcm_dpo/beta": 0.030817052349448204, "fcm_dpo/delta": -0.09736991673707962, "fcm_dpo/margin": 28.78680992126465, "fcm_dpo/q_t": 0.33300113677978516, "grad_norm": 21.175722122192383, "learning_rate": 3.415778361095226e-08, "logits/chosen": 0.5440159440040588, "logits/rejected": 0.5005948543548584, "logps/chosen": -120.5440673828125, "logps/ref_chosen": -74.78173828125, "logps/ref_rejected": -92.63499450683594, "logps/rejected": -167.18414306640625, "loss": 0.9237, "margin_dpo/margin_mean": 28.786806106567383, "margin_dpo/margin_std": 35.51923370361328, "step": 562 }, { "KL/chosen_KL_mean": -40.64170837402344, "KL/mean": -53.593021392822266, "KL/rejected_KL_mean": -66.54432678222656, "KL/std": 33.413021087646484, "epoch": 0.8510959939531368, "fcm_dpo/beta": 0.030957389622926712, "fcm_dpo/delta": -0.0022011548280715942, "fcm_dpo/margin": 25.90262222290039, "fcm_dpo/q_t": 0.34764784574508667, "grad_norm": 23.740327835083008, "learning_rate": 3.349364905389032e-08, "logits/chosen": 0.5938107967376709, "logits/rejected": 0.5400828123092651, "logps/chosen": -90.8402099609375, "logps/ref_chosen": -50.19850158691406, "logps/ref_rejected": -66.76687622070312, "logps/rejected": -133.3112030029297, "loss": 1.0339, "margin_dpo/margin_mean": 25.90262222290039, "margin_dpo/margin_std": 38.862754821777344, "step": 563 }, { "KL/chosen_KL_mean": -39.63123321533203, "KL/mean": -55.32246398925781, "KL/rejected_KL_mean": -71.01370239257812, "KL/std": 33.93794250488281, "epoch": 0.8526077097505669, "fcm_dpo/beta": 0.030182164162397385, "fcm_dpo/delta": -0.16239605844020844, "fcm_dpo/margin": 31.38246726989746, "fcm_dpo/q_t": 0.31941738724708557, "grad_norm": 17.085735321044922, "learning_rate": 3.283557064487785e-08, "logits/chosen": 0.4971418082714081, "logits/rejected": 0.46422910690307617, "logps/chosen": -95.37208557128906, "logps/ref_chosen": -55.7408447265625, "logps/ref_rejected": -74.82323455810547, "logps/rejected": -145.83694458007812, "loss": 0.9018, "margin_dpo/margin_mean": 31.38246726989746, "margin_dpo/margin_std": 37.687286376953125, "step": 564 }, { "KL/chosen_KL_mean": -51.22956848144531, "KL/mean": -62.0440673828125, "KL/rejected_KL_mean": -72.85856628417969, "KL/std": 36.59965515136719, "epoch": 0.854119425547997, "fcm_dpo/beta": 0.030782857909798622, "fcm_dpo/delta": 0.14088793098926544, "fcm_dpo/margin": 21.62899398803711, "fcm_dpo/q_t": 0.3720448613166809, "grad_norm": 19.96152687072754, "learning_rate": 3.218356679178252e-08, "logits/chosen": 0.563768744468689, "logits/rejected": 0.5072394013404846, "logps/chosen": -109.56695556640625, "logps/ref_chosen": -58.33738327026367, "logps/ref_rejected": -78.31776428222656, "logps/rejected": -151.17633056640625, "loss": 1.06, "margin_dpo/margin_mean": 21.628995895385742, "margin_dpo/margin_std": 34.05863571166992, "step": 565 }, { "KL/chosen_KL_mean": -44.82415008544922, "KL/mean": -56.39575958251953, "KL/rejected_KL_mean": -67.96736145019531, "KL/std": 34.829559326171875, "epoch": 0.8556311413454271, "fcm_dpo/beta": 0.03129101172089577, "fcm_dpo/delta": 0.08050192892551422, "fcm_dpo/margin": 23.143218994140625, "fcm_dpo/q_t": 0.3658328950405121, "grad_norm": 26.74419593811035, "learning_rate": 3.1537655732553764e-08, "logits/chosen": 0.536413311958313, "logits/rejected": 0.5143975615501404, "logps/chosen": -116.04789733886719, "logps/ref_chosen": -71.22373962402344, "logps/ref_rejected": -71.11601257324219, "logps/rejected": -139.0833740234375, "loss": 1.1239, "margin_dpo/margin_mean": 23.143218994140625, "margin_dpo/margin_std": 40.9520149230957, "step": 566 }, { "KL/chosen_KL_mean": -44.22145080566406, "KL/mean": -57.35389709472656, "KL/rejected_KL_mean": -70.48634338378906, "KL/std": 35.569793701171875, "epoch": 0.8571428571428571, "fcm_dpo/beta": 0.03084486722946167, "fcm_dpo/delta": -0.013498269021511078, "fcm_dpo/margin": 26.26490020751953, "fcm_dpo/q_t": 0.3430580496788025, "grad_norm": 16.386756896972656, "learning_rate": 3.089785553471233e-08, "logits/chosen": 0.5396600365638733, "logits/rejected": 0.4434698820114136, "logps/chosen": -96.8907241821289, "logps/ref_chosen": -52.669273376464844, "logps/ref_rejected": -74.34785461425781, "logps/rejected": -144.83419799804688, "loss": 0.9424, "margin_dpo/margin_mean": 26.2648983001709, "margin_dpo/margin_std": 32.46692657470703, "step": 567 }, { "KL/chosen_KL_mean": -41.46443176269531, "KL/mean": -56.34814453125, "KL/rejected_KL_mean": -71.23185729980469, "KL/std": 33.74592208862305, "epoch": 0.8586545729402872, "fcm_dpo/beta": 0.03083047643303871, "fcm_dpo/delta": -0.13012617826461792, "fcm_dpo/margin": 29.767419815063477, "fcm_dpo/q_t": 0.3229033350944519, "grad_norm": 19.885181427001953, "learning_rate": 3.026418409484513e-08, "logits/chosen": 0.5687329173088074, "logits/rejected": 0.4785163998603821, "logps/chosen": -93.6424331665039, "logps/ref_chosen": -52.178001403808594, "logps/ref_rejected": -85.8277587890625, "logps/rejected": -157.05963134765625, "loss": 0.8686, "margin_dpo/margin_mean": 29.767419815063477, "margin_dpo/margin_std": 32.10868835449219, "step": 568 }, { "KL/chosen_KL_mean": -46.65216827392578, "KL/mean": -57.41333770751953, "KL/rejected_KL_mean": -68.17449951171875, "KL/std": 34.74163818359375, "epoch": 0.8601662887377173, "fcm_dpo/beta": 0.030656516551971436, "fcm_dpo/delta": 0.14884671568870544, "fcm_dpo/margin": 21.522335052490234, "fcm_dpo/q_t": 0.37315359711647034, "grad_norm": 20.921825408935547, "learning_rate": 2.963665913810451e-08, "logits/chosen": 0.46112173795700073, "logits/rejected": 0.4297167658805847, "logps/chosen": -109.30142211914062, "logps/ref_chosen": -62.649261474609375, "logps/ref_rejected": -75.4298324584961, "logps/rejected": -143.60433959960938, "loss": 1.0718, "margin_dpo/margin_mean": 21.522335052490234, "margin_dpo/margin_std": 34.24241256713867, "step": 569 }, { "KL/chosen_KL_mean": -40.823997497558594, "KL/mean": -58.369590759277344, "KL/rejected_KL_mean": -75.9151840209961, "KL/std": 33.9276123046875, "epoch": 0.8616780045351474, "fcm_dpo/beta": 0.030290057882666588, "fcm_dpo/delta": -0.2935383915901184, "fcm_dpo/margin": 35.091190338134766, "fcm_dpo/q_t": 0.2969120740890503, "grad_norm": 17.00665283203125, "learning_rate": 2.9015298217712453e-08, "logits/chosen": 0.4849631190299988, "logits/rejected": 0.39886677265167236, "logps/chosen": -90.86579132080078, "logps/ref_chosen": -50.04179382324219, "logps/ref_rejected": -78.27146911621094, "logps/rejected": -154.1866455078125, "loss": 0.8096, "margin_dpo/margin_mean": 35.091190338134766, "margin_dpo/margin_std": 35.0093994140625, "step": 570 }, { "KL/chosen_KL_mean": -45.955867767333984, "KL/mean": -55.82860565185547, "KL/rejected_KL_mean": -65.70135498046875, "KL/std": 32.06074142456055, "epoch": 0.8631897203325775, "fcm_dpo/beta": 0.030671749264001846, "fcm_dpo/delta": 0.20318175852298737, "fcm_dpo/margin": 19.74549102783203, "fcm_dpo/q_t": 0.375748872756958, "grad_norm": 21.15029525756836, "learning_rate": 2.840011871446962e-08, "logits/chosen": 0.5245357751846313, "logits/rejected": 0.487672358751297, "logps/chosen": -99.61268615722656, "logps/ref_chosen": -53.65681457519531, "logps/ref_rejected": -66.13298034667969, "logps/rejected": -131.83433532714844, "loss": 1.0978, "margin_dpo/margin_mean": 19.74549102783203, "margin_dpo/margin_std": 32.583404541015625, "step": 571 }, { "KL/chosen_KL_mean": -44.28086853027344, "KL/mean": -56.86238098144531, "KL/rejected_KL_mean": -69.44389343261719, "KL/std": 33.44170379638672, "epoch": 0.8647014361300076, "fcm_dpo/beta": 0.030863357707858086, "fcm_dpo/delta": 0.025278106331825256, "fcm_dpo/margin": 25.163036346435547, "fcm_dpo/q_t": 0.346957266330719, "grad_norm": 21.679777145385742, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 0.5327567458152771, "logits/rejected": 0.5728912353515625, "logps/chosen": -119.09879302978516, "logps/ref_chosen": -74.81792449951172, "logps/ref_rejected": -65.88681030273438, "logps/rejected": -135.33070373535156, "loss": 0.9486, "margin_dpo/margin_mean": 25.163036346435547, "margin_dpo/margin_std": 31.716094970703125, "step": 572 }, { "KL/chosen_KL_mean": -50.17157745361328, "KL/mean": -61.936317443847656, "KL/rejected_KL_mean": -73.7010498046875, "KL/std": 36.69175338745117, "epoch": 0.8662131519274376, "fcm_dpo/beta": 0.03104744479060173, "fcm_dpo/delta": 0.074785977602005, "fcm_dpo/margin": 23.529462814331055, "fcm_dpo/q_t": 0.3677501380443573, "grad_norm": 26.699995040893555, "learning_rate": 2.718837261761528e-08, "logits/chosen": 0.519621729850769, "logits/rejected": 0.4729001522064209, "logps/chosen": -118.89722442626953, "logps/ref_chosen": -68.72564697265625, "logps/ref_rejected": -88.16201782226562, "logps/rejected": -161.86306762695312, "loss": 1.1149, "margin_dpo/margin_mean": 23.529464721679688, "margin_dpo/margin_std": 41.90831756591797, "step": 573 }, { "KL/chosen_KL_mean": -44.603668212890625, "KL/mean": -59.30667495727539, "KL/rejected_KL_mean": -74.00968170166016, "KL/std": 35.03868103027344, "epoch": 0.8677248677248677, "fcm_dpo/beta": 0.031052954494953156, "fcm_dpo/delta": -0.12524369359016418, "fcm_dpo/margin": 29.406003952026367, "fcm_dpo/q_t": 0.32094401121139526, "grad_norm": 16.99439811706543, "learning_rate": 2.659183991914696e-08, "logits/chosen": 0.5790042877197266, "logits/rejected": 0.5058803558349609, "logps/chosen": -100.91706848144531, "logps/ref_chosen": -56.31340026855469, "logps/ref_rejected": -83.91553497314453, "logps/rejected": -157.9252166748047, "loss": 0.8578, "margin_dpo/margin_mean": 29.406003952026367, "margin_dpo/margin_std": 31.216388702392578, "step": 574 }, { "KL/chosen_KL_mean": -44.00529479980469, "KL/mean": -53.75147247314453, "KL/rejected_KL_mean": -63.49766540527344, "KL/std": 32.914180755615234, "epoch": 0.8692365835222978, "fcm_dpo/beta": 0.03111717477440834, "fcm_dpo/delta": 0.20531663298606873, "fcm_dpo/margin": 19.49237060546875, "fcm_dpo/q_t": 0.3899354934692383, "grad_norm": 22.751699447631836, "learning_rate": 2.600155642716606e-08, "logits/chosen": 0.5909614562988281, "logits/rejected": 0.5080176591873169, "logps/chosen": -108.58941650390625, "logps/ref_chosen": -64.5841293334961, "logps/ref_rejected": -93.47034454345703, "logps/rejected": -156.968017578125, "loss": 1.159, "margin_dpo/margin_mean": 19.492368698120117, "margin_dpo/margin_std": 37.30479049682617, "step": 575 }, { "KL/chosen_KL_mean": -39.35127258300781, "KL/mean": -53.455238342285156, "KL/rejected_KL_mean": -67.5592041015625, "KL/std": 34.92966079711914, "epoch": 0.8707482993197279, "fcm_dpo/beta": 0.03119221143424511, "fcm_dpo/delta": -0.08927100151777267, "fcm_dpo/margin": 28.207935333251953, "fcm_dpo/q_t": 0.3355104923248291, "grad_norm": 18.904682159423828, "learning_rate": 2.5417538653170754e-08, "logits/chosen": 0.558881402015686, "logits/rejected": 0.4506233334541321, "logps/chosen": -92.63179779052734, "logps/ref_chosen": -53.28052520751953, "logps/ref_rejected": -84.2000503540039, "logps/rejected": -151.75924682617188, "loss": 0.9491, "margin_dpo/margin_mean": 28.20793342590332, "margin_dpo/margin_std": 36.800270080566406, "step": 576 }, { "KL/chosen_KL_mean": -43.53594207763672, "KL/mean": -54.1453857421875, "KL/rejected_KL_mean": -64.75482940673828, "KL/std": 32.73902130126953, "epoch": 0.872260015117158, "fcm_dpo/beta": 0.03182173892855644, "fcm_dpo/delta": 0.13401471078395844, "fcm_dpo/margin": 21.218887329101562, "fcm_dpo/q_t": 0.3719717860221863, "grad_norm": 21.348833084106445, "learning_rate": 2.4839802933393607e-08, "logits/chosen": 0.5364026427268982, "logits/rejected": 0.5159157514572144, "logps/chosen": -105.86062622070312, "logps/ref_chosen": -62.32468795776367, "logps/ref_rejected": -67.300537109375, "logps/rejected": -132.05535888671875, "loss": 1.1081, "margin_dpo/margin_mean": 21.218889236450195, "margin_dpo/margin_std": 37.124244689941406, "step": 577 }, { "KL/chosen_KL_mean": -43.72724151611328, "KL/mean": -52.62541961669922, "KL/rejected_KL_mean": -61.52360534667969, "KL/std": 33.30746078491211, "epoch": 0.873771730914588, "fcm_dpo/beta": 0.033036790788173676, "fcm_dpo/delta": 0.22592979669570923, "fcm_dpo/margin": 17.796363830566406, "fcm_dpo/q_t": 0.3949415981769562, "grad_norm": 21.883607864379883, "learning_rate": 2.4268365428344733e-08, "logits/chosen": 0.5855566263198853, "logits/rejected": 0.5576690435409546, "logps/chosen": -100.3828125, "logps/ref_chosen": -56.65557861328125, "logps/ref_rejected": -68.21835327148438, "logps/rejected": -129.74195861816406, "loss": 1.1512, "margin_dpo/margin_mean": 17.796363830566406, "margin_dpo/margin_std": 34.408897399902344, "step": 578 }, { "KL/chosen_KL_mean": -44.7183837890625, "KL/mean": -60.99937438964844, "KL/rejected_KL_mean": -77.28036499023438, "KL/std": 35.42094421386719, "epoch": 0.8752834467120182, "fcm_dpo/beta": 0.03220862150192261, "fcm_dpo/delta": -0.2783782482147217, "fcm_dpo/margin": 32.561981201171875, "fcm_dpo/q_t": 0.30090928077697754, "grad_norm": 17.46132469177246, "learning_rate": 2.3703242122359357e-08, "logits/chosen": 0.4664205312728882, "logits/rejected": 0.43535664677619934, "logps/chosen": -101.52804565429688, "logps/ref_chosen": -56.809661865234375, "logps/ref_rejected": -68.09613037109375, "logps/rejected": -145.37649536132812, "loss": 0.825, "margin_dpo/margin_mean": 32.56198501586914, "margin_dpo/margin_std": 34.768150329589844, "step": 579 }, { "KL/chosen_KL_mean": -44.27873992919922, "KL/mean": -55.907466888427734, "KL/rejected_KL_mean": -67.53619384765625, "KL/std": 36.50806427001953, "epoch": 0.8767951625094482, "fcm_dpo/beta": 0.032273322343826294, "fcm_dpo/delta": 0.05180711671710014, "fcm_dpo/margin": 23.257455825805664, "fcm_dpo/q_t": 0.3628014028072357, "grad_norm": 23.451509475708008, "learning_rate": 2.3144448823151392e-08, "logits/chosen": 0.49390000104904175, "logits/rejected": 0.43802976608276367, "logps/chosen": -101.97885131835938, "logps/ref_chosen": -57.70011520385742, "logps/ref_rejected": -77.90664672851562, "logps/rejected": -145.44284057617188, "loss": 1.0788, "margin_dpo/margin_mean": 23.25745391845703, "margin_dpo/margin_std": 38.16685485839844, "step": 580 }, { "KL/chosen_KL_mean": -50.066375732421875, "KL/mean": -60.091793060302734, "KL/rejected_KL_mean": -70.11721801757812, "KL/std": 32.76508331298828, "epoch": 0.8783068783068783, "fcm_dpo/beta": 0.03265048936009407, "fcm_dpo/delta": 0.1556333750486374, "fcm_dpo/margin": 20.05084228515625, "fcm_dpo/q_t": 0.37884268164634705, "grad_norm": 22.65587043762207, "learning_rate": 2.259200116137039e-08, "logits/chosen": 0.5524417161941528, "logits/rejected": 0.48380810022354126, "logps/chosen": -109.39872741699219, "logps/ref_chosen": -59.332359313964844, "logps/ref_rejected": -83.64482116699219, "logps/rejected": -153.7620391845703, "loss": 1.1162, "margin_dpo/margin_mean": 20.05084228515625, "margin_dpo/margin_std": 35.896873474121094, "step": 581 }, { "KL/chosen_KL_mean": -44.435325622558594, "KL/mean": -57.18058776855469, "KL/rejected_KL_mean": -69.92584228515625, "KL/std": 34.72920227050781, "epoch": 0.8798185941043084, "fcm_dpo/beta": 0.032961271703243256, "fcm_dpo/delta": -0.0438670888543129, "fcm_dpo/margin": 25.49052619934082, "fcm_dpo/q_t": 0.341183066368103, "grad_norm": 22.326169967651367, "learning_rate": 2.204591459016525e-08, "logits/chosen": 0.5303821563720703, "logits/rejected": 0.557191014289856, "logps/chosen": -108.59818267822266, "logps/ref_chosen": -64.16285705566406, "logps/ref_rejected": -58.632896423339844, "logps/rejected": -128.55874633789062, "loss": 0.9808, "margin_dpo/margin_mean": 25.490524291992188, "margin_dpo/margin_std": 35.270545959472656, "step": 582 }, { "KL/chosen_KL_mean": -44.09051513671875, "KL/mean": -56.927040100097656, "KL/rejected_KL_mean": -69.76355743408203, "KL/std": 34.18093490600586, "epoch": 0.8813303099017384, "fcm_dpo/beta": 0.032621078193187714, "fcm_dpo/delta": -0.04094298556447029, "fcm_dpo/margin": 25.67304801940918, "fcm_dpo/q_t": 0.3495996296405792, "grad_norm": 24.98419761657715, "learning_rate": 2.1506204384751064e-08, "logits/chosen": 0.6439502239227295, "logits/rejected": 0.5294203162193298, "logps/chosen": -95.96290588378906, "logps/ref_chosen": -51.87239456176758, "logps/ref_rejected": -83.86331176757812, "logps/rejected": -153.62686157226562, "loss": 1.0603, "margin_dpo/margin_mean": 25.673049926757812, "margin_dpo/margin_std": 40.889869689941406, "step": 583 }, { "KL/chosen_KL_mean": -39.34344482421875, "KL/mean": -51.7713508605957, "KL/rejected_KL_mean": -64.19924926757812, "KL/std": 32.398521423339844, "epoch": 0.8828420256991686, "fcm_dpo/beta": 0.03228888660669327, "fcm_dpo/delta": -0.004207976162433624, "fcm_dpo/margin": 24.855804443359375, "fcm_dpo/q_t": 0.3564729690551758, "grad_norm": 23.26299285888672, "learning_rate": 2.09728856419826e-08, "logits/chosen": 0.6478234529495239, "logits/rejected": 0.5402784943580627, "logps/chosen": -85.91484069824219, "logps/ref_chosen": -46.571388244628906, "logps/ref_rejected": -80.67969512939453, "logps/rejected": -144.87893676757812, "loss": 1.0805, "margin_dpo/margin_mean": 24.855804443359375, "margin_dpo/margin_std": 40.67819595336914, "step": 584 }, { "KL/chosen_KL_mean": -45.52735900878906, "KL/mean": -55.15117645263672, "KL/rejected_KL_mean": -64.77499389648438, "KL/std": 35.69919967651367, "epoch": 0.8843537414965986, "fcm_dpo/beta": 0.033551305532455444, "fcm_dpo/delta": 0.1630236804485321, "fcm_dpo/margin": 19.247638702392578, "fcm_dpo/q_t": 0.3736453950405121, "grad_norm": 20.46152114868164, "learning_rate": 2.044597327993153e-08, "logits/chosen": 0.4899832606315613, "logits/rejected": 0.44771748781204224, "logps/chosen": -103.65189361572266, "logps/ref_chosen": -58.124534606933594, "logps/ref_rejected": -79.00538635253906, "logps/rejected": -143.78038024902344, "loss": 1.1302, "margin_dpo/margin_mean": 19.247638702392578, "margin_dpo/margin_std": 35.1866569519043, "step": 585 }, { "KL/chosen_KL_mean": -44.563514709472656, "KL/mean": -58.280799865722656, "KL/rejected_KL_mean": -71.99807739257812, "KL/std": 34.64778518676758, "epoch": 0.8858654572940288, "fcm_dpo/beta": 0.03286353871226311, "fcm_dpo/delta": -0.11315355449914932, "fcm_dpo/margin": 27.4345703125, "fcm_dpo/q_t": 0.32458969950675964, "grad_norm": 23.63680648803711, "learning_rate": 1.9925482037469187e-08, "logits/chosen": 0.5588226914405823, "logits/rejected": 0.5063307881355286, "logps/chosen": -98.66515350341797, "logps/ref_chosen": -54.10163879394531, "logps/ref_rejected": -63.72113037109375, "logps/rejected": -135.71920776367188, "loss": 0.8879, "margin_dpo/margin_mean": 27.4345703125, "margin_dpo/margin_std": 31.724838256835938, "step": 586 }, { "KL/chosen_KL_mean": -42.331520080566406, "KL/mean": -55.22560119628906, "KL/rejected_KL_mean": -68.11968994140625, "KL/std": 35.381317138671875, "epoch": 0.8873771730914588, "fcm_dpo/beta": 0.03263702988624573, "fcm_dpo/delta": -0.04549926146864891, "fcm_dpo/margin": 25.788166046142578, "fcm_dpo/q_t": 0.3424980342388153, "grad_norm": 22.418487548828125, "learning_rate": 1.9411426473854687e-08, "logits/chosen": 0.5717095136642456, "logits/rejected": 0.5568786859512329, "logps/chosen": -105.74871826171875, "logps/ref_chosen": -63.41719436645508, "logps/ref_rejected": -63.47003936767578, "logps/rejected": -131.5897216796875, "loss": 1.0842, "margin_dpo/margin_mean": 25.788166046142578, "margin_dpo/margin_std": 42.68872833251953, "step": 587 }, { "KL/chosen_KL_mean": -45.627567291259766, "KL/mean": -58.562164306640625, "KL/rejected_KL_mean": -71.49676513671875, "KL/std": 34.04606628417969, "epoch": 0.8888888888888888, "fcm_dpo/beta": 0.03227134793996811, "fcm_dpo/delta": -0.038421668112277985, "fcm_dpo/margin": 25.86919403076172, "fcm_dpo/q_t": 0.3437250256538391, "grad_norm": 23.81206703186035, "learning_rate": 1.890382096832699e-08, "logits/chosen": 0.574599027633667, "logits/rejected": 0.5247448682785034, "logps/chosen": -107.82859802246094, "logps/ref_chosen": -62.20103454589844, "logps/ref_rejected": -82.10249328613281, "logps/rejected": -153.59925842285156, "loss": 0.9862, "margin_dpo/margin_mean": 25.869192123413086, "margin_dpo/margin_std": 36.37947082519531, "step": 588 }, { "KL/chosen_KL_mean": -39.77646255493164, "KL/mean": -54.80766677856445, "KL/rejected_KL_mean": -69.8388671875, "KL/std": 35.71990966796875, "epoch": 0.890400604686319, "fcm_dpo/beta": 0.03161867707967758, "fcm_dpo/delta": -0.1660689115524292, "fcm_dpo/margin": 30.06241226196289, "fcm_dpo/q_t": 0.3181713819503784, "grad_norm": 20.573936462402344, "learning_rate": 1.840267971970344e-08, "logits/chosen": 0.5346547961235046, "logits/rejected": 0.5033892393112183, "logps/chosen": -96.49008178710938, "logps/ref_chosen": -56.71361541748047, "logps/ref_rejected": -76.7366943359375, "logps/rejected": -146.5755615234375, "loss": 0.8598, "margin_dpo/margin_mean": 30.06241226196289, "margin_dpo/margin_std": 33.88391876220703, "step": 589 }, { "KL/chosen_KL_mean": -46.1741828918457, "KL/mean": -61.5664176940918, "KL/rejected_KL_mean": -76.95864868164062, "KL/std": 34.06959533691406, "epoch": 0.891912320483749, "fcm_dpo/beta": 0.030643416568636894, "fcm_dpo/delta": -0.15798437595367432, "fcm_dpo/margin": 30.784465789794922, "fcm_dpo/q_t": 0.32272064685821533, "grad_norm": 19.49903678894043, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 0.46632716059684753, "logits/rejected": 0.4379855990409851, "logps/chosen": -112.68800354003906, "logps/ref_chosen": -66.5138168334961, "logps/ref_rejected": -85.70820617675781, "logps/rejected": -162.66685485839844, "loss": 0.9171, "margin_dpo/margin_mean": 30.784465789794922, "margin_dpo/margin_std": 38.08427047729492, "step": 590 }, { "KL/chosen_KL_mean": -38.659080505371094, "KL/mean": -54.095245361328125, "KL/rejected_KL_mean": -69.53141784667969, "KL/std": 35.19484329223633, "epoch": 0.8934240362811792, "fcm_dpo/beta": 0.02937409281730652, "fcm_dpo/delta": -0.12251757830381393, "fcm_dpo/margin": 30.872339248657227, "fcm_dpo/q_t": 0.328896701335907, "grad_norm": 19.901981353759766, "learning_rate": 1.7419845883949098e-08, "logits/chosen": 0.62455815076828, "logits/rejected": 0.5571376085281372, "logps/chosen": -99.35626220703125, "logps/ref_chosen": -60.697181701660156, "logps/ref_rejected": -86.12278747558594, "logps/rejected": -155.65420532226562, "loss": 0.9763, "margin_dpo/margin_mean": 30.872339248657227, "margin_dpo/margin_std": 40.98619079589844, "step": 591 }, { "KL/chosen_KL_mean": -39.370513916015625, "KL/mean": -52.374717712402344, "KL/rejected_KL_mean": -65.37891387939453, "KL/std": 33.04375457763672, "epoch": 0.8949357520786092, "fcm_dpo/beta": 0.029393857344985008, "fcm_dpo/delta": 0.036264337599277496, "fcm_dpo/margin": 26.008399963378906, "fcm_dpo/q_t": 0.35765397548675537, "grad_norm": 18.51282501220703, "learning_rate": 1.6938180788793556e-08, "logits/chosen": 0.5873167514801025, "logits/rejected": 0.4715259373188019, "logps/chosen": -90.60784912109375, "logps/ref_chosen": -51.237327575683594, "logps/ref_rejected": -81.60242462158203, "logps/rejected": -146.98133850097656, "loss": 0.9725, "margin_dpo/margin_mean": 26.008399963378906, "margin_dpo/margin_std": 34.70623779296875, "step": 592 }, { "KL/chosen_KL_mean": -38.42186737060547, "KL/mean": -50.738250732421875, "KL/rejected_KL_mean": -63.05463790893555, "KL/std": 35.53569793701172, "epoch": 0.8964474678760394, "fcm_dpo/beta": 0.03002101369202137, "fcm_dpo/delta": 0.06538180261850357, "fcm_dpo/margin": 24.632770538330078, "fcm_dpo/q_t": 0.3631042242050171, "grad_norm": 21.09285545349121, "learning_rate": 1.6463034933723336e-08, "logits/chosen": 0.5370926856994629, "logits/rejected": 0.4419124722480774, "logps/chosen": -80.50186920166016, "logps/ref_chosen": -42.08000183105469, "logps/ref_rejected": -68.47499084472656, "logps/rejected": -131.52963256835938, "loss": 1.0906, "margin_dpo/margin_mean": 24.632770538330078, "margin_dpo/margin_std": 41.283905029296875, "step": 593 }, { "KL/chosen_KL_mean": -46.77622985839844, "KL/mean": -57.62593078613281, "KL/rejected_KL_mean": -68.47563171386719, "KL/std": 34.27396011352539, "epoch": 0.8979591836734694, "fcm_dpo/beta": 0.03068956546485424, "fcm_dpo/delta": 0.14385400712490082, "fcm_dpo/margin": 21.699413299560547, "fcm_dpo/q_t": 0.3688500225543976, "grad_norm": 18.898942947387695, "learning_rate": 1.5994421609589385e-08, "logits/chosen": 0.45002150535583496, "logits/rejected": 0.43023163080215454, "logps/chosen": -110.43489837646484, "logps/ref_chosen": -63.658668518066406, "logps/ref_rejected": -70.35597229003906, "logps/rejected": -138.83160400390625, "loss": 1.0382, "margin_dpo/margin_mean": 21.699413299560547, "margin_dpo/margin_std": 32.530616760253906, "step": 594 }, { "KL/chosen_KL_mean": -42.06338119506836, "KL/mean": -58.380043029785156, "KL/rejected_KL_mean": -74.69670104980469, "KL/std": 35.81193542480469, "epoch": 0.8994708994708994, "fcm_dpo/beta": 0.030249230563640594, "fcm_dpo/delta": -0.20712698996067047, "fcm_dpo/margin": 32.63332748413086, "fcm_dpo/q_t": 0.31433260440826416, "grad_norm": 17.953968048095703, "learning_rate": 1.553235392451377e-08, "logits/chosen": 0.5894097089767456, "logits/rejected": 0.4952569007873535, "logps/chosen": -98.28213500976562, "logps/ref_chosen": -56.21875762939453, "logps/ref_rejected": -83.95773315429688, "logps/rejected": -158.65444946289062, "loss": 0.912, "margin_dpo/margin_mean": 32.633331298828125, "margin_dpo/margin_std": 40.300350189208984, "step": 595 }, { "KL/chosen_KL_mean": -47.10646057128906, "KL/mean": -55.662208557128906, "KL/rejected_KL_mean": -64.21795654296875, "KL/std": 32.69890594482422, "epoch": 0.9009826152683296, "fcm_dpo/beta": 0.0309823639690876, "fcm_dpo/delta": 0.2856805920600891, "fcm_dpo/margin": 17.111492156982422, "fcm_dpo/q_t": 0.4075329303741455, "grad_norm": 21.329530715942383, "learning_rate": 1.507684480352292e-08, "logits/chosen": 0.45342981815338135, "logits/rejected": 0.46392911672592163, "logps/chosen": -115.58734130859375, "logps/ref_chosen": -68.48088073730469, "logps/ref_rejected": -61.732967376708984, "logps/rejected": -125.950927734375, "loss": 1.1914, "margin_dpo/margin_mean": 17.111494064331055, "margin_dpo/margin_std": 37.079368591308594, "step": 596 }, { "KL/chosen_KL_mean": -37.21601867675781, "KL/mean": -49.4849853515625, "KL/rejected_KL_mean": -61.75395202636719, "KL/std": 31.241012573242188, "epoch": 0.9024943310657596, "fcm_dpo/beta": 0.03160978853702545, "fcm_dpo/delta": 0.026392444968223572, "fcm_dpo/margin": 24.53792381286621, "fcm_dpo/q_t": 0.36042332649230957, "grad_norm": 18.138704299926758, "learning_rate": 1.4627906988186111e-08, "logits/chosen": 0.5056836605072021, "logits/rejected": 0.4817969799041748, "logps/chosen": -86.07353210449219, "logps/ref_chosen": -48.85750961303711, "logps/ref_rejected": -55.068084716796875, "logps/rejected": -116.82203674316406, "loss": 1.0104, "margin_dpo/margin_mean": 24.537925720214844, "margin_dpo/margin_std": 36.30146026611328, "step": 597 }, { "KL/chosen_KL_mean": -49.368431091308594, "KL/mean": -57.680335998535156, "KL/rejected_KL_mean": -65.99224090576172, "KL/std": 34.18096160888672, "epoch": 0.9040060468631897, "fcm_dpo/beta": 0.03309684246778488, "fcm_dpo/delta": 0.2630840837955475, "fcm_dpo/margin": 16.62381362915039, "fcm_dpo/q_t": 0.40100929141044617, "grad_norm": 39.50839614868164, "learning_rate": 1.4185553036259095e-08, "logits/chosen": 0.5584957599639893, "logits/rejected": 0.4789044260978699, "logps/chosen": -108.25558471679688, "logps/ref_chosen": -58.88715362548828, "logps/ref_rejected": -81.43145751953125, "logps/rejected": -147.4237060546875, "loss": 1.204, "margin_dpo/margin_mean": 16.62381362915039, "margin_dpo/margin_std": 36.20482635498047, "step": 598 }, { "KL/chosen_KL_mean": -50.590545654296875, "KL/mean": -60.70787048339844, "KL/rejected_KL_mean": -70.8251953125, "KL/std": 31.916885375976562, "epoch": 0.9055177626606198, "fcm_dpo/beta": 0.03424448519945145, "fcm_dpo/delta": 0.11299724876880646, "fcm_dpo/margin": 20.234647750854492, "fcm_dpo/q_t": 0.3777085244655609, "grad_norm": 25.235319137573242, "learning_rate": 1.3749795321332885e-08, "logits/chosen": 0.619079053401947, "logits/rejected": 0.5716946721076965, "logps/chosen": -108.19773864746094, "logps/ref_chosen": -57.60719299316406, "logps/ref_rejected": -71.80469512939453, "logps/rejected": -142.6298828125, "loss": 1.1176, "margin_dpo/margin_mean": 20.234647750854492, "margin_dpo/margin_std": 36.37825012207031, "step": 599 }, { "KL/chosen_KL_mean": -46.038330078125, "KL/mean": -56.578067779541016, "KL/rejected_KL_mean": -67.11781311035156, "KL/std": 35.091941833496094, "epoch": 0.9070294784580499, "fcm_dpo/beta": 0.03507785126566887, "fcm_dpo/delta": 0.0583866685628891, "fcm_dpo/margin": 21.0794734954834, "fcm_dpo/q_t": 0.36688366532325745, "grad_norm": 26.915437698364258, "learning_rate": 1.3320646032487393e-08, "logits/chosen": 0.591224193572998, "logits/rejected": 0.5398635864257812, "logps/chosen": -104.48063659667969, "logps/ref_chosen": -58.44231414794922, "logps/ref_rejected": -83.64639282226562, "logps/rejected": -150.76419067382812, "loss": 1.1057, "margin_dpo/margin_mean": 21.0794734954834, "margin_dpo/margin_std": 35.62135314941406, "step": 600 }, { "KL/chosen_KL_mean": -38.50769805908203, "KL/mean": -52.836692810058594, "KL/rejected_KL_mean": -67.16569519042969, "KL/std": 36.2598991394043, "epoch": 0.90854119425548, "fcm_dpo/beta": 0.033360324800014496, "fcm_dpo/delta": -0.17645078897476196, "fcm_dpo/margin": 28.657991409301758, "fcm_dpo/q_t": 0.3295706808567047, "grad_norm": 18.96521759033203, "learning_rate": 1.2898117173950868e-08, "logits/chosen": 0.5417522192001343, "logits/rejected": 0.4645710289478302, "logps/chosen": -94.10202026367188, "logps/ref_chosen": -55.59432601928711, "logps/ref_rejected": -83.68630981445312, "logps/rejected": -150.8520050048828, "loss": 0.9661, "margin_dpo/margin_mean": 28.65799331665039, "margin_dpo/margin_std": 38.92144012451172, "step": 601 }, { "KL/chosen_KL_mean": -34.5461311340332, "KL/mean": -48.17620086669922, "KL/rejected_KL_mean": -61.8062629699707, "KL/std": 32.06867218017578, "epoch": 0.91005291005291, "fcm_dpo/beta": 0.033003613352775574, "fcm_dpo/delta": -0.10947325825691223, "fcm_dpo/margin": 27.260128021240234, "fcm_dpo/q_t": 0.32961416244506836, "grad_norm": 19.130779266357422, "learning_rate": 1.2482220564763667e-08, "logits/chosen": 0.5375270843505859, "logits/rejected": 0.5018444061279297, "logps/chosen": -90.89531707763672, "logps/ref_chosen": -56.349185943603516, "logps/ref_rejected": -71.9959716796875, "logps/rejected": -133.80223083496094, "loss": 0.9007, "margin_dpo/margin_mean": 27.260128021240234, "margin_dpo/margin_std": 32.607025146484375, "step": 602 }, { "KL/chosen_KL_mean": -38.932960510253906, "KL/mean": -52.24224853515625, "KL/rejected_KL_mean": -65.55152893066406, "KL/std": 31.548992156982422, "epoch": 0.9115646258503401, "fcm_dpo/beta": 0.03229742497205734, "fcm_dpo/delta": -0.06548966467380524, "fcm_dpo/margin": 26.618579864501953, "fcm_dpo/q_t": 0.34004509449005127, "grad_norm": 21.881919860839844, "learning_rate": 1.2072967838448051e-08, "logits/chosen": 0.49233826994895935, "logits/rejected": 0.4304012656211853, "logps/chosen": -92.10134887695312, "logps/ref_chosen": -53.16838836669922, "logps/ref_rejected": -73.8604736328125, "logps/rejected": -139.41201782226562, "loss": 0.9581, "margin_dpo/margin_mean": 26.618579864501953, "margin_dpo/margin_std": 35.89923095703125, "step": 603 }, { "KL/chosen_KL_mean": -38.09632873535156, "KL/mean": -50.37153244018555, "KL/rejected_KL_mean": -62.64673614501953, "KL/std": 33.445926666259766, "epoch": 0.9130763416477702, "fcm_dpo/beta": 0.0323166623711586, "fcm_dpo/delta": 0.0071356454864144325, "fcm_dpo/margin": 24.550411224365234, "fcm_dpo/q_t": 0.35812100768089294, "grad_norm": 20.806612014770508, "learning_rate": 1.1670370442682459e-08, "logits/chosen": 0.4992177486419678, "logits/rejected": 0.500224232673645, "logps/chosen": -110.74574279785156, "logps/ref_chosen": -72.64942169189453, "logps/ref_rejected": -69.8792724609375, "logps/rejected": -132.5260009765625, "loss": 1.0781, "margin_dpo/margin_mean": 24.550411224365234, "margin_dpo/margin_std": 41.21388244628906, "step": 604 }, { "KL/chosen_KL_mean": -45.63953399658203, "KL/mean": -57.18891906738281, "KL/rejected_KL_mean": -68.7383041381836, "KL/std": 32.24746322631836, "epoch": 0.9145880574452003, "fcm_dpo/beta": 0.032349593937397, "fcm_dpo/delta": 0.05652875080704689, "fcm_dpo/margin": 23.098766326904297, "fcm_dpo/q_t": 0.3580436110496521, "grad_norm": 22.0025577545166, "learning_rate": 1.1274439638981532e-08, "logits/chosen": 0.5846168398857117, "logits/rejected": 0.522821843624115, "logps/chosen": -107.25238037109375, "logps/ref_chosen": -61.61284637451172, "logps/ref_rejected": -79.34398651123047, "logps/rejected": -148.08229064941406, "loss": 1.0632, "margin_dpo/margin_mean": 23.098766326904297, "margin_dpo/margin_std": 36.80876922607422, "step": 605 }, { "KL/chosen_KL_mean": -39.366722106933594, "KL/mean": -52.45832824707031, "KL/rejected_KL_mean": -65.54991912841797, "KL/std": 31.701202392578125, "epoch": 0.9160997732426304, "fcm_dpo/beta": 0.03257810324430466, "fcm_dpo/delta": -0.05840463563799858, "fcm_dpo/margin": 26.183189392089844, "fcm_dpo/q_t": 0.3414803743362427, "grad_norm": 19.46663475036621, "learning_rate": 1.0885186502381016e-08, "logits/chosen": 0.5263932943344116, "logits/rejected": 0.45905801653862, "logps/chosen": -93.83096313476562, "logps/ref_chosen": -54.46424102783203, "logps/ref_rejected": -79.62708282470703, "logps/rejected": -145.177001953125, "loss": 0.9523, "margin_dpo/margin_mean": 26.183189392089844, "margin_dpo/margin_std": 34.3382568359375, "step": 606 }, { "KL/chosen_KL_mean": -44.537776947021484, "KL/mean": -58.091129302978516, "KL/rejected_KL_mean": -71.64448547363281, "KL/std": 35.20539855957031, "epoch": 0.9176114890400605, "fcm_dpo/beta": 0.03151046857237816, "fcm_dpo/delta": -0.06450414657592773, "fcm_dpo/margin": 27.106712341308594, "fcm_dpo/q_t": 0.3402349054813385, "grad_norm": 20.594440460205078, "learning_rate": 1.0502621921127774e-08, "logits/chosen": 0.4834175109863281, "logits/rejected": 0.4526243209838867, "logps/chosen": -107.39864349365234, "logps/ref_chosen": -62.86086654663086, "logps/ref_rejected": -72.5501937866211, "logps/rejected": -144.19467163085938, "loss": 0.9645, "margin_dpo/margin_mean": 27.106712341308594, "margin_dpo/margin_std": 35.469757080078125, "step": 607 }, { "KL/chosen_KL_mean": -46.28819274902344, "KL/mean": -56.1732177734375, "KL/rejected_KL_mean": -66.05824279785156, "KL/std": 33.02288055419922, "epoch": 0.9191232048374905, "fcm_dpo/beta": 0.032674677670001984, "fcm_dpo/delta": 0.16450561583042145, "fcm_dpo/margin": 19.770050048828125, "fcm_dpo/q_t": 0.3787013590335846, "grad_norm": 23.87982177734375, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 0.5198084115982056, "logits/rejected": 0.4453343152999878, "logps/chosen": -109.46891784667969, "logps/ref_chosen": -63.18071746826172, "logps/ref_rejected": -99.15888214111328, "logps/rejected": -165.21713256835938, "loss": 1.0811, "margin_dpo/margin_mean": 19.77005386352539, "margin_dpo/margin_std": 33.525691986083984, "step": 608 }, { "KL/chosen_KL_mean": -38.11963653564453, "KL/mean": -53.10977554321289, "KL/rejected_KL_mean": -68.09991455078125, "KL/std": 33.28239440917969, "epoch": 0.9206349206349206, "fcm_dpo/beta": 0.03173757344484329, "fcm_dpo/delta": -0.1720995157957077, "fcm_dpo/margin": 29.980270385742188, "fcm_dpo/q_t": 0.30832743644714355, "grad_norm": 18.805438995361328, "learning_rate": 9.757601041885694e-09, "logits/chosen": 0.6107975244522095, "logits/rejected": 0.5735124349594116, "logps/chosen": -86.74285888671875, "logps/ref_chosen": -48.62322235107422, "logps/ref_rejected": -68.28271484375, "logps/rejected": -136.38262939453125, "loss": 0.8461, "margin_dpo/margin_mean": 29.980270385742188, "margin_dpo/margin_std": 30.857749938964844, "step": 609 }, { "KL/chosen_KL_mean": -44.18012237548828, "KL/mean": -58.19615936279297, "KL/rejected_KL_mean": -72.21221160888672, "KL/std": 35.15657043457031, "epoch": 0.9221466364323507, "fcm_dpo/beta": 0.03127070516347885, "fcm_dpo/delta": -0.08477940410375595, "fcm_dpo/margin": 28.032087326049805, "fcm_dpo/q_t": 0.33978596329689026, "grad_norm": 20.88776969909668, "learning_rate": 9.395165583732379e-09, "logits/chosen": 0.5239899754524231, "logits/rejected": 0.5249122381210327, "logps/chosen": -116.84524536132812, "logps/ref_chosen": -72.66513061523438, "logps/ref_rejected": -87.15310668945312, "logps/rejected": -159.36532592773438, "loss": 0.9586, "margin_dpo/margin_mean": 28.032085418701172, "margin_dpo/margin_std": 37.857051849365234, "step": 610 }, { "KL/chosen_KL_mean": -41.32282257080078, "KL/mean": -51.38006591796875, "KL/rejected_KL_mean": -61.43731689453125, "KL/std": 32.634307861328125, "epoch": 0.9236583522297808, "fcm_dpo/beta": 0.031811684370040894, "fcm_dpo/delta": 0.17101894319057465, "fcm_dpo/margin": 20.11448860168457, "fcm_dpo/q_t": 0.37627357244491577, "grad_norm": 21.9239501953125, "learning_rate": 9.03946036001449e-09, "logits/chosen": 0.5745566487312317, "logits/rejected": 0.5245550870895386, "logps/chosen": -89.63140106201172, "logps/ref_chosen": -48.30857849121094, "logps/ref_rejected": -70.6141128540039, "logps/rejected": -132.0514373779297, "loss": 1.0601, "margin_dpo/margin_mean": 20.114486694335938, "margin_dpo/margin_std": 31.439861297607422, "step": 611 }, { "KL/chosen_KL_mean": -45.562416076660156, "KL/mean": -59.08216857910156, "KL/rejected_KL_mean": -72.6019287109375, "KL/std": 36.2523193359375, "epoch": 0.9251700680272109, "fcm_dpo/beta": 0.03201327472925186, "fcm_dpo/delta": -0.07186618447303772, "fcm_dpo/margin": 27.039508819580078, "fcm_dpo/q_t": 0.33631759881973267, "grad_norm": 19.976383209228516, "learning_rate": 8.690495320571839e-09, "logits/chosen": 0.45067673921585083, "logits/rejected": 0.3812732696533203, "logps/chosen": -106.79397583007812, "logps/ref_chosen": -61.23155975341797, "logps/ref_rejected": -94.37979888916016, "logps/rejected": -166.98171997070312, "loss": 0.9933, "margin_dpo/margin_mean": 27.039508819580078, "margin_dpo/margin_std": 38.99497985839844, "step": 612 }, { "KL/chosen_KL_mean": -37.76877975463867, "KL/mean": -54.28990936279297, "KL/rejected_KL_mean": -70.81103515625, "KL/std": 36.1613883972168, "epoch": 0.926681783824641, "fcm_dpo/beta": 0.030983448028564453, "fcm_dpo/delta": -0.24858498573303223, "fcm_dpo/margin": 33.042259216308594, "fcm_dpo/q_t": 0.30614638328552246, "grad_norm": 17.632335662841797, "learning_rate": 8.348280226706722e-09, "logits/chosen": 0.4682348370552063, "logits/rejected": 0.4623478651046753, "logps/chosen": -91.75189208984375, "logps/ref_chosen": -53.98310852050781, "logps/ref_rejected": -58.32208251953125, "logps/rejected": -129.13311767578125, "loss": 0.8478, "margin_dpo/margin_mean": 33.042259216308594, "margin_dpo/margin_std": 36.33470153808594, "step": 613 }, { "KL/chosen_KL_mean": -43.43042755126953, "KL/mean": -56.76459503173828, "KL/rejected_KL_mean": -70.0987548828125, "KL/std": 31.019271850585938, "epoch": 0.9281934996220711, "fcm_dpo/beta": 0.0304144024848938, "fcm_dpo/delta": -0.012328855693340302, "fcm_dpo/margin": 26.6683349609375, "fcm_dpo/q_t": 0.3338313102722168, "grad_norm": 20.849937438964844, "learning_rate": 8.012824650910937e-09, "logits/chosen": 0.5582789182662964, "logits/rejected": 0.5460919737815857, "logps/chosen": -103.6734619140625, "logps/ref_chosen": -60.24303436279297, "logps/ref_rejected": -72.26258850097656, "logps/rejected": -142.36134338378906, "loss": 0.9255, "margin_dpo/margin_mean": 26.6683349609375, "margin_dpo/margin_std": 31.444583892822266, "step": 614 }, { "KL/chosen_KL_mean": -44.83760070800781, "KL/mean": -59.19696807861328, "KL/rejected_KL_mean": -73.55632019042969, "KL/std": 37.00213623046875, "epoch": 0.9297052154195011, "fcm_dpo/beta": 0.0297965407371521, "fcm_dpo/delta": -0.062363844364881516, "fcm_dpo/margin": 28.718719482421875, "fcm_dpo/q_t": 0.3398233652114868, "grad_norm": 21.655941009521484, "learning_rate": 7.684137976598088e-09, "logits/chosen": 0.514799952507019, "logits/rejected": 0.4654197692871094, "logps/chosen": -116.93228149414062, "logps/ref_chosen": -72.09467315673828, "logps/ref_rejected": -104.02980041503906, "logps/rejected": -177.58612060546875, "loss": 0.9957, "margin_dpo/margin_mean": 28.718719482421875, "margin_dpo/margin_std": 40.58180236816406, "step": 615 }, { "KL/chosen_KL_mean": -44.345245361328125, "KL/mean": -55.76415252685547, "KL/rejected_KL_mean": -67.18305969238281, "KL/std": 32.98649215698242, "epoch": 0.9312169312169312, "fcm_dpo/beta": 0.030331624671816826, "fcm_dpo/delta": 0.11543683707714081, "fcm_dpo/margin": 22.837812423706055, "fcm_dpo/q_t": 0.370138943195343, "grad_norm": 20.064226150512695, "learning_rate": 7.36222939784098e-09, "logits/chosen": 0.5760443210601807, "logits/rejected": 0.49106380343437195, "logps/chosen": -102.87596893310547, "logps/ref_chosen": -58.530723571777344, "logps/ref_rejected": -75.48025512695312, "logps/rejected": -142.66331481933594, "loss": 1.0384, "margin_dpo/margin_mean": 22.837814331054688, "margin_dpo/margin_std": 35.295989990234375, "step": 616 }, { "KL/chosen_KL_mean": -48.91425323486328, "KL/mean": -60.303802490234375, "KL/rejected_KL_mean": -71.693359375, "KL/std": 34.04789733886719, "epoch": 0.9327286470143613, "fcm_dpo/beta": 0.03068365529179573, "fcm_dpo/delta": 0.10637275129556656, "fcm_dpo/margin": 22.77910614013672, "fcm_dpo/q_t": 0.36347612738609314, "grad_norm": 21.133838653564453, "learning_rate": 7.047107919114586e-09, "logits/chosen": 0.5448430776596069, "logits/rejected": 0.4960317611694336, "logps/chosen": -106.5229263305664, "logps/ref_chosen": -57.608673095703125, "logps/ref_rejected": -81.22109985351562, "logps/rejected": -152.91445922851562, "loss": 1.0212, "margin_dpo/margin_mean": 22.77910614013672, "margin_dpo/margin_std": 32.204254150390625, "step": 617 }, { "KL/chosen_KL_mean": -41.57891845703125, "KL/mean": -54.18458557128906, "KL/rejected_KL_mean": -66.79025268554688, "KL/std": 35.886566162109375, "epoch": 0.9342403628117913, "fcm_dpo/beta": 0.03101673163473606, "fcm_dpo/delta": 0.017431527376174927, "fcm_dpo/margin": 25.21134376525879, "fcm_dpo/q_t": 0.349711537361145, "grad_norm": 21.669893264770508, "learning_rate": 6.738782355044048e-09, "logits/chosen": 0.5267188549041748, "logits/rejected": 0.42298799753189087, "logps/chosen": -98.27485656738281, "logps/ref_chosen": -56.69594192504883, "logps/ref_rejected": -85.92362976074219, "logps/rejected": -152.71389770507812, "loss": 0.9609, "margin_dpo/margin_mean": 25.211345672607422, "margin_dpo/margin_std": 33.00782775878906, "step": 618 }, { "KL/chosen_KL_mean": -39.777366638183594, "KL/mean": -52.436004638671875, "KL/rejected_KL_mean": -65.09465026855469, "KL/std": 32.83123016357422, "epoch": 0.9357520786092215, "fcm_dpo/beta": 0.03125225752592087, "fcm_dpo/delta": 0.009025529026985168, "fcm_dpo/margin": 25.317279815673828, "fcm_dpo/q_t": 0.348273903131485, "grad_norm": 18.896320343017578, "learning_rate": 6.437261330158206e-09, "logits/chosen": 0.6078675985336304, "logits/rejected": 0.5344126224517822, "logps/chosen": -93.83578491210938, "logps/ref_chosen": -54.05841827392578, "logps/ref_rejected": -83.55493927001953, "logps/rejected": -148.64959716796875, "loss": 0.965, "margin_dpo/margin_mean": 25.317279815673828, "margin_dpo/margin_std": 33.63002014160156, "step": 619 }, { "KL/chosen_KL_mean": -39.8848991394043, "KL/mean": -51.43940734863281, "KL/rejected_KL_mean": -62.99391555786133, "KL/std": 31.84799575805664, "epoch": 0.9372637944066515, "fcm_dpo/beta": 0.03225833922624588, "fcm_dpo/delta": 0.05094073712825775, "fcm_dpo/margin": 23.109012603759766, "fcm_dpo/q_t": 0.36446672677993774, "grad_norm": 25.099145889282227, "learning_rate": 6.142553278648238e-09, "logits/chosen": 0.5527920722961426, "logits/rejected": 0.5471271276473999, "logps/chosen": -103.2546157836914, "logps/ref_chosen": -63.36971664428711, "logps/ref_rejected": -65.68269348144531, "logps/rejected": -128.67660522460938, "loss": 1.038, "margin_dpo/margin_mean": 23.109012603759766, "margin_dpo/margin_std": 34.14104461669922, "step": 620 }, { "KL/chosen_KL_mean": -42.45032501220703, "KL/mean": -53.225311279296875, "KL/rejected_KL_mean": -64.00030517578125, "KL/std": 32.297000885009766, "epoch": 0.9387755102040817, "fcm_dpo/beta": 0.032468922436237335, "fcm_dpo/delta": 0.10621249675750732, "fcm_dpo/margin": 21.549976348876953, "fcm_dpo/q_t": 0.3690629005432129, "grad_norm": 22.792800903320312, "learning_rate": 5.854666444131934e-09, "logits/chosen": 0.5810672044754028, "logits/rejected": 0.47503602504730225, "logps/chosen": -94.77154541015625, "logps/ref_chosen": -52.321224212646484, "logps/ref_rejected": -88.09001159667969, "logps/rejected": -152.09030151367188, "loss": 1.0761, "margin_dpo/margin_mean": 21.549978256225586, "margin_dpo/margin_std": 35.361324310302734, "step": 621 }, { "KL/chosen_KL_mean": -46.312774658203125, "KL/mean": -57.30717468261719, "KL/rejected_KL_mean": -68.30157470703125, "KL/std": 35.315643310546875, "epoch": 0.9402872260015117, "fcm_dpo/beta": 0.0329909548163414, "fcm_dpo/delta": 0.07938125729560852, "fcm_dpo/margin": 21.988800048828125, "fcm_dpo/q_t": 0.3594774603843689, "grad_norm": 20.948509216308594, "learning_rate": 5.573608879422875e-09, "logits/chosen": 0.49009227752685547, "logits/rejected": 0.4497200846672058, "logps/chosen": -106.17823791503906, "logps/ref_chosen": -59.86545944213867, "logps/ref_rejected": -81.86668395996094, "logps/rejected": -150.1682586669922, "loss": 0.9982, "margin_dpo/margin_mean": 21.988800048828125, "margin_dpo/margin_std": 30.735015869140625, "step": 622 }, { "KL/chosen_KL_mean": -49.466217041015625, "KL/mean": -62.440582275390625, "KL/rejected_KL_mean": -75.41496276855469, "KL/std": 35.49492263793945, "epoch": 0.9417989417989417, "fcm_dpo/beta": 0.03263657167553902, "fcm_dpo/delta": -0.051398035138845444, "fcm_dpo/margin": 25.94874382019043, "fcm_dpo/q_t": 0.3343699276447296, "grad_norm": 18.90989875793457, "learning_rate": 5.299388446305342e-09, "logits/chosen": 0.5144941210746765, "logits/rejected": 0.44776612520217896, "logps/chosen": -116.83467102050781, "logps/ref_chosen": -67.36846160888672, "logps/ref_rejected": -82.02733612060547, "logps/rejected": -157.44229125976562, "loss": 0.9179, "margin_dpo/margin_mean": 25.948741912841797, "margin_dpo/margin_std": 31.110553741455078, "step": 623 }, { "KL/chosen_KL_mean": -40.11148452758789, "KL/mean": -53.995384216308594, "KL/rejected_KL_mean": -67.8792724609375, "KL/std": 34.82365417480469, "epoch": 0.9433106575963719, "fcm_dpo/beta": 0.032003238797187805, "fcm_dpo/delta": -0.09814178943634033, "fcm_dpo/margin": 27.767793655395508, "fcm_dpo/q_t": 0.33920931816101074, "grad_norm": 19.443069458007812, "learning_rate": 5.03201281531429e-09, "logits/chosen": 0.5250840187072754, "logits/rejected": 0.42520371079444885, "logps/chosen": -91.13803100585938, "logps/ref_chosen": -51.02655029296875, "logps/ref_rejected": -76.49203491210938, "logps/rejected": -144.37130737304688, "loss": 0.9565, "margin_dpo/margin_mean": 27.767793655395508, "margin_dpo/margin_std": 37.009613037109375, "step": 624 }, { "KL/chosen_KL_mean": -43.644832611083984, "KL/mean": -53.9796028137207, "KL/rejected_KL_mean": -64.31437683105469, "KL/std": 33.18622589111328, "epoch": 0.9448223733938019, "fcm_dpo/beta": 0.032724544405937195, "fcm_dpo/delta": 0.1318553388118744, "fcm_dpo/margin": 20.669544219970703, "fcm_dpo/q_t": 0.3774385452270508, "grad_norm": 20.581443786621094, "learning_rate": 4.7714894655209174e-09, "logits/chosen": 0.5876431465148926, "logits/rejected": 0.5019428133964539, "logps/chosen": -97.85244750976562, "logps/ref_chosen": -54.20761489868164, "logps/ref_rejected": -84.93669128417969, "logps/rejected": -149.25106811523438, "loss": 1.1065, "margin_dpo/margin_mean": 20.669546127319336, "margin_dpo/margin_std": 36.78896713256836, "step": 625 }, { "KL/chosen_KL_mean": -40.374717712402344, "KL/mean": -54.71905517578125, "KL/rejected_KL_mean": -69.06339263916016, "KL/std": 36.232276916503906, "epoch": 0.9463340891912321, "fcm_dpo/beta": 0.03213457390666008, "fcm_dpo/delta": -0.13456374406814575, "fcm_dpo/margin": 28.68867301940918, "fcm_dpo/q_t": 0.3437504768371582, "grad_norm": 20.765810012817383, "learning_rate": 4.517825684323323e-09, "logits/chosen": 0.6345266699790955, "logits/rejected": 0.5021851658821106, "logps/chosen": -85.43673706054688, "logps/ref_chosen": -45.06201934814453, "logps/ref_rejected": -89.66368103027344, "logps/rejected": -158.72708129882812, "loss": 1.0454, "margin_dpo/margin_mean": 28.688674926757812, "margin_dpo/margin_std": 44.237403869628906, "step": 626 }, { "KL/chosen_KL_mean": -43.914947509765625, "KL/mean": -60.0136833190918, "KL/rejected_KL_mean": -76.11241149902344, "KL/std": 34.76005554199219, "epoch": 0.9478458049886621, "fcm_dpo/beta": 0.03084169700741768, "fcm_dpo/delta": -0.21548502147197723, "fcm_dpo/margin": 32.19746398925781, "fcm_dpo/q_t": 0.3146284818649292, "grad_norm": 19.66023063659668, "learning_rate": 4.271028567242818e-09, "logits/chosen": 0.4840816259384155, "logits/rejected": 0.3665580451488495, "logps/chosen": -102.70600891113281, "logps/ref_chosen": -58.791053771972656, "logps/ref_rejected": -94.90802001953125, "logps/rejected": -171.0204315185547, "loss": 0.897, "margin_dpo/margin_mean": 32.19746398925781, "margin_dpo/margin_std": 38.98145294189453, "step": 627 }, { "KL/chosen_KL_mean": -37.61805725097656, "KL/mean": -53.94782257080078, "KL/rejected_KL_mean": -70.277587890625, "KL/std": 35.664676666259766, "epoch": 0.9493575207860923, "fcm_dpo/beta": 0.03022720292210579, "fcm_dpo/delta": -0.21070462465286255, "fcm_dpo/margin": 32.65953063964844, "fcm_dpo/q_t": 0.3106863796710968, "grad_norm": 21.5578556060791, "learning_rate": 4.0311050177251895e-09, "logits/chosen": 0.5580893158912659, "logits/rejected": 0.5208135843276978, "logps/chosen": -90.421630859375, "logps/ref_chosen": -52.80357360839844, "logps/ref_rejected": -76.49468994140625, "logps/rejected": -146.77227783203125, "loss": 0.9537, "margin_dpo/margin_mean": 32.65953063964844, "margin_dpo/margin_std": 39.72731018066406, "step": 628 }, { "KL/chosen_KL_mean": -40.470176696777344, "KL/mean": -53.423011779785156, "KL/rejected_KL_mean": -66.37583923339844, "KL/std": 33.411163330078125, "epoch": 0.9508692365835223, "fcm_dpo/beta": 0.029644351452589035, "fcm_dpo/delta": 0.034037984907627106, "fcm_dpo/margin": 25.905670166015625, "fcm_dpo/q_t": 0.3487117290496826, "grad_norm": 18.10139274597168, "learning_rate": 3.798061746947995e-09, "logits/chosen": 0.5535793304443359, "logits/rejected": 0.5457119941711426, "logps/chosen": -111.18766784667969, "logps/ref_chosen": -70.71749877929688, "logps/ref_rejected": -78.96273803710938, "logps/rejected": -145.33859252929688, "loss": 0.9226, "margin_dpo/margin_mean": 25.905670166015625, "margin_dpo/margin_std": 30.098609924316406, "step": 629 }, { "KL/chosen_KL_mean": -42.22637939453125, "KL/mean": -56.090721130371094, "KL/rejected_KL_mean": -69.95506286621094, "KL/std": 37.00312042236328, "epoch": 0.9523809523809523, "fcm_dpo/beta": 0.029658004641532898, "fcm_dpo/delta": -0.025315992534160614, "fcm_dpo/margin": 27.728683471679688, "fcm_dpo/q_t": 0.3414066731929779, "grad_norm": 15.803033828735352, "learning_rate": 3.5719052736323806e-09, "logits/chosen": 0.49068087339401245, "logits/rejected": 0.4432094693183899, "logps/chosen": -98.42779541015625, "logps/ref_chosen": -56.201412200927734, "logps/ref_rejected": -74.69807434082031, "logps/rejected": -144.65313720703125, "loss": 0.9297, "margin_dpo/margin_mean": 27.728683471679688, "margin_dpo/margin_std": 33.98920822143555, "step": 630 }, { "KL/chosen_KL_mean": -38.312564849853516, "KL/mean": -54.70954895019531, "KL/rejected_KL_mean": -71.10653686523438, "KL/std": 36.32965087890625, "epoch": 0.9538926681783825, "fcm_dpo/beta": 0.02823655493557453, "fcm_dpo/delta": -0.14881953597068787, "fcm_dpo/margin": 32.793975830078125, "fcm_dpo/q_t": 0.3277819752693176, "grad_norm": 19.586389541625977, "learning_rate": 3.352641923861144e-09, "logits/chosen": 0.6357178688049316, "logits/rejected": 0.5238248109817505, "logps/chosen": -97.13316345214844, "logps/ref_chosen": -58.82059860229492, "logps/ref_rejected": -96.51437377929688, "logps/rejected": -167.62091064453125, "loss": 0.9319, "margin_dpo/margin_mean": 32.793975830078125, "margin_dpo/margin_std": 40.02543258666992, "step": 631 }, { "KL/chosen_KL_mean": -40.46467590332031, "KL/mean": -57.193267822265625, "KL/rejected_KL_mean": -73.92185974121094, "KL/std": 34.62277603149414, "epoch": 0.9554043839758125, "fcm_dpo/beta": 0.027888979762792587, "fcm_dpo/delta": -0.14702674746513367, "fcm_dpo/margin": 33.45717239379883, "fcm_dpo/q_t": 0.31367796659469604, "grad_norm": 16.896800994873047, "learning_rate": 3.140277830901428e-09, "logits/chosen": 0.5784220695495605, "logits/rejected": 0.5536450147628784, "logps/chosen": -99.250732421875, "logps/ref_chosen": -58.786048889160156, "logps/ref_rejected": -67.21923828125, "logps/rejected": -141.14109802246094, "loss": 0.8562, "margin_dpo/margin_mean": 33.457176208496094, "margin_dpo/margin_std": 35.489871978759766, "step": 632 }, { "KL/chosen_KL_mean": -37.61022186279297, "KL/mean": -49.89120101928711, "KL/rejected_KL_mean": -62.17218780517578, "KL/std": 33.52429962158203, "epoch": 0.9569160997732427, "fcm_dpo/beta": 0.0282943956553936, "fcm_dpo/delta": 0.11205098032951355, "fcm_dpo/margin": 24.561969757080078, "fcm_dpo/q_t": 0.36849457025527954, "grad_norm": 16.958282470703125, "learning_rate": 2.9348189350335007e-09, "logits/chosen": 0.5199195742607117, "logits/rejected": 0.45689111948013306, "logps/chosen": -89.74041748046875, "logps/ref_chosen": -52.13019561767578, "logps/ref_rejected": -67.23016357421875, "logps/rejected": -129.40234375, "loss": 1.0335, "margin_dpo/margin_mean": 24.561969757080078, "margin_dpo/margin_std": 37.097103118896484, "step": 633 }, { "KL/chosen_KL_mean": -43.55609130859375, "KL/mean": -50.937355041503906, "KL/rejected_KL_mean": -58.31862258911133, "KL/std": 31.610820770263672, "epoch": 0.9584278155706727, "fcm_dpo/beta": 0.030082430690526962, "fcm_dpo/delta": 0.37042272090911865, "fcm_dpo/margin": 14.762529373168945, "fcm_dpo/q_t": 0.420715868473053, "grad_norm": 26.369741439819336, "learning_rate": 2.736270983384276e-09, "logits/chosen": 0.5791378021240234, "logits/rejected": 0.5895206928253174, "logps/chosen": -104.535888671875, "logps/ref_chosen": -60.97979736328125, "logps/ref_rejected": -58.50825119018555, "logps/rejected": -116.82687377929688, "loss": 1.2388, "margin_dpo/margin_mean": 14.762530326843262, "margin_dpo/margin_std": 35.889408111572266, "step": 634 }, { "KL/chosen_KL_mean": -53.46791076660156, "KL/mean": -63.21743392944336, "KL/rejected_KL_mean": -72.96694946289062, "KL/std": 33.996768951416016, "epoch": 0.9599395313681028, "fcm_dpo/beta": 0.0314885713160038, "fcm_dpo/delta": 0.1977309286594391, "fcm_dpo/margin": 19.499042510986328, "fcm_dpo/q_t": 0.39308854937553406, "grad_norm": 22.93577766418457, "learning_rate": 2.5446395297668287e-09, "logits/chosen": 0.4295782148838043, "logits/rejected": 0.365118145942688, "logps/chosen": -119.44093322753906, "logps/ref_chosen": -65.9730224609375, "logps/ref_rejected": -85.61317443847656, "logps/rejected": -158.5801239013672, "loss": 1.2012, "margin_dpo/margin_mean": 19.499042510986328, "margin_dpo/margin_std": 40.97630310058594, "step": 635 }, { "KL/chosen_KL_mean": -40.943885803222656, "KL/mean": -55.528175354003906, "KL/rejected_KL_mean": -70.11245727539062, "KL/std": 35.467247009277344, "epoch": 0.9614512471655329, "fcm_dpo/beta": 0.0310262069106102, "fcm_dpo/delta": -0.11683852970600128, "fcm_dpo/margin": 29.168590545654297, "fcm_dpo/q_t": 0.32491472363471985, "grad_norm": 18.515304565429688, "learning_rate": 2.359929934524829e-09, "logits/chosen": 0.5118346810340881, "logits/rejected": 0.42026370763778687, "logps/chosen": -90.08405303955078, "logps/ref_chosen": -49.140167236328125, "logps/ref_rejected": -81.26971435546875, "logps/rejected": -151.38217163085938, "loss": 0.8938, "margin_dpo/margin_mean": 29.168590545654297, "margin_dpo/margin_std": 34.66674041748047, "step": 636 }, { "KL/chosen_KL_mean": -53.92256164550781, "KL/mean": -64.56072998046875, "KL/rejected_KL_mean": -75.19889831542969, "KL/std": 32.787139892578125, "epoch": 0.9629629629629629, "fcm_dpo/beta": 0.031797170639038086, "fcm_dpo/delta": 0.13112817704677582, "fcm_dpo/margin": 21.27634048461914, "fcm_dpo/q_t": 0.3831368088722229, "grad_norm": 23.125991821289062, "learning_rate": 2.1821473643827137e-09, "logits/chosen": 0.5066337585449219, "logits/rejected": 0.4331105053424835, "logps/chosen": -127.61914825439453, "logps/ref_chosen": -73.69658660888672, "logps/ref_rejected": -83.01487731933594, "logps/rejected": -158.21377563476562, "loss": 1.1488, "margin_dpo/margin_mean": 21.27634048461914, "margin_dpo/margin_std": 40.99974822998047, "step": 637 }, { "KL/chosen_KL_mean": -48.4141731262207, "KL/mean": -61.80738067626953, "KL/rejected_KL_mean": -75.20059204101562, "KL/std": 37.1383171081543, "epoch": 0.9644746787603931, "fcm_dpo/beta": 0.03168731927871704, "fcm_dpo/delta": -0.05334305390715599, "fcm_dpo/margin": 26.78641700744629, "fcm_dpo/q_t": 0.34307920932769775, "grad_norm": 21.305564880371094, "learning_rate": 2.0112967923011646e-09, "logits/chosen": 0.5187181830406189, "logits/rejected": 0.4680929481983185, "logps/chosen": -111.19575500488281, "logps/ref_chosen": -62.78158187866211, "logps/ref_rejected": -85.40478515625, "logps/rejected": -160.60537719726562, "loss": 0.9674, "margin_dpo/margin_mean": 26.786415100097656, "margin_dpo/margin_std": 36.208106994628906, "step": 638 }, { "KL/chosen_KL_mean": -42.945159912109375, "KL/mean": -56.8536262512207, "KL/rejected_KL_mean": -70.76209259033203, "KL/std": 36.39090347290039, "epoch": 0.9659863945578231, "fcm_dpo/beta": 0.03127972036600113, "fcm_dpo/delta": -0.07683076709508896, "fcm_dpo/margin": 27.81693458557129, "fcm_dpo/q_t": 0.33929571509361267, "grad_norm": 21.127824783325195, "learning_rate": 1.847382997337943e-09, "logits/chosen": 0.5249953269958496, "logits/rejected": 0.41127103567123413, "logps/chosen": -96.71174621582031, "logps/ref_chosen": -53.76658630371094, "logps/ref_rejected": -72.30009460449219, "logps/rejected": -143.06219482421875, "loss": 0.9446, "margin_dpo/margin_mean": 27.81693458557129, "margin_dpo/margin_std": 36.36324691772461, "step": 639 }, { "KL/chosen_KL_mean": -44.95122528076172, "KL/mean": -56.254302978515625, "KL/rejected_KL_mean": -67.55738830566406, "KL/std": 34.88379669189453, "epoch": 0.9674981103552532, "fcm_dpo/beta": 0.03140745311975479, "fcm_dpo/delta": 0.0969884991645813, "fcm_dpo/margin": 22.606157302856445, "fcm_dpo/q_t": 0.35654008388519287, "grad_norm": 20.348535537719727, "learning_rate": 1.690410564514244e-09, "logits/chosen": 0.5839003920555115, "logits/rejected": 0.5229287147521973, "logps/chosen": -96.36900329589844, "logps/ref_chosen": -51.41777801513672, "logps/ref_rejected": -77.27879333496094, "logps/rejected": -144.836181640625, "loss": 1.0716, "margin_dpo/margin_mean": 22.606155395507812, "margin_dpo/margin_std": 36.692012786865234, "step": 640 }, { "KL/chosen_KL_mean": -45.83881378173828, "KL/mean": -57.58781814575195, "KL/rejected_KL_mean": -69.33682250976562, "KL/std": 32.90306854248047, "epoch": 0.9690098261526833, "fcm_dpo/beta": 0.03183242306113243, "fcm_dpo/delta": 0.0562346875667572, "fcm_dpo/margin": 23.498010635375977, "fcm_dpo/q_t": 0.35128700733184814, "grad_norm": 21.091442108154297, "learning_rate": 1.5403838846864692e-09, "logits/chosen": 0.5104883909225464, "logits/rejected": 0.4826532006263733, "logps/chosen": -116.89347839355469, "logps/ref_chosen": -71.0546646118164, "logps/ref_rejected": -82.2440185546875, "logps/rejected": -151.58084106445312, "loss": 0.9694, "margin_dpo/margin_mean": 23.498008728027344, "margin_dpo/margin_std": 31.388883590698242, "step": 641 }, { "KL/chosen_KL_mean": -48.326541900634766, "KL/mean": -58.25520706176758, "KL/rejected_KL_mean": -68.18387603759766, "KL/std": 31.67426300048828, "epoch": 0.9705215419501134, "fcm_dpo/beta": 0.032394275069236755, "fcm_dpo/delta": 0.16641943156719208, "fcm_dpo/margin": 19.857332229614258, "fcm_dpo/q_t": 0.38337334990501404, "grad_norm": 25.411165237426758, "learning_rate": 1.3973071544233218e-09, "logits/chosen": 0.4777846038341522, "logits/rejected": 0.4858616590499878, "logps/chosen": -117.25581359863281, "logps/ref_chosen": -68.92927551269531, "logps/ref_rejected": -70.85682678222656, "logps/rejected": -139.04071044921875, "loss": 1.1461, "margin_dpo/margin_mean": 19.857332229614258, "margin_dpo/margin_std": 36.84928894042969, "step": 642 }, { "KL/chosen_KL_mean": -46.31542205810547, "KL/mean": -57.94270324707031, "KL/rejected_KL_mean": -69.56997680664062, "KL/std": 32.99165725708008, "epoch": 0.9720332577475435, "fcm_dpo/beta": 0.03313422203063965, "fcm_dpo/delta": 0.03186196833848953, "fcm_dpo/margin": 23.254549026489258, "fcm_dpo/q_t": 0.36055275797843933, "grad_norm": 36.48023986816406, "learning_rate": 1.261184375888541e-09, "logits/chosen": 0.460068941116333, "logits/rejected": 0.3696553409099579, "logps/chosen": -111.62445831298828, "logps/ref_chosen": -65.30903625488281, "logps/ref_rejected": -83.61613464355469, "logps/rejected": -153.1861114501953, "loss": 1.0505, "margin_dpo/margin_mean": 23.25455093383789, "margin_dpo/margin_std": 36.687599182128906, "step": 643 }, { "KL/chosen_KL_mean": -41.07984924316406, "KL/mean": -51.94877624511719, "KL/rejected_KL_mean": -62.81770324707031, "KL/std": 33.44035339355469, "epoch": 0.9735449735449735, "fcm_dpo/beta": 0.03386765718460083, "fcm_dpo/delta": 0.06720557808876038, "fcm_dpo/margin": 21.737850189208984, "fcm_dpo/q_t": 0.3682977557182312, "grad_norm": 21.03730010986328, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 0.5970633029937744, "logits/rejected": 0.5632849931716919, "logps/chosen": -92.08245086669922, "logps/ref_chosen": -51.002601623535156, "logps/ref_rejected": -64.46372985839844, "logps/rejected": -127.28143310546875, "loss": 1.1474, "margin_dpo/margin_mean": 21.737850189208984, "margin_dpo/margin_std": 40.25080108642578, "step": 644 }, { "KL/chosen_KL_mean": -45.624202728271484, "KL/mean": -57.04932403564453, "KL/rejected_KL_mean": -68.47444152832031, "KL/std": 34.80279541015625, "epoch": 0.9750566893424036, "fcm_dpo/beta": 0.03370298445224762, "fcm_dpo/delta": 0.03188609704375267, "fcm_dpo/margin": 22.850234985351562, "fcm_dpo/q_t": 0.3569079339504242, "grad_norm": 22.33905029296875, "learning_rate": 1.0098157099674987e-09, "logits/chosen": 0.47687435150146484, "logits/rejected": 0.4549427032470703, "logps/chosen": -106.58761596679688, "logps/ref_chosen": -60.963409423828125, "logps/ref_rejected": -69.73353576660156, "logps/rejected": -138.20797729492188, "loss": 1.019, "margin_dpo/margin_mean": 22.850234985351562, "margin_dpo/margin_std": 34.275508880615234, "step": 645 }, { "KL/chosen_KL_mean": -48.04452133178711, "KL/mean": -59.946693420410156, "KL/rejected_KL_mean": -71.84886169433594, "KL/std": 37.24139404296875, "epoch": 0.9765684051398337, "fcm_dpo/beta": 0.03389802575111389, "fcm_dpo/delta": -0.007534122560173273, "fcm_dpo/margin": 23.804344177246094, "fcm_dpo/q_t": 0.3509964048862457, "grad_norm": 22.17904281616211, "learning_rate": 8.945768539031783e-10, "logits/chosen": 0.5630219578742981, "logits/rejected": 0.5091748237609863, "logps/chosen": -110.3345947265625, "logps/ref_chosen": -62.290069580078125, "logps/ref_rejected": -85.54812622070312, "logps/rejected": -157.39698791503906, "loss": 1.0346, "margin_dpo/margin_mean": 23.804344177246094, "margin_dpo/margin_std": 36.64860916137695, "step": 646 }, { "KL/chosen_KL_mean": -47.405487060546875, "KL/mean": -63.31657028198242, "KL/rejected_KL_mean": -79.22764587402344, "KL/std": 36.01445007324219, "epoch": 0.9780801209372638, "fcm_dpo/beta": 0.032675907015800476, "fcm_dpo/delta": -0.2671160101890564, "fcm_dpo/margin": 31.82217025756836, "fcm_dpo/q_t": 0.30921974778175354, "grad_norm": 23.366548538208008, "learning_rate": 7.863060120144316e-10, "logits/chosen": 0.576119601726532, "logits/rejected": 0.4795302748680115, "logps/chosen": -114.92135620117188, "logps/ref_chosen": -67.515869140625, "logps/ref_rejected": -101.50871276855469, "logps/rejected": -180.73635864257812, "loss": 0.8618, "margin_dpo/margin_mean": 31.82217025756836, "margin_dpo/margin_std": 36.936439514160156, "step": 647 }, { "KL/chosen_KL_mean": -47.49691390991211, "KL/mean": -60.25159454345703, "KL/rejected_KL_mean": -73.00627136230469, "KL/std": 32.82928466796875, "epoch": 0.9795918367346939, "fcm_dpo/beta": 0.03184132277965546, "fcm_dpo/delta": -0.01447652280330658, "fcm_dpo/margin": 25.509355545043945, "fcm_dpo/q_t": 0.34164726734161377, "grad_norm": 17.782150268554688, "learning_rate": 6.850062128694045e-10, "logits/chosen": 0.49999916553497314, "logits/rejected": 0.4332047700881958, "logps/chosen": -112.09284973144531, "logps/ref_chosen": -64.59593963623047, "logps/ref_rejected": -83.384033203125, "logps/rejected": -156.3903045654297, "loss": 1.0102, "margin_dpo/margin_mean": 25.509355545043945, "margin_dpo/margin_std": 36.32646942138672, "step": 648 }, { "KL/chosen_KL_mean": -44.42068099975586, "KL/mean": -57.07667541503906, "KL/rejected_KL_mean": -69.732666015625, "KL/std": 33.58146286010742, "epoch": 0.981103552532124, "fcm_dpo/beta": 0.031717028468847275, "fcm_dpo/delta": -0.004996992647647858, "fcm_dpo/margin": 25.311986923217773, "fcm_dpo/q_t": 0.35203224420547485, "grad_norm": 25.448650360107422, "learning_rate": 5.906802900412788e-10, "logits/chosen": 0.5485668182373047, "logits/rejected": 0.48975634574890137, "logps/chosen": -93.73033142089844, "logps/ref_chosen": -49.30964660644531, "logps/ref_rejected": -73.73710632324219, "logps/rejected": -143.4697723388672, "loss": 1.0337, "margin_dpo/margin_mean": 25.31198501586914, "margin_dpo/margin_std": 37.88409423828125, "step": 649 }, { "KL/chosen_KL_mean": -42.31673049926758, "KL/mean": -54.86392593383789, "KL/rejected_KL_mean": -67.41111755371094, "KL/std": 35.4691162109375, "epoch": 0.982615268329554, "fcm_dpo/beta": 0.03204188868403435, "fcm_dpo/delta": -0.004477673210203648, "fcm_dpo/margin": 25.094390869140625, "fcm_dpo/q_t": 0.3569669723510742, "grad_norm": 25.507871627807617, "learning_rate": 5.033308820289184e-10, "logits/chosen": 0.6226690411567688, "logits/rejected": 0.5567650198936462, "logps/chosen": -97.37998962402344, "logps/ref_chosen": -55.06325912475586, "logps/ref_rejected": -77.39610290527344, "logps/rejected": -144.80722045898438, "loss": 1.0672, "margin_dpo/margin_mean": 25.094392776489258, "margin_dpo/margin_std": 40.48807144165039, "step": 650 }, { "KL/chosen_KL_mean": -41.40019226074219, "KL/mean": -52.56417465209961, "KL/rejected_KL_mean": -63.7281608581543, "KL/std": 34.437225341796875, "epoch": 0.9841269841269841, "fcm_dpo/beta": 0.032618556171655655, "fcm_dpo/delta": 0.0751466155052185, "fcm_dpo/margin": 22.327966690063477, "fcm_dpo/q_t": 0.3638933598995209, "grad_norm": 21.54810905456543, "learning_rate": 4.2296043218295606e-10, "logits/chosen": 0.5694529414176941, "logits/rejected": 0.4965120255947113, "logps/chosen": -95.4653549194336, "logps/ref_chosen": -54.065162658691406, "logps/ref_rejected": -77.79080200195312, "logps/rejected": -141.5189666748047, "loss": 1.0293, "margin_dpo/margin_mean": 22.327966690063477, "margin_dpo/margin_std": 33.20568084716797, "step": 651 }, { "KL/chosen_KL_mean": -49.35215759277344, "KL/mean": -60.68022918701172, "KL/rejected_KL_mean": -72.00830078125, "KL/std": 34.67872619628906, "epoch": 0.9856386999244142, "fcm_dpo/beta": 0.03263973072171211, "fcm_dpo/delta": 0.06523266434669495, "fcm_dpo/margin": 22.656145095825195, "fcm_dpo/q_t": 0.3670682907104492, "grad_norm": 22.150224685668945, "learning_rate": 3.4957118863768176e-10, "logits/chosen": 0.5378252863883972, "logits/rejected": 0.48126500844955444, "logps/chosen": -112.99246215820312, "logps/ref_chosen": -63.64030456542969, "logps/ref_rejected": -78.86882019042969, "logps/rejected": -150.87713623046875, "loss": 1.0563, "margin_dpo/margin_mean": 22.656145095825195, "margin_dpo/margin_std": 36.34861755371094, "step": 652 }, { "KL/chosen_KL_mean": -43.989662170410156, "KL/mean": -57.336952209472656, "KL/rejected_KL_mean": -70.68423461914062, "KL/std": 34.78733825683594, "epoch": 0.9871504157218443, "fcm_dpo/beta": 0.03256607800722122, "fcm_dpo/delta": -0.07587750256061554, "fcm_dpo/margin": 26.694576263427734, "fcm_dpo/q_t": 0.34083616733551025, "grad_norm": 22.887805938720703, "learning_rate": 2.831652042480093e-10, "logits/chosen": 0.5100860595703125, "logits/rejected": 0.4632833003997803, "logps/chosen": -105.65803527832031, "logps/ref_chosen": -61.668373107910156, "logps/ref_rejected": -73.83012390136719, "logps/rejected": -144.51437377929688, "loss": 0.9963, "margin_dpo/margin_mean": 26.694576263427734, "margin_dpo/margin_std": 38.514312744140625, "step": 653 }, { "KL/chosen_KL_mean": -45.31245803833008, "KL/mean": -55.97831344604492, "KL/rejected_KL_mean": -66.64417266845703, "KL/std": 36.80150604248047, "epoch": 0.9886621315192744, "fcm_dpo/beta": 0.03246813267469406, "fcm_dpo/delta": 0.11256799101829529, "fcm_dpo/margin": 21.331714630126953, "fcm_dpo/q_t": 0.3787607252597809, "grad_norm": 22.46723747253418, "learning_rate": 2.2374433653205016e-10, "logits/chosen": 0.5046300888061523, "logits/rejected": 0.40765130519866943, "logps/chosen": -102.88072204589844, "logps/ref_chosen": -57.568267822265625, "logps/ref_rejected": -87.74789428710938, "logps/rejected": -154.39205932617188, "loss": 1.0765, "margin_dpo/margin_mean": 21.33171272277832, "margin_dpo/margin_std": 34.9078254699707, "step": 654 }, { "KL/chosen_KL_mean": -37.20157241821289, "KL/mean": -51.45652770996094, "KL/rejected_KL_mean": -65.71148681640625, "KL/std": 33.084171295166016, "epoch": 0.9901738473167044, "fcm_dpo/beta": 0.031920626759529114, "fcm_dpo/delta": -0.13033875823020935, "fcm_dpo/margin": 28.50991439819336, "fcm_dpo/q_t": 0.3177955448627472, "grad_norm": 18.4930362701416, "learning_rate": 1.7131024761923852e-10, "logits/chosen": 0.521104097366333, "logits/rejected": 0.4239880442619324, "logps/chosen": -89.34872436523438, "logps/ref_chosen": -52.14714813232422, "logps/ref_rejected": -80.85014343261719, "logps/rejected": -146.56161499023438, "loss": 0.8502, "margin_dpo/margin_mean": 28.509910583496094, "margin_dpo/margin_std": 28.062389373779297, "step": 655 }, { "KL/chosen_KL_mean": -45.38775634765625, "KL/mean": -59.60160827636719, "KL/rejected_KL_mean": -73.81546020507812, "KL/std": 35.18117141723633, "epoch": 0.9916855631141346, "fcm_dpo/beta": 0.03162258490920067, "fcm_dpo/delta": -0.10963227599859238, "fcm_dpo/margin": 28.427703857421875, "fcm_dpo/q_t": 0.33033907413482666, "grad_norm": 17.709102630615234, "learning_rate": 1.2586440420372934e-10, "logits/chosen": 0.46768325567245483, "logits/rejected": 0.4151872396469116, "logps/chosen": -118.64448547363281, "logps/ref_chosen": -73.25672912597656, "logps/ref_rejected": -85.35127258300781, "logps/rejected": -159.16673278808594, "loss": 0.904, "margin_dpo/margin_mean": 28.427701950073242, "margin_dpo/margin_std": 34.47346878051758, "step": 656 }, { "KL/chosen_KL_mean": -42.539817810058594, "KL/mean": -58.39175796508789, "KL/rejected_KL_mean": -74.24369812011719, "KL/std": 38.77171325683594, "epoch": 0.9931972789115646, "fcm_dpo/beta": 0.030581142753362656, "fcm_dpo/delta": -0.18895968794822693, "fcm_dpo/margin": 31.703880310058594, "fcm_dpo/q_t": 0.3181675672531128, "grad_norm": 19.23728370666504, "learning_rate": 8.740807750345913e-11, "logits/chosen": 0.6302456855773926, "logits/rejected": 0.540096640586853, "logps/chosen": -92.26321411132812, "logps/ref_chosen": -49.72339630126953, "logps/ref_rejected": -75.1568603515625, "logps/rejected": -149.4005584716797, "loss": 0.9203, "margin_dpo/margin_mean": 31.703880310058594, "margin_dpo/margin_std": 39.745079040527344, "step": 657 }, { "KL/chosen_KL_mean": -46.40203094482422, "KL/mean": -57.525978088378906, "KL/rejected_KL_mean": -68.64991760253906, "KL/std": 37.033355712890625, "epoch": 0.9947089947089947, "fcm_dpo/beta": 0.03115946799516678, "fcm_dpo/delta": 0.11264770478010178, "fcm_dpo/margin": 22.24788475036621, "fcm_dpo/q_t": 0.3732909858226776, "grad_norm": 24.012834548950195, "learning_rate": 5.594234322453539e-11, "logits/chosen": 0.5549105405807495, "logits/rejected": 0.5142882466316223, "logps/chosen": -109.44837951660156, "logps/ref_chosen": -63.04634094238281, "logps/ref_rejected": -83.44963073730469, "logps/rejected": -152.09954833984375, "loss": 1.1425, "margin_dpo/margin_mean": 22.24788475036621, "margin_dpo/margin_std": 41.10093688964844, "step": 658 }, { "KL/chosen_KL_mean": -46.43489456176758, "KL/mean": -56.98037338256836, "KL/rejected_KL_mean": -67.52584838867188, "KL/std": 34.07621765136719, "epoch": 0.9962207105064248, "fcm_dpo/beta": 0.03144437074661255, "fcm_dpo/delta": 0.14566412568092346, "fcm_dpo/margin": 21.09095573425293, "fcm_dpo/q_t": 0.3753587007522583, "grad_norm": 21.728607177734375, "learning_rate": 3.146808153123293e-11, "logits/chosen": 0.6010191440582275, "logits/rejected": 0.5353198051452637, "logps/chosen": -101.51509094238281, "logps/ref_chosen": -55.0802001953125, "logps/ref_rejected": -71.91049194335938, "logps/rejected": -139.43634033203125, "loss": 1.1074, "margin_dpo/margin_mean": 21.09095573425293, "margin_dpo/margin_std": 36.32421112060547, "step": 659 }, { "KL/chosen_KL_mean": -45.47954559326172, "KL/mean": -59.227901458740234, "KL/rejected_KL_mean": -72.97624969482422, "KL/std": 34.6148796081543, "epoch": 0.9977324263038548, "fcm_dpo/beta": 0.031728580594062805, "fcm_dpo/delta": -0.0792866051197052, "fcm_dpo/margin": 27.49671173095703, "fcm_dpo/q_t": 0.32858672738075256, "grad_norm": 19.706825256347656, "learning_rate": 1.3985977021235829e-11, "logits/chosen": 0.6279199123382568, "logits/rejected": 0.5537869930267334, "logps/chosen": -100.00546264648438, "logps/ref_chosen": -54.525917053222656, "logps/ref_rejected": -81.23604583740234, "logps/rejected": -154.21229553222656, "loss": 0.8971, "margin_dpo/margin_mean": 27.49671173095703, "margin_dpo/margin_std": 32.02090835571289, "step": 660 }, { "KL/chosen_KL_mean": -51.72168731689453, "KL/mean": -62.73213195800781, "KL/rejected_KL_mean": -73.74258422851562, "KL/std": 36.27117919921875, "epoch": 0.999244142101285, "fcm_dpo/beta": 0.032111577689647675, "fcm_dpo/delta": 0.09923127293586731, "fcm_dpo/margin": 22.020889282226562, "fcm_dpo/q_t": 0.365009069442749, "grad_norm": 22.926618576049805, "learning_rate": 3.4965187065971735e-12, "logits/chosen": 0.49313268065452576, "logits/rejected": 0.40937820076942444, "logps/chosen": -112.09432983398438, "logps/ref_chosen": -60.37263870239258, "logps/ref_rejected": -77.42874145507812, "logps/rejected": -151.17132568359375, "loss": 1.0897, "margin_dpo/margin_mean": 22.020889282226562, "margin_dpo/margin_std": 37.274497985839844, "step": 661 }, { "epoch": 0.999244142101285, "step": 661, "total_flos": 0.0, "train_loss": 1.068178587349388, "train_runtime": 1640.4212, "train_samples_per_second": 25.808, "train_steps_per_second": 0.403 } ], "logging_steps": 1, "max_steps": 661, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }