{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9989528795811519, "eval_steps": 200, "global_step": 477, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": -0.04536247253417969, "KL/mean": -0.05389120429754257, "KL/rejected_KL_mean": -0.06241607666015625, "KL/std": 0.5513200759887695, "epoch": 0.0020942408376963353, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01704825460910797, "fcm_dpo/q_t": 0.499957412481308, "grad_norm": 14.618651390075684, "learning_rate": 0.0, "logits/chosen": 1.9854813814163208, "logits/rejected": 1.7737188339233398, "logps/chosen": -267.0936584472656, "logps/ref_chosen": -267.04827880859375, "logps/ref_rejected": -219.76657104492188, "logps/rejected": -219.82899475097656, "loss": 5.5446, "margin_dpo/margin_mean": 0.01704716682434082, "margin_dpo/margin_std": 0.7467707991600037, "step": 1 }, { "KL/chosen_KL_mean": 0.07550811767578125, "KL/mean": 0.03060000389814377, "KL/rejected_KL_mean": -0.014301300048828125, "KL/std": 0.6403332352638245, "epoch": 0.004188481675392671, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08979703485965729, "fcm_dpo/q_t": 0.4997755289077759, "grad_norm": 15.140149116516113, "learning_rate": 1.0416666666666666e-08, "logits/chosen": 2.1993117332458496, "logits/rejected": 2.1368322372436523, "logps/chosen": -256.5101623535156, "logps/ref_chosen": -256.585693359375, "logps/ref_rejected": -239.92742919921875, "logps/rejected": -239.94171142578125, "loss": 5.5417, "margin_dpo/margin_mean": 0.08979731798171997, "margin_dpo/margin_std": 0.9745746850967407, "step": 2 }, { "KL/chosen_KL_mean": 0.00390625, "KL/mean": -0.015328258275985718, "KL/rejected_KL_mean": -0.03455162048339844, "KL/std": 0.625785768032074, "epoch": 0.0062827225130890054, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03845793008804321, "fcm_dpo/q_t": 0.4999038279056549, "grad_norm": 14.649983406066895, "learning_rate": 2.083333333333333e-08, "logits/chosen": 2.2561516761779785, "logits/rejected": 2.2956507205963135, "logps/chosen": -265.66497802734375, "logps/ref_chosen": -265.66888427734375, "logps/ref_rejected": -284.3778991699219, "logps/rejected": -284.4124755859375, "loss": 5.5437, "margin_dpo/margin_mean": 0.03845754265785217, "margin_dpo/margin_std": 0.8079719543457031, "step": 3 }, { "KL/chosen_KL_mean": 0.07561492919921875, "KL/mean": 0.07756663113832474, "KL/rejected_KL_mean": 0.07953453063964844, "KL/std": 0.6961261630058289, "epoch": 0.008376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.003922984004020691, "fcm_dpo/q_t": 0.5000097751617432, "grad_norm": 15.700592041015625, "learning_rate": 3.125e-08, "logits/chosen": 1.8715617656707764, "logits/rejected": 1.8110389709472656, "logps/chosen": -319.431884765625, "logps/ref_chosen": -319.50750732421875, "logps/ref_rejected": -296.22314453125, "logps/rejected": -296.1436462402344, "loss": 5.5455, "margin_dpo/margin_mean": -0.003923103213310242, "margin_dpo/margin_std": 1.0326601266860962, "step": 4 }, { "KL/chosen_KL_mean": -0.003139495849609375, "KL/mean": 0.02464383840560913, "KL/rejected_KL_mean": 0.05242919921875, "KL/std": 0.6424128413200378, "epoch": 0.010471204188481676, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0555671751499176, "fcm_dpo/q_t": 0.5001389384269714, "grad_norm": 15.680787086486816, "learning_rate": 4.166666666666666e-08, "logits/chosen": 1.9470841884613037, "logits/rejected": 1.9636645317077637, "logps/chosen": -292.8761291503906, "logps/ref_chosen": -292.87298583984375, "logps/ref_rejected": -270.3802795410156, "logps/rejected": -270.327880859375, "loss": 5.5475, "margin_dpo/margin_mean": -0.05556736886501312, "margin_dpo/margin_std": 0.878207802772522, "step": 5 }, { "KL/chosen_KL_mean": 0.00652313232421875, "KL/mean": 0.008331656455993652, "KL/rejected_KL_mean": 0.0101470947265625, "KL/std": 0.5935744643211365, "epoch": 0.012565445026178011, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0036214888095855713, "fcm_dpo/q_t": 0.5000090599060059, "grad_norm": 15.472807884216309, "learning_rate": 5.208333333333333e-08, "logits/chosen": 2.1412649154663086, "logits/rejected": 2.078343152999878, "logps/chosen": -280.2789306640625, "logps/ref_chosen": -280.28546142578125, "logps/ref_rejected": -242.7611541748047, "logps/rejected": -242.75100708007812, "loss": 5.5454, "margin_dpo/margin_mean": -0.0036224573850631714, "margin_dpo/margin_std": 0.7638767957687378, "step": 6 }, { "KL/chosen_KL_mean": -0.004520416259765625, "KL/mean": 0.03132270276546478, "KL/rejected_KL_mean": 0.06715965270996094, "KL/std": 0.6809048652648926, "epoch": 0.014659685863874346, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0716867744922638, "fcm_dpo/q_t": 0.5001791715621948, "grad_norm": 15.662071228027344, "learning_rate": 6.25e-08, "logits/chosen": 1.8676385879516602, "logits/rejected": 1.7663965225219727, "logps/chosen": -255.11024475097656, "logps/ref_chosen": -255.10572814941406, "logps/ref_rejected": -251.15402221679688, "logps/rejected": -251.08685302734375, "loss": 5.5482, "margin_dpo/margin_mean": -0.07168655097484589, "margin_dpo/margin_std": 0.9891082644462585, "step": 7 }, { "KL/chosen_KL_mean": -0.05980682373046875, "KL/mean": -0.008247092366218567, "KL/rejected_KL_mean": 0.043323516845703125, "KL/std": 0.7069255709648132, "epoch": 0.016753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.10313859581947327, "fcm_dpo/q_t": 0.5002578496932983, "grad_norm": 15.891398429870605, "learning_rate": 7.291666666666667e-08, "logits/chosen": 2.325185537338257, "logits/rejected": 2.135742664337158, "logps/chosen": -345.24139404296875, "logps/ref_chosen": -345.18157958984375, "logps/ref_rejected": -277.3522644042969, "logps/rejected": -277.3089599609375, "loss": 5.5494, "margin_dpo/margin_mean": -0.10313853621482849, "margin_dpo/margin_std": 1.0472654104232788, "step": 8 }, { "KL/chosen_KL_mean": -0.05593681335449219, "KL/mean": -0.04206794500350952, "KL/rejected_KL_mean": -0.0281982421875, "KL/std": 0.6409730315208435, "epoch": 0.018848167539267015, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.027742207050323486, "fcm_dpo/q_t": 0.5000693798065186, "grad_norm": 14.116764068603516, "learning_rate": 8.333333333333333e-08, "logits/chosen": 2.199899435043335, "logits/rejected": 2.1797549724578857, "logps/chosen": -276.7862854003906, "logps/ref_chosen": -276.7303161621094, "logps/ref_rejected": -266.19952392578125, "logps/rejected": -266.2276916503906, "loss": 5.5464, "margin_dpo/margin_mean": -0.027742385864257812, "margin_dpo/margin_std": 0.8435088396072388, "step": 9 }, { "KL/chosen_KL_mean": -0.027740478515625, "KL/mean": -0.010075248777866364, "KL/rejected_KL_mean": 0.00759124755859375, "KL/std": 0.5887585282325745, "epoch": 0.020942408376963352, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.035333022475242615, "fcm_dpo/q_t": 0.5000883340835571, "grad_norm": 13.785626411437988, "learning_rate": 9.375e-08, "logits/chosen": 1.9849461317062378, "logits/rejected": 2.041294813156128, "logps/chosen": -246.5900115966797, "logps/ref_chosen": -246.562255859375, "logps/ref_rejected": -224.70401000976562, "logps/rejected": -224.6964111328125, "loss": 5.5467, "margin_dpo/margin_mean": -0.03533339500427246, "margin_dpo/margin_std": 0.7649649381637573, "step": 10 }, { "KL/chosen_KL_mean": 0.059051513671875, "KL/mean": 0.010689198970794678, "KL/rejected_KL_mean": -0.03765869140625, "KL/std": 0.6329272389411926, "epoch": 0.023036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09670479595661163, "fcm_dpo/q_t": 0.499758243560791, "grad_norm": 15.459443092346191, "learning_rate": 1.0416666666666667e-07, "logits/chosen": 1.970166802406311, "logits/rejected": 1.9005579948425293, "logps/chosen": -290.4420166015625, "logps/ref_chosen": -290.5010986328125, "logps/ref_rejected": -253.9559326171875, "logps/rejected": -253.99354553222656, "loss": 5.5414, "margin_dpo/margin_mean": 0.09670477360486984, "margin_dpo/margin_std": 0.9046503305435181, "step": 11 }, { "KL/chosen_KL_mean": 0.040882110595703125, "KL/mean": 0.030767112970352173, "KL/rejected_KL_mean": 0.020654678344726562, "KL/std": 0.614155650138855, "epoch": 0.025130890052356022, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02022460103034973, "fcm_dpo/q_t": 0.4999494254589081, "grad_norm": 14.839339256286621, "learning_rate": 1.1458333333333332e-07, "logits/chosen": 1.5951333045959473, "logits/rejected": 1.579640507698059, "logps/chosen": -278.4776611328125, "logps/ref_chosen": -278.5185241699219, "logps/ref_rejected": -273.56207275390625, "logps/rejected": -273.54144287109375, "loss": 5.5444, "margin_dpo/margin_mean": 0.020224541425704956, "margin_dpo/margin_std": 0.8575910925865173, "step": 12 }, { "KL/chosen_KL_mean": -0.10135078430175781, "KL/mean": -0.05781416594982147, "KL/rejected_KL_mean": -0.014280319213867188, "KL/std": 0.5546329617500305, "epoch": 0.027225130890052355, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08706018328666687, "fcm_dpo/q_t": 0.5002176761627197, "grad_norm": 14.821869850158691, "learning_rate": 1.25e-07, "logits/chosen": 1.8845791816711426, "logits/rejected": 2.0167646408081055, "logps/chosen": -264.75848388671875, "logps/ref_chosen": -264.6571350097656, "logps/ref_rejected": -274.3492736816406, "logps/rejected": -274.3635559082031, "loss": 5.5487, "margin_dpo/margin_mean": -0.08706018328666687, "margin_dpo/margin_std": 0.7981866002082825, "step": 13 }, { "KL/chosen_KL_mean": 0.0448760986328125, "KL/mean": -0.011957794427871704, "KL/rejected_KL_mean": -0.0688018798828125, "KL/std": 0.5699095129966736, "epoch": 0.02931937172774869, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11367706954479218, "fcm_dpo/q_t": 0.4997158348560333, "grad_norm": 15.356264114379883, "learning_rate": 1.3541666666666666e-07, "logits/chosen": 2.0706424713134766, "logits/rejected": 2.1114251613616943, "logps/chosen": -274.0697326660156, "logps/ref_chosen": -274.1145935058594, "logps/ref_rejected": -261.0951843261719, "logps/rejected": -261.1639709472656, "loss": 5.5407, "margin_dpo/margin_mean": 0.11367689073085785, "margin_dpo/margin_std": 0.7376527786254883, "step": 14 }, { "KL/chosen_KL_mean": 0.02631378173828125, "KL/mean": 0.055424295365810394, "KL/rejected_KL_mean": 0.08452606201171875, "KL/std": 0.6501664519309998, "epoch": 0.031413612565445025, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05820421874523163, "fcm_dpo/q_t": 0.5001454949378967, "grad_norm": 16.085472106933594, "learning_rate": 1.4583333333333335e-07, "logits/chosen": 2.1451964378356934, "logits/rejected": 2.0267412662506104, "logps/chosen": -288.55841064453125, "logps/ref_chosen": -288.584716796875, "logps/ref_rejected": -224.8094482421875, "logps/rejected": -224.72491455078125, "loss": 5.5476, "margin_dpo/margin_mean": -0.05820445716381073, "margin_dpo/margin_std": 0.8958753943443298, "step": 15 }, { "KL/chosen_KL_mean": 0.00731658935546875, "KL/mean": 0.004071265459060669, "KL/rejected_KL_mean": 0.00083160400390625, "KL/std": 0.6132818460464478, "epoch": 0.033507853403141365, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.006486058235168457, "fcm_dpo/q_t": 0.4999837875366211, "grad_norm": 16.61728858947754, "learning_rate": 1.5624999999999999e-07, "logits/chosen": 1.8354623317718506, "logits/rejected": 1.6800379753112793, "logps/chosen": -283.8358154296875, "logps/ref_chosen": -283.8431091308594, "logps/ref_rejected": -248.63645935058594, "logps/rejected": -248.6356201171875, "loss": 5.545, "margin_dpo/margin_mean": 0.006484881043434143, "margin_dpo/margin_std": 0.8758120536804199, "step": 16 }, { "KL/chosen_KL_mean": 0.07244110107421875, "KL/mean": 0.06645964086055756, "KL/rejected_KL_mean": 0.06048583984375, "KL/std": 0.5597188472747803, "epoch": 0.0356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011956393718719482, "fcm_dpo/q_t": 0.49997010827064514, "grad_norm": 15.15675163269043, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 1.8342028856277466, "logits/rejected": 1.786647915840149, "logps/chosen": -254.56114196777344, "logps/ref_chosen": -254.6335906982422, "logps/ref_rejected": -245.3981170654297, "logps/rejected": -245.33763122558594, "loss": 5.5448, "margin_dpo/margin_mean": 0.011956632137298584, "margin_dpo/margin_std": 0.7229968309402466, "step": 17 }, { "KL/chosen_KL_mean": -0.0007228851318359375, "KL/mean": 0.06241019070148468, "KL/rejected_KL_mean": 0.1255321502685547, "KL/std": 0.6303836107254028, "epoch": 0.03769633507853403, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12625613808631897, "fcm_dpo/q_t": 0.5003156065940857, "grad_norm": 15.067169189453125, "learning_rate": 1.7708333333333334e-07, "logits/chosen": 1.8873498439788818, "logits/rejected": 1.8011493682861328, "logps/chosen": -287.750244140625, "logps/ref_chosen": -287.7495422363281, "logps/ref_rejected": -273.03363037109375, "logps/rejected": -272.9080810546875, "loss": 5.5503, "margin_dpo/margin_mean": -0.12625686824321747, "margin_dpo/margin_std": 0.896041214466095, "step": 18 }, { "KL/chosen_KL_mean": -0.023553848266601562, "KL/mean": -0.029172401875257492, "KL/rejected_KL_mean": -0.03479766845703125, "KL/std": 0.59148770570755, "epoch": 0.039790575916230364, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011247612535953522, "fcm_dpo/q_t": 0.4999718964099884, "grad_norm": 14.825756072998047, "learning_rate": 1.875e-07, "logits/chosen": 1.9977781772613525, "logits/rejected": 1.9584683179855347, "logps/chosen": -273.4274597167969, "logps/ref_chosen": -273.4039001464844, "logps/ref_rejected": -260.09722900390625, "logps/rejected": -260.13201904296875, "loss": 5.5448, "margin_dpo/margin_mean": 0.011247284710407257, "margin_dpo/margin_std": 0.773921549320221, "step": 19 }, { "KL/chosen_KL_mean": 0.07031822204589844, "KL/mean": 0.049061864614486694, "KL/rejected_KL_mean": 0.0278167724609375, "KL/std": 0.6591533422470093, "epoch": 0.041884816753926704, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.042512863874435425, "fcm_dpo/q_t": 0.4998937249183655, "grad_norm": 15.503080368041992, "learning_rate": 1.9791666666666664e-07, "logits/chosen": 1.8373280763626099, "logits/rejected": 1.7833504676818848, "logps/chosen": -292.89178466796875, "logps/ref_chosen": -292.96209716796875, "logps/ref_rejected": -256.602783203125, "logps/rejected": -256.574951171875, "loss": 5.5436, "margin_dpo/margin_mean": 0.042513057589530945, "margin_dpo/margin_std": 0.8287696838378906, "step": 20 }, { "KL/chosen_KL_mean": -0.009965896606445312, "KL/mean": 0.0010740011930465698, "KL/rejected_KL_mean": 0.012109756469726562, "KL/std": 0.6445425152778625, "epoch": 0.04397905759162304, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.022067517042160034, "fcm_dpo/q_t": 0.500055193901062, "grad_norm": 15.471956253051758, "learning_rate": 2.0833333333333333e-07, "logits/chosen": 2.0407376289367676, "logits/rejected": 1.9655239582061768, "logps/chosen": -292.3371887207031, "logps/ref_chosen": -292.3271789550781, "logps/ref_rejected": -274.20635986328125, "logps/rejected": -274.19427490234375, "loss": 5.5461, "margin_dpo/margin_mean": -0.022068947553634644, "margin_dpo/margin_std": 0.8329710960388184, "step": 21 }, { "KL/chosen_KL_mean": 0.09007644653320312, "KL/mean": 0.04034734144806862, "KL/rejected_KL_mean": -0.009380340576171875, "KL/std": 0.5518816709518433, "epoch": 0.04607329842931937, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09946237504482269, "fcm_dpo/q_t": 0.49975138902664185, "grad_norm": 13.731009483337402, "learning_rate": 2.1875e-07, "logits/chosen": 2.002246856689453, "logits/rejected": 2.0066237449645996, "logps/chosen": -253.17613220214844, "logps/ref_chosen": -253.2662353515625, "logps/ref_rejected": -241.4681396484375, "logps/rejected": -241.4775390625, "loss": 5.5413, "margin_dpo/margin_mean": 0.09946297109127045, "margin_dpo/margin_std": 0.7452505230903625, "step": 22 }, { "KL/chosen_KL_mean": 0.020843505859375, "KL/mean": 0.05904676765203476, "KL/rejected_KL_mean": 0.0972442626953125, "KL/std": 0.7336594462394714, "epoch": 0.048167539267015703, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07639683783054352, "fcm_dpo/q_t": 0.5001909732818604, "grad_norm": 15.244080543518066, "learning_rate": 2.2916666666666663e-07, "logits/chosen": 1.8692981004714966, "logits/rejected": 1.8475254774093628, "logps/chosen": -275.4007263183594, "logps/ref_chosen": -275.4216003417969, "logps/ref_rejected": -264.44024658203125, "logps/rejected": -264.3429870605469, "loss": 5.5484, "margin_dpo/margin_mean": -0.07639613747596741, "margin_dpo/margin_std": 1.02485191822052, "step": 23 }, { "KL/chosen_KL_mean": -0.03753471374511719, "KL/mean": -0.006300576031208038, "KL/rejected_KL_mean": 0.024938583374023438, "KL/std": 0.6965287923812866, "epoch": 0.050261780104712044, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06248579919338226, "fcm_dpo/q_t": 0.5001561641693115, "grad_norm": 16.480966567993164, "learning_rate": 2.3958333333333335e-07, "logits/chosen": 1.9098074436187744, "logits/rejected": 1.9367527961730957, "logps/chosen": -293.41510009765625, "logps/ref_chosen": -293.3775634765625, "logps/ref_rejected": -301.59014892578125, "logps/rejected": -301.5652160644531, "loss": 5.5478, "margin_dpo/margin_mean": -0.062486037611961365, "margin_dpo/margin_std": 0.9366015791893005, "step": 24 }, { "KL/chosen_KL_mean": 0.07129096984863281, "KL/mean": 0.03325209766626358, "KL/rejected_KL_mean": -0.004787445068359375, "KL/std": 0.5905327200889587, "epoch": 0.05235602094240838, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07607276737689972, "fcm_dpo/q_t": 0.49980980157852173, "grad_norm": 14.5157470703125, "learning_rate": 2.5e-07, "logits/chosen": 1.721674919128418, "logits/rejected": 1.756920576095581, "logps/chosen": -240.86239624023438, "logps/ref_chosen": -240.93368530273438, "logps/ref_rejected": -235.3230743408203, "logps/rejected": -235.32786560058594, "loss": 5.5422, "margin_dpo/margin_mean": 0.07607249915599823, "margin_dpo/margin_std": 0.806990921497345, "step": 25 }, { "KL/chosen_KL_mean": 0.06521224975585938, "KL/mean": 0.023147452622652054, "KL/rejected_KL_mean": -0.018918991088867188, "KL/std": 0.6094150543212891, "epoch": 0.05445026178010471, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08413281291723251, "fcm_dpo/q_t": 0.49978965520858765, "grad_norm": 15.577874183654785, "learning_rate": 2.604166666666667e-07, "logits/chosen": 1.7217767238616943, "logits/rejected": 1.7037371397018433, "logps/chosen": -273.53009033203125, "logps/ref_chosen": -273.5953369140625, "logps/ref_rejected": -234.49148559570312, "logps/rejected": -234.51040649414062, "loss": 5.5419, "margin_dpo/margin_mean": 0.08413246273994446, "margin_dpo/margin_std": 0.7173627614974976, "step": 26 }, { "KL/chosen_KL_mean": 0.07335662841796875, "KL/mean": 0.025179728865623474, "KL/rejected_KL_mean": -0.022989273071289062, "KL/std": 0.6592622995376587, "epoch": 0.05654450261780105, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09633858501911163, "fcm_dpo/q_t": 0.49975913763046265, "grad_norm": 15.121481895446777, "learning_rate": 2.708333333333333e-07, "logits/chosen": 2.2555246353149414, "logits/rejected": 2.032055616378784, "logps/chosen": -314.1905212402344, "logps/ref_chosen": -314.26385498046875, "logps/ref_rejected": -242.06735229492188, "logps/rejected": -242.09031677246094, "loss": 5.5414, "margin_dpo/margin_mean": 0.09633757174015045, "margin_dpo/margin_std": 0.8887685537338257, "step": 27 }, { "KL/chosen_KL_mean": 0.1266918182373047, "KL/mean": 0.03975718468427658, "KL/rejected_KL_mean": -0.047176361083984375, "KL/std": 0.581898033618927, "epoch": 0.05863874345549738, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1738647222518921, "fcm_dpo/q_t": 0.49956533312797546, "grad_norm": 15.378426551818848, "learning_rate": 2.8125e-07, "logits/chosen": 2.039022922515869, "logits/rejected": 2.04673171043396, "logps/chosen": -240.6121368408203, "logps/ref_chosen": -240.7388153076172, "logps/ref_rejected": -237.8331298828125, "logps/rejected": -237.8803253173828, "loss": 5.5383, "margin_dpo/margin_mean": 0.1738651543855667, "margin_dpo/margin_std": 0.8048579692840576, "step": 28 }, { "KL/chosen_KL_mean": 0.07528114318847656, "KL/mean": 0.04480874538421631, "KL/rejected_KL_mean": 0.014341354370117188, "KL/std": 0.6803877949714661, "epoch": 0.060732984293193716, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06094604730606079, "fcm_dpo/q_t": 0.4998476505279541, "grad_norm": 14.76402759552002, "learning_rate": 2.916666666666667e-07, "logits/chosen": 1.9146788120269775, "logits/rejected": 1.794429898262024, "logps/chosen": -307.754150390625, "logps/ref_chosen": -307.82940673828125, "logps/ref_rejected": -281.8109130859375, "logps/rejected": -281.79656982421875, "loss": 5.5428, "margin_dpo/margin_mean": 0.06094469130039215, "margin_dpo/margin_std": 0.9543280005455017, "step": 29 }, { "KL/chosen_KL_mean": 0.03249931335449219, "KL/mean": 0.02733471244573593, "KL/rejected_KL_mean": 0.022161483764648438, "KL/std": 0.5754698514938354, "epoch": 0.06282722513089005, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.010352656245231628, "fcm_dpo/q_t": 0.4999741315841675, "grad_norm": 14.39789867401123, "learning_rate": 3.020833333333333e-07, "logits/chosen": 1.7222105264663696, "logits/rejected": 1.7153657674789429, "logps/chosen": -247.9812774658203, "logps/ref_chosen": -248.01376342773438, "logps/ref_rejected": -259.0014953613281, "logps/rejected": -258.9793395996094, "loss": 5.5448, "margin_dpo/margin_mean": 0.010352000594139099, "margin_dpo/margin_std": 0.806208610534668, "step": 30 }, { "KL/chosen_KL_mean": 0.10100173950195312, "KL/mean": 0.04322759807109833, "KL/rejected_KL_mean": -0.014553070068359375, "KL/std": 0.5859971046447754, "epoch": 0.06492146596858639, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11553975939750671, "fcm_dpo/q_t": 0.49971115589141846, "grad_norm": 13.664345741271973, "learning_rate": 3.1249999999999997e-07, "logits/chosen": 1.9450936317443848, "logits/rejected": 1.8830111026763916, "logps/chosen": -239.96180725097656, "logps/ref_chosen": -240.0628204345703, "logps/ref_rejected": -201.91021728515625, "logps/rejected": -201.9247589111328, "loss": 5.5406, "margin_dpo/margin_mean": 0.11553928256034851, "margin_dpo/margin_std": 0.751106321811676, "step": 31 }, { "KL/chosen_KL_mean": 0.15114593505859375, "KL/mean": 0.07302998751401901, "KL/rejected_KL_mean": -0.005084991455078125, "KL/std": 0.657054603099823, "epoch": 0.06701570680628273, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1562250405550003, "fcm_dpo/q_t": 0.49960944056510925, "grad_norm": 15.576884269714355, "learning_rate": 3.2291666666666666e-07, "logits/chosen": 1.918774962425232, "logits/rejected": 1.8419352769851685, "logps/chosen": -293.9810791015625, "logps/ref_chosen": -294.1322326660156, "logps/ref_rejected": -265.8150634765625, "logps/rejected": -265.8201599121094, "loss": 5.539, "margin_dpo/margin_mean": 0.15622559189796448, "margin_dpo/margin_std": 0.8563711047172546, "step": 32 }, { "KL/chosen_KL_mean": 0.24457740783691406, "KL/mean": 0.1883525848388672, "KL/rejected_KL_mean": 0.13211822509765625, "KL/std": 0.6640927195549011, "epoch": 0.06910994764397906, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11245375871658325, "fcm_dpo/q_t": 0.4997188448905945, "grad_norm": 14.962160110473633, "learning_rate": 3.333333333333333e-07, "logits/chosen": 1.9812177419662476, "logits/rejected": 1.9455798864364624, "logps/chosen": -275.4105224609375, "logps/ref_chosen": -275.6551208496094, "logps/ref_rejected": -253.5972137451172, "logps/rejected": -253.465087890625, "loss": 5.5408, "margin_dpo/margin_mean": 0.11245402693748474, "margin_dpo/margin_std": 0.9471386075019836, "step": 33 }, { "KL/chosen_KL_mean": 0.24161911010742188, "KL/mean": 0.1785203069448471, "KL/rejected_KL_mean": 0.11542320251464844, "KL/std": 0.7460881471633911, "epoch": 0.0712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12618765234947205, "fcm_dpo/q_t": 0.49968454241752625, "grad_norm": 17.706989288330078, "learning_rate": 3.4375e-07, "logits/chosen": 2.1137003898620605, "logits/rejected": 2.008443593978882, "logps/chosen": -332.7284851074219, "logps/ref_chosen": -332.9701232910156, "logps/ref_rejected": -278.2213439941406, "logps/rejected": -278.10595703125, "loss": 5.5402, "margin_dpo/margin_mean": 0.12618720531463623, "margin_dpo/margin_std": 0.9816871881484985, "step": 34 }, { "KL/chosen_KL_mean": 0.1520099639892578, "KL/mean": 0.13822707533836365, "KL/rejected_KL_mean": 0.12445068359375, "KL/std": 0.6340049505233765, "epoch": 0.07329842931937172, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02756369858980179, "fcm_dpo/q_t": 0.49993109703063965, "grad_norm": 13.682244300842285, "learning_rate": 3.541666666666667e-07, "logits/chosen": 1.7967126369476318, "logits/rejected": 1.8555289506912231, "logps/chosen": -258.27685546875, "logps/ref_chosen": -258.42889404296875, "logps/ref_rejected": -259.5119934082031, "logps/rejected": -259.3875427246094, "loss": 5.5442, "margin_dpo/margin_mean": 0.02756364643573761, "margin_dpo/margin_std": 0.8654166460037231, "step": 35 }, { "KL/chosen_KL_mean": 0.4261493682861328, "KL/mean": 0.22535449266433716, "KL/rejected_KL_mean": 0.024557113647460938, "KL/std": 0.79774010181427, "epoch": 0.07539267015706806, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40159767866134644, "fcm_dpo/q_t": 0.4989960491657257, "grad_norm": 17.678897857666016, "learning_rate": 3.645833333333333e-07, "logits/chosen": 1.9235519170761108, "logits/rejected": 1.8702714443206787, "logps/chosen": -290.1932067871094, "logps/ref_chosen": -290.6193542480469, "logps/ref_rejected": -280.2503967285156, "logps/rejected": -280.2258605957031, "loss": 5.5292, "margin_dpo/margin_mean": 0.40159714221954346, "margin_dpo/margin_std": 1.0169625282287598, "step": 36 }, { "KL/chosen_KL_mean": 0.29315185546875, "KL/mean": 0.20537252724170685, "KL/rejected_KL_mean": 0.11758995056152344, "KL/std": 0.7490401864051819, "epoch": 0.0774869109947644, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1755593866109848, "fcm_dpo/q_t": 0.4995611608028412, "grad_norm": 15.22133731842041, "learning_rate": 3.75e-07, "logits/chosen": 1.8605339527130127, "logits/rejected": 1.8499176502227783, "logps/chosen": -303.2796630859375, "logps/ref_chosen": -303.57281494140625, "logps/ref_rejected": -294.56878662109375, "logps/rejected": -294.451171875, "loss": 5.5383, "margin_dpo/margin_mean": 0.1755588799715042, "margin_dpo/margin_std": 1.0125519037246704, "step": 37 }, { "KL/chosen_KL_mean": 0.3296089172363281, "KL/mean": 0.2202475368976593, "KL/rejected_KL_mean": 0.1108856201171875, "KL/std": 0.763922393321991, "epoch": 0.07958115183246073, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21871574223041534, "fcm_dpo/q_t": 0.49945324659347534, "grad_norm": 16.027008056640625, "learning_rate": 3.8541666666666665e-07, "logits/chosen": 1.8954304456710815, "logits/rejected": 1.9512522220611572, "logps/chosen": -272.223876953125, "logps/ref_chosen": -272.5534973144531, "logps/ref_rejected": -266.9136047363281, "logps/rejected": -266.802734375, "loss": 5.5365, "margin_dpo/margin_mean": 0.21871612966060638, "margin_dpo/margin_std": 0.9921301603317261, "step": 38 }, { "KL/chosen_KL_mean": 0.5544986724853516, "KL/mean": 0.3141814172267914, "KL/rejected_KL_mean": 0.07386398315429688, "KL/std": 0.77060467004776, "epoch": 0.08167539267015707, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.48063284158706665, "fcm_dpo/q_t": 0.4987984299659729, "grad_norm": 15.96513557434082, "learning_rate": 3.958333333333333e-07, "logits/chosen": 1.7163671255111694, "logits/rejected": 1.6469464302062988, "logps/chosen": -264.83203125, "logps/ref_chosen": -265.38653564453125, "logps/ref_rejected": -244.34774780273438, "logps/rejected": -244.27392578125, "loss": 5.5261, "margin_dpo/margin_mean": 0.48063212633132935, "margin_dpo/margin_std": 0.8729540109634399, "step": 39 }, { "KL/chosen_KL_mean": 0.4694805145263672, "KL/mean": 0.27980729937553406, "KL/rejected_KL_mean": 0.0901336669921875, "KL/std": 0.7455384135246277, "epoch": 0.08376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.37933290004730225, "fcm_dpo/q_t": 0.4990517199039459, "grad_norm": 14.94265365600586, "learning_rate": 4.0625e-07, "logits/chosen": 1.7584089040756226, "logits/rejected": 1.744396448135376, "logps/chosen": -274.42431640625, "logps/ref_chosen": -274.893798828125, "logps/ref_rejected": -244.3120880126953, "logps/rejected": -244.22195434570312, "loss": 5.5301, "margin_dpo/margin_mean": 0.37933290004730225, "margin_dpo/margin_std": 0.9794178605079651, "step": 40 }, { "KL/chosen_KL_mean": 0.383026123046875, "KL/mean": 0.278035968542099, "KL/rejected_KL_mean": 0.1730499267578125, "KL/std": 0.7958909869194031, "epoch": 0.08586387434554973, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20995977520942688, "fcm_dpo/q_t": 0.4994751214981079, "grad_norm": 15.472199440002441, "learning_rate": 4.1666666666666667e-07, "logits/chosen": 2.015687942504883, "logits/rejected": 1.9452970027923584, "logps/chosen": -284.4161071777344, "logps/ref_chosen": -284.7991027832031, "logps/ref_rejected": -248.8882598876953, "logps/rejected": -248.71519470214844, "loss": 5.5369, "margin_dpo/margin_mean": 0.2099606841802597, "margin_dpo/margin_std": 1.0590481758117676, "step": 41 }, { "KL/chosen_KL_mean": 0.49579811096191406, "KL/mean": 0.3066490590572357, "KL/rejected_KL_mean": 0.11749649047851562, "KL/std": 0.7752858400344849, "epoch": 0.08795811518324607, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3783026337623596, "fcm_dpo/q_t": 0.4990542531013489, "grad_norm": 14.61644172668457, "learning_rate": 4.270833333333333e-07, "logits/chosen": 1.8188252449035645, "logits/rejected": 1.8732718229293823, "logps/chosen": -245.3438720703125, "logps/ref_chosen": -245.83966064453125, "logps/ref_rejected": -259.2870788574219, "logps/rejected": -259.1695861816406, "loss": 5.5302, "margin_dpo/margin_mean": 0.3783027231693268, "margin_dpo/margin_std": 1.054323434829712, "step": 42 }, { "KL/chosen_KL_mean": 0.4672527313232422, "KL/mean": 0.31816428899765015, "KL/rejected_KL_mean": 0.1690826416015625, "KL/std": 0.8266512155532837, "epoch": 0.09005235602094241, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.29816895723342896, "fcm_dpo/q_t": 0.49925461411476135, "grad_norm": 15.704439163208008, "learning_rate": 4.375e-07, "logits/chosen": 1.7950459718704224, "logits/rejected": 1.7692830562591553, "logps/chosen": -307.9302978515625, "logps/ref_chosen": -308.39752197265625, "logps/ref_rejected": -270.3797912597656, "logps/rejected": -270.2107238769531, "loss": 5.5334, "margin_dpo/margin_mean": 0.2981690466403961, "margin_dpo/margin_std": 1.0226835012435913, "step": 43 }, { "KL/chosen_KL_mean": 0.7249507904052734, "KL/mean": 0.47041821479797363, "KL/rejected_KL_mean": 0.2158832550048828, "KL/std": 0.8857836127281189, "epoch": 0.09214659685863874, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5090519785881042, "fcm_dpo/q_t": 0.49872738122940063, "grad_norm": 15.815662384033203, "learning_rate": 4.479166666666667e-07, "logits/chosen": 1.9265743494033813, "logits/rejected": 1.9330271482467651, "logps/chosen": -274.63677978515625, "logps/ref_chosen": -275.36175537109375, "logps/ref_rejected": -279.7540588378906, "logps/rejected": -279.5381774902344, "loss": 5.525, "margin_dpo/margin_mean": 0.5090534090995789, "margin_dpo/margin_std": 1.1619629859924316, "step": 44 }, { "KL/chosen_KL_mean": 0.7300357818603516, "KL/mean": 0.4381192922592163, "KL/rejected_KL_mean": 0.1461963653564453, "KL/std": 1.0366795063018799, "epoch": 0.09424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.583842396736145, "fcm_dpo/q_t": 0.4985404908657074, "grad_norm": 16.86652183532715, "learning_rate": 4.5833333333333327e-07, "logits/chosen": 1.8563649654388428, "logits/rejected": 1.8279879093170166, "logps/chosen": -314.39013671875, "logps/ref_chosen": -315.1202087402344, "logps/ref_rejected": -290.34210205078125, "logps/rejected": -290.1958923339844, "loss": 5.522, "margin_dpo/margin_mean": 0.5838421583175659, "margin_dpo/margin_std": 1.3088961839675903, "step": 45 }, { "KL/chosen_KL_mean": 0.8227634429931641, "KL/mean": 0.49031269550323486, "KL/rejected_KL_mean": 0.15786361694335938, "KL/std": 1.0303242206573486, "epoch": 0.09633507853403141, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6648975014686584, "fcm_dpo/q_t": 0.49833783507347107, "grad_norm": 17.709739685058594, "learning_rate": 4.6874999999999996e-07, "logits/chosen": 1.6669872999191284, "logits/rejected": 1.668416142463684, "logps/chosen": -292.1178283691406, "logps/ref_chosen": -292.94061279296875, "logps/ref_rejected": -264.0979309082031, "logps/rejected": -263.9400634765625, "loss": 5.5188, "margin_dpo/margin_mean": 0.6648980975151062, "margin_dpo/margin_std": 1.3021197319030762, "step": 46 }, { "KL/chosen_KL_mean": 0.6330909729003906, "KL/mean": 0.4338260293006897, "KL/rejected_KL_mean": 0.23456192016601562, "KL/std": 1.0144609212875366, "epoch": 0.09842931937172775, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3985220491886139, "fcm_dpo/q_t": 0.4990037679672241, "grad_norm": 14.713279724121094, "learning_rate": 4.791666666666667e-07, "logits/chosen": 1.8358039855957031, "logits/rejected": 1.9415535926818848, "logps/chosen": -254.21958923339844, "logps/ref_chosen": -254.85269165039062, "logps/ref_rejected": -255.88380432128906, "logps/rejected": -255.6492156982422, "loss": 5.5295, "margin_dpo/margin_mean": 0.3985212445259094, "margin_dpo/margin_std": 1.4352999925613403, "step": 47 }, { "KL/chosen_KL_mean": 0.7173347473144531, "KL/mean": 0.39810723066329956, "KL/rejected_KL_mean": 0.07886505126953125, "KL/std": 1.0975290536880493, "epoch": 0.10052356020942409, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.638471245765686, "fcm_dpo/q_t": 0.49840396642684937, "grad_norm": 14.869832038879395, "learning_rate": 4.895833333333333e-07, "logits/chosen": 1.8391691446304321, "logits/rejected": 1.82039213180542, "logps/chosen": -261.7904052734375, "logps/ref_chosen": -262.5077209472656, "logps/ref_rejected": -239.25320434570312, "logps/rejected": -239.17434692382812, "loss": 5.5199, "margin_dpo/margin_mean": 0.6384709477424622, "margin_dpo/margin_std": 1.3757047653198242, "step": 48 }, { "KL/chosen_KL_mean": 0.9507026672363281, "KL/mean": 0.6429872512817383, "KL/rejected_KL_mean": 0.33528709411621094, "KL/std": 1.1882615089416504, "epoch": 0.10261780104712041, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.61541748046875, "fcm_dpo/q_t": 0.49846160411834717, "grad_norm": 15.200157165527344, "learning_rate": 5e-07, "logits/chosen": 1.7383028268814087, "logits/rejected": 1.8577663898468018, "logps/chosen": -248.57717895507812, "logps/ref_chosen": -249.5278778076172, "logps/ref_rejected": -254.6465301513672, "logps/rejected": -254.311279296875, "loss": 5.5209, "margin_dpo/margin_mean": 0.615418553352356, "margin_dpo/margin_std": 1.670445442199707, "step": 49 }, { "KL/chosen_KL_mean": 0.9584369659423828, "KL/mean": 0.714724063873291, "KL/rejected_KL_mean": 0.4709930419921875, "KL/std": 1.2188169956207275, "epoch": 0.10471204188481675, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.48745203018188477, "fcm_dpo/q_t": 0.4987815022468567, "grad_norm": 14.659168243408203, "learning_rate": 4.999932966293553e-07, "logits/chosen": 1.902108907699585, "logits/rejected": 1.9085142612457275, "logps/chosen": -291.54327392578125, "logps/ref_chosen": -292.501708984375, "logps/ref_rejected": -294.6496887207031, "logps/rejected": -294.1787109375, "loss": 5.526, "margin_dpo/margin_mean": 0.48745256662368774, "margin_dpo/margin_std": 1.5779855251312256, "step": 50 }, { "KL/chosen_KL_mean": 0.9927196502685547, "KL/mean": 0.6409055590629578, "KL/rejected_KL_mean": 0.28908538818359375, "KL/std": 1.3663586378097534, "epoch": 0.1068062827225131, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7036239504814148, "fcm_dpo/q_t": 0.498241126537323, "grad_norm": 14.71286678314209, "learning_rate": 4.999731868769026e-07, "logits/chosen": 1.5570727586746216, "logits/rejected": 1.6826603412628174, "logps/chosen": -258.869140625, "logps/ref_chosen": -259.8619079589844, "logps/ref_rejected": -293.1994934082031, "logps/rejected": -292.910400390625, "loss": 5.5175, "margin_dpo/margin_mean": 0.703624963760376, "margin_dpo/margin_std": 1.8807909488677979, "step": 51 }, { "KL/chosen_KL_mean": 1.238037109375, "KL/mean": 0.7984499335289001, "KL/rejected_KL_mean": 0.35886383056640625, "KL/std": 1.439669132232666, "epoch": 0.10890052356020942, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8791704773902893, "fcm_dpo/q_t": 0.4978024363517761, "grad_norm": 17.054243087768555, "learning_rate": 4.99939671821067e-07, "logits/chosen": 2.004744052886963, "logits/rejected": 1.925959587097168, "logps/chosen": -297.1872863769531, "logps/ref_chosen": -298.42529296875, "logps/ref_rejected": -278.5128479003906, "logps/rejected": -278.1539611816406, "loss": 5.5105, "margin_dpo/margin_mean": 0.8791694045066833, "margin_dpo/margin_std": 1.9268198013305664, "step": 52 }, { "KL/chosen_KL_mean": 1.1849250793457031, "KL/mean": 0.8460350632667542, "KL/rejected_KL_mean": 0.5071334838867188, "KL/std": 1.4284039735794067, "epoch": 0.11099476439790576, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6777990460395813, "fcm_dpo/q_t": 0.49830570816993713, "grad_norm": 15.994030952453613, "learning_rate": 4.998927532591591e-07, "logits/chosen": 1.5852761268615723, "logits/rejected": 1.6465017795562744, "logps/chosen": -274.1235656738281, "logps/ref_chosen": -275.3085021972656, "logps/ref_rejected": -267.97381591796875, "logps/rejected": -267.4667053222656, "loss": 5.5185, "margin_dpo/margin_mean": 0.6777995824813843, "margin_dpo/margin_std": 1.7592477798461914, "step": 53 }, { "KL/chosen_KL_mean": 1.2797298431396484, "KL/mean": 0.9198053479194641, "KL/rejected_KL_mean": 0.5598831176757812, "KL/std": 1.4647531509399414, "epoch": 0.1130890052356021, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.719858705997467, "fcm_dpo/q_t": 0.4982006251811981, "grad_norm": 14.507118225097656, "learning_rate": 4.998324337072792e-07, "logits/chosen": 1.733615756034851, "logits/rejected": 1.765891432762146, "logps/chosen": -296.8675842285156, "logps/ref_chosen": -298.1473388671875, "logps/ref_rejected": -269.7539367675781, "logps/rejected": -269.19403076171875, "loss": 5.5169, "margin_dpo/margin_mean": 0.7198596000671387, "margin_dpo/margin_std": 1.952254295349121, "step": 54 }, { "KL/chosen_KL_mean": 1.1664409637451172, "KL/mean": 0.8456940650939941, "KL/rejected_KL_mean": 0.5249443054199219, "KL/std": 1.3166602849960327, "epoch": 0.11518324607329843, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6415013074874878, "fcm_dpo/q_t": 0.4983964264392853, "grad_norm": 12.762544631958008, "learning_rate": 4.997587164001815e-07, "logits/chosen": 1.521553874015808, "logits/rejected": 1.5612958669662476, "logps/chosen": -215.5835418701172, "logps/ref_chosen": -216.74998474121094, "logps/ref_rejected": -202.7234649658203, "logps/rejected": -202.19850158691406, "loss": 5.5199, "margin_dpo/margin_mean": 0.6415011882781982, "margin_dpo/margin_std": 1.7675597667694092, "step": 55 }, { "KL/chosen_KL_mean": 1.2932567596435547, "KL/mean": 0.7695432305335999, "KL/rejected_KL_mean": 0.2458324432373047, "KL/std": 1.81564199924469, "epoch": 0.11727748691099477, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0474203824996948, "fcm_dpo/q_t": 0.4973820149898529, "grad_norm": 14.452275276184082, "learning_rate": 4.996716052911017e-07, "logits/chosen": 1.5290578603744507, "logits/rejected": 1.615087866783142, "logps/chosen": -242.39816284179688, "logps/ref_chosen": -243.69140625, "logps/ref_rejected": -234.970947265625, "logps/rejected": -234.72509765625, "loss": 5.504, "margin_dpo/margin_mean": 1.0474200248718262, "margin_dpo/margin_std": 2.32503604888916, "step": 56 }, { "KL/chosen_KL_mean": 1.7666187286376953, "KL/mean": 1.0542978048324585, "KL/rejected_KL_mean": 0.34197998046875, "KL/std": 1.7339706420898438, "epoch": 0.1193717277486911, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4246318340301514, "fcm_dpo/q_t": 0.4964395761489868, "grad_norm": 17.599245071411133, "learning_rate": 4.99571105051544e-07, "logits/chosen": 2.0374603271484375, "logits/rejected": 1.935616135597229, "logps/chosen": -308.9797058105469, "logps/ref_chosen": -310.746337890625, "logps/ref_rejected": -270.6407165527344, "logps/rejected": -270.2987060546875, "loss": 5.489, "margin_dpo/margin_mean": 1.424631118774414, "margin_dpo/margin_std": 2.1392083168029785, "step": 57 }, { "KL/chosen_KL_mean": 1.3137035369873047, "KL/mean": 0.8551568388938904, "KL/rejected_KL_mean": 0.3966217041015625, "KL/std": 1.615386962890625, "epoch": 0.12146596858638743, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9170898199081421, "fcm_dpo/q_t": 0.49770745635032654, "grad_norm": 14.97243881225586, "learning_rate": 4.994572210710314e-07, "logits/chosen": 1.744799256324768, "logits/rejected": 1.7298442125320435, "logps/chosen": -257.4992980957031, "logps/ref_chosen": -258.8130187988281, "logps/ref_rejected": -268.2101745605469, "logps/rejected": -267.8135681152344, "loss": 5.509, "margin_dpo/margin_mean": 0.917088508605957, "margin_dpo/margin_std": 1.958569884300232, "step": 58 }, { "KL/chosen_KL_mean": 1.4245281219482422, "KL/mean": 1.0160744190216064, "KL/rejected_KL_mean": 0.6076259613037109, "KL/std": 1.923659086227417, "epoch": 0.12356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8169017434120178, "fcm_dpo/q_t": 0.4979582726955414, "grad_norm": 15.254434585571289, "learning_rate": 4.993299594568162e-07, "logits/chosen": 1.7835378646850586, "logits/rejected": 1.7226471900939941, "logps/chosen": -275.8169250488281, "logps/ref_chosen": -277.2414245605469, "logps/ref_rejected": -259.13519287109375, "logps/rejected": -258.5275573730469, "loss": 5.5133, "margin_dpo/margin_mean": 0.8169001340866089, "margin_dpo/margin_std": 2.5672593116760254, "step": 59 }, { "KL/chosen_KL_mean": 1.5428886413574219, "KL/mean": 0.9007205367088318, "KL/rejected_KL_mean": 0.2585639953613281, "KL/std": 1.732983112335205, "epoch": 0.1256544502617801, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2843198776245117, "fcm_dpo/q_t": 0.496789813041687, "grad_norm": 14.663458824157715, "learning_rate": 4.991893270335525e-07, "logits/chosen": 1.8974318504333496, "logits/rejected": 1.943682312965393, "logps/chosen": -249.80296325683594, "logps/ref_chosen": -251.34585571289062, "logps/ref_rejected": -251.4120635986328, "logps/rejected": -251.1534881591797, "loss": 5.4946, "margin_dpo/margin_mean": 1.2843196392059326, "margin_dpo/margin_std": 2.2694783210754395, "step": 60 }, { "KL/chosen_KL_mean": 1.6408882141113281, "KL/mean": 1.103342056274414, "KL/rejected_KL_mean": 0.5658016204833984, "KL/std": 1.8047294616699219, "epoch": 0.12774869109947645, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.075092077255249, "fcm_dpo/q_t": 0.49731284379959106, "grad_norm": 15.019508361816406, "learning_rate": 4.990353313429303e-07, "logits/chosen": 1.7713136672973633, "logits/rejected": 1.725186824798584, "logps/chosen": -267.34173583984375, "logps/ref_chosen": -268.98260498046875, "logps/ref_rejected": -249.34474182128906, "logps/rejected": -248.7789306640625, "loss": 5.503, "margin_dpo/margin_mean": 1.0750921964645386, "margin_dpo/margin_std": 2.428819417953491, "step": 61 }, { "KL/chosen_KL_mean": 2.0969371795654297, "KL/mean": 1.4681521654129028, "KL/rejected_KL_mean": 0.8393688201904297, "KL/std": 2.045714855194092, "epoch": 0.12984293193717278, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2575669288635254, "fcm_dpo/q_t": 0.4968569278717041, "grad_norm": 14.69105052947998, "learning_rate": 4.988679806432711e-07, "logits/chosen": 1.9424972534179688, "logits/rejected": 1.9180412292480469, "logps/chosen": -262.1065673828125, "logps/ref_chosen": -264.2034912109375, "logps/ref_rejected": -260.2911376953125, "logps/rejected": -259.4517822265625, "loss": 5.4958, "margin_dpo/margin_mean": 1.2575657367706299, "margin_dpo/margin_std": 2.6261508464813232, "step": 62 }, { "KL/chosen_KL_mean": 1.8828144073486328, "KL/mean": 1.2346196174621582, "KL/rejected_KL_mean": 0.5864162445068359, "KL/std": 2.0843284130096436, "epoch": 0.1319371727748691, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2964017391204834, "fcm_dpo/q_t": 0.49675989151000977, "grad_norm": 14.768925666809082, "learning_rate": 4.986872839090852e-07, "logits/chosen": 1.7671276330947876, "logits/rejected": 1.7914533615112305, "logps/chosen": -269.5854797363281, "logps/ref_chosen": -271.4682922363281, "logps/ref_rejected": -273.0804748535156, "logps/rejected": -272.4940490722656, "loss": 5.4943, "margin_dpo/margin_mean": 1.2964012622833252, "margin_dpo/margin_std": 2.744537591934204, "step": 63 }, { "KL/chosen_KL_mean": 2.306966781616211, "KL/mean": 1.525729775428772, "KL/rejected_KL_mean": 0.7444896697998047, "KL/std": 2.388784170150757, "epoch": 0.13403141361256546, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5624661445617676, "fcm_dpo/q_t": 0.4960950016975403, "grad_norm": 15.698229789733887, "learning_rate": 4.9849325083059e-07, "logits/chosen": 1.7998967170715332, "logits/rejected": 1.8028191328048706, "logps/chosen": -271.29644775390625, "logps/ref_chosen": -273.6033935546875, "logps/ref_rejected": -259.54107666015625, "logps/rejected": -258.79656982421875, "loss": 5.484, "margin_dpo/margin_mean": 1.5624663829803467, "margin_dpo/margin_std": 3.152646064758301, "step": 64 }, { "KL/chosen_KL_mean": 2.3221817016601562, "KL/mean": 1.769188404083252, "KL/rejected_KL_mean": 1.2161998748779297, "KL/std": 2.1231086254119873, "epoch": 0.13612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1059739589691162, "fcm_dpo/q_t": 0.4972355365753174, "grad_norm": 14.570204734802246, "learning_rate": 4.982858918131906e-07, "logits/chosen": 1.8644115924835205, "logits/rejected": 1.962466835975647, "logps/chosen": -256.19549560546875, "logps/ref_chosen": -258.5176696777344, "logps/ref_rejected": -264.9944763183594, "logps/rejected": -263.77825927734375, "loss": 5.5019, "margin_dpo/margin_mean": 1.1059731245040894, "margin_dpo/margin_std": 2.760225534439087, "step": 65 }, { "KL/chosen_KL_mean": 2.2520103454589844, "KL/mean": 1.5462473630905151, "KL/rejected_KL_mean": 0.8404960632324219, "KL/std": 2.6777021884918213, "epoch": 0.1382198952879581, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.411508321762085, "fcm_dpo/q_t": 0.49647241830825806, "grad_norm": 14.818696022033691, "learning_rate": 4.980652179769217e-07, "logits/chosen": 1.9537615776062012, "logits/rejected": 2.05753493309021, "logps/chosen": -265.2574157714844, "logps/ref_chosen": -267.5094299316406, "logps/ref_rejected": -277.03228759765625, "logps/rejected": -276.1917724609375, "loss": 5.4903, "margin_dpo/margin_mean": 1.4115078449249268, "margin_dpo/margin_std": 3.628791332244873, "step": 66 }, { "KL/chosen_KL_mean": 2.579130172729492, "KL/mean": 1.9668623208999634, "KL/rejected_KL_mean": 1.354593276977539, "KL/std": 2.6205031871795654, "epoch": 0.14031413612565444, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.224534511566162, "fcm_dpo/q_t": 0.49693942070007324, "grad_norm": 14.398809432983398, "learning_rate": 4.978312411558517e-07, "logits/chosen": 1.6984463930130005, "logits/rejected": 1.705766201019287, "logps/chosen": -254.8858184814453, "logps/ref_chosen": -257.4649353027344, "logps/ref_rejected": -247.29161071777344, "logps/rejected": -245.93704223632812, "loss": 5.4975, "margin_dpo/margin_mean": 1.2245352268218994, "margin_dpo/margin_std": 3.272260904312134, "step": 67 }, { "KL/chosen_KL_mean": 2.4789600372314453, "KL/mean": 1.6513326168060303, "KL/rejected_KL_mean": 0.8237094879150391, "KL/std": 2.9045519828796387, "epoch": 0.1424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6552443504333496, "fcm_dpo/q_t": 0.4958641231060028, "grad_norm": 14.775374412536621, "learning_rate": 4.975839738974473e-07, "logits/chosen": 1.700913667678833, "logits/rejected": 1.6477922201156616, "logps/chosen": -288.0052185058594, "logps/ref_chosen": -290.4841613769531, "logps/ref_rejected": -260.8398132324219, "logps/rejected": -260.01611328125, "loss": 5.4809, "margin_dpo/margin_mean": 1.6552444696426392, "margin_dpo/margin_std": 3.8591935634613037, "step": 68 }, { "KL/chosen_KL_mean": 2.830690383911133, "KL/mean": 1.7516111135482788, "KL/rejected_KL_mean": 0.6725292205810547, "KL/std": 2.9886691570281982, "epoch": 0.14450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.1581647396087646, "fcm_dpo/q_t": 0.4946069121360779, "grad_norm": 15.165844917297363, "learning_rate": 4.97323429461901e-07, "logits/chosen": 1.8554754257202148, "logits/rejected": 1.893682599067688, "logps/chosen": -254.806884765625, "logps/ref_chosen": -257.6375732421875, "logps/ref_rejected": -235.6666717529297, "logps/rejected": -234.994140625, "loss": 5.4608, "margin_dpo/margin_mean": 2.1581649780273438, "margin_dpo/margin_std": 3.7994115352630615, "step": 69 }, { "KL/chosen_KL_mean": 3.446706771850586, "KL/mean": 2.358335494995117, "KL/rejected_KL_mean": 1.2699604034423828, "KL/std": 3.0744781494140625, "epoch": 0.14659685863874344, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.176751136779785, "fcm_dpo/q_t": 0.4945610463619232, "grad_norm": 15.59477424621582, "learning_rate": 4.970496218214204e-07, "logits/chosen": 1.6959162950515747, "logits/rejected": 1.6481292247772217, "logps/chosen": -284.01849365234375, "logps/ref_chosen": -287.4651794433594, "logps/ref_rejected": -253.4770965576172, "logps/rejected": -252.20713806152344, "loss": 5.4602, "margin_dpo/margin_mean": 2.1767518520355225, "margin_dpo/margin_std": 3.827840566635132, "step": 70 }, { "KL/chosen_KL_mean": 3.4551048278808594, "KL/mean": 2.5311903953552246, "KL/rejected_KL_mean": 1.6072845458984375, "KL/std": 3.245633840560913, "epoch": 0.1486910994764398, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.847829818725586, "fcm_dpo/q_t": 0.4953823685646057, "grad_norm": 14.982972145080566, "learning_rate": 4.967625656594781e-07, "logits/chosen": 1.964093804359436, "logits/rejected": 1.9719585180282593, "logps/chosen": -277.4803161621094, "logps/ref_chosen": -280.9353942871094, "logps/ref_rejected": -275.998779296875, "logps/rejected": -274.3915100097656, "loss": 5.4736, "margin_dpo/margin_mean": 1.8478305339813232, "margin_dpo/margin_std": 4.226970195770264, "step": 71 }, { "KL/chosen_KL_mean": 3.068317413330078, "KL/mean": 2.0340206623077393, "KL/rejected_KL_mean": 0.9997177124023438, "KL/std": 3.0637056827545166, "epoch": 0.15078534031413612, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0686075687408447, "fcm_dpo/q_t": 0.4948310852050781, "grad_norm": 15.693150520324707, "learning_rate": 4.964622763700252e-07, "logits/chosen": 1.7748774290084839, "logits/rejected": 1.9190484285354614, "logps/chosen": -227.07720947265625, "logps/ref_chosen": -230.14553833007812, "logps/ref_rejected": -253.37661743164062, "logps/rejected": -252.37689208984375, "loss": 5.4646, "margin_dpo/margin_mean": 2.068606376647949, "margin_dpo/margin_std": 3.9270946979522705, "step": 72 }, { "KL/chosen_KL_mean": 3.863943099975586, "KL/mean": 2.885406494140625, "KL/rejected_KL_mean": 1.9068660736083984, "KL/std": 3.2366580963134766, "epoch": 0.15287958115183245, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9570707082748413, "fcm_dpo/q_t": 0.49510952830314636, "grad_norm": 14.728974342346191, "learning_rate": 4.961487700566646e-07, "logits/chosen": 1.6069412231445312, "logits/rejected": 1.621825933456421, "logps/chosen": -261.0023498535156, "logps/ref_chosen": -264.86627197265625, "logps/ref_rejected": -245.21304321289062, "logps/rejected": -243.30616760253906, "loss": 5.4694, "margin_dpo/margin_mean": 1.957069993019104, "margin_dpo/margin_std": 4.404537677764893, "step": 73 }, { "KL/chosen_KL_mean": 3.792757034301758, "KL/mean": 2.8318533897399902, "KL/rejected_KL_mean": 1.8709602355957031, "KL/std": 3.567683458328247, "epoch": 0.1549738219895288, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.921801209449768, "fcm_dpo/q_t": 0.4951982796192169, "grad_norm": 15.760645866394043, "learning_rate": 4.958220635317885e-07, "logits/chosen": 1.634965419769287, "logits/rejected": 1.6836990118026733, "logps/chosen": -330.7607421875, "logps/ref_chosen": -334.55352783203125, "logps/ref_rejected": -325.8403015136719, "logps/rejected": -323.9693298339844, "loss": 5.471, "margin_dpo/margin_mean": 1.9218016862869263, "margin_dpo/margin_std": 4.589147567749023, "step": 74 }, { "KL/chosen_KL_mean": 4.494831085205078, "KL/mean": 3.165982723236084, "KL/rejected_KL_mean": 1.8371295928955078, "KL/std": 3.5341873168945312, "epoch": 0.15706806282722513, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.657700538635254, "fcm_dpo/q_t": 0.4933589696884155, "grad_norm": 15.213676452636719, "learning_rate": 4.954821743156767e-07, "logits/chosen": 1.6492834091186523, "logits/rejected": 1.7265129089355469, "logps/chosen": -255.92706298828125, "logps/ref_chosen": -260.421875, "logps/ref_rejected": -273.0882263183594, "logps/rejected": -271.2510986328125, "loss": 5.4415, "margin_dpo/margin_mean": 2.657698631286621, "margin_dpo/margin_std": 4.120148658752441, "step": 75 }, { "KL/chosen_KL_mean": 4.497751235961914, "KL/mean": 3.0013351440429688, "KL/rejected_KL_mean": 1.5049152374267578, "KL/std": 3.860976457595825, "epoch": 0.15916230366492146, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.992838144302368, "fcm_dpo/q_t": 0.4925258159637451, "grad_norm": 16.776811599731445, "learning_rate": 4.951291206355559e-07, "logits/chosen": 1.6961089372634888, "logits/rejected": 1.6757653951644897, "logps/chosen": -267.2377014160156, "logps/ref_chosen": -271.7354431152344, "logps/ref_rejected": -262.64111328125, "logps/rejected": -261.13623046875, "loss": 5.429, "margin_dpo/margin_mean": 2.992838144302368, "margin_dpo/margin_std": 4.868131637573242, "step": 76 }, { "KL/chosen_KL_mean": 4.730262756347656, "KL/mean": 3.5549404621124268, "KL/rejected_KL_mean": 2.3796234130859375, "KL/std": 4.021265029907227, "epoch": 0.1612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.3506383895874023, "fcm_dpo/q_t": 0.4941290318965912, "grad_norm": 14.854421615600586, "learning_rate": 4.947629214246236e-07, "logits/chosen": 1.7227365970611572, "logits/rejected": 1.5612424612045288, "logps/chosen": -289.94036865234375, "logps/ref_chosen": -294.6706237792969, "logps/ref_rejected": -247.6900177001953, "logps/rejected": -245.3104248046875, "loss": 5.4545, "margin_dpo/margin_mean": 2.350637674331665, "margin_dpo/margin_std": 5.070836067199707, "step": 77 }, { "KL/chosen_KL_mean": 5.183525085449219, "KL/mean": 3.701960325241089, "KL/rejected_KL_mean": 2.2203903198242188, "KL/std": 3.653484344482422, "epoch": 0.16335078534031414, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9631338119506836, "fcm_dpo/q_t": 0.49259960651397705, "grad_norm": 14.6358060836792, "learning_rate": 4.943835963210323e-07, "logits/chosen": 1.7129117250442505, "logits/rejected": 1.6462018489837646, "logps/chosen": -242.96875, "logps/ref_chosen": -248.1522979736328, "logps/ref_rejected": -209.12808227539062, "logps/rejected": -206.9076690673828, "loss": 5.4303, "margin_dpo/margin_mean": 2.963132619857788, "margin_dpo/margin_std": 5.155320167541504, "step": 78 }, { "KL/chosen_KL_mean": 4.984601974487305, "KL/mean": 3.4764275550842285, "KL/rejected_KL_mean": 1.9682502746582031, "KL/std": 4.8389410972595215, "epoch": 0.16544502617801046, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.016352653503418, "fcm_dpo/q_t": 0.4924572706222534, "grad_norm": 16.72459602355957, "learning_rate": 4.939911656668361e-07, "logits/chosen": 1.8387343883514404, "logits/rejected": 1.9029288291931152, "logps/chosen": -254.48977661132812, "logps/ref_chosen": -259.4743957519531, "logps/ref_rejected": -246.9627227783203, "logps/rejected": -244.99447631835938, "loss": 5.4301, "margin_dpo/margin_mean": 3.016352891921997, "margin_dpo/margin_std": 6.287435531616211, "step": 79 }, { "KL/chosen_KL_mean": 5.328868865966797, "KL/mean": 4.002456188201904, "KL/rejected_KL_mean": 2.6760520935058594, "KL/std": 4.738064289093018, "epoch": 0.16753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.652810573577881, "fcm_dpo/q_t": 0.4933757185935974, "grad_norm": 15.615460395812988, "learning_rate": 4.935856505068998e-07, "logits/chosen": 1.5117417573928833, "logits/rejected": 1.4887609481811523, "logps/chosen": -280.59765625, "logps/ref_chosen": -285.926513671875, "logps/ref_rejected": -253.77996826171875, "logps/rejected": -251.1039276123047, "loss": 5.4436, "margin_dpo/margin_mean": 2.652811288833618, "margin_dpo/margin_std": 5.693536758422852, "step": 80 }, { "KL/chosen_KL_mean": 5.9272003173828125, "KL/mean": 4.476429462432861, "KL/rejected_KL_mean": 3.025655746459961, "KL/std": 4.470715522766113, "epoch": 0.16963350785340314, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9015376567840576, "fcm_dpo/q_t": 0.49275198578834534, "grad_norm": 14.92487907409668, "learning_rate": 4.93167072587771e-07, "logits/chosen": 1.745436429977417, "logits/rejected": 1.6883275508880615, "logps/chosen": -256.6669921875, "logps/ref_chosen": -262.5942077636719, "logps/ref_rejected": -236.687744140625, "logps/rejected": -233.66207885742188, "loss": 5.4343, "margin_dpo/margin_mean": 2.9015393257141113, "margin_dpo/margin_std": 6.303562641143799, "step": 81 }, { "KL/chosen_KL_mean": 6.398937225341797, "KL/mean": 4.894276142120361, "KL/rejected_KL_mean": 3.3896026611328125, "KL/std": 4.335615634918213, "epoch": 0.17172774869109947, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.009336233139038, "fcm_dpo/q_t": 0.4924847185611725, "grad_norm": 15.691886901855469, "learning_rate": 4.92735454356513e-07, "logits/chosen": 1.8371354341506958, "logits/rejected": 1.7929863929748535, "logps/chosen": -264.64422607421875, "logps/ref_chosen": -271.04315185546875, "logps/ref_rejected": -234.03359985351562, "logps/rejected": -230.64398193359375, "loss": 5.4292, "margin_dpo/margin_mean": 3.009336233139038, "margin_dpo/margin_std": 5.506655216217041, "step": 82 }, { "KL/chosen_KL_mean": 6.437170028686523, "KL/mean": 4.668962001800537, "KL/rejected_KL_mean": 2.9007644653320312, "KL/std": 4.886711597442627, "epoch": 0.17382198952879582, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.5364108085632324, "fcm_dpo/q_t": 0.4911731779575348, "grad_norm": 15.337501525878906, "learning_rate": 4.922908189595017e-07, "logits/chosen": 1.7183247804641724, "logits/rejected": 1.793430209159851, "logps/chosen": -259.8481140136719, "logps/ref_chosen": -266.2852783203125, "logps/ref_rejected": -274.32501220703125, "logps/rejected": -271.42425537109375, "loss": 5.4096, "margin_dpo/margin_mean": 3.536411762237549, "margin_dpo/margin_std": 6.47735071182251, "step": 83 }, { "KL/chosen_KL_mean": 6.663084030151367, "KL/mean": 5.296658039093018, "KL/rejected_KL_mean": 3.9302330017089844, "KL/std": 5.117173671722412, "epoch": 0.17591623036649215, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.7328550815582275, "fcm_dpo/q_t": 0.493173211812973, "grad_norm": 15.184418678283691, "learning_rate": 4.918331902411841e-07, "logits/chosen": 1.8807992935180664, "logits/rejected": 1.8695476055145264, "logps/chosen": -280.1175231933594, "logps/ref_chosen": -286.7806091308594, "logps/ref_rejected": -276.1176452636719, "logps/rejected": -272.18743896484375, "loss": 5.4415, "margin_dpo/margin_mean": 2.732853651046753, "margin_dpo/margin_std": 6.627993583679199, "step": 84 }, { "KL/chosen_KL_mean": 6.175373077392578, "KL/mean": 5.1813273429870605, "KL/rejected_KL_mean": 4.187278747558594, "KL/std": 4.361845970153809, "epoch": 0.17801047120418848, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9880964756011963, "fcm_dpo/q_t": 0.4950335621833801, "grad_norm": 13.90788459777832, "learning_rate": 4.913625927427995e-07, "logits/chosen": 1.8967005014419556, "logits/rejected": 1.9972286224365234, "logps/chosen": -228.8544921875, "logps/ref_chosen": -235.02987670898438, "logps/ref_rejected": -259.80999755859375, "logps/rejected": -255.62274169921875, "loss": 5.4693, "margin_dpo/margin_mean": 1.9880964756011963, "margin_dpo/margin_std": 5.34002161026001, "step": 85 }, { "KL/chosen_KL_mean": 7.878423690795898, "KL/mean": 6.022519111633301, "KL/rejected_KL_mean": 4.166624069213867, "KL/std": 5.585781574249268, "epoch": 0.18010471204188483, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.711801052093506, "fcm_dpo/q_t": 0.4907352328300476, "grad_norm": 16.465648651123047, "learning_rate": 4.908790517010636e-07, "logits/chosen": 1.9428482055664062, "logits/rejected": 1.995483160018921, "logps/chosen": -293.434326171875, "logps/ref_chosen": -301.312744140625, "logps/ref_rejected": -288.8368225097656, "logps/rejected": -284.6701965332031, "loss": 5.404, "margin_dpo/margin_mean": 3.7118024826049805, "margin_dpo/margin_std": 7.516912460327148, "step": 86 }, { "KL/chosen_KL_mean": 8.02880859375, "KL/mean": 5.975081443786621, "KL/rejected_KL_mean": 3.921346664428711, "KL/std": 5.473031044006348, "epoch": 0.18219895287958116, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.10746431350708, "fcm_dpo/q_t": 0.48974609375, "grad_norm": 15.32911205291748, "learning_rate": 4.903825930468148e-07, "logits/chosen": 1.7233219146728516, "logits/rejected": 1.7099597454071045, "logps/chosen": -257.6371765136719, "logps/ref_chosen": -265.6659851074219, "logps/ref_rejected": -242.0697479248047, "logps/rejected": -238.14840698242188, "loss": 5.387, "margin_dpo/margin_mean": 4.10746431350708, "margin_dpo/margin_std": 6.42798376083374, "step": 87 }, { "KL/chosen_KL_mean": 7.276971817016602, "KL/mean": 5.6905622482299805, "KL/rejected_KL_mean": 4.104146957397461, "KL/std": 5.471114158630371, "epoch": 0.18429319371727748, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1728219985961914, "fcm_dpo/q_t": 0.4920782744884491, "grad_norm": 14.059208869934082, "learning_rate": 4.898732434036243e-07, "logits/chosen": 1.7752293348312378, "logits/rejected": 1.7928274869918823, "logps/chosen": -253.44447326660156, "logps/ref_chosen": -260.721435546875, "logps/ref_rejected": -262.6593322753906, "logps/rejected": -258.55517578125, "loss": 5.4247, "margin_dpo/margin_mean": 3.1728243827819824, "margin_dpo/margin_std": 7.120820999145508, "step": 88 }, { "KL/chosen_KL_mean": 8.954307556152344, "KL/mean": 7.050387859344482, "KL/rejected_KL_mean": 5.14647102355957, "KL/std": 5.224122524261475, "epoch": 0.18638743455497384, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.8078417778015137, "fcm_dpo/q_t": 0.4904939532279968, "grad_norm": 15.816296577453613, "learning_rate": 4.893510300863676e-07, "logits/chosen": 1.7604267597198486, "logits/rejected": 1.7203480005264282, "logps/chosen": -248.0485076904297, "logps/ref_chosen": -257.0028076171875, "logps/ref_rejected": -254.66232299804688, "logps/rejected": -249.515869140625, "loss": 5.3988, "margin_dpo/margin_mean": 3.807842493057251, "margin_dpo/margin_std": 6.273983955383301, "step": 89 }, { "KL/chosen_KL_mean": 8.962364196777344, "KL/mean": 7.273670196533203, "KL/rejected_KL_mean": 5.58497428894043, "KL/std": 5.62357759475708, "epoch": 0.18848167539267016, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3773882389068604, "fcm_dpo/q_t": 0.49156898260116577, "grad_norm": 15.256770133972168, "learning_rate": 4.8881598109976e-07, "logits/chosen": 2.0314841270446777, "logits/rejected": 1.976723551750183, "logps/chosen": -289.4480285644531, "logps/ref_chosen": -298.4103698730469, "logps/ref_rejected": -262.8761901855469, "logps/rejected": -257.2912292480469, "loss": 5.4166, "margin_dpo/margin_mean": 3.3773880004882812, "margin_dpo/margin_std": 6.8097686767578125, "step": 90 }, { "KL/chosen_KL_mean": 8.793439865112305, "KL/mean": 7.109142780303955, "KL/rejected_KL_mean": 5.424846649169922, "KL/std": 5.423921585083008, "epoch": 0.1905759162303665, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.368590831756592, "fcm_dpo/q_t": 0.4915948510169983, "grad_norm": 14.69740104675293, "learning_rate": 4.882681251368548e-07, "logits/chosen": 1.6287318468093872, "logits/rejected": 1.6709989309310913, "logps/chosen": -218.73043823242188, "logps/ref_chosen": -227.5238800048828, "logps/ref_rejected": -227.11428833007812, "logps/rejected": -221.689453125, "loss": 5.4173, "margin_dpo/margin_mean": 3.368590831756592, "margin_dpo/margin_std": 7.276670455932617, "step": 91 }, { "KL/chosen_KL_mean": 9.003055572509766, "KL/mean": 7.0606303215026855, "KL/rejected_KL_mean": 5.118196487426758, "KL/std": 6.609164714813232, "epoch": 0.19267015706806281, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.8848636150360107, "fcm_dpo/q_t": 0.4903002381324768, "grad_norm": 15.576513290405273, "learning_rate": 4.877074915775048e-07, "logits/chosen": 1.769035816192627, "logits/rejected": 1.6670876741409302, "logps/chosen": -266.3791809082031, "logps/ref_chosen": -275.3822326660156, "logps/ref_rejected": -269.8356018066406, "logps/rejected": -264.7174072265625, "loss": 5.3976, "margin_dpo/margin_mean": 3.8848628997802734, "margin_dpo/margin_std": 7.702731132507324, "step": 92 }, { "KL/chosen_KL_mean": 9.199054718017578, "KL/mean": 7.457064151763916, "KL/rejected_KL_mean": 5.715080261230469, "KL/std": 5.752642631530762, "epoch": 0.19476439790575917, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.483973503112793, "fcm_dpo/q_t": 0.4913066029548645, "grad_norm": 14.088767051696777, "learning_rate": 4.871341104867864e-07, "logits/chosen": 1.7151176929473877, "logits/rejected": 1.6953684091567993, "logps/chosen": -218.91732788085938, "logps/ref_chosen": -228.1163787841797, "logps/ref_rejected": -228.26266479492188, "logps/rejected": -222.54757690429688, "loss": 5.4126, "margin_dpo/margin_mean": 3.4839727878570557, "margin_dpo/margin_std": 7.183119773864746, "step": 93 }, { "KL/chosen_KL_mean": 10.13239860534668, "KL/mean": 8.115608215332031, "KL/rejected_KL_mean": 6.098814010620117, "KL/std": 6.654098987579346, "epoch": 0.1968586387434555, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.033576965332031, "fcm_dpo/q_t": 0.489940881729126, "grad_norm": 15.566554069519043, "learning_rate": 4.865480126133871e-07, "logits/chosen": 1.7142863273620605, "logits/rejected": 1.8038783073425293, "logps/chosen": -273.89483642578125, "logps/ref_chosen": -284.0272521972656, "logps/ref_rejected": -282.1720275878906, "logps/rejected": -276.0732116699219, "loss": 5.3938, "margin_dpo/margin_mean": 4.0335774421691895, "margin_dpo/margin_std": 8.711073875427246, "step": 94 }, { "KL/chosen_KL_mean": 9.834869384765625, "KL/mean": 7.612403869628906, "KL/rejected_KL_mean": 5.389949798583984, "KL/std": 6.842440128326416, "epoch": 0.19895287958115182, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.444918632507324, "fcm_dpo/q_t": 0.4889160394668579, "grad_norm": 15.96788215637207, "learning_rate": 4.859492293879573e-07, "logits/chosen": 1.8246318101882935, "logits/rejected": 1.7539443969726562, "logps/chosen": -293.2149658203125, "logps/ref_chosen": -303.0498352050781, "logps/ref_rejected": -259.8980712890625, "logps/rejected": -254.50814819335938, "loss": 5.3776, "margin_dpo/margin_mean": 4.444916725158691, "margin_dpo/margin_std": 8.767796516418457, "step": 95 }, { "KL/chosen_KL_mean": 9.86693000793457, "KL/mean": 7.7021965980529785, "KL/rejected_KL_mean": 5.537467956542969, "KL/std": 6.970180511474609, "epoch": 0.20104712041884817, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.329456329345703, "fcm_dpo/q_t": 0.4892018437385559, "grad_norm": 15.623520851135254, "learning_rate": 4.853377929214243e-07, "logits/chosen": 1.7852537631988525, "logits/rejected": 1.7468805313110352, "logps/chosen": -260.7207336425781, "logps/ref_chosen": -270.58770751953125, "logps/ref_rejected": -238.90069580078125, "logps/rejected": -233.36322021484375, "loss": 5.3808, "margin_dpo/margin_mean": 4.329455375671387, "margin_dpo/margin_std": 7.833381175994873, "step": 96 }, { "KL/chosen_KL_mean": 9.781450271606445, "KL/mean": 7.512989521026611, "KL/rejected_KL_mean": 5.244525909423828, "KL/std": 6.864675521850586, "epoch": 0.2031413612565445, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.53691291809082, "fcm_dpo/q_t": 0.4886978268623352, "grad_norm": 15.31660270690918, "learning_rate": 4.847137360032699e-07, "logits/chosen": 1.8315813541412354, "logits/rejected": 1.7155046463012695, "logps/chosen": -286.40411376953125, "logps/ref_chosen": -296.1855773925781, "logps/ref_rejected": -260.35333251953125, "logps/rejected": -255.10879516601562, "loss": 5.3753, "margin_dpo/margin_mean": 4.5369133949279785, "margin_dpo/margin_std": 9.493367195129395, "step": 97 }, { "KL/chosen_KL_mean": 10.113018035888672, "KL/mean": 7.6967668533325195, "KL/rejected_KL_mean": 5.280508041381836, "KL/std": 7.416313648223877, "epoch": 0.20523560209424083, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.8325090408325195, "fcm_dpo/q_t": 0.48795896768569946, "grad_norm": 15.75379467010498, "learning_rate": 4.84077092099773e-07, "logits/chosen": 2.001626491546631, "logits/rejected": 1.9910184144973755, "logps/chosen": -267.6023254394531, "logps/ref_chosen": -277.7153625488281, "logps/ref_rejected": -272.9094543457031, "logps/rejected": -267.6289367675781, "loss": 5.3648, "margin_dpo/margin_mean": 4.8325090408325195, "margin_dpo/margin_std": 9.47249984741211, "step": 98 }, { "KL/chosen_KL_mean": 10.356014251708984, "KL/mean": 7.7745161056518555, "KL/rejected_KL_mean": 5.193023681640625, "KL/std": 7.610881328582764, "epoch": 0.20732984293193718, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.162997722625732, "fcm_dpo/q_t": 0.48712167143821716, "grad_norm": 15.03431224822998, "learning_rate": 4.834278953522137e-07, "logits/chosen": 1.7922710180282593, "logits/rejected": 1.7652747631072998, "logps/chosen": -262.1817932128906, "logps/ref_chosen": -272.5378112792969, "logps/ref_rejected": -244.6329345703125, "logps/rejected": -239.43991088867188, "loss": 5.3527, "margin_dpo/margin_mean": 5.162997722625732, "margin_dpo/margin_std": 10.040070533752441, "step": 99 }, { "KL/chosen_KL_mean": 9.370292663574219, "KL/mean": 7.144068717956543, "KL/rejected_KL_mean": 4.917854309082031, "KL/std": 7.839417457580566, "epoch": 0.2094240837696335, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.452436923980713, "fcm_dpo/q_t": 0.4889054000377655, "grad_norm": 15.50412368774414, "learning_rate": 4.827661805750437e-07, "logits/chosen": 1.7104381322860718, "logits/rejected": 1.6327906847000122, "logps/chosen": -280.44207763671875, "logps/ref_chosen": -289.8123474121094, "logps/ref_rejected": -274.29071044921875, "logps/rejected": -269.3728332519531, "loss": 5.3786, "margin_dpo/margin_mean": 4.452437400817871, "margin_dpo/margin_std": 9.089731216430664, "step": 100 }, { "KL/chosen_KL_mean": 9.941091537475586, "KL/mean": 7.673428535461426, "KL/rejected_KL_mean": 5.405767440795898, "KL/std": 8.093482971191406, "epoch": 0.21151832460732983, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.535319805145264, "fcm_dpo/q_t": 0.48869484663009644, "grad_norm": 15.68911075592041, "learning_rate": 4.820919832540181e-07, "logits/chosen": 1.7467799186706543, "logits/rejected": 1.9193204641342163, "logps/chosen": -235.77203369140625, "logps/ref_chosen": -245.71311950683594, "logps/ref_rejected": -272.08935546875, "logps/rejected": -266.68359375, "loss": 5.3766, "margin_dpo/margin_mean": 4.5353193283081055, "margin_dpo/margin_std": 9.767685890197754, "step": 101 }, { "KL/chosen_KL_mean": 10.382354736328125, "KL/mean": 7.193247318267822, "KL/rejected_KL_mean": 4.004146575927734, "KL/std": 7.537840843200684, "epoch": 0.2136125654450262, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.378210067749023, "fcm_dpo/q_t": 0.4840970039367676, "grad_norm": 15.268579483032227, "learning_rate": 4.814053395442932e-07, "logits/chosen": 1.5934196710586548, "logits/rejected": 1.6801108121871948, "logps/chosen": -199.56005859375, "logps/ref_chosen": -209.94239807128906, "logps/ref_rejected": -230.66897583007812, "logps/rejected": -226.66481018066406, "loss": 5.3031, "margin_dpo/margin_mean": 6.378209114074707, "margin_dpo/margin_std": 8.972631454467773, "step": 102 }, { "KL/chosen_KL_mean": 9.419265747070312, "KL/mean": 7.0547308921813965, "KL/rejected_KL_mean": 4.690199851989746, "KL/std": 8.282856941223145, "epoch": 0.2157068062827225, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.729057312011719, "fcm_dpo/q_t": 0.48822057247161865, "grad_norm": 16.341489791870117, "learning_rate": 4.807062862684873e-07, "logits/chosen": 1.7020974159240723, "logits/rejected": 1.7979882955551147, "logps/chosen": -241.88037109375, "logps/ref_chosen": -251.2996368408203, "logps/ref_rejected": -277.4847412109375, "logps/rejected": -272.7945251464844, "loss": 5.3694, "margin_dpo/margin_mean": 4.729056358337402, "margin_dpo/margin_std": 10.193910598754883, "step": 103 }, { "KL/chosen_KL_mean": 8.38427734375, "KL/mean": 6.163862705230713, "KL/rejected_KL_mean": 3.943441390991211, "KL/std": 8.543988227844238, "epoch": 0.21780104712041884, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.440829753875732, "fcm_dpo/q_t": 0.4889433681964874, "grad_norm": 14.319323539733887, "learning_rate": 4.799948609147061e-07, "logits/chosen": 1.757680892944336, "logits/rejected": 1.796667218208313, "logps/chosen": -249.39512634277344, "logps/ref_chosen": -257.7794189453125, "logps/ref_rejected": -227.91546630859375, "logps/rejected": -223.97201538085938, "loss": 5.3825, "margin_dpo/margin_mean": 4.440830230712891, "margin_dpo/margin_std": 11.020442962646484, "step": 104 }, { "KL/chosen_KL_mean": 10.291145324707031, "KL/mean": 6.228615760803223, "KL/rejected_KL_mean": 2.1660938262939453, "KL/std": 9.67591381072998, "epoch": 0.2198952879581152, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.125062942504883, "fcm_dpo/q_t": 0.47975796461105347, "grad_norm": 17.875959396362305, "learning_rate": 4.792711016345321e-07, "logits/chosen": 1.8722009658813477, "logits/rejected": 1.8510180711746216, "logps/chosen": -301.8028869628906, "logps/ref_chosen": -312.09405517578125, "logps/ref_rejected": -277.97332763671875, "logps/rejected": -275.8072204589844, "loss": 5.2393, "margin_dpo/margin_mean": 8.125061988830566, "margin_dpo/margin_std": 10.691678047180176, "step": 105 }, { "KL/chosen_KL_mean": 6.734123229980469, "KL/mean": 3.977590560913086, "KL/rejected_KL_mean": 1.221048355102539, "KL/std": 8.91702938079834, "epoch": 0.22198952879581152, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.513080596923828, "fcm_dpo/q_t": 0.48627668619155884, "grad_norm": 16.416852951049805, "learning_rate": 4.785350472409791e-07, "logits/chosen": 1.6931140422821045, "logits/rejected": 1.6858336925506592, "logps/chosen": -278.6322326660156, "logps/ref_chosen": -285.3663330078125, "logps/ref_rejected": -262.2779541015625, "logps/rejected": -261.056884765625, "loss": 5.3415, "margin_dpo/margin_mean": 5.513082027435303, "margin_dpo/margin_std": 11.19965934753418, "step": 106 }, { "KL/chosen_KL_mean": 9.352890014648438, "KL/mean": 5.674660682678223, "KL/rejected_KL_mean": 1.9964275360107422, "KL/std": 9.54844856262207, "epoch": 0.22408376963350785, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.356465816497803, "fcm_dpo/q_t": 0.48166128993034363, "grad_norm": 16.4990234375, "learning_rate": 4.777867372064105e-07, "logits/chosen": 1.9865798950195312, "logits/rejected": 2.064659833908081, "logps/chosen": -286.9407043457031, "logps/ref_chosen": -296.2935791015625, "logps/ref_rejected": -291.5162353515625, "logps/rejected": -289.519775390625, "loss": 5.2719, "margin_dpo/margin_mean": 7.356466293334961, "margin_dpo/margin_std": 11.690131187438965, "step": 107 }, { "KL/chosen_KL_mean": 9.395183563232422, "KL/mean": 5.331313610076904, "KL/rejected_KL_mean": 1.267435073852539, "KL/std": 8.979668617248535, "epoch": 0.2261780104712042, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.127754211425781, "fcm_dpo/q_t": 0.47975727915763855, "grad_norm": 16.080812454223633, "learning_rate": 4.770262116604223e-07, "logits/chosen": 1.9511507749557495, "logits/rejected": 1.8253110647201538, "logps/chosen": -278.7291564941406, "logps/ref_chosen": -288.12432861328125, "logps/ref_rejected": -232.35720825195312, "logps/rejected": -231.08978271484375, "loss": 5.2387, "margin_dpo/margin_mean": 8.127753257751465, "margin_dpo/margin_std": 10.594194412231445, "step": 108 }, { "KL/chosen_KL_mean": 8.109546661376953, "KL/mean": 3.8482954502105713, "KL/rejected_KL_mean": -0.4129467010498047, "KL/std": 10.208244323730469, "epoch": 0.22827225130890053, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.522490501403809, "fcm_dpo/q_t": 0.4788376986980438, "grad_norm": 17.691587448120117, "learning_rate": 4.7625351138769166e-07, "logits/chosen": 1.5900017023086548, "logits/rejected": 1.5221468210220337, "logps/chosen": -309.0920104980469, "logps/ref_chosen": -317.2015686035156, "logps/ref_rejected": -278.0653381347656, "logps/rejected": -278.478271484375, "loss": 5.2299, "margin_dpo/margin_mean": 8.522488594055176, "margin_dpo/margin_std": 12.911687850952148, "step": 109 }, { "KL/chosen_KL_mean": 8.590599060058594, "KL/mean": 4.59503173828125, "KL/rejected_KL_mean": 0.5994701385498047, "KL/std": 10.171649932861328, "epoch": 0.23036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.9911394119262695, "fcm_dpo/q_t": 0.48011514544487, "grad_norm": 15.929461479187012, "learning_rate": 4.75468677825789e-07, "logits/chosen": 1.877347469329834, "logits/rejected": 1.881067156791687, "logps/chosen": -254.34884643554688, "logps/ref_chosen": -262.939453125, "logps/ref_rejected": -257.82275390625, "logps/rejected": -257.2232971191406, "loss": 5.2504, "margin_dpo/margin_mean": 7.9911394119262695, "margin_dpo/margin_std": 13.278604507446289, "step": 110 }, { "KL/chosen_KL_mean": 9.09178352355957, "KL/mean": 4.779868125915527, "KL/rejected_KL_mean": 0.4679603576660156, "KL/std": 11.26836109161377, "epoch": 0.2324607329842932, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.623827934265137, "fcm_dpo/q_t": 0.4785763621330261, "grad_norm": 18.236873626708984, "learning_rate": 4.7467175306295647e-07, "logits/chosen": 1.8864248991012573, "logits/rejected": 1.7907862663269043, "logps/chosen": -310.06201171875, "logps/ref_chosen": -319.15380859375, "logps/ref_rejected": -281.7733154296875, "logps/rejected": -281.30535888671875, "loss": 5.2262, "margin_dpo/margin_mean": 8.62382698059082, "margin_dpo/margin_std": 12.825776100158691, "step": 111 }, { "KL/chosen_KL_mean": 5.521121978759766, "KL/mean": 3.3389346599578857, "KL/rejected_KL_mean": 1.156747817993164, "KL/std": 10.577245712280273, "epoch": 0.23455497382198953, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.364378452301025, "fcm_dpo/q_t": 0.48913276195526123, "grad_norm": 14.586549758911133, "learning_rate": 4.7386277983585053e-07, "logits/chosen": 1.85859215259552, "logits/rejected": 1.967081904411316, "logps/chosen": -244.97930908203125, "logps/ref_chosen": -250.50042724609375, "logps/ref_rejected": -268.32635498046875, "logps/rejected": -267.16961669921875, "loss": 5.3963, "margin_dpo/margin_mean": 4.364377975463867, "margin_dpo/margin_std": 14.259346008300781, "step": 112 }, { "KL/chosen_KL_mean": 6.750080108642578, "KL/mean": 1.6503314971923828, "KL/rejected_KL_mean": -3.449411392211914, "KL/std": 11.677305221557617, "epoch": 0.23664921465968586, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.199491500854492, "fcm_dpo/q_t": 0.47468462586402893, "grad_norm": 16.903900146484375, "learning_rate": 4.7304180152725024e-07, "logits/chosen": 1.6014283895492554, "logits/rejected": 1.636258602142334, "logps/chosen": -272.6515197753906, "logps/ref_chosen": -279.4015808105469, "logps/ref_rejected": -266.6392822265625, "logps/rejected": -270.0887145996094, "loss": 5.17, "margin_dpo/margin_mean": 10.199490547180176, "margin_dpo/margin_std": 14.30715560913086, "step": 113 }, { "KL/chosen_KL_mean": 5.200408935546875, "KL/mean": 2.613903522491455, "KL/rejected_KL_mean": 0.02741241455078125, "KL/std": 10.772842407226562, "epoch": 0.2387434554973822, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.172990322113037, "fcm_dpo/q_t": 0.4871406853199005, "grad_norm": 15.496525764465332, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 1.6856958866119385, "logits/rejected": 1.6697858572006226, "logps/chosen": -249.06007385253906, "logps/ref_chosen": -254.26048278808594, "logps/ref_rejected": -241.8769989013672, "logps/rejected": -241.84957885742188, "loss": 5.3599, "margin_dpo/margin_mean": 5.172990322113037, "margin_dpo/margin_std": 13.062189102172852, "step": 114 }, { "KL/chosen_KL_mean": 5.646036148071289, "KL/mean": 2.675328493118286, "KL/rejected_KL_mean": -0.2953834533691406, "KL/std": 11.957541465759277, "epoch": 0.24083769633507854, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.941422939300537, "fcm_dpo/q_t": 0.48527175188064575, "grad_norm": 15.836464881896973, "learning_rate": 4.7136400641330245e-07, "logits/chosen": 1.7326550483703613, "logits/rejected": 1.7668747901916504, "logps/chosen": -283.1584777832031, "logps/ref_chosen": -288.80450439453125, "logps/ref_rejected": -266.8573913574219, "logps/rejected": -267.15277099609375, "loss": 5.3344, "margin_dpo/margin_mean": 5.941421985626221, "margin_dpo/margin_std": 14.47954273223877, "step": 115 }, { "KL/chosen_KL_mean": 6.954935073852539, "KL/mean": 2.911851644515991, "KL/rejected_KL_mean": -1.1312294006347656, "KL/std": 11.175189971923828, "epoch": 0.24293193717277486, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.086163520812988, "fcm_dpo/q_t": 0.47990795969963074, "grad_norm": 16.428619384765625, "learning_rate": 4.70507279583015e-07, "logits/chosen": 1.6866003274917603, "logits/rejected": 1.6639567613601685, "logps/chosen": -265.23504638671875, "logps/ref_chosen": -272.18994140625, "logps/ref_rejected": -253.14334106445312, "logps/rejected": -254.27456665039062, "loss": 5.2496, "margin_dpo/margin_mean": 8.086163520812988, "margin_dpo/margin_std": 14.127702713012695, "step": 116 }, { "KL/chosen_KL_mean": 5.766324996948242, "KL/mean": 1.491805076599121, "KL/rejected_KL_mean": -2.782712936401367, "KL/std": 11.109378814697266, "epoch": 0.2450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.549039840698242, "fcm_dpo/q_t": 0.4787558615207672, "grad_norm": 15.702656745910645, "learning_rate": 4.6963872761652834e-07, "logits/chosen": 1.5461552143096924, "logits/rejected": 1.5219987630844116, "logps/chosen": -244.6363067626953, "logps/ref_chosen": -250.4026336669922, "logps/ref_rejected": -229.63853454589844, "logps/rejected": -232.4212646484375, "loss": 5.2298, "margin_dpo/margin_mean": 8.549040794372559, "margin_dpo/margin_std": 12.602401733398438, "step": 117 }, { "KL/chosen_KL_mean": 5.609523773193359, "KL/mean": 1.0851911306381226, "KL/rejected_KL_mean": -3.439149856567383, "KL/std": 12.804605484008789, "epoch": 0.24712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.048680305480957, "fcm_dpo/q_t": 0.4775315523147583, "grad_norm": 20.030120849609375, "learning_rate": 4.687583970916486e-07, "logits/chosen": 1.7872297763824463, "logits/rejected": 1.8385579586029053, "logps/chosen": -252.786376953125, "logps/ref_chosen": -258.3959045410156, "logps/ref_rejected": -268.84228515625, "logps/rejected": -272.2814636230469, "loss": 5.2143, "margin_dpo/margin_mean": 9.048680305480957, "margin_dpo/margin_std": 14.543923377990723, "step": 118 }, { "KL/chosen_KL_mean": 6.068717956542969, "KL/mean": 2.253279685974121, "KL/rejected_KL_mean": -1.5621623992919922, "KL/std": 14.264659881591797, "epoch": 0.24921465968586387, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.630876541137695, "fcm_dpo/q_t": 0.48115378618240356, "grad_norm": 15.853008270263672, "learning_rate": 4.6786633521783005e-07, "logits/chosen": 1.8772937059402466, "logits/rejected": 1.8533079624176025, "logps/chosen": -311.23382568359375, "logps/ref_chosen": -317.30255126953125, "logps/ref_rejected": -300.47552490234375, "logps/rejected": -302.0376892089844, "loss": 5.2793, "margin_dpo/margin_mean": 7.630877494812012, "margin_dpo/margin_std": 16.963003158569336, "step": 119 }, { "KL/chosen_KL_mean": 6.961023330688477, "KL/mean": 2.4848945140838623, "KL/rejected_KL_mean": -1.9912357330322266, "KL/std": 13.374311447143555, "epoch": 0.2513089005235602, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.952258110046387, "fcm_dpo/q_t": 0.4778059124946594, "grad_norm": 16.386856079101562, "learning_rate": 4.669625898336438e-07, "logits/chosen": 2.0444533824920654, "logits/rejected": 2.03185772895813, "logps/chosen": -298.29339599609375, "logps/ref_chosen": -305.2544250488281, "logps/ref_rejected": -259.40716552734375, "logps/rejected": -261.39837646484375, "loss": 5.2241, "margin_dpo/margin_mean": 8.952259063720703, "margin_dpo/margin_std": 16.330432891845703, "step": 120 }, { "KL/chosen_KL_mean": 3.5496482849121094, "KL/mean": 0.8425604104995728, "KL/rejected_KL_mean": -1.8645248413085938, "KL/std": 11.928267478942871, "epoch": 0.2534031413612565, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.414175987243652, "fcm_dpo/q_t": 0.48656564950942993, "grad_norm": 15.524222373962402, "learning_rate": 4.6604720940421207e-07, "logits/chosen": 1.5826594829559326, "logits/rejected": 1.6027311086654663, "logps/chosen": -213.091064453125, "logps/ref_chosen": -216.64073181152344, "logps/ref_rejected": -224.36422729492188, "logps/rejected": -226.22874450683594, "loss": 5.3549, "margin_dpo/margin_mean": 5.414175987243652, "margin_dpo/margin_std": 14.486465454101562, "step": 121 }, { "KL/chosen_KL_mean": 5.696184158325195, "KL/mean": 1.9863008260726929, "KL/rejected_KL_mean": -1.7235946655273438, "KL/std": 14.075178146362305, "epoch": 0.2554973821989529, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.419771671295166, "fcm_dpo/q_t": 0.4816259443759918, "grad_norm": 15.780983924865723, "learning_rate": 4.651202430186092e-07, "logits/chosen": 1.6831539869308472, "logits/rejected": 1.6807143688201904, "logps/chosen": -263.12261962890625, "logps/ref_chosen": -268.8188171386719, "logps/ref_rejected": -272.55853271484375, "logps/rejected": -274.2821044921875, "loss": 5.2877, "margin_dpo/margin_mean": 7.419772624969482, "margin_dpo/margin_std": 17.622047424316406, "step": 122 }, { "KL/chosen_KL_mean": 8.446487426757812, "KL/mean": 3.178858757019043, "KL/rejected_KL_mean": -2.088764190673828, "KL/std": 15.126729011535645, "epoch": 0.25759162303664923, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.535255432128906, "fcm_dpo/q_t": 0.4739333391189575, "grad_norm": 18.45299530029297, "learning_rate": 4.6418174038722924e-07, "logits/chosen": 1.8908565044403076, "logits/rejected": 1.8965411186218262, "logps/chosen": -313.2452697753906, "logps/ref_chosen": -321.6917419433594, "logps/ref_rejected": -279.1517333984375, "logps/rejected": -281.240478515625, "loss": 5.1704, "margin_dpo/margin_mean": 10.535255432128906, "margin_dpo/margin_std": 18.469093322753906, "step": 123 }, { "KL/chosen_KL_mean": 6.057577133178711, "KL/mean": 1.2710437774658203, "KL/rejected_KL_mean": -3.515501022338867, "KL/std": 14.487335205078125, "epoch": 0.25968586387434556, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.573090553283691, "fcm_dpo/q_t": 0.47625818848609924, "grad_norm": 17.11802864074707, "learning_rate": 4.6323175183912023e-07, "logits/chosen": 1.8385354280471802, "logits/rejected": 1.7378027439117432, "logps/chosen": -265.61761474609375, "logps/ref_chosen": -271.6752014160156, "logps/ref_rejected": -221.85577392578125, "logps/rejected": -225.3712921142578, "loss": 5.2017, "margin_dpo/margin_mean": 9.573091506958008, "margin_dpo/margin_std": 16.087444305419922, "step": 124 }, { "KL/chosen_KL_mean": 5.609090805053711, "KL/mean": 0.8871803283691406, "KL/rejected_KL_mean": -3.8347320556640625, "KL/std": 14.195379257202148, "epoch": 0.2617801047120419, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.443817138671875, "fcm_dpo/q_t": 0.4766680598258972, "grad_norm": 16.373830795288086, "learning_rate": 4.6227032831928483e-07, "logits/chosen": 1.627115249633789, "logits/rejected": 1.6331775188446045, "logps/chosen": -264.7899475097656, "logps/ref_chosen": -270.3990478515625, "logps/ref_rejected": -264.16925048828125, "logps/rejected": -268.00396728515625, "loss": 5.2127, "margin_dpo/margin_mean": 9.443815231323242, "margin_dpo/margin_std": 18.61271095275879, "step": 125 }, { "KL/chosen_KL_mean": 6.839256286621094, "KL/mean": 1.4984056949615479, "KL/rejected_KL_mean": -3.8424301147460938, "KL/std": 15.581066131591797, "epoch": 0.2638743455497382, "fcm_dpo/beta": 0.010156591422855854, "fcm_dpo/delta": 0.022172488272190094, "fcm_dpo/margin": 10.681697845458984, "fcm_dpo/q_t": 0.47326067090034485, "grad_norm": 16.927207946777344, "learning_rate": 4.612975213859487e-07, "logits/chosen": 1.724934458732605, "logits/rejected": 1.699325442314148, "logps/chosen": -306.41204833984375, "logps/ref_chosen": -313.25128173828125, "logps/ref_rejected": -283.0433349609375, "logps/rejected": -286.8857421875, "loss": 5.1609, "margin_dpo/margin_mean": 10.681696891784668, "margin_dpo/margin_std": 18.312498092651367, "step": 126 }, { "KL/chosen_KL_mean": 8.03831672668457, "KL/mean": 1.868345022201538, "KL/rejected_KL_mean": -4.301609039306641, "KL/std": 16.05044937133789, "epoch": 0.26596858638743454, "fcm_dpo/beta": 0.010299422778189182, "fcm_dpo/delta": 0.04627181217074394, "fcm_dpo/margin": 12.339926719665527, "fcm_dpo/q_t": 0.4686408042907715, "grad_norm": 17.30465316772461, "learning_rate": 4.603133832077953e-07, "logits/chosen": 1.6262192726135254, "logits/rejected": 1.548923373222351, "logps/chosen": -290.5256652832031, "logps/ref_chosen": -298.5639953613281, "logps/ref_rejected": -274.021728515625, "logps/rejected": -278.3233337402344, "loss": 5.0892, "margin_dpo/margin_mean": 12.339926719665527, "margin_dpo/margin_std": 18.16909408569336, "step": 127 }, { "KL/chosen_KL_mean": 8.606454849243164, "KL/mean": 1.7851275205612183, "KL/rejected_KL_mean": -5.036199569702148, "KL/std": 17.000141143798828, "epoch": 0.2680628272251309, "fcm_dpo/beta": 0.010715160518884659, "fcm_dpo/delta": 0.023808792233467102, "fcm_dpo/margin": 13.642653465270996, "fcm_dpo/q_t": 0.46404626965522766, "grad_norm": 17.175697326660156, "learning_rate": 4.5931796656116837e-07, "logits/chosen": 1.6542246341705322, "logits/rejected": 1.690248966217041, "logps/chosen": -248.5244140625, "logps/ref_chosen": -257.130859375, "logps/ref_rejected": -247.71636962890625, "logps/rejected": -252.75254821777344, "loss": 5.0312, "margin_dpo/margin_mean": 13.642655372619629, "margin_dpo/margin_std": 20.29867935180664, "step": 128 }, { "KL/chosen_KL_mean": 8.863290786743164, "KL/mean": 2.632221221923828, "KL/rejected_KL_mean": -3.5988426208496094, "KL/std": 17.10430908203125, "epoch": 0.27015706806282724, "fcm_dpo/beta": 0.010969205759465694, "fcm_dpo/delta": 0.04741775244474411, "fcm_dpo/margin": 12.46213150024414, "fcm_dpo/q_t": 0.4665260910987854, "grad_norm": 17.762378692626953, "learning_rate": 4.5831132482724193e-07, "logits/chosen": 1.6477328538894653, "logits/rejected": 1.6887083053588867, "logps/chosen": -286.0972595214844, "logps/ref_chosen": -294.9605712890625, "logps/ref_rejected": -272.6249694824219, "logps/rejected": -276.22381591796875, "loss": 5.0714, "margin_dpo/margin_mean": 12.46213150024414, "margin_dpo/margin_std": 20.405351638793945, "step": 129 }, { "KL/chosen_KL_mean": 7.510087966918945, "KL/mean": 1.6374802589416504, "KL/rejected_KL_mean": -4.235134124755859, "KL/std": 15.58251667022705, "epoch": 0.27225130890052357, "fcm_dpo/beta": 0.011182182468473911, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.745227813720703, "fcm_dpo/q_t": 0.4675550162792206, "grad_norm": 22.702251434326172, "learning_rate": 4.5729351198915705e-07, "logits/chosen": 1.811161756515503, "logits/rejected": 1.8970874547958374, "logps/chosen": -268.93212890625, "logps/ref_chosen": -276.4422302246094, "logps/ref_rejected": -290.241455078125, "logps/rejected": -294.4765930175781, "loss": 5.085, "margin_dpo/margin_mean": 11.745227813720703, "margin_dpo/margin_std": 19.256053924560547, "step": 130 }, { "KL/chosen_KL_mean": 5.034919738769531, "KL/mean": 0.19638243317604065, "KL/rejected_KL_mean": -4.64215087890625, "KL/std": 16.58043098449707, "epoch": 0.2743455497382199, "fcm_dpo/beta": 0.011311189271509647, "fcm_dpo/delta": 0.02286328561604023, "fcm_dpo/margin": 9.677083015441895, "fcm_dpo/q_t": 0.4732462167739868, "grad_norm": 19.667495727539062, "learning_rate": 4.5626458262912735e-07, "logits/chosen": 1.5953093767166138, "logits/rejected": 1.5892738103866577, "logps/chosen": -299.789794921875, "logps/ref_chosen": -304.82470703125, "logps/ref_rejected": -289.9726867675781, "logps/rejected": -294.61480712890625, "loss": 5.1813, "margin_dpo/margin_mean": 9.677083015441895, "margin_dpo/margin_std": 21.382164001464844, "step": 131 }, { "KL/chosen_KL_mean": 7.070009231567383, "KL/mean": 0.573898434638977, "KL/rejected_KL_mean": -5.922212600708008, "KL/std": 18.59527587890625, "epoch": 0.2764397905759162, "fcm_dpo/beta": 0.011468597687780857, "fcm_dpo/delta": 0.032270729541778564, "fcm_dpo/margin": 12.99222183227539, "fcm_dpo/q_t": 0.4638496935367584, "grad_norm": 20.325056076049805, "learning_rate": 4.5522459192551166e-07, "logits/chosen": 1.4823801517486572, "logits/rejected": 1.5927780866622925, "logps/chosen": -253.24472045898438, "logps/ref_chosen": -260.31475830078125, "logps/ref_rejected": -280.93756103515625, "logps/rejected": -286.8597717285156, "loss": 5.0492, "margin_dpo/margin_mean": 12.992222785949707, "margin_dpo/margin_std": 23.399463653564453, "step": 132 }, { "KL/chosen_KL_mean": 7.743677139282227, "KL/mean": 1.0414981842041016, "KL/rejected_KL_mean": -5.660671234130859, "KL/std": 17.203453063964844, "epoch": 0.27853403141361255, "fcm_dpo/beta": 0.011720841750502586, "fcm_dpo/delta": 0.014634376391768456, "fcm_dpo/margin": 13.40435791015625, "fcm_dpo/q_t": 0.4613617956638336, "grad_norm": 18.77317237854004, "learning_rate": 4.541735956498554e-07, "logits/chosen": 1.5894581079483032, "logits/rejected": 1.5555914640426636, "logps/chosen": -271.37689208984375, "logps/ref_chosen": -279.12060546875, "logps/ref_rejected": -247.22982788085938, "logps/rejected": -252.8905029296875, "loss": 5.0094, "margin_dpo/margin_mean": 13.40435791015625, "margin_dpo/margin_std": 21.21170997619629, "step": 133 }, { "KL/chosen_KL_mean": 1.6970939636230469, "KL/mean": -3.0371992588043213, "KL/rejected_KL_mean": -7.77149772644043, "KL/std": 17.570058822631836, "epoch": 0.2806282722513089, "fcm_dpo/beta": 0.011929137632250786, "fcm_dpo/delta": 0.022021804004907608, "fcm_dpo/margin": 9.46860408782959, "fcm_dpo/q_t": 0.4723929166793823, "grad_norm": 24.406070709228516, "learning_rate": 4.5311165016389914e-07, "logits/chosen": 1.8009541034698486, "logits/rejected": 1.8369526863098145, "logps/chosen": -313.9070129394531, "logps/ref_chosen": -315.6040954589844, "logps/ref_rejected": -291.45635986328125, "logps/rejected": -299.22784423828125, "loss": 5.169, "margin_dpo/margin_mean": 9.46860408782959, "margin_dpo/margin_std": 20.294403076171875, "step": 134 }, { "KL/chosen_KL_mean": 5.026142120361328, "KL/mean": -1.5437160730361938, "KL/rejected_KL_mean": -8.113567352294922, "KL/std": 18.06447410583496, "epoch": 0.28272251308900526, "fcm_dpo/beta": 0.012354599311947823, "fcm_dpo/delta": 0.05824194476008415, "fcm_dpo/margin": 13.139720916748047, "fcm_dpo/q_t": 0.4605092704296112, "grad_norm": 20.61244773864746, "learning_rate": 4.520388124165564e-07, "logits/chosen": 1.43618643283844, "logits/rejected": 1.419420838356018, "logps/chosen": -277.8468322753906, "logps/ref_chosen": -282.87298583984375, "logps/ref_rejected": -265.0682373046875, "logps/rejected": -273.1817932128906, "loss": 4.9883, "margin_dpo/margin_mean": 13.13971996307373, "margin_dpo/margin_std": 19.358867645263672, "step": 135 }, { "KL/chosen_KL_mean": 4.645238876342773, "KL/mean": -1.692807674407959, "KL/rejected_KL_mean": -8.030860900878906, "KL/std": 17.679540634155273, "epoch": 0.2848167539267016, "fcm_dpo/beta": 0.012806904502213001, "fcm_dpo/delta": 0.03608924150466919, "fcm_dpo/margin": 12.676102638244629, "fcm_dpo/q_t": 0.4605832099914551, "grad_norm": 23.0947322845459, "learning_rate": 4.5095513994085974e-07, "logits/chosen": 1.490594744682312, "logits/rejected": 1.6552977561950684, "logps/chosen": -258.7966613769531, "logps/ref_chosen": -263.4419250488281, "logps/ref_rejected": -251.43150329589844, "logps/rejected": -259.46234130859375, "loss": 5.0083, "margin_dpo/margin_mean": 12.676101684570312, "margin_dpo/margin_std": 21.883665084838867, "step": 136 }, { "KL/chosen_KL_mean": 2.6373767852783203, "KL/mean": -3.0409834384918213, "KL/rejected_KL_mean": -8.719337463378906, "KL/std": 19.185440063476562, "epoch": 0.2869109947643979, "fcm_dpo/beta": 0.01320512406527996, "fcm_dpo/delta": 0.06166880577802658, "fcm_dpo/margin": 11.356720924377441, "fcm_dpo/q_t": 0.46354466676712036, "grad_norm": 23.587217330932617, "learning_rate": 4.498606908508753e-07, "logits/chosen": 1.5931267738342285, "logits/rejected": 1.5913976430892944, "logps/chosen": -292.63336181640625, "logps/ref_chosen": -295.2707214355469, "logps/ref_rejected": -283.9851989746094, "logps/rejected": -292.70458984375, "loss": 5.0482, "margin_dpo/margin_mean": 11.356720924377441, "margin_dpo/margin_std": 20.716747283935547, "step": 137 }, { "KL/chosen_KL_mean": 1.1577720642089844, "KL/mean": -4.458102703094482, "KL/rejected_KL_mean": -10.073986053466797, "KL/std": 18.46007537841797, "epoch": 0.28900523560209423, "fcm_dpo/beta": 0.01372732874006033, "fcm_dpo/delta": 0.00854338239878416, "fcm_dpo/margin": 11.231751441955566, "fcm_dpo/q_t": 0.4628608822822571, "grad_norm": 22.117218017578125, "learning_rate": 4.487555238385862e-07, "logits/chosen": 1.4696087837219238, "logits/rejected": 1.460563063621521, "logps/chosen": -281.4001159667969, "logps/ref_chosen": -282.5578918457031, "logps/ref_rejected": -262.4772644042969, "logps/rejected": -272.55126953125, "loss": 5.0642, "margin_dpo/margin_mean": 11.231752395629883, "margin_dpo/margin_std": 23.57673454284668, "step": 138 }, { "KL/chosen_KL_mean": 2.1462669372558594, "KL/mean": -1.9316972494125366, "KL/rejected_KL_mean": -6.009664535522461, "KL/std": 19.203439712524414, "epoch": 0.29109947643979056, "fcm_dpo/beta": 0.01388666033744812, "fcm_dpo/delta": 0.0230011697858572, "fcm_dpo/margin": 8.15593147277832, "fcm_dpo/q_t": 0.4726797342300415, "grad_norm": 22.397058486938477, "learning_rate": 4.476396981707453e-07, "logits/chosen": 1.6937816143035889, "logits/rejected": 1.7701420783996582, "logps/chosen": -261.5045471191406, "logps/ref_chosen": -263.65081787109375, "logps/ref_rejected": -266.2420959472656, "logps/rejected": -272.25177001953125, "loss": 5.217, "margin_dpo/margin_mean": 8.155933380126953, "margin_dpo/margin_std": 23.77894401550293, "step": 139 }, { "KL/chosen_KL_mean": 5.735929489135742, "KL/mean": -1.7937304973602295, "KL/rejected_KL_mean": -9.323375701904297, "KL/std": 19.06015396118164, "epoch": 0.2931937172774869, "fcm_dpo/beta": 0.014193753711879253, "fcm_dpo/delta": 0.055229805409908295, "fcm_dpo/margin": 15.059309005737305, "fcm_dpo/q_t": 0.4484260678291321, "grad_norm": 21.740379333496094, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 1.7132786512374878, "logits/rejected": 1.6778305768966675, "logps/chosen": -266.6465148925781, "logps/ref_chosen": -272.3824462890625, "logps/ref_rejected": -248.96444702148438, "logps/rejected": -258.287841796875, "loss": 4.8335, "margin_dpo/margin_mean": 15.059309005737305, "margin_dpo/margin_std": 21.372209548950195, "step": 140 }, { "KL/chosen_KL_mean": 3.1934146881103516, "KL/mean": -3.918694257736206, "KL/rejected_KL_mean": -11.030794143676758, "KL/std": 20.240835189819336, "epoch": 0.29528795811518327, "fcm_dpo/beta": 0.014990235678851604, "fcm_dpo/delta": 0.04028759524226189, "fcm_dpo/margin": 14.224204063415527, "fcm_dpo/q_t": 0.4487554430961609, "grad_norm": 33.108272552490234, "learning_rate": 4.453763107901675e-07, "logits/chosen": 1.7549117803573608, "logits/rejected": 1.8129609823226929, "logps/chosen": -284.87762451171875, "logps/ref_chosen": -288.071044921875, "logps/ref_rejected": -274.83477783203125, "logps/rejected": -285.86553955078125, "loss": 4.8777, "margin_dpo/margin_mean": 14.224204063415527, "margin_dpo/margin_std": 24.404754638671875, "step": 141 }, { "KL/chosen_KL_mean": 5.986888885498047, "KL/mean": -0.9714588522911072, "KL/rejected_KL_mean": -7.929801940917969, "KL/std": 19.997905731201172, "epoch": 0.2973821989528796, "fcm_dpo/beta": 0.015509484335780144, "fcm_dpo/delta": 0.04578714445233345, "fcm_dpo/margin": 13.916696548461914, "fcm_dpo/q_t": 0.44861719012260437, "grad_norm": 25.78396224975586, "learning_rate": 4.4422887045602674e-07, "logits/chosen": 1.5206505060195923, "logits/rejected": 1.406781792640686, "logps/chosen": -283.00494384765625, "logps/ref_chosen": -288.9918518066406, "logps/ref_rejected": -228.26071166992188, "logps/rejected": -236.1905059814453, "loss": 4.8791, "margin_dpo/margin_mean": 13.916695594787598, "margin_dpo/margin_std": 24.268972396850586, "step": 142 }, { "KL/chosen_KL_mean": 5.79694938659668, "KL/mean": -2.3009872436523438, "KL/rejected_KL_mean": -10.398920059204102, "KL/std": 21.152034759521484, "epoch": 0.2994764397905759, "fcm_dpo/beta": 0.016140975058078766, "fcm_dpo/delta": 0.07417097687721252, "fcm_dpo/margin": 16.19586181640625, "fcm_dpo/q_t": 0.438321053981781, "grad_norm": 29.62942123413086, "learning_rate": 4.4307101421701755e-07, "logits/chosen": 1.4169150590896606, "logits/rejected": 1.4356873035430908, "logps/chosen": -283.6775207519531, "logps/ref_chosen": -289.4744873046875, "logps/ref_rejected": -265.130615234375, "logps/rejected": -275.5295715332031, "loss": 4.7569, "margin_dpo/margin_mean": 16.195859909057617, "margin_dpo/margin_std": 26.71329689025879, "step": 143 }, { "KL/chosen_KL_mean": 7.127115249633789, "KL/mean": 0.3925158381462097, "KL/rejected_KL_mean": -6.342063903808594, "KL/std": 20.461181640625, "epoch": 0.30157068062827225, "fcm_dpo/beta": 0.01703416183590889, "fcm_dpo/delta": 0.06011633202433586, "fcm_dpo/margin": 13.469175338745117, "fcm_dpo/q_t": 0.4462902843952179, "grad_norm": 28.699085235595703, "learning_rate": 4.419028041654559e-07, "logits/chosen": 1.70430588722229, "logits/rejected": 1.688096046447754, "logps/chosen": -283.4592590332031, "logps/ref_chosen": -290.5863342285156, "logps/ref_rejected": -263.41046142578125, "logps/rejected": -269.7525329589844, "loss": 4.8609, "margin_dpo/margin_mean": 13.4691743850708, "margin_dpo/margin_std": 24.161060333251953, "step": 144 }, { "KL/chosen_KL_mean": 7.376377105712891, "KL/mean": -1.2903668880462646, "KL/rejected_KL_mean": -9.957107543945312, "KL/std": 19.41018295288086, "epoch": 0.3036649214659686, "fcm_dpo/beta": 0.01746426150202751, "fcm_dpo/delta": -0.00025831814855337143, "fcm_dpo/margin": 17.333486557006836, "fcm_dpo/q_t": 0.4287872910499573, "grad_norm": 27.49614715576172, "learning_rate": 4.4072430294890166e-07, "logits/chosen": 1.5508968830108643, "logits/rejected": 1.4520549774169922, "logps/chosen": -257.16387939453125, "logps/ref_chosen": -264.5402526855469, "logps/ref_rejected": -214.7958221435547, "logps/rejected": -224.7529296875, "loss": 4.6247, "margin_dpo/margin_mean": 17.333484649658203, "margin_dpo/margin_std": 24.21767234802246, "step": 145 }, { "KL/chosen_KL_mean": 3.2908153533935547, "KL/mean": -3.671146869659424, "KL/rejected_KL_mean": -10.633108139038086, "KL/std": 19.569717407226562, "epoch": 0.3057591623036649, "fcm_dpo/beta": 0.017695514485239983, "fcm_dpo/delta": 0.050446767359972, "fcm_dpo/margin": 13.923921585083008, "fcm_dpo/q_t": 0.44180774688720703, "grad_norm": 46.415531158447266, "learning_rate": 4.395355737667985e-07, "logits/chosen": 1.5119810104370117, "logits/rejected": 1.626979947090149, "logps/chosen": -266.8889465332031, "logps/ref_chosen": -270.17974853515625, "logps/ref_rejected": -261.2323913574219, "logps/rejected": -271.865478515625, "loss": 4.8104, "margin_dpo/margin_mean": 13.92392349243164, "margin_dpo/margin_std": 23.901309967041016, "step": 146 }, { "KL/chosen_KL_mean": 1.9331626892089844, "KL/mean": -3.9541797637939453, "KL/rejected_KL_mean": -9.841531753540039, "KL/std": 21.822551727294922, "epoch": 0.3078534031413613, "fcm_dpo/beta": 0.018427424132823944, "fcm_dpo/delta": 0.02790326438844204, "fcm_dpo/margin": 11.774698257446289, "fcm_dpo/q_t": 0.4492885172367096, "grad_norm": 29.35231590270996, "learning_rate": 4.3833668036708483e-07, "logits/chosen": 1.5314257144927979, "logits/rejected": 1.5078436136245728, "logps/chosen": -280.8474426269531, "logps/ref_chosen": -282.7806091308594, "logps/ref_rejected": -249.23399353027344, "logps/rejected": -259.0755310058594, "loss": 4.9819, "margin_dpo/margin_mean": 11.774696350097656, "margin_dpo/margin_std": 26.265121459960938, "step": 147 }, { "KL/chosen_KL_mean": 2.4545116424560547, "KL/mean": -3.7157320976257324, "KL/rejected_KL_mean": -9.885980606079102, "KL/std": 20.505939483642578, "epoch": 0.3099476439790576, "fcm_dpo/beta": 0.01864941045641899, "fcm_dpo/delta": 0.018809720873832703, "fcm_dpo/margin": 12.340494155883789, "fcm_dpo/q_t": 0.4464648365974426, "grad_norm": 30.132938385009766, "learning_rate": 4.3712768704277524e-07, "logits/chosen": 1.5904818773269653, "logits/rejected": 1.6076817512512207, "logps/chosen": -226.46279907226562, "logps/ref_chosen": -228.9173126220703, "logps/ref_rejected": -226.81222534179688, "logps/rejected": -236.69822692871094, "loss": 4.914, "margin_dpo/margin_mean": 12.340494155883789, "margin_dpo/margin_std": 25.5549373626709, "step": 148 }, { "KL/chosen_KL_mean": 4.127834320068359, "KL/mean": -3.567676544189453, "KL/rejected_KL_mean": -11.263181686401367, "KL/std": 21.19050407409668, "epoch": 0.31204188481675393, "fcm_dpo/beta": 0.019410088658332825, "fcm_dpo/delta": 0.05117492750287056, "fcm_dpo/margin": 15.391016006469727, "fcm_dpo/q_t": 0.4308072626590729, "grad_norm": 32.52208709716797, "learning_rate": 4.3590865862851263e-07, "logits/chosen": 1.9011356830596924, "logits/rejected": 1.8911197185516357, "logps/chosen": -306.95806884765625, "logps/ref_chosen": -311.0859375, "logps/ref_rejected": -303.225341796875, "logps/rejected": -314.4885559082031, "loss": 4.6801, "margin_dpo/margin_mean": 15.39101791381836, "margin_dpo/margin_std": 25.0798282623291, "step": 149 }, { "KL/chosen_KL_mean": 1.6150665283203125, "KL/mean": -5.609472751617432, "KL/rejected_KL_mean": -12.834024429321289, "KL/std": 20.40045166015625, "epoch": 0.31413612565445026, "fcm_dpo/beta": 0.020297512412071228, "fcm_dpo/delta": 0.07883753627538681, "fcm_dpo/margin": 14.449088096618652, "fcm_dpo/q_t": 0.4320872724056244, "grad_norm": 32.55631637573242, "learning_rate": 4.346796604970912e-07, "logits/chosen": 1.540358066558838, "logits/rejected": 1.5534651279449463, "logps/chosen": -268.86834716796875, "logps/ref_chosen": -270.48345947265625, "logps/ref_rejected": -269.0491638183594, "logps/rejected": -281.8831481933594, "loss": 4.7022, "margin_dpo/margin_mean": 14.449089050292969, "margin_dpo/margin_std": 24.21239471435547, "step": 150 }, { "KL/chosen_KL_mean": 5.48326301574707, "KL/mean": -5.53632926940918, "KL/rejected_KL_mean": -16.55592155456543, "KL/std": 22.382789611816406, "epoch": 0.3162303664921466, "fcm_dpo/beta": 0.020343992859125137, "fcm_dpo/delta": -0.05105812847614288, "fcm_dpo/margin": 22.03917121887207, "fcm_dpo/q_t": 0.3968949019908905, "grad_norm": 35.151790618896484, "learning_rate": 4.3344075855595097e-07, "logits/chosen": 1.4414728879928589, "logits/rejected": 1.447618007659912, "logps/chosen": -282.68243408203125, "logps/ref_chosen": -288.16571044921875, "logps/ref_rejected": -264.7412414550781, "logps/rejected": -281.29718017578125, "loss": 4.2681, "margin_dpo/margin_mean": 22.039173126220703, "margin_dpo/margin_std": 27.89799690246582, "step": 151 }, { "KL/chosen_KL_mean": 3.7835044860839844, "KL/mean": -4.452916145324707, "KL/rejected_KL_mean": -12.689334869384766, "KL/std": 22.493972778320312, "epoch": 0.3183246073298429, "fcm_dpo/beta": 0.020089194178581238, "fcm_dpo/delta": 0.01719110645353794, "fcm_dpo/margin": 16.472835540771484, "fcm_dpo/q_t": 0.4237143099308014, "grad_norm": 39.374393463134766, "learning_rate": 4.3219201924364323e-07, "logits/chosen": 1.5648020505905151, "logits/rejected": 1.6551036834716797, "logps/chosen": -256.1549987792969, "logps/ref_chosen": -259.9385070800781, "logps/ref_rejected": -266.2453308105469, "logps/rejected": -278.9346923828125, "loss": 4.6358, "margin_dpo/margin_mean": 16.47283363342285, "margin_dpo/margin_std": 26.881832122802734, "step": 152 }, { "KL/chosen_KL_mean": 2.586366653442383, "KL/mean": -9.49244213104248, "KL/rejected_KL_mean": -21.571243286132812, "KL/std": 23.96262550354004, "epoch": 0.3204188481675393, "fcm_dpo/beta": 0.019452663138508797, "fcm_dpo/delta": -0.07453102618455887, "fcm_dpo/margin": 24.157608032226562, "fcm_dpo/q_t": 0.3905426859855652, "grad_norm": 40.339393615722656, "learning_rate": 4.309335095262675e-07, "logits/chosen": 1.6833831071853638, "logits/rejected": 1.7517095804214478, "logps/chosen": -262.9906311035156, "logps/ref_chosen": -265.5769958496094, "logps/ref_rejected": -277.6246032714844, "logps/rejected": -299.19586181640625, "loss": 4.1366, "margin_dpo/margin_mean": 24.157608032226562, "margin_dpo/margin_std": 26.23731803894043, "step": 153 }, { "KL/chosen_KL_mean": 0.8299999237060547, "KL/mean": -7.864226818084717, "KL/rejected_KL_mean": -16.558452606201172, "KL/std": 24.469377517700195, "epoch": 0.3225130890052356, "fcm_dpo/beta": 0.019323289394378662, "fcm_dpo/delta": 0.002240743488073349, "fcm_dpo/margin": 17.38846206665039, "fcm_dpo/q_t": 0.42226171493530273, "grad_norm": 36.89560317993164, "learning_rate": 4.2966529689388064e-07, "logits/chosen": 1.360990047454834, "logits/rejected": 1.327060341835022, "logps/chosen": -281.44342041015625, "logps/ref_chosen": -282.2734069824219, "logps/ref_rejected": -261.5135498046875, "logps/rejected": -278.072021484375, "loss": 4.6639, "margin_dpo/margin_mean": 17.38846206665039, "margin_dpo/margin_std": 30.103727340698242, "step": 154 }, { "KL/chosen_KL_mean": -0.40489959716796875, "KL/mean": -6.861968994140625, "KL/rejected_KL_mean": -13.319042205810547, "KL/std": 22.191448211669922, "epoch": 0.32460732984293195, "fcm_dpo/beta": 0.019459176808595657, "fcm_dpo/delta": 0.032827116549015045, "fcm_dpo/margin": 12.914140701293945, "fcm_dpo/q_t": 0.44211363792419434, "grad_norm": 35.24588394165039, "learning_rate": 4.2838744935687716e-07, "logits/chosen": 1.6091328859329224, "logits/rejected": 1.7022383213043213, "logps/chosen": -279.116455078125, "logps/ref_chosen": -278.7115783691406, "logps/ref_rejected": -298.0945739746094, "logps/rejected": -311.41363525390625, "loss": 4.8997, "margin_dpo/margin_mean": 12.914140701293945, "margin_dpo/margin_std": 27.220930099487305, "step": 155 }, { "KL/chosen_KL_mean": 1.2807083129882812, "KL/mean": -9.533502578735352, "KL/rejected_KL_mean": -20.34770965576172, "KL/std": 24.442045211791992, "epoch": 0.3267015706806283, "fcm_dpo/beta": 0.01958465948700905, "fcm_dpo/delta": -0.024999193847179413, "fcm_dpo/margin": 21.628427505493164, "fcm_dpo/q_t": 0.4033975303173065, "grad_norm": 31.40673828125, "learning_rate": 4.271000354423425e-07, "logits/chosen": 1.6639182567596436, "logits/rejected": 1.694714903831482, "logps/chosen": -267.0704345703125, "logps/ref_chosen": -268.35113525390625, "logps/ref_rejected": -241.9660186767578, "logps/rejected": -262.313720703125, "loss": 4.4131, "margin_dpo/margin_mean": 21.628427505493164, "margin_dpo/margin_std": 32.28349304199219, "step": 156 }, { "KL/chosen_KL_mean": -0.10972976684570312, "KL/mean": -7.63818883895874, "KL/rejected_KL_mean": -15.166645050048828, "KL/std": 23.81902503967285, "epoch": 0.3287958115183246, "fcm_dpo/beta": 0.019391320645809174, "fcm_dpo/delta": 0.02052324265241623, "fcm_dpo/margin": 15.056900024414062, "fcm_dpo/q_t": 0.43288618326187134, "grad_norm": 46.412200927734375, "learning_rate": 4.258031241903777e-07, "logits/chosen": 1.4068256616592407, "logits/rejected": 1.4659873247146606, "logps/chosen": -265.51336669921875, "logps/ref_chosen": -265.40362548828125, "logps/ref_rejected": -257.82440185546875, "logps/rejected": -272.99102783203125, "loss": 4.7695, "margin_dpo/margin_mean": 15.056900024414062, "margin_dpo/margin_std": 27.764768600463867, "step": 157 }, { "KL/chosen_KL_mean": 1.0236034393310547, "KL/mean": -7.27507209777832, "KL/rejected_KL_mean": -15.573741912841797, "KL/std": 23.505292892456055, "epoch": 0.3308900523560209, "fcm_dpo/beta": 0.020102355629205704, "fcm_dpo/delta": 0.02125120908021927, "fcm_dpo/margin": 16.59735107421875, "fcm_dpo/q_t": 0.42306697368621826, "grad_norm": 35.24538803100586, "learning_rate": 4.2449678515039743e-07, "logits/chosen": 1.2856690883636475, "logits/rejected": 1.317069411277771, "logps/chosen": -280.1387023925781, "logps/ref_chosen": -281.1622619628906, "logps/ref_rejected": -268.2899475097656, "logps/rejected": -283.8636779785156, "loss": 4.6476, "margin_dpo/margin_mean": 16.597352981567383, "margin_dpo/margin_std": 27.941625595092773, "step": 158 }, { "KL/chosen_KL_mean": 0.037982940673828125, "KL/mean": -6.434523582458496, "KL/rejected_KL_mean": -12.907020568847656, "KL/std": 24.362865447998047, "epoch": 0.33298429319371725, "fcm_dpo/beta": 0.019898485392332077, "fcm_dpo/delta": -0.001014402136206627, "fcm_dpo/margin": 12.94499397277832, "fcm_dpo/q_t": 0.44250720739364624, "grad_norm": 39.46014404296875, "learning_rate": 4.2318108837739986e-07, "logits/chosen": 1.5753355026245117, "logits/rejected": 1.46786367893219, "logps/chosen": -311.28704833984375, "logps/ref_chosen": -311.32501220703125, "logps/ref_rejected": -248.85350036621094, "logps/rejected": -261.760498046875, "loss": 5.012, "margin_dpo/margin_mean": 12.94499397277832, "margin_dpo/margin_std": 32.133689880371094, "step": 159 }, { "KL/chosen_KL_mean": 3.813638687133789, "KL/mean": -6.125799655914307, "KL/rejected_KL_mean": -16.06523895263672, "KL/std": 24.1287899017334, "epoch": 0.33507853403141363, "fcm_dpo/beta": 0.01965140737593174, "fcm_dpo/delta": -0.023824896663427353, "fcm_dpo/margin": 19.878883361816406, "fcm_dpo/q_t": 0.41106274724006653, "grad_norm": 50.02199172973633, "learning_rate": 4.218561044282098e-07, "logits/chosen": 1.4759557247161865, "logits/rejected": 1.4768081903457642, "logps/chosen": -263.29534912109375, "logps/ref_chosen": -267.10894775390625, "logps/ref_rejected": -258.66162109375, "logps/rejected": -274.72686767578125, "loss": 4.4502, "margin_dpo/margin_mean": 19.878883361816406, "margin_dpo/margin_std": 27.98908805847168, "step": 160 }, { "KL/chosen_KL_mean": 3.346097946166992, "KL/mean": -6.985053539276123, "KL/rejected_KL_mean": -17.31619644165039, "KL/std": 25.046649932861328, "epoch": 0.33717277486910996, "fcm_dpo/beta": 0.019791923463344574, "fcm_dpo/delta": -0.009587782435119152, "fcm_dpo/margin": 20.662309646606445, "fcm_dpo/q_t": 0.40747061371803284, "grad_norm": 42.32925033569336, "learning_rate": 4.2052190435769554e-07, "logits/chosen": 1.7133197784423828, "logits/rejected": 1.6081936359405518, "logps/chosen": -298.5462646484375, "logps/ref_chosen": -301.89239501953125, "logps/ref_rejected": -247.645263671875, "logps/rejected": -264.9614562988281, "loss": 4.4483, "margin_dpo/margin_mean": 20.662309646606445, "margin_dpo/margin_std": 31.163915634155273, "step": 161 }, { "KL/chosen_KL_mean": 4.65040397644043, "KL/mean": -4.603542327880859, "KL/rejected_KL_mean": -13.857484817504883, "KL/std": 24.12982940673828, "epoch": 0.3392670157068063, "fcm_dpo/beta": 0.01887052319943905, "fcm_dpo/delta": -0.011755827814340591, "fcm_dpo/margin": 18.507890701293945, "fcm_dpo/q_t": 0.4189423620700836, "grad_norm": 36.829593658447266, "learning_rate": 4.1917855971495763e-07, "logits/chosen": 1.603858232498169, "logits/rejected": 1.5859763622283936, "logps/chosen": -283.5826416015625, "logps/ref_chosen": -288.2330627441406, "logps/ref_rejected": -252.5148162841797, "logps/rejected": -266.3722839355469, "loss": 4.5906, "margin_dpo/margin_mean": 18.507892608642578, "margin_dpo/margin_std": 29.093059539794922, "step": 162 }, { "KL/chosen_KL_mean": 3.3226470947265625, "KL/mean": -7.70544958114624, "KL/rejected_KL_mean": -18.73355484008789, "KL/std": 23.491344451904297, "epoch": 0.3413612565445026, "fcm_dpo/beta": 0.019159583374857903, "fcm_dpo/delta": -0.024082660675048828, "fcm_dpo/margin": 22.05620002746582, "fcm_dpo/q_t": 0.40153104066848755, "grad_norm": 37.20431137084961, "learning_rate": 4.1782614253949255e-07, "logits/chosen": 1.6068838834762573, "logits/rejected": 1.5826996564865112, "logps/chosen": -283.4411926269531, "logps/ref_chosen": -286.7638244628906, "logps/ref_rejected": -262.8302917480469, "logps/rejected": -281.5638122558594, "loss": 4.2868, "margin_dpo/margin_mean": 22.056196212768555, "margin_dpo/margin_std": 26.158645629882812, "step": 163 }, { "KL/chosen_KL_mean": -3.850292205810547, "KL/mean": -13.29897403717041, "KL/rejected_KL_mean": -22.747655868530273, "KL/std": 25.1888370513916, "epoch": 0.34345549738219894, "fcm_dpo/beta": 0.018868062645196915, "fcm_dpo/delta": -0.02091181091964245, "fcm_dpo/margin": 18.897357940673828, "fcm_dpo/q_t": 0.4178180396556854, "grad_norm": 32.7020378112793, "learning_rate": 4.164647253573289e-07, "logits/chosen": 1.732231616973877, "logits/rejected": 1.7001898288726807, "logps/chosen": -264.1287841796875, "logps/ref_chosen": -260.2784729003906, "logps/ref_rejected": -217.10556030273438, "logps/rejected": -239.8532257080078, "loss": 4.5442, "margin_dpo/margin_mean": 18.897357940673828, "margin_dpo/margin_std": 28.706422805786133, "step": 164 }, { "KL/chosen_KL_mean": 0.1812763214111328, "KL/mean": -7.839303970336914, "KL/rejected_KL_mean": -15.859880447387695, "KL/std": 23.385269165039062, "epoch": 0.34554973821989526, "fcm_dpo/beta": 0.01889457367360592, "fcm_dpo/delta": 0.024164600297808647, "fcm_dpo/margin": 16.041149139404297, "fcm_dpo/q_t": 0.4299403131008148, "grad_norm": 36.616493225097656, "learning_rate": 4.1509438117713863e-07, "logits/chosen": 1.7898118495941162, "logits/rejected": 1.8426463603973389, "logps/chosen": -270.373046875, "logps/ref_chosen": -270.5543212890625, "logps/ref_rejected": -259.80218505859375, "logps/rejected": -275.66204833984375, "loss": 4.6946, "margin_dpo/margin_mean": 16.041149139404297, "margin_dpo/margin_std": 26.979326248168945, "step": 165 }, { "KL/chosen_KL_mean": -2.423433303833008, "KL/mean": -9.657724380493164, "KL/rejected_KL_mean": -16.892017364501953, "KL/std": 25.662090301513672, "epoch": 0.34764397905759165, "fcm_dpo/beta": 0.018830828368663788, "fcm_dpo/delta": 0.014171226881444454, "fcm_dpo/margin": 14.468596458435059, "fcm_dpo/q_t": 0.4390319287776947, "grad_norm": 32.48873519897461, "learning_rate": 4.137151834863213e-07, "logits/chosen": 1.5254061222076416, "logits/rejected": 1.5039020776748657, "logps/chosen": -272.7178649902344, "logps/ref_chosen": -270.29443359375, "logps/ref_rejected": -230.40151977539062, "logps/rejected": -247.29351806640625, "loss": 4.9205, "margin_dpo/margin_mean": 14.468595504760742, "margin_dpo/margin_std": 33.09213638305664, "step": 166 }, { "KL/chosen_KL_mean": -2.779369354248047, "KL/mean": -11.354179382324219, "KL/rejected_KL_mean": -19.92898941040039, "KL/std": 25.046934127807617, "epoch": 0.34973821989528797, "fcm_dpo/beta": 0.01971183903515339, "fcm_dpo/delta": 0.028924930840730667, "fcm_dpo/margin": 17.14962387084961, "fcm_dpo/q_t": 0.42319583892822266, "grad_norm": 34.71977233886719, "learning_rate": 4.123272062470633e-07, "logits/chosen": 1.7265127897262573, "logits/rejected": 1.6851749420166016, "logps/chosen": -276.2399597167969, "logps/ref_chosen": -273.4605712890625, "logps/ref_rejected": -248.86997985839844, "logps/rejected": -268.7989501953125, "loss": 4.6612, "margin_dpo/margin_mean": 17.14962387084961, "margin_dpo/margin_std": 30.061779022216797, "step": 167 }, { "KL/chosen_KL_mean": 0.3898735046386719, "KL/mean": -10.855436325073242, "KL/rejected_KL_mean": -22.100749969482422, "KL/std": 27.144193649291992, "epoch": 0.3518324607329843, "fcm_dpo/beta": 0.01884337142109871, "fcm_dpo/delta": -0.07109567523002625, "fcm_dpo/margin": 22.490629196166992, "fcm_dpo/q_t": 0.4015496075153351, "grad_norm": 35.232513427734375, "learning_rate": 4.1093052389237174e-07, "logits/chosen": 1.7364689111709595, "logits/rejected": 1.7493667602539062, "logps/chosen": -305.3348388671875, "logps/ref_chosen": -305.7247009277344, "logps/ref_rejected": -290.16650390625, "logps/rejected": -312.26727294921875, "loss": 4.3533, "margin_dpo/margin_mean": 22.490631103515625, "margin_dpo/margin_std": 29.51881217956543, "step": 168 }, { "KL/chosen_KL_mean": -0.20895767211914062, "KL/mean": -13.058430671691895, "KL/rejected_KL_mean": -25.90790557861328, "KL/std": 26.704998016357422, "epoch": 0.3539267015706806, "fcm_dpo/beta": 0.017862707376480103, "fcm_dpo/delta": -0.09691692888736725, "fcm_dpo/margin": 25.698944091796875, "fcm_dpo/q_t": 0.3947722017765045, "grad_norm": 29.24917221069336, "learning_rate": 4.0952521132208267e-07, "logits/chosen": 1.2933053970336914, "logits/rejected": 1.3155572414398193, "logps/chosen": -252.3792266845703, "logps/ref_chosen": -252.1702880859375, "logps/ref_rejected": -233.34750366210938, "logps/rejected": -259.2554016113281, "loss": 4.2295, "margin_dpo/margin_mean": 25.698944091796875, "margin_dpo/margin_std": 29.882102966308594, "step": 169 }, { "KL/chosen_KL_mean": 0.6535549163818359, "KL/mean": -10.402743339538574, "KL/rejected_KL_mean": -21.459043502807617, "KL/std": 25.29845428466797, "epoch": 0.35602094240837695, "fcm_dpo/beta": 0.017152002081274986, "fcm_dpo/delta": 0.02156105265021324, "fcm_dpo/margin": 22.11260223388672, "fcm_dpo/q_t": 0.41253462433815, "grad_norm": 37.989990234375, "learning_rate": 4.081113438988443e-07, "logits/chosen": 1.4381834268569946, "logits/rejected": 1.4736790657043457, "logps/chosen": -301.8248596191406, "logps/ref_chosen": -302.4783935546875, "logps/ref_rejected": -259.40777587890625, "logps/rejected": -280.8668212890625, "loss": 4.4294, "margin_dpo/margin_mean": 22.112600326538086, "margin_dpo/margin_std": 30.194551467895508, "step": 170 }, { "KL/chosen_KL_mean": 0.401123046875, "KL/mean": -10.984270095825195, "KL/rejected_KL_mean": -22.36966896057129, "KL/std": 25.969249725341797, "epoch": 0.3581151832460733, "fcm_dpo/beta": 0.016981348395347595, "fcm_dpo/delta": -0.0327591747045517, "fcm_dpo/margin": 22.770790100097656, "fcm_dpo/q_t": 0.41038233041763306, "grad_norm": 33.292850494384766, "learning_rate": 4.0668899744407567e-07, "logits/chosen": 1.3699983358383179, "logits/rejected": 1.4796205759048462, "logps/chosen": -250.24560546875, "logps/ref_chosen": -250.646728515625, "logps/ref_rejected": -259.1659851074219, "logps/rejected": -281.5356750488281, "loss": 4.409, "margin_dpo/margin_mean": 22.770788192749023, "margin_dpo/margin_std": 29.17012596130371, "step": 171 }, { "KL/chosen_KL_mean": 1.918487548828125, "KL/mean": -5.691974639892578, "KL/rejected_KL_mean": -13.302434921264648, "KL/std": 26.688800811767578, "epoch": 0.36020942408376966, "fcm_dpo/beta": 0.017365295439958572, "fcm_dpo/delta": 0.04691143333911896, "fcm_dpo/margin": 15.220909118652344, "fcm_dpo/q_t": 0.4379444122314453, "grad_norm": 31.966800689697266, "learning_rate": 4.0525824823390043e-07, "logits/chosen": 1.688840627670288, "logits/rejected": 1.6983768939971924, "logps/chosen": -327.1064453125, "logps/ref_chosen": -329.02490234375, "logps/ref_rejected": -293.0215759277344, "logps/rejected": -306.3240051269531, "loss": 4.8557, "margin_dpo/margin_mean": 15.220909118652344, "margin_dpo/margin_std": 30.862401962280273, "step": 172 }, { "KL/chosen_KL_mean": 1.177724838256836, "KL/mean": -8.230151176452637, "KL/rejected_KL_mean": -17.638029098510742, "KL/std": 26.464139938354492, "epoch": 0.362303664921466, "fcm_dpo/beta": 0.017997927963733673, "fcm_dpo/delta": 0.020180756226181984, "fcm_dpo/margin": 18.815750122070312, "fcm_dpo/q_t": 0.4219967722892761, "grad_norm": 31.154638290405273, "learning_rate": 4.0381917299505686e-07, "logits/chosen": 1.525386095046997, "logits/rejected": 1.496148705482483, "logps/chosen": -288.962890625, "logps/ref_chosen": -290.140625, "logps/ref_rejected": -271.474853515625, "logps/rejected": -289.11285400390625, "loss": 4.5925, "margin_dpo/margin_mean": 18.815750122070312, "margin_dpo/margin_std": 29.17994499206543, "step": 173 }, { "KL/chosen_KL_mean": 2.210672378540039, "KL/mean": -9.635655403137207, "KL/rejected_KL_mean": -21.481983184814453, "KL/std": 25.624143600463867, "epoch": 0.3643979057591623, "fcm_dpo/beta": 0.017743976786732674, "fcm_dpo/delta": -0.022125719115138054, "fcm_dpo/margin": 23.69266128540039, "fcm_dpo/q_t": 0.40354806184768677, "grad_norm": 42.779747009277344, "learning_rate": 4.0237184890078243e-07, "logits/chosen": 1.5114091634750366, "logits/rejected": 1.4241846799850464, "logps/chosen": -327.0965881347656, "logps/ref_chosen": -329.3072509765625, "logps/ref_rejected": -256.6839599609375, "logps/rejected": -278.16595458984375, "loss": 4.3127, "margin_dpo/margin_mean": 23.692659378051758, "margin_dpo/margin_std": 29.146427154541016, "step": 174 }, { "KL/chosen_KL_mean": -0.10210800170898438, "KL/mean": -9.668460845947266, "KL/rejected_KL_mean": -19.23480987548828, "KL/std": 26.648395538330078, "epoch": 0.36649214659685864, "fcm_dpo/beta": 0.017704615369439125, "fcm_dpo/delta": 0.0033662477508187294, "fcm_dpo/margin": 19.132694244384766, "fcm_dpo/q_t": 0.42134740948677063, "grad_norm": 32.94378662109375, "learning_rate": 4.00916353566676e-07, "logits/chosen": 1.6281156539916992, "logits/rejected": 1.6203536987304688, "logps/chosen": -276.0043640136719, "logps/ref_chosen": -275.9023132324219, "logps/ref_rejected": -281.4163513183594, "logps/rejected": -300.65118408203125, "loss": 4.6253, "margin_dpo/margin_mean": 19.1326961517334, "margin_dpo/margin_std": 31.65574836730957, "step": 175 }, { "KL/chosen_KL_mean": -1.8588676452636719, "KL/mean": -8.88139533996582, "KL/rejected_KL_mean": -15.903926849365234, "KL/std": 26.206148147583008, "epoch": 0.36858638743455496, "fcm_dpo/beta": 0.017541371285915375, "fcm_dpo/delta": -0.027952371165156364, "fcm_dpo/margin": 14.045061111450195, "fcm_dpo/q_t": 0.4421733021736145, "grad_norm": 33.93702697753906, "learning_rate": 3.994527650465352e-07, "logits/chosen": 1.3971831798553467, "logits/rejected": 1.4549391269683838, "logps/chosen": -245.423095703125, "logps/ref_chosen": -243.56423950195312, "logps/ref_rejected": -239.12905883789062, "logps/rejected": -255.03297424316406, "loss": 4.9662, "margin_dpo/margin_mean": 14.045061111450195, "margin_dpo/margin_std": 31.961017608642578, "step": 176 }, { "KL/chosen_KL_mean": -2.177288055419922, "KL/mean": -9.260589599609375, "KL/rejected_KL_mean": -16.343902587890625, "KL/std": 26.89461326599121, "epoch": 0.3706806282722513, "fcm_dpo/beta": 0.017628297209739685, "fcm_dpo/delta": 0.03455434739589691, "fcm_dpo/margin": 14.166617393493652, "fcm_dpo/q_t": 0.4423142969608307, "grad_norm": 62.61311340332031, "learning_rate": 3.979811618281705e-07, "logits/chosen": 1.5508711338043213, "logits/rejected": 1.4726340770721436, "logps/chosen": -295.48095703125, "logps/ref_chosen": -293.30364990234375, "logps/ref_rejected": -292.293701171875, "logps/rejected": -308.6376037597656, "loss": 4.9989, "margin_dpo/margin_mean": 14.166617393493652, "margin_dpo/margin_std": 34.50107192993164, "step": 177 }, { "KL/chosen_KL_mean": 2.100444793701172, "KL/mean": -9.132177352905273, "KL/rejected_KL_mean": -20.36479949951172, "KL/std": 28.20733070373535, "epoch": 0.37277486910994767, "fcm_dpo/beta": 0.017436780035495758, "fcm_dpo/delta": -0.02598407492041588, "fcm_dpo/margin": 22.465251922607422, "fcm_dpo/q_t": 0.4109533429145813, "grad_norm": 36.81523132324219, "learning_rate": 3.9650162282919654e-07, "logits/chosen": 1.6348121166229248, "logits/rejected": 1.6586459875106812, "logps/chosen": -274.0257873535156, "logps/ref_chosen": -276.126220703125, "logps/ref_rejected": -240.31179809570312, "logps/rejected": -260.6766052246094, "loss": 4.4651, "margin_dpo/margin_mean": 22.46525001525879, "margin_dpo/margin_std": 33.34449768066406, "step": 178 }, { "KL/chosen_KL_mean": 1.2245655059814453, "KL/mean": -6.5566229820251465, "KL/rejected_KL_mean": -14.337806701660156, "KL/std": 27.405574798583984, "epoch": 0.374869109947644, "fcm_dpo/beta": 0.01756238378584385, "fcm_dpo/delta": 0.03585870563983917, "fcm_dpo/margin": 15.56238079071045, "fcm_dpo/q_t": 0.4376697540283203, "grad_norm": 37.79298782348633, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 1.316421389579773, "logits/rejected": 1.4284144639968872, "logps/chosen": -248.6378631591797, "logps/ref_chosen": -249.8624267578125, "logps/ref_rejected": -271.18603515625, "logps/rejected": -285.52386474609375, "loss": 4.8619, "margin_dpo/margin_mean": 15.562381744384766, "margin_dpo/margin_std": 31.931869506835938, "step": 179 }, { "KL/chosen_KL_mean": 2.530637741088867, "KL/mean": -7.931081295013428, "KL/rejected_KL_mean": -18.39280128479004, "KL/std": 26.43503189086914, "epoch": 0.3769633507853403, "fcm_dpo/beta": 0.017923174425959587, "fcm_dpo/delta": -0.00823952816426754, "fcm_dpo/margin": 20.92344856262207, "fcm_dpo/q_t": 0.4146185517311096, "grad_norm": 44.96994400024414, "learning_rate": 3.935190552834828e-07, "logits/chosen": 1.5395066738128662, "logits/rejected": 1.5768020153045654, "logps/chosen": -258.13848876953125, "logps/ref_chosen": -260.66912841796875, "logps/ref_rejected": -229.25491333007812, "logps/rejected": -247.64773559570312, "loss": 4.5056, "margin_dpo/margin_mean": 20.92344856262207, "margin_dpo/margin_std": 31.313364028930664, "step": 180 }, { "KL/chosen_KL_mean": 4.5495452880859375, "KL/mean": -5.502880096435547, "KL/rejected_KL_mean": -15.555295944213867, "KL/std": 26.449636459350586, "epoch": 0.37905759162303665, "fcm_dpo/beta": 0.017637405544519424, "fcm_dpo/delta": -0.005057952366769314, "fcm_dpo/margin": 20.104846954345703, "fcm_dpo/q_t": 0.4186038374900818, "grad_norm": 36.245113372802734, "learning_rate": 3.920161866827889e-07, "logits/chosen": 1.5702896118164062, "logits/rejected": 1.54368257522583, "logps/chosen": -286.3390197753906, "logps/ref_chosen": -290.8885803222656, "logps/ref_rejected": -275.1922302246094, "logps/rejected": -290.7475280761719, "loss": 4.5959, "margin_dpo/margin_mean": 20.104846954345703, "margin_dpo/margin_std": 32.911582946777344, "step": 181 }, { "KL/chosen_KL_mean": 2.2509727478027344, "KL/mean": -10.04803466796875, "KL/rejected_KL_mean": -22.347047805786133, "KL/std": 26.909276962280273, "epoch": 0.381151832460733, "fcm_dpo/beta": 0.017666857689619064, "fcm_dpo/delta": -0.036299802362918854, "fcm_dpo/margin": 24.598037719726562, "fcm_dpo/q_t": 0.3986150324344635, "grad_norm": 47.16189193725586, "learning_rate": 3.90505702185e-07, "logits/chosen": 1.46827232837677, "logits/rejected": 1.4841911792755127, "logps/chosen": -268.31585693359375, "logps/ref_chosen": -270.56683349609375, "logps/ref_rejected": -260.3321533203125, "logps/rejected": -282.6791687011719, "loss": 4.2547, "margin_dpo/margin_mean": 24.598037719726562, "margin_dpo/margin_std": 28.5032958984375, "step": 182 }, { "KL/chosen_KL_mean": 2.6596717834472656, "KL/mean": -7.440838813781738, "KL/rejected_KL_mean": -17.541349411010742, "KL/std": 26.812868118286133, "epoch": 0.3832460732984293, "fcm_dpo/beta": 0.01747625507414341, "fcm_dpo/delta": 0.048673342913389206, "fcm_dpo/margin": 20.20101547241211, "fcm_dpo/q_t": 0.4192884564399719, "grad_norm": 35.38688659667969, "learning_rate": 3.889876827928156e-07, "logits/chosen": 1.5745729207992554, "logits/rejected": 1.6223291158676147, "logps/chosen": -258.3683166503906, "logps/ref_chosen": -261.0279846191406, "logps/ref_rejected": -240.72361755371094, "logps/rejected": -258.2649841308594, "loss": 4.5729, "margin_dpo/margin_mean": 20.20101547241211, "margin_dpo/margin_std": 32.505943298339844, "step": 183 }, { "KL/chosen_KL_mean": 6.955654144287109, "KL/mean": -7.538846492767334, "KL/rejected_KL_mean": -22.033349990844727, "KL/std": 29.865995407104492, "epoch": 0.38534031413612563, "fcm_dpo/beta": 0.016830746084451675, "fcm_dpo/delta": -0.09457923471927643, "fcm_dpo/margin": 28.989009857177734, "fcm_dpo/q_t": 0.3883908689022064, "grad_norm": 36.70336151123047, "learning_rate": 3.874622099130087e-07, "logits/chosen": 1.4726858139038086, "logits/rejected": 1.4355816841125488, "logps/chosen": -302.1282043457031, "logps/ref_chosen": -309.0838317871094, "logps/ref_rejected": -263.6435241699219, "logps/rejected": -285.6768798828125, "loss": 4.1719, "margin_dpo/margin_mean": 28.989009857177734, "margin_dpo/margin_std": 35.74262237548828, "step": 184 }, { "KL/chosen_KL_mean": 1.0868282318115234, "KL/mean": -9.744384765625, "KL/rejected_KL_mean": -20.57560157775879, "KL/std": 28.781902313232422, "epoch": 0.387434554973822, "fcm_dpo/beta": 0.0165347158908844, "fcm_dpo/delta": 0.006427157670259476, "fcm_dpo/margin": 21.66242218017578, "fcm_dpo/q_t": 0.41854044795036316, "grad_norm": 39.357120513916016, "learning_rate": 3.859293653520604e-07, "logits/chosen": 1.453484058380127, "logits/rejected": 1.5226836204528809, "logps/chosen": -264.6148681640625, "logps/ref_chosen": -265.70172119140625, "logps/ref_rejected": -257.5074157714844, "logps/rejected": -278.0830078125, "loss": 4.5754, "margin_dpo/margin_mean": 21.662424087524414, "margin_dpo/margin_std": 34.37306594848633, "step": 185 }, { "KL/chosen_KL_mean": -0.38506507873535156, "KL/mean": -10.96863842010498, "KL/rejected_KL_mean": -21.55221176147461, "KL/std": 27.273874282836914, "epoch": 0.38952879581151834, "fcm_dpo/beta": 0.016737408936023712, "fcm_dpo/delta": 0.014673182740807533, "fcm_dpo/margin": 21.167146682739258, "fcm_dpo/q_t": 0.4183846414089203, "grad_norm": 47.73256301879883, "learning_rate": 3.8438923131177237e-07, "logits/chosen": 1.437438726425171, "logits/rejected": 1.4502127170562744, "logps/chosen": -292.11004638671875, "logps/ref_chosen": -291.7249755859375, "logps/ref_rejected": -273.54010009765625, "logps/rejected": -295.09234619140625, "loss": 4.5356, "margin_dpo/margin_mean": 21.16714859008789, "margin_dpo/margin_std": 31.127275466918945, "step": 186 }, { "KL/chosen_KL_mean": 0.13139915466308594, "KL/mean": -10.805803298950195, "KL/rejected_KL_mean": -21.74300193786621, "KL/std": 29.678203582763672, "epoch": 0.39162303664921466, "fcm_dpo/beta": 0.016892198473215103, "fcm_dpo/delta": 0.031411200761795044, "fcm_dpo/margin": 21.874393463134766, "fcm_dpo/q_t": 0.4164515733718872, "grad_norm": 37.13667678833008, "learning_rate": 3.828418903848593e-07, "logits/chosen": 1.5591447353363037, "logits/rejected": 1.5824084281921387, "logps/chosen": -321.5704650878906, "logps/ref_chosen": -321.70184326171875, "logps/ref_rejected": -261.8386535644531, "logps/rejected": -283.5816650390625, "loss": 4.5952, "margin_dpo/margin_mean": 21.874393463134766, "margin_dpo/margin_std": 37.063011169433594, "step": 187 }, { "KL/chosen_KL_mean": -1.804117202758789, "KL/mean": -12.928169250488281, "KL/rejected_KL_mean": -24.052230834960938, "KL/std": 29.605234146118164, "epoch": 0.393717277486911, "fcm_dpo/beta": 0.01713397167623043, "fcm_dpo/delta": -0.009998919442296028, "fcm_dpo/margin": 22.248119354248047, "fcm_dpo/q_t": 0.41325312852859497, "grad_norm": 33.81171798706055, "learning_rate": 3.812874255505191e-07, "logits/chosen": 1.3632234334945679, "logits/rejected": 1.333018183708191, "logps/chosen": -257.71954345703125, "logps/ref_chosen": -255.91542053222656, "logps/ref_rejected": -245.1119842529297, "logps/rejected": -269.1642150878906, "loss": 4.5532, "margin_dpo/margin_mean": 22.248119354248047, "margin_dpo/margin_std": 36.244773864746094, "step": 188 }, { "KL/chosen_KL_mean": 3.7180252075195312, "KL/mean": -9.177162170410156, "KL/rejected_KL_mean": -22.07235336303711, "KL/std": 29.054580688476562, "epoch": 0.3958115183246073, "fcm_dpo/beta": 0.016553619876503944, "fcm_dpo/delta": -0.0703524798154831, "fcm_dpo/margin": 25.790372848510742, "fcm_dpo/q_t": 0.40066149830818176, "grad_norm": 28.8972225189209, "learning_rate": 3.797259201699833e-07, "logits/chosen": 1.5498346090316772, "logits/rejected": 1.5829347372055054, "logps/chosen": -262.300537109375, "logps/ref_chosen": -266.0185546875, "logps/ref_rejected": -272.862548828125, "logps/rejected": -294.9349060058594, "loss": 4.2716, "margin_dpo/margin_mean": 25.790372848510742, "margin_dpo/margin_std": 29.882930755615234, "step": 189 }, { "KL/chosen_KL_mean": 2.1718368530273438, "KL/mean": -10.560308456420898, "KL/rejected_KL_mean": -23.292451858520508, "KL/std": 28.673686981201172, "epoch": 0.39790575916230364, "fcm_dpo/beta": 0.016140490770339966, "fcm_dpo/delta": -0.011915532872080803, "fcm_dpo/margin": 25.464290618896484, "fcm_dpo/q_t": 0.40527617931365967, "grad_norm": 30.04618263244629, "learning_rate": 3.781574579820464e-07, "logits/chosen": 1.4085410833358765, "logits/rejected": 1.3426133394241333, "logps/chosen": -247.9791259765625, "logps/ref_chosen": -250.15098571777344, "logps/ref_rejected": -223.34393310546875, "logps/rejected": -246.63636779785156, "loss": 4.3943, "margin_dpo/margin_mean": 25.464290618896484, "margin_dpo/margin_std": 35.534278869628906, "step": 190 }, { "KL/chosen_KL_mean": 1.7022075653076172, "KL/mean": -8.630979537963867, "KL/rejected_KL_mean": -18.964176177978516, "KL/std": 29.75131607055664, "epoch": 0.4, "fcm_dpo/beta": 0.01615564338862896, "fcm_dpo/delta": 0.02544761821627617, "fcm_dpo/margin": 20.6663818359375, "fcm_dpo/q_t": 0.4240559935569763, "grad_norm": 41.732566833496094, "learning_rate": 3.765821230985757e-07, "logits/chosen": 1.316603660583496, "logits/rejected": 1.376888632774353, "logps/chosen": -236.68960571289062, "logps/ref_chosen": -238.3917999267578, "logps/ref_rejected": -243.70272827148438, "logps/rejected": -262.6669006347656, "loss": 4.6813, "margin_dpo/margin_mean": 20.6663818359375, "margin_dpo/margin_std": 37.0047607421875, "step": 191 }, { "KL/chosen_KL_mean": -2.1696548461914062, "KL/mean": -11.991387367248535, "KL/rejected_KL_mean": -21.8131046295166, "KL/std": 29.120777130126953, "epoch": 0.40209424083769635, "fcm_dpo/beta": 0.016227317973971367, "fcm_dpo/delta": -0.005627312697470188, "fcm_dpo/margin": 19.643455505371094, "fcm_dpo/q_t": 0.42661046981811523, "grad_norm": 39.09224319458008, "learning_rate": 3.75e-07, "logits/chosen": 1.5734920501708984, "logits/rejected": 1.653855562210083, "logps/chosen": -258.806640625, "logps/ref_chosen": -256.6369934082031, "logps/ref_rejected": -256.8894348144531, "logps/rejected": -278.7025146484375, "loss": 4.7535, "margin_dpo/margin_mean": 19.64345359802246, "margin_dpo/margin_std": 36.65842819213867, "step": 192 }, { "KL/chosen_KL_mean": -5.266847610473633, "KL/mean": -15.703073501586914, "KL/rejected_KL_mean": -26.139293670654297, "KL/std": 31.600704193115234, "epoch": 0.4041884816753927, "fcm_dpo/beta": 0.016515541821718216, "fcm_dpo/delta": 0.025459565222263336, "fcm_dpo/margin": 20.872455596923828, "fcm_dpo/q_t": 0.42145636677742004, "grad_norm": 37.17631149291992, "learning_rate": 3.734111735307796e-07, "logits/chosen": 1.4087841510772705, "logits/rejected": 1.5019336938858032, "logps/chosen": -278.0664978027344, "logps/ref_chosen": -272.7996826171875, "logps/ref_rejected": -289.2391052246094, "logps/rejected": -315.3783874511719, "loss": 4.6456, "margin_dpo/margin_mean": 20.87245750427246, "margin_dpo/margin_std": 36.63458251953125, "step": 193 }, { "KL/chosen_KL_mean": -4.638954162597656, "KL/mean": -12.322515487670898, "KL/rejected_KL_mean": -20.006086349487305, "KL/std": 31.54027557373047, "epoch": 0.406282722513089, "fcm_dpo/beta": 0.016818445175886154, "fcm_dpo/delta": 0.04274335503578186, "fcm_dpo/margin": 15.367134094238281, "fcm_dpo/q_t": 0.4427103102207184, "grad_norm": 36.010684967041016, "learning_rate": 3.7181572889485623e-07, "logits/chosen": 1.3490233421325684, "logits/rejected": 1.3899866342544556, "logps/chosen": -285.947998046875, "logps/ref_chosen": -281.30902099609375, "logps/ref_rejected": -246.3243408203125, "logps/rejected": -266.3304443359375, "loss": 4.9282, "margin_dpo/margin_mean": 15.367136001586914, "margin_dpo/margin_std": 34.865966796875, "step": 194 }, { "KL/chosen_KL_mean": -4.831840515136719, "KL/mean": -12.867500305175781, "KL/rejected_KL_mean": -20.903165817260742, "KL/std": 30.93072509765625, "epoch": 0.4083769633507853, "fcm_dpo/beta": 0.01715153455734253, "fcm_dpo/delta": -0.0032990314066410065, "fcm_dpo/margin": 16.071332931518555, "fcm_dpo/q_t": 0.4376472234725952, "grad_norm": 35.62577438354492, "learning_rate": 3.7021375165108377e-07, "logits/chosen": 1.548952579498291, "logits/rejected": 1.6486254930496216, "logps/chosen": -271.68994140625, "logps/ref_chosen": -266.8581237792969, "logps/ref_rejected": -276.0855712890625, "logps/rejected": -296.9887390136719, "loss": 4.8675, "margin_dpo/margin_mean": 16.071334838867188, "margin_dpo/margin_std": 34.527015686035156, "step": 195 }, { "KL/chosen_KL_mean": 0.8048229217529297, "KL/mean": -11.171314239501953, "KL/rejected_KL_mean": -23.14745330810547, "KL/std": 29.260290145874023, "epoch": 0.41047120418848165, "fcm_dpo/beta": 0.0168392863124609, "fcm_dpo/delta": -0.062426064163446426, "fcm_dpo/margin": 23.952272415161133, "fcm_dpo/q_t": 0.4076106548309326, "grad_norm": 47.47340393066406, "learning_rate": 3.6860532770864005e-07, "logits/chosen": 1.3038650751113892, "logits/rejected": 1.4030379056930542, "logps/chosen": -266.19012451171875, "logps/ref_chosen": -266.99493408203125, "logps/ref_rejected": -248.60513305664062, "logps/rejected": -271.7525939941406, "loss": 4.4078, "margin_dpo/margin_mean": 23.952272415161133, "margin_dpo/margin_std": 32.61539077758789, "step": 196 }, { "KL/chosen_KL_mean": 3.457935333251953, "KL/mean": -11.847572326660156, "KL/rejected_KL_mean": -27.153076171875, "KL/std": 32.25026321411133, "epoch": 0.41256544502617803, "fcm_dpo/beta": 0.015746818855404854, "fcm_dpo/delta": -0.11807969212532043, "fcm_dpo/margin": 30.611000061035156, "fcm_dpo/q_t": 0.38884419202804565, "grad_norm": 45.38121795654297, "learning_rate": 3.6699054332241985e-07, "logits/chosen": 1.2113600969314575, "logits/rejected": 1.1563668251037598, "logps/chosen": -299.5318603515625, "logps/ref_chosen": -302.98980712890625, "logps/ref_rejected": -262.8002624511719, "logps/rejected": -289.95330810546875, "loss": 4.1683, "margin_dpo/margin_mean": 30.611000061035156, "margin_dpo/margin_std": 34.92479705810547, "step": 197 }, { "KL/chosen_KL_mean": 0.10808563232421875, "KL/mean": -11.575591087341309, "KL/rejected_KL_mean": -23.25926399230957, "KL/std": 32.306583404541016, "epoch": 0.41465968586387436, "fcm_dpo/beta": 0.014680261723697186, "fcm_dpo/delta": 0.001502858940511942, "fcm_dpo/margin": 23.36733627319336, "fcm_dpo/q_t": 0.42125099897384644, "grad_norm": 32.45808029174805, "learning_rate": 3.653694850884091e-07, "logits/chosen": 1.5946077108383179, "logits/rejected": 1.6851319074630737, "logps/chosen": -296.10369873046875, "logps/ref_chosen": -296.2117614746094, "logps/ref_rejected": -289.8357238769531, "logps/rejected": -313.0950012207031, "loss": 4.6621, "margin_dpo/margin_mean": 23.36733627319336, "margin_dpo/margin_std": 41.27313995361328, "step": 198 }, { "KL/chosen_KL_mean": -1.3021049499511719, "KL/mean": -13.162481307983398, "KL/rejected_KL_mean": -25.02284812927246, "KL/std": 30.66471290588379, "epoch": 0.4167539267015707, "fcm_dpo/beta": 0.015121981501579285, "fcm_dpo/delta": 0.013484282419085503, "fcm_dpo/margin": 23.72073745727539, "fcm_dpo/q_t": 0.4176020324230194, "grad_norm": 30.408153533935547, "learning_rate": 3.6374223993904124e-07, "logits/chosen": 1.2783989906311035, "logits/rejected": 1.2025794982910156, "logps/chosen": -256.5947265625, "logps/ref_chosen": -255.29263305664062, "logps/ref_rejected": -212.69761657714844, "logps/rejected": -237.720458984375, "loss": 4.5483, "margin_dpo/margin_mean": 23.720739364624023, "margin_dpo/margin_std": 37.188148498535156, "step": 199 }, { "KL/chosen_KL_mean": -0.3975563049316406, "KL/mean": -10.419899940490723, "KL/rejected_KL_mean": -20.442241668701172, "KL/std": 32.0081787109375, "epoch": 0.418848167539267, "fcm_dpo/beta": 0.015231948345899582, "fcm_dpo/delta": 0.03270227462053299, "fcm_dpo/margin": 20.044692993164062, "fcm_dpo/q_t": 0.43182697892189026, "grad_norm": 41.86520767211914, "learning_rate": 3.621088951385353e-07, "logits/chosen": 1.7400442361831665, "logits/rejected": 1.739106297492981, "logps/chosen": -313.9339599609375, "logps/ref_chosen": -313.5364074707031, "logps/ref_rejected": -274.349609375, "logps/rejected": -294.79180908203125, "loss": 4.8731, "margin_dpo/margin_mean": 20.044694900512695, "margin_dpo/margin_std": 44.38846969604492, "step": 200 }, { "epoch": 0.418848167539267, "eval_KL/chosen_KL_mean": -4.467589855194092, "eval_KL/mean": -16.012392044067383, "eval_KL/rejected_KL_mean": -27.557188034057617, "eval_KL/std": 31.657482147216797, "eval_fcm_dpo/beta": 0.015443911775946617, "eval_logits/chosen": 1.6197923421859741, "eval_logits/rejected": 1.661108136177063, "eval_logps/chosen": -284.88427734375, "eval_logps/ref_chosen": -280.41668701171875, "eval_logps/ref_rejected": -264.6455383300781, "eval_logps/rejected": -292.2027282714844, "eval_loss": 0.5750734806060791, "eval_margin_dpo/margin_mean": 23.089601516723633, "eval_margin_dpo/margin_std": 36.13518524169922, "eval_runtime": 89.255, "eval_samples_per_second": 22.408, "eval_steps_per_second": 2.801, "step": 200 }, { "KL/chosen_KL_mean": -2.7801513671875, "KL/mean": -15.753643035888672, "KL/rejected_KL_mean": -28.727128982543945, "KL/std": 32.468223571777344, "epoch": 0.42094240837696334, "fcm_dpo/beta": 0.015363471582531929, "fcm_dpo/delta": -0.03582913428544998, "fcm_dpo/margin": 25.946979522705078, "fcm_dpo/q_t": 0.40801653265953064, "grad_norm": 39.52208709716797, "learning_rate": 3.604695382782159e-07, "logits/chosen": 1.3580992221832275, "logits/rejected": 1.3939217329025269, "logps/chosen": -265.4873352050781, "logps/ref_chosen": -262.7071838378906, "logps/ref_rejected": -243.85879516601562, "logps/rejected": -272.5859375, "loss": 4.422, "margin_dpo/margin_mean": 25.946975708007812, "margin_dpo/margin_std": 35.81454849243164, "step": 201 }, { "KL/chosen_KL_mean": -5.743253707885742, "KL/mean": -17.584720611572266, "KL/rejected_KL_mean": -29.426183700561523, "KL/std": 32.779457092285156, "epoch": 0.42303664921465967, "fcm_dpo/beta": 0.014960610307753086, "fcm_dpo/delta": -0.013768346980214119, "fcm_dpo/margin": 23.68292808532715, "fcm_dpo/q_t": 0.4190349876880646, "grad_norm": 34.390716552734375, "learning_rate": 3.588242572718162e-07, "logits/chosen": 1.5304558277130127, "logits/rejected": 1.508581280708313, "logps/chosen": -270.0299987792969, "logps/ref_chosen": -264.2867431640625, "logps/ref_rejected": -231.6894989013672, "logps/rejected": -261.1156921386719, "loss": 4.6531, "margin_dpo/margin_mean": 23.68292808532715, "margin_dpo/margin_std": 41.36975860595703, "step": 202 }, { "KL/chosen_KL_mean": -4.744565963745117, "KL/mean": -14.712820053100586, "KL/rejected_KL_mean": -24.681068420410156, "KL/std": 31.517353057861328, "epoch": 0.42513089005235605, "fcm_dpo/beta": 0.015301337465643883, "fcm_dpo/delta": 0.06527502834796906, "fcm_dpo/margin": 19.936498641967773, "fcm_dpo/q_t": 0.4290742576122284, "grad_norm": 43.57932662963867, "learning_rate": 3.571731403507635e-07, "logits/chosen": 1.5578314065933228, "logits/rejected": 1.5329176187515259, "logps/chosen": -276.0287780761719, "logps/ref_chosen": -271.2842102050781, "logps/ref_rejected": -247.95462036132812, "logps/rejected": -272.63568115234375, "loss": 4.6656, "margin_dpo/margin_mean": 19.936500549316406, "margin_dpo/margin_std": 32.206573486328125, "step": 203 }, { "KL/chosen_KL_mean": -2.7664566040039062, "KL/mean": -16.75481605529785, "KL/rejected_KL_mean": -30.743175506591797, "KL/std": 32.51542663574219, "epoch": 0.4272251308900524, "fcm_dpo/beta": 0.015280654653906822, "fcm_dpo/delta": -0.029274964705109596, "fcm_dpo/margin": 27.976722717285156, "fcm_dpo/q_t": 0.40244799852371216, "grad_norm": 32.696537017822266, "learning_rate": 3.5551627605944746e-07, "logits/chosen": 1.5064336061477661, "logits/rejected": 1.4716084003448486, "logps/chosen": -312.89691162109375, "logps/ref_chosen": -310.13043212890625, "logps/ref_rejected": -267.3905334472656, "logps/rejected": -298.1336669921875, "loss": 4.3423, "margin_dpo/margin_mean": 27.97671890258789, "margin_dpo/margin_std": 38.091148376464844, "step": 204 }, { "KL/chosen_KL_mean": -1.1357479095458984, "KL/mean": -14.039722442626953, "KL/rejected_KL_mean": -26.94369125366211, "KL/std": 31.764497756958008, "epoch": 0.4293193717277487, "fcm_dpo/beta": 0.01524931751191616, "fcm_dpo/delta": -0.023136526346206665, "fcm_dpo/margin": 25.80794906616211, "fcm_dpo/q_t": 0.4099942147731781, "grad_norm": 34.16581344604492, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 1.4873193502426147, "logits/rejected": 1.484212875366211, "logps/chosen": -278.2964172363281, "logps/ref_chosen": -277.16064453125, "logps/ref_rejected": -292.91815185546875, "logps/rejected": -319.8618469238281, "loss": 4.4749, "margin_dpo/margin_mean": 25.807947158813477, "margin_dpo/margin_std": 37.94681167602539, "step": 205 }, { "KL/chosen_KL_mean": -1.1707172393798828, "KL/mean": -12.986665725708008, "KL/rejected_KL_mean": -24.802610397338867, "KL/std": 32.0893440246582, "epoch": 0.431413612565445, "fcm_dpo/beta": 0.014862453565001488, "fcm_dpo/delta": 0.019112059846520424, "fcm_dpo/margin": 23.63189125061035, "fcm_dpo/q_t": 0.41891059279441833, "grad_norm": 34.6976318359375, "learning_rate": 3.5218566107988867e-07, "logits/chosen": 1.7092478275299072, "logits/rejected": 1.8270341157913208, "logps/chosen": -287.95330810546875, "logps/ref_chosen": -286.7825927734375, "logps/ref_rejected": -289.08270263671875, "logps/rejected": -313.8852844238281, "loss": 4.5861, "margin_dpo/margin_mean": 23.63189125061035, "margin_dpo/margin_std": 37.21580123901367, "step": 206 }, { "KL/chosen_KL_mean": -4.060152053833008, "KL/mean": -14.630226135253906, "KL/rejected_KL_mean": -25.20030975341797, "KL/std": 33.9632568359375, "epoch": 0.43350785340314135, "fcm_dpo/beta": 0.01515961904078722, "fcm_dpo/delta": 0.015925094485282898, "fcm_dpo/margin": 21.140159606933594, "fcm_dpo/q_t": 0.42776355147361755, "grad_norm": 44.10590362548828, "learning_rate": 3.505120890024195e-07, "logits/chosen": 1.4632537364959717, "logits/rejected": 1.5669593811035156, "logps/chosen": -270.5924072265625, "logps/ref_chosen": -266.5322570800781, "logps/ref_rejected": -275.20672607421875, "logps/rejected": -300.40704345703125, "loss": 4.8052, "margin_dpo/margin_mean": 21.14015769958496, "margin_dpo/margin_std": 43.635372161865234, "step": 207 }, { "KL/chosen_KL_mean": -3.625772476196289, "KL/mean": -16.000215530395508, "KL/rejected_KL_mean": -28.374656677246094, "KL/std": 33.34564208984375, "epoch": 0.4356020942408377, "fcm_dpo/beta": 0.01525797974318266, "fcm_dpo/delta": -0.04257369041442871, "fcm_dpo/margin": 24.74889373779297, "fcm_dpo/q_t": 0.41384872794151306, "grad_norm": 31.783222198486328, "learning_rate": 3.4883312676665534e-07, "logits/chosen": 1.755733847618103, "logits/rejected": 1.76398766040802, "logps/chosen": -276.4271240234375, "logps/ref_chosen": -272.80133056640625, "logps/ref_rejected": -288.4167785644531, "logps/rejected": -316.7914123535156, "loss": 4.539, "margin_dpo/margin_mean": 24.748897552490234, "margin_dpo/margin_std": 39.13401412963867, "step": 208 }, { "KL/chosen_KL_mean": -7.097347259521484, "KL/mean": -17.537128448486328, "KL/rejected_KL_mean": -27.976924896240234, "KL/std": 33.336891174316406, "epoch": 0.437696335078534, "fcm_dpo/beta": 0.015181426890194416, "fcm_dpo/delta": 0.0385538712143898, "fcm_dpo/margin": 20.87958335876465, "fcm_dpo/q_t": 0.4282674193382263, "grad_norm": 31.456403732299805, "learning_rate": 3.4714886441024573e-07, "logits/chosen": 1.5062553882598877, "logits/rejected": 1.359464406967163, "logps/chosen": -317.7767639160156, "logps/ref_chosen": -310.679443359375, "logps/ref_rejected": -259.3792419433594, "logps/rejected": -287.3561706542969, "loss": 4.7444, "margin_dpo/margin_mean": 20.879581451416016, "margin_dpo/margin_std": 38.3335075378418, "step": 209 }, { "KL/chosen_KL_mean": -2.737762451171875, "KL/mean": -15.045697212219238, "KL/rejected_KL_mean": -27.353635787963867, "KL/std": 33.89357376098633, "epoch": 0.4397905759162304, "fcm_dpo/beta": 0.015006419271230698, "fcm_dpo/delta": -0.031001610681414604, "fcm_dpo/margin": 24.615867614746094, "fcm_dpo/q_t": 0.4150693118572235, "grad_norm": 37.6225471496582, "learning_rate": 3.454593922550693e-07, "logits/chosen": 1.5236557722091675, "logits/rejected": 1.564916729927063, "logps/chosen": -277.7059020996094, "logps/ref_chosen": -274.9681701660156, "logps/ref_rejected": -283.346923828125, "logps/rejected": -310.7005615234375, "loss": 4.5562, "margin_dpo/margin_mean": 24.61586570739746, "margin_dpo/margin_std": 39.228519439697266, "step": 210 }, { "KL/chosen_KL_mean": -0.10589599609375, "KL/mean": -16.420366287231445, "KL/rejected_KL_mean": -32.73482131958008, "KL/std": 33.75396728515625, "epoch": 0.4418848167539267, "fcm_dpo/beta": 0.014337250031530857, "fcm_dpo/delta": -0.10005457699298859, "fcm_dpo/margin": 32.6289176940918, "fcm_dpo/q_t": 0.39139655232429504, "grad_norm": 36.14860534667969, "learning_rate": 3.4376480090239047e-07, "logits/chosen": 1.4577884674072266, "logits/rejected": 1.4486171007156372, "logps/chosen": -266.385498046875, "logps/ref_chosen": -266.2795715332031, "logps/ref_rejected": -249.7120819091797, "logps/rejected": -282.4468994140625, "loss": 4.1621, "margin_dpo/margin_mean": 32.6289176940918, "margin_dpo/margin_std": 35.4855842590332, "step": 211 }, { "KL/chosen_KL_mean": -3.3957462310791016, "KL/mean": -15.396642684936523, "KL/rejected_KL_mean": -27.397533416748047, "KL/std": 33.133121490478516, "epoch": 0.44397905759162304, "fcm_dpo/beta": 0.013785447925329208, "fcm_dpo/delta": 0.005094792228192091, "fcm_dpo/margin": 24.001785278320312, "fcm_dpo/q_t": 0.42262667417526245, "grad_norm": 38.02641677856445, "learning_rate": 3.4206518122800055e-07, "logits/chosen": 1.4611730575561523, "logits/rejected": 1.5041024684906006, "logps/chosen": -268.1485595703125, "logps/ref_chosen": -264.7528076171875, "logps/ref_rejected": -241.02842712402344, "logps/rejected": -268.42596435546875, "loss": 4.6317, "margin_dpo/margin_mean": 24.001785278320312, "margin_dpo/margin_std": 39.277198791503906, "step": 212 }, { "KL/chosen_KL_mean": -7.741588592529297, "KL/mean": -18.942157745361328, "KL/rejected_KL_mean": -30.142728805541992, "KL/std": 33.720516204833984, "epoch": 0.44607329842931936, "fcm_dpo/beta": 0.014078613370656967, "fcm_dpo/delta": 0.019538499414920807, "fcm_dpo/margin": 22.401147842407227, "fcm_dpo/q_t": 0.4286879301071167, "grad_norm": 31.262754440307617, "learning_rate": 3.403606243773448e-07, "logits/chosen": 1.4571382999420166, "logits/rejected": 1.5783281326293945, "logps/chosen": -301.88018798828125, "logps/ref_chosen": -294.13861083984375, "logps/ref_rejected": -303.11669921875, "logps/rejected": -333.2593994140625, "loss": 4.697, "margin_dpo/margin_mean": 22.40114974975586, "margin_dpo/margin_std": 39.48596954345703, "step": 213 }, { "KL/chosen_KL_mean": -17.921554565429688, "KL/mean": -28.23214340209961, "KL/rejected_KL_mean": -38.54274368286133, "KL/std": 32.626007080078125, "epoch": 0.4481675392670157, "fcm_dpo/beta": 0.014258643612265587, "fcm_dpo/delta": 0.014283804222941399, "fcm_dpo/margin": 20.62118148803711, "fcm_dpo/q_t": 0.4328727722167969, "grad_norm": 50.17946243286133, "learning_rate": 3.3865122176063385e-07, "logits/chosen": 1.553182601928711, "logits/rejected": 1.7039114236831665, "logps/chosen": -284.4983215332031, "logps/ref_chosen": -266.5767517089844, "logps/ref_rejected": -291.2264709472656, "logps/rejected": -329.76922607421875, "loss": 4.7629, "margin_dpo/margin_mean": 20.62118148803711, "margin_dpo/margin_std": 38.329471588134766, "step": 214 }, { "KL/chosen_KL_mean": -11.867902755737305, "KL/mean": -23.135061264038086, "KL/rejected_KL_mean": -34.4022216796875, "KL/std": 36.483604431152344, "epoch": 0.450261780104712, "fcm_dpo/beta": 0.014291242696344852, "fcm_dpo/delta": 0.011042824946343899, "fcm_dpo/margin": 22.534318923950195, "fcm_dpo/q_t": 0.4279981255531311, "grad_norm": 32.39078903198242, "learning_rate": 3.3693706504794243e-07, "logits/chosen": 1.4903463125228882, "logits/rejected": 1.4890713691711426, "logps/chosen": -296.9086608886719, "logps/ref_chosen": -285.040771484375, "logps/ref_rejected": -258.0254821777344, "logps/rejected": -292.42767333984375, "loss": 4.7469, "margin_dpo/margin_mean": 22.534318923950195, "margin_dpo/margin_std": 43.56327819824219, "step": 215 }, { "KL/chosen_KL_mean": -11.920280456542969, "KL/mean": -24.491779327392578, "KL/rejected_KL_mean": -37.06327438354492, "KL/std": 37.01415252685547, "epoch": 0.4523560209424084, "fcm_dpo/beta": 0.01426931843161583, "fcm_dpo/delta": -0.02282012067735195, "fcm_dpo/margin": 25.143007278442383, "fcm_dpo/q_t": 0.4173295199871063, "grad_norm": 62.473167419433594, "learning_rate": 3.3521824616429284e-07, "logits/chosen": 1.5284839868545532, "logits/rejected": 1.5369490385055542, "logps/chosen": -344.6716613769531, "logps/ref_chosen": -332.7513732910156, "logps/ref_rejected": -302.9167785644531, "logps/rejected": -339.98004150390625, "loss": 4.6568, "margin_dpo/margin_mean": 25.143007278442383, "margin_dpo/margin_std": 44.69744873046875, "step": 216 }, { "KL/chosen_KL_mean": -12.519142150878906, "KL/mean": -29.5145263671875, "KL/rejected_KL_mean": -46.50989532470703, "KL/std": 36.492759704589844, "epoch": 0.4544502617801047, "fcm_dpo/beta": 0.013744605705142021, "fcm_dpo/delta": -0.07050801068544388, "fcm_dpo/margin": 33.990753173828125, "fcm_dpo/q_t": 0.3944663405418396, "grad_norm": 28.30463981628418, "learning_rate": 3.334948572847253e-07, "logits/chosen": 1.7275793552398682, "logits/rejected": 1.7330214977264404, "logps/chosen": -298.9870910644531, "logps/ref_chosen": -286.46795654296875, "logps/ref_rejected": -253.99298095703125, "logps/rejected": -300.50286865234375, "loss": 4.258, "margin_dpo/margin_mean": 33.990753173828125, "margin_dpo/margin_std": 44.993526458740234, "step": 217 }, { "KL/chosen_KL_mean": -9.264612197875977, "KL/mean": -22.980953216552734, "KL/rejected_KL_mean": -36.697288513183594, "KL/std": 36.396488189697266, "epoch": 0.45654450261780105, "fcm_dpo/beta": 0.013674240559339523, "fcm_dpo/delta": 0.025067811831831932, "fcm_dpo/margin": 27.43267059326172, "fcm_dpo/q_t": 0.4136398434638977, "grad_norm": 41.536502838134766, "learning_rate": 3.317669908293554e-07, "logits/chosen": 1.6624709367752075, "logits/rejected": 1.7311898469924927, "logps/chosen": -320.64312744140625, "logps/ref_chosen": -311.3785400390625, "logps/ref_rejected": -290.3092956542969, "logps/rejected": -327.006591796875, "loss": 4.4824, "margin_dpo/margin_mean": 27.432668685913086, "margin_dpo/margin_std": 40.15641784667969, "step": 218 }, { "KL/chosen_KL_mean": -9.294855117797852, "KL/mean": -23.00322723388672, "KL/rejected_KL_mean": -36.71159362792969, "KL/std": 34.37981414794922, "epoch": 0.4586387434554974, "fcm_dpo/beta": 0.013447243720293045, "fcm_dpo/delta": -0.02139398269355297, "fcm_dpo/margin": 27.41674041748047, "fcm_dpo/q_t": 0.41440269351005554, "grad_norm": 38.7296257019043, "learning_rate": 3.300347394584172e-07, "logits/chosen": 1.2187390327453613, "logits/rejected": 1.2701640129089355, "logps/chosen": -268.6277770996094, "logps/ref_chosen": -259.33294677734375, "logps/ref_rejected": -261.7661437988281, "logps/rejected": -298.47772216796875, "loss": 4.5068, "margin_dpo/margin_mean": 27.4167423248291, "margin_dpo/margin_std": 40.53258514404297, "step": 219 }, { "KL/chosen_KL_mean": -10.534852981567383, "KL/mean": -24.904674530029297, "KL/rejected_KL_mean": -39.274497985839844, "KL/std": 36.653446197509766, "epoch": 0.4607329842931937, "fcm_dpo/beta": 0.013439134694635868, "fcm_dpo/delta": 0.014226877130568027, "fcm_dpo/margin": 28.73965835571289, "fcm_dpo/q_t": 0.41149696707725525, "grad_norm": 49.266822814941406, "learning_rate": 3.2829819606729477e-07, "logits/chosen": 1.702681303024292, "logits/rejected": 1.652176022529602, "logps/chosen": -316.7392578125, "logps/ref_chosen": -306.2043762207031, "logps/ref_rejected": -256.7926330566406, "logps/rejected": -296.067138671875, "loss": 4.488, "margin_dpo/margin_mean": 28.73965835571289, "margin_dpo/margin_std": 44.04676818847656, "step": 220 }, { "KL/chosen_KL_mean": -11.413686752319336, "KL/mean": -22.628131866455078, "KL/rejected_KL_mean": -33.842586517333984, "KL/std": 36.744773864746094, "epoch": 0.46282722513089003, "fcm_dpo/beta": 0.01356799341738224, "fcm_dpo/delta": -0.017365112900733948, "fcm_dpo/margin": 22.428897857666016, "fcm_dpo/q_t": 0.4291653335094452, "grad_norm": 30.916912078857422, "learning_rate": 3.265574537815398e-07, "logits/chosen": 1.1171040534973145, "logits/rejected": 1.2264795303344727, "logps/chosen": -304.7456359863281, "logps/ref_chosen": -293.3319396972656, "logps/ref_rejected": -282.7599182128906, "logps/rejected": -316.60247802734375, "loss": 4.7429, "margin_dpo/margin_mean": 22.428895950317383, "margin_dpo/margin_std": 41.00771713256836, "step": 221 }, { "KL/chosen_KL_mean": -7.823530197143555, "KL/mean": -22.639671325683594, "KL/rejected_KL_mean": -37.455810546875, "KL/std": 35.31937026977539, "epoch": 0.4649214659685864, "fcm_dpo/beta": 0.01314732525497675, "fcm_dpo/delta": -0.012120349332690239, "fcm_dpo/margin": 29.632274627685547, "fcm_dpo/q_t": 0.4094116687774658, "grad_norm": 40.30009078979492, "learning_rate": 3.248126059518784e-07, "logits/chosen": 1.564622402191162, "logits/rejected": 1.480457067489624, "logps/chosen": -294.8296813964844, "logps/ref_chosen": -287.0061340332031, "logps/ref_rejected": -233.23130798339844, "logps/rejected": -270.6871337890625, "loss": 4.4124, "margin_dpo/margin_mean": 29.63227653503418, "margin_dpo/margin_std": 39.13923645019531, "step": 222 }, { "KL/chosen_KL_mean": -8.294769287109375, "KL/mean": -24.715295791625977, "KL/rejected_KL_mean": -41.13582992553711, "KL/std": 38.33906936645508, "epoch": 0.46701570680628274, "fcm_dpo/beta": 0.013019093312323093, "fcm_dpo/delta": -0.029901059344410896, "fcm_dpo/margin": 32.841064453125, "fcm_dpo/q_t": 0.40152767300605774, "grad_norm": 29.832014083862305, "learning_rate": 3.230637461492043e-07, "logits/chosen": 1.3996307849884033, "logits/rejected": 1.2953250408172607, "logps/chosen": -284.1100769042969, "logps/ref_chosen": -275.8153076171875, "logps/ref_rejected": -238.54286193847656, "logps/rejected": -279.6786804199219, "loss": 4.3452, "margin_dpo/margin_mean": 32.841064453125, "margin_dpo/margin_std": 43.7483024597168, "step": 223 }, { "KL/chosen_KL_mean": -9.125528335571289, "KL/mean": -25.0452823638916, "KL/rejected_KL_mean": -40.96503448486328, "KL/std": 37.11698532104492, "epoch": 0.46910994764397906, "fcm_dpo/beta": 0.012906144373118877, "fcm_dpo/delta": -0.042829155921936035, "fcm_dpo/margin": 31.839508056640625, "fcm_dpo/q_t": 0.404636412858963, "grad_norm": 44.4716682434082, "learning_rate": 3.213109681595612e-07, "logits/chosen": 1.397386908531189, "logits/rejected": 1.4601026773452759, "logps/chosen": -248.22592163085938, "logps/ref_chosen": -239.1003875732422, "logps/ref_rejected": -230.31900024414062, "logps/rejected": -271.2840270996094, "loss": 4.36, "margin_dpo/margin_mean": 31.839508056640625, "margin_dpo/margin_std": 40.82142639160156, "step": 224 }, { "KL/chosen_KL_mean": -15.982580184936523, "KL/mean": -27.515350341796875, "KL/rejected_KL_mean": -39.04811477661133, "KL/std": 38.05552673339844, "epoch": 0.4712041884816754, "fcm_dpo/beta": 0.012642517685890198, "fcm_dpo/delta": 0.04203230142593384, "fcm_dpo/margin": 23.065526962280273, "fcm_dpo/q_t": 0.4347875416278839, "grad_norm": 31.074045181274414, "learning_rate": 3.1955436597911315e-07, "logits/chosen": 1.4961891174316406, "logits/rejected": 1.5179147720336914, "logps/chosen": -316.31195068359375, "logps/ref_chosen": -300.3293762207031, "logps/ref_rejected": -332.132568359375, "logps/rejected": -371.1806640625, "loss": 4.8525, "margin_dpo/margin_mean": 23.065526962280273, "margin_dpo/margin_std": 49.160301208496094, "step": 225 }, { "KL/chosen_KL_mean": -11.568771362304688, "KL/mean": -22.770845413208008, "KL/rejected_KL_mean": -33.97290802001953, "KL/std": 36.24320602416992, "epoch": 0.4732984293193717, "fcm_dpo/beta": 0.013132845051586628, "fcm_dpo/delta": 0.046728260815143585, "fcm_dpo/margin": 22.404129028320312, "fcm_dpo/q_t": 0.43278542160987854, "grad_norm": 34.16708755493164, "learning_rate": 3.1779403380910425e-07, "logits/chosen": 1.3392904996871948, "logits/rejected": 1.3999265432357788, "logps/chosen": -256.98095703125, "logps/ref_chosen": -245.41213989257812, "logps/ref_rejected": -251.72999572753906, "logps/rejected": -285.7029113769531, "loss": 4.78, "margin_dpo/margin_mean": 22.404129028320312, "margin_dpo/margin_std": 43.30206298828125, "step": 226 }, { "KL/chosen_KL_mean": -10.773397445678711, "KL/mean": -26.87171173095703, "KL/rejected_KL_mean": -42.97003173828125, "KL/std": 38.47583770751953, "epoch": 0.47539267015706804, "fcm_dpo/beta": 0.013252967968583107, "fcm_dpo/delta": -0.028668414801359177, "fcm_dpo/margin": 32.19664001464844, "fcm_dpo/q_t": 0.40182405710220337, "grad_norm": 31.617626190185547, "learning_rate": 3.160300660508064e-07, "logits/chosen": 1.4435663223266602, "logits/rejected": 1.4892001152038574, "logps/chosen": -287.6492004394531, "logps/ref_chosen": -276.87579345703125, "logps/ref_rejected": -259.0455017089844, "logps/rejected": -302.0155334472656, "loss": 4.369, "margin_dpo/margin_mean": 32.19664001464844, "margin_dpo/margin_std": 44.42658615112305, "step": 227 }, { "KL/chosen_KL_mean": -8.808744430541992, "KL/mean": -25.228464126586914, "KL/rejected_KL_mean": -41.648189544677734, "KL/std": 37.78360366821289, "epoch": 0.4774869109947644, "fcm_dpo/beta": 0.013077094219624996, "fcm_dpo/delta": -0.03130736202001572, "fcm_dpo/margin": 32.83942794799805, "fcm_dpo/q_t": 0.4017367660999298, "grad_norm": 32.4455680847168, "learning_rate": 3.1426255730045695e-07, "logits/chosen": 1.5388902425765991, "logits/rejected": 1.5339179039001465, "logps/chosen": -312.77978515625, "logps/ref_chosen": -303.9710693359375, "logps/ref_rejected": -255.62033081054688, "logps/rejected": -297.26849365234375, "loss": 4.3349, "margin_dpo/margin_mean": 32.83942794799805, "margin_dpo/margin_std": 43.40449523925781, "step": 228 }, { "KL/chosen_KL_mean": -8.927270889282227, "KL/mean": -29.90787696838379, "KL/rejected_KL_mean": -50.888465881347656, "KL/std": 39.940574645996094, "epoch": 0.47958115183246075, "fcm_dpo/beta": 0.012126508168876171, "fcm_dpo/delta": -0.11578944325447083, "fcm_dpo/margin": 41.961185455322266, "fcm_dpo/q_t": 0.38210585713386536, "grad_norm": 28.49382209777832, "learning_rate": 3.1249160234418644e-07, "logits/chosen": 1.2846533060073853, "logits/rejected": 1.2547979354858398, "logps/chosen": -290.6231994628906, "logps/ref_chosen": -281.6959533691406, "logps/ref_rejected": -258.93353271484375, "logps/rejected": -309.822021484375, "loss": 4.0764, "margin_dpo/margin_mean": 41.96118927001953, "margin_dpo/margin_std": 45.12513732910156, "step": 229 }, { "KL/chosen_KL_mean": -9.807109832763672, "KL/mean": -26.030256271362305, "KL/rejected_KL_mean": -42.253395080566406, "KL/std": 38.47943878173828, "epoch": 0.4816753926701571, "fcm_dpo/beta": 0.011823546141386032, "fcm_dpo/delta": 0.016186799854040146, "fcm_dpo/margin": 32.4462776184082, "fcm_dpo/q_t": 0.4109889566898346, "grad_norm": 45.17549133300781, "learning_rate": 3.1071729615293424e-07, "logits/chosen": 1.3623743057250977, "logits/rejected": 1.4585875272750854, "logps/chosen": -233.76364135742188, "logps/ref_chosen": -223.95655822753906, "logps/ref_rejected": -238.95187377929688, "logps/rejected": -281.20526123046875, "loss": 4.4637, "margin_dpo/margin_mean": 32.44627380371094, "margin_dpo/margin_std": 46.24778747558594, "step": 230 }, { "KL/chosen_KL_mean": -15.893941879272461, "KL/mean": -28.720094680786133, "KL/rejected_KL_mean": -41.54625701904297, "KL/std": 38.19879150390625, "epoch": 0.4837696335078534, "fcm_dpo/beta": 0.011899704113602638, "fcm_dpo/delta": 0.015257255174219608, "fcm_dpo/margin": 25.652320861816406, "fcm_dpo/q_t": 0.4285806715488434, "grad_norm": 38.40047073364258, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 1.301490306854248, "logits/rejected": 1.358456015586853, "logps/chosen": -330.81854248046875, "logps/ref_chosen": -314.92462158203125, "logps/ref_rejected": -279.6347351074219, "logps/rejected": -321.1809997558594, "loss": 4.6728, "margin_dpo/margin_mean": 25.652320861816406, "margin_dpo/margin_std": 42.17586898803711, "step": 231 }, { "KL/chosen_KL_mean": -8.095703125, "KL/mean": -23.29351806640625, "KL/rejected_KL_mean": -38.491336822509766, "KL/std": 40.04689025878906, "epoch": 0.48586387434554973, "fcm_dpo/beta": 0.01189206913113594, "fcm_dpo/delta": -0.0057932473719120026, "fcm_dpo/margin": 30.395639419555664, "fcm_dpo/q_t": 0.41665101051330566, "grad_norm": 28.25246810913086, "learning_rate": 3.071590108427243e-07, "logits/chosen": 1.3400001525878906, "logits/rejected": 1.3999887704849243, "logps/chosen": -271.4153137207031, "logps/ref_chosen": -263.3196105957031, "logps/ref_rejected": -247.13612365722656, "logps/rejected": -285.62744140625, "loss": 4.5218, "margin_dpo/margin_mean": 30.395639419555664, "margin_dpo/margin_std": 44.760887145996094, "step": 232 }, { "KL/chosen_KL_mean": -7.75877571105957, "KL/mean": -23.7706356048584, "KL/rejected_KL_mean": -39.78249740600586, "KL/std": 39.010414123535156, "epoch": 0.48795811518324606, "fcm_dpo/beta": 0.011896791867911816, "fcm_dpo/delta": -0.004583900794386864, "fcm_dpo/margin": 32.023719787597656, "fcm_dpo/q_t": 0.41239315271377563, "grad_norm": 34.30588150024414, "learning_rate": 3.05375222543809e-07, "logits/chosen": 1.3542277812957764, "logits/rejected": 1.4231150150299072, "logps/chosen": -287.1209716796875, "logps/ref_chosen": -279.36224365234375, "logps/ref_rejected": -264.46112060546875, "logps/rejected": -304.24365234375, "loss": 4.4553, "margin_dpo/margin_mean": 32.023719787597656, "margin_dpo/margin_std": 45.20970153808594, "step": 233 }, { "KL/chosen_KL_mean": -7.031229019165039, "KL/mean": -23.30236053466797, "KL/rejected_KL_mean": -39.5734977722168, "KL/std": 41.138160705566406, "epoch": 0.4900523560209424, "fcm_dpo/beta": 0.011804355308413506, "fcm_dpo/delta": -0.015027280896902084, "fcm_dpo/margin": 32.54227066040039, "fcm_dpo/q_t": 0.41214871406555176, "grad_norm": 32.93385314941406, "learning_rate": 3.035884646397637e-07, "logits/chosen": 1.3371347188949585, "logits/rejected": 1.4137632846832275, "logps/chosen": -293.4803466796875, "logps/ref_chosen": -286.4491271972656, "logps/ref_rejected": -294.5469665527344, "logps/rejected": -334.1204833984375, "loss": 4.5281, "margin_dpo/margin_mean": 32.54227066040039, "margin_dpo/margin_std": 51.168392181396484, "step": 234 }, { "KL/chosen_KL_mean": -8.793224334716797, "KL/mean": -23.923728942871094, "KL/rejected_KL_mean": -39.054237365722656, "KL/std": 42.044288635253906, "epoch": 0.49214659685863876, "fcm_dpo/beta": 0.011966248974204063, "fcm_dpo/delta": 0.005010334774851799, "fcm_dpo/margin": 30.261001586914062, "fcm_dpo/q_t": 0.4166721701622009, "grad_norm": 55.811973571777344, "learning_rate": 3.017988329489923e-07, "logits/chosen": 1.4549720287322998, "logits/rejected": 1.4591089487075806, "logps/chosen": -291.1532897949219, "logps/ref_chosen": -282.36004638671875, "logps/ref_rejected": -243.79052734375, "logps/rejected": -282.8447570800781, "loss": 4.5759, "margin_dpo/margin_mean": 30.261001586914062, "margin_dpo/margin_std": 49.17083740234375, "step": 235 }, { "KL/chosen_KL_mean": -2.9586658477783203, "KL/mean": -20.14302635192871, "KL/rejected_KL_mean": -37.327392578125, "KL/std": 39.80377197265625, "epoch": 0.4942408376963351, "fcm_dpo/beta": 0.011891270987689495, "fcm_dpo/delta": -0.009616704657673836, "fcm_dpo/margin": 34.368717193603516, "fcm_dpo/q_t": 0.40520286560058594, "grad_norm": 30.121191024780273, "learning_rate": 3.000064234440111e-07, "logits/chosen": 1.392760992050171, "logits/rejected": 1.4052550792694092, "logps/chosen": -282.09808349609375, "logps/ref_chosen": -279.139404296875, "logps/ref_rejected": -240.5850067138672, "logps/rejected": -277.9123840332031, "loss": 4.3816, "margin_dpo/margin_mean": 34.368717193603516, "margin_dpo/margin_std": 46.1241340637207, "step": 236 }, { "KL/chosen_KL_mean": -5.928140640258789, "KL/mean": -20.729507446289062, "KL/rejected_KL_mean": -35.530879974365234, "KL/std": 36.886966705322266, "epoch": 0.4963350785340314, "fcm_dpo/beta": 0.011840139515697956, "fcm_dpo/delta": 0.009517230093479156, "fcm_dpo/margin": 29.60274314880371, "fcm_dpo/q_t": 0.41943880915641785, "grad_norm": 30.60716438293457, "learning_rate": 2.9821133224630223e-07, "logits/chosen": 1.3946075439453125, "logits/rejected": 1.4704986810684204, "logps/chosen": -264.7396545410156, "logps/ref_chosen": -258.8114929199219, "logps/ref_rejected": -266.3898010253906, "logps/rejected": -301.920654296875, "loss": 4.5759, "margin_dpo/margin_mean": 29.60274314880371, "margin_dpo/margin_std": 46.73874282836914, "step": 237 }, { "KL/chosen_KL_mean": -6.701227188110352, "KL/mean": -23.007665634155273, "KL/rejected_KL_mean": -39.31410598754883, "KL/std": 41.285335540771484, "epoch": 0.49842931937172774, "fcm_dpo/beta": 0.011498241685330868, "fcm_dpo/delta": -0.030815942212939262, "fcm_dpo/margin": 32.61288070678711, "fcm_dpo/q_t": 0.4142690896987915, "grad_norm": 40.07395935058594, "learning_rate": 2.964136556211588e-07, "logits/chosen": 1.3910374641418457, "logits/rejected": 1.3155585527420044, "logps/chosen": -308.76739501953125, "logps/ref_chosen": -302.0661926269531, "logps/ref_rejected": -265.5518798828125, "logps/rejected": -304.865966796875, "loss": 4.495, "margin_dpo/margin_mean": 32.61288070678711, "margin_dpo/margin_std": 47.61798095703125, "step": 238 }, { "KL/chosen_KL_mean": -3.102130889892578, "KL/mean": -17.093172073364258, "KL/rejected_KL_mean": -31.084213256835938, "KL/std": 40.21778869628906, "epoch": 0.5005235602094241, "fcm_dpo/beta": 0.011761010624468327, "fcm_dpo/delta": 0.07378143817186356, "fcm_dpo/margin": 27.982086181640625, "fcm_dpo/q_t": 0.42560938000679016, "grad_norm": 30.5093994140625, "learning_rate": 2.946134899725226e-07, "logits/chosen": 1.6279447078704834, "logits/rejected": 1.7356630563735962, "logps/chosen": -262.1004943847656, "logps/ref_chosen": -258.99835205078125, "logps/ref_rejected": -273.7834777832031, "logps/rejected": -304.86767578125, "loss": 4.6858, "margin_dpo/margin_mean": 27.982086181640625, "margin_dpo/margin_std": 50.87156295776367, "step": 239 }, { "KL/chosen_KL_mean": -0.7970199584960938, "KL/mean": -18.575305938720703, "KL/rejected_KL_mean": -36.353599548339844, "KL/std": 39.7580680847168, "epoch": 0.5026178010471204, "fcm_dpo/beta": 0.011806718073785305, "fcm_dpo/delta": -0.05412173643708229, "fcm_dpo/margin": 35.556583404541016, "fcm_dpo/q_t": 0.40297773480415344, "grad_norm": 44.02696990966797, "learning_rate": 2.9281093183781403e-07, "logits/chosen": 1.1961954832077026, "logits/rejected": 1.2669832706451416, "logps/chosen": -276.320068359375, "logps/ref_chosen": -275.5230407714844, "logps/ref_rejected": -255.50784301757812, "logps/rejected": -291.8614196777344, "loss": 4.3285, "margin_dpo/margin_mean": 35.556583404541016, "margin_dpo/margin_std": 45.55504608154297, "step": 240 }, { "KL/chosen_KL_mean": -6.538116455078125, "KL/mean": -19.914806365966797, "KL/rejected_KL_mean": -33.29149627685547, "KL/std": 40.47013854980469, "epoch": 0.5047120418848168, "fcm_dpo/beta": 0.011729340068995953, "fcm_dpo/delta": 0.025532079860568047, "fcm_dpo/margin": 26.753385543823242, "fcm_dpo/q_t": 0.42871057987213135, "grad_norm": 31.722482681274414, "learning_rate": 2.910060778827554e-07, "logits/chosen": 1.733362078666687, "logits/rejected": 1.7891788482666016, "logps/chosen": -259.901611328125, "logps/ref_chosen": -253.36349487304688, "logps/ref_rejected": -247.1916961669922, "logps/rejected": -280.4831848144531, "loss": 4.7253, "margin_dpo/margin_mean": 26.753385543823242, "margin_dpo/margin_std": 49.15824890136719, "step": 241 }, { "KL/chosen_KL_mean": -3.924562454223633, "KL/mean": -20.041032791137695, "KL/rejected_KL_mean": -36.15748977661133, "KL/std": 39.7971305847168, "epoch": 0.506806282722513, "fcm_dpo/beta": 0.011834909208118916, "fcm_dpo/delta": -0.010150094516575336, "fcm_dpo/margin": 32.23292922973633, "fcm_dpo/q_t": 0.4113154411315918, "grad_norm": 33.408729553222656, "learning_rate": 2.891990248961871e-07, "logits/chosen": 1.470890760421753, "logits/rejected": 1.4131429195404053, "logps/chosen": -268.9126281738281, "logps/ref_chosen": -264.9880676269531, "logps/ref_rejected": -243.07838439941406, "logps/rejected": -279.2358703613281, "loss": 4.4475, "margin_dpo/margin_mean": 32.23292541503906, "margin_dpo/margin_std": 44.627681732177734, "step": 242 }, { "KL/chosen_KL_mean": 0.30600738525390625, "KL/mean": -18.52072525024414, "KL/rejected_KL_mean": -37.34746551513672, "KL/std": 39.642032623291016, "epoch": 0.5089005235602094, "fcm_dpo/beta": 0.011598140932619572, "fcm_dpo/delta": -0.0700819194316864, "fcm_dpo/margin": 37.65347671508789, "fcm_dpo/q_t": 0.3989962637424469, "grad_norm": 53.24457931518555, "learning_rate": 2.873898697848762e-07, "logits/chosen": 1.5711965560913086, "logits/rejected": 1.6385666131973267, "logps/chosen": -315.5646667480469, "logps/ref_chosen": -315.8706970214844, "logps/ref_rejected": -304.3729248046875, "logps/rejected": -341.72039794921875, "loss": 4.2608, "margin_dpo/margin_mean": 37.65347671508789, "margin_dpo/margin_std": 42.59384536743164, "step": 243 }, { "KL/chosen_KL_mean": -3.296337127685547, "KL/mean": -22.78225326538086, "KL/rejected_KL_mean": -42.26817321777344, "KL/std": 42.10606384277344, "epoch": 0.5109947643979058, "fcm_dpo/beta": 0.01082875207066536, "fcm_dpo/delta": -0.05993076041340828, "fcm_dpo/margin": 38.971832275390625, "fcm_dpo/q_t": 0.401917040348053, "grad_norm": 27.999408721923828, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 1.5478038787841797, "logits/rejected": 1.5114996433258057, "logps/chosen": -314.39349365234375, "logps/ref_chosen": -311.09716796875, "logps/ref_rejected": -266.01959228515625, "logps/rejected": -308.2877502441406, "loss": 4.318, "margin_dpo/margin_mean": 38.97182846069336, "margin_dpo/margin_std": 48.343570709228516, "step": 244 }, { "KL/chosen_KL_mean": -3.196439743041992, "KL/mean": -22.358665466308594, "KL/rejected_KL_mean": -41.520896911621094, "KL/std": 41.44492721557617, "epoch": 0.5130890052356021, "fcm_dpo/beta": 0.010494336485862732, "fcm_dpo/delta": -0.050034236162900925, "fcm_dpo/margin": 38.3244743347168, "fcm_dpo/q_t": 0.4060443937778473, "grad_norm": 41.397159576416016, "learning_rate": 2.837656413735479e-07, "logits/chosen": 1.4871952533721924, "logits/rejected": 1.3748258352279663, "logps/chosen": -287.9369201660156, "logps/ref_chosen": -284.740478515625, "logps/ref_rejected": -237.64117431640625, "logps/rejected": -279.1620788574219, "loss": 4.3706, "margin_dpo/margin_mean": 38.3244743347168, "margin_dpo/margin_std": 47.9886474609375, "step": 245 }, { "KL/chosen_KL_mean": -12.974367141723633, "KL/mean": -25.873023986816406, "KL/rejected_KL_mean": -38.77168273925781, "KL/std": 40.77077865600586, "epoch": 0.5151832460732985, "fcm_dpo/beta": 0.01014913059771061, "fcm_dpo/delta": -0.009277934208512306, "fcm_dpo/margin": 25.79730987548828, "fcm_dpo/q_t": 0.43964654207229614, "grad_norm": 43.22949981689453, "learning_rate": 2.8195076242990116e-07, "logits/chosen": 1.5551692247390747, "logits/rejected": 1.5850450992584229, "logps/chosen": -285.5872497558594, "logps/ref_chosen": -272.6128845214844, "logps/ref_rejected": -249.99725341796875, "logps/rejected": -288.7689514160156, "loss": 4.845, "margin_dpo/margin_mean": 25.79730987548828, "margin_dpo/margin_std": 50.12140655517578, "step": 246 }, { "KL/chosen_KL_mean": -5.976814270019531, "KL/mean": -22.756752014160156, "KL/rejected_KL_mean": -39.53668975830078, "KL/std": 41.12067413330078, "epoch": 0.5172774869109947, "fcm_dpo/beta": 0.01021606009453535, "fcm_dpo/delta": 0.005496193654835224, "fcm_dpo/margin": 33.559879302978516, "fcm_dpo/q_t": 0.4197867214679718, "grad_norm": 30.316707611083984, "learning_rate": 2.801341700638307e-07, "logits/chosen": 1.4950445890426636, "logits/rejected": 1.507032871246338, "logps/chosen": -280.4126281738281, "logps/ref_chosen": -274.435791015625, "logps/ref_rejected": -257.8125305175781, "logps/rejected": -297.3492126464844, "loss": 4.5279, "margin_dpo/margin_mean": 33.55988311767578, "margin_dpo/margin_std": 47.255096435546875, "step": 247 }, { "KL/chosen_KL_mean": -10.627542495727539, "KL/mean": -24.254194259643555, "KL/rejected_KL_mean": -37.88084411621094, "KL/std": 40.86634826660156, "epoch": 0.5193717277486911, "fcm_dpo/beta": 0.010291523300111294, "fcm_dpo/delta": 0.038031596690416336, "fcm_dpo/margin": 27.25330352783203, "fcm_dpo/q_t": 0.4346015453338623, "grad_norm": 38.1819953918457, "learning_rate": 2.7831596169367227e-07, "logits/chosen": 1.2884039878845215, "logits/rejected": 1.3203765153884888, "logps/chosen": -271.9388732910156, "logps/ref_chosen": -261.3113098144531, "logps/ref_rejected": -231.152587890625, "logps/rejected": -269.033447265625, "loss": 4.7694, "margin_dpo/margin_mean": 27.25330352783203, "margin_dpo/margin_std": 49.18940353393555, "step": 248 }, { "KL/chosen_KL_mean": -12.927343368530273, "KL/mean": -29.005685806274414, "KL/rejected_KL_mean": -45.08403015136719, "KL/std": 43.595123291015625, "epoch": 0.5214659685863874, "fcm_dpo/beta": 0.010600361973047256, "fcm_dpo/delta": 0.020274803042411804, "fcm_dpo/margin": 32.15668869018555, "fcm_dpo/q_t": 0.4210623502731323, "grad_norm": 40.379180908203125, "learning_rate": 2.7649623482442274e-07, "logits/chosen": 1.4619112014770508, "logits/rejected": 1.4200809001922607, "logps/chosen": -281.39459228515625, "logps/ref_chosen": -268.46722412109375, "logps/ref_rejected": -240.27597045898438, "logps/rejected": -285.3599853515625, "loss": 4.5757, "margin_dpo/margin_mean": 32.15668487548828, "margin_dpo/margin_std": 49.13870620727539, "step": 249 }, { "KL/chosen_KL_mean": -13.616073608398438, "KL/mean": -29.948217391967773, "KL/rejected_KL_mean": -46.280357360839844, "KL/std": 44.8901481628418, "epoch": 0.5235602094240838, "fcm_dpo/beta": 0.010583215393126011, "fcm_dpo/delta": -0.0072011156007647514, "fcm_dpo/margin": 32.6642951965332, "fcm_dpo/q_t": 0.4200817942619324, "grad_norm": 29.176288604736328, "learning_rate": 2.7467508704251135e-07, "logits/chosen": 1.3896605968475342, "logits/rejected": 1.4943935871124268, "logps/chosen": -298.7212829589844, "logps/ref_chosen": -285.1051940917969, "logps/ref_rejected": -281.2331848144531, "logps/rejected": -327.5135498046875, "loss": 4.6045, "margin_dpo/margin_mean": 32.6642951965332, "margin_dpo/margin_std": 52.9317512512207, "step": 250 }, { "KL/chosen_KL_mean": -10.912569046020508, "KL/mean": -27.458873748779297, "KL/rejected_KL_mean": -44.00518035888672, "KL/std": 42.60402297973633, "epoch": 0.5256544502617801, "fcm_dpo/beta": 0.010662700049579144, "fcm_dpo/delta": -0.015465203672647476, "fcm_dpo/margin": 33.09260559082031, "fcm_dpo/q_t": 0.4187396764755249, "grad_norm": 41.07495880126953, "learning_rate": 2.7285261601056697e-07, "logits/chosen": 1.4050275087356567, "logits/rejected": 1.3634721040725708, "logps/chosen": -285.02520751953125, "logps/ref_chosen": -274.1126403808594, "logps/ref_rejected": -256.7730407714844, "logps/rejected": -300.7782287597656, "loss": 4.5358, "margin_dpo/margin_mean": 33.09260940551758, "margin_dpo/margin_std": 48.139583587646484, "step": 251 }, { "KL/chosen_KL_mean": -17.89266586303711, "KL/mean": -35.21458053588867, "KL/rejected_KL_mean": -52.53649139404297, "KL/std": 44.71033477783203, "epoch": 0.5277486910994764, "fcm_dpo/beta": 0.010413442738354206, "fcm_dpo/delta": 0.01656465046107769, "fcm_dpo/margin": 34.643829345703125, "fcm_dpo/q_t": 0.4168463945388794, "grad_norm": 30.107173919677734, "learning_rate": 2.7102891946217994e-07, "logits/chosen": 1.5433881282806396, "logits/rejected": 1.514957070350647, "logps/chosen": -307.4583740234375, "logps/ref_chosen": -289.56573486328125, "logps/ref_rejected": -277.9891662597656, "logps/rejected": -330.525634765625, "loss": 4.5739, "margin_dpo/margin_mean": 34.643829345703125, "margin_dpo/margin_std": 54.72654342651367, "step": 252 }, { "KL/chosen_KL_mean": -16.55995750427246, "KL/mean": -30.391183853149414, "KL/rejected_KL_mean": -44.222415924072266, "KL/std": 44.476783752441406, "epoch": 0.5298429319371728, "fcm_dpo/beta": 0.010737106204032898, "fcm_dpo/delta": 0.025508765131235123, "fcm_dpo/margin": 27.662458419799805, "fcm_dpo/q_t": 0.43178489804267883, "grad_norm": 37.88166427612305, "learning_rate": 2.692040951966617e-07, "logits/chosen": 1.434678554534912, "logits/rejected": 1.423270583152771, "logps/chosen": -286.743896484375, "logps/ref_chosen": -270.1839294433594, "logps/ref_rejected": -245.86949157714844, "logps/rejected": -290.0918884277344, "loss": 4.7617, "margin_dpo/margin_mean": 27.662458419799805, "margin_dpo/margin_std": 51.88008499145508, "step": 253 }, { "KL/chosen_KL_mean": -12.813804626464844, "KL/mean": -30.503990173339844, "KL/rejected_KL_mean": -48.194175720214844, "KL/std": 44.81682586669922, "epoch": 0.5319371727748691, "fcm_dpo/beta": 0.011003278195858002, "fcm_dpo/delta": -0.01815183088183403, "fcm_dpo/margin": 35.380367279052734, "fcm_dpo/q_t": 0.4096793532371521, "grad_norm": 29.679153442382812, "learning_rate": 2.6737824107379947e-07, "logits/chosen": 1.3948644399642944, "logits/rejected": 1.383762001991272, "logps/chosen": -273.500244140625, "logps/ref_chosen": -260.6864318847656, "logps/ref_rejected": -245.99508666992188, "logps/rejected": -294.1892395019531, "loss": 4.4201, "margin_dpo/margin_mean": 35.38037109375, "margin_dpo/margin_std": 45.590354919433594, "step": 254 }, { "KL/chosen_KL_mean": -11.986688613891602, "KL/mean": -31.540464401245117, "KL/rejected_KL_mean": -51.09423828125, "KL/std": 45.224788665771484, "epoch": 0.5340314136125655, "fcm_dpo/beta": 0.010793380439281464, "fcm_dpo/delta": -0.023381978273391724, "fcm_dpo/margin": 39.1075553894043, "fcm_dpo/q_t": 0.40392202138900757, "grad_norm": 34.99132537841797, "learning_rate": 2.655514550086086e-07, "logits/chosen": 1.6440770626068115, "logits/rejected": 1.5638293027877808, "logps/chosen": -309.9005126953125, "logps/ref_chosen": -297.9138488769531, "logps/ref_rejected": -253.39952087402344, "logps/rejected": -304.4937744140625, "loss": 4.4037, "margin_dpo/margin_mean": 39.10755157470703, "margin_dpo/margin_std": 56.7020378112793, "step": 255 }, { "KL/chosen_KL_mean": -10.440006256103516, "KL/mean": -30.1831111907959, "KL/rejected_KL_mean": -49.92621612548828, "KL/std": 46.450164794921875, "epoch": 0.5361256544502618, "fcm_dpo/beta": 0.010454785078763962, "fcm_dpo/delta": -0.03796786814928055, "fcm_dpo/margin": 39.486209869384766, "fcm_dpo/q_t": 0.40512558817863464, "grad_norm": 36.39244842529297, "learning_rate": 2.6372383496608186e-07, "logits/chosen": 1.5219961404800415, "logits/rejected": 1.4941329956054688, "logps/chosen": -320.36639404296875, "logps/ref_chosen": -309.9263916015625, "logps/ref_rejected": -253.452392578125, "logps/rejected": -303.3786315917969, "loss": 4.4543, "margin_dpo/margin_mean": 39.486209869384766, "margin_dpo/margin_std": 58.31575393676758, "step": 256 }, { "KL/chosen_KL_mean": -14.06297492980957, "KL/mean": -31.715511322021484, "KL/rejected_KL_mean": -49.3680534362793, "KL/std": 46.08407974243164, "epoch": 0.5382198952879581, "fcm_dpo/beta": 0.010138208046555519, "fcm_dpo/delta": 0.018022824078798294, "fcm_dpo/margin": 35.30508804321289, "fcm_dpo/q_t": 0.41767239570617676, "grad_norm": 50.86438751220703, "learning_rate": 2.618954789559356e-07, "logits/chosen": 1.1277216672897339, "logits/rejected": 1.1563029289245605, "logps/chosen": -273.8766174316406, "logps/ref_chosen": -259.8136291503906, "logps/ref_rejected": -245.84947204589844, "logps/rejected": -295.217529296875, "loss": 4.5203, "margin_dpo/margin_mean": 35.30508804321289, "margin_dpo/margin_std": 51.38655471801758, "step": 257 }, { "KL/chosen_KL_mean": -12.599967956542969, "KL/mean": -32.08304977416992, "KL/rejected_KL_mean": -51.566131591796875, "KL/std": 46.551483154296875, "epoch": 0.5403141361256545, "fcm_dpo/beta": 0.01018352247774601, "fcm_dpo/delta": 0.0029114307835698128, "fcm_dpo/margin": 38.966163635253906, "fcm_dpo/q_t": 0.40727880597114563, "grad_norm": 31.537353515625, "learning_rate": 2.600664850273538e-07, "logits/chosen": 1.556702733039856, "logits/rejected": 1.6439810991287231, "logps/chosen": -277.95361328125, "logps/ref_chosen": -265.35369873046875, "logps/ref_rejected": -247.9174346923828, "logps/rejected": -299.4835510253906, "loss": 4.3671, "margin_dpo/margin_mean": 38.966163635253906, "margin_dpo/margin_std": 49.094520568847656, "step": 258 }, { "KL/chosen_KL_mean": -12.489116668701172, "KL/mean": -30.218482971191406, "KL/rejected_KL_mean": -47.94784164428711, "KL/std": 46.43803787231445, "epoch": 0.5424083769633508, "fcm_dpo/beta": 0.010553291067481041, "fcm_dpo/delta": 0.02626843750476837, "fcm_dpo/margin": 35.45872497558594, "fcm_dpo/q_t": 0.41366690397262573, "grad_norm": 40.70254135131836, "learning_rate": 2.582369512637302e-07, "logits/chosen": 1.5837359428405762, "logits/rejected": 1.5908241271972656, "logps/chosen": -300.71234130859375, "logps/ref_chosen": -288.2232360839844, "logps/ref_rejected": -270.7300109863281, "logps/rejected": -318.6778564453125, "loss": 4.5045, "margin_dpo/margin_mean": 35.45872497558594, "margin_dpo/margin_std": 54.217803955078125, "step": 259 }, { "KL/chosen_KL_mean": -16.213422775268555, "KL/mean": -24.49829864501953, "KL/rejected_KL_mean": -32.783172607421875, "KL/std": 44.51939392089844, "epoch": 0.5445026178010471, "fcm_dpo/beta": 0.010747314430773258, "fcm_dpo/delta": 0.023694217205047607, "fcm_dpo/margin": 16.569740295410156, "fcm_dpo/q_t": 0.45913174748420715, "grad_norm": 36.17821502685547, "learning_rate": 2.5640697577740815e-07, "logits/chosen": 1.4569004774093628, "logits/rejected": 1.4987921714782715, "logps/chosen": -297.8331298828125, "logps/ref_chosen": -281.61968994140625, "logps/ref_rejected": -271.03973388671875, "logps/rejected": -303.8229064941406, "loss": 5.1896, "margin_dpo/margin_mean": 16.569740295410156, "margin_dpo/margin_std": 52.53853988647461, "step": 260 }, { "KL/chosen_KL_mean": -20.783327102661133, "KL/mean": -36.140594482421875, "KL/rejected_KL_mean": -51.497886657714844, "KL/std": 45.89060974121094, "epoch": 0.5465968586387434, "fcm_dpo/beta": 0.010736201889812946, "fcm_dpo/delta": 0.01374741643667221, "fcm_dpo/margin": 30.714555740356445, "fcm_dpo/q_t": 0.4246878921985626, "grad_norm": 40.811885833740234, "learning_rate": 2.5457665670441937e-07, "logits/chosen": 1.2206922769546509, "logits/rejected": 1.370550274848938, "logps/chosen": -263.2464904785156, "logps/ref_chosen": -242.4631805419922, "logps/ref_rejected": -244.0020751953125, "logps/rejected": -295.4999694824219, "loss": 4.7007, "margin_dpo/margin_mean": 30.714557647705078, "margin_dpo/margin_std": 56.56787109375, "step": 261 }, { "KL/chosen_KL_mean": -13.065511703491211, "KL/mean": -32.089962005615234, "KL/rejected_KL_mean": -51.114410400390625, "KL/std": 47.31406021118164, "epoch": 0.5486910994764398, "fcm_dpo/beta": 0.01091125700622797, "fcm_dpo/delta": -0.01627585105597973, "fcm_dpo/margin": 38.04890441894531, "fcm_dpo/q_t": 0.4048381447792053, "grad_norm": 55.11283874511719, "learning_rate": 2.527460921992209e-07, "logits/chosen": 1.606964111328125, "logits/rejected": 1.6369879245758057, "logps/chosen": -305.0541076660156, "logps/ref_chosen": -291.98858642578125, "logps/ref_rejected": -273.35955810546875, "logps/rejected": -324.4739685058594, "loss": 4.3713, "margin_dpo/margin_mean": 38.04890441894531, "margin_dpo/margin_std": 51.87273406982422, "step": 262 }, { "KL/chosen_KL_mean": -20.500993728637695, "KL/mean": -36.50056457519531, "KL/rejected_KL_mean": -52.50013732910156, "KL/std": 47.64826965332031, "epoch": 0.5507853403141362, "fcm_dpo/beta": 0.010693997144699097, "fcm_dpo/delta": -0.013701440766453743, "fcm_dpo/margin": 31.9991455078125, "fcm_dpo/q_t": 0.42212849855422974, "grad_norm": 45.294193267822266, "learning_rate": 2.509153804294318e-07, "logits/chosen": 1.2619881629943848, "logits/rejected": 1.2392959594726562, "logps/chosen": -295.4894714355469, "logps/ref_chosen": -274.98846435546875, "logps/ref_rejected": -254.75782775878906, "logps/rejected": -307.25799560546875, "loss": 4.6377, "margin_dpo/margin_mean": 31.9991455078125, "margin_dpo/margin_std": 54.831886291503906, "step": 263 }, { "KL/chosen_KL_mean": -11.085941314697266, "KL/mean": -31.268726348876953, "KL/rejected_KL_mean": -51.45151138305664, "KL/std": 47.041046142578125, "epoch": 0.5528795811518324, "fcm_dpo/beta": 0.010340879671275616, "fcm_dpo/delta": -0.05381663143634796, "fcm_dpo/margin": 40.36557388305664, "fcm_dpo/q_t": 0.4039081037044525, "grad_norm": 27.04505729675293, "learning_rate": 2.4908461957056825e-07, "logits/chosen": 1.2077322006225586, "logits/rejected": 1.276128888130188, "logps/chosen": -262.7121276855469, "logps/ref_chosen": -251.62615966796875, "logps/ref_rejected": -251.9549560546875, "logps/rejected": -303.406494140625, "loss": 4.3325, "margin_dpo/margin_mean": 40.36557388305664, "margin_dpo/margin_std": 51.57044982910156, "step": 264 }, { "KL/chosen_KL_mean": -17.214717864990234, "KL/mean": -36.08303451538086, "KL/rejected_KL_mean": -54.95136260986328, "KL/std": 49.44145584106445, "epoch": 0.5549738219895288, "fcm_dpo/beta": 0.009950151666998863, "fcm_dpo/delta": -0.030162349343299866, "fcm_dpo/margin": 37.73666000366211, "fcm_dpo/q_t": 0.41336631774902344, "grad_norm": 37.25914764404297, "learning_rate": 2.4725390780077905e-07, "logits/chosen": 1.388809323310852, "logits/rejected": 1.4408124685287476, "logps/chosen": -295.15130615234375, "logps/ref_chosen": -277.9366149902344, "logps/ref_rejected": -262.689453125, "logps/rejected": -317.6408386230469, "loss": 4.4984, "margin_dpo/margin_mean": 37.73666000366211, "margin_dpo/margin_std": 55.681739807128906, "step": 265 }, { "KL/chosen_KL_mean": -7.3596954345703125, "KL/mean": -26.975372314453125, "KL/rejected_KL_mean": -46.5910530090332, "KL/std": 47.52385330200195, "epoch": 0.5570680628272251, "fcm_dpo/beta": 0.00993617344647646, "fcm_dpo/delta": -0.021311882883310318, "fcm_dpo/margin": 39.231361389160156, "fcm_dpo/q_t": 0.40893828868865967, "grad_norm": 37.004058837890625, "learning_rate": 2.454233432955807e-07, "logits/chosen": 1.655153512954712, "logits/rejected": 1.652661681175232, "logps/chosen": -277.80462646484375, "logps/ref_chosen": -270.4449157714844, "logps/ref_rejected": -259.5942077636719, "logps/rejected": -306.1852722167969, "loss": 4.4086, "margin_dpo/margin_mean": 39.231361389160156, "margin_dpo/margin_std": 50.556400299072266, "step": 266 }, { "KL/chosen_KL_mean": -15.378364562988281, "KL/mean": -31.202293395996094, "KL/rejected_KL_mean": -47.02621841430664, "KL/std": 48.46784973144531, "epoch": 0.5591623036649215, "fcm_dpo/beta": 0.009753599762916565, "fcm_dpo/delta": 0.008063675835728645, "fcm_dpo/margin": 31.647851943969727, "fcm_dpo/q_t": 0.42852890491485596, "grad_norm": 35.43317413330078, "learning_rate": 2.435930242225919e-07, "logits/chosen": 1.227034330368042, "logits/rejected": 1.2889069318771362, "logps/chosen": -285.21826171875, "logps/ref_chosen": -269.83990478515625, "logps/ref_rejected": -246.33746337890625, "logps/rejected": -293.36370849609375, "loss": 4.7415, "margin_dpo/margin_mean": 31.64785385131836, "margin_dpo/margin_std": 59.13050079345703, "step": 267 }, { "KL/chosen_KL_mean": -9.642990112304688, "KL/mean": -28.5567626953125, "KL/rejected_KL_mean": -47.47053527832031, "KL/std": 47.91365432739258, "epoch": 0.5612565445026177, "fcm_dpo/beta": 0.00979677215218544, "fcm_dpo/delta": -0.005948986858129501, "fcm_dpo/margin": 37.827545166015625, "fcm_dpo/q_t": 0.41443774104118347, "grad_norm": 34.83292770385742, "learning_rate": 2.4176304873626984e-07, "logits/chosen": 1.2267543077468872, "logits/rejected": 1.1969324350357056, "logps/chosen": -307.17694091796875, "logps/ref_chosen": -297.533935546875, "logps/ref_rejected": -270.3407897949219, "logps/rejected": -317.8113098144531, "loss": 4.4696, "margin_dpo/margin_mean": 37.827545166015625, "margin_dpo/margin_std": 52.79344177246094, "step": 268 }, { "KL/chosen_KL_mean": -13.821935653686523, "KL/mean": -30.641128540039062, "KL/rejected_KL_mean": -47.4603271484375, "KL/std": 48.29499053955078, "epoch": 0.5633507853403141, "fcm_dpo/beta": 0.010049771517515182, "fcm_dpo/delta": 0.039144884794950485, "fcm_dpo/margin": 33.638389587402344, "fcm_dpo/q_t": 0.42163753509521484, "grad_norm": 34.716251373291016, "learning_rate": 2.399335149726463e-07, "logits/chosen": 1.0486133098602295, "logits/rejected": 1.081622838973999, "logps/chosen": -254.16078186035156, "logps/ref_chosen": -240.33883666992188, "logps/ref_rejected": -220.71717834472656, "logps/rejected": -268.177490234375, "loss": 4.6492, "margin_dpo/margin_mean": 33.63838577270508, "margin_dpo/margin_std": 58.49000930786133, "step": 269 }, { "KL/chosen_KL_mean": -9.618762969970703, "KL/mean": -30.344589233398438, "KL/rejected_KL_mean": -51.070404052734375, "KL/std": 50.43669128417969, "epoch": 0.5654450261780105, "fcm_dpo/beta": 0.009877322241663933, "fcm_dpo/delta": -0.04414429888129234, "fcm_dpo/margin": 41.451622009277344, "fcm_dpo/q_t": 0.4053528308868408, "grad_norm": 51.390342712402344, "learning_rate": 2.381045210440644e-07, "logits/chosen": 1.7349985837936401, "logits/rejected": 1.7574037313461304, "logps/chosen": -318.5118408203125, "logps/ref_chosen": -308.8930969238281, "logps/ref_rejected": -278.47869873046875, "logps/rejected": -329.5491027832031, "loss": 4.4327, "margin_dpo/margin_mean": 41.451622009277344, "margin_dpo/margin_std": 60.15433883666992, "step": 270 }, { "KL/chosen_KL_mean": -10.520090103149414, "KL/mean": -29.58258819580078, "KL/rejected_KL_mean": -48.645076751708984, "KL/std": 47.74767303466797, "epoch": 0.5675392670157068, "fcm_dpo/beta": 0.010011949576437473, "fcm_dpo/delta": 0.017741503193974495, "fcm_dpo/margin": 38.12498474121094, "fcm_dpo/q_t": 0.4125012457370758, "grad_norm": 32.7165641784668, "learning_rate": 2.3627616503391812e-07, "logits/chosen": 1.1843674182891846, "logits/rejected": 1.1827080249786377, "logps/chosen": -285.94830322265625, "logps/ref_chosen": -275.42822265625, "logps/ref_rejected": -251.50906372070312, "logps/rejected": -300.1541442871094, "loss": 4.447, "margin_dpo/margin_mean": 38.12498474121094, "margin_dpo/margin_std": 52.87849044799805, "step": 271 }, { "KL/chosen_KL_mean": -8.865665435791016, "KL/mean": -26.257308959960938, "KL/rejected_KL_mean": -43.648956298828125, "KL/std": 47.73676681518555, "epoch": 0.5696335078534032, "fcm_dpo/beta": 0.009815176017582417, "fcm_dpo/delta": -0.016911154612898827, "fcm_dpo/margin": 34.78328323364258, "fcm_dpo/q_t": 0.4211246073246002, "grad_norm": 58.63067626953125, "learning_rate": 2.344485449913914e-07, "logits/chosen": 1.4511405229568481, "logits/rejected": 1.4676002264022827, "logps/chosen": -298.2484130859375, "logps/ref_chosen": -289.38275146484375, "logps/ref_rejected": -274.895751953125, "logps/rejected": -318.5447082519531, "loss": 4.5905, "margin_dpo/margin_mean": 34.78328323364258, "margin_dpo/margin_std": 54.094459533691406, "step": 272 }, { "KL/chosen_KL_mean": -8.876047134399414, "KL/mean": -28.207664489746094, "KL/rejected_KL_mean": -47.53927993774414, "KL/std": 48.09898376464844, "epoch": 0.5717277486910994, "fcm_dpo/beta": 0.009568422101438046, "fcm_dpo/delta": -0.0034137628972530365, "fcm_dpo/margin": 38.663230895996094, "fcm_dpo/q_t": 0.414844810962677, "grad_norm": 34.78086853027344, "learning_rate": 2.3262175892620062e-07, "logits/chosen": 1.498274803161621, "logits/rejected": 1.618941307067871, "logps/chosen": -293.8645324707031, "logps/ref_chosen": -284.98846435546875, "logps/ref_rejected": -273.5077819824219, "logps/rejected": -321.04705810546875, "loss": 4.5119, "margin_dpo/margin_mean": 38.663230895996094, "margin_dpo/margin_std": 57.82013702392578, "step": 273 }, { "KL/chosen_KL_mean": -9.224098205566406, "KL/mean": -30.08123016357422, "KL/rejected_KL_mean": -50.938358306884766, "KL/std": 50.3647346496582, "epoch": 0.5738219895287958, "fcm_dpo/beta": 0.00954630970954895, "fcm_dpo/delta": -0.04250919446349144, "fcm_dpo/margin": 41.71424865722656, "fcm_dpo/q_t": 0.4074460566043854, "grad_norm": 29.297210693359375, "learning_rate": 2.3079590480333827e-07, "logits/chosen": 1.4107565879821777, "logits/rejected": 1.4762794971466064, "logps/chosen": -271.9036865234375, "logps/ref_chosen": -262.6795959472656, "logps/ref_rejected": -236.13792419433594, "logps/rejected": -287.0762939453125, "loss": 4.4076, "margin_dpo/margin_mean": 41.71424865722656, "margin_dpo/margin_std": 56.30377960205078, "step": 274 }, { "KL/chosen_KL_mean": -5.721000671386719, "KL/mean": -30.27288246154785, "KL/rejected_KL_mean": -54.82476043701172, "KL/std": 48.94614791870117, "epoch": 0.5759162303664922, "fcm_dpo/beta": 0.009182119742035866, "fcm_dpo/delta": -0.053633034229278564, "fcm_dpo/margin": 49.103763580322266, "fcm_dpo/q_t": 0.3949816823005676, "grad_norm": 27.509553909301758, "learning_rate": 2.2897108053782e-07, "logits/chosen": 1.151371955871582, "logits/rejected": 1.2531877756118774, "logps/chosen": -252.1204833984375, "logps/ref_chosen": -246.39950561523438, "logps/ref_rejected": -247.4836883544922, "logps/rejected": -302.3084716796875, "loss": 4.2006, "margin_dpo/margin_mean": 49.10376739501953, "margin_dpo/margin_std": 55.38058853149414, "step": 275 }, { "KL/chosen_KL_mean": -2.3746280670166016, "KL/mean": -24.551910400390625, "KL/rejected_KL_mean": -46.72920227050781, "KL/std": 48.10188674926758, "epoch": 0.5780104712041885, "fcm_dpo/beta": 0.008951807394623756, "fcm_dpo/delta": 0.0025863340124487877, "fcm_dpo/margin": 44.354576110839844, "fcm_dpo/q_t": 0.407490611076355, "grad_norm": 35.141151428222656, "learning_rate": 2.2714738398943308e-07, "logits/chosen": 1.5304458141326904, "logits/rejected": 1.6467180252075195, "logps/chosen": -291.2947082519531, "logps/ref_chosen": -288.9200439453125, "logps/ref_rejected": -291.69305419921875, "logps/rejected": -338.4222412109375, "loss": 4.3461, "margin_dpo/margin_mean": 44.354576110839844, "margin_dpo/margin_std": 53.57950210571289, "step": 276 }, { "KL/chosen_KL_mean": -12.331165313720703, "KL/mean": -28.60528564453125, "KL/rejected_KL_mean": -44.8794059753418, "KL/std": 48.65975570678711, "epoch": 0.5801047120418849, "fcm_dpo/beta": 0.009269548580050468, "fcm_dpo/delta": 0.03773127496242523, "fcm_dpo/margin": 32.548255920410156, "fcm_dpo/q_t": 0.429889976978302, "grad_norm": 36.12635040283203, "learning_rate": 2.2532491295748865e-07, "logits/chosen": 1.2161855697631836, "logits/rejected": 1.2608983516693115, "logps/chosen": -269.4160461425781, "logps/ref_chosen": -257.08489990234375, "logps/ref_rejected": -251.61532592773438, "logps/rejected": -296.4947509765625, "loss": 4.7665, "margin_dpo/margin_mean": 32.548255920410156, "margin_dpo/margin_std": 61.996192932128906, "step": 277 }, { "KL/chosen_KL_mean": -17.341049194335938, "KL/mean": -31.862226486206055, "KL/rejected_KL_mean": -46.38340759277344, "KL/std": 50.42668914794922, "epoch": 0.5821989528795811, "fcm_dpo/beta": 0.009464469738304615, "fcm_dpo/delta": 0.029068736359477043, "fcm_dpo/margin": 29.042362213134766, "fcm_dpo/q_t": 0.43703651428222656, "grad_norm": 35.64056396484375, "learning_rate": 2.2350376517557726e-07, "logits/chosen": 1.3662313222885132, "logits/rejected": 1.3283146619796753, "logps/chosen": -278.0823974609375, "logps/ref_chosen": -260.7413635253906, "logps/ref_rejected": -227.4325408935547, "logps/rejected": -273.81597900390625, "loss": 4.8913, "margin_dpo/margin_mean": 29.042362213134766, "margin_dpo/margin_std": 62.812740325927734, "step": 278 }, { "KL/chosen_KL_mean": -9.475412368774414, "KL/mean": -32.11738586425781, "KL/rejected_KL_mean": -54.75935363769531, "KL/std": 52.19489288330078, "epoch": 0.5842931937172775, "fcm_dpo/beta": 0.009224911220371723, "fcm_dpo/delta": -0.04645578935742378, "fcm_dpo/margin": 45.283931732177734, "fcm_dpo/q_t": 0.4031781256198883, "grad_norm": 27.29572296142578, "learning_rate": 2.2168403830632769e-07, "logits/chosen": 1.2522985935211182, "logits/rejected": 1.3427064418792725, "logps/chosen": -312.4229736328125, "logps/ref_chosen": -302.947509765625, "logps/ref_rejected": -294.1771545410156, "logps/rejected": -348.9365234375, "loss": 4.3233, "margin_dpo/margin_mean": 45.283931732177734, "margin_dpo/margin_std": 56.65925216674805, "step": 279 }, { "KL/chosen_KL_mean": -9.627082824707031, "KL/mean": -30.115880966186523, "KL/rejected_KL_mean": -50.60468673706055, "KL/std": 48.38449478149414, "epoch": 0.5863874345549738, "fcm_dpo/beta": 0.009083016775548458, "fcm_dpo/delta": -0.004786573350429535, "fcm_dpo/margin": 40.97761917114258, "fcm_dpo/q_t": 0.41387462615966797, "grad_norm": 31.47639274597168, "learning_rate": 2.1986582993616925e-07, "logits/chosen": 1.4386847019195557, "logits/rejected": 1.5247678756713867, "logps/chosen": -267.5251770019531, "logps/ref_chosen": -257.8981018066406, "logps/ref_rejected": -244.9084014892578, "logps/rejected": -295.5130920410156, "loss": 4.4853, "margin_dpo/margin_mean": 40.97761917114258, "margin_dpo/margin_std": 59.07532501220703, "step": 280 }, { "KL/chosen_KL_mean": -19.928136825561523, "KL/mean": -39.731937408447266, "KL/rejected_KL_mean": -59.53572463989258, "KL/std": 52.304969787597656, "epoch": 0.5884816753926702, "fcm_dpo/beta": 0.009135991334915161, "fcm_dpo/delta": 0.00806823093444109, "fcm_dpo/margin": 39.607601165771484, "fcm_dpo/q_t": 0.41674911975860596, "grad_norm": 35.33727264404297, "learning_rate": 2.1804923757009882e-07, "logits/chosen": 1.4383091926574707, "logits/rejected": 1.5045735836029053, "logps/chosen": -310.122802734375, "logps/ref_chosen": -290.1946716308594, "logps/ref_rejected": -290.98883056640625, "logps/rejected": -350.52459716796875, "loss": 4.4937, "margin_dpo/margin_mean": 39.607601165771484, "margin_dpo/margin_std": 55.200523376464844, "step": 281 }, { "KL/chosen_KL_mean": -17.756895065307617, "KL/mean": -35.99581527709961, "KL/rejected_KL_mean": -54.2347412109375, "KL/std": 52.38850402832031, "epoch": 0.5905759162303665, "fcm_dpo/beta": 0.009173966012895107, "fcm_dpo/delta": 0.009089872241020203, "fcm_dpo/margin": 36.47785949707031, "fcm_dpo/q_t": 0.4234859347343445, "grad_norm": 35.32107162475586, "learning_rate": 2.1623435862645205e-07, "logits/chosen": 1.3886326551437378, "logits/rejected": 1.4161839485168457, "logps/chosen": -326.6743469238281, "logps/ref_chosen": -308.91741943359375, "logps/ref_rejected": -272.2177734375, "logps/rejected": -326.4525146484375, "loss": 4.6586, "margin_dpo/margin_mean": 36.47785568237305, "margin_dpo/margin_std": 63.51108932495117, "step": 282 }, { "KL/chosen_KL_mean": -20.84634017944336, "KL/mean": -40.495811462402344, "KL/rejected_KL_mean": -60.14530944824219, "KL/std": 49.9395751953125, "epoch": 0.5926701570680628, "fcm_dpo/beta": 0.00926823541522026, "fcm_dpo/delta": -0.016706613823771477, "fcm_dpo/margin": 39.29896926879883, "fcm_dpo/q_t": 0.41592517495155334, "grad_norm": 27.957590103149414, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 1.6090337038040161, "logits/rejected": 1.678878903388977, "logps/chosen": -268.86834716796875, "logps/ref_chosen": -248.02200317382812, "logps/ref_rejected": -234.78541564941406, "logps/rejected": -294.93072509765625, "loss": 4.5319, "margin_dpo/margin_mean": 39.29896926879883, "margin_dpo/margin_std": 58.933406829833984, "step": 283 }, { "KL/chosen_KL_mean": -22.433128356933594, "KL/mean": -41.53519058227539, "KL/rejected_KL_mean": -60.63725662231445, "KL/std": 53.89200973510742, "epoch": 0.5947643979057592, "fcm_dpo/beta": 0.009125684387981892, "fcm_dpo/delta": 0.02019267901778221, "fcm_dpo/margin": 38.204124450683594, "fcm_dpo/q_t": 0.4207829535007477, "grad_norm": 50.85140609741211, "learning_rate": 2.1261013021512378e-07, "logits/chosen": 1.3252496719360352, "logits/rejected": 1.3272849321365356, "logps/chosen": -286.92236328125, "logps/ref_chosen": -264.4892578125, "logps/ref_rejected": -255.43092346191406, "logps/rejected": -316.0682067871094, "loss": 4.6678, "margin_dpo/margin_mean": 38.20412063598633, "margin_dpo/margin_std": 68.52159881591797, "step": 284 }, { "KL/chosen_KL_mean": -20.986644744873047, "KL/mean": -36.80175018310547, "KL/rejected_KL_mean": -52.61685562133789, "KL/std": 50.02957534790039, "epoch": 0.5968586387434555, "fcm_dpo/beta": 0.009410517290234566, "fcm_dpo/delta": 0.0025642849504947662, "fcm_dpo/margin": 31.630218505859375, "fcm_dpo/q_t": 0.43106475472450256, "grad_norm": 50.25341033935547, "learning_rate": 2.1080097510381294e-07, "logits/chosen": 1.4484590291976929, "logits/rejected": 1.4547747373580933, "logps/chosen": -322.455322265625, "logps/ref_chosen": -301.46868896484375, "logps/ref_rejected": -274.2380676269531, "logps/rejected": -326.85491943359375, "loss": 4.7508, "margin_dpo/margin_mean": 31.630218505859375, "margin_dpo/margin_std": 56.27755355834961, "step": 285 }, { "KL/chosen_KL_mean": -25.049720764160156, "KL/mean": -41.187744140625, "KL/rejected_KL_mean": -57.32576370239258, "KL/std": 51.383689880371094, "epoch": 0.5989528795811518, "fcm_dpo/beta": 0.009583317674696445, "fcm_dpo/delta": 0.06181109696626663, "fcm_dpo/margin": 32.276039123535156, "fcm_dpo/q_t": 0.4300505518913269, "grad_norm": 32.433380126953125, "learning_rate": 2.089939221172446e-07, "logits/chosen": 1.3072019815444946, "logits/rejected": 1.3753267526626587, "logps/chosen": -291.6087951660156, "logps/ref_chosen": -266.5590515136719, "logps/ref_rejected": -280.12066650390625, "logps/rejected": -337.4464416503906, "loss": 4.7853, "margin_dpo/margin_mean": 32.276039123535156, "margin_dpo/margin_std": 64.37699127197266, "step": 286 }, { "KL/chosen_KL_mean": -22.22498321533203, "KL/mean": -40.70927810668945, "KL/rejected_KL_mean": -59.19358825683594, "KL/std": 48.768131256103516, "epoch": 0.6010471204188481, "fcm_dpo/beta": 0.009829613380134106, "fcm_dpo/delta": 0.00442691333591938, "fcm_dpo/margin": 36.968597412109375, "fcm_dpo/q_t": 0.4159740209579468, "grad_norm": 43.9468879699707, "learning_rate": 2.0718906816218595e-07, "logits/chosen": 1.1281629800796509, "logits/rejected": 1.1733816862106323, "logps/chosen": -292.16131591796875, "logps/ref_chosen": -269.93634033203125, "logps/ref_rejected": -232.32330322265625, "logps/rejected": -291.5168762207031, "loss": 4.5792, "margin_dpo/margin_mean": 36.968597412109375, "margin_dpo/margin_std": 60.603633880615234, "step": 287 }, { "KL/chosen_KL_mean": -23.372922897338867, "KL/mean": -41.4660758972168, "KL/rejected_KL_mean": -59.5592155456543, "KL/std": 51.44970703125, "epoch": 0.6031413612565445, "fcm_dpo/beta": 0.010073695331811905, "fcm_dpo/delta": 0.008239217102527618, "fcm_dpo/margin": 36.18629837036133, "fcm_dpo/q_t": 0.4177555441856384, "grad_norm": 47.44202423095703, "learning_rate": 2.053865100274774e-07, "logits/chosen": 1.233814001083374, "logits/rejected": 1.1790906190872192, "logps/chosen": -307.41839599609375, "logps/ref_chosen": -284.04547119140625, "logps/ref_rejected": -261.19378662109375, "logps/rejected": -320.75299072265625, "loss": 4.5936, "margin_dpo/margin_mean": 36.18629837036133, "margin_dpo/margin_std": 60.974952697753906, "step": 288 }, { "KL/chosen_KL_mean": -30.569286346435547, "KL/mean": -43.77549362182617, "KL/rejected_KL_mean": -56.981693267822266, "KL/std": 53.21607971191406, "epoch": 0.6052356020942409, "fcm_dpo/beta": 0.010000809095799923, "fcm_dpo/delta": 0.03143392875790596, "fcm_dpo/margin": 26.41241455078125, "fcm_dpo/q_t": 0.43992340564727783, "grad_norm": 31.09710693359375, "learning_rate": 2.035863443788411e-07, "logits/chosen": 1.3066461086273193, "logits/rejected": 1.2719676494598389, "logps/chosen": -353.85052490234375, "logps/ref_chosen": -323.28125, "logps/ref_rejected": -274.2093200683594, "logps/rejected": -331.1910400390625, "loss": 4.9493, "margin_dpo/margin_mean": 26.412412643432617, "margin_dpo/margin_std": 62.61479187011719, "step": 289 }, { "KL/chosen_KL_mean": -23.792905807495117, "KL/mean": -39.8603401184082, "KL/rejected_KL_mean": -55.92776107788086, "KL/std": 51.54084396362305, "epoch": 0.6073298429319371, "fcm_dpo/beta": 0.010087518952786922, "fcm_dpo/delta": 0.01475964393466711, "fcm_dpo/margin": 32.134864807128906, "fcm_dpo/q_t": 0.4261983335018158, "grad_norm": 38.690677642822266, "learning_rate": 2.0178866775369774e-07, "logits/chosen": 1.2787078619003296, "logits/rejected": 1.2383533716201782, "logps/chosen": -341.2283020019531, "logps/ref_chosen": -317.4353942871094, "logps/ref_rejected": -309.32403564453125, "logps/rejected": -365.2518005371094, "loss": 4.7017, "margin_dpo/margin_mean": 32.13486862182617, "margin_dpo/margin_std": 57.67361831665039, "step": 290 }, { "KL/chosen_KL_mean": -21.22112464904785, "KL/mean": -44.63714599609375, "KL/rejected_KL_mean": -68.05317687988281, "KL/std": 55.67799377441406, "epoch": 0.6094240837696335, "fcm_dpo/beta": 0.009846209548413754, "fcm_dpo/delta": -0.09073906391859055, "fcm_dpo/margin": 46.8320426940918, "fcm_dpo/q_t": 0.39452454447746277, "grad_norm": 32.533935546875, "learning_rate": 1.9999357655598891e-07, "logits/chosen": 1.2153387069702148, "logits/rejected": 1.377948522567749, "logps/chosen": -286.7293395996094, "logps/ref_chosen": -265.50823974609375, "logps/ref_rejected": -275.1380310058594, "logps/rejected": -343.191162109375, "loss": 4.2942, "margin_dpo/margin_mean": 46.83203887939453, "margin_dpo/margin_std": 62.62254333496094, "step": 291 }, { "KL/chosen_KL_mean": -18.6783504486084, "KL/mean": -37.69143295288086, "KL/rejected_KL_mean": -56.70452117919922, "KL/std": 52.398887634277344, "epoch": 0.6115183246073298, "fcm_dpo/beta": 0.00950497854501009, "fcm_dpo/delta": 0.006776423193514347, "fcm_dpo/margin": 38.02617645263672, "fcm_dpo/q_t": 0.4163859188556671, "grad_norm": 45.77560806274414, "learning_rate": 1.9820116705100775e-07, "logits/chosen": 1.3917005062103271, "logits/rejected": 1.514042854309082, "logps/chosen": -268.57965087890625, "logps/ref_chosen": -249.9013214111328, "logps/ref_rejected": -274.58245849609375, "logps/rejected": -331.2869873046875, "loss": 4.5507, "margin_dpo/margin_mean": 38.026180267333984, "margin_dpo/margin_std": 58.06243133544922, "step": 292 }, { "KL/chosen_KL_mean": -26.171293258666992, "KL/mean": -46.48382568359375, "KL/rejected_KL_mean": -66.79635620117188, "KL/std": 55.30024719238281, "epoch": 0.6136125654450262, "fcm_dpo/beta": 0.009691167622804642, "fcm_dpo/delta": 0.005665135104209185, "fcm_dpo/margin": 40.625057220458984, "fcm_dpo/q_t": 0.4102424681186676, "grad_norm": 60.529232025146484, "learning_rate": 1.9641153536023642e-07, "logits/chosen": 1.3179248571395874, "logits/rejected": 1.2775920629501343, "logps/chosen": -322.47723388671875, "logps/ref_chosen": -296.30596923828125, "logps/ref_rejected": -268.0640869140625, "logps/rejected": -334.8604431152344, "loss": 4.4681, "margin_dpo/margin_mean": 40.625057220458984, "margin_dpo/margin_std": 60.73282241821289, "step": 293 }, { "KL/chosen_KL_mean": -22.822816848754883, "KL/mean": -39.891197204589844, "KL/rejected_KL_mean": -56.959564208984375, "KL/std": 52.99144744873047, "epoch": 0.6157068062827226, "fcm_dpo/beta": 0.009667360223829746, "fcm_dpo/delta": 0.016613639891147614, "fcm_dpo/margin": 34.136741638183594, "fcm_dpo/q_t": 0.4245774447917938, "grad_norm": 58.12077713012695, "learning_rate": 1.9462477745619106e-07, "logits/chosen": 1.2750738859176636, "logits/rejected": 1.3111969232559204, "logps/chosen": -256.705810546875, "logps/ref_chosen": -233.8829803466797, "logps/ref_rejected": -224.14334106445312, "logps/rejected": -281.1029052734375, "loss": 4.6867, "margin_dpo/margin_mean": 34.136741638183594, "margin_dpo/margin_std": 60.988094329833984, "step": 294 }, { "KL/chosen_KL_mean": -24.004697799682617, "KL/mean": -43.55730438232422, "KL/rejected_KL_mean": -63.10991668701172, "KL/std": 53.0479850769043, "epoch": 0.6178010471204188, "fcm_dpo/beta": 0.009808500297367573, "fcm_dpo/delta": 0.016834016889333725, "fcm_dpo/margin": 39.105220794677734, "fcm_dpo/q_t": 0.41373884677886963, "grad_norm": 38.24657440185547, "learning_rate": 1.928409891572757e-07, "logits/chosen": 1.404849886894226, "logits/rejected": 1.4178699254989624, "logps/chosen": -268.11602783203125, "logps/ref_chosen": -244.111328125, "logps/ref_rejected": -240.18373107910156, "logps/rejected": -303.29364013671875, "loss": 4.5514, "margin_dpo/margin_mean": 39.105220794677734, "margin_dpo/margin_std": 64.37168884277344, "step": 295 }, { "KL/chosen_KL_mean": -18.98125648498535, "KL/mean": -46.19329833984375, "KL/rejected_KL_mean": -73.40534210205078, "KL/std": 55.29228973388672, "epoch": 0.6198952879581152, "fcm_dpo/beta": 0.009237255901098251, "fcm_dpo/delta": -0.1366957724094391, "fcm_dpo/margin": 54.424095153808594, "fcm_dpo/q_t": 0.3833378553390503, "grad_norm": 30.917573928833008, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 1.2253468036651611, "logits/rejected": 1.3825082778930664, "logps/chosen": -303.6618347167969, "logps/ref_chosen": -284.6805725097656, "logps/ref_rejected": -282.1923828125, "logps/rejected": -355.5976867675781, "loss": 4.1243, "margin_dpo/margin_mean": 54.424095153808594, "margin_dpo/margin_std": 63.56785583496094, "step": 296 }, { "KL/chosen_KL_mean": -21.348806381225586, "KL/mean": -42.41645431518555, "KL/rejected_KL_mean": -63.48410415649414, "KL/std": 54.160221099853516, "epoch": 0.6219895287958115, "fcm_dpo/beta": 0.008797341957688332, "fcm_dpo/delta": 0.007480676285922527, "fcm_dpo/margin": 42.135292053222656, "fcm_dpo/q_t": 0.4149421155452728, "grad_norm": 26.944902420043945, "learning_rate": 1.8928270384706582e-07, "logits/chosen": 1.695941686630249, "logits/rejected": 1.707463026046753, "logps/chosen": -301.57745361328125, "logps/ref_chosen": -280.2286376953125, "logps/ref_rejected": -280.8172912597656, "logps/rejected": -344.3014221191406, "loss": 4.5039, "margin_dpo/margin_mean": 42.135292053222656, "margin_dpo/margin_std": 61.98017883300781, "step": 297 }, { "KL/chosen_KL_mean": -15.223262786865234, "KL/mean": -38.87238311767578, "KL/rejected_KL_mean": -62.52149963378906, "KL/std": 56.35419845581055, "epoch": 0.6240837696335079, "fcm_dpo/beta": 0.008612115867435932, "fcm_dpo/delta": -0.07574725151062012, "fcm_dpo/margin": 47.29825210571289, "fcm_dpo/q_t": 0.40708616375923157, "grad_norm": 34.92477798461914, "learning_rate": 1.875083976558136e-07, "logits/chosen": 1.3770982027053833, "logits/rejected": 1.3817055225372314, "logps/chosen": -312.22357177734375, "logps/ref_chosen": -297.0003356933594, "logps/ref_rejected": -279.1561279296875, "logps/rejected": -341.6776123046875, "loss": 4.4116, "margin_dpo/margin_mean": 47.29825210571289, "margin_dpo/margin_std": 66.13207244873047, "step": 298 }, { "KL/chosen_KL_mean": -24.688032150268555, "KL/mean": -43.32847213745117, "KL/rejected_KL_mean": -61.96891403198242, "KL/std": 54.59784698486328, "epoch": 0.6261780104712041, "fcm_dpo/beta": 0.008580111898481846, "fcm_dpo/delta": 0.032745346426963806, "fcm_dpo/margin": 37.28089141845703, "fcm_dpo/q_t": 0.42528051137924194, "grad_norm": 30.84089469909668, "learning_rate": 1.8573744269954297e-07, "logits/chosen": 1.281267523765564, "logits/rejected": 1.3654969930648804, "logps/chosen": -296.15948486328125, "logps/ref_chosen": -271.471435546875, "logps/ref_rejected": -268.0870056152344, "logps/rejected": -330.0559387207031, "loss": 4.6057, "margin_dpo/margin_mean": 37.28089141845703, "margin_dpo/margin_std": 54.77768325805664, "step": 299 }, { "KL/chosen_KL_mean": -25.011089324951172, "KL/mean": -44.76543426513672, "KL/rejected_KL_mean": -64.519775390625, "KL/std": 54.050819396972656, "epoch": 0.6282722513089005, "fcm_dpo/beta": 0.008571874350309372, "fcm_dpo/delta": 0.0033765954431146383, "fcm_dpo/margin": 39.508697509765625, "fcm_dpo/q_t": 0.42205578088760376, "grad_norm": 38.74055480957031, "learning_rate": 1.839699339491937e-07, "logits/chosen": 1.3771365880966187, "logits/rejected": 1.364328145980835, "logps/chosen": -331.36895751953125, "logps/ref_chosen": -306.35784912109375, "logps/ref_rejected": -268.64776611328125, "logps/rejected": -333.16754150390625, "loss": 4.6241, "margin_dpo/margin_mean": 39.508697509765625, "margin_dpo/margin_std": 66.10736846923828, "step": 300 }, { "KL/chosen_KL_mean": -23.9780216217041, "KL/mean": -43.05587387084961, "KL/rejected_KL_mean": -62.13372802734375, "KL/std": 54.78178787231445, "epoch": 0.6303664921465969, "fcm_dpo/beta": 0.008896047249436378, "fcm_dpo/delta": 0.06255801022052765, "fcm_dpo/margin": 38.155696868896484, "fcm_dpo/q_t": 0.4224284887313843, "grad_norm": 37.444976806640625, "learning_rate": 1.8220596619089573e-07, "logits/chosen": 1.1716094017028809, "logits/rejected": 1.2080966234207153, "logps/chosen": -293.97052001953125, "logps/ref_chosen": -269.9924621582031, "logps/ref_rejected": -271.3135681152344, "logps/rejected": -333.44732666015625, "loss": 4.6205, "margin_dpo/margin_mean": 38.15570068359375, "margin_dpo/margin_std": 63.96094512939453, "step": 301 }, { "KL/chosen_KL_mean": -22.293458938598633, "KL/mean": -44.63550567626953, "KL/rejected_KL_mean": -66.9775390625, "KL/std": 53.870208740234375, "epoch": 0.6324607329842932, "fcm_dpo/beta": 0.00888870283961296, "fcm_dpo/delta": -0.025246508419513702, "fcm_dpo/margin": 44.684078216552734, "fcm_dpo/q_t": 0.4077514708042145, "grad_norm": 31.096065521240234, "learning_rate": 1.8044563402088682e-07, "logits/chosen": 1.2770832777023315, "logits/rejected": 1.3494573831558228, "logps/chosen": -282.7541809082031, "logps/ref_chosen": -260.460693359375, "logps/ref_rejected": -254.2005615234375, "logps/rejected": -321.1781005859375, "loss": 4.4448, "margin_dpo/margin_mean": 44.684078216552734, "margin_dpo/margin_std": 64.02107238769531, "step": 302 }, { "KL/chosen_KL_mean": -20.06652069091797, "KL/mean": -41.20815658569336, "KL/rejected_KL_mean": -62.349796295166016, "KL/std": 54.60296630859375, "epoch": 0.6345549738219896, "fcm_dpo/beta": 0.008872736245393753, "fcm_dpo/delta": -0.021030310541391373, "fcm_dpo/margin": 42.28326416015625, "fcm_dpo/q_t": 0.4127102792263031, "grad_norm": 37.00385284423828, "learning_rate": 1.7868903184043885e-07, "logits/chosen": 1.320084571838379, "logits/rejected": 1.3712267875671387, "logps/chosen": -316.16278076171875, "logps/ref_chosen": -296.09625244140625, "logps/ref_rejected": -264.7806701660156, "logps/rejected": -327.13043212890625, "loss": 4.5436, "margin_dpo/margin_mean": 42.28326416015625, "margin_dpo/margin_std": 66.04558563232422, "step": 303 }, { "KL/chosen_KL_mean": -30.992998123168945, "KL/mean": -49.216251373291016, "KL/rejected_KL_mean": -67.43950653076172, "KL/std": 58.00931167602539, "epoch": 0.6366492146596858, "fcm_dpo/beta": 0.009011980146169662, "fcm_dpo/delta": 0.0739751011133194, "fcm_dpo/margin": 36.44649887084961, "fcm_dpo/q_t": 0.42482835054397583, "grad_norm": 35.08708572387695, "learning_rate": 1.7693625385079574e-07, "logits/chosen": 1.2776747941970825, "logits/rejected": 1.2732635736465454, "logps/chosen": -312.84246826171875, "logps/ref_chosen": -281.8494873046875, "logps/ref_rejected": -234.3673858642578, "logps/rejected": -301.806884765625, "loss": 4.6685, "margin_dpo/margin_mean": 36.44649887084961, "margin_dpo/margin_std": 63.821224212646484, "step": 304 }, { "KL/chosen_KL_mean": -15.93623161315918, "KL/mean": -42.71954345703125, "KL/rejected_KL_mean": -69.50286865234375, "KL/std": 57.875244140625, "epoch": 0.6387434554973822, "fcm_dpo/beta": 0.008949695155024529, "fcm_dpo/delta": -0.08474086225032806, "fcm_dpo/margin": 53.56663513183594, "fcm_dpo/q_t": 0.38978996872901917, "grad_norm": 42.071353912353516, "learning_rate": 1.7518739404812155e-07, "logits/chosen": 1.227604627609253, "logits/rejected": 1.3309612274169922, "logps/chosen": -263.9789733886719, "logps/ref_chosen": -248.0427703857422, "logps/ref_rejected": -263.464599609375, "logps/rejected": -332.96746826171875, "loss": 4.2059, "margin_dpo/margin_mean": 53.56663513183594, "margin_dpo/margin_std": 65.27417755126953, "step": 305 }, { "KL/chosen_KL_mean": -14.433404922485352, "KL/mean": -37.7215576171875, "KL/rejected_KL_mean": -61.00971603393555, "KL/std": 54.80039596557617, "epoch": 0.6408376963350786, "fcm_dpo/beta": 0.008622164838016033, "fcm_dpo/delta": -0.028978414833545685, "fcm_dpo/margin": 46.576324462890625, "fcm_dpo/q_t": 0.4068741798400879, "grad_norm": 40.38009262084961, "learning_rate": 1.7344254621846017e-07, "logits/chosen": 1.4350454807281494, "logits/rejected": 1.3580363988876343, "logps/chosen": -325.7193298339844, "logps/ref_chosen": -311.2859191894531, "logps/ref_rejected": -268.4949645996094, "logps/rejected": -329.5046691894531, "loss": 4.4003, "margin_dpo/margin_mean": 46.576324462890625, "margin_dpo/margin_std": 62.669273376464844, "step": 306 }, { "KL/chosen_KL_mean": -20.74403190612793, "KL/mean": -46.11383056640625, "KL/rejected_KL_mean": -71.48363494873047, "KL/std": 58.11615753173828, "epoch": 0.6429319371727749, "fcm_dpo/beta": 0.008324475027620792, "fcm_dpo/delta": -0.06760591268539429, "fcm_dpo/margin": 50.73961639404297, "fcm_dpo/q_t": 0.4010232090950012, "grad_norm": 22.734024047851562, "learning_rate": 1.717018039327053e-07, "logits/chosen": 1.370535135269165, "logits/rejected": 1.3642425537109375, "logps/chosen": -290.95648193359375, "logps/ref_chosen": -270.21246337890625, "logps/ref_rejected": -239.1932830810547, "logps/rejected": -310.6769104003906, "loss": 4.3113, "margin_dpo/margin_mean": 50.73961639404297, "margin_dpo/margin_std": 61.37603759765625, "step": 307 }, { "KL/chosen_KL_mean": -27.225780487060547, "KL/mean": -44.34769821166992, "KL/rejected_KL_mean": -61.46961212158203, "KL/std": 55.797428131103516, "epoch": 0.6450261780104712, "fcm_dpo/beta": 0.008255185559391975, "fcm_dpo/delta": 0.05357668548822403, "fcm_dpo/margin": 34.24382781982422, "fcm_dpo/q_t": 0.4344278872013092, "grad_norm": 30.90519142150879, "learning_rate": 1.699652605415828e-07, "logits/chosen": 1.472908854484558, "logits/rejected": 1.48015296459198, "logps/chosen": -315.74859619140625, "logps/ref_chosen": -288.5228271484375, "logps/ref_rejected": -253.90597534179688, "logps/rejected": -315.3755798339844, "loss": 4.779, "margin_dpo/margin_mean": 34.24382781982422, "margin_dpo/margin_std": 63.58550262451172, "step": 308 }, { "KL/chosen_KL_mean": -25.030376434326172, "KL/mean": -50.54033660888672, "KL/rejected_KL_mean": -76.05027770996094, "KL/std": 59.059818267822266, "epoch": 0.6471204188481675, "fcm_dpo/beta": 0.008353800512850285, "fcm_dpo/delta": -0.02783842757344246, "fcm_dpo/margin": 51.019893646240234, "fcm_dpo/q_t": 0.4025571048259735, "grad_norm": 28.824357986450195, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 1.3302932977676392, "logits/rejected": 1.4493520259857178, "logps/chosen": -296.92510986328125, "logps/ref_chosen": -271.8947448730469, "logps/ref_rejected": -260.5709228515625, "logps/rejected": -336.62115478515625, "loss": 4.3499, "margin_dpo/margin_mean": 51.019893646240234, "margin_dpo/margin_std": 68.8276138305664, "step": 309 }, { "KL/chosen_KL_mean": -30.154001235961914, "KL/mean": -50.38459396362305, "KL/rejected_KL_mean": -70.61518859863281, "KL/std": 54.72597885131836, "epoch": 0.6492146596858639, "fcm_dpo/beta": 0.008251826278865337, "fcm_dpo/delta": 0.011738432571291924, "fcm_dpo/margin": 40.46120071411133, "fcm_dpo/q_t": 0.42262619733810425, "grad_norm": 37.7224006652832, "learning_rate": 1.6650514271527465e-07, "logits/chosen": 1.2993278503417969, "logits/rejected": 1.412429690361023, "logps/chosen": -300.3294677734375, "logps/ref_chosen": -270.17547607421875, "logps/ref_rejected": -257.6781311035156, "logps/rejected": -328.2933349609375, "loss": 4.5676, "margin_dpo/margin_mean": 40.46120071411133, "margin_dpo/margin_std": 59.413360595703125, "step": 310 }, { "KL/chosen_KL_mean": -28.951461791992188, "KL/mean": -51.364383697509766, "KL/rejected_KL_mean": -73.77729797363281, "KL/std": 58.013572692871094, "epoch": 0.6513089005235602, "fcm_dpo/beta": 0.008316735737025738, "fcm_dpo/delta": -0.003663080744445324, "fcm_dpo/margin": 44.82583236694336, "fcm_dpo/q_t": 0.413865327835083, "grad_norm": 39.35993957519531, "learning_rate": 1.647817538357072e-07, "logits/chosen": 1.110593557357788, "logits/rejected": 1.2548357248306274, "logps/chosen": -292.7137145996094, "logps/ref_chosen": -263.76226806640625, "logps/ref_rejected": -263.8758850097656, "logps/rejected": -337.6531677246094, "loss": 4.5594, "margin_dpo/margin_mean": 44.82583236694336, "margin_dpo/margin_std": 71.53900909423828, "step": 311 }, { "KL/chosen_KL_mean": -26.776235580444336, "KL/mean": -48.77787780761719, "KL/rejected_KL_mean": -70.77953338623047, "KL/std": 55.27427291870117, "epoch": 0.6534031413612565, "fcm_dpo/beta": 0.00823201984167099, "fcm_dpo/delta": 0.00931546464562416, "fcm_dpo/margin": 44.003299713134766, "fcm_dpo/q_t": 0.4157472848892212, "grad_norm": 34.14425277709961, "learning_rate": 1.6306293495205755e-07, "logits/chosen": 1.1987491846084595, "logits/rejected": 1.1471325159072876, "logps/chosen": -301.4486389160156, "logps/ref_chosen": -274.67242431640625, "logps/ref_rejected": -242.1553955078125, "logps/rejected": -312.9349365234375, "loss": 4.5322, "margin_dpo/margin_mean": 44.00330352783203, "margin_dpo/margin_std": 65.34341430664062, "step": 312 }, { "KL/chosen_KL_mean": -29.452743530273438, "KL/mean": -50.3410530090332, "KL/rejected_KL_mean": -71.2293701171875, "KL/std": 59.430816650390625, "epoch": 0.6554973821989529, "fcm_dpo/beta": 0.008578747510910034, "fcm_dpo/delta": 0.008986297994852066, "fcm_dpo/margin": 41.77661895751953, "fcm_dpo/q_t": 0.41794195771217346, "grad_norm": 33.07001876831055, "learning_rate": 1.6134877823936607e-07, "logits/chosen": 1.4117703437805176, "logits/rejected": 1.4809421300888062, "logps/chosen": -324.6405944824219, "logps/ref_chosen": -295.1878662109375, "logps/ref_rejected": -271.3431091308594, "logps/rejected": -342.5724792480469, "loss": 4.613, "margin_dpo/margin_mean": 41.77661895751953, "margin_dpo/margin_std": 68.8209457397461, "step": 313 }, { "KL/chosen_KL_mean": -27.462417602539062, "KL/mean": -49.15557098388672, "KL/rejected_KL_mean": -70.84872436523438, "KL/std": 57.39149475097656, "epoch": 0.6575916230366492, "fcm_dpo/beta": 0.008496898226439953, "fcm_dpo/delta": 0.0017744582146406174, "fcm_dpo/margin": 43.38631820678711, "fcm_dpo/q_t": 0.41470271348953247, "grad_norm": 45.454280853271484, "learning_rate": 1.5963937562265522e-07, "logits/chosen": 1.3141942024230957, "logits/rejected": 1.3588166236877441, "logps/chosen": -317.23028564453125, "logps/ref_chosen": -289.767822265625, "logps/ref_rejected": -267.0824279785156, "logps/rejected": -337.93115234375, "loss": 4.5466, "margin_dpo/margin_mean": 43.386322021484375, "margin_dpo/margin_std": 67.73545837402344, "step": 314 }, { "KL/chosen_KL_mean": -24.688133239746094, "KL/mean": -48.83763885498047, "KL/rejected_KL_mean": -72.98715209960938, "KL/std": 58.236576080322266, "epoch": 0.6596858638743456, "fcm_dpo/beta": 0.008282303810119629, "fcm_dpo/delta": -0.027523932978510857, "fcm_dpo/margin": 48.29901885986328, "fcm_dpo/q_t": 0.4080940783023834, "grad_norm": 32.36628723144531, "learning_rate": 1.5793481877199943e-07, "logits/chosen": 1.6165395975112915, "logits/rejected": 1.6356583833694458, "logps/chosen": -323.107177734375, "logps/ref_chosen": -298.4190673828125, "logps/ref_rejected": -266.64447021484375, "logps/rejected": -339.6316223144531, "loss": 4.4189, "margin_dpo/margin_mean": 48.29901885986328, "margin_dpo/margin_std": 68.33442687988281, "step": 315 }, { "KL/chosen_KL_mean": -29.04701042175293, "KL/mean": -54.41446304321289, "KL/rejected_KL_mean": -79.78190612792969, "KL/std": 59.92026138305664, "epoch": 0.6617801047120419, "fcm_dpo/beta": 0.008162661455571651, "fcm_dpo/delta": -0.015288690105080605, "fcm_dpo/margin": 50.734893798828125, "fcm_dpo/q_t": 0.40513941645622253, "grad_norm": 32.51222610473633, "learning_rate": 1.562351990976095e-07, "logits/chosen": 1.397526741027832, "logits/rejected": 1.3919577598571777, "logps/chosen": -331.2768859863281, "logps/ref_chosen": -302.2298583984375, "logps/ref_rejected": -272.7162170410156, "logps/rejected": -352.49810791015625, "loss": 4.372, "margin_dpo/margin_mean": 50.734893798828125, "margin_dpo/margin_std": 69.03914642333984, "step": 316 }, { "KL/chosen_KL_mean": -22.076305389404297, "KL/mean": -44.926109313964844, "KL/rejected_KL_mean": -67.77590942382812, "KL/std": 56.510860443115234, "epoch": 0.6638743455497382, "fcm_dpo/beta": 0.008222212083637714, "fcm_dpo/delta": 0.02461521327495575, "fcm_dpo/margin": 45.69961166381836, "fcm_dpo/q_t": 0.4130440354347229, "grad_norm": 63.09319305419922, "learning_rate": 1.5454060774493065e-07, "logits/chosen": 1.3081531524658203, "logits/rejected": 1.2788610458374023, "logps/chosen": -268.92974853515625, "logps/ref_chosen": -246.85342407226562, "logps/ref_rejected": -216.5264892578125, "logps/rejected": -284.3023681640625, "loss": 4.4627, "margin_dpo/margin_mean": 45.69961166381836, "margin_dpo/margin_std": 64.08443450927734, "step": 317 }, { "KL/chosen_KL_mean": -25.16353416442871, "KL/mean": -48.39657974243164, "KL/rejected_KL_mean": -71.6296157836914, "KL/std": 58.039100646972656, "epoch": 0.6659685863874345, "fcm_dpo/beta": 0.008192427456378937, "fcm_dpo/delta": -0.007180863991379738, "fcm_dpo/margin": 46.466087341308594, "fcm_dpo/q_t": 0.41059932112693787, "grad_norm": 40.38105392456055, "learning_rate": 1.5285113558975427e-07, "logits/chosen": 1.3573622703552246, "logits/rejected": 1.4426628351211548, "logps/chosen": -287.8131408691406, "logps/ref_chosen": -262.649658203125, "logps/ref_rejected": -251.24945068359375, "logps/rejected": -322.8790588378906, "loss": 4.4393, "margin_dpo/margin_mean": 46.466087341308594, "margin_dpo/margin_std": 62.907737731933594, "step": 318 }, { "KL/chosen_KL_mean": -21.974462509155273, "KL/mean": -47.03016662597656, "KL/rejected_KL_mean": -72.08587646484375, "KL/std": 57.22808074951172, "epoch": 0.6680628272251309, "fcm_dpo/beta": 0.008105763234198093, "fcm_dpo/delta": -0.0071238186210393906, "fcm_dpo/margin": 50.111412048339844, "fcm_dpo/q_t": 0.4045659303665161, "grad_norm": 28.36729621887207, "learning_rate": 1.5116687323334464e-07, "logits/chosen": 1.3648066520690918, "logits/rejected": 1.3974896669387817, "logps/chosen": -315.36077880859375, "logps/ref_chosen": -293.3863220214844, "logps/ref_rejected": -238.86026000976562, "logps/rejected": -310.9461364746094, "loss": 4.3064, "margin_dpo/margin_mean": 50.11140823364258, "margin_dpo/margin_std": 57.791046142578125, "step": 319 }, { "KL/chosen_KL_mean": -20.64314079284668, "KL/mean": -44.01919937133789, "KL/rejected_KL_mean": -67.39525604248047, "KL/std": 57.318016052246094, "epoch": 0.6701570680628273, "fcm_dpo/beta": 0.008033184334635735, "fcm_dpo/delta": -0.04559609666466713, "fcm_dpo/margin": 46.752113342285156, "fcm_dpo/q_t": 0.4125458002090454, "grad_norm": 42.626216888427734, "learning_rate": 1.4948791099758052e-07, "logits/chosen": 1.6326038837432861, "logits/rejected": 1.6842557191848755, "logps/chosen": -316.14697265625, "logps/ref_chosen": -295.5038146972656, "logps/ref_rejected": -278.524169921875, "logps/rejected": -345.91943359375, "loss": 4.5235, "margin_dpo/margin_mean": 46.752113342285156, "margin_dpo/margin_std": 71.97604370117188, "step": 320 }, { "KL/chosen_KL_mean": -23.548904418945312, "KL/mean": -43.685184478759766, "KL/rejected_KL_mean": -63.82146453857422, "KL/std": 57.974605560302734, "epoch": 0.6722513089005235, "fcm_dpo/beta": 0.00815974734723568, "fcm_dpo/delta": 0.04450944438576698, "fcm_dpo/margin": 40.27255630493164, "fcm_dpo/q_t": 0.4240015149116516, "grad_norm": 26.314739227294922, "learning_rate": 1.478143389201113e-07, "logits/chosen": 1.7631984949111938, "logits/rejected": 1.7144174575805664, "logps/chosen": -304.3042297363281, "logps/ref_chosen": -280.7553405761719, "logps/ref_rejected": -237.47286987304688, "logps/rejected": -301.2943420410156, "loss": 4.6297, "margin_dpo/margin_mean": 40.272560119628906, "margin_dpo/margin_std": 66.70619201660156, "step": 321 }, { "KL/chosen_KL_mean": -22.824190139770508, "KL/mean": -47.798336029052734, "KL/rejected_KL_mean": -72.77247619628906, "KL/std": 59.09068298339844, "epoch": 0.6743455497382199, "fcm_dpo/beta": 0.008056925609707832, "fcm_dpo/delta": -0.038321636617183685, "fcm_dpo/margin": 49.94829177856445, "fcm_dpo/q_t": 0.4062731862068176, "grad_norm": 44.12809371948242, "learning_rate": 1.461462467495284e-07, "logits/chosen": 1.397947072982788, "logits/rejected": 1.4019372463226318, "logps/chosen": -321.9154968261719, "logps/ref_chosen": -299.09130859375, "logps/ref_rejected": -268.9637756347656, "logps/rejected": -341.7362365722656, "loss": 4.3814, "margin_dpo/margin_mean": 49.94829177856445, "margin_dpo/margin_std": 66.34510040283203, "step": 322 }, { "KL/chosen_KL_mean": -28.13748550415039, "KL/mean": -45.56864547729492, "KL/rejected_KL_mean": -62.999813079833984, "KL/std": 56.300010681152344, "epoch": 0.6764397905759162, "fcm_dpo/beta": 0.007967677898705006, "fcm_dpo/delta": 0.022964956238865852, "fcm_dpo/margin": 34.86231994628906, "fcm_dpo/q_t": 0.4358903765678406, "grad_norm": 35.1325798034668, "learning_rate": 1.4448372394055246e-07, "logits/chosen": 1.5975879430770874, "logits/rejected": 1.5509735345840454, "logps/chosen": -304.8335266113281, "logps/ref_chosen": -276.696044921875, "logps/ref_rejected": -225.78790283203125, "logps/rejected": -288.7877197265625, "loss": 4.8489, "margin_dpo/margin_mean": 34.86231994628906, "margin_dpo/margin_std": 71.83058166503906, "step": 323 }, { "KL/chosen_KL_mean": -16.53660011291504, "KL/mean": -44.17595291137695, "KL/rejected_KL_mean": -71.8153076171875, "KL/std": 57.79463577270508, "epoch": 0.6785340314136126, "fcm_dpo/beta": 0.007844395935535431, "fcm_dpo/delta": -0.03678688779473305, "fcm_dpo/margin": 55.27870178222656, "fcm_dpo/q_t": 0.39868438243865967, "grad_norm": 50.061527252197266, "learning_rate": 1.428268596492364e-07, "logits/chosen": 1.36971914768219, "logits/rejected": 1.4502313137054443, "logps/chosen": -247.12161254882812, "logps/ref_chosen": -230.58499145507812, "logps/ref_rejected": -229.8074493408203, "logps/rejected": -301.6227722167969, "loss": 4.2292, "margin_dpo/margin_mean": 55.27870178222656, "margin_dpo/margin_std": 59.602325439453125, "step": 324 }, { "KL/chosen_KL_mean": -20.096792221069336, "KL/mean": -42.533958435058594, "KL/rejected_KL_mean": -64.97111511230469, "KL/std": 61.21430206298828, "epoch": 0.680628272251309, "fcm_dpo/beta": 0.007813180796802044, "fcm_dpo/delta": 0.014284295961260796, "fcm_dpo/margin": 44.87432861328125, "fcm_dpo/q_t": 0.4205002188682556, "grad_norm": 40.60066604614258, "learning_rate": 1.4117574272818386e-07, "logits/chosen": 1.3730337619781494, "logits/rejected": 1.4071730375289917, "logps/chosen": -290.6528015136719, "logps/ref_chosen": -270.5559997558594, "logps/ref_rejected": -266.328369140625, "logps/rejected": -331.2994689941406, "loss": 4.6294, "margin_dpo/margin_mean": 44.87432861328125, "margin_dpo/margin_std": 76.99175262451172, "step": 325 }, { "KL/chosen_KL_mean": -17.595491409301758, "KL/mean": -40.99762725830078, "KL/rejected_KL_mean": -64.3997573852539, "KL/std": 55.9461669921875, "epoch": 0.6827225130890052, "fcm_dpo/beta": 0.008045146241784096, "fcm_dpo/delta": 0.023972200229763985, "fcm_dpo/margin": 46.80427932739258, "fcm_dpo/q_t": 0.4126897156238556, "grad_norm": 40.11039733886719, "learning_rate": 1.3953046172178413e-07, "logits/chosen": 1.284023642539978, "logits/rejected": 1.3685574531555176, "logps/chosen": -250.83407592773438, "logps/ref_chosen": -233.2386016845703, "logps/ref_rejected": -258.154541015625, "logps/rejected": -322.5542907714844, "loss": 4.5018, "margin_dpo/margin_mean": 46.80427932739258, "margin_dpo/margin_std": 70.84291076660156, "step": 326 }, { "KL/chosen_KL_mean": -8.933675765991211, "KL/mean": -34.981788635253906, "KL/rejected_KL_mean": -61.029911041259766, "KL/std": 56.793663024902344, "epoch": 0.6848167539267016, "fcm_dpo/beta": 0.008016958832740784, "fcm_dpo/delta": -0.019539086148142815, "fcm_dpo/margin": 52.096229553222656, "fcm_dpo/q_t": 0.4032677114009857, "grad_norm": 28.974864959716797, "learning_rate": 1.3789110486146468e-07, "logits/chosen": 1.364997148513794, "logits/rejected": 1.3957648277282715, "logps/chosen": -277.4798583984375, "logps/ref_chosen": -268.5461730957031, "logps/ref_rejected": -266.5316467285156, "logps/rejected": -327.5615234375, "loss": 4.3732, "margin_dpo/margin_mean": 52.096229553222656, "margin_dpo/margin_std": 69.97504425048828, "step": 327 }, { "KL/chosen_KL_mean": -21.03065299987793, "KL/mean": -43.62373352050781, "KL/rejected_KL_mean": -66.21681213378906, "KL/std": 57.95714569091797, "epoch": 0.6869109947643979, "fcm_dpo/beta": 0.008020462468266487, "fcm_dpo/delta": 0.03876934573054314, "fcm_dpo/margin": 45.1861572265625, "fcm_dpo/q_t": 0.41611042618751526, "grad_norm": 53.43826675415039, "learning_rate": 1.362577600609588e-07, "logits/chosen": 1.451033353805542, "logits/rejected": 1.4784150123596191, "logps/chosen": -315.90740966796875, "logps/ref_chosen": -294.8768005371094, "logps/ref_rejected": -281.09771728515625, "logps/rejected": -347.3145446777344, "loss": 4.5141, "margin_dpo/margin_mean": 45.1861572265625, "margin_dpo/margin_std": 66.3680648803711, "step": 328 }, { "KL/chosen_KL_mean": -28.838703155517578, "KL/mean": -48.4996337890625, "KL/rejected_KL_mean": -68.16056823730469, "KL/std": 63.01567840576172, "epoch": 0.6890052356020943, "fcm_dpo/beta": 0.008267353288829327, "fcm_dpo/delta": 0.016651635989546776, "fcm_dpo/margin": 39.321868896484375, "fcm_dpo/q_t": 0.42634207010269165, "grad_norm": 32.08344268798828, "learning_rate": 1.3463051491159093e-07, "logits/chosen": 1.3955703973770142, "logits/rejected": 1.4034247398376465, "logps/chosen": -341.136474609375, "logps/ref_chosen": -312.2977600097656, "logps/ref_rejected": -304.6536560058594, "logps/rejected": -372.814208984375, "loss": 4.7474, "margin_dpo/margin_mean": 39.321868896484375, "margin_dpo/margin_std": 76.04838562011719, "step": 329 }, { "KL/chosen_KL_mean": -25.502676010131836, "KL/mean": -42.34815216064453, "KL/rejected_KL_mean": -59.193626403808594, "KL/std": 58.84164810180664, "epoch": 0.6910994764397905, "fcm_dpo/beta": 0.008485514670610428, "fcm_dpo/delta": 0.038778625428676605, "fcm_dpo/margin": 33.690956115722656, "fcm_dpo/q_t": 0.43506842851638794, "grad_norm": 48.36201095581055, "learning_rate": 1.3300945667758012e-07, "logits/chosen": 1.2198834419250488, "logits/rejected": 1.2783613204956055, "logps/chosen": -319.7068786621094, "logps/ref_chosen": -294.2042236328125, "logps/ref_rejected": -296.2102355957031, "logps/rejected": -355.4039001464844, "loss": 4.8353, "margin_dpo/margin_mean": 33.69095230102539, "margin_dpo/margin_std": 71.08140563964844, "step": 330 }, { "KL/chosen_KL_mean": -18.539098739624023, "KL/mean": -39.45647430419922, "KL/rejected_KL_mean": -60.37385559082031, "KL/std": 58.204124450683594, "epoch": 0.6931937172774869, "fcm_dpo/beta": 0.008582251146435738, "fcm_dpo/delta": 0.0032618592958897352, "fcm_dpo/margin": 41.83475112915039, "fcm_dpo/q_t": 0.4177251160144806, "grad_norm": 35.6547737121582, "learning_rate": 1.3139467229135998e-07, "logits/chosen": 1.4187893867492676, "logits/rejected": 1.4320850372314453, "logps/chosen": -286.85174560546875, "logps/ref_chosen": -268.3126525878906, "logps/ref_rejected": -255.5543212890625, "logps/rejected": -315.92816162109375, "loss": 4.5625, "margin_dpo/margin_mean": 41.834747314453125, "margin_dpo/margin_std": 66.47994995117188, "step": 331 }, { "KL/chosen_KL_mean": -19.064308166503906, "KL/mean": -45.1978645324707, "KL/rejected_KL_mean": -71.33141326904297, "KL/std": 59.10099411010742, "epoch": 0.6952879581151833, "fcm_dpo/beta": 0.00829013716429472, "fcm_dpo/delta": -0.03619233891367912, "fcm_dpo/margin": 52.2671012878418, "fcm_dpo/q_t": 0.4004819691181183, "grad_norm": 35.77095031738281, "learning_rate": 1.2978624834891626e-07, "logits/chosen": 1.3465325832366943, "logits/rejected": 1.3774622678756714, "logps/chosen": -282.8147277832031, "logps/ref_chosen": -263.75042724609375, "logps/ref_rejected": -234.5144500732422, "logps/rejected": -305.8458251953125, "loss": 4.3568, "margin_dpo/margin_mean": 52.2671012878418, "margin_dpo/margin_std": 72.7847900390625, "step": 332 }, { "KL/chosen_KL_mean": -18.448471069335938, "KL/mean": -38.11326599121094, "KL/rejected_KL_mean": -57.778076171875, "KL/std": 56.81516647338867, "epoch": 0.6973821989528796, "fcm_dpo/beta": 0.008496787399053574, "fcm_dpo/delta": 0.03937835618853569, "fcm_dpo/margin": 39.329593658447266, "fcm_dpo/q_t": 0.42261365056037903, "grad_norm": 51.38772964477539, "learning_rate": 1.281842711051438e-07, "logits/chosen": 1.4903069734573364, "logits/rejected": 1.4716383218765259, "logps/chosen": -305.264892578125, "logps/ref_chosen": -286.81646728515625, "logps/ref_rejected": -260.4169616699219, "logps/rejected": -318.1950378417969, "loss": 4.6185, "margin_dpo/margin_mean": 39.3296012878418, "margin_dpo/margin_std": 64.0001449584961, "step": 333 }, { "KL/chosen_KL_mean": -16.50014305114746, "KL/mean": -40.04962158203125, "KL/rejected_KL_mean": -63.59909439086914, "KL/std": 59.867618560791016, "epoch": 0.6994764397905759, "fcm_dpo/beta": 0.008586030453443527, "fcm_dpo/delta": -0.03872523456811905, "fcm_dpo/margin": 47.09894561767578, "fcm_dpo/q_t": 0.40821734070777893, "grad_norm": 45.0473747253418, "learning_rate": 1.2658882646922033e-07, "logits/chosen": 1.2563035488128662, "logits/rejected": 1.2555757761001587, "logps/chosen": -309.8110656738281, "logps/ref_chosen": -293.3109130859375, "logps/ref_rejected": -264.1449279785156, "logps/rejected": -327.7440185546875, "loss": 4.525, "margin_dpo/margin_mean": 47.09895324707031, "margin_dpo/margin_std": 74.84980773925781, "step": 334 }, { "KL/chosen_KL_mean": -29.5706729888916, "KL/mean": -50.683982849121094, "KL/rejected_KL_mean": -71.79728698730469, "KL/std": 58.2430305480957, "epoch": 0.7015706806282722, "fcm_dpo/beta": 0.008247178047895432, "fcm_dpo/delta": -0.010653029195964336, "fcm_dpo/margin": 42.226619720458984, "fcm_dpo/q_t": 0.4203599989414215, "grad_norm": 31.517383575439453, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 1.1738187074661255, "logits/rejected": 1.2567086219787598, "logps/chosen": -299.3270263671875, "logps/ref_chosen": -269.75634765625, "logps/ref_rejected": -241.08438110351562, "logps/rejected": -312.88165283203125, "loss": 4.6694, "margin_dpo/margin_mean": 42.226619720458984, "margin_dpo/margin_std": 74.86090850830078, "step": 335 }, { "KL/chosen_KL_mean": -22.192955017089844, "KL/mean": -42.18242645263672, "KL/rejected_KL_mean": -62.171897888183594, "KL/std": 58.28977584838867, "epoch": 0.7036649214659686, "fcm_dpo/beta": 0.00838224682956934, "fcm_dpo/delta": 0.029553044587373734, "fcm_dpo/margin": 39.978946685791016, "fcm_dpo/q_t": 0.4237605035305023, "grad_norm": 39.035865783691406, "learning_rate": 1.2341787690142435e-07, "logits/chosen": 1.3819284439086914, "logits/rejected": 1.4114774465560913, "logps/chosen": -259.56475830078125, "logps/ref_chosen": -237.37179565429688, "logps/ref_rejected": -253.5463409423828, "logps/rejected": -315.71826171875, "loss": 4.6372, "margin_dpo/margin_mean": 39.978946685791016, "margin_dpo/margin_std": 67.77619171142578, "step": 336 }, { "KL/chosen_KL_mean": -26.687501907348633, "KL/mean": -49.499855041503906, "KL/rejected_KL_mean": -72.31220245361328, "KL/std": 57.879478454589844, "epoch": 0.7057591623036649, "fcm_dpo/beta": 0.008477190509438515, "fcm_dpo/delta": -0.020977266132831573, "fcm_dpo/margin": 45.62470626831055, "fcm_dpo/q_t": 0.4121153652667999, "grad_norm": 46.522640228271484, "learning_rate": 1.2184254201795363e-07, "logits/chosen": 1.2859256267547607, "logits/rejected": 1.3341453075408936, "logps/chosen": -287.9951477050781, "logps/ref_chosen": -261.3076477050781, "logps/ref_rejected": -252.3170166015625, "logps/rejected": -324.6291809082031, "loss": 4.5192, "margin_dpo/margin_mean": 45.62470626831055, "margin_dpo/margin_std": 72.3380126953125, "step": 337 }, { "KL/chosen_KL_mean": -11.782232284545898, "KL/mean": -39.000064849853516, "KL/rejected_KL_mean": -66.21788787841797, "KL/std": 60.04515838623047, "epoch": 0.7078534031413612, "fcm_dpo/beta": 0.008186925202608109, "fcm_dpo/delta": -0.04778764396905899, "fcm_dpo/margin": 54.43565368652344, "fcm_dpo/q_t": 0.39773881435394287, "grad_norm": 27.990068435668945, "learning_rate": 1.202740798300168e-07, "logits/chosen": 1.5228660106658936, "logits/rejected": 1.5129181146621704, "logps/chosen": -280.553955078125, "logps/ref_chosen": -268.7717590332031, "logps/ref_rejected": -232.3352813720703, "logps/rejected": -298.55316162109375, "loss": 4.2827, "margin_dpo/margin_mean": 54.43565368652344, "margin_dpo/margin_std": 70.96080780029297, "step": 338 }, { "KL/chosen_KL_mean": -20.091079711914062, "KL/mean": -47.560150146484375, "KL/rejected_KL_mean": -75.02922058105469, "KL/std": 61.880332946777344, "epoch": 0.7099476439790576, "fcm_dpo/beta": 0.007898853160440922, "fcm_dpo/delta": -0.035964664071798325, "fcm_dpo/margin": 54.938148498535156, "fcm_dpo/q_t": 0.3994678258895874, "grad_norm": 42.823509216308594, "learning_rate": 1.1871257444948096e-07, "logits/chosen": 1.256564736366272, "logits/rejected": 1.224652647972107, "logps/chosen": -313.7042236328125, "logps/ref_chosen": -293.6131286621094, "logps/ref_rejected": -281.87738037109375, "logps/rejected": -356.9066162109375, "loss": 4.3348, "margin_dpo/margin_mean": 54.938148498535156, "margin_dpo/margin_std": 74.72753143310547, "step": 339 }, { "KL/chosen_KL_mean": -25.699125289916992, "KL/mean": -48.140480041503906, "KL/rejected_KL_mean": -70.58184814453125, "KL/std": 61.5643424987793, "epoch": 0.7120418848167539, "fcm_dpo/beta": 0.00794089213013649, "fcm_dpo/delta": 0.04462888091802597, "fcm_dpo/margin": 44.882720947265625, "fcm_dpo/q_t": 0.41869547963142395, "grad_norm": 25.24543571472168, "learning_rate": 1.1715810961514072e-07, "logits/chosen": 0.9644868969917297, "logits/rejected": 1.0749053955078125, "logps/chosen": -277.4561767578125, "logps/ref_chosen": -251.75704956054688, "logps/ref_rejected": -255.76441955566406, "logps/rejected": -326.34625244140625, "loss": 4.5771, "margin_dpo/margin_mean": 44.88272476196289, "margin_dpo/margin_std": 71.99385070800781, "step": 340 }, { "KL/chosen_KL_mean": -38.51675033569336, "KL/mean": -56.909446716308594, "KL/rejected_KL_mean": -75.30213928222656, "KL/std": 64.48269653320312, "epoch": 0.7141361256544503, "fcm_dpo/beta": 0.008220081217586994, "fcm_dpo/delta": 0.029346494004130363, "fcm_dpo/margin": 36.785396575927734, "fcm_dpo/q_t": 0.43140214681625366, "grad_norm": 49.63078689575195, "learning_rate": 1.1561076868822755e-07, "logits/chosen": 1.3204576969146729, "logits/rejected": 1.4248378276824951, "logps/chosen": -345.1643981933594, "logps/ref_chosen": -306.64764404296875, "logps/ref_rejected": -301.4248352050781, "logps/rejected": -376.72698974609375, "loss": 4.8246, "margin_dpo/margin_mean": 36.78539276123047, "margin_dpo/margin_std": 76.30755615234375, "step": 341 }, { "KL/chosen_KL_mean": -22.643159866333008, "KL/mean": -45.68241882324219, "KL/rejected_KL_mean": -68.7216796875, "KL/std": 61.156394958496094, "epoch": 0.7162303664921466, "fcm_dpo/beta": 0.00822380930185318, "fcm_dpo/delta": -0.035644009709358215, "fcm_dpo/margin": 46.07852554321289, "fcm_dpo/q_t": 0.41292059421539307, "grad_norm": 28.56557846069336, "learning_rate": 1.1407063464793965e-07, "logits/chosen": 1.4937210083007812, "logits/rejected": 1.5514423847198486, "logps/chosen": -285.31207275390625, "logps/ref_chosen": -262.6689147949219, "logps/ref_rejected": -260.8409118652344, "logps/rejected": -329.5625915527344, "loss": 4.4882, "margin_dpo/margin_mean": 46.078521728515625, "margin_dpo/margin_std": 67.86953735351562, "step": 342 }, { "KL/chosen_KL_mean": -23.73531723022461, "KL/mean": -44.780399322509766, "KL/rejected_KL_mean": -65.82548522949219, "KL/std": 60.82894515991211, "epoch": 0.7183246073298429, "fcm_dpo/beta": 0.008099392056465149, "fcm_dpo/delta": 0.030330460518598557, "fcm_dpo/margin": 42.09016799926758, "fcm_dpo/q_t": 0.4220288395881653, "grad_norm": 30.66365623474121, "learning_rate": 1.125377900869913e-07, "logits/chosen": 1.606204867362976, "logits/rejected": 1.6237492561340332, "logps/chosen": -323.36126708984375, "logps/ref_chosen": -299.6259765625, "logps/ref_rejected": -261.2701416015625, "logps/rejected": -327.09564208984375, "loss": 4.6368, "margin_dpo/margin_mean": 42.09016418457031, "margin_dpo/margin_std": 72.11952209472656, "step": 343 }, { "KL/chosen_KL_mean": -24.897951126098633, "KL/mean": -49.33934783935547, "KL/rejected_KL_mean": -73.7807388305664, "KL/std": 62.17938232421875, "epoch": 0.7204188481675393, "fcm_dpo/beta": 0.00840503815561533, "fcm_dpo/delta": -0.0125031853094697, "fcm_dpo/margin": 48.882781982421875, "fcm_dpo/q_t": 0.4062635004520416, "grad_norm": 35.609527587890625, "learning_rate": 1.110123172071844e-07, "logits/chosen": 1.2275300025939941, "logits/rejected": 1.2526569366455078, "logps/chosen": -315.8483581542969, "logps/ref_chosen": -290.950439453125, "logps/ref_rejected": -255.15374755859375, "logps/rejected": -328.93450927734375, "loss": 4.4319, "margin_dpo/margin_mean": 48.882781982421875, "margin_dpo/margin_std": 70.47264099121094, "step": 344 }, { "KL/chosen_KL_mean": -21.042957305908203, "KL/mean": -41.27716827392578, "KL/rejected_KL_mean": -61.51138687133789, "KL/std": 59.13972091674805, "epoch": 0.7225130890052356, "fcm_dpo/beta": 0.00830150581896305, "fcm_dpo/delta": 0.030441733077168465, "fcm_dpo/margin": 40.46842956542969, "fcm_dpo/q_t": 0.42245370149612427, "grad_norm": 37.91876220703125, "learning_rate": 1.09494297815e-07, "logits/chosen": 1.3884456157684326, "logits/rejected": 1.4642207622528076, "logps/chosen": -294.308837890625, "logps/ref_chosen": -273.2658996582031, "logps/ref_rejected": -263.9664306640625, "logps/rejected": -325.4778137207031, "loss": 4.5937, "margin_dpo/margin_mean": 40.46843338012695, "margin_dpo/margin_std": 62.32899856567383, "step": 345 }, { "KL/chosen_KL_mean": -33.96916198730469, "KL/mean": -57.84877014160156, "KL/rejected_KL_mean": -81.7283706665039, "KL/std": 61.43087387084961, "epoch": 0.724607329842932, "fcm_dpo/beta": 0.007993143051862717, "fcm_dpo/delta": -0.08858863264322281, "fcm_dpo/margin": 47.75920104980469, "fcm_dpo/q_t": 0.41108354926109314, "grad_norm": 34.72679138183594, "learning_rate": 1.0798381331721107e-07, "logits/chosen": 1.302663803100586, "logits/rejected": 1.3537399768829346, "logps/chosen": -296.65130615234375, "logps/ref_chosen": -262.6821594238281, "logps/ref_rejected": -226.46859741210938, "logps/rejected": -308.19696044921875, "loss": 4.4971, "margin_dpo/margin_mean": 47.75920867919922, "margin_dpo/margin_std": 67.78038024902344, "step": 346 }, { "KL/chosen_KL_mean": -30.05909538269043, "KL/mean": -51.16781234741211, "KL/rejected_KL_mean": -72.27653503417969, "KL/std": 61.078372955322266, "epoch": 0.7267015706806282, "fcm_dpo/beta": 0.007945908233523369, "fcm_dpo/delta": 0.0373678132891655, "fcm_dpo/margin": 42.21742630004883, "fcm_dpo/q_t": 0.4221636950969696, "grad_norm": 30.520584106445312, "learning_rate": 1.0648094471651722e-07, "logits/chosen": 1.356184720993042, "logits/rejected": 1.3794606924057007, "logps/chosen": -293.8836669921875, "logps/ref_chosen": -263.8246154785156, "logps/ref_rejected": -237.9121551513672, "logps/rejected": -310.188720703125, "loss": 4.6194, "margin_dpo/margin_mean": 42.217430114746094, "margin_dpo/margin_std": 69.29716491699219, "step": 347 }, { "KL/chosen_KL_mean": -23.193214416503906, "KL/mean": -41.58753204345703, "KL/rejected_KL_mean": -59.98184585571289, "KL/std": 60.363502502441406, "epoch": 0.7287958115183246, "fcm_dpo/beta": 0.008062899112701416, "fcm_dpo/delta": 0.030613910406827927, "fcm_dpo/margin": 36.78863525390625, "fcm_dpo/q_t": 0.43188756704330444, "grad_norm": 31.7449951171875, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 1.1868278980255127, "logits/rejected": 1.2123560905456543, "logps/chosen": -301.3465881347656, "logps/ref_chosen": -278.1533203125, "logps/ref_rejected": -263.39691162109375, "logps/rejected": -323.3787536621094, "loss": 4.7326, "margin_dpo/margin_mean": 36.788631439208984, "margin_dpo/margin_std": 64.37882995605469, "step": 348 }, { "KL/chosen_KL_mean": -26.564422607421875, "KL/mean": -52.15089416503906, "KL/rejected_KL_mean": -77.73736572265625, "KL/std": 64.47464752197266, "epoch": 0.7308900523560209, "fcm_dpo/beta": 0.00800681859254837, "fcm_dpo/delta": -0.05323091894388199, "fcm_dpo/margin": 51.17293930053711, "fcm_dpo/q_t": 0.40616288781166077, "grad_norm": 27.40471649169922, "learning_rate": 1.0349837717080347e-07, "logits/chosen": 1.6410548686981201, "logits/rejected": 1.6932549476623535, "logps/chosen": -345.01953125, "logps/ref_chosen": -318.455078125, "logps/ref_rejected": -288.5691833496094, "logps/rejected": -366.3065185546875, "loss": 4.4206, "margin_dpo/margin_mean": 51.172943115234375, "margin_dpo/margin_std": 72.1238784790039, "step": 349 }, { "KL/chosen_KL_mean": -29.941728591918945, "KL/mean": -52.43879318237305, "KL/rejected_KL_mean": -74.93585205078125, "KL/std": 63.54440689086914, "epoch": 0.7329842931937173, "fcm_dpo/beta": 0.00789142120629549, "fcm_dpo/delta": -0.034058570861816406, "fcm_dpo/margin": 44.99413299560547, "fcm_dpo/q_t": 0.41845959424972534, "grad_norm": 35.786888122558594, "learning_rate": 1.0201883817182949e-07, "logits/chosen": 1.4704442024230957, "logits/rejected": 1.4845540523529053, "logps/chosen": -314.13092041015625, "logps/ref_chosen": -284.1891784667969, "logps/ref_rejected": -247.94589233398438, "logps/rejected": -322.8817443847656, "loss": 4.616, "margin_dpo/margin_mean": 44.99413299560547, "margin_dpo/margin_std": 74.6503677368164, "step": 350 }, { "KL/chosen_KL_mean": -27.57063102722168, "KL/mean": -43.758853912353516, "KL/rejected_KL_mean": -59.947086334228516, "KL/std": 60.97875213623047, "epoch": 0.7350785340314137, "fcm_dpo/beta": 0.0077229635789990425, "fcm_dpo/delta": 0.0042969174683094025, "fcm_dpo/margin": 32.376461029052734, "fcm_dpo/q_t": 0.4431162476539612, "grad_norm": 37.373355865478516, "learning_rate": 1.0054723495346482e-07, "logits/chosen": 1.3978153467178345, "logits/rejected": 1.3665990829467773, "logps/chosen": -330.87872314453125, "logps/ref_chosen": -303.30810546875, "logps/ref_rejected": -260.1866455078125, "logps/rejected": -320.13372802734375, "loss": 4.9431, "margin_dpo/margin_mean": 32.376461029052734, "margin_dpo/margin_std": 73.97547149658203, "step": 351 }, { "KL/chosen_KL_mean": -24.06132698059082, "KL/mean": -49.409461975097656, "KL/rejected_KL_mean": -74.75761413574219, "KL/std": 62.60258865356445, "epoch": 0.7371727748691099, "fcm_dpo/beta": 0.007590837776660919, "fcm_dpo/delta": -0.04276890680193901, "fcm_dpo/margin": 50.69627380371094, "fcm_dpo/q_t": 0.4107002317905426, "grad_norm": 30.83437156677246, "learning_rate": 9.908364643332398e-08, "logits/chosen": 1.3391189575195312, "logits/rejected": 1.5173217058181763, "logps/chosen": -270.8684387207031, "logps/ref_chosen": -246.80709838867188, "logps/ref_rejected": -253.13223266601562, "logps/rejected": -327.8898620605469, "loss": 4.4599, "margin_dpo/margin_mean": 50.69627380371094, "margin_dpo/margin_std": 70.11902618408203, "step": 352 }, { "KL/chosen_KL_mean": -26.750572204589844, "KL/mean": -52.218353271484375, "KL/rejected_KL_mean": -77.6861343383789, "KL/std": 61.98667526245117, "epoch": 0.7392670157068063, "fcm_dpo/beta": 0.007327005732804537, "fcm_dpo/delta": -0.033968254923820496, "fcm_dpo/margin": 50.9355583190918, "fcm_dpo/q_t": 0.41317909955978394, "grad_norm": 31.25110626220703, "learning_rate": 9.76281510992176e-08, "logits/chosen": 1.1865570545196533, "logits/rejected": 1.2616384029388428, "logps/chosen": -288.3520202636719, "logps/ref_chosen": -261.6014709472656, "logps/ref_rejected": -262.74761962890625, "logps/rejected": -340.4337463378906, "loss": 4.4683, "margin_dpo/margin_mean": 50.9355583190918, "margin_dpo/margin_std": 70.31192016601562, "step": 353 }, { "KL/chosen_KL_mean": -27.61041259765625, "KL/mean": -42.82272720336914, "KL/rejected_KL_mean": -58.0350456237793, "KL/std": 61.23716354370117, "epoch": 0.7413612565445026, "fcm_dpo/beta": 0.007398069370537996, "fcm_dpo/delta": 0.042706623673439026, "fcm_dpo/margin": 30.42462158203125, "fcm_dpo/q_t": 0.4486212134361267, "grad_norm": 37.50423049926758, "learning_rate": 9.618082700494318e-08, "logits/chosen": 1.2743968963623047, "logits/rejected": 1.3830703496932983, "logps/chosen": -279.75885009765625, "logps/ref_chosen": -252.14840698242188, "logps/ref_rejected": -244.1017608642578, "logps/rejected": -302.13677978515625, "loss": 5.0227, "margin_dpo/margin_mean": 30.42462158203125, "margin_dpo/margin_std": 76.54853820800781, "step": 354 }, { "KL/chosen_KL_mean": -28.34946060180664, "KL/mean": -53.75284194946289, "KL/rejected_KL_mean": -79.15621948242188, "KL/std": 63.44832992553711, "epoch": 0.743455497382199, "fcm_dpo/beta": 0.007384137716144323, "fcm_dpo/delta": -0.009487075731158257, "fcm_dpo/margin": 50.806766510009766, "fcm_dpo/q_t": 0.41258612275123596, "grad_norm": 30.895038604736328, "learning_rate": 9.474175176609956e-08, "logits/chosen": 1.4611732959747314, "logits/rejected": 1.5832594633102417, "logps/chosen": -314.5119934082031, "logps/ref_chosen": -286.16253662109375, "logps/ref_rejected": -272.3275146484375, "logps/rejected": -351.4837646484375, "loss": 4.481, "margin_dpo/margin_mean": 50.806766510009766, "margin_dpo/margin_std": 72.48587036132812, "step": 355 }, { "KL/chosen_KL_mean": -21.091541290283203, "KL/mean": -41.81569290161133, "KL/rejected_KL_mean": -62.53986358642578, "KL/std": 58.343475341796875, "epoch": 0.7455497382198953, "fcm_dpo/beta": 0.007514188997447491, "fcm_dpo/delta": 0.029240809381008148, "fcm_dpo/margin": 41.44832229614258, "fcm_dpo/q_t": 0.42722052335739136, "grad_norm": 34.9584846496582, "learning_rate": 9.331100255592436e-08, "logits/chosen": 1.315157175064087, "logits/rejected": 1.3648688793182373, "logps/chosen": -219.2443084716797, "logps/ref_chosen": -198.1527862548828, "logps/ref_rejected": -209.80569458007812, "logps/rejected": -272.3455505371094, "loss": 4.6401, "margin_dpo/margin_mean": 41.44831848144531, "margin_dpo/margin_std": 64.64385986328125, "step": 356 }, { "KL/chosen_KL_mean": -31.06587028503418, "KL/mean": -53.74104309082031, "KL/rejected_KL_mean": -76.41620635986328, "KL/std": 59.96897888183594, "epoch": 0.7476439790575916, "fcm_dpo/beta": 0.007646047510206699, "fcm_dpo/delta": 0.012101225554943085, "fcm_dpo/margin": 45.350341796875, "fcm_dpo/q_t": 0.4210367798805237, "grad_norm": 38.74076461791992, "learning_rate": 9.18886561011557e-08, "logits/chosen": 1.3575685024261475, "logits/rejected": 1.412861943244934, "logps/chosen": -287.5730285644531, "logps/ref_chosen": -256.50714111328125, "logps/ref_rejected": -237.40769958496094, "logps/rejected": -313.82391357421875, "loss": 4.6022, "margin_dpo/margin_mean": 45.350341796875, "margin_dpo/margin_std": 73.01123809814453, "step": 357 }, { "KL/chosen_KL_mean": -20.08008575439453, "KL/mean": -45.896324157714844, "KL/rejected_KL_mean": -71.71257019042969, "KL/std": 62.05603790283203, "epoch": 0.749738219895288, "fcm_dpo/beta": 0.007647480815649033, "fcm_dpo/delta": -0.023323874920606613, "fcm_dpo/margin": 51.63248825073242, "fcm_dpo/q_t": 0.40921440720558167, "grad_norm": 26.73249626159668, "learning_rate": 9.047478867791731e-08, "logits/chosen": 1.5963877439498901, "logits/rejected": 1.5494128465652466, "logps/chosen": -312.5273132324219, "logps/ref_chosen": -292.447265625, "logps/ref_rejected": -255.47872924804688, "logps/rejected": -327.1913146972656, "loss": 4.4678, "margin_dpo/margin_mean": 51.63249588012695, "margin_dpo/margin_std": 76.36697387695312, "step": 358 }, { "KL/chosen_KL_mean": -20.328577041625977, "KL/mean": -44.36174774169922, "KL/rejected_KL_mean": -68.39491271972656, "KL/std": 62.28682327270508, "epoch": 0.7518324607329843, "fcm_dpo/beta": 0.007573855575174093, "fcm_dpo/delta": 0.012442296370863914, "fcm_dpo/margin": 48.06633377075195, "fcm_dpo/q_t": 0.4150635302066803, "grad_norm": 29.436304092407227, "learning_rate": 8.906947610762825e-08, "logits/chosen": 1.5220872163772583, "logits/rejected": 1.492361068725586, "logps/chosen": -314.08856201171875, "logps/ref_chosen": -293.7599792480469, "logps/ref_rejected": -259.4142761230469, "logps/rejected": -327.8092041015625, "loss": 4.4952, "margin_dpo/margin_mean": 48.06633377075195, "margin_dpo/margin_std": 68.04971313476562, "step": 359 }, { "KL/chosen_KL_mean": -29.602924346923828, "KL/mean": -50.32772445678711, "KL/rejected_KL_mean": -71.05253601074219, "KL/std": 62.25556564331055, "epoch": 0.7539267015706806, "fcm_dpo/beta": 0.00775428581982851, "fcm_dpo/delta": 0.03547399491071701, "fcm_dpo/margin": 41.44960021972656, "fcm_dpo/q_t": 0.4252384901046753, "grad_norm": 25.99398422241211, "learning_rate": 8.76727937529367e-08, "logits/chosen": 1.428528904914856, "logits/rejected": 1.3796058893203735, "logps/chosen": -330.51666259765625, "logps/ref_chosen": -300.91375732421875, "logps/ref_rejected": -256.4080810546875, "logps/rejected": -327.4606018066406, "loss": 4.6291, "margin_dpo/margin_mean": 41.44960021972656, "margin_dpo/margin_std": 65.40933227539062, "step": 360 }, { "KL/chosen_KL_mean": -17.207592010498047, "KL/mean": -45.10061264038086, "KL/rejected_KL_mean": -72.99363708496094, "KL/std": 63.61145782470703, "epoch": 0.7560209424083769, "fcm_dpo/beta": 0.007767850067466497, "fcm_dpo/delta": -0.0348568931221962, "fcm_dpo/margin": 55.78605270385742, "fcm_dpo/q_t": 0.40066930651664734, "grad_norm": 28.439056396484375, "learning_rate": 8.628481651367875e-08, "logits/chosen": 0.9440096616744995, "logits/rejected": 1.0340206623077393, "logps/chosen": -273.76788330078125, "logps/ref_chosen": -256.5603332519531, "logps/ref_rejected": -271.043701171875, "logps/rejected": -344.03729248046875, "loss": 4.3285, "margin_dpo/margin_mean": 55.78605270385742, "margin_dpo/margin_std": 75.44465637207031, "step": 361 }, { "KL/chosen_KL_mean": -23.494083404541016, "KL/mean": -46.04978942871094, "KL/rejected_KL_mean": -68.60549926757812, "KL/std": 58.13932800292969, "epoch": 0.7581151832460733, "fcm_dpo/beta": 0.007708454504609108, "fcm_dpo/delta": 0.01965925097465515, "fcm_dpo/margin": 45.111412048339844, "fcm_dpo/q_t": 0.4195477068424225, "grad_norm": 69.2497787475586, "learning_rate": 8.490561882286135e-08, "logits/chosen": 1.4437376260757446, "logits/rejected": 1.3562790155410767, "logps/chosen": -318.194580078125, "logps/ref_chosen": -294.700439453125, "logps/ref_rejected": -239.67922973632812, "logps/rejected": -308.28472900390625, "loss": 4.5829, "margin_dpo/margin_mean": 45.11141586303711, "margin_dpo/margin_std": 71.23416900634766, "step": 362 }, { "KL/chosen_KL_mean": -22.342344284057617, "KL/mean": -49.643497467041016, "KL/rejected_KL_mean": -76.94465637207031, "KL/std": 64.40416717529297, "epoch": 0.7602094240837697, "fcm_dpo/beta": 0.007676102686673403, "fcm_dpo/delta": -0.020748523995280266, "fcm_dpo/margin": 54.60232162475586, "fcm_dpo/q_t": 0.40338873863220215, "grad_norm": 36.314090728759766, "learning_rate": 8.353527464267104e-08, "logits/chosen": 1.3692359924316406, "logits/rejected": 1.3743070363998413, "logps/chosen": -316.0182189941406, "logps/ref_chosen": -293.67584228515625, "logps/ref_rejected": -260.85675048828125, "logps/rejected": -337.8014221191406, "loss": 4.3686, "margin_dpo/margin_mean": 54.60232162475586, "margin_dpo/margin_std": 74.3442611694336, "step": 363 }, { "KL/chosen_KL_mean": -39.35921096801758, "KL/mean": -59.600502014160156, "KL/rejected_KL_mean": -79.84181213378906, "KL/std": 62.217586517333984, "epoch": 0.762303664921466, "fcm_dpo/beta": 0.00767554622143507, "fcm_dpo/delta": 0.027282362803816795, "fcm_dpo/margin": 40.48260498046875, "fcm_dpo/q_t": 0.4288344383239746, "grad_norm": 29.953201293945312, "learning_rate": 8.217385746050742e-08, "logits/chosen": 1.0847784280776978, "logits/rejected": 1.1279144287109375, "logps/chosen": -318.2546081542969, "logps/ref_chosen": -278.8953857421875, "logps/ref_rejected": -283.8898620605469, "logps/rejected": -363.73162841796875, "loss": 4.7261, "margin_dpo/margin_mean": 40.48260498046875, "margin_dpo/margin_std": 72.49263000488281, "step": 364 }, { "KL/chosen_KL_mean": -23.04935073852539, "KL/mean": -48.64548873901367, "KL/rejected_KL_mean": -74.24163818359375, "KL/std": 63.86309051513672, "epoch": 0.7643979057591623, "fcm_dpo/beta": 0.007841559126973152, "fcm_dpo/delta": -0.0018529277294874191, "fcm_dpo/margin": 51.192283630371094, "fcm_dpo/q_t": 0.4081072509288788, "grad_norm": 41.78466796875, "learning_rate": 8.082144028504231e-08, "logits/chosen": 1.339031457901001, "logits/rejected": 1.3882323503494263, "logps/chosen": -288.6932678222656, "logps/ref_chosen": -265.6439208984375, "logps/ref_rejected": -255.40138244628906, "logps/rejected": -329.64300537109375, "loss": 4.4171, "margin_dpo/margin_mean": 51.192283630371094, "margin_dpo/margin_std": 72.82061004638672, "step": 365 }, { "KL/chosen_KL_mean": -27.14842987060547, "KL/mean": -52.876365661621094, "KL/rejected_KL_mean": -78.60430908203125, "KL/std": 63.30706024169922, "epoch": 0.7664921465968586, "fcm_dpo/beta": 0.007694230880588293, "fcm_dpo/delta": 0.003679636400192976, "fcm_dpo/margin": 51.45587158203125, "fcm_dpo/q_t": 0.40837374329566956, "grad_norm": 37.82375717163086, "learning_rate": 7.947809564230445e-08, "logits/chosen": 1.4413959980010986, "logits/rejected": 1.4565752744674683, "logps/chosen": -307.3729248046875, "logps/ref_chosen": -280.2244873046875, "logps/ref_rejected": -248.72366333007812, "logps/rejected": -327.3279724121094, "loss": 4.4214, "margin_dpo/margin_mean": 51.45586395263672, "margin_dpo/margin_std": 70.79524230957031, "step": 366 }, { "KL/chosen_KL_mean": -23.835987091064453, "KL/mean": -50.60634231567383, "KL/rejected_KL_mean": -77.37670135498047, "KL/std": 62.218441009521484, "epoch": 0.768586387434555, "fcm_dpo/beta": 0.007677263580262661, "fcm_dpo/delta": -0.01209146250039339, "fcm_dpo/margin": 53.54071807861328, "fcm_dpo/q_t": 0.40493521094322205, "grad_norm": 41.89437484741211, "learning_rate": 7.814389557179016e-08, "logits/chosen": 1.3644987344741821, "logits/rejected": 1.3721001148223877, "logps/chosen": -316.77264404296875, "logps/ref_chosen": -292.9366760253906, "logps/ref_rejected": -261.88763427734375, "logps/rejected": -339.2643127441406, "loss": 4.3442, "margin_dpo/margin_mean": 53.54071807861328, "margin_dpo/margin_std": 68.99994659423828, "step": 367 }, { "KL/chosen_KL_mean": -18.622913360595703, "KL/mean": -48.8709602355957, "KL/rejected_KL_mean": -79.11900329589844, "KL/std": 64.39937591552734, "epoch": 0.7706806282722513, "fcm_dpo/beta": 0.0074094366282224655, "fcm_dpo/delta": -0.08462084829807281, "fcm_dpo/margin": 60.49608612060547, "fcm_dpo/q_t": 0.3952578604221344, "grad_norm": 39.50826644897461, "learning_rate": 7.681891162260015e-08, "logits/chosen": 1.1243423223495483, "logits/rejected": 1.1874594688415527, "logps/chosen": -303.5286865234375, "logps/ref_chosen": -284.90576171875, "logps/ref_rejected": -256.59197998046875, "logps/rejected": -335.71099853515625, "loss": 4.2209, "margin_dpo/margin_mean": 60.49608612060547, "margin_dpo/margin_std": 68.90132141113281, "step": 368 }, { "KL/chosen_KL_mean": -23.85970687866211, "KL/mean": -45.43791198730469, "KL/rejected_KL_mean": -67.01612091064453, "KL/std": 61.029144287109375, "epoch": 0.7727748691099476, "fcm_dpo/beta": 0.007211884018033743, "fcm_dpo/delta": 0.0024794889613986015, "fcm_dpo/margin": 43.15642166137695, "fcm_dpo/q_t": 0.4275315999984741, "grad_norm": 32.8515625, "learning_rate": 7.550321484960251e-08, "logits/chosen": 1.3259495496749878, "logits/rejected": 1.3368303775787354, "logps/chosen": -299.775634765625, "logps/ref_chosen": -275.91595458984375, "logps/ref_rejected": -264.79736328125, "logps/rejected": -331.8134765625, "loss": 4.6791, "margin_dpo/margin_mean": 43.15641403198242, "margin_dpo/margin_std": 72.02017974853516, "step": 369 }, { "KL/chosen_KL_mean": -24.69597816467285, "KL/mean": -51.460182189941406, "KL/rejected_KL_mean": -78.22439575195312, "KL/std": 63.278751373291016, "epoch": 0.774869109947644, "fcm_dpo/beta": 0.007086962461471558, "fcm_dpo/delta": -0.035770583897829056, "fcm_dpo/margin": 53.528411865234375, "fcm_dpo/q_t": 0.41154617071151733, "grad_norm": 23.188228607177734, "learning_rate": 7.419687580962222e-08, "logits/chosen": 1.1783413887023926, "logits/rejected": 1.2379190921783447, "logps/chosen": -269.63800048828125, "logps/ref_chosen": -244.9420166015625, "logps/ref_rejected": -235.61978149414062, "logps/rejected": -313.8441467285156, "loss": 4.4625, "margin_dpo/margin_mean": 53.528411865234375, "margin_dpo/margin_std": 76.80668640136719, "step": 370 }, { "KL/chosen_KL_mean": -28.699094772338867, "KL/mean": -49.59718704223633, "KL/rejected_KL_mean": -70.49527740478516, "KL/std": 62.270164489746094, "epoch": 0.7769633507853403, "fcm_dpo/beta": 0.00724080391228199, "fcm_dpo/delta": 0.04666786640882492, "fcm_dpo/margin": 41.79617691040039, "fcm_dpo/q_t": 0.42907005548477173, "grad_norm": 46.07411193847656, "learning_rate": 7.289996455765748e-08, "logits/chosen": 1.2584155797958374, "logits/rejected": 1.245367169380188, "logps/chosen": -318.0489501953125, "logps/ref_chosen": -289.3498229980469, "logps/ref_rejected": -243.70205688476562, "logps/rejected": -314.19732666015625, "loss": 4.688, "margin_dpo/margin_mean": 41.79617691040039, "margin_dpo/margin_std": 69.91704559326172, "step": 371 }, { "KL/chosen_KL_mean": -24.16677474975586, "KL/mean": -54.58926010131836, "KL/rejected_KL_mean": -85.01174926757812, "KL/std": 65.71511840820312, "epoch": 0.7790575916230367, "fcm_dpo/beta": 0.00708200316876173, "fcm_dpo/delta": -0.033582255244255066, "fcm_dpo/margin": 60.84496307373047, "fcm_dpo/q_t": 0.3992455005645752, "grad_norm": 35.841552734375, "learning_rate": 7.161255064312283e-08, "logits/chosen": 1.5282446146011353, "logits/rejected": 1.519301414489746, "logps/chosen": -345.8202819824219, "logps/ref_chosen": -321.65350341796875, "logps/ref_rejected": -282.73773193359375, "logps/rejected": -367.74945068359375, "loss": 4.284, "margin_dpo/margin_mean": 60.84495544433594, "margin_dpo/margin_std": 72.75962829589844, "step": 372 }, { "KL/chosen_KL_mean": -25.180253982543945, "KL/mean": -52.30124282836914, "KL/rejected_KL_mean": -79.42222595214844, "KL/std": 66.4149398803711, "epoch": 0.7811518324607329, "fcm_dpo/beta": 0.0070937094278633595, "fcm_dpo/delta": 0.015240887179970741, "fcm_dpo/margin": 54.241973876953125, "fcm_dpo/q_t": 0.4105536639690399, "grad_norm": 41.76082229614258, "learning_rate": 7.033470310611945e-08, "logits/chosen": 1.276640772819519, "logits/rejected": 1.2320808172225952, "logps/chosen": -334.9307556152344, "logps/ref_chosen": -309.75048828125, "logps/ref_rejected": -267.41925048828125, "logps/rejected": -346.8414611816406, "loss": 4.4296, "margin_dpo/margin_mean": 54.24197769165039, "margin_dpo/margin_std": 72.85505676269531, "step": 373 }, { "KL/chosen_KL_mean": -26.792285919189453, "KL/mean": -46.979881286621094, "KL/rejected_KL_mean": -67.16746520996094, "KL/std": 63.515899658203125, "epoch": 0.7832460732984293, "fcm_dpo/beta": 0.0071641867980360985, "fcm_dpo/delta": 0.017971232533454895, "fcm_dpo/margin": 40.37517547607422, "fcm_dpo/q_t": 0.43322664499282837, "grad_norm": 45.671104431152344, "learning_rate": 6.906649047373245e-08, "logits/chosen": 1.430773138999939, "logits/rejected": 1.4034602642059326, "logps/chosen": -337.7864074707031, "logps/ref_chosen": -310.9941101074219, "logps/ref_rejected": -279.94915771484375, "logps/rejected": -347.11663818359375, "loss": 4.7503, "margin_dpo/margin_mean": 40.37517547607422, "margin_dpo/margin_std": 72.25169372558594, "step": 374 }, { "KL/chosen_KL_mean": -26.015838623046875, "KL/mean": -50.00862503051758, "KL/rejected_KL_mean": -74.00141906738281, "KL/std": 61.2200927734375, "epoch": 0.7853403141361257, "fcm_dpo/beta": 0.007448050193488598, "fcm_dpo/delta": 0.01717124693095684, "fcm_dpo/margin": 47.985595703125, "fcm_dpo/q_t": 0.4177404046058655, "grad_norm": 30.2922306060791, "learning_rate": 6.780798075635675e-08, "logits/chosen": 1.535935640335083, "logits/rejected": 1.4619958400726318, "logps/chosen": -335.19927978515625, "logps/ref_chosen": -309.18341064453125, "logps/ref_rejected": -256.9963073730469, "logps/rejected": -330.9977111816406, "loss": 4.5554, "margin_dpo/margin_mean": 47.985599517822266, "margin_dpo/margin_std": 75.4139175415039, "step": 375 }, { "KL/chosen_KL_mean": -31.192398071289062, "KL/mean": -53.65875244140625, "KL/rejected_KL_mean": -76.12509155273438, "KL/std": 64.35493469238281, "epoch": 0.787434554973822, "fcm_dpo/beta": 0.007259708363562822, "fcm_dpo/delta": -0.045814137905836105, "fcm_dpo/margin": 44.93268585205078, "fcm_dpo/q_t": 0.4243854880332947, "grad_norm": 31.516435623168945, "learning_rate": 6.655924144404906e-08, "logits/chosen": 1.553802728652954, "logits/rejected": 1.6345727443695068, "logps/chosen": -313.46759033203125, "logps/ref_chosen": -282.275146484375, "logps/ref_rejected": -255.17941284179688, "logps/rejected": -331.30450439453125, "loss": 4.6516, "margin_dpo/margin_mean": 44.93268966674805, "margin_dpo/margin_std": 73.13008117675781, "step": 376 }, { "KL/chosen_KL_mean": -28.07622528076172, "KL/mean": -49.12818908691406, "KL/rejected_KL_mean": -70.18014526367188, "KL/std": 61.01256561279297, "epoch": 0.7895287958115184, "fcm_dpo/beta": 0.0072228261269629, "fcm_dpo/delta": 0.04093307629227638, "fcm_dpo/margin": 42.103912353515625, "fcm_dpo/q_t": 0.4301294684410095, "grad_norm": 46.87041473388672, "learning_rate": 6.532033950290885e-08, "logits/chosen": 1.4122331142425537, "logits/rejected": 1.4553221464157104, "logps/chosen": -320.9534606933594, "logps/ref_chosen": -292.8772277832031, "logps/ref_rejected": -266.78509521484375, "logps/rejected": -336.9652099609375, "loss": 4.7059, "margin_dpo/margin_mean": 42.103912353515625, "margin_dpo/margin_std": 73.83411407470703, "step": 377 }, { "KL/chosen_KL_mean": -28.152355194091797, "KL/mean": -49.32393264770508, "KL/rejected_KL_mean": -70.49549865722656, "KL/std": 63.85035705566406, "epoch": 0.7916230366492146, "fcm_dpo/beta": 0.007573708426207304, "fcm_dpo/delta": 0.04489870369434357, "fcm_dpo/margin": 42.34314727783203, "fcm_dpo/q_t": 0.42605486512184143, "grad_norm": 34.113529205322266, "learning_rate": 6.409134137148736e-08, "logits/chosen": 1.2811505794525146, "logits/rejected": 1.3435945510864258, "logps/chosen": -301.0460510253906, "logps/ref_chosen": -272.8937072753906, "logps/ref_rejected": -293.058349609375, "logps/rejected": -363.5538635253906, "loss": 4.6475, "margin_dpo/margin_mean": 42.34314727783203, "margin_dpo/margin_std": 69.5819091796875, "step": 378 }, { "KL/chosen_KL_mean": -26.611143112182617, "KL/mean": -48.94253158569336, "KL/rejected_KL_mean": -71.27391815185547, "KL/std": 60.81068420410156, "epoch": 0.793717277486911, "fcm_dpo/beta": 0.007608911953866482, "fcm_dpo/delta": 0.010132553055882454, "fcm_dpo/margin": 44.66277313232422, "fcm_dpo/q_t": 0.42162272334098816, "grad_norm": 48.30427551269531, "learning_rate": 6.28723129572247e-08, "logits/chosen": 1.328374981880188, "logits/rejected": 1.3346821069717407, "logps/chosen": -285.75274658203125, "logps/ref_chosen": -259.1416015625, "logps/ref_rejected": -229.64996337890625, "logps/rejected": -300.9239196777344, "loss": 4.6286, "margin_dpo/margin_mean": 44.662776947021484, "margin_dpo/margin_std": 75.78076171875, "step": 379 }, { "KL/chosen_KL_mean": -32.59101104736328, "KL/mean": -55.79750442504883, "KL/rejected_KL_mean": -79.00399780273438, "KL/std": 64.7029037475586, "epoch": 0.7958115183246073, "fcm_dpo/beta": 0.007695009466260672, "fcm_dpo/delta": -0.0028763366863131523, "fcm_dpo/margin": 46.41297912597656, "fcm_dpo/q_t": 0.4187002182006836, "grad_norm": 36.6245231628418, "learning_rate": 6.166331963291519e-08, "logits/chosen": 1.2380932569503784, "logits/rejected": 1.159091830253601, "logps/chosen": -328.9289855957031, "logps/ref_chosen": -296.33795166015625, "logps/ref_rejected": -283.7359924316406, "logps/rejected": -362.739990234375, "loss": 4.5558, "margin_dpo/margin_mean": 46.41297912597656, "margin_dpo/margin_std": 72.37862396240234, "step": 380 }, { "KL/chosen_KL_mean": -31.32663345336914, "KL/mean": -56.64055252075195, "KL/rejected_KL_mean": -81.9544677734375, "KL/std": 64.01226806640625, "epoch": 0.7979057591623037, "fcm_dpo/beta": 0.007697650697082281, "fcm_dpo/delta": 0.010398047976195812, "fcm_dpo/margin": 50.62784194946289, "fcm_dpo/q_t": 0.41058602929115295, "grad_norm": 37.08723449707031, "learning_rate": 6.046442623320145e-08, "logits/chosen": 1.106405258178711, "logits/rejected": 1.1405856609344482, "logps/chosen": -280.2998046875, "logps/ref_chosen": -248.97320556640625, "logps/ref_rejected": -259.5140075683594, "logps/rejected": -341.4684753417969, "loss": 4.4377, "margin_dpo/margin_mean": 50.62784194946289, "margin_dpo/margin_std": 72.49524688720703, "step": 381 }, { "KL/chosen_KL_mean": -19.252281188964844, "KL/mean": -50.923675537109375, "KL/rejected_KL_mean": -82.59505462646484, "KL/std": 64.12224578857422, "epoch": 0.8, "fcm_dpo/beta": 0.007423033006489277, "fcm_dpo/delta": -0.10267797112464905, "fcm_dpo/margin": 63.3427734375, "fcm_dpo/q_t": 0.3905338943004608, "grad_norm": 35.46590805053711, "learning_rate": 5.9275697051098275e-08, "logits/chosen": 1.2661614418029785, "logits/rejected": 1.4135240316390991, "logps/chosen": -299.9245910644531, "logps/ref_chosen": -280.67230224609375, "logps/ref_rejected": -272.4359130859375, "logps/rejected": -355.03094482421875, "loss": 4.1828, "margin_dpo/margin_mean": 63.3427734375, "margin_dpo/margin_std": 72.86822509765625, "step": 382 }, { "KL/chosen_KL_mean": -27.320846557617188, "KL/mean": -50.53596878051758, "KL/rejected_KL_mean": -73.75108337402344, "KL/std": 61.96195602416992, "epoch": 0.8020942408376963, "fcm_dpo/beta": 0.0072087873704731464, "fcm_dpo/delta": 0.022169094532728195, "fcm_dpo/margin": 46.43023681640625, "fcm_dpo/q_t": 0.42245474457740784, "grad_norm": 53.77853775024414, "learning_rate": 5.809719583454414e-08, "logits/chosen": 0.8659422397613525, "logits/rejected": 0.9049512147903442, "logps/chosen": -294.97930908203125, "logps/ref_chosen": -267.658447265625, "logps/ref_rejected": -262.9810791015625, "logps/rejected": -336.732177734375, "loss": 4.5948, "margin_dpo/margin_mean": 46.43023681640625, "margin_dpo/margin_std": 71.89701080322266, "step": 383 }, { "KL/chosen_KL_mean": -32.981590270996094, "KL/mean": -50.9521369934082, "KL/rejected_KL_mean": -68.92266845703125, "KL/std": 61.502723693847656, "epoch": 0.8041884816753927, "fcm_dpo/beta": 0.0074386936612427235, "fcm_dpo/delta": 0.04118882864713669, "fcm_dpo/margin": 35.94107437133789, "fcm_dpo/q_t": 0.43864670395851135, "grad_norm": 63.32637405395508, "learning_rate": 5.6928985782982524e-08, "logits/chosen": 1.4148743152618408, "logits/rejected": 1.5957931280136108, "logps/chosen": -299.2281494140625, "logps/ref_chosen": -266.2465515136719, "logps/ref_rejected": -281.1450500488281, "logps/rejected": -350.0677490234375, "loss": 4.8763, "margin_dpo/margin_mean": 35.94107437133789, "margin_dpo/margin_std": 77.21729278564453, "step": 384 }, { "KL/chosen_KL_mean": -29.206470489501953, "KL/mean": -52.04616165161133, "KL/rejected_KL_mean": -74.8858413696289, "KL/std": 62.6688232421875, "epoch": 0.806282722513089, "fcm_dpo/beta": 0.007602612022310495, "fcm_dpo/delta": 0.018680794164538383, "fcm_dpo/margin": 45.679378509521484, "fcm_dpo/q_t": 0.4202457666397095, "grad_norm": 30.920242309570312, "learning_rate": 5.57711295439732e-08, "logits/chosen": 1.284320592880249, "logits/rejected": 1.2636933326721191, "logps/chosen": -303.5123291015625, "logps/ref_chosen": -274.30584716796875, "logps/ref_rejected": -242.94247436523438, "logps/rejected": -317.82830810546875, "loss": 4.5784, "margin_dpo/margin_mean": 45.679378509521484, "margin_dpo/margin_std": 72.00262451171875, "step": 385 }, { "KL/chosen_KL_mean": -22.99639129638672, "KL/mean": -51.93415069580078, "KL/rejected_KL_mean": -80.87190246582031, "KL/std": 67.61793518066406, "epoch": 0.8083769633507853, "fcm_dpo/beta": 0.0072924308478832245, "fcm_dpo/delta": -0.063094861805439, "fcm_dpo/margin": 57.87550735473633, "fcm_dpo/q_t": 0.40138837695121765, "grad_norm": 26.728208541870117, "learning_rate": 5.4623689209832484e-08, "logits/chosen": 1.487105369567871, "logits/rejected": 1.5577136278152466, "logps/chosen": -332.2628173828125, "logps/ref_chosen": -309.2664489746094, "logps/ref_rejected": -298.4182434082031, "logps/rejected": -379.2901306152344, "loss": 4.3501, "margin_dpo/margin_mean": 57.87551498413086, "margin_dpo/margin_std": 74.2111587524414, "step": 386 }, { "KL/chosen_KL_mean": -24.85624122619629, "KL/mean": -51.09400939941406, "KL/rejected_KL_mean": -77.33177947998047, "KL/std": 64.32537078857422, "epoch": 0.8104712041884817, "fcm_dpo/beta": 0.007194596342742443, "fcm_dpo/delta": -0.014300383627414703, "fcm_dpo/margin": 52.47553253173828, "fcm_dpo/q_t": 0.4129951596260071, "grad_norm": 38.05221939086914, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 1.3143092393875122, "logits/rejected": 1.3520739078521729, "logps/chosen": -288.87713623046875, "logps/ref_chosen": -264.0209045410156, "logps/ref_rejected": -264.3447570800781, "logps/rejected": -341.676513671875, "loss": 4.4934, "margin_dpo/margin_mean": 52.47553253173828, "margin_dpo/margin_std": 76.32706451416016, "step": 387 }, { "KL/chosen_KL_mean": -36.69129180908203, "KL/mean": -56.626888275146484, "KL/rejected_KL_mean": -76.56248474121094, "KL/std": 62.020477294921875, "epoch": 0.812565445026178, "fcm_dpo/beta": 0.007130505982786417, "fcm_dpo/delta": 0.0102801863104105, "fcm_dpo/margin": 39.87118148803711, "fcm_dpo/q_t": 0.43438929319381714, "grad_norm": 73.09330749511719, "learning_rate": 5.2360301829254745e-08, "logits/chosen": 1.2919379472732544, "logits/rejected": 1.3333224058151245, "logps/chosen": -299.9101257324219, "logps/ref_chosen": -263.2188415527344, "logps/ref_rejected": -236.08546447753906, "logps/rejected": -312.6479187011719, "loss": 4.8043, "margin_dpo/margin_mean": 39.87118148803711, "margin_dpo/margin_std": 77.06546783447266, "step": 388 }, { "KL/chosen_KL_mean": -27.810565948486328, "KL/mean": -54.52623748779297, "KL/rejected_KL_mean": -81.2419204711914, "KL/std": 61.5937614440918, "epoch": 0.8146596858638744, "fcm_dpo/beta": 0.007093273103237152, "fcm_dpo/delta": -0.0129660964012146, "fcm_dpo/margin": 53.43135452270508, "fcm_dpo/q_t": 0.411451131105423, "grad_norm": 34.63319778442383, "learning_rate": 5.1244476161413806e-08, "logits/chosen": 1.5643380880355835, "logits/rejected": 1.5534164905548096, "logps/chosen": -301.6084899902344, "logps/ref_chosen": -273.79791259765625, "logps/ref_rejected": -242.4090118408203, "logps/rejected": -323.6509094238281, "loss": 4.4611, "margin_dpo/margin_mean": 53.43135070800781, "margin_dpo/margin_std": 74.87136840820312, "step": 389 }, { "KL/chosen_KL_mean": -28.37144660949707, "KL/mean": -54.154396057128906, "KL/rejected_KL_mean": -79.9373550415039, "KL/std": 61.98411178588867, "epoch": 0.8167539267015707, "fcm_dpo/beta": 0.007195638492703438, "fcm_dpo/delta": -0.010860946960747242, "fcm_dpo/margin": 51.56591033935547, "fcm_dpo/q_t": 0.4138355553150177, "grad_norm": 31.387096405029297, "learning_rate": 5.013930914912476e-08, "logits/chosen": 1.2626863718032837, "logits/rejected": 1.3200889825820923, "logps/chosen": -303.6866149902344, "logps/ref_chosen": -275.3151550292969, "logps/ref_rejected": -278.1514587402344, "logps/rejected": -358.0888366699219, "loss": 4.5442, "margin_dpo/margin_mean": 51.56591033935547, "margin_dpo/margin_std": 80.60596466064453, "step": 390 }, { "KL/chosen_KL_mean": -34.32966613769531, "KL/mean": -58.90526580810547, "KL/rejected_KL_mean": -83.48086547851562, "KL/std": 65.26823425292969, "epoch": 0.818848167539267, "fcm_dpo/beta": 0.007021555211395025, "fcm_dpo/delta": -0.014854553155601025, "fcm_dpo/margin": 49.15119171142578, "fcm_dpo/q_t": 0.4196704626083374, "grad_norm": 25.932903289794922, "learning_rate": 4.904486005914027e-08, "logits/chosen": 1.3541665077209473, "logits/rejected": 1.3399139642715454, "logps/chosen": -309.52899169921875, "logps/ref_chosen": -275.1993408203125, "logps/ref_rejected": -261.32122802734375, "logps/rejected": -344.80206298828125, "loss": 4.5185, "margin_dpo/margin_mean": 49.15119934082031, "margin_dpo/margin_std": 67.3121337890625, "step": 391 }, { "KL/chosen_KL_mean": -27.980928421020508, "KL/mean": -59.93144989013672, "KL/rejected_KL_mean": -91.88199615478516, "KL/std": 65.07289123535156, "epoch": 0.8209424083769633, "fcm_dpo/beta": 0.006673329975455999, "fcm_dpo/delta": -0.07043564319610596, "fcm_dpo/margin": 63.901065826416016, "fcm_dpo/q_t": 0.3997809886932373, "grad_norm": 30.282054901123047, "learning_rate": 4.796118758344353e-08, "logits/chosen": 1.322252869606018, "logits/rejected": 1.3225462436676025, "logps/chosen": -327.8086853027344, "logps/ref_chosen": -299.8277587890625, "logps/ref_rejected": -251.24798583984375, "logps/rejected": -343.1300048828125, "loss": 4.2429, "margin_dpo/margin_mean": 63.901065826416016, "margin_dpo/margin_std": 69.360107421875, "step": 392 }, { "KL/chosen_KL_mean": -27.79674530029297, "KL/mean": -50.54197311401367, "KL/rejected_KL_mean": -73.28719329833984, "KL/std": 62.20020294189453, "epoch": 0.8230366492146597, "fcm_dpo/beta": 0.006766752805560827, "fcm_dpo/delta": 0.036494992673397064, "fcm_dpo/margin": 45.490455627441406, "fcm_dpo/q_t": 0.4280647039413452, "grad_norm": 31.35349464416504, "learning_rate": 4.688834983610082e-08, "logits/chosen": 1.3172001838684082, "logits/rejected": 1.2955164909362793, "logps/chosen": -308.2578430175781, "logps/ref_chosen": -280.4610900878906, "logps/ref_rejected": -228.34823608398438, "logps/rejected": -301.63543701171875, "loss": 4.6621, "margin_dpo/margin_mean": 45.49045181274414, "margin_dpo/margin_std": 74.2087631225586, "step": 393 }, { "KL/chosen_KL_mean": -23.677614212036133, "KL/mean": -46.4250602722168, "KL/rejected_KL_mean": -69.17251586914062, "KL/std": 62.090328216552734, "epoch": 0.8251308900523561, "fcm_dpo/beta": 0.00686721783131361, "fcm_dpo/delta": 0.022358842194080353, "fcm_dpo/margin": 45.494895935058594, "fcm_dpo/q_t": 0.4278073012828827, "grad_norm": 32.50846862792969, "learning_rate": 4.582640435014459e-08, "logits/chosen": 1.4290770292282104, "logits/rejected": 1.5399553775787354, "logps/chosen": -343.2777099609375, "logps/ref_chosen": -319.60009765625, "logps/ref_rejected": -316.42822265625, "logps/rejected": -385.6007385253906, "loss": 4.6475, "margin_dpo/margin_mean": 45.494895935058594, "margin_dpo/margin_std": 71.92081451416016, "step": 394 }, { "KL/chosen_KL_mean": -32.48970031738281, "KL/mean": -56.57379913330078, "KL/rejected_KL_mean": -80.65789794921875, "KL/std": 61.502525329589844, "epoch": 0.8272251308900523, "fcm_dpo/beta": 0.006931242998689413, "fcm_dpo/delta": 0.008774535730481148, "fcm_dpo/margin": 48.16818618774414, "fcm_dpo/q_t": 0.42228472232818604, "grad_norm": 29.18105697631836, "learning_rate": 4.477540807448832e-08, "logits/chosen": 1.1601319313049316, "logits/rejected": 1.2798205614089966, "logps/chosen": -292.9075622558594, "logps/ref_chosen": -260.4178466796875, "logps/ref_rejected": -270.04498291015625, "logps/rejected": -350.702880859375, "loss": 4.5643, "margin_dpo/margin_mean": 48.16818618774414, "margin_dpo/margin_std": 69.345458984375, "step": 395 }, { "KL/chosen_KL_mean": -32.005760192871094, "KL/mean": -56.341041564941406, "KL/rejected_KL_mean": -80.67633819580078, "KL/std": 63.819515228271484, "epoch": 0.8293193717277487, "fcm_dpo/beta": 0.0071111805737018585, "fcm_dpo/delta": 0.021238427609205246, "fcm_dpo/margin": 48.670570373535156, "fcm_dpo/q_t": 0.4193718731403351, "grad_norm": 32.6487922668457, "learning_rate": 4.373541737087263e-08, "logits/chosen": 1.2952619791030884, "logits/rejected": 1.279961109161377, "logps/chosen": -312.7087707519531, "logps/ref_chosen": -280.7030029296875, "logps/ref_rejected": -250.76385498046875, "logps/rejected": -331.4402160644531, "loss": 4.5757, "margin_dpo/margin_mean": 48.670570373535156, "margin_dpo/margin_std": 76.09983825683594, "step": 396 }, { "KL/chosen_KL_mean": -27.94131088256836, "KL/mean": -51.001068115234375, "KL/rejected_KL_mean": -74.06082916259766, "KL/std": 64.63763427734375, "epoch": 0.831413612565445, "fcm_dpo/beta": 0.007109126076102257, "fcm_dpo/delta": -0.01439717784523964, "fcm_dpo/margin": 46.119537353515625, "fcm_dpo/q_t": 0.4233459234237671, "grad_norm": 28.73206329345703, "learning_rate": 4.270648801084295e-08, "logits/chosen": 1.0899914503097534, "logits/rejected": 1.1841706037521362, "logps/chosen": -328.8277587890625, "logps/ref_chosen": -300.8864440917969, "logps/ref_rejected": -288.5682067871094, "logps/rejected": -362.6290588378906, "loss": 4.6325, "margin_dpo/margin_mean": 46.119537353515625, "margin_dpo/margin_std": 72.10877990722656, "step": 397 }, { "KL/chosen_KL_mean": -41.151214599609375, "KL/mean": -58.75756072998047, "KL/rejected_KL_mean": -76.3638916015625, "KL/std": 65.13799285888672, "epoch": 0.8335078534031414, "fcm_dpo/beta": 0.007091197185218334, "fcm_dpo/delta": 0.02512446977198124, "fcm_dpo/margin": 35.21266174316406, "fcm_dpo/q_t": 0.4427448511123657, "grad_norm": 40.92946243286133, "learning_rate": 4.168867517275806e-08, "logits/chosen": 1.3057215213775635, "logits/rejected": 1.3617304563522339, "logps/chosen": -334.45355224609375, "logps/ref_chosen": -293.3023681640625, "logps/ref_rejected": -270.0457763671875, "logps/rejected": -346.40966796875, "loss": 4.9548, "margin_dpo/margin_mean": 35.21266555786133, "margin_dpo/margin_std": 82.94670867919922, "step": 398 }, { "KL/chosen_KL_mean": -35.81355285644531, "KL/mean": -57.71054458618164, "KL/rejected_KL_mean": -79.6075439453125, "KL/std": 65.38770294189453, "epoch": 0.8356020942408376, "fcm_dpo/beta": 0.007297034841030836, "fcm_dpo/delta": 0.05020921677350998, "fcm_dpo/margin": 43.794002532958984, "fcm_dpo/q_t": 0.4259675145149231, "grad_norm": 44.98393630981445, "learning_rate": 4.0682033438831584e-08, "logits/chosen": 1.135213851928711, "logits/rejected": 1.2031030654907227, "logps/chosen": -319.2127990722656, "logps/ref_chosen": -283.39923095703125, "logps/ref_rejected": -264.5092468261719, "logps/rejected": -344.1167907714844, "loss": 4.6477, "margin_dpo/margin_mean": 43.79399871826172, "margin_dpo/margin_std": 71.06166076660156, "step": 399 }, { "KL/chosen_KL_mean": -30.493812561035156, "KL/mean": -55.4969367980957, "KL/rejected_KL_mean": -80.50005340576172, "KL/std": 61.17449951171875, "epoch": 0.837696335078534, "fcm_dpo/beta": 0.007266245782375336, "fcm_dpo/delta": -0.05055814981460571, "fcm_dpo/margin": 50.00624084472656, "fcm_dpo/q_t": 0.4153440296649933, "grad_norm": 34.71428680419922, "learning_rate": 3.968661679220467e-08, "logits/chosen": 1.3628064393997192, "logits/rejected": 1.4308254718780518, "logps/chosen": -284.9616394042969, "logps/ref_chosen": -254.4678192138672, "logps/ref_rejected": -252.00326538085938, "logps/rejected": -332.5032958984375, "loss": 4.5255, "margin_dpo/margin_mean": 50.00624084472656, "margin_dpo/margin_std": 72.89318084716797, "step": 400 }, { "epoch": 0.837696335078534, "eval_KL/chosen_KL_mean": -33.885189056396484, "eval_KL/mean": -57.455745697021484, "eval_KL/rejected_KL_mean": -81.02631378173828, "eval_KL/std": 62.96332550048828, "eval_fcm_dpo/beta": 0.007154672406613827, "eval_logits/chosen": 1.4236695766448975, "eval_logits/rejected": 1.4793931245803833, "eval_logps/chosen": -314.3018798828125, "eval_logps/ref_chosen": -280.41668701171875, "eval_logps/ref_rejected": -264.6455383300781, "eval_logps/rejected": -345.6718444824219, "eval_loss": 0.5765824317932129, "eval_margin_dpo/margin_mean": 47.141117095947266, "eval_margin_dpo/margin_std": 72.3220443725586, "eval_runtime": 89.2477, "eval_samples_per_second": 22.41, "eval_steps_per_second": 2.801, "step": 400 }, { "KL/chosen_KL_mean": -27.291719436645508, "KL/mean": -58.15824508666992, "KL/rejected_KL_mean": -89.0247802734375, "KL/std": 66.10443878173828, "epoch": 0.8397905759162304, "fcm_dpo/beta": 0.007040434516966343, "fcm_dpo/delta": -0.036518603563308716, "fcm_dpo/margin": 61.733055114746094, "fcm_dpo/q_t": 0.398317813873291, "grad_norm": 32.57052230834961, "learning_rate": 3.8702478614051345e-08, "logits/chosen": 1.1346800327301025, "logits/rejected": 1.1599088907241821, "logps/chosen": -337.89129638671875, "logps/ref_chosen": -310.5995788574219, "logps/ref_rejected": -292.7754821777344, "logps/rejected": -381.8002624511719, "loss": 4.2511, "margin_dpo/margin_mean": 61.733055114746094, "margin_dpo/margin_std": 72.1438980102539, "step": 401 }, { "KL/chosen_KL_mean": -29.142030715942383, "KL/mean": -53.469818115234375, "KL/rejected_KL_mean": -77.79762268066406, "KL/std": 64.34076690673828, "epoch": 0.8418848167539267, "fcm_dpo/beta": 0.006985440384596586, "fcm_dpo/delta": 0.031272128224372864, "fcm_dpo/margin": 48.65558624267578, "fcm_dpo/q_t": 0.4210313558578491, "grad_norm": 36.524269104003906, "learning_rate": 3.772967168071517e-08, "logits/chosen": 1.4216690063476562, "logits/rejected": 1.3480905294418335, "logps/chosen": -328.993408203125, "logps/ref_chosen": -299.8513488769531, "logps/ref_rejected": -278.5664367675781, "logps/rejected": -356.36407470703125, "loss": 4.5789, "margin_dpo/margin_mean": 48.65558624267578, "margin_dpo/margin_std": 74.5606460571289, "step": 402 }, { "KL/chosen_KL_mean": -24.964340209960938, "KL/mean": -58.60805892944336, "KL/rejected_KL_mean": -92.25179290771484, "KL/std": 66.777099609375, "epoch": 0.8439790575916231, "fcm_dpo/beta": 0.006979155354201794, "fcm_dpo/delta": -0.07276086509227753, "fcm_dpo/margin": 67.28743743896484, "fcm_dpo/q_t": 0.38980188965797424, "grad_norm": 27.25344467163086, "learning_rate": 3.676824816087978e-08, "logits/chosen": 1.4216952323913574, "logits/rejected": 1.4643619060516357, "logps/chosen": -327.39593505859375, "logps/ref_chosen": -302.4316101074219, "logps/ref_rejected": -270.951171875, "logps/rejected": -363.20294189453125, "loss": 4.1007, "margin_dpo/margin_mean": 67.28743743896484, "margin_dpo/margin_std": 65.54295349121094, "step": 403 }, { "KL/chosen_KL_mean": -37.125144958496094, "KL/mean": -60.501609802246094, "KL/rejected_KL_mean": -83.87809753417969, "KL/std": 66.83379364013672, "epoch": 0.8460732984293193, "fcm_dpo/beta": 0.006721503101289272, "fcm_dpo/delta": 0.018977034837007523, "fcm_dpo/margin": 46.75294494628906, "fcm_dpo/q_t": 0.42682117223739624, "grad_norm": 30.03093147277832, "learning_rate": 3.581825961277074e-08, "logits/chosen": 1.3763082027435303, "logits/rejected": 1.4138517379760742, "logps/chosen": -327.02825927734375, "logps/ref_chosen": -289.9031066894531, "logps/ref_rejected": -252.84918212890625, "logps/rejected": -336.7272644042969, "loss": 4.6706, "margin_dpo/margin_mean": 46.7529411315918, "margin_dpo/margin_std": 77.15641021728516, "step": 404 }, { "KL/chosen_KL_mean": -28.577688217163086, "KL/mean": -55.34074020385742, "KL/rejected_KL_mean": -82.10379028320312, "KL/std": 65.560302734375, "epoch": 0.8481675392670157, "fcm_dpo/beta": 0.006971847265958786, "fcm_dpo/delta": 0.026871800422668457, "fcm_dpo/margin": 53.52610397338867, "fcm_dpo/q_t": 0.41349077224731445, "grad_norm": 28.93546485900879, "learning_rate": 3.487975698139084e-08, "logits/chosen": 1.4755507707595825, "logits/rejected": 1.6662366390228271, "logps/chosen": -276.7517395019531, "logps/ref_chosen": -248.17404174804688, "logps/ref_rejected": -252.23764038085938, "logps/rejected": -334.3414306640625, "loss": 4.4438, "margin_dpo/margin_mean": 53.52610397338867, "margin_dpo/margin_std": 71.0295181274414, "step": 405 }, { "KL/chosen_KL_mean": -35.79462432861328, "KL/mean": -57.796234130859375, "KL/rejected_KL_mean": -79.79784393310547, "KL/std": 65.24603271484375, "epoch": 0.8502617801047121, "fcm_dpo/beta": 0.006913081742823124, "fcm_dpo/delta": -0.011782095767557621, "fcm_dpo/margin": 44.00320816040039, "fcm_dpo/q_t": 0.4291381239891052, "grad_norm": 35.615928649902344, "learning_rate": 3.3952790595787986e-08, "logits/chosen": 1.3034807443618774, "logits/rejected": 1.3731592893600464, "logps/chosen": -312.21075439453125, "logps/ref_chosen": -276.4161376953125, "logps/ref_rejected": -264.0082702636719, "logps/rejected": -343.80609130859375, "loss": 4.7119, "margin_dpo/margin_mean": 44.00320816040039, "margin_dpo/margin_std": 75.86014556884766, "step": 406 }, { "KL/chosen_KL_mean": -42.82422637939453, "KL/mean": -67.25929260253906, "KL/rejected_KL_mean": -91.69435119628906, "KL/std": 67.51566314697266, "epoch": 0.8523560209424084, "fcm_dpo/beta": 0.007086477242410183, "fcm_dpo/delta": 0.05532386898994446, "fcm_dpo/margin": 48.87013244628906, "fcm_dpo/q_t": 0.42036789655685425, "grad_norm": 31.873289108276367, "learning_rate": 3.303741016635614e-08, "logits/chosen": 1.3776185512542725, "logits/rejected": 1.3903013467788696, "logps/chosen": -300.2615966796875, "logps/ref_chosen": -257.4373474121094, "logps/ref_rejected": -219.66390991210938, "logps/rejected": -311.3582458496094, "loss": 4.5654, "margin_dpo/margin_mean": 48.87013244628906, "margin_dpo/margin_std": 76.1953125, "step": 407 }, { "KL/chosen_KL_mean": -31.4367733001709, "KL/mean": -56.73390579223633, "KL/rejected_KL_mean": -82.03103637695312, "KL/std": 63.85590362548828, "epoch": 0.8544502617801047, "fcm_dpo/beta": 0.007128621451556683, "fcm_dpo/delta": -0.04133890941739082, "fcm_dpo/margin": 50.59425354003906, "fcm_dpo/q_t": 0.4156496524810791, "grad_norm": 29.58357048034668, "learning_rate": 3.2133664782169944e-08, "logits/chosen": 1.2520235776901245, "logits/rejected": 1.363689661026001, "logps/chosen": -320.0901794433594, "logps/ref_chosen": -288.6533508300781, "logps/ref_rejected": -291.54107666015625, "logps/rejected": -373.5721130371094, "loss": 4.5092, "margin_dpo/margin_mean": 50.59425735473633, "margin_dpo/margin_std": 70.98474884033203, "step": 408 }, { "KL/chosen_KL_mean": -34.46295166015625, "KL/mean": -61.470436096191406, "KL/rejected_KL_mean": -88.47792053222656, "KL/std": 66.3189468383789, "epoch": 0.856544502617801, "fcm_dpo/beta": 0.006866151466965675, "fcm_dpo/delta": -0.02705734595656395, "fcm_dpo/margin": 54.014949798583984, "fcm_dpo/q_t": 0.41403132677078247, "grad_norm": 29.867446899414062, "learning_rate": 3.12416029083514e-08, "logits/chosen": 1.4281820058822632, "logits/rejected": 1.5338895320892334, "logps/chosen": -312.2003479003906, "logps/ref_chosen": -277.7373962402344, "logps/ref_rejected": -272.988525390625, "logps/rejected": -361.4664306640625, "loss": 4.5003, "margin_dpo/margin_mean": 54.014949798583984, "margin_dpo/margin_std": 79.91426086425781, "step": 409 }, { "KL/chosen_KL_mean": -33.24790573120117, "KL/mean": -56.598358154296875, "KL/rejected_KL_mean": -79.9488296508789, "KL/std": 65.97520446777344, "epoch": 0.8586387434554974, "fcm_dpo/beta": 0.007001395337283611, "fcm_dpo/delta": 0.03865911066532135, "fcm_dpo/margin": 46.70093536376953, "fcm_dpo/q_t": 0.42443960905075073, "grad_norm": 45.927207946777344, "learning_rate": 3.036127238347164e-08, "logits/chosen": 1.6032644510269165, "logits/rejected": 1.5830140113830566, "logps/chosen": -307.468994140625, "logps/ref_chosen": -274.2210998535156, "logps/ref_rejected": -264.33465576171875, "logps/rejected": -344.28350830078125, "loss": 4.6541, "margin_dpo/margin_mean": 46.700931549072266, "margin_dpo/margin_std": 79.43925476074219, "step": 410 }, { "KL/chosen_KL_mean": -28.699905395507812, "KL/mean": -55.76493835449219, "KL/rejected_KL_mean": -82.82997131347656, "KL/std": 66.32817077636719, "epoch": 0.8607329842931937, "fcm_dpo/beta": 0.006910680327564478, "fcm_dpo/delta": -0.04498377442359924, "fcm_dpo/margin": 54.130069732666016, "fcm_dpo/q_t": 0.41248053312301636, "grad_norm": 34.258750915527344, "learning_rate": 2.9492720416985e-08, "logits/chosen": 1.4129570722579956, "logits/rejected": 1.47940194606781, "logps/chosen": -301.5679016113281, "logps/ref_chosen": -272.86798095703125, "logps/ref_rejected": -250.35772705078125, "logps/rejected": -333.18768310546875, "loss": 4.4382, "margin_dpo/margin_mean": 54.130069732666016, "margin_dpo/margin_std": 68.8840103149414, "step": 411 }, { "KL/chosen_KL_mean": -40.81800842285156, "KL/mean": -63.90482711791992, "KL/rejected_KL_mean": -86.99164581298828, "KL/std": 66.9229507446289, "epoch": 0.86282722513089, "fcm_dpo/beta": 0.006773846223950386, "fcm_dpo/delta": 0.02097306028008461, "fcm_dpo/margin": 46.17364501953125, "fcm_dpo/q_t": 0.4274871349334717, "grad_norm": 28.87379264831543, "learning_rate": 2.863599358669755e-08, "logits/chosen": 1.1528704166412354, "logits/rejected": 1.2533339262008667, "logps/chosen": -308.7372131347656, "logps/ref_chosen": -267.919189453125, "logps/ref_rejected": -271.73162841796875, "logps/rejected": -358.7232360839844, "loss": 4.6837, "margin_dpo/margin_mean": 46.17365264892578, "margin_dpo/margin_std": 78.83636474609375, "step": 412 }, { "KL/chosen_KL_mean": -29.904508590698242, "KL/mean": -51.50232696533203, "KL/rejected_KL_mean": -73.10014343261719, "KL/std": 62.79712677001953, "epoch": 0.8649214659685864, "fcm_dpo/beta": 0.007038842886686325, "fcm_dpo/delta": 0.048431336879730225, "fcm_dpo/margin": 43.19563674926758, "fcm_dpo/q_t": 0.4297616183757782, "grad_norm": 32.770263671875, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 1.2540414333343506, "logits/rejected": 1.3082278966903687, "logps/chosen": -294.8625793457031, "logps/ref_chosen": -264.95806884765625, "logps/ref_rejected": -267.40093994140625, "logps/rejected": -340.5010986328125, "loss": 4.6649, "margin_dpo/margin_mean": 43.19563674926758, "margin_dpo/margin_std": 71.008056640625, "step": 413 }, { "KL/chosen_KL_mean": -38.727088928222656, "KL/mean": -63.03623962402344, "KL/rejected_KL_mean": -87.34538269042969, "KL/std": 66.72122192382812, "epoch": 0.8670157068062827, "fcm_dpo/beta": 0.00721154548227787, "fcm_dpo/delta": -0.01244377251714468, "fcm_dpo/margin": 48.6182861328125, "fcm_dpo/q_t": 0.4196760654449463, "grad_norm": 36.20530700683594, "learning_rate": 2.6958198472749717e-08, "logits/chosen": 1.3935970067977905, "logits/rejected": 1.4582998752593994, "logps/chosen": -328.73284912109375, "logps/ref_chosen": -290.00579833984375, "logps/ref_rejected": -266.8810119628906, "logps/rejected": -354.22637939453125, "loss": 4.6178, "margin_dpo/margin_mean": 48.6182861328125, "margin_dpo/margin_std": 81.58920288085938, "step": 414 }, { "KL/chosen_KL_mean": -27.320758819580078, "KL/mean": -57.90191650390625, "KL/rejected_KL_mean": -88.48309326171875, "KL/std": 65.74607849121094, "epoch": 0.8691099476439791, "fcm_dpo/beta": 0.007084012031555176, "fcm_dpo/delta": -0.03487085923552513, "fcm_dpo/margin": 61.16233444213867, "fcm_dpo/q_t": 0.3997090756893158, "grad_norm": 34.59502029418945, "learning_rate": 2.613722016414943e-08, "logits/chosen": 1.477209210395813, "logits/rejected": 1.5739178657531738, "logps/chosen": -317.13177490234375, "logps/ref_chosen": -289.81103515625, "logps/ref_rejected": -278.2758483886719, "logps/rejected": -366.7590026855469, "loss": 4.2901, "margin_dpo/margin_mean": 61.162330627441406, "margin_dpo/margin_std": 76.23521423339844, "step": 415 }, { "KL/chosen_KL_mean": -27.77629280090332, "KL/mean": -56.491310119628906, "KL/rejected_KL_mean": -85.2063217163086, "KL/std": 65.02974700927734, "epoch": 0.8712041884816754, "fcm_dpo/beta": 0.006733311805874109, "fcm_dpo/delta": -0.02400980517268181, "fcm_dpo/margin": 57.43003463745117, "fcm_dpo/q_t": 0.40963587164878845, "grad_norm": 38.45534896850586, "learning_rate": 2.5328246937043525e-08, "logits/chosen": 1.4707696437835693, "logits/rejected": 1.4490132331848145, "logps/chosen": -332.42510986328125, "logps/ref_chosen": -304.6488037109375, "logps/ref_rejected": -264.2703857421875, "logps/rejected": -349.4767150878906, "loss": 4.4291, "margin_dpo/margin_mean": 57.430030822753906, "margin_dpo/margin_std": 77.3693618774414, "step": 416 }, { "KL/chosen_KL_mean": -32.69208526611328, "KL/mean": -57.48160934448242, "KL/rejected_KL_mean": -82.27113342285156, "KL/std": 64.19945526123047, "epoch": 0.8732984293193717, "fcm_dpo/beta": 0.006742675788700581, "fcm_dpo/delta": -0.0234349574893713, "fcm_dpo/margin": 49.579044342041016, "fcm_dpo/q_t": 0.42191338539123535, "grad_norm": 58.41111373901367, "learning_rate": 2.4531322174210973e-08, "logits/chosen": 1.5242427587509155, "logits/rejected": 1.6251673698425293, "logps/chosen": -334.58099365234375, "logps/ref_chosen": -301.8889465332031, "logps/ref_rejected": -275.0347595214844, "logps/rejected": -357.305908203125, "loss": 4.5991, "margin_dpo/margin_mean": 49.579044342041016, "margin_dpo/margin_std": 75.97627258300781, "step": 417 }, { "KL/chosen_KL_mean": -42.28622055053711, "KL/mean": -63.07315444946289, "KL/rejected_KL_mean": -83.86009216308594, "KL/std": 65.60322570800781, "epoch": 0.875392670157068, "fcm_dpo/beta": 0.006690526846796274, "fcm_dpo/delta": -0.006242621690034866, "fcm_dpo/margin": 41.57387924194336, "fcm_dpo/q_t": 0.4347190856933594, "grad_norm": 55.8277587890625, "learning_rate": 2.3746488612308295e-08, "logits/chosen": 1.2606562376022339, "logits/rejected": 1.276036262512207, "logps/chosen": -311.5335693359375, "logps/ref_chosen": -269.2473449707031, "logps/ref_rejected": -275.5192565917969, "logps/rejected": -359.37933349609375, "loss": 4.7622, "margin_dpo/margin_mean": 41.57387924194336, "margin_dpo/margin_std": 70.28833770751953, "step": 418 }, { "KL/chosen_KL_mean": -39.38565444946289, "KL/mean": -68.76937866210938, "KL/rejected_KL_mean": -98.15310668945312, "KL/std": 70.06245422363281, "epoch": 0.8774869109947644, "fcm_dpo/beta": 0.006503076758235693, "fcm_dpo/delta": -0.03890489414334297, "fcm_dpo/margin": 58.76746368408203, "fcm_dpo/q_t": 0.4106443524360657, "grad_norm": 28.051912307739258, "learning_rate": 2.297378833957761e-08, "logits/chosen": 1.5848643779754639, "logits/rejected": 1.584818959236145, "logps/chosen": -330.54608154296875, "logps/ref_chosen": -291.1604309082031, "logps/ref_rejected": -245.18325805664062, "logps/rejected": -343.3363342285156, "loss": 4.4578, "margin_dpo/margin_mean": 58.7674674987793, "margin_dpo/margin_std": 82.36515808105469, "step": 419 }, { "KL/chosen_KL_mean": -35.29539489746094, "KL/mean": -63.34731674194336, "KL/rejected_KL_mean": -91.39923858642578, "KL/std": 68.02963256835938, "epoch": 0.8795811518324608, "fcm_dpo/beta": 0.006553223356604576, "fcm_dpo/delta": 0.03296438604593277, "fcm_dpo/margin": 56.10383224487305, "fcm_dpo/q_t": 0.41389018297195435, "grad_norm": 35.01494216918945, "learning_rate": 2.2213262793589482e-08, "logits/chosen": 1.1775184869766235, "logits/rejected": 1.1260844469070435, "logps/chosen": -293.7652282714844, "logps/ref_chosen": -258.4698181152344, "logps/ref_rejected": -242.916015625, "logps/rejected": -334.31524658203125, "loss": 4.4722, "margin_dpo/margin_mean": 56.10383224487305, "margin_dpo/margin_std": 78.17017364501953, "step": 420 }, { "KL/chosen_KL_mean": -25.225984573364258, "KL/mean": -54.59425354003906, "KL/rejected_KL_mean": -83.9625244140625, "KL/std": 63.646209716796875, "epoch": 0.881675392670157, "fcm_dpo/beta": 0.0065411971881985664, "fcm_dpo/delta": -0.014971577562391758, "fcm_dpo/margin": 58.73655700683594, "fcm_dpo/q_t": 0.4093979597091675, "grad_norm": 28.420188903808594, "learning_rate": 2.1464952759020856e-08, "logits/chosen": 1.2410991191864014, "logits/rejected": 1.2269824743270874, "logps/chosen": -315.36676025390625, "logps/ref_chosen": -290.1407775878906, "logps/ref_rejected": -239.65676879882812, "logps/rejected": -323.6192626953125, "loss": 4.3795, "margin_dpo/margin_mean": 58.73655319213867, "margin_dpo/margin_std": 71.42195892333984, "step": 421 }, { "KL/chosen_KL_mean": -37.09423828125, "KL/mean": -58.779449462890625, "KL/rejected_KL_mean": -80.46466827392578, "KL/std": 65.69966888427734, "epoch": 0.8837696335078534, "fcm_dpo/beta": 0.00664010364562273, "fcm_dpo/delta": 0.034144721925258636, "fcm_dpo/margin": 43.37041473388672, "fcm_dpo/q_t": 0.4327212870121002, "grad_norm": 29.3281192779541, "learning_rate": 2.07288983654679e-08, "logits/chosen": 1.3102211952209473, "logits/rejected": 1.3857343196868896, "logps/chosen": -314.0302734375, "logps/ref_chosen": -276.93603515625, "logps/ref_rejected": -250.398681640625, "logps/rejected": -330.86334228515625, "loss": 4.774, "margin_dpo/margin_mean": 43.37041473388672, "margin_dpo/margin_std": 80.89669799804688, "step": 422 }, { "KL/chosen_KL_mean": -30.344362258911133, "KL/mean": -60.24177169799805, "KL/rejected_KL_mean": -90.13917541503906, "KL/std": 67.41627502441406, "epoch": 0.8858638743455497, "fcm_dpo/beta": 0.0066752671264112, "fcm_dpo/delta": 0.0005025333957746625, "fcm_dpo/margin": 59.79481887817383, "fcm_dpo/q_t": 0.40733617544174194, "grad_norm": 56.790443420410156, "learning_rate": 2.0005139085293942e-08, "logits/chosen": 1.2415132522583008, "logits/rejected": 1.3153170347213745, "logps/chosen": -315.7744445800781, "logps/ref_chosen": -285.43011474609375, "logps/ref_rejected": -258.0743408203125, "logps/rejected": -348.2135009765625, "loss": 4.399, "margin_dpo/margin_mean": 59.79481887817383, "margin_dpo/margin_std": 80.7657699584961, "step": 423 }, { "KL/chosen_KL_mean": -25.805326461791992, "KL/mean": -54.49253845214844, "KL/rejected_KL_mean": -83.17976379394531, "KL/std": 63.28171920776367, "epoch": 0.8879581151832461, "fcm_dpo/beta": 0.006660280749201775, "fcm_dpo/delta": -0.011748640798032284, "fcm_dpo/margin": 57.37444305419922, "fcm_dpo/q_t": 0.4099368453025818, "grad_norm": 46.51303482055664, "learning_rate": 1.9293713731512673e-08, "logits/chosen": 1.2232285737991333, "logits/rejected": 1.1983013153076172, "logps/chosen": -325.1503601074219, "logps/ref_chosen": -299.3450012207031, "logps/ref_rejected": -244.60169982910156, "logps/rejected": -327.78143310546875, "loss": 4.3857, "margin_dpo/margin_mean": 57.374446868896484, "margin_dpo/margin_std": 69.36468505859375, "step": 424 }, { "KL/chosen_KL_mean": -32.66620635986328, "KL/mean": -57.438941955566406, "KL/rejected_KL_mean": -82.21167755126953, "KL/std": 67.45172882080078, "epoch": 0.8900523560209425, "fcm_dpo/beta": 0.006631086580455303, "fcm_dpo/delta": -0.012979666702449322, "fcm_dpo/margin": 49.54546356201172, "fcm_dpo/q_t": 0.42322254180908203, "grad_norm": 40.446041107177734, "learning_rate": 1.8594660455706763e-08, "logits/chosen": 1.4363305568695068, "logits/rejected": 1.5310245752334595, "logps/chosen": -305.615966796875, "logps/ref_chosen": -272.9497375488281, "logps/ref_rejected": -274.6335754394531, "logps/rejected": -356.84527587890625, "loss": 4.6004, "margin_dpo/margin_mean": 49.54546356201172, "margin_dpo/margin_std": 73.35726165771484, "step": 425 }, { "KL/chosen_KL_mean": -28.64447784423828, "KL/mean": -55.604576110839844, "KL/rejected_KL_mean": -82.56466674804688, "KL/std": 65.25291442871094, "epoch": 0.8921465968586387, "fcm_dpo/beta": 0.006651397328823805, "fcm_dpo/delta": 0.012115703895688057, "fcm_dpo/margin": 53.92020034790039, "fcm_dpo/q_t": 0.41588619351387024, "grad_norm": 22.520782470703125, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 1.3158924579620361, "logits/rejected": 1.3581820726394653, "logps/chosen": -310.5860595703125, "logps/ref_chosen": -281.9415588378906, "logps/ref_rejected": -253.20486450195312, "logps/rejected": -335.76953125, "loss": 4.4642, "margin_dpo/margin_mean": 53.92020034790039, "margin_dpo/margin_std": 70.84208679199219, "step": 426 }, { "KL/chosen_KL_mean": -26.13115692138672, "KL/mean": -61.82663345336914, "KL/rejected_KL_mean": -97.52210998535156, "KL/std": 67.89302062988281, "epoch": 0.8942408376963351, "fcm_dpo/beta": 0.006481249816715717, "fcm_dpo/delta": -0.09418084472417831, "fcm_dpo/margin": 71.39095306396484, "fcm_dpo/q_t": 0.39159178733825684, "grad_norm": 26.449934005737305, "learning_rate": 1.7233819424956247e-08, "logits/chosen": 1.446837067604065, "logits/rejected": 1.4505691528320312, "logps/chosen": -306.74774169921875, "logps/ref_chosen": -280.6166076660156, "logps/ref_rejected": -263.7532958984375, "logps/rejected": -361.275390625, "loss": 4.1412, "margin_dpo/margin_mean": 71.39096069335938, "margin_dpo/margin_std": 72.03829956054688, "step": 427 }, { "KL/chosen_KL_mean": -33.17949676513672, "KL/mean": -66.57071685791016, "KL/rejected_KL_mean": -99.96192169189453, "KL/std": 68.0640869140625, "epoch": 0.8963350785340314, "fcm_dpo/beta": 0.006181970238685608, "fcm_dpo/delta": -0.013892674818634987, "fcm_dpo/margin": 66.78241729736328, "fcm_dpo/q_t": 0.4024965763092041, "grad_norm": 58.5177001953125, "learning_rate": 1.6572104647786245e-08, "logits/chosen": 1.4094966650009155, "logits/rejected": 1.5273640155792236, "logps/chosen": -319.34173583984375, "logps/ref_chosen": -286.1622314453125, "logps/ref_rejected": -273.3803405761719, "logps/rejected": -373.34222412109375, "loss": 4.2663, "margin_dpo/margin_mean": 66.78242492675781, "margin_dpo/margin_std": 70.78731536865234, "step": 428 }, { "KL/chosen_KL_mean": -33.086944580078125, "KL/mean": -59.44505310058594, "KL/rejected_KL_mean": -85.80316162109375, "KL/std": 65.49327850341797, "epoch": 0.8984293193717278, "fcm_dpo/beta": 0.006138901691883802, "fcm_dpo/delta": 0.019139215350151062, "fcm_dpo/margin": 52.71621322631836, "fcm_dpo/q_t": 0.42456236481666565, "grad_norm": 30.656179428100586, "learning_rate": 1.5922907900227017e-08, "logits/chosen": 1.4257725477218628, "logits/rejected": 1.4433765411376953, "logps/chosen": -299.42352294921875, "logps/ref_chosen": -266.3365783691406, "logps/ref_rejected": -255.01956176757812, "logps/rejected": -340.82275390625, "loss": 4.6478, "margin_dpo/margin_mean": 52.71621322631836, "margin_dpo/margin_std": 85.40750122070312, "step": 429 }, { "KL/chosen_KL_mean": -33.892616271972656, "KL/mean": -56.6580696105957, "KL/rejected_KL_mean": -79.42350769042969, "KL/std": 64.94258880615234, "epoch": 0.900523560209424, "fcm_dpo/beta": 0.006354113575071096, "fcm_dpo/delta": 0.03459186851978302, "fcm_dpo/margin": 45.53089141845703, "fcm_dpo/q_t": 0.43239158391952515, "grad_norm": 65.36149597167969, "learning_rate": 1.5286263996730026e-08, "logits/chosen": 1.452754020690918, "logits/rejected": 1.4913965463638306, "logps/chosen": -316.0439758300781, "logps/ref_chosen": -282.1513977050781, "logps/ref_rejected": -263.4792175292969, "logps/rejected": -342.9027099609375, "loss": 4.7431, "margin_dpo/margin_mean": 45.53089141845703, "margin_dpo/margin_std": 78.78829956054688, "step": 430 }, { "KL/chosen_KL_mean": -36.276119232177734, "KL/mean": -58.02353286743164, "KL/rejected_KL_mean": -79.77095031738281, "KL/std": 65.9626693725586, "epoch": 0.9026178010471204, "fcm_dpo/beta": 0.00650543300434947, "fcm_dpo/delta": 0.033233776688575745, "fcm_dpo/margin": 43.49483108520508, "fcm_dpo/q_t": 0.43427538871765137, "grad_norm": 36.24375534057617, "learning_rate": 1.4662207078575684e-08, "logits/chosen": 1.3554952144622803, "logits/rejected": 1.517827033996582, "logps/chosen": -304.9892578125, "logps/ref_chosen": -268.713134765625, "logps/ref_rejected": -266.13446044921875, "logps/rejected": -345.90545654296875, "loss": 4.7386, "margin_dpo/margin_mean": 43.49483108520508, "margin_dpo/margin_std": 75.53327941894531, "step": 431 }, { "KL/chosen_KL_mean": -26.729814529418945, "KL/mean": -55.14447021484375, "KL/rejected_KL_mean": -83.55911254882812, "KL/std": 66.13988494873047, "epoch": 0.9047120418848168, "fcm_dpo/beta": 0.006465256679803133, "fcm_dpo/delta": 0.0010647280141711235, "fcm_dpo/margin": 56.82931137084961, "fcm_dpo/q_t": 0.41424909234046936, "grad_norm": 35.50051498413086, "learning_rate": 1.40507706120426e-08, "logits/chosen": 1.6082763671875, "logits/rejected": 1.6648359298706055, "logps/chosen": -310.7140197753906, "logps/ref_chosen": -283.9842224121094, "logps/ref_rejected": -253.9568634033203, "logps/rejected": -337.5159912109375, "loss": 4.4608, "margin_dpo/margin_mean": 56.82931137084961, "margin_dpo/margin_std": 77.23971557617188, "step": 432 }, { "KL/chosen_KL_mean": -42.06611633300781, "KL/mean": -64.81314086914062, "KL/rejected_KL_mean": -87.56016540527344, "KL/std": 65.60404205322266, "epoch": 0.9068062827225131, "fcm_dpo/beta": 0.006759243551641703, "fcm_dpo/delta": 0.09632132947444916, "fcm_dpo/margin": 45.49404525756836, "fcm_dpo/q_t": 0.4279315173625946, "grad_norm": 28.762327194213867, "learning_rate": 1.345198738661285e-08, "logits/chosen": 1.344286322593689, "logits/rejected": 1.3697078227996826, "logps/chosen": -284.5093078613281, "logps/ref_chosen": -242.4431915283203, "logps/ref_rejected": -252.87979125976562, "logps/rejected": -340.4399719238281, "loss": 4.6848, "margin_dpo/margin_mean": 45.49404525756836, "margin_dpo/margin_std": 76.451171875, "step": 433 }, { "KL/chosen_KL_mean": -35.92133712768555, "KL/mean": -62.846778869628906, "KL/rejected_KL_mean": -89.77222442626953, "KL/std": 66.37069702148438, "epoch": 0.9089005235602095, "fcm_dpo/beta": 0.007066050078719854, "fcm_dpo/delta": -0.013744387775659561, "fcm_dpo/margin": 53.85087966918945, "fcm_dpo/q_t": 0.4120463728904724, "grad_norm": 35.42103958129883, "learning_rate": 1.2865889513213628e-08, "logits/chosen": 1.4268447160720825, "logits/rejected": 1.3628696203231812, "logps/chosen": -324.2055358886719, "logps/ref_chosen": -288.2841796875, "logps/ref_rejected": -252.9123992919922, "logps/rejected": -342.6846008300781, "loss": 4.4621, "margin_dpo/margin_mean": 53.85087585449219, "margin_dpo/margin_std": 76.14476776123047, "step": 434 }, { "KL/chosen_KL_mean": -29.39164924621582, "KL/mean": -53.528568267822266, "KL/rejected_KL_mean": -77.66548156738281, "KL/std": 65.62640380859375, "epoch": 0.9109947643979057, "fcm_dpo/beta": 0.007086035795509815, "fcm_dpo/delta": 0.030129510909318924, "fcm_dpo/margin": 48.273834228515625, "fcm_dpo/q_t": 0.4203599691390991, "grad_norm": 55.130271911621094, "learning_rate": 1.2292508422495157e-08, "logits/chosen": 1.293613314628601, "logits/rejected": 1.3832406997680664, "logps/chosen": -278.406494140625, "logps/ref_chosen": -249.01483154296875, "logps/ref_rejected": -243.81192016601562, "logps/rejected": -321.4773864746094, "loss": 4.5701, "margin_dpo/margin_mean": 48.273834228515625, "margin_dpo/margin_std": 73.19772338867188, "step": 435 }, { "KL/chosen_KL_mean": -35.94551467895508, "KL/mean": -52.956241607666016, "KL/rejected_KL_mean": -69.96697998046875, "KL/std": 66.97096252441406, "epoch": 0.9130890052356021, "fcm_dpo/beta": 0.007382245734333992, "fcm_dpo/delta": 0.0650118738412857, "fcm_dpo/margin": 34.02146911621094, "fcm_dpo/q_t": 0.442565381526947, "grad_norm": 40.14704895019531, "learning_rate": 1.1731874863145142e-08, "logits/chosen": 1.345801591873169, "logits/rejected": 1.448590636253357, "logps/chosen": -348.8756103515625, "logps/ref_chosen": -312.9300537109375, "logps/ref_rejected": -296.2721862792969, "logps/rejected": -366.2391357421875, "loss": 4.9019, "margin_dpo/margin_mean": 34.02146530151367, "margin_dpo/margin_std": 74.24811553955078, "step": 436 }, { "KL/chosen_KL_mean": -28.4786434173584, "KL/mean": -57.46525955200195, "KL/rejected_KL_mean": -86.45186614990234, "KL/std": 68.7561264038086, "epoch": 0.9151832460732985, "fcm_dpo/beta": 0.007375634275376797, "fcm_dpo/delta": -0.06073632091283798, "fcm_dpo/margin": 57.973209381103516, "fcm_dpo/q_t": 0.4013356864452362, "grad_norm": 23.203168869018555, "learning_rate": 1.118401890024001e-08, "logits/chosen": 1.555214762687683, "logits/rejected": 1.6089116334915161, "logps/chosen": -296.7925720214844, "logps/ref_chosen": -268.31390380859375, "logps/ref_rejected": -268.7603759765625, "logps/rejected": -355.2122497558594, "loss": 4.3614, "margin_dpo/margin_mean": 57.973209381103516, "margin_dpo/margin_std": 78.40283966064453, "step": 437 }, { "KL/chosen_KL_mean": -43.95945358276367, "KL/mean": -57.19584655761719, "KL/rejected_KL_mean": -70.4322509765625, "KL/std": 64.86258697509766, "epoch": 0.9172774869109948, "fcm_dpo/beta": 0.007335146423429251, "fcm_dpo/delta": 0.05353505536913872, "fcm_dpo/margin": 26.47280502319336, "fcm_dpo/q_t": 0.4555557370185852, "grad_norm": 98.37814331054688, "learning_rate": 1.06489699136324e-08, "logits/chosen": 1.3550448417663574, "logits/rejected": 1.4457849264144897, "logps/chosen": -299.19134521484375, "logps/ref_chosen": -255.23190307617188, "logps/ref_rejected": -241.1186981201172, "logps/rejected": -311.55096435546875, "loss": 5.1403, "margin_dpo/margin_mean": 26.47280502319336, "margin_dpo/margin_std": 78.69203186035156, "step": 438 }, { "KL/chosen_KL_mean": -35.10698318481445, "KL/mean": -59.143768310546875, "KL/rejected_KL_mean": -83.18055725097656, "KL/std": 63.55727767944336, "epoch": 0.9193717277486911, "fcm_dpo/beta": 0.007670318242162466, "fcm_dpo/delta": 0.03191578388214111, "fcm_dpo/margin": 48.07355880737305, "fcm_dpo/q_t": 0.41562914848327637, "grad_norm": 33.346168518066406, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 1.137187123298645, "logits/rejected": 1.151048183441162, "logps/chosen": -286.08734130859375, "logps/ref_chosen": -250.9803466796875, "logps/ref_rejected": -239.9314422607422, "logps/rejected": -323.11199951171875, "loss": 4.5363, "margin_dpo/margin_mean": 48.07355880737305, "margin_dpo/margin_std": 75.46719360351562, "step": 439 }, { "KL/chosen_KL_mean": -42.483943939208984, "KL/mean": -61.24757766723633, "KL/rejected_KL_mean": -80.01121520996094, "KL/std": 63.740665435791016, "epoch": 0.9214659685863874, "fcm_dpo/beta": 0.007664527744054794, "fcm_dpo/delta": -0.01909954845905304, "fcm_dpo/margin": 37.52727127075195, "fcm_dpo/q_t": 0.43392854928970337, "grad_norm": 37.99457931518555, "learning_rate": 9.617406953185136e-09, "logits/chosen": 1.2969262599945068, "logits/rejected": 1.2244068384170532, "logps/chosen": -343.4866943359375, "logps/ref_chosen": -301.0027770996094, "logps/ref_rejected": -263.628662109375, "logps/rejected": -343.639892578125, "loss": 4.8331, "margin_dpo/margin_mean": 37.52727127075195, "margin_dpo/margin_std": 74.27835845947266, "step": 440 }, { "KL/chosen_KL_mean": -30.726608276367188, "KL/mean": -59.48487854003906, "KL/rejected_KL_mean": -88.24314880371094, "KL/std": 66.66978454589844, "epoch": 0.9235602094240838, "fcm_dpo/beta": 0.007572128903120756, "fcm_dpo/delta": -0.03766479343175888, "fcm_dpo/margin": 57.516536712646484, "fcm_dpo/q_t": 0.39927002787590027, "grad_norm": 33.259742736816406, "learning_rate": 9.12094829893642e-09, "logits/chosen": 1.4339779615402222, "logits/rejected": 1.5300768613815308, "logps/chosen": -331.57537841796875, "logps/ref_chosen": -300.8487854003906, "logps/ref_rejected": -296.7118835449219, "logps/rejected": -384.9550476074219, "loss": 4.286, "margin_dpo/margin_mean": 57.516536712646484, "margin_dpo/margin_std": 68.08244323730469, "step": 441 }, { "KL/chosen_KL_mean": -44.049869537353516, "KL/mean": -65.7066879272461, "KL/rejected_KL_mean": -87.36351776123047, "KL/std": 64.52679443359375, "epoch": 0.9256544502617801, "fcm_dpo/beta": 0.0074118864722549915, "fcm_dpo/delta": 0.05567052215337753, "fcm_dpo/margin": 43.31364440917969, "fcm_dpo/q_t": 0.4267332851886749, "grad_norm": 33.63667678833008, "learning_rate": 8.637407257200496e-09, "logits/chosen": 1.642959475517273, "logits/rejected": 1.716474175453186, "logps/chosen": -315.6982727050781, "logps/ref_chosen": -271.6484069824219, "logps/ref_rejected": -238.21197509765625, "logps/rejected": -325.57550048828125, "loss": 4.7617, "margin_dpo/margin_mean": 43.31364440917969, "margin_dpo/margin_std": 84.8089599609375, "step": 442 }, { "KL/chosen_KL_mean": -36.81932830810547, "KL/mean": -59.354122161865234, "KL/rejected_KL_mean": -81.88893127441406, "KL/std": 64.16720581054688, "epoch": 0.9277486910994764, "fcm_dpo/beta": 0.007686508819460869, "fcm_dpo/delta": -0.03169297054409981, "fcm_dpo/margin": 45.06959915161133, "fcm_dpo/q_t": 0.41995733976364136, "grad_norm": 34.606407165527344, "learning_rate": 8.166809758815895e-09, "logits/chosen": 1.3172075748443604, "logits/rejected": 1.4071624279022217, "logps/chosen": -303.1158447265625, "logps/ref_chosen": -266.2965087890625, "logps/ref_rejected": -263.7326965332031, "logps/rejected": -345.6216125488281, "loss": 4.6215, "margin_dpo/margin_mean": 45.06959915161133, "margin_dpo/margin_std": 72.51322174072266, "step": 443 }, { "KL/chosen_KL_mean": -34.71281814575195, "KL/mean": -59.032806396484375, "KL/rejected_KL_mean": -83.35279846191406, "KL/std": 67.0949478149414, "epoch": 0.9298429319371728, "fcm_dpo/beta": 0.007581553887575865, "fcm_dpo/delta": 0.031975001096725464, "fcm_dpo/margin": 48.639984130859375, "fcm_dpo/q_t": 0.41638487577438354, "grad_norm": 32.99140548706055, "learning_rate": 7.709181040498253e-09, "logits/chosen": 1.17976975440979, "logits/rejected": 1.1861872673034668, "logps/chosen": -327.07354736328125, "logps/ref_chosen": -292.36077880859375, "logps/ref_rejected": -264.35662841796875, "logps/rejected": -347.70941162109375, "loss": 4.5585, "margin_dpo/margin_mean": 48.639984130859375, "margin_dpo/margin_std": 79.28902435302734, "step": 444 }, { "KL/chosen_KL_mean": -34.34645080566406, "KL/mean": -54.194400787353516, "KL/rejected_KL_mean": -74.0423583984375, "KL/std": 66.02217102050781, "epoch": 0.9319371727748691, "fcm_dpo/beta": 0.0076246075332164764, "fcm_dpo/delta": -0.032751768827438354, "fcm_dpo/margin": 39.69590759277344, "fcm_dpo/q_t": 0.4303748905658722, "grad_norm": 33.18951416015625, "learning_rate": 7.2645456434869965e-09, "logits/chosen": 1.361595869064331, "logits/rejected": 1.4701972007751465, "logps/chosen": -281.0662536621094, "logps/ref_chosen": -246.71978759765625, "logps/ref_rejected": -261.339599609375, "logps/rejected": -335.3819885253906, "loss": 4.7777, "margin_dpo/margin_mean": 39.6959114074707, "margin_dpo/margin_std": 73.78507995605469, "step": 445 }, { "KL/chosen_KL_mean": -28.681175231933594, "KL/mean": -55.73102951049805, "KL/rejected_KL_mean": -82.78089141845703, "KL/std": 64.98340606689453, "epoch": 0.9340314136125655, "fcm_dpo/beta": 0.007325790822505951, "fcm_dpo/delta": -0.04906530678272247, "fcm_dpo/margin": 54.099708557128906, "fcm_dpo/q_t": 0.4080902338027954, "grad_norm": 31.139144897460938, "learning_rate": 6.832927412229017e-09, "logits/chosen": 1.2609220743179321, "logits/rejected": 1.2591335773468018, "logps/chosen": -327.2398986816406, "logps/ref_chosen": -298.5587158203125, "logps/ref_rejected": -264.5877685546875, "logps/rejected": -347.36865234375, "loss": 4.44, "margin_dpo/margin_mean": 54.09970474243164, "margin_dpo/margin_std": 76.38221740722656, "step": 446 }, { "KL/chosen_KL_mean": -23.04488182067871, "KL/mean": -51.995361328125, "KL/rejected_KL_mean": -80.94583892822266, "KL/std": 63.761531829833984, "epoch": 0.9361256544502617, "fcm_dpo/beta": 0.0070063467137515545, "fcm_dpo/delta": -0.038511306047439575, "fcm_dpo/margin": 57.90095138549805, "fcm_dpo/q_t": 0.40471604466438293, "grad_norm": 32.2551383972168, "learning_rate": 6.414349493100129e-09, "logits/chosen": 1.236978530883789, "logits/rejected": 1.2315425872802734, "logps/chosen": -272.8981018066406, "logps/ref_chosen": -249.8532257080078, "logps/ref_rejected": -233.2117156982422, "logps/rejected": -314.1575622558594, "loss": 4.3238, "margin_dpo/margin_mean": 57.90095138549805, "margin_dpo/margin_std": 69.3619384765625, "step": 447 }, { "KL/chosen_KL_mean": -29.741437911987305, "KL/mean": -55.09192657470703, "KL/rejected_KL_mean": -80.4424057006836, "KL/std": 65.35220336914062, "epoch": 0.9382198952879581, "fcm_dpo/beta": 0.0069551835767924786, "fcm_dpo/delta": 0.005020767450332642, "fcm_dpo/margin": 50.700958251953125, "fcm_dpo/q_t": 0.41787856817245483, "grad_norm": 34.935394287109375, "learning_rate": 6.0088343331638756e-09, "logits/chosen": 1.4007723331451416, "logits/rejected": 1.5684510469436646, "logps/chosen": -288.56304931640625, "logps/ref_chosen": -258.8216247558594, "logps/ref_rejected": -260.4953308105469, "logps/rejected": -340.9377136230469, "loss": 4.5225, "margin_dpo/margin_mean": 50.70096206665039, "margin_dpo/margin_std": 72.33600616455078, "step": 448 }, { "KL/chosen_KL_mean": -28.044179916381836, "KL/mean": -56.71196746826172, "KL/rejected_KL_mean": -85.3797607421875, "KL/std": 64.18692016601562, "epoch": 0.9403141361256544, "fcm_dpo/beta": 0.006888584233820438, "fcm_dpo/delta": -0.0480492040514946, "fcm_dpo/margin": 57.33557891845703, "fcm_dpo/q_t": 0.4075410068035126, "grad_norm": 32.0770263671875, "learning_rate": 5.616403678967624e-09, "logits/chosen": 1.5268685817718506, "logits/rejected": 1.429668664932251, "logps/chosen": -352.2607116699219, "logps/ref_chosen": -324.2165222167969, "logps/ref_rejected": -237.05453491210938, "logps/rejected": -322.4342956542969, "loss": 4.3933, "margin_dpo/margin_mean": 57.33557891845703, "margin_dpo/margin_std": 74.86048889160156, "step": 449 }, { "KL/chosen_KL_mean": -40.39637756347656, "KL/mean": -62.09858322143555, "KL/rejected_KL_mean": -83.80079650878906, "KL/std": 66.7348403930664, "epoch": 0.9424083769633508, "fcm_dpo/beta": 0.006802456919103861, "fcm_dpo/delta": 0.038337331265211105, "fcm_dpo/margin": 43.4044189453125, "fcm_dpo/q_t": 0.43125033378601074, "grad_norm": 47.554039001464844, "learning_rate": 5.2370785753763356e-09, "logits/chosen": 1.3350361585617065, "logits/rejected": 1.3207178115844727, "logps/chosen": -318.7976379394531, "logps/ref_chosen": -278.4012451171875, "logps/ref_rejected": -247.72671508789062, "logps/rejected": -331.5274963378906, "loss": 4.7082, "margin_dpo/margin_mean": 43.4044189453125, "margin_dpo/margin_std": 73.86143493652344, "step": 450 }, { "KL/chosen_KL_mean": -30.948543548583984, "KL/mean": -56.13187789916992, "KL/rejected_KL_mean": -81.31521606445312, "KL/std": 63.70576095581055, "epoch": 0.9445026178010472, "fcm_dpo/beta": 0.006816812325268984, "fcm_dpo/delta": -0.019252382218837738, "fcm_dpo/margin": 50.366668701171875, "fcm_dpo/q_t": 0.4198485314846039, "grad_norm": 39.90604019165039, "learning_rate": 4.8708793644441086e-09, "logits/chosen": 1.2547870874404907, "logits/rejected": 1.267533779144287, "logps/chosen": -323.0431823730469, "logps/ref_chosen": -292.0946350097656, "logps/ref_rejected": -252.24237060546875, "logps/rejected": -333.5575866699219, "loss": 4.5659, "margin_dpo/margin_mean": 50.366668701171875, "margin_dpo/margin_std": 73.88423156738281, "step": 451 }, { "KL/chosen_KL_mean": -25.739990234375, "KL/mean": -49.663002014160156, "KL/rejected_KL_mean": -73.58600616455078, "KL/std": 66.58033752441406, "epoch": 0.9465968586387434, "fcm_dpo/beta": 0.006885976064950228, "fcm_dpo/delta": 0.03865534067153931, "fcm_dpo/margin": 47.84601593017578, "fcm_dpo/q_t": 0.423822820186615, "grad_norm": 40.99658203125, "learning_rate": 4.517825684323323e-09, "logits/chosen": 1.3628993034362793, "logits/rejected": 1.4249635934829712, "logps/chosen": -316.3507995605469, "logps/ref_chosen": -290.61077880859375, "logps/ref_rejected": -279.8963928222656, "logps/rejected": -353.4823913574219, "loss": 4.5878, "margin_dpo/margin_mean": 47.84601593017578, "margin_dpo/margin_std": 71.62435150146484, "step": 452 }, { "KL/chosen_KL_mean": -32.94917297363281, "KL/mean": -59.938636779785156, "KL/rejected_KL_mean": -86.9281005859375, "KL/std": 65.81941223144531, "epoch": 0.9486910994764398, "fcm_dpo/beta": 0.007084976881742477, "fcm_dpo/delta": 0.01817309856414795, "fcm_dpo/margin": 53.97893142700195, "fcm_dpo/q_t": 0.4109514057636261, "grad_norm": 33.803531646728516, "learning_rate": 4.1779364682113794e-09, "logits/chosen": 1.623923897743225, "logits/rejected": 1.6035189628601074, "logps/chosen": -349.2644348144531, "logps/ref_chosen": -316.3152770996094, "logps/ref_rejected": -300.5086975097656, "logps/rejected": -387.43682861328125, "loss": 4.4253, "margin_dpo/margin_mean": 53.97893142700195, "margin_dpo/margin_std": 73.46149444580078, "step": 453 }, { "KL/chosen_KL_mean": -32.817420959472656, "KL/mean": -57.83567428588867, "KL/rejected_KL_mean": -82.85394287109375, "KL/std": 68.1610107421875, "epoch": 0.9507853403141361, "fcm_dpo/beta": 0.007199456449598074, "fcm_dpo/delta": 0.015694206580519676, "fcm_dpo/margin": 50.036521911621094, "fcm_dpo/q_t": 0.4167366623878479, "grad_norm": 30.35013198852539, "learning_rate": 3.851229943335393e-09, "logits/chosen": 1.6498425006866455, "logits/rejected": 1.7537225484848022, "logps/chosen": -323.3627014160156, "logps/ref_chosen": -290.5452575683594, "logps/ref_rejected": -297.2102355957031, "logps/rejected": -380.0641784667969, "loss": 4.5173, "margin_dpo/margin_mean": 50.036521911621094, "margin_dpo/margin_std": 72.81925201416016, "step": 454 }, { "KL/chosen_KL_mean": -27.860380172729492, "KL/mean": -48.78271484375, "KL/rejected_KL_mean": -69.70503997802734, "KL/std": 62.85211944580078, "epoch": 0.9528795811518325, "fcm_dpo/beta": 0.00735099334269762, "fcm_dpo/delta": 0.02359689027070999, "fcm_dpo/margin": 41.844661712646484, "fcm_dpo/q_t": 0.4287131726741791, "grad_norm": 35.1033821105957, "learning_rate": 3.5377236299748147e-09, "logits/chosen": 1.2487902641296387, "logits/rejected": 1.3486948013305664, "logps/chosen": -291.2807312011719, "logps/ref_chosen": -263.4203186035156, "logps/ref_rejected": -270.30291748046875, "logps/rejected": -340.0079650878906, "loss": 4.7048, "margin_dpo/margin_mean": 41.844661712646484, "margin_dpo/margin_std": 73.86407470703125, "step": 455 }, { "KL/chosen_KL_mean": -34.925262451171875, "KL/mean": -64.77821350097656, "KL/rejected_KL_mean": -94.63116455078125, "KL/std": 69.82489013671875, "epoch": 0.9549738219895288, "fcm_dpo/beta": 0.007192356511950493, "fcm_dpo/delta": -0.0638280063867569, "fcm_dpo/margin": 59.705894470214844, "fcm_dpo/q_t": 0.4018319845199585, "grad_norm": 27.928911209106445, "learning_rate": 3.2374343405217884e-09, "logits/chosen": 1.2568565607070923, "logits/rejected": 1.4191408157348633, "logps/chosen": -346.9626159667969, "logps/ref_chosen": -312.0373840332031, "logps/ref_rejected": -290.50946044921875, "logps/rejected": -385.140625, "loss": 4.3587, "margin_dpo/margin_mean": 59.705902099609375, "margin_dpo/margin_std": 82.88359832763672, "step": 456 }, { "KL/chosen_KL_mean": -35.84705352783203, "KL/mean": -63.143150329589844, "KL/rejected_KL_mean": -90.43922424316406, "KL/std": 68.66735076904297, "epoch": 0.9570680628272251, "fcm_dpo/beta": 0.007003502920269966, "fcm_dpo/delta": -0.007018011063337326, "fcm_dpo/margin": 54.59217834472656, "fcm_dpo/q_t": 0.41150468587875366, "grad_norm": 32.43729782104492, "learning_rate": 2.9503781785795713e-09, "logits/chosen": 1.2495930194854736, "logits/rejected": 1.2768305540084839, "logps/chosen": -333.01904296875, "logps/ref_chosen": -297.1719970703125, "logps/ref_rejected": -269.15557861328125, "logps/rejected": -359.59478759765625, "loss": 4.4655, "margin_dpo/margin_mean": 54.59218215942383, "margin_dpo/margin_std": 79.75926208496094, "step": 457 }, { "KL/chosen_KL_mean": -32.20441818237305, "KL/mean": -56.76567459106445, "KL/rejected_KL_mean": -81.32694244384766, "KL/std": 69.28995513916016, "epoch": 0.9591623036649215, "fcm_dpo/beta": 0.006989713758230209, "fcm_dpo/delta": -0.0033550746738910675, "fcm_dpo/margin": 49.122520446777344, "fcm_dpo/q_t": 0.420844703912735, "grad_norm": 28.704254150390625, "learning_rate": 2.6765705380989432e-09, "logits/chosen": 1.3223989009857178, "logits/rejected": 1.24267578125, "logps/chosen": -319.5693359375, "logps/ref_chosen": -287.3648986816406, "logps/ref_rejected": -271.1472473144531, "logps/rejected": -352.4742126464844, "loss": 4.5797, "margin_dpo/margin_mean": 49.122528076171875, "margin_dpo/margin_std": 75.25064086914062, "step": 458 }, { "KL/chosen_KL_mean": -37.28789138793945, "KL/mean": -56.56632614135742, "KL/rejected_KL_mean": -75.84475708007812, "KL/std": 65.46312713623047, "epoch": 0.9612565445026178, "fcm_dpo/beta": 0.007017179392278194, "fcm_dpo/delta": 0.01031123474240303, "fcm_dpo/margin": 38.55687713623047, "fcm_dpo/q_t": 0.4377577304840088, "grad_norm": 32.177268981933594, "learning_rate": 2.416026102552732e-09, "logits/chosen": 1.3852252960205078, "logits/rejected": 1.47627854347229, "logps/chosen": -325.8940124511719, "logps/ref_chosen": -288.6061096191406, "logps/ref_rejected": -285.6366271972656, "logps/rejected": -361.48138427734375, "loss": 4.8559, "margin_dpo/margin_mean": 38.5568733215332, "margin_dpo/margin_std": 78.94550323486328, "step": 459 }, { "KL/chosen_KL_mean": -36.56329345703125, "KL/mean": -55.490535736083984, "KL/rejected_KL_mean": -74.41775512695312, "KL/std": 64.80435180664062, "epoch": 0.9633507853403142, "fcm_dpo/beta": 0.00710292998701334, "fcm_dpo/delta": 0.03049231693148613, "fcm_dpo/margin": 37.854469299316406, "fcm_dpo/q_t": 0.4383412301540375, "grad_norm": 50.06414031982422, "learning_rate": 2.168758844148272e-09, "logits/chosen": 1.137605905532837, "logits/rejected": 1.060865879058838, "logps/chosen": -346.8152160644531, "logps/ref_chosen": -310.251953125, "logps/ref_rejected": -252.962158203125, "logps/rejected": -327.3798828125, "loss": 4.8889, "margin_dpo/margin_mean": 37.85446548461914, "margin_dpo/margin_std": 81.85502624511719, "step": 460 }, { "KL/chosen_KL_mean": -33.92215347290039, "KL/mean": -57.918094635009766, "KL/rejected_KL_mean": -81.91403198242188, "KL/std": 67.5086898803711, "epoch": 0.9654450261780104, "fcm_dpo/beta": 0.007304798811674118, "fcm_dpo/delta": 0.017830081284046173, "fcm_dpo/margin": 47.99187469482422, "fcm_dpo/q_t": 0.4192803204059601, "grad_norm": 34.50105285644531, "learning_rate": 1.9347820230782295e-09, "logits/chosen": 1.1698627471923828, "logits/rejected": 1.2976014614105225, "logps/chosen": -270.7026062011719, "logps/ref_chosen": -236.7804412841797, "logps/ref_rejected": -229.67550659179688, "logps/rejected": -311.58953857421875, "loss": 4.6163, "margin_dpo/margin_mean": 47.991878509521484, "margin_dpo/margin_std": 81.20181274414062, "step": 461 }, { "KL/chosen_KL_mean": -27.43246078491211, "KL/mean": -57.93980026245117, "KL/rejected_KL_mean": -88.4471435546875, "KL/std": 66.90789794921875, "epoch": 0.9675392670157068, "fcm_dpo/beta": 0.007108363322913647, "fcm_dpo/delta": -0.03596530854701996, "fcm_dpo/margin": 61.01466369628906, "fcm_dpo/q_t": 0.39972108602523804, "grad_norm": 35.206180572509766, "learning_rate": 1.7141081868094209e-09, "logits/chosen": 1.2888528108596802, "logits/rejected": 1.1797453165054321, "logps/chosen": -362.7359619140625, "logps/ref_chosen": -335.30352783203125, "logps/ref_rejected": -250.6796875, "logps/rejected": -339.1268005371094, "loss": 4.3303, "margin_dpo/margin_mean": 61.01466369628906, "margin_dpo/margin_std": 82.29486083984375, "step": 462 }, { "KL/chosen_KL_mean": -35.447837829589844, "KL/mean": -57.846675872802734, "KL/rejected_KL_mean": -80.2455062866211, "KL/std": 66.26893615722656, "epoch": 0.9696335078534032, "fcm_dpo/beta": 0.007227907422930002, "fcm_dpo/delta": 0.04812032729387283, "fcm_dpo/margin": 44.79767608642578, "fcm_dpo/q_t": 0.4250917136669159, "grad_norm": 43.74908447265625, "learning_rate": 1.5067491694100153e-09, "logits/chosen": 1.3669559955596924, "logits/rejected": 1.2001994848251343, "logps/chosen": -323.6680908203125, "logps/ref_chosen": -288.22021484375, "logps/ref_rejected": -227.96673583984375, "logps/rejected": -308.21221923828125, "loss": 4.638, "margin_dpo/margin_mean": 44.79767608642578, "margin_dpo/margin_std": 73.12406921386719, "step": 463 }, { "KL/chosen_KL_mean": -37.204872131347656, "KL/mean": -60.64276123046875, "KL/rejected_KL_mean": -84.08065795898438, "KL/std": 65.3951187133789, "epoch": 0.9717277486910995, "fcm_dpo/beta": 0.007382219657301903, "fcm_dpo/delta": 0.016385518014431, "fcm_dpo/margin": 46.87579345703125, "fcm_dpo/q_t": 0.4192740321159363, "grad_norm": 43.08910369873047, "learning_rate": 1.3127160909147672e-09, "logits/chosen": 1.5911651849746704, "logits/rejected": 1.6315275430679321, "logps/chosen": -297.54180908203125, "logps/ref_chosen": -260.3369140625, "logps/ref_rejected": -254.2083740234375, "logps/rejected": -338.2890319824219, "loss": 4.592, "margin_dpo/margin_mean": 46.87579345703125, "margin_dpo/margin_std": 73.90274047851562, "step": 464 }, { "KL/chosen_KL_mean": -22.133838653564453, "KL/mean": -52.00831604003906, "KL/rejected_KL_mean": -81.88279724121094, "KL/std": 66.21720886230469, "epoch": 0.9738219895287958, "fcm_dpo/beta": 0.007099932059645653, "fcm_dpo/delta": -0.08648946136236191, "fcm_dpo/margin": 59.74894714355469, "fcm_dpo/q_t": 0.4017646610736847, "grad_norm": 31.485698699951172, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 1.4218883514404297, "logits/rejected": 1.4663176536560059, "logps/chosen": -305.2033386230469, "logps/ref_chosen": -283.0694885253906, "logps/ref_rejected": -289.1370544433594, "logps/rejected": -371.0198669433594, "loss": 4.3439, "margin_dpo/margin_mean": 59.74894714355469, "margin_dpo/margin_std": 73.79570770263672, "step": 465 }, { "KL/chosen_KL_mean": -23.852914810180664, "KL/mean": -55.77590560913086, "KL/rejected_KL_mean": -87.69889068603516, "KL/std": 66.72920227050781, "epoch": 0.9759162303664921, "fcm_dpo/beta": 0.006767213344573975, "fcm_dpo/delta": -0.034620895981788635, "fcm_dpo/margin": 63.845977783203125, "fcm_dpo/q_t": 0.3984140157699585, "grad_norm": 30.930160522460938, "learning_rate": 9.64668657069706e-10, "logits/chosen": 0.9917327761650085, "logits/rejected": 1.0448675155639648, "logps/chosen": -280.7379455566406, "logps/ref_chosen": -256.8850402832031, "logps/ref_rejected": -247.0802459716797, "logps/rejected": -334.77911376953125, "loss": 4.2378, "margin_dpo/margin_mean": 63.845977783203125, "margin_dpo/margin_std": 71.75048828125, "step": 466 }, { "KL/chosen_KL_mean": -34.90070724487305, "KL/mean": -57.99346160888672, "KL/rejected_KL_mean": -81.08621215820312, "KL/std": 66.7815933227539, "epoch": 0.9780104712041885, "fcm_dpo/beta": 0.006723516620695591, "fcm_dpo/delta": -0.011592521332204342, "fcm_dpo/margin": 46.18550109863281, "fcm_dpo/q_t": 0.4279315769672394, "grad_norm": 39.73870086669922, "learning_rate": 8.106729664475176e-10, "logits/chosen": 0.9874340295791626, "logits/rejected": 1.1375449895858765, "logps/chosen": -292.4247741699219, "logps/ref_chosen": -257.5240783691406, "logps/ref_rejected": -274.82122802734375, "logps/rejected": -355.9074401855469, "loss": 4.6955, "margin_dpo/margin_mean": 46.18550109863281, "margin_dpo/margin_std": 78.61346435546875, "step": 467 }, { "KL/chosen_KL_mean": -26.964384078979492, "KL/mean": -48.01728439331055, "KL/rejected_KL_mean": -69.07018280029297, "KL/std": 64.16717529296875, "epoch": 0.9801047120418848, "fcm_dpo/beta": 0.00669306144118309, "fcm_dpo/delta": -0.02148795872926712, "fcm_dpo/margin": 42.105796813964844, "fcm_dpo/q_t": 0.43420475721359253, "grad_norm": 32.607810974121094, "learning_rate": 6.700405431837585e-10, "logits/chosen": 1.34190833568573, "logits/rejected": 1.229614019393921, "logps/chosen": -339.2240905761719, "logps/ref_chosen": -312.2597351074219, "logps/ref_rejected": -259.74383544921875, "logps/rejected": -328.81396484375, "loss": 4.7303, "margin_dpo/margin_mean": 42.105796813964844, "margin_dpo/margin_std": 69.45880889892578, "step": 468 }, { "KL/chosen_KL_mean": -23.92185401916504, "KL/mean": -50.92070007324219, "KL/rejected_KL_mean": -77.9195327758789, "KL/std": 67.88216400146484, "epoch": 0.9821989528795811, "fcm_dpo/beta": 0.006680009420961142, "fcm_dpo/delta": 0.014251125045120716, "fcm_dpo/margin": 53.997684478759766, "fcm_dpo/q_t": 0.41561082005500793, "grad_norm": 36.88972091674805, "learning_rate": 5.427789289685347e-10, "logits/chosen": 1.1206728219985962, "logits/rejected": 1.022853136062622, "logps/chosen": -340.96160888671875, "logps/ref_chosen": -317.03973388671875, "logps/ref_rejected": -260.12628173828125, "logps/rejected": -338.0457763671875, "loss": 4.5264, "margin_dpo/margin_mean": 53.997684478759766, "margin_dpo/margin_std": 80.5838623046875, "step": 469 }, { "KL/chosen_KL_mean": -23.201324462890625, "KL/mean": -53.28938293457031, "KL/rejected_KL_mean": -83.3774185180664, "KL/std": 67.57502746582031, "epoch": 0.9842931937172775, "fcm_dpo/beta": 0.006601809058338404, "fcm_dpo/delta": -0.05593057721853256, "fcm_dpo/margin": 60.17609405517578, "fcm_dpo/q_t": 0.4073055684566498, "grad_norm": 27.246021270751953, "learning_rate": 4.288949484559934e-10, "logits/chosen": 1.1400357484817505, "logits/rejected": 1.0457278490066528, "logps/chosen": -327.59527587890625, "logps/ref_chosen": -304.3939514160156, "logps/ref_rejected": -257.6587219238281, "logps/rejected": -341.0361328125, "loss": 4.4029, "margin_dpo/margin_mean": 60.17609405517578, "margin_dpo/margin_std": 78.45281982421875, "step": 470 }, { "KL/chosen_KL_mean": -34.258522033691406, "KL/mean": -55.35606002807617, "KL/rejected_KL_mean": -76.4535903930664, "KL/std": 67.38356018066406, "epoch": 0.9863874345549738, "fcm_dpo/beta": 0.006539772264659405, "fcm_dpo/delta": 0.0638960599899292, "fcm_dpo/margin": 42.195072174072266, "fcm_dpo/q_t": 0.4357315003871918, "grad_norm": 50.0222053527832, "learning_rate": 3.2839470889836627e-10, "logits/chosen": 1.1337032318115234, "logits/rejected": 1.2113916873931885, "logps/chosen": -319.2023620605469, "logps/ref_chosen": -284.94384765625, "logps/ref_rejected": -297.4357604980469, "logps/rejected": -373.88934326171875, "loss": 4.7498, "margin_dpo/margin_mean": 42.195072174072266, "margin_dpo/margin_std": 73.13125610351562, "step": 471 }, { "KL/chosen_KL_mean": -25.31435775756836, "KL/mean": -54.23596954345703, "KL/rejected_KL_mean": -83.1575927734375, "KL/std": 65.6238784790039, "epoch": 0.9884816753926702, "fcm_dpo/beta": 0.006686407141387463, "fcm_dpo/delta": 0.01352601032704115, "fcm_dpo/margin": 57.843223571777344, "fcm_dpo/q_t": 0.4107360243797302, "grad_norm": 40.486175537109375, "learning_rate": 2.412835998185092e-10, "logits/chosen": 1.2406048774719238, "logits/rejected": 1.3722997903823853, "logps/chosen": -257.66864013671875, "logps/ref_chosen": -232.35427856445312, "logps/ref_rejected": -245.6057891845703, "logps/rejected": -328.7633972167969, "loss": 4.419, "margin_dpo/margin_mean": 57.843223571777344, "margin_dpo/margin_std": 77.69920349121094, "step": 472 }, { "KL/chosen_KL_mean": -30.094152450561523, "KL/mean": -59.20981979370117, "KL/rejected_KL_mean": -88.32546997070312, "KL/std": 66.05684661865234, "epoch": 0.9905759162303664, "fcm_dpo/beta": 0.006787514314055443, "fcm_dpo/delta": -0.01905285194516182, "fcm_dpo/margin": 58.23133087158203, "fcm_dpo/q_t": 0.4079660177230835, "grad_norm": 28.650922775268555, "learning_rate": 1.6756629272085544e-10, "logits/chosen": 1.3480632305145264, "logits/rejected": 1.4126392602920532, "logps/chosen": -308.10101318359375, "logps/ref_chosen": -278.0068664550781, "logps/ref_rejected": -257.19256591796875, "logps/rejected": -345.51806640625, "loss": 4.3959, "margin_dpo/margin_mean": 58.23133087158203, "margin_dpo/margin_std": 76.88653564453125, "step": 473 }, { "KL/chosen_KL_mean": -40.19635009765625, "KL/mean": -61.384925842285156, "KL/rejected_KL_mean": -82.57350158691406, "KL/std": 64.97396087646484, "epoch": 0.9926701570680628, "fcm_dpo/beta": 0.0065875062718987465, "fcm_dpo/delta": -0.00273904949426651, "fcm_dpo/margin": 42.37714385986328, "fcm_dpo/q_t": 0.43389174342155457, "grad_norm": 62.54719924926758, "learning_rate": 1.072467408408384e-10, "logits/chosen": 1.2016754150390625, "logits/rejected": 1.2527438402175903, "logps/chosen": -320.953857421875, "logps/ref_chosen": -280.75750732421875, "logps/ref_rejected": -264.34295654296875, "logps/rejected": -346.91644287109375, "loss": 4.735, "margin_dpo/margin_mean": 42.37714385986328, "margin_dpo/margin_std": 68.2033462524414, "step": 474 }, { "KL/chosen_KL_mean": -28.445863723754883, "KL/mean": -51.21686553955078, "KL/rejected_KL_mean": -73.98786926269531, "KL/std": 62.809024810791016, "epoch": 0.9947643979057592, "fcm_dpo/beta": 0.006702399346977472, "fcm_dpo/delta": 0.014713454991579056, "fcm_dpo/margin": 45.54199981689453, "fcm_dpo/q_t": 0.4287365674972534, "grad_norm": 34.98037338256836, "learning_rate": 6.032817893297793e-11, "logits/chosen": 1.175693392753601, "logits/rejected": 1.3002114295959473, "logps/chosen": -269.9172058105469, "logps/ref_chosen": -241.4713592529297, "logps/ref_rejected": -244.0782928466797, "logps/rejected": -318.066162109375, "loss": 4.6671, "margin_dpo/margin_mean": 45.54199981689453, "margin_dpo/margin_std": 73.11616516113281, "step": 475 }, { "KL/chosen_KL_mean": -19.955669403076172, "KL/mean": -46.316307067871094, "KL/rejected_KL_mean": -72.67695617675781, "KL/std": 63.86593246459961, "epoch": 0.9968586387434555, "fcm_dpo/beta": 0.006906186696141958, "fcm_dpo/delta": 0.03595799580216408, "fcm_dpo/margin": 52.72127914428711, "fcm_dpo/q_t": 0.4155328571796417, "grad_norm": 29.642295837402344, "learning_rate": 2.6813123097352287e-11, "logits/chosen": 1.409799575805664, "logits/rejected": 1.418018102645874, "logps/chosen": -338.7300109863281, "logps/ref_chosen": -318.7742919921875, "logps/ref_rejected": -293.66595458984375, "logps/rejected": -366.3429260253906, "loss": 4.5041, "margin_dpo/margin_mean": 52.72127914428711, "margin_dpo/margin_std": 75.55950164794922, "step": 476 }, { "KL/chosen_KL_mean": -30.197856903076172, "KL/mean": -58.40687942504883, "KL/rejected_KL_mean": -86.61589050292969, "KL/std": 67.73674011230469, "epoch": 0.9989528795811519, "fcm_dpo/beta": 0.007070903200656176, "fcm_dpo/delta": 0.0004990696907043457, "fcm_dpo/margin": 56.41804122924805, "fcm_dpo/q_t": 0.40811702609062195, "grad_norm": 33.42003631591797, "learning_rate": 6.7033706447061635e-12, "logits/chosen": 1.2682021856307983, "logits/rejected": 1.3051879405975342, "logps/chosen": -296.1335754394531, "logps/ref_chosen": -265.9356994628906, "logps/ref_rejected": -243.6986083984375, "logps/rejected": -330.31451416015625, "loss": 4.4395, "margin_dpo/margin_mean": 56.41804122924805, "margin_dpo/margin_std": 81.06288146972656, "step": 477 }, { "epoch": 0.9989528795811519, "step": 477, "total_flos": 0.0, "train_loss": 4.806692141406941, "train_runtime": 6998.3828, "train_samples_per_second": 8.736, "train_steps_per_second": 0.068 } ], "logging_steps": 1, "max_steps": 477, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }