{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9989528795811519, "eval_steps": 200, "global_step": 477, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": -0.05443763732910156, "KL/mean": -0.04046645760536194, "KL/rejected_KL_mean": -0.026498794555664062, "KL/std": 0.47091516852378845, "epoch": 0.0020942408376963353, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02793481945991516, "fcm_dpo/q_t": 0.500069797039032, "grad_norm": 28.59322738647461, "learning_rate": 0.0, "logits/chosen": -0.5898098945617676, "logits/rejected": -0.604260265827179, "logps/chosen": -275.28570556640625, "logps/ref_chosen": -275.2312927246094, "logps/ref_rejected": -222.9380340576172, "logps/rejected": -222.96453857421875, "loss": 5.5463, "margin_dpo/margin_mean": -0.02793477475643158, "margin_dpo/margin_std": 0.5724214911460876, "step": 1 }, { "KL/chosen_KL_mean": 0.04458808898925781, "KL/mean": 0.0374295711517334, "KL/rejected_KL_mean": 0.030269622802734375, "KL/std": 0.49036121368408203, "epoch": 0.004188481675392671, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014312177896499634, "fcm_dpo/q_t": 0.4999642074108124, "grad_norm": 27.875282287597656, "learning_rate": 1.0416666666666666e-08, "logits/chosen": -0.6574729681015015, "logits/rejected": -0.6464410424232483, "logps/chosen": -264.7165222167969, "logps/ref_chosen": -264.7611083984375, "logps/ref_rejected": -242.5597686767578, "logps/rejected": -242.52951049804688, "loss": 5.5446, "margin_dpo/margin_mean": 0.014312252402305603, "margin_dpo/margin_std": 0.6423971652984619, "step": 2 }, { "KL/chosen_KL_mean": -0.00445556640625, "KL/mean": -0.034483589231967926, "KL/rejected_KL_mean": -0.06451225280761719, "KL/std": 0.4977545440196991, "epoch": 0.0062827225130890054, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0600539892911911, "fcm_dpo/q_t": 0.4998498558998108, "grad_norm": 25.79319953918457, "learning_rate": 2.083333333333333e-08, "logits/chosen": -0.6840305328369141, "logits/rejected": -0.735154390335083, "logps/chosen": -274.1063537597656, "logps/ref_chosen": -274.1018981933594, "logps/ref_rejected": -286.5882568359375, "logps/rejected": -286.65277099609375, "loss": 5.5428, "margin_dpo/margin_mean": 0.060050785541534424, "margin_dpo/margin_std": 0.7622858881950378, "step": 3 }, { "KL/chosen_KL_mean": 0.017669677734375, "KL/mean": -0.01155301183462143, "KL/rejected_KL_mean": -0.04076957702636719, "KL/std": 0.5529603958129883, "epoch": 0.008376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05844058096408844, "fcm_dpo/q_t": 0.4998539090156555, "grad_norm": 31.700010299682617, "learning_rate": 3.125e-08, "logits/chosen": -0.6172097325325012, "logits/rejected": -0.611587405204773, "logps/chosen": -329.8205871582031, "logps/ref_chosen": -329.8382568359375, "logps/ref_rejected": -303.2850646972656, "logps/rejected": -303.3258361816406, "loss": 5.5429, "margin_dpo/margin_mean": 0.05844086408615112, "margin_dpo/margin_std": 0.7090508937835693, "step": 4 }, { "KL/chosen_KL_mean": 0.07551956176757812, "KL/mean": 0.07383932173252106, "KL/rejected_KL_mean": 0.07216453552246094, "KL/std": 0.533249020576477, "epoch": 0.010471204188481676, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0033641159534454346, "fcm_dpo/q_t": 0.49999159574508667, "grad_norm": 29.5593318939209, "learning_rate": 4.166666666666666e-08, "logits/chosen": -0.5748654007911682, "logits/rejected": -0.591567873954773, "logps/chosen": -301.6634216308594, "logps/ref_chosen": -301.7389221191406, "logps/ref_rejected": -274.7654724121094, "logps/rejected": -274.69329833984375, "loss": 5.5451, "margin_dpo/margin_mean": 0.003364160656929016, "margin_dpo/margin_std": 0.7680634260177612, "step": 5 }, { "KL/chosen_KL_mean": 0.05218315124511719, "KL/mean": 0.025937549769878387, "KL/rejected_KL_mean": -0.00030517578125, "KL/std": 0.5092122554779053, "epoch": 0.012565445026178011, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05248446762561798, "fcm_dpo/q_t": 0.49986881017684937, "grad_norm": 28.216808319091797, "learning_rate": 5.208333333333333e-08, "logits/chosen": -0.6817427277565002, "logits/rejected": -0.644816517829895, "logps/chosen": -285.6424865722656, "logps/ref_chosen": -285.6946716308594, "logps/ref_rejected": -245.8200225830078, "logps/rejected": -245.82032775878906, "loss": 5.5431, "margin_dpo/margin_mean": 0.05248543620109558, "margin_dpo/margin_std": 0.6747213006019592, "step": 6 }, { "KL/chosen_KL_mean": -0.06369400024414062, "KL/mean": -0.026505768299102783, "KL/rejected_KL_mean": 0.010677337646484375, "KL/std": 0.5207120776176453, "epoch": 0.014659685863874346, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07438454031944275, "fcm_dpo/q_t": 0.5001859664916992, "grad_norm": 28.569318771362305, "learning_rate": 6.25e-08, "logits/chosen": -0.5781703591346741, "logits/rejected": -0.6110429167747498, "logps/chosen": -264.7191467285156, "logps/ref_chosen": -264.65545654296875, "logps/ref_rejected": -253.10305786132812, "logps/rejected": -253.09237670898438, "loss": 5.5482, "margin_dpo/margin_mean": -0.0743849128484726, "margin_dpo/margin_std": 0.7227006554603577, "step": 7 }, { "KL/chosen_KL_mean": -0.0766448974609375, "KL/mean": -0.04857754707336426, "KL/rejected_KL_mean": -0.020505905151367188, "KL/std": 0.5684447288513184, "epoch": 0.016753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05614340305328369, "fcm_dpo/q_t": 0.5001404285430908, "grad_norm": 30.8109073638916, "learning_rate": 7.291666666666667e-08, "logits/chosen": -0.683033287525177, "logits/rejected": -0.6907529234886169, "logps/chosen": -354.2654113769531, "logps/ref_chosen": -354.1887512207031, "logps/ref_rejected": -282.9112243652344, "logps/rejected": -282.93170166015625, "loss": 5.5475, "margin_dpo/margin_mean": -0.05614197254180908, "margin_dpo/margin_std": 0.7655779123306274, "step": 8 }, { "KL/chosen_KL_mean": -0.01226043701171875, "KL/mean": -0.01405593752861023, "KL/rejected_KL_mean": -0.015859603881835938, "KL/std": 0.49738261103630066, "epoch": 0.018848167539267015, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.00360184907913208, "fcm_dpo/q_t": 0.4999909996986389, "grad_norm": 27.886255264282227, "learning_rate": 8.333333333333333e-08, "logits/chosen": -0.6334417462348938, "logits/rejected": -0.6513369083404541, "logps/chosen": -285.5624694824219, "logps/ref_chosen": -285.5502014160156, "logps/ref_rejected": -267.99664306640625, "logps/rejected": -268.01251220703125, "loss": 5.5451, "margin_dpo/margin_mean": 0.0036015063524246216, "margin_dpo/margin_std": 0.6722431778907776, "step": 9 }, { "KL/chosen_KL_mean": -0.012775421142578125, "KL/mean": -0.01617148518562317, "KL/rejected_KL_mean": -0.019565582275390625, "KL/std": 0.4946087896823883, "epoch": 0.020942408376963352, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.00680088996887207, "fcm_dpo/q_t": 0.4999829828739166, "grad_norm": 26.488203048706055, "learning_rate": 9.375e-08, "logits/chosen": -0.68070387840271, "logits/rejected": -0.6747847199440002, "logps/chosen": -251.9251708984375, "logps/ref_chosen": -251.91238403320312, "logps/ref_rejected": -226.45260620117188, "logps/rejected": -226.47216796875, "loss": 5.5449, "margin_dpo/margin_mean": 0.006801143288612366, "margin_dpo/margin_std": 0.651378870010376, "step": 10 }, { "KL/chosen_KL_mean": 0.04965972900390625, "KL/mean": 0.05433201044797897, "KL/rejected_KL_mean": 0.05899620056152344, "KL/std": 0.48608970642089844, "epoch": 0.023036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.009331181645393372, "fcm_dpo/q_t": 0.5000233054161072, "grad_norm": 29.016658782958984, "learning_rate": 1.0416666666666667e-07, "logits/chosen": -0.5983105897903442, "logits/rejected": -0.6540327072143555, "logps/chosen": -301.0338134765625, "logps/ref_chosen": -301.08343505859375, "logps/ref_rejected": -259.546630859375, "logps/rejected": -259.4876403808594, "loss": 5.5456, "margin_dpo/margin_mean": -0.009330227971076965, "margin_dpo/margin_std": 0.6587917804718018, "step": 11 }, { "KL/chosen_KL_mean": -0.02674102783203125, "KL/mean": -0.08727707713842392, "KL/rejected_KL_mean": -0.14780616760253906, "KL/std": 0.5344446897506714, "epoch": 0.025130890052356022, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12106941640377045, "fcm_dpo/q_t": 0.49969732761383057, "grad_norm": 30.082069396972656, "learning_rate": 1.1458333333333332e-07, "logits/chosen": -0.5738207101821899, "logits/rejected": -0.5335919857025146, "logps/chosen": -287.5748596191406, "logps/ref_chosen": -287.548095703125, "logps/ref_rejected": -277.37945556640625, "logps/rejected": -277.5272521972656, "loss": 5.5404, "margin_dpo/margin_mean": 0.12106986343860626, "margin_dpo/margin_std": 0.7516077756881714, "step": 12 }, { "KL/chosen_KL_mean": -0.03533744812011719, "KL/mean": -0.07945731282234192, "KL/rejected_KL_mean": -0.12357711791992188, "KL/std": 0.48769837617874146, "epoch": 0.027225130890052355, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08824285864830017, "fcm_dpo/q_t": 0.4997794032096863, "grad_norm": 27.216075897216797, "learning_rate": 1.25e-07, "logits/chosen": -0.653461217880249, "logits/rejected": -0.6608254313468933, "logps/chosen": -270.7017517089844, "logps/ref_chosen": -270.6664123535156, "logps/ref_rejected": -274.6546936035156, "logps/rejected": -274.77825927734375, "loss": 5.5417, "margin_dpo/margin_mean": 0.08824336528778076, "margin_dpo/margin_std": 0.6230098009109497, "step": 13 }, { "KL/chosen_KL_mean": -0.006622314453125, "KL/mean": -0.017597533762454987, "KL/rejected_KL_mean": -0.028568267822265625, "KL/std": 0.5604071617126465, "epoch": 0.02931937172774869, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02194879949092865, "fcm_dpo/q_t": 0.49994513392448425, "grad_norm": 28.335783004760742, "learning_rate": 1.3541666666666666e-07, "logits/chosen": -0.6175643801689148, "logits/rejected": -0.6454805731773376, "logps/chosen": -281.5998229980469, "logps/ref_chosen": -281.59320068359375, "logps/ref_rejected": -263.52215576171875, "logps/rejected": -263.55072021484375, "loss": 5.5444, "margin_dpo/margin_mean": 0.02194814383983612, "margin_dpo/margin_std": 0.8026820421218872, "step": 14 }, { "KL/chosen_KL_mean": 0.022657394409179688, "KL/mean": -0.011080130934715271, "KL/rejected_KL_mean": -0.04481697082519531, "KL/std": 0.47758567333221436, "epoch": 0.031413612565445025, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06748214364051819, "fcm_dpo/q_t": 0.49983131885528564, "grad_norm": 30.399776458740234, "learning_rate": 1.4583333333333335e-07, "logits/chosen": -0.6331017017364502, "logits/rejected": -0.644238293170929, "logps/chosen": -298.4307861328125, "logps/ref_chosen": -298.45343017578125, "logps/ref_rejected": -227.17832946777344, "logps/rejected": -227.22314453125, "loss": 5.5425, "margin_dpo/margin_mean": 0.06748141348361969, "margin_dpo/margin_std": 0.6817619800567627, "step": 15 }, { "KL/chosen_KL_mean": 0.066680908203125, "KL/mean": 0.02534003183245659, "KL/rejected_KL_mean": -0.016004562377929688, "KL/std": 0.5315845012664795, "epoch": 0.033507853403141365, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08268719166517258, "fcm_dpo/q_t": 0.49979326128959656, "grad_norm": 30.135879516601562, "learning_rate": 1.5624999999999999e-07, "logits/chosen": -0.5993279218673706, "logits/rejected": -0.5962104797363281, "logps/chosen": -293.8999328613281, "logps/ref_chosen": -293.96661376953125, "logps/ref_rejected": -250.78443908691406, "logps/rejected": -250.80044555664062, "loss": 5.5419, "margin_dpo/margin_mean": 0.08268788456916809, "margin_dpo/margin_std": 0.7702502012252808, "step": 16 }, { "KL/chosen_KL_mean": 0.043094635009765625, "KL/mean": 0.030858159065246582, "KL/rejected_KL_mean": 0.018617630004882812, "KL/std": 0.4950779974460602, "epoch": 0.0356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02447649836540222, "fcm_dpo/q_t": 0.4999387860298157, "grad_norm": 27.67669105529785, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5722163319587708, "logits/rejected": -0.5985718965530396, "logps/chosen": -262.35089111328125, "logps/ref_chosen": -262.39398193359375, "logps/ref_rejected": -248.500244140625, "logps/rejected": -248.48162841796875, "loss": 5.5443, "margin_dpo/margin_mean": 0.024476364254951477, "margin_dpo/margin_std": 0.6870174407958984, "step": 17 }, { "KL/chosen_KL_mean": 0.02175140380859375, "KL/mean": -0.027004405856132507, "KL/rejected_KL_mean": -0.07576942443847656, "KL/std": 0.5336968898773193, "epoch": 0.03769633507853403, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09750780463218689, "fcm_dpo/q_t": 0.49975621700286865, "grad_norm": 29.709609985351562, "learning_rate": 1.7708333333333334e-07, "logits/chosen": -0.6183552742004395, "logits/rejected": -0.6245881915092468, "logps/chosen": -293.6875, "logps/ref_chosen": -293.709228515625, "logps/ref_rejected": -274.5875244140625, "logps/rejected": -274.663330078125, "loss": 5.5413, "margin_dpo/margin_mean": 0.09750711917877197, "margin_dpo/margin_std": 0.7472649216651917, "step": 18 }, { "KL/chosen_KL_mean": 0.034343719482421875, "KL/mean": 0.04221701622009277, "KL/rejected_KL_mean": 0.05010414123535156, "KL/std": 0.4715411067008972, "epoch": 0.039790575916230364, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.015760570764541626, "fcm_dpo/q_t": 0.5000393986701965, "grad_norm": 28.117584228515625, "learning_rate": 1.875e-07, "logits/chosen": -0.6209872961044312, "logits/rejected": -0.6168931722640991, "logps/chosen": -280.2313232421875, "logps/ref_chosen": -280.26568603515625, "logps/ref_rejected": -259.9742736816406, "logps/rejected": -259.9241638183594, "loss": 5.5459, "margin_dpo/margin_mean": -0.01576043665409088, "margin_dpo/margin_std": 0.6759567260742188, "step": 19 }, { "KL/chosen_KL_mean": 0.1446208953857422, "KL/mean": 0.09324289113283157, "KL/rejected_KL_mean": 0.04186439514160156, "KL/std": 0.5209404826164246, "epoch": 0.041884816753926704, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10276265442371368, "fcm_dpo/q_t": 0.49974310398101807, "grad_norm": 29.73322868347168, "learning_rate": 1.9791666666666664e-07, "logits/chosen": -0.6250026822090149, "logits/rejected": -0.6576300263404846, "logps/chosen": -303.7508239746094, "logps/ref_chosen": -303.8954162597656, "logps/ref_rejected": -260.214599609375, "logps/rejected": -260.1727600097656, "loss": 5.5411, "margin_dpo/margin_mean": 0.10276344418525696, "margin_dpo/margin_std": 0.7132169008255005, "step": 20 }, { "KL/chosen_KL_mean": 0.1494312286376953, "KL/mean": 0.05350485444068909, "KL/rejected_KL_mean": -0.04242897033691406, "KL/std": 0.5489564538002014, "epoch": 0.04397905759162304, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19186845421791077, "fcm_dpo/q_t": 0.49952030181884766, "grad_norm": 34.93364715576172, "learning_rate": 2.0833333333333333e-07, "logits/chosen": -0.643299400806427, "logits/rejected": -0.670960545539856, "logps/chosen": -301.3840637207031, "logps/ref_chosen": -301.5334777832031, "logps/ref_rejected": -280.28900146484375, "logps/rejected": -280.3314208984375, "loss": 5.5376, "margin_dpo/margin_mean": 0.1918683797121048, "margin_dpo/margin_std": 0.7002838850021362, "step": 21 }, { "KL/chosen_KL_mean": 0.08012008666992188, "KL/mean": 0.055122010409832, "KL/rejected_KL_mean": 0.030115127563476562, "KL/std": 0.5340926051139832, "epoch": 0.04607329842931937, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.050002023577690125, "fcm_dpo/q_t": 0.4998749792575836, "grad_norm": 25.30601692199707, "learning_rate": 2.1875e-07, "logits/chosen": -0.6519259810447693, "logits/rejected": -0.6534304022789001, "logps/chosen": -259.9150390625, "logps/ref_chosen": -259.9951477050781, "logps/ref_rejected": -243.0721435546875, "logps/rejected": -243.04205322265625, "loss": 5.5432, "margin_dpo/margin_mean": 0.05000239610671997, "margin_dpo/margin_std": 0.7330818176269531, "step": 22 }, { "KL/chosen_KL_mean": 0.08829116821289062, "KL/mean": 0.03367228806018829, "KL/rejected_KL_mean": -0.0209503173828125, "KL/std": 0.5275300145149231, "epoch": 0.048167539267015703, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10923662781715393, "fcm_dpo/q_t": 0.49972692131996155, "grad_norm": 27.844369888305664, "learning_rate": 2.2916666666666663e-07, "logits/chosen": -0.6188793778419495, "logits/rejected": -0.6528036594390869, "logps/chosen": -282.09246826171875, "logps/ref_chosen": -282.1807556152344, "logps/ref_rejected": -265.0758056640625, "logps/rejected": -265.09674072265625, "loss": 5.5409, "margin_dpo/margin_mean": 0.10923708975315094, "margin_dpo/margin_std": 0.6296786665916443, "step": 23 }, { "KL/chosen_KL_mean": 0.15508651733398438, "KL/mean": 0.03027293086051941, "KL/rejected_KL_mean": -0.09454917907714844, "KL/std": 0.5798429846763611, "epoch": 0.050261780104712044, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24962344765663147, "fcm_dpo/q_t": 0.49937596917152405, "grad_norm": 30.161964416503906, "learning_rate": 2.3958333333333335e-07, "logits/chosen": -0.6673365831375122, "logits/rejected": -0.579649031162262, "logps/chosen": -301.0245361328125, "logps/ref_chosen": -301.17962646484375, "logps/ref_rejected": -302.12786865234375, "logps/rejected": -302.222412109375, "loss": 5.5353, "margin_dpo/margin_mean": 0.24962244927883148, "margin_dpo/margin_std": 0.759114682674408, "step": 24 }, { "KL/chosen_KL_mean": 0.19303131103515625, "KL/mean": 0.11812683194875717, "KL/rejected_KL_mean": 0.04321861267089844, "KL/std": 0.5453072190284729, "epoch": 0.05235602094240838, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14981667697429657, "fcm_dpo/q_t": 0.49962544441223145, "grad_norm": 27.210227966308594, "learning_rate": 2.5e-07, "logits/chosen": -0.602566123008728, "logits/rejected": -0.6109638214111328, "logps/chosen": -246.55343627929688, "logps/ref_chosen": -246.74649047851562, "logps/ref_rejected": -235.55638122558594, "logps/rejected": -235.5131378173828, "loss": 5.5392, "margin_dpo/margin_mean": 0.14981591701507568, "margin_dpo/margin_std": 0.7190626859664917, "step": 25 }, { "KL/chosen_KL_mean": 0.21875381469726562, "KL/mean": 0.13972681760787964, "KL/rejected_KL_mean": 0.06070518493652344, "KL/std": 0.5684024691581726, "epoch": 0.05445026178010471, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15804807841777802, "fcm_dpo/q_t": 0.4996049404144287, "grad_norm": 28.61292266845703, "learning_rate": 2.604166666666667e-07, "logits/chosen": -0.6518624424934387, "logits/rejected": -0.6661399602890015, "logps/chosen": -281.97686767578125, "logps/ref_chosen": -282.1955871582031, "logps/ref_rejected": -235.3135528564453, "logps/rejected": -235.25283813476562, "loss": 5.5389, "margin_dpo/margin_mean": 0.15804804861545563, "margin_dpo/margin_std": 0.7699212431907654, "step": 26 }, { "KL/chosen_KL_mean": 0.2905387878417969, "KL/mean": 0.16653002798557281, "KL/rejected_KL_mean": 0.042522430419921875, "KL/std": 0.6718394160270691, "epoch": 0.05654450261780105, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24801474809646606, "fcm_dpo/q_t": 0.4993799924850464, "grad_norm": 27.75027084350586, "learning_rate": 2.708333333333333e-07, "logits/chosen": -0.6572801470756531, "logits/rejected": -0.6775311827659607, "logps/chosen": -323.5657958984375, "logps/ref_chosen": -323.8563537597656, "logps/ref_rejected": -245.968017578125, "logps/rejected": -245.9254913330078, "loss": 5.5354, "margin_dpo/margin_mean": 0.24801427125930786, "margin_dpo/margin_std": 0.9611263275146484, "step": 27 }, { "KL/chosen_KL_mean": 0.23924636840820312, "KL/mean": 0.16367939114570618, "KL/rejected_KL_mean": 0.088104248046875, "KL/std": 0.585762083530426, "epoch": 0.05863874345549738, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1511506289243698, "fcm_dpo/q_t": 0.4996221363544464, "grad_norm": 26.398744583129883, "learning_rate": 2.8125e-07, "logits/chosen": -0.6180057525634766, "logits/rejected": -0.6270272731781006, "logps/chosen": -248.00747680664062, "logps/ref_chosen": -248.24673461914062, "logps/ref_rejected": -240.0382080078125, "logps/rejected": -239.95013427734375, "loss": 5.5392, "margin_dpo/margin_mean": 0.15115085244178772, "margin_dpo/margin_std": 0.8178046941757202, "step": 28 }, { "KL/chosen_KL_mean": 0.3459911346435547, "KL/mean": 0.24326342344284058, "KL/rejected_KL_mean": 0.140533447265625, "KL/std": 0.6447778940200806, "epoch": 0.060732984293193716, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20546211302280426, "fcm_dpo/q_t": 0.49948635697364807, "grad_norm": 30.924745559692383, "learning_rate": 2.916666666666667e-07, "logits/chosen": -0.5906850695610046, "logits/rejected": -0.6126277446746826, "logps/chosen": -317.910400390625, "logps/ref_chosen": -318.2564392089844, "logps/ref_rejected": -286.75848388671875, "logps/rejected": -286.6180114746094, "loss": 5.537, "margin_dpo/margin_mean": 0.2054625153541565, "margin_dpo/margin_std": 0.8641461133956909, "step": 29 }, { "KL/chosen_KL_mean": 0.3351631164550781, "KL/mean": 0.2142288237810135, "KL/rejected_KL_mean": 0.09329414367675781, "KL/std": 0.6597960591316223, "epoch": 0.06282722513089005, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24187538027763367, "fcm_dpo/q_t": 0.49939531087875366, "grad_norm": 28.964019775390625, "learning_rate": 3.020833333333333e-07, "logits/chosen": -0.6030691266059875, "logits/rejected": -0.6204848885536194, "logps/chosen": -252.71395874023438, "logps/ref_chosen": -253.0491485595703, "logps/ref_rejected": -261.30029296875, "logps/rejected": -261.20697021484375, "loss": 5.5356, "margin_dpo/margin_mean": 0.24187535047531128, "margin_dpo/margin_std": 0.8954479694366455, "step": 30 }, { "KL/chosen_KL_mean": 0.3509845733642578, "KL/mean": 0.256618857383728, "KL/rejected_KL_mean": 0.16225242614746094, "KL/std": 0.726455569267273, "epoch": 0.06492146596858639, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18873509764671326, "fcm_dpo/q_t": 0.499528169631958, "grad_norm": 25.20211410522461, "learning_rate": 3.1249999999999997e-07, "logits/chosen": -0.6544696688652039, "logits/rejected": -0.6891241073608398, "logps/chosen": -247.80206298828125, "logps/ref_chosen": -248.15301513671875, "logps/ref_rejected": -203.17703247070312, "logps/rejected": -203.01480102539062, "loss": 5.5377, "margin_dpo/margin_mean": 0.1887347549200058, "margin_dpo/margin_std": 0.9368091225624084, "step": 31 }, { "KL/chosen_KL_mean": 0.5199089050292969, "KL/mean": 0.3558507561683655, "KL/rejected_KL_mean": 0.1917896270751953, "KL/std": 0.686607837677002, "epoch": 0.06701570680628273, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.32811465859413147, "fcm_dpo/q_t": 0.4991797208786011, "grad_norm": 29.66213607788086, "learning_rate": 3.2291666666666666e-07, "logits/chosen": -0.6054428219795227, "logits/rejected": -0.6095200777053833, "logps/chosen": -305.0200500488281, "logps/ref_chosen": -305.5399475097656, "logps/ref_rejected": -267.6527099609375, "logps/rejected": -267.4609069824219, "loss": 5.5322, "margin_dpo/margin_mean": 0.3281139135360718, "margin_dpo/margin_std": 0.8977186679840088, "step": 32 }, { "KL/chosen_KL_mean": 0.6279258728027344, "KL/mean": 0.4263458847999573, "KL/rejected_KL_mean": 0.2247791290283203, "KL/std": 0.8318162560462952, "epoch": 0.06910994764397906, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4031338691711426, "fcm_dpo/q_t": 0.4989922046661377, "grad_norm": 28.33131217956543, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.6404253840446472, "logits/rejected": -0.6528037190437317, "logps/chosen": -285.6055908203125, "logps/ref_chosen": -286.2335205078125, "logps/ref_rejected": -255.38748168945312, "logps/rejected": -255.16270446777344, "loss": 5.5292, "margin_dpo/margin_mean": 0.40313342213630676, "margin_dpo/margin_std": 1.0211021900177002, "step": 33 }, { "KL/chosen_KL_mean": 0.7115211486816406, "KL/mean": 0.37938636541366577, "KL/rejected_KL_mean": 0.04724884033203125, "KL/std": 1.0121040344238281, "epoch": 0.0712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6642796993255615, "fcm_dpo/q_t": 0.49833938479423523, "grad_norm": 31.110395431518555, "learning_rate": 3.4375e-07, "logits/chosen": -0.6285020112991333, "logits/rejected": -0.6383810043334961, "logps/chosen": -340.8804931640625, "logps/ref_chosen": -341.5920104980469, "logps/ref_rejected": -278.8866882324219, "logps/rejected": -278.8394470214844, "loss": 5.5188, "margin_dpo/margin_mean": 0.6642793416976929, "margin_dpo/margin_std": 1.1777315139770508, "step": 34 }, { "KL/chosen_KL_mean": 0.6163215637207031, "KL/mean": 0.4386853873729706, "KL/rejected_KL_mean": 0.26105690002441406, "KL/std": 1.0218110084533691, "epoch": 0.07329842931937172, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.35526472330093384, "fcm_dpo/q_t": 0.4991118907928467, "grad_norm": 26.492263793945312, "learning_rate": 3.541666666666667e-07, "logits/chosen": -0.6304330229759216, "logits/rejected": -0.6505473256111145, "logps/chosen": -264.4632263183594, "logps/ref_chosen": -265.0795593261719, "logps/ref_rejected": -264.4876708984375, "logps/rejected": -264.22662353515625, "loss": 5.5312, "margin_dpo/margin_mean": 0.3552646338939667, "margin_dpo/margin_std": 1.282766580581665, "step": 35 }, { "KL/chosen_KL_mean": 0.8185024261474609, "KL/mean": 0.5113904476165771, "KL/rejected_KL_mean": 0.2042827606201172, "KL/std": 1.0968939065933228, "epoch": 0.07539267015706806, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6142216920852661, "fcm_dpo/q_t": 0.49846455454826355, "grad_norm": 31.88554573059082, "learning_rate": 3.645833333333333e-07, "logits/chosen": -0.5922667980194092, "logits/rejected": -0.6093138456344604, "logps/chosen": -296.50762939453125, "logps/ref_chosen": -297.3261413574219, "logps/ref_rejected": -282.09515380859375, "logps/rejected": -281.890869140625, "loss": 5.5209, "margin_dpo/margin_mean": 0.6142206788063049, "margin_dpo/margin_std": 1.4092711210250854, "step": 36 }, { "KL/chosen_KL_mean": 0.7976875305175781, "KL/mean": 0.5208411812782288, "KL/rejected_KL_mean": 0.24400711059570312, "KL/std": 1.1177294254302979, "epoch": 0.0774869109947644, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5536773204803467, "fcm_dpo/q_t": 0.49861592054367065, "grad_norm": 30.80563735961914, "learning_rate": 3.75e-07, "logits/chosen": -0.6051210761070251, "logits/rejected": -0.6199328899383545, "logps/chosen": -313.236328125, "logps/ref_chosen": -314.0340270996094, "logps/ref_rejected": -299.3437805175781, "logps/rejected": -299.09979248046875, "loss": 5.5233, "margin_dpo/margin_mean": 0.5536766052246094, "margin_dpo/margin_std": 1.531832218170166, "step": 37 }, { "KL/chosen_KL_mean": 0.9667644500732422, "KL/mean": 0.6877572536468506, "KL/rejected_KL_mean": 0.4087562561035156, "KL/std": 1.1976516246795654, "epoch": 0.07958115183246073, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5580183267593384, "fcm_dpo/q_t": 0.49860504269599915, "grad_norm": 28.460735321044922, "learning_rate": 3.8541666666666665e-07, "logits/chosen": -0.6329147219657898, "logits/rejected": -0.6447269916534424, "logps/chosen": -281.5744323730469, "logps/ref_chosen": -282.54119873046875, "logps/ref_rejected": -269.7773132324219, "logps/rejected": -269.3685302734375, "loss": 5.5231, "margin_dpo/margin_mean": 0.5580188632011414, "margin_dpo/margin_std": 1.5111653804779053, "step": 38 }, { "KL/chosen_KL_mean": 1.3426704406738281, "KL/mean": 0.6943768262863159, "KL/rejected_KL_mean": 0.04608726501464844, "KL/std": 1.4975932836532593, "epoch": 0.08167539267015707, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2965832948684692, "fcm_dpo/q_t": 0.49675899744033813, "grad_norm": 29.592939376831055, "learning_rate": 3.958333333333333e-07, "logits/chosen": -0.6133351922035217, "logits/rejected": -0.6270293593406677, "logps/chosen": -275.4302673339844, "logps/ref_chosen": -276.7729187011719, "logps/ref_rejected": -249.95889282226562, "logps/rejected": -249.91278076171875, "loss": 5.4938, "margin_dpo/margin_mean": 1.2965826988220215, "margin_dpo/margin_std": 1.7473759651184082, "step": 39 }, { "KL/chosen_KL_mean": 1.0723190307617188, "KL/mean": 0.676328182220459, "KL/rejected_KL_mean": 0.2803363800048828, "KL/std": 1.3713598251342773, "epoch": 0.08376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7919808030128479, "fcm_dpo/q_t": 0.49802035093307495, "grad_norm": 27.339101791381836, "learning_rate": 4.0625e-07, "logits/chosen": -0.6305021047592163, "logits/rejected": -0.6675591468811035, "logps/chosen": -283.2347717285156, "logps/ref_chosen": -284.30706787109375, "logps/ref_rejected": -244.4459991455078, "logps/rejected": -244.16563415527344, "loss": 5.5139, "margin_dpo/margin_mean": 0.7919806241989136, "margin_dpo/margin_std": 1.7995089292526245, "step": 40 }, { "KL/chosen_KL_mean": 1.1165599822998047, "KL/mean": 0.7140932679176331, "KL/rejected_KL_mean": 0.3116283416748047, "KL/std": 1.5022192001342773, "epoch": 0.08586387434554973, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8049377202987671, "fcm_dpo/q_t": 0.49798786640167236, "grad_norm": 30.32787322998047, "learning_rate": 4.1666666666666667e-07, "logits/chosen": -0.6256983876228333, "logits/rejected": -0.6507747769355774, "logps/chosen": -292.6986389160156, "logps/ref_chosen": -293.8151550292969, "logps/ref_rejected": -252.16815185546875, "logps/rejected": -251.85653686523438, "loss": 5.5134, "margin_dpo/margin_mean": 0.8049373626708984, "margin_dpo/margin_std": 1.754093050956726, "step": 41 }, { "KL/chosen_KL_mean": 1.208505630493164, "KL/mean": 0.7730397582054138, "KL/rejected_KL_mean": 0.33758544921875, "KL/std": 1.6781818866729736, "epoch": 0.08795811518324607, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8709163665771484, "fcm_dpo/q_t": 0.4978230595588684, "grad_norm": 27.442352294921875, "learning_rate": 4.270833333333333e-07, "logits/chosen": -0.632148265838623, "logits/rejected": -0.6499121189117432, "logps/chosen": -251.55172729492188, "logps/ref_chosen": -252.76023864746094, "logps/ref_rejected": -261.0414733886719, "logps/rejected": -260.7038879394531, "loss": 5.511, "margin_dpo/margin_mean": 0.8709160089492798, "margin_dpo/margin_std": 2.178144693374634, "step": 42 }, { "KL/chosen_KL_mean": 1.2755184173583984, "KL/mean": 0.6735259890556335, "KL/rejected_KL_mean": 0.07152366638183594, "KL/std": 1.878867268562317, "epoch": 0.09005235602094241, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2040010690689087, "fcm_dpo/q_t": 0.4969903826713562, "grad_norm": 30.027923583984375, "learning_rate": 4.375e-07, "logits/chosen": -0.5912747383117676, "logits/rejected": -0.60638427734375, "logps/chosen": -315.5592041015625, "logps/ref_chosen": -316.8347473144531, "logps/ref_rejected": -273.7649230957031, "logps/rejected": -273.693359375, "loss": 5.4977, "margin_dpo/margin_mean": 1.2040014266967773, "margin_dpo/margin_std": 2.2428250312805176, "step": 43 }, { "KL/chosen_KL_mean": 1.544687271118164, "KL/mean": 0.6966075897216797, "KL/rejected_KL_mean": -0.15147972106933594, "KL/std": 2.2357659339904785, "epoch": 0.09214659685863874, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.696164846420288, "fcm_dpo/q_t": 0.49576056003570557, "grad_norm": 30.86996078491211, "learning_rate": 4.479166666666667e-07, "logits/chosen": -0.5922445058822632, "logits/rejected": -0.5905203223228455, "logps/chosen": -285.33099365234375, "logps/ref_chosen": -286.8757019042969, "logps/ref_rejected": -282.4681396484375, "logps/rejected": -282.6196594238281, "loss": 5.4786, "margin_dpo/margin_mean": 1.696164846420288, "margin_dpo/margin_std": 3.0431594848632812, "step": 44 }, { "KL/chosen_KL_mean": 1.7514400482177734, "KL/mean": 1.0452030897140503, "KL/rejected_KL_mean": 0.33896636962890625, "KL/std": 2.3070363998413086, "epoch": 0.09424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4124752283096313, "fcm_dpo/q_t": 0.49646997451782227, "grad_norm": 28.983610153198242, "learning_rate": 4.5833333333333327e-07, "logits/chosen": -0.6918033957481384, "logits/rejected": -0.7159820199012756, "logps/chosen": -322.511962890625, "logps/ref_chosen": -324.2633972167969, "logps/ref_rejected": -293.09466552734375, "logps/rejected": -292.7557067871094, "loss": 5.4897, "margin_dpo/margin_mean": 1.412475347518921, "margin_dpo/margin_std": 2.8487558364868164, "step": 45 }, { "KL/chosen_KL_mean": 1.676168441772461, "KL/mean": 0.9087100028991699, "KL/rejected_KL_mean": 0.14125442504882812, "KL/std": 2.392753839492798, "epoch": 0.09633507853403141, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5349185466766357, "fcm_dpo/q_t": 0.4961636960506439, "grad_norm": 30.31817626953125, "learning_rate": 4.6874999999999996e-07, "logits/chosen": -0.6293231248855591, "logits/rejected": -0.643032968044281, "logps/chosen": -296.6595764160156, "logps/ref_chosen": -298.3357238769531, "logps/ref_rejected": -267.66204833984375, "logps/rejected": -267.520751953125, "loss": 5.4847, "margin_dpo/margin_mean": 1.5349185466766357, "margin_dpo/margin_std": 2.6055755615234375, "step": 46 }, { "KL/chosen_KL_mean": 1.4842796325683594, "KL/mean": 0.9708749055862427, "KL/rejected_KL_mean": 0.4574623107910156, "KL/std": 2.3160178661346436, "epoch": 0.09842931937172775, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0268166065216064, "fcm_dpo/q_t": 0.49743345379829407, "grad_norm": 26.303861618041992, "learning_rate": 4.791666666666667e-07, "logits/chosen": -0.6067807078361511, "logits/rejected": -0.627469539642334, "logps/chosen": -261.0826416015625, "logps/ref_chosen": -262.5669250488281, "logps/ref_rejected": -258.70989990234375, "logps/rejected": -258.2524108886719, "loss": 5.5054, "margin_dpo/margin_mean": 1.0268152952194214, "margin_dpo/margin_std": 3.289706230163574, "step": 47 }, { "KL/chosen_KL_mean": 1.8775215148925781, "KL/mean": 1.1411019563674927, "KL/rejected_KL_mean": 0.4046955108642578, "KL/std": 2.5993833541870117, "epoch": 0.10052356020942409, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4728214740753174, "fcm_dpo/q_t": 0.49632006883621216, "grad_norm": 27.548425674438477, "learning_rate": 4.895833333333333e-07, "logits/chosen": -0.6069186925888062, "logits/rejected": -0.631554126739502, "logps/chosen": -267.61572265625, "logps/ref_chosen": -269.4932556152344, "logps/ref_rejected": -241.888916015625, "logps/rejected": -241.48423767089844, "loss": 5.4876, "margin_dpo/margin_mean": 1.4728212356567383, "margin_dpo/margin_std": 3.0481390953063965, "step": 48 }, { "KL/chosen_KL_mean": 2.2043685913085938, "KL/mean": 1.3517543077468872, "KL/rejected_KL_mean": 0.4991436004638672, "KL/std": 2.6367743015289307, "epoch": 0.10261780104712041, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7052276134490967, "fcm_dpo/q_t": 0.4957382082939148, "grad_norm": 27.595603942871094, "learning_rate": 5e-07, "logits/chosen": -0.6787562966346741, "logits/rejected": -0.6629523038864136, "logps/chosen": -255.68011474609375, "logps/ref_chosen": -257.8844909667969, "logps/ref_rejected": -256.8912048339844, "logps/rejected": -256.3920593261719, "loss": 5.4785, "margin_dpo/margin_mean": 1.7052276134490967, "margin_dpo/margin_std": 3.387364625930786, "step": 49 }, { "KL/chosen_KL_mean": 1.974029541015625, "KL/mean": 1.4002656936645508, "KL/rejected_KL_mean": 0.826507568359375, "KL/std": 3.0437886714935303, "epoch": 0.10471204188481675, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.147522211074829, "fcm_dpo/q_t": 0.49713265895843506, "grad_norm": 27.980398178100586, "learning_rate": 4.999932966293553e-07, "logits/chosen": -0.633937418460846, "logits/rejected": -0.6559914350509644, "logps/chosen": -299.6548156738281, "logps/ref_chosen": -301.62884521484375, "logps/ref_rejected": -298.2716064453125, "logps/rejected": -297.4450988769531, "loss": 5.5011, "margin_dpo/margin_mean": 1.1475228071212769, "margin_dpo/margin_std": 3.857665777206421, "step": 50 }, { "KL/chosen_KL_mean": 2.1326828002929688, "KL/mean": 1.1401723623275757, "KL/rejected_KL_mean": 0.1476593017578125, "KL/std": 3.5523159503936768, "epoch": 0.1068062827225131, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9850341081619263, "fcm_dpo/q_t": 0.4950404763221741, "grad_norm": 29.13739585876465, "learning_rate": 4.999731868769026e-07, "logits/chosen": -0.6125076413154602, "logits/rejected": -0.6065410375595093, "logps/chosen": -267.2397155761719, "logps/ref_chosen": -269.37237548828125, "logps/ref_rejected": -297.0167541503906, "logps/rejected": -296.8690490722656, "loss": 5.4685, "margin_dpo/margin_mean": 1.9850351810455322, "margin_dpo/margin_std": 4.482949256896973, "step": 51 }, { "KL/chosen_KL_mean": 2.7480640411376953, "KL/mean": 1.210340976715088, "KL/rejected_KL_mean": -0.3273906707763672, "KL/std": 3.7830421924591064, "epoch": 0.10890052356020942, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.0754599571228027, "fcm_dpo/q_t": 0.49231770634651184, "grad_norm": 30.282175064086914, "learning_rate": 4.99939671821067e-07, "logits/chosen": -0.6546225547790527, "logits/rejected": -0.661052942276001, "logps/chosen": -304.15472412109375, "logps/ref_chosen": -306.9028015136719, "logps/ref_rejected": -281.24737548828125, "logps/rejected": -281.5747375488281, "loss": 5.4256, "margin_dpo/margin_mean": 3.075460195541382, "margin_dpo/margin_std": 4.735509395599365, "step": 52 }, { "KL/chosen_KL_mean": 2.7503910064697266, "KL/mean": 1.6823850870132446, "KL/rejected_KL_mean": 0.6143932342529297, "KL/std": 4.089887619018555, "epoch": 0.11099476439790576, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.136000633239746, "fcm_dpo/q_t": 0.49466413259506226, "grad_norm": 29.992427825927734, "learning_rate": 4.998927532591591e-07, "logits/chosen": -0.6430725455284119, "logits/rejected": -0.6831983327865601, "logps/chosen": -283.2255554199219, "logps/ref_chosen": -285.9759521484375, "logps/ref_rejected": -273.9073486328125, "logps/rejected": -273.29296875, "loss": 5.4633, "margin_dpo/margin_mean": 2.136000156402588, "margin_dpo/margin_std": 5.067354679107666, "step": 53 }, { "KL/chosen_KL_mean": 2.6715049743652344, "KL/mean": 1.728779911994934, "KL/rejected_KL_mean": 0.7860507965087891, "KL/std": 4.300192356109619, "epoch": 0.1130890052356021, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8854495286941528, "fcm_dpo/q_t": 0.4952917695045471, "grad_norm": 26.448970794677734, "learning_rate": 4.998324337072792e-07, "logits/chosen": -0.6894734501838684, "logits/rejected": -0.6976596713066101, "logps/chosen": -303.8331298828125, "logps/ref_chosen": -306.504638671875, "logps/ref_rejected": -272.67431640625, "logps/rejected": -271.8882751464844, "loss": 5.4733, "margin_dpo/margin_mean": 1.8854488134384155, "margin_dpo/margin_std": 5.352308750152588, "step": 54 }, { "KL/chosen_KL_mean": 1.8048782348632812, "KL/mean": 0.6566358804702759, "KL/rejected_KL_mean": -0.49161338806152344, "KL/std": 3.7019548416137695, "epoch": 0.11518324607329843, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.296487808227539, "fcm_dpo/q_t": 0.494264155626297, "grad_norm": 24.89438819885254, "learning_rate": 4.997587164001815e-07, "logits/chosen": -0.6316308975219727, "logits/rejected": -0.6350709199905396, "logps/chosen": -220.52525329589844, "logps/ref_chosen": -222.33013916015625, "logps/ref_rejected": -206.59571838378906, "logps/rejected": -207.0873260498047, "loss": 5.4564, "margin_dpo/margin_mean": 2.2964882850646973, "margin_dpo/margin_std": 4.970940113067627, "step": 55 }, { "KL/chosen_KL_mean": 2.7975711822509766, "KL/mean": 1.2214081287384033, "KL/rejected_KL_mean": -0.35475921630859375, "KL/std": 4.772672653198242, "epoch": 0.11727748691099477, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1523289680480957, "fcm_dpo/q_t": 0.4921315908432007, "grad_norm": 27.497465133666992, "learning_rate": 4.996716052911017e-07, "logits/chosen": -0.6138368844985962, "logits/rejected": -0.6288657188415527, "logps/chosen": -247.6805877685547, "logps/ref_chosen": -250.47816467285156, "logps/ref_rejected": -228.25848388671875, "logps/rejected": -228.61325073242188, "loss": 5.4244, "margin_dpo/margin_mean": 3.1523287296295166, "margin_dpo/margin_std": 5.94834041595459, "step": 56 }, { "KL/chosen_KL_mean": 3.8411941528320312, "KL/mean": 1.778662919998169, "KL/rejected_KL_mean": -0.2838726043701172, "KL/std": 4.771164417266846, "epoch": 0.1193717277486911, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.125072479248047, "fcm_dpo/q_t": 0.48969873785972595, "grad_norm": 30.785301208496094, "learning_rate": 4.99571105051544e-07, "logits/chosen": -0.6994465589523315, "logits/rejected": -0.6702845096588135, "logps/chosen": -311.2783508300781, "logps/ref_chosen": -315.1195373535156, "logps/ref_rejected": -272.755615234375, "logps/rejected": -273.03948974609375, "loss": 5.3852, "margin_dpo/margin_mean": 4.125072002410889, "margin_dpo/margin_std": 5.533132553100586, "step": 57 }, { "KL/chosen_KL_mean": 2.443532943725586, "KL/mean": 1.277455449104309, "KL/rejected_KL_mean": 0.11137962341308594, "KL/std": 4.433131217956543, "epoch": 0.12146596858638743, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.3321542739868164, "fcm_dpo/q_t": 0.49417322874069214, "grad_norm": 27.45433235168457, "learning_rate": 4.994572210710314e-07, "logits/chosen": -0.6160473823547363, "logits/rejected": -0.6402156352996826, "logps/chosen": -262.7381591796875, "logps/ref_chosen": -265.1816711425781, "logps/ref_rejected": -268.2203369140625, "logps/rejected": -268.10894775390625, "loss": 5.456, "margin_dpo/margin_mean": 2.3321540355682373, "margin_dpo/margin_std": 5.656576156616211, "step": 58 }, { "KL/chosen_KL_mean": 2.1098995208740234, "KL/mean": 1.2807146310806274, "KL/rejected_KL_mean": 0.4515361785888672, "KL/std": 5.291559219360352, "epoch": 0.12356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6583688259124756, "fcm_dpo/q_t": 0.49586057662963867, "grad_norm": 29.933673858642578, "learning_rate": 4.993299594568162e-07, "logits/chosen": -0.6065900921821594, "logits/rejected": -0.5998431444168091, "logps/chosen": -284.2440490722656, "logps/ref_chosen": -286.35394287109375, "logps/ref_rejected": -260.6757507324219, "logps/rejected": -260.2242126464844, "loss": 5.485, "margin_dpo/margin_mean": 1.6583688259124756, "margin_dpo/margin_std": 7.080748558044434, "step": 59 }, { "KL/chosen_KL_mean": 2.876659393310547, "KL/mean": 1.1580764055252075, "KL/rejected_KL_mean": -0.5605106353759766, "KL/std": 5.181823253631592, "epoch": 0.1256544502617801, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4371724128723145, "fcm_dpo/q_t": 0.49141788482666016, "grad_norm": 28.205102920532227, "learning_rate": 4.991893270335525e-07, "logits/chosen": -0.687022864818573, "logits/rejected": -0.7121449708938599, "logps/chosen": -255.8719482421875, "logps/ref_chosen": -258.74859619140625, "logps/ref_rejected": -255.04893493652344, "logps/rejected": -255.60943603515625, "loss": 5.4144, "margin_dpo/margin_mean": 3.4371743202209473, "margin_dpo/margin_std": 7.137161731719971, "step": 60 }, { "KL/chosen_KL_mean": 3.009805679321289, "KL/mean": 1.4096331596374512, "KL/rejected_KL_mean": -0.19052886962890625, "KL/std": 5.557041645050049, "epoch": 0.12774869109947645, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.200338363647461, "fcm_dpo/q_t": 0.4920124113559723, "grad_norm": 29.904375076293945, "learning_rate": 4.990353313429303e-07, "logits/chosen": -0.6517542004585266, "logits/rejected": -0.668937087059021, "logps/chosen": -275.4580993652344, "logps/ref_chosen": -278.4678955078125, "logps/ref_rejected": -252.02720642089844, "logps/rejected": -252.21771240234375, "loss": 5.4249, "margin_dpo/margin_mean": 3.2003390789031982, "margin_dpo/margin_std": 7.634793281555176, "step": 61 }, { "KL/chosen_KL_mean": 3.993856430053711, "KL/mean": 2.264561414718628, "KL/rejected_KL_mean": 0.5352687835693359, "KL/std": 5.766189098358154, "epoch": 0.12984293193717278, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.458585739135742, "fcm_dpo/q_t": 0.49137410521507263, "grad_norm": 26.703571319580078, "learning_rate": 4.988679806432711e-07, "logits/chosen": -0.6075385808944702, "logits/rejected": -0.6490654945373535, "logps/chosen": -268.9304504394531, "logps/ref_chosen": -272.92431640625, "logps/ref_rejected": -260.7935485839844, "logps/rejected": -260.2582702636719, "loss": 5.4139, "margin_dpo/margin_mean": 3.458585739135742, "margin_dpo/margin_std": 7.508395195007324, "step": 62 }, { "KL/chosen_KL_mean": 3.311128616333008, "KL/mean": 1.574736475944519, "KL/rejected_KL_mean": -0.1616668701171875, "KL/std": 5.540885925292969, "epoch": 0.1319371727748691, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.472792863845825, "fcm_dpo/q_t": 0.4913320541381836, "grad_norm": 28.29745864868164, "learning_rate": 4.986872839090852e-07, "logits/chosen": -0.6676257848739624, "logits/rejected": -0.6745007038116455, "logps/chosen": -273.7778625488281, "logps/ref_chosen": -277.0889892578125, "logps/ref_rejected": -273.3413391113281, "logps/rejected": -273.50299072265625, "loss": 5.4129, "margin_dpo/margin_mean": 3.4727916717529297, "margin_dpo/margin_std": 7.09786319732666, "step": 63 }, { "KL/chosen_KL_mean": 4.158119201660156, "KL/mean": 1.989778757095337, "KL/rejected_KL_mean": -0.1785602569580078, "KL/std": 6.2935590744018555, "epoch": 0.13403141361256546, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.336686611175537, "fcm_dpo/q_t": 0.4891809821128845, "grad_norm": 28.39674186706543, "learning_rate": 4.9849325083059e-07, "logits/chosen": -0.6363846063613892, "logits/rejected": -0.6319074630737305, "logps/chosen": -279.6663818359375, "logps/ref_chosen": -283.8244934082031, "logps/ref_rejected": -263.29351806640625, "logps/rejected": -263.4720764160156, "loss": 5.381, "margin_dpo/margin_mean": 4.336686611175537, "margin_dpo/margin_std": 8.268450736999512, "step": 64 }, { "KL/chosen_KL_mean": 3.4316043853759766, "KL/mean": 1.7727466821670532, "KL/rejected_KL_mean": 0.11389350891113281, "KL/std": 5.717898368835449, "epoch": 0.13612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3177053928375244, "fcm_dpo/q_t": 0.49171876907348633, "grad_norm": 27.772890090942383, "learning_rate": 4.982858918131906e-07, "logits/chosen": -0.696242094039917, "logits/rejected": -0.6643742918968201, "logps/chosen": -261.4383239746094, "logps/ref_chosen": -264.8699645996094, "logps/ref_rejected": -268.5076904296875, "logps/rejected": -268.393798828125, "loss": 5.4193, "margin_dpo/margin_mean": 3.317704439163208, "margin_dpo/margin_std": 7.324665069580078, "step": 65 }, { "KL/chosen_KL_mean": 3.131479263305664, "KL/mean": 1.0381340980529785, "KL/rejected_KL_mean": -1.0552043914794922, "KL/std": 7.445024490356445, "epoch": 0.1382198952879581, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.186667442321777, "fcm_dpo/q_t": 0.4895693361759186, "grad_norm": 27.830961227416992, "learning_rate": 4.980652179769217e-07, "logits/chosen": -0.6735014319419861, "logits/rejected": -0.6931784749031067, "logps/chosen": -269.7968444824219, "logps/ref_chosen": -272.9283142089844, "logps/ref_rejected": -280.94696044921875, "logps/rejected": -282.0021667480469, "loss": 5.3901, "margin_dpo/margin_mean": 4.186666965484619, "margin_dpo/margin_std": 10.143901824951172, "step": 66 }, { "KL/chosen_KL_mean": 4.024763107299805, "KL/mean": 2.15836501121521, "KL/rejected_KL_mean": 0.29196739196777344, "KL/std": 7.101201057434082, "epoch": 0.14031413612565444, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.7328035831451416, "fcm_dpo/q_t": 0.49068784713745117, "grad_norm": 25.62310791015625, "learning_rate": 4.978312411558517e-07, "logits/chosen": -0.6985008716583252, "logits/rejected": -0.7321080565452576, "logps/chosen": -262.16217041015625, "logps/ref_chosen": -266.18695068359375, "logps/ref_rejected": -250.17405700683594, "logps/rejected": -249.88209533691406, "loss": 5.4053, "margin_dpo/margin_mean": 3.7328031063079834, "margin_dpo/margin_std": 8.70307445526123, "step": 67 }, { "KL/chosen_KL_mean": 2.9730300903320312, "KL/mean": 0.6432544589042664, "KL/rejected_KL_mean": -1.6865234375, "KL/std": 8.092517852783203, "epoch": 0.1424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.65955924987793, "fcm_dpo/q_t": 0.4884108304977417, "grad_norm": 28.221860885620117, "learning_rate": 4.975839738974473e-07, "logits/chosen": -0.68064284324646, "logits/rejected": -0.6935175657272339, "logps/chosen": -294.9655456542969, "logps/ref_chosen": -297.9385986328125, "logps/ref_rejected": -261.5141296386719, "logps/rejected": -263.20062255859375, "loss": 5.373, "margin_dpo/margin_mean": 4.659559726715088, "margin_dpo/margin_std": 10.586312294006348, "step": 68 }, { "KL/chosen_KL_mean": 3.898843765258789, "KL/mean": 0.8900386095046997, "KL/rejected_KL_mean": -2.1187610626220703, "KL/std": 7.852825164794922, "epoch": 0.14450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.017608642578125, "fcm_dpo/q_t": 0.48500216007232666, "grad_norm": 28.46092414855957, "learning_rate": 4.97323429461901e-07, "logits/chosen": -0.7054140567779541, "logits/rejected": -0.7388426065444946, "logps/chosen": -261.71868896484375, "logps/ref_chosen": -265.6175231933594, "logps/ref_rejected": -236.8287353515625, "logps/rejected": -238.947509765625, "loss": 5.3179, "margin_dpo/margin_mean": 6.017608642578125, "margin_dpo/margin_std": 9.605355262756348, "step": 69 }, { "KL/chosen_KL_mean": 4.312845230102539, "KL/mean": 1.372788429260254, "KL/rejected_KL_mean": -1.5672664642333984, "KL/std": 8.651166915893555, "epoch": 0.14659685863874344, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.880106449127197, "fcm_dpo/q_t": 0.4853578507900238, "grad_norm": 28.572994232177734, "learning_rate": 4.970496218214204e-07, "logits/chosen": -0.6805024147033691, "logits/rejected": -0.7144273519515991, "logps/chosen": -291.9130859375, "logps/ref_chosen": -296.2259216308594, "logps/ref_rejected": -254.68496704101562, "logps/rejected": -256.252197265625, "loss": 5.3263, "margin_dpo/margin_mean": 5.8801045417785645, "margin_dpo/margin_std": 10.940442085266113, "step": 70 }, { "KL/chosen_KL_mean": 5.105655670166016, "KL/mean": 2.5064656734466553, "KL/rejected_KL_mean": -0.09272193908691406, "KL/std": 9.356485366821289, "epoch": 0.1486910994764398, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.19838285446167, "fcm_dpo/q_t": 0.48704421520233154, "grad_norm": 28.81658172607422, "learning_rate": 4.967625656594781e-07, "logits/chosen": -0.6425278782844543, "logits/rejected": -0.630567729473114, "logps/chosen": -283.82159423828125, "logps/ref_chosen": -288.92724609375, "logps/ref_rejected": -278.6405334472656, "logps/rejected": -278.7332458496094, "loss": 5.3565, "margin_dpo/margin_mean": 5.198383331298828, "margin_dpo/margin_std": 12.317588806152344, "step": 71 }, { "KL/chosen_KL_mean": 3.2886581420898438, "KL/mean": 0.7096061706542969, "KL/rejected_KL_mean": -1.8694496154785156, "KL/std": 8.412724494934082, "epoch": 0.15078534031413612, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.158100128173828, "fcm_dpo/q_t": 0.48713764548301697, "grad_norm": 28.224729537963867, "learning_rate": 4.964622763700252e-07, "logits/chosen": -0.6948481202125549, "logits/rejected": -0.7061551809310913, "logps/chosen": -233.7566375732422, "logps/ref_chosen": -237.0452880859375, "logps/ref_rejected": -252.7946319580078, "logps/rejected": -254.66407775878906, "loss": 5.354, "margin_dpo/margin_mean": 5.158100128173828, "margin_dpo/margin_std": 10.428442001342773, "step": 72 }, { "KL/chosen_KL_mean": 4.298160552978516, "KL/mean": 1.9073495864868164, "KL/rejected_KL_mean": -0.48345375061035156, "KL/std": 8.987354278564453, "epoch": 0.15287958115183245, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.781611442565918, "fcm_dpo/q_t": 0.48807430267333984, "grad_norm": 27.856719970703125, "learning_rate": 4.961487700566646e-07, "logits/chosen": -0.6573762893676758, "logits/rejected": -0.6748519539833069, "logps/chosen": -268.75494384765625, "logps/ref_chosen": -273.0531005859375, "logps/ref_rejected": -246.8330841064453, "logps/rejected": -247.3165283203125, "loss": 5.3727, "margin_dpo/margin_mean": 4.781611442565918, "margin_dpo/margin_std": 12.327835083007812, "step": 73 }, { "KL/chosen_KL_mean": 3.3608856201171875, "KL/mean": 1.0955123901367188, "KL/rejected_KL_mean": -1.1698570251464844, "KL/std": 9.356354713439941, "epoch": 0.1549738219895288, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.530752182006836, "fcm_dpo/q_t": 0.48871979117393494, "grad_norm": 30.288177490234375, "learning_rate": 4.958220635317885e-07, "logits/chosen": -0.7295544147491455, "logits/rejected": -0.7080293893814087, "logps/chosen": -338.9210205078125, "logps/ref_chosen": -342.2818908691406, "logps/ref_rejected": -330.0293884277344, "logps/rejected": -331.1991882324219, "loss": 5.3808, "margin_dpo/margin_mean": 4.530751705169678, "margin_dpo/margin_std": 11.701099395751953, "step": 74 }, { "KL/chosen_KL_mean": 4.55201530456543, "KL/mean": 1.2454302310943604, "KL/rejected_KL_mean": -2.061159133911133, "KL/std": 9.440528869628906, "epoch": 0.15706806282722513, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.613184928894043, "fcm_dpo/q_t": 0.4835236966609955, "grad_norm": 29.505849838256836, "learning_rate": 4.954821743156767e-07, "logits/chosen": -0.6339945197105408, "logits/rejected": -0.6352519989013672, "logps/chosen": -262.3121337890625, "logps/ref_chosen": -266.8641662597656, "logps/ref_rejected": -276.8699951171875, "logps/rejected": -278.9311828613281, "loss": 5.2986, "margin_dpo/margin_mean": 6.613184452056885, "margin_dpo/margin_std": 11.05253791809082, "step": 75 }, { "KL/chosen_KL_mean": 4.158893585205078, "KL/mean": 0.5089197158813477, "KL/rejected_KL_mean": -3.1410446166992188, "KL/std": 9.98751163482666, "epoch": 0.15916230366492146, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.2999348640441895, "fcm_dpo/q_t": 0.4818638563156128, "grad_norm": 29.17070198059082, "learning_rate": 4.951291206355559e-07, "logits/chosen": -0.7195841670036316, "logits/rejected": -0.7272598743438721, "logps/chosen": -277.01568603515625, "logps/ref_chosen": -281.174560546875, "logps/ref_rejected": -263.6067199707031, "logps/rejected": -266.7477722167969, "loss": 5.2763, "margin_dpo/margin_mean": 7.299935340881348, "margin_dpo/margin_std": 12.812186241149902, "step": 76 }, { "KL/chosen_KL_mean": 3.6959686279296875, "KL/mean": 0.8079696893692017, "KL/rejected_KL_mean": -2.0800418853759766, "KL/std": 11.053658485412598, "epoch": 0.1612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.776014804840088, "fcm_dpo/q_t": 0.48565778136253357, "grad_norm": 32.9556884765625, "learning_rate": 4.947629214246236e-07, "logits/chosen": -0.564210057258606, "logits/rejected": -0.5733780860900879, "logps/chosen": -302.39935302734375, "logps/ref_chosen": -306.09527587890625, "logps/ref_rejected": -253.49569702148438, "logps/rejected": -255.57574462890625, "loss": 5.3386, "margin_dpo/margin_mean": 5.77601432800293, "margin_dpo/margin_std": 13.992176055908203, "step": 77 }, { "KL/chosen_KL_mean": 3.927644729614258, "KL/mean": -0.44709402322769165, "KL/rejected_KL_mean": -4.821832656860352, "KL/std": 11.035544395446777, "epoch": 0.16335078534031414, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.749481201171875, "fcm_dpo/q_t": 0.4782907962799072, "grad_norm": 29.68499755859375, "learning_rate": 4.943835963210323e-07, "logits/chosen": -0.6741328239440918, "logits/rejected": -0.6679074764251709, "logps/chosen": -252.97471618652344, "logps/ref_chosen": -256.90234375, "logps/ref_rejected": -211.57154846191406, "logps/rejected": -216.39340209960938, "loss": 5.2247, "margin_dpo/margin_mean": 8.749481201171875, "margin_dpo/margin_std": 14.491506576538086, "step": 78 }, { "KL/chosen_KL_mean": 3.0539989471435547, "KL/mean": -1.0648549795150757, "KL/rejected_KL_mean": -5.183706283569336, "KL/std": 11.793110847473145, "epoch": 0.16544502617801046, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.237702369689941, "fcm_dpo/q_t": 0.47956132888793945, "grad_norm": 29.673128128051758, "learning_rate": 4.939911656668361e-07, "logits/chosen": -0.6683970093727112, "logits/rejected": -0.6898422241210938, "logps/chosen": -263.2195739746094, "logps/ref_chosen": -266.2735595703125, "logps/ref_rejected": -251.57257080078125, "logps/rejected": -256.75628662109375, "loss": 5.2466, "margin_dpo/margin_mean": 8.237702369689941, "margin_dpo/margin_std": 15.015762329101562, "step": 79 }, { "KL/chosen_KL_mean": 1.7665481567382812, "KL/mean": -1.5028220415115356, "KL/rejected_KL_mean": -4.772197723388672, "KL/std": 11.338288307189941, "epoch": 0.16753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.538745880126953, "fcm_dpo/q_t": 0.4837731420993805, "grad_norm": 28.959936141967773, "learning_rate": 4.935856505068998e-07, "logits/chosen": -0.6686798334121704, "logits/rejected": -0.6984753608703613, "logps/chosen": -286.0844421386719, "logps/ref_chosen": -287.8509826660156, "logps/ref_rejected": -256.0766296386719, "logps/rejected": -260.84881591796875, "loss": 5.3077, "margin_dpo/margin_mean": 6.538746356964111, "margin_dpo/margin_std": 13.19536304473877, "step": 80 }, { "KL/chosen_KL_mean": 2.343219757080078, "KL/mean": -1.2019355297088623, "KL/rejected_KL_mean": -4.747091293334961, "KL/std": 12.21540355682373, "epoch": 0.16963350785340314, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.090311050415039, "fcm_dpo/q_t": 0.4824158251285553, "grad_norm": 28.09380340576172, "learning_rate": 4.93167072587771e-07, "logits/chosen": -0.639075756072998, "logits/rejected": -0.6379343271255493, "logps/chosen": -266.17999267578125, "logps/ref_chosen": -268.5232238769531, "logps/ref_rejected": -237.81137084960938, "logps/rejected": -242.5584716796875, "loss": 5.2961, "margin_dpo/margin_mean": 7.090311527252197, "margin_dpo/margin_std": 16.69711685180664, "step": 81 }, { "KL/chosen_KL_mean": 2.3644046783447266, "KL/mean": -1.5669686794281006, "KL/rejected_KL_mean": -5.49833869934082, "KL/std": 12.20776653289795, "epoch": 0.17172774869109947, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.862754821777344, "fcm_dpo/q_t": 0.4805517792701721, "grad_norm": 27.840482711791992, "learning_rate": 4.92735454356513e-07, "logits/chosen": -0.7332218885421753, "logits/rejected": -0.7412782907485962, "logps/chosen": -276.9995422363281, "logps/ref_chosen": -279.36395263671875, "logps/ref_rejected": -236.51365661621094, "logps/rejected": -242.01199340820312, "loss": 5.2652, "margin_dpo/margin_mean": 7.862753391265869, "margin_dpo/margin_std": 15.764188766479492, "step": 82 }, { "KL/chosen_KL_mean": 0.2912254333496094, "KL/mean": -4.046411514282227, "KL/rejected_KL_mean": -8.384056091308594, "KL/std": 13.611123085021973, "epoch": 0.17382198952879582, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.675275802612305, "fcm_dpo/q_t": 0.4784584641456604, "grad_norm": 30.742509841918945, "learning_rate": 4.922908189595017e-07, "logits/chosen": -0.6875962615013123, "logits/rejected": -0.670936107635498, "logps/chosen": -273.92803955078125, "logps/ref_chosen": -274.21923828125, "logps/ref_rejected": -276.2212219238281, "logps/rejected": -284.6052551269531, "loss": 5.2421, "margin_dpo/margin_mean": 8.675275802612305, "margin_dpo/margin_std": 18.091768264770508, "step": 83 }, { "KL/chosen_KL_mean": 0.62109375, "KL/mean": -3.0680785179138184, "KL/rejected_KL_mean": -6.757253646850586, "KL/std": 13.313782691955566, "epoch": 0.17591623036649215, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.378340721130371, "fcm_dpo/q_t": 0.4817003905773163, "grad_norm": 29.882280349731445, "learning_rate": 4.918331902411841e-07, "logits/chosen": -0.7385432124137878, "logits/rejected": -0.7533390522003174, "logps/chosen": -293.7764587402344, "logps/ref_chosen": -294.3975524902344, "logps/ref_rejected": -279.81884765625, "logps/rejected": -286.5760803222656, "loss": 5.2865, "margin_dpo/margin_mean": 7.378340721130371, "margin_dpo/margin_std": 16.830930709838867, "step": 84 }, { "KL/chosen_KL_mean": -1.5447559356689453, "KL/mean": -4.431204795837402, "KL/rejected_KL_mean": -7.317655563354492, "KL/std": 12.28141975402832, "epoch": 0.17801047120418848, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.7729082107543945, "fcm_dpo/q_t": 0.48567745089530945, "grad_norm": 29.436279296875, "learning_rate": 4.913625927427995e-07, "logits/chosen": -0.6729092597961426, "logits/rejected": -0.6811429262161255, "logps/chosen": -245.2069549560547, "logps/ref_chosen": -243.66220092773438, "logps/ref_rejected": -263.9421691894531, "logps/rejected": -271.25982666015625, "loss": 5.3443, "margin_dpo/margin_mean": 5.772907257080078, "margin_dpo/margin_std": 15.37360954284668, "step": 85 }, { "KL/chosen_KL_mean": 1.2405624389648438, "KL/mean": -3.23270845413208, "KL/rejected_KL_mean": -7.705974578857422, "KL/std": 13.27261734008789, "epoch": 0.18010471204188483, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.946547508239746, "fcm_dpo/q_t": 0.47779226303100586, "grad_norm": 35.33269500732422, "learning_rate": 4.908790517010636e-07, "logits/chosen": -0.6816504001617432, "logits/rejected": -0.6742711067199707, "logps/chosen": -308.19012451171875, "logps/ref_chosen": -309.4306945800781, "logps/ref_rejected": -290.91278076171875, "logps/rejected": -298.6187744140625, "loss": 5.227, "margin_dpo/margin_mean": 8.94654655456543, "margin_dpo/margin_std": 17.33509063720703, "step": 86 }, { "KL/chosen_KL_mean": -0.1603870391845703, "KL/mean": -5.2997941970825195, "KL/rejected_KL_mean": -10.439201354980469, "KL/std": 15.276622772216797, "epoch": 0.18219895287958116, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.278806686401367, "fcm_dpo/q_t": 0.4746187925338745, "grad_norm": 29.66010856628418, "learning_rate": 4.903825930468148e-07, "logits/chosen": -0.7536052465438843, "logits/rejected": -0.747842013835907, "logps/chosen": -278.1881103515625, "logps/ref_chosen": -278.0277099609375, "logps/ref_rejected": -245.70123291015625, "logps/rejected": -256.1404113769531, "loss": 5.1829, "margin_dpo/margin_mean": 10.27880573272705, "margin_dpo/margin_std": 18.86574363708496, "step": 87 }, { "KL/chosen_KL_mean": -2.069059371948242, "KL/mean": -6.533992290496826, "KL/rejected_KL_mean": -10.998920440673828, "KL/std": 15.059403419494629, "epoch": 0.18429319371727748, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.929862976074219, "fcm_dpo/q_t": 0.4779248833656311, "grad_norm": 28.824947357177734, "learning_rate": 4.898732434036243e-07, "logits/chosen": -0.7699937224388123, "logits/rejected": -0.786287248134613, "logps/chosen": -268.5838623046875, "logps/ref_chosen": -266.5148010253906, "logps/ref_rejected": -265.90081787109375, "logps/rejected": -276.89971923828125, "loss": 5.2342, "margin_dpo/margin_mean": 8.929862976074219, "margin_dpo/margin_std": 19.146970748901367, "step": 88 }, { "KL/chosen_KL_mean": 0.019695281982421875, "KL/mean": -4.6023640632629395, "KL/rejected_KL_mean": -9.224431991577148, "KL/std": 14.222090721130371, "epoch": 0.18638743455497384, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.244122505187988, "fcm_dpo/q_t": 0.4771014451980591, "grad_norm": 30.576059341430664, "learning_rate": 4.893510300863676e-07, "logits/chosen": -0.7487041354179382, "logits/rejected": -0.7396792769432068, "logps/chosen": -265.6695861816406, "logps/ref_chosen": -265.6893005371094, "logps/ref_rejected": -251.49314880371094, "logps/rejected": -260.7175598144531, "loss": 5.2212, "margin_dpo/margin_mean": 9.244121551513672, "margin_dpo/margin_std": 18.273433685302734, "step": 89 }, { "KL/chosen_KL_mean": -1.189279556274414, "KL/mean": -5.295719146728516, "KL/rejected_KL_mean": -9.402151107788086, "KL/std": 14.302242279052734, "epoch": 0.18848167539267016, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.212868690490723, "fcm_dpo/q_t": 0.47964057326316833, "grad_norm": 29.81761360168457, "learning_rate": 4.8881598109976e-07, "logits/chosen": -0.7451858520507812, "logits/rejected": -0.7541492581367493, "logps/chosen": -308.6142883300781, "logps/ref_chosen": -307.4250183105469, "logps/ref_rejected": -265.7172546386719, "logps/rejected": -275.119384765625, "loss": 5.2572, "margin_dpo/margin_mean": 8.212869644165039, "margin_dpo/margin_std": 17.621723175048828, "step": 90 }, { "KL/chosen_KL_mean": -2.184307098388672, "KL/mean": -6.940348148345947, "KL/rejected_KL_mean": -11.696388244628906, "KL/std": 14.84321403503418, "epoch": 0.1905759162303665, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.512090682983398, "fcm_dpo/q_t": 0.4765446186065674, "grad_norm": 32.97541427612305, "learning_rate": 4.882681251368548e-07, "logits/chosen": -0.6837319135665894, "logits/rejected": -0.7010940909385681, "logps/chosen": -237.92529296875, "logps/ref_chosen": -235.74098205566406, "logps/ref_rejected": -226.6428985595703, "logps/rejected": -238.33929443359375, "loss": 5.2161, "margin_dpo/margin_mean": 9.512090682983398, "margin_dpo/margin_std": 19.28335189819336, "step": 91 }, { "KL/chosen_KL_mean": -3.1826534271240234, "KL/mean": -8.19556713104248, "KL/rejected_KL_mean": -13.208467483520508, "KL/std": 17.41613006591797, "epoch": 0.19267015706806281, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.025823593139648, "fcm_dpo/q_t": 0.47521984577178955, "grad_norm": 33.7353401184082, "learning_rate": 4.877074915775048e-07, "logits/chosen": -0.7360749840736389, "logits/rejected": -0.7194338440895081, "logps/chosen": -286.63018798828125, "logps/ref_chosen": -283.4475402832031, "logps/ref_rejected": -273.134033203125, "logps/rejected": -286.3424987792969, "loss": 5.2016, "margin_dpo/margin_mean": 10.025823593139648, "margin_dpo/margin_std": 21.176162719726562, "step": 92 }, { "KL/chosen_KL_mean": -2.424844741821289, "KL/mean": -7.178375720977783, "KL/rejected_KL_mean": -11.93189811706543, "KL/std": 15.435590744018555, "epoch": 0.19476439790575917, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.507054328918457, "fcm_dpo/q_t": 0.4765450060367584, "grad_norm": 28.56822395324707, "learning_rate": 4.871341104867864e-07, "logits/chosen": -0.739149272441864, "logits/rejected": -0.7635706067085266, "logps/chosen": -235.76199340820312, "logps/ref_chosen": -233.33714294433594, "logps/ref_rejected": -230.54273986816406, "logps/rejected": -242.47462463378906, "loss": 5.215, "margin_dpo/margin_mean": 9.507055282592773, "margin_dpo/margin_std": 19.51372528076172, "step": 93 }, { "KL/chosen_KL_mean": -2.452543258666992, "KL/mean": -7.112139701843262, "KL/rejected_KL_mean": -11.771745681762695, "KL/std": 16.46709442138672, "epoch": 0.1968586387434555, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.31920051574707, "fcm_dpo/q_t": 0.4769359230995178, "grad_norm": 31.440210342407227, "learning_rate": 4.865480126133871e-07, "logits/chosen": -0.6917638182640076, "logits/rejected": -0.7137020230293274, "logps/chosen": -297.1053466796875, "logps/ref_chosen": -294.6528015136719, "logps/ref_rejected": -283.657958984375, "logps/rejected": -295.42974853515625, "loss": 5.2323, "margin_dpo/margin_mean": 9.319201469421387, "margin_dpo/margin_std": 21.84139633178711, "step": 94 }, { "KL/chosen_KL_mean": -3.3814926147460938, "KL/mean": -8.43643569946289, "KL/rejected_KL_mean": -13.491373062133789, "KL/std": 18.077224731445312, "epoch": 0.19895287958115182, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.109877586364746, "fcm_dpo/q_t": 0.47511792182922363, "grad_norm": 33.59014892578125, "learning_rate": 4.859492293879573e-07, "logits/chosen": -0.7207019925117493, "logits/rejected": -0.741344153881073, "logps/chosen": -315.05120849609375, "logps/ref_chosen": -311.6697082519531, "logps/ref_rejected": -262.7471923828125, "logps/rejected": -276.23858642578125, "loss": 5.2086, "margin_dpo/margin_mean": 10.109875679016113, "margin_dpo/margin_std": 22.540496826171875, "step": 95 }, { "KL/chosen_KL_mean": -4.737796783447266, "KL/mean": -9.849888801574707, "KL/rejected_KL_mean": -14.961980819702148, "KL/std": 19.32669448852539, "epoch": 0.20104712041884817, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.224173545837402, "fcm_dpo/q_t": 0.4748472571372986, "grad_norm": 37.64308547973633, "learning_rate": 4.853377929214243e-07, "logits/chosen": -0.7003790736198425, "logits/rejected": -0.712866485118866, "logps/chosen": -287.29376220703125, "logps/ref_chosen": -282.55596923828125, "logps/ref_rejected": -242.71588134765625, "logps/rejected": -257.6778564453125, "loss": 5.205, "margin_dpo/margin_mean": 10.224174499511719, "margin_dpo/margin_std": 23.328632354736328, "step": 96 }, { "KL/chosen_KL_mean": -4.413461685180664, "KL/mean": -10.396341323852539, "KL/rejected_KL_mean": -16.37922477722168, "KL/std": 17.304428100585938, "epoch": 0.2031413612565445, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.965749740600586, "fcm_dpo/q_t": 0.47052669525146484, "grad_norm": 32.066131591796875, "learning_rate": 4.847137360032699e-07, "logits/chosen": -0.7437675595283508, "logits/rejected": -0.7308604717254639, "logps/chosen": -307.99127197265625, "logps/ref_chosen": -303.57781982421875, "logps/ref_rejected": -264.22491455078125, "logps/rejected": -280.6041564941406, "loss": 5.1338, "margin_dpo/margin_mean": 11.965749740600586, "margin_dpo/margin_std": 22.346803665161133, "step": 97 }, { "KL/chosen_KL_mean": -4.811254501342773, "KL/mean": -11.00230884552002, "KL/rejected_KL_mean": -17.19336700439453, "KL/std": 18.988019943237305, "epoch": 0.20523560209424083, "fcm_dpo/beta": 0.01010863482952118, "fcm_dpo/delta": 0.024685721844434738, "fcm_dpo/margin": 12.38210678100586, "fcm_dpo/q_t": 0.4695638120174408, "grad_norm": 34.899879455566406, "learning_rate": 4.84077092099773e-07, "logits/chosen": -0.771165132522583, "logits/rejected": -0.7832750082015991, "logps/chosen": -291.6416015625, "logps/ref_chosen": -286.8303527832031, "logps/ref_rejected": -278.08331298828125, "logps/rejected": -295.27667236328125, "loss": 5.1274, "margin_dpo/margin_mean": 12.382107734680176, "margin_dpo/margin_std": 22.810455322265625, "step": 98 }, { "KL/chosen_KL_mean": -5.324771881103516, "KL/mean": -11.61580753326416, "KL/rejected_KL_mean": -17.906835556030273, "KL/std": 19.748458862304688, "epoch": 0.20732984293193718, "fcm_dpo/beta": 0.010278570465743542, "fcm_dpo/delta": 0.01541962381452322, "fcm_dpo/margin": 12.582066535949707, "fcm_dpo/q_t": 0.4683588445186615, "grad_norm": 33.6032600402832, "learning_rate": 4.834278953522137e-07, "logits/chosen": -0.7313593626022339, "logits/rejected": -0.7438006401062012, "logps/chosen": -285.2459716796875, "logps/ref_chosen": -279.92120361328125, "logps/ref_rejected": -250.3365478515625, "logps/rejected": -268.2433776855469, "loss": 5.127, "margin_dpo/margin_mean": 12.582066535949707, "margin_dpo/margin_std": 27.021068572998047, "step": 99 }, { "KL/chosen_KL_mean": -7.78460693359375, "KL/mean": -14.085283279418945, "KL/rejected_KL_mean": -20.385950088500977, "KL/std": 21.98175811767578, "epoch": 0.2094240837696335, "fcm_dpo/beta": 0.010492552071809769, "fcm_dpo/delta": 0.02715744450688362, "fcm_dpo/margin": 12.601343154907227, "fcm_dpo/q_t": 0.4678565263748169, "grad_norm": 37.04026794433594, "learning_rate": 4.827661805750437e-07, "logits/chosen": -0.7743514776229858, "logits/rejected": -0.7868786454200745, "logps/chosen": -304.6123046875, "logps/ref_chosen": -296.8276672363281, "logps/ref_rejected": -275.56146240234375, "logps/rejected": -295.9474182128906, "loss": 5.1047, "margin_dpo/margin_mean": 12.601341247558594, "margin_dpo/margin_std": 24.13147735595703, "step": 100 }, { "KL/chosen_KL_mean": -5.093202590942383, "KL/mean": -12.606334686279297, "KL/rejected_KL_mean": -20.11946678161621, "KL/std": 22.76483154296875, "epoch": 0.21151832460732983, "fcm_dpo/beta": 0.010632730089128017, "fcm_dpo/delta": 0.02228887937963009, "fcm_dpo/margin": 15.02625846862793, "fcm_dpo/q_t": 0.46118059754371643, "grad_norm": 34.257389068603516, "learning_rate": 4.820919832540181e-07, "logits/chosen": -0.7760720252990723, "logits/rejected": -0.7899767160415649, "logps/chosen": -257.8352355957031, "logps/ref_chosen": -252.74203491210938, "logps/ref_rejected": -276.4185485839844, "logps/rejected": -296.53802490234375, "loss": 5.0165, "margin_dpo/margin_mean": 15.026259422302246, "margin_dpo/margin_std": 26.347774505615234, "step": 101 }, { "KL/chosen_KL_mean": -5.112276077270508, "KL/mean": -12.508711814880371, "KL/rejected_KL_mean": -19.905141830444336, "KL/std": 19.70211410522461, "epoch": 0.2136125654450262, "fcm_dpo/beta": 0.010945435613393784, "fcm_dpo/delta": 0.04265664890408516, "fcm_dpo/margin": 14.792869567871094, "fcm_dpo/q_t": 0.4603489637374878, "grad_norm": 34.85258102416992, "learning_rate": 4.814053395442932e-07, "logits/chosen": -0.7522796988487244, "logits/rejected": -0.7484122514724731, "logps/chosen": -224.666015625, "logps/ref_chosen": -219.5537109375, "logps/ref_rejected": -231.90853881835938, "logps/rejected": -251.81369018554688, "loss": 5.0038, "margin_dpo/margin_mean": 14.792869567871094, "margin_dpo/margin_std": 24.94087791442871, "step": 102 }, { "KL/chosen_KL_mean": -4.756736755371094, "KL/mean": -11.560209274291992, "KL/rejected_KL_mean": -18.363676071166992, "KL/std": 21.59006690979004, "epoch": 0.2157068062827225, "fcm_dpo/beta": 0.011328417807817459, "fcm_dpo/delta": 0.033762745559215546, "fcm_dpo/margin": 13.606931686401367, "fcm_dpo/q_t": 0.46292123198509216, "grad_norm": 36.85382843017578, "learning_rate": 4.807062862684873e-07, "logits/chosen": -0.7776579856872559, "logits/rejected": -0.7746756672859192, "logps/chosen": -264.4317932128906, "logps/ref_chosen": -259.6750793457031, "logps/ref_rejected": -278.7400817871094, "logps/rejected": -297.103759765625, "loss": 5.0428, "margin_dpo/margin_mean": 13.606931686401367, "margin_dpo/margin_std": 25.625690460205078, "step": 103 }, { "KL/chosen_KL_mean": -9.137903213500977, "KL/mean": -14.100648880004883, "KL/rejected_KL_mean": -19.06338119506836, "KL/std": 22.391727447509766, "epoch": 0.21780104712041884, "fcm_dpo/beta": 0.011419950984418392, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.92547607421875, "fcm_dpo/q_t": 0.47238224744796753, "grad_norm": 37.83346176147461, "learning_rate": 4.799948609147061e-07, "logits/chosen": -0.7624853253364563, "logits/rejected": -0.7678467631340027, "logps/chosen": -277.11212158203125, "logps/ref_chosen": -267.9741516113281, "logps/ref_rejected": -230.5306396484375, "logps/rejected": -249.59400939941406, "loss": 5.1971, "margin_dpo/margin_mean": 9.92547607421875, "margin_dpo/margin_std": 26.146018981933594, "step": 104 }, { "KL/chosen_KL_mean": -5.228021621704102, "KL/mean": -15.545706748962402, "KL/rejected_KL_mean": -25.8634033203125, "KL/std": 25.212154388427734, "epoch": 0.2198952879581152, "fcm_dpo/beta": 0.011541016399860382, "fcm_dpo/delta": 0.012324552051723003, "fcm_dpo/margin": 20.63536834716797, "fcm_dpo/q_t": 0.4421055316925049, "grad_norm": 39.61387252807617, "learning_rate": 4.792711016345321e-07, "logits/chosen": -0.764461874961853, "logits/rejected": -0.7763689756393433, "logps/chosen": -327.48284912109375, "logps/ref_chosen": -322.25482177734375, "logps/ref_rejected": -279.02978515625, "logps/rejected": -304.8931579589844, "loss": 4.7572, "margin_dpo/margin_mean": 20.63536834716797, "margin_dpo/margin_std": 27.043453216552734, "step": 105 }, { "KL/chosen_KL_mean": -12.485261917114258, "KL/mean": -18.77321434020996, "KL/rejected_KL_mean": -25.061166763305664, "KL/std": 23.474252700805664, "epoch": 0.22198952879581152, "fcm_dpo/beta": 0.011548945680260658, "fcm_dpo/delta": 0.006849272176623344, "fcm_dpo/margin": 12.575901985168457, "fcm_dpo/q_t": 0.46476516127586365, "grad_norm": 44.48746871948242, "learning_rate": 4.785350472409791e-07, "logits/chosen": -0.7404403686523438, "logits/rejected": -0.7760816812515259, "logps/chosen": -308.64306640625, "logps/ref_chosen": -296.15777587890625, "logps/ref_rejected": -266.2691650390625, "logps/rejected": -291.3303527832031, "loss": 5.1058, "margin_dpo/margin_mean": 12.575901985168457, "margin_dpo/margin_std": 29.33730697631836, "step": 106 }, { "KL/chosen_KL_mean": -4.258321762084961, "KL/mean": -14.100729942321777, "KL/rejected_KL_mean": -23.943143844604492, "KL/std": 22.039447784423828, "epoch": 0.22408376963350785, "fcm_dpo/beta": 0.011992116458714008, "fcm_dpo/delta": 0.07569944858551025, "fcm_dpo/margin": 19.6848087310791, "fcm_dpo/q_t": 0.44321852922439575, "grad_norm": 44.562828063964844, "learning_rate": 4.777867372064105e-07, "logits/chosen": -0.7750982642173767, "logits/rejected": -0.7679177522659302, "logps/chosen": -311.254638671875, "logps/ref_chosen": -306.996337890625, "logps/ref_rejected": -296.79412841796875, "logps/rejected": -320.73724365234375, "loss": 4.7705, "margin_dpo/margin_mean": 19.68480682373047, "margin_dpo/margin_std": 27.532594680786133, "step": 107 }, { "KL/chosen_KL_mean": -5.4611663818359375, "KL/mean": -14.556838989257812, "KL/rejected_KL_mean": -23.65252113342285, "KL/std": 25.107391357421875, "epoch": 0.2261780104712042, "fcm_dpo/beta": 0.012616816908121109, "fcm_dpo/delta": 0.041766583919525146, "fcm_dpo/margin": 18.191373825073242, "fcm_dpo/q_t": 0.44564181566238403, "grad_norm": 127.99562072753906, "learning_rate": 4.770262116604223e-07, "logits/chosen": -0.7681426405906677, "logits/rejected": -0.7800797820091248, "logps/chosen": -300.61383056640625, "logps/ref_chosen": -295.1526794433594, "logps/ref_rejected": -235.974853515625, "logps/rejected": -259.62738037109375, "loss": 4.8423, "margin_dpo/margin_mean": 18.19137191772461, "margin_dpo/margin_std": 30.33808708190918, "step": 108 }, { "KL/chosen_KL_mean": -8.192440032958984, "KL/mean": -18.310985565185547, "KL/rejected_KL_mean": -28.429523468017578, "KL/std": 25.256664276123047, "epoch": 0.22827225130890053, "fcm_dpo/beta": 0.013039087876677513, "fcm_dpo/delta": 0.0516541451215744, "fcm_dpo/margin": 20.237075805664062, "fcm_dpo/q_t": 0.4380655884742737, "grad_norm": 48.19257736206055, "learning_rate": 4.7625351138769166e-07, "logits/chosen": -0.7955527901649475, "logits/rejected": -0.7929811477661133, "logps/chosen": -334.11724853515625, "logps/ref_chosen": -325.9248046875, "logps/ref_rejected": -279.15423583984375, "logps/rejected": -307.5837707519531, "loss": 4.7317, "margin_dpo/margin_mean": 20.237077713012695, "margin_dpo/margin_std": 31.672195434570312, "step": 109 }, { "KL/chosen_KL_mean": -8.757614135742188, "KL/mean": -18.272449493408203, "KL/rejected_KL_mean": -27.78728675842285, "KL/std": 27.215492248535156, "epoch": 0.23036649214659685, "fcm_dpo/beta": 0.013565942645072937, "fcm_dpo/delta": 0.046711843460798264, "fcm_dpo/margin": 19.029666900634766, "fcm_dpo/q_t": 0.43923109769821167, "grad_norm": 44.99413299560547, "learning_rate": 4.75468677825789e-07, "logits/chosen": -0.8082597851753235, "logits/rejected": -0.7972973585128784, "logps/chosen": -283.1968078613281, "logps/ref_chosen": -274.439208984375, "logps/ref_rejected": -260.0552062988281, "logps/rejected": -287.8424987792969, "loss": 4.7868, "margin_dpo/margin_mean": 19.029666900634766, "margin_dpo/margin_std": 33.67991256713867, "step": 110 }, { "KL/chosen_KL_mean": -9.568780899047852, "KL/mean": -19.672412872314453, "KL/rejected_KL_mean": -29.776050567626953, "KL/std": 27.85650062561035, "epoch": 0.2324607329842932, "fcm_dpo/beta": 0.013938742689788342, "fcm_dpo/delta": 0.017976418137550354, "fcm_dpo/margin": 20.20726776123047, "fcm_dpo/q_t": 0.4342862367630005, "grad_norm": 50.70798873901367, "learning_rate": 4.7467175306295647e-07, "logits/chosen": -0.8316294550895691, "logits/rejected": -0.8110882639884949, "logps/chosen": -338.804931640625, "logps/ref_chosen": -329.2361755371094, "logps/ref_rejected": -287.82830810546875, "logps/rejected": -317.6043395996094, "loss": 4.7335, "margin_dpo/margin_mean": 20.20726776123047, "margin_dpo/margin_std": 34.256099700927734, "step": 111 }, { "KL/chosen_KL_mean": -15.03680419921875, "KL/mean": -21.586559295654297, "KL/rejected_KL_mean": -28.13631820678711, "KL/std": 26.71904182434082, "epoch": 0.23455497382198953, "fcm_dpo/beta": 0.013969598338007927, "fcm_dpo/delta": 0.00925416685640812, "fcm_dpo/margin": 13.099512100219727, "fcm_dpo/q_t": 0.4575166404247284, "grad_norm": 55.44192123413086, "learning_rate": 4.7386277983585053e-07, "logits/chosen": -0.7383916974067688, "logits/rejected": -0.7683345675468445, "logps/chosen": -272.09619140625, "logps/ref_chosen": -257.0593566894531, "logps/ref_rejected": -272.9595031738281, "logps/rejected": -301.0958557128906, "loss": 5.1261, "margin_dpo/margin_mean": 13.099512100219727, "margin_dpo/margin_std": 35.875091552734375, "step": 112 }, { "KL/chosen_KL_mean": -12.496185302734375, "KL/mean": -24.628555297851562, "KL/rejected_KL_mean": -36.76092529296875, "KL/std": 30.702579498291016, "epoch": 0.23664921465968586, "fcm_dpo/beta": 0.013736647553741932, "fcm_dpo/delta": -0.03855419531464577, "fcm_dpo/margin": 24.26474380493164, "fcm_dpo/q_t": 0.42343804240226746, "grad_norm": 53.458412170410156, "learning_rate": 4.7304180152725024e-07, "logits/chosen": -0.8085811138153076, "logits/rejected": -0.8125609159469604, "logps/chosen": -298.537841796875, "logps/ref_chosen": -286.0416564941406, "logps/ref_rejected": -270.374267578125, "logps/rejected": -307.13519287109375, "loss": 4.6065, "margin_dpo/margin_mean": 24.264739990234375, "margin_dpo/margin_std": 38.84653854370117, "step": 113 }, { "KL/chosen_KL_mean": -16.22774887084961, "KL/mean": -22.99600601196289, "KL/rejected_KL_mean": -29.764259338378906, "KL/std": 27.5384521484375, "epoch": 0.2387434554973822, "fcm_dpo/beta": 0.013897491618990898, "fcm_dpo/delta": 0.028407400473952293, "fcm_dpo/margin": 13.536506652832031, "fcm_dpo/q_t": 0.4553212821483612, "grad_norm": 53.780792236328125, "learning_rate": 4.7220886216373085e-07, "logits/chosen": -0.8557870388031006, "logits/rejected": -0.8575012683868408, "logps/chosen": -276.2362060546875, "logps/ref_chosen": -260.0084533691406, "logps/ref_rejected": -246.67190551757812, "logps/rejected": -276.4361572265625, "loss": 5.0764, "margin_dpo/margin_mean": 13.536508560180664, "margin_dpo/margin_std": 34.91930389404297, "step": 114 }, { "KL/chosen_KL_mean": -17.749019622802734, "KL/mean": -24.61688232421875, "KL/rejected_KL_mean": -31.484745025634766, "KL/std": 30.253429412841797, "epoch": 0.24083769633507854, "fcm_dpo/beta": 0.013822369277477264, "fcm_dpo/delta": -0.01768554374575615, "fcm_dpo/margin": 13.73572826385498, "fcm_dpo/q_t": 0.4572725296020508, "grad_norm": 59.642250061035156, "learning_rate": 4.7136400641330245e-07, "logits/chosen": -0.848541796207428, "logits/rejected": -0.8091092705726624, "logps/chosen": -317.1719970703125, "logps/ref_chosen": -299.4229736328125, "logps/ref_rejected": -272.1186828613281, "logps/rejected": -303.60345458984375, "loss": 5.1056, "margin_dpo/margin_mean": 13.735727310180664, "margin_dpo/margin_std": 38.05146408081055, "step": 115 }, { "KL/chosen_KL_mean": -13.734819412231445, "KL/mean": -22.632715225219727, "KL/rejected_KL_mean": -31.530614852905273, "KL/std": 28.271486282348633, "epoch": 0.24293193717277486, "fcm_dpo/beta": 0.014031772501766682, "fcm_dpo/delta": 0.0374310202896595, "fcm_dpo/margin": 17.79579734802246, "fcm_dpo/q_t": 0.4412615895271301, "grad_norm": 51.95737075805664, "learning_rate": 4.70507279583015e-07, "logits/chosen": -0.8612798452377319, "logits/rejected": -0.8237401843070984, "logps/chosen": -292.9987487792969, "logps/ref_chosen": -279.263916015625, "logps/ref_rejected": -253.6192169189453, "logps/rejected": -285.14984130859375, "loss": 4.845, "margin_dpo/margin_mean": 17.795799255371094, "margin_dpo/margin_std": 34.34822463989258, "step": 116 }, { "KL/chosen_KL_mean": -16.676069259643555, "KL/mean": -26.847412109375, "KL/rejected_KL_mean": -37.01875305175781, "KL/std": 26.84279441833496, "epoch": 0.2450261780104712, "fcm_dpo/beta": 0.014229537919163704, "fcm_dpo/delta": -0.02480526641011238, "fcm_dpo/margin": 20.34267807006836, "fcm_dpo/q_t": 0.43198737502098083, "grad_norm": 57.02861785888672, "learning_rate": 4.6963872761652834e-07, "logits/chosen": -0.8200348615646362, "logits/rejected": -0.8224045038223267, "logps/chosen": -275.90093994140625, "logps/ref_chosen": -259.2248840332031, "logps/ref_rejected": -229.3042755126953, "logps/rejected": -266.32305908203125, "loss": 4.7122, "margin_dpo/margin_mean": 20.34267807006836, "margin_dpo/margin_std": 31.05514907836914, "step": 117 }, { "KL/chosen_KL_mean": -22.062801361083984, "KL/mean": -33.46045684814453, "KL/rejected_KL_mean": -44.85810852050781, "KL/std": 33.74718475341797, "epoch": 0.24712041884816754, "fcm_dpo/beta": 0.01404194999486208, "fcm_dpo/delta": -0.0025224993005394936, "fcm_dpo/margin": 22.795305252075195, "fcm_dpo/q_t": 0.42747047543525696, "grad_norm": 59.17610168457031, "learning_rate": 4.687583970916486e-07, "logits/chosen": -0.8000219464302063, "logits/rejected": -0.78817218542099, "logps/chosen": -289.1335144042969, "logps/ref_chosen": -267.0707092285156, "logps/ref_rejected": -272.7322082519531, "logps/rejected": -317.59027099609375, "loss": 4.7095, "margin_dpo/margin_mean": 22.795305252075195, "margin_dpo/margin_std": 41.762020111083984, "step": 118 }, { "KL/chosen_KL_mean": -26.935039520263672, "KL/mean": -35.30348587036133, "KL/rejected_KL_mean": -43.671932220458984, "KL/std": 33.14229965209961, "epoch": 0.24921465968586387, "fcm_dpo/beta": 0.014066150411963463, "fcm_dpo/delta": 0.007536234334111214, "fcm_dpo/margin": 16.736879348754883, "fcm_dpo/q_t": 0.44677790999412537, "grad_norm": 61.08521270751953, "learning_rate": 4.6786633521783005e-07, "logits/chosen": -0.8812325596809387, "logits/rejected": -0.8813977241516113, "logps/chosen": -351.6116943359375, "logps/ref_chosen": -324.6766357421875, "logps/ref_rejected": -306.0322265625, "logps/rejected": -349.70416259765625, "loss": 5.0037, "margin_dpo/margin_mean": 16.73687744140625, "margin_dpo/margin_std": 40.269229888916016, "step": 119 }, { "KL/chosen_KL_mean": -23.385488510131836, "KL/mean": -32.74627685546875, "KL/rejected_KL_mean": -42.10707092285156, "KL/std": 31.856447219848633, "epoch": 0.2513089005235602, "fcm_dpo/beta": 0.013914698734879494, "fcm_dpo/delta": -0.018124153837561607, "fcm_dpo/margin": 18.72157859802246, "fcm_dpo/q_t": 0.44076991081237793, "grad_norm": 54.6278076171875, "learning_rate": 4.669625898336438e-07, "logits/chosen": -0.8311907052993774, "logits/rejected": -0.8439901471138, "logps/chosen": -338.6472473144531, "logps/ref_chosen": -315.2617492675781, "logps/ref_rejected": -265.32501220703125, "logps/rejected": -307.4320983886719, "loss": 4.9043, "margin_dpo/margin_mean": 18.72157859802246, "margin_dpo/margin_std": 39.45494842529297, "step": 120 }, { "KL/chosen_KL_mean": -27.836078643798828, "KL/mean": -34.881961822509766, "KL/rejected_KL_mean": -41.9278564453125, "KL/std": 30.415508270263672, "epoch": 0.2534031413612565, "fcm_dpo/beta": 0.014034891501069069, "fcm_dpo/delta": 0.04258127138018608, "fcm_dpo/margin": 14.091768264770508, "fcm_dpo/q_t": 0.4556926190853119, "grad_norm": 70.71554565429688, "learning_rate": 4.6604720940421207e-07, "logits/chosen": -0.8554950952529907, "logits/rejected": -0.8707772493362427, "logps/chosen": -250.83218383789062, "logps/ref_chosen": -222.99609375, "logps/ref_rejected": -226.92860412597656, "logps/rejected": -268.8564453125, "loss": 5.0876, "margin_dpo/margin_mean": 14.091768264770508, "margin_dpo/margin_std": 38.64990234375, "step": 121 }, { "KL/chosen_KL_mean": -28.256216049194336, "KL/mean": -37.67997741699219, "KL/rejected_KL_mean": -47.10374069213867, "KL/std": 35.624656677246094, "epoch": 0.2554973821989529, "fcm_dpo/beta": 0.014204684644937515, "fcm_dpo/delta": -0.0006600477499887347, "fcm_dpo/margin": 18.847537994384766, "fcm_dpo/q_t": 0.4392344355583191, "grad_norm": 65.4048080444336, "learning_rate": 4.651202430186092e-07, "logits/chosen": -0.9069045186042786, "logits/rejected": -0.8689112663269043, "logps/chosen": -304.28253173828125, "logps/ref_chosen": -276.02630615234375, "logps/ref_rejected": -277.97418212890625, "logps/rejected": -325.0779113769531, "loss": 4.9306, "margin_dpo/margin_mean": 18.847537994384766, "margin_dpo/margin_std": 43.85592269897461, "step": 122 }, { "KL/chosen_KL_mean": -21.264556884765625, "KL/mean": -33.50984191894531, "KL/rejected_KL_mean": -45.75513458251953, "KL/std": 34.91896438598633, "epoch": 0.25759162303664923, "fcm_dpo/beta": 0.014176392927765846, "fcm_dpo/delta": -0.01918771117925644, "fcm_dpo/margin": 24.49056625366211, "fcm_dpo/q_t": 0.41995978355407715, "grad_norm": 63.334815979003906, "learning_rate": 4.6418174038722924e-07, "logits/chosen": -0.8300783634185791, "logits/rejected": -0.8240232467651367, "logps/chosen": -349.419189453125, "logps/ref_chosen": -328.1546325683594, "logps/ref_rejected": -280.6911315917969, "logps/rejected": -326.4462890625, "loss": 4.6042, "margin_dpo/margin_mean": 24.490564346313477, "margin_dpo/margin_std": 40.045387268066406, "step": 123 }, { "KL/chosen_KL_mean": -22.935548782348633, "KL/mean": -33.28853988647461, "KL/rejected_KL_mean": -43.64153289794922, "KL/std": 33.49969482421875, "epoch": 0.25968586387434556, "fcm_dpo/beta": 0.013861719518899918, "fcm_dpo/delta": -0.010028579272329807, "fcm_dpo/margin": 20.705991744995117, "fcm_dpo/q_t": 0.4327910542488098, "grad_norm": 66.91712951660156, "learning_rate": 4.6323175183912023e-07, "logits/chosen": -0.8569639325141907, "logits/rejected": -0.82343590259552, "logps/chosen": -298.6317443847656, "logps/ref_chosen": -275.6961975097656, "logps/ref_rejected": -225.361572265625, "logps/rejected": -269.0030822753906, "loss": 4.7679, "margin_dpo/margin_mean": 20.705991744995117, "margin_dpo/margin_std": 37.01435089111328, "step": 124 }, { "KL/chosen_KL_mean": -24.424352645874023, "KL/mean": -34.008094787597656, "KL/rejected_KL_mean": -43.59183883666992, "KL/std": 34.53946304321289, "epoch": 0.2617801047120419, "fcm_dpo/beta": 0.013881498947739601, "fcm_dpo/delta": -0.015007440000772476, "fcm_dpo/margin": 19.167491912841797, "fcm_dpo/q_t": 0.4410821795463562, "grad_norm": 67.82720184326172, "learning_rate": 4.6227032831928483e-07, "logits/chosen": -0.8077659606933594, "logits/rejected": -0.7630800008773804, "logps/chosen": -302.4941101074219, "logps/ref_chosen": -278.06976318359375, "logps/ref_rejected": -265.63873291015625, "logps/rejected": -309.2305908203125, "loss": 4.9802, "margin_dpo/margin_mean": 19.16749382019043, "margin_dpo/margin_std": 47.179725646972656, "step": 125 }, { "KL/chosen_KL_mean": -20.58001136779785, "KL/mean": -31.229753494262695, "KL/rejected_KL_mean": -41.87950134277344, "KL/std": 35.75967788696289, "epoch": 0.2638743455497382, "fcm_dpo/beta": 0.013768978416919708, "fcm_dpo/delta": 0.018436769023537636, "fcm_dpo/margin": 21.299489974975586, "fcm_dpo/q_t": 0.4325592815876007, "grad_norm": 67.8341064453125, "learning_rate": 4.612975213859487e-07, "logits/chosen": -0.8235605359077454, "logits/rejected": -0.8356186151504517, "logps/chosen": -341.9761047363281, "logps/ref_chosen": -321.3960876464844, "logps/ref_rejected": -285.37664794921875, "logps/rejected": -327.256103515625, "loss": 4.7876, "margin_dpo/margin_mean": 21.299489974975586, "margin_dpo/margin_std": 41.721378326416016, "step": 126 }, { "KL/chosen_KL_mean": -18.004310607910156, "KL/mean": -30.651229858398438, "KL/rejected_KL_mean": -43.29817199707031, "KL/std": 36.886497497558594, "epoch": 0.26596858638743454, "fcm_dpo/beta": 0.01383438054472208, "fcm_dpo/delta": -0.06396237015724182, "fcm_dpo/margin": 25.293867111206055, "fcm_dpo/q_t": 0.42087969183921814, "grad_norm": 68.39196014404297, "learning_rate": 4.603133832077953e-07, "logits/chosen": -0.8953632116317749, "logits/rejected": -0.845830500125885, "logps/chosen": -324.5630798339844, "logps/ref_chosen": -306.55877685546875, "logps/ref_rejected": -274.8651428222656, "logps/rejected": -318.163330078125, "loss": 4.6431, "margin_dpo/margin_mean": 25.293869018554688, "margin_dpo/margin_std": 41.969058990478516, "step": 127 }, { "KL/chosen_KL_mean": -12.162698745727539, "KL/mean": -26.459373474121094, "KL/rejected_KL_mean": -40.75605010986328, "KL/std": 38.046356201171875, "epoch": 0.2680628272251309, "fcm_dpo/beta": 0.01290260162204504, "fcm_dpo/delta": -0.04873104393482208, "fcm_dpo/margin": 28.59334945678711, "fcm_dpo/q_t": 0.4156576991081238, "grad_norm": 50.67338180541992, "learning_rate": 4.5931796656116837e-07, "logits/chosen": -0.7757830023765564, "logits/rejected": -0.7743943929672241, "logps/chosen": -277.56005859375, "logps/ref_chosen": -265.3973693847656, "logps/ref_rejected": -250.9737548828125, "logps/rejected": -291.72979736328125, "loss": 4.5472, "margin_dpo/margin_mean": 28.59334945678711, "margin_dpo/margin_std": 45.63541030883789, "step": 128 }, { "KL/chosen_KL_mean": -13.00929069519043, "KL/mean": -25.526004791259766, "KL/rejected_KL_mean": -38.042720794677734, "KL/std": 35.623844146728516, "epoch": 0.27015706806282724, "fcm_dpo/beta": 0.012600157409906387, "fcm_dpo/delta": -0.035920120775699615, "fcm_dpo/margin": 25.033435821533203, "fcm_dpo/q_t": 0.42803478240966797, "grad_norm": 64.80278778076172, "learning_rate": 4.5831132482724193e-07, "logits/chosen": -0.8124359846115112, "logits/rejected": -0.8133723139762878, "logps/chosen": -316.167724609375, "logps/ref_chosen": -303.158447265625, "logps/ref_rejected": -275.9891052246094, "logps/rejected": -314.0318298339844, "loss": 4.7028, "margin_dpo/margin_mean": 25.03343391418457, "margin_dpo/margin_std": 44.670989990234375, "step": 129 }, { "KL/chosen_KL_mean": -14.777093887329102, "KL/mean": -26.76191520690918, "KL/rejected_KL_mean": -38.746734619140625, "KL/std": 34.12180709838867, "epoch": 0.27225130890052357, "fcm_dpo/beta": 0.01246493961662054, "fcm_dpo/delta": 0.026410242542624474, "fcm_dpo/margin": 23.96963119506836, "fcm_dpo/q_t": 0.4313068389892578, "grad_norm": 67.26573181152344, "learning_rate": 4.5729351198915705e-07, "logits/chosen": -0.7755675911903381, "logits/rejected": -0.8148837685585022, "logps/chosen": -301.1844177246094, "logps/ref_chosen": -286.4073486328125, "logps/ref_rejected": -294.38665771484375, "logps/rejected": -333.1334228515625, "loss": 4.7318, "margin_dpo/margin_mean": 23.96963119506836, "margin_dpo/margin_std": 43.78425216674805, "step": 130 }, { "KL/chosen_KL_mean": -17.329355239868164, "KL/mean": -27.278770446777344, "KL/rejected_KL_mean": -37.22817611694336, "KL/std": 37.274391174316406, "epoch": 0.2743455497382199, "fcm_dpo/beta": 0.01295158639550209, "fcm_dpo/delta": 0.03556347265839577, "fcm_dpo/margin": 19.898832321166992, "fcm_dpo/q_t": 0.44100892543792725, "grad_norm": 71.38201904296875, "learning_rate": 4.5626458262912735e-07, "logits/chosen": -0.8666002750396729, "logits/rejected": -0.8158248662948608, "logps/chosen": -328.8944091796875, "logps/ref_chosen": -311.5650634765625, "logps/ref_rejected": -291.62432861328125, "logps/rejected": -328.8525390625, "loss": 4.9313, "margin_dpo/margin_mean": 19.898832321166992, "margin_dpo/margin_std": 44.99689483642578, "step": 131 }, { "KL/chosen_KL_mean": -13.132219314575195, "KL/mean": -27.211585998535156, "KL/rejected_KL_mean": -41.290950775146484, "KL/std": 37.97029113769531, "epoch": 0.2764397905759162, "fcm_dpo/beta": 0.013177872635424137, "fcm_dpo/delta": -0.022556345909833908, "fcm_dpo/margin": 28.158733367919922, "fcm_dpo/q_t": 0.4168829023838043, "grad_norm": 86.56053924560547, "learning_rate": 4.5522459192551166e-07, "logits/chosen": -0.8170298933982849, "logits/rejected": -0.8009600639343262, "logps/chosen": -283.2140197753906, "logps/ref_chosen": -270.0818176269531, "logps/ref_rejected": -284.3084411621094, "logps/rejected": -325.5993957519531, "loss": 4.6122, "margin_dpo/margin_mean": 28.158733367919922, "margin_dpo/margin_std": 48.239803314208984, "step": 132 }, { "KL/chosen_KL_mean": -14.192306518554688, "KL/mean": -27.241323471069336, "KL/rejected_KL_mean": -40.290340423583984, "KL/std": 36.0876350402832, "epoch": 0.27853403141361255, "fcm_dpo/beta": 0.012713887728750706, "fcm_dpo/delta": -0.01462561171501875, "fcm_dpo/margin": 26.098039627075195, "fcm_dpo/q_t": 0.4238502085208893, "grad_norm": 55.55501174926758, "learning_rate": 4.541735956498554e-07, "logits/chosen": -0.8563445806503296, "logits/rejected": -0.8587122559547424, "logps/chosen": -299.81365966796875, "logps/ref_chosen": -285.6213684082031, "logps/ref_rejected": -251.19386291503906, "logps/rejected": -291.48419189453125, "loss": 4.6574, "margin_dpo/margin_mean": 26.098039627075195, "margin_dpo/margin_std": 44.765777587890625, "step": 133 }, { "KL/chosen_KL_mean": -25.40647315979004, "KL/mean": -35.70955276489258, "KL/rejected_KL_mean": -46.01262664794922, "KL/std": 37.336177825927734, "epoch": 0.2806282722513089, "fcm_dpo/beta": 0.01256817951798439, "fcm_dpo/delta": -0.014759279787540436, "fcm_dpo/margin": 20.606157302856445, "fcm_dpo/q_t": 0.43928804993629456, "grad_norm": 53.031646728515625, "learning_rate": 4.5311165016389914e-07, "logits/chosen": -0.8518386483192444, "logits/rejected": -0.8516722917556763, "logps/chosen": -344.3273010253906, "logps/ref_chosen": -318.92083740234375, "logps/ref_rejected": -293.1894836425781, "logps/rejected": -339.20208740234375, "loss": 4.8628, "margin_dpo/margin_mean": 20.606155395507812, "margin_dpo/margin_std": 41.08130645751953, "step": 134 }, { "KL/chosen_KL_mean": -21.542409896850586, "KL/mean": -33.6240234375, "KL/rejected_KL_mean": -45.70563888549805, "KL/std": 38.229713439941406, "epoch": 0.28272251308900526, "fcm_dpo/beta": 0.012417946010828018, "fcm_dpo/delta": -0.01869252882897854, "fcm_dpo/margin": 24.16322898864746, "fcm_dpo/q_t": 0.4295629858970642, "grad_norm": 71.39109802246094, "learning_rate": 4.520388124165564e-07, "logits/chosen": -0.7575780153274536, "logits/rejected": -0.7987253665924072, "logps/chosen": -314.3641662597656, "logps/ref_chosen": -292.8217468261719, "logps/ref_rejected": -269.2896728515625, "logps/rejected": -314.995361328125, "loss": 4.7495, "margin_dpo/margin_mean": 24.16322898864746, "margin_dpo/margin_std": 42.811241149902344, "step": 135 }, { "KL/chosen_KL_mean": -23.46628189086914, "KL/mean": -37.211727142333984, "KL/rejected_KL_mean": -50.957183837890625, "KL/std": 39.38726043701172, "epoch": 0.2848167539267016, "fcm_dpo/beta": 0.012112447991967201, "fcm_dpo/delta": -0.032658714801073074, "fcm_dpo/margin": 27.49091339111328, "fcm_dpo/q_t": 0.4244113862514496, "grad_norm": 68.2113265991211, "learning_rate": 4.5095513994085974e-07, "logits/chosen": -0.8045265078544617, "logits/rejected": -0.7990159392356873, "logps/chosen": -296.3188171386719, "logps/ref_chosen": -272.8525390625, "logps/ref_rejected": -252.68202209472656, "logps/rejected": -303.63922119140625, "loss": 4.6882, "margin_dpo/margin_mean": 27.49091339111328, "margin_dpo/margin_std": 49.20502853393555, "step": 136 }, { "KL/chosen_KL_mean": -27.184465408325195, "KL/mean": -38.95940017700195, "KL/rejected_KL_mean": -50.734336853027344, "KL/std": 41.108158111572266, "epoch": 0.2869109947643979, "fcm_dpo/beta": 0.011971182189881802, "fcm_dpo/delta": -0.009735875762999058, "fcm_dpo/margin": 23.549869537353516, "fcm_dpo/q_t": 0.43623051047325134, "grad_norm": 58.9945068359375, "learning_rate": 4.498606908508753e-07, "logits/chosen": -0.865675687789917, "logits/rejected": -0.8515555262565613, "logps/chosen": -327.9366760253906, "logps/ref_chosen": -300.7522277832031, "logps/ref_rejected": -286.1935119628906, "logps/rejected": -336.9278869628906, "loss": 4.816, "margin_dpo/margin_mean": 23.54987144470215, "margin_dpo/margin_std": 47.051055908203125, "step": 137 }, { "KL/chosen_KL_mean": -24.572195053100586, "KL/mean": -37.86249923706055, "KL/rejected_KL_mean": -51.15279006958008, "KL/std": 41.30282211303711, "epoch": 0.28900523560209423, "fcm_dpo/beta": 0.011851192452013493, "fcm_dpo/delta": 0.0014142822474241257, "fcm_dpo/margin": 26.580608367919922, "fcm_dpo/q_t": 0.43020325899124146, "grad_norm": 57.485626220703125, "learning_rate": 4.487555238385862e-07, "logits/chosen": -0.770022988319397, "logits/rejected": -0.7489472031593323, "logps/chosen": -313.5091552734375, "logps/ref_chosen": -288.9369812011719, "logps/ref_rejected": -263.7076416015625, "logps/rejected": -314.8604431152344, "loss": 4.7718, "margin_dpo/margin_mean": 26.580608367919922, "margin_dpo/margin_std": 53.411861419677734, "step": 138 }, { "KL/chosen_KL_mean": -23.968965530395508, "KL/mean": -33.10724639892578, "KL/rejected_KL_mean": -42.24552917480469, "KL/std": 37.81224060058594, "epoch": 0.29109947643979056, "fcm_dpo/beta": 0.011845817789435387, "fcm_dpo/delta": 0.011396631598472595, "fcm_dpo/margin": 18.276565551757812, "fcm_dpo/q_t": 0.45067816972732544, "grad_norm": 62.92908477783203, "learning_rate": 4.476396981707453e-07, "logits/chosen": -0.8066970109939575, "logits/rejected": -0.8366684913635254, "logps/chosen": -294.0132751464844, "logps/ref_chosen": -270.0443115234375, "logps/ref_rejected": -267.3226013183594, "logps/rejected": -309.568115234375, "loss": 5.0408, "margin_dpo/margin_mean": 18.276565551757812, "margin_dpo/margin_std": 45.98987579345703, "step": 139 }, { "KL/chosen_KL_mean": -23.92278289794922, "KL/mean": -38.3636360168457, "KL/rejected_KL_mean": -52.80449676513672, "KL/std": 41.12156295776367, "epoch": 0.2931937172774869, "fcm_dpo/beta": 0.011858167126774788, "fcm_dpo/delta": -0.036420539021492004, "fcm_dpo/margin": 28.881723403930664, "fcm_dpo/q_t": 0.4207428991794586, "grad_norm": 62.07208251953125, "learning_rate": 4.4651327368569684e-07, "logits/chosen": -0.892544150352478, "logits/rejected": -0.8596428632736206, "logps/chosen": -306.87835693359375, "logps/ref_chosen": -282.9555969238281, "logps/ref_rejected": -251.17181396484375, "logps/rejected": -303.976318359375, "loss": 4.5731, "margin_dpo/margin_mean": 28.881723403930664, "margin_dpo/margin_std": 43.74916458129883, "step": 140 }, { "KL/chosen_KL_mean": -23.3916072845459, "KL/mean": -38.7611198425293, "KL/rejected_KL_mean": -54.1306266784668, "KL/std": 40.98480224609375, "epoch": 0.29528795811518327, "fcm_dpo/beta": 0.01144687831401825, "fcm_dpo/delta": -0.031413860619068146, "fcm_dpo/margin": 30.739025115966797, "fcm_dpo/q_t": 0.4180885851383209, "grad_norm": 58.84656524658203, "learning_rate": 4.453763107901675e-07, "logits/chosen": -0.8045722842216492, "logits/rejected": -0.7984085083007812, "logps/chosen": -319.6917724609375, "logps/ref_chosen": -296.3001708984375, "logps/ref_rejected": -279.8486633300781, "logps/rejected": -333.979248046875, "loss": 4.6001, "margin_dpo/margin_mean": 30.739025115966797, "margin_dpo/margin_std": 50.728179931640625, "step": 141 }, { "KL/chosen_KL_mean": -25.941381454467773, "KL/mean": -38.507537841796875, "KL/rejected_KL_mean": -51.07369613647461, "KL/std": 42.384246826171875, "epoch": 0.2973821989528796, "fcm_dpo/beta": 0.011349150910973549, "fcm_dpo/delta": -0.005858212243765593, "fcm_dpo/margin": 25.132293701171875, "fcm_dpo/q_t": 0.4345892667770386, "grad_norm": 58.35767364501953, "learning_rate": 4.4422887045602674e-07, "logits/chosen": -0.8434002995491028, "logits/rejected": -0.8453127145767212, "logps/chosen": -326.50726318359375, "logps/ref_chosen": -300.56585693359375, "logps/ref_rejected": -231.43316650390625, "logps/rejected": -282.5068359375, "loss": 4.8534, "margin_dpo/margin_mean": 25.132295608520508, "margin_dpo/margin_std": 53.17611312866211, "step": 142 }, { "KL/chosen_KL_mean": -27.10928726196289, "KL/mean": -42.68886947631836, "KL/rejected_KL_mean": -58.268463134765625, "KL/std": 41.513275146484375, "epoch": 0.2994764397905759, "fcm_dpo/beta": 0.010968414135277271, "fcm_dpo/delta": -0.057835567742586136, "fcm_dpo/margin": 31.15917205810547, "fcm_dpo/q_t": 0.42122355103492737, "grad_norm": 57.15718078613281, "learning_rate": 4.4307101421701755e-07, "logits/chosen": -0.8285028338432312, "logits/rejected": -0.8069489002227783, "logps/chosen": -323.84161376953125, "logps/ref_chosen": -296.73236083984375, "logps/ref_rejected": -266.45257568359375, "logps/rejected": -324.7210693359375, "loss": 4.5791, "margin_dpo/margin_mean": 31.15917205810547, "margin_dpo/margin_std": 47.604496002197266, "step": 143 }, { "KL/chosen_KL_mean": -29.586715698242188, "KL/mean": -42.25658416748047, "KL/rejected_KL_mean": -54.92646026611328, "KL/std": 44.707191467285156, "epoch": 0.30157068062827225, "fcm_dpo/beta": 0.010684658773243427, "fcm_dpo/delta": -0.006853851955384016, "fcm_dpo/margin": 25.339746475219727, "fcm_dpo/q_t": 0.4385736882686615, "grad_norm": 52.378929138183594, "learning_rate": 4.419028041654559e-07, "logits/chosen": -0.8826330304145813, "logits/rejected": -0.8678580522537231, "logps/chosen": -328.43072509765625, "logps/ref_chosen": -298.843994140625, "logps/ref_rejected": -266.120849609375, "logps/rejected": -321.04730224609375, "loss": 4.8628, "margin_dpo/margin_mean": 25.33974838256836, "margin_dpo/margin_std": 52.99594497680664, "step": 144 }, { "KL/chosen_KL_mean": -25.344947814941406, "KL/mean": -42.59326934814453, "KL/rejected_KL_mean": -59.84159469604492, "KL/std": 41.871673583984375, "epoch": 0.3036649214659686, "fcm_dpo/beta": 0.01033711526542902, "fcm_dpo/delta": -0.07404731959104538, "fcm_dpo/margin": 34.49665069580078, "fcm_dpo/q_t": 0.41699421405792236, "grad_norm": 55.13352584838867, "learning_rate": 4.4072430294890166e-07, "logits/chosen": -0.8775890469551086, "logits/rejected": -0.8793016076087952, "logps/chosen": -301.0978088378906, "logps/ref_chosen": -275.7528381347656, "logps/ref_rejected": -214.74807739257812, "logps/rejected": -274.58966064453125, "loss": 4.5283, "margin_dpo/margin_mean": 34.49665069580078, "margin_dpo/margin_std": 49.740142822265625, "step": 145 }, { "KL/chosen_KL_mean": -35.13841247558594, "KL/mean": -51.969417572021484, "KL/rejected_KL_mean": -68.8004150390625, "KL/std": 45.30426788330078, "epoch": 0.3057591623036649, "fcm_dpo/beta": 0.010088625364005566, "fcm_dpo/delta": -0.013300293125212193, "fcm_dpo/margin": 33.662017822265625, "fcm_dpo/q_t": 0.4215752184391022, "grad_norm": 58.701690673828125, "learning_rate": 4.395355737667985e-07, "logits/chosen": -0.8522456288337708, "logits/rejected": -0.8450497984886169, "logps/chosen": -312.23663330078125, "logps/ref_chosen": -277.09820556640625, "logps/ref_rejected": -265.41046142578125, "logps/rejected": -334.2108459472656, "loss": 4.5732, "margin_dpo/margin_mean": 33.66202163696289, "margin_dpo/margin_std": 50.35722351074219, "step": 146 }, { "KL/chosen_KL_mean": -37.00726318359375, "KL/mean": -49.70895767211914, "KL/rejected_KL_mean": -62.41065216064453, "KL/std": 45.74989318847656, "epoch": 0.3078534031413613, "fcm_dpo/beta": 0.010042143985629082, "fcm_dpo/delta": -0.03261498734354973, "fcm_dpo/margin": 25.403392791748047, "fcm_dpo/q_t": 0.44056451320648193, "grad_norm": 73.16529846191406, "learning_rate": 4.3833668036708483e-07, "logits/chosen": -0.8551448583602905, "logits/rejected": -0.8509636521339417, "logps/chosen": -328.42584228515625, "logps/ref_chosen": -291.4185791015625, "logps/ref_rejected": -253.43051147460938, "logps/rejected": -315.84112548828125, "loss": 4.9002, "margin_dpo/margin_mean": 25.403392791748047, "margin_dpo/margin_std": 51.63163375854492, "step": 147 }, { "KL/chosen_KL_mean": -39.83953857421875, "KL/mean": -53.04331588745117, "KL/rejected_KL_mean": -66.24710083007812, "KL/std": 47.25356674194336, "epoch": 0.3099476439790576, "fcm_dpo/beta": 0.009962659329175949, "fcm_dpo/delta": 0.04581095650792122, "fcm_dpo/margin": 26.407564163208008, "fcm_dpo/q_t": 0.4414011240005493, "grad_norm": 59.49394226074219, "learning_rate": 4.3712768704277524e-07, "logits/chosen": -0.9048643112182617, "logits/rejected": -0.9068287014961243, "logps/chosen": -276.5880432128906, "logps/ref_chosen": -236.74850463867188, "logps/ref_rejected": -231.4674072265625, "logps/rejected": -297.7144775390625, "loss": 4.9041, "margin_dpo/margin_mean": 26.407562255859375, "margin_dpo/margin_std": 59.63642120361328, "step": 148 }, { "KL/chosen_KL_mean": -41.80915069580078, "KL/mean": -59.93357849121094, "KL/rejected_KL_mean": -78.05801391601562, "KL/std": 44.09037399291992, "epoch": 0.31204188481675393, "fcm_dpo/beta": 0.010042773559689522, "fcm_dpo/delta": -0.01487391721457243, "fcm_dpo/margin": 36.248878479003906, "fcm_dpo/q_t": 0.4150589406490326, "grad_norm": 70.69794464111328, "learning_rate": 4.3590865862851263e-07, "logits/chosen": -0.8720038533210754, "logits/rejected": -0.8556748628616333, "logps/chosen": -361.7376403808594, "logps/ref_chosen": -319.9284973144531, "logps/ref_rejected": -308.20233154296875, "logps/rejected": -386.2603454589844, "loss": 4.4518, "margin_dpo/margin_mean": 36.248878479003906, "margin_dpo/margin_std": 48.31082534790039, "step": 149 }, { "KL/chosen_KL_mean": -50.01300048828125, "KL/mean": -63.49955749511719, "KL/rejected_KL_mean": -76.98611450195312, "KL/std": 46.77826690673828, "epoch": 0.31413612565445026, "fcm_dpo/beta": 0.010052911005914211, "fcm_dpo/delta": -0.0005218717269599438, "fcm_dpo/margin": 26.973114013671875, "fcm_dpo/q_t": 0.4368109703063965, "grad_norm": 59.045284271240234, "learning_rate": 4.346796604970912e-07, "logits/chosen": -0.8413239121437073, "logits/rejected": -0.8248977661132812, "logps/chosen": -326.33123779296875, "logps/ref_chosen": -276.3182373046875, "logps/ref_rejected": -273.02215576171875, "logps/rejected": -350.00823974609375, "loss": 4.8136, "margin_dpo/margin_mean": 26.97311019897461, "margin_dpo/margin_std": 52.10138702392578, "step": 150 }, { "KL/chosen_KL_mean": -46.77452087402344, "KL/mean": -66.79248809814453, "KL/rejected_KL_mean": -86.81045532226562, "KL/std": 46.96579360961914, "epoch": 0.3162303664921466, "fcm_dpo/beta": 0.009873086586594582, "fcm_dpo/delta": -0.04695701599121094, "fcm_dpo/margin": 40.035926818847656, "fcm_dpo/q_t": 0.40897297859191895, "grad_norm": 81.46723175048828, "learning_rate": 4.3344075855595097e-07, "logits/chosen": -0.8678506016731262, "logits/rejected": -0.8601349592208862, "logps/chosen": -344.0873718261719, "logps/ref_chosen": -297.31280517578125, "logps/ref_rejected": -266.1003723144531, "logps/rejected": -352.91082763671875, "loss": 4.4272, "margin_dpo/margin_mean": 40.03593063354492, "margin_dpo/margin_std": 55.97598648071289, "step": 151 }, { "KL/chosen_KL_mean": -37.11475372314453, "KL/mean": -55.36869812011719, "KL/rejected_KL_mean": -73.62263488769531, "KL/std": 49.92714309692383, "epoch": 0.3183246073298429, "fcm_dpo/beta": 0.009343523532152176, "fcm_dpo/delta": -0.04294053837656975, "fcm_dpo/margin": 36.50787353515625, "fcm_dpo/q_t": 0.4220908284187317, "grad_norm": 58.823143005371094, "learning_rate": 4.3219201924364323e-07, "logits/chosen": -0.880072832107544, "logits/rejected": -0.8768435716629028, "logps/chosen": -307.36175537109375, "logps/ref_chosen": -270.2470397949219, "logps/ref_rejected": -269.7749328613281, "logps/rejected": -343.3975830078125, "loss": 4.6093, "margin_dpo/margin_mean": 36.50787353515625, "margin_dpo/margin_std": 58.238258361816406, "step": 152 }, { "KL/chosen_KL_mean": -42.54793167114258, "KL/mean": -67.99817657470703, "KL/rejected_KL_mean": -93.44841003417969, "KL/std": 53.13435363769531, "epoch": 0.3204188481675393, "fcm_dpo/beta": 0.008867593482136726, "fcm_dpo/delta": -0.1064264178276062, "fcm_dpo/margin": 50.900489807128906, "fcm_dpo/q_t": 0.3951815664768219, "grad_norm": 61.95179748535156, "learning_rate": 4.309335095262675e-07, "logits/chosen": -0.8620288372039795, "logits/rejected": -0.8489632606506348, "logps/chosen": -316.3269958496094, "logps/ref_chosen": -273.779052734375, "logps/ref_rejected": -280.9530944824219, "logps/rejected": -374.4014892578125, "loss": 4.1803, "margin_dpo/margin_mean": 50.90049362182617, "margin_dpo/margin_std": 55.47189712524414, "step": 153 }, { "KL/chosen_KL_mean": -41.75373458862305, "KL/mean": -61.51596450805664, "KL/rejected_KL_mean": -81.27819061279297, "KL/std": 55.93893051147461, "epoch": 0.3225130890052356, "fcm_dpo/beta": 0.008387413807213306, "fcm_dpo/delta": -0.043754659593105316, "fcm_dpo/margin": 39.52445983886719, "fcm_dpo/q_t": 0.4235866665840149, "grad_norm": 64.17202758789062, "learning_rate": 4.2966529689388064e-07, "logits/chosen": -0.9106939435005188, "logits/rejected": -0.8913959860801697, "logps/chosen": -331.6569519042969, "logps/ref_chosen": -289.9031982421875, "logps/ref_rejected": -261.5166320800781, "logps/rejected": -342.7948303222656, "loss": 4.6569, "margin_dpo/margin_mean": 39.52445983886719, "margin_dpo/margin_std": 66.74655151367188, "step": 154 }, { "KL/chosen_KL_mean": -45.44877624511719, "KL/mean": -62.710758209228516, "KL/rejected_KL_mean": -79.9727554321289, "KL/std": 53.82670974731445, "epoch": 0.32460732984293195, "fcm_dpo/beta": 0.008284756913781166, "fcm_dpo/delta": 0.005428697448223829, "fcm_dpo/margin": 34.52397918701172, "fcm_dpo/q_t": 0.43525511026382446, "grad_norm": 67.82777404785156, "learning_rate": 4.2838744935687716e-07, "logits/chosen": -0.8425617814064026, "logits/rejected": -0.8398025035858154, "logps/chosen": -331.3099670410156, "logps/ref_chosen": -285.8612060546875, "logps/ref_rejected": -300.1272888183594, "logps/rejected": -380.10003662109375, "loss": 4.7548, "margin_dpo/margin_mean": 34.52397918701172, "margin_dpo/margin_std": 62.337467193603516, "step": 155 }, { "KL/chosen_KL_mean": -40.039371490478516, "KL/mean": -63.92396926879883, "KL/rejected_KL_mean": -87.80857849121094, "KL/std": 56.946449279785156, "epoch": 0.3267015706806283, "fcm_dpo/beta": 0.008112832903862, "fcm_dpo/delta": -0.08456975221633911, "fcm_dpo/margin": 47.769203186035156, "fcm_dpo/q_t": 0.41121193766593933, "grad_norm": 50.074623107910156, "learning_rate": 4.271000354423425e-07, "logits/chosen": -0.8586286902427673, "logits/rejected": -0.8539766073226929, "logps/chosen": -319.0748596191406, "logps/ref_chosen": -279.0354919433594, "logps/ref_rejected": -244.2198486328125, "logps/rejected": -332.0284423828125, "loss": 4.4928, "margin_dpo/margin_mean": 47.769203186035156, "margin_dpo/margin_std": 71.89561462402344, "step": 156 }, { "KL/chosen_KL_mean": -43.30219268798828, "KL/mean": -62.22343826293945, "KL/rejected_KL_mean": -81.14466857910156, "KL/std": 56.71215057373047, "epoch": 0.3287958115183246, "fcm_dpo/beta": 0.007705520838499069, "fcm_dpo/delta": -0.007637109607458115, "fcm_dpo/margin": 37.84247589111328, "fcm_dpo/q_t": 0.4317454695701599, "grad_norm": 59.33927917480469, "learning_rate": 4.258031241903777e-07, "logits/chosen": -0.9295624494552612, "logits/rejected": -0.9276785254478455, "logps/chosen": -314.13250732421875, "logps/ref_chosen": -270.830322265625, "logps/ref_rejected": -259.08319091796875, "logps/rejected": -340.2278747558594, "loss": 4.7054, "margin_dpo/margin_mean": 37.84247589111328, "margin_dpo/margin_std": 61.984832763671875, "step": 157 }, { "KL/chosen_KL_mean": -42.21181869506836, "KL/mean": -62.8458137512207, "KL/rejected_KL_mean": -83.47979736328125, "KL/std": 57.90792465209961, "epoch": 0.3308900523560209, "fcm_dpo/beta": 0.007645631209015846, "fcm_dpo/delta": -0.030163168907165527, "fcm_dpo/margin": 41.267974853515625, "fcm_dpo/q_t": 0.4256693720817566, "grad_norm": 70.06487274169922, "learning_rate": 4.2449678515039743e-07, "logits/chosen": -0.8575935959815979, "logits/rejected": -0.8421310186386108, "logps/chosen": -332.1781311035156, "logps/ref_chosen": -289.9663391113281, "logps/ref_rejected": -271.335693359375, "logps/rejected": -354.8154602050781, "loss": 4.6531, "margin_dpo/margin_mean": 41.267974853515625, "margin_dpo/margin_std": 65.3279037475586, "step": 158 }, { "KL/chosen_KL_mean": -51.21124267578125, "KL/mean": -63.622154235839844, "KL/rejected_KL_mean": -76.03305053710938, "KL/std": 54.81487274169922, "epoch": 0.33298429319371725, "fcm_dpo/beta": 0.007572945207357407, "fcm_dpo/delta": 0.028481464833021164, "fcm_dpo/margin": 24.821823120117188, "fcm_dpo/q_t": 0.45764467120170593, "grad_norm": 76.31413269042969, "learning_rate": 4.2318108837739986e-07, "logits/chosen": -0.9422204494476318, "logits/rejected": -0.8938316702842712, "logps/chosen": -372.589599609375, "logps/ref_chosen": -321.37835693359375, "logps/ref_rejected": -250.45652770996094, "logps/rejected": -326.4895935058594, "loss": 5.1345, "margin_dpo/margin_mean": 24.821823120117188, "margin_dpo/margin_std": 70.63139343261719, "step": 159 }, { "KL/chosen_KL_mean": -41.84352111816406, "KL/mean": -66.21701049804688, "KL/rejected_KL_mean": -90.59048461914062, "KL/std": 57.292686462402344, "epoch": 0.33507853403141363, "fcm_dpo/beta": 0.007637364789843559, "fcm_dpo/delta": -0.02358035184442997, "fcm_dpo/margin": 48.74696350097656, "fcm_dpo/q_t": 0.4126546382904053, "grad_norm": 54.5738525390625, "learning_rate": 4.218561044282098e-07, "logits/chosen": -0.8767799139022827, "logits/rejected": -0.8782601952552795, "logps/chosen": -318.12701416015625, "logps/ref_chosen": -276.28350830078125, "logps/ref_rejected": -262.7477722167969, "logps/rejected": -353.3382568359375, "loss": 4.3985, "margin_dpo/margin_mean": 48.74696350097656, "margin_dpo/margin_std": 58.39624786376953, "step": 160 }, { "KL/chosen_KL_mean": -50.77786636352539, "KL/mean": -72.51949310302734, "KL/rejected_KL_mean": -94.26113891601562, "KL/std": 60.983299255371094, "epoch": 0.33717277486910996, "fcm_dpo/beta": 0.007549113593995571, "fcm_dpo/delta": -0.019491776823997498, "fcm_dpo/margin": 43.4832649230957, "fcm_dpo/q_t": 0.4252697825431824, "grad_norm": 69.1987075805664, "learning_rate": 4.2052190435769554e-07, "logits/chosen": -0.8953877687454224, "logits/rejected": -0.8785583972930908, "logps/chosen": -361.2706604003906, "logps/ref_chosen": -310.4927978515625, "logps/ref_rejected": -250.25347900390625, "logps/rejected": -344.5146179199219, "loss": 4.6394, "margin_dpo/margin_mean": 43.48326873779297, "margin_dpo/margin_std": 71.65884399414062, "step": 161 }, { "KL/chosen_KL_mean": -47.96656036376953, "KL/mean": -67.32000732421875, "KL/rejected_KL_mean": -86.6734619140625, "KL/std": 56.53418731689453, "epoch": 0.3392670157068063, "fcm_dpo/beta": 0.007303684484213591, "fcm_dpo/delta": -0.009029125794768333, "fcm_dpo/margin": 38.7069091796875, "fcm_dpo/q_t": 0.43392330408096313, "grad_norm": 63.3481559753418, "learning_rate": 4.1917855971495763e-07, "logits/chosen": -0.8692272901535034, "logits/rejected": -0.8613343238830566, "logps/chosen": -344.0770263671875, "logps/ref_chosen": -296.1105041503906, "logps/ref_rejected": -253.4247589111328, "logps/rejected": -340.0982360839844, "loss": 4.7335, "margin_dpo/margin_mean": 38.7069091796875, "margin_dpo/margin_std": 64.84658813476562, "step": 162 }, { "KL/chosen_KL_mean": -51.703914642333984, "KL/mean": -73.81686401367188, "KL/rejected_KL_mean": -95.9298095703125, "KL/std": 55.789180755615234, "epoch": 0.3413612565445026, "fcm_dpo/beta": 0.007537513505667448, "fcm_dpo/delta": 0.01612706668674946, "fcm_dpo/margin": 44.22589111328125, "fcm_dpo/q_t": 0.42153996229171753, "grad_norm": 68.57307434082031, "learning_rate": 4.1782614253949255e-07, "logits/chosen": -0.8940120339393616, "logits/rejected": -0.8899393081665039, "logps/chosen": -345.2038269042969, "logps/ref_chosen": -293.4999084472656, "logps/ref_rejected": -266.7116394042969, "logps/rejected": -362.6415100097656, "loss": 4.5561, "margin_dpo/margin_mean": 44.22589111328125, "margin_dpo/margin_std": 61.894046783447266, "step": 163 }, { "KL/chosen_KL_mean": -60.08200454711914, "KL/mean": -80.8095703125, "KL/rejected_KL_mean": -101.5371322631836, "KL/std": 61.46062469482422, "epoch": 0.34345549738219894, "fcm_dpo/beta": 0.0075241196900606155, "fcm_dpo/delta": -0.012014145031571388, "fcm_dpo/margin": 41.455135345458984, "fcm_dpo/q_t": 0.42829614877700806, "grad_norm": 60.644813537597656, "learning_rate": 4.164647253573289e-07, "logits/chosen": -0.8615102171897888, "logits/rejected": -0.8748703598976135, "logps/chosen": -327.13153076171875, "logps/ref_chosen": -267.04949951171875, "logps/ref_rejected": -215.9768829345703, "logps/rejected": -317.5140075683594, "loss": 4.6851, "margin_dpo/margin_mean": 41.45513153076172, "margin_dpo/margin_std": 70.61214447021484, "step": 164 }, { "KL/chosen_KL_mean": -49.37308120727539, "KL/mean": -63.531410217285156, "KL/rejected_KL_mean": -77.68974304199219, "KL/std": 52.045166015625, "epoch": 0.34554973821989526, "fcm_dpo/beta": 0.0073788780719041824, "fcm_dpo/delta": -0.004405612125992775, "fcm_dpo/margin": 28.3166561126709, "fcm_dpo/q_t": 0.4508272409439087, "grad_norm": 80.6587142944336, "learning_rate": 4.1509438117713863e-07, "logits/chosen": -0.8801563382148743, "logits/rejected": -0.8534449934959412, "logps/chosen": -327.4346008300781, "logps/ref_chosen": -278.06146240234375, "logps/ref_rejected": -260.4288635253906, "logps/rejected": -338.11865234375, "loss": 4.9606, "margin_dpo/margin_mean": 28.31665802001953, "margin_dpo/margin_std": 59.50096893310547, "step": 165 }, { "KL/chosen_KL_mean": -45.839622497558594, "KL/mean": -64.79864501953125, "KL/rejected_KL_mean": -83.75764465332031, "KL/std": 55.16622543334961, "epoch": 0.34764397905759165, "fcm_dpo/beta": 0.007531080860644579, "fcm_dpo/delta": 0.06673787534236908, "fcm_dpo/margin": 37.91801071166992, "fcm_dpo/q_t": 0.43460068106651306, "grad_norm": 61.27188491821289, "learning_rate": 4.137151834863213e-07, "logits/chosen": -0.8442600965499878, "logits/rejected": -0.809601366519928, "logps/chosen": -321.7886962890625, "logps/ref_chosen": -275.9490661621094, "logps/ref_rejected": -232.13473510742188, "logps/rejected": -315.8923645019531, "loss": 4.7461, "margin_dpo/margin_mean": 37.91801452636719, "margin_dpo/margin_std": 67.47180938720703, "step": 166 }, { "KL/chosen_KL_mean": -56.37845993041992, "KL/mean": -77.35397338867188, "KL/rejected_KL_mean": -98.32949829101562, "KL/std": 60.27275848388672, "epoch": 0.34973821989528797, "fcm_dpo/beta": 0.007845787331461906, "fcm_dpo/delta": 0.00037165917456150055, "fcm_dpo/margin": 41.95103073120117, "fcm_dpo/q_t": 0.4235421419143677, "grad_norm": 73.83167266845703, "learning_rate": 4.123272062470633e-07, "logits/chosen": -0.8352253437042236, "logits/rejected": -0.8208370208740234, "logps/chosen": -336.92987060546875, "logps/ref_chosen": -280.5514221191406, "logps/ref_rejected": -255.2896728515625, "logps/rejected": -353.6191711425781, "loss": 4.6424, "margin_dpo/margin_mean": 41.95103073120117, "margin_dpo/margin_std": 69.98577117919922, "step": 167 }, { "KL/chosen_KL_mean": -55.98714828491211, "KL/mean": -79.81024169921875, "KL/rejected_KL_mean": -103.6333236694336, "KL/std": 63.85597610473633, "epoch": 0.3518324607329843, "fcm_dpo/beta": 0.0075763678178191185, "fcm_dpo/delta": -0.04661968722939491, "fcm_dpo/margin": 47.646175384521484, "fcm_dpo/q_t": 0.4152388274669647, "grad_norm": 115.36837005615234, "learning_rate": 4.1093052389237174e-07, "logits/chosen": -0.8275383710861206, "logits/rejected": -0.7986543774604797, "logps/chosen": -371.7853698730469, "logps/ref_chosen": -315.7982177734375, "logps/ref_rejected": -291.48406982421875, "logps/rejected": -395.11737060546875, "loss": 4.5223, "margin_dpo/margin_mean": 47.64617156982422, "margin_dpo/margin_std": 70.27813720703125, "step": 168 }, { "KL/chosen_KL_mean": -55.89307403564453, "KL/mean": -84.45292663574219, "KL/rejected_KL_mean": -113.01277160644531, "KL/std": 64.26335144042969, "epoch": 0.3539267015706806, "fcm_dpo/beta": 0.0072573344223201275, "fcm_dpo/delta": -0.09110445529222488, "fcm_dpo/margin": 57.11970520019531, "fcm_dpo/q_t": 0.4041180908679962, "grad_norm": 88.24478912353516, "learning_rate": 4.0952521132208267e-07, "logits/chosen": -0.8238295912742615, "logits/rejected": -0.8293823599815369, "logps/chosen": -316.95733642578125, "logps/ref_chosen": -261.06427001953125, "logps/ref_rejected": -235.40663146972656, "logps/rejected": -348.41943359375, "loss": 4.3353, "margin_dpo/margin_mean": 57.11970520019531, "margin_dpo/margin_std": 70.19021606445312, "step": 169 }, { "KL/chosen_KL_mean": -56.9837760925293, "KL/mean": -80.9883804321289, "KL/rejected_KL_mean": -104.99298095703125, "KL/std": 67.8764877319336, "epoch": 0.35602094240837695, "fcm_dpo/beta": 0.006932388059794903, "fcm_dpo/delta": -0.0026629664935171604, "fcm_dpo/margin": 48.00920104980469, "fcm_dpo/q_t": 0.4246680736541748, "grad_norm": 63.6226806640625, "learning_rate": 4.081113438988443e-07, "logits/chosen": -0.8085747957229614, "logits/rejected": -0.8082205653190613, "logps/chosen": -365.95098876953125, "logps/ref_chosen": -308.96722412109375, "logps/ref_rejected": -263.8466796875, "logps/rejected": -368.83966064453125, "loss": 4.6691, "margin_dpo/margin_mean": 48.00919723510742, "margin_dpo/margin_std": 86.9151840209961, "step": 170 }, { "KL/chosen_KL_mean": -52.376243591308594, "KL/mean": -80.6495132446289, "KL/rejected_KL_mean": -108.92279052734375, "KL/std": 65.0744857788086, "epoch": 0.3581151832460733, "fcm_dpo/beta": 0.006713580340147018, "fcm_dpo/delta": -0.06937012076377869, "fcm_dpo/margin": 56.54655456542969, "fcm_dpo/q_t": 0.41104498505592346, "grad_norm": 81.89509582519531, "learning_rate": 4.0668899744407567e-07, "logits/chosen": -0.8374527096748352, "logits/rejected": -0.8505634069442749, "logps/chosen": -311.2652587890625, "logps/ref_chosen": -258.8890380859375, "logps/ref_rejected": -262.19140625, "logps/rejected": -371.11419677734375, "loss": 4.3971, "margin_dpo/margin_mean": 56.54655456542969, "margin_dpo/margin_std": 69.68020629882812, "step": 171 }, { "KL/chosen_KL_mean": -56.283409118652344, "KL/mean": -72.89511108398438, "KL/rejected_KL_mean": -89.50682067871094, "KL/std": 59.91847610473633, "epoch": 0.36020942408376966, "fcm_dpo/beta": 0.006686648353934288, "fcm_dpo/delta": 0.03564419597387314, "fcm_dpo/margin": 33.223411560058594, "fcm_dpo/q_t": 0.447513610124588, "grad_norm": 90.56462097167969, "learning_rate": 4.0525824823390043e-07, "logits/chosen": -0.8602209687232971, "logits/rejected": -0.8738598227500916, "logps/chosen": -395.3057556152344, "logps/ref_chosen": -339.0223388671875, "logps/ref_rejected": -295.78759765625, "logps/rejected": -385.29443359375, "loss": 4.955, "margin_dpo/margin_mean": 33.22341537475586, "margin_dpo/margin_std": 74.95690155029297, "step": 172 }, { "KL/chosen_KL_mean": -58.3000602722168, "KL/mean": -80.27344512939453, "KL/rejected_KL_mean": -102.24684143066406, "KL/std": 63.42533874511719, "epoch": 0.362303664921466, "fcm_dpo/beta": 0.0067140692844986916, "fcm_dpo/delta": -0.014882495626807213, "fcm_dpo/margin": 43.94677734375, "fcm_dpo/q_t": 0.43222668766975403, "grad_norm": 49.26839828491211, "learning_rate": 4.0381917299505686e-07, "logits/chosen": -0.88132643699646, "logits/rejected": -0.8753910660743713, "logps/chosen": -358.4115295410156, "logps/ref_chosen": -300.1114501953125, "logps/ref_rejected": -273.78460693359375, "logps/rejected": -376.03143310546875, "loss": 4.7171, "margin_dpo/margin_mean": 43.946781158447266, "margin_dpo/margin_std": 74.02472686767578, "step": 173 }, { "KL/chosen_KL_mean": -59.025245666503906, "KL/mean": -84.17713165283203, "KL/rejected_KL_mean": -109.32901000976562, "KL/std": 63.81253433227539, "epoch": 0.3643979057591623, "fcm_dpo/beta": 0.006741848308593035, "fcm_dpo/delta": 0.010899264365434647, "fcm_dpo/margin": 50.303775787353516, "fcm_dpo/q_t": 0.419879287481308, "grad_norm": 70.78109741210938, "learning_rate": 4.0237184890078243e-07, "logits/chosen": -0.8532913327217102, "logits/rejected": -0.8315014839172363, "logps/chosen": -394.0791015625, "logps/ref_chosen": -335.0538635253906, "logps/ref_rejected": -257.4646911621094, "logps/rejected": -366.7936706542969, "loss": 4.5267, "margin_dpo/margin_mean": 50.303775787353516, "margin_dpo/margin_std": 70.04698944091797, "step": 174 }, { "KL/chosen_KL_mean": -71.4371109008789, "KL/mean": -94.12638854980469, "KL/rejected_KL_mean": -116.815673828125, "KL/std": 69.9000473022461, "epoch": 0.36649214659685864, "fcm_dpo/beta": 0.006678528618067503, "fcm_dpo/delta": -0.048241063952445984, "fcm_dpo/margin": 45.37855529785156, "fcm_dpo/q_t": 0.432115375995636, "grad_norm": 63.40766143798828, "learning_rate": 4.00916353566676e-07, "logits/chosen": -0.8552145957946777, "logits/rejected": -0.8552535772323608, "logps/chosen": -355.8326721191406, "logps/ref_chosen": -284.39556884765625, "logps/ref_rejected": -283.3876647949219, "logps/rejected": -400.20330810546875, "loss": 4.7497, "margin_dpo/margin_mean": 45.37855529785156, "margin_dpo/margin_std": 81.3629379272461, "step": 175 }, { "KL/chosen_KL_mean": -65.49942016601562, "KL/mean": -83.41252136230469, "KL/rejected_KL_mean": -101.32563781738281, "KL/std": 64.79817199707031, "epoch": 0.36858638743455496, "fcm_dpo/beta": 0.006564862094819546, "fcm_dpo/delta": 0.004861448425799608, "fcm_dpo/margin": 35.82622528076172, "fcm_dpo/q_t": 0.4460398554801941, "grad_norm": 69.52070617675781, "learning_rate": 3.994527650465352e-07, "logits/chosen": -0.8273112773895264, "logits/rejected": -0.832463264465332, "logps/chosen": -317.31219482421875, "logps/ref_chosen": -251.81280517578125, "logps/ref_rejected": -242.05328369140625, "logps/rejected": -343.3789367675781, "loss": 4.9677, "margin_dpo/margin_mean": 35.82622528076172, "margin_dpo/margin_std": 83.5301513671875, "step": 176 }, { "KL/chosen_KL_mean": -72.5023193359375, "KL/mean": -90.53446960449219, "KL/rejected_KL_mean": -108.56661224365234, "KL/std": 65.00486755371094, "epoch": 0.3706806282722513, "fcm_dpo/beta": 0.006526026874780655, "fcm_dpo/delta": 0.008697102777659893, "fcm_dpo/margin": 36.06428909301758, "fcm_dpo/q_t": 0.44549474120140076, "grad_norm": 63.3103141784668, "learning_rate": 3.979811618281705e-07, "logits/chosen": -0.9081556797027588, "logits/rejected": -0.8767774105072021, "logps/chosen": -371.14874267578125, "logps/ref_chosen": -298.6463928222656, "logps/ref_rejected": -295.66534423828125, "logps/rejected": -404.2319641113281, "loss": 4.9694, "margin_dpo/margin_mean": 36.06428909301758, "margin_dpo/margin_std": 79.99176025390625, "step": 177 }, { "KL/chosen_KL_mean": -60.73016357421875, "KL/mean": -88.53191375732422, "KL/rejected_KL_mean": -116.33367156982422, "KL/std": 69.4561538696289, "epoch": 0.37277486910994767, "fcm_dpo/beta": 0.006494695320725441, "fcm_dpo/delta": -0.011969889514148235, "fcm_dpo/margin": 55.60350799560547, "fcm_dpo/q_t": 0.41702789068222046, "grad_norm": 54.077415466308594, "learning_rate": 3.9650162282919654e-07, "logits/chosen": -0.8326258659362793, "logits/rejected": -0.8247045874595642, "logps/chosen": -346.98779296875, "logps/ref_chosen": -286.2576599121094, "logps/ref_rejected": -243.97491455078125, "logps/rejected": -360.3085632324219, "loss": 4.4967, "margin_dpo/margin_mean": 55.60350799560547, "margin_dpo/margin_std": 79.79682922363281, "step": 178 }, { "KL/chosen_KL_mean": -70.25981903076172, "KL/mean": -92.2850341796875, "KL/rejected_KL_mean": -114.31026458740234, "KL/std": 69.08260345458984, "epoch": 0.374869109947644, "fcm_dpo/beta": 0.006461943034082651, "fcm_dpo/delta": -0.009359869174659252, "fcm_dpo/margin": 44.050445556640625, "fcm_dpo/q_t": 0.4336419403553009, "grad_norm": 72.00945281982422, "learning_rate": 3.9501422739279953e-07, "logits/chosen": -0.8239375352859497, "logits/rejected": -0.8243504166603088, "logps/chosen": -329.99688720703125, "logps/ref_chosen": -259.737060546875, "logps/ref_rejected": -277.8813171386719, "logps/rejected": -392.19158935546875, "loss": 4.7656, "margin_dpo/margin_mean": 44.05044174194336, "margin_dpo/margin_std": 78.93538665771484, "step": 179 }, { "KL/chosen_KL_mean": -71.84423828125, "KL/mean": -98.30107879638672, "KL/rejected_KL_mean": -124.75789642333984, "KL/std": 66.42559814453125, "epoch": 0.3769633507853403, "fcm_dpo/beta": 0.006444687955081463, "fcm_dpo/delta": -0.013784291222691536, "fcm_dpo/margin": 52.91366958618164, "fcm_dpo/q_t": 0.42224639654159546, "grad_norm": 78.1937026977539, "learning_rate": 3.935190552834828e-07, "logits/chosen": -0.8528253436088562, "logits/rejected": -0.8767141699790955, "logps/chosen": -339.15313720703125, "logps/ref_chosen": -267.30889892578125, "logps/ref_rejected": -230.4376983642578, "logps/rejected": -355.19561767578125, "loss": 4.5563, "margin_dpo/margin_mean": 52.91366958618164, "margin_dpo/margin_std": 78.59088134765625, "step": 180 }, { "KL/chosen_KL_mean": -83.44524383544922, "KL/mean": -106.03001403808594, "KL/rejected_KL_mean": -128.61477661132812, "KL/std": 69.1161880493164, "epoch": 0.37905759162303665, "fcm_dpo/beta": 0.006374704651534557, "fcm_dpo/delta": 0.011338986456394196, "fcm_dpo/margin": 45.169532775878906, "fcm_dpo/q_t": 0.4334385395050049, "grad_norm": 76.543212890625, "learning_rate": 3.920161866827889e-07, "logits/chosen": -0.8567452430725098, "logits/rejected": -0.8574219346046448, "logps/chosen": -383.9366149902344, "logps/ref_chosen": -300.49139404296875, "logps/ref_rejected": -278.98284912109375, "logps/rejected": -407.5976257324219, "loss": 4.7631, "margin_dpo/margin_mean": 45.169532775878906, "margin_dpo/margin_std": 83.6402587890625, "step": 181 }, { "KL/chosen_KL_mean": -84.8715591430664, "KL/mean": -116.56951904296875, "KL/rejected_KL_mean": -148.26747131347656, "KL/std": 69.70707702636719, "epoch": 0.381151832460733, "fcm_dpo/beta": 0.006300204433500767, "fcm_dpo/delta": -0.08057114481925964, "fcm_dpo/margin": 63.39593505859375, "fcm_dpo/q_t": 0.40720659494400024, "grad_norm": 86.77213287353516, "learning_rate": 3.90505702185e-07, "logits/chosen": -0.8535815477371216, "logits/rejected": -0.877009391784668, "logps/chosen": -364.3696594238281, "logps/ref_chosen": -279.4981689453125, "logps/ref_rejected": -263.6926574707031, "logps/rejected": -411.9601135253906, "loss": 4.3766, "margin_dpo/margin_mean": 63.39593505859375, "margin_dpo/margin_std": 80.48890686035156, "step": 182 }, { "KL/chosen_KL_mean": -83.44680786132812, "KL/mean": -110.89114379882812, "KL/rejected_KL_mean": -138.33546447753906, "KL/std": 72.1974105834961, "epoch": 0.3832460732984293, "fcm_dpo/beta": 0.0060380203649401665, "fcm_dpo/delta": 0.019243724644184113, "fcm_dpo/margin": 54.8886604309082, "fcm_dpo/q_t": 0.4239034652709961, "grad_norm": 63.45649719238281, "learning_rate": 3.889876827928156e-07, "logits/chosen": -0.8888998031616211, "logits/rejected": -0.8892589807510376, "logps/chosen": -354.6525573730469, "logps/ref_chosen": -271.2057189941406, "logps/ref_rejected": -243.91549682617188, "logps/rejected": -382.2509460449219, "loss": 4.6048, "margin_dpo/margin_mean": 54.8886604309082, "margin_dpo/margin_std": 87.88034057617188, "step": 183 }, { "KL/chosen_KL_mean": -82.51263427734375, "KL/mean": -119.50811767578125, "KL/rejected_KL_mean": -156.5036163330078, "KL/std": 79.93283081054688, "epoch": 0.38534031413612563, "fcm_dpo/beta": 0.005751221906393766, "fcm_dpo/delta": -0.10398561507463455, "fcm_dpo/margin": 73.99097442626953, "fcm_dpo/q_t": 0.400656133890152, "grad_norm": 59.62614822387695, "learning_rate": 3.874622099130087e-07, "logits/chosen": -0.9280270934104919, "logits/rejected": -0.9025266766548157, "logps/chosen": -400.95843505859375, "logps/ref_chosen": -318.4457702636719, "logps/ref_rejected": -266.640869140625, "logps/rejected": -423.14447021484375, "loss": 4.3096, "margin_dpo/margin_mean": 73.990966796875, "margin_dpo/margin_std": 93.23393249511719, "step": 184 }, { "KL/chosen_KL_mean": -96.60318756103516, "KL/mean": -123.63733673095703, "KL/rejected_KL_mean": -150.67147827148438, "KL/std": 84.32405090332031, "epoch": 0.387434554973822, "fcm_dpo/beta": 0.005654371343553066, "fcm_dpo/delta": 0.013005169108510017, "fcm_dpo/margin": 54.06830978393555, "fcm_dpo/q_t": 0.42959728837013245, "grad_norm": 81.64273834228516, "learning_rate": 3.859293653520604e-07, "logits/chosen": -0.8964764475822449, "logits/rejected": -0.8889636397361755, "logps/chosen": -370.9120178222656, "logps/ref_chosen": -274.308837890625, "logps/ref_rejected": -260.7274169921875, "logps/rejected": -411.3988952636719, "loss": 4.6766, "margin_dpo/margin_mean": 54.06830978393555, "margin_dpo/margin_std": 89.609130859375, "step": 185 }, { "KL/chosen_KL_mean": -102.949951171875, "KL/mean": -129.26104736328125, "KL/rejected_KL_mean": -155.57212829589844, "KL/std": 75.34392547607422, "epoch": 0.38952879581151834, "fcm_dpo/beta": 0.0056923748925328255, "fcm_dpo/delta": -0.009608768858015537, "fcm_dpo/margin": 52.622169494628906, "fcm_dpo/q_t": 0.42984485626220703, "grad_norm": 63.738101959228516, "learning_rate": 3.8438923131177237e-07, "logits/chosen": -0.9191085696220398, "logits/rejected": -0.9174607396125793, "logps/chosen": -401.955322265625, "logps/ref_chosen": -299.00537109375, "logps/ref_rejected": -274.4014587402344, "logps/rejected": -429.97357177734375, "loss": 4.6882, "margin_dpo/margin_mean": 52.622169494628906, "margin_dpo/margin_std": 85.38917541503906, "step": 186 }, { "KL/chosen_KL_mean": -112.53751373291016, "KL/mean": -135.30740356445312, "KL/rejected_KL_mean": -158.07728576660156, "KL/std": 80.25025939941406, "epoch": 0.39162303664921466, "fcm_dpo/beta": 0.005760915577411652, "fcm_dpo/delta": 0.046012088656425476, "fcm_dpo/margin": 45.53976821899414, "fcm_dpo/q_t": 0.4372592568397522, "grad_norm": 98.26136016845703, "learning_rate": 3.828418903848593e-07, "logits/chosen": -0.8597753047943115, "logits/rejected": -0.8412461876869202, "logps/chosen": -442.3628234863281, "logps/ref_chosen": -329.8253173828125, "logps/ref_rejected": -263.73175048828125, "logps/rejected": -421.8090515136719, "loss": 4.8902, "margin_dpo/margin_mean": 45.53976821899414, "margin_dpo/margin_std": 97.40721893310547, "step": 187 }, { "KL/chosen_KL_mean": -101.80508422851562, "KL/mean": -128.22146606445312, "KL/rejected_KL_mean": -154.6378173828125, "KL/std": 82.52928924560547, "epoch": 0.393717277486911, "fcm_dpo/beta": 0.005808525253087282, "fcm_dpo/delta": -0.002299257554113865, "fcm_dpo/margin": 52.83273696899414, "fcm_dpo/q_t": 0.43033504486083984, "grad_norm": 82.98176574707031, "learning_rate": 3.812874255505191e-07, "logits/chosen": -0.8709114789962769, "logits/rejected": -0.8592625260353088, "logps/chosen": -364.8106689453125, "logps/ref_chosen": -263.005615234375, "logps/ref_rejected": -247.08668518066406, "logps/rejected": -401.7244873046875, "loss": 4.7836, "margin_dpo/margin_mean": 52.83273696899414, "margin_dpo/margin_std": 103.43292236328125, "step": 188 }, { "KL/chosen_KL_mean": -88.54216003417969, "KL/mean": -119.99163818359375, "KL/rejected_KL_mean": -151.44114685058594, "KL/std": 81.04593658447266, "epoch": 0.3958115183246073, "fcm_dpo/beta": 0.005683474242687225, "fcm_dpo/delta": -0.05930778384208679, "fcm_dpo/margin": 62.898963928222656, "fcm_dpo/q_t": 0.41748058795928955, "grad_norm": 80.47416687011719, "learning_rate": 3.797259201699833e-07, "logits/chosen": -0.8913860321044922, "logits/rejected": -0.8976064920425415, "logps/chosen": -361.5025634765625, "logps/ref_chosen": -272.96038818359375, "logps/ref_rejected": -275.13238525390625, "logps/rejected": -426.5735168457031, "loss": 4.503, "margin_dpo/margin_mean": 62.898963928222656, "margin_dpo/margin_std": 89.14469909667969, "step": 189 }, { "KL/chosen_KL_mean": -90.09049987792969, "KL/mean": -119.01117706298828, "KL/rejected_KL_mean": -147.93182373046875, "KL/std": 78.23188781738281, "epoch": 0.39790575916230364, "fcm_dpo/beta": 0.005622372031211853, "fcm_dpo/delta": 0.02536490187048912, "fcm_dpo/margin": 57.84133529663086, "fcm_dpo/q_t": 0.4240185022354126, "grad_norm": 57.77339172363281, "learning_rate": 3.781574579820464e-07, "logits/chosen": -0.884880542755127, "logits/rejected": -0.8418174386024475, "logps/chosen": -347.8880615234375, "logps/ref_chosen": -257.79754638671875, "logps/ref_rejected": -225.2164306640625, "logps/rejected": -373.14825439453125, "loss": 4.5946, "margin_dpo/margin_mean": 57.84133529663086, "margin_dpo/margin_std": 86.56686401367188, "step": 190 }, { "KL/chosen_KL_mean": -82.45503234863281, "KL/mean": -112.23388671875, "KL/rejected_KL_mean": -142.0127410888672, "KL/std": 79.87920379638672, "epoch": 0.4, "fcm_dpo/beta": 0.005705860909074545, "fcm_dpo/delta": -0.009763660840690136, "fcm_dpo/margin": 59.557716369628906, "fcm_dpo/q_t": 0.4211188554763794, "grad_norm": 77.27084350585938, "learning_rate": 3.765821230985757e-07, "logits/chosen": -0.9118991494178772, "logits/rejected": -0.9118152856826782, "logps/chosen": -326.3135681152344, "logps/ref_chosen": -243.8585205078125, "logps/ref_rejected": -245.12136840820312, "logps/rejected": -387.1341247558594, "loss": 4.5937, "margin_dpo/margin_mean": 59.55772018432617, "margin_dpo/margin_std": 94.2480239868164, "step": 191 }, { "KL/chosen_KL_mean": -96.14141845703125, "KL/mean": -120.22209167480469, "KL/rejected_KL_mean": -144.30276489257812, "KL/std": 74.34886932373047, "epoch": 0.40209424083769635, "fcm_dpo/beta": 0.005650951061397791, "fcm_dpo/delta": -0.001796043012291193, "fcm_dpo/margin": 48.16132354736328, "fcm_dpo/q_t": 0.43616360425949097, "grad_norm": 85.8841781616211, "learning_rate": 3.75e-07, "logits/chosen": -0.8480498194694519, "logits/rejected": -0.8291639685630798, "logps/chosen": -363.121337890625, "logps/ref_chosen": -266.9799499511719, "logps/ref_rejected": -260.1697082519531, "logps/rejected": -404.472412109375, "loss": 4.7714, "margin_dpo/margin_mean": 48.16132354736328, "margin_dpo/margin_std": 84.6034927368164, "step": 192 }, { "KL/chosen_KL_mean": -110.70806884765625, "KL/mean": -140.7467498779297, "KL/rejected_KL_mean": -170.78546142578125, "KL/std": 84.00237274169922, "epoch": 0.4041884816753927, "fcm_dpo/beta": 0.00559256924316287, "fcm_dpo/delta": -0.024292614310979843, "fcm_dpo/margin": 60.07738494873047, "fcm_dpo/q_t": 0.4226481318473816, "grad_norm": 79.06672668457031, "learning_rate": 3.734111735307796e-07, "logits/chosen": -0.9012806415557861, "logits/rejected": -0.8710051774978638, "logps/chosen": -390.9613037109375, "logps/ref_chosen": -280.25323486328125, "logps/ref_rejected": -291.0348815917969, "logps/rejected": -461.8203430175781, "loss": 4.5952, "margin_dpo/margin_mean": 60.07738494873047, "margin_dpo/margin_std": 93.01579284667969, "step": 193 }, { "KL/chosen_KL_mean": -119.90608215332031, "KL/mean": -141.48089599609375, "KL/rejected_KL_mean": -163.0557098388672, "KL/std": 80.69340515136719, "epoch": 0.406282722513089, "fcm_dpo/beta": 0.0055352929048240185, "fcm_dpo/delta": 0.009452687576413155, "fcm_dpo/margin": 43.14963150024414, "fcm_dpo/q_t": 0.44507163763046265, "grad_norm": 83.3067398071289, "learning_rate": 3.7181572889485623e-07, "logits/chosen": -0.8725247383117676, "logits/rejected": -0.8608178496360779, "logps/chosen": -408.0455322265625, "logps/ref_chosen": -288.13946533203125, "logps/ref_rejected": -251.31529235839844, "logps/rejected": -414.3709716796875, "loss": 4.8879, "margin_dpo/margin_mean": 43.14963150024414, "margin_dpo/margin_std": 87.68283081054688, "step": 194 }, { "KL/chosen_KL_mean": -126.77201843261719, "KL/mean": -146.34735107421875, "KL/rejected_KL_mean": -165.9226837158203, "KL/std": 80.25843811035156, "epoch": 0.4083769633507853, "fcm_dpo/beta": 0.005730912555009127, "fcm_dpo/delta": 0.06066245585680008, "fcm_dpo/margin": 39.1506462097168, "fcm_dpo/q_t": 0.44943714141845703, "grad_norm": 81.02775573730469, "learning_rate": 3.7021375165108377e-07, "logits/chosen": -0.8785435557365417, "logits/rejected": -0.8827537894248962, "logps/chosen": -400.77264404296875, "logps/ref_chosen": -274.0006408691406, "logps/ref_rejected": -280.22723388671875, "logps/rejected": -446.14996337890625, "loss": 4.9734, "margin_dpo/margin_mean": 39.1506462097168, "margin_dpo/margin_std": 91.88790893554688, "step": 195 }, { "KL/chosen_KL_mean": -105.7139892578125, "KL/mean": -136.76834106445312, "KL/rejected_KL_mean": -167.8226776123047, "KL/std": 83.81902313232422, "epoch": 0.41047120418848165, "fcm_dpo/beta": 0.005761865060776472, "fcm_dpo/delta": -0.02817281149327755, "fcm_dpo/margin": 62.10868453979492, "fcm_dpo/q_t": 0.4175757169723511, "grad_norm": 77.755859375, "learning_rate": 3.6860532770864005e-07, "logits/chosen": -0.8623827695846558, "logits/rejected": -0.8653976321220398, "logps/chosen": -380.61468505859375, "logps/ref_chosen": -274.90069580078125, "logps/ref_rejected": -248.7281951904297, "logps/rejected": -416.5509033203125, "loss": 4.5488, "margin_dpo/margin_mean": 62.10868453979492, "margin_dpo/margin_std": 96.91848754882812, "step": 196 }, { "KL/chosen_KL_mean": -113.64167785644531, "KL/mean": -146.5870361328125, "KL/rejected_KL_mean": -179.53237915039062, "KL/std": 85.77356719970703, "epoch": 0.41256544502617803, "fcm_dpo/beta": 0.005556808784604073, "fcm_dpo/delta": -0.06945441663265228, "fcm_dpo/margin": 65.89070129394531, "fcm_dpo/q_t": 0.4156161844730377, "grad_norm": 101.23210144042969, "learning_rate": 3.6699054332241985e-07, "logits/chosen": -0.8872265219688416, "logits/rejected": -0.8704795837402344, "logps/chosen": -423.176513671875, "logps/ref_chosen": -309.5348205566406, "logps/ref_rejected": -264.3179931640625, "logps/rejected": -443.85040283203125, "loss": 4.5075, "margin_dpo/margin_mean": 65.89070129394531, "margin_dpo/margin_std": 94.22657012939453, "step": 197 }, { "KL/chosen_KL_mean": -101.82292175292969, "KL/mean": -137.69024658203125, "KL/rejected_KL_mean": -173.55758666992188, "KL/std": 86.97761535644531, "epoch": 0.41465968586387436, "fcm_dpo/beta": 0.005258793011307716, "fcm_dpo/delta": -0.04758100211620331, "fcm_dpo/margin": 71.73465728759766, "fcm_dpo/q_t": 0.41255441308021545, "grad_norm": 57.259925842285156, "learning_rate": 3.653694850884091e-07, "logits/chosen": -0.8913165330886841, "logits/rejected": -0.8648770451545715, "logps/chosen": -402.8363952636719, "logps/ref_chosen": -301.0134582519531, "logps/ref_rejected": -292.84185791015625, "logps/rejected": -466.3994140625, "loss": 4.4359, "margin_dpo/margin_mean": 71.73465728759766, "margin_dpo/margin_std": 96.48065948486328, "step": 198 }, { "KL/chosen_KL_mean": -107.54205322265625, "KL/mean": -136.6196746826172, "KL/rejected_KL_mean": -165.6973114013672, "KL/std": 78.0857925415039, "epoch": 0.4167539267015707, "fcm_dpo/beta": 0.005284877959638834, "fcm_dpo/delta": 0.017607517540454865, "fcm_dpo/margin": 58.15523910522461, "fcm_dpo/q_t": 0.42847687005996704, "grad_norm": 71.95082092285156, "learning_rate": 3.6374223993904124e-07, "logits/chosen": -0.8611741065979004, "logits/rejected": -0.817335307598114, "logps/chosen": -372.1479187011719, "logps/ref_chosen": -264.6058654785156, "logps/ref_rejected": -214.9014892578125, "logps/rejected": -380.5987854003906, "loss": 4.6595, "margin_dpo/margin_mean": 58.155242919921875, "margin_dpo/margin_std": 94.97657775878906, "step": 199 }, { "KL/chosen_KL_mean": -113.04508972167969, "KL/mean": -136.30540466308594, "KL/rejected_KL_mean": -159.56573486328125, "KL/std": 83.99603271484375, "epoch": 0.418848167539267, "fcm_dpo/beta": 0.00527481734752655, "fcm_dpo/delta": -0.007252190262079239, "fcm_dpo/margin": 46.5206184387207, "fcm_dpo/q_t": 0.4431796073913574, "grad_norm": 118.91153717041016, "learning_rate": 3.621088951385353e-07, "logits/chosen": -0.9174572229385376, "logits/rejected": -0.8933016061782837, "logps/chosen": -437.20391845703125, "logps/ref_chosen": -324.1588134765625, "logps/ref_rejected": -277.80218505859375, "logps/rejected": -437.367919921875, "loss": 4.959, "margin_dpo/margin_mean": 46.52062225341797, "margin_dpo/margin_std": 107.42146301269531, "step": 200 }, { "epoch": 0.418848167539267, "eval_KL/chosen_KL_mean": -112.52688598632812, "eval_KL/mean": -140.7783966064453, "eval_KL/rejected_KL_mean": -169.0298614501953, "eval_KL/std": 83.77688598632812, "eval_fcm_dpo/beta": 0.005265017040073872, "eval_logits/chosen": -0.8929662704467773, "eval_logits/rejected": -0.8754425644874573, "eval_logps/chosen": -400.35369873046875, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -435.9598693847656, "eval_loss": 0.5908177495002747, "eval_margin_dpo/margin_mean": 56.50297546386719, "eval_margin_dpo/margin_std": 93.3614501953125, "eval_runtime": 78.0509, "eval_samples_per_second": 25.624, "eval_steps_per_second": 3.203, "step": 200 }, { "KL/chosen_KL_mean": -113.03704071044922, "KL/mean": -142.47430419921875, "KL/rejected_KL_mean": -171.91156005859375, "KL/std": 85.64922332763672, "epoch": 0.42094240837696334, "fcm_dpo/beta": 0.005312280263751745, "fcm_dpo/delta": -0.0002721776254475117, "fcm_dpo/margin": 58.8745231628418, "fcm_dpo/q_t": 0.4275677502155304, "grad_norm": 103.92790985107422, "learning_rate": 3.604695382782159e-07, "logits/chosen": -0.8811385631561279, "logits/rejected": -0.8665943145751953, "logps/chosen": -384.5327453613281, "logps/ref_chosen": -271.49566650390625, "logps/ref_rejected": -245.71414184570312, "logps/rejected": -417.62567138671875, "loss": 4.6766, "margin_dpo/margin_mean": 58.8745231628418, "margin_dpo/margin_std": 98.85687255859375, "step": 201 }, { "KL/chosen_KL_mean": -106.796630859375, "KL/mean": -135.2665557861328, "KL/rejected_KL_mean": -163.73648071289062, "KL/std": 82.64311218261719, "epoch": 0.42303664921465967, "fcm_dpo/beta": 0.005215301178395748, "fcm_dpo/delta": -0.007278631906956434, "fcm_dpo/margin": 56.93985366821289, "fcm_dpo/q_t": 0.4311002492904663, "grad_norm": 85.00394439697266, "learning_rate": 3.588242572718162e-07, "logits/chosen": -0.8881719708442688, "logits/rejected": -0.8747313022613525, "logps/chosen": -378.8945617675781, "logps/ref_chosen": -272.0979309082031, "logps/ref_rejected": -235.94805908203125, "logps/rejected": -399.6845703125, "loss": 4.7285, "margin_dpo/margin_mean": 56.93985366821289, "margin_dpo/margin_std": 100.85719299316406, "step": 202 }, { "KL/chosen_KL_mean": -109.50445556640625, "KL/mean": -132.5556182861328, "KL/rejected_KL_mean": -155.60678100585938, "KL/std": 81.00904846191406, "epoch": 0.42513089005235605, "fcm_dpo/beta": 0.0052531929686665535, "fcm_dpo/delta": 0.02188851684331894, "fcm_dpo/margin": 46.10232925415039, "fcm_dpo/q_t": 0.4425586462020874, "grad_norm": 88.16043853759766, "learning_rate": 3.571731403507635e-07, "logits/chosen": -0.8671609163284302, "logits/rejected": -0.8737747669219971, "logps/chosen": -389.72662353515625, "logps/ref_chosen": -280.2221374511719, "logps/ref_rejected": -251.79798889160156, "logps/rejected": -407.40478515625, "loss": 4.8388, "margin_dpo/margin_mean": 46.10232925415039, "margin_dpo/margin_std": 84.27774047851562, "step": 203 }, { "KL/chosen_KL_mean": -97.49296569824219, "KL/mean": -128.00454711914062, "KL/rejected_KL_mean": -158.51614379882812, "KL/std": 79.80975341796875, "epoch": 0.4272251308900524, "fcm_dpo/beta": 0.005305150058120489, "fcm_dpo/delta": 0.006939949467778206, "fcm_dpo/margin": 61.023170471191406, "fcm_dpo/q_t": 0.42425647377967834, "grad_norm": 79.25830841064453, "learning_rate": 3.5551627605944746e-07, "logits/chosen": -0.9088394045829773, "logits/rejected": -0.8813303709030151, "logps/chosen": -416.28900146484375, "logps/ref_chosen": -318.7960510253906, "logps/ref_rejected": -269.69921875, "logps/rejected": -428.21533203125, "loss": 4.5763, "margin_dpo/margin_mean": 61.023170471191406, "margin_dpo/margin_std": 89.2691879272461, "step": 204 }, { "KL/chosen_KL_mean": -97.44485473632812, "KL/mean": -133.10438537597656, "KL/rejected_KL_mean": -168.76394653320312, "KL/std": 87.8086166381836, "epoch": 0.4293193717277487, "fcm_dpo/beta": 0.005317580420523882, "fcm_dpo/delta": -0.030647534877061844, "fcm_dpo/margin": 71.319091796875, "fcm_dpo/q_t": 0.4137461483478546, "grad_norm": 81.98143005371094, "learning_rate": 3.5385375325047163e-07, "logits/chosen": -0.8436496257781982, "logits/rejected": -0.8121804594993591, "logps/chosen": -381.2069091796875, "logps/ref_chosen": -283.7620544433594, "logps/ref_rejected": -297.69439697265625, "logps/rejected": -466.4584045410156, "loss": 4.4529, "margin_dpo/margin_mean": 71.31909942626953, "margin_dpo/margin_std": 99.79702758789062, "step": 205 }, { "KL/chosen_KL_mean": -106.53814697265625, "KL/mean": -134.1820831298828, "KL/rejected_KL_mean": -161.8260040283203, "KL/std": 88.01177978515625, "epoch": 0.431413612565445, "fcm_dpo/beta": 0.005157336127012968, "fcm_dpo/delta": -0.01993716135621071, "fcm_dpo/margin": 55.28785705566406, "fcm_dpo/q_t": 0.4342867136001587, "grad_norm": 79.73020935058594, "learning_rate": 3.5218566107988867e-07, "logits/chosen": -0.8805474042892456, "logits/rejected": -0.9017478823661804, "logps/chosen": -400.20196533203125, "logps/ref_chosen": -293.66387939453125, "logps/ref_rejected": -291.3056640625, "logps/rejected": -453.13165283203125, "loss": 4.7925, "margin_dpo/margin_mean": 55.28785705566406, "margin_dpo/margin_std": 103.69197082519531, "step": 206 }, { "KL/chosen_KL_mean": -99.68655395507812, "KL/mean": -127.54710388183594, "KL/rejected_KL_mean": -155.40768432617188, "KL/std": 94.75768280029297, "epoch": 0.43350785340314135, "fcm_dpo/beta": 0.005091050639748573, "fcm_dpo/delta": -0.01264068391174078, "fcm_dpo/margin": 55.72111511230469, "fcm_dpo/q_t": 0.4338788688182831, "grad_norm": 86.8250961303711, "learning_rate": 3.505120890024195e-07, "logits/chosen": -0.8275532722473145, "logits/rejected": -0.8351794481277466, "logps/chosen": -370.22161865234375, "logps/ref_chosen": -270.5350646972656, "logps/ref_rejected": -278.7747497558594, "logps/rejected": -434.18243408203125, "loss": 4.8577, "margin_dpo/margin_mean": 55.72111511230469, "margin_dpo/margin_std": 117.74676513671875, "step": 207 }, { "KL/chosen_KL_mean": -105.37781524658203, "KL/mean": -140.57730102539062, "KL/rejected_KL_mean": -175.77679443359375, "KL/std": 94.64569854736328, "epoch": 0.4356020942408377, "fcm_dpo/beta": 0.005056449212133884, "fcm_dpo/delta": -0.029296772554516792, "fcm_dpo/margin": 70.39897155761719, "fcm_dpo/q_t": 0.4191908836364746, "grad_norm": 78.56848907470703, "learning_rate": 3.4883312676665534e-07, "logits/chosen": -0.8986639380455017, "logits/rejected": -0.8506540060043335, "logps/chosen": -384.9605712890625, "logps/ref_chosen": -279.582763671875, "logps/ref_rejected": -290.041015625, "logps/rejected": -465.8177795410156, "loss": 4.5134, "margin_dpo/margin_mean": 70.39897155761719, "margin_dpo/margin_std": 102.34636688232422, "step": 208 }, { "KL/chosen_KL_mean": -119.72714233398438, "KL/mean": -140.69674682617188, "KL/rejected_KL_mean": -161.66632080078125, "KL/std": 89.64631652832031, "epoch": 0.437696335078534, "fcm_dpo/beta": 0.005014811176806688, "fcm_dpo/delta": 0.030134355649352074, "fcm_dpo/margin": 41.939170837402344, "fcm_dpo/q_t": 0.4505680203437805, "grad_norm": 74.9662094116211, "learning_rate": 3.4714886441024573e-07, "logits/chosen": -0.816275954246521, "logits/rejected": -0.8116741180419922, "logps/chosen": -438.59967041015625, "logps/ref_chosen": -318.8725280761719, "logps/ref_rejected": -270.64324951171875, "logps/rejected": -432.3095703125, "loss": 4.9977, "margin_dpo/margin_mean": 41.939170837402344, "margin_dpo/margin_std": 98.76667785644531, "step": 209 }, { "KL/chosen_KL_mean": -100.22869873046875, "KL/mean": -132.45101928710938, "KL/rejected_KL_mean": -164.67333984375, "KL/std": 86.8050537109375, "epoch": 0.4397905759162304, "fcm_dpo/beta": 0.005045687779784203, "fcm_dpo/delta": -0.00024997442960739136, "fcm_dpo/margin": 64.44464111328125, "fcm_dpo/q_t": 0.4242168068885803, "grad_norm": 76.91278839111328, "learning_rate": 3.454593922550693e-07, "logits/chosen": -0.8455177545547485, "logits/rejected": -0.8349478244781494, "logps/chosen": -383.3689880371094, "logps/ref_chosen": -283.14031982421875, "logps/ref_rejected": -287.2986755371094, "logps/rejected": -451.97198486328125, "loss": 4.605, "margin_dpo/margin_mean": 64.44464111328125, "margin_dpo/margin_std": 99.05096435546875, "step": 210 }, { "KL/chosen_KL_mean": -90.99891662597656, "KL/mean": -128.32627868652344, "KL/rejected_KL_mean": -165.65362548828125, "KL/std": 85.75665283203125, "epoch": 0.4418848167539267, "fcm_dpo/beta": 0.004931907169520855, "fcm_dpo/delta": -0.0706772655248642, "fcm_dpo/margin": 74.65471649169922, "fcm_dpo/q_t": 0.41302767395973206, "grad_norm": 84.34500885009766, "learning_rate": 3.4376480090239047e-07, "logits/chosen": -0.8675280213356018, "logits/rejected": -0.8453372716903687, "logps/chosen": -367.42181396484375, "logps/ref_chosen": -276.4228515625, "logps/ref_rejected": -252.40603637695312, "logps/rejected": -418.0596923828125, "loss": 4.4164, "margin_dpo/margin_mean": 74.65471649169922, "margin_dpo/margin_std": 88.93736267089844, "step": 211 }, { "KL/chosen_KL_mean": -103.40162658691406, "KL/mean": -126.14122009277344, "KL/rejected_KL_mean": -148.88082885742188, "KL/std": 79.97456359863281, "epoch": 0.44397905759162304, "fcm_dpo/beta": 0.0047816019505262375, "fcm_dpo/delta": 0.0027326405979692936, "fcm_dpo/margin": 45.479190826416016, "fcm_dpo/q_t": 0.44804397225379944, "grad_norm": 68.33330535888672, "learning_rate": 3.4206518122800055e-07, "logits/chosen": -0.8594481945037842, "logits/rejected": -0.8569087982177734, "logps/chosen": -375.107177734375, "logps/ref_chosen": -271.7055358886719, "logps/ref_rejected": -241.18511962890625, "logps/rejected": -390.06597900390625, "loss": 4.9313, "margin_dpo/margin_mean": 45.479190826416016, "margin_dpo/margin_std": 90.3721694946289, "step": 212 }, { "KL/chosen_KL_mean": -110.37431335449219, "KL/mean": -135.97512817382812, "KL/rejected_KL_mean": -161.57594299316406, "KL/std": 97.51154327392578, "epoch": 0.44607329842931936, "fcm_dpo/beta": 0.004937951453030109, "fcm_dpo/delta": 0.034974243491888046, "fcm_dpo/margin": 51.201629638671875, "fcm_dpo/q_t": 0.44280922412872314, "grad_norm": 66.99137115478516, "learning_rate": 3.403606243773448e-07, "logits/chosen": -0.8500252962112427, "logits/rejected": -0.8643661141395569, "logps/chosen": -412.6719665527344, "logps/ref_chosen": -302.2976379394531, "logps/ref_rejected": -303.6202087402344, "logps/rejected": -465.1961364746094, "loss": 4.874, "margin_dpo/margin_mean": 51.201629638671875, "margin_dpo/margin_std": 106.69091796875, "step": 213 }, { "KL/chosen_KL_mean": -113.40092468261719, "KL/mean": -147.15130615234375, "KL/rejected_KL_mean": -180.90167236328125, "KL/std": 88.04362487792969, "epoch": 0.4481675392670157, "fcm_dpo/beta": 0.004911818541586399, "fcm_dpo/delta": -0.007429293356835842, "fcm_dpo/margin": 67.50076293945312, "fcm_dpo/q_t": 0.4230693280696869, "grad_norm": 136.9283905029297, "learning_rate": 3.3865122176063385e-07, "logits/chosen": -0.8651580810546875, "logits/rejected": -0.8647922873497009, "logps/chosen": -385.5335388183594, "logps/ref_chosen": -272.13262939453125, "logps/ref_rejected": -294.82354736328125, "logps/rejected": -475.7251892089844, "loss": 4.5422, "margin_dpo/margin_mean": 67.50076293945312, "margin_dpo/margin_std": 91.87295532226562, "step": 214 }, { "KL/chosen_KL_mean": -122.17478942871094, "KL/mean": -142.13975524902344, "KL/rejected_KL_mean": -162.104736328125, "KL/std": 98.62191009521484, "epoch": 0.450261780104712, "fcm_dpo/beta": 0.004899848252534866, "fcm_dpo/delta": -0.021466845646500587, "fcm_dpo/margin": 39.92993927001953, "fcm_dpo/q_t": 0.45550331473350525, "grad_norm": 86.05668640136719, "learning_rate": 3.3693706504794243e-07, "logits/chosen": -0.9035891890525818, "logits/rejected": -0.8835508823394775, "logps/chosen": -413.5531005859375, "logps/ref_chosen": -291.3782958984375, "logps/ref_rejected": -261.05792236328125, "logps/rejected": -423.1626281738281, "loss": 5.0789, "margin_dpo/margin_mean": 39.92993927001953, "margin_dpo/margin_std": 105.15361785888672, "step": 215 }, { "KL/chosen_KL_mean": -113.71073913574219, "KL/mean": -144.38978576660156, "KL/rejected_KL_mean": -175.06881713867188, "KL/std": 93.9161376953125, "epoch": 0.4523560209424084, "fcm_dpo/beta": 0.004838687367737293, "fcm_dpo/delta": -0.0036501260474324226, "fcm_dpo/margin": 61.358089447021484, "fcm_dpo/q_t": 0.4316501319408417, "grad_norm": 91.55261993408203, "learning_rate": 3.3521824616429284e-07, "logits/chosen": -0.9281271696090698, "logits/rejected": -0.920557975769043, "logps/chosen": -452.2161560058594, "logps/ref_chosen": -338.50543212890625, "logps/ref_rejected": -305.76104736328125, "logps/rejected": -480.829833984375, "loss": 4.721, "margin_dpo/margin_mean": 61.358089447021484, "margin_dpo/margin_std": 107.93409729003906, "step": 216 }, { "KL/chosen_KL_mean": -120.99028015136719, "KL/mean": -158.77626037597656, "KL/rejected_KL_mean": -196.56224060058594, "KL/std": 99.00094604492188, "epoch": 0.4544502617801047, "fcm_dpo/beta": 0.004801592789590359, "fcm_dpo/delta": -0.013928312808275223, "fcm_dpo/margin": 75.57192993164062, "fcm_dpo/q_t": 0.4170202612876892, "grad_norm": 60.47035217285156, "learning_rate": 3.334948572847253e-07, "logits/chosen": -0.8282374739646912, "logits/rejected": -0.7875553369522095, "logps/chosen": -414.5400695800781, "logps/ref_chosen": -293.5498046875, "logps/ref_rejected": -256.7830810546875, "logps/rejected": -453.34527587890625, "loss": 4.5201, "margin_dpo/margin_mean": 75.57192993164062, "margin_dpo/margin_std": 112.79179382324219, "step": 217 }, { "KL/chosen_KL_mean": -120.90603637695312, "KL/mean": -155.9094696044922, "KL/rejected_KL_mean": -190.91290283203125, "KL/std": 98.04553985595703, "epoch": 0.45654450261780105, "fcm_dpo/beta": 0.00477956049144268, "fcm_dpo/delta": 0.015686698257923126, "fcm_dpo/margin": 70.00685119628906, "fcm_dpo/q_t": 0.4229409098625183, "grad_norm": 81.7936019897461, "learning_rate": 3.317669908293554e-07, "logits/chosen": -0.8649943470954895, "logits/rejected": -0.8808245658874512, "logps/chosen": -441.4853820800781, "logps/ref_chosen": -320.579345703125, "logps/ref_rejected": -294.0381164550781, "logps/rejected": -484.95098876953125, "loss": 4.576, "margin_dpo/margin_mean": 70.00685119628906, "margin_dpo/margin_std": 107.72746276855469, "step": 218 }, { "KL/chosen_KL_mean": -120.40277099609375, "KL/mean": -156.69488525390625, "KL/rejected_KL_mean": -192.9869842529297, "KL/std": 93.7581558227539, "epoch": 0.4586387434554974, "fcm_dpo/beta": 0.0046853115782141685, "fcm_dpo/delta": -0.0629052147269249, "fcm_dpo/margin": 72.58422088623047, "fcm_dpo/q_t": 0.42137354612350464, "grad_norm": 91.5604019165039, "learning_rate": 3.300347394584172e-07, "logits/chosen": -0.8439445495605469, "logits/rejected": -0.8702387809753418, "logps/chosen": -388.8214111328125, "logps/ref_chosen": -268.4186096191406, "logps/ref_rejected": -265.7808837890625, "logps/rejected": -458.7678527832031, "loss": 4.6062, "margin_dpo/margin_mean": 72.58422088623047, "margin_dpo/margin_std": 113.50650024414062, "step": 219 }, { "KL/chosen_KL_mean": -122.6368408203125, "KL/mean": -160.08717346191406, "KL/rejected_KL_mean": -197.53753662109375, "KL/std": 94.09228515625, "epoch": 0.4607329842931937, "fcm_dpo/beta": 0.00458397064357996, "fcm_dpo/delta": 0.006707936059683561, "fcm_dpo/margin": 74.90068817138672, "fcm_dpo/q_t": 0.4198194146156311, "grad_norm": 66.64740753173828, "learning_rate": 3.2829819606729477e-07, "logits/chosen": -0.9155178070068359, "logits/rejected": -0.8841468095779419, "logps/chosen": -435.5233154296875, "logps/ref_chosen": -312.8864440917969, "logps/ref_rejected": -259.5191955566406, "logps/rejected": -457.05670166015625, "loss": 4.5305, "margin_dpo/margin_mean": 74.90068817138672, "margin_dpo/margin_std": 107.72557830810547, "step": 220 }, { "KL/chosen_KL_mean": -135.22842407226562, "KL/mean": -166.27349853515625, "KL/rejected_KL_mean": -197.3185272216797, "KL/std": 99.48567962646484, "epoch": 0.46282722513089003, "fcm_dpo/beta": 0.004662664607167244, "fcm_dpo/delta": -0.014790516346693039, "fcm_dpo/margin": 62.09008026123047, "fcm_dpo/q_t": 0.43384507298469543, "grad_norm": 71.38502502441406, "learning_rate": 3.265574537815398e-07, "logits/chosen": -0.8199173212051392, "logits/rejected": -0.8298671245574951, "logps/chosen": -435.5542907714844, "logps/ref_chosen": -300.32586669921875, "logps/ref_rejected": -286.312255859375, "logps/rejected": -483.6307678222656, "loss": 4.7667, "margin_dpo/margin_mean": 62.09008026123047, "margin_dpo/margin_std": 114.19816589355469, "step": 221 }, { "KL/chosen_KL_mean": -132.97164916992188, "KL/mean": -162.91835021972656, "KL/rejected_KL_mean": -192.86505126953125, "KL/std": 89.18901062011719, "epoch": 0.4649214659685864, "fcm_dpo/beta": 0.004599052481353283, "fcm_dpo/delta": 0.0036037182435393333, "fcm_dpo/margin": 59.89339065551758, "fcm_dpo/q_t": 0.4353114664554596, "grad_norm": 79.5625991821289, "learning_rate": 3.248126059518784e-07, "logits/chosen": -0.9127927422523499, "logits/rejected": -0.8806558847427368, "logps/chosen": -430.0830383300781, "logps/ref_chosen": -297.1113586425781, "logps/ref_rejected": -235.53146362304688, "logps/rejected": -428.396484375, "loss": 4.7627, "margin_dpo/margin_mean": 59.89339065551758, "margin_dpo/margin_std": 105.86309814453125, "step": 222 }, { "KL/chosen_KL_mean": -142.52178955078125, "KL/mean": -175.47720336914062, "KL/rejected_KL_mean": -208.4326171875, "KL/std": 96.71148681640625, "epoch": 0.46701570680628274, "fcm_dpo/beta": 0.004580706357955933, "fcm_dpo/delta": -0.0016177373472601175, "fcm_dpo/margin": 65.91083526611328, "fcm_dpo/q_t": 0.42940470576286316, "grad_norm": 90.15703582763672, "learning_rate": 3.230637461492043e-07, "logits/chosen": -0.8521276712417603, "logits/rejected": -0.8184829950332642, "logps/chosen": -428.9368896484375, "logps/ref_chosen": -286.41510009765625, "logps/ref_rejected": -241.1181640625, "logps/rejected": -449.5508117675781, "loss": 4.6955, "margin_dpo/margin_mean": 65.91083526611328, "margin_dpo/margin_std": 111.98548126220703, "step": 223 }, { "KL/chosen_KL_mean": -137.31179809570312, "KL/mean": -171.8541717529297, "KL/rejected_KL_mean": -206.39657592773438, "KL/std": 92.82299041748047, "epoch": 0.46910994764397906, "fcm_dpo/beta": 0.0045858751982450485, "fcm_dpo/delta": -0.019513994455337524, "fcm_dpo/margin": 69.08476257324219, "fcm_dpo/q_t": 0.4261767268180847, "grad_norm": 83.94529724121094, "learning_rate": 3.213109681595612e-07, "logits/chosen": -0.8192363977432251, "logits/rejected": -0.836337149143219, "logps/chosen": -386.80413818359375, "logps/ref_chosen": -249.49234008789062, "logps/ref_rejected": -233.10752868652344, "logps/rejected": -439.50408935546875, "loss": 4.6084, "margin_dpo/margin_mean": 69.08476257324219, "margin_dpo/margin_std": 102.27616119384766, "step": 224 }, { "KL/chosen_KL_mean": -160.03781127929688, "KL/mean": -191.45758056640625, "KL/rejected_KL_mean": -222.87734985351562, "KL/std": 104.55242156982422, "epoch": 0.4712041884816754, "fcm_dpo/beta": 0.004542301408946514, "fcm_dpo/delta": 0.026309635490179062, "fcm_dpo/margin": 62.83953857421875, "fcm_dpo/q_t": 0.4349502921104431, "grad_norm": 82.01798248291016, "learning_rate": 3.1955436597911315e-07, "logits/chosen": -0.8548203706741333, "logits/rejected": -0.8286932706832886, "logps/chosen": -471.8962097167969, "logps/ref_chosen": -311.8583679199219, "logps/ref_rejected": -336.8523864746094, "logps/rejected": -559.729736328125, "loss": 4.8119, "margin_dpo/margin_mean": 62.83953857421875, "margin_dpo/margin_std": 125.79779052734375, "step": 225 }, { "KL/chosen_KL_mean": -149.28944396972656, "KL/mean": -175.85955810546875, "KL/rejected_KL_mean": -202.42967224121094, "KL/std": 98.32502746582031, "epoch": 0.4732984293193717, "fcm_dpo/beta": 0.0046951621770858765, "fcm_dpo/delta": 0.028832029551267624, "fcm_dpo/margin": 53.14021301269531, "fcm_dpo/q_t": 0.44293373823165894, "grad_norm": 128.95950317382812, "learning_rate": 3.1779403380910425e-07, "logits/chosen": -0.8814980387687683, "logits/rejected": -0.8697863221168518, "logps/chosen": -401.49072265625, "logps/ref_chosen": -252.20123291015625, "logps/ref_rejected": -254.41162109375, "logps/rejected": -456.8412780761719, "loss": 4.9116, "margin_dpo/margin_mean": 53.14021301269531, "margin_dpo/margin_std": 117.36741638183594, "step": 226 }, { "KL/chosen_KL_mean": -137.6566162109375, "KL/mean": -174.5099334716797, "KL/rejected_KL_mean": -211.36326599121094, "KL/std": 95.45277404785156, "epoch": 0.47539267015706804, "fcm_dpo/beta": 0.004618903622031212, "fcm_dpo/delta": -0.009899847209453583, "fcm_dpo/margin": 73.70665740966797, "fcm_dpo/q_t": 0.42046865820884705, "grad_norm": 75.73706817626953, "learning_rate": 3.160300660508064e-07, "logits/chosen": -0.8298121690750122, "logits/rejected": -0.8260351419448853, "logps/chosen": -422.916015625, "logps/ref_chosen": -285.25946044921875, "logps/ref_rejected": -261.3220520019531, "logps/rejected": -472.685302734375, "loss": 4.5876, "margin_dpo/margin_mean": 73.70665740966797, "margin_dpo/margin_std": 115.51934814453125, "step": 227 }, { "KL/chosen_KL_mean": -124.57246398925781, "KL/mean": -160.07022094726562, "KL/rejected_KL_mean": -195.56796264648438, "KL/std": 91.25104522705078, "epoch": 0.4774869109947644, "fcm_dpo/beta": 0.004695215728133917, "fcm_dpo/delta": -0.007118703797459602, "fcm_dpo/margin": 70.99549102783203, "fcm_dpo/q_t": 0.422940731048584, "grad_norm": 94.28306579589844, "learning_rate": 3.1426255730045695e-07, "logits/chosen": -0.8550602197647095, "logits/rejected": -0.8216891288757324, "logps/chosen": -438.3912658691406, "logps/ref_chosen": -313.81878662109375, "logps/ref_rejected": -258.07061767578125, "logps/rejected": -453.6386413574219, "loss": 4.5961, "margin_dpo/margin_mean": 70.99549865722656, "margin_dpo/margin_std": 109.40447235107422, "step": 228 }, { "KL/chosen_KL_mean": -136.54019165039062, "KL/mean": -179.30322265625, "KL/rejected_KL_mean": -222.06625366210938, "KL/std": 98.02401733398438, "epoch": 0.47958115183246075, "fcm_dpo/beta": 0.0044661094434559345, "fcm_dpo/delta": -0.0841682031750679, "fcm_dpo/margin": 85.52606964111328, "fcm_dpo/q_t": 0.409898966550827, "grad_norm": 84.5251235961914, "learning_rate": 3.1249160234418644e-07, "logits/chosen": -0.8241239190101624, "logits/rejected": -0.8290253281593323, "logps/chosen": -428.5109558105469, "logps/ref_chosen": -291.9707946777344, "logps/ref_rejected": -263.42059326171875, "logps/rejected": -485.4868469238281, "loss": 4.4071, "margin_dpo/margin_mean": 85.52606964111328, "margin_dpo/margin_std": 106.6671142578125, "step": 229 }, { "KL/chosen_KL_mean": -126.73824310302734, "KL/mean": -161.21286010742188, "KL/rejected_KL_mean": -195.6875, "KL/std": 102.5704574584961, "epoch": 0.4816753926701571, "fcm_dpo/beta": 0.00430025951936841, "fcm_dpo/delta": -0.009406418539583683, "fcm_dpo/margin": 68.94926452636719, "fcm_dpo/q_t": 0.43157389760017395, "grad_norm": 98.18880462646484, "learning_rate": 3.1071729615293424e-07, "logits/chosen": -0.8713237643241882, "logits/rejected": -0.8723396062850952, "logps/chosen": -359.99835205078125, "logps/ref_chosen": -233.2601318359375, "logps/ref_rejected": -238.922119140625, "logps/rejected": -434.6096496582031, "loss": 4.7, "margin_dpo/margin_mean": 68.94926452636719, "margin_dpo/margin_std": 116.23746490478516, "step": 230 }, { "KL/chosen_KL_mean": -126.22959899902344, "KL/mean": -155.4517822265625, "KL/rejected_KL_mean": -184.67396545410156, "KL/std": 88.7551040649414, "epoch": 0.4837696335078534, "fcm_dpo/beta": 0.004372628405690193, "fcm_dpo/delta": 0.038051147013902664, "fcm_dpo/margin": 58.44437026977539, "fcm_dpo/q_t": 0.43943536281585693, "grad_norm": 81.86189270019531, "learning_rate": 3.0893973387735683e-07, "logits/chosen": -0.8475738167762756, "logits/rejected": -0.8374499082565308, "logps/chosen": -448.384765625, "logps/ref_chosen": -322.1551818847656, "logps/ref_rejected": -280.97613525390625, "logps/rejected": -465.650146484375, "loss": 4.7883, "margin_dpo/margin_mean": 58.44437026977539, "margin_dpo/margin_std": 99.115478515625, "step": 231 }, { "KL/chosen_KL_mean": -125.79296875, "KL/mean": -156.00680541992188, "KL/rejected_KL_mean": -186.22064208984375, "KL/std": 96.2881088256836, "epoch": 0.48586387434554973, "fcm_dpo/beta": 0.004428422078490257, "fcm_dpo/delta": 0.006661738269031048, "fcm_dpo/margin": 60.42768096923828, "fcm_dpo/q_t": 0.43733125925064087, "grad_norm": 86.83077239990234, "learning_rate": 3.071590108427243e-07, "logits/chosen": -0.8142211437225342, "logits/rejected": -0.7967170476913452, "logps/chosen": -397.5367736816406, "logps/ref_chosen": -271.7437744140625, "logps/ref_rejected": -249.94981384277344, "logps/rejected": -436.1704406738281, "loss": 4.8043, "margin_dpo/margin_mean": 60.427677154541016, "margin_dpo/margin_std": 109.37926483154297, "step": 232 }, { "KL/chosen_KL_mean": -114.01582336425781, "KL/mean": -150.8794403076172, "KL/rejected_KL_mean": -187.74307250976562, "KL/std": 90.44303131103516, "epoch": 0.48795811518324606, "fcm_dpo/beta": 0.004417214542627335, "fcm_dpo/delta": -0.0340757891535759, "fcm_dpo/margin": 73.72722625732422, "fcm_dpo/q_t": 0.4235742688179016, "grad_norm": 75.02037811279297, "learning_rate": 3.05375222543809e-07, "logits/chosen": -0.8514370322227478, "logits/rejected": -0.8423394560813904, "logps/chosen": -399.3581848144531, "logps/ref_chosen": -285.3423156738281, "logps/ref_rejected": -266.34320068359375, "logps/rejected": -454.08624267578125, "loss": 4.5624, "margin_dpo/margin_mean": 73.72722625732422, "margin_dpo/margin_std": 98.64462280273438, "step": 233 }, { "KL/chosen_KL_mean": -114.3414077758789, "KL/mean": -146.4625244140625, "KL/rejected_KL_mean": -178.58364868164062, "KL/std": 90.29096984863281, "epoch": 0.4900523560209424, "fcm_dpo/beta": 0.004416363779455423, "fcm_dpo/delta": 0.037968117743730545, "fcm_dpo/margin": 64.24222564697266, "fcm_dpo/q_t": 0.4340072572231293, "grad_norm": 104.63549041748047, "learning_rate": 3.035884646397637e-07, "logits/chosen": -0.8319493532180786, "logits/rejected": -0.8197311758995056, "logps/chosen": -409.2471618652344, "logps/ref_chosen": -294.9057312011719, "logps/ref_rejected": -299.37054443359375, "logps/rejected": -477.9541931152344, "loss": 4.7523, "margin_dpo/margin_mean": 64.24222564697266, "margin_dpo/margin_std": 115.23040008544922, "step": 234 }, { "KL/chosen_KL_mean": -115.91151428222656, "KL/mean": -149.10227966308594, "KL/rejected_KL_mean": -182.29306030273438, "KL/std": 96.69937133789062, "epoch": 0.49214659685863876, "fcm_dpo/beta": 0.004568690899759531, "fcm_dpo/delta": 0.02875841036438942, "fcm_dpo/margin": 66.38154602050781, "fcm_dpo/q_t": 0.4287912845611572, "grad_norm": 80.04872131347656, "learning_rate": 3.017988329489923e-07, "logits/chosen": -0.8510345816612244, "logits/rejected": -0.8468698263168335, "logps/chosen": -405.40911865234375, "logps/ref_chosen": -289.49755859375, "logps/ref_rejected": -247.55076599121094, "logps/rejected": -429.8438415527344, "loss": 4.6593, "margin_dpo/margin_mean": 66.38154602050781, "margin_dpo/margin_std": 106.72305297851562, "step": 235 }, { "KL/chosen_KL_mean": -108.14529418945312, "KL/mean": -142.6337432861328, "KL/rejected_KL_mean": -177.12220764160156, "KL/std": 94.26268005371094, "epoch": 0.4942408376963351, "fcm_dpo/beta": 0.004645336419343948, "fcm_dpo/delta": 0.005949061363935471, "fcm_dpo/margin": 68.97691345214844, "fcm_dpo/q_t": 0.4257998764514923, "grad_norm": 63.3080940246582, "learning_rate": 3.000064234440111e-07, "logits/chosen": -0.8691312670707703, "logits/rejected": -0.8709132671356201, "logps/chosen": -397.0299377441406, "logps/ref_chosen": -288.8846435546875, "logps/ref_rejected": -242.0452880859375, "logps/rejected": -419.16748046875, "loss": 4.6332, "margin_dpo/margin_mean": 68.97691345214844, "margin_dpo/margin_std": 111.18565368652344, "step": 236 }, { "KL/chosen_KL_mean": -112.85758972167969, "KL/mean": -146.0996856689453, "KL/rejected_KL_mean": -179.341796875, "KL/std": 80.62757110595703, "epoch": 0.4963350785340314, "fcm_dpo/beta": 0.004666031338274479, "fcm_dpo/delta": 0.014529314823448658, "fcm_dpo/margin": 66.48421478271484, "fcm_dpo/q_t": 0.42739030718803406, "grad_norm": 97.25940704345703, "learning_rate": 2.9821133224630223e-07, "logits/chosen": -0.8377714157104492, "logits/rejected": -0.8173346519470215, "logps/chosen": -378.3363037109375, "logps/ref_chosen": -265.47869873046875, "logps/ref_rejected": -267.9891357421875, "logps/rejected": -447.3309326171875, "loss": 4.6172, "margin_dpo/margin_mean": 66.48422241210938, "margin_dpo/margin_std": 99.97410583496094, "step": 237 }, { "KL/chosen_KL_mean": -119.30398559570312, "KL/mean": -151.64341735839844, "KL/rejected_KL_mean": -183.98287963867188, "KL/std": 90.0185546875, "epoch": 0.49842931937172774, "fcm_dpo/beta": 0.004634943325072527, "fcm_dpo/delta": -0.022365430369973183, "fcm_dpo/margin": 64.67887878417969, "fcm_dpo/q_t": 0.43083974719047546, "grad_norm": 97.27252197265625, "learning_rate": 2.964136556211588e-07, "logits/chosen": -0.8281529545783997, "logits/rejected": -0.7977491617202759, "logps/chosen": -431.306640625, "logps/ref_chosen": -312.0026550292969, "logps/ref_rejected": -270.0257263183594, "logps/rejected": -454.0085754394531, "loss": 4.6711, "margin_dpo/margin_mean": 64.67887878417969, "margin_dpo/margin_std": 103.0881576538086, "step": 238 }, { "KL/chosen_KL_mean": -106.01922607421875, "KL/mean": -135.8646240234375, "KL/rejected_KL_mean": -165.71002197265625, "KL/std": 91.91876220703125, "epoch": 0.5005235602094241, "fcm_dpo/beta": 0.004592178855091333, "fcm_dpo/delta": 0.017465800046920776, "fcm_dpo/margin": 59.690773010253906, "fcm_dpo/q_t": 0.43653061985969543, "grad_norm": 95.7003402709961, "learning_rate": 2.946134899725226e-07, "logits/chosen": -0.837583065032959, "logits/rejected": -0.8786738514900208, "logps/chosen": -373.18646240234375, "logps/ref_chosen": -267.167236328125, "logps/ref_rejected": -275.99468994140625, "logps/rejected": -441.7046813964844, "loss": 4.8234, "margin_dpo/margin_mean": 59.690773010253906, "margin_dpo/margin_std": 118.9461441040039, "step": 239 }, { "KL/chosen_KL_mean": -104.64605712890625, "KL/mean": -141.4502410888672, "KL/rejected_KL_mean": -178.25445556640625, "KL/std": 90.54521179199219, "epoch": 0.5026178010471204, "fcm_dpo/beta": 0.004639900289475918, "fcm_dpo/delta": 0.008587292395532131, "fcm_dpo/margin": 73.60839080810547, "fcm_dpo/q_t": 0.42163485288619995, "grad_norm": 61.846168518066406, "learning_rate": 2.9281093183781403e-07, "logits/chosen": -0.8871752023696899, "logits/rejected": -0.8817972540855408, "logps/chosen": -390.62567138671875, "logps/ref_chosen": -285.9796142578125, "logps/ref_rejected": -256.8258056640625, "logps/rejected": -435.0802307128906, "loss": 4.5299, "margin_dpo/margin_mean": 73.60839080810547, "margin_dpo/margin_std": 106.27149963378906, "step": 240 }, { "KL/chosen_KL_mean": -121.47247314453125, "KL/mean": -149.4210968017578, "KL/rejected_KL_mean": -177.36972045898438, "KL/std": 95.52164459228516, "epoch": 0.5047120418848168, "fcm_dpo/beta": 0.004665836691856384, "fcm_dpo/delta": -0.0019118525087833405, "fcm_dpo/margin": 55.897247314453125, "fcm_dpo/q_t": 0.44022229313850403, "grad_norm": 55.96895217895508, "learning_rate": 2.910060778827554e-07, "logits/chosen": -0.7976040840148926, "logits/rejected": -0.773979127407074, "logps/chosen": -382.9893493652344, "logps/ref_chosen": -261.516845703125, "logps/ref_rejected": -250.2250518798828, "logps/rejected": -427.59478759765625, "loss": 4.8268, "margin_dpo/margin_mean": 55.897247314453125, "margin_dpo/margin_std": 105.24579620361328, "step": 241 }, { "KL/chosen_KL_mean": -109.387451171875, "KL/mean": -138.6495361328125, "KL/rejected_KL_mean": -167.91165161132812, "KL/std": 94.68675994873047, "epoch": 0.506806282722513, "fcm_dpo/beta": 0.004686980042606592, "fcm_dpo/delta": 0.028351204469799995, "fcm_dpo/margin": 58.5241813659668, "fcm_dpo/q_t": 0.4375962018966675, "grad_norm": 69.91267395019531, "learning_rate": 2.891990248961871e-07, "logits/chosen": -0.8706403374671936, "logits/rejected": -0.8489304184913635, "logps/chosen": -379.90142822265625, "logps/ref_chosen": -270.51397705078125, "logps/ref_rejected": -244.8560791015625, "logps/rejected": -412.7677001953125, "loss": 4.7544, "margin_dpo/margin_mean": 58.52418518066406, "margin_dpo/margin_std": 103.48672485351562, "step": 242 }, { "KL/chosen_KL_mean": -106.11418914794922, "KL/mean": -139.8125762939453, "KL/rejected_KL_mean": -173.510986328125, "KL/std": 88.66896057128906, "epoch": 0.5089005235602094, "fcm_dpo/beta": 0.004790696781128645, "fcm_dpo/delta": -0.01932116597890854, "fcm_dpo/margin": 67.39679718017578, "fcm_dpo/q_t": 0.4278520345687866, "grad_norm": 73.60661315917969, "learning_rate": 2.873898697848762e-07, "logits/chosen": -0.8736940622329712, "logits/rejected": -0.8598724603652954, "logps/chosen": -430.7962646484375, "logps/ref_chosen": -324.68206787109375, "logps/ref_rejected": -307.1111755371094, "logps/rejected": -480.62213134765625, "loss": 4.6179, "margin_dpo/margin_mean": 67.39679718017578, "margin_dpo/margin_std": 106.52936553955078, "step": 243 }, { "KL/chosen_KL_mean": -112.37324523925781, "KL/mean": -146.05435180664062, "KL/rejected_KL_mean": -179.73544311523438, "KL/std": 87.45367431640625, "epoch": 0.5109947643979058, "fcm_dpo/beta": 0.0046486621722579, "fcm_dpo/delta": -0.014583555981516838, "fcm_dpo/margin": 67.36219024658203, "fcm_dpo/q_t": 0.42639923095703125, "grad_norm": 69.93437957763672, "learning_rate": 2.8557870956832133e-07, "logits/chosen": -0.861052930355072, "logits/rejected": -0.8165071606636047, "logps/chosen": -431.35247802734375, "logps/ref_chosen": -318.979248046875, "logps/ref_rejected": -269.67572021484375, "logps/rejected": -449.4111633300781, "loss": 4.6283, "margin_dpo/margin_mean": 67.36219024658203, "margin_dpo/margin_std": 100.07360076904297, "step": 244 }, { "KL/chosen_KL_mean": -108.40132141113281, "KL/mean": -140.7490997314453, "KL/rejected_KL_mean": -173.09686279296875, "KL/std": 80.6366958618164, "epoch": 0.5130890052356021, "fcm_dpo/beta": 0.004631055053323507, "fcm_dpo/delta": -0.014426006004214287, "fcm_dpo/margin": 64.69554138183594, "fcm_dpo/q_t": 0.42882615327835083, "grad_norm": 97.62570190429688, "learning_rate": 2.837656413735479e-07, "logits/chosen": -0.880599856376648, "logits/rejected": -0.8871563076972961, "logps/chosen": -403.29937744140625, "logps/ref_chosen": -294.8980712890625, "logps/ref_rejected": -239.8111114501953, "logps/rejected": -412.907958984375, "loss": 4.6392, "margin_dpo/margin_mean": 64.6955337524414, "margin_dpo/margin_std": 92.92266845703125, "step": 245 }, { "KL/chosen_KL_mean": -128.1768341064453, "KL/mean": -153.97454833984375, "KL/rejected_KL_mean": -179.77224731445312, "KL/std": 83.72673034667969, "epoch": 0.5151832460732985, "fcm_dpo/beta": 0.004627012647688389, "fcm_dpo/delta": 0.025665562599897385, "fcm_dpo/margin": 51.595394134521484, "fcm_dpo/q_t": 0.44522804021835327, "grad_norm": 74.21184539794922, "learning_rate": 2.8195076242990116e-07, "logits/chosen": -0.8659073114395142, "logits/rejected": -0.8697488307952881, "logps/chosen": -408.86224365234375, "logps/ref_chosen": -280.6854248046875, "logps/ref_rejected": -253.65382385253906, "logps/rejected": -433.426025390625, "loss": 4.8917, "margin_dpo/margin_mean": 51.595394134521484, "margin_dpo/margin_std": 106.63719177246094, "step": 246 }, { "KL/chosen_KL_mean": -124.93351745605469, "KL/mean": -155.44081115722656, "KL/rejected_KL_mean": -185.94808959960938, "KL/std": 90.46797180175781, "epoch": 0.5172774869109947, "fcm_dpo/beta": 0.00462773721665144, "fcm_dpo/delta": -0.03191925212740898, "fcm_dpo/margin": 61.01456832885742, "fcm_dpo/q_t": 0.4344664216041565, "grad_norm": 82.88591766357422, "learning_rate": 2.801341700638307e-07, "logits/chosen": -0.8671166896820068, "logits/rejected": -0.8641787171363831, "logps/chosen": -406.0426025390625, "logps/ref_chosen": -281.1091003417969, "logps/ref_rejected": -260.3700866699219, "logps/rejected": -446.318115234375, "loss": 4.736, "margin_dpo/margin_mean": 61.01456832885742, "margin_dpo/margin_std": 101.25210571289062, "step": 247 }, { "KL/chosen_KL_mean": -126.19422149658203, "KL/mean": -153.88031005859375, "KL/rejected_KL_mean": -181.56640625, "KL/std": 91.33638763427734, "epoch": 0.5193717277486911, "fcm_dpo/beta": 0.004508162848651409, "fcm_dpo/delta": -0.018666669726371765, "fcm_dpo/margin": 55.37216567993164, "fcm_dpo/q_t": 0.441021203994751, "grad_norm": 84.99152374267578, "learning_rate": 2.7831596169367227e-07, "logits/chosen": -0.8335204124450684, "logits/rejected": -0.8391017913818359, "logps/chosen": -396.5125732421875, "logps/ref_chosen": -270.318359375, "logps/ref_rejected": -233.46778869628906, "logps/rejected": -415.0342102050781, "loss": 4.8294, "margin_dpo/margin_mean": 55.37216567993164, "margin_dpo/margin_std": 100.47364807128906, "step": 248 }, { "KL/chosen_KL_mean": -144.4632110595703, "KL/mean": -169.6751251220703, "KL/rejected_KL_mean": -194.88705444335938, "KL/std": 96.40766143798828, "epoch": 0.5214659685863874, "fcm_dpo/beta": 0.0045876894146203995, "fcm_dpo/delta": 0.0523696206510067, "fcm_dpo/margin": 50.42381286621094, "fcm_dpo/q_t": 0.4456580579280853, "grad_norm": 70.08956146240234, "learning_rate": 2.7649623482442274e-07, "logits/chosen": -0.8723926544189453, "logits/rejected": -0.8454214930534363, "logps/chosen": -420.2720642089844, "logps/ref_chosen": -275.8088684082031, "logps/ref_rejected": -243.45138549804688, "logps/rejected": -438.33843994140625, "loss": 4.9733, "margin_dpo/margin_mean": 50.42381286621094, "margin_dpo/margin_std": 120.47492980957031, "step": 249 }, { "KL/chosen_KL_mean": -147.3335418701172, "KL/mean": -185.5907440185547, "KL/rejected_KL_mean": -223.84793090820312, "KL/std": 113.06914520263672, "epoch": 0.5235602094240838, "fcm_dpo/beta": 0.004560520872473717, "fcm_dpo/delta": -0.05923834070563316, "fcm_dpo/margin": 76.51438903808594, "fcm_dpo/q_t": 0.4200321435928345, "grad_norm": 77.23839569091797, "learning_rate": 2.7467508704251135e-07, "logits/chosen": -0.8727688193321228, "logits/rejected": -0.8649075031280518, "logps/chosen": -439.8280944824219, "logps/ref_chosen": -292.4945373535156, "logps/ref_rejected": -284.2869567871094, "logps/rejected": -508.13482666015625, "loss": 4.6123, "margin_dpo/margin_mean": 76.51438903808594, "margin_dpo/margin_std": 127.33292388916016, "step": 250 }, { "KL/chosen_KL_mean": -137.64854431152344, "KL/mean": -172.20977783203125, "KL/rejected_KL_mean": -206.77101135253906, "KL/std": 109.15238189697266, "epoch": 0.5256544502617801, "fcm_dpo/beta": 0.004414653405547142, "fcm_dpo/delta": -0.04583483934402466, "fcm_dpo/margin": 69.12245178222656, "fcm_dpo/q_t": 0.433123379945755, "grad_norm": 99.51129913330078, "learning_rate": 2.7285261601056697e-07, "logits/chosen": -0.8708928823471069, "logits/rejected": -0.8535991311073303, "logps/chosen": -419.3851318359375, "logps/ref_chosen": -281.736572265625, "logps/ref_rejected": -255.9419708251953, "logps/rejected": -462.7129821777344, "loss": 4.7186, "margin_dpo/margin_mean": 69.12245178222656, "margin_dpo/margin_std": 121.37847900390625, "step": 251 }, { "KL/chosen_KL_mean": -150.81507873535156, "KL/mean": -186.76541137695312, "KL/rejected_KL_mean": -222.71575927734375, "KL/std": 103.89976501464844, "epoch": 0.5277486910994764, "fcm_dpo/beta": 0.004231244325637817, "fcm_dpo/delta": -0.02651745080947876, "fcm_dpo/margin": 71.90065002441406, "fcm_dpo/q_t": 0.4281459152698517, "grad_norm": 55.069679260253906, "learning_rate": 2.7102891946217994e-07, "logits/chosen": -0.9138679504394531, "logits/rejected": -0.8856754302978516, "logps/chosen": -446.7825622558594, "logps/ref_chosen": -295.9674072265625, "logps/ref_rejected": -280.111572265625, "logps/rejected": -502.82733154296875, "loss": 4.6809, "margin_dpo/margin_mean": 71.90065002441406, "margin_dpo/margin_std": 118.01815032958984, "step": 252 }, { "KL/chosen_KL_mean": -162.62567138671875, "KL/mean": -196.12326049804688, "KL/rejected_KL_mean": -229.620849609375, "KL/std": 113.87892150878906, "epoch": 0.5298429319371728, "fcm_dpo/beta": 0.004193556495010853, "fcm_dpo/delta": -0.004063830710947514, "fcm_dpo/margin": 66.99517822265625, "fcm_dpo/q_t": 0.43573546409606934, "grad_norm": 64.6878890991211, "learning_rate": 2.692040951966617e-07, "logits/chosen": -0.8893136978149414, "logits/rejected": -0.8789063692092896, "logps/chosen": -439.69793701171875, "logps/ref_chosen": -277.072265625, "logps/ref_rejected": -247.31643676757812, "logps/rejected": -476.9372863769531, "loss": 4.8051, "margin_dpo/margin_mean": 66.99517822265625, "margin_dpo/margin_std": 127.16492462158203, "step": 253 }, { "KL/chosen_KL_mean": -155.98829650878906, "KL/mean": -191.45701599121094, "KL/rejected_KL_mean": -226.92575073242188, "KL/std": 111.76846313476562, "epoch": 0.5319371727748691, "fcm_dpo/beta": 0.00419703358784318, "fcm_dpo/delta": -0.028339367359876633, "fcm_dpo/margin": 70.93742370605469, "fcm_dpo/q_t": 0.43127933144569397, "grad_norm": 61.05302047729492, "learning_rate": 2.6737824107379947e-07, "logits/chosen": -0.8169664740562439, "logits/rejected": -0.7953270673751831, "logps/chosen": -425.9361572265625, "logps/ref_chosen": -269.9478454589844, "logps/ref_rejected": -249.45005798339844, "logps/rejected": -476.37579345703125, "loss": 4.701, "margin_dpo/margin_mean": 70.93742370605469, "margin_dpo/margin_std": 119.53722381591797, "step": 254 }, { "KL/chosen_KL_mean": -152.83241271972656, "KL/mean": -194.11044311523438, "KL/rejected_KL_mean": -235.38848876953125, "KL/std": 110.81584930419922, "epoch": 0.5340314136125655, "fcm_dpo/beta": 0.004109766334295273, "fcm_dpo/delta": -0.013929899781942368, "fcm_dpo/margin": 82.55609130859375, "fcm_dpo/q_t": 0.4215297996997833, "grad_norm": 83.60579681396484, "learning_rate": 2.655514550086086e-07, "logits/chosen": -0.842009961605072, "logits/rejected": -0.8048016428947449, "logps/chosen": -459.4877014160156, "logps/ref_chosen": -306.6552734375, "logps/ref_rejected": -254.47528076171875, "logps/rejected": -489.8637390136719, "loss": 4.6142, "margin_dpo/margin_mean": 82.55609893798828, "margin_dpo/margin_std": 136.48744201660156, "step": 255 }, { "KL/chosen_KL_mean": -155.2074432373047, "KL/mean": -192.99703979492188, "KL/rejected_KL_mean": -230.78665161132812, "KL/std": 112.66366577148438, "epoch": 0.5361256544502618, "fcm_dpo/beta": 0.003994643688201904, "fcm_dpo/delta": -0.06211162358522415, "fcm_dpo/margin": 75.57919311523438, "fcm_dpo/q_t": 0.42736783623695374, "grad_norm": 89.4056167602539, "learning_rate": 2.6372383496608186e-07, "logits/chosen": -0.8736791014671326, "logits/rejected": -0.8580365180969238, "logps/chosen": -478.9256591796875, "logps/ref_chosen": -323.7181701660156, "logps/ref_rejected": -254.1871337890625, "logps/rejected": -484.9737854003906, "loss": 4.7341, "margin_dpo/margin_mean": 75.57919311523438, "margin_dpo/margin_std": 127.94612884521484, "step": 256 }, { "KL/chosen_KL_mean": -156.5220184326172, "KL/mean": -196.63253784179688, "KL/rejected_KL_mean": -236.7430419921875, "KL/std": 107.44888305664062, "epoch": 0.5382198952879581, "fcm_dpo/beta": 0.003933961037546396, "fcm_dpo/delta": 0.035381704568862915, "fcm_dpo/margin": 80.22103881835938, "fcm_dpo/q_t": 0.4262939393520355, "grad_norm": 69.32088470458984, "learning_rate": 2.618954789559356e-07, "logits/chosen": -0.857764720916748, "logits/rejected": -0.8485167622566223, "logps/chosen": -423.73406982421875, "logps/ref_chosen": -267.21209716796875, "logps/ref_rejected": -249.12579345703125, "logps/rejected": -485.86883544921875, "loss": 4.6149, "margin_dpo/margin_mean": 80.22103881835938, "margin_dpo/margin_std": 123.36433410644531, "step": 257 }, { "KL/chosen_KL_mean": -161.85557556152344, "KL/mean": -204.7264404296875, "KL/rejected_KL_mean": -247.5973358154297, "KL/std": 105.87276458740234, "epoch": 0.5403141361256545, "fcm_dpo/beta": 0.0038711142260581255, "fcm_dpo/delta": -0.05333390459418297, "fcm_dpo/margin": 85.74173736572266, "fcm_dpo/q_t": 0.4211893677711487, "grad_norm": 59.2056770324707, "learning_rate": 2.600664850273538e-07, "logits/chosen": -0.8781223297119141, "logits/rejected": -0.8470626473426819, "logps/chosen": -439.53826904296875, "logps/ref_chosen": -277.6827392578125, "logps/ref_rejected": -250.73385620117188, "logps/rejected": -498.3311767578125, "loss": 4.5362, "margin_dpo/margin_mean": 85.74173736572266, "margin_dpo/margin_std": 114.76727294921875, "step": 258 }, { "KL/chosen_KL_mean": -154.62147521972656, "KL/mean": -193.3902587890625, "KL/rejected_KL_mean": -232.1590576171875, "KL/std": 112.23604583740234, "epoch": 0.5424083769633508, "fcm_dpo/beta": 0.0038862007204443216, "fcm_dpo/delta": 0.00880090519785881, "fcm_dpo/margin": 77.53759765625, "fcm_dpo/q_t": 0.4306592345237732, "grad_norm": 74.10926055908203, "learning_rate": 2.582369512637302e-07, "logits/chosen": -0.8891596794128418, "logits/rejected": -0.8823885321617126, "logps/chosen": -449.2314453125, "logps/ref_chosen": -294.6099853515625, "logps/ref_rejected": -272.2725830078125, "logps/rejected": -504.431640625, "loss": 4.6745, "margin_dpo/margin_mean": 77.53760528564453, "margin_dpo/margin_std": 126.21345520019531, "step": 259 }, { "KL/chosen_KL_mean": -165.0343780517578, "KL/mean": -180.45216369628906, "KL/rejected_KL_mean": -195.8699951171875, "KL/std": 95.3846206665039, "epoch": 0.5445026178010471, "fcm_dpo/beta": 0.003850242355838418, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.835628509521484, "fcm_dpo/q_t": 0.47213658690452576, "grad_norm": 68.87381744384766, "learning_rate": 2.5640697577740815e-07, "logits/chosen": -0.8860092163085938, "logits/rejected": -0.8798715472221375, "logps/chosen": -455.8914794921875, "logps/ref_chosen": -290.85711669921875, "logps/ref_rejected": -277.5970153808594, "logps/rejected": -473.4670104980469, "loss": 5.273, "margin_dpo/margin_mean": 30.83563232421875, "margin_dpo/margin_std": 111.492431640625, "step": 260 }, { "KL/chosen_KL_mean": -178.47940063476562, "KL/mean": -209.2558135986328, "KL/rejected_KL_mean": -240.03221130371094, "KL/std": 104.58702087402344, "epoch": 0.5465968586387434, "fcm_dpo/beta": 0.0038688078057020903, "fcm_dpo/delta": 0.007620506454259157, "fcm_dpo/margin": 61.552818298339844, "fcm_dpo/q_t": 0.44480204582214355, "grad_norm": 58.4471549987793, "learning_rate": 2.5457665670441937e-07, "logits/chosen": -0.7781463861465454, "logits/rejected": -0.7877897620201111, "logps/chosen": -429.6116027832031, "logps/ref_chosen": -251.13223266601562, "logps/ref_rejected": -244.76016235351562, "logps/rejected": -484.7923583984375, "loss": 4.9127, "margin_dpo/margin_mean": 61.552818298339844, "margin_dpo/margin_std": 131.39944458007812, "step": 261 }, { "KL/chosen_KL_mean": -166.16873168945312, "KL/mean": -208.1382598876953, "KL/rejected_KL_mean": -250.1077880859375, "KL/std": 105.5480728149414, "epoch": 0.5486910994764398, "fcm_dpo/beta": 0.003931888844817877, "fcm_dpo/delta": -0.0042154379189014435, "fcm_dpo/margin": 83.93904113769531, "fcm_dpo/q_t": 0.42236289381980896, "grad_norm": 64.91275024414062, "learning_rate": 2.527460921992209e-07, "logits/chosen": -0.8072105646133423, "logits/rejected": -0.7947403788566589, "logps/chosen": -465.8905029296875, "logps/ref_chosen": -299.7217712402344, "logps/ref_rejected": -277.0969543457031, "logps/rejected": -527.2047119140625, "loss": 4.55, "margin_dpo/margin_mean": 83.93904113769531, "margin_dpo/margin_std": 116.44705963134766, "step": 262 }, { "KL/chosen_KL_mean": -173.62734985351562, "KL/mean": -207.27186584472656, "KL/rejected_KL_mean": -240.91635131835938, "KL/std": 108.08099365234375, "epoch": 0.5507853403141362, "fcm_dpo/beta": 0.0038349460810422897, "fcm_dpo/delta": -0.013341857120394707, "fcm_dpo/margin": 67.28901672363281, "fcm_dpo/q_t": 0.43885383009910583, "grad_norm": 56.40167999267578, "learning_rate": 2.509153804294318e-07, "logits/chosen": -0.813066840171814, "logits/rejected": -0.7947369813919067, "logps/chosen": -453.5799560546875, "logps/ref_chosen": -279.95257568359375, "logps/ref_rejected": -256.5327453613281, "logps/rejected": -497.4490661621094, "loss": 4.8042, "margin_dpo/margin_mean": 67.28901672363281, "margin_dpo/margin_std": 117.43939208984375, "step": 263 }, { "KL/chosen_KL_mean": -166.5303955078125, "KL/mean": -209.10252380371094, "KL/rejected_KL_mean": -251.6746826171875, "KL/std": 108.8144302368164, "epoch": 0.5528795811518324, "fcm_dpo/beta": 0.0037783209700137377, "fcm_dpo/delta": -0.005399790126830339, "fcm_dpo/margin": 85.1442642211914, "fcm_dpo/q_t": 0.42472028732299805, "grad_norm": 69.8880386352539, "learning_rate": 2.4908461957056825e-07, "logits/chosen": -0.8240674734115601, "logits/rejected": -0.8171445727348328, "logps/chosen": -427.06549072265625, "logps/ref_chosen": -260.53509521484375, "logps/ref_rejected": -255.53799438476562, "logps/rejected": -507.212646484375, "loss": 4.5679, "margin_dpo/margin_mean": 85.1442642211914, "margin_dpo/margin_std": 118.58462524414062, "step": 264 }, { "KL/chosen_KL_mean": -170.61724853515625, "KL/mean": -213.73841857910156, "KL/rejected_KL_mean": -256.859619140625, "KL/std": 115.99640655517578, "epoch": 0.5549738219895288, "fcm_dpo/beta": 0.003773211035877466, "fcm_dpo/delta": -0.012947505339980125, "fcm_dpo/margin": 86.2424087524414, "fcm_dpo/q_t": 0.424190491437912, "grad_norm": 53.74254608154297, "learning_rate": 2.4725390780077905e-07, "logits/chosen": -0.8812572956085205, "logits/rejected": -0.8822892308235168, "logps/chosen": -454.33026123046875, "logps/ref_chosen": -283.7130432128906, "logps/ref_rejected": -270.3209533691406, "logps/rejected": -527.1806030273438, "loss": 4.5936, "margin_dpo/margin_mean": 86.2424087524414, "margin_dpo/margin_std": 130.76058959960938, "step": 265 }, { "KL/chosen_KL_mean": -154.7050323486328, "KL/mean": -199.395751953125, "KL/rejected_KL_mean": -244.08648681640625, "KL/std": 108.90046691894531, "epoch": 0.5570680628272251, "fcm_dpo/beta": 0.0037543033249676228, "fcm_dpo/delta": -0.023054908961057663, "fcm_dpo/margin": 89.3814468383789, "fcm_dpo/q_t": 0.4214840829372406, "grad_norm": 75.81797790527344, "learning_rate": 2.454233432955807e-07, "logits/chosen": -0.9104347229003906, "logits/rejected": -0.8814761638641357, "logps/chosen": -432.8043212890625, "logps/ref_chosen": -278.09930419921875, "logps/ref_rejected": -260.6734619140625, "logps/rejected": -504.75994873046875, "loss": 4.5175, "margin_dpo/margin_mean": 89.3814468383789, "margin_dpo/margin_std": 117.37722778320312, "step": 266 }, { "KL/chosen_KL_mean": -182.2346649169922, "KL/mean": -214.27894592285156, "KL/rejected_KL_mean": -246.32321166992188, "KL/std": 112.4916763305664, "epoch": 0.5591623036649215, "fcm_dpo/beta": 0.0037524704821407795, "fcm_dpo/delta": 0.04089874029159546, "fcm_dpo/margin": 64.08850860595703, "fcm_dpo/q_t": 0.44417688250541687, "grad_norm": 61.133914947509766, "learning_rate": 2.435930242225919e-07, "logits/chosen": -0.8439475893974304, "logits/rejected": -0.8501834869384766, "logps/chosen": -462.5679016113281, "logps/ref_chosen": -280.33319091796875, "logps/ref_rejected": -247.78099060058594, "logps/rejected": -494.1041564941406, "loss": 4.8717, "margin_dpo/margin_mean": 64.08850860595703, "margin_dpo/margin_std": 123.6415786743164, "step": 267 }, { "KL/chosen_KL_mean": -163.2926025390625, "KL/mean": -205.69686889648438, "KL/rejected_KL_mean": -248.1011199951172, "KL/std": 111.8238296508789, "epoch": 0.5612565445026177, "fcm_dpo/beta": 0.003843301208689809, "fcm_dpo/delta": 0.0008840234950184822, "fcm_dpo/margin": 84.80854034423828, "fcm_dpo/q_t": 0.42401742935180664, "grad_norm": 68.5013656616211, "learning_rate": 2.4176304873626984e-07, "logits/chosen": -0.8200739026069641, "logits/rejected": -0.7995492815971375, "logps/chosen": -467.4713134765625, "logps/ref_chosen": -304.1787109375, "logps/ref_rejected": -272.80316162109375, "logps/rejected": -520.904296875, "loss": 4.5785, "margin_dpo/margin_mean": 84.80854034423828, "margin_dpo/margin_std": 123.63158416748047, "step": 268 }, { "KL/chosen_KL_mean": -163.21693420410156, "KL/mean": -194.07969665527344, "KL/rejected_KL_mean": -224.94247436523438, "KL/std": 99.83782196044922, "epoch": 0.5633507853403141, "fcm_dpo/beta": 0.003911444451659918, "fcm_dpo/delta": 0.05910227820277214, "fcm_dpo/margin": 61.725563049316406, "fcm_dpo/q_t": 0.4430586099624634, "grad_norm": 87.58391571044922, "learning_rate": 2.399335149726463e-07, "logits/chosen": -0.8339531421661377, "logits/rejected": -0.8274367451667786, "logps/chosen": -413.0619812011719, "logps/ref_chosen": -249.84512329101562, "logps/ref_rejected": -223.37356567382812, "logps/rejected": -448.3160400390625, "loss": 4.867, "margin_dpo/margin_mean": 61.725563049316406, "margin_dpo/margin_std": 122.53068542480469, "step": 269 }, { "KL/chosen_KL_mean": -168.89588928222656, "KL/mean": -206.75592041015625, "KL/rejected_KL_mean": -244.61593627929688, "KL/std": 114.02747344970703, "epoch": 0.5654450261780105, "fcm_dpo/beta": 0.004022765904664993, "fcm_dpo/delta": 0.02421419322490692, "fcm_dpo/margin": 75.72004699707031, "fcm_dpo/q_t": 0.43018603324890137, "grad_norm": 64.52881622314453, "learning_rate": 2.381045210440644e-07, "logits/chosen": -0.9270073175430298, "logits/rejected": -0.9333524703979492, "logps/chosen": -487.458251953125, "logps/ref_chosen": -318.5623779296875, "logps/ref_rejected": -281.1880798339844, "logps/rejected": -525.8040161132812, "loss": 4.7208, "margin_dpo/margin_mean": 75.72004699707031, "margin_dpo/margin_std": 138.9295654296875, "step": 270 }, { "KL/chosen_KL_mean": -167.02001953125, "KL/mean": -201.0181884765625, "KL/rejected_KL_mean": -235.01632690429688, "KL/std": 112.24214935302734, "epoch": 0.5675392670157068, "fcm_dpo/beta": 0.00413860660046339, "fcm_dpo/delta": 0.023125268518924713, "fcm_dpo/margin": 67.99632263183594, "fcm_dpo/q_t": 0.4350787103176117, "grad_norm": 74.27021789550781, "learning_rate": 2.3627616503391812e-07, "logits/chosen": -0.7775533199310303, "logits/rejected": -0.7718729972839355, "logps/chosen": -451.1247253417969, "logps/ref_chosen": -284.104736328125, "logps/ref_rejected": -253.9580535888672, "logps/rejected": -488.97442626953125, "loss": 4.7303, "margin_dpo/margin_mean": 67.99632263183594, "margin_dpo/margin_std": 115.17955017089844, "step": 271 }, { "KL/chosen_KL_mean": -157.8939666748047, "KL/mean": -197.00889587402344, "KL/rejected_KL_mean": -236.12380981445312, "KL/std": 115.54269409179688, "epoch": 0.5696335078534032, "fcm_dpo/beta": 0.004133047070354223, "fcm_dpo/delta": -0.008325421251356602, "fcm_dpo/margin": 78.22982788085938, "fcm_dpo/q_t": 0.42606019973754883, "grad_norm": 58.807376861572266, "learning_rate": 2.344485449913914e-07, "logits/chosen": -0.8697534203529358, "logits/rejected": -0.8527165651321411, "logps/chosen": -455.25299072265625, "logps/ref_chosen": -297.3590087890625, "logps/ref_rejected": -279.20196533203125, "logps/rejected": -515.3258056640625, "loss": 4.6817, "margin_dpo/margin_mean": 78.22982788085938, "margin_dpo/margin_std": 139.56898498535156, "step": 272 }, { "KL/chosen_KL_mean": -158.95762634277344, "KL/mean": -199.33731079101562, "KL/rejected_KL_mean": -239.71701049804688, "KL/std": 105.8140640258789, "epoch": 0.5717277486910994, "fcm_dpo/beta": 0.004161749500781298, "fcm_dpo/delta": 0.013979857787489891, "fcm_dpo/margin": 80.75938415527344, "fcm_dpo/q_t": 0.42355871200561523, "grad_norm": 85.51101684570312, "learning_rate": 2.3262175892620062e-07, "logits/chosen": -0.8654077053070068, "logits/rejected": -0.8781574964523315, "logps/chosen": -452.16339111328125, "logps/ref_chosen": -293.20574951171875, "logps/ref_rejected": -274.7646789550781, "logps/rejected": -514.481689453125, "loss": 4.6187, "margin_dpo/margin_mean": 80.75936889648438, "margin_dpo/margin_std": 134.3832244873047, "step": 273 }, { "KL/chosen_KL_mean": -156.28903198242188, "KL/mean": -206.505859375, "KL/rejected_KL_mean": -256.7226867675781, "KL/std": 118.53926086425781, "epoch": 0.5738219895287958, "fcm_dpo/beta": 0.004055425524711609, "fcm_dpo/delta": -0.060026153922080994, "fcm_dpo/margin": 100.43367767333984, "fcm_dpo/q_t": 0.40615060925483704, "grad_norm": 54.804561614990234, "learning_rate": 2.3079590480333827e-07, "logits/chosen": -0.8232536315917969, "logits/rejected": -0.7915986776351929, "logps/chosen": -426.84771728515625, "logps/ref_chosen": -270.55865478515625, "logps/ref_rejected": -239.47048950195312, "logps/rejected": -496.19317626953125, "loss": 4.3666, "margin_dpo/margin_mean": 100.43367767333984, "margin_dpo/margin_std": 134.73028564453125, "step": 274 }, { "KL/chosen_KL_mean": -144.99569702148438, "KL/mean": -192.33421325683594, "KL/rejected_KL_mean": -239.67274475097656, "KL/std": 111.46051025390625, "epoch": 0.5759162303664922, "fcm_dpo/beta": 0.003937349654734135, "fcm_dpo/delta": -0.02433641441166401, "fcm_dpo/margin": 94.67705535888672, "fcm_dpo/q_t": 0.4138934016227722, "grad_norm": 48.31379699707031, "learning_rate": 2.2897108053782e-07, "logits/chosen": -0.8615505695343018, "logits/rejected": -0.8430612683296204, "logps/chosen": -395.31494140625, "logps/ref_chosen": -250.31922912597656, "logps/ref_rejected": -249.3187255859375, "logps/rejected": -488.991455078125, "loss": 4.4327, "margin_dpo/margin_mean": 94.67705535888672, "margin_dpo/margin_std": 124.92469787597656, "step": 275 }, { "KL/chosen_KL_mean": -152.50244140625, "KL/mean": -198.66302490234375, "KL/rejected_KL_mean": -244.82359313964844, "KL/std": 118.38050842285156, "epoch": 0.5780104712041885, "fcm_dpo/beta": 0.003826065920293331, "fcm_dpo/delta": -0.004235291387885809, "fcm_dpo/margin": 92.32111358642578, "fcm_dpo/q_t": 0.4181724786758423, "grad_norm": 62.1019287109375, "learning_rate": 2.2714738398943308e-07, "logits/chosen": -0.926426887512207, "logits/rejected": -0.9037998914718628, "logps/chosen": -450.1335144042969, "logps/ref_chosen": -297.6310729980469, "logps/ref_rejected": -295.225830078125, "logps/rejected": -540.0493774414062, "loss": 4.5166, "margin_dpo/margin_mean": 92.32112121582031, "margin_dpo/margin_std": 130.4283905029297, "step": 276 }, { "KL/chosen_KL_mean": -176.41529846191406, "KL/mean": -207.66116333007812, "KL/rejected_KL_mean": -238.90696716308594, "KL/std": 117.95841979980469, "epoch": 0.5801047120418849, "fcm_dpo/beta": 0.0038804924115538597, "fcm_dpo/delta": 0.004320847801864147, "fcm_dpo/margin": 62.49165725708008, "fcm_dpo/q_t": 0.44412410259246826, "grad_norm": 92.11759948730469, "learning_rate": 2.2532491295748865e-07, "logits/chosen": -0.8612473011016846, "logits/rejected": -0.8609473705291748, "logps/chosen": -442.7757568359375, "logps/ref_chosen": -266.3604736328125, "logps/ref_rejected": -253.36767578125, "logps/rejected": -492.2746276855469, "loss": 4.9139, "margin_dpo/margin_mean": 62.49165344238281, "margin_dpo/margin_std": 134.01393127441406, "step": 277 }, { "KL/chosen_KL_mean": -189.94204711914062, "KL/mean": -214.97512817382812, "KL/rejected_KL_mean": -240.0082550048828, "KL/std": 124.60601806640625, "epoch": 0.5821989528795811, "fcm_dpo/beta": 0.003933742176741362, "fcm_dpo/delta": 0.009704717434942722, "fcm_dpo/margin": 50.06622314453125, "fcm_dpo/q_t": 0.45811718702316284, "grad_norm": 66.18705749511719, "learning_rate": 2.2350376517557726e-07, "logits/chosen": -0.8861530423164368, "logits/rejected": -0.8472384810447693, "logps/chosen": -457.34930419921875, "logps/ref_chosen": -267.40728759765625, "logps/ref_rejected": -229.5758514404297, "logps/rejected": -469.5841369628906, "loss": 5.1834, "margin_dpo/margin_mean": 50.06621551513672, "margin_dpo/margin_std": 154.64942932128906, "step": 278 }, { "KL/chosen_KL_mean": -173.27903747558594, "KL/mean": -222.08213806152344, "KL/rejected_KL_mean": -270.8852233886719, "KL/std": 131.5963897705078, "epoch": 0.5842931937172775, "fcm_dpo/beta": 0.0038174251094460487, "fcm_dpo/delta": -0.06448332965373993, "fcm_dpo/margin": 97.60619354248047, "fcm_dpo/q_t": 0.4153634309768677, "grad_norm": 117.3631591796875, "learning_rate": 2.2168403830632769e-07, "logits/chosen": -0.8119494915008545, "logits/rejected": -0.7960024476051331, "logps/chosen": -486.6468200683594, "logps/ref_chosen": -313.3677978515625, "logps/ref_rejected": -299.1744384765625, "logps/rejected": -570.0596923828125, "loss": 4.5018, "margin_dpo/margin_mean": 97.60618591308594, "margin_dpo/margin_std": 146.39854431152344, "step": 279 }, { "KL/chosen_KL_mean": -160.8683319091797, "KL/mean": -198.24644470214844, "KL/rejected_KL_mean": -235.62454223632812, "KL/std": 106.50706481933594, "epoch": 0.5863874345549738, "fcm_dpo/beta": 0.003840838558971882, "fcm_dpo/delta": 0.046352073550224304, "fcm_dpo/margin": 74.75621032714844, "fcm_dpo/q_t": 0.43327394127845764, "grad_norm": 51.44595718383789, "learning_rate": 2.1986582993616925e-07, "logits/chosen": -0.8889455199241638, "logits/rejected": -0.9012906551361084, "logps/chosen": -426.4241638183594, "logps/ref_chosen": -265.5558166503906, "logps/ref_rejected": -247.1573944091797, "logps/rejected": -482.781982421875, "loss": 4.73, "margin_dpo/margin_mean": 74.75621032714844, "margin_dpo/margin_std": 131.78170776367188, "step": 280 }, { "KL/chosen_KL_mean": -190.2058868408203, "KL/mean": -233.55860900878906, "KL/rejected_KL_mean": -276.91131591796875, "KL/std": 123.68882751464844, "epoch": 0.5884816753926702, "fcm_dpo/beta": 0.0038038038183003664, "fcm_dpo/delta": -0.03660136088728905, "fcm_dpo/margin": 86.70545959472656, "fcm_dpo/q_t": 0.42573872208595276, "grad_norm": 55.746944427490234, "learning_rate": 2.1804923757009882e-07, "logits/chosen": -0.8607216477394104, "logits/rejected": -0.8701354265213013, "logps/chosen": -485.5054931640625, "logps/ref_chosen": -295.2995910644531, "logps/ref_rejected": -293.80877685546875, "logps/rejected": -570.7200927734375, "loss": 4.6412, "margin_dpo/margin_mean": 86.70547485351562, "margin_dpo/margin_std": 145.6509552001953, "step": 281 }, { "KL/chosen_KL_mean": -186.29031372070312, "KL/mean": -226.07289123535156, "KL/rejected_KL_mean": -265.8554992675781, "KL/std": 120.34805297851562, "epoch": 0.5905759162303665, "fcm_dpo/beta": 0.003840283490717411, "fcm_dpo/delta": 0.045822735875844955, "fcm_dpo/margin": 79.565185546875, "fcm_dpo/q_t": 0.4292564392089844, "grad_norm": 84.25727081298828, "learning_rate": 2.1623435862645205e-07, "logits/chosen": -0.8561784625053406, "logits/rejected": -0.8544781804084778, "logps/chosen": -504.927490234375, "logps/ref_chosen": -318.63714599609375, "logps/ref_rejected": -273.5943603515625, "logps/rejected": -539.4498291015625, "loss": 4.6817, "margin_dpo/margin_mean": 79.565185546875, "margin_dpo/margin_std": 134.56092834472656, "step": 282 }, { "KL/chosen_KL_mean": -190.91969299316406, "KL/mean": -228.52896118164062, "KL/rejected_KL_mean": -266.13824462890625, "KL/std": 117.83441925048828, "epoch": 0.5926701570680628, "fcm_dpo/beta": 0.0039502596482634544, "fcm_dpo/delta": -0.001253846101462841, "fcm_dpo/margin": 75.21856689453125, "fcm_dpo/q_t": 0.43303558230400085, "grad_norm": 61.768646240234375, "learning_rate": 2.1442129043167873e-07, "logits/chosen": -0.8777590990066528, "logits/rejected": -0.8666056394577026, "logps/chosen": -445.58026123046875, "logps/ref_chosen": -254.66053771972656, "logps/ref_rejected": -236.8627166748047, "logps/rejected": -503.0010070800781, "loss": 4.7327, "margin_dpo/margin_mean": 75.21856689453125, "margin_dpo/margin_std": 134.68295288085938, "step": 283 }, { "KL/chosen_KL_mean": -199.0657501220703, "KL/mean": -239.81517028808594, "KL/rejected_KL_mean": -280.5645751953125, "KL/std": 125.28971099853516, "epoch": 0.5947643979057592, "fcm_dpo/beta": 0.0038177259266376495, "fcm_dpo/delta": -0.03368262201547623, "fcm_dpo/margin": 81.49883270263672, "fcm_dpo/q_t": 0.4287629723548889, "grad_norm": 65.97899627685547, "learning_rate": 2.1261013021512378e-07, "logits/chosen": -0.850072979927063, "logits/rejected": -0.8169124126434326, "logps/chosen": -472.4210205078125, "logps/ref_chosen": -273.355224609375, "logps/ref_rejected": -259.84759521484375, "logps/rejected": -540.4122314453125, "loss": 4.732, "margin_dpo/margin_mean": 81.49883270263672, "margin_dpo/margin_std": 149.6010284423828, "step": 284 }, { "KL/chosen_KL_mean": -206.79808044433594, "KL/mean": -235.41844177246094, "KL/rejected_KL_mean": -264.038818359375, "KL/std": 118.39861297607422, "epoch": 0.5968586387434555, "fcm_dpo/beta": 0.00384325347840786, "fcm_dpo/delta": 0.00668741762638092, "fcm_dpo/margin": 57.24073028564453, "fcm_dpo/q_t": 0.4500173330307007, "grad_norm": 73.33934783935547, "learning_rate": 2.1080097510381294e-07, "logits/chosen": -0.8570336103439331, "logits/rejected": -0.85038161277771, "logps/chosen": -516.6002807617188, "logps/ref_chosen": -309.8022155761719, "logps/ref_rejected": -279.11846923828125, "logps/rejected": -543.1572875976562, "loss": 5.0613, "margin_dpo/margin_mean": 57.24073028564453, "margin_dpo/margin_std": 148.24229431152344, "step": 285 }, { "KL/chosen_KL_mean": -200.6256866455078, "KL/mean": -238.26754760742188, "KL/rejected_KL_mean": -275.909423828125, "KL/std": 126.28824615478516, "epoch": 0.5989528795811518, "fcm_dpo/beta": 0.0037976307794451714, "fcm_dpo/delta": 0.0033064894378185272, "fcm_dpo/margin": 75.28372192382812, "fcm_dpo/q_t": 0.43621984124183655, "grad_norm": 100.08869934082031, "learning_rate": 2.089939221172446e-07, "logits/chosen": -0.8294232487678528, "logits/rejected": -0.8224241733551025, "logps/chosen": -472.09124755859375, "logps/ref_chosen": -271.4655456542969, "logps/ref_rejected": -279.531494140625, "logps/rejected": -555.44091796875, "loss": 4.7871, "margin_dpo/margin_mean": 75.28372192382812, "margin_dpo/margin_std": 145.39089965820312, "step": 286 }, { "KL/chosen_KL_mean": -205.491943359375, "KL/mean": -242.21157836914062, "KL/rejected_KL_mean": -278.93121337890625, "KL/std": 122.3520736694336, "epoch": 0.6010471204188481, "fcm_dpo/beta": 0.00381746469065547, "fcm_dpo/delta": -0.0022038668394088745, "fcm_dpo/margin": 73.43925476074219, "fcm_dpo/q_t": 0.4349508583545685, "grad_norm": 90.85375213623047, "learning_rate": 2.0718906816218595e-07, "logits/chosen": -0.853791356086731, "logits/rejected": -0.8370929956436157, "logps/chosen": -482.5851745605469, "logps/ref_chosen": -277.0932312011719, "logps/ref_rejected": -233.55599975585938, "logps/rejected": -512.4871826171875, "loss": 4.8313, "margin_dpo/margin_mean": 73.43925476074219, "margin_dpo/margin_std": 150.7272186279297, "step": 287 }, { "KL/chosen_KL_mean": -201.13818359375, "KL/mean": -238.72735595703125, "KL/rejected_KL_mean": -276.3164978027344, "KL/std": 125.49322509765625, "epoch": 0.6031413612565445, "fcm_dpo/beta": 0.00389170553535223, "fcm_dpo/delta": 0.011016139760613441, "fcm_dpo/margin": 75.17829132080078, "fcm_dpo/q_t": 0.43270570039749146, "grad_norm": 89.63787841796875, "learning_rate": 2.053865100274774e-07, "logits/chosen": -0.8803871273994446, "logits/rejected": -0.8856675028800964, "logps/chosen": -494.306396484375, "logps/ref_chosen": -293.1681823730469, "logps/ref_rejected": -263.4059143066406, "logps/rejected": -539.722412109375, "loss": 4.7564, "margin_dpo/margin_mean": 75.17829132080078, "margin_dpo/margin_std": 140.85711669921875, "step": 288 }, { "KL/chosen_KL_mean": -218.83831787109375, "KL/mean": -242.6974639892578, "KL/rejected_KL_mean": -266.5566101074219, "KL/std": 120.14765930175781, "epoch": 0.6052356020942409, "fcm_dpo/beta": 0.0038702820893377066, "fcm_dpo/delta": 0.004241670481860638, "fcm_dpo/margin": 47.71830368041992, "fcm_dpo/q_t": 0.45726847648620605, "grad_norm": 69.4831771850586, "learning_rate": 2.035863443788411e-07, "logits/chosen": -0.8640040159225464, "logits/rejected": -0.8451948165893555, "logps/chosen": -548.7957153320312, "logps/ref_chosen": -329.9574279785156, "logps/ref_rejected": -276.7565002441406, "logps/rejected": -543.3131103515625, "loss": 5.1264, "margin_dpo/margin_mean": 47.71830368041992, "margin_dpo/margin_std": 134.6345977783203, "step": 289 }, { "KL/chosen_KL_mean": -200.6356658935547, "KL/mean": -238.46449279785156, "KL/rejected_KL_mean": -276.29327392578125, "KL/std": 125.73330688476562, "epoch": 0.6073298429319371, "fcm_dpo/beta": 0.0037858528085052967, "fcm_dpo/delta": -0.024414364248514175, "fcm_dpo/margin": 75.65762329101562, "fcm_dpo/q_t": 0.4347863495349884, "grad_norm": 63.50288391113281, "learning_rate": 2.0178866775369774e-07, "logits/chosen": -0.8688110113143921, "logits/rejected": -0.8021400570869446, "logps/chosen": -525.3047485351562, "logps/ref_chosen": -324.6690673828125, "logps/ref_rejected": -311.8439636230469, "logps/rejected": -588.1372680664062, "loss": 4.8057, "margin_dpo/margin_mean": 75.65762329101562, "margin_dpo/margin_std": 148.36460876464844, "step": 290 }, { "KL/chosen_KL_mean": -192.0414276123047, "KL/mean": -240.45831298828125, "KL/rejected_KL_mean": -288.8752136230469, "KL/std": 125.70685577392578, "epoch": 0.6094240837696335, "fcm_dpo/beta": 0.0037065076176077127, "fcm_dpo/delta": -0.04703481122851372, "fcm_dpo/margin": 96.83379364013672, "fcm_dpo/q_t": 0.41666173934936523, "grad_norm": 62.59638214111328, "learning_rate": 1.9999357655598891e-07, "logits/chosen": -0.8442043662071228, "logits/rejected": -0.8339248299598694, "logps/chosen": -466.18548583984375, "logps/ref_chosen": -274.1440734863281, "logps/ref_rejected": -278.07208251953125, "logps/rejected": -566.9473266601562, "loss": 4.5137, "margin_dpo/margin_mean": 96.83379364013672, "margin_dpo/margin_std": 137.9904327392578, "step": 291 }, { "KL/chosen_KL_mean": -184.49508666992188, "KL/mean": -225.76486206054688, "KL/rejected_KL_mean": -267.03466796875, "KL/std": 128.42660522460938, "epoch": 0.6115183246073298, "fcm_dpo/beta": 0.003625905141234398, "fcm_dpo/delta": 0.005497709847986698, "fcm_dpo/margin": 82.53958129882812, "fcm_dpo/q_t": 0.4316753149032593, "grad_norm": 77.57347869873047, "learning_rate": 1.9820116705100775e-07, "logits/chosen": -0.8392362594604492, "logits/rejected": -0.8352776169776917, "logps/chosen": -443.8587951660156, "logps/ref_chosen": -259.3636779785156, "logps/ref_rejected": -279.30218505859375, "logps/rejected": -546.3368530273438, "loss": 4.734, "margin_dpo/margin_mean": 82.53958129882812, "margin_dpo/margin_std": 148.260986328125, "step": 292 }, { "KL/chosen_KL_mean": -198.5830078125, "KL/mean": -244.17982482910156, "KL/rejected_KL_mean": -289.776611328125, "KL/std": 132.66905212402344, "epoch": 0.6136125654450262, "fcm_dpo/beta": 0.003596771974116564, "fcm_dpo/delta": -0.03300413116812706, "fcm_dpo/margin": 91.19361877441406, "fcm_dpo/q_t": 0.4237741231918335, "grad_norm": 80.38446044921875, "learning_rate": 1.9641153536023642e-07, "logits/chosen": -0.9296824932098389, "logits/rejected": -0.8887497186660767, "logps/chosen": -502.35382080078125, "logps/ref_chosen": -303.77081298828125, "logps/ref_rejected": -270.07513427734375, "logps/rejected": -559.8517456054688, "loss": 4.5962, "margin_dpo/margin_mean": 91.19361877441406, "margin_dpo/margin_std": 136.49072265625, "step": 293 }, { "KL/chosen_KL_mean": -188.66331481933594, "KL/mean": -226.62930297851562, "KL/rejected_KL_mean": -264.59527587890625, "KL/std": 125.87104034423828, "epoch": 0.6157068062827226, "fcm_dpo/beta": 0.0035779972095042467, "fcm_dpo/delta": 0.004179379902780056, "fcm_dpo/margin": 75.93196105957031, "fcm_dpo/q_t": 0.4386540353298187, "grad_norm": 63.021671295166016, "learning_rate": 1.9462477745619106e-07, "logits/chosen": -0.837997317314148, "logits/rejected": -0.8435596823692322, "logps/chosen": -428.90167236328125, "logps/ref_chosen": -240.23831176757812, "logps/ref_rejected": -229.187744140625, "logps/rejected": -493.78302001953125, "loss": 4.8281, "margin_dpo/margin_mean": 75.93196105957031, "margin_dpo/margin_std": 151.36424255371094, "step": 294 }, { "KL/chosen_KL_mean": -184.72071838378906, "KL/mean": -222.63592529296875, "KL/rejected_KL_mean": -260.5511779785156, "KL/std": 110.61774444580078, "epoch": 0.6178010471204188, "fcm_dpo/beta": 0.0036432219203561544, "fcm_dpo/delta": 0.03179817646741867, "fcm_dpo/margin": 75.83045196533203, "fcm_dpo/q_t": 0.4368009567260742, "grad_norm": 66.03601837158203, "learning_rate": 1.928409891572757e-07, "logits/chosen": -0.8257983922958374, "logits/rejected": -0.8278652429580688, "logps/chosen": -435.73040771484375, "logps/ref_chosen": -251.00970458984375, "logps/ref_rejected": -244.15142822265625, "logps/rejected": -504.7026062011719, "loss": 4.7656, "margin_dpo/margin_mean": 75.83045196533203, "margin_dpo/margin_std": 136.33737182617188, "step": 295 }, { "KL/chosen_KL_mean": -192.8872528076172, "KL/mean": -244.12698364257812, "KL/rejected_KL_mean": -295.36669921875, "KL/std": 126.94731140136719, "epoch": 0.6198952879581152, "fcm_dpo/beta": 0.0035800463519990444, "fcm_dpo/delta": -0.06783003360033035, "fcm_dpo/margin": 102.47940826416016, "fcm_dpo/q_t": 0.41508862376213074, "grad_norm": 56.43666076660156, "learning_rate": 1.9106026612264315e-07, "logits/chosen": -0.8277036547660828, "logits/rejected": -0.7932828664779663, "logps/chosen": -486.76788330078125, "logps/ref_chosen": -293.880615234375, "logps/ref_rejected": -283.4175720214844, "logps/rejected": -578.7842407226562, "loss": 4.5237, "margin_dpo/margin_mean": 102.47940826416016, "margin_dpo/margin_std": 155.32875061035156, "step": 296 }, { "KL/chosen_KL_mean": -191.3452911376953, "KL/mean": -227.1857147216797, "KL/rejected_KL_mean": -263.0261535644531, "KL/std": 114.53092956542969, "epoch": 0.6219895287958115, "fcm_dpo/beta": 0.0034403197932988405, "fcm_dpo/delta": -0.010270388796925545, "fcm_dpo/margin": 71.68084716796875, "fcm_dpo/q_t": 0.4418776333332062, "grad_norm": 72.24175262451172, "learning_rate": 1.8928270384706582e-07, "logits/chosen": -0.9125591516494751, "logits/rejected": -0.9058645963668823, "logps/chosen": -480.8053894042969, "logps/ref_chosen": -289.4600830078125, "logps/ref_rejected": -283.69110107421875, "logps/rejected": -546.7172241210938, "loss": 4.8233, "margin_dpo/margin_mean": 71.68084716796875, "margin_dpo/margin_std": 126.96052551269531, "step": 297 }, { "KL/chosen_KL_mean": -180.29208374023438, "KL/mean": -228.24099731445312, "KL/rejected_KL_mean": -276.1899108886719, "KL/std": 122.46294403076172, "epoch": 0.6240837696335079, "fcm_dpo/beta": 0.0034188858699053526, "fcm_dpo/delta": -0.038795486092567444, "fcm_dpo/margin": 95.89783477783203, "fcm_dpo/q_t": 0.4239664673805237, "grad_norm": 59.304649353027344, "learning_rate": 1.875083976558136e-07, "logits/chosen": -0.8275625109672546, "logits/rejected": -0.8134176135063171, "logps/chosen": -486.80706787109375, "logps/ref_chosen": -306.5150146484375, "logps/ref_rejected": -280.6969909667969, "logps/rejected": -556.8869018554688, "loss": 4.6075, "margin_dpo/margin_mean": 95.89783477783203, "margin_dpo/margin_std": 146.3650360107422, "step": 298 }, { "KL/chosen_KL_mean": -206.77996826171875, "KL/mean": -242.58572387695312, "KL/rejected_KL_mean": -278.3914489746094, "KL/std": 122.42256164550781, "epoch": 0.6261780104712041, "fcm_dpo/beta": 0.003332537831738591, "fcm_dpo/delta": 0.0029420889914035797, "fcm_dpo/margin": 71.61149597167969, "fcm_dpo/q_t": 0.44390517473220825, "grad_norm": 48.41752624511719, "learning_rate": 1.8573744269954297e-07, "logits/chosen": -0.8100135326385498, "logits/rejected": -0.7927089333534241, "logps/chosen": -488.1437683105469, "logps/ref_chosen": -281.36376953125, "logps/ref_rejected": -270.39508056640625, "logps/rejected": -548.7865600585938, "loss": 4.8385, "margin_dpo/margin_mean": 71.61149597167969, "margin_dpo/margin_std": 119.70535278320312, "step": 299 }, { "KL/chosen_KL_mean": -212.83030700683594, "KL/mean": -249.0923614501953, "KL/rejected_KL_mean": -285.35443115234375, "KL/std": 132.61465454101562, "epoch": 0.6282722513089005, "fcm_dpo/beta": 0.00347234308719635, "fcm_dpo/delta": 0.05787147581577301, "fcm_dpo/margin": 72.52412414550781, "fcm_dpo/q_t": 0.4418582320213318, "grad_norm": 63.956878662109375, "learning_rate": 1.839699339491937e-07, "logits/chosen": -0.8397761583328247, "logits/rejected": -0.8176465034484863, "logps/chosen": -527.666015625, "logps/ref_chosen": -314.83575439453125, "logps/ref_rejected": -269.1154479980469, "logps/rejected": -554.4698486328125, "loss": 4.8907, "margin_dpo/margin_mean": 72.52412414550781, "margin_dpo/margin_std": 154.9835968017578, "step": 300 }, { "KL/chosen_KL_mean": -199.2548828125, "KL/mean": -235.5850372314453, "KL/rejected_KL_mean": -271.9151916503906, "KL/std": 123.72215270996094, "epoch": 0.6303664921465969, "fcm_dpo/beta": 0.0036008688621222973, "fcm_dpo/delta": 0.018657753244042397, "fcm_dpo/margin": 72.66029357910156, "fcm_dpo/q_t": 0.4398142397403717, "grad_norm": 48.57915115356445, "learning_rate": 1.8220596619089573e-07, "logits/chosen": -0.8675850629806519, "logits/rejected": -0.8598907589912415, "logps/chosen": -479.1494140625, "logps/ref_chosen": -279.89453125, "logps/ref_rejected": -271.6694641113281, "logps/rejected": -543.5846557617188, "loss": 4.8263, "margin_dpo/margin_mean": 72.6603012084961, "margin_dpo/margin_std": 138.31997680664062, "step": 301 }, { "KL/chosen_KL_mean": -193.967529296875, "KL/mean": -242.45010375976562, "KL/rejected_KL_mean": -290.93267822265625, "KL/std": 130.0727081298828, "epoch": 0.6324607329842932, "fcm_dpo/beta": 0.0035522691905498505, "fcm_dpo/delta": -0.026153553277254105, "fcm_dpo/margin": 96.96512603759766, "fcm_dpo/q_t": 0.41999223828315735, "grad_norm": 60.82284927368164, "learning_rate": 1.8044563402088682e-07, "logits/chosen": -0.8301497101783752, "logits/rejected": -0.8174096345901489, "logps/chosen": -465.2993469238281, "logps/ref_chosen": -271.3318176269531, "logps/ref_rejected": -256.5587158203125, "logps/rejected": -547.4913940429688, "loss": 4.5419, "margin_dpo/margin_mean": 96.96513366699219, "margin_dpo/margin_std": 141.36141967773438, "step": 302 }, { "KL/chosen_KL_mean": -196.64830017089844, "KL/mean": -238.0702362060547, "KL/rejected_KL_mean": -279.4921569824219, "KL/std": 119.1928939819336, "epoch": 0.6345549738219896, "fcm_dpo/beta": 0.003514134092256427, "fcm_dpo/delta": -0.009628637693822384, "fcm_dpo/margin": 82.84388732910156, "fcm_dpo/q_t": 0.4314088523387909, "grad_norm": 71.00035095214844, "learning_rate": 1.7868903184043885e-07, "logits/chosen": -0.813162624835968, "logits/rejected": -0.789709210395813, "logps/chosen": -501.5293273925781, "logps/ref_chosen": -304.88104248046875, "logps/ref_rejected": -269.063720703125, "logps/rejected": -548.555908203125, "loss": 4.6955, "margin_dpo/margin_mean": 82.84388732910156, "margin_dpo/margin_std": 131.7555389404297, "step": 303 }, { "KL/chosen_KL_mean": -209.07229614257812, "KL/mean": -253.82827758789062, "KL/rejected_KL_mean": -298.584228515625, "KL/std": 142.47499084472656, "epoch": 0.6366492146596858, "fcm_dpo/beta": 0.003528868081048131, "fcm_dpo/delta": 0.03502277284860611, "fcm_dpo/margin": 89.51190185546875, "fcm_dpo/q_t": 0.42928797006607056, "grad_norm": 62.49873733520508, "learning_rate": 1.7693625385079574e-07, "logits/chosen": -0.8146846294403076, "logits/rejected": -0.8206278681755066, "logps/chosen": -499.7832946777344, "logps/ref_chosen": -290.7109680175781, "logps/ref_rejected": -237.6885986328125, "logps/rejected": -536.2728271484375, "loss": 4.681, "margin_dpo/margin_mean": 89.51190185546875, "margin_dpo/margin_std": 158.9447784423828, "step": 304 }, { "KL/chosen_KL_mean": -181.38778686523438, "KL/mean": -248.25180053710938, "KL/rejected_KL_mean": -315.1158142089844, "KL/std": 137.24256896972656, "epoch": 0.6387434554973822, "fcm_dpo/beta": 0.0033798436634242535, "fcm_dpo/delta": -0.12808626890182495, "fcm_dpo/margin": 133.72805786132812, "fcm_dpo/q_t": 0.396328866481781, "grad_norm": 83.69937896728516, "learning_rate": 1.7518739404812155e-07, "logits/chosen": -0.864880383014679, "logits/rejected": -0.8337042927742004, "logps/chosen": -437.8717956542969, "logps/ref_chosen": -256.4839782714844, "logps/ref_rejected": -266.4063415527344, "logps/rejected": -581.5220947265625, "loss": 4.2357, "margin_dpo/margin_mean": 133.72805786132812, "margin_dpo/margin_std": 159.96530151367188, "step": 305 }, { "KL/chosen_KL_mean": -192.40550231933594, "KL/mean": -232.20289611816406, "KL/rejected_KL_mean": -272.00030517578125, "KL/std": 119.71340942382812, "epoch": 0.6408376963350786, "fcm_dpo/beta": 0.0031991961877793074, "fcm_dpo/delta": -0.0037949923425912857, "fcm_dpo/margin": 79.59476470947266, "fcm_dpo/q_t": 0.43988752365112305, "grad_norm": 56.67892837524414, "learning_rate": 1.7344254621846017e-07, "logits/chosen": -0.8629582524299622, "logits/rejected": -0.8421335220336914, "logps/chosen": -513.0547485351562, "logps/ref_chosen": -320.6492004394531, "logps/ref_rejected": -273.36773681640625, "logps/rejected": -545.3679809570312, "loss": 4.7771, "margin_dpo/margin_mean": 79.59476470947266, "margin_dpo/margin_std": 127.43058013916016, "step": 306 }, { "KL/chosen_KL_mean": -220.43991088867188, "KL/mean": -261.191650390625, "KL/rejected_KL_mean": -301.943359375, "KL/std": 134.89817810058594, "epoch": 0.6429319371727749, "fcm_dpo/beta": 0.0032339305616915226, "fcm_dpo/delta": 0.020983368158340454, "fcm_dpo/margin": 81.5034408569336, "fcm_dpo/q_t": 0.4373091161251068, "grad_norm": 75.93914031982422, "learning_rate": 1.717018039327053e-07, "logits/chosen": -0.7927024960517883, "logits/rejected": -0.8132081031799316, "logps/chosen": -499.89410400390625, "logps/ref_chosen": -279.4541931152344, "logps/ref_rejected": -240.3796844482422, "logps/rejected": -542.3230590820312, "loss": 4.7445, "margin_dpo/margin_mean": 81.5034408569336, "margin_dpo/margin_std": 128.4541015625, "step": 307 }, { "KL/chosen_KL_mean": -240.52467346191406, "KL/mean": -269.99462890625, "KL/rejected_KL_mean": -299.4645690917969, "KL/std": 134.7300262451172, "epoch": 0.6450261780104712, "fcm_dpo/beta": 0.0033486539032310247, "fcm_dpo/delta": 0.0349406898021698, "fcm_dpo/margin": 58.93989944458008, "fcm_dpo/q_t": 0.45389360189437866, "grad_norm": 67.70240020751953, "learning_rate": 1.699652605415828e-07, "logits/chosen": -0.8504436612129211, "logits/rejected": -0.8614956140518188, "logps/chosen": -537.123046875, "logps/ref_chosen": -296.598388671875, "logps/ref_rejected": -258.6953430175781, "logps/rejected": -558.1598510742188, "loss": 5.03, "margin_dpo/margin_mean": 58.93989944458008, "margin_dpo/margin_std": 142.15562438964844, "step": 308 }, { "KL/chosen_KL_mean": -226.738525390625, "KL/mean": -276.4004211425781, "KL/rejected_KL_mean": -326.06231689453125, "KL/std": 132.22732543945312, "epoch": 0.6471204188481675, "fcm_dpo/beta": 0.003401699708774686, "fcm_dpo/delta": 0.012193283997476101, "fcm_dpo/margin": 99.32380676269531, "fcm_dpo/q_t": 0.4232228100299835, "grad_norm": 67.57255554199219, "learning_rate": 1.6823300917064458e-07, "logits/chosen": -0.8523184061050415, "logits/rejected": -0.8619851469993591, "logps/chosen": -508.12664794921875, "logps/ref_chosen": -281.3881530761719, "logps/ref_rejected": -262.458740234375, "logps/rejected": -588.5210571289062, "loss": 4.5812, "margin_dpo/margin_mean": 99.32379150390625, "margin_dpo/margin_std": 157.58938598632812, "step": 309 }, { "KL/chosen_KL_mean": -215.33587646484375, "KL/mean": -265.105712890625, "KL/rejected_KL_mean": -314.87554931640625, "KL/std": 138.22256469726562, "epoch": 0.6492146596858639, "fcm_dpo/beta": 0.003396518062800169, "fcm_dpo/delta": -0.010896272957324982, "fcm_dpo/margin": 99.53965759277344, "fcm_dpo/q_t": 0.42174240946769714, "grad_norm": 65.10143280029297, "learning_rate": 1.6650514271527465e-07, "logits/chosen": -0.8462041616439819, "logits/rejected": -0.8201505541801453, "logps/chosen": -494.52313232421875, "logps/ref_chosen": -279.1872863769531, "logps/ref_rejected": -261.8279724121094, "logps/rejected": -576.7034912109375, "loss": 4.5525, "margin_dpo/margin_mean": 99.53965759277344, "margin_dpo/margin_std": 145.5522918701172, "step": 310 }, { "KL/chosen_KL_mean": -224.9881134033203, "KL/mean": -273.8262023925781, "KL/rejected_KL_mean": -322.664306640625, "KL/std": 144.56954956054688, "epoch": 0.6513089005235602, "fcm_dpo/beta": 0.0033532544039189816, "fcm_dpo/delta": -0.006738318130373955, "fcm_dpo/margin": 97.67620849609375, "fcm_dpo/q_t": 0.4256284534931183, "grad_norm": 72.03314208984375, "learning_rate": 1.647817538357072e-07, "logits/chosen": -0.8415148854255676, "logits/rejected": -0.8207394480705261, "logps/chosen": -496.3862609863281, "logps/ref_chosen": -271.39813232421875, "logps/ref_rejected": -266.12701416015625, "logps/rejected": -588.7913208007812, "loss": 4.6702, "margin_dpo/margin_mean": 97.67620849609375, "margin_dpo/margin_std": 168.5247039794922, "step": 311 }, { "KL/chosen_KL_mean": -223.4862823486328, "KL/mean": -267.6231689453125, "KL/rejected_KL_mean": -311.76007080078125, "KL/std": 139.96295166015625, "epoch": 0.6534031413612565, "fcm_dpo/beta": 0.003314582398161292, "fcm_dpo/delta": -0.019153205677866936, "fcm_dpo/margin": 88.27379608154297, "fcm_dpo/q_t": 0.43480026721954346, "grad_norm": 102.6431655883789, "learning_rate": 1.6306293495205755e-07, "logits/chosen": -0.8480397462844849, "logits/rejected": -0.8192158341407776, "logps/chosen": -505.8712463378906, "logps/ref_chosen": -282.3850402832031, "logps/ref_rejected": -246.35389709472656, "logps/rejected": -558.114013671875, "loss": 4.8413, "margin_dpo/margin_mean": 88.27378845214844, "margin_dpo/margin_std": 183.87039184570312, "step": 312 }, { "KL/chosen_KL_mean": -232.0310516357422, "KL/mean": -274.9604797363281, "KL/rejected_KL_mean": -317.889892578125, "KL/std": 148.04339599609375, "epoch": 0.6554973821989529, "fcm_dpo/beta": 0.0033472059294581413, "fcm_dpo/delta": -6.6105276346206665e-06, "fcm_dpo/margin": 85.85885620117188, "fcm_dpo/q_t": 0.437633752822876, "grad_norm": 60.15560531616211, "learning_rate": 1.6134877823936607e-07, "logits/chosen": -0.8933418393135071, "logits/rejected": -0.8773050308227539, "logps/chosen": -535.6619262695312, "logps/ref_chosen": -303.630859375, "logps/ref_rejected": -273.1156921386719, "logps/rejected": -591.005615234375, "loss": 4.8209, "margin_dpo/margin_mean": 85.85885620117188, "margin_dpo/margin_std": 170.81617736816406, "step": 313 }, { "KL/chosen_KL_mean": -220.53932189941406, "KL/mean": -266.6279296875, "KL/rejected_KL_mean": -312.716552734375, "KL/std": 129.1936492919922, "epoch": 0.6575916230366492, "fcm_dpo/beta": 0.00330774555914104, "fcm_dpo/delta": 0.004261258989572525, "fcm_dpo/margin": 92.1772232055664, "fcm_dpo/q_t": 0.4281480610370636, "grad_norm": 62.16466522216797, "learning_rate": 1.5963937562265522e-07, "logits/chosen": -0.8956902623176575, "logits/rejected": -0.8859307169914246, "logps/chosen": -522.8436279296875, "logps/ref_chosen": -302.3042907714844, "logps/ref_rejected": -273.6416015625, "logps/rejected": -586.358154296875, "loss": 4.6366, "margin_dpo/margin_mean": 92.1772232055664, "margin_dpo/margin_std": 139.03298950195312, "step": 314 }, { "KL/chosen_KL_mean": -219.21998596191406, "KL/mean": -273.90521240234375, "KL/rejected_KL_mean": -328.5904541015625, "KL/std": 133.44537353515625, "epoch": 0.6596858638743456, "fcm_dpo/beta": 0.003281098324805498, "fcm_dpo/delta": -0.02703600376844406, "fcm_dpo/margin": 109.3704833984375, "fcm_dpo/q_t": 0.41646474599838257, "grad_norm": 83.7893295288086, "learning_rate": 1.5793481877199943e-07, "logits/chosen": -0.8796918392181396, "logits/rejected": -0.8612196445465088, "logps/chosen": -521.9492797851562, "logps/ref_chosen": -302.729248046875, "logps/ref_rejected": -270.26910400390625, "logps/rejected": -598.859619140625, "loss": 4.4913, "margin_dpo/margin_mean": 109.37049102783203, "margin_dpo/margin_std": 153.03680419921875, "step": 315 }, { "KL/chosen_KL_mean": -230.92750549316406, "KL/mean": -278.43603515625, "KL/rejected_KL_mean": -325.944580078125, "KL/std": 137.39004516601562, "epoch": 0.6617801047120419, "fcm_dpo/beta": 0.003247649408876896, "fcm_dpo/delta": 0.0010586827993392944, "fcm_dpo/margin": 95.01707458496094, "fcm_dpo/q_t": 0.42847543954849243, "grad_norm": 57.49567413330078, "learning_rate": 1.562351990976095e-07, "logits/chosen": -0.899001955986023, "logits/rejected": -0.8805509209632874, "logps/chosen": -541.4981689453125, "logps/ref_chosen": -310.5706481933594, "logps/ref_rejected": -272.9354553222656, "logps/rejected": -598.8800659179688, "loss": 4.6608, "margin_dpo/margin_mean": 95.0170669555664, "margin_dpo/margin_std": 155.4547882080078, "step": 316 }, { "KL/chosen_KL_mean": -204.7845001220703, "KL/mean": -249.20443725585938, "KL/rejected_KL_mean": -293.6243591308594, "KL/std": 122.82341003417969, "epoch": 0.6638743455497382, "fcm_dpo/beta": 0.003311583073809743, "fcm_dpo/delta": 0.05764345824718475, "fcm_dpo/margin": 88.83987426757812, "fcm_dpo/q_t": 0.4310056269168854, "grad_norm": 67.7326431274414, "learning_rate": 1.5454060774493065e-07, "logits/chosen": -0.8872113823890686, "logits/rejected": -0.8559964895248413, "logps/chosen": -458.68487548828125, "logps/ref_chosen": -253.90036010742188, "logps/ref_rejected": -218.74078369140625, "logps/rejected": -512.3651123046875, "loss": 4.6495, "margin_dpo/margin_mean": 88.83988189697266, "margin_dpo/margin_std": 131.25308227539062, "step": 317 }, { "KL/chosen_KL_mean": -213.66708374023438, "KL/mean": -270.26947021484375, "KL/rejected_KL_mean": -326.87188720703125, "KL/std": 135.44711303710938, "epoch": 0.6659685863874345, "fcm_dpo/beta": 0.003273027017712593, "fcm_dpo/delta": -0.044196274131536484, "fcm_dpo/margin": 113.20478820800781, "fcm_dpo/q_t": 0.4135770797729492, "grad_norm": 75.95118713378906, "learning_rate": 1.5285113558975427e-07, "logits/chosen": -0.9056518077850342, "logits/rejected": -0.8769103288650513, "logps/chosen": -484.4899597167969, "logps/ref_chosen": -270.8228759765625, "logps/ref_rejected": -255.30972290039062, "logps/rejected": -582.1815795898438, "loss": 4.4434, "margin_dpo/margin_mean": 113.20478820800781, "margin_dpo/margin_std": 149.13006591796875, "step": 318 }, { "KL/chosen_KL_mean": -231.12075805664062, "KL/mean": -284.0456237792969, "KL/rejected_KL_mean": -336.970458984375, "KL/std": 135.72552490234375, "epoch": 0.6680628272251309, "fcm_dpo/beta": 0.0032491928432136774, "fcm_dpo/delta": 0.006155331619083881, "fcm_dpo/margin": 105.84971618652344, "fcm_dpo/q_t": 0.4182014465332031, "grad_norm": 47.199466705322266, "learning_rate": 1.5116687323334464e-07, "logits/chosen": -0.8861814141273499, "logits/rejected": -0.8539734482765198, "logps/chosen": -532.1235961914062, "logps/ref_chosen": -301.0028076171875, "logps/ref_rejected": -242.39002990722656, "logps/rejected": -579.3605346679688, "loss": 4.4814, "margin_dpo/margin_mean": 105.8497085571289, "margin_dpo/margin_std": 133.62188720703125, "step": 319 }, { "KL/chosen_KL_mean": -230.05516052246094, "KL/mean": -276.97979736328125, "KL/rejected_KL_mean": -323.9044189453125, "KL/std": 141.29583740234375, "epoch": 0.6701570680628273, "fcm_dpo/beta": 0.003300814889371395, "fcm_dpo/delta": 0.019661886617541313, "fcm_dpo/margin": 93.84925842285156, "fcm_dpo/q_t": 0.42925646901130676, "grad_norm": 68.85742950439453, "learning_rate": 1.4948791099758052e-07, "logits/chosen": -0.855350911617279, "logits/rejected": -0.8459681272506714, "logps/chosen": -533.677734375, "logps/ref_chosen": -303.6225891113281, "logps/ref_rejected": -280.85174560546875, "logps/rejected": -604.7561645507812, "loss": 4.6843, "margin_dpo/margin_mean": 93.84925842285156, "margin_dpo/margin_std": 161.83663940429688, "step": 320 }, { "KL/chosen_KL_mean": -245.5141143798828, "KL/mean": -282.2264099121094, "KL/rejected_KL_mean": -318.938720703125, "KL/std": 139.87318420410156, "epoch": 0.6722513089005235, "fcm_dpo/beta": 0.0033561254385858774, "fcm_dpo/delta": 0.00966968759894371, "fcm_dpo/margin": 73.42462158203125, "fcm_dpo/q_t": 0.4437595009803772, "grad_norm": 83.1728286743164, "learning_rate": 1.478143389201113e-07, "logits/chosen": -0.8710657954216003, "logits/rejected": -0.8343586325645447, "logps/chosen": -534.4999389648438, "logps/ref_chosen": -288.98583984375, "logps/ref_rejected": -241.1822052001953, "logps/rejected": -560.1209106445312, "loss": 4.9071, "margin_dpo/margin_mean": 73.42462158203125, "margin_dpo/margin_std": 156.1352996826172, "step": 321 }, { "KL/chosen_KL_mean": -258.0967712402344, "KL/mean": -300.7222595214844, "KL/rejected_KL_mean": -343.3477783203125, "KL/std": 140.47586059570312, "epoch": 0.6743455497382199, "fcm_dpo/beta": 0.0033736673649400473, "fcm_dpo/delta": -0.00021159928292036057, "fcm_dpo/margin": 85.25101470947266, "fcm_dpo/q_t": 0.4336232841014862, "grad_norm": 78.91681671142578, "learning_rate": 1.461462467495284e-07, "logits/chosen": -0.9282336831092834, "logits/rejected": -0.8913758397102356, "logps/chosen": -566.6402587890625, "logps/ref_chosen": -308.54345703125, "logps/ref_rejected": -269.7995910644531, "logps/rejected": -613.1473388671875, "loss": 4.7848, "margin_dpo/margin_mean": 85.25101470947266, "margin_dpo/margin_std": 164.19068908691406, "step": 322 }, { "KL/chosen_KL_mean": -243.4940185546875, "KL/mean": -277.5121765136719, "KL/rejected_KL_mean": -311.53033447265625, "KL/std": 131.02301025390625, "epoch": 0.6764397905759162, "fcm_dpo/beta": 0.0033622642513364553, "fcm_dpo/delta": -0.0014576911926269531, "fcm_dpo/margin": 68.03628540039062, "fcm_dpo/q_t": 0.4480839669704437, "grad_norm": 83.17462158203125, "learning_rate": 1.4448372394055246e-07, "logits/chosen": -0.8875012993812561, "logits/rejected": -0.8782394528388977, "logps/chosen": -525.9876708984375, "logps/ref_chosen": -282.49365234375, "logps/ref_rejected": -227.7105255126953, "logps/rejected": -539.2408447265625, "loss": 4.963, "margin_dpo/margin_mean": 68.03628540039062, "margin_dpo/margin_std": 155.5357208251953, "step": 323 }, { "KL/chosen_KL_mean": -221.84173583984375, "KL/mean": -273.66180419921875, "KL/rejected_KL_mean": -325.4819030761719, "KL/std": 140.5488739013672, "epoch": 0.6785340314136126, "fcm_dpo/beta": 0.0032545297872275114, "fcm_dpo/delta": -0.03785352408885956, "fcm_dpo/margin": 103.64017486572266, "fcm_dpo/q_t": 0.4208952486515045, "grad_norm": 73.04549407958984, "learning_rate": 1.428268596492364e-07, "logits/chosen": -0.8201410174369812, "logits/rejected": -0.8138597011566162, "logps/chosen": -461.18011474609375, "logps/ref_chosen": -239.33836364746094, "logps/ref_rejected": -230.53775024414062, "logps/rejected": -556.0196533203125, "loss": 4.5327, "margin_dpo/margin_mean": 103.64017486572266, "margin_dpo/margin_std": 140.1855010986328, "step": 324 }, { "KL/chosen_KL_mean": -243.33395385742188, "KL/mean": -289.827392578125, "KL/rejected_KL_mean": -336.32080078125, "KL/std": 146.8050537109375, "epoch": 0.680628272251309, "fcm_dpo/beta": 0.003209222573786974, "fcm_dpo/delta": -0.013333482667803764, "fcm_dpo/margin": 92.98680114746094, "fcm_dpo/q_t": 0.4324532747268677, "grad_norm": 74.4027328491211, "learning_rate": 1.4117574272818386e-07, "logits/chosen": -0.8348155617713928, "logits/rejected": -0.8183823227882385, "logps/chosen": -523.962890625, "logps/ref_chosen": -280.62896728515625, "logps/ref_rejected": -270.5085754394531, "logps/rejected": -606.829345703125, "loss": 4.7618, "margin_dpo/margin_mean": 92.98680114746094, "margin_dpo/margin_std": 173.72450256347656, "step": 325 }, { "KL/chosen_KL_mean": -229.41400146484375, "KL/mean": -278.8251647949219, "KL/rejected_KL_mean": -328.236328125, "KL/std": 138.20025634765625, "epoch": 0.6827225130890052, "fcm_dpo/beta": 0.003266323357820511, "fcm_dpo/delta": 0.0035640615969896317, "fcm_dpo/margin": 98.82231903076172, "fcm_dpo/q_t": 0.4255218505859375, "grad_norm": 87.20018768310547, "learning_rate": 1.3953046172178413e-07, "logits/chosen": -0.9498153328895569, "logits/rejected": -0.940109372138977, "logps/chosen": -470.401123046875, "logps/ref_chosen": -240.9871368408203, "logps/ref_rejected": -261.0238342285156, "logps/rejected": -589.2601318359375, "loss": 4.6263, "margin_dpo/margin_mean": 98.82231903076172, "margin_dpo/margin_std": 158.7273406982422, "step": 326 }, { "KL/chosen_KL_mean": -219.74449157714844, "KL/mean": -273.5055236816406, "KL/rejected_KL_mean": -327.26654052734375, "KL/std": 132.50523376464844, "epoch": 0.6848167539267016, "fcm_dpo/beta": 0.0032268206123262644, "fcm_dpo/delta": -0.021940872073173523, "fcm_dpo/margin": 107.5220718383789, "fcm_dpo/q_t": 0.42012837529182434, "grad_norm": 73.7422103881836, "learning_rate": 1.3789110486146468e-07, "logits/chosen": -0.8853604793548584, "logits/rejected": -0.8608740568161011, "logps/chosen": -499.2645263671875, "logps/ref_chosen": -279.52001953125, "logps/ref_rejected": -269.51824951171875, "logps/rejected": -596.7847900390625, "loss": 4.5388, "margin_dpo/margin_mean": 107.5220718383789, "margin_dpo/margin_std": 155.50799560546875, "step": 327 }, { "KL/chosen_KL_mean": -241.4763641357422, "KL/mean": -290.4848327636719, "KL/rejected_KL_mean": -339.4931945800781, "KL/std": 136.66537475585938, "epoch": 0.6869109947643979, "fcm_dpo/beta": 0.0031155140604823828, "fcm_dpo/delta": -0.02170320227742195, "fcm_dpo/margin": 98.01687622070312, "fcm_dpo/q_t": 0.42870888113975525, "grad_norm": 52.95525360107422, "learning_rate": 1.362577600609588e-07, "logits/chosen": -0.8597935438156128, "logits/rejected": -0.8578972220420837, "logps/chosen": -542.5098266601562, "logps/ref_chosen": -301.033447265625, "logps/ref_rejected": -284.2101135253906, "logps/rejected": -623.7033081054688, "loss": 4.6399, "margin_dpo/margin_mean": 98.01687622070312, "margin_dpo/margin_std": 149.0442352294922, "step": 328 }, { "KL/chosen_KL_mean": -256.7224426269531, "KL/mean": -309.82830810546875, "KL/rejected_KL_mean": -362.9341125488281, "KL/std": 172.28172302246094, "epoch": 0.6890052356020943, "fcm_dpo/beta": 0.0030725565738976, "fcm_dpo/delta": -0.036677245050668716, "fcm_dpo/margin": 106.211669921875, "fcm_dpo/q_t": 0.42879265546798706, "grad_norm": 60.5069694519043, "learning_rate": 1.3463051491159093e-07, "logits/chosen": -0.8657263517379761, "logits/rejected": -0.8348018527030945, "logps/chosen": -576.7113037109375, "logps/ref_chosen": -319.9888610839844, "logps/ref_rejected": -307.5588684082031, "logps/rejected": -670.4929809570312, "loss": 4.6954, "margin_dpo/margin_mean": 106.21165466308594, "margin_dpo/margin_std": 189.08535766601562, "step": 329 }, { "KL/chosen_KL_mean": -247.13705444335938, "KL/mean": -297.3825378417969, "KL/rejected_KL_mean": -347.6280517578125, "KL/std": 152.5355224609375, "epoch": 0.6910994764397905, "fcm_dpo/beta": 0.00300287501886487, "fcm_dpo/delta": -0.02195155620574951, "fcm_dpo/margin": 100.49103546142578, "fcm_dpo/q_t": 0.42996516823768616, "grad_norm": 74.84465789794922, "learning_rate": 1.3300945667758012e-07, "logits/chosen": -0.8545703291893005, "logits/rejected": -0.861172080039978, "logps/chosen": -548.2518310546875, "logps/ref_chosen": -301.11474609375, "logps/ref_rejected": -299.673095703125, "logps/rejected": -647.3011474609375, "loss": 4.6614, "margin_dpo/margin_mean": 100.49103546142578, "margin_dpo/margin_std": 156.20176696777344, "step": 330 }, { "KL/chosen_KL_mean": -239.11048889160156, "KL/mean": -285.6376647949219, "KL/rejected_KL_mean": -332.16485595703125, "KL/std": 151.9279022216797, "epoch": 0.6931937172774869, "fcm_dpo/beta": 0.003041815711185336, "fcm_dpo/delta": 0.047408707439899445, "fcm_dpo/margin": 93.05432891845703, "fcm_dpo/q_t": 0.4363221824169159, "grad_norm": 69.54380798339844, "learning_rate": 1.3139467229135998e-07, "logits/chosen": -0.8673312664031982, "logits/rejected": -0.8451822996139526, "logps/chosen": -516.7019653320312, "logps/ref_chosen": -277.59149169921875, "logps/ref_rejected": -256.025634765625, "logps/rejected": -588.1904296875, "loss": 4.7977, "margin_dpo/margin_mean": 93.05433654785156, "margin_dpo/margin_std": 185.36294555664062, "step": 331 }, { "KL/chosen_KL_mean": -246.1035919189453, "KL/mean": -294.6551208496094, "KL/rejected_KL_mean": -343.2066650390625, "KL/std": 146.26475524902344, "epoch": 0.6952879581151833, "fcm_dpo/beta": 0.003083170158788562, "fcm_dpo/delta": 0.0028741867281496525, "fcm_dpo/margin": 97.10308837890625, "fcm_dpo/q_t": 0.4316597878932953, "grad_norm": 78.57029724121094, "learning_rate": 1.2978624834891626e-07, "logits/chosen": -0.8808923363685608, "logits/rejected": -0.8539749383926392, "logps/chosen": -516.0772705078125, "logps/ref_chosen": -269.97369384765625, "logps/ref_rejected": -235.03164672851562, "logps/rejected": -578.2383422851562, "loss": 4.7816, "margin_dpo/margin_mean": 97.10308837890625, "margin_dpo/margin_std": 189.04632568359375, "step": 332 }, { "KL/chosen_KL_mean": -242.9016571044922, "KL/mean": -284.91748046875, "KL/rejected_KL_mean": -326.9332580566406, "KL/std": 140.8754119873047, "epoch": 0.6973821989528796, "fcm_dpo/beta": 0.0031835411209613085, "fcm_dpo/delta": 0.03588256612420082, "fcm_dpo/margin": 84.03166961669922, "fcm_dpo/q_t": 0.4383671283721924, "grad_norm": 63.44430160522461, "learning_rate": 1.281842711051438e-07, "logits/chosen": -0.9540477991104126, "logits/rejected": -0.9159016609191895, "logps/chosen": -539.6646728515625, "logps/ref_chosen": -296.76300048828125, "logps/ref_rejected": -265.97991943359375, "logps/rejected": -592.9132080078125, "loss": 4.808, "margin_dpo/margin_mean": 84.03166198730469, "margin_dpo/margin_std": 161.39566040039062, "step": 333 }, { "KL/chosen_KL_mean": -236.5258026123047, "KL/mean": -283.31243896484375, "KL/rejected_KL_mean": -330.0990295410156, "KL/std": 143.63352966308594, "epoch": 0.6994764397905759, "fcm_dpo/beta": 0.0031920385081321, "fcm_dpo/delta": -0.011491209268569946, "fcm_dpo/margin": 93.57325744628906, "fcm_dpo/q_t": 0.43058937788009644, "grad_norm": 82.48439025878906, "learning_rate": 1.2658882646922033e-07, "logits/chosen": -0.8545200824737549, "logits/rejected": -0.8265820145606995, "logps/chosen": -537.5625610351562, "logps/ref_chosen": -301.0367431640625, "logps/ref_rejected": -268.87652587890625, "logps/rejected": -598.9755859375, "loss": 4.7247, "margin_dpo/margin_mean": 93.57325744628906, "margin_dpo/margin_std": 160.36260986328125, "step": 334 }, { "KL/chosen_KL_mean": -257.5293273925781, "KL/mean": -303.568359375, "KL/rejected_KL_mean": -349.6073913574219, "KL/std": 131.11956787109375, "epoch": 0.7015706806282722, "fcm_dpo/beta": 0.003155500628054142, "fcm_dpo/delta": 0.008357154205441475, "fcm_dpo/margin": 92.07801818847656, "fcm_dpo/q_t": 0.43331050872802734, "grad_norm": 83.25331115722656, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.8444750308990479, "logits/rejected": -0.8382730484008789, "logps/chosen": -533.662109375, "logps/ref_chosen": -276.13275146484375, "logps/ref_rejected": -243.44203186035156, "logps/rejected": -593.04931640625, "loss": 4.7586, "margin_dpo/margin_mean": 92.07801818847656, "margin_dpo/margin_std": 167.62142944335938, "step": 335 }, { "KL/chosen_KL_mean": -253.9612579345703, "KL/mean": -301.3621520996094, "KL/rejected_KL_mean": -348.76300048828125, "KL/std": 167.3890380859375, "epoch": 0.7036649214659686, "fcm_dpo/beta": 0.0031551639549434185, "fcm_dpo/delta": -0.024496689438819885, "fcm_dpo/margin": 94.80172729492188, "fcm_dpo/q_t": 0.43416231870651245, "grad_norm": 78.45794677734375, "learning_rate": 1.2341787690142435e-07, "logits/chosen": -0.8579354286193848, "logits/rejected": -0.7981801629066467, "logps/chosen": -500.22393798828125, "logps/ref_chosen": -246.2626495361328, "logps/ref_rejected": -261.0617980957031, "logps/rejected": -609.8248901367188, "loss": 4.7654, "margin_dpo/margin_mean": 94.80172729492188, "margin_dpo/margin_std": 181.49172973632812, "step": 336 }, { "KL/chosen_KL_mean": -232.01927185058594, "KL/mean": -287.64947509765625, "KL/rejected_KL_mean": -343.2796630859375, "KL/std": 140.15916442871094, "epoch": 0.7057591623036649, "fcm_dpo/beta": 0.0031573735177516937, "fcm_dpo/delta": -0.02179955132305622, "fcm_dpo/margin": 111.26039123535156, "fcm_dpo/q_t": 0.418662965297699, "grad_norm": 85.16871643066406, "learning_rate": 1.2184254201795363e-07, "logits/chosen": -0.873991072177887, "logits/rejected": -0.8448648452758789, "logps/chosen": -499.0130615234375, "logps/ref_chosen": -266.9937744140625, "logps/ref_rejected": -253.015625, "logps/rejected": -596.2952270507812, "loss": 4.5231, "margin_dpo/margin_mean": 111.26039123535156, "margin_dpo/margin_std": 158.54148864746094, "step": 337 }, { "KL/chosen_KL_mean": -224.66920471191406, "KL/mean": -278.88800048828125, "KL/rejected_KL_mean": -333.1067810058594, "KL/std": 147.4876708984375, "epoch": 0.7078534031413612, "fcm_dpo/beta": 0.003021388780325651, "fcm_dpo/delta": -0.016518374904990196, "fcm_dpo/margin": 108.43759155273438, "fcm_dpo/q_t": 0.4243887960910797, "grad_norm": 113.49219512939453, "learning_rate": 1.202740798300168e-07, "logits/chosen": -0.874689519405365, "logits/rejected": -0.8554161190986633, "logps/chosen": -501.26177978515625, "logps/ref_chosen": -276.5925598144531, "logps/ref_rejected": -233.979248046875, "logps/rejected": -567.0860595703125, "loss": 4.5851, "margin_dpo/margin_mean": 108.4375991821289, "margin_dpo/margin_std": 163.255859375, "step": 338 }, { "KL/chosen_KL_mean": -231.06105041503906, "KL/mean": -289.5965576171875, "KL/rejected_KL_mean": -348.13214111328125, "KL/std": 148.72731018066406, "epoch": 0.7099476439790576, "fcm_dpo/beta": 0.0030209606047719717, "fcm_dpo/delta": -0.028637029230594635, "fcm_dpo/margin": 117.071044921875, "fcm_dpo/q_t": 0.4185601770877838, "grad_norm": 68.01354217529297, "learning_rate": 1.1871257444948096e-07, "logits/chosen": -0.8962268233299255, "logits/rejected": -0.880439817905426, "logps/chosen": -534.5888061523438, "logps/ref_chosen": -303.5277404785156, "logps/ref_rejected": -283.11676025390625, "logps/rejected": -631.2489013671875, "loss": 4.5252, "margin_dpo/margin_mean": 117.071044921875, "margin_dpo/margin_std": 171.6200408935547, "step": 339 }, { "KL/chosen_KL_mean": -246.8760986328125, "KL/mean": -289.718017578125, "KL/rejected_KL_mean": -332.5599670410156, "KL/std": 158.76173400878906, "epoch": 0.7120418848167539, "fcm_dpo/beta": 0.0029858858324587345, "fcm_dpo/delta": 0.03210965916514397, "fcm_dpo/margin": 85.68387603759766, "fcm_dpo/q_t": 0.44048047065734863, "grad_norm": 123.83653259277344, "learning_rate": 1.1715810961514072e-07, "logits/chosen": -0.8333358764648438, "logits/rejected": -0.8279672861099243, "logps/chosen": -508.40185546875, "logps/ref_chosen": -261.5257568359375, "logps/ref_rejected": -259.39862060546875, "logps/rejected": -591.9585571289062, "loss": 4.9309, "margin_dpo/margin_mean": 85.68387603759766, "margin_dpo/margin_std": 192.60916137695312, "step": 340 }, { "KL/chosen_KL_mean": -277.3525390625, "KL/mean": -318.425048828125, "KL/rejected_KL_mean": -359.4975891113281, "KL/std": 176.37754821777344, "epoch": 0.7141361256544503, "fcm_dpo/beta": 0.00305312592536211, "fcm_dpo/delta": 0.0065925233066082, "fcm_dpo/margin": 82.14506530761719, "fcm_dpo/q_t": 0.44349971413612366, "grad_norm": 63.160316467285156, "learning_rate": 1.1561076868822755e-07, "logits/chosen": -0.8536667227745056, "logits/rejected": -0.8266552686691284, "logps/chosen": -593.2560424804688, "logps/ref_chosen": -315.903564453125, "logps/ref_rejected": -308.02392578125, "logps/rejected": -667.521484375, "loss": 4.985, "margin_dpo/margin_mean": 82.14506530761719, "margin_dpo/margin_std": 199.24632263183594, "step": 341 }, { "KL/chosen_KL_mean": -232.24563598632812, "KL/mean": -278.6511535644531, "KL/rejected_KL_mean": -325.056640625, "KL/std": 141.9452667236328, "epoch": 0.7162303664921466, "fcm_dpo/beta": 0.0031279418617486954, "fcm_dpo/delta": 0.03749464452266693, "fcm_dpo/margin": 92.81096649169922, "fcm_dpo/q_t": 0.4330783486366272, "grad_norm": 101.11497497558594, "learning_rate": 1.1407063464793965e-07, "logits/chosen": -0.8642405271530151, "logits/rejected": -0.8597648739814758, "logps/chosen": -501.42431640625, "logps/ref_chosen": -269.17864990234375, "logps/ref_rejected": -260.8977355957031, "logps/rejected": -585.954345703125, "loss": 4.7053, "margin_dpo/margin_mean": 92.81096649169922, "margin_dpo/margin_std": 154.14793395996094, "step": 342 }, { "KL/chosen_KL_mean": -244.72955322265625, "KL/mean": -282.2915954589844, "KL/rejected_KL_mean": -319.8536376953125, "KL/std": 139.9641876220703, "epoch": 0.7183246073298429, "fcm_dpo/beta": 0.003217648947611451, "fcm_dpo/delta": 0.0457039512693882, "fcm_dpo/margin": 75.12411499023438, "fcm_dpo/q_t": 0.4435161352157593, "grad_norm": 102.95614624023438, "learning_rate": 1.125377900869913e-07, "logits/chosen": -0.859501302242279, "logits/rejected": -0.8335797786712646, "logps/chosen": -555.4495239257812, "logps/ref_chosen": -310.719970703125, "logps/ref_rejected": -263.5224914550781, "logps/rejected": -583.3761596679688, "loss": 4.8569, "margin_dpo/margin_mean": 75.12410736083984, "margin_dpo/margin_std": 147.29708862304688, "step": 343 }, { "KL/chosen_KL_mean": -240.10861206054688, "KL/mean": -283.4324951171875, "KL/rejected_KL_mean": -326.75640869140625, "KL/std": 134.30642700195312, "epoch": 0.7204188481675393, "fcm_dpo/beta": 0.0033485074527561665, "fcm_dpo/delta": 0.016562025994062424, "fcm_dpo/margin": 86.64774322509766, "fcm_dpo/q_t": 0.43293195962905884, "grad_norm": 90.93418884277344, "learning_rate": 1.110123172071844e-07, "logits/chosen": -0.8508941531181335, "logits/rejected": -0.8322650790214539, "logps/chosen": -541.9085693359375, "logps/ref_chosen": -301.7999267578125, "logps/ref_rejected": -257.9061584472656, "logps/rejected": -584.6625366210938, "loss": 4.7599, "margin_dpo/margin_mean": 86.64774322509766, "margin_dpo/margin_std": 159.52081298828125, "step": 344 }, { "KL/chosen_KL_mean": -227.98085021972656, "KL/mean": -269.60870361328125, "KL/rejected_KL_mean": -311.236572265625, "KL/std": 137.84278869628906, "epoch": 0.7225130890052356, "fcm_dpo/beta": 0.003270794404670596, "fcm_dpo/delta": -0.04042273014783859, "fcm_dpo/margin": 83.2557373046875, "fcm_dpo/q_t": 0.4363884925842285, "grad_norm": 61.989376068115234, "learning_rate": 1.09494297815e-07, "logits/chosen": -0.8617357611656189, "logits/rejected": -0.86259526014328, "logps/chosen": -510.9992980957031, "logps/ref_chosen": -283.0184326171875, "logps/ref_rejected": -266.8457336425781, "logps/rejected": -578.082275390625, "loss": 4.772, "margin_dpo/margin_mean": 83.25572967529297, "margin_dpo/margin_std": 138.2306365966797, "step": 345 }, { "KL/chosen_KL_mean": -242.52352905273438, "KL/mean": -294.254638671875, "KL/rejected_KL_mean": -345.9857177734375, "KL/std": 155.80648803710938, "epoch": 0.724607329842932, "fcm_dpo/beta": 0.0030984547920525074, "fcm_dpo/delta": -0.06594164669513702, "fcm_dpo/margin": 103.4621810913086, "fcm_dpo/q_t": 0.4260784685611725, "grad_norm": 74.91835021972656, "learning_rate": 1.0798381331721107e-07, "logits/chosen": -0.9356507062911987, "logits/rejected": -0.883700966835022, "logps/chosen": -510.96478271484375, "logps/ref_chosen": -268.44122314453125, "logps/ref_rejected": -227.8225860595703, "logps/rejected": -573.8082885742188, "loss": 4.6604, "margin_dpo/margin_mean": 103.4621810913086, "margin_dpo/margin_std": 165.5200958251953, "step": 346 }, { "KL/chosen_KL_mean": -224.31581115722656, "KL/mean": -266.2325439453125, "KL/rejected_KL_mean": -308.1493225097656, "KL/std": 134.7196044921875, "epoch": 0.7267015706806282, "fcm_dpo/beta": 0.003096605185419321, "fcm_dpo/delta": 0.03434152901172638, "fcm_dpo/margin": 83.83348846435547, "fcm_dpo/q_t": 0.43899720907211304, "grad_norm": 97.60405731201172, "learning_rate": 1.0648094471651722e-07, "logits/chosen": -0.7862703800201416, "logits/rejected": -0.8122860193252563, "logps/chosen": -498.0193786621094, "logps/ref_chosen": -273.70355224609375, "logps/ref_rejected": -243.65521240234375, "logps/rejected": -551.8045043945312, "loss": 4.805, "margin_dpo/margin_mean": 83.83348846435547, "margin_dpo/margin_std": 149.16348266601562, "step": 347 }, { "KL/chosen_KL_mean": -224.01858520507812, "KL/mean": -263.1257019042969, "KL/rejected_KL_mean": -302.2328186035156, "KL/std": 143.7584686279297, "epoch": 0.7287958115183246, "fcm_dpo/beta": 0.0031398944556713104, "fcm_dpo/delta": 0.013123849406838417, "fcm_dpo/margin": 78.21422576904297, "fcm_dpo/q_t": 0.44496262073516846, "grad_norm": 60.390602111816406, "learning_rate": 1.0498577260720048e-07, "logits/chosen": -0.8914543390274048, "logits/rejected": -0.8751494884490967, "logps/chosen": -509.6600036621094, "logps/ref_chosen": -285.64141845703125, "logps/ref_rejected": -265.6270446777344, "logps/rejected": -567.85986328125, "loss": 4.8846, "margin_dpo/margin_mean": 78.2142333984375, "margin_dpo/margin_std": 162.25692749023438, "step": 348 }, { "KL/chosen_KL_mean": -223.03997802734375, "KL/mean": -281.030517578125, "KL/rejected_KL_mean": -339.02105712890625, "KL/std": 154.15811157226562, "epoch": 0.7308900523560209, "fcm_dpo/beta": 0.003127102041617036, "fcm_dpo/delta": -0.03881394863128662, "fcm_dpo/margin": 115.98108673095703, "fcm_dpo/q_t": 0.41518571972846985, "grad_norm": 71.31893157958984, "learning_rate": 1.0349837717080347e-07, "logits/chosen": -0.8437027335166931, "logits/rejected": -0.8291211724281311, "logps/chosen": -551.3574829101562, "logps/ref_chosen": -328.3175048828125, "logps/ref_rejected": -292.37872314453125, "logps/rejected": -631.3997802734375, "loss": 4.5358, "margin_dpo/margin_mean": 115.98108673095703, "margin_dpo/margin_std": 175.9039306640625, "step": 349 }, { "KL/chosen_KL_mean": -236.86434936523438, "KL/mean": -285.975830078125, "KL/rejected_KL_mean": -335.08734130859375, "KL/std": 149.2736053466797, "epoch": 0.7329842931937173, "fcm_dpo/beta": 0.003120012581348419, "fcm_dpo/delta": 0.012096697464585304, "fcm_dpo/margin": 98.22299194335938, "fcm_dpo/q_t": 0.42966753244400024, "grad_norm": 92.44560241699219, "learning_rate": 1.0201883817182949e-07, "logits/chosen": -0.8361937999725342, "logits/rejected": -0.8472439646720886, "logps/chosen": -529.6690063476562, "logps/ref_chosen": -292.8046569824219, "logps/ref_rejected": -250.35504150390625, "logps/rejected": -585.4423217773438, "loss": 4.6988, "margin_dpo/margin_mean": 98.2229995727539, "margin_dpo/margin_std": 171.5211944580078, "step": 350 }, { "KL/chosen_KL_mean": -232.821044921875, "KL/mean": -262.088134765625, "KL/rejected_KL_mean": -291.3552551269531, "KL/std": 137.2724609375, "epoch": 0.7350785340314137, "fcm_dpo/beta": 0.0031031551770865917, "fcm_dpo/delta": 0.0010303156450390816, "fcm_dpo/margin": 58.534210205078125, "fcm_dpo/q_t": 0.45783162117004395, "grad_norm": 53.25884246826172, "learning_rate": 1.0054723495346482e-07, "logits/chosen": -0.8983533382415771, "logits/rejected": -0.8776514530181885, "logps/chosen": -544.7100219726562, "logps/ref_chosen": -311.8890380859375, "logps/ref_rejected": -263.59033203125, "logps/rejected": -554.945556640625, "loss": 5.1347, "margin_dpo/margin_mean": 58.534210205078125, "margin_dpo/margin_std": 169.1748046875, "step": 351 }, { "KL/chosen_KL_mean": -216.96815490722656, "KL/mean": -275.911865234375, "KL/rejected_KL_mean": -334.8555603027344, "KL/std": 151.17433166503906, "epoch": 0.7371727748691099, "fcm_dpo/beta": 0.0030314219184219837, "fcm_dpo/delta": -0.031058115884661674, "fcm_dpo/margin": 117.88742065429688, "fcm_dpo/q_t": 0.4165811240673065, "grad_norm": 51.545772552490234, "learning_rate": 9.908364643332398e-08, "logits/chosen": -0.8226331472396851, "logits/rejected": -0.7918906211853027, "logps/chosen": -471.87603759765625, "logps/ref_chosen": -254.9078826904297, "logps/ref_rejected": -257.1688232421875, "logps/rejected": -592.0244140625, "loss": 4.5028, "margin_dpo/margin_mean": 117.88742065429688, "margin_dpo/margin_std": 162.4287872314453, "step": 352 }, { "KL/chosen_KL_mean": -233.33163452148438, "KL/mean": -281.92626953125, "KL/rejected_KL_mean": -330.5209045410156, "KL/std": 144.54786682128906, "epoch": 0.7392670157068063, "fcm_dpo/beta": 0.002987043932080269, "fcm_dpo/delta": -0.027638321742415428, "fcm_dpo/margin": 97.18927764892578, "fcm_dpo/q_t": 0.43258699774742126, "grad_norm": 134.96693420410156, "learning_rate": 9.76281510992176e-08, "logits/chosen": -0.8499235510826111, "logits/rejected": -0.8441857695579529, "logps/chosen": -503.70770263671875, "logps/ref_chosen": -270.3760681152344, "logps/ref_rejected": -264.65234375, "logps/rejected": -595.1732177734375, "loss": 4.721, "margin_dpo/margin_mean": 97.18927001953125, "margin_dpo/margin_std": 162.46739196777344, "step": 353 }, { "KL/chosen_KL_mean": -230.3787841796875, "KL/mean": -269.59588623046875, "KL/rejected_KL_mean": -308.8129577636719, "KL/std": 154.2929229736328, "epoch": 0.7413612565445026, "fcm_dpo/beta": 0.003026010002940893, "fcm_dpo/delta": 0.03974125534296036, "fcm_dpo/margin": 78.43419647216797, "fcm_dpo/q_t": 0.4459616541862488, "grad_norm": 68.59556579589844, "learning_rate": 9.618082700494318e-08, "logits/chosen": -0.838015079498291, "logits/rejected": -0.8697560429573059, "logps/chosen": -488.02734375, "logps/ref_chosen": -257.6485595703125, "logps/ref_rejected": -246.94203186035156, "logps/rejected": -555.7550048828125, "loss": 4.9336, "margin_dpo/margin_mean": 78.43421173095703, "margin_dpo/margin_std": 173.0082550048828, "step": 354 }, { "KL/chosen_KL_mean": -236.45423889160156, "KL/mean": -288.6083984375, "KL/rejected_KL_mean": -340.7625732421875, "KL/std": 159.89952087402344, "epoch": 0.743455497382199, "fcm_dpo/beta": 0.0030326605774462223, "fcm_dpo/delta": 0.006990807130932808, "fcm_dpo/margin": 104.30831909179688, "fcm_dpo/q_t": 0.42733824253082275, "grad_norm": 103.6706771850586, "learning_rate": 9.474175176609956e-08, "logits/chosen": -0.8978556394577026, "logits/rejected": -0.8931483030319214, "logps/chosen": -529.8076171875, "logps/ref_chosen": -293.35333251953125, "logps/ref_rejected": -275.6051940917969, "logps/rejected": -616.3677368164062, "loss": 4.7378, "margin_dpo/margin_mean": 104.30831909179688, "margin_dpo/margin_std": 191.57334899902344, "step": 355 }, { "KL/chosen_KL_mean": -218.8563690185547, "KL/mean": -256.1012268066406, "KL/rejected_KL_mean": -293.34613037109375, "KL/std": 133.4541473388672, "epoch": 0.7455497382198953, "fcm_dpo/beta": 0.0030662843491882086, "fcm_dpo/delta": 0.013114407658576965, "fcm_dpo/margin": 74.48975372314453, "fcm_dpo/q_t": 0.4463121294975281, "grad_norm": 82.64666748046875, "learning_rate": 9.331100255592436e-08, "logits/chosen": -0.8024091124534607, "logits/rejected": -0.8281019926071167, "logps/chosen": -423.11187744140625, "logps/ref_chosen": -204.25550842285156, "logps/ref_rejected": -213.467529296875, "logps/rejected": -506.8136291503906, "loss": 4.8785, "margin_dpo/margin_mean": 74.48975372314453, "margin_dpo/margin_std": 138.68128967285156, "step": 356 }, { "KL/chosen_KL_mean": -243.69976806640625, "KL/mean": -292.585205078125, "KL/rejected_KL_mean": -341.47064208984375, "KL/std": 151.95843505859375, "epoch": 0.7476439790575916, "fcm_dpo/beta": 0.0030811114702373743, "fcm_dpo/delta": -0.00847620703279972, "fcm_dpo/margin": 97.77091979980469, "fcm_dpo/q_t": 0.4313168227672577, "grad_norm": 77.11827850341797, "learning_rate": 9.18886561011557e-08, "logits/chosen": -0.7848935723304749, "logits/rejected": -0.7811943888664246, "logps/chosen": -510.0703125, "logps/ref_chosen": -266.3705749511719, "logps/ref_rejected": -239.04490661621094, "logps/rejected": -580.5155639648438, "loss": 4.7273, "margin_dpo/margin_mean": 97.77091979980469, "margin_dpo/margin_std": 172.3494873046875, "step": 357 }, { "KL/chosen_KL_mean": -218.75701904296875, "KL/mean": -275.1843566894531, "KL/rejected_KL_mean": -331.6116943359375, "KL/std": 139.84754943847656, "epoch": 0.749738219895288, "fcm_dpo/beta": 0.0030688412953168154, "fcm_dpo/delta": -0.01982707343995571, "fcm_dpo/margin": 112.85469818115234, "fcm_dpo/q_t": 0.41943082213401794, "grad_norm": 91.5672836303711, "learning_rate": 9.047478867791731e-08, "logits/chosen": -0.8832120895385742, "logits/rejected": -0.8587401509284973, "logps/chosen": -517.9044799804688, "logps/ref_chosen": -299.1474609375, "logps/ref_rejected": -257.2531433105469, "logps/rejected": -588.8648071289062, "loss": 4.5521, "margin_dpo/margin_mean": 112.85469818115234, "margin_dpo/margin_std": 167.56106567382812, "step": 358 }, { "KL/chosen_KL_mean": -230.01956176757812, "KL/mean": -279.5022888183594, "KL/rejected_KL_mean": -328.9850158691406, "KL/std": 144.20054626464844, "epoch": 0.7518324607329843, "fcm_dpo/beta": 0.0030328300781548023, "fcm_dpo/delta": -0.007977485656738281, "fcm_dpo/margin": 98.9654312133789, "fcm_dpo/q_t": 0.4301950931549072, "grad_norm": 80.08256530761719, "learning_rate": 8.906947610762825e-08, "logits/chosen": -0.8496133089065552, "logits/rejected": -0.8573009967803955, "logps/chosen": -533.0174560546875, "logps/ref_chosen": -302.99786376953125, "logps/ref_rejected": -260.4137268066406, "logps/rejected": -589.3987426757812, "loss": 4.7148, "margin_dpo/margin_mean": 98.9654312133789, "margin_dpo/margin_std": 172.8538055419922, "step": 359 }, { "KL/chosen_KL_mean": -234.53375244140625, "KL/mean": -281.4641418457031, "KL/rejected_KL_mean": -328.39453125, "KL/std": 157.81326293945312, "epoch": 0.7539267015706806, "fcm_dpo/beta": 0.0029506399296224117, "fcm_dpo/delta": -0.040261730551719666, "fcm_dpo/margin": 93.8608169555664, "fcm_dpo/q_t": 0.4368816614151001, "grad_norm": 72.47200775146484, "learning_rate": 8.76727937529367e-08, "logits/chosen": -0.8603860139846802, "logits/rejected": -0.8375499844551086, "logps/chosen": -544.1452026367188, "logps/ref_chosen": -309.6114501953125, "logps/ref_rejected": -256.64031982421875, "logps/rejected": -585.0348510742188, "loss": 4.8678, "margin_dpo/margin_mean": 93.8608169555664, "margin_dpo/margin_std": 188.05604553222656, "step": 360 }, { "KL/chosen_KL_mean": -199.42823791503906, "KL/mean": -260.2237548828125, "KL/rejected_KL_mean": -321.0193176269531, "KL/std": 154.24195861816406, "epoch": 0.7560209424083769, "fcm_dpo/beta": 0.0029223968740552664, "fcm_dpo/delta": -0.005785183981060982, "fcm_dpo/margin": 121.59111022949219, "fcm_dpo/q_t": 0.4192415475845337, "grad_norm": 47.21493911743164, "learning_rate": 8.628481651367875e-08, "logits/chosen": -0.7954362630844116, "logits/rejected": -0.7759599089622498, "logps/chosen": -462.8079833984375, "logps/ref_chosen": -263.3797607421875, "logps/ref_rejected": -271.18157958984375, "logps/rejected": -592.2008666992188, "loss": 4.5488, "margin_dpo/margin_mean": 121.59109497070312, "margin_dpo/margin_std": 189.3097381591797, "step": 361 }, { "KL/chosen_KL_mean": -222.83444213867188, "KL/mean": -270.36297607421875, "KL/rejected_KL_mean": -317.89154052734375, "KL/std": 126.81389617919922, "epoch": 0.7581151832460733, "fcm_dpo/beta": 0.0029204203747212887, "fcm_dpo/delta": 0.0328575074672699, "fcm_dpo/margin": 95.0571060180664, "fcm_dpo/q_t": 0.43568283319473267, "grad_norm": 72.48114776611328, "learning_rate": 8.490561882286135e-08, "logits/chosen": -0.8405733704566956, "logits/rejected": -0.822902262210846, "logps/chosen": -526.0927734375, "logps/ref_chosen": -303.2583923339844, "logps/ref_rejected": -243.22891235351562, "logps/rejected": -561.1204223632812, "loss": 4.7016, "margin_dpo/margin_mean": 95.05711364746094, "margin_dpo/margin_std": 146.4384765625, "step": 362 }, { "KL/chosen_KL_mean": -226.85987854003906, "KL/mean": -280.93182373046875, "KL/rejected_KL_mean": -335.0037536621094, "KL/std": 155.6280517578125, "epoch": 0.7602094240837697, "fcm_dpo/beta": 0.0029316996224224567, "fcm_dpo/delta": -0.0422758013010025, "fcm_dpo/margin": 108.14390563964844, "fcm_dpo/q_t": 0.427703857421875, "grad_norm": 81.47772216796875, "learning_rate": 8.353527464267104e-08, "logits/chosen": -0.8603069186210632, "logits/rejected": -0.8154016137123108, "logps/chosen": -530.2070922851562, "logps/ref_chosen": -303.34722900390625, "logps/ref_rejected": -262.05419921875, "logps/rejected": -597.0579223632812, "loss": 4.6611, "margin_dpo/margin_mean": 108.14390563964844, "margin_dpo/margin_std": 177.33998107910156, "step": 363 }, { "KL/chosen_KL_mean": -257.1740417480469, "KL/mean": -305.13641357421875, "KL/rejected_KL_mean": -353.0987243652344, "KL/std": 165.08631896972656, "epoch": 0.762303664921466, "fcm_dpo/beta": 0.0028783483430743217, "fcm_dpo/delta": 0.0030604167841374874, "fcm_dpo/margin": 95.92469787597656, "fcm_dpo/q_t": 0.43887603282928467, "grad_norm": 71.76043701171875, "learning_rate": 8.217385746050742e-08, "logits/chosen": -0.8362113833427429, "logits/rejected": -0.8350046873092651, "logps/chosen": -542.7178344726562, "logps/ref_chosen": -285.54376220703125, "logps/ref_rejected": -284.84619140625, "logps/rejected": -637.9449462890625, "loss": 4.8453, "margin_dpo/margin_mean": 95.92469024658203, "margin_dpo/margin_std": 200.49720764160156, "step": 364 }, { "KL/chosen_KL_mean": -238.0579376220703, "KL/mean": -284.226806640625, "KL/rejected_KL_mean": -330.3956604003906, "KL/std": 158.41650390625, "epoch": 0.7643979057591623, "fcm_dpo/beta": 0.0029075169004499912, "fcm_dpo/delta": 0.017943935468792915, "fcm_dpo/margin": 92.33773803710938, "fcm_dpo/q_t": 0.43834882974624634, "grad_norm": 86.59075164794922, "learning_rate": 8.082144028504231e-08, "logits/chosen": -0.8370366096496582, "logits/rejected": -0.835037112236023, "logps/chosen": -512.8457641601562, "logps/ref_chosen": -274.7878112792969, "logps/ref_rejected": -256.5738220214844, "logps/rejected": -586.9694213867188, "loss": 4.8049, "margin_dpo/margin_mean": 92.33773803710938, "margin_dpo/margin_std": 173.68531799316406, "step": 365 }, { "KL/chosen_KL_mean": -224.99862670898438, "KL/mean": -273.92193603515625, "KL/rejected_KL_mean": -322.84521484375, "KL/std": 138.87074279785156, "epoch": 0.7664921465968586, "fcm_dpo/beta": 0.002992146648466587, "fcm_dpo/delta": 0.0403238870203495, "fcm_dpo/margin": 97.8465576171875, "fcm_dpo/q_t": 0.43188464641571045, "grad_norm": 69.0442886352539, "learning_rate": 7.947809564230445e-08, "logits/chosen": -0.8111105561256409, "logits/rejected": -0.8164764046669006, "logps/chosen": -511.64825439453125, "logps/ref_chosen": -286.6496276855469, "logps/ref_rejected": -251.97140502929688, "logps/rejected": -574.816650390625, "loss": 4.7061, "margin_dpo/margin_mean": 97.8465576171875, "margin_dpo/margin_std": 166.59800720214844, "step": 366 }, { "KL/chosen_KL_mean": -225.3688201904297, "KL/mean": -279.73223876953125, "KL/rejected_KL_mean": -334.09564208984375, "KL/std": 145.0337371826172, "epoch": 0.768586387434555, "fcm_dpo/beta": 0.002969947410747409, "fcm_dpo/delta": -0.022818509489297867, "fcm_dpo/margin": 108.726806640625, "fcm_dpo/q_t": 0.42565882205963135, "grad_norm": 117.63641357421875, "learning_rate": 7.814389557179016e-08, "logits/chosen": -0.834640383720398, "logits/rejected": -0.8060700297355652, "logps/chosen": -527.3137817382812, "logps/ref_chosen": -301.9449768066406, "logps/ref_rejected": -265.5677185058594, "logps/rejected": -599.663330078125, "loss": 4.6168, "margin_dpo/margin_mean": 108.72679901123047, "margin_dpo/margin_std": 170.5776824951172, "step": 367 }, { "KL/chosen_KL_mean": -209.96340942382812, "KL/mean": -275.3994445800781, "KL/rejected_KL_mean": -340.83544921875, "KL/std": 145.23342895507812, "epoch": 0.7706806282722513, "fcm_dpo/beta": 0.0029076621867716312, "fcm_dpo/delta": -0.05346748232841492, "fcm_dpo/margin": 130.87203979492188, "fcm_dpo/q_t": 0.4109519422054291, "grad_norm": 84.33660888671875, "learning_rate": 7.681891162260015e-08, "logits/chosen": -0.8131657838821411, "logits/rejected": -0.8158013820648193, "logps/chosen": -504.5899353027344, "logps/ref_chosen": -294.62652587890625, "logps/ref_rejected": -258.7628479003906, "logps/rejected": -599.5983276367188, "loss": 4.3871, "margin_dpo/margin_mean": 130.87203979492188, "margin_dpo/margin_std": 156.81239318847656, "step": 368 }, { "KL/chosen_KL_mean": -225.32662963867188, "KL/mean": -272.04248046875, "KL/rejected_KL_mean": -318.75836181640625, "KL/std": 135.3495635986328, "epoch": 0.7727748691099476, "fcm_dpo/beta": 0.002850665245205164, "fcm_dpo/delta": 0.019876670092344284, "fcm_dpo/margin": 93.43172454833984, "fcm_dpo/q_t": 0.43696144223213196, "grad_norm": 69.02677154541016, "learning_rate": 7.550321484960251e-08, "logits/chosen": -0.8901256322860718, "logits/rejected": -0.8731386065483093, "logps/chosen": -507.8323974609375, "logps/ref_chosen": -282.5057373046875, "logps/ref_rejected": -266.41607666015625, "logps/rejected": -585.1744384765625, "loss": 4.7213, "margin_dpo/margin_mean": 93.43172454833984, "margin_dpo/margin_std": 143.53280639648438, "step": 369 }, { "KL/chosen_KL_mean": -217.40830993652344, "KL/mean": -273.412109375, "KL/rejected_KL_mean": -329.4158630371094, "KL/std": 149.15206909179688, "epoch": 0.774869109947644, "fcm_dpo/beta": 0.002886255271732807, "fcm_dpo/delta": -0.009045472368597984, "fcm_dpo/margin": 112.00758361816406, "fcm_dpo/q_t": 0.4255586564540863, "grad_norm": 58.84016799926758, "learning_rate": 7.419687580962222e-08, "logits/chosen": -0.8674653172492981, "logits/rejected": -0.8908781409263611, "logps/chosen": -468.41473388671875, "logps/ref_chosen": -251.00640869140625, "logps/ref_rejected": -238.12542724609375, "logps/rejected": -567.541259765625, "loss": 4.6418, "margin_dpo/margin_mean": 112.00758361816406, "margin_dpo/margin_std": 187.31283569335938, "step": 370 }, { "KL/chosen_KL_mean": -239.92498779296875, "KL/mean": -274.255126953125, "KL/rejected_KL_mean": -308.5852966308594, "KL/std": 141.81640625, "epoch": 0.7769633507853403, "fcm_dpo/beta": 0.0028966227546334267, "fcm_dpo/delta": 0.007331667002290487, "fcm_dpo/margin": 68.66031646728516, "fcm_dpo/q_t": 0.4531916677951813, "grad_norm": 86.75391387939453, "learning_rate": 7.289996455765748e-08, "logits/chosen": -0.8269072771072388, "logits/rejected": -0.8060893416404724, "logps/chosen": -536.5841064453125, "logps/ref_chosen": -296.6591491699219, "logps/ref_rejected": -251.14675903320312, "logps/rejected": -559.7320556640625, "loss": 4.9955, "margin_dpo/margin_mean": 68.66031646728516, "margin_dpo/margin_std": 151.32308959960938, "step": 371 }, { "KL/chosen_KL_mean": -232.09654235839844, "KL/mean": -288.6462097167969, "KL/rejected_KL_mean": -345.19586181640625, "KL/std": 156.81259155273438, "epoch": 0.7790575916230367, "fcm_dpo/beta": 0.0028610192239284515, "fcm_dpo/delta": 0.002475677989423275, "fcm_dpo/margin": 113.09928894042969, "fcm_dpo/q_t": 0.4260843098163605, "grad_norm": 66.6779556274414, "learning_rate": 7.161255064312283e-08, "logits/chosen": -0.7979535460472107, "logits/rejected": -0.783908486366272, "logps/chosen": -563.4679565429688, "logps/ref_chosen": -331.3714599609375, "logps/ref_rejected": -285.56805419921875, "logps/rejected": -630.763916015625, "loss": 4.6371, "margin_dpo/margin_mean": 113.09928894042969, "margin_dpo/margin_std": 179.6610107421875, "step": 372 }, { "KL/chosen_KL_mean": -219.7714385986328, "KL/mean": -270.5410461425781, "KL/rejected_KL_mean": -321.31060791015625, "KL/std": 135.90286254882812, "epoch": 0.7811518324607329, "fcm_dpo/beta": 0.002902068430557847, "fcm_dpo/delta": 0.01581491529941559, "fcm_dpo/margin": 101.53917694091797, "fcm_dpo/q_t": 0.4303041994571686, "grad_norm": 62.56334686279297, "learning_rate": 7.033470310611945e-08, "logits/chosen": -0.8850827217102051, "logits/rejected": -0.8500010371208191, "logps/chosen": -541.7144165039062, "logps/ref_chosen": -321.9429931640625, "logps/ref_rejected": -271.2288513183594, "logps/rejected": -592.53955078125, "loss": 4.6699, "margin_dpo/margin_mean": 101.53917694091797, "margin_dpo/margin_std": 156.63763427734375, "step": 373 }, { "KL/chosen_KL_mean": -232.57269287109375, "KL/mean": -277.5435791015625, "KL/rejected_KL_mean": -322.514404296875, "KL/std": 155.8538818359375, "epoch": 0.7832460732984293, "fcm_dpo/beta": 0.0028754910454154015, "fcm_dpo/delta": -0.02850567176938057, "fcm_dpo/margin": 89.94171142578125, "fcm_dpo/q_t": 0.43950414657592773, "grad_norm": 53.21077346801758, "learning_rate": 6.906649047373245e-08, "logits/chosen": -0.883882462978363, "logits/rejected": -0.8781501650810242, "logps/chosen": -551.7412719726562, "logps/ref_chosen": -319.1685485839844, "logps/ref_rejected": -284.6263732910156, "logps/rejected": -607.1407470703125, "loss": 4.8022, "margin_dpo/margin_mean": 89.94171142578125, "margin_dpo/margin_std": 160.33763122558594, "step": 374 }, { "KL/chosen_KL_mean": -242.3306121826172, "KL/mean": -279.80706787109375, "KL/rejected_KL_mean": -317.2834777832031, "KL/std": 129.10975646972656, "epoch": 0.7853403141361257, "fcm_dpo/beta": 0.0028963014483451843, "fcm_dpo/delta": 0.03359980508685112, "fcm_dpo/margin": 74.95287322998047, "fcm_dpo/q_t": 0.4491427540779114, "grad_norm": 106.22050476074219, "learning_rate": 6.780798075635675e-08, "logits/chosen": -0.8753329515457153, "logits/rejected": -0.8530557155609131, "logps/chosen": -557.2063598632812, "logps/ref_chosen": -314.87579345703125, "logps/ref_rejected": -259.1965026855469, "logps/rejected": -576.47998046875, "loss": 4.9298, "margin_dpo/margin_mean": 74.95287322998047, "margin_dpo/margin_std": 153.65232849121094, "step": 375 }, { "KL/chosen_KL_mean": -236.53941345214844, "KL/mean": -286.677734375, "KL/rejected_KL_mean": -336.8160400390625, "KL/std": 152.2532958984375, "epoch": 0.787434554973822, "fcm_dpo/beta": 0.003028448671102524, "fcm_dpo/delta": 0.047010768204927444, "fcm_dpo/margin": 100.27667236328125, "fcm_dpo/q_t": 0.4310030937194824, "grad_norm": 92.208251953125, "learning_rate": 6.655924144404906e-08, "logits/chosen": -0.8538745045661926, "logits/rejected": -0.8539252281188965, "logps/chosen": -524.212646484375, "logps/ref_chosen": -287.6732482910156, "logps/ref_rejected": -256.6697082519531, "logps/rejected": -593.4857788085938, "loss": 4.7153, "margin_dpo/margin_mean": 100.27667236328125, "margin_dpo/margin_std": 179.34959411621094, "step": 376 }, { "KL/chosen_KL_mean": -250.41348266601562, "KL/mean": -287.26507568359375, "KL/rejected_KL_mean": -324.11663818359375, "KL/std": 149.1571807861328, "epoch": 0.7895287958115184, "fcm_dpo/beta": 0.003050284693017602, "fcm_dpo/delta": 0.007031728979200125, "fcm_dpo/margin": 73.703125, "fcm_dpo/q_t": 0.4477493166923523, "grad_norm": 58.59090042114258, "learning_rate": 6.532033950290885e-08, "logits/chosen": -0.8329107761383057, "logits/rejected": -0.8246560096740723, "logps/chosen": -555.6749267578125, "logps/ref_chosen": -305.261474609375, "logps/ref_rejected": -271.8887023925781, "logps/rejected": -596.00537109375, "loss": 4.9961, "margin_dpo/margin_mean": 73.703125, "margin_dpo/margin_std": 176.01364135742188, "step": 377 }, { "KL/chosen_KL_mean": -239.36642456054688, "KL/mean": -284.56353759765625, "KL/rejected_KL_mean": -329.7606506347656, "KL/std": 154.51390075683594, "epoch": 0.7916230366492146, "fcm_dpo/beta": 0.0031418083235621452, "fcm_dpo/delta": 0.04664887487888336, "fcm_dpo/margin": 90.39424896240234, "fcm_dpo/q_t": 0.43551376461982727, "grad_norm": 92.76583862304688, "learning_rate": 6.409134137148736e-08, "logits/chosen": -0.8306005001068115, "logits/rejected": -0.8188484311103821, "logps/chosen": -520.89599609375, "logps/ref_chosen": -281.5295715332031, "logps/ref_rejected": -296.980224609375, "logps/rejected": -626.7409057617188, "loss": 4.7842, "margin_dpo/margin_mean": 90.39423370361328, "margin_dpo/margin_std": 172.24591064453125, "step": 378 }, { "KL/chosen_KL_mean": -224.11880493164062, "KL/mean": -268.5871276855469, "KL/rejected_KL_mean": -313.055419921875, "KL/std": 126.2266616821289, "epoch": 0.793717277486911, "fcm_dpo/beta": 0.003183132503181696, "fcm_dpo/delta": 0.004846029914915562, "fcm_dpo/margin": 88.9366226196289, "fcm_dpo/q_t": 0.4342249631881714, "grad_norm": 70.53761291503906, "learning_rate": 6.28723129572247e-08, "logits/chosen": -0.8911904096603394, "logits/rejected": -0.8721175789833069, "logps/chosen": -489.1995849609375, "logps/ref_chosen": -265.0807800292969, "logps/ref_rejected": -230.58932495117188, "logps/rejected": -543.6447143554688, "loss": 4.7555, "margin_dpo/margin_mean": 88.93663024902344, "margin_dpo/margin_std": 160.14614868164062, "step": 379 }, { "KL/chosen_KL_mean": -241.3148193359375, "KL/mean": -293.77069091796875, "KL/rejected_KL_mean": -346.2265319824219, "KL/std": 157.62762451171875, "epoch": 0.7958115183246073, "fcm_dpo/beta": 0.003145367605611682, "fcm_dpo/delta": -0.042871177196502686, "fcm_dpo/margin": 104.91173553466797, "fcm_dpo/q_t": 0.4266766607761383, "grad_norm": 98.90926361083984, "learning_rate": 6.166331963291519e-08, "logits/chosen": -0.8594827651977539, "logits/rejected": -0.834078848361969, "logps/chosen": -547.2232055664062, "logps/ref_chosen": -305.90838623046875, "logps/ref_rejected": -286.5906677246094, "logps/rejected": -632.8171997070312, "loss": 4.696, "margin_dpo/margin_mean": 104.91173553466797, "margin_dpo/margin_std": 186.09226989746094, "step": 380 }, { "KL/chosen_KL_mean": -238.97413635253906, "KL/mean": -295.09765625, "KL/rejected_KL_mean": -351.2211608886719, "KL/std": 172.82876586914062, "epoch": 0.7979057591623037, "fcm_dpo/beta": 0.0030606756918132305, "fcm_dpo/delta": -0.03527439385652542, "fcm_dpo/margin": 112.24700164794922, "fcm_dpo/q_t": 0.42362287640571594, "grad_norm": 90.26060485839844, "learning_rate": 6.046442623320145e-08, "logits/chosen": -0.8346625566482544, "logits/rejected": -0.8009909391403198, "logps/chosen": -491.8448181152344, "logps/ref_chosen": -252.87066650390625, "logps/ref_rejected": -261.1927490234375, "logps/rejected": -612.4138793945312, "loss": 4.5969, "margin_dpo/margin_mean": 112.24700927734375, "margin_dpo/margin_std": 181.87994384765625, "step": 381 }, { "KL/chosen_KL_mean": -227.16079711914062, "KL/mean": -288.4083251953125, "KL/rejected_KL_mean": -349.6558837890625, "KL/std": 143.6719970703125, "epoch": 0.8, "fcm_dpo/beta": 0.002987783635035157, "fcm_dpo/delta": -0.017105957493185997, "fcm_dpo/margin": 122.49507141113281, "fcm_dpo/q_t": 0.4151462912559509, "grad_norm": 70.07193756103516, "learning_rate": 5.9275697051098275e-08, "logits/chosen": -0.8694351315498352, "logits/rejected": -0.8645322322845459, "logps/chosen": -516.3722534179688, "logps/ref_chosen": -289.2114562988281, "logps/ref_rejected": -278.45751953125, "logps/rejected": -628.1133422851562, "loss": 4.4638, "margin_dpo/margin_mean": 122.49507141113281, "margin_dpo/margin_std": 168.80638122558594, "step": 382 }, { "KL/chosen_KL_mean": -224.1322784423828, "KL/mean": -280.5713195800781, "KL/rejected_KL_mean": -337.0103759765625, "KL/std": 159.5373992919922, "epoch": 0.8020942408376963, "fcm_dpo/beta": 0.0029694954864680767, "fcm_dpo/delta": -0.028972981497645378, "fcm_dpo/margin": 112.87809753417969, "fcm_dpo/q_t": 0.4229624271392822, "grad_norm": 93.5083999633789, "learning_rate": 5.809719583454414e-08, "logits/chosen": -0.8512469530105591, "logits/rejected": -0.8341723680496216, "logps/chosen": -497.76312255859375, "logps/ref_chosen": -273.630859375, "logps/ref_rejected": -261.44024658203125, "logps/rejected": -598.4506225585938, "loss": 4.6078, "margin_dpo/margin_mean": 112.87809753417969, "margin_dpo/margin_std": 181.7798309326172, "step": 383 }, { "KL/chosen_KL_mean": -233.4850616455078, "KL/mean": -281.0430603027344, "KL/rejected_KL_mean": -328.6011047363281, "KL/std": 128.45787048339844, "epoch": 0.8041884816753927, "fcm_dpo/beta": 0.002953949151560664, "fcm_dpo/delta": 0.040556252002716064, "fcm_dpo/margin": 95.11605834960938, "fcm_dpo/q_t": 0.4346542954444885, "grad_norm": 70.20569610595703, "learning_rate": 5.6928985782982524e-08, "logits/chosen": -0.8760598301887512, "logits/rejected": -0.8713008165359497, "logps/chosen": -508.0550537109375, "logps/ref_chosen": -274.5699462890625, "logps/ref_rejected": -285.8253479003906, "logps/rejected": -614.4264526367188, "loss": 4.7248, "margin_dpo/margin_mean": 95.11605834960938, "margin_dpo/margin_std": 159.0487060546875, "step": 384 }, { "KL/chosen_KL_mean": -238.326416015625, "KL/mean": -279.248779296875, "KL/rejected_KL_mean": -320.1711730957031, "KL/std": 137.06533813476562, "epoch": 0.806282722513089, "fcm_dpo/beta": 0.0030650359112769365, "fcm_dpo/delta": 0.028704283758997917, "fcm_dpo/margin": 81.8447494506836, "fcm_dpo/q_t": 0.44181060791015625, "grad_norm": 69.25728607177734, "learning_rate": 5.57711295439732e-08, "logits/chosen": -0.8149707913398743, "logits/rejected": -0.8097137212753296, "logps/chosen": -522.47705078125, "logps/ref_chosen": -284.150634765625, "logps/ref_rejected": -244.87921142578125, "logps/rejected": -565.0503540039062, "loss": 4.8258, "margin_dpo/margin_mean": 81.84474182128906, "margin_dpo/margin_std": 149.96339416503906, "step": 385 }, { "KL/chosen_KL_mean": -229.14710998535156, "KL/mean": -288.1037292480469, "KL/rejected_KL_mean": -347.060302734375, "KL/std": 148.16629028320312, "epoch": 0.8083769633507853, "fcm_dpo/beta": 0.002984508639201522, "fcm_dpo/delta": -0.04910660907626152, "fcm_dpo/margin": 117.91322326660156, "fcm_dpo/q_t": 0.4191738963127136, "grad_norm": 86.30573272705078, "learning_rate": 5.4623689209832484e-08, "logits/chosen": -0.7966516613960266, "logits/rejected": -0.795097291469574, "logps/chosen": -549.3233642578125, "logps/ref_chosen": -320.1762390136719, "logps/ref_rejected": -302.05023193359375, "logps/rejected": -649.1105346679688, "loss": 4.5273, "margin_dpo/margin_mean": 117.91322326660156, "margin_dpo/margin_std": 162.8836212158203, "step": 386 }, { "KL/chosen_KL_mean": -247.12374877929688, "KL/mean": -293.9572448730469, "KL/rejected_KL_mean": -340.79071044921875, "KL/std": 143.6245880126953, "epoch": 0.8104712041884817, "fcm_dpo/beta": 0.0029312213882803917, "fcm_dpo/delta": -0.015069102868437767, "fcm_dpo/margin": 93.66694641113281, "fcm_dpo/q_t": 0.43505552411079407, "grad_norm": 69.55741882324219, "learning_rate": 5.3486726314303175e-08, "logits/chosen": -0.8296703696250916, "logits/rejected": -0.8330702185630798, "logps/chosen": -519.4038696289062, "logps/ref_chosen": -272.2801513671875, "logps/ref_rejected": -265.1615905761719, "logps/rejected": -605.9522705078125, "loss": 4.8091, "margin_dpo/margin_mean": 93.66694641113281, "margin_dpo/margin_std": 173.94976806640625, "step": 387 }, { "KL/chosen_KL_mean": -262.6239318847656, "KL/mean": -296.395263671875, "KL/rejected_KL_mean": -330.1665954589844, "KL/std": 146.9426727294922, "epoch": 0.812565445026178, "fcm_dpo/beta": 0.002879001200199127, "fcm_dpo/delta": -0.010795612819492817, "fcm_dpo/margin": 67.54266357421875, "fcm_dpo/q_t": 0.45520469546318054, "grad_norm": 92.43110656738281, "learning_rate": 5.2360301829254745e-08, "logits/chosen": -0.8211464285850525, "logits/rejected": -0.8075721859931946, "logps/chosen": -535.1552734375, "logps/ref_chosen": -272.5313415527344, "logps/ref_rejected": -239.55735778808594, "logps/rejected": -569.7239379882812, "loss": 5.0687, "margin_dpo/margin_mean": 67.54266357421875, "margin_dpo/margin_std": 172.599853515625, "step": 388 }, { "KL/chosen_KL_mean": -257.0192565917969, "KL/mean": -309.4371643066406, "KL/rejected_KL_mean": -361.8551025390625, "KL/std": 156.31715393066406, "epoch": 0.8146596858638744, "fcm_dpo/beta": 0.0028533104341477156, "fcm_dpo/delta": -0.017784729599952698, "fcm_dpo/margin": 104.83583068847656, "fcm_dpo/q_t": 0.43158379197120667, "grad_norm": 70.29999542236328, "learning_rate": 5.1244476161413806e-08, "logits/chosen": -0.8539035320281982, "logits/rejected": -0.8511427640914917, "logps/chosen": -538.1085205078125, "logps/ref_chosen": -281.0892639160156, "logps/ref_rejected": -246.50045776367188, "logps/rejected": -608.3555908203125, "loss": 4.7168, "margin_dpo/margin_mean": 104.83582305908203, "margin_dpo/margin_std": 178.91343688964844, "step": 389 }, { "KL/chosen_KL_mean": -247.18812561035156, "KL/mean": -299.47222900390625, "KL/rejected_KL_mean": -351.75634765625, "KL/std": 147.55088806152344, "epoch": 0.8167539267015707, "fcm_dpo/beta": 0.0028385238256305456, "fcm_dpo/delta": -0.008637780323624611, "fcm_dpo/margin": 104.5682373046875, "fcm_dpo/q_t": 0.43057528138160706, "grad_norm": 55.768348693847656, "learning_rate": 5.013930914912476e-08, "logits/chosen": -0.8642117977142334, "logits/rejected": -0.8738707900047302, "logps/chosen": -531.1755981445312, "logps/ref_chosen": -283.98748779296875, "logps/ref_rejected": -283.465087890625, "logps/rejected": -635.221435546875, "loss": 4.6778, "margin_dpo/margin_mean": 104.5682373046875, "margin_dpo/margin_std": 167.7999725341797, "step": 390 }, { "KL/chosen_KL_mean": -262.1589660644531, "KL/mean": -318.81219482421875, "KL/rejected_KL_mean": -375.4654541015625, "KL/std": 163.68447875976562, "epoch": 0.818848167539267, "fcm_dpo/beta": 0.002770882099866867, "fcm_dpo/delta": -0.04480758681893349, "fcm_dpo/margin": 113.30651092529297, "fcm_dpo/q_t": 0.4273582696914673, "grad_norm": 74.78135681152344, "learning_rate": 4.904486005914027e-08, "logits/chosen": -0.8259506821632385, "logits/rejected": -0.8088208436965942, "logps/chosen": -546.0203857421875, "logps/ref_chosen": -283.86138916015625, "logps/ref_rejected": -263.5093688964844, "logps/rejected": -638.974853515625, "loss": 4.6416, "margin_dpo/margin_mean": 113.3065185546875, "margin_dpo/margin_std": 179.85752868652344, "step": 391 }, { "KL/chosen_KL_mean": -242.8234100341797, "KL/mean": -304.9492492675781, "KL/rejected_KL_mean": -367.0750732421875, "KL/std": 145.44244384765625, "epoch": 0.8209424083769633, "fcm_dpo/beta": 0.002657483099028468, "fcm_dpo/delta": -0.030804261565208435, "fcm_dpo/margin": 124.25167083740234, "fcm_dpo/q_t": 0.4213445782661438, "grad_norm": 74.31720733642578, "learning_rate": 4.796118758344353e-08, "logits/chosen": -0.8100302219390869, "logits/rejected": -0.8226779103279114, "logps/chosen": -552.8934326171875, "logps/ref_chosen": -310.070068359375, "logps/ref_rejected": -252.89817810058594, "logps/rejected": -619.9732055664062, "loss": 4.5373, "margin_dpo/margin_mean": 124.25165557861328, "margin_dpo/margin_std": 161.54566955566406, "step": 392 }, { "KL/chosen_KL_mean": -244.78565979003906, "KL/mean": -289.96575927734375, "KL/rejected_KL_mean": -335.14581298828125, "KL/std": 140.73870849609375, "epoch": 0.8230366492146597, "fcm_dpo/beta": 0.0027394951321184635, "fcm_dpo/delta": 0.06814486533403397, "fcm_dpo/margin": 90.36016082763672, "fcm_dpo/q_t": 0.4422205984592438, "grad_norm": 57.734832763671875, "learning_rate": 4.688834983610082e-08, "logits/chosen": -0.8401621580123901, "logits/rejected": -0.820095419883728, "logps/chosen": -531.5012817382812, "logps/ref_chosen": -286.7156677246094, "logps/ref_rejected": -230.00357055664062, "logps/rejected": -565.1493530273438, "loss": 4.8246, "margin_dpo/margin_mean": 90.36016845703125, "margin_dpo/margin_std": 166.87240600585938, "step": 393 }, { "KL/chosen_KL_mean": -249.3653564453125, "KL/mean": -294.98590087890625, "KL/rejected_KL_mean": -340.6064453125, "KL/std": 166.85951232910156, "epoch": 0.8251308900523561, "fcm_dpo/beta": 0.002803670009598136, "fcm_dpo/delta": 0.0016157319769263268, "fcm_dpo/margin": 91.24108123779297, "fcm_dpo/q_t": 0.44337010383605957, "grad_norm": 77.8697280883789, "learning_rate": 4.582640435014459e-08, "logits/chosen": -0.8874049782752991, "logits/rejected": -0.8782488107681274, "logps/chosen": -575.3588256835938, "logps/ref_chosen": -325.9934387207031, "logps/ref_rejected": -317.42706298828125, "logps/rejected": -658.0335083007812, "loss": 4.8929, "margin_dpo/margin_mean": 91.24108123779297, "margin_dpo/margin_std": 193.69656372070312, "step": 394 }, { "KL/chosen_KL_mean": -237.0347442626953, "KL/mean": -299.3041687011719, "KL/rejected_KL_mean": -361.5736083984375, "KL/std": 156.9813232421875, "epoch": 0.8272251308900523, "fcm_dpo/beta": 0.0028244995046406984, "fcm_dpo/delta": -0.0022739041596651077, "fcm_dpo/margin": 124.53888702392578, "fcm_dpo/q_t": 0.42142555117607117, "grad_norm": 63.480865478515625, "learning_rate": 4.477540807448832e-08, "logits/chosen": -0.8299747109413147, "logits/rejected": -0.8384482860565186, "logps/chosen": -505.9355773925781, "logps/ref_chosen": -268.90081787109375, "logps/ref_rejected": -272.85809326171875, "logps/rejected": -634.4317626953125, "loss": 4.545, "margin_dpo/margin_mean": 124.53888702392578, "margin_dpo/margin_std": 189.89027404785156, "step": 395 }, { "KL/chosen_KL_mean": -254.85581970214844, "KL/mean": -299.94439697265625, "KL/rejected_KL_mean": -345.032958984375, "KL/std": 150.55859375, "epoch": 0.8293193717277487, "fcm_dpo/beta": 0.0028044539503753185, "fcm_dpo/delta": 0.0007072770968079567, "fcm_dpo/margin": 90.17716217041016, "fcm_dpo/q_t": 0.44121721386909485, "grad_norm": 89.61280822753906, "learning_rate": 4.373541737087263e-08, "logits/chosen": -0.846572995185852, "logits/rejected": -0.8284200429916382, "logps/chosen": -546.0541381835938, "logps/ref_chosen": -291.19830322265625, "logps/ref_rejected": -253.2803955078125, "logps/rejected": -598.3133544921875, "loss": 4.8408, "margin_dpo/margin_mean": 90.17715454101562, "margin_dpo/margin_std": 173.43312072753906, "step": 396 }, { "KL/chosen_KL_mean": -244.94935607910156, "KL/mean": -288.22979736328125, "KL/rejected_KL_mean": -331.51025390625, "KL/std": 147.65113830566406, "epoch": 0.831413612565445, "fcm_dpo/beta": 0.0027613083366304636, "fcm_dpo/delta": -0.03639310598373413, "fcm_dpo/margin": 86.56092071533203, "fcm_dpo/q_t": 0.4450097680091858, "grad_norm": 56.71054458618164, "learning_rate": 4.270648801084295e-08, "logits/chosen": -0.8588390946388245, "logits/rejected": -0.8406790494918823, "logps/chosen": -554.7717895507812, "logps/ref_chosen": -309.8224182128906, "logps/ref_rejected": -291.9057922363281, "logps/rejected": -623.4160766601562, "loss": 4.9138, "margin_dpo/margin_mean": 86.5609130859375, "margin_dpo/margin_std": 175.44119262695312, "step": 397 }, { "KL/chosen_KL_mean": -269.9547424316406, "KL/mean": -307.1197509765625, "KL/rejected_KL_mean": -344.2847595214844, "KL/std": 160.36767578125, "epoch": 0.8335078534031414, "fcm_dpo/beta": 0.0027223476208746433, "fcm_dpo/delta": 0.005399319343268871, "fcm_dpo/margin": 74.3299789428711, "fcm_dpo/q_t": 0.4516116678714752, "grad_norm": 67.60769653320312, "learning_rate": 4.168867517275806e-08, "logits/chosen": -0.7625999450683594, "logits/rejected": -0.8003882169723511, "logps/chosen": -567.768310546875, "logps/ref_chosen": -297.8135070800781, "logps/ref_rejected": -270.5025634765625, "logps/rejected": -614.7872924804688, "loss": 5.1248, "margin_dpo/margin_mean": 74.32998657226562, "margin_dpo/margin_std": 205.70623779296875, "step": 398 }, { "KL/chosen_KL_mean": -258.96551513671875, "KL/mean": -308.9136047363281, "KL/rejected_KL_mean": -358.8616027832031, "KL/std": 152.3387451171875, "epoch": 0.8356020942408376, "fcm_dpo/beta": 0.0027614845894277096, "fcm_dpo/delta": 0.038348905742168427, "fcm_dpo/margin": 99.89608764648438, "fcm_dpo/q_t": 0.43669405579566956, "grad_norm": 90.15144348144531, "learning_rate": 4.0682033438831584e-08, "logits/chosen": -0.8555898070335388, "logits/rejected": -0.8154149651527405, "logps/chosen": -551.8123168945312, "logps/ref_chosen": -292.8467712402344, "logps/ref_rejected": -268.3638916015625, "logps/rejected": -627.2255249023438, "loss": 4.7632, "margin_dpo/margin_mean": 99.89608764648438, "margin_dpo/margin_std": 176.52777099609375, "step": 399 }, { "KL/chosen_KL_mean": -247.63442993164062, "KL/mean": -294.90032958984375, "KL/rejected_KL_mean": -342.166259765625, "KL/std": 147.7308349609375, "epoch": 0.837696335078534, "fcm_dpo/beta": 0.002795381471514702, "fcm_dpo/delta": -0.007865451276302338, "fcm_dpo/margin": 94.53181457519531, "fcm_dpo/q_t": 0.4383159875869751, "grad_norm": 59.41498947143555, "learning_rate": 3.968661679220467e-08, "logits/chosen": -0.890003502368927, "logits/rejected": -0.8898519277572632, "logps/chosen": -511.31085205078125, "logps/ref_chosen": -263.6763916015625, "logps/ref_rejected": -258.67266845703125, "logps/rejected": -600.8389282226562, "loss": 4.8212, "margin_dpo/margin_mean": 94.53180694580078, "margin_dpo/margin_std": 177.65081787109375, "step": 400 }, { "epoch": 0.837696335078534, "eval_KL/chosen_KL_mean": -251.34669494628906, "eval_KL/mean": -301.0162048339844, "eval_KL/rejected_KL_mean": -350.6857604980469, "eval_KL/std": 151.94033813476562, "eval_fcm_dpo/beta": 0.002804369665682316, "eval_logits/chosen": -0.8488364815711975, "eval_logits/rejected": -0.8314180374145508, "eval_logps/chosen": -539.1735229492188, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -617.61572265625, "eval_loss": 0.5985068678855896, "eval_margin_dpo/margin_mean": 99.33906555175781, "eval_margin_dpo/margin_std": 172.5637664794922, "eval_runtime": 78.0551, "eval_samples_per_second": 25.623, "eval_steps_per_second": 3.203, "step": 400 }, { "KL/chosen_KL_mean": -244.6216583251953, "KL/mean": -295.5447082519531, "KL/rejected_KL_mean": -346.4677734375, "KL/std": 154.69151306152344, "epoch": 0.8397905759162304, "fcm_dpo/beta": 0.002830990357324481, "fcm_dpo/delta": 0.01728871278464794, "fcm_dpo/margin": 101.84611511230469, "fcm_dpo/q_t": 0.4332505464553833, "grad_norm": 111.5696792602539, "learning_rate": 3.8702478614051345e-08, "logits/chosen": -0.8225564956665039, "logits/rejected": -0.8207831382751465, "logps/chosen": -562.9069213867188, "logps/ref_chosen": -318.2853088378906, "logps/ref_rejected": -293.75225830078125, "logps/rejected": -640.2200317382812, "loss": 4.7265, "margin_dpo/margin_mean": 101.84611511230469, "margin_dpo/margin_std": 175.7081298828125, "step": 401 }, { "KL/chosen_KL_mean": -242.03074645996094, "KL/mean": -288.13262939453125, "KL/rejected_KL_mean": -334.2344665527344, "KL/std": 158.20025634765625, "epoch": 0.8418848167539267, "fcm_dpo/beta": 0.002913672011345625, "fcm_dpo/delta": 0.06168531998991966, "fcm_dpo/margin": 92.20372772216797, "fcm_dpo/q_t": 0.4407009482383728, "grad_norm": 75.40254974365234, "learning_rate": 3.772967168071517e-08, "logits/chosen": -0.8898206949234009, "logits/rejected": -0.8673676252365112, "logps/chosen": -551.4586181640625, "logps/ref_chosen": -309.4278564453125, "logps/ref_rejected": -282.0279846191406, "logps/rejected": -616.262451171875, "loss": 4.8418, "margin_dpo/margin_mean": 92.20372772216797, "margin_dpo/margin_std": 192.0637969970703, "step": 402 }, { "KL/chosen_KL_mean": -232.39500427246094, "KL/mean": -302.0286560058594, "KL/rejected_KL_mean": -371.66229248046875, "KL/std": 162.0281982421875, "epoch": 0.8439790575916231, "fcm_dpo/beta": 0.002955038333311677, "fcm_dpo/delta": -0.06371486186981201, "fcm_dpo/margin": 139.26730346679688, "fcm_dpo/q_t": 0.4056738615036011, "grad_norm": 72.25170135498047, "learning_rate": 3.676824816087978e-08, "logits/chosen": -0.8628606200218201, "logits/rejected": -0.8384885787963867, "logps/chosen": -541.4235229492188, "logps/ref_chosen": -309.0284729003906, "logps/ref_rejected": -272.9622497558594, "logps/rejected": -644.62451171875, "loss": 4.3167, "margin_dpo/margin_mean": 139.26730346679688, "margin_dpo/margin_std": 168.85223388671875, "step": 403 }, { "KL/chosen_KL_mean": -252.7679443359375, "KL/mean": -304.2862548828125, "KL/rejected_KL_mean": -355.80450439453125, "KL/std": 164.53274536132812, "epoch": 0.8460732984293193, "fcm_dpo/beta": 0.0028354106470942497, "fcm_dpo/delta": -0.011737764813005924, "fcm_dpo/margin": 103.03657531738281, "fcm_dpo/q_t": 0.4333052933216095, "grad_norm": 85.11771392822266, "learning_rate": 3.581825961277074e-08, "logits/chosen": -0.8758622407913208, "logits/rejected": -0.8507161140441895, "logps/chosen": -550.0516357421875, "logps/ref_chosen": -297.2837219238281, "logps/ref_rejected": -256.99041748046875, "logps/rejected": -612.7949829101562, "loss": 4.7764, "margin_dpo/margin_mean": 103.03657531738281, "margin_dpo/margin_std": 195.86697387695312, "step": 404 }, { "KL/chosen_KL_mean": -246.62994384765625, "KL/mean": -303.05877685546875, "KL/rejected_KL_mean": -359.48760986328125, "KL/std": 159.86688232421875, "epoch": 0.8481675392670157, "fcm_dpo/beta": 0.002817755099385977, "fcm_dpo/delta": 0.008052084594964981, "fcm_dpo/margin": 112.85768127441406, "fcm_dpo/q_t": 0.4265621304512024, "grad_norm": 53.79868698120117, "learning_rate": 3.487975698139084e-08, "logits/chosen": -0.8158195614814758, "logits/rejected": -0.8253852128982544, "logps/chosen": -504.5953063964844, "logps/ref_chosen": -257.96533203125, "logps/ref_rejected": -255.811279296875, "logps/rejected": -615.2988891601562, "loss": 4.6274, "margin_dpo/margin_mean": 112.85768127441406, "margin_dpo/margin_std": 179.63681030273438, "step": 405 }, { "KL/chosen_KL_mean": -264.41668701171875, "KL/mean": -302.35516357421875, "KL/rejected_KL_mean": -340.2936706542969, "KL/std": 148.21942138671875, "epoch": 0.8502617801047121, "fcm_dpo/beta": 0.0028648902662098408, "fcm_dpo/delta": 0.0225369930267334, "fcm_dpo/margin": 75.87698364257812, "fcm_dpo/q_t": 0.4482402503490448, "grad_norm": 95.1996078491211, "learning_rate": 3.3952790595787986e-08, "logits/chosen": -0.8537810444831848, "logits/rejected": -0.8314708471298218, "logps/chosen": -549.5977172851562, "logps/ref_chosen": -285.1810607910156, "logps/ref_rejected": -264.41351318359375, "logps/rejected": -604.7071533203125, "loss": 4.9276, "margin_dpo/margin_mean": 75.87698364257812, "margin_dpo/margin_std": 154.45828247070312, "step": 406 }, { "KL/chosen_KL_mean": -265.21063232421875, "KL/mean": -317.1808776855469, "KL/rejected_KL_mean": -369.151123046875, "KL/std": 160.95401000976562, "epoch": 0.8523560209424084, "fcm_dpo/beta": 0.002887902082875371, "fcm_dpo/delta": -0.0053447941318154335, "fcm_dpo/margin": 103.94047546386719, "fcm_dpo/q_t": 0.43184512853622437, "grad_norm": 55.63980484008789, "learning_rate": 3.303741016635614e-08, "logits/chosen": -0.8277978897094727, "logits/rejected": -0.8441427946090698, "logps/chosen": -530.44873046875, "logps/ref_chosen": -265.23809814453125, "logps/ref_rejected": -219.0631561279297, "logps/rejected": -588.2142944335938, "loss": 4.7298, "margin_dpo/margin_mean": 103.94047546386719, "margin_dpo/margin_std": 186.37350463867188, "step": 407 }, { "KL/chosen_KL_mean": -241.85614013671875, "KL/mean": -296.291748046875, "KL/rejected_KL_mean": -350.7274169921875, "KL/std": 152.62454223632812, "epoch": 0.8544502617801047, "fcm_dpo/beta": 0.0028567167464643717, "fcm_dpo/delta": -0.04666806012392044, "fcm_dpo/margin": 108.8712158203125, "fcm_dpo/q_t": 0.4278115928173065, "grad_norm": 70.93869018554688, "learning_rate": 3.2133664782169944e-08, "logits/chosen": -0.8690704703330994, "logits/rejected": -0.8604199886322021, "logps/chosen": -538.8287353515625, "logps/ref_chosen": -296.9726257324219, "logps/ref_rejected": -295.4786376953125, "logps/rejected": -646.2060546875, "loss": 4.6472, "margin_dpo/margin_mean": 108.8712158203125, "margin_dpo/margin_std": 166.20672607421875, "step": 408 }, { "KL/chosen_KL_mean": -249.7004852294922, "KL/mean": -310.64984130859375, "KL/rejected_KL_mean": -371.59918212890625, "KL/std": 195.6245574951172, "epoch": 0.856544502617801, "fcm_dpo/beta": 0.0027475841343402863, "fcm_dpo/delta": -0.02165559120476246, "fcm_dpo/margin": 121.8986587524414, "fcm_dpo/q_t": 0.43092209100723267, "grad_norm": 49.7144889831543, "learning_rate": 3.12416029083514e-08, "logits/chosen": -0.8461386561393738, "logits/rejected": -0.8305491209030151, "logps/chosen": -537.079833984375, "logps/ref_chosen": -287.37933349609375, "logps/ref_rejected": -275.80291748046875, "logps/rejected": -647.402099609375, "loss": 4.7179, "margin_dpo/margin_mean": 121.8986587524414, "margin_dpo/margin_std": 237.97662353515625, "step": 409 }, { "KL/chosen_KL_mean": -243.9736785888672, "KL/mean": -292.99066162109375, "KL/rejected_KL_mean": -342.0075988769531, "KL/std": 148.37387084960938, "epoch": 0.8586387434554974, "fcm_dpo/beta": 0.0027260768692940474, "fcm_dpo/delta": 0.008642399683594704, "fcm_dpo/margin": 98.03392791748047, "fcm_dpo/q_t": 0.43822577595710754, "grad_norm": 100.64157104492188, "learning_rate": 3.036127238347164e-08, "logits/chosen": -0.8542839288711548, "logits/rejected": -0.8603944778442383, "logps/chosen": -525.7538452148438, "logps/ref_chosen": -281.7801818847656, "logps/ref_rejected": -266.7550354003906, "logps/rejected": -608.7626342773438, "loss": 4.7851, "margin_dpo/margin_mean": 98.03392791748047, "margin_dpo/margin_std": 175.8046417236328, "step": 410 }, { "KL/chosen_KL_mean": -243.66639709472656, "KL/mean": -300.5707702636719, "KL/rejected_KL_mean": -357.4750671386719, "KL/std": 162.13427734375, "epoch": 0.8607329842931937, "fcm_dpo/beta": 0.0027524821925908327, "fcm_dpo/delta": -0.009361675009131432, "fcm_dpo/margin": 113.80868530273438, "fcm_dpo/q_t": 0.4285432696342468, "grad_norm": 46.32007598876953, "learning_rate": 2.9492720416985e-08, "logits/chosen": -0.8611247539520264, "logits/rejected": -0.8141701817512512, "logps/chosen": -525.2536010742188, "logps/ref_chosen": -281.5872497558594, "logps/ref_rejected": -254.78916931152344, "logps/rejected": -612.2642822265625, "loss": 4.6081, "margin_dpo/margin_mean": 113.8086929321289, "margin_dpo/margin_std": 163.32958984375, "step": 411 }, { "KL/chosen_KL_mean": -265.3537292480469, "KL/mean": -312.93505859375, "KL/rejected_KL_mean": -360.51641845703125, "KL/std": 155.24221801757812, "epoch": 0.86282722513089, "fcm_dpo/beta": 0.002621360355988145, "fcm_dpo/delta": -0.052118320018053055, "fcm_dpo/margin": 95.16270446777344, "fcm_dpo/q_t": 0.44208434224128723, "grad_norm": 89.77355194091797, "learning_rate": 2.863599358669755e-08, "logits/chosen": -0.8341819643974304, "logits/rejected": -0.8465805649757385, "logps/chosen": -541.8878784179688, "logps/ref_chosen": -276.5341796875, "logps/ref_rejected": -273.8751220703125, "logps/rejected": -634.3915405273438, "loss": 4.8408, "margin_dpo/margin_mean": 95.16270446777344, "margin_dpo/margin_std": 173.06126403808594, "step": 412 }, { "KL/chosen_KL_mean": -248.24635314941406, "KL/mean": -292.7178649902344, "KL/rejected_KL_mean": -337.1893005371094, "KL/std": 152.73260498046875, "epoch": 0.8649214659685864, "fcm_dpo/beta": 0.0026765938382595778, "fcm_dpo/delta": 0.06977185606956482, "fcm_dpo/margin": 88.94297790527344, "fcm_dpo/q_t": 0.44494855403900146, "grad_norm": 95.87439727783203, "learning_rate": 2.7791137836269158e-08, "logits/chosen": -0.8483795523643494, "logits/rejected": -0.8444381952285767, "logps/chosen": -519.5209350585938, "logps/ref_chosen": -271.2745666503906, "logps/ref_rejected": -270.16912841796875, "logps/rejected": -607.3585205078125, "loss": 4.8712, "margin_dpo/margin_mean": 88.94297790527344, "margin_dpo/margin_std": 177.28707885742188, "step": 413 }, { "KL/chosen_KL_mean": -251.517578125, "KL/mean": -309.97723388671875, "KL/rejected_KL_mean": -368.4368591308594, "KL/std": 155.47898864746094, "epoch": 0.8670157068062827, "fcm_dpo/beta": 0.0027606477960944176, "fcm_dpo/delta": -0.01383502408862114, "fcm_dpo/margin": 116.91925811767578, "fcm_dpo/q_t": 0.4259103536605835, "grad_norm": 60.382442474365234, "learning_rate": 2.6958198472749717e-08, "logits/chosen": -0.889758288860321, "logits/rejected": -0.8836604952812195, "logps/chosen": -548.6326293945312, "logps/ref_chosen": -297.11505126953125, "logps/ref_rejected": -271.7034606933594, "logps/rejected": -640.1403198242188, "loss": 4.6044, "margin_dpo/margin_mean": 116.91926574707031, "margin_dpo/margin_std": 176.09750366210938, "step": 414 }, { "KL/chosen_KL_mean": -243.7269744873047, "KL/mean": -295.453125, "KL/rejected_KL_mean": -347.1793212890625, "KL/std": 143.5938720703125, "epoch": 0.8691099476439791, "fcm_dpo/beta": 0.002773047424852848, "fcm_dpo/delta": 0.0341414213180542, "fcm_dpo/margin": 103.45230865478516, "fcm_dpo/q_t": 0.4327102601528168, "grad_norm": 61.61573028564453, "learning_rate": 2.613722016414943e-08, "logits/chosen": -0.8744388222694397, "logits/rejected": -0.8600894808769226, "logps/chosen": -541.419677734375, "logps/ref_chosen": -297.6926574707031, "logps/ref_rejected": -279.0503234863281, "logps/rejected": -626.2296752929688, "loss": 4.6806, "margin_dpo/margin_mean": 103.45231628417969, "margin_dpo/margin_std": 161.2438507080078, "step": 415 }, { "KL/chosen_KL_mean": -235.6446075439453, "KL/mean": -296.1462097167969, "KL/rejected_KL_mean": -356.6477966308594, "KL/std": 144.9048614501953, "epoch": 0.8712041884816754, "fcm_dpo/beta": 0.0027621614281088114, "fcm_dpo/delta": -0.010026758536696434, "fcm_dpo/margin": 121.0031967163086, "fcm_dpo/q_t": 0.42156314849853516, "grad_norm": 54.852943420410156, "learning_rate": 2.5328246937043525e-08, "logits/chosen": -0.8805437088012695, "logits/rejected": -0.8778148293495178, "logps/chosen": -547.47021484375, "logps/ref_chosen": -311.8255615234375, "logps/ref_rejected": -268.6170654296875, "logps/rejected": -625.2648315429688, "loss": 4.5416, "margin_dpo/margin_mean": 121.0031967163086, "margin_dpo/margin_std": 169.40794372558594, "step": 416 }, { "KL/chosen_KL_mean": -257.8272705078125, "KL/mean": -309.54876708984375, "KL/rejected_KL_mean": -361.27032470703125, "KL/std": 162.28265380859375, "epoch": 0.8732984293193717, "fcm_dpo/beta": 0.0027372105978429317, "fcm_dpo/delta": -0.038760267198085785, "fcm_dpo/margin": 103.44303894042969, "fcm_dpo/q_t": 0.4357483386993408, "grad_norm": 67.29402160644531, "learning_rate": 2.4531322174210973e-08, "logits/chosen": -0.8284046649932861, "logits/rejected": -0.816261887550354, "logps/chosen": -568.2640991210938, "logps/ref_chosen": -310.43682861328125, "logps/ref_rejected": -277.15283203125, "logps/rejected": -638.4231567382812, "loss": 4.7596, "margin_dpo/margin_mean": 103.44303894042969, "margin_dpo/margin_std": 180.10482788085938, "step": 417 }, { "KL/chosen_KL_mean": -267.19671630859375, "KL/mean": -322.150634765625, "KL/rejected_KL_mean": -377.1046142578125, "KL/std": 174.22171020507812, "epoch": 0.875392670157068, "fcm_dpo/beta": 0.0026559322141110897, "fcm_dpo/delta": -0.0309746153652668, "fcm_dpo/margin": 109.90789794921875, "fcm_dpo/q_t": 0.4333530366420746, "grad_norm": 83.46846008300781, "learning_rate": 2.3746488612308295e-08, "logits/chosen": -0.8226134181022644, "logits/rejected": -0.8000130653381348, "logps/chosen": -545.6926879882812, "logps/ref_chosen": -278.49591064453125, "logps/ref_rejected": -276.56671142578125, "logps/rejected": -653.67138671875, "loss": 4.7553, "margin_dpo/margin_mean": 109.90789794921875, "margin_dpo/margin_std": 201.9710235595703, "step": 418 }, { "KL/chosen_KL_mean": -270.344482421875, "KL/mean": -320.5840148925781, "KL/rejected_KL_mean": -370.82354736328125, "KL/std": 162.86825561523438, "epoch": 0.8774869109947644, "fcm_dpo/beta": 0.0026421635411679745, "fcm_dpo/delta": 0.00882997177541256, "fcm_dpo/margin": 100.47907257080078, "fcm_dpo/q_t": 0.43709927797317505, "grad_norm": 75.29345703125, "learning_rate": 2.297378833957761e-08, "logits/chosen": -0.8669901490211487, "logits/rejected": -0.8375420570373535, "logps/chosen": -569.2447509765625, "logps/ref_chosen": -298.9002380371094, "logps/ref_rejected": -246.1540985107422, "logps/rejected": -616.9776611328125, "loss": 4.8279, "margin_dpo/margin_mean": 100.47908020019531, "margin_dpo/margin_std": 196.37474060058594, "step": 419 }, { "KL/chosen_KL_mean": -257.9861145019531, "KL/mean": -322.0567626953125, "KL/rejected_KL_mean": -386.12738037109375, "KL/std": 174.96910095214844, "epoch": 0.8795811518324608, "fcm_dpo/beta": 0.002635931596159935, "fcm_dpo/delta": -0.010491513647139072, "fcm_dpo/margin": 128.1412811279297, "fcm_dpo/q_t": 0.4231594204902649, "grad_norm": 73.04295349121094, "learning_rate": 2.2213262793589482e-08, "logits/chosen": -0.8123749494552612, "logits/rejected": -0.7837444543838501, "logps/chosen": -522.5469970703125, "logps/ref_chosen": -264.5608825683594, "logps/ref_rejected": -245.67031860351562, "logps/rejected": -631.7977294921875, "loss": 4.6445, "margin_dpo/margin_mean": 128.14129638671875, "margin_dpo/margin_std": 220.80238342285156, "step": 420 }, { "KL/chosen_KL_mean": -239.97381591796875, "KL/mean": -294.1003723144531, "KL/rejected_KL_mean": -348.2269592285156, "KL/std": 146.87315368652344, "epoch": 0.881675392670157, "fcm_dpo/beta": 0.0026289948727935553, "fcm_dpo/delta": 0.016483336687088013, "fcm_dpo/margin": 108.2531509399414, "fcm_dpo/q_t": 0.4323281943798065, "grad_norm": 102.15831756591797, "learning_rate": 2.1464952759020856e-08, "logits/chosen": -0.8939631581306458, "logits/rejected": -0.8694461584091187, "logps/chosen": -537.6788330078125, "logps/ref_chosen": -297.70501708984375, "logps/ref_rejected": -243.74771118164062, "logps/rejected": -591.9746704101562, "loss": 4.6963, "margin_dpo/margin_mean": 108.2531509399414, "margin_dpo/margin_std": 168.7926788330078, "step": 421 }, { "KL/chosen_KL_mean": -253.19100952148438, "KL/mean": -304.7960205078125, "KL/rejected_KL_mean": -356.4010925292969, "KL/std": 160.6505889892578, "epoch": 0.8837696335078534, "fcm_dpo/beta": 0.002700560726225376, "fcm_dpo/delta": 0.02611323446035385, "fcm_dpo/margin": 103.21012878417969, "fcm_dpo/q_t": 0.43617942929267883, "grad_norm": 58.64923858642578, "learning_rate": 2.07288983654679e-08, "logits/chosen": -0.7524704933166504, "logits/rejected": -0.7914985418319702, "logps/chosen": -541.5497436523438, "logps/ref_chosen": -288.3587646484375, "logps/ref_rejected": -256.4377746582031, "logps/rejected": -612.8389282226562, "loss": 4.7559, "margin_dpo/margin_mean": 103.21012878417969, "margin_dpo/margin_std": 184.8786163330078, "step": 422 }, { "KL/chosen_KL_mean": -259.6864318847656, "KL/mean": -316.55615234375, "KL/rejected_KL_mean": -373.4259033203125, "KL/std": 150.36436462402344, "epoch": 0.8858638743455497, "fcm_dpo/beta": 0.0027262200601398945, "fcm_dpo/delta": 0.01529497466981411, "fcm_dpo/margin": 113.73950958251953, "fcm_dpo/q_t": 0.42795345187187195, "grad_norm": 71.0254898071289, "learning_rate": 2.0005139085293942e-08, "logits/chosen": -0.8736763000488281, "logits/rejected": -0.850985050201416, "logps/chosen": -555.6934814453125, "logps/ref_chosen": -296.00701904296875, "logps/ref_rejected": -261.3480529785156, "logps/rejected": -634.7738647460938, "loss": 4.6501, "margin_dpo/margin_mean": 113.73950958251953, "margin_dpo/margin_std": 184.51678466796875, "step": 423 }, { "KL/chosen_KL_mean": -236.1890411376953, "KL/mean": -291.9098205566406, "KL/rejected_KL_mean": -347.6305847167969, "KL/std": 145.9111328125, "epoch": 0.8879581151832461, "fcm_dpo/beta": 0.002728747669607401, "fcm_dpo/delta": 0.0019058811012655497, "fcm_dpo/margin": 111.44156646728516, "fcm_dpo/q_t": 0.4285033643245697, "grad_norm": 88.48895263671875, "learning_rate": 1.9293713731512673e-08, "logits/chosen": -0.8554025292396545, "logits/rejected": -0.8399917483329773, "logps/chosen": -545.6109008789062, "logps/ref_chosen": -309.421875, "logps/ref_rejected": -249.14886474609375, "logps/rejected": -596.7794799804688, "loss": 4.6199, "margin_dpo/margin_mean": 111.44155883789062, "margin_dpo/margin_std": 159.28919982910156, "step": 424 }, { "KL/chosen_KL_mean": -249.6146240234375, "KL/mean": -301.1488342285156, "KL/rejected_KL_mean": -352.6830139160156, "KL/std": 154.60667419433594, "epoch": 0.8900523560209425, "fcm_dpo/beta": 0.002732752589508891, "fcm_dpo/delta": -0.018169663846492767, "fcm_dpo/margin": 103.06843566894531, "fcm_dpo/q_t": 0.43511763215065, "grad_norm": 70.74845886230469, "learning_rate": 1.8594660455706763e-08, "logits/chosen": -0.837730884552002, "logits/rejected": -0.8395782709121704, "logps/chosen": -530.1237182617188, "logps/ref_chosen": -280.50909423828125, "logps/ref_rejected": -276.8252258300781, "logps/rejected": -629.5082397460938, "loss": 4.7381, "margin_dpo/margin_mean": 103.06842803955078, "margin_dpo/margin_std": 173.00927734375, "step": 425 }, { "KL/chosen_KL_mean": -250.49110412597656, "KL/mean": -305.4483947753906, "KL/rejected_KL_mean": -360.4056701660156, "KL/std": 157.17343139648438, "epoch": 0.8921465968586387, "fcm_dpo/beta": 0.002728844992816448, "fcm_dpo/delta": -0.007583186496049166, "fcm_dpo/margin": 109.91456604003906, "fcm_dpo/q_t": 0.42983826994895935, "grad_norm": 78.41447448730469, "learning_rate": 1.7908016745981856e-08, "logits/chosen": -0.8673391342163086, "logits/rejected": -0.8460459113121033, "logps/chosen": -543.2763061523438, "logps/ref_chosen": -292.78521728515625, "logps/ref_rejected": -255.62698364257812, "logps/rejected": -616.0326538085938, "loss": 4.6647, "margin_dpo/margin_mean": 109.91456604003906, "margin_dpo/margin_std": 170.11758422851562, "step": 426 }, { "KL/chosen_KL_mean": -247.78399658203125, "KL/mean": -326.1251220703125, "KL/rejected_KL_mean": -404.4662170410156, "KL/std": 178.78623962402344, "epoch": 0.8942408376963351, "fcm_dpo/beta": 0.002634434960782528, "fcm_dpo/delta": -0.09540139138698578, "fcm_dpo/margin": 156.68223571777344, "fcm_dpo/q_t": 0.40496763586997986, "grad_norm": 97.31462860107422, "learning_rate": 1.7233819424956247e-08, "logits/chosen": -0.8368352055549622, "logits/rejected": -0.7985715866088867, "logps/chosen": -536.552734375, "logps/ref_chosen": -288.7687072753906, "logps/ref_rejected": -268.4986572265625, "logps/rejected": -672.9649658203125, "loss": 4.3996, "margin_dpo/margin_mean": 156.68223571777344, "margin_dpo/margin_std": 214.5924072265625, "step": 427 }, { "KL/chosen_KL_mean": -279.4367370605469, "KL/mean": -345.8340759277344, "KL/rejected_KL_mean": -412.2314453125, "KL/std": 175.0027313232422, "epoch": 0.8963350785340314, "fcm_dpo/beta": 0.002493727719411254, "fcm_dpo/delta": -0.009016228839755058, "fcm_dpo/margin": 132.79470825195312, "fcm_dpo/q_t": 0.4228391945362091, "grad_norm": 58.75352096557617, "learning_rate": 1.6572104647786245e-08, "logits/chosen": -0.803855836391449, "logits/rejected": -0.834683895111084, "logps/chosen": -574.9577026367188, "logps/ref_chosen": -295.5209655761719, "logps/ref_rejected": -275.71026611328125, "logps/rejected": -687.9417724609375, "loss": 4.5616, "margin_dpo/margin_mean": 132.79470825195312, "margin_dpo/margin_std": 189.15499877929688, "step": 428 }, { "KL/chosen_KL_mean": -259.98162841796875, "KL/mean": -314.2020263671875, "KL/rejected_KL_mean": -368.4223937988281, "KL/std": 167.9196319580078, "epoch": 0.8984293193717278, "fcm_dpo/beta": 0.0024902529548853636, "fcm_dpo/delta": 0.018422594293951988, "fcm_dpo/margin": 108.44074249267578, "fcm_dpo/q_t": 0.4369828701019287, "grad_norm": 66.33638000488281, "learning_rate": 1.5922907900227017e-08, "logits/chosen": -0.8072247505187988, "logits/rejected": -0.8141688108444214, "logps/chosen": -534.3739624023438, "logps/ref_chosen": -274.392333984375, "logps/ref_rejected": -258.574462890625, "logps/rejected": -626.9968872070312, "loss": 4.8132, "margin_dpo/margin_mean": 108.44074249267578, "margin_dpo/margin_std": 207.27586364746094, "step": 429 }, { "KL/chosen_KL_mean": -254.06158447265625, "KL/mean": -306.5641174316406, "KL/rejected_KL_mean": -359.066650390625, "KL/std": 159.74868774414062, "epoch": 0.900523560209424, "fcm_dpo/beta": 0.0025652057956904173, "fcm_dpo/delta": 0.017119091004133224, "fcm_dpo/margin": 105.00508117675781, "fcm_dpo/q_t": 0.4378116726875305, "grad_norm": 76.95693969726562, "learning_rate": 1.5286263996730026e-08, "logits/chosen": -0.8991668820381165, "logits/rejected": -0.8685553073883057, "logps/chosen": -542.8007202148438, "logps/ref_chosen": -288.7391357421875, "logps/ref_rejected": -268.6106262207031, "logps/rejected": -627.6773071289062, "loss": 4.7653, "margin_dpo/margin_mean": 105.00508880615234, "margin_dpo/margin_std": 181.61851501464844, "step": 430 }, { "KL/chosen_KL_mean": -256.83135986328125, "KL/mean": -298.0772399902344, "KL/rejected_KL_mean": -339.3231201171875, "KL/std": 148.50230407714844, "epoch": 0.9026178010471204, "fcm_dpo/beta": 0.0025556902401149273, "fcm_dpo/delta": -0.009694522246718407, "fcm_dpo/margin": 82.49180603027344, "fcm_dpo/q_t": 0.45022091269493103, "grad_norm": 64.07176971435547, "learning_rate": 1.4662207078575684e-08, "logits/chosen": -0.8577299118041992, "logits/rejected": -0.8271062970161438, "logps/chosen": -532.5560913085938, "logps/ref_chosen": -275.7247314453125, "logps/ref_rejected": -268.91729736328125, "logps/rejected": -608.2404174804688, "loss": 4.9432, "margin_dpo/margin_mean": 82.4917984008789, "margin_dpo/margin_std": 168.22735595703125, "step": 431 }, { "KL/chosen_KL_mean": -235.85765075683594, "KL/mean": -293.31103515625, "KL/rejected_KL_mean": -350.76446533203125, "KL/std": 162.96881103515625, "epoch": 0.9047120418848168, "fcm_dpo/beta": 0.0025669136084616184, "fcm_dpo/delta": 0.02326844446361065, "fcm_dpo/margin": 114.90679931640625, "fcm_dpo/q_t": 0.4326931834220886, "grad_norm": 106.33002471923828, "learning_rate": 1.40507706120426e-08, "logits/chosen": -0.8967832922935486, "logits/rejected": -0.8778641819953918, "logps/chosen": -527.2777709960938, "logps/ref_chosen": -291.42010498046875, "logps/ref_rejected": -255.48202514648438, "logps/rejected": -606.2464599609375, "loss": 4.7029, "margin_dpo/margin_mean": 114.90681457519531, "margin_dpo/margin_std": 193.0474090576172, "step": 432 }, { "KL/chosen_KL_mean": -263.96636962890625, "KL/mean": -314.3001403808594, "KL/rejected_KL_mean": -364.6339111328125, "KL/std": 153.60552978515625, "epoch": 0.9068062827225131, "fcm_dpo/beta": 0.0026351790875196457, "fcm_dpo/delta": 0.05364841967821121, "fcm_dpo/margin": 100.6675796508789, "fcm_dpo/q_t": 0.43835511803627014, "grad_norm": 52.755271911621094, "learning_rate": 1.345198738661285e-08, "logits/chosen": -0.8388685584068298, "logits/rejected": -0.8329813480377197, "logps/chosen": -510.1932373046875, "logps/ref_chosen": -246.2268829345703, "logps/ref_rejected": -253.65924072265625, "logps/rejected": -618.2931518554688, "loss": 4.7584, "margin_dpo/margin_mean": 100.6675796508789, "margin_dpo/margin_std": 172.32330322265625, "step": 433 }, { "KL/chosen_KL_mean": -281.077880859375, "KL/mean": -328.9410400390625, "KL/rejected_KL_mean": -376.80419921875, "KL/std": 159.2793731689453, "epoch": 0.9089005235602095, "fcm_dpo/beta": 0.002758192829787731, "fcm_dpo/delta": 0.04489844664931297, "fcm_dpo/margin": 95.72633361816406, "fcm_dpo/q_t": 0.4392809271812439, "grad_norm": 82.58972930908203, "learning_rate": 1.2865889513213628e-08, "logits/chosen": -0.8396630883216858, "logits/rejected": -0.8432857394218445, "logps/chosen": -576.5396728515625, "logps/ref_chosen": -295.4618225097656, "logps/ref_rejected": -256.2254333496094, "logps/rejected": -633.0296630859375, "loss": 4.7972, "margin_dpo/margin_mean": 95.72633361816406, "margin_dpo/margin_std": 174.9141387939453, "step": 434 }, { "KL/chosen_KL_mean": -246.79896545410156, "KL/mean": -306.7959899902344, "KL/rejected_KL_mean": -366.79302978515625, "KL/std": 164.5160369873047, "epoch": 0.9109947643979057, "fcm_dpo/beta": 0.0027714024763554335, "fcm_dpo/delta": -0.030311204493045807, "fcm_dpo/margin": 119.99405670166016, "fcm_dpo/q_t": 0.425067663192749, "grad_norm": 90.30838775634766, "learning_rate": 1.2292508422495157e-08, "logits/chosen": -0.8386386632919312, "logits/rejected": -0.8267545104026794, "logps/chosen": -507.537353515625, "logps/ref_chosen": -260.7384033203125, "logps/ref_rejected": -248.5688018798828, "logps/rejected": -615.36181640625, "loss": 4.5755, "margin_dpo/margin_mean": 119.99406433105469, "margin_dpo/margin_std": 176.0413055419922, "step": 435 }, { "KL/chosen_KL_mean": -276.3723449707031, "KL/mean": -319.248046875, "KL/rejected_KL_mean": -362.1237487792969, "KL/std": 180.16555786132812, "epoch": 0.9130890052356021, "fcm_dpo/beta": 0.0027568363584578037, "fcm_dpo/delta": 0.03259842097759247, "fcm_dpo/margin": 85.7514419555664, "fcm_dpo/q_t": 0.4493623971939087, "grad_norm": 97.92943572998047, "learning_rate": 1.1731874863145142e-08, "logits/chosen": -0.8305177092552185, "logits/rejected": -0.8246312141418457, "logps/chosen": -595.69482421875, "logps/ref_chosen": -319.3224792480469, "logps/ref_rejected": -299.30322265625, "logps/rejected": -661.4270629882812, "loss": 4.9529, "margin_dpo/margin_mean": 85.7514419555664, "margin_dpo/margin_std": 200.18887329101562, "step": 436 }, { "KL/chosen_KL_mean": -256.4522399902344, "KL/mean": -318.58221435546875, "KL/rejected_KL_mean": -380.7121887207031, "KL/std": 179.59681701660156, "epoch": 0.9151832460732985, "fcm_dpo/beta": 0.002750533865764737, "fcm_dpo/delta": -0.027415964752435684, "fcm_dpo/margin": 124.25997924804688, "fcm_dpo/q_t": 0.42281562089920044, "grad_norm": 56.36848831176758, "learning_rate": 1.118401890024001e-08, "logits/chosen": -0.8621474504470825, "logits/rejected": -0.8434783220291138, "logps/chosen": -535.281005859375, "logps/ref_chosen": -278.82879638671875, "logps/ref_rejected": -272.55303955078125, "logps/rejected": -653.2651977539062, "loss": 4.6259, "margin_dpo/margin_mean": 124.25997924804688, "margin_dpo/margin_std": 206.07656860351562, "step": 437 }, { "KL/chosen_KL_mean": -268.6119384765625, "KL/mean": -298.52435302734375, "KL/rejected_KL_mean": -328.4367370605469, "KL/std": 162.5165557861328, "epoch": 0.9172774869109948, "fcm_dpo/beta": 0.0027467007748782635, "fcm_dpo/delta": -0.0029242374002933502, "fcm_dpo/margin": 59.82483673095703, "fcm_dpo/q_t": 0.461994469165802, "grad_norm": 66.8003158569336, "learning_rate": 1.06489699136324e-08, "logits/chosen": -0.8481613993644714, "logits/rejected": -0.8591020107269287, "logps/chosen": -527.9309692382812, "logps/ref_chosen": -259.31903076171875, "logps/ref_rejected": -240.99581909179688, "logps/rejected": -569.4325561523438, "loss": 5.1956, "margin_dpo/margin_mean": 59.82484817504883, "margin_dpo/margin_std": 185.36160278320312, "step": 438 }, { "KL/chosen_KL_mean": -255.05923461914062, "KL/mean": -310.136474609375, "KL/rejected_KL_mean": -365.21368408203125, "KL/std": 151.9537353515625, "epoch": 0.9193717277486911, "fcm_dpo/beta": 0.002724123653024435, "fcm_dpo/delta": 0.012698237784206867, "fcm_dpo/margin": 110.15446472167969, "fcm_dpo/q_t": 0.4297390878200531, "grad_norm": 113.9437255859375, "learning_rate": 1.0126756596375685e-08, "logits/chosen": -0.8277400135993958, "logits/rejected": -0.824100136756897, "logps/chosen": -512.18359375, "logps/ref_chosen": -257.1243896484375, "logps/ref_rejected": -243.20416259765625, "logps/rejected": -608.4178466796875, "loss": 4.6466, "margin_dpo/margin_mean": 110.15446472167969, "margin_dpo/margin_std": 167.56851196289062, "step": 439 }, { "KL/chosen_KL_mean": -277.35302734375, "KL/mean": -317.02935791015625, "KL/rejected_KL_mean": -356.7057189941406, "KL/std": 143.4852294921875, "epoch": 0.9214659685863874, "fcm_dpo/beta": 0.0028076376765966415, "fcm_dpo/delta": 0.030055997893214226, "fcm_dpo/margin": 79.3526840209961, "fcm_dpo/q_t": 0.44725722074508667, "grad_norm": 59.82633590698242, "learning_rate": 9.617406953185136e-09, "logits/chosen": -0.8821266889572144, "logits/rejected": -0.873234748840332, "logps/chosen": -584.884521484375, "logps/ref_chosen": -307.5315246582031, "logps/ref_rejected": -264.3540954589844, "logps/rejected": -621.059814453125, "loss": 4.9115, "margin_dpo/margin_mean": 79.3526840209961, "margin_dpo/margin_std": 156.04428100585938, "step": 440 }, { "KL/chosen_KL_mean": -263.2364807128906, "KL/mean": -323.9188537597656, "KL/rejected_KL_mean": -384.6011962890625, "KL/std": 157.65415954589844, "epoch": 0.9235602094240838, "fcm_dpo/beta": 0.0027934680692851543, "fcm_dpo/delta": -0.028223905712366104, "fcm_dpo/margin": 121.36473083496094, "fcm_dpo/q_t": 0.42069217562675476, "grad_norm": 88.51628112792969, "learning_rate": 9.12094829893642e-09, "logits/chosen": -0.8281688094139099, "logits/rejected": -0.821442723274231, "logps/chosen": -573.218505859375, "logps/ref_chosen": -309.9819641113281, "logps/ref_rejected": -297.4968566894531, "logps/rejected": -682.0980224609375, "loss": 4.5291, "margin_dpo/margin_mean": 121.36473083496094, "margin_dpo/margin_std": 165.20242309570312, "step": 441 }, { "KL/chosen_KL_mean": -285.0911560058594, "KL/mean": -330.148681640625, "KL/rejected_KL_mean": -375.20623779296875, "KL/std": 159.1475067138672, "epoch": 0.9256544502617801, "fcm_dpo/beta": 0.0027609432581812143, "fcm_dpo/delta": 0.0063867755234241486, "fcm_dpo/margin": 90.11505126953125, "fcm_dpo/q_t": 0.4421730935573578, "grad_norm": 53.26344680786133, "learning_rate": 8.637407257200496e-09, "logits/chosen": -0.9114494323730469, "logits/rejected": -0.8700456023216248, "logps/chosen": -564.0703125, "logps/ref_chosen": -278.9791564941406, "logps/ref_rejected": -242.87310791015625, "logps/rejected": -618.079345703125, "loss": 4.8762, "margin_dpo/margin_mean": 90.11505126953125, "margin_dpo/margin_std": 186.69032287597656, "step": 442 }, { "KL/chosen_KL_mean": -268.91424560546875, "KL/mean": -314.4470520019531, "KL/rejected_KL_mean": -359.9798889160156, "KL/std": 142.87657165527344, "epoch": 0.9277486910994764, "fcm_dpo/beta": 0.0027999957092106342, "fcm_dpo/delta": -0.00435899943113327, "fcm_dpo/margin": 91.06564331054688, "fcm_dpo/q_t": 0.4397067725658417, "grad_norm": 64.01789855957031, "learning_rate": 8.166809758815895e-09, "logits/chosen": -0.801997721195221, "logits/rejected": -0.8227972388267517, "logps/chosen": -542.4733276367188, "logps/ref_chosen": -273.5590515136719, "logps/ref_rejected": -264.0199279785156, "logps/rejected": -623.9998779296875, "loss": 4.8378, "margin_dpo/margin_mean": 91.0656509399414, "margin_dpo/margin_std": 170.69918823242188, "step": 443 }, { "KL/chosen_KL_mean": -255.1349334716797, "KL/mean": -311.1309509277344, "KL/rejected_KL_mean": -367.12701416015625, "KL/std": 171.04638671875, "epoch": 0.9298429319371728, "fcm_dpo/beta": 0.0027402995619922876, "fcm_dpo/delta": -0.02012881450355053, "fcm_dpo/margin": 111.99202728271484, "fcm_dpo/q_t": 0.42987489700317383, "grad_norm": 78.47362518310547, "learning_rate": 7.709181040498253e-09, "logits/chosen": -0.817521333694458, "logits/rejected": -0.8048192858695984, "logps/chosen": -553.2791137695312, "logps/ref_chosen": -298.1441955566406, "logps/ref_rejected": -268.0572814941406, "logps/rejected": -635.1842041015625, "loss": 4.6927, "margin_dpo/margin_mean": 111.99202728271484, "margin_dpo/margin_std": 193.68740844726562, "step": 444 }, { "KL/chosen_KL_mean": -263.73199462890625, "KL/mean": -311.5459289550781, "KL/rejected_KL_mean": -359.3598327636719, "KL/std": 162.5752716064453, "epoch": 0.9319371727748691, "fcm_dpo/beta": 0.0027151252143085003, "fcm_dpo/delta": -0.023878220468759537, "fcm_dpo/margin": 95.62787628173828, "fcm_dpo/q_t": 0.4410240650177002, "grad_norm": 60.90433883666992, "learning_rate": 7.2645456434869965e-09, "logits/chosen": -0.8886433243751526, "logits/rejected": -0.8941999673843384, "logps/chosen": -518.2726440429688, "logps/ref_chosen": -254.54067993164062, "logps/ref_rejected": -264.2445983886719, "logps/rejected": -623.6044311523438, "loss": 4.8625, "margin_dpo/margin_mean": 95.62787628173828, "margin_dpo/margin_std": 188.66397094726562, "step": 445 }, { "KL/chosen_KL_mean": -256.6874084472656, "KL/mean": -311.60101318359375, "KL/rejected_KL_mean": -366.5146179199219, "KL/std": 157.3093719482422, "epoch": 0.9340314136125655, "fcm_dpo/beta": 0.002662414452061057, "fcm_dpo/delta": 0.01298682950437069, "fcm_dpo/margin": 109.82720947265625, "fcm_dpo/q_t": 0.4332526922225952, "grad_norm": 72.25860595703125, "learning_rate": 6.832927412229017e-09, "logits/chosen": -0.8085878491401672, "logits/rejected": -0.8123000860214233, "logps/chosen": -563.409912109375, "logps/ref_chosen": -306.72247314453125, "logps/ref_rejected": -266.3735656738281, "logps/rejected": -632.88818359375, "loss": 4.7131, "margin_dpo/margin_mean": 109.82720947265625, "margin_dpo/margin_std": 187.44256591796875, "step": 446 }, { "KL/chosen_KL_mean": -251.7252960205078, "KL/mean": -314.72784423828125, "KL/rejected_KL_mean": -377.7303771972656, "KL/std": 160.4945068359375, "epoch": 0.9361256544502617, "fcm_dpo/beta": 0.0026090561877936125, "fcm_dpo/delta": -0.04460753872990608, "fcm_dpo/margin": 126.00508880615234, "fcm_dpo/q_t": 0.42350420355796814, "grad_norm": 80.54020690917969, "learning_rate": 6.414349493100129e-09, "logits/chosen": -0.8133091330528259, "logits/rejected": -0.8063684105873108, "logps/chosen": -512.2425537109375, "logps/ref_chosen": -260.51727294921875, "logps/ref_rejected": -236.47061157226562, "logps/rejected": -614.2009887695312, "loss": 4.5824, "margin_dpo/margin_mean": 126.00509643554688, "margin_dpo/margin_std": 186.56752014160156, "step": 447 }, { "KL/chosen_KL_mean": -270.5550842285156, "KL/mean": -327.3743896484375, "KL/rejected_KL_mean": -384.19366455078125, "KL/std": 155.24166870117188, "epoch": 0.9382198952879581, "fcm_dpo/beta": 0.0025703529827296734, "fcm_dpo/delta": -0.01508631743490696, "fcm_dpo/margin": 113.63859558105469, "fcm_dpo/q_t": 0.43178319931030273, "grad_norm": 54.260047912597656, "learning_rate": 6.0088343331638756e-09, "logits/chosen": -0.8144208192825317, "logits/rejected": -0.8058769702911377, "logps/chosen": -539.3421020507812, "logps/ref_chosen": -268.78704833984375, "logps/ref_rejected": -262.1703796386719, "logps/rejected": -646.364013671875, "loss": 4.6702, "margin_dpo/margin_mean": 113.63859558105469, "margin_dpo/margin_std": 171.42291259765625, "step": 448 }, { "KL/chosen_KL_mean": -256.49884033203125, "KL/mean": -310.0642395019531, "KL/rejected_KL_mean": -363.62969970703125, "KL/std": 157.3622589111328, "epoch": 0.9403141361256544, "fcm_dpo/beta": 0.002572092227637768, "fcm_dpo/delta": -0.0028328632470220327, "fcm_dpo/margin": 107.13084411621094, "fcm_dpo/q_t": 0.433704674243927, "grad_norm": 97.39542388916016, "learning_rate": 5.616403678967624e-09, "logits/chosen": -0.9088627696037292, "logits/rejected": -0.8921771049499512, "logps/chosen": -587.4502563476562, "logps/ref_chosen": -330.9514465332031, "logps/ref_rejected": -239.76974487304688, "logps/rejected": -603.3994140625, "loss": 4.77, "margin_dpo/margin_mean": 107.13084411621094, "margin_dpo/margin_std": 188.76866149902344, "step": 449 }, { "KL/chosen_KL_mean": -279.85650634765625, "KL/mean": -327.69305419921875, "KL/rejected_KL_mean": -375.5296325683594, "KL/std": 164.77511596679688, "epoch": 0.9424083769633508, "fcm_dpo/beta": 0.0025807241909205914, "fcm_dpo/delta": 0.017238600179553032, "fcm_dpo/margin": 95.67314147949219, "fcm_dpo/q_t": 0.44215500354766846, "grad_norm": 72.74471282958984, "learning_rate": 5.2370785753763356e-09, "logits/chosen": -0.8057252764701843, "logits/rejected": -0.7975594997406006, "logps/chosen": -564.1218872070312, "logps/ref_chosen": -284.26544189453125, "logps/ref_rejected": -250.5401611328125, "logps/rejected": -626.06982421875, "loss": 4.8131, "margin_dpo/margin_mean": 95.67314147949219, "margin_dpo/margin_std": 164.05947875976562, "step": 450 }, { "KL/chosen_KL_mean": -282.51611328125, "KL/mean": -326.5624084472656, "KL/rejected_KL_mean": -370.60870361328125, "KL/std": 158.82203674316406, "epoch": 0.9445026178010472, "fcm_dpo/beta": 0.002585146576166153, "fcm_dpo/delta": 0.0006589877884835005, "fcm_dpo/margin": 88.09262084960938, "fcm_dpo/q_t": 0.44617974758148193, "grad_norm": 76.11978912353516, "learning_rate": 4.8708793644441086e-09, "logits/chosen": -0.8257752656936646, "logits/rejected": -0.7940558791160583, "logps/chosen": -584.8370361328125, "logps/ref_chosen": -302.3209228515625, "logps/ref_rejected": -254.09747314453125, "logps/rejected": -624.7062377929688, "loss": 4.8982, "margin_dpo/margin_mean": 88.09262084960938, "margin_dpo/margin_std": 171.1018524169922, "step": 451 }, { "KL/chosen_KL_mean": -255.13052368164062, "KL/mean": -322.3986511230469, "KL/rejected_KL_mean": -389.6668395996094, "KL/std": 191.00897216796875, "epoch": 0.9465968586387434, "fcm_dpo/beta": 0.002510292222723365, "fcm_dpo/delta": -0.04761781170964241, "fcm_dpo/margin": 134.53628540039062, "fcm_dpo/q_t": 0.425809383392334, "grad_norm": 56.017059326171875, "learning_rate": 4.517825684323323e-09, "logits/chosen": -0.8782578706741333, "logits/rejected": -0.8507087826728821, "logps/chosen": -554.522705078125, "logps/ref_chosen": -299.39215087890625, "logps/ref_rejected": -284.3475036621094, "logps/rejected": -674.0143432617188, "loss": 4.6199, "margin_dpo/margin_mean": 134.53628540039062, "margin_dpo/margin_std": 222.55630493164062, "step": 452 }, { "KL/chosen_KL_mean": -276.0933837890625, "KL/mean": -331.9483642578125, "KL/rejected_KL_mean": -387.80328369140625, "KL/std": 161.8867950439453, "epoch": 0.9486910994764398, "fcm_dpo/beta": 0.002537027932703495, "fcm_dpo/delta": 0.02657966874539852, "fcm_dpo/margin": 111.70990753173828, "fcm_dpo/q_t": 0.43421390652656555, "grad_norm": 85.80929565429688, "learning_rate": 4.1779364682113794e-09, "logits/chosen": -0.8239843249320984, "logits/rejected": -0.814884603023529, "logps/chosen": -600.7450561523438, "logps/ref_chosen": -324.6517028808594, "logps/ref_rejected": -304.1527099609375, "logps/rejected": -691.9559936523438, "loss": 4.7206, "margin_dpo/margin_mean": 111.70991516113281, "margin_dpo/margin_std": 191.12188720703125, "step": 453 }, { "KL/chosen_KL_mean": -268.03387451171875, "KL/mean": -332.931396484375, "KL/rejected_KL_mean": -397.8288269042969, "KL/std": 183.07398986816406, "epoch": 0.9507853403141361, "fcm_dpo/beta": 0.0025168166030198336, "fcm_dpo/delta": -0.019286856055259705, "fcm_dpo/margin": 129.79495239257812, "fcm_dpo/q_t": 0.4242181181907654, "grad_norm": 61.91704177856445, "learning_rate": 3.851229943335393e-09, "logits/chosen": -0.8545857667922974, "logits/rejected": -0.8613591194152832, "logps/chosen": -567.6456298828125, "logps/ref_chosen": -299.6117248535156, "logps/ref_rejected": -303.74224853515625, "logps/rejected": -701.5711059570312, "loss": 4.6043, "margin_dpo/margin_mean": 129.7949676513672, "margin_dpo/margin_std": 201.04190063476562, "step": 454 }, { "KL/chosen_KL_mean": -260.20513916015625, "KL/mean": -300.1604919433594, "KL/rejected_KL_mean": -340.1158447265625, "KL/std": 160.56234741210938, "epoch": 0.9528795811518325, "fcm_dpo/beta": 0.002559874439612031, "fcm_dpo/delta": 0.018644150346517563, "fcm_dpo/margin": 79.91071319580078, "fcm_dpo/q_t": 0.4540133774280548, "grad_norm": 80.64703369140625, "learning_rate": 3.5377236299748147e-09, "logits/chosen": -0.8247178196907043, "logits/rejected": -0.8339167833328247, "logps/chosen": -533.8168334960938, "logps/ref_chosen": -273.6116943359375, "logps/ref_rejected": -274.4293518066406, "logps/rejected": -614.545166015625, "loss": 5.0062, "margin_dpo/margin_mean": 79.91071319580078, "margin_dpo/margin_std": 183.53799438476562, "step": 455 }, { "KL/chosen_KL_mean": -286.6998291015625, "KL/mean": -344.54241943359375, "KL/rejected_KL_mean": -402.385009765625, "KL/std": 184.26705932617188, "epoch": 0.9549738219895288, "fcm_dpo/beta": 0.0026082973927259445, "fcm_dpo/delta": 0.026392972096800804, "fcm_dpo/margin": 115.68516540527344, "fcm_dpo/q_t": 0.4336429834365845, "grad_norm": 63.203765869140625, "learning_rate": 3.2374343405217884e-09, "logits/chosen": -0.7551023364067078, "logits/rejected": -0.7697523236274719, "logps/chosen": -608.8717651367188, "logps/ref_chosen": -322.17193603515625, "logps/ref_rejected": -294.54461669921875, "logps/rejected": -696.9296264648438, "loss": 4.7445, "margin_dpo/margin_mean": 115.68516540527344, "margin_dpo/margin_std": 216.65805053710938, "step": 456 }, { "KL/chosen_KL_mean": -280.3325500488281, "KL/mean": -341.10992431640625, "KL/rejected_KL_mean": -401.88720703125, "KL/std": 178.5426483154297, "epoch": 0.9570680628272251, "fcm_dpo/beta": 0.002582404762506485, "fcm_dpo/delta": -0.02019866183400154, "fcm_dpo/margin": 121.55464172363281, "fcm_dpo/q_t": 0.42459535598754883, "grad_norm": 68.02951049804688, "learning_rate": 2.9503781785795713e-09, "logits/chosen": -0.8246585130691528, "logits/rejected": -0.826050341129303, "logps/chosen": -588.1287841796875, "logps/ref_chosen": -307.7962341308594, "logps/ref_rejected": -274.5501403808594, "logps/rejected": -676.4373168945312, "loss": 4.6975, "margin_dpo/margin_mean": 121.55464172363281, "margin_dpo/margin_std": 210.71910095214844, "step": 457 }, { "KL/chosen_KL_mean": -275.7753601074219, "KL/mean": -325.453857421875, "KL/rejected_KL_mean": -375.1323547363281, "KL/std": 178.79847717285156, "epoch": 0.9591623036649215, "fcm_dpo/beta": 0.002581921638920903, "fcm_dpo/delta": 0.01107083261013031, "fcm_dpo/margin": 99.35697937011719, "fcm_dpo/q_t": 0.44110581278800964, "grad_norm": 55.92534637451172, "learning_rate": 2.6765705380989432e-09, "logits/chosen": -0.8420109152793884, "logits/rejected": -0.8187612891197205, "logps/chosen": -572.8070068359375, "logps/ref_chosen": -297.0316467285156, "logps/ref_rejected": -276.1112365722656, "logps/rejected": -651.2435913085938, "loss": 4.8562, "margin_dpo/margin_mean": 99.35697937011719, "margin_dpo/margin_std": 197.4419708251953, "step": 458 }, { "KL/chosen_KL_mean": -262.0546569824219, "KL/mean": -305.3924865722656, "KL/rejected_KL_mean": -348.73028564453125, "KL/std": 151.888916015625, "epoch": 0.9612565445026178, "fcm_dpo/beta": 0.002643053652718663, "fcm_dpo/delta": 0.04129417985677719, "fcm_dpo/margin": 86.67559814453125, "fcm_dpo/q_t": 0.4476607143878937, "grad_norm": 63.60211944580078, "learning_rate": 2.416026102552732e-09, "logits/chosen": -0.9004296064376831, "logits/rejected": -0.8917874097824097, "logps/chosen": -555.5799560546875, "logps/ref_chosen": -293.5252990722656, "logps/ref_rejected": -289.30126953125, "logps/rejected": -638.031494140625, "loss": 4.924, "margin_dpo/margin_mean": 86.67559814453125, "margin_dpo/margin_std": 182.57568359375, "step": 459 }, { "KL/chosen_KL_mean": -271.2803039550781, "KL/mean": -317.36767578125, "KL/rejected_KL_mean": -363.455078125, "KL/std": 146.47784423828125, "epoch": 0.9633507853403142, "fcm_dpo/beta": 0.0027003977447748184, "fcm_dpo/delta": 0.012249761261045933, "fcm_dpo/margin": 92.17478942871094, "fcm_dpo/q_t": 0.4414074122905731, "grad_norm": 90.7199478149414, "learning_rate": 2.168758844148272e-09, "logits/chosen": -0.8628847002983093, "logits/rejected": -0.8590608835220337, "logps/chosen": -590.0607299804688, "logps/ref_chosen": -318.7803649902344, "logps/ref_rejected": -258.7906799316406, "logps/rejected": -622.2457885742188, "loss": 4.8383, "margin_dpo/margin_mean": 92.17478942871094, "margin_dpo/margin_std": 174.8225555419922, "step": 460 }, { "KL/chosen_KL_mean": -266.7901306152344, "KL/mean": -318.44488525390625, "KL/rejected_KL_mean": -370.0997009277344, "KL/std": 165.7501220703125, "epoch": 0.9654450261780104, "fcm_dpo/beta": 0.002741393866017461, "fcm_dpo/delta": 0.0116231394931674, "fcm_dpo/margin": 103.30960083007812, "fcm_dpo/q_t": 0.4361855983734131, "grad_norm": 88.08728790283203, "learning_rate": 1.9347820230782295e-09, "logits/chosen": -0.8326213955879211, "logits/rejected": -0.8544647097587585, "logps/chosen": -510.7001037597656, "logps/ref_chosen": -243.9099884033203, "logps/ref_rejected": -232.6382293701172, "logps/rejected": -602.7379760742188, "loss": 4.8029, "margin_dpo/margin_mean": 103.30960083007812, "margin_dpo/margin_std": 200.97842407226562, "step": 461 }, { "KL/chosen_KL_mean": -260.3146057128906, "KL/mean": -323.3621826171875, "KL/rejected_KL_mean": -386.40972900390625, "KL/std": 158.8644561767578, "epoch": 0.9675392670157068, "fcm_dpo/beta": 0.002706458792090416, "fcm_dpo/delta": -0.016491197049617767, "fcm_dpo/margin": 126.09507751464844, "fcm_dpo/q_t": 0.42121651768684387, "grad_norm": 65.64117431640625, "learning_rate": 1.7141081868094209e-09, "logits/chosen": -0.8658933043479919, "logits/rejected": -0.8126181364059448, "logps/chosen": -604.4056396484375, "logps/ref_chosen": -344.09100341796875, "logps/ref_rejected": -252.45037841796875, "logps/rejected": -638.8601684570312, "loss": 4.5488, "margin_dpo/margin_mean": 126.09507751464844, "margin_dpo/margin_std": 184.6616668701172, "step": 462 }, { "KL/chosen_KL_mean": -263.3692321777344, "KL/mean": -310.8293151855469, "KL/rejected_KL_mean": -358.28936767578125, "KL/std": 168.63275146484375, "epoch": 0.9696335078534032, "fcm_dpo/beta": 0.0027042459696531296, "fcm_dpo/delta": -0.0022338179405778646, "fcm_dpo/margin": 94.9201431274414, "fcm_dpo/q_t": 0.4414103031158447, "grad_norm": 89.20877075195312, "learning_rate": 1.5067491694100153e-09, "logits/chosen": -0.8476633429527283, "logits/rejected": -0.8041394352912903, "logps/chosen": -560.5116577148438, "logps/ref_chosen": -297.1424560546875, "logps/ref_rejected": -234.0208282470703, "logps/rejected": -592.3101806640625, "loss": 4.8758, "margin_dpo/margin_mean": 94.92015075683594, "margin_dpo/margin_std": 198.87655639648438, "step": 463 }, { "KL/chosen_KL_mean": -274.56988525390625, "KL/mean": -323.9346618652344, "KL/rejected_KL_mean": -373.29937744140625, "KL/std": 161.66122436523438, "epoch": 0.9717277486910995, "fcm_dpo/beta": 0.002703792415559292, "fcm_dpo/delta": 0.016853541135787964, "fcm_dpo/margin": 98.72955322265625, "fcm_dpo/q_t": 0.43772199749946594, "grad_norm": 65.47233581542969, "learning_rate": 1.3127160909147672e-09, "logits/chosen": -0.8353570699691772, "logits/rejected": -0.8507075905799866, "logps/chosen": -540.2806396484375, "logps/ref_chosen": -265.71075439453125, "logps/ref_rejected": -256.4108581542969, "logps/rejected": -629.7102661132812, "loss": 4.7798, "margin_dpo/margin_mean": 98.72955322265625, "margin_dpo/margin_std": 175.61636352539062, "step": 464 }, { "KL/chosen_KL_mean": -252.164306640625, "KL/mean": -317.9529724121094, "KL/rejected_KL_mean": -383.7416076660156, "KL/std": 157.08651733398438, "epoch": 0.9738219895287958, "fcm_dpo/beta": 0.002617460675537586, "fcm_dpo/delta": -0.06499166041612625, "fcm_dpo/margin": 131.57730102539062, "fcm_dpo/q_t": 0.4203506112098694, "grad_norm": 61.7449951171875, "learning_rate": 1.1320193567288527e-09, "logits/chosen": -0.9151032567024231, "logits/rejected": -0.8762094974517822, "logps/chosen": -545.3170166015625, "logps/ref_chosen": -293.1527404785156, "logps/ref_rejected": -293.70947265625, "logps/rejected": -677.4511108398438, "loss": 4.5364, "margin_dpo/margin_mean": 131.57730102539062, "margin_dpo/margin_std": 182.81796264648438, "step": 465 }, { "KL/chosen_KL_mean": -238.81729125976562, "KL/mean": -307.2685546875, "KL/rejected_KL_mean": -375.71978759765625, "KL/std": 156.15011596679688, "epoch": 0.9759162303664921, "fcm_dpo/beta": 0.002551088575273752, "fcm_dpo/delta": 0.0003731958568096161, "fcm_dpo/margin": 136.90252685546875, "fcm_dpo/q_t": 0.4190123379230499, "grad_norm": 84.08358764648438, "learning_rate": 9.64668657069706e-10, "logits/chosen": -0.8227401375770569, "logits/rejected": -0.7646010518074036, "logps/chosen": -500.29486083984375, "logps/ref_chosen": -261.4775695800781, "logps/ref_rejected": -248.36282348632812, "logps/rejected": -624.0826416015625, "loss": 4.4914, "margin_dpo/margin_mean": 136.90252685546875, "margin_dpo/margin_std": 183.17254638671875, "step": 466 }, { "KL/chosen_KL_mean": -265.2427978515625, "KL/mean": -315.06036376953125, "KL/rejected_KL_mean": -364.8779602050781, "KL/std": 160.9639892578125, "epoch": 0.9780104712041885, "fcm_dpo/beta": 0.002554756123572588, "fcm_dpo/delta": -0.012596940621733665, "fcm_dpo/margin": 99.6351547241211, "fcm_dpo/q_t": 0.4416211247444153, "grad_norm": 81.79702758789062, "learning_rate": 8.106729664475176e-10, "logits/chosen": -0.8072253465652466, "logits/rejected": -0.7998838424682617, "logps/chosen": -531.5970458984375, "logps/ref_chosen": -266.354248046875, "logps/ref_rejected": -277.76324462890625, "logps/rejected": -642.6411743164062, "loss": 4.8475, "margin_dpo/margin_mean": 99.6351547241211, "margin_dpo/margin_std": 196.27394104003906, "step": 467 }, { "KL/chosen_KL_mean": -268.8735046386719, "KL/mean": -305.91766357421875, "KL/rejected_KL_mean": -342.9618225097656, "KL/std": 154.4098358154297, "epoch": 0.9801047120418848, "fcm_dpo/beta": 0.002624538727104664, "fcm_dpo/delta": 0.048930246382951736, "fcm_dpo/margin": 74.08831024169922, "fcm_dpo/q_t": 0.45356136560440063, "grad_norm": 70.1420669555664, "learning_rate": 6.700405431837585e-10, "logits/chosen": -0.8905247449874878, "logits/rejected": -0.8599618673324585, "logps/chosen": -586.8367309570312, "logps/ref_chosen": -317.9631652832031, "logps/ref_rejected": -261.8744201660156, "logps/rejected": -604.8362426757812, "loss": 5.0869, "margin_dpo/margin_mean": 74.08831024169922, "margin_dpo/margin_std": 198.05606079101562, "step": 468 }, { "KL/chosen_KL_mean": -249.63702392578125, "KL/mean": -310.2472839355469, "KL/rejected_KL_mean": -370.8575744628906, "KL/std": 164.26007080078125, "epoch": 0.9821989528795811, "fcm_dpo/beta": 0.0026474245823919773, "fcm_dpo/delta": -0.009328970685601234, "fcm_dpo/margin": 121.22054290771484, "fcm_dpo/q_t": 0.4254051744937897, "grad_norm": 38.51364517211914, "learning_rate": 5.427789289685347e-10, "logits/chosen": -0.8271424174308777, "logits/rejected": -0.8110153079032898, "logps/chosen": -574.5238647460938, "logps/ref_chosen": -324.8868103027344, "logps/ref_rejected": -264.0421447753906, "logps/rejected": -634.8997802734375, "loss": 4.6194, "margin_dpo/margin_mean": 121.22053527832031, "margin_dpo/margin_std": 189.99745178222656, "step": 469 }, { "KL/chosen_KL_mean": -251.842529296875, "KL/mean": -312.7073974609375, "KL/rejected_KL_mean": -373.57232666015625, "KL/std": 160.9497833251953, "epoch": 0.9842931937172775, "fcm_dpo/beta": 0.0025979301426559687, "fcm_dpo/delta": -0.04329398646950722, "fcm_dpo/margin": 121.72976684570312, "fcm_dpo/q_t": 0.4263451397418976, "grad_norm": 64.70094299316406, "learning_rate": 4.288949484559934e-10, "logits/chosen": -0.8176441788673401, "logits/rejected": -0.8153640031814575, "logps/chosen": -566.5467529296875, "logps/ref_chosen": -314.7042541503906, "logps/ref_rejected": -259.2276611328125, "logps/rejected": -632.7999877929688, "loss": 4.619, "margin_dpo/margin_mean": 121.72976684570312, "margin_dpo/margin_std": 182.20755004882812, "step": 470 }, { "KL/chosen_KL_mean": -273.6257629394531, "KL/mean": -332.0756530761719, "KL/rejected_KL_mean": -390.52557373046875, "KL/std": 192.36666870117188, "epoch": 0.9863874345549738, "fcm_dpo/beta": 0.0025639727246016264, "fcm_dpo/delta": -0.009167149662971497, "fcm_dpo/margin": 116.89979553222656, "fcm_dpo/q_t": 0.4352128505706787, "grad_norm": 86.51072692871094, "learning_rate": 3.2839470889836627e-10, "logits/chosen": -0.8735991716384888, "logits/rejected": -0.8589140176773071, "logps/chosen": -566.20068359375, "logps/ref_chosen": -292.5748291015625, "logps/ref_rejected": -298.7525329589844, "logps/rejected": -689.278076171875, "loss": 4.7466, "margin_dpo/margin_mean": 116.89979553222656, "margin_dpo/margin_std": 217.3658905029297, "step": 471 }, { "KL/chosen_KL_mean": -249.77090454101562, "KL/mean": -314.00897216796875, "KL/rejected_KL_mean": -378.2470397949219, "KL/std": 159.54998779296875, "epoch": 0.9884816753926702, "fcm_dpo/beta": 0.0025038199964910746, "fcm_dpo/delta": -0.012427094392478466, "fcm_dpo/margin": 128.47613525390625, "fcm_dpo/q_t": 0.4258285164833069, "grad_norm": 79.63658905029297, "learning_rate": 2.412835998185092e-10, "logits/chosen": -0.8494762778282166, "logits/rejected": -0.8620040416717529, "logps/chosen": -493.144775390625, "logps/ref_chosen": -243.37380981445312, "logps/ref_rejected": -251.12109375, "logps/rejected": -629.3681640625, "loss": 4.5936, "margin_dpo/margin_mean": 128.47613525390625, "margin_dpo/margin_std": 189.0301971435547, "step": 472 }, { "KL/chosen_KL_mean": -261.2845153808594, "KL/mean": -321.57684326171875, "KL/rejected_KL_mean": -381.86920166015625, "KL/std": 147.27406311035156, "epoch": 0.9905759162303664, "fcm_dpo/beta": 0.0025835472624748945, "fcm_dpo/delta": 0.03919682651758194, "fcm_dpo/margin": 120.58467102050781, "fcm_dpo/q_t": 0.4267313778400421, "grad_norm": 50.619747161865234, "learning_rate": 1.6756629272085544e-10, "logits/chosen": -0.8187922835350037, "logits/rejected": -0.816412091255188, "logps/chosen": -547.6131591796875, "logps/ref_chosen": -286.3286437988281, "logps/ref_rejected": -258.6535339355469, "logps/rejected": -640.5227661132812, "loss": 4.583, "margin_dpo/margin_mean": 120.58467102050781, "margin_dpo/margin_std": 162.5080108642578, "step": 473 }, { "KL/chosen_KL_mean": -267.7467041015625, "KL/mean": -321.2333984375, "KL/rejected_KL_mean": -374.72015380859375, "KL/std": 164.40090942382812, "epoch": 0.9926701570680628, "fcm_dpo/beta": 0.002585589187219739, "fcm_dpo/delta": 0.016463520005345345, "fcm_dpo/margin": 106.97345733642578, "fcm_dpo/q_t": 0.43555474281311035, "grad_norm": 92.50374603271484, "learning_rate": 1.072467408408384e-10, "logits/chosen": -0.8593537211418152, "logits/rejected": -0.8538248538970947, "logps/chosen": -555.8363037109375, "logps/ref_chosen": -288.08966064453125, "logps/ref_rejected": -266.69696044921875, "logps/rejected": -641.4171142578125, "loss": 4.7239, "margin_dpo/margin_mean": 106.97344970703125, "margin_dpo/margin_std": 171.1471405029297, "step": 474 }, { "KL/chosen_KL_mean": -246.10379028320312, "KL/mean": -295.0997314453125, "KL/rejected_KL_mean": -344.09564208984375, "KL/std": 147.87771606445312, "epoch": 0.9947643979057592, "fcm_dpo/beta": 0.0025957191828638315, "fcm_dpo/delta": -0.01816142536699772, "fcm_dpo/margin": 97.99187469482422, "fcm_dpo/q_t": 0.4415433406829834, "grad_norm": 66.68531799316406, "learning_rate": 6.032817893297793e-11, "logits/chosen": -0.8354889750480652, "logits/rejected": -0.8441330790519714, "logps/chosen": -502.10687255859375, "logps/ref_chosen": -256.0030517578125, "logps/ref_rejected": -244.50660705566406, "logps/rejected": -588.6022338867188, "loss": 4.806, "margin_dpo/margin_mean": 97.99187469482422, "margin_dpo/margin_std": 172.44944763183594, "step": 475 }, { "KL/chosen_KL_mean": -248.5246124267578, "KL/mean": -301.4661865234375, "KL/rejected_KL_mean": -354.40777587890625, "KL/std": 148.5124969482422, "epoch": 0.9968586387434555, "fcm_dpo/beta": 0.002539725974202156, "fcm_dpo/delta": -0.031470753252506256, "fcm_dpo/margin": 105.8831558227539, "fcm_dpo/q_t": 0.43887490034103394, "grad_norm": 69.0814208984375, "learning_rate": 2.6813123097352287e-11, "logits/chosen": -0.9057525396347046, "logits/rejected": -0.8594703674316406, "logps/chosen": -569.9921264648438, "logps/ref_chosen": -321.467529296875, "logps/ref_rejected": -295.0592956542969, "logps/rejected": -649.4671630859375, "loss": 4.7909, "margin_dpo/margin_mean": 105.8831558227539, "margin_dpo/margin_std": 185.74710083007812, "step": 476 }, { "KL/chosen_KL_mean": -270.9020690917969, "KL/mean": -323.1831359863281, "KL/rejected_KL_mean": -375.4642333984375, "KL/std": 173.52398681640625, "epoch": 0.9989528795811519, "fcm_dpo/beta": 0.002541655208915472, "fcm_dpo/delta": -0.0096766147762537, "fcm_dpo/margin": 104.5621566772461, "fcm_dpo/q_t": 0.4383348822593689, "grad_norm": 120.1622085571289, "learning_rate": 6.7033706447061635e-12, "logits/chosen": -0.7790302634239197, "logits/rejected": -0.7840569019317627, "logps/chosen": -547.696044921875, "logps/ref_chosen": -276.7939758300781, "logps/ref_rejected": -244.82919311523438, "logps/rejected": -620.2933959960938, "loss": 4.8684, "margin_dpo/margin_mean": 104.5621566772461, "margin_dpo/margin_std": 218.2042236328125, "step": 477 }, { "epoch": 0.9989528795811519, "step": 477, "total_flos": 0.0, "train_loss": 4.871757778231703, "train_runtime": 6533.4358, "train_samples_per_second": 9.357, "train_steps_per_second": 0.073 } ], "logging_steps": 1, "max_steps": 477, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }