{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9989528795811519, "eval_steps": 200, "global_step": 477, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": -0.05443763732910156, "KL/mean": -0.04046645760536194, "KL/rejected_KL_mean": -0.026498794555664062, "KL/std": 0.47091516852378845, "epoch": 0.0020942408376963353, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02793481945991516, "fcm_dpo/q_t": 0.500069797039032, "grad_norm": 28.589929580688477, "learning_rate": 0.0, "logits/chosen": -0.5898098945617676, "logits/rejected": -0.604260265827179, "logps/chosen": -275.28570556640625, "logps/ref_chosen": -275.2312927246094, "logps/ref_rejected": -222.9380340576172, "logps/rejected": -222.96453857421875, "loss": 5.5463, "margin_dpo/margin_mean": -0.02793477475643158, "margin_dpo/margin_std": 0.5724214911460876, "step": 1 }, { "KL/chosen_KL_mean": 0.04458808898925781, "KL/mean": 0.0374295711517334, "KL/rejected_KL_mean": 0.030269622802734375, "KL/std": 0.49036121368408203, "epoch": 0.004188481675392671, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014312177896499634, "fcm_dpo/q_t": 0.4999642074108124, "grad_norm": 27.881349563598633, "learning_rate": 1.0416666666666666e-08, "logits/chosen": -0.6574729681015015, "logits/rejected": -0.6464410424232483, "logps/chosen": -264.7165222167969, "logps/ref_chosen": -264.7611083984375, "logps/ref_rejected": -242.5597686767578, "logps/rejected": -242.52951049804688, "loss": 5.5446, "margin_dpo/margin_mean": 0.014312252402305603, "margin_dpo/margin_std": 0.6423971652984619, "step": 2 }, { "KL/chosen_KL_mean": -0.015283584594726562, "KL/mean": -0.027470149099826813, "KL/rejected_KL_mean": -0.03966331481933594, "KL/std": 0.4722851514816284, "epoch": 0.0062827225130890054, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02437432110309601, "fcm_dpo/q_t": 0.49993908405303955, "grad_norm": 25.772430419921875, "learning_rate": 2.083333333333333e-08, "logits/chosen": -0.6839795112609863, "logits/rejected": -0.7351416349411011, "logps/chosen": -274.1171875, "logps/ref_chosen": -274.1018981933594, "logps/ref_rejected": -286.5882568359375, "logps/rejected": -286.6279296875, "loss": 5.5442, "margin_dpo/margin_mean": 0.024372458457946777, "margin_dpo/margin_std": 0.6266783475875854, "step": 3 }, { "KL/chosen_KL_mean": 0.04073333740234375, "KL/mean": -0.02466951310634613, "KL/rejected_KL_mean": -0.0900726318359375, "KL/std": 0.5628562569618225, "epoch": 0.008376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13081058859825134, "fcm_dpo/q_t": 0.49967294931411743, "grad_norm": 31.76841926574707, "learning_rate": 3.125e-08, "logits/chosen": -0.6164096593856812, "logits/rejected": -0.61078280210495, "logps/chosen": -329.7975158691406, "logps/ref_chosen": -329.8382568359375, "logps/ref_rejected": -303.2850646972656, "logps/rejected": -303.3750915527344, "loss": 5.54, "margin_dpo/margin_mean": 0.1308116316795349, "margin_dpo/margin_std": 0.7712123990058899, "step": 4 }, { "KL/chosen_KL_mean": -0.004119873046875, "KL/mean": 0.020755663514137268, "KL/rejected_KL_mean": 0.045635223388671875, "KL/std": 0.533147931098938, "epoch": 0.010471204188481676, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04974430799484253, "fcm_dpo/q_t": 0.5001243352890015, "grad_norm": 29.5401554107666, "learning_rate": 4.166666666666666e-08, "logits/chosen": -0.5749531984329224, "logits/rejected": -0.5917044281959534, "logps/chosen": -301.7430419921875, "logps/ref_chosen": -301.7389221191406, "logps/ref_rejected": -274.7654724121094, "logps/rejected": -274.7198181152344, "loss": 5.5472, "margin_dpo/margin_mean": -0.049745798110961914, "margin_dpo/margin_std": 0.8052065968513489, "step": 5 }, { "KL/chosen_KL_mean": -0.016300201416015625, "KL/mean": 0.0033769533038139343, "KL/rejected_KL_mean": 0.023052215576171875, "KL/std": 0.5462159514427185, "epoch": 0.012565445026178011, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03935401141643524, "fcm_dpo/q_t": 0.5000984072685242, "grad_norm": 28.32386589050293, "learning_rate": 5.208333333333333e-08, "logits/chosen": -0.6760701537132263, "logits/rejected": -0.6386649012565613, "logps/chosen": -285.7109680175781, "logps/ref_chosen": -285.6946716308594, "logps/ref_rejected": -245.8200225830078, "logps/rejected": -245.79696655273438, "loss": 5.5468, "margin_dpo/margin_mean": -0.03935439884662628, "margin_dpo/margin_std": 0.7249687314033508, "step": 6 }, { "KL/chosen_KL_mean": -0.08362960815429688, "KL/mean": -0.03842785954475403, "KL/rejected_KL_mean": 0.006763458251953125, "KL/std": 0.4993395209312439, "epoch": 0.014659685863874346, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09039899706840515, "fcm_dpo/q_t": 0.5002260208129883, "grad_norm": 28.561147689819336, "learning_rate": 6.25e-08, "logits/chosen": -0.5755224823951721, "logits/rejected": -0.6081489324569702, "logps/chosen": -264.7390441894531, "logps/ref_chosen": -264.65545654296875, "logps/ref_rejected": -253.10305786132812, "logps/rejected": -253.09628295898438, "loss": 5.5488, "margin_dpo/margin_mean": -0.09039971232414246, "margin_dpo/margin_std": 0.6376783847808838, "step": 7 }, { "KL/chosen_KL_mean": 0.041156768798828125, "KL/mean": -0.013764090836048126, "KL/rejected_KL_mean": -0.06868171691894531, "KL/std": 0.5888418555259705, "epoch": 0.016753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10983617603778839, "fcm_dpo/q_t": 0.49972546100616455, "grad_norm": 30.76198387145996, "learning_rate": 7.291666666666667e-08, "logits/chosen": -0.6753714680671692, "logits/rejected": -0.681854784488678, "logps/chosen": -354.14764404296875, "logps/ref_chosen": -354.1887512207031, "logps/ref_rejected": -282.9112243652344, "logps/rejected": -282.9798889160156, "loss": 5.5409, "margin_dpo/margin_mean": 0.10983628034591675, "margin_dpo/margin_std": 0.7891420125961304, "step": 8 }, { "KL/chosen_KL_mean": 0.0035858154296875, "KL/mean": 0.005865432322025299, "KL/rejected_KL_mean": 0.008136749267578125, "KL/std": 0.5102388858795166, "epoch": 0.018848167539267015, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.004547819495201111, "fcm_dpo/q_t": 0.5000113844871521, "grad_norm": 27.88920021057129, "learning_rate": 8.333333333333333e-08, "logits/chosen": -0.6402922868728638, "logits/rejected": -0.6586679220199585, "logps/chosen": -285.546630859375, "logps/ref_chosen": -285.5502014160156, "logps/ref_rejected": -267.99664306640625, "logps/rejected": -267.988525390625, "loss": 5.5454, "margin_dpo/margin_mean": -0.004548460245132446, "margin_dpo/margin_std": 0.7229992747306824, "step": 9 }, { "KL/chosen_KL_mean": 0.06154441833496094, "KL/mean": 0.019467219710350037, "KL/rejected_KL_mean": -0.022617340087890625, "KL/std": 0.4935854971408844, "epoch": 0.020942408376963352, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08416536450386047, "fcm_dpo/q_t": 0.49978959560394287, "grad_norm": 26.521408081054688, "learning_rate": 9.375e-08, "logits/chosen": -0.6825000047683716, "logits/rejected": -0.6766196489334106, "logps/chosen": -251.850830078125, "logps/ref_chosen": -251.91238403320312, "logps/ref_rejected": -226.45260620117188, "logps/rejected": -226.4752197265625, "loss": 5.5419, "margin_dpo/margin_mean": 0.08416460454463959, "margin_dpo/margin_std": 0.7020270228385925, "step": 10 }, { "KL/chosen_KL_mean": 0.02030181884765625, "KL/mean": 0.06047465652227402, "KL/rejected_KL_mean": 0.10064697265625, "KL/std": 0.5014757513999939, "epoch": 0.023036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08033938705921173, "fcm_dpo/q_t": 0.5002008080482483, "grad_norm": 28.9848575592041, "learning_rate": 1.0416666666666667e-07, "logits/chosen": -0.5878165364265442, "logits/rejected": -0.6417873501777649, "logps/chosen": -301.06317138671875, "logps/ref_chosen": -301.08343505859375, "logps/ref_rejected": -259.546630859375, "logps/rejected": -259.44598388671875, "loss": 5.5484, "margin_dpo/margin_mean": -0.08033882081508636, "margin_dpo/margin_std": 0.6635822653770447, "step": 11 }, { "KL/chosen_KL_mean": -0.0308685302734375, "KL/mean": -0.08398044854402542, "KL/rejected_KL_mean": -0.1370849609375, "KL/std": 0.5727797746658325, "epoch": 0.025130890052356022, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1062176376581192, "fcm_dpo/q_t": 0.4997345209121704, "grad_norm": 30.093599319458008, "learning_rate": 1.1458333333333332e-07, "logits/chosen": -0.5787484645843506, "logits/rejected": -0.538584291934967, "logps/chosen": -287.5789794921875, "logps/ref_chosen": -287.548095703125, "logps/ref_rejected": -277.37945556640625, "logps/rejected": -277.51654052734375, "loss": 5.541, "margin_dpo/margin_mean": 0.10621786117553711, "margin_dpo/margin_std": 0.8124967813491821, "step": 12 }, { "KL/chosen_KL_mean": 0.04111671447753906, "KL/mean": -0.03685136139392853, "KL/rejected_KL_mean": -0.11482620239257812, "KL/std": 0.49719753861427307, "epoch": 0.027225130890052355, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15594592690467834, "fcm_dpo/q_t": 0.49961015582084656, "grad_norm": 27.183223724365234, "learning_rate": 1.25e-07, "logits/chosen": -0.6483482122421265, "logits/rejected": -0.6553612947463989, "logps/chosen": -270.62530517578125, "logps/ref_chosen": -270.6664123535156, "logps/ref_rejected": -274.6546936035156, "logps/rejected": -274.76953125, "loss": 5.539, "margin_dpo/margin_mean": 0.15594591200351715, "margin_dpo/margin_std": 0.6842634677886963, "step": 13 }, { "KL/chosen_KL_mean": 0.08417701721191406, "KL/mean": 0.03670407831668854, "KL/rejected_KL_mean": -0.010768890380859375, "KL/std": 0.5663655400276184, "epoch": 0.02931937172774869, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09495711326599121, "fcm_dpo/q_t": 0.4997625946998596, "grad_norm": 28.36115264892578, "learning_rate": 1.3541666666666666e-07, "logits/chosen": -0.6208177208900452, "logits/rejected": -0.6492733955383301, "logps/chosen": -281.5090637207031, "logps/ref_chosen": -281.59320068359375, "logps/ref_rejected": -263.52215576171875, "logps/rejected": -263.532958984375, "loss": 5.5414, "margin_dpo/margin_mean": 0.09495654702186584, "margin_dpo/margin_std": 0.7788461446762085, "step": 14 }, { "KL/chosen_KL_mean": -0.000659942626953125, "KL/mean": 0.006451621651649475, "KL/rejected_KL_mean": 0.013570785522460938, "KL/std": 0.5095148086547852, "epoch": 0.031413612565445025, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014226287603378296, "fcm_dpo/q_t": 0.5000355839729309, "grad_norm": 30.425567626953125, "learning_rate": 1.4583333333333335e-07, "logits/chosen": -0.6453888416290283, "logits/rejected": -0.6586918830871582, "logps/chosen": -298.4541015625, "logps/ref_chosen": -298.45343017578125, "logps/ref_rejected": -227.17832946777344, "logps/rejected": -227.16476440429688, "loss": 5.5458, "margin_dpo/margin_mean": -0.014227613806724548, "margin_dpo/margin_std": 0.7101153135299683, "step": 15 }, { "KL/chosen_KL_mean": 0.062591552734375, "KL/mean": 0.027167625725269318, "KL/rejected_KL_mean": -0.008253097534179688, "KL/std": 0.5141140222549438, "epoch": 0.033507853403141365, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07084225118160248, "fcm_dpo/q_t": 0.4998228847980499, "grad_norm": 30.161983489990234, "learning_rate": 1.5624999999999999e-07, "logits/chosen": -0.6102164387702942, "logits/rejected": -0.60810387134552, "logps/chosen": -293.9040222167969, "logps/ref_chosen": -293.96661376953125, "logps/ref_rejected": -250.78443908691406, "logps/rejected": -250.79269409179688, "loss": 5.5424, "margin_dpo/margin_mean": 0.07084193825721741, "margin_dpo/margin_std": 0.7234665751457214, "step": 16 }, { "KL/chosen_KL_mean": 0.08296775817871094, "KL/mean": 0.07325167208909988, "KL/rejected_KL_mean": 0.06353187561035156, "KL/std": 0.48913800716400146, "epoch": 0.0356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.019437596201896667, "fcm_dpo/q_t": 0.49995142221450806, "grad_norm": 27.688617706298828, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5614761114120483, "logits/rejected": -0.5870969891548157, "logps/chosen": -262.3110656738281, "logps/ref_chosen": -262.39398193359375, "logps/ref_rejected": -248.500244140625, "logps/rejected": -248.43670654296875, "loss": 5.5444, "margin_dpo/margin_mean": 0.019437864422798157, "margin_dpo/margin_std": 0.6257155537605286, "step": 17 }, { "KL/chosen_KL_mean": -0.00669097900390625, "KL/mean": -0.022606775164604187, "KL/rejected_KL_mean": -0.038524627685546875, "KL/std": 0.49792757630348206, "epoch": 0.03769633507853403, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03182762861251831, "fcm_dpo/q_t": 0.49992042779922485, "grad_norm": 29.69187355041504, "learning_rate": 1.7708333333333334e-07, "logits/chosen": -0.6175638437271118, "logits/rejected": -0.6237700581550598, "logps/chosen": -293.7159423828125, "logps/ref_chosen": -293.709228515625, "logps/ref_rejected": -274.5875244140625, "logps/rejected": -274.6260681152344, "loss": 5.544, "margin_dpo/margin_mean": 0.03182883560657501, "margin_dpo/margin_std": 0.6748430728912354, "step": 18 }, { "KL/chosen_KL_mean": 0.03246307373046875, "KL/mean": 0.049572668969631195, "KL/rejected_KL_mean": 0.06668472290039062, "KL/std": 0.5367411375045776, "epoch": 0.039790575916230364, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.034227266907691956, "fcm_dpo/q_t": 0.5000855326652527, "grad_norm": 28.177858352661133, "learning_rate": 1.875e-07, "logits/chosen": -0.6283941268920898, "logits/rejected": -0.6246621608734131, "logps/chosen": -280.23321533203125, "logps/ref_chosen": -280.26568603515625, "logps/ref_rejected": -259.9742736816406, "logps/rejected": -259.9075927734375, "loss": 5.5466, "margin_dpo/margin_mean": -0.03422766923904419, "margin_dpo/margin_std": 0.7041956782341003, "step": 19 }, { "KL/chosen_KL_mean": 0.17637062072753906, "KL/mean": 0.1533583253622055, "KL/rejected_KL_mean": 0.1303424835205078, "KL/std": 0.54378741979599, "epoch": 0.041884816753926704, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04602688550949097, "fcm_dpo/q_t": 0.4998849332332611, "grad_norm": 29.69119644165039, "learning_rate": 1.9791666666666664e-07, "logits/chosen": -0.6134122610092163, "logits/rejected": -0.6440813541412354, "logps/chosen": -303.71905517578125, "logps/ref_chosen": -303.8954162597656, "logps/ref_rejected": -260.214599609375, "logps/rejected": -260.0842590332031, "loss": 5.5434, "margin_dpo/margin_mean": 0.04602590203285217, "margin_dpo/margin_std": 0.8291985392570496, "step": 20 }, { "KL/chosen_KL_mean": 0.09269332885742188, "KL/mean": 0.03890705853700638, "KL/rejected_KL_mean": -0.01488494873046875, "KL/std": 0.5381225347518921, "epoch": 0.04397905759162304, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10758335888385773, "fcm_dpo/q_t": 0.49973100423812866, "grad_norm": 34.960567474365234, "learning_rate": 2.0833333333333333e-07, "logits/chosen": -0.6435227394104004, "logits/rejected": -0.671474277973175, "logps/chosen": -301.4407958984375, "logps/ref_chosen": -301.5334777832031, "logps/ref_rejected": -280.28900146484375, "logps/rejected": -280.3038635253906, "loss": 5.5409, "margin_dpo/margin_mean": 0.10758353769779205, "margin_dpo/margin_std": 0.7066560387611389, "step": 21 }, { "KL/chosen_KL_mean": 0.023479461669921875, "KL/mean": 0.034437984228134155, "KL/rejected_KL_mean": 0.04539680480957031, "KL/std": 0.5244945287704468, "epoch": 0.04607329842931937, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0219249427318573, "fcm_dpo/q_t": 0.5000548362731934, "grad_norm": 25.298784255981445, "learning_rate": 2.1875e-07, "logits/chosen": -0.6568877696990967, "logits/rejected": -0.658561110496521, "logps/chosen": -259.9716491699219, "logps/ref_chosen": -259.9951477050781, "logps/ref_rejected": -243.0721435546875, "logps/rejected": -243.02676391601562, "loss": 5.5461, "margin_dpo/margin_mean": -0.021924346685409546, "margin_dpo/margin_std": 0.713553249835968, "step": 22 }, { "KL/chosen_KL_mean": 0.09302520751953125, "KL/mean": 0.048005327582359314, "KL/rejected_KL_mean": 0.0029850006103515625, "KL/std": 0.5733331441879272, "epoch": 0.048167539267015703, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09003856778144836, "fcm_dpo/q_t": 0.4997749328613281, "grad_norm": 28.17792320251465, "learning_rate": 2.2916666666666663e-07, "logits/chosen": -0.6223236918449402, "logits/rejected": -0.6566183567047119, "logps/chosen": -282.0877380371094, "logps/ref_chosen": -282.1807556152344, "logps/ref_rejected": -265.0758056640625, "logps/rejected": -265.07281494140625, "loss": 5.5416, "margin_dpo/margin_mean": 0.09003835916519165, "margin_dpo/margin_std": 0.8055158853530884, "step": 23 }, { "KL/chosen_KL_mean": 0.18692398071289062, "KL/mean": 0.0784425437450409, "KL/rejected_KL_mean": -0.0300445556640625, "KL/std": 0.5381811857223511, "epoch": 0.050261780104712044, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21695753931999207, "fcm_dpo/q_t": 0.49945762753486633, "grad_norm": 29.881175994873047, "learning_rate": 2.3958333333333335e-07, "logits/chosen": -0.6491536498069763, "logits/rejected": -0.5612991452217102, "logps/chosen": -300.99267578125, "logps/ref_chosen": -301.17962646484375, "logps/ref_rejected": -302.12786865234375, "logps/rejected": -302.15789794921875, "loss": 5.5366, "margin_dpo/margin_mean": 0.21695764362812042, "margin_dpo/margin_std": 0.681999921798706, "step": 24 }, { "KL/chosen_KL_mean": 0.1545238494873047, "KL/mean": 0.11132223159074783, "KL/rejected_KL_mean": 0.06811237335205078, "KL/std": 0.5142117738723755, "epoch": 0.05235602094240838, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08640654385089874, "fcm_dpo/q_t": 0.499783992767334, "grad_norm": 27.276826858520508, "learning_rate": 2.5e-07, "logits/chosen": -0.5908087491989136, "logits/rejected": -0.5986031293869019, "logps/chosen": -246.59197998046875, "logps/ref_chosen": -246.74649047851562, "logps/ref_rejected": -235.55638122558594, "logps/rejected": -235.48825073242188, "loss": 5.5418, "margin_dpo/margin_mean": 0.08640624582767487, "margin_dpo/margin_std": 0.6666019558906555, "step": 25 }, { "KL/chosen_KL_mean": 0.22557830810546875, "KL/mean": 0.16001597046852112, "KL/rejected_KL_mean": 0.09445762634277344, "KL/std": 0.6255167126655579, "epoch": 0.05445026178010471, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13112373650074005, "fcm_dpo/q_t": 0.49967217445373535, "grad_norm": 28.602731704711914, "learning_rate": 2.604166666666667e-07, "logits/chosen": -0.646224319934845, "logits/rejected": -0.6599107384681702, "logps/chosen": -281.97003173828125, "logps/ref_chosen": -282.1955871582031, "logps/ref_rejected": -235.3135528564453, "logps/rejected": -235.21910095214844, "loss": 5.54, "margin_dpo/margin_mean": 0.13112422823905945, "margin_dpo/margin_std": 0.8185635805130005, "step": 26 }, { "KL/chosen_KL_mean": 0.3226490020751953, "KL/mean": 0.2239154875278473, "KL/rejected_KL_mean": 0.1251850128173828, "KL/std": 0.6283989548683167, "epoch": 0.05654450261780105, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19746924936771393, "fcm_dpo/q_t": 0.4995063543319702, "grad_norm": 27.78577423095703, "learning_rate": 2.708333333333333e-07, "logits/chosen": -0.6513367891311646, "logits/rejected": -0.6705080270767212, "logps/chosen": -323.53369140625, "logps/ref_chosen": -323.8563537597656, "logps/ref_rejected": -245.968017578125, "logps/rejected": -245.84283447265625, "loss": 5.5374, "margin_dpo/margin_mean": 0.19746987521648407, "margin_dpo/margin_std": 0.8737579584121704, "step": 27 }, { "KL/chosen_KL_mean": 0.2768516540527344, "KL/mean": 0.19153092801570892, "KL/rejected_KL_mean": 0.10620689392089844, "KL/std": 0.5605363845825195, "epoch": 0.05863874345549738, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1706477254629135, "fcm_dpo/q_t": 0.49957337975502014, "grad_norm": 26.49478530883789, "learning_rate": 2.8125e-07, "logits/chosen": -0.6174635887145996, "logits/rejected": -0.6264654397964478, "logps/chosen": -247.9698944091797, "logps/ref_chosen": -248.24673461914062, "logps/ref_rejected": -240.0382080078125, "logps/rejected": -239.9320068359375, "loss": 5.5384, "margin_dpo/margin_mean": 0.17064793407917023, "margin_dpo/margin_std": 0.7738831043243408, "step": 28 }, { "KL/chosen_KL_mean": 0.29106712341308594, "KL/mean": 0.19878274202346802, "KL/rejected_KL_mean": 0.10648536682128906, "KL/std": 0.5895746350288391, "epoch": 0.060732984293193716, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18458740413188934, "fcm_dpo/q_t": 0.4995385706424713, "grad_norm": 29.785554885864258, "learning_rate": 2.916666666666667e-07, "logits/chosen": -0.5842040777206421, "logits/rejected": -0.6051267385482788, "logps/chosen": -317.9653625488281, "logps/ref_chosen": -318.2564392089844, "logps/ref_rejected": -286.75848388671875, "logps/rejected": -286.65203857421875, "loss": 5.5379, "margin_dpo/margin_mean": 0.18458794057369232, "margin_dpo/margin_std": 0.7843542098999023, "step": 29 }, { "KL/chosen_KL_mean": 0.3651390075683594, "KL/mean": 0.22121500968933105, "KL/rejected_KL_mean": 0.07729148864746094, "KL/std": 0.6174904704093933, "epoch": 0.06282722513089005, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.28784090280532837, "fcm_dpo/q_t": 0.4992803931236267, "grad_norm": 28.858478546142578, "learning_rate": 3.020833333333333e-07, "logits/chosen": -0.6003649830818176, "logits/rejected": -0.6176969408988953, "logps/chosen": -252.68399047851562, "logps/ref_chosen": -253.0491485595703, "logps/ref_rejected": -261.30029296875, "logps/rejected": -261.2229919433594, "loss": 5.5337, "margin_dpo/margin_mean": 0.2878407835960388, "margin_dpo/margin_std": 0.8439157605171204, "step": 30 }, { "KL/chosen_KL_mean": 0.41502952575683594, "KL/mean": 0.2872907519340515, "KL/rejected_KL_mean": 0.1595611572265625, "KL/std": 0.7194770574569702, "epoch": 0.06492146596858639, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25546789169311523, "fcm_dpo/q_t": 0.4993613660335541, "grad_norm": 25.050519943237305, "learning_rate": 3.1249999999999997e-07, "logits/chosen": -0.6552233099937439, "logits/rejected": -0.6901096105575562, "logps/chosen": -247.73800659179688, "logps/ref_chosen": -248.15301513671875, "logps/ref_rejected": -203.17703247070312, "logps/rejected": -203.01748657226562, "loss": 5.5351, "margin_dpo/margin_mean": 0.25546786189079285, "margin_dpo/margin_std": 0.900585949420929, "step": 31 }, { "KL/chosen_KL_mean": 0.45583343505859375, "KL/mean": 0.34075260162353516, "KL/rejected_KL_mean": 0.22568130493164062, "KL/std": 0.723097026348114, "epoch": 0.06701570680628273, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2301485389471054, "fcm_dpo/q_t": 0.49942466616630554, "grad_norm": 29.70162582397461, "learning_rate": 3.2291666666666666e-07, "logits/chosen": -0.6078366637229919, "logits/rejected": -0.6122292876243591, "logps/chosen": -305.0841369628906, "logps/ref_chosen": -305.5399475097656, "logps/ref_rejected": -267.6527099609375, "logps/rejected": -267.4270324707031, "loss": 5.5361, "margin_dpo/margin_mean": 0.23014913499355316, "margin_dpo/margin_std": 0.9483384490013123, "step": 32 }, { "KL/chosen_KL_mean": 0.5750503540039062, "KL/mean": 0.37175193428993225, "KL/rejected_KL_mean": 0.16846084594726562, "KL/std": 0.9059445858001709, "epoch": 0.06910994764397906, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4065783619880676, "fcm_dpo/q_t": 0.49898362159729004, "grad_norm": 28.4556941986084, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.640916109085083, "logits/rejected": -0.6534519195556641, "logps/chosen": -285.6584777832031, "logps/ref_chosen": -286.2335205078125, "logps/ref_rejected": -255.38748168945312, "logps/rejected": -255.2190399169922, "loss": 5.5291, "margin_dpo/margin_mean": 0.4065782427787781, "margin_dpo/margin_std": 1.132638931274414, "step": 33 }, { "KL/chosen_KL_mean": 0.7782306671142578, "KL/mean": 0.35225048661231995, "KL/rejected_KL_mean": -0.07373237609863281, "KL/std": 1.0198003053665161, "epoch": 0.0712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8519618511199951, "fcm_dpo/q_t": 0.49787020683288574, "grad_norm": 31.113811492919922, "learning_rate": 3.4375e-07, "logits/chosen": -0.6230222582817078, "logits/rejected": -0.6323559284210205, "logps/chosen": -340.8137512207031, "logps/ref_chosen": -341.5920104980469, "logps/ref_rejected": -278.8866882324219, "logps/rejected": -278.9604187011719, "loss": 5.5113, "margin_dpo/margin_mean": 0.8519620299339294, "margin_dpo/margin_std": 1.140161156654358, "step": 34 }, { "KL/chosen_KL_mean": 0.6100578308105469, "KL/mean": 0.45359697937965393, "KL/rejected_KL_mean": 0.2971305847167969, "KL/std": 0.9627859592437744, "epoch": 0.07329842931937172, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.31292349100112915, "fcm_dpo/q_t": 0.4992177188396454, "grad_norm": 26.534446716308594, "learning_rate": 3.541666666666667e-07, "logits/chosen": -0.6332972049713135, "logits/rejected": -0.6537303924560547, "logps/chosen": -264.469482421875, "logps/ref_chosen": -265.0795593261719, "logps/ref_rejected": -264.4876708984375, "logps/rejected": -264.1905517578125, "loss": 5.5328, "margin_dpo/margin_mean": 0.3129233419895172, "margin_dpo/margin_std": 1.3062032461166382, "step": 35 }, { "KL/chosen_KL_mean": 0.8854637145996094, "KL/mean": 0.5023590326309204, "KL/rejected_KL_mean": 0.11924552917480469, "KL/std": 1.133253812789917, "epoch": 0.07539267015706806, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7662180662155151, "fcm_dpo/q_t": 0.4980846047401428, "grad_norm": 31.921070098876953, "learning_rate": 3.645833333333333e-07, "logits/chosen": -0.6024245023727417, "logits/rejected": -0.6200557947158813, "logps/chosen": -296.440673828125, "logps/ref_chosen": -297.3261413574219, "logps/ref_rejected": -282.09515380859375, "logps/rejected": -281.97589111328125, "loss": 5.5148, "margin_dpo/margin_mean": 0.7662180066108704, "margin_dpo/margin_std": 1.4552934169769287, "step": 36 }, { "KL/chosen_KL_mean": 0.7702865600585938, "KL/mean": 0.4680500328540802, "KL/rejected_KL_mean": 0.1658344268798828, "KL/std": 1.1320182085037231, "epoch": 0.0774869109947644, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6044507026672363, "fcm_dpo/q_t": 0.4984889626502991, "grad_norm": 31.38983917236328, "learning_rate": 3.75e-07, "logits/chosen": -0.6020391583442688, "logits/rejected": -0.6165008544921875, "logps/chosen": -313.26373291015625, "logps/ref_chosen": -314.0340270996094, "logps/ref_rejected": -299.3437805175781, "logps/rejected": -299.1779479980469, "loss": 5.5213, "margin_dpo/margin_mean": 0.6044489741325378, "margin_dpo/margin_std": 1.5211684703826904, "step": 37 }, { "KL/chosen_KL_mean": 1.0551738739013672, "KL/mean": 0.7106471061706543, "KL/rejected_KL_mean": 0.36612892150878906, "KL/std": 1.2290730476379395, "epoch": 0.07958115183246073, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.689054012298584, "fcm_dpo/q_t": 0.4982774555683136, "grad_norm": 28.488887786865234, "learning_rate": 3.8541666666666665e-07, "logits/chosen": -0.6347458362579346, "logits/rejected": -0.6467496156692505, "logps/chosen": -281.48602294921875, "logps/ref_chosen": -282.54119873046875, "logps/ref_rejected": -269.7773132324219, "logps/rejected": -269.4111633300781, "loss": 5.5179, "margin_dpo/margin_mean": 0.6890544891357422, "margin_dpo/margin_std": 1.4637418985366821, "step": 38 }, { "KL/chosen_KL_mean": 1.2901058197021484, "KL/mean": 0.679212212562561, "KL/rejected_KL_mean": 0.06832122802734375, "KL/std": 1.531196117401123, "epoch": 0.08167539267015707, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2217795848846436, "fcm_dpo/q_t": 0.49694594740867615, "grad_norm": 29.57316017150879, "learning_rate": 3.958333333333333e-07, "logits/chosen": -0.6168429851531982, "logits/rejected": -0.6308571100234985, "logps/chosen": -275.48284912109375, "logps/ref_chosen": -276.7729187011719, "logps/ref_rejected": -249.95889282226562, "logps/rejected": -249.89056396484375, "loss": 5.4968, "margin_dpo/margin_mean": 1.221779465675354, "margin_dpo/margin_std": 1.7274365425109863, "step": 39 }, { "KL/chosen_KL_mean": 1.0404300689697266, "KL/mean": 0.7055494785308838, "KL/rejected_KL_mean": 0.3706798553466797, "KL/std": 1.3352773189544678, "epoch": 0.08376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6697390675544739, "fcm_dpo/q_t": 0.4983258545398712, "grad_norm": 27.396766662597656, "learning_rate": 4.0625e-07, "logits/chosen": -0.6229257583618164, "logits/rejected": -0.6586186289787292, "logps/chosen": -283.26666259765625, "logps/ref_chosen": -284.30706787109375, "logps/ref_rejected": -244.4459991455078, "logps/rejected": -244.07530212402344, "loss": 5.5188, "margin_dpo/margin_mean": 0.6697392463684082, "margin_dpo/margin_std": 1.7444779872894287, "step": 40 }, { "KL/chosen_KL_mean": 1.0908069610595703, "KL/mean": 0.7018202543258667, "KL/rejected_KL_mean": 0.31284141540527344, "KL/std": 1.4677762985229492, "epoch": 0.08586387434554973, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7779622077941895, "fcm_dpo/q_t": 0.49805524945259094, "grad_norm": 30.210500717163086, "learning_rate": 4.1666666666666667e-07, "logits/chosen": -0.6286399364471436, "logits/rejected": -0.6542515754699707, "logps/chosen": -292.724365234375, "logps/ref_chosen": -293.8151550292969, "logps/ref_rejected": -252.16815185546875, "logps/rejected": -251.85531616210938, "loss": 5.5144, "margin_dpo/margin_mean": 0.7779629826545715, "margin_dpo/margin_std": 1.7769787311553955, "step": 41 }, { "KL/chosen_KL_mean": 1.2728042602539062, "KL/mean": 0.7940388917922974, "KL/rejected_KL_mean": 0.31528282165527344, "KL/std": 1.7132642269134521, "epoch": 0.08795811518324607, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9575194716453552, "fcm_dpo/q_t": 0.4976065456867218, "grad_norm": 27.403196334838867, "learning_rate": 4.270833333333333e-07, "logits/chosen": -0.6356105804443359, "logits/rejected": -0.6535137891769409, "logps/chosen": -251.4874725341797, "logps/ref_chosen": -252.76023864746094, "logps/ref_rejected": -261.0414733886719, "logps/rejected": -260.7261962890625, "loss": 5.5075, "margin_dpo/margin_mean": 0.95751953125, "margin_dpo/margin_std": 2.236936092376709, "step": 42 }, { "KL/chosen_KL_mean": 1.2807998657226562, "KL/mean": 0.6640318632125854, "KL/rejected_KL_mean": 0.04726409912109375, "KL/std": 1.9099464416503906, "epoch": 0.09005235602094241, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.233534812927246, "fcm_dpo/q_t": 0.4969165027141571, "grad_norm": 29.962818145751953, "learning_rate": 4.375e-07, "logits/chosen": -0.6014646887779236, "logits/rejected": -0.6178789138793945, "logps/chosen": -315.553955078125, "logps/ref_chosen": -316.8347473144531, "logps/ref_rejected": -273.7649230957031, "logps/rejected": -273.7176513671875, "loss": 5.4966, "margin_dpo/margin_mean": 1.2335340976715088, "margin_dpo/margin_std": 2.2836263179779053, "step": 43 }, { "KL/chosen_KL_mean": 1.5793323516845703, "KL/mean": 0.7775524854660034, "KL/rejected_KL_mean": -0.02422332763671875, "KL/std": 2.20059871673584, "epoch": 0.09214659685863874, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6035535335540771, "fcm_dpo/q_t": 0.49599185585975647, "grad_norm": 30.827856063842773, "learning_rate": 4.479166666666667e-07, "logits/chosen": -0.5976197719573975, "logits/rejected": -0.5963581800460815, "logps/chosen": -285.2963562011719, "logps/ref_chosen": -286.8757019042969, "logps/ref_rejected": -282.4681396484375, "logps/rejected": -282.49237060546875, "loss": 5.4822, "margin_dpo/margin_mean": 1.6035535335540771, "margin_dpo/margin_std": 2.972196340560913, "step": 44 }, { "KL/chosen_KL_mean": 1.7299308776855469, "KL/mean": 1.0418126583099365, "KL/rejected_KL_mean": 0.35369300842285156, "KL/std": 2.250051975250244, "epoch": 0.09424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3762341737747192, "fcm_dpo/q_t": 0.4965605139732361, "grad_norm": 28.878467559814453, "learning_rate": 4.5833333333333327e-07, "logits/chosen": -0.688926100730896, "logits/rejected": -0.7126062512397766, "logps/chosen": -322.533447265625, "logps/ref_chosen": -324.2633972167969, "logps/ref_rejected": -293.09466552734375, "logps/rejected": -292.740966796875, "loss": 5.4912, "margin_dpo/margin_mean": 1.376233696937561, "margin_dpo/margin_std": 2.813117742538452, "step": 45 }, { "KL/chosen_KL_mean": 1.6946258544921875, "KL/mean": 0.9494956135749817, "KL/rejected_KL_mean": 0.20435523986816406, "KL/std": 2.433075189590454, "epoch": 0.09633507853403141, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4902722835540771, "fcm_dpo/q_t": 0.4962751865386963, "grad_norm": 30.186275482177734, "learning_rate": 4.6874999999999996e-07, "logits/chosen": -0.6262366771697998, "logits/rejected": -0.6397067904472351, "logps/chosen": -296.64111328125, "logps/ref_chosen": -298.3357238769531, "logps/ref_rejected": -267.66204833984375, "logps/rejected": -267.4576721191406, "loss": 5.4865, "margin_dpo/margin_mean": 1.4902722835540771, "margin_dpo/margin_std": 2.6397438049316406, "step": 46 }, { "KL/chosen_KL_mean": 1.5152873992919922, "KL/mean": 1.0190459489822388, "KL/rejected_KL_mean": 0.5227985382080078, "KL/std": 2.292161464691162, "epoch": 0.09842931937172775, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9924949407577515, "fcm_dpo/q_t": 0.4975191056728363, "grad_norm": 26.3062801361084, "learning_rate": 4.791666666666667e-07, "logits/chosen": -0.6064034700393677, "logits/rejected": -0.6271387934684753, "logps/chosen": -261.0516357421875, "logps/ref_chosen": -262.5669250488281, "logps/ref_rejected": -258.70989990234375, "logps/rejected": -258.18707275390625, "loss": 5.5068, "margin_dpo/margin_mean": 0.9924947619438171, "margin_dpo/margin_std": 3.2504637241363525, "step": 47 }, { "KL/chosen_KL_mean": 1.820608139038086, "KL/mean": 1.0710346698760986, "KL/rejected_KL_mean": 0.32146644592285156, "KL/std": 2.579556465148926, "epoch": 0.10052356020942409, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4991388320922852, "fcm_dpo/q_t": 0.4962541460990906, "grad_norm": 27.537498474121094, "learning_rate": 4.895833333333333e-07, "logits/chosen": -0.6123904585838318, "logits/rejected": -0.637353777885437, "logps/chosen": -267.67266845703125, "logps/ref_chosen": -269.4932556152344, "logps/ref_rejected": -241.888916015625, "logps/rejected": -241.56744384765625, "loss": 5.4865, "margin_dpo/margin_mean": 1.4991381168365479, "margin_dpo/margin_std": 2.9511523246765137, "step": 48 }, { "KL/chosen_KL_mean": 2.2462005615234375, "KL/mean": 1.3681857585906982, "KL/rejected_KL_mean": 0.49018096923828125, "KL/std": 2.621547222137451, "epoch": 0.10261780104712041, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7560153007507324, "fcm_dpo/q_t": 0.4956113398075104, "grad_norm": 27.621536254882812, "learning_rate": 5e-07, "logits/chosen": -0.664618194103241, "logits/rejected": -0.6490676999092102, "logps/chosen": -255.63827514648438, "logps/ref_chosen": -257.8844909667969, "logps/ref_rejected": -256.8912048339844, "logps/rejected": -256.4010009765625, "loss": 5.4765, "margin_dpo/margin_mean": 1.7560160160064697, "margin_dpo/margin_std": 3.3718807697296143, "step": 49 }, { "KL/chosen_KL_mean": 1.9965705871582031, "KL/mean": 1.436202883720398, "KL/rejected_KL_mean": 0.8758354187011719, "KL/std": 2.9903759956359863, "epoch": 0.10471204188481675, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1207308769226074, "fcm_dpo/q_t": 0.4971994161605835, "grad_norm": 27.850839614868164, "learning_rate": 4.999932966293553e-07, "logits/chosen": -0.6214494109153748, "logits/rejected": -0.642817497253418, "logps/chosen": -299.63226318359375, "logps/ref_chosen": -301.62884521484375, "logps/ref_rejected": -298.2716064453125, "logps/rejected": -297.3957824707031, "loss": 5.5021, "margin_dpo/margin_mean": 1.1207301616668701, "margin_dpo/margin_std": 3.682468891143799, "step": 50 }, { "KL/chosen_KL_mean": 2.1178417205810547, "KL/mean": 1.1750433444976807, "KL/rejected_KL_mean": 0.23224449157714844, "KL/std": 3.547469139099121, "epoch": 0.1068062827225131, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8856074810028076, "fcm_dpo/q_t": 0.49528858065605164, "grad_norm": 29.11636734008789, "learning_rate": 4.999731868769026e-07, "logits/chosen": -0.6039535403251648, "logits/rejected": -0.5983304381370544, "logps/chosen": -267.2545166015625, "logps/ref_chosen": -269.37237548828125, "logps/ref_rejected": -297.0167541503906, "logps/rejected": -296.7845153808594, "loss": 5.4725, "margin_dpo/margin_mean": 1.8856079578399658, "margin_dpo/margin_std": 4.448060512542725, "step": 51 }, { "KL/chosen_KL_mean": 2.7982959747314453, "KL/mean": 1.2421739101409912, "KL/rejected_KL_mean": -0.3139476776123047, "KL/std": 3.763486862182617, "epoch": 0.10890052356020942, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1122491359710693, "fcm_dpo/q_t": 0.49222618341445923, "grad_norm": 30.2871150970459, "learning_rate": 4.99939671821067e-07, "logits/chosen": -0.6552134156227112, "logits/rejected": -0.6618373990058899, "logps/chosen": -304.1044921875, "logps/ref_chosen": -306.9028015136719, "logps/ref_rejected": -281.24737548828125, "logps/rejected": -281.561279296875, "loss": 5.4241, "margin_dpo/margin_mean": 3.1122488975524902, "margin_dpo/margin_std": 4.741540431976318, "step": 52 }, { "KL/chosen_KL_mean": 2.7981128692626953, "KL/mean": 1.7105417251586914, "KL/rejected_KL_mean": 0.6229763031005859, "KL/std": 4.075300216674805, "epoch": 0.11099476439790576, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.1751344203948975, "fcm_dpo/q_t": 0.49456626176834106, "grad_norm": 30.046728134155273, "learning_rate": 4.998927532591591e-07, "logits/chosen": -0.6532727479934692, "logits/rejected": -0.6938633322715759, "logps/chosen": -283.1778564453125, "logps/ref_chosen": -285.9759521484375, "logps/ref_rejected": -273.9073486328125, "logps/rejected": -273.28436279296875, "loss": 5.4616, "margin_dpo/margin_mean": 2.17513370513916, "margin_dpo/margin_std": 5.007012367248535, "step": 53 }, { "KL/chosen_KL_mean": 2.679128646850586, "KL/mean": 1.732741355895996, "KL/rejected_KL_mean": 0.7863483428955078, "KL/std": 4.326586723327637, "epoch": 0.1130890052356021, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8927712440490723, "fcm_dpo/q_t": 0.4952733814716339, "grad_norm": 26.467914581298828, "learning_rate": 4.998324337072792e-07, "logits/chosen": -0.690001368522644, "logits/rejected": -0.6982678174972534, "logps/chosen": -303.8254699707031, "logps/ref_chosen": -306.504638671875, "logps/ref_rejected": -272.67431640625, "logps/rejected": -271.8879699707031, "loss": 5.473, "margin_dpo/margin_mean": 1.89277184009552, "margin_dpo/margin_std": 5.428363800048828, "step": 54 }, { "KL/chosen_KL_mean": 1.8250560760498047, "KL/mean": 0.6974931955337524, "KL/rejected_KL_mean": -0.43007469177246094, "KL/std": 3.710113525390625, "epoch": 0.11518324607329843, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.255131244659424, "fcm_dpo/q_t": 0.49436742067337036, "grad_norm": 24.798433303833008, "learning_rate": 4.997587164001815e-07, "logits/chosen": -0.6320837736129761, "logits/rejected": -0.635603129863739, "logps/chosen": -220.50509643554688, "logps/ref_chosen": -222.33013916015625, "logps/ref_rejected": -206.59571838378906, "logps/rejected": -207.0258026123047, "loss": 5.458, "margin_dpo/margin_mean": 2.255131959915161, "margin_dpo/margin_std": 4.947845458984375, "step": 55 }, { "KL/chosen_KL_mean": 2.8548545837402344, "KL/mean": 1.2973169088363647, "KL/rejected_KL_mean": -0.26021766662597656, "KL/std": 4.783367156982422, "epoch": 0.11727748691099477, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1150808334350586, "fcm_dpo/q_t": 0.49222424626350403, "grad_norm": 27.468914031982422, "learning_rate": 4.996716052911017e-07, "logits/chosen": -0.613375723361969, "logits/rejected": -0.6283761262893677, "logps/chosen": -247.62330627441406, "logps/ref_chosen": -250.47816467285156, "logps/ref_rejected": -228.25848388671875, "logps/rejected": -228.51870727539062, "loss": 5.4259, "margin_dpo/margin_mean": 3.115081310272217, "margin_dpo/margin_std": 5.996671676635742, "step": 56 }, { "KL/chosen_KL_mean": 3.899179458618164, "KL/mean": 1.7846465110778809, "KL/rejected_KL_mean": -0.3298816680908203, "KL/std": 4.845379829406738, "epoch": 0.1193717277486911, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.229069709777832, "fcm_dpo/q_t": 0.4894401431083679, "grad_norm": 30.87196159362793, "learning_rate": 4.99571105051544e-07, "logits/chosen": -0.7000694274902344, "logits/rejected": -0.6709764003753662, "logps/chosen": -311.2203369140625, "logps/ref_chosen": -315.1195373535156, "logps/ref_rejected": -272.755615234375, "logps/rejected": -273.0854797363281, "loss": 5.3813, "margin_dpo/margin_mean": 4.229069709777832, "margin_dpo/margin_std": 5.674056529998779, "step": 57 }, { "KL/chosen_KL_mean": 2.4763240814208984, "KL/mean": 1.3078148365020752, "KL/rejected_KL_mean": 0.13930511474609375, "KL/std": 4.4075093269348145, "epoch": 0.12146596858638743, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.337027072906494, "fcm_dpo/q_t": 0.494160920381546, "grad_norm": 27.59241485595703, "learning_rate": 4.994572210710314e-07, "logits/chosen": -0.6156286597251892, "logits/rejected": -0.6398040652275085, "logps/chosen": -262.705322265625, "logps/ref_chosen": -265.1816711425781, "logps/ref_rejected": -268.2203369140625, "logps/rejected": -268.0810241699219, "loss": 5.4557, "margin_dpo/margin_mean": 2.3370275497436523, "margin_dpo/margin_std": 5.594066619873047, "step": 58 }, { "KL/chosen_KL_mean": 2.1144371032714844, "KL/mean": 1.332619071006775, "KL/rejected_KL_mean": 0.5508079528808594, "KL/std": 5.286384582519531, "epoch": 0.12356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5636342763900757, "fcm_dpo/q_t": 0.4960969388484955, "grad_norm": 30.200071334838867, "learning_rate": 4.993299594568162e-07, "logits/chosen": -0.6096245646476746, "logits/rejected": -0.603198766708374, "logps/chosen": -284.2395324707031, "logps/ref_chosen": -286.35394287109375, "logps/ref_rejected": -260.6757507324219, "logps/rejected": -260.12493896484375, "loss": 5.4887, "margin_dpo/margin_mean": 1.5636343955993652, "margin_dpo/margin_std": 7.07401180267334, "step": 59 }, { "KL/chosen_KL_mean": 2.8281993865966797, "KL/mean": 1.1985671520233154, "KL/rejected_KL_mean": -0.43106842041015625, "KL/std": 5.1828532218933105, "epoch": 0.1256544502617801, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.2592642307281494, "fcm_dpo/q_t": 0.49186116456985474, "grad_norm": 28.115793228149414, "learning_rate": 4.991893270335525e-07, "logits/chosen": -0.6829241514205933, "logits/rejected": -0.7077679634094238, "logps/chosen": -255.92039489746094, "logps/ref_chosen": -258.74859619140625, "logps/ref_rejected": -255.04893493652344, "logps/rejected": -255.47998046875, "loss": 5.4214, "margin_dpo/margin_mean": 3.2592647075653076, "margin_dpo/margin_std": 7.173513412475586, "step": 60 }, { "KL/chosen_KL_mean": 3.054656982421875, "KL/mean": 1.4622215032577515, "KL/rejected_KL_mean": -0.13021087646484375, "KL/std": 5.500098705291748, "epoch": 0.12774869109947645, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.184875965118408, "fcm_dpo/q_t": 0.492051899433136, "grad_norm": 29.790935516357422, "learning_rate": 4.990353313429303e-07, "logits/chosen": -0.6545593738555908, "logits/rejected": -0.6720077395439148, "logps/chosen": -275.41326904296875, "logps/ref_chosen": -278.4678955078125, "logps/ref_rejected": -252.02720642089844, "logps/rejected": -252.1573944091797, "loss": 5.4254, "margin_dpo/margin_mean": 3.184875965118408, "margin_dpo/margin_std": 7.548828125, "step": 61 }, { "KL/chosen_KL_mean": 3.975238800048828, "KL/mean": 2.29765248298645, "KL/rejected_KL_mean": 0.6200656890869141, "KL/std": 5.711119174957275, "epoch": 0.12984293193717278, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3551716804504395, "fcm_dpo/q_t": 0.49163132905960083, "grad_norm": 26.7069091796875, "learning_rate": 4.988679806432711e-07, "logits/chosen": -0.6063894629478455, "logits/rejected": -0.6477152705192566, "logps/chosen": -268.9490661621094, "logps/ref_chosen": -272.92431640625, "logps/ref_rejected": -260.7935485839844, "logps/rejected": -260.1734924316406, "loss": 5.4178, "margin_dpo/margin_mean": 3.3551721572875977, "margin_dpo/margin_std": 7.388532638549805, "step": 62 }, { "KL/chosen_KL_mean": 3.386953353881836, "KL/mean": 1.6372132301330566, "KL/rejected_KL_mean": -0.11253166198730469, "KL/std": 5.6143670082092285, "epoch": 0.1319371727748691, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4994750022888184, "fcm_dpo/q_t": 0.4912656843662262, "grad_norm": 28.21631622314453, "learning_rate": 4.986872839090852e-07, "logits/chosen": -0.6588255167007446, "logits/rejected": -0.6650886535644531, "logps/chosen": -273.7020568847656, "logps/ref_chosen": -277.0889892578125, "logps/ref_rejected": -273.3413391113281, "logps/rejected": -273.453857421875, "loss": 5.4121, "margin_dpo/margin_mean": 3.4994747638702393, "margin_dpo/margin_std": 7.25181770324707, "step": 63 }, { "KL/chosen_KL_mean": 4.168355941772461, "KL/mean": 2.0174996852874756, "KL/rejected_KL_mean": -0.13335037231445312, "KL/std": 6.28251314163208, "epoch": 0.13403141361256546, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.301701545715332, "fcm_dpo/q_t": 0.48926711082458496, "grad_norm": 28.383853912353516, "learning_rate": 4.9849325083059e-07, "logits/chosen": -0.6357841491699219, "logits/rejected": -0.6312809586524963, "logps/chosen": -279.6561279296875, "logps/ref_chosen": -283.8244934082031, "logps/ref_rejected": -263.29351806640625, "logps/rejected": -263.4268798828125, "loss": 5.3824, "margin_dpo/margin_mean": 4.301700592041016, "margin_dpo/margin_std": 8.25213623046875, "step": 64 }, { "KL/chosen_KL_mean": 3.4636306762695312, "KL/mean": 1.8112151622772217, "KL/rejected_KL_mean": 0.15880584716796875, "KL/std": 5.756163597106934, "epoch": 0.13612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3048229217529297, "fcm_dpo/q_t": 0.4917513132095337, "grad_norm": 27.825502395629883, "learning_rate": 4.982858918131906e-07, "logits/chosen": -0.7022321820259094, "logits/rejected": -0.6702457070350647, "logps/chosen": -261.40631103515625, "logps/ref_chosen": -264.8699645996094, "logps/ref_rejected": -268.5076904296875, "logps/rejected": -268.348876953125, "loss": 5.4199, "margin_dpo/margin_mean": 3.3048219680786133, "margin_dpo/margin_std": 7.397376537322998, "step": 65 }, { "KL/chosen_KL_mean": 3.1544952392578125, "KL/mean": 1.0874898433685303, "KL/rejected_KL_mean": -0.9795093536376953, "KL/std": 7.427680015563965, "epoch": 0.1382198952879581, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.133997440338135, "fcm_dpo/q_t": 0.4896990954875946, "grad_norm": 27.89104461669922, "learning_rate": 4.980652179769217e-07, "logits/chosen": -0.6843776702880859, "logits/rejected": -0.704654335975647, "logps/chosen": -269.7738342285156, "logps/ref_chosen": -272.9283142089844, "logps/ref_rejected": -280.94696044921875, "logps/rejected": -281.9264831542969, "loss": 5.3922, "margin_dpo/margin_mean": 4.133997917175293, "margin_dpo/margin_std": 10.16850471496582, "step": 66 }, { "KL/chosen_KL_mean": 3.9878292083740234, "KL/mean": 2.169799566268921, "KL/rejected_KL_mean": 0.3517570495605469, "KL/std": 7.131447792053223, "epoch": 0.14031413612565444, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.6360793113708496, "fcm_dpo/q_t": 0.4909287989139557, "grad_norm": 25.531761169433594, "learning_rate": 4.978312411558517e-07, "logits/chosen": -0.6986119747161865, "logits/rejected": -0.7322227954864502, "logps/chosen": -262.1990966796875, "logps/ref_chosen": -266.18695068359375, "logps/ref_rejected": -250.17405700683594, "logps/rejected": -249.82232666015625, "loss": 5.4094, "margin_dpo/margin_mean": 3.6360793113708496, "margin_dpo/margin_std": 8.844635963439941, "step": 67 }, { "KL/chosen_KL_mean": 3.0627079010009766, "KL/mean": 0.719912588596344, "KL/rejected_KL_mean": -1.6228828430175781, "KL/std": 8.057466506958008, "epoch": 0.1424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.685594081878662, "fcm_dpo/q_t": 0.488344669342041, "grad_norm": 27.940174102783203, "learning_rate": 4.975839738974473e-07, "logits/chosen": -0.6814758777618408, "logits/rejected": -0.6943728923797607, "logps/chosen": -294.8758850097656, "logps/ref_chosen": -297.9385986328125, "logps/ref_rejected": -261.5141296386719, "logps/rejected": -263.1369934082031, "loss": 5.3719, "margin_dpo/margin_mean": 4.685595512390137, "margin_dpo/margin_std": 10.544692039489746, "step": 68 }, { "KL/chosen_KL_mean": 3.918996810913086, "KL/mean": 0.8691984415054321, "KL/rejected_KL_mean": -2.1805973052978516, "KL/std": 7.882665634155273, "epoch": 0.14450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.09959602355957, "fcm_dpo/q_t": 0.48479950428009033, "grad_norm": 28.539579391479492, "learning_rate": 4.97323429461901e-07, "logits/chosen": -0.6883444786071777, "logits/rejected": -0.719520092010498, "logps/chosen": -261.69854736328125, "logps/ref_chosen": -265.6175231933594, "logps/ref_rejected": -236.8287353515625, "logps/rejected": -239.00933837890625, "loss": 5.3149, "margin_dpo/margin_mean": 6.09959602355957, "margin_dpo/margin_std": 9.689201354980469, "step": 69 }, { "KL/chosen_KL_mean": 4.362461090087891, "KL/mean": 1.4138339757919312, "KL/rejected_KL_mean": -1.5347957611083984, "KL/std": 8.69215202331543, "epoch": 0.14659685863874344, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.897259712219238, "fcm_dpo/q_t": 0.48531487584114075, "grad_norm": 28.618436813354492, "learning_rate": 4.970496218214204e-07, "logits/chosen": -0.6700032353401184, "logits/rejected": -0.7025509476661682, "logps/chosen": -291.86346435546875, "logps/ref_chosen": -296.2259216308594, "logps/ref_rejected": -254.68496704101562, "logps/rejected": -256.2197570800781, "loss": 5.3258, "margin_dpo/margin_mean": 5.897259712219238, "margin_dpo/margin_std": 10.952218055725098, "step": 70 }, { "KL/chosen_KL_mean": 5.089008331298828, "KL/mean": 2.5183145999908447, "KL/rejected_KL_mean": -0.05237388610839844, "KL/std": 9.370198249816895, "epoch": 0.1486910994764398, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.141389846801758, "fcm_dpo/q_t": 0.4871872365474701, "grad_norm": 28.453304290771484, "learning_rate": 4.967625656594781e-07, "logits/chosen": -0.6482348442077637, "logits/rejected": -0.6368006467819214, "logps/chosen": -283.8382263183594, "logps/ref_chosen": -288.92724609375, "logps/ref_rejected": -278.6405334472656, "logps/rejected": -278.69287109375, "loss": 5.3587, "margin_dpo/margin_mean": 5.141388893127441, "margin_dpo/margin_std": 12.29645824432373, "step": 71 }, { "KL/chosen_KL_mean": 3.3922672271728516, "KL/mean": 0.7981147766113281, "KL/rejected_KL_mean": -1.7960338592529297, "KL/std": 8.441047668457031, "epoch": 0.15078534031413612, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.188295364379883, "fcm_dpo/q_t": 0.48706212639808655, "grad_norm": 28.272216796875, "learning_rate": 4.964622763700252e-07, "logits/chosen": -0.6958415508270264, "logits/rejected": -0.7072215676307678, "logps/chosen": -233.65304565429688, "logps/ref_chosen": -237.0452880859375, "logps/ref_rejected": -252.7946319580078, "logps/rejected": -254.59066772460938, "loss": 5.3528, "margin_dpo/margin_mean": 5.188295364379883, "margin_dpo/margin_std": 10.39709186553955, "step": 72 }, { "KL/chosen_KL_mean": 4.309686660766602, "KL/mean": 1.9369821548461914, "KL/rejected_KL_mean": -0.43572235107421875, "KL/std": 8.945561408996582, "epoch": 0.15287958115183245, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.74540901184082, "fcm_dpo/q_t": 0.4881640076637268, "grad_norm": 27.904314041137695, "learning_rate": 4.961487700566646e-07, "logits/chosen": -0.6588871479034424, "logits/rejected": -0.6765344142913818, "logps/chosen": -268.743408203125, "logps/ref_chosen": -273.0531005859375, "logps/ref_rejected": -246.8330841064453, "logps/rejected": -247.26881408691406, "loss": 5.3739, "margin_dpo/margin_mean": 4.745408058166504, "margin_dpo/margin_std": 12.261763572692871, "step": 73 }, { "KL/chosen_KL_mean": 3.3888206481933594, "KL/mean": 1.1340489387512207, "KL/rejected_KL_mean": -1.1207256317138672, "KL/std": 9.382542610168457, "epoch": 0.1549738219895288, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.5095534324646, "fcm_dpo/q_t": 0.48877227306365967, "grad_norm": 30.305757522583008, "learning_rate": 4.958220635317885e-07, "logits/chosen": -0.7255840301513672, "logits/rejected": -0.7040022611618042, "logps/chosen": -338.8930969238281, "logps/ref_chosen": -342.2818908691406, "logps/ref_rejected": -330.0293884277344, "logps/rejected": -331.15008544921875, "loss": 5.3817, "margin_dpo/margin_mean": 4.509552478790283, "margin_dpo/margin_std": 11.70348834991455, "step": 74 }, { "KL/chosen_KL_mean": 4.573873519897461, "KL/mean": 1.2532798051834106, "KL/rejected_KL_mean": -2.0673141479492188, "KL/std": 9.401840209960938, "epoch": 0.15706806282722513, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.64119291305542, "fcm_dpo/q_t": 0.4834534525871277, "grad_norm": 29.507638931274414, "learning_rate": 4.954821743156767e-07, "logits/chosen": -0.6492323279380798, "logits/rejected": -0.6508209109306335, "logps/chosen": -262.290283203125, "logps/ref_chosen": -266.8641662597656, "logps/ref_rejected": -276.8699951171875, "logps/rejected": -278.93731689453125, "loss": 5.2973, "margin_dpo/margin_mean": 6.641193389892578, "margin_dpo/margin_std": 10.963482856750488, "step": 75 }, { "KL/chosen_KL_mean": 4.076141357421875, "KL/mean": 0.49301546812057495, "KL/rejected_KL_mean": -3.0901031494140625, "KL/std": 9.934422492980957, "epoch": 0.15916230366492146, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.166243553161621, "fcm_dpo/q_t": 0.48219114542007446, "grad_norm": 29.160371780395508, "learning_rate": 4.951291206355559e-07, "logits/chosen": -0.7237194180488586, "logits/rejected": -0.731726884841919, "logps/chosen": -277.09844970703125, "logps/ref_chosen": -281.174560546875, "logps/ref_rejected": -263.6067199707031, "logps/rejected": -266.6968078613281, "loss": 5.2812, "margin_dpo/margin_mean": 7.166243553161621, "margin_dpo/margin_std": 12.741656303405762, "step": 76 }, { "KL/chosen_KL_mean": 3.6962413787841797, "KL/mean": 0.7907732725143433, "KL/rejected_KL_mean": -2.1146984100341797, "KL/std": 11.032858848571777, "epoch": 0.1612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.81093692779541, "fcm_dpo/q_t": 0.48557212948799133, "grad_norm": 33.117801666259766, "learning_rate": 4.947629214246236e-07, "logits/chosen": -0.5612236857414246, "logits/rejected": -0.5700373649597168, "logps/chosen": -302.3990478515625, "logps/ref_chosen": -306.09527587890625, "logps/ref_rejected": -253.49569702148438, "logps/rejected": -255.61038208007812, "loss": 5.3373, "margin_dpo/margin_mean": 5.810935974121094, "margin_dpo/margin_std": 14.009510040283203, "step": 77 }, { "KL/chosen_KL_mean": 3.9560623168945312, "KL/mean": -0.42408621311187744, "KL/rejected_KL_mean": -4.804229736328125, "KL/std": 11.058279037475586, "epoch": 0.16335078534031414, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.760295867919922, "fcm_dpo/q_t": 0.4782665967941284, "grad_norm": 29.5765323638916, "learning_rate": 4.943835963210323e-07, "logits/chosen": -0.6872852444648743, "logits/rejected": -0.6824613213539124, "logps/chosen": -252.9462890625, "logps/ref_chosen": -256.90234375, "logps/ref_rejected": -211.57154846191406, "logps/rejected": -216.37579345703125, "loss": 5.2243, "margin_dpo/margin_mean": 8.760295867919922, "margin_dpo/margin_std": 14.509359359741211, "step": 78 }, { "KL/chosen_KL_mean": 3.094745635986328, "KL/mean": -1.0207563638687134, "KL/rejected_KL_mean": -5.136257171630859, "KL/std": 11.751511573791504, "epoch": 0.16544502617801046, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.231000900268555, "fcm_dpo/q_t": 0.47957462072372437, "grad_norm": 29.715789794921875, "learning_rate": 4.939911656668361e-07, "logits/chosen": -0.6479459404945374, "logits/rejected": -0.6681416630744934, "logps/chosen": -263.1788330078125, "logps/ref_chosen": -266.2735595703125, "logps/ref_rejected": -251.57257080078125, "logps/rejected": -256.7088317871094, "loss": 5.2467, "margin_dpo/margin_mean": 8.231000900268555, "margin_dpo/margin_std": 14.964641571044922, "step": 79 }, { "KL/chosen_KL_mean": 1.8658809661865234, "KL/mean": -1.5003409385681152, "KL/rejected_KL_mean": -4.866567611694336, "KL/std": 11.38090991973877, "epoch": 0.16753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.732439994812012, "fcm_dpo/q_t": 0.48329317569732666, "grad_norm": 29.368417739868164, "learning_rate": 4.935856505068998e-07, "logits/chosen": -0.6606127619743347, "logits/rejected": -0.6897295713424683, "logps/chosen": -285.9851379394531, "logps/ref_chosen": -287.8509826660156, "logps/ref_rejected": -256.0766296386719, "logps/rejected": -260.94317626953125, "loss": 5.3003, "margin_dpo/margin_mean": 6.732440948486328, "margin_dpo/margin_std": 13.253829002380371, "step": 80 }, { "KL/chosen_KL_mean": 2.3929805755615234, "KL/mean": -1.201179027557373, "KL/rejected_KL_mean": -4.795331954956055, "KL/std": 12.221700668334961, "epoch": 0.16963350785340314, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.188300132751465, "fcm_dpo/q_t": 0.4821750223636627, "grad_norm": 28.137069702148438, "learning_rate": 4.93167072587771e-07, "logits/chosen": -0.639922022819519, "logits/rejected": -0.6389482021331787, "logps/chosen": -266.1302490234375, "logps/ref_chosen": -268.5232238769531, "logps/ref_rejected": -237.81137084960938, "logps/rejected": -242.606689453125, "loss": 5.2926, "margin_dpo/margin_mean": 7.188301086425781, "margin_dpo/margin_std": 16.771045684814453, "step": 81 }, { "KL/chosen_KL_mean": 2.4245166778564453, "KL/mean": -1.560113549232483, "KL/rejected_KL_mean": -5.544746398925781, "KL/std": 12.169999122619629, "epoch": 0.17172774869109947, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.9692702293396, "fcm_dpo/q_t": 0.48028764128685, "grad_norm": 27.79278564453125, "learning_rate": 4.92735454356513e-07, "logits/chosen": -0.7311412692070007, "logits/rejected": -0.7389302849769592, "logps/chosen": -276.939453125, "logps/ref_chosen": -279.36395263671875, "logps/ref_rejected": -236.51365661621094, "logps/rejected": -242.0584259033203, "loss": 5.2612, "margin_dpo/margin_mean": 7.969269275665283, "margin_dpo/margin_std": 15.728371620178223, "step": 82 }, { "KL/chosen_KL_mean": 0.2986888885498047, "KL/mean": -4.056522846221924, "KL/rejected_KL_mean": -8.411733627319336, "KL/std": 13.574930191040039, "epoch": 0.17382198952879582, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.710416793823242, "fcm_dpo/q_t": 0.4783723056316376, "grad_norm": 30.74686050415039, "learning_rate": 4.922908189595017e-07, "logits/chosen": -0.6830888390541077, "logits/rejected": -0.6660097241401672, "logps/chosen": -273.9205322265625, "logps/ref_chosen": -274.21923828125, "logps/ref_rejected": -276.2212219238281, "logps/rejected": -284.6329345703125, "loss": 5.2404, "margin_dpo/margin_mean": 8.71041488647461, "margin_dpo/margin_std": 18.00052261352539, "step": 83 }, { "KL/chosen_KL_mean": 0.45940589904785156, "KL/mean": -3.1783106327056885, "KL/rejected_KL_mean": -6.816030502319336, "KL/std": 13.303352355957031, "epoch": 0.17591623036649215, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.275437355041504, "fcm_dpo/q_t": 0.4819525480270386, "grad_norm": 29.886608123779297, "learning_rate": 4.918331902411841e-07, "logits/chosen": -0.7300011515617371, "logits/rejected": -0.7440951466560364, "logps/chosen": -293.9381408691406, "logps/ref_chosen": -294.3975524902344, "logps/ref_rejected": -279.81884765625, "logps/rejected": -286.6348571777344, "loss": 5.2907, "margin_dpo/margin_mean": 7.275436878204346, "margin_dpo/margin_std": 16.904508590698242, "step": 84 }, { "KL/chosen_KL_mean": -1.547414779663086, "KL/mean": -4.410724639892578, "KL/rejected_KL_mean": -7.274040222167969, "KL/std": 12.268994331359863, "epoch": 0.17801047120418848, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.726627349853516, "fcm_dpo/q_t": 0.4857904314994812, "grad_norm": 29.356456756591797, "learning_rate": 4.913625927427995e-07, "logits/chosen": -0.6729651093482971, "logits/rejected": -0.681312620639801, "logps/chosen": -245.20960998535156, "logps/ref_chosen": -243.66220092773438, "logps/ref_rejected": -263.9421691894531, "logps/rejected": -271.2162170410156, "loss": 5.3462, "margin_dpo/margin_mean": 5.726626873016357, "margin_dpo/margin_std": 15.388749122619629, "step": 85 }, { "KL/chosen_KL_mean": 1.1150398254394531, "KL/mean": -3.287755012512207, "KL/rejected_KL_mean": -7.690544128417969, "KL/std": 13.255000114440918, "epoch": 0.18010471204188483, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.805594444274902, "fcm_dpo/q_t": 0.47813674807548523, "grad_norm": 35.038631439208984, "learning_rate": 4.908790517010636e-07, "logits/chosen": -0.6957335472106934, "logits/rejected": -0.689771294593811, "logps/chosen": -308.315673828125, "logps/ref_chosen": -309.4306945800781, "logps/ref_rejected": -290.91278076171875, "logps/rejected": -298.60333251953125, "loss": 5.2323, "margin_dpo/margin_mean": 8.805594444274902, "margin_dpo/margin_std": 17.30478286743164, "step": 86 }, { "KL/chosen_KL_mean": -0.07946395874023438, "KL/mean": -5.250776290893555, "KL/rejected_KL_mean": -10.42209243774414, "KL/std": 15.329278945922852, "epoch": 0.18219895287958116, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.342628479003906, "fcm_dpo/q_t": 0.47446227073669434, "grad_norm": 29.616622924804688, "learning_rate": 4.903825930468148e-07, "logits/chosen": -0.7527617812156677, "logits/rejected": -0.7467857599258423, "logps/chosen": -278.107177734375, "logps/ref_chosen": -278.0277099609375, "logps/ref_rejected": -245.70123291015625, "logps/rejected": -256.1233215332031, "loss": 5.1807, "margin_dpo/margin_mean": 10.34262752532959, "margin_dpo/margin_std": 18.891633987426758, "step": 87 }, { "KL/chosen_KL_mean": -2.065469741821289, "KL/mean": -6.552918910980225, "KL/rejected_KL_mean": -11.040365219116211, "KL/std": 15.065473556518555, "epoch": 0.18429319371727748, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.974905014038086, "fcm_dpo/q_t": 0.47781187295913696, "grad_norm": 28.785079956054688, "learning_rate": 4.898732434036243e-07, "logits/chosen": -0.7596137523651123, "logits/rejected": -0.7751888632774353, "logps/chosen": -268.58026123046875, "logps/ref_chosen": -266.5148010253906, "logps/ref_rejected": -265.90081787109375, "logps/rejected": -276.9411926269531, "loss": 5.2321, "margin_dpo/margin_mean": 8.974905014038086, "margin_dpo/margin_std": 19.049829483032227, "step": 88 }, { "KL/chosen_KL_mean": 0.024187088012695312, "KL/mean": -4.604069709777832, "KL/rejected_KL_mean": -9.232324600219727, "KL/std": 14.210840225219727, "epoch": 0.18638743455497384, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.256507873535156, "fcm_dpo/q_t": 0.47706693410873413, "grad_norm": 30.544334411621094, "learning_rate": 4.893510300863676e-07, "logits/chosen": -0.7496181726455688, "logits/rejected": -0.7401238083839417, "logps/chosen": -265.66510009765625, "logps/ref_chosen": -265.6893005371094, "logps/ref_rejected": -251.49314880371094, "logps/rejected": -260.7254638671875, "loss": 5.2206, "margin_dpo/margin_mean": 9.256505966186523, "margin_dpo/margin_std": 18.1968994140625, "step": 89 }, { "KL/chosen_KL_mean": -1.1707649230957031, "KL/mean": -5.307417869567871, "KL/rejected_KL_mean": -9.444063186645508, "KL/std": 14.324629783630371, "epoch": 0.18848167539267016, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.273297309875488, "fcm_dpo/q_t": 0.47948795557022095, "grad_norm": 29.870534896850586, "learning_rate": 4.8881598109976e-07, "logits/chosen": -0.7418705224990845, "logits/rejected": -0.7506077885627747, "logps/chosen": -308.59576416015625, "logps/ref_chosen": -307.4250183105469, "logps/ref_rejected": -265.7172546386719, "logps/rejected": -275.16131591796875, "loss": 5.255, "margin_dpo/margin_mean": 8.273297309875488, "margin_dpo/margin_std": 17.62283706665039, "step": 90 }, { "KL/chosen_KL_mean": -2.0556297302246094, "KL/mean": -6.845937252044678, "KL/rejected_KL_mean": -11.636245727539062, "KL/std": 14.768171310424805, "epoch": 0.1905759162303665, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.580626487731934, "fcm_dpo/q_t": 0.4763721525669098, "grad_norm": 32.8757438659668, "learning_rate": 4.882681251368548e-07, "logits/chosen": -0.6754969358444214, "logits/rejected": -0.6923132538795471, "logps/chosen": -237.79660034179688, "logps/ref_chosen": -235.74098205566406, "logps/ref_rejected": -226.6428985595703, "logps/rejected": -238.27914428710938, "loss": 5.213, "margin_dpo/margin_mean": 9.580625534057617, "margin_dpo/margin_std": 19.14481544494629, "step": 91 }, { "KL/chosen_KL_mean": -3.073078155517578, "KL/mean": -8.132434844970703, "KL/rejected_KL_mean": -13.191781997680664, "KL/std": 17.528228759765625, "epoch": 0.19267015706806281, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.118701934814453, "fcm_dpo/q_t": 0.47499901056289673, "grad_norm": 33.32277297973633, "learning_rate": 4.877074915775048e-07, "logits/chosen": -0.7399584054946899, "logits/rejected": -0.7235689759254456, "logps/chosen": -286.5206298828125, "logps/ref_chosen": -283.4475402832031, "logps/ref_rejected": -273.134033203125, "logps/rejected": -286.3258056640625, "loss": 5.1989, "margin_dpo/margin_mean": 10.118700981140137, "margin_dpo/margin_std": 21.361736297607422, "step": 92 }, { "KL/chosen_KL_mean": -2.37042236328125, "KL/mean": -7.15866756439209, "KL/rejected_KL_mean": -11.946908950805664, "KL/std": 15.452308654785156, "epoch": 0.19476439790575917, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.576484680175781, "fcm_dpo/q_t": 0.47637003660202026, "grad_norm": 28.583599090576172, "learning_rate": 4.871341104867864e-07, "logits/chosen": -0.7276648879051208, "logits/rejected": -0.7507765293121338, "logps/chosen": -235.70758056640625, "logps/ref_chosen": -233.33714294433594, "logps/ref_rejected": -230.54273986816406, "logps/rejected": -242.48963928222656, "loss": 5.2119, "margin_dpo/margin_mean": 9.576486587524414, "margin_dpo/margin_std": 19.396862030029297, "step": 93 }, { "KL/chosen_KL_mean": -2.370729446411133, "KL/mean": -7.069452285766602, "KL/rejected_KL_mean": -11.768178939819336, "KL/std": 16.41796875, "epoch": 0.1968586387434555, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.397444725036621, "fcm_dpo/q_t": 0.47674643993377686, "grad_norm": 31.590362548828125, "learning_rate": 4.865480126133871e-07, "logits/chosen": -0.6804681420326233, "logits/rejected": -0.701062798500061, "logps/chosen": -297.0235290527344, "logps/ref_chosen": -294.6528015136719, "logps/ref_rejected": -283.657958984375, "logps/rejected": -295.4261779785156, "loss": 5.229, "margin_dpo/margin_mean": 9.397444725036621, "margin_dpo/margin_std": 21.770397186279297, "step": 94 }, { "KL/chosen_KL_mean": -3.2703723907470703, "KL/mean": -8.370147705078125, "KL/rejected_KL_mean": -13.469928741455078, "KL/std": 18.059005737304688, "epoch": 0.19895287958115182, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.199569702148438, "fcm_dpo/q_t": 0.47488659620285034, "grad_norm": 33.58381652832031, "learning_rate": 4.859492293879573e-07, "logits/chosen": -0.7297713756561279, "logits/rejected": -0.7513625621795654, "logps/chosen": -314.9400634765625, "logps/ref_chosen": -311.6697082519531, "logps/ref_rejected": -262.7471923828125, "logps/rejected": -276.2171325683594, "loss": 5.2047, "margin_dpo/margin_mean": 10.199568748474121, "margin_dpo/margin_std": 22.428693771362305, "step": 95 }, { "KL/chosen_KL_mean": -4.741254806518555, "KL/mean": -9.848160743713379, "KL/rejected_KL_mean": -14.955074310302734, "KL/std": 19.324003219604492, "epoch": 0.20104712041884817, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.213811874389648, "fcm_dpo/q_t": 0.47487515211105347, "grad_norm": 32.13194274902344, "learning_rate": 4.853377929214243e-07, "logits/chosen": -0.7099624276161194, "logits/rejected": -0.7225924134254456, "logps/chosen": -287.2972412109375, "logps/ref_chosen": -282.55596923828125, "logps/ref_rejected": -242.71588134765625, "logps/rejected": -257.67095947265625, "loss": 5.2055, "margin_dpo/margin_mean": 10.213811874389648, "margin_dpo/margin_std": 23.361492156982422, "step": 96 }, { "KL/chosen_KL_mean": -4.432626724243164, "KL/mean": -10.405980110168457, "KL/rejected_KL_mean": -16.379331588745117, "KL/std": 17.279687881469727, "epoch": 0.2031413612565445, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.946697235107422, "fcm_dpo/q_t": 0.4705726206302643, "grad_norm": 32.038726806640625, "learning_rate": 4.847137360032699e-07, "logits/chosen": -0.7478024959564209, "logits/rejected": -0.7351346015930176, "logps/chosen": -308.0104064941406, "logps/ref_chosen": -303.57781982421875, "logps/ref_rejected": -264.22491455078125, "logps/rejected": -280.6042785644531, "loss": 5.134, "margin_dpo/margin_mean": 11.946695327758789, "margin_dpo/margin_std": 22.253314971923828, "step": 97 }, { "KL/chosen_KL_mean": -4.91217041015625, "KL/mean": -11.056485176086426, "KL/rejected_KL_mean": -17.200815200805664, "KL/std": 18.921445846557617, "epoch": 0.20523560209424083, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.288653373718262, "fcm_dpo/q_t": 0.46991679072380066, "grad_norm": 34.590091705322266, "learning_rate": 4.84077092099773e-07, "logits/chosen": -0.773546576499939, "logits/rejected": -0.7858595848083496, "logps/chosen": -291.7425231933594, "logps/ref_chosen": -286.8303527832031, "logps/ref_rejected": -278.08331298828125, "logps/rejected": -295.28411865234375, "loss": 5.1334, "margin_dpo/margin_mean": 12.288653373718262, "margin_dpo/margin_std": 22.73413848876953, "step": 98 }, { "KL/chosen_KL_mean": -5.357412338256836, "KL/mean": -11.63817310333252, "KL/rejected_KL_mean": -17.918926239013672, "KL/std": 19.809310913085938, "epoch": 0.20732984293193718, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.561515808105469, "fcm_dpo/q_t": 0.46920517086982727, "grad_norm": 32.64725112915039, "learning_rate": 4.834278953522137e-07, "logits/chosen": -0.7336536645889282, "logits/rejected": -0.7463732957839966, "logps/chosen": -285.27862548828125, "logps/ref_chosen": -279.92120361328125, "logps/ref_rejected": -250.3365478515625, "logps/rejected": -268.2554626464844, "loss": 5.1374, "margin_dpo/margin_mean": 12.561513900756836, "margin_dpo/margin_std": 27.08693504333496, "step": 99 }, { "KL/chosen_KL_mean": -7.981771469116211, "KL/mean": -14.22314167022705, "KL/rejected_KL_mean": -20.464502334594727, "KL/std": 22.023868560791016, "epoch": 0.2094240837696335, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.482728958129883, "fcm_dpo/q_t": 0.46943894028663635, "grad_norm": 35.665283203125, "learning_rate": 4.827661805750437e-07, "logits/chosen": -0.7793423533439636, "logits/rejected": -0.792355477809906, "logps/chosen": -304.8094482421875, "logps/ref_chosen": -296.8276672363281, "logps/ref_rejected": -275.56146240234375, "logps/rejected": -296.0259704589844, "loss": 5.1259, "margin_dpo/margin_mean": 12.482728958129883, "margin_dpo/margin_std": 24.164777755737305, "step": 100 }, { "KL/chosen_KL_mean": -5.265691757202148, "KL/mean": -12.787820816040039, "KL/rejected_KL_mean": -20.309947967529297, "KL/std": 22.83624267578125, "epoch": 0.21151832460732983, "fcm_dpo/beta": 0.010065114125609398, "fcm_dpo/delta": 0.012955582700669765, "fcm_dpo/margin": 15.044249534606934, "fcm_dpo/q_t": 0.46306145191192627, "grad_norm": 32.623504638671875, "learning_rate": 4.820919832540181e-07, "logits/chosen": -0.7783185839653015, "logits/rejected": -0.7923865914344788, "logps/chosen": -258.0077209472656, "logps/ref_chosen": -252.74203491210938, "logps/ref_rejected": -276.4185485839844, "logps/rejected": -296.728515625, "loss": 5.0388, "margin_dpo/margin_mean": 15.044251441955566, "margin_dpo/margin_std": 26.482027053833008, "step": 101 }, { "KL/chosen_KL_mean": -5.2807159423828125, "KL/mean": -12.680944442749023, "KL/rejected_KL_mean": -20.0811710357666, "KL/std": 19.747940063476562, "epoch": 0.2136125654450262, "fcm_dpo/beta": 0.010104183107614517, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.800459861755371, "fcm_dpo/q_t": 0.4631401300430298, "grad_norm": 32.70088577270508, "learning_rate": 4.814053395442932e-07, "logits/chosen": -0.748501718044281, "logits/rejected": -0.7446244359016418, "logps/chosen": -224.8344268798828, "logps/ref_chosen": -219.5537109375, "logps/ref_rejected": -231.90853881835938, "logps/rejected": -251.98971557617188, "loss": 5.0391, "margin_dpo/margin_mean": 14.800461769104004, "margin_dpo/margin_std": 24.98674201965332, "step": 102 }, { "KL/chosen_KL_mean": -4.796598434448242, "KL/mean": -11.604303359985352, "KL/rejected_KL_mean": -18.41200828552246, "KL/std": 21.641616821289062, "epoch": 0.2157068062827225, "fcm_dpo/beta": 0.010104183107614517, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.615408897399902, "fcm_dpo/q_t": 0.46653062105178833, "grad_norm": 33.16029357910156, "learning_rate": 4.807062862684873e-07, "logits/chosen": -0.7775754332542419, "logits/rejected": -0.7745975852012634, "logps/chosen": -264.4716491699219, "logps/ref_chosen": -259.6750793457031, "logps/ref_rejected": -278.7400817871094, "logps/rejected": -297.1520690917969, "loss": 5.087, "margin_dpo/margin_mean": 13.615410804748535, "margin_dpo/margin_std": 25.769948959350586, "step": 103 }, { "KL/chosen_KL_mean": -9.229019165039062, "KL/mean": -14.208805084228516, "KL/rejected_KL_mean": -19.188581466674805, "KL/std": 22.482053756713867, "epoch": 0.21780104712041884, "fcm_dpo/beta": 0.010104183107614517, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.959566116333008, "fcm_dpo/q_t": 0.4753492772579193, "grad_norm": 33.56724166870117, "learning_rate": 4.799948609147061e-07, "logits/chosen": -0.7666143178939819, "logits/rejected": -0.7724973559379578, "logps/chosen": -277.20318603515625, "logps/ref_chosen": -267.9741516113281, "logps/ref_rejected": -230.5306396484375, "logps/rejected": -249.7191925048828, "loss": 5.2263, "margin_dpo/margin_mean": 9.959567070007324, "margin_dpo/margin_std": 26.30889129638672, "step": 104 }, { "KL/chosen_KL_mean": -5.367591857910156, "KL/mean": -15.618803977966309, "KL/rejected_KL_mean": -25.870025634765625, "KL/std": 25.24723243713379, "epoch": 0.2198952879581152, "fcm_dpo/beta": 0.010265516117215157, "fcm_dpo/delta": 0.03177732229232788, "fcm_dpo/margin": 20.502422332763672, "fcm_dpo/q_t": 0.44872522354125977, "grad_norm": 35.49409484863281, "learning_rate": 4.792711016345321e-07, "logits/chosen": -0.7507005929946899, "logits/rejected": -0.7593094706535339, "logps/chosen": -327.6224365234375, "logps/ref_chosen": -322.25482177734375, "logps/ref_rejected": -279.02978515625, "logps/rejected": -304.8998107910156, "loss": 4.8339, "margin_dpo/margin_mean": 20.502424240112305, "margin_dpo/margin_std": 27.04251480102539, "step": 105 }, { "KL/chosen_KL_mean": -12.565797805786133, "KL/mean": -18.833768844604492, "KL/rejected_KL_mean": -25.101734161376953, "KL/std": 23.593149185180664, "epoch": 0.22198952879581152, "fcm_dpo/beta": 0.010364342480897903, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.535932540893555, "fcm_dpo/q_t": 0.46829015016555786, "grad_norm": 39.71345901489258, "learning_rate": 4.785350472409791e-07, "logits/chosen": -0.7475220561027527, "logits/rejected": -0.7846206426620483, "logps/chosen": -308.7236022949219, "logps/ref_chosen": -296.15777587890625, "logps/ref_rejected": -266.2691650390625, "logps/rejected": -291.3708801269531, "loss": 5.141, "margin_dpo/margin_mean": 12.535932540893555, "margin_dpo/margin_std": 29.484294891357422, "step": 106 }, { "KL/chosen_KL_mean": -4.345104217529297, "KL/mean": -14.212882995605469, "KL/rejected_KL_mean": -24.08066749572754, "KL/std": 22.098743438720703, "epoch": 0.22408376963350785, "fcm_dpo/beta": 0.010364342480897903, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.735557556152344, "fcm_dpo/q_t": 0.45006412267684937, "grad_norm": 39.31448745727539, "learning_rate": 4.777867372064105e-07, "logits/chosen": -0.7761357426643372, "logits/rejected": -0.7689798474311829, "logps/chosen": -311.3414001464844, "logps/ref_chosen": -306.996337890625, "logps/ref_rejected": -296.79412841796875, "logps/rejected": -320.8747863769531, "loss": 4.8556, "margin_dpo/margin_mean": 19.735557556152344, "margin_dpo/margin_std": 27.56524658203125, "step": 107 }, { "KL/chosen_KL_mean": -5.493007659912109, "KL/mean": -14.60330867767334, "KL/rejected_KL_mean": -23.713605880737305, "KL/std": 25.141176223754883, "epoch": 0.2261780104712042, "fcm_dpo/beta": 0.010364342480897903, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.220611572265625, "fcm_dpo/q_t": 0.45436638593673706, "grad_norm": 105.63594055175781, "learning_rate": 4.770262116604223e-07, "logits/chosen": -0.7658447027206421, "logits/rejected": -0.7774515151977539, "logps/chosen": -300.6456604003906, "logps/ref_chosen": -295.1526794433594, "logps/ref_rejected": -235.974853515625, "logps/rejected": -259.6884460449219, "loss": 4.9372, "margin_dpo/margin_mean": 18.220611572265625, "margin_dpo/margin_std": 30.389331817626953, "step": 108 }, { "KL/chosen_KL_mean": -8.395013809204102, "KL/mean": -18.529293060302734, "KL/rejected_KL_mean": -28.663572311401367, "KL/std": 25.285425186157227, "epoch": 0.22827225130890053, "fcm_dpo/beta": 0.010364342480897903, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.268556594848633, "fcm_dpo/q_t": 0.4494061768054962, "grad_norm": 39.464012145996094, "learning_rate": 4.7625351138769166e-07, "logits/chosen": -0.7855874300003052, "logits/rejected": -0.7813213467597961, "logps/chosen": -334.31982421875, "logps/ref_chosen": -325.9248046875, "logps/ref_rejected": -279.15423583984375, "logps/rejected": -307.81781005859375, "loss": 4.8598, "margin_dpo/margin_mean": 20.2685546875, "margin_dpo/margin_std": 31.691814422607422, "step": 109 }, { "KL/chosen_KL_mean": -8.880786895751953, "KL/mean": -18.487215042114258, "KL/rejected_KL_mean": -28.093652725219727, "KL/std": 27.44156837463379, "epoch": 0.23036649214659685, "fcm_dpo/beta": 0.010364342480897903, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.212865829467773, "fcm_dpo/q_t": 0.45193198323249817, "grad_norm": 35.9247932434082, "learning_rate": 4.75468677825789e-07, "logits/chosen": -0.8031843900680542, "logits/rejected": -0.7917722463607788, "logps/chosen": -283.32000732421875, "logps/ref_chosen": -274.439208984375, "logps/ref_rejected": -260.0552062988281, "logps/rejected": -288.14886474609375, "loss": 4.9139, "margin_dpo/margin_mean": 19.212865829467773, "margin_dpo/margin_std": 33.88175964355469, "step": 110 }, { "KL/chosen_KL_mean": -9.789043426513672, "KL/mean": -19.974393844604492, "KL/rejected_KL_mean": -30.159746170043945, "KL/std": 28.102462768554688, "epoch": 0.2324607329842932, "fcm_dpo/beta": 0.010417740792036057, "fcm_dpo/delta": 0.017835039645433426, "fcm_dpo/margin": 20.370704650878906, "fcm_dpo/q_t": 0.4491615891456604, "grad_norm": 39.79193878173828, "learning_rate": 4.7467175306295647e-07, "logits/chosen": -0.8448382616043091, "logits/rejected": -0.8246790170669556, "logps/chosen": -339.02520751953125, "logps/ref_chosen": -329.2361755371094, "logps/ref_rejected": -287.82830810546875, "logps/rejected": -317.988037109375, "loss": 4.8781, "margin_dpo/margin_mean": 20.370704650878906, "margin_dpo/margin_std": 34.60567092895508, "step": 111 }, { "KL/chosen_KL_mean": -15.530427932739258, "KL/mean": -22.135120391845703, "KL/rejected_KL_mean": -28.739818572998047, "KL/std": 27.021099090576172, "epoch": 0.23455497382198953, "fcm_dpo/beta": 0.010554548352956772, "fcm_dpo/delta": 0.006524841766804457, "fcm_dpo/margin": 13.209390640258789, "fcm_dpo/q_t": 0.46666276454925537, "grad_norm": 41.61723327636719, "learning_rate": 4.7386277983585053e-07, "logits/chosen": -0.7402675151824951, "logits/rejected": -0.7702385783195496, "logps/chosen": -272.5898132324219, "logps/ref_chosen": -257.0593566894531, "logps/ref_rejected": -272.9595031738281, "logps/rejected": -301.6993103027344, "loss": 5.1718, "margin_dpo/margin_mean": 13.209390640258789, "margin_dpo/margin_std": 36.302940368652344, "step": 112 }, { "KL/chosen_KL_mean": -12.931312561035156, "KL/mean": -25.255403518676758, "KL/rejected_KL_mean": -37.579498291015625, "KL/std": 31.106075286865234, "epoch": 0.23664921465968586, "fcm_dpo/beta": 0.010605855844914913, "fcm_dpo/delta": 0.008770487271249294, "fcm_dpo/margin": 24.648176193237305, "fcm_dpo/q_t": 0.43847501277923584, "grad_norm": 42.69111251831055, "learning_rate": 4.7304180152725024e-07, "logits/chosen": -0.801536500453949, "logits/rejected": -0.8040984869003296, "logps/chosen": -298.97296142578125, "logps/ref_chosen": -286.0416564941406, "logps/ref_rejected": -270.374267578125, "logps/rejected": -307.9537658691406, "loss": 4.7466, "margin_dpo/margin_mean": 24.648176193237305, "margin_dpo/margin_std": 39.36397171020508, "step": 113 }, { "KL/chosen_KL_mean": -17.070255279541016, "KL/mean": -23.929719924926758, "KL/rejected_KL_mean": -30.78919219970703, "KL/std": 28.087270736694336, "epoch": 0.2387434554973822, "fcm_dpo/beta": 0.01064271479845047, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.718945503234863, "fcm_dpo/q_t": 0.46457406878471375, "grad_norm": 42.539310455322266, "learning_rate": 4.7220886216373085e-07, "logits/chosen": -0.8641890287399292, "logits/rejected": -0.8660076856613159, "logps/chosen": -277.0787353515625, "logps/ref_chosen": -260.0084533691406, "logps/ref_rejected": -246.67190551757812, "logps/rejected": -277.4610900878906, "loss": 5.1358, "margin_dpo/margin_mean": 13.718944549560547, "margin_dpo/margin_std": 35.54159927368164, "step": 114 }, { "KL/chosen_KL_mean": -18.9876766204834, "KL/mean": -25.9890079498291, "KL/rejected_KL_mean": -32.99033737182617, "KL/std": 30.873493194580078, "epoch": 0.24083769633507854, "fcm_dpo/beta": 0.010650977492332458, "fcm_dpo/delta": 0.0012931342935189605, "fcm_dpo/margin": 14.002666473388672, "fcm_dpo/q_t": 0.4652654826641083, "grad_norm": 46.39145278930664, "learning_rate": 4.7136400641330245e-07, "logits/chosen": -0.856718122959137, "logits/rejected": -0.817272424697876, "logps/chosen": -318.41064453125, "logps/ref_chosen": -299.4229736328125, "logps/ref_rejected": -272.1186828613281, "logps/rejected": -305.1090393066406, "loss": 5.1502, "margin_dpo/margin_mean": 14.002666473388672, "margin_dpo/margin_std": 38.7987174987793, "step": 115 }, { "KL/chosen_KL_mean": -15.55940055847168, "KL/mean": -24.596479415893555, "KL/rejected_KL_mean": -33.633567810058594, "KL/std": 28.916170120239258, "epoch": 0.24293193717277486, "fcm_dpo/beta": 0.010653730481863022, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.074167251586914, "fcm_dpo/q_t": 0.4535205662250519, "grad_norm": 40.90885925292969, "learning_rate": 4.70507279583015e-07, "logits/chosen": -0.8591501116752625, "logits/rejected": -0.8206615447998047, "logps/chosen": -294.8233337402344, "logps/ref_chosen": -279.263916015625, "logps/ref_rejected": -253.6192169189453, "logps/rejected": -287.2527770996094, "loss": 4.9579, "margin_dpo/margin_mean": 18.074169158935547, "margin_dpo/margin_std": 35.237911224365234, "step": 116 }, { "KL/chosen_KL_mean": -18.99713897705078, "KL/mean": -29.38507652282715, "KL/rejected_KL_mean": -39.77302169799805, "KL/std": 27.603759765625, "epoch": 0.2450261780104712, "fcm_dpo/beta": 0.010707004927098751, "fcm_dpo/delta": 0.0019599078223109245, "fcm_dpo/margin": 20.775876998901367, "fcm_dpo/q_t": 0.44658583402633667, "grad_norm": 51.663761138916016, "learning_rate": 4.6963872761652834e-07, "logits/chosen": -0.8078407645225525, "logits/rejected": -0.8070172071456909, "logps/chosen": -278.2220153808594, "logps/ref_chosen": -259.2248840332031, "logps/ref_rejected": -229.3042755126953, "logps/rejected": -269.0773010253906, "loss": 4.8498, "margin_dpo/margin_mean": 20.775875091552734, "margin_dpo/margin_std": 31.987648010253906, "step": 117 }, { "KL/chosen_KL_mean": -25.057470321655273, "KL/mean": -36.63970184326172, "KL/rejected_KL_mean": -48.22193145751953, "KL/std": 34.87174987792969, "epoch": 0.24712041884816754, "fcm_dpo/beta": 0.010724391788244247, "fcm_dpo/delta": 0.009311456233263016, "fcm_dpo/margin": 23.16446304321289, "fcm_dpo/q_t": 0.44190675020217896, "grad_norm": 48.20987319946289, "learning_rate": 4.687583970916486e-07, "logits/chosen": -0.8008370399475098, "logits/rejected": -0.7879800200462341, "logps/chosen": -292.128173828125, "logps/ref_chosen": -267.0707092285156, "logps/ref_rejected": -272.7322082519531, "logps/rejected": -320.9541320800781, "loss": 4.833, "margin_dpo/margin_mean": 23.164464950561523, "margin_dpo/margin_std": 43.23383712768555, "step": 118 }, { "KL/chosen_KL_mean": -30.545703887939453, "KL/mean": -39.11827087402344, "KL/rejected_KL_mean": -47.69084930419922, "KL/std": 34.09941482543945, "epoch": 0.24921465968586387, "fcm_dpo/beta": 0.010751364752650261, "fcm_dpo/delta": 0.0010554725304245949, "fcm_dpo/margin": 17.145132064819336, "fcm_dpo/q_t": 0.45693254470825195, "grad_norm": 48.39207077026367, "learning_rate": 4.6786633521783005e-07, "logits/chosen": -0.8896147012710571, "logits/rejected": -0.8895549774169922, "logps/chosen": -355.2223815917969, "logps/ref_chosen": -324.6766357421875, "logps/ref_rejected": -306.0322265625, "logps/rejected": -353.72308349609375, "loss": 5.0604, "margin_dpo/margin_mean": 17.145130157470703, "margin_dpo/margin_std": 41.501827239990234, "step": 119 }, { "KL/chosen_KL_mean": -27.356718063354492, "KL/mean": -37.051090240478516, "KL/rejected_KL_mean": -46.745460510253906, "KL/std": 33.13863754272461, "epoch": 0.2513089005235602, "fcm_dpo/beta": 0.010827319696545601, "fcm_dpo/delta": 0.009227736853063107, "fcm_dpo/margin": 19.38873863220215, "fcm_dpo/q_t": 0.4509553909301758, "grad_norm": 44.932960510253906, "learning_rate": 4.669625898336438e-07, "logits/chosen": -0.8416849970817566, "logits/rejected": -0.8531475067138672, "logps/chosen": -342.6184997558594, "logps/ref_chosen": -315.2617492675781, "logps/ref_rejected": -265.32501220703125, "logps/rejected": -312.0704345703125, "loss": 4.9718, "margin_dpo/margin_mean": 19.38874053955078, "margin_dpo/margin_std": 41.262054443359375, "step": 120 }, { "KL/chosen_KL_mean": -31.884214401245117, "KL/mean": -38.95367431640625, "KL/rejected_KL_mean": -46.02313995361328, "KL/std": 31.924612045288086, "epoch": 0.2534031413612565, "fcm_dpo/beta": 0.010839032009243965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.138921737670898, "fcm_dpo/q_t": 0.464445561170578, "grad_norm": 60.977882385253906, "learning_rate": 4.6604720940421207e-07, "logits/chosen": -0.8611897230148315, "logits/rejected": -0.875971257686615, "logps/chosen": -254.8802947998047, "logps/ref_chosen": -222.99609375, "logps/ref_rejected": -226.92860412597656, "logps/rejected": -272.95172119140625, "loss": 5.1531, "margin_dpo/margin_mean": 14.138921737670898, "margin_dpo/margin_std": 40.55924606323242, "step": 121 }, { "KL/chosen_KL_mean": -32.263099670410156, "KL/mean": -42.11972427368164, "KL/rejected_KL_mean": -51.97634506225586, "KL/std": 37.34040451049805, "epoch": 0.2554973821989529, "fcm_dpo/beta": 0.010961100459098816, "fcm_dpo/delta": 0.01673881895840168, "fcm_dpo/margin": 19.7132511138916, "fcm_dpo/q_t": 0.44972339272499084, "grad_norm": 58.791770935058594, "learning_rate": 4.651202430186092e-07, "logits/chosen": -0.9088048934936523, "logits/rejected": -0.8704884648323059, "logps/chosen": -308.2893981933594, "logps/ref_chosen": -276.02630615234375, "logps/ref_rejected": -277.97418212890625, "logps/rejected": -329.9505310058594, "loss": 4.9927, "margin_dpo/margin_mean": 19.713253021240234, "margin_dpo/margin_std": 46.560882568359375, "step": 122 }, { "KL/chosen_KL_mean": -25.356151580810547, "KL/mean": -38.11783218383789, "KL/rejected_KL_mean": -50.8795280456543, "KL/std": 36.9088249206543, "epoch": 0.25759162303664923, "fcm_dpo/beta": 0.011069193482398987, "fcm_dpo/delta": 0.024628084152936935, "fcm_dpo/margin": 25.523366928100586, "fcm_dpo/q_t": 0.4335595369338989, "grad_norm": 52.54579544067383, "learning_rate": 4.6418174038722924e-07, "logits/chosen": -0.827944278717041, "logits/rejected": -0.818522036075592, "logps/chosen": -353.5107421875, "logps/ref_chosen": -328.1546325683594, "logps/ref_rejected": -280.6911315917969, "logps/rejected": -331.5706481933594, "loss": 4.7223, "margin_dpo/margin_mean": 25.523366928100586, "margin_dpo/margin_std": 42.302734375, "step": 123 }, { "KL/chosen_KL_mean": -26.102346420288086, "KL/mean": -37.06385040283203, "KL/rejected_KL_mean": -48.025367736816406, "KL/std": 35.34791564941406, "epoch": 0.25968586387434556, "fcm_dpo/beta": 0.011221332475543022, "fcm_dpo/delta": 0.0027072872035205364, "fcm_dpo/margin": 21.92302131652832, "fcm_dpo/q_t": 0.44157105684280396, "grad_norm": 55.5904426574707, "learning_rate": 4.6323175183912023e-07, "logits/chosen": -0.8537352085113525, "logits/rejected": -0.818031907081604, "logps/chosen": -301.7985534667969, "logps/ref_chosen": -275.6961975097656, "logps/ref_rejected": -225.361572265625, "logps/rejected": -273.386962890625, "loss": 4.8314, "margin_dpo/margin_mean": 21.923023223876953, "margin_dpo/margin_std": 38.72266387939453, "step": 124 }, { "KL/chosen_KL_mean": -27.758771896362305, "KL/mean": -37.67707061767578, "KL/rejected_KL_mean": -47.595367431640625, "KL/std": 36.744564056396484, "epoch": 0.2617801047120419, "fcm_dpo/beta": 0.01130264438688755, "fcm_dpo/delta": 0.0206233449280262, "fcm_dpo/margin": 19.836593627929688, "fcm_dpo/q_t": 0.44899219274520874, "grad_norm": 58.402217864990234, "learning_rate": 4.6227032831928483e-07, "logits/chosen": -0.8152500987052917, "logits/rejected": -0.7693508863449097, "logps/chosen": -305.8285217285156, "logps/ref_chosen": -278.06976318359375, "logps/ref_rejected": -265.63873291015625, "logps/rejected": -313.234130859375, "loss": 5.0234, "margin_dpo/margin_mean": 19.836593627929688, "margin_dpo/margin_std": 50.21840286254883, "step": 125 }, { "KL/chosen_KL_mean": -23.98495864868164, "KL/mean": -35.167537689208984, "KL/rejected_KL_mean": -46.350120544433594, "KL/std": 38.36277770996094, "epoch": 0.2638743455497382, "fcm_dpo/beta": 0.011509076692163944, "fcm_dpo/delta": 0.01648152619600296, "fcm_dpo/margin": 22.365156173706055, "fcm_dpo/q_t": 0.44002822041511536, "grad_norm": 56.53205871582031, "learning_rate": 4.612975213859487e-07, "logits/chosen": -0.8276666402816772, "logits/rejected": -0.8373324275016785, "logps/chosen": -345.38104248046875, "logps/ref_chosen": -321.3960876464844, "logps/ref_rejected": -285.37664794921875, "logps/rejected": -331.72674560546875, "loss": 4.8493, "margin_dpo/margin_mean": 22.365154266357422, "margin_dpo/margin_std": 44.759033203125, "step": 126 }, { "KL/chosen_KL_mean": -22.506771087646484, "KL/mean": -35.692604064941406, "KL/rejected_KL_mean": -48.87844467163086, "KL/std": 39.546775817871094, "epoch": 0.26596858638743454, "fcm_dpo/beta": 0.011503228917717934, "fcm_dpo/delta": -0.021350204944610596, "fcm_dpo/margin": 26.37167739868164, "fcm_dpo/q_t": 0.43013378977775574, "grad_norm": 56.40062713623047, "learning_rate": 4.603133832077953e-07, "logits/chosen": -0.9043377637863159, "logits/rejected": -0.8537869453430176, "logps/chosen": -329.0655212402344, "logps/ref_chosen": -306.55877685546875, "logps/ref_rejected": -274.8651428222656, "logps/rejected": -323.74359130859375, "loss": 4.7248, "margin_dpo/margin_mean": 26.371681213378906, "margin_dpo/margin_std": 45.39989471435547, "step": 127 }, { "KL/chosen_KL_mean": -17.200231552124023, "KL/mean": -32.34983825683594, "KL/rejected_KL_mean": -47.49944305419922, "KL/std": 41.117652893066406, "epoch": 0.2680628272251309, "fcm_dpo/beta": 0.011355388909578323, "fcm_dpo/delta": 0.0041136001236736774, "fcm_dpo/margin": 30.299209594726562, "fcm_dpo/q_t": 0.4208820164203644, "grad_norm": 53.725135803222656, "learning_rate": 4.5931796656116837e-07, "logits/chosen": -0.7902308702468872, "logits/rejected": -0.7869139909744263, "logps/chosen": -282.59759521484375, "logps/ref_chosen": -265.3973693847656, "logps/ref_rejected": -250.9737548828125, "logps/rejected": -298.47320556640625, "loss": 4.5971, "margin_dpo/margin_mean": 30.299209594726562, "margin_dpo/margin_std": 49.29853820800781, "step": 128 }, { "KL/chosen_KL_mean": -18.242015838623047, "KL/mean": -31.862220764160156, "KL/rejected_KL_mean": -45.482425689697266, "KL/std": 38.571449279785156, "epoch": 0.27015706806282724, "fcm_dpo/beta": 0.011395599693059921, "fcm_dpo/delta": -0.014722984284162521, "fcm_dpo/margin": 27.24040985107422, "fcm_dpo/q_t": 0.4288609027862549, "grad_norm": 56.34138488769531, "learning_rate": 4.5831132482724193e-07, "logits/chosen": -0.8255420327186584, "logits/rejected": -0.8241854310035706, "logps/chosen": -321.40045166015625, "logps/ref_chosen": -303.158447265625, "logps/ref_rejected": -275.9891052246094, "logps/rejected": -321.47149658203125, "loss": 4.7074, "margin_dpo/margin_mean": 27.24040985107422, "margin_dpo/margin_std": 48.436065673828125, "step": 129 }, { "KL/chosen_KL_mean": -20.89402961730957, "KL/mean": -34.46953201293945, "KL/rejected_KL_mean": -48.04502487182617, "KL/std": 38.12776565551758, "epoch": 0.27225130890052357, "fcm_dpo/beta": 0.011203363537788391, "fcm_dpo/delta": -0.00045868102461099625, "fcm_dpo/margin": 27.150991439819336, "fcm_dpo/q_t": 0.4299607276916504, "grad_norm": 62.90643310546875, "learning_rate": 4.5729351198915705e-07, "logits/chosen": -0.7972059845924377, "logits/rejected": -0.8352242112159729, "logps/chosen": -307.3013610839844, "logps/ref_chosen": -286.4073486328125, "logps/ref_rejected": -294.38665771484375, "logps/rejected": -342.4317321777344, "loss": 4.7155, "margin_dpo/margin_mean": 27.1509952545166, "margin_dpo/margin_std": 48.800880432128906, "step": 130 }, { "KL/chosen_KL_mean": -25.511310577392578, "KL/mean": -36.389923095703125, "KL/rejected_KL_mean": -47.268531799316406, "KL/std": 42.743492126464844, "epoch": 0.2743455497382199, "fcm_dpo/beta": 0.011295843869447708, "fcm_dpo/delta": 0.008524668402969837, "fcm_dpo/margin": 21.75722312927246, "fcm_dpo/q_t": 0.4437254071235657, "grad_norm": 66.18495178222656, "learning_rate": 4.5626458262912735e-07, "logits/chosen": -0.8689754009246826, "logits/rejected": -0.8165445327758789, "logps/chosen": -337.0763854980469, "logps/ref_chosen": -311.5650634765625, "logps/ref_rejected": -291.62432861328125, "logps/rejected": -338.89288330078125, "loss": 4.9757, "margin_dpo/margin_mean": 21.75722312927246, "margin_dpo/margin_std": 51.93965148925781, "step": 131 }, { "KL/chosen_KL_mean": -21.638566970825195, "KL/mean": -36.89574432373047, "KL/rejected_KL_mean": -52.152923583984375, "KL/std": 42.402366638183594, "epoch": 0.2764397905759162, "fcm_dpo/beta": 0.01130670215934515, "fcm_dpo/delta": -0.03159793093800545, "fcm_dpo/margin": 30.514366149902344, "fcm_dpo/q_t": 0.42199385166168213, "grad_norm": 70.13489532470703, "learning_rate": 4.5522459192551166e-07, "logits/chosen": -0.8335902690887451, "logits/rejected": -0.8173937797546387, "logps/chosen": -291.72039794921875, "logps/ref_chosen": -270.0818176269531, "logps/ref_rejected": -284.3084411621094, "logps/rejected": -336.46136474609375, "loss": 4.658, "margin_dpo/margin_mean": 30.514366149902344, "margin_dpo/margin_std": 53.337528228759766, "step": 132 }, { "KL/chosen_KL_mean": -24.91779327392578, "KL/mean": -39.1947135925293, "KL/rejected_KL_mean": -53.47163009643555, "KL/std": 39.884578704833984, "epoch": 0.27853403141361255, "fcm_dpo/beta": 0.011096285656094551, "fcm_dpo/delta": -0.002665869891643524, "fcm_dpo/margin": 28.553842544555664, "fcm_dpo/q_t": 0.42722228169441223, "grad_norm": 47.607017517089844, "learning_rate": 4.541735956498554e-07, "logits/chosen": -0.8624144196510315, "logits/rejected": -0.8618667721748352, "logps/chosen": -310.5391540527344, "logps/ref_chosen": -285.6213684082031, "logps/ref_rejected": -251.19386291503906, "logps/rejected": -304.66546630859375, "loss": 4.6841, "margin_dpo/margin_mean": 28.553842544555664, "margin_dpo/margin_std": 49.61962127685547, "step": 133 }, { "KL/chosen_KL_mean": -37.28353500366211, "KL/mean": -47.89252471923828, "KL/rejected_KL_mean": -58.50152587890625, "KL/std": 41.15799331665039, "epoch": 0.2806282722513089, "fcm_dpo/beta": 0.010984513908624649, "fcm_dpo/delta": -0.00815630704164505, "fcm_dpo/margin": 21.21798324584961, "fcm_dpo/q_t": 0.4452402591705322, "grad_norm": 54.9819450378418, "learning_rate": 4.5311165016389914e-07, "logits/chosen": -0.8601773977279663, "logits/rejected": -0.8593323230743408, "logps/chosen": -356.204345703125, "logps/ref_chosen": -318.92083740234375, "logps/ref_rejected": -293.1894836425781, "logps/rejected": -351.69097900390625, "loss": 4.9206, "margin_dpo/margin_mean": 21.21798324584961, "margin_dpo/margin_std": 44.41059875488281, "step": 134 }, { "KL/chosen_KL_mean": -34.964073181152344, "KL/mean": -47.551513671875, "KL/rejected_KL_mean": -60.138946533203125, "KL/std": 43.32963943481445, "epoch": 0.28272251308900526, "fcm_dpo/beta": 0.011102442629635334, "fcm_dpo/delta": 0.0017053596675395966, "fcm_dpo/margin": 25.174869537353516, "fcm_dpo/q_t": 0.4345686733722687, "grad_norm": 65.74720001220703, "learning_rate": 4.520388124165564e-07, "logits/chosen": -0.7585154175758362, "logits/rejected": -0.798126757144928, "logps/chosen": -327.7857971191406, "logps/ref_chosen": -292.8217468261719, "logps/ref_rejected": -269.2896728515625, "logps/rejected": -329.42864990234375, "loss": 4.8261, "margin_dpo/margin_mean": 25.174869537353516, "margin_dpo/margin_std": 48.767791748046875, "step": 135 }, { "KL/chosen_KL_mean": -37.39518356323242, "KL/mean": -51.40061950683594, "KL/rejected_KL_mean": -65.40606689453125, "KL/std": 43.074127197265625, "epoch": 0.2848167539267016, "fcm_dpo/beta": 0.010944095440208912, "fcm_dpo/delta": -0.015430422499775887, "fcm_dpo/margin": 28.01087760925293, "fcm_dpo/q_t": 0.4303656220436096, "grad_norm": 75.27836608886719, "learning_rate": 4.5095513994085974e-07, "logits/chosen": -0.8118724226951599, "logits/rejected": -0.806470513343811, "logps/chosen": -310.2476806640625, "logps/ref_chosen": -272.8525390625, "logps/ref_rejected": -252.68202209472656, "logps/rejected": -318.08807373046875, "loss": 4.7693, "margin_dpo/margin_mean": 28.01087760925293, "margin_dpo/margin_std": 54.0770378112793, "step": 136 }, { "KL/chosen_KL_mean": -41.546897888183594, "KL/mean": -54.02006149291992, "KL/rejected_KL_mean": -66.49323272705078, "KL/std": 45.12068176269531, "epoch": 0.2869109947643979, "fcm_dpo/beta": 0.010924424976110458, "fcm_dpo/delta": 0.014189784415066242, "fcm_dpo/margin": 24.94634437561035, "fcm_dpo/q_t": 0.43837597966194153, "grad_norm": 66.66802978515625, "learning_rate": 4.498606908508753e-07, "logits/chosen": -0.859586775302887, "logits/rejected": -0.8458654880523682, "logps/chosen": -342.2991027832031, "logps/ref_chosen": -300.7522277832031, "logps/ref_rejected": -286.1935119628906, "logps/rejected": -352.686767578125, "loss": 4.8497, "margin_dpo/margin_mean": 24.94634437561035, "margin_dpo/margin_std": 51.542877197265625, "step": 137 }, { "KL/chosen_KL_mean": -38.696102142333984, "KL/mean": -53.19694900512695, "KL/rejected_KL_mean": -67.69778442382812, "KL/std": 46.134708404541016, "epoch": 0.28900523560209423, "fcm_dpo/beta": 0.011035628616809845, "fcm_dpo/delta": 0.011557774618268013, "fcm_dpo/margin": 29.00168228149414, "fcm_dpo/q_t": 0.42893779277801514, "grad_norm": 61.13792419433594, "learning_rate": 4.487555238385862e-07, "logits/chosen": -0.7798894047737122, "logits/rejected": -0.7592524290084839, "logps/chosen": -327.63311767578125, "logps/ref_chosen": -288.9369812011719, "logps/ref_rejected": -263.7076416015625, "logps/rejected": -331.40545654296875, "loss": 4.7942, "margin_dpo/margin_mean": 29.00168228149414, "margin_dpo/margin_std": 60.18000030517578, "step": 138 }, { "KL/chosen_KL_mean": -35.5364990234375, "KL/mean": -45.61208724975586, "KL/rejected_KL_mean": -55.68767547607422, "KL/std": 41.366214752197266, "epoch": 0.29109947643979056, "fcm_dpo/beta": 0.01104138232767582, "fcm_dpo/delta": -0.008695616386830807, "fcm_dpo/margin": 20.151180267333984, "fcm_dpo/q_t": 0.4494304060935974, "grad_norm": 69.4866714477539, "learning_rate": 4.476396981707453e-07, "logits/chosen": -0.8099366426467896, "logits/rejected": -0.8398616313934326, "logps/chosen": -305.580810546875, "logps/ref_chosen": -270.0443115234375, "logps/ref_rejected": -267.3226013183594, "logps/rejected": -323.01025390625, "loss": 5.0408, "margin_dpo/margin_mean": 20.151180267333984, "margin_dpo/margin_std": 51.15916442871094, "step": 139 }, { "KL/chosen_KL_mean": -33.201412200927734, "KL/mean": -48.6481819152832, "KL/rejected_KL_mean": -64.09495544433594, "KL/std": 43.8369255065918, "epoch": 0.2931937172774869, "fcm_dpo/beta": 0.010931570082902908, "fcm_dpo/delta": -0.034940119832754135, "fcm_dpo/margin": 30.893543243408203, "fcm_dpo/q_t": 0.4216638505458832, "grad_norm": 62.90419006347656, "learning_rate": 4.4651327368569684e-07, "logits/chosen": -0.8739014863967896, "logits/rejected": -0.8388884663581848, "logps/chosen": -316.156982421875, "logps/ref_chosen": -282.9555969238281, "logps/ref_rejected": -251.17181396484375, "logps/rejected": -315.2667541503906, "loss": 4.596, "margin_dpo/margin_mean": 30.893543243408203, "margin_dpo/margin_std": 47.828704833984375, "step": 140 }, { "KL/chosen_KL_mean": -31.26448631286621, "KL/mean": -47.31875228881836, "KL/rejected_KL_mean": -63.37302017211914, "KL/std": 45.12312316894531, "epoch": 0.29528795811518327, "fcm_dpo/beta": 0.010553342290222645, "fcm_dpo/delta": -0.02428608201444149, "fcm_dpo/margin": 32.10853958129883, "fcm_dpo/q_t": 0.42113417387008667, "grad_norm": 68.2318344116211, "learning_rate": 4.453763107901675e-07, "logits/chosen": -0.8127657771110535, "logits/rejected": -0.8073655366897583, "logps/chosen": -327.56463623046875, "logps/ref_chosen": -296.3001708984375, "logps/ref_rejected": -279.8486633300781, "logps/rejected": -343.2216796875, "loss": 4.6544, "margin_dpo/margin_mean": 32.10853958129883, "margin_dpo/margin_std": 55.96519470214844, "step": 141 }, { "KL/chosen_KL_mean": -32.01213836669922, "KL/mean": -45.19526672363281, "KL/rejected_KL_mean": -58.378387451171875, "KL/std": 45.81889343261719, "epoch": 0.2973821989528796, "fcm_dpo/beta": 0.01045268028974533, "fcm_dpo/delta": -0.0036399501841515303, "fcm_dpo/margin": 26.366239547729492, "fcm_dpo/q_t": 0.4370165467262268, "grad_norm": 58.871788024902344, "learning_rate": 4.4422887045602674e-07, "logits/chosen": -0.8336638808250427, "logits/rejected": -0.8351055979728699, "logps/chosen": -332.5780029296875, "logps/ref_chosen": -300.56585693359375, "logps/ref_rejected": -231.43316650390625, "logps/rejected": -289.8115539550781, "loss": 4.8816, "margin_dpo/margin_mean": 26.366239547729492, "margin_dpo/margin_std": 56.89813995361328, "step": 142 }, { "KL/chosen_KL_mean": -31.096927642822266, "KL/mean": -47.20490264892578, "KL/rejected_KL_mean": -63.3128776550293, "KL/std": 43.217559814453125, "epoch": 0.2994764397905759, "fcm_dpo/beta": 0.01030801422894001, "fcm_dpo/delta": -0.026560742408037186, "fcm_dpo/margin": 32.2159538269043, "fcm_dpo/q_t": 0.42342373728752136, "grad_norm": 58.806697845458984, "learning_rate": 4.4307101421701755e-07, "logits/chosen": -0.8223780393600464, "logits/rejected": -0.8012036681175232, "logps/chosen": -327.82928466796875, "logps/ref_chosen": -296.73236083984375, "logps/ref_rejected": -266.45257568359375, "logps/rejected": -329.7654724121094, "loss": 4.6057, "margin_dpo/margin_mean": 32.2159538269043, "margin_dpo/margin_std": 50.72016143798828, "step": 143 }, { "KL/chosen_KL_mean": -31.219467163085938, "KL/mean": -44.27275848388672, "KL/rejected_KL_mean": -57.326053619384766, "KL/std": 46.5449333190918, "epoch": 0.30157068062827225, "fcm_dpo/beta": 0.010053442791104317, "fcm_dpo/delta": -0.015576949343085289, "fcm_dpo/margin": 26.106597900390625, "fcm_dpo/q_t": 0.44039374589920044, "grad_norm": 52.784488677978516, "learning_rate": 4.419028041654559e-07, "logits/chosen": -0.8913372755050659, "logits/rejected": -0.8779230117797852, "logps/chosen": -330.06341552734375, "logps/ref_chosen": -298.843994140625, "logps/ref_rejected": -266.120849609375, "logps/rejected": -323.4468994140625, "loss": 4.8861, "margin_dpo/margin_mean": 26.10660171508789, "margin_dpo/margin_std": 56.03483963012695, "step": 144 }, { "KL/chosen_KL_mean": -27.039196014404297, "KL/mean": -43.98484420776367, "KL/rejected_KL_mean": -60.930477142333984, "KL/std": 42.91300964355469, "epoch": 0.3036649214659686, "fcm_dpo/beta": 0.00988059677183628, "fcm_dpo/delta": -0.03598638251423836, "fcm_dpo/margin": 33.89128112792969, "fcm_dpo/q_t": 0.42246097326278687, "grad_norm": 58.97926712036133, "learning_rate": 4.4072430294890166e-07, "logits/chosen": -0.8730674982070923, "logits/rejected": -0.8762320876121521, "logps/chosen": -302.79205322265625, "logps/ref_chosen": -275.7528381347656, "logps/ref_rejected": -214.74807739257812, "logps/rejected": -275.6785583496094, "loss": 4.6006, "margin_dpo/margin_mean": 33.89128112792969, "margin_dpo/margin_std": 52.56437683105469, "step": 145 }, { "KL/chosen_KL_mean": -36.02202606201172, "KL/mean": -52.752845764160156, "KL/rejected_KL_mean": -69.48365783691406, "KL/std": 47.3927001953125, "epoch": 0.3057591623036649, "fcm_dpo/beta": 0.009669817984104156, "fcm_dpo/delta": -0.026783179491758347, "fcm_dpo/margin": 33.461639404296875, "fcm_dpo/q_t": 0.42534470558166504, "grad_norm": 59.773223876953125, "learning_rate": 4.395355737667985e-07, "logits/chosen": -0.8538410067558289, "logits/rejected": -0.848624587059021, "logps/chosen": -313.1202392578125, "logps/ref_chosen": -277.09820556640625, "logps/ref_rejected": -265.41046142578125, "logps/rejected": -334.89410400390625, "loss": 4.6348, "margin_dpo/margin_mean": 33.461639404296875, "margin_dpo/margin_std": 52.627410888671875, "step": 146 }, { "KL/chosen_KL_mean": -37.466670989990234, "KL/mean": -50.719993591308594, "KL/rejected_KL_mean": -63.97330856323242, "KL/std": 49.072593688964844, "epoch": 0.3078534031413613, "fcm_dpo/beta": 0.009586220607161522, "fcm_dpo/delta": -0.023667726665735245, "fcm_dpo/margin": 26.50663185119629, "fcm_dpo/q_t": 0.4408421814441681, "grad_norm": 54.904624938964844, "learning_rate": 4.3833668036708483e-07, "logits/chosen": -0.83968186378479, "logits/rejected": -0.8350346088409424, "logps/chosen": -328.88525390625, "logps/ref_chosen": -291.4185791015625, "logps/ref_rejected": -253.43051147460938, "logps/rejected": -317.40380859375, "loss": 4.9045, "margin_dpo/margin_mean": 26.506629943847656, "margin_dpo/margin_std": 54.914512634277344, "step": 147 }, { "KL/chosen_KL_mean": -35.33280944824219, "KL/mean": -49.60417175292969, "KL/rejected_KL_mean": -63.87554168701172, "KL/std": 51.94282913208008, "epoch": 0.3099476439790576, "fcm_dpo/beta": 0.009600591845810413, "fcm_dpo/delta": 0.039563555270433426, "fcm_dpo/margin": 28.542724609375, "fcm_dpo/q_t": 0.43925410509109497, "grad_norm": 56.438453674316406, "learning_rate": 4.3712768704277524e-07, "logits/chosen": -0.9007651209831238, "logits/rejected": -0.9030435681343079, "logps/chosen": -272.0813293457031, "logps/ref_chosen": -236.74850463867188, "logps/ref_rejected": -231.4674072265625, "logps/rejected": -295.3429260253906, "loss": 4.8703, "margin_dpo/margin_mean": 28.542724609375, "margin_dpo/margin_std": 62.54542922973633, "step": 148 }, { "KL/chosen_KL_mean": -41.490516662597656, "KL/mean": -60.27011489868164, "KL/rejected_KL_mean": -79.04971313476562, "KL/std": 47.53480911254883, "epoch": 0.31204188481675393, "fcm_dpo/beta": 0.009786082431674004, "fcm_dpo/delta": 0.0004139197990298271, "fcm_dpo/margin": 37.55918884277344, "fcm_dpo/q_t": 0.4152625799179077, "grad_norm": 60.076107025146484, "learning_rate": 4.3590865862851263e-07, "logits/chosen": -0.8776776790618896, "logits/rejected": -0.8625072836875916, "logps/chosen": -361.41900634765625, "logps/ref_chosen": -319.9284973144531, "logps/ref_rejected": -308.20233154296875, "logps/rejected": -387.2520751953125, "loss": 4.4703, "margin_dpo/margin_mean": 37.55918884277344, "margin_dpo/margin_std": 52.95542526245117, "step": 149 }, { "KL/chosen_KL_mean": -47.29694366455078, "KL/mean": -62.3214225769043, "KL/rejected_KL_mean": -77.34589385986328, "KL/std": 50.34033966064453, "epoch": 0.31413612565445026, "fcm_dpo/beta": 0.009834048338234425, "fcm_dpo/delta": 0.013316557742655277, "fcm_dpo/margin": 30.048938751220703, "fcm_dpo/q_t": 0.4316878020763397, "grad_norm": 59.99736022949219, "learning_rate": 4.346796604970912e-07, "logits/chosen": -0.8376021981239319, "logits/rejected": -0.8213092088699341, "logps/chosen": -323.61517333984375, "logps/ref_chosen": -276.3182373046875, "logps/ref_rejected": -273.02215576171875, "logps/rejected": -350.3680114746094, "loss": 4.7488, "margin_dpo/margin_mean": 30.04894256591797, "margin_dpo/margin_std": 55.58155059814453, "step": 150 }, { "KL/chosen_KL_mean": -44.79521560668945, "KL/mean": -66.85020446777344, "KL/rejected_KL_mean": -88.90518951416016, "KL/std": 50.85723876953125, "epoch": 0.3162303664921466, "fcm_dpo/beta": 0.009750737808644772, "fcm_dpo/delta": -0.03168368339538574, "fcm_dpo/margin": 44.109981536865234, "fcm_dpo/q_t": 0.4018837809562683, "grad_norm": 62.32393264770508, "learning_rate": 4.3344075855595097e-07, "logits/chosen": -0.8728746771812439, "logits/rejected": -0.8648634552955627, "logps/chosen": -342.1080322265625, "logps/ref_chosen": -297.31280517578125, "logps/ref_rejected": -266.1003723144531, "logps/rejected": -355.0055847167969, "loss": 4.3372, "margin_dpo/margin_mean": 44.109981536865234, "margin_dpo/margin_std": 58.83431625366211, "step": 151 }, { "KL/chosen_KL_mean": -39.10889434814453, "KL/mean": -57.962406158447266, "KL/rejected_KL_mean": -76.81593322753906, "KL/std": 53.661624908447266, "epoch": 0.3183246073298429, "fcm_dpo/beta": 0.009311305359005928, "fcm_dpo/delta": -0.05312967672944069, "fcm_dpo/margin": 37.7070198059082, "fcm_dpo/q_t": 0.421106219291687, "grad_norm": 54.742496490478516, "learning_rate": 4.3219201924364323e-07, "logits/chosen": -0.8810481429100037, "logits/rejected": -0.8775116205215454, "logps/chosen": -309.35589599609375, "logps/ref_chosen": -270.2470397949219, "logps/ref_rejected": -269.7749328613281, "logps/rejected": -346.59088134765625, "loss": 4.6232, "margin_dpo/margin_mean": 37.7070198059082, "margin_dpo/margin_std": 62.859439849853516, "step": 152 }, { "KL/chosen_KL_mean": -46.92686462402344, "KL/mean": -73.7924575805664, "KL/rejected_KL_mean": -100.65806579589844, "KL/std": 58.3165397644043, "epoch": 0.3204188481675393, "fcm_dpo/beta": 0.008776971139013767, "fcm_dpo/delta": -0.09027046710252762, "fcm_dpo/margin": 53.731204986572266, "fcm_dpo/q_t": 0.39192867279052734, "grad_norm": 56.017066955566406, "learning_rate": 4.309335095262675e-07, "logits/chosen": -0.8752989172935486, "logits/rejected": -0.8623684048652649, "logps/chosen": -320.7059326171875, "logps/ref_chosen": -273.779052734375, "logps/ref_rejected": -280.9530944824219, "logps/rejected": -381.6112060546875, "loss": 4.1495, "margin_dpo/margin_mean": 53.731204986572266, "margin_dpo/margin_std": 60.12602996826172, "step": 153 }, { "KL/chosen_KL_mean": -47.703121185302734, "KL/mean": -68.70870208740234, "KL/rejected_KL_mean": -89.71429443359375, "KL/std": 60.09400177001953, "epoch": 0.3225130890052356, "fcm_dpo/beta": 0.0084590008482337, "fcm_dpo/delta": -0.023284776136279106, "fcm_dpo/margin": 42.011173248291016, "fcm_dpo/q_t": 0.41954493522644043, "grad_norm": 59.05760955810547, "learning_rate": 4.2966529689388064e-07, "logits/chosen": -0.9124890565872192, "logits/rejected": -0.8928197622299194, "logps/chosen": -337.6063232421875, "logps/ref_chosen": -289.9031982421875, "logps/ref_rejected": -261.5166320800781, "logps/rejected": -351.2309265136719, "loss": 4.6048, "margin_dpo/margin_mean": 42.011173248291016, "margin_dpo/margin_std": 70.03306579589844, "step": 154 }, { "KL/chosen_KL_mean": -54.11407470703125, "KL/mean": -72.34459686279297, "KL/rejected_KL_mean": -90.57511901855469, "KL/std": 57.328651428222656, "epoch": 0.32460732984293195, "fcm_dpo/beta": 0.008467341773211956, "fcm_dpo/delta": 0.010241080075502396, "fcm_dpo/margin": 36.461036682128906, "fcm_dpo/q_t": 0.4311079978942871, "grad_norm": 69.09295654296875, "learning_rate": 4.2838744935687716e-07, "logits/chosen": -0.8517003655433655, "logits/rejected": -0.848478376865387, "logps/chosen": -339.97528076171875, "logps/ref_chosen": -285.8612060546875, "logps/ref_rejected": -300.1272888183594, "logps/rejected": -390.702392578125, "loss": 4.7183, "margin_dpo/margin_mean": 36.461036682128906, "margin_dpo/margin_std": 65.83427429199219, "step": 155 }, { "KL/chosen_KL_mean": -50.88059997558594, "KL/mean": -76.42092895507812, "KL/rejected_KL_mean": -101.96127319335938, "KL/std": 61.5806884765625, "epoch": 0.3267015706806283, "fcm_dpo/beta": 0.008312368765473366, "fcm_dpo/delta": -0.07811664044857025, "fcm_dpo/margin": 51.08067321777344, "fcm_dpo/q_t": 0.4043055474758148, "grad_norm": 58.513641357421875, "learning_rate": 4.271000354423425e-07, "logits/chosen": -0.8742014765739441, "logits/rejected": -0.8701291084289551, "logps/chosen": -329.91607666015625, "logps/ref_chosen": -279.0354919433594, "logps/ref_rejected": -244.2198486328125, "logps/rejected": -346.1811218261719, "loss": 4.4398, "margin_dpo/margin_mean": 51.08067321777344, "margin_dpo/margin_std": 77.06956481933594, "step": 156 }, { "KL/chosen_KL_mean": -54.794883728027344, "KL/mean": -74.77871704101562, "KL/rejected_KL_mean": -94.76253509521484, "KL/std": 61.28529357910156, "epoch": 0.3287958115183246, "fcm_dpo/beta": 0.007772133685648441, "fcm_dpo/delta": -0.0433635450899601, "fcm_dpo/margin": 39.9676513671875, "fcm_dpo/q_t": 0.4275224804878235, "grad_norm": 53.20646286010742, "learning_rate": 4.258031241903777e-07, "logits/chosen": -0.9390982985496521, "logits/rejected": -0.937108039855957, "logps/chosen": -325.62518310546875, "logps/ref_chosen": -270.830322265625, "logps/ref_rejected": -259.08319091796875, "logps/rejected": -353.84576416015625, "loss": 4.6787, "margin_dpo/margin_mean": 39.9676513671875, "margin_dpo/margin_std": 66.26377868652344, "step": 157 }, { "KL/chosen_KL_mean": -54.49103546142578, "KL/mean": -77.38906860351562, "KL/rejected_KL_mean": -100.28709411621094, "KL/std": 62.60470199584961, "epoch": 0.3308900523560209, "fcm_dpo/beta": 0.007661835290491581, "fcm_dpo/delta": -0.022122088819742203, "fcm_dpo/margin": 45.79604721069336, "fcm_dpo/q_t": 0.4180770218372345, "grad_norm": 55.99773406982422, "learning_rate": 4.2449678515039743e-07, "logits/chosen": -0.8606072068214417, "logits/rejected": -0.8443244695663452, "logps/chosen": -344.4573669433594, "logps/ref_chosen": -289.9663391113281, "logps/ref_rejected": -271.335693359375, "logps/rejected": -371.6227722167969, "loss": 4.5509, "margin_dpo/margin_mean": 45.796051025390625, "margin_dpo/margin_std": 67.24652862548828, "step": 158 }, { "KL/chosen_KL_mean": -60.751251220703125, "KL/mean": -74.46627807617188, "KL/rejected_KL_mean": -88.18130493164062, "KL/std": 58.07416534423828, "epoch": 0.33298429319371725, "fcm_dpo/beta": 0.007585161365568638, "fcm_dpo/delta": 0.009133652783930302, "fcm_dpo/margin": 27.430057525634766, "fcm_dpo/q_t": 0.4531330466270447, "grad_norm": 74.61146545410156, "learning_rate": 4.2318108837739986e-07, "logits/chosen": -0.9520941972732544, "logits/rejected": -0.9025680422782898, "logps/chosen": -382.12957763671875, "logps/ref_chosen": -321.37835693359375, "logps/ref_rejected": -250.45652770996094, "logps/rejected": -338.63787841796875, "loss": 5.0799, "margin_dpo/margin_mean": 27.430057525634766, "margin_dpo/margin_std": 72.122802734375, "step": 159 }, { "KL/chosen_KL_mean": -50.046653747558594, "KL/mean": -75.42059326171875, "KL/rejected_KL_mean": -100.79450988769531, "KL/std": 61.544471740722656, "epoch": 0.33507853403141363, "fcm_dpo/beta": 0.00758227426558733, "fcm_dpo/delta": -0.000992432702332735, "fcm_dpo/margin": 50.74785614013672, "fcm_dpo/q_t": 0.41024836897850037, "grad_norm": 55.165855407714844, "learning_rate": 4.218561044282098e-07, "logits/chosen": -0.8842563629150391, "logits/rejected": -0.8855242133140564, "logps/chosen": -326.3301696777344, "logps/ref_chosen": -276.28350830078125, "logps/ref_rejected": -262.7477722167969, "logps/rejected": -363.54229736328125, "loss": 4.3613, "margin_dpo/margin_mean": 50.74785614013672, "margin_dpo/margin_std": 59.35392761230469, "step": 160 }, { "KL/chosen_KL_mean": -59.74705123901367, "KL/mean": -82.03538513183594, "KL/rejected_KL_mean": -104.32372283935547, "KL/std": 64.49849700927734, "epoch": 0.33717277486910996, "fcm_dpo/beta": 0.00754944933578372, "fcm_dpo/delta": -0.02097606658935547, "fcm_dpo/margin": 44.5766716003418, "fcm_dpo/q_t": 0.42417094111442566, "grad_norm": 75.2176284790039, "learning_rate": 4.2052190435769554e-07, "logits/chosen": -0.8945316076278687, "logits/rejected": -0.8761196136474609, "logps/chosen": -370.2398376464844, "logps/ref_chosen": -310.4927978515625, "logps/ref_rejected": -250.25347900390625, "logps/rejected": -354.57720947265625, "loss": 4.6438, "margin_dpo/margin_mean": 44.5766716003418, "margin_dpo/margin_std": 75.5418701171875, "step": 161 }, { "KL/chosen_KL_mean": -54.763946533203125, "KL/mean": -75.6114273071289, "KL/rejected_KL_mean": -96.45890808105469, "KL/std": 60.076480865478516, "epoch": 0.3392670157068063, "fcm_dpo/beta": 0.00734911672770977, "fcm_dpo/delta": -0.007708599790930748, "fcm_dpo/margin": 41.69496154785156, "fcm_dpo/q_t": 0.42868733406066895, "grad_norm": 45.79068374633789, "learning_rate": 4.1917855971495763e-07, "logits/chosen": -0.8724191784858704, "logits/rejected": -0.8640581965446472, "logps/chosen": -350.8743896484375, "logps/ref_chosen": -296.1105041503906, "logps/ref_rejected": -253.4247589111328, "logps/rejected": -349.8836669921875, "loss": 4.6753, "margin_dpo/margin_mean": 41.69496536254883, "margin_dpo/margin_std": 67.41392517089844, "step": 162 }, { "KL/chosen_KL_mean": -59.032291412353516, "KL/mean": -82.56084442138672, "KL/rejected_KL_mean": -106.08937072753906, "KL/std": 59.548545837402344, "epoch": 0.3413612565445026, "fcm_dpo/beta": 0.00737118162214756, "fcm_dpo/delta": -0.02219436690211296, "fcm_dpo/margin": 47.05707550048828, "fcm_dpo/q_t": 0.4185625910758972, "grad_norm": 65.94065856933594, "learning_rate": 4.1782614253949255e-07, "logits/chosen": -0.9133932590484619, "logits/rejected": -0.9096333384513855, "logps/chosen": -352.5321960449219, "logps/ref_chosen": -293.4999084472656, "logps/ref_rejected": -266.7116394042969, "logps/rejected": -372.801025390625, "loss": 4.5254, "margin_dpo/margin_mean": 47.05707550048828, "margin_dpo/margin_std": 65.4988021850586, "step": 163 }, { "KL/chosen_KL_mean": -69.27760314941406, "KL/mean": -90.69017028808594, "KL/rejected_KL_mean": -112.10274505615234, "KL/std": 66.29005432128906, "epoch": 0.34345549738219894, "fcm_dpo/beta": 0.007323879282921553, "fcm_dpo/delta": -0.013420582748949528, "fcm_dpo/margin": 42.82514572143555, "fcm_dpo/q_t": 0.42865079641342163, "grad_norm": 72.55785369873047, "learning_rate": 4.164647253573289e-07, "logits/chosen": -0.86087965965271, "logits/rejected": -0.8710927963256836, "logps/chosen": -336.3271179199219, "logps/ref_chosen": -267.04949951171875, "logps/ref_rejected": -215.9768829345703, "logps/rejected": -328.0796203613281, "loss": 4.7004, "margin_dpo/margin_mean": 42.82514572143555, "margin_dpo/margin_std": 75.33882904052734, "step": 164 }, { "KL/chosen_KL_mean": -57.162994384765625, "KL/mean": -72.81074523925781, "KL/rejected_KL_mean": -88.45850372314453, "KL/std": 55.15323257446289, "epoch": 0.34554973821989526, "fcm_dpo/beta": 0.007216047961264849, "fcm_dpo/delta": 0.00300390413030982, "fcm_dpo/margin": 31.295488357543945, "fcm_dpo/q_t": 0.4470052123069763, "grad_norm": 65.5068588256836, "learning_rate": 4.1509438117713863e-07, "logits/chosen": -0.8794437646865845, "logits/rejected": -0.8518754243850708, "logps/chosen": -335.2244873046875, "logps/ref_chosen": -278.06146240234375, "logps/ref_rejected": -260.4288635253906, "logps/rejected": -348.88739013671875, "loss": 4.908, "margin_dpo/margin_mean": 31.295486450195312, "margin_dpo/margin_std": 61.843997955322266, "step": 165 }, { "KL/chosen_KL_mean": -51.96410369873047, "KL/mean": -71.39328002929688, "KL/rejected_KL_mean": -90.82244873046875, "KL/std": 58.12259292602539, "epoch": 0.34764397905759165, "fcm_dpo/beta": 0.007127226330339909, "fcm_dpo/delta": -0.0192001573741436, "fcm_dpo/margin": 38.85832977294922, "fcm_dpo/q_t": 0.43620848655700684, "grad_norm": 58.496612548828125, "learning_rate": 4.137151834863213e-07, "logits/chosen": -0.8499168157577515, "logits/rejected": -0.8151109218597412, "logps/chosen": -327.91314697265625, "logps/ref_chosen": -275.9490661621094, "logps/ref_rejected": -232.13473510742188, "logps/rejected": -322.9571838378906, "loss": 4.776, "margin_dpo/margin_mean": 38.85832977294922, "margin_dpo/margin_std": 71.73967742919922, "step": 166 }, { "KL/chosen_KL_mean": -60.47264099121094, "KL/mean": -83.17312622070312, "KL/rejected_KL_mean": -105.87361145019531, "KL/std": 63.884830474853516, "epoch": 0.34973821989528797, "fcm_dpo/beta": 0.007206224836409092, "fcm_dpo/delta": 0.01624479703605175, "fcm_dpo/margin": 45.400970458984375, "fcm_dpo/q_t": 0.4238249361515045, "grad_norm": 78.14469146728516, "learning_rate": 4.123272062470633e-07, "logits/chosen": -0.8364863395690918, "logits/rejected": -0.8216645121574402, "logps/chosen": -341.0240478515625, "logps/ref_chosen": -280.5514221191406, "logps/ref_rejected": -255.2896728515625, "logps/rejected": -361.1632995605469, "loss": 4.6281, "margin_dpo/margin_mean": 45.400970458984375, "margin_dpo/margin_std": 73.75985717773438, "step": 167 }, { "KL/chosen_KL_mean": -62.252197265625, "KL/mean": -86.59355163574219, "KL/rejected_KL_mean": -110.93489837646484, "KL/std": 66.37821960449219, "epoch": 0.3518324607329843, "fcm_dpo/beta": 0.007193275727331638, "fcm_dpo/delta": -0.006395729724317789, "fcm_dpo/margin": 48.682701110839844, "fcm_dpo/q_t": 0.41802850365638733, "grad_norm": 124.62866973876953, "learning_rate": 4.1093052389237174e-07, "logits/chosen": -0.8164465427398682, "logits/rejected": -0.7877838611602783, "logps/chosen": -378.0504150390625, "logps/ref_chosen": -315.7982177734375, "logps/ref_rejected": -291.48406982421875, "logps/rejected": -402.4189758300781, "loss": 4.5506, "margin_dpo/margin_mean": 48.682701110839844, "margin_dpo/margin_std": 73.01609802246094, "step": 168 }, { "KL/chosen_KL_mean": -57.24802017211914, "KL/mean": -87.19869232177734, "KL/rejected_KL_mean": -117.14934539794922, "KL/std": 67.05406188964844, "epoch": 0.3539267015706806, "fcm_dpo/beta": 0.007014545612037182, "fcm_dpo/delta": -0.061939746141433716, "fcm_dpo/margin": 59.90133285522461, "fcm_dpo/q_t": 0.40285056829452515, "grad_norm": 89.98390197753906, "learning_rate": 4.0952521132208267e-07, "logits/chosen": -0.8162607550621033, "logits/rejected": -0.8252590894699097, "logps/chosen": -318.3122863769531, "logps/ref_chosen": -261.06427001953125, "logps/ref_rejected": -235.40663146972656, "logps/rejected": -352.55596923828125, "loss": 4.3106, "margin_dpo/margin_mean": 59.90133285522461, "margin_dpo/margin_std": 72.81208801269531, "step": 169 }, { "KL/chosen_KL_mean": -56.95838165283203, "KL/mean": -81.45191955566406, "KL/rejected_KL_mean": -105.94544982910156, "KL/std": 70.3679428100586, "epoch": 0.35602094240837695, "fcm_dpo/beta": 0.006829574704170227, "fcm_dpo/delta": 0.006389938294887543, "fcm_dpo/margin": 48.9870719909668, "fcm_dpo/q_t": 0.42464104294776917, "grad_norm": 79.10480499267578, "learning_rate": 4.081113438988443e-07, "logits/chosen": -0.7996067404747009, "logits/rejected": -0.8019770383834839, "logps/chosen": -365.9256286621094, "logps/ref_chosen": -308.96722412109375, "logps/ref_rejected": -263.8466796875, "logps/rejected": -369.7921142578125, "loss": 4.6634, "margin_dpo/margin_mean": 48.98707580566406, "margin_dpo/margin_std": 88.27198791503906, "step": 170 }, { "KL/chosen_KL_mean": -48.412353515625, "KL/mean": -77.40523529052734, "KL/rejected_KL_mean": -106.39811706542969, "KL/std": 66.31096649169922, "epoch": 0.3581151832460733, "fcm_dpo/beta": 0.006731792353093624, "fcm_dpo/delta": -0.03171330690383911, "fcm_dpo/margin": 57.98576354980469, "fcm_dpo/q_t": 0.40904250741004944, "grad_norm": 76.98529052734375, "learning_rate": 4.0668899744407567e-07, "logits/chosen": -0.8132805824279785, "logits/rejected": -0.8281208872795105, "logps/chosen": -307.3013916015625, "logps/ref_chosen": -258.8890380859375, "logps/ref_rejected": -262.19140625, "logps/rejected": -368.5894775390625, "loss": 4.3738, "margin_dpo/margin_mean": 57.98576354980469, "margin_dpo/margin_std": 71.38015747070312, "step": 171 }, { "KL/chosen_KL_mean": -54.201725006103516, "KL/mean": -70.12786865234375, "KL/rejected_KL_mean": -86.05400085449219, "KL/std": 60.457340240478516, "epoch": 0.36020942408376966, "fcm_dpo/beta": 0.006770864129066467, "fcm_dpo/delta": 0.021626032888889313, "fcm_dpo/margin": 31.85228157043457, "fcm_dpo/q_t": 0.4491257965564728, "grad_norm": 90.36054229736328, "learning_rate": 4.0525824823390043e-07, "logits/chosen": -0.8342244625091553, "logits/rejected": -0.850119411945343, "logps/chosen": -393.22406005859375, "logps/ref_chosen": -339.0223388671875, "logps/ref_rejected": -295.78759765625, "logps/rejected": -381.84161376953125, "loss": 4.9797, "margin_dpo/margin_mean": 31.852283477783203, "margin_dpo/margin_std": 74.22462463378906, "step": 172 }, { "KL/chosen_KL_mean": -56.15380096435547, "KL/mean": -77.50936889648438, "KL/rejected_KL_mean": -98.86494445800781, "KL/std": 64.10746765136719, "epoch": 0.362303664921466, "fcm_dpo/beta": 0.006724307779222727, "fcm_dpo/delta": -0.025682823732495308, "fcm_dpo/margin": 42.711151123046875, "fcm_dpo/q_t": 0.43435436487197876, "grad_norm": 104.70634460449219, "learning_rate": 4.0381917299505686e-07, "logits/chosen": -0.8539585471153259, "logits/rejected": -0.8501390814781189, "logps/chosen": -356.2652282714844, "logps/ref_chosen": -300.1114501953125, "logps/ref_rejected": -273.78460693359375, "logps/rejected": -372.6495666503906, "loss": 4.7538, "margin_dpo/margin_mean": 42.711151123046875, "margin_dpo/margin_std": 76.24950408935547, "step": 173 }, { "KL/chosen_KL_mean": -56.64954376220703, "KL/mean": -83.61141967773438, "KL/rejected_KL_mean": -110.57327270507812, "KL/std": 65.21598815917969, "epoch": 0.3643979057591623, "fcm_dpo/beta": 0.00670324731618166, "fcm_dpo/delta": 0.006629655137658119, "fcm_dpo/margin": 53.923744201660156, "fcm_dpo/q_t": 0.41451215744018555, "grad_norm": 70.95905303955078, "learning_rate": 4.0237184890078243e-07, "logits/chosen": -0.8421565294265747, "logits/rejected": -0.8234596848487854, "logps/chosen": -391.703369140625, "logps/ref_chosen": -335.0538635253906, "logps/ref_rejected": -257.4646911621094, "logps/rejected": -368.0379638671875, "loss": 4.4474, "margin_dpo/margin_mean": 53.923744201660156, "margin_dpo/margin_std": 69.16473388671875, "step": 174 }, { "KL/chosen_KL_mean": -73.34457397460938, "KL/mean": -97.99015808105469, "KL/rejected_KL_mean": -122.63572692871094, "KL/std": 74.52360534667969, "epoch": 0.36649214659685864, "fcm_dpo/beta": 0.006607607007026672, "fcm_dpo/delta": -0.04977422207593918, "fcm_dpo/margin": 49.29114532470703, "fcm_dpo/q_t": 0.42731624841690063, "grad_norm": 61.65116882324219, "learning_rate": 4.00916353566676e-07, "logits/chosen": -0.8486623764038086, "logits/rejected": -0.8486083149909973, "logps/chosen": -357.7401428222656, "logps/ref_chosen": -284.39556884765625, "logps/ref_rejected": -283.3876647949219, "logps/rejected": -406.02337646484375, "loss": 4.6827, "margin_dpo/margin_mean": 49.29115295410156, "margin_dpo/margin_std": 83.50072479248047, "step": 175 }, { "KL/chosen_KL_mean": -77.07302856445312, "KL/mean": -96.13621520996094, "KL/rejected_KL_mean": -115.19940185546875, "KL/std": 70.39031219482422, "epoch": 0.36858638743455496, "fcm_dpo/beta": 0.006414460018277168, "fcm_dpo/delta": -0.01322941668331623, "fcm_dpo/margin": 38.12636184692383, "fcm_dpo/q_t": 0.44413357973098755, "grad_norm": 84.40768432617188, "learning_rate": 3.994527650465352e-07, "logits/chosen": -0.8279335498809814, "logits/rejected": -0.8326810002326965, "logps/chosen": -328.8858337402344, "logps/ref_chosen": -251.81280517578125, "logps/ref_rejected": -242.05328369140625, "logps/rejected": -357.2526550292969, "loss": 4.9587, "margin_dpo/margin_mean": 38.12635803222656, "margin_dpo/margin_std": 88.72412872314453, "step": 176 }, { "KL/chosen_KL_mean": -91.10723876953125, "KL/mean": -110.1352310180664, "KL/rejected_KL_mean": -129.1632080078125, "KL/std": 69.8150634765625, "epoch": 0.3706806282722513, "fcm_dpo/beta": 0.006443779915571213, "fcm_dpo/delta": 0.014410565607249737, "fcm_dpo/margin": 38.05598449707031, "fcm_dpo/q_t": 0.4435146152973175, "grad_norm": 86.33255004882812, "learning_rate": 3.979811618281705e-07, "logits/chosen": -0.9177388548851013, "logits/rejected": -0.8854184746742249, "logps/chosen": -389.7536315917969, "logps/ref_chosen": -298.6463928222656, "logps/ref_rejected": -295.66534423828125, "logps/rejected": -424.8285217285156, "loss": 4.9676, "margin_dpo/margin_mean": 38.05598068237305, "margin_dpo/margin_std": 86.24855041503906, "step": 177 }, { "KL/chosen_KL_mean": -78.96491241455078, "KL/mean": -106.84060668945312, "KL/rejected_KL_mean": -134.71629333496094, "KL/std": 75.7911376953125, "epoch": 0.37277486910994767, "fcm_dpo/beta": 0.006407511420547962, "fcm_dpo/delta": -0.014248996041715145, "fcm_dpo/margin": 55.751380920410156, "fcm_dpo/q_t": 0.4190220236778259, "grad_norm": 78.22713470458984, "learning_rate": 3.9650162282919654e-07, "logits/chosen": -0.8397620320320129, "logits/rejected": -0.8298240303993225, "logps/chosen": -365.22259521484375, "logps/ref_chosen": -286.2576599121094, "logps/ref_rejected": -243.97491455078125, "logps/rejected": -378.69122314453125, "loss": 4.5792, "margin_dpo/margin_mean": 55.751380920410156, "margin_dpo/margin_std": 90.83074951171875, "step": 178 }, { "KL/chosen_KL_mean": -84.91183471679688, "KL/mean": -111.3154067993164, "KL/rejected_KL_mean": -137.71897888183594, "KL/std": 76.51798248291016, "epoch": 0.374869109947644, "fcm_dpo/beta": 0.006282145623117685, "fcm_dpo/delta": -0.02392883226275444, "fcm_dpo/margin": 52.80714797973633, "fcm_dpo/q_t": 0.42275673151016235, "grad_norm": 82.36101531982422, "learning_rate": 3.9501422739279953e-07, "logits/chosen": -0.8264894485473633, "logits/rejected": -0.825171709060669, "logps/chosen": -344.64892578125, "logps/ref_chosen": -259.737060546875, "logps/ref_rejected": -277.8813171386719, "logps/rejected": -415.6003112792969, "loss": 4.6261, "margin_dpo/margin_mean": 52.80714797973633, "margin_dpo/margin_std": 83.93762969970703, "step": 179 }, { "KL/chosen_KL_mean": -87.58425903320312, "KL/mean": -113.29141235351562, "KL/rejected_KL_mean": -138.99856567382812, "KL/std": 72.59463500976562, "epoch": 0.3769633507853403, "fcm_dpo/beta": 0.006279164459556341, "fcm_dpo/delta": 0.013550683856010437, "fcm_dpo/margin": 51.414302825927734, "fcm_dpo/q_t": 0.4277075529098511, "grad_norm": 102.1092758178711, "learning_rate": 3.935190552834828e-07, "logits/chosen": -0.8694858551025391, "logits/rejected": -0.8932338356971741, "logps/chosen": -354.8931579589844, "logps/ref_chosen": -267.30889892578125, "logps/ref_rejected": -230.4376983642578, "logps/rejected": -369.436279296875, "loss": 4.6452, "margin_dpo/margin_mean": 51.41429901123047, "margin_dpo/margin_std": 85.73615264892578, "step": 180 }, { "KL/chosen_KL_mean": -95.89186096191406, "KL/mean": -118.81684112548828, "KL/rejected_KL_mean": -141.7418212890625, "KL/std": 74.99784088134766, "epoch": 0.37905759162303665, "fcm_dpo/beta": 0.006303788162767887, "fcm_dpo/delta": 0.002721858210861683, "fcm_dpo/margin": 45.849979400634766, "fcm_dpo/q_t": 0.43346357345581055, "grad_norm": 80.63015747070312, "learning_rate": 3.920161866827889e-07, "logits/chosen": -0.857856810092926, "logits/rejected": -0.856174886226654, "logps/chosen": -396.38323974609375, "logps/ref_chosen": -300.49139404296875, "logps/ref_rejected": -278.98284912109375, "logps/rejected": -420.72467041015625, "loss": 4.7731, "margin_dpo/margin_mean": 45.849979400634766, "margin_dpo/margin_std": 86.88410186767578, "step": 181 }, { "KL/chosen_KL_mean": -90.28527069091797, "KL/mean": -123.19674682617188, "KL/rejected_KL_mean": -156.1082000732422, "KL/std": 74.51239013671875, "epoch": 0.381151832460733, "fcm_dpo/beta": 0.00616471329703927, "fcm_dpo/delta": -0.06726302951574326, "fcm_dpo/margin": 65.82292938232422, "fcm_dpo/q_t": 0.40653538703918457, "grad_norm": 70.58544158935547, "learning_rate": 3.90505702185e-07, "logits/chosen": -0.8527212142944336, "logits/rejected": -0.8740633726119995, "logps/chosen": -369.7834167480469, "logps/ref_chosen": -279.4981689453125, "logps/ref_rejected": -263.6926574707031, "logps/rejected": -419.8008117675781, "loss": 4.3893, "margin_dpo/margin_mean": 65.82292938232422, "margin_dpo/margin_std": 87.4244613647461, "step": 182 }, { "KL/chosen_KL_mean": -88.05329895019531, "KL/mean": -115.47635650634766, "KL/rejected_KL_mean": -142.8994140625, "KL/std": 75.31936645507812, "epoch": 0.3832460732984293, "fcm_dpo/beta": 0.006069937255233526, "fcm_dpo/delta": 0.017182346433401108, "fcm_dpo/margin": 54.846107482910156, "fcm_dpo/q_t": 0.42420393228530884, "grad_norm": 76.18987274169922, "learning_rate": 3.889876827928156e-07, "logits/chosen": -0.8800171613693237, "logits/rejected": -0.8790852427482605, "logps/chosen": -359.259033203125, "logps/ref_chosen": -271.2057189941406, "logps/ref_rejected": -243.91549682617188, "logps/rejected": -386.8149108886719, "loss": 4.6149, "margin_dpo/margin_mean": 54.846107482910156, "margin_dpo/margin_std": 89.88021087646484, "step": 183 }, { "KL/chosen_KL_mean": -83.444580078125, "KL/mean": -122.65955352783203, "KL/rejected_KL_mean": -161.87454223632812, "KL/std": 84.31695556640625, "epoch": 0.38534031413612563, "fcm_dpo/beta": 0.00580790126696229, "fcm_dpo/delta": -0.07667214423418045, "fcm_dpo/margin": 78.4299545288086, "fcm_dpo/q_t": 0.39444056153297424, "grad_norm": 72.20956420898438, "learning_rate": 3.874622099130087e-07, "logits/chosen": -0.929442822933197, "logits/rejected": -0.9039252996444702, "logps/chosen": -401.89031982421875, "logps/ref_chosen": -318.4457702636719, "logps/ref_rejected": -266.640869140625, "logps/rejected": -428.5154113769531, "loss": 4.2258, "margin_dpo/margin_mean": 78.4299545288086, "margin_dpo/margin_std": 94.27364349365234, "step": 184 }, { "KL/chosen_KL_mean": -90.38389587402344, "KL/mean": -119.19790649414062, "KL/rejected_KL_mean": -148.01194763183594, "KL/std": 88.37677764892578, "epoch": 0.387434554973822, "fcm_dpo/beta": 0.005671571008861065, "fcm_dpo/delta": -0.00434712041169405, "fcm_dpo/margin": 57.628047943115234, "fcm_dpo/q_t": 0.42499980330467224, "grad_norm": 110.43806457519531, "learning_rate": 3.859293653520604e-07, "logits/chosen": -0.8981891870498657, "logits/rejected": -0.8912757635116577, "logps/chosen": -364.69268798828125, "logps/ref_chosen": -274.308837890625, "logps/ref_rejected": -260.7274169921875, "logps/rejected": -408.7393493652344, "loss": 4.63, "margin_dpo/margin_mean": 57.62804412841797, "margin_dpo/margin_std": 93.35250091552734, "step": 185 }, { "KL/chosen_KL_mean": -89.22308349609375, "KL/mean": -115.18913269042969, "KL/rejected_KL_mean": -141.15518188476562, "KL/std": 79.0693359375, "epoch": 0.38952879581151834, "fcm_dpo/beta": 0.005763459485024214, "fcm_dpo/delta": 0.013718879781663418, "fcm_dpo/margin": 51.932098388671875, "fcm_dpo/q_t": 0.4305761456489563, "grad_norm": 92.03166961669922, "learning_rate": 3.8438923131177237e-07, "logits/chosen": -0.9018892049789429, "logits/rejected": -0.9007343649864197, "logps/chosen": -388.22845458984375, "logps/ref_chosen": -299.00537109375, "logps/ref_rejected": -274.4014587402344, "logps/rejected": -415.5566101074219, "loss": 4.6983, "margin_dpo/margin_mean": 51.932098388671875, "margin_dpo/margin_std": 86.81627655029297, "step": 186 }, { "KL/chosen_KL_mean": -96.13522338867188, "KL/mean": -119.4954605102539, "KL/rejected_KL_mean": -142.85569763183594, "KL/std": 82.50918579101562, "epoch": 0.39162303664921466, "fcm_dpo/beta": 0.0057730223052203655, "fcm_dpo/delta": 0.004285093396902084, "fcm_dpo/margin": 46.72047805786133, "fcm_dpo/q_t": 0.43553128838539124, "grad_norm": 85.7626953125, "learning_rate": 3.828418903848593e-07, "logits/chosen": -0.8542200922966003, "logits/rejected": -0.8379209637641907, "logps/chosen": -425.96051025390625, "logps/ref_chosen": -329.8253173828125, "logps/ref_rejected": -263.73175048828125, "logps/rejected": -406.58746337890625, "loss": 4.8707, "margin_dpo/margin_mean": 46.72047805786133, "margin_dpo/margin_std": 97.68164825439453, "step": 187 }, { "KL/chosen_KL_mean": -85.97469329833984, "KL/mean": -112.66793823242188, "KL/rejected_KL_mean": -139.36119079589844, "KL/std": 84.3029556274414, "epoch": 0.393717277486911, "fcm_dpo/beta": 0.0057581509463489056, "fcm_dpo/delta": -0.01138460636138916, "fcm_dpo/margin": 53.38649368286133, "fcm_dpo/q_t": 0.43061012029647827, "grad_norm": 67.76911926269531, "learning_rate": 3.812874255505191e-07, "logits/chosen": -0.8586629629135132, "logits/rejected": -0.8478258848190308, "logps/chosen": -348.9803161621094, "logps/ref_chosen": -263.005615234375, "logps/ref_rejected": -247.08668518066406, "logps/rejected": -386.4478454589844, "loss": 4.7691, "margin_dpo/margin_mean": 53.38649368286133, "margin_dpo/margin_std": 103.5580825805664, "step": 188 }, { "KL/chosen_KL_mean": -68.06344604492188, "KL/mean": -101.9673080444336, "KL/rejected_KL_mean": -135.8711700439453, "KL/std": 82.10550689697266, "epoch": 0.3958115183246073, "fcm_dpo/beta": 0.00560600683093071, "fcm_dpo/delta": -0.03640016168355942, "fcm_dpo/margin": 67.8077392578125, "fcm_dpo/q_t": 0.4118382930755615, "grad_norm": 69.77622985839844, "learning_rate": 3.797259201699833e-07, "logits/chosen": -0.880431592464447, "logits/rejected": -0.8870152235031128, "logps/chosen": -341.0238342285156, "logps/ref_chosen": -272.96038818359375, "logps/ref_rejected": -275.13238525390625, "logps/rejected": -411.0035705566406, "loss": 4.4049, "margin_dpo/margin_mean": 67.80773162841797, "margin_dpo/margin_std": 85.12550354003906, "step": 189 }, { "KL/chosen_KL_mean": -74.26818084716797, "KL/mean": -101.46267700195312, "KL/rejected_KL_mean": -128.65719604492188, "KL/std": 77.80656433105469, "epoch": 0.39790575916230364, "fcm_dpo/beta": 0.005546726752072573, "fcm_dpo/delta": -0.009571824222803116, "fcm_dpo/margin": 54.38901138305664, "fcm_dpo/q_t": 0.4297546446323395, "grad_norm": 93.16829681396484, "learning_rate": 3.781574579820464e-07, "logits/chosen": -0.8819738030433655, "logits/rejected": -0.841058611869812, "logps/chosen": -332.06573486328125, "logps/ref_chosen": -257.79754638671875, "logps/ref_rejected": -225.2164306640625, "logps/rejected": -353.8736572265625, "loss": 4.657, "margin_dpo/margin_mean": 54.38901138305664, "margin_dpo/margin_std": 85.22750091552734, "step": 190 }, { "KL/chosen_KL_mean": -66.46964263916016, "KL/mean": -94.91885375976562, "KL/rejected_KL_mean": -123.3680648803711, "KL/std": 79.13052368164062, "epoch": 0.4, "fcm_dpo/beta": 0.0055416785180568695, "fcm_dpo/delta": 0.0067793321795761585, "fcm_dpo/margin": 56.89841842651367, "fcm_dpo/q_t": 0.4267407953739166, "grad_norm": 61.709129333496094, "learning_rate": 3.765821230985757e-07, "logits/chosen": -0.9132006764411926, "logits/rejected": -0.9146799445152283, "logps/chosen": -310.3281555175781, "logps/ref_chosen": -243.8585205078125, "logps/ref_rejected": -245.12136840820312, "logps/rejected": -368.48944091796875, "loss": 4.6375, "margin_dpo/margin_mean": 56.89841842651367, "margin_dpo/margin_std": 91.43920135498047, "step": 191 }, { "KL/chosen_KL_mean": -79.08763122558594, "KL/mean": -101.60113525390625, "KL/rejected_KL_mean": -124.11463928222656, "KL/std": 76.82417297363281, "epoch": 0.40209424083769635, "fcm_dpo/beta": 0.0056621660478413105, "fcm_dpo/delta": 0.029165830463171005, "fcm_dpo/margin": 45.02701187133789, "fcm_dpo/q_t": 0.44025444984436035, "grad_norm": 73.44487762451172, "learning_rate": 3.75e-07, "logits/chosen": -0.8472084999084473, "logits/rejected": -0.8306607604026794, "logps/chosen": -346.06756591796875, "logps/ref_chosen": -266.9799499511719, "logps/ref_rejected": -260.1697082519531, "logps/rejected": -384.2843017578125, "loss": 4.824, "margin_dpo/margin_mean": 45.02701187133789, "margin_dpo/margin_std": 83.66022491455078, "step": 192 }, { "KL/chosen_KL_mean": -94.40106964111328, "KL/mean": -121.47966003417969, "KL/rejected_KL_mean": -148.55825805664062, "KL/std": 83.79367065429688, "epoch": 0.4041884816753927, "fcm_dpo/beta": 0.005736154038459063, "fcm_dpo/delta": 0.005536096170544624, "fcm_dpo/margin": 54.157196044921875, "fcm_dpo/q_t": 0.42890992760658264, "grad_norm": 72.95458221435547, "learning_rate": 3.734111735307796e-07, "logits/chosen": -0.8859913945198059, "logits/rejected": -0.8564121723175049, "logps/chosen": -374.654296875, "logps/ref_chosen": -280.25323486328125, "logps/ref_rejected": -291.0348815917969, "logps/rejected": -439.5931396484375, "loss": 4.7054, "margin_dpo/margin_mean": 54.157196044921875, "margin_dpo/margin_std": 94.62388610839844, "step": 193 }, { "KL/chosen_KL_mean": -92.71996307373047, "KL/mean": -116.64244842529297, "KL/rejected_KL_mean": -140.56492614746094, "KL/std": 82.40608978271484, "epoch": 0.406282722513089, "fcm_dpo/beta": 0.00563084939494729, "fcm_dpo/delta": -0.02582578919827938, "fcm_dpo/margin": 47.844966888427734, "fcm_dpo/q_t": 0.43761441111564636, "grad_norm": 99.4243392944336, "learning_rate": 3.7181572889485623e-07, "logits/chosen": -0.8625931739807129, "logits/rejected": -0.8507343530654907, "logps/chosen": -380.8594055175781, "logps/ref_chosen": -288.13946533203125, "logps/ref_rejected": -251.31529235839844, "logps/rejected": -391.8802185058594, "loss": 4.7859, "margin_dpo/margin_mean": 47.844966888427734, "margin_dpo/margin_std": 85.31089782714844, "step": 194 }, { "KL/chosen_KL_mean": -97.91946411132812, "KL/mean": -119.41505432128906, "KL/rejected_KL_mean": -140.91062927246094, "KL/std": 80.02857971191406, "epoch": 0.4083769633507853, "fcm_dpo/beta": 0.005613365676254034, "fcm_dpo/delta": -0.0017420882359147072, "fcm_dpo/margin": 42.99113464355469, "fcm_dpo/q_t": 0.44507643580436707, "grad_norm": 73.5301513671875, "learning_rate": 3.7021375165108377e-07, "logits/chosen": -0.8627254366874695, "logits/rejected": -0.8665878176689148, "logps/chosen": -371.92010498046875, "logps/ref_chosen": -274.0006408691406, "logps/ref_rejected": -280.22723388671875, "logps/rejected": -421.13787841796875, "loss": 4.8972, "margin_dpo/margin_mean": 42.99113464355469, "margin_dpo/margin_std": 88.47681427001953, "step": 195 }, { "KL/chosen_KL_mean": -82.35733032226562, "KL/mean": -109.65511322021484, "KL/rejected_KL_mean": -136.95291137695312, "KL/std": 82.34046936035156, "epoch": 0.41047120418848165, "fcm_dpo/beta": 0.005610158666968346, "fcm_dpo/delta": -0.0006429278291761875, "fcm_dpo/margin": 54.595584869384766, "fcm_dpo/q_t": 0.42899465560913086, "grad_norm": 76.89717864990234, "learning_rate": 3.6860532770864005e-07, "logits/chosen": -0.8541459441184998, "logits/rejected": -0.8584092855453491, "logps/chosen": -357.25799560546875, "logps/ref_chosen": -274.90069580078125, "logps/ref_rejected": -248.7281951904297, "logps/rejected": -385.6811218261719, "loss": 4.6883, "margin_dpo/margin_mean": 54.595584869384766, "margin_dpo/margin_std": 92.98194122314453, "step": 196 }, { "KL/chosen_KL_mean": -92.13257598876953, "KL/mean": -123.7594985961914, "KL/rejected_KL_mean": -155.38644409179688, "KL/std": 82.77664184570312, "epoch": 0.41256544502617803, "fcm_dpo/beta": 0.00550233107060194, "fcm_dpo/delta": -0.03711705654859543, "fcm_dpo/margin": 63.25384521484375, "fcm_dpo/q_t": 0.4192579984664917, "grad_norm": 90.26445770263672, "learning_rate": 3.6699054332241985e-07, "logits/chosen": -0.8871943950653076, "logits/rejected": -0.8706080913543701, "logps/chosen": -401.66741943359375, "logps/ref_chosen": -309.5348205566406, "logps/ref_rejected": -264.3179931640625, "logps/rejected": -419.7044372558594, "loss": 4.54, "margin_dpo/margin_mean": 63.25384521484375, "margin_dpo/margin_std": 89.81802368164062, "step": 197 }, { "KL/chosen_KL_mean": -87.10678100585938, "KL/mean": -119.78683471679688, "KL/rejected_KL_mean": -152.46690368652344, "KL/std": 88.0517807006836, "epoch": 0.41465968586387436, "fcm_dpo/beta": 0.005422430112957954, "fcm_dpo/delta": 0.002035717014223337, "fcm_dpo/margin": 65.360107421875, "fcm_dpo/q_t": 0.41849538683891296, "grad_norm": 61.8067512512207, "learning_rate": 3.653694850884091e-07, "logits/chosen": -0.8917360305786133, "logits/rejected": -0.8658944964408875, "logps/chosen": -388.1202392578125, "logps/ref_chosen": -301.0134582519531, "logps/ref_rejected": -292.84185791015625, "logps/rejected": -445.3087158203125, "loss": 4.5293, "margin_dpo/margin_mean": 65.36009979248047, "margin_dpo/margin_std": 97.01239013671875, "step": 198 }, { "KL/chosen_KL_mean": -97.43031311035156, "KL/mean": -124.94877624511719, "KL/rejected_KL_mean": -152.46722412109375, "KL/std": 83.84455871582031, "epoch": 0.4167539267015707, "fcm_dpo/beta": 0.0055153826251626015, "fcm_dpo/delta": 0.018592506647109985, "fcm_dpo/margin": 55.036903381347656, "fcm_dpo/q_t": 0.4302946627140045, "grad_norm": 96.4948501586914, "learning_rate": 3.6374223993904124e-07, "logits/chosen": -0.8654062747955322, "logits/rejected": -0.8202941417694092, "logps/chosen": -362.03619384765625, "logps/ref_chosen": -264.6058654785156, "logps/ref_rejected": -214.9014892578125, "logps/rejected": -367.36871337890625, "loss": 4.6993, "margin_dpo/margin_mean": 55.03690719604492, "margin_dpo/margin_std": 95.58619689941406, "step": 199 }, { "KL/chosen_KL_mean": -107.61213684082031, "KL/mean": -131.58660888671875, "KL/rejected_KL_mean": -155.56106567382812, "KL/std": 85.42179870605469, "epoch": 0.418848167539267, "fcm_dpo/beta": 0.005476394202560186, "fcm_dpo/delta": -0.020585589110851288, "fcm_dpo/margin": 47.948944091796875, "fcm_dpo/q_t": 0.4395952820777893, "grad_norm": 106.97425842285156, "learning_rate": 3.621088951385353e-07, "logits/chosen": -0.9183908700942993, "logits/rejected": -0.8932676315307617, "logps/chosen": -431.7709655761719, "logps/ref_chosen": -324.1588134765625, "logps/ref_rejected": -277.80218505859375, "logps/rejected": -433.36328125, "loss": 4.9173, "margin_dpo/margin_mean": 47.948944091796875, "margin_dpo/margin_std": 107.63665008544922, "step": 200 }, { "epoch": 0.418848167539267, "eval_KL/chosen_KL_mean": -118.4775161743164, "eval_KL/mean": -146.10684204101562, "eval_KL/rejected_KL_mean": -173.73617553710938, "eval_KL/std": 87.68894958496094, "eval_fcm_dpo/beta": 0.005431666970252991, "eval_logits/chosen": -0.9033797979354858, "eval_logits/rejected": -0.8850231170654297, "eval_logps/chosen": -406.3043212890625, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -440.666259765625, "eval_loss": 0.5937886834144592, "eval_margin_dpo/margin_mean": 55.258663177490234, "eval_margin_dpo/margin_std": 96.29785919189453, "eval_runtime": 78.3994, "eval_samples_per_second": 25.51, "eval_steps_per_second": 3.189, "step": 200 }, { "KL/chosen_KL_mean": -114.04182434082031, "KL/mean": -142.33677673339844, "KL/rejected_KL_mean": -170.63168334960938, "KL/std": 90.65510559082031, "epoch": 0.42094240837696334, "fcm_dpo/beta": 0.005450923927128315, "fcm_dpo/delta": -0.003941504284739494, "fcm_dpo/margin": 56.58985900878906, "fcm_dpo/q_t": 0.4291457533836365, "grad_norm": 69.15496826171875, "learning_rate": 3.604695382782159e-07, "logits/chosen": -0.8902816772460938, "logits/rejected": -0.8751832246780396, "logps/chosen": -385.53753662109375, "logps/ref_chosen": -271.49566650390625, "logps/ref_rejected": -245.71414184570312, "logps/rejected": -416.3458251953125, "loss": 4.7244, "margin_dpo/margin_mean": 56.58985900878906, "margin_dpo/margin_std": 102.68513488769531, "step": 201 }, { "KL/chosen_KL_mean": -119.89775848388672, "KL/mean": -146.6784210205078, "KL/rejected_KL_mean": -173.45907592773438, "KL/std": 88.81134796142578, "epoch": 0.42303664921465967, "fcm_dpo/beta": 0.005403751973062754, "fcm_dpo/delta": 9.850505739450455e-05, "fcm_dpo/margin": 53.561317443847656, "fcm_dpo/q_t": 0.43379461765289307, "grad_norm": 89.58255767822266, "learning_rate": 3.588242572718162e-07, "logits/chosen": -0.9021543264389038, "logits/rejected": -0.8883557915687561, "logps/chosen": -391.9956970214844, "logps/ref_chosen": -272.0979309082031, "logps/ref_rejected": -235.94805908203125, "logps/rejected": -409.4071350097656, "loss": 4.7836, "margin_dpo/margin_mean": 53.561317443847656, "margin_dpo/margin_std": 102.62480163574219, "step": 202 }, { "KL/chosen_KL_mean": -130.4102783203125, "KL/mean": -151.12998962402344, "KL/rejected_KL_mean": -171.84970092773438, "KL/std": 87.56797790527344, "epoch": 0.42513089005235605, "fcm_dpo/beta": 0.00546378456056118, "fcm_dpo/delta": 0.014473012648522854, "fcm_dpo/margin": 41.439414978027344, "fcm_dpo/q_t": 0.44730111956596375, "grad_norm": 102.8392562866211, "learning_rate": 3.571731403507635e-07, "logits/chosen": -0.8795283436775208, "logits/rejected": -0.8848550319671631, "logps/chosen": -410.6324462890625, "logps/ref_chosen": -280.2221374511719, "logps/ref_rejected": -251.79798889160156, "logps/rejected": -423.6476745605469, "loss": 4.9634, "margin_dpo/margin_mean": 41.439414978027344, "margin_dpo/margin_std": 95.13355255126953, "step": 203 }, { "KL/chosen_KL_mean": -132.2213592529297, "KL/mean": -160.99591064453125, "KL/rejected_KL_mean": -189.77044677734375, "KL/std": 87.84689331054688, "epoch": 0.4272251308900524, "fcm_dpo/beta": 0.005502759013324976, "fcm_dpo/delta": 0.007985341362655163, "fcm_dpo/margin": 57.549095153808594, "fcm_dpo/q_t": 0.42689085006713867, "grad_norm": 125.6895751953125, "learning_rate": 3.5551627605944746e-07, "logits/chosen": -0.9232524633407593, "logits/rejected": -0.8936776518821716, "logps/chosen": -451.0173645019531, "logps/ref_chosen": -318.7960510253906, "logps/ref_rejected": -269.69921875, "logps/rejected": -459.4697265625, "loss": 4.671, "margin_dpo/margin_mean": 57.549102783203125, "margin_dpo/margin_std": 98.45879364013672, "step": 204 }, { "KL/chosen_KL_mean": -136.3469696044922, "KL/mean": -168.44393920898438, "KL/rejected_KL_mean": -200.54092407226562, "KL/std": 92.91146850585938, "epoch": 0.4293193717277487, "fcm_dpo/beta": 0.005452723708003759, "fcm_dpo/delta": -0.045753512531518936, "fcm_dpo/margin": 64.19393920898438, "fcm_dpo/q_t": 0.421352356672287, "grad_norm": 128.93753051757812, "learning_rate": 3.5385375325047163e-07, "logits/chosen": -0.8728401064872742, "logits/rejected": -0.841010570526123, "logps/chosen": -420.10906982421875, "logps/ref_chosen": -283.7620544433594, "logps/ref_rejected": -297.69439697265625, "logps/rejected": -498.2353210449219, "loss": 4.6025, "margin_dpo/margin_mean": 64.19393920898438, "margin_dpo/margin_std": 103.74689483642578, "step": 205 }, { "KL/chosen_KL_mean": -142.62896728515625, "KL/mean": -167.98255920410156, "KL/rejected_KL_mean": -193.33612060546875, "KL/std": 92.03362274169922, "epoch": 0.431413612565445, "fcm_dpo/beta": 0.005307645071297884, "fcm_dpo/delta": -0.0065862322226166725, "fcm_dpo/margin": 50.70718002319336, "fcm_dpo/q_t": 0.43794897198677063, "grad_norm": 107.9933090209961, "learning_rate": 3.5218566107988867e-07, "logits/chosen": -0.895841121673584, "logits/rejected": -0.9164851903915405, "logps/chosen": -436.2928466796875, "logps/ref_chosen": -293.66387939453125, "logps/ref_rejected": -291.3056640625, "logps/rejected": -484.6418151855469, "loss": 4.8515, "margin_dpo/margin_mean": 50.70718002319336, "margin_dpo/margin_std": 103.65234375, "step": 206 }, { "KL/chosen_KL_mean": -129.6578826904297, "KL/mean": -155.3979034423828, "KL/rejected_KL_mean": -181.13790893554688, "KL/std": 103.08729553222656, "epoch": 0.43350785340314135, "fcm_dpo/beta": 0.005296694580465555, "fcm_dpo/delta": 0.01233250554651022, "fcm_dpo/margin": 51.480037689208984, "fcm_dpo/q_t": 0.43691202998161316, "grad_norm": 71.85640716552734, "learning_rate": 3.505120890024195e-07, "logits/chosen": -0.8434122800827026, "logits/rejected": -0.8525888323783875, "logps/chosen": -400.1929931640625, "logps/ref_chosen": -270.5350646972656, "logps/ref_rejected": -278.7747497558594, "logps/rejected": -459.91265869140625, "loss": 4.9192, "margin_dpo/margin_mean": 51.480037689208984, "margin_dpo/margin_std": 117.58082580566406, "step": 207 }, { "KL/chosen_KL_mean": -137.4970703125, "KL/mean": -168.57623291015625, "KL/rejected_KL_mean": -199.65541076660156, "KL/std": 101.80587005615234, "epoch": 0.4356020942408377, "fcm_dpo/beta": 0.0053030322305858135, "fcm_dpo/delta": -0.015698019415140152, "fcm_dpo/margin": 62.15835952758789, "fcm_dpo/q_t": 0.42619529366493225, "grad_norm": 76.81256103515625, "learning_rate": 3.4883312676665534e-07, "logits/chosen": -0.9078364968299866, "logits/rejected": -0.8600209355354309, "logps/chosen": -417.0798034667969, "logps/ref_chosen": -279.582763671875, "logps/ref_rejected": -290.041015625, "logps/rejected": -489.6964111328125, "loss": 4.6342, "margin_dpo/margin_mean": 62.15835952758789, "margin_dpo/margin_std": 105.14048767089844, "step": 208 }, { "KL/chosen_KL_mean": -139.73585510253906, "KL/mean": -159.71441650390625, "KL/rejected_KL_mean": -179.6929931640625, "KL/std": 94.56495666503906, "epoch": 0.437696335078534, "fcm_dpo/beta": 0.005294729024171829, "fcm_dpo/delta": 0.011330154724419117, "fcm_dpo/margin": 39.9571533203125, "fcm_dpo/q_t": 0.4502761662006378, "grad_norm": 107.32901763916016, "learning_rate": 3.4714886441024573e-07, "logits/chosen": -0.8233649730682373, "logits/rejected": -0.8163270950317383, "logps/chosen": -458.6083679199219, "logps/ref_chosen": -318.8725280761719, "logps/ref_rejected": -270.64324951171875, "logps/rejected": -450.33624267578125, "loss": 5.0317, "margin_dpo/margin_mean": 39.9571533203125, "margin_dpo/margin_std": 99.94036102294922, "step": 209 }, { "KL/chosen_KL_mean": -127.69154357910156, "KL/mean": -158.0561981201172, "KL/rejected_KL_mean": -188.4208526611328, "KL/std": 90.50584411621094, "epoch": 0.4397905759162304, "fcm_dpo/beta": 0.0052696773782372475, "fcm_dpo/delta": -0.023277752101421356, "fcm_dpo/margin": 60.729339599609375, "fcm_dpo/q_t": 0.4259607493877411, "grad_norm": 83.70458984375, "learning_rate": 3.454593922550693e-07, "logits/chosen": -0.8556398153305054, "logits/rejected": -0.8458552360534668, "logps/chosen": -410.83184814453125, "logps/ref_chosen": -283.14031982421875, "logps/ref_rejected": -287.2986755371094, "logps/rejected": -475.71954345703125, "loss": 4.6405, "margin_dpo/margin_mean": 60.729331970214844, "margin_dpo/margin_std": 97.90538024902344, "step": 210 }, { "KL/chosen_KL_mean": -111.53688049316406, "KL/mean": -149.20333862304688, "KL/rejected_KL_mean": -186.8698272705078, "KL/std": 91.30079650878906, "epoch": 0.4418848167539267, "fcm_dpo/beta": 0.005068170838057995, "fcm_dpo/delta": -0.047747448086738586, "fcm_dpo/margin": 75.33293151855469, "fcm_dpo/q_t": 0.4104166328907013, "grad_norm": 83.38289642333984, "learning_rate": 3.4376480090239047e-07, "logits/chosen": -0.8847814798355103, "logits/rejected": -0.8608183860778809, "logps/chosen": -387.9597473144531, "logps/ref_chosen": -276.4228515625, "logps/ref_rejected": -252.40603637695312, "logps/rejected": -439.2758483886719, "loss": 4.3908, "margin_dpo/margin_mean": 75.33293151855469, "margin_dpo/margin_std": 90.77347564697266, "step": 211 }, { "KL/chosen_KL_mean": -120.57503509521484, "KL/mean": -145.82603454589844, "KL/rejected_KL_mean": -171.07704162597656, "KL/std": 89.40132141113281, "epoch": 0.44397905759162304, "fcm_dpo/beta": 0.005025006365031004, "fcm_dpo/delta": 0.008119989186525345, "fcm_dpo/margin": 50.502010345458984, "fcm_dpo/q_t": 0.4400903582572937, "grad_norm": 74.55123138427734, "learning_rate": 3.4206518122800055e-07, "logits/chosen": -0.8634406328201294, "logits/rejected": -0.8582726716995239, "logps/chosen": -392.28057861328125, "logps/ref_chosen": -271.7055358886719, "logps/ref_rejected": -241.18511962890625, "logps/rejected": -412.26220703125, "loss": 4.8539, "margin_dpo/margin_mean": 50.50200653076172, "margin_dpo/margin_std": 98.8302001953125, "step": 212 }, { "KL/chosen_KL_mean": -118.58316802978516, "KL/mean": -146.1175537109375, "KL/rejected_KL_mean": -173.6519317626953, "KL/std": 103.11312103271484, "epoch": 0.44607329842931936, "fcm_dpo/beta": 0.0050498321652412415, "fcm_dpo/delta": -0.005192122422158718, "fcm_dpo/margin": 55.06875991821289, "fcm_dpo/q_t": 0.43775975704193115, "grad_norm": 82.58657836914062, "learning_rate": 3.403606243773448e-07, "logits/chosen": -0.8551130294799805, "logits/rejected": -0.8685761094093323, "logps/chosen": -420.88079833984375, "logps/ref_chosen": -302.2976379394531, "logps/ref_rejected": -303.6202087402344, "logps/rejected": -477.2721252441406, "loss": 4.8435, "margin_dpo/margin_mean": 55.06875991821289, "margin_dpo/margin_std": 114.744384765625, "step": 213 }, { "KL/chosen_KL_mean": -125.1562728881836, "KL/mean": -158.9164581298828, "KL/rejected_KL_mean": -192.6766357421875, "KL/std": 91.4408950805664, "epoch": 0.4481675392670157, "fcm_dpo/beta": 0.004997848533093929, "fcm_dpo/delta": -0.02767416462302208, "fcm_dpo/margin": 67.52037811279297, "fcm_dpo/q_t": 0.42145538330078125, "grad_norm": 90.98759460449219, "learning_rate": 3.3865122176063385e-07, "logits/chosen": -0.8687431216239929, "logits/rejected": -0.8687351942062378, "logps/chosen": -397.2889099121094, "logps/ref_chosen": -272.13262939453125, "logps/ref_rejected": -294.82354736328125, "logps/rejected": -487.50018310546875, "loss": 4.535, "margin_dpo/margin_mean": 67.5203857421875, "margin_dpo/margin_std": 91.6003189086914, "step": 214 }, { "KL/chosen_KL_mean": -128.5072479248047, "KL/mean": -150.73866271972656, "KL/rejected_KL_mean": -172.97007751464844, "KL/std": 100.23798370361328, "epoch": 0.450261780104712, "fcm_dpo/beta": 0.004917190410196781, "fcm_dpo/delta": -0.015194178558886051, "fcm_dpo/margin": 44.46282958984375, "fcm_dpo/q_t": 0.45000001788139343, "grad_norm": 86.00048828125, "learning_rate": 3.3693706504794243e-07, "logits/chosen": -0.9164157509803772, "logits/rejected": -0.8946561813354492, "logps/chosen": -419.8855285644531, "logps/ref_chosen": -291.3782958984375, "logps/ref_rejected": -261.05792236328125, "logps/rejected": -434.02801513671875, "loss": 4.9817, "margin_dpo/margin_mean": 44.46282958984375, "margin_dpo/margin_std": 101.5790786743164, "step": 215 }, { "KL/chosen_KL_mean": -115.26673889160156, "KL/mean": -144.14666748046875, "KL/rejected_KL_mean": -173.02658081054688, "KL/std": 91.47891998291016, "epoch": 0.4523560209424084, "fcm_dpo/beta": 0.004956618882715702, "fcm_dpo/delta": 0.033862657845020294, "fcm_dpo/margin": 57.759849548339844, "fcm_dpo/q_t": 0.4338792860507965, "grad_norm": 67.37702178955078, "learning_rate": 3.3521824616429284e-07, "logits/chosen": -0.9285804033279419, "logits/rejected": -0.9208376407623291, "logps/chosen": -453.7721862792969, "logps/ref_chosen": -338.50543212890625, "logps/ref_rejected": -305.76104736328125, "logps/rejected": -478.7876281738281, "loss": 4.7734, "margin_dpo/margin_mean": 57.759849548339844, "margin_dpo/margin_std": 107.90360260009766, "step": 216 }, { "KL/chosen_KL_mean": -118.00670623779297, "KL/mean": -152.04202270507812, "KL/rejected_KL_mean": -186.07737731933594, "KL/std": 93.6181869506836, "epoch": 0.4544502617801047, "fcm_dpo/beta": 0.00493368785828352, "fcm_dpo/delta": -0.02944992296397686, "fcm_dpo/margin": 68.07068634033203, "fcm_dpo/q_t": 0.4224163889884949, "grad_norm": 73.08317565917969, "learning_rate": 3.334948572847253e-07, "logits/chosen": -0.8377935886383057, "logits/rejected": -0.7964776754379272, "logps/chosen": -411.5565185546875, "logps/ref_chosen": -293.5498046875, "logps/ref_rejected": -256.7830810546875, "logps/rejected": -442.8605041503906, "loss": 4.5865, "margin_dpo/margin_mean": 68.07068634033203, "margin_dpo/margin_std": 105.03534698486328, "step": 217 }, { "KL/chosen_KL_mean": -114.50249481201172, "KL/mean": -144.8571319580078, "KL/rejected_KL_mean": -175.2117462158203, "KL/std": 92.68726348876953, "epoch": 0.45654450261780105, "fcm_dpo/beta": 0.0048797354102134705, "fcm_dpo/delta": -0.008044907823204994, "fcm_dpo/margin": 60.709266662597656, "fcm_dpo/q_t": 0.4313804805278778, "grad_norm": 64.33407592773438, "learning_rate": 3.317669908293554e-07, "logits/chosen": -0.8685387372970581, "logits/rejected": -0.8849347829818726, "logps/chosen": -435.08184814453125, "logps/ref_chosen": -320.579345703125, "logps/ref_rejected": -294.0381164550781, "logps/rejected": -469.2499084472656, "loss": 4.6864, "margin_dpo/margin_mean": 60.70926284790039, "margin_dpo/margin_std": 99.89984130859375, "step": 218 }, { "KL/chosen_KL_mean": -108.34378051757812, "KL/mean": -139.96334838867188, "KL/rejected_KL_mean": -171.58291625976562, "KL/std": 83.42886352539062, "epoch": 0.4586387434554974, "fcm_dpo/beta": 0.004817603155970573, "fcm_dpo/delta": -0.010519558563828468, "fcm_dpo/margin": 63.2391471862793, "fcm_dpo/q_t": 0.4293725788593292, "grad_norm": 86.39237976074219, "learning_rate": 3.300347394584172e-07, "logits/chosen": -0.847774863243103, "logits/rejected": -0.8739175796508789, "logps/chosen": -376.76239013671875, "logps/ref_chosen": -268.4186096191406, "logps/ref_rejected": -265.7808837890625, "logps/rejected": -437.36376953125, "loss": 4.6705, "margin_dpo/margin_mean": 63.2391471862793, "margin_dpo/margin_std": 100.10897827148438, "step": 219 }, { "KL/chosen_KL_mean": -109.47624206542969, "KL/mean": -141.2556610107422, "KL/rejected_KL_mean": -173.03509521484375, "KL/std": 82.5869369506836, "epoch": 0.4607329842931937, "fcm_dpo/beta": 0.004888718016445637, "fcm_dpo/delta": 0.03557638078927994, "fcm_dpo/margin": 63.5588493347168, "fcm_dpo/q_t": 0.4272876977920532, "grad_norm": 79.07698822021484, "learning_rate": 3.2829819606729477e-07, "logits/chosen": -0.9154024124145508, "logits/rejected": -0.8836403489112854, "logps/chosen": -422.3627014160156, "logps/ref_chosen": -312.8864440917969, "logps/ref_rejected": -259.5191955566406, "logps/rejected": -432.5542907714844, "loss": 4.5985, "margin_dpo/margin_mean": 63.55884552001953, "margin_dpo/margin_std": 90.30359649658203, "step": 220 }, { "KL/chosen_KL_mean": -113.12982940673828, "KL/mean": -138.3195037841797, "KL/rejected_KL_mean": -163.50918579101562, "KL/std": 86.43636322021484, "epoch": 0.46282722513089003, "fcm_dpo/beta": 0.004951215349137783, "fcm_dpo/delta": -0.004505933728069067, "fcm_dpo/margin": 50.37934112548828, "fcm_dpo/q_t": 0.4432408809661865, "grad_norm": 70.59790802001953, "learning_rate": 3.265574537815398e-07, "logits/chosen": -0.8173964023590088, "logits/rejected": -0.8285130262374878, "logps/chosen": -413.4556579589844, "logps/ref_chosen": -300.32586669921875, "logps/ref_rejected": -286.312255859375, "logps/rejected": -449.8214111328125, "loss": 4.8711, "margin_dpo/margin_mean": 50.37934112548828, "margin_dpo/margin_std": 101.99186706542969, "step": 221 }, { "KL/chosen_KL_mean": -106.68103790283203, "KL/mean": -134.0705108642578, "KL/rejected_KL_mean": -161.45997619628906, "KL/std": 78.9898681640625, "epoch": 0.4649214659685864, "fcm_dpo/beta": 0.004972555674612522, "fcm_dpo/delta": 0.014529617503285408, "fcm_dpo/margin": 54.77894592285156, "fcm_dpo/q_t": 0.43578585982322693, "grad_norm": 83.3537826538086, "learning_rate": 3.248126059518784e-07, "logits/chosen": -0.9040961265563965, "logits/rejected": -0.8702518939971924, "logps/chosen": -403.79241943359375, "logps/ref_chosen": -297.1113586425781, "logps/ref_rejected": -235.53146362304688, "logps/rejected": -396.991455078125, "loss": 4.7331, "margin_dpo/margin_mean": 54.7789421081543, "margin_dpo/margin_std": 88.59034729003906, "step": 222 }, { "KL/chosen_KL_mean": -112.10569763183594, "KL/mean": -143.2996826171875, "KL/rejected_KL_mean": -174.49366760253906, "KL/std": 86.74018096923828, "epoch": 0.46701570680628274, "fcm_dpo/beta": 0.005028898827731609, "fcm_dpo/delta": 0.010601907968521118, "fcm_dpo/margin": 62.38795852661133, "fcm_dpo/q_t": 0.4269620180130005, "grad_norm": 95.17221069335938, "learning_rate": 3.230637461492043e-07, "logits/chosen": -0.8466207385063171, "logits/rejected": -0.8117715716362, "logps/chosen": -398.52081298828125, "logps/ref_chosen": -286.41510009765625, "logps/ref_rejected": -241.1181640625, "logps/rejected": -415.61181640625, "loss": 4.6314, "margin_dpo/margin_mean": 62.38795852661133, "margin_dpo/margin_std": 97.02510070800781, "step": 223 }, { "KL/chosen_KL_mean": -103.6304702758789, "KL/mean": -138.8751220703125, "KL/rejected_KL_mean": -174.11976623535156, "KL/std": 85.77777099609375, "epoch": 0.46910994764397906, "fcm_dpo/beta": 0.005118512082844973, "fcm_dpo/delta": 0.005692493170499802, "fcm_dpo/margin": 70.48930358886719, "fcm_dpo/q_t": 0.4164232611656189, "grad_norm": 69.0962905883789, "learning_rate": 3.213109681595612e-07, "logits/chosen": -0.8163408637046814, "logits/rejected": -0.8316698670387268, "logps/chosen": -353.1227722167969, "logps/ref_chosen": -249.49234008789062, "logps/ref_rejected": -233.10752868652344, "logps/rejected": -407.2273254394531, "loss": 4.4561, "margin_dpo/margin_mean": 70.48930358886719, "margin_dpo/margin_std": 91.14543151855469, "step": 224 }, { "KL/chosen_KL_mean": -123.72078704833984, "KL/mean": -154.9224395751953, "KL/rejected_KL_mean": -186.1240997314453, "KL/std": 93.72308349609375, "epoch": 0.4712041884816754, "fcm_dpo/beta": 0.005051084794104099, "fcm_dpo/delta": 0.011575451120734215, "fcm_dpo/margin": 62.4033088684082, "fcm_dpo/q_t": 0.42811909317970276, "grad_norm": 72.46212005615234, "learning_rate": 3.1955436597911315e-07, "logits/chosen": -0.8486655354499817, "logits/rejected": -0.821964681148529, "logps/chosen": -435.57916259765625, "logps/ref_chosen": -311.8583679199219, "logps/ref_rejected": -336.8523864746094, "logps/rejected": -522.9765014648438, "loss": 4.6715, "margin_dpo/margin_mean": 62.40331268310547, "margin_dpo/margin_std": 105.1444091796875, "step": 225 }, { "KL/chosen_KL_mean": -112.75259399414062, "KL/mean": -141.30508422851562, "KL/rejected_KL_mean": -169.85757446289062, "KL/std": 89.33605194091797, "epoch": 0.4732984293193717, "fcm_dpo/beta": 0.005086014047265053, "fcm_dpo/delta": -0.0021250159479677677, "fcm_dpo/margin": 57.104957580566406, "fcm_dpo/q_t": 0.4333423972129822, "grad_norm": 56.582576751708984, "learning_rate": 3.1779403380910425e-07, "logits/chosen": -0.8840992450714111, "logits/rejected": -0.8711143136024475, "logps/chosen": -364.9538269042969, "logps/ref_chosen": -252.20123291015625, "logps/ref_rejected": -254.41162109375, "logps/rejected": -424.2691650390625, "loss": 4.7484, "margin_dpo/margin_mean": 57.104957580566406, "margin_dpo/margin_std": 105.84939575195312, "step": 226 }, { "KL/chosen_KL_mean": -117.15921020507812, "KL/mean": -151.5059814453125, "KL/rejected_KL_mean": -185.85281372070312, "KL/std": 86.63410949707031, "epoch": 0.47539267015706804, "fcm_dpo/beta": 0.0050234198570251465, "fcm_dpo/delta": -0.021003495901823044, "fcm_dpo/margin": 68.6935806274414, "fcm_dpo/q_t": 0.41943684220314026, "grad_norm": 72.25760650634766, "learning_rate": 3.160300660508064e-07, "logits/chosen": -0.8448896408081055, "logits/rejected": -0.8412713408470154, "logps/chosen": -402.4186706542969, "logps/ref_chosen": -285.25946044921875, "logps/ref_rejected": -261.3220520019531, "logps/rejected": -447.1748352050781, "loss": 4.5729, "margin_dpo/margin_mean": 68.6935806274414, "margin_dpo/margin_std": 106.60133361816406, "step": 227 }, { "KL/chosen_KL_mean": -108.93597412109375, "KL/mean": -143.87466430664062, "KL/rejected_KL_mean": -178.81336975097656, "KL/std": 85.76429748535156, "epoch": 0.4774869109947644, "fcm_dpo/beta": 0.005010589491575956, "fcm_dpo/delta": -0.023567285388708115, "fcm_dpo/margin": 69.87741088867188, "fcm_dpo/q_t": 0.4189654588699341, "grad_norm": 60.705631256103516, "learning_rate": 3.1426255730045695e-07, "logits/chosen": -0.8538888692855835, "logits/rejected": -0.8198533654212952, "logps/chosen": -422.7547607421875, "logps/ref_chosen": -313.81878662109375, "logps/ref_rejected": -258.07061767578125, "logps/rejected": -436.8840026855469, "loss": 4.511, "margin_dpo/margin_mean": 69.87741088867188, "margin_dpo/margin_std": 96.2317886352539, "step": 228 }, { "KL/chosen_KL_mean": -128.9109344482422, "KL/mean": -168.88983154296875, "KL/rejected_KL_mean": -208.8687286376953, "KL/std": 94.05682373046875, "epoch": 0.47958115183246075, "fcm_dpo/beta": 0.00481749139726162, "fcm_dpo/delta": -0.03823835402727127, "fcm_dpo/margin": 79.95777130126953, "fcm_dpo/q_t": 0.4103056490421295, "grad_norm": 84.75572967529297, "learning_rate": 3.1249160234418644e-07, "logits/chosen": -0.8427700400352478, "logits/rejected": -0.8486419916152954, "logps/chosen": -420.8816833496094, "logps/ref_chosen": -291.9707946777344, "logps/ref_rejected": -263.42059326171875, "logps/rejected": -472.2893371582031, "loss": 4.4112, "margin_dpo/margin_mean": 79.95777893066406, "margin_dpo/margin_std": 104.22063446044922, "step": 229 }, { "KL/chosen_KL_mean": -126.17520141601562, "KL/mean": -160.9757843017578, "KL/rejected_KL_mean": -195.77639770507812, "KL/std": 99.210205078125, "epoch": 0.4816753926701571, "fcm_dpo/beta": 0.004781956318765879, "fcm_dpo/delta": 0.01782013103365898, "fcm_dpo/margin": 69.6011734008789, "fcm_dpo/q_t": 0.42428159713745117, "grad_norm": 75.2130355834961, "learning_rate": 3.1071729615293424e-07, "logits/chosen": -0.8815920948982239, "logits/rejected": -0.8813921213150024, "logps/chosen": -359.435302734375, "logps/ref_chosen": -233.2601318359375, "logps/ref_rejected": -238.922119140625, "logps/rejected": -434.698486328125, "loss": 4.6226, "margin_dpo/margin_mean": 69.6011734008789, "margin_dpo/margin_std": 115.32308197021484, "step": 230 }, { "KL/chosen_KL_mean": -137.8847198486328, "KL/mean": -166.61566162109375, "KL/rejected_KL_mean": -195.3466033935547, "KL/std": 88.32843780517578, "epoch": 0.4837696335078534, "fcm_dpo/beta": 0.004834107588976622, "fcm_dpo/delta": 0.005063319578766823, "fcm_dpo/margin": 57.46186828613281, "fcm_dpo/q_t": 0.4348789155483246, "grad_norm": 71.28133392333984, "learning_rate": 3.0893973387735683e-07, "logits/chosen": -0.8586534261703491, "logits/rejected": -0.8459566235542297, "logps/chosen": -460.0399169921875, "logps/ref_chosen": -322.1551818847656, "logps/ref_rejected": -280.97613525390625, "logps/rejected": -476.3227844238281, "loss": 4.7507, "margin_dpo/margin_mean": 57.461883544921875, "margin_dpo/margin_std": 99.15318298339844, "step": 231 }, { "KL/chosen_KL_mean": -136.840576171875, "KL/mean": -166.6940155029297, "KL/rejected_KL_mean": -196.54745483398438, "KL/std": 97.31487274169922, "epoch": 0.48586387434554973, "fcm_dpo/beta": 0.004791594110429287, "fcm_dpo/delta": -0.025890890508890152, "fcm_dpo/margin": 59.70685577392578, "fcm_dpo/q_t": 0.43394574522972107, "grad_norm": 77.55052947998047, "learning_rate": 3.071590108427243e-07, "logits/chosen": -0.833568811416626, "logits/rejected": -0.8151888251304626, "logps/chosen": -408.5843505859375, "logps/ref_chosen": -271.7437744140625, "logps/ref_rejected": -249.94981384277344, "logps/rejected": -446.4972839355469, "loss": 4.7829, "margin_dpo/margin_mean": 59.70685958862305, "margin_dpo/margin_std": 108.40725708007812, "step": 232 }, { "KL/chosen_KL_mean": -142.8009033203125, "KL/mean": -177.56996154785156, "KL/rejected_KL_mean": -212.33900451660156, "KL/std": 95.32579803466797, "epoch": 0.48795811518324606, "fcm_dpo/beta": 0.004678625613451004, "fcm_dpo/delta": -0.037981029599905014, "fcm_dpo/margin": 69.5381088256836, "fcm_dpo/q_t": 0.42466118931770325, "grad_norm": 63.76099395751953, "learning_rate": 3.05375222543809e-07, "logits/chosen": -0.8736680746078491, "logits/rejected": -0.8643991947174072, "logps/chosen": -428.14324951171875, "logps/ref_chosen": -285.3423156738281, "logps/ref_rejected": -266.34320068359375, "logps/rejected": -478.68218994140625, "loss": 4.6045, "margin_dpo/margin_mean": 69.5381088256836, "margin_dpo/margin_std": 104.16007232666016, "step": 233 }, { "KL/chosen_KL_mean": -138.01681518554688, "KL/mean": -173.01083374023438, "KL/rejected_KL_mean": -208.0048370361328, "KL/std": 98.38790130615234, "epoch": 0.4900523560209424, "fcm_dpo/beta": 0.004634845070540905, "fcm_dpo/delta": 0.012818412855267525, "fcm_dpo/margin": 69.98805236816406, "fcm_dpo/q_t": 0.42441120743751526, "grad_norm": 64.4365005493164, "learning_rate": 3.035884646397637e-07, "logits/chosen": -0.8443531394004822, "logits/rejected": -0.8304785490036011, "logps/chosen": -432.92254638671875, "logps/ref_chosen": -294.9057312011719, "logps/ref_rejected": -299.37054443359375, "logps/rejected": -507.37542724609375, "loss": 4.6463, "margin_dpo/margin_mean": 69.98805236816406, "margin_dpo/margin_std": 114.38517761230469, "step": 234 }, { "KL/chosen_KL_mean": -149.86141967773438, "KL/mean": -180.7827606201172, "KL/rejected_KL_mean": -211.7041015625, "KL/std": 103.66209411621094, "epoch": 0.49214659685863876, "fcm_dpo/beta": 0.0047287228517234325, "fcm_dpo/delta": 0.039121147245168686, "fcm_dpo/margin": 61.84267044067383, "fcm_dpo/q_t": 0.4319824278354645, "grad_norm": 76.6629638671875, "learning_rate": 3.017988329489923e-07, "logits/chosen": -0.863947868347168, "logits/rejected": -0.855870246887207, "logps/chosen": -439.3590087890625, "logps/ref_chosen": -289.49755859375, "logps/ref_rejected": -247.55076599121094, "logps/rejected": -459.2548522949219, "loss": 4.7503, "margin_dpo/margin_mean": 61.84267044067383, "margin_dpo/margin_std": 115.53366088867188, "step": 235 }, { "KL/chosen_KL_mean": -145.44284057617188, "KL/mean": -175.6923828125, "KL/rejected_KL_mean": -205.94192504882812, "KL/std": 100.11052703857422, "epoch": 0.4942408376963351, "fcm_dpo/beta": 0.004794456996023655, "fcm_dpo/delta": -0.02471328340470791, "fcm_dpo/margin": 60.49909591674805, "fcm_dpo/q_t": 0.43273258209228516, "grad_norm": 72.24032592773438, "learning_rate": 3.000064234440111e-07, "logits/chosen": -0.8775507807731628, "logits/rejected": -0.8777625560760498, "logps/chosen": -434.3274841308594, "logps/ref_chosen": -288.8846435546875, "logps/ref_rejected": -242.0452880859375, "logps/rejected": -447.9872131347656, "loss": 4.7912, "margin_dpo/margin_mean": 60.49909210205078, "margin_dpo/margin_std": 114.68053436279297, "step": 236 }, { "KL/chosen_KL_mean": -158.35903930664062, "KL/mean": -189.57205200195312, "KL/rejected_KL_mean": -220.7850341796875, "KL/std": 89.05985260009766, "epoch": 0.4963350785340314, "fcm_dpo/beta": 0.004720585886389017, "fcm_dpo/delta": 0.015051835216581821, "fcm_dpo/margin": 62.426025390625, "fcm_dpo/q_t": 0.4318310618400574, "grad_norm": 90.479248046875, "learning_rate": 2.9821133224630223e-07, "logits/chosen": -0.8398151397705078, "logits/rejected": -0.8189886212348938, "logps/chosen": -423.8377685546875, "logps/ref_chosen": -265.47869873046875, "logps/ref_rejected": -267.9891357421875, "logps/rejected": -488.774169921875, "loss": 4.7226, "margin_dpo/margin_mean": 62.426025390625, "margin_dpo/margin_std": 109.10443115234375, "step": 237 }, { "KL/chosen_KL_mean": -163.52406311035156, "KL/mean": -198.2044677734375, "KL/rejected_KL_mean": -232.88482666015625, "KL/std": 100.38282775878906, "epoch": 0.49842931937172774, "fcm_dpo/beta": 0.0047994209453463554, "fcm_dpo/delta": 0.0028489064425230026, "fcm_dpo/margin": 69.36079406738281, "fcm_dpo/q_t": 0.42530369758605957, "grad_norm": 77.99054718017578, "learning_rate": 2.964136556211588e-07, "logits/chosen": -0.8484768867492676, "logits/rejected": -0.8160721659660339, "logps/chosen": -475.5267333984375, "logps/ref_chosen": -312.0026550292969, "logps/ref_rejected": -270.0257263183594, "logps/rejected": -502.9105224609375, "loss": 4.6355, "margin_dpo/margin_mean": 69.36080169677734, "margin_dpo/margin_std": 117.8523941040039, "step": 238 }, { "KL/chosen_KL_mean": -161.76864624023438, "KL/mean": -191.57293701171875, "KL/rejected_KL_mean": -221.37716674804688, "KL/std": 100.97264862060547, "epoch": 0.5005235602094241, "fcm_dpo/beta": 0.004840420559048653, "fcm_dpo/delta": 0.02600950561463833, "fcm_dpo/margin": 59.608524322509766, "fcm_dpo/q_t": 0.4337758719921112, "grad_norm": 65.61921691894531, "learning_rate": 2.946134899725226e-07, "logits/chosen": -0.8408248424530029, "logits/rejected": -0.880803108215332, "logps/chosen": -428.93585205078125, "logps/ref_chosen": -267.167236328125, "logps/ref_rejected": -275.99468994140625, "logps/rejected": -497.37188720703125, "loss": 4.8741, "margin_dpo/margin_mean": 59.608524322509766, "margin_dpo/margin_std": 132.66128540039062, "step": 239 }, { "KL/chosen_KL_mean": -158.13919067382812, "KL/mean": -196.2527313232422, "KL/rejected_KL_mean": -234.36627197265625, "KL/std": 103.12637329101562, "epoch": 0.5026178010471204, "fcm_dpo/beta": 0.0048082321882247925, "fcm_dpo/delta": -0.015010528266429901, "fcm_dpo/margin": 76.22711181640625, "fcm_dpo/q_t": 0.4153556823730469, "grad_norm": 79.14559173583984, "learning_rate": 2.9281093183781403e-07, "logits/chosen": -0.9028453230857849, "logits/rejected": -0.8958457708358765, "logps/chosen": -444.1188049316406, "logps/ref_chosen": -285.9796142578125, "logps/ref_rejected": -256.8258056640625, "logps/rejected": -491.1921081542969, "loss": 4.4889, "margin_dpo/margin_mean": 76.22711181640625, "margin_dpo/margin_std": 109.41165924072266, "step": 240 }, { "KL/chosen_KL_mean": -180.80047607421875, "KL/mean": -206.89230346679688, "KL/rejected_KL_mean": -232.984130859375, "KL/std": 111.80062866210938, "epoch": 0.5047120418848168, "fcm_dpo/beta": 0.004808627534657717, "fcm_dpo/delta": -0.006103941705077887, "fcm_dpo/margin": 52.18364715576172, "fcm_dpo/q_t": 0.4433174133300781, "grad_norm": 88.64216613769531, "learning_rate": 2.910060778827554e-07, "logits/chosen": -0.8175089955329895, "logits/rejected": -0.7948691844940186, "logps/chosen": -442.31732177734375, "logps/ref_chosen": -261.516845703125, "logps/ref_rejected": -250.2250518798828, "logps/rejected": -483.20916748046875, "loss": 4.9968, "margin_dpo/margin_mean": 52.18364715576172, "margin_dpo/margin_std": 127.83100128173828, "step": 241 }, { "KL/chosen_KL_mean": -161.50523376464844, "KL/mean": -195.77635192871094, "KL/rejected_KL_mean": -230.0474395751953, "KL/std": 106.44923400878906, "epoch": 0.506806282722513, "fcm_dpo/beta": 0.004742825403809547, "fcm_dpo/delta": -0.012836568988859653, "fcm_dpo/margin": 68.5422134399414, "fcm_dpo/q_t": 0.42594078183174133, "grad_norm": 88.80677032470703, "learning_rate": 2.891990248961871e-07, "logits/chosen": -0.8814034461975098, "logits/rejected": -0.856747031211853, "logps/chosen": -432.01922607421875, "logps/ref_chosen": -270.51397705078125, "logps/ref_rejected": -244.8560791015625, "logps/rejected": -474.9035339355469, "loss": 4.6268, "margin_dpo/margin_mean": 68.5422134399414, "margin_dpo/margin_std": 111.54056549072266, "step": 242 }, { "KL/chosen_KL_mean": -153.52972412109375, "KL/mean": -195.6244659423828, "KL/rejected_KL_mean": -237.71920776367188, "KL/std": 106.60342407226562, "epoch": 0.5089005235602094, "fcm_dpo/beta": 0.0047475434839725494, "fcm_dpo/delta": -0.034220486879348755, "fcm_dpo/margin": 84.18949127197266, "fcm_dpo/q_t": 0.41172146797180176, "grad_norm": 102.534912109375, "learning_rate": 2.873898697848762e-07, "logits/chosen": -0.8814060091972351, "logits/rejected": -0.8652539849281311, "logps/chosen": -478.21173095703125, "logps/ref_chosen": -324.68206787109375, "logps/ref_rejected": -307.1111755371094, "logps/rejected": -544.8303833007812, "loss": 4.4595, "margin_dpo/margin_mean": 84.18949127197266, "margin_dpo/margin_std": 128.6507110595703, "step": 243 }, { "KL/chosen_KL_mean": -152.66043090820312, "KL/mean": -192.0305633544922, "KL/rejected_KL_mean": -231.4007110595703, "KL/std": 100.23159790039062, "epoch": 0.5109947643979058, "fcm_dpo/beta": 0.004584737122058868, "fcm_dpo/delta": -0.017737431451678276, "fcm_dpo/margin": 78.74027252197266, "fcm_dpo/q_t": 0.4154585003852844, "grad_norm": 77.94464111328125, "learning_rate": 2.8557870956832133e-07, "logits/chosen": -0.863549530506134, "logits/rejected": -0.8185126185417175, "logps/chosen": -471.6396789550781, "logps/ref_chosen": -318.979248046875, "logps/ref_rejected": -269.67572021484375, "logps/rejected": -501.0764465332031, "loss": 4.5167, "margin_dpo/margin_mean": 78.74027252197266, "margin_dpo/margin_std": 112.84671020507812, "step": 244 }, { "KL/chosen_KL_mean": -143.22264099121094, "KL/mean": -182.23419189453125, "KL/rejected_KL_mean": -221.24574279785156, "KL/std": 96.40372467041016, "epoch": 0.5130890052356021, "fcm_dpo/beta": 0.004495944362133741, "fcm_dpo/delta": -0.03692924976348877, "fcm_dpo/margin": 78.02310943603516, "fcm_dpo/q_t": 0.41797515749931335, "grad_norm": 59.87691879272461, "learning_rate": 2.837656413735479e-07, "logits/chosen": -0.8684712648391724, "logits/rejected": -0.8744341731071472, "logps/chosen": -438.1206970214844, "logps/ref_chosen": -294.8980712890625, "logps/ref_rejected": -239.8111114501953, "logps/rejected": -461.0568542480469, "loss": 4.5402, "margin_dpo/margin_mean": 78.02310943603516, "margin_dpo/margin_std": 112.71392822265625, "step": 245 }, { "KL/chosen_KL_mean": -157.49560546875, "KL/mean": -186.96458435058594, "KL/rejected_KL_mean": -216.43356323242188, "KL/std": 101.2094497680664, "epoch": 0.5151832460732985, "fcm_dpo/beta": 0.004423782229423523, "fcm_dpo/delta": -0.0013894562143832445, "fcm_dpo/margin": 58.93793487548828, "fcm_dpo/q_t": 0.44051581621170044, "grad_norm": 90.08110809326172, "learning_rate": 2.8195076242990116e-07, "logits/chosen": -0.8500803709030151, "logits/rejected": -0.8542726039886475, "logps/chosen": -438.1810302734375, "logps/ref_chosen": -280.6854248046875, "logps/ref_rejected": -253.65382385253906, "logps/rejected": -470.0873718261719, "loss": 4.8966, "margin_dpo/margin_mean": 58.93793487548828, "margin_dpo/margin_std": 127.81275177001953, "step": 246 }, { "KL/chosen_KL_mean": -140.9234619140625, "KL/mean": -177.16494750976562, "KL/rejected_KL_mean": -213.40638732910156, "KL/std": 101.20616149902344, "epoch": 0.5172774869109947, "fcm_dpo/beta": 0.0044204313308000565, "fcm_dpo/delta": -0.004756400361657143, "fcm_dpo/margin": 72.48292541503906, "fcm_dpo/q_t": 0.4261864423751831, "grad_norm": 62.47346878051758, "learning_rate": 2.801341700638307e-07, "logits/chosen": -0.8428160548210144, "logits/rejected": -0.8401237726211548, "logps/chosen": -422.0325927734375, "logps/ref_chosen": -281.1091003417969, "logps/ref_rejected": -260.3700866699219, "logps/rejected": -473.7765197753906, "loss": 4.6164, "margin_dpo/margin_mean": 72.48292541503906, "margin_dpo/margin_std": 109.14017486572266, "step": 247 }, { "KL/chosen_KL_mean": -138.6498565673828, "KL/mean": -173.0037384033203, "KL/rejected_KL_mean": -207.35763549804688, "KL/std": 101.77959442138672, "epoch": 0.5193717277486911, "fcm_dpo/beta": 0.004360491875559092, "fcm_dpo/delta": -0.01580795831978321, "fcm_dpo/margin": 68.70777893066406, "fcm_dpo/q_t": 0.4298118054866791, "grad_norm": 68.65995788574219, "learning_rate": 2.7831596169367227e-07, "logits/chosen": -0.813584566116333, "logits/rejected": -0.8213151693344116, "logps/chosen": -408.9682312011719, "logps/ref_chosen": -270.318359375, "logps/ref_rejected": -233.46778869628906, "logps/rejected": -440.825439453125, "loss": 4.6974, "margin_dpo/margin_mean": 68.70777893066406, "margin_dpo/margin_std": 112.36260986328125, "step": 248 }, { "KL/chosen_KL_mean": -151.7535400390625, "KL/mean": -181.5082550048828, "KL/rejected_KL_mean": -211.26295471191406, "KL/std": 104.59439849853516, "epoch": 0.5214659685863874, "fcm_dpo/beta": 0.004381960723549128, "fcm_dpo/delta": 0.020416682586073875, "fcm_dpo/margin": 59.50940704345703, "fcm_dpo/q_t": 0.4388493001461029, "grad_norm": 62.078365325927734, "learning_rate": 2.7649623482442274e-07, "logits/chosen": -0.8454968333244324, "logits/rejected": -0.8196409344673157, "logps/chosen": -427.56243896484375, "logps/ref_chosen": -275.8088684082031, "logps/ref_rejected": -243.45138549804688, "logps/rejected": -454.7143249511719, "loss": 4.8702, "margin_dpo/margin_mean": 59.50940704345703, "margin_dpo/margin_std": 124.977783203125, "step": 249 }, { "KL/chosen_KL_mean": -146.8738250732422, "KL/mean": -185.1446533203125, "KL/rejected_KL_mean": -223.41546630859375, "KL/std": 114.59971618652344, "epoch": 0.5235602094240838, "fcm_dpo/beta": 0.004331628791987896, "fcm_dpo/delta": -0.035680629312992096, "fcm_dpo/margin": 76.54167938232422, "fcm_dpo/q_t": 0.4232044219970703, "grad_norm": 52.845611572265625, "learning_rate": 2.7467508704251135e-07, "logits/chosen": -0.8485902547836304, "logits/rejected": -0.8440597057342529, "logps/chosen": -439.36834716796875, "logps/ref_chosen": -292.4945373535156, "logps/ref_rejected": -284.2869567871094, "logps/rejected": -507.70245361328125, "loss": 4.6384, "margin_dpo/margin_mean": 76.54167938232422, "margin_dpo/margin_std": 125.18535614013672, "step": 250 }, { "KL/chosen_KL_mean": -128.29568481445312, "KL/mean": -163.9156951904297, "KL/rejected_KL_mean": -199.53573608398438, "KL/std": 107.50858306884766, "epoch": 0.5256544502617801, "fcm_dpo/beta": 0.004250232130289078, "fcm_dpo/delta": -0.027963757514953613, "fcm_dpo/margin": 71.24006652832031, "fcm_dpo/q_t": 0.43231555819511414, "grad_norm": 63.01066970825195, "learning_rate": 2.7285261601056697e-07, "logits/chosen": -0.8441932201385498, "logits/rejected": -0.8291101455688477, "logps/chosen": -410.03228759765625, "logps/ref_chosen": -281.736572265625, "logps/ref_rejected": -255.9419708251953, "logps/rejected": -455.47772216796875, "loss": 4.6987, "margin_dpo/margin_mean": 71.24006652832031, "margin_dpo/margin_std": 119.5110092163086, "step": 251 }, { "KL/chosen_KL_mean": -138.88409423828125, "KL/mean": -173.60968017578125, "KL/rejected_KL_mean": -208.3352508544922, "KL/std": 102.1337661743164, "epoch": 0.5277486910994764, "fcm_dpo/beta": 0.004174236208200455, "fcm_dpo/delta": -0.011869529262185097, "fcm_dpo/margin": 69.45118713378906, "fcm_dpo/q_t": 0.43159157037734985, "grad_norm": 62.90778732299805, "learning_rate": 2.7102891946217994e-07, "logits/chosen": -0.8931678533554077, "logits/rejected": -0.8665968179702759, "logps/chosen": -434.85150146484375, "logps/ref_chosen": -295.9674072265625, "logps/ref_rejected": -280.111572265625, "logps/rejected": -488.44683837890625, "loss": 4.7147, "margin_dpo/margin_mean": 69.45118713378906, "margin_dpo/margin_std": 117.09326171875, "step": 252 }, { "KL/chosen_KL_mean": -144.9630584716797, "KL/mean": -177.68711853027344, "KL/rejected_KL_mean": -210.41119384765625, "KL/std": 112.56256103515625, "epoch": 0.5298429319371728, "fcm_dpo/beta": 0.00418475829064846, "fcm_dpo/delta": 0.008287381380796432, "fcm_dpo/margin": 65.44812774658203, "fcm_dpo/q_t": 0.4371293783187866, "grad_norm": 56.794734954833984, "learning_rate": 2.692040951966617e-07, "logits/chosen": -0.8609364628791809, "logits/rejected": -0.8533194661140442, "logps/chosen": -422.0353088378906, "logps/ref_chosen": -277.072265625, "logps/ref_rejected": -247.31643676757812, "logps/rejected": -457.7276306152344, "loss": 4.812, "margin_dpo/margin_mean": 65.44812774658203, "margin_dpo/margin_std": 124.55648040771484, "step": 253 }, { "KL/chosen_KL_mean": -136.63442993164062, "KL/mean": -172.9150390625, "KL/rejected_KL_mean": -209.19570922851562, "KL/std": 108.1693115234375, "epoch": 0.5319371727748691, "fcm_dpo/beta": 0.0041767992079257965, "fcm_dpo/delta": -0.02163619175553322, "fcm_dpo/margin": 72.561279296875, "fcm_dpo/q_t": 0.430123507976532, "grad_norm": 50.65597152709961, "learning_rate": 2.6737824107379947e-07, "logits/chosen": -0.8018378019332886, "logits/rejected": -0.7838313579559326, "logps/chosen": -406.5822448730469, "logps/ref_chosen": -269.9478454589844, "logps/ref_rejected": -249.45005798339844, "logps/rejected": -458.645751953125, "loss": 4.6683, "margin_dpo/margin_mean": 72.56128692626953, "margin_dpo/margin_std": 116.72087097167969, "step": 254 }, { "KL/chosen_KL_mean": -130.73049926757812, "KL/mean": -171.3893280029297, "KL/rejected_KL_mean": -212.04815673828125, "KL/std": 108.50567626953125, "epoch": 0.5340314136125655, "fcm_dpo/beta": 0.004186869598925114, "fcm_dpo/delta": 0.02620495669543743, "fcm_dpo/margin": 81.31766510009766, "fcm_dpo/q_t": 0.42155683040618896, "grad_norm": 64.20274353027344, "learning_rate": 2.655514550086086e-07, "logits/chosen": -0.8257165551185608, "logits/rejected": -0.792091429233551, "logps/chosen": -437.38580322265625, "logps/ref_chosen": -306.6552734375, "logps/ref_rejected": -254.47528076171875, "logps/rejected": -466.5234069824219, "loss": 4.5829, "margin_dpo/margin_mean": 81.31766510009766, "margin_dpo/margin_std": 127.40750122070312, "step": 255 }, { "KL/chosen_KL_mean": -134.72642517089844, "KL/mean": -171.73529052734375, "KL/rejected_KL_mean": -208.74420166015625, "KL/std": 108.54015350341797, "epoch": 0.5361256544502618, "fcm_dpo/beta": 0.004111228045076132, "fcm_dpo/delta": -0.056310322135686874, "fcm_dpo/margin": 74.01776123046875, "fcm_dpo/q_t": 0.4271259903907776, "grad_norm": 98.39281463623047, "learning_rate": 2.6372383496608186e-07, "logits/chosen": -0.85221928358078, "logits/rejected": -0.8382247686386108, "logps/chosen": -458.4446105957031, "logps/ref_chosen": -323.7181701660156, "logps/ref_rejected": -254.1871337890625, "logps/rejected": -462.9313049316406, "loss": 4.7255, "margin_dpo/margin_mean": 74.01776123046875, "margin_dpo/margin_std": 125.59674835205078, "step": 256 }, { "KL/chosen_KL_mean": -137.28506469726562, "KL/mean": -176.9401092529297, "KL/rejected_KL_mean": -216.59512329101562, "KL/std": 103.74283599853516, "epoch": 0.5382198952879581, "fcm_dpo/beta": 0.004046399611979723, "fcm_dpo/delta": 0.012473970651626587, "fcm_dpo/margin": 79.31005859375, "fcm_dpo/q_t": 0.4253496825695038, "grad_norm": 80.60918426513672, "learning_rate": 2.618954789559356e-07, "logits/chosen": -0.8426645994186401, "logits/rejected": -0.8348857164382935, "logps/chosen": -404.4971618652344, "logps/ref_chosen": -267.21209716796875, "logps/ref_rejected": -249.12579345703125, "logps/rejected": -465.72088623046875, "loss": 4.5916, "margin_dpo/margin_mean": 79.31005859375, "margin_dpo/margin_std": 119.5707778930664, "step": 257 }, { "KL/chosen_KL_mean": -145.31399536132812, "KL/mean": -187.23464965820312, "KL/rejected_KL_mean": -229.1552734375, "KL/std": 105.5355224609375, "epoch": 0.5403141361256545, "fcm_dpo/beta": 0.003976684529334307, "fcm_dpo/delta": -0.035320185124874115, "fcm_dpo/margin": 83.84129333496094, "fcm_dpo/q_t": 0.4211466908454895, "grad_norm": 66.4306411743164, "learning_rate": 2.600664850273538e-07, "logits/chosen": -0.8702799677848816, "logits/rejected": -0.8408257961273193, "logps/chosen": -422.9967346191406, "logps/ref_chosen": -277.6827392578125, "logps/ref_rejected": -250.73385620117188, "logps/rejected": -479.8891296386719, "loss": 4.5294, "margin_dpo/margin_mean": 83.8412857055664, "margin_dpo/margin_std": 111.361328125, "step": 258 }, { "KL/chosen_KL_mean": -138.1536102294922, "KL/mean": -175.6834716796875, "KL/rejected_KL_mean": -213.21331787109375, "KL/std": 108.60619354248047, "epoch": 0.5424083769633508, "fcm_dpo/beta": 0.003963233437389135, "fcm_dpo/delta": 0.0012955646961927414, "fcm_dpo/margin": 75.05972290039062, "fcm_dpo/q_t": 0.4318161606788635, "grad_norm": 80.89569091796875, "learning_rate": 2.582369512637302e-07, "logits/chosen": -0.8907499313354492, "logits/rejected": -0.885222315788269, "logps/chosen": -432.7635498046875, "logps/ref_chosen": -294.6099853515625, "logps/ref_rejected": -272.2725830078125, "logps/rejected": -485.4858703613281, "loss": 4.6869, "margin_dpo/margin_mean": 75.05972290039062, "margin_dpo/margin_std": 123.2925033569336, "step": 259 }, { "KL/chosen_KL_mean": -151.42819213867188, "KL/mean": -169.2168426513672, "KL/rejected_KL_mean": -187.00550842285156, "KL/std": 95.43949127197266, "epoch": 0.5445026178010471, "fcm_dpo/beta": 0.003953398671001196, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.57732391357422, "fcm_dpo/q_t": 0.4671649634838104, "grad_norm": 74.01944732666016, "learning_rate": 2.5640697577740815e-07, "logits/chosen": -0.8799023628234863, "logits/rejected": -0.873131513595581, "logps/chosen": -442.2852783203125, "logps/ref_chosen": -290.85711669921875, "logps/ref_rejected": -277.5970153808594, "logps/rejected": -464.6025390625, "loss": 5.2063, "margin_dpo/margin_mean": 35.57732009887695, "margin_dpo/margin_std": 112.50765228271484, "step": 260 }, { "KL/chosen_KL_mean": -163.0951690673828, "KL/mean": -195.61669921875, "KL/rejected_KL_mean": -228.1382293701172, "KL/std": 109.09551239013672, "epoch": 0.5465968586387434, "fcm_dpo/beta": 0.003954585175961256, "fcm_dpo/delta": 0.004022921435534954, "fcm_dpo/margin": 65.04306030273438, "fcm_dpo/q_t": 0.44060125946998596, "grad_norm": 52.7375373840332, "learning_rate": 2.5457665670441937e-07, "logits/chosen": -0.778756856918335, "logits/rejected": -0.787772536277771, "logps/chosen": -414.2274169921875, "logps/ref_chosen": -251.13223266601562, "logps/ref_rejected": -244.76016235351562, "logps/rejected": -472.89837646484375, "loss": 4.8579, "margin_dpo/margin_mean": 65.04306030273438, "margin_dpo/margin_std": 130.55770874023438, "step": 261 }, { "KL/chosen_KL_mean": -157.522216796875, "KL/mean": -198.93121337890625, "KL/rejected_KL_mean": -240.34019470214844, "KL/std": 108.54368591308594, "epoch": 0.5486910994764398, "fcm_dpo/beta": 0.003956255502998829, "fcm_dpo/delta": -0.019527418538928032, "fcm_dpo/margin": 82.81796264648438, "fcm_dpo/q_t": 0.4236336648464203, "grad_norm": 64.078125, "learning_rate": 2.527460921992209e-07, "logits/chosen": -0.8164616823196411, "logits/rejected": -0.8031435012817383, "logps/chosen": -457.2440185546875, "logps/ref_chosen": -299.7217712402344, "logps/ref_rejected": -277.0969543457031, "logps/rejected": -517.4371948242188, "loss": 4.5771, "margin_dpo/margin_mean": 82.81796264648438, "margin_dpo/margin_std": 120.71702575683594, "step": 262 }, { "KL/chosen_KL_mean": -167.35940551757812, "KL/mean": -204.05819702148438, "KL/rejected_KL_mean": -240.75698852539062, "KL/std": 112.38304138183594, "epoch": 0.5507853403141362, "fcm_dpo/beta": 0.0039150831289589405, "fcm_dpo/delta": 0.004613391123712063, "fcm_dpo/margin": 73.39756774902344, "fcm_dpo/q_t": 0.4326966404914856, "grad_norm": 57.92847442626953, "learning_rate": 2.509153804294318e-07, "logits/chosen": -0.8228314518928528, "logits/rejected": -0.8032459020614624, "logps/chosen": -447.3119812011719, "logps/ref_chosen": -279.95257568359375, "logps/ref_rejected": -256.5327453613281, "logps/rejected": -497.28973388671875, "loss": 4.7408, "margin_dpo/margin_mean": 73.39756774902344, "margin_dpo/margin_std": 127.13219451904297, "step": 263 }, { "KL/chosen_KL_mean": -162.80029296875, "KL/mean": -206.62985229492188, "KL/rejected_KL_mean": -250.45938110351562, "KL/std": 113.47503662109375, "epoch": 0.5528795811518324, "fcm_dpo/beta": 0.003873462090268731, "fcm_dpo/delta": -0.020922593772411346, "fcm_dpo/margin": 87.65908813476562, "fcm_dpo/q_t": 0.42167770862579346, "grad_norm": 74.20809936523438, "learning_rate": 2.4908461957056825e-07, "logits/chosen": -0.8265196084976196, "logits/rejected": -0.8174469470977783, "logps/chosen": -423.3354187011719, "logps/ref_chosen": -260.53509521484375, "logps/ref_rejected": -255.53799438476562, "logps/rejected": -505.9973449707031, "loss": 4.5581, "margin_dpo/margin_mean": 87.65908813476562, "margin_dpo/margin_std": 128.83778381347656, "step": 264 }, { "KL/chosen_KL_mean": -166.21475219726562, "KL/mean": -212.19488525390625, "KL/rejected_KL_mean": -258.17498779296875, "KL/std": 123.5406265258789, "epoch": 0.5549738219895288, "fcm_dpo/beta": 0.0038373975548893213, "fcm_dpo/delta": -0.007137385196983814, "fcm_dpo/margin": 91.96025848388672, "fcm_dpo/q_t": 0.4191870093345642, "grad_norm": 64.72450256347656, "learning_rate": 2.4725390780077905e-07, "logits/chosen": -0.8913218379020691, "logits/rejected": -0.8904725909233093, "logps/chosen": -449.9277648925781, "logps/ref_chosen": -283.7130432128906, "logps/ref_rejected": -270.3209533691406, "logps/rejected": -528.4959716796875, "loss": 4.5638, "margin_dpo/margin_mean": 91.96025848388672, "margin_dpo/margin_std": 144.87387084960938, "step": 265 }, { "KL/chosen_KL_mean": -152.7468719482422, "KL/mean": -199.51541137695312, "KL/rejected_KL_mean": -246.28395080566406, "KL/std": 112.15851593017578, "epoch": 0.5570680628272251, "fcm_dpo/beta": 0.0038449072744697332, "fcm_dpo/delta": -0.0055340006947517395, "fcm_dpo/margin": 93.53707885742188, "fcm_dpo/q_t": 0.41636204719543457, "grad_norm": 73.08821868896484, "learning_rate": 2.454233432955807e-07, "logits/chosen": -0.9144778847694397, "logits/rejected": -0.8848609328269958, "logps/chosen": -430.84613037109375, "logps/ref_chosen": -278.09930419921875, "logps/ref_rejected": -260.6734619140625, "logps/rejected": -506.9573974609375, "loss": 4.4522, "margin_dpo/margin_mean": 93.53707885742188, "margin_dpo/margin_std": 120.03593444824219, "step": 266 }, { "KL/chosen_KL_mean": -182.0021514892578, "KL/mean": -217.25442504882812, "KL/rejected_KL_mean": -252.5067138671875, "KL/std": 117.42387390136719, "epoch": 0.5591623036649215, "fcm_dpo/beta": 0.003810013411566615, "fcm_dpo/delta": -0.003673842176795006, "fcm_dpo/margin": 70.50455474853516, "fcm_dpo/q_t": 0.4374660849571228, "grad_norm": 74.9322738647461, "learning_rate": 2.435930242225919e-07, "logits/chosen": -0.8645923733711243, "logits/rejected": -0.8707435131072998, "logps/chosen": -462.3353271484375, "logps/ref_chosen": -280.33319091796875, "logps/ref_rejected": -247.78099060058594, "logps/rejected": -500.28765869140625, "loss": 4.8086, "margin_dpo/margin_mean": 70.50454711914062, "margin_dpo/margin_std": 132.0673065185547, "step": 267 }, { "KL/chosen_KL_mean": -172.97525024414062, "KL/mean": -217.17364501953125, "KL/rejected_KL_mean": -261.3720703125, "KL/std": 120.18418884277344, "epoch": 0.5612565445026177, "fcm_dpo/beta": 0.0038873879238963127, "fcm_dpo/delta": 0.034165654331445694, "fcm_dpo/margin": 88.39683532714844, "fcm_dpo/q_t": 0.42039138078689575, "grad_norm": 72.54732513427734, "learning_rate": 2.4176304873626984e-07, "logits/chosen": -0.829031229019165, "logits/rejected": -0.807607889175415, "logps/chosen": -477.1539611816406, "logps/ref_chosen": -304.1787109375, "logps/ref_rejected": -272.80316162109375, "logps/rejected": -534.1752319335938, "loss": 4.5311, "margin_dpo/margin_mean": 88.39683532714844, "margin_dpo/margin_std": 126.67704010009766, "step": 268 }, { "KL/chosen_KL_mean": -176.60040283203125, "KL/mean": -205.09878540039062, "KL/rejected_KL_mean": -233.59716796875, "KL/std": 104.05487060546875, "epoch": 0.5633507853403141, "fcm_dpo/beta": 0.003917751833796501, "fcm_dpo/delta": 0.0035603800788521767, "fcm_dpo/margin": 56.99675750732422, "fcm_dpo/q_t": 0.4470731317996979, "grad_norm": 71.9229965209961, "learning_rate": 2.399335149726463e-07, "logits/chosen": -0.8421822786331177, "logits/rejected": -0.835168719291687, "logps/chosen": -426.4455261230469, "logps/ref_chosen": -249.84512329101562, "logps/ref_rejected": -223.37356567382812, "logps/rejected": -456.9707336425781, "loss": 4.9429, "margin_dpo/margin_mean": 56.996761322021484, "margin_dpo/margin_std": 125.362548828125, "step": 269 }, { "KL/chosen_KL_mean": -185.8561248779297, "KL/mean": -224.6792755126953, "KL/rejected_KL_mean": -263.5024108886719, "KL/std": 120.34756469726562, "epoch": 0.5654450261780105, "fcm_dpo/beta": 0.003944146912544966, "fcm_dpo/delta": 0.01949712075293064, "fcm_dpo/margin": 77.64627838134766, "fcm_dpo/q_t": 0.43020257353782654, "grad_norm": 67.843505859375, "learning_rate": 2.381045210440644e-07, "logits/chosen": -0.934403657913208, "logits/rejected": -0.9376578330993652, "logps/chosen": -504.4185485839844, "logps/ref_chosen": -318.5623779296875, "logps/ref_rejected": -281.1880798339844, "logps/rejected": -544.6904296875, "loss": 4.7418, "margin_dpo/margin_mean": 77.64627838134766, "margin_dpo/margin_std": 149.40438842773438, "step": 270 }, { "KL/chosen_KL_mean": -181.69509887695312, "KL/mean": -218.57923889160156, "KL/rejected_KL_mean": -255.46336364746094, "KL/std": 120.86827850341797, "epoch": 0.5675392670157068, "fcm_dpo/beta": 0.003990632947534323, "fcm_dpo/delta": 0.0031273523345589638, "fcm_dpo/margin": 73.76829528808594, "fcm_dpo/q_t": 0.4324439764022827, "grad_norm": 58.05919647216797, "learning_rate": 2.3627616503391812e-07, "logits/chosen": -0.7795909643173218, "logits/rejected": -0.7716021537780762, "logps/chosen": -465.7997741699219, "logps/ref_chosen": -284.104736328125, "logps/ref_rejected": -253.9580535888672, "logps/rejected": -509.42144775390625, "loss": 4.7087, "margin_dpo/margin_mean": 73.76829528808594, "margin_dpo/margin_std": 125.9541015625, "step": 271 }, { "KL/chosen_KL_mean": -169.88497924804688, "KL/mean": -210.8214111328125, "KL/rejected_KL_mean": -251.75787353515625, "KL/std": 119.86404418945312, "epoch": 0.5696335078534032, "fcm_dpo/beta": 0.0039629037491977215, "fcm_dpo/delta": -0.0006313333287835121, "fcm_dpo/margin": 81.87287902832031, "fcm_dpo/q_t": 0.42572107911109924, "grad_norm": 86.46068572998047, "learning_rate": 2.344485449913914e-07, "logits/chosen": -0.883471667766571, "logits/rejected": -0.8656666874885559, "logps/chosen": -467.2440185546875, "logps/ref_chosen": -297.3590087890625, "logps/ref_rejected": -279.20196533203125, "logps/rejected": -530.9597778320312, "loss": 4.678, "margin_dpo/margin_mean": 81.87287902832031, "margin_dpo/margin_std": 145.54736328125, "step": 272 }, { "KL/chosen_KL_mean": -174.3992919921875, "KL/mean": -216.345947265625, "KL/rejected_KL_mean": -258.2926025390625, "KL/std": 115.58932495117188, "epoch": 0.5717277486910994, "fcm_dpo/beta": 0.003995942883193493, "fcm_dpo/delta": 0.009679936803877354, "fcm_dpo/margin": 83.89334106445312, "fcm_dpo/q_t": 0.4239039421081543, "grad_norm": 88.02112579345703, "learning_rate": 2.3262175892620062e-07, "logits/chosen": -0.8663466572761536, "logits/rejected": -0.8779019117355347, "logps/chosen": -467.60504150390625, "logps/ref_chosen": -293.20574951171875, "logps/ref_rejected": -274.7646789550781, "logps/rejected": -533.0572509765625, "loss": 4.6362, "margin_dpo/margin_mean": 83.89334869384766, "margin_dpo/margin_std": 143.1727752685547, "step": 273 }, { "KL/chosen_KL_mean": -172.38304138183594, "KL/mean": -221.9813690185547, "KL/rejected_KL_mean": -271.5797119140625, "KL/std": 125.11678314208984, "epoch": 0.5738219895287958, "fcm_dpo/beta": 0.0040048351511359215, "fcm_dpo/delta": 0.0025398635771125555, "fcm_dpo/margin": 99.1966552734375, "fcm_dpo/q_t": 0.4098568260669708, "grad_norm": 81.96195220947266, "learning_rate": 2.3079590480333827e-07, "logits/chosen": -0.8212272524833679, "logits/rejected": -0.78861403465271, "logps/chosen": -442.94171142578125, "logps/ref_chosen": -270.55865478515625, "logps/ref_rejected": -239.47048950195312, "logps/rejected": -511.0502014160156, "loss": 4.4372, "margin_dpo/margin_mean": 99.19667053222656, "margin_dpo/margin_std": 144.70790100097656, "step": 274 }, { "KL/chosen_KL_mean": -157.3403778076172, "KL/mean": -207.17703247070312, "KL/rejected_KL_mean": -257.013671875, "KL/std": 118.32302856445312, "epoch": 0.5759162303664922, "fcm_dpo/beta": 0.003956434316933155, "fcm_dpo/delta": -0.030254080891609192, "fcm_dpo/margin": 99.67333221435547, "fcm_dpo/q_t": 0.40914809703826904, "grad_norm": 58.78046417236328, "learning_rate": 2.2897108053782e-07, "logits/chosen": -0.875052809715271, "logits/rejected": -0.8559980988502502, "logps/chosen": -407.65960693359375, "logps/ref_chosen": -250.31922912597656, "logps/ref_rejected": -249.3187255859375, "logps/rejected": -506.3324279785156, "loss": 4.3663, "margin_dpo/margin_mean": 99.67330932617188, "margin_dpo/margin_std": 125.57425689697266, "step": 275 }, { "KL/chosen_KL_mean": -171.91647338867188, "KL/mean": -217.97848510742188, "KL/rejected_KL_mean": -264.04046630859375, "KL/std": 126.98786926269531, "epoch": 0.5780104712041885, "fcm_dpo/beta": 0.0038329358212649822, "fcm_dpo/delta": -0.035872187465429306, "fcm_dpo/margin": 92.1240005493164, "fcm_dpo/q_t": 0.4182504713535309, "grad_norm": 55.775657653808594, "learning_rate": 2.2714738398943308e-07, "logits/chosen": -0.9235668182373047, "logits/rejected": -0.9017929434776306, "logps/chosen": -469.54754638671875, "logps/ref_chosen": -297.6310729980469, "logps/ref_rejected": -295.225830078125, "logps/rejected": -559.2662963867188, "loss": 4.556, "margin_dpo/margin_mean": 92.1240005493164, "margin_dpo/margin_std": 140.80569458007812, "step": 276 }, { "KL/chosen_KL_mean": -193.89830017089844, "KL/mean": -225.9439697265625, "KL/rejected_KL_mean": -257.9896240234375, "KL/std": 126.7256088256836, "epoch": 0.5801047120418849, "fcm_dpo/beta": 0.003798598423600197, "fcm_dpo/delta": -0.014602910727262497, "fcm_dpo/margin": 64.09132385253906, "fcm_dpo/q_t": 0.4443640112876892, "grad_norm": 64.50138092041016, "learning_rate": 2.2532491295748865e-07, "logits/chosen": -0.8662960529327393, "logits/rejected": -0.8660153746604919, "logps/chosen": -460.2587890625, "logps/ref_chosen": -266.3604736328125, "logps/ref_rejected": -253.36767578125, "logps/rejected": -511.3572998046875, "loss": 4.9367, "margin_dpo/margin_mean": 64.09132385253906, "margin_dpo/margin_std": 143.8057861328125, "step": 277 }, { "KL/chosen_KL_mean": -202.2888641357422, "KL/mean": -230.39736938476562, "KL/rejected_KL_mean": -258.5058898925781, "KL/std": 130.2160186767578, "epoch": 0.5821989528795811, "fcm_dpo/beta": 0.0037986235693097115, "fcm_dpo/delta": 0.011226484552025795, "fcm_dpo/margin": 56.217010498046875, "fcm_dpo/q_t": 0.4543144702911377, "grad_norm": 96.25747680664062, "learning_rate": 2.2350376517557726e-07, "logits/chosen": -0.8956958651542664, "logits/rejected": -0.8570727109909058, "logps/chosen": -469.6961669921875, "logps/ref_chosen": -267.40728759765625, "logps/ref_rejected": -229.5758514404297, "logps/rejected": -488.08172607421875, "loss": 5.1072, "margin_dpo/margin_mean": 56.217010498046875, "margin_dpo/margin_std": 157.5133514404297, "step": 278 }, { "KL/chosen_KL_mean": -185.69024658203125, "KL/mean": -241.31065368652344, "KL/rejected_KL_mean": -296.9310607910156, "KL/std": 138.99148559570312, "epoch": 0.5842931937172775, "fcm_dpo/beta": 0.0037318726535886526, "fcm_dpo/delta": -0.05649794638156891, "fcm_dpo/margin": 111.24081420898438, "fcm_dpo/q_t": 0.405769944190979, "grad_norm": 66.29354858398438, "learning_rate": 2.2168403830632769e-07, "logits/chosen": -0.8187253475189209, "logits/rejected": -0.8022856116294861, "logps/chosen": -499.05804443359375, "logps/ref_chosen": -313.3677978515625, "logps/ref_rejected": -299.1744384765625, "logps/rejected": -596.10546875, "loss": 4.3506, "margin_dpo/margin_mean": 111.24081420898438, "margin_dpo/margin_std": 146.99954223632812, "step": 279 }, { "KL/chosen_KL_mean": -182.4557342529297, "KL/mean": -221.92530822753906, "KL/rejected_KL_mean": -261.3948669433594, "KL/std": 114.89640045166016, "epoch": 0.5863874345549738, "fcm_dpo/beta": 0.0036900462582707405, "fcm_dpo/delta": 0.03023211471736431, "fcm_dpo/margin": 78.93912506103516, "fcm_dpo/q_t": 0.43237319588661194, "grad_norm": 65.89375305175781, "learning_rate": 2.1986582993616925e-07, "logits/chosen": -0.8743321895599365, "logits/rejected": -0.8858817219734192, "logps/chosen": -448.0115661621094, "logps/ref_chosen": -265.5558166503906, "logps/ref_rejected": -247.1573944091797, "logps/rejected": -508.55224609375, "loss": 4.7364, "margin_dpo/margin_mean": 78.93912506103516, "margin_dpo/margin_std": 143.53533935546875, "step": 280 }, { "KL/chosen_KL_mean": -207.59323120117188, "KL/mean": -253.11624145507812, "KL/rejected_KL_mean": -298.63922119140625, "KL/std": 127.79008483886719, "epoch": 0.5884816753926702, "fcm_dpo/beta": 0.0036901538260281086, "fcm_dpo/delta": -0.028504468500614166, "fcm_dpo/margin": 91.0459976196289, "fcm_dpo/q_t": 0.4240657687187195, "grad_norm": 67.67650604248047, "learning_rate": 2.1804923757009882e-07, "logits/chosen": -0.8543596267700195, "logits/rejected": -0.8635410070419312, "logps/chosen": -502.892822265625, "logps/ref_chosen": -295.2995910644531, "logps/ref_rejected": -293.80877685546875, "logps/rejected": -592.447998046875, "loss": 4.6098, "margin_dpo/margin_mean": 91.04598236083984, "margin_dpo/margin_std": 144.34996032714844, "step": 281 }, { "KL/chosen_KL_mean": -202.552490234375, "KL/mean": -242.41110229492188, "KL/rejected_KL_mean": -282.2697448730469, "KL/std": 125.7529067993164, "epoch": 0.5905759162303665, "fcm_dpo/beta": 0.003655828535556793, "fcm_dpo/delta": 0.0036571845412254333, "fcm_dpo/margin": 79.71723937988281, "fcm_dpo/q_t": 0.43203699588775635, "grad_norm": 72.81260681152344, "learning_rate": 2.1623435862645205e-07, "logits/chosen": -0.8544440269470215, "logits/rejected": -0.8522729873657227, "logps/chosen": -521.1896362304688, "logps/ref_chosen": -318.63714599609375, "logps/ref_rejected": -273.5943603515625, "logps/rejected": -555.8641357421875, "loss": 4.7281, "margin_dpo/margin_mean": 79.71723937988281, "margin_dpo/margin_std": 139.59954833984375, "step": 282 }, { "KL/chosen_KL_mean": -202.61456298828125, "KL/mean": -244.5399932861328, "KL/rejected_KL_mean": -286.4654541015625, "KL/std": 124.06411743164062, "epoch": 0.5926701570680628, "fcm_dpo/beta": 0.0036506266333162785, "fcm_dpo/delta": -0.0035746542271226645, "fcm_dpo/margin": 83.85090637207031, "fcm_dpo/q_t": 0.43061453104019165, "grad_norm": 57.49915313720703, "learning_rate": 2.1442129043167873e-07, "logits/chosen": -0.876941442489624, "logits/rejected": -0.8653864860534668, "logps/chosen": -457.2751159667969, "logps/ref_chosen": -254.66053771972656, "logps/ref_rejected": -236.8627166748047, "logps/rejected": -523.3281860351562, "loss": 4.6959, "margin_dpo/margin_mean": 83.85089111328125, "margin_dpo/margin_std": 145.12371826171875, "step": 283 }, { "KL/chosen_KL_mean": -210.0465545654297, "KL/mean": -254.60189819335938, "KL/rejected_KL_mean": -299.1572265625, "KL/std": 129.64718627929688, "epoch": 0.5947643979057592, "fcm_dpo/beta": 0.0035921428352594376, "fcm_dpo/delta": -0.02668537199497223, "fcm_dpo/margin": 89.11066436767578, "fcm_dpo/q_t": 0.42657744884490967, "grad_norm": 73.3239517211914, "learning_rate": 2.1261013021512378e-07, "logits/chosen": -0.8376402258872986, "logits/rejected": -0.8031928539276123, "logps/chosen": -483.4017639160156, "logps/ref_chosen": -273.355224609375, "logps/ref_rejected": -259.84759521484375, "logps/rejected": -559.0048217773438, "loss": 4.6655, "margin_dpo/margin_mean": 89.11066436767578, "margin_dpo/margin_std": 149.5060577392578, "step": 284 }, { "KL/chosen_KL_mean": -217.2931365966797, "KL/mean": -246.5795135498047, "KL/rejected_KL_mean": -275.86590576171875, "KL/std": 122.36019897460938, "epoch": 0.5968586387434555, "fcm_dpo/beta": 0.0035818694159388542, "fcm_dpo/delta": 0.003046293742954731, "fcm_dpo/margin": 58.572792053222656, "fcm_dpo/q_t": 0.4514738619327545, "grad_norm": 84.60891723632812, "learning_rate": 2.1080097510381294e-07, "logits/chosen": -0.8543550372123718, "logits/rejected": -0.847979724407196, "logps/chosen": -527.0953369140625, "logps/ref_chosen": -309.8022155761719, "logps/ref_rejected": -279.11846923828125, "logps/rejected": -554.984375, "loss": 5.0642, "margin_dpo/margin_mean": 58.572792053222656, "margin_dpo/margin_std": 150.84938049316406, "step": 285 }, { "KL/chosen_KL_mean": -208.0546112060547, "KL/mean": -246.29867553710938, "KL/rejected_KL_mean": -284.5427551269531, "KL/std": 131.73287963867188, "epoch": 0.5989528795811518, "fcm_dpo/beta": 0.00358386617153883, "fcm_dpo/delta": 0.004873398691415787, "fcm_dpo/margin": 76.48812103271484, "fcm_dpo/q_t": 0.43887919187545776, "grad_norm": 92.04210662841797, "learning_rate": 2.089939221172446e-07, "logits/chosen": -0.8385077118873596, "logits/rejected": -0.8326424360275269, "logps/chosen": -479.5201721191406, "logps/ref_chosen": -271.4655456542969, "logps/ref_rejected": -279.531494140625, "logps/rejected": -564.07421875, "loss": 4.83, "margin_dpo/margin_mean": 76.48812103271484, "margin_dpo/margin_std": 154.2538604736328, "step": 286 }, { "KL/chosen_KL_mean": -207.88980102539062, "KL/mean": -244.87991333007812, "KL/rejected_KL_mean": -281.87005615234375, "KL/std": 124.23129272460938, "epoch": 0.6010471204188481, "fcm_dpo/beta": 0.003638399764895439, "fcm_dpo/delta": 0.026291579008102417, "fcm_dpo/margin": 73.98025512695312, "fcm_dpo/q_t": 0.4376610815525055, "grad_norm": 68.3472900390625, "learning_rate": 2.0718906816218595e-07, "logits/chosen": -0.8518833518028259, "logits/rejected": -0.8357435464859009, "logps/chosen": -484.9830322265625, "logps/ref_chosen": -277.0932312011719, "logps/ref_rejected": -233.55599975585938, "logps/rejected": -515.426025390625, "loss": 4.859, "margin_dpo/margin_mean": 73.98026275634766, "margin_dpo/margin_std": 155.37527465820312, "step": 287 }, { "KL/chosen_KL_mean": -199.41354370117188, "KL/mean": -238.12754821777344, "KL/rejected_KL_mean": -276.84161376953125, "KL/std": 129.22042846679688, "epoch": 0.6031413612565445, "fcm_dpo/beta": 0.003688263474032283, "fcm_dpo/delta": -0.005784134846180677, "fcm_dpo/margin": 77.42807006835938, "fcm_dpo/q_t": 0.43434223532676697, "grad_norm": 77.68144226074219, "learning_rate": 2.053865100274774e-07, "logits/chosen": -0.8734298944473267, "logits/rejected": -0.8790357708930969, "logps/chosen": -492.58172607421875, "logps/ref_chosen": -293.1681823730469, "logps/ref_rejected": -263.4059143066406, "logps/rejected": -540.2474975585938, "loss": 4.7674, "margin_dpo/margin_mean": 77.4280776977539, "margin_dpo/margin_std": 144.46633911132812, "step": 288 }, { "KL/chosen_KL_mean": -219.11532592773438, "KL/mean": -241.2076873779297, "KL/rejected_KL_mean": -263.3000183105469, "KL/std": 120.56001281738281, "epoch": 0.6052356020942409, "fcm_dpo/beta": 0.00366892758756876, "fcm_dpo/delta": 0.015712561085820198, "fcm_dpo/margin": 44.1847038269043, "fcm_dpo/q_t": 0.46217232942581177, "grad_norm": 70.22594451904297, "learning_rate": 2.035863443788411e-07, "logits/chosen": -0.8561855554580688, "logits/rejected": -0.8383423089981079, "logps/chosen": -549.07275390625, "logps/ref_chosen": -329.9574279785156, "logps/ref_rejected": -276.7565002441406, "logps/rejected": -540.0565185546875, "loss": 5.1915, "margin_dpo/margin_mean": 44.1847038269043, "margin_dpo/margin_std": 136.41799926757812, "step": 289 }, { "KL/chosen_KL_mean": -193.29393005371094, "KL/mean": -233.5602569580078, "KL/rejected_KL_mean": -273.82659912109375, "KL/std": 127.18586730957031, "epoch": 0.6073298429319371, "fcm_dpo/beta": 0.0036051059141755104, "fcm_dpo/delta": -0.03102608397603035, "fcm_dpo/margin": 80.5326919555664, "fcm_dpo/q_t": 0.43328824639320374, "grad_norm": 100.93278503417969, "learning_rate": 2.0178866775369774e-07, "logits/chosen": -0.8621543049812317, "logits/rejected": -0.7959156036376953, "logps/chosen": -517.9629516601562, "logps/ref_chosen": -324.6690673828125, "logps/ref_rejected": -311.8439636230469, "logps/rejected": -585.6705932617188, "loss": 4.7497, "margin_dpo/margin_mean": 80.5326919555664, "margin_dpo/margin_std": 144.66436767578125, "step": 290 }, { "KL/chosen_KL_mean": -185.8102264404297, "KL/mean": -235.31077575683594, "KL/rejected_KL_mean": -284.8113098144531, "KL/std": 127.50725555419922, "epoch": 0.6094240837696335, "fcm_dpo/beta": 0.003556034294888377, "fcm_dpo/delta": -0.027959764003753662, "fcm_dpo/margin": 99.00109100341797, "fcm_dpo/q_t": 0.41859322786331177, "grad_norm": 57.9321174621582, "learning_rate": 1.9999357655598891e-07, "logits/chosen": -0.8408191204071045, "logits/rejected": -0.8317016959190369, "logps/chosen": -459.95428466796875, "logps/ref_chosen": -274.1440734863281, "logps/ref_rejected": -278.07208251953125, "logps/rejected": -562.8833618164062, "loss": 4.5248, "margin_dpo/margin_mean": 99.00109100341797, "margin_dpo/margin_std": 140.4064178466797, "step": 291 }, { "KL/chosen_KL_mean": -173.9268798828125, "KL/mean": -216.89697265625, "KL/rejected_KL_mean": -259.8670349121094, "KL/std": 126.9601821899414, "epoch": 0.6115183246073298, "fcm_dpo/beta": 0.0034741119015961885, "fcm_dpo/delta": -0.024277152493596077, "fcm_dpo/margin": 85.94014739990234, "fcm_dpo/q_t": 0.4312286972999573, "grad_norm": 64.79742431640625, "learning_rate": 1.9820116705100775e-07, "logits/chosen": -0.8414206504821777, "logits/rejected": -0.8373555541038513, "logps/chosen": -433.2906188964844, "logps/ref_chosen": -259.3636779785156, "logps/ref_rejected": -279.30218505859375, "logps/rejected": -539.1692504882812, "loss": 4.7164, "margin_dpo/margin_mean": 85.94014739990234, "margin_dpo/margin_std": 148.6029815673828, "step": 292 }, { "KL/chosen_KL_mean": -196.158203125, "KL/mean": -242.4661102294922, "KL/rejected_KL_mean": -288.7740783691406, "KL/std": 135.72119140625, "epoch": 0.6136125654450262, "fcm_dpo/beta": 0.003440189640969038, "fcm_dpo/delta": -0.006982447579503059, "fcm_dpo/margin": 92.61585998535156, "fcm_dpo/q_t": 0.42598748207092285, "grad_norm": 62.069820404052734, "learning_rate": 1.9641153536023642e-07, "logits/chosen": -0.9244207143783569, "logits/rejected": -0.8833814859390259, "logps/chosen": -499.92901611328125, "logps/ref_chosen": -303.77081298828125, "logps/ref_rejected": -270.07513427734375, "logps/rejected": -558.8491821289062, "loss": 4.6122, "margin_dpo/margin_mean": 92.61585998535156, "margin_dpo/margin_std": 139.9814910888672, "step": 293 }, { "KL/chosen_KL_mean": -186.78228759765625, "KL/mean": -225.4820556640625, "KL/rejected_KL_mean": -264.18182373046875, "KL/std": 131.13441467285156, "epoch": 0.6157068062827226, "fcm_dpo/beta": 0.003445146605372429, "fcm_dpo/delta": -0.002983933314681053, "fcm_dpo/margin": 77.39950561523438, "fcm_dpo/q_t": 0.4392993748188019, "grad_norm": 56.854644775390625, "learning_rate": 1.9462477745619106e-07, "logits/chosen": -0.8256325125694275, "logits/rejected": -0.8303864002227783, "logps/chosen": -427.02056884765625, "logps/ref_chosen": -240.23831176757812, "logps/ref_rejected": -229.187744140625, "logps/rejected": -493.3695373535156, "loss": 4.8415, "margin_dpo/margin_mean": 77.39949798583984, "margin_dpo/margin_std": 154.39572143554688, "step": 294 }, { "KL/chosen_KL_mean": -188.3880615234375, "KL/mean": -226.49118041992188, "KL/rejected_KL_mean": -264.59429931640625, "KL/std": 112.24237060546875, "epoch": 0.6178010471204188, "fcm_dpo/beta": 0.003445256035774946, "fcm_dpo/delta": 0.014424074441194534, "fcm_dpo/margin": 76.20626068115234, "fcm_dpo/q_t": 0.4396905303001404, "grad_norm": 69.74424743652344, "learning_rate": 1.928409891572757e-07, "logits/chosen": -0.8317238688468933, "logits/rejected": -0.8348393440246582, "logps/chosen": -439.3977966308594, "logps/ref_chosen": -251.00970458984375, "logps/ref_rejected": -244.15142822265625, "logps/rejected": -508.74578857421875, "loss": 4.7987, "margin_dpo/margin_mean": 76.20626068115234, "margin_dpo/margin_std": 140.69558715820312, "step": 295 }, { "KL/chosen_KL_mean": -192.59164428710938, "KL/mean": -244.67086791992188, "KL/rejected_KL_mean": -296.7500915527344, "KL/std": 131.72547912597656, "epoch": 0.6198952879581152, "fcm_dpo/beta": 0.0034456930588930845, "fcm_dpo/delta": -0.02413531392812729, "fcm_dpo/margin": 104.15848541259766, "fcm_dpo/q_t": 0.417242169380188, "grad_norm": 68.13108825683594, "learning_rate": 1.9106026612264315e-07, "logits/chosen": -0.8389942646026611, "logits/rejected": -0.8055410981178284, "logps/chosen": -486.47222900390625, "logps/ref_chosen": -293.880615234375, "logps/ref_rejected": -283.4175720214844, "logps/rejected": -580.167724609375, "loss": 4.539, "margin_dpo/margin_mean": 104.15848541259766, "margin_dpo/margin_std": 157.52789306640625, "step": 296 }, { "KL/chosen_KL_mean": -189.89254760742188, "KL/mean": -227.05242919921875, "KL/rejected_KL_mean": -264.21234130859375, "KL/std": 118.02452087402344, "epoch": 0.6219895287958115, "fcm_dpo/beta": 0.003379724221304059, "fcm_dpo/delta": -0.012697052210569382, "fcm_dpo/margin": 74.31977844238281, "fcm_dpo/q_t": 0.44128429889678955, "grad_norm": 60.725318908691406, "learning_rate": 1.8928270384706582e-07, "logits/chosen": -0.9069033265113831, "logits/rejected": -0.9000098705291748, "logps/chosen": -479.35260009765625, "logps/ref_chosen": -289.4600830078125, "logps/ref_rejected": -283.69110107421875, "logps/rejected": -547.9033813476562, "loss": 4.8268, "margin_dpo/margin_mean": 74.31977844238281, "margin_dpo/margin_std": 135.9732666015625, "step": 297 }, { "KL/chosen_KL_mean": -180.1241455078125, "KL/mean": -226.821044921875, "KL/rejected_KL_mean": -273.5179443359375, "KL/std": 125.18611907958984, "epoch": 0.6240837696335079, "fcm_dpo/beta": 0.0033408179879188538, "fcm_dpo/delta": -0.020929735153913498, "fcm_dpo/margin": 93.3937759399414, "fcm_dpo/q_t": 0.4280543327331543, "grad_norm": 89.00557708740234, "learning_rate": 1.875083976558136e-07, "logits/chosen": -0.8199465870857239, "logits/rejected": -0.8056501150131226, "logps/chosen": -486.6391296386719, "logps/ref_chosen": -306.5150146484375, "logps/ref_rejected": -280.6969909667969, "logps/rejected": -554.21484375, "loss": 4.6812, "margin_dpo/margin_mean": 93.3937759399414, "margin_dpo/margin_std": 156.3482666015625, "step": 298 }, { "KL/chosen_KL_mean": -210.533447265625, "KL/mean": -247.99322509765625, "KL/rejected_KL_mean": -285.4530334472656, "KL/std": 129.0129852294922, "epoch": 0.6261780104712041, "fcm_dpo/beta": 0.0033021531999111176, "fcm_dpo/delta": -0.0064115277491509914, "fcm_dpo/margin": 74.91960906982422, "fcm_dpo/q_t": 0.442142128944397, "grad_norm": 70.9146957397461, "learning_rate": 1.8573744269954297e-07, "logits/chosen": -0.8116376399993896, "logits/rejected": -0.7935671210289001, "logps/chosen": -491.8971862792969, "logps/ref_chosen": -281.36376953125, "logps/ref_rejected": -270.39508056640625, "logps/rejected": -555.84814453125, "loss": 4.8285, "margin_dpo/margin_mean": 74.91960906982422, "margin_dpo/margin_std": 130.8252410888672, "step": 299 }, { "KL/chosen_KL_mean": -212.4322509765625, "KL/mean": -248.7073211669922, "KL/rejected_KL_mean": -284.982421875, "KL/std": 135.9843292236328, "epoch": 0.6282722513089005, "fcm_dpo/beta": 0.00330313085578382, "fcm_dpo/delta": 0.010395125485956669, "fcm_dpo/margin": 72.5501480102539, "fcm_dpo/q_t": 0.4443802237510681, "grad_norm": 66.76209259033203, "learning_rate": 1.839699339491937e-07, "logits/chosen": -0.836089551448822, "logits/rejected": -0.8139994144439697, "logps/chosen": -527.26806640625, "logps/ref_chosen": -314.83575439453125, "logps/ref_rejected": -269.1154479980469, "logps/rejected": -554.0978393554688, "loss": 4.9016, "margin_dpo/margin_mean": 72.5501480102539, "margin_dpo/margin_std": 154.16453552246094, "step": 300 }, { "KL/chosen_KL_mean": -201.32894897460938, "KL/mean": -238.62901306152344, "KL/rejected_KL_mean": -275.9290771484375, "KL/std": 128.91531372070312, "epoch": 0.6303664921465969, "fcm_dpo/beta": 0.003320886753499508, "fcm_dpo/delta": -0.009585805237293243, "fcm_dpo/margin": 74.60012817382812, "fcm_dpo/q_t": 0.4424639344215393, "grad_norm": 81.5901870727539, "learning_rate": 1.8220596619089573e-07, "logits/chosen": -0.8685322999954224, "logits/rejected": -0.8592681884765625, "logps/chosen": -481.2235107421875, "logps/ref_chosen": -279.89453125, "logps/ref_rejected": -271.6694641113281, "logps/rejected": -547.5985107421875, "loss": 4.8592, "margin_dpo/margin_mean": 74.60012817382812, "margin_dpo/margin_std": 144.12062072753906, "step": 301 }, { "KL/chosen_KL_mean": -197.4744415283203, "KL/mean": -246.68585205078125, "KL/rejected_KL_mean": -295.89727783203125, "KL/std": 135.5497589111328, "epoch": 0.6324607329842932, "fcm_dpo/beta": 0.0033070500940084457, "fcm_dpo/delta": 0.005456558428704739, "fcm_dpo/margin": 98.42284393310547, "fcm_dpo/q_t": 0.42443469166755676, "grad_norm": 70.69303131103516, "learning_rate": 1.8044563402088682e-07, "logits/chosen": -0.8222125172615051, "logits/rejected": -0.8099421262741089, "logps/chosen": -468.8062744140625, "logps/ref_chosen": -271.3318176269531, "logps/ref_rejected": -256.5587158203125, "logps/rejected": -552.4559936523438, "loss": 4.5808, "margin_dpo/margin_mean": 98.42283630371094, "margin_dpo/margin_std": 144.48379516601562, "step": 302 }, { "KL/chosen_KL_mean": -200.82537841796875, "KL/mean": -243.748291015625, "KL/rejected_KL_mean": -286.67120361328125, "KL/std": 122.43389892578125, "epoch": 0.6345549738219896, "fcm_dpo/beta": 0.0033165114000439644, "fcm_dpo/delta": 0.004780970048159361, "fcm_dpo/margin": 85.84585571289062, "fcm_dpo/q_t": 0.4328896403312683, "grad_norm": 66.35929870605469, "learning_rate": 1.7868903184043885e-07, "logits/chosen": -0.8214887380599976, "logits/rejected": -0.7972728610038757, "logps/chosen": -505.7063903808594, "logps/ref_chosen": -304.88104248046875, "logps/ref_rejected": -269.063720703125, "logps/rejected": -555.7349853515625, "loss": 4.6998, "margin_dpo/margin_mean": 85.84585571289062, "margin_dpo/margin_std": 135.42276000976562, "step": 303 }, { "KL/chosen_KL_mean": -215.84197998046875, "KL/mean": -262.8927001953125, "KL/rejected_KL_mean": -309.9433898925781, "KL/std": 149.1090850830078, "epoch": 0.6366492146596858, "fcm_dpo/beta": 0.003333859611302614, "fcm_dpo/delta": 0.005845099687576294, "fcm_dpo/margin": 94.10143280029297, "fcm_dpo/q_t": 0.42921072244644165, "grad_norm": 54.69770812988281, "learning_rate": 1.7693625385079574e-07, "logits/chosen": -0.8201622366905212, "logits/rejected": -0.8260871171951294, "logps/chosen": -506.55291748046875, "logps/ref_chosen": -290.7109680175781, "logps/ref_rejected": -237.6885986328125, "logps/rejected": -547.6320190429688, "loss": 4.677, "margin_dpo/margin_mean": 94.10143280029297, "margin_dpo/margin_std": 163.25584411621094, "step": 304 }, { "KL/chosen_KL_mean": -180.0332489013672, "KL/mean": -250.61903381347656, "KL/rejected_KL_mean": -321.204833984375, "KL/std": 140.91082763671875, "epoch": 0.6387434554973822, "fcm_dpo/beta": 0.0031983419321477413, "fcm_dpo/delta": -0.1049448549747467, "fcm_dpo/margin": 141.17156982421875, "fcm_dpo/q_t": 0.3961653709411621, "grad_norm": 65.096923828125, "learning_rate": 1.7518739404812155e-07, "logits/chosen": -0.8557301759719849, "logits/rejected": -0.8239875435829163, "logps/chosen": -436.51727294921875, "logps/ref_chosen": -256.4839782714844, "logps/ref_rejected": -266.4063415527344, "logps/rejected": -587.6111450195312, "loss": 4.2208, "margin_dpo/margin_mean": 141.17156982421875, "margin_dpo/margin_std": 163.42698669433594, "step": 305 }, { "KL/chosen_KL_mean": -197.95596313476562, "KL/mean": -238.4625244140625, "KL/rejected_KL_mean": -278.9691162109375, "KL/std": 122.23258209228516, "epoch": 0.6408376963350786, "fcm_dpo/beta": 0.0031039295718073845, "fcm_dpo/delta": 0.02336675301194191, "fcm_dpo/margin": 81.01317596435547, "fcm_dpo/q_t": 0.4409124553203583, "grad_norm": 70.48225402832031, "learning_rate": 1.7344254621846017e-07, "logits/chosen": -0.8595868945121765, "logits/rejected": -0.8387668132781982, "logps/chosen": -518.6051635742188, "logps/ref_chosen": -320.6492004394531, "logps/ref_rejected": -273.36773681640625, "logps/rejected": -552.3368530273438, "loss": 4.7914, "margin_dpo/margin_mean": 81.01317596435547, "margin_dpo/margin_std": 134.45468139648438, "step": 306 }, { "KL/chosen_KL_mean": -226.18310546875, "KL/mean": -269.3814392089844, "KL/rejected_KL_mean": -312.57977294921875, "KL/std": 138.75106811523438, "epoch": 0.6429319371727749, "fcm_dpo/beta": 0.00315967109054327, "fcm_dpo/delta": 0.011747002601623535, "fcm_dpo/margin": 86.39667510986328, "fcm_dpo/q_t": 0.4350773096084595, "grad_norm": 107.51629638671875, "learning_rate": 1.717018039327053e-07, "logits/chosen": -0.797162652015686, "logits/rejected": -0.8184522986412048, "logps/chosen": -505.6372985839844, "logps/ref_chosen": -279.4541931152344, "logps/ref_rejected": -240.3796844482422, "logps/rejected": -552.95947265625, "loss": 4.7194, "margin_dpo/margin_mean": 86.39667510986328, "margin_dpo/margin_std": 132.84042358398438, "step": 307 }, { "KL/chosen_KL_mean": -245.1857147216797, "KL/mean": -275.275146484375, "KL/rejected_KL_mean": -305.36456298828125, "KL/std": 142.4264373779297, "epoch": 0.6450261780104712, "fcm_dpo/beta": 0.003162857610732317, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.1788330078125, "fcm_dpo/q_t": 0.4557945728302002, "grad_norm": 66.74758911132812, "learning_rate": 1.699652605415828e-07, "logits/chosen": -0.8476360440254211, "logits/rejected": -0.8570371866226196, "logps/chosen": -541.7840576171875, "logps/ref_chosen": -296.598388671875, "logps/ref_rejected": -258.6953430175781, "logps/rejected": -564.0598754882812, "loss": 5.0646, "margin_dpo/margin_mean": 60.1788330078125, "margin_dpo/margin_std": 152.62615966796875, "step": 308 }, { "KL/chosen_KL_mean": -229.7421112060547, "KL/mean": -281.80914306640625, "KL/rejected_KL_mean": -333.8762512207031, "KL/std": 138.70144653320312, "epoch": 0.6471204188481675, "fcm_dpo/beta": 0.0031696951482445, "fcm_dpo/delta": 0.008564796298742294, "fcm_dpo/margin": 104.13411712646484, "fcm_dpo/q_t": 0.4243236780166626, "grad_norm": 69.46084594726562, "learning_rate": 1.6823300917064458e-07, "logits/chosen": -0.8503900766372681, "logits/rejected": -0.8593624234199524, "logps/chosen": -511.1302185058594, "logps/ref_chosen": -281.3881530761719, "logps/ref_rejected": -262.458740234375, "logps/rejected": -596.3349609375, "loss": 4.583, "margin_dpo/margin_mean": 104.13410949707031, "margin_dpo/margin_std": 161.4528045654297, "step": 309 }, { "KL/chosen_KL_mean": -222.09640502929688, "KL/mean": -270.9717712402344, "KL/rejected_KL_mean": -319.84710693359375, "KL/std": 141.6556396484375, "epoch": 0.6492146596858639, "fcm_dpo/beta": 0.003191425697878003, "fcm_dpo/delta": 0.008555184118449688, "fcm_dpo/margin": 97.75069427490234, "fcm_dpo/q_t": 0.4272443950176239, "grad_norm": 97.37775421142578, "learning_rate": 1.6650514271527465e-07, "logits/chosen": -0.8479313850402832, "logits/rejected": -0.8224340081214905, "logps/chosen": -501.2836608886719, "logps/ref_chosen": -279.1872863769531, "logps/ref_rejected": -261.8279724121094, "logps/rejected": -581.675048828125, "loss": 4.6234, "margin_dpo/margin_mean": 97.75069427490234, "margin_dpo/margin_std": 146.51434326171875, "step": 310 }, { "KL/chosen_KL_mean": -227.7700958251953, "KL/mean": -280.1990661621094, "KL/rejected_KL_mean": -332.6280212402344, "KL/std": 150.89422607421875, "epoch": 0.6513089005235602, "fcm_dpo/beta": 0.0031548929400742054, "fcm_dpo/delta": -0.02223263308405876, "fcm_dpo/margin": 104.85791015625, "fcm_dpo/q_t": 0.42464131116867065, "grad_norm": 55.97704315185547, "learning_rate": 1.647817538357072e-07, "logits/chosen": -0.8462434411048889, "logits/rejected": -0.8253843784332275, "logps/chosen": -499.1682434082031, "logps/ref_chosen": -271.39813232421875, "logps/ref_rejected": -266.12701416015625, "logps/rejected": -598.7550659179688, "loss": 4.6293, "margin_dpo/margin_mean": 104.85791015625, "margin_dpo/margin_std": 171.730224609375, "step": 311 }, { "KL/chosen_KL_mean": -225.2976837158203, "KL/mean": -271.5206604003906, "KL/rejected_KL_mean": -317.74365234375, "KL/std": 148.50408935546875, "epoch": 0.6534031413612565, "fcm_dpo/beta": 0.0031063989736139774, "fcm_dpo/delta": -0.02017890475690365, "fcm_dpo/margin": 92.44593811035156, "fcm_dpo/q_t": 0.4358768165111542, "grad_norm": 69.38884735107422, "learning_rate": 1.6306293495205755e-07, "logits/chosen": -0.8498347997665405, "logits/rejected": -0.8207939863204956, "logps/chosen": -507.6827087402344, "logps/ref_chosen": -282.3850402832031, "logps/ref_rejected": -246.35389709472656, "logps/rejected": -564.0975341796875, "loss": 4.8437, "margin_dpo/margin_mean": 92.44593048095703, "margin_dpo/margin_std": 191.8460693359375, "step": 312 }, { "KL/chosen_KL_mean": -228.06698608398438, "KL/mean": -272.4660949707031, "KL/rejected_KL_mean": -316.86517333984375, "KL/std": 152.6967315673828, "epoch": 0.6554973821989529, "fcm_dpo/beta": 0.0031254414934664965, "fcm_dpo/delta": -0.0015552723780274391, "fcm_dpo/margin": 88.7981948852539, "fcm_dpo/q_t": 0.43964749574661255, "grad_norm": 53.70850372314453, "learning_rate": 1.6134877823936607e-07, "logits/chosen": -0.894835352897644, "logits/rejected": -0.8788830637931824, "logps/chosen": -531.6978759765625, "logps/ref_chosen": -303.630859375, "logps/ref_rejected": -273.1156921386719, "logps/rejected": -589.9808349609375, "loss": 4.844, "margin_dpo/margin_mean": 88.7981948852539, "margin_dpo/margin_std": 178.9306640625, "step": 313 }, { "KL/chosen_KL_mean": -218.431640625, "KL/mean": -264.6053466796875, "KL/rejected_KL_mean": -310.779052734375, "KL/std": 135.10806274414062, "epoch": 0.6575916230366492, "fcm_dpo/beta": 0.0031253828201442957, "fcm_dpo/delta": 0.024184562265872955, "fcm_dpo/margin": 92.347412109375, "fcm_dpo/q_t": 0.43235987424850464, "grad_norm": 63.3831901550293, "learning_rate": 1.5963937562265522e-07, "logits/chosen": -0.8976191282272339, "logits/rejected": -0.8891264200210571, "logps/chosen": -520.7359619140625, "logps/ref_chosen": -302.3042907714844, "logps/ref_rejected": -273.6416015625, "logps/rejected": -584.4207153320312, "loss": 4.7032, "margin_dpo/margin_mean": 92.34739685058594, "margin_dpo/margin_std": 150.6728057861328, "step": 314 }, { "KL/chosen_KL_mean": -211.95986938476562, "KL/mean": -268.8974609375, "KL/rejected_KL_mean": -325.8350524902344, "KL/std": 141.79452514648438, "epoch": 0.6596858638743456, "fcm_dpo/beta": 0.0031580040231347084, "fcm_dpo/delta": -0.00586454663425684, "fcm_dpo/margin": 113.87518310546875, "fcm_dpo/q_t": 0.4168047308921814, "grad_norm": 71.96776580810547, "learning_rate": 1.5793481877199943e-07, "logits/chosen": -0.8908346891403198, "logits/rejected": -0.8735600113868713, "logps/chosen": -514.6891479492188, "logps/ref_chosen": -302.729248046875, "logps/ref_rejected": -270.26910400390625, "logps/rejected": -596.1041259765625, "loss": 4.4891, "margin_dpo/margin_mean": 113.87518310546875, "margin_dpo/margin_std": 158.78358459472656, "step": 315 }, { "KL/chosen_KL_mean": -222.13168334960938, "KL/mean": -270.41119384765625, "KL/rejected_KL_mean": -318.69073486328125, "KL/std": 140.0904541015625, "epoch": 0.6617801047120419, "fcm_dpo/beta": 0.003170891897752881, "fcm_dpo/delta": 0.0005847662687301636, "fcm_dpo/margin": 96.55903625488281, "fcm_dpo/q_t": 0.4291931390762329, "grad_norm": 62.24413299560547, "learning_rate": 1.562351990976095e-07, "logits/chosen": -0.9054620265960693, "logits/rejected": -0.8874289393424988, "logps/chosen": -532.702392578125, "logps/ref_chosen": -310.5706481933594, "logps/ref_rejected": -272.9354553222656, "logps/rejected": -591.626220703125, "loss": 4.6598, "margin_dpo/margin_mean": 96.55903625488281, "margin_dpo/margin_std": 156.25230407714844, "step": 316 }, { "KL/chosen_KL_mean": -193.6438446044922, "KL/mean": -238.2861328125, "KL/rejected_KL_mean": -282.9284362792969, "KL/std": 125.384033203125, "epoch": 0.6638743455497382, "fcm_dpo/beta": 0.0031651100143790245, "fcm_dpo/delta": 0.015250356867909431, "fcm_dpo/margin": 89.2845458984375, "fcm_dpo/q_t": 0.43306419253349304, "grad_norm": 67.99295043945312, "learning_rate": 1.5454060774493065e-07, "logits/chosen": -0.8674439787864685, "logits/rejected": -0.8366461992263794, "logps/chosen": -447.5442199707031, "logps/ref_chosen": -253.90036010742188, "logps/ref_rejected": -218.74078369140625, "logps/rejected": -501.669189453125, "loss": 4.68, "margin_dpo/margin_mean": 89.2845458984375, "margin_dpo/margin_std": 131.8083038330078, "step": 317 }, { "KL/chosen_KL_mean": -200.83892822265625, "KL/mean": -259.72735595703125, "KL/rejected_KL_mean": -318.61578369140625, "KL/std": 138.72280883789062, "epoch": 0.6659685863874345, "fcm_dpo/beta": 0.0031195408664643764, "fcm_dpo/delta": -0.029568474739789963, "fcm_dpo/margin": 117.77685546875, "fcm_dpo/q_t": 0.41461455821990967, "grad_norm": 64.10457611083984, "learning_rate": 1.5285113558975427e-07, "logits/chosen": -0.9008802771568298, "logits/rejected": -0.8715929388999939, "logps/chosen": -471.66180419921875, "logps/ref_chosen": -270.8228759765625, "logps/ref_rejected": -255.30972290039062, "logps/rejected": -573.925537109375, "loss": 4.4471, "margin_dpo/margin_mean": 117.77685546875, "margin_dpo/margin_std": 154.478515625, "step": 318 }, { "KL/chosen_KL_mean": -211.363037109375, "KL/mean": -264.632568359375, "KL/rejected_KL_mean": -317.902099609375, "KL/std": 134.839599609375, "epoch": 0.6680628272251309, "fcm_dpo/beta": 0.0031336878892034292, "fcm_dpo/delta": 0.01707865111529827, "fcm_dpo/margin": 106.53904724121094, "fcm_dpo/q_t": 0.42066890001296997, "grad_norm": 84.68042755126953, "learning_rate": 1.5116687323334464e-07, "logits/chosen": -0.8805448412895203, "logits/rejected": -0.8484068512916565, "logps/chosen": -512.3659057617188, "logps/ref_chosen": -301.0028076171875, "logps/ref_rejected": -242.39002990722656, "logps/rejected": -560.2921142578125, "loss": 4.506, "margin_dpo/margin_mean": 106.53904724121094, "margin_dpo/margin_std": 134.68338012695312, "step": 319 }, { "KL/chosen_KL_mean": -210.13380432128906, "KL/mean": -259.4587707519531, "KL/rejected_KL_mean": -308.78369140625, "KL/std": 143.7012176513672, "epoch": 0.6701570680628273, "fcm_dpo/beta": 0.0031071542762219906, "fcm_dpo/delta": -0.02263573370873928, "fcm_dpo/margin": 98.64984893798828, "fcm_dpo/q_t": 0.4299119710922241, "grad_norm": 75.7774658203125, "learning_rate": 1.4948791099758052e-07, "logits/chosen": -0.8548473715782166, "logits/rejected": -0.8449010848999023, "logps/chosen": -513.7564086914062, "logps/ref_chosen": -303.6225891113281, "logps/ref_rejected": -280.85174560546875, "logps/rejected": -589.6353759765625, "loss": 4.6904, "margin_dpo/margin_mean": 98.64985656738281, "margin_dpo/margin_std": 171.7944793701172, "step": 320 }, { "KL/chosen_KL_mean": -231.89138793945312, "KL/mean": -268.456787109375, "KL/rejected_KL_mean": -305.02215576171875, "KL/std": 141.75418090820312, "epoch": 0.6722513089005235, "fcm_dpo/beta": 0.0031160078942775726, "fcm_dpo/delta": 0.01728174462914467, "fcm_dpo/margin": 73.13074493408203, "fcm_dpo/q_t": 0.4479866623878479, "grad_norm": 60.38873291015625, "learning_rate": 1.478143389201113e-07, "logits/chosen": -0.865630567073822, "logits/rejected": -0.8288436532020569, "logps/chosen": -520.877197265625, "logps/ref_chosen": -288.98583984375, "logps/ref_rejected": -241.1822052001953, "logps/rejected": -546.204345703125, "loss": 4.9345, "margin_dpo/margin_mean": 73.13074493408203, "margin_dpo/margin_std": 156.70712280273438, "step": 321 }, { "KL/chosen_KL_mean": -239.51849365234375, "KL/mean": -283.0441589355469, "KL/rejected_KL_mean": -326.56982421875, "KL/std": 145.20364379882812, "epoch": 0.6743455497382199, "fcm_dpo/beta": 0.0031907029915601015, "fcm_dpo/delta": 0.04024318605661392, "fcm_dpo/margin": 87.05138397216797, "fcm_dpo/q_t": 0.4357774257659912, "grad_norm": 65.08485412597656, "learning_rate": 1.461462467495284e-07, "logits/chosen": -0.930508017539978, "logits/rejected": -0.8933470845222473, "logps/chosen": -548.0618896484375, "logps/ref_chosen": -308.54345703125, "logps/ref_rejected": -269.7995910644531, "logps/rejected": -596.369384765625, "loss": 4.8048, "margin_dpo/margin_mean": 87.05138397216797, "margin_dpo/margin_std": 171.3778076171875, "step": 322 }, { "KL/chosen_KL_mean": -233.23052978515625, "KL/mean": -264.7674560546875, "KL/rejected_KL_mean": -296.30438232421875, "KL/std": 134.792236328125, "epoch": 0.6764397905759162, "fcm_dpo/beta": 0.003232902381569147, "fcm_dpo/delta": -0.0018465555040165782, "fcm_dpo/margin": 63.07388687133789, "fcm_dpo/q_t": 0.4540364444255829, "grad_norm": 92.90402221679688, "learning_rate": 1.4448372394055246e-07, "logits/chosen": -0.8878564834594727, "logits/rejected": -0.8803995847702026, "logps/chosen": -515.72412109375, "logps/ref_chosen": -282.49365234375, "logps/ref_rejected": -227.7105255126953, "logps/rejected": -524.014892578125, "loss": 5.0631, "margin_dpo/margin_mean": 63.073883056640625, "margin_dpo/margin_std": 164.00965881347656, "step": 323 }, { "KL/chosen_KL_mean": -203.592041015625, "KL/mean": -258.118896484375, "KL/rejected_KL_mean": -312.64569091796875, "KL/std": 143.88446044921875, "epoch": 0.6785340314136126, "fcm_dpo/beta": 0.0031611351296305656, "fcm_dpo/delta": -0.020262351259589195, "fcm_dpo/margin": 109.05362701416016, "fcm_dpo/q_t": 0.41922521591186523, "grad_norm": 52.81629943847656, "learning_rate": 1.428268596492364e-07, "logits/chosen": -0.8115453124046326, "logits/rejected": -0.8045988082885742, "logps/chosen": -442.930419921875, "logps/ref_chosen": -239.33836364746094, "logps/ref_rejected": -230.53775024414062, "logps/rejected": -543.1834716796875, "loss": 4.5051, "margin_dpo/margin_mean": 109.05361938476562, "margin_dpo/margin_std": 144.5257568359375, "step": 324 }, { "KL/chosen_KL_mean": -234.3408203125, "KL/mean": -280.7336120605469, "KL/rejected_KL_mean": -327.1264343261719, "KL/std": 150.88209533691406, "epoch": 0.680628272251309, "fcm_dpo/beta": 0.003149219322949648, "fcm_dpo/delta": -0.012197908945381641, "fcm_dpo/margin": 92.78559875488281, "fcm_dpo/q_t": 0.4342680275440216, "grad_norm": 94.48786163330078, "learning_rate": 1.4117574272818386e-07, "logits/chosen": -0.8268110156059265, "logits/rejected": -0.8105596303939819, "logps/chosen": -514.9697875976562, "logps/ref_chosen": -280.62896728515625, "logps/ref_rejected": -270.5085754394531, "logps/rejected": -597.6349487304688, "loss": 4.8091, "margin_dpo/margin_mean": 92.78559875488281, "margin_dpo/margin_std": 185.6398468017578, "step": 325 }, { "KL/chosen_KL_mean": -214.26693725585938, "KL/mean": -265.3741455078125, "KL/rejected_KL_mean": -316.4813232421875, "KL/std": 142.83740234375, "epoch": 0.6827225130890052, "fcm_dpo/beta": 0.0032117185182869434, "fcm_dpo/delta": 0.028104102239012718, "fcm_dpo/margin": 102.21440124511719, "fcm_dpo/q_t": 0.4247254431247711, "grad_norm": 64.72186279296875, "learning_rate": 1.3953046172178413e-07, "logits/chosen": -0.945293664932251, "logits/rejected": -0.9358657598495483, "logps/chosen": -455.2540588378906, "logps/ref_chosen": -240.9871368408203, "logps/ref_rejected": -261.0238342285156, "logps/rejected": -577.5051879882812, "loss": 4.6272, "margin_dpo/margin_mean": 102.21440124511719, "margin_dpo/margin_std": 167.8172149658203, "step": 326 }, { "KL/chosen_KL_mean": -207.91552734375, "KL/mean": -264.1645812988281, "KL/rejected_KL_mean": -320.4136657714844, "KL/std": 134.82386779785156, "epoch": 0.6848167539267016, "fcm_dpo/beta": 0.0031878852751106024, "fcm_dpo/delta": -0.03201831877231598, "fcm_dpo/margin": 112.49812316894531, "fcm_dpo/q_t": 0.4176490604877472, "grad_norm": 94.11993408203125, "learning_rate": 1.3789110486146468e-07, "logits/chosen": -0.8883708119392395, "logits/rejected": -0.8645241856575012, "logps/chosen": -487.435546875, "logps/ref_chosen": -279.52001953125, "logps/ref_rejected": -269.51824951171875, "logps/rejected": -589.931884765625, "loss": 4.5059, "margin_dpo/margin_mean": 112.49813079833984, "margin_dpo/margin_std": 157.23403930664062, "step": 327 }, { "KL/chosen_KL_mean": -229.1092529296875, "KL/mean": -280.052490234375, "KL/rejected_KL_mean": -330.9957275390625, "KL/std": 139.11770629882812, "epoch": 0.6869109947643979, "fcm_dpo/beta": 0.003118137363344431, "fcm_dpo/delta": -0.0110612902790308, "fcm_dpo/margin": 101.88642883300781, "fcm_dpo/q_t": 0.42611968517303467, "grad_norm": 63.17850112915039, "learning_rate": 1.362577600609588e-07, "logits/chosen": -0.8533894419670105, "logits/rejected": -0.8511139154434204, "logps/chosen": -530.1427612304688, "logps/ref_chosen": -301.033447265625, "logps/ref_rejected": -284.2101135253906, "logps/rejected": -615.205810546875, "loss": 4.6045, "margin_dpo/margin_mean": 101.88642883300781, "margin_dpo/margin_std": 151.49839782714844, "step": 328 }, { "KL/chosen_KL_mean": -243.52928161621094, "KL/mean": -300.4892883300781, "KL/rejected_KL_mean": -357.4493103027344, "KL/std": 178.26580810546875, "epoch": 0.6890052356020943, "fcm_dpo/beta": 0.0030765547417104244, "fcm_dpo/delta": -0.023377757519483566, "fcm_dpo/margin": 113.92003631591797, "fcm_dpo/q_t": 0.4244258999824524, "grad_norm": 67.04179382324219, "learning_rate": 1.3463051491159093e-07, "logits/chosen": -0.8606047630310059, "logits/rejected": -0.8284364938735962, "logps/chosen": -563.5181274414062, "logps/ref_chosen": -319.9888610839844, "logps/ref_rejected": -307.5588684082031, "logps/rejected": -665.0081787109375, "loss": 4.6515, "margin_dpo/margin_mean": 113.92002868652344, "margin_dpo/margin_std": 201.48175048828125, "step": 329 }, { "KL/chosen_KL_mean": -238.3928985595703, "KL/mean": -291.460205078125, "KL/rejected_KL_mean": -344.5274963378906, "KL/std": 156.5318603515625, "epoch": 0.6910994764397905, "fcm_dpo/beta": 0.0030223994981497526, "fcm_dpo/delta": -0.023758048191666603, "fcm_dpo/margin": 106.13461303710938, "fcm_dpo/q_t": 0.42642486095428467, "grad_norm": 73.99193572998047, "learning_rate": 1.3300945667758012e-07, "logits/chosen": -0.857183039188385, "logits/rejected": -0.8630150556564331, "logps/chosen": -539.5076904296875, "logps/ref_chosen": -301.11474609375, "logps/ref_rejected": -299.673095703125, "logps/rejected": -644.2005615234375, "loss": 4.6121, "margin_dpo/margin_mean": 106.13459777832031, "margin_dpo/margin_std": 160.67657470703125, "step": 330 }, { "KL/chosen_KL_mean": -226.96383666992188, "KL/mean": -274.009765625, "KL/rejected_KL_mean": -321.0557556152344, "KL/std": 153.6221466064453, "epoch": 0.6931937172774869, "fcm_dpo/beta": 0.0030192735139280558, "fcm_dpo/delta": 0.01395251415669918, "fcm_dpo/margin": 94.09191131591797, "fcm_dpo/q_t": 0.4358104467391968, "grad_norm": 80.74335479736328, "learning_rate": 1.3139467229135998e-07, "logits/chosen": -0.8754410743713379, "logits/rejected": -0.8539817333221436, "logps/chosen": -504.5553283691406, "logps/ref_chosen": -277.59149169921875, "logps/ref_rejected": -256.025634765625, "logps/rejected": -577.0813598632812, "loss": 4.7924, "margin_dpo/margin_mean": 94.09191131591797, "margin_dpo/margin_std": 185.90357971191406, "step": 331 }, { "KL/chosen_KL_mean": -240.19212341308594, "KL/mean": -288.86236572265625, "KL/rejected_KL_mean": -337.5325927734375, "KL/std": 150.13291931152344, "epoch": 0.6952879581151833, "fcm_dpo/beta": 0.003028081264346838, "fcm_dpo/delta": -0.003697561100125313, "fcm_dpo/margin": 97.34046936035156, "fcm_dpo/q_t": 0.4329964518547058, "grad_norm": 66.77661895751953, "learning_rate": 1.2978624834891626e-07, "logits/chosen": -0.8756115436553955, "logits/rejected": -0.8486796617507935, "logps/chosen": -510.16583251953125, "logps/ref_chosen": -269.97369384765625, "logps/ref_rejected": -235.03164672851562, "logps/rejected": -572.564208984375, "loss": 4.7937, "margin_dpo/margin_mean": 97.34046936035156, "margin_dpo/margin_std": 192.24783325195312, "step": 332 }, { "KL/chosen_KL_mean": -243.05345153808594, "KL/mean": -284.6350402832031, "KL/rejected_KL_mean": -326.21661376953125, "KL/std": 145.56326293945312, "epoch": 0.6973821989528796, "fcm_dpo/beta": 0.0030364254489541054, "fcm_dpo/delta": 0.02071891352534294, "fcm_dpo/margin": 83.16316986083984, "fcm_dpo/q_t": 0.4418914318084717, "grad_norm": 94.2875747680664, "learning_rate": 1.281842711051438e-07, "logits/chosen": -0.9469788074493408, "logits/rejected": -0.9081500172615051, "logps/chosen": -539.8164672851562, "logps/ref_chosen": -296.76300048828125, "logps/ref_rejected": -265.97991943359375, "logps/rejected": -592.1965942382812, "loss": 4.8583, "margin_dpo/margin_mean": 83.16317749023438, "margin_dpo/margin_std": 168.8401641845703, "step": 333 }, { "KL/chosen_KL_mean": -237.4409942626953, "KL/mean": -285.7542724609375, "KL/rejected_KL_mean": -334.0675354003906, "KL/std": 146.4021453857422, "epoch": 0.6994764397905759, "fcm_dpo/beta": 0.0030563974287360907, "fcm_dpo/delta": -0.020071666687726974, "fcm_dpo/margin": 96.62651062011719, "fcm_dpo/q_t": 0.43138545751571655, "grad_norm": 58.2845573425293, "learning_rate": 1.2658882646922033e-07, "logits/chosen": -0.8468398451805115, "logits/rejected": -0.817967414855957, "logps/chosen": -538.4777221679688, "logps/ref_chosen": -301.0367431640625, "logps/ref_rejected": -268.87652587890625, "logps/rejected": -602.9440307617188, "loss": 4.7228, "margin_dpo/margin_mean": 96.62650299072266, "margin_dpo/margin_std": 164.7143096923828, "step": 334 }, { "KL/chosen_KL_mean": -256.3838806152344, "KL/mean": -304.9598388671875, "KL/rejected_KL_mean": -353.535888671875, "KL/std": 136.20851135253906, "epoch": 0.7015706806282722, "fcm_dpo/beta": 0.0029724808409810066, "fcm_dpo/delta": -0.027509108185768127, "fcm_dpo/margin": 97.15202331542969, "fcm_dpo/q_t": 0.43334853649139404, "grad_norm": 62.15879821777344, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.8522113561630249, "logits/rejected": -0.8463060259819031, "logps/chosen": -532.5166625976562, "logps/ref_chosen": -276.13275146484375, "logps/ref_rejected": -243.44203186035156, "logps/rejected": -596.9779052734375, "loss": 4.7403, "margin_dpo/margin_mean": 97.15201568603516, "margin_dpo/margin_std": 172.3384246826172, "step": 335 }, { "KL/chosen_KL_mean": -251.550048828125, "KL/mean": -300.40789794921875, "KL/rejected_KL_mean": -349.2657165527344, "KL/std": 172.0433349609375, "epoch": 0.7036649214659686, "fcm_dpo/beta": 0.0029860164504498243, "fcm_dpo/delta": 0.013530546799302101, "fcm_dpo/margin": 97.7156753540039, "fcm_dpo/q_t": 0.4363878071308136, "grad_norm": 74.54422760009766, "learning_rate": 1.2341787690142435e-07, "logits/chosen": -0.8671432733535767, "logits/rejected": -0.8070932030677795, "logps/chosen": -497.81268310546875, "logps/ref_chosen": -246.2626495361328, "logps/ref_rejected": -261.0617980957031, "logps/rejected": -610.3275756835938, "loss": 4.7949, "margin_dpo/margin_mean": 97.7156753540039, "margin_dpo/margin_std": 191.96246337890625, "step": 336 }, { "KL/chosen_KL_mean": -237.27195739746094, "KL/mean": -291.9794006347656, "KL/rejected_KL_mean": -346.6868896484375, "KL/std": 143.3297882080078, "epoch": 0.7057591623036649, "fcm_dpo/beta": 0.0029976258520036936, "fcm_dpo/delta": -0.021170055493712425, "fcm_dpo/margin": 109.41490936279297, "fcm_dpo/q_t": 0.42369288206100464, "grad_norm": 77.50701904296875, "learning_rate": 1.2184254201795363e-07, "logits/chosen": -0.8679603338241577, "logits/rejected": -0.8385904431343079, "logps/chosen": -504.2657470703125, "logps/ref_chosen": -266.9937744140625, "logps/ref_rejected": -253.015625, "logps/rejected": -599.7025146484375, "loss": 4.5946, "margin_dpo/margin_mean": 109.41490936279297, "margin_dpo/margin_std": 163.00048828125, "step": 337 }, { "KL/chosen_KL_mean": -235.5108642578125, "KL/mean": -288.2745056152344, "KL/rejected_KL_mean": -341.03814697265625, "KL/std": 152.8268280029297, "epoch": 0.7078534031413612, "fcm_dpo/beta": 0.0029197270050644875, "fcm_dpo/delta": -0.005821624770760536, "fcm_dpo/margin": 105.52726745605469, "fcm_dpo/q_t": 0.4291449189186096, "grad_norm": 75.93712615966797, "learning_rate": 1.202740798300168e-07, "logits/chosen": -0.8869338631629944, "logits/rejected": -0.8708446025848389, "logps/chosen": -512.1034545898438, "logps/ref_chosen": -276.5925598144531, "logps/ref_rejected": -233.979248046875, "logps/rejected": -575.017333984375, "loss": 4.6638, "margin_dpo/margin_mean": 105.52726745605469, "margin_dpo/margin_std": 172.4578094482422, "step": 338 }, { "KL/chosen_KL_mean": -237.4051513671875, "KL/mean": -297.82196044921875, "KL/rejected_KL_mean": -358.23876953125, "KL/std": 157.0062255859375, "epoch": 0.7099476439790576, "fcm_dpo/beta": 0.0029296469874680042, "fcm_dpo/delta": -0.021757004782557487, "fcm_dpo/margin": 120.8335952758789, "fcm_dpo/q_t": 0.41886672377586365, "grad_norm": 64.22445678710938, "learning_rate": 1.1871257444948096e-07, "logits/chosen": -0.9070875644683838, "logits/rejected": -0.8940801620483398, "logps/chosen": -540.9329223632812, "logps/ref_chosen": -303.5277404785156, "logps/ref_rejected": -283.11676025390625, "logps/rejected": -641.35546875, "loss": 4.5489, "margin_dpo/margin_mean": 120.83360290527344, "margin_dpo/margin_std": 184.66961669921875, "step": 339 }, { "KL/chosen_KL_mean": -246.29815673828125, "KL/mean": -293.8956604003906, "KL/rejected_KL_mean": -341.4931640625, "KL/std": 165.5948486328125, "epoch": 0.7120418848167539, "fcm_dpo/beta": 0.0028934727888554335, "fcm_dpo/delta": 0.01472179964184761, "fcm_dpo/margin": 95.19498443603516, "fcm_dpo/q_t": 0.43556487560272217, "grad_norm": 89.77741241455078, "learning_rate": 1.1715810961514072e-07, "logits/chosen": -0.8359464406967163, "logits/rejected": -0.8298177719116211, "logps/chosen": -507.82391357421875, "logps/ref_chosen": -261.5257568359375, "logps/ref_rejected": -259.39862060546875, "logps/rejected": -600.8917236328125, "loss": 4.88, "margin_dpo/margin_mean": 95.19498443603516, "margin_dpo/margin_std": 201.7469482421875, "step": 340 }, { "KL/chosen_KL_mean": -288.7161865234375, "KL/mean": -333.5431213378906, "KL/rejected_KL_mean": -378.3700256347656, "KL/std": 184.27386474609375, "epoch": 0.7141361256544503, "fcm_dpo/beta": 0.002914420096203685, "fcm_dpo/delta": -0.00907097663730383, "fcm_dpo/margin": 89.65385437011719, "fcm_dpo/q_t": 0.4414941966533661, "grad_norm": 75.35196685791016, "learning_rate": 1.1561076868822755e-07, "logits/chosen": -0.8593753576278687, "logits/rejected": -0.8326888680458069, "logps/chosen": -604.6196899414062, "logps/ref_chosen": -315.903564453125, "logps/ref_rejected": -308.02392578125, "logps/rejected": -686.3939819335938, "loss": 4.9406, "margin_dpo/margin_mean": 89.65385437011719, "margin_dpo/margin_std": 208.85565185546875, "step": 341 }, { "KL/chosen_KL_mean": -238.28636169433594, "KL/mean": -286.4154052734375, "KL/rejected_KL_mean": -334.54443359375, "KL/std": 143.09034729003906, "epoch": 0.7162303664921466, "fcm_dpo/beta": 0.002888055983930826, "fcm_dpo/delta": -0.005859066266566515, "fcm_dpo/margin": 96.25808715820312, "fcm_dpo/q_t": 0.4352652430534363, "grad_norm": 82.94450378417969, "learning_rate": 1.1407063464793965e-07, "logits/chosen": -0.8524309992790222, "logits/rejected": -0.8473560810089111, "logps/chosen": -507.4649963378906, "logps/ref_chosen": -269.17864990234375, "logps/ref_rejected": -260.8977355957031, "logps/rejected": -595.442138671875, "loss": 4.7214, "margin_dpo/margin_mean": 96.25808715820312, "margin_dpo/margin_std": 158.40966796875, "step": 342 }, { "KL/chosen_KL_mean": -251.1302032470703, "KL/mean": -289.5614318847656, "KL/rejected_KL_mean": -327.99267578125, "KL/std": 144.47100830078125, "epoch": 0.7183246073298429, "fcm_dpo/beta": 0.0028825635090470314, "fcm_dpo/delta": 0.007316844072192907, "fcm_dpo/margin": 76.86248779296875, "fcm_dpo/q_t": 0.4476861357688904, "grad_norm": 78.4572525024414, "learning_rate": 1.125377900869913e-07, "logits/chosen": -0.8445292711257935, "logits/rejected": -0.8152074813842773, "logps/chosen": -561.8502197265625, "logps/ref_chosen": -310.719970703125, "logps/ref_rejected": -263.5224914550781, "logps/rejected": -591.51513671875, "loss": 4.9124, "margin_dpo/margin_mean": 76.86248779296875, "margin_dpo/margin_std": 157.17347717285156, "step": 343 }, { "KL/chosen_KL_mean": -247.63592529296875, "KL/mean": -293.60247802734375, "KL/rejected_KL_mean": -339.5689697265625, "KL/std": 140.92300415039062, "epoch": 0.7204188481675393, "fcm_dpo/beta": 0.0028950828127563, "fcm_dpo/delta": -0.003131479024887085, "fcm_dpo/margin": 91.93303680419922, "fcm_dpo/q_t": 0.4373406767845154, "grad_norm": 72.97909545898438, "learning_rate": 1.110123172071844e-07, "logits/chosen": -0.8511172533035278, "logits/rejected": -0.8328704237937927, "logps/chosen": -549.4358520507812, "logps/ref_chosen": -301.7999267578125, "logps/ref_rejected": -257.9061584472656, "logps/rejected": -597.4751586914062, "loss": 4.7968, "margin_dpo/margin_mean": 91.93302917480469, "margin_dpo/margin_std": 167.35589599609375, "step": 344 }, { "KL/chosen_KL_mean": -232.2271728515625, "KL/mean": -273.35272216796875, "KL/rejected_KL_mean": -314.4782409667969, "KL/std": 139.15582275390625, "epoch": 0.7225130890052356, "fcm_dpo/beta": 0.0028755811508744955, "fcm_dpo/delta": -0.004357065074145794, "fcm_dpo/margin": 82.25109100341797, "fcm_dpo/q_t": 0.444185733795166, "grad_norm": 76.50686645507812, "learning_rate": 1.09494297815e-07, "logits/chosen": -0.8535346984863281, "logits/rejected": -0.8539408445358276, "logps/chosen": -515.24560546875, "logps/ref_chosen": -283.0184326171875, "logps/ref_rejected": -266.8457336425781, "logps/rejected": -581.323974609375, "loss": 4.8509, "margin_dpo/margin_mean": 82.25108337402344, "margin_dpo/margin_std": 142.70669555664062, "step": 345 }, { "KL/chosen_KL_mean": -243.66827392578125, "KL/mean": -296.9436950683594, "KL/rejected_KL_mean": -350.21917724609375, "KL/std": 160.29945373535156, "epoch": 0.724607329842932, "fcm_dpo/beta": 0.0028548564296215773, "fcm_dpo/delta": -0.013966667465865612, "fcm_dpo/margin": 106.5509033203125, "fcm_dpo/q_t": 0.4294491708278656, "grad_norm": 50.279972076416016, "learning_rate": 1.0798381331721107e-07, "logits/chosen": -0.9294825792312622, "logits/rejected": -0.87758469581604, "logps/chosen": -512.1094970703125, "logps/ref_chosen": -268.44122314453125, "logps/ref_rejected": -227.8225860595703, "logps/rejected": -578.041748046875, "loss": 4.666, "margin_dpo/margin_mean": 106.5509033203125, "margin_dpo/margin_std": 163.5757293701172, "step": 346 }, { "KL/chosen_KL_mean": -224.2758331298828, "KL/mean": -267.6025390625, "KL/rejected_KL_mean": -310.92919921875, "KL/std": 138.48739624023438, "epoch": 0.7267015706806282, "fcm_dpo/beta": 0.002854297636076808, "fcm_dpo/delta": 0.007548038847744465, "fcm_dpo/margin": 86.65332794189453, "fcm_dpo/q_t": 0.4415615499019623, "grad_norm": 83.579833984375, "learning_rate": 1.0648094471651722e-07, "logits/chosen": -0.7760555148124695, "logits/rejected": -0.8024675250053406, "logps/chosen": -497.9794006347656, "logps/ref_chosen": -273.70355224609375, "logps/ref_rejected": -243.65521240234375, "logps/rejected": -554.5843505859375, "loss": 4.8312, "margin_dpo/margin_mean": 86.65332794189453, "margin_dpo/margin_std": 157.4734344482422, "step": 347 }, { "KL/chosen_KL_mean": -220.09567260742188, "KL/mean": -258.8450622558594, "KL/rejected_KL_mean": -297.594482421875, "KL/std": 147.97817993164062, "epoch": 0.7287958115183246, "fcm_dpo/beta": 0.0028738253749907017, "fcm_dpo/delta": 0.0110637117177248, "fcm_dpo/margin": 77.49883270263672, "fcm_dpo/q_t": 0.4496265649795532, "grad_norm": 75.31076049804688, "learning_rate": 1.0498577260720048e-07, "logits/chosen": -0.892849326133728, "logits/rejected": -0.8771178722381592, "logps/chosen": -505.7370910644531, "logps/ref_chosen": -285.64141845703125, "logps/ref_rejected": -265.6270446777344, "logps/rejected": -563.2215576171875, "loss": 4.9376, "margin_dpo/margin_mean": 77.49882507324219, "margin_dpo/margin_std": 166.94793701171875, "step": 348 }, { "KL/chosen_KL_mean": -220.64576721191406, "KL/mean": -279.0647277832031, "KL/rejected_KL_mean": -337.48370361328125, "KL/std": 160.11648559570312, "epoch": 0.7308900523560209, "fcm_dpo/beta": 0.0028767730109393597, "fcm_dpo/delta": -0.013212434016168118, "fcm_dpo/margin": 116.83795928955078, "fcm_dpo/q_t": 0.42093393206596375, "grad_norm": 66.55834197998047, "learning_rate": 1.0349837717080347e-07, "logits/chosen": -0.8378231525421143, "logits/rejected": -0.8238610625267029, "logps/chosen": -548.9632568359375, "logps/ref_chosen": -328.3175048828125, "logps/ref_rejected": -292.37872314453125, "logps/rejected": -629.8624267578125, "loss": 4.6037, "margin_dpo/margin_mean": 116.83795928955078, "margin_dpo/margin_std": 184.72499084472656, "step": 349 }, { "KL/chosen_KL_mean": -237.5376434326172, "KL/mean": -285.01214599609375, "KL/rejected_KL_mean": -332.4866027832031, "KL/std": 148.0513916015625, "epoch": 0.7329842931937173, "fcm_dpo/beta": 0.0028601842932403088, "fcm_dpo/delta": 0.004661480896174908, "fcm_dpo/margin": 94.94898986816406, "fcm_dpo/q_t": 0.43736475706100464, "grad_norm": 80.1038818359375, "learning_rate": 1.0201883817182949e-07, "logits/chosen": -0.8289676904678345, "logits/rejected": -0.8410451412200928, "logps/chosen": -530.34228515625, "logps/ref_chosen": -292.8046569824219, "logps/ref_rejected": -250.35504150390625, "logps/rejected": -582.8416748046875, "loss": 4.7774, "margin_dpo/margin_mean": 94.94898986816406, "margin_dpo/margin_std": 173.54653930664062, "step": 350 }, { "KL/chosen_KL_mean": -223.345458984375, "KL/mean": -254.21771240234375, "KL/rejected_KL_mean": -285.0899658203125, "KL/std": 138.79812622070312, "epoch": 0.7350785340314137, "fcm_dpo/beta": 0.0028854927513748407, "fcm_dpo/delta": 0.013731059618294239, "fcm_dpo/margin": 61.744510650634766, "fcm_dpo/q_t": 0.4585198163986206, "grad_norm": 70.86201477050781, "learning_rate": 1.0054723495346482e-07, "logits/chosen": -0.8867701292037964, "logits/rejected": -0.8659510612487793, "logps/chosen": -535.2344970703125, "logps/ref_chosen": -311.8890380859375, "logps/ref_rejected": -263.59033203125, "logps/rejected": -548.6802978515625, "loss": 5.113, "margin_dpo/margin_mean": 61.744510650634766, "margin_dpo/margin_std": 169.5723876953125, "step": 351 }, { "KL/chosen_KL_mean": -210.1036376953125, "KL/mean": -268.3479309082031, "KL/rejected_KL_mean": -326.5922546386719, "KL/std": 149.0188446044922, "epoch": 0.7371727748691099, "fcm_dpo/beta": 0.0028885467909276485, "fcm_dpo/delta": -0.00935527216643095, "fcm_dpo/margin": 116.48863220214844, "fcm_dpo/q_t": 0.42103269696235657, "grad_norm": 57.36745834350586, "learning_rate": 9.908364643332398e-08, "logits/chosen": -0.8118300437927246, "logits/rejected": -0.7807732820510864, "logps/chosen": -465.0115051269531, "logps/ref_chosen": -254.9078826904297, "logps/ref_rejected": -257.1688232421875, "logps/rejected": -583.7610473632812, "loss": 4.5527, "margin_dpo/margin_mean": 116.48863983154297, "margin_dpo/margin_std": 165.00392150878906, "step": 352 }, { "KL/chosen_KL_mean": -229.11819458007812, "KL/mean": -279.8221740722656, "KL/rejected_KL_mean": -330.526123046875, "KL/std": 147.36636352539062, "epoch": 0.7392670157068063, "fcm_dpo/beta": 0.0028685785364359617, "fcm_dpo/delta": -0.013993466272950172, "fcm_dpo/margin": 101.40792846679688, "fcm_dpo/q_t": 0.4323953688144684, "grad_norm": 93.38015747070312, "learning_rate": 9.76281510992176e-08, "logits/chosen": -0.8352831602096558, "logits/rejected": -0.8298249840736389, "logps/chosen": -499.4942626953125, "logps/ref_chosen": -270.3760681152344, "logps/ref_rejected": -264.65234375, "logps/rejected": -595.178466796875, "loss": 4.7011, "margin_dpo/margin_mean": 101.40792846679688, "margin_dpo/margin_std": 164.58172607421875, "step": 353 }, { "KL/chosen_KL_mean": -226.212646484375, "KL/mean": -265.5102233886719, "KL/rejected_KL_mean": -304.8078308105469, "KL/std": 154.871826171875, "epoch": 0.7413612565445026, "fcm_dpo/beta": 0.002862285589799285, "fcm_dpo/delta": 0.012168960645794868, "fcm_dpo/margin": 78.59518432617188, "fcm_dpo/q_t": 0.4484597146511078, "grad_norm": 77.23882293701172, "learning_rate": 9.618082700494318e-08, "logits/chosen": -0.8314879536628723, "logits/rejected": -0.8639715909957886, "logps/chosen": -483.8612365722656, "logps/ref_chosen": -257.6485595703125, "logps/ref_rejected": -246.94203186035156, "logps/rejected": -551.7498779296875, "loss": 4.9439, "margin_dpo/margin_mean": 78.59518432617188, "margin_dpo/margin_std": 171.47744750976562, "step": 354 }, { "KL/chosen_KL_mean": -228.62823486328125, "KL/mean": -280.390625, "KL/rejected_KL_mean": -332.1530456542969, "KL/std": 161.08724975585938, "epoch": 0.743455497382199, "fcm_dpo/beta": 0.0028833611868321896, "fcm_dpo/delta": 0.009556437842547894, "fcm_dpo/margin": 103.52484893798828, "fcm_dpo/q_t": 0.431077778339386, "grad_norm": 55.02450942993164, "learning_rate": 9.474175176609956e-08, "logits/chosen": -0.8792758584022522, "logits/rejected": -0.8736676573753357, "logps/chosen": -521.9815673828125, "logps/ref_chosen": -293.35333251953125, "logps/ref_rejected": -275.6051940917969, "logps/rejected": -607.7582397460938, "loss": 4.7879, "margin_dpo/margin_mean": 103.52484893798828, "margin_dpo/margin_std": 198.6239013671875, "step": 355 }, { "KL/chosen_KL_mean": -216.16531372070312, "KL/mean": -253.62515258789062, "KL/rejected_KL_mean": -291.08502197265625, "KL/std": 134.0445556640625, "epoch": 0.7455497382198953, "fcm_dpo/beta": 0.0028942120261490345, "fcm_dpo/delta": -0.0024351300671696663, "fcm_dpo/margin": 74.91969299316406, "fcm_dpo/q_t": 0.44854995608329773, "grad_norm": 71.81739044189453, "learning_rate": 9.331100255592436e-08, "logits/chosen": -0.7871492505073547, "logits/rejected": -0.8126851320266724, "logps/chosen": -420.4208068847656, "logps/ref_chosen": -204.25550842285156, "logps/ref_rejected": -213.467529296875, "logps/rejected": -504.552490234375, "loss": 4.9034, "margin_dpo/margin_mean": 74.91969299316406, "margin_dpo/margin_std": 139.6249542236328, "step": 356 }, { "KL/chosen_KL_mean": -239.94329833984375, "KL/mean": -289.7812805175781, "KL/rejected_KL_mean": -339.6192932128906, "KL/std": 154.73379516601562, "epoch": 0.7476439790575916, "fcm_dpo/beta": 0.00291292741894722, "fcm_dpo/delta": 0.021629903465509415, "fcm_dpo/margin": 99.67597961425781, "fcm_dpo/q_t": 0.433463990688324, "grad_norm": 69.52693939208984, "learning_rate": 9.18886561011557e-08, "logits/chosen": -0.7813151478767395, "logits/rejected": -0.7780415415763855, "logps/chosen": -506.3138427734375, "logps/ref_chosen": -266.3705749511719, "logps/ref_rejected": -239.04490661621094, "logps/rejected": -578.6642456054688, "loss": 4.7521, "margin_dpo/margin_mean": 99.67597961425781, "margin_dpo/margin_std": 178.00946044921875, "step": 357 }, { "KL/chosen_KL_mean": -217.33197021484375, "KL/mean": -272.29388427734375, "KL/rejected_KL_mean": -327.2558288574219, "KL/std": 141.83389282226562, "epoch": 0.749738219895288, "fcm_dpo/beta": 0.00295689282938838, "fcm_dpo/delta": 0.014568077400326729, "fcm_dpo/margin": 109.92384338378906, "fcm_dpo/q_t": 0.42453300952911377, "grad_norm": 83.9714126586914, "learning_rate": 9.047478867791731e-08, "logits/chosen": -0.8699600696563721, "logits/rejected": -0.8453044891357422, "logps/chosen": -516.4794311523438, "logps/ref_chosen": -299.1474609375, "logps/ref_rejected": -257.2531433105469, "logps/rejected": -584.509033203125, "loss": 4.6161, "margin_dpo/margin_mean": 109.92383575439453, "margin_dpo/margin_std": 175.24615478515625, "step": 358 }, { "KL/chosen_KL_mean": -230.4601593017578, "KL/mean": -278.14129638671875, "KL/rejected_KL_mean": -325.82244873046875, "KL/std": 147.65829467773438, "epoch": 0.7518324607329843, "fcm_dpo/beta": 0.0029695353005081415, "fcm_dpo/delta": 0.0013146661221981049, "fcm_dpo/margin": 95.36227416992188, "fcm_dpo/q_t": 0.43455877900123596, "grad_norm": 70.06732177734375, "learning_rate": 8.906947610762825e-08, "logits/chosen": -0.8403257131576538, "logits/rejected": -0.8475351333618164, "logps/chosen": -533.4580078125, "logps/ref_chosen": -302.99786376953125, "logps/ref_rejected": -260.4137268066406, "logps/rejected": -586.2362060546875, "loss": 4.7681, "margin_dpo/margin_mean": 95.36227416992188, "margin_dpo/margin_std": 175.98709106445312, "step": 359 }, { "KL/chosen_KL_mean": -232.94993591308594, "KL/mean": -278.8365783691406, "KL/rejected_KL_mean": -324.7232360839844, "KL/std": 160.8883056640625, "epoch": 0.7539267015706806, "fcm_dpo/beta": 0.0029307494405657053, "fcm_dpo/delta": -0.03134762495756149, "fcm_dpo/margin": 91.77330780029297, "fcm_dpo/q_t": 0.43876707553863525, "grad_norm": 60.748844146728516, "learning_rate": 8.76727937529367e-08, "logits/chosen": -0.8615961074829102, "logits/rejected": -0.8414865136146545, "logps/chosen": -542.5614013671875, "logps/ref_chosen": -309.6114501953125, "logps/ref_rejected": -256.64031982421875, "logps/rejected": -581.363525390625, "loss": 4.8976, "margin_dpo/margin_mean": 91.77330780029297, "margin_dpo/margin_std": 189.3778076171875, "step": 360 }, { "KL/chosen_KL_mean": -196.89012145996094, "KL/mean": -256.0363464355469, "KL/rejected_KL_mean": -315.18255615234375, "KL/std": 154.51834106445312, "epoch": 0.7560209424083769, "fcm_dpo/beta": 0.0029320104513317347, "fcm_dpo/delta": 0.015726841986179352, "fcm_dpo/margin": 118.29248046875, "fcm_dpo/q_t": 0.421514093875885, "grad_norm": 64.89899444580078, "learning_rate": 8.628481651367875e-08, "logits/chosen": -0.803794264793396, "logits/rejected": -0.7853226065635681, "logps/chosen": -460.2698669433594, "logps/ref_chosen": -263.3797607421875, "logps/ref_rejected": -271.18157958984375, "logps/rejected": -586.3641357421875, "loss": 4.5926, "margin_dpo/margin_mean": 118.29248046875, "margin_dpo/margin_std": 193.5961456298828, "step": 361 }, { "KL/chosen_KL_mean": -219.24996948242188, "KL/mean": -266.0592956542969, "KL/rejected_KL_mean": -312.86865234375, "KL/std": 129.30587768554688, "epoch": 0.7581151832460733, "fcm_dpo/beta": 0.002948556561022997, "fcm_dpo/delta": 0.006594155449420214, "fcm_dpo/margin": 93.61869812011719, "fcm_dpo/q_t": 0.43606704473495483, "grad_norm": 72.42217254638672, "learning_rate": 8.490561882286135e-08, "logits/chosen": -0.8361051082611084, "logits/rejected": -0.81890469789505, "logps/chosen": -522.5083618164062, "logps/ref_chosen": -303.2583923339844, "logps/ref_rejected": -243.22891235351562, "logps/rejected": -556.0975341796875, "loss": 4.7161, "margin_dpo/margin_mean": 93.61870574951172, "margin_dpo/margin_std": 146.4253387451172, "step": 362 }, { "KL/chosen_KL_mean": -224.68502807617188, "KL/mean": -277.4530029296875, "KL/rejected_KL_mean": -330.220947265625, "KL/std": 156.96578979492188, "epoch": 0.7602094240837697, "fcm_dpo/beta": 0.002929537557065487, "fcm_dpo/delta": -0.03551473096013069, "fcm_dpo/margin": 105.53596496582031, "fcm_dpo/q_t": 0.42976054549217224, "grad_norm": 71.42375183105469, "learning_rate": 8.353527464267104e-08, "logits/chosen": -0.8412438035011292, "logits/rejected": -0.7956109642982483, "logps/chosen": -528.0322875976562, "logps/ref_chosen": -303.34722900390625, "logps/ref_rejected": -262.05419921875, "logps/rejected": -592.2752075195312, "loss": 4.7065, "margin_dpo/margin_mean": 105.53596496582031, "margin_dpo/margin_std": 184.98667907714844, "step": 363 }, { "KL/chosen_KL_mean": -254.75527954101562, "KL/mean": -303.8872375488281, "KL/rejected_KL_mean": -353.0191650390625, "KL/std": 167.35247802734375, "epoch": 0.762303664921466, "fcm_dpo/beta": 0.002902527106925845, "fcm_dpo/delta": 0.02750171534717083, "fcm_dpo/margin": 98.26390838623047, "fcm_dpo/q_t": 0.43715596199035645, "grad_norm": 70.5536117553711, "learning_rate": 8.217385746050742e-08, "logits/chosen": -0.8307126760482788, "logits/rejected": -0.8298197984695435, "logps/chosen": -540.2990112304688, "logps/ref_chosen": -285.54376220703125, "logps/ref_rejected": -284.84619140625, "logps/rejected": -637.8654174804688, "loss": 4.8201, "margin_dpo/margin_mean": 98.263916015625, "margin_dpo/margin_std": 200.9420623779297, "step": 364 }, { "KL/chosen_KL_mean": -238.25563049316406, "KL/mean": -282.6103515625, "KL/rejected_KL_mean": -326.96502685546875, "KL/std": 162.19873046875, "epoch": 0.7643979057591623, "fcm_dpo/beta": 0.002954564755782485, "fcm_dpo/delta": 0.009645760990679264, "fcm_dpo/margin": 88.70938110351562, "fcm_dpo/q_t": 0.44021692872047424, "grad_norm": 70.69696807861328, "learning_rate": 8.082144028504231e-08, "logits/chosen": -0.8417816758155823, "logits/rejected": -0.8418366312980652, "logps/chosen": -513.04345703125, "logps/ref_chosen": -274.7878112792969, "logps/ref_rejected": -256.5738220214844, "logps/rejected": -583.5388793945312, "loss": 4.8457, "margin_dpo/margin_mean": 88.70938110351562, "margin_dpo/margin_std": 176.14007568359375, "step": 365 }, { "KL/chosen_KL_mean": -222.59596252441406, "KL/mean": -271.5234680175781, "KL/rejected_KL_mean": -320.450927734375, "KL/std": 141.25015258789062, "epoch": 0.7664921465968586, "fcm_dpo/beta": 0.0030219045002013445, "fcm_dpo/delta": 0.037596531212329865, "fcm_dpo/margin": 97.85496520996094, "fcm_dpo/q_t": 0.4315846860408783, "grad_norm": 70.44728088378906, "learning_rate": 7.947809564230445e-08, "logits/chosen": -0.7966880798339844, "logits/rejected": -0.8008946776390076, "logps/chosen": -509.24560546875, "logps/ref_chosen": -286.6496276855469, "logps/ref_rejected": -251.97140502929688, "logps/rejected": -572.42236328125, "loss": 4.717, "margin_dpo/margin_mean": 97.85496520996094, "margin_dpo/margin_std": 171.53121948242188, "step": 366 }, { "KL/chosen_KL_mean": -226.7736358642578, "KL/mean": -278.9706115722656, "KL/rejected_KL_mean": -331.16766357421875, "KL/std": 146.42007446289062, "epoch": 0.768586387434555, "fcm_dpo/beta": 0.00305999256670475, "fcm_dpo/delta": 0.010715863667428493, "fcm_dpo/margin": 104.39400482177734, "fcm_dpo/q_t": 0.4274834990501404, "grad_norm": 82.6015625, "learning_rate": 7.814389557179016e-08, "logits/chosen": -0.8351811766624451, "logits/rejected": -0.8077444434165955, "logps/chosen": -528.7186279296875, "logps/ref_chosen": -301.9449768066406, "logps/ref_rejected": -265.5677185058594, "logps/rejected": -596.7353515625, "loss": 4.6422, "margin_dpo/margin_mean": 104.39400482177734, "margin_dpo/margin_std": 168.87966918945312, "step": 367 }, { "KL/chosen_KL_mean": -209.43209838867188, "KL/mean": -276.49969482421875, "KL/rejected_KL_mean": -343.56732177734375, "KL/std": 147.89605712890625, "epoch": 0.7706806282722513, "fcm_dpo/beta": 0.0030316186603158712, "fcm_dpo/delta": -0.04178418964147568, "fcm_dpo/margin": 134.13523864746094, "fcm_dpo/q_t": 0.405577689409256, "grad_norm": 76.57606506347656, "learning_rate": 7.681891162260015e-08, "logits/chosen": -0.7988978028297424, "logits/rejected": -0.8013542890548706, "logps/chosen": -504.05865478515625, "logps/ref_chosen": -294.62652587890625, "logps/ref_rejected": -258.7628479003906, "logps/rejected": -602.3301391601562, "loss": 4.3154, "margin_dpo/margin_mean": 134.13523864746094, "margin_dpo/margin_std": 154.99842834472656, "step": 368 }, { "KL/chosen_KL_mean": -225.83438110351562, "KL/mean": -273.8876647949219, "KL/rejected_KL_mean": -321.9409484863281, "KL/std": 137.2335205078125, "epoch": 0.7727748691099476, "fcm_dpo/beta": 0.0030030980706214905, "fcm_dpo/delta": 0.01607823558151722, "fcm_dpo/margin": 96.10658264160156, "fcm_dpo/q_t": 0.4320639371871948, "grad_norm": 82.7309341430664, "learning_rate": 7.550321484960251e-08, "logits/chosen": -0.8810120820999146, "logits/rejected": -0.8641123175621033, "logps/chosen": -508.340087890625, "logps/ref_chosen": -282.5057373046875, "logps/ref_rejected": -266.41607666015625, "logps/rejected": -588.3570556640625, "loss": 4.6647, "margin_dpo/margin_mean": 96.10657501220703, "margin_dpo/margin_std": 144.0449981689453, "step": 369 }, { "KL/chosen_KL_mean": -215.18609619140625, "KL/mean": -274.7394104003906, "KL/rejected_KL_mean": -334.2928161621094, "KL/std": 153.53570556640625, "epoch": 0.774869109947644, "fcm_dpo/beta": 0.003025064943358302, "fcm_dpo/delta": -0.0023751449771225452, "fcm_dpo/margin": 119.10672760009766, "fcm_dpo/q_t": 0.4178846776485443, "grad_norm": 62.49055480957031, "learning_rate": 7.419687580962222e-08, "logits/chosen": -0.8580411076545715, "logits/rejected": -0.8811969757080078, "logps/chosen": -466.1924743652344, "logps/ref_chosen": -251.00640869140625, "logps/ref_rejected": -238.12542724609375, "logps/rejected": -572.418212890625, "loss": 4.5629, "margin_dpo/margin_mean": 119.10673522949219, "margin_dpo/margin_std": 193.12779235839844, "step": 370 }, { "KL/chosen_KL_mean": -250.4105682373047, "KL/mean": -285.2479248046875, "KL/rejected_KL_mean": -320.0852355957031, "KL/std": 149.4564666748047, "epoch": 0.7769633507853403, "fcm_dpo/beta": 0.003027756232768297, "fcm_dpo/delta": -0.0017022611573338509, "fcm_dpo/margin": 69.6746597290039, "fcm_dpo/q_t": 0.450680136680603, "grad_norm": 105.20964050292969, "learning_rate": 7.289996455765748e-08, "logits/chosen": -0.8298521041870117, "logits/rejected": -0.8093293309211731, "logps/chosen": -547.0697021484375, "logps/ref_chosen": -296.6591491699219, "logps/ref_rejected": -251.14675903320312, "logps/rejected": -571.23193359375, "loss": 4.9843, "margin_dpo/margin_mean": 69.67466735839844, "margin_dpo/margin_std": 154.35647583007812, "step": 371 }, { "KL/chosen_KL_mean": -240.75746154785156, "KL/mean": -299.1548156738281, "KL/rejected_KL_mean": -357.55224609375, "KL/std": 165.2203826904297, "epoch": 0.7790575916230367, "fcm_dpo/beta": 0.0029513647314161062, "fcm_dpo/delta": -0.024020541459321976, "fcm_dpo/margin": 116.7947998046875, "fcm_dpo/q_t": 0.4221827983856201, "grad_norm": 70.41334533691406, "learning_rate": 7.161255064312283e-08, "logits/chosen": -0.7842466235160828, "logits/rejected": -0.7680008411407471, "logps/chosen": -572.12890625, "logps/ref_chosen": -331.3714599609375, "logps/ref_rejected": -285.56805419921875, "logps/rejected": -643.1203002929688, "loss": 4.6121, "margin_dpo/margin_mean": 116.7947998046875, "margin_dpo/margin_std": 187.99862670898438, "step": 372 }, { "KL/chosen_KL_mean": -226.25857543945312, "KL/mean": -281.29608154296875, "KL/rejected_KL_mean": -336.3335876464844, "KL/std": 142.73548889160156, "epoch": 0.7811518324607329, "fcm_dpo/beta": 0.002992741297930479, "fcm_dpo/delta": 0.03157367929816246, "fcm_dpo/margin": 110.07496643066406, "fcm_dpo/q_t": 0.42270153760910034, "grad_norm": 86.4478988647461, "learning_rate": 7.033470310611945e-08, "logits/chosen": -0.8854488730430603, "logits/rejected": -0.8503902554512024, "logps/chosen": -548.2015380859375, "logps/ref_chosen": -321.9429931640625, "logps/ref_rejected": -271.2288513183594, "logps/rejected": -607.5624389648438, "loss": 4.5869, "margin_dpo/margin_mean": 110.0749740600586, "margin_dpo/margin_std": 165.2940216064453, "step": 373 }, { "KL/chosen_KL_mean": -236.50103759765625, "KL/mean": -282.9904479980469, "KL/rejected_KL_mean": -329.4798278808594, "KL/std": 160.59323120117188, "epoch": 0.7832460732984293, "fcm_dpo/beta": 0.0029890371952205896, "fcm_dpo/delta": -0.015482398681342602, "fcm_dpo/margin": 92.97879028320312, "fcm_dpo/q_t": 0.43591830134391785, "grad_norm": 74.21908569335938, "learning_rate": 6.906649047373245e-08, "logits/chosen": -0.8732917904853821, "logits/rejected": -0.8661702871322632, "logps/chosen": -555.6695556640625, "logps/ref_chosen": -319.1685485839844, "logps/ref_rejected": -284.6263732910156, "logps/rejected": -614.1061401367188, "loss": 4.773, "margin_dpo/margin_mean": 92.97879028320312, "margin_dpo/margin_std": 169.21636962890625, "step": 374 }, { "KL/chosen_KL_mean": -245.4715576171875, "KL/mean": -289.0046691894531, "KL/rejected_KL_mean": -332.53778076171875, "KL/std": 133.46920776367188, "epoch": 0.7853403141361257, "fcm_dpo/beta": 0.0029952474869787693, "fcm_dpo/delta": -0.0004676980897784233, "fcm_dpo/margin": 87.06620788574219, "fcm_dpo/q_t": 0.43925830721855164, "grad_norm": 74.76742553710938, "learning_rate": 6.780798075635675e-08, "logits/chosen": -0.8679319620132446, "logits/rejected": -0.8446568250656128, "logps/chosen": -560.3473510742188, "logps/ref_chosen": -314.87579345703125, "logps/ref_rejected": -259.1965026855469, "logps/rejected": -591.7342529296875, "loss": 4.8, "margin_dpo/margin_mean": 87.06620788574219, "margin_dpo/margin_std": 155.70870971679688, "step": 375 }, { "KL/chosen_KL_mean": -241.67770385742188, "KL/mean": -294.02313232421875, "KL/rejected_KL_mean": -346.36859130859375, "KL/std": 157.73382568359375, "epoch": 0.787434554973822, "fcm_dpo/beta": 0.0029938980005681515, "fcm_dpo/delta": -0.005739666521549225, "fcm_dpo/margin": 104.69088745117188, "fcm_dpo/q_t": 0.42898622155189514, "grad_norm": 100.38687133789062, "learning_rate": 6.655924144404906e-08, "logits/chosen": -0.8551640510559082, "logits/rejected": -0.8558748364448547, "logps/chosen": -529.3509521484375, "logps/ref_chosen": -287.6732482910156, "logps/ref_rejected": -256.6697082519531, "logps/rejected": -603.0382690429688, "loss": 4.7103, "margin_dpo/margin_mean": 104.69087982177734, "margin_dpo/margin_std": 189.6014404296875, "step": 376 }, { "KL/chosen_KL_mean": -258.047119140625, "KL/mean": -295.9388732910156, "KL/rejected_KL_mean": -333.83062744140625, "KL/std": 157.74354553222656, "epoch": 0.7895287958115184, "fcm_dpo/beta": 0.00297717796638608, "fcm_dpo/delta": 0.003270770888775587, "fcm_dpo/margin": 75.78350830078125, "fcm_dpo/q_t": 0.4478086233139038, "grad_norm": 64.10608673095703, "learning_rate": 6.532033950290885e-08, "logits/chosen": -0.8300923109054565, "logits/rejected": -0.8211076855659485, "logps/chosen": -563.30859375, "logps/ref_chosen": -305.261474609375, "logps/ref_rejected": -271.8887023925781, "logps/rejected": -605.7192993164062, "loss": 4.9974, "margin_dpo/margin_mean": 75.78350830078125, "margin_dpo/margin_std": 182.6964874267578, "step": 377 }, { "KL/chosen_KL_mean": -248.6783905029297, "KL/mean": -296.946533203125, "KL/rejected_KL_mean": -345.21466064453125, "KL/std": 163.25067138671875, "epoch": 0.7916230366492146, "fcm_dpo/beta": 0.0030068312771618366, "fcm_dpo/delta": 0.02455754391849041, "fcm_dpo/margin": 96.5362548828125, "fcm_dpo/q_t": 0.4341698884963989, "grad_norm": 74.27600860595703, "learning_rate": 6.409134137148736e-08, "logits/chosen": -0.8239084482192993, "logits/rejected": -0.8121182322502136, "logps/chosen": -530.2080078125, "logps/ref_chosen": -281.5295715332031, "logps/ref_rejected": -296.980224609375, "logps/rejected": -642.1949462890625, "loss": 4.7595, "margin_dpo/margin_mean": 96.5362548828125, "margin_dpo/margin_std": 179.10707092285156, "step": 378 }, { "KL/chosen_KL_mean": -228.6240234375, "KL/mean": -274.6946105957031, "KL/rejected_KL_mean": -320.76519775390625, "KL/std": 132.6265869140625, "epoch": 0.793717277486911, "fcm_dpo/beta": 0.0030300889629870653, "fcm_dpo/delta": -0.00830287579447031, "fcm_dpo/margin": 92.14117431640625, "fcm_dpo/q_t": 0.43474820256233215, "grad_norm": 89.22589874267578, "learning_rate": 6.28723129572247e-08, "logits/chosen": -0.8947068452835083, "logits/rejected": -0.8764788508415222, "logps/chosen": -493.704833984375, "logps/ref_chosen": -265.0807800292969, "logps/ref_rejected": -230.58932495117188, "logps/rejected": -551.3544921875, "loss": 4.7549, "margin_dpo/margin_mean": 92.14117431640625, "margin_dpo/margin_std": 164.5974578857422, "step": 379 }, { "KL/chosen_KL_mean": -243.8111572265625, "KL/mean": -299.34429931640625, "KL/rejected_KL_mean": -354.87744140625, "KL/std": 164.056640625, "epoch": 0.7958115183246073, "fcm_dpo/beta": 0.0029976628720760345, "fcm_dpo/delta": -0.021797433495521545, "fcm_dpo/margin": 111.06632995605469, "fcm_dpo/q_t": 0.4255141019821167, "grad_norm": 91.8367691040039, "learning_rate": 6.166331963291519e-08, "logits/chosen": -0.8527557253837585, "logits/rejected": -0.8262585997581482, "logps/chosen": -549.7195434570312, "logps/ref_chosen": -305.90838623046875, "logps/ref_rejected": -286.5906677246094, "logps/rejected": -641.4681396484375, "loss": 4.6676, "margin_dpo/margin_mean": 111.06632232666016, "margin_dpo/margin_std": 190.367919921875, "step": 380 }, { "KL/chosen_KL_mean": -243.97262573242188, "KL/mean": -303.4189147949219, "KL/rejected_KL_mean": -362.86517333984375, "KL/std": 184.5408477783203, "epoch": 0.7979057591623037, "fcm_dpo/beta": 0.002953719347715378, "fcm_dpo/delta": -0.021564383059740067, "fcm_dpo/margin": 118.892578125, "fcm_dpo/q_t": 0.422404944896698, "grad_norm": 64.48091888427734, "learning_rate": 6.046442623320145e-08, "logits/chosen": -0.8266147375106812, "logits/rejected": -0.7924824357032776, "logps/chosen": -496.84326171875, "logps/ref_chosen": -252.87066650390625, "logps/ref_rejected": -261.1927490234375, "logps/rejected": -624.0579223632812, "loss": 4.5793, "margin_dpo/margin_mean": 118.892578125, "margin_dpo/margin_std": 191.17279052734375, "step": 381 }, { "KL/chosen_KL_mean": -236.02357482910156, "KL/mean": -297.5960998535156, "KL/rejected_KL_mean": -359.1686706542969, "KL/std": 151.47557067871094, "epoch": 0.8, "fcm_dpo/beta": 0.0028901936020702124, "fcm_dpo/delta": -0.010631639510393143, "fcm_dpo/margin": 123.14509582519531, "fcm_dpo/q_t": 0.4171634614467621, "grad_norm": 66.18559265136719, "learning_rate": 5.9275697051098275e-08, "logits/chosen": -0.8696933388710022, "logits/rejected": -0.8662832379341125, "logps/chosen": -525.2350463867188, "logps/ref_chosen": -289.2114562988281, "logps/ref_rejected": -278.45751953125, "logps/rejected": -637.626220703125, "loss": 4.4974, "margin_dpo/margin_mean": 123.14509582519531, "margin_dpo/margin_std": 173.98348999023438, "step": 382 }, { "KL/chosen_KL_mean": -237.31979370117188, "KL/mean": -290.00091552734375, "KL/rejected_KL_mean": -342.68206787109375, "KL/std": 168.33383178710938, "epoch": 0.8020942408376963, "fcm_dpo/beta": 0.0028781089931726456, "fcm_dpo/delta": -0.025008056312799454, "fcm_dpo/margin": 105.36231994628906, "fcm_dpo/q_t": 0.43072545528411865, "grad_norm": 75.10661315917969, "learning_rate": 5.809719583454414e-08, "logits/chosen": -0.850218653678894, "logits/rejected": -0.8339353203773499, "logps/chosen": -510.9505920410156, "logps/ref_chosen": -273.630859375, "logps/ref_rejected": -261.44024658203125, "logps/rejected": -604.122314453125, "loss": 4.7315, "margin_dpo/margin_mean": 105.36231231689453, "margin_dpo/margin_std": 192.60629272460938, "step": 383 }, { "KL/chosen_KL_mean": -241.386474609375, "KL/mean": -290.0650939941406, "KL/rejected_KL_mean": -338.7437438964844, "KL/std": 136.2803192138672, "epoch": 0.8041884816753927, "fcm_dpo/beta": 0.0028522873762995005, "fcm_dpo/delta": 0.014860259369015694, "fcm_dpo/margin": 97.35729217529297, "fcm_dpo/q_t": 0.4356921911239624, "grad_norm": 54.56941223144531, "learning_rate": 5.6928985782982524e-08, "logits/chosen": -0.8740838766098022, "logits/rejected": -0.8694276213645935, "logps/chosen": -515.9564208984375, "logps/ref_chosen": -274.5699462890625, "logps/ref_rejected": -285.8253479003906, "logps/rejected": -624.569091796875, "loss": 4.7479, "margin_dpo/margin_mean": 97.35729217529297, "margin_dpo/margin_std": 170.97972106933594, "step": 384 }, { "KL/chosen_KL_mean": -248.646484375, "KL/mean": -288.6224365234375, "KL/rejected_KL_mean": -328.598388671875, "KL/std": 142.26695251464844, "epoch": 0.806282722513089, "fcm_dpo/beta": 0.002904432127252221, "fcm_dpo/delta": 0.019007494673132896, "fcm_dpo/margin": 79.95191955566406, "fcm_dpo/q_t": 0.44628220796585083, "grad_norm": 70.91700744628906, "learning_rate": 5.57711295439732e-08, "logits/chosen": -0.8183778524398804, "logits/rejected": -0.8135970830917358, "logps/chosen": -532.797119140625, "logps/ref_chosen": -284.150634765625, "logps/ref_rejected": -244.87921142578125, "logps/rejected": -573.4776000976562, "loss": 4.8742, "margin_dpo/margin_mean": 79.95191955566406, "margin_dpo/margin_std": 152.34213256835938, "step": 385 }, { "KL/chosen_KL_mean": -239.798095703125, "KL/mean": -304.68206787109375, "KL/rejected_KL_mean": -369.5660095214844, "KL/std": 161.51425170898438, "epoch": 0.8083769633507853, "fcm_dpo/beta": 0.0028078884351998568, "fcm_dpo/delta": -0.05693187564611435, "fcm_dpo/margin": 129.7678985595703, "fcm_dpo/q_t": 0.4165182411670685, "grad_norm": 78.76203155517578, "learning_rate": 5.4623689209832484e-08, "logits/chosen": -0.7965095639228821, "logits/rejected": -0.7943077087402344, "logps/chosen": -559.974365234375, "logps/ref_chosen": -320.1762390136719, "logps/ref_rejected": -302.05023193359375, "logps/rejected": -671.6162109375, "loss": 4.503, "margin_dpo/margin_mean": 129.7678985595703, "margin_dpo/margin_std": 177.5083770751953, "step": 386 }, { "KL/chosen_KL_mean": -255.77442932128906, "KL/mean": -303.63458251953125, "KL/rejected_KL_mean": -351.4947204589844, "KL/std": 153.2172088623047, "epoch": 0.8104712041884817, "fcm_dpo/beta": 0.0027528819628059864, "fcm_dpo/delta": -0.027687324211001396, "fcm_dpo/margin": 95.72029113769531, "fcm_dpo/q_t": 0.43756136298179626, "grad_norm": 80.19580078125, "learning_rate": 5.3486726314303175e-08, "logits/chosen": -0.8275306224822998, "logits/rejected": -0.8312669396400452, "logps/chosen": -528.0545654296875, "logps/ref_chosen": -272.2801513671875, "logps/ref_rejected": -265.1615905761719, "logps/rejected": -616.6563110351562, "loss": 4.8415, "margin_dpo/margin_mean": 95.72029113769531, "margin_dpo/margin_std": 183.86953735351562, "step": 387 }, { "KL/chosen_KL_mean": -273.6151123046875, "KL/mean": -306.04327392578125, "KL/rejected_KL_mean": -338.471435546875, "KL/std": 158.35028076171875, "epoch": 0.812565445026178, "fcm_dpo/beta": 0.0026982480194419622, "fcm_dpo/delta": -0.011122354306280613, "fcm_dpo/margin": 64.85627746582031, "fcm_dpo/q_t": 0.45950567722320557, "grad_norm": 84.51703643798828, "learning_rate": 5.2360301829254745e-08, "logits/chosen": -0.8161114454269409, "logits/rejected": -0.8020353317260742, "logps/chosen": -546.146484375, "logps/ref_chosen": -272.5313415527344, "logps/ref_rejected": -239.55735778808594, "logps/rejected": -578.02880859375, "loss": 5.1513, "margin_dpo/margin_mean": 64.85627746582031, "margin_dpo/margin_std": 188.13427734375, "step": 388 }, { "KL/chosen_KL_mean": -266.1781005859375, "KL/mean": -321.2748718261719, "KL/rejected_KL_mean": -376.37164306640625, "KL/std": 165.8982391357422, "epoch": 0.8146596858638744, "fcm_dpo/beta": 0.0026822565123438835, "fcm_dpo/delta": -0.018625523895025253, "fcm_dpo/margin": 110.19355010986328, "fcm_dpo/q_t": 0.43248510360717773, "grad_norm": 62.38093948364258, "learning_rate": 5.1244476161413806e-08, "logits/chosen": -0.8523331880569458, "logits/rejected": -0.8490720987319946, "logps/chosen": -547.267333984375, "logps/ref_chosen": -281.0892639160156, "logps/ref_rejected": -246.50045776367188, "logps/rejected": -622.8720703125, "loss": 4.7258, "margin_dpo/margin_mean": 110.19355773925781, "margin_dpo/margin_std": 189.8052978515625, "step": 389 }, { "KL/chosen_KL_mean": -259.96453857421875, "KL/mean": -313.5345458984375, "KL/rejected_KL_mean": -367.10455322265625, "KL/std": 155.7751922607422, "epoch": 0.8167539267015707, "fcm_dpo/beta": 0.0026583157014101744, "fcm_dpo/delta": -0.00272680283524096, "fcm_dpo/margin": 107.14004516601562, "fcm_dpo/q_t": 0.4331052899360657, "grad_norm": 59.64759063720703, "learning_rate": 5.013930914912476e-08, "logits/chosen": -0.8624210357666016, "logits/rejected": -0.8728106021881104, "logps/chosen": -543.9520263671875, "logps/ref_chosen": -283.98748779296875, "logps/ref_rejected": -283.465087890625, "logps/rejected": -650.569580078125, "loss": 4.7094, "margin_dpo/margin_mean": 107.14004516601562, "margin_dpo/margin_std": 176.22068786621094, "step": 390 }, { "KL/chosen_KL_mean": -267.4212951660156, "KL/mean": -325.6793518066406, "KL/rejected_KL_mean": -383.93743896484375, "KL/std": 176.0823516845703, "epoch": 0.818848167539267, "fcm_dpo/beta": 0.002617758931592107, "fcm_dpo/delta": -0.02285085991024971, "fcm_dpo/margin": 116.51616668701172, "fcm_dpo/q_t": 0.42977476119995117, "grad_norm": 69.12674713134766, "learning_rate": 4.904486005914027e-08, "logits/chosen": -0.8110455274581909, "logits/rejected": -0.7923341393470764, "logps/chosen": -551.2826538085938, "logps/ref_chosen": -283.86138916015625, "logps/ref_rejected": -263.5093688964844, "logps/rejected": -647.44677734375, "loss": 4.6727, "margin_dpo/margin_mean": 116.51615905761719, "margin_dpo/margin_std": 188.8794708251953, "step": 391 }, { "KL/chosen_KL_mean": -254.72386169433594, "KL/mean": -318.38873291015625, "KL/rejected_KL_mean": -382.05364990234375, "KL/std": 156.4735870361328, "epoch": 0.8209424083769633, "fcm_dpo/beta": 0.0025636577047407627, "fcm_dpo/delta": -0.012513198889791965, "fcm_dpo/margin": 127.32978820800781, "fcm_dpo/q_t": 0.42234691977500916, "grad_norm": 87.62789916992188, "learning_rate": 4.796118758344353e-08, "logits/chosen": -0.8127277493476868, "logits/rejected": -0.8246631026268005, "logps/chosen": -564.7939453125, "logps/ref_chosen": -310.070068359375, "logps/ref_rejected": -252.89817810058594, "logps/rejected": -634.9518432617188, "loss": 4.5707, "margin_dpo/margin_mean": 127.32978820800781, "margin_dpo/margin_std": 177.67172241210938, "step": 392 }, { "KL/chosen_KL_mean": -254.2708740234375, "KL/mean": -301.04620361328125, "KL/rejected_KL_mean": -347.821533203125, "KL/std": 150.13577270507812, "epoch": 0.8230366492146597, "fcm_dpo/beta": 0.0025904097128659487, "fcm_dpo/delta": 0.024187462404370308, "fcm_dpo/margin": 93.55062103271484, "fcm_dpo/q_t": 0.4434916079044342, "grad_norm": 52.19322204589844, "learning_rate": 4.688834983610082e-08, "logits/chosen": -0.8437351584434509, "logits/rejected": -0.8238535523414612, "logps/chosen": -540.9865112304688, "logps/ref_chosen": -286.7156677246094, "logps/ref_rejected": -230.00357055664062, "logps/rejected": -577.8250732421875, "loss": 4.8433, "margin_dpo/margin_mean": 93.55061340332031, "margin_dpo/margin_std": 174.47572326660156, "step": 393 }, { "KL/chosen_KL_mean": -257.57318115234375, "KL/mean": -304.79718017578125, "KL/rejected_KL_mean": -352.02117919921875, "KL/std": 179.62074279785156, "epoch": 0.8251308900523561, "fcm_dpo/beta": 0.0026478366926312447, "fcm_dpo/delta": 0.02115212380886078, "fcm_dpo/margin": 94.44804382324219, "fcm_dpo/q_t": 0.444901704788208, "grad_norm": 57.13371658325195, "learning_rate": 4.582640435014459e-08, "logits/chosen": -0.8858999609947205, "logits/rejected": -0.875397801399231, "logps/chosen": -583.566650390625, "logps/ref_chosen": -325.9934387207031, "logps/ref_rejected": -317.42706298828125, "logps/rejected": -669.4482421875, "loss": 4.9197, "margin_dpo/margin_mean": 94.44804382324219, "margin_dpo/margin_std": 205.414306640625, "step": 394 }, { "KL/chosen_KL_mean": -244.72979736328125, "KL/mean": -310.8471984863281, "KL/rejected_KL_mean": -376.9646301269531, "KL/std": 167.30059814453125, "epoch": 0.8272251308900523, "fcm_dpo/beta": 0.002669533481821418, "fcm_dpo/delta": -0.015710730105638504, "fcm_dpo/margin": 132.23486328125, "fcm_dpo/q_t": 0.4210633933544159, "grad_norm": 63.38975524902344, "learning_rate": 4.477540807448832e-08, "logits/chosen": -0.8237651586532593, "logits/rejected": -0.8312820792198181, "logps/chosen": -513.630615234375, "logps/ref_chosen": -268.90081787109375, "logps/ref_rejected": -272.85809326171875, "logps/rejected": -649.82275390625, "loss": 4.5299, "margin_dpo/margin_mean": 132.23486328125, "margin_dpo/margin_std": 197.66409301757812, "step": 395 }, { "KL/chosen_KL_mean": -267.392333984375, "KL/mean": -313.00030517578125, "KL/rejected_KL_mean": -358.6082763671875, "KL/std": 159.95083618164062, "epoch": 0.8293193717277487, "fcm_dpo/beta": 0.002658443059772253, "fcm_dpo/delta": 0.013091212138533592, "fcm_dpo/margin": 91.2159194946289, "fcm_dpo/q_t": 0.443788081407547, "grad_norm": 103.75393676757812, "learning_rate": 4.373541737087263e-08, "logits/chosen": -0.833990752696991, "logits/rejected": -0.8136544823646545, "logps/chosen": -558.5906372070312, "logps/ref_chosen": -291.19830322265625, "logps/ref_rejected": -253.2803955078125, "logps/rejected": -611.888671875, "loss": 4.8872, "margin_dpo/margin_mean": 91.2159194946289, "margin_dpo/margin_std": 185.2631072998047, "step": 396 }, { "KL/chosen_KL_mean": -252.65269470214844, "KL/mean": -297.8766174316406, "KL/rejected_KL_mean": -343.1005859375, "KL/std": 157.75360107421875, "epoch": 0.831413612565445, "fcm_dpo/beta": 0.002626030705869198, "fcm_dpo/delta": -0.026181943714618683, "fcm_dpo/margin": 90.44788360595703, "fcm_dpo/q_t": 0.4456873834133148, "grad_norm": 70.29791259765625, "learning_rate": 4.270648801084295e-08, "logits/chosen": -0.8465428352355957, "logits/rejected": -0.8269012570381165, "logps/chosen": -562.47509765625, "logps/ref_chosen": -309.8224182128906, "logps/ref_rejected": -291.9057922363281, "logps/rejected": -635.00634765625, "loss": 4.9391, "margin_dpo/margin_mean": 90.44788360595703, "margin_dpo/margin_std": 190.6138153076172, "step": 397 }, { "KL/chosen_KL_mean": -283.8010559082031, "KL/mean": -324.195068359375, "KL/rejected_KL_mean": -364.5890197753906, "KL/std": 173.2881317138672, "epoch": 0.8335078534031414, "fcm_dpo/beta": 0.0025994465686380863, "fcm_dpo/delta": -0.006987897679209709, "fcm_dpo/margin": 80.7879638671875, "fcm_dpo/q_t": 0.44956809282302856, "grad_norm": 79.59461212158203, "learning_rate": 4.168867517275806e-08, "logits/chosen": -0.7517906427383423, "logits/rejected": -0.7874580025672913, "logps/chosen": -581.6146240234375, "logps/ref_chosen": -297.8135070800781, "logps/ref_rejected": -270.5025634765625, "logps/rejected": -635.0916137695312, "loss": 5.1179, "margin_dpo/margin_mean": 80.78797149658203, "margin_dpo/margin_std": 224.45187377929688, "step": 398 }, { "KL/chosen_KL_mean": -264.94573974609375, "KL/mean": -315.5485534667969, "KL/rejected_KL_mean": -366.15130615234375, "KL/std": 161.73268127441406, "epoch": 0.8356020942408376, "fcm_dpo/beta": 0.0026253818068653345, "fcm_dpo/delta": 0.022293759509921074, "fcm_dpo/margin": 101.20550537109375, "fcm_dpo/q_t": 0.4391327202320099, "grad_norm": 73.089111328125, "learning_rate": 4.0682033438831584e-08, "logits/chosen": -0.8615554571151733, "logits/rejected": -0.8228447437286377, "logps/chosen": -557.7925415039062, "logps/ref_chosen": -292.8467712402344, "logps/ref_rejected": -268.3638916015625, "logps/rejected": -634.51513671875, "loss": 4.8136, "margin_dpo/margin_mean": 101.20550537109375, "margin_dpo/margin_std": 190.82528686523438, "step": 399 }, { "KL/chosen_KL_mean": -255.96389770507812, "KL/mean": -306.29583740234375, "KL/rejected_KL_mean": -356.6277770996094, "KL/std": 157.36746215820312, "epoch": 0.837696335078534, "fcm_dpo/beta": 0.002646203152835369, "fcm_dpo/delta": 0.005138233304023743, "fcm_dpo/margin": 100.66392517089844, "fcm_dpo/q_t": 0.43769270181655884, "grad_norm": 51.399471282958984, "learning_rate": 3.968661679220467e-08, "logits/chosen": -0.8845930695533752, "logits/rejected": -0.8838775753974915, "logps/chosen": -519.6402587890625, "logps/ref_chosen": -263.6763916015625, "logps/ref_rejected": -258.67266845703125, "logps/rejected": -615.3004760742188, "loss": 4.8145, "margin_dpo/margin_mean": 100.66392517089844, "margin_dpo/margin_std": 187.87037658691406, "step": 400 }, { "epoch": 0.837696335078534, "eval_KL/chosen_KL_mean": -259.0677185058594, "eval_KL/mean": -311.7682800292969, "eval_KL/rejected_KL_mean": -364.4688720703125, "eval_KL/std": 162.36822509765625, "eval_fcm_dpo/beta": 0.0026530208997428417, "eval_logits/chosen": -0.8539494872093201, "eval_logits/rejected": -0.8364132642745972, "eval_logps/chosen": -546.8944702148438, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -631.3988647460938, "eval_loss": 0.5982628464698792, "eval_margin_dpo/margin_mean": 105.40113830566406, "eval_margin_dpo/margin_std": 182.74659729003906, "eval_runtime": 78.7931, "eval_samples_per_second": 25.383, "eval_steps_per_second": 3.173, "step": 400 }, { "KL/chosen_KL_mean": -256.0751953125, "KL/mean": -309.08087158203125, "KL/rejected_KL_mean": -362.0865478515625, "KL/std": 163.9885711669922, "epoch": 0.8397905759162304, "fcm_dpo/beta": 0.002689874265342951, "fcm_dpo/delta": 0.024891149252653122, "fcm_dpo/margin": 106.01134490966797, "fcm_dpo/q_t": 0.4338783919811249, "grad_norm": 69.87895202636719, "learning_rate": 3.8702478614051345e-08, "logits/chosen": -0.83061283826828, "logits/rejected": -0.8284586668014526, "logps/chosen": -574.3605346679688, "logps/ref_chosen": -318.2853088378906, "logps/ref_rejected": -293.75225830078125, "logps/rejected": -655.8388061523438, "loss": 4.7405, "margin_dpo/margin_mean": 106.01134490966797, "margin_dpo/margin_std": 185.73927307128906, "step": 401 }, { "KL/chosen_KL_mean": -247.47999572753906, "KL/mean": -295.99700927734375, "KL/rejected_KL_mean": -344.5140075683594, "KL/std": 166.7805938720703, "epoch": 0.8418848167539267, "fcm_dpo/beta": 0.0027063796296715736, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.03401184082031, "fcm_dpo/q_t": 0.4415395259857178, "grad_norm": 67.62411499023438, "learning_rate": 3.772967168071517e-08, "logits/chosen": -0.8851345181465149, "logits/rejected": -0.8621891736984253, "logps/chosen": -556.9078369140625, "logps/ref_chosen": -309.4278564453125, "logps/ref_rejected": -282.0279846191406, "logps/rejected": -626.5419921875, "loss": 4.8518, "margin_dpo/margin_mean": 97.03401184082031, "margin_dpo/margin_std": 202.4774169921875, "step": 402 }, { "KL/chosen_KL_mean": -236.76856994628906, "KL/mean": -308.6864318847656, "KL/rejected_KL_mean": -380.604248046875, "KL/std": 171.2498779296875, "epoch": 0.8439790575916231, "fcm_dpo/beta": 0.002709330292418599, "fcm_dpo/delta": -0.025865260511636734, "fcm_dpo/margin": 143.83566284179688, "fcm_dpo/q_t": 0.41079238057136536, "grad_norm": 60.24233627319336, "learning_rate": 3.676824816087978e-08, "logits/chosen": -0.8610972166061401, "logits/rejected": -0.835875928401947, "logps/chosen": -545.7970581054688, "logps/ref_chosen": -309.0284729003906, "logps/ref_rejected": -272.9622497558594, "logps/rejected": -653.5665283203125, "loss": 4.3735, "margin_dpo/margin_mean": 143.83567810058594, "margin_dpo/margin_std": 179.44992065429688, "step": 403 }, { "KL/chosen_KL_mean": -261.0257873535156, "KL/mean": -315.1200256347656, "KL/rejected_KL_mean": -369.2142028808594, "KL/std": 173.9381866455078, "epoch": 0.8460732984293193, "fcm_dpo/beta": 0.002677470212802291, "fcm_dpo/delta": 0.0036769555881619453, "fcm_dpo/margin": 108.18839263916016, "fcm_dpo/q_t": 0.4339839220046997, "grad_norm": 78.65907287597656, "learning_rate": 3.581825961277074e-08, "logits/chosen": -0.8783624172210693, "logits/rejected": -0.8541516661643982, "logps/chosen": -558.3095703125, "logps/ref_chosen": -297.2837219238281, "logps/ref_rejected": -256.99041748046875, "logps/rejected": -626.20458984375, "loss": 4.7808, "margin_dpo/margin_mean": 108.18839263916016, "margin_dpo/margin_std": 206.70828247070312, "step": 404 }, { "KL/chosen_KL_mean": -249.36727905273438, "KL/mean": -309.5423889160156, "KL/rejected_KL_mean": -369.7174987792969, "KL/std": 169.79800415039062, "epoch": 0.8481675392670157, "fcm_dpo/beta": 0.002700282959267497, "fcm_dpo/delta": 0.026766503229737282, "fcm_dpo/margin": 120.3502197265625, "fcm_dpo/q_t": 0.42523056268692017, "grad_norm": 61.90435028076172, "learning_rate": 3.487975698139084e-08, "logits/chosen": -0.8048846125602722, "logits/rejected": -0.8137521147727966, "logps/chosen": -507.3326110839844, "logps/ref_chosen": -257.96533203125, "logps/ref_rejected": -255.811279296875, "logps/rejected": -625.52880859375, "loss": 4.6128, "margin_dpo/margin_mean": 120.3502197265625, "margin_dpo/margin_std": 191.45315551757812, "step": 405 }, { "KL/chosen_KL_mean": -269.34295654296875, "KL/mean": -309.04095458984375, "KL/rejected_KL_mean": -348.739013671875, "KL/std": 154.44068908691406, "epoch": 0.8502617801047121, "fcm_dpo/beta": 0.002717172261327505, "fcm_dpo/delta": 0.003170239506289363, "fcm_dpo/margin": 79.39606475830078, "fcm_dpo/q_t": 0.4483673572540283, "grad_norm": 113.7933349609375, "learning_rate": 3.3952790595787986e-08, "logits/chosen": -0.8405861258506775, "logits/rejected": -0.8171727061271667, "logps/chosen": -554.5239868164062, "logps/ref_chosen": -285.1810607910156, "logps/ref_rejected": -264.41351318359375, "logps/rejected": -613.1525268554688, "loss": 4.9484, "margin_dpo/margin_mean": 79.39606475830078, "margin_dpo/margin_std": 170.13916015625, "step": 406 }, { "KL/chosen_KL_mean": -265.46820068359375, "KL/mean": -321.43084716796875, "KL/rejected_KL_mean": -377.3935546875, "KL/std": 168.72860717773438, "epoch": 0.8523560209424084, "fcm_dpo/beta": 0.0027630478143692017, "fcm_dpo/delta": 0.02725648693740368, "fcm_dpo/margin": 111.92539978027344, "fcm_dpo/q_t": 0.4298943281173706, "grad_norm": 55.85504913330078, "learning_rate": 3.303741016635614e-08, "logits/chosen": -0.8258509635925293, "logits/rejected": -0.8421826362609863, "logps/chosen": -530.706298828125, "logps/ref_chosen": -265.23809814453125, "logps/ref_rejected": -219.0631561279297, "logps/rejected": -596.4567260742188, "loss": 4.7059, "margin_dpo/margin_mean": 111.9253921508789, "margin_dpo/margin_std": 197.7037353515625, "step": 407 }, { "KL/chosen_KL_mean": -247.27427673339844, "KL/mean": -302.8316345214844, "KL/rejected_KL_mean": -358.3890686035156, "KL/std": 157.02703857421875, "epoch": 0.8544502617801047, "fcm_dpo/beta": 0.0027716769836843014, "fcm_dpo/delta": -0.04041987285017967, "fcm_dpo/margin": 111.11476135253906, "fcm_dpo/q_t": 0.42840081453323364, "grad_norm": 79.1960678100586, "learning_rate": 3.2133664782169944e-08, "logits/chosen": -0.8630377054214478, "logits/rejected": -0.8538559079170227, "logps/chosen": -544.2468872070312, "logps/ref_chosen": -296.9726257324219, "logps/ref_rejected": -295.4786376953125, "logps/rejected": -653.8676147460938, "loss": 4.6593, "margin_dpo/margin_mean": 111.11476135253906, "margin_dpo/margin_std": 172.04241943359375, "step": 408 }, { "KL/chosen_KL_mean": -260.24188232421875, "KL/mean": -322.407958984375, "KL/rejected_KL_mean": -384.5740051269531, "KL/std": 207.52333068847656, "epoch": 0.856544502617801, "fcm_dpo/beta": 0.002639027778059244, "fcm_dpo/delta": -0.04701896011829376, "fcm_dpo/margin": 124.33211517333984, "fcm_dpo/q_t": 0.4329832196235657, "grad_norm": 53.74481201171875, "learning_rate": 3.12416029083514e-08, "logits/chosen": -0.839493989944458, "logits/rejected": -0.8232097029685974, "logps/chosen": -547.6212768554688, "logps/ref_chosen": -287.37933349609375, "logps/ref_rejected": -275.80291748046875, "logps/rejected": -660.3768310546875, "loss": 4.759, "margin_dpo/margin_mean": 124.33211517333984, "margin_dpo/margin_std": 254.77313232421875, "step": 409 }, { "KL/chosen_KL_mean": -251.41685485839844, "KL/mean": -302.3774108886719, "KL/rejected_KL_mean": -353.33795166015625, "KL/std": 163.56126403808594, "epoch": 0.8586387434554974, "fcm_dpo/beta": 0.0026220958679914474, "fcm_dpo/delta": 0.018517207354307175, "fcm_dpo/margin": 101.92111206054688, "fcm_dpo/q_t": 0.43905141949653625, "grad_norm": 59.67084884643555, "learning_rate": 3.036127238347164e-08, "logits/chosen": -0.8541938662528992, "logits/rejected": -0.8597331047058105, "logps/chosen": -533.197021484375, "logps/ref_chosen": -281.7801818847656, "logps/ref_rejected": -266.7550354003906, "logps/rejected": -620.093017578125, "loss": 4.8163, "margin_dpo/margin_mean": 101.92111206054688, "margin_dpo/margin_std": 195.57972717285156, "step": 410 }, { "KL/chosen_KL_mean": -244.28140258789062, "KL/mean": -301.5091857910156, "KL/rejected_KL_mean": -358.7369079589844, "KL/std": 168.92539978027344, "epoch": 0.8607329842931937, "fcm_dpo/beta": 0.0026748264208436012, "fcm_dpo/delta": 0.0014014113694429398, "fcm_dpo/margin": 114.45552825927734, "fcm_dpo/q_t": 0.4303073585033417, "grad_norm": 85.47948455810547, "learning_rate": 2.9492720416985e-08, "logits/chosen": -0.8583596348762512, "logits/rejected": -0.8106494545936584, "logps/chosen": -525.86865234375, "logps/ref_chosen": -281.5872497558594, "logps/ref_rejected": -254.78916931152344, "logps/rejected": -613.526123046875, "loss": 4.6366, "margin_dpo/margin_mean": 114.45552825927734, "margin_dpo/margin_std": 170.4075927734375, "step": 411 }, { "KL/chosen_KL_mean": -273.9906921386719, "KL/mean": -319.8974304199219, "KL/rejected_KL_mean": -365.8041687011719, "KL/std": 163.98312377929688, "epoch": 0.86282722513089, "fcm_dpo/beta": 0.002576531609520316, "fcm_dpo/delta": -0.026781653985381126, "fcm_dpo/margin": 91.8134994506836, "fcm_dpo/q_t": 0.44552770256996155, "grad_norm": 70.79202270507812, "learning_rate": 2.863599358669755e-08, "logits/chosen": -0.8241173624992371, "logits/rejected": -0.8364276885986328, "logps/chosen": -550.5248413085938, "logps/ref_chosen": -276.5341796875, "logps/ref_rejected": -273.8751220703125, "logps/rejected": -639.6793212890625, "loss": 4.9052, "margin_dpo/margin_mean": 91.8134994506836, "margin_dpo/margin_std": 183.9867401123047, "step": 412 }, { "KL/chosen_KL_mean": -252.1434783935547, "KL/mean": -297.56536865234375, "KL/rejected_KL_mean": -342.98724365234375, "KL/std": 162.1449737548828, "epoch": 0.8649214659685864, "fcm_dpo/beta": 0.0025809926446527243, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.84378051757812, "fcm_dpo/q_t": 0.44537127017974854, "grad_norm": 60.38282775878906, "learning_rate": 2.7791137836269158e-08, "logits/chosen": -0.840187132358551, "logits/rejected": -0.8357990384101868, "logps/chosen": -523.4180297851562, "logps/ref_chosen": -271.2745666503906, "logps/ref_rejected": -270.16912841796875, "logps/rejected": -613.1563720703125, "loss": 4.8861, "margin_dpo/margin_mean": 90.84378051757812, "margin_dpo/margin_std": 185.5474395751953, "step": 413 }, { "KL/chosen_KL_mean": -254.0613250732422, "KL/mean": -316.70928955078125, "KL/rejected_KL_mean": -379.35723876953125, "KL/std": 163.72683715820312, "epoch": 0.8670157068062827, "fcm_dpo/beta": 0.0025707168970257044, "fcm_dpo/delta": -0.01698092371225357, "fcm_dpo/margin": 125.29594421386719, "fcm_dpo/q_t": 0.4260947108268738, "grad_norm": 62.23833084106445, "learning_rate": 2.6958198472749717e-08, "logits/chosen": -0.8793128728866577, "logits/rejected": -0.872239351272583, "logps/chosen": -551.1763305664062, "logps/ref_chosen": -297.11505126953125, "logps/ref_rejected": -271.7034606933594, "logps/rejected": -651.060791015625, "loss": 4.6003, "margin_dpo/margin_mean": 125.29594421386719, "margin_dpo/margin_std": 187.06222534179688, "step": 414 }, { "KL/chosen_KL_mean": -251.2029266357422, "KL/mean": -303.4581604003906, "KL/rejected_KL_mean": -355.71331787109375, "KL/std": 149.83013916015625, "epoch": 0.8691099476439791, "fcm_dpo/beta": 0.0025737180840224028, "fcm_dpo/delta": 0.01915055885910988, "fcm_dpo/margin": 104.51042938232422, "fcm_dpo/q_t": 0.4367385804653168, "grad_norm": 63.56855773925781, "learning_rate": 2.613722016414943e-08, "logits/chosen": -0.871803343296051, "logits/rejected": -0.8581423163414001, "logps/chosen": -548.8956298828125, "logps/ref_chosen": -297.6926574707031, "logps/ref_rejected": -279.0503234863281, "logps/rejected": -634.7637329101562, "loss": 4.7351, "margin_dpo/margin_mean": 104.51043701171875, "margin_dpo/margin_std": 170.8563995361328, "step": 415 }, { "KL/chosen_KL_mean": -236.1012725830078, "KL/mean": -297.1469421386719, "KL/rejected_KL_mean": -358.192626953125, "KL/std": 151.40481567382812, "epoch": 0.8712041884816754, "fcm_dpo/beta": 0.002560949884355068, "fcm_dpo/delta": -5.1200266170781106e-05, "fcm_dpo/margin": 122.09134674072266, "fcm_dpo/q_t": 0.42640596628189087, "grad_norm": 39.975067138671875, "learning_rate": 2.5328246937043525e-08, "logits/chosen": -0.8686580061912537, "logits/rejected": -0.8653780817985535, "logps/chosen": -547.9268188476562, "logps/ref_chosen": -311.8255615234375, "logps/ref_rejected": -268.6170654296875, "logps/rejected": -626.8096923828125, "loss": 4.6017, "margin_dpo/margin_mean": 122.09133911132812, "margin_dpo/margin_std": 176.49038696289062, "step": 416 }, { "KL/chosen_KL_mean": -256.92279052734375, "KL/mean": -309.4592590332031, "KL/rejected_KL_mean": -361.9957275390625, "KL/std": 169.861083984375, "epoch": 0.8732984293193717, "fcm_dpo/beta": 0.0025655170902609825, "fcm_dpo/delta": -0.02516508474946022, "fcm_dpo/margin": 105.07295227050781, "fcm_dpo/q_t": 0.43865251541137695, "grad_norm": 58.286231994628906, "learning_rate": 2.4531322174210973e-08, "logits/chosen": -0.824512779712677, "logits/rejected": -0.8112680315971375, "logps/chosen": -567.3595581054688, "logps/ref_chosen": -310.43682861328125, "logps/ref_rejected": -277.15283203125, "logps/rejected": -639.1485595703125, "loss": 4.8026, "margin_dpo/margin_mean": 105.07294464111328, "margin_dpo/margin_std": 191.708740234375, "step": 417 }, { "KL/chosen_KL_mean": -271.6235046386719, "KL/mean": -324.97332763671875, "KL/rejected_KL_mean": -378.3231506347656, "KL/std": 181.0856170654297, "epoch": 0.875392670157068, "fcm_dpo/beta": 0.0025076887104660273, "fcm_dpo/delta": -0.01675930619239807, "fcm_dpo/margin": 106.69969177246094, "fcm_dpo/q_t": 0.43919453024864197, "grad_norm": 80.68965911865234, "learning_rate": 2.3746488612308295e-08, "logits/chosen": -0.8077826499938965, "logits/rejected": -0.7858579158782959, "logps/chosen": -550.1194458007812, "logps/ref_chosen": -278.49591064453125, "logps/ref_rejected": -276.56671142578125, "logps/rejected": -654.889892578125, "loss": 4.8247, "margin_dpo/margin_mean": 106.69967651367188, "margin_dpo/margin_std": 208.5244598388672, "step": 418 }, { "KL/chosen_KL_mean": -267.290771484375, "KL/mean": -320.2489013671875, "KL/rejected_KL_mean": -373.2070007324219, "KL/std": 170.7522430419922, "epoch": 0.8774869109947644, "fcm_dpo/beta": 0.0025136300828307867, "fcm_dpo/delta": 0.009540699422359467, "fcm_dpo/margin": 105.91624450683594, "fcm_dpo/q_t": 0.4366569519042969, "grad_norm": 75.40731048583984, "learning_rate": 2.297378833957761e-08, "logits/chosen": -0.8613269329071045, "logits/rejected": -0.8316982388496399, "logps/chosen": -566.1910400390625, "logps/ref_chosen": -298.9002380371094, "logps/ref_rejected": -246.1540985107422, "logps/rejected": -619.3611450195312, "loss": 4.8257, "margin_dpo/margin_mean": 105.91624450683594, "margin_dpo/margin_std": 208.2015838623047, "step": 419 }, { "KL/chosen_KL_mean": -260.46661376953125, "KL/mean": -325.86968994140625, "KL/rejected_KL_mean": -391.2727966308594, "KL/std": 182.0087432861328, "epoch": 0.8795811518324608, "fcm_dpo/beta": 0.0025134901516139507, "fcm_dpo/delta": -0.011981809511780739, "fcm_dpo/margin": 130.80618286132812, "fcm_dpo/q_t": 0.4250591993331909, "grad_norm": 66.69501495361328, "learning_rate": 2.2213262793589482e-08, "logits/chosen": -0.8003321290016174, "logits/rejected": -0.771695613861084, "logps/chosen": -525.0275268554688, "logps/ref_chosen": -264.5608825683594, "logps/ref_rejected": -245.67031860351562, "logps/rejected": -636.9431762695312, "loss": 4.6755, "margin_dpo/margin_mean": 130.80618286132812, "margin_dpo/margin_std": 231.65260314941406, "step": 420 }, { "KL/chosen_KL_mean": -239.69073486328125, "KL/mean": -296.38812255859375, "KL/rejected_KL_mean": -353.08551025390625, "KL/std": 154.60948181152344, "epoch": 0.881675392670157, "fcm_dpo/beta": 0.002471539657562971, "fcm_dpo/delta": -0.02126682549715042, "fcm_dpo/margin": 113.39478302001953, "fcm_dpo/q_t": 0.4333392381668091, "grad_norm": 67.4413070678711, "learning_rate": 2.1464952759020856e-08, "logits/chosen": -0.8876159191131592, "logits/rejected": -0.8631241917610168, "logps/chosen": -537.395751953125, "logps/ref_chosen": -297.70501708984375, "logps/ref_rejected": -243.74771118164062, "logps/rejected": -596.8331909179688, "loss": 4.7061, "margin_dpo/margin_mean": 113.394775390625, "margin_dpo/margin_std": 176.9335479736328, "step": 421 }, { "KL/chosen_KL_mean": -253.8042449951172, "KL/mean": -306.0145263671875, "KL/rejected_KL_mean": -358.22479248046875, "KL/std": 167.76966857910156, "epoch": 0.8837696335078534, "fcm_dpo/beta": 0.002462256234139204, "fcm_dpo/delta": 0.015522689558565617, "fcm_dpo/margin": 104.4205322265625, "fcm_dpo/q_t": 0.44079747796058655, "grad_norm": 51.60731887817383, "learning_rate": 2.07288983654679e-08, "logits/chosen": -0.7430369257926941, "logits/rejected": -0.7820625901222229, "logps/chosen": -542.1630249023438, "logps/ref_chosen": -288.3587646484375, "logps/ref_rejected": -256.4377746582031, "logps/rejected": -614.66259765625, "loss": 4.805, "margin_dpo/margin_mean": 104.42054748535156, "margin_dpo/margin_std": 192.31918334960938, "step": 422 }, { "KL/chosen_KL_mean": -258.586181640625, "KL/mean": -319.3099670410156, "KL/rejected_KL_mean": -380.03375244140625, "KL/std": 160.17820739746094, "epoch": 0.8858638743455497, "fcm_dpo/beta": 0.0024836647789925337, "fcm_dpo/delta": 0.003166984301060438, "fcm_dpo/margin": 121.4476318359375, "fcm_dpo/q_t": 0.42924964427948, "grad_norm": 62.990482330322266, "learning_rate": 2.0005139085293942e-08, "logits/chosen": -0.8718672394752502, "logits/rejected": -0.849821925163269, "logps/chosen": -554.5932006835938, "logps/ref_chosen": -296.00701904296875, "logps/ref_rejected": -261.3480529785156, "logps/rejected": -641.3818359375, "loss": 4.6526, "margin_dpo/margin_mean": 121.44761657714844, "margin_dpo/margin_std": 191.81884765625, "step": 423 }, { "KL/chosen_KL_mean": -238.3405303955078, "KL/mean": -292.3135986328125, "KL/rejected_KL_mean": -346.28668212890625, "KL/std": 150.8682098388672, "epoch": 0.8879581151832461, "fcm_dpo/beta": 0.0024709829594939947, "fcm_dpo/delta": -0.0036196252331137657, "fcm_dpo/margin": 107.9461669921875, "fcm_dpo/q_t": 0.4369615614414215, "grad_norm": 80.2655258178711, "learning_rate": 1.9293713731512673e-08, "logits/chosen": -0.8483703136444092, "logits/rejected": -0.8329979777336121, "logps/chosen": -547.7623901367188, "logps/ref_chosen": -309.421875, "logps/ref_rejected": -249.14886474609375, "logps/rejected": -595.4356079101562, "loss": 4.734, "margin_dpo/margin_mean": 107.9461669921875, "margin_dpo/margin_std": 167.56033325195312, "step": 424 }, { "KL/chosen_KL_mean": -247.47341918945312, "KL/mean": -297.63507080078125, "KL/rejected_KL_mean": -347.79669189453125, "KL/std": 160.0552520751953, "epoch": 0.8900523560209425, "fcm_dpo/beta": 0.0024773648474365473, "fcm_dpo/delta": -0.0067837778478860855, "fcm_dpo/margin": 100.32325744628906, "fcm_dpo/q_t": 0.44239264726638794, "grad_norm": 47.60661697387695, "learning_rate": 1.8594660455706763e-08, "logits/chosen": -0.8335504531860352, "logits/rejected": -0.8358312249183655, "logps/chosen": -527.9825439453125, "logps/ref_chosen": -280.50909423828125, "logps/ref_rejected": -276.8252258300781, "logps/rejected": -624.6218872070312, "loss": 4.8279, "margin_dpo/margin_mean": 100.32325744628906, "margin_dpo/margin_std": 179.0926971435547, "step": 425 }, { "KL/chosen_KL_mean": -253.93557739257812, "KL/mean": -309.0635986328125, "KL/rejected_KL_mean": -364.19158935546875, "KL/std": 163.70797729492188, "epoch": 0.8921465968586387, "fcm_dpo/beta": 0.002501498442143202, "fcm_dpo/delta": 0.027417544275522232, "fcm_dpo/margin": 110.25602722167969, "fcm_dpo/q_t": 0.4352967441082001, "grad_norm": 61.759361267089844, "learning_rate": 1.7908016745981856e-08, "logits/chosen": -0.8617511987686157, "logits/rejected": -0.8399728536605835, "logps/chosen": -546.7207641601562, "logps/ref_chosen": -292.78521728515625, "logps/ref_rejected": -255.62698364257812, "logps/rejected": -619.8185424804688, "loss": 4.733, "margin_dpo/margin_mean": 110.25602722167969, "margin_dpo/margin_std": 181.19354248046875, "step": 426 }, { "KL/chosen_KL_mean": -242.61668395996094, "KL/mean": -321.30316162109375, "KL/rejected_KL_mean": -399.9896240234375, "KL/std": 182.28509521484375, "epoch": 0.8942408376963351, "fcm_dpo/beta": 0.002506428863853216, "fcm_dpo/delta": -0.04454171657562256, "fcm_dpo/margin": 157.37294006347656, "fcm_dpo/q_t": 0.40898868441581726, "grad_norm": 74.9897232055664, "learning_rate": 1.7233819424956247e-08, "logits/chosen": -0.8332107663154602, "logits/rejected": -0.7955859303474426, "logps/chosen": -531.3854370117188, "logps/ref_chosen": -288.7687072753906, "logps/ref_rejected": -268.4986572265625, "logps/rejected": -668.48828125, "loss": 4.4275, "margin_dpo/margin_mean": 157.37295532226562, "margin_dpo/margin_std": 218.2144775390625, "step": 427 }, { "KL/chosen_KL_mean": -274.7869873046875, "KL/mean": -343.9637451171875, "KL/rejected_KL_mean": -413.14044189453125, "KL/std": 183.8684539794922, "epoch": 0.8963350785340314, "fcm_dpo/beta": 0.0024281598161906004, "fcm_dpo/delta": -0.008501682430505753, "fcm_dpo/margin": 138.35345458984375, "fcm_dpo/q_t": 0.42190542817115784, "grad_norm": 59.844512939453125, "learning_rate": 1.6572104647786245e-08, "logits/chosen": -0.791337788105011, "logits/rejected": -0.8210979700088501, "logps/chosen": -570.3079833984375, "logps/ref_chosen": -295.5209655761719, "logps/ref_rejected": -275.71026611328125, "logps/rejected": -688.8507080078125, "loss": 4.5482, "margin_dpo/margin_mean": 138.35345458984375, "margin_dpo/margin_std": 195.07057189941406, "step": 428 }, { "KL/chosen_KL_mean": -264.47540283203125, "KL/mean": -318.24371337890625, "KL/rejected_KL_mean": -372.01202392578125, "KL/std": 177.7375946044922, "epoch": 0.8984293193717278, "fcm_dpo/beta": 0.0024204240180552006, "fcm_dpo/delta": 0.012468225322663784, "fcm_dpo/margin": 107.53662109375, "fcm_dpo/q_t": 0.4388344883918762, "grad_norm": 59.98493194580078, "learning_rate": 1.5922907900227017e-08, "logits/chosen": -0.7992820739746094, "logits/rejected": -0.8066202402114868, "logps/chosen": -538.8677368164062, "logps/ref_chosen": -274.392333984375, "logps/ref_rejected": -258.574462890625, "logps/rejected": -630.5865478515625, "loss": 4.862, "margin_dpo/margin_mean": 107.53662109375, "margin_dpo/margin_std": 216.66375732421875, "step": 429 }, { "KL/chosen_KL_mean": -253.07960510253906, "KL/mean": -308.39447021484375, "KL/rejected_KL_mean": -363.7093505859375, "KL/std": 166.74659729003906, "epoch": 0.900523560209424, "fcm_dpo/beta": 0.0024694507010281086, "fcm_dpo/delta": 0.014632913284003735, "fcm_dpo/margin": 110.62974548339844, "fcm_dpo/q_t": 0.43679171800613403, "grad_norm": 72.91577911376953, "learning_rate": 1.5286263996730026e-08, "logits/chosen": -0.8794166445732117, "logits/rejected": -0.847719132900238, "logps/chosen": -541.8187255859375, "logps/ref_chosen": -288.7391357421875, "logps/ref_rejected": -268.6106262207031, "logps/rejected": -632.3199462890625, "loss": 4.756, "margin_dpo/margin_mean": 110.62974548339844, "margin_dpo/margin_std": 189.39857482910156, "step": 430 }, { "KL/chosen_KL_mean": -250.58224487304688, "KL/mean": -295.65850830078125, "KL/rejected_KL_mean": -340.7347106933594, "KL/std": 156.20620727539062, "epoch": 0.9026178010471204, "fcm_dpo/beta": 0.002479498041793704, "fcm_dpo/delta": -0.0046107154339551926, "fcm_dpo/margin": 90.15248107910156, "fcm_dpo/q_t": 0.447639524936676, "grad_norm": 50.818077087402344, "learning_rate": 1.4662207078575684e-08, "logits/chosen": -0.853556215763092, "logits/rejected": -0.8220837712287903, "logps/chosen": -526.3070068359375, "logps/ref_chosen": -275.7247314453125, "logps/ref_rejected": -268.91729736328125, "logps/rejected": -609.6519775390625, "loss": 4.9115, "margin_dpo/margin_mean": 90.1524658203125, "margin_dpo/margin_std": 178.93893432617188, "step": 431 }, { "KL/chosen_KL_mean": -233.45767211914062, "KL/mean": -291.3896484375, "KL/rejected_KL_mean": -349.32171630859375, "KL/std": 166.04766845703125, "epoch": 0.9047120418848168, "fcm_dpo/beta": 0.0024858207907527685, "fcm_dpo/delta": 0.01790335588157177, "fcm_dpo/margin": 115.8640365600586, "fcm_dpo/q_t": 0.4340059757232666, "grad_norm": 57.34628677368164, "learning_rate": 1.40507706120426e-08, "logits/chosen": -0.887722373008728, "logits/rejected": -0.8687385320663452, "logps/chosen": -524.8777465820312, "logps/ref_chosen": -291.42010498046875, "logps/ref_rejected": -255.48202514648438, "logps/rejected": -604.8037109375, "loss": 4.7118, "margin_dpo/margin_mean": 115.8640365600586, "margin_dpo/margin_std": 193.5400390625, "step": 432 }, { "KL/chosen_KL_mean": -264.11077880859375, "KL/mean": -312.563720703125, "KL/rejected_KL_mean": -361.0166931152344, "KL/std": 158.07310485839844, "epoch": 0.9068062827225131, "fcm_dpo/beta": 0.002510627033188939, "fcm_dpo/delta": 0.007871107198297977, "fcm_dpo/margin": 96.9058609008789, "fcm_dpo/q_t": 0.44348740577697754, "grad_norm": 75.00899505615234, "learning_rate": 1.345198738661285e-08, "logits/chosen": -0.8320577144622803, "logits/rejected": -0.8262537717819214, "logps/chosen": -510.33770751953125, "logps/ref_chosen": -246.2268829345703, "logps/ref_rejected": -253.65924072265625, "logps/rejected": -614.6759033203125, "loss": 4.8285, "margin_dpo/margin_mean": 96.9058609008789, "margin_dpo/margin_std": 178.91293334960938, "step": 433 }, { "KL/chosen_KL_mean": -274.1569519042969, "KL/mean": -325.5293884277344, "KL/rejected_KL_mean": -376.90179443359375, "KL/std": 164.99172973632812, "epoch": 0.9089005235602095, "fcm_dpo/beta": 0.002546996111050248, "fcm_dpo/delta": 0.02380567416548729, "fcm_dpo/margin": 102.74482727050781, "fcm_dpo/q_t": 0.43898072838783264, "grad_norm": 74.7905044555664, "learning_rate": 1.2865889513213628e-08, "logits/chosen": -0.8248652219772339, "logits/rejected": -0.8263947367668152, "logps/chosen": -569.6187744140625, "logps/ref_chosen": -295.4618225097656, "logps/ref_rejected": -256.2254333496094, "logps/rejected": -633.127197265625, "loss": 4.7812, "margin_dpo/margin_mean": 102.74482727050781, "margin_dpo/margin_std": 179.2010498046875, "step": 434 }, { "KL/chosen_KL_mean": -244.2108917236328, "KL/mean": -307.3104248046875, "KL/rejected_KL_mean": -370.40997314453125, "KL/std": 169.79331970214844, "epoch": 0.9109947643979057, "fcm_dpo/beta": 0.0025554713793098927, "fcm_dpo/delta": -0.02224489115178585, "fcm_dpo/margin": 126.19908142089844, "fcm_dpo/q_t": 0.42764154076576233, "grad_norm": 70.11904907226562, "learning_rate": 1.2292508422495157e-08, "logits/chosen": -0.8194464445114136, "logits/rejected": -0.8064465522766113, "logps/chosen": -504.9492492675781, "logps/ref_chosen": -260.7384033203125, "logps/ref_rejected": -248.5688018798828, "logps/rejected": -618.978759765625, "loss": 4.6041, "margin_dpo/margin_mean": 126.19908142089844, "margin_dpo/margin_std": 186.30056762695312, "step": 435 }, { "KL/chosen_KL_mean": -273.79022216796875, "KL/mean": -318.10125732421875, "KL/rejected_KL_mean": -362.4123840332031, "KL/std": 188.65408325195312, "epoch": 0.9130890052356021, "fcm_dpo/beta": 0.002511697355657816, "fcm_dpo/delta": -0.003989637829363346, "fcm_dpo/margin": 88.62214660644531, "fcm_dpo/q_t": 0.4518389403820038, "grad_norm": 63.78618240356445, "learning_rate": 1.1731874863145142e-08, "logits/chosen": -0.81931072473526, "logits/rejected": -0.8128557205200195, "logps/chosen": -593.1127319335938, "logps/ref_chosen": -319.3224792480469, "logps/ref_rejected": -299.30322265625, "logps/rejected": -661.715576171875, "loss": 4.9787, "margin_dpo/margin_mean": 88.62214660644531, "margin_dpo/margin_std": 211.4658966064453, "step": 436 }, { "KL/chosen_KL_mean": -249.11143493652344, "KL/mean": -312.220703125, "KL/rejected_KL_mean": -375.3299865722656, "KL/std": 183.77027893066406, "epoch": 0.9151832460732985, "fcm_dpo/beta": 0.0025260450784116983, "fcm_dpo/delta": 0.0047508468851447105, "fcm_dpo/margin": 126.21853637695312, "fcm_dpo/q_t": 0.42712879180908203, "grad_norm": 65.6551742553711, "learning_rate": 1.118401890024001e-08, "logits/chosen": -0.8493269085884094, "logits/rejected": -0.8305841684341431, "logps/chosen": -527.940185546875, "logps/ref_chosen": -278.82879638671875, "logps/ref_rejected": -272.55303955078125, "logps/rejected": -647.8829956054688, "loss": 4.6504, "margin_dpo/margin_mean": 126.21853637695312, "margin_dpo/margin_std": 204.38760375976562, "step": 437 }, { "KL/chosen_KL_mean": -263.64837646484375, "KL/mean": -293.3003845214844, "KL/rejected_KL_mean": -322.9523620605469, "KL/std": 168.18655395507812, "epoch": 0.9172774869109948, "fcm_dpo/beta": 0.0025404624175280333, "fcm_dpo/delta": 0.007852256298065186, "fcm_dpo/margin": 59.30400466918945, "fcm_dpo/q_t": 0.46499937772750854, "grad_norm": 64.0697021484375, "learning_rate": 1.06489699136324e-08, "logits/chosen": -0.8310567736625671, "logits/rejected": -0.840906023979187, "logps/chosen": -522.9674682617188, "logps/ref_chosen": -259.31903076171875, "logps/ref_rejected": -240.99581909179688, "logps/rejected": -563.9481811523438, "loss": 5.2207, "margin_dpo/margin_mean": 59.303993225097656, "margin_dpo/margin_std": 191.96351623535156, "step": 438 }, { "KL/chosen_KL_mean": -252.53065490722656, "KL/mean": -308.7279357910156, "KL/rejected_KL_mean": -364.92523193359375, "KL/std": 159.78070068359375, "epoch": 0.9193717277486911, "fcm_dpo/beta": 0.0025289033073931932, "fcm_dpo/delta": -0.005491324700415134, "fcm_dpo/margin": 112.39454650878906, "fcm_dpo/q_t": 0.4330964684486389, "grad_norm": 69.76829528808594, "learning_rate": 1.0126756596375685e-08, "logits/chosen": -0.8097501993179321, "logits/rejected": -0.8053499460220337, "logps/chosen": -509.65509033203125, "logps/ref_chosen": -257.1243896484375, "logps/ref_rejected": -243.20416259765625, "logps/rejected": -608.1293334960938, "loss": 4.6923, "margin_dpo/margin_mean": 112.39454650878906, "margin_dpo/margin_std": 177.54962158203125, "step": 439 }, { "KL/chosen_KL_mean": -271.7328186035156, "KL/mean": -313.9856872558594, "KL/rejected_KL_mean": -356.2385559082031, "KL/std": 148.13238525390625, "epoch": 0.9214659685863874, "fcm_dpo/beta": 0.0025394840631633997, "fcm_dpo/delta": 0.01721479743719101, "fcm_dpo/margin": 84.5057601928711, "fcm_dpo/q_t": 0.4492788314819336, "grad_norm": 85.56015014648438, "learning_rate": 9.617406953185136e-09, "logits/chosen": -0.8681034445762634, "logits/rejected": -0.8587910532951355, "logps/chosen": -579.2643432617188, "logps/ref_chosen": -307.5315246582031, "logps/ref_rejected": -264.3540954589844, "logps/rejected": -620.5927124023438, "loss": 4.9114, "margin_dpo/margin_mean": 84.5057601928711, "margin_dpo/margin_std": 158.16221618652344, "step": 440 }, { "KL/chosen_KL_mean": -251.61834716796875, "KL/mean": -314.0332336425781, "KL/rejected_KL_mean": -376.44805908203125, "KL/std": 164.15542602539062, "epoch": 0.9235602094240838, "fcm_dpo/beta": 0.0025592236779630184, "fcm_dpo/delta": -0.005991383455693722, "fcm_dpo/margin": 124.82971954345703, "fcm_dpo/q_t": 0.425258606672287, "grad_norm": 64.42757415771484, "learning_rate": 9.12094829893642e-09, "logits/chosen": -0.8236322402954102, "logits/rejected": -0.8174566626548767, "logps/chosen": -561.6002807617188, "logps/ref_chosen": -309.9819641113281, "logps/ref_rejected": -297.4968566894531, "logps/rejected": -673.9448852539062, "loss": 4.5766, "margin_dpo/margin_mean": 124.82971954345703, "margin_dpo/margin_std": 175.52149963378906, "step": 441 }, { "KL/chosen_KL_mean": -281.2906494140625, "KL/mean": -326.7481994628906, "KL/rejected_KL_mean": -372.2057189941406, "KL/std": 168.01153564453125, "epoch": 0.9256544502617801, "fcm_dpo/beta": 0.002559695625677705, "fcm_dpo/delta": 0.007131714373826981, "fcm_dpo/margin": 90.91503143310547, "fcm_dpo/q_t": 0.44628316164016724, "grad_norm": 72.68498229980469, "learning_rate": 8.637407257200496e-09, "logits/chosen": -0.8905078768730164, "logits/rejected": -0.8481354713439941, "logps/chosen": -560.2698364257812, "logps/ref_chosen": -278.9791564941406, "logps/ref_rejected": -242.87310791015625, "logps/rejected": -615.0787963867188, "loss": 4.9229, "margin_dpo/margin_mean": 90.91503143310547, "margin_dpo/margin_std": 197.5365753173828, "step": 442 }, { "KL/chosen_KL_mean": -262.447021484375, "KL/mean": -311.3345947265625, "KL/rejected_KL_mean": -360.22210693359375, "KL/std": 150.85263061523438, "epoch": 0.9277486910994764, "fcm_dpo/beta": 0.002550204750150442, "fcm_dpo/delta": -0.021942952647805214, "fcm_dpo/margin": 97.77508544921875, "fcm_dpo/q_t": 0.44047561287879944, "grad_norm": 80.34606170654297, "learning_rate": 8.166809758815895e-09, "logits/chosen": -0.7915633320808411, "logits/rejected": -0.8125393390655518, "logps/chosen": -536.006103515625, "logps/ref_chosen": -273.5590515136719, "logps/ref_rejected": -264.0199279785156, "logps/rejected": -624.2420043945312, "loss": 4.8309, "margin_dpo/margin_mean": 97.77508544921875, "margin_dpo/margin_std": 177.55967712402344, "step": 443 }, { "KL/chosen_KL_mean": -255.01412963867188, "KL/mean": -312.3970031738281, "KL/rejected_KL_mean": -369.7798767089844, "KL/std": 177.91873168945312, "epoch": 0.9298429319371728, "fcm_dpo/beta": 0.002535287756472826, "fcm_dpo/delta": 0.018113907426595688, "fcm_dpo/margin": 114.76580047607422, "fcm_dpo/q_t": 0.43355995416641235, "grad_norm": 61.830787658691406, "learning_rate": 7.709181040498253e-09, "logits/chosen": -0.7957402467727661, "logits/rejected": -0.7812896370887756, "logps/chosen": -553.1583251953125, "logps/ref_chosen": -298.1441955566406, "logps/ref_rejected": -268.0572814941406, "logps/rejected": -637.837158203125, "loss": 4.7328, "margin_dpo/margin_mean": 114.76579284667969, "margin_dpo/margin_std": 204.9227294921875, "step": 444 }, { "KL/chosen_KL_mean": -255.86961364746094, "KL/mean": -306.81475830078125, "KL/rejected_KL_mean": -357.7599182128906, "KL/std": 171.42166137695312, "epoch": 0.9319371727748691, "fcm_dpo/beta": 0.0025360870640724897, "fcm_dpo/delta": -0.026608020067214966, "fcm_dpo/margin": 101.89030456542969, "fcm_dpo/q_t": 0.44091877341270447, "grad_norm": 75.62554168701172, "learning_rate": 7.2645456434869965e-09, "logits/chosen": -0.876228392124176, "logits/rejected": -0.8805335760116577, "logps/chosen": -510.4102783203125, "logps/ref_chosen": -254.54067993164062, "logps/ref_rejected": -264.2445983886719, "logps/rejected": -622.0044555664062, "loss": 4.8553, "margin_dpo/margin_mean": 101.89029693603516, "margin_dpo/margin_std": 197.63458251953125, "step": 445 }, { "KL/chosen_KL_mean": -250.0303955078125, "KL/mean": -306.152587890625, "KL/rejected_KL_mean": -362.2747497558594, "KL/std": 161.83197021484375, "epoch": 0.9340314136125655, "fcm_dpo/beta": 0.002476347843185067, "fcm_dpo/delta": -0.010473774746060371, "fcm_dpo/margin": 112.24434661865234, "fcm_dpo/q_t": 0.4361517131328583, "grad_norm": 49.15285110473633, "learning_rate": 6.832927412229017e-09, "logits/chosen": -0.7973624467849731, "logits/rejected": -0.8008688688278198, "logps/chosen": -556.7528686523438, "logps/ref_chosen": -306.72247314453125, "logps/ref_rejected": -266.3735656738281, "logps/rejected": -628.6482543945312, "loss": 4.7508, "margin_dpo/margin_mean": 112.24434661865234, "margin_dpo/margin_std": 195.67247009277344, "step": 446 }, { "KL/chosen_KL_mean": -246.97784423828125, "KL/mean": -308.9893798828125, "KL/rejected_KL_mean": -371.0009460449219, "KL/std": 164.9072265625, "epoch": 0.9361256544502617, "fcm_dpo/beta": 0.002457038965076208, "fcm_dpo/delta": -0.005904083605855703, "fcm_dpo/margin": 124.02310180664062, "fcm_dpo/q_t": 0.4291154742240906, "grad_norm": 69.2431640625, "learning_rate": 6.414349493100129e-09, "logits/chosen": -0.7988148331642151, "logits/rejected": -0.7919713258743286, "logps/chosen": -507.4951171875, "logps/ref_chosen": -260.51727294921875, "logps/ref_rejected": -236.47061157226562, "logps/rejected": -607.4715576171875, "loss": 4.6405, "margin_dpo/margin_mean": 124.02310180664062, "margin_dpo/margin_std": 185.88461303710938, "step": 447 }, { "KL/chosen_KL_mean": -260.0938415527344, "KL/mean": -320.5010070800781, "KL/rejected_KL_mean": -380.908203125, "KL/std": 160.903076171875, "epoch": 0.9382198952879581, "fcm_dpo/beta": 0.0024337496142834425, "fcm_dpo/delta": -0.020262887701392174, "fcm_dpo/margin": 120.81436157226562, "fcm_dpo/q_t": 0.43088439106941223, "grad_norm": 60.84456253051758, "learning_rate": 6.0088343331638756e-09, "logits/chosen": -0.8034505248069763, "logits/rejected": -0.7938514351844788, "logps/chosen": -528.880859375, "logps/ref_chosen": -268.78704833984375, "logps/ref_rejected": -262.1703796386719, "logps/rejected": -643.0785522460938, "loss": 4.6526, "margin_dpo/margin_mean": 120.81436157226562, "margin_dpo/margin_std": 174.54934692382812, "step": 448 }, { "KL/chosen_KL_mean": -247.74948120117188, "KL/mean": -304.65057373046875, "KL/rejected_KL_mean": -361.5516052246094, "KL/std": 163.8621826171875, "epoch": 0.9403141361256544, "fcm_dpo/beta": 0.002439489820972085, "fcm_dpo/delta": 0.02136923372745514, "fcm_dpo/margin": 113.80213928222656, "fcm_dpo/q_t": 0.4335501194000244, "grad_norm": 85.0506362915039, "learning_rate": 5.616403678967624e-09, "logits/chosen": -0.9094984531402588, "logits/rejected": -0.8936970829963684, "logps/chosen": -578.700927734375, "logps/ref_chosen": -330.9514465332031, "logps/ref_rejected": -239.76974487304688, "logps/rejected": -601.3213500976562, "loss": 4.76, "margin_dpo/margin_mean": 113.80213928222656, "margin_dpo/margin_std": 199.1973114013672, "step": 449 }, { "KL/chosen_KL_mean": -270.36553955078125, "KL/mean": -320.8754577636719, "KL/rejected_KL_mean": -371.3853454589844, "KL/std": 169.2615203857422, "epoch": 0.9424083769633508, "fcm_dpo/beta": 0.0024614077992737293, "fcm_dpo/delta": -0.004837765824049711, "fcm_dpo/margin": 101.0197982788086, "fcm_dpo/q_t": 0.44178810715675354, "grad_norm": 61.14320373535156, "learning_rate": 5.2370785753763356e-09, "logits/chosen": -0.8040182590484619, "logits/rejected": -0.7969501614570618, "logps/chosen": -554.6309814453125, "logps/ref_chosen": -284.26544189453125, "logps/ref_rejected": -250.5401611328125, "logps/rejected": -621.925537109375, "loss": 4.8008, "margin_dpo/margin_mean": 101.0197982788086, "margin_dpo/margin_std": 170.01528930664062, "step": 450 }, { "KL/chosen_KL_mean": -279.3070068359375, "KL/mean": -320.45257568359375, "KL/rejected_KL_mean": -361.5981140136719, "KL/std": 158.63836669921875, "epoch": 0.9445026178010472, "fcm_dpo/beta": 0.0024547951761633158, "fcm_dpo/delta": 0.0002841167151927948, "fcm_dpo/margin": 82.29108428955078, "fcm_dpo/q_t": 0.4520818889141083, "grad_norm": 81.18460845947266, "learning_rate": 4.8708793644441086e-09, "logits/chosen": -0.8152300715446472, "logits/rejected": -0.7828786373138428, "logps/chosen": -581.6279296875, "logps/ref_chosen": -302.3209228515625, "logps/ref_rejected": -254.09747314453125, "logps/rejected": -615.6956176757812, "loss": 4.9711, "margin_dpo/margin_mean": 82.29108428955078, "margin_dpo/margin_std": 171.24998474121094, "step": 451 }, { "KL/chosen_KL_mean": -255.58740234375, "KL/mean": -323.3721008300781, "KL/rejected_KL_mean": -391.1568603515625, "KL/std": 196.24913024902344, "epoch": 0.9465968586387434, "fcm_dpo/beta": 0.002398530952632427, "fcm_dpo/delta": -0.03113538958132267, "fcm_dpo/margin": 135.5694580078125, "fcm_dpo/q_t": 0.42863401770591736, "grad_norm": 50.24382781982422, "learning_rate": 4.517825684323323e-09, "logits/chosen": -0.8734545707702637, "logits/rejected": -0.846143364906311, "logps/chosen": -554.9795532226562, "logps/ref_chosen": -299.39215087890625, "logps/ref_rejected": -284.3475036621094, "logps/rejected": -675.50439453125, "loss": 4.6565, "margin_dpo/margin_mean": 135.5694580078125, "margin_dpo/margin_std": 229.8317108154297, "step": 452 }, { "KL/chosen_KL_mean": -275.34716796875, "KL/mean": -332.12860107421875, "KL/rejected_KL_mean": -388.9101257324219, "KL/std": 167.65248107910156, "epoch": 0.9486910994764398, "fcm_dpo/beta": 0.0024174817372113466, "fcm_dpo/delta": 0.01900402083992958, "fcm_dpo/margin": 113.56294250488281, "fcm_dpo/q_t": 0.4362477958202362, "grad_norm": 65.24600982666016, "learning_rate": 4.1779364682113794e-09, "logits/chosen": -0.8175550699234009, "logits/rejected": -0.8079196214675903, "logps/chosen": -599.9988403320312, "logps/ref_chosen": -324.6517028808594, "logps/ref_rejected": -304.1527099609375, "logps/rejected": -693.0628051757812, "loss": 4.7582, "margin_dpo/margin_mean": 113.56295013427734, "margin_dpo/margin_std": 204.01402282714844, "step": 453 }, { "KL/chosen_KL_mean": -261.29437255859375, "KL/mean": -329.093994140625, "KL/rejected_KL_mean": -396.8935546875, "KL/std": 192.02064514160156, "epoch": 0.9507853403141361, "fcm_dpo/beta": 0.002420138567686081, "fcm_dpo/delta": -0.012020056135952473, "fcm_dpo/margin": 135.59918212890625, "fcm_dpo/q_t": 0.42454957962036133, "grad_norm": 73.93843841552734, "learning_rate": 3.851229943335393e-09, "logits/chosen": -0.8420263528823853, "logits/rejected": -0.8489396572113037, "logps/chosen": -560.9061279296875, "logps/ref_chosen": -299.6117248535156, "logps/ref_rejected": -303.74224853515625, "logps/rejected": -700.6358032226562, "loss": 4.6019, "margin_dpo/margin_mean": 135.59918212890625, "margin_dpo/margin_std": 211.62002563476562, "step": 454 }, { "KL/chosen_KL_mean": -254.47279357910156, "KL/mean": -297.5650939941406, "KL/rejected_KL_mean": -340.65740966796875, "KL/std": 164.42532348632812, "epoch": 0.9528795811518325, "fcm_dpo/beta": 0.002410230925306678, "fcm_dpo/delta": -0.0064119016751646996, "fcm_dpo/margin": 86.18463897705078, "fcm_dpo/q_t": 0.45259642601013184, "grad_norm": 60.235843658447266, "learning_rate": 3.5377236299748147e-09, "logits/chosen": -0.8099350333213806, "logits/rejected": -0.8183305263519287, "logps/chosen": -528.08447265625, "logps/ref_chosen": -273.6116943359375, "logps/ref_rejected": -274.4293518066406, "logps/rejected": -615.0867919921875, "loss": 4.9766, "margin_dpo/margin_mean": 86.18463897705078, "margin_dpo/margin_std": 188.7678985595703, "step": 455 }, { "KL/chosen_KL_mean": -284.4190979003906, "KL/mean": -342.2475280761719, "KL/rejected_KL_mean": -400.07598876953125, "KL/std": 191.12667846679688, "epoch": 0.9549738219895288, "fcm_dpo/beta": 0.0024038502015173435, "fcm_dpo/delta": -0.0050141820684075356, "fcm_dpo/margin": 115.6568374633789, "fcm_dpo/q_t": 0.4384552836418152, "grad_norm": 72.66372680664062, "learning_rate": 3.2374343405217884e-09, "logits/chosen": -0.7380684614181519, "logits/rejected": -0.7515442967414856, "logps/chosen": -606.591064453125, "logps/ref_chosen": -322.17193603515625, "logps/ref_rejected": -294.54461669921875, "logps/rejected": -694.6205444335938, "loss": 4.7994, "margin_dpo/margin_mean": 115.6568374633789, "margin_dpo/margin_std": 224.9918212890625, "step": 456 }, { "KL/chosen_KL_mean": -275.81463623046875, "KL/mean": -339.070068359375, "KL/rejected_KL_mean": -402.3255615234375, "KL/std": 183.56442260742188, "epoch": 0.9570680628272251, "fcm_dpo/beta": 0.0023992531932890415, "fcm_dpo/delta": 0.008432567119598389, "fcm_dpo/margin": 126.51089477539062, "fcm_dpo/q_t": 0.4272172451019287, "grad_norm": 64.16017150878906, "learning_rate": 2.9503781785795713e-09, "logits/chosen": -0.8137893080711365, "logits/rejected": -0.8142549991607666, "logps/chosen": -583.6109008789062, "logps/ref_chosen": -307.7962341308594, "logps/ref_rejected": -274.5501403808594, "logps/rejected": -676.8756713867188, "loss": 4.6952, "margin_dpo/margin_mean": 126.51089477539062, "margin_dpo/margin_std": 209.9931182861328, "step": 457 }, { "KL/chosen_KL_mean": -270.3670349121094, "KL/mean": -323.4082946777344, "KL/rejected_KL_mean": -376.44952392578125, "KL/std": 184.91030883789062, "epoch": 0.9591623036649215, "fcm_dpo/beta": 0.002431598724797368, "fcm_dpo/delta": 0.017081839963793755, "fcm_dpo/margin": 106.08248901367188, "fcm_dpo/q_t": 0.4409106373786926, "grad_norm": 64.75917053222656, "learning_rate": 2.6765705380989432e-09, "logits/chosen": -0.8364468812942505, "logits/rejected": -0.8125419020652771, "logps/chosen": -567.398681640625, "logps/ref_chosen": -297.0316467285156, "logps/ref_rejected": -276.1112365722656, "logps/rejected": -652.560791015625, "loss": 4.8427, "margin_dpo/margin_mean": 106.08248901367188, "margin_dpo/margin_std": 206.88600158691406, "step": 458 }, { "KL/chosen_KL_mean": -255.69676208496094, "KL/mean": -301.064697265625, "KL/rejected_KL_mean": -346.4325866699219, "KL/std": 156.7518768310547, "epoch": 0.9612565445026178, "fcm_dpo/beta": 0.002469552680850029, "fcm_dpo/delta": 0.03057432547211647, "fcm_dpo/margin": 90.73584747314453, "fcm_dpo/q_t": 0.44875121116638184, "grad_norm": 76.80902862548828, "learning_rate": 2.416026102552732e-09, "logits/chosen": -0.8981096148490906, "logits/rejected": -0.8896926641464233, "logps/chosen": -549.2220458984375, "logps/ref_chosen": -293.5252990722656, "logps/ref_rejected": -289.30126953125, "logps/rejected": -635.7338256835938, "loss": 4.9246, "margin_dpo/margin_mean": 90.73584747314453, "margin_dpo/margin_std": 187.47048950195312, "step": 459 }, { "KL/chosen_KL_mean": -263.8416442871094, "KL/mean": -310.0146789550781, "KL/rejected_KL_mean": -356.18768310546875, "KL/std": 150.35638427734375, "epoch": 0.9633507853403142, "fcm_dpo/beta": 0.0025106584653258324, "fcm_dpo/delta": 0.018712403252720833, "fcm_dpo/margin": 92.34601593017578, "fcm_dpo/q_t": 0.44546929001808167, "grad_norm": 55.63397979736328, "learning_rate": 2.168758844148272e-09, "logits/chosen": -0.8555883169174194, "logits/rejected": -0.8532405495643616, "logps/chosen": -582.6220092773438, "logps/ref_chosen": -318.7803649902344, "logps/ref_rejected": -258.7906799316406, "logps/rejected": -614.9783325195312, "loss": 4.8808, "margin_dpo/margin_mean": 92.34600830078125, "margin_dpo/margin_std": 182.1141357421875, "step": 460 }, { "KL/chosen_KL_mean": -259.0890197753906, "KL/mean": -315.6316833496094, "KL/rejected_KL_mean": -372.1743469238281, "KL/std": 172.29824829101562, "epoch": 0.9654450261780104, "fcm_dpo/beta": 0.002565775066614151, "fcm_dpo/delta": 0.022872814908623695, "fcm_dpo/margin": 113.08531188964844, "fcm_dpo/q_t": 0.4341578781604767, "grad_norm": 60.84050750732422, "learning_rate": 1.9347820230782295e-09, "logits/chosen": -0.81836998462677, "logits/rejected": -0.8396799564361572, "logps/chosen": -502.9990234375, "logps/ref_chosen": -243.9099884033203, "logps/ref_rejected": -232.6382293701172, "logps/rejected": -604.8125610351562, "loss": 4.7443, "margin_dpo/margin_mean": 113.08531188964844, "margin_dpo/margin_std": 204.30162048339844, "step": 461 }, { "KL/chosen_KL_mean": -262.39306640625, "KL/mean": -323.4773864746094, "KL/rejected_KL_mean": -384.56170654296875, "KL/std": 163.00491333007812, "epoch": 0.9675392670157068, "fcm_dpo/beta": 0.002616667188704014, "fcm_dpo/delta": 0.022976616397500038, "fcm_dpo/margin": 122.16867065429688, "fcm_dpo/q_t": 0.4264695942401886, "grad_norm": 67.28462219238281, "learning_rate": 1.7141081868094209e-09, "logits/chosen": -0.8513904213905334, "logits/rejected": -0.7988516688346863, "logps/chosen": -606.4840698242188, "logps/ref_chosen": -344.09100341796875, "logps/ref_rejected": -252.45037841796875, "logps/rejected": -637.0120849609375, "loss": 4.6085, "margin_dpo/margin_mean": 122.16866302490234, "margin_dpo/margin_std": 185.75375366210938, "step": 462 }, { "KL/chosen_KL_mean": -260.8435974121094, "KL/mean": -307.8876037597656, "KL/rejected_KL_mean": -354.93157958984375, "KL/std": 176.79953002929688, "epoch": 0.9696335078534032, "fcm_dpo/beta": 0.002635349053889513, "fcm_dpo/delta": 0.00047087250277400017, "fcm_dpo/margin": 94.08798217773438, "fcm_dpo/q_t": 0.4436490535736084, "grad_norm": 87.89910888671875, "learning_rate": 1.5067491694100153e-09, "logits/chosen": -0.8419432640075684, "logits/rejected": -0.7984883189201355, "logps/chosen": -557.986083984375, "logps/ref_chosen": -297.1424560546875, "logps/ref_rejected": -234.0208282470703, "logps/rejected": -588.9524536132812, "loss": 4.9332, "margin_dpo/margin_mean": 94.08797454833984, "margin_dpo/margin_std": 211.70285034179688, "step": 463 }, { "KL/chosen_KL_mean": -269.11505126953125, "KL/mean": -319.7918701171875, "KL/rejected_KL_mean": -370.46868896484375, "KL/std": 167.46054077148438, "epoch": 0.9717277486910995, "fcm_dpo/beta": 0.0026447109412401915, "fcm_dpo/delta": 0.0007200678810477257, "fcm_dpo/margin": 101.35370635986328, "fcm_dpo/q_t": 0.43749621510505676, "grad_norm": 58.30624008178711, "learning_rate": 1.3127160909147672e-09, "logits/chosen": -0.8257731199264526, "logits/rejected": -0.8404132723808289, "logps/chosen": -534.8258056640625, "logps/ref_chosen": -265.71075439453125, "logps/ref_rejected": -256.4108581542969, "logps/rejected": -626.8795776367188, "loss": 4.7914, "margin_dpo/margin_mean": 101.35370635986328, "margin_dpo/margin_std": 186.4237823486328, "step": 464 }, { "KL/chosen_KL_mean": -246.98736572265625, "KL/mean": -310.69110107421875, "KL/rejected_KL_mean": -374.39483642578125, "KL/std": 160.25502014160156, "epoch": 0.9738219895287958, "fcm_dpo/beta": 0.002568161580711603, "fcm_dpo/delta": -0.03626430779695511, "fcm_dpo/margin": 127.40748596191406, "fcm_dpo/q_t": 0.42483144998550415, "grad_norm": 69.23091888427734, "learning_rate": 1.1320193567288527e-09, "logits/chosen": -0.8961881995201111, "logits/rejected": -0.8565638065338135, "logps/chosen": -540.1400756835938, "logps/ref_chosen": -293.1527404785156, "logps/ref_rejected": -293.70947265625, "logps/rejected": -668.1043701171875, "loss": 4.5974, "margin_dpo/margin_mean": 127.40748596191406, "margin_dpo/margin_std": 186.34690856933594, "step": 465 }, { "KL/chosen_KL_mean": -234.4508819580078, "KL/mean": -300.031982421875, "KL/rejected_KL_mean": -365.6130676269531, "KL/std": 157.27207946777344, "epoch": 0.9759162303664921, "fcm_dpo/beta": 0.0025337152183055878, "fcm_dpo/delta": -0.0182378850877285, "fcm_dpo/margin": 131.1621551513672, "fcm_dpo/q_t": 0.4229571223258972, "grad_norm": 57.641929626464844, "learning_rate": 9.64668657069706e-10, "logits/chosen": -0.8110611438751221, "logits/rejected": -0.7530339956283569, "logps/chosen": -495.928466796875, "logps/ref_chosen": -261.4775695800781, "logps/ref_rejected": -248.36282348632812, "logps/rejected": -613.9758911132812, "loss": 4.5442, "margin_dpo/margin_mean": 131.1621551513672, "margin_dpo/margin_std": 181.0853271484375, "step": 466 }, { "KL/chosen_KL_mean": -262.05810546875, "KL/mean": -312.107421875, "KL/rejected_KL_mean": -362.1567687988281, "KL/std": 167.3401641845703, "epoch": 0.9780104712041885, "fcm_dpo/beta": 0.0024951626546680927, "fcm_dpo/delta": -0.015738535672426224, "fcm_dpo/margin": 100.09864807128906, "fcm_dpo/q_t": 0.4429374635219574, "grad_norm": 75.20722961425781, "learning_rate": 8.106729664475176e-10, "logits/chosen": -0.8004065752029419, "logits/rejected": -0.7925856709480286, "logps/chosen": -528.412353515625, "logps/ref_chosen": -266.354248046875, "logps/ref_rejected": -277.76324462890625, "logps/rejected": -639.9200439453125, "loss": 4.8735, "margin_dpo/margin_mean": 100.09864807128906, "margin_dpo/margin_std": 204.54727172851562, "step": 467 }, { "KL/chosen_KL_mean": -262.4240417480469, "KL/mean": -304.2724914550781, "KL/rejected_KL_mean": -346.12091064453125, "KL/std": 160.87689208984375, "epoch": 0.9801047120418848, "fcm_dpo/beta": 0.0025261654518544674, "fcm_dpo/delta": 0.0298687182366848, "fcm_dpo/margin": 83.69690704345703, "fcm_dpo/q_t": 0.4495968222618103, "grad_norm": 70.44644165039062, "learning_rate": 6.700405431837585e-10, "logits/chosen": -0.8833749294281006, "logits/rejected": -0.8521139025688171, "logps/chosen": -580.38720703125, "logps/ref_chosen": -317.9631652832031, "logps/ref_rejected": -261.8744201660156, "logps/rejected": -607.9953002929688, "loss": 5.0089, "margin_dpo/margin_mean": 83.69690704345703, "margin_dpo/margin_std": 198.9070281982422, "step": 468 }, { "KL/chosen_KL_mean": -246.26190185546875, "KL/mean": -308.28466796875, "KL/rejected_KL_mean": -370.3074951171875, "KL/std": 168.65597534179688, "epoch": 0.9821989528795811, "fcm_dpo/beta": 0.0025742819998413324, "fcm_dpo/delta": 0.003947127610445023, "fcm_dpo/margin": 124.04557800292969, "fcm_dpo/q_t": 0.42625945806503296, "grad_norm": 52.03805923461914, "learning_rate": 5.427789289685347e-10, "logits/chosen": -0.8123364448547363, "logits/rejected": -0.795385479927063, "logps/chosen": -571.148681640625, "logps/ref_chosen": -324.8868103027344, "logps/ref_rejected": -264.0421447753906, "logps/rejected": -634.349609375, "loss": 4.6346, "margin_dpo/margin_mean": 124.04557800292969, "margin_dpo/margin_std": 199.66555786132812, "step": 469 }, { "KL/chosen_KL_mean": -248.92559814453125, "KL/mean": -309.4586181640625, "KL/rejected_KL_mean": -369.9916076660156, "KL/std": 166.44065856933594, "epoch": 0.9842931937172775, "fcm_dpo/beta": 0.0025518168695271015, "fcm_dpo/delta": -0.006509192753583193, "fcm_dpo/margin": 121.06607818603516, "fcm_dpo/q_t": 0.4286976456642151, "grad_norm": 82.49861145019531, "learning_rate": 4.288949484559934e-10, "logits/chosen": -0.8041696548461914, "logits/rejected": -0.8020331263542175, "logps/chosen": -563.6297607421875, "logps/ref_chosen": -314.7042541503906, "logps/ref_rejected": -259.2276611328125, "logps/rejected": -629.2192993164062, "loss": 4.6557, "margin_dpo/margin_mean": 121.06607818603516, "margin_dpo/margin_std": 191.92596435546875, "step": 470 }, { "KL/chosen_KL_mean": -265.6898498535156, "KL/mean": -325.8057556152344, "KL/rejected_KL_mean": -385.921630859375, "KL/std": 194.99688720703125, "epoch": 0.9863874345549738, "fcm_dpo/beta": 0.0025341196451336145, "fcm_dpo/delta": -0.01957012340426445, "fcm_dpo/margin": 120.23179626464844, "fcm_dpo/q_t": 0.43427878618240356, "grad_norm": 76.36981201171875, "learning_rate": 3.2839470889836627e-10, "logits/chosen": -0.8662016987800598, "logits/rejected": -0.8508539199829102, "logps/chosen": -558.2646484375, "logps/ref_chosen": -292.5748291015625, "logps/ref_rejected": -298.7525329589844, "logps/rejected": -684.6741333007812, "loss": 4.7386, "margin_dpo/margin_mean": 120.23179626464844, "margin_dpo/margin_std": 221.61729431152344, "step": 471 }, { "KL/chosen_KL_mean": -239.04766845703125, "KL/mean": -306.37890625, "KL/rejected_KL_mean": -373.710205078125, "KL/std": 166.82215881347656, "epoch": 0.9884816753926702, "fcm_dpo/beta": 0.0024978769943118095, "fcm_dpo/delta": 0.0006190729327499866, "fcm_dpo/margin": 134.66250610351562, "fcm_dpo/q_t": 0.4230276346206665, "grad_norm": 88.5049057006836, "learning_rate": 2.412835998185092e-10, "logits/chosen": -0.8472277522087097, "logits/rejected": -0.8603392839431763, "logps/chosen": -482.4215087890625, "logps/ref_chosen": -243.37380981445312, "logps/ref_rejected": -251.12109375, "logps/rejected": -624.831298828125, "loss": 4.5638, "margin_dpo/margin_mean": 134.66250610351562, "margin_dpo/margin_std": 198.72836303710938, "step": 472 }, { "KL/chosen_KL_mean": -256.0351867675781, "KL/mean": -317.4332275390625, "KL/rejected_KL_mean": -378.8313293457031, "KL/std": 152.7645263671875, "epoch": 0.9905759162303664, "fcm_dpo/beta": 0.0025325005408376455, "fcm_dpo/delta": 0.027150386944413185, "fcm_dpo/margin": 122.79612731933594, "fcm_dpo/q_t": 0.4268743097782135, "grad_norm": 49.66973114013672, "learning_rate": 1.6756629272085544e-10, "logits/chosen": -0.7995487451553345, "logits/rejected": -0.7962360978126526, "logps/chosen": -542.36376953125, "logps/ref_chosen": -286.3286437988281, "logps/ref_rejected": -258.6535339355469, "logps/rejected": -637.4849243164062, "loss": 4.5916, "margin_dpo/margin_mean": 122.79612731933594, "margin_dpo/margin_std": 171.07659912109375, "step": 473 }, { "KL/chosen_KL_mean": -259.002685546875, "KL/mean": -313.4521179199219, "KL/rejected_KL_mean": -367.90155029296875, "KL/std": 170.04241943359375, "epoch": 0.9926701570680628, "fcm_dpo/beta": 0.002547986339777708, "fcm_dpo/delta": -0.010679654777050018, "fcm_dpo/margin": 108.89888000488281, "fcm_dpo/q_t": 0.4356432557106018, "grad_norm": 75.55198669433594, "learning_rate": 1.072467408408384e-10, "logits/chosen": -0.8527939319610596, "logits/rejected": -0.8474220633506775, "logps/chosen": -547.0923461914062, "logps/ref_chosen": -288.08966064453125, "logps/ref_rejected": -266.69696044921875, "logps/rejected": -634.5985717773438, "loss": 4.7386, "margin_dpo/margin_mean": 108.89888763427734, "margin_dpo/margin_std": 182.4081573486328, "step": 474 }, { "KL/chosen_KL_mean": -241.1262969970703, "KL/mean": -290.3180236816406, "KL/rejected_KL_mean": -339.50970458984375, "KL/std": 152.7301483154297, "epoch": 0.9947643979057592, "fcm_dpo/beta": 0.0025214108172804117, "fcm_dpo/delta": -0.01645851507782936, "fcm_dpo/margin": 98.38336181640625, "fcm_dpo/q_t": 0.4433209002017975, "grad_norm": 57.4051628112793, "learning_rate": 6.032817893297793e-11, "logits/chosen": -0.8317741751670837, "logits/rejected": -0.84046870470047, "logps/chosen": -497.1293640136719, "logps/ref_chosen": -256.0030517578125, "logps/ref_rejected": -244.50660705566406, "logps/rejected": -584.0162963867188, "loss": 4.824, "margin_dpo/margin_mean": 98.38336181640625, "margin_dpo/margin_std": 176.57955932617188, "step": 475 }, { "KL/chosen_KL_mean": -242.7122039794922, "KL/mean": -295.69720458984375, "KL/rejected_KL_mean": -348.68218994140625, "KL/std": 153.21731567382812, "epoch": 0.9968586387434555, "fcm_dpo/beta": 0.0024653631262481213, "fcm_dpo/delta": -0.029824629426002502, "fcm_dpo/margin": 105.96993255615234, "fcm_dpo/q_t": 0.44073987007141113, "grad_norm": 54.92226028442383, "learning_rate": 2.6813123097352287e-11, "logits/chosen": -0.8974910378456116, "logits/rejected": -0.8506391048431396, "logps/chosen": -564.1797485351562, "logps/ref_chosen": -321.467529296875, "logps/ref_rejected": -295.0592956542969, "logps/rejected": -643.7415161132812, "loss": 4.8239, "margin_dpo/margin_mean": 105.96994018554688, "margin_dpo/margin_std": 196.14195251464844, "step": 476 }, { "KL/chosen_KL_mean": -268.98797607421875, "KL/mean": -318.58404541015625, "KL/rejected_KL_mean": -368.18011474609375, "KL/std": 177.54359436035156, "epoch": 0.9989528795811519, "fcm_dpo/beta": 0.002445072866976261, "fcm_dpo/delta": -0.009669403545558453, "fcm_dpo/margin": 99.1921615600586, "fcm_dpo/q_t": 0.44362398982048035, "grad_norm": 69.17218017578125, "learning_rate": 6.7033706447061635e-12, "logits/chosen": -0.7706684470176697, "logits/rejected": -0.7763949632644653, "logps/chosen": -545.7819213867188, "logps/ref_chosen": -276.7939758300781, "logps/ref_rejected": -244.82919311523438, "logps/rejected": -613.00927734375, "loss": 4.9399, "margin_dpo/margin_mean": 99.19215393066406, "margin_dpo/margin_std": 223.55364990234375, "step": 477 }, { "epoch": 0.9989528795811519, "step": 477, "total_flos": 0.0, "train_loss": 4.88172167052263, "train_runtime": 7274.147, "train_samples_per_second": 8.404, "train_steps_per_second": 0.066 } ], "logging_steps": 1, "max_steps": 477, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }