{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9989528795811519, "eval_steps": 200, "global_step": 477, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": -0.05443763732910156, "KL/mean": -0.04046645760536194, "KL/rejected_KL_mean": -0.026498794555664062, "KL/std": 0.47091516852378845, "epoch": 0.0020942408376963353, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02793481945991516, "fcm_dpo/q_t": 0.500069797039032, "grad_norm": 28.589427947998047, "learning_rate": 0.0, "logits/chosen": -0.5898098945617676, "logits/rejected": -0.604260265827179, "logps/chosen": -275.28570556640625, "logps/ref_chosen": -275.2312927246094, "logps/ref_rejected": -222.9380340576172, "logps/rejected": -222.96453857421875, "loss": 5.5463, "margin_dpo/margin_mean": -0.02793477475643158, "margin_dpo/margin_std": 0.5724214911460876, "step": 1 }, { "KL/chosen_KL_mean": 0.04458808898925781, "KL/mean": 0.0374295711517334, "KL/rejected_KL_mean": 0.030269622802734375, "KL/std": 0.49036121368408203, "epoch": 0.004188481675392671, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014312177896499634, "fcm_dpo/q_t": 0.4999642074108124, "grad_norm": 27.880699157714844, "learning_rate": 1.0416666666666666e-08, "logits/chosen": -0.6574729681015015, "logits/rejected": -0.6464410424232483, "logps/chosen": -264.7165222167969, "logps/ref_chosen": -264.7611083984375, "logps/ref_rejected": -242.5597686767578, "logps/rejected": -242.52951049804688, "loss": 5.5446, "margin_dpo/margin_mean": 0.014312252402305603, "margin_dpo/margin_std": 0.6423971652984619, "step": 2 }, { "KL/chosen_KL_mean": 0.02648162841796875, "KL/mean": -0.001500651240348816, "KL/rejected_KL_mean": -0.029489517211914062, "KL/std": 0.49069744348526, "epoch": 0.0062827225130890054, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05598026514053345, "fcm_dpo/q_t": 0.499860018491745, "grad_norm": 25.799875259399414, "learning_rate": 2.083333333333333e-08, "logits/chosen": -0.6840239763259888, "logits/rejected": -0.7351561784744263, "logps/chosen": -274.0754089355469, "logps/ref_chosen": -274.1018981933594, "logps/ref_rejected": -286.5882568359375, "logps/rejected": -286.61773681640625, "loss": 5.543, "margin_dpo/margin_mean": 0.05597783625125885, "margin_dpo/margin_std": 0.6958938241004944, "step": 3 }, { "KL/chosen_KL_mean": -0.060821533203125, "KL/mean": -0.044119708240032196, "KL/rejected_KL_mean": -0.027416229248046875, "KL/std": 0.5412886142730713, "epoch": 0.008376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03339381515979767, "fcm_dpo/q_t": 0.5000834465026855, "grad_norm": 31.741003036499023, "learning_rate": 3.125e-08, "logits/chosen": -0.6172194480895996, "logits/rejected": -0.6116077899932861, "logps/chosen": -329.8990478515625, "logps/ref_chosen": -329.8382568359375, "logps/ref_rejected": -303.2850646972656, "logps/rejected": -303.3124694824219, "loss": 5.5466, "margin_dpo/margin_mean": -0.0333935022354126, "margin_dpo/margin_std": 0.7534641623497009, "step": 4 }, { "KL/chosen_KL_mean": -0.06523323059082031, "KL/mean": -0.0311996191740036, "KL/rejected_KL_mean": 0.0028362274169921875, "KL/std": 0.5405311584472656, "epoch": 0.010471204188481676, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06806734204292297, "fcm_dpo/q_t": 0.500170111656189, "grad_norm": 29.559877395629883, "learning_rate": 4.166666666666666e-08, "logits/chosen": -0.5798951387405396, "logits/rejected": -0.5972731113433838, "logps/chosen": -301.80419921875, "logps/ref_chosen": -301.7389221191406, "logps/ref_rejected": -274.7654724121094, "logps/rejected": -274.76263427734375, "loss": 5.548, "margin_dpo/margin_mean": -0.0680682510137558, "margin_dpo/margin_std": 0.787623941898346, "step": 5 }, { "KL/chosen_KL_mean": 0.036251068115234375, "KL/mean": -0.007517039775848389, "KL/rejected_KL_mean": -0.0512847900390625, "KL/std": 0.5245298743247986, "epoch": 0.012565445026178011, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08753737807273865, "fcm_dpo/q_t": 0.49978113174438477, "grad_norm": 28.30576515197754, "learning_rate": 5.208333333333333e-08, "logits/chosen": -0.6791407465934753, "logits/rejected": -0.6419384479522705, "logps/chosen": -285.658447265625, "logps/ref_chosen": -285.6946716308594, "logps/ref_rejected": -245.8200225830078, "logps/rejected": -245.87130737304688, "loss": 5.5417, "margin_dpo/margin_mean": 0.08753781020641327, "margin_dpo/margin_std": 0.7099345326423645, "step": 6 }, { "KL/chosen_KL_mean": -0.04565238952636719, "KL/mean": 0.0033334270119667053, "KL/rejected_KL_mean": 0.05230522155761719, "KL/std": 0.5084267854690552, "epoch": 0.014659685863874346, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0979558676481247, "fcm_dpo/q_t": 0.5002449154853821, "grad_norm": 28.62909698486328, "learning_rate": 6.25e-08, "logits/chosen": -0.5669029951095581, "logits/rejected": -0.5987157225608826, "logps/chosen": -264.7010803222656, "logps/ref_chosen": -264.65545654296875, "logps/ref_rejected": -253.10305786132812, "logps/rejected": -253.05076599121094, "loss": 5.5491, "margin_dpo/margin_mean": -0.09795580804347992, "margin_dpo/margin_std": 0.6526715755462646, "step": 7 }, { "KL/chosen_KL_mean": 0.029354095458984375, "KL/mean": 0.002108544111251831, "KL/rejected_KL_mean": -0.025133132934570312, "KL/std": 0.5323061943054199, "epoch": 0.016753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.054483771324157715, "fcm_dpo/q_t": 0.49986377358436584, "grad_norm": 30.765409469604492, "learning_rate": 7.291666666666667e-08, "logits/chosen": -0.6719302535057068, "logits/rejected": -0.6778964996337891, "logps/chosen": -354.159423828125, "logps/ref_chosen": -354.1887512207031, "logps/ref_rejected": -282.9112243652344, "logps/rejected": -282.93634033203125, "loss": 5.5431, "margin_dpo/margin_mean": 0.054484546184539795, "margin_dpo/margin_std": 0.7297343015670776, "step": 8 }, { "KL/chosen_KL_mean": 0.05601310729980469, "KL/mean": 0.03541021794080734, "KL/rejected_KL_mean": 0.014802932739257812, "KL/std": 0.5015314817428589, "epoch": 0.018848167539267015, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04120449721813202, "fcm_dpo/q_t": 0.49989697337150574, "grad_norm": 27.886104583740234, "learning_rate": 8.333333333333333e-08, "logits/chosen": -0.6346057653427124, "logits/rejected": -0.6525917053222656, "logps/chosen": -285.49420166015625, "logps/ref_chosen": -285.5502014160156, "logps/ref_rejected": -267.99664306640625, "logps/rejected": -267.9818420410156, "loss": 5.5436, "margin_dpo/margin_mean": 0.04120419919490814, "margin_dpo/margin_std": 0.6990031003952026, "step": 9 }, { "KL/chosen_KL_mean": 0.046550750732421875, "KL/mean": 0.020397096872329712, "KL/rejected_KL_mean": -0.0057582855224609375, "KL/std": 0.5216137170791626, "epoch": 0.020942408376963352, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.052318453788757324, "fcm_dpo/q_t": 0.499869167804718, "grad_norm": 26.553926467895508, "learning_rate": 9.375e-08, "logits/chosen": -0.691142201423645, "logits/rejected": -0.6861263513565063, "logps/chosen": -251.8658447265625, "logps/ref_chosen": -251.91238403320312, "logps/ref_rejected": -226.45260620117188, "logps/rejected": -226.4583740234375, "loss": 5.5431, "margin_dpo/margin_mean": 0.05231724679470062, "margin_dpo/margin_std": 0.6777650713920593, "step": 10 }, { "KL/chosen_KL_mean": 0.05559539794921875, "KL/mean": 0.08593348413705826, "KL/rejected_KL_mean": 0.11626625061035156, "KL/std": 0.5352065563201904, "epoch": 0.023036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06067343056201935, "fcm_dpo/q_t": 0.5001516938209534, "grad_norm": 28.990291595458984, "learning_rate": 1.0416666666666667e-07, "logits/chosen": -0.5975240468978882, "logits/rejected": -0.6529561281204224, "logps/chosen": -301.02789306640625, "logps/ref_chosen": -301.08343505859375, "logps/ref_rejected": -259.546630859375, "logps/rejected": -259.43035888671875, "loss": 5.5477, "margin_dpo/margin_mean": -0.060673072934150696, "margin_dpo/margin_std": 0.7372453212738037, "step": 11 }, { "KL/chosen_KL_mean": -0.03789520263671875, "KL/mean": -0.051166288554668427, "KL/rejected_KL_mean": -0.06442451477050781, "KL/std": 0.5343046188354492, "epoch": 0.025130890052356022, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.026523694396018982, "fcm_dpo/q_t": 0.499933660030365, "grad_norm": 30.13033103942871, "learning_rate": 1.1458333333333332e-07, "logits/chosen": -0.5826385021209717, "logits/rejected": -0.5425857901573181, "logps/chosen": -287.58599853515625, "logps/ref_chosen": -287.548095703125, "logps/ref_rejected": -277.37945556640625, "logps/rejected": -277.44390869140625, "loss": 5.5442, "margin_dpo/margin_mean": 0.026523590087890625, "margin_dpo/margin_std": 0.7421956062316895, "step": 12 }, { "KL/chosen_KL_mean": 0.05934333801269531, "KL/mean": -0.022284463047981262, "KL/rejected_KL_mean": -0.10392189025878906, "KL/std": 0.43461257219314575, "epoch": 0.027225130890052355, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1632615029811859, "fcm_dpo/q_t": 0.4995918273925781, "grad_norm": 27.26964569091797, "learning_rate": 1.25e-07, "logits/chosen": -0.6637694239616394, "logits/rejected": -0.6718661785125732, "logps/chosen": -270.6070556640625, "logps/ref_chosen": -270.6664123535156, "logps/ref_rejected": -274.6546936035156, "logps/rejected": -274.7586364746094, "loss": 5.5387, "margin_dpo/margin_mean": 0.1632617861032486, "margin_dpo/margin_std": 0.5879830121994019, "step": 13 }, { "KL/chosen_KL_mean": 0.01528167724609375, "KL/mean": -0.007226079702377319, "KL/rejected_KL_mean": -0.029737472534179688, "KL/std": 0.5344897508621216, "epoch": 0.02931937172774869, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04503127932548523, "fcm_dpo/q_t": 0.4998874068260193, "grad_norm": 28.33542823791504, "learning_rate": 1.3541666666666666e-07, "logits/chosen": -0.6277254223823547, "logits/rejected": -0.657189130783081, "logps/chosen": -281.5779113769531, "logps/ref_chosen": -281.59320068359375, "logps/ref_rejected": -263.52215576171875, "logps/rejected": -263.5519104003906, "loss": 5.5434, "margin_dpo/margin_mean": 0.04503123462200165, "margin_dpo/margin_std": 0.7003819942474365, "step": 14 }, { "KL/chosen_KL_mean": 0.0198822021484375, "KL/mean": 0.006983160972595215, "KL/rejected_KL_mean": -0.005916595458984375, "KL/std": 0.4857458770275116, "epoch": 0.031413612565445025, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02580079436302185, "fcm_dpo/q_t": 0.499935507774353, "grad_norm": 30.2783145904541, "learning_rate": 1.4583333333333335e-07, "logits/chosen": -0.6451765298843384, "logits/rejected": -0.6582185626029968, "logps/chosen": -298.4335632324219, "logps/ref_chosen": -298.45343017578125, "logps/ref_rejected": -227.17832946777344, "logps/rejected": -227.1842498779297, "loss": 5.5442, "margin_dpo/margin_mean": 0.025799661874771118, "margin_dpo/margin_std": 0.7161386609077454, "step": 15 }, { "KL/chosen_KL_mean": 0.011865615844726562, "KL/mean": 0.002268832176923752, "KL/rejected_KL_mean": -0.007328033447265625, "KL/std": 0.5652431845664978, "epoch": 0.033507853403141365, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01919480413198471, "fcm_dpo/q_t": 0.49995195865631104, "grad_norm": 30.200963973999023, "learning_rate": 1.5624999999999999e-07, "logits/chosen": -0.5971285104751587, "logits/rejected": -0.5938217043876648, "logps/chosen": -293.9547424316406, "logps/ref_chosen": -293.96661376953125, "logps/ref_rejected": -250.78443908691406, "logps/rejected": -250.79176330566406, "loss": 5.5445, "margin_dpo/margin_mean": 0.019194379448890686, "margin_dpo/margin_std": 0.802629828453064, "step": 16 }, { "KL/chosen_KL_mean": -0.02978515625, "KL/mean": 0.001806497573852539, "KL/rejected_KL_mean": 0.0334014892578125, "KL/std": 0.4761771261692047, "epoch": 0.0356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06319683790206909, "fcm_dpo/q_t": 0.5001580119132996, "grad_norm": 27.68961524963379, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5712617635726929, "logits/rejected": -0.5975736379623413, "logps/chosen": -262.4237976074219, "logps/ref_chosen": -262.39398193359375, "logps/ref_rejected": -248.500244140625, "logps/rejected": -248.4668426513672, "loss": 5.5478, "margin_dpo/margin_mean": -0.06319691240787506, "margin_dpo/margin_std": 0.6851055026054382, "step": 17 }, { "KL/chosen_KL_mean": 0.13873863220214844, "KL/mean": 0.06383218616247177, "KL/rejected_KL_mean": -0.011079788208007812, "KL/std": 0.5538792014122009, "epoch": 0.03769633507853403, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14981205761432648, "fcm_dpo/q_t": 0.49962547421455383, "grad_norm": 29.6390438079834, "learning_rate": 1.7708333333333334e-07, "logits/chosen": -0.6094870567321777, "logits/rejected": -0.6156261563301086, "logps/chosen": -293.57049560546875, "logps/ref_chosen": -293.709228515625, "logps/ref_rejected": -274.5875244140625, "logps/rejected": -274.5986328125, "loss": 5.5392, "margin_dpo/margin_mean": 0.14981183409690857, "margin_dpo/margin_std": 0.7385837435722351, "step": 18 }, { "KL/chosen_KL_mean": 0.0306243896484375, "KL/mean": 0.023819290101528168, "KL/rejected_KL_mean": 0.017019271850585938, "KL/std": 0.44108718633651733, "epoch": 0.039790575916230364, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01360417902469635, "fcm_dpo/q_t": 0.49996596574783325, "grad_norm": 28.12141227722168, "learning_rate": 1.875e-07, "logits/chosen": -0.6333426833152771, "logits/rejected": -0.6301394701004028, "logps/chosen": -280.2350158691406, "logps/ref_chosen": -280.26568603515625, "logps/ref_rejected": -259.9742736816406, "logps/rejected": -259.9572448730469, "loss": 5.5447, "margin_dpo/margin_mean": 0.013604193925857544, "margin_dpo/margin_std": 0.5601605176925659, "step": 19 }, { "KL/chosen_KL_mean": 0.15392303466796875, "KL/mean": 0.09969540685415268, "KL/rejected_KL_mean": 0.04547691345214844, "KL/std": 0.5148954391479492, "epoch": 0.041884816753926704, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10843174159526825, "fcm_dpo/q_t": 0.4997289180755615, "grad_norm": 29.663406372070312, "learning_rate": 1.9791666666666664e-07, "logits/chosen": -0.61493980884552, "logits/rejected": -0.6458949446678162, "logps/chosen": -303.7414855957031, "logps/ref_chosen": -303.8954162597656, "logps/ref_rejected": -260.214599609375, "logps/rejected": -260.16912841796875, "loss": 5.5409, "margin_dpo/margin_mean": 0.10843226313591003, "margin_dpo/margin_std": 0.7343800067901611, "step": 20 }, { "KL/chosen_KL_mean": 0.13170433044433594, "KL/mean": 0.06188298016786575, "KL/rejected_KL_mean": -0.007947921752929688, "KL/std": 0.5429558753967285, "epoch": 0.04397905759162304, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13965456187725067, "fcm_dpo/q_t": 0.49965083599090576, "grad_norm": 34.97311019897461, "learning_rate": 2.0833333333333333e-07, "logits/chosen": -0.6538794636726379, "logits/rejected": -0.6838114857673645, "logps/chosen": -301.40179443359375, "logps/ref_chosen": -301.5334777832031, "logps/ref_rejected": -280.28900146484375, "logps/rejected": -280.29693603515625, "loss": 5.5397, "margin_dpo/margin_mean": 0.13965462148189545, "margin_dpo/margin_std": 0.7473960518836975, "step": 21 }, { "KL/chosen_KL_mean": 0.095489501953125, "KL/mean": 0.06919194012880325, "KL/rejected_KL_mean": 0.04288482666015625, "KL/std": 0.5348218679428101, "epoch": 0.04607329842931937, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05260805785655975, "fcm_dpo/q_t": 0.4998684525489807, "grad_norm": 25.295730590820312, "learning_rate": 2.1875e-07, "logits/chosen": -0.6623374819755554, "logits/rejected": -0.6642951369285583, "logps/chosen": -259.8996276855469, "logps/ref_chosen": -259.9951477050781, "logps/ref_rejected": -243.0721435546875, "logps/rejected": -243.02928161621094, "loss": 5.5431, "margin_dpo/margin_mean": 0.052607640624046326, "margin_dpo/margin_std": 0.7113056182861328, "step": 22 }, { "KL/chosen_KL_mean": 0.09666061401367188, "KL/mean": 0.03947530686855316, "KL/rejected_KL_mean": -0.0177154541015625, "KL/std": 0.5528764128684998, "epoch": 0.048167539267015703, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11437445878982544, "fcm_dpo/q_t": 0.49971407651901245, "grad_norm": 28.02526092529297, "learning_rate": 2.2916666666666663e-07, "logits/chosen": -0.6213913559913635, "logits/rejected": -0.6555658578872681, "logps/chosen": -282.0841064453125, "logps/ref_chosen": -282.1807556152344, "logps/ref_rejected": -265.0758056640625, "logps/rejected": -265.093505859375, "loss": 5.5407, "margin_dpo/margin_mean": 0.11437398195266724, "margin_dpo/margin_std": 0.7158402800559998, "step": 23 }, { "KL/chosen_KL_mean": 0.16210174560546875, "KL/mean": 0.04678072780370712, "KL/rejected_KL_mean": -0.06854248046875, "KL/std": 0.6121370792388916, "epoch": 0.050261780104712044, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2306310385465622, "fcm_dpo/q_t": 0.49942344427108765, "grad_norm": 29.789752960205078, "learning_rate": 2.3958333333333335e-07, "logits/chosen": -0.6437051296234131, "logits/rejected": -0.5558314323425293, "logps/chosen": -301.01751708984375, "logps/ref_chosen": -301.17962646484375, "logps/ref_rejected": -302.12786865234375, "logps/rejected": -302.1964111328125, "loss": 5.536, "margin_dpo/margin_mean": 0.23063026368618011, "margin_dpo/margin_std": 0.8269215226173401, "step": 24 }, { "KL/chosen_KL_mean": 0.15172958374023438, "KL/mean": 0.07193604111671448, "KL/rejected_KL_mean": -0.007861137390136719, "KL/std": 0.49663278460502625, "epoch": 0.05235602094240838, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1595902144908905, "fcm_dpo/q_t": 0.49960100650787354, "grad_norm": 26.53129005432129, "learning_rate": 2.5e-07, "logits/chosen": -0.5964261293411255, "logits/rejected": -0.6044997572898865, "logps/chosen": -246.59474182128906, "logps/ref_chosen": -246.74649047851562, "logps/ref_rejected": -235.55638122558594, "logps/rejected": -235.56422424316406, "loss": 5.5389, "margin_dpo/margin_mean": 0.15958915650844574, "margin_dpo/margin_std": 0.7234926819801331, "step": 25 }, { "KL/chosen_KL_mean": 0.2514305114746094, "KL/mean": 0.16334398090839386, "KL/rejected_KL_mean": 0.07526397705078125, "KL/std": 0.6091904044151306, "epoch": 0.05445026178010471, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17616990208625793, "fcm_dpo/q_t": 0.499559611082077, "grad_norm": 28.695537567138672, "learning_rate": 2.604166666666667e-07, "logits/chosen": -0.647978663444519, "logits/rejected": -0.6618956327438354, "logps/chosen": -281.94415283203125, "logps/ref_chosen": -282.1955871582031, "logps/ref_rejected": -235.3135528564453, "logps/rejected": -235.23829650878906, "loss": 5.5382, "margin_dpo/margin_mean": 0.17617014050483704, "margin_dpo/margin_std": 0.8417377471923828, "step": 26 }, { "KL/chosen_KL_mean": 0.3227558135986328, "KL/mean": 0.2342260479927063, "KL/rejected_KL_mean": 0.1456890106201172, "KL/std": 0.6314464211463928, "epoch": 0.05654450261780105, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17707379162311554, "fcm_dpo/q_t": 0.49955734610557556, "grad_norm": 27.701112747192383, "learning_rate": 2.708333333333333e-07, "logits/chosen": -0.6549860239028931, "logits/rejected": -0.6748118996620178, "logps/chosen": -323.5335693359375, "logps/ref_chosen": -323.8563537597656, "logps/ref_rejected": -245.968017578125, "logps/rejected": -245.82235717773438, "loss": 5.5382, "margin_dpo/margin_mean": 0.1770738810300827, "margin_dpo/margin_std": 0.9032553434371948, "step": 27 }, { "KL/chosen_KL_mean": 0.233856201171875, "KL/mean": 0.17051860690116882, "KL/rejected_KL_mean": 0.107177734375, "KL/std": 0.5839200615882874, "epoch": 0.05863874345549738, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12668462097644806, "fcm_dpo/q_t": 0.49968329071998596, "grad_norm": 26.368717193603516, "learning_rate": 2.8125e-07, "logits/chosen": -0.6276756525039673, "logits/rejected": -0.6372566819190979, "logps/chosen": -248.01287841796875, "logps/ref_chosen": -248.24673461914062, "logps/ref_rejected": -240.0382080078125, "logps/rejected": -239.9310302734375, "loss": 5.5402, "margin_dpo/margin_mean": 0.1266840100288391, "margin_dpo/margin_std": 0.8177756071090698, "step": 28 }, { "KL/chosen_KL_mean": 0.33344268798828125, "KL/mean": 0.25681155920028687, "KL/rejected_KL_mean": 0.18017005920410156, "KL/std": 0.6312374472618103, "epoch": 0.060732984293193716, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15327195823192596, "fcm_dpo/q_t": 0.4996168315410614, "grad_norm": 29.685964584350586, "learning_rate": 2.916666666666667e-07, "logits/chosen": -0.5926003456115723, "logits/rejected": -0.6149235367774963, "logps/chosen": -317.9229736328125, "logps/ref_chosen": -318.2564392089844, "logps/ref_rejected": -286.75848388671875, "logps/rejected": -286.5783386230469, "loss": 5.5391, "margin_dpo/margin_mean": 0.15327222645282745, "margin_dpo/margin_std": 0.8355914354324341, "step": 29 }, { "KL/chosen_KL_mean": 0.37634849548339844, "KL/mean": 0.2421349585056305, "KL/rejected_KL_mean": 0.10791778564453125, "KL/std": 0.6748695373535156, "epoch": 0.06282722513089005, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2684309482574463, "fcm_dpo/q_t": 0.4993289113044739, "grad_norm": 28.966293334960938, "learning_rate": 3.020833333333333e-07, "logits/chosen": -0.589932918548584, "logits/rejected": -0.6068093180656433, "logps/chosen": -252.6727752685547, "logps/ref_chosen": -253.0491485595703, "logps/ref_rejected": -261.30029296875, "logps/rejected": -261.1923828125, "loss": 5.5345, "margin_dpo/margin_mean": 0.26843076944351196, "margin_dpo/margin_std": 0.9395996332168579, "step": 30 }, { "KL/chosen_KL_mean": 0.4201011657714844, "KL/mean": 0.3030571937561035, "KL/rejected_KL_mean": 0.1860198974609375, "KL/std": 0.709516167640686, "epoch": 0.06492146596858639, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23408280313014984, "fcm_dpo/q_t": 0.49941486120224, "grad_norm": 25.04664421081543, "learning_rate": 3.1249999999999997e-07, "logits/chosen": -0.6467900276184082, "logits/rejected": -0.680314302444458, "logps/chosen": -247.73292541503906, "logps/ref_chosen": -248.15301513671875, "logps/ref_rejected": -203.17703247070312, "logps/rejected": -202.9910125732422, "loss": 5.5359, "margin_dpo/margin_mean": 0.23408250510692596, "margin_dpo/margin_std": 0.9148648977279663, "step": 31 }, { "KL/chosen_KL_mean": 0.4879474639892578, "KL/mean": 0.3524807095527649, "KL/rejected_KL_mean": 0.21702194213867188, "KL/std": 0.7370665073394775, "epoch": 0.06701570680628273, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2709151804447174, "fcm_dpo/q_t": 0.49932271242141724, "grad_norm": 29.720426559448242, "learning_rate": 3.2291666666666666e-07, "logits/chosen": -0.6020100116729736, "logits/rejected": -0.6054341793060303, "logps/chosen": -305.052001953125, "logps/ref_chosen": -305.5399475097656, "logps/ref_rejected": -267.6527099609375, "logps/rejected": -267.4356689453125, "loss": 5.5345, "margin_dpo/margin_mean": 0.27091488242149353, "margin_dpo/margin_std": 0.9720284938812256, "step": 32 }, { "KL/chosen_KL_mean": 0.6097984313964844, "KL/mean": 0.38870200514793396, "KL/rejected_KL_mean": 0.1676158905029297, "KL/std": 0.8518388271331787, "epoch": 0.06910994764397906, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.44216686487197876, "fcm_dpo/q_t": 0.4988946318626404, "grad_norm": 28.51626205444336, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.633573591709137, "logits/rejected": -0.6452749371528625, "logps/chosen": -285.62371826171875, "logps/ref_chosen": -286.2335205078125, "logps/ref_rejected": -255.38748168945312, "logps/rejected": -255.21986389160156, "loss": 5.5276, "margin_dpo/margin_mean": 0.44216734170913696, "margin_dpo/margin_std": 1.0235333442687988, "step": 33 }, { "KL/chosen_KL_mean": 0.7400722503662109, "KL/mean": 0.4082331657409668, "KL/rejected_KL_mean": 0.07639503479003906, "KL/std": 1.0116835832595825, "epoch": 0.0712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.663679838180542, "fcm_dpo/q_t": 0.4983408749103546, "grad_norm": 31.122045516967773, "learning_rate": 3.4375e-07, "logits/chosen": -0.6284201741218567, "logits/rejected": -0.6383717060089111, "logps/chosen": -340.8519287109375, "logps/ref_chosen": -341.5920104980469, "logps/ref_rejected": -278.8866882324219, "logps/rejected": -278.8102722167969, "loss": 5.5188, "margin_dpo/margin_mean": 0.6636798977851868, "margin_dpo/margin_std": 1.1839109659194946, "step": 34 }, { "KL/chosen_KL_mean": 0.6229438781738281, "KL/mean": 0.46992263197898865, "KL/rejected_KL_mean": 0.3169078826904297, "KL/std": 1.0069925785064697, "epoch": 0.07329842931937172, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3060385286808014, "fcm_dpo/q_t": 0.4992349147796631, "grad_norm": 26.427873611450195, "learning_rate": 3.541666666666667e-07, "logits/chosen": -0.6407392024993896, "logits/rejected": -0.6617733836174011, "logps/chosen": -264.45660400390625, "logps/ref_chosen": -265.0795593261719, "logps/ref_rejected": -264.4876708984375, "logps/rejected": -264.1707763671875, "loss": 5.5332, "margin_dpo/margin_mean": 0.30603715777397156, "margin_dpo/margin_std": 1.3842066526412964, "step": 35 }, { "KL/chosen_KL_mean": 0.9210777282714844, "KL/mean": 0.5656237602233887, "KL/rejected_KL_mean": 0.21016883850097656, "KL/std": 1.1084519624710083, "epoch": 0.07539267015706806, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7109137177467346, "fcm_dpo/q_t": 0.49822285771369934, "grad_norm": 32.040306091308594, "learning_rate": 3.645833333333333e-07, "logits/chosen": -0.600266695022583, "logits/rejected": -0.61773681640625, "logps/chosen": -296.405029296875, "logps/ref_chosen": -297.3261413574219, "logps/ref_rejected": -282.09515380859375, "logps/rejected": -281.8849792480469, "loss": 5.517, "margin_dpo/margin_mean": 0.7109129428863525, "margin_dpo/margin_std": 1.4326907396316528, "step": 36 }, { "KL/chosen_KL_mean": 0.7100296020507812, "KL/mean": 0.45437169075012207, "KL/rejected_KL_mean": 0.1987323760986328, "KL/std": 1.1140685081481934, "epoch": 0.0774869109947644, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5112793445587158, "fcm_dpo/q_t": 0.4987218976020813, "grad_norm": 31.217262268066406, "learning_rate": 3.75e-07, "logits/chosen": -0.5984426140785217, "logits/rejected": -0.612653911113739, "logps/chosen": -313.3240051269531, "logps/ref_chosen": -314.0340270996094, "logps/ref_rejected": -299.3437805175781, "logps/rejected": -299.14501953125, "loss": 5.525, "margin_dpo/margin_mean": 0.5112783312797546, "margin_dpo/margin_std": 1.474202036857605, "step": 37 }, { "KL/chosen_KL_mean": 1.0367469787597656, "KL/mean": 0.7218881845474243, "KL/rejected_KL_mean": 0.40703392028808594, "KL/std": 1.2584364414215088, "epoch": 0.07958115183246073, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6297140121459961, "fcm_dpo/q_t": 0.49842581152915955, "grad_norm": 28.42009162902832, "learning_rate": 3.8541666666666665e-07, "logits/chosen": -0.6392098069190979, "logits/rejected": -0.6514177322387695, "logps/chosen": -281.50445556640625, "logps/ref_chosen": -282.54119873046875, "logps/ref_rejected": -269.7773132324219, "logps/rejected": -269.3702697753906, "loss": 5.5203, "margin_dpo/margin_mean": 0.6297144889831543, "margin_dpo/margin_std": 1.5614598989486694, "step": 38 }, { "KL/chosen_KL_mean": 1.3400955200195312, "KL/mean": 0.716141402721405, "KL/rejected_KL_mean": 0.092193603515625, "KL/std": 1.4875259399414062, "epoch": 0.08167539267015707, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2479068040847778, "fcm_dpo/q_t": 0.4968806505203247, "grad_norm": 29.515811920166016, "learning_rate": 3.958333333333333e-07, "logits/chosen": -0.6168644428253174, "logits/rejected": -0.6308912634849548, "logps/chosen": -275.432861328125, "logps/ref_chosen": -276.7729187011719, "logps/ref_rejected": -249.95889282226562, "logps/rejected": -249.86668395996094, "loss": 5.4958, "margin_dpo/margin_mean": 1.2479069232940674, "margin_dpo/margin_std": 1.717916488647461, "step": 39 }, { "KL/chosen_KL_mean": 1.0269756317138672, "KL/mean": 0.6691527962684631, "KL/rejected_KL_mean": 0.3113231658935547, "KL/std": 1.3725290298461914, "epoch": 0.08376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.715646505355835, "fcm_dpo/q_t": 0.49821117520332336, "grad_norm": 27.384668350219727, "learning_rate": 4.0625e-07, "logits/chosen": -0.6227149367332458, "logits/rejected": -0.6583834290504456, "logps/chosen": -283.28009033203125, "logps/ref_chosen": -284.30706787109375, "logps/ref_rejected": -244.4459991455078, "logps/rejected": -244.1346435546875, "loss": 5.5169, "margin_dpo/margin_mean": 0.7156472206115723, "margin_dpo/margin_std": 1.7450072765350342, "step": 40 }, { "KL/chosen_KL_mean": 1.1292037963867188, "KL/mean": 0.7421854734420776, "KL/rejected_KL_mean": 0.3551807403564453, "KL/std": 1.465866208076477, "epoch": 0.08586387434554973, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7740218639373779, "fcm_dpo/q_t": 0.49806511402130127, "grad_norm": 30.4445743560791, "learning_rate": 4.1666666666666667e-07, "logits/chosen": -0.6232025027275085, "logits/rejected": -0.647884726524353, "logps/chosen": -292.68597412109375, "logps/ref_chosen": -293.8151550292969, "logps/ref_rejected": -252.16815185546875, "logps/rejected": -251.81297302246094, "loss": 5.5146, "margin_dpo/margin_mean": 0.7740228176116943, "margin_dpo/margin_std": 1.7456119060516357, "step": 41 }, { "KL/chosen_KL_mean": 1.1923046112060547, "KL/mean": 0.7407265901565552, "KL/rejected_KL_mean": 0.2891521453857422, "KL/std": 1.7247297763824463, "epoch": 0.08795811518324607, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9031492471694946, "fcm_dpo/q_t": 0.49774235486984253, "grad_norm": 27.42459487915039, "learning_rate": 4.270833333333333e-07, "logits/chosen": -0.6371679902076721, "logits/rejected": -0.6549400687217712, "logps/chosen": -251.5679473876953, "logps/ref_chosen": -252.76023864746094, "logps/ref_rejected": -261.0414733886719, "logps/rejected": -260.7523193359375, "loss": 5.5097, "margin_dpo/margin_mean": 0.9031498432159424, "margin_dpo/margin_std": 2.27445387840271, "step": 42 }, { "KL/chosen_KL_mean": 1.2597465515136719, "KL/mean": 0.6688581705093384, "KL/rejected_KL_mean": 0.07796669006347656, "KL/std": 1.8619308471679688, "epoch": 0.09005235602094241, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1817760467529297, "fcm_dpo/q_t": 0.4970458745956421, "grad_norm": 29.90841293334961, "learning_rate": 4.375e-07, "logits/chosen": -0.5890940427780151, "logits/rejected": -0.6039578914642334, "logps/chosen": -315.5749816894531, "logps/ref_chosen": -316.8347473144531, "logps/ref_rejected": -273.7649230957031, "logps/rejected": -273.68695068359375, "loss": 5.4986, "margin_dpo/margin_mean": 1.1817759275436401, "margin_dpo/margin_std": 2.1939334869384766, "step": 43 }, { "KL/chosen_KL_mean": 1.512136459350586, "KL/mean": 0.7135680913925171, "KL/rejected_KL_mean": -0.08500099182128906, "KL/std": 2.2240757942199707, "epoch": 0.09214659685863874, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5971362590789795, "fcm_dpo/q_t": 0.49600785970687866, "grad_norm": 30.842592239379883, "learning_rate": 4.479166666666667e-07, "logits/chosen": -0.5955119132995605, "logits/rejected": -0.5941088199615479, "logps/chosen": -285.3635559082031, "logps/ref_chosen": -286.8757019042969, "logps/ref_rejected": -282.4681396484375, "logps/rejected": -282.55316162109375, "loss": 5.4825, "margin_dpo/margin_mean": 1.5971367359161377, "margin_dpo/margin_std": 3.021836996078491, "step": 44 }, { "KL/chosen_KL_mean": 1.7010555267333984, "KL/mean": 1.0267201662063599, "KL/rejected_KL_mean": 0.35239219665527344, "KL/std": 2.252026319503784, "epoch": 0.09424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3486623764038086, "fcm_dpo/q_t": 0.49662941694259644, "grad_norm": 28.998353958129883, "learning_rate": 4.5833333333333327e-07, "logits/chosen": -0.6837330460548401, "logits/rejected": -0.7065256237983704, "logps/chosen": -322.5623474121094, "logps/ref_chosen": -324.2633972167969, "logps/ref_rejected": -293.09466552734375, "logps/rejected": -292.7422790527344, "loss": 5.4922, "margin_dpo/margin_mean": 1.3486623764038086, "margin_dpo/margin_std": 2.807202100753784, "step": 45 }, { "KL/chosen_KL_mean": 1.6978473663330078, "KL/mean": 0.9566701650619507, "KL/rejected_KL_mean": 0.2154865264892578, "KL/std": 2.4581615924835205, "epoch": 0.09633507853403141, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4823609590530396, "fcm_dpo/q_t": 0.4962949752807617, "grad_norm": 30.333192825317383, "learning_rate": 4.6874999999999996e-07, "logits/chosen": -0.6262620687484741, "logits/rejected": -0.6398423314094543, "logps/chosen": -296.63787841796875, "logps/ref_chosen": -298.3357238769531, "logps/ref_rejected": -267.66204833984375, "logps/rejected": -267.4465637207031, "loss": 5.4868, "margin_dpo/margin_mean": 1.482360601425171, "margin_dpo/margin_std": 2.6219446659088135, "step": 46 }, { "KL/chosen_KL_mean": 1.5099334716796875, "KL/mean": 0.9814417362213135, "KL/rejected_KL_mean": 0.45295143127441406, "KL/std": 2.342404842376709, "epoch": 0.09842931937172775, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0569857358932495, "fcm_dpo/q_t": 0.4973580241203308, "grad_norm": 26.228063583374023, "learning_rate": 4.791666666666667e-07, "logits/chosen": -0.601626992225647, "logits/rejected": -0.6220702528953552, "logps/chosen": -261.0569763183594, "logps/ref_chosen": -262.5669250488281, "logps/ref_rejected": -258.70989990234375, "logps/rejected": -258.2569274902344, "loss": 5.5042, "margin_dpo/margin_mean": 1.0569851398468018, "margin_dpo/margin_std": 3.353296995162964, "step": 47 }, { "KL/chosen_KL_mean": 1.8159637451171875, "KL/mean": 1.0512405633926392, "KL/rejected_KL_mean": 0.28652191162109375, "KL/std": 2.5599913597106934, "epoch": 0.10052356020942409, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.529449224472046, "fcm_dpo/q_t": 0.4961783289909363, "grad_norm": 27.525484085083008, "learning_rate": 4.895833333333333e-07, "logits/chosen": -0.6145200133323669, "logits/rejected": -0.6397130489349365, "logps/chosen": -267.67730712890625, "logps/ref_chosen": -269.4932556152344, "logps/ref_rejected": -241.888916015625, "logps/rejected": -241.60238647460938, "loss": 5.4853, "margin_dpo/margin_mean": 1.529449224472046, "margin_dpo/margin_std": 3.0062029361724854, "step": 48 }, { "KL/chosen_KL_mean": 2.173074722290039, "KL/mean": 1.3489664793014526, "KL/rejected_KL_mean": 0.5248641967773438, "KL/std": 2.607396125793457, "epoch": 0.10261780104712041, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6482032537460327, "fcm_dpo/q_t": 0.49588078260421753, "grad_norm": 27.708139419555664, "learning_rate": 5e-07, "logits/chosen": -0.6658346652984619, "logits/rejected": -0.6502586007118225, "logps/chosen": -255.71141052246094, "logps/ref_chosen": -257.8844909667969, "logps/ref_rejected": -256.8912048339844, "logps/rejected": -256.3663330078125, "loss": 5.4807, "margin_dpo/margin_mean": 1.6482040882110596, "margin_dpo/margin_std": 3.3653252124786377, "step": 49 }, { "KL/chosen_KL_mean": 1.9860305786132812, "KL/mean": 1.4248580932617188, "KL/rejected_KL_mean": 0.8636798858642578, "KL/std": 3.0206358432769775, "epoch": 0.10471204188481675, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1223528385162354, "fcm_dpo/q_t": 0.4971952736377716, "grad_norm": 28.091381072998047, "learning_rate": 4.999932966293553e-07, "logits/chosen": -0.6322698593139648, "logits/rejected": -0.6543733477592468, "logps/chosen": -299.642822265625, "logps/ref_chosen": -301.62884521484375, "logps/ref_rejected": -298.2716064453125, "logps/rejected": -297.4079284667969, "loss": 5.502, "margin_dpo/margin_mean": 1.1223524808883667, "margin_dpo/margin_std": 3.72768497467041, "step": 50 }, { "KL/chosen_KL_mean": 2.0781002044677734, "KL/mean": 1.165189504623413, "KL/rejected_KL_mean": 0.2522716522216797, "KL/std": 3.6036739349365234, "epoch": 0.1068062827225131, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.825838565826416, "fcm_dpo/q_t": 0.4954381287097931, "grad_norm": 29.186372756958008, "learning_rate": 4.999731868769026e-07, "logits/chosen": -0.6121972799301147, "logits/rejected": -0.6063085794448853, "logps/chosen": -267.2942810058594, "logps/ref_chosen": -269.37237548828125, "logps/ref_rejected": -297.0167541503906, "logps/rejected": -296.76446533203125, "loss": 5.4749, "margin_dpo/margin_mean": 1.8258390426635742, "margin_dpo/margin_std": 4.549215793609619, "step": 51 }, { "KL/chosen_KL_mean": 2.7024993896484375, "KL/mean": 1.1765809059143066, "KL/rejected_KL_mean": -0.3493385314941406, "KL/std": 3.7803289890289307, "epoch": 0.10890052356020942, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.0518381595611572, "fcm_dpo/q_t": 0.4923774003982544, "grad_norm": 30.223594665527344, "learning_rate": 4.99939671821067e-07, "logits/chosen": -0.659646213054657, "logits/rejected": -0.6663126945495605, "logps/chosen": -304.2002868652344, "logps/ref_chosen": -306.9028015136719, "logps/ref_rejected": -281.24737548828125, "logps/rejected": -281.5967102050781, "loss": 5.4265, "margin_dpo/margin_mean": 3.051837682723999, "margin_dpo/margin_std": 4.751425266265869, "step": 52 }, { "KL/chosen_KL_mean": 2.787456512451172, "KL/mean": 1.679011583328247, "KL/rejected_KL_mean": 0.5705699920654297, "KL/std": 4.107968807220459, "epoch": 0.11099476439790576, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.216883659362793, "fcm_dpo/q_t": 0.49446216225624084, "grad_norm": 30.71049690246582, "learning_rate": 4.998927532591591e-07, "logits/chosen": -0.65622878074646, "logits/rejected": -0.6969560980796814, "logps/chosen": -283.1884765625, "logps/ref_chosen": -285.9759521484375, "logps/ref_rejected": -273.9073486328125, "logps/rejected": -273.3367614746094, "loss": 5.4601, "margin_dpo/margin_mean": 2.216883659362793, "margin_dpo/margin_std": 5.178028106689453, "step": 53 }, { "KL/chosen_KL_mean": 2.6346054077148438, "KL/mean": 1.7063802480697632, "KL/rejected_KL_mean": 0.7781620025634766, "KL/std": 4.234417915344238, "epoch": 0.1130890052356021, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8564293384552002, "fcm_dpo/q_t": 0.49536406993865967, "grad_norm": 26.411357879638672, "learning_rate": 4.998324337072792e-07, "logits/chosen": -0.6882258653640747, "logits/rejected": -0.6963566541671753, "logps/chosen": -303.8699951171875, "logps/ref_chosen": -306.504638671875, "logps/ref_rejected": -272.67431640625, "logps/rejected": -271.8961486816406, "loss": 5.4744, "margin_dpo/margin_mean": 1.856428861618042, "margin_dpo/margin_std": 5.311679840087891, "step": 54 }, { "KL/chosen_KL_mean": 1.7031211853027344, "KL/mean": 0.5867319107055664, "KL/rejected_KL_mean": -0.5296592712402344, "KL/std": 3.6960041522979736, "epoch": 0.11518324607329843, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.2327768802642822, "fcm_dpo/q_t": 0.49442318081855774, "grad_norm": 24.854549407958984, "learning_rate": 4.997587164001815e-07, "logits/chosen": -0.6439815163612366, "logits/rejected": -0.6484256982803345, "logps/chosen": -220.6270294189453, "logps/ref_chosen": -222.33013916015625, "logps/ref_rejected": -206.59571838378906, "logps/rejected": -207.12538146972656, "loss": 5.4588, "margin_dpo/margin_mean": 2.2327778339385986, "margin_dpo/margin_std": 4.8868231773376465, "step": 55 }, { "KL/chosen_KL_mean": 2.8303165435791016, "KL/mean": 1.238706350326538, "KL/rejected_KL_mean": -0.3529033660888672, "KL/std": 4.7677001953125, "epoch": 0.11727748691099477, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1832265853881836, "fcm_dpo/q_t": 0.49205446243286133, "grad_norm": 27.35767936706543, "learning_rate": 4.996716052911017e-07, "logits/chosen": -0.6080012917518616, "logits/rejected": -0.622451663017273, "logps/chosen": -247.64785766601562, "logps/ref_chosen": -250.47816467285156, "logps/ref_rejected": -228.25848388671875, "logps/rejected": -228.61138916015625, "loss": 5.4231, "margin_dpo/margin_mean": 3.1832268238067627, "margin_dpo/margin_std": 5.896901607513428, "step": 56 }, { "KL/chosen_KL_mean": 3.8403587341308594, "KL/mean": 1.7561249732971191, "KL/rejected_KL_mean": -0.32811737060546875, "KL/std": 4.8524932861328125, "epoch": 0.1193717277486911, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.16848611831665, "fcm_dpo/q_t": 0.48959094285964966, "grad_norm": 30.88191032409668, "learning_rate": 4.99571105051544e-07, "logits/chosen": -0.6883333921432495, "logits/rejected": -0.6583897471427917, "logps/chosen": -311.2791442871094, "logps/ref_chosen": -315.1195373535156, "logps/ref_rejected": -272.755615234375, "logps/rejected": -273.083740234375, "loss": 5.3837, "margin_dpo/margin_mean": 4.168484687805176, "margin_dpo/margin_std": 5.704001426696777, "step": 57 }, { "KL/chosen_KL_mean": 2.4969558715820312, "KL/mean": 1.277438998222351, "KL/rejected_KL_mean": 0.05792045593261719, "KL/std": 4.529829025268555, "epoch": 0.12146596858638743, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.439037322998047, "fcm_dpo/q_t": 0.49390649795532227, "grad_norm": 27.652095794677734, "learning_rate": 4.994572210710314e-07, "logits/chosen": -0.6133895516395569, "logits/rejected": -0.6374213099479675, "logps/chosen": -262.6847229003906, "logps/ref_chosen": -265.1816711425781, "logps/ref_rejected": -268.2203369140625, "logps/rejected": -268.16241455078125, "loss": 5.4518, "margin_dpo/margin_mean": 2.439037799835205, "margin_dpo/margin_std": 5.721737861633301, "step": 58 }, { "KL/chosen_KL_mean": 2.1310195922851562, "KL/mean": 1.3431408405303955, "KL/rejected_KL_mean": 0.555267333984375, "KL/std": 5.277097225189209, "epoch": 0.12356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5757672786712646, "fcm_dpo/q_t": 0.4960674047470093, "grad_norm": 29.76839828491211, "learning_rate": 4.993299594568162e-07, "logits/chosen": -0.6114040613174438, "logits/rejected": -0.6053056120872498, "logps/chosen": -284.22296142578125, "logps/ref_chosen": -286.35394287109375, "logps/ref_rejected": -260.6757507324219, "logps/rejected": -260.1204833984375, "loss": 5.4881, "margin_dpo/margin_mean": 1.5757670402526855, "margin_dpo/margin_std": 7.032860279083252, "step": 59 }, { "KL/chosen_KL_mean": 2.8480281829833984, "KL/mean": 1.117538332939148, "KL/rejected_KL_mean": -0.6129531860351562, "KL/std": 5.176889419555664, "epoch": 0.1256544502617801, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4609785079956055, "fcm_dpo/q_t": 0.49135878682136536, "grad_norm": 28.109230041503906, "learning_rate": 4.991893270335525e-07, "logits/chosen": -0.6809808611869812, "logits/rejected": -0.7057399749755859, "logps/chosen": -255.9005889892578, "logps/ref_chosen": -258.74859619140625, "logps/ref_rejected": -255.04893493652344, "logps/rejected": -255.661865234375, "loss": 5.4135, "margin_dpo/margin_mean": 3.460980176925659, "margin_dpo/margin_std": 7.187258720397949, "step": 60 }, { "KL/chosen_KL_mean": 3.034067153930664, "KL/mean": 1.4287720918655396, "KL/rejected_KL_mean": -0.17651939392089844, "KL/std": 5.567894458770752, "epoch": 0.12774869109947645, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.210594892501831, "fcm_dpo/q_t": 0.49198687076568604, "grad_norm": 29.902263641357422, "learning_rate": 4.990353313429303e-07, "logits/chosen": -0.654046356678009, "logits/rejected": -0.6714726686477661, "logps/chosen": -275.433837890625, "logps/ref_chosen": -278.4678955078125, "logps/ref_rejected": -252.02720642089844, "logps/rejected": -252.20370483398438, "loss": 5.4244, "margin_dpo/margin_mean": 3.210594654083252, "margin_dpo/margin_std": 7.630921840667725, "step": 61 }, { "KL/chosen_KL_mean": 3.9335670471191406, "KL/mean": 2.268831968307495, "KL/rejected_KL_mean": 0.6041011810302734, "KL/std": 5.7209367752075195, "epoch": 0.12984293193717278, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3294646739959717, "fcm_dpo/q_t": 0.49169519543647766, "grad_norm": 26.6936092376709, "learning_rate": 4.988679806432711e-07, "logits/chosen": -0.6085910797119141, "logits/rejected": -0.6502899527549744, "logps/chosen": -268.9907531738281, "logps/ref_chosen": -272.92431640625, "logps/ref_rejected": -260.7935485839844, "logps/rejected": -260.189453125, "loss": 5.4188, "margin_dpo/margin_mean": 3.3294646739959717, "margin_dpo/margin_std": 7.3894362449646, "step": 62 }, { "KL/chosen_KL_mean": 3.3459014892578125, "KL/mean": 1.5705658197402954, "KL/rejected_KL_mean": -0.20477867126464844, "KL/std": 5.626295566558838, "epoch": 0.1319371727748691, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.550675868988037, "fcm_dpo/q_t": 0.49113813042640686, "grad_norm": 28.22681999206543, "learning_rate": 4.986872839090852e-07, "logits/chosen": -0.6682570576667786, "logits/rejected": -0.6751868724822998, "logps/chosen": -273.74310302734375, "logps/ref_chosen": -277.0889892578125, "logps/ref_rejected": -273.3413391113281, "logps/rejected": -273.5461120605469, "loss": 5.4101, "margin_dpo/margin_mean": 3.5506744384765625, "margin_dpo/margin_std": 7.26901912689209, "step": 63 }, { "KL/chosen_KL_mean": 4.068166732788086, "KL/mean": 1.9456909894943237, "KL/rejected_KL_mean": -0.17678260803222656, "KL/std": 6.259605884552002, "epoch": 0.13403141361256546, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.244952201843262, "fcm_dpo/q_t": 0.4894087612628937, "grad_norm": 28.343130111694336, "learning_rate": 4.9849325083059e-07, "logits/chosen": -0.6286708116531372, "logits/rejected": -0.6235312819480896, "logps/chosen": -279.7563171386719, "logps/ref_chosen": -283.8244934082031, "logps/ref_rejected": -263.29351806640625, "logps/rejected": -263.4703063964844, "loss": 5.3846, "margin_dpo/margin_mean": 4.244952201843262, "margin_dpo/margin_std": 8.277649879455566, "step": 64 }, { "KL/chosen_KL_mean": 3.4773826599121094, "KL/mean": 1.7928293943405151, "KL/rejected_KL_mean": 0.10827827453613281, "KL/std": 5.792224407196045, "epoch": 0.13612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3691048622131348, "fcm_dpo/q_t": 0.49159109592437744, "grad_norm": 27.745553970336914, "learning_rate": 4.982858918131906e-07, "logits/chosen": -0.7000569701194763, "logits/rejected": -0.6681522727012634, "logps/chosen": -261.3925476074219, "logps/ref_chosen": -264.8699645996094, "logps/ref_rejected": -268.5076904296875, "logps/rejected": -268.3994140625, "loss": 5.4174, "margin_dpo/margin_mean": 3.3691043853759766, "margin_dpo/margin_std": 7.4207940101623535, "step": 65 }, { "KL/chosen_KL_mean": 3.1046905517578125, "KL/mean": 1.0091890096664429, "KL/rejected_KL_mean": -1.0862960815429688, "KL/std": 7.481138229370117, "epoch": 0.1382198952879581, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.190981864929199, "fcm_dpo/q_t": 0.48955801129341125, "grad_norm": 27.39222526550293, "learning_rate": 4.980652179769217e-07, "logits/chosen": -0.6774306893348694, "logits/rejected": -0.6973876357078552, "logps/chosen": -269.8236389160156, "logps/ref_chosen": -272.9283142089844, "logps/ref_rejected": -280.94696044921875, "logps/rejected": -282.03326416015625, "loss": 5.39, "margin_dpo/margin_mean": 4.190980911254883, "margin_dpo/margin_std": 10.186588287353516, "step": 66 }, { "KL/chosen_KL_mean": 4.062643051147461, "KL/mean": 2.216336250305176, "KL/rejected_KL_mean": 0.370025634765625, "KL/std": 7.127995014190674, "epoch": 0.14031413612565444, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.6926159858703613, "fcm_dpo/q_t": 0.49078747630119324, "grad_norm": 25.49480438232422, "learning_rate": 4.978312411558517e-07, "logits/chosen": -0.6880432963371277, "logits/rejected": -0.7203540802001953, "logps/chosen": -262.12432861328125, "logps/ref_chosen": -266.18695068359375, "logps/ref_rejected": -250.17405700683594, "logps/rejected": -249.80404663085938, "loss": 5.4069, "margin_dpo/margin_mean": 3.692615509033203, "margin_dpo/margin_std": 8.700530052185059, "step": 67 }, { "KL/chosen_KL_mean": 3.052156448364258, "KL/mean": 0.7123284935951233, "KL/rejected_KL_mean": -1.627492904663086, "KL/std": 8.075217247009277, "epoch": 0.1424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.679654121398926, "fcm_dpo/q_t": 0.4883608818054199, "grad_norm": 27.840843200683594, "learning_rate": 4.975839738974473e-07, "logits/chosen": -0.6769158840179443, "logits/rejected": -0.6895080208778381, "logps/chosen": -294.8864440917969, "logps/ref_chosen": -297.9385986328125, "logps/ref_rejected": -261.5141296386719, "logps/rejected": -263.1416015625, "loss": 5.3723, "margin_dpo/margin_mean": 4.679654598236084, "margin_dpo/margin_std": 10.612685203552246, "step": 68 }, { "KL/chosen_KL_mean": 3.876893997192383, "KL/mean": 0.8488324284553528, "KL/rejected_KL_mean": -2.1792163848876953, "KL/std": 7.927192211151123, "epoch": 0.14450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.056118011474609, "fcm_dpo/q_t": 0.48490819334983826, "grad_norm": 28.45749855041504, "learning_rate": 4.97323429461901e-07, "logits/chosen": -0.6810729503631592, "logits/rejected": -0.7114135026931763, "logps/chosen": -261.7406311035156, "logps/ref_chosen": -265.6175231933594, "logps/ref_rejected": -236.8287353515625, "logps/rejected": -239.00794982910156, "loss": 5.3166, "margin_dpo/margin_mean": 6.056118488311768, "margin_dpo/margin_std": 9.694916725158691, "step": 69 }, { "KL/chosen_KL_mean": 4.329980850219727, "KL/mean": 1.4181368350982666, "KL/rejected_KL_mean": -1.4937019348144531, "KL/std": 8.636222839355469, "epoch": 0.14659685863874344, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.823677062988281, "fcm_dpo/q_t": 0.48549583554267883, "grad_norm": 28.639202117919922, "learning_rate": 4.970496218214204e-07, "logits/chosen": -0.6776444315910339, "logits/rejected": -0.7112736701965332, "logps/chosen": -291.8959655761719, "logps/ref_chosen": -296.2259216308594, "logps/ref_rejected": -254.68496704101562, "logps/rejected": -256.17864990234375, "loss": 5.3285, "margin_dpo/margin_mean": 5.823675632476807, "margin_dpo/margin_std": 10.906417846679688, "step": 70 }, { "KL/chosen_KL_mean": 5.135082244873047, "KL/mean": 2.540114402770996, "KL/rejected_KL_mean": -0.054851531982421875, "KL/std": 9.344449043273926, "epoch": 0.1486910994764398, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.189933776855469, "fcm_dpo/q_t": 0.48706474900245667, "grad_norm": 28.435436248779297, "learning_rate": 4.967625656594781e-07, "logits/chosen": -0.6521185636520386, "logits/rejected": -0.6411818265914917, "logps/chosen": -283.7921447753906, "logps/ref_chosen": -288.92724609375, "logps/ref_rejected": -278.6405334472656, "logps/rejected": -278.69537353515625, "loss": 5.3567, "margin_dpo/margin_mean": 5.189933776855469, "margin_dpo/margin_std": 12.259124755859375, "step": 71 }, { "KL/chosen_KL_mean": 3.259723663330078, "KL/mean": 0.7097737789154053, "KL/rejected_KL_mean": -1.8401775360107422, "KL/std": 8.4075288772583, "epoch": 0.15078534031413612, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.099894046783447, "fcm_dpo/q_t": 0.4872819781303406, "grad_norm": 28.07781982421875, "learning_rate": 4.964622763700252e-07, "logits/chosen": -0.6942219734191895, "logits/rejected": -0.7054851651191711, "logps/chosen": -233.7855682373047, "logps/ref_chosen": -237.0452880859375, "logps/ref_rejected": -252.7946319580078, "logps/rejected": -254.63479614257812, "loss": 5.3565, "margin_dpo/margin_mean": 5.0998945236206055, "margin_dpo/margin_std": 10.471480369567871, "step": 72 }, { "KL/chosen_KL_mean": 4.394702911376953, "KL/mean": 1.990799069404602, "KL/rejected_KL_mean": -0.41310882568359375, "KL/std": 8.965617179870605, "epoch": 0.15287958115183245, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.80781888961792, "fcm_dpo/q_t": 0.48800960183143616, "grad_norm": 27.92042350769043, "learning_rate": 4.961487700566646e-07, "logits/chosen": -0.6583269238471985, "logits/rejected": -0.6759439706802368, "logps/chosen": -268.65838623046875, "logps/ref_chosen": -273.0531005859375, "logps/ref_rejected": -246.8330841064453, "logps/rejected": -247.24620056152344, "loss": 5.3714, "margin_dpo/margin_mean": 4.8078179359436035, "margin_dpo/margin_std": 12.222956657409668, "step": 73 }, { "KL/chosen_KL_mean": 3.3620052337646484, "KL/mean": 1.1552921533584595, "KL/rejected_KL_mean": -1.051422119140625, "KL/std": 9.413747787475586, "epoch": 0.1549738219895288, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.413438320159912, "fcm_dpo/q_t": 0.48901113867759705, "grad_norm": 30.360273361206055, "learning_rate": 4.958220635317885e-07, "logits/chosen": -0.7333360910415649, "logits/rejected": -0.7118159532546997, "logps/chosen": -338.9198913574219, "logps/ref_chosen": -342.2818908691406, "logps/ref_rejected": -330.0293884277344, "logps/rejected": -331.08074951171875, "loss": 5.3855, "margin_dpo/margin_mean": 4.41343879699707, "margin_dpo/margin_std": 11.758991241455078, "step": 74 }, { "KL/chosen_KL_mean": 4.601959228515625, "KL/mean": 1.333653211593628, "KL/rejected_KL_mean": -1.9346542358398438, "KL/std": 9.369311332702637, "epoch": 0.15706806282722513, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.536617279052734, "fcm_dpo/q_t": 0.48371294140815735, "grad_norm": 29.381935119628906, "learning_rate": 4.954821743156767e-07, "logits/chosen": -0.6344682574272156, "logits/rejected": -0.6357614398002625, "logps/chosen": -262.2621765136719, "logps/ref_chosen": -266.8641662597656, "logps/ref_rejected": -276.8699951171875, "logps/rejected": -278.8046875, "loss": 5.3013, "margin_dpo/margin_mean": 6.536618709564209, "margin_dpo/margin_std": 10.953182220458984, "step": 75 }, { "KL/chosen_KL_mean": 4.06199836730957, "KL/mean": 0.48307615518569946, "KL/rejected_KL_mean": -3.095844268798828, "KL/std": 9.946473121643066, "epoch": 0.15916230366492146, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.157842636108398, "fcm_dpo/q_t": 0.4822140634059906, "grad_norm": 29.176395416259766, "learning_rate": 4.951291206355559e-07, "logits/chosen": -0.7243552803993225, "logits/rejected": -0.7324966788291931, "logps/chosen": -277.1125793457031, "logps/ref_chosen": -281.174560546875, "logps/ref_rejected": -263.6067199707031, "logps/rejected": -266.7025451660156, "loss": 5.2816, "margin_dpo/margin_mean": 7.157844543457031, "margin_dpo/margin_std": 12.785238265991211, "step": 76 }, { "KL/chosen_KL_mean": 3.7531566619873047, "KL/mean": 0.8536058068275452, "KL/rejected_KL_mean": -2.0459423065185547, "KL/std": 11.038262367248535, "epoch": 0.1612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.799106121063232, "fcm_dpo/q_t": 0.485598623752594, "grad_norm": 33.110633850097656, "learning_rate": 4.947629214246236e-07, "logits/chosen": -0.554699718952179, "logits/rejected": -0.5627188682556152, "logps/chosen": -302.34210205078125, "logps/ref_chosen": -306.09527587890625, "logps/ref_rejected": -253.49569702148438, "logps/rejected": -255.54164123535156, "loss": 5.3375, "margin_dpo/margin_mean": 5.799104690551758, "margin_dpo/margin_std": 13.92863655090332, "step": 77 }, { "KL/chosen_KL_mean": 3.8339462280273438, "KL/mean": -0.49760693311691284, "KL/rejected_KL_mean": -4.829154968261719, "KL/std": 11.0235595703125, "epoch": 0.16335078534031414, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.663091659545898, "fcm_dpo/q_t": 0.47850561141967773, "grad_norm": 29.756555557250977, "learning_rate": 4.943835963210323e-07, "logits/chosen": -0.6848552227020264, "logits/rejected": -0.6797273755073547, "logps/chosen": -253.0684051513672, "logps/ref_chosen": -256.90234375, "logps/ref_rejected": -211.57154846191406, "logps/rejected": -216.40072631835938, "loss": 5.228, "margin_dpo/margin_mean": 8.663091659545898, "margin_dpo/margin_std": 14.5060453414917, "step": 78 }, { "KL/chosen_KL_mean": 3.0965023040771484, "KL/mean": -1.057416319847107, "KL/rejected_KL_mean": -5.211338043212891, "KL/std": 11.727227210998535, "epoch": 0.16544502617801046, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.307835578918457, "fcm_dpo/q_t": 0.4793843924999237, "grad_norm": 29.793476104736328, "learning_rate": 4.939911656668361e-07, "logits/chosen": -0.6616241335868835, "logits/rejected": -0.6827108263969421, "logps/chosen": -263.17706298828125, "logps/ref_chosen": -266.2735595703125, "logps/ref_rejected": -251.57257080078125, "logps/rejected": -256.7839050292969, "loss": 5.2436, "margin_dpo/margin_mean": 8.307835578918457, "margin_dpo/margin_std": 14.920553207397461, "step": 79 }, { "KL/chosen_KL_mean": 1.841531753540039, "KL/mean": -1.4569592475891113, "KL/rejected_KL_mean": -4.755453109741211, "KL/std": 11.351509094238281, "epoch": 0.16753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.596972465515137, "fcm_dpo/q_t": 0.48362815380096436, "grad_norm": 29.06841278076172, "learning_rate": 4.935856505068998e-07, "logits/chosen": -0.6586440801620483, "logits/rejected": -0.6877402663230896, "logps/chosen": -286.00946044921875, "logps/ref_chosen": -287.8509826660156, "logps/ref_rejected": -256.0766296386719, "logps/rejected": -260.8320617675781, "loss": 5.3056, "margin_dpo/margin_mean": 6.596972465515137, "margin_dpo/margin_std": 13.254508972167969, "step": 80 }, { "KL/chosen_KL_mean": 2.3833045959472656, "KL/mean": -1.1887388229370117, "KL/rejected_KL_mean": -4.760778427124023, "KL/std": 12.248699188232422, "epoch": 0.16963350785340314, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.144074440002441, "fcm_dpo/q_t": 0.48228341341018677, "grad_norm": 28.06484031677246, "learning_rate": 4.93167072587771e-07, "logits/chosen": -0.6383923888206482, "logits/rejected": -0.637448787689209, "logps/chosen": -266.1399230957031, "logps/ref_chosen": -268.5232238769531, "logps/ref_rejected": -237.81137084960938, "logps/rejected": -242.57215881347656, "loss": 5.2942, "margin_dpo/margin_mean": 7.1440749168396, "margin_dpo/margin_std": 16.711862564086914, "step": 81 }, { "KL/chosen_KL_mean": 2.4359188079833984, "KL/mean": -1.5335369110107422, "KL/rejected_KL_mean": -5.50299072265625, "KL/std": 12.241569519042969, "epoch": 0.17172774869109947, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.938918590545654, "fcm_dpo/q_t": 0.4803643226623535, "grad_norm": 27.75286293029785, "learning_rate": 4.92735454356513e-07, "logits/chosen": -0.7282431721687317, "logits/rejected": -0.7358072996139526, "logps/chosen": -276.9280090332031, "logps/ref_chosen": -279.36395263671875, "logps/ref_rejected": -236.51365661621094, "logps/rejected": -242.01666259765625, "loss": 5.2623, "margin_dpo/margin_mean": 7.9389190673828125, "margin_dpo/margin_std": 15.775489807128906, "step": 82 }, { "KL/chosen_KL_mean": 0.3072986602783203, "KL/mean": -4.027276039123535, "KL/rejected_KL_mean": -8.361845016479492, "KL/std": 13.603336334228516, "epoch": 0.17382198952879582, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.669137954711914, "fcm_dpo/q_t": 0.4784778654575348, "grad_norm": 30.750106811523438, "learning_rate": 4.922908189595017e-07, "logits/chosen": -0.6986796855926514, "logits/rejected": -0.6824924945831299, "logps/chosen": -273.91192626953125, "logps/ref_chosen": -274.21923828125, "logps/ref_rejected": -276.2212219238281, "logps/rejected": -284.58306884765625, "loss": 5.2426, "margin_dpo/margin_mean": 8.669137001037598, "margin_dpo/margin_std": 18.15301513671875, "step": 83 }, { "KL/chosen_KL_mean": 0.5734672546386719, "KL/mean": -3.0715651512145996, "KL/rejected_KL_mean": -6.716592788696289, "KL/std": 13.341272354125977, "epoch": 0.17591623036649215, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.290060997009277, "fcm_dpo/q_t": 0.48191478848457336, "grad_norm": 29.86717987060547, "learning_rate": 4.918331902411841e-07, "logits/chosen": -0.7325999140739441, "logits/rejected": -0.7469091415405273, "logps/chosen": -293.8240966796875, "logps/ref_chosen": -294.3975524902344, "logps/ref_rejected": -279.81884765625, "logps/rejected": -286.5354309082031, "loss": 5.29, "margin_dpo/margin_mean": 7.290060520172119, "margin_dpo/margin_std": 16.862646102905273, "step": 84 }, { "KL/chosen_KL_mean": -1.566385269165039, "KL/mean": -4.428199768066406, "KL/rejected_KL_mean": -7.290014266967773, "KL/std": 12.298978805541992, "epoch": 0.17801047120418848, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.723620414733887, "fcm_dpo/q_t": 0.48579931259155273, "grad_norm": 29.314868927001953, "learning_rate": 4.913625927427995e-07, "logits/chosen": -0.6648886203765869, "logits/rejected": -0.6732238531112671, "logps/chosen": -245.22857666015625, "logps/ref_chosen": -243.66220092773438, "logps/ref_rejected": -263.9421691894531, "logps/rejected": -271.2322082519531, "loss": 5.3466, "margin_dpo/margin_mean": 5.723620414733887, "margin_dpo/margin_std": 15.493175506591797, "step": 85 }, { "KL/chosen_KL_mean": 1.2148284912109375, "KL/mean": -3.267716646194458, "KL/rejected_KL_mean": -7.75025749206543, "KL/std": 13.284515380859375, "epoch": 0.18010471204188483, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.965092658996582, "fcm_dpo/q_t": 0.4777436852455139, "grad_norm": 34.879512786865234, "learning_rate": 4.908790517010636e-07, "logits/chosen": -0.691657543182373, "logits/rejected": -0.6852882504463196, "logps/chosen": -308.2158508300781, "logps/ref_chosen": -309.4306945800781, "logps/ref_rejected": -290.91278076171875, "logps/rejected": -298.66302490234375, "loss": 5.2262, "margin_dpo/margin_mean": 8.965091705322266, "margin_dpo/margin_std": 17.340694427490234, "step": 86 }, { "KL/chosen_KL_mean": -0.16178321838378906, "KL/mean": -5.300088882446289, "KL/rejected_KL_mean": -10.438394546508789, "KL/std": 15.289347648620605, "epoch": 0.18219895287958116, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.276609420776367, "fcm_dpo/q_t": 0.47462573647499084, "grad_norm": 29.600753784179688, "learning_rate": 4.903825930468148e-07, "logits/chosen": -0.7515206933021545, "logits/rejected": -0.7456373572349548, "logps/chosen": -278.1894836425781, "logps/ref_chosen": -278.0277099609375, "logps/ref_rejected": -245.70123291015625, "logps/rejected": -256.13958740234375, "loss": 5.1833, "margin_dpo/margin_mean": 10.276611328125, "margin_dpo/margin_std": 18.967639923095703, "step": 87 }, { "KL/chosen_KL_mean": -2.0936412811279297, "KL/mean": -6.569038391113281, "KL/rejected_KL_mean": -11.044445037841797, "KL/std": 15.05484676361084, "epoch": 0.18429319371727748, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.950809478759766, "fcm_dpo/q_t": 0.477876216173172, "grad_norm": 28.82327651977539, "learning_rate": 4.898732434036243e-07, "logits/chosen": -0.7612143754959106, "logits/rejected": -0.7769263386726379, "logps/chosen": -268.60845947265625, "logps/ref_chosen": -266.5148010253906, "logps/ref_rejected": -265.90081787109375, "logps/rejected": -276.94525146484375, "loss": 5.2333, "margin_dpo/margin_mean": 8.950811386108398, "margin_dpo/margin_std": 19.136362075805664, "step": 88 }, { "KL/chosen_KL_mean": 0.00550079345703125, "KL/mean": -4.640643119812012, "KL/rejected_KL_mean": -9.286785125732422, "KL/std": 14.196423530578613, "epoch": 0.18638743455497384, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.292278289794922, "fcm_dpo/q_t": 0.4769834876060486, "grad_norm": 30.551815032958984, "learning_rate": 4.893510300863676e-07, "logits/chosen": -0.7438966631889343, "logits/rejected": -0.7343195080757141, "logps/chosen": -265.68377685546875, "logps/ref_chosen": -265.6893005371094, "logps/ref_rejected": -251.49314880371094, "logps/rejected": -260.7799377441406, "loss": 5.2193, "margin_dpo/margin_mean": 9.292276382446289, "margin_dpo/margin_std": 18.199703216552734, "step": 89 }, { "KL/chosen_KL_mean": -1.1707630157470703, "KL/mean": -5.281407356262207, "KL/rejected_KL_mean": -9.39205551147461, "KL/std": 14.30504322052002, "epoch": 0.18848167539267016, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.221304893493652, "fcm_dpo/q_t": 0.4796198904514313, "grad_norm": 29.808265686035156, "learning_rate": 4.8881598109976e-07, "logits/chosen": -0.7398777008056641, "logits/rejected": -0.7484868764877319, "logps/chosen": -308.59576416015625, "logps/ref_chosen": -307.4250183105469, "logps/ref_rejected": -265.7172546386719, "logps/rejected": -275.10931396484375, "loss": 5.2571, "margin_dpo/margin_mean": 8.221305847167969, "margin_dpo/margin_std": 17.676063537597656, "step": 90 }, { "KL/chosen_KL_mean": -2.1329708099365234, "KL/mean": -6.8853864669799805, "KL/rejected_KL_mean": -11.637798309326172, "KL/std": 14.826963424682617, "epoch": 0.1905759162303665, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.50483512878418, "fcm_dpo/q_t": 0.47655919194221497, "grad_norm": 32.95356750488281, "learning_rate": 4.882681251368548e-07, "logits/chosen": -0.6793421506881714, "logits/rejected": -0.6965538859367371, "logps/chosen": -237.87396240234375, "logps/ref_chosen": -235.74098205566406, "logps/ref_rejected": -226.6428985595703, "logps/rejected": -238.2807159423828, "loss": 5.2162, "margin_dpo/margin_mean": 9.504833221435547, "margin_dpo/margin_std": 19.24348258972168, "step": 91 }, { "KL/chosen_KL_mean": -3.134540557861328, "KL/mean": -8.218965530395508, "KL/rejected_KL_mean": -13.303377151489258, "KL/std": 17.46491050720215, "epoch": 0.19267015706806281, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.168837547302246, "fcm_dpo/q_t": 0.47487232089042664, "grad_norm": 33.593353271484375, "learning_rate": 4.877074915775048e-07, "logits/chosen": -0.7426879405975342, "logits/rejected": -0.7264429330825806, "logps/chosen": -286.58209228515625, "logps/ref_chosen": -283.4475402832031, "logps/ref_rejected": -273.134033203125, "logps/rejected": -286.4374084472656, "loss": 5.1966, "margin_dpo/margin_mean": 10.168838500976562, "margin_dpo/margin_std": 21.30776596069336, "step": 92 }, { "KL/chosen_KL_mean": -2.5298290252685547, "KL/mean": -7.310698986053467, "KL/rejected_KL_mean": -12.091564178466797, "KL/std": 15.477099418640137, "epoch": 0.19476439790575917, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.56174087524414, "fcm_dpo/q_t": 0.4764096438884735, "grad_norm": 28.567649841308594, "learning_rate": 4.871341104867864e-07, "logits/chosen": -0.7310028076171875, "logits/rejected": -0.7545213103294373, "logps/chosen": -235.86695861816406, "logps/ref_chosen": -233.33714294433594, "logps/ref_rejected": -230.54273986816406, "logps/rejected": -242.63427734375, "loss": 5.2129, "margin_dpo/margin_mean": 9.561741828918457, "margin_dpo/margin_std": 19.498998641967773, "step": 93 }, { "KL/chosen_KL_mean": -2.3857421875, "KL/mean": -7.097320079803467, "KL/rejected_KL_mean": -11.808910369873047, "KL/std": 16.434885025024414, "epoch": 0.1968586387434555, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.423165321350098, "fcm_dpo/q_t": 0.47669246792793274, "grad_norm": 31.399282455444336, "learning_rate": 4.865480126133871e-07, "logits/chosen": -0.6810243725776672, "logits/rejected": -0.701765775680542, "logps/chosen": -297.03851318359375, "logps/ref_chosen": -294.6528015136719, "logps/ref_rejected": -283.657958984375, "logps/rejected": -295.4669189453125, "loss": 5.2283, "margin_dpo/margin_mean": 9.423166275024414, "margin_dpo/margin_std": 21.879398345947266, "step": 94 }, { "KL/chosen_KL_mean": -3.322275161743164, "KL/mean": -8.415876388549805, "KL/rejected_KL_mean": -13.50948715209961, "KL/std": 18.067832946777344, "epoch": 0.19895287958115182, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.187210083007812, "fcm_dpo/q_t": 0.474922239780426, "grad_norm": 33.59168243408203, "learning_rate": 4.859492293879573e-07, "logits/chosen": -0.7242058515548706, "logits/rejected": -0.745286226272583, "logps/chosen": -314.9919738769531, "logps/ref_chosen": -311.6697082519531, "logps/ref_rejected": -262.7471923828125, "logps/rejected": -276.2567138671875, "loss": 5.2054, "margin_dpo/margin_mean": 10.187210083007812, "margin_dpo/margin_std": 22.50189208984375, "step": 95 }, { "KL/chosen_KL_mean": -4.764711380004883, "KL/mean": -9.865034103393555, "KL/rejected_KL_mean": -14.965354919433594, "KL/std": 19.30546760559082, "epoch": 0.20104712041884817, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.20064640045166, "fcm_dpo/q_t": 0.4748993217945099, "grad_norm": 32.950992584228516, "learning_rate": 4.853377929214243e-07, "logits/chosen": -0.7030908465385437, "logits/rejected": -0.7156802415847778, "logps/chosen": -287.3207092285156, "logps/ref_chosen": -282.55596923828125, "logps/ref_rejected": -242.71588134765625, "logps/rejected": -257.68121337890625, "loss": 5.2057, "margin_dpo/margin_mean": 10.200647354125977, "margin_dpo/margin_std": 23.298458099365234, "step": 96 }, { "KL/chosen_KL_mean": -4.412008285522461, "KL/mean": -10.414529800415039, "KL/rejected_KL_mean": -16.41705322265625, "KL/std": 17.29755401611328, "epoch": 0.2031413612565445, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.005033493041992, "fcm_dpo/q_t": 0.4704352617263794, "grad_norm": 32.43635940551758, "learning_rate": 4.847137360032699e-07, "logits/chosen": -0.7456763982772827, "logits/rejected": -0.7329200506210327, "logps/chosen": -307.98980712890625, "logps/ref_chosen": -303.57781982421875, "logps/ref_rejected": -264.22491455078125, "logps/rejected": -280.6419982910156, "loss": 5.1326, "margin_dpo/margin_mean": 12.005033493041992, "margin_dpo/margin_std": 22.390636444091797, "step": 97 }, { "KL/chosen_KL_mean": -4.835039138793945, "KL/mean": -10.993499755859375, "KL/rejected_KL_mean": -17.151966094970703, "KL/std": 18.9281005859375, "epoch": 0.20523560209424083, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.31692886352539, "fcm_dpo/q_t": 0.4698448181152344, "grad_norm": 34.54245376586914, "learning_rate": 4.84077092099773e-07, "logits/chosen": -0.7799739837646484, "logits/rejected": -0.7929960489273071, "logps/chosen": -291.6654052734375, "logps/ref_chosen": -286.8303527832031, "logps/ref_rejected": -278.08331298828125, "logps/rejected": -295.23529052734375, "loss": 5.1325, "margin_dpo/margin_mean": 12.31692886352539, "margin_dpo/margin_std": 22.748985290527344, "step": 98 }, { "KL/chosen_KL_mean": -5.303714752197266, "KL/mean": -11.64534854888916, "KL/rejected_KL_mean": -17.986982345581055, "KL/std": 19.881885528564453, "epoch": 0.20732984293193718, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.683259963989258, "fcm_dpo/q_t": 0.4689215123653412, "grad_norm": 32.6298942565918, "learning_rate": 4.834278953522137e-07, "logits/chosen": -0.7404739260673523, "logits/rejected": -0.7537968754768372, "logps/chosen": -285.22491455078125, "logps/ref_chosen": -279.92120361328125, "logps/ref_rejected": -250.3365478515625, "logps/rejected": -268.32354736328125, "loss": 5.1338, "margin_dpo/margin_mean": 12.683258056640625, "margin_dpo/margin_std": 27.28083610534668, "step": 99 }, { "KL/chosen_KL_mean": -7.890239715576172, "KL/mean": -14.145411491394043, "KL/rejected_KL_mean": -20.400583267211914, "KL/std": 21.990659713745117, "epoch": 0.2094240837696335, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.510345458984375, "fcm_dpo/q_t": 0.4693695604801178, "grad_norm": 35.569637298583984, "learning_rate": 4.827661805750437e-07, "logits/chosen": -0.7786294221878052, "logits/rejected": -0.7915595173835754, "logps/chosen": -304.7179260253906, "logps/ref_chosen": -296.8276672363281, "logps/ref_rejected": -275.56146240234375, "logps/rejected": -295.9620666503906, "loss": 5.1248, "margin_dpo/margin_mean": 12.510345458984375, "margin_dpo/margin_std": 24.131572723388672, "step": 100 }, { "KL/chosen_KL_mean": -5.328624725341797, "KL/mean": -12.77022647857666, "KL/rejected_KL_mean": -20.21182632446289, "KL/std": 22.71660041809082, "epoch": 0.21151832460732983, "fcm_dpo/beta": 0.010064744390547276, "fcm_dpo/delta": 0.012882252223789692, "fcm_dpo/margin": 14.883186340332031, "fcm_dpo/q_t": 0.4634419083595276, "grad_norm": 32.82856369018555, "learning_rate": 4.820919832540181e-07, "logits/chosen": -0.7667604684829712, "logits/rejected": -0.7804319262504578, "logps/chosen": -258.0706481933594, "logps/ref_chosen": -252.74203491210938, "logps/ref_rejected": -276.4185485839844, "logps/rejected": -296.6304016113281, "loss": 5.0441, "margin_dpo/margin_mean": 14.883186340332031, "margin_dpo/margin_std": 26.3232421875, "step": 101 }, { "KL/chosen_KL_mean": -5.145671844482422, "KL/mean": -12.559816360473633, "KL/rejected_KL_mean": -19.973960876464844, "KL/std": 19.77225112915039, "epoch": 0.2136125654450262, "fcm_dpo/beta": 0.010103590786457062, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.828295707702637, "fcm_dpo/q_t": 0.4630769193172455, "grad_norm": 32.874755859375, "learning_rate": 4.814053395442932e-07, "logits/chosen": -0.7466103434562683, "logits/rejected": -0.7427460551261902, "logps/chosen": -224.69940185546875, "logps/ref_chosen": -219.5537109375, "logps/ref_rejected": -231.90853881835938, "logps/rejected": -251.88250732421875, "loss": 5.0383, "margin_dpo/margin_mean": 14.828296661376953, "margin_dpo/margin_std": 25.028118133544922, "step": 102 }, { "KL/chosen_KL_mean": -4.823522567749023, "KL/mean": -11.646297454833984, "KL/rejected_KL_mean": -18.46908187866211, "KL/std": 21.67599868774414, "epoch": 0.2157068062827225, "fcm_dpo/beta": 0.010103590786457062, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.645557403564453, "fcm_dpo/q_t": 0.4664449691772461, "grad_norm": 33.129119873046875, "learning_rate": 4.807062862684873e-07, "logits/chosen": -0.7692456245422363, "logits/rejected": -0.7661339044570923, "logps/chosen": -264.4985656738281, "logps/ref_chosen": -259.6750793457031, "logps/ref_rejected": -278.7400817871094, "logps/rejected": -297.20916748046875, "loss": 5.0855, "margin_dpo/margin_mean": 13.64555835723877, "margin_dpo/margin_std": 25.76202392578125, "step": 103 }, { "KL/chosen_KL_mean": -9.094964981079102, "KL/mean": -14.14463996887207, "KL/rejected_KL_mean": -19.194313049316406, "KL/std": 22.547658920288086, "epoch": 0.21780104712041884, "fcm_dpo/beta": 0.010103590786457062, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.099353790283203, "fcm_dpo/q_t": 0.475015789270401, "grad_norm": 33.46672821044922, "learning_rate": 4.799948609147061e-07, "logits/chosen": -0.7669414281845093, "logits/rejected": -0.7730291485786438, "logps/chosen": -277.06915283203125, "logps/ref_chosen": -267.9741516113281, "logps/ref_rejected": -230.5306396484375, "logps/rejected": -249.7249298095703, "loss": 5.2216, "margin_dpo/margin_mean": 10.099355697631836, "margin_dpo/margin_std": 26.440128326416016, "step": 104 }, { "KL/chosen_KL_mean": -5.365236282348633, "KL/mean": -15.661399841308594, "KL/rejected_KL_mean": -25.957565307617188, "KL/std": 25.220060348510742, "epoch": 0.2198952879581152, "fcm_dpo/beta": 0.01026467327028513, "fcm_dpo/delta": 0.03175576403737068, "fcm_dpo/margin": 20.592317581176758, "fcm_dpo/q_t": 0.4485164284706116, "grad_norm": 35.88564682006836, "learning_rate": 4.792711016345321e-07, "logits/chosen": -0.7539578676223755, "logits/rejected": -0.7634439468383789, "logps/chosen": -327.6200866699219, "logps/ref_chosen": -322.25482177734375, "logps/ref_rejected": -279.02978515625, "logps/rejected": -304.9873352050781, "loss": 4.8304, "margin_dpo/margin_mean": 20.592321395874023, "margin_dpo/margin_std": 27.02099609375, "step": 105 }, { "KL/chosen_KL_mean": -12.540826797485352, "KL/mean": -18.82832145690918, "KL/rejected_KL_mean": -25.115812301635742, "KL/std": 23.56497573852539, "epoch": 0.22198952879581152, "fcm_dpo/beta": 0.010363556444644928, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.57497787475586, "fcm_dpo/q_t": 0.46819162368774414, "grad_norm": 39.138980865478516, "learning_rate": 4.785350472409791e-07, "logits/chosen": -0.7461320757865906, "logits/rejected": -0.7831467390060425, "logps/chosen": -308.6986389160156, "logps/ref_chosen": -296.15777587890625, "logps/ref_rejected": -266.2691650390625, "logps/rejected": -291.3849792480469, "loss": 5.1392, "margin_dpo/margin_mean": 12.57497787475586, "margin_dpo/margin_std": 29.43160629272461, "step": 106 }, { "KL/chosen_KL_mean": -4.413387298583984, "KL/mean": -14.257648468017578, "KL/rejected_KL_mean": -24.101911544799805, "KL/std": 22.089332580566406, "epoch": 0.22408376963350785, "fcm_dpo/beta": 0.010363556444644928, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.688522338867188, "fcm_dpo/q_t": 0.45019635558128357, "grad_norm": 39.04673767089844, "learning_rate": 4.777867372064105e-07, "logits/chosen": -0.779481053352356, "logits/rejected": -0.7725954055786133, "logps/chosen": -311.4096984863281, "logps/ref_chosen": -306.996337890625, "logps/ref_rejected": -296.79412841796875, "logps/rejected": -320.89599609375, "loss": 4.8578, "margin_dpo/margin_mean": 19.688522338867188, "margin_dpo/margin_std": 27.66748046875, "step": 107 }, { "KL/chosen_KL_mean": -5.592203140258789, "KL/mean": -14.71112060546875, "KL/rejected_KL_mean": -23.830047607421875, "KL/std": 25.131715774536133, "epoch": 0.2261780104712042, "fcm_dpo/beta": 0.010363556444644928, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.23786163330078, "fcm_dpo/q_t": 0.4543207287788391, "grad_norm": 105.16789245605469, "learning_rate": 4.770262116604223e-07, "logits/chosen": -0.7518104314804077, "logits/rejected": -0.7614055871963501, "logps/chosen": -300.744873046875, "logps/ref_chosen": -295.1526794433594, "logps/ref_rejected": -235.974853515625, "logps/rejected": -259.8049011230469, "loss": 4.9361, "margin_dpo/margin_mean": 18.23785972595215, "margin_dpo/margin_std": 30.31496810913086, "step": 108 }, { "KL/chosen_KL_mean": -8.415664672851562, "KL/mean": -18.587873458862305, "KL/rejected_KL_mean": -28.760072708129883, "KL/std": 25.343238830566406, "epoch": 0.22827225130890053, "fcm_dpo/beta": 0.010363556444644928, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.344406127929688, "fcm_dpo/q_t": 0.4492321312427521, "grad_norm": 39.30132293701172, "learning_rate": 4.7625351138769166e-07, "logits/chosen": -0.794326901435852, "logits/rejected": -0.7915245294570923, "logps/chosen": -334.3404541015625, "logps/ref_chosen": -325.9248046875, "logps/ref_rejected": -279.15423583984375, "logps/rejected": -307.914306640625, "loss": 4.8574, "margin_dpo/margin_mean": 20.344402313232422, "margin_dpo/margin_std": 31.736478805541992, "step": 109 }, { "KL/chosen_KL_mean": -8.976770401000977, "KL/mean": -18.56324005126953, "KL/rejected_KL_mean": -28.149715423583984, "KL/std": 27.376617431640625, "epoch": 0.23036649214659685, "fcm_dpo/beta": 0.010363556444644928, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.172945022583008, "fcm_dpo/q_t": 0.4520251750946045, "grad_norm": 35.762718200683594, "learning_rate": 4.75468677825789e-07, "logits/chosen": -0.7997367978096008, "logits/rejected": -0.7880566716194153, "logps/chosen": -283.4159851074219, "logps/ref_chosen": -274.439208984375, "logps/ref_rejected": -260.0552062988281, "logps/rejected": -288.2049255371094, "loss": 4.9146, "margin_dpo/margin_mean": 19.172948837280273, "margin_dpo/margin_std": 33.746768951416016, "step": 110 }, { "KL/chosen_KL_mean": -9.660625457763672, "KL/mean": -19.830608367919922, "KL/rejected_KL_mean": -30.00059700012207, "KL/std": 28.046205520629883, "epoch": 0.2324607329842932, "fcm_dpo/beta": 0.010417782701551914, "fcm_dpo/delta": 0.017871737480163574, "fcm_dpo/margin": 20.33997344970703, "fcm_dpo/q_t": 0.44924646615982056, "grad_norm": 39.768409729003906, "learning_rate": 4.7467175306295647e-07, "logits/chosen": -0.8416504859924316, "logits/rejected": -0.8213886022567749, "logps/chosen": -338.8967590332031, "logps/ref_chosen": -329.2361755371094, "logps/ref_rejected": -287.82830810546875, "logps/rejected": -317.82891845703125, "loss": 4.8784, "margin_dpo/margin_mean": 20.3399715423584, "margin_dpo/margin_std": 34.47478485107422, "step": 111 }, { "KL/chosen_KL_mean": -15.551521301269531, "KL/mean": -22.11650848388672, "KL/rejected_KL_mean": -28.68149185180664, "KL/std": 27.066593170166016, "epoch": 0.23455497382198953, "fcm_dpo/beta": 0.010552349500358105, "fcm_dpo/delta": 0.006255615968257189, "fcm_dpo/margin": 13.12997055053711, "fcm_dpo/q_t": 0.46686533093452454, "grad_norm": 41.6264533996582, "learning_rate": 4.7386277983585053e-07, "logits/chosen": -0.7422491312026978, "logits/rejected": -0.7724015712738037, "logps/chosen": -272.61090087890625, "logps/ref_chosen": -257.0593566894531, "logps/ref_rejected": -272.9595031738281, "logps/rejected": -301.6409912109375, "loss": 5.176, "margin_dpo/margin_mean": 13.129972457885742, "margin_dpo/margin_std": 36.441139221191406, "step": 112 }, { "KL/chosen_KL_mean": -13.025020599365234, "KL/mean": -25.310611724853516, "KL/rejected_KL_mean": -37.59620666503906, "KL/std": 31.059711456298828, "epoch": 0.23664921465968586, "fcm_dpo/beta": 0.010606877505779266, "fcm_dpo/delta": 0.009444285184144974, "fcm_dpo/margin": 24.57118034362793, "fcm_dpo/q_t": 0.4386439323425293, "grad_norm": 43.13325500488281, "learning_rate": 4.7304180152725024e-07, "logits/chosen": -0.8121810555458069, "logits/rejected": -0.8162283301353455, "logps/chosen": -299.06671142578125, "logps/ref_chosen": -286.0416564941406, "logps/ref_rejected": -270.374267578125, "logps/rejected": -307.9704895019531, "loss": 4.7489, "margin_dpo/margin_mean": 24.571176528930664, "margin_dpo/margin_std": 39.310142517089844, "step": 113 }, { "KL/chosen_KL_mean": -16.87484359741211, "KL/mean": -23.812885284423828, "KL/rejected_KL_mean": -30.750930786132812, "KL/std": 28.061574935913086, "epoch": 0.2387434554973822, "fcm_dpo/beta": 0.010645664297044277, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.876100540161133, "fcm_dpo/q_t": 0.46417757868766785, "grad_norm": 42.24106979370117, "learning_rate": 4.7220886216373085e-07, "logits/chosen": -0.8521884679794312, "logits/rejected": -0.8534945249557495, "logps/chosen": -276.88330078125, "logps/ref_chosen": -260.0084533691406, "logps/ref_rejected": -246.67190551757812, "logps/rejected": -277.4228210449219, "loss": 5.1292, "margin_dpo/margin_mean": 13.876102447509766, "margin_dpo/margin_std": 35.51622009277344, "step": 114 }, { "KL/chosen_KL_mean": -19.06077766418457, "KL/mean": -26.02539825439453, "KL/rejected_KL_mean": -32.990013122558594, "KL/std": 30.866111755371094, "epoch": 0.24083769633507854, "fcm_dpo/beta": 0.01065509207546711, "fcm_dpo/delta": 0.0014752836432307959, "fcm_dpo/margin": 13.929231643676758, "fcm_dpo/q_t": 0.4654514193534851, "grad_norm": 46.440879821777344, "learning_rate": 4.7136400641330245e-07, "logits/chosen": -0.8538047671318054, "logits/rejected": -0.8142991662025452, "logps/chosen": -318.4837341308594, "logps/ref_chosen": -299.4229736328125, "logps/ref_rejected": -272.1186828613281, "logps/rejected": -305.10870361328125, "loss": 5.1532, "margin_dpo/margin_mean": 13.929231643676758, "margin_dpo/margin_std": 38.84375762939453, "step": 115 }, { "KL/chosen_KL_mean": -15.394144058227539, "KL/mean": -24.405548095703125, "KL/rejected_KL_mean": -33.416954040527344, "KL/std": 28.890514373779297, "epoch": 0.24293193717277486, "fcm_dpo/beta": 0.010658235289156437, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.022811889648438, "fcm_dpo/q_t": 0.4536454677581787, "grad_norm": 41.08831024169922, "learning_rate": 4.70507279583015e-07, "logits/chosen": -0.8548110127449036, "logits/rejected": -0.8159914016723633, "logps/chosen": -294.6580810546875, "logps/ref_chosen": -279.263916015625, "logps/ref_rejected": -253.6192169189453, "logps/rejected": -287.0361633300781, "loss": 4.9585, "margin_dpo/margin_mean": 18.02281379699707, "margin_dpo/margin_std": 35.096229553222656, "step": 116 }, { "KL/chosen_KL_mean": -19.165630340576172, "KL/mean": -29.62946128845215, "KL/rejected_KL_mean": -40.093284606933594, "KL/std": 27.6377010345459, "epoch": 0.2450261780104712, "fcm_dpo/beta": 0.010705700144171715, "fcm_dpo/delta": 0.0005413433536887169, "fcm_dpo/margin": 20.927654266357422, "fcm_dpo/q_t": 0.446196049451828, "grad_norm": 45.51142501831055, "learning_rate": 4.6963872761652834e-07, "logits/chosen": -0.8168615698814392, "logits/rejected": -0.8167919516563416, "logps/chosen": -278.3905029296875, "logps/ref_chosen": -259.2248840332031, "logps/ref_rejected": -229.3042755126953, "logps/rejected": -269.3975524902344, "loss": 4.8441, "margin_dpo/margin_mean": 20.927654266357422, "margin_dpo/margin_std": 31.943593978881836, "step": 117 }, { "KL/chosen_KL_mean": -25.37116813659668, "KL/mean": -36.948516845703125, "KL/rejected_KL_mean": -48.525875091552734, "KL/std": 34.855682373046875, "epoch": 0.24712041884816754, "fcm_dpo/beta": 0.010717739351093769, "fcm_dpo/delta": 0.009510215371847153, "fcm_dpo/margin": 23.154699325561523, "fcm_dpo/q_t": 0.44198188185691833, "grad_norm": 48.185428619384766, "learning_rate": 4.687583970916486e-07, "logits/chosen": -0.8087795972824097, "logits/rejected": -0.7963286638259888, "logps/chosen": -292.44183349609375, "logps/ref_chosen": -267.0707092285156, "logps/ref_rejected": -272.7322082519531, "logps/rejected": -321.258056640625, "loss": 4.8341, "margin_dpo/margin_mean": 23.154701232910156, "margin_dpo/margin_std": 43.30189895629883, "step": 118 }, { "KL/chosen_KL_mean": -30.972999572753906, "KL/mean": -39.472694396972656, "KL/rejected_KL_mean": -47.972389221191406, "KL/std": 34.111595153808594, "epoch": 0.24921465968586387, "fcm_dpo/beta": 0.010796227492392063, "fcm_dpo/delta": 0.012761089019477367, "fcm_dpo/margin": 16.9993896484375, "fcm_dpo/q_t": 0.457156240940094, "grad_norm": 48.64939880371094, "learning_rate": 4.6786633521783005e-07, "logits/chosen": -0.8885324597358704, "logits/rejected": -0.8882258534431458, "logps/chosen": -355.649658203125, "logps/ref_chosen": -324.6766357421875, "logps/ref_rejected": -306.0322265625, "logps/rejected": -354.004638671875, "loss": 5.0637, "margin_dpo/margin_mean": 16.999387741088867, "margin_dpo/margin_std": 41.50002670288086, "step": 119 }, { "KL/chosen_KL_mean": -27.831636428833008, "KL/mean": -37.551856994628906, "KL/rejected_KL_mean": -47.27207565307617, "KL/std": 33.026878356933594, "epoch": 0.2513089005235602, "fcm_dpo/beta": 0.010917214676737785, "fcm_dpo/delta": 0.007896372117102146, "fcm_dpo/margin": 19.44042205810547, "fcm_dpo/q_t": 0.450439453125, "grad_norm": 45.14885711669922, "learning_rate": 4.669625898336438e-07, "logits/chosen": -0.8366523385047913, "logits/rejected": -0.8472779393196106, "logps/chosen": -343.0933837890625, "logps/ref_chosen": -315.2617492675781, "logps/ref_rejected": -265.32501220703125, "logps/rejected": -312.5970764160156, "loss": 4.9651, "margin_dpo/margin_mean": 19.440420150756836, "margin_dpo/margin_std": 41.05113983154297, "step": 120 }, { "KL/chosen_KL_mean": -32.51570510864258, "KL/mean": -39.51547622680664, "KL/rejected_KL_mean": -46.51525115966797, "KL/std": 31.86375617980957, "epoch": 0.2534031413612565, "fcm_dpo/beta": 0.01092331949621439, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.999553680419922, "fcm_dpo/q_t": 0.4645032286643982, "grad_norm": 62.9486083984375, "learning_rate": 4.6604720940421207e-07, "logits/chosen": -0.8580417633056641, "logits/rejected": -0.8725881576538086, "logps/chosen": -255.5117950439453, "logps/ref_chosen": -222.99609375, "logps/ref_rejected": -226.92860412597656, "logps/rejected": -273.44384765625, "loss": 5.156, "margin_dpo/margin_mean": 13.999553680419922, "margin_dpo/margin_std": 40.4022102355957, "step": 121 }, { "KL/chosen_KL_mean": -33.064552307128906, "KL/mean": -42.89902114868164, "KL/rejected_KL_mean": -52.733489990234375, "KL/std": 37.320213317871094, "epoch": 0.2554973821989529, "fcm_dpo/beta": 0.011041234247386456, "fcm_dpo/delta": 0.0161091610789299, "fcm_dpo/margin": 19.668941497802734, "fcm_dpo/q_t": 0.4495151937007904, "grad_norm": 60.40846252441406, "learning_rate": 4.651202430186092e-07, "logits/chosen": -0.90716952085495, "logits/rejected": -0.8687220811843872, "logps/chosen": -309.09088134765625, "logps/ref_chosen": -276.02630615234375, "logps/ref_rejected": -277.97418212890625, "logps/rejected": -330.7076721191406, "loss": 4.9921, "margin_dpo/margin_mean": 19.6689453125, "margin_dpo/margin_std": 46.50227355957031, "step": 122 }, { "KL/chosen_KL_mean": -25.650867462158203, "KL/mean": -38.428531646728516, "KL/rejected_KL_mean": -51.206214904785156, "KL/std": 36.957576751708984, "epoch": 0.25759162303664923, "fcm_dpo/beta": 0.011142628267407417, "fcm_dpo/delta": 0.023120440542697906, "fcm_dpo/margin": 25.55533218383789, "fcm_dpo/q_t": 0.43309932947158813, "grad_norm": 53.674034118652344, "learning_rate": 4.6418174038722924e-07, "logits/chosen": -0.8390490412712097, "logits/rejected": -0.8307358026504517, "logps/chosen": -353.80548095703125, "logps/ref_chosen": -328.1546325683594, "logps/ref_rejected": -280.6911315917969, "logps/rejected": -331.8973693847656, "loss": 4.7193, "margin_dpo/margin_mean": 25.55533218383789, "margin_dpo/margin_std": 42.476070404052734, "step": 123 }, { "KL/chosen_KL_mean": -26.246152877807617, "KL/mean": -37.12230682373047, "KL/rejected_KL_mean": -47.998477935791016, "KL/std": 35.47646713256836, "epoch": 0.25968586387434556, "fcm_dpo/beta": 0.01128486730158329, "fcm_dpo/delta": 0.002518674358725548, "fcm_dpo/margin": 21.752336502075195, "fcm_dpo/q_t": 0.44173774123191833, "grad_norm": 57.442718505859375, "learning_rate": 4.6323175183912023e-07, "logits/chosen": -0.8511196374893188, "logits/rejected": -0.8153136372566223, "logps/chosen": -301.9423828125, "logps/ref_chosen": -275.6961975097656, "logps/ref_rejected": -225.361572265625, "logps/rejected": -273.36004638671875, "loss": 4.8391, "margin_dpo/margin_mean": 21.752336502075195, "margin_dpo/margin_std": 39.0461540222168, "step": 124 }, { "KL/chosen_KL_mean": -27.938440322875977, "KL/mean": -37.887481689453125, "KL/rejected_KL_mean": -47.83651351928711, "KL/std": 36.877479553222656, "epoch": 0.2617801047120419, "fcm_dpo/beta": 0.01136357244104147, "fcm_dpo/delta": 0.019005879759788513, "fcm_dpo/margin": 19.8980712890625, "fcm_dpo/q_t": 0.4486285448074341, "grad_norm": 57.97393798828125, "learning_rate": 4.6227032831928483e-07, "logits/chosen": -0.8193236589431763, "logits/rejected": -0.7735443711280823, "logps/chosen": -306.0082092285156, "logps/ref_chosen": -278.06976318359375, "logps/ref_rejected": -265.63873291015625, "logps/rejected": -313.4752197265625, "loss": 5.0226, "margin_dpo/margin_mean": 19.8980712890625, "margin_dpo/margin_std": 50.3729362487793, "step": 125 }, { "KL/chosen_KL_mean": -23.977537155151367, "KL/mean": -35.20510482788086, "KL/rejected_KL_mean": -46.43267822265625, "KL/std": 38.37907028198242, "epoch": 0.2638743455497382, "fcm_dpo/beta": 0.011555435135960579, "fcm_dpo/delta": 0.015333190560340881, "fcm_dpo/margin": 22.455135345458984, "fcm_dpo/q_t": 0.4395897090435028, "grad_norm": 56.987953186035156, "learning_rate": 4.612975213859487e-07, "logits/chosen": -0.8197309970855713, "logits/rejected": -0.8281855583190918, "logps/chosen": -345.3736267089844, "logps/ref_chosen": -321.3960876464844, "logps/ref_rejected": -285.37664794921875, "logps/rejected": -331.8092956542969, "loss": 4.8436, "margin_dpo/margin_mean": 22.455137252807617, "margin_dpo/margin_std": 44.75693893432617, "step": 126 }, { "KL/chosen_KL_mean": -22.364826202392578, "KL/mean": -35.611881256103516, "KL/rejected_KL_mean": -48.85894775390625, "KL/std": 39.49049758911133, "epoch": 0.26596858638743454, "fcm_dpo/beta": 0.011540120467543602, "fcm_dpo/delta": -0.02305636554956436, "fcm_dpo/margin": 26.49411964416504, "fcm_dpo/q_t": 0.4295908808708191, "grad_norm": 56.03866195678711, "learning_rate": 4.603133832077953e-07, "logits/chosen": -0.9088628888130188, "logits/rejected": -0.858267605304718, "logps/chosen": -328.923583984375, "logps/ref_chosen": -306.55877685546875, "logps/ref_rejected": -274.8651428222656, "logps/rejected": -323.72412109375, "loss": 4.7184, "margin_dpo/margin_mean": 26.494121551513672, "margin_dpo/margin_std": 45.3493766784668, "step": 127 }, { "KL/chosen_KL_mean": -17.152809143066406, "KL/mean": -32.300350189208984, "KL/rejected_KL_mean": -47.44788360595703, "KL/std": 41.129150390625, "epoch": 0.2680628272251309, "fcm_dpo/beta": 0.011374343186616898, "fcm_dpo/delta": 0.0034487280063331127, "fcm_dpo/margin": 30.295085906982422, "fcm_dpo/q_t": 0.4207966923713684, "grad_norm": 53.232566833496094, "learning_rate": 4.5931796656116837e-07, "logits/chosen": -0.7918947339057922, "logits/rejected": -0.7883634567260742, "logps/chosen": -282.5501708984375, "logps/ref_chosen": -265.3973693847656, "logps/ref_rejected": -250.9737548828125, "logps/rejected": -298.421630859375, "loss": 4.5958, "margin_dpo/margin_mean": 30.29508399963379, "margin_dpo/margin_std": 49.26608657836914, "step": 128 }, { "KL/chosen_KL_mean": -18.030426025390625, "KL/mean": -31.652067184448242, "KL/rejected_KL_mean": -45.27369689941406, "KL/std": 38.48607635498047, "epoch": 0.27015706806282724, "fcm_dpo/beta": 0.011414232663810253, "fcm_dpo/delta": -0.015051654540002346, "fcm_dpo/margin": 27.243267059326172, "fcm_dpo/q_t": 0.4287635087966919, "grad_norm": 53.86652374267578, "learning_rate": 4.5831132482724193e-07, "logits/chosen": -0.8241338133811951, "logits/rejected": -0.8225374221801758, "logps/chosen": -321.1888427734375, "logps/ref_chosen": -303.158447265625, "logps/ref_rejected": -275.9891052246094, "logps/rejected": -321.2628173828125, "loss": 4.7053, "margin_dpo/margin_mean": 27.243270874023438, "margin_dpo/margin_std": 48.319923400878906, "step": 129 }, { "KL/chosen_KL_mean": -20.707996368408203, "KL/mean": -34.25269317626953, "KL/rejected_KL_mean": -47.7973747253418, "KL/std": 38.006534576416016, "epoch": 0.27225130890052357, "fcm_dpo/beta": 0.011222862638533115, "fcm_dpo/delta": -4.155188798904419e-05, "fcm_dpo/margin": 27.08936309814453, "fcm_dpo/q_t": 0.43000417947769165, "grad_norm": 65.6941146850586, "learning_rate": 4.5729351198915705e-07, "logits/chosen": -0.8021270036697388, "logits/rejected": -0.8400847911834717, "logps/chosen": -307.1153259277344, "logps/ref_chosen": -286.4073486328125, "logps/ref_rejected": -294.38665771484375, "logps/rejected": -342.1840515136719, "loss": 4.7133, "margin_dpo/margin_mean": 27.089365005493164, "margin_dpo/margin_std": 48.50556564331055, "step": 130 }, { "KL/chosen_KL_mean": -25.196441650390625, "KL/mean": -36.10236358642578, "KL/rejected_KL_mean": -47.00827407836914, "KL/std": 42.682491302490234, "epoch": 0.2743455497382199, "fcm_dpo/beta": 0.01131611317396164, "fcm_dpo/delta": 0.008455941453576088, "fcm_dpo/margin": 21.811826705932617, "fcm_dpo/q_t": 0.44346877932548523, "grad_norm": 65.5090103149414, "learning_rate": 4.5626458262912735e-07, "logits/chosen": -0.8783740997314453, "logits/rejected": -0.8263896703720093, "logps/chosen": -336.7615051269531, "logps/ref_chosen": -311.5650634765625, "logps/ref_rejected": -291.62432861328125, "logps/rejected": -338.63262939453125, "loss": 4.971, "margin_dpo/margin_mean": 21.811826705932617, "margin_dpo/margin_std": 51.76227951049805, "step": 131 }, { "KL/chosen_KL_mean": -21.0625, "KL/mean": -36.35379409790039, "KL/rejected_KL_mean": -51.645084381103516, "KL/std": 42.20893478393555, "epoch": 0.2764397905759162, "fcm_dpo/beta": 0.011329811997711658, "fcm_dpo/delta": -0.031071798875927925, "fcm_dpo/margin": 30.582592010498047, "fcm_dpo/q_t": 0.42160820960998535, "grad_norm": 69.17401885986328, "learning_rate": 4.5522459192551166e-07, "logits/chosen": -0.821100115776062, "logits/rejected": -0.8047419190406799, "logps/chosen": -291.14434814453125, "logps/ref_chosen": -270.0818176269531, "logps/ref_rejected": -284.3084411621094, "logps/rejected": -335.95355224609375, "loss": 4.6525, "margin_dpo/margin_mean": 30.582592010498047, "margin_dpo/margin_std": 53.2596321105957, "step": 132 }, { "KL/chosen_KL_mean": -24.293067932128906, "KL/mean": -38.521732330322266, "KL/rejected_KL_mean": -52.75040054321289, "KL/std": 39.787479400634766, "epoch": 0.27853403141361255, "fcm_dpo/beta": 0.011118058115243912, "fcm_dpo/delta": -0.0034186802804470062, "fcm_dpo/margin": 28.45734405517578, "fcm_dpo/q_t": 0.42733392119407654, "grad_norm": 47.61275863647461, "learning_rate": 4.541735956498554e-07, "logits/chosen": -0.8711456656455994, "logits/rejected": -0.8716740012168884, "logps/chosen": -309.9144287109375, "logps/ref_chosen": -285.6213684082031, "logps/ref_rejected": -251.19386291503906, "logps/rejected": -303.9442443847656, "loss": 4.687, "margin_dpo/margin_mean": 28.45734405517578, "margin_dpo/margin_std": 49.6102409362793, "step": 133 }, { "KL/chosen_KL_mean": -36.26886749267578, "KL/mean": -46.98722457885742, "KL/rejected_KL_mean": -57.70557403564453, "KL/std": 41.17924499511719, "epoch": 0.2806282722513089, "fcm_dpo/beta": 0.010998277924954891, "fcm_dpo/delta": -0.009528097696602345, "fcm_dpo/margin": 21.43670654296875, "fcm_dpo/q_t": 0.44455987215042114, "grad_norm": 54.783790588378906, "learning_rate": 4.5311165016389914e-07, "logits/chosen": -0.8688364028930664, "logits/rejected": -0.8680624961853027, "logps/chosen": -355.1896667480469, "logps/ref_chosen": -318.92083740234375, "logps/ref_rejected": -293.1894836425781, "logps/rejected": -350.8950500488281, "loss": 4.9126, "margin_dpo/margin_mean": 21.43670654296875, "margin_dpo/margin_std": 44.431854248046875, "step": 134 }, { "KL/chosen_KL_mean": -33.41929244995117, "KL/mean": -45.94215393066406, "KL/rejected_KL_mean": -58.465023040771484, "KL/std": 43.048866271972656, "epoch": 0.28272251308900526, "fcm_dpo/beta": 0.011012822389602661, "fcm_dpo/delta": -0.011346682906150818, "fcm_dpo/margin": 25.045719146728516, "fcm_dpo/q_t": 0.43511971831321716, "grad_norm": 64.9057388305664, "learning_rate": 4.520388124165564e-07, "logits/chosen": -0.7530911564826965, "logits/rejected": -0.7919206619262695, "logps/chosen": -326.24102783203125, "logps/ref_chosen": -292.8217468261719, "logps/ref_rejected": -269.2896728515625, "logps/rejected": -327.7547302246094, "loss": 4.8295, "margin_dpo/margin_mean": 25.04572296142578, "margin_dpo/margin_std": 48.300167083740234, "step": 135 }, { "KL/chosen_KL_mean": -35.22936248779297, "KL/mean": -49.50828552246094, "KL/rejected_KL_mean": -63.787208557128906, "KL/std": 43.139225006103516, "epoch": 0.2848167539267016, "fcm_dpo/beta": 0.010835347697138786, "fcm_dpo/delta": -0.016382994130253792, "fcm_dpo/margin": 28.557849884033203, "fcm_dpo/q_t": 0.42960232496261597, "grad_norm": 70.99027252197266, "learning_rate": 4.5095513994085974e-07, "logits/chosen": -0.8074231147766113, "logits/rejected": -0.8016695976257324, "logps/chosen": -308.0819091796875, "logps/ref_chosen": -272.8525390625, "logps/ref_rejected": -252.68202209472656, "logps/rejected": -316.4692077636719, "loss": 4.752, "margin_dpo/margin_mean": 28.557851791381836, "margin_dpo/margin_std": 53.87636947631836, "step": 136 }, { "KL/chosen_KL_mean": -39.36355972290039, "KL/mean": -51.80638122558594, "KL/rejected_KL_mean": -64.24920654296875, "KL/std": 45.04750442504883, "epoch": 0.2869109947643979, "fcm_dpo/beta": 0.010752377100288868, "fcm_dpo/delta": 0.006089036352932453, "fcm_dpo/margin": 24.88566780090332, "fcm_dpo/q_t": 0.43942371010780334, "grad_norm": 61.39313507080078, "learning_rate": 4.498606908508753e-07, "logits/chosen": -0.8615954518318176, "logits/rejected": -0.8479353189468384, "logps/chosen": -340.11578369140625, "logps/ref_chosen": -300.7522277832031, "logps/ref_rejected": -286.1935119628906, "logps/rejected": -350.4427185058594, "loss": 4.8589, "margin_dpo/margin_mean": 24.88566780090332, "margin_dpo/margin_std": 51.68086624145508, "step": 137 }, { "KL/chosen_KL_mean": -36.756832122802734, "KL/mean": -51.21699523925781, "KL/rejected_KL_mean": -65.67715454101562, "KL/std": 46.030006408691406, "epoch": 0.28900523560209423, "fcm_dpo/beta": 0.010876420885324478, "fcm_dpo/delta": 0.0157970879226923, "fcm_dpo/margin": 28.920316696166992, "fcm_dpo/q_t": 0.4301770329475403, "grad_norm": 61.19403076171875, "learning_rate": 4.487555238385862e-07, "logits/chosen": -0.7898247241973877, "logits/rejected": -0.7696442604064941, "logps/chosen": -325.6938171386719, "logps/ref_chosen": -288.9369812011719, "logps/ref_rejected": -263.7076416015625, "logps/rejected": -329.3847961425781, "loss": 4.8013, "margin_dpo/margin_mean": 28.920316696166992, "margin_dpo/margin_std": 60.27662658691406, "step": 138 }, { "KL/chosen_KL_mean": -33.69660186767578, "KL/mean": -43.981441497802734, "KL/rejected_KL_mean": -54.26628494262695, "KL/std": 41.30760955810547, "epoch": 0.29109947643979056, "fcm_dpo/beta": 0.010893644765019417, "fcm_dpo/delta": -0.007741590961813927, "fcm_dpo/margin": 20.569684982299805, "fcm_dpo/q_t": 0.4489022493362427, "grad_norm": 62.34196472167969, "learning_rate": 4.476396981707453e-07, "logits/chosen": -0.8070241808891296, "logits/rejected": -0.8368173241615295, "logps/chosen": -303.74090576171875, "logps/ref_chosen": -270.0443115234375, "logps/ref_rejected": -267.3226013183594, "logps/rejected": -321.5888977050781, "loss": 5.0213, "margin_dpo/margin_mean": 20.569684982299805, "margin_dpo/margin_std": 50.81401824951172, "step": 139 }, { "KL/chosen_KL_mean": -31.74407386779785, "KL/mean": -47.28278350830078, "KL/rejected_KL_mean": -62.821502685546875, "KL/std": 43.96257019042969, "epoch": 0.2931937172774869, "fcm_dpo/beta": 0.010792272165417671, "fcm_dpo/delta": -0.03338925540447235, "fcm_dpo/margin": 31.077434539794922, "fcm_dpo/q_t": 0.422089159488678, "grad_norm": 58.86613464355469, "learning_rate": 4.4651327368569684e-07, "logits/chosen": -0.883962869644165, "logits/rejected": -0.8495863676071167, "logps/chosen": -314.69964599609375, "logps/ref_chosen": -282.9555969238281, "logps/ref_rejected": -251.17181396484375, "logps/rejected": -313.9932861328125, "loss": 4.5964, "margin_dpo/margin_mean": 31.077436447143555, "margin_dpo/margin_std": 47.86009216308594, "step": 140 }, { "KL/chosen_KL_mean": -30.595308303833008, "KL/mean": -46.62080383300781, "KL/rejected_KL_mean": -62.64628982543945, "KL/std": 45.07204818725586, "epoch": 0.29528795811518327, "fcm_dpo/beta": 0.010443806648254395, "fcm_dpo/delta": -0.02163812890648842, "fcm_dpo/margin": 32.050987243652344, "fcm_dpo/q_t": 0.4220162034034729, "grad_norm": 65.77468872070312, "learning_rate": 4.453763107901675e-07, "logits/chosen": -0.8113507628440857, "logits/rejected": -0.8057313561439514, "logps/chosen": -326.89544677734375, "logps/ref_chosen": -296.3001708984375, "logps/ref_rejected": -279.8486633300781, "logps/rejected": -342.4949645996094, "loss": 4.6623, "margin_dpo/margin_mean": 32.05098342895508, "margin_dpo/margin_std": 56.02928161621094, "step": 141 }, { "KL/chosen_KL_mean": -31.103105545043945, "KL/mean": -44.4693603515625, "KL/rejected_KL_mean": -57.835609436035156, "KL/std": 45.70125198364258, "epoch": 0.2973821989528796, "fcm_dpo/beta": 0.010357570834457874, "fcm_dpo/delta": -0.00366947241127491, "fcm_dpo/margin": 26.73249626159668, "fcm_dpo/q_t": 0.43658334016799927, "grad_norm": 60.299896240234375, "learning_rate": 4.4422887045602674e-07, "logits/chosen": -0.8335211277008057, "logits/rejected": -0.8346509337425232, "logps/chosen": -331.6689453125, "logps/ref_chosen": -300.56585693359375, "logps/ref_rejected": -231.43316650390625, "logps/rejected": -289.2687683105469, "loss": 4.8729, "margin_dpo/margin_mean": 26.73249626159668, "margin_dpo/margin_std": 57.07372283935547, "step": 142 }, { "KL/chosen_KL_mean": -31.008270263671875, "KL/mean": -47.164031982421875, "KL/rejected_KL_mean": -63.31980895996094, "KL/std": 43.575416564941406, "epoch": 0.2994764397905759, "fcm_dpo/beta": 0.010205346159636974, "fcm_dpo/delta": -0.026661261916160583, "fcm_dpo/margin": 32.3115348815918, "fcm_dpo/q_t": 0.4239623248577118, "grad_norm": 55.74364471435547, "learning_rate": 4.4307101421701755e-07, "logits/chosen": -0.8169223666191101, "logits/rejected": -0.794075071811676, "logps/chosen": -327.7406005859375, "logps/ref_chosen": -296.73236083984375, "logps/ref_rejected": -266.45257568359375, "logps/rejected": -329.7724304199219, "loss": 4.6115, "margin_dpo/margin_mean": 32.3115348815918, "margin_dpo/margin_std": 51.006839752197266, "step": 143 }, { "KL/chosen_KL_mean": -31.154144287109375, "KL/mean": -44.3969612121582, "KL/rejected_KL_mean": -57.63978958129883, "KL/std": 46.97825241088867, "epoch": 0.30157068062827225, "fcm_dpo/beta": 0.00994843989610672, "fcm_dpo/delta": -0.016245760023593903, "fcm_dpo/margin": 26.485647201538086, "fcm_dpo/q_t": 0.44019222259521484, "grad_norm": 52.55347442626953, "learning_rate": 4.419028041654559e-07, "logits/chosen": -0.8847938179969788, "logits/rejected": -0.8704959750175476, "logps/chosen": -329.9981689453125, "logps/ref_chosen": -298.843994140625, "logps/ref_rejected": -266.120849609375, "logps/rejected": -323.7606201171875, "loss": 4.8825, "margin_dpo/margin_mean": 26.48564910888672, "margin_dpo/margin_std": 56.594215393066406, "step": 144 }, { "KL/chosen_KL_mean": -27.540496826171875, "KL/mean": -44.87952423095703, "KL/rejected_KL_mean": -62.218544006347656, "KL/std": 43.72524642944336, "epoch": 0.3036649214659686, "fcm_dpo/beta": 0.009765357710421085, "fcm_dpo/delta": -0.04112963750958443, "fcm_dpo/margin": 34.678043365478516, "fcm_dpo/q_t": 0.42156580090522766, "grad_norm": 56.111358642578125, "learning_rate": 4.4072430294890166e-07, "logits/chosen": -0.8678039312362671, "logits/rejected": -0.8692808747291565, "logps/chosen": -303.2933654785156, "logps/ref_chosen": -275.7528381347656, "logps/ref_rejected": -214.74807739257812, "logps/rejected": -276.96661376953125, "loss": 4.591, "margin_dpo/margin_mean": 34.678043365478516, "margin_dpo/margin_std": 53.235076904296875, "step": 145 }, { "KL/chosen_KL_mean": -36.69147491455078, "KL/mean": -53.71113586425781, "KL/rejected_KL_mean": -70.73080444335938, "KL/std": 48.02214813232422, "epoch": 0.3057591623036649, "fcm_dpo/beta": 0.009542811661958694, "fcm_dpo/delta": -0.024183155968785286, "fcm_dpo/margin": 34.03933334350586, "fcm_dpo/q_t": 0.4251405894756317, "grad_norm": 62.798160552978516, "learning_rate": 4.395355737667985e-07, "logits/chosen": -0.8429842591285706, "logits/rejected": -0.8354379534721375, "logps/chosen": -313.7897033691406, "logps/ref_chosen": -277.09820556640625, "logps/ref_rejected": -265.41046142578125, "logps/rejected": -336.1412658691406, "loss": 4.6317, "margin_dpo/margin_mean": 34.03933334350586, "margin_dpo/margin_std": 53.72346496582031, "step": 146 }, { "KL/chosen_KL_mean": -38.27781677246094, "KL/mean": -51.62920379638672, "KL/rejected_KL_mean": -64.98057556152344, "KL/std": 49.75927734375, "epoch": 0.3078534031413613, "fcm_dpo/beta": 0.009431717917323112, "fcm_dpo/delta": -0.03413345292210579, "fcm_dpo/margin": 26.7027530670166, "fcm_dpo/q_t": 0.44130846858024597, "grad_norm": 56.538326263427734, "learning_rate": 4.3833668036708483e-07, "logits/chosen": -0.8454154133796692, "logits/rejected": -0.8405569195747375, "logps/chosen": -329.6964111328125, "logps/ref_chosen": -291.4185791015625, "logps/ref_rejected": -253.43051147460938, "logps/rejected": -318.41107177734375, "loss": 4.9113, "margin_dpo/margin_mean": 26.7027530670166, "margin_dpo/margin_std": 55.443599700927734, "step": 147 }, { "KL/chosen_KL_mean": -37.951026916503906, "KL/mean": -52.09737777709961, "KL/rejected_KL_mean": -66.24372863769531, "KL/std": 52.47624206542969, "epoch": 0.3099476439790576, "fcm_dpo/beta": 0.009357105009257793, "fcm_dpo/delta": 0.03023623488843441, "fcm_dpo/margin": 28.292701721191406, "fcm_dpo/q_t": 0.44127410650253296, "grad_norm": 59.94662094116211, "learning_rate": 4.3712768704277524e-07, "logits/chosen": -0.9010493755340576, "logits/rejected": -0.9029269218444824, "logps/chosen": -274.69952392578125, "logps/ref_chosen": -236.74850463867188, "logps/ref_rejected": -231.4674072265625, "logps/rejected": -297.7110900878906, "loss": 4.9011, "margin_dpo/margin_mean": 28.292701721191406, "margin_dpo/margin_std": 63.87792205810547, "step": 148 }, { "KL/chosen_KL_mean": -42.97991180419922, "KL/mean": -62.22007369995117, "KL/rejected_KL_mean": -81.46024322509766, "KL/std": 48.12861633300781, "epoch": 0.31204188481675393, "fcm_dpo/beta": 0.00953738298267126, "fcm_dpo/delta": 0.0038804998621344566, "fcm_dpo/margin": 38.4803352355957, "fcm_dpo/q_t": 0.41513967514038086, "grad_norm": 60.234527587890625, "learning_rate": 4.3590865862851263e-07, "logits/chosen": -0.8748192191123962, "logits/rejected": -0.8590805530548096, "logps/chosen": -362.9084167480469, "logps/ref_chosen": -319.9284973144531, "logps/ref_rejected": -308.20233154296875, "logps/rejected": -389.66259765625, "loss": 4.4618, "margin_dpo/margin_mean": 38.4803352355957, "margin_dpo/margin_std": 53.24681854248047, "step": 149 }, { "KL/chosen_KL_mean": -50.06086349487305, "KL/mean": -64.94522094726562, "KL/rejected_KL_mean": -79.8295669555664, "KL/std": 50.34340286254883, "epoch": 0.31413612565445026, "fcm_dpo/beta": 0.009615819901227951, "fcm_dpo/delta": 0.017674576491117477, "fcm_dpo/margin": 29.768699645996094, "fcm_dpo/q_t": 0.43368738889694214, "grad_norm": 67.94093322753906, "learning_rate": 4.346796604970912e-07, "logits/chosen": -0.839390754699707, "logits/rejected": -0.822877824306488, "logps/chosen": -326.37908935546875, "logps/ref_chosen": -276.3182373046875, "logps/ref_rejected": -273.02215576171875, "logps/rejected": -352.8516845703125, "loss": 4.7679, "margin_dpo/margin_mean": 29.768701553344727, "margin_dpo/margin_std": 55.33635711669922, "step": 150 }, { "KL/chosen_KL_mean": -46.528472900390625, "KL/mean": -68.42727661132812, "KL/rejected_KL_mean": -90.32608032226562, "KL/std": 51.12382507324219, "epoch": 0.3162303664921466, "fcm_dpo/beta": 0.009591327980160713, "fcm_dpo/delta": -0.02124784141778946, "fcm_dpo/margin": 43.797607421875, "fcm_dpo/q_t": 0.40414297580718994, "grad_norm": 72.9853744506836, "learning_rate": 4.3344075855595097e-07, "logits/chosen": -0.8729795217514038, "logits/rejected": -0.8649865984916687, "logps/chosen": -343.84130859375, "logps/ref_chosen": -297.31280517578125, "logps/ref_rejected": -266.1003723144531, "logps/rejected": -356.4264831542969, "loss": 4.3725, "margin_dpo/margin_mean": 43.797607421875, "margin_dpo/margin_std": 59.94606399536133, "step": 151 }, { "KL/chosen_KL_mean": -40.6783561706543, "KL/mean": -59.743812561035156, "KL/rejected_KL_mean": -78.80927276611328, "KL/std": 54.26198959350586, "epoch": 0.3183246073298429, "fcm_dpo/beta": 0.009200290776789188, "fcm_dpo/delta": -0.05215620622038841, "fcm_dpo/margin": 38.13091278076172, "fcm_dpo/q_t": 0.4210585653781891, "grad_norm": 58.28529739379883, "learning_rate": 4.3219201924364323e-07, "logits/chosen": -0.87758469581604, "logits/rejected": -0.8735326528549194, "logps/chosen": -310.92535400390625, "logps/ref_chosen": -270.2470397949219, "logps/ref_rejected": -269.7749328613281, "logps/rejected": -348.5841979980469, "loss": 4.6203, "margin_dpo/margin_mean": 38.130916595458984, "margin_dpo/margin_std": 63.328189849853516, "step": 152 }, { "KL/chosen_KL_mean": -48.53391647338867, "KL/mean": -75.49513244628906, "KL/rejected_KL_mean": -102.45635986328125, "KL/std": 58.56542205810547, "epoch": 0.3204188481675393, "fcm_dpo/beta": 0.008681675419211388, "fcm_dpo/delta": -0.08731570094823837, "fcm_dpo/margin": 53.92245864868164, "fcm_dpo/q_t": 0.3927898108959198, "grad_norm": 58.35908126831055, "learning_rate": 4.309335095262675e-07, "logits/chosen": -0.8716917037963867, "logits/rejected": -0.858845055103302, "logps/chosen": -322.31298828125, "logps/ref_chosen": -273.779052734375, "logps/ref_rejected": -280.9530944824219, "logps/rejected": -383.40948486328125, "loss": 4.1636, "margin_dpo/margin_mean": 53.92245864868164, "margin_dpo/margin_std": 61.229759216308594, "step": 153 }, { "KL/chosen_KL_mean": -48.57218551635742, "KL/mean": -69.53777313232422, "KL/rejected_KL_mean": -90.50334930419922, "KL/std": 60.53078842163086, "epoch": 0.3225130890052356, "fcm_dpo/beta": 0.008394555188715458, "fcm_dpo/delta": -0.020893538370728493, "fcm_dpo/margin": 41.9311637878418, "fcm_dpo/q_t": 0.4201442301273346, "grad_norm": 72.38925170898438, "learning_rate": 4.2966529689388064e-07, "logits/chosen": -0.9051564335823059, "logits/rejected": -0.8851950764656067, "logps/chosen": -338.4753723144531, "logps/ref_chosen": -289.9031982421875, "logps/ref_rejected": -261.5166320800781, "logps/rejected": -352.02001953125, "loss": 4.6266, "margin_dpo/margin_mean": 41.9311637878418, "margin_dpo/margin_std": 71.46162414550781, "step": 154 }, { "KL/chosen_KL_mean": -53.91712188720703, "KL/mean": -72.27357482910156, "KL/rejected_KL_mean": -90.63002014160156, "KL/std": 57.95890808105469, "epoch": 0.32460732984293195, "fcm_dpo/beta": 0.008404220454394817, "fcm_dpo/delta": 0.010009624063968658, "fcm_dpo/margin": 36.712890625, "fcm_dpo/q_t": 0.43113821744918823, "grad_norm": 70.87203979492188, "learning_rate": 4.2838744935687716e-07, "logits/chosen": -0.8529869914054871, "logits/rejected": -0.8497177958488464, "logps/chosen": -339.7783203125, "logps/ref_chosen": -285.8612060546875, "logps/ref_rejected": -300.1272888183594, "logps/rejected": -390.75732421875, "loss": 4.7199, "margin_dpo/margin_mean": 36.71288299560547, "margin_dpo/margin_std": 66.47289276123047, "step": 155 }, { "KL/chosen_KL_mean": -49.25990676879883, "KL/mean": -74.67170715332031, "KL/rejected_KL_mean": -100.0835189819336, "KL/std": 61.44602584838867, "epoch": 0.3267015706806283, "fcm_dpo/beta": 0.008251436054706573, "fcm_dpo/delta": -0.07570717483758926, "fcm_dpo/margin": 50.82361602783203, "fcm_dpo/q_t": 0.405461847782135, "grad_norm": 55.39389419555664, "learning_rate": 4.271000354423425e-07, "logits/chosen": -0.8789178133010864, "logits/rejected": -0.8763436675071716, "logps/chosen": -328.2953796386719, "logps/ref_chosen": -279.0354919433594, "logps/ref_rejected": -244.2198486328125, "logps/rejected": -344.3033447265625, "loss": 4.4554, "margin_dpo/margin_mean": 50.82362365722656, "margin_dpo/margin_std": 77.3794937133789, "step": 156 }, { "KL/chosen_KL_mean": -52.569740295410156, "KL/mean": -72.73023986816406, "KL/rejected_KL_mean": -92.89073181152344, "KL/std": 61.27772521972656, "epoch": 0.3287958115183246, "fcm_dpo/beta": 0.007722177542746067, "fcm_dpo/delta": -0.04277050867676735, "fcm_dpo/margin": 40.32099533081055, "fcm_dpo/q_t": 0.42746734619140625, "grad_norm": 55.638614654541016, "learning_rate": 4.258031241903777e-07, "logits/chosen": -0.9371187686920166, "logits/rejected": -0.9353018999099731, "logps/chosen": -323.4000549316406, "logps/ref_chosen": -270.830322265625, "logps/ref_rejected": -259.08319091796875, "logps/rejected": -351.9739685058594, "loss": 4.6749, "margin_dpo/margin_mean": 40.32099533081055, "margin_dpo/margin_std": 66.63133239746094, "step": 157 }, { "KL/chosen_KL_mean": -51.89894485473633, "KL/mean": -74.62893676757812, "KL/rejected_KL_mean": -97.35890197753906, "KL/std": 62.313785552978516, "epoch": 0.3308900523560209, "fcm_dpo/beta": 0.007630071137100458, "fcm_dpo/delta": -0.01720864325761795, "fcm_dpo/margin": 45.4599494934082, "fcm_dpo/q_t": 0.41904595494270325, "grad_norm": 56.36635208129883, "learning_rate": 4.2449678515039743e-07, "logits/chosen": -0.8542776107788086, "logits/rejected": -0.8377709984779358, "logps/chosen": -341.8652648925781, "logps/ref_chosen": -289.9663391113281, "logps/ref_rejected": -271.335693359375, "logps/rejected": -368.694580078125, "loss": 4.5601, "margin_dpo/margin_mean": 45.4599494934082, "margin_dpo/margin_std": 67.09923553466797, "step": 158 }, { "KL/chosen_KL_mean": -59.03363800048828, "KL/mean": -72.34220123291016, "KL/rejected_KL_mean": -85.65077209472656, "KL/std": 57.92660140991211, "epoch": 0.33298429319371725, "fcm_dpo/beta": 0.007565178908407688, "fcm_dpo/delta": 0.008062109351158142, "fcm_dpo/margin": 26.61713409423828, "fcm_dpo/q_t": 0.45483458042144775, "grad_norm": 76.1928939819336, "learning_rate": 4.2318108837739986e-07, "logits/chosen": -0.947105348110199, "logits/rejected": -0.8976367712020874, "logps/chosen": -380.4119873046875, "logps/ref_chosen": -321.37835693359375, "logps/ref_rejected": -250.45652770996094, "logps/rejected": -336.1072692871094, "loss": 5.1068, "margin_dpo/margin_mean": 26.61713409423828, "margin_dpo/margin_std": 72.70268249511719, "step": 159 }, { "KL/chosen_KL_mean": -48.30318832397461, "KL/mean": -73.82183837890625, "KL/rejected_KL_mean": -99.34048461914062, "KL/std": 61.44125747680664, "epoch": 0.33507853403141363, "fcm_dpo/beta": 0.007519013714045286, "fcm_dpo/delta": -0.013213069178164005, "fcm_dpo/margin": 51.03730773925781, "fcm_dpo/q_t": 0.41031157970428467, "grad_norm": 55.092437744140625, "learning_rate": 4.218561044282098e-07, "logits/chosen": -0.8815222978591919, "logits/rejected": -0.8827785849571228, "logps/chosen": -324.586669921875, "logps/ref_chosen": -276.28350830078125, "logps/ref_rejected": -262.7477722167969, "logps/rejected": -362.0882568359375, "loss": 4.3609, "margin_dpo/margin_mean": 51.03730773925781, "margin_dpo/margin_std": 59.32886505126953, "step": 160 }, { "KL/chosen_KL_mean": -59.115379333496094, "KL/mean": -81.7115707397461, "KL/rejected_KL_mean": -104.30776977539062, "KL/std": 64.58517456054688, "epoch": 0.33717277486910996, "fcm_dpo/beta": 0.007456418592482805, "fcm_dpo/delta": -0.019674938172101974, "fcm_dpo/margin": 45.19239044189453, "fcm_dpo/q_t": 0.4239409565925598, "grad_norm": 75.19351959228516, "learning_rate": 4.2052190435769554e-07, "logits/chosen": -0.9016933441162109, "logits/rejected": -0.8840854167938232, "logps/chosen": -369.6081848144531, "logps/ref_chosen": -310.4927978515625, "logps/ref_rejected": -250.25347900390625, "logps/rejected": -354.5612487792969, "loss": 4.6386, "margin_dpo/margin_mean": 45.19239044189453, "margin_dpo/margin_std": 76.1778564453125, "step": 161 }, { "KL/chosen_KL_mean": -54.95409393310547, "KL/mean": -75.90670776367188, "KL/rejected_KL_mean": -96.85931396484375, "KL/std": 60.112361907958984, "epoch": 0.3392670157068063, "fcm_dpo/beta": 0.007265571039170027, "fcm_dpo/delta": -0.0053212931379675865, "fcm_dpo/margin": 41.90523147583008, "fcm_dpo/q_t": 0.429195761680603, "grad_norm": 51.2578125, "learning_rate": 4.1917855971495763e-07, "logits/chosen": -0.8682603240013123, "logits/rejected": -0.8593798875808716, "logps/chosen": -351.0645751953125, "logps/ref_chosen": -296.1105041503906, "logps/ref_rejected": -253.4247589111328, "logps/rejected": -350.2840881347656, "loss": 4.6797, "margin_dpo/margin_mean": 41.90523147583008, "margin_dpo/margin_std": 67.80030822753906, "step": 162 }, { "KL/chosen_KL_mean": -60.050079345703125, "KL/mean": -83.6735610961914, "KL/rejected_KL_mean": -107.29702758789062, "KL/std": 59.25013732910156, "epoch": 0.3413612565445026, "fcm_dpo/beta": 0.007304114755243063, "fcm_dpo/delta": -0.019453469663858414, "fcm_dpo/margin": 47.2469482421875, "fcm_dpo/q_t": 0.41881659626960754, "grad_norm": 63.22408676147461, "learning_rate": 4.1782614253949255e-07, "logits/chosen": -0.9068223237991333, "logits/rejected": -0.9022581577301025, "logps/chosen": -353.54998779296875, "logps/ref_chosen": -293.4999084472656, "logps/ref_rejected": -266.7116394042969, "logps/rejected": -374.0086975097656, "loss": 4.524, "margin_dpo/margin_mean": 47.2469482421875, "margin_dpo/margin_std": 64.95793914794922, "step": 163 }, { "KL/chosen_KL_mean": -71.46724700927734, "KL/mean": -92.78965759277344, "KL/rejected_KL_mean": -114.11207580566406, "KL/std": 66.23802947998047, "epoch": 0.34345549738219894, "fcm_dpo/beta": 0.007272779941558838, "fcm_dpo/delta": -0.011227300390601158, "fcm_dpo/margin": 42.64482116699219, "fcm_dpo/q_t": 0.4294374883174896, "grad_norm": 73.2182846069336, "learning_rate": 4.164647253573289e-07, "logits/chosen": -0.8723958730697632, "logits/rejected": -0.8833568096160889, "logps/chosen": -338.5167541503906, "logps/ref_chosen": -267.04949951171875, "logps/ref_rejected": -215.9768829345703, "logps/rejected": -330.08892822265625, "loss": 4.7089, "margin_dpo/margin_mean": 42.64482116699219, "margin_dpo/margin_std": 75.42022705078125, "step": 164 }, { "KL/chosen_KL_mean": -60.704872131347656, "KL/mean": -76.39503479003906, "KL/rejected_KL_mean": -92.085205078125, "KL/std": 55.665733337402344, "epoch": 0.34554973821989526, "fcm_dpo/beta": 0.007172515615820885, "fcm_dpo/delta": 0.003400525078177452, "fcm_dpo/margin": 31.38031768798828, "fcm_dpo/q_t": 0.44702640175819397, "grad_norm": 70.69437408447266, "learning_rate": 4.1509438117713863e-07, "logits/chosen": -0.9045889973640442, "logits/rejected": -0.877446711063385, "logps/chosen": -338.766357421875, "logps/ref_chosen": -278.06146240234375, "logps/ref_rejected": -260.4288635253906, "logps/rejected": -352.5140686035156, "loss": 4.9154, "margin_dpo/margin_mean": 31.38031768798828, "margin_dpo/margin_std": 62.93089294433594, "step": 165 }, { "KL/chosen_KL_mean": -56.28936767578125, "KL/mean": -76.24816131591797, "KL/rejected_KL_mean": -96.20693969726562, "KL/std": 58.851802825927734, "epoch": 0.34764397905759165, "fcm_dpo/beta": 0.007088161073625088, "fcm_dpo/delta": -0.018855443224310875, "fcm_dpo/margin": 39.91756820678711, "fcm_dpo/q_t": 0.43469470739364624, "grad_norm": 65.55958557128906, "learning_rate": 4.137151834863213e-07, "logits/chosen": -0.8408677577972412, "logits/rejected": -0.8041284084320068, "logps/chosen": -332.2384338378906, "logps/ref_chosen": -275.9490661621094, "logps/ref_rejected": -232.13473510742188, "logps/rejected": -328.3417053222656, "loss": 4.7531, "margin_dpo/margin_mean": 39.91757583618164, "margin_dpo/margin_std": 71.8862075805664, "step": 166 }, { "KL/chosen_KL_mean": -66.38935852050781, "KL/mean": -88.67193603515625, "KL/rejected_KL_mean": -110.95450592041016, "KL/std": 64.25645446777344, "epoch": 0.34973821989528797, "fcm_dpo/beta": 0.007166431751102209, "fcm_dpo/delta": 0.017246726900339127, "fcm_dpo/margin": 44.56515884399414, "fcm_dpo/q_t": 0.4254525899887085, "grad_norm": 65.34365844726562, "learning_rate": 4.123272062470633e-07, "logits/chosen": -0.8350700736045837, "logits/rejected": -0.8198394179344177, "logps/chosen": -346.9407653808594, "logps/ref_chosen": -280.5514221191406, "logps/ref_rejected": -255.2896728515625, "logps/rejected": -366.24420166015625, "loss": 4.6514, "margin_dpo/margin_mean": 44.56515884399414, "margin_dpo/margin_std": 73.58280181884766, "step": 167 }, { "KL/chosen_KL_mean": -68.78351593017578, "KL/mean": -92.84286499023438, "KL/rejected_KL_mean": -116.9021987915039, "KL/std": 67.12002563476562, "epoch": 0.3518324607329843, "fcm_dpo/beta": 0.007110526319593191, "fcm_dpo/delta": -0.016713347285985947, "fcm_dpo/margin": 48.118675231933594, "fcm_dpo/q_t": 0.42008110880851746, "grad_norm": 129.8765869140625, "learning_rate": 4.1093052389237174e-07, "logits/chosen": -0.829613447189331, "logits/rejected": -0.8005003333091736, "logps/chosen": -384.58172607421875, "logps/ref_chosen": -315.7982177734375, "logps/ref_rejected": -291.48406982421875, "logps/rejected": -408.38629150390625, "loss": 4.5751, "margin_dpo/margin_mean": 48.118675231933594, "margin_dpo/margin_std": 73.80198669433594, "step": 168 }, { "KL/chosen_KL_mean": -65.46214294433594, "KL/mean": -94.92474365234375, "KL/rejected_KL_mean": -124.38734436035156, "KL/std": 68.11466979980469, "epoch": 0.3539267015706806, "fcm_dpo/beta": 0.006964054889976978, "fcm_dpo/delta": -0.05502891540527344, "fcm_dpo/margin": 58.92519760131836, "fcm_dpo/q_t": 0.40527647733688354, "grad_norm": 64.64112091064453, "learning_rate": 4.0952521132208267e-07, "logits/chosen": -0.826874852180481, "logits/rejected": -0.8327090740203857, "logps/chosen": -326.5263977050781, "logps/ref_chosen": -261.06427001953125, "logps/ref_rejected": -235.40663146972656, "logps/rejected": -359.7939453125, "loss": 4.3467, "margin_dpo/margin_mean": 58.925201416015625, "margin_dpo/margin_std": 73.77999877929688, "step": 169 }, { "KL/chosen_KL_mean": -64.50227355957031, "KL/mean": -88.4448013305664, "KL/rejected_KL_mean": -112.3873062133789, "KL/std": 70.87551879882812, "epoch": 0.35602094240837695, "fcm_dpo/beta": 0.006804569158703089, "fcm_dpo/delta": 0.01784161478281021, "fcm_dpo/margin": 47.88505172729492, "fcm_dpo/q_t": 0.4266098141670227, "grad_norm": 65.7511978149414, "learning_rate": 4.081113438988443e-07, "logits/chosen": -0.8047292828559875, "logits/rejected": -0.8058911561965942, "logps/chosen": -373.4695129394531, "logps/ref_chosen": -308.96722412109375, "logps/ref_rejected": -263.8466796875, "logps/rejected": -376.2340087890625, "loss": 4.6903, "margin_dpo/margin_mean": 47.88505172729492, "margin_dpo/margin_std": 88.50460815429688, "step": 170 }, { "KL/chosen_KL_mean": -54.57660675048828, "KL/mean": -83.85132598876953, "KL/rejected_KL_mean": -113.12605285644531, "KL/std": 67.29131317138672, "epoch": 0.3581151832460733, "fcm_dpo/beta": 0.006733529269695282, "fcm_dpo/delta": -0.03675668314099312, "fcm_dpo/margin": 58.549434661865234, "fcm_dpo/q_t": 0.40839463472366333, "grad_norm": 64.48705291748047, "learning_rate": 4.0668899744407567e-07, "logits/chosen": -0.8200169801712036, "logits/rejected": -0.8341271877288818, "logps/chosen": -313.46563720703125, "logps/ref_chosen": -258.8890380859375, "logps/ref_rejected": -262.19140625, "logps/rejected": -375.31744384765625, "loss": 4.378, "margin_dpo/margin_mean": 58.5494384765625, "margin_dpo/margin_std": 74.06875610351562, "step": 171 }, { "KL/chosen_KL_mean": -60.651363372802734, "KL/mean": -76.64715576171875, "KL/rejected_KL_mean": -92.64295959472656, "KL/std": 60.878604888916016, "epoch": 0.36020942408376966, "fcm_dpo/beta": 0.006755727343261242, "fcm_dpo/delta": 0.020635303109884262, "fcm_dpo/margin": 31.991588592529297, "fcm_dpo/q_t": 0.44898900389671326, "grad_norm": 55.07887268066406, "learning_rate": 4.0525824823390043e-07, "logits/chosen": -0.8459972143173218, "logits/rejected": -0.8613450527191162, "logps/chosen": -399.6737060546875, "logps/ref_chosen": -339.0223388671875, "logps/ref_rejected": -295.78759765625, "logps/rejected": -388.4305419921875, "loss": 4.9779, "margin_dpo/margin_mean": 31.991588592529297, "margin_dpo/margin_std": 74.38744354248047, "step": 172 }, { "KL/chosen_KL_mean": -57.42732620239258, "KL/mean": -79.39176940917969, "KL/rejected_KL_mean": -101.35624694824219, "KL/std": 64.20226287841797, "epoch": 0.362303664921466, "fcm_dpo/beta": 0.006686392240226269, "fcm_dpo/delta": -0.030118033289909363, "fcm_dpo/margin": 43.928924560546875, "fcm_dpo/q_t": 0.43261343240737915, "grad_norm": 80.68160247802734, "learning_rate": 4.0381917299505686e-07, "logits/chosen": -0.855338990688324, "logits/rejected": -0.8514031171798706, "logps/chosen": -357.5387878417969, "logps/ref_chosen": -300.1114501953125, "logps/ref_rejected": -273.78460693359375, "logps/rejected": -375.140869140625, "loss": 4.7279, "margin_dpo/margin_mean": 43.928924560546875, "margin_dpo/margin_std": 75.71229553222656, "step": 173 }, { "KL/chosen_KL_mean": -57.29583740234375, "KL/mean": -83.97393035888672, "KL/rejected_KL_mean": -110.65203857421875, "KL/std": 64.6370849609375, "epoch": 0.3643979057591623, "fcm_dpo/beta": 0.006674854550510645, "fcm_dpo/delta": 0.012298239395022392, "fcm_dpo/margin": 53.3561897277832, "fcm_dpo/q_t": 0.4156748950481415, "grad_norm": 72.79425811767578, "learning_rate": 4.0237184890078243e-07, "logits/chosen": -0.8256723284721375, "logits/rejected": -0.805625855922699, "logps/chosen": -392.3497009277344, "logps/ref_chosen": -335.0538635253906, "logps/ref_rejected": -257.4646911621094, "logps/rejected": -368.11669921875, "loss": 4.4548, "margin_dpo/margin_mean": 53.3561897277832, "margin_dpo/margin_std": 67.76395416259766, "step": 174 }, { "KL/chosen_KL_mean": -72.9652328491211, "KL/mean": -96.99482727050781, "KL/rejected_KL_mean": -121.0244140625, "KL/std": 73.48253631591797, "epoch": 0.36649214659685864, "fcm_dpo/beta": 0.006608583033084869, "fcm_dpo/delta": -0.04623459279537201, "fcm_dpo/margin": 48.05918884277344, "fcm_dpo/q_t": 0.4289584159851074, "grad_norm": 79.97168731689453, "learning_rate": 4.00916353566676e-07, "logits/chosen": -0.8397626280784607, "logits/rejected": -0.8402819633483887, "logps/chosen": -357.36083984375, "logps/ref_chosen": -284.39556884765625, "logps/ref_rejected": -283.3876647949219, "logps/rejected": -404.41204833984375, "loss": 4.7035, "margin_dpo/margin_mean": 48.05918884277344, "margin_dpo/margin_std": 82.69506072998047, "step": 175 }, { "KL/chosen_KL_mean": -70.91928100585938, "KL/mean": -89.49232482910156, "KL/rejected_KL_mean": -108.06539916992188, "KL/std": 68.54695129394531, "epoch": 0.36858638743455496, "fcm_dpo/beta": 0.006419216748327017, "fcm_dpo/delta": -0.016511470079421997, "fcm_dpo/margin": 37.14612579345703, "fcm_dpo/q_t": 0.4452558159828186, "grad_norm": 69.9654312133789, "learning_rate": 3.994527650465352e-07, "logits/chosen": -0.8216432929039001, "logits/rejected": -0.8281827569007874, "logps/chosen": -322.7320861816406, "logps/ref_chosen": -251.81280517578125, "logps/ref_rejected": -242.05328369140625, "logps/rejected": -350.1186828613281, "loss": 4.9657, "margin_dpo/margin_mean": 37.14612579345703, "margin_dpo/margin_std": 86.01189422607422, "step": 176 }, { "KL/chosen_KL_mean": -77.08590698242188, "KL/mean": -96.39353942871094, "KL/rejected_KL_mean": -115.70115661621094, "KL/std": 66.26946258544922, "epoch": 0.3706806282722513, "fcm_dpo/beta": 0.006435380782932043, "fcm_dpo/delta": 0.015266492031514645, "fcm_dpo/margin": 38.615257263183594, "fcm_dpo/q_t": 0.44256123900413513, "grad_norm": 64.47611999511719, "learning_rate": 3.979811618281705e-07, "logits/chosen": -0.8922350406646729, "logits/rejected": -0.860856831073761, "logps/chosen": -375.7322998046875, "logps/ref_chosen": -298.6463928222656, "logps/ref_rejected": -295.66534423828125, "logps/rejected": -411.36651611328125, "loss": 4.9252, "margin_dpo/margin_mean": 38.615257263183594, "margin_dpo/margin_std": 81.1719970703125, "step": 177 }, { "KL/chosen_KL_mean": -62.48854064941406, "KL/mean": -90.35904693603516, "KL/rejected_KL_mean": -118.22955322265625, "KL/std": 71.95285034179688, "epoch": 0.37277486910994767, "fcm_dpo/beta": 0.006367242429405451, "fcm_dpo/delta": -0.016469581052660942, "fcm_dpo/margin": 55.74102020263672, "fcm_dpo/q_t": 0.41873496770858765, "grad_norm": 55.844093322753906, "learning_rate": 3.9650162282919654e-07, "logits/chosen": -0.8221363425254822, "logits/rejected": -0.8154042363166809, "logps/chosen": -348.7462158203125, "logps/ref_chosen": -286.2576599121094, "logps/ref_rejected": -243.97491455078125, "logps/rejected": -362.2044677734375, "loss": 4.5338, "margin_dpo/margin_mean": 55.74102020263672, "margin_dpo/margin_std": 83.90606689453125, "step": 178 }, { "KL/chosen_KL_mean": -69.41419219970703, "KL/mean": -93.62001037597656, "KL/rejected_KL_mean": -117.8258056640625, "KL/std": 72.83280181884766, "epoch": 0.374869109947644, "fcm_dpo/beta": 0.006365994457155466, "fcm_dpo/delta": -0.003825224470347166, "fcm_dpo/margin": 48.41162109375, "fcm_dpo/q_t": 0.42837071418762207, "grad_norm": 88.72013854980469, "learning_rate": 3.9501422739279953e-07, "logits/chosen": -0.8086428642272949, "logits/rejected": -0.8089961409568787, "logps/chosen": -329.1512451171875, "logps/ref_chosen": -259.737060546875, "logps/ref_rejected": -277.8813171386719, "logps/rejected": -395.7071533203125, "loss": 4.6943, "margin_dpo/margin_mean": 48.41161346435547, "margin_dpo/margin_std": 81.75579071044922, "step": 179 }, { "KL/chosen_KL_mean": -75.5199966430664, "KL/mean": -102.80406188964844, "KL/rejected_KL_mean": -130.08815002441406, "KL/std": 69.5558090209961, "epoch": 0.3769633507853403, "fcm_dpo/beta": 0.006364882458001375, "fcm_dpo/delta": 0.006663701497018337, "fcm_dpo/margin": 54.568145751953125, "fcm_dpo/q_t": 0.421146959066391, "grad_norm": 86.5185317993164, "learning_rate": 3.935190552834828e-07, "logits/chosen": -0.8502231240272522, "logits/rejected": -0.8749803900718689, "logps/chosen": -342.8288879394531, "logps/ref_chosen": -267.30889892578125, "logps/ref_rejected": -230.4376983642578, "logps/rejected": -360.5258483886719, "loss": 4.5368, "margin_dpo/margin_mean": 54.568138122558594, "margin_dpo/margin_std": 80.2123031616211, "step": 180 }, { "KL/chosen_KL_mean": -86.32603454589844, "KL/mean": -108.53008270263672, "KL/rejected_KL_mean": -130.734130859375, "KL/std": 71.4383544921875, "epoch": 0.37905759162303665, "fcm_dpo/beta": 0.006331674288958311, "fcm_dpo/delta": 0.0014796480536460876, "fcm_dpo/margin": 44.40808868408203, "fcm_dpo/q_t": 0.43516260385513306, "grad_norm": 98.99745178222656, "learning_rate": 3.920161866827889e-07, "logits/chosen": -0.8576675057411194, "logits/rejected": -0.8596795201301575, "logps/chosen": -386.81744384765625, "logps/ref_chosen": -300.49139404296875, "logps/ref_rejected": -278.98284912109375, "logps/rejected": -409.71697998046875, "loss": 4.7925, "margin_dpo/margin_mean": 44.40808868408203, "margin_dpo/margin_std": 85.0110855102539, "step": 181 }, { "KL/chosen_KL_mean": -83.20874786376953, "KL/mean": -115.09024810791016, "KL/rejected_KL_mean": -146.97177124023438, "KL/std": 71.0909652709961, "epoch": 0.381151832460733, "fcm_dpo/beta": 0.0062592774629592896, "fcm_dpo/delta": -0.056663818657398224, "fcm_dpo/margin": 63.763023376464844, "fcm_dpo/q_t": 0.40782204270362854, "grad_norm": 63.953773498535156, "learning_rate": 3.90505702185e-07, "logits/chosen": -0.837653398513794, "logits/rejected": -0.8604555130004883, "logps/chosen": -362.7068786621094, "logps/ref_chosen": -279.4981689453125, "logps/ref_rejected": -263.6926574707031, "logps/rejected": -410.6643981933594, "loss": 4.3798, "margin_dpo/margin_mean": 63.76302719116211, "margin_dpo/margin_std": 82.07465362548828, "step": 182 }, { "KL/chosen_KL_mean": -85.31684875488281, "KL/mean": -112.9171371459961, "KL/rejected_KL_mean": -140.51742553710938, "KL/std": 73.43064880371094, "epoch": 0.3832460732984293, "fcm_dpo/beta": 0.006128930952399969, "fcm_dpo/delta": 0.017954856157302856, "fcm_dpo/margin": 55.20054626464844, "fcm_dpo/q_t": 0.4230417311191559, "grad_norm": 52.553104400634766, "learning_rate": 3.889876827928156e-07, "logits/chosen": -0.8802267909049988, "logits/rejected": -0.8798730373382568, "logps/chosen": -356.5225524902344, "logps/ref_chosen": -271.2057189941406, "logps/ref_rejected": -243.91549682617188, "logps/rejected": -384.4328918457031, "loss": 4.6031, "margin_dpo/margin_mean": 55.20054626464844, "margin_dpo/margin_std": 89.55872344970703, "step": 183 }, { "KL/chosen_KL_mean": -81.1993179321289, "KL/mean": -120.1291732788086, "KL/rejected_KL_mean": -159.05902099609375, "KL/std": 81.69471740722656, "epoch": 0.38534031413612563, "fcm_dpo/beta": 0.005876118782907724, "fcm_dpo/delta": -0.07986614853143692, "fcm_dpo/margin": 77.8597183227539, "fcm_dpo/q_t": 0.3941153287887573, "grad_norm": 65.2461929321289, "learning_rate": 3.874622099130087e-07, "logits/chosen": -0.9231401085853577, "logits/rejected": -0.897407591342926, "logps/chosen": -399.64508056640625, "logps/ref_chosen": -318.4457702636719, "logps/ref_rejected": -266.640869140625, "logps/rejected": -425.69989013671875, "loss": 4.21, "margin_dpo/margin_mean": 77.8597183227539, "margin_dpo/margin_std": 92.04389190673828, "step": 184 }, { "KL/chosen_KL_mean": -98.42439270019531, "KL/mean": -126.96806335449219, "KL/rejected_KL_mean": -155.51171875, "KL/std": 88.64209747314453, "epoch": 0.387434554973822, "fcm_dpo/beta": 0.005731166340410709, "fcm_dpo/delta": -0.018187183886766434, "fcm_dpo/margin": 57.087345123291016, "fcm_dpo/q_t": 0.42507031559944153, "grad_norm": 85.08312225341797, "learning_rate": 3.859293653520604e-07, "logits/chosen": -0.8975915312767029, "logits/rejected": -0.889983594417572, "logps/chosen": -372.7332458496094, "logps/ref_chosen": -274.308837890625, "logps/ref_rejected": -260.7274169921875, "logps/rejected": -416.2391357421875, "loss": 4.6385, "margin_dpo/margin_mean": 57.087345123291016, "margin_dpo/margin_std": 94.04627227783203, "step": 185 }, { "KL/chosen_KL_mean": -101.5023193359375, "KL/mean": -128.28860473632812, "KL/rejected_KL_mean": -155.07489013671875, "KL/std": 79.86894989013672, "epoch": 0.38952879581151834, "fcm_dpo/beta": 0.005754129029810429, "fcm_dpo/delta": 0.00872084405273199, "fcm_dpo/margin": 53.57258987426758, "fcm_dpo/q_t": 0.42865413427352905, "grad_norm": 102.2063980102539, "learning_rate": 3.8438923131177237e-07, "logits/chosen": -0.9187572002410889, "logits/rejected": -0.9172680974006653, "logps/chosen": -400.5076904296875, "logps/ref_chosen": -299.00537109375, "logps/ref_rejected": -274.4014587402344, "logps/rejected": -429.47637939453125, "loss": 4.682, "margin_dpo/margin_mean": 53.57258605957031, "margin_dpo/margin_std": 89.2789535522461, "step": 186 }, { "KL/chosen_KL_mean": -112.88499450683594, "KL/mean": -136.58340454101562, "KL/rejected_KL_mean": -160.28182983398438, "KL/std": 84.63465118408203, "epoch": 0.39162303664921466, "fcm_dpo/beta": 0.0057259490713477135, "fcm_dpo/delta": -0.0088438019156456, "fcm_dpo/margin": 47.3968505859375, "fcm_dpo/q_t": 0.4351907968521118, "grad_norm": 104.48662567138672, "learning_rate": 3.828418903848593e-07, "logits/chosen": -0.8610119223594666, "logits/rejected": -0.8431413769721985, "logps/chosen": -442.7103271484375, "logps/ref_chosen": -329.8253173828125, "logps/ref_rejected": -263.73175048828125, "logps/rejected": -424.0135803222656, "loss": 4.8913, "margin_dpo/margin_mean": 47.396854400634766, "margin_dpo/margin_std": 102.80694580078125, "step": 187 }, { "KL/chosen_KL_mean": -101.29950714111328, "KL/mean": -128.2642364501953, "KL/rejected_KL_mean": -155.22898864746094, "KL/std": 87.04512023925781, "epoch": 0.393717277486911, "fcm_dpo/beta": 0.005684699863195419, "fcm_dpo/delta": -0.011355758644640446, "fcm_dpo/margin": 53.92945861816406, "fcm_dpo/q_t": 0.43097540736198425, "grad_norm": 83.95048522949219, "learning_rate": 3.812874255505191e-07, "logits/chosen": -0.8679317235946655, "logits/rejected": -0.8562878370285034, "logps/chosen": -364.3051452636719, "logps/ref_chosen": -263.005615234375, "logps/ref_rejected": -247.08668518066406, "logps/rejected": -402.315673828125, "loss": 4.7949, "margin_dpo/margin_mean": 53.92945861816406, "margin_dpo/margin_std": 108.377685546875, "step": 188 }, { "KL/chosen_KL_mean": -84.28831481933594, "KL/mean": -118.6055908203125, "KL/rejected_KL_mean": -152.9228973388672, "KL/std": 86.47982788085938, "epoch": 0.3958115183246073, "fcm_dpo/beta": 0.00552529888227582, "fcm_dpo/delta": -0.042275551706552505, "fcm_dpo/margin": 68.63457489013672, "fcm_dpo/q_t": 0.4125663638114929, "grad_norm": 86.17080688476562, "learning_rate": 3.797259201699833e-07, "logits/chosen": -0.8880312442779541, "logits/rejected": -0.8941810131072998, "logps/chosen": -357.24871826171875, "logps/ref_chosen": -272.96038818359375, "logps/ref_rejected": -275.13238525390625, "logps/rejected": -428.0552978515625, "loss": 4.4343, "margin_dpo/margin_mean": 68.63457489013672, "margin_dpo/margin_std": 91.30592346191406, "step": 189 }, { "KL/chosen_KL_mean": -82.46324157714844, "KL/mean": -112.3152847290039, "KL/rejected_KL_mean": -142.16734313964844, "KL/std": 79.62899780273438, "epoch": 0.39790575916230364, "fcm_dpo/beta": 0.0054209912195801735, "fcm_dpo/delta": -0.019242707639932632, "fcm_dpo/margin": 59.70409393310547, "fcm_dpo/q_t": 0.4244186282157898, "grad_norm": 67.50397491455078, "learning_rate": 3.781574579820464e-07, "logits/chosen": -0.8963736891746521, "logits/rejected": -0.8552230000495911, "logps/chosen": -340.2608337402344, "logps/ref_chosen": -257.79754638671875, "logps/ref_rejected": -225.2164306640625, "logps/rejected": -367.3837585449219, "loss": 4.5841, "margin_dpo/margin_mean": 59.70409393310547, "margin_dpo/margin_std": 87.59053039550781, "step": 190 }, { "KL/chosen_KL_mean": -74.05329132080078, "KL/mean": -102.2022705078125, "KL/rejected_KL_mean": -130.35125732421875, "KL/std": 79.56758117675781, "epoch": 0.4, "fcm_dpo/beta": 0.005430443678051233, "fcm_dpo/delta": 0.0181720070540905, "fcm_dpo/margin": 56.29796600341797, "fcm_dpo/q_t": 0.42908400297164917, "grad_norm": 76.7206802368164, "learning_rate": 3.765821230985757e-07, "logits/chosen": -0.9064384698867798, "logits/rejected": -0.9073519706726074, "logps/chosen": -317.9118347167969, "logps/ref_chosen": -243.8585205078125, "logps/ref_rejected": -245.12136840820312, "logps/rejected": -375.47265625, "loss": 4.6776, "margin_dpo/margin_mean": 56.29796600341797, "margin_dpo/margin_std": 94.13633728027344, "step": 191 }, { "KL/chosen_KL_mean": -82.34419250488281, "KL/mean": -105.46398162841797, "KL/rejected_KL_mean": -128.58377075195312, "KL/std": 74.32576751708984, "epoch": 0.40209424083769635, "fcm_dpo/beta": 0.005527670495212078, "fcm_dpo/delta": 0.020868487656116486, "fcm_dpo/margin": 46.239559173583984, "fcm_dpo/q_t": 0.44014275074005127, "grad_norm": 95.28579711914062, "learning_rate": 3.75e-07, "logits/chosen": -0.8447480797767639, "logits/rejected": -0.8279674053192139, "logps/chosen": -349.3241271972656, "logps/ref_chosen": -266.9799499511719, "logps/ref_rejected": -260.1697082519531, "logps/rejected": -388.7534484863281, "loss": 4.807, "margin_dpo/margin_mean": 46.23956298828125, "margin_dpo/margin_std": 82.4874038696289, "step": 192 }, { "KL/chosen_KL_mean": -95.75445556640625, "KL/mean": -122.55574035644531, "KL/rejected_KL_mean": -149.35702514648438, "KL/std": 84.06842041015625, "epoch": 0.4041884816753927, "fcm_dpo/beta": 0.005618045572191477, "fcm_dpo/delta": 0.020845647901296616, "fcm_dpo/margin": 53.602569580078125, "fcm_dpo/q_t": 0.4309660792350769, "grad_norm": 59.146549224853516, "learning_rate": 3.734111735307796e-07, "logits/chosen": -0.8828919529914856, "logits/rejected": -0.8536106944084167, "logps/chosen": -376.0076904296875, "logps/ref_chosen": -280.25323486328125, "logps/ref_rejected": -291.0348815917969, "logps/rejected": -440.39190673828125, "loss": 4.7189, "margin_dpo/margin_mean": 53.602569580078125, "margin_dpo/margin_std": 93.06439208984375, "step": 193 }, { "KL/chosen_KL_mean": -94.51910400390625, "KL/mean": -116.68457794189453, "KL/rejected_KL_mean": -138.85008239746094, "KL/std": 80.56071472167969, "epoch": 0.406282722513089, "fcm_dpo/beta": 0.00563051225617528, "fcm_dpo/delta": -0.008683041669428349, "fcm_dpo/margin": 44.33095932006836, "fcm_dpo/q_t": 0.4425777792930603, "grad_norm": 60.31032180786133, "learning_rate": 3.7181572889485623e-07, "logits/chosen": -0.8641772270202637, "logits/rejected": -0.8540708422660828, "logps/chosen": -382.6585693359375, "logps/ref_chosen": -288.13946533203125, "logps/ref_rejected": -251.31529235839844, "logps/rejected": -390.1653747558594, "loss": 4.8567, "margin_dpo/margin_mean": 44.33095932006836, "margin_dpo/margin_std": 86.52152252197266, "step": 194 }, { "KL/chosen_KL_mean": -95.61801147460938, "KL/mean": -117.46638488769531, "KL/rejected_KL_mean": -139.3147735595703, "KL/std": 76.0776596069336, "epoch": 0.4083769633507853, "fcm_dpo/beta": 0.005567906890064478, "fcm_dpo/delta": -0.02082628197968006, "fcm_dpo/margin": 43.6967658996582, "fcm_dpo/q_t": 0.44454753398895264, "grad_norm": 55.601314544677734, "learning_rate": 3.7021375165108377e-07, "logits/chosen": -0.8613829612731934, "logits/rejected": -0.865749716758728, "logps/chosen": -369.6186218261719, "logps/ref_chosen": -274.0006408691406, "logps/ref_rejected": -280.22723388671875, "logps/rejected": -419.5419921875, "loss": 4.8687, "margin_dpo/margin_mean": 43.6967658996582, "margin_dpo/margin_std": 84.557373046875, "step": 195 }, { "KL/chosen_KL_mean": -85.62928771972656, "KL/mean": -110.2017822265625, "KL/rejected_KL_mean": -134.7742919921875, "KL/std": 79.14740753173828, "epoch": 0.41047120418848165, "fcm_dpo/beta": 0.005564381834119558, "fcm_dpo/delta": 0.017413640394806862, "fcm_dpo/margin": 49.14500045776367, "fcm_dpo/q_t": 0.4364781379699707, "grad_norm": 77.14463806152344, "learning_rate": 3.6860532770864005e-07, "logits/chosen": -0.8474195003509521, "logits/rejected": -0.8541578650474548, "logps/chosen": -360.52996826171875, "logps/ref_chosen": -274.90069580078125, "logps/ref_rejected": -248.7281951904297, "logps/rejected": -383.5024719238281, "loss": 4.7765, "margin_dpo/margin_mean": 49.14500045776367, "margin_dpo/margin_std": 88.91536712646484, "step": 196 }, { "KL/chosen_KL_mean": -91.18795776367188, "KL/mean": -121.93099212646484, "KL/rejected_KL_mean": -152.67404174804688, "KL/std": 80.69728088378906, "epoch": 0.41256544502617803, "fcm_dpo/beta": 0.00554257445037365, "fcm_dpo/delta": -0.030014852061867714, "fcm_dpo/margin": 61.48606491088867, "fcm_dpo/q_t": 0.4207172095775604, "grad_norm": 74.54379272460938, "learning_rate": 3.6699054332241985e-07, "logits/chosen": -0.8761681318283081, "logits/rejected": -0.8605803847312927, "logps/chosen": -400.7228088378906, "logps/ref_chosen": -309.5348205566406, "logps/ref_rejected": -264.3179931640625, "logps/rejected": -416.99200439453125, "loss": 4.5427, "margin_dpo/margin_mean": 61.48606491088867, "margin_dpo/margin_std": 85.9816665649414, "step": 197 }, { "KL/chosen_KL_mean": -89.14295959472656, "KL/mean": -122.2314453125, "KL/rejected_KL_mean": -155.31993103027344, "KL/std": 85.83655548095703, "epoch": 0.41465968586387436, "fcm_dpo/beta": 0.00542075652629137, "fcm_dpo/delta": -0.011349959298968315, "fcm_dpo/margin": 66.17696380615234, "fcm_dpo/q_t": 0.4171242415904999, "grad_norm": 58.74266052246094, "learning_rate": 3.653694850884091e-07, "logits/chosen": -0.8896830081939697, "logits/rejected": -0.8643697500228882, "logps/chosen": -390.1564025878906, "logps/ref_chosen": -301.0134582519531, "logps/ref_rejected": -292.84185791015625, "logps/rejected": -448.1617736816406, "loss": 4.5038, "margin_dpo/margin_mean": 66.17696380615234, "margin_dpo/margin_std": 95.03239440917969, "step": 198 }, { "KL/chosen_KL_mean": -100.06723022460938, "KL/mean": -126.93196868896484, "KL/rejected_KL_mean": -153.79669189453125, "KL/std": 82.8907241821289, "epoch": 0.4167539267015707, "fcm_dpo/beta": 0.005536025855690241, "fcm_dpo/delta": 0.02695927582681179, "fcm_dpo/margin": 53.729469299316406, "fcm_dpo/q_t": 0.43211570382118225, "grad_norm": 89.9813461303711, "learning_rate": 3.6374223993904124e-07, "logits/chosen": -0.8591454029083252, "logits/rejected": -0.8150307536125183, "logps/chosen": -364.6730651855469, "logps/ref_chosen": -264.6058654785156, "logps/ref_rejected": -214.9014892578125, "logps/rejected": -368.69818115234375, "loss": 4.7389, "margin_dpo/margin_mean": 53.729469299316406, "margin_dpo/margin_std": 98.36550903320312, "step": 199 }, { "KL/chosen_KL_mean": -110.9617919921875, "KL/mean": -136.12791442871094, "KL/rejected_KL_mean": -161.29400634765625, "KL/std": 87.3275146484375, "epoch": 0.418848167539267, "fcm_dpo/beta": 0.005466805305331945, "fcm_dpo/delta": -0.027961192652583122, "fcm_dpo/margin": 50.332218170166016, "fcm_dpo/q_t": 0.4367090165615082, "grad_norm": 66.81365966796875, "learning_rate": 3.621088951385353e-07, "logits/chosen": -0.9206555485725403, "logits/rejected": -0.8960063457489014, "logps/chosen": -435.1206359863281, "logps/ref_chosen": -324.1588134765625, "logps/ref_rejected": -277.80218505859375, "logps/rejected": -439.09625244140625, "loss": 4.8804, "margin_dpo/margin_mean": 50.332218170166016, "margin_dpo/margin_std": 108.61395263671875, "step": 200 }, { "epoch": 0.418848167539267, "eval_KL/chosen_KL_mean": -123.2765121459961, "eval_KL/mean": -151.9739227294922, "eval_KL/rejected_KL_mean": -180.67132568359375, "eval_KL/std": 89.34925842285156, "eval_fcm_dpo/beta": 0.0054236906580626965, "eval_logits/chosen": -0.9016394019126892, "eval_logits/rejected": -0.8836301565170288, "eval_logps/chosen": -411.103271484375, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -447.601318359375, "eval_loss": 0.5898867249488831, "eval_margin_dpo/margin_mean": 57.394798278808594, "eval_margin_dpo/margin_std": 97.59921264648438, "eval_runtime": 78.4876, "eval_samples_per_second": 25.482, "eval_steps_per_second": 3.185, "step": 200 }, { "KL/chosen_KL_mean": -121.671142578125, "KL/mean": -151.29833984375, "KL/rejected_KL_mean": -180.92556762695312, "KL/std": 91.87706756591797, "epoch": 0.42094240837696334, "fcm_dpo/beta": 0.005429561249911785, "fcm_dpo/delta": -0.0011716950684785843, "fcm_dpo/margin": 59.25444030761719, "fcm_dpo/q_t": 0.4263926148414612, "grad_norm": 70.4216537475586, "learning_rate": 3.604695382782159e-07, "logits/chosen": -0.889416515827179, "logits/rejected": -0.8737778663635254, "logps/chosen": -393.1668395996094, "logps/ref_chosen": -271.49566650390625, "logps/ref_rejected": -245.71414184570312, "logps/rejected": -426.63970947265625, "loss": 4.6884, "margin_dpo/margin_mean": 59.25444030761719, "margin_dpo/margin_std": 104.75772857666016, "step": 201 }, { "KL/chosen_KL_mean": -129.14830017089844, "KL/mean": -156.79586791992188, "KL/rejected_KL_mean": -184.44342041015625, "KL/std": 89.80362701416016, "epoch": 0.42303664921465967, "fcm_dpo/beta": 0.005406467244029045, "fcm_dpo/delta": -0.0008845115080475807, "fcm_dpo/margin": 55.29514694213867, "fcm_dpo/q_t": 0.43192172050476074, "grad_norm": 83.05462646484375, "learning_rate": 3.588242572718162e-07, "logits/chosen": -0.9037338495254517, "logits/rejected": -0.8901877999305725, "logps/chosen": -401.2462158203125, "logps/ref_chosen": -272.0979309082031, "logps/ref_rejected": -235.94805908203125, "logps/rejected": -420.3915100097656, "loss": 4.7591, "margin_dpo/margin_mean": 55.29515075683594, "margin_dpo/margin_std": 104.44528198242188, "step": 202 }, { "KL/chosen_KL_mean": -139.50340270996094, "KL/mean": -160.76329040527344, "KL/rejected_KL_mean": -182.0231475830078, "KL/std": 89.24750518798828, "epoch": 0.42513089005235605, "fcm_dpo/beta": 0.005487063899636269, "fcm_dpo/delta": 0.012710193172097206, "fcm_dpo/margin": 42.51974105834961, "fcm_dpo/q_t": 0.4455467462539673, "grad_norm": 63.943870544433594, "learning_rate": 3.571731403507635e-07, "logits/chosen": -0.8693050146102905, "logits/rejected": -0.8741556406021118, "logps/chosen": -419.7255554199219, "logps/ref_chosen": -280.2221374511719, "logps/ref_rejected": -251.79798889160156, "logps/rejected": -433.8211669921875, "loss": 4.931, "margin_dpo/margin_mean": 42.51974105834961, "margin_dpo/margin_std": 92.08596801757812, "step": 203 }, { "KL/chosen_KL_mean": -137.26512145996094, "KL/mean": -167.29208374023438, "KL/rejected_KL_mean": -197.31906127929688, "KL/std": 89.45562744140625, "epoch": 0.4272251308900524, "fcm_dpo/beta": 0.005518666002899408, "fcm_dpo/delta": 0.016294872388243675, "fcm_dpo/margin": 60.05392074584961, "fcm_dpo/q_t": 0.4236374497413635, "grad_norm": 85.55860900878906, "learning_rate": 3.5551627605944746e-07, "logits/chosen": -0.929052472114563, "logits/rejected": -0.9007848501205444, "logps/chosen": -456.0611572265625, "logps/ref_chosen": -318.7960510253906, "logps/ref_rejected": -269.69921875, "logps/rejected": -467.0182800292969, "loss": 4.6312, "margin_dpo/margin_mean": 60.05392074584961, "margin_dpo/margin_std": 100.22209167480469, "step": 204 }, { "KL/chosen_KL_mean": -137.09326171875, "KL/mean": -172.9523468017578, "KL/rejected_KL_mean": -208.8114776611328, "KL/std": 96.3641357421875, "epoch": 0.4293193717277487, "fcm_dpo/beta": 0.005441513378173113, "fcm_dpo/delta": -0.044503822922706604, "fcm_dpo/margin": 71.71823120117188, "fcm_dpo/q_t": 0.4126250743865967, "grad_norm": 90.81995391845703, "learning_rate": 3.5385375325047163e-07, "logits/chosen": -0.8676726818084717, "logits/rejected": -0.8353573679924011, "logps/chosen": -420.8553466796875, "logps/ref_chosen": -283.7620544433594, "logps/ref_rejected": -297.69439697265625, "logps/rejected": -506.50592041015625, "loss": 4.4849, "margin_dpo/margin_mean": 71.71823120117188, "margin_dpo/margin_std": 108.88336181640625, "step": 205 }, { "KL/chosen_KL_mean": -141.72567749023438, "KL/mean": -169.52108764648438, "KL/rejected_KL_mean": -197.31649780273438, "KL/std": 93.6350326538086, "epoch": 0.431413612565445, "fcm_dpo/beta": 0.005305927712470293, "fcm_dpo/delta": -0.018429771065711975, "fcm_dpo/margin": 55.590843200683594, "fcm_dpo/q_t": 0.43249496817588806, "grad_norm": 58.64113998413086, "learning_rate": 3.5218566107988867e-07, "logits/chosen": -0.8933624029159546, "logits/rejected": -0.913768470287323, "logps/chosen": -435.3895263671875, "logps/ref_chosen": -293.66387939453125, "logps/ref_rejected": -291.3056640625, "logps/rejected": -488.6221923828125, "loss": 4.7898, "margin_dpo/margin_mean": 55.59084701538086, "margin_dpo/margin_std": 106.94378662109375, "step": 206 }, { "KL/chosen_KL_mean": -130.13616943359375, "KL/mean": -158.40179443359375, "KL/rejected_KL_mean": -186.66741943359375, "KL/std": 104.5025634765625, "epoch": 0.43350785340314135, "fcm_dpo/beta": 0.005238719284534454, "fcm_dpo/delta": -0.0088468287140131, "fcm_dpo/margin": 56.53125, "fcm_dpo/q_t": 0.43153464794158936, "grad_norm": 73.89046478271484, "learning_rate": 3.505120890024195e-07, "logits/chosen": -0.8374500870704651, "logits/rejected": -0.8462930917739868, "logps/chosen": -400.6712646484375, "logps/ref_chosen": -270.5350646972656, "logps/ref_rejected": -278.7747497558594, "logps/rejected": -465.44219970703125, "loss": 4.8468, "margin_dpo/margin_mean": 56.53125, "margin_dpo/margin_std": 121.36986541748047, "step": 207 }, { "KL/chosen_KL_mean": -134.00111389160156, "KL/mean": -169.24815368652344, "KL/rejected_KL_mean": -204.4951934814453, "KL/std": 102.28126525878906, "epoch": 0.4356020942408377, "fcm_dpo/beta": 0.005222794134169817, "fcm_dpo/delta": -0.006807409692555666, "fcm_dpo/margin": 70.49407196044922, "fcm_dpo/q_t": 0.417470782995224, "grad_norm": 77.3393325805664, "learning_rate": 3.4883312676665534e-07, "logits/chosen": -0.9094988703727722, "logits/rejected": -0.8614720106124878, "logps/chosen": -413.5838928222656, "logps/ref_chosen": -279.582763671875, "logps/ref_rejected": -290.041015625, "logps/rejected": -494.53619384765625, "loss": 4.5102, "margin_dpo/margin_mean": 70.49407196044922, "margin_dpo/margin_std": 107.34097290039062, "step": 208 }, { "KL/chosen_KL_mean": -138.67898559570312, "KL/mean": -159.94009399414062, "KL/rejected_KL_mean": -181.2012176513672, "KL/std": 95.46649169921875, "epoch": 0.437696335078534, "fcm_dpo/beta": 0.005218043457716703, "fcm_dpo/delta": 0.007631334010511637, "fcm_dpo/margin": 42.522220611572266, "fcm_dpo/q_t": 0.44813573360443115, "grad_norm": 67.46821594238281, "learning_rate": 3.4714886441024573e-07, "logits/chosen": -0.8228191137313843, "logits/rejected": -0.8171289563179016, "logps/chosen": -457.5514831542969, "logps/ref_chosen": -318.8725280761719, "logps/ref_rejected": -270.64324951171875, "logps/rejected": -451.8444519042969, "loss": 5.0119, "margin_dpo/margin_mean": 42.522220611572266, "margin_dpo/margin_std": 104.65240478515625, "step": 209 }, { "KL/chosen_KL_mean": -121.98692321777344, "KL/mean": -152.7623291015625, "KL/rejected_KL_mean": -183.5377655029297, "KL/std": 90.5138168334961, "epoch": 0.4397905759162304, "fcm_dpo/beta": 0.005136261694133282, "fcm_dpo/delta": -0.03048218972980976, "fcm_dpo/margin": 61.550846099853516, "fcm_dpo/q_t": 0.42682385444641113, "grad_norm": 72.6939697265625, "learning_rate": 3.454593922550693e-07, "logits/chosen": -0.842387318611145, "logits/rejected": -0.832798182964325, "logps/chosen": -405.12725830078125, "logps/ref_chosen": -283.14031982421875, "logps/ref_rejected": -287.2986755371094, "logps/rejected": -470.8364562988281, "loss": 4.6659, "margin_dpo/margin_mean": 61.55085372924805, "margin_dpo/margin_std": 103.3311767578125, "step": 210 }, { "KL/chosen_KL_mean": -105.573974609375, "KL/mean": -145.5406494140625, "KL/rejected_KL_mean": -185.50733947753906, "KL/std": 91.00731658935547, "epoch": 0.4418848167539267, "fcm_dpo/beta": 0.004942171275615692, "fcm_dpo/delta": -0.0607127919793129, "fcm_dpo/margin": 79.93335723876953, "fcm_dpo/q_t": 0.4069315791130066, "grad_norm": 55.6853141784668, "learning_rate": 3.4376480090239047e-07, "logits/chosen": -0.8768879771232605, "logits/rejected": -0.8537665605545044, "logps/chosen": -381.9968566894531, "logps/ref_chosen": -276.4228515625, "logps/ref_rejected": -252.40603637695312, "logps/rejected": -437.913330078125, "loss": 4.3436, "margin_dpo/margin_mean": 79.93335723876953, "margin_dpo/margin_std": 91.45414733886719, "step": 211 }, { "KL/chosen_KL_mean": -121.54786682128906, "KL/mean": -145.3662109375, "KL/rejected_KL_mean": -169.1845703125, "KL/std": 86.90359497070312, "epoch": 0.44397905759162304, "fcm_dpo/beta": 0.004877158906310797, "fcm_dpo/delta": 0.010021396912634373, "fcm_dpo/margin": 47.63668441772461, "fcm_dpo/q_t": 0.4448275566101074, "grad_norm": 75.4878158569336, "learning_rate": 3.4206518122800055e-07, "logits/chosen": -0.8545178174972534, "logits/rejected": -0.8506518602371216, "logps/chosen": -393.25341796875, "logps/ref_chosen": -271.7055358886719, "logps/ref_rejected": -241.18511962890625, "logps/rejected": -410.36968994140625, "loss": 4.9094, "margin_dpo/margin_mean": 47.636680603027344, "margin_dpo/margin_std": 95.85389709472656, "step": 212 }, { "KL/chosen_KL_mean": -122.48336029052734, "KL/mean": -149.56399536132812, "KL/rejected_KL_mean": -176.64462280273438, "KL/std": 102.08607482910156, "epoch": 0.44607329842931936, "fcm_dpo/beta": 0.004948125220835209, "fcm_dpo/delta": 0.013094035908579826, "fcm_dpo/margin": 54.16126251220703, "fcm_dpo/q_t": 0.43963873386383057, "grad_norm": 72.34474182128906, "learning_rate": 3.403606243773448e-07, "logits/chosen": -0.8604366779327393, "logits/rejected": -0.8748980760574341, "logps/chosen": -424.781005859375, "logps/ref_chosen": -302.2976379394531, "logps/ref_rejected": -303.6202087402344, "logps/rejected": -480.2647705078125, "loss": 4.8561, "margin_dpo/margin_mean": 54.16126251220703, "margin_dpo/margin_std": 113.01664733886719, "step": 213 }, { "KL/chosen_KL_mean": -131.97894287109375, "KL/mean": -166.79592895507812, "KL/rejected_KL_mean": -201.61289978027344, "KL/std": 91.53814697265625, "epoch": 0.4481675392670157, "fcm_dpo/beta": 0.004879150539636612, "fcm_dpo/delta": -0.0352763831615448, "fcm_dpo/margin": 69.63395690917969, "fcm_dpo/q_t": 0.42116767168045044, "grad_norm": 63.54450988769531, "learning_rate": 3.3865122176063385e-07, "logits/chosen": -0.8635294437408447, "logits/rejected": -0.8634909391403198, "logps/chosen": -404.111572265625, "logps/ref_chosen": -272.13262939453125, "logps/ref_rejected": -294.82354736328125, "logps/rejected": -496.4364013671875, "loss": 4.5288, "margin_dpo/margin_mean": 69.63395690917969, "margin_dpo/margin_std": 93.9524154663086, "step": 214 }, { "KL/chosen_KL_mean": -138.74960327148438, "KL/mean": -161.1497039794922, "KL/rejected_KL_mean": -183.54978942871094, "KL/std": 103.77374267578125, "epoch": 0.450261780104712, "fcm_dpo/beta": 0.004789126571267843, "fcm_dpo/delta": -0.021435966715216637, "fcm_dpo/margin": 44.8001823425293, "fcm_dpo/q_t": 0.45110857486724854, "grad_norm": 62.86751174926758, "learning_rate": 3.3693706504794243e-07, "logits/chosen": -0.9035426378250122, "logits/rejected": -0.8817762136459351, "logps/chosen": -430.1279296875, "logps/ref_chosen": -291.3782958984375, "logps/ref_rejected": -261.05792236328125, "logps/rejected": -444.60772705078125, "loss": 5.0051, "margin_dpo/margin_mean": 44.80018997192383, "margin_dpo/margin_std": 105.21973419189453, "step": 215 }, { "KL/chosen_KL_mean": -127.76866149902344, "KL/mean": -159.33363342285156, "KL/rejected_KL_mean": -190.89862060546875, "KL/std": 96.72543334960938, "epoch": 0.4523560209424084, "fcm_dpo/beta": 0.004820346832275391, "fcm_dpo/delta": 0.024416334927082062, "fcm_dpo/margin": 63.12995529174805, "fcm_dpo/q_t": 0.4305679202079773, "grad_norm": 56.78171920776367, "learning_rate": 3.3521824616429284e-07, "logits/chosen": -0.9260469675064087, "logits/rejected": -0.9183626174926758, "logps/chosen": -466.27410888671875, "logps/ref_chosen": -338.50543212890625, "logps/ref_rejected": -305.76104736328125, "logps/rejected": -496.6596374511719, "loss": 4.7264, "margin_dpo/margin_mean": 63.12995147705078, "margin_dpo/margin_std": 113.52439880371094, "step": 216 }, { "KL/chosen_KL_mean": -135.17062377929688, "KL/mean": -173.59078979492188, "KL/rejected_KL_mean": -212.01097106933594, "KL/std": 99.70831298828125, "epoch": 0.4544502617801047, "fcm_dpo/beta": 0.004769713152199984, "fcm_dpo/delta": -0.017181845381855965, "fcm_dpo/margin": 76.84034729003906, "fcm_dpo/q_t": 0.4159371256828308, "grad_norm": 62.81724548339844, "learning_rate": 3.334948572847253e-07, "logits/chosen": -0.836664617061615, "logits/rejected": -0.7955836653709412, "logps/chosen": -428.720458984375, "logps/ref_chosen": -293.5498046875, "logps/ref_rejected": -256.7830810546875, "logps/rejected": -468.7940673828125, "loss": 4.5008, "margin_dpo/margin_mean": 76.84033966064453, "margin_dpo/margin_std": 112.65450286865234, "step": 217 }, { "KL/chosen_KL_mean": -133.9468231201172, "KL/mean": -168.5343780517578, "KL/rejected_KL_mean": -203.12193298339844, "KL/std": 99.32330322265625, "epoch": 0.45654450261780105, "fcm_dpo/beta": 0.0047028930857777596, "fcm_dpo/delta": -0.024641798809170723, "fcm_dpo/margin": 69.17512512207031, "fcm_dpo/q_t": 0.4250248372554779, "grad_norm": 64.38352966308594, "learning_rate": 3.317669908293554e-07, "logits/chosen": -0.862998902797699, "logits/rejected": -0.8793630003929138, "logps/chosen": -454.5261535644531, "logps/ref_chosen": -320.579345703125, "logps/ref_rejected": -294.0381164550781, "logps/rejected": -497.1600646972656, "loss": 4.615, "margin_dpo/margin_mean": 69.17512512207031, "margin_dpo/margin_std": 110.75273895263672, "step": 218 }, { "KL/chosen_KL_mean": -132.96096801757812, "KL/mean": -169.79696655273438, "KL/rejected_KL_mean": -206.63299560546875, "KL/std": 95.11730194091797, "epoch": 0.4586387434554974, "fcm_dpo/beta": 0.00460631912574172, "fcm_dpo/delta": -0.01590793766081333, "fcm_dpo/margin": 73.6720199584961, "fcm_dpo/q_t": 0.4217400550842285, "grad_norm": 67.673828125, "learning_rate": 3.300347394584172e-07, "logits/chosen": -0.839383065700531, "logits/rejected": -0.8644520044326782, "logps/chosen": -401.3796081542969, "logps/ref_chosen": -268.4186096191406, "logps/ref_rejected": -265.7808837890625, "logps/rejected": -472.413818359375, "loss": 4.5894, "margin_dpo/margin_mean": 73.6720199584961, "margin_dpo/margin_std": 113.50521850585938, "step": 219 }, { "KL/chosen_KL_mean": -137.07662963867188, "KL/mean": -173.6854705810547, "KL/rejected_KL_mean": -210.2943115234375, "KL/std": 93.0467300415039, "epoch": 0.4607329842931937, "fcm_dpo/beta": 0.004609918221831322, "fcm_dpo/delta": 0.0026855701580643654, "fcm_dpo/margin": 73.21768188476562, "fcm_dpo/q_t": 0.42106613516807556, "grad_norm": 70.86668395996094, "learning_rate": 3.2829819606729477e-07, "logits/chosen": -0.9124230742454529, "logits/rejected": -0.8802636861801147, "logps/chosen": -449.9631042480469, "logps/ref_chosen": -312.8864440917969, "logps/ref_rejected": -259.5191955566406, "logps/rejected": -469.8135070800781, "loss": 4.5431, "margin_dpo/margin_mean": 73.21768188476562, "margin_dpo/margin_std": 104.53365325927734, "step": 220 }, { "KL/chosen_KL_mean": -149.1865692138672, "KL/mean": -178.74742126464844, "KL/rejected_KL_mean": -208.3082733154297, "KL/std": 99.30208587646484, "epoch": 0.46282722513089003, "fcm_dpo/beta": 0.004629157017916441, "fcm_dpo/delta": -0.01543284673243761, "fcm_dpo/margin": 59.121707916259766, "fcm_dpo/q_t": 0.43726086616516113, "grad_norm": 62.723228454589844, "learning_rate": 3.265574537815398e-07, "logits/chosen": -0.8228099942207336, "logits/rejected": -0.8340501189231873, "logps/chosen": -449.5124206542969, "logps/ref_chosen": -300.32586669921875, "logps/ref_rejected": -286.312255859375, "logps/rejected": -494.62054443359375, "loss": 4.8241, "margin_dpo/margin_mean": 59.12171173095703, "margin_dpo/margin_std": 115.42534637451172, "step": 221 }, { "KL/chosen_KL_mean": -143.39520263671875, "KL/mean": -173.56765747070312, "KL/rejected_KL_mean": -203.74014282226562, "KL/std": 89.37479400634766, "epoch": 0.4649214659685864, "fcm_dpo/beta": 0.004566497169435024, "fcm_dpo/delta": -0.0049129268154501915, "fcm_dpo/margin": 60.34489822387695, "fcm_dpo/q_t": 0.4353586733341217, "grad_norm": 55.062538146972656, "learning_rate": 3.248126059518784e-07, "logits/chosen": -0.9121435880661011, "logits/rejected": -0.8791247606277466, "logps/chosen": -440.506591796875, "logps/ref_chosen": -297.1113586425781, "logps/ref_rejected": -235.53146362304688, "logps/rejected": -439.2715759277344, "loss": 4.7509, "margin_dpo/margin_mean": 60.34489822387695, "margin_dpo/margin_std": 104.17533874511719, "step": 222 }, { "KL/chosen_KL_mean": -149.42726135253906, "KL/mean": -182.06930541992188, "KL/rejected_KL_mean": -214.7113494873047, "KL/std": 97.39741516113281, "epoch": 0.46701570680628274, "fcm_dpo/beta": 0.004558564629405737, "fcm_dpo/delta": 0.011139356531202793, "fcm_dpo/margin": 65.28407287597656, "fcm_dpo/q_t": 0.4302670955657959, "grad_norm": 64.61984252929688, "learning_rate": 3.230637461492043e-07, "logits/chosen": -0.8495522737503052, "logits/rejected": -0.8148607611656189, "logps/chosen": -435.8423767089844, "logps/ref_chosen": -286.41510009765625, "logps/ref_rejected": -241.1181640625, "logps/rejected": -455.82952880859375, "loss": 4.7017, "margin_dpo/margin_mean": 65.28407287597656, "margin_dpo/margin_std": 111.41921997070312, "step": 223 }, { "KL/chosen_KL_mean": -136.95799255371094, "KL/mean": -172.8470916748047, "KL/rejected_KL_mean": -208.7362060546875, "KL/std": 96.30474090576172, "epoch": 0.46910994764397906, "fcm_dpo/beta": 0.004641786217689514, "fcm_dpo/delta": 0.006216204725205898, "fcm_dpo/margin": 71.77820587158203, "fcm_dpo/q_t": 0.4229627847671509, "grad_norm": 65.5974349975586, "learning_rate": 3.213109681595612e-07, "logits/chosen": -0.8177117705345154, "logits/rejected": -0.8344778418540955, "logps/chosen": -386.4503173828125, "logps/ref_chosen": -249.49234008789062, "logps/ref_rejected": -233.10752868652344, "logps/rejected": -441.84375, "loss": 4.5747, "margin_dpo/margin_mean": 71.77820587158203, "margin_dpo/margin_std": 106.76852416992188, "step": 224 }, { "KL/chosen_KL_mean": -156.29876708984375, "KL/mean": -189.03250122070312, "KL/rejected_KL_mean": -221.76626586914062, "KL/std": 103.037841796875, "epoch": 0.4712041884816754, "fcm_dpo/beta": 0.004633888602256775, "fcm_dpo/delta": 0.014746258035302162, "fcm_dpo/margin": 65.46749877929688, "fcm_dpo/q_t": 0.4310811161994934, "grad_norm": 77.01375579833984, "learning_rate": 3.1955436597911315e-07, "logits/chosen": -0.8554367423057556, "logits/rejected": -0.8290306925773621, "logps/chosen": -468.1571350097656, "logps/ref_chosen": -311.8583679199219, "logps/ref_rejected": -336.8523864746094, "logps/rejected": -558.61865234375, "loss": 4.7305, "margin_dpo/margin_mean": 65.46749877929688, "margin_dpo/margin_std": 119.62830352783203, "step": 225 }, { "KL/chosen_KL_mean": -141.25518798828125, "KL/mean": -169.96444702148438, "KL/rejected_KL_mean": -198.6737060546875, "KL/std": 97.53821563720703, "epoch": 0.4732984293193717, "fcm_dpo/beta": 0.0046873511746525764, "fcm_dpo/delta": 0.0078051392920315266, "fcm_dpo/margin": 57.41852569580078, "fcm_dpo/q_t": 0.43802326917648315, "grad_norm": 74.30442810058594, "learning_rate": 3.1779403380910425e-07, "logits/chosen": -0.876971423625946, "logits/rejected": -0.864268958568573, "logps/chosen": -393.4564514160156, "logps/ref_chosen": -252.20123291015625, "logps/ref_rejected": -254.41162109375, "logps/rejected": -453.0853576660156, "loss": 4.8147, "margin_dpo/margin_mean": 57.418521881103516, "margin_dpo/margin_std": 112.63554382324219, "step": 226 }, { "KL/chosen_KL_mean": -135.2330780029297, "KL/mean": -172.74185180664062, "KL/rejected_KL_mean": -210.25062561035156, "KL/std": 94.74110412597656, "epoch": 0.47539267015706804, "fcm_dpo/beta": 0.004633747041225433, "fcm_dpo/delta": -0.023657258599996567, "fcm_dpo/margin": 75.01754760742188, "fcm_dpo/q_t": 0.4188823401927948, "grad_norm": 61.293704986572266, "learning_rate": 3.160300660508064e-07, "logits/chosen": -0.8379367589950562, "logits/rejected": -0.8345897197723389, "logps/chosen": -420.4925231933594, "logps/ref_chosen": -285.25946044921875, "logps/ref_rejected": -261.3220520019531, "logps/rejected": -471.57269287109375, "loss": 4.559, "margin_dpo/margin_mean": 75.01754760742188, "margin_dpo/margin_std": 114.54286193847656, "step": 227 }, { "KL/chosen_KL_mean": -120.16371154785156, "KL/mean": -156.32989501953125, "KL/rejected_KL_mean": -192.4960479736328, "KL/std": 90.50350189208984, "epoch": 0.4774869109947644, "fcm_dpo/beta": 0.004585505463182926, "fcm_dpo/delta": -0.025612030178308487, "fcm_dpo/margin": 72.33232879638672, "fcm_dpo/q_t": 0.4227723479270935, "grad_norm": 55.890357971191406, "learning_rate": 3.1426255730045695e-07, "logits/chosen": -0.8586771488189697, "logits/rejected": -0.8261110782623291, "logps/chosen": -433.9825439453125, "logps/ref_chosen": -313.81878662109375, "logps/ref_rejected": -258.07061767578125, "logps/rejected": -450.56671142578125, "loss": 4.5548, "margin_dpo/margin_mean": 72.33232879638672, "margin_dpo/margin_std": 101.38350677490234, "step": 228 }, { "KL/chosen_KL_mean": -135.5593719482422, "KL/mean": -177.83876037597656, "KL/rejected_KL_mean": -220.11810302734375, "KL/std": 98.42047119140625, "epoch": 0.47958115183246075, "fcm_dpo/beta": 0.004426400177180767, "fcm_dpo/delta": -0.029844271019101143, "fcm_dpo/margin": 84.5587158203125, "fcm_dpo/q_t": 0.41256657242774963, "grad_norm": 52.477874755859375, "learning_rate": 3.1249160234418644e-07, "logits/chosen": -0.8296700716018677, "logits/rejected": -0.8337705135345459, "logps/chosen": -427.5301513671875, "logps/ref_chosen": -291.9707946777344, "logps/ref_rejected": -263.42059326171875, "logps/rejected": -483.5386657714844, "loss": 4.4323, "margin_dpo/margin_mean": 84.55870819091797, "margin_dpo/margin_std": 109.96800994873047, "step": 229 }, { "KL/chosen_KL_mean": -121.74411010742188, "KL/mean": -160.2207489013672, "KL/rejected_KL_mean": -198.69741821289062, "KL/std": 105.12950897216797, "epoch": 0.4816753926701571, "fcm_dpo/beta": 0.004394059535115957, "fcm_dpo/delta": 0.0015674427850171924, "fcm_dpo/margin": 76.95330047607422, "fcm_dpo/q_t": 0.4227007031440735, "grad_norm": 66.97175598144531, "learning_rate": 3.1071729615293424e-07, "logits/chosen": -0.8868385553359985, "logits/rejected": -0.8867266178131104, "logps/chosen": -355.00421142578125, "logps/ref_chosen": -233.2601318359375, "logps/ref_rejected": -238.922119140625, "logps/rejected": -437.6195373535156, "loss": 4.5802, "margin_dpo/margin_mean": 76.95330047607422, "margin_dpo/margin_std": 119.8840560913086, "step": 230 }, { "KL/chosen_KL_mean": -133.48675537109375, "KL/mean": -164.26443481445312, "KL/rejected_KL_mean": -195.0421142578125, "KL/std": 91.5148696899414, "epoch": 0.4837696335078534, "fcm_dpo/beta": 0.00437714671716094, "fcm_dpo/delta": -0.008870087563991547, "fcm_dpo/margin": 61.55538558959961, "fcm_dpo/q_t": 0.4362969696521759, "grad_norm": 66.45602416992188, "learning_rate": 3.0893973387735683e-07, "logits/chosen": -0.8645358681678772, "logits/rejected": -0.8530266880989075, "logps/chosen": -455.64190673828125, "logps/ref_chosen": -322.1551818847656, "logps/ref_rejected": -280.97613525390625, "logps/rejected": -476.01824951171875, "loss": 4.7541, "margin_dpo/margin_mean": 61.55538558959961, "margin_dpo/margin_std": 102.5092544555664, "step": 231 }, { "KL/chosen_KL_mean": -132.1063995361328, "KL/mean": -162.6656951904297, "KL/rejected_KL_mean": -193.22494506835938, "KL/std": 99.24211120605469, "epoch": 0.48586387434554973, "fcm_dpo/beta": 0.004379937890917063, "fcm_dpo/delta": -0.0022955876775085926, "fcm_dpo/margin": 61.11854934692383, "fcm_dpo/q_t": 0.43785691261291504, "grad_norm": 64.07685852050781, "learning_rate": 3.071590108427243e-07, "logits/chosen": -0.8338838219642639, "logits/rejected": -0.8158622980117798, "logps/chosen": -403.8501892089844, "logps/ref_chosen": -271.7437744140625, "logps/ref_rejected": -249.94981384277344, "logps/rejected": -443.17474365234375, "loss": 4.8322, "margin_dpo/margin_mean": 61.11854934692383, "margin_dpo/margin_std": 115.72997283935547, "step": 232 }, { "KL/chosen_KL_mean": -122.5298080444336, "KL/mean": -161.3760986328125, "KL/rejected_KL_mean": -200.22239685058594, "KL/std": 95.90873718261719, "epoch": 0.48795811518324606, "fcm_dpo/beta": 0.004333424381911755, "fcm_dpo/delta": -0.03165815770626068, "fcm_dpo/margin": 77.6926040649414, "fcm_dpo/q_t": 0.42159727215766907, "grad_norm": 53.214111328125, "learning_rate": 3.05375222543809e-07, "logits/chosen": -0.8735519051551819, "logits/rejected": -0.8635944128036499, "logps/chosen": -407.87213134765625, "logps/ref_chosen": -285.3423156738281, "logps/ref_rejected": -266.34320068359375, "logps/rejected": -466.56561279296875, "loss": 4.5516, "margin_dpo/margin_mean": 77.69261169433594, "margin_dpo/margin_std": 106.49396514892578, "step": 233 }, { "KL/chosen_KL_mean": -126.59469604492188, "KL/mean": -158.7538604736328, "KL/rejected_KL_mean": -190.9130859375, "KL/std": 95.13108825683594, "epoch": 0.4900523560209424, "fcm_dpo/beta": 0.004321113228797913, "fcm_dpo/delta": 0.043769948184490204, "fcm_dpo/margin": 64.3183822631836, "fcm_dpo/q_t": 0.43559810519218445, "grad_norm": 65.22840118408203, "learning_rate": 3.035884646397637e-07, "logits/chosen": -0.8416929244995117, "logits/rejected": -0.829358696937561, "logps/chosen": -421.5003662109375, "logps/ref_chosen": -294.9057312011719, "logps/ref_rejected": -299.37054443359375, "logps/rejected": -490.28363037109375, "loss": 4.7916, "margin_dpo/margin_mean": 64.3183822631836, "margin_dpo/margin_std": 121.3981704711914, "step": 234 }, { "KL/chosen_KL_mean": -123.97207641601562, "KL/mean": -159.31024169921875, "KL/rejected_KL_mean": -194.6483917236328, "KL/std": 103.93309020996094, "epoch": 0.49214659685863876, "fcm_dpo/beta": 0.004475614987313747, "fcm_dpo/delta": 0.024762175977230072, "fcm_dpo/margin": 70.67631530761719, "fcm_dpo/q_t": 0.42616766691207886, "grad_norm": 74.38615417480469, "learning_rate": 3.017988329489923e-07, "logits/chosen": -0.864960789680481, "logits/rejected": -0.8573201894760132, "logps/chosen": -413.46966552734375, "logps/ref_chosen": -289.49755859375, "logps/ref_rejected": -247.55076599121094, "logps/rejected": -442.1991882324219, "loss": 4.6269, "margin_dpo/margin_mean": 70.67632293701172, "margin_dpo/margin_std": 112.42530822753906, "step": 235 }, { "KL/chosen_KL_mean": -116.880126953125, "KL/mean": -151.97642517089844, "KL/rejected_KL_mean": -187.0727081298828, "KL/std": 98.38629913330078, "epoch": 0.4942408376963351, "fcm_dpo/beta": 0.004550088196992874, "fcm_dpo/delta": 0.01439427025616169, "fcm_dpo/margin": 70.19258880615234, "fcm_dpo/q_t": 0.4265892505645752, "grad_norm": 86.40446472167969, "learning_rate": 3.000064234440111e-07, "logits/chosen": -0.8829069137573242, "logits/rejected": -0.8841407895088196, "logps/chosen": -405.7647399902344, "logps/ref_chosen": -288.8846435546875, "logps/ref_rejected": -242.0452880859375, "logps/rejected": -429.11798095703125, "loss": 4.6436, "margin_dpo/margin_mean": 70.19258880615234, "margin_dpo/margin_std": 115.69688415527344, "step": 236 }, { "KL/chosen_KL_mean": -123.7160873413086, "KL/mean": -159.17050170898438, "KL/rejected_KL_mean": -194.62493896484375, "KL/std": 86.97819519042969, "epoch": 0.4963350785340314, "fcm_dpo/beta": 0.0045979805290699005, "fcm_dpo/delta": 0.013288134709000587, "fcm_dpo/margin": 70.9088363647461, "fcm_dpo/q_t": 0.42442870140075684, "grad_norm": 69.7810287475586, "learning_rate": 2.9821133224630223e-07, "logits/chosen": -0.8455889821052551, "logits/rejected": -0.8238015174865723, "logps/chosen": -389.19482421875, "logps/ref_chosen": -265.47869873046875, "logps/ref_rejected": -267.9891357421875, "logps/rejected": -462.61407470703125, "loss": 4.5715, "margin_dpo/margin_mean": 70.90882873535156, "margin_dpo/margin_std": 103.5340576171875, "step": 237 }, { "KL/chosen_KL_mean": -132.91195678710938, "KL/mean": -167.68841552734375, "KL/rejected_KL_mean": -202.46484375, "KL/std": 96.82723236083984, "epoch": 0.49842931937172774, "fcm_dpo/beta": 0.004613780416548252, "fcm_dpo/delta": 0.0007776557467877865, "fcm_dpo/margin": 69.55287170410156, "fcm_dpo/q_t": 0.42712903022766113, "grad_norm": 51.249977111816406, "learning_rate": 2.964136556211588e-07, "logits/chosen": -0.853753924369812, "logits/rejected": -0.8224870562553406, "logps/chosen": -444.91461181640625, "logps/ref_chosen": -312.0026550292969, "logps/ref_rejected": -270.0257263183594, "logps/rejected": -472.4905700683594, "loss": 4.6223, "margin_dpo/margin_mean": 69.5528793334961, "margin_dpo/margin_std": 110.1138687133789, "step": 238 }, { "KL/chosen_KL_mean": -122.88169860839844, "KL/mean": -154.25526428222656, "KL/rejected_KL_mean": -185.62881469726562, "KL/std": 101.83222961425781, "epoch": 0.5005235602094241, "fcm_dpo/beta": 0.004658076446503401, "fcm_dpo/delta": 0.014860610477626324, "fcm_dpo/margin": 62.74710464477539, "fcm_dpo/q_t": 0.4337066411972046, "grad_norm": 62.06940460205078, "learning_rate": 2.946134899725226e-07, "logits/chosen": -0.8369898796081543, "logits/rejected": -0.876994788646698, "logps/chosen": -390.0489501953125, "logps/ref_chosen": -267.167236328125, "logps/ref_rejected": -275.99468994140625, "logps/rejected": -461.6235046386719, "loss": 4.8319, "margin_dpo/margin_mean": 62.74710464477539, "margin_dpo/margin_std": 131.92681884765625, "step": 239 }, { "KL/chosen_KL_mean": -123.39981079101562, "KL/mean": -161.76980590820312, "KL/rejected_KL_mean": -200.13980102539062, "KL/std": 98.65313720703125, "epoch": 0.5026178010471204, "fcm_dpo/beta": 0.004674379248172045, "fcm_dpo/delta": 0.002928507514297962, "fcm_dpo/margin": 76.73999786376953, "fcm_dpo/q_t": 0.41863059997558594, "grad_norm": 49.32747268676758, "learning_rate": 2.9281093183781403e-07, "logits/chosen": -0.9084842801094055, "logits/rejected": -0.9017030596733093, "logps/chosen": -409.3794250488281, "logps/ref_chosen": -285.9796142578125, "logps/ref_rejected": -256.8258056640625, "logps/rejected": -456.9656066894531, "loss": 4.5228, "margin_dpo/margin_mean": 76.73999786376953, "margin_dpo/margin_std": 116.18621826171875, "step": 240 }, { "KL/chosen_KL_mean": -143.24009704589844, "KL/mean": -172.29185485839844, "KL/rejected_KL_mean": -201.3436279296875, "KL/std": 106.31816101074219, "epoch": 0.5047120418848168, "fcm_dpo/beta": 0.004635634366422892, "fcm_dpo/delta": -0.013472635298967361, "fcm_dpo/margin": 58.10354995727539, "fcm_dpo/q_t": 0.43844515085220337, "grad_norm": 55.63737869262695, "learning_rate": 2.910060778827554e-07, "logits/chosen": -0.8202676773071289, "logits/rejected": -0.7962172031402588, "logps/chosen": -404.7569580078125, "logps/ref_chosen": -261.516845703125, "logps/ref_rejected": -250.2250518798828, "logps/rejected": -451.5686950683594, "loss": 4.8474, "margin_dpo/margin_mean": 58.10354995727539, "margin_dpo/margin_std": 116.29940032958984, "step": 241 }, { "KL/chosen_KL_mean": -130.06700134277344, "KL/mean": -162.6671600341797, "KL/rejected_KL_mean": -195.267333984375, "KL/std": 102.92440795898438, "epoch": 0.506806282722513, "fcm_dpo/beta": 0.004585296381264925, "fcm_dpo/delta": 0.0017472277395427227, "fcm_dpo/margin": 65.20032501220703, "fcm_dpo/q_t": 0.43209636211395264, "grad_norm": 80.01802062988281, "learning_rate": 2.891990248961871e-07, "logits/chosen": -0.8808521032333374, "logits/rejected": -0.8558878302574158, "logps/chosen": -400.5809631347656, "logps/ref_chosen": -270.51397705078125, "logps/ref_rejected": -244.8560791015625, "logps/rejected": -440.1234130859375, "loss": 4.6885, "margin_dpo/margin_mean": 65.20032501220703, "margin_dpo/margin_std": 109.88291931152344, "step": 242 }, { "KL/chosen_KL_mean": -125.00543212890625, "KL/mean": -165.06704711914062, "KL/rejected_KL_mean": -205.12864685058594, "KL/std": 104.33854675292969, "epoch": 0.5089005235602094, "fcm_dpo/beta": 0.004604042042046785, "fcm_dpo/delta": -0.03992704302072525, "fcm_dpo/margin": 80.12322998046875, "fcm_dpo/q_t": 0.41883641481399536, "grad_norm": 76.32345581054688, "learning_rate": 2.873898697848762e-07, "logits/chosen": -0.8886011838912964, "logits/rejected": -0.8729423880577087, "logps/chosen": -449.6875, "logps/ref_chosen": -324.68206787109375, "logps/ref_rejected": -307.1111755371094, "logps/rejected": -512.2398071289062, "loss": 4.5412, "margin_dpo/margin_mean": 80.12322998046875, "margin_dpo/margin_std": 127.59658813476562, "step": 243 }, { "KL/chosen_KL_mean": -128.8868408203125, "KL/mean": -168.5599365234375, "KL/rejected_KL_mean": -208.23304748535156, "KL/std": 96.4882583618164, "epoch": 0.5109947643979058, "fcm_dpo/beta": 0.004474720451980829, "fcm_dpo/delta": 0.001254271250218153, "fcm_dpo/margin": 79.34623718261719, "fcm_dpo/q_t": 0.41681310534477234, "grad_norm": 47.35638427734375, "learning_rate": 2.8557870956832133e-07, "logits/chosen": -0.8795413970947266, "logits/rejected": -0.8352590203285217, "logps/chosen": -447.8660583496094, "logps/ref_chosen": -318.979248046875, "logps/ref_rejected": -269.67572021484375, "logps/rejected": -477.90875244140625, "loss": 4.5117, "margin_dpo/margin_mean": 79.34623718261719, "margin_dpo/margin_std": 108.80554962158203, "step": 244 }, { "KL/chosen_KL_mean": -125.91683197021484, "KL/mean": -165.79470825195312, "KL/rejected_KL_mean": -205.67257690429688, "KL/std": 94.03182220458984, "epoch": 0.5130890052356021, "fcm_dpo/beta": 0.004462458658963442, "fcm_dpo/delta": -0.01995830610394478, "fcm_dpo/margin": 79.75574493408203, "fcm_dpo/q_t": 0.4166679382324219, "grad_norm": 62.6284065246582, "learning_rate": 2.837656413735479e-07, "logits/chosen": -0.8734516501426697, "logits/rejected": -0.879128634929657, "logps/chosen": -420.81488037109375, "logps/ref_chosen": -294.8980712890625, "logps/ref_rejected": -239.8111114501953, "logps/rejected": -445.483642578125, "loss": 4.4922, "margin_dpo/margin_mean": 79.7557373046875, "margin_dpo/margin_std": 107.47772979736328, "step": 245 }, { "KL/chosen_KL_mean": -151.3753204345703, "KL/mean": -178.75440979003906, "KL/rejected_KL_mean": -206.1334686279297, "KL/std": 95.8738021850586, "epoch": 0.5151832460732985, "fcm_dpo/beta": 0.004442715086042881, "fcm_dpo/delta": 0.012672241777181625, "fcm_dpo/margin": 54.75814437866211, "fcm_dpo/q_t": 0.4447598457336426, "grad_norm": 64.96891021728516, "learning_rate": 2.8195076242990116e-07, "logits/chosen": -0.8741089701652527, "logits/rejected": -0.8788989186286926, "logps/chosen": -432.06072998046875, "logps/ref_chosen": -280.6854248046875, "logps/ref_rejected": -253.65382385253906, "logps/rejected": -459.78729248046875, "loss": 4.9389, "margin_dpo/margin_mean": 54.75814437866211, "margin_dpo/margin_std": 124.61536407470703, "step": 246 }, { "KL/chosen_KL_mean": -139.81202697753906, "KL/mean": -175.6849365234375, "KL/rejected_KL_mean": -211.55783081054688, "KL/std": 100.8326416015625, "epoch": 0.5172774869109947, "fcm_dpo/beta": 0.0044505903497338295, "fcm_dpo/delta": -0.016747113317251205, "fcm_dpo/margin": 71.74579620361328, "fcm_dpo/q_t": 0.4267142415046692, "grad_norm": 47.79685592651367, "learning_rate": 2.801341700638307e-07, "logits/chosen": -0.8639557361602783, "logits/rejected": -0.861725926399231, "logps/chosen": -420.9211120605469, "logps/ref_chosen": -281.1091003417969, "logps/ref_rejected": -260.3700866699219, "logps/rejected": -471.92791748046875, "loss": 4.6184, "margin_dpo/margin_mean": 71.74579620361328, "margin_dpo/margin_std": 109.62867736816406, "step": 247 }, { "KL/chosen_KL_mean": -145.56988525390625, "KL/mean": -179.25965881347656, "KL/rejected_KL_mean": -212.94943237304688, "KL/std": 101.54124450683594, "epoch": 0.5193717277486911, "fcm_dpo/beta": 0.004346329253166914, "fcm_dpo/delta": -0.013487712480127811, "fcm_dpo/margin": 67.37956237792969, "fcm_dpo/q_t": 0.4315546154975891, "grad_norm": 69.30085754394531, "learning_rate": 2.7831596169367227e-07, "logits/chosen": -0.8273190259933472, "logits/rejected": -0.8335063457489014, "logps/chosen": -415.88824462890625, "logps/ref_chosen": -270.318359375, "logps/ref_rejected": -233.46778869628906, "logps/rejected": -446.417236328125, "loss": 4.7203, "margin_dpo/margin_mean": 67.37956237792969, "margin_dpo/margin_std": 112.68716430664062, "step": 248 }, { "KL/chosen_KL_mean": -163.27923583984375, "KL/mean": -192.41412353515625, "KL/rejected_KL_mean": -221.5490264892578, "KL/std": 105.23775482177734, "epoch": 0.5214659685863874, "fcm_dpo/beta": 0.00435510091483593, "fcm_dpo/delta": -0.0018205934902653098, "fcm_dpo/margin": 58.26978302001953, "fcm_dpo/q_t": 0.4405788779258728, "grad_norm": 58.962860107421875, "learning_rate": 2.7649623482442274e-07, "logits/chosen": -0.859215497970581, "logits/rejected": -0.8319091200828552, "logps/chosen": -439.08807373046875, "logps/ref_chosen": -275.8088684082031, "logps/ref_rejected": -243.45138549804688, "logps/rejected": -465.0003967285156, "loss": 4.9087, "margin_dpo/margin_mean": 58.26978302001953, "margin_dpo/margin_std": 128.27325439453125, "step": 249 }, { "KL/chosen_KL_mean": -163.93161010742188, "KL/mean": -204.99388122558594, "KL/rejected_KL_mean": -246.05612182617188, "KL/std": 120.12943267822266, "epoch": 0.5235602094240838, "fcm_dpo/beta": 0.004266043193638325, "fcm_dpo/delta": -0.04007497429847717, "fcm_dpo/margin": 82.12449645996094, "fcm_dpo/q_t": 0.41968196630477905, "grad_norm": 79.0232925415039, "learning_rate": 2.7467508704251135e-07, "logits/chosen": -0.8501719236373901, "logits/rejected": -0.8420849442481995, "logps/chosen": -456.4261474609375, "logps/ref_chosen": -292.4945373535156, "logps/ref_rejected": -284.2869567871094, "logps/rejected": -530.3430786132812, "loss": 4.6, "margin_dpo/margin_mean": 82.1244888305664, "margin_dpo/margin_std": 134.7095947265625, "step": 250 }, { "KL/chosen_KL_mean": -151.3125, "KL/mean": -189.22445678710938, "KL/rejected_KL_mean": -227.1363983154297, "KL/std": 116.46218872070312, "epoch": 0.5256544502617801, "fcm_dpo/beta": 0.0041464585810899734, "fcm_dpo/delta": -0.04177861660718918, "fcm_dpo/margin": 75.82389831542969, "fcm_dpo/q_t": 0.4313172996044159, "grad_norm": 75.89229583740234, "learning_rate": 2.7285261601056697e-07, "logits/chosen": -0.8566744327545166, "logits/rejected": -0.8396670818328857, "logps/chosen": -433.04913330078125, "logps/ref_chosen": -281.736572265625, "logps/ref_rejected": -255.9419708251953, "logps/rejected": -483.078369140625, "loss": 4.6892, "margin_dpo/margin_mean": 75.82389831542969, "margin_dpo/margin_std": 130.15245056152344, "step": 251 }, { "KL/chosen_KL_mean": -169.72479248046875, "KL/mean": -205.39056396484375, "KL/rejected_KL_mean": -241.0563201904297, "KL/std": 109.83949279785156, "epoch": 0.5277486910994764, "fcm_dpo/beta": 0.004018429201096296, "fcm_dpo/delta": -0.022251684218645096, "fcm_dpo/margin": 71.3315200805664, "fcm_dpo/q_t": 0.4324949383735657, "grad_norm": 50.09172058105469, "learning_rate": 2.7102891946217994e-07, "logits/chosen": -0.9038177132606506, "logits/rejected": -0.8767113089561462, "logps/chosen": -465.6922302246094, "logps/ref_chosen": -295.9674072265625, "logps/ref_rejected": -280.111572265625, "logps/rejected": -521.1679077148438, "loss": 4.7294, "margin_dpo/margin_mean": 71.3315200805664, "margin_dpo/margin_std": 121.27824401855469, "step": 252 }, { "KL/chosen_KL_mean": -177.96763610839844, "KL/mean": -212.9547119140625, "KL/rejected_KL_mean": -247.94180297851562, "KL/std": 117.46932220458984, "epoch": 0.5298429319371728, "fcm_dpo/beta": 0.004015140701085329, "fcm_dpo/delta": -0.0008969169575721025, "fcm_dpo/margin": 69.97415924072266, "fcm_dpo/q_t": 0.43637773394584656, "grad_norm": 63.2220458984375, "learning_rate": 2.692040951966617e-07, "logits/chosen": -0.8821121454238892, "logits/rejected": -0.8729538917541504, "logps/chosen": -455.0398864746094, "logps/ref_chosen": -277.072265625, "logps/ref_rejected": -247.31643676757812, "logps/rejected": -495.25823974609375, "loss": 4.8047, "margin_dpo/margin_mean": 69.97415924072266, "margin_dpo/margin_std": 132.85340881347656, "step": 253 }, { "KL/chosen_KL_mean": -166.27993774414062, "KL/mean": -205.32247924804688, "KL/rejected_KL_mean": -244.36500549316406, "KL/std": 116.28435516357422, "epoch": 0.5319371727748691, "fcm_dpo/beta": 0.003979699220508337, "fcm_dpo/delta": -0.027800586074590683, "fcm_dpo/margin": 78.0850830078125, "fcm_dpo/q_t": 0.4289184510707855, "grad_norm": 44.38166427612305, "learning_rate": 2.6737824107379947e-07, "logits/chosen": -0.8131484389305115, "logits/rejected": -0.7929275035858154, "logps/chosen": -436.227783203125, "logps/ref_chosen": -269.9478454589844, "logps/ref_rejected": -249.45005798339844, "logps/rejected": -493.8150634765625, "loss": 4.6575, "margin_dpo/margin_mean": 78.0850830078125, "margin_dpo/margin_std": 126.3444595336914, "step": 254 }, { "KL/chosen_KL_mean": -165.02828979492188, "KL/mean": -207.98382568359375, "KL/rejected_KL_mean": -250.9393768310547, "KL/std": 116.85551452636719, "epoch": 0.5340314136125655, "fcm_dpo/beta": 0.003907862119376659, "fcm_dpo/delta": -0.0021270355209708214, "fcm_dpo/margin": 85.91107940673828, "fcm_dpo/q_t": 0.42185455560684204, "grad_norm": 69.25841522216797, "learning_rate": 2.655514550086086e-07, "logits/chosen": -0.8334129452705383, "logits/rejected": -0.7977045178413391, "logps/chosen": -471.6835632324219, "logps/ref_chosen": -306.6552734375, "logps/ref_rejected": -254.47528076171875, "logps/rejected": -505.4146728515625, "loss": 4.6419, "margin_dpo/margin_mean": 85.91107940673828, "margin_dpo/margin_std": 146.6105194091797, "step": 255 }, { "KL/chosen_KL_mean": -167.70700073242188, "KL/mean": -206.95205688476562, "KL/rejected_KL_mean": -246.19708251953125, "KL/std": 115.3568344116211, "epoch": 0.5361256544502618, "fcm_dpo/beta": 0.0038304305635392666, "fcm_dpo/delta": -0.04873408377170563, "fcm_dpo/margin": 78.49009704589844, "fcm_dpo/q_t": 0.4281355142593384, "grad_norm": 82.34991455078125, "learning_rate": 2.6372383496608186e-07, "logits/chosen": -0.8674604296684265, "logits/rejected": -0.8532135486602783, "logps/chosen": -491.4251708984375, "logps/ref_chosen": -323.7181701660156, "logps/ref_rejected": -254.1871337890625, "logps/rejected": -500.38421630859375, "loss": 4.7412, "margin_dpo/margin_mean": 78.49009704589844, "margin_dpo/margin_std": 136.25772094726562, "step": 256 }, { "KL/chosen_KL_mean": -172.4775390625, "KL/mean": -212.53933715820312, "KL/rejected_KL_mean": -252.6011199951172, "KL/std": 113.29838562011719, "epoch": 0.5382198952879581, "fcm_dpo/beta": 0.003783945692703128, "fcm_dpo/delta": 0.01767011545598507, "fcm_dpo/margin": 80.12359619140625, "fcm_dpo/q_t": 0.4296230971813202, "grad_norm": 52.96334457397461, "learning_rate": 2.618954789559356e-07, "logits/chosen": -0.8517543077468872, "logits/rejected": -0.8445267081260681, "logps/chosen": -439.6896057128906, "logps/ref_chosen": -267.21209716796875, "logps/ref_rejected": -249.12579345703125, "logps/rejected": -501.7268981933594, "loss": 4.6621, "margin_dpo/margin_mean": 80.12359619140625, "margin_dpo/margin_std": 130.36715698242188, "step": 257 }, { "KL/chosen_KL_mean": -185.05252075195312, "KL/mean": -227.99496459960938, "KL/rejected_KL_mean": -270.9373779296875, "KL/std": 111.19657135009766, "epoch": 0.5403141361256545, "fcm_dpo/beta": 0.00376452156342566, "fcm_dpo/delta": -0.018048834055662155, "fcm_dpo/margin": 85.88488006591797, "fcm_dpo/q_t": 0.42411839962005615, "grad_norm": 59.09746551513672, "learning_rate": 2.600664850273538e-07, "logits/chosen": -0.8708582520484924, "logits/rejected": -0.8416208624839783, "logps/chosen": -462.7352600097656, "logps/ref_chosen": -277.6827392578125, "logps/ref_rejected": -250.73385620117188, "logps/rejected": -521.6712036132812, "loss": 4.5813, "margin_dpo/margin_mean": 85.88487243652344, "margin_dpo/margin_std": 122.19969177246094, "step": 258 }, { "KL/chosen_KL_mean": -170.76246643066406, "KL/mean": -212.5333709716797, "KL/rejected_KL_mean": -254.30429077148438, "KL/std": 120.29609680175781, "epoch": 0.5424083769633508, "fcm_dpo/beta": 0.003755849553272128, "fcm_dpo/delta": -0.010320892557501793, "fcm_dpo/margin": 83.54182434082031, "fcm_dpo/q_t": 0.4279811978340149, "grad_norm": 49.8717041015625, "learning_rate": 2.582369512637302e-07, "logits/chosen": -0.8988098502159119, "logits/rejected": -0.8933365941047668, "logps/chosen": -465.3724365234375, "logps/ref_chosen": -294.6099853515625, "logps/ref_rejected": -272.2725830078125, "logps/rejected": -526.576904296875, "loss": 4.6393, "margin_dpo/margin_mean": 83.54182434082031, "margin_dpo/margin_std": 132.25244140625, "step": 259 }, { "KL/chosen_KL_mean": -185.67103576660156, "KL/mean": -202.73085021972656, "KL/rejected_KL_mean": -219.79066467285156, "KL/std": 103.56588745117188, "epoch": 0.5445026178010471, "fcm_dpo/beta": 0.0037252292968332767, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.1196174621582, "fcm_dpo/q_t": 0.4704400599002838, "grad_norm": 56.41184616088867, "learning_rate": 2.5640697577740815e-07, "logits/chosen": -0.8784099221229553, "logits/rejected": -0.8720245361328125, "logps/chosen": -476.5281677246094, "logps/ref_chosen": -290.85711669921875, "logps/ref_rejected": -277.5970153808594, "logps/rejected": -497.3876953125, "loss": 5.2616, "margin_dpo/margin_mean": 34.1196174621582, "margin_dpo/margin_std": 120.5584945678711, "step": 260 }, { "KL/chosen_KL_mean": -196.43760681152344, "KL/mean": -230.16085815429688, "KL/rejected_KL_mean": -263.88409423828125, "KL/std": 111.6231918334961, "epoch": 0.5465968586387434, "fcm_dpo/beta": 0.003716195933520794, "fcm_dpo/delta": -0.004061264451593161, "fcm_dpo/margin": 67.44648742675781, "fcm_dpo/q_t": 0.44201305508613586, "grad_norm": 55.86997985839844, "learning_rate": 2.5457665670441937e-07, "logits/chosen": -0.7669353485107422, "logits/rejected": -0.7763051986694336, "logps/chosen": -447.56982421875, "logps/ref_chosen": -251.13223266601562, "logps/ref_rejected": -244.76016235351562, "logps/rejected": -508.64422607421875, "loss": 4.8721, "margin_dpo/margin_mean": 67.44648742675781, "margin_dpo/margin_std": 137.79544067382812, "step": 261 }, { "KL/chosen_KL_mean": -178.6384735107422, "KL/mean": -224.44729614257812, "KL/rejected_KL_mean": -270.2561340332031, "KL/std": 111.68685150146484, "epoch": 0.5486910994764398, "fcm_dpo/beta": 0.0037381243892014027, "fcm_dpo/delta": -0.0032047489657998085, "fcm_dpo/margin": 91.61766052246094, "fcm_dpo/q_t": 0.419685423374176, "grad_norm": 63.28798294067383, "learning_rate": 2.527460921992209e-07, "logits/chosen": -0.814434289932251, "logits/rejected": -0.8024383187294006, "logps/chosen": -478.36029052734375, "logps/ref_chosen": -299.7217712402344, "logps/ref_rejected": -277.0969543457031, "logps/rejected": -547.3530883789062, "loss": 4.5031, "margin_dpo/margin_mean": 91.61766052246094, "margin_dpo/margin_std": 122.15998077392578, "step": 262 }, { "KL/chosen_KL_mean": -189.57598876953125, "KL/mean": -228.04983520507812, "KL/rejected_KL_mean": -266.523681640625, "KL/std": 116.67737579345703, "epoch": 0.5507853403141362, "fcm_dpo/beta": 0.003702682675793767, "fcm_dpo/delta": 0.0006384910084307194, "fcm_dpo/margin": 76.94770050048828, "fcm_dpo/q_t": 0.4331601560115814, "grad_norm": 52.963157653808594, "learning_rate": 2.509153804294318e-07, "logits/chosen": -0.8113653063774109, "logits/rejected": -0.7922741174697876, "logps/chosen": -469.5285949707031, "logps/ref_chosen": -279.95257568359375, "logps/ref_rejected": -256.5327453613281, "logps/rejected": -523.0564575195312, "loss": 4.7282, "margin_dpo/margin_mean": 76.94770050048828, "margin_dpo/margin_std": 126.67123413085938, "step": 263 }, { "KL/chosen_KL_mean": -180.07493591308594, "KL/mean": -227.3055877685547, "KL/rejected_KL_mean": -274.53619384765625, "KL/std": 121.89452362060547, "epoch": 0.5528795811518324, "fcm_dpo/beta": 0.0036615305580198765, "fcm_dpo/delta": -0.010648549534380436, "fcm_dpo/margin": 94.46124267578125, "fcm_dpo/q_t": 0.4205635190010071, "grad_norm": 50.7412109375, "learning_rate": 2.4908461957056825e-07, "logits/chosen": -0.8269824981689453, "logits/rejected": -0.818806529045105, "logps/chosen": -440.6100769042969, "logps/ref_chosen": -260.53509521484375, "logps/ref_rejected": -255.53799438476562, "logps/rejected": -530.07421875, "loss": 4.5297, "margin_dpo/margin_mean": 94.46124267578125, "margin_dpo/margin_std": 134.4732666015625, "step": 264 }, { "KL/chosen_KL_mean": -182.9898223876953, "KL/mean": -233.20361328125, "KL/rejected_KL_mean": -283.4173889160156, "KL/std": 130.7704620361328, "epoch": 0.5549738219895288, "fcm_dpo/beta": 0.0036417150404304266, "fcm_dpo/delta": -0.012156824581325054, "fcm_dpo/margin": 100.42760467529297, "fcm_dpo/q_t": 0.41630399227142334, "grad_norm": 58.42458724975586, "learning_rate": 2.4725390780077905e-07, "logits/chosen": -0.9007827043533325, "logits/rejected": -0.9030391573905945, "logps/chosen": -466.7028503417969, "logps/ref_chosen": -283.7130432128906, "logps/ref_rejected": -270.3209533691406, "logps/rejected": -553.7383422851562, "loss": 4.5194, "margin_dpo/margin_mean": 100.42760467529297, "margin_dpo/margin_std": 151.31163024902344, "step": 265 }, { "KL/chosen_KL_mean": -172.0806884765625, "KL/mean": -221.55726623535156, "KL/rejected_KL_mean": -271.0338439941406, "KL/std": 119.99907684326172, "epoch": 0.5570680628272251, "fcm_dpo/beta": 0.0035795229487121105, "fcm_dpo/delta": -0.0383283831179142, "fcm_dpo/margin": 98.95314025878906, "fcm_dpo/q_t": 0.41741737723350525, "grad_norm": 44.32012176513672, "learning_rate": 2.454233432955807e-07, "logits/chosen": -0.9105031490325928, "logits/rejected": -0.881584107875824, "logps/chosen": -450.17999267578125, "logps/ref_chosen": -278.09930419921875, "logps/ref_rejected": -260.6734619140625, "logps/rejected": -531.7073364257812, "loss": 4.4727, "margin_dpo/margin_mean": 98.95314025878906, "margin_dpo/margin_std": 129.08531188964844, "step": 266 }, { "KL/chosen_KL_mean": -203.26300048828125, "KL/mean": -240.33253479003906, "KL/rejected_KL_mean": -277.4020080566406, "KL/std": 123.59054565429688, "epoch": 0.5591623036649215, "fcm_dpo/beta": 0.0035598650574684143, "fcm_dpo/delta": 0.02088256925344467, "fcm_dpo/margin": 74.1390151977539, "fcm_dpo/q_t": 0.43908679485321045, "grad_norm": 63.60444259643555, "learning_rate": 2.435930242225919e-07, "logits/chosen": -0.8473813533782959, "logits/rejected": -0.8534542322158813, "logps/chosen": -483.59619140625, "logps/ref_chosen": -280.33319091796875, "logps/ref_rejected": -247.78099060058594, "logps/rejected": -525.1829833984375, "loss": 4.8265, "margin_dpo/margin_mean": 74.1390151977539, "margin_dpo/margin_std": 141.98263549804688, "step": 267 }, { "KL/chosen_KL_mean": -191.51849365234375, "KL/mean": -238.06063842773438, "KL/rejected_KL_mean": -284.60284423828125, "KL/std": 125.77220916748047, "epoch": 0.5612565445026177, "fcm_dpo/beta": 0.0036618567537516356, "fcm_dpo/delta": 0.031537555158138275, "fcm_dpo/margin": 93.08435821533203, "fcm_dpo/q_t": 0.4210575520992279, "grad_norm": 70.60133361816406, "learning_rate": 2.4176304873626984e-07, "logits/chosen": -0.827730119228363, "logits/rejected": -0.8081169128417969, "logps/chosen": -495.69720458984375, "logps/ref_chosen": -304.1787109375, "logps/ref_rejected": -272.80316162109375, "logps/rejected": -557.4060668945312, "loss": 4.5456, "margin_dpo/margin_mean": 93.08436584472656, "margin_dpo/margin_std": 135.3468475341797, "step": 268 }, { "KL/chosen_KL_mean": -192.0048828125, "KL/mean": -225.10169982910156, "KL/rejected_KL_mean": -258.198486328125, "KL/std": 111.24128723144531, "epoch": 0.5633507853403141, "fcm_dpo/beta": 0.0036840138491243124, "fcm_dpo/delta": 0.0019121436635032296, "fcm_dpo/margin": 66.193603515625, "fcm_dpo/q_t": 0.44279053807258606, "grad_norm": 64.69379425048828, "learning_rate": 2.399335149726463e-07, "logits/chosen": -0.8461325764656067, "logits/rejected": -0.8401869535446167, "logps/chosen": -441.85003662109375, "logps/ref_chosen": -249.84512329101562, "logps/ref_rejected": -223.37356567382812, "logps/rejected": -481.57208251953125, "loss": 4.8866, "margin_dpo/margin_mean": 66.193603515625, "margin_dpo/margin_std": 138.15615844726562, "step": 269 }, { "KL/chosen_KL_mean": -198.5923309326172, "KL/mean": -241.0689697265625, "KL/rejected_KL_mean": -283.5456237792969, "KL/std": 129.37863159179688, "epoch": 0.5654450261780105, "fcm_dpo/beta": 0.003672501305118203, "fcm_dpo/delta": -0.003966245800256729, "fcm_dpo/margin": 84.95326232910156, "fcm_dpo/q_t": 0.42902982234954834, "grad_norm": 59.16077423095703, "learning_rate": 2.381045210440644e-07, "logits/chosen": -0.9263371825218201, "logits/rejected": -0.9309374690055847, "logps/chosen": -517.1547241210938, "logps/ref_chosen": -318.5623779296875, "logps/ref_rejected": -281.1880798339844, "logps/rejected": -564.7337036132812, "loss": 4.731, "margin_dpo/margin_mean": 84.95326232910156, "margin_dpo/margin_std": 162.09225463867188, "step": 270 }, { "KL/chosen_KL_mean": -195.5090789794922, "KL/mean": -234.53392028808594, "KL/rejected_KL_mean": -273.5587158203125, "KL/std": 128.2386474609375, "epoch": 0.5675392670157068, "fcm_dpo/beta": 0.0036695911549031734, "fcm_dpo/delta": 0.001470465213060379, "fcm_dpo/margin": 78.04962921142578, "fcm_dpo/q_t": 0.43377920985221863, "grad_norm": 59.94120407104492, "learning_rate": 2.3627616503391812e-07, "logits/chosen": -0.7686310410499573, "logits/rejected": -0.761326789855957, "logps/chosen": -479.6138000488281, "logps/ref_chosen": -284.104736328125, "logps/ref_rejected": -253.9580535888672, "logps/rejected": -527.5167846679688, "loss": 4.727, "margin_dpo/margin_mean": 78.04962921142578, "margin_dpo/margin_std": 134.4047393798828, "step": 271 }, { "KL/chosen_KL_mean": -186.4130859375, "KL/mean": -229.1510009765625, "KL/rejected_KL_mean": -271.888916015625, "KL/std": 129.54705810546875, "epoch": 0.5696335078534032, "fcm_dpo/beta": 0.003663167357444763, "fcm_dpo/delta": 0.006455127615481615, "fcm_dpo/margin": 85.47587585449219, "fcm_dpo/q_t": 0.42850711941719055, "grad_norm": 56.258522033691406, "learning_rate": 2.344485449913914e-07, "logits/chosen": -0.8799334168434143, "logits/rejected": -0.8621615171432495, "logps/chosen": -483.7720642089844, "logps/ref_chosen": -297.3590087890625, "logps/ref_rejected": -279.20196533203125, "logps/rejected": -551.0909423828125, "loss": 4.7078, "margin_dpo/margin_mean": 85.47587585449219, "margin_dpo/margin_std": 155.64697265625, "step": 272 }, { "KL/chosen_KL_mean": -185.26217651367188, "KL/mean": -228.59132385253906, "KL/rejected_KL_mean": -271.9204406738281, "KL/std": 119.84618377685547, "epoch": 0.5717277486910994, "fcm_dpo/beta": 0.003680173773318529, "fcm_dpo/delta": 0.0036273770965635777, "fcm_dpo/margin": 86.65824127197266, "fcm_dpo/q_t": 0.4274797737598419, "grad_norm": 75.49598693847656, "learning_rate": 2.3262175892620062e-07, "logits/chosen": -0.8711019158363342, "logits/rejected": -0.8843865394592285, "logps/chosen": -478.4679260253906, "logps/ref_chosen": -293.20574951171875, "logps/ref_rejected": -274.7646789550781, "logps/rejected": -546.68505859375, "loss": 4.6788, "margin_dpo/margin_mean": 86.65824127197266, "margin_dpo/margin_std": 152.3551025390625, "step": 273 }, { "KL/chosen_KL_mean": -184.4656982421875, "KL/mean": -236.48504638671875, "KL/rejected_KL_mean": -288.50445556640625, "KL/std": 131.9507598876953, "epoch": 0.5738219895287958, "fcm_dpo/beta": 0.0037050426471978426, "fcm_dpo/delta": -0.001804053783416748, "fcm_dpo/margin": 104.03874206542969, "fcm_dpo/q_t": 0.41177284717559814, "grad_norm": 57.44676208496094, "learning_rate": 2.3079590480333827e-07, "logits/chosen": -0.8160399794578552, "logits/rejected": -0.784659206867218, "logps/chosen": -455.0243835449219, "logps/ref_chosen": -270.55865478515625, "logps/ref_rejected": -239.47048950195312, "logps/rejected": -527.9749145507812, "loss": 4.4432, "margin_dpo/margin_mean": 104.03874206542969, "margin_dpo/margin_std": 148.03915405273438, "step": 274 }, { "KL/chosen_KL_mean": -167.6212921142578, "KL/mean": -221.64671325683594, "KL/rejected_KL_mean": -275.67218017578125, "KL/std": 129.25466918945312, "epoch": 0.5759162303664922, "fcm_dpo/beta": 0.0036281857173889875, "fcm_dpo/delta": -0.03448822721838951, "fcm_dpo/margin": 108.0508804321289, "fcm_dpo/q_t": 0.4102928340435028, "grad_norm": 60.525047302246094, "learning_rate": 2.2897108053782e-07, "logits/chosen": -0.873982310295105, "logits/rejected": -0.8553178906440735, "logps/chosen": -417.94049072265625, "logps/ref_chosen": -250.31922912597656, "logps/ref_rejected": -249.3187255859375, "logps/rejected": -524.9908447265625, "loss": 4.3886, "margin_dpo/margin_mean": 108.05087280273438, "margin_dpo/margin_std": 142.07064819335938, "step": 275 }, { "KL/chosen_KL_mean": -172.93948364257812, "KL/mean": -224.42706298828125, "KL/rejected_KL_mean": -275.91461181640625, "KL/std": 131.497314453125, "epoch": 0.5780104712041885, "fcm_dpo/beta": 0.0035385219380259514, "fcm_dpo/delta": -0.009656044654548168, "fcm_dpo/margin": 102.97510528564453, "fcm_dpo/q_t": 0.4158563017845154, "grad_norm": 54.36017608642578, "learning_rate": 2.2714738398943308e-07, "logits/chosen": -0.9190772771835327, "logits/rejected": -0.8974164724349976, "logps/chosen": -470.5705261230469, "logps/ref_chosen": -297.6310729980469, "logps/ref_rejected": -295.225830078125, "logps/rejected": -571.1404418945312, "loss": 4.4991, "margin_dpo/margin_mean": 102.97510528564453, "margin_dpo/margin_std": 147.03050231933594, "step": 276 }, { "KL/chosen_KL_mean": -197.20071411132812, "KL/mean": -231.30577087402344, "KL/rejected_KL_mean": -265.41082763671875, "KL/std": 134.91824340820312, "epoch": 0.5801047120418849, "fcm_dpo/beta": 0.003561137244105339, "fcm_dpo/delta": -0.004946920555084944, "fcm_dpo/margin": 68.21009826660156, "fcm_dpo/q_t": 0.44467830657958984, "grad_norm": 63.98385238647461, "learning_rate": 2.2532491295748865e-07, "logits/chosen": -0.8566328287124634, "logits/rejected": -0.8565943837165833, "logps/chosen": -463.5611877441406, "logps/ref_chosen": -266.3604736328125, "logps/ref_rejected": -253.36767578125, "logps/rejected": -518.7784423828125, "loss": 4.9356, "margin_dpo/margin_mean": 68.21009826660156, "margin_dpo/margin_std": 153.74371337890625, "step": 277 }, { "KL/chosen_KL_mean": -204.8040771484375, "KL/mean": -233.74908447265625, "KL/rejected_KL_mean": -262.694091796875, "KL/std": 139.94210815429688, "epoch": 0.5821989528795811, "fcm_dpo/beta": 0.003580951364710927, "fcm_dpo/delta": 0.01175983902066946, "fcm_dpo/margin": 57.88999557495117, "fcm_dpo/q_t": 0.4574360251426697, "grad_norm": 59.146480560302734, "learning_rate": 2.2350376517557726e-07, "logits/chosen": -0.8874562382698059, "logits/rejected": -0.8490403890609741, "logps/chosen": -472.21136474609375, "logps/ref_chosen": -267.40728759765625, "logps/ref_rejected": -229.5758514404297, "logps/rejected": -492.2698974609375, "loss": 5.1767, "margin_dpo/margin_mean": 57.88999557495117, "margin_dpo/margin_std": 176.36570739746094, "step": 278 }, { "KL/chosen_KL_mean": -189.5259552001953, "KL/mean": -242.93582153320312, "KL/rejected_KL_mean": -296.345703125, "KL/std": 145.5375213623047, "epoch": 0.5842931937172775, "fcm_dpo/beta": 0.003567744279280305, "fcm_dpo/delta": -0.021797548979520798, "fcm_dpo/margin": 106.81974029541016, "fcm_dpo/q_t": 0.4145677089691162, "grad_norm": 66.13142395019531, "learning_rate": 2.2168403830632769e-07, "logits/chosen": -0.8134815096855164, "logits/rejected": -0.7979801893234253, "logps/chosen": -502.89373779296875, "logps/ref_chosen": -313.3677978515625, "logps/ref_rejected": -299.1744384765625, "logps/rejected": -595.5202026367188, "loss": 4.4922, "margin_dpo/margin_mean": 106.81974792480469, "margin_dpo/margin_std": 161.84104919433594, "step": 279 }, { "KL/chosen_KL_mean": -174.68370056152344, "KL/mean": -217.16433715820312, "KL/rejected_KL_mean": -259.64495849609375, "KL/std": 118.23523712158203, "epoch": 0.5863874345549738, "fcm_dpo/beta": 0.0035581968259066343, "fcm_dpo/delta": 0.018335789442062378, "fcm_dpo/margin": 84.96123504638672, "fcm_dpo/q_t": 0.4296843409538269, "grad_norm": 67.77641296386719, "learning_rate": 2.1986582993616925e-07, "logits/chosen": -0.8782325983047485, "logits/rejected": -0.8902976512908936, "logps/chosen": -440.239501953125, "logps/ref_chosen": -265.5558166503906, "logps/ref_rejected": -247.1573944091797, "logps/rejected": -506.8023681640625, "loss": 4.6825, "margin_dpo/margin_mean": 84.96123504638672, "margin_dpo/margin_std": 143.5811004638672, "step": 280 }, { "KL/chosen_KL_mean": -199.01400756835938, "KL/mean": -250.08953857421875, "KL/rejected_KL_mean": -301.16510009765625, "KL/std": 136.9486083984375, "epoch": 0.5884816753926702, "fcm_dpo/beta": 0.0035306517966091633, "fcm_dpo/delta": -0.02944459766149521, "fcm_dpo/margin": 102.15110778808594, "fcm_dpo/q_t": 0.4185321033000946, "grad_norm": 62.55031967163086, "learning_rate": 2.1804923757009882e-07, "logits/chosen": -0.8659012913703918, "logits/rejected": -0.8745157122612, "logps/chosen": -494.3135986328125, "logps/ref_chosen": -295.2995910644531, "logps/ref_rejected": -293.80877685546875, "logps/rejected": -594.9738159179688, "loss": 4.5101, "margin_dpo/margin_mean": 102.15110778808594, "margin_dpo/margin_std": 148.16885375976562, "step": 281 }, { "KL/chosen_KL_mean": -205.15968322753906, "KL/mean": -245.86717224121094, "KL/rejected_KL_mean": -286.5746765136719, "KL/std": 131.70346069335938, "epoch": 0.5905759162303665, "fcm_dpo/beta": 0.003522593295201659, "fcm_dpo/delta": 0.00954904779791832, "fcm_dpo/margin": 81.4150161743164, "fcm_dpo/q_t": 0.43364566564559937, "grad_norm": 60.06401062011719, "learning_rate": 2.1623435862645205e-07, "logits/chosen": -0.8671073913574219, "logits/rejected": -0.8696990013122559, "logps/chosen": -523.7968139648438, "logps/ref_chosen": -318.63714599609375, "logps/ref_rejected": -273.5943603515625, "logps/rejected": -560.1690673828125, "loss": 4.7488, "margin_dpo/margin_mean": 81.4150161743164, "margin_dpo/margin_std": 147.11825561523438, "step": 282 }, { "KL/chosen_KL_mean": -196.23243713378906, "KL/mean": -240.39724731445312, "KL/rejected_KL_mean": -284.5620422363281, "KL/std": 129.15927124023438, "epoch": 0.5926701570680628, "fcm_dpo/beta": 0.003505897242575884, "fcm_dpo/delta": -0.027185089886188507, "fcm_dpo/margin": 88.32962036132812, "fcm_dpo/q_t": 0.4301910996437073, "grad_norm": 63.45745849609375, "learning_rate": 2.1442129043167873e-07, "logits/chosen": -0.8812756538391113, "logits/rejected": -0.8700328469276428, "logps/chosen": -450.8929748535156, "logps/ref_chosen": -254.66053771972656, "logps/ref_rejected": -236.8627166748047, "logps/rejected": -521.4248046875, "loss": 4.6848, "margin_dpo/margin_mean": 88.32962036132812, "margin_dpo/margin_std": 149.92486572265625, "step": 283 }, { "KL/chosen_KL_mean": -211.7559814453125, "KL/mean": -256.141357421875, "KL/rejected_KL_mean": -300.5267333984375, "KL/std": 144.52191162109375, "epoch": 0.5947643979057592, "fcm_dpo/beta": 0.003395251464098692, "fcm_dpo/delta": -0.023953504860401154, "fcm_dpo/margin": 88.77078247070312, "fcm_dpo/q_t": 0.43172183632850647, "grad_norm": 57.34799575805664, "learning_rate": 2.1261013021512378e-07, "logits/chosen": -0.844742476940155, "logits/rejected": -0.8104969263076782, "logps/chosen": -485.1112365722656, "logps/ref_chosen": -273.355224609375, "logps/ref_rejected": -259.84759521484375, "logps/rejected": -560.3743896484375, "loss": 4.7799, "margin_dpo/margin_mean": 88.77078247070312, "margin_dpo/margin_std": 171.4403533935547, "step": 284 }, { "KL/chosen_KL_mean": -215.08908081054688, "KL/mean": -244.9665069580078, "KL/rejected_KL_mean": -274.8439636230469, "KL/std": 130.01708984375, "epoch": 0.5968586387434555, "fcm_dpo/beta": 0.0033849114552140236, "fcm_dpo/delta": -0.0046186684630811214, "fcm_dpo/margin": 59.75489807128906, "fcm_dpo/q_t": 0.45396003127098083, "grad_norm": 75.9342041015625, "learning_rate": 2.1080097510381294e-07, "logits/chosen": -0.8575014472007751, "logits/rejected": -0.8521921634674072, "logps/chosen": -524.891357421875, "logps/ref_chosen": -309.8022155761719, "logps/ref_rejected": -279.11846923828125, "logps/rejected": -553.96240234375, "loss": 5.0888, "margin_dpo/margin_mean": 59.75489807128906, "margin_dpo/margin_std": 159.50062561035156, "step": 285 }, { "KL/chosen_KL_mean": -198.20655822753906, "KL/mean": -242.35784912109375, "KL/rejected_KL_mean": -286.50909423828125, "KL/std": 141.2283172607422, "epoch": 0.5989528795811518, "fcm_dpo/beta": 0.0033742974046617746, "fcm_dpo/delta": 0.00259046396240592, "fcm_dpo/margin": 88.30255126953125, "fcm_dpo/q_t": 0.4332633316516876, "grad_norm": 82.05410766601562, "learning_rate": 2.089939221172446e-07, "logits/chosen": -0.8378498554229736, "logits/rejected": -0.8318293690681458, "logps/chosen": -469.6720886230469, "logps/ref_chosen": -271.4655456542969, "logps/ref_rejected": -279.531494140625, "logps/rejected": -566.0405883789062, "loss": 4.7228, "margin_dpo/margin_mean": 88.30255126953125, "margin_dpo/margin_std": 159.4552459716797, "step": 286 }, { "KL/chosen_KL_mean": -207.5540008544922, "KL/mean": -247.44190979003906, "KL/rejected_KL_mean": -287.329833984375, "KL/std": 134.13143920898438, "epoch": 0.6010471204188481, "fcm_dpo/beta": 0.0034193575847893953, "fcm_dpo/delta": 0.0249363761395216, "fcm_dpo/margin": 79.77581024169922, "fcm_dpo/q_t": 0.43721669912338257, "grad_norm": 56.872581481933594, "learning_rate": 2.0718906816218595e-07, "logits/chosen": -0.8712465167045593, "logits/rejected": -0.8559173941612244, "logps/chosen": -484.647216796875, "logps/ref_chosen": -277.0932312011719, "logps/ref_rejected": -233.55599975585938, "logps/rejected": -520.8858032226562, "loss": 4.8451, "margin_dpo/margin_mean": 79.77581787109375, "margin_dpo/margin_std": 164.70887756347656, "step": 287 }, { "KL/chosen_KL_mean": -210.85910034179688, "KL/mean": -248.73919677734375, "KL/rejected_KL_mean": -286.6192932128906, "KL/std": 138.9389190673828, "epoch": 0.6031413612565445, "fcm_dpo/beta": 0.003475298173725605, "fcm_dpo/delta": 0.0075583611615002155, "fcm_dpo/margin": 75.76020812988281, "fcm_dpo/q_t": 0.4398469924926758, "grad_norm": 84.63459014892578, "learning_rate": 2.053865100274774e-07, "logits/chosen": -0.8746079206466675, "logits/rejected": -0.8799384832382202, "logps/chosen": -504.0272521972656, "logps/ref_chosen": -293.1681823730469, "logps/ref_rejected": -263.4059143066406, "logps/rejected": -550.0252075195312, "loss": 4.8465, "margin_dpo/margin_mean": 75.76020812988281, "margin_dpo/margin_std": 154.83770751953125, "step": 288 }, { "KL/chosen_KL_mean": -228.95736694335938, "KL/mean": -254.9104766845703, "KL/rejected_KL_mean": -280.86358642578125, "KL/std": 126.50141906738281, "epoch": 0.6052356020942409, "fcm_dpo/beta": 0.0034878358710557222, "fcm_dpo/delta": 0.018204133957624435, "fcm_dpo/margin": 51.90618896484375, "fcm_dpo/q_t": 0.4578736424446106, "grad_norm": 80.30821228027344, "learning_rate": 2.035863443788411e-07, "logits/chosen": -0.8596778512001038, "logits/rejected": -0.841497540473938, "logps/chosen": -558.914794921875, "logps/ref_chosen": -329.9574279785156, "logps/ref_rejected": -276.7565002441406, "logps/rejected": -557.6200561523438, "loss": 5.1116, "margin_dpo/margin_mean": 51.906192779541016, "margin_dpo/margin_std": 142.21298217773438, "step": 289 }, { "KL/chosen_KL_mean": -208.2900848388672, "KL/mean": -250.78472900390625, "KL/rejected_KL_mean": -293.2793273925781, "KL/std": 139.5697784423828, "epoch": 0.6073298429319371, "fcm_dpo/beta": 0.0034028813242912292, "fcm_dpo/delta": -0.049578987061977386, "fcm_dpo/margin": 84.98924255371094, "fcm_dpo/q_t": 0.43361252546310425, "grad_norm": 59.59135818481445, "learning_rate": 2.0178866775369774e-07, "logits/chosen": -0.8690849542617798, "logits/rejected": -0.8019747138023376, "logps/chosen": -532.9591064453125, "logps/ref_chosen": -324.6690673828125, "logps/ref_rejected": -311.8439636230469, "logps/rejected": -605.123291015625, "loss": 4.7878, "margin_dpo/margin_mean": 84.98924255371094, "margin_dpo/margin_std": 162.67477416992188, "step": 290 }, { "KL/chosen_KL_mean": -200.08258056640625, "KL/mean": -254.6800079345703, "KL/rejected_KL_mean": -309.2774658203125, "KL/std": 136.1592559814453, "epoch": 0.6094240837696335, "fcm_dpo/beta": 0.0033203570637851954, "fcm_dpo/delta": -0.027233559638261795, "fcm_dpo/margin": 109.19487762451172, "fcm_dpo/q_t": 0.4158145785331726, "grad_norm": 57.8904914855957, "learning_rate": 1.9999357655598891e-07, "logits/chosen": -0.8527657985687256, "logits/rejected": -0.8431987762451172, "logps/chosen": -474.2266540527344, "logps/ref_chosen": -274.1440734863281, "logps/ref_rejected": -278.07208251953125, "logps/rejected": -587.3495483398438, "loss": 4.477, "margin_dpo/margin_mean": 109.19487762451172, "margin_dpo/margin_std": 146.18112182617188, "step": 291 }, { "KL/chosen_KL_mean": -194.33355712890625, "KL/mean": -239.71888732910156, "KL/rejected_KL_mean": -285.10418701171875, "KL/std": 141.9927978515625, "epoch": 0.6115183246073298, "fcm_dpo/beta": 0.003264216473326087, "fcm_dpo/delta": -0.02208729460835457, "fcm_dpo/margin": 90.77063751220703, "fcm_dpo/q_t": 0.43203163146972656, "grad_norm": 83.5884017944336, "learning_rate": 1.9820116705100775e-07, "logits/chosen": -0.8462182283401489, "logits/rejected": -0.8420131206512451, "logps/chosen": -453.69720458984375, "logps/ref_chosen": -259.3636779785156, "logps/ref_rejected": -279.30218505859375, "logps/rejected": -564.4063720703125, "loss": 4.7423, "margin_dpo/margin_mean": 90.77064514160156, "margin_dpo/margin_std": 164.15216064453125, "step": 292 }, { "KL/chosen_KL_mean": -218.1024169921875, "KL/mean": -264.27685546875, "KL/rejected_KL_mean": -310.4512939453125, "KL/std": 145.62364196777344, "epoch": 0.6136125654450262, "fcm_dpo/beta": 0.0032458994537591934, "fcm_dpo/delta": 0.0010393057018518448, "fcm_dpo/margin": 92.3488540649414, "fcm_dpo/q_t": 0.430684357881546, "grad_norm": 79.72573852539062, "learning_rate": 1.9641153536023642e-07, "logits/chosen": -0.9407028555870056, "logits/rejected": -0.901149570941925, "logps/chosen": -521.8732299804688, "logps/ref_chosen": -303.77081298828125, "logps/ref_rejected": -270.07513427734375, "logps/rejected": -580.5263671875, "loss": 4.6868, "margin_dpo/margin_mean": 92.34884643554688, "margin_dpo/margin_std": 150.49273681640625, "step": 293 }, { "KL/chosen_KL_mean": -210.60379028320312, "KL/mean": -250.68609619140625, "KL/rejected_KL_mean": -290.7684020996094, "KL/std": 138.80445861816406, "epoch": 0.6157068062827226, "fcm_dpo/beta": 0.0032662583980709314, "fcm_dpo/delta": 0.0030375849455595016, "fcm_dpo/margin": 80.16455841064453, "fcm_dpo/q_t": 0.440270334482193, "grad_norm": 77.52326965332031, "learning_rate": 1.9462477745619106e-07, "logits/chosen": -0.8381096720695496, "logits/rejected": -0.8436415195465088, "logps/chosen": -450.8421325683594, "logps/ref_chosen": -240.23831176757812, "logps/ref_rejected": -229.187744140625, "logps/rejected": -519.9561157226562, "loss": 4.8661, "margin_dpo/margin_mean": 80.16456604003906, "margin_dpo/margin_std": 165.5260467529297, "step": 294 }, { "KL/chosen_KL_mean": -204.40306091308594, "KL/mean": -245.8980712890625, "KL/rejected_KL_mean": -287.39306640625, "KL/std": 122.65670013427734, "epoch": 0.6178010471204188, "fcm_dpo/beta": 0.0032912057358771563, "fcm_dpo/delta": 0.014641467481851578, "fcm_dpo/margin": 82.98999786376953, "fcm_dpo/q_t": 0.4375506639480591, "grad_norm": 59.422271728515625, "learning_rate": 1.928409891572757e-07, "logits/chosen": -0.8383684158325195, "logits/rejected": -0.8414021134376526, "logps/chosen": -455.4127502441406, "logps/ref_chosen": -251.00970458984375, "logps/ref_rejected": -244.15142822265625, "logps/rejected": -531.5444946289062, "loss": 4.7802, "margin_dpo/margin_mean": 82.98999786376953, "margin_dpo/margin_std": 152.8289794921875, "step": 295 }, { "KL/chosen_KL_mean": -216.45693969726562, "KL/mean": -269.9961853027344, "KL/rejected_KL_mean": -323.5354309082031, "KL/std": 143.4501953125, "epoch": 0.6198952879581152, "fcm_dpo/beta": 0.0032748132944107056, "fcm_dpo/delta": -0.014164071530103683, "fcm_dpo/margin": 107.0785140991211, "fcm_dpo/q_t": 0.4195513427257538, "grad_norm": 66.60144805908203, "learning_rate": 1.9106026612264315e-07, "logits/chosen": -0.8362563848495483, "logits/rejected": -0.8021262884140015, "logps/chosen": -510.3375244140625, "logps/ref_chosen": -293.880615234375, "logps/ref_rejected": -283.4175720214844, "logps/rejected": -606.9530639648438, "loss": 4.5964, "margin_dpo/margin_mean": 107.0785140991211, "margin_dpo/margin_std": 174.71786499023438, "step": 296 }, { "KL/chosen_KL_mean": -209.06858825683594, "KL/mean": -250.95046997070312, "KL/rejected_KL_mean": -292.8323974609375, "KL/std": 129.31927490234375, "epoch": 0.6219895287958115, "fcm_dpo/beta": 0.003221014980226755, "fcm_dpo/delta": -0.01936129853129387, "fcm_dpo/margin": 83.7637710571289, "fcm_dpo/q_t": 0.4371768832206726, "grad_norm": 52.13896179199219, "learning_rate": 1.8928270384706582e-07, "logits/chosen": -0.9062499403953552, "logits/rejected": -0.899086058139801, "logps/chosen": -498.5286560058594, "logps/ref_chosen": -289.4600830078125, "logps/ref_rejected": -283.69110107421875, "logps/rejected": -576.5234375, "loss": 4.7666, "margin_dpo/margin_mean": 83.7637710571289, "margin_dpo/margin_std": 145.26136779785156, "step": 297 }, { "KL/chosen_KL_mean": -202.0797576904297, "KL/mean": -250.1943817138672, "KL/rejected_KL_mean": -298.3089904785156, "KL/std": 134.8616943359375, "epoch": 0.6240837696335079, "fcm_dpo/beta": 0.0031903949566185474, "fcm_dpo/delta": -0.01658639684319496, "fcm_dpo/margin": 96.22921752929688, "fcm_dpo/q_t": 0.42945045232772827, "grad_norm": 80.38378143310547, "learning_rate": 1.875083976558136e-07, "logits/chosen": -0.8216419219970703, "logits/rejected": -0.8078317642211914, "logps/chosen": -508.5947570800781, "logps/ref_chosen": -306.5150146484375, "logps/ref_rejected": -280.6969909667969, "logps/rejected": -579.0059204101562, "loss": 4.7, "margin_dpo/margin_mean": 96.2292251586914, "margin_dpo/margin_std": 163.79393005371094, "step": 298 }, { "KL/chosen_KL_mean": -233.1396484375, "KL/mean": -269.0777893066406, "KL/rejected_KL_mean": -305.0159606933594, "KL/std": 135.54824829101562, "epoch": 0.6261780104712041, "fcm_dpo/beta": 0.003153471741825342, "fcm_dpo/delta": -0.007434126455336809, "fcm_dpo/margin": 71.87633514404297, "fcm_dpo/q_t": 0.4467780888080597, "grad_norm": 91.21902465820312, "learning_rate": 1.8573744269954297e-07, "logits/chosen": -0.811547040939331, "logits/rejected": -0.7952200174331665, "logps/chosen": -514.5033569335938, "logps/ref_chosen": -281.36376953125, "logps/ref_rejected": -270.39508056640625, "logps/rejected": -575.4110107421875, "loss": 4.9011, "margin_dpo/margin_mean": 71.87632751464844, "margin_dpo/margin_std": 136.6454620361328, "step": 299 }, { "KL/chosen_KL_mean": -228.64112854003906, "KL/mean": -268.7874755859375, "KL/rejected_KL_mean": -308.933837890625, "KL/std": 145.60882568359375, "epoch": 0.6282722513089005, "fcm_dpo/beta": 0.003157139290124178, "fcm_dpo/delta": 0.0067957015708088875, "fcm_dpo/margin": 80.2927017211914, "fcm_dpo/q_t": 0.44123968482017517, "grad_norm": 61.41617965698242, "learning_rate": 1.839699339491937e-07, "logits/chosen": -0.84067302942276, "logits/rejected": -0.8198051452636719, "logps/chosen": -543.4768676757812, "logps/ref_chosen": -314.83575439453125, "logps/ref_rejected": -269.1154479980469, "logps/rejected": -578.0492553710938, "loss": 4.8649, "margin_dpo/margin_mean": 80.2927017211914, "margin_dpo/margin_std": 165.27395629882812, "step": 300 }, { "KL/chosen_KL_mean": -213.63665771484375, "KL/mean": -253.96006774902344, "KL/rejected_KL_mean": -294.28350830078125, "KL/std": 134.56875610351562, "epoch": 0.6303664921465969, "fcm_dpo/beta": 0.003165181027725339, "fcm_dpo/delta": -0.007671603467315435, "fcm_dpo/margin": 80.64684295654297, "fcm_dpo/q_t": 0.44037818908691406, "grad_norm": 58.440460205078125, "learning_rate": 1.8220596619089573e-07, "logits/chosen": -0.869897723197937, "logits/rejected": -0.8631291389465332, "logps/chosen": -493.5311584472656, "logps/ref_chosen": -279.89453125, "logps/ref_rejected": -271.6694641113281, "logps/rejected": -565.9530029296875, "loss": 4.8103, "margin_dpo/margin_mean": 80.64684295654297, "margin_dpo/margin_std": 145.11534118652344, "step": 301 }, { "KL/chosen_KL_mean": -213.73731994628906, "KL/mean": -264.9085388183594, "KL/rejected_KL_mean": -316.0797424316406, "KL/std": 142.73170471191406, "epoch": 0.6324607329842932, "fcm_dpo/beta": 0.003172217635437846, "fcm_dpo/delta": 0.01391267403960228, "fcm_dpo/margin": 102.34239959716797, "fcm_dpo/q_t": 0.4247261881828308, "grad_norm": 64.62631225585938, "learning_rate": 1.8044563402088682e-07, "logits/chosen": -0.8292580842971802, "logits/rejected": -0.8176696300506592, "logps/chosen": -485.06915283203125, "logps/ref_chosen": -271.3318176269531, "logps/ref_rejected": -256.5587158203125, "logps/rejected": -572.638427734375, "loss": 4.5893, "margin_dpo/margin_mean": 102.34239959716797, "margin_dpo/margin_std": 152.8368377685547, "step": 302 }, { "KL/chosen_KL_mean": -208.92112731933594, "KL/mean": -253.95260620117188, "KL/rejected_KL_mean": -298.9841003417969, "KL/std": 129.33070373535156, "epoch": 0.6345549738219896, "fcm_dpo/beta": 0.0031979738269001245, "fcm_dpo/delta": 0.008892138488590717, "fcm_dpo/margin": 90.06295013427734, "fcm_dpo/q_t": 0.43192505836486816, "grad_norm": 69.94568634033203, "learning_rate": 1.7868903184043885e-07, "logits/chosen": -0.8113217353820801, "logits/rejected": -0.7879561185836792, "logps/chosen": -513.8021240234375, "logps/ref_chosen": -304.88104248046875, "logps/ref_rejected": -269.063720703125, "logps/rejected": -568.0478515625, "loss": 4.6939, "margin_dpo/margin_mean": 90.06295776367188, "margin_dpo/margin_std": 140.77880859375, "step": 303 }, { "KL/chosen_KL_mean": -224.17652893066406, "KL/mean": -273.39581298828125, "KL/rejected_KL_mean": -322.6150817871094, "KL/std": 155.12173461914062, "epoch": 0.6366492146596858, "fcm_dpo/beta": 0.0032359191682189703, "fcm_dpo/delta": 0.02149886079132557, "fcm_dpo/margin": 98.43850708007812, "fcm_dpo/q_t": 0.42846569418907166, "grad_norm": 74.36817169189453, "learning_rate": 1.7693625385079574e-07, "logits/chosen": -0.8236927390098572, "logits/rejected": -0.8324135541915894, "logps/chosen": -514.8875122070312, "logps/ref_chosen": -290.7109680175781, "logps/ref_rejected": -237.6885986328125, "logps/rejected": -560.3036499023438, "loss": 4.6536, "margin_dpo/margin_mean": 98.43850708007812, "margin_dpo/margin_std": 167.7232666015625, "step": 304 }, { "KL/chosen_KL_mean": -188.64137268066406, "KL/mean": -263.8195495605469, "KL/rejected_KL_mean": -338.9976806640625, "KL/std": 151.70013427734375, "epoch": 0.6387434554973822, "fcm_dpo/beta": 0.003107309341430664, "fcm_dpo/delta": -0.11182599514722824, "fcm_dpo/margin": 150.35633850097656, "fcm_dpo/q_t": 0.393195241689682, "grad_norm": 81.83216094970703, "learning_rate": 1.7518739404812155e-07, "logits/chosen": -0.8657177686691284, "logits/rejected": -0.8329824805259705, "logps/chosen": -445.1253356933594, "logps/ref_chosen": -256.4839782714844, "logps/ref_rejected": -266.4063415527344, "logps/rejected": -605.404052734375, "loss": 4.1802, "margin_dpo/margin_mean": 150.3563232421875, "margin_dpo/margin_std": 173.1336212158203, "step": 305 }, { "KL/chosen_KL_mean": -202.8968505859375, "KL/mean": -247.1907501220703, "KL/rejected_KL_mean": -291.484619140625, "KL/std": 131.15960693359375, "epoch": 0.6408376963350786, "fcm_dpo/beta": 0.002977159805595875, "fcm_dpo/delta": 0.012585463933646679, "fcm_dpo/margin": 88.5877914428711, "fcm_dpo/q_t": 0.43810799717903137, "grad_norm": 64.3017349243164, "learning_rate": 1.7344254621846017e-07, "logits/chosen": -0.8670382499694824, "logits/rejected": -0.8467966914176941, "logps/chosen": -523.5460815429688, "logps/ref_chosen": -320.6492004394531, "logps/ref_rejected": -273.36773681640625, "logps/rejected": -564.8523559570312, "loss": 4.7537, "margin_dpo/margin_mean": 88.5877914428711, "margin_dpo/margin_std": 140.4833984375, "step": 306 }, { "KL/chosen_KL_mean": -236.40028381347656, "KL/mean": -281.1076965332031, "KL/rejected_KL_mean": -325.81512451171875, "KL/std": 148.4873504638672, "epoch": 0.6429319371727749, "fcm_dpo/beta": 0.0030362734105437994, "fcm_dpo/delta": 0.03187718614935875, "fcm_dpo/margin": 89.4148178100586, "fcm_dpo/q_t": 0.4356860816478729, "grad_norm": 75.47786712646484, "learning_rate": 1.717018039327053e-07, "logits/chosen": -0.7897435426712036, "logits/rejected": -0.8119577765464783, "logps/chosen": -515.8544921875, "logps/ref_chosen": -279.4541931152344, "logps/ref_rejected": -240.3796844482422, "logps/rejected": -566.19482421875, "loss": 4.7277, "margin_dpo/margin_mean": 89.41481018066406, "margin_dpo/margin_std": 140.82554626464844, "step": 307 }, { "KL/chosen_KL_mean": -255.62368774414062, "KL/mean": -287.2620544433594, "KL/rejected_KL_mean": -318.9004211425781, "KL/std": 149.1934051513672, "epoch": 0.6450261780104712, "fcm_dpo/beta": 0.003119693137705326, "fcm_dpo/delta": 0.012247689999639988, "fcm_dpo/margin": 63.276702880859375, "fcm_dpo/q_t": 0.45396554470062256, "grad_norm": 68.79566955566406, "learning_rate": 1.699652605415828e-07, "logits/chosen": -0.8509783744812012, "logits/rejected": -0.8624202013015747, "logps/chosen": -552.22216796875, "logps/ref_chosen": -296.598388671875, "logps/ref_rejected": -258.6953430175781, "logps/rejected": -577.5957641601562, "loss": 5.0384, "margin_dpo/margin_mean": 63.27670669555664, "margin_dpo/margin_std": 155.48910522460938, "step": 308 }, { "KL/chosen_KL_mean": -236.53305053710938, "KL/mean": -292.6006774902344, "KL/rejected_KL_mean": -348.6683044433594, "KL/std": 146.83604431152344, "epoch": 0.6471204188481675, "fcm_dpo/beta": 0.0031316163949668407, "fcm_dpo/delta": 0.0024306168779730797, "fcm_dpo/margin": 112.1352310180664, "fcm_dpo/q_t": 0.42018356919288635, "grad_norm": 74.16272735595703, "learning_rate": 1.6823300917064458e-07, "logits/chosen": -0.8479481339454651, "logits/rejected": -0.8579108715057373, "logps/chosen": -517.921142578125, "logps/ref_chosen": -281.3881530761719, "logps/ref_rejected": -262.458740234375, "logps/rejected": -611.1270141601562, "loss": 4.5344, "margin_dpo/margin_mean": 112.1352310180664, "margin_dpo/margin_std": 170.53929138183594, "step": 309 }, { "KL/chosen_KL_mean": -227.86915588378906, "KL/mean": -281.9316711425781, "KL/rejected_KL_mean": -335.99420166015625, "KL/std": 152.35032653808594, "epoch": 0.6492146596858639, "fcm_dpo/beta": 0.003158912993967533, "fcm_dpo/delta": 0.0094301737844944, "fcm_dpo/margin": 108.12503051757812, "fcm_dpo/q_t": 0.4215067923069, "grad_norm": 76.43192291259766, "learning_rate": 1.6650514271527465e-07, "logits/chosen": -0.8392739295959473, "logits/rejected": -0.8133707642555237, "logps/chosen": -507.056396484375, "logps/ref_chosen": -279.1872863769531, "logps/ref_rejected": -261.8279724121094, "logps/rejected": -597.8222045898438, "loss": 4.5592, "margin_dpo/margin_mean": 108.12503814697266, "margin_dpo/margin_std": 162.43666076660156, "step": 310 }, { "KL/chosen_KL_mean": -230.60015869140625, "KL/mean": -286.6460266113281, "KL/rejected_KL_mean": -342.69189453125, "KL/std": 164.47862243652344, "epoch": 0.6513089005235602, "fcm_dpo/beta": 0.0031031672842800617, "fcm_dpo/delta": -0.01641257479786873, "fcm_dpo/margin": 112.0916976928711, "fcm_dpo/q_t": 0.42211809754371643, "grad_norm": 68.3413314819336, "learning_rate": 1.647817538357072e-07, "logits/chosen": -0.8347179889678955, "logits/rejected": -0.8130024671554565, "logps/chosen": -501.9983215332031, "logps/ref_chosen": -271.39813232421875, "logps/ref_rejected": -266.12701416015625, "logps/rejected": -608.8189086914062, "loss": 4.6098, "margin_dpo/margin_mean": 112.0916976928711, "margin_dpo/margin_std": 184.5821075439453, "step": 311 }, { "KL/chosen_KL_mean": -227.80303955078125, "KL/mean": -276.14068603515625, "KL/rejected_KL_mean": -324.4783630371094, "KL/std": 157.450439453125, "epoch": 0.6534031413612565, "fcm_dpo/beta": 0.0030568826477974653, "fcm_dpo/delta": -0.028012195602059364, "fcm_dpo/margin": 96.67533111572266, "fcm_dpo/q_t": 0.43515488505363464, "grad_norm": 106.50956726074219, "learning_rate": 1.6306293495205755e-07, "logits/chosen": -0.8552649617195129, "logits/rejected": -0.8285101652145386, "logps/chosen": -510.18804931640625, "logps/ref_chosen": -282.3850402832031, "logps/ref_rejected": -246.35389709472656, "logps/rejected": -570.8322143554688, "loss": 4.8594, "margin_dpo/margin_mean": 96.67532348632812, "margin_dpo/margin_std": 206.41854858398438, "step": 312 }, { "KL/chosen_KL_mean": -230.12075805664062, "KL/mean": -276.99615478515625, "KL/rejected_KL_mean": -323.8715515136719, "KL/std": 160.83921813964844, "epoch": 0.6554973821989529, "fcm_dpo/beta": 0.003049192950129509, "fcm_dpo/delta": -0.01281303446739912, "fcm_dpo/margin": 93.75079345703125, "fcm_dpo/q_t": 0.43882137537002563, "grad_norm": 66.034912109375, "learning_rate": 1.6134877823936607e-07, "logits/chosen": -0.8895430564880371, "logits/rejected": -0.8738856911659241, "logps/chosen": -533.7515869140625, "logps/ref_chosen": -303.630859375, "logps/ref_rejected": -273.1156921386719, "logps/rejected": -596.9872436523438, "loss": 4.8408, "margin_dpo/margin_mean": 93.75078582763672, "margin_dpo/margin_std": 189.20486450195312, "step": 313 }, { "KL/chosen_KL_mean": -223.54823303222656, "KL/mean": -268.31048583984375, "KL/rejected_KL_mean": -313.0727233886719, "KL/std": 137.15122985839844, "epoch": 0.6575916230366492, "fcm_dpo/beta": 0.003023975295946002, "fcm_dpo/delta": 0.006585121154785156, "fcm_dpo/margin": 89.52447509765625, "fcm_dpo/q_t": 0.43667665123939514, "grad_norm": 73.49964141845703, "learning_rate": 1.5963937562265522e-07, "logits/chosen": -0.8998604416847229, "logits/rejected": -0.8924080729484558, "logps/chosen": -525.8524780273438, "logps/ref_chosen": -302.3042907714844, "logps/ref_rejected": -273.6416015625, "logps/rejected": -586.71435546875, "loss": 4.7536, "margin_dpo/margin_mean": 89.52449035644531, "margin_dpo/margin_std": 151.22608947753906, "step": 314 }, { "KL/chosen_KL_mean": -219.18116760253906, "KL/mean": -277.6825256347656, "KL/rejected_KL_mean": -336.18389892578125, "KL/std": 145.4262237548828, "epoch": 0.6596858638743456, "fcm_dpo/beta": 0.0030265850946307182, "fcm_dpo/delta": -0.0040887510403990746, "fcm_dpo/margin": 117.00270080566406, "fcm_dpo/q_t": 0.41837024688720703, "grad_norm": 76.63323211669922, "learning_rate": 1.5793481877199943e-07, "logits/chosen": -0.8940035700798035, "logits/rejected": -0.8783378005027771, "logps/chosen": -521.9104614257812, "logps/ref_chosen": -302.729248046875, "logps/ref_rejected": -270.26910400390625, "logps/rejected": -606.4529418945312, "loss": 4.5038, "margin_dpo/margin_mean": 117.00270080566406, "margin_dpo/margin_std": 163.93301391601562, "step": 315 }, { "KL/chosen_KL_mean": -229.85618591308594, "KL/mean": -278.86376953125, "KL/rejected_KL_mean": -327.8713684082031, "KL/std": 147.73422241210938, "epoch": 0.6617801047120419, "fcm_dpo/beta": 0.003056867513805628, "fcm_dpo/delta": -0.0008974131196737289, "fcm_dpo/margin": 98.01517486572266, "fcm_dpo/q_t": 0.4309207797050476, "grad_norm": 59.27046585083008, "learning_rate": 1.562351990976095e-07, "logits/chosen": -0.8997865915298462, "logits/rejected": -0.8820158243179321, "logps/chosen": -540.4267578125, "logps/ref_chosen": -310.5706481933594, "logps/ref_rejected": -272.9354553222656, "logps/rejected": -600.8068237304688, "loss": 4.7006, "margin_dpo/margin_mean": 98.01517486572266, "margin_dpo/margin_std": 166.4867401123047, "step": 316 }, { "KL/chosen_KL_mean": -198.8494873046875, "KL/mean": -245.5550537109375, "KL/rejected_KL_mean": -292.2606506347656, "KL/std": 128.78086853027344, "epoch": 0.6638743455497382, "fcm_dpo/beta": 0.0030271485447883606, "fcm_dpo/delta": 0.016471808776259422, "fcm_dpo/margin": 93.41114807128906, "fcm_dpo/q_t": 0.43299350142478943, "grad_norm": 58.140594482421875, "learning_rate": 1.5454060774493065e-07, "logits/chosen": -0.8841041326522827, "logits/rejected": -0.8535327911376953, "logps/chosen": -452.7498474121094, "logps/ref_chosen": -253.90036010742188, "logps/ref_rejected": -218.74078369140625, "logps/rejected": -511.00140380859375, "loss": 4.6731, "margin_dpo/margin_mean": 93.41114807128906, "margin_dpo/margin_std": 136.35061645507812, "step": 317 }, { "KL/chosen_KL_mean": -207.8227996826172, "KL/mean": -265.3223876953125, "KL/rejected_KL_mean": -322.82196044921875, "KL/std": 141.49267578125, "epoch": 0.6659685863874345, "fcm_dpo/beta": 0.003031826112419367, "fcm_dpo/delta": -0.009629029780626297, "fcm_dpo/margin": 114.99917602539062, "fcm_dpo/q_t": 0.4194708466529846, "grad_norm": 75.79177856445312, "learning_rate": 1.5285113558975427e-07, "logits/chosen": -0.8975743055343628, "logits/rejected": -0.8688497543334961, "logps/chosen": -478.6456604003906, "logps/ref_chosen": -270.8228759765625, "logps/ref_rejected": -255.30972290039062, "logps/rejected": -578.1317138671875, "loss": 4.5028, "margin_dpo/margin_mean": 114.99917602539062, "margin_dpo/margin_std": 158.4157257080078, "step": 318 }, { "KL/chosen_KL_mean": -224.45407104492188, "KL/mean": -277.3719787597656, "KL/rejected_KL_mean": -330.28985595703125, "KL/std": 138.86293029785156, "epoch": 0.6680628272251309, "fcm_dpo/beta": 0.0030600870959460735, "fcm_dpo/delta": 0.018474522978067398, "fcm_dpo/margin": 105.83576202392578, "fcm_dpo/q_t": 0.4230358898639679, "grad_norm": 106.81951141357422, "learning_rate": 1.5116687323334464e-07, "logits/chosen": -0.8801316022872925, "logits/rejected": -0.8496555685997009, "logps/chosen": -525.4569091796875, "logps/ref_chosen": -301.0028076171875, "logps/ref_rejected": -242.39002990722656, "logps/rejected": -572.6798706054688, "loss": 4.5445, "margin_dpo/margin_mean": 105.83576202392578, "margin_dpo/margin_std": 139.6443328857422, "step": 319 }, { "KL/chosen_KL_mean": -224.6094512939453, "KL/mean": -275.99322509765625, "KL/rejected_KL_mean": -327.37701416015625, "KL/std": 150.7802734375, "epoch": 0.6701570680628273, "fcm_dpo/beta": 0.0030319918878376484, "fcm_dpo/delta": -0.024170946329832077, "fcm_dpo/margin": 102.76758575439453, "fcm_dpo/q_t": 0.4289899170398712, "grad_norm": 80.37665557861328, "learning_rate": 1.4948791099758052e-07, "logits/chosen": -0.8487673401832581, "logits/rejected": -0.8394469618797302, "logps/chosen": -528.2320556640625, "logps/ref_chosen": -303.6225891113281, "logps/ref_rejected": -280.85174560546875, "logps/rejected": -608.228759765625, "loss": 4.6669, "margin_dpo/margin_mean": 102.76758575439453, "margin_dpo/margin_std": 172.9017333984375, "step": 320 }, { "KL/chosen_KL_mean": -247.0533447265625, "KL/mean": -284.8410339355469, "KL/rejected_KL_mean": -322.62872314453125, "KL/std": 149.05075073242188, "epoch": 0.6722513089005235, "fcm_dpo/beta": 0.0030333874747157097, "fcm_dpo/delta": 0.012165244668722153, "fcm_dpo/margin": 75.57539367675781, "fcm_dpo/q_t": 0.44790154695510864, "grad_norm": 91.25624084472656, "learning_rate": 1.478143389201113e-07, "logits/chosen": -0.8742708563804626, "logits/rejected": -0.8403862714767456, "logps/chosen": -536.0391845703125, "logps/ref_chosen": -288.98583984375, "logps/ref_rejected": -241.1822052001953, "logps/rejected": -563.8109741210938, "loss": 4.9434, "margin_dpo/margin_mean": 75.57539367675781, "margin_dpo/margin_std": 165.88363647460938, "step": 321 }, { "KL/chosen_KL_mean": -255.3909912109375, "KL/mean": -299.4102783203125, "KL/rejected_KL_mean": -343.4295349121094, "KL/std": 151.14059448242188, "epoch": 0.6743455497382199, "fcm_dpo/beta": 0.003069436876103282, "fcm_dpo/delta": 0.02577303536236286, "fcm_dpo/margin": 88.03853607177734, "fcm_dpo/q_t": 0.43747204542160034, "grad_norm": 70.48045349121094, "learning_rate": 1.461462467495284e-07, "logits/chosen": -0.9231587052345276, "logits/rejected": -0.8860714435577393, "logps/chosen": -563.9344482421875, "logps/ref_chosen": -308.54345703125, "logps/ref_rejected": -269.7995910644531, "logps/rejected": -613.2291259765625, "loss": 4.8283, "margin_dpo/margin_mean": 88.03853607177734, "margin_dpo/margin_std": 176.32192993164062, "step": 322 }, { "KL/chosen_KL_mean": -243.8632354736328, "KL/mean": -279.10198974609375, "KL/rejected_KL_mean": -314.3406677246094, "KL/std": 137.9376983642578, "epoch": 0.6764397905759162, "fcm_dpo/beta": 0.003103929106146097, "fcm_dpo/delta": -0.00034204451367259026, "fcm_dpo/margin": 70.47740936279297, "fcm_dpo/q_t": 0.4506058394908905, "grad_norm": 71.5815200805664, "learning_rate": 1.4448372394055246e-07, "logits/chosen": -0.8818837404251099, "logits/rejected": -0.8731890320777893, "logps/chosen": -526.3568725585938, "logps/ref_chosen": -282.49365234375, "logps/ref_rejected": -227.7105255126953, "logps/rejected": -542.0511474609375, "loss": 5.0069, "margin_dpo/margin_mean": 70.4774169921875, "margin_dpo/margin_std": 171.1042022705078, "step": 323 }, { "KL/chosen_KL_mean": -218.2962188720703, "KL/mean": -277.8293151855469, "KL/rejected_KL_mean": -337.3623962402344, "KL/std": 150.20140075683594, "epoch": 0.6785340314136126, "fcm_dpo/beta": 0.0030384839046746492, "fcm_dpo/delta": -0.01760367676615715, "fcm_dpo/margin": 119.06621551513672, "fcm_dpo/q_t": 0.41548827290534973, "grad_norm": 79.55120086669922, "learning_rate": 1.428268596492364e-07, "logits/chosen": -0.8202590346336365, "logits/rejected": -0.8138710260391235, "logps/chosen": -457.63458251953125, "logps/ref_chosen": -239.33836364746094, "logps/ref_rejected": -230.53775024414062, "logps/rejected": -567.900146484375, "loss": 4.4636, "margin_dpo/margin_mean": 119.06622314453125, "margin_dpo/margin_std": 155.9190673828125, "step": 324 }, { "KL/chosen_KL_mean": -250.08969116210938, "KL/mean": -300.6783142089844, "KL/rejected_KL_mean": -351.26702880859375, "KL/std": 157.67953491210938, "epoch": 0.680628272251309, "fcm_dpo/beta": 0.003035879461094737, "fcm_dpo/delta": -0.01274973526597023, "fcm_dpo/margin": 101.17728424072266, "fcm_dpo/q_t": 0.4310661554336548, "grad_norm": 85.70771789550781, "learning_rate": 1.4117574272818386e-07, "logits/chosen": -0.826617956161499, "logits/rejected": -0.8100321888923645, "logps/chosen": -530.7186889648438, "logps/ref_chosen": -280.62896728515625, "logps/ref_rejected": -270.5085754394531, "logps/rejected": -621.7755737304688, "loss": 4.7572, "margin_dpo/margin_mean": 101.17729187011719, "margin_dpo/margin_std": 193.80123901367188, "step": 325 }, { "KL/chosen_KL_mean": -235.35711669921875, "KL/mean": -287.36279296875, "KL/rejected_KL_mean": -339.36846923828125, "KL/std": 148.6804656982422, "epoch": 0.6827225130890052, "fcm_dpo/beta": 0.0030440862756222486, "fcm_dpo/delta": -0.008422894403338432, "fcm_dpo/margin": 104.01131439208984, "fcm_dpo/q_t": 0.42731618881225586, "grad_norm": 72.86016082763672, "learning_rate": 1.3953046172178413e-07, "logits/chosen": -0.9463378190994263, "logits/rejected": -0.9375435709953308, "logps/chosen": -476.3442687988281, "logps/ref_chosen": -240.9871368408203, "logps/ref_rejected": -261.0238342285156, "logps/rejected": -600.3922729492188, "loss": 4.6561, "margin_dpo/margin_mean": 104.01131439208984, "margin_dpo/margin_std": 171.9400634765625, "step": 326 }, { "KL/chosen_KL_mean": -227.19876098632812, "KL/mean": -282.76019287109375, "KL/rejected_KL_mean": -338.32159423828125, "KL/std": 140.7682342529297, "epoch": 0.6848167539267016, "fcm_dpo/beta": 0.002978120930492878, "fcm_dpo/delta": -0.02210163325071335, "fcm_dpo/margin": 111.1228256225586, "fcm_dpo/q_t": 0.4241907000541687, "grad_norm": 72.43849182128906, "learning_rate": 1.3789110486146468e-07, "logits/chosen": -0.8795799016952515, "logits/rejected": -0.8552607893943787, "logps/chosen": -506.71881103515625, "logps/ref_chosen": -279.52001953125, "logps/ref_rejected": -269.51824951171875, "logps/rejected": -607.8397827148438, "loss": 4.5967, "margin_dpo/margin_mean": 111.12281799316406, "margin_dpo/margin_std": 169.77825927734375, "step": 327 }, { "KL/chosen_KL_mean": -247.49966430664062, "KL/mean": -300.46051025390625, "KL/rejected_KL_mean": -353.42138671875, "KL/std": 146.58609008789062, "epoch": 0.6869109947643979, "fcm_dpo/beta": 0.002920475322753191, "fcm_dpo/delta": -0.01834581233561039, "fcm_dpo/margin": 105.92172241210938, "fcm_dpo/q_t": 0.4280359447002411, "grad_norm": 65.31753540039062, "learning_rate": 1.362577600609588e-07, "logits/chosen": -0.8584927320480347, "logits/rejected": -0.8567357063293457, "logps/chosen": -548.5330810546875, "logps/ref_chosen": -301.033447265625, "logps/ref_rejected": -284.2101135253906, "logps/rejected": -637.6315307617188, "loss": 4.6345, "margin_dpo/margin_mean": 105.92172241210938, "margin_dpo/margin_std": 160.9266357421875, "step": 328 }, { "KL/chosen_KL_mean": -265.75604248046875, "KL/mean": -323.16357421875, "KL/rejected_KL_mean": -380.5710754394531, "KL/std": 184.89715576171875, "epoch": 0.6890052356020943, "fcm_dpo/beta": 0.002896467689424753, "fcm_dpo/delta": -0.018565185368061066, "fcm_dpo/margin": 114.81501770019531, "fcm_dpo/q_t": 0.42783403396606445, "grad_norm": 60.461456298828125, "learning_rate": 1.3463051491159093e-07, "logits/chosen": -0.8645045161247253, "logits/rejected": -0.8325575590133667, "logps/chosen": -585.744873046875, "logps/ref_chosen": -319.9888610839844, "logps/ref_rejected": -307.5588684082031, "logps/rejected": -688.1299438476562, "loss": 4.6877, "margin_dpo/margin_mean": 114.81501770019531, "margin_dpo/margin_std": 206.36744689941406, "step": 329 }, { "KL/chosen_KL_mean": -262.357177734375, "KL/mean": -316.2622985839844, "KL/rejected_KL_mean": -370.1673583984375, "KL/std": 162.6302490234375, "epoch": 0.6910994764397905, "fcm_dpo/beta": 0.002852683886885643, "fcm_dpo/delta": -0.024464130401611328, "fcm_dpo/margin": 107.81021118164062, "fcm_dpo/q_t": 0.42945441603660583, "grad_norm": 61.41120529174805, "learning_rate": 1.3300945667758012e-07, "logits/chosen": -0.8586999177932739, "logits/rejected": -0.8646368384361267, "logps/chosen": -563.471923828125, "logps/ref_chosen": -301.11474609375, "logps/ref_rejected": -299.673095703125, "logps/rejected": -669.8404541015625, "loss": 4.6601, "margin_dpo/margin_mean": 107.8102035522461, "margin_dpo/margin_std": 173.99789428710938, "step": 330 }, { "KL/chosen_KL_mean": -256.2668151855469, "KL/mean": -304.8879089355469, "KL/rejected_KL_mean": -353.509033203125, "KL/std": 162.7855987548828, "epoch": 0.6931937172774869, "fcm_dpo/beta": 0.002842208370566368, "fcm_dpo/delta": 0.018526801839470863, "fcm_dpo/margin": 97.24219512939453, "fcm_dpo/q_t": 0.4376268982887268, "grad_norm": 100.0017318725586, "learning_rate": 1.3139467229135998e-07, "logits/chosen": -0.8795989155769348, "logits/rejected": -0.8588756322860718, "logps/chosen": -533.8583374023438, "logps/ref_chosen": -277.59149169921875, "logps/ref_rejected": -256.025634765625, "logps/rejected": -609.53466796875, "loss": 4.8138, "margin_dpo/margin_mean": 97.24219512939453, "margin_dpo/margin_std": 195.071044921875, "step": 331 }, { "KL/chosen_KL_mean": -267.27197265625, "KL/mean": -321.0538024902344, "KL/rejected_KL_mean": -374.8355712890625, "KL/std": 159.4349365234375, "epoch": 0.6952879581151833, "fcm_dpo/beta": 0.0028672670014202595, "fcm_dpo/delta": 0.008170833811163902, "fcm_dpo/margin": 107.56364440917969, "fcm_dpo/q_t": 0.429614782333374, "grad_norm": 68.8551025390625, "learning_rate": 1.2978624834891626e-07, "logits/chosen": -0.8785598874092102, "logits/rejected": -0.8509567379951477, "logps/chosen": -537.2456665039062, "logps/ref_chosen": -269.97369384765625, "logps/ref_rejected": -235.03164672851562, "logps/rejected": -609.8673095703125, "loss": 4.7519, "margin_dpo/margin_mean": 107.56364440917969, "margin_dpo/margin_std": 203.162109375, "step": 332 }, { "KL/chosen_KL_mean": -265.45159912109375, "KL/mean": -310.3097229003906, "KL/rejected_KL_mean": -355.1678466796875, "KL/std": 151.80691528320312, "epoch": 0.6973821989528796, "fcm_dpo/beta": 0.0028877186123281717, "fcm_dpo/delta": 0.016247637569904327, "fcm_dpo/margin": 89.71623992919922, "fcm_dpo/q_t": 0.44060900807380676, "grad_norm": 65.02811431884766, "learning_rate": 1.281842711051438e-07, "logits/chosen": -0.9548963308334351, "logits/rejected": -0.9161214232444763, "logps/chosen": -562.214599609375, "logps/ref_chosen": -296.76300048828125, "logps/ref_rejected": -265.97991943359375, "logps/rejected": -621.1477661132812, "loss": 4.8365, "margin_dpo/margin_mean": 89.71623229980469, "margin_dpo/margin_std": 177.90646362304688, "step": 333 }, { "KL/chosen_KL_mean": -252.63279724121094, "KL/mean": -302.4210205078125, "KL/rejected_KL_mean": -352.20928955078125, "KL/std": 154.0194854736328, "epoch": 0.6994764397905759, "fcm_dpo/beta": 0.002904269378632307, "fcm_dpo/delta": -0.021205034106969833, "fcm_dpo/margin": 99.57649993896484, "fcm_dpo/q_t": 0.4326726198196411, "grad_norm": 75.01168823242188, "learning_rate": 1.2658882646922033e-07, "logits/chosen": -0.8493959903717041, "logits/rejected": -0.8207319378852844, "logps/chosen": -553.6695556640625, "logps/ref_chosen": -301.0367431640625, "logps/ref_rejected": -268.87652587890625, "logps/rejected": -621.0858154296875, "loss": 4.7444, "margin_dpo/margin_mean": 99.57649993896484, "margin_dpo/margin_std": 173.79196166992188, "step": 334 }, { "KL/chosen_KL_mean": -280.1068115234375, "KL/mean": -328.50701904296875, "KL/rejected_KL_mean": -376.90716552734375, "KL/std": 143.94677734375, "epoch": 0.7015706806282722, "fcm_dpo/beta": 0.002826720243319869, "fcm_dpo/delta": -0.024722017347812653, "fcm_dpo/margin": 96.80036926269531, "fcm_dpo/q_t": 0.43696001172065735, "grad_norm": 82.47087860107422, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.8456929922103882, "logits/rejected": -0.8398817777633667, "logps/chosen": -556.239501953125, "logps/ref_chosen": -276.13275146484375, "logps/ref_rejected": -243.44203186035156, "logps/rejected": -620.3491821289062, "loss": 4.7968, "margin_dpo/margin_mean": 96.80036926269531, "margin_dpo/margin_std": 181.85284423828125, "step": 335 }, { "KL/chosen_KL_mean": -272.6168212890625, "KL/mean": -323.9999694824219, "KL/rejected_KL_mean": -375.383056640625, "KL/std": 182.46585083007812, "epoch": 0.7036649214659686, "fcm_dpo/beta": 0.0028306483291089535, "fcm_dpo/delta": 0.011404670774936676, "fcm_dpo/margin": 102.76631164550781, "fcm_dpo/q_t": 0.4366564452648163, "grad_norm": 60.21648025512695, "learning_rate": 1.2341787690142435e-07, "logits/chosen": -0.8676019906997681, "logits/rejected": -0.8084288835525513, "logps/chosen": -518.8794555664062, "logps/ref_chosen": -246.2626495361328, "logps/ref_rejected": -261.0617980957031, "logps/rejected": -636.4448852539062, "loss": 4.791, "margin_dpo/margin_mean": 102.76629638671875, "margin_dpo/margin_std": 199.2318115234375, "step": 336 }, { "KL/chosen_KL_mean": -257.2256164550781, "KL/mean": -313.9550476074219, "KL/rejected_KL_mean": -370.6844177246094, "KL/std": 151.99803161621094, "epoch": 0.7057591623036649, "fcm_dpo/beta": 0.0028662250842899084, "fcm_dpo/delta": -0.0013907505199313164, "fcm_dpo/margin": 113.45879364013672, "fcm_dpo/q_t": 0.42453762888908386, "grad_norm": 79.62832641601562, "learning_rate": 1.2184254201795363e-07, "logits/chosen": -0.8662607669830322, "logits/rejected": -0.8372821807861328, "logps/chosen": -524.2194213867188, "logps/ref_chosen": -266.9937744140625, "logps/ref_rejected": -253.015625, "logps/rejected": -623.7000732421875, "loss": 4.599, "margin_dpo/margin_mean": 113.45879364013672, "margin_dpo/margin_std": 171.08424377441406, "step": 337 }, { "KL/chosen_KL_mean": -248.36929321289062, "KL/mean": -305.20367431640625, "KL/rejected_KL_mean": -362.03802490234375, "KL/std": 163.98251342773438, "epoch": 0.7078534031413612, "fcm_dpo/beta": 0.0028280254919081926, "fcm_dpo/delta": 0.005961664952337742, "fcm_dpo/margin": 113.66871643066406, "fcm_dpo/q_t": 0.4267999529838562, "grad_norm": 75.13379669189453, "learning_rate": 1.202740798300168e-07, "logits/chosen": -0.8829590678215027, "logits/rejected": -0.86571204662323, "logps/chosen": -524.9619140625, "logps/ref_chosen": -276.5925598144531, "logps/ref_rejected": -233.979248046875, "logps/rejected": -596.0172119140625, "loss": 4.6272, "margin_dpo/margin_mean": 113.66871643066406, "margin_dpo/margin_std": 182.07052612304688, "step": 338 }, { "KL/chosen_KL_mean": -256.4414978027344, "KL/mean": -320.28216552734375, "KL/rejected_KL_mean": -384.1228332519531, "KL/std": 166.31675720214844, "epoch": 0.7099476439790576, "fcm_dpo/beta": 0.002872914308682084, "fcm_dpo/delta": -0.01606258563697338, "fcm_dpo/margin": 127.6812973022461, "fcm_dpo/q_t": 0.4165673553943634, "grad_norm": 53.9896125793457, "learning_rate": 1.1871257444948096e-07, "logits/chosen": -0.9026840925216675, "logits/rejected": -0.88820880651474, "logps/chosen": -559.96923828125, "logps/ref_chosen": -303.5277404785156, "logps/ref_rejected": -283.11676025390625, "logps/rejected": -667.239501953125, "loss": 4.5267, "margin_dpo/margin_mean": 127.68128967285156, "margin_dpo/margin_std": 196.2003173828125, "step": 339 }, { "KL/chosen_KL_mean": -263.4041748046875, "KL/mean": -310.89080810546875, "KL/rejected_KL_mean": -358.37738037109375, "KL/std": 174.05343627929688, "epoch": 0.7120418848167539, "fcm_dpo/beta": 0.002833343343809247, "fcm_dpo/delta": 0.010575986467301846, "fcm_dpo/margin": 94.97323608398438, "fcm_dpo/q_t": 0.4373157024383545, "grad_norm": 76.64401245117188, "learning_rate": 1.1715810961514072e-07, "logits/chosen": -0.83310467004776, "logits/rejected": -0.8270291090011597, "logps/chosen": -524.929931640625, "logps/ref_chosen": -261.5257568359375, "logps/ref_rejected": -259.39862060546875, "logps/rejected": -617.7760009765625, "loss": 4.9296, "margin_dpo/margin_mean": 94.97323608398438, "margin_dpo/margin_std": 215.53196716308594, "step": 340 }, { "KL/chosen_KL_mean": -300.6253662109375, "KL/mean": -347.34564208984375, "KL/rejected_KL_mean": -394.0658264160156, "KL/std": 195.77174377441406, "epoch": 0.7141361256544503, "fcm_dpo/beta": 0.002833633916452527, "fcm_dpo/delta": -0.014535056427121162, "fcm_dpo/margin": 93.44046020507812, "fcm_dpo/q_t": 0.4412190616130829, "grad_norm": 72.96549987792969, "learning_rate": 1.1561076868822755e-07, "logits/chosen": -0.8564797043800354, "logits/rejected": -0.8294931650161743, "logps/chosen": -616.5289306640625, "logps/ref_chosen": -315.903564453125, "logps/ref_rejected": -308.02392578125, "logps/rejected": -702.0897827148438, "loss": 4.9579, "margin_dpo/margin_mean": 93.44046020507812, "margin_dpo/margin_std": 222.5795440673828, "step": 341 }, { "KL/chosen_KL_mean": -252.14891052246094, "KL/mean": -301.0910949707031, "KL/rejected_KL_mean": -350.0333251953125, "KL/std": 153.94277954101562, "epoch": 0.7162303664921466, "fcm_dpo/beta": 0.002802258124575019, "fcm_dpo/delta": -0.010795130394399166, "fcm_dpo/margin": 97.88438415527344, "fcm_dpo/q_t": 0.4367576539516449, "grad_norm": 81.44354248046875, "learning_rate": 1.1407063464793965e-07, "logits/chosen": -0.8587002754211426, "logits/rejected": -0.8541080355644226, "logps/chosen": -521.3275756835938, "logps/ref_chosen": -269.17864990234375, "logps/ref_rejected": -260.8977355957031, "logps/rejected": -610.9310913085938, "loss": 4.7627, "margin_dpo/margin_mean": 97.88438415527344, "margin_dpo/margin_std": 172.59506225585938, "step": 342 }, { "KL/chosen_KL_mean": -263.67144775390625, "KL/mean": -303.8117370605469, "KL/rejected_KL_mean": -343.95196533203125, "KL/std": 153.24267578125, "epoch": 0.7183246073298429, "fcm_dpo/beta": 0.0027811508625745773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.28048706054688, "fcm_dpo/q_t": 0.4475141167640686, "grad_norm": 61.77683639526367, "learning_rate": 1.125377900869913e-07, "logits/chosen": -0.8490421772003174, "logits/rejected": -0.8213639259338379, "logps/chosen": -574.3914794921875, "logps/ref_chosen": -310.719970703125, "logps/ref_rejected": -263.5224914550781, "logps/rejected": -607.4744873046875, "loss": 4.9068, "margin_dpo/margin_mean": 80.2804946899414, "margin_dpo/margin_std": 163.10105895996094, "step": 343 }, { "KL/chosen_KL_mean": -261.09722900390625, "KL/mean": -305.764404296875, "KL/rejected_KL_mean": -350.4315490722656, "KL/std": 147.62881469726562, "epoch": 0.7204188481675393, "fcm_dpo/beta": 0.0027887485921382904, "fcm_dpo/delta": 0.008171796798706055, "fcm_dpo/margin": 89.3343505859375, "fcm_dpo/q_t": 0.44169238209724426, "grad_norm": 75.60264587402344, "learning_rate": 1.110123172071844e-07, "logits/chosen": -0.8601065874099731, "logits/rejected": -0.8441787958145142, "logps/chosen": -562.8970947265625, "logps/ref_chosen": -301.7999267578125, "logps/ref_rejected": -257.9061584472656, "logps/rejected": -608.3377075195312, "loss": 4.8542, "margin_dpo/margin_mean": 89.33435821533203, "margin_dpo/margin_std": 173.32949829101562, "step": 344 }, { "KL/chosen_KL_mean": -239.99986267089844, "KL/mean": -284.1513366699219, "KL/rejected_KL_mean": -328.30279541015625, "KL/std": 147.23294067382812, "epoch": 0.7225130890052356, "fcm_dpo/beta": 0.002783895004540682, "fcm_dpo/delta": -0.00986623577773571, "fcm_dpo/margin": 88.30291748046875, "fcm_dpo/q_t": 0.4421829581260681, "grad_norm": 68.73944091796875, "learning_rate": 1.09494297815e-07, "logits/chosen": -0.8500536680221558, "logits/rejected": -0.8508837223052979, "logps/chosen": -523.018310546875, "logps/ref_chosen": -283.0184326171875, "logps/ref_rejected": -266.8457336425781, "logps/rejected": -595.1485595703125, "loss": 4.8153, "margin_dpo/margin_mean": 88.30291748046875, "margin_dpo/margin_std": 145.37179565429688, "step": 345 }, { "KL/chosen_KL_mean": -255.714111328125, "KL/mean": -312.4100036621094, "KL/rejected_KL_mean": -369.10595703125, "KL/std": 172.77389526367188, "epoch": 0.724607329842932, "fcm_dpo/beta": 0.0027374359779059887, "fcm_dpo/delta": -0.02758317068219185, "fcm_dpo/margin": 113.39186096191406, "fcm_dpo/q_t": 0.42870837450027466, "grad_norm": 54.13254928588867, "learning_rate": 1.0798381331721107e-07, "logits/chosen": -0.936150848865509, "logits/rejected": -0.8845096230506897, "logps/chosen": -524.1553344726562, "logps/ref_chosen": -268.44122314453125, "logps/ref_rejected": -227.8225860595703, "logps/rejected": -596.9285278320312, "loss": 4.6752, "margin_dpo/margin_mean": 113.3918685913086, "margin_dpo/margin_std": 180.54847717285156, "step": 346 }, { "KL/chosen_KL_mean": -238.4910125732422, "KL/mean": -283.69293212890625, "KL/rejected_KL_mean": -328.8948669433594, "KL/std": 144.58145141601562, "epoch": 0.7267015706806282, "fcm_dpo/beta": 0.0027357658836990595, "fcm_dpo/delta": 0.012428262270987034, "fcm_dpo/margin": 90.40386199951172, "fcm_dpo/q_t": 0.44158247113227844, "grad_norm": 50.78943634033203, "learning_rate": 1.0648094471651722e-07, "logits/chosen": -0.783841609954834, "logits/rejected": -0.8095725178718567, "logps/chosen": -512.194580078125, "logps/ref_chosen": -273.70355224609375, "logps/ref_rejected": -243.65521240234375, "logps/rejected": -572.550048828125, "loss": 4.8291, "margin_dpo/margin_mean": 90.40386199951172, "margin_dpo/margin_std": 160.6577911376953, "step": 347 }, { "KL/chosen_KL_mean": -237.40325927734375, "KL/mean": -279.7298583984375, "KL/rejected_KL_mean": -322.0564880371094, "KL/std": 156.852783203125, "epoch": 0.7287958115183246, "fcm_dpo/beta": 0.0027791352476924658, "fcm_dpo/delta": 0.022717544808983803, "fcm_dpo/margin": 84.6531982421875, "fcm_dpo/q_t": 0.4474527835845947, "grad_norm": 81.65412139892578, "learning_rate": 1.0498577260720048e-07, "logits/chosen": -0.8968985676765442, "logits/rejected": -0.8811699748039246, "logps/chosen": -523.044677734375, "logps/ref_chosen": -285.64141845703125, "logps/ref_rejected": -265.6270446777344, "logps/rejected": -587.6835327148438, "loss": 4.9125, "margin_dpo/margin_mean": 84.6531982421875, "margin_dpo/margin_std": 180.24642944335938, "step": 348 }, { "KL/chosen_KL_mean": -240.90472412109375, "KL/mean": -305.62078857421875, "KL/rejected_KL_mean": -370.3368225097656, "KL/std": 170.03091430664062, "epoch": 0.7308900523560209, "fcm_dpo/beta": 0.002779011381790042, "fcm_dpo/delta": -0.02953748032450676, "fcm_dpo/margin": 129.43206787109375, "fcm_dpo/q_t": 0.4158233404159546, "grad_norm": 68.60745239257812, "learning_rate": 1.0349837717080347e-07, "logits/chosen": -0.8537794351577759, "logits/rejected": -0.8392589092254639, "logps/chosen": -569.2222290039062, "logps/ref_chosen": -328.3175048828125, "logps/ref_rejected": -292.37872314453125, "logps/rejected": -662.715576171875, "loss": 4.5281, "margin_dpo/margin_mean": 129.43206787109375, "margin_dpo/margin_std": 191.28211975097656, "step": 349 }, { "KL/chosen_KL_mean": -255.6253662109375, "KL/mean": -307.3200378417969, "KL/rejected_KL_mean": -359.0146179199219, "KL/std": 162.4384002685547, "epoch": 0.7329842931937173, "fcm_dpo/beta": 0.0027364445850253105, "fcm_dpo/delta": 0.014540414325892925, "fcm_dpo/margin": 103.38926696777344, "fcm_dpo/q_t": 0.4352550506591797, "grad_norm": 70.40982818603516, "learning_rate": 1.0201883817182949e-07, "logits/chosen": -0.8301476240158081, "logits/rejected": -0.8409773111343384, "logps/chosen": -548.4300537109375, "logps/ref_chosen": -292.8046569824219, "logps/ref_rejected": -250.35504150390625, "logps/rejected": -609.3696899414062, "loss": 4.7581, "margin_dpo/margin_mean": 103.38926696777344, "margin_dpo/margin_std": 189.09571838378906, "step": 350 }, { "KL/chosen_KL_mean": -248.666748046875, "KL/mean": -277.6920166015625, "KL/rejected_KL_mean": -306.71728515625, "KL/std": 148.222900390625, "epoch": 0.7350785340314137, "fcm_dpo/beta": 0.002761303447186947, "fcm_dpo/delta": 0.001408736570738256, "fcm_dpo/margin": 58.05055618286133, "fcm_dpo/q_t": 0.4626092314720154, "grad_norm": 71.92450714111328, "learning_rate": 1.0054723495346482e-07, "logits/chosen": -0.9074581861495972, "logits/rejected": -0.887728214263916, "logps/chosen": -560.5557861328125, "logps/ref_chosen": -311.8890380859375, "logps/ref_rejected": -263.59033203125, "logps/rejected": -570.3076171875, "loss": 5.1991, "margin_dpo/margin_mean": 58.05055618286133, "margin_dpo/margin_std": 185.35238647460938, "step": 351 }, { "KL/chosen_KL_mean": -232.7962188720703, "KL/mean": -293.2824401855469, "KL/rejected_KL_mean": -353.7686767578125, "KL/std": 162.10598754882812, "epoch": 0.7371727748691099, "fcm_dpo/beta": 0.0027385905850678682, "fcm_dpo/delta": -0.024517323821783066, "fcm_dpo/margin": 120.97250366210938, "fcm_dpo/q_t": 0.4226635694503784, "grad_norm": 84.17012023925781, "learning_rate": 9.908364643332398e-08, "logits/chosen": -0.8231208324432373, "logits/rejected": -0.7927186489105225, "logps/chosen": -487.7040710449219, "logps/ref_chosen": -254.9078826904297, "logps/ref_rejected": -257.1688232421875, "logps/rejected": -610.9375, "loss": 4.5778, "margin_dpo/margin_mean": 120.97249603271484, "margin_dpo/margin_std": 177.3806610107422, "step": 352 }, { "KL/chosen_KL_mean": -249.73309326171875, "KL/mean": -302.8935241699219, "KL/rejected_KL_mean": -356.053955078125, "KL/std": 158.14662170410156, "epoch": 0.7392670157068063, "fcm_dpo/beta": 0.0027025165036320686, "fcm_dpo/delta": -0.016819603741168976, "fcm_dpo/margin": 106.32086181640625, "fcm_dpo/q_t": 0.4334595799446106, "grad_norm": 64.48303985595703, "learning_rate": 9.76281510992176e-08, "logits/chosen": -0.8482782244682312, "logits/rejected": -0.8419215083122253, "logps/chosen": -520.109130859375, "logps/ref_chosen": -270.3760681152344, "logps/ref_rejected": -264.65234375, "logps/rejected": -620.706298828125, "loss": 4.7294, "margin_dpo/margin_mean": 106.32086181640625, "margin_dpo/margin_std": 179.3717041015625, "step": 353 }, { "KL/chosen_KL_mean": -246.82254028320312, "KL/mean": -288.2264099121094, "KL/rejected_KL_mean": -329.6302795410156, "KL/std": 169.2479705810547, "epoch": 0.7413612565445026, "fcm_dpo/beta": 0.0026852269656956196, "fcm_dpo/delta": 0.00828171893954277, "fcm_dpo/margin": 82.80773162841797, "fcm_dpo/q_t": 0.4493102431297302, "grad_norm": 70.67339324951172, "learning_rate": 9.618082700494318e-08, "logits/chosen": -0.8379197716712952, "logits/rejected": -0.8698081970214844, "logps/chosen": -504.4710998535156, "logps/ref_chosen": -257.6485595703125, "logps/ref_rejected": -246.94203186035156, "logps/rejected": -576.5723266601562, "loss": 4.9535, "margin_dpo/margin_mean": 82.8077392578125, "margin_dpo/margin_std": 183.56936645507812, "step": 354 }, { "KL/chosen_KL_mean": -248.17721557617188, "KL/mean": -303.8741455078125, "KL/rejected_KL_mean": -359.571044921875, "KL/std": 174.03460693359375, "epoch": 0.743455497382199, "fcm_dpo/beta": 0.002684948965907097, "fcm_dpo/delta": -0.00029130373150110245, "fcm_dpo/margin": 111.39378356933594, "fcm_dpo/q_t": 0.4308469891548157, "grad_norm": 72.10034942626953, "learning_rate": 9.474175176609956e-08, "logits/chosen": -0.8968920111656189, "logits/rejected": -0.8910172581672668, "logps/chosen": -541.5305786132812, "logps/ref_chosen": -293.35333251953125, "logps/ref_rejected": -275.6051940917969, "logps/rejected": -635.1762084960938, "loss": 4.7794, "margin_dpo/margin_mean": 111.39378356933594, "margin_dpo/margin_std": 211.11050415039062, "step": 355 }, { "KL/chosen_KL_mean": -233.09683227539062, "KL/mean": -273.4365234375, "KL/rejected_KL_mean": -313.7762145996094, "KL/std": 145.1485595703125, "epoch": 0.7455497382198953, "fcm_dpo/beta": 0.002696634503081441, "fcm_dpo/delta": -0.0037465146742761135, "fcm_dpo/margin": 80.67941284179688, "fcm_dpo/q_t": 0.4487796127796173, "grad_norm": 62.512733459472656, "learning_rate": 9.331100255592436e-08, "logits/chosen": -0.8034874796867371, "logits/rejected": -0.8288117051124573, "logps/chosen": -437.35235595703125, "logps/ref_chosen": -204.25550842285156, "logps/ref_rejected": -213.467529296875, "logps/rejected": -527.2437744140625, "loss": 4.897, "margin_dpo/margin_mean": 80.67941284179688, "margin_dpo/margin_std": 147.65928649902344, "step": 356 }, { "KL/chosen_KL_mean": -260.0800476074219, "KL/mean": -313.5126647949219, "KL/rejected_KL_mean": -366.9453430175781, "KL/std": 166.6250762939453, "epoch": 0.7476439790575916, "fcm_dpo/beta": 0.002710919128730893, "fcm_dpo/delta": 0.02739933878183365, "fcm_dpo/margin": 106.86528015136719, "fcm_dpo/q_t": 0.4340450167655945, "grad_norm": 56.6240348815918, "learning_rate": 9.18886561011557e-08, "logits/chosen": -0.7838116884231567, "logits/rejected": -0.779089629650116, "logps/chosen": -526.4506225585938, "logps/ref_chosen": -266.3705749511719, "logps/ref_rejected": -239.04490661621094, "logps/rejected": -605.990234375, "loss": 4.7505, "margin_dpo/margin_mean": 106.86528015136719, "margin_dpo/margin_std": 189.58627319335938, "step": 357 }, { "KL/chosen_KL_mean": -231.25390625, "KL/mean": -294.31298828125, "KL/rejected_KL_mean": -357.3720703125, "KL/std": 153.28192138671875, "epoch": 0.749738219895288, "fcm_dpo/beta": 0.0027492064982652664, "fcm_dpo/delta": -0.007799042388796806, "fcm_dpo/margin": 126.11814880371094, "fcm_dpo/q_t": 0.419543981552124, "grad_norm": 46.20698547363281, "learning_rate": 9.047478867791731e-08, "logits/chosen": -0.8915876150131226, "logits/rejected": -0.8662262558937073, "logps/chosen": -530.4014282226562, "logps/ref_chosen": -299.1474609375, "logps/ref_rejected": -257.2531433105469, "logps/rejected": -614.625244140625, "loss": 4.5507, "margin_dpo/margin_mean": 126.11814880371094, "margin_dpo/margin_std": 187.35885620117188, "step": 358 }, { "KL/chosen_KL_mean": -250.6073455810547, "KL/mean": -302.3663024902344, "KL/rejected_KL_mean": -354.12530517578125, "KL/std": 158.33580017089844, "epoch": 0.7518324607329843, "fcm_dpo/beta": 0.002736097201704979, "fcm_dpo/delta": 0.006915869656950235, "fcm_dpo/margin": 103.51795959472656, "fcm_dpo/q_t": 0.4344536066055298, "grad_norm": 89.7514877319336, "learning_rate": 8.906947610762825e-08, "logits/chosen": -0.8484093546867371, "logits/rejected": -0.8544447422027588, "logps/chosen": -553.605224609375, "logps/ref_chosen": -302.99786376953125, "logps/ref_rejected": -260.4137268066406, "logps/rejected": -614.5390014648438, "loss": 4.7663, "margin_dpo/margin_mean": 103.51795959472656, "margin_dpo/margin_std": 189.6798553466797, "step": 359 }, { "KL/chosen_KL_mean": -255.82347106933594, "KL/mean": -305.854736328125, "KL/rejected_KL_mean": -355.885986328125, "KL/std": 177.97512817382812, "epoch": 0.7539267015706806, "fcm_dpo/beta": 0.0026915487833321095, "fcm_dpo/delta": -0.03379403427243233, "fcm_dpo/margin": 100.0625, "fcm_dpo/q_t": 0.43899649381637573, "grad_norm": 75.62610626220703, "learning_rate": 8.76727937529367e-08, "logits/chosen": -0.8806153535842896, "logits/rejected": -0.8590009808540344, "logps/chosen": -565.4349365234375, "logps/ref_chosen": -309.6114501953125, "logps/ref_rejected": -256.64031982421875, "logps/rejected": -612.5263061523438, "loss": 4.9075, "margin_dpo/margin_mean": 100.06248474121094, "margin_dpo/margin_std": 208.209716796875, "step": 360 }, { "KL/chosen_KL_mean": -218.59913635253906, "KL/mean": -284.7613830566406, "KL/rejected_KL_mean": -350.92364501953125, "KL/std": 172.47113037109375, "epoch": 0.7560209424083769, "fcm_dpo/beta": 0.0026670051738619804, "fcm_dpo/delta": -0.0043861037120223045, "fcm_dpo/margin": 132.32452392578125, "fcm_dpo/q_t": 0.42048534750938416, "grad_norm": 59.663902282714844, "learning_rate": 8.628481651367875e-08, "logits/chosen": -0.8187331557273865, "logits/rejected": -0.7988158464431763, "logps/chosen": -481.9788818359375, "logps/ref_chosen": -263.3797607421875, "logps/ref_rejected": -271.18157958984375, "logps/rejected": -622.105224609375, "loss": 4.5702, "margin_dpo/margin_mean": 132.3245086669922, "margin_dpo/margin_std": 212.18954467773438, "step": 361 }, { "KL/chosen_KL_mean": -246.6825408935547, "KL/mean": -296.739501953125, "KL/rejected_KL_mean": -346.7964782714844, "KL/std": 138.58175659179688, "epoch": 0.7581151832460733, "fcm_dpo/beta": 0.002679286291822791, "fcm_dpo/delta": 0.015072029083967209, "fcm_dpo/margin": 100.11392211914062, "fcm_dpo/q_t": 0.43785110116004944, "grad_norm": 77.70549774169922, "learning_rate": 8.490561882286135e-08, "logits/chosen": -0.8581762909889221, "logits/rejected": -0.8390923142433167, "logps/chosen": -549.94091796875, "logps/ref_chosen": -303.2583923339844, "logps/ref_rejected": -243.22891235351562, "logps/rejected": -590.0253295898438, "loss": 4.7371, "margin_dpo/margin_mean": 100.11392974853516, "margin_dpo/margin_std": 159.69705200195312, "step": 362 }, { "KL/chosen_KL_mean": -253.08914184570312, "KL/mean": -310.4933776855469, "KL/rejected_KL_mean": -367.8975830078125, "KL/std": 171.24012756347656, "epoch": 0.7602094240837697, "fcm_dpo/beta": 0.00266785966232419, "fcm_dpo/delta": -0.03768566623330116, "fcm_dpo/margin": 114.80848693847656, "fcm_dpo/q_t": 0.4307466149330139, "grad_norm": 73.5361099243164, "learning_rate": 8.353527464267104e-08, "logits/chosen": -0.8588560819625854, "logits/rejected": -0.8144607543945312, "logps/chosen": -556.4363403320312, "logps/ref_chosen": -303.34722900390625, "logps/ref_rejected": -262.05419921875, "logps/rejected": -629.9518432617188, "loss": 4.7016, "margin_dpo/margin_mean": 114.80850219726562, "margin_dpo/margin_std": 196.30714416503906, "step": 363 }, { "KL/chosen_KL_mean": -289.6898193359375, "KL/mean": -345.4079895019531, "KL/rejected_KL_mean": -401.1260986328125, "KL/std": 190.25381469726562, "epoch": 0.762303664921466, "fcm_dpo/beta": 0.0026328519452363253, "fcm_dpo/delta": 0.019132010638713837, "fcm_dpo/margin": 111.43631744384766, "fcm_dpo/q_t": 0.4358139634132385, "grad_norm": 59.363731384277344, "learning_rate": 8.217385746050742e-08, "logits/chosen": -0.8394103646278381, "logits/rejected": -0.8337091207504272, "logps/chosen": -575.2335815429688, "logps/ref_chosen": -285.54376220703125, "logps/ref_rejected": -284.84619140625, "logps/rejected": -685.9723510742188, "loss": 4.8079, "margin_dpo/margin_mean": 111.43632507324219, "margin_dpo/margin_std": 226.2152862548828, "step": 364 }, { "KL/chosen_KL_mean": -271.29132080078125, "KL/mean": -321.2420349121094, "KL/rejected_KL_mean": -371.1927185058594, "KL/std": 180.25765991210938, "epoch": 0.7643979057591623, "fcm_dpo/beta": 0.0026724659837782383, "fcm_dpo/delta": 0.018962977454066277, "fcm_dpo/margin": 99.90141296386719, "fcm_dpo/q_t": 0.4393257200717926, "grad_norm": 63.602535247802734, "learning_rate": 8.082144028504231e-08, "logits/chosen": -0.8441622257232666, "logits/rejected": -0.8413698077201843, "logps/chosen": -546.0791015625, "logps/ref_chosen": -274.7878112792969, "logps/ref_rejected": -256.5738220214844, "logps/rejected": -627.7665405273438, "loss": 4.8409, "margin_dpo/margin_mean": 99.90141296386719, "margin_dpo/margin_std": 199.26466369628906, "step": 365 }, { "KL/chosen_KL_mean": -253.7181396484375, "KL/mean": -306.461669921875, "KL/rejected_KL_mean": -359.2052307128906, "KL/std": 156.46954345703125, "epoch": 0.7664921465968586, "fcm_dpo/beta": 0.00269645219668746, "fcm_dpo/delta": 0.010923715308308601, "fcm_dpo/margin": 105.4870834350586, "fcm_dpo/q_t": 0.4338816702365875, "grad_norm": 58.37995910644531, "learning_rate": 7.947809564230445e-08, "logits/chosen": -0.8080151677131653, "logits/rejected": -0.8108749985694885, "logps/chosen": -540.3677368164062, "logps/ref_chosen": -286.6496276855469, "logps/ref_rejected": -251.97140502929688, "logps/rejected": -611.1766357421875, "loss": 4.7406, "margin_dpo/margin_mean": 105.48709106445312, "margin_dpo/margin_std": 187.03518676757812, "step": 366 }, { "KL/chosen_KL_mean": -259.25201416015625, "KL/mean": -319.07611083984375, "KL/rejected_KL_mean": -378.90032958984375, "KL/std": 164.353759765625, "epoch": 0.768586387434555, "fcm_dpo/beta": 0.0026909676380455494, "fcm_dpo/delta": -0.009831960313022137, "fcm_dpo/margin": 119.64830017089844, "fcm_dpo/q_t": 0.426592618227005, "grad_norm": 85.58404541015625, "learning_rate": 7.814389557179016e-08, "logits/chosen": -0.8485161066055298, "logits/rejected": -0.8188115954399109, "logps/chosen": -561.1969604492188, "logps/ref_chosen": -301.9449768066406, "logps/ref_rejected": -265.5677185058594, "logps/rejected": -644.468017578125, "loss": 4.6244, "margin_dpo/margin_mean": 119.64830017089844, "margin_dpo/margin_std": 188.09889221191406, "step": 367 }, { "KL/chosen_KL_mean": -246.48849487304688, "KL/mean": -317.7361145019531, "KL/rejected_KL_mean": -388.98382568359375, "KL/std": 162.717529296875, "epoch": 0.7706806282722513, "fcm_dpo/beta": 0.0026594342198222876, "fcm_dpo/delta": -0.0268840454518795, "fcm_dpo/margin": 142.4953155517578, "fcm_dpo/q_t": 0.41193145513534546, "grad_norm": 58.12229537963867, "learning_rate": 7.681891162260015e-08, "logits/chosen": -0.8149774074554443, "logits/rejected": -0.815480649471283, "logps/chosen": -541.1150512695312, "logps/ref_chosen": -294.62652587890625, "logps/ref_rejected": -258.7628479003906, "logps/rejected": -647.7466430664062, "loss": 4.3926, "margin_dpo/margin_mean": 142.4953155517578, "margin_dpo/margin_std": 172.4111785888672, "step": 368 }, { "KL/chosen_KL_mean": -265.4953918457031, "KL/mean": -314.00823974609375, "KL/rejected_KL_mean": -362.5211181640625, "KL/std": 154.16493225097656, "epoch": 0.7727748691099476, "fcm_dpo/beta": 0.002672000788152218, "fcm_dpo/delta": 0.02517055906355381, "fcm_dpo/margin": 97.02571105957031, "fcm_dpo/q_t": 0.4393949508666992, "grad_norm": 68.1326904296875, "learning_rate": 7.550321484960251e-08, "logits/chosen": -0.8995909094810486, "logits/rejected": -0.8832194209098816, "logps/chosen": -548.0011596679688, "logps/ref_chosen": -282.5057373046875, "logps/ref_rejected": -266.41607666015625, "logps/rejected": -628.9371948242188, "loss": 4.7696, "margin_dpo/margin_mean": 97.02571105957031, "margin_dpo/margin_std": 162.9827423095703, "step": 369 }, { "KL/chosen_KL_mean": -254.54122924804688, "KL/mean": -315.56475830078125, "KL/rejected_KL_mean": -376.5882873535156, "KL/std": 168.5694122314453, "epoch": 0.774869109947644, "fcm_dpo/beta": 0.002689562737941742, "fcm_dpo/delta": -0.00773870013654232, "fcm_dpo/margin": 122.04708099365234, "fcm_dpo/q_t": 0.42569032311439514, "grad_norm": 69.9754638671875, "learning_rate": 7.419687580962222e-08, "logits/chosen": -0.86855548620224, "logits/rejected": -0.8914240002632141, "logps/chosen": -505.5476379394531, "logps/ref_chosen": -251.00640869140625, "logps/ref_rejected": -238.12542724609375, "logps/rejected": -614.7136840820312, "loss": 4.6562, "margin_dpo/margin_mean": 122.04708099365234, "margin_dpo/margin_std": 212.94515991210938, "step": 370 }, { "KL/chosen_KL_mean": -287.0211181640625, "KL/mean": -323.5873107910156, "KL/rejected_KL_mean": -360.153564453125, "KL/std": 164.57492065429688, "epoch": 0.7769633507853403, "fcm_dpo/beta": 0.002693257061764598, "fcm_dpo/delta": 0.002763350959867239, "fcm_dpo/margin": 73.13240051269531, "fcm_dpo/q_t": 0.45326054096221924, "grad_norm": 97.06198120117188, "learning_rate": 7.289996455765748e-08, "logits/chosen": -0.8332430720329285, "logits/rejected": -0.8098735213279724, "logps/chosen": -583.6802978515625, "logps/ref_chosen": -296.6591491699219, "logps/ref_rejected": -251.14675903320312, "logps/rejected": -611.30029296875, "loss": 5.0419, "margin_dpo/margin_mean": 73.13240814208984, "margin_dpo/margin_std": 174.42578125, "step": 371 }, { "KL/chosen_KL_mean": -277.73712158203125, "KL/mean": -339.80938720703125, "KL/rejected_KL_mean": -401.88165283203125, "KL/std": 183.5233612060547, "epoch": 0.7790575916230367, "fcm_dpo/beta": 0.002633734606206417, "fcm_dpo/delta": -0.028473343700170517, "fcm_dpo/margin": 124.14448547363281, "fcm_dpo/q_t": 0.4261954724788666, "grad_norm": 73.80138397216797, "learning_rate": 7.161255064312283e-08, "logits/chosen": -0.8024683594703674, "logits/rejected": -0.7857841849327087, "logps/chosen": -609.1085815429688, "logps/ref_chosen": -331.3714599609375, "logps/ref_rejected": -285.56805419921875, "logps/rejected": -687.44970703125, "loss": 4.6629, "margin_dpo/margin_mean": 124.14448547363281, "margin_dpo/margin_std": 206.26226806640625, "step": 372 }, { "KL/chosen_KL_mean": -258.4107666015625, "KL/mean": -317.982421875, "KL/rejected_KL_mean": -377.5540771484375, "KL/std": 160.34129333496094, "epoch": 0.7811518324607329, "fcm_dpo/beta": 0.0026560514234006405, "fcm_dpo/delta": 0.02417863719165325, "fcm_dpo/margin": 119.14329528808594, "fcm_dpo/q_t": 0.42621058225631714, "grad_norm": 91.65182495117188, "learning_rate": 7.033470310611945e-08, "logits/chosen": -0.8905776143074036, "logits/rejected": -0.8529370427131653, "logps/chosen": -580.353759765625, "logps/ref_chosen": -321.9429931640625, "logps/ref_rejected": -271.2288513183594, "logps/rejected": -648.782958984375, "loss": 4.6392, "margin_dpo/margin_mean": 119.14329528808594, "margin_dpo/margin_std": 189.48521423339844, "step": 373 }, { "KL/chosen_KL_mean": -271.4115295410156, "KL/mean": -323.5093994140625, "KL/rejected_KL_mean": -375.6073303222656, "KL/std": 175.6970672607422, "epoch": 0.7832460732984293, "fcm_dpo/beta": 0.00263632507994771, "fcm_dpo/delta": -0.01818685419857502, "fcm_dpo/margin": 104.19580078125, "fcm_dpo/q_t": 0.4362541735172272, "grad_norm": 75.91381072998047, "learning_rate": 6.906649047373245e-08, "logits/chosen": -0.8902040719985962, "logits/rejected": -0.8820330500602722, "logps/chosen": -590.580078125, "logps/ref_chosen": -319.1685485839844, "logps/ref_rejected": -284.6263732910156, "logps/rejected": -660.233642578125, "loss": 4.7608, "margin_dpo/margin_mean": 104.19580078125, "margin_dpo/margin_std": 180.38076782226562, "step": 374 }, { "KL/chosen_KL_mean": -282.55670166015625, "KL/mean": -325.5635986328125, "KL/rejected_KL_mean": -368.570556640625, "KL/std": 148.60000610351562, "epoch": 0.7853403141361257, "fcm_dpo/beta": 0.002658943645656109, "fcm_dpo/delta": 0.01239805668592453, "fcm_dpo/margin": 86.01384735107422, "fcm_dpo/q_t": 0.4467378258705139, "grad_norm": 81.2287826538086, "learning_rate": 6.780798075635675e-08, "logits/chosen": -0.8825744986534119, "logits/rejected": -0.8607974052429199, "logps/chosen": -597.4324340820312, "logps/ref_chosen": -314.87579345703125, "logps/ref_rejected": -259.1965026855469, "logps/rejected": -627.7670288085938, "loss": 4.9062, "margin_dpo/margin_mean": 86.01384735107422, "margin_dpo/margin_std": 172.94680786132812, "step": 375 }, { "KL/chosen_KL_mean": -277.1960754394531, "KL/mean": -333.73248291015625, "KL/rejected_KL_mean": -390.2689208984375, "KL/std": 171.84317016601562, "epoch": 0.787434554973822, "fcm_dpo/beta": 0.0026825799141079187, "fcm_dpo/delta": 0.002811199054121971, "fcm_dpo/margin": 113.07284545898438, "fcm_dpo/q_t": 0.4311826527118683, "grad_norm": 76.82144165039062, "learning_rate": 6.655924144404906e-08, "logits/chosen": -0.8568066954612732, "logits/rejected": -0.85575270652771, "logps/chosen": -564.8693237304688, "logps/ref_chosen": -287.6732482910156, "logps/ref_rejected": -256.6697082519531, "logps/rejected": -646.9385986328125, "loss": 4.7243, "margin_dpo/margin_mean": 113.07284545898438, "margin_dpo/margin_std": 204.332763671875, "step": 376 }, { "KL/chosen_KL_mean": -292.6242370605469, "KL/mean": -330.6067199707031, "KL/rejected_KL_mean": -368.5892333984375, "KL/std": 170.7978973388672, "epoch": 0.7895287958115184, "fcm_dpo/beta": 0.002660011174157262, "fcm_dpo/delta": -0.003608726430684328, "fcm_dpo/margin": 75.96499633789062, "fcm_dpo/q_t": 0.45309317111968994, "grad_norm": 70.9164810180664, "learning_rate": 6.532033950290885e-08, "logits/chosen": -0.8469290137290955, "logits/rejected": -0.8385406136512756, "logps/chosen": -597.8856811523438, "logps/ref_chosen": -305.261474609375, "logps/ref_rejected": -271.8887023925781, "logps/rejected": -640.4779052734375, "loss": 5.0753, "margin_dpo/margin_mean": 75.96499633789062, "margin_dpo/margin_std": 200.70811462402344, "step": 377 }, { "KL/chosen_KL_mean": -277.7027893066406, "KL/mean": -330.92620849609375, "KL/rejected_KL_mean": -384.1496276855469, "KL/std": 179.26226806640625, "epoch": 0.7916230366492146, "fcm_dpo/beta": 0.0026616579852998257, "fcm_dpo/delta": 0.008543765172362328, "fcm_dpo/margin": 106.44680786132812, "fcm_dpo/q_t": 0.4352142810821533, "grad_norm": 92.10003662109375, "learning_rate": 6.409134137148736e-08, "logits/chosen": -0.8278205394744873, "logits/rejected": -0.8163886070251465, "logps/chosen": -559.232421875, "logps/ref_chosen": -281.5295715332031, "logps/ref_rejected": -296.980224609375, "logps/rejected": -681.1298828125, "loss": 4.775, "margin_dpo/margin_mean": 106.44680786132812, "margin_dpo/margin_std": 199.52207946777344, "step": 378 }, { "KL/chosen_KL_mean": -257.5284423828125, "KL/mean": -305.9362487792969, "KL/rejected_KL_mean": -354.3440856933594, "KL/std": 143.47189331054688, "epoch": 0.793717277486911, "fcm_dpo/beta": 0.002679901197552681, "fcm_dpo/delta": -0.0017141085118055344, "fcm_dpo/margin": 96.81559753417969, "fcm_dpo/q_t": 0.43951869010925293, "grad_norm": 81.0946044921875, "learning_rate": 6.28723129572247e-08, "logits/chosen": -0.9057261943817139, "logits/rejected": -0.8876835703849792, "logps/chosen": -522.6092529296875, "logps/ref_chosen": -265.0807800292969, "logps/ref_rejected": -230.58932495117188, "logps/rejected": -584.9334106445312, "loss": 4.8098, "margin_dpo/margin_mean": 96.81559753417969, "margin_dpo/margin_std": 180.41207885742188, "step": 379 }, { "KL/chosen_KL_mean": -278.8175964355469, "KL/mean": -334.8206787109375, "KL/rejected_KL_mean": -390.8237609863281, "KL/std": 182.3302459716797, "epoch": 0.7958115183246073, "fcm_dpo/beta": 0.002679501660168171, "fcm_dpo/delta": -0.008481554687023163, "fcm_dpo/margin": 112.0062026977539, "fcm_dpo/q_t": 0.43370163440704346, "grad_norm": 61.04739761352539, "learning_rate": 6.166331963291519e-08, "logits/chosen": -0.8648520708084106, "logits/rejected": -0.8380929231643677, "logps/chosen": -584.7259521484375, "logps/ref_chosen": -305.90838623046875, "logps/ref_rejected": -286.5906677246094, "logps/rejected": -677.4144287109375, "loss": 4.7804, "margin_dpo/margin_mean": 112.0062026977539, "margin_dpo/margin_std": 214.64852905273438, "step": 380 }, { "KL/chosen_KL_mean": -273.33929443359375, "KL/mean": -337.74554443359375, "KL/rejected_KL_mean": -402.1517333984375, "KL/std": 202.25027465820312, "epoch": 0.7979057591623037, "fcm_dpo/beta": 0.002631543204188347, "fcm_dpo/delta": -0.025047196075320244, "fcm_dpo/margin": 128.8124542236328, "fcm_dpo/q_t": 0.42539119720458984, "grad_norm": 61.73163604736328, "learning_rate": 6.046442623320145e-08, "logits/chosen": -0.832200288772583, "logits/rejected": -0.797419548034668, "logps/chosen": -526.2099609375, "logps/ref_chosen": -252.87066650390625, "logps/ref_rejected": -261.1927490234375, "logps/rejected": -663.344482421875, "loss": 4.6148, "margin_dpo/margin_mean": 128.8124542236328, "margin_dpo/margin_std": 213.1539306640625, "step": 381 }, { "KL/chosen_KL_mean": -264.1448059082031, "KL/mean": -332.416748046875, "KL/rejected_KL_mean": -400.688720703125, "KL/std": 163.4908447265625, "epoch": 0.8, "fcm_dpo/beta": 0.002580613363534212, "fcm_dpo/delta": -0.01013551652431488, "fcm_dpo/margin": 136.54393005371094, "fcm_dpo/q_t": 0.4181453287601471, "grad_norm": 60.10068130493164, "learning_rate": 5.9275697051098275e-08, "logits/chosen": -0.8769955635070801, "logits/rejected": -0.8728847503662109, "logps/chosen": -553.3562622070312, "logps/ref_chosen": -289.2114562988281, "logps/ref_rejected": -278.45751953125, "logps/rejected": -679.1461791992188, "loss": 4.4954, "margin_dpo/margin_mean": 136.54393005371094, "margin_dpo/margin_std": 189.81167602539062, "step": 382 }, { "KL/chosen_KL_mean": -260.4679870605469, "KL/mean": -319.3662109375, "KL/rejected_KL_mean": -378.2645263671875, "KL/std": 184.4472198486328, "epoch": 0.8020942408376963, "fcm_dpo/beta": 0.00258796033449471, "fcm_dpo/delta": -0.011181109584867954, "fcm_dpo/margin": 117.7965087890625, "fcm_dpo/q_t": 0.430372029542923, "grad_norm": 89.09895324707031, "learning_rate": 5.809719583454414e-08, "logits/chosen": -0.8481532335281372, "logits/rejected": -0.8308265209197998, "logps/chosen": -534.0987548828125, "logps/ref_chosen": -273.630859375, "logps/ref_rejected": -261.44024658203125, "logps/rejected": -639.7047119140625, "loss": 4.7125, "margin_dpo/margin_mean": 117.79650115966797, "margin_dpo/margin_std": 209.50341796875, "step": 383 }, { "KL/chosen_KL_mean": -264.8871154785156, "KL/mean": -317.6487731933594, "KL/rejected_KL_mean": -370.4104309082031, "KL/std": 146.82882690429688, "epoch": 0.8041884816753927, "fcm_dpo/beta": 0.002574121579527855, "fcm_dpo/delta": 0.014372201636433601, "fcm_dpo/margin": 105.52330017089844, "fcm_dpo/q_t": 0.437011182308197, "grad_norm": 57.85554122924805, "learning_rate": 5.6928985782982524e-08, "logits/chosen": -0.8842571973800659, "logits/rejected": -0.8786624073982239, "logps/chosen": -539.4570922851562, "logps/ref_chosen": -274.5699462890625, "logps/ref_rejected": -285.8253479003906, "logps/rejected": -656.2357177734375, "loss": 4.7566, "margin_dpo/margin_mean": 105.5232925415039, "margin_dpo/margin_std": 184.34634399414062, "step": 384 }, { "KL/chosen_KL_mean": -270.7925720214844, "KL/mean": -314.6628112792969, "KL/rejected_KL_mean": -358.5330810546875, "KL/std": 152.50241088867188, "epoch": 0.806282722513089, "fcm_dpo/beta": 0.0025990111753344536, "fcm_dpo/delta": 0.009375514462590218, "fcm_dpo/margin": 87.74047088623047, "fcm_dpo/q_t": 0.44724568724632263, "grad_norm": 70.02297973632812, "learning_rate": 5.57711295439732e-08, "logits/chosen": -0.821690559387207, "logits/rejected": -0.8161090612411499, "logps/chosen": -554.943115234375, "logps/ref_chosen": -284.150634765625, "logps/ref_rejected": -244.87921142578125, "logps/rejected": -603.4122314453125, "loss": 4.8863, "margin_dpo/margin_mean": 87.74047088623047, "margin_dpo/margin_std": 168.0803985595703, "step": 385 }, { "KL/chosen_KL_mean": -259.23101806640625, "KL/mean": -329.41278076171875, "KL/rejected_KL_mean": -399.594482421875, "KL/std": 172.50108337402344, "epoch": 0.8083769633507853, "fcm_dpo/beta": 0.002526678144931793, "fcm_dpo/delta": -0.04324443265795708, "fcm_dpo/margin": 140.36346435546875, "fcm_dpo/q_t": 0.4185819625854492, "grad_norm": 79.33147430419922, "learning_rate": 5.4623689209832484e-08, "logits/chosen": -0.7993844151496887, "logits/rejected": -0.7968799471855164, "logps/chosen": -579.4072875976562, "logps/ref_chosen": -320.1762390136719, "logps/ref_rejected": -302.05023193359375, "logps/rejected": -701.644775390625, "loss": 4.5125, "margin_dpo/margin_mean": 140.36346435546875, "margin_dpo/margin_std": 189.78298950195312, "step": 386 }, { "KL/chosen_KL_mean": -282.47479248046875, "KL/mean": -334.10858154296875, "KL/rejected_KL_mean": -385.74237060546875, "KL/std": 162.93824768066406, "epoch": 0.8104712041884817, "fcm_dpo/beta": 0.002494890708476305, "fcm_dpo/delta": -0.023709315806627274, "fcm_dpo/margin": 103.26756286621094, "fcm_dpo/q_t": 0.4388362765312195, "grad_norm": 74.36798858642578, "learning_rate": 5.3486726314303175e-08, "logits/chosen": -0.8310315012931824, "logits/rejected": -0.8349242210388184, "logps/chosen": -554.7550048828125, "logps/ref_chosen": -272.2801513671875, "logps/ref_rejected": -265.1615905761719, "logps/rejected": -650.9039306640625, "loss": 4.8558, "margin_dpo/margin_mean": 103.26756286621094, "margin_dpo/margin_std": 199.19708251953125, "step": 387 }, { "KL/chosen_KL_mean": -298.3812561035156, "KL/mean": -334.0774230957031, "KL/rejected_KL_mean": -369.7735595703125, "KL/std": 169.716552734375, "epoch": 0.812565445026178, "fcm_dpo/beta": 0.0024554524570703506, "fcm_dpo/delta": -0.007999473251402378, "fcm_dpo/margin": 71.392333984375, "fcm_dpo/q_t": 0.45962756872177124, "grad_norm": 65.51069641113281, "learning_rate": 5.2360301829254745e-08, "logits/chosen": -0.8141230344772339, "logits/rejected": -0.7994064092636108, "logps/chosen": -570.91259765625, "logps/ref_chosen": -272.5313415527344, "logps/ref_rejected": -239.55735778808594, "logps/rejected": -609.3309936523438, "loss": 5.1277, "margin_dpo/margin_mean": 71.392333984375, "margin_dpo/margin_std": 200.36790466308594, "step": 388 }, { "KL/chosen_KL_mean": -292.8984069824219, "KL/mean": -350.0514831542969, "KL/rejected_KL_mean": -407.2045593261719, "KL/std": 179.6480255126953, "epoch": 0.8146596858638744, "fcm_dpo/beta": 0.0024243236985057592, "fcm_dpo/delta": -0.027813490480184555, "fcm_dpo/margin": 114.30616760253906, "fcm_dpo/q_t": 0.43652477860450745, "grad_norm": 57.472259521484375, "learning_rate": 5.1244476161413806e-08, "logits/chosen": -0.8450219631195068, "logits/rejected": -0.8411790132522583, "logps/chosen": -573.9876708984375, "logps/ref_chosen": -281.0892639160156, "logps/ref_rejected": -246.50045776367188, "logps/rejected": -653.7050170898438, "loss": 4.779, "margin_dpo/margin_mean": 114.30615997314453, "margin_dpo/margin_std": 204.97317504882812, "step": 389 }, { "KL/chosen_KL_mean": -281.30267333984375, "KL/mean": -337.25286865234375, "KL/rejected_KL_mean": -393.20306396484375, "KL/std": 167.31585693359375, "epoch": 0.8167539267015707, "fcm_dpo/beta": 0.002404128899797797, "fcm_dpo/delta": 0.0032799788750708103, "fcm_dpo/margin": 111.90047454833984, "fcm_dpo/q_t": 0.4366511106491089, "grad_norm": 69.60022735595703, "learning_rate": 5.013930914912476e-08, "logits/chosen": -0.8703796863555908, "logits/rejected": -0.8816381096839905, "logps/chosen": -565.2901000976562, "logps/ref_chosen": -283.98748779296875, "logps/ref_rejected": -283.465087890625, "logps/rejected": -676.6681518554688, "loss": 4.745, "margin_dpo/margin_mean": 111.90047454833984, "margin_dpo/margin_std": 186.23097229003906, "step": 390 }, { "KL/chosen_KL_mean": -291.63238525390625, "KL/mean": -353.4014892578125, "KL/rejected_KL_mean": -415.1705627441406, "KL/std": 187.97467041015625, "epoch": 0.818848167539267, "fcm_dpo/beta": 0.0024001263082027435, "fcm_dpo/delta": -0.009568718262016773, "fcm_dpo/margin": 123.53820037841797, "fcm_dpo/q_t": 0.43175381422042847, "grad_norm": 60.939510345458984, "learning_rate": 4.904486005914027e-08, "logits/chosen": -0.8228967785835266, "logits/rejected": -0.8025352954864502, "logps/chosen": -575.4937744140625, "logps/ref_chosen": -283.86138916015625, "logps/ref_rejected": -263.5093688964844, "logps/rejected": -678.679931640625, "loss": 4.6976, "margin_dpo/margin_mean": 123.53820037841797, "margin_dpo/margin_std": 205.3511962890625, "step": 391 }, { "KL/chosen_KL_mean": -268.03955078125, "KL/mean": -335.4253234863281, "KL/rejected_KL_mean": -402.8111267089844, "KL/std": 162.87490844726562, "epoch": 0.8209424083769633, "fcm_dpo/beta": 0.0023622428998351097, "fcm_dpo/delta": -0.008839438669383526, "fcm_dpo/margin": 134.77159118652344, "fcm_dpo/q_t": 0.42433956265449524, "grad_norm": 79.00080871582031, "learning_rate": 4.796118758344353e-08, "logits/chosen": -0.8106127977371216, "logits/rejected": -0.8217307329177856, "logps/chosen": -578.1096801757812, "logps/ref_chosen": -310.070068359375, "logps/ref_rejected": -252.89817810058594, "logps/rejected": -655.7092895507812, "loss": 4.571, "margin_dpo/margin_mean": 134.77157592773438, "margin_dpo/margin_std": 180.40658569335938, "step": 392 }, { "KL/chosen_KL_mean": -274.3587646484375, "KL/mean": -322.0811767578125, "KL/rejected_KL_mean": -369.8035888671875, "KL/std": 162.0053253173828, "epoch": 0.8230366492146597, "fcm_dpo/beta": 0.0023829475976526737, "fcm_dpo/delta": 0.016615940257906914, "fcm_dpo/margin": 95.44483947753906, "fcm_dpo/q_t": 0.4470600485801697, "grad_norm": 48.30846405029297, "learning_rate": 4.688834983610082e-08, "logits/chosen": -0.8459432721138, "logits/rejected": -0.8264710903167725, "logps/chosen": -561.0744018554688, "logps/ref_chosen": -286.7156677246094, "logps/ref_rejected": -230.00357055664062, "logps/rejected": -599.8071899414062, "loss": 4.8972, "margin_dpo/margin_mean": 95.44483947753906, "margin_dpo/margin_std": 188.01112365722656, "step": 393 }, { "KL/chosen_KL_mean": -277.00994873046875, "KL/mean": -327.65032958984375, "KL/rejected_KL_mean": -378.290771484375, "KL/std": 194.45957946777344, "epoch": 0.8251308900523561, "fcm_dpo/beta": 0.002444605343043804, "fcm_dpo/delta": 0.03289797902107239, "fcm_dpo/margin": 101.28084564208984, "fcm_dpo/q_t": 0.44555556774139404, "grad_norm": 50.116294860839844, "learning_rate": 4.582640435014459e-08, "logits/chosen": -0.891754686832428, "logits/rejected": -0.8810494542121887, "logps/chosen": -603.0033569335938, "logps/ref_chosen": -325.9934387207031, "logps/ref_rejected": -317.42706298828125, "logps/rejected": -695.7178344726562, "loss": 4.9184, "margin_dpo/margin_mean": 101.28085327148438, "margin_dpo/margin_std": 220.1911163330078, "step": 394 }, { "KL/chosen_KL_mean": -264.396240234375, "KL/mean": -332.3326110839844, "KL/rejected_KL_mean": -400.2690124511719, "KL/std": 177.35574340820312, "epoch": 0.8272251308900523, "fcm_dpo/beta": 0.002485373057425022, "fcm_dpo/delta": -0.001986829563975334, "fcm_dpo/margin": 135.87278747558594, "fcm_dpo/q_t": 0.4249032437801361, "grad_norm": 77.19000244140625, "learning_rate": 4.477540807448832e-08, "logits/chosen": -0.8276509642601013, "logits/rejected": -0.8361413478851318, "logps/chosen": -533.2970581054688, "logps/ref_chosen": -268.90081787109375, "logps/ref_rejected": -272.85809326171875, "logps/rejected": -673.127197265625, "loss": 4.5902, "margin_dpo/margin_mean": 135.87278747558594, "margin_dpo/margin_std": 214.613037109375, "step": 395 }, { "KL/chosen_KL_mean": -279.2574462890625, "KL/mean": -331.1311950683594, "KL/rejected_KL_mean": -383.0049133300781, "KL/std": 168.5165557861328, "epoch": 0.8293193717277487, "fcm_dpo/beta": 0.0024622506462037563, "fcm_dpo/delta": -0.0033884027507156134, "fcm_dpo/margin": 103.74746704101562, "fcm_dpo/q_t": 0.4406885802745819, "grad_norm": 85.6566162109375, "learning_rate": 4.373541737087263e-08, "logits/chosen": -0.8391107320785522, "logits/rejected": -0.817350447177887, "logps/chosen": -570.4557495117188, "logps/ref_chosen": -291.19830322265625, "logps/ref_rejected": -253.2803955078125, "logps/rejected": -636.2852783203125, "loss": 4.814, "margin_dpo/margin_mean": 103.7474594116211, "margin_dpo/margin_std": 189.7008514404297, "step": 396 }, { "KL/chosen_KL_mean": -270.15325927734375, "KL/mean": -316.8970947265625, "KL/rejected_KL_mean": -363.64093017578125, "KL/std": 164.19032287597656, "epoch": 0.831413612565445, "fcm_dpo/beta": 0.002420744625851512, "fcm_dpo/delta": -0.03247765451669693, "fcm_dpo/margin": 93.48770141601562, "fcm_dpo/q_t": 0.4481962025165558, "grad_norm": 84.03921508789062, "learning_rate": 4.270648801084295e-08, "logits/chosen": -0.8543325066566467, "logits/rejected": -0.8361637592315674, "logps/chosen": -579.9756469726562, "logps/ref_chosen": -309.8224182128906, "logps/ref_rejected": -291.9057922363281, "logps/rejected": -655.5467529296875, "loss": 4.9515, "margin_dpo/margin_mean": 93.48770141601562, "margin_dpo/margin_std": 195.09881591796875, "step": 397 }, { "KL/chosen_KL_mean": -302.65869140625, "KL/mean": -342.31829833984375, "KL/rejected_KL_mean": -381.97796630859375, "KL/std": 184.55848693847656, "epoch": 0.8335078534031414, "fcm_dpo/beta": 0.002387051470577717, "fcm_dpo/delta": -0.00552708376199007, "fcm_dpo/margin": 79.3193130493164, "fcm_dpo/q_t": 0.4543570280075073, "grad_norm": 74.99633026123047, "learning_rate": 4.168867517275806e-08, "logits/chosen": -0.7610083818435669, "logits/rejected": -0.7987161874771118, "logps/chosen": -600.47216796875, "logps/ref_chosen": -297.8135070800781, "logps/ref_rejected": -270.5025634765625, "logps/rejected": -652.4805297851562, "loss": 5.1802, "margin_dpo/margin_mean": 79.3193130493164, "margin_dpo/margin_std": 238.6734161376953, "step": 398 }, { "KL/chosen_KL_mean": -282.90521240234375, "KL/mean": -336.2668762207031, "KL/rejected_KL_mean": -389.6285400390625, "KL/std": 171.79843139648438, "epoch": 0.8356020942408376, "fcm_dpo/beta": 0.0023906768765300512, "fcm_dpo/delta": 0.0051354942843317986, "fcm_dpo/margin": 106.72332763671875, "fcm_dpo/q_t": 0.44120585918426514, "grad_norm": 64.98973083496094, "learning_rate": 4.0682033438831584e-08, "logits/chosen": -0.8578112125396729, "logits/rejected": -0.8178203701972961, "logps/chosen": -575.751953125, "logps/ref_chosen": -292.8467712402344, "logps/ref_rejected": -268.3638916015625, "logps/rejected": -657.992431640625, "loss": 4.8244, "margin_dpo/margin_mean": 106.72332763671875, "margin_dpo/margin_std": 198.6634063720703, "step": 399 }, { "KL/chosen_KL_mean": -276.2769775390625, "KL/mean": -327.45770263671875, "KL/rejected_KL_mean": -378.6385192871094, "KL/std": 167.0556182861328, "epoch": 0.837696335078534, "fcm_dpo/beta": 0.0023928708396852016, "fcm_dpo/delta": 0.004267351236194372, "fcm_dpo/margin": 102.36151885986328, "fcm_dpo/q_t": 0.4427049458026886, "grad_norm": 52.65484619140625, "learning_rate": 3.968661679220467e-08, "logits/chosen": -0.8961995840072632, "logits/rejected": -0.8956049680709839, "logps/chosen": -539.953369140625, "logps/ref_chosen": -263.6763916015625, "logps/ref_rejected": -258.67266845703125, "logps/rejected": -637.3111572265625, "loss": 4.8661, "margin_dpo/margin_mean": 102.36151123046875, "margin_dpo/margin_std": 198.69558715820312, "step": 400 }, { "epoch": 0.837696335078534, "eval_KL/chosen_KL_mean": -276.4910583496094, "eval_KL/mean": -331.89849853515625, "eval_KL/rejected_KL_mean": -387.3060302734375, "eval_KL/std": 172.48843383789062, "eval_fcm_dpo/beta": 0.0024002783466130495, "eval_logits/chosen": -0.858095645904541, "eval_logits/rejected": -0.8404949307441711, "eval_logps/chosen": -564.31787109375, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -654.2360229492188, "eval_loss": 0.601615309715271, "eval_margin_dpo/margin_mean": 110.81495666503906, "eval_margin_dpo/margin_std": 193.406005859375, "eval_runtime": 78.4354, "eval_samples_per_second": 25.499, "eval_steps_per_second": 3.187, "step": 400 }, { "KL/chosen_KL_mean": -277.6224670410156, "KL/mean": -332.4400329589844, "KL/rejected_KL_mean": -387.2575988769531, "KL/std": 175.7390594482422, "epoch": 0.8397905759162304, "fcm_dpo/beta": 0.002420206554234028, "fcm_dpo/delta": 0.020259615033864975, "fcm_dpo/margin": 109.63511657714844, "fcm_dpo/q_t": 0.4383876919746399, "grad_norm": 74.74368286132812, "learning_rate": 3.8702478614051345e-08, "logits/chosen": -0.8335940837860107, "logits/rejected": -0.8319975137710571, "logps/chosen": -595.9077758789062, "logps/ref_chosen": -318.2853088378906, "logps/ref_rejected": -293.75225830078125, "logps/rejected": -681.009765625, "loss": 4.7963, "margin_dpo/margin_mean": 109.63511657714844, "margin_dpo/margin_std": 200.65774536132812, "step": 401 }, { "KL/chosen_KL_mean": -263.85992431640625, "KL/mean": -316.65972900390625, "KL/rejected_KL_mean": -369.45953369140625, "KL/std": 180.7909698486328, "epoch": 0.8418848167539267, "fcm_dpo/beta": 0.0024443857837468386, "fcm_dpo/delta": 0.009977946057915688, "fcm_dpo/margin": 105.59957885742188, "fcm_dpo/q_t": 0.44268375635147095, "grad_norm": 62.405208587646484, "learning_rate": 3.772967168071517e-08, "logits/chosen": -0.8932153582572937, "logits/rejected": -0.8693395256996155, "logps/chosen": -573.2877807617188, "logps/ref_chosen": -309.4278564453125, "logps/ref_rejected": -282.0279846191406, "logps/rejected": -651.487548828125, "loss": 4.8532, "margin_dpo/margin_mean": 105.59957885742188, "margin_dpo/margin_std": 217.751953125, "step": 402 }, { "KL/chosen_KL_mean": -258.3434753417969, "KL/mean": -334.11083984375, "KL/rejected_KL_mean": -409.8782043457031, "KL/std": 184.3195037841797, "epoch": 0.8439790575916231, "fcm_dpo/beta": 0.00247446377761662, "fcm_dpo/delta": -0.014491792768239975, "fcm_dpo/margin": 151.5347442626953, "fcm_dpo/q_t": 0.4138992726802826, "grad_norm": 57.879676818847656, "learning_rate": 3.676824816087978e-08, "logits/chosen": -0.866585373878479, "logits/rejected": -0.8403845429420471, "logps/chosen": -567.3720092773438, "logps/ref_chosen": -309.0284729003906, "logps/ref_rejected": -272.9622497558594, "logps/rejected": -682.8404541015625, "loss": 4.4146, "margin_dpo/margin_mean": 151.5347442626953, "margin_dpo/margin_std": 193.26220703125, "step": 403 }, { "KL/chosen_KL_mean": -280.8930358886719, "KL/mean": -337.9317321777344, "KL/rejected_KL_mean": -394.9703674316406, "KL/std": 184.8937225341797, "epoch": 0.8460732984293193, "fcm_dpo/beta": 0.002477236557751894, "fcm_dpo/delta": 0.02862975373864174, "fcm_dpo/margin": 114.07733154296875, "fcm_dpo/q_t": 0.4357641041278839, "grad_norm": 80.73699188232422, "learning_rate": 3.581825961277074e-08, "logits/chosen": -0.8829959630966187, "logits/rejected": -0.858309268951416, "logps/chosen": -578.1767578125, "logps/ref_chosen": -297.2837219238281, "logps/ref_rejected": -256.99041748046875, "logps/rejected": -651.9608154296875, "loss": 4.7975, "margin_dpo/margin_mean": 114.07733917236328, "margin_dpo/margin_std": 220.4342041015625, "step": 404 }, { "KL/chosen_KL_mean": -271.5806579589844, "KL/mean": -334.3476257324219, "KL/rejected_KL_mean": -397.1145324707031, "KL/std": 183.08030700683594, "epoch": 0.8481675392670157, "fcm_dpo/beta": 0.002512460108846426, "fcm_dpo/delta": 0.023881835862994194, "fcm_dpo/margin": 125.53388977050781, "fcm_dpo/q_t": 0.42782172560691833, "grad_norm": 49.44511795043945, "learning_rate": 3.487975698139084e-08, "logits/chosen": -0.8111081123352051, "logits/rejected": -0.8217229843139648, "logps/chosen": -529.5459594726562, "logps/ref_chosen": -257.96533203125, "logps/ref_rejected": -255.811279296875, "logps/rejected": -652.92578125, "loss": 4.6444, "margin_dpo/margin_mean": 125.53388977050781, "margin_dpo/margin_std": 206.29136657714844, "step": 405 }, { "KL/chosen_KL_mean": -288.9252014160156, "KL/mean": -329.91778564453125, "KL/rejected_KL_mean": -370.910400390625, "KL/std": 164.0752410888672, "epoch": 0.8502617801047121, "fcm_dpo/beta": 0.002553413389250636, "fcm_dpo/delta": 0.010630952194333076, "fcm_dpo/margin": 81.9852294921875, "fcm_dpo/q_t": 0.45008060336112976, "grad_norm": 70.0001220703125, "learning_rate": 3.3952790595787986e-08, "logits/chosen": -0.852975606918335, "logits/rejected": -0.8305176496505737, "logps/chosen": -574.106201171875, "logps/ref_chosen": -285.1810607910156, "logps/ref_rejected": -264.41351318359375, "logps/rejected": -635.3239135742188, "loss": 4.9526, "margin_dpo/margin_mean": 81.9852294921875, "margin_dpo/margin_std": 172.13865661621094, "step": 406 }, { "KL/chosen_KL_mean": -290.98028564453125, "KL/mean": -351.85919189453125, "KL/rejected_KL_mean": -412.73809814453125, "KL/std": 183.7567596435547, "epoch": 0.8523560209424084, "fcm_dpo/beta": 0.002567433984950185, "fcm_dpo/delta": 0.007098458707332611, "fcm_dpo/margin": 121.75782012939453, "fcm_dpo/q_t": 0.42956987023353577, "grad_norm": 69.03553009033203, "learning_rate": 3.303741016635614e-08, "logits/chosen": -0.8222445249557495, "logits/rejected": -0.8362969160079956, "logps/chosen": -556.2183837890625, "logps/ref_chosen": -265.23809814453125, "logps/ref_rejected": -219.0631561279297, "logps/rejected": -631.80126953125, "loss": 4.7024, "margin_dpo/margin_mean": 121.75782012939453, "margin_dpo/margin_std": 215.36898803710938, "step": 407 }, { "KL/chosen_KL_mean": -265.6532287597656, "KL/mean": -325.81756591796875, "KL/rejected_KL_mean": -385.98187255859375, "KL/std": 170.1363067626953, "epoch": 0.8544502617801047, "fcm_dpo/beta": 0.002568720607087016, "fcm_dpo/delta": -0.03308972716331482, "fcm_dpo/margin": 120.32862091064453, "fcm_dpo/q_t": 0.42802876234054565, "grad_norm": 64.20223236083984, "learning_rate": 3.2133664782169944e-08, "logits/chosen": -0.8611224293708801, "logits/rejected": -0.8501880764961243, "logps/chosen": -562.6258544921875, "logps/ref_chosen": -296.9726257324219, "logps/ref_rejected": -295.4786376953125, "logps/rejected": -681.46044921875, "loss": 4.6499, "margin_dpo/margin_mean": 120.32861328125, "margin_dpo/margin_std": 186.1289520263672, "step": 408 }, { "KL/chosen_KL_mean": -277.5410461425781, "KL/mean": -346.1307678222656, "KL/rejected_KL_mean": -414.720458984375, "KL/std": 232.1706085205078, "epoch": 0.856544502617801, "fcm_dpo/beta": 0.0024434137158095837, "fcm_dpo/delta": -0.05346183478832245, "fcm_dpo/margin": 137.17942810058594, "fcm_dpo/q_t": 0.4328618347644806, "grad_norm": 49.34114074707031, "learning_rate": 3.12416029083514e-08, "logits/chosen": -0.8433458209037781, "logits/rejected": -0.8271778225898743, "logps/chosen": -564.92041015625, "logps/ref_chosen": -287.37933349609375, "logps/ref_rejected": -275.80291748046875, "logps/rejected": -690.5233154296875, "loss": 4.748, "margin_dpo/margin_mean": 137.17942810058594, "margin_dpo/margin_std": 281.4482727050781, "step": 409 }, { "KL/chosen_KL_mean": -273.635986328125, "KL/mean": -326.820068359375, "KL/rejected_KL_mean": -380.00408935546875, "KL/std": 171.7606658935547, "epoch": 0.8586387434554974, "fcm_dpo/beta": 0.0024228268302977085, "fcm_dpo/delta": 0.015623921528458595, "fcm_dpo/margin": 106.36808013916016, "fcm_dpo/q_t": 0.44091394543647766, "grad_norm": 69.07209777832031, "learning_rate": 3.036127238347164e-08, "logits/chosen": -0.8576352596282959, "logits/rejected": -0.8624677062034607, "logps/chosen": -555.4161987304688, "logps/ref_chosen": -281.7801818847656, "logps/ref_rejected": -266.7550354003906, "logps/rejected": -646.7590942382812, "loss": 4.8306, "margin_dpo/margin_mean": 106.36808013916016, "margin_dpo/margin_std": 204.33482360839844, "step": 410 }, { "KL/chosen_KL_mean": -265.7773742675781, "KL/mean": -329.0021057128906, "KL/rejected_KL_mean": -392.2268371582031, "KL/std": 182.93353271484375, "epoch": 0.8607329842931937, "fcm_dpo/beta": 0.0024366420693695545, "fcm_dpo/delta": -0.019304735586047173, "fcm_dpo/margin": 126.44949340820312, "fcm_dpo/q_t": 0.4297502040863037, "grad_norm": 61.002811431884766, "learning_rate": 2.9492720416985e-08, "logits/chosen": -0.8678796291351318, "logits/rejected": -0.8195263743400574, "logps/chosen": -547.3646240234375, "logps/ref_chosen": -281.5872497558594, "logps/ref_rejected": -254.78916931152344, "logps/rejected": -647.0159912109375, "loss": 4.6211, "margin_dpo/margin_mean": 126.44949340820312, "margin_dpo/margin_std": 180.95140075683594, "step": 411 }, { "KL/chosen_KL_mean": -296.2814636230469, "KL/mean": -348.8402099609375, "KL/rejected_KL_mean": -401.39898681640625, "KL/std": 175.22732543945312, "epoch": 0.86282722513089, "fcm_dpo/beta": 0.0023367248941212893, "fcm_dpo/delta": -0.03149264678359032, "fcm_dpo/margin": 105.11750030517578, "fcm_dpo/q_t": 0.44317078590393066, "grad_norm": 65.44154357910156, "learning_rate": 2.863599358669755e-08, "logits/chosen": -0.8309513926506042, "logits/rejected": -0.8433988094329834, "logps/chosen": -572.8156127929688, "logps/ref_chosen": -276.5341796875, "logps/ref_rejected": -273.8751220703125, "logps/rejected": -675.2740478515625, "loss": 4.8567, "margin_dpo/margin_mean": 105.11750030517578, "margin_dpo/margin_std": 195.50192260742188, "step": 412 }, { "KL/chosen_KL_mean": -279.3658752441406, "KL/mean": -327.3970947265625, "KL/rejected_KL_mean": -375.42840576171875, "KL/std": 175.02883911132812, "epoch": 0.8649214659685864, "fcm_dpo/beta": 0.0023326242808252573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.0625, "fcm_dpo/q_t": 0.44770365953445435, "grad_norm": 88.38508605957031, "learning_rate": 2.7791137836269158e-08, "logits/chosen": -0.8533222079277039, "logits/rejected": -0.8485733866691589, "logps/chosen": -550.6405029296875, "logps/ref_chosen": -271.2745666503906, "logps/ref_rejected": -270.16912841796875, "logps/rejected": -645.5975341796875, "loss": 4.9113, "margin_dpo/margin_mean": 96.0625, "margin_dpo/margin_std": 199.59288024902344, "step": 413 }, { "KL/chosen_KL_mean": -281.5064392089844, "KL/mean": -345.09735107421875, "KL/rejected_KL_mean": -408.6882629394531, "KL/std": 176.5061798095703, "epoch": 0.8670157068062827, "fcm_dpo/beta": 0.0023213180247694254, "fcm_dpo/delta": -0.015394072979688644, "fcm_dpo/margin": 127.18183898925781, "fcm_dpo/q_t": 0.4320528507232666, "grad_norm": 54.224853515625, "learning_rate": 2.6958198472749717e-08, "logits/chosen": -0.8887584805488586, "logits/rejected": -0.8804943561553955, "logps/chosen": -578.6215209960938, "logps/ref_chosen": -297.11505126953125, "logps/ref_rejected": -271.7034606933594, "logps/rejected": -680.3917236328125, "loss": 4.6761, "margin_dpo/margin_mean": 127.18183898925781, "margin_dpo/margin_std": 199.06312561035156, "step": 414 }, { "KL/chosen_KL_mean": -272.0690002441406, "KL/mean": -331.60137939453125, "KL/rejected_KL_mean": -391.1337890625, "KL/std": 162.61656188964844, "epoch": 0.8691099476439791, "fcm_dpo/beta": 0.002339128637686372, "fcm_dpo/delta": 0.029414832592010498, "fcm_dpo/margin": 119.0648193359375, "fcm_dpo/q_t": 0.4344155192375183, "grad_norm": 72.77362060546875, "learning_rate": 2.613722016414943e-08, "logits/chosen": -0.8714032173156738, "logits/rejected": -0.8565488457679749, "logps/chosen": -569.7616577148438, "logps/ref_chosen": -297.6926574707031, "logps/ref_rejected": -279.0503234863281, "logps/rejected": -670.18408203125, "loss": 4.6898, "margin_dpo/margin_mean": 119.0648193359375, "margin_dpo/margin_std": 182.08590698242188, "step": 415 }, { "KL/chosen_KL_mean": -260.75152587890625, "KL/mean": -327.1971435546875, "KL/rejected_KL_mean": -393.6427001953125, "KL/std": 163.0526885986328, "epoch": 0.8712041884816754, "fcm_dpo/beta": 0.002332664094865322, "fcm_dpo/delta": -0.0067732506431639194, "fcm_dpo/margin": 132.8911590576172, "fcm_dpo/q_t": 0.42698273062705994, "grad_norm": 51.488826751708984, "learning_rate": 2.5328246937043525e-08, "logits/chosen": -0.8797720670700073, "logits/rejected": -0.8755515217781067, "logps/chosen": -572.5771484375, "logps/ref_chosen": -311.8255615234375, "logps/ref_rejected": -268.6170654296875, "logps/rejected": -662.259765625, "loss": 4.6047, "margin_dpo/margin_mean": 132.8911590576172, "margin_dpo/margin_std": 190.5181121826172, "step": 416 }, { "KL/chosen_KL_mean": -290.8223571777344, "KL/mean": -347.7958068847656, "KL/rejected_KL_mean": -404.7692565917969, "KL/std": 185.7816162109375, "epoch": 0.8732984293193717, "fcm_dpo/beta": 0.0023309113457798958, "fcm_dpo/delta": -0.02380407229065895, "fcm_dpo/margin": 113.94691467285156, "fcm_dpo/q_t": 0.43974053859710693, "grad_norm": 60.413761138916016, "learning_rate": 2.4531322174210973e-08, "logits/chosen": -0.8300318717956543, "logits/rejected": -0.8154677748680115, "logps/chosen": -601.2591552734375, "logps/ref_chosen": -310.43682861328125, "logps/ref_rejected": -277.15283203125, "logps/rejected": -681.922119140625, "loss": 4.8109, "margin_dpo/margin_mean": 113.94691467285156, "margin_dpo/margin_std": 207.6744842529297, "step": 417 }, { "KL/chosen_KL_mean": -301.2762145996094, "KL/mean": -361.7869873046875, "KL/rejected_KL_mean": -422.29779052734375, "KL/std": 202.2158660888672, "epoch": 0.875392670157068, "fcm_dpo/beta": 0.00228428328409791, "fcm_dpo/delta": -0.011082015931606293, "fcm_dpo/margin": 121.02159881591797, "fcm_dpo/q_t": 0.43776950240135193, "grad_norm": 64.88748931884766, "learning_rate": 2.3746488612308295e-08, "logits/chosen": -0.8160441517829895, "logits/rejected": -0.7922072410583496, "logps/chosen": -579.7721557617188, "logps/ref_chosen": -278.49591064453125, "logps/ref_rejected": -276.56671142578125, "logps/rejected": -698.864501953125, "loss": 4.8021, "margin_dpo/margin_mean": 121.02159118652344, "margin_dpo/margin_std": 235.40972900390625, "step": 418 }, { "KL/chosen_KL_mean": -301.7872314453125, "KL/mean": -356.77294921875, "KL/rejected_KL_mean": -411.7586669921875, "KL/std": 187.64405822753906, "epoch": 0.8774869109947644, "fcm_dpo/beta": 0.002300471067428589, "fcm_dpo/delta": 0.014020832255482674, "fcm_dpo/margin": 109.97142791748047, "fcm_dpo/q_t": 0.44006672501564026, "grad_norm": 68.1669692993164, "learning_rate": 2.297378833957761e-08, "logits/chosen": -0.8629166483879089, "logits/rejected": -0.8316540122032166, "logps/chosen": -600.6875, "logps/ref_chosen": -298.9002380371094, "logps/ref_rejected": -246.1540985107422, "logps/rejected": -657.9127197265625, "loss": 4.8853, "margin_dpo/margin_mean": 109.97142791748047, "margin_dpo/margin_std": 232.2672119140625, "step": 419 }, { "KL/chosen_KL_mean": -289.37896728515625, "KL/mean": -360.5723876953125, "KL/rejected_KL_mean": -431.765869140625, "KL/std": 196.79029846191406, "epoch": 0.8795811518324608, "fcm_dpo/beta": 0.0023048303555697203, "fcm_dpo/delta": -0.011320296674966812, "fcm_dpo/margin": 142.38693237304688, "fcm_dpo/q_t": 0.4256310760974884, "grad_norm": 87.50121307373047, "learning_rate": 2.2213262793589482e-08, "logits/chosen": -0.8016407489776611, "logits/rejected": -0.7715350389480591, "logps/chosen": -553.9398193359375, "logps/ref_chosen": -264.5608825683594, "logps/ref_rejected": -245.67031860351562, "logps/rejected": -677.4362182617188, "loss": 4.672, "margin_dpo/margin_mean": 142.38693237304688, "margin_dpo/margin_std": 250.87786865234375, "step": 420 }, { "KL/chosen_KL_mean": -267.2189636230469, "KL/mean": -327.4419860839844, "KL/rejected_KL_mean": -387.6650390625, "KL/std": 166.16717529296875, "epoch": 0.881675392670157, "fcm_dpo/beta": 0.002284942427650094, "fcm_dpo/delta": -0.0033614085987210274, "fcm_dpo/margin": 120.44603729248047, "fcm_dpo/q_t": 0.4343334436416626, "grad_norm": 60.55808639526367, "learning_rate": 2.1464952759020856e-08, "logits/chosen": -0.8927887082099915, "logits/rejected": -0.8663312792778015, "logps/chosen": -564.9240112304688, "logps/ref_chosen": -297.70501708984375, "logps/ref_rejected": -243.74771118164062, "logps/rejected": -631.4127197265625, "loss": 4.718, "margin_dpo/margin_mean": 120.44603729248047, "margin_dpo/margin_std": 188.74952697753906, "step": 421 }, { "KL/chosen_KL_mean": -283.3934020996094, "KL/mean": -340.1562194824219, "KL/rejected_KL_mean": -396.9190368652344, "KL/std": 182.96835327148438, "epoch": 0.8837696335078534, "fcm_dpo/beta": 0.0022875501308590174, "fcm_dpo/delta": 0.014606714248657227, "fcm_dpo/margin": 113.52565002441406, "fcm_dpo/q_t": 0.44040507078170776, "grad_norm": 53.32109069824219, "learning_rate": 2.07288983654679e-08, "logits/chosen": -0.7491770386695862, "logits/rejected": -0.7867967486381531, "logps/chosen": -571.7521362304688, "logps/ref_chosen": -288.3587646484375, "logps/ref_rejected": -256.4377746582031, "logps/rejected": -653.3568115234375, "loss": 4.8024, "margin_dpo/margin_mean": 113.5256576538086, "margin_dpo/margin_std": 209.61355590820312, "step": 422 }, { "KL/chosen_KL_mean": -292.1402893066406, "KL/mean": -356.8420715332031, "KL/rejected_KL_mean": -421.54388427734375, "KL/std": 173.6347198486328, "epoch": 0.8858638743455497, "fcm_dpo/beta": 0.0023075914941728115, "fcm_dpo/delta": 0.0020584806334227324, "fcm_dpo/margin": 129.40359497070312, "fcm_dpo/q_t": 0.43022823333740234, "grad_norm": 63.76276397705078, "learning_rate": 2.0005139085293942e-08, "logits/chosen": -0.8720574378967285, "logits/rejected": -0.8487915396690369, "logps/chosen": -588.1472778320312, "logps/ref_chosen": -296.00701904296875, "logps/ref_rejected": -261.3480529785156, "logps/rejected": -682.8919067382812, "loss": 4.6646, "margin_dpo/margin_mean": 129.40359497070312, "margin_dpo/margin_std": 206.61529541015625, "step": 423 }, { "KL/chosen_KL_mean": -270.66021728515625, "KL/mean": -328.9430847167969, "KL/rejected_KL_mean": -387.2259826660156, "KL/std": 167.630615234375, "epoch": 0.8879581151832461, "fcm_dpo/beta": 0.0022989388089627028, "fcm_dpo/delta": 0.0014383550733327866, "fcm_dpo/margin": 116.56575775146484, "fcm_dpo/q_t": 0.4369730055332184, "grad_norm": 85.71529388427734, "learning_rate": 1.9293713731512673e-08, "logits/chosen": -0.8570412397384644, "logits/rejected": -0.8399197459220886, "logps/chosen": -580.0821533203125, "logps/ref_chosen": -309.421875, "logps/ref_rejected": -249.14886474609375, "logps/rejected": -636.3748779296875, "loss": 4.7347, "margin_dpo/margin_mean": 116.56575775146484, "margin_dpo/margin_std": 184.96189880371094, "step": 424 }, { "KL/chosen_KL_mean": -277.16748046875, "KL/mean": -334.2838439941406, "KL/rejected_KL_mean": -391.40020751953125, "KL/std": 176.4136199951172, "epoch": 0.8900523560209425, "fcm_dpo/beta": 0.0022896535228937864, "fcm_dpo/delta": -0.02011648565530777, "fcm_dpo/margin": 114.23271942138672, "fcm_dpo/q_t": 0.4396037459373474, "grad_norm": 79.22269439697266, "learning_rate": 1.8594660455706763e-08, "logits/chosen": -0.8274625539779663, "logits/rejected": -0.8282160758972168, "logps/chosen": -557.6765747070312, "logps/ref_chosen": -280.50909423828125, "logps/ref_rejected": -276.8252258300781, "logps/rejected": -668.2254028320312, "loss": 4.7899, "margin_dpo/margin_mean": 114.23271942138672, "margin_dpo/margin_std": 195.52053833007812, "step": 425 }, { "KL/chosen_KL_mean": -282.42633056640625, "KL/mean": -342.55267333984375, "KL/rejected_KL_mean": -402.678955078125, "KL/std": 177.6331329345703, "epoch": 0.8921465968586387, "fcm_dpo/beta": 0.002310623647645116, "fcm_dpo/delta": 0.025467144325375557, "fcm_dpo/margin": 120.25263977050781, "fcm_dpo/q_t": 0.4347745478153229, "grad_norm": 77.7251205444336, "learning_rate": 1.7908016745981856e-08, "logits/chosen": -0.8655619621276855, "logits/rejected": -0.8427607417106628, "logps/chosen": -575.2116088867188, "logps/ref_chosen": -292.78521728515625, "logps/ref_rejected": -255.62698364257812, "logps/rejected": -658.3059692382812, "loss": 4.7273, "margin_dpo/margin_mean": 120.25263977050781, "margin_dpo/margin_std": 196.75177001953125, "step": 426 }, { "KL/chosen_KL_mean": -276.68865966796875, "KL/mean": -362.1603088378906, "KL/rejected_KL_mean": -447.6319274902344, "KL/std": 200.92494201660156, "epoch": 0.8942408376963351, "fcm_dpo/beta": 0.002332492033019662, "fcm_dpo/delta": -0.03227193281054497, "fcm_dpo/margin": 170.94326782226562, "fcm_dpo/q_t": 0.40855950117111206, "grad_norm": 60.52691650390625, "learning_rate": 1.7233819424956247e-08, "logits/chosen": -0.8385801315307617, "logits/rejected": -0.7992298603057861, "logps/chosen": -565.4573974609375, "logps/ref_chosen": -288.7687072753906, "logps/ref_rejected": -268.4986572265625, "logps/rejected": -716.130615234375, "loss": 4.4378, "margin_dpo/margin_mean": 170.94326782226562, "margin_dpo/margin_std": 245.36048889160156, "step": 427 }, { "KL/chosen_KL_mean": -311.8055419921875, "KL/mean": -385.68841552734375, "KL/rejected_KL_mean": -459.5712890625, "KL/std": 200.10256958007812, "epoch": 0.8963350785340314, "fcm_dpo/beta": 0.002250659978017211, "fcm_dpo/delta": -0.008135773241519928, "fcm_dpo/margin": 147.7657470703125, "fcm_dpo/q_t": 0.42285412549972534, "grad_norm": 60.747161865234375, "learning_rate": 1.6572104647786245e-08, "logits/chosen": -0.7936046123504639, "logits/rejected": -0.8243399858474731, "logps/chosen": -607.3265380859375, "logps/ref_chosen": -295.5209655761719, "logps/ref_rejected": -275.71026611328125, "logps/rejected": -735.2815551757812, "loss": 4.5594, "margin_dpo/margin_mean": 147.7657470703125, "margin_dpo/margin_std": 209.39695739746094, "step": 428 }, { "KL/chosen_KL_mean": -295.0547790527344, "KL/mean": -354.29083251953125, "KL/rejected_KL_mean": -413.52691650390625, "KL/std": 192.97251892089844, "epoch": 0.8984293193717278, "fcm_dpo/beta": 0.0022739970590919256, "fcm_dpo/delta": 0.019945936277508736, "fcm_dpo/margin": 118.47213745117188, "fcm_dpo/q_t": 0.4369737505912781, "grad_norm": 73.8173599243164, "learning_rate": 1.5922907900227017e-08, "logits/chosen": -0.8066105246543884, "logits/rejected": -0.8140644431114197, "logps/chosen": -569.4470825195312, "logps/ref_chosen": -274.392333984375, "logps/ref_rejected": -258.574462890625, "logps/rejected": -672.1014404296875, "loss": 4.837, "margin_dpo/margin_mean": 118.47213745117188, "margin_dpo/margin_std": 233.86431884765625, "step": 429 }, { "KL/chosen_KL_mean": -285.7272033691406, "KL/mean": -344.11572265625, "KL/rejected_KL_mean": -402.5043029785156, "KL/std": 180.56253051757812, "epoch": 0.900523560209424, "fcm_dpo/beta": 0.002290163654834032, "fcm_dpo/delta": 0.002819681540131569, "fcm_dpo/margin": 116.77705383300781, "fcm_dpo/q_t": 0.43817266821861267, "grad_norm": 60.57477951049805, "learning_rate": 1.5286263996730026e-08, "logits/chosen": -0.8805914521217346, "logits/rejected": -0.8473300337791443, "logps/chosen": -574.46630859375, "logps/ref_chosen": -288.7391357421875, "logps/ref_rejected": -268.6106262207031, "logps/rejected": -671.1148681640625, "loss": 4.7739, "margin_dpo/margin_mean": 116.77705383300781, "margin_dpo/margin_std": 204.91702270507812, "step": 430 }, { "KL/chosen_KL_mean": -282.6735534667969, "KL/mean": -331.5006408691406, "KL/rejected_KL_mean": -380.3277893066406, "KL/std": 168.19338989257812, "epoch": 0.9026178010471204, "fcm_dpo/beta": 0.002297281287610531, "fcm_dpo/delta": -0.006534409709274769, "fcm_dpo/margin": 97.65420532226562, "fcm_dpo/q_t": 0.4476650655269623, "grad_norm": 55.633140563964844, "learning_rate": 1.4662207078575684e-08, "logits/chosen": -0.8622323870658875, "logits/rejected": -0.8313441872596741, "logps/chosen": -558.3983154296875, "logps/ref_chosen": -275.7247314453125, "logps/ref_rejected": -268.91729736328125, "logps/rejected": -649.2450561523438, "loss": 4.9127, "margin_dpo/margin_mean": 97.6541976928711, "margin_dpo/margin_std": 194.69580078125, "step": 431 }, { "KL/chosen_KL_mean": -268.3981628417969, "KL/mean": -328.0491027832031, "KL/rejected_KL_mean": -387.7000732421875, "KL/std": 181.62322998046875, "epoch": 0.9047120418848168, "fcm_dpo/beta": 0.002279035747051239, "fcm_dpo/delta": 0.004813372157514095, "fcm_dpo/margin": 119.30191040039062, "fcm_dpo/q_t": 0.4380475580692291, "grad_norm": 44.37526321411133, "learning_rate": 1.40507706120426e-08, "logits/chosen": -0.8959624767303467, "logits/rejected": -0.8759222626686096, "logps/chosen": -559.8182373046875, "logps/ref_chosen": -291.42010498046875, "logps/ref_rejected": -255.48202514648438, "logps/rejected": -643.1820678710938, "loss": 4.7857, "margin_dpo/margin_mean": 119.30191802978516, "margin_dpo/margin_std": 219.4633331298828, "step": 432 }, { "KL/chosen_KL_mean": -298.1172790527344, "KL/mean": -351.10052490234375, "KL/rejected_KL_mean": -404.083740234375, "KL/std": 174.59242248535156, "epoch": 0.9068062827225131, "fcm_dpo/beta": 0.002294522011652589, "fcm_dpo/delta": 0.004826649557799101, "fcm_dpo/margin": 105.96644592285156, "fcm_dpo/q_t": 0.44354549050331116, "grad_norm": 57.427310943603516, "learning_rate": 1.345198738661285e-08, "logits/chosen": -0.8302546143531799, "logits/rejected": -0.8239802718162537, "logps/chosen": -544.3441772460938, "logps/ref_chosen": -246.2268829345703, "logps/ref_rejected": -253.65924072265625, "logps/rejected": -657.7430419921875, "loss": 4.8246, "margin_dpo/margin_mean": 105.96646118164062, "margin_dpo/margin_std": 192.65756225585938, "step": 433 }, { "KL/chosen_KL_mean": -320.85400390625, "KL/mean": -370.788818359375, "KL/rejected_KL_mean": -420.7236633300781, "KL/std": 183.7316131591797, "epoch": 0.9089005235602095, "fcm_dpo/beta": 0.002314929384738207, "fcm_dpo/delta": 0.020489579066634178, "fcm_dpo/margin": 99.86967468261719, "fcm_dpo/q_t": 0.44652947783470154, "grad_norm": 60.26441955566406, "learning_rate": 1.2865889513213628e-08, "logits/chosen": -0.8335898518562317, "logits/rejected": -0.8363132476806641, "logps/chosen": -616.3158569335938, "logps/ref_chosen": -295.4618225097656, "logps/ref_rejected": -256.2254333496094, "logps/rejected": -676.9491577148438, "loss": 4.9074, "margin_dpo/margin_mean": 99.86965942382812, "margin_dpo/margin_std": 205.02581787109375, "step": 434 }, { "KL/chosen_KL_mean": -272.2348937988281, "KL/mean": -340.4631652832031, "KL/rejected_KL_mean": -408.6913757324219, "KL/std": 186.8828125, "epoch": 0.9109947643979057, "fcm_dpo/beta": 0.002351157832890749, "fcm_dpo/delta": -0.009057372808456421, "fcm_dpo/margin": 136.4564971923828, "fcm_dpo/q_t": 0.42885512113571167, "grad_norm": 69.38368225097656, "learning_rate": 1.2292508422495157e-08, "logits/chosen": -0.8288702368736267, "logits/rejected": -0.8155333995819092, "logps/chosen": -532.9733276367188, "logps/ref_chosen": -260.7384033203125, "logps/ref_rejected": -248.5688018798828, "logps/rejected": -657.2601928710938, "loss": 4.6169, "margin_dpo/margin_mean": 136.4564971923828, "margin_dpo/margin_std": 204.81259155273438, "step": 435 }, { "KL/chosen_KL_mean": -311.447265625, "KL/mean": -359.7391662597656, "KL/rejected_KL_mean": -408.03106689453125, "KL/std": 206.40972900390625, "epoch": 0.9130890052356021, "fcm_dpo/beta": 0.00231517874635756, "fcm_dpo/delta": 0.004760443232953548, "fcm_dpo/margin": 96.58380889892578, "fcm_dpo/q_t": 0.4519736170768738, "grad_norm": 69.52943420410156, "learning_rate": 1.1731874863145142e-08, "logits/chosen": -0.8272717595100403, "logits/rejected": -0.8191923499107361, "logps/chosen": -630.769775390625, "logps/ref_chosen": -319.3224792480469, "logps/ref_rejected": -299.30322265625, "logps/rejected": -707.3342895507812, "loss": 4.9815, "margin_dpo/margin_mean": 96.58382415771484, "margin_dpo/margin_std": 230.41339111328125, "step": 436 }, { "KL/chosen_KL_mean": -283.3936767578125, "KL/mean": -354.4276123046875, "KL/rejected_KL_mean": -425.4615783691406, "KL/std": 201.54818725585938, "epoch": 0.9151832460732985, "fcm_dpo/beta": 0.0023110881447792053, "fcm_dpo/delta": -0.01546804141253233, "fcm_dpo/margin": 142.06787109375, "fcm_dpo/q_t": 0.42517486214637756, "grad_norm": 48.614498138427734, "learning_rate": 1.118401890024001e-08, "logits/chosen": -0.8567025065422058, "logits/rejected": -0.8355041742324829, "logps/chosen": -562.2224731445312, "logps/ref_chosen": -278.82879638671875, "logps/ref_rejected": -272.55303955078125, "logps/rejected": -698.0146484375, "loss": 4.6208, "margin_dpo/margin_mean": 142.06787109375, "margin_dpo/margin_std": 224.5172882080078, "step": 437 }, { "KL/chosen_KL_mean": -299.3929443359375, "KL/mean": -329.7269592285156, "KL/rejected_KL_mean": -360.06103515625, "KL/std": 184.3596649169922, "epoch": 0.9172774869109948, "fcm_dpo/beta": 0.0022973110899329185, "fcm_dpo/delta": -0.003291675355285406, "fcm_dpo/margin": 60.66808319091797, "fcm_dpo/q_t": 0.4677131474018097, "grad_norm": 77.67984771728516, "learning_rate": 1.06489699136324e-08, "logits/chosen": -0.8413230776786804, "logits/rejected": -0.851213812828064, "logps/chosen": -558.7120361328125, "logps/ref_chosen": -259.31903076171875, "logps/ref_rejected": -240.99581909179688, "logps/rejected": -601.0568237304688, "loss": 5.2507, "margin_dpo/margin_mean": 60.66808319091797, "margin_dpo/margin_std": 206.700439453125, "step": 438 }, { "KL/chosen_KL_mean": -289.4859619140625, "KL/mean": -349.2972412109375, "KL/rejected_KL_mean": -409.1085510253906, "KL/std": 174.05442810058594, "epoch": 0.9193717277486911, "fcm_dpo/beta": 0.0022938335314393044, "fcm_dpo/delta": -0.0010132784955203533, "fcm_dpo/margin": 119.62257385253906, "fcm_dpo/q_t": 0.43509790301322937, "grad_norm": 79.5361099243164, "learning_rate": 1.0126756596375685e-08, "logits/chosen": -0.8212806582450867, "logits/rejected": -0.8165966868400574, "logps/chosen": -546.6103515625, "logps/ref_chosen": -257.1243896484375, "logps/ref_rejected": -243.20416259765625, "logps/rejected": -652.3126220703125, "loss": 4.7176, "margin_dpo/margin_mean": 119.62257385253906, "margin_dpo/margin_std": 190.63784790039062, "step": 439 }, { "KL/chosen_KL_mean": -306.56158447265625, "KL/mean": -353.7540283203125, "KL/rejected_KL_mean": -400.946533203125, "KL/std": 162.243408203125, "epoch": 0.9214659685863874, "fcm_dpo/beta": 0.0023019982036203146, "fcm_dpo/delta": 0.015137860551476479, "fcm_dpo/margin": 94.38501739501953, "fcm_dpo/q_t": 0.4487031102180481, "grad_norm": 66.66883850097656, "learning_rate": 9.617406953185136e-09, "logits/chosen": -0.8774101734161377, "logits/rejected": -0.8676927089691162, "logps/chosen": -614.0930786132812, "logps/ref_chosen": -307.5315246582031, "logps/ref_rejected": -264.3540954589844, "logps/rejected": -665.3006591796875, "loss": 4.9081, "margin_dpo/margin_mean": 94.38501739501953, "margin_dpo/margin_std": 178.12355041503906, "step": 440 }, { "KL/chosen_KL_mean": -290.2575378417969, "KL/mean": -360.6673583984375, "KL/rejected_KL_mean": -431.07720947265625, "KL/std": 180.72862243652344, "epoch": 0.9235602094240838, "fcm_dpo/beta": 0.0023078853264451027, "fcm_dpo/delta": -0.01445065625011921, "fcm_dpo/margin": 140.8197479248047, "fcm_dpo/q_t": 0.4237060546875, "grad_norm": 75.2840805053711, "learning_rate": 9.12094829893642e-09, "logits/chosen": -0.8152427077293396, "logits/rejected": -0.8081190586090088, "logps/chosen": -600.239501953125, "logps/ref_chosen": -309.9819641113281, "logps/ref_rejected": -297.4968566894531, "logps/rejected": -728.5740966796875, "loss": 4.5509, "margin_dpo/margin_mean": 140.8197479248047, "margin_dpo/margin_std": 188.0748748779297, "step": 441 }, { "KL/chosen_KL_mean": -317.641357421875, "KL/mean": -365.92034912109375, "KL/rejected_KL_mean": -414.19927978515625, "KL/std": 183.26416015625, "epoch": 0.9256544502617801, "fcm_dpo/beta": 0.002325834007933736, "fcm_dpo/delta": 0.026067446917295456, "fcm_dpo/margin": 96.55789184570312, "fcm_dpo/q_t": 0.44816458225250244, "grad_norm": 75.7071533203125, "learning_rate": 8.637407257200496e-09, "logits/chosen": -0.8991624712944031, "logits/rejected": -0.8569780588150024, "logps/chosen": -596.62060546875, "logps/ref_chosen": -278.9791564941406, "logps/ref_rejected": -242.87310791015625, "logps/rejected": -657.0723876953125, "loss": 4.9514, "margin_dpo/margin_mean": 96.55789184570312, "margin_dpo/margin_std": 217.6230010986328, "step": 442 }, { "KL/chosen_KL_mean": -293.0943298339844, "KL/mean": -347.28619384765625, "KL/rejected_KL_mean": -401.47808837890625, "KL/std": 162.5325927734375, "epoch": 0.9277486910994764, "fcm_dpo/beta": 0.0023306708317250013, "fcm_dpo/delta": -0.019472528249025345, "fcm_dpo/margin": 108.38372802734375, "fcm_dpo/q_t": 0.439905047416687, "grad_norm": 51.0541877746582, "learning_rate": 8.166809758815895e-09, "logits/chosen": -0.7947054505348206, "logits/rejected": -0.8145523071289062, "logps/chosen": -566.6533813476562, "logps/ref_chosen": -273.5590515136719, "logps/ref_rejected": -264.0199279785156, "logps/rejected": -665.4979858398438, "loss": 4.8112, "margin_dpo/margin_mean": 108.38371276855469, "margin_dpo/margin_std": 190.74830627441406, "step": 443 }, { "KL/chosen_KL_mean": -290.04559326171875, "KL/mean": -350.908203125, "KL/rejected_KL_mean": -411.7707824707031, "KL/std": 195.36392211914062, "epoch": 0.9298429319371728, "fcm_dpo/beta": 0.002323201159015298, "fcm_dpo/delta": 0.019285790622234344, "fcm_dpo/margin": 121.72520446777344, "fcm_dpo/q_t": 0.4358820617198944, "grad_norm": 63.36241149902344, "learning_rate": 7.709181040498253e-09, "logits/chosen": -0.8152733445167542, "logits/rejected": -0.8027286529541016, "logps/chosen": -588.1897583007812, "logps/ref_chosen": -298.1441955566406, "logps/ref_rejected": -268.0572814941406, "logps/rejected": -679.8280639648438, "loss": 4.7722, "margin_dpo/margin_mean": 121.72520446777344, "margin_dpo/margin_std": 227.64364624023438, "step": 444 }, { "KL/chosen_KL_mean": -290.1022033691406, "KL/mean": -345.72955322265625, "KL/rejected_KL_mean": -401.35687255859375, "KL/std": 185.03326416015625, "epoch": 0.9319371727748691, "fcm_dpo/beta": 0.0023218824062496424, "fcm_dpo/delta": -0.025263221934437752, "fcm_dpo/margin": 111.25466918945312, "fcm_dpo/q_t": 0.4408957064151764, "grad_norm": 66.18413543701172, "learning_rate": 7.2645456434869965e-09, "logits/chosen": -0.8810672163963318, "logits/rejected": -0.8841793537139893, "logps/chosen": -544.6429443359375, "logps/ref_chosen": -254.54067993164062, "logps/ref_rejected": -264.2445983886719, "logps/rejected": -665.6015014648438, "loss": 4.8478, "margin_dpo/margin_mean": 111.25466918945312, "margin_dpo/margin_std": 212.44100952148438, "step": 445 }, { "KL/chosen_KL_mean": -286.95159912109375, "KL/mean": -347.5931396484375, "KL/rejected_KL_mean": -408.2346496582031, "KL/std": 178.93850708007812, "epoch": 0.9340314136125655, "fcm_dpo/beta": 0.0022728617768734694, "fcm_dpo/delta": -0.010188797488808632, "fcm_dpo/margin": 121.28301239013672, "fcm_dpo/q_t": 0.43673738837242126, "grad_norm": 47.07920837402344, "learning_rate": 6.832927412229017e-09, "logits/chosen": -0.7991313338279724, "logits/rejected": -0.803611159324646, "logps/chosen": -593.674072265625, "logps/ref_chosen": -306.72247314453125, "logps/ref_rejected": -266.3735656738281, "logps/rejected": -674.6082153320312, "loss": 4.755, "margin_dpo/margin_mean": 121.28301239013672, "margin_dpo/margin_std": 211.34518432617188, "step": 446 }, { "KL/chosen_KL_mean": -279.7216491699219, "KL/mean": -348.5470886230469, "KL/rejected_KL_mean": -417.3725280761719, "KL/std": 182.1820526123047, "epoch": 0.9361256544502617, "fcm_dpo/beta": 0.0022493728902190924, "fcm_dpo/delta": -0.012385351583361626, "fcm_dpo/margin": 137.65090942382812, "fcm_dpo/q_t": 0.42796117067337036, "grad_norm": 61.105281829833984, "learning_rate": 6.414349493100129e-09, "logits/chosen": -0.8082762956619263, "logits/rejected": -0.7996385097503662, "logps/chosen": -540.2389526367188, "logps/ref_chosen": -260.51727294921875, "logps/ref_rejected": -236.47061157226562, "logps/rejected": -653.8431396484375, "loss": 4.6265, "margin_dpo/margin_mean": 137.65089416503906, "margin_dpo/margin_std": 203.60350036621094, "step": 447 }, { "KL/chosen_KL_mean": -301.380859375, "KL/mean": -365.99871826171875, "KL/rejected_KL_mean": -430.6165466308594, "KL/std": 176.27540588378906, "epoch": 0.9382198952879581, "fcm_dpo/beta": 0.0022281836718320847, "fcm_dpo/delta": -0.011953938752412796, "fcm_dpo/margin": 129.2357177734375, "fcm_dpo/q_t": 0.43249234557151794, "grad_norm": 69.26811981201172, "learning_rate": 6.0088343331638756e-09, "logits/chosen": -0.809171736240387, "logits/rejected": -0.798628568649292, "logps/chosen": -570.1679077148438, "logps/ref_chosen": -268.78704833984375, "logps/ref_rejected": -262.1703796386719, "logps/rejected": -692.7869873046875, "loss": 4.6706, "margin_dpo/margin_mean": 129.23573303222656, "margin_dpo/margin_std": 189.93157958984375, "step": 448 }, { "KL/chosen_KL_mean": -287.81689453125, "KL/mean": -346.68743896484375, "KL/rejected_KL_mean": -405.5579833984375, "KL/std": 176.11363220214844, "epoch": 0.9403141361256544, "fcm_dpo/beta": 0.0022455663420259953, "fcm_dpo/delta": 0.027673836797475815, "fcm_dpo/margin": 117.74109649658203, "fcm_dpo/q_t": 0.43690261244773865, "grad_norm": 67.57733154296875, "learning_rate": 5.616403678967624e-09, "logits/chosen": -0.9088640809059143, "logits/rejected": -0.8916957378387451, "logps/chosen": -618.768310546875, "logps/ref_chosen": -330.9514465332031, "logps/ref_rejected": -239.76974487304688, "logps/rejected": -645.3276977539062, "loss": 4.7859, "margin_dpo/margin_mean": 117.7410888671875, "margin_dpo/margin_std": 208.05458068847656, "step": 449 }, { "KL/chosen_KL_mean": -308.811767578125, "KL/mean": -362.2073974609375, "KL/rejected_KL_mean": -415.60302734375, "KL/std": 188.40576171875, "epoch": 0.9424083769633508, "fcm_dpo/beta": 0.0022853235714137554, "fcm_dpo/delta": 0.005974981002509594, "fcm_dpo/margin": 106.791259765625, "fcm_dpo/q_t": 0.4430885910987854, "grad_norm": 47.89699935913086, "learning_rate": 5.2370785753763356e-09, "logits/chosen": -0.8070337772369385, "logits/rejected": -0.7972606420516968, "logps/chosen": -593.0771484375, "logps/ref_chosen": -284.26544189453125, "logps/ref_rejected": -250.5401611328125, "logps/rejected": -666.1432495117188, "loss": 4.8339, "margin_dpo/margin_mean": 106.791259765625, "margin_dpo/margin_std": 189.6084442138672, "step": 450 }, { "KL/chosen_KL_mean": -319.8152770996094, "KL/mean": -364.75592041015625, "KL/rejected_KL_mean": -409.6966247558594, "KL/std": 178.8978729248047, "epoch": 0.9445026178010472, "fcm_dpo/beta": 0.002270276425406337, "fcm_dpo/delta": -0.012955044396221638, "fcm_dpo/margin": 89.88134765625, "fcm_dpo/q_t": 0.4515979290008545, "grad_norm": 90.45854949951172, "learning_rate": 4.8708793644441086e-09, "logits/chosen": -0.8183082938194275, "logits/rejected": -0.7841957807540894, "logps/chosen": -622.1361694335938, "logps/ref_chosen": -302.3209228515625, "logps/ref_rejected": -254.09747314453125, "logps/rejected": -663.7941284179688, "loss": 4.9745, "margin_dpo/margin_mean": 89.88135528564453, "margin_dpo/margin_std": 191.08470153808594, "step": 451 }, { "KL/chosen_KL_mean": -290.7073059082031, "KL/mean": -364.1423034667969, "KL/rejected_KL_mean": -437.57733154296875, "KL/std": 219.24545288085938, "epoch": 0.9465968586387434, "fcm_dpo/beta": 0.002213032217696309, "fcm_dpo/delta": -0.030287623405456543, "fcm_dpo/margin": 146.87002563476562, "fcm_dpo/q_t": 0.4293460249900818, "grad_norm": 59.19186782836914, "learning_rate": 4.517825684323323e-09, "logits/chosen": -0.8647107481956482, "logits/rejected": -0.8348664045333862, "logps/chosen": -590.0994262695312, "logps/ref_chosen": -299.39215087890625, "logps/ref_rejected": -284.3475036621094, "logps/rejected": -721.9248046875, "loss": 4.6751, "margin_dpo/margin_mean": 146.87001037597656, "margin_dpo/margin_std": 258.558349609375, "step": 452 }, { "KL/chosen_KL_mean": -314.28314208984375, "KL/mean": -375.9656066894531, "KL/rejected_KL_mean": -437.6480712890625, "KL/std": 184.8981170654297, "epoch": 0.9486910994764398, "fcm_dpo/beta": 0.0022319061681628227, "fcm_dpo/delta": 0.018467098474502563, "fcm_dpo/margin": 123.36490631103516, "fcm_dpo/q_t": 0.4363798201084137, "grad_norm": 54.6486930847168, "learning_rate": 4.1779364682113794e-09, "logits/chosen": -0.8098393082618713, "logits/rejected": -0.7993586659431458, "logps/chosen": -638.934814453125, "logps/ref_chosen": -324.6517028808594, "logps/ref_rejected": -304.1527099609375, "logps/rejected": -741.80078125, "loss": 4.7523, "margin_dpo/margin_mean": 123.36491394042969, "margin_dpo/margin_std": 219.076904296875, "step": 453 }, { "KL/chosen_KL_mean": -299.9391174316406, "KL/mean": -372.45458984375, "KL/rejected_KL_mean": -444.969970703125, "KL/std": 207.7763671875, "epoch": 0.9507853403141361, "fcm_dpo/beta": 0.002227374352514744, "fcm_dpo/delta": -0.015013519674539566, "fcm_dpo/margin": 145.03086853027344, "fcm_dpo/q_t": 0.4254119396209717, "grad_norm": 63.96196365356445, "learning_rate": 3.851229943335393e-09, "logits/chosen": -0.8499652743339539, "logits/rejected": -0.8552207946777344, "logps/chosen": -599.5509033203125, "logps/ref_chosen": -299.6117248535156, "logps/ref_rejected": -303.74224853515625, "logps/rejected": -748.7122802734375, "loss": 4.6208, "margin_dpo/margin_mean": 145.03086853027344, "margin_dpo/margin_std": 229.23269653320312, "step": 454 }, { "KL/chosen_KL_mean": -292.7987976074219, "KL/mean": -337.0995178222656, "KL/rejected_KL_mean": -381.40020751953125, "KL/std": 183.76556396484375, "epoch": 0.9528795811518325, "fcm_dpo/beta": 0.0022193007171154022, "fcm_dpo/delta": 0.00029542212723754346, "fcm_dpo/margin": 88.60137939453125, "fcm_dpo/q_t": 0.45545342564582825, "grad_norm": 84.49998474121094, "learning_rate": 3.5377236299748147e-09, "logits/chosen": -0.8165888786315918, "logits/rejected": -0.8248051404953003, "logps/chosen": -566.4105224609375, "logps/ref_chosen": -273.6116943359375, "logps/ref_rejected": -274.4293518066406, "logps/rejected": -655.82958984375, "loss": 5.0176, "margin_dpo/margin_mean": 88.60137939453125, "margin_dpo/margin_std": 203.42779541015625, "step": 455 }, { "KL/chosen_KL_mean": -318.74468994140625, "KL/mean": -383.7198791503906, "KL/rejected_KL_mean": -448.6950988769531, "KL/std": 208.6256866455078, "epoch": 0.9549738219895288, "fcm_dpo/beta": 0.0022203458938747644, "fcm_dpo/delta": -0.008752606809139252, "fcm_dpo/margin": 129.9503936767578, "fcm_dpo/q_t": 0.4367368221282959, "grad_norm": 81.42244720458984, "learning_rate": 3.2374343405217884e-09, "logits/chosen": -0.7438274621963501, "logits/rejected": -0.7582361698150635, "logps/chosen": -640.9166259765625, "logps/ref_chosen": -322.17193603515625, "logps/ref_rejected": -294.54461669921875, "logps/rejected": -743.2396240234375, "loss": 4.7733, "margin_dpo/margin_mean": 129.95040893554688, "margin_dpo/margin_std": 247.870849609375, "step": 456 }, { "KL/chosen_KL_mean": -315.93475341796875, "KL/mean": -383.9335632324219, "KL/rejected_KL_mean": -451.9323425292969, "KL/std": 205.77975463867188, "epoch": 0.9570680628272251, "fcm_dpo/beta": 0.0022161712404340506, "fcm_dpo/delta": 0.010278364643454552, "fcm_dpo/margin": 135.99758911132812, "fcm_dpo/q_t": 0.4274827241897583, "grad_norm": 58.002906799316406, "learning_rate": 2.9503781785795713e-09, "logits/chosen": -0.8158080577850342, "logits/rejected": -0.8166444301605225, "logps/chosen": -623.73095703125, "logps/ref_chosen": -307.7962341308594, "logps/ref_rejected": -274.5501403808594, "logps/rejected": -726.4824829101562, "loss": 4.7231, "margin_dpo/margin_mean": 135.99757385253906, "margin_dpo/margin_std": 236.98309326171875, "step": 457 }, { "KL/chosen_KL_mean": -307.9447021484375, "KL/mean": -363.36785888671875, "KL/rejected_KL_mean": -418.79107666015625, "KL/std": 202.58853149414062, "epoch": 0.9591623036649215, "fcm_dpo/beta": 0.002248093718662858, "fcm_dpo/delta": 0.02113131433725357, "fcm_dpo/margin": 110.84642028808594, "fcm_dpo/q_t": 0.4432235360145569, "grad_norm": 47.68861770629883, "learning_rate": 2.6765705380989432e-09, "logits/chosen": -0.8353589177131653, "logits/rejected": -0.8087614178657532, "logps/chosen": -604.976318359375, "logps/ref_chosen": -297.0316467285156, "logps/ref_rejected": -276.1112365722656, "logps/rejected": -694.9022827148438, "loss": 4.8799, "margin_dpo/margin_mean": 110.84642028808594, "margin_dpo/margin_std": 226.8346710205078, "step": 458 }, { "KL/chosen_KL_mean": -289.771484375, "KL/mean": -339.0989990234375, "KL/rejected_KL_mean": -388.4265441894531, "KL/std": 170.22535705566406, "epoch": 0.9612565445026178, "fcm_dpo/beta": 0.002288884948939085, "fcm_dpo/delta": 0.03183550387620926, "fcm_dpo/margin": 98.65508270263672, "fcm_dpo/q_t": 0.44840818643569946, "grad_norm": 57.28891372680664, "learning_rate": 2.416026102552732e-09, "logits/chosen": -0.8987852931022644, "logits/rejected": -0.8900645971298218, "logps/chosen": -583.2967529296875, "logps/ref_chosen": -293.5252990722656, "logps/ref_rejected": -289.30126953125, "logps/rejected": -677.7278442382812, "loss": 4.9211, "margin_dpo/margin_mean": 98.65508270263672, "margin_dpo/margin_std": 204.67257690429688, "step": 459 }, { "KL/chosen_KL_mean": -304.1756591796875, "KL/mean": -353.0399169921875, "KL/rejected_KL_mean": -401.90423583984375, "KL/std": 166.08837890625, "epoch": 0.9633507853403142, "fcm_dpo/beta": 0.0023258451838046312, "fcm_dpo/delta": 0.0077877165749669075, "fcm_dpo/margin": 97.72859954833984, "fcm_dpo/q_t": 0.44643843173980713, "grad_norm": 67.08515930175781, "learning_rate": 2.168758844148272e-09, "logits/chosen": -0.8612726330757141, "logits/rejected": -0.8576234579086304, "logps/chosen": -622.9559936523438, "logps/ref_chosen": -318.7803649902344, "logps/ref_rejected": -258.7906799316406, "logps/rejected": -660.6948852539062, "loss": 4.9022, "margin_dpo/margin_mean": 97.72859954833984, "margin_dpo/margin_std": 199.7197265625, "step": 460 }, { "KL/chosen_KL_mean": -294.57061767578125, "KL/mean": -356.40191650390625, "KL/rejected_KL_mean": -418.23309326171875, "KL/std": 190.1246795654297, "epoch": 0.9654450261780104, "fcm_dpo/beta": 0.002357393503189087, "fcm_dpo/delta": 0.020651711151003838, "fcm_dpo/margin": 123.66246032714844, "fcm_dpo/q_t": 0.4340013265609741, "grad_norm": 62.43252182006836, "learning_rate": 1.9347820230782295e-09, "logits/chosen": -0.817859947681427, "logits/rejected": -0.838330864906311, "logps/chosen": -538.4805908203125, "logps/ref_chosen": -243.9099884033203, "logps/ref_rejected": -232.6382293701172, "logps/rejected": -650.871337890625, "loss": 4.7388, "margin_dpo/margin_mean": 123.66246032714844, "margin_dpo/margin_std": 221.1026611328125, "step": 461 }, { "KL/chosen_KL_mean": -294.6795959472656, "KL/mean": -362.9072570800781, "KL/rejected_KL_mean": -431.13494873046875, "KL/std": 176.5676727294922, "epoch": 0.9675392670157068, "fcm_dpo/beta": 0.002386984182521701, "fcm_dpo/delta": 0.009113818407058716, "fcm_dpo/margin": 136.45535278320312, "fcm_dpo/q_t": 0.4248298108577728, "grad_norm": 64.27290344238281, "learning_rate": 1.7141081868094209e-09, "logits/chosen": -0.8587002158164978, "logits/rejected": -0.8031215667724609, "logps/chosen": -638.7705078125, "logps/ref_chosen": -344.09100341796875, "logps/ref_rejected": -252.45037841796875, "logps/rejected": -683.5853271484375, "loss": 4.5764, "margin_dpo/margin_mean": 136.45535278320312, "margin_dpo/margin_std": 198.38424682617188, "step": 462 }, { "KL/chosen_KL_mean": -298.1247253417969, "KL/mean": -349.39569091796875, "KL/rejected_KL_mean": -400.66668701171875, "KL/std": 196.35546875, "epoch": 0.9696335078534032, "fcm_dpo/beta": 0.0023907949216663837, "fcm_dpo/delta": -0.0031707417219877243, "fcm_dpo/margin": 102.54196166992188, "fcm_dpo/q_t": 0.4443747103214264, "grad_norm": 63.04395294189453, "learning_rate": 1.5067491694100153e-09, "logits/chosen": -0.841509222984314, "logits/rejected": -0.7965072393417358, "logps/chosen": -595.2672119140625, "logps/ref_chosen": -297.1424560546875, "logps/ref_rejected": -234.0208282470703, "logps/rejected": -634.6875610351562, "loss": 4.9414, "margin_dpo/margin_mean": 102.54196166992188, "margin_dpo/margin_std": 233.15066528320312, "step": 463 }, { "KL/chosen_KL_mean": -303.274658203125, "KL/mean": -358.65533447265625, "KL/rejected_KL_mean": -414.0360412597656, "KL/std": 182.3558807373047, "epoch": 0.9717277486910995, "fcm_dpo/beta": 0.002398580312728882, "fcm_dpo/delta": 0.004773720167577267, "fcm_dpo/margin": 110.76134490966797, "fcm_dpo/q_t": 0.43780282139778137, "grad_norm": 74.20742797851562, "learning_rate": 1.3127160909147672e-09, "logits/chosen": -0.8260834217071533, "logits/rejected": -0.8394900560379028, "logps/chosen": -568.9854125976562, "logps/ref_chosen": -265.71075439453125, "logps/ref_rejected": -256.4108581542969, "logps/rejected": -670.4468994140625, "loss": 4.7798, "margin_dpo/margin_mean": 110.76134490966797, "margin_dpo/margin_std": 196.59963989257812, "step": 464 }, { "KL/chosen_KL_mean": -284.39501953125, "KL/mean": -354.4608459472656, "KL/rejected_KL_mean": -424.52667236328125, "KL/std": 174.48228454589844, "epoch": 0.9738219895287958, "fcm_dpo/beta": 0.0023324608337134123, "fcm_dpo/delta": -0.03364231437444687, "fcm_dpo/margin": 140.13165283203125, "fcm_dpo/q_t": 0.42524397373199463, "grad_norm": 59.864524841308594, "learning_rate": 1.1320193567288527e-09, "logits/chosen": -0.9016032814979553, "logits/rejected": -0.8606617450714111, "logps/chosen": -577.5477294921875, "logps/ref_chosen": -293.1527404785156, "logps/ref_rejected": -293.70947265625, "logps/rejected": -718.2361450195312, "loss": 4.5923, "margin_dpo/margin_mean": 140.13165283203125, "margin_dpo/margin_std": 205.0211181640625, "step": 465 }, { "KL/chosen_KL_mean": -270.7398986816406, "KL/mean": -341.1874084472656, "KL/rejected_KL_mean": -411.63494873046875, "KL/std": 175.04595947265625, "epoch": 0.9759162303664921, "fcm_dpo/beta": 0.002325007924810052, "fcm_dpo/delta": 0.006551696918904781, "fcm_dpo/margin": 140.89501953125, "fcm_dpo/q_t": 0.425047367811203, "grad_norm": 67.3387680053711, "learning_rate": 9.64668657069706e-10, "logits/chosen": -0.8157132863998413, "logits/rejected": -0.7559111714363098, "logps/chosen": -532.2174682617188, "logps/ref_chosen": -261.4775695800781, "logps/ref_rejected": -248.36282348632812, "logps/rejected": -659.9977416992188, "loss": 4.5809, "margin_dpo/margin_mean": 140.89501953125, "margin_dpo/margin_std": 209.25616455078125, "step": 466 }, { "KL/chosen_KL_mean": -294.3692932128906, "KL/mean": -352.0955810546875, "KL/rejected_KL_mean": -409.8218688964844, "KL/std": 185.36192321777344, "epoch": 0.9780104712041885, "fcm_dpo/beta": 0.0023117591626942158, "fcm_dpo/delta": -0.017675073817372322, "fcm_dpo/margin": 115.45263671875, "fcm_dpo/q_t": 0.43935906887054443, "grad_norm": 70.38238525390625, "learning_rate": 8.106729664475176e-10, "logits/chosen": -0.8092154860496521, "logits/rejected": -0.7998495101928711, "logps/chosen": -560.7235107421875, "logps/ref_chosen": -266.354248046875, "logps/ref_rejected": -277.76324462890625, "logps/rejected": -687.5851440429688, "loss": 4.8215, "margin_dpo/margin_mean": 115.45262908935547, "margin_dpo/margin_std": 225.523193359375, "step": 467 }, { "KL/chosen_KL_mean": -300.00885009765625, "KL/mean": -342.2411193847656, "KL/rejected_KL_mean": -384.4734191894531, "KL/std": 174.18258666992188, "epoch": 0.9801047120418848, "fcm_dpo/beta": 0.00232279347255826, "fcm_dpo/delta": 0.02214384824037552, "fcm_dpo/margin": 84.46463012695312, "fcm_dpo/q_t": 0.45253071188926697, "grad_norm": 91.43498992919922, "learning_rate": 6.700405431837585e-10, "logits/chosen": -0.8857331275939941, "logits/rejected": -0.8545862436294556, "logps/chosen": -617.9720458984375, "logps/ref_chosen": -317.9631652832031, "logps/ref_rejected": -261.8744201660156, "logps/rejected": -646.3478393554688, "loss": 5.0639, "margin_dpo/margin_mean": 84.46461486816406, "margin_dpo/margin_std": 214.5466766357422, "step": 468 }, { "KL/chosen_KL_mean": -282.91015625, "KL/mean": -349.28240966796875, "KL/rejected_KL_mean": -415.6546630859375, "KL/std": 187.77842712402344, "epoch": 0.9821989528795811, "fcm_dpo/beta": 0.0023529785685241222, "fcm_dpo/delta": -0.0062449611723423, "fcm_dpo/margin": 132.7445068359375, "fcm_dpo/q_t": 0.4277336299419403, "grad_norm": 43.829017639160156, "learning_rate": 5.427789289685347e-10, "logits/chosen": -0.8155910968780518, "logits/rejected": -0.7978315353393555, "logps/chosen": -607.7969360351562, "logps/ref_chosen": -324.8868103027344, "logps/ref_rejected": -264.0421447753906, "logps/rejected": -679.6968383789062, "loss": 4.6583, "margin_dpo/margin_mean": 132.7445068359375, "margin_dpo/margin_std": 217.20770263671875, "step": 469 }, { "KL/chosen_KL_mean": -283.00579833984375, "KL/mean": -351.0141296386719, "KL/rejected_KL_mean": -419.0224609375, "KL/std": 183.21075439453125, "epoch": 0.9842931937172775, "fcm_dpo/beta": 0.0023244861513376236, "fcm_dpo/delta": -0.011775792576372623, "fcm_dpo/margin": 136.0166778564453, "fcm_dpo/q_t": 0.427010178565979, "grad_norm": 60.516151428222656, "learning_rate": 4.288949484559934e-10, "logits/chosen": -0.8107550144195557, "logits/rejected": -0.8079327344894409, "logps/chosen": -597.7100830078125, "logps/ref_chosen": -314.7042541503906, "logps/ref_rejected": -259.2276611328125, "logps/rejected": -678.2501220703125, "loss": 4.6326, "margin_dpo/margin_mean": 136.0166778564453, "margin_dpo/margin_std": 210.85792541503906, "step": 470 }, { "KL/chosen_KL_mean": -300.8998107910156, "KL/mean": -368.5666809082031, "KL/rejected_KL_mean": -436.23358154296875, "KL/std": 217.97096252441406, "epoch": 0.9863874345549738, "fcm_dpo/beta": 0.002303960733115673, "fcm_dpo/delta": -0.020671039819717407, "fcm_dpo/margin": 135.33377075195312, "fcm_dpo/q_t": 0.4327457547187805, "grad_norm": 58.01945114135742, "learning_rate": 3.2839470889836627e-10, "logits/chosen": -0.860909640789032, "logits/rejected": -0.8448034524917603, "logps/chosen": -593.4747314453125, "logps/ref_chosen": -292.5748291015625, "logps/ref_rejected": -298.7525329589844, "logps/rejected": -734.9861450195312, "loss": 4.7235, "margin_dpo/margin_mean": 135.33377075195312, "margin_dpo/margin_std": 248.5484619140625, "step": 471 }, { "KL/chosen_KL_mean": -275.46673583984375, "KL/mean": -348.41241455078125, "KL/rejected_KL_mean": -421.358154296875, "KL/std": 182.7490234375, "epoch": 0.9884816753926702, "fcm_dpo/beta": 0.0022689655888825655, "fcm_dpo/delta": 0.00046153878793120384, "fcm_dpo/margin": 145.8914337158203, "fcm_dpo/q_t": 0.4243175983428955, "grad_norm": 60.666263580322266, "learning_rate": 2.412835998185092e-10, "logits/chosen": -0.8494566082954407, "logits/rejected": -0.8609163165092468, "logps/chosen": -518.840576171875, "logps/ref_chosen": -243.37380981445312, "logps/ref_rejected": -251.12109375, "logps/rejected": -672.479248046875, "loss": 4.5793, "margin_dpo/margin_mean": 145.8914337158203, "margin_dpo/margin_std": 217.2068328857422, "step": 472 }, { "KL/chosen_KL_mean": -293.68231201171875, "KL/mean": -360.2905578613281, "KL/rejected_KL_mean": -426.89886474609375, "KL/std": 168.0052490234375, "epoch": 0.9905759162303664, "fcm_dpo/beta": 0.0023105614818632603, "fcm_dpo/delta": 0.030130187049508095, "fcm_dpo/margin": 133.216552734375, "fcm_dpo/q_t": 0.42758381366729736, "grad_norm": 48.50118637084961, "learning_rate": 1.6756629272085544e-10, "logits/chosen": -0.7985444664955139, "logits/rejected": -0.7936415076255798, "logps/chosen": -580.010986328125, "logps/ref_chosen": -286.3286437988281, "logps/ref_rejected": -258.6535339355469, "logps/rejected": -685.5524291992188, "loss": 4.5982, "margin_dpo/margin_mean": 133.216552734375, "margin_dpo/margin_std": 184.37120056152344, "step": 473 }, { "KL/chosen_KL_mean": -295.7257385253906, "KL/mean": -356.5581359863281, "KL/rejected_KL_mean": -417.39056396484375, "KL/std": 190.985595703125, "epoch": 0.9926701570680628, "fcm_dpo/beta": 0.002310484182089567, "fcm_dpo/delta": -0.016346491873264313, "fcm_dpo/margin": 121.6648178100586, "fcm_dpo/q_t": 0.4347348213195801, "grad_norm": 64.4442138671875, "learning_rate": 1.072467408408384e-10, "logits/chosen": -0.8563570976257324, "logits/rejected": -0.8490430116653442, "logps/chosen": -583.8153686523438, "logps/ref_chosen": -288.08966064453125, "logps/ref_rejected": -266.69696044921875, "logps/rejected": -684.0875854492188, "loss": 4.7213, "margin_dpo/margin_mean": 121.6648178100586, "margin_dpo/margin_std": 197.94935607910156, "step": 474 }, { "KL/chosen_KL_mean": -274.5186462402344, "KL/mean": -328.12591552734375, "KL/rejected_KL_mean": -381.73321533203125, "KL/std": 163.69903564453125, "epoch": 0.9947643979057592, "fcm_dpo/beta": 0.002282880013808608, "fcm_dpo/delta": -0.018598873168230057, "fcm_dpo/margin": 107.21459197998047, "fcm_dpo/q_t": 0.44367164373397827, "grad_norm": 66.61784362792969, "learning_rate": 6.032817893297793e-11, "logits/chosen": -0.8387801051139832, "logits/rejected": -0.8456286191940308, "logps/chosen": -530.521728515625, "logps/ref_chosen": -256.0030517578125, "logps/ref_rejected": -244.50660705566406, "logps/rejected": -626.2398071289062, "loss": 4.8252, "margin_dpo/margin_mean": 107.21459197998047, "margin_dpo/margin_std": 188.63526916503906, "step": 475 }, { "KL/chosen_KL_mean": -278.682861328125, "KL/mean": -335.3662109375, "KL/rejected_KL_mean": -392.0495910644531, "KL/std": 165.678466796875, "epoch": 0.9968586387434555, "fcm_dpo/beta": 0.002227974822744727, "fcm_dpo/delta": -0.037535179406404495, "fcm_dpo/margin": 113.36674499511719, "fcm_dpo/q_t": 0.4426191449165344, "grad_norm": 48.4283447265625, "learning_rate": 2.6813123097352287e-11, "logits/chosen": -0.8977000713348389, "logits/rejected": -0.8495609164237976, "logps/chosen": -600.1503295898438, "logps/ref_chosen": -321.467529296875, "logps/ref_rejected": -295.0592956542969, "logps/rejected": -687.1089477539062, "loss": 4.8402, "margin_dpo/margin_mean": 113.36674499511719, "margin_dpo/margin_std": 209.83367919921875, "step": 476 }, { "KL/chosen_KL_mean": -304.5742492675781, "KL/mean": -361.22320556640625, "KL/rejected_KL_mean": -417.8720703125, "KL/std": 196.73390197753906, "epoch": 0.9989528795811519, "fcm_dpo/beta": 0.002197434427216649, "fcm_dpo/delta": -0.011369161307811737, "fcm_dpo/margin": 113.29786682128906, "fcm_dpo/q_t": 0.44230157136917114, "grad_norm": 89.01945495605469, "learning_rate": 6.7033706447061635e-12, "logits/chosen": -0.7661444544792175, "logits/rejected": -0.7697646021842957, "logps/chosen": -581.3682250976562, "logps/ref_chosen": -276.7939758300781, "logps/ref_rejected": -244.82919311523438, "logps/rejected": -662.7012939453125, "loss": 4.914, "margin_dpo/margin_mean": 113.29786682128906, "margin_dpo/margin_std": 247.40846252441406, "step": 477 }, { "epoch": 0.9989528795811519, "step": 477, "total_flos": 0.0, "train_loss": 4.88491684035935, "train_runtime": 6567.2262, "train_samples_per_second": 9.309, "train_steps_per_second": 0.073 } ], "logging_steps": 1, "max_steps": 477, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }