W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4-20260429-230725
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-ult…/trainer_state.json
ModelHub XC e8efbe1ca9 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4-20260429-230725
Source: Original Platform
2026-05-20 08:27:04 +08:00

11057 lines
414 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": -0.05443763732910156,
"KL/mean": -0.04046645760536194,
"KL/rejected_KL_mean": -0.026498794555664062,
"KL/std": 0.47091516852378845,
"epoch": 0.0020942408376963353,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02793481945991516,
"fcm_dpo/q_t": 0.500069797039032,
"grad_norm": 28.589427947998047,
"learning_rate": 0.0,
"logits/chosen": -0.5898098945617676,
"logits/rejected": -0.604260265827179,
"logps/chosen": -275.28570556640625,
"logps/ref_chosen": -275.2312927246094,
"logps/ref_rejected": -222.9380340576172,
"logps/rejected": -222.96453857421875,
"loss": 5.5463,
"margin_dpo/margin_mean": -0.02793477475643158,
"margin_dpo/margin_std": 0.5724214911460876,
"step": 1
},
{
"KL/chosen_KL_mean": 0.04458808898925781,
"KL/mean": 0.0374295711517334,
"KL/rejected_KL_mean": 0.030269622802734375,
"KL/std": 0.49036121368408203,
"epoch": 0.004188481675392671,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014312177896499634,
"fcm_dpo/q_t": 0.4999642074108124,
"grad_norm": 27.880699157714844,
"learning_rate": 1.0416666666666666e-08,
"logits/chosen": -0.6574729681015015,
"logits/rejected": -0.6464410424232483,
"logps/chosen": -264.7165222167969,
"logps/ref_chosen": -264.7611083984375,
"logps/ref_rejected": -242.5597686767578,
"logps/rejected": -242.52951049804688,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.014312252402305603,
"margin_dpo/margin_std": 0.6423971652984619,
"step": 2
},
{
"KL/chosen_KL_mean": 0.02648162841796875,
"KL/mean": -0.001500651240348816,
"KL/rejected_KL_mean": -0.029489517211914062,
"KL/std": 0.49069744348526,
"epoch": 0.0062827225130890054,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05598026514053345,
"fcm_dpo/q_t": 0.499860018491745,
"grad_norm": 25.799875259399414,
"learning_rate": 2.083333333333333e-08,
"logits/chosen": -0.6840239763259888,
"logits/rejected": -0.7351561784744263,
"logps/chosen": -274.0754089355469,
"logps/ref_chosen": -274.1018981933594,
"logps/ref_rejected": -286.5882568359375,
"logps/rejected": -286.61773681640625,
"loss": 5.543,
"margin_dpo/margin_mean": 0.05597783625125885,
"margin_dpo/margin_std": 0.6958938241004944,
"step": 3
},
{
"KL/chosen_KL_mean": -0.060821533203125,
"KL/mean": -0.044119708240032196,
"KL/rejected_KL_mean": -0.027416229248046875,
"KL/std": 0.5412886142730713,
"epoch": 0.008376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03339381515979767,
"fcm_dpo/q_t": 0.5000834465026855,
"grad_norm": 31.741003036499023,
"learning_rate": 3.125e-08,
"logits/chosen": -0.6172194480895996,
"logits/rejected": -0.6116077899932861,
"logps/chosen": -329.8990478515625,
"logps/ref_chosen": -329.8382568359375,
"logps/ref_rejected": -303.2850646972656,
"logps/rejected": -303.3124694824219,
"loss": 5.5466,
"margin_dpo/margin_mean": -0.0333935022354126,
"margin_dpo/margin_std": 0.7534641623497009,
"step": 4
},
{
"KL/chosen_KL_mean": -0.06523323059082031,
"KL/mean": -0.0311996191740036,
"KL/rejected_KL_mean": 0.0028362274169921875,
"KL/std": 0.5405311584472656,
"epoch": 0.010471204188481676,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06806734204292297,
"fcm_dpo/q_t": 0.500170111656189,
"grad_norm": 29.559877395629883,
"learning_rate": 4.166666666666666e-08,
"logits/chosen": -0.5798951387405396,
"logits/rejected": -0.5972731113433838,
"logps/chosen": -301.80419921875,
"logps/ref_chosen": -301.7389221191406,
"logps/ref_rejected": -274.7654724121094,
"logps/rejected": -274.76263427734375,
"loss": 5.548,
"margin_dpo/margin_mean": -0.0680682510137558,
"margin_dpo/margin_std": 0.787623941898346,
"step": 5
},
{
"KL/chosen_KL_mean": 0.036251068115234375,
"KL/mean": -0.007517039775848389,
"KL/rejected_KL_mean": -0.0512847900390625,
"KL/std": 0.5245298743247986,
"epoch": 0.012565445026178011,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08753737807273865,
"fcm_dpo/q_t": 0.49978113174438477,
"grad_norm": 28.30576515197754,
"learning_rate": 5.208333333333333e-08,
"logits/chosen": -0.6791407465934753,
"logits/rejected": -0.6419384479522705,
"logps/chosen": -285.658447265625,
"logps/ref_chosen": -285.6946716308594,
"logps/ref_rejected": -245.8200225830078,
"logps/rejected": -245.87130737304688,
"loss": 5.5417,
"margin_dpo/margin_mean": 0.08753781020641327,
"margin_dpo/margin_std": 0.7099345326423645,
"step": 6
},
{
"KL/chosen_KL_mean": -0.04565238952636719,
"KL/mean": 0.0033334270119667053,
"KL/rejected_KL_mean": 0.05230522155761719,
"KL/std": 0.5084267854690552,
"epoch": 0.014659685863874346,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0979558676481247,
"fcm_dpo/q_t": 0.5002449154853821,
"grad_norm": 28.62909698486328,
"learning_rate": 6.25e-08,
"logits/chosen": -0.5669029951095581,
"logits/rejected": -0.5987157225608826,
"logps/chosen": -264.7010803222656,
"logps/ref_chosen": -264.65545654296875,
"logps/ref_rejected": -253.10305786132812,
"logps/rejected": -253.05076599121094,
"loss": 5.5491,
"margin_dpo/margin_mean": -0.09795580804347992,
"margin_dpo/margin_std": 0.6526715755462646,
"step": 7
},
{
"KL/chosen_KL_mean": 0.029354095458984375,
"KL/mean": 0.002108544111251831,
"KL/rejected_KL_mean": -0.025133132934570312,
"KL/std": 0.5323061943054199,
"epoch": 0.016753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.054483771324157715,
"fcm_dpo/q_t": 0.49986377358436584,
"grad_norm": 30.765409469604492,
"learning_rate": 7.291666666666667e-08,
"logits/chosen": -0.6719302535057068,
"logits/rejected": -0.6778964996337891,
"logps/chosen": -354.159423828125,
"logps/ref_chosen": -354.1887512207031,
"logps/ref_rejected": -282.9112243652344,
"logps/rejected": -282.93634033203125,
"loss": 5.5431,
"margin_dpo/margin_mean": 0.054484546184539795,
"margin_dpo/margin_std": 0.7297343015670776,
"step": 8
},
{
"KL/chosen_KL_mean": 0.05601310729980469,
"KL/mean": 0.03541021794080734,
"KL/rejected_KL_mean": 0.014802932739257812,
"KL/std": 0.5015314817428589,
"epoch": 0.018848167539267015,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04120449721813202,
"fcm_dpo/q_t": 0.49989697337150574,
"grad_norm": 27.886104583740234,
"learning_rate": 8.333333333333333e-08,
"logits/chosen": -0.6346057653427124,
"logits/rejected": -0.6525917053222656,
"logps/chosen": -285.49420166015625,
"logps/ref_chosen": -285.5502014160156,
"logps/ref_rejected": -267.99664306640625,
"logps/rejected": -267.9818420410156,
"loss": 5.5436,
"margin_dpo/margin_mean": 0.04120419919490814,
"margin_dpo/margin_std": 0.6990031003952026,
"step": 9
},
{
"KL/chosen_KL_mean": 0.046550750732421875,
"KL/mean": 0.020397096872329712,
"KL/rejected_KL_mean": -0.0057582855224609375,
"KL/std": 0.5216137170791626,
"epoch": 0.020942408376963352,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.052318453788757324,
"fcm_dpo/q_t": 0.499869167804718,
"grad_norm": 26.553926467895508,
"learning_rate": 9.375e-08,
"logits/chosen": -0.691142201423645,
"logits/rejected": -0.6861263513565063,
"logps/chosen": -251.8658447265625,
"logps/ref_chosen": -251.91238403320312,
"logps/ref_rejected": -226.45260620117188,
"logps/rejected": -226.4583740234375,
"loss": 5.5431,
"margin_dpo/margin_mean": 0.05231724679470062,
"margin_dpo/margin_std": 0.6777650713920593,
"step": 10
},
{
"KL/chosen_KL_mean": 0.05559539794921875,
"KL/mean": 0.08593348413705826,
"KL/rejected_KL_mean": 0.11626625061035156,
"KL/std": 0.5352065563201904,
"epoch": 0.023036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06067343056201935,
"fcm_dpo/q_t": 0.5001516938209534,
"grad_norm": 28.990291595458984,
"learning_rate": 1.0416666666666667e-07,
"logits/chosen": -0.5975240468978882,
"logits/rejected": -0.6529561281204224,
"logps/chosen": -301.02789306640625,
"logps/ref_chosen": -301.08343505859375,
"logps/ref_rejected": -259.546630859375,
"logps/rejected": -259.43035888671875,
"loss": 5.5477,
"margin_dpo/margin_mean": -0.060673072934150696,
"margin_dpo/margin_std": 0.7372453212738037,
"step": 11
},
{
"KL/chosen_KL_mean": -0.03789520263671875,
"KL/mean": -0.051166288554668427,
"KL/rejected_KL_mean": -0.06442451477050781,
"KL/std": 0.5343046188354492,
"epoch": 0.025130890052356022,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.026523694396018982,
"fcm_dpo/q_t": 0.499933660030365,
"grad_norm": 30.13033103942871,
"learning_rate": 1.1458333333333332e-07,
"logits/chosen": -0.5826385021209717,
"logits/rejected": -0.5425857901573181,
"logps/chosen": -287.58599853515625,
"logps/ref_chosen": -287.548095703125,
"logps/ref_rejected": -277.37945556640625,
"logps/rejected": -277.44390869140625,
"loss": 5.5442,
"margin_dpo/margin_mean": 0.026523590087890625,
"margin_dpo/margin_std": 0.7421956062316895,
"step": 12
},
{
"KL/chosen_KL_mean": 0.05934333801269531,
"KL/mean": -0.022284463047981262,
"KL/rejected_KL_mean": -0.10392189025878906,
"KL/std": 0.43461257219314575,
"epoch": 0.027225130890052355,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1632615029811859,
"fcm_dpo/q_t": 0.4995918273925781,
"grad_norm": 27.26964569091797,
"learning_rate": 1.25e-07,
"logits/chosen": -0.6637694239616394,
"logits/rejected": -0.6718661785125732,
"logps/chosen": -270.6070556640625,
"logps/ref_chosen": -270.6664123535156,
"logps/ref_rejected": -274.6546936035156,
"logps/rejected": -274.7586364746094,
"loss": 5.5387,
"margin_dpo/margin_mean": 0.1632617861032486,
"margin_dpo/margin_std": 0.5879830121994019,
"step": 13
},
{
"KL/chosen_KL_mean": 0.01528167724609375,
"KL/mean": -0.007226079702377319,
"KL/rejected_KL_mean": -0.029737472534179688,
"KL/std": 0.5344897508621216,
"epoch": 0.02931937172774869,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04503127932548523,
"fcm_dpo/q_t": 0.4998874068260193,
"grad_norm": 28.33542823791504,
"learning_rate": 1.3541666666666666e-07,
"logits/chosen": -0.6277254223823547,
"logits/rejected": -0.657189130783081,
"logps/chosen": -281.5779113769531,
"logps/ref_chosen": -281.59320068359375,
"logps/ref_rejected": -263.52215576171875,
"logps/rejected": -263.5519104003906,
"loss": 5.5434,
"margin_dpo/margin_mean": 0.04503123462200165,
"margin_dpo/margin_std": 0.7003819942474365,
"step": 14
},
{
"KL/chosen_KL_mean": 0.0198822021484375,
"KL/mean": 0.006983160972595215,
"KL/rejected_KL_mean": -0.005916595458984375,
"KL/std": 0.4857458770275116,
"epoch": 0.031413612565445025,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02580079436302185,
"fcm_dpo/q_t": 0.499935507774353,
"grad_norm": 30.2783145904541,
"learning_rate": 1.4583333333333335e-07,
"logits/chosen": -0.6451765298843384,
"logits/rejected": -0.6582185626029968,
"logps/chosen": -298.4335632324219,
"logps/ref_chosen": -298.45343017578125,
"logps/ref_rejected": -227.17832946777344,
"logps/rejected": -227.1842498779297,
"loss": 5.5442,
"margin_dpo/margin_mean": 0.025799661874771118,
"margin_dpo/margin_std": 0.7161386609077454,
"step": 15
},
{
"KL/chosen_KL_mean": 0.011865615844726562,
"KL/mean": 0.002268832176923752,
"KL/rejected_KL_mean": -0.007328033447265625,
"KL/std": 0.5652431845664978,
"epoch": 0.033507853403141365,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01919480413198471,
"fcm_dpo/q_t": 0.49995195865631104,
"grad_norm": 30.200963973999023,
"learning_rate": 1.5624999999999999e-07,
"logits/chosen": -0.5971285104751587,
"logits/rejected": -0.5938217043876648,
"logps/chosen": -293.9547424316406,
"logps/ref_chosen": -293.96661376953125,
"logps/ref_rejected": -250.78443908691406,
"logps/rejected": -250.79176330566406,
"loss": 5.5445,
"margin_dpo/margin_mean": 0.019194379448890686,
"margin_dpo/margin_std": 0.802629828453064,
"step": 16
},
{
"KL/chosen_KL_mean": -0.02978515625,
"KL/mean": 0.001806497573852539,
"KL/rejected_KL_mean": 0.0334014892578125,
"KL/std": 0.4761771261692047,
"epoch": 0.0356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06319683790206909,
"fcm_dpo/q_t": 0.5001580119132996,
"grad_norm": 27.68961524963379,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5712617635726929,
"logits/rejected": -0.5975736379623413,
"logps/chosen": -262.4237976074219,
"logps/ref_chosen": -262.39398193359375,
"logps/ref_rejected": -248.500244140625,
"logps/rejected": -248.4668426513672,
"loss": 5.5478,
"margin_dpo/margin_mean": -0.06319691240787506,
"margin_dpo/margin_std": 0.6851055026054382,
"step": 17
},
{
"KL/chosen_KL_mean": 0.13873863220214844,
"KL/mean": 0.06383218616247177,
"KL/rejected_KL_mean": -0.011079788208007812,
"KL/std": 0.5538792014122009,
"epoch": 0.03769633507853403,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14981205761432648,
"fcm_dpo/q_t": 0.49962547421455383,
"grad_norm": 29.6390438079834,
"learning_rate": 1.7708333333333334e-07,
"logits/chosen": -0.6094870567321777,
"logits/rejected": -0.6156261563301086,
"logps/chosen": -293.57049560546875,
"logps/ref_chosen": -293.709228515625,
"logps/ref_rejected": -274.5875244140625,
"logps/rejected": -274.5986328125,
"loss": 5.5392,
"margin_dpo/margin_mean": 0.14981183409690857,
"margin_dpo/margin_std": 0.7385837435722351,
"step": 18
},
{
"KL/chosen_KL_mean": 0.0306243896484375,
"KL/mean": 0.023819290101528168,
"KL/rejected_KL_mean": 0.017019271850585938,
"KL/std": 0.44108718633651733,
"epoch": 0.039790575916230364,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01360417902469635,
"fcm_dpo/q_t": 0.49996596574783325,
"grad_norm": 28.12141227722168,
"learning_rate": 1.875e-07,
"logits/chosen": -0.6333426833152771,
"logits/rejected": -0.6301394701004028,
"logps/chosen": -280.2350158691406,
"logps/ref_chosen": -280.26568603515625,
"logps/ref_rejected": -259.9742736816406,
"logps/rejected": -259.9572448730469,
"loss": 5.5447,
"margin_dpo/margin_mean": 0.013604193925857544,
"margin_dpo/margin_std": 0.5601605176925659,
"step": 19
},
{
"KL/chosen_KL_mean": 0.15392303466796875,
"KL/mean": 0.09969540685415268,
"KL/rejected_KL_mean": 0.04547691345214844,
"KL/std": 0.5148954391479492,
"epoch": 0.041884816753926704,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10843174159526825,
"fcm_dpo/q_t": 0.4997289180755615,
"grad_norm": 29.663406372070312,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": -0.61493980884552,
"logits/rejected": -0.6458949446678162,
"logps/chosen": -303.7414855957031,
"logps/ref_chosen": -303.8954162597656,
"logps/ref_rejected": -260.214599609375,
"logps/rejected": -260.16912841796875,
"loss": 5.5409,
"margin_dpo/margin_mean": 0.10843226313591003,
"margin_dpo/margin_std": 0.7343800067901611,
"step": 20
},
{
"KL/chosen_KL_mean": 0.13170433044433594,
"KL/mean": 0.06188298016786575,
"KL/rejected_KL_mean": -0.007947921752929688,
"KL/std": 0.5429558753967285,
"epoch": 0.04397905759162304,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13965456187725067,
"fcm_dpo/q_t": 0.49965083599090576,
"grad_norm": 34.97311019897461,
"learning_rate": 2.0833333333333333e-07,
"logits/chosen": -0.6538794636726379,
"logits/rejected": -0.6838114857673645,
"logps/chosen": -301.40179443359375,
"logps/ref_chosen": -301.5334777832031,
"logps/ref_rejected": -280.28900146484375,
"logps/rejected": -280.29693603515625,
"loss": 5.5397,
"margin_dpo/margin_mean": 0.13965462148189545,
"margin_dpo/margin_std": 0.7473960518836975,
"step": 21
},
{
"KL/chosen_KL_mean": 0.095489501953125,
"KL/mean": 0.06919194012880325,
"KL/rejected_KL_mean": 0.04288482666015625,
"KL/std": 0.5348218679428101,
"epoch": 0.04607329842931937,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05260805785655975,
"fcm_dpo/q_t": 0.4998684525489807,
"grad_norm": 25.295730590820312,
"learning_rate": 2.1875e-07,
"logits/chosen": -0.6623374819755554,
"logits/rejected": -0.6642951369285583,
"logps/chosen": -259.8996276855469,
"logps/ref_chosen": -259.9951477050781,
"logps/ref_rejected": -243.0721435546875,
"logps/rejected": -243.02928161621094,
"loss": 5.5431,
"margin_dpo/margin_mean": 0.052607640624046326,
"margin_dpo/margin_std": 0.7113056182861328,
"step": 22
},
{
"KL/chosen_KL_mean": 0.09666061401367188,
"KL/mean": 0.03947530686855316,
"KL/rejected_KL_mean": -0.0177154541015625,
"KL/std": 0.5528764128684998,
"epoch": 0.048167539267015703,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11437445878982544,
"fcm_dpo/q_t": 0.49971407651901245,
"grad_norm": 28.02526092529297,
"learning_rate": 2.2916666666666663e-07,
"logits/chosen": -0.6213913559913635,
"logits/rejected": -0.6555658578872681,
"logps/chosen": -282.0841064453125,
"logps/ref_chosen": -282.1807556152344,
"logps/ref_rejected": -265.0758056640625,
"logps/rejected": -265.093505859375,
"loss": 5.5407,
"margin_dpo/margin_mean": 0.11437398195266724,
"margin_dpo/margin_std": 0.7158402800559998,
"step": 23
},
{
"KL/chosen_KL_mean": 0.16210174560546875,
"KL/mean": 0.04678072780370712,
"KL/rejected_KL_mean": -0.06854248046875,
"KL/std": 0.6121370792388916,
"epoch": 0.050261780104712044,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2306310385465622,
"fcm_dpo/q_t": 0.49942344427108765,
"grad_norm": 29.789752960205078,
"learning_rate": 2.3958333333333335e-07,
"logits/chosen": -0.6437051296234131,
"logits/rejected": -0.5558314323425293,
"logps/chosen": -301.01751708984375,
"logps/ref_chosen": -301.17962646484375,
"logps/ref_rejected": -302.12786865234375,
"logps/rejected": -302.1964111328125,
"loss": 5.536,
"margin_dpo/margin_mean": 0.23063026368618011,
"margin_dpo/margin_std": 0.8269215226173401,
"step": 24
},
{
"KL/chosen_KL_mean": 0.15172958374023438,
"KL/mean": 0.07193604111671448,
"KL/rejected_KL_mean": -0.007861137390136719,
"KL/std": 0.49663278460502625,
"epoch": 0.05235602094240838,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1595902144908905,
"fcm_dpo/q_t": 0.49960100650787354,
"grad_norm": 26.53129005432129,
"learning_rate": 2.5e-07,
"logits/chosen": -0.5964261293411255,
"logits/rejected": -0.6044997572898865,
"logps/chosen": -246.59474182128906,
"logps/ref_chosen": -246.74649047851562,
"logps/ref_rejected": -235.55638122558594,
"logps/rejected": -235.56422424316406,
"loss": 5.5389,
"margin_dpo/margin_mean": 0.15958915650844574,
"margin_dpo/margin_std": 0.7234926819801331,
"step": 25
},
{
"KL/chosen_KL_mean": 0.2514305114746094,
"KL/mean": 0.16334398090839386,
"KL/rejected_KL_mean": 0.07526397705078125,
"KL/std": 0.6091904044151306,
"epoch": 0.05445026178010471,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17616990208625793,
"fcm_dpo/q_t": 0.499559611082077,
"grad_norm": 28.695537567138672,
"learning_rate": 2.604166666666667e-07,
"logits/chosen": -0.647978663444519,
"logits/rejected": -0.6618956327438354,
"logps/chosen": -281.94415283203125,
"logps/ref_chosen": -282.1955871582031,
"logps/ref_rejected": -235.3135528564453,
"logps/rejected": -235.23829650878906,
"loss": 5.5382,
"margin_dpo/margin_mean": 0.17617014050483704,
"margin_dpo/margin_std": 0.8417377471923828,
"step": 26
},
{
"KL/chosen_KL_mean": 0.3227558135986328,
"KL/mean": 0.2342260479927063,
"KL/rejected_KL_mean": 0.1456890106201172,
"KL/std": 0.6314464211463928,
"epoch": 0.05654450261780105,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17707379162311554,
"fcm_dpo/q_t": 0.49955734610557556,
"grad_norm": 27.701112747192383,
"learning_rate": 2.708333333333333e-07,
"logits/chosen": -0.6549860239028931,
"logits/rejected": -0.6748118996620178,
"logps/chosen": -323.5335693359375,
"logps/ref_chosen": -323.8563537597656,
"logps/ref_rejected": -245.968017578125,
"logps/rejected": -245.82235717773438,
"loss": 5.5382,
"margin_dpo/margin_mean": 0.1770738810300827,
"margin_dpo/margin_std": 0.9032553434371948,
"step": 27
},
{
"KL/chosen_KL_mean": 0.233856201171875,
"KL/mean": 0.17051860690116882,
"KL/rejected_KL_mean": 0.107177734375,
"KL/std": 0.5839200615882874,
"epoch": 0.05863874345549738,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12668462097644806,
"fcm_dpo/q_t": 0.49968329071998596,
"grad_norm": 26.368717193603516,
"learning_rate": 2.8125e-07,
"logits/chosen": -0.6276756525039673,
"logits/rejected": -0.6372566819190979,
"logps/chosen": -248.01287841796875,
"logps/ref_chosen": -248.24673461914062,
"logps/ref_rejected": -240.0382080078125,
"logps/rejected": -239.9310302734375,
"loss": 5.5402,
"margin_dpo/margin_mean": 0.1266840100288391,
"margin_dpo/margin_std": 0.8177756071090698,
"step": 28
},
{
"KL/chosen_KL_mean": 0.33344268798828125,
"KL/mean": 0.25681155920028687,
"KL/rejected_KL_mean": 0.18017005920410156,
"KL/std": 0.6312374472618103,
"epoch": 0.060732984293193716,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15327195823192596,
"fcm_dpo/q_t": 0.4996168315410614,
"grad_norm": 29.685964584350586,
"learning_rate": 2.916666666666667e-07,
"logits/chosen": -0.5926003456115723,
"logits/rejected": -0.6149235367774963,
"logps/chosen": -317.9229736328125,
"logps/ref_chosen": -318.2564392089844,
"logps/ref_rejected": -286.75848388671875,
"logps/rejected": -286.5783386230469,
"loss": 5.5391,
"margin_dpo/margin_mean": 0.15327222645282745,
"margin_dpo/margin_std": 0.8355914354324341,
"step": 29
},
{
"KL/chosen_KL_mean": 0.37634849548339844,
"KL/mean": 0.2421349585056305,
"KL/rejected_KL_mean": 0.10791778564453125,
"KL/std": 0.6748695373535156,
"epoch": 0.06282722513089005,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2684309482574463,
"fcm_dpo/q_t": 0.4993289113044739,
"grad_norm": 28.966293334960938,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": -0.589932918548584,
"logits/rejected": -0.6068093180656433,
"logps/chosen": -252.6727752685547,
"logps/ref_chosen": -253.0491485595703,
"logps/ref_rejected": -261.30029296875,
"logps/rejected": -261.1923828125,
"loss": 5.5345,
"margin_dpo/margin_mean": 0.26843076944351196,
"margin_dpo/margin_std": 0.9395996332168579,
"step": 30
},
{
"KL/chosen_KL_mean": 0.4201011657714844,
"KL/mean": 0.3030571937561035,
"KL/rejected_KL_mean": 0.1860198974609375,
"KL/std": 0.709516167640686,
"epoch": 0.06492146596858639,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23408280313014984,
"fcm_dpo/q_t": 0.49941486120224,
"grad_norm": 25.04664421081543,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": -0.6467900276184082,
"logits/rejected": -0.680314302444458,
"logps/chosen": -247.73292541503906,
"logps/ref_chosen": -248.15301513671875,
"logps/ref_rejected": -203.17703247070312,
"logps/rejected": -202.9910125732422,
"loss": 5.5359,
"margin_dpo/margin_mean": 0.23408250510692596,
"margin_dpo/margin_std": 0.9148648977279663,
"step": 31
},
{
"KL/chosen_KL_mean": 0.4879474639892578,
"KL/mean": 0.3524807095527649,
"KL/rejected_KL_mean": 0.21702194213867188,
"KL/std": 0.7370665073394775,
"epoch": 0.06701570680628273,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2709151804447174,
"fcm_dpo/q_t": 0.49932271242141724,
"grad_norm": 29.720426559448242,
"learning_rate": 3.2291666666666666e-07,
"logits/chosen": -0.6020100116729736,
"logits/rejected": -0.6054341793060303,
"logps/chosen": -305.052001953125,
"logps/ref_chosen": -305.5399475097656,
"logps/ref_rejected": -267.6527099609375,
"logps/rejected": -267.4356689453125,
"loss": 5.5345,
"margin_dpo/margin_mean": 0.27091488242149353,
"margin_dpo/margin_std": 0.9720284938812256,
"step": 32
},
{
"KL/chosen_KL_mean": 0.6097984313964844,
"KL/mean": 0.38870200514793396,
"KL/rejected_KL_mean": 0.1676158905029297,
"KL/std": 0.8518388271331787,
"epoch": 0.06910994764397906,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.44216686487197876,
"fcm_dpo/q_t": 0.4988946318626404,
"grad_norm": 28.51626205444336,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.633573591709137,
"logits/rejected": -0.6452749371528625,
"logps/chosen": -285.62371826171875,
"logps/ref_chosen": -286.2335205078125,
"logps/ref_rejected": -255.38748168945312,
"logps/rejected": -255.21986389160156,
"loss": 5.5276,
"margin_dpo/margin_mean": 0.44216734170913696,
"margin_dpo/margin_std": 1.0235333442687988,
"step": 33
},
{
"KL/chosen_KL_mean": 0.7400722503662109,
"KL/mean": 0.4082331657409668,
"KL/rejected_KL_mean": 0.07639503479003906,
"KL/std": 1.0116835832595825,
"epoch": 0.0712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.663679838180542,
"fcm_dpo/q_t": 0.4983408749103546,
"grad_norm": 31.122045516967773,
"learning_rate": 3.4375e-07,
"logits/chosen": -0.6284201741218567,
"logits/rejected": -0.6383717060089111,
"logps/chosen": -340.8519287109375,
"logps/ref_chosen": -341.5920104980469,
"logps/ref_rejected": -278.8866882324219,
"logps/rejected": -278.8102722167969,
"loss": 5.5188,
"margin_dpo/margin_mean": 0.6636798977851868,
"margin_dpo/margin_std": 1.1839109659194946,
"step": 34
},
{
"KL/chosen_KL_mean": 0.6229438781738281,
"KL/mean": 0.46992263197898865,
"KL/rejected_KL_mean": 0.3169078826904297,
"KL/std": 1.0069925785064697,
"epoch": 0.07329842931937172,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3060385286808014,
"fcm_dpo/q_t": 0.4992349147796631,
"grad_norm": 26.427873611450195,
"learning_rate": 3.541666666666667e-07,
"logits/chosen": -0.6407392024993896,
"logits/rejected": -0.6617733836174011,
"logps/chosen": -264.45660400390625,
"logps/ref_chosen": -265.0795593261719,
"logps/ref_rejected": -264.4876708984375,
"logps/rejected": -264.1707763671875,
"loss": 5.5332,
"margin_dpo/margin_mean": 0.30603715777397156,
"margin_dpo/margin_std": 1.3842066526412964,
"step": 35
},
{
"KL/chosen_KL_mean": 0.9210777282714844,
"KL/mean": 0.5656237602233887,
"KL/rejected_KL_mean": 0.21016883850097656,
"KL/std": 1.1084519624710083,
"epoch": 0.07539267015706806,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7109137177467346,
"fcm_dpo/q_t": 0.49822285771369934,
"grad_norm": 32.040306091308594,
"learning_rate": 3.645833333333333e-07,
"logits/chosen": -0.600266695022583,
"logits/rejected": -0.61773681640625,
"logps/chosen": -296.405029296875,
"logps/ref_chosen": -297.3261413574219,
"logps/ref_rejected": -282.09515380859375,
"logps/rejected": -281.8849792480469,
"loss": 5.517,
"margin_dpo/margin_mean": 0.7109129428863525,
"margin_dpo/margin_std": 1.4326907396316528,
"step": 36
},
{
"KL/chosen_KL_mean": 0.7100296020507812,
"KL/mean": 0.45437169075012207,
"KL/rejected_KL_mean": 0.1987323760986328,
"KL/std": 1.1140685081481934,
"epoch": 0.0774869109947644,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5112793445587158,
"fcm_dpo/q_t": 0.4987218976020813,
"grad_norm": 31.217262268066406,
"learning_rate": 3.75e-07,
"logits/chosen": -0.5984426140785217,
"logits/rejected": -0.612653911113739,
"logps/chosen": -313.3240051269531,
"logps/ref_chosen": -314.0340270996094,
"logps/ref_rejected": -299.3437805175781,
"logps/rejected": -299.14501953125,
"loss": 5.525,
"margin_dpo/margin_mean": 0.5112783312797546,
"margin_dpo/margin_std": 1.474202036857605,
"step": 37
},
{
"KL/chosen_KL_mean": 1.0367469787597656,
"KL/mean": 0.7218881845474243,
"KL/rejected_KL_mean": 0.40703392028808594,
"KL/std": 1.2584364414215088,
"epoch": 0.07958115183246073,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6297140121459961,
"fcm_dpo/q_t": 0.49842581152915955,
"grad_norm": 28.42009162902832,
"learning_rate": 3.8541666666666665e-07,
"logits/chosen": -0.6392098069190979,
"logits/rejected": -0.6514177322387695,
"logps/chosen": -281.50445556640625,
"logps/ref_chosen": -282.54119873046875,
"logps/ref_rejected": -269.7773132324219,
"logps/rejected": -269.3702697753906,
"loss": 5.5203,
"margin_dpo/margin_mean": 0.6297144889831543,
"margin_dpo/margin_std": 1.5614598989486694,
"step": 38
},
{
"KL/chosen_KL_mean": 1.3400955200195312,
"KL/mean": 0.716141402721405,
"KL/rejected_KL_mean": 0.092193603515625,
"KL/std": 1.4875259399414062,
"epoch": 0.08167539267015707,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2479068040847778,
"fcm_dpo/q_t": 0.4968806505203247,
"grad_norm": 29.515811920166016,
"learning_rate": 3.958333333333333e-07,
"logits/chosen": -0.6168644428253174,
"logits/rejected": -0.6308912634849548,
"logps/chosen": -275.432861328125,
"logps/ref_chosen": -276.7729187011719,
"logps/ref_rejected": -249.95889282226562,
"logps/rejected": -249.86668395996094,
"loss": 5.4958,
"margin_dpo/margin_mean": 1.2479069232940674,
"margin_dpo/margin_std": 1.717916488647461,
"step": 39
},
{
"KL/chosen_KL_mean": 1.0269756317138672,
"KL/mean": 0.6691527962684631,
"KL/rejected_KL_mean": 0.3113231658935547,
"KL/std": 1.3725290298461914,
"epoch": 0.08376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.715646505355835,
"fcm_dpo/q_t": 0.49821117520332336,
"grad_norm": 27.384668350219727,
"learning_rate": 4.0625e-07,
"logits/chosen": -0.6227149367332458,
"logits/rejected": -0.6583834290504456,
"logps/chosen": -283.28009033203125,
"logps/ref_chosen": -284.30706787109375,
"logps/ref_rejected": -244.4459991455078,
"logps/rejected": -244.1346435546875,
"loss": 5.5169,
"margin_dpo/margin_mean": 0.7156472206115723,
"margin_dpo/margin_std": 1.7450072765350342,
"step": 40
},
{
"KL/chosen_KL_mean": 1.1292037963867188,
"KL/mean": 0.7421854734420776,
"KL/rejected_KL_mean": 0.3551807403564453,
"KL/std": 1.465866208076477,
"epoch": 0.08586387434554973,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7740218639373779,
"fcm_dpo/q_t": 0.49806511402130127,
"grad_norm": 30.4445743560791,
"learning_rate": 4.1666666666666667e-07,
"logits/chosen": -0.6232025027275085,
"logits/rejected": -0.647884726524353,
"logps/chosen": -292.68597412109375,
"logps/ref_chosen": -293.8151550292969,
"logps/ref_rejected": -252.16815185546875,
"logps/rejected": -251.81297302246094,
"loss": 5.5146,
"margin_dpo/margin_mean": 0.7740228176116943,
"margin_dpo/margin_std": 1.7456119060516357,
"step": 41
},
{
"KL/chosen_KL_mean": 1.1923046112060547,
"KL/mean": 0.7407265901565552,
"KL/rejected_KL_mean": 0.2891521453857422,
"KL/std": 1.7247297763824463,
"epoch": 0.08795811518324607,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9031492471694946,
"fcm_dpo/q_t": 0.49774235486984253,
"grad_norm": 27.42459487915039,
"learning_rate": 4.270833333333333e-07,
"logits/chosen": -0.6371679902076721,
"logits/rejected": -0.6549400687217712,
"logps/chosen": -251.5679473876953,
"logps/ref_chosen": -252.76023864746094,
"logps/ref_rejected": -261.0414733886719,
"logps/rejected": -260.7523193359375,
"loss": 5.5097,
"margin_dpo/margin_mean": 0.9031498432159424,
"margin_dpo/margin_std": 2.27445387840271,
"step": 42
},
{
"KL/chosen_KL_mean": 1.2597465515136719,
"KL/mean": 0.6688581705093384,
"KL/rejected_KL_mean": 0.07796669006347656,
"KL/std": 1.8619308471679688,
"epoch": 0.09005235602094241,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1817760467529297,
"fcm_dpo/q_t": 0.4970458745956421,
"grad_norm": 29.90841293334961,
"learning_rate": 4.375e-07,
"logits/chosen": -0.5890940427780151,
"logits/rejected": -0.6039578914642334,
"logps/chosen": -315.5749816894531,
"logps/ref_chosen": -316.8347473144531,
"logps/ref_rejected": -273.7649230957031,
"logps/rejected": -273.68695068359375,
"loss": 5.4986,
"margin_dpo/margin_mean": 1.1817759275436401,
"margin_dpo/margin_std": 2.1939334869384766,
"step": 43
},
{
"KL/chosen_KL_mean": 1.512136459350586,
"KL/mean": 0.7135680913925171,
"KL/rejected_KL_mean": -0.08500099182128906,
"KL/std": 2.2240757942199707,
"epoch": 0.09214659685863874,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5971362590789795,
"fcm_dpo/q_t": 0.49600785970687866,
"grad_norm": 30.842592239379883,
"learning_rate": 4.479166666666667e-07,
"logits/chosen": -0.5955119132995605,
"logits/rejected": -0.5941088199615479,
"logps/chosen": -285.3635559082031,
"logps/ref_chosen": -286.8757019042969,
"logps/ref_rejected": -282.4681396484375,
"logps/rejected": -282.55316162109375,
"loss": 5.4825,
"margin_dpo/margin_mean": 1.5971367359161377,
"margin_dpo/margin_std": 3.021836996078491,
"step": 44
},
{
"KL/chosen_KL_mean": 1.7010555267333984,
"KL/mean": 1.0267201662063599,
"KL/rejected_KL_mean": 0.35239219665527344,
"KL/std": 2.252026319503784,
"epoch": 0.09424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3486623764038086,
"fcm_dpo/q_t": 0.49662941694259644,
"grad_norm": 28.998353958129883,
"learning_rate": 4.5833333333333327e-07,
"logits/chosen": -0.6837330460548401,
"logits/rejected": -0.7065256237983704,
"logps/chosen": -322.5623474121094,
"logps/ref_chosen": -324.2633972167969,
"logps/ref_rejected": -293.09466552734375,
"logps/rejected": -292.7422790527344,
"loss": 5.4922,
"margin_dpo/margin_mean": 1.3486623764038086,
"margin_dpo/margin_std": 2.807202100753784,
"step": 45
},
{
"KL/chosen_KL_mean": 1.6978473663330078,
"KL/mean": 0.9566701650619507,
"KL/rejected_KL_mean": 0.2154865264892578,
"KL/std": 2.4581615924835205,
"epoch": 0.09633507853403141,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4823609590530396,
"fcm_dpo/q_t": 0.4962949752807617,
"grad_norm": 30.333192825317383,
"learning_rate": 4.6874999999999996e-07,
"logits/chosen": -0.6262620687484741,
"logits/rejected": -0.6398423314094543,
"logps/chosen": -296.63787841796875,
"logps/ref_chosen": -298.3357238769531,
"logps/ref_rejected": -267.66204833984375,
"logps/rejected": -267.4465637207031,
"loss": 5.4868,
"margin_dpo/margin_mean": 1.482360601425171,
"margin_dpo/margin_std": 2.6219446659088135,
"step": 46
},
{
"KL/chosen_KL_mean": 1.5099334716796875,
"KL/mean": 0.9814417362213135,
"KL/rejected_KL_mean": 0.45295143127441406,
"KL/std": 2.342404842376709,
"epoch": 0.09842931937172775,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0569857358932495,
"fcm_dpo/q_t": 0.4973580241203308,
"grad_norm": 26.228063583374023,
"learning_rate": 4.791666666666667e-07,
"logits/chosen": -0.601626992225647,
"logits/rejected": -0.6220702528953552,
"logps/chosen": -261.0569763183594,
"logps/ref_chosen": -262.5669250488281,
"logps/ref_rejected": -258.70989990234375,
"logps/rejected": -258.2569274902344,
"loss": 5.5042,
"margin_dpo/margin_mean": 1.0569851398468018,
"margin_dpo/margin_std": 3.353296995162964,
"step": 47
},
{
"KL/chosen_KL_mean": 1.8159637451171875,
"KL/mean": 1.0512405633926392,
"KL/rejected_KL_mean": 0.28652191162109375,
"KL/std": 2.5599913597106934,
"epoch": 0.10052356020942409,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.529449224472046,
"fcm_dpo/q_t": 0.4961783289909363,
"grad_norm": 27.525484085083008,
"learning_rate": 4.895833333333333e-07,
"logits/chosen": -0.6145200133323669,
"logits/rejected": -0.6397130489349365,
"logps/chosen": -267.67730712890625,
"logps/ref_chosen": -269.4932556152344,
"logps/ref_rejected": -241.888916015625,
"logps/rejected": -241.60238647460938,
"loss": 5.4853,
"margin_dpo/margin_mean": 1.529449224472046,
"margin_dpo/margin_std": 3.0062029361724854,
"step": 48
},
{
"KL/chosen_KL_mean": 2.173074722290039,
"KL/mean": 1.3489664793014526,
"KL/rejected_KL_mean": 0.5248641967773438,
"KL/std": 2.607396125793457,
"epoch": 0.10261780104712041,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6482032537460327,
"fcm_dpo/q_t": 0.49588078260421753,
"grad_norm": 27.708139419555664,
"learning_rate": 5e-07,
"logits/chosen": -0.6658346652984619,
"logits/rejected": -0.6502586007118225,
"logps/chosen": -255.71141052246094,
"logps/ref_chosen": -257.8844909667969,
"logps/ref_rejected": -256.8912048339844,
"logps/rejected": -256.3663330078125,
"loss": 5.4807,
"margin_dpo/margin_mean": 1.6482040882110596,
"margin_dpo/margin_std": 3.3653252124786377,
"step": 49
},
{
"KL/chosen_KL_mean": 1.9860305786132812,
"KL/mean": 1.4248580932617188,
"KL/rejected_KL_mean": 0.8636798858642578,
"KL/std": 3.0206358432769775,
"epoch": 0.10471204188481675,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1223528385162354,
"fcm_dpo/q_t": 0.4971952736377716,
"grad_norm": 28.091381072998047,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": -0.6322698593139648,
"logits/rejected": -0.6543733477592468,
"logps/chosen": -299.642822265625,
"logps/ref_chosen": -301.62884521484375,
"logps/ref_rejected": -298.2716064453125,
"logps/rejected": -297.4079284667969,
"loss": 5.502,
"margin_dpo/margin_mean": 1.1223524808883667,
"margin_dpo/margin_std": 3.72768497467041,
"step": 50
},
{
"KL/chosen_KL_mean": 2.0781002044677734,
"KL/mean": 1.165189504623413,
"KL/rejected_KL_mean": 0.2522716522216797,
"KL/std": 3.6036739349365234,
"epoch": 0.1068062827225131,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.825838565826416,
"fcm_dpo/q_t": 0.4954381287097931,
"grad_norm": 29.186372756958008,
"learning_rate": 4.999731868769026e-07,
"logits/chosen": -0.6121972799301147,
"logits/rejected": -0.6063085794448853,
"logps/chosen": -267.2942810058594,
"logps/ref_chosen": -269.37237548828125,
"logps/ref_rejected": -297.0167541503906,
"logps/rejected": -296.76446533203125,
"loss": 5.4749,
"margin_dpo/margin_mean": 1.8258390426635742,
"margin_dpo/margin_std": 4.549215793609619,
"step": 51
},
{
"KL/chosen_KL_mean": 2.7024993896484375,
"KL/mean": 1.1765809059143066,
"KL/rejected_KL_mean": -0.3493385314941406,
"KL/std": 3.7803289890289307,
"epoch": 0.10890052356020942,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.0518381595611572,
"fcm_dpo/q_t": 0.4923774003982544,
"grad_norm": 30.223594665527344,
"learning_rate": 4.99939671821067e-07,
"logits/chosen": -0.659646213054657,
"logits/rejected": -0.6663126945495605,
"logps/chosen": -304.2002868652344,
"logps/ref_chosen": -306.9028015136719,
"logps/ref_rejected": -281.24737548828125,
"logps/rejected": -281.5967102050781,
"loss": 5.4265,
"margin_dpo/margin_mean": 3.051837682723999,
"margin_dpo/margin_std": 4.751425266265869,
"step": 52
},
{
"KL/chosen_KL_mean": 2.787456512451172,
"KL/mean": 1.679011583328247,
"KL/rejected_KL_mean": 0.5705699920654297,
"KL/std": 4.107968807220459,
"epoch": 0.11099476439790576,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.216883659362793,
"fcm_dpo/q_t": 0.49446216225624084,
"grad_norm": 30.71049690246582,
"learning_rate": 4.998927532591591e-07,
"logits/chosen": -0.65622878074646,
"logits/rejected": -0.6969560980796814,
"logps/chosen": -283.1884765625,
"logps/ref_chosen": -285.9759521484375,
"logps/ref_rejected": -273.9073486328125,
"logps/rejected": -273.3367614746094,
"loss": 5.4601,
"margin_dpo/margin_mean": 2.216883659362793,
"margin_dpo/margin_std": 5.178028106689453,
"step": 53
},
{
"KL/chosen_KL_mean": 2.6346054077148438,
"KL/mean": 1.7063802480697632,
"KL/rejected_KL_mean": 0.7781620025634766,
"KL/std": 4.234417915344238,
"epoch": 0.1130890052356021,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8564293384552002,
"fcm_dpo/q_t": 0.49536406993865967,
"grad_norm": 26.411357879638672,
"learning_rate": 4.998324337072792e-07,
"logits/chosen": -0.6882258653640747,
"logits/rejected": -0.6963566541671753,
"logps/chosen": -303.8699951171875,
"logps/ref_chosen": -306.504638671875,
"logps/ref_rejected": -272.67431640625,
"logps/rejected": -271.8961486816406,
"loss": 5.4744,
"margin_dpo/margin_mean": 1.856428861618042,
"margin_dpo/margin_std": 5.311679840087891,
"step": 54
},
{
"KL/chosen_KL_mean": 1.7031211853027344,
"KL/mean": 0.5867319107055664,
"KL/rejected_KL_mean": -0.5296592712402344,
"KL/std": 3.6960041522979736,
"epoch": 0.11518324607329843,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.2327768802642822,
"fcm_dpo/q_t": 0.49442318081855774,
"grad_norm": 24.854549407958984,
"learning_rate": 4.997587164001815e-07,
"logits/chosen": -0.6439815163612366,
"logits/rejected": -0.6484256982803345,
"logps/chosen": -220.6270294189453,
"logps/ref_chosen": -222.33013916015625,
"logps/ref_rejected": -206.59571838378906,
"logps/rejected": -207.12538146972656,
"loss": 5.4588,
"margin_dpo/margin_mean": 2.2327778339385986,
"margin_dpo/margin_std": 4.8868231773376465,
"step": 55
},
{
"KL/chosen_KL_mean": 2.8303165435791016,
"KL/mean": 1.238706350326538,
"KL/rejected_KL_mean": -0.3529033660888672,
"KL/std": 4.7677001953125,
"epoch": 0.11727748691099477,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.1832265853881836,
"fcm_dpo/q_t": 0.49205446243286133,
"grad_norm": 27.35767936706543,
"learning_rate": 4.996716052911017e-07,
"logits/chosen": -0.6080012917518616,
"logits/rejected": -0.622451663017273,
"logps/chosen": -247.64785766601562,
"logps/ref_chosen": -250.47816467285156,
"logps/ref_rejected": -228.25848388671875,
"logps/rejected": -228.61138916015625,
"loss": 5.4231,
"margin_dpo/margin_mean": 3.1832268238067627,
"margin_dpo/margin_std": 5.896901607513428,
"step": 56
},
{
"KL/chosen_KL_mean": 3.8403587341308594,
"KL/mean": 1.7561249732971191,
"KL/rejected_KL_mean": -0.32811737060546875,
"KL/std": 4.8524932861328125,
"epoch": 0.1193717277486911,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.16848611831665,
"fcm_dpo/q_t": 0.48959094285964966,
"grad_norm": 30.88191032409668,
"learning_rate": 4.99571105051544e-07,
"logits/chosen": -0.6883333921432495,
"logits/rejected": -0.6583897471427917,
"logps/chosen": -311.2791442871094,
"logps/ref_chosen": -315.1195373535156,
"logps/ref_rejected": -272.755615234375,
"logps/rejected": -273.083740234375,
"loss": 5.3837,
"margin_dpo/margin_mean": 4.168484687805176,
"margin_dpo/margin_std": 5.704001426696777,
"step": 57
},
{
"KL/chosen_KL_mean": 2.4969558715820312,
"KL/mean": 1.277438998222351,
"KL/rejected_KL_mean": 0.05792045593261719,
"KL/std": 4.529829025268555,
"epoch": 0.12146596858638743,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.439037322998047,
"fcm_dpo/q_t": 0.49390649795532227,
"grad_norm": 27.652095794677734,
"learning_rate": 4.994572210710314e-07,
"logits/chosen": -0.6133895516395569,
"logits/rejected": -0.6374213099479675,
"logps/chosen": -262.6847229003906,
"logps/ref_chosen": -265.1816711425781,
"logps/ref_rejected": -268.2203369140625,
"logps/rejected": -268.16241455078125,
"loss": 5.4518,
"margin_dpo/margin_mean": 2.439037799835205,
"margin_dpo/margin_std": 5.721737861633301,
"step": 58
},
{
"KL/chosen_KL_mean": 2.1310195922851562,
"KL/mean": 1.3431408405303955,
"KL/rejected_KL_mean": 0.555267333984375,
"KL/std": 5.277097225189209,
"epoch": 0.12356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5757672786712646,
"fcm_dpo/q_t": 0.4960674047470093,
"grad_norm": 29.76839828491211,
"learning_rate": 4.993299594568162e-07,
"logits/chosen": -0.6114040613174438,
"logits/rejected": -0.6053056120872498,
"logps/chosen": -284.22296142578125,
"logps/ref_chosen": -286.35394287109375,
"logps/ref_rejected": -260.6757507324219,
"logps/rejected": -260.1204833984375,
"loss": 5.4881,
"margin_dpo/margin_mean": 1.5757670402526855,
"margin_dpo/margin_std": 7.032860279083252,
"step": 59
},
{
"KL/chosen_KL_mean": 2.8480281829833984,
"KL/mean": 1.117538332939148,
"KL/rejected_KL_mean": -0.6129531860351562,
"KL/std": 5.176889419555664,
"epoch": 0.1256544502617801,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.4609785079956055,
"fcm_dpo/q_t": 0.49135878682136536,
"grad_norm": 28.109230041503906,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": -0.6809808611869812,
"logits/rejected": -0.7057399749755859,
"logps/chosen": -255.9005889892578,
"logps/ref_chosen": -258.74859619140625,
"logps/ref_rejected": -255.04893493652344,
"logps/rejected": -255.661865234375,
"loss": 5.4135,
"margin_dpo/margin_mean": 3.460980176925659,
"margin_dpo/margin_std": 7.187258720397949,
"step": 60
},
{
"KL/chosen_KL_mean": 3.034067153930664,
"KL/mean": 1.4287720918655396,
"KL/rejected_KL_mean": -0.17651939392089844,
"KL/std": 5.567894458770752,
"epoch": 0.12774869109947645,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.210594892501831,
"fcm_dpo/q_t": 0.49198687076568604,
"grad_norm": 29.902263641357422,
"learning_rate": 4.990353313429303e-07,
"logits/chosen": -0.654046356678009,
"logits/rejected": -0.6714726686477661,
"logps/chosen": -275.433837890625,
"logps/ref_chosen": -278.4678955078125,
"logps/ref_rejected": -252.02720642089844,
"logps/rejected": -252.20370483398438,
"loss": 5.4244,
"margin_dpo/margin_mean": 3.210594654083252,
"margin_dpo/margin_std": 7.630921840667725,
"step": 61
},
{
"KL/chosen_KL_mean": 3.9335670471191406,
"KL/mean": 2.268831968307495,
"KL/rejected_KL_mean": 0.6041011810302734,
"KL/std": 5.7209367752075195,
"epoch": 0.12984293193717278,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3294646739959717,
"fcm_dpo/q_t": 0.49169519543647766,
"grad_norm": 26.6936092376709,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": -0.6085910797119141,
"logits/rejected": -0.6502899527549744,
"logps/chosen": -268.9907531738281,
"logps/ref_chosen": -272.92431640625,
"logps/ref_rejected": -260.7935485839844,
"logps/rejected": -260.189453125,
"loss": 5.4188,
"margin_dpo/margin_mean": 3.3294646739959717,
"margin_dpo/margin_std": 7.3894362449646,
"step": 62
},
{
"KL/chosen_KL_mean": 3.3459014892578125,
"KL/mean": 1.5705658197402954,
"KL/rejected_KL_mean": -0.20477867126464844,
"KL/std": 5.626295566558838,
"epoch": 0.1319371727748691,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.550675868988037,
"fcm_dpo/q_t": 0.49113813042640686,
"grad_norm": 28.22681999206543,
"learning_rate": 4.986872839090852e-07,
"logits/chosen": -0.6682570576667786,
"logits/rejected": -0.6751868724822998,
"logps/chosen": -273.74310302734375,
"logps/ref_chosen": -277.0889892578125,
"logps/ref_rejected": -273.3413391113281,
"logps/rejected": -273.5461120605469,
"loss": 5.4101,
"margin_dpo/margin_mean": 3.5506744384765625,
"margin_dpo/margin_std": 7.26901912689209,
"step": 63
},
{
"KL/chosen_KL_mean": 4.068166732788086,
"KL/mean": 1.9456909894943237,
"KL/rejected_KL_mean": -0.17678260803222656,
"KL/std": 6.259605884552002,
"epoch": 0.13403141361256546,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.244952201843262,
"fcm_dpo/q_t": 0.4894087612628937,
"grad_norm": 28.343130111694336,
"learning_rate": 4.9849325083059e-07,
"logits/chosen": -0.6286708116531372,
"logits/rejected": -0.6235312819480896,
"logps/chosen": -279.7563171386719,
"logps/ref_chosen": -283.8244934082031,
"logps/ref_rejected": -263.29351806640625,
"logps/rejected": -263.4703063964844,
"loss": 5.3846,
"margin_dpo/margin_mean": 4.244952201843262,
"margin_dpo/margin_std": 8.277649879455566,
"step": 64
},
{
"KL/chosen_KL_mean": 3.4773826599121094,
"KL/mean": 1.7928293943405151,
"KL/rejected_KL_mean": 0.10827827453613281,
"KL/std": 5.792224407196045,
"epoch": 0.13612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3691048622131348,
"fcm_dpo/q_t": 0.49159109592437744,
"grad_norm": 27.745553970336914,
"learning_rate": 4.982858918131906e-07,
"logits/chosen": -0.7000569701194763,
"logits/rejected": -0.6681522727012634,
"logps/chosen": -261.3925476074219,
"logps/ref_chosen": -264.8699645996094,
"logps/ref_rejected": -268.5076904296875,
"logps/rejected": -268.3994140625,
"loss": 5.4174,
"margin_dpo/margin_mean": 3.3691043853759766,
"margin_dpo/margin_std": 7.4207940101623535,
"step": 65
},
{
"KL/chosen_KL_mean": 3.1046905517578125,
"KL/mean": 1.0091890096664429,
"KL/rejected_KL_mean": -1.0862960815429688,
"KL/std": 7.481138229370117,
"epoch": 0.1382198952879581,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.190981864929199,
"fcm_dpo/q_t": 0.48955801129341125,
"grad_norm": 27.39222526550293,
"learning_rate": 4.980652179769217e-07,
"logits/chosen": -0.6774306893348694,
"logits/rejected": -0.6973876357078552,
"logps/chosen": -269.8236389160156,
"logps/ref_chosen": -272.9283142089844,
"logps/ref_rejected": -280.94696044921875,
"logps/rejected": -282.03326416015625,
"loss": 5.39,
"margin_dpo/margin_mean": 4.190980911254883,
"margin_dpo/margin_std": 10.186588287353516,
"step": 66
},
{
"KL/chosen_KL_mean": 4.062643051147461,
"KL/mean": 2.216336250305176,
"KL/rejected_KL_mean": 0.370025634765625,
"KL/std": 7.127995014190674,
"epoch": 0.14031413612565444,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.6926159858703613,
"fcm_dpo/q_t": 0.49078747630119324,
"grad_norm": 25.49480438232422,
"learning_rate": 4.978312411558517e-07,
"logits/chosen": -0.6880432963371277,
"logits/rejected": -0.7203540802001953,
"logps/chosen": -262.12432861328125,
"logps/ref_chosen": -266.18695068359375,
"logps/ref_rejected": -250.17405700683594,
"logps/rejected": -249.80404663085938,
"loss": 5.4069,
"margin_dpo/margin_mean": 3.692615509033203,
"margin_dpo/margin_std": 8.700530052185059,
"step": 67
},
{
"KL/chosen_KL_mean": 3.052156448364258,
"KL/mean": 0.7123284935951233,
"KL/rejected_KL_mean": -1.627492904663086,
"KL/std": 8.075217247009277,
"epoch": 0.1424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.679654121398926,
"fcm_dpo/q_t": 0.4883608818054199,
"grad_norm": 27.840843200683594,
"learning_rate": 4.975839738974473e-07,
"logits/chosen": -0.6769158840179443,
"logits/rejected": -0.6895080208778381,
"logps/chosen": -294.8864440917969,
"logps/ref_chosen": -297.9385986328125,
"logps/ref_rejected": -261.5141296386719,
"logps/rejected": -263.1416015625,
"loss": 5.3723,
"margin_dpo/margin_mean": 4.679654598236084,
"margin_dpo/margin_std": 10.612685203552246,
"step": 68
},
{
"KL/chosen_KL_mean": 3.876893997192383,
"KL/mean": 0.8488324284553528,
"KL/rejected_KL_mean": -2.1792163848876953,
"KL/std": 7.927192211151123,
"epoch": 0.14450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.056118011474609,
"fcm_dpo/q_t": 0.48490819334983826,
"grad_norm": 28.45749855041504,
"learning_rate": 4.97323429461901e-07,
"logits/chosen": -0.6810729503631592,
"logits/rejected": -0.7114135026931763,
"logps/chosen": -261.7406311035156,
"logps/ref_chosen": -265.6175231933594,
"logps/ref_rejected": -236.8287353515625,
"logps/rejected": -239.00794982910156,
"loss": 5.3166,
"margin_dpo/margin_mean": 6.056118488311768,
"margin_dpo/margin_std": 9.694916725158691,
"step": 69
},
{
"KL/chosen_KL_mean": 4.329980850219727,
"KL/mean": 1.4181368350982666,
"KL/rejected_KL_mean": -1.4937019348144531,
"KL/std": 8.636222839355469,
"epoch": 0.14659685863874344,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.823677062988281,
"fcm_dpo/q_t": 0.48549583554267883,
"grad_norm": 28.639202117919922,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": -0.6776444315910339,
"logits/rejected": -0.7112736701965332,
"logps/chosen": -291.8959655761719,
"logps/ref_chosen": -296.2259216308594,
"logps/ref_rejected": -254.68496704101562,
"logps/rejected": -256.17864990234375,
"loss": 5.3285,
"margin_dpo/margin_mean": 5.823675632476807,
"margin_dpo/margin_std": 10.906417846679688,
"step": 70
},
{
"KL/chosen_KL_mean": 5.135082244873047,
"KL/mean": 2.540114402770996,
"KL/rejected_KL_mean": -0.054851531982421875,
"KL/std": 9.344449043273926,
"epoch": 0.1486910994764398,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.189933776855469,
"fcm_dpo/q_t": 0.48706474900245667,
"grad_norm": 28.435436248779297,
"learning_rate": 4.967625656594781e-07,
"logits/chosen": -0.6521185636520386,
"logits/rejected": -0.6411818265914917,
"logps/chosen": -283.7921447753906,
"logps/ref_chosen": -288.92724609375,
"logps/ref_rejected": -278.6405334472656,
"logps/rejected": -278.69537353515625,
"loss": 5.3567,
"margin_dpo/margin_mean": 5.189933776855469,
"margin_dpo/margin_std": 12.259124755859375,
"step": 71
},
{
"KL/chosen_KL_mean": 3.259723663330078,
"KL/mean": 0.7097737789154053,
"KL/rejected_KL_mean": -1.8401775360107422,
"KL/std": 8.4075288772583,
"epoch": 0.15078534031413612,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.099894046783447,
"fcm_dpo/q_t": 0.4872819781303406,
"grad_norm": 28.07781982421875,
"learning_rate": 4.964622763700252e-07,
"logits/chosen": -0.6942219734191895,
"logits/rejected": -0.7054851651191711,
"logps/chosen": -233.7855682373047,
"logps/ref_chosen": -237.0452880859375,
"logps/ref_rejected": -252.7946319580078,
"logps/rejected": -254.63479614257812,
"loss": 5.3565,
"margin_dpo/margin_mean": 5.0998945236206055,
"margin_dpo/margin_std": 10.471480369567871,
"step": 72
},
{
"KL/chosen_KL_mean": 4.394702911376953,
"KL/mean": 1.990799069404602,
"KL/rejected_KL_mean": -0.41310882568359375,
"KL/std": 8.965617179870605,
"epoch": 0.15287958115183245,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.80781888961792,
"fcm_dpo/q_t": 0.48800960183143616,
"grad_norm": 27.92042350769043,
"learning_rate": 4.961487700566646e-07,
"logits/chosen": -0.6583269238471985,
"logits/rejected": -0.6759439706802368,
"logps/chosen": -268.65838623046875,
"logps/ref_chosen": -273.0531005859375,
"logps/ref_rejected": -246.8330841064453,
"logps/rejected": -247.24620056152344,
"loss": 5.3714,
"margin_dpo/margin_mean": 4.8078179359436035,
"margin_dpo/margin_std": 12.222956657409668,
"step": 73
},
{
"KL/chosen_KL_mean": 3.3620052337646484,
"KL/mean": 1.1552921533584595,
"KL/rejected_KL_mean": -1.051422119140625,
"KL/std": 9.413747787475586,
"epoch": 0.1549738219895288,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.413438320159912,
"fcm_dpo/q_t": 0.48901113867759705,
"grad_norm": 30.360273361206055,
"learning_rate": 4.958220635317885e-07,
"logits/chosen": -0.7333360910415649,
"logits/rejected": -0.7118159532546997,
"logps/chosen": -338.9198913574219,
"logps/ref_chosen": -342.2818908691406,
"logps/ref_rejected": -330.0293884277344,
"logps/rejected": -331.08074951171875,
"loss": 5.3855,
"margin_dpo/margin_mean": 4.41343879699707,
"margin_dpo/margin_std": 11.758991241455078,
"step": 74
},
{
"KL/chosen_KL_mean": 4.601959228515625,
"KL/mean": 1.333653211593628,
"KL/rejected_KL_mean": -1.9346542358398438,
"KL/std": 9.369311332702637,
"epoch": 0.15706806282722513,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.536617279052734,
"fcm_dpo/q_t": 0.48371294140815735,
"grad_norm": 29.381935119628906,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": -0.6344682574272156,
"logits/rejected": -0.6357614398002625,
"logps/chosen": -262.2621765136719,
"logps/ref_chosen": -266.8641662597656,
"logps/ref_rejected": -276.8699951171875,
"logps/rejected": -278.8046875,
"loss": 5.3013,
"margin_dpo/margin_mean": 6.536618709564209,
"margin_dpo/margin_std": 10.953182220458984,
"step": 75
},
{
"KL/chosen_KL_mean": 4.06199836730957,
"KL/mean": 0.48307615518569946,
"KL/rejected_KL_mean": -3.095844268798828,
"KL/std": 9.946473121643066,
"epoch": 0.15916230366492146,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.157842636108398,
"fcm_dpo/q_t": 0.4822140634059906,
"grad_norm": 29.176395416259766,
"learning_rate": 4.951291206355559e-07,
"logits/chosen": -0.7243552803993225,
"logits/rejected": -0.7324966788291931,
"logps/chosen": -277.1125793457031,
"logps/ref_chosen": -281.174560546875,
"logps/ref_rejected": -263.6067199707031,
"logps/rejected": -266.7025451660156,
"loss": 5.2816,
"margin_dpo/margin_mean": 7.157844543457031,
"margin_dpo/margin_std": 12.785238265991211,
"step": 76
},
{
"KL/chosen_KL_mean": 3.7531566619873047,
"KL/mean": 0.8536058068275452,
"KL/rejected_KL_mean": -2.0459423065185547,
"KL/std": 11.038262367248535,
"epoch": 0.1612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.799106121063232,
"fcm_dpo/q_t": 0.485598623752594,
"grad_norm": 33.110633850097656,
"learning_rate": 4.947629214246236e-07,
"logits/chosen": -0.554699718952179,
"logits/rejected": -0.5627188682556152,
"logps/chosen": -302.34210205078125,
"logps/ref_chosen": -306.09527587890625,
"logps/ref_rejected": -253.49569702148438,
"logps/rejected": -255.54164123535156,
"loss": 5.3375,
"margin_dpo/margin_mean": 5.799104690551758,
"margin_dpo/margin_std": 13.92863655090332,
"step": 77
},
{
"KL/chosen_KL_mean": 3.8339462280273438,
"KL/mean": -0.49760693311691284,
"KL/rejected_KL_mean": -4.829154968261719,
"KL/std": 11.0235595703125,
"epoch": 0.16335078534031414,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.663091659545898,
"fcm_dpo/q_t": 0.47850561141967773,
"grad_norm": 29.756555557250977,
"learning_rate": 4.943835963210323e-07,
"logits/chosen": -0.6848552227020264,
"logits/rejected": -0.6797273755073547,
"logps/chosen": -253.0684051513672,
"logps/ref_chosen": -256.90234375,
"logps/ref_rejected": -211.57154846191406,
"logps/rejected": -216.40072631835938,
"loss": 5.228,
"margin_dpo/margin_mean": 8.663091659545898,
"margin_dpo/margin_std": 14.5060453414917,
"step": 78
},
{
"KL/chosen_KL_mean": 3.0965023040771484,
"KL/mean": -1.057416319847107,
"KL/rejected_KL_mean": -5.211338043212891,
"KL/std": 11.727227210998535,
"epoch": 0.16544502617801046,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.307835578918457,
"fcm_dpo/q_t": 0.4793843924999237,
"grad_norm": 29.793476104736328,
"learning_rate": 4.939911656668361e-07,
"logits/chosen": -0.6616241335868835,
"logits/rejected": -0.6827108263969421,
"logps/chosen": -263.17706298828125,
"logps/ref_chosen": -266.2735595703125,
"logps/ref_rejected": -251.57257080078125,
"logps/rejected": -256.7839050292969,
"loss": 5.2436,
"margin_dpo/margin_mean": 8.307835578918457,
"margin_dpo/margin_std": 14.920553207397461,
"step": 79
},
{
"KL/chosen_KL_mean": 1.841531753540039,
"KL/mean": -1.4569592475891113,
"KL/rejected_KL_mean": -4.755453109741211,
"KL/std": 11.351509094238281,
"epoch": 0.16753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.596972465515137,
"fcm_dpo/q_t": 0.48362815380096436,
"grad_norm": 29.06841278076172,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": -0.6586440801620483,
"logits/rejected": -0.6877402663230896,
"logps/chosen": -286.00946044921875,
"logps/ref_chosen": -287.8509826660156,
"logps/ref_rejected": -256.0766296386719,
"logps/rejected": -260.8320617675781,
"loss": 5.3056,
"margin_dpo/margin_mean": 6.596972465515137,
"margin_dpo/margin_std": 13.254508972167969,
"step": 80
},
{
"KL/chosen_KL_mean": 2.3833045959472656,
"KL/mean": -1.1887388229370117,
"KL/rejected_KL_mean": -4.760778427124023,
"KL/std": 12.248699188232422,
"epoch": 0.16963350785340314,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.144074440002441,
"fcm_dpo/q_t": 0.48228341341018677,
"grad_norm": 28.06484031677246,
"learning_rate": 4.93167072587771e-07,
"logits/chosen": -0.6383923888206482,
"logits/rejected": -0.637448787689209,
"logps/chosen": -266.1399230957031,
"logps/ref_chosen": -268.5232238769531,
"logps/ref_rejected": -237.81137084960938,
"logps/rejected": -242.57215881347656,
"loss": 5.2942,
"margin_dpo/margin_mean": 7.1440749168396,
"margin_dpo/margin_std": 16.711862564086914,
"step": 81
},
{
"KL/chosen_KL_mean": 2.4359188079833984,
"KL/mean": -1.5335369110107422,
"KL/rejected_KL_mean": -5.50299072265625,
"KL/std": 12.241569519042969,
"epoch": 0.17172774869109947,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.938918590545654,
"fcm_dpo/q_t": 0.4803643226623535,
"grad_norm": 27.75286293029785,
"learning_rate": 4.92735454356513e-07,
"logits/chosen": -0.7282431721687317,
"logits/rejected": -0.7358072996139526,
"logps/chosen": -276.9280090332031,
"logps/ref_chosen": -279.36395263671875,
"logps/ref_rejected": -236.51365661621094,
"logps/rejected": -242.01666259765625,
"loss": 5.2623,
"margin_dpo/margin_mean": 7.9389190673828125,
"margin_dpo/margin_std": 15.775489807128906,
"step": 82
},
{
"KL/chosen_KL_mean": 0.3072986602783203,
"KL/mean": -4.027276039123535,
"KL/rejected_KL_mean": -8.361845016479492,
"KL/std": 13.603336334228516,
"epoch": 0.17382198952879582,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.669137954711914,
"fcm_dpo/q_t": 0.4784778654575348,
"grad_norm": 30.750106811523438,
"learning_rate": 4.922908189595017e-07,
"logits/chosen": -0.6986796855926514,
"logits/rejected": -0.6824924945831299,
"logps/chosen": -273.91192626953125,
"logps/ref_chosen": -274.21923828125,
"logps/ref_rejected": -276.2212219238281,
"logps/rejected": -284.58306884765625,
"loss": 5.2426,
"margin_dpo/margin_mean": 8.669137001037598,
"margin_dpo/margin_std": 18.15301513671875,
"step": 83
},
{
"KL/chosen_KL_mean": 0.5734672546386719,
"KL/mean": -3.0715651512145996,
"KL/rejected_KL_mean": -6.716592788696289,
"KL/std": 13.341272354125977,
"epoch": 0.17591623036649215,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.290060997009277,
"fcm_dpo/q_t": 0.48191478848457336,
"grad_norm": 29.86717987060547,
"learning_rate": 4.918331902411841e-07,
"logits/chosen": -0.7325999140739441,
"logits/rejected": -0.7469091415405273,
"logps/chosen": -293.8240966796875,
"logps/ref_chosen": -294.3975524902344,
"logps/ref_rejected": -279.81884765625,
"logps/rejected": -286.5354309082031,
"loss": 5.29,
"margin_dpo/margin_mean": 7.290060520172119,
"margin_dpo/margin_std": 16.862646102905273,
"step": 84
},
{
"KL/chosen_KL_mean": -1.566385269165039,
"KL/mean": -4.428199768066406,
"KL/rejected_KL_mean": -7.290014266967773,
"KL/std": 12.298978805541992,
"epoch": 0.17801047120418848,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.723620414733887,
"fcm_dpo/q_t": 0.48579931259155273,
"grad_norm": 29.314868927001953,
"learning_rate": 4.913625927427995e-07,
"logits/chosen": -0.6648886203765869,
"logits/rejected": -0.6732238531112671,
"logps/chosen": -245.22857666015625,
"logps/ref_chosen": -243.66220092773438,
"logps/ref_rejected": -263.9421691894531,
"logps/rejected": -271.2322082519531,
"loss": 5.3466,
"margin_dpo/margin_mean": 5.723620414733887,
"margin_dpo/margin_std": 15.493175506591797,
"step": 85
},
{
"KL/chosen_KL_mean": 1.2148284912109375,
"KL/mean": -3.267716646194458,
"KL/rejected_KL_mean": -7.75025749206543,
"KL/std": 13.284515380859375,
"epoch": 0.18010471204188483,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.965092658996582,
"fcm_dpo/q_t": 0.4777436852455139,
"grad_norm": 34.879512786865234,
"learning_rate": 4.908790517010636e-07,
"logits/chosen": -0.691657543182373,
"logits/rejected": -0.6852882504463196,
"logps/chosen": -308.2158508300781,
"logps/ref_chosen": -309.4306945800781,
"logps/ref_rejected": -290.91278076171875,
"logps/rejected": -298.66302490234375,
"loss": 5.2262,
"margin_dpo/margin_mean": 8.965091705322266,
"margin_dpo/margin_std": 17.340694427490234,
"step": 86
},
{
"KL/chosen_KL_mean": -0.16178321838378906,
"KL/mean": -5.300088882446289,
"KL/rejected_KL_mean": -10.438394546508789,
"KL/std": 15.289347648620605,
"epoch": 0.18219895287958116,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.276609420776367,
"fcm_dpo/q_t": 0.47462573647499084,
"grad_norm": 29.600753784179688,
"learning_rate": 4.903825930468148e-07,
"logits/chosen": -0.7515206933021545,
"logits/rejected": -0.7456373572349548,
"logps/chosen": -278.1894836425781,
"logps/ref_chosen": -278.0277099609375,
"logps/ref_rejected": -245.70123291015625,
"logps/rejected": -256.13958740234375,
"loss": 5.1833,
"margin_dpo/margin_mean": 10.276611328125,
"margin_dpo/margin_std": 18.967639923095703,
"step": 87
},
{
"KL/chosen_KL_mean": -2.0936412811279297,
"KL/mean": -6.569038391113281,
"KL/rejected_KL_mean": -11.044445037841797,
"KL/std": 15.05484676361084,
"epoch": 0.18429319371727748,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.950809478759766,
"fcm_dpo/q_t": 0.477876216173172,
"grad_norm": 28.82327651977539,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": -0.7612143754959106,
"logits/rejected": -0.7769263386726379,
"logps/chosen": -268.60845947265625,
"logps/ref_chosen": -266.5148010253906,
"logps/ref_rejected": -265.90081787109375,
"logps/rejected": -276.94525146484375,
"loss": 5.2333,
"margin_dpo/margin_mean": 8.950811386108398,
"margin_dpo/margin_std": 19.136362075805664,
"step": 88
},
{
"KL/chosen_KL_mean": 0.00550079345703125,
"KL/mean": -4.640643119812012,
"KL/rejected_KL_mean": -9.286785125732422,
"KL/std": 14.196423530578613,
"epoch": 0.18638743455497384,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.292278289794922,
"fcm_dpo/q_t": 0.4769834876060486,
"grad_norm": 30.551815032958984,
"learning_rate": 4.893510300863676e-07,
"logits/chosen": -0.7438966631889343,
"logits/rejected": -0.7343195080757141,
"logps/chosen": -265.68377685546875,
"logps/ref_chosen": -265.6893005371094,
"logps/ref_rejected": -251.49314880371094,
"logps/rejected": -260.7799377441406,
"loss": 5.2193,
"margin_dpo/margin_mean": 9.292276382446289,
"margin_dpo/margin_std": 18.199703216552734,
"step": 89
},
{
"KL/chosen_KL_mean": -1.1707630157470703,
"KL/mean": -5.281407356262207,
"KL/rejected_KL_mean": -9.39205551147461,
"KL/std": 14.30504322052002,
"epoch": 0.18848167539267016,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.221304893493652,
"fcm_dpo/q_t": 0.4796198904514313,
"grad_norm": 29.808265686035156,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": -0.7398777008056641,
"logits/rejected": -0.7484868764877319,
"logps/chosen": -308.59576416015625,
"logps/ref_chosen": -307.4250183105469,
"logps/ref_rejected": -265.7172546386719,
"logps/rejected": -275.10931396484375,
"loss": 5.2571,
"margin_dpo/margin_mean": 8.221305847167969,
"margin_dpo/margin_std": 17.676063537597656,
"step": 90
},
{
"KL/chosen_KL_mean": -2.1329708099365234,
"KL/mean": -6.8853864669799805,
"KL/rejected_KL_mean": -11.637798309326172,
"KL/std": 14.826963424682617,
"epoch": 0.1905759162303665,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.50483512878418,
"fcm_dpo/q_t": 0.47655919194221497,
"grad_norm": 32.95356750488281,
"learning_rate": 4.882681251368548e-07,
"logits/chosen": -0.6793421506881714,
"logits/rejected": -0.6965538859367371,
"logps/chosen": -237.87396240234375,
"logps/ref_chosen": -235.74098205566406,
"logps/ref_rejected": -226.6428985595703,
"logps/rejected": -238.2807159423828,
"loss": 5.2162,
"margin_dpo/margin_mean": 9.504833221435547,
"margin_dpo/margin_std": 19.24348258972168,
"step": 91
},
{
"KL/chosen_KL_mean": -3.134540557861328,
"KL/mean": -8.218965530395508,
"KL/rejected_KL_mean": -13.303377151489258,
"KL/std": 17.46491050720215,
"epoch": 0.19267015706806281,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.168837547302246,
"fcm_dpo/q_t": 0.47487232089042664,
"grad_norm": 33.593353271484375,
"learning_rate": 4.877074915775048e-07,
"logits/chosen": -0.7426879405975342,
"logits/rejected": -0.7264429330825806,
"logps/chosen": -286.58209228515625,
"logps/ref_chosen": -283.4475402832031,
"logps/ref_rejected": -273.134033203125,
"logps/rejected": -286.4374084472656,
"loss": 5.1966,
"margin_dpo/margin_mean": 10.168838500976562,
"margin_dpo/margin_std": 21.30776596069336,
"step": 92
},
{
"KL/chosen_KL_mean": -2.5298290252685547,
"KL/mean": -7.310698986053467,
"KL/rejected_KL_mean": -12.091564178466797,
"KL/std": 15.477099418640137,
"epoch": 0.19476439790575917,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.56174087524414,
"fcm_dpo/q_t": 0.4764096438884735,
"grad_norm": 28.567649841308594,
"learning_rate": 4.871341104867864e-07,
"logits/chosen": -0.7310028076171875,
"logits/rejected": -0.7545213103294373,
"logps/chosen": -235.86695861816406,
"logps/ref_chosen": -233.33714294433594,
"logps/ref_rejected": -230.54273986816406,
"logps/rejected": -242.63427734375,
"loss": 5.2129,
"margin_dpo/margin_mean": 9.561741828918457,
"margin_dpo/margin_std": 19.498998641967773,
"step": 93
},
{
"KL/chosen_KL_mean": -2.3857421875,
"KL/mean": -7.097320079803467,
"KL/rejected_KL_mean": -11.808910369873047,
"KL/std": 16.434885025024414,
"epoch": 0.1968586387434555,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.423165321350098,
"fcm_dpo/q_t": 0.47669246792793274,
"grad_norm": 31.399282455444336,
"learning_rate": 4.865480126133871e-07,
"logits/chosen": -0.6810243725776672,
"logits/rejected": -0.701765775680542,
"logps/chosen": -297.03851318359375,
"logps/ref_chosen": -294.6528015136719,
"logps/ref_rejected": -283.657958984375,
"logps/rejected": -295.4669189453125,
"loss": 5.2283,
"margin_dpo/margin_mean": 9.423166275024414,
"margin_dpo/margin_std": 21.879398345947266,
"step": 94
},
{
"KL/chosen_KL_mean": -3.322275161743164,
"KL/mean": -8.415876388549805,
"KL/rejected_KL_mean": -13.50948715209961,
"KL/std": 18.067832946777344,
"epoch": 0.19895287958115182,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.187210083007812,
"fcm_dpo/q_t": 0.474922239780426,
"grad_norm": 33.59168243408203,
"learning_rate": 4.859492293879573e-07,
"logits/chosen": -0.7242058515548706,
"logits/rejected": -0.745286226272583,
"logps/chosen": -314.9919738769531,
"logps/ref_chosen": -311.6697082519531,
"logps/ref_rejected": -262.7471923828125,
"logps/rejected": -276.2567138671875,
"loss": 5.2054,
"margin_dpo/margin_mean": 10.187210083007812,
"margin_dpo/margin_std": 22.50189208984375,
"step": 95
},
{
"KL/chosen_KL_mean": -4.764711380004883,
"KL/mean": -9.865034103393555,
"KL/rejected_KL_mean": -14.965354919433594,
"KL/std": 19.30546760559082,
"epoch": 0.20104712041884817,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.20064640045166,
"fcm_dpo/q_t": 0.4748993217945099,
"grad_norm": 32.950992584228516,
"learning_rate": 4.853377929214243e-07,
"logits/chosen": -0.7030908465385437,
"logits/rejected": -0.7156802415847778,
"logps/chosen": -287.3207092285156,
"logps/ref_chosen": -282.55596923828125,
"logps/ref_rejected": -242.71588134765625,
"logps/rejected": -257.68121337890625,
"loss": 5.2057,
"margin_dpo/margin_mean": 10.200647354125977,
"margin_dpo/margin_std": 23.298458099365234,
"step": 96
},
{
"KL/chosen_KL_mean": -4.412008285522461,
"KL/mean": -10.414529800415039,
"KL/rejected_KL_mean": -16.41705322265625,
"KL/std": 17.29755401611328,
"epoch": 0.2031413612565445,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.005033493041992,
"fcm_dpo/q_t": 0.4704352617263794,
"grad_norm": 32.43635940551758,
"learning_rate": 4.847137360032699e-07,
"logits/chosen": -0.7456763982772827,
"logits/rejected": -0.7329200506210327,
"logps/chosen": -307.98980712890625,
"logps/ref_chosen": -303.57781982421875,
"logps/ref_rejected": -264.22491455078125,
"logps/rejected": -280.6419982910156,
"loss": 5.1326,
"margin_dpo/margin_mean": 12.005033493041992,
"margin_dpo/margin_std": 22.390636444091797,
"step": 97
},
{
"KL/chosen_KL_mean": -4.835039138793945,
"KL/mean": -10.993499755859375,
"KL/rejected_KL_mean": -17.151966094970703,
"KL/std": 18.9281005859375,
"epoch": 0.20523560209424083,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.31692886352539,
"fcm_dpo/q_t": 0.4698448181152344,
"grad_norm": 34.54245376586914,
"learning_rate": 4.84077092099773e-07,
"logits/chosen": -0.7799739837646484,
"logits/rejected": -0.7929960489273071,
"logps/chosen": -291.6654052734375,
"logps/ref_chosen": -286.8303527832031,
"logps/ref_rejected": -278.08331298828125,
"logps/rejected": -295.23529052734375,
"loss": 5.1325,
"margin_dpo/margin_mean": 12.31692886352539,
"margin_dpo/margin_std": 22.748985290527344,
"step": 98
},
{
"KL/chosen_KL_mean": -5.303714752197266,
"KL/mean": -11.64534854888916,
"KL/rejected_KL_mean": -17.986982345581055,
"KL/std": 19.881885528564453,
"epoch": 0.20732984293193718,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.683259963989258,
"fcm_dpo/q_t": 0.4689215123653412,
"grad_norm": 32.6298942565918,
"learning_rate": 4.834278953522137e-07,
"logits/chosen": -0.7404739260673523,
"logits/rejected": -0.7537968754768372,
"logps/chosen": -285.22491455078125,
"logps/ref_chosen": -279.92120361328125,
"logps/ref_rejected": -250.3365478515625,
"logps/rejected": -268.32354736328125,
"loss": 5.1338,
"margin_dpo/margin_mean": 12.683258056640625,
"margin_dpo/margin_std": 27.28083610534668,
"step": 99
},
{
"KL/chosen_KL_mean": -7.890239715576172,
"KL/mean": -14.145411491394043,
"KL/rejected_KL_mean": -20.400583267211914,
"KL/std": 21.990659713745117,
"epoch": 0.2094240837696335,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.510345458984375,
"fcm_dpo/q_t": 0.4693695604801178,
"grad_norm": 35.569637298583984,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": -0.7786294221878052,
"logits/rejected": -0.7915595173835754,
"logps/chosen": -304.7179260253906,
"logps/ref_chosen": -296.8276672363281,
"logps/ref_rejected": -275.56146240234375,
"logps/rejected": -295.9620666503906,
"loss": 5.1248,
"margin_dpo/margin_mean": 12.510345458984375,
"margin_dpo/margin_std": 24.131572723388672,
"step": 100
},
{
"KL/chosen_KL_mean": -5.328624725341797,
"KL/mean": -12.77022647857666,
"KL/rejected_KL_mean": -20.21182632446289,
"KL/std": 22.71660041809082,
"epoch": 0.21151832460732983,
"fcm_dpo/beta": 0.010064744390547276,
"fcm_dpo/delta": 0.012882252223789692,
"fcm_dpo/margin": 14.883186340332031,
"fcm_dpo/q_t": 0.4634419083595276,
"grad_norm": 32.82856369018555,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": -0.7667604684829712,
"logits/rejected": -0.7804319262504578,
"logps/chosen": -258.0706481933594,
"logps/ref_chosen": -252.74203491210938,
"logps/ref_rejected": -276.4185485839844,
"logps/rejected": -296.6304016113281,
"loss": 5.0441,
"margin_dpo/margin_mean": 14.883186340332031,
"margin_dpo/margin_std": 26.3232421875,
"step": 101
},
{
"KL/chosen_KL_mean": -5.145671844482422,
"KL/mean": -12.559816360473633,
"KL/rejected_KL_mean": -19.973960876464844,
"KL/std": 19.77225112915039,
"epoch": 0.2136125654450262,
"fcm_dpo/beta": 0.010103590786457062,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 14.828295707702637,
"fcm_dpo/q_t": 0.4630769193172455,
"grad_norm": 32.874755859375,
"learning_rate": 4.814053395442932e-07,
"logits/chosen": -0.7466103434562683,
"logits/rejected": -0.7427460551261902,
"logps/chosen": -224.69940185546875,
"logps/ref_chosen": -219.5537109375,
"logps/ref_rejected": -231.90853881835938,
"logps/rejected": -251.88250732421875,
"loss": 5.0383,
"margin_dpo/margin_mean": 14.828296661376953,
"margin_dpo/margin_std": 25.028118133544922,
"step": 102
},
{
"KL/chosen_KL_mean": -4.823522567749023,
"KL/mean": -11.646297454833984,
"KL/rejected_KL_mean": -18.46908187866211,
"KL/std": 21.67599868774414,
"epoch": 0.2157068062827225,
"fcm_dpo/beta": 0.010103590786457062,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.645557403564453,
"fcm_dpo/q_t": 0.4664449691772461,
"grad_norm": 33.129119873046875,
"learning_rate": 4.807062862684873e-07,
"logits/chosen": -0.7692456245422363,
"logits/rejected": -0.7661339044570923,
"logps/chosen": -264.4985656738281,
"logps/ref_chosen": -259.6750793457031,
"logps/ref_rejected": -278.7400817871094,
"logps/rejected": -297.20916748046875,
"loss": 5.0855,
"margin_dpo/margin_mean": 13.64555835723877,
"margin_dpo/margin_std": 25.76202392578125,
"step": 103
},
{
"KL/chosen_KL_mean": -9.094964981079102,
"KL/mean": -14.14463996887207,
"KL/rejected_KL_mean": -19.194313049316406,
"KL/std": 22.547658920288086,
"epoch": 0.21780104712041884,
"fcm_dpo/beta": 0.010103590786457062,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.099353790283203,
"fcm_dpo/q_t": 0.475015789270401,
"grad_norm": 33.46672821044922,
"learning_rate": 4.799948609147061e-07,
"logits/chosen": -0.7669414281845093,
"logits/rejected": -0.7730291485786438,
"logps/chosen": -277.06915283203125,
"logps/ref_chosen": -267.9741516113281,
"logps/ref_rejected": -230.5306396484375,
"logps/rejected": -249.7249298095703,
"loss": 5.2216,
"margin_dpo/margin_mean": 10.099355697631836,
"margin_dpo/margin_std": 26.440128326416016,
"step": 104
},
{
"KL/chosen_KL_mean": -5.365236282348633,
"KL/mean": -15.661399841308594,
"KL/rejected_KL_mean": -25.957565307617188,
"KL/std": 25.220060348510742,
"epoch": 0.2198952879581152,
"fcm_dpo/beta": 0.01026467327028513,
"fcm_dpo/delta": 0.03175576403737068,
"fcm_dpo/margin": 20.592317581176758,
"fcm_dpo/q_t": 0.4485164284706116,
"grad_norm": 35.88564682006836,
"learning_rate": 4.792711016345321e-07,
"logits/chosen": -0.7539578676223755,
"logits/rejected": -0.7634439468383789,
"logps/chosen": -327.6200866699219,
"logps/ref_chosen": -322.25482177734375,
"logps/ref_rejected": -279.02978515625,
"logps/rejected": -304.9873352050781,
"loss": 4.8304,
"margin_dpo/margin_mean": 20.592321395874023,
"margin_dpo/margin_std": 27.02099609375,
"step": 105
},
{
"KL/chosen_KL_mean": -12.540826797485352,
"KL/mean": -18.82832145690918,
"KL/rejected_KL_mean": -25.115812301635742,
"KL/std": 23.56497573852539,
"epoch": 0.22198952879581152,
"fcm_dpo/beta": 0.010363556444644928,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.57497787475586,
"fcm_dpo/q_t": 0.46819162368774414,
"grad_norm": 39.138980865478516,
"learning_rate": 4.785350472409791e-07,
"logits/chosen": -0.7461320757865906,
"logits/rejected": -0.7831467390060425,
"logps/chosen": -308.6986389160156,
"logps/ref_chosen": -296.15777587890625,
"logps/ref_rejected": -266.2691650390625,
"logps/rejected": -291.3849792480469,
"loss": 5.1392,
"margin_dpo/margin_mean": 12.57497787475586,
"margin_dpo/margin_std": 29.43160629272461,
"step": 106
},
{
"KL/chosen_KL_mean": -4.413387298583984,
"KL/mean": -14.257648468017578,
"KL/rejected_KL_mean": -24.101911544799805,
"KL/std": 22.089332580566406,
"epoch": 0.22408376963350785,
"fcm_dpo/beta": 0.010363556444644928,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.688522338867188,
"fcm_dpo/q_t": 0.45019635558128357,
"grad_norm": 39.04673767089844,
"learning_rate": 4.777867372064105e-07,
"logits/chosen": -0.779481053352356,
"logits/rejected": -0.7725954055786133,
"logps/chosen": -311.4096984863281,
"logps/ref_chosen": -306.996337890625,
"logps/ref_rejected": -296.79412841796875,
"logps/rejected": -320.89599609375,
"loss": 4.8578,
"margin_dpo/margin_mean": 19.688522338867188,
"margin_dpo/margin_std": 27.66748046875,
"step": 107
},
{
"KL/chosen_KL_mean": -5.592203140258789,
"KL/mean": -14.71112060546875,
"KL/rejected_KL_mean": -23.830047607421875,
"KL/std": 25.131715774536133,
"epoch": 0.2261780104712042,
"fcm_dpo/beta": 0.010363556444644928,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.23786163330078,
"fcm_dpo/q_t": 0.4543207287788391,
"grad_norm": 105.16789245605469,
"learning_rate": 4.770262116604223e-07,
"logits/chosen": -0.7518104314804077,
"logits/rejected": -0.7614055871963501,
"logps/chosen": -300.744873046875,
"logps/ref_chosen": -295.1526794433594,
"logps/ref_rejected": -235.974853515625,
"logps/rejected": -259.8049011230469,
"loss": 4.9361,
"margin_dpo/margin_mean": 18.23785972595215,
"margin_dpo/margin_std": 30.31496810913086,
"step": 108
},
{
"KL/chosen_KL_mean": -8.415664672851562,
"KL/mean": -18.587873458862305,
"KL/rejected_KL_mean": -28.760072708129883,
"KL/std": 25.343238830566406,
"epoch": 0.22827225130890053,
"fcm_dpo/beta": 0.010363556444644928,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.344406127929688,
"fcm_dpo/q_t": 0.4492321312427521,
"grad_norm": 39.30132293701172,
"learning_rate": 4.7625351138769166e-07,
"logits/chosen": -0.794326901435852,
"logits/rejected": -0.7915245294570923,
"logps/chosen": -334.3404541015625,
"logps/ref_chosen": -325.9248046875,
"logps/ref_rejected": -279.15423583984375,
"logps/rejected": -307.914306640625,
"loss": 4.8574,
"margin_dpo/margin_mean": 20.344402313232422,
"margin_dpo/margin_std": 31.736478805541992,
"step": 109
},
{
"KL/chosen_KL_mean": -8.976770401000977,
"KL/mean": -18.56324005126953,
"KL/rejected_KL_mean": -28.149715423583984,
"KL/std": 27.376617431640625,
"epoch": 0.23036649214659685,
"fcm_dpo/beta": 0.010363556444644928,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.172945022583008,
"fcm_dpo/q_t": 0.4520251750946045,
"grad_norm": 35.762718200683594,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": -0.7997367978096008,
"logits/rejected": -0.7880566716194153,
"logps/chosen": -283.4159851074219,
"logps/ref_chosen": -274.439208984375,
"logps/ref_rejected": -260.0552062988281,
"logps/rejected": -288.2049255371094,
"loss": 4.9146,
"margin_dpo/margin_mean": 19.172948837280273,
"margin_dpo/margin_std": 33.746768951416016,
"step": 110
},
{
"KL/chosen_KL_mean": -9.660625457763672,
"KL/mean": -19.830608367919922,
"KL/rejected_KL_mean": -30.00059700012207,
"KL/std": 28.046205520629883,
"epoch": 0.2324607329842932,
"fcm_dpo/beta": 0.010417782701551914,
"fcm_dpo/delta": 0.017871737480163574,
"fcm_dpo/margin": 20.33997344970703,
"fcm_dpo/q_t": 0.44924646615982056,
"grad_norm": 39.768409729003906,
"learning_rate": 4.7467175306295647e-07,
"logits/chosen": -0.8416504859924316,
"logits/rejected": -0.8213886022567749,
"logps/chosen": -338.8967590332031,
"logps/ref_chosen": -329.2361755371094,
"logps/ref_rejected": -287.82830810546875,
"logps/rejected": -317.82891845703125,
"loss": 4.8784,
"margin_dpo/margin_mean": 20.3399715423584,
"margin_dpo/margin_std": 34.47478485107422,
"step": 111
},
{
"KL/chosen_KL_mean": -15.551521301269531,
"KL/mean": -22.11650848388672,
"KL/rejected_KL_mean": -28.68149185180664,
"KL/std": 27.066593170166016,
"epoch": 0.23455497382198953,
"fcm_dpo/beta": 0.010552349500358105,
"fcm_dpo/delta": 0.006255615968257189,
"fcm_dpo/margin": 13.12997055053711,
"fcm_dpo/q_t": 0.46686533093452454,
"grad_norm": 41.6264533996582,
"learning_rate": 4.7386277983585053e-07,
"logits/chosen": -0.7422491312026978,
"logits/rejected": -0.7724015712738037,
"logps/chosen": -272.61090087890625,
"logps/ref_chosen": -257.0593566894531,
"logps/ref_rejected": -272.9595031738281,
"logps/rejected": -301.6409912109375,
"loss": 5.176,
"margin_dpo/margin_mean": 13.129972457885742,
"margin_dpo/margin_std": 36.441139221191406,
"step": 112
},
{
"KL/chosen_KL_mean": -13.025020599365234,
"KL/mean": -25.310611724853516,
"KL/rejected_KL_mean": -37.59620666503906,
"KL/std": 31.059711456298828,
"epoch": 0.23664921465968586,
"fcm_dpo/beta": 0.010606877505779266,
"fcm_dpo/delta": 0.009444285184144974,
"fcm_dpo/margin": 24.57118034362793,
"fcm_dpo/q_t": 0.4386439323425293,
"grad_norm": 43.13325500488281,
"learning_rate": 4.7304180152725024e-07,
"logits/chosen": -0.8121810555458069,
"logits/rejected": -0.8162283301353455,
"logps/chosen": -299.06671142578125,
"logps/ref_chosen": -286.0416564941406,
"logps/ref_rejected": -270.374267578125,
"logps/rejected": -307.9704895019531,
"loss": 4.7489,
"margin_dpo/margin_mean": 24.571176528930664,
"margin_dpo/margin_std": 39.310142517089844,
"step": 113
},
{
"KL/chosen_KL_mean": -16.87484359741211,
"KL/mean": -23.812885284423828,
"KL/rejected_KL_mean": -30.750930786132812,
"KL/std": 28.061574935913086,
"epoch": 0.2387434554973822,
"fcm_dpo/beta": 0.010645664297044277,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.876100540161133,
"fcm_dpo/q_t": 0.46417757868766785,
"grad_norm": 42.24106979370117,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": -0.8521884679794312,
"logits/rejected": -0.8534945249557495,
"logps/chosen": -276.88330078125,
"logps/ref_chosen": -260.0084533691406,
"logps/ref_rejected": -246.67190551757812,
"logps/rejected": -277.4228210449219,
"loss": 5.1292,
"margin_dpo/margin_mean": 13.876102447509766,
"margin_dpo/margin_std": 35.51622009277344,
"step": 114
},
{
"KL/chosen_KL_mean": -19.06077766418457,
"KL/mean": -26.02539825439453,
"KL/rejected_KL_mean": -32.990013122558594,
"KL/std": 30.866111755371094,
"epoch": 0.24083769633507854,
"fcm_dpo/beta": 0.01065509207546711,
"fcm_dpo/delta": 0.0014752836432307959,
"fcm_dpo/margin": 13.929231643676758,
"fcm_dpo/q_t": 0.4654514193534851,
"grad_norm": 46.440879821777344,
"learning_rate": 4.7136400641330245e-07,
"logits/chosen": -0.8538047671318054,
"logits/rejected": -0.8142991662025452,
"logps/chosen": -318.4837341308594,
"logps/ref_chosen": -299.4229736328125,
"logps/ref_rejected": -272.1186828613281,
"logps/rejected": -305.10870361328125,
"loss": 5.1532,
"margin_dpo/margin_mean": 13.929231643676758,
"margin_dpo/margin_std": 38.84375762939453,
"step": 115
},
{
"KL/chosen_KL_mean": -15.394144058227539,
"KL/mean": -24.405548095703125,
"KL/rejected_KL_mean": -33.416954040527344,
"KL/std": 28.890514373779297,
"epoch": 0.24293193717277486,
"fcm_dpo/beta": 0.010658235289156437,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.022811889648438,
"fcm_dpo/q_t": 0.4536454677581787,
"grad_norm": 41.08831024169922,
"learning_rate": 4.70507279583015e-07,
"logits/chosen": -0.8548110127449036,
"logits/rejected": -0.8159914016723633,
"logps/chosen": -294.6580810546875,
"logps/ref_chosen": -279.263916015625,
"logps/ref_rejected": -253.6192169189453,
"logps/rejected": -287.0361633300781,
"loss": 4.9585,
"margin_dpo/margin_mean": 18.02281379699707,
"margin_dpo/margin_std": 35.096229553222656,
"step": 116
},
{
"KL/chosen_KL_mean": -19.165630340576172,
"KL/mean": -29.62946128845215,
"KL/rejected_KL_mean": -40.093284606933594,
"KL/std": 27.6377010345459,
"epoch": 0.2450261780104712,
"fcm_dpo/beta": 0.010705700144171715,
"fcm_dpo/delta": 0.0005413433536887169,
"fcm_dpo/margin": 20.927654266357422,
"fcm_dpo/q_t": 0.446196049451828,
"grad_norm": 45.51142501831055,
"learning_rate": 4.6963872761652834e-07,
"logits/chosen": -0.8168615698814392,
"logits/rejected": -0.8167919516563416,
"logps/chosen": -278.3905029296875,
"logps/ref_chosen": -259.2248840332031,
"logps/ref_rejected": -229.3042755126953,
"logps/rejected": -269.3975524902344,
"loss": 4.8441,
"margin_dpo/margin_mean": 20.927654266357422,
"margin_dpo/margin_std": 31.943593978881836,
"step": 117
},
{
"KL/chosen_KL_mean": -25.37116813659668,
"KL/mean": -36.948516845703125,
"KL/rejected_KL_mean": -48.525875091552734,
"KL/std": 34.855682373046875,
"epoch": 0.24712041884816754,
"fcm_dpo/beta": 0.010717739351093769,
"fcm_dpo/delta": 0.009510215371847153,
"fcm_dpo/margin": 23.154699325561523,
"fcm_dpo/q_t": 0.44198188185691833,
"grad_norm": 48.185428619384766,
"learning_rate": 4.687583970916486e-07,
"logits/chosen": -0.8087795972824097,
"logits/rejected": -0.7963286638259888,
"logps/chosen": -292.44183349609375,
"logps/ref_chosen": -267.0707092285156,
"logps/ref_rejected": -272.7322082519531,
"logps/rejected": -321.258056640625,
"loss": 4.8341,
"margin_dpo/margin_mean": 23.154701232910156,
"margin_dpo/margin_std": 43.30189895629883,
"step": 118
},
{
"KL/chosen_KL_mean": -30.972999572753906,
"KL/mean": -39.472694396972656,
"KL/rejected_KL_mean": -47.972389221191406,
"KL/std": 34.111595153808594,
"epoch": 0.24921465968586387,
"fcm_dpo/beta": 0.010796227492392063,
"fcm_dpo/delta": 0.012761089019477367,
"fcm_dpo/margin": 16.9993896484375,
"fcm_dpo/q_t": 0.457156240940094,
"grad_norm": 48.64939880371094,
"learning_rate": 4.6786633521783005e-07,
"logits/chosen": -0.8885324597358704,
"logits/rejected": -0.8882258534431458,
"logps/chosen": -355.649658203125,
"logps/ref_chosen": -324.6766357421875,
"logps/ref_rejected": -306.0322265625,
"logps/rejected": -354.004638671875,
"loss": 5.0637,
"margin_dpo/margin_mean": 16.999387741088867,
"margin_dpo/margin_std": 41.50002670288086,
"step": 119
},
{
"KL/chosen_KL_mean": -27.831636428833008,
"KL/mean": -37.551856994628906,
"KL/rejected_KL_mean": -47.27207565307617,
"KL/std": 33.026878356933594,
"epoch": 0.2513089005235602,
"fcm_dpo/beta": 0.010917214676737785,
"fcm_dpo/delta": 0.007896372117102146,
"fcm_dpo/margin": 19.44042205810547,
"fcm_dpo/q_t": 0.450439453125,
"grad_norm": 45.14885711669922,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": -0.8366523385047913,
"logits/rejected": -0.8472779393196106,
"logps/chosen": -343.0933837890625,
"logps/ref_chosen": -315.2617492675781,
"logps/ref_rejected": -265.32501220703125,
"logps/rejected": -312.5970764160156,
"loss": 4.9651,
"margin_dpo/margin_mean": 19.440420150756836,
"margin_dpo/margin_std": 41.05113983154297,
"step": 120
},
{
"KL/chosen_KL_mean": -32.51570510864258,
"KL/mean": -39.51547622680664,
"KL/rejected_KL_mean": -46.51525115966797,
"KL/std": 31.86375617980957,
"epoch": 0.2534031413612565,
"fcm_dpo/beta": 0.01092331949621439,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.999553680419922,
"fcm_dpo/q_t": 0.4645032286643982,
"grad_norm": 62.9486083984375,
"learning_rate": 4.6604720940421207e-07,
"logits/chosen": -0.8580417633056641,
"logits/rejected": -0.8725881576538086,
"logps/chosen": -255.5117950439453,
"logps/ref_chosen": -222.99609375,
"logps/ref_rejected": -226.92860412597656,
"logps/rejected": -273.44384765625,
"loss": 5.156,
"margin_dpo/margin_mean": 13.999553680419922,
"margin_dpo/margin_std": 40.4022102355957,
"step": 121
},
{
"KL/chosen_KL_mean": -33.064552307128906,
"KL/mean": -42.89902114868164,
"KL/rejected_KL_mean": -52.733489990234375,
"KL/std": 37.320213317871094,
"epoch": 0.2554973821989529,
"fcm_dpo/beta": 0.011041234247386456,
"fcm_dpo/delta": 0.0161091610789299,
"fcm_dpo/margin": 19.668941497802734,
"fcm_dpo/q_t": 0.4495151937007904,
"grad_norm": 60.40846252441406,
"learning_rate": 4.651202430186092e-07,
"logits/chosen": -0.90716952085495,
"logits/rejected": -0.8687220811843872,
"logps/chosen": -309.09088134765625,
"logps/ref_chosen": -276.02630615234375,
"logps/ref_rejected": -277.97418212890625,
"logps/rejected": -330.7076721191406,
"loss": 4.9921,
"margin_dpo/margin_mean": 19.6689453125,
"margin_dpo/margin_std": 46.50227355957031,
"step": 122
},
{
"KL/chosen_KL_mean": -25.650867462158203,
"KL/mean": -38.428531646728516,
"KL/rejected_KL_mean": -51.206214904785156,
"KL/std": 36.957576751708984,
"epoch": 0.25759162303664923,
"fcm_dpo/beta": 0.011142628267407417,
"fcm_dpo/delta": 0.023120440542697906,
"fcm_dpo/margin": 25.55533218383789,
"fcm_dpo/q_t": 0.43309932947158813,
"grad_norm": 53.674034118652344,
"learning_rate": 4.6418174038722924e-07,
"logits/chosen": -0.8390490412712097,
"logits/rejected": -0.8307358026504517,
"logps/chosen": -353.80548095703125,
"logps/ref_chosen": -328.1546325683594,
"logps/ref_rejected": -280.6911315917969,
"logps/rejected": -331.8973693847656,
"loss": 4.7193,
"margin_dpo/margin_mean": 25.55533218383789,
"margin_dpo/margin_std": 42.476070404052734,
"step": 123
},
{
"KL/chosen_KL_mean": -26.246152877807617,
"KL/mean": -37.12230682373047,
"KL/rejected_KL_mean": -47.998477935791016,
"KL/std": 35.47646713256836,
"epoch": 0.25968586387434556,
"fcm_dpo/beta": 0.01128486730158329,
"fcm_dpo/delta": 0.002518674358725548,
"fcm_dpo/margin": 21.752336502075195,
"fcm_dpo/q_t": 0.44173774123191833,
"grad_norm": 57.442718505859375,
"learning_rate": 4.6323175183912023e-07,
"logits/chosen": -0.8511196374893188,
"logits/rejected": -0.8153136372566223,
"logps/chosen": -301.9423828125,
"logps/ref_chosen": -275.6961975097656,
"logps/ref_rejected": -225.361572265625,
"logps/rejected": -273.36004638671875,
"loss": 4.8391,
"margin_dpo/margin_mean": 21.752336502075195,
"margin_dpo/margin_std": 39.0461540222168,
"step": 124
},
{
"KL/chosen_KL_mean": -27.938440322875977,
"KL/mean": -37.887481689453125,
"KL/rejected_KL_mean": -47.83651351928711,
"KL/std": 36.877479553222656,
"epoch": 0.2617801047120419,
"fcm_dpo/beta": 0.01136357244104147,
"fcm_dpo/delta": 0.019005879759788513,
"fcm_dpo/margin": 19.8980712890625,
"fcm_dpo/q_t": 0.4486285448074341,
"grad_norm": 57.97393798828125,
"learning_rate": 4.6227032831928483e-07,
"logits/chosen": -0.8193236589431763,
"logits/rejected": -0.7735443711280823,
"logps/chosen": -306.0082092285156,
"logps/ref_chosen": -278.06976318359375,
"logps/ref_rejected": -265.63873291015625,
"logps/rejected": -313.4752197265625,
"loss": 5.0226,
"margin_dpo/margin_mean": 19.8980712890625,
"margin_dpo/margin_std": 50.3729362487793,
"step": 125
},
{
"KL/chosen_KL_mean": -23.977537155151367,
"KL/mean": -35.20510482788086,
"KL/rejected_KL_mean": -46.43267822265625,
"KL/std": 38.37907028198242,
"epoch": 0.2638743455497382,
"fcm_dpo/beta": 0.011555435135960579,
"fcm_dpo/delta": 0.015333190560340881,
"fcm_dpo/margin": 22.455135345458984,
"fcm_dpo/q_t": 0.4395897090435028,
"grad_norm": 56.987953186035156,
"learning_rate": 4.612975213859487e-07,
"logits/chosen": -0.8197309970855713,
"logits/rejected": -0.8281855583190918,
"logps/chosen": -345.3736267089844,
"logps/ref_chosen": -321.3960876464844,
"logps/ref_rejected": -285.37664794921875,
"logps/rejected": -331.8092956542969,
"loss": 4.8436,
"margin_dpo/margin_mean": 22.455137252807617,
"margin_dpo/margin_std": 44.75693893432617,
"step": 126
},
{
"KL/chosen_KL_mean": -22.364826202392578,
"KL/mean": -35.611881256103516,
"KL/rejected_KL_mean": -48.85894775390625,
"KL/std": 39.49049758911133,
"epoch": 0.26596858638743454,
"fcm_dpo/beta": 0.011540120467543602,
"fcm_dpo/delta": -0.02305636554956436,
"fcm_dpo/margin": 26.49411964416504,
"fcm_dpo/q_t": 0.4295908808708191,
"grad_norm": 56.03866195678711,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": -0.9088628888130188,
"logits/rejected": -0.858267605304718,
"logps/chosen": -328.923583984375,
"logps/ref_chosen": -306.55877685546875,
"logps/ref_rejected": -274.8651428222656,
"logps/rejected": -323.72412109375,
"loss": 4.7184,
"margin_dpo/margin_mean": 26.494121551513672,
"margin_dpo/margin_std": 45.3493766784668,
"step": 127
},
{
"KL/chosen_KL_mean": -17.152809143066406,
"KL/mean": -32.300350189208984,
"KL/rejected_KL_mean": -47.44788360595703,
"KL/std": 41.129150390625,
"epoch": 0.2680628272251309,
"fcm_dpo/beta": 0.011374343186616898,
"fcm_dpo/delta": 0.0034487280063331127,
"fcm_dpo/margin": 30.295085906982422,
"fcm_dpo/q_t": 0.4207966923713684,
"grad_norm": 53.232566833496094,
"learning_rate": 4.5931796656116837e-07,
"logits/chosen": -0.7918947339057922,
"logits/rejected": -0.7883634567260742,
"logps/chosen": -282.5501708984375,
"logps/ref_chosen": -265.3973693847656,
"logps/ref_rejected": -250.9737548828125,
"logps/rejected": -298.421630859375,
"loss": 4.5958,
"margin_dpo/margin_mean": 30.29508399963379,
"margin_dpo/margin_std": 49.26608657836914,
"step": 128
},
{
"KL/chosen_KL_mean": -18.030426025390625,
"KL/mean": -31.652067184448242,
"KL/rejected_KL_mean": -45.27369689941406,
"KL/std": 38.48607635498047,
"epoch": 0.27015706806282724,
"fcm_dpo/beta": 0.011414232663810253,
"fcm_dpo/delta": -0.015051654540002346,
"fcm_dpo/margin": 27.243267059326172,
"fcm_dpo/q_t": 0.4287635087966919,
"grad_norm": 53.86652374267578,
"learning_rate": 4.5831132482724193e-07,
"logits/chosen": -0.8241338133811951,
"logits/rejected": -0.8225374221801758,
"logps/chosen": -321.1888427734375,
"logps/ref_chosen": -303.158447265625,
"logps/ref_rejected": -275.9891052246094,
"logps/rejected": -321.2628173828125,
"loss": 4.7053,
"margin_dpo/margin_mean": 27.243270874023438,
"margin_dpo/margin_std": 48.319923400878906,
"step": 129
},
{
"KL/chosen_KL_mean": -20.707996368408203,
"KL/mean": -34.25269317626953,
"KL/rejected_KL_mean": -47.7973747253418,
"KL/std": 38.006534576416016,
"epoch": 0.27225130890052357,
"fcm_dpo/beta": 0.011222862638533115,
"fcm_dpo/delta": -4.155188798904419e-05,
"fcm_dpo/margin": 27.08936309814453,
"fcm_dpo/q_t": 0.43000417947769165,
"grad_norm": 65.6941146850586,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": -0.8021270036697388,
"logits/rejected": -0.8400847911834717,
"logps/chosen": -307.1153259277344,
"logps/ref_chosen": -286.4073486328125,
"logps/ref_rejected": -294.38665771484375,
"logps/rejected": -342.1840515136719,
"loss": 4.7133,
"margin_dpo/margin_mean": 27.089365005493164,
"margin_dpo/margin_std": 48.50556564331055,
"step": 130
},
{
"KL/chosen_KL_mean": -25.196441650390625,
"KL/mean": -36.10236358642578,
"KL/rejected_KL_mean": -47.00827407836914,
"KL/std": 42.682491302490234,
"epoch": 0.2743455497382199,
"fcm_dpo/beta": 0.01131611317396164,
"fcm_dpo/delta": 0.008455941453576088,
"fcm_dpo/margin": 21.811826705932617,
"fcm_dpo/q_t": 0.44346877932548523,
"grad_norm": 65.5090103149414,
"learning_rate": 4.5626458262912735e-07,
"logits/chosen": -0.8783740997314453,
"logits/rejected": -0.8263896703720093,
"logps/chosen": -336.7615051269531,
"logps/ref_chosen": -311.5650634765625,
"logps/ref_rejected": -291.62432861328125,
"logps/rejected": -338.63262939453125,
"loss": 4.971,
"margin_dpo/margin_mean": 21.811826705932617,
"margin_dpo/margin_std": 51.76227951049805,
"step": 131
},
{
"KL/chosen_KL_mean": -21.0625,
"KL/mean": -36.35379409790039,
"KL/rejected_KL_mean": -51.645084381103516,
"KL/std": 42.20893478393555,
"epoch": 0.2764397905759162,
"fcm_dpo/beta": 0.011329811997711658,
"fcm_dpo/delta": -0.031071798875927925,
"fcm_dpo/margin": 30.582592010498047,
"fcm_dpo/q_t": 0.42160820960998535,
"grad_norm": 69.17401885986328,
"learning_rate": 4.5522459192551166e-07,
"logits/chosen": -0.821100115776062,
"logits/rejected": -0.8047419190406799,
"logps/chosen": -291.14434814453125,
"logps/ref_chosen": -270.0818176269531,
"logps/ref_rejected": -284.3084411621094,
"logps/rejected": -335.95355224609375,
"loss": 4.6525,
"margin_dpo/margin_mean": 30.582592010498047,
"margin_dpo/margin_std": 53.2596321105957,
"step": 132
},
{
"KL/chosen_KL_mean": -24.293067932128906,
"KL/mean": -38.521732330322266,
"KL/rejected_KL_mean": -52.75040054321289,
"KL/std": 39.787479400634766,
"epoch": 0.27853403141361255,
"fcm_dpo/beta": 0.011118058115243912,
"fcm_dpo/delta": -0.0034186802804470062,
"fcm_dpo/margin": 28.45734405517578,
"fcm_dpo/q_t": 0.42733392119407654,
"grad_norm": 47.61275863647461,
"learning_rate": 4.541735956498554e-07,
"logits/chosen": -0.8711456656455994,
"logits/rejected": -0.8716740012168884,
"logps/chosen": -309.9144287109375,
"logps/ref_chosen": -285.6213684082031,
"logps/ref_rejected": -251.19386291503906,
"logps/rejected": -303.9442443847656,
"loss": 4.687,
"margin_dpo/margin_mean": 28.45734405517578,
"margin_dpo/margin_std": 49.6102409362793,
"step": 133
},
{
"KL/chosen_KL_mean": -36.26886749267578,
"KL/mean": -46.98722457885742,
"KL/rejected_KL_mean": -57.70557403564453,
"KL/std": 41.17924499511719,
"epoch": 0.2806282722513089,
"fcm_dpo/beta": 0.010998277924954891,
"fcm_dpo/delta": -0.009528097696602345,
"fcm_dpo/margin": 21.43670654296875,
"fcm_dpo/q_t": 0.44455987215042114,
"grad_norm": 54.783790588378906,
"learning_rate": 4.5311165016389914e-07,
"logits/chosen": -0.8688364028930664,
"logits/rejected": -0.8680624961853027,
"logps/chosen": -355.1896667480469,
"logps/ref_chosen": -318.92083740234375,
"logps/ref_rejected": -293.1894836425781,
"logps/rejected": -350.8950500488281,
"loss": 4.9126,
"margin_dpo/margin_mean": 21.43670654296875,
"margin_dpo/margin_std": 44.431854248046875,
"step": 134
},
{
"KL/chosen_KL_mean": -33.41929244995117,
"KL/mean": -45.94215393066406,
"KL/rejected_KL_mean": -58.465023040771484,
"KL/std": 43.048866271972656,
"epoch": 0.28272251308900526,
"fcm_dpo/beta": 0.011012822389602661,
"fcm_dpo/delta": -0.011346682906150818,
"fcm_dpo/margin": 25.045719146728516,
"fcm_dpo/q_t": 0.43511971831321716,
"grad_norm": 64.9057388305664,
"learning_rate": 4.520388124165564e-07,
"logits/chosen": -0.7530911564826965,
"logits/rejected": -0.7919206619262695,
"logps/chosen": -326.24102783203125,
"logps/ref_chosen": -292.8217468261719,
"logps/ref_rejected": -269.2896728515625,
"logps/rejected": -327.7547302246094,
"loss": 4.8295,
"margin_dpo/margin_mean": 25.04572296142578,
"margin_dpo/margin_std": 48.300167083740234,
"step": 135
},
{
"KL/chosen_KL_mean": -35.22936248779297,
"KL/mean": -49.50828552246094,
"KL/rejected_KL_mean": -63.787208557128906,
"KL/std": 43.139225006103516,
"epoch": 0.2848167539267016,
"fcm_dpo/beta": 0.010835347697138786,
"fcm_dpo/delta": -0.016382994130253792,
"fcm_dpo/margin": 28.557849884033203,
"fcm_dpo/q_t": 0.42960232496261597,
"grad_norm": 70.99027252197266,
"learning_rate": 4.5095513994085974e-07,
"logits/chosen": -0.8074231147766113,
"logits/rejected": -0.8016695976257324,
"logps/chosen": -308.0819091796875,
"logps/ref_chosen": -272.8525390625,
"logps/ref_rejected": -252.68202209472656,
"logps/rejected": -316.4692077636719,
"loss": 4.752,
"margin_dpo/margin_mean": 28.557851791381836,
"margin_dpo/margin_std": 53.87636947631836,
"step": 136
},
{
"KL/chosen_KL_mean": -39.36355972290039,
"KL/mean": -51.80638122558594,
"KL/rejected_KL_mean": -64.24920654296875,
"KL/std": 45.04750442504883,
"epoch": 0.2869109947643979,
"fcm_dpo/beta": 0.010752377100288868,
"fcm_dpo/delta": 0.006089036352932453,
"fcm_dpo/margin": 24.88566780090332,
"fcm_dpo/q_t": 0.43942371010780334,
"grad_norm": 61.39313507080078,
"learning_rate": 4.498606908508753e-07,
"logits/chosen": -0.8615954518318176,
"logits/rejected": -0.8479353189468384,
"logps/chosen": -340.11578369140625,
"logps/ref_chosen": -300.7522277832031,
"logps/ref_rejected": -286.1935119628906,
"logps/rejected": -350.4427185058594,
"loss": 4.8589,
"margin_dpo/margin_mean": 24.88566780090332,
"margin_dpo/margin_std": 51.68086624145508,
"step": 137
},
{
"KL/chosen_KL_mean": -36.756832122802734,
"KL/mean": -51.21699523925781,
"KL/rejected_KL_mean": -65.67715454101562,
"KL/std": 46.030006408691406,
"epoch": 0.28900523560209423,
"fcm_dpo/beta": 0.010876420885324478,
"fcm_dpo/delta": 0.0157970879226923,
"fcm_dpo/margin": 28.920316696166992,
"fcm_dpo/q_t": 0.4301770329475403,
"grad_norm": 61.19403076171875,
"learning_rate": 4.487555238385862e-07,
"logits/chosen": -0.7898247241973877,
"logits/rejected": -0.7696442604064941,
"logps/chosen": -325.6938171386719,
"logps/ref_chosen": -288.9369812011719,
"logps/ref_rejected": -263.7076416015625,
"logps/rejected": -329.3847961425781,
"loss": 4.8013,
"margin_dpo/margin_mean": 28.920316696166992,
"margin_dpo/margin_std": 60.27662658691406,
"step": 138
},
{
"KL/chosen_KL_mean": -33.69660186767578,
"KL/mean": -43.981441497802734,
"KL/rejected_KL_mean": -54.26628494262695,
"KL/std": 41.30760955810547,
"epoch": 0.29109947643979056,
"fcm_dpo/beta": 0.010893644765019417,
"fcm_dpo/delta": -0.007741590961813927,
"fcm_dpo/margin": 20.569684982299805,
"fcm_dpo/q_t": 0.4489022493362427,
"grad_norm": 62.34196472167969,
"learning_rate": 4.476396981707453e-07,
"logits/chosen": -0.8070241808891296,
"logits/rejected": -0.8368173241615295,
"logps/chosen": -303.74090576171875,
"logps/ref_chosen": -270.0443115234375,
"logps/ref_rejected": -267.3226013183594,
"logps/rejected": -321.5888977050781,
"loss": 5.0213,
"margin_dpo/margin_mean": 20.569684982299805,
"margin_dpo/margin_std": 50.81401824951172,
"step": 139
},
{
"KL/chosen_KL_mean": -31.74407386779785,
"KL/mean": -47.28278350830078,
"KL/rejected_KL_mean": -62.821502685546875,
"KL/std": 43.96257019042969,
"epoch": 0.2931937172774869,
"fcm_dpo/beta": 0.010792272165417671,
"fcm_dpo/delta": -0.03338925540447235,
"fcm_dpo/margin": 31.077434539794922,
"fcm_dpo/q_t": 0.422089159488678,
"grad_norm": 58.86613464355469,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": -0.883962869644165,
"logits/rejected": -0.8495863676071167,
"logps/chosen": -314.69964599609375,
"logps/ref_chosen": -282.9555969238281,
"logps/ref_rejected": -251.17181396484375,
"logps/rejected": -313.9932861328125,
"loss": 4.5964,
"margin_dpo/margin_mean": 31.077436447143555,
"margin_dpo/margin_std": 47.86009216308594,
"step": 140
},
{
"KL/chosen_KL_mean": -30.595308303833008,
"KL/mean": -46.62080383300781,
"KL/rejected_KL_mean": -62.64628982543945,
"KL/std": 45.07204818725586,
"epoch": 0.29528795811518327,
"fcm_dpo/beta": 0.010443806648254395,
"fcm_dpo/delta": -0.02163812890648842,
"fcm_dpo/margin": 32.050987243652344,
"fcm_dpo/q_t": 0.4220162034034729,
"grad_norm": 65.77468872070312,
"learning_rate": 4.453763107901675e-07,
"logits/chosen": -0.8113507628440857,
"logits/rejected": -0.8057313561439514,
"logps/chosen": -326.89544677734375,
"logps/ref_chosen": -296.3001708984375,
"logps/ref_rejected": -279.8486633300781,
"logps/rejected": -342.4949645996094,
"loss": 4.6623,
"margin_dpo/margin_mean": 32.05098342895508,
"margin_dpo/margin_std": 56.02928161621094,
"step": 141
},
{
"KL/chosen_KL_mean": -31.103105545043945,
"KL/mean": -44.4693603515625,
"KL/rejected_KL_mean": -57.835609436035156,
"KL/std": 45.70125198364258,
"epoch": 0.2973821989528796,
"fcm_dpo/beta": 0.010357570834457874,
"fcm_dpo/delta": -0.00366947241127491,
"fcm_dpo/margin": 26.73249626159668,
"fcm_dpo/q_t": 0.43658334016799927,
"grad_norm": 60.299896240234375,
"learning_rate": 4.4422887045602674e-07,
"logits/chosen": -0.8335211277008057,
"logits/rejected": -0.8346509337425232,
"logps/chosen": -331.6689453125,
"logps/ref_chosen": -300.56585693359375,
"logps/ref_rejected": -231.43316650390625,
"logps/rejected": -289.2687683105469,
"loss": 4.8729,
"margin_dpo/margin_mean": 26.73249626159668,
"margin_dpo/margin_std": 57.07372283935547,
"step": 142
},
{
"KL/chosen_KL_mean": -31.008270263671875,
"KL/mean": -47.164031982421875,
"KL/rejected_KL_mean": -63.31980895996094,
"KL/std": 43.575416564941406,
"epoch": 0.2994764397905759,
"fcm_dpo/beta": 0.010205346159636974,
"fcm_dpo/delta": -0.026661261916160583,
"fcm_dpo/margin": 32.3115348815918,
"fcm_dpo/q_t": 0.4239623248577118,
"grad_norm": 55.74364471435547,
"learning_rate": 4.4307101421701755e-07,
"logits/chosen": -0.8169223666191101,
"logits/rejected": -0.794075071811676,
"logps/chosen": -327.7406005859375,
"logps/ref_chosen": -296.73236083984375,
"logps/ref_rejected": -266.45257568359375,
"logps/rejected": -329.7724304199219,
"loss": 4.6115,
"margin_dpo/margin_mean": 32.3115348815918,
"margin_dpo/margin_std": 51.006839752197266,
"step": 143
},
{
"KL/chosen_KL_mean": -31.154144287109375,
"KL/mean": -44.3969612121582,
"KL/rejected_KL_mean": -57.63978958129883,
"KL/std": 46.97825241088867,
"epoch": 0.30157068062827225,
"fcm_dpo/beta": 0.00994843989610672,
"fcm_dpo/delta": -0.016245760023593903,
"fcm_dpo/margin": 26.485647201538086,
"fcm_dpo/q_t": 0.44019222259521484,
"grad_norm": 52.55347442626953,
"learning_rate": 4.419028041654559e-07,
"logits/chosen": -0.8847938179969788,
"logits/rejected": -0.8704959750175476,
"logps/chosen": -329.9981689453125,
"logps/ref_chosen": -298.843994140625,
"logps/ref_rejected": -266.120849609375,
"logps/rejected": -323.7606201171875,
"loss": 4.8825,
"margin_dpo/margin_mean": 26.48564910888672,
"margin_dpo/margin_std": 56.594215393066406,
"step": 144
},
{
"KL/chosen_KL_mean": -27.540496826171875,
"KL/mean": -44.87952423095703,
"KL/rejected_KL_mean": -62.218544006347656,
"KL/std": 43.72524642944336,
"epoch": 0.3036649214659686,
"fcm_dpo/beta": 0.009765357710421085,
"fcm_dpo/delta": -0.04112963750958443,
"fcm_dpo/margin": 34.678043365478516,
"fcm_dpo/q_t": 0.42156580090522766,
"grad_norm": 56.111358642578125,
"learning_rate": 4.4072430294890166e-07,
"logits/chosen": -0.8678039312362671,
"logits/rejected": -0.8692808747291565,
"logps/chosen": -303.2933654785156,
"logps/ref_chosen": -275.7528381347656,
"logps/ref_rejected": -214.74807739257812,
"logps/rejected": -276.96661376953125,
"loss": 4.591,
"margin_dpo/margin_mean": 34.678043365478516,
"margin_dpo/margin_std": 53.235076904296875,
"step": 145
},
{
"KL/chosen_KL_mean": -36.69147491455078,
"KL/mean": -53.71113586425781,
"KL/rejected_KL_mean": -70.73080444335938,
"KL/std": 48.02214813232422,
"epoch": 0.3057591623036649,
"fcm_dpo/beta": 0.009542811661958694,
"fcm_dpo/delta": -0.024183155968785286,
"fcm_dpo/margin": 34.03933334350586,
"fcm_dpo/q_t": 0.4251405894756317,
"grad_norm": 62.798160552978516,
"learning_rate": 4.395355737667985e-07,
"logits/chosen": -0.8429842591285706,
"logits/rejected": -0.8354379534721375,
"logps/chosen": -313.7897033691406,
"logps/ref_chosen": -277.09820556640625,
"logps/ref_rejected": -265.41046142578125,
"logps/rejected": -336.1412658691406,
"loss": 4.6317,
"margin_dpo/margin_mean": 34.03933334350586,
"margin_dpo/margin_std": 53.72346496582031,
"step": 146
},
{
"KL/chosen_KL_mean": -38.27781677246094,
"KL/mean": -51.62920379638672,
"KL/rejected_KL_mean": -64.98057556152344,
"KL/std": 49.75927734375,
"epoch": 0.3078534031413613,
"fcm_dpo/beta": 0.009431717917323112,
"fcm_dpo/delta": -0.03413345292210579,
"fcm_dpo/margin": 26.7027530670166,
"fcm_dpo/q_t": 0.44130846858024597,
"grad_norm": 56.538326263427734,
"learning_rate": 4.3833668036708483e-07,
"logits/chosen": -0.8454154133796692,
"logits/rejected": -0.8405569195747375,
"logps/chosen": -329.6964111328125,
"logps/ref_chosen": -291.4185791015625,
"logps/ref_rejected": -253.43051147460938,
"logps/rejected": -318.41107177734375,
"loss": 4.9113,
"margin_dpo/margin_mean": 26.7027530670166,
"margin_dpo/margin_std": 55.443599700927734,
"step": 147
},
{
"KL/chosen_KL_mean": -37.951026916503906,
"KL/mean": -52.09737777709961,
"KL/rejected_KL_mean": -66.24372863769531,
"KL/std": 52.47624206542969,
"epoch": 0.3099476439790576,
"fcm_dpo/beta": 0.009357105009257793,
"fcm_dpo/delta": 0.03023623488843441,
"fcm_dpo/margin": 28.292701721191406,
"fcm_dpo/q_t": 0.44127410650253296,
"grad_norm": 59.94662094116211,
"learning_rate": 4.3712768704277524e-07,
"logits/chosen": -0.9010493755340576,
"logits/rejected": -0.9029269218444824,
"logps/chosen": -274.69952392578125,
"logps/ref_chosen": -236.74850463867188,
"logps/ref_rejected": -231.4674072265625,
"logps/rejected": -297.7110900878906,
"loss": 4.9011,
"margin_dpo/margin_mean": 28.292701721191406,
"margin_dpo/margin_std": 63.87792205810547,
"step": 148
},
{
"KL/chosen_KL_mean": -42.97991180419922,
"KL/mean": -62.22007369995117,
"KL/rejected_KL_mean": -81.46024322509766,
"KL/std": 48.12861633300781,
"epoch": 0.31204188481675393,
"fcm_dpo/beta": 0.00953738298267126,
"fcm_dpo/delta": 0.0038804998621344566,
"fcm_dpo/margin": 38.4803352355957,
"fcm_dpo/q_t": 0.41513967514038086,
"grad_norm": 60.234527587890625,
"learning_rate": 4.3590865862851263e-07,
"logits/chosen": -0.8748192191123962,
"logits/rejected": -0.8590805530548096,
"logps/chosen": -362.9084167480469,
"logps/ref_chosen": -319.9284973144531,
"logps/ref_rejected": -308.20233154296875,
"logps/rejected": -389.66259765625,
"loss": 4.4618,
"margin_dpo/margin_mean": 38.4803352355957,
"margin_dpo/margin_std": 53.24681854248047,
"step": 149
},
{
"KL/chosen_KL_mean": -50.06086349487305,
"KL/mean": -64.94522094726562,
"KL/rejected_KL_mean": -79.8295669555664,
"KL/std": 50.34340286254883,
"epoch": 0.31413612565445026,
"fcm_dpo/beta": 0.009615819901227951,
"fcm_dpo/delta": 0.017674576491117477,
"fcm_dpo/margin": 29.768699645996094,
"fcm_dpo/q_t": 0.43368738889694214,
"grad_norm": 67.94093322753906,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": -0.839390754699707,
"logits/rejected": -0.822877824306488,
"logps/chosen": -326.37908935546875,
"logps/ref_chosen": -276.3182373046875,
"logps/ref_rejected": -273.02215576171875,
"logps/rejected": -352.8516845703125,
"loss": 4.7679,
"margin_dpo/margin_mean": 29.768701553344727,
"margin_dpo/margin_std": 55.33635711669922,
"step": 150
},
{
"KL/chosen_KL_mean": -46.528472900390625,
"KL/mean": -68.42727661132812,
"KL/rejected_KL_mean": -90.32608032226562,
"KL/std": 51.12382507324219,
"epoch": 0.3162303664921466,
"fcm_dpo/beta": 0.009591327980160713,
"fcm_dpo/delta": -0.02124784141778946,
"fcm_dpo/margin": 43.797607421875,
"fcm_dpo/q_t": 0.40414297580718994,
"grad_norm": 72.9853744506836,
"learning_rate": 4.3344075855595097e-07,
"logits/chosen": -0.8729795217514038,
"logits/rejected": -0.8649865984916687,
"logps/chosen": -343.84130859375,
"logps/ref_chosen": -297.31280517578125,
"logps/ref_rejected": -266.1003723144531,
"logps/rejected": -356.4264831542969,
"loss": 4.3725,
"margin_dpo/margin_mean": 43.797607421875,
"margin_dpo/margin_std": 59.94606399536133,
"step": 151
},
{
"KL/chosen_KL_mean": -40.6783561706543,
"KL/mean": -59.743812561035156,
"KL/rejected_KL_mean": -78.80927276611328,
"KL/std": 54.26198959350586,
"epoch": 0.3183246073298429,
"fcm_dpo/beta": 0.009200290776789188,
"fcm_dpo/delta": -0.05215620622038841,
"fcm_dpo/margin": 38.13091278076172,
"fcm_dpo/q_t": 0.4210585653781891,
"grad_norm": 58.28529739379883,
"learning_rate": 4.3219201924364323e-07,
"logits/chosen": -0.87758469581604,
"logits/rejected": -0.8735326528549194,
"logps/chosen": -310.92535400390625,
"logps/ref_chosen": -270.2470397949219,
"logps/ref_rejected": -269.7749328613281,
"logps/rejected": -348.5841979980469,
"loss": 4.6203,
"margin_dpo/margin_mean": 38.130916595458984,
"margin_dpo/margin_std": 63.328189849853516,
"step": 152
},
{
"KL/chosen_KL_mean": -48.53391647338867,
"KL/mean": -75.49513244628906,
"KL/rejected_KL_mean": -102.45635986328125,
"KL/std": 58.56542205810547,
"epoch": 0.3204188481675393,
"fcm_dpo/beta": 0.008681675419211388,
"fcm_dpo/delta": -0.08731570094823837,
"fcm_dpo/margin": 53.92245864868164,
"fcm_dpo/q_t": 0.3927898108959198,
"grad_norm": 58.35908126831055,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": -0.8716917037963867,
"logits/rejected": -0.858845055103302,
"logps/chosen": -322.31298828125,
"logps/ref_chosen": -273.779052734375,
"logps/ref_rejected": -280.9530944824219,
"logps/rejected": -383.40948486328125,
"loss": 4.1636,
"margin_dpo/margin_mean": 53.92245864868164,
"margin_dpo/margin_std": 61.229759216308594,
"step": 153
},
{
"KL/chosen_KL_mean": -48.57218551635742,
"KL/mean": -69.53777313232422,
"KL/rejected_KL_mean": -90.50334930419922,
"KL/std": 60.53078842163086,
"epoch": 0.3225130890052356,
"fcm_dpo/beta": 0.008394555188715458,
"fcm_dpo/delta": -0.020893538370728493,
"fcm_dpo/margin": 41.9311637878418,
"fcm_dpo/q_t": 0.4201442301273346,
"grad_norm": 72.38925170898438,
"learning_rate": 4.2966529689388064e-07,
"logits/chosen": -0.9051564335823059,
"logits/rejected": -0.8851950764656067,
"logps/chosen": -338.4753723144531,
"logps/ref_chosen": -289.9031982421875,
"logps/ref_rejected": -261.5166320800781,
"logps/rejected": -352.02001953125,
"loss": 4.6266,
"margin_dpo/margin_mean": 41.9311637878418,
"margin_dpo/margin_std": 71.46162414550781,
"step": 154
},
{
"KL/chosen_KL_mean": -53.91712188720703,
"KL/mean": -72.27357482910156,
"KL/rejected_KL_mean": -90.63002014160156,
"KL/std": 57.95890808105469,
"epoch": 0.32460732984293195,
"fcm_dpo/beta": 0.008404220454394817,
"fcm_dpo/delta": 0.010009624063968658,
"fcm_dpo/margin": 36.712890625,
"fcm_dpo/q_t": 0.43113821744918823,
"grad_norm": 70.87203979492188,
"learning_rate": 4.2838744935687716e-07,
"logits/chosen": -0.8529869914054871,
"logits/rejected": -0.8497177958488464,
"logps/chosen": -339.7783203125,
"logps/ref_chosen": -285.8612060546875,
"logps/ref_rejected": -300.1272888183594,
"logps/rejected": -390.75732421875,
"loss": 4.7199,
"margin_dpo/margin_mean": 36.71288299560547,
"margin_dpo/margin_std": 66.47289276123047,
"step": 155
},
{
"KL/chosen_KL_mean": -49.25990676879883,
"KL/mean": -74.67170715332031,
"KL/rejected_KL_mean": -100.0835189819336,
"KL/std": 61.44602584838867,
"epoch": 0.3267015706806283,
"fcm_dpo/beta": 0.008251436054706573,
"fcm_dpo/delta": -0.07570717483758926,
"fcm_dpo/margin": 50.82361602783203,
"fcm_dpo/q_t": 0.405461847782135,
"grad_norm": 55.39389419555664,
"learning_rate": 4.271000354423425e-07,
"logits/chosen": -0.8789178133010864,
"logits/rejected": -0.8763436675071716,
"logps/chosen": -328.2953796386719,
"logps/ref_chosen": -279.0354919433594,
"logps/ref_rejected": -244.2198486328125,
"logps/rejected": -344.3033447265625,
"loss": 4.4554,
"margin_dpo/margin_mean": 50.82362365722656,
"margin_dpo/margin_std": 77.3794937133789,
"step": 156
},
{
"KL/chosen_KL_mean": -52.569740295410156,
"KL/mean": -72.73023986816406,
"KL/rejected_KL_mean": -92.89073181152344,
"KL/std": 61.27772521972656,
"epoch": 0.3287958115183246,
"fcm_dpo/beta": 0.007722177542746067,
"fcm_dpo/delta": -0.04277050867676735,
"fcm_dpo/margin": 40.32099533081055,
"fcm_dpo/q_t": 0.42746734619140625,
"grad_norm": 55.638614654541016,
"learning_rate": 4.258031241903777e-07,
"logits/chosen": -0.9371187686920166,
"logits/rejected": -0.9353018999099731,
"logps/chosen": -323.4000549316406,
"logps/ref_chosen": -270.830322265625,
"logps/ref_rejected": -259.08319091796875,
"logps/rejected": -351.9739685058594,
"loss": 4.6749,
"margin_dpo/margin_mean": 40.32099533081055,
"margin_dpo/margin_std": 66.63133239746094,
"step": 157
},
{
"KL/chosen_KL_mean": -51.89894485473633,
"KL/mean": -74.62893676757812,
"KL/rejected_KL_mean": -97.35890197753906,
"KL/std": 62.313785552978516,
"epoch": 0.3308900523560209,
"fcm_dpo/beta": 0.007630071137100458,
"fcm_dpo/delta": -0.01720864325761795,
"fcm_dpo/margin": 45.4599494934082,
"fcm_dpo/q_t": 0.41904595494270325,
"grad_norm": 56.36635208129883,
"learning_rate": 4.2449678515039743e-07,
"logits/chosen": -0.8542776107788086,
"logits/rejected": -0.8377709984779358,
"logps/chosen": -341.8652648925781,
"logps/ref_chosen": -289.9663391113281,
"logps/ref_rejected": -271.335693359375,
"logps/rejected": -368.694580078125,
"loss": 4.5601,
"margin_dpo/margin_mean": 45.4599494934082,
"margin_dpo/margin_std": 67.09923553466797,
"step": 158
},
{
"KL/chosen_KL_mean": -59.03363800048828,
"KL/mean": -72.34220123291016,
"KL/rejected_KL_mean": -85.65077209472656,
"KL/std": 57.92660140991211,
"epoch": 0.33298429319371725,
"fcm_dpo/beta": 0.007565178908407688,
"fcm_dpo/delta": 0.008062109351158142,
"fcm_dpo/margin": 26.61713409423828,
"fcm_dpo/q_t": 0.45483458042144775,
"grad_norm": 76.1928939819336,
"learning_rate": 4.2318108837739986e-07,
"logits/chosen": -0.947105348110199,
"logits/rejected": -0.8976367712020874,
"logps/chosen": -380.4119873046875,
"logps/ref_chosen": -321.37835693359375,
"logps/ref_rejected": -250.45652770996094,
"logps/rejected": -336.1072692871094,
"loss": 5.1068,
"margin_dpo/margin_mean": 26.61713409423828,
"margin_dpo/margin_std": 72.70268249511719,
"step": 159
},
{
"KL/chosen_KL_mean": -48.30318832397461,
"KL/mean": -73.82183837890625,
"KL/rejected_KL_mean": -99.34048461914062,
"KL/std": 61.44125747680664,
"epoch": 0.33507853403141363,
"fcm_dpo/beta": 0.007519013714045286,
"fcm_dpo/delta": -0.013213069178164005,
"fcm_dpo/margin": 51.03730773925781,
"fcm_dpo/q_t": 0.41031157970428467,
"grad_norm": 55.092437744140625,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": -0.8815222978591919,
"logits/rejected": -0.8827785849571228,
"logps/chosen": -324.586669921875,
"logps/ref_chosen": -276.28350830078125,
"logps/ref_rejected": -262.7477722167969,
"logps/rejected": -362.0882568359375,
"loss": 4.3609,
"margin_dpo/margin_mean": 51.03730773925781,
"margin_dpo/margin_std": 59.32886505126953,
"step": 160
},
{
"KL/chosen_KL_mean": -59.115379333496094,
"KL/mean": -81.7115707397461,
"KL/rejected_KL_mean": -104.30776977539062,
"KL/std": 64.58517456054688,
"epoch": 0.33717277486910996,
"fcm_dpo/beta": 0.007456418592482805,
"fcm_dpo/delta": -0.019674938172101974,
"fcm_dpo/margin": 45.19239044189453,
"fcm_dpo/q_t": 0.4239409565925598,
"grad_norm": 75.19351959228516,
"learning_rate": 4.2052190435769554e-07,
"logits/chosen": -0.9016933441162109,
"logits/rejected": -0.8840854167938232,
"logps/chosen": -369.6081848144531,
"logps/ref_chosen": -310.4927978515625,
"logps/ref_rejected": -250.25347900390625,
"logps/rejected": -354.5612487792969,
"loss": 4.6386,
"margin_dpo/margin_mean": 45.19239044189453,
"margin_dpo/margin_std": 76.1778564453125,
"step": 161
},
{
"KL/chosen_KL_mean": -54.95409393310547,
"KL/mean": -75.90670776367188,
"KL/rejected_KL_mean": -96.85931396484375,
"KL/std": 60.112361907958984,
"epoch": 0.3392670157068063,
"fcm_dpo/beta": 0.007265571039170027,
"fcm_dpo/delta": -0.0053212931379675865,
"fcm_dpo/margin": 41.90523147583008,
"fcm_dpo/q_t": 0.429195761680603,
"grad_norm": 51.2578125,
"learning_rate": 4.1917855971495763e-07,
"logits/chosen": -0.8682603240013123,
"logits/rejected": -0.8593798875808716,
"logps/chosen": -351.0645751953125,
"logps/ref_chosen": -296.1105041503906,
"logps/ref_rejected": -253.4247589111328,
"logps/rejected": -350.2840881347656,
"loss": 4.6797,
"margin_dpo/margin_mean": 41.90523147583008,
"margin_dpo/margin_std": 67.80030822753906,
"step": 162
},
{
"KL/chosen_KL_mean": -60.050079345703125,
"KL/mean": -83.6735610961914,
"KL/rejected_KL_mean": -107.29702758789062,
"KL/std": 59.25013732910156,
"epoch": 0.3413612565445026,
"fcm_dpo/beta": 0.007304114755243063,
"fcm_dpo/delta": -0.019453469663858414,
"fcm_dpo/margin": 47.2469482421875,
"fcm_dpo/q_t": 0.41881659626960754,
"grad_norm": 63.22408676147461,
"learning_rate": 4.1782614253949255e-07,
"logits/chosen": -0.9068223237991333,
"logits/rejected": -0.9022581577301025,
"logps/chosen": -353.54998779296875,
"logps/ref_chosen": -293.4999084472656,
"logps/ref_rejected": -266.7116394042969,
"logps/rejected": -374.0086975097656,
"loss": 4.524,
"margin_dpo/margin_mean": 47.2469482421875,
"margin_dpo/margin_std": 64.95793914794922,
"step": 163
},
{
"KL/chosen_KL_mean": -71.46724700927734,
"KL/mean": -92.78965759277344,
"KL/rejected_KL_mean": -114.11207580566406,
"KL/std": 66.23802947998047,
"epoch": 0.34345549738219894,
"fcm_dpo/beta": 0.007272779941558838,
"fcm_dpo/delta": -0.011227300390601158,
"fcm_dpo/margin": 42.64482116699219,
"fcm_dpo/q_t": 0.4294374883174896,
"grad_norm": 73.2182846069336,
"learning_rate": 4.164647253573289e-07,
"logits/chosen": -0.8723958730697632,
"logits/rejected": -0.8833568096160889,
"logps/chosen": -338.5167541503906,
"logps/ref_chosen": -267.04949951171875,
"logps/ref_rejected": -215.9768829345703,
"logps/rejected": -330.08892822265625,
"loss": 4.7089,
"margin_dpo/margin_mean": 42.64482116699219,
"margin_dpo/margin_std": 75.42022705078125,
"step": 164
},
{
"KL/chosen_KL_mean": -60.704872131347656,
"KL/mean": -76.39503479003906,
"KL/rejected_KL_mean": -92.085205078125,
"KL/std": 55.665733337402344,
"epoch": 0.34554973821989526,
"fcm_dpo/beta": 0.007172515615820885,
"fcm_dpo/delta": 0.003400525078177452,
"fcm_dpo/margin": 31.38031768798828,
"fcm_dpo/q_t": 0.44702640175819397,
"grad_norm": 70.69437408447266,
"learning_rate": 4.1509438117713863e-07,
"logits/chosen": -0.9045889973640442,
"logits/rejected": -0.877446711063385,
"logps/chosen": -338.766357421875,
"logps/ref_chosen": -278.06146240234375,
"logps/ref_rejected": -260.4288635253906,
"logps/rejected": -352.5140686035156,
"loss": 4.9154,
"margin_dpo/margin_mean": 31.38031768798828,
"margin_dpo/margin_std": 62.93089294433594,
"step": 165
},
{
"KL/chosen_KL_mean": -56.28936767578125,
"KL/mean": -76.24816131591797,
"KL/rejected_KL_mean": -96.20693969726562,
"KL/std": 58.851802825927734,
"epoch": 0.34764397905759165,
"fcm_dpo/beta": 0.007088161073625088,
"fcm_dpo/delta": -0.018855443224310875,
"fcm_dpo/margin": 39.91756820678711,
"fcm_dpo/q_t": 0.43469470739364624,
"grad_norm": 65.55958557128906,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": -0.8408677577972412,
"logits/rejected": -0.8041284084320068,
"logps/chosen": -332.2384338378906,
"logps/ref_chosen": -275.9490661621094,
"logps/ref_rejected": -232.13473510742188,
"logps/rejected": -328.3417053222656,
"loss": 4.7531,
"margin_dpo/margin_mean": 39.91757583618164,
"margin_dpo/margin_std": 71.8862075805664,
"step": 166
},
{
"KL/chosen_KL_mean": -66.38935852050781,
"KL/mean": -88.67193603515625,
"KL/rejected_KL_mean": -110.95450592041016,
"KL/std": 64.25645446777344,
"epoch": 0.34973821989528797,
"fcm_dpo/beta": 0.007166431751102209,
"fcm_dpo/delta": 0.017246726900339127,
"fcm_dpo/margin": 44.56515884399414,
"fcm_dpo/q_t": 0.4254525899887085,
"grad_norm": 65.34365844726562,
"learning_rate": 4.123272062470633e-07,
"logits/chosen": -0.8350700736045837,
"logits/rejected": -0.8198394179344177,
"logps/chosen": -346.9407653808594,
"logps/ref_chosen": -280.5514221191406,
"logps/ref_rejected": -255.2896728515625,
"logps/rejected": -366.24420166015625,
"loss": 4.6514,
"margin_dpo/margin_mean": 44.56515884399414,
"margin_dpo/margin_std": 73.58280181884766,
"step": 167
},
{
"KL/chosen_KL_mean": -68.78351593017578,
"KL/mean": -92.84286499023438,
"KL/rejected_KL_mean": -116.9021987915039,
"KL/std": 67.12002563476562,
"epoch": 0.3518324607329843,
"fcm_dpo/beta": 0.007110526319593191,
"fcm_dpo/delta": -0.016713347285985947,
"fcm_dpo/margin": 48.118675231933594,
"fcm_dpo/q_t": 0.42008110880851746,
"grad_norm": 129.8765869140625,
"learning_rate": 4.1093052389237174e-07,
"logits/chosen": -0.829613447189331,
"logits/rejected": -0.8005003333091736,
"logps/chosen": -384.58172607421875,
"logps/ref_chosen": -315.7982177734375,
"logps/ref_rejected": -291.48406982421875,
"logps/rejected": -408.38629150390625,
"loss": 4.5751,
"margin_dpo/margin_mean": 48.118675231933594,
"margin_dpo/margin_std": 73.80198669433594,
"step": 168
},
{
"KL/chosen_KL_mean": -65.46214294433594,
"KL/mean": -94.92474365234375,
"KL/rejected_KL_mean": -124.38734436035156,
"KL/std": 68.11466979980469,
"epoch": 0.3539267015706806,
"fcm_dpo/beta": 0.006964054889976978,
"fcm_dpo/delta": -0.05502891540527344,
"fcm_dpo/margin": 58.92519760131836,
"fcm_dpo/q_t": 0.40527647733688354,
"grad_norm": 64.64112091064453,
"learning_rate": 4.0952521132208267e-07,
"logits/chosen": -0.826874852180481,
"logits/rejected": -0.8327090740203857,
"logps/chosen": -326.5263977050781,
"logps/ref_chosen": -261.06427001953125,
"logps/ref_rejected": -235.40663146972656,
"logps/rejected": -359.7939453125,
"loss": 4.3467,
"margin_dpo/margin_mean": 58.925201416015625,
"margin_dpo/margin_std": 73.77999877929688,
"step": 169
},
{
"KL/chosen_KL_mean": -64.50227355957031,
"KL/mean": -88.4448013305664,
"KL/rejected_KL_mean": -112.3873062133789,
"KL/std": 70.87551879882812,
"epoch": 0.35602094240837695,
"fcm_dpo/beta": 0.006804569158703089,
"fcm_dpo/delta": 0.01784161478281021,
"fcm_dpo/margin": 47.88505172729492,
"fcm_dpo/q_t": 0.4266098141670227,
"grad_norm": 65.7511978149414,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": -0.8047292828559875,
"logits/rejected": -0.8058911561965942,
"logps/chosen": -373.4695129394531,
"logps/ref_chosen": -308.96722412109375,
"logps/ref_rejected": -263.8466796875,
"logps/rejected": -376.2340087890625,
"loss": 4.6903,
"margin_dpo/margin_mean": 47.88505172729492,
"margin_dpo/margin_std": 88.50460815429688,
"step": 170
},
{
"KL/chosen_KL_mean": -54.57660675048828,
"KL/mean": -83.85132598876953,
"KL/rejected_KL_mean": -113.12605285644531,
"KL/std": 67.29131317138672,
"epoch": 0.3581151832460733,
"fcm_dpo/beta": 0.006733529269695282,
"fcm_dpo/delta": -0.03675668314099312,
"fcm_dpo/margin": 58.549434661865234,
"fcm_dpo/q_t": 0.40839463472366333,
"grad_norm": 64.48705291748047,
"learning_rate": 4.0668899744407567e-07,
"logits/chosen": -0.8200169801712036,
"logits/rejected": -0.8341271877288818,
"logps/chosen": -313.46563720703125,
"logps/ref_chosen": -258.8890380859375,
"logps/ref_rejected": -262.19140625,
"logps/rejected": -375.31744384765625,
"loss": 4.378,
"margin_dpo/margin_mean": 58.5494384765625,
"margin_dpo/margin_std": 74.06875610351562,
"step": 171
},
{
"KL/chosen_KL_mean": -60.651363372802734,
"KL/mean": -76.64715576171875,
"KL/rejected_KL_mean": -92.64295959472656,
"KL/std": 60.878604888916016,
"epoch": 0.36020942408376966,
"fcm_dpo/beta": 0.006755727343261242,
"fcm_dpo/delta": 0.020635303109884262,
"fcm_dpo/margin": 31.991588592529297,
"fcm_dpo/q_t": 0.44898900389671326,
"grad_norm": 55.07887268066406,
"learning_rate": 4.0525824823390043e-07,
"logits/chosen": -0.8459972143173218,
"logits/rejected": -0.8613450527191162,
"logps/chosen": -399.6737060546875,
"logps/ref_chosen": -339.0223388671875,
"logps/ref_rejected": -295.78759765625,
"logps/rejected": -388.4305419921875,
"loss": 4.9779,
"margin_dpo/margin_mean": 31.991588592529297,
"margin_dpo/margin_std": 74.38744354248047,
"step": 172
},
{
"KL/chosen_KL_mean": -57.42732620239258,
"KL/mean": -79.39176940917969,
"KL/rejected_KL_mean": -101.35624694824219,
"KL/std": 64.20226287841797,
"epoch": 0.362303664921466,
"fcm_dpo/beta": 0.006686392240226269,
"fcm_dpo/delta": -0.030118033289909363,
"fcm_dpo/margin": 43.928924560546875,
"fcm_dpo/q_t": 0.43261343240737915,
"grad_norm": 80.68160247802734,
"learning_rate": 4.0381917299505686e-07,
"logits/chosen": -0.855338990688324,
"logits/rejected": -0.8514031171798706,
"logps/chosen": -357.5387878417969,
"logps/ref_chosen": -300.1114501953125,
"logps/ref_rejected": -273.78460693359375,
"logps/rejected": -375.140869140625,
"loss": 4.7279,
"margin_dpo/margin_mean": 43.928924560546875,
"margin_dpo/margin_std": 75.71229553222656,
"step": 173
},
{
"KL/chosen_KL_mean": -57.29583740234375,
"KL/mean": -83.97393035888672,
"KL/rejected_KL_mean": -110.65203857421875,
"KL/std": 64.6370849609375,
"epoch": 0.3643979057591623,
"fcm_dpo/beta": 0.006674854550510645,
"fcm_dpo/delta": 0.012298239395022392,
"fcm_dpo/margin": 53.3561897277832,
"fcm_dpo/q_t": 0.4156748950481415,
"grad_norm": 72.79425811767578,
"learning_rate": 4.0237184890078243e-07,
"logits/chosen": -0.8256723284721375,
"logits/rejected": -0.805625855922699,
"logps/chosen": -392.3497009277344,
"logps/ref_chosen": -335.0538635253906,
"logps/ref_rejected": -257.4646911621094,
"logps/rejected": -368.11669921875,
"loss": 4.4548,
"margin_dpo/margin_mean": 53.3561897277832,
"margin_dpo/margin_std": 67.76395416259766,
"step": 174
},
{
"KL/chosen_KL_mean": -72.9652328491211,
"KL/mean": -96.99482727050781,
"KL/rejected_KL_mean": -121.0244140625,
"KL/std": 73.48253631591797,
"epoch": 0.36649214659685864,
"fcm_dpo/beta": 0.006608583033084869,
"fcm_dpo/delta": -0.04623459279537201,
"fcm_dpo/margin": 48.05918884277344,
"fcm_dpo/q_t": 0.4289584159851074,
"grad_norm": 79.97168731689453,
"learning_rate": 4.00916353566676e-07,
"logits/chosen": -0.8397626280784607,
"logits/rejected": -0.8402819633483887,
"logps/chosen": -357.36083984375,
"logps/ref_chosen": -284.39556884765625,
"logps/ref_rejected": -283.3876647949219,
"logps/rejected": -404.41204833984375,
"loss": 4.7035,
"margin_dpo/margin_mean": 48.05918884277344,
"margin_dpo/margin_std": 82.69506072998047,
"step": 175
},
{
"KL/chosen_KL_mean": -70.91928100585938,
"KL/mean": -89.49232482910156,
"KL/rejected_KL_mean": -108.06539916992188,
"KL/std": 68.54695129394531,
"epoch": 0.36858638743455496,
"fcm_dpo/beta": 0.006419216748327017,
"fcm_dpo/delta": -0.016511470079421997,
"fcm_dpo/margin": 37.14612579345703,
"fcm_dpo/q_t": 0.4452558159828186,
"grad_norm": 69.9654312133789,
"learning_rate": 3.994527650465352e-07,
"logits/chosen": -0.8216432929039001,
"logits/rejected": -0.8281827569007874,
"logps/chosen": -322.7320861816406,
"logps/ref_chosen": -251.81280517578125,
"logps/ref_rejected": -242.05328369140625,
"logps/rejected": -350.1186828613281,
"loss": 4.9657,
"margin_dpo/margin_mean": 37.14612579345703,
"margin_dpo/margin_std": 86.01189422607422,
"step": 176
},
{
"KL/chosen_KL_mean": -77.08590698242188,
"KL/mean": -96.39353942871094,
"KL/rejected_KL_mean": -115.70115661621094,
"KL/std": 66.26946258544922,
"epoch": 0.3706806282722513,
"fcm_dpo/beta": 0.006435380782932043,
"fcm_dpo/delta": 0.015266492031514645,
"fcm_dpo/margin": 38.615257263183594,
"fcm_dpo/q_t": 0.44256123900413513,
"grad_norm": 64.47611999511719,
"learning_rate": 3.979811618281705e-07,
"logits/chosen": -0.8922350406646729,
"logits/rejected": -0.860856831073761,
"logps/chosen": -375.7322998046875,
"logps/ref_chosen": -298.6463928222656,
"logps/ref_rejected": -295.66534423828125,
"logps/rejected": -411.36651611328125,
"loss": 4.9252,
"margin_dpo/margin_mean": 38.615257263183594,
"margin_dpo/margin_std": 81.1719970703125,
"step": 177
},
{
"KL/chosen_KL_mean": -62.48854064941406,
"KL/mean": -90.35904693603516,
"KL/rejected_KL_mean": -118.22955322265625,
"KL/std": 71.95285034179688,
"epoch": 0.37277486910994767,
"fcm_dpo/beta": 0.006367242429405451,
"fcm_dpo/delta": -0.016469581052660942,
"fcm_dpo/margin": 55.74102020263672,
"fcm_dpo/q_t": 0.41873496770858765,
"grad_norm": 55.844093322753906,
"learning_rate": 3.9650162282919654e-07,
"logits/chosen": -0.8221363425254822,
"logits/rejected": -0.8154042363166809,
"logps/chosen": -348.7462158203125,
"logps/ref_chosen": -286.2576599121094,
"logps/ref_rejected": -243.97491455078125,
"logps/rejected": -362.2044677734375,
"loss": 4.5338,
"margin_dpo/margin_mean": 55.74102020263672,
"margin_dpo/margin_std": 83.90606689453125,
"step": 178
},
{
"KL/chosen_KL_mean": -69.41419219970703,
"KL/mean": -93.62001037597656,
"KL/rejected_KL_mean": -117.8258056640625,
"KL/std": 72.83280181884766,
"epoch": 0.374869109947644,
"fcm_dpo/beta": 0.006365994457155466,
"fcm_dpo/delta": -0.003825224470347166,
"fcm_dpo/margin": 48.41162109375,
"fcm_dpo/q_t": 0.42837071418762207,
"grad_norm": 88.72013854980469,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": -0.8086428642272949,
"logits/rejected": -0.8089961409568787,
"logps/chosen": -329.1512451171875,
"logps/ref_chosen": -259.737060546875,
"logps/ref_rejected": -277.8813171386719,
"logps/rejected": -395.7071533203125,
"loss": 4.6943,
"margin_dpo/margin_mean": 48.41161346435547,
"margin_dpo/margin_std": 81.75579071044922,
"step": 179
},
{
"KL/chosen_KL_mean": -75.5199966430664,
"KL/mean": -102.80406188964844,
"KL/rejected_KL_mean": -130.08815002441406,
"KL/std": 69.5558090209961,
"epoch": 0.3769633507853403,
"fcm_dpo/beta": 0.006364882458001375,
"fcm_dpo/delta": 0.006663701497018337,
"fcm_dpo/margin": 54.568145751953125,
"fcm_dpo/q_t": 0.421146959066391,
"grad_norm": 86.5185317993164,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": -0.8502231240272522,
"logits/rejected": -0.8749803900718689,
"logps/chosen": -342.8288879394531,
"logps/ref_chosen": -267.30889892578125,
"logps/ref_rejected": -230.4376983642578,
"logps/rejected": -360.5258483886719,
"loss": 4.5368,
"margin_dpo/margin_mean": 54.568138122558594,
"margin_dpo/margin_std": 80.2123031616211,
"step": 180
},
{
"KL/chosen_KL_mean": -86.32603454589844,
"KL/mean": -108.53008270263672,
"KL/rejected_KL_mean": -130.734130859375,
"KL/std": 71.4383544921875,
"epoch": 0.37905759162303665,
"fcm_dpo/beta": 0.006331674288958311,
"fcm_dpo/delta": 0.0014796480536460876,
"fcm_dpo/margin": 44.40808868408203,
"fcm_dpo/q_t": 0.43516260385513306,
"grad_norm": 98.99745178222656,
"learning_rate": 3.920161866827889e-07,
"logits/chosen": -0.8576675057411194,
"logits/rejected": -0.8596795201301575,
"logps/chosen": -386.81744384765625,
"logps/ref_chosen": -300.49139404296875,
"logps/ref_rejected": -278.98284912109375,
"logps/rejected": -409.71697998046875,
"loss": 4.7925,
"margin_dpo/margin_mean": 44.40808868408203,
"margin_dpo/margin_std": 85.0110855102539,
"step": 181
},
{
"KL/chosen_KL_mean": -83.20874786376953,
"KL/mean": -115.09024810791016,
"KL/rejected_KL_mean": -146.97177124023438,
"KL/std": 71.0909652709961,
"epoch": 0.381151832460733,
"fcm_dpo/beta": 0.0062592774629592896,
"fcm_dpo/delta": -0.056663818657398224,
"fcm_dpo/margin": 63.763023376464844,
"fcm_dpo/q_t": 0.40782204270362854,
"grad_norm": 63.953773498535156,
"learning_rate": 3.90505702185e-07,
"logits/chosen": -0.837653398513794,
"logits/rejected": -0.8604555130004883,
"logps/chosen": -362.7068786621094,
"logps/ref_chosen": -279.4981689453125,
"logps/ref_rejected": -263.6926574707031,
"logps/rejected": -410.6643981933594,
"loss": 4.3798,
"margin_dpo/margin_mean": 63.76302719116211,
"margin_dpo/margin_std": 82.07465362548828,
"step": 182
},
{
"KL/chosen_KL_mean": -85.31684875488281,
"KL/mean": -112.9171371459961,
"KL/rejected_KL_mean": -140.51742553710938,
"KL/std": 73.43064880371094,
"epoch": 0.3832460732984293,
"fcm_dpo/beta": 0.006128930952399969,
"fcm_dpo/delta": 0.017954856157302856,
"fcm_dpo/margin": 55.20054626464844,
"fcm_dpo/q_t": 0.4230417311191559,
"grad_norm": 52.553104400634766,
"learning_rate": 3.889876827928156e-07,
"logits/chosen": -0.8802267909049988,
"logits/rejected": -0.8798730373382568,
"logps/chosen": -356.5225524902344,
"logps/ref_chosen": -271.2057189941406,
"logps/ref_rejected": -243.91549682617188,
"logps/rejected": -384.4328918457031,
"loss": 4.6031,
"margin_dpo/margin_mean": 55.20054626464844,
"margin_dpo/margin_std": 89.55872344970703,
"step": 183
},
{
"KL/chosen_KL_mean": -81.1993179321289,
"KL/mean": -120.1291732788086,
"KL/rejected_KL_mean": -159.05902099609375,
"KL/std": 81.69471740722656,
"epoch": 0.38534031413612563,
"fcm_dpo/beta": 0.005876118782907724,
"fcm_dpo/delta": -0.07986614853143692,
"fcm_dpo/margin": 77.8597183227539,
"fcm_dpo/q_t": 0.3941153287887573,
"grad_norm": 65.2461929321289,
"learning_rate": 3.874622099130087e-07,
"logits/chosen": -0.9231401085853577,
"logits/rejected": -0.897407591342926,
"logps/chosen": -399.64508056640625,
"logps/ref_chosen": -318.4457702636719,
"logps/ref_rejected": -266.640869140625,
"logps/rejected": -425.69989013671875,
"loss": 4.21,
"margin_dpo/margin_mean": 77.8597183227539,
"margin_dpo/margin_std": 92.04389190673828,
"step": 184
},
{
"KL/chosen_KL_mean": -98.42439270019531,
"KL/mean": -126.96806335449219,
"KL/rejected_KL_mean": -155.51171875,
"KL/std": 88.64209747314453,
"epoch": 0.387434554973822,
"fcm_dpo/beta": 0.005731166340410709,
"fcm_dpo/delta": -0.018187183886766434,
"fcm_dpo/margin": 57.087345123291016,
"fcm_dpo/q_t": 0.42507031559944153,
"grad_norm": 85.08312225341797,
"learning_rate": 3.859293653520604e-07,
"logits/chosen": -0.8975915312767029,
"logits/rejected": -0.889983594417572,
"logps/chosen": -372.7332458496094,
"logps/ref_chosen": -274.308837890625,
"logps/ref_rejected": -260.7274169921875,
"logps/rejected": -416.2391357421875,
"loss": 4.6385,
"margin_dpo/margin_mean": 57.087345123291016,
"margin_dpo/margin_std": 94.04627227783203,
"step": 185
},
{
"KL/chosen_KL_mean": -101.5023193359375,
"KL/mean": -128.28860473632812,
"KL/rejected_KL_mean": -155.07489013671875,
"KL/std": 79.86894989013672,
"epoch": 0.38952879581151834,
"fcm_dpo/beta": 0.005754129029810429,
"fcm_dpo/delta": 0.00872084405273199,
"fcm_dpo/margin": 53.57258987426758,
"fcm_dpo/q_t": 0.42865413427352905,
"grad_norm": 102.2063980102539,
"learning_rate": 3.8438923131177237e-07,
"logits/chosen": -0.9187572002410889,
"logits/rejected": -0.9172680974006653,
"logps/chosen": -400.5076904296875,
"logps/ref_chosen": -299.00537109375,
"logps/ref_rejected": -274.4014587402344,
"logps/rejected": -429.47637939453125,
"loss": 4.682,
"margin_dpo/margin_mean": 53.57258605957031,
"margin_dpo/margin_std": 89.2789535522461,
"step": 186
},
{
"KL/chosen_KL_mean": -112.88499450683594,
"KL/mean": -136.58340454101562,
"KL/rejected_KL_mean": -160.28182983398438,
"KL/std": 84.63465118408203,
"epoch": 0.39162303664921466,
"fcm_dpo/beta": 0.0057259490713477135,
"fcm_dpo/delta": -0.0088438019156456,
"fcm_dpo/margin": 47.3968505859375,
"fcm_dpo/q_t": 0.4351907968521118,
"grad_norm": 104.48662567138672,
"learning_rate": 3.828418903848593e-07,
"logits/chosen": -0.8610119223594666,
"logits/rejected": -0.8431413769721985,
"logps/chosen": -442.7103271484375,
"logps/ref_chosen": -329.8253173828125,
"logps/ref_rejected": -263.73175048828125,
"logps/rejected": -424.0135803222656,
"loss": 4.8913,
"margin_dpo/margin_mean": 47.396854400634766,
"margin_dpo/margin_std": 102.80694580078125,
"step": 187
},
{
"KL/chosen_KL_mean": -101.29950714111328,
"KL/mean": -128.2642364501953,
"KL/rejected_KL_mean": -155.22898864746094,
"KL/std": 87.04512023925781,
"epoch": 0.393717277486911,
"fcm_dpo/beta": 0.005684699863195419,
"fcm_dpo/delta": -0.011355758644640446,
"fcm_dpo/margin": 53.92945861816406,
"fcm_dpo/q_t": 0.43097540736198425,
"grad_norm": 83.95048522949219,
"learning_rate": 3.812874255505191e-07,
"logits/chosen": -0.8679317235946655,
"logits/rejected": -0.8562878370285034,
"logps/chosen": -364.3051452636719,
"logps/ref_chosen": -263.005615234375,
"logps/ref_rejected": -247.08668518066406,
"logps/rejected": -402.315673828125,
"loss": 4.7949,
"margin_dpo/margin_mean": 53.92945861816406,
"margin_dpo/margin_std": 108.377685546875,
"step": 188
},
{
"KL/chosen_KL_mean": -84.28831481933594,
"KL/mean": -118.6055908203125,
"KL/rejected_KL_mean": -152.9228973388672,
"KL/std": 86.47982788085938,
"epoch": 0.3958115183246073,
"fcm_dpo/beta": 0.00552529888227582,
"fcm_dpo/delta": -0.042275551706552505,
"fcm_dpo/margin": 68.63457489013672,
"fcm_dpo/q_t": 0.4125663638114929,
"grad_norm": 86.17080688476562,
"learning_rate": 3.797259201699833e-07,
"logits/chosen": -0.8880312442779541,
"logits/rejected": -0.8941810131072998,
"logps/chosen": -357.24871826171875,
"logps/ref_chosen": -272.96038818359375,
"logps/ref_rejected": -275.13238525390625,
"logps/rejected": -428.0552978515625,
"loss": 4.4343,
"margin_dpo/margin_mean": 68.63457489013672,
"margin_dpo/margin_std": 91.30592346191406,
"step": 189
},
{
"KL/chosen_KL_mean": -82.46324157714844,
"KL/mean": -112.3152847290039,
"KL/rejected_KL_mean": -142.16734313964844,
"KL/std": 79.62899780273438,
"epoch": 0.39790575916230364,
"fcm_dpo/beta": 0.0054209912195801735,
"fcm_dpo/delta": -0.019242707639932632,
"fcm_dpo/margin": 59.70409393310547,
"fcm_dpo/q_t": 0.4244186282157898,
"grad_norm": 67.50397491455078,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": -0.8963736891746521,
"logits/rejected": -0.8552230000495911,
"logps/chosen": -340.2608337402344,
"logps/ref_chosen": -257.79754638671875,
"logps/ref_rejected": -225.2164306640625,
"logps/rejected": -367.3837585449219,
"loss": 4.5841,
"margin_dpo/margin_mean": 59.70409393310547,
"margin_dpo/margin_std": 87.59053039550781,
"step": 190
},
{
"KL/chosen_KL_mean": -74.05329132080078,
"KL/mean": -102.2022705078125,
"KL/rejected_KL_mean": -130.35125732421875,
"KL/std": 79.56758117675781,
"epoch": 0.4,
"fcm_dpo/beta": 0.005430443678051233,
"fcm_dpo/delta": 0.0181720070540905,
"fcm_dpo/margin": 56.29796600341797,
"fcm_dpo/q_t": 0.42908400297164917,
"grad_norm": 76.7206802368164,
"learning_rate": 3.765821230985757e-07,
"logits/chosen": -0.9064384698867798,
"logits/rejected": -0.9073519706726074,
"logps/chosen": -317.9118347167969,
"logps/ref_chosen": -243.8585205078125,
"logps/ref_rejected": -245.12136840820312,
"logps/rejected": -375.47265625,
"loss": 4.6776,
"margin_dpo/margin_mean": 56.29796600341797,
"margin_dpo/margin_std": 94.13633728027344,
"step": 191
},
{
"KL/chosen_KL_mean": -82.34419250488281,
"KL/mean": -105.46398162841797,
"KL/rejected_KL_mean": -128.58377075195312,
"KL/std": 74.32576751708984,
"epoch": 0.40209424083769635,
"fcm_dpo/beta": 0.005527670495212078,
"fcm_dpo/delta": 0.020868487656116486,
"fcm_dpo/margin": 46.239559173583984,
"fcm_dpo/q_t": 0.44014275074005127,
"grad_norm": 95.28579711914062,
"learning_rate": 3.75e-07,
"logits/chosen": -0.8447480797767639,
"logits/rejected": -0.8279674053192139,
"logps/chosen": -349.3241271972656,
"logps/ref_chosen": -266.9799499511719,
"logps/ref_rejected": -260.1697082519531,
"logps/rejected": -388.7534484863281,
"loss": 4.807,
"margin_dpo/margin_mean": 46.23956298828125,
"margin_dpo/margin_std": 82.4874038696289,
"step": 192
},
{
"KL/chosen_KL_mean": -95.75445556640625,
"KL/mean": -122.55574035644531,
"KL/rejected_KL_mean": -149.35702514648438,
"KL/std": 84.06842041015625,
"epoch": 0.4041884816753927,
"fcm_dpo/beta": 0.005618045572191477,
"fcm_dpo/delta": 0.020845647901296616,
"fcm_dpo/margin": 53.602569580078125,
"fcm_dpo/q_t": 0.4309660792350769,
"grad_norm": 59.146549224853516,
"learning_rate": 3.734111735307796e-07,
"logits/chosen": -0.8828919529914856,
"logits/rejected": -0.8536106944084167,
"logps/chosen": -376.0076904296875,
"logps/ref_chosen": -280.25323486328125,
"logps/ref_rejected": -291.0348815917969,
"logps/rejected": -440.39190673828125,
"loss": 4.7189,
"margin_dpo/margin_mean": 53.602569580078125,
"margin_dpo/margin_std": 93.06439208984375,
"step": 193
},
{
"KL/chosen_KL_mean": -94.51910400390625,
"KL/mean": -116.68457794189453,
"KL/rejected_KL_mean": -138.85008239746094,
"KL/std": 80.56071472167969,
"epoch": 0.406282722513089,
"fcm_dpo/beta": 0.00563051225617528,
"fcm_dpo/delta": -0.008683041669428349,
"fcm_dpo/margin": 44.33095932006836,
"fcm_dpo/q_t": 0.4425777792930603,
"grad_norm": 60.31032180786133,
"learning_rate": 3.7181572889485623e-07,
"logits/chosen": -0.8641772270202637,
"logits/rejected": -0.8540708422660828,
"logps/chosen": -382.6585693359375,
"logps/ref_chosen": -288.13946533203125,
"logps/ref_rejected": -251.31529235839844,
"logps/rejected": -390.1653747558594,
"loss": 4.8567,
"margin_dpo/margin_mean": 44.33095932006836,
"margin_dpo/margin_std": 86.52152252197266,
"step": 194
},
{
"KL/chosen_KL_mean": -95.61801147460938,
"KL/mean": -117.46638488769531,
"KL/rejected_KL_mean": -139.3147735595703,
"KL/std": 76.0776596069336,
"epoch": 0.4083769633507853,
"fcm_dpo/beta": 0.005567906890064478,
"fcm_dpo/delta": -0.02082628197968006,
"fcm_dpo/margin": 43.6967658996582,
"fcm_dpo/q_t": 0.44454753398895264,
"grad_norm": 55.601314544677734,
"learning_rate": 3.7021375165108377e-07,
"logits/chosen": -0.8613829612731934,
"logits/rejected": -0.865749716758728,
"logps/chosen": -369.6186218261719,
"logps/ref_chosen": -274.0006408691406,
"logps/ref_rejected": -280.22723388671875,
"logps/rejected": -419.5419921875,
"loss": 4.8687,
"margin_dpo/margin_mean": 43.6967658996582,
"margin_dpo/margin_std": 84.557373046875,
"step": 195
},
{
"KL/chosen_KL_mean": -85.62928771972656,
"KL/mean": -110.2017822265625,
"KL/rejected_KL_mean": -134.7742919921875,
"KL/std": 79.14740753173828,
"epoch": 0.41047120418848165,
"fcm_dpo/beta": 0.005564381834119558,
"fcm_dpo/delta": 0.017413640394806862,
"fcm_dpo/margin": 49.14500045776367,
"fcm_dpo/q_t": 0.4364781379699707,
"grad_norm": 77.14463806152344,
"learning_rate": 3.6860532770864005e-07,
"logits/chosen": -0.8474195003509521,
"logits/rejected": -0.8541578650474548,
"logps/chosen": -360.52996826171875,
"logps/ref_chosen": -274.90069580078125,
"logps/ref_rejected": -248.7281951904297,
"logps/rejected": -383.5024719238281,
"loss": 4.7765,
"margin_dpo/margin_mean": 49.14500045776367,
"margin_dpo/margin_std": 88.91536712646484,
"step": 196
},
{
"KL/chosen_KL_mean": -91.18795776367188,
"KL/mean": -121.93099212646484,
"KL/rejected_KL_mean": -152.67404174804688,
"KL/std": 80.69728088378906,
"epoch": 0.41256544502617803,
"fcm_dpo/beta": 0.00554257445037365,
"fcm_dpo/delta": -0.030014852061867714,
"fcm_dpo/margin": 61.48606491088867,
"fcm_dpo/q_t": 0.4207172095775604,
"grad_norm": 74.54379272460938,
"learning_rate": 3.6699054332241985e-07,
"logits/chosen": -0.8761681318283081,
"logits/rejected": -0.8605803847312927,
"logps/chosen": -400.7228088378906,
"logps/ref_chosen": -309.5348205566406,
"logps/ref_rejected": -264.3179931640625,
"logps/rejected": -416.99200439453125,
"loss": 4.5427,
"margin_dpo/margin_mean": 61.48606491088867,
"margin_dpo/margin_std": 85.9816665649414,
"step": 197
},
{
"KL/chosen_KL_mean": -89.14295959472656,
"KL/mean": -122.2314453125,
"KL/rejected_KL_mean": -155.31993103027344,
"KL/std": 85.83655548095703,
"epoch": 0.41465968586387436,
"fcm_dpo/beta": 0.00542075652629137,
"fcm_dpo/delta": -0.011349959298968315,
"fcm_dpo/margin": 66.17696380615234,
"fcm_dpo/q_t": 0.4171242415904999,
"grad_norm": 58.74266052246094,
"learning_rate": 3.653694850884091e-07,
"logits/chosen": -0.8896830081939697,
"logits/rejected": -0.8643697500228882,
"logps/chosen": -390.1564025878906,
"logps/ref_chosen": -301.0134582519531,
"logps/ref_rejected": -292.84185791015625,
"logps/rejected": -448.1617736816406,
"loss": 4.5038,
"margin_dpo/margin_mean": 66.17696380615234,
"margin_dpo/margin_std": 95.03239440917969,
"step": 198
},
{
"KL/chosen_KL_mean": -100.06723022460938,
"KL/mean": -126.93196868896484,
"KL/rejected_KL_mean": -153.79669189453125,
"KL/std": 82.8907241821289,
"epoch": 0.4167539267015707,
"fcm_dpo/beta": 0.005536025855690241,
"fcm_dpo/delta": 0.02695927582681179,
"fcm_dpo/margin": 53.729469299316406,
"fcm_dpo/q_t": 0.43211570382118225,
"grad_norm": 89.9813461303711,
"learning_rate": 3.6374223993904124e-07,
"logits/chosen": -0.8591454029083252,
"logits/rejected": -0.8150307536125183,
"logps/chosen": -364.6730651855469,
"logps/ref_chosen": -264.6058654785156,
"logps/ref_rejected": -214.9014892578125,
"logps/rejected": -368.69818115234375,
"loss": 4.7389,
"margin_dpo/margin_mean": 53.729469299316406,
"margin_dpo/margin_std": 98.36550903320312,
"step": 199
},
{
"KL/chosen_KL_mean": -110.9617919921875,
"KL/mean": -136.12791442871094,
"KL/rejected_KL_mean": -161.29400634765625,
"KL/std": 87.3275146484375,
"epoch": 0.418848167539267,
"fcm_dpo/beta": 0.005466805305331945,
"fcm_dpo/delta": -0.027961192652583122,
"fcm_dpo/margin": 50.332218170166016,
"fcm_dpo/q_t": 0.4367090165615082,
"grad_norm": 66.81365966796875,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": -0.9206555485725403,
"logits/rejected": -0.8960063457489014,
"logps/chosen": -435.1206359863281,
"logps/ref_chosen": -324.1588134765625,
"logps/ref_rejected": -277.80218505859375,
"logps/rejected": -439.09625244140625,
"loss": 4.8804,
"margin_dpo/margin_mean": 50.332218170166016,
"margin_dpo/margin_std": 108.61395263671875,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_KL/chosen_KL_mean": -123.2765121459961,
"eval_KL/mean": -151.9739227294922,
"eval_KL/rejected_KL_mean": -180.67132568359375,
"eval_KL/std": 89.34925842285156,
"eval_fcm_dpo/beta": 0.0054236906580626965,
"eval_logits/chosen": -0.9016394019126892,
"eval_logits/rejected": -0.8836301565170288,
"eval_logps/chosen": -411.103271484375,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -447.601318359375,
"eval_loss": 0.5898867249488831,
"eval_margin_dpo/margin_mean": 57.394798278808594,
"eval_margin_dpo/margin_std": 97.59921264648438,
"eval_runtime": 78.4876,
"eval_samples_per_second": 25.482,
"eval_steps_per_second": 3.185,
"step": 200
},
{
"KL/chosen_KL_mean": -121.671142578125,
"KL/mean": -151.29833984375,
"KL/rejected_KL_mean": -180.92556762695312,
"KL/std": 91.87706756591797,
"epoch": 0.42094240837696334,
"fcm_dpo/beta": 0.005429561249911785,
"fcm_dpo/delta": -0.0011716950684785843,
"fcm_dpo/margin": 59.25444030761719,
"fcm_dpo/q_t": 0.4263926148414612,
"grad_norm": 70.4216537475586,
"learning_rate": 3.604695382782159e-07,
"logits/chosen": -0.889416515827179,
"logits/rejected": -0.8737778663635254,
"logps/chosen": -393.1668395996094,
"logps/ref_chosen": -271.49566650390625,
"logps/ref_rejected": -245.71414184570312,
"logps/rejected": -426.63970947265625,
"loss": 4.6884,
"margin_dpo/margin_mean": 59.25444030761719,
"margin_dpo/margin_std": 104.75772857666016,
"step": 201
},
{
"KL/chosen_KL_mean": -129.14830017089844,
"KL/mean": -156.79586791992188,
"KL/rejected_KL_mean": -184.44342041015625,
"KL/std": 89.80362701416016,
"epoch": 0.42303664921465967,
"fcm_dpo/beta": 0.005406467244029045,
"fcm_dpo/delta": -0.0008845115080475807,
"fcm_dpo/margin": 55.29514694213867,
"fcm_dpo/q_t": 0.43192172050476074,
"grad_norm": 83.05462646484375,
"learning_rate": 3.588242572718162e-07,
"logits/chosen": -0.9037338495254517,
"logits/rejected": -0.8901877999305725,
"logps/chosen": -401.2462158203125,
"logps/ref_chosen": -272.0979309082031,
"logps/ref_rejected": -235.94805908203125,
"logps/rejected": -420.3915100097656,
"loss": 4.7591,
"margin_dpo/margin_mean": 55.29515075683594,
"margin_dpo/margin_std": 104.44528198242188,
"step": 202
},
{
"KL/chosen_KL_mean": -139.50340270996094,
"KL/mean": -160.76329040527344,
"KL/rejected_KL_mean": -182.0231475830078,
"KL/std": 89.24750518798828,
"epoch": 0.42513089005235605,
"fcm_dpo/beta": 0.005487063899636269,
"fcm_dpo/delta": 0.012710193172097206,
"fcm_dpo/margin": 42.51974105834961,
"fcm_dpo/q_t": 0.4455467462539673,
"grad_norm": 63.943870544433594,
"learning_rate": 3.571731403507635e-07,
"logits/chosen": -0.8693050146102905,
"logits/rejected": -0.8741556406021118,
"logps/chosen": -419.7255554199219,
"logps/ref_chosen": -280.2221374511719,
"logps/ref_rejected": -251.79798889160156,
"logps/rejected": -433.8211669921875,
"loss": 4.931,
"margin_dpo/margin_mean": 42.51974105834961,
"margin_dpo/margin_std": 92.08596801757812,
"step": 203
},
{
"KL/chosen_KL_mean": -137.26512145996094,
"KL/mean": -167.29208374023438,
"KL/rejected_KL_mean": -197.31906127929688,
"KL/std": 89.45562744140625,
"epoch": 0.4272251308900524,
"fcm_dpo/beta": 0.005518666002899408,
"fcm_dpo/delta": 0.016294872388243675,
"fcm_dpo/margin": 60.05392074584961,
"fcm_dpo/q_t": 0.4236374497413635,
"grad_norm": 85.55860900878906,
"learning_rate": 3.5551627605944746e-07,
"logits/chosen": -0.929052472114563,
"logits/rejected": -0.9007848501205444,
"logps/chosen": -456.0611572265625,
"logps/ref_chosen": -318.7960510253906,
"logps/ref_rejected": -269.69921875,
"logps/rejected": -467.0182800292969,
"loss": 4.6312,
"margin_dpo/margin_mean": 60.05392074584961,
"margin_dpo/margin_std": 100.22209167480469,
"step": 204
},
{
"KL/chosen_KL_mean": -137.09326171875,
"KL/mean": -172.9523468017578,
"KL/rejected_KL_mean": -208.8114776611328,
"KL/std": 96.3641357421875,
"epoch": 0.4293193717277487,
"fcm_dpo/beta": 0.005441513378173113,
"fcm_dpo/delta": -0.044503822922706604,
"fcm_dpo/margin": 71.71823120117188,
"fcm_dpo/q_t": 0.4126250743865967,
"grad_norm": 90.81995391845703,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": -0.8676726818084717,
"logits/rejected": -0.8353573679924011,
"logps/chosen": -420.8553466796875,
"logps/ref_chosen": -283.7620544433594,
"logps/ref_rejected": -297.69439697265625,
"logps/rejected": -506.50592041015625,
"loss": 4.4849,
"margin_dpo/margin_mean": 71.71823120117188,
"margin_dpo/margin_std": 108.88336181640625,
"step": 205
},
{
"KL/chosen_KL_mean": -141.72567749023438,
"KL/mean": -169.52108764648438,
"KL/rejected_KL_mean": -197.31649780273438,
"KL/std": 93.6350326538086,
"epoch": 0.431413612565445,
"fcm_dpo/beta": 0.005305927712470293,
"fcm_dpo/delta": -0.018429771065711975,
"fcm_dpo/margin": 55.590843200683594,
"fcm_dpo/q_t": 0.43249496817588806,
"grad_norm": 58.64113998413086,
"learning_rate": 3.5218566107988867e-07,
"logits/chosen": -0.8933624029159546,
"logits/rejected": -0.913768470287323,
"logps/chosen": -435.3895263671875,
"logps/ref_chosen": -293.66387939453125,
"logps/ref_rejected": -291.3056640625,
"logps/rejected": -488.6221923828125,
"loss": 4.7898,
"margin_dpo/margin_mean": 55.59084701538086,
"margin_dpo/margin_std": 106.94378662109375,
"step": 206
},
{
"KL/chosen_KL_mean": -130.13616943359375,
"KL/mean": -158.40179443359375,
"KL/rejected_KL_mean": -186.66741943359375,
"KL/std": 104.5025634765625,
"epoch": 0.43350785340314135,
"fcm_dpo/beta": 0.005238719284534454,
"fcm_dpo/delta": -0.0088468287140131,
"fcm_dpo/margin": 56.53125,
"fcm_dpo/q_t": 0.43153464794158936,
"grad_norm": 73.89046478271484,
"learning_rate": 3.505120890024195e-07,
"logits/chosen": -0.8374500870704651,
"logits/rejected": -0.8462930917739868,
"logps/chosen": -400.6712646484375,
"logps/ref_chosen": -270.5350646972656,
"logps/ref_rejected": -278.7747497558594,
"logps/rejected": -465.44219970703125,
"loss": 4.8468,
"margin_dpo/margin_mean": 56.53125,
"margin_dpo/margin_std": 121.36986541748047,
"step": 207
},
{
"KL/chosen_KL_mean": -134.00111389160156,
"KL/mean": -169.24815368652344,
"KL/rejected_KL_mean": -204.4951934814453,
"KL/std": 102.28126525878906,
"epoch": 0.4356020942408377,
"fcm_dpo/beta": 0.005222794134169817,
"fcm_dpo/delta": -0.006807409692555666,
"fcm_dpo/margin": 70.49407196044922,
"fcm_dpo/q_t": 0.417470782995224,
"grad_norm": 77.3393325805664,
"learning_rate": 3.4883312676665534e-07,
"logits/chosen": -0.9094988703727722,
"logits/rejected": -0.8614720106124878,
"logps/chosen": -413.5838928222656,
"logps/ref_chosen": -279.582763671875,
"logps/ref_rejected": -290.041015625,
"logps/rejected": -494.53619384765625,
"loss": 4.5102,
"margin_dpo/margin_mean": 70.49407196044922,
"margin_dpo/margin_std": 107.34097290039062,
"step": 208
},
{
"KL/chosen_KL_mean": -138.67898559570312,
"KL/mean": -159.94009399414062,
"KL/rejected_KL_mean": -181.2012176513672,
"KL/std": 95.46649169921875,
"epoch": 0.437696335078534,
"fcm_dpo/beta": 0.005218043457716703,
"fcm_dpo/delta": 0.007631334010511637,
"fcm_dpo/margin": 42.522220611572266,
"fcm_dpo/q_t": 0.44813573360443115,
"grad_norm": 67.46821594238281,
"learning_rate": 3.4714886441024573e-07,
"logits/chosen": -0.8228191137313843,
"logits/rejected": -0.8171289563179016,
"logps/chosen": -457.5514831542969,
"logps/ref_chosen": -318.8725280761719,
"logps/ref_rejected": -270.64324951171875,
"logps/rejected": -451.8444519042969,
"loss": 5.0119,
"margin_dpo/margin_mean": 42.522220611572266,
"margin_dpo/margin_std": 104.65240478515625,
"step": 209
},
{
"KL/chosen_KL_mean": -121.98692321777344,
"KL/mean": -152.7623291015625,
"KL/rejected_KL_mean": -183.5377655029297,
"KL/std": 90.5138168334961,
"epoch": 0.4397905759162304,
"fcm_dpo/beta": 0.005136261694133282,
"fcm_dpo/delta": -0.03048218972980976,
"fcm_dpo/margin": 61.550846099853516,
"fcm_dpo/q_t": 0.42682385444641113,
"grad_norm": 72.6939697265625,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": -0.842387318611145,
"logits/rejected": -0.832798182964325,
"logps/chosen": -405.12725830078125,
"logps/ref_chosen": -283.14031982421875,
"logps/ref_rejected": -287.2986755371094,
"logps/rejected": -470.8364562988281,
"loss": 4.6659,
"margin_dpo/margin_mean": 61.55085372924805,
"margin_dpo/margin_std": 103.3311767578125,
"step": 210
},
{
"KL/chosen_KL_mean": -105.573974609375,
"KL/mean": -145.5406494140625,
"KL/rejected_KL_mean": -185.50733947753906,
"KL/std": 91.00731658935547,
"epoch": 0.4418848167539267,
"fcm_dpo/beta": 0.004942171275615692,
"fcm_dpo/delta": -0.0607127919793129,
"fcm_dpo/margin": 79.93335723876953,
"fcm_dpo/q_t": 0.4069315791130066,
"grad_norm": 55.6853141784668,
"learning_rate": 3.4376480090239047e-07,
"logits/chosen": -0.8768879771232605,
"logits/rejected": -0.8537665605545044,
"logps/chosen": -381.9968566894531,
"logps/ref_chosen": -276.4228515625,
"logps/ref_rejected": -252.40603637695312,
"logps/rejected": -437.913330078125,
"loss": 4.3436,
"margin_dpo/margin_mean": 79.93335723876953,
"margin_dpo/margin_std": 91.45414733886719,
"step": 211
},
{
"KL/chosen_KL_mean": -121.54786682128906,
"KL/mean": -145.3662109375,
"KL/rejected_KL_mean": -169.1845703125,
"KL/std": 86.90359497070312,
"epoch": 0.44397905759162304,
"fcm_dpo/beta": 0.004877158906310797,
"fcm_dpo/delta": 0.010021396912634373,
"fcm_dpo/margin": 47.63668441772461,
"fcm_dpo/q_t": 0.4448275566101074,
"grad_norm": 75.4878158569336,
"learning_rate": 3.4206518122800055e-07,
"logits/chosen": -0.8545178174972534,
"logits/rejected": -0.8506518602371216,
"logps/chosen": -393.25341796875,
"logps/ref_chosen": -271.7055358886719,
"logps/ref_rejected": -241.18511962890625,
"logps/rejected": -410.36968994140625,
"loss": 4.9094,
"margin_dpo/margin_mean": 47.636680603027344,
"margin_dpo/margin_std": 95.85389709472656,
"step": 212
},
{
"KL/chosen_KL_mean": -122.48336029052734,
"KL/mean": -149.56399536132812,
"KL/rejected_KL_mean": -176.64462280273438,
"KL/std": 102.08607482910156,
"epoch": 0.44607329842931936,
"fcm_dpo/beta": 0.004948125220835209,
"fcm_dpo/delta": 0.013094035908579826,
"fcm_dpo/margin": 54.16126251220703,
"fcm_dpo/q_t": 0.43963873386383057,
"grad_norm": 72.34474182128906,
"learning_rate": 3.403606243773448e-07,
"logits/chosen": -0.8604366779327393,
"logits/rejected": -0.8748980760574341,
"logps/chosen": -424.781005859375,
"logps/ref_chosen": -302.2976379394531,
"logps/ref_rejected": -303.6202087402344,
"logps/rejected": -480.2647705078125,
"loss": 4.8561,
"margin_dpo/margin_mean": 54.16126251220703,
"margin_dpo/margin_std": 113.01664733886719,
"step": 213
},
{
"KL/chosen_KL_mean": -131.97894287109375,
"KL/mean": -166.79592895507812,
"KL/rejected_KL_mean": -201.61289978027344,
"KL/std": 91.53814697265625,
"epoch": 0.4481675392670157,
"fcm_dpo/beta": 0.004879150539636612,
"fcm_dpo/delta": -0.0352763831615448,
"fcm_dpo/margin": 69.63395690917969,
"fcm_dpo/q_t": 0.42116767168045044,
"grad_norm": 63.54450988769531,
"learning_rate": 3.3865122176063385e-07,
"logits/chosen": -0.8635294437408447,
"logits/rejected": -0.8634909391403198,
"logps/chosen": -404.111572265625,
"logps/ref_chosen": -272.13262939453125,
"logps/ref_rejected": -294.82354736328125,
"logps/rejected": -496.4364013671875,
"loss": 4.5288,
"margin_dpo/margin_mean": 69.63395690917969,
"margin_dpo/margin_std": 93.9524154663086,
"step": 214
},
{
"KL/chosen_KL_mean": -138.74960327148438,
"KL/mean": -161.1497039794922,
"KL/rejected_KL_mean": -183.54978942871094,
"KL/std": 103.77374267578125,
"epoch": 0.450261780104712,
"fcm_dpo/beta": 0.004789126571267843,
"fcm_dpo/delta": -0.021435966715216637,
"fcm_dpo/margin": 44.8001823425293,
"fcm_dpo/q_t": 0.45110857486724854,
"grad_norm": 62.86751174926758,
"learning_rate": 3.3693706504794243e-07,
"logits/chosen": -0.9035426378250122,
"logits/rejected": -0.8817762136459351,
"logps/chosen": -430.1279296875,
"logps/ref_chosen": -291.3782958984375,
"logps/ref_rejected": -261.05792236328125,
"logps/rejected": -444.60772705078125,
"loss": 5.0051,
"margin_dpo/margin_mean": 44.80018997192383,
"margin_dpo/margin_std": 105.21973419189453,
"step": 215
},
{
"KL/chosen_KL_mean": -127.76866149902344,
"KL/mean": -159.33363342285156,
"KL/rejected_KL_mean": -190.89862060546875,
"KL/std": 96.72543334960938,
"epoch": 0.4523560209424084,
"fcm_dpo/beta": 0.004820346832275391,
"fcm_dpo/delta": 0.024416334927082062,
"fcm_dpo/margin": 63.12995529174805,
"fcm_dpo/q_t": 0.4305679202079773,
"grad_norm": 56.78171920776367,
"learning_rate": 3.3521824616429284e-07,
"logits/chosen": -0.9260469675064087,
"logits/rejected": -0.9183626174926758,
"logps/chosen": -466.27410888671875,
"logps/ref_chosen": -338.50543212890625,
"logps/ref_rejected": -305.76104736328125,
"logps/rejected": -496.6596374511719,
"loss": 4.7264,
"margin_dpo/margin_mean": 63.12995147705078,
"margin_dpo/margin_std": 113.52439880371094,
"step": 216
},
{
"KL/chosen_KL_mean": -135.17062377929688,
"KL/mean": -173.59078979492188,
"KL/rejected_KL_mean": -212.01097106933594,
"KL/std": 99.70831298828125,
"epoch": 0.4544502617801047,
"fcm_dpo/beta": 0.004769713152199984,
"fcm_dpo/delta": -0.017181845381855965,
"fcm_dpo/margin": 76.84034729003906,
"fcm_dpo/q_t": 0.4159371256828308,
"grad_norm": 62.81724548339844,
"learning_rate": 3.334948572847253e-07,
"logits/chosen": -0.836664617061615,
"logits/rejected": -0.7955836653709412,
"logps/chosen": -428.720458984375,
"logps/ref_chosen": -293.5498046875,
"logps/ref_rejected": -256.7830810546875,
"logps/rejected": -468.7940673828125,
"loss": 4.5008,
"margin_dpo/margin_mean": 76.84033966064453,
"margin_dpo/margin_std": 112.65450286865234,
"step": 217
},
{
"KL/chosen_KL_mean": -133.9468231201172,
"KL/mean": -168.5343780517578,
"KL/rejected_KL_mean": -203.12193298339844,
"KL/std": 99.32330322265625,
"epoch": 0.45654450261780105,
"fcm_dpo/beta": 0.0047028930857777596,
"fcm_dpo/delta": -0.024641798809170723,
"fcm_dpo/margin": 69.17512512207031,
"fcm_dpo/q_t": 0.4250248372554779,
"grad_norm": 64.38352966308594,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": -0.862998902797699,
"logits/rejected": -0.8793630003929138,
"logps/chosen": -454.5261535644531,
"logps/ref_chosen": -320.579345703125,
"logps/ref_rejected": -294.0381164550781,
"logps/rejected": -497.1600646972656,
"loss": 4.615,
"margin_dpo/margin_mean": 69.17512512207031,
"margin_dpo/margin_std": 110.75273895263672,
"step": 218
},
{
"KL/chosen_KL_mean": -132.96096801757812,
"KL/mean": -169.79696655273438,
"KL/rejected_KL_mean": -206.63299560546875,
"KL/std": 95.11730194091797,
"epoch": 0.4586387434554974,
"fcm_dpo/beta": 0.00460631912574172,
"fcm_dpo/delta": -0.01590793766081333,
"fcm_dpo/margin": 73.6720199584961,
"fcm_dpo/q_t": 0.4217400550842285,
"grad_norm": 67.673828125,
"learning_rate": 3.300347394584172e-07,
"logits/chosen": -0.839383065700531,
"logits/rejected": -0.8644520044326782,
"logps/chosen": -401.3796081542969,
"logps/ref_chosen": -268.4186096191406,
"logps/ref_rejected": -265.7808837890625,
"logps/rejected": -472.413818359375,
"loss": 4.5894,
"margin_dpo/margin_mean": 73.6720199584961,
"margin_dpo/margin_std": 113.50521850585938,
"step": 219
},
{
"KL/chosen_KL_mean": -137.07662963867188,
"KL/mean": -173.6854705810547,
"KL/rejected_KL_mean": -210.2943115234375,
"KL/std": 93.0467300415039,
"epoch": 0.4607329842931937,
"fcm_dpo/beta": 0.004609918221831322,
"fcm_dpo/delta": 0.0026855701580643654,
"fcm_dpo/margin": 73.21768188476562,
"fcm_dpo/q_t": 0.42106613516807556,
"grad_norm": 70.86668395996094,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": -0.9124230742454529,
"logits/rejected": -0.8802636861801147,
"logps/chosen": -449.9631042480469,
"logps/ref_chosen": -312.8864440917969,
"logps/ref_rejected": -259.5191955566406,
"logps/rejected": -469.8135070800781,
"loss": 4.5431,
"margin_dpo/margin_mean": 73.21768188476562,
"margin_dpo/margin_std": 104.53365325927734,
"step": 220
},
{
"KL/chosen_KL_mean": -149.1865692138672,
"KL/mean": -178.74742126464844,
"KL/rejected_KL_mean": -208.3082733154297,
"KL/std": 99.30208587646484,
"epoch": 0.46282722513089003,
"fcm_dpo/beta": 0.004629157017916441,
"fcm_dpo/delta": -0.01543284673243761,
"fcm_dpo/margin": 59.121707916259766,
"fcm_dpo/q_t": 0.43726086616516113,
"grad_norm": 62.723228454589844,
"learning_rate": 3.265574537815398e-07,
"logits/chosen": -0.8228099942207336,
"logits/rejected": -0.8340501189231873,
"logps/chosen": -449.5124206542969,
"logps/ref_chosen": -300.32586669921875,
"logps/ref_rejected": -286.312255859375,
"logps/rejected": -494.62054443359375,
"loss": 4.8241,
"margin_dpo/margin_mean": 59.12171173095703,
"margin_dpo/margin_std": 115.42534637451172,
"step": 221
},
{
"KL/chosen_KL_mean": -143.39520263671875,
"KL/mean": -173.56765747070312,
"KL/rejected_KL_mean": -203.74014282226562,
"KL/std": 89.37479400634766,
"epoch": 0.4649214659685864,
"fcm_dpo/beta": 0.004566497169435024,
"fcm_dpo/delta": -0.0049129268154501915,
"fcm_dpo/margin": 60.34489822387695,
"fcm_dpo/q_t": 0.4353586733341217,
"grad_norm": 55.062538146972656,
"learning_rate": 3.248126059518784e-07,
"logits/chosen": -0.9121435880661011,
"logits/rejected": -0.8791247606277466,
"logps/chosen": -440.506591796875,
"logps/ref_chosen": -297.1113586425781,
"logps/ref_rejected": -235.53146362304688,
"logps/rejected": -439.2715759277344,
"loss": 4.7509,
"margin_dpo/margin_mean": 60.34489822387695,
"margin_dpo/margin_std": 104.17533874511719,
"step": 222
},
{
"KL/chosen_KL_mean": -149.42726135253906,
"KL/mean": -182.06930541992188,
"KL/rejected_KL_mean": -214.7113494873047,
"KL/std": 97.39741516113281,
"epoch": 0.46701570680628274,
"fcm_dpo/beta": 0.004558564629405737,
"fcm_dpo/delta": 0.011139356531202793,
"fcm_dpo/margin": 65.28407287597656,
"fcm_dpo/q_t": 0.4302670955657959,
"grad_norm": 64.61984252929688,
"learning_rate": 3.230637461492043e-07,
"logits/chosen": -0.8495522737503052,
"logits/rejected": -0.8148607611656189,
"logps/chosen": -435.8423767089844,
"logps/ref_chosen": -286.41510009765625,
"logps/ref_rejected": -241.1181640625,
"logps/rejected": -455.82952880859375,
"loss": 4.7017,
"margin_dpo/margin_mean": 65.28407287597656,
"margin_dpo/margin_std": 111.41921997070312,
"step": 223
},
{
"KL/chosen_KL_mean": -136.95799255371094,
"KL/mean": -172.8470916748047,
"KL/rejected_KL_mean": -208.7362060546875,
"KL/std": 96.30474090576172,
"epoch": 0.46910994764397906,
"fcm_dpo/beta": 0.004641786217689514,
"fcm_dpo/delta": 0.006216204725205898,
"fcm_dpo/margin": 71.77820587158203,
"fcm_dpo/q_t": 0.4229627847671509,
"grad_norm": 65.5974349975586,
"learning_rate": 3.213109681595612e-07,
"logits/chosen": -0.8177117705345154,
"logits/rejected": -0.8344778418540955,
"logps/chosen": -386.4503173828125,
"logps/ref_chosen": -249.49234008789062,
"logps/ref_rejected": -233.10752868652344,
"logps/rejected": -441.84375,
"loss": 4.5747,
"margin_dpo/margin_mean": 71.77820587158203,
"margin_dpo/margin_std": 106.76852416992188,
"step": 224
},
{
"KL/chosen_KL_mean": -156.29876708984375,
"KL/mean": -189.03250122070312,
"KL/rejected_KL_mean": -221.76626586914062,
"KL/std": 103.037841796875,
"epoch": 0.4712041884816754,
"fcm_dpo/beta": 0.004633888602256775,
"fcm_dpo/delta": 0.014746258035302162,
"fcm_dpo/margin": 65.46749877929688,
"fcm_dpo/q_t": 0.4310811161994934,
"grad_norm": 77.01375579833984,
"learning_rate": 3.1955436597911315e-07,
"logits/chosen": -0.8554367423057556,
"logits/rejected": -0.8290306925773621,
"logps/chosen": -468.1571350097656,
"logps/ref_chosen": -311.8583679199219,
"logps/ref_rejected": -336.8523864746094,
"logps/rejected": -558.61865234375,
"loss": 4.7305,
"margin_dpo/margin_mean": 65.46749877929688,
"margin_dpo/margin_std": 119.62830352783203,
"step": 225
},
{
"KL/chosen_KL_mean": -141.25518798828125,
"KL/mean": -169.96444702148438,
"KL/rejected_KL_mean": -198.6737060546875,
"KL/std": 97.53821563720703,
"epoch": 0.4732984293193717,
"fcm_dpo/beta": 0.0046873511746525764,
"fcm_dpo/delta": 0.0078051392920315266,
"fcm_dpo/margin": 57.41852569580078,
"fcm_dpo/q_t": 0.43802326917648315,
"grad_norm": 74.30442810058594,
"learning_rate": 3.1779403380910425e-07,
"logits/chosen": -0.876971423625946,
"logits/rejected": -0.864268958568573,
"logps/chosen": -393.4564514160156,
"logps/ref_chosen": -252.20123291015625,
"logps/ref_rejected": -254.41162109375,
"logps/rejected": -453.0853576660156,
"loss": 4.8147,
"margin_dpo/margin_mean": 57.418521881103516,
"margin_dpo/margin_std": 112.63554382324219,
"step": 226
},
{
"KL/chosen_KL_mean": -135.2330780029297,
"KL/mean": -172.74185180664062,
"KL/rejected_KL_mean": -210.25062561035156,
"KL/std": 94.74110412597656,
"epoch": 0.47539267015706804,
"fcm_dpo/beta": 0.004633747041225433,
"fcm_dpo/delta": -0.023657258599996567,
"fcm_dpo/margin": 75.01754760742188,
"fcm_dpo/q_t": 0.4188823401927948,
"grad_norm": 61.293704986572266,
"learning_rate": 3.160300660508064e-07,
"logits/chosen": -0.8379367589950562,
"logits/rejected": -0.8345897197723389,
"logps/chosen": -420.4925231933594,
"logps/ref_chosen": -285.25946044921875,
"logps/ref_rejected": -261.3220520019531,
"logps/rejected": -471.57269287109375,
"loss": 4.559,
"margin_dpo/margin_mean": 75.01754760742188,
"margin_dpo/margin_std": 114.54286193847656,
"step": 227
},
{
"KL/chosen_KL_mean": -120.16371154785156,
"KL/mean": -156.32989501953125,
"KL/rejected_KL_mean": -192.4960479736328,
"KL/std": 90.50350189208984,
"epoch": 0.4774869109947644,
"fcm_dpo/beta": 0.004585505463182926,
"fcm_dpo/delta": -0.025612030178308487,
"fcm_dpo/margin": 72.33232879638672,
"fcm_dpo/q_t": 0.4227723479270935,
"grad_norm": 55.890357971191406,
"learning_rate": 3.1426255730045695e-07,
"logits/chosen": -0.8586771488189697,
"logits/rejected": -0.8261110782623291,
"logps/chosen": -433.9825439453125,
"logps/ref_chosen": -313.81878662109375,
"logps/ref_rejected": -258.07061767578125,
"logps/rejected": -450.56671142578125,
"loss": 4.5548,
"margin_dpo/margin_mean": 72.33232879638672,
"margin_dpo/margin_std": 101.38350677490234,
"step": 228
},
{
"KL/chosen_KL_mean": -135.5593719482422,
"KL/mean": -177.83876037597656,
"KL/rejected_KL_mean": -220.11810302734375,
"KL/std": 98.42047119140625,
"epoch": 0.47958115183246075,
"fcm_dpo/beta": 0.004426400177180767,
"fcm_dpo/delta": -0.029844271019101143,
"fcm_dpo/margin": 84.5587158203125,
"fcm_dpo/q_t": 0.41256657242774963,
"grad_norm": 52.477874755859375,
"learning_rate": 3.1249160234418644e-07,
"logits/chosen": -0.8296700716018677,
"logits/rejected": -0.8337705135345459,
"logps/chosen": -427.5301513671875,
"logps/ref_chosen": -291.9707946777344,
"logps/ref_rejected": -263.42059326171875,
"logps/rejected": -483.5386657714844,
"loss": 4.4323,
"margin_dpo/margin_mean": 84.55870819091797,
"margin_dpo/margin_std": 109.96800994873047,
"step": 229
},
{
"KL/chosen_KL_mean": -121.74411010742188,
"KL/mean": -160.2207489013672,
"KL/rejected_KL_mean": -198.69741821289062,
"KL/std": 105.12950897216797,
"epoch": 0.4816753926701571,
"fcm_dpo/beta": 0.004394059535115957,
"fcm_dpo/delta": 0.0015674427850171924,
"fcm_dpo/margin": 76.95330047607422,
"fcm_dpo/q_t": 0.4227007031440735,
"grad_norm": 66.97175598144531,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": -0.8868385553359985,
"logits/rejected": -0.8867266178131104,
"logps/chosen": -355.00421142578125,
"logps/ref_chosen": -233.2601318359375,
"logps/ref_rejected": -238.922119140625,
"logps/rejected": -437.6195373535156,
"loss": 4.5802,
"margin_dpo/margin_mean": 76.95330047607422,
"margin_dpo/margin_std": 119.8840560913086,
"step": 230
},
{
"KL/chosen_KL_mean": -133.48675537109375,
"KL/mean": -164.26443481445312,
"KL/rejected_KL_mean": -195.0421142578125,
"KL/std": 91.5148696899414,
"epoch": 0.4837696335078534,
"fcm_dpo/beta": 0.00437714671716094,
"fcm_dpo/delta": -0.008870087563991547,
"fcm_dpo/margin": 61.55538558959961,
"fcm_dpo/q_t": 0.4362969696521759,
"grad_norm": 66.45602416992188,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": -0.8645358681678772,
"logits/rejected": -0.8530266880989075,
"logps/chosen": -455.64190673828125,
"logps/ref_chosen": -322.1551818847656,
"logps/ref_rejected": -280.97613525390625,
"logps/rejected": -476.01824951171875,
"loss": 4.7541,
"margin_dpo/margin_mean": 61.55538558959961,
"margin_dpo/margin_std": 102.5092544555664,
"step": 231
},
{
"KL/chosen_KL_mean": -132.1063995361328,
"KL/mean": -162.6656951904297,
"KL/rejected_KL_mean": -193.22494506835938,
"KL/std": 99.24211120605469,
"epoch": 0.48586387434554973,
"fcm_dpo/beta": 0.004379937890917063,
"fcm_dpo/delta": -0.0022955876775085926,
"fcm_dpo/margin": 61.11854934692383,
"fcm_dpo/q_t": 0.43785691261291504,
"grad_norm": 64.07685852050781,
"learning_rate": 3.071590108427243e-07,
"logits/chosen": -0.8338838219642639,
"logits/rejected": -0.8158622980117798,
"logps/chosen": -403.8501892089844,
"logps/ref_chosen": -271.7437744140625,
"logps/ref_rejected": -249.94981384277344,
"logps/rejected": -443.17474365234375,
"loss": 4.8322,
"margin_dpo/margin_mean": 61.11854934692383,
"margin_dpo/margin_std": 115.72997283935547,
"step": 232
},
{
"KL/chosen_KL_mean": -122.5298080444336,
"KL/mean": -161.3760986328125,
"KL/rejected_KL_mean": -200.22239685058594,
"KL/std": 95.90873718261719,
"epoch": 0.48795811518324606,
"fcm_dpo/beta": 0.004333424381911755,
"fcm_dpo/delta": -0.03165815770626068,
"fcm_dpo/margin": 77.6926040649414,
"fcm_dpo/q_t": 0.42159727215766907,
"grad_norm": 53.214111328125,
"learning_rate": 3.05375222543809e-07,
"logits/chosen": -0.8735519051551819,
"logits/rejected": -0.8635944128036499,
"logps/chosen": -407.87213134765625,
"logps/ref_chosen": -285.3423156738281,
"logps/ref_rejected": -266.34320068359375,
"logps/rejected": -466.56561279296875,
"loss": 4.5516,
"margin_dpo/margin_mean": 77.69261169433594,
"margin_dpo/margin_std": 106.49396514892578,
"step": 233
},
{
"KL/chosen_KL_mean": -126.59469604492188,
"KL/mean": -158.7538604736328,
"KL/rejected_KL_mean": -190.9130859375,
"KL/std": 95.13108825683594,
"epoch": 0.4900523560209424,
"fcm_dpo/beta": 0.004321113228797913,
"fcm_dpo/delta": 0.043769948184490204,
"fcm_dpo/margin": 64.3183822631836,
"fcm_dpo/q_t": 0.43559810519218445,
"grad_norm": 65.22840118408203,
"learning_rate": 3.035884646397637e-07,
"logits/chosen": -0.8416929244995117,
"logits/rejected": -0.829358696937561,
"logps/chosen": -421.5003662109375,
"logps/ref_chosen": -294.9057312011719,
"logps/ref_rejected": -299.37054443359375,
"logps/rejected": -490.28363037109375,
"loss": 4.7916,
"margin_dpo/margin_mean": 64.3183822631836,
"margin_dpo/margin_std": 121.3981704711914,
"step": 234
},
{
"KL/chosen_KL_mean": -123.97207641601562,
"KL/mean": -159.31024169921875,
"KL/rejected_KL_mean": -194.6483917236328,
"KL/std": 103.93309020996094,
"epoch": 0.49214659685863876,
"fcm_dpo/beta": 0.004475614987313747,
"fcm_dpo/delta": 0.024762175977230072,
"fcm_dpo/margin": 70.67631530761719,
"fcm_dpo/q_t": 0.42616766691207886,
"grad_norm": 74.38615417480469,
"learning_rate": 3.017988329489923e-07,
"logits/chosen": -0.864960789680481,
"logits/rejected": -0.8573201894760132,
"logps/chosen": -413.46966552734375,
"logps/ref_chosen": -289.49755859375,
"logps/ref_rejected": -247.55076599121094,
"logps/rejected": -442.1991882324219,
"loss": 4.6269,
"margin_dpo/margin_mean": 70.67632293701172,
"margin_dpo/margin_std": 112.42530822753906,
"step": 235
},
{
"KL/chosen_KL_mean": -116.880126953125,
"KL/mean": -151.97642517089844,
"KL/rejected_KL_mean": -187.0727081298828,
"KL/std": 98.38629913330078,
"epoch": 0.4942408376963351,
"fcm_dpo/beta": 0.004550088196992874,
"fcm_dpo/delta": 0.01439427025616169,
"fcm_dpo/margin": 70.19258880615234,
"fcm_dpo/q_t": 0.4265892505645752,
"grad_norm": 86.40446472167969,
"learning_rate": 3.000064234440111e-07,
"logits/chosen": -0.8829069137573242,
"logits/rejected": -0.8841407895088196,
"logps/chosen": -405.7647399902344,
"logps/ref_chosen": -288.8846435546875,
"logps/ref_rejected": -242.0452880859375,
"logps/rejected": -429.11798095703125,
"loss": 4.6436,
"margin_dpo/margin_mean": 70.19258880615234,
"margin_dpo/margin_std": 115.69688415527344,
"step": 236
},
{
"KL/chosen_KL_mean": -123.7160873413086,
"KL/mean": -159.17050170898438,
"KL/rejected_KL_mean": -194.62493896484375,
"KL/std": 86.97819519042969,
"epoch": 0.4963350785340314,
"fcm_dpo/beta": 0.0045979805290699005,
"fcm_dpo/delta": 0.013288134709000587,
"fcm_dpo/margin": 70.9088363647461,
"fcm_dpo/q_t": 0.42442870140075684,
"grad_norm": 69.7810287475586,
"learning_rate": 2.9821133224630223e-07,
"logits/chosen": -0.8455889821052551,
"logits/rejected": -0.8238015174865723,
"logps/chosen": -389.19482421875,
"logps/ref_chosen": -265.47869873046875,
"logps/ref_rejected": -267.9891357421875,
"logps/rejected": -462.61407470703125,
"loss": 4.5715,
"margin_dpo/margin_mean": 70.90882873535156,
"margin_dpo/margin_std": 103.5340576171875,
"step": 237
},
{
"KL/chosen_KL_mean": -132.91195678710938,
"KL/mean": -167.68841552734375,
"KL/rejected_KL_mean": -202.46484375,
"KL/std": 96.82723236083984,
"epoch": 0.49842931937172774,
"fcm_dpo/beta": 0.004613780416548252,
"fcm_dpo/delta": 0.0007776557467877865,
"fcm_dpo/margin": 69.55287170410156,
"fcm_dpo/q_t": 0.42712903022766113,
"grad_norm": 51.249977111816406,
"learning_rate": 2.964136556211588e-07,
"logits/chosen": -0.853753924369812,
"logits/rejected": -0.8224870562553406,
"logps/chosen": -444.91461181640625,
"logps/ref_chosen": -312.0026550292969,
"logps/ref_rejected": -270.0257263183594,
"logps/rejected": -472.4905700683594,
"loss": 4.6223,
"margin_dpo/margin_mean": 69.5528793334961,
"margin_dpo/margin_std": 110.1138687133789,
"step": 238
},
{
"KL/chosen_KL_mean": -122.88169860839844,
"KL/mean": -154.25526428222656,
"KL/rejected_KL_mean": -185.62881469726562,
"KL/std": 101.83222961425781,
"epoch": 0.5005235602094241,
"fcm_dpo/beta": 0.004658076446503401,
"fcm_dpo/delta": 0.014860610477626324,
"fcm_dpo/margin": 62.74710464477539,
"fcm_dpo/q_t": 0.4337066411972046,
"grad_norm": 62.06940460205078,
"learning_rate": 2.946134899725226e-07,
"logits/chosen": -0.8369898796081543,
"logits/rejected": -0.876994788646698,
"logps/chosen": -390.0489501953125,
"logps/ref_chosen": -267.167236328125,
"logps/ref_rejected": -275.99468994140625,
"logps/rejected": -461.6235046386719,
"loss": 4.8319,
"margin_dpo/margin_mean": 62.74710464477539,
"margin_dpo/margin_std": 131.92681884765625,
"step": 239
},
{
"KL/chosen_KL_mean": -123.39981079101562,
"KL/mean": -161.76980590820312,
"KL/rejected_KL_mean": -200.13980102539062,
"KL/std": 98.65313720703125,
"epoch": 0.5026178010471204,
"fcm_dpo/beta": 0.004674379248172045,
"fcm_dpo/delta": 0.002928507514297962,
"fcm_dpo/margin": 76.73999786376953,
"fcm_dpo/q_t": 0.41863059997558594,
"grad_norm": 49.32747268676758,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": -0.9084842801094055,
"logits/rejected": -0.9017030596733093,
"logps/chosen": -409.3794250488281,
"logps/ref_chosen": -285.9796142578125,
"logps/ref_rejected": -256.8258056640625,
"logps/rejected": -456.9656066894531,
"loss": 4.5228,
"margin_dpo/margin_mean": 76.73999786376953,
"margin_dpo/margin_std": 116.18621826171875,
"step": 240
},
{
"KL/chosen_KL_mean": -143.24009704589844,
"KL/mean": -172.29185485839844,
"KL/rejected_KL_mean": -201.3436279296875,
"KL/std": 106.31816101074219,
"epoch": 0.5047120418848168,
"fcm_dpo/beta": 0.004635634366422892,
"fcm_dpo/delta": -0.013472635298967361,
"fcm_dpo/margin": 58.10354995727539,
"fcm_dpo/q_t": 0.43844515085220337,
"grad_norm": 55.63737869262695,
"learning_rate": 2.910060778827554e-07,
"logits/chosen": -0.8202676773071289,
"logits/rejected": -0.7962172031402588,
"logps/chosen": -404.7569580078125,
"logps/ref_chosen": -261.516845703125,
"logps/ref_rejected": -250.2250518798828,
"logps/rejected": -451.5686950683594,
"loss": 4.8474,
"margin_dpo/margin_mean": 58.10354995727539,
"margin_dpo/margin_std": 116.29940032958984,
"step": 241
},
{
"KL/chosen_KL_mean": -130.06700134277344,
"KL/mean": -162.6671600341797,
"KL/rejected_KL_mean": -195.267333984375,
"KL/std": 102.92440795898438,
"epoch": 0.506806282722513,
"fcm_dpo/beta": 0.004585296381264925,
"fcm_dpo/delta": 0.0017472277395427227,
"fcm_dpo/margin": 65.20032501220703,
"fcm_dpo/q_t": 0.43209636211395264,
"grad_norm": 80.01802062988281,
"learning_rate": 2.891990248961871e-07,
"logits/chosen": -0.8808521032333374,
"logits/rejected": -0.8558878302574158,
"logps/chosen": -400.5809631347656,
"logps/ref_chosen": -270.51397705078125,
"logps/ref_rejected": -244.8560791015625,
"logps/rejected": -440.1234130859375,
"loss": 4.6885,
"margin_dpo/margin_mean": 65.20032501220703,
"margin_dpo/margin_std": 109.88291931152344,
"step": 242
},
{
"KL/chosen_KL_mean": -125.00543212890625,
"KL/mean": -165.06704711914062,
"KL/rejected_KL_mean": -205.12864685058594,
"KL/std": 104.33854675292969,
"epoch": 0.5089005235602094,
"fcm_dpo/beta": 0.004604042042046785,
"fcm_dpo/delta": -0.03992704302072525,
"fcm_dpo/margin": 80.12322998046875,
"fcm_dpo/q_t": 0.41883641481399536,
"grad_norm": 76.32345581054688,
"learning_rate": 2.873898697848762e-07,
"logits/chosen": -0.8886011838912964,
"logits/rejected": -0.8729423880577087,
"logps/chosen": -449.6875,
"logps/ref_chosen": -324.68206787109375,
"logps/ref_rejected": -307.1111755371094,
"logps/rejected": -512.2398071289062,
"loss": 4.5412,
"margin_dpo/margin_mean": 80.12322998046875,
"margin_dpo/margin_std": 127.59658813476562,
"step": 243
},
{
"KL/chosen_KL_mean": -128.8868408203125,
"KL/mean": -168.5599365234375,
"KL/rejected_KL_mean": -208.23304748535156,
"KL/std": 96.4882583618164,
"epoch": 0.5109947643979058,
"fcm_dpo/beta": 0.004474720451980829,
"fcm_dpo/delta": 0.001254271250218153,
"fcm_dpo/margin": 79.34623718261719,
"fcm_dpo/q_t": 0.41681310534477234,
"grad_norm": 47.35638427734375,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": -0.8795413970947266,
"logits/rejected": -0.8352590203285217,
"logps/chosen": -447.8660583496094,
"logps/ref_chosen": -318.979248046875,
"logps/ref_rejected": -269.67572021484375,
"logps/rejected": -477.90875244140625,
"loss": 4.5117,
"margin_dpo/margin_mean": 79.34623718261719,
"margin_dpo/margin_std": 108.80554962158203,
"step": 244
},
{
"KL/chosen_KL_mean": -125.91683197021484,
"KL/mean": -165.79470825195312,
"KL/rejected_KL_mean": -205.67257690429688,
"KL/std": 94.03182220458984,
"epoch": 0.5130890052356021,
"fcm_dpo/beta": 0.004462458658963442,
"fcm_dpo/delta": -0.01995830610394478,
"fcm_dpo/margin": 79.75574493408203,
"fcm_dpo/q_t": 0.4166679382324219,
"grad_norm": 62.6284065246582,
"learning_rate": 2.837656413735479e-07,
"logits/chosen": -0.8734516501426697,
"logits/rejected": -0.879128634929657,
"logps/chosen": -420.81488037109375,
"logps/ref_chosen": -294.8980712890625,
"logps/ref_rejected": -239.8111114501953,
"logps/rejected": -445.483642578125,
"loss": 4.4922,
"margin_dpo/margin_mean": 79.7557373046875,
"margin_dpo/margin_std": 107.47772979736328,
"step": 245
},
{
"KL/chosen_KL_mean": -151.3753204345703,
"KL/mean": -178.75440979003906,
"KL/rejected_KL_mean": -206.1334686279297,
"KL/std": 95.8738021850586,
"epoch": 0.5151832460732985,
"fcm_dpo/beta": 0.004442715086042881,
"fcm_dpo/delta": 0.012672241777181625,
"fcm_dpo/margin": 54.75814437866211,
"fcm_dpo/q_t": 0.4447598457336426,
"grad_norm": 64.96891021728516,
"learning_rate": 2.8195076242990116e-07,
"logits/chosen": -0.8741089701652527,
"logits/rejected": -0.8788989186286926,
"logps/chosen": -432.06072998046875,
"logps/ref_chosen": -280.6854248046875,
"logps/ref_rejected": -253.65382385253906,
"logps/rejected": -459.78729248046875,
"loss": 4.9389,
"margin_dpo/margin_mean": 54.75814437866211,
"margin_dpo/margin_std": 124.61536407470703,
"step": 246
},
{
"KL/chosen_KL_mean": -139.81202697753906,
"KL/mean": -175.6849365234375,
"KL/rejected_KL_mean": -211.55783081054688,
"KL/std": 100.8326416015625,
"epoch": 0.5172774869109947,
"fcm_dpo/beta": 0.0044505903497338295,
"fcm_dpo/delta": -0.016747113317251205,
"fcm_dpo/margin": 71.74579620361328,
"fcm_dpo/q_t": 0.4267142415046692,
"grad_norm": 47.79685592651367,
"learning_rate": 2.801341700638307e-07,
"logits/chosen": -0.8639557361602783,
"logits/rejected": -0.861725926399231,
"logps/chosen": -420.9211120605469,
"logps/ref_chosen": -281.1091003417969,
"logps/ref_rejected": -260.3700866699219,
"logps/rejected": -471.92791748046875,
"loss": 4.6184,
"margin_dpo/margin_mean": 71.74579620361328,
"margin_dpo/margin_std": 109.62867736816406,
"step": 247
},
{
"KL/chosen_KL_mean": -145.56988525390625,
"KL/mean": -179.25965881347656,
"KL/rejected_KL_mean": -212.94943237304688,
"KL/std": 101.54124450683594,
"epoch": 0.5193717277486911,
"fcm_dpo/beta": 0.004346329253166914,
"fcm_dpo/delta": -0.013487712480127811,
"fcm_dpo/margin": 67.37956237792969,
"fcm_dpo/q_t": 0.4315546154975891,
"grad_norm": 69.30085754394531,
"learning_rate": 2.7831596169367227e-07,
"logits/chosen": -0.8273190259933472,
"logits/rejected": -0.8335063457489014,
"logps/chosen": -415.88824462890625,
"logps/ref_chosen": -270.318359375,
"logps/ref_rejected": -233.46778869628906,
"logps/rejected": -446.417236328125,
"loss": 4.7203,
"margin_dpo/margin_mean": 67.37956237792969,
"margin_dpo/margin_std": 112.68716430664062,
"step": 248
},
{
"KL/chosen_KL_mean": -163.27923583984375,
"KL/mean": -192.41412353515625,
"KL/rejected_KL_mean": -221.5490264892578,
"KL/std": 105.23775482177734,
"epoch": 0.5214659685863874,
"fcm_dpo/beta": 0.00435510091483593,
"fcm_dpo/delta": -0.0018205934902653098,
"fcm_dpo/margin": 58.26978302001953,
"fcm_dpo/q_t": 0.4405788779258728,
"grad_norm": 58.962860107421875,
"learning_rate": 2.7649623482442274e-07,
"logits/chosen": -0.859215497970581,
"logits/rejected": -0.8319091200828552,
"logps/chosen": -439.08807373046875,
"logps/ref_chosen": -275.8088684082031,
"logps/ref_rejected": -243.45138549804688,
"logps/rejected": -465.0003967285156,
"loss": 4.9087,
"margin_dpo/margin_mean": 58.26978302001953,
"margin_dpo/margin_std": 128.27325439453125,
"step": 249
},
{
"KL/chosen_KL_mean": -163.93161010742188,
"KL/mean": -204.99388122558594,
"KL/rejected_KL_mean": -246.05612182617188,
"KL/std": 120.12943267822266,
"epoch": 0.5235602094240838,
"fcm_dpo/beta": 0.004266043193638325,
"fcm_dpo/delta": -0.04007497429847717,
"fcm_dpo/margin": 82.12449645996094,
"fcm_dpo/q_t": 0.41968196630477905,
"grad_norm": 79.0232925415039,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": -0.8501719236373901,
"logits/rejected": -0.8420849442481995,
"logps/chosen": -456.4261474609375,
"logps/ref_chosen": -292.4945373535156,
"logps/ref_rejected": -284.2869567871094,
"logps/rejected": -530.3430786132812,
"loss": 4.6,
"margin_dpo/margin_mean": 82.1244888305664,
"margin_dpo/margin_std": 134.7095947265625,
"step": 250
},
{
"KL/chosen_KL_mean": -151.3125,
"KL/mean": -189.22445678710938,
"KL/rejected_KL_mean": -227.1363983154297,
"KL/std": 116.46218872070312,
"epoch": 0.5256544502617801,
"fcm_dpo/beta": 0.0041464585810899734,
"fcm_dpo/delta": -0.04177861660718918,
"fcm_dpo/margin": 75.82389831542969,
"fcm_dpo/q_t": 0.4313172996044159,
"grad_norm": 75.89229583740234,
"learning_rate": 2.7285261601056697e-07,
"logits/chosen": -0.8566744327545166,
"logits/rejected": -0.8396670818328857,
"logps/chosen": -433.04913330078125,
"logps/ref_chosen": -281.736572265625,
"logps/ref_rejected": -255.9419708251953,
"logps/rejected": -483.078369140625,
"loss": 4.6892,
"margin_dpo/margin_mean": 75.82389831542969,
"margin_dpo/margin_std": 130.15245056152344,
"step": 251
},
{
"KL/chosen_KL_mean": -169.72479248046875,
"KL/mean": -205.39056396484375,
"KL/rejected_KL_mean": -241.0563201904297,
"KL/std": 109.83949279785156,
"epoch": 0.5277486910994764,
"fcm_dpo/beta": 0.004018429201096296,
"fcm_dpo/delta": -0.022251684218645096,
"fcm_dpo/margin": 71.3315200805664,
"fcm_dpo/q_t": 0.4324949383735657,
"grad_norm": 50.09172058105469,
"learning_rate": 2.7102891946217994e-07,
"logits/chosen": -0.9038177132606506,
"logits/rejected": -0.8767113089561462,
"logps/chosen": -465.6922302246094,
"logps/ref_chosen": -295.9674072265625,
"logps/ref_rejected": -280.111572265625,
"logps/rejected": -521.1679077148438,
"loss": 4.7294,
"margin_dpo/margin_mean": 71.3315200805664,
"margin_dpo/margin_std": 121.27824401855469,
"step": 252
},
{
"KL/chosen_KL_mean": -177.96763610839844,
"KL/mean": -212.9547119140625,
"KL/rejected_KL_mean": -247.94180297851562,
"KL/std": 117.46932220458984,
"epoch": 0.5298429319371728,
"fcm_dpo/beta": 0.004015140701085329,
"fcm_dpo/delta": -0.0008969169575721025,
"fcm_dpo/margin": 69.97415924072266,
"fcm_dpo/q_t": 0.43637773394584656,
"grad_norm": 63.2220458984375,
"learning_rate": 2.692040951966617e-07,
"logits/chosen": -0.8821121454238892,
"logits/rejected": -0.8729538917541504,
"logps/chosen": -455.0398864746094,
"logps/ref_chosen": -277.072265625,
"logps/ref_rejected": -247.31643676757812,
"logps/rejected": -495.25823974609375,
"loss": 4.8047,
"margin_dpo/margin_mean": 69.97415924072266,
"margin_dpo/margin_std": 132.85340881347656,
"step": 253
},
{
"KL/chosen_KL_mean": -166.27993774414062,
"KL/mean": -205.32247924804688,
"KL/rejected_KL_mean": -244.36500549316406,
"KL/std": 116.28435516357422,
"epoch": 0.5319371727748691,
"fcm_dpo/beta": 0.003979699220508337,
"fcm_dpo/delta": -0.027800586074590683,
"fcm_dpo/margin": 78.0850830078125,
"fcm_dpo/q_t": 0.4289184510707855,
"grad_norm": 44.38166427612305,
"learning_rate": 2.6737824107379947e-07,
"logits/chosen": -0.8131484389305115,
"logits/rejected": -0.7929275035858154,
"logps/chosen": -436.227783203125,
"logps/ref_chosen": -269.9478454589844,
"logps/ref_rejected": -249.45005798339844,
"logps/rejected": -493.8150634765625,
"loss": 4.6575,
"margin_dpo/margin_mean": 78.0850830078125,
"margin_dpo/margin_std": 126.3444595336914,
"step": 254
},
{
"KL/chosen_KL_mean": -165.02828979492188,
"KL/mean": -207.98382568359375,
"KL/rejected_KL_mean": -250.9393768310547,
"KL/std": 116.85551452636719,
"epoch": 0.5340314136125655,
"fcm_dpo/beta": 0.003907862119376659,
"fcm_dpo/delta": -0.0021270355209708214,
"fcm_dpo/margin": 85.91107940673828,
"fcm_dpo/q_t": 0.42185455560684204,
"grad_norm": 69.25841522216797,
"learning_rate": 2.655514550086086e-07,
"logits/chosen": -0.8334129452705383,
"logits/rejected": -0.7977045178413391,
"logps/chosen": -471.6835632324219,
"logps/ref_chosen": -306.6552734375,
"logps/ref_rejected": -254.47528076171875,
"logps/rejected": -505.4146728515625,
"loss": 4.6419,
"margin_dpo/margin_mean": 85.91107940673828,
"margin_dpo/margin_std": 146.6105194091797,
"step": 255
},
{
"KL/chosen_KL_mean": -167.70700073242188,
"KL/mean": -206.95205688476562,
"KL/rejected_KL_mean": -246.19708251953125,
"KL/std": 115.3568344116211,
"epoch": 0.5361256544502618,
"fcm_dpo/beta": 0.0038304305635392666,
"fcm_dpo/delta": -0.04873408377170563,
"fcm_dpo/margin": 78.49009704589844,
"fcm_dpo/q_t": 0.4281355142593384,
"grad_norm": 82.34991455078125,
"learning_rate": 2.6372383496608186e-07,
"logits/chosen": -0.8674604296684265,
"logits/rejected": -0.8532135486602783,
"logps/chosen": -491.4251708984375,
"logps/ref_chosen": -323.7181701660156,
"logps/ref_rejected": -254.1871337890625,
"logps/rejected": -500.38421630859375,
"loss": 4.7412,
"margin_dpo/margin_mean": 78.49009704589844,
"margin_dpo/margin_std": 136.25772094726562,
"step": 256
},
{
"KL/chosen_KL_mean": -172.4775390625,
"KL/mean": -212.53933715820312,
"KL/rejected_KL_mean": -252.6011199951172,
"KL/std": 113.29838562011719,
"epoch": 0.5382198952879581,
"fcm_dpo/beta": 0.003783945692703128,
"fcm_dpo/delta": 0.01767011545598507,
"fcm_dpo/margin": 80.12359619140625,
"fcm_dpo/q_t": 0.4296230971813202,
"grad_norm": 52.96334457397461,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": -0.8517543077468872,
"logits/rejected": -0.8445267081260681,
"logps/chosen": -439.6896057128906,
"logps/ref_chosen": -267.21209716796875,
"logps/ref_rejected": -249.12579345703125,
"logps/rejected": -501.7268981933594,
"loss": 4.6621,
"margin_dpo/margin_mean": 80.12359619140625,
"margin_dpo/margin_std": 130.36715698242188,
"step": 257
},
{
"KL/chosen_KL_mean": -185.05252075195312,
"KL/mean": -227.99496459960938,
"KL/rejected_KL_mean": -270.9373779296875,
"KL/std": 111.19657135009766,
"epoch": 0.5403141361256545,
"fcm_dpo/beta": 0.00376452156342566,
"fcm_dpo/delta": -0.018048834055662155,
"fcm_dpo/margin": 85.88488006591797,
"fcm_dpo/q_t": 0.42411839962005615,
"grad_norm": 59.09746551513672,
"learning_rate": 2.600664850273538e-07,
"logits/chosen": -0.8708582520484924,
"logits/rejected": -0.8416208624839783,
"logps/chosen": -462.7352600097656,
"logps/ref_chosen": -277.6827392578125,
"logps/ref_rejected": -250.73385620117188,
"logps/rejected": -521.6712036132812,
"loss": 4.5813,
"margin_dpo/margin_mean": 85.88487243652344,
"margin_dpo/margin_std": 122.19969177246094,
"step": 258
},
{
"KL/chosen_KL_mean": -170.76246643066406,
"KL/mean": -212.5333709716797,
"KL/rejected_KL_mean": -254.30429077148438,
"KL/std": 120.29609680175781,
"epoch": 0.5424083769633508,
"fcm_dpo/beta": 0.003755849553272128,
"fcm_dpo/delta": -0.010320892557501793,
"fcm_dpo/margin": 83.54182434082031,
"fcm_dpo/q_t": 0.4279811978340149,
"grad_norm": 49.8717041015625,
"learning_rate": 2.582369512637302e-07,
"logits/chosen": -0.8988098502159119,
"logits/rejected": -0.8933365941047668,
"logps/chosen": -465.3724365234375,
"logps/ref_chosen": -294.6099853515625,
"logps/ref_rejected": -272.2725830078125,
"logps/rejected": -526.576904296875,
"loss": 4.6393,
"margin_dpo/margin_mean": 83.54182434082031,
"margin_dpo/margin_std": 132.25244140625,
"step": 259
},
{
"KL/chosen_KL_mean": -185.67103576660156,
"KL/mean": -202.73085021972656,
"KL/rejected_KL_mean": -219.79066467285156,
"KL/std": 103.56588745117188,
"epoch": 0.5445026178010471,
"fcm_dpo/beta": 0.0037252292968332767,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 34.1196174621582,
"fcm_dpo/q_t": 0.4704400599002838,
"grad_norm": 56.41184616088867,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": -0.8784099221229553,
"logits/rejected": -0.8720245361328125,
"logps/chosen": -476.5281677246094,
"logps/ref_chosen": -290.85711669921875,
"logps/ref_rejected": -277.5970153808594,
"logps/rejected": -497.3876953125,
"loss": 5.2616,
"margin_dpo/margin_mean": 34.1196174621582,
"margin_dpo/margin_std": 120.5584945678711,
"step": 260
},
{
"KL/chosen_KL_mean": -196.43760681152344,
"KL/mean": -230.16085815429688,
"KL/rejected_KL_mean": -263.88409423828125,
"KL/std": 111.6231918334961,
"epoch": 0.5465968586387434,
"fcm_dpo/beta": 0.003716195933520794,
"fcm_dpo/delta": -0.004061264451593161,
"fcm_dpo/margin": 67.44648742675781,
"fcm_dpo/q_t": 0.44201305508613586,
"grad_norm": 55.86997985839844,
"learning_rate": 2.5457665670441937e-07,
"logits/chosen": -0.7669353485107422,
"logits/rejected": -0.7763051986694336,
"logps/chosen": -447.56982421875,
"logps/ref_chosen": -251.13223266601562,
"logps/ref_rejected": -244.76016235351562,
"logps/rejected": -508.64422607421875,
"loss": 4.8721,
"margin_dpo/margin_mean": 67.44648742675781,
"margin_dpo/margin_std": 137.79544067382812,
"step": 261
},
{
"KL/chosen_KL_mean": -178.6384735107422,
"KL/mean": -224.44729614257812,
"KL/rejected_KL_mean": -270.2561340332031,
"KL/std": 111.68685150146484,
"epoch": 0.5486910994764398,
"fcm_dpo/beta": 0.0037381243892014027,
"fcm_dpo/delta": -0.0032047489657998085,
"fcm_dpo/margin": 91.61766052246094,
"fcm_dpo/q_t": 0.419685423374176,
"grad_norm": 63.28798294067383,
"learning_rate": 2.527460921992209e-07,
"logits/chosen": -0.814434289932251,
"logits/rejected": -0.8024383187294006,
"logps/chosen": -478.36029052734375,
"logps/ref_chosen": -299.7217712402344,
"logps/ref_rejected": -277.0969543457031,
"logps/rejected": -547.3530883789062,
"loss": 4.5031,
"margin_dpo/margin_mean": 91.61766052246094,
"margin_dpo/margin_std": 122.15998077392578,
"step": 262
},
{
"KL/chosen_KL_mean": -189.57598876953125,
"KL/mean": -228.04983520507812,
"KL/rejected_KL_mean": -266.523681640625,
"KL/std": 116.67737579345703,
"epoch": 0.5507853403141362,
"fcm_dpo/beta": 0.003702682675793767,
"fcm_dpo/delta": 0.0006384910084307194,
"fcm_dpo/margin": 76.94770050048828,
"fcm_dpo/q_t": 0.4331601560115814,
"grad_norm": 52.963157653808594,
"learning_rate": 2.509153804294318e-07,
"logits/chosen": -0.8113653063774109,
"logits/rejected": -0.7922741174697876,
"logps/chosen": -469.5285949707031,
"logps/ref_chosen": -279.95257568359375,
"logps/ref_rejected": -256.5327453613281,
"logps/rejected": -523.0564575195312,
"loss": 4.7282,
"margin_dpo/margin_mean": 76.94770050048828,
"margin_dpo/margin_std": 126.67123413085938,
"step": 263
},
{
"KL/chosen_KL_mean": -180.07493591308594,
"KL/mean": -227.3055877685547,
"KL/rejected_KL_mean": -274.53619384765625,
"KL/std": 121.89452362060547,
"epoch": 0.5528795811518324,
"fcm_dpo/beta": 0.0036615305580198765,
"fcm_dpo/delta": -0.010648549534380436,
"fcm_dpo/margin": 94.46124267578125,
"fcm_dpo/q_t": 0.4205635190010071,
"grad_norm": 50.7412109375,
"learning_rate": 2.4908461957056825e-07,
"logits/chosen": -0.8269824981689453,
"logits/rejected": -0.818806529045105,
"logps/chosen": -440.6100769042969,
"logps/ref_chosen": -260.53509521484375,
"logps/ref_rejected": -255.53799438476562,
"logps/rejected": -530.07421875,
"loss": 4.5297,
"margin_dpo/margin_mean": 94.46124267578125,
"margin_dpo/margin_std": 134.4732666015625,
"step": 264
},
{
"KL/chosen_KL_mean": -182.9898223876953,
"KL/mean": -233.20361328125,
"KL/rejected_KL_mean": -283.4173889160156,
"KL/std": 130.7704620361328,
"epoch": 0.5549738219895288,
"fcm_dpo/beta": 0.0036417150404304266,
"fcm_dpo/delta": -0.012156824581325054,
"fcm_dpo/margin": 100.42760467529297,
"fcm_dpo/q_t": 0.41630399227142334,
"grad_norm": 58.42458724975586,
"learning_rate": 2.4725390780077905e-07,
"logits/chosen": -0.9007827043533325,
"logits/rejected": -0.9030391573905945,
"logps/chosen": -466.7028503417969,
"logps/ref_chosen": -283.7130432128906,
"logps/ref_rejected": -270.3209533691406,
"logps/rejected": -553.7383422851562,
"loss": 4.5194,
"margin_dpo/margin_mean": 100.42760467529297,
"margin_dpo/margin_std": 151.31163024902344,
"step": 265
},
{
"KL/chosen_KL_mean": -172.0806884765625,
"KL/mean": -221.55726623535156,
"KL/rejected_KL_mean": -271.0338439941406,
"KL/std": 119.99907684326172,
"epoch": 0.5570680628272251,
"fcm_dpo/beta": 0.0035795229487121105,
"fcm_dpo/delta": -0.0383283831179142,
"fcm_dpo/margin": 98.95314025878906,
"fcm_dpo/q_t": 0.41741737723350525,
"grad_norm": 44.32012176513672,
"learning_rate": 2.454233432955807e-07,
"logits/chosen": -0.9105031490325928,
"logits/rejected": -0.881584107875824,
"logps/chosen": -450.17999267578125,
"logps/ref_chosen": -278.09930419921875,
"logps/ref_rejected": -260.6734619140625,
"logps/rejected": -531.7073364257812,
"loss": 4.4727,
"margin_dpo/margin_mean": 98.95314025878906,
"margin_dpo/margin_std": 129.08531188964844,
"step": 266
},
{
"KL/chosen_KL_mean": -203.26300048828125,
"KL/mean": -240.33253479003906,
"KL/rejected_KL_mean": -277.4020080566406,
"KL/std": 123.59054565429688,
"epoch": 0.5591623036649215,
"fcm_dpo/beta": 0.0035598650574684143,
"fcm_dpo/delta": 0.02088256925344467,
"fcm_dpo/margin": 74.1390151977539,
"fcm_dpo/q_t": 0.43908679485321045,
"grad_norm": 63.60444259643555,
"learning_rate": 2.435930242225919e-07,
"logits/chosen": -0.8473813533782959,
"logits/rejected": -0.8534542322158813,
"logps/chosen": -483.59619140625,
"logps/ref_chosen": -280.33319091796875,
"logps/ref_rejected": -247.78099060058594,
"logps/rejected": -525.1829833984375,
"loss": 4.8265,
"margin_dpo/margin_mean": 74.1390151977539,
"margin_dpo/margin_std": 141.98263549804688,
"step": 267
},
{
"KL/chosen_KL_mean": -191.51849365234375,
"KL/mean": -238.06063842773438,
"KL/rejected_KL_mean": -284.60284423828125,
"KL/std": 125.77220916748047,
"epoch": 0.5612565445026177,
"fcm_dpo/beta": 0.0036618567537516356,
"fcm_dpo/delta": 0.031537555158138275,
"fcm_dpo/margin": 93.08435821533203,
"fcm_dpo/q_t": 0.4210575520992279,
"grad_norm": 70.60133361816406,
"learning_rate": 2.4176304873626984e-07,
"logits/chosen": -0.827730119228363,
"logits/rejected": -0.8081169128417969,
"logps/chosen": -495.69720458984375,
"logps/ref_chosen": -304.1787109375,
"logps/ref_rejected": -272.80316162109375,
"logps/rejected": -557.4060668945312,
"loss": 4.5456,
"margin_dpo/margin_mean": 93.08436584472656,
"margin_dpo/margin_std": 135.3468475341797,
"step": 268
},
{
"KL/chosen_KL_mean": -192.0048828125,
"KL/mean": -225.10169982910156,
"KL/rejected_KL_mean": -258.198486328125,
"KL/std": 111.24128723144531,
"epoch": 0.5633507853403141,
"fcm_dpo/beta": 0.0036840138491243124,
"fcm_dpo/delta": 0.0019121436635032296,
"fcm_dpo/margin": 66.193603515625,
"fcm_dpo/q_t": 0.44279053807258606,
"grad_norm": 64.69379425048828,
"learning_rate": 2.399335149726463e-07,
"logits/chosen": -0.8461325764656067,
"logits/rejected": -0.8401869535446167,
"logps/chosen": -441.85003662109375,
"logps/ref_chosen": -249.84512329101562,
"logps/ref_rejected": -223.37356567382812,
"logps/rejected": -481.57208251953125,
"loss": 4.8866,
"margin_dpo/margin_mean": 66.193603515625,
"margin_dpo/margin_std": 138.15615844726562,
"step": 269
},
{
"KL/chosen_KL_mean": -198.5923309326172,
"KL/mean": -241.0689697265625,
"KL/rejected_KL_mean": -283.5456237792969,
"KL/std": 129.37863159179688,
"epoch": 0.5654450261780105,
"fcm_dpo/beta": 0.003672501305118203,
"fcm_dpo/delta": -0.003966245800256729,
"fcm_dpo/margin": 84.95326232910156,
"fcm_dpo/q_t": 0.42902982234954834,
"grad_norm": 59.16077423095703,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": -0.9263371825218201,
"logits/rejected": -0.9309374690055847,
"logps/chosen": -517.1547241210938,
"logps/ref_chosen": -318.5623779296875,
"logps/ref_rejected": -281.1880798339844,
"logps/rejected": -564.7337036132812,
"loss": 4.731,
"margin_dpo/margin_mean": 84.95326232910156,
"margin_dpo/margin_std": 162.09225463867188,
"step": 270
},
{
"KL/chosen_KL_mean": -195.5090789794922,
"KL/mean": -234.53392028808594,
"KL/rejected_KL_mean": -273.5587158203125,
"KL/std": 128.2386474609375,
"epoch": 0.5675392670157068,
"fcm_dpo/beta": 0.0036695911549031734,
"fcm_dpo/delta": 0.001470465213060379,
"fcm_dpo/margin": 78.04962921142578,
"fcm_dpo/q_t": 0.43377920985221863,
"grad_norm": 59.94120407104492,
"learning_rate": 2.3627616503391812e-07,
"logits/chosen": -0.7686310410499573,
"logits/rejected": -0.761326789855957,
"logps/chosen": -479.6138000488281,
"logps/ref_chosen": -284.104736328125,
"logps/ref_rejected": -253.9580535888672,
"logps/rejected": -527.5167846679688,
"loss": 4.727,
"margin_dpo/margin_mean": 78.04962921142578,
"margin_dpo/margin_std": 134.4047393798828,
"step": 271
},
{
"KL/chosen_KL_mean": -186.4130859375,
"KL/mean": -229.1510009765625,
"KL/rejected_KL_mean": -271.888916015625,
"KL/std": 129.54705810546875,
"epoch": 0.5696335078534032,
"fcm_dpo/beta": 0.003663167357444763,
"fcm_dpo/delta": 0.006455127615481615,
"fcm_dpo/margin": 85.47587585449219,
"fcm_dpo/q_t": 0.42850711941719055,
"grad_norm": 56.258522033691406,
"learning_rate": 2.344485449913914e-07,
"logits/chosen": -0.8799334168434143,
"logits/rejected": -0.8621615171432495,
"logps/chosen": -483.7720642089844,
"logps/ref_chosen": -297.3590087890625,
"logps/ref_rejected": -279.20196533203125,
"logps/rejected": -551.0909423828125,
"loss": 4.7078,
"margin_dpo/margin_mean": 85.47587585449219,
"margin_dpo/margin_std": 155.64697265625,
"step": 272
},
{
"KL/chosen_KL_mean": -185.26217651367188,
"KL/mean": -228.59132385253906,
"KL/rejected_KL_mean": -271.9204406738281,
"KL/std": 119.84618377685547,
"epoch": 0.5717277486910994,
"fcm_dpo/beta": 0.003680173773318529,
"fcm_dpo/delta": 0.0036273770965635777,
"fcm_dpo/margin": 86.65824127197266,
"fcm_dpo/q_t": 0.4274797737598419,
"grad_norm": 75.49598693847656,
"learning_rate": 2.3262175892620062e-07,
"logits/chosen": -0.8711019158363342,
"logits/rejected": -0.8843865394592285,
"logps/chosen": -478.4679260253906,
"logps/ref_chosen": -293.20574951171875,
"logps/ref_rejected": -274.7646789550781,
"logps/rejected": -546.68505859375,
"loss": 4.6788,
"margin_dpo/margin_mean": 86.65824127197266,
"margin_dpo/margin_std": 152.3551025390625,
"step": 273
},
{
"KL/chosen_KL_mean": -184.4656982421875,
"KL/mean": -236.48504638671875,
"KL/rejected_KL_mean": -288.50445556640625,
"KL/std": 131.9507598876953,
"epoch": 0.5738219895287958,
"fcm_dpo/beta": 0.0037050426471978426,
"fcm_dpo/delta": -0.001804053783416748,
"fcm_dpo/margin": 104.03874206542969,
"fcm_dpo/q_t": 0.41177284717559814,
"grad_norm": 57.44676208496094,
"learning_rate": 2.3079590480333827e-07,
"logits/chosen": -0.8160399794578552,
"logits/rejected": -0.784659206867218,
"logps/chosen": -455.0243835449219,
"logps/ref_chosen": -270.55865478515625,
"logps/ref_rejected": -239.47048950195312,
"logps/rejected": -527.9749145507812,
"loss": 4.4432,
"margin_dpo/margin_mean": 104.03874206542969,
"margin_dpo/margin_std": 148.03915405273438,
"step": 274
},
{
"KL/chosen_KL_mean": -167.6212921142578,
"KL/mean": -221.64671325683594,
"KL/rejected_KL_mean": -275.67218017578125,
"KL/std": 129.25466918945312,
"epoch": 0.5759162303664922,
"fcm_dpo/beta": 0.0036281857173889875,
"fcm_dpo/delta": -0.03448822721838951,
"fcm_dpo/margin": 108.0508804321289,
"fcm_dpo/q_t": 0.4102928340435028,
"grad_norm": 60.525047302246094,
"learning_rate": 2.2897108053782e-07,
"logits/chosen": -0.873982310295105,
"logits/rejected": -0.8553178906440735,
"logps/chosen": -417.94049072265625,
"logps/ref_chosen": -250.31922912597656,
"logps/ref_rejected": -249.3187255859375,
"logps/rejected": -524.9908447265625,
"loss": 4.3886,
"margin_dpo/margin_mean": 108.05087280273438,
"margin_dpo/margin_std": 142.07064819335938,
"step": 275
},
{
"KL/chosen_KL_mean": -172.93948364257812,
"KL/mean": -224.42706298828125,
"KL/rejected_KL_mean": -275.91461181640625,
"KL/std": 131.497314453125,
"epoch": 0.5780104712041885,
"fcm_dpo/beta": 0.0035385219380259514,
"fcm_dpo/delta": -0.009656044654548168,
"fcm_dpo/margin": 102.97510528564453,
"fcm_dpo/q_t": 0.4158563017845154,
"grad_norm": 54.36017608642578,
"learning_rate": 2.2714738398943308e-07,
"logits/chosen": -0.9190772771835327,
"logits/rejected": -0.8974164724349976,
"logps/chosen": -470.5705261230469,
"logps/ref_chosen": -297.6310729980469,
"logps/ref_rejected": -295.225830078125,
"logps/rejected": -571.1404418945312,
"loss": 4.4991,
"margin_dpo/margin_mean": 102.97510528564453,
"margin_dpo/margin_std": 147.03050231933594,
"step": 276
},
{
"KL/chosen_KL_mean": -197.20071411132812,
"KL/mean": -231.30577087402344,
"KL/rejected_KL_mean": -265.41082763671875,
"KL/std": 134.91824340820312,
"epoch": 0.5801047120418849,
"fcm_dpo/beta": 0.003561137244105339,
"fcm_dpo/delta": -0.004946920555084944,
"fcm_dpo/margin": 68.21009826660156,
"fcm_dpo/q_t": 0.44467830657958984,
"grad_norm": 63.98385238647461,
"learning_rate": 2.2532491295748865e-07,
"logits/chosen": -0.8566328287124634,
"logits/rejected": -0.8565943837165833,
"logps/chosen": -463.5611877441406,
"logps/ref_chosen": -266.3604736328125,
"logps/ref_rejected": -253.36767578125,
"logps/rejected": -518.7784423828125,
"loss": 4.9356,
"margin_dpo/margin_mean": 68.21009826660156,
"margin_dpo/margin_std": 153.74371337890625,
"step": 277
},
{
"KL/chosen_KL_mean": -204.8040771484375,
"KL/mean": -233.74908447265625,
"KL/rejected_KL_mean": -262.694091796875,
"KL/std": 139.94210815429688,
"epoch": 0.5821989528795811,
"fcm_dpo/beta": 0.003580951364710927,
"fcm_dpo/delta": 0.01175983902066946,
"fcm_dpo/margin": 57.88999557495117,
"fcm_dpo/q_t": 0.4574360251426697,
"grad_norm": 59.146480560302734,
"learning_rate": 2.2350376517557726e-07,
"logits/chosen": -0.8874562382698059,
"logits/rejected": -0.8490403890609741,
"logps/chosen": -472.21136474609375,
"logps/ref_chosen": -267.40728759765625,
"logps/ref_rejected": -229.5758514404297,
"logps/rejected": -492.2698974609375,
"loss": 5.1767,
"margin_dpo/margin_mean": 57.88999557495117,
"margin_dpo/margin_std": 176.36570739746094,
"step": 278
},
{
"KL/chosen_KL_mean": -189.5259552001953,
"KL/mean": -242.93582153320312,
"KL/rejected_KL_mean": -296.345703125,
"KL/std": 145.5375213623047,
"epoch": 0.5842931937172775,
"fcm_dpo/beta": 0.003567744279280305,
"fcm_dpo/delta": -0.021797548979520798,
"fcm_dpo/margin": 106.81974029541016,
"fcm_dpo/q_t": 0.4145677089691162,
"grad_norm": 66.13142395019531,
"learning_rate": 2.2168403830632769e-07,
"logits/chosen": -0.8134815096855164,
"logits/rejected": -0.7979801893234253,
"logps/chosen": -502.89373779296875,
"logps/ref_chosen": -313.3677978515625,
"logps/ref_rejected": -299.1744384765625,
"logps/rejected": -595.5202026367188,
"loss": 4.4922,
"margin_dpo/margin_mean": 106.81974792480469,
"margin_dpo/margin_std": 161.84104919433594,
"step": 279
},
{
"KL/chosen_KL_mean": -174.68370056152344,
"KL/mean": -217.16433715820312,
"KL/rejected_KL_mean": -259.64495849609375,
"KL/std": 118.23523712158203,
"epoch": 0.5863874345549738,
"fcm_dpo/beta": 0.0035581968259066343,
"fcm_dpo/delta": 0.018335789442062378,
"fcm_dpo/margin": 84.96123504638672,
"fcm_dpo/q_t": 0.4296843409538269,
"grad_norm": 67.77641296386719,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": -0.8782325983047485,
"logits/rejected": -0.8902976512908936,
"logps/chosen": -440.239501953125,
"logps/ref_chosen": -265.5558166503906,
"logps/ref_rejected": -247.1573944091797,
"logps/rejected": -506.8023681640625,
"loss": 4.6825,
"margin_dpo/margin_mean": 84.96123504638672,
"margin_dpo/margin_std": 143.5811004638672,
"step": 280
},
{
"KL/chosen_KL_mean": -199.01400756835938,
"KL/mean": -250.08953857421875,
"KL/rejected_KL_mean": -301.16510009765625,
"KL/std": 136.9486083984375,
"epoch": 0.5884816753926702,
"fcm_dpo/beta": 0.0035306517966091633,
"fcm_dpo/delta": -0.02944459766149521,
"fcm_dpo/margin": 102.15110778808594,
"fcm_dpo/q_t": 0.4185321033000946,
"grad_norm": 62.55031967163086,
"learning_rate": 2.1804923757009882e-07,
"logits/chosen": -0.8659012913703918,
"logits/rejected": -0.8745157122612,
"logps/chosen": -494.3135986328125,
"logps/ref_chosen": -295.2995910644531,
"logps/ref_rejected": -293.80877685546875,
"logps/rejected": -594.9738159179688,
"loss": 4.5101,
"margin_dpo/margin_mean": 102.15110778808594,
"margin_dpo/margin_std": 148.16885375976562,
"step": 281
},
{
"KL/chosen_KL_mean": -205.15968322753906,
"KL/mean": -245.86717224121094,
"KL/rejected_KL_mean": -286.5746765136719,
"KL/std": 131.70346069335938,
"epoch": 0.5905759162303665,
"fcm_dpo/beta": 0.003522593295201659,
"fcm_dpo/delta": 0.00954904779791832,
"fcm_dpo/margin": 81.4150161743164,
"fcm_dpo/q_t": 0.43364566564559937,
"grad_norm": 60.06401062011719,
"learning_rate": 2.1623435862645205e-07,
"logits/chosen": -0.8671073913574219,
"logits/rejected": -0.8696990013122559,
"logps/chosen": -523.7968139648438,
"logps/ref_chosen": -318.63714599609375,
"logps/ref_rejected": -273.5943603515625,
"logps/rejected": -560.1690673828125,
"loss": 4.7488,
"margin_dpo/margin_mean": 81.4150161743164,
"margin_dpo/margin_std": 147.11825561523438,
"step": 282
},
{
"KL/chosen_KL_mean": -196.23243713378906,
"KL/mean": -240.39724731445312,
"KL/rejected_KL_mean": -284.5620422363281,
"KL/std": 129.15927124023438,
"epoch": 0.5926701570680628,
"fcm_dpo/beta": 0.003505897242575884,
"fcm_dpo/delta": -0.027185089886188507,
"fcm_dpo/margin": 88.32962036132812,
"fcm_dpo/q_t": 0.4301910996437073,
"grad_norm": 63.45745849609375,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": -0.8812756538391113,
"logits/rejected": -0.8700328469276428,
"logps/chosen": -450.8929748535156,
"logps/ref_chosen": -254.66053771972656,
"logps/ref_rejected": -236.8627166748047,
"logps/rejected": -521.4248046875,
"loss": 4.6848,
"margin_dpo/margin_mean": 88.32962036132812,
"margin_dpo/margin_std": 149.92486572265625,
"step": 283
},
{
"KL/chosen_KL_mean": -211.7559814453125,
"KL/mean": -256.141357421875,
"KL/rejected_KL_mean": -300.5267333984375,
"KL/std": 144.52191162109375,
"epoch": 0.5947643979057592,
"fcm_dpo/beta": 0.003395251464098692,
"fcm_dpo/delta": -0.023953504860401154,
"fcm_dpo/margin": 88.77078247070312,
"fcm_dpo/q_t": 0.43172183632850647,
"grad_norm": 57.34799575805664,
"learning_rate": 2.1261013021512378e-07,
"logits/chosen": -0.844742476940155,
"logits/rejected": -0.8104969263076782,
"logps/chosen": -485.1112365722656,
"logps/ref_chosen": -273.355224609375,
"logps/ref_rejected": -259.84759521484375,
"logps/rejected": -560.3743896484375,
"loss": 4.7799,
"margin_dpo/margin_mean": 88.77078247070312,
"margin_dpo/margin_std": 171.4403533935547,
"step": 284
},
{
"KL/chosen_KL_mean": -215.08908081054688,
"KL/mean": -244.9665069580078,
"KL/rejected_KL_mean": -274.8439636230469,
"KL/std": 130.01708984375,
"epoch": 0.5968586387434555,
"fcm_dpo/beta": 0.0033849114552140236,
"fcm_dpo/delta": -0.0046186684630811214,
"fcm_dpo/margin": 59.75489807128906,
"fcm_dpo/q_t": 0.45396003127098083,
"grad_norm": 75.9342041015625,
"learning_rate": 2.1080097510381294e-07,
"logits/chosen": -0.8575014472007751,
"logits/rejected": -0.8521921634674072,
"logps/chosen": -524.891357421875,
"logps/ref_chosen": -309.8022155761719,
"logps/ref_rejected": -279.11846923828125,
"logps/rejected": -553.96240234375,
"loss": 5.0888,
"margin_dpo/margin_mean": 59.75489807128906,
"margin_dpo/margin_std": 159.50062561035156,
"step": 285
},
{
"KL/chosen_KL_mean": -198.20655822753906,
"KL/mean": -242.35784912109375,
"KL/rejected_KL_mean": -286.50909423828125,
"KL/std": 141.2283172607422,
"epoch": 0.5989528795811518,
"fcm_dpo/beta": 0.0033742974046617746,
"fcm_dpo/delta": 0.00259046396240592,
"fcm_dpo/margin": 88.30255126953125,
"fcm_dpo/q_t": 0.4332633316516876,
"grad_norm": 82.05410766601562,
"learning_rate": 2.089939221172446e-07,
"logits/chosen": -0.8378498554229736,
"logits/rejected": -0.8318293690681458,
"logps/chosen": -469.6720886230469,
"logps/ref_chosen": -271.4655456542969,
"logps/ref_rejected": -279.531494140625,
"logps/rejected": -566.0405883789062,
"loss": 4.7228,
"margin_dpo/margin_mean": 88.30255126953125,
"margin_dpo/margin_std": 159.4552459716797,
"step": 286
},
{
"KL/chosen_KL_mean": -207.5540008544922,
"KL/mean": -247.44190979003906,
"KL/rejected_KL_mean": -287.329833984375,
"KL/std": 134.13143920898438,
"epoch": 0.6010471204188481,
"fcm_dpo/beta": 0.0034193575847893953,
"fcm_dpo/delta": 0.0249363761395216,
"fcm_dpo/margin": 79.77581024169922,
"fcm_dpo/q_t": 0.43721669912338257,
"grad_norm": 56.872581481933594,
"learning_rate": 2.0718906816218595e-07,
"logits/chosen": -0.8712465167045593,
"logits/rejected": -0.8559173941612244,
"logps/chosen": -484.647216796875,
"logps/ref_chosen": -277.0932312011719,
"logps/ref_rejected": -233.55599975585938,
"logps/rejected": -520.8858032226562,
"loss": 4.8451,
"margin_dpo/margin_mean": 79.77581787109375,
"margin_dpo/margin_std": 164.70887756347656,
"step": 287
},
{
"KL/chosen_KL_mean": -210.85910034179688,
"KL/mean": -248.73919677734375,
"KL/rejected_KL_mean": -286.6192932128906,
"KL/std": 138.9389190673828,
"epoch": 0.6031413612565445,
"fcm_dpo/beta": 0.003475298173725605,
"fcm_dpo/delta": 0.0075583611615002155,
"fcm_dpo/margin": 75.76020812988281,
"fcm_dpo/q_t": 0.4398469924926758,
"grad_norm": 84.63459014892578,
"learning_rate": 2.053865100274774e-07,
"logits/chosen": -0.8746079206466675,
"logits/rejected": -0.8799384832382202,
"logps/chosen": -504.0272521972656,
"logps/ref_chosen": -293.1681823730469,
"logps/ref_rejected": -263.4059143066406,
"logps/rejected": -550.0252075195312,
"loss": 4.8465,
"margin_dpo/margin_mean": 75.76020812988281,
"margin_dpo/margin_std": 154.83770751953125,
"step": 288
},
{
"KL/chosen_KL_mean": -228.95736694335938,
"KL/mean": -254.9104766845703,
"KL/rejected_KL_mean": -280.86358642578125,
"KL/std": 126.50141906738281,
"epoch": 0.6052356020942409,
"fcm_dpo/beta": 0.0034878358710557222,
"fcm_dpo/delta": 0.018204133957624435,
"fcm_dpo/margin": 51.90618896484375,
"fcm_dpo/q_t": 0.4578736424446106,
"grad_norm": 80.30821228027344,
"learning_rate": 2.035863443788411e-07,
"logits/chosen": -0.8596778512001038,
"logits/rejected": -0.841497540473938,
"logps/chosen": -558.914794921875,
"logps/ref_chosen": -329.9574279785156,
"logps/ref_rejected": -276.7565002441406,
"logps/rejected": -557.6200561523438,
"loss": 5.1116,
"margin_dpo/margin_mean": 51.906192779541016,
"margin_dpo/margin_std": 142.21298217773438,
"step": 289
},
{
"KL/chosen_KL_mean": -208.2900848388672,
"KL/mean": -250.78472900390625,
"KL/rejected_KL_mean": -293.2793273925781,
"KL/std": 139.5697784423828,
"epoch": 0.6073298429319371,
"fcm_dpo/beta": 0.0034028813242912292,
"fcm_dpo/delta": -0.049578987061977386,
"fcm_dpo/margin": 84.98924255371094,
"fcm_dpo/q_t": 0.43361252546310425,
"grad_norm": 59.59135818481445,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": -0.8690849542617798,
"logits/rejected": -0.8019747138023376,
"logps/chosen": -532.9591064453125,
"logps/ref_chosen": -324.6690673828125,
"logps/ref_rejected": -311.8439636230469,
"logps/rejected": -605.123291015625,
"loss": 4.7878,
"margin_dpo/margin_mean": 84.98924255371094,
"margin_dpo/margin_std": 162.67477416992188,
"step": 290
},
{
"KL/chosen_KL_mean": -200.08258056640625,
"KL/mean": -254.6800079345703,
"KL/rejected_KL_mean": -309.2774658203125,
"KL/std": 136.1592559814453,
"epoch": 0.6094240837696335,
"fcm_dpo/beta": 0.0033203570637851954,
"fcm_dpo/delta": -0.027233559638261795,
"fcm_dpo/margin": 109.19487762451172,
"fcm_dpo/q_t": 0.4158145785331726,
"grad_norm": 57.8904914855957,
"learning_rate": 1.9999357655598891e-07,
"logits/chosen": -0.8527657985687256,
"logits/rejected": -0.8431987762451172,
"logps/chosen": -474.2266540527344,
"logps/ref_chosen": -274.1440734863281,
"logps/ref_rejected": -278.07208251953125,
"logps/rejected": -587.3495483398438,
"loss": 4.477,
"margin_dpo/margin_mean": 109.19487762451172,
"margin_dpo/margin_std": 146.18112182617188,
"step": 291
},
{
"KL/chosen_KL_mean": -194.33355712890625,
"KL/mean": -239.71888732910156,
"KL/rejected_KL_mean": -285.10418701171875,
"KL/std": 141.9927978515625,
"epoch": 0.6115183246073298,
"fcm_dpo/beta": 0.003264216473326087,
"fcm_dpo/delta": -0.02208729460835457,
"fcm_dpo/margin": 90.77063751220703,
"fcm_dpo/q_t": 0.43203163146972656,
"grad_norm": 83.5884017944336,
"learning_rate": 1.9820116705100775e-07,
"logits/chosen": -0.8462182283401489,
"logits/rejected": -0.8420131206512451,
"logps/chosen": -453.69720458984375,
"logps/ref_chosen": -259.3636779785156,
"logps/ref_rejected": -279.30218505859375,
"logps/rejected": -564.4063720703125,
"loss": 4.7423,
"margin_dpo/margin_mean": 90.77064514160156,
"margin_dpo/margin_std": 164.15216064453125,
"step": 292
},
{
"KL/chosen_KL_mean": -218.1024169921875,
"KL/mean": -264.27685546875,
"KL/rejected_KL_mean": -310.4512939453125,
"KL/std": 145.62364196777344,
"epoch": 0.6136125654450262,
"fcm_dpo/beta": 0.0032458994537591934,
"fcm_dpo/delta": 0.0010393057018518448,
"fcm_dpo/margin": 92.3488540649414,
"fcm_dpo/q_t": 0.430684357881546,
"grad_norm": 79.72573852539062,
"learning_rate": 1.9641153536023642e-07,
"logits/chosen": -0.9407028555870056,
"logits/rejected": -0.901149570941925,
"logps/chosen": -521.8732299804688,
"logps/ref_chosen": -303.77081298828125,
"logps/ref_rejected": -270.07513427734375,
"logps/rejected": -580.5263671875,
"loss": 4.6868,
"margin_dpo/margin_mean": 92.34884643554688,
"margin_dpo/margin_std": 150.49273681640625,
"step": 293
},
{
"KL/chosen_KL_mean": -210.60379028320312,
"KL/mean": -250.68609619140625,
"KL/rejected_KL_mean": -290.7684020996094,
"KL/std": 138.80445861816406,
"epoch": 0.6157068062827226,
"fcm_dpo/beta": 0.0032662583980709314,
"fcm_dpo/delta": 0.0030375849455595016,
"fcm_dpo/margin": 80.16455841064453,
"fcm_dpo/q_t": 0.440270334482193,
"grad_norm": 77.52326965332031,
"learning_rate": 1.9462477745619106e-07,
"logits/chosen": -0.8381096720695496,
"logits/rejected": -0.8436415195465088,
"logps/chosen": -450.8421325683594,
"logps/ref_chosen": -240.23831176757812,
"logps/ref_rejected": -229.187744140625,
"logps/rejected": -519.9561157226562,
"loss": 4.8661,
"margin_dpo/margin_mean": 80.16456604003906,
"margin_dpo/margin_std": 165.5260467529297,
"step": 294
},
{
"KL/chosen_KL_mean": -204.40306091308594,
"KL/mean": -245.8980712890625,
"KL/rejected_KL_mean": -287.39306640625,
"KL/std": 122.65670013427734,
"epoch": 0.6178010471204188,
"fcm_dpo/beta": 0.0032912057358771563,
"fcm_dpo/delta": 0.014641467481851578,
"fcm_dpo/margin": 82.98999786376953,
"fcm_dpo/q_t": 0.4375506639480591,
"grad_norm": 59.422271728515625,
"learning_rate": 1.928409891572757e-07,
"logits/chosen": -0.8383684158325195,
"logits/rejected": -0.8414021134376526,
"logps/chosen": -455.4127502441406,
"logps/ref_chosen": -251.00970458984375,
"logps/ref_rejected": -244.15142822265625,
"logps/rejected": -531.5444946289062,
"loss": 4.7802,
"margin_dpo/margin_mean": 82.98999786376953,
"margin_dpo/margin_std": 152.8289794921875,
"step": 295
},
{
"KL/chosen_KL_mean": -216.45693969726562,
"KL/mean": -269.9961853027344,
"KL/rejected_KL_mean": -323.5354309082031,
"KL/std": 143.4501953125,
"epoch": 0.6198952879581152,
"fcm_dpo/beta": 0.0032748132944107056,
"fcm_dpo/delta": -0.014164071530103683,
"fcm_dpo/margin": 107.0785140991211,
"fcm_dpo/q_t": 0.4195513427257538,
"grad_norm": 66.60144805908203,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": -0.8362563848495483,
"logits/rejected": -0.8021262884140015,
"logps/chosen": -510.3375244140625,
"logps/ref_chosen": -293.880615234375,
"logps/ref_rejected": -283.4175720214844,
"logps/rejected": -606.9530639648438,
"loss": 4.5964,
"margin_dpo/margin_mean": 107.0785140991211,
"margin_dpo/margin_std": 174.71786499023438,
"step": 296
},
{
"KL/chosen_KL_mean": -209.06858825683594,
"KL/mean": -250.95046997070312,
"KL/rejected_KL_mean": -292.8323974609375,
"KL/std": 129.31927490234375,
"epoch": 0.6219895287958115,
"fcm_dpo/beta": 0.003221014980226755,
"fcm_dpo/delta": -0.01936129853129387,
"fcm_dpo/margin": 83.7637710571289,
"fcm_dpo/q_t": 0.4371768832206726,
"grad_norm": 52.13896179199219,
"learning_rate": 1.8928270384706582e-07,
"logits/chosen": -0.9062499403953552,
"logits/rejected": -0.899086058139801,
"logps/chosen": -498.5286560058594,
"logps/ref_chosen": -289.4600830078125,
"logps/ref_rejected": -283.69110107421875,
"logps/rejected": -576.5234375,
"loss": 4.7666,
"margin_dpo/margin_mean": 83.7637710571289,
"margin_dpo/margin_std": 145.26136779785156,
"step": 297
},
{
"KL/chosen_KL_mean": -202.0797576904297,
"KL/mean": -250.1943817138672,
"KL/rejected_KL_mean": -298.3089904785156,
"KL/std": 134.8616943359375,
"epoch": 0.6240837696335079,
"fcm_dpo/beta": 0.0031903949566185474,
"fcm_dpo/delta": -0.01658639684319496,
"fcm_dpo/margin": 96.22921752929688,
"fcm_dpo/q_t": 0.42945045232772827,
"grad_norm": 80.38378143310547,
"learning_rate": 1.875083976558136e-07,
"logits/chosen": -0.8216419219970703,
"logits/rejected": -0.8078317642211914,
"logps/chosen": -508.5947570800781,
"logps/ref_chosen": -306.5150146484375,
"logps/ref_rejected": -280.6969909667969,
"logps/rejected": -579.0059204101562,
"loss": 4.7,
"margin_dpo/margin_mean": 96.2292251586914,
"margin_dpo/margin_std": 163.79393005371094,
"step": 298
},
{
"KL/chosen_KL_mean": -233.1396484375,
"KL/mean": -269.0777893066406,
"KL/rejected_KL_mean": -305.0159606933594,
"KL/std": 135.54824829101562,
"epoch": 0.6261780104712041,
"fcm_dpo/beta": 0.003153471741825342,
"fcm_dpo/delta": -0.007434126455336809,
"fcm_dpo/margin": 71.87633514404297,
"fcm_dpo/q_t": 0.4467780888080597,
"grad_norm": 91.21902465820312,
"learning_rate": 1.8573744269954297e-07,
"logits/chosen": -0.811547040939331,
"logits/rejected": -0.7952200174331665,
"logps/chosen": -514.5033569335938,
"logps/ref_chosen": -281.36376953125,
"logps/ref_rejected": -270.39508056640625,
"logps/rejected": -575.4110107421875,
"loss": 4.9011,
"margin_dpo/margin_mean": 71.87632751464844,
"margin_dpo/margin_std": 136.6454620361328,
"step": 299
},
{
"KL/chosen_KL_mean": -228.64112854003906,
"KL/mean": -268.7874755859375,
"KL/rejected_KL_mean": -308.933837890625,
"KL/std": 145.60882568359375,
"epoch": 0.6282722513089005,
"fcm_dpo/beta": 0.003157139290124178,
"fcm_dpo/delta": 0.0067957015708088875,
"fcm_dpo/margin": 80.2927017211914,
"fcm_dpo/q_t": 0.44123968482017517,
"grad_norm": 61.41617965698242,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": -0.84067302942276,
"logits/rejected": -0.8198051452636719,
"logps/chosen": -543.4768676757812,
"logps/ref_chosen": -314.83575439453125,
"logps/ref_rejected": -269.1154479980469,
"logps/rejected": -578.0492553710938,
"loss": 4.8649,
"margin_dpo/margin_mean": 80.2927017211914,
"margin_dpo/margin_std": 165.27395629882812,
"step": 300
},
{
"KL/chosen_KL_mean": -213.63665771484375,
"KL/mean": -253.96006774902344,
"KL/rejected_KL_mean": -294.28350830078125,
"KL/std": 134.56875610351562,
"epoch": 0.6303664921465969,
"fcm_dpo/beta": 0.003165181027725339,
"fcm_dpo/delta": -0.007671603467315435,
"fcm_dpo/margin": 80.64684295654297,
"fcm_dpo/q_t": 0.44037818908691406,
"grad_norm": 58.440460205078125,
"learning_rate": 1.8220596619089573e-07,
"logits/chosen": -0.869897723197937,
"logits/rejected": -0.8631291389465332,
"logps/chosen": -493.5311584472656,
"logps/ref_chosen": -279.89453125,
"logps/ref_rejected": -271.6694641113281,
"logps/rejected": -565.9530029296875,
"loss": 4.8103,
"margin_dpo/margin_mean": 80.64684295654297,
"margin_dpo/margin_std": 145.11534118652344,
"step": 301
},
{
"KL/chosen_KL_mean": -213.73731994628906,
"KL/mean": -264.9085388183594,
"KL/rejected_KL_mean": -316.0797424316406,
"KL/std": 142.73170471191406,
"epoch": 0.6324607329842932,
"fcm_dpo/beta": 0.003172217635437846,
"fcm_dpo/delta": 0.01391267403960228,
"fcm_dpo/margin": 102.34239959716797,
"fcm_dpo/q_t": 0.4247261881828308,
"grad_norm": 64.62631225585938,
"learning_rate": 1.8044563402088682e-07,
"logits/chosen": -0.8292580842971802,
"logits/rejected": -0.8176696300506592,
"logps/chosen": -485.06915283203125,
"logps/ref_chosen": -271.3318176269531,
"logps/ref_rejected": -256.5587158203125,
"logps/rejected": -572.638427734375,
"loss": 4.5893,
"margin_dpo/margin_mean": 102.34239959716797,
"margin_dpo/margin_std": 152.8368377685547,
"step": 302
},
{
"KL/chosen_KL_mean": -208.92112731933594,
"KL/mean": -253.95260620117188,
"KL/rejected_KL_mean": -298.9841003417969,
"KL/std": 129.33070373535156,
"epoch": 0.6345549738219896,
"fcm_dpo/beta": 0.0031979738269001245,
"fcm_dpo/delta": 0.008892138488590717,
"fcm_dpo/margin": 90.06295013427734,
"fcm_dpo/q_t": 0.43192505836486816,
"grad_norm": 69.94568634033203,
"learning_rate": 1.7868903184043885e-07,
"logits/chosen": -0.8113217353820801,
"logits/rejected": -0.7879561185836792,
"logps/chosen": -513.8021240234375,
"logps/ref_chosen": -304.88104248046875,
"logps/ref_rejected": -269.063720703125,
"logps/rejected": -568.0478515625,
"loss": 4.6939,
"margin_dpo/margin_mean": 90.06295776367188,
"margin_dpo/margin_std": 140.77880859375,
"step": 303
},
{
"KL/chosen_KL_mean": -224.17652893066406,
"KL/mean": -273.39581298828125,
"KL/rejected_KL_mean": -322.6150817871094,
"KL/std": 155.12173461914062,
"epoch": 0.6366492146596858,
"fcm_dpo/beta": 0.0032359191682189703,
"fcm_dpo/delta": 0.02149886079132557,
"fcm_dpo/margin": 98.43850708007812,
"fcm_dpo/q_t": 0.42846569418907166,
"grad_norm": 74.36817169189453,
"learning_rate": 1.7693625385079574e-07,
"logits/chosen": -0.8236927390098572,
"logits/rejected": -0.8324135541915894,
"logps/chosen": -514.8875122070312,
"logps/ref_chosen": -290.7109680175781,
"logps/ref_rejected": -237.6885986328125,
"logps/rejected": -560.3036499023438,
"loss": 4.6536,
"margin_dpo/margin_mean": 98.43850708007812,
"margin_dpo/margin_std": 167.7232666015625,
"step": 304
},
{
"KL/chosen_KL_mean": -188.64137268066406,
"KL/mean": -263.8195495605469,
"KL/rejected_KL_mean": -338.9976806640625,
"KL/std": 151.70013427734375,
"epoch": 0.6387434554973822,
"fcm_dpo/beta": 0.003107309341430664,
"fcm_dpo/delta": -0.11182599514722824,
"fcm_dpo/margin": 150.35633850097656,
"fcm_dpo/q_t": 0.393195241689682,
"grad_norm": 81.83216094970703,
"learning_rate": 1.7518739404812155e-07,
"logits/chosen": -0.8657177686691284,
"logits/rejected": -0.8329824805259705,
"logps/chosen": -445.1253356933594,
"logps/ref_chosen": -256.4839782714844,
"logps/ref_rejected": -266.4063415527344,
"logps/rejected": -605.404052734375,
"loss": 4.1802,
"margin_dpo/margin_mean": 150.3563232421875,
"margin_dpo/margin_std": 173.1336212158203,
"step": 305
},
{
"KL/chosen_KL_mean": -202.8968505859375,
"KL/mean": -247.1907501220703,
"KL/rejected_KL_mean": -291.484619140625,
"KL/std": 131.15960693359375,
"epoch": 0.6408376963350786,
"fcm_dpo/beta": 0.002977159805595875,
"fcm_dpo/delta": 0.012585463933646679,
"fcm_dpo/margin": 88.5877914428711,
"fcm_dpo/q_t": 0.43810799717903137,
"grad_norm": 64.3017349243164,
"learning_rate": 1.7344254621846017e-07,
"logits/chosen": -0.8670382499694824,
"logits/rejected": -0.8467966914176941,
"logps/chosen": -523.5460815429688,
"logps/ref_chosen": -320.6492004394531,
"logps/ref_rejected": -273.36773681640625,
"logps/rejected": -564.8523559570312,
"loss": 4.7537,
"margin_dpo/margin_mean": 88.5877914428711,
"margin_dpo/margin_std": 140.4833984375,
"step": 306
},
{
"KL/chosen_KL_mean": -236.40028381347656,
"KL/mean": -281.1076965332031,
"KL/rejected_KL_mean": -325.81512451171875,
"KL/std": 148.4873504638672,
"epoch": 0.6429319371727749,
"fcm_dpo/beta": 0.0030362734105437994,
"fcm_dpo/delta": 0.03187718614935875,
"fcm_dpo/margin": 89.4148178100586,
"fcm_dpo/q_t": 0.4356860816478729,
"grad_norm": 75.47786712646484,
"learning_rate": 1.717018039327053e-07,
"logits/chosen": -0.7897435426712036,
"logits/rejected": -0.8119577765464783,
"logps/chosen": -515.8544921875,
"logps/ref_chosen": -279.4541931152344,
"logps/ref_rejected": -240.3796844482422,
"logps/rejected": -566.19482421875,
"loss": 4.7277,
"margin_dpo/margin_mean": 89.41481018066406,
"margin_dpo/margin_std": 140.82554626464844,
"step": 307
},
{
"KL/chosen_KL_mean": -255.62368774414062,
"KL/mean": -287.2620544433594,
"KL/rejected_KL_mean": -318.9004211425781,
"KL/std": 149.1934051513672,
"epoch": 0.6450261780104712,
"fcm_dpo/beta": 0.003119693137705326,
"fcm_dpo/delta": 0.012247689999639988,
"fcm_dpo/margin": 63.276702880859375,
"fcm_dpo/q_t": 0.45396554470062256,
"grad_norm": 68.79566955566406,
"learning_rate": 1.699652605415828e-07,
"logits/chosen": -0.8509783744812012,
"logits/rejected": -0.8624202013015747,
"logps/chosen": -552.22216796875,
"logps/ref_chosen": -296.598388671875,
"logps/ref_rejected": -258.6953430175781,
"logps/rejected": -577.5957641601562,
"loss": 5.0384,
"margin_dpo/margin_mean": 63.27670669555664,
"margin_dpo/margin_std": 155.48910522460938,
"step": 308
},
{
"KL/chosen_KL_mean": -236.53305053710938,
"KL/mean": -292.6006774902344,
"KL/rejected_KL_mean": -348.6683044433594,
"KL/std": 146.83604431152344,
"epoch": 0.6471204188481675,
"fcm_dpo/beta": 0.0031316163949668407,
"fcm_dpo/delta": 0.0024306168779730797,
"fcm_dpo/margin": 112.1352310180664,
"fcm_dpo/q_t": 0.42018356919288635,
"grad_norm": 74.16272735595703,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": -0.8479481339454651,
"logits/rejected": -0.8579108715057373,
"logps/chosen": -517.921142578125,
"logps/ref_chosen": -281.3881530761719,
"logps/ref_rejected": -262.458740234375,
"logps/rejected": -611.1270141601562,
"loss": 4.5344,
"margin_dpo/margin_mean": 112.1352310180664,
"margin_dpo/margin_std": 170.53929138183594,
"step": 309
},
{
"KL/chosen_KL_mean": -227.86915588378906,
"KL/mean": -281.9316711425781,
"KL/rejected_KL_mean": -335.99420166015625,
"KL/std": 152.35032653808594,
"epoch": 0.6492146596858639,
"fcm_dpo/beta": 0.003158912993967533,
"fcm_dpo/delta": 0.0094301737844944,
"fcm_dpo/margin": 108.12503051757812,
"fcm_dpo/q_t": 0.4215067923069,
"grad_norm": 76.43192291259766,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": -0.8392739295959473,
"logits/rejected": -0.8133707642555237,
"logps/chosen": -507.056396484375,
"logps/ref_chosen": -279.1872863769531,
"logps/ref_rejected": -261.8279724121094,
"logps/rejected": -597.8222045898438,
"loss": 4.5592,
"margin_dpo/margin_mean": 108.12503814697266,
"margin_dpo/margin_std": 162.43666076660156,
"step": 310
},
{
"KL/chosen_KL_mean": -230.60015869140625,
"KL/mean": -286.6460266113281,
"KL/rejected_KL_mean": -342.69189453125,
"KL/std": 164.47862243652344,
"epoch": 0.6513089005235602,
"fcm_dpo/beta": 0.0031031672842800617,
"fcm_dpo/delta": -0.01641257479786873,
"fcm_dpo/margin": 112.0916976928711,
"fcm_dpo/q_t": 0.42211809754371643,
"grad_norm": 68.3413314819336,
"learning_rate": 1.647817538357072e-07,
"logits/chosen": -0.8347179889678955,
"logits/rejected": -0.8130024671554565,
"logps/chosen": -501.9983215332031,
"logps/ref_chosen": -271.39813232421875,
"logps/ref_rejected": -266.12701416015625,
"logps/rejected": -608.8189086914062,
"loss": 4.6098,
"margin_dpo/margin_mean": 112.0916976928711,
"margin_dpo/margin_std": 184.5821075439453,
"step": 311
},
{
"KL/chosen_KL_mean": -227.80303955078125,
"KL/mean": -276.14068603515625,
"KL/rejected_KL_mean": -324.4783630371094,
"KL/std": 157.450439453125,
"epoch": 0.6534031413612565,
"fcm_dpo/beta": 0.0030568826477974653,
"fcm_dpo/delta": -0.028012195602059364,
"fcm_dpo/margin": 96.67533111572266,
"fcm_dpo/q_t": 0.43515488505363464,
"grad_norm": 106.50956726074219,
"learning_rate": 1.6306293495205755e-07,
"logits/chosen": -0.8552649617195129,
"logits/rejected": -0.8285101652145386,
"logps/chosen": -510.18804931640625,
"logps/ref_chosen": -282.3850402832031,
"logps/ref_rejected": -246.35389709472656,
"logps/rejected": -570.8322143554688,
"loss": 4.8594,
"margin_dpo/margin_mean": 96.67532348632812,
"margin_dpo/margin_std": 206.41854858398438,
"step": 312
},
{
"KL/chosen_KL_mean": -230.12075805664062,
"KL/mean": -276.99615478515625,
"KL/rejected_KL_mean": -323.8715515136719,
"KL/std": 160.83921813964844,
"epoch": 0.6554973821989529,
"fcm_dpo/beta": 0.003049192950129509,
"fcm_dpo/delta": -0.01281303446739912,
"fcm_dpo/margin": 93.75079345703125,
"fcm_dpo/q_t": 0.43882137537002563,
"grad_norm": 66.034912109375,
"learning_rate": 1.6134877823936607e-07,
"logits/chosen": -0.8895430564880371,
"logits/rejected": -0.8738856911659241,
"logps/chosen": -533.7515869140625,
"logps/ref_chosen": -303.630859375,
"logps/ref_rejected": -273.1156921386719,
"logps/rejected": -596.9872436523438,
"loss": 4.8408,
"margin_dpo/margin_mean": 93.75078582763672,
"margin_dpo/margin_std": 189.20486450195312,
"step": 313
},
{
"KL/chosen_KL_mean": -223.54823303222656,
"KL/mean": -268.31048583984375,
"KL/rejected_KL_mean": -313.0727233886719,
"KL/std": 137.15122985839844,
"epoch": 0.6575916230366492,
"fcm_dpo/beta": 0.003023975295946002,
"fcm_dpo/delta": 0.006585121154785156,
"fcm_dpo/margin": 89.52447509765625,
"fcm_dpo/q_t": 0.43667665123939514,
"grad_norm": 73.49964141845703,
"learning_rate": 1.5963937562265522e-07,
"logits/chosen": -0.8998604416847229,
"logits/rejected": -0.8924080729484558,
"logps/chosen": -525.8524780273438,
"logps/ref_chosen": -302.3042907714844,
"logps/ref_rejected": -273.6416015625,
"logps/rejected": -586.71435546875,
"loss": 4.7536,
"margin_dpo/margin_mean": 89.52449035644531,
"margin_dpo/margin_std": 151.22608947753906,
"step": 314
},
{
"KL/chosen_KL_mean": -219.18116760253906,
"KL/mean": -277.6825256347656,
"KL/rejected_KL_mean": -336.18389892578125,
"KL/std": 145.4262237548828,
"epoch": 0.6596858638743456,
"fcm_dpo/beta": 0.0030265850946307182,
"fcm_dpo/delta": -0.0040887510403990746,
"fcm_dpo/margin": 117.00270080566406,
"fcm_dpo/q_t": 0.41837024688720703,
"grad_norm": 76.63323211669922,
"learning_rate": 1.5793481877199943e-07,
"logits/chosen": -0.8940035700798035,
"logits/rejected": -0.8783378005027771,
"logps/chosen": -521.9104614257812,
"logps/ref_chosen": -302.729248046875,
"logps/ref_rejected": -270.26910400390625,
"logps/rejected": -606.4529418945312,
"loss": 4.5038,
"margin_dpo/margin_mean": 117.00270080566406,
"margin_dpo/margin_std": 163.93301391601562,
"step": 315
},
{
"KL/chosen_KL_mean": -229.85618591308594,
"KL/mean": -278.86376953125,
"KL/rejected_KL_mean": -327.8713684082031,
"KL/std": 147.73422241210938,
"epoch": 0.6617801047120419,
"fcm_dpo/beta": 0.003056867513805628,
"fcm_dpo/delta": -0.0008974131196737289,
"fcm_dpo/margin": 98.01517486572266,
"fcm_dpo/q_t": 0.4309207797050476,
"grad_norm": 59.27046585083008,
"learning_rate": 1.562351990976095e-07,
"logits/chosen": -0.8997865915298462,
"logits/rejected": -0.8820158243179321,
"logps/chosen": -540.4267578125,
"logps/ref_chosen": -310.5706481933594,
"logps/ref_rejected": -272.9354553222656,
"logps/rejected": -600.8068237304688,
"loss": 4.7006,
"margin_dpo/margin_mean": 98.01517486572266,
"margin_dpo/margin_std": 166.4867401123047,
"step": 316
},
{
"KL/chosen_KL_mean": -198.8494873046875,
"KL/mean": -245.5550537109375,
"KL/rejected_KL_mean": -292.2606506347656,
"KL/std": 128.78086853027344,
"epoch": 0.6638743455497382,
"fcm_dpo/beta": 0.0030271485447883606,
"fcm_dpo/delta": 0.016471808776259422,
"fcm_dpo/margin": 93.41114807128906,
"fcm_dpo/q_t": 0.43299350142478943,
"grad_norm": 58.140594482421875,
"learning_rate": 1.5454060774493065e-07,
"logits/chosen": -0.8841041326522827,
"logits/rejected": -0.8535327911376953,
"logps/chosen": -452.7498474121094,
"logps/ref_chosen": -253.90036010742188,
"logps/ref_rejected": -218.74078369140625,
"logps/rejected": -511.00140380859375,
"loss": 4.6731,
"margin_dpo/margin_mean": 93.41114807128906,
"margin_dpo/margin_std": 136.35061645507812,
"step": 317
},
{
"KL/chosen_KL_mean": -207.8227996826172,
"KL/mean": -265.3223876953125,
"KL/rejected_KL_mean": -322.82196044921875,
"KL/std": 141.49267578125,
"epoch": 0.6659685863874345,
"fcm_dpo/beta": 0.003031826112419367,
"fcm_dpo/delta": -0.009629029780626297,
"fcm_dpo/margin": 114.99917602539062,
"fcm_dpo/q_t": 0.4194708466529846,
"grad_norm": 75.79177856445312,
"learning_rate": 1.5285113558975427e-07,
"logits/chosen": -0.8975743055343628,
"logits/rejected": -0.8688497543334961,
"logps/chosen": -478.6456604003906,
"logps/ref_chosen": -270.8228759765625,
"logps/ref_rejected": -255.30972290039062,
"logps/rejected": -578.1317138671875,
"loss": 4.5028,
"margin_dpo/margin_mean": 114.99917602539062,
"margin_dpo/margin_std": 158.4157257080078,
"step": 318
},
{
"KL/chosen_KL_mean": -224.45407104492188,
"KL/mean": -277.3719787597656,
"KL/rejected_KL_mean": -330.28985595703125,
"KL/std": 138.86293029785156,
"epoch": 0.6680628272251309,
"fcm_dpo/beta": 0.0030600870959460735,
"fcm_dpo/delta": 0.018474522978067398,
"fcm_dpo/margin": 105.83576202392578,
"fcm_dpo/q_t": 0.4230358898639679,
"grad_norm": 106.81951141357422,
"learning_rate": 1.5116687323334464e-07,
"logits/chosen": -0.8801316022872925,
"logits/rejected": -0.8496555685997009,
"logps/chosen": -525.4569091796875,
"logps/ref_chosen": -301.0028076171875,
"logps/ref_rejected": -242.39002990722656,
"logps/rejected": -572.6798706054688,
"loss": 4.5445,
"margin_dpo/margin_mean": 105.83576202392578,
"margin_dpo/margin_std": 139.6443328857422,
"step": 319
},
{
"KL/chosen_KL_mean": -224.6094512939453,
"KL/mean": -275.99322509765625,
"KL/rejected_KL_mean": -327.37701416015625,
"KL/std": 150.7802734375,
"epoch": 0.6701570680628273,
"fcm_dpo/beta": 0.0030319918878376484,
"fcm_dpo/delta": -0.024170946329832077,
"fcm_dpo/margin": 102.76758575439453,
"fcm_dpo/q_t": 0.4289899170398712,
"grad_norm": 80.37665557861328,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": -0.8487673401832581,
"logits/rejected": -0.8394469618797302,
"logps/chosen": -528.2320556640625,
"logps/ref_chosen": -303.6225891113281,
"logps/ref_rejected": -280.85174560546875,
"logps/rejected": -608.228759765625,
"loss": 4.6669,
"margin_dpo/margin_mean": 102.76758575439453,
"margin_dpo/margin_std": 172.9017333984375,
"step": 320
},
{
"KL/chosen_KL_mean": -247.0533447265625,
"KL/mean": -284.8410339355469,
"KL/rejected_KL_mean": -322.62872314453125,
"KL/std": 149.05075073242188,
"epoch": 0.6722513089005235,
"fcm_dpo/beta": 0.0030333874747157097,
"fcm_dpo/delta": 0.012165244668722153,
"fcm_dpo/margin": 75.57539367675781,
"fcm_dpo/q_t": 0.44790154695510864,
"grad_norm": 91.25624084472656,
"learning_rate": 1.478143389201113e-07,
"logits/chosen": -0.8742708563804626,
"logits/rejected": -0.8403862714767456,
"logps/chosen": -536.0391845703125,
"logps/ref_chosen": -288.98583984375,
"logps/ref_rejected": -241.1822052001953,
"logps/rejected": -563.8109741210938,
"loss": 4.9434,
"margin_dpo/margin_mean": 75.57539367675781,
"margin_dpo/margin_std": 165.88363647460938,
"step": 321
},
{
"KL/chosen_KL_mean": -255.3909912109375,
"KL/mean": -299.4102783203125,
"KL/rejected_KL_mean": -343.4295349121094,
"KL/std": 151.14059448242188,
"epoch": 0.6743455497382199,
"fcm_dpo/beta": 0.003069436876103282,
"fcm_dpo/delta": 0.02577303536236286,
"fcm_dpo/margin": 88.03853607177734,
"fcm_dpo/q_t": 0.43747204542160034,
"grad_norm": 70.48045349121094,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": -0.9231587052345276,
"logits/rejected": -0.8860714435577393,
"logps/chosen": -563.9344482421875,
"logps/ref_chosen": -308.54345703125,
"logps/ref_rejected": -269.7995910644531,
"logps/rejected": -613.2291259765625,
"loss": 4.8283,
"margin_dpo/margin_mean": 88.03853607177734,
"margin_dpo/margin_std": 176.32192993164062,
"step": 322
},
{
"KL/chosen_KL_mean": -243.8632354736328,
"KL/mean": -279.10198974609375,
"KL/rejected_KL_mean": -314.3406677246094,
"KL/std": 137.9376983642578,
"epoch": 0.6764397905759162,
"fcm_dpo/beta": 0.003103929106146097,
"fcm_dpo/delta": -0.00034204451367259026,
"fcm_dpo/margin": 70.47740936279297,
"fcm_dpo/q_t": 0.4506058394908905,
"grad_norm": 71.5815200805664,
"learning_rate": 1.4448372394055246e-07,
"logits/chosen": -0.8818837404251099,
"logits/rejected": -0.8731890320777893,
"logps/chosen": -526.3568725585938,
"logps/ref_chosen": -282.49365234375,
"logps/ref_rejected": -227.7105255126953,
"logps/rejected": -542.0511474609375,
"loss": 5.0069,
"margin_dpo/margin_mean": 70.4774169921875,
"margin_dpo/margin_std": 171.1042022705078,
"step": 323
},
{
"KL/chosen_KL_mean": -218.2962188720703,
"KL/mean": -277.8293151855469,
"KL/rejected_KL_mean": -337.3623962402344,
"KL/std": 150.20140075683594,
"epoch": 0.6785340314136126,
"fcm_dpo/beta": 0.0030384839046746492,
"fcm_dpo/delta": -0.01760367676615715,
"fcm_dpo/margin": 119.06621551513672,
"fcm_dpo/q_t": 0.41548827290534973,
"grad_norm": 79.55120086669922,
"learning_rate": 1.428268596492364e-07,
"logits/chosen": -0.8202590346336365,
"logits/rejected": -0.8138710260391235,
"logps/chosen": -457.63458251953125,
"logps/ref_chosen": -239.33836364746094,
"logps/ref_rejected": -230.53775024414062,
"logps/rejected": -567.900146484375,
"loss": 4.4636,
"margin_dpo/margin_mean": 119.06622314453125,
"margin_dpo/margin_std": 155.9190673828125,
"step": 324
},
{
"KL/chosen_KL_mean": -250.08969116210938,
"KL/mean": -300.6783142089844,
"KL/rejected_KL_mean": -351.26702880859375,
"KL/std": 157.67953491210938,
"epoch": 0.680628272251309,
"fcm_dpo/beta": 0.003035879461094737,
"fcm_dpo/delta": -0.01274973526597023,
"fcm_dpo/margin": 101.17728424072266,
"fcm_dpo/q_t": 0.4310661554336548,
"grad_norm": 85.70771789550781,
"learning_rate": 1.4117574272818386e-07,
"logits/chosen": -0.826617956161499,
"logits/rejected": -0.8100321888923645,
"logps/chosen": -530.7186889648438,
"logps/ref_chosen": -280.62896728515625,
"logps/ref_rejected": -270.5085754394531,
"logps/rejected": -621.7755737304688,
"loss": 4.7572,
"margin_dpo/margin_mean": 101.17729187011719,
"margin_dpo/margin_std": 193.80123901367188,
"step": 325
},
{
"KL/chosen_KL_mean": -235.35711669921875,
"KL/mean": -287.36279296875,
"KL/rejected_KL_mean": -339.36846923828125,
"KL/std": 148.6804656982422,
"epoch": 0.6827225130890052,
"fcm_dpo/beta": 0.0030440862756222486,
"fcm_dpo/delta": -0.008422894403338432,
"fcm_dpo/margin": 104.01131439208984,
"fcm_dpo/q_t": 0.42731618881225586,
"grad_norm": 72.86016082763672,
"learning_rate": 1.3953046172178413e-07,
"logits/chosen": -0.9463378190994263,
"logits/rejected": -0.9375435709953308,
"logps/chosen": -476.3442687988281,
"logps/ref_chosen": -240.9871368408203,
"logps/ref_rejected": -261.0238342285156,
"logps/rejected": -600.3922729492188,
"loss": 4.6561,
"margin_dpo/margin_mean": 104.01131439208984,
"margin_dpo/margin_std": 171.9400634765625,
"step": 326
},
{
"KL/chosen_KL_mean": -227.19876098632812,
"KL/mean": -282.76019287109375,
"KL/rejected_KL_mean": -338.32159423828125,
"KL/std": 140.7682342529297,
"epoch": 0.6848167539267016,
"fcm_dpo/beta": 0.002978120930492878,
"fcm_dpo/delta": -0.02210163325071335,
"fcm_dpo/margin": 111.1228256225586,
"fcm_dpo/q_t": 0.4241907000541687,
"grad_norm": 72.43849182128906,
"learning_rate": 1.3789110486146468e-07,
"logits/chosen": -0.8795799016952515,
"logits/rejected": -0.8552607893943787,
"logps/chosen": -506.71881103515625,
"logps/ref_chosen": -279.52001953125,
"logps/ref_rejected": -269.51824951171875,
"logps/rejected": -607.8397827148438,
"loss": 4.5967,
"margin_dpo/margin_mean": 111.12281799316406,
"margin_dpo/margin_std": 169.77825927734375,
"step": 327
},
{
"KL/chosen_KL_mean": -247.49966430664062,
"KL/mean": -300.46051025390625,
"KL/rejected_KL_mean": -353.42138671875,
"KL/std": 146.58609008789062,
"epoch": 0.6869109947643979,
"fcm_dpo/beta": 0.002920475322753191,
"fcm_dpo/delta": -0.01834581233561039,
"fcm_dpo/margin": 105.92172241210938,
"fcm_dpo/q_t": 0.4280359447002411,
"grad_norm": 65.31753540039062,
"learning_rate": 1.362577600609588e-07,
"logits/chosen": -0.8584927320480347,
"logits/rejected": -0.8567357063293457,
"logps/chosen": -548.5330810546875,
"logps/ref_chosen": -301.033447265625,
"logps/ref_rejected": -284.2101135253906,
"logps/rejected": -637.6315307617188,
"loss": 4.6345,
"margin_dpo/margin_mean": 105.92172241210938,
"margin_dpo/margin_std": 160.9266357421875,
"step": 328
},
{
"KL/chosen_KL_mean": -265.75604248046875,
"KL/mean": -323.16357421875,
"KL/rejected_KL_mean": -380.5710754394531,
"KL/std": 184.89715576171875,
"epoch": 0.6890052356020943,
"fcm_dpo/beta": 0.002896467689424753,
"fcm_dpo/delta": -0.018565185368061066,
"fcm_dpo/margin": 114.81501770019531,
"fcm_dpo/q_t": 0.42783403396606445,
"grad_norm": 60.461456298828125,
"learning_rate": 1.3463051491159093e-07,
"logits/chosen": -0.8645045161247253,
"logits/rejected": -0.8325575590133667,
"logps/chosen": -585.744873046875,
"logps/ref_chosen": -319.9888610839844,
"logps/ref_rejected": -307.5588684082031,
"logps/rejected": -688.1299438476562,
"loss": 4.6877,
"margin_dpo/margin_mean": 114.81501770019531,
"margin_dpo/margin_std": 206.36744689941406,
"step": 329
},
{
"KL/chosen_KL_mean": -262.357177734375,
"KL/mean": -316.2622985839844,
"KL/rejected_KL_mean": -370.1673583984375,
"KL/std": 162.6302490234375,
"epoch": 0.6910994764397905,
"fcm_dpo/beta": 0.002852683886885643,
"fcm_dpo/delta": -0.024464130401611328,
"fcm_dpo/margin": 107.81021118164062,
"fcm_dpo/q_t": 0.42945441603660583,
"grad_norm": 61.41120529174805,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": -0.8586999177932739,
"logits/rejected": -0.8646368384361267,
"logps/chosen": -563.471923828125,
"logps/ref_chosen": -301.11474609375,
"logps/ref_rejected": -299.673095703125,
"logps/rejected": -669.8404541015625,
"loss": 4.6601,
"margin_dpo/margin_mean": 107.8102035522461,
"margin_dpo/margin_std": 173.99789428710938,
"step": 330
},
{
"KL/chosen_KL_mean": -256.2668151855469,
"KL/mean": -304.8879089355469,
"KL/rejected_KL_mean": -353.509033203125,
"KL/std": 162.7855987548828,
"epoch": 0.6931937172774869,
"fcm_dpo/beta": 0.002842208370566368,
"fcm_dpo/delta": 0.018526801839470863,
"fcm_dpo/margin": 97.24219512939453,
"fcm_dpo/q_t": 0.4376268982887268,
"grad_norm": 100.0017318725586,
"learning_rate": 1.3139467229135998e-07,
"logits/chosen": -0.8795989155769348,
"logits/rejected": -0.8588756322860718,
"logps/chosen": -533.8583374023438,
"logps/ref_chosen": -277.59149169921875,
"logps/ref_rejected": -256.025634765625,
"logps/rejected": -609.53466796875,
"loss": 4.8138,
"margin_dpo/margin_mean": 97.24219512939453,
"margin_dpo/margin_std": 195.071044921875,
"step": 331
},
{
"KL/chosen_KL_mean": -267.27197265625,
"KL/mean": -321.0538024902344,
"KL/rejected_KL_mean": -374.8355712890625,
"KL/std": 159.4349365234375,
"epoch": 0.6952879581151833,
"fcm_dpo/beta": 0.0028672670014202595,
"fcm_dpo/delta": 0.008170833811163902,
"fcm_dpo/margin": 107.56364440917969,
"fcm_dpo/q_t": 0.429614782333374,
"grad_norm": 68.8551025390625,
"learning_rate": 1.2978624834891626e-07,
"logits/chosen": -0.8785598874092102,
"logits/rejected": -0.8509567379951477,
"logps/chosen": -537.2456665039062,
"logps/ref_chosen": -269.97369384765625,
"logps/ref_rejected": -235.03164672851562,
"logps/rejected": -609.8673095703125,
"loss": 4.7519,
"margin_dpo/margin_mean": 107.56364440917969,
"margin_dpo/margin_std": 203.162109375,
"step": 332
},
{
"KL/chosen_KL_mean": -265.45159912109375,
"KL/mean": -310.3097229003906,
"KL/rejected_KL_mean": -355.1678466796875,
"KL/std": 151.80691528320312,
"epoch": 0.6973821989528796,
"fcm_dpo/beta": 0.0028877186123281717,
"fcm_dpo/delta": 0.016247637569904327,
"fcm_dpo/margin": 89.71623992919922,
"fcm_dpo/q_t": 0.44060900807380676,
"grad_norm": 65.02811431884766,
"learning_rate": 1.281842711051438e-07,
"logits/chosen": -0.9548963308334351,
"logits/rejected": -0.9161214232444763,
"logps/chosen": -562.214599609375,
"logps/ref_chosen": -296.76300048828125,
"logps/ref_rejected": -265.97991943359375,
"logps/rejected": -621.1477661132812,
"loss": 4.8365,
"margin_dpo/margin_mean": 89.71623229980469,
"margin_dpo/margin_std": 177.90646362304688,
"step": 333
},
{
"KL/chosen_KL_mean": -252.63279724121094,
"KL/mean": -302.4210205078125,
"KL/rejected_KL_mean": -352.20928955078125,
"KL/std": 154.0194854736328,
"epoch": 0.6994764397905759,
"fcm_dpo/beta": 0.002904269378632307,
"fcm_dpo/delta": -0.021205034106969833,
"fcm_dpo/margin": 99.57649993896484,
"fcm_dpo/q_t": 0.4326726198196411,
"grad_norm": 75.01168823242188,
"learning_rate": 1.2658882646922033e-07,
"logits/chosen": -0.8493959903717041,
"logits/rejected": -0.8207319378852844,
"logps/chosen": -553.6695556640625,
"logps/ref_chosen": -301.0367431640625,
"logps/ref_rejected": -268.87652587890625,
"logps/rejected": -621.0858154296875,
"loss": 4.7444,
"margin_dpo/margin_mean": 99.57649993896484,
"margin_dpo/margin_std": 173.79196166992188,
"step": 334
},
{
"KL/chosen_KL_mean": -280.1068115234375,
"KL/mean": -328.50701904296875,
"KL/rejected_KL_mean": -376.90716552734375,
"KL/std": 143.94677734375,
"epoch": 0.7015706806282722,
"fcm_dpo/beta": 0.002826720243319869,
"fcm_dpo/delta": -0.024722017347812653,
"fcm_dpo/margin": 96.80036926269531,
"fcm_dpo/q_t": 0.43696001172065735,
"grad_norm": 82.47087860107422,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.8456929922103882,
"logits/rejected": -0.8398817777633667,
"logps/chosen": -556.239501953125,
"logps/ref_chosen": -276.13275146484375,
"logps/ref_rejected": -243.44203186035156,
"logps/rejected": -620.3491821289062,
"loss": 4.7968,
"margin_dpo/margin_mean": 96.80036926269531,
"margin_dpo/margin_std": 181.85284423828125,
"step": 335
},
{
"KL/chosen_KL_mean": -272.6168212890625,
"KL/mean": -323.9999694824219,
"KL/rejected_KL_mean": -375.383056640625,
"KL/std": 182.46585083007812,
"epoch": 0.7036649214659686,
"fcm_dpo/beta": 0.0028306483291089535,
"fcm_dpo/delta": 0.011404670774936676,
"fcm_dpo/margin": 102.76631164550781,
"fcm_dpo/q_t": 0.4366564452648163,
"grad_norm": 60.21648025512695,
"learning_rate": 1.2341787690142435e-07,
"logits/chosen": -0.8676019906997681,
"logits/rejected": -0.8084288835525513,
"logps/chosen": -518.8794555664062,
"logps/ref_chosen": -246.2626495361328,
"logps/ref_rejected": -261.0617980957031,
"logps/rejected": -636.4448852539062,
"loss": 4.791,
"margin_dpo/margin_mean": 102.76629638671875,
"margin_dpo/margin_std": 199.2318115234375,
"step": 336
},
{
"KL/chosen_KL_mean": -257.2256164550781,
"KL/mean": -313.9550476074219,
"KL/rejected_KL_mean": -370.6844177246094,
"KL/std": 151.99803161621094,
"epoch": 0.7057591623036649,
"fcm_dpo/beta": 0.0028662250842899084,
"fcm_dpo/delta": -0.0013907505199313164,
"fcm_dpo/margin": 113.45879364013672,
"fcm_dpo/q_t": 0.42453762888908386,
"grad_norm": 79.62832641601562,
"learning_rate": 1.2184254201795363e-07,
"logits/chosen": -0.8662607669830322,
"logits/rejected": -0.8372821807861328,
"logps/chosen": -524.2194213867188,
"logps/ref_chosen": -266.9937744140625,
"logps/ref_rejected": -253.015625,
"logps/rejected": -623.7000732421875,
"loss": 4.599,
"margin_dpo/margin_mean": 113.45879364013672,
"margin_dpo/margin_std": 171.08424377441406,
"step": 337
},
{
"KL/chosen_KL_mean": -248.36929321289062,
"KL/mean": -305.20367431640625,
"KL/rejected_KL_mean": -362.03802490234375,
"KL/std": 163.98251342773438,
"epoch": 0.7078534031413612,
"fcm_dpo/beta": 0.0028280254919081926,
"fcm_dpo/delta": 0.005961664952337742,
"fcm_dpo/margin": 113.66871643066406,
"fcm_dpo/q_t": 0.4267999529838562,
"grad_norm": 75.13379669189453,
"learning_rate": 1.202740798300168e-07,
"logits/chosen": -0.8829590678215027,
"logits/rejected": -0.86571204662323,
"logps/chosen": -524.9619140625,
"logps/ref_chosen": -276.5925598144531,
"logps/ref_rejected": -233.979248046875,
"logps/rejected": -596.0172119140625,
"loss": 4.6272,
"margin_dpo/margin_mean": 113.66871643066406,
"margin_dpo/margin_std": 182.07052612304688,
"step": 338
},
{
"KL/chosen_KL_mean": -256.4414978027344,
"KL/mean": -320.28216552734375,
"KL/rejected_KL_mean": -384.1228332519531,
"KL/std": 166.31675720214844,
"epoch": 0.7099476439790576,
"fcm_dpo/beta": 0.002872914308682084,
"fcm_dpo/delta": -0.01606258563697338,
"fcm_dpo/margin": 127.6812973022461,
"fcm_dpo/q_t": 0.4165673553943634,
"grad_norm": 53.9896125793457,
"learning_rate": 1.1871257444948096e-07,
"logits/chosen": -0.9026840925216675,
"logits/rejected": -0.88820880651474,
"logps/chosen": -559.96923828125,
"logps/ref_chosen": -303.5277404785156,
"logps/ref_rejected": -283.11676025390625,
"logps/rejected": -667.239501953125,
"loss": 4.5267,
"margin_dpo/margin_mean": 127.68128967285156,
"margin_dpo/margin_std": 196.2003173828125,
"step": 339
},
{
"KL/chosen_KL_mean": -263.4041748046875,
"KL/mean": -310.89080810546875,
"KL/rejected_KL_mean": -358.37738037109375,
"KL/std": 174.05343627929688,
"epoch": 0.7120418848167539,
"fcm_dpo/beta": 0.002833343343809247,
"fcm_dpo/delta": 0.010575986467301846,
"fcm_dpo/margin": 94.97323608398438,
"fcm_dpo/q_t": 0.4373157024383545,
"grad_norm": 76.64401245117188,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": -0.83310467004776,
"logits/rejected": -0.8270291090011597,
"logps/chosen": -524.929931640625,
"logps/ref_chosen": -261.5257568359375,
"logps/ref_rejected": -259.39862060546875,
"logps/rejected": -617.7760009765625,
"loss": 4.9296,
"margin_dpo/margin_mean": 94.97323608398438,
"margin_dpo/margin_std": 215.53196716308594,
"step": 340
},
{
"KL/chosen_KL_mean": -300.6253662109375,
"KL/mean": -347.34564208984375,
"KL/rejected_KL_mean": -394.0658264160156,
"KL/std": 195.77174377441406,
"epoch": 0.7141361256544503,
"fcm_dpo/beta": 0.002833633916452527,
"fcm_dpo/delta": -0.014535056427121162,
"fcm_dpo/margin": 93.44046020507812,
"fcm_dpo/q_t": 0.4412190616130829,
"grad_norm": 72.96549987792969,
"learning_rate": 1.1561076868822755e-07,
"logits/chosen": -0.8564797043800354,
"logits/rejected": -0.8294931650161743,
"logps/chosen": -616.5289306640625,
"logps/ref_chosen": -315.903564453125,
"logps/ref_rejected": -308.02392578125,
"logps/rejected": -702.0897827148438,
"loss": 4.9579,
"margin_dpo/margin_mean": 93.44046020507812,
"margin_dpo/margin_std": 222.5795440673828,
"step": 341
},
{
"KL/chosen_KL_mean": -252.14891052246094,
"KL/mean": -301.0910949707031,
"KL/rejected_KL_mean": -350.0333251953125,
"KL/std": 153.94277954101562,
"epoch": 0.7162303664921466,
"fcm_dpo/beta": 0.002802258124575019,
"fcm_dpo/delta": -0.010795130394399166,
"fcm_dpo/margin": 97.88438415527344,
"fcm_dpo/q_t": 0.4367576539516449,
"grad_norm": 81.44354248046875,
"learning_rate": 1.1407063464793965e-07,
"logits/chosen": -0.8587002754211426,
"logits/rejected": -0.8541080355644226,
"logps/chosen": -521.3275756835938,
"logps/ref_chosen": -269.17864990234375,
"logps/ref_rejected": -260.8977355957031,
"logps/rejected": -610.9310913085938,
"loss": 4.7627,
"margin_dpo/margin_mean": 97.88438415527344,
"margin_dpo/margin_std": 172.59506225585938,
"step": 342
},
{
"KL/chosen_KL_mean": -263.67144775390625,
"KL/mean": -303.8117370605469,
"KL/rejected_KL_mean": -343.95196533203125,
"KL/std": 153.24267578125,
"epoch": 0.7183246073298429,
"fcm_dpo/beta": 0.0027811508625745773,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 80.28048706054688,
"fcm_dpo/q_t": 0.4475141167640686,
"grad_norm": 61.77683639526367,
"learning_rate": 1.125377900869913e-07,
"logits/chosen": -0.8490421772003174,
"logits/rejected": -0.8213639259338379,
"logps/chosen": -574.3914794921875,
"logps/ref_chosen": -310.719970703125,
"logps/ref_rejected": -263.5224914550781,
"logps/rejected": -607.4744873046875,
"loss": 4.9068,
"margin_dpo/margin_mean": 80.2804946899414,
"margin_dpo/margin_std": 163.10105895996094,
"step": 343
},
{
"KL/chosen_KL_mean": -261.09722900390625,
"KL/mean": -305.764404296875,
"KL/rejected_KL_mean": -350.4315490722656,
"KL/std": 147.62881469726562,
"epoch": 0.7204188481675393,
"fcm_dpo/beta": 0.0027887485921382904,
"fcm_dpo/delta": 0.008171796798706055,
"fcm_dpo/margin": 89.3343505859375,
"fcm_dpo/q_t": 0.44169238209724426,
"grad_norm": 75.60264587402344,
"learning_rate": 1.110123172071844e-07,
"logits/chosen": -0.8601065874099731,
"logits/rejected": -0.8441787958145142,
"logps/chosen": -562.8970947265625,
"logps/ref_chosen": -301.7999267578125,
"logps/ref_rejected": -257.9061584472656,
"logps/rejected": -608.3377075195312,
"loss": 4.8542,
"margin_dpo/margin_mean": 89.33435821533203,
"margin_dpo/margin_std": 173.32949829101562,
"step": 344
},
{
"KL/chosen_KL_mean": -239.99986267089844,
"KL/mean": -284.1513366699219,
"KL/rejected_KL_mean": -328.30279541015625,
"KL/std": 147.23294067382812,
"epoch": 0.7225130890052356,
"fcm_dpo/beta": 0.002783895004540682,
"fcm_dpo/delta": -0.00986623577773571,
"fcm_dpo/margin": 88.30291748046875,
"fcm_dpo/q_t": 0.4421829581260681,
"grad_norm": 68.73944091796875,
"learning_rate": 1.09494297815e-07,
"logits/chosen": -0.8500536680221558,
"logits/rejected": -0.8508837223052979,
"logps/chosen": -523.018310546875,
"logps/ref_chosen": -283.0184326171875,
"logps/ref_rejected": -266.8457336425781,
"logps/rejected": -595.1485595703125,
"loss": 4.8153,
"margin_dpo/margin_mean": 88.30291748046875,
"margin_dpo/margin_std": 145.37179565429688,
"step": 345
},
{
"KL/chosen_KL_mean": -255.714111328125,
"KL/mean": -312.4100036621094,
"KL/rejected_KL_mean": -369.10595703125,
"KL/std": 172.77389526367188,
"epoch": 0.724607329842932,
"fcm_dpo/beta": 0.0027374359779059887,
"fcm_dpo/delta": -0.02758317068219185,
"fcm_dpo/margin": 113.39186096191406,
"fcm_dpo/q_t": 0.42870837450027466,
"grad_norm": 54.13254928588867,
"learning_rate": 1.0798381331721107e-07,
"logits/chosen": -0.936150848865509,
"logits/rejected": -0.8845096230506897,
"logps/chosen": -524.1553344726562,
"logps/ref_chosen": -268.44122314453125,
"logps/ref_rejected": -227.8225860595703,
"logps/rejected": -596.9285278320312,
"loss": 4.6752,
"margin_dpo/margin_mean": 113.3918685913086,
"margin_dpo/margin_std": 180.54847717285156,
"step": 346
},
{
"KL/chosen_KL_mean": -238.4910125732422,
"KL/mean": -283.69293212890625,
"KL/rejected_KL_mean": -328.8948669433594,
"KL/std": 144.58145141601562,
"epoch": 0.7267015706806282,
"fcm_dpo/beta": 0.0027357658836990595,
"fcm_dpo/delta": 0.012428262270987034,
"fcm_dpo/margin": 90.40386199951172,
"fcm_dpo/q_t": 0.44158247113227844,
"grad_norm": 50.78943634033203,
"learning_rate": 1.0648094471651722e-07,
"logits/chosen": -0.783841609954834,
"logits/rejected": -0.8095725178718567,
"logps/chosen": -512.194580078125,
"logps/ref_chosen": -273.70355224609375,
"logps/ref_rejected": -243.65521240234375,
"logps/rejected": -572.550048828125,
"loss": 4.8291,
"margin_dpo/margin_mean": 90.40386199951172,
"margin_dpo/margin_std": 160.6577911376953,
"step": 347
},
{
"KL/chosen_KL_mean": -237.40325927734375,
"KL/mean": -279.7298583984375,
"KL/rejected_KL_mean": -322.0564880371094,
"KL/std": 156.852783203125,
"epoch": 0.7287958115183246,
"fcm_dpo/beta": 0.0027791352476924658,
"fcm_dpo/delta": 0.022717544808983803,
"fcm_dpo/margin": 84.6531982421875,
"fcm_dpo/q_t": 0.4474527835845947,
"grad_norm": 81.65412139892578,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": -0.8968985676765442,
"logits/rejected": -0.8811699748039246,
"logps/chosen": -523.044677734375,
"logps/ref_chosen": -285.64141845703125,
"logps/ref_rejected": -265.6270446777344,
"logps/rejected": -587.6835327148438,
"loss": 4.9125,
"margin_dpo/margin_mean": 84.6531982421875,
"margin_dpo/margin_std": 180.24642944335938,
"step": 348
},
{
"KL/chosen_KL_mean": -240.90472412109375,
"KL/mean": -305.62078857421875,
"KL/rejected_KL_mean": -370.3368225097656,
"KL/std": 170.03091430664062,
"epoch": 0.7308900523560209,
"fcm_dpo/beta": 0.002779011381790042,
"fcm_dpo/delta": -0.02953748032450676,
"fcm_dpo/margin": 129.43206787109375,
"fcm_dpo/q_t": 0.4158233404159546,
"grad_norm": 68.60745239257812,
"learning_rate": 1.0349837717080347e-07,
"logits/chosen": -0.8537794351577759,
"logits/rejected": -0.8392589092254639,
"logps/chosen": -569.2222290039062,
"logps/ref_chosen": -328.3175048828125,
"logps/ref_rejected": -292.37872314453125,
"logps/rejected": -662.715576171875,
"loss": 4.5281,
"margin_dpo/margin_mean": 129.43206787109375,
"margin_dpo/margin_std": 191.28211975097656,
"step": 349
},
{
"KL/chosen_KL_mean": -255.6253662109375,
"KL/mean": -307.3200378417969,
"KL/rejected_KL_mean": -359.0146179199219,
"KL/std": 162.4384002685547,
"epoch": 0.7329842931937173,
"fcm_dpo/beta": 0.0027364445850253105,
"fcm_dpo/delta": 0.014540414325892925,
"fcm_dpo/margin": 103.38926696777344,
"fcm_dpo/q_t": 0.4352550506591797,
"grad_norm": 70.40982818603516,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": -0.8301476240158081,
"logits/rejected": -0.8409773111343384,
"logps/chosen": -548.4300537109375,
"logps/ref_chosen": -292.8046569824219,
"logps/ref_rejected": -250.35504150390625,
"logps/rejected": -609.3696899414062,
"loss": 4.7581,
"margin_dpo/margin_mean": 103.38926696777344,
"margin_dpo/margin_std": 189.09571838378906,
"step": 350
},
{
"KL/chosen_KL_mean": -248.666748046875,
"KL/mean": -277.6920166015625,
"KL/rejected_KL_mean": -306.71728515625,
"KL/std": 148.222900390625,
"epoch": 0.7350785340314137,
"fcm_dpo/beta": 0.002761303447186947,
"fcm_dpo/delta": 0.001408736570738256,
"fcm_dpo/margin": 58.05055618286133,
"fcm_dpo/q_t": 0.4626092314720154,
"grad_norm": 71.92450714111328,
"learning_rate": 1.0054723495346482e-07,
"logits/chosen": -0.9074581861495972,
"logits/rejected": -0.887728214263916,
"logps/chosen": -560.5557861328125,
"logps/ref_chosen": -311.8890380859375,
"logps/ref_rejected": -263.59033203125,
"logps/rejected": -570.3076171875,
"loss": 5.1991,
"margin_dpo/margin_mean": 58.05055618286133,
"margin_dpo/margin_std": 185.35238647460938,
"step": 351
},
{
"KL/chosen_KL_mean": -232.7962188720703,
"KL/mean": -293.2824401855469,
"KL/rejected_KL_mean": -353.7686767578125,
"KL/std": 162.10598754882812,
"epoch": 0.7371727748691099,
"fcm_dpo/beta": 0.0027385905850678682,
"fcm_dpo/delta": -0.024517323821783066,
"fcm_dpo/margin": 120.97250366210938,
"fcm_dpo/q_t": 0.4226635694503784,
"grad_norm": 84.17012023925781,
"learning_rate": 9.908364643332398e-08,
"logits/chosen": -0.8231208324432373,
"logits/rejected": -0.7927186489105225,
"logps/chosen": -487.7040710449219,
"logps/ref_chosen": -254.9078826904297,
"logps/ref_rejected": -257.1688232421875,
"logps/rejected": -610.9375,
"loss": 4.5778,
"margin_dpo/margin_mean": 120.97249603271484,
"margin_dpo/margin_std": 177.3806610107422,
"step": 352
},
{
"KL/chosen_KL_mean": -249.73309326171875,
"KL/mean": -302.8935241699219,
"KL/rejected_KL_mean": -356.053955078125,
"KL/std": 158.14662170410156,
"epoch": 0.7392670157068063,
"fcm_dpo/beta": 0.0027025165036320686,
"fcm_dpo/delta": -0.016819603741168976,
"fcm_dpo/margin": 106.32086181640625,
"fcm_dpo/q_t": 0.4334595799446106,
"grad_norm": 64.48303985595703,
"learning_rate": 9.76281510992176e-08,
"logits/chosen": -0.8482782244682312,
"logits/rejected": -0.8419215083122253,
"logps/chosen": -520.109130859375,
"logps/ref_chosen": -270.3760681152344,
"logps/ref_rejected": -264.65234375,
"logps/rejected": -620.706298828125,
"loss": 4.7294,
"margin_dpo/margin_mean": 106.32086181640625,
"margin_dpo/margin_std": 179.3717041015625,
"step": 353
},
{
"KL/chosen_KL_mean": -246.82254028320312,
"KL/mean": -288.2264099121094,
"KL/rejected_KL_mean": -329.6302795410156,
"KL/std": 169.2479705810547,
"epoch": 0.7413612565445026,
"fcm_dpo/beta": 0.0026852269656956196,
"fcm_dpo/delta": 0.00828171893954277,
"fcm_dpo/margin": 82.80773162841797,
"fcm_dpo/q_t": 0.4493102431297302,
"grad_norm": 70.67339324951172,
"learning_rate": 9.618082700494318e-08,
"logits/chosen": -0.8379197716712952,
"logits/rejected": -0.8698081970214844,
"logps/chosen": -504.4710998535156,
"logps/ref_chosen": -257.6485595703125,
"logps/ref_rejected": -246.94203186035156,
"logps/rejected": -576.5723266601562,
"loss": 4.9535,
"margin_dpo/margin_mean": 82.8077392578125,
"margin_dpo/margin_std": 183.56936645507812,
"step": 354
},
{
"KL/chosen_KL_mean": -248.17721557617188,
"KL/mean": -303.8741455078125,
"KL/rejected_KL_mean": -359.571044921875,
"KL/std": 174.03460693359375,
"epoch": 0.743455497382199,
"fcm_dpo/beta": 0.002684948965907097,
"fcm_dpo/delta": -0.00029130373150110245,
"fcm_dpo/margin": 111.39378356933594,
"fcm_dpo/q_t": 0.4308469891548157,
"grad_norm": 72.10034942626953,
"learning_rate": 9.474175176609956e-08,
"logits/chosen": -0.8968920111656189,
"logits/rejected": -0.8910172581672668,
"logps/chosen": -541.5305786132812,
"logps/ref_chosen": -293.35333251953125,
"logps/ref_rejected": -275.6051940917969,
"logps/rejected": -635.1762084960938,
"loss": 4.7794,
"margin_dpo/margin_mean": 111.39378356933594,
"margin_dpo/margin_std": 211.11050415039062,
"step": 355
},
{
"KL/chosen_KL_mean": -233.09683227539062,
"KL/mean": -273.4365234375,
"KL/rejected_KL_mean": -313.7762145996094,
"KL/std": 145.1485595703125,
"epoch": 0.7455497382198953,
"fcm_dpo/beta": 0.002696634503081441,
"fcm_dpo/delta": -0.0037465146742761135,
"fcm_dpo/margin": 80.67941284179688,
"fcm_dpo/q_t": 0.4487796127796173,
"grad_norm": 62.512733459472656,
"learning_rate": 9.331100255592436e-08,
"logits/chosen": -0.8034874796867371,
"logits/rejected": -0.8288117051124573,
"logps/chosen": -437.35235595703125,
"logps/ref_chosen": -204.25550842285156,
"logps/ref_rejected": -213.467529296875,
"logps/rejected": -527.2437744140625,
"loss": 4.897,
"margin_dpo/margin_mean": 80.67941284179688,
"margin_dpo/margin_std": 147.65928649902344,
"step": 356
},
{
"KL/chosen_KL_mean": -260.0800476074219,
"KL/mean": -313.5126647949219,
"KL/rejected_KL_mean": -366.9453430175781,
"KL/std": 166.6250762939453,
"epoch": 0.7476439790575916,
"fcm_dpo/beta": 0.002710919128730893,
"fcm_dpo/delta": 0.02739933878183365,
"fcm_dpo/margin": 106.86528015136719,
"fcm_dpo/q_t": 0.4340450167655945,
"grad_norm": 56.6240348815918,
"learning_rate": 9.18886561011557e-08,
"logits/chosen": -0.7838116884231567,
"logits/rejected": -0.779089629650116,
"logps/chosen": -526.4506225585938,
"logps/ref_chosen": -266.3705749511719,
"logps/ref_rejected": -239.04490661621094,
"logps/rejected": -605.990234375,
"loss": 4.7505,
"margin_dpo/margin_mean": 106.86528015136719,
"margin_dpo/margin_std": 189.58627319335938,
"step": 357
},
{
"KL/chosen_KL_mean": -231.25390625,
"KL/mean": -294.31298828125,
"KL/rejected_KL_mean": -357.3720703125,
"KL/std": 153.28192138671875,
"epoch": 0.749738219895288,
"fcm_dpo/beta": 0.0027492064982652664,
"fcm_dpo/delta": -0.007799042388796806,
"fcm_dpo/margin": 126.11814880371094,
"fcm_dpo/q_t": 0.419543981552124,
"grad_norm": 46.20698547363281,
"learning_rate": 9.047478867791731e-08,
"logits/chosen": -0.8915876150131226,
"logits/rejected": -0.8662262558937073,
"logps/chosen": -530.4014282226562,
"logps/ref_chosen": -299.1474609375,
"logps/ref_rejected": -257.2531433105469,
"logps/rejected": -614.625244140625,
"loss": 4.5507,
"margin_dpo/margin_mean": 126.11814880371094,
"margin_dpo/margin_std": 187.35885620117188,
"step": 358
},
{
"KL/chosen_KL_mean": -250.6073455810547,
"KL/mean": -302.3663024902344,
"KL/rejected_KL_mean": -354.12530517578125,
"KL/std": 158.33580017089844,
"epoch": 0.7518324607329843,
"fcm_dpo/beta": 0.002736097201704979,
"fcm_dpo/delta": 0.006915869656950235,
"fcm_dpo/margin": 103.51795959472656,
"fcm_dpo/q_t": 0.4344536066055298,
"grad_norm": 89.7514877319336,
"learning_rate": 8.906947610762825e-08,
"logits/chosen": -0.8484093546867371,
"logits/rejected": -0.8544447422027588,
"logps/chosen": -553.605224609375,
"logps/ref_chosen": -302.99786376953125,
"logps/ref_rejected": -260.4137268066406,
"logps/rejected": -614.5390014648438,
"loss": 4.7663,
"margin_dpo/margin_mean": 103.51795959472656,
"margin_dpo/margin_std": 189.6798553466797,
"step": 359
},
{
"KL/chosen_KL_mean": -255.82347106933594,
"KL/mean": -305.854736328125,
"KL/rejected_KL_mean": -355.885986328125,
"KL/std": 177.97512817382812,
"epoch": 0.7539267015706806,
"fcm_dpo/beta": 0.0026915487833321095,
"fcm_dpo/delta": -0.03379403427243233,
"fcm_dpo/margin": 100.0625,
"fcm_dpo/q_t": 0.43899649381637573,
"grad_norm": 75.62610626220703,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": -0.8806153535842896,
"logits/rejected": -0.8590009808540344,
"logps/chosen": -565.4349365234375,
"logps/ref_chosen": -309.6114501953125,
"logps/ref_rejected": -256.64031982421875,
"logps/rejected": -612.5263061523438,
"loss": 4.9075,
"margin_dpo/margin_mean": 100.06248474121094,
"margin_dpo/margin_std": 208.209716796875,
"step": 360
},
{
"KL/chosen_KL_mean": -218.59913635253906,
"KL/mean": -284.7613830566406,
"KL/rejected_KL_mean": -350.92364501953125,
"KL/std": 172.47113037109375,
"epoch": 0.7560209424083769,
"fcm_dpo/beta": 0.0026670051738619804,
"fcm_dpo/delta": -0.0043861037120223045,
"fcm_dpo/margin": 132.32452392578125,
"fcm_dpo/q_t": 0.42048534750938416,
"grad_norm": 59.663902282714844,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": -0.8187331557273865,
"logits/rejected": -0.7988158464431763,
"logps/chosen": -481.9788818359375,
"logps/ref_chosen": -263.3797607421875,
"logps/ref_rejected": -271.18157958984375,
"logps/rejected": -622.105224609375,
"loss": 4.5702,
"margin_dpo/margin_mean": 132.3245086669922,
"margin_dpo/margin_std": 212.18954467773438,
"step": 361
},
{
"KL/chosen_KL_mean": -246.6825408935547,
"KL/mean": -296.739501953125,
"KL/rejected_KL_mean": -346.7964782714844,
"KL/std": 138.58175659179688,
"epoch": 0.7581151832460733,
"fcm_dpo/beta": 0.002679286291822791,
"fcm_dpo/delta": 0.015072029083967209,
"fcm_dpo/margin": 100.11392211914062,
"fcm_dpo/q_t": 0.43785110116004944,
"grad_norm": 77.70549774169922,
"learning_rate": 8.490561882286135e-08,
"logits/chosen": -0.8581762909889221,
"logits/rejected": -0.8390923142433167,
"logps/chosen": -549.94091796875,
"logps/ref_chosen": -303.2583923339844,
"logps/ref_rejected": -243.22891235351562,
"logps/rejected": -590.0253295898438,
"loss": 4.7371,
"margin_dpo/margin_mean": 100.11392974853516,
"margin_dpo/margin_std": 159.69705200195312,
"step": 362
},
{
"KL/chosen_KL_mean": -253.08914184570312,
"KL/mean": -310.4933776855469,
"KL/rejected_KL_mean": -367.8975830078125,
"KL/std": 171.24012756347656,
"epoch": 0.7602094240837697,
"fcm_dpo/beta": 0.00266785966232419,
"fcm_dpo/delta": -0.03768566623330116,
"fcm_dpo/margin": 114.80848693847656,
"fcm_dpo/q_t": 0.4307466149330139,
"grad_norm": 73.5361099243164,
"learning_rate": 8.353527464267104e-08,
"logits/chosen": -0.8588560819625854,
"logits/rejected": -0.8144607543945312,
"logps/chosen": -556.4363403320312,
"logps/ref_chosen": -303.34722900390625,
"logps/ref_rejected": -262.05419921875,
"logps/rejected": -629.9518432617188,
"loss": 4.7016,
"margin_dpo/margin_mean": 114.80850219726562,
"margin_dpo/margin_std": 196.30714416503906,
"step": 363
},
{
"KL/chosen_KL_mean": -289.6898193359375,
"KL/mean": -345.4079895019531,
"KL/rejected_KL_mean": -401.1260986328125,
"KL/std": 190.25381469726562,
"epoch": 0.762303664921466,
"fcm_dpo/beta": 0.0026328519452363253,
"fcm_dpo/delta": 0.019132010638713837,
"fcm_dpo/margin": 111.43631744384766,
"fcm_dpo/q_t": 0.4358139634132385,
"grad_norm": 59.363731384277344,
"learning_rate": 8.217385746050742e-08,
"logits/chosen": -0.8394103646278381,
"logits/rejected": -0.8337091207504272,
"logps/chosen": -575.2335815429688,
"logps/ref_chosen": -285.54376220703125,
"logps/ref_rejected": -284.84619140625,
"logps/rejected": -685.9723510742188,
"loss": 4.8079,
"margin_dpo/margin_mean": 111.43632507324219,
"margin_dpo/margin_std": 226.2152862548828,
"step": 364
},
{
"KL/chosen_KL_mean": -271.29132080078125,
"KL/mean": -321.2420349121094,
"KL/rejected_KL_mean": -371.1927185058594,
"KL/std": 180.25765991210938,
"epoch": 0.7643979057591623,
"fcm_dpo/beta": 0.0026724659837782383,
"fcm_dpo/delta": 0.018962977454066277,
"fcm_dpo/margin": 99.90141296386719,
"fcm_dpo/q_t": 0.4393257200717926,
"grad_norm": 63.602535247802734,
"learning_rate": 8.082144028504231e-08,
"logits/chosen": -0.8441622257232666,
"logits/rejected": -0.8413698077201843,
"logps/chosen": -546.0791015625,
"logps/ref_chosen": -274.7878112792969,
"logps/ref_rejected": -256.5738220214844,
"logps/rejected": -627.7665405273438,
"loss": 4.8409,
"margin_dpo/margin_mean": 99.90141296386719,
"margin_dpo/margin_std": 199.26466369628906,
"step": 365
},
{
"KL/chosen_KL_mean": -253.7181396484375,
"KL/mean": -306.461669921875,
"KL/rejected_KL_mean": -359.2052307128906,
"KL/std": 156.46954345703125,
"epoch": 0.7664921465968586,
"fcm_dpo/beta": 0.00269645219668746,
"fcm_dpo/delta": 0.010923715308308601,
"fcm_dpo/margin": 105.4870834350586,
"fcm_dpo/q_t": 0.4338816702365875,
"grad_norm": 58.37995910644531,
"learning_rate": 7.947809564230445e-08,
"logits/chosen": -0.8080151677131653,
"logits/rejected": -0.8108749985694885,
"logps/chosen": -540.3677368164062,
"logps/ref_chosen": -286.6496276855469,
"logps/ref_rejected": -251.97140502929688,
"logps/rejected": -611.1766357421875,
"loss": 4.7406,
"margin_dpo/margin_mean": 105.48709106445312,
"margin_dpo/margin_std": 187.03518676757812,
"step": 366
},
{
"KL/chosen_KL_mean": -259.25201416015625,
"KL/mean": -319.07611083984375,
"KL/rejected_KL_mean": -378.90032958984375,
"KL/std": 164.353759765625,
"epoch": 0.768586387434555,
"fcm_dpo/beta": 0.0026909676380455494,
"fcm_dpo/delta": -0.009831960313022137,
"fcm_dpo/margin": 119.64830017089844,
"fcm_dpo/q_t": 0.426592618227005,
"grad_norm": 85.58404541015625,
"learning_rate": 7.814389557179016e-08,
"logits/chosen": -0.8485161066055298,
"logits/rejected": -0.8188115954399109,
"logps/chosen": -561.1969604492188,
"logps/ref_chosen": -301.9449768066406,
"logps/ref_rejected": -265.5677185058594,
"logps/rejected": -644.468017578125,
"loss": 4.6244,
"margin_dpo/margin_mean": 119.64830017089844,
"margin_dpo/margin_std": 188.09889221191406,
"step": 367
},
{
"KL/chosen_KL_mean": -246.48849487304688,
"KL/mean": -317.7361145019531,
"KL/rejected_KL_mean": -388.98382568359375,
"KL/std": 162.717529296875,
"epoch": 0.7706806282722513,
"fcm_dpo/beta": 0.0026594342198222876,
"fcm_dpo/delta": -0.0268840454518795,
"fcm_dpo/margin": 142.4953155517578,
"fcm_dpo/q_t": 0.41193145513534546,
"grad_norm": 58.12229537963867,
"learning_rate": 7.681891162260015e-08,
"logits/chosen": -0.8149774074554443,
"logits/rejected": -0.815480649471283,
"logps/chosen": -541.1150512695312,
"logps/ref_chosen": -294.62652587890625,
"logps/ref_rejected": -258.7628479003906,
"logps/rejected": -647.7466430664062,
"loss": 4.3926,
"margin_dpo/margin_mean": 142.4953155517578,
"margin_dpo/margin_std": 172.4111785888672,
"step": 368
},
{
"KL/chosen_KL_mean": -265.4953918457031,
"KL/mean": -314.00823974609375,
"KL/rejected_KL_mean": -362.5211181640625,
"KL/std": 154.16493225097656,
"epoch": 0.7727748691099476,
"fcm_dpo/beta": 0.002672000788152218,
"fcm_dpo/delta": 0.02517055906355381,
"fcm_dpo/margin": 97.02571105957031,
"fcm_dpo/q_t": 0.4393949508666992,
"grad_norm": 68.1326904296875,
"learning_rate": 7.550321484960251e-08,
"logits/chosen": -0.8995909094810486,
"logits/rejected": -0.8832194209098816,
"logps/chosen": -548.0011596679688,
"logps/ref_chosen": -282.5057373046875,
"logps/ref_rejected": -266.41607666015625,
"logps/rejected": -628.9371948242188,
"loss": 4.7696,
"margin_dpo/margin_mean": 97.02571105957031,
"margin_dpo/margin_std": 162.9827423095703,
"step": 369
},
{
"KL/chosen_KL_mean": -254.54122924804688,
"KL/mean": -315.56475830078125,
"KL/rejected_KL_mean": -376.5882873535156,
"KL/std": 168.5694122314453,
"epoch": 0.774869109947644,
"fcm_dpo/beta": 0.002689562737941742,
"fcm_dpo/delta": -0.00773870013654232,
"fcm_dpo/margin": 122.04708099365234,
"fcm_dpo/q_t": 0.42569032311439514,
"grad_norm": 69.9754638671875,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": -0.86855548620224,
"logits/rejected": -0.8914240002632141,
"logps/chosen": -505.5476379394531,
"logps/ref_chosen": -251.00640869140625,
"logps/ref_rejected": -238.12542724609375,
"logps/rejected": -614.7136840820312,
"loss": 4.6562,
"margin_dpo/margin_mean": 122.04708099365234,
"margin_dpo/margin_std": 212.94515991210938,
"step": 370
},
{
"KL/chosen_KL_mean": -287.0211181640625,
"KL/mean": -323.5873107910156,
"KL/rejected_KL_mean": -360.153564453125,
"KL/std": 164.57492065429688,
"epoch": 0.7769633507853403,
"fcm_dpo/beta": 0.002693257061764598,
"fcm_dpo/delta": 0.002763350959867239,
"fcm_dpo/margin": 73.13240051269531,
"fcm_dpo/q_t": 0.45326054096221924,
"grad_norm": 97.06198120117188,
"learning_rate": 7.289996455765748e-08,
"logits/chosen": -0.8332430720329285,
"logits/rejected": -0.8098735213279724,
"logps/chosen": -583.6802978515625,
"logps/ref_chosen": -296.6591491699219,
"logps/ref_rejected": -251.14675903320312,
"logps/rejected": -611.30029296875,
"loss": 5.0419,
"margin_dpo/margin_mean": 73.13240814208984,
"margin_dpo/margin_std": 174.42578125,
"step": 371
},
{
"KL/chosen_KL_mean": -277.73712158203125,
"KL/mean": -339.80938720703125,
"KL/rejected_KL_mean": -401.88165283203125,
"KL/std": 183.5233612060547,
"epoch": 0.7790575916230367,
"fcm_dpo/beta": 0.002633734606206417,
"fcm_dpo/delta": -0.028473343700170517,
"fcm_dpo/margin": 124.14448547363281,
"fcm_dpo/q_t": 0.4261954724788666,
"grad_norm": 73.80138397216797,
"learning_rate": 7.161255064312283e-08,
"logits/chosen": -0.8024683594703674,
"logits/rejected": -0.7857841849327087,
"logps/chosen": -609.1085815429688,
"logps/ref_chosen": -331.3714599609375,
"logps/ref_rejected": -285.56805419921875,
"logps/rejected": -687.44970703125,
"loss": 4.6629,
"margin_dpo/margin_mean": 124.14448547363281,
"margin_dpo/margin_std": 206.26226806640625,
"step": 372
},
{
"KL/chosen_KL_mean": -258.4107666015625,
"KL/mean": -317.982421875,
"KL/rejected_KL_mean": -377.5540771484375,
"KL/std": 160.34129333496094,
"epoch": 0.7811518324607329,
"fcm_dpo/beta": 0.0026560514234006405,
"fcm_dpo/delta": 0.02417863719165325,
"fcm_dpo/margin": 119.14329528808594,
"fcm_dpo/q_t": 0.42621058225631714,
"grad_norm": 91.65182495117188,
"learning_rate": 7.033470310611945e-08,
"logits/chosen": -0.8905776143074036,
"logits/rejected": -0.8529370427131653,
"logps/chosen": -580.353759765625,
"logps/ref_chosen": -321.9429931640625,
"logps/ref_rejected": -271.2288513183594,
"logps/rejected": -648.782958984375,
"loss": 4.6392,
"margin_dpo/margin_mean": 119.14329528808594,
"margin_dpo/margin_std": 189.48521423339844,
"step": 373
},
{
"KL/chosen_KL_mean": -271.4115295410156,
"KL/mean": -323.5093994140625,
"KL/rejected_KL_mean": -375.6073303222656,
"KL/std": 175.6970672607422,
"epoch": 0.7832460732984293,
"fcm_dpo/beta": 0.00263632507994771,
"fcm_dpo/delta": -0.01818685419857502,
"fcm_dpo/margin": 104.19580078125,
"fcm_dpo/q_t": 0.4362541735172272,
"grad_norm": 75.91381072998047,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": -0.8902040719985962,
"logits/rejected": -0.8820330500602722,
"logps/chosen": -590.580078125,
"logps/ref_chosen": -319.1685485839844,
"logps/ref_rejected": -284.6263732910156,
"logps/rejected": -660.233642578125,
"loss": 4.7608,
"margin_dpo/margin_mean": 104.19580078125,
"margin_dpo/margin_std": 180.38076782226562,
"step": 374
},
{
"KL/chosen_KL_mean": -282.55670166015625,
"KL/mean": -325.5635986328125,
"KL/rejected_KL_mean": -368.570556640625,
"KL/std": 148.60000610351562,
"epoch": 0.7853403141361257,
"fcm_dpo/beta": 0.002658943645656109,
"fcm_dpo/delta": 0.01239805668592453,
"fcm_dpo/margin": 86.01384735107422,
"fcm_dpo/q_t": 0.4467378258705139,
"grad_norm": 81.2287826538086,
"learning_rate": 6.780798075635675e-08,
"logits/chosen": -0.8825744986534119,
"logits/rejected": -0.8607974052429199,
"logps/chosen": -597.4324340820312,
"logps/ref_chosen": -314.87579345703125,
"logps/ref_rejected": -259.1965026855469,
"logps/rejected": -627.7670288085938,
"loss": 4.9062,
"margin_dpo/margin_mean": 86.01384735107422,
"margin_dpo/margin_std": 172.94680786132812,
"step": 375
},
{
"KL/chosen_KL_mean": -277.1960754394531,
"KL/mean": -333.73248291015625,
"KL/rejected_KL_mean": -390.2689208984375,
"KL/std": 171.84317016601562,
"epoch": 0.787434554973822,
"fcm_dpo/beta": 0.0026825799141079187,
"fcm_dpo/delta": 0.002811199054121971,
"fcm_dpo/margin": 113.07284545898438,
"fcm_dpo/q_t": 0.4311826527118683,
"grad_norm": 76.82144165039062,
"learning_rate": 6.655924144404906e-08,
"logits/chosen": -0.8568066954612732,
"logits/rejected": -0.85575270652771,
"logps/chosen": -564.8693237304688,
"logps/ref_chosen": -287.6732482910156,
"logps/ref_rejected": -256.6697082519531,
"logps/rejected": -646.9385986328125,
"loss": 4.7243,
"margin_dpo/margin_mean": 113.07284545898438,
"margin_dpo/margin_std": 204.332763671875,
"step": 376
},
{
"KL/chosen_KL_mean": -292.6242370605469,
"KL/mean": -330.6067199707031,
"KL/rejected_KL_mean": -368.5892333984375,
"KL/std": 170.7978973388672,
"epoch": 0.7895287958115184,
"fcm_dpo/beta": 0.002660011174157262,
"fcm_dpo/delta": -0.003608726430684328,
"fcm_dpo/margin": 75.96499633789062,
"fcm_dpo/q_t": 0.45309317111968994,
"grad_norm": 70.9164810180664,
"learning_rate": 6.532033950290885e-08,
"logits/chosen": -0.8469290137290955,
"logits/rejected": -0.8385406136512756,
"logps/chosen": -597.8856811523438,
"logps/ref_chosen": -305.261474609375,
"logps/ref_rejected": -271.8887023925781,
"logps/rejected": -640.4779052734375,
"loss": 5.0753,
"margin_dpo/margin_mean": 75.96499633789062,
"margin_dpo/margin_std": 200.70811462402344,
"step": 377
},
{
"KL/chosen_KL_mean": -277.7027893066406,
"KL/mean": -330.92620849609375,
"KL/rejected_KL_mean": -384.1496276855469,
"KL/std": 179.26226806640625,
"epoch": 0.7916230366492146,
"fcm_dpo/beta": 0.0026616579852998257,
"fcm_dpo/delta": 0.008543765172362328,
"fcm_dpo/margin": 106.44680786132812,
"fcm_dpo/q_t": 0.4352142810821533,
"grad_norm": 92.10003662109375,
"learning_rate": 6.409134137148736e-08,
"logits/chosen": -0.8278205394744873,
"logits/rejected": -0.8163886070251465,
"logps/chosen": -559.232421875,
"logps/ref_chosen": -281.5295715332031,
"logps/ref_rejected": -296.980224609375,
"logps/rejected": -681.1298828125,
"loss": 4.775,
"margin_dpo/margin_mean": 106.44680786132812,
"margin_dpo/margin_std": 199.52207946777344,
"step": 378
},
{
"KL/chosen_KL_mean": -257.5284423828125,
"KL/mean": -305.9362487792969,
"KL/rejected_KL_mean": -354.3440856933594,
"KL/std": 143.47189331054688,
"epoch": 0.793717277486911,
"fcm_dpo/beta": 0.002679901197552681,
"fcm_dpo/delta": -0.0017141085118055344,
"fcm_dpo/margin": 96.81559753417969,
"fcm_dpo/q_t": 0.43951869010925293,
"grad_norm": 81.0946044921875,
"learning_rate": 6.28723129572247e-08,
"logits/chosen": -0.9057261943817139,
"logits/rejected": -0.8876835703849792,
"logps/chosen": -522.6092529296875,
"logps/ref_chosen": -265.0807800292969,
"logps/ref_rejected": -230.58932495117188,
"logps/rejected": -584.9334106445312,
"loss": 4.8098,
"margin_dpo/margin_mean": 96.81559753417969,
"margin_dpo/margin_std": 180.41207885742188,
"step": 379
},
{
"KL/chosen_KL_mean": -278.8175964355469,
"KL/mean": -334.8206787109375,
"KL/rejected_KL_mean": -390.8237609863281,
"KL/std": 182.3302459716797,
"epoch": 0.7958115183246073,
"fcm_dpo/beta": 0.002679501660168171,
"fcm_dpo/delta": -0.008481554687023163,
"fcm_dpo/margin": 112.0062026977539,
"fcm_dpo/q_t": 0.43370163440704346,
"grad_norm": 61.04739761352539,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": -0.8648520708084106,
"logits/rejected": -0.8380929231643677,
"logps/chosen": -584.7259521484375,
"logps/ref_chosen": -305.90838623046875,
"logps/ref_rejected": -286.5906677246094,
"logps/rejected": -677.4144287109375,
"loss": 4.7804,
"margin_dpo/margin_mean": 112.0062026977539,
"margin_dpo/margin_std": 214.64852905273438,
"step": 380
},
{
"KL/chosen_KL_mean": -273.33929443359375,
"KL/mean": -337.74554443359375,
"KL/rejected_KL_mean": -402.1517333984375,
"KL/std": 202.25027465820312,
"epoch": 0.7979057591623037,
"fcm_dpo/beta": 0.002631543204188347,
"fcm_dpo/delta": -0.025047196075320244,
"fcm_dpo/margin": 128.8124542236328,
"fcm_dpo/q_t": 0.42539119720458984,
"grad_norm": 61.73163604736328,
"learning_rate": 6.046442623320145e-08,
"logits/chosen": -0.832200288772583,
"logits/rejected": -0.797419548034668,
"logps/chosen": -526.2099609375,
"logps/ref_chosen": -252.87066650390625,
"logps/ref_rejected": -261.1927490234375,
"logps/rejected": -663.344482421875,
"loss": 4.6148,
"margin_dpo/margin_mean": 128.8124542236328,
"margin_dpo/margin_std": 213.1539306640625,
"step": 381
},
{
"KL/chosen_KL_mean": -264.1448059082031,
"KL/mean": -332.416748046875,
"KL/rejected_KL_mean": -400.688720703125,
"KL/std": 163.4908447265625,
"epoch": 0.8,
"fcm_dpo/beta": 0.002580613363534212,
"fcm_dpo/delta": -0.01013551652431488,
"fcm_dpo/margin": 136.54393005371094,
"fcm_dpo/q_t": 0.4181453287601471,
"grad_norm": 60.10068130493164,
"learning_rate": 5.9275697051098275e-08,
"logits/chosen": -0.8769955635070801,
"logits/rejected": -0.8728847503662109,
"logps/chosen": -553.3562622070312,
"logps/ref_chosen": -289.2114562988281,
"logps/ref_rejected": -278.45751953125,
"logps/rejected": -679.1461791992188,
"loss": 4.4954,
"margin_dpo/margin_mean": 136.54393005371094,
"margin_dpo/margin_std": 189.81167602539062,
"step": 382
},
{
"KL/chosen_KL_mean": -260.4679870605469,
"KL/mean": -319.3662109375,
"KL/rejected_KL_mean": -378.2645263671875,
"KL/std": 184.4472198486328,
"epoch": 0.8020942408376963,
"fcm_dpo/beta": 0.00258796033449471,
"fcm_dpo/delta": -0.011181109584867954,
"fcm_dpo/margin": 117.7965087890625,
"fcm_dpo/q_t": 0.430372029542923,
"grad_norm": 89.09895324707031,
"learning_rate": 5.809719583454414e-08,
"logits/chosen": -0.8481532335281372,
"logits/rejected": -0.8308265209197998,
"logps/chosen": -534.0987548828125,
"logps/ref_chosen": -273.630859375,
"logps/ref_rejected": -261.44024658203125,
"logps/rejected": -639.7047119140625,
"loss": 4.7125,
"margin_dpo/margin_mean": 117.79650115966797,
"margin_dpo/margin_std": 209.50341796875,
"step": 383
},
{
"KL/chosen_KL_mean": -264.8871154785156,
"KL/mean": -317.6487731933594,
"KL/rejected_KL_mean": -370.4104309082031,
"KL/std": 146.82882690429688,
"epoch": 0.8041884816753927,
"fcm_dpo/beta": 0.002574121579527855,
"fcm_dpo/delta": 0.014372201636433601,
"fcm_dpo/margin": 105.52330017089844,
"fcm_dpo/q_t": 0.437011182308197,
"grad_norm": 57.85554122924805,
"learning_rate": 5.6928985782982524e-08,
"logits/chosen": -0.8842571973800659,
"logits/rejected": -0.8786624073982239,
"logps/chosen": -539.4570922851562,
"logps/ref_chosen": -274.5699462890625,
"logps/ref_rejected": -285.8253479003906,
"logps/rejected": -656.2357177734375,
"loss": 4.7566,
"margin_dpo/margin_mean": 105.5232925415039,
"margin_dpo/margin_std": 184.34634399414062,
"step": 384
},
{
"KL/chosen_KL_mean": -270.7925720214844,
"KL/mean": -314.6628112792969,
"KL/rejected_KL_mean": -358.5330810546875,
"KL/std": 152.50241088867188,
"epoch": 0.806282722513089,
"fcm_dpo/beta": 0.0025990111753344536,
"fcm_dpo/delta": 0.009375514462590218,
"fcm_dpo/margin": 87.74047088623047,
"fcm_dpo/q_t": 0.44724568724632263,
"grad_norm": 70.02297973632812,
"learning_rate": 5.57711295439732e-08,
"logits/chosen": -0.821690559387207,
"logits/rejected": -0.8161090612411499,
"logps/chosen": -554.943115234375,
"logps/ref_chosen": -284.150634765625,
"logps/ref_rejected": -244.87921142578125,
"logps/rejected": -603.4122314453125,
"loss": 4.8863,
"margin_dpo/margin_mean": 87.74047088623047,
"margin_dpo/margin_std": 168.0803985595703,
"step": 385
},
{
"KL/chosen_KL_mean": -259.23101806640625,
"KL/mean": -329.41278076171875,
"KL/rejected_KL_mean": -399.594482421875,
"KL/std": 172.50108337402344,
"epoch": 0.8083769633507853,
"fcm_dpo/beta": 0.002526678144931793,
"fcm_dpo/delta": -0.04324443265795708,
"fcm_dpo/margin": 140.36346435546875,
"fcm_dpo/q_t": 0.4185819625854492,
"grad_norm": 79.33147430419922,
"learning_rate": 5.4623689209832484e-08,
"logits/chosen": -0.7993844151496887,
"logits/rejected": -0.7968799471855164,
"logps/chosen": -579.4072875976562,
"logps/ref_chosen": -320.1762390136719,
"logps/ref_rejected": -302.05023193359375,
"logps/rejected": -701.644775390625,
"loss": 4.5125,
"margin_dpo/margin_mean": 140.36346435546875,
"margin_dpo/margin_std": 189.78298950195312,
"step": 386
},
{
"KL/chosen_KL_mean": -282.47479248046875,
"KL/mean": -334.10858154296875,
"KL/rejected_KL_mean": -385.74237060546875,
"KL/std": 162.93824768066406,
"epoch": 0.8104712041884817,
"fcm_dpo/beta": 0.002494890708476305,
"fcm_dpo/delta": -0.023709315806627274,
"fcm_dpo/margin": 103.26756286621094,
"fcm_dpo/q_t": 0.4388362765312195,
"grad_norm": 74.36798858642578,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": -0.8310315012931824,
"logits/rejected": -0.8349242210388184,
"logps/chosen": -554.7550048828125,
"logps/ref_chosen": -272.2801513671875,
"logps/ref_rejected": -265.1615905761719,
"logps/rejected": -650.9039306640625,
"loss": 4.8558,
"margin_dpo/margin_mean": 103.26756286621094,
"margin_dpo/margin_std": 199.19708251953125,
"step": 387
},
{
"KL/chosen_KL_mean": -298.3812561035156,
"KL/mean": -334.0774230957031,
"KL/rejected_KL_mean": -369.7735595703125,
"KL/std": 169.716552734375,
"epoch": 0.812565445026178,
"fcm_dpo/beta": 0.0024554524570703506,
"fcm_dpo/delta": -0.007999473251402378,
"fcm_dpo/margin": 71.392333984375,
"fcm_dpo/q_t": 0.45962756872177124,
"grad_norm": 65.51069641113281,
"learning_rate": 5.2360301829254745e-08,
"logits/chosen": -0.8141230344772339,
"logits/rejected": -0.7994064092636108,
"logps/chosen": -570.91259765625,
"logps/ref_chosen": -272.5313415527344,
"logps/ref_rejected": -239.55735778808594,
"logps/rejected": -609.3309936523438,
"loss": 5.1277,
"margin_dpo/margin_mean": 71.392333984375,
"margin_dpo/margin_std": 200.36790466308594,
"step": 388
},
{
"KL/chosen_KL_mean": -292.8984069824219,
"KL/mean": -350.0514831542969,
"KL/rejected_KL_mean": -407.2045593261719,
"KL/std": 179.6480255126953,
"epoch": 0.8146596858638744,
"fcm_dpo/beta": 0.0024243236985057592,
"fcm_dpo/delta": -0.027813490480184555,
"fcm_dpo/margin": 114.30616760253906,
"fcm_dpo/q_t": 0.43652477860450745,
"grad_norm": 57.472259521484375,
"learning_rate": 5.1244476161413806e-08,
"logits/chosen": -0.8450219631195068,
"logits/rejected": -0.8411790132522583,
"logps/chosen": -573.9876708984375,
"logps/ref_chosen": -281.0892639160156,
"logps/ref_rejected": -246.50045776367188,
"logps/rejected": -653.7050170898438,
"loss": 4.779,
"margin_dpo/margin_mean": 114.30615997314453,
"margin_dpo/margin_std": 204.97317504882812,
"step": 389
},
{
"KL/chosen_KL_mean": -281.30267333984375,
"KL/mean": -337.25286865234375,
"KL/rejected_KL_mean": -393.20306396484375,
"KL/std": 167.31585693359375,
"epoch": 0.8167539267015707,
"fcm_dpo/beta": 0.002404128899797797,
"fcm_dpo/delta": 0.0032799788750708103,
"fcm_dpo/margin": 111.90047454833984,
"fcm_dpo/q_t": 0.4366511106491089,
"grad_norm": 69.60022735595703,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": -0.8703796863555908,
"logits/rejected": -0.8816381096839905,
"logps/chosen": -565.2901000976562,
"logps/ref_chosen": -283.98748779296875,
"logps/ref_rejected": -283.465087890625,
"logps/rejected": -676.6681518554688,
"loss": 4.745,
"margin_dpo/margin_mean": 111.90047454833984,
"margin_dpo/margin_std": 186.23097229003906,
"step": 390
},
{
"KL/chosen_KL_mean": -291.63238525390625,
"KL/mean": -353.4014892578125,
"KL/rejected_KL_mean": -415.1705627441406,
"KL/std": 187.97467041015625,
"epoch": 0.818848167539267,
"fcm_dpo/beta": 0.0024001263082027435,
"fcm_dpo/delta": -0.009568718262016773,
"fcm_dpo/margin": 123.53820037841797,
"fcm_dpo/q_t": 0.43175381422042847,
"grad_norm": 60.939510345458984,
"learning_rate": 4.904486005914027e-08,
"logits/chosen": -0.8228967785835266,
"logits/rejected": -0.8025352954864502,
"logps/chosen": -575.4937744140625,
"logps/ref_chosen": -283.86138916015625,
"logps/ref_rejected": -263.5093688964844,
"logps/rejected": -678.679931640625,
"loss": 4.6976,
"margin_dpo/margin_mean": 123.53820037841797,
"margin_dpo/margin_std": 205.3511962890625,
"step": 391
},
{
"KL/chosen_KL_mean": -268.03955078125,
"KL/mean": -335.4253234863281,
"KL/rejected_KL_mean": -402.8111267089844,
"KL/std": 162.87490844726562,
"epoch": 0.8209424083769633,
"fcm_dpo/beta": 0.0023622428998351097,
"fcm_dpo/delta": -0.008839438669383526,
"fcm_dpo/margin": 134.77159118652344,
"fcm_dpo/q_t": 0.42433956265449524,
"grad_norm": 79.00080871582031,
"learning_rate": 4.796118758344353e-08,
"logits/chosen": -0.8106127977371216,
"logits/rejected": -0.8217307329177856,
"logps/chosen": -578.1096801757812,
"logps/ref_chosen": -310.070068359375,
"logps/ref_rejected": -252.89817810058594,
"logps/rejected": -655.7092895507812,
"loss": 4.571,
"margin_dpo/margin_mean": 134.77157592773438,
"margin_dpo/margin_std": 180.40658569335938,
"step": 392
},
{
"KL/chosen_KL_mean": -274.3587646484375,
"KL/mean": -322.0811767578125,
"KL/rejected_KL_mean": -369.8035888671875,
"KL/std": 162.0053253173828,
"epoch": 0.8230366492146597,
"fcm_dpo/beta": 0.0023829475976526737,
"fcm_dpo/delta": 0.016615940257906914,
"fcm_dpo/margin": 95.44483947753906,
"fcm_dpo/q_t": 0.4470600485801697,
"grad_norm": 48.30846405029297,
"learning_rate": 4.688834983610082e-08,
"logits/chosen": -0.8459432721138,
"logits/rejected": -0.8264710903167725,
"logps/chosen": -561.0744018554688,
"logps/ref_chosen": -286.7156677246094,
"logps/ref_rejected": -230.00357055664062,
"logps/rejected": -599.8071899414062,
"loss": 4.8972,
"margin_dpo/margin_mean": 95.44483947753906,
"margin_dpo/margin_std": 188.01112365722656,
"step": 393
},
{
"KL/chosen_KL_mean": -277.00994873046875,
"KL/mean": -327.65032958984375,
"KL/rejected_KL_mean": -378.290771484375,
"KL/std": 194.45957946777344,
"epoch": 0.8251308900523561,
"fcm_dpo/beta": 0.002444605343043804,
"fcm_dpo/delta": 0.03289797902107239,
"fcm_dpo/margin": 101.28084564208984,
"fcm_dpo/q_t": 0.44555556774139404,
"grad_norm": 50.116294860839844,
"learning_rate": 4.582640435014459e-08,
"logits/chosen": -0.891754686832428,
"logits/rejected": -0.8810494542121887,
"logps/chosen": -603.0033569335938,
"logps/ref_chosen": -325.9934387207031,
"logps/ref_rejected": -317.42706298828125,
"logps/rejected": -695.7178344726562,
"loss": 4.9184,
"margin_dpo/margin_mean": 101.28085327148438,
"margin_dpo/margin_std": 220.1911163330078,
"step": 394
},
{
"KL/chosen_KL_mean": -264.396240234375,
"KL/mean": -332.3326110839844,
"KL/rejected_KL_mean": -400.2690124511719,
"KL/std": 177.35574340820312,
"epoch": 0.8272251308900523,
"fcm_dpo/beta": 0.002485373057425022,
"fcm_dpo/delta": -0.001986829563975334,
"fcm_dpo/margin": 135.87278747558594,
"fcm_dpo/q_t": 0.4249032437801361,
"grad_norm": 77.19000244140625,
"learning_rate": 4.477540807448832e-08,
"logits/chosen": -0.8276509642601013,
"logits/rejected": -0.8361413478851318,
"logps/chosen": -533.2970581054688,
"logps/ref_chosen": -268.90081787109375,
"logps/ref_rejected": -272.85809326171875,
"logps/rejected": -673.127197265625,
"loss": 4.5902,
"margin_dpo/margin_mean": 135.87278747558594,
"margin_dpo/margin_std": 214.613037109375,
"step": 395
},
{
"KL/chosen_KL_mean": -279.2574462890625,
"KL/mean": -331.1311950683594,
"KL/rejected_KL_mean": -383.0049133300781,
"KL/std": 168.5165557861328,
"epoch": 0.8293193717277487,
"fcm_dpo/beta": 0.0024622506462037563,
"fcm_dpo/delta": -0.0033884027507156134,
"fcm_dpo/margin": 103.74746704101562,
"fcm_dpo/q_t": 0.4406885802745819,
"grad_norm": 85.6566162109375,
"learning_rate": 4.373541737087263e-08,
"logits/chosen": -0.8391107320785522,
"logits/rejected": -0.817350447177887,
"logps/chosen": -570.4557495117188,
"logps/ref_chosen": -291.19830322265625,
"logps/ref_rejected": -253.2803955078125,
"logps/rejected": -636.2852783203125,
"loss": 4.814,
"margin_dpo/margin_mean": 103.7474594116211,
"margin_dpo/margin_std": 189.7008514404297,
"step": 396
},
{
"KL/chosen_KL_mean": -270.15325927734375,
"KL/mean": -316.8970947265625,
"KL/rejected_KL_mean": -363.64093017578125,
"KL/std": 164.19032287597656,
"epoch": 0.831413612565445,
"fcm_dpo/beta": 0.002420744625851512,
"fcm_dpo/delta": -0.03247765451669693,
"fcm_dpo/margin": 93.48770141601562,
"fcm_dpo/q_t": 0.4481962025165558,
"grad_norm": 84.03921508789062,
"learning_rate": 4.270648801084295e-08,
"logits/chosen": -0.8543325066566467,
"logits/rejected": -0.8361637592315674,
"logps/chosen": -579.9756469726562,
"logps/ref_chosen": -309.8224182128906,
"logps/ref_rejected": -291.9057922363281,
"logps/rejected": -655.5467529296875,
"loss": 4.9515,
"margin_dpo/margin_mean": 93.48770141601562,
"margin_dpo/margin_std": 195.09881591796875,
"step": 397
},
{
"KL/chosen_KL_mean": -302.65869140625,
"KL/mean": -342.31829833984375,
"KL/rejected_KL_mean": -381.97796630859375,
"KL/std": 184.55848693847656,
"epoch": 0.8335078534031414,
"fcm_dpo/beta": 0.002387051470577717,
"fcm_dpo/delta": -0.00552708376199007,
"fcm_dpo/margin": 79.3193130493164,
"fcm_dpo/q_t": 0.4543570280075073,
"grad_norm": 74.99633026123047,
"learning_rate": 4.168867517275806e-08,
"logits/chosen": -0.7610083818435669,
"logits/rejected": -0.7987161874771118,
"logps/chosen": -600.47216796875,
"logps/ref_chosen": -297.8135070800781,
"logps/ref_rejected": -270.5025634765625,
"logps/rejected": -652.4805297851562,
"loss": 5.1802,
"margin_dpo/margin_mean": 79.3193130493164,
"margin_dpo/margin_std": 238.6734161376953,
"step": 398
},
{
"KL/chosen_KL_mean": -282.90521240234375,
"KL/mean": -336.2668762207031,
"KL/rejected_KL_mean": -389.6285400390625,
"KL/std": 171.79843139648438,
"epoch": 0.8356020942408376,
"fcm_dpo/beta": 0.0023906768765300512,
"fcm_dpo/delta": 0.0051354942843317986,
"fcm_dpo/margin": 106.72332763671875,
"fcm_dpo/q_t": 0.44120585918426514,
"grad_norm": 64.98973083496094,
"learning_rate": 4.0682033438831584e-08,
"logits/chosen": -0.8578112125396729,
"logits/rejected": -0.8178203701972961,
"logps/chosen": -575.751953125,
"logps/ref_chosen": -292.8467712402344,
"logps/ref_rejected": -268.3638916015625,
"logps/rejected": -657.992431640625,
"loss": 4.8244,
"margin_dpo/margin_mean": 106.72332763671875,
"margin_dpo/margin_std": 198.6634063720703,
"step": 399
},
{
"KL/chosen_KL_mean": -276.2769775390625,
"KL/mean": -327.45770263671875,
"KL/rejected_KL_mean": -378.6385192871094,
"KL/std": 167.0556182861328,
"epoch": 0.837696335078534,
"fcm_dpo/beta": 0.0023928708396852016,
"fcm_dpo/delta": 0.004267351236194372,
"fcm_dpo/margin": 102.36151885986328,
"fcm_dpo/q_t": 0.4427049458026886,
"grad_norm": 52.65484619140625,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": -0.8961995840072632,
"logits/rejected": -0.8956049680709839,
"logps/chosen": -539.953369140625,
"logps/ref_chosen": -263.6763916015625,
"logps/ref_rejected": -258.67266845703125,
"logps/rejected": -637.3111572265625,
"loss": 4.8661,
"margin_dpo/margin_mean": 102.36151123046875,
"margin_dpo/margin_std": 198.69558715820312,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_KL/chosen_KL_mean": -276.4910583496094,
"eval_KL/mean": -331.89849853515625,
"eval_KL/rejected_KL_mean": -387.3060302734375,
"eval_KL/std": 172.48843383789062,
"eval_fcm_dpo/beta": 0.0024002783466130495,
"eval_logits/chosen": -0.858095645904541,
"eval_logits/rejected": -0.8404949307441711,
"eval_logps/chosen": -564.31787109375,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -654.2360229492188,
"eval_loss": 0.601615309715271,
"eval_margin_dpo/margin_mean": 110.81495666503906,
"eval_margin_dpo/margin_std": 193.406005859375,
"eval_runtime": 78.4354,
"eval_samples_per_second": 25.499,
"eval_steps_per_second": 3.187,
"step": 400
},
{
"KL/chosen_KL_mean": -277.6224670410156,
"KL/mean": -332.4400329589844,
"KL/rejected_KL_mean": -387.2575988769531,
"KL/std": 175.7390594482422,
"epoch": 0.8397905759162304,
"fcm_dpo/beta": 0.002420206554234028,
"fcm_dpo/delta": 0.020259615033864975,
"fcm_dpo/margin": 109.63511657714844,
"fcm_dpo/q_t": 0.4383876919746399,
"grad_norm": 74.74368286132812,
"learning_rate": 3.8702478614051345e-08,
"logits/chosen": -0.8335940837860107,
"logits/rejected": -0.8319975137710571,
"logps/chosen": -595.9077758789062,
"logps/ref_chosen": -318.2853088378906,
"logps/ref_rejected": -293.75225830078125,
"logps/rejected": -681.009765625,
"loss": 4.7963,
"margin_dpo/margin_mean": 109.63511657714844,
"margin_dpo/margin_std": 200.65774536132812,
"step": 401
},
{
"KL/chosen_KL_mean": -263.85992431640625,
"KL/mean": -316.65972900390625,
"KL/rejected_KL_mean": -369.45953369140625,
"KL/std": 180.7909698486328,
"epoch": 0.8418848167539267,
"fcm_dpo/beta": 0.0024443857837468386,
"fcm_dpo/delta": 0.009977946057915688,
"fcm_dpo/margin": 105.59957885742188,
"fcm_dpo/q_t": 0.44268375635147095,
"grad_norm": 62.405208587646484,
"learning_rate": 3.772967168071517e-08,
"logits/chosen": -0.8932153582572937,
"logits/rejected": -0.8693395256996155,
"logps/chosen": -573.2877807617188,
"logps/ref_chosen": -309.4278564453125,
"logps/ref_rejected": -282.0279846191406,
"logps/rejected": -651.487548828125,
"loss": 4.8532,
"margin_dpo/margin_mean": 105.59957885742188,
"margin_dpo/margin_std": 217.751953125,
"step": 402
},
{
"KL/chosen_KL_mean": -258.3434753417969,
"KL/mean": -334.11083984375,
"KL/rejected_KL_mean": -409.8782043457031,
"KL/std": 184.3195037841797,
"epoch": 0.8439790575916231,
"fcm_dpo/beta": 0.00247446377761662,
"fcm_dpo/delta": -0.014491792768239975,
"fcm_dpo/margin": 151.5347442626953,
"fcm_dpo/q_t": 0.4138992726802826,
"grad_norm": 57.879676818847656,
"learning_rate": 3.676824816087978e-08,
"logits/chosen": -0.866585373878479,
"logits/rejected": -0.8403845429420471,
"logps/chosen": -567.3720092773438,
"logps/ref_chosen": -309.0284729003906,
"logps/ref_rejected": -272.9622497558594,
"logps/rejected": -682.8404541015625,
"loss": 4.4146,
"margin_dpo/margin_mean": 151.5347442626953,
"margin_dpo/margin_std": 193.26220703125,
"step": 403
},
{
"KL/chosen_KL_mean": -280.8930358886719,
"KL/mean": -337.9317321777344,
"KL/rejected_KL_mean": -394.9703674316406,
"KL/std": 184.8937225341797,
"epoch": 0.8460732984293193,
"fcm_dpo/beta": 0.002477236557751894,
"fcm_dpo/delta": 0.02862975373864174,
"fcm_dpo/margin": 114.07733154296875,
"fcm_dpo/q_t": 0.4357641041278839,
"grad_norm": 80.73699188232422,
"learning_rate": 3.581825961277074e-08,
"logits/chosen": -0.8829959630966187,
"logits/rejected": -0.858309268951416,
"logps/chosen": -578.1767578125,
"logps/ref_chosen": -297.2837219238281,
"logps/ref_rejected": -256.99041748046875,
"logps/rejected": -651.9608154296875,
"loss": 4.7975,
"margin_dpo/margin_mean": 114.07733917236328,
"margin_dpo/margin_std": 220.4342041015625,
"step": 404
},
{
"KL/chosen_KL_mean": -271.5806579589844,
"KL/mean": -334.3476257324219,
"KL/rejected_KL_mean": -397.1145324707031,
"KL/std": 183.08030700683594,
"epoch": 0.8481675392670157,
"fcm_dpo/beta": 0.002512460108846426,
"fcm_dpo/delta": 0.023881835862994194,
"fcm_dpo/margin": 125.53388977050781,
"fcm_dpo/q_t": 0.42782172560691833,
"grad_norm": 49.44511795043945,
"learning_rate": 3.487975698139084e-08,
"logits/chosen": -0.8111081123352051,
"logits/rejected": -0.8217229843139648,
"logps/chosen": -529.5459594726562,
"logps/ref_chosen": -257.96533203125,
"logps/ref_rejected": -255.811279296875,
"logps/rejected": -652.92578125,
"loss": 4.6444,
"margin_dpo/margin_mean": 125.53388977050781,
"margin_dpo/margin_std": 206.29136657714844,
"step": 405
},
{
"KL/chosen_KL_mean": -288.9252014160156,
"KL/mean": -329.91778564453125,
"KL/rejected_KL_mean": -370.910400390625,
"KL/std": 164.0752410888672,
"epoch": 0.8502617801047121,
"fcm_dpo/beta": 0.002553413389250636,
"fcm_dpo/delta": 0.010630952194333076,
"fcm_dpo/margin": 81.9852294921875,
"fcm_dpo/q_t": 0.45008060336112976,
"grad_norm": 70.0001220703125,
"learning_rate": 3.3952790595787986e-08,
"logits/chosen": -0.852975606918335,
"logits/rejected": -0.8305176496505737,
"logps/chosen": -574.106201171875,
"logps/ref_chosen": -285.1810607910156,
"logps/ref_rejected": -264.41351318359375,
"logps/rejected": -635.3239135742188,
"loss": 4.9526,
"margin_dpo/margin_mean": 81.9852294921875,
"margin_dpo/margin_std": 172.13865661621094,
"step": 406
},
{
"KL/chosen_KL_mean": -290.98028564453125,
"KL/mean": -351.85919189453125,
"KL/rejected_KL_mean": -412.73809814453125,
"KL/std": 183.7567596435547,
"epoch": 0.8523560209424084,
"fcm_dpo/beta": 0.002567433984950185,
"fcm_dpo/delta": 0.007098458707332611,
"fcm_dpo/margin": 121.75782012939453,
"fcm_dpo/q_t": 0.42956987023353577,
"grad_norm": 69.03553009033203,
"learning_rate": 3.303741016635614e-08,
"logits/chosen": -0.8222445249557495,
"logits/rejected": -0.8362969160079956,
"logps/chosen": -556.2183837890625,
"logps/ref_chosen": -265.23809814453125,
"logps/ref_rejected": -219.0631561279297,
"logps/rejected": -631.80126953125,
"loss": 4.7024,
"margin_dpo/margin_mean": 121.75782012939453,
"margin_dpo/margin_std": 215.36898803710938,
"step": 407
},
{
"KL/chosen_KL_mean": -265.6532287597656,
"KL/mean": -325.81756591796875,
"KL/rejected_KL_mean": -385.98187255859375,
"KL/std": 170.1363067626953,
"epoch": 0.8544502617801047,
"fcm_dpo/beta": 0.002568720607087016,
"fcm_dpo/delta": -0.03308972716331482,
"fcm_dpo/margin": 120.32862091064453,
"fcm_dpo/q_t": 0.42802876234054565,
"grad_norm": 64.20223236083984,
"learning_rate": 3.2133664782169944e-08,
"logits/chosen": -0.8611224293708801,
"logits/rejected": -0.8501880764961243,
"logps/chosen": -562.6258544921875,
"logps/ref_chosen": -296.9726257324219,
"logps/ref_rejected": -295.4786376953125,
"logps/rejected": -681.46044921875,
"loss": 4.6499,
"margin_dpo/margin_mean": 120.32861328125,
"margin_dpo/margin_std": 186.1289520263672,
"step": 408
},
{
"KL/chosen_KL_mean": -277.5410461425781,
"KL/mean": -346.1307678222656,
"KL/rejected_KL_mean": -414.720458984375,
"KL/std": 232.1706085205078,
"epoch": 0.856544502617801,
"fcm_dpo/beta": 0.0024434137158095837,
"fcm_dpo/delta": -0.05346183478832245,
"fcm_dpo/margin": 137.17942810058594,
"fcm_dpo/q_t": 0.4328618347644806,
"grad_norm": 49.34114074707031,
"learning_rate": 3.12416029083514e-08,
"logits/chosen": -0.8433458209037781,
"logits/rejected": -0.8271778225898743,
"logps/chosen": -564.92041015625,
"logps/ref_chosen": -287.37933349609375,
"logps/ref_rejected": -275.80291748046875,
"logps/rejected": -690.5233154296875,
"loss": 4.748,
"margin_dpo/margin_mean": 137.17942810058594,
"margin_dpo/margin_std": 281.4482727050781,
"step": 409
},
{
"KL/chosen_KL_mean": -273.635986328125,
"KL/mean": -326.820068359375,
"KL/rejected_KL_mean": -380.00408935546875,
"KL/std": 171.7606658935547,
"epoch": 0.8586387434554974,
"fcm_dpo/beta": 0.0024228268302977085,
"fcm_dpo/delta": 0.015623921528458595,
"fcm_dpo/margin": 106.36808013916016,
"fcm_dpo/q_t": 0.44091394543647766,
"grad_norm": 69.07209777832031,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": -0.8576352596282959,
"logits/rejected": -0.8624677062034607,
"logps/chosen": -555.4161987304688,
"logps/ref_chosen": -281.7801818847656,
"logps/ref_rejected": -266.7550354003906,
"logps/rejected": -646.7590942382812,
"loss": 4.8306,
"margin_dpo/margin_mean": 106.36808013916016,
"margin_dpo/margin_std": 204.33482360839844,
"step": 410
},
{
"KL/chosen_KL_mean": -265.7773742675781,
"KL/mean": -329.0021057128906,
"KL/rejected_KL_mean": -392.2268371582031,
"KL/std": 182.93353271484375,
"epoch": 0.8607329842931937,
"fcm_dpo/beta": 0.0024366420693695545,
"fcm_dpo/delta": -0.019304735586047173,
"fcm_dpo/margin": 126.44949340820312,
"fcm_dpo/q_t": 0.4297502040863037,
"grad_norm": 61.002811431884766,
"learning_rate": 2.9492720416985e-08,
"logits/chosen": -0.8678796291351318,
"logits/rejected": -0.8195263743400574,
"logps/chosen": -547.3646240234375,
"logps/ref_chosen": -281.5872497558594,
"logps/ref_rejected": -254.78916931152344,
"logps/rejected": -647.0159912109375,
"loss": 4.6211,
"margin_dpo/margin_mean": 126.44949340820312,
"margin_dpo/margin_std": 180.95140075683594,
"step": 411
},
{
"KL/chosen_KL_mean": -296.2814636230469,
"KL/mean": -348.8402099609375,
"KL/rejected_KL_mean": -401.39898681640625,
"KL/std": 175.22732543945312,
"epoch": 0.86282722513089,
"fcm_dpo/beta": 0.0023367248941212893,
"fcm_dpo/delta": -0.03149264678359032,
"fcm_dpo/margin": 105.11750030517578,
"fcm_dpo/q_t": 0.44317078590393066,
"grad_norm": 65.44154357910156,
"learning_rate": 2.863599358669755e-08,
"logits/chosen": -0.8309513926506042,
"logits/rejected": -0.8433988094329834,
"logps/chosen": -572.8156127929688,
"logps/ref_chosen": -276.5341796875,
"logps/ref_rejected": -273.8751220703125,
"logps/rejected": -675.2740478515625,
"loss": 4.8567,
"margin_dpo/margin_mean": 105.11750030517578,
"margin_dpo/margin_std": 195.50192260742188,
"step": 412
},
{
"KL/chosen_KL_mean": -279.3658752441406,
"KL/mean": -327.3970947265625,
"KL/rejected_KL_mean": -375.42840576171875,
"KL/std": 175.02883911132812,
"epoch": 0.8649214659685864,
"fcm_dpo/beta": 0.0023326242808252573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 96.0625,
"fcm_dpo/q_t": 0.44770365953445435,
"grad_norm": 88.38508605957031,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": -0.8533222079277039,
"logits/rejected": -0.8485733866691589,
"logps/chosen": -550.6405029296875,
"logps/ref_chosen": -271.2745666503906,
"logps/ref_rejected": -270.16912841796875,
"logps/rejected": -645.5975341796875,
"loss": 4.9113,
"margin_dpo/margin_mean": 96.0625,
"margin_dpo/margin_std": 199.59288024902344,
"step": 413
},
{
"KL/chosen_KL_mean": -281.5064392089844,
"KL/mean": -345.09735107421875,
"KL/rejected_KL_mean": -408.6882629394531,
"KL/std": 176.5061798095703,
"epoch": 0.8670157068062827,
"fcm_dpo/beta": 0.0023213180247694254,
"fcm_dpo/delta": -0.015394072979688644,
"fcm_dpo/margin": 127.18183898925781,
"fcm_dpo/q_t": 0.4320528507232666,
"grad_norm": 54.224853515625,
"learning_rate": 2.6958198472749717e-08,
"logits/chosen": -0.8887584805488586,
"logits/rejected": -0.8804943561553955,
"logps/chosen": -578.6215209960938,
"logps/ref_chosen": -297.11505126953125,
"logps/ref_rejected": -271.7034606933594,
"logps/rejected": -680.3917236328125,
"loss": 4.6761,
"margin_dpo/margin_mean": 127.18183898925781,
"margin_dpo/margin_std": 199.06312561035156,
"step": 414
},
{
"KL/chosen_KL_mean": -272.0690002441406,
"KL/mean": -331.60137939453125,
"KL/rejected_KL_mean": -391.1337890625,
"KL/std": 162.61656188964844,
"epoch": 0.8691099476439791,
"fcm_dpo/beta": 0.002339128637686372,
"fcm_dpo/delta": 0.029414832592010498,
"fcm_dpo/margin": 119.0648193359375,
"fcm_dpo/q_t": 0.4344155192375183,
"grad_norm": 72.77362060546875,
"learning_rate": 2.613722016414943e-08,
"logits/chosen": -0.8714032173156738,
"logits/rejected": -0.8565488457679749,
"logps/chosen": -569.7616577148438,
"logps/ref_chosen": -297.6926574707031,
"logps/ref_rejected": -279.0503234863281,
"logps/rejected": -670.18408203125,
"loss": 4.6898,
"margin_dpo/margin_mean": 119.0648193359375,
"margin_dpo/margin_std": 182.08590698242188,
"step": 415
},
{
"KL/chosen_KL_mean": -260.75152587890625,
"KL/mean": -327.1971435546875,
"KL/rejected_KL_mean": -393.6427001953125,
"KL/std": 163.0526885986328,
"epoch": 0.8712041884816754,
"fcm_dpo/beta": 0.002332664094865322,
"fcm_dpo/delta": -0.0067732506431639194,
"fcm_dpo/margin": 132.8911590576172,
"fcm_dpo/q_t": 0.42698273062705994,
"grad_norm": 51.488826751708984,
"learning_rate": 2.5328246937043525e-08,
"logits/chosen": -0.8797720670700073,
"logits/rejected": -0.8755515217781067,
"logps/chosen": -572.5771484375,
"logps/ref_chosen": -311.8255615234375,
"logps/ref_rejected": -268.6170654296875,
"logps/rejected": -662.259765625,
"loss": 4.6047,
"margin_dpo/margin_mean": 132.8911590576172,
"margin_dpo/margin_std": 190.5181121826172,
"step": 416
},
{
"KL/chosen_KL_mean": -290.8223571777344,
"KL/mean": -347.7958068847656,
"KL/rejected_KL_mean": -404.7692565917969,
"KL/std": 185.7816162109375,
"epoch": 0.8732984293193717,
"fcm_dpo/beta": 0.0023309113457798958,
"fcm_dpo/delta": -0.02380407229065895,
"fcm_dpo/margin": 113.94691467285156,
"fcm_dpo/q_t": 0.43974053859710693,
"grad_norm": 60.413761138916016,
"learning_rate": 2.4531322174210973e-08,
"logits/chosen": -0.8300318717956543,
"logits/rejected": -0.8154677748680115,
"logps/chosen": -601.2591552734375,
"logps/ref_chosen": -310.43682861328125,
"logps/ref_rejected": -277.15283203125,
"logps/rejected": -681.922119140625,
"loss": 4.8109,
"margin_dpo/margin_mean": 113.94691467285156,
"margin_dpo/margin_std": 207.6744842529297,
"step": 417
},
{
"KL/chosen_KL_mean": -301.2762145996094,
"KL/mean": -361.7869873046875,
"KL/rejected_KL_mean": -422.29779052734375,
"KL/std": 202.2158660888672,
"epoch": 0.875392670157068,
"fcm_dpo/beta": 0.00228428328409791,
"fcm_dpo/delta": -0.011082015931606293,
"fcm_dpo/margin": 121.02159881591797,
"fcm_dpo/q_t": 0.43776950240135193,
"grad_norm": 64.88748931884766,
"learning_rate": 2.3746488612308295e-08,
"logits/chosen": -0.8160441517829895,
"logits/rejected": -0.7922072410583496,
"logps/chosen": -579.7721557617188,
"logps/ref_chosen": -278.49591064453125,
"logps/ref_rejected": -276.56671142578125,
"logps/rejected": -698.864501953125,
"loss": 4.8021,
"margin_dpo/margin_mean": 121.02159118652344,
"margin_dpo/margin_std": 235.40972900390625,
"step": 418
},
{
"KL/chosen_KL_mean": -301.7872314453125,
"KL/mean": -356.77294921875,
"KL/rejected_KL_mean": -411.7586669921875,
"KL/std": 187.64405822753906,
"epoch": 0.8774869109947644,
"fcm_dpo/beta": 0.002300471067428589,
"fcm_dpo/delta": 0.014020832255482674,
"fcm_dpo/margin": 109.97142791748047,
"fcm_dpo/q_t": 0.44006672501564026,
"grad_norm": 68.1669692993164,
"learning_rate": 2.297378833957761e-08,
"logits/chosen": -0.8629166483879089,
"logits/rejected": -0.8316540122032166,
"logps/chosen": -600.6875,
"logps/ref_chosen": -298.9002380371094,
"logps/ref_rejected": -246.1540985107422,
"logps/rejected": -657.9127197265625,
"loss": 4.8853,
"margin_dpo/margin_mean": 109.97142791748047,
"margin_dpo/margin_std": 232.2672119140625,
"step": 419
},
{
"KL/chosen_KL_mean": -289.37896728515625,
"KL/mean": -360.5723876953125,
"KL/rejected_KL_mean": -431.765869140625,
"KL/std": 196.79029846191406,
"epoch": 0.8795811518324608,
"fcm_dpo/beta": 0.0023048303555697203,
"fcm_dpo/delta": -0.011320296674966812,
"fcm_dpo/margin": 142.38693237304688,
"fcm_dpo/q_t": 0.4256310760974884,
"grad_norm": 87.50121307373047,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": -0.8016407489776611,
"logits/rejected": -0.7715350389480591,
"logps/chosen": -553.9398193359375,
"logps/ref_chosen": -264.5608825683594,
"logps/ref_rejected": -245.67031860351562,
"logps/rejected": -677.4362182617188,
"loss": 4.672,
"margin_dpo/margin_mean": 142.38693237304688,
"margin_dpo/margin_std": 250.87786865234375,
"step": 420
},
{
"KL/chosen_KL_mean": -267.2189636230469,
"KL/mean": -327.4419860839844,
"KL/rejected_KL_mean": -387.6650390625,
"KL/std": 166.16717529296875,
"epoch": 0.881675392670157,
"fcm_dpo/beta": 0.002284942427650094,
"fcm_dpo/delta": -0.0033614085987210274,
"fcm_dpo/margin": 120.44603729248047,
"fcm_dpo/q_t": 0.4343334436416626,
"grad_norm": 60.55808639526367,
"learning_rate": 2.1464952759020856e-08,
"logits/chosen": -0.8927887082099915,
"logits/rejected": -0.8663312792778015,
"logps/chosen": -564.9240112304688,
"logps/ref_chosen": -297.70501708984375,
"logps/ref_rejected": -243.74771118164062,
"logps/rejected": -631.4127197265625,
"loss": 4.718,
"margin_dpo/margin_mean": 120.44603729248047,
"margin_dpo/margin_std": 188.74952697753906,
"step": 421
},
{
"KL/chosen_KL_mean": -283.3934020996094,
"KL/mean": -340.1562194824219,
"KL/rejected_KL_mean": -396.9190368652344,
"KL/std": 182.96835327148438,
"epoch": 0.8837696335078534,
"fcm_dpo/beta": 0.0022875501308590174,
"fcm_dpo/delta": 0.014606714248657227,
"fcm_dpo/margin": 113.52565002441406,
"fcm_dpo/q_t": 0.44040507078170776,
"grad_norm": 53.32109069824219,
"learning_rate": 2.07288983654679e-08,
"logits/chosen": -0.7491770386695862,
"logits/rejected": -0.7867967486381531,
"logps/chosen": -571.7521362304688,
"logps/ref_chosen": -288.3587646484375,
"logps/ref_rejected": -256.4377746582031,
"logps/rejected": -653.3568115234375,
"loss": 4.8024,
"margin_dpo/margin_mean": 113.5256576538086,
"margin_dpo/margin_std": 209.61355590820312,
"step": 422
},
{
"KL/chosen_KL_mean": -292.1402893066406,
"KL/mean": -356.8420715332031,
"KL/rejected_KL_mean": -421.54388427734375,
"KL/std": 173.6347198486328,
"epoch": 0.8858638743455497,
"fcm_dpo/beta": 0.0023075914941728115,
"fcm_dpo/delta": 0.0020584806334227324,
"fcm_dpo/margin": 129.40359497070312,
"fcm_dpo/q_t": 0.43022823333740234,
"grad_norm": 63.76276397705078,
"learning_rate": 2.0005139085293942e-08,
"logits/chosen": -0.8720574378967285,
"logits/rejected": -0.8487915396690369,
"logps/chosen": -588.1472778320312,
"logps/ref_chosen": -296.00701904296875,
"logps/ref_rejected": -261.3480529785156,
"logps/rejected": -682.8919067382812,
"loss": 4.6646,
"margin_dpo/margin_mean": 129.40359497070312,
"margin_dpo/margin_std": 206.61529541015625,
"step": 423
},
{
"KL/chosen_KL_mean": -270.66021728515625,
"KL/mean": -328.9430847167969,
"KL/rejected_KL_mean": -387.2259826660156,
"KL/std": 167.630615234375,
"epoch": 0.8879581151832461,
"fcm_dpo/beta": 0.0022989388089627028,
"fcm_dpo/delta": 0.0014383550733327866,
"fcm_dpo/margin": 116.56575775146484,
"fcm_dpo/q_t": 0.4369730055332184,
"grad_norm": 85.71529388427734,
"learning_rate": 1.9293713731512673e-08,
"logits/chosen": -0.8570412397384644,
"logits/rejected": -0.8399197459220886,
"logps/chosen": -580.0821533203125,
"logps/ref_chosen": -309.421875,
"logps/ref_rejected": -249.14886474609375,
"logps/rejected": -636.3748779296875,
"loss": 4.7347,
"margin_dpo/margin_mean": 116.56575775146484,
"margin_dpo/margin_std": 184.96189880371094,
"step": 424
},
{
"KL/chosen_KL_mean": -277.16748046875,
"KL/mean": -334.2838439941406,
"KL/rejected_KL_mean": -391.40020751953125,
"KL/std": 176.4136199951172,
"epoch": 0.8900523560209425,
"fcm_dpo/beta": 0.0022896535228937864,
"fcm_dpo/delta": -0.02011648565530777,
"fcm_dpo/margin": 114.23271942138672,
"fcm_dpo/q_t": 0.4396037459373474,
"grad_norm": 79.22269439697266,
"learning_rate": 1.8594660455706763e-08,
"logits/chosen": -0.8274625539779663,
"logits/rejected": -0.8282160758972168,
"logps/chosen": -557.6765747070312,
"logps/ref_chosen": -280.50909423828125,
"logps/ref_rejected": -276.8252258300781,
"logps/rejected": -668.2254028320312,
"loss": 4.7899,
"margin_dpo/margin_mean": 114.23271942138672,
"margin_dpo/margin_std": 195.52053833007812,
"step": 425
},
{
"KL/chosen_KL_mean": -282.42633056640625,
"KL/mean": -342.55267333984375,
"KL/rejected_KL_mean": -402.678955078125,
"KL/std": 177.6331329345703,
"epoch": 0.8921465968586387,
"fcm_dpo/beta": 0.002310623647645116,
"fcm_dpo/delta": 0.025467144325375557,
"fcm_dpo/margin": 120.25263977050781,
"fcm_dpo/q_t": 0.4347745478153229,
"grad_norm": 77.7251205444336,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": -0.8655619621276855,
"logits/rejected": -0.8427607417106628,
"logps/chosen": -575.2116088867188,
"logps/ref_chosen": -292.78521728515625,
"logps/ref_rejected": -255.62698364257812,
"logps/rejected": -658.3059692382812,
"loss": 4.7273,
"margin_dpo/margin_mean": 120.25263977050781,
"margin_dpo/margin_std": 196.75177001953125,
"step": 426
},
{
"KL/chosen_KL_mean": -276.68865966796875,
"KL/mean": -362.1603088378906,
"KL/rejected_KL_mean": -447.6319274902344,
"KL/std": 200.92494201660156,
"epoch": 0.8942408376963351,
"fcm_dpo/beta": 0.002332492033019662,
"fcm_dpo/delta": -0.03227193281054497,
"fcm_dpo/margin": 170.94326782226562,
"fcm_dpo/q_t": 0.40855950117111206,
"grad_norm": 60.52691650390625,
"learning_rate": 1.7233819424956247e-08,
"logits/chosen": -0.8385801315307617,
"logits/rejected": -0.7992298603057861,
"logps/chosen": -565.4573974609375,
"logps/ref_chosen": -288.7687072753906,
"logps/ref_rejected": -268.4986572265625,
"logps/rejected": -716.130615234375,
"loss": 4.4378,
"margin_dpo/margin_mean": 170.94326782226562,
"margin_dpo/margin_std": 245.36048889160156,
"step": 427
},
{
"KL/chosen_KL_mean": -311.8055419921875,
"KL/mean": -385.68841552734375,
"KL/rejected_KL_mean": -459.5712890625,
"KL/std": 200.10256958007812,
"epoch": 0.8963350785340314,
"fcm_dpo/beta": 0.002250659978017211,
"fcm_dpo/delta": -0.008135773241519928,
"fcm_dpo/margin": 147.7657470703125,
"fcm_dpo/q_t": 0.42285412549972534,
"grad_norm": 60.747161865234375,
"learning_rate": 1.6572104647786245e-08,
"logits/chosen": -0.7936046123504639,
"logits/rejected": -0.8243399858474731,
"logps/chosen": -607.3265380859375,
"logps/ref_chosen": -295.5209655761719,
"logps/ref_rejected": -275.71026611328125,
"logps/rejected": -735.2815551757812,
"loss": 4.5594,
"margin_dpo/margin_mean": 147.7657470703125,
"margin_dpo/margin_std": 209.39695739746094,
"step": 428
},
{
"KL/chosen_KL_mean": -295.0547790527344,
"KL/mean": -354.29083251953125,
"KL/rejected_KL_mean": -413.52691650390625,
"KL/std": 192.97251892089844,
"epoch": 0.8984293193717278,
"fcm_dpo/beta": 0.0022739970590919256,
"fcm_dpo/delta": 0.019945936277508736,
"fcm_dpo/margin": 118.47213745117188,
"fcm_dpo/q_t": 0.4369737505912781,
"grad_norm": 73.8173599243164,
"learning_rate": 1.5922907900227017e-08,
"logits/chosen": -0.8066105246543884,
"logits/rejected": -0.8140644431114197,
"logps/chosen": -569.4470825195312,
"logps/ref_chosen": -274.392333984375,
"logps/ref_rejected": -258.574462890625,
"logps/rejected": -672.1014404296875,
"loss": 4.837,
"margin_dpo/margin_mean": 118.47213745117188,
"margin_dpo/margin_std": 233.86431884765625,
"step": 429
},
{
"KL/chosen_KL_mean": -285.7272033691406,
"KL/mean": -344.11572265625,
"KL/rejected_KL_mean": -402.5043029785156,
"KL/std": 180.56253051757812,
"epoch": 0.900523560209424,
"fcm_dpo/beta": 0.002290163654834032,
"fcm_dpo/delta": 0.002819681540131569,
"fcm_dpo/margin": 116.77705383300781,
"fcm_dpo/q_t": 0.43817266821861267,
"grad_norm": 60.57477951049805,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": -0.8805914521217346,
"logits/rejected": -0.8473300337791443,
"logps/chosen": -574.46630859375,
"logps/ref_chosen": -288.7391357421875,
"logps/ref_rejected": -268.6106262207031,
"logps/rejected": -671.1148681640625,
"loss": 4.7739,
"margin_dpo/margin_mean": 116.77705383300781,
"margin_dpo/margin_std": 204.91702270507812,
"step": 430
},
{
"KL/chosen_KL_mean": -282.6735534667969,
"KL/mean": -331.5006408691406,
"KL/rejected_KL_mean": -380.3277893066406,
"KL/std": 168.19338989257812,
"epoch": 0.9026178010471204,
"fcm_dpo/beta": 0.002297281287610531,
"fcm_dpo/delta": -0.006534409709274769,
"fcm_dpo/margin": 97.65420532226562,
"fcm_dpo/q_t": 0.4476650655269623,
"grad_norm": 55.633140563964844,
"learning_rate": 1.4662207078575684e-08,
"logits/chosen": -0.8622323870658875,
"logits/rejected": -0.8313441872596741,
"logps/chosen": -558.3983154296875,
"logps/ref_chosen": -275.7247314453125,
"logps/ref_rejected": -268.91729736328125,
"logps/rejected": -649.2450561523438,
"loss": 4.9127,
"margin_dpo/margin_mean": 97.6541976928711,
"margin_dpo/margin_std": 194.69580078125,
"step": 431
},
{
"KL/chosen_KL_mean": -268.3981628417969,
"KL/mean": -328.0491027832031,
"KL/rejected_KL_mean": -387.7000732421875,
"KL/std": 181.62322998046875,
"epoch": 0.9047120418848168,
"fcm_dpo/beta": 0.002279035747051239,
"fcm_dpo/delta": 0.004813372157514095,
"fcm_dpo/margin": 119.30191040039062,
"fcm_dpo/q_t": 0.4380475580692291,
"grad_norm": 44.37526321411133,
"learning_rate": 1.40507706120426e-08,
"logits/chosen": -0.8959624767303467,
"logits/rejected": -0.8759222626686096,
"logps/chosen": -559.8182373046875,
"logps/ref_chosen": -291.42010498046875,
"logps/ref_rejected": -255.48202514648438,
"logps/rejected": -643.1820678710938,
"loss": 4.7857,
"margin_dpo/margin_mean": 119.30191802978516,
"margin_dpo/margin_std": 219.4633331298828,
"step": 432
},
{
"KL/chosen_KL_mean": -298.1172790527344,
"KL/mean": -351.10052490234375,
"KL/rejected_KL_mean": -404.083740234375,
"KL/std": 174.59242248535156,
"epoch": 0.9068062827225131,
"fcm_dpo/beta": 0.002294522011652589,
"fcm_dpo/delta": 0.004826649557799101,
"fcm_dpo/margin": 105.96644592285156,
"fcm_dpo/q_t": 0.44354549050331116,
"grad_norm": 57.427310943603516,
"learning_rate": 1.345198738661285e-08,
"logits/chosen": -0.8302546143531799,
"logits/rejected": -0.8239802718162537,
"logps/chosen": -544.3441772460938,
"logps/ref_chosen": -246.2268829345703,
"logps/ref_rejected": -253.65924072265625,
"logps/rejected": -657.7430419921875,
"loss": 4.8246,
"margin_dpo/margin_mean": 105.96646118164062,
"margin_dpo/margin_std": 192.65756225585938,
"step": 433
},
{
"KL/chosen_KL_mean": -320.85400390625,
"KL/mean": -370.788818359375,
"KL/rejected_KL_mean": -420.7236633300781,
"KL/std": 183.7316131591797,
"epoch": 0.9089005235602095,
"fcm_dpo/beta": 0.002314929384738207,
"fcm_dpo/delta": 0.020489579066634178,
"fcm_dpo/margin": 99.86967468261719,
"fcm_dpo/q_t": 0.44652947783470154,
"grad_norm": 60.26441955566406,
"learning_rate": 1.2865889513213628e-08,
"logits/chosen": -0.8335898518562317,
"logits/rejected": -0.8363132476806641,
"logps/chosen": -616.3158569335938,
"logps/ref_chosen": -295.4618225097656,
"logps/ref_rejected": -256.2254333496094,
"logps/rejected": -676.9491577148438,
"loss": 4.9074,
"margin_dpo/margin_mean": 99.86965942382812,
"margin_dpo/margin_std": 205.02581787109375,
"step": 434
},
{
"KL/chosen_KL_mean": -272.2348937988281,
"KL/mean": -340.4631652832031,
"KL/rejected_KL_mean": -408.6913757324219,
"KL/std": 186.8828125,
"epoch": 0.9109947643979057,
"fcm_dpo/beta": 0.002351157832890749,
"fcm_dpo/delta": -0.009057372808456421,
"fcm_dpo/margin": 136.4564971923828,
"fcm_dpo/q_t": 0.42885512113571167,
"grad_norm": 69.38368225097656,
"learning_rate": 1.2292508422495157e-08,
"logits/chosen": -0.8288702368736267,
"logits/rejected": -0.8155333995819092,
"logps/chosen": -532.9733276367188,
"logps/ref_chosen": -260.7384033203125,
"logps/ref_rejected": -248.5688018798828,
"logps/rejected": -657.2601928710938,
"loss": 4.6169,
"margin_dpo/margin_mean": 136.4564971923828,
"margin_dpo/margin_std": 204.81259155273438,
"step": 435
},
{
"KL/chosen_KL_mean": -311.447265625,
"KL/mean": -359.7391662597656,
"KL/rejected_KL_mean": -408.03106689453125,
"KL/std": 206.40972900390625,
"epoch": 0.9130890052356021,
"fcm_dpo/beta": 0.00231517874635756,
"fcm_dpo/delta": 0.004760443232953548,
"fcm_dpo/margin": 96.58380889892578,
"fcm_dpo/q_t": 0.4519736170768738,
"grad_norm": 69.52943420410156,
"learning_rate": 1.1731874863145142e-08,
"logits/chosen": -0.8272717595100403,
"logits/rejected": -0.8191923499107361,
"logps/chosen": -630.769775390625,
"logps/ref_chosen": -319.3224792480469,
"logps/ref_rejected": -299.30322265625,
"logps/rejected": -707.3342895507812,
"loss": 4.9815,
"margin_dpo/margin_mean": 96.58382415771484,
"margin_dpo/margin_std": 230.41339111328125,
"step": 436
},
{
"KL/chosen_KL_mean": -283.3936767578125,
"KL/mean": -354.4276123046875,
"KL/rejected_KL_mean": -425.4615783691406,
"KL/std": 201.54818725585938,
"epoch": 0.9151832460732985,
"fcm_dpo/beta": 0.0023110881447792053,
"fcm_dpo/delta": -0.01546804141253233,
"fcm_dpo/margin": 142.06787109375,
"fcm_dpo/q_t": 0.42517486214637756,
"grad_norm": 48.614498138427734,
"learning_rate": 1.118401890024001e-08,
"logits/chosen": -0.8567025065422058,
"logits/rejected": -0.8355041742324829,
"logps/chosen": -562.2224731445312,
"logps/ref_chosen": -278.82879638671875,
"logps/ref_rejected": -272.55303955078125,
"logps/rejected": -698.0146484375,
"loss": 4.6208,
"margin_dpo/margin_mean": 142.06787109375,
"margin_dpo/margin_std": 224.5172882080078,
"step": 437
},
{
"KL/chosen_KL_mean": -299.3929443359375,
"KL/mean": -329.7269592285156,
"KL/rejected_KL_mean": -360.06103515625,
"KL/std": 184.3596649169922,
"epoch": 0.9172774869109948,
"fcm_dpo/beta": 0.0022973110899329185,
"fcm_dpo/delta": -0.003291675355285406,
"fcm_dpo/margin": 60.66808319091797,
"fcm_dpo/q_t": 0.4677131474018097,
"grad_norm": 77.67984771728516,
"learning_rate": 1.06489699136324e-08,
"logits/chosen": -0.8413230776786804,
"logits/rejected": -0.851213812828064,
"logps/chosen": -558.7120361328125,
"logps/ref_chosen": -259.31903076171875,
"logps/ref_rejected": -240.99581909179688,
"logps/rejected": -601.0568237304688,
"loss": 5.2507,
"margin_dpo/margin_mean": 60.66808319091797,
"margin_dpo/margin_std": 206.700439453125,
"step": 438
},
{
"KL/chosen_KL_mean": -289.4859619140625,
"KL/mean": -349.2972412109375,
"KL/rejected_KL_mean": -409.1085510253906,
"KL/std": 174.05442810058594,
"epoch": 0.9193717277486911,
"fcm_dpo/beta": 0.0022938335314393044,
"fcm_dpo/delta": -0.0010132784955203533,
"fcm_dpo/margin": 119.62257385253906,
"fcm_dpo/q_t": 0.43509790301322937,
"grad_norm": 79.5361099243164,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": -0.8212806582450867,
"logits/rejected": -0.8165966868400574,
"logps/chosen": -546.6103515625,
"logps/ref_chosen": -257.1243896484375,
"logps/ref_rejected": -243.20416259765625,
"logps/rejected": -652.3126220703125,
"loss": 4.7176,
"margin_dpo/margin_mean": 119.62257385253906,
"margin_dpo/margin_std": 190.63784790039062,
"step": 439
},
{
"KL/chosen_KL_mean": -306.56158447265625,
"KL/mean": -353.7540283203125,
"KL/rejected_KL_mean": -400.946533203125,
"KL/std": 162.243408203125,
"epoch": 0.9214659685863874,
"fcm_dpo/beta": 0.0023019982036203146,
"fcm_dpo/delta": 0.015137860551476479,
"fcm_dpo/margin": 94.38501739501953,
"fcm_dpo/q_t": 0.4487031102180481,
"grad_norm": 66.66883850097656,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": -0.8774101734161377,
"logits/rejected": -0.8676927089691162,
"logps/chosen": -614.0930786132812,
"logps/ref_chosen": -307.5315246582031,
"logps/ref_rejected": -264.3540954589844,
"logps/rejected": -665.3006591796875,
"loss": 4.9081,
"margin_dpo/margin_mean": 94.38501739501953,
"margin_dpo/margin_std": 178.12355041503906,
"step": 440
},
{
"KL/chosen_KL_mean": -290.2575378417969,
"KL/mean": -360.6673583984375,
"KL/rejected_KL_mean": -431.07720947265625,
"KL/std": 180.72862243652344,
"epoch": 0.9235602094240838,
"fcm_dpo/beta": 0.0023078853264451027,
"fcm_dpo/delta": -0.01445065625011921,
"fcm_dpo/margin": 140.8197479248047,
"fcm_dpo/q_t": 0.4237060546875,
"grad_norm": 75.2840805053711,
"learning_rate": 9.12094829893642e-09,
"logits/chosen": -0.8152427077293396,
"logits/rejected": -0.8081190586090088,
"logps/chosen": -600.239501953125,
"logps/ref_chosen": -309.9819641113281,
"logps/ref_rejected": -297.4968566894531,
"logps/rejected": -728.5740966796875,
"loss": 4.5509,
"margin_dpo/margin_mean": 140.8197479248047,
"margin_dpo/margin_std": 188.0748748779297,
"step": 441
},
{
"KL/chosen_KL_mean": -317.641357421875,
"KL/mean": -365.92034912109375,
"KL/rejected_KL_mean": -414.19927978515625,
"KL/std": 183.26416015625,
"epoch": 0.9256544502617801,
"fcm_dpo/beta": 0.002325834007933736,
"fcm_dpo/delta": 0.026067446917295456,
"fcm_dpo/margin": 96.55789184570312,
"fcm_dpo/q_t": 0.44816458225250244,
"grad_norm": 75.7071533203125,
"learning_rate": 8.637407257200496e-09,
"logits/chosen": -0.8991624712944031,
"logits/rejected": -0.8569780588150024,
"logps/chosen": -596.62060546875,
"logps/ref_chosen": -278.9791564941406,
"logps/ref_rejected": -242.87310791015625,
"logps/rejected": -657.0723876953125,
"loss": 4.9514,
"margin_dpo/margin_mean": 96.55789184570312,
"margin_dpo/margin_std": 217.6230010986328,
"step": 442
},
{
"KL/chosen_KL_mean": -293.0943298339844,
"KL/mean": -347.28619384765625,
"KL/rejected_KL_mean": -401.47808837890625,
"KL/std": 162.5325927734375,
"epoch": 0.9277486910994764,
"fcm_dpo/beta": 0.0023306708317250013,
"fcm_dpo/delta": -0.019472528249025345,
"fcm_dpo/margin": 108.38372802734375,
"fcm_dpo/q_t": 0.439905047416687,
"grad_norm": 51.0541877746582,
"learning_rate": 8.166809758815895e-09,
"logits/chosen": -0.7947054505348206,
"logits/rejected": -0.8145523071289062,
"logps/chosen": -566.6533813476562,
"logps/ref_chosen": -273.5590515136719,
"logps/ref_rejected": -264.0199279785156,
"logps/rejected": -665.4979858398438,
"loss": 4.8112,
"margin_dpo/margin_mean": 108.38371276855469,
"margin_dpo/margin_std": 190.74830627441406,
"step": 443
},
{
"KL/chosen_KL_mean": -290.04559326171875,
"KL/mean": -350.908203125,
"KL/rejected_KL_mean": -411.7707824707031,
"KL/std": 195.36392211914062,
"epoch": 0.9298429319371728,
"fcm_dpo/beta": 0.002323201159015298,
"fcm_dpo/delta": 0.019285790622234344,
"fcm_dpo/margin": 121.72520446777344,
"fcm_dpo/q_t": 0.4358820617198944,
"grad_norm": 63.36241149902344,
"learning_rate": 7.709181040498253e-09,
"logits/chosen": -0.8152733445167542,
"logits/rejected": -0.8027286529541016,
"logps/chosen": -588.1897583007812,
"logps/ref_chosen": -298.1441955566406,
"logps/ref_rejected": -268.0572814941406,
"logps/rejected": -679.8280639648438,
"loss": 4.7722,
"margin_dpo/margin_mean": 121.72520446777344,
"margin_dpo/margin_std": 227.64364624023438,
"step": 444
},
{
"KL/chosen_KL_mean": -290.1022033691406,
"KL/mean": -345.72955322265625,
"KL/rejected_KL_mean": -401.35687255859375,
"KL/std": 185.03326416015625,
"epoch": 0.9319371727748691,
"fcm_dpo/beta": 0.0023218824062496424,
"fcm_dpo/delta": -0.025263221934437752,
"fcm_dpo/margin": 111.25466918945312,
"fcm_dpo/q_t": 0.4408957064151764,
"grad_norm": 66.18413543701172,
"learning_rate": 7.2645456434869965e-09,
"logits/chosen": -0.8810672163963318,
"logits/rejected": -0.8841793537139893,
"logps/chosen": -544.6429443359375,
"logps/ref_chosen": -254.54067993164062,
"logps/ref_rejected": -264.2445983886719,
"logps/rejected": -665.6015014648438,
"loss": 4.8478,
"margin_dpo/margin_mean": 111.25466918945312,
"margin_dpo/margin_std": 212.44100952148438,
"step": 445
},
{
"KL/chosen_KL_mean": -286.95159912109375,
"KL/mean": -347.5931396484375,
"KL/rejected_KL_mean": -408.2346496582031,
"KL/std": 178.93850708007812,
"epoch": 0.9340314136125655,
"fcm_dpo/beta": 0.0022728617768734694,
"fcm_dpo/delta": -0.010188797488808632,
"fcm_dpo/margin": 121.28301239013672,
"fcm_dpo/q_t": 0.43673738837242126,
"grad_norm": 47.07920837402344,
"learning_rate": 6.832927412229017e-09,
"logits/chosen": -0.7991313338279724,
"logits/rejected": -0.803611159324646,
"logps/chosen": -593.674072265625,
"logps/ref_chosen": -306.72247314453125,
"logps/ref_rejected": -266.3735656738281,
"logps/rejected": -674.6082153320312,
"loss": 4.755,
"margin_dpo/margin_mean": 121.28301239013672,
"margin_dpo/margin_std": 211.34518432617188,
"step": 446
},
{
"KL/chosen_KL_mean": -279.7216491699219,
"KL/mean": -348.5470886230469,
"KL/rejected_KL_mean": -417.3725280761719,
"KL/std": 182.1820526123047,
"epoch": 0.9361256544502617,
"fcm_dpo/beta": 0.0022493728902190924,
"fcm_dpo/delta": -0.012385351583361626,
"fcm_dpo/margin": 137.65090942382812,
"fcm_dpo/q_t": 0.42796117067337036,
"grad_norm": 61.105281829833984,
"learning_rate": 6.414349493100129e-09,
"logits/chosen": -0.8082762956619263,
"logits/rejected": -0.7996385097503662,
"logps/chosen": -540.2389526367188,
"logps/ref_chosen": -260.51727294921875,
"logps/ref_rejected": -236.47061157226562,
"logps/rejected": -653.8431396484375,
"loss": 4.6265,
"margin_dpo/margin_mean": 137.65089416503906,
"margin_dpo/margin_std": 203.60350036621094,
"step": 447
},
{
"KL/chosen_KL_mean": -301.380859375,
"KL/mean": -365.99871826171875,
"KL/rejected_KL_mean": -430.6165466308594,
"KL/std": 176.27540588378906,
"epoch": 0.9382198952879581,
"fcm_dpo/beta": 0.0022281836718320847,
"fcm_dpo/delta": -0.011953938752412796,
"fcm_dpo/margin": 129.2357177734375,
"fcm_dpo/q_t": 0.43249234557151794,
"grad_norm": 69.26811981201172,
"learning_rate": 6.0088343331638756e-09,
"logits/chosen": -0.809171736240387,
"logits/rejected": -0.798628568649292,
"logps/chosen": -570.1679077148438,
"logps/ref_chosen": -268.78704833984375,
"logps/ref_rejected": -262.1703796386719,
"logps/rejected": -692.7869873046875,
"loss": 4.6706,
"margin_dpo/margin_mean": 129.23573303222656,
"margin_dpo/margin_std": 189.93157958984375,
"step": 448
},
{
"KL/chosen_KL_mean": -287.81689453125,
"KL/mean": -346.68743896484375,
"KL/rejected_KL_mean": -405.5579833984375,
"KL/std": 176.11363220214844,
"epoch": 0.9403141361256544,
"fcm_dpo/beta": 0.0022455663420259953,
"fcm_dpo/delta": 0.027673836797475815,
"fcm_dpo/margin": 117.74109649658203,
"fcm_dpo/q_t": 0.43690261244773865,
"grad_norm": 67.57733154296875,
"learning_rate": 5.616403678967624e-09,
"logits/chosen": -0.9088640809059143,
"logits/rejected": -0.8916957378387451,
"logps/chosen": -618.768310546875,
"logps/ref_chosen": -330.9514465332031,
"logps/ref_rejected": -239.76974487304688,
"logps/rejected": -645.3276977539062,
"loss": 4.7859,
"margin_dpo/margin_mean": 117.7410888671875,
"margin_dpo/margin_std": 208.05458068847656,
"step": 449
},
{
"KL/chosen_KL_mean": -308.811767578125,
"KL/mean": -362.2073974609375,
"KL/rejected_KL_mean": -415.60302734375,
"KL/std": 188.40576171875,
"epoch": 0.9424083769633508,
"fcm_dpo/beta": 0.0022853235714137554,
"fcm_dpo/delta": 0.005974981002509594,
"fcm_dpo/margin": 106.791259765625,
"fcm_dpo/q_t": 0.4430885910987854,
"grad_norm": 47.89699935913086,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": -0.8070337772369385,
"logits/rejected": -0.7972606420516968,
"logps/chosen": -593.0771484375,
"logps/ref_chosen": -284.26544189453125,
"logps/ref_rejected": -250.5401611328125,
"logps/rejected": -666.1432495117188,
"loss": 4.8339,
"margin_dpo/margin_mean": 106.791259765625,
"margin_dpo/margin_std": 189.6084442138672,
"step": 450
},
{
"KL/chosen_KL_mean": -319.8152770996094,
"KL/mean": -364.75592041015625,
"KL/rejected_KL_mean": -409.6966247558594,
"KL/std": 178.8978729248047,
"epoch": 0.9445026178010472,
"fcm_dpo/beta": 0.002270276425406337,
"fcm_dpo/delta": -0.012955044396221638,
"fcm_dpo/margin": 89.88134765625,
"fcm_dpo/q_t": 0.4515979290008545,
"grad_norm": 90.45854949951172,
"learning_rate": 4.8708793644441086e-09,
"logits/chosen": -0.8183082938194275,
"logits/rejected": -0.7841957807540894,
"logps/chosen": -622.1361694335938,
"logps/ref_chosen": -302.3209228515625,
"logps/ref_rejected": -254.09747314453125,
"logps/rejected": -663.7941284179688,
"loss": 4.9745,
"margin_dpo/margin_mean": 89.88135528564453,
"margin_dpo/margin_std": 191.08470153808594,
"step": 451
},
{
"KL/chosen_KL_mean": -290.7073059082031,
"KL/mean": -364.1423034667969,
"KL/rejected_KL_mean": -437.57733154296875,
"KL/std": 219.24545288085938,
"epoch": 0.9465968586387434,
"fcm_dpo/beta": 0.002213032217696309,
"fcm_dpo/delta": -0.030287623405456543,
"fcm_dpo/margin": 146.87002563476562,
"fcm_dpo/q_t": 0.4293460249900818,
"grad_norm": 59.19186782836914,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": -0.8647107481956482,
"logits/rejected": -0.8348664045333862,
"logps/chosen": -590.0994262695312,
"logps/ref_chosen": -299.39215087890625,
"logps/ref_rejected": -284.3475036621094,
"logps/rejected": -721.9248046875,
"loss": 4.6751,
"margin_dpo/margin_mean": 146.87001037597656,
"margin_dpo/margin_std": 258.558349609375,
"step": 452
},
{
"KL/chosen_KL_mean": -314.28314208984375,
"KL/mean": -375.9656066894531,
"KL/rejected_KL_mean": -437.6480712890625,
"KL/std": 184.8981170654297,
"epoch": 0.9486910994764398,
"fcm_dpo/beta": 0.0022319061681628227,
"fcm_dpo/delta": 0.018467098474502563,
"fcm_dpo/margin": 123.36490631103516,
"fcm_dpo/q_t": 0.4363798201084137,
"grad_norm": 54.6486930847168,
"learning_rate": 4.1779364682113794e-09,
"logits/chosen": -0.8098393082618713,
"logits/rejected": -0.7993586659431458,
"logps/chosen": -638.934814453125,
"logps/ref_chosen": -324.6517028808594,
"logps/ref_rejected": -304.1527099609375,
"logps/rejected": -741.80078125,
"loss": 4.7523,
"margin_dpo/margin_mean": 123.36491394042969,
"margin_dpo/margin_std": 219.076904296875,
"step": 453
},
{
"KL/chosen_KL_mean": -299.9391174316406,
"KL/mean": -372.45458984375,
"KL/rejected_KL_mean": -444.969970703125,
"KL/std": 207.7763671875,
"epoch": 0.9507853403141361,
"fcm_dpo/beta": 0.002227374352514744,
"fcm_dpo/delta": -0.015013519674539566,
"fcm_dpo/margin": 145.03086853027344,
"fcm_dpo/q_t": 0.4254119396209717,
"grad_norm": 63.96196365356445,
"learning_rate": 3.851229943335393e-09,
"logits/chosen": -0.8499652743339539,
"logits/rejected": -0.8552207946777344,
"logps/chosen": -599.5509033203125,
"logps/ref_chosen": -299.6117248535156,
"logps/ref_rejected": -303.74224853515625,
"logps/rejected": -748.7122802734375,
"loss": 4.6208,
"margin_dpo/margin_mean": 145.03086853027344,
"margin_dpo/margin_std": 229.23269653320312,
"step": 454
},
{
"KL/chosen_KL_mean": -292.7987976074219,
"KL/mean": -337.0995178222656,
"KL/rejected_KL_mean": -381.40020751953125,
"KL/std": 183.76556396484375,
"epoch": 0.9528795811518325,
"fcm_dpo/beta": 0.0022193007171154022,
"fcm_dpo/delta": 0.00029542212723754346,
"fcm_dpo/margin": 88.60137939453125,
"fcm_dpo/q_t": 0.45545342564582825,
"grad_norm": 84.49998474121094,
"learning_rate": 3.5377236299748147e-09,
"logits/chosen": -0.8165888786315918,
"logits/rejected": -0.8248051404953003,
"logps/chosen": -566.4105224609375,
"logps/ref_chosen": -273.6116943359375,
"logps/ref_rejected": -274.4293518066406,
"logps/rejected": -655.82958984375,
"loss": 5.0176,
"margin_dpo/margin_mean": 88.60137939453125,
"margin_dpo/margin_std": 203.42779541015625,
"step": 455
},
{
"KL/chosen_KL_mean": -318.74468994140625,
"KL/mean": -383.7198791503906,
"KL/rejected_KL_mean": -448.6950988769531,
"KL/std": 208.6256866455078,
"epoch": 0.9549738219895288,
"fcm_dpo/beta": 0.0022203458938747644,
"fcm_dpo/delta": -0.008752606809139252,
"fcm_dpo/margin": 129.9503936767578,
"fcm_dpo/q_t": 0.4367368221282959,
"grad_norm": 81.42244720458984,
"learning_rate": 3.2374343405217884e-09,
"logits/chosen": -0.7438274621963501,
"logits/rejected": -0.7582361698150635,
"logps/chosen": -640.9166259765625,
"logps/ref_chosen": -322.17193603515625,
"logps/ref_rejected": -294.54461669921875,
"logps/rejected": -743.2396240234375,
"loss": 4.7733,
"margin_dpo/margin_mean": 129.95040893554688,
"margin_dpo/margin_std": 247.870849609375,
"step": 456
},
{
"KL/chosen_KL_mean": -315.93475341796875,
"KL/mean": -383.9335632324219,
"KL/rejected_KL_mean": -451.9323425292969,
"KL/std": 205.77975463867188,
"epoch": 0.9570680628272251,
"fcm_dpo/beta": 0.0022161712404340506,
"fcm_dpo/delta": 0.010278364643454552,
"fcm_dpo/margin": 135.99758911132812,
"fcm_dpo/q_t": 0.4274827241897583,
"grad_norm": 58.002906799316406,
"learning_rate": 2.9503781785795713e-09,
"logits/chosen": -0.8158080577850342,
"logits/rejected": -0.8166444301605225,
"logps/chosen": -623.73095703125,
"logps/ref_chosen": -307.7962341308594,
"logps/ref_rejected": -274.5501403808594,
"logps/rejected": -726.4824829101562,
"loss": 4.7231,
"margin_dpo/margin_mean": 135.99757385253906,
"margin_dpo/margin_std": 236.98309326171875,
"step": 457
},
{
"KL/chosen_KL_mean": -307.9447021484375,
"KL/mean": -363.36785888671875,
"KL/rejected_KL_mean": -418.79107666015625,
"KL/std": 202.58853149414062,
"epoch": 0.9591623036649215,
"fcm_dpo/beta": 0.002248093718662858,
"fcm_dpo/delta": 0.02113131433725357,
"fcm_dpo/margin": 110.84642028808594,
"fcm_dpo/q_t": 0.4432235360145569,
"grad_norm": 47.68861770629883,
"learning_rate": 2.6765705380989432e-09,
"logits/chosen": -0.8353589177131653,
"logits/rejected": -0.8087614178657532,
"logps/chosen": -604.976318359375,
"logps/ref_chosen": -297.0316467285156,
"logps/ref_rejected": -276.1112365722656,
"logps/rejected": -694.9022827148438,
"loss": 4.8799,
"margin_dpo/margin_mean": 110.84642028808594,
"margin_dpo/margin_std": 226.8346710205078,
"step": 458
},
{
"KL/chosen_KL_mean": -289.771484375,
"KL/mean": -339.0989990234375,
"KL/rejected_KL_mean": -388.4265441894531,
"KL/std": 170.22535705566406,
"epoch": 0.9612565445026178,
"fcm_dpo/beta": 0.002288884948939085,
"fcm_dpo/delta": 0.03183550387620926,
"fcm_dpo/margin": 98.65508270263672,
"fcm_dpo/q_t": 0.44840818643569946,
"grad_norm": 57.28891372680664,
"learning_rate": 2.416026102552732e-09,
"logits/chosen": -0.8987852931022644,
"logits/rejected": -0.8900645971298218,
"logps/chosen": -583.2967529296875,
"logps/ref_chosen": -293.5252990722656,
"logps/ref_rejected": -289.30126953125,
"logps/rejected": -677.7278442382812,
"loss": 4.9211,
"margin_dpo/margin_mean": 98.65508270263672,
"margin_dpo/margin_std": 204.67257690429688,
"step": 459
},
{
"KL/chosen_KL_mean": -304.1756591796875,
"KL/mean": -353.0399169921875,
"KL/rejected_KL_mean": -401.90423583984375,
"KL/std": 166.08837890625,
"epoch": 0.9633507853403142,
"fcm_dpo/beta": 0.0023258451838046312,
"fcm_dpo/delta": 0.0077877165749669075,
"fcm_dpo/margin": 97.72859954833984,
"fcm_dpo/q_t": 0.44643843173980713,
"grad_norm": 67.08515930175781,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": -0.8612726330757141,
"logits/rejected": -0.8576234579086304,
"logps/chosen": -622.9559936523438,
"logps/ref_chosen": -318.7803649902344,
"logps/ref_rejected": -258.7906799316406,
"logps/rejected": -660.6948852539062,
"loss": 4.9022,
"margin_dpo/margin_mean": 97.72859954833984,
"margin_dpo/margin_std": 199.7197265625,
"step": 460
},
{
"KL/chosen_KL_mean": -294.57061767578125,
"KL/mean": -356.40191650390625,
"KL/rejected_KL_mean": -418.23309326171875,
"KL/std": 190.1246795654297,
"epoch": 0.9654450261780104,
"fcm_dpo/beta": 0.002357393503189087,
"fcm_dpo/delta": 0.020651711151003838,
"fcm_dpo/margin": 123.66246032714844,
"fcm_dpo/q_t": 0.4340013265609741,
"grad_norm": 62.43252182006836,
"learning_rate": 1.9347820230782295e-09,
"logits/chosen": -0.817859947681427,
"logits/rejected": -0.838330864906311,
"logps/chosen": -538.4805908203125,
"logps/ref_chosen": -243.9099884033203,
"logps/ref_rejected": -232.6382293701172,
"logps/rejected": -650.871337890625,
"loss": 4.7388,
"margin_dpo/margin_mean": 123.66246032714844,
"margin_dpo/margin_std": 221.1026611328125,
"step": 461
},
{
"KL/chosen_KL_mean": -294.6795959472656,
"KL/mean": -362.9072570800781,
"KL/rejected_KL_mean": -431.13494873046875,
"KL/std": 176.5676727294922,
"epoch": 0.9675392670157068,
"fcm_dpo/beta": 0.002386984182521701,
"fcm_dpo/delta": 0.009113818407058716,
"fcm_dpo/margin": 136.45535278320312,
"fcm_dpo/q_t": 0.4248298108577728,
"grad_norm": 64.27290344238281,
"learning_rate": 1.7141081868094209e-09,
"logits/chosen": -0.8587002158164978,
"logits/rejected": -0.8031215667724609,
"logps/chosen": -638.7705078125,
"logps/ref_chosen": -344.09100341796875,
"logps/ref_rejected": -252.45037841796875,
"logps/rejected": -683.5853271484375,
"loss": 4.5764,
"margin_dpo/margin_mean": 136.45535278320312,
"margin_dpo/margin_std": 198.38424682617188,
"step": 462
},
{
"KL/chosen_KL_mean": -298.1247253417969,
"KL/mean": -349.39569091796875,
"KL/rejected_KL_mean": -400.66668701171875,
"KL/std": 196.35546875,
"epoch": 0.9696335078534032,
"fcm_dpo/beta": 0.0023907949216663837,
"fcm_dpo/delta": -0.0031707417219877243,
"fcm_dpo/margin": 102.54196166992188,
"fcm_dpo/q_t": 0.4443747103214264,
"grad_norm": 63.04395294189453,
"learning_rate": 1.5067491694100153e-09,
"logits/chosen": -0.841509222984314,
"logits/rejected": -0.7965072393417358,
"logps/chosen": -595.2672119140625,
"logps/ref_chosen": -297.1424560546875,
"logps/ref_rejected": -234.0208282470703,
"logps/rejected": -634.6875610351562,
"loss": 4.9414,
"margin_dpo/margin_mean": 102.54196166992188,
"margin_dpo/margin_std": 233.15066528320312,
"step": 463
},
{
"KL/chosen_KL_mean": -303.274658203125,
"KL/mean": -358.65533447265625,
"KL/rejected_KL_mean": -414.0360412597656,
"KL/std": 182.3558807373047,
"epoch": 0.9717277486910995,
"fcm_dpo/beta": 0.002398580312728882,
"fcm_dpo/delta": 0.004773720167577267,
"fcm_dpo/margin": 110.76134490966797,
"fcm_dpo/q_t": 0.43780282139778137,
"grad_norm": 74.20742797851562,
"learning_rate": 1.3127160909147672e-09,
"logits/chosen": -0.8260834217071533,
"logits/rejected": -0.8394900560379028,
"logps/chosen": -568.9854125976562,
"logps/ref_chosen": -265.71075439453125,
"logps/ref_rejected": -256.4108581542969,
"logps/rejected": -670.4468994140625,
"loss": 4.7798,
"margin_dpo/margin_mean": 110.76134490966797,
"margin_dpo/margin_std": 196.59963989257812,
"step": 464
},
{
"KL/chosen_KL_mean": -284.39501953125,
"KL/mean": -354.4608459472656,
"KL/rejected_KL_mean": -424.52667236328125,
"KL/std": 174.48228454589844,
"epoch": 0.9738219895287958,
"fcm_dpo/beta": 0.0023324608337134123,
"fcm_dpo/delta": -0.03364231437444687,
"fcm_dpo/margin": 140.13165283203125,
"fcm_dpo/q_t": 0.42524397373199463,
"grad_norm": 59.864524841308594,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": -0.9016032814979553,
"logits/rejected": -0.8606617450714111,
"logps/chosen": -577.5477294921875,
"logps/ref_chosen": -293.1527404785156,
"logps/ref_rejected": -293.70947265625,
"logps/rejected": -718.2361450195312,
"loss": 4.5923,
"margin_dpo/margin_mean": 140.13165283203125,
"margin_dpo/margin_std": 205.0211181640625,
"step": 465
},
{
"KL/chosen_KL_mean": -270.7398986816406,
"KL/mean": -341.1874084472656,
"KL/rejected_KL_mean": -411.63494873046875,
"KL/std": 175.04595947265625,
"epoch": 0.9759162303664921,
"fcm_dpo/beta": 0.002325007924810052,
"fcm_dpo/delta": 0.006551696918904781,
"fcm_dpo/margin": 140.89501953125,
"fcm_dpo/q_t": 0.425047367811203,
"grad_norm": 67.3387680053711,
"learning_rate": 9.64668657069706e-10,
"logits/chosen": -0.8157132863998413,
"logits/rejected": -0.7559111714363098,
"logps/chosen": -532.2174682617188,
"logps/ref_chosen": -261.4775695800781,
"logps/ref_rejected": -248.36282348632812,
"logps/rejected": -659.9977416992188,
"loss": 4.5809,
"margin_dpo/margin_mean": 140.89501953125,
"margin_dpo/margin_std": 209.25616455078125,
"step": 466
},
{
"KL/chosen_KL_mean": -294.3692932128906,
"KL/mean": -352.0955810546875,
"KL/rejected_KL_mean": -409.8218688964844,
"KL/std": 185.36192321777344,
"epoch": 0.9780104712041885,
"fcm_dpo/beta": 0.0023117591626942158,
"fcm_dpo/delta": -0.017675073817372322,
"fcm_dpo/margin": 115.45263671875,
"fcm_dpo/q_t": 0.43935906887054443,
"grad_norm": 70.38238525390625,
"learning_rate": 8.106729664475176e-10,
"logits/chosen": -0.8092154860496521,
"logits/rejected": -0.7998495101928711,
"logps/chosen": -560.7235107421875,
"logps/ref_chosen": -266.354248046875,
"logps/ref_rejected": -277.76324462890625,
"logps/rejected": -687.5851440429688,
"loss": 4.8215,
"margin_dpo/margin_mean": 115.45262908935547,
"margin_dpo/margin_std": 225.523193359375,
"step": 467
},
{
"KL/chosen_KL_mean": -300.00885009765625,
"KL/mean": -342.2411193847656,
"KL/rejected_KL_mean": -384.4734191894531,
"KL/std": 174.18258666992188,
"epoch": 0.9801047120418848,
"fcm_dpo/beta": 0.00232279347255826,
"fcm_dpo/delta": 0.02214384824037552,
"fcm_dpo/margin": 84.46463012695312,
"fcm_dpo/q_t": 0.45253071188926697,
"grad_norm": 91.43498992919922,
"learning_rate": 6.700405431837585e-10,
"logits/chosen": -0.8857331275939941,
"logits/rejected": -0.8545862436294556,
"logps/chosen": -617.9720458984375,
"logps/ref_chosen": -317.9631652832031,
"logps/ref_rejected": -261.8744201660156,
"logps/rejected": -646.3478393554688,
"loss": 5.0639,
"margin_dpo/margin_mean": 84.46461486816406,
"margin_dpo/margin_std": 214.5466766357422,
"step": 468
},
{
"KL/chosen_KL_mean": -282.91015625,
"KL/mean": -349.28240966796875,
"KL/rejected_KL_mean": -415.6546630859375,
"KL/std": 187.77842712402344,
"epoch": 0.9821989528795811,
"fcm_dpo/beta": 0.0023529785685241222,
"fcm_dpo/delta": -0.0062449611723423,
"fcm_dpo/margin": 132.7445068359375,
"fcm_dpo/q_t": 0.4277336299419403,
"grad_norm": 43.829017639160156,
"learning_rate": 5.427789289685347e-10,
"logits/chosen": -0.8155910968780518,
"logits/rejected": -0.7978315353393555,
"logps/chosen": -607.7969360351562,
"logps/ref_chosen": -324.8868103027344,
"logps/ref_rejected": -264.0421447753906,
"logps/rejected": -679.6968383789062,
"loss": 4.6583,
"margin_dpo/margin_mean": 132.7445068359375,
"margin_dpo/margin_std": 217.20770263671875,
"step": 469
},
{
"KL/chosen_KL_mean": -283.00579833984375,
"KL/mean": -351.0141296386719,
"KL/rejected_KL_mean": -419.0224609375,
"KL/std": 183.21075439453125,
"epoch": 0.9842931937172775,
"fcm_dpo/beta": 0.0023244861513376236,
"fcm_dpo/delta": -0.011775792576372623,
"fcm_dpo/margin": 136.0166778564453,
"fcm_dpo/q_t": 0.427010178565979,
"grad_norm": 60.516151428222656,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": -0.8107550144195557,
"logits/rejected": -0.8079327344894409,
"logps/chosen": -597.7100830078125,
"logps/ref_chosen": -314.7042541503906,
"logps/ref_rejected": -259.2276611328125,
"logps/rejected": -678.2501220703125,
"loss": 4.6326,
"margin_dpo/margin_mean": 136.0166778564453,
"margin_dpo/margin_std": 210.85792541503906,
"step": 470
},
{
"KL/chosen_KL_mean": -300.8998107910156,
"KL/mean": -368.5666809082031,
"KL/rejected_KL_mean": -436.23358154296875,
"KL/std": 217.97096252441406,
"epoch": 0.9863874345549738,
"fcm_dpo/beta": 0.002303960733115673,
"fcm_dpo/delta": -0.020671039819717407,
"fcm_dpo/margin": 135.33377075195312,
"fcm_dpo/q_t": 0.4327457547187805,
"grad_norm": 58.01945114135742,
"learning_rate": 3.2839470889836627e-10,
"logits/chosen": -0.860909640789032,
"logits/rejected": -0.8448034524917603,
"logps/chosen": -593.4747314453125,
"logps/ref_chosen": -292.5748291015625,
"logps/ref_rejected": -298.7525329589844,
"logps/rejected": -734.9861450195312,
"loss": 4.7235,
"margin_dpo/margin_mean": 135.33377075195312,
"margin_dpo/margin_std": 248.5484619140625,
"step": 471
},
{
"KL/chosen_KL_mean": -275.46673583984375,
"KL/mean": -348.41241455078125,
"KL/rejected_KL_mean": -421.358154296875,
"KL/std": 182.7490234375,
"epoch": 0.9884816753926702,
"fcm_dpo/beta": 0.0022689655888825655,
"fcm_dpo/delta": 0.00046153878793120384,
"fcm_dpo/margin": 145.8914337158203,
"fcm_dpo/q_t": 0.4243175983428955,
"grad_norm": 60.666263580322266,
"learning_rate": 2.412835998185092e-10,
"logits/chosen": -0.8494566082954407,
"logits/rejected": -0.8609163165092468,
"logps/chosen": -518.840576171875,
"logps/ref_chosen": -243.37380981445312,
"logps/ref_rejected": -251.12109375,
"logps/rejected": -672.479248046875,
"loss": 4.5793,
"margin_dpo/margin_mean": 145.8914337158203,
"margin_dpo/margin_std": 217.2068328857422,
"step": 472
},
{
"KL/chosen_KL_mean": -293.68231201171875,
"KL/mean": -360.2905578613281,
"KL/rejected_KL_mean": -426.89886474609375,
"KL/std": 168.0052490234375,
"epoch": 0.9905759162303664,
"fcm_dpo/beta": 0.0023105614818632603,
"fcm_dpo/delta": 0.030130187049508095,
"fcm_dpo/margin": 133.216552734375,
"fcm_dpo/q_t": 0.42758381366729736,
"grad_norm": 48.50118637084961,
"learning_rate": 1.6756629272085544e-10,
"logits/chosen": -0.7985444664955139,
"logits/rejected": -0.7936415076255798,
"logps/chosen": -580.010986328125,
"logps/ref_chosen": -286.3286437988281,
"logps/ref_rejected": -258.6535339355469,
"logps/rejected": -685.5524291992188,
"loss": 4.5982,
"margin_dpo/margin_mean": 133.216552734375,
"margin_dpo/margin_std": 184.37120056152344,
"step": 473
},
{
"KL/chosen_KL_mean": -295.7257385253906,
"KL/mean": -356.5581359863281,
"KL/rejected_KL_mean": -417.39056396484375,
"KL/std": 190.985595703125,
"epoch": 0.9926701570680628,
"fcm_dpo/beta": 0.002310484182089567,
"fcm_dpo/delta": -0.016346491873264313,
"fcm_dpo/margin": 121.6648178100586,
"fcm_dpo/q_t": 0.4347348213195801,
"grad_norm": 64.4442138671875,
"learning_rate": 1.072467408408384e-10,
"logits/chosen": -0.8563570976257324,
"logits/rejected": -0.8490430116653442,
"logps/chosen": -583.8153686523438,
"logps/ref_chosen": -288.08966064453125,
"logps/ref_rejected": -266.69696044921875,
"logps/rejected": -684.0875854492188,
"loss": 4.7213,
"margin_dpo/margin_mean": 121.6648178100586,
"margin_dpo/margin_std": 197.94935607910156,
"step": 474
},
{
"KL/chosen_KL_mean": -274.5186462402344,
"KL/mean": -328.12591552734375,
"KL/rejected_KL_mean": -381.73321533203125,
"KL/std": 163.69903564453125,
"epoch": 0.9947643979057592,
"fcm_dpo/beta": 0.002282880013808608,
"fcm_dpo/delta": -0.018598873168230057,
"fcm_dpo/margin": 107.21459197998047,
"fcm_dpo/q_t": 0.44367164373397827,
"grad_norm": 66.61784362792969,
"learning_rate": 6.032817893297793e-11,
"logits/chosen": -0.8387801051139832,
"logits/rejected": -0.8456286191940308,
"logps/chosen": -530.521728515625,
"logps/ref_chosen": -256.0030517578125,
"logps/ref_rejected": -244.50660705566406,
"logps/rejected": -626.2398071289062,
"loss": 4.8252,
"margin_dpo/margin_mean": 107.21459197998047,
"margin_dpo/margin_std": 188.63526916503906,
"step": 475
},
{
"KL/chosen_KL_mean": -278.682861328125,
"KL/mean": -335.3662109375,
"KL/rejected_KL_mean": -392.0495910644531,
"KL/std": 165.678466796875,
"epoch": 0.9968586387434555,
"fcm_dpo/beta": 0.002227974822744727,
"fcm_dpo/delta": -0.037535179406404495,
"fcm_dpo/margin": 113.36674499511719,
"fcm_dpo/q_t": 0.4426191449165344,
"grad_norm": 48.4283447265625,
"learning_rate": 2.6813123097352287e-11,
"logits/chosen": -0.8977000713348389,
"logits/rejected": -0.8495609164237976,
"logps/chosen": -600.1503295898438,
"logps/ref_chosen": -321.467529296875,
"logps/ref_rejected": -295.0592956542969,
"logps/rejected": -687.1089477539062,
"loss": 4.8402,
"margin_dpo/margin_mean": 113.36674499511719,
"margin_dpo/margin_std": 209.83367919921875,
"step": 476
},
{
"KL/chosen_KL_mean": -304.5742492675781,
"KL/mean": -361.22320556640625,
"KL/rejected_KL_mean": -417.8720703125,
"KL/std": 196.73390197753906,
"epoch": 0.9989528795811519,
"fcm_dpo/beta": 0.002197434427216649,
"fcm_dpo/delta": -0.011369161307811737,
"fcm_dpo/margin": 113.29786682128906,
"fcm_dpo/q_t": 0.44230157136917114,
"grad_norm": 89.01945495605469,
"learning_rate": 6.7033706447061635e-12,
"logits/chosen": -0.7661444544792175,
"logits/rejected": -0.7697646021842957,
"logps/chosen": -581.3682250976562,
"logps/ref_chosen": -276.7939758300781,
"logps/ref_rejected": -244.82919311523438,
"logps/rejected": -662.7012939453125,
"loss": 4.914,
"margin_dpo/margin_mean": 113.29786682128906,
"margin_dpo/margin_std": 247.40846252441406,
"step": 477
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 4.88491684035935,
"train_runtime": 6567.2262,
"train_samples_per_second": 9.309,
"train_steps_per_second": 0.073
}
],
"logging_steps": 1,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink