W-61/qwen3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.6-20260430-165125
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
qwen3-8b-base-new-dpo-ultra…/trainer_state.json
ModelHub XC 5e269964b3 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/qwen3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.6-20260430-165125
Source: Original Platform
2026-06-17 18:55:37 +08:00

11015 lines
411 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": -0.04536247253417969,
"KL/mean": -0.05389120429754257,
"KL/rejected_KL_mean": -0.06241607666015625,
"KL/std": 0.5513200759887695,
"epoch": 0.0020942408376963353,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01704825460910797,
"fcm_dpo/q_t": 0.499957412481308,
"grad_norm": 14.618375778198242,
"learning_rate": 0.0,
"logits/chosen": 1.9854813814163208,
"logits/rejected": 1.7737188339233398,
"logps/chosen": -267.0936584472656,
"logps/ref_chosen": -267.04827880859375,
"logps/ref_rejected": -219.76657104492188,
"logps/rejected": -219.82899475097656,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.01704716682434082,
"margin_dpo/margin_std": 0.7467707991600037,
"step": 1
},
{
"KL/chosen_KL_mean": 0.07550811767578125,
"KL/mean": 0.03060000389814377,
"KL/rejected_KL_mean": -0.014301300048828125,
"KL/std": 0.6403332352638245,
"epoch": 0.004188481675392671,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08979703485965729,
"fcm_dpo/q_t": 0.4997755289077759,
"grad_norm": 15.139668464660645,
"learning_rate": 1.0416666666666666e-08,
"logits/chosen": 2.1993117332458496,
"logits/rejected": 2.1368322372436523,
"logps/chosen": -256.5101623535156,
"logps/ref_chosen": -256.585693359375,
"logps/ref_rejected": -239.92742919921875,
"logps/rejected": -239.94171142578125,
"loss": 5.5417,
"margin_dpo/margin_mean": 0.08979731798171997,
"margin_dpo/margin_std": 0.9745746850967407,
"step": 2
},
{
"KL/chosen_KL_mean": -0.04197883605957031,
"KL/mean": -0.04229529947042465,
"KL/rejected_KL_mean": -0.04261016845703125,
"KL/std": 0.6228253245353699,
"epoch": 0.0062827225130890054,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0006283968687057495,
"fcm_dpo/q_t": 0.49999845027923584,
"grad_norm": 14.598505973815918,
"learning_rate": 2.083333333333333e-08,
"logits/chosen": 2.2566211223602295,
"logits/rejected": 2.2960007190704346,
"logps/chosen": -265.71087646484375,
"logps/ref_chosen": -265.66888427734375,
"logps/ref_rejected": -284.3778991699219,
"logps/rejected": -284.4205322265625,
"loss": 5.5452,
"margin_dpo/margin_mean": 0.0006298869848251343,
"margin_dpo/margin_std": 0.8659858107566833,
"step": 3
},
{
"KL/chosen_KL_mean": 0.003345489501953125,
"KL/mean": -0.003935039043426514,
"KL/rejected_KL_mean": -0.011201858520507812,
"KL/std": 0.6224133968353271,
"epoch": 0.008376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01454666256904602,
"fcm_dpo/q_t": 0.4999636113643646,
"grad_norm": 15.804097175598145,
"learning_rate": 3.125e-08,
"logits/chosen": 1.9357259273529053,
"logits/rejected": 1.8748366832733154,
"logps/chosen": -319.5041198730469,
"logps/ref_chosen": -319.50750732421875,
"logps/ref_rejected": -296.22314453125,
"logps/rejected": -296.2343444824219,
"loss": 5.5447,
"margin_dpo/margin_mean": 0.014546826481819153,
"margin_dpo/margin_std": 0.8421736359596252,
"step": 4
},
{
"KL/chosen_KL_mean": 0.02983856201171875,
"KL/mean": 0.04574793577194214,
"KL/rejected_KL_mean": 0.0616607666015625,
"KL/std": 0.601071298122406,
"epoch": 0.010471204188481676,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0318109393119812,
"fcm_dpo/q_t": 0.5000795125961304,
"grad_norm": 15.678240776062012,
"learning_rate": 4.166666666666666e-08,
"logits/chosen": 1.8491719961166382,
"logits/rejected": 1.851250410079956,
"logps/chosen": -292.8431396484375,
"logps/ref_chosen": -292.87298583984375,
"logps/ref_rejected": -270.3802795410156,
"logps/rejected": -270.3186340332031,
"loss": 5.5465,
"margin_dpo/margin_mean": -0.03181034326553345,
"margin_dpo/margin_std": 0.9028931260108948,
"step": 5
},
{
"KL/chosen_KL_mean": 0.018728256225585938,
"KL/mean": 0.017683424055576324,
"KL/rejected_KL_mean": 0.0166473388671875,
"KL/std": 0.6028397083282471,
"epoch": 0.012565445026178011,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.002083107829093933,
"fcm_dpo/q_t": 0.49999481439590454,
"grad_norm": 15.508500099182129,
"learning_rate": 5.208333333333333e-08,
"logits/chosen": 2.140822649002075,
"logits/rejected": 2.076831340789795,
"logps/chosen": -280.2667236328125,
"logps/ref_chosen": -280.28546142578125,
"logps/ref_rejected": -242.7611541748047,
"logps/rejected": -242.74449157714844,
"loss": 5.5452,
"margin_dpo/margin_mean": 0.0020822137594223022,
"margin_dpo/margin_std": 0.8873875141143799,
"step": 6
},
{
"KL/chosen_KL_mean": 0.002033233642578125,
"KL/mean": 0.03911462053656578,
"KL/rejected_KL_mean": 0.0761871337890625,
"KL/std": 0.6071482300758362,
"epoch": 0.014659685863874346,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07415292412042618,
"fcm_dpo/q_t": 0.5001853704452515,
"grad_norm": 15.60063648223877,
"learning_rate": 6.25e-08,
"logits/chosen": 2.155738115310669,
"logits/rejected": 2.0826399326324463,
"logps/chosen": -255.1036834716797,
"logps/ref_chosen": -255.10572814941406,
"logps/ref_rejected": -251.15402221679688,
"logps/rejected": -251.07785034179688,
"loss": 5.5482,
"margin_dpo/margin_mean": -0.07415376603603363,
"margin_dpo/margin_std": 0.8465871214866638,
"step": 7
},
{
"KL/chosen_KL_mean": -0.026432037353515625,
"KL/mean": 0.003932002931833267,
"KL/rejected_KL_mean": 0.03430938720703125,
"KL/std": 0.7194060683250427,
"epoch": 0.016753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06074821203947067,
"fcm_dpo/q_t": 0.5001518726348877,
"grad_norm": 15.915802001953125,
"learning_rate": 7.291666666666667e-08,
"logits/chosen": 2.315553903579712,
"logits/rejected": 2.1242852210998535,
"logps/chosen": -345.2080078125,
"logps/ref_chosen": -345.18157958984375,
"logps/ref_rejected": -277.3522644042969,
"logps/rejected": -277.3179931640625,
"loss": 5.5477,
"margin_dpo/margin_mean": -0.06074722111225128,
"margin_dpo/margin_std": 1.0226110219955444,
"step": 8
},
{
"KL/chosen_KL_mean": -0.046703338623046875,
"KL/mean": -0.05963290482759476,
"KL/rejected_KL_mean": -0.07256317138671875,
"KL/std": 0.6330029964447021,
"epoch": 0.018848167539267015,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.025863781571388245,
"fcm_dpo/q_t": 0.4999352991580963,
"grad_norm": 14.079815864562988,
"learning_rate": 8.333333333333333e-08,
"logits/chosen": 2.0162174701690674,
"logits/rejected": 1.9880930185317993,
"logps/chosen": -276.77703857421875,
"logps/ref_chosen": -276.7303161621094,
"logps/ref_rejected": -266.19952392578125,
"logps/rejected": -266.2720642089844,
"loss": 5.5442,
"margin_dpo/margin_mean": 0.02586349844932556,
"margin_dpo/margin_std": 0.8693335056304932,
"step": 9
},
{
"KL/chosen_KL_mean": -0.02152252197265625,
"KL/mean": 0.0014857053756713867,
"KL/rejected_KL_mean": 0.024494171142578125,
"KL/std": 0.5658552050590515,
"epoch": 0.020942408376963352,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04602077603340149,
"fcm_dpo/q_t": 0.5001150369644165,
"grad_norm": 13.844391822814941,
"learning_rate": 9.375e-08,
"logits/chosen": 1.8030073642730713,
"logits/rejected": 1.8423787355422974,
"logps/chosen": -246.5837860107422,
"logps/ref_chosen": -246.562255859375,
"logps/ref_rejected": -224.70401000976562,
"logps/rejected": -224.67950439453125,
"loss": 5.5471,
"margin_dpo/margin_mean": -0.0460197776556015,
"margin_dpo/margin_std": 0.7490549087524414,
"step": 10
},
{
"KL/chosen_KL_mean": -0.0169677734375,
"KL/mean": -0.038240786641836166,
"KL/rejected_KL_mean": -0.05950736999511719,
"KL/std": 0.6162736415863037,
"epoch": 0.023036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.042532242834568024,
"fcm_dpo/q_t": 0.4998936653137207,
"grad_norm": 15.514847755432129,
"learning_rate": 1.0416666666666667e-07,
"logits/chosen": 2.0664830207824707,
"logits/rejected": 2.0137524604797363,
"logps/chosen": -290.51806640625,
"logps/ref_chosen": -290.5010986328125,
"logps/ref_rejected": -253.9559326171875,
"logps/rejected": -254.01541137695312,
"loss": 5.5436,
"margin_dpo/margin_mean": 0.042532920837402344,
"margin_dpo/margin_std": 0.9051563143730164,
"step": 11
},
{
"KL/chosen_KL_mean": 0.0077972412109375,
"KL/mean": -0.016895174980163574,
"KL/rejected_KL_mean": -0.04158592224121094,
"KL/std": 0.6474024653434753,
"epoch": 0.025130890052356022,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04937353730201721,
"fcm_dpo/q_t": 0.49987655878067017,
"grad_norm": 14.951948165893555,
"learning_rate": 1.1458333333333332e-07,
"logits/chosen": 1.7271878719329834,
"logits/rejected": 1.7156167030334473,
"logps/chosen": -278.5107421875,
"logps/ref_chosen": -278.5185241699219,
"logps/ref_rejected": -273.56207275390625,
"logps/rejected": -273.6036682128906,
"loss": 5.5433,
"margin_dpo/margin_mean": 0.049374401569366455,
"margin_dpo/margin_std": 0.8798114061355591,
"step": 12
},
{
"KL/chosen_KL_mean": -0.017948150634765625,
"KL/mean": -0.02015083283185959,
"KL/rejected_KL_mean": -0.0223541259765625,
"KL/std": 0.5520142912864685,
"epoch": 0.027225130890052355,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.00440196692943573,
"fcm_dpo/q_t": 0.49998903274536133,
"grad_norm": 14.794624328613281,
"learning_rate": 1.25e-07,
"logits/chosen": 1.9182813167572021,
"logits/rejected": 2.0527210235595703,
"logps/chosen": -264.6750793457031,
"logps/ref_chosen": -264.6571350097656,
"logps/ref_rejected": -274.3492736816406,
"logps/rejected": -274.3716735839844,
"loss": 5.5451,
"margin_dpo/margin_mean": 0.004402056336402893,
"margin_dpo/margin_std": 0.862347424030304,
"step": 13
},
{
"KL/chosen_KL_mean": 0.053466796875,
"KL/mean": 0.00025077909231185913,
"KL/rejected_KL_mean": -0.052967071533203125,
"KL/std": 0.5912115573883057,
"epoch": 0.02931937172774869,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10642853379249573,
"fcm_dpo/q_t": 0.49973392486572266,
"grad_norm": 15.18773078918457,
"learning_rate": 1.3541666666666666e-07,
"logits/chosen": 2.0496034622192383,
"logits/rejected": 2.0889523029327393,
"logps/chosen": -274.06109619140625,
"logps/ref_chosen": -274.1145935058594,
"logps/ref_rejected": -261.0951843261719,
"logps/rejected": -261.1481628417969,
"loss": 5.541,
"margin_dpo/margin_mean": 0.10642816126346588,
"margin_dpo/margin_std": 0.8385832905769348,
"step": 14
},
{
"KL/chosen_KL_mean": -0.01445770263671875,
"KL/mean": -0.0046601369976997375,
"KL/rejected_KL_mean": 0.00513458251953125,
"KL/std": 0.5840538740158081,
"epoch": 0.031413612565445025,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.019589528441429138,
"fcm_dpo/q_t": 0.5000489950180054,
"grad_norm": 16.10891342163086,
"learning_rate": 1.4583333333333335e-07,
"logits/chosen": 2.1259677410125732,
"logits/rejected": 2.005582094192505,
"logps/chosen": -288.5991516113281,
"logps/ref_chosen": -288.584716796875,
"logps/ref_rejected": -224.8094482421875,
"logps/rejected": -224.8043212890625,
"loss": 5.546,
"margin_dpo/margin_mean": -0.01959007978439331,
"margin_dpo/margin_std": 0.8089416027069092,
"step": 15
},
{
"KL/chosen_KL_mean": 0.035327911376953125,
"KL/mean": 0.012278169393539429,
"KL/rejected_KL_mean": -0.010770797729492188,
"KL/std": 0.600157618522644,
"epoch": 0.033507853403141365,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04610002040863037,
"fcm_dpo/q_t": 0.4998847544193268,
"grad_norm": 16.78972816467285,
"learning_rate": 1.5624999999999999e-07,
"logits/chosen": 2.119436502456665,
"logits/rejected": 1.9895243644714355,
"logps/chosen": -283.80780029296875,
"logps/ref_chosen": -283.8431091308594,
"logps/ref_rejected": -248.63645935058594,
"logps/rejected": -248.64723205566406,
"loss": 5.5434,
"margin_dpo/margin_mean": 0.04610002040863037,
"margin_dpo/margin_std": 0.8715221881866455,
"step": 16
},
{
"KL/chosen_KL_mean": 0.05317878723144531,
"KL/mean": 0.01209481805562973,
"KL/rejected_KL_mean": -0.02899169921875,
"KL/std": 0.5728281736373901,
"epoch": 0.0356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08215896785259247,
"fcm_dpo/q_t": 0.499794602394104,
"grad_norm": 15.140898704528809,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": 1.8333988189697266,
"logits/rejected": 1.786064624786377,
"logps/chosen": -254.5803985595703,
"logps/ref_chosen": -254.6335906982422,
"logps/ref_rejected": -245.3981170654297,
"logps/rejected": -245.4271240234375,
"loss": 5.542,
"margin_dpo/margin_mean": 0.08215945959091187,
"margin_dpo/margin_std": 0.855034351348877,
"step": 17
},
{
"KL/chosen_KL_mean": -0.07054710388183594,
"KL/mean": -0.010584428906440735,
"KL/rejected_KL_mean": 0.04937553405761719,
"KL/std": 0.6170819997787476,
"epoch": 0.03769633507853403,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.11993196606636047,
"fcm_dpo/q_t": 0.5002998113632202,
"grad_norm": 15.51552963256836,
"learning_rate": 1.7708333333333334e-07,
"logits/chosen": 1.833971619606018,
"logits/rejected": 1.7458878755569458,
"logps/chosen": -287.820068359375,
"logps/ref_chosen": -287.7495422363281,
"logps/ref_rejected": -273.03363037109375,
"logps/rejected": -272.9842529296875,
"loss": 5.5501,
"margin_dpo/margin_mean": -0.11993266642093658,
"margin_dpo/margin_std": 0.842696487903595,
"step": 18
},
{
"KL/chosen_KL_mean": -0.050800323486328125,
"KL/mean": -0.01509222760796547,
"KL/rejected_KL_mean": 0.020612716674804688,
"KL/std": 0.6209220886230469,
"epoch": 0.039790575916230364,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07141081243753433,
"fcm_dpo/q_t": 0.5001784563064575,
"grad_norm": 14.881866455078125,
"learning_rate": 1.875e-07,
"logits/chosen": 2.0522818565368652,
"logits/rejected": 2.0182793140411377,
"logps/chosen": -273.4547119140625,
"logps/ref_chosen": -273.4039001464844,
"logps/ref_rejected": -260.09722900390625,
"logps/rejected": -260.0766296386719,
"loss": 5.5481,
"margin_dpo/margin_mean": -0.07141060382127762,
"margin_dpo/margin_std": 0.7947637438774109,
"step": 19
},
{
"KL/chosen_KL_mean": -0.011041641235351562,
"KL/mean": -0.049523673951625824,
"KL/rejected_KL_mean": -0.08799934387207031,
"KL/std": 0.5975103974342346,
"epoch": 0.041884816753926704,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07696382701396942,
"fcm_dpo/q_t": 0.49980759620666504,
"grad_norm": 15.470925331115723,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": 1.8817099332809448,
"logits/rejected": 1.8345247507095337,
"logps/chosen": -292.97314453125,
"logps/ref_chosen": -292.96209716796875,
"logps/ref_rejected": -256.602783203125,
"logps/rejected": -256.6907958984375,
"loss": 5.5422,
"margin_dpo/margin_mean": 0.07696413993835449,
"margin_dpo/margin_std": 0.8018198013305664,
"step": 20
},
{
"KL/chosen_KL_mean": -0.0027065277099609375,
"KL/mean": 0.011075474321842194,
"KL/rejected_KL_mean": 0.02484893798828125,
"KL/std": 0.6228327751159668,
"epoch": 0.04397905759162304,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02755419909954071,
"fcm_dpo/q_t": 0.5000689029693604,
"grad_norm": 15.547636032104492,
"learning_rate": 2.0833333333333333e-07,
"logits/chosen": 2.1370320320129395,
"logits/rejected": 2.0824661254882812,
"logps/chosen": -292.3299255371094,
"logps/ref_chosen": -292.3271789550781,
"logps/ref_rejected": -274.20635986328125,
"logps/rejected": -274.1815185546875,
"loss": 5.5464,
"margin_dpo/margin_mean": -0.02755562961101532,
"margin_dpo/margin_std": 0.8300514817237854,
"step": 21
},
{
"KL/chosen_KL_mean": 0.06711769104003906,
"KL/mean": -0.007669083774089813,
"KL/rejected_KL_mean": -0.08245086669921875,
"KL/std": 0.5317828059196472,
"epoch": 0.04607329842931937,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1495741754770279,
"fcm_dpo/q_t": 0.499626100063324,
"grad_norm": 13.725032806396484,
"learning_rate": 2.1875e-07,
"logits/chosen": 2.115772008895874,
"logits/rejected": 2.1254642009735107,
"logps/chosen": -253.1990966796875,
"logps/ref_chosen": -253.2662353515625,
"logps/ref_rejected": -241.4681396484375,
"logps/rejected": -241.55059814453125,
"loss": 5.5393,
"margin_dpo/margin_mean": 0.14957471191883087,
"margin_dpo/margin_std": 0.7799807786941528,
"step": 22
},
{
"KL/chosen_KL_mean": -0.03246879577636719,
"KL/mean": -0.02023410052061081,
"KL/rejected_KL_mean": -0.00799560546875,
"KL/std": 0.6498331427574158,
"epoch": 0.048167539267015703,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02447943389415741,
"fcm_dpo/q_t": 0.5000612139701843,
"grad_norm": 15.345940589904785,
"learning_rate": 2.2916666666666663e-07,
"logits/chosen": 2.0507888793945312,
"logits/rejected": 2.0380196571350098,
"logps/chosen": -275.45404052734375,
"logps/ref_chosen": -275.4216003417969,
"logps/ref_rejected": -264.44024658203125,
"logps/rejected": -264.4482727050781,
"loss": 5.5462,
"margin_dpo/margin_mean": -0.024479106068611145,
"margin_dpo/margin_std": 0.8917433023452759,
"step": 23
},
{
"KL/chosen_KL_mean": 0.10337448120117188,
"KL/mean": 0.08301132917404175,
"KL/rejected_KL_mean": 0.06264495849609375,
"KL/std": 0.6015381813049316,
"epoch": 0.050261780104712044,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.040723443031311035,
"fcm_dpo/q_t": 0.49989819526672363,
"grad_norm": 16.448129653930664,
"learning_rate": 2.3958333333333335e-07,
"logits/chosen": 1.785954236984253,
"logits/rejected": 1.8136179447174072,
"logps/chosen": -293.2742004394531,
"logps/ref_chosen": -293.3775634765625,
"logps/ref_rejected": -301.59014892578125,
"logps/rejected": -301.5274963378906,
"loss": 5.5436,
"margin_dpo/margin_mean": 0.040722861886024475,
"margin_dpo/margin_std": 0.8323075771331787,
"step": 24
},
{
"KL/chosen_KL_mean": 0.0938873291015625,
"KL/mean": 0.043157801032066345,
"KL/rejected_KL_mean": -0.007564544677734375,
"KL/std": 0.5598914623260498,
"epoch": 0.05235602094240838,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10144254565238953,
"fcm_dpo/q_t": 0.4997463524341583,
"grad_norm": 14.537904739379883,
"learning_rate": 2.5e-07,
"logits/chosen": 1.7452150583267212,
"logits/rejected": 1.7821879386901855,
"logps/chosen": -240.8397979736328,
"logps/ref_chosen": -240.93368530273438,
"logps/ref_rejected": -235.3230743408203,
"logps/rejected": -235.33062744140625,
"loss": 5.5412,
"margin_dpo/margin_mean": 0.10144302248954773,
"margin_dpo/margin_std": 0.6517617106437683,
"step": 25
},
{
"KL/chosen_KL_mean": 0.03214263916015625,
"KL/mean": 0.013769354671239853,
"KL/rejected_KL_mean": -0.004604339599609375,
"KL/std": 0.5574186444282532,
"epoch": 0.05445026178010471,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03675615042448044,
"fcm_dpo/q_t": 0.49990811944007874,
"grad_norm": 15.54818344116211,
"learning_rate": 2.604166666666667e-07,
"logits/chosen": 1.6013331413269043,
"logits/rejected": 1.569954752922058,
"logps/chosen": -273.5631408691406,
"logps/ref_chosen": -273.5953369140625,
"logps/ref_rejected": -234.49148559570312,
"logps/rejected": -234.49609375,
"loss": 5.5438,
"margin_dpo/margin_mean": 0.036755383014678955,
"margin_dpo/margin_std": 0.6695701479911804,
"step": 26
},
{
"KL/chosen_KL_mean": 0.028787612915039062,
"KL/mean": 0.015575557947158813,
"KL/rejected_KL_mean": 0.00238037109375,
"KL/std": 0.6904422640800476,
"epoch": 0.05654450261780105,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.026397570967674255,
"fcm_dpo/q_t": 0.49993401765823364,
"grad_norm": 15.199823379516602,
"learning_rate": 2.708333333333333e-07,
"logits/chosen": 2.25942063331604,
"logits/rejected": 2.0365524291992188,
"logps/chosen": -314.235107421875,
"logps/ref_chosen": -314.26385498046875,
"logps/ref_rejected": -242.06735229492188,
"logps/rejected": -242.06497192382812,
"loss": 5.5442,
"margin_dpo/margin_mean": 0.02639685571193695,
"margin_dpo/margin_std": 0.8937135934829712,
"step": 27
},
{
"KL/chosen_KL_mean": 0.0720367431640625,
"KL/mean": 0.05425500497221947,
"KL/rejected_KL_mean": 0.03647041320800781,
"KL/std": 0.5645954012870789,
"epoch": 0.05863874345549738,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03556331247091293,
"fcm_dpo/q_t": 0.4999110698699951,
"grad_norm": 15.47055435180664,
"learning_rate": 2.8125e-07,
"logits/chosen": 2.1194231510162354,
"logits/rejected": 2.1313040256500244,
"logps/chosen": -240.66676330566406,
"logps/ref_chosen": -240.7388153076172,
"logps/ref_rejected": -237.8331298828125,
"logps/rejected": -237.79666137695312,
"loss": 5.5438,
"margin_dpo/margin_mean": 0.035564124584198,
"margin_dpo/margin_std": 0.7161281704902649,
"step": 28
},
{
"KL/chosen_KL_mean": 0.11529350280761719,
"KL/mean": 0.05905672162771225,
"KL/rejected_KL_mean": 0.002834320068359375,
"KL/std": 0.643757164478302,
"epoch": 0.060732984293193716,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11245884001255035,
"fcm_dpo/q_t": 0.49971887469291687,
"grad_norm": 14.707746505737305,
"learning_rate": 2.916666666666667e-07,
"logits/chosen": 2.1459810733795166,
"logits/rejected": 2.06351900100708,
"logps/chosen": -307.714111328125,
"logps/ref_chosen": -307.82940673828125,
"logps/ref_rejected": -281.8109130859375,
"logps/rejected": -281.8080749511719,
"loss": 5.5408,
"margin_dpo/margin_mean": 0.11245732009410858,
"margin_dpo/margin_std": 0.9345981478691101,
"step": 29
},
{
"KL/chosen_KL_mean": 0.024847030639648438,
"KL/mean": 0.031680867075920105,
"KL/rejected_KL_mean": 0.038501739501953125,
"KL/std": 0.6008696556091309,
"epoch": 0.06282722513089005,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.013647228479385376,
"fcm_dpo/q_t": 0.5000340938568115,
"grad_norm": 14.452679634094238,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": 1.5792551040649414,
"logits/rejected": 1.5667495727539062,
"logps/chosen": -247.98892211914062,
"logps/ref_chosen": -248.01376342773438,
"logps/ref_rejected": -259.0014953613281,
"logps/rejected": -258.9629821777344,
"loss": 5.5458,
"margin_dpo/margin_mean": -0.013647839426994324,
"margin_dpo/margin_std": 0.7875149250030518,
"step": 30
},
{
"KL/chosen_KL_mean": 0.11716651916503906,
"KL/mean": 0.03221321105957031,
"KL/rejected_KL_mean": -0.05274772644042969,
"KL/std": 0.6022624969482422,
"epoch": 0.06492146596858639,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.16990646719932556,
"fcm_dpo/q_t": 0.4995752274990082,
"grad_norm": 13.671382904052734,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": 1.8226486444473267,
"logits/rejected": 1.7482107877731323,
"logps/chosen": -239.94564819335938,
"logps/ref_chosen": -240.0628204345703,
"logps/ref_rejected": -201.91021728515625,
"logps/rejected": -201.96295166015625,
"loss": 5.5384,
"margin_dpo/margin_mean": 0.16990603506565094,
"margin_dpo/margin_std": 0.7710446119308472,
"step": 31
},
{
"KL/chosen_KL_mean": 0.1855602264404297,
"KL/mean": 0.13113217055797577,
"KL/rejected_KL_mean": 0.07670211791992188,
"KL/std": 0.6365066170692444,
"epoch": 0.06701570680628273,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10885399580001831,
"fcm_dpo/q_t": 0.49972790479660034,
"grad_norm": 15.790374755859375,
"learning_rate": 3.2291666666666666e-07,
"logits/chosen": 1.9913053512573242,
"logits/rejected": 1.9235936403274536,
"logps/chosen": -293.9466857910156,
"logps/ref_chosen": -294.1322326660156,
"logps/ref_rejected": -265.8150634765625,
"logps/rejected": -265.7383728027344,
"loss": 5.5409,
"margin_dpo/margin_mean": 0.10885459184646606,
"margin_dpo/margin_std": 0.7468684911727905,
"step": 32
},
{
"KL/chosen_KL_mean": 0.29471588134765625,
"KL/mean": 0.17659299075603485,
"KL/rejected_KL_mean": 0.058460235595703125,
"KL/std": 0.698663055896759,
"epoch": 0.06910994764397906,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23624789714813232,
"fcm_dpo/q_t": 0.49940937757492065,
"grad_norm": 14.96553897857666,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": 1.9445685148239136,
"logits/rejected": 1.90474534034729,
"logps/chosen": -275.36041259765625,
"logps/ref_chosen": -275.6551208496094,
"logps/ref_rejected": -253.5972137451172,
"logps/rejected": -253.53875732421875,
"loss": 5.5358,
"margin_dpo/margin_mean": 0.23624709248542786,
"margin_dpo/margin_std": 1.0003958940505981,
"step": 33
},
{
"KL/chosen_KL_mean": 0.15773773193359375,
"KL/mean": 0.08135133981704712,
"KL/rejected_KL_mean": 0.0049591064453125,
"KL/std": 0.7322280406951904,
"epoch": 0.0712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15276679396629333,
"fcm_dpo/q_t": 0.4996180534362793,
"grad_norm": 17.711772918701172,
"learning_rate": 3.4375e-07,
"logits/chosen": 1.9582128524780273,
"logits/rejected": 1.8374905586242676,
"logps/chosen": -332.8123474121094,
"logps/ref_chosen": -332.9701232910156,
"logps/ref_rejected": -278.2213439941406,
"logps/rejected": -278.2164306640625,
"loss": 5.5392,
"margin_dpo/margin_mean": 0.1527663916349411,
"margin_dpo/margin_std": 0.99702388048172,
"step": 34
},
{
"KL/chosen_KL_mean": 0.2559928894042969,
"KL/mean": 0.22444842755794525,
"KL/rejected_KL_mean": 0.1929035186767578,
"KL/std": 0.6824629902839661,
"epoch": 0.07329842931937172,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06308795511722565,
"fcm_dpo/q_t": 0.49984225630760193,
"grad_norm": 13.625845909118652,
"learning_rate": 3.541666666666667e-07,
"logits/chosen": 1.95424222946167,
"logits/rejected": 2.0227391719818115,
"logps/chosen": -258.17291259765625,
"logps/ref_chosen": -258.42889404296875,
"logps/ref_rejected": -259.5119934082031,
"logps/rejected": -259.3190612792969,
"loss": 5.5427,
"margin_dpo/margin_mean": 0.06308749318122864,
"margin_dpo/margin_std": 0.8707388043403625,
"step": 35
},
{
"KL/chosen_KL_mean": 0.3726043701171875,
"KL/mean": 0.23814818263053894,
"KL/rejected_KL_mean": 0.10369491577148438,
"KL/std": 0.7664194107055664,
"epoch": 0.07539267015706806,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2689109444618225,
"fcm_dpo/q_t": 0.49932774901390076,
"grad_norm": 17.59621810913086,
"learning_rate": 3.645833333333333e-07,
"logits/chosen": 1.8321075439453125,
"logits/rejected": 1.7767295837402344,
"logps/chosen": -290.24676513671875,
"logps/ref_chosen": -290.6193542480469,
"logps/ref_rejected": -280.2503967285156,
"logps/rejected": -280.1466979980469,
"loss": 5.5346,
"margin_dpo/margin_mean": 0.26891028881073,
"margin_dpo/margin_std": 1.0319002866744995,
"step": 36
},
{
"KL/chosen_KL_mean": 0.3753223419189453,
"KL/mean": 0.2544947862625122,
"KL/rejected_KL_mean": 0.13367462158203125,
"KL/std": 0.7211363911628723,
"epoch": 0.0774869109947644,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24163669347763062,
"fcm_dpo/q_t": 0.4993959367275238,
"grad_norm": 15.294231414794922,
"learning_rate": 3.75e-07,
"logits/chosen": 1.7585910558700562,
"logits/rejected": 1.740007996559143,
"logps/chosen": -303.197509765625,
"logps/ref_chosen": -303.57281494140625,
"logps/ref_rejected": -294.56878662109375,
"logps/rejected": -294.4350891113281,
"loss": 5.5356,
"margin_dpo/margin_mean": 0.24163639545440674,
"margin_dpo/margin_std": 0.9587388038635254,
"step": 37
},
{
"KL/chosen_KL_mean": 0.30801963806152344,
"KL/mean": 0.2135545313358307,
"KL/rejected_KL_mean": 0.11908149719238281,
"KL/std": 0.7989519238471985,
"epoch": 0.07958115183246073,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1889396458864212,
"fcm_dpo/q_t": 0.4995276629924774,
"grad_norm": 15.346175193786621,
"learning_rate": 3.8541666666666665e-07,
"logits/chosen": 1.7355865240097046,
"logits/rejected": 1.786137580871582,
"logps/chosen": -272.2454833984375,
"logps/ref_chosen": -272.5534973144531,
"logps/ref_rejected": -266.9136047363281,
"logps/rejected": -266.7945251464844,
"loss": 5.5378,
"margin_dpo/margin_mean": 0.18893998861312866,
"margin_dpo/margin_std": 1.0406105518341064,
"step": 38
},
{
"KL/chosen_KL_mean": 0.512542724609375,
"KL/mean": 0.30288946628570557,
"KL/rejected_KL_mean": 0.09322929382324219,
"KL/std": 0.7305359244346619,
"epoch": 0.08167539267015707,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4193153977394104,
"fcm_dpo/q_t": 0.4989517331123352,
"grad_norm": 15.872249603271484,
"learning_rate": 3.958333333333333e-07,
"logits/chosen": 1.7924615144729614,
"logits/rejected": 1.7269222736358643,
"logps/chosen": -264.87396240234375,
"logps/ref_chosen": -265.38653564453125,
"logps/ref_rejected": -244.34774780273438,
"logps/rejected": -244.25454711914062,
"loss": 5.5285,
"margin_dpo/margin_mean": 0.41931572556495667,
"margin_dpo/margin_std": 0.9724928140640259,
"step": 39
},
{
"KL/chosen_KL_mean": 0.5598220825195312,
"KL/mean": 0.3143317401409149,
"KL/rejected_KL_mean": 0.06884765625,
"KL/std": 0.7974763512611389,
"epoch": 0.08376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.49096524715423584,
"fcm_dpo/q_t": 0.49877268075942993,
"grad_norm": 14.932929992675781,
"learning_rate": 4.0625e-07,
"logits/chosen": 1.794917345046997,
"logits/rejected": 1.783827543258667,
"logps/chosen": -274.333984375,
"logps/ref_chosen": -274.893798828125,
"logps/ref_rejected": -244.3120880126953,
"logps/rejected": -244.2432403564453,
"loss": 5.5257,
"margin_dpo/margin_mean": 0.4909658432006836,
"margin_dpo/margin_std": 1.0364124774932861,
"step": 40
},
{
"KL/chosen_KL_mean": 0.4529266357421875,
"KL/mean": 0.3263741731643677,
"KL/rejected_KL_mean": 0.19981765747070312,
"KL/std": 0.7767796516418457,
"epoch": 0.08586387434554973,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.25309914350509644,
"fcm_dpo/q_t": 0.49936726689338684,
"grad_norm": 15.412158966064453,
"learning_rate": 4.1666666666666667e-07,
"logits/chosen": 2.0711989402770996,
"logits/rejected": 2.011732816696167,
"logps/chosen": -284.3462219238281,
"logps/ref_chosen": -284.7991027832031,
"logps/ref_rejected": -248.8882598876953,
"logps/rejected": -248.68844604492188,
"loss": 5.5352,
"margin_dpo/margin_mean": 0.25309881567955017,
"margin_dpo/margin_std": 1.022508978843689,
"step": 41
},
{
"KL/chosen_KL_mean": 0.43947601318359375,
"KL/mean": 0.2273520678281784,
"KL/rejected_KL_mean": 0.015222549438476562,
"KL/std": 0.812735378742218,
"epoch": 0.08795811518324607,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4242541790008545,
"fcm_dpo/q_t": 0.4989393949508667,
"grad_norm": 14.70630168914795,
"learning_rate": 4.270833333333333e-07,
"logits/chosen": 1.9239072799682617,
"logits/rejected": 1.9785937070846558,
"logps/chosen": -245.4001922607422,
"logps/ref_chosen": -245.83966064453125,
"logps/ref_rejected": -259.2870788574219,
"logps/rejected": -259.2718505859375,
"loss": 5.5284,
"margin_dpo/margin_mean": 0.42425358295440674,
"margin_dpo/margin_std": 1.125490427017212,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5615425109863281,
"KL/mean": 0.32219600677490234,
"KL/rejected_KL_mean": 0.08284378051757812,
"KL/std": 0.8720331192016602,
"epoch": 0.09005235602094241,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4786991775035858,
"fcm_dpo/q_t": 0.4988033175468445,
"grad_norm": 15.70401668548584,
"learning_rate": 4.375e-07,
"logits/chosen": 1.87813138961792,
"logits/rejected": 1.861478328704834,
"logps/chosen": -307.83599853515625,
"logps/ref_chosen": -308.39752197265625,
"logps/ref_rejected": -270.3797912597656,
"logps/rejected": -270.2969665527344,
"loss": 5.5262,
"margin_dpo/margin_mean": 0.4787002503871918,
"margin_dpo/margin_std": 1.0944887399673462,
"step": 43
},
{
"KL/chosen_KL_mean": 0.6350421905517578,
"KL/mean": 0.4450918138027191,
"KL/rejected_KL_mean": 0.2551383972167969,
"KL/std": 0.9023072719573975,
"epoch": 0.09214659685863874,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3798975348472595,
"fcm_dpo/q_t": 0.4990502893924713,
"grad_norm": 15.863829612731934,
"learning_rate": 4.479166666666667e-07,
"logits/chosen": 1.9397392272949219,
"logits/rejected": 1.9450037479400635,
"logps/chosen": -274.7266845703125,
"logps/ref_chosen": -275.36175537109375,
"logps/ref_rejected": -279.7540588378906,
"logps/rejected": -279.49896240234375,
"loss": 5.5302,
"margin_dpo/margin_mean": 0.3798980116844177,
"margin_dpo/margin_std": 1.2212566137313843,
"step": 44
},
{
"KL/chosen_KL_mean": 0.7438888549804688,
"KL/mean": 0.46234309673309326,
"KL/rejected_KL_mean": 0.18078994750976562,
"KL/std": 1.0001219511032104,
"epoch": 0.09424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5631004571914673,
"fcm_dpo/q_t": 0.4985923171043396,
"grad_norm": 16.84543228149414,
"learning_rate": 4.5833333333333327e-07,
"logits/chosen": 1.862227439880371,
"logits/rejected": 1.8337498903274536,
"logps/chosen": -314.3763427734375,
"logps/ref_chosen": -315.1202087402344,
"logps/ref_rejected": -290.34210205078125,
"logps/rejected": -290.1612854003906,
"loss": 5.5229,
"margin_dpo/margin_mean": 0.5630996227264404,
"margin_dpo/margin_std": 1.3207401037216187,
"step": 45
},
{
"KL/chosen_KL_mean": 0.7039508819580078,
"KL/mean": 0.383242130279541,
"KL/rejected_KL_mean": 0.06253814697265625,
"KL/std": 1.0930442810058594,
"epoch": 0.09633507853403141,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.641418993473053,
"fcm_dpo/q_t": 0.4983965754508972,
"grad_norm": 16.388086318969727,
"learning_rate": 4.6874999999999996e-07,
"logits/chosen": 1.6258591413497925,
"logits/rejected": 1.6258472204208374,
"logps/chosen": -292.23663330078125,
"logps/ref_chosen": -292.94061279296875,
"logps/ref_rejected": -264.0979309082031,
"logps/rejected": -264.035400390625,
"loss": 5.5198,
"margin_dpo/margin_mean": 0.6414193511009216,
"margin_dpo/margin_std": 1.4341840744018555,
"step": 46
},
{
"KL/chosen_KL_mean": 0.6179580688476562,
"KL/mean": 0.4481770098209381,
"KL/rejected_KL_mean": 0.2783985137939453,
"KL/std": 0.9199997186660767,
"epoch": 0.09842931937172775,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3395555317401886,
"fcm_dpo/q_t": 0.4991511404514313,
"grad_norm": 13.547813415527344,
"learning_rate": 4.791666666666667e-07,
"logits/chosen": 1.856745719909668,
"logits/rejected": 1.963684320449829,
"logps/chosen": -254.23472595214844,
"logps/ref_chosen": -254.85269165039062,
"logps/ref_rejected": -255.88380432128906,
"logps/rejected": -255.60537719726562,
"loss": 5.5318,
"margin_dpo/margin_mean": 0.3395553231239319,
"margin_dpo/margin_std": 1.2476435899734497,
"step": 47
},
{
"KL/chosen_KL_mean": 0.7965354919433594,
"KL/mean": 0.48358938097953796,
"KL/rejected_KL_mean": 0.1706409454345703,
"KL/std": 1.1975098848342896,
"epoch": 0.10052356020942409,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6258994340896606,
"fcm_dpo/q_t": 0.49843546748161316,
"grad_norm": 14.78850269317627,
"learning_rate": 4.895833333333333e-07,
"logits/chosen": 1.9663748741149902,
"logits/rejected": 1.955999493598938,
"logps/chosen": -261.711181640625,
"logps/ref_chosen": -262.5077209472656,
"logps/ref_rejected": -239.25320434570312,
"logps/rejected": -239.08258056640625,
"loss": 5.5204,
"margin_dpo/margin_mean": 0.625899076461792,
"margin_dpo/margin_std": 1.5170960426330566,
"step": 48
},
{
"KL/chosen_KL_mean": 0.8709144592285156,
"KL/mean": 0.6166729927062988,
"KL/rejected_KL_mean": 0.3624401092529297,
"KL/std": 1.174827218055725,
"epoch": 0.10261780104712041,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5084781646728516,
"fcm_dpo/q_t": 0.49872881174087524,
"grad_norm": 15.39860725402832,
"learning_rate": 5e-07,
"logits/chosen": 1.9863102436065674,
"logits/rejected": 2.099377393722534,
"logps/chosen": -248.65695190429688,
"logps/ref_chosen": -249.5278778076172,
"logps/ref_rejected": -254.6465301513672,
"logps/rejected": -254.2841033935547,
"loss": 5.5251,
"margin_dpo/margin_mean": 0.5084781646728516,
"margin_dpo/margin_std": 1.5813007354736328,
"step": 49
},
{
"KL/chosen_KL_mean": 0.9026336669921875,
"KL/mean": 0.7201584577560425,
"KL/rejected_KL_mean": 0.5376853942871094,
"KL/std": 1.1079752445220947,
"epoch": 0.10471204188481675,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3649483919143677,
"fcm_dpo/q_t": 0.49908769130706787,
"grad_norm": 14.629812240600586,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": 1.9577043056488037,
"logits/rejected": 1.9702847003936768,
"logps/chosen": -291.59906005859375,
"logps/ref_chosen": -292.501708984375,
"logps/ref_rejected": -294.6496887207031,
"logps/rejected": -294.11199951171875,
"loss": 5.5308,
"margin_dpo/margin_mean": 0.36494964361190796,
"margin_dpo/margin_std": 1.3937370777130127,
"step": 50
},
{
"KL/chosen_KL_mean": 0.9896411895751953,
"KL/mean": 0.6506175994873047,
"KL/rejected_KL_mean": 0.3115806579589844,
"KL/std": 1.320156216621399,
"epoch": 0.1068062827225131,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6780645251274109,
"fcm_dpo/q_t": 0.49830490350723267,
"grad_norm": 14.741464614868164,
"learning_rate": 4.999731868769026e-07,
"logits/chosen": 1.3491064310073853,
"logits/rejected": 1.4821522235870361,
"logps/chosen": -258.87225341796875,
"logps/ref_chosen": -259.8619079589844,
"logps/ref_rejected": -293.1994934082031,
"logps/rejected": -292.887939453125,
"loss": 5.5185,
"margin_dpo/margin_mean": 0.6780642867088318,
"margin_dpo/margin_std": 1.7821853160858154,
"step": 51
},
{
"KL/chosen_KL_mean": 1.2401237487792969,
"KL/mean": 0.7772451043128967,
"KL/rejected_KL_mean": 0.3143730163574219,
"KL/std": 1.382419466972351,
"epoch": 0.10890052356020942,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9257485270500183,
"fcm_dpo/q_t": 0.497685968875885,
"grad_norm": 16.996623992919922,
"learning_rate": 4.99939671821067e-07,
"logits/chosen": 1.9146828651428223,
"logits/rejected": 1.8284406661987305,
"logps/chosen": -297.1852111816406,
"logps/ref_chosen": -298.42529296875,
"logps/ref_rejected": -278.5128479003906,
"logps/rejected": -278.1984558105469,
"loss": 5.5086,
"margin_dpo/margin_mean": 0.9257479906082153,
"margin_dpo/margin_std": 1.7723982334136963,
"step": 52
},
{
"KL/chosen_KL_mean": 1.19000244140625,
"KL/mean": 0.8038426637649536,
"KL/rejected_KL_mean": 0.4176826477050781,
"KL/std": 1.428905725479126,
"epoch": 0.11099476439790576,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7723150849342346,
"fcm_dpo/q_t": 0.4980694651603699,
"grad_norm": 15.993934631347656,
"learning_rate": 4.998927532591591e-07,
"logits/chosen": 1.8577872514724731,
"logits/rejected": 1.927185297012329,
"logps/chosen": -274.1184997558594,
"logps/ref_chosen": -275.3085021972656,
"logps/ref_rejected": -267.97381591796875,
"logps/rejected": -267.55615234375,
"loss": 5.5148,
"margin_dpo/margin_mean": 0.7723162770271301,
"margin_dpo/margin_std": 1.8218241930007935,
"step": 53
},
{
"KL/chosen_KL_mean": 1.3046531677246094,
"KL/mean": 0.9638843536376953,
"KL/rejected_KL_mean": 0.62310791015625,
"KL/std": 1.5169343948364258,
"epoch": 0.1130890052356021,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6815524697303772,
"fcm_dpo/q_t": 0.4982964098453522,
"grad_norm": 14.538448333740234,
"learning_rate": 4.998324337072792e-07,
"logits/chosen": 1.6982616186141968,
"logits/rejected": 1.7266590595245361,
"logps/chosen": -296.8426818847656,
"logps/ref_chosen": -298.1473388671875,
"logps/ref_rejected": -269.7539367675781,
"logps/rejected": -269.1308288574219,
"loss": 5.5185,
"margin_dpo/margin_mean": 0.6815527081489563,
"margin_dpo/margin_std": 2.04019832611084,
"step": 54
},
{
"KL/chosen_KL_mean": 1.1596031188964844,
"KL/mean": 0.8441203832626343,
"KL/rejected_KL_mean": 0.5286293029785156,
"KL/std": 1.3791707754135132,
"epoch": 0.11518324607329843,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6309798955917358,
"fcm_dpo/q_t": 0.4984227120876312,
"grad_norm": 12.82992172241211,
"learning_rate": 4.997587164001815e-07,
"logits/chosen": 1.7965575456619263,
"logits/rejected": 1.8571105003356934,
"logps/chosen": -215.5903778076172,
"logps/ref_chosen": -216.74998474121094,
"logps/ref_rejected": -202.7234649658203,
"logps/rejected": -202.19483947753906,
"loss": 5.5203,
"margin_dpo/margin_mean": 0.6309804320335388,
"margin_dpo/margin_std": 1.8771089315414429,
"step": 55
},
{
"KL/chosen_KL_mean": 1.3335132598876953,
"KL/mean": 0.8361304998397827,
"KL/rejected_KL_mean": 0.3387489318847656,
"KL/std": 1.7761727571487427,
"epoch": 0.11727748691099477,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9947671294212341,
"fcm_dpo/q_t": 0.4975135326385498,
"grad_norm": 14.402202606201172,
"learning_rate": 4.996716052911017e-07,
"logits/chosen": 1.4954833984375,
"logits/rejected": 1.577931523323059,
"logps/chosen": -242.35787963867188,
"logps/ref_chosen": -243.69140625,
"logps/ref_rejected": -234.970947265625,
"logps/rejected": -234.63218688964844,
"loss": 5.5061,
"margin_dpo/margin_mean": 0.9947657585144043,
"margin_dpo/margin_std": 2.2741339206695557,
"step": 56
},
{
"KL/chosen_KL_mean": 1.683126449584961,
"KL/mean": 0.9615461230278015,
"KL/rejected_KL_mean": 0.23997116088867188,
"KL/std": 1.7344322204589844,
"epoch": 0.1193717277486911,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4431445598602295,
"fcm_dpo/q_t": 0.4963931739330292,
"grad_norm": 17.59752655029297,
"learning_rate": 4.99571105051544e-07,
"logits/chosen": 2.170443296432495,
"logits/rejected": 2.078343152999878,
"logps/chosen": -309.0632019042969,
"logps/ref_chosen": -310.746337890625,
"logps/ref_rejected": -270.6407165527344,
"logps/rejected": -270.4007263183594,
"loss": 5.4882,
"margin_dpo/margin_mean": 1.44314444065094,
"margin_dpo/margin_std": 2.150846004486084,
"step": 57
},
{
"KL/chosen_KL_mean": 1.324371337890625,
"KL/mean": 0.7948843240737915,
"KL/rejected_KL_mean": 0.26540565490722656,
"KL/std": 1.598447322845459,
"epoch": 0.12146596858638743,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.058971643447876,
"fcm_dpo/q_t": 0.497352659702301,
"grad_norm": 14.892204284667969,
"learning_rate": 4.994572210710314e-07,
"logits/chosen": 1.8611407279968262,
"logits/rejected": 1.8537086248397827,
"logps/chosen": -257.4886169433594,
"logps/ref_chosen": -258.8130187988281,
"logps/ref_rejected": -268.2101745605469,
"logps/rejected": -267.9447937011719,
"loss": 5.5034,
"margin_dpo/margin_mean": 1.0589708089828491,
"margin_dpo/margin_std": 1.9082849025726318,
"step": 58
},
{
"KL/chosen_KL_mean": 1.4427604675292969,
"KL/mean": 1.0179864168167114,
"KL/rejected_KL_mean": 0.5932121276855469,
"KL/std": 1.8750296831130981,
"epoch": 0.12356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8495504260063171,
"fcm_dpo/q_t": 0.49787670373916626,
"grad_norm": 15.2084321975708,
"learning_rate": 4.993299594568162e-07,
"logits/chosen": 1.5610744953155518,
"logits/rejected": 1.4733085632324219,
"logps/chosen": -275.7986755371094,
"logps/ref_chosen": -277.2414245605469,
"logps/ref_rejected": -259.13519287109375,
"logps/rejected": -258.5419921875,
"loss": 5.512,
"margin_dpo/margin_mean": 0.8495496511459351,
"margin_dpo/margin_std": 2.5182838439941406,
"step": 59
},
{
"KL/chosen_KL_mean": 1.5604877471923828,
"KL/mean": 0.9622378945350647,
"KL/rejected_KL_mean": 0.3640003204345703,
"KL/std": 1.7057533264160156,
"epoch": 0.1256544502617801,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.196488857269287,
"fcm_dpo/q_t": 0.4970093071460724,
"grad_norm": 14.753252029418945,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": 1.9633570909500122,
"logits/rejected": 2.014233112335205,
"logps/chosen": -249.78536987304688,
"logps/ref_chosen": -251.34585571289062,
"logps/ref_rejected": -251.4120635986328,
"logps/rejected": -251.04806518554688,
"loss": 5.4981,
"margin_dpo/margin_mean": 1.1964893341064453,
"margin_dpo/margin_std": 2.2795042991638184,
"step": 60
},
{
"KL/chosen_KL_mean": 1.6346378326416016,
"KL/mean": 1.08052659034729,
"KL/rejected_KL_mean": 0.5264148712158203,
"KL/std": 1.824796199798584,
"epoch": 0.12774869109947645,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.108229637145996,
"fcm_dpo/q_t": 0.49723008275032043,
"grad_norm": 15.012340545654297,
"learning_rate": 4.990353313429303e-07,
"logits/chosen": 1.9480056762695312,
"logits/rejected": 1.9169143438339233,
"logps/chosen": -267.34796142578125,
"logps/ref_chosen": -268.98260498046875,
"logps/ref_rejected": -249.34474182128906,
"logps/rejected": -248.81832885742188,
"loss": 5.5017,
"margin_dpo/margin_mean": 1.1082285642623901,
"margin_dpo/margin_std": 2.4616637229919434,
"step": 61
},
{
"KL/chosen_KL_mean": 2.027973175048828,
"KL/mean": 1.4072773456573486,
"KL/rejected_KL_mean": 0.7865867614746094,
"KL/std": 2.0561020374298096,
"epoch": 0.12984293193717278,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.241378903388977,
"fcm_dpo/q_t": 0.4968973994255066,
"grad_norm": 14.663749694824219,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 1.9495458602905273,
"logits/rejected": 1.9271520376205444,
"logps/chosen": -262.1755676269531,
"logps/ref_chosen": -264.2034912109375,
"logps/ref_rejected": -260.2911376953125,
"logps/rejected": -259.50457763671875,
"loss": 5.4964,
"margin_dpo/margin_mean": 1.2413785457611084,
"margin_dpo/margin_std": 2.655601739883423,
"step": 62
},
{
"KL/chosen_KL_mean": 1.924692153930664,
"KL/mean": 1.2763150930404663,
"KL/rejected_KL_mean": 0.6279335021972656,
"KL/std": 2.1142983436584473,
"epoch": 0.1319371727748691,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2967690229415894,
"fcm_dpo/q_t": 0.4967590272426605,
"grad_norm": 14.749189376831055,
"learning_rate": 4.986872839090852e-07,
"logits/chosen": 1.6582473516464233,
"logits/rejected": 1.6768556833267212,
"logps/chosen": -269.5436096191406,
"logps/ref_chosen": -271.4682922363281,
"logps/ref_rejected": -273.0804748535156,
"logps/rejected": -272.4525451660156,
"loss": 5.4944,
"margin_dpo/margin_mean": 1.2967699766159058,
"margin_dpo/margin_std": 2.7775895595550537,
"step": 63
},
{
"KL/chosen_KL_mean": 2.337747573852539,
"KL/mean": 1.6104967594146729,
"KL/rejected_KL_mean": 0.8832416534423828,
"KL/std": 2.3649282455444336,
"epoch": 0.13403141361256546,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4545021057128906,
"fcm_dpo/q_t": 0.4963648319244385,
"grad_norm": 15.6937255859375,
"learning_rate": 4.9849325083059e-07,
"logits/chosen": 1.9031604528427124,
"logits/rejected": 1.9173095226287842,
"logps/chosen": -271.2656555175781,
"logps/ref_chosen": -273.6033935546875,
"logps/ref_rejected": -259.54107666015625,
"logps/rejected": -258.6578369140625,
"loss": 5.4882,
"margin_dpo/margin_mean": 1.4545015096664429,
"margin_dpo/margin_std": 3.0799834728240967,
"step": 64
},
{
"KL/chosen_KL_mean": 2.314014434814453,
"KL/mean": 1.737621784210205,
"KL/rejected_KL_mean": 1.1612358093261719,
"KL/std": 2.1046664714813232,
"epoch": 0.13612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.152785301208496,
"fcm_dpo/q_t": 0.4971183240413666,
"grad_norm": 14.508123397827148,
"learning_rate": 4.982858918131906e-07,
"logits/chosen": 1.8851243257522583,
"logits/rejected": 1.983131766319275,
"logps/chosen": -256.2036437988281,
"logps/ref_chosen": -258.5176696777344,
"logps/ref_rejected": -264.9944763183594,
"logps/rejected": -263.833251953125,
"loss": 5.4999,
"margin_dpo/margin_mean": 1.152784824371338,
"margin_dpo/margin_std": 2.61080265045166,
"step": 65
},
{
"KL/chosen_KL_mean": 2.190641403198242,
"KL/mean": 1.5029757022857666,
"KL/rejected_KL_mean": 0.8153171539306641,
"KL/std": 2.748277425765991,
"epoch": 0.1382198952879581,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3753265142440796,
"fcm_dpo/q_t": 0.49656298756599426,
"grad_norm": 14.806829452514648,
"learning_rate": 4.980652179769217e-07,
"logits/chosen": 1.7718381881713867,
"logits/rejected": 1.8650543689727783,
"logps/chosen": -265.31878662109375,
"logps/ref_chosen": -267.5094299316406,
"logps/ref_rejected": -277.03228759765625,
"logps/rejected": -276.2169494628906,
"loss": 5.4917,
"margin_dpo/margin_mean": 1.3753262758255005,
"margin_dpo/margin_std": 3.608642816543579,
"step": 66
},
{
"KL/chosen_KL_mean": 2.578306198120117,
"KL/mean": 1.970463752746582,
"KL/rejected_KL_mean": 1.3626251220703125,
"KL/std": 2.6167731285095215,
"epoch": 0.14031413612565444,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.215672492980957,
"fcm_dpo/q_t": 0.4969613552093506,
"grad_norm": 14.19619083404541,
"learning_rate": 4.978312411558517e-07,
"logits/chosen": 1.9167455434799194,
"logits/rejected": 1.9535115957260132,
"logps/chosen": -254.88662719726562,
"logps/ref_chosen": -257.4649353027344,
"logps/ref_rejected": -247.29161071777344,
"logps/rejected": -245.9290008544922,
"loss": 5.4978,
"margin_dpo/margin_mean": 1.2156729698181152,
"margin_dpo/margin_std": 3.270585775375366,
"step": 67
},
{
"KL/chosen_KL_mean": 2.5522289276123047,
"KL/mean": 1.6995009183883667,
"KL/rejected_KL_mean": 0.8467826843261719,
"KL/std": 2.9510626792907715,
"epoch": 0.1424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7054537534713745,
"fcm_dpo/q_t": 0.4957389235496521,
"grad_norm": 14.784210205078125,
"learning_rate": 4.975839738974473e-07,
"logits/chosen": 1.7252765893936157,
"logits/rejected": 1.674182653427124,
"logps/chosen": -287.93194580078125,
"logps/ref_chosen": -290.4841613769531,
"logps/ref_rejected": -260.8398132324219,
"logps/rejected": -259.9930419921875,
"loss": 5.4789,
"margin_dpo/margin_mean": 1.7054539918899536,
"margin_dpo/margin_std": 3.846391201019287,
"step": 68
},
{
"KL/chosen_KL_mean": 2.9039573669433594,
"KL/mean": 1.8334503173828125,
"KL/rejected_KL_mean": 0.7629432678222656,
"KL/std": 2.94232439994812,
"epoch": 0.14450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.141019821166992,
"fcm_dpo/q_t": 0.49464961886405945,
"grad_norm": 14.979423522949219,
"learning_rate": 4.97323429461901e-07,
"logits/chosen": 2.0321993827819824,
"logits/rejected": 2.0943429470062256,
"logps/chosen": -254.73361206054688,
"logps/ref_chosen": -257.6375732421875,
"logps/ref_rejected": -235.6666717529297,
"logps/rejected": -234.9037322998047,
"loss": 5.4615,
"margin_dpo/margin_mean": 2.141019344329834,
"margin_dpo/margin_std": 3.7320189476013184,
"step": 69
},
{
"KL/chosen_KL_mean": 3.424154281616211,
"KL/mean": 2.3216376304626465,
"KL/rejected_KL_mean": 1.2191219329833984,
"KL/std": 3.025865077972412,
"epoch": 0.14659685863874344,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.2050423622131348,
"fcm_dpo/q_t": 0.49449020624160767,
"grad_norm": 15.603200912475586,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": 1.8569802045822144,
"logits/rejected": 1.8316940069198608,
"logps/chosen": -284.0410461425781,
"logps/ref_chosen": -287.4651794433594,
"logps/ref_rejected": -253.4770965576172,
"logps/rejected": -252.25799560546875,
"loss": 5.4589,
"margin_dpo/margin_mean": 2.2050423622131348,
"margin_dpo/margin_std": 3.663863182067871,
"step": 70
},
{
"KL/chosen_KL_mean": 3.570659637451172,
"KL/mean": 2.601686954498291,
"KL/rejected_KL_mean": 1.6327152252197266,
"KL/std": 3.250652313232422,
"epoch": 0.1486910994764398,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9379527568817139,
"fcm_dpo/q_t": 0.4951571822166443,
"grad_norm": 14.96021556854248,
"learning_rate": 4.967625656594781e-07,
"logits/chosen": 1.9369959831237793,
"logits/rejected": 1.9424176216125488,
"logps/chosen": -277.36474609375,
"logps/ref_chosen": -280.9353942871094,
"logps/ref_rejected": -275.998779296875,
"logps/rejected": -274.3660888671875,
"loss": 5.47,
"margin_dpo/margin_mean": 1.9379535913467407,
"margin_dpo/margin_std": 4.208340644836426,
"step": 71
},
{
"KL/chosen_KL_mean": 2.9796676635742188,
"KL/mean": 1.987913966178894,
"KL/rejected_KL_mean": 0.9961509704589844,
"KL/std": 3.054412603378296,
"epoch": 0.15078534031413612,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.983521580696106,
"fcm_dpo/q_t": 0.4950438439846039,
"grad_norm": 14.67113971710205,
"learning_rate": 4.964622763700252e-07,
"logits/chosen": 1.721984624862671,
"logits/rejected": 1.8629076480865479,
"logps/chosen": -227.16586303710938,
"logps/ref_chosen": -230.14553833007812,
"logps/ref_rejected": -253.37661743164062,
"logps/rejected": -252.3804473876953,
"loss": 5.468,
"margin_dpo/margin_mean": 1.9835209846496582,
"margin_dpo/margin_std": 3.9971396923065186,
"step": 72
},
{
"KL/chosen_KL_mean": 3.7841663360595703,
"KL/mean": 2.8107807636260986,
"KL/rejected_KL_mean": 1.8373908996582031,
"KL/std": 3.2371246814727783,
"epoch": 0.15287958115183245,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9467737674713135,
"fcm_dpo/q_t": 0.49513500928878784,
"grad_norm": 14.997169494628906,
"learning_rate": 4.961487700566646e-07,
"logits/chosen": 1.6057504415512085,
"logits/rejected": 1.6186208724975586,
"logps/chosen": -261.0821228027344,
"logps/ref_chosen": -264.86627197265625,
"logps/ref_rejected": -245.21304321289062,
"logps/rejected": -243.37564086914062,
"loss": 5.4698,
"margin_dpo/margin_mean": 1.9467735290527344,
"margin_dpo/margin_std": 4.460379600524902,
"step": 73
},
{
"KL/chosen_KL_mean": 3.76385498046875,
"KL/mean": 2.8343586921691895,
"KL/rejected_KL_mean": 1.9048748016357422,
"KL/std": 3.5186123847961426,
"epoch": 0.1549738219895288,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.858985185623169,
"fcm_dpo/q_t": 0.4953550696372986,
"grad_norm": 15.689414024353027,
"learning_rate": 4.958220635317885e-07,
"logits/chosen": 1.8440401554107666,
"logits/rejected": 1.8993223905563354,
"logps/chosen": -330.7896728515625,
"logps/ref_chosen": -334.55352783203125,
"logps/ref_rejected": -325.8403015136719,
"logps/rejected": -323.9353942871094,
"loss": 5.4735,
"margin_dpo/margin_mean": 1.8589857816696167,
"margin_dpo/margin_std": 4.583897590637207,
"step": 74
},
{
"KL/chosen_KL_mean": 4.565900802612305,
"KL/mean": 3.20298433303833,
"KL/rejected_KL_mean": 1.8400688171386719,
"KL/std": 3.6198770999908447,
"epoch": 0.15706806282722513,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.7258455753326416,
"fcm_dpo/q_t": 0.4931892156600952,
"grad_norm": 15.345573425292969,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 1.6761469841003418,
"logits/rejected": 1.7539714574813843,
"logps/chosen": -255.85597229003906,
"logps/ref_chosen": -260.421875,
"logps/ref_rejected": -273.0882263183594,
"logps/rejected": -271.2481689453125,
"loss": 5.4389,
"margin_dpo/margin_mean": 2.7258448600769043,
"margin_dpo/margin_std": 4.2095537185668945,
"step": 75
},
{
"KL/chosen_KL_mean": 4.54121208190918,
"KL/mean": 3.065985918045044,
"KL/rejected_KL_mean": 1.590768814086914,
"KL/std": 3.8259737491607666,
"epoch": 0.15916230366492146,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.950458526611328,
"fcm_dpo/q_t": 0.4926307797431946,
"grad_norm": 16.780370712280273,
"learning_rate": 4.951291206355559e-07,
"logits/chosen": 1.8369426727294922,
"logits/rejected": 1.8256155252456665,
"logps/chosen": -267.1942443847656,
"logps/ref_chosen": -271.7354431152344,
"logps/ref_rejected": -262.64111328125,
"logps/rejected": -261.0503845214844,
"loss": 5.4305,
"margin_dpo/margin_mean": 2.9504590034484863,
"margin_dpo/margin_std": 4.740379333496094,
"step": 76
},
{
"KL/chosen_KL_mean": 4.897563934326172,
"KL/mean": 3.6664695739746094,
"KL/rejected_KL_mean": 2.4353809356689453,
"KL/std": 4.139870643615723,
"epoch": 0.1612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.4621849060058594,
"fcm_dpo/q_t": 0.4938510060310364,
"grad_norm": 14.8322172164917,
"learning_rate": 4.947629214246236e-07,
"logits/chosen": 1.782356858253479,
"logits/rejected": 1.629751443862915,
"logps/chosen": -289.7730407714844,
"logps/ref_chosen": -294.6706237792969,
"logps/ref_rejected": -247.6900177001953,
"logps/rejected": -245.25465393066406,
"loss": 5.4502,
"margin_dpo/margin_mean": 2.4621849060058594,
"margin_dpo/margin_std": 5.111874103546143,
"step": 77
},
{
"KL/chosen_KL_mean": 5.139194488525391,
"KL/mean": 3.663858413696289,
"KL/rejected_KL_mean": 2.1885147094726562,
"KL/std": 3.7755632400512695,
"epoch": 0.16335078534031414,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.950676679611206,
"fcm_dpo/q_t": 0.49263134598731995,
"grad_norm": 14.61767578125,
"learning_rate": 4.943835963210323e-07,
"logits/chosen": 1.6119264364242554,
"logits/rejected": 1.5382490158081055,
"logps/chosen": -243.01309204101562,
"logps/ref_chosen": -248.1522979736328,
"logps/ref_rejected": -209.12808227539062,
"logps/rejected": -206.9395294189453,
"loss": 5.4309,
"margin_dpo/margin_mean": 2.950676441192627,
"margin_dpo/margin_std": 5.283993721008301,
"step": 78
},
{
"KL/chosen_KL_mean": 5.191509246826172,
"KL/mean": 3.6148133277893066,
"KL/rejected_KL_mean": 2.038118362426758,
"KL/std": 4.859566688537598,
"epoch": 0.16544502617801046,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.153383493423462,
"fcm_dpo/q_t": 0.49211859703063965,
"grad_norm": 16.667619705200195,
"learning_rate": 4.939911656668361e-07,
"logits/chosen": 1.5719172954559326,
"logits/rejected": 1.6212188005447388,
"logps/chosen": -254.28286743164062,
"logps/ref_chosen": -259.4743957519531,
"logps/ref_rejected": -246.9627227783203,
"logps/rejected": -244.92459106445312,
"loss": 5.4248,
"margin_dpo/margin_mean": 3.153383255004883,
"margin_dpo/margin_std": 6.35499382019043,
"step": 79
},
{
"KL/chosen_KL_mean": 5.269865036010742,
"KL/mean": 3.982539653778076,
"KL/rejected_KL_mean": 2.6952075958251953,
"KL/std": 4.611552715301514,
"epoch": 0.16753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.5746498107910156,
"fcm_dpo/q_t": 0.49357014894485474,
"grad_norm": 16.91059684753418,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": 1.7765568494796753,
"logits/rejected": 1.7734700441360474,
"logps/chosen": -280.65667724609375,
"logps/ref_chosen": -285.926513671875,
"logps/ref_rejected": -253.77996826171875,
"logps/rejected": -251.0847625732422,
"loss": 5.4465,
"margin_dpo/margin_mean": 2.574650287628174,
"margin_dpo/margin_std": 5.556662559509277,
"step": 80
},
{
"KL/chosen_KL_mean": 5.917991638183594,
"KL/mean": 4.468988418579102,
"KL/rejected_KL_mean": 3.019977569580078,
"KL/std": 4.51953125,
"epoch": 0.16963350785340314,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.898013114929199,
"fcm_dpo/q_t": 0.4927605092525482,
"grad_norm": 14.584118843078613,
"learning_rate": 4.93167072587771e-07,
"logits/chosen": 1.8349915742874146,
"logits/rejected": 1.7867443561553955,
"logps/chosen": -256.67620849609375,
"logps/ref_chosen": -262.5942077636719,
"logps/ref_rejected": -236.687744140625,
"logps/rejected": -233.66775512695312,
"loss": 5.4345,
"margin_dpo/margin_mean": 2.898014545440674,
"margin_dpo/margin_std": 6.337032794952393,
"step": 81
},
{
"KL/chosen_KL_mean": 6.365209579467773,
"KL/mean": 4.8737077713012695,
"KL/rejected_KL_mean": 3.3821964263916016,
"KL/std": 4.350286960601807,
"epoch": 0.17172774869109947,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.983015298843384,
"fcm_dpo/q_t": 0.4925503730773926,
"grad_norm": 17.14452362060547,
"learning_rate": 4.92735454356513e-07,
"logits/chosen": 1.762351632118225,
"logits/rejected": 1.7092797756195068,
"logps/chosen": -264.677978515625,
"logps/ref_chosen": -271.04315185546875,
"logps/ref_rejected": -234.03359985351562,
"logps/rejected": -230.65138244628906,
"loss": 5.4303,
"margin_dpo/margin_mean": 2.9830164909362793,
"margin_dpo/margin_std": 5.512236595153809,
"step": 82
},
{
"KL/chosen_KL_mean": 6.469331741333008,
"KL/mean": 4.712227821350098,
"KL/rejected_KL_mean": 2.9551239013671875,
"KL/std": 4.9338226318359375,
"epoch": 0.17382198952879582,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.514209032058716,
"fcm_dpo/q_t": 0.49122947454452515,
"grad_norm": 15.297825813293457,
"learning_rate": 4.922908189595017e-07,
"logits/chosen": 1.737414836883545,
"logits/rejected": 1.8145241737365723,
"logps/chosen": -259.8159484863281,
"logps/ref_chosen": -266.2852783203125,
"logps/ref_rejected": -274.32501220703125,
"logps/rejected": -271.369873046875,
"loss": 5.4108,
"margin_dpo/margin_mean": 3.5142087936401367,
"margin_dpo/margin_std": 6.680100440979004,
"step": 83
},
{
"KL/chosen_KL_mean": 6.714977264404297,
"KL/mean": 5.295899391174316,
"KL/rejected_KL_mean": 3.876821517944336,
"KL/std": 5.049211502075195,
"epoch": 0.17591623036649215,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.8381547927856445,
"fcm_dpo/q_t": 0.4929097890853882,
"grad_norm": 15.236591339111328,
"learning_rate": 4.918331902411841e-07,
"logits/chosen": 1.9121453762054443,
"logits/rejected": 1.904983639717102,
"logps/chosen": -280.0656433105469,
"logps/ref_chosen": -286.7806091308594,
"logps/ref_rejected": -276.1176452636719,
"logps/rejected": -272.2408142089844,
"loss": 5.4371,
"margin_dpo/margin_mean": 2.83815336227417,
"margin_dpo/margin_std": 6.488803386688232,
"step": 84
},
{
"KL/chosen_KL_mean": 6.354665756225586,
"KL/mean": 5.3685302734375,
"KL/rejected_KL_mean": 4.382389068603516,
"KL/std": 4.390091419219971,
"epoch": 0.17801047120418848,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.97226881980896,
"fcm_dpo/q_t": 0.49507302045822144,
"grad_norm": 14.166430473327637,
"learning_rate": 4.913625927427995e-07,
"logits/chosen": 1.931032419204712,
"logits/rejected": 2.031745195388794,
"logps/chosen": -228.67518615722656,
"logps/ref_chosen": -235.02987670898438,
"logps/ref_rejected": -259.80999755859375,
"logps/rejected": -255.4276123046875,
"loss": 5.4701,
"margin_dpo/margin_mean": 1.9722685813903809,
"margin_dpo/margin_std": 5.482989311218262,
"step": 85
},
{
"KL/chosen_KL_mean": 7.849931716918945,
"KL/mean": 6.066666603088379,
"KL/rejected_KL_mean": 4.28339958190918,
"KL/std": 5.507645130157471,
"epoch": 0.18010471204188483,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.5665531158447266,
"fcm_dpo/q_t": 0.4910971522331238,
"grad_norm": 16.2667236328125,
"learning_rate": 4.908790517010636e-07,
"logits/chosen": 1.887448787689209,
"logits/rejected": 1.934370994567871,
"logps/chosen": -293.46282958984375,
"logps/ref_chosen": -301.312744140625,
"logps/ref_rejected": -288.8368225097656,
"logps/rejected": -284.55340576171875,
"loss": 5.4096,
"margin_dpo/margin_mean": 3.5665528774261475,
"margin_dpo/margin_std": 7.414119720458984,
"step": 86
},
{
"KL/chosen_KL_mean": 8.08189582824707,
"KL/mean": 6.034160137176514,
"KL/rejected_KL_mean": 3.986419677734375,
"KL/std": 5.5277419090271,
"epoch": 0.18219895287958116,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.095483779907227,
"fcm_dpo/q_t": 0.48977530002593994,
"grad_norm": 15.365828514099121,
"learning_rate": 4.903825930468148e-07,
"logits/chosen": 1.7935822010040283,
"logits/rejected": 1.788144588470459,
"logps/chosen": -257.5841064453125,
"logps/ref_chosen": -265.6659851074219,
"logps/ref_rejected": -242.0697479248047,
"logps/rejected": -238.08334350585938,
"loss": 5.3875,
"margin_dpo/margin_mean": 4.09548282623291,
"margin_dpo/margin_std": 6.456631183624268,
"step": 87
},
{
"KL/chosen_KL_mean": 7.397937774658203,
"KL/mean": 5.798396587371826,
"KL/rejected_KL_mean": 4.198854446411133,
"KL/std": 5.472133636474609,
"epoch": 0.18429319371727748,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.199070692062378,
"fcm_dpo/q_t": 0.49201327562332153,
"grad_norm": 14.061400413513184,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 1.8010516166687012,
"logits/rejected": 1.820248007774353,
"logps/chosen": -253.32351684570312,
"logps/ref_chosen": -260.721435546875,
"logps/ref_rejected": -262.6593322753906,
"logps/rejected": -258.46044921875,
"loss": 5.4234,
"margin_dpo/margin_mean": 3.1990714073181152,
"margin_dpo/margin_std": 6.9819793701171875,
"step": 88
},
{
"KL/chosen_KL_mean": 9.051471710205078,
"KL/mean": 7.091951370239258,
"KL/rejected_KL_mean": 5.132427215576172,
"KL/std": 5.247347831726074,
"epoch": 0.18638743455497384,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.9190514087677,
"fcm_dpo/q_t": 0.49021562933921814,
"grad_norm": 15.85186767578125,
"learning_rate": 4.893510300863676e-07,
"logits/chosen": 1.770682454109192,
"logits/rejected": 1.7325642108917236,
"logps/chosen": -247.95135498046875,
"logps/ref_chosen": -257.0028076171875,
"logps/ref_rejected": -254.66232299804688,
"logps/rejected": -249.52989196777344,
"loss": 5.3944,
"margin_dpo/margin_mean": 3.9190523624420166,
"margin_dpo/margin_std": 6.255563259124756,
"step": 89
},
{
"KL/chosen_KL_mean": 8.974075317382812,
"KL/mean": 7.279847145080566,
"KL/rejected_KL_mean": 5.585620880126953,
"KL/std": 5.636865139007568,
"epoch": 0.18848167539267016,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3884599208831787,
"fcm_dpo/q_t": 0.49154117703437805,
"grad_norm": 15.255399703979492,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": 2.067680597305298,
"logits/rejected": 2.0169973373413086,
"logps/chosen": -289.4363098144531,
"logps/ref_chosen": -298.4103698730469,
"logps/ref_rejected": -262.8761901855469,
"logps/rejected": -257.2905578613281,
"loss": 5.4163,
"margin_dpo/margin_mean": 3.3884589672088623,
"margin_dpo/margin_std": 6.8790998458862305,
"step": 90
},
{
"KL/chosen_KL_mean": 8.779237747192383,
"KL/mean": 7.078490734100342,
"KL/rejected_KL_mean": 5.377744674682617,
"KL/std": 5.447061538696289,
"epoch": 0.1905759162303665,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.40149188041687,
"fcm_dpo/q_t": 0.4915134608745575,
"grad_norm": 14.696788787841797,
"learning_rate": 4.882681251368548e-07,
"logits/chosen": 1.5244275331497192,
"logits/rejected": 1.557888150215149,
"logps/chosen": -218.74464416503906,
"logps/ref_chosen": -227.5238800048828,
"logps/ref_rejected": -227.11428833007812,
"logps/rejected": -221.73654174804688,
"loss": 5.4161,
"margin_dpo/margin_mean": 3.4014906883239746,
"margin_dpo/margin_std": 7.320674896240234,
"step": 91
},
{
"KL/chosen_KL_mean": 8.950838088989258,
"KL/mean": 7.051262378692627,
"KL/rejected_KL_mean": 5.151678085327148,
"KL/std": 6.589147090911865,
"epoch": 0.19267015706806281,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.7991676330566406,
"fcm_dpo/q_t": 0.4905143678188324,
"grad_norm": 15.557400703430176,
"learning_rate": 4.877074915775048e-07,
"logits/chosen": 1.8932842016220093,
"logits/rejected": 1.7986698150634766,
"logps/chosen": -266.431396484375,
"logps/ref_chosen": -275.3822326660156,
"logps/ref_rejected": -269.8356018066406,
"logps/rejected": -264.68389892578125,
"loss": 5.401,
"margin_dpo/margin_mean": 3.7991671562194824,
"margin_dpo/margin_std": 7.670883655548096,
"step": 92
},
{
"KL/chosen_KL_mean": 9.275632858276367,
"KL/mean": 7.499587059020996,
"KL/rejected_KL_mean": 5.723550796508789,
"KL/std": 5.7577009201049805,
"epoch": 0.19476439790575917,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.5520870685577393,
"fcm_dpo/q_t": 0.49113625288009644,
"grad_norm": 14.2173490524292,
"learning_rate": 4.871341104867864e-07,
"logits/chosen": 1.7212178707122803,
"logits/rejected": 1.702576994895935,
"logps/chosen": -218.84075927734375,
"logps/ref_chosen": -228.1163787841797,
"logps/ref_rejected": -228.26266479492188,
"logps/rejected": -222.53909301757812,
"loss": 5.41,
"margin_dpo/margin_mean": 3.5520858764648438,
"margin_dpo/margin_std": 7.237574100494385,
"step": 93
},
{
"KL/chosen_KL_mean": 10.041501998901367,
"KL/mean": 8.051166534423828,
"KL/rejected_KL_mean": 6.060827255249023,
"KL/std": 6.666971206665039,
"epoch": 0.1968586387434555,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.980672597885132,
"fcm_dpo/q_t": 0.4900735318660736,
"grad_norm": 15.548222541809082,
"learning_rate": 4.865480126133871e-07,
"logits/chosen": 1.9226418733596802,
"logits/rejected": 2.030660629272461,
"logps/chosen": -273.9857177734375,
"logps/ref_chosen": -284.0272521972656,
"logps/ref_rejected": -282.1720275878906,
"logps/rejected": -276.1112060546875,
"loss": 5.3958,
"margin_dpo/margin_mean": 3.980672836303711,
"margin_dpo/margin_std": 8.678937911987305,
"step": 94
},
{
"KL/chosen_KL_mean": 9.779237747192383,
"KL/mean": 7.586515426635742,
"KL/rejected_KL_mean": 5.393795013427734,
"KL/std": 6.788585186004639,
"epoch": 0.19895287958115182,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.385440349578857,
"fcm_dpo/q_t": 0.48906460404396057,
"grad_norm": 15.942317962646484,
"learning_rate": 4.859492293879573e-07,
"logits/chosen": 1.79828679561615,
"logits/rejected": 1.7253429889678955,
"logps/chosen": -293.2705993652344,
"logps/ref_chosen": -303.0498352050781,
"logps/ref_rejected": -259.8980712890625,
"logps/rejected": -254.50428771972656,
"loss": 5.3798,
"margin_dpo/margin_mean": 4.385439395904541,
"margin_dpo/margin_std": 8.744900703430176,
"step": 95
},
{
"KL/chosen_KL_mean": 9.869930267333984,
"KL/mean": 7.763474464416504,
"KL/rejected_KL_mean": 5.657018661499023,
"KL/std": 7.008248329162598,
"epoch": 0.20104712041884817,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.212911605834961,
"fcm_dpo/q_t": 0.48949211835861206,
"grad_norm": 15.15978717803955,
"learning_rate": 4.853377929214243e-07,
"logits/chosen": 1.7972484827041626,
"logits/rejected": 1.7564634084701538,
"logps/chosen": -260.7177734375,
"logps/ref_chosen": -270.58770751953125,
"logps/ref_rejected": -238.90069580078125,
"logps/rejected": -233.24366760253906,
"loss": 5.3855,
"margin_dpo/margin_mean": 4.212912559509277,
"margin_dpo/margin_std": 7.945834159851074,
"step": 96
},
{
"KL/chosen_KL_mean": 9.695920944213867,
"KL/mean": 7.339411735534668,
"KL/rejected_KL_mean": 4.982898712158203,
"KL/std": 6.886740684509277,
"epoch": 0.2031413612565445,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.713016033172607,
"fcm_dpo/q_t": 0.488258957862854,
"grad_norm": 15.31843376159668,
"learning_rate": 4.847137360032699e-07,
"logits/chosen": 1.8880928754806519,
"logits/rejected": 1.7756001949310303,
"logps/chosen": -286.4896545410156,
"logps/ref_chosen": -296.1855773925781,
"logps/ref_rejected": -260.35333251953125,
"logps/rejected": -255.3704376220703,
"loss": 5.3685,
"margin_dpo/margin_mean": 4.713016033172607,
"margin_dpo/margin_std": 9.550836563110352,
"step": 97
},
{
"KL/chosen_KL_mean": 10.064401626586914,
"KL/mean": 7.7170209884643555,
"KL/rejected_KL_mean": 5.369632720947266,
"KL/std": 7.359187126159668,
"epoch": 0.20523560209424083,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.694765090942383,
"fcm_dpo/q_t": 0.488303005695343,
"grad_norm": 15.714393615722656,
"learning_rate": 4.84077092099773e-07,
"logits/chosen": 2.0168633460998535,
"logits/rejected": 2.0064096450805664,
"logps/chosen": -267.65093994140625,
"logps/ref_chosen": -277.7153625488281,
"logps/ref_rejected": -272.9094543457031,
"logps/rejected": -267.5398254394531,
"loss": 5.3704,
"margin_dpo/margin_mean": 4.694764614105225,
"margin_dpo/margin_std": 9.518474578857422,
"step": 98
},
{
"KL/chosen_KL_mean": 10.338481903076172,
"KL/mean": 7.729951858520508,
"KL/rejected_KL_mean": 5.121423721313477,
"KL/std": 7.738285064697266,
"epoch": 0.20732984293193718,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.217060089111328,
"fcm_dpo/q_t": 0.4869866967201233,
"grad_norm": 15.05201530456543,
"learning_rate": 4.834278953522137e-07,
"logits/chosen": 1.743818759918213,
"logits/rejected": 1.7173069715499878,
"logps/chosen": -262.1993408203125,
"logps/ref_chosen": -272.5378112792969,
"logps/ref_rejected": -244.6329345703125,
"logps/rejected": -239.51150512695312,
"loss": 5.3509,
"margin_dpo/margin_mean": 5.21705961227417,
"margin_dpo/margin_std": 10.1746244430542,
"step": 99
},
{
"KL/chosen_KL_mean": 9.348031997680664,
"KL/mean": 7.089855670928955,
"KL/rejected_KL_mean": 4.831691741943359,
"KL/std": 7.804023742675781,
"epoch": 0.2094240837696335,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.516352653503418,
"fcm_dpo/q_t": 0.48874592781066895,
"grad_norm": 15.512487411499023,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": 1.6178163290023804,
"logits/rejected": 1.5342121124267578,
"logps/chosen": -280.4643249511719,
"logps/ref_chosen": -289.8123474121094,
"logps/ref_rejected": -274.29071044921875,
"logps/rejected": -269.4590148925781,
"loss": 5.376,
"margin_dpo/margin_mean": 4.516352653503418,
"margin_dpo/margin_std": 9.030591011047363,
"step": 100
},
{
"KL/chosen_KL_mean": 9.87614631652832,
"KL/mean": 7.632713794708252,
"KL/rejected_KL_mean": 5.389278411865234,
"KL/std": 8.039362907409668,
"epoch": 0.21151832460732983,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.486861705780029,
"fcm_dpo/q_t": 0.4888134002685547,
"grad_norm": 15.381603240966797,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 1.7923437356948853,
"logits/rejected": 1.9661798477172852,
"logps/chosen": -235.83697509765625,
"logps/ref_chosen": -245.71311950683594,
"logps/ref_rejected": -272.08935546875,
"logps/rejected": -266.7000732421875,
"loss": 5.3784,
"margin_dpo/margin_mean": 4.486861705780029,
"margin_dpo/margin_std": 9.73862075805664,
"step": 101
},
{
"KL/chosen_KL_mean": 10.340808868408203,
"KL/mean": 7.144167900085449,
"KL/rejected_KL_mean": 3.9475269317626953,
"KL/std": 7.482494354248047,
"epoch": 0.2136125654450262,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.393280982971191,
"fcm_dpo/q_t": 0.48405927419662476,
"grad_norm": 15.228653907775879,
"learning_rate": 4.814053395442932e-07,
"logits/chosen": 1.5331921577453613,
"logits/rejected": 1.6188983917236328,
"logps/chosen": -199.6016082763672,
"logps/ref_chosen": -209.94239807128906,
"logps/ref_rejected": -230.66897583007812,
"logps/rejected": -226.721435546875,
"loss": 5.3023,
"margin_dpo/margin_mean": 6.393280506134033,
"margin_dpo/margin_std": 8.819887161254883,
"step": 102
},
{
"KL/chosen_KL_mean": 9.326803207397461,
"KL/mean": 6.885707378387451,
"KL/rejected_KL_mean": 4.444611549377441,
"KL/std": 8.337653160095215,
"epoch": 0.2157068062827225,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.882187843322754,
"fcm_dpo/q_t": 0.4878406524658203,
"grad_norm": 15.703271865844727,
"learning_rate": 4.807062862684873e-07,
"logits/chosen": 1.6679704189300537,
"logits/rejected": 1.7627195119857788,
"logps/chosen": -241.9728240966797,
"logps/ref_chosen": -251.2996368408203,
"logps/ref_rejected": -277.4847412109375,
"logps/rejected": -273.0401306152344,
"loss": 5.3636,
"margin_dpo/margin_mean": 4.882188320159912,
"margin_dpo/margin_std": 10.275616645812988,
"step": 103
},
{
"KL/chosen_KL_mean": 8.242347717285156,
"KL/mean": 6.029875755310059,
"KL/rejected_KL_mean": 3.8173999786376953,
"KL/std": 8.634225845336914,
"epoch": 0.21780104712041884,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.424940586090088,
"fcm_dpo/q_t": 0.4889841675758362,
"grad_norm": 14.384554862976074,
"learning_rate": 4.799948609147061e-07,
"logits/chosen": 1.7679015398025513,
"logits/rejected": 1.8099502325057983,
"logps/chosen": -249.53707885742188,
"logps/ref_chosen": -257.7794189453125,
"logps/ref_rejected": -227.91546630859375,
"logps/rejected": -224.09805297851562,
"loss": 5.3835,
"margin_dpo/margin_mean": 4.42494010925293,
"margin_dpo/margin_std": 11.217126846313477,
"step": 104
},
{
"KL/chosen_KL_mean": 10.295242309570312,
"KL/mean": 6.219893455505371,
"KL/rejected_KL_mean": 2.1445579528808594,
"KL/std": 9.750238418579102,
"epoch": 0.2198952879581152,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.1506929397583,
"fcm_dpo/q_t": 0.47969570755958557,
"grad_norm": 17.80915069580078,
"learning_rate": 4.792711016345321e-07,
"logits/chosen": 1.8179645538330078,
"logits/rejected": 1.78440260887146,
"logps/chosen": -301.79876708984375,
"logps/ref_chosen": -312.09405517578125,
"logps/ref_rejected": -277.97332763671875,
"logps/rejected": -275.8287658691406,
"loss": 5.2387,
"margin_dpo/margin_mean": 8.1506929397583,
"margin_dpo/margin_std": 10.811229705810547,
"step": 105
},
{
"KL/chosen_KL_mean": 6.591794967651367,
"KL/mean": 3.870194911956787,
"KL/rejected_KL_mean": 1.1485862731933594,
"KL/std": 8.947371482849121,
"epoch": 0.22198952879581152,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.443210124969482,
"fcm_dpo/q_t": 0.48644956946372986,
"grad_norm": 28.13164710998535,
"learning_rate": 4.785350472409791e-07,
"logits/chosen": 1.9726589918136597,
"logits/rejected": 2.0139129161834717,
"logps/chosen": -278.7745361328125,
"logps/ref_chosen": -285.3663330078125,
"logps/ref_rejected": -262.2779541015625,
"logps/rejected": -261.1293640136719,
"loss": 5.3443,
"margin_dpo/margin_mean": 5.443212032318115,
"margin_dpo/margin_std": 11.273680686950684,
"step": 106
},
{
"KL/chosen_KL_mean": 9.141887664794922,
"KL/mean": 5.479947566986084,
"KL/rejected_KL_mean": 1.8180046081542969,
"KL/std": 9.478837013244629,
"epoch": 0.22408376963350785,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.3238911628723145,
"fcm_dpo/q_t": 0.48174697160720825,
"grad_norm": 16.708341598510742,
"learning_rate": 4.777867372064105e-07,
"logits/chosen": 1.8046985864639282,
"logits/rejected": 1.8737220764160156,
"logps/chosen": -287.1517028808594,
"logps/ref_chosen": -296.2935791015625,
"logps/ref_rejected": -291.5162353515625,
"logps/rejected": -289.6982116699219,
"loss": 5.2726,
"margin_dpo/margin_mean": 7.323891639709473,
"margin_dpo/margin_std": 11.570927619934082,
"step": 107
},
{
"KL/chosen_KL_mean": 9.301300048828125,
"KL/mean": 5.2102484703063965,
"KL/rejected_KL_mean": 1.11920166015625,
"KL/std": 9.038829803466797,
"epoch": 0.2261780104712042,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.182097434997559,
"fcm_dpo/q_t": 0.479623943567276,
"grad_norm": 16.092702865600586,
"learning_rate": 4.770262116604223e-07,
"logits/chosen": 1.8490670919418335,
"logits/rejected": 1.7061842679977417,
"logps/chosen": -278.82305908203125,
"logps/ref_chosen": -288.12432861328125,
"logps/ref_rejected": -232.35720825195312,
"logps/rejected": -231.23802185058594,
"loss": 5.2367,
"margin_dpo/margin_mean": 8.182095527648926,
"margin_dpo/margin_std": 10.616247177124023,
"step": 108
},
{
"KL/chosen_KL_mean": 7.901805877685547,
"KL/mean": 3.661217212677002,
"KL/rejected_KL_mean": -0.5793647766113281,
"KL/std": 10.176899909973145,
"epoch": 0.22827225130890053,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.481165885925293,
"fcm_dpo/q_t": 0.47893649339675903,
"grad_norm": 17.60049057006836,
"learning_rate": 4.7625351138769166e-07,
"logits/chosen": 1.7414685487747192,
"logits/rejected": 1.6945842504501343,
"logps/chosen": -309.29974365234375,
"logps/ref_chosen": -317.2015686035156,
"logps/ref_rejected": -278.0653381347656,
"logps/rejected": -278.6446838378906,
"loss": 5.2316,
"margin_dpo/margin_mean": 8.481164932250977,
"margin_dpo/margin_std": 12.964422225952148,
"step": 109
},
{
"KL/chosen_KL_mean": 8.356040954589844,
"KL/mean": 4.442471027374268,
"KL/rejected_KL_mean": 0.5289173126220703,
"KL/std": 10.09596061706543,
"epoch": 0.23036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.827131748199463,
"fcm_dpo/q_t": 0.4805219769477844,
"grad_norm": 16.00581932067871,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": 1.8129990100860596,
"logits/rejected": 1.8113503456115723,
"logps/chosen": -254.58340454101562,
"logps/ref_chosen": -262.939453125,
"logps/ref_rejected": -257.82275390625,
"logps/rejected": -257.2938232421875,
"loss": 5.2562,
"margin_dpo/margin_mean": 7.827132701873779,
"margin_dpo/margin_std": 13.084668159484863,
"step": 110
},
{
"KL/chosen_KL_mean": 8.985616683959961,
"KL/mean": 4.6226325035095215,
"KL/rejected_KL_mean": 0.25965309143066406,
"KL/std": 11.262866973876953,
"epoch": 0.2324607329842932,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.725970268249512,
"fcm_dpo/q_t": 0.47832146286964417,
"grad_norm": 18.12258529663086,
"learning_rate": 4.7467175306295647e-07,
"logits/chosen": 1.8897958993911743,
"logits/rejected": 1.7945163249969482,
"logps/chosen": -310.168212890625,
"logps/ref_chosen": -319.15380859375,
"logps/ref_rejected": -281.7733154296875,
"logps/rejected": -281.513671875,
"loss": 5.2221,
"margin_dpo/margin_mean": 8.725970268249512,
"margin_dpo/margin_std": 12.794700622558594,
"step": 111
},
{
"KL/chosen_KL_mean": 5.339027404785156,
"KL/mean": 3.1850929260253906,
"KL/rejected_KL_mean": 1.0311565399169922,
"KL/std": 10.531282424926758,
"epoch": 0.23455497382198953,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.3078765869140625,
"fcm_dpo/q_t": 0.4892743229866028,
"grad_norm": 14.627169609069824,
"learning_rate": 4.7386277983585053e-07,
"logits/chosen": 1.8687463998794556,
"logits/rejected": 1.9776508808135986,
"logps/chosen": -245.16140747070312,
"logps/ref_chosen": -250.50042724609375,
"logps/ref_rejected": -268.32635498046875,
"logps/rejected": -267.29522705078125,
"loss": 5.3983,
"margin_dpo/margin_mean": 4.307877063751221,
"margin_dpo/margin_std": 14.214605331420898,
"step": 112
},
{
"KL/chosen_KL_mean": 6.826242446899414,
"KL/mean": 1.7497713565826416,
"KL/rejected_KL_mean": -3.3266963958740234,
"KL/std": 11.603326797485352,
"epoch": 0.23664921465968586,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.152936935424805,
"fcm_dpo/q_t": 0.4747968018054962,
"grad_norm": 16.86988067626953,
"learning_rate": 4.7304180152725024e-07,
"logits/chosen": 1.5450334548950195,
"logits/rejected": 1.574907660484314,
"logps/chosen": -272.5753479003906,
"logps/ref_chosen": -279.4015808105469,
"logps/ref_rejected": -266.6392822265625,
"logps/rejected": -269.96600341796875,
"loss": 5.1714,
"margin_dpo/margin_mean": 10.152936935424805,
"margin_dpo/margin_std": 14.223891258239746,
"step": 113
},
{
"KL/chosen_KL_mean": 5.255594253540039,
"KL/mean": 2.662837266921997,
"KL/rejected_KL_mean": 0.07008552551269531,
"KL/std": 10.680999755859375,
"epoch": 0.2387434554973822,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.1855058670043945,
"fcm_dpo/q_t": 0.4871106743812561,
"grad_norm": 15.769734382629395,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 1.8406256437301636,
"logits/rejected": 1.8326904773712158,
"logps/chosen": -249.00489807128906,
"logps/ref_chosen": -254.26048278808594,
"logps/ref_rejected": -241.8769989013672,
"logps/rejected": -241.80690002441406,
"loss": 5.3594,
"margin_dpo/margin_mean": 5.185506343841553,
"margin_dpo/margin_std": 13.097881317138672,
"step": 114
},
{
"KL/chosen_KL_mean": 5.537559509277344,
"KL/mean": 2.6346588134765625,
"KL/rejected_KL_mean": -0.2682456970214844,
"KL/std": 11.89238452911377,
"epoch": 0.24083769633507854,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.805802822113037,
"fcm_dpo/q_t": 0.4856095016002655,
"grad_norm": 15.853104591369629,
"learning_rate": 4.7136400641330245e-07,
"logits/chosen": 1.6168692111968994,
"logits/rejected": 1.6461868286132812,
"logps/chosen": -283.2669372558594,
"logps/ref_chosen": -288.80450439453125,
"logps/ref_rejected": -266.8573913574219,
"logps/rejected": -267.1256103515625,
"loss": 5.3395,
"margin_dpo/margin_mean": 5.805802822113037,
"margin_dpo/margin_std": 14.407818794250488,
"step": 115
},
{
"KL/chosen_KL_mean": 7.151908874511719,
"KL/mean": 3.055176258087158,
"KL/rejected_KL_mean": -1.0415592193603516,
"KL/std": 11.208464622497559,
"epoch": 0.24293193717277486,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.193467140197754,
"fcm_dpo/q_t": 0.47964489459991455,
"grad_norm": 16.368608474731445,
"learning_rate": 4.70507279583015e-07,
"logits/chosen": 1.656557559967041,
"logits/rejected": 1.6314194202423096,
"logps/chosen": -265.0380554199219,
"logps/ref_chosen": -272.18994140625,
"logps/ref_rejected": -253.14334106445312,
"logps/rejected": -254.1848907470703,
"loss": 5.2458,
"margin_dpo/margin_mean": 8.193467140197754,
"margin_dpo/margin_std": 14.211151123046875,
"step": 116
},
{
"KL/chosen_KL_mean": 5.650030136108398,
"KL/mean": 1.4515990018844604,
"KL/rejected_KL_mean": -2.746824264526367,
"KL/std": 11.022621154785156,
"epoch": 0.2450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.39685344696045,
"fcm_dpo/q_t": 0.47912970185279846,
"grad_norm": 16.54109764099121,
"learning_rate": 4.6963872761652834e-07,
"logits/chosen": 1.7056045532226562,
"logits/rejected": 1.704630970954895,
"logps/chosen": -244.75259399414062,
"logps/ref_chosen": -250.4026336669922,
"logps/ref_rejected": -229.63853454589844,
"logps/rejected": -232.38536071777344,
"loss": 5.2352,
"margin_dpo/margin_mean": 8.39685344696045,
"margin_dpo/margin_std": 12.509329795837402,
"step": 117
},
{
"KL/chosen_KL_mean": 5.426443099975586,
"KL/mean": 0.9376038312911987,
"KL/rejected_KL_mean": -3.5512428283691406,
"KL/std": 12.780660629272461,
"epoch": 0.24712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.977682113647461,
"fcm_dpo/q_t": 0.47770825028419495,
"grad_norm": 20.744915008544922,
"learning_rate": 4.687583970916486e-07,
"logits/chosen": 1.6222083568572998,
"logits/rejected": 1.6639357805252075,
"logps/chosen": -252.96945190429688,
"logps/ref_chosen": -258.3959045410156,
"logps/ref_rejected": -268.84228515625,
"logps/rejected": -272.3935546875,
"loss": 5.2174,
"margin_dpo/margin_mean": 8.977681159973145,
"margin_dpo/margin_std": 14.679039001464844,
"step": 118
},
{
"KL/chosen_KL_mean": 6.003448486328125,
"KL/mean": 2.179342269897461,
"KL/rejected_KL_mean": -1.6447715759277344,
"KL/std": 14.2131986618042,
"epoch": 0.24921465968586387,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.648217678070068,
"fcm_dpo/q_t": 0.48110345005989075,
"grad_norm": 15.949928283691406,
"learning_rate": 4.6786633521783005e-07,
"logits/chosen": 1.9601776599884033,
"logits/rejected": 1.9405962228775024,
"logps/chosen": -311.2991027832031,
"logps/ref_chosen": -317.30255126953125,
"logps/ref_rejected": -300.47552490234375,
"logps/rejected": -302.1202697753906,
"loss": 5.2785,
"margin_dpo/margin_mean": 7.648219108581543,
"margin_dpo/margin_std": 16.88101577758789,
"step": 119
},
{
"KL/chosen_KL_mean": 6.925140380859375,
"KL/mean": 2.351628303527832,
"KL/rejected_KL_mean": -2.2218856811523438,
"KL/std": 13.34640121459961,
"epoch": 0.2513089005235602,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.147025108337402,
"fcm_dpo/q_t": 0.47732555866241455,
"grad_norm": 16.257707595825195,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": 1.8242895603179932,
"logits/rejected": 1.7785429954528809,
"logps/chosen": -298.32928466796875,
"logps/ref_chosen": -305.2544250488281,
"logps/ref_rejected": -259.40716552734375,
"logps/rejected": -261.6290283203125,
"loss": 5.2168,
"margin_dpo/margin_mean": 9.147024154663086,
"margin_dpo/margin_std": 16.313743591308594,
"step": 120
},
{
"KL/chosen_KL_mean": 3.4570655822753906,
"KL/mean": 0.7536537647247314,
"KL/rejected_KL_mean": -1.9497547149658203,
"KL/std": 11.887968063354492,
"epoch": 0.2534031413612565,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.4068145751953125,
"fcm_dpo/q_t": 0.48658356070518494,
"grad_norm": 14.691038131713867,
"learning_rate": 4.6604720940421207e-07,
"logits/chosen": 1.5685416460037231,
"logits/rejected": 1.587632656097412,
"logps/chosen": -213.18365478515625,
"logps/ref_chosen": -216.64073181152344,
"logps/ref_rejected": -224.36422729492188,
"logps/rejected": -226.31396484375,
"loss": 5.3553,
"margin_dpo/margin_mean": 5.406813621520996,
"margin_dpo/margin_std": 14.467510223388672,
"step": 121
},
{
"KL/chosen_KL_mean": 5.674816131591797,
"KL/mean": 2.0157768726348877,
"KL/rejected_KL_mean": -1.6432666778564453,
"KL/std": 14.010313034057617,
"epoch": 0.2554973821989529,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.318086624145508,
"fcm_dpo/q_t": 0.4818747043609619,
"grad_norm": 15.785666465759277,
"learning_rate": 4.651202430186092e-07,
"logits/chosen": 1.884055495262146,
"logits/rejected": 1.871678113937378,
"logps/chosen": -263.14398193359375,
"logps/ref_chosen": -268.8188171386719,
"logps/ref_rejected": -272.55853271484375,
"logps/rejected": -274.2017822265625,
"loss": 5.2913,
"margin_dpo/margin_mean": 7.318086624145508,
"margin_dpo/margin_std": 17.51386260986328,
"step": 122
},
{
"KL/chosen_KL_mean": 8.699848175048828,
"KL/mean": 3.4266481399536133,
"KL/rejected_KL_mean": -1.8465538024902344,
"KL/std": 14.962854385375977,
"epoch": 0.25759162303664923,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.546406745910645,
"fcm_dpo/q_t": 0.4738994240760803,
"grad_norm": 18.218210220336914,
"learning_rate": 4.6418174038722924e-07,
"logits/chosen": 1.7776902914047241,
"logits/rejected": 1.7658780813217163,
"logps/chosen": -312.99188232421875,
"logps/ref_chosen": -321.6917419433594,
"logps/ref_rejected": -279.1517333984375,
"logps/rejected": -280.9982604980469,
"loss": 5.1692,
"margin_dpo/margin_mean": 10.546405792236328,
"margin_dpo/margin_std": 18.25002670288086,
"step": 123
},
{
"KL/chosen_KL_mean": 6.083673477172852,
"KL/mean": 1.3417143821716309,
"KL/rejected_KL_mean": -3.4002513885498047,
"KL/std": 14.426389694213867,
"epoch": 0.25968586387434556,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.483942985534668,
"fcm_dpo/q_t": 0.47647592425346375,
"grad_norm": 16.518327713012695,
"learning_rate": 4.6323175183912023e-07,
"logits/chosen": 1.7295470237731934,
"logits/rejected": 1.6208832263946533,
"logps/chosen": -265.591552734375,
"logps/ref_chosen": -271.6752014160156,
"logps/ref_rejected": -221.85577392578125,
"logps/rejected": -225.25601196289062,
"loss": 5.2046,
"margin_dpo/margin_mean": 9.483943939208984,
"margin_dpo/margin_std": 15.952499389648438,
"step": 124
},
{
"KL/chosen_KL_mean": 5.441370010375977,
"KL/mean": 0.8098728656768799,
"KL/rejected_KL_mean": -3.821634292602539,
"KL/std": 14.032330513000488,
"epoch": 0.2617801047120419,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.263002395629883,
"fcm_dpo/q_t": 0.4771071970462799,
"grad_norm": 16.489973068237305,
"learning_rate": 4.6227032831928483e-07,
"logits/chosen": 1.5670815706253052,
"logits/rejected": 1.5696477890014648,
"logps/chosen": -264.95770263671875,
"logps/ref_chosen": -270.3990478515625,
"logps/ref_rejected": -264.16925048828125,
"logps/rejected": -267.99090576171875,
"loss": 5.2188,
"margin_dpo/margin_mean": 9.263002395629883,
"margin_dpo/margin_std": 18.39293670654297,
"step": 125
},
{
"KL/chosen_KL_mean": 6.915998458862305,
"KL/mean": 1.5826067924499512,
"KL/rejected_KL_mean": -3.7507762908935547,
"KL/std": 15.615608215332031,
"epoch": 0.2638743455497382,
"fcm_dpo/beta": 0.010336995124816895,
"fcm_dpo/delta": 0.047238241881132126,
"fcm_dpo/margin": 10.66677474975586,
"fcm_dpo/q_t": 0.4729771614074707,
"grad_norm": 17.296894073486328,
"learning_rate": 4.612975213859487e-07,
"logits/chosen": 1.8302912712097168,
"logits/rejected": 1.8179588317871094,
"logps/chosen": -306.3352966308594,
"logps/ref_chosen": -313.25128173828125,
"logps/ref_rejected": -283.0433349609375,
"logps/rejected": -286.79412841796875,
"loss": 5.1563,
"margin_dpo/margin_mean": 10.66677474975586,
"margin_dpo/margin_std": 18.422670364379883,
"step": 126
},
{
"KL/chosen_KL_mean": 8.044687271118164,
"KL/mean": 1.8930842876434326,
"KL/rejected_KL_mean": -4.258506774902344,
"KL/std": 16.056201934814453,
"epoch": 0.26596858638743454,
"fcm_dpo/beta": 0.010639732703566551,
"fcm_dpo/delta": 0.09435366094112396,
"fcm_dpo/margin": 12.30319881439209,
"fcm_dpo/q_t": 0.46779361367225647,
"grad_norm": 57.42796325683594,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 1.7370554208755493,
"logits/rejected": 1.6627271175384521,
"logps/chosen": -290.519287109375,
"logps/ref_chosen": -298.5639953613281,
"logps/ref_rejected": -274.021728515625,
"logps/rejected": -278.2802429199219,
"loss": 5.0748,
"margin_dpo/margin_mean": 12.303199768066406,
"margin_dpo/margin_std": 18.242170333862305,
"step": 127
},
{
"KL/chosen_KL_mean": 8.76463508605957,
"KL/mean": 1.9856010675430298,
"KL/rejected_KL_mean": -4.793434143066406,
"KL/std": 16.885038375854492,
"epoch": 0.2680628272251309,
"fcm_dpo/beta": 0.011522623710334301,
"fcm_dpo/delta": 0.04720144718885422,
"fcm_dpo/margin": 13.558062553405762,
"fcm_dpo/q_t": 0.4617130160331726,
"grad_norm": 18.26601219177246,
"learning_rate": 4.5931796656116837e-07,
"logits/chosen": 1.786920189857483,
"logits/rejected": 1.8387939929962158,
"logps/chosen": -248.36624145507812,
"logps/ref_chosen": -257.130859375,
"logps/ref_rejected": -247.71636962890625,
"logps/rejected": -252.5098114013672,
"loss": 4.9995,
"margin_dpo/margin_mean": 13.558062553405762,
"margin_dpo/margin_std": 20.108369827270508,
"step": 128
},
{
"KL/chosen_KL_mean": 9.020750045776367,
"KL/mean": 2.7810397148132324,
"KL/rejected_KL_mean": -3.4586639404296875,
"KL/std": 16.992502212524414,
"epoch": 0.27015706806282724,
"fcm_dpo/beta": 0.012070320546627045,
"fcm_dpo/delta": 0.09296167641878128,
"fcm_dpo/margin": 12.47940444946289,
"fcm_dpo/q_t": 0.4633902609348297,
"grad_norm": 19.298423767089844,
"learning_rate": 4.5831132482724193e-07,
"logits/chosen": 1.652014970779419,
"logits/rejected": 1.6925582885742188,
"logps/chosen": -285.9397888183594,
"logps/ref_chosen": -294.9605712890625,
"logps/ref_rejected": -272.6249694824219,
"logps/rejected": -276.0836181640625,
"loss": 5.0286,
"margin_dpo/margin_mean": 12.47940444946289,
"margin_dpo/margin_std": 20.265625,
"step": 129
},
{
"KL/chosen_KL_mean": 7.594112396240234,
"KL/mean": 1.8055014610290527,
"KL/rejected_KL_mean": -3.9831180572509766,
"KL/std": 15.501653671264648,
"epoch": 0.27225130890052357,
"fcm_dpo/beta": 0.012528286315500736,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 11.577238082885742,
"fcm_dpo/q_t": 0.4642757475376129,
"grad_norm": 25.295881271362305,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": 1.7699538469314575,
"logits/rejected": 1.8514341115951538,
"logps/chosen": -268.8481140136719,
"logps/ref_chosen": -276.4422302246094,
"logps/ref_rejected": -290.241455078125,
"logps/rejected": -294.2245788574219,
"loss": 5.0455,
"margin_dpo/margin_mean": 11.577238082885742,
"margin_dpo/margin_std": 19.18193244934082,
"step": 130
},
{
"KL/chosen_KL_mean": 4.91627311706543,
"KL/mean": 0.08451282978057861,
"KL/rejected_KL_mean": -4.747249603271484,
"KL/std": 16.430450439453125,
"epoch": 0.2743455497382199,
"fcm_dpo/beta": 0.012808036059141159,
"fcm_dpo/delta": 0.04387928918004036,
"fcm_dpo/margin": 9.663520812988281,
"fcm_dpo/q_t": 0.47001218795776367,
"grad_norm": 22.026689529418945,
"learning_rate": 4.5626458262912735e-07,
"logits/chosen": 1.614654779434204,
"logits/rejected": 1.6121958494186401,
"logps/chosen": -299.9084167480469,
"logps/ref_chosen": -304.82470703125,
"logps/ref_rejected": -289.9726867675781,
"logps/rejected": -294.7199401855469,
"loss": 5.1431,
"margin_dpo/margin_mean": 9.663520812988281,
"margin_dpo/margin_std": 21.14334487915039,
"step": 131
},
{
"KL/chosen_KL_mean": 7.009241104125977,
"KL/mean": 0.5452959537506104,
"KL/rejected_KL_mean": -5.91864013671875,
"KL/std": 18.429950714111328,
"epoch": 0.2764397905759162,
"fcm_dpo/beta": 0.013177389279007912,
"fcm_dpo/delta": 0.0732303261756897,
"fcm_dpo/margin": 12.927886009216309,
"fcm_dpo/q_t": 0.45908477902412415,
"grad_norm": 22.78095054626465,
"learning_rate": 4.5522459192551166e-07,
"logits/chosen": 1.464105486869812,
"logits/rejected": 1.5743765830993652,
"logps/chosen": -253.30551147460938,
"logps/ref_chosen": -260.31475830078125,
"logps/ref_rejected": -280.93756103515625,
"logps/rejected": -286.856201171875,
"loss": 4.992,
"margin_dpo/margin_mean": 12.927887916564941,
"margin_dpo/margin_std": 23.27849006652832,
"step": 132
},
{
"KL/chosen_KL_mean": 7.630144119262695,
"KL/mean": 0.963181734085083,
"KL/rejected_KL_mean": -5.70378303527832,
"KL/std": 17.05446434020996,
"epoch": 0.27853403141361255,
"fcm_dpo/beta": 0.014581998810172081,
"fcm_dpo/delta": 0.12239662557840347,
"fcm_dpo/margin": 13.333932876586914,
"fcm_dpo/q_t": 0.45295092463493347,
"grad_norm": 22.359704971313477,
"learning_rate": 4.541735956498554e-07,
"logits/chosen": 1.6703652143478394,
"logits/rejected": 1.6475379467010498,
"logps/chosen": -271.4904479980469,
"logps/ref_chosen": -279.12060546875,
"logps/ref_rejected": -247.22982788085938,
"logps/rejected": -252.93360900878906,
"loss": 4.9045,
"margin_dpo/margin_mean": 13.33393383026123,
"margin_dpo/margin_std": 20.930585861206055,
"step": 133
},
{
"KL/chosen_KL_mean": 1.8229942321777344,
"KL/mean": -2.830925703048706,
"KL/rejected_KL_mean": -7.484846115112305,
"KL/std": 17.411767959594727,
"epoch": 0.2806282722513089,
"fcm_dpo/beta": 0.01576686091721058,
"fcm_dpo/delta": 0.08350040018558502,
"fcm_dpo/margin": 9.307854652404785,
"fcm_dpo/q_t": 0.46470966935157776,
"grad_norm": 33.37110137939453,
"learning_rate": 4.5311165016389914e-07,
"logits/chosen": 1.7755942344665527,
"logits/rejected": 1.8094536066055298,
"logps/chosen": -313.78106689453125,
"logps/ref_chosen": -315.6040954589844,
"logps/ref_rejected": -291.45635986328125,
"logps/rejected": -298.9411926269531,
"loss": 5.0808,
"margin_dpo/margin_mean": 9.307854652404785,
"margin_dpo/margin_std": 19.93376350402832,
"step": 134
},
{
"KL/chosen_KL_mean": 5.3419647216796875,
"KL/mean": -1.218588948249817,
"KL/rejected_KL_mean": -7.779151916503906,
"KL/std": 17.860780715942383,
"epoch": 0.28272251308900526,
"fcm_dpo/beta": 0.017457405105233192,
"fcm_dpo/delta": 0.13837897777557373,
"fcm_dpo/margin": 13.12112045288086,
"fcm_dpo/q_t": 0.4456590414047241,
"grad_norm": 27.369003295898438,
"learning_rate": 4.520388124165564e-07,
"logits/chosen": 1.5128746032714844,
"logits/rejected": 1.5043995380401611,
"logps/chosen": -277.531005859375,
"logps/ref_chosen": -282.87298583984375,
"logps/ref_rejected": -265.0682373046875,
"logps/rejected": -272.84735107421875,
"loss": 4.8075,
"margin_dpo/margin_mean": 13.121119499206543,
"margin_dpo/margin_std": 19.0527400970459,
"step": 135
},
{
"KL/chosen_KL_mean": 4.886213302612305,
"KL/mean": -1.2634704113006592,
"KL/rejected_KL_mean": -7.4131622314453125,
"KL/std": 17.349163055419922,
"epoch": 0.2848167539267016,
"fcm_dpo/beta": 0.020285794511437416,
"fcm_dpo/delta": 0.1794380247592926,
"fcm_dpo/margin": 12.299372673034668,
"fcm_dpo/q_t": 0.44218307733535767,
"grad_norm": 34.979557037353516,
"learning_rate": 4.5095513994085974e-07,
"logits/chosen": 1.5496782064437866,
"logits/rejected": 1.7162466049194336,
"logps/chosen": -258.5556945800781,
"logps/ref_chosen": -263.4419250488281,
"logps/ref_rejected": -251.43150329589844,
"logps/rejected": -258.8446350097656,
"loss": 4.8138,
"margin_dpo/margin_mean": 12.299372673034668,
"margin_dpo/margin_std": 21.40203094482422,
"step": 136
},
{
"KL/chosen_KL_mean": 3.0768299102783203,
"KL/mean": -2.479499101638794,
"KL/rejected_KL_mean": -8.035833358764648,
"KL/std": 18.846830368041992,
"epoch": 0.2869109947643979,
"fcm_dpo/beta": 0.02238212712109089,
"fcm_dpo/delta": 0.16478125751018524,
"fcm_dpo/margin": 11.112651824951172,
"fcm_dpo/q_t": 0.44189003109931946,
"grad_norm": 36.73350524902344,
"learning_rate": 4.498606908508753e-07,
"logits/chosen": 1.6065609455108643,
"logits/rejected": 1.597914695739746,
"logps/chosen": -292.19390869140625,
"logps/ref_chosen": -295.2707214355469,
"logps/ref_rejected": -283.9851989746094,
"logps/rejected": -292.02105712890625,
"loss": 4.8242,
"margin_dpo/margin_mean": 11.112653732299805,
"margin_dpo/margin_std": 20.317659378051758,
"step": 137
},
{
"KL/chosen_KL_mean": 1.7046375274658203,
"KL/mean": -3.7809078693389893,
"KL/rejected_KL_mean": -9.266460418701172,
"KL/std": 18.058706283569336,
"epoch": 0.28900523560209423,
"fcm_dpo/beta": 0.026084531098604202,
"fcm_dpo/delta": 0.20727840065956116,
"fcm_dpo/margin": 10.971098899841309,
"fcm_dpo/q_t": 0.4376677870750427,
"grad_norm": 40.71832275390625,
"learning_rate": 4.487555238385862e-07,
"logits/chosen": 1.6645891666412354,
"logits/rejected": 1.663257360458374,
"logps/chosen": -280.8532409667969,
"logps/ref_chosen": -282.5578918457031,
"logps/ref_rejected": -262.4772644042969,
"logps/rejected": -271.74371337890625,
"loss": 4.8585,
"margin_dpo/margin_mean": 10.971098899841309,
"margin_dpo/margin_std": 22.973066329956055,
"step": 138
},
{
"KL/chosen_KL_mean": 2.924623489379883,
"KL/mean": -1.1854338645935059,
"KL/rejected_KL_mean": -5.295494079589844,
"KL/std": 18.88577651977539,
"epoch": 0.29109947643979056,
"fcm_dpo/beta": 0.03053542599081993,
"fcm_dpo/delta": 0.18156927824020386,
"fcm_dpo/margin": 8.220112800598145,
"fcm_dpo/q_t": 0.44587525725364685,
"grad_norm": 49.938262939453125,
"learning_rate": 4.476396981707453e-07,
"logits/chosen": 1.568600058555603,
"logits/rejected": 1.6356700658798218,
"logps/chosen": -260.7261657714844,
"logps/ref_chosen": -263.65081787109375,
"logps/ref_rejected": -266.2420959472656,
"logps/rejected": -271.53759765625,
"loss": 5.1004,
"margin_dpo/margin_mean": 8.220112800598145,
"margin_dpo/margin_std": 23.391185760498047,
"step": 139
},
{
"KL/chosen_KL_mean": 6.302621841430664,
"KL/mean": -1.1070319414138794,
"KL/rejected_KL_mean": -8.516683578491211,
"KL/std": 18.66071319580078,
"epoch": 0.2931937172774869,
"fcm_dpo/beta": 0.03383221849799156,
"fcm_dpo/delta": 0.05534835532307625,
"fcm_dpo/margin": 14.81930923461914,
"fcm_dpo/q_t": 0.39205098152160645,
"grad_norm": 47.0718994140625,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 1.5065391063690186,
"logits/rejected": 1.4494394063949585,
"logps/chosen": -266.0798034667969,
"logps/ref_chosen": -272.3824462890625,
"logps/ref_rejected": -248.96444702148438,
"logps/rejected": -257.48114013671875,
"loss": 4.2848,
"margin_dpo/margin_mean": 14.819307327270508,
"margin_dpo/margin_std": 20.897306442260742,
"step": 140
},
{
"KL/chosen_KL_mean": 3.9792404174804688,
"KL/mean": -3.0592873096466064,
"KL/rejected_KL_mean": -10.097808837890625,
"KL/std": 19.82098388671875,
"epoch": 0.29528795811518327,
"fcm_dpo/beta": 0.03550390154123306,
"fcm_dpo/delta": 0.10487215965986252,
"fcm_dpo/margin": 14.077037811279297,
"fcm_dpo/q_t": 0.393840491771698,
"grad_norm": 75.06885528564453,
"learning_rate": 4.453763107901675e-07,
"logits/chosen": 1.6357208490371704,
"logits/rejected": 1.6846797466278076,
"logps/chosen": -284.0918273925781,
"logps/ref_chosen": -288.071044921875,
"logps/ref_rejected": -274.83477783203125,
"logps/rejected": -284.93255615234375,
"loss": 4.4699,
"margin_dpo/margin_mean": 14.077035903930664,
"margin_dpo/margin_std": 23.960243225097656,
"step": 141
},
{
"KL/chosen_KL_mean": 6.928815841674805,
"KL/mean": 0.08577963709831238,
"KL/rejected_KL_mean": -6.757253646850586,
"KL/std": 19.40366554260254,
"epoch": 0.2973821989528796,
"fcm_dpo/beta": 0.03750036656856537,
"fcm_dpo/delta": 0.029122523963451385,
"fcm_dpo/margin": 13.686077117919922,
"fcm_dpo/q_t": 0.3947056531906128,
"grad_norm": 54.780887603759766,
"learning_rate": 4.4422887045602674e-07,
"logits/chosen": 1.6604958772659302,
"logits/rejected": 1.5612449645996094,
"logps/chosen": -282.06304931640625,
"logps/ref_chosen": -288.9918518066406,
"logps/ref_rejected": -228.26071166992188,
"logps/rejected": -235.01795959472656,
"loss": 4.4883,
"margin_dpo/margin_mean": 13.686078071594238,
"margin_dpo/margin_std": 23.471479415893555,
"step": 142
},
{
"KL/chosen_KL_mean": 6.831939697265625,
"KL/mean": -1.064931869506836,
"KL/rejected_KL_mean": -8.961799621582031,
"KL/std": 20.378860473632812,
"epoch": 0.2994764397905759,
"fcm_dpo/beta": 0.03808540105819702,
"fcm_dpo/delta": -0.003338230773806572,
"fcm_dpo/margin": 15.793725967407227,
"fcm_dpo/q_t": 0.3775142729282379,
"grad_norm": 60.79389953613281,
"learning_rate": 4.4307101421701755e-07,
"logits/chosen": 1.5778990983963013,
"logits/rejected": 1.6198242902755737,
"logps/chosen": -282.6425476074219,
"logps/ref_chosen": -289.4744873046875,
"logps/ref_rejected": -265.130615234375,
"logps/rejected": -274.0924377441406,
"loss": 4.3248,
"margin_dpo/margin_mean": 15.793725967407227,
"margin_dpo/margin_std": 25.55742835998535,
"step": 143
},
{
"KL/chosen_KL_mean": 8.030105590820312,
"KL/mean": 1.6460278034210205,
"KL/rejected_KL_mean": -4.7380523681640625,
"KL/std": 19.515775680541992,
"epoch": 0.30157068062827225,
"fcm_dpo/beta": 0.036904361099004745,
"fcm_dpo/delta": 0.008406475186347961,
"fcm_dpo/margin": 12.768148422241211,
"fcm_dpo/q_t": 0.4033263921737671,
"grad_norm": 57.53997802734375,
"learning_rate": 4.419028041654559e-07,
"logits/chosen": 1.6351606845855713,
"logits/rejected": 1.612158179283142,
"logps/chosen": -282.5562744140625,
"logps/ref_chosen": -290.5863342285156,
"logps/ref_rejected": -263.41046142578125,
"logps/rejected": -268.1485290527344,
"loss": 4.5598,
"margin_dpo/margin_mean": 12.768148422241211,
"margin_dpo/margin_std": 23.036344528198242,
"step": 144
},
{
"KL/chosen_KL_mean": 8.423906326293945,
"KL/mean": 0.10684564709663391,
"KL/rejected_KL_mean": -8.210210800170898,
"KL/std": 18.5434627532959,
"epoch": 0.3036649214659686,
"fcm_dpo/beta": 0.0359405092895031,
"fcm_dpo/delta": -0.07233719527721405,
"fcm_dpo/margin": 16.634119033813477,
"fcm_dpo/q_t": 0.37615370750427246,
"grad_norm": 46.92060470581055,
"learning_rate": 4.4072430294890166e-07,
"logits/chosen": 1.5900194644927979,
"logits/rejected": 1.497979760169983,
"logps/chosen": -256.1163330078125,
"logps/ref_chosen": -264.5402526855469,
"logps/ref_rejected": -214.7958221435547,
"logps/rejected": -223.0060272216797,
"loss": 4.1937,
"margin_dpo/margin_mean": 16.634119033813477,
"margin_dpo/margin_std": 23.260997772216797,
"step": 145
},
{
"KL/chosen_KL_mean": 4.434247970581055,
"KL/mean": -1.973922610282898,
"KL/rejected_KL_mean": -8.382087707519531,
"KL/std": 18.502222061157227,
"epoch": 0.3057591623036649,
"fcm_dpo/beta": 0.03668835386633873,
"fcm_dpo/delta": 0.06929504871368408,
"fcm_dpo/margin": 12.816330909729004,
"fcm_dpo/q_t": 0.4019925892353058,
"grad_norm": 77.25863647460938,
"learning_rate": 4.395355737667985e-07,
"logits/chosen": 1.4586389064788818,
"logits/rejected": 1.5658501386642456,
"logps/chosen": -265.74554443359375,
"logps/ref_chosen": -270.17974853515625,
"logps/ref_rejected": -261.2323913574219,
"logps/rejected": -269.6144714355469,
"loss": 4.5585,
"margin_dpo/margin_mean": 12.816332817077637,
"margin_dpo/margin_std": 22.71735191345215,
"step": 146
},
{
"KL/chosen_KL_mean": 3.596536636352539,
"KL/mean": -1.9176825284957886,
"KL/rejected_KL_mean": -7.431909561157227,
"KL/std": 20.64834213256836,
"epoch": 0.3078534031413613,
"fcm_dpo/beta": 0.03882746770977974,
"fcm_dpo/delta": 0.024022063240408897,
"fcm_dpo/margin": 11.028444290161133,
"fcm_dpo/q_t": 0.41412436962127686,
"grad_norm": 59.746368408203125,
"learning_rate": 4.3833668036708483e-07,
"logits/chosen": 1.4037699699401855,
"logits/rejected": 1.367514729499817,
"logps/chosen": -279.1840515136719,
"logps/ref_chosen": -282.7806091308594,
"logps/ref_rejected": -249.23399353027344,
"logps/rejected": -256.6659240722656,
"loss": 4.9488,
"margin_dpo/margin_mean": 11.028443336486816,
"margin_dpo/margin_std": 25.061433792114258,
"step": 147
},
{
"KL/chosen_KL_mean": 4.780019760131836,
"KL/mean": -1.0210793018341064,
"KL/rejected_KL_mean": -6.822177886962891,
"KL/std": 18.859333038330078,
"epoch": 0.3099476439790576,
"fcm_dpo/beta": 0.03977343067526817,
"fcm_dpo/delta": 0.08418260514736176,
"fcm_dpo/margin": 11.602202415466309,
"fcm_dpo/q_t": 0.40774232149124146,
"grad_norm": 69.44275665283203,
"learning_rate": 4.3712768704277524e-07,
"logits/chosen": 1.4985826015472412,
"logits/rejected": 1.509657621383667,
"logps/chosen": -224.13729858398438,
"logps/ref_chosen": -228.9173126220703,
"logps/ref_rejected": -226.81222534179688,
"logps/rejected": -233.63441467285156,
"loss": 4.7155,
"margin_dpo/margin_mean": 11.602203369140625,
"margin_dpo/margin_std": 23.362445831298828,
"step": 148
},
{
"KL/chosen_KL_mean": 6.897031784057617,
"KL/mean": -0.17732220888137817,
"KL/rejected_KL_mean": -7.251680374145508,
"KL/std": 19.505613327026367,
"epoch": 0.31204188481675393,
"fcm_dpo/beta": 0.04087590053677559,
"fcm_dpo/delta": 0.021851126104593277,
"fcm_dpo/margin": 14.14870548248291,
"fcm_dpo/q_t": 0.3834710717201233,
"grad_norm": 63.1515998840332,
"learning_rate": 4.3590865862851263e-07,
"logits/chosen": 1.524064302444458,
"logits/rejected": 1.4874056577682495,
"logps/chosen": -304.18890380859375,
"logps/ref_chosen": -311.0859375,
"logps/ref_rejected": -303.225341796875,
"logps/rejected": -310.47705078125,
"loss": 4.3311,
"margin_dpo/margin_mean": 14.14870548248291,
"margin_dpo/margin_std": 22.75583267211914,
"step": 149
},
{
"KL/chosen_KL_mean": 5.022129058837891,
"KL/mean": -1.7816307544708252,
"KL/rejected_KL_mean": -8.585397720336914,
"KL/std": 18.76820182800293,
"epoch": 0.31413612565445026,
"fcm_dpo/beta": 0.042426105588674545,
"fcm_dpo/delta": 0.022796878591179848,
"fcm_dpo/margin": 13.607522964477539,
"fcm_dpo/q_t": 0.3835679888725281,
"grad_norm": 65.22611999511719,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": 1.5500192642211914,
"logits/rejected": 1.561482310295105,
"logps/chosen": -265.4613342285156,
"logps/ref_chosen": -270.48345947265625,
"logps/ref_rejected": -269.0491638183594,
"logps/rejected": -277.634521484375,
"loss": 4.3796,
"margin_dpo/margin_mean": 13.607524871826172,
"margin_dpo/margin_std": 22.5236759185791,
"step": 150
},
{
"KL/chosen_KL_mean": 8.34054183959961,
"KL/mean": -1.2480887174606323,
"KL/rejected_KL_mean": -10.836723327636719,
"KL/std": 20.108816146850586,
"epoch": 0.3162303664921466,
"fcm_dpo/beta": 0.03923875465989113,
"fcm_dpo/delta": -0.16742047667503357,
"fcm_dpo/margin": 19.177261352539062,
"fcm_dpo/q_t": 0.3483484983444214,
"grad_norm": 89.0177001953125,
"learning_rate": 4.3344075855595097e-07,
"logits/chosen": 1.5356732606887817,
"logits/rejected": 1.5543012619018555,
"logps/chosen": -279.8251647949219,
"logps/ref_chosen": -288.16571044921875,
"logps/ref_rejected": -264.7412414550781,
"logps/rejected": -275.5779724121094,
"loss": 3.9405,
"margin_dpo/margin_mean": 19.177261352539062,
"margin_dpo/margin_std": 25.098867416381836,
"step": 151
},
{
"KL/chosen_KL_mean": 7.048564910888672,
"KL/mean": -0.2566535472869873,
"KL/rejected_KL_mean": -7.561862945556641,
"KL/std": 20.26637077331543,
"epoch": 0.3183246073298429,
"fcm_dpo/beta": 0.03661992773413658,
"fcm_dpo/delta": -0.01711050420999527,
"fcm_dpo/margin": 14.61042594909668,
"fcm_dpo/q_t": 0.3898727297782898,
"grad_norm": 71.59041595458984,
"learning_rate": 4.3219201924364323e-07,
"logits/chosen": 1.3534401655197144,
"logits/rejected": 1.4261000156402588,
"logps/chosen": -252.88995361328125,
"logps/ref_chosen": -259.9385070800781,
"logps/ref_rejected": -266.2453308105469,
"logps/rejected": -273.8072509765625,
"loss": 4.45,
"margin_dpo/margin_mean": 14.610424995422363,
"margin_dpo/margin_std": 24.417827606201172,
"step": 152
},
{
"KL/chosen_KL_mean": 5.933012008666992,
"KL/mean": -5.1869964599609375,
"KL/rejected_KL_mean": -16.3070125579834,
"KL/std": 21.712013244628906,
"epoch": 0.3204188481675393,
"fcm_dpo/beta": 0.03363032639026642,
"fcm_dpo/delta": -0.16363948583602905,
"fcm_dpo/margin": 22.240018844604492,
"fcm_dpo/q_t": 0.34008270502090454,
"grad_norm": 49.01840591430664,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 1.5906455516815186,
"logits/rejected": 1.6551276445388794,
"logps/chosen": -259.6440124511719,
"logps/ref_chosen": -265.5769958496094,
"logps/ref_rejected": -277.6246032714844,
"logps/rejected": -293.931640625,
"loss": 3.637,
"margin_dpo/margin_mean": 22.240018844604492,
"margin_dpo/margin_std": 23.46230697631836,
"step": 153
},
{
"KL/chosen_KL_mean": 3.613248825073242,
"KL/mean": -4.316822052001953,
"KL/rejected_KL_mean": -12.246877670288086,
"KL/std": 22.21419906616211,
"epoch": 0.3225130890052356,
"fcm_dpo/beta": 0.032934922724962234,
"fcm_dpo/delta": 0.02667096257209778,
"fcm_dpo/margin": 15.860122680664062,
"fcm_dpo/q_t": 0.38882002234458923,
"grad_norm": 52.11634063720703,
"learning_rate": 4.2966529689388064e-07,
"logits/chosen": 1.4579464197158813,
"logits/rejected": 1.4288320541381836,
"logps/chosen": -278.66015625,
"logps/ref_chosen": -282.2734069824219,
"logps/ref_rejected": -261.5135498046875,
"logps/rejected": -273.76043701171875,
"loss": 4.4732,
"margin_dpo/margin_mean": 15.860122680664062,
"margin_dpo/margin_std": 26.72602081298828,
"step": 154
},
{
"KL/chosen_KL_mean": 1.7544536590576172,
"KL/mean": -4.108672142028809,
"KL/rejected_KL_mean": -9.971803665161133,
"KL/std": 20.209108352661133,
"epoch": 0.32460732984293195,
"fcm_dpo/beta": 0.034309010952711105,
"fcm_dpo/delta": 0.07300734519958496,
"fcm_dpo/margin": 11.726253509521484,
"fcm_dpo/q_t": 0.4163626432418823,
"grad_norm": 60.788536071777344,
"learning_rate": 4.2838744935687716e-07,
"logits/chosen": 1.4265573024749756,
"logits/rejected": 1.5197659730911255,
"logps/chosen": -276.9571228027344,
"logps/ref_chosen": -278.7115783691406,
"logps/ref_rejected": -298.0945739746094,
"logps/rejected": -308.06640625,
"loss": 4.8173,
"margin_dpo/margin_mean": 11.7262544631958,
"margin_dpo/margin_std": 24.825592041015625,
"step": 155
},
{
"KL/chosen_KL_mean": 2.966684341430664,
"KL/mean": -6.782113075256348,
"KL/rejected_KL_mean": -16.53091049194336,
"KL/std": 22.180252075195312,
"epoch": 0.3267015706806283,
"fcm_dpo/beta": 0.03426394239068031,
"fcm_dpo/delta": -0.07360708713531494,
"fcm_dpo/margin": 19.497600555419922,
"fcm_dpo/q_t": 0.3636919856071472,
"grad_norm": 49.924827575683594,
"learning_rate": 4.271000354423425e-07,
"logits/chosen": 1.6021648645401,
"logits/rejected": 1.6253923177719116,
"logps/chosen": -265.38446044921875,
"logps/ref_chosen": -268.35113525390625,
"logps/ref_rejected": -241.9660186767578,
"logps/rejected": -258.4969482421875,
"loss": 4.1912,
"margin_dpo/margin_mean": 19.49759864807129,
"margin_dpo/margin_std": 29.38343048095703,
"step": 156
},
{
"KL/chosen_KL_mean": 1.2138385772705078,
"KL/mean": -5.53639554977417,
"KL/rejected_KL_mean": -12.286630630493164,
"KL/std": 21.31039047241211,
"epoch": 0.3287958115183246,
"fcm_dpo/beta": 0.033198121935129166,
"fcm_dpo/delta": 0.017839806154370308,
"fcm_dpo/margin": 13.50045394897461,
"fcm_dpo/q_t": 0.40580445528030396,
"grad_norm": 58.3088264465332,
"learning_rate": 4.258031241903777e-07,
"logits/chosen": 1.3480119705200195,
"logits/rejected": 1.4023487567901611,
"logps/chosen": -264.1897888183594,
"logps/ref_chosen": -265.40362548828125,
"logps/ref_rejected": -257.82440185546875,
"logps/rejected": -270.11102294921875,
"loss": 4.6207,
"margin_dpo/margin_mean": 13.50045394897461,
"margin_dpo/margin_std": 24.77008056640625,
"step": 157
},
{
"KL/chosen_KL_mean": 1.459920883178711,
"KL/mean": -5.981760501861572,
"KL/rejected_KL_mean": -13.423439025878906,
"KL/std": 20.990612030029297,
"epoch": 0.3308900523560209,
"fcm_dpo/beta": 0.0347001887857914,
"fcm_dpo/delta": 0.019363276660442352,
"fcm_dpo/margin": 14.8833646774292,
"fcm_dpo/q_t": 0.39081138372421265,
"grad_norm": 54.962223052978516,
"learning_rate": 4.2449678515039743e-07,
"logits/chosen": 1.6265257596969604,
"logits/rejected": 1.6831825971603394,
"logps/chosen": -279.7023620605469,
"logps/ref_chosen": -281.1622619628906,
"logps/ref_rejected": -268.2899475097656,
"logps/rejected": -281.71337890625,
"loss": 4.4528,
"margin_dpo/margin_mean": 14.883365631103516,
"margin_dpo/margin_std": 24.7315616607666,
"step": 158
},
{
"KL/chosen_KL_mean": 0.3786163330078125,
"KL/mean": -5.6010661125183105,
"KL/rejected_KL_mean": -11.580739974975586,
"KL/std": 22.01244354248047,
"epoch": 0.33298429319371725,
"fcm_dpo/beta": 0.03370116651058197,
"fcm_dpo/delta": -0.017574762925505638,
"fcm_dpo/margin": 11.95935344696045,
"fcm_dpo/q_t": 0.4201620817184448,
"grad_norm": 61.2664794921875,
"learning_rate": 4.2318108837739986e-07,
"logits/chosen": 1.600152611732483,
"logits/rejected": 1.492041826248169,
"logps/chosen": -310.9463806152344,
"logps/ref_chosen": -311.32501220703125,
"logps/ref_rejected": -248.85350036621094,
"logps/rejected": -260.4342346191406,
"loss": 5.0339,
"margin_dpo/margin_mean": 11.95935344696045,
"margin_dpo/margin_std": 29.016633987426758,
"step": 159
},
{
"KL/chosen_KL_mean": 3.298036575317383,
"KL/mean": -5.638082027435303,
"KL/rejected_KL_mean": -14.574199676513672,
"KL/std": 21.566913604736328,
"epoch": 0.33507853403141363,
"fcm_dpo/beta": 0.03293953090906143,
"fcm_dpo/delta": -0.03199426829814911,
"fcm_dpo/margin": 17.872236251831055,
"fcm_dpo/q_t": 0.3772663474082947,
"grad_norm": 53.90055847167969,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": 1.5303587913513184,
"logits/rejected": 1.5337125062942505,
"logps/chosen": -263.8109130859375,
"logps/ref_chosen": -267.10894775390625,
"logps/ref_rejected": -258.66162109375,
"logps/rejected": -273.2358093261719,
"loss": 4.161,
"margin_dpo/margin_mean": 17.872238159179688,
"margin_dpo/margin_std": 24.7597713470459,
"step": 160
},
{
"KL/chosen_KL_mean": 3.1708030700683594,
"KL/mean": -6.072520732879639,
"KL/rejected_KL_mean": -15.315837860107422,
"KL/std": 22.220367431640625,
"epoch": 0.33717277486910996,
"fcm_dpo/beta": 0.033242467790842056,
"fcm_dpo/delta": -0.01598743163049221,
"fcm_dpo/margin": 18.486648559570312,
"fcm_dpo/q_t": 0.37375712394714355,
"grad_norm": 54.25385665893555,
"learning_rate": 4.2052190435769554e-07,
"logits/chosen": 1.8315788507461548,
"logits/rejected": 1.737775206565857,
"logps/chosen": -298.7215881347656,
"logps/ref_chosen": -301.89239501953125,
"logps/ref_rejected": -247.645263671875,
"logps/rejected": -262.9610900878906,
"loss": 4.2261,
"margin_dpo/margin_mean": 18.486648559570312,
"margin_dpo/margin_std": 27.791015625,
"step": 161
},
{
"KL/chosen_KL_mean": 4.625587463378906,
"KL/mean": -3.6662158966064453,
"KL/rejected_KL_mean": -11.95802116394043,
"KL/std": 21.571430206298828,
"epoch": 0.3392670157068063,
"fcm_dpo/beta": 0.03174804151058197,
"fcm_dpo/delta": 0.019307700917124748,
"fcm_dpo/margin": 16.583608627319336,
"fcm_dpo/q_t": 0.3874986171722412,
"grad_norm": 53.21702575683594,
"learning_rate": 4.1917855971495763e-07,
"logits/chosen": 1.5175679922103882,
"logits/rejected": 1.4912818670272827,
"logps/chosen": -283.60748291015625,
"logps/ref_chosen": -288.2330627441406,
"logps/ref_rejected": -252.5148162841797,
"logps/rejected": -264.47283935546875,
"loss": 4.3715,
"margin_dpo/margin_mean": 16.583608627319336,
"margin_dpo/margin_std": 25.756649017333984,
"step": 162
},
{
"KL/chosen_KL_mean": 2.9671707153320312,
"KL/mean": -6.372570991516113,
"KL/rejected_KL_mean": -15.712316513061523,
"KL/std": 20.523710250854492,
"epoch": 0.3413612565445026,
"fcm_dpo/beta": 0.03297853097319603,
"fcm_dpo/delta": -0.0184785183519125,
"fcm_dpo/margin": 18.67947769165039,
"fcm_dpo/q_t": 0.3669191002845764,
"grad_norm": 81.9054183959961,
"learning_rate": 4.1782614253949255e-07,
"logits/chosen": 1.7574456930160522,
"logits/rejected": 1.7493940591812134,
"logps/chosen": -283.796630859375,
"logps/ref_chosen": -286.7638244628906,
"logps/ref_rejected": -262.8302917480469,
"logps/rejected": -278.5425720214844,
"loss": 3.9888,
"margin_dpo/margin_mean": 18.67947769165039,
"margin_dpo/margin_std": 22.76688575744629,
"step": 163
},
{
"KL/chosen_KL_mean": -2.706052780151367,
"KL/mean": -10.915844917297363,
"KL/rejected_KL_mean": -19.125640869140625,
"KL/std": 21.754802703857422,
"epoch": 0.34345549738219894,
"fcm_dpo/beta": 0.032983679324388504,
"fcm_dpo/delta": 0.002447988837957382,
"fcm_dpo/margin": 16.41958236694336,
"fcm_dpo/q_t": 0.3856901526451111,
"grad_norm": 52.84718322753906,
"learning_rate": 4.164647253573289e-07,
"logits/chosen": 1.7213536500930786,
"logits/rejected": 1.680013656616211,
"logps/chosen": -262.9845275878906,
"logps/ref_chosen": -260.2784729003906,
"logps/ref_rejected": -217.10556030273438,
"logps/rejected": -236.231201171875,
"loss": 4.2968,
"margin_dpo/margin_mean": 16.41958236694336,
"margin_dpo/margin_std": 24.709409713745117,
"step": 164
},
{
"KL/chosen_KL_mean": 1.5090866088867188,
"KL/mean": -5.225419521331787,
"KL/rejected_KL_mean": -11.95993423461914,
"KL/std": 20.277297973632812,
"epoch": 0.34554973821989526,
"fcm_dpo/beta": 0.034100648015737534,
"fcm_dpo/delta": 0.061964474618434906,
"fcm_dpo/margin": 13.469009399414062,
"fcm_dpo/q_t": 0.4033600986003876,
"grad_norm": 61.006935119628906,
"learning_rate": 4.1509438117713863e-07,
"logits/chosen": 1.6673741340637207,
"logits/rejected": 1.7128586769104004,
"logps/chosen": -269.04522705078125,
"logps/ref_chosen": -270.5543212890625,
"logps/ref_rejected": -259.80218505859375,
"logps/rejected": -271.7620849609375,
"loss": 4.5383,
"margin_dpo/margin_mean": 13.469008445739746,
"margin_dpo/margin_std": 23.338573455810547,
"step": 165
},
{
"KL/chosen_KL_mean": 0.41426753997802734,
"KL/mean": -5.925065517425537,
"KL/rejected_KL_mean": -12.26440143585205,
"KL/std": 22.263172149658203,
"epoch": 0.34764397905759165,
"fcm_dpo/beta": 0.036087341606616974,
"fcm_dpo/delta": 0.15015417337417603,
"fcm_dpo/margin": 12.678671836853027,
"fcm_dpo/q_t": 0.4126741290092468,
"grad_norm": 66.5831298828125,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 1.5135771036148071,
"logits/rejected": 1.4846245050430298,
"logps/chosen": -269.88018798828125,
"logps/ref_chosen": -270.29443359375,
"logps/ref_rejected": -230.40151977539062,
"logps/rejected": -242.66590881347656,
"loss": 4.9011,
"margin_dpo/margin_mean": 12.678671836853027,
"margin_dpo/margin_std": 28.675046920776367,
"step": 166
},
{
"KL/chosen_KL_mean": 0.8034381866455078,
"KL/mean": -6.434817790985107,
"KL/rejected_KL_mean": -13.673067092895508,
"KL/std": 21.16307830810547,
"epoch": 0.34973821989528797,
"fcm_dpo/beta": 0.03901813179254532,
"fcm_dpo/delta": -0.016899898648262024,
"fcm_dpo/margin": 14.476500511169434,
"fcm_dpo/q_t": 0.38619351387023926,
"grad_norm": 68.85694122314453,
"learning_rate": 4.123272062470633e-07,
"logits/chosen": 1.7194454669952393,
"logits/rejected": 1.6758253574371338,
"logps/chosen": -272.65716552734375,
"logps/ref_chosen": -273.4605712890625,
"logps/ref_rejected": -248.86997985839844,
"logps/rejected": -262.5430603027344,
"loss": 4.4764,
"margin_dpo/margin_mean": 14.476500511169434,
"margin_dpo/margin_std": 25.108524322509766,
"step": 167
},
{
"KL/chosen_KL_mean": 4.459650039672852,
"KL/mean": -4.971283912658691,
"KL/rejected_KL_mean": -14.402233123779297,
"KL/std": 23.291791915893555,
"epoch": 0.3518324607329843,
"fcm_dpo/beta": 0.035170651972293854,
"fcm_dpo/delta": -0.1423185020685196,
"fcm_dpo/margin": 18.861879348754883,
"fcm_dpo/q_t": 0.36044758558273315,
"grad_norm": 66.56237030029297,
"learning_rate": 4.1093052389237174e-07,
"logits/chosen": 1.5125325918197632,
"logits/rejected": 1.4943667650222778,
"logps/chosen": -301.2650451660156,
"logps/ref_chosen": -305.7247009277344,
"logps/ref_rejected": -290.16650390625,
"logps/rejected": -304.5687561035156,
"loss": 4.0711,
"margin_dpo/margin_mean": 18.86187744140625,
"margin_dpo/margin_std": 25.238630294799805,
"step": 168
},
{
"KL/chosen_KL_mean": 3.7789669036865234,
"KL/mean": -7.170663833618164,
"KL/rejected_KL_mean": -18.12029457092285,
"KL/std": 22.5257568359375,
"epoch": 0.3539267015706806,
"fcm_dpo/beta": 0.03189728409051895,
"fcm_dpo/delta": -0.1565765142440796,
"fcm_dpo/margin": 21.899253845214844,
"fcm_dpo/q_t": 0.35491833090782166,
"grad_norm": 41.584468841552734,
"learning_rate": 4.0952521132208267e-07,
"logits/chosen": 1.3543460369110107,
"logits/rejected": 1.3774365186691284,
"logps/chosen": -248.39129638671875,
"logps/ref_chosen": -252.1702880859375,
"logps/ref_rejected": -233.34750366210938,
"logps/rejected": -251.46780395507812,
"loss": 3.866,
"margin_dpo/margin_mean": 21.89925765991211,
"margin_dpo/margin_std": 25.406164169311523,
"step": 169
},
{
"KL/chosen_KL_mean": 4.566259384155273,
"KL/mean": -4.576227188110352,
"KL/rejected_KL_mean": -13.718713760375977,
"KL/std": 21.150253295898438,
"epoch": 0.35602094240837695,
"fcm_dpo/beta": 0.030278891324996948,
"fcm_dpo/delta": 0.049296632409095764,
"fcm_dpo/margin": 18.284975051879883,
"fcm_dpo/q_t": 0.3811044692993164,
"grad_norm": 54.09820556640625,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": 1.4744322299957275,
"logits/rejected": 1.5083649158477783,
"logps/chosen": -297.9121398925781,
"logps/ref_chosen": -302.4783935546875,
"logps/ref_rejected": -259.40777587890625,
"logps/rejected": -273.1264953613281,
"loss": 4.1314,
"margin_dpo/margin_mean": 18.284975051879883,
"margin_dpo/margin_std": 24.58019256591797,
"step": 170
},
{
"KL/chosen_KL_mean": 4.070655822753906,
"KL/mean": -5.274190902709961,
"KL/rejected_KL_mean": -14.619039535522461,
"KL/std": 21.39913558959961,
"epoch": 0.3581151832460733,
"fcm_dpo/beta": 0.03019765205681324,
"fcm_dpo/delta": -0.03337910771369934,
"fcm_dpo/margin": 18.689701080322266,
"fcm_dpo/q_t": 0.3792933225631714,
"grad_norm": 53.6861572265625,
"learning_rate": 4.0668899744407567e-07,
"logits/chosen": 1.3581851720809937,
"logits/rejected": 1.465482234954834,
"logps/chosen": -246.57606506347656,
"logps/ref_chosen": -250.646728515625,
"logps/ref_rejected": -259.1659851074219,
"logps/rejected": -273.7850341796875,
"loss": 4.1222,
"margin_dpo/margin_mean": 18.689701080322266,
"margin_dpo/margin_std": 23.99151611328125,
"step": 171
},
{
"KL/chosen_KL_mean": 4.425405502319336,
"KL/mean": -1.8122390508651733,
"KL/rejected_KL_mean": -8.049894332885742,
"KL/std": 22.412261962890625,
"epoch": 0.36020942408376966,
"fcm_dpo/beta": 0.032538753002882004,
"fcm_dpo/delta": 0.1456703394651413,
"fcm_dpo/margin": 12.475287437438965,
"fcm_dpo/q_t": 0.41149380803108215,
"grad_norm": 57.02654266357422,
"learning_rate": 4.0525824823390043e-07,
"logits/chosen": 1.8623565435409546,
"logits/rejected": 1.883276343345642,
"logps/chosen": -324.59954833984375,
"logps/ref_chosen": -329.02490234375,
"logps/ref_rejected": -293.0215759277344,
"logps/rejected": -301.0714416503906,
"loss": 4.7384,
"margin_dpo/margin_mean": 12.475286483764648,
"margin_dpo/margin_std": 25.173187255859375,
"step": 172
},
{
"KL/chosen_KL_mean": 3.771291732788086,
"KL/mean": -4.03558349609375,
"KL/rejected_KL_mean": -11.84245491027832,
"KL/std": 21.920373916625977,
"epoch": 0.362303664921466,
"fcm_dpo/beta": 0.03458525240421295,
"fcm_dpo/delta": 0.0016709789633750916,
"fcm_dpo/margin": 15.613749504089355,
"fcm_dpo/q_t": 0.3865172863006592,
"grad_norm": 50.34199523925781,
"learning_rate": 4.0381917299505686e-07,
"logits/chosen": 1.5832735300064087,
"logits/rejected": 1.5554523468017578,
"logps/chosen": -286.36932373046875,
"logps/ref_chosen": -290.140625,
"logps/ref_rejected": -271.474853515625,
"logps/rejected": -283.3172912597656,
"loss": 4.3357,
"margin_dpo/margin_mean": 15.613749504089355,
"margin_dpo/margin_std": 24.040199279785156,
"step": 173
},
{
"KL/chosen_KL_mean": 3.915590286254883,
"KL/mean": -5.95166540145874,
"KL/rejected_KL_mean": -15.818925857543945,
"KL/std": 21.474803924560547,
"epoch": 0.3643979057591623,
"fcm_dpo/beta": 0.033181384205818176,
"fcm_dpo/delta": -0.06058121472597122,
"fcm_dpo/margin": 19.734516143798828,
"fcm_dpo/q_t": 0.3638980984687805,
"grad_norm": 54.13778305053711,
"learning_rate": 4.0237184890078243e-07,
"logits/chosen": 1.6420636177062988,
"logits/rejected": 1.568307876586914,
"logps/chosen": -325.39166259765625,
"logps/ref_chosen": -329.3072509765625,
"logps/ref_rejected": -256.6839599609375,
"logps/rejected": -272.50286865234375,
"loss": 3.9372,
"margin_dpo/margin_mean": 19.734516143798828,
"margin_dpo/margin_std": 24.167448043823242,
"step": 174
},
{
"KL/chosen_KL_mean": 1.5434303283691406,
"KL/mean": -6.360366344451904,
"KL/rejected_KL_mean": -14.264167785644531,
"KL/std": 22.346118927001953,
"epoch": 0.36649214659685864,
"fcm_dpo/beta": 0.03351647034287453,
"fcm_dpo/delta": 0.07235424220561981,
"fcm_dpo/margin": 15.807592391967773,
"fcm_dpo/q_t": 0.38753843307495117,
"grad_norm": 66.53447723388672,
"learning_rate": 4.00916353566676e-07,
"logits/chosen": 1.6045417785644531,
"logits/rejected": 1.592195987701416,
"logps/chosen": -274.35882568359375,
"logps/ref_chosen": -275.9023132324219,
"logps/ref_rejected": -281.4163513183594,
"logps/rejected": -295.6805114746094,
"loss": 4.4513,
"margin_dpo/margin_mean": 15.807592391967773,
"margin_dpo/margin_std": 26.658302307128906,
"step": 175
},
{
"KL/chosen_KL_mean": -0.5240001678466797,
"KL/mean": -6.366352081298828,
"KL/rejected_KL_mean": -12.208709716796875,
"KL/std": 21.937408447265625,
"epoch": 0.36858638743455496,
"fcm_dpo/beta": 0.035578951239585876,
"fcm_dpo/delta": 0.00773615762591362,
"fcm_dpo/margin": 11.684707641601562,
"fcm_dpo/q_t": 0.41137951612472534,
"grad_norm": 64.17352294921875,
"learning_rate": 3.994527650465352e-07,
"logits/chosen": 1.6549088954925537,
"logits/rejected": 1.727834939956665,
"logps/chosen": -244.08819580078125,
"logps/ref_chosen": -243.56423950195312,
"logps/ref_rejected": -239.12905883789062,
"logps/rejected": -251.33773803710938,
"loss": 4.9509,
"margin_dpo/margin_mean": 11.684707641601562,
"margin_dpo/margin_std": 26.69280433654785,
"step": 176
},
{
"KL/chosen_KL_mean": -0.5834159851074219,
"KL/mean": -6.346553802490234,
"KL/rejected_KL_mean": -12.109701156616211,
"KL/std": 22.373661041259766,
"epoch": 0.3706806282722513,
"fcm_dpo/beta": 0.035139717161655426,
"fcm_dpo/delta": 0.015670977532863617,
"fcm_dpo/margin": 11.526307106018066,
"fcm_dpo/q_t": 0.4153884947299957,
"grad_norm": 86.48933410644531,
"learning_rate": 3.979811618281705e-07,
"logits/chosen": 1.7052347660064697,
"logits/rejected": 1.6299563646316528,
"logps/chosen": -293.8870849609375,
"logps/ref_chosen": -293.30364990234375,
"logps/ref_rejected": -292.293701171875,
"logps/rejected": -304.4034118652344,
"loss": 5.0891,
"margin_dpo/margin_mean": 11.52630615234375,
"margin_dpo/margin_std": 28.712135314941406,
"step": 177
},
{
"KL/chosen_KL_mean": 2.7671356201171875,
"KL/mean": -6.616493225097656,
"KL/rejected_KL_mean": -16.000125885009766,
"KL/std": 23.80596923828125,
"epoch": 0.37277486910994767,
"fcm_dpo/beta": 0.033524710685014725,
"fcm_dpo/delta": -0.08717905730009079,
"fcm_dpo/margin": 18.767263412475586,
"fcm_dpo/q_t": 0.37235215306282043,
"grad_norm": 50.03008270263672,
"learning_rate": 3.9650162282919654e-07,
"logits/chosen": 1.4354071617126465,
"logits/rejected": 1.439162254333496,
"logps/chosen": -273.3591003417969,
"logps/ref_chosen": -276.126220703125,
"logps/ref_rejected": -240.31179809570312,
"logps/rejected": -256.3119201660156,
"loss": 4.1963,
"margin_dpo/margin_mean": 18.767263412475586,
"margin_dpo/margin_std": 28.032089233398438,
"step": 178
},
{
"KL/chosen_KL_mean": 2.1550731658935547,
"KL/mean": -4.787572860717773,
"KL/rejected_KL_mean": -11.730218887329102,
"KL/std": 22.734161376953125,
"epoch": 0.374869109947644,
"fcm_dpo/beta": 0.03260195255279541,
"fcm_dpo/delta": 0.021392395719885826,
"fcm_dpo/margin": 13.885309219360352,
"fcm_dpo/q_t": 0.40605348348617554,
"grad_norm": 64.03956604003906,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 1.4492707252502441,
"logits/rejected": 1.559048056602478,
"logps/chosen": -247.70736694335938,
"logps/ref_chosen": -249.8624267578125,
"logps/ref_rejected": -271.18603515625,
"logps/rejected": -282.9162902832031,
"loss": 4.6721,
"margin_dpo/margin_mean": 13.885309219360352,
"margin_dpo/margin_std": 26.38463592529297,
"step": 179
},
{
"KL/chosen_KL_mean": 2.6901912689208984,
"KL/mean": -6.234161376953125,
"KL/rejected_KL_mean": -15.158517837524414,
"KL/std": 22.213472366333008,
"epoch": 0.3769633507853403,
"fcm_dpo/beta": 0.03353189304471016,
"fcm_dpo/delta": 0.0003486163914203644,
"fcm_dpo/margin": 17.848716735839844,
"fcm_dpo/q_t": 0.3777158558368683,
"grad_norm": 54.19392395019531,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": 1.423414707183838,
"logits/rejected": 1.4377055168151855,
"logps/chosen": -257.9789123535156,
"logps/ref_chosen": -260.66912841796875,
"logps/ref_rejected": -229.25491333007812,
"logps/rejected": -244.41343688964844,
"loss": 4.1846,
"margin_dpo/margin_mean": 17.848716735839844,
"margin_dpo/margin_std": 25.64480972290039,
"step": 180
},
{
"KL/chosen_KL_mean": 3.9794750213623047,
"KL/mean": -4.322617530822754,
"KL/rejected_KL_mean": -12.624713897705078,
"KL/std": 22.12684440612793,
"epoch": 0.37905759162303665,
"fcm_dpo/beta": 0.0333593524992466,
"fcm_dpo/delta": 0.047833461314439774,
"fcm_dpo/margin": 16.604196548461914,
"fcm_dpo/q_t": 0.38562846183776855,
"grad_norm": 54.91704559326172,
"learning_rate": 3.920161866827889e-07,
"logits/chosen": 1.6338317394256592,
"logits/rejected": 1.6056936979293823,
"logps/chosen": -286.90911865234375,
"logps/ref_chosen": -290.8885803222656,
"logps/ref_rejected": -275.1922302246094,
"logps/rejected": -287.81695556640625,
"loss": 4.4188,
"margin_dpo/margin_mean": 16.60419273376465,
"margin_dpo/margin_std": 27.110607147216797,
"step": 181
},
{
"KL/chosen_KL_mean": 2.504199981689453,
"KL/mean": -7.788280010223389,
"KL/rejected_KL_mean": -18.080759048461914,
"KL/std": 22.3680362701416,
"epoch": 0.381151832460733,
"fcm_dpo/beta": 0.033523909747600555,
"fcm_dpo/delta": -0.09699434787034988,
"fcm_dpo/margin": 20.58496856689453,
"fcm_dpo/q_t": 0.3516695499420166,
"grad_norm": 48.38198471069336,
"learning_rate": 3.90505702185e-07,
"logits/chosen": 1.529535174369812,
"logits/rejected": 1.5425692796707153,
"logps/chosen": -268.0626525878906,
"logps/ref_chosen": -270.56683349609375,
"logps/ref_rejected": -260.3321533203125,
"logps/rejected": -278.41290283203125,
"loss": 3.8327,
"margin_dpo/margin_mean": 20.58496856689453,
"margin_dpo/margin_std": 23.530336380004883,
"step": 182
},
{
"KL/chosen_KL_mean": 3.2648563385009766,
"KL/mean": -5.201539039611816,
"KL/rejected_KL_mean": -13.667932510375977,
"KL/std": 22.122039794921875,
"epoch": 0.3832460732984293,
"fcm_dpo/beta": 0.03249736875295639,
"fcm_dpo/delta": 0.05231393501162529,
"fcm_dpo/margin": 16.932781219482422,
"fcm_dpo/q_t": 0.3850289285182953,
"grad_norm": 52.90995407104492,
"learning_rate": 3.889876827928156e-07,
"logits/chosen": 1.479565978050232,
"logits/rejected": 1.5102664232254028,
"logps/chosen": -257.76312255859375,
"logps/ref_chosen": -261.0279846191406,
"logps/ref_rejected": -240.72361755371094,
"logps/rejected": -254.39157104492188,
"loss": 4.3147,
"margin_dpo/margin_mean": 16.932781219482422,
"margin_dpo/margin_std": 26.4365291595459,
"step": 183
},
{
"KL/chosen_KL_mean": 7.062267303466797,
"KL/mean": -5.364647388458252,
"KL/rejected_KL_mean": -17.791555404663086,
"KL/std": 24.86423110961914,
"epoch": 0.38534031413612563,
"fcm_dpo/beta": 0.03018895350396633,
"fcm_dpo/delta": -0.1675216257572174,
"fcm_dpo/margin": 24.85382652282715,
"fcm_dpo/q_t": 0.3426292836666107,
"grad_norm": 55.49262619018555,
"learning_rate": 3.874622099130087e-07,
"logits/chosen": 1.6249607801437378,
"logits/rejected": 1.5996079444885254,
"logps/chosen": -302.0216064453125,
"logps/ref_chosen": -309.0838317871094,
"logps/ref_rejected": -263.6435241699219,
"logps/rejected": -281.43511962890625,
"loss": 3.7688,
"margin_dpo/margin_mean": 24.85382652282715,
"margin_dpo/margin_std": 29.232576370239258,
"step": 184
},
{
"KL/chosen_KL_mean": 1.7405548095703125,
"KL/mean": -6.731338977813721,
"KL/rejected_KL_mean": -15.203241348266602,
"KL/std": 23.19679832458496,
"epoch": 0.387434554973822,
"fcm_dpo/beta": 0.02973821386694908,
"fcm_dpo/delta": 0.045723918825387955,
"fcm_dpo/margin": 16.943790435791016,
"fcm_dpo/q_t": 0.39501094818115234,
"grad_norm": 51.1673583984375,
"learning_rate": 3.859293653520604e-07,
"logits/chosen": 1.5313293933868408,
"logits/rejected": 1.5998423099517822,
"logps/chosen": -263.9611511230469,
"logps/ref_chosen": -265.70172119140625,
"logps/ref_rejected": -257.5074157714844,
"logps/rejected": -272.7106628417969,
"loss": 4.4361,
"margin_dpo/margin_mean": 16.943788528442383,
"margin_dpo/margin_std": 27.48980140686035,
"step": 185
},
{
"KL/chosen_KL_mean": -0.06787872314453125,
"KL/mean": -8.355184555053711,
"KL/rejected_KL_mean": -16.64249038696289,
"KL/std": 22.052623748779297,
"epoch": 0.38952879581151834,
"fcm_dpo/beta": 0.031189240515232086,
"fcm_dpo/delta": 0.0864519476890564,
"fcm_dpo/margin": 16.57461929321289,
"fcm_dpo/q_t": 0.39101552963256836,
"grad_norm": 57.871768951416016,
"learning_rate": 3.8438923131177237e-07,
"logits/chosen": 1.5475683212280273,
"logits/rejected": 1.5632997751235962,
"logps/chosen": -291.7928466796875,
"logps/ref_chosen": -291.7249755859375,
"logps/ref_rejected": -273.54010009765625,
"logps/rejected": -290.1826171875,
"loss": 4.3362,
"margin_dpo/margin_mean": 16.574621200561523,
"margin_dpo/margin_std": 25.190021514892578,
"step": 186
},
{
"KL/chosen_KL_mean": 0.42089271545410156,
"KL/mean": -8.181295394897461,
"KL/rejected_KL_mean": -16.783475875854492,
"KL/std": 24.009702682495117,
"epoch": 0.39162303664921466,
"fcm_dpo/beta": 0.03242059797048569,
"fcm_dpo/delta": 0.044073861092329025,
"fcm_dpo/margin": 17.204360961914062,
"fcm_dpo/q_t": 0.3884267807006836,
"grad_norm": 65.55802917480469,
"learning_rate": 3.828418903848593e-07,
"logits/chosen": 1.5783560276031494,
"logits/rejected": 1.5967649221420288,
"logps/chosen": -321.2809753417969,
"logps/ref_chosen": -321.70184326171875,
"logps/ref_rejected": -261.8386535644531,
"logps/rejected": -278.62213134765625,
"loss": 4.5248,
"margin_dpo/margin_mean": 17.204364776611328,
"margin_dpo/margin_std": 30.568544387817383,
"step": 187
},
{
"KL/chosen_KL_mean": -1.0099678039550781,
"KL/mean": -9.896193504333496,
"KL/rejected_KL_mean": -18.782432556152344,
"KL/std": 23.633604049682617,
"epoch": 0.393717277486911,
"fcm_dpo/beta": 0.03295998275279999,
"fcm_dpo/delta": 0.013461939990520477,
"fcm_dpo/margin": 17.772464752197266,
"fcm_dpo/q_t": 0.3803802728652954,
"grad_norm": 51.01662826538086,
"learning_rate": 3.812874255505191e-07,
"logits/chosen": 1.4776414632797241,
"logits/rejected": 1.45059335231781,
"logps/chosen": -256.9253845214844,
"logps/ref_chosen": -255.91542053222656,
"logps/ref_rejected": -245.1119842529297,
"logps/rejected": -263.8944091796875,
"loss": 4.4022,
"margin_dpo/margin_mean": 17.772464752197266,
"margin_dpo/margin_std": 29.367900848388672,
"step": 188
},
{
"KL/chosen_KL_mean": 4.5848846435546875,
"KL/mean": -5.746577739715576,
"KL/rejected_KL_mean": -16.078039169311523,
"KL/std": 22.845165252685547,
"epoch": 0.3958115183246073,
"fcm_dpo/beta": 0.03189515694975853,
"fcm_dpo/delta": -0.12400280684232712,
"fcm_dpo/margin": 20.662931442260742,
"fcm_dpo/q_t": 0.3588719069957733,
"grad_norm": 48.38937759399414,
"learning_rate": 3.797259201699833e-07,
"logits/chosen": 1.689395785331726,
"logits/rejected": 1.7200841903686523,
"logps/chosen": -261.4336853027344,
"logps/ref_chosen": -266.0185546875,
"logps/ref_rejected": -272.862548828125,
"logps/rejected": -288.9405822753906,
"loss": 3.8595,
"margin_dpo/margin_mean": 20.662931442260742,
"margin_dpo/margin_std": 23.503387451171875,
"step": 189
},
{
"KL/chosen_KL_mean": 3.0656661987304688,
"KL/mean": -6.925785064697266,
"KL/rejected_KL_mean": -16.917236328125,
"KL/std": 22.68586540222168,
"epoch": 0.39790575916230364,
"fcm_dpo/beta": 0.03057757578790188,
"fcm_dpo/delta": -0.012582660652697086,
"fcm_dpo/margin": 19.982908248901367,
"fcm_dpo/q_t": 0.37064746022224426,
"grad_norm": 48.570960998535156,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": 1.4705171585083008,
"logits/rejected": 1.4011236429214478,
"logps/chosen": -247.08529663085938,
"logps/ref_chosen": -250.15098571777344,
"logps/ref_rejected": -223.34393310546875,
"logps/rejected": -240.26116943359375,
"loss": 4.1479,
"margin_dpo/margin_mean": 19.982908248901367,
"margin_dpo/margin_std": 28.228897094726562,
"step": 190
},
{
"KL/chosen_KL_mean": 3.550985336303711,
"KL/mean": -4.874011516571045,
"KL/rejected_KL_mean": -13.299015045166016,
"KL/std": 23.36663055419922,
"epoch": 0.4,
"fcm_dpo/beta": 0.030532926321029663,
"fcm_dpo/delta": 0.032537586987018585,
"fcm_dpo/margin": 16.849985122680664,
"fcm_dpo/q_t": 0.39333850145339966,
"grad_norm": 63.48810958862305,
"learning_rate": 3.765821230985757e-07,
"logits/chosen": 1.395258903503418,
"logits/rejected": 1.4522275924682617,
"logps/chosen": -234.8408203125,
"logps/ref_chosen": -238.3917999267578,
"logps/ref_rejected": -243.70272827148438,
"logps/rejected": -257.0017395019531,
"loss": 4.4708,
"margin_dpo/margin_mean": 16.84998321533203,
"margin_dpo/margin_std": 28.85344696044922,
"step": 191
},
{
"KL/chosen_KL_mean": 0.2355213165283203,
"KL/mean": -7.245480060577393,
"KL/rejected_KL_mean": -14.726480484008789,
"KL/std": 22.562896728515625,
"epoch": 0.40209424083769635,
"fcm_dpo/beta": 0.03104531019926071,
"fcm_dpo/delta": 0.005957015790045261,
"fcm_dpo/margin": 14.96200180053711,
"fcm_dpo/q_t": 0.40129604935646057,
"grad_norm": 58.2281379699707,
"learning_rate": 3.75e-07,
"logits/chosen": 1.4725991487503052,
"logits/rejected": 1.5391660928726196,
"logps/chosen": -256.4014587402344,
"logps/ref_chosen": -256.6369934082031,
"logps/ref_rejected": -256.8894348144531,
"logps/rejected": -271.61590576171875,
"loss": 4.6748,
"margin_dpo/margin_mean": 14.962002754211426,
"margin_dpo/margin_std": 28.23845100402832,
"step": 192
},
{
"KL/chosen_KL_mean": -1.5852489471435547,
"KL/mean": -9.660032272338867,
"KL/rejected_KL_mean": -17.73480796813965,
"KL/std": 24.343469619750977,
"epoch": 0.4041884816753927,
"fcm_dpo/beta": 0.03241005539894104,
"fcm_dpo/delta": 0.08015380799770355,
"fcm_dpo/margin": 16.149559020996094,
"fcm_dpo/q_t": 0.3922501802444458,
"grad_norm": 71.72376251220703,
"learning_rate": 3.734111735307796e-07,
"logits/chosen": 1.6393442153930664,
"logits/rejected": 1.7344868183135986,
"logps/chosen": -274.3849182128906,
"logps/ref_chosen": -272.7996826171875,
"logps/ref_rejected": -289.2391052246094,
"logps/rejected": -306.973876953125,
"loss": 4.463,
"margin_dpo/margin_mean": 16.149559020996094,
"margin_dpo/margin_std": 27.722713470458984,
"step": 193
},
{
"KL/chosen_KL_mean": -0.7121067047119141,
"KL/mean": -6.904905319213867,
"KL/rejected_KL_mean": -13.097705841064453,
"KL/std": 24.412935256958008,
"epoch": 0.406282722513089,
"fcm_dpo/beta": 0.03421565517783165,
"fcm_dpo/delta": 0.06331188976764679,
"fcm_dpo/margin": 12.385600090026855,
"fcm_dpo/q_t": 0.4166790843009949,
"grad_norm": 57.3293571472168,
"learning_rate": 3.7181572889485623e-07,
"logits/chosen": 1.179120659828186,
"logits/rejected": 1.2023894786834717,
"logps/chosen": -282.0211486816406,
"logps/ref_chosen": -281.30902099609375,
"logps/ref_rejected": -246.3243408203125,
"logps/rejected": -259.42205810546875,
"loss": 4.8153,
"margin_dpo/margin_mean": 12.385601043701172,
"margin_dpo/margin_std": 26.814218521118164,
"step": 194
},
{
"KL/chosen_KL_mean": -1.6187248229980469,
"KL/mean": -7.541670799255371,
"KL/rejected_KL_mean": -13.464611053466797,
"KL/std": 24.115158081054688,
"epoch": 0.4083769633507853,
"fcm_dpo/beta": 0.0353362001478672,
"fcm_dpo/delta": 0.02736041322350502,
"fcm_dpo/margin": 11.845905303955078,
"fcm_dpo/q_t": 0.4158301055431366,
"grad_norm": 78.81611633300781,
"learning_rate": 3.7021375165108377e-07,
"logits/chosen": 1.621145248413086,
"logits/rejected": 1.7222257852554321,
"logps/chosen": -268.476806640625,
"logps/ref_chosen": -266.8581237792969,
"logps/ref_rejected": -276.0855712890625,
"logps/rejected": -289.5501708984375,
"loss": 4.9263,
"margin_dpo/margin_mean": 11.845905303955078,
"margin_dpo/margin_std": 27.449527740478516,
"step": 195
},
{
"KL/chosen_KL_mean": 3.640165328979492,
"KL/mean": -5.483175277709961,
"KL/rejected_KL_mean": -14.606521606445312,
"KL/std": 22.341976165771484,
"epoch": 0.41047120418848165,
"fcm_dpo/beta": 0.03500073403120041,
"fcm_dpo/delta": -0.04595329985022545,
"fcm_dpo/margin": 18.246681213378906,
"fcm_dpo/q_t": 0.3699175715446472,
"grad_norm": 57.45576477050781,
"learning_rate": 3.6860532770864005e-07,
"logits/chosen": 1.2707561254501343,
"logits/rejected": 1.3572243452072144,
"logps/chosen": -263.35479736328125,
"logps/ref_chosen": -266.99493408203125,
"logps/ref_rejected": -248.60513305664062,
"logps/rejected": -263.211669921875,
"loss": 4.1055,
"margin_dpo/margin_mean": 18.246681213378906,
"margin_dpo/margin_std": 24.88081932067871,
"step": 196
},
{
"KL/chosen_KL_mean": 5.81505012512207,
"KL/mean": -6.082913398742676,
"KL/rejected_KL_mean": -17.98088264465332,
"KL/std": 24.73003387451172,
"epoch": 0.41256544502617803,
"fcm_dpo/beta": 0.03207242116332054,
"fcm_dpo/delta": -0.2244684398174286,
"fcm_dpo/margin": 23.795928955078125,
"fcm_dpo/q_t": 0.3399311900138855,
"grad_norm": 54.1199836730957,
"learning_rate": 3.6699054332241985e-07,
"logits/chosen": 1.4284682273864746,
"logits/rejected": 1.386784315109253,
"logps/chosen": -297.17474365234375,
"logps/ref_chosen": -302.98980712890625,
"logps/ref_rejected": -262.8002624511719,
"logps/rejected": -280.7811279296875,
"loss": 3.7765,
"margin_dpo/margin_mean": 23.795928955078125,
"margin_dpo/margin_std": 26.546613693237305,
"step": 197
},
{
"KL/chosen_KL_mean": 3.993558883666992,
"KL/mean": -5.0717973709106445,
"KL/rejected_KL_mean": -14.137165069580078,
"KL/std": 24.779373168945312,
"epoch": 0.41465968586387436,
"fcm_dpo/beta": 0.028974642977118492,
"fcm_dpo/delta": 0.0227762870490551,
"fcm_dpo/margin": 18.130712509155273,
"fcm_dpo/q_t": 0.38926345109939575,
"grad_norm": 48.55694580078125,
"learning_rate": 3.653694850884091e-07,
"logits/chosen": 1.6023445129394531,
"logits/rejected": 1.686340570449829,
"logps/chosen": -292.2182312011719,
"logps/ref_chosen": -296.2117614746094,
"logps/ref_rejected": -289.8357238769531,
"logps/rejected": -303.972900390625,
"loss": 4.4946,
"margin_dpo/margin_mean": 18.130712509155273,
"margin_dpo/margin_std": 31.369131088256836,
"step": 198
},
{
"KL/chosen_KL_mean": 3.8633079528808594,
"KL/mean": -5.587414741516113,
"KL/rejected_KL_mean": -15.03813362121582,
"KL/std": 23.215652465820312,
"epoch": 0.4167539267015707,
"fcm_dpo/beta": 0.02959287539124489,
"fcm_dpo/delta": -0.016490591689944267,
"fcm_dpo/margin": 18.901432037353516,
"fcm_dpo/q_t": 0.3825532793998718,
"grad_norm": 49.97153854370117,
"learning_rate": 3.6374223993904124e-07,
"logits/chosen": 1.3078348636627197,
"logits/rejected": 1.2245980501174927,
"logps/chosen": -251.4293212890625,
"logps/ref_chosen": -255.29263305664062,
"logps/ref_rejected": -212.69761657714844,
"logps/rejected": -227.73574829101562,
"loss": 4.2515,
"margin_dpo/margin_mean": 18.901432037353516,
"margin_dpo/margin_std": 28.002967834472656,
"step": 199
},
{
"KL/chosen_KL_mean": 5.087657928466797,
"KL/mean": -3.0331709384918213,
"KL/rejected_KL_mean": -11.153993606567383,
"KL/std": 24.184293746948242,
"epoch": 0.418848167539267,
"fcm_dpo/beta": 0.03048442304134369,
"fcm_dpo/delta": 0.10826665163040161,
"fcm_dpo/margin": 16.24165916442871,
"fcm_dpo/q_t": 0.40207067131996155,
"grad_norm": 62.7784538269043,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": 1.7090070247650146,
"logits/rejected": 1.6965177059173584,
"logps/chosen": -308.4487609863281,
"logps/ref_chosen": -313.5364074707031,
"logps/ref_rejected": -274.349609375,
"logps/rejected": -285.5035400390625,
"loss": 4.7548,
"margin_dpo/margin_mean": 16.241661071777344,
"margin_dpo/margin_std": 33.140193939208984,
"step": 200
},
{
"KL/chosen_KL_mean": 2.934652328491211,
"KL/mean": -6.4366374015808105,
"KL/rejected_KL_mean": -15.807924270629883,
"KL/std": 24.043773651123047,
"epoch": 0.42094240837696334,
"fcm_dpo/beta": 0.03153421729803085,
"fcm_dpo/delta": -0.04986376315355301,
"fcm_dpo/margin": 18.742576599121094,
"fcm_dpo/q_t": 0.37566325068473816,
"grad_norm": 48.46647262573242,
"learning_rate": 3.604695382782159e-07,
"logits/chosen": 1.3615880012512207,
"logits/rejected": 1.3875739574432373,
"logps/chosen": -259.77252197265625,
"logps/ref_chosen": -262.7071838378906,
"logps/ref_rejected": -243.85879516601562,
"logps/rejected": -259.6667175292969,
"loss": 4.21,
"margin_dpo/margin_mean": 18.742576599121094,
"margin_dpo/margin_std": 26.73308563232422,
"step": 201
},
{
"KL/chosen_KL_mean": 1.704813003540039,
"KL/mean": -7.445696830749512,
"KL/rejected_KL_mean": -16.596208572387695,
"KL/std": 24.027362823486328,
"epoch": 0.42303664921465967,
"fcm_dpo/beta": 0.030500078573822975,
"fcm_dpo/delta": -0.014350108802318573,
"fcm_dpo/margin": 18.301023483276367,
"fcm_dpo/q_t": 0.3851291239261627,
"grad_norm": 65.09416961669922,
"learning_rate": 3.588242572718162e-07,
"logits/chosen": 1.4482910633087158,
"logits/rejected": 1.4086792469024658,
"logps/chosen": -262.5819396972656,
"logps/ref_chosen": -264.2867431640625,
"logps/ref_rejected": -231.6894989013672,
"logps/rejected": -248.2857208251953,
"loss": 4.4414,
"margin_dpo/margin_mean": 18.301021575927734,
"margin_dpo/margin_std": 30.050230026245117,
"step": 202
},
{
"KL/chosen_KL_mean": 1.4567317962646484,
"KL/mean": -5.678568363189697,
"KL/rejected_KL_mean": -12.81386947631836,
"KL/std": 23.273643493652344,
"epoch": 0.42513089005235605,
"fcm_dpo/beta": 0.031557291746139526,
"fcm_dpo/delta": 0.10629908740520477,
"fcm_dpo/margin": 14.270602226257324,
"fcm_dpo/q_t": 0.4037012755870819,
"grad_norm": 58.54388427734375,
"learning_rate": 3.571731403507635e-07,
"logits/chosen": 1.5311411619186401,
"logits/rejected": 1.4974974393844604,
"logps/chosen": -269.82745361328125,
"logps/ref_chosen": -271.2842102050781,
"logps/ref_rejected": -247.95462036132812,
"logps/rejected": -260.76849365234375,
"loss": 4.4926,
"margin_dpo/margin_mean": 14.27060317993164,
"margin_dpo/margin_std": 23.92316246032715,
"step": 203
},
{
"KL/chosen_KL_mean": 4.097650527954102,
"KL/mean": -6.428679466247559,
"KL/rejected_KL_mean": -16.954998016357422,
"KL/std": 24.10330581665039,
"epoch": 0.4272251308900524,
"fcm_dpo/beta": 0.031438861042261124,
"fcm_dpo/delta": -0.06763069331645966,
"fcm_dpo/margin": 21.05264663696289,
"fcm_dpo/q_t": 0.3640082776546478,
"grad_norm": 48.709930419921875,
"learning_rate": 3.5551627605944746e-07,
"logits/chosen": 1.4213337898254395,
"logits/rejected": 1.3685266971588135,
"logps/chosen": -306.0328063964844,
"logps/ref_chosen": -310.13043212890625,
"logps/ref_rejected": -267.3905334472656,
"logps/rejected": -284.34552001953125,
"loss": 3.9993,
"margin_dpo/margin_mean": 21.05264663696289,
"margin_dpo/margin_std": 28.11899185180664,
"step": 204
},
{
"KL/chosen_KL_mean": 5.760053634643555,
"KL/mean": -4.147031784057617,
"KL/rejected_KL_mean": -14.054115295410156,
"KL/std": 23.153827667236328,
"epoch": 0.4293193717277487,
"fcm_dpo/beta": 0.031499359756708145,
"fcm_dpo/delta": -0.02703755721449852,
"fcm_dpo/margin": 19.81418228149414,
"fcm_dpo/q_t": 0.3702598810195923,
"grad_norm": 52.241336822509766,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 1.4928886890411377,
"logits/rejected": 1.487642765045166,
"logps/chosen": -271.4006042480469,
"logps/ref_chosen": -277.16064453125,
"logps/ref_rejected": -292.91815185546875,
"logps/rejected": -306.9722900390625,
"loss": 4.1179,
"margin_dpo/margin_mean": 19.814180374145508,
"margin_dpo/margin_std": 26.955835342407227,
"step": 205
},
{
"KL/chosen_KL_mean": 5.546255111694336,
"KL/mean": -3.0547800064086914,
"KL/rejected_KL_mean": -11.655815124511719,
"KL/std": 22.979106903076172,
"epoch": 0.431413612565445,
"fcm_dpo/beta": 0.030934946611523628,
"fcm_dpo/delta": 0.06948360800743103,
"fcm_dpo/margin": 17.202072143554688,
"fcm_dpo/q_t": 0.3871549367904663,
"grad_norm": 49.26683807373047,
"learning_rate": 3.5218566107988867e-07,
"logits/chosen": 1.606423020362854,
"logits/rejected": 1.7115347385406494,
"logps/chosen": -281.236328125,
"logps/ref_chosen": -286.7825927734375,
"logps/ref_rejected": -289.08270263671875,
"logps/rejected": -300.738525390625,
"loss": 4.2983,
"margin_dpo/margin_mean": 17.202072143554688,
"margin_dpo/margin_std": 25.05809783935547,
"step": 206
},
{
"KL/chosen_KL_mean": 3.277751922607422,
"KL/mean": -4.28781270980835,
"KL/rejected_KL_mean": -11.853385925292969,
"KL/std": 24.55677604675293,
"epoch": 0.43350785340314135,
"fcm_dpo/beta": 0.03269320726394653,
"fcm_dpo/delta": 0.04207862913608551,
"fcm_dpo/margin": 15.13114070892334,
"fcm_dpo/q_t": 0.40015873312950134,
"grad_norm": 55.763126373291016,
"learning_rate": 3.505120890024195e-07,
"logits/chosen": 1.4150789976119995,
"logits/rejected": 1.5169289112091064,
"logps/chosen": -263.2544860839844,
"logps/ref_chosen": -266.5322570800781,
"logps/ref_rejected": -275.20672607421875,
"logps/rejected": -287.06011962890625,
"loss": 4.774,
"margin_dpo/margin_mean": 15.13114070892334,
"margin_dpo/margin_std": 31.045963287353516,
"step": 207
},
{
"KL/chosen_KL_mean": 3.792125701904297,
"KL/mean": -5.186357498168945,
"KL/rejected_KL_mean": -14.164838790893555,
"KL/std": 23.937915802001953,
"epoch": 0.4356020942408377,
"fcm_dpo/beta": 0.03304613381624222,
"fcm_dpo/delta": -0.04791893810033798,
"fcm_dpo/margin": 17.956958770751953,
"fcm_dpo/q_t": 0.37966203689575195,
"grad_norm": 57.23706817626953,
"learning_rate": 3.4883312676665534e-07,
"logits/chosen": 1.6336067914962769,
"logits/rejected": 1.6375012397766113,
"logps/chosen": -269.00921630859375,
"logps/ref_chosen": -272.80133056640625,
"logps/ref_rejected": -288.4167785644531,
"logps/rejected": -302.58160400390625,
"loss": 4.2962,
"margin_dpo/margin_mean": 17.956960678100586,
"margin_dpo/margin_std": 28.082860946655273,
"step": 208
},
{
"KL/chosen_KL_mean": 1.9714374542236328,
"KL/mean": -5.760955333709717,
"KL/rejected_KL_mean": -13.493354797363281,
"KL/std": 23.7287540435791,
"epoch": 0.437696335078534,
"fcm_dpo/beta": 0.03246966749429703,
"fcm_dpo/delta": 0.029187699779868126,
"fcm_dpo/margin": 15.46480941772461,
"fcm_dpo/q_t": 0.3972010016441345,
"grad_norm": 54.213714599609375,
"learning_rate": 3.4714886441024573e-07,
"logits/chosen": 1.59073805809021,
"logits/rejected": 1.4431016445159912,
"logps/chosen": -308.7079772949219,
"logps/ref_chosen": -310.679443359375,
"logps/ref_rejected": -259.3792419433594,
"logps/rejected": -272.8725891113281,
"loss": 4.55,
"margin_dpo/margin_mean": 15.464810371398926,
"margin_dpo/margin_std": 26.98141860961914,
"step": 209
},
{
"KL/chosen_KL_mean": 4.238682746887207,
"KL/mean": -4.264247894287109,
"KL/rejected_KL_mean": -12.767183303833008,
"KL/std": 24.33123207092285,
"epoch": 0.4397905759162304,
"fcm_dpo/beta": 0.03341270983219147,
"fcm_dpo/delta": 0.030814504250884056,
"fcm_dpo/margin": 17.005861282348633,
"fcm_dpo/q_t": 0.3825717568397522,
"grad_norm": 50.88881301879883,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": 1.312711238861084,
"logits/rejected": 1.3513705730438232,
"logps/chosen": -270.7294616699219,
"logps/ref_chosen": -274.9681701660156,
"logps/ref_rejected": -283.346923828125,
"logps/rejected": -296.1141357421875,
"loss": 4.3537,
"margin_dpo/margin_mean": 17.005861282348633,
"margin_dpo/margin_std": 27.453842163085938,
"step": 210
},
{
"KL/chosen_KL_mean": 7.164724349975586,
"KL/mean": -4.725644588470459,
"KL/rejected_KL_mean": -16.61601448059082,
"KL/std": 24.55815315246582,
"epoch": 0.4418848167539267,
"fcm_dpo/beta": 0.03139013797044754,
"fcm_dpo/delta": -0.16069211065769196,
"fcm_dpo/margin": 23.780738830566406,
"fcm_dpo/q_t": 0.3442986011505127,
"grad_norm": 44.99844741821289,
"learning_rate": 3.4376480090239047e-07,
"logits/chosen": 1.4098219871520996,
"logits/rejected": 1.3897714614868164,
"logps/chosen": -259.1148681640625,
"logps/ref_chosen": -266.2795715332031,
"logps/ref_rejected": -249.7120819091797,
"logps/rejected": -266.328125,
"loss": 3.7348,
"margin_dpo/margin_mean": 23.78073501586914,
"margin_dpo/margin_std": 26.240310668945312,
"step": 211
},
{
"KL/chosen_KL_mean": 5.324275970458984,
"KL/mean": -2.900439739227295,
"KL/rejected_KL_mean": -11.125146865844727,
"KL/std": 22.97644805908203,
"epoch": 0.44397905759162304,
"fcm_dpo/beta": 0.029926881194114685,
"fcm_dpo/delta": 0.05482662841677666,
"fcm_dpo/margin": 16.449420928955078,
"fcm_dpo/q_t": 0.3944656550884247,
"grad_norm": 44.68327331542969,
"learning_rate": 3.4206518122800055e-07,
"logits/chosen": 1.4912159442901611,
"logits/rejected": 1.5235073566436768,
"logps/chosen": -259.42852783203125,
"logps/ref_chosen": -264.7528076171875,
"logps/ref_rejected": -241.02842712402344,
"logps/rejected": -252.153564453125,
"loss": 4.4221,
"margin_dpo/margin_mean": 16.449420928955078,
"margin_dpo/margin_std": 26.83086585998535,
"step": 212
},
{
"KL/chosen_KL_mean": 3.322385787963867,
"KL/mean": -4.5970659255981445,
"KL/rejected_KL_mean": -12.51651382446289,
"KL/std": 23.645183563232422,
"epoch": 0.44607329842931936,
"fcm_dpo/beta": 0.031889475882053375,
"fcm_dpo/delta": 0.09982357919216156,
"fcm_dpo/margin": 15.838902473449707,
"fcm_dpo/q_t": 0.3998870551586151,
"grad_norm": 55.86567306518555,
"learning_rate": 3.403606243773448e-07,
"logits/chosen": 1.6490623950958252,
"logits/rejected": 1.7675503492355347,
"logps/chosen": -290.81622314453125,
"logps/ref_chosen": -294.13861083984375,
"logps/ref_rejected": -303.11669921875,
"logps/rejected": -315.63323974609375,
"loss": 4.4696,
"margin_dpo/margin_mean": 15.838902473449707,
"margin_dpo/margin_std": 27.366634368896484,
"step": 213
},
{
"KL/chosen_KL_mean": -1.0719566345214844,
"KL/mean": -9.300503730773926,
"KL/rejected_KL_mean": -17.529054641723633,
"KL/std": 24.037235260009766,
"epoch": 0.4481675392670157,
"fcm_dpo/beta": 0.03453436866402626,
"fcm_dpo/delta": 0.03112040087580681,
"fcm_dpo/margin": 16.457096099853516,
"fcm_dpo/q_t": 0.3864825665950775,
"grad_norm": 65.27860260009766,
"learning_rate": 3.3865122176063385e-07,
"logits/chosen": 1.5908783674240112,
"logits/rejected": 1.7395843267440796,
"logps/chosen": -267.6487121582031,
"logps/ref_chosen": -266.5767517089844,
"logps/ref_rejected": -291.2264709472656,
"logps/rejected": -308.75555419921875,
"loss": 4.403,
"margin_dpo/margin_mean": 16.457094192504883,
"margin_dpo/margin_std": 27.721385955810547,
"step": 214
},
{
"KL/chosen_KL_mean": 1.1925544738769531,
"KL/mean": -6.978704929351807,
"KL/rejected_KL_mean": -15.149971008300781,
"KL/std": 25.942358016967773,
"epoch": 0.450261780104712,
"fcm_dpo/beta": 0.03314455598592758,
"fcm_dpo/delta": -0.054589588195085526,
"fcm_dpo/margin": 16.3425235748291,
"fcm_dpo/q_t": 0.3942345380783081,
"grad_norm": 55.492122650146484,
"learning_rate": 3.3693706504794243e-07,
"logits/chosen": 1.6796056032180786,
"logits/rejected": 1.6907001733779907,
"logps/chosen": -283.84820556640625,
"logps/ref_chosen": -285.040771484375,
"logps/ref_rejected": -258.0254821777344,
"logps/rejected": -273.1754455566406,
"loss": 4.5728,
"margin_dpo/margin_mean": 16.3425235748291,
"margin_dpo/margin_std": 29.976999282836914,
"step": 215
},
{
"KL/chosen_KL_mean": 3.9657115936279297,
"KL/mean": -5.047689437866211,
"KL/rejected_KL_mean": -14.06108283996582,
"KL/std": 24.962993621826172,
"epoch": 0.4523560209424084,
"fcm_dpo/beta": 0.03226002678275108,
"fcm_dpo/delta": 0.018793560564517975,
"fcm_dpo/margin": 18.02680015563965,
"fcm_dpo/q_t": 0.37889283895492554,
"grad_norm": 59.685245513916016,
"learning_rate": 3.3521824616429284e-07,
"logits/chosen": 1.8646385669708252,
"logits/rejected": 1.8905611038208008,
"logps/chosen": -328.7856750488281,
"logps/ref_chosen": -332.7513732910156,
"logps/ref_rejected": -302.9167785644531,
"logps/rejected": -316.9778137207031,
"loss": 4.3924,
"margin_dpo/margin_mean": 18.02680206298828,
"margin_dpo/margin_std": 29.033185958862305,
"step": 216
},
{
"KL/chosen_KL_mean": 0.4787712097167969,
"KL/mean": -11.578567504882812,
"KL/rejected_KL_mean": -23.635900497436523,
"KL/std": 26.74181365966797,
"epoch": 0.4544502617801047,
"fcm_dpo/beta": 0.03131940960884094,
"fcm_dpo/delta": -0.1656835675239563,
"fcm_dpo/margin": 24.114669799804688,
"fcm_dpo/q_t": 0.3509507179260254,
"grad_norm": 47.78200149536133,
"learning_rate": 3.334948572847253e-07,
"logits/chosen": 1.795779824256897,
"logits/rejected": 1.7963271141052246,
"logps/chosen": -285.9891662597656,
"logps/ref_chosen": -286.46795654296875,
"logps/ref_rejected": -253.99298095703125,
"logps/rejected": -277.6288757324219,
"loss": 3.9438,
"margin_dpo/margin_mean": 24.114669799804688,
"margin_dpo/margin_std": 32.39649200439453,
"step": 217
},
{
"KL/chosen_KL_mean": 4.0630035400390625,
"KL/mean": -5.699796676635742,
"KL/rejected_KL_mean": -15.462593078613281,
"KL/std": 25.529905319213867,
"epoch": 0.45654450261780105,
"fcm_dpo/beta": 0.02972830832004547,
"fcm_dpo/delta": 0.019646622240543365,
"fcm_dpo/margin": 19.525588989257812,
"fcm_dpo/q_t": 0.3794595003128052,
"grad_norm": 48.438018798828125,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 1.6519176959991455,
"logits/rejected": 1.7010029554367065,
"logps/chosen": -307.3155212402344,
"logps/ref_chosen": -311.3785400390625,
"logps/ref_rejected": -290.3092956542969,
"logps/rejected": -305.77191162109375,
"loss": 4.2268,
"margin_dpo/margin_mean": 19.525588989257812,
"margin_dpo/margin_std": 29.03296661376953,
"step": 218
},
{
"KL/chosen_KL_mean": 3.8179988861083984,
"KL/mean": -6.301147937774658,
"KL/rejected_KL_mean": -16.42029571533203,
"KL/std": 23.378128051757812,
"epoch": 0.4586387434554974,
"fcm_dpo/beta": 0.02980736270546913,
"fcm_dpo/delta": -0.006483299657702446,
"fcm_dpo/margin": 20.238300323486328,
"fcm_dpo/q_t": 0.37255263328552246,
"grad_norm": 42.6064338684082,
"learning_rate": 3.300347394584172e-07,
"logits/chosen": 1.4093046188354492,
"logits/rejected": 1.4627782106399536,
"logps/chosen": -255.51490783691406,
"logps/ref_chosen": -259.33294677734375,
"logps/ref_rejected": -261.7661437988281,
"logps/rejected": -278.1864013671875,
"loss": 4.1426,
"margin_dpo/margin_mean": 20.23830223083496,
"margin_dpo/margin_std": 27.70301628112793,
"step": 219
},
{
"KL/chosen_KL_mean": 3.390085220336914,
"KL/mean": -7.234885215759277,
"KL/rejected_KL_mean": -17.859859466552734,
"KL/std": 25.234189987182617,
"epoch": 0.4607329842931937,
"fcm_dpo/beta": 0.029233120381832123,
"fcm_dpo/delta": -0.02283724956214428,
"fcm_dpo/margin": 21.249942779541016,
"fcm_dpo/q_t": 0.3688524663448334,
"grad_norm": 46.714271545410156,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": 1.8225642442703247,
"logits/rejected": 1.7747106552124023,
"logps/chosen": -302.8143005371094,
"logps/ref_chosen": -306.2043762207031,
"logps/ref_rejected": -256.7926330566406,
"logps/rejected": -274.6524658203125,
"loss": 4.1038,
"margin_dpo/margin_mean": 21.24994468688965,
"margin_dpo/margin_std": 29.586055755615234,
"step": 220
},
{
"KL/chosen_KL_mean": 1.3392181396484375,
"KL/mean": -6.753597259521484,
"KL/rejected_KL_mean": -14.84642219543457,
"KL/std": 25.3695125579834,
"epoch": 0.46282722513089003,
"fcm_dpo/beta": 0.0307632889598608,
"fcm_dpo/delta": 0.10439582169055939,
"fcm_dpo/margin": 16.185644149780273,
"fcm_dpo/q_t": 0.39704880118370056,
"grad_norm": 55.72863006591797,
"learning_rate": 3.265574537815398e-07,
"logits/chosen": 1.3814107179641724,
"logits/rejected": 1.4919143915176392,
"logps/chosen": -291.99273681640625,
"logps/ref_chosen": -293.3319396972656,
"logps/ref_rejected": -282.7599182128906,
"logps/rejected": -297.6063232421875,
"loss": 4.5227,
"margin_dpo/margin_mean": 16.18564224243164,
"margin_dpo/margin_std": 28.6278076171875,
"step": 221
},
{
"KL/chosen_KL_mean": 2.481769561767578,
"KL/mean": -7.462711811065674,
"KL/rejected_KL_mean": -17.407188415527344,
"KL/std": 24.014915466308594,
"epoch": 0.4649214659685864,
"fcm_dpo/beta": 0.030535133555531502,
"fcm_dpo/delta": -0.011182472109794617,
"fcm_dpo/margin": 19.888965606689453,
"fcm_dpo/q_t": 0.3723604679107666,
"grad_norm": 50.793792724609375,
"learning_rate": 3.248126059518784e-07,
"logits/chosen": 1.49371337890625,
"logits/rejected": 1.3759980201721191,
"logps/chosen": -284.5243835449219,
"logps/ref_chosen": -287.0061340332031,
"logps/ref_rejected": -233.23130798339844,
"logps/rejected": -250.63851928710938,
"loss": 4.1239,
"margin_dpo/margin_mean": 19.888965606689453,
"margin_dpo/margin_std": 26.840036392211914,
"step": 222
},
{
"KL/chosen_KL_mean": 1.7029399871826172,
"KL/mean": -9.730170249938965,
"KL/rejected_KL_mean": -21.16328239440918,
"KL/std": 27.028404235839844,
"epoch": 0.46701570680628274,
"fcm_dpo/beta": 0.02971070259809494,
"fcm_dpo/delta": -0.08874759078025818,
"fcm_dpo/margin": 22.866230010986328,
"fcm_dpo/q_t": 0.3580406904220581,
"grad_norm": 43.08765411376953,
"learning_rate": 3.230637461492043e-07,
"logits/chosen": 1.57514488697052,
"logits/rejected": 1.4774196147918701,
"logps/chosen": -274.11236572265625,
"logps/ref_chosen": -275.8153076171875,
"logps/ref_rejected": -238.54286193847656,
"logps/rejected": -259.70611572265625,
"loss": 3.9717,
"margin_dpo/margin_mean": 22.866230010986328,
"margin_dpo/margin_std": 29.15939712524414,
"step": 223
},
{
"KL/chosen_KL_mean": 0.9415683746337891,
"KL/mean": -10.042745590209961,
"KL/rejected_KL_mean": -21.027055740356445,
"KL/std": 25.473079681396484,
"epoch": 0.46910994764397906,
"fcm_dpo/beta": 0.02928377129137516,
"fcm_dpo/delta": -0.04776962101459503,
"fcm_dpo/margin": 21.9686222076416,
"fcm_dpo/q_t": 0.3629598915576935,
"grad_norm": 43.5782470703125,
"learning_rate": 3.213109681595612e-07,
"logits/chosen": 1.6065367460250854,
"logits/rejected": 1.6727735996246338,
"logps/chosen": -238.1588134765625,
"logps/ref_chosen": -239.1003875732422,
"logps/ref_rejected": -230.31900024414062,
"logps/rejected": -251.34605407714844,
"loss": 3.9708,
"margin_dpo/margin_mean": 21.9686222076416,
"margin_dpo/margin_std": 26.808597564697266,
"step": 224
},
{
"KL/chosen_KL_mean": -2.3414173126220703,
"KL/mean": -10.983441352844238,
"KL/rejected_KL_mean": -19.625471115112305,
"KL/std": 27.00665855407715,
"epoch": 0.4712041884816754,
"fcm_dpo/beta": 0.027854181826114655,
"fcm_dpo/delta": 0.051747214049100876,
"fcm_dpo/margin": 17.28403663635254,
"fcm_dpo/q_t": 0.4044538736343384,
"grad_norm": 50.141746520996094,
"learning_rate": 3.1955436597911315e-07,
"logits/chosen": 1.6040029525756836,
"logits/rejected": 1.6216011047363281,
"logps/chosen": -302.6707763671875,
"logps/ref_chosen": -300.3293762207031,
"logps/ref_rejected": -332.132568359375,
"logps/rejected": -351.758056640625,
"loss": 4.6892,
"margin_dpo/margin_mean": 17.284034729003906,
"margin_dpo/margin_std": 33.72834777832031,
"step": 225
},
{
"KL/chosen_KL_mean": -1.0356464385986328,
"KL/mean": -8.687429428100586,
"KL/rejected_KL_mean": -16.339204788208008,
"KL/std": 24.889102935791016,
"epoch": 0.4732984293193717,
"fcm_dpo/beta": 0.03033391945064068,
"fcm_dpo/delta": 0.0942978709936142,
"fcm_dpo/margin": 15.30355453491211,
"fcm_dpo/q_t": 0.40447574853897095,
"grad_norm": 55.16220474243164,
"learning_rate": 3.1779403380910425e-07,
"logits/chosen": 1.3315622806549072,
"logits/rejected": 1.3884360790252686,
"logps/chosen": -246.4477996826172,
"logps/ref_chosen": -245.41213989257812,
"logps/ref_rejected": -251.72999572753906,
"logps/rejected": -268.0691833496094,
"loss": 4.6364,
"margin_dpo/margin_mean": 15.30355453491211,
"margin_dpo/margin_std": 29.265268325805664,
"step": 226
},
{
"KL/chosen_KL_mean": -0.8187618255615234,
"KL/mean": -11.41288948059082,
"KL/rejected_KL_mean": -22.00701332092285,
"KL/std": 26.464872360229492,
"epoch": 0.47539267015706804,
"fcm_dpo/beta": 0.03090839646756649,
"fcm_dpo/delta": -0.05948437750339508,
"fcm_dpo/margin": 21.188251495361328,
"fcm_dpo/q_t": 0.36408165097236633,
"grad_norm": 51.02029800415039,
"learning_rate": 3.160300660508064e-07,
"logits/chosen": 1.3621947765350342,
"logits/rejected": 1.3967113494873047,
"logps/chosen": -277.694580078125,
"logps/ref_chosen": -276.87579345703125,
"logps/ref_rejected": -259.0455017089844,
"logps/rejected": -281.052490234375,
"loss": 4.1226,
"margin_dpo/margin_mean": 21.18825340270996,
"margin_dpo/margin_std": 29.894878387451172,
"step": 227
},
{
"KL/chosen_KL_mean": 0.9791069030761719,
"KL/mean": -10.024959564208984,
"KL/rejected_KL_mean": -21.029029846191406,
"KL/std": 25.749013900756836,
"epoch": 0.4774869109947644,
"fcm_dpo/beta": 0.0297478549182415,
"fcm_dpo/delta": -0.05890221893787384,
"fcm_dpo/margin": 22.008134841918945,
"fcm_dpo/q_t": 0.3642002046108246,
"grad_norm": 55.73063659667969,
"learning_rate": 3.1426255730045695e-07,
"logits/chosen": 1.686366319656372,
"logits/rejected": 1.6875087022781372,
"logps/chosen": -302.991943359375,
"logps/ref_chosen": -303.9710693359375,
"logps/ref_rejected": -255.62033081054688,
"logps/rejected": -276.64935302734375,
"loss": 4.0223,
"margin_dpo/margin_mean": 22.00813865661621,
"margin_dpo/margin_std": 28.885726928710938,
"step": 228
},
{
"KL/chosen_KL_mean": 0.8257102966308594,
"KL/mean": -12.805261611938477,
"KL/rejected_KL_mean": -26.436227798461914,
"KL/std": 26.923622131347656,
"epoch": 0.47958115183246075,
"fcm_dpo/beta": 0.02663799375295639,
"fcm_dpo/delta": -0.13948528468608856,
"fcm_dpo/margin": 27.261943817138672,
"fcm_dpo/q_t": 0.3465552031993866,
"grad_norm": 56.9528694152832,
"learning_rate": 3.1249160234418644e-07,
"logits/chosen": 1.4259783029556274,
"logits/rejected": 1.3962115049362183,
"logps/chosen": -280.8702392578125,
"logps/ref_chosen": -281.6959533691406,
"logps/ref_rejected": -258.93353271484375,
"logps/rejected": -285.3697814941406,
"loss": 3.7582,
"margin_dpo/margin_mean": 27.261943817138672,
"margin_dpo/margin_std": 30.23539161682129,
"step": 229
},
{
"KL/chosen_KL_mean": -0.2526702880859375,
"KL/mean": -11.950700759887695,
"KL/rejected_KL_mean": -23.64872932434082,
"KL/std": 26.261003494262695,
"epoch": 0.4816753926701571,
"fcm_dpo/beta": 0.02576565369963646,
"fcm_dpo/delta": -0.005254525691270828,
"fcm_dpo/margin": 23.39605140686035,
"fcm_dpo/q_t": 0.37064623832702637,
"grad_norm": 47.49397277832031,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": 1.2236592769622803,
"logits/rejected": 1.302830696105957,
"logps/chosen": -224.20921325683594,
"logps/ref_chosen": -223.95655822753906,
"logps/ref_rejected": -238.95187377929688,
"logps/rejected": -262.6005859375,
"loss": 4.1101,
"margin_dpo/margin_mean": 23.39605140686035,
"margin_dpo/margin_std": 31.021326065063477,
"step": 230
},
{
"KL/chosen_KL_mean": -2.974292755126953,
"KL/mean": -12.331928253173828,
"KL/rejected_KL_mean": -21.68956184387207,
"KL/std": 25.90277671813965,
"epoch": 0.4837696335078534,
"fcm_dpo/beta": 0.026006178930401802,
"fcm_dpo/delta": 0.07183073461055756,
"fcm_dpo/margin": 18.71527099609375,
"fcm_dpo/q_t": 0.3939741849899292,
"grad_norm": 40.720008850097656,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 1.339698314666748,
"logits/rejected": 1.3909707069396973,
"logps/chosen": -317.8988952636719,
"logps/ref_chosen": -314.92462158203125,
"logps/ref_rejected": -279.6347351074219,
"logps/rejected": -301.32427978515625,
"loss": 4.3163,
"margin_dpo/margin_mean": 18.715272903442383,
"margin_dpo/margin_std": 27.25091552734375,
"step": 231
},
{
"KL/chosen_KL_mean": -0.09930038452148438,
"KL/mean": -10.04248046875,
"KL/rejected_KL_mean": -19.98565673828125,
"KL/std": 27.593305587768555,
"epoch": 0.48586387434554973,
"fcm_dpo/beta": 0.026716448366642,
"fcm_dpo/delta": -0.0027984483167529106,
"fcm_dpo/margin": 19.886362075805664,
"fcm_dpo/q_t": 0.3892497420310974,
"grad_norm": 57.157737731933594,
"learning_rate": 3.071590108427243e-07,
"logits/chosen": 1.283789873123169,
"logits/rejected": 1.3314770460128784,
"logps/chosen": -263.41888427734375,
"logps/ref_chosen": -263.3196105957031,
"logps/ref_rejected": -247.13612365722656,
"logps/rejected": -267.12176513671875,
"loss": 4.3119,
"margin_dpo/margin_mean": 19.886363983154297,
"margin_dpo/margin_std": 29.738393783569336,
"step": 232
},
{
"KL/chosen_KL_mean": 0.6233539581298828,
"KL/mean": -10.353264808654785,
"KL/rejected_KL_mean": -21.32988929748535,
"KL/std": 26.082984924316406,
"epoch": 0.48795811518324606,
"fcm_dpo/beta": 0.02723000943660736,
"fcm_dpo/delta": 0.0005163960158824921,
"fcm_dpo/margin": 21.95323371887207,
"fcm_dpo/q_t": 0.37384089827537537,
"grad_norm": 48.244415283203125,
"learning_rate": 3.05375222543809e-07,
"logits/chosen": 1.3779985904693604,
"logits/rejected": 1.439375877380371,
"logps/chosen": -278.7388610839844,
"logps/ref_chosen": -279.36224365234375,
"logps/ref_rejected": -264.46112060546875,
"logps/rejected": -285.791015625,
"loss": 4.1144,
"margin_dpo/margin_mean": 21.953237533569336,
"margin_dpo/margin_std": 29.3866024017334,
"step": 233
},
{
"KL/chosen_KL_mean": 0.3291606903076172,
"KL/mean": -11.041275978088379,
"KL/rejected_KL_mean": -22.411699295043945,
"KL/std": 27.928560256958008,
"epoch": 0.4900523560209424,
"fcm_dpo/beta": 0.027022581547498703,
"fcm_dpo/delta": -0.015877969563007355,
"fcm_dpo/margin": 22.74085807800293,
"fcm_dpo/q_t": 0.3736668825149536,
"grad_norm": 44.880882263183594,
"learning_rate": 3.035884646397637e-07,
"logits/chosen": 1.440163254737854,
"logits/rejected": 1.5084267854690552,
"logps/chosen": -286.1199645996094,
"logps/ref_chosen": -286.4491271972656,
"logps/ref_rejected": -294.5469665527344,
"logps/rejected": -316.95867919921875,
"loss": 4.2289,
"margin_dpo/margin_mean": 22.740859985351562,
"margin_dpo/margin_std": 34.04836654663086,
"step": 234
},
{
"KL/chosen_KL_mean": -2.050537109375,
"KL/mean": -12.448473930358887,
"KL/rejected_KL_mean": -22.84640884399414,
"KL/std": 29.29195785522461,
"epoch": 0.49214659685863876,
"fcm_dpo/beta": 0.027660487219691277,
"fcm_dpo/delta": 0.023279912769794464,
"fcm_dpo/margin": 20.79585838317871,
"fcm_dpo/q_t": 0.380502313375473,
"grad_norm": 52.81705856323242,
"learning_rate": 3.017988329489923e-07,
"logits/chosen": 1.4513448476791382,
"logits/rejected": 1.4381203651428223,
"logps/chosen": -284.4106140136719,
"logps/ref_chosen": -282.36004638671875,
"logps/ref_rejected": -243.79052734375,
"logps/rejected": -266.6369323730469,
"loss": 4.3436,
"margin_dpo/margin_mean": 20.79585838317871,
"margin_dpo/margin_std": 32.966270446777344,
"step": 235
},
{
"KL/chosen_KL_mean": 1.3165740966796875,
"KL/mean": -10.353586196899414,
"KL/rejected_KL_mean": -22.023738861083984,
"KL/std": 27.25334930419922,
"epoch": 0.4942408376963351,
"fcm_dpo/beta": 0.027351150289177895,
"fcm_dpo/delta": -0.04152906313538551,
"fcm_dpo/margin": 23.340309143066406,
"fcm_dpo/q_t": 0.36344993114471436,
"grad_norm": 42.070194244384766,
"learning_rate": 3.000064234440111e-07,
"logits/chosen": 1.2549378871917725,
"logits/rejected": 1.2370033264160156,
"logps/chosen": -277.8228454589844,
"logps/ref_chosen": -279.139404296875,
"logps/ref_rejected": -240.5850067138672,
"logps/rejected": -262.6087646484375,
"loss": 4.068,
"margin_dpo/margin_mean": 23.340309143066406,
"margin_dpo/margin_std": 30.97328758239746,
"step": 236
},
{
"KL/chosen_KL_mean": -1.8713130950927734,
"KL/mean": -11.904808044433594,
"KL/rejected_KL_mean": -21.938297271728516,
"KL/std": 24.791580200195312,
"epoch": 0.4963350785340314,
"fcm_dpo/beta": 0.02666321024298668,
"fcm_dpo/delta": -0.00504470057785511,
"fcm_dpo/margin": 20.06699562072754,
"fcm_dpo/q_t": 0.38698601722717285,
"grad_norm": 51.159584045410156,
"learning_rate": 2.9821133224630223e-07,
"logits/chosen": 1.4548025131225586,
"logits/rejected": 1.5232490301132202,
"logps/chosen": -260.6828308105469,
"logps/ref_chosen": -258.8114929199219,
"logps/ref_rejected": -266.3898010253906,
"logps/rejected": -288.32806396484375,
"loss": 4.3179,
"margin_dpo/margin_mean": 20.06699562072754,
"margin_dpo/margin_std": 30.49276351928711,
"step": 237
},
{
"KL/chosen_KL_mean": -2.533510208129883,
"KL/mean": -13.915240287780762,
"KL/rejected_KL_mean": -25.296966552734375,
"KL/std": 28.526103973388672,
"epoch": 0.49842931937172774,
"fcm_dpo/beta": 0.025130130350589752,
"fcm_dpo/delta": -0.050658006221055984,
"fcm_dpo/margin": 22.763446807861328,
"fcm_dpo/q_t": 0.3809536397457123,
"grad_norm": 44.547672271728516,
"learning_rate": 2.964136556211588e-07,
"logits/chosen": 1.3475905656814575,
"logits/rejected": 1.2551782131195068,
"logps/chosen": -304.5997009277344,
"logps/ref_chosen": -302.0661926269531,
"logps/ref_rejected": -265.5518798828125,
"logps/rejected": -290.8488464355469,
"loss": 4.2063,
"margin_dpo/margin_mean": 22.763446807861328,
"margin_dpo/margin_std": 32.277366638183594,
"step": 238
},
{
"KL/chosen_KL_mean": 0.11390304565429688,
"KL/mean": -9.70567798614502,
"KL/rejected_KL_mean": -19.525257110595703,
"KL/std": 27.92169761657715,
"epoch": 0.5005235602094241,
"fcm_dpo/beta": 0.025815239176154137,
"fcm_dpo/delta": 0.09881044924259186,
"fcm_dpo/margin": 19.639163970947266,
"fcm_dpo/q_t": 0.3956132233142853,
"grad_norm": 50.240867614746094,
"learning_rate": 2.946134899725226e-07,
"logits/chosen": 1.5159615278244019,
"logits/rejected": 1.6065120697021484,
"logps/chosen": -258.88446044921875,
"logps/ref_chosen": -258.99835205078125,
"logps/ref_rejected": -273.7834777832031,
"logps/rejected": -293.3087463378906,
"loss": 4.4926,
"margin_dpo/margin_mean": 19.639163970947266,
"margin_dpo/margin_std": 34.4343147277832,
"step": 239
},
{
"KL/chosen_KL_mean": 0.9601726531982422,
"KL/mean": -11.192224502563477,
"KL/rejected_KL_mean": -23.34463119506836,
"KL/std": 27.793094635009766,
"epoch": 0.5026178010471204,
"fcm_dpo/beta": 0.025913547724485397,
"fcm_dpo/delta": -0.08534112572669983,
"fcm_dpo/margin": 24.3048095703125,
"fcm_dpo/q_t": 0.36542585492134094,
"grad_norm": 50.47843933105469,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": 1.328792691230774,
"logits/rejected": 1.3966224193572998,
"logps/chosen": -274.5628967285156,
"logps/ref_chosen": -275.5230407714844,
"logps/ref_rejected": -255.50784301757812,
"logps/rejected": -278.85247802734375,
"loss": 3.9814,
"margin_dpo/margin_mean": 24.3048095703125,
"margin_dpo/margin_std": 30.22945213317871,
"step": 240
},
{
"KL/chosen_KL_mean": -2.142576217651367,
"KL/mean": -11.555323600769043,
"KL/rejected_KL_mean": -20.968074798583984,
"KL/std": 28.288204193115234,
"epoch": 0.5047120418848168,
"fcm_dpo/beta": 0.02569686621427536,
"fcm_dpo/delta": 0.06423062831163406,
"fcm_dpo/margin": 18.825511932373047,
"fcm_dpo/q_t": 0.40052130818367004,
"grad_norm": 43.8680419921875,
"learning_rate": 2.910060778827554e-07,
"logits/chosen": 1.573228120803833,
"logits/rejected": 1.6145110130310059,
"logps/chosen": -255.50607299804688,
"logps/ref_chosen": -253.36349487304688,
"logps/ref_rejected": -247.1916961669922,
"logps/rejected": -268.1597595214844,
"loss": 4.5423,
"margin_dpo/margin_mean": 18.825511932373047,
"margin_dpo/margin_std": 33.42321014404297,
"step": 241
},
{
"KL/chosen_KL_mean": -0.8480720520019531,
"KL/mean": -12.20622444152832,
"KL/rejected_KL_mean": -23.564376831054688,
"KL/std": 27.829221725463867,
"epoch": 0.506806282722513,
"fcm_dpo/beta": 0.026592141017317772,
"fcm_dpo/delta": -0.006941572763025761,
"fcm_dpo/margin": 22.71630096435547,
"fcm_dpo/q_t": 0.3716428279876709,
"grad_norm": 40.12123107910156,
"learning_rate": 2.891990248961871e-07,
"logits/chosen": 1.5100977420806885,
"logits/rejected": 1.4358429908752441,
"logps/chosen": -265.8361511230469,
"logps/ref_chosen": -264.9880676269531,
"logps/ref_rejected": -243.07838439941406,
"logps/rejected": -266.64276123046875,
"loss": 4.0944,
"margin_dpo/margin_mean": 22.71630096435547,
"margin_dpo/margin_std": 29.884212493896484,
"step": 242
},
{
"KL/chosen_KL_mean": 1.9819889068603516,
"KL/mean": -11.120399475097656,
"KL/rejected_KL_mean": -24.22279930114746,
"KL/std": 27.46685791015625,
"epoch": 0.5089005235602094,
"fcm_dpo/beta": 0.025906052440404892,
"fcm_dpo/delta": -0.08735494315624237,
"fcm_dpo/margin": 26.204782485961914,
"fcm_dpo/q_t": 0.35590317845344543,
"grad_norm": 55.49812316894531,
"learning_rate": 2.873898697848762e-07,
"logits/chosen": 1.5263855457305908,
"logits/rejected": 1.5749152898788452,
"logps/chosen": -313.8887023925781,
"logps/ref_chosen": -315.8706970214844,
"logps/ref_rejected": -304.3729248046875,
"logps/rejected": -328.595703125,
"loss": 3.8786,
"margin_dpo/margin_mean": 26.204784393310547,
"margin_dpo/margin_std": 29.490520477294922,
"step": 243
},
{
"KL/chosen_KL_mean": 0.8133888244628906,
"KL/mean": -12.799525260925293,
"KL/rejected_KL_mean": -26.412450790405273,
"KL/std": 28.79197120666504,
"epoch": 0.5109947643979058,
"fcm_dpo/beta": 0.023789148777723312,
"fcm_dpo/delta": -0.05430053174495697,
"fcm_dpo/margin": 27.22583770751953,
"fcm_dpo/q_t": 0.36135512590408325,
"grad_norm": 37.841976165771484,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 1.4552251100540161,
"logits/rejected": 1.4061833620071411,
"logps/chosen": -310.2837829589844,
"logps/ref_chosen": -311.09716796875,
"logps/ref_rejected": -266.01959228515625,
"logps/rejected": -292.4320373535156,
"loss": 3.9512,
"margin_dpo/margin_mean": 27.22583770751953,
"margin_dpo/margin_std": 32.62396240234375,
"step": 244
},
{
"KL/chosen_KL_mean": 1.0648555755615234,
"KL/mean": -12.222169876098633,
"KL/rejected_KL_mean": -25.509199142456055,
"KL/std": 28.52521514892578,
"epoch": 0.5130890052356021,
"fcm_dpo/beta": 0.023074612021446228,
"fcm_dpo/delta": -0.08860162645578384,
"fcm_dpo/margin": 26.574054718017578,
"fcm_dpo/q_t": 0.36784839630126953,
"grad_norm": 45.462127685546875,
"learning_rate": 2.837656413735479e-07,
"logits/chosen": 1.5822170972824097,
"logits/rejected": 1.4620615243911743,
"logps/chosen": -283.67559814453125,
"logps/ref_chosen": -284.740478515625,
"logps/ref_rejected": -237.64117431640625,
"logps/rejected": -263.150390625,
"loss": 4.0302,
"margin_dpo/margin_mean": 26.574054718017578,
"margin_dpo/margin_std": 32.760372161865234,
"step": 245
},
{
"KL/chosen_KL_mean": -5.959909439086914,
"KL/mean": -14.731524467468262,
"KL/rejected_KL_mean": -23.503145217895508,
"KL/std": 28.082195281982422,
"epoch": 0.5151832460732985,
"fcm_dpo/beta": 0.02217438444495201,
"fcm_dpo/delta": 0.033019039779901505,
"fcm_dpo/margin": 17.543224334716797,
"fcm_dpo/q_t": 0.4182347059249878,
"grad_norm": 46.06009292602539,
"learning_rate": 2.8195076242990116e-07,
"logits/chosen": 1.49590265750885,
"logits/rejected": 1.5044995546340942,
"logps/chosen": -278.57281494140625,
"logps/ref_chosen": -272.6128845214844,
"logps/ref_rejected": -249.99725341796875,
"logps/rejected": -273.50042724609375,
"loss": 4.7293,
"margin_dpo/margin_mean": 17.54322624206543,
"margin_dpo/margin_std": 34.45924758911133,
"step": 246
},
{
"KL/chosen_KL_mean": -1.1401596069335938,
"KL/mean": -12.122604370117188,
"KL/rejected_KL_mean": -23.10504722595215,
"KL/std": 27.030519485473633,
"epoch": 0.5172774869109947,
"fcm_dpo/beta": 0.023736296221613884,
"fcm_dpo/delta": 0.08034541457891464,
"fcm_dpo/margin": 21.964893341064453,
"fcm_dpo/q_t": 0.3879540264606476,
"grad_norm": 40.82952880859375,
"learning_rate": 2.801341700638307e-07,
"logits/chosen": 1.4822200536727905,
"logits/rejected": 1.4732378721237183,
"logps/chosen": -275.5759582519531,
"logps/ref_chosen": -274.435791015625,
"logps/ref_rejected": -257.8125305175781,
"logps/rejected": -280.9175720214844,
"loss": 4.261,
"margin_dpo/margin_mean": 21.964893341064453,
"margin_dpo/margin_std": 31.085176467895508,
"step": 247
},
{
"KL/chosen_KL_mean": -2.6777801513671875,
"KL/mean": -12.569023132324219,
"KL/rejected_KL_mean": -22.46026611328125,
"KL/std": 27.558944702148438,
"epoch": 0.5193717277486911,
"fcm_dpo/beta": 0.02531389333307743,
"fcm_dpo/delta": 0.10261654108762741,
"fcm_dpo/margin": 19.782489776611328,
"fcm_dpo/q_t": 0.39476096630096436,
"grad_norm": 46.36613845825195,
"learning_rate": 2.7831596169367227e-07,
"logits/chosen": 1.4401912689208984,
"logits/rejected": 1.4732575416564941,
"logps/chosen": -263.9891052246094,
"logps/ref_chosen": -261.3113098144531,
"logps/ref_rejected": -231.152587890625,
"logps/rejected": -253.61285400390625,
"loss": 4.4323,
"margin_dpo/margin_mean": 19.782487869262695,
"margin_dpo/margin_std": 32.60723876953125,
"step": 248
},
{
"KL/chosen_KL_mean": -3.753652572631836,
"KL/mean": -14.500252723693848,
"KL/rejected_KL_mean": -25.246868133544922,
"KL/std": 29.098094940185547,
"epoch": 0.5214659685863874,
"fcm_dpo/beta": 0.02699856646358967,
"fcm_dpo/delta": 0.018965115770697594,
"fcm_dpo/margin": 21.49321746826172,
"fcm_dpo/q_t": 0.3801358938217163,
"grad_norm": 43.70502471923828,
"learning_rate": 2.7649623482442274e-07,
"logits/chosen": 1.4656652212142944,
"logits/rejected": 1.406292200088501,
"logps/chosen": -272.22088623046875,
"logps/ref_chosen": -268.46722412109375,
"logps/ref_rejected": -240.27597045898438,
"logps/rejected": -265.5228271484375,
"loss": 4.2513,
"margin_dpo/margin_mean": 21.49321746826172,
"margin_dpo/margin_std": 32.12462615966797,
"step": 249
},
{
"KL/chosen_KL_mean": -3.9061927795410156,
"KL/mean": -14.475398063659668,
"KL/rejected_KL_mean": -25.04460334777832,
"KL/std": 29.591577529907227,
"epoch": 0.5235602094240838,
"fcm_dpo/beta": 0.02638782374560833,
"fcm_dpo/delta": 0.04420147091150284,
"fcm_dpo/margin": 21.13842010498047,
"fcm_dpo/q_t": 0.3843858242034912,
"grad_norm": 46.38828659057617,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": 1.5810189247131348,
"logits/rejected": 1.6897047758102417,
"logps/chosen": -289.01141357421875,
"logps/ref_chosen": -285.1051940917969,
"logps/ref_rejected": -281.2331848144531,
"logps/rejected": -306.2778015136719,
"loss": 4.4161,
"margin_dpo/margin_mean": 21.13842010498047,
"margin_dpo/margin_std": 34.87960433959961,
"step": 250
},
{
"KL/chosen_KL_mean": -1.2985305786132812,
"KL/mean": -12.381001472473145,
"KL/rejected_KL_mean": -23.463476181030273,
"KL/std": 27.799116134643555,
"epoch": 0.5256544502617801,
"fcm_dpo/beta": 0.02752423658967018,
"fcm_dpo/delta": -0.06013893708586693,
"fcm_dpo/margin": 22.164939880371094,
"fcm_dpo/q_t": 0.373883992433548,
"grad_norm": 53.40812301635742,
"learning_rate": 2.7285261601056697e-07,
"logits/chosen": 1.5160138607025146,
"logits/rejected": 1.4690115451812744,
"logps/chosen": -275.4111633300781,
"logps/ref_chosen": -274.1126403808594,
"logps/ref_rejected": -256.7730407714844,
"logps/rejected": -280.2365417480469,
"loss": 4.1646,
"margin_dpo/margin_mean": 22.16494369506836,
"margin_dpo/margin_std": 31.104576110839844,
"step": 251
},
{
"KL/chosen_KL_mean": -5.993812561035156,
"KL/mean": -17.212871551513672,
"KL/rejected_KL_mean": -28.431922912597656,
"KL/std": 29.184667587280273,
"epoch": 0.5277486910994764,
"fcm_dpo/beta": 0.025960441678762436,
"fcm_dpo/delta": 0.015204920433461666,
"fcm_dpo/margin": 22.438114166259766,
"fcm_dpo/q_t": 0.37905487418174744,
"grad_norm": 48.171077728271484,
"learning_rate": 2.7102891946217994e-07,
"logits/chosen": 1.4628584384918213,
"logits/rejected": 1.422957181930542,
"logps/chosen": -295.5595397949219,
"logps/ref_chosen": -289.56573486328125,
"logps/ref_rejected": -277.9891662597656,
"logps/rejected": -306.42108154296875,
"loss": 4.3431,
"margin_dpo/margin_mean": 22.438114166259766,
"margin_dpo/margin_std": 34.84325408935547,
"step": 252
},
{
"KL/chosen_KL_mean": -5.769189834594727,
"KL/mean": -15.143193244934082,
"KL/rejected_KL_mean": -24.517194747924805,
"KL/std": 29.447696685791016,
"epoch": 0.5298429319371728,
"fcm_dpo/beta": 0.026602119207382202,
"fcm_dpo/delta": -0.0013262918218970299,
"fcm_dpo/margin": 18.748003005981445,
"fcm_dpo/q_t": 0.3968650698661804,
"grad_norm": 63.81908416748047,
"learning_rate": 2.692040951966617e-07,
"logits/chosen": 1.346488118171692,
"logits/rejected": 1.3190160989761353,
"logps/chosen": -275.953125,
"logps/ref_chosen": -270.1839294433594,
"logps/ref_rejected": -245.86949157714844,
"logps/rejected": -270.38665771484375,
"loss": 4.538,
"margin_dpo/margin_mean": 18.748003005981445,
"margin_dpo/margin_std": 33.28840255737305,
"step": 253
},
{
"KL/chosen_KL_mean": -2.1069259643554688,
"KL/mean": -13.89623737335205,
"KL/rejected_KL_mean": -25.685544967651367,
"KL/std": 29.125656127929688,
"epoch": 0.5319371727748691,
"fcm_dpo/beta": 0.027346130460500717,
"fcm_dpo/delta": -0.049901798367500305,
"fcm_dpo/margin": 23.5786190032959,
"fcm_dpo/q_t": 0.36554965376853943,
"grad_norm": 62.93083572387695,
"learning_rate": 2.6737824107379947e-07,
"logits/chosen": 1.3603527545928955,
"logits/rejected": 1.3280959129333496,
"logps/chosen": -262.7933654785156,
"logps/ref_chosen": -260.6864318847656,
"logps/ref_rejected": -245.99508666992188,
"logps/rejected": -271.6806335449219,
"loss": 4.0305,
"margin_dpo/margin_mean": 23.57862091064453,
"margin_dpo/margin_std": 29.09947967529297,
"step": 254
},
{
"KL/chosen_KL_mean": -0.32198143005371094,
"KL/mean": -13.63015365600586,
"KL/rejected_KL_mean": -26.938322067260742,
"KL/std": 29.864192962646484,
"epoch": 0.5340314136125655,
"fcm_dpo/beta": 0.025404341518878937,
"fcm_dpo/delta": -0.080487459897995,
"fcm_dpo/margin": 26.616352081298828,
"fcm_dpo/q_t": 0.3620305359363556,
"grad_norm": 40.981224060058594,
"learning_rate": 2.655514550086086e-07,
"logits/chosen": 1.6588364839553833,
"logits/rejected": 1.5612589120864868,
"logps/chosen": -298.23583984375,
"logps/ref_chosen": -297.9138488769531,
"logps/ref_rejected": -253.39952087402344,
"logps/rejected": -280.33782958984375,
"loss": 4.0428,
"margin_dpo/margin_mean": 26.616350173950195,
"margin_dpo/margin_std": 36.11050796508789,
"step": 255
},
{
"KL/chosen_KL_mean": 0.7054767608642578,
"KL/mean": -12.810953140258789,
"KL/rejected_KL_mean": -26.32737922668457,
"KL/std": 31.35456085205078,
"epoch": 0.5361256544502618,
"fcm_dpo/beta": 0.024050066247582436,
"fcm_dpo/delta": -0.055629149079322815,
"fcm_dpo/margin": 27.032861709594727,
"fcm_dpo/q_t": 0.36533451080322266,
"grad_norm": 44.06679916381836,
"learning_rate": 2.6372383496608186e-07,
"logits/chosen": 1.5450178384780884,
"logits/rejected": 1.4958531856536865,
"logps/chosen": -309.22088623046875,
"logps/ref_chosen": -309.9263916015625,
"logps/ref_rejected": -253.452392578125,
"logps/rejected": -279.77978515625,
"loss": 4.1981,
"margin_dpo/margin_mean": 27.032861709594727,
"margin_dpo/margin_std": 38.51383590698242,
"step": 256
},
{
"KL/chosen_KL_mean": -2.0746002197265625,
"KL/mean": -13.949150085449219,
"KL/rejected_KL_mean": -25.823705673217773,
"KL/std": 30.354991912841797,
"epoch": 0.5382198952879581,
"fcm_dpo/beta": 0.023041877895593643,
"fcm_dpo/delta": 0.055370353162288666,
"fcm_dpo/margin": 23.749109268188477,
"fcm_dpo/q_t": 0.38487961888313293,
"grad_norm": 57.64748764038086,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 1.1348298788070679,
"logits/rejected": 1.152570128440857,
"logps/chosen": -261.8882141113281,
"logps/ref_chosen": -259.8136291503906,
"logps/ref_rejected": -245.84947204589844,
"logps/rejected": -271.6731872558594,
"loss": 4.1992,
"margin_dpo/margin_mean": 23.749109268188477,
"margin_dpo/margin_std": 33.00218200683594,
"step": 257
},
{
"KL/chosen_KL_mean": -1.9968814849853516,
"KL/mean": -14.562836647033691,
"KL/rejected_KL_mean": -27.128786087036133,
"KL/std": 30.502214431762695,
"epoch": 0.5403141361256545,
"fcm_dpo/beta": 0.023513339459896088,
"fcm_dpo/delta": 0.00881824642419815,
"fcm_dpo/margin": 25.13190269470215,
"fcm_dpo/q_t": 0.3715367615222931,
"grad_norm": 46.1146354675293,
"learning_rate": 2.600664850273538e-07,
"logits/chosen": 1.6273773908615112,
"logits/rejected": 1.7062067985534668,
"logps/chosen": -267.3505554199219,
"logps/ref_chosen": -265.35369873046875,
"logps/ref_rejected": -247.9174346923828,
"logps/rejected": -275.04620361328125,
"loss": 4.0437,
"margin_dpo/margin_mean": 25.13190269470215,
"margin_dpo/margin_std": 31.523521423339844,
"step": 258
},
{
"KL/chosen_KL_mean": 0.11016464233398438,
"KL/mean": -12.010380744934082,
"KL/rejected_KL_mean": -24.13092041015625,
"KL/std": 30.323959350585938,
"epoch": 0.5424083769633508,
"fcm_dpo/beta": 0.024552009999752045,
"fcm_dpo/delta": 0.004387339577078819,
"fcm_dpo/margin": 24.241092681884766,
"fcm_dpo/q_t": 0.3753139078617096,
"grad_norm": 51.9947395324707,
"learning_rate": 2.582369512637302e-07,
"logits/chosen": 1.5842348337173462,
"logits/rejected": 1.5799920558929443,
"logps/chosen": -288.1130676269531,
"logps/ref_chosen": -288.2232360839844,
"logps/ref_rejected": -270.7300109863281,
"logps/rejected": -294.8609619140625,
"loss": 4.1724,
"margin_dpo/margin_mean": 24.241092681884766,
"margin_dpo/margin_std": 35.039058685302734,
"step": 259
},
{
"KL/chosen_KL_mean": -3.2845516204833984,
"KL/mean": -9.023161888122559,
"KL/rejected_KL_mean": -14.761768341064453,
"KL/std": 28.911399841308594,
"epoch": 0.5445026178010471,
"fcm_dpo/beta": 0.02567979507148266,
"fcm_dpo/delta": 0.1519329994916916,
"fcm_dpo/margin": 11.477216720581055,
"fcm_dpo/q_t": 0.43951642513275146,
"grad_norm": 71.44678497314453,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": 1.722202181816101,
"logits/rejected": 1.764743447303772,
"logps/chosen": -284.9042663574219,
"logps/ref_chosen": -281.61968994140625,
"logps/ref_rejected": -271.03973388671875,
"logps/rejected": -285.801513671875,
"loss": 5.1871,
"margin_dpo/margin_mean": 11.477216720581055,
"margin_dpo/margin_std": 33.64004135131836,
"step": 260
},
{
"KL/chosen_KL_mean": -6.1820526123046875,
"KL/mean": -16.529911041259766,
"KL/rejected_KL_mean": -26.877765655517578,
"KL/std": 29.89157485961914,
"epoch": 0.5465968586387434,
"fcm_dpo/beta": 0.027530504390597343,
"fcm_dpo/delta": 0.029914624989032745,
"fcm_dpo/margin": 20.69570541381836,
"fcm_dpo/q_t": 0.3858029246330261,
"grad_norm": 57.450679779052734,
"learning_rate": 2.5457665670441937e-07,
"logits/chosen": 1.2852181196212769,
"logits/rejected": 1.4253342151641846,
"logps/chosen": -248.6452178955078,
"logps/ref_chosen": -242.4631805419922,
"logps/ref_rejected": -244.0020751953125,
"logps/rejected": -270.87982177734375,
"loss": 4.4957,
"margin_dpo/margin_mean": 20.69570541381836,
"margin_dpo/margin_std": 36.37822723388672,
"step": 261
},
{
"KL/chosen_KL_mean": -0.7161979675292969,
"KL/mean": -13.208702087402344,
"KL/rejected_KL_mean": -25.701202392578125,
"KL/std": 30.572877883911133,
"epoch": 0.5486910994764398,
"fcm_dpo/beta": 0.027186822146177292,
"fcm_dpo/delta": -0.08552050590515137,
"fcm_dpo/margin": 24.984996795654297,
"fcm_dpo/q_t": 0.36049196124076843,
"grad_norm": 50.54449462890625,
"learning_rate": 2.527460921992209e-07,
"logits/chosen": 1.580787181854248,
"logits/rejected": 1.5897417068481445,
"logps/chosen": -292.704833984375,
"logps/ref_chosen": -291.98858642578125,
"logps/ref_rejected": -273.35955810546875,
"logps/rejected": -299.060791015625,
"loss": 3.9733,
"margin_dpo/margin_mean": 24.984994888305664,
"margin_dpo/margin_std": 32.54281997680664,
"step": 262
},
{
"KL/chosen_KL_mean": -4.733024597167969,
"KL/mean": -15.698508262634277,
"KL/rejected_KL_mean": -26.663997650146484,
"KL/std": 30.50117301940918,
"epoch": 0.5507853403141362,
"fcm_dpo/beta": 0.025977104902267456,
"fcm_dpo/delta": -0.03878769278526306,
"fcm_dpo/margin": 21.930967330932617,
"fcm_dpo/q_t": 0.3833428621292114,
"grad_norm": 55.72367858886719,
"learning_rate": 2.509153804294318e-07,
"logits/chosen": 1.263526439666748,
"logits/rejected": 1.2350153923034668,
"logps/chosen": -279.72149658203125,
"logps/ref_chosen": -274.98846435546875,
"logps/ref_rejected": -254.75782775878906,
"logps/rejected": -281.4218444824219,
"loss": 4.3527,
"margin_dpo/margin_mean": 21.930967330932617,
"margin_dpo/margin_std": 35.15817642211914,
"step": 263
},
{
"KL/chosen_KL_mean": 0.7876987457275391,
"KL/mean": -12.341998100280762,
"KL/rejected_KL_mean": -25.4716854095459,
"KL/std": 30.41027069091797,
"epoch": 0.5528795811518324,
"fcm_dpo/beta": 0.02479763887822628,
"fcm_dpo/delta": -0.056634265929460526,
"fcm_dpo/margin": 26.259387969970703,
"fcm_dpo/q_t": 0.36550551652908325,
"grad_norm": 40.78193664550781,
"learning_rate": 2.4908461957056825e-07,
"logits/chosen": 1.1997872591018677,
"logits/rejected": 1.2514468431472778,
"logps/chosen": -250.83847045898438,
"logps/ref_chosen": -251.62615966796875,
"logps/ref_rejected": -251.9549560546875,
"logps/rejected": -277.4266357421875,
"loss": 4.0046,
"margin_dpo/margin_mean": 26.259389877319336,
"margin_dpo/margin_std": 34.020729064941406,
"step": 264
},
{
"KL/chosen_KL_mean": -3.2062511444091797,
"KL/mean": -15.04905891418457,
"KL/rejected_KL_mean": -26.89187240600586,
"KL/std": 31.76069450378418,
"epoch": 0.5549738219895288,
"fcm_dpo/beta": 0.02372550778090954,
"fcm_dpo/delta": -0.01052355207502842,
"fcm_dpo/margin": 23.685611724853516,
"fcm_dpo/q_t": 0.38212448358535767,
"grad_norm": 45.85237121582031,
"learning_rate": 2.4725390780077905e-07,
"logits/chosen": 1.3538559675216675,
"logits/rejected": 1.3851525783538818,
"logps/chosen": -281.1428527832031,
"logps/ref_chosen": -277.9366149902344,
"logps/ref_rejected": -262.689453125,
"logps/rejected": -289.5813293457031,
"loss": 4.2632,
"margin_dpo/margin_mean": 23.685611724853516,
"margin_dpo/margin_std": 34.29460144042969,
"step": 265
},
{
"KL/chosen_KL_mean": 2.9426498413085938,
"KL/mean": -9.96561336517334,
"KL/rejected_KL_mean": -22.873876571655273,
"KL/std": 30.54420280456543,
"epoch": 0.5570680628272251,
"fcm_dpo/beta": 0.02399318292737007,
"fcm_dpo/delta": -0.022777840495109558,
"fcm_dpo/margin": 25.816530227661133,
"fcm_dpo/q_t": 0.3668290376663208,
"grad_norm": 43.122013092041016,
"learning_rate": 2.454233432955807e-07,
"logits/chosen": 1.5934802293777466,
"logits/rejected": 1.5779345035552979,
"logps/chosen": -267.50225830078125,
"logps/ref_chosen": -270.4449157714844,
"logps/ref_rejected": -259.5942077636719,
"logps/rejected": -282.4681091308594,
"loss": 4.0291,
"margin_dpo/margin_mean": 25.816532135009766,
"margin_dpo/margin_std": 32.49258041381836,
"step": 266
},
{
"KL/chosen_KL_mean": -2.7025794982910156,
"KL/mean": -13.08747386932373,
"KL/rejected_KL_mean": -23.472375869750977,
"KL/std": 30.725170135498047,
"epoch": 0.5591623036649215,
"fcm_dpo/beta": 0.02411152422428131,
"fcm_dpo/delta": 0.051255080848932266,
"fcm_dpo/margin": 20.769792556762695,
"fcm_dpo/q_t": 0.3949718773365021,
"grad_norm": 46.152915954589844,
"learning_rate": 2.435930242225919e-07,
"logits/chosen": 1.2272932529449463,
"logits/rejected": 1.2689601182937622,
"logps/chosen": -272.54248046875,
"logps/ref_chosen": -269.83990478515625,
"logps/ref_rejected": -246.33746337890625,
"logps/rejected": -269.8098449707031,
"loss": 4.5142,
"margin_dpo/margin_mean": 20.769792556762695,
"margin_dpo/margin_std": 35.88163757324219,
"step": 267
},
{
"KL/chosen_KL_mean": 0.5022964477539062,
"KL/mean": -12.046917915344238,
"KL/rejected_KL_mean": -24.59613037109375,
"KL/std": 31.01312828063965,
"epoch": 0.5612565445026177,
"fcm_dpo/beta": 0.02405945211648941,
"fcm_dpo/delta": -0.05520617961883545,
"fcm_dpo/margin": 25.09842872619629,
"fcm_dpo/q_t": 0.3740164637565613,
"grad_norm": 44.2439079284668,
"learning_rate": 2.4176304873626984e-07,
"logits/chosen": 1.409890055656433,
"logits/rejected": 1.375052809715271,
"logps/chosen": -297.0316467285156,
"logps/ref_chosen": -297.533935546875,
"logps/ref_rejected": -270.3407897949219,
"logps/rejected": -294.9368896484375,
"loss": 4.1233,
"margin_dpo/margin_mean": 25.09842872619629,
"margin_dpo/margin_std": 34.37728500366211,
"step": 268
},
{
"KL/chosen_KL_mean": -1.2247867584228516,
"KL/mean": -11.996273040771484,
"KL/rejected_KL_mean": -22.76775550842285,
"KL/std": 30.386701583862305,
"epoch": 0.5633507853403141,
"fcm_dpo/beta": 0.02467486262321472,
"fcm_dpo/delta": 0.070290707051754,
"fcm_dpo/margin": 21.5429744720459,
"fcm_dpo/q_t": 0.3866848051548004,
"grad_norm": 84.861572265625,
"learning_rate": 2.399335149726463e-07,
"logits/chosen": 1.200810432434082,
"logits/rejected": 1.2272669076919556,
"logps/chosen": -241.5636444091797,
"logps/ref_chosen": -240.33883666992188,
"logps/ref_rejected": -220.71717834472656,
"logps/rejected": -243.48492431640625,
"loss": 4.4276,
"margin_dpo/margin_mean": 21.542972564697266,
"margin_dpo/margin_std": 35.83372497558594,
"step": 269
},
{
"KL/chosen_KL_mean": 0.8291587829589844,
"KL/mean": -11.934760093688965,
"KL/rejected_KL_mean": -24.69868278503418,
"KL/std": 31.864492416381836,
"epoch": 0.5654450261780105,
"fcm_dpo/beta": 0.024463558569550514,
"fcm_dpo/delta": -0.027698107063770294,
"fcm_dpo/margin": 25.527835845947266,
"fcm_dpo/q_t": 0.36969655752182007,
"grad_norm": 49.83897018432617,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 1.6544300317764282,
"logits/rejected": 1.634549617767334,
"logps/chosen": -308.06396484375,
"logps/ref_chosen": -308.8930969238281,
"logps/ref_rejected": -278.47869873046875,
"logps/rejected": -303.1773986816406,
"loss": 4.2113,
"margin_dpo/margin_mean": 25.527835845947266,
"margin_dpo/margin_std": 37.18598556518555,
"step": 270
},
{
"KL/chosen_KL_mean": -0.1355876922607422,
"KL/mean": -12.542990684509277,
"KL/rejected_KL_mean": -24.950380325317383,
"KL/std": 30.933940887451172,
"epoch": 0.5675392670157068,
"fcm_dpo/beta": 0.02482183650135994,
"fcm_dpo/delta": -0.018352961167693138,
"fcm_dpo/margin": 24.814781188964844,
"fcm_dpo/q_t": 0.3741310238838196,
"grad_norm": 50.110206604003906,
"learning_rate": 2.3627616503391812e-07,
"logits/chosen": 1.38817298412323,
"logits/rejected": 1.3879042863845825,
"logps/chosen": -275.5638122558594,
"logps/ref_chosen": -275.42822265625,
"logps/ref_rejected": -251.50906372070312,
"logps/rejected": -276.45941162109375,
"loss": 4.132,
"margin_dpo/margin_mean": 24.814781188964844,
"margin_dpo/margin_std": 34.804603576660156,
"step": 271
},
{
"KL/chosen_KL_mean": 1.4148082733154297,
"KL/mean": -9.921167373657227,
"KL/rejected_KL_mean": -21.25714874267578,
"KL/std": 30.931760787963867,
"epoch": 0.5696335078534032,
"fcm_dpo/beta": 0.023737920448184013,
"fcm_dpo/delta": -0.0019292645156383514,
"fcm_dpo/margin": 22.671958923339844,
"fcm_dpo/q_t": 0.386726975440979,
"grad_norm": 82.14847564697266,
"learning_rate": 2.344485449913914e-07,
"logits/chosen": 1.6356867551803589,
"logits/rejected": 1.651943564414978,
"logps/chosen": -287.96795654296875,
"logps/ref_chosen": -289.38275146484375,
"logps/ref_rejected": -274.895751953125,
"logps/rejected": -296.15289306640625,
"loss": 4.3293,
"margin_dpo/margin_mean": 22.671958923339844,
"margin_dpo/margin_std": 34.74875259399414,
"step": 272
},
{
"KL/chosen_KL_mean": 0.9697151184082031,
"KL/mean": -11.799010276794434,
"KL/rejected_KL_mean": -24.567733764648438,
"KL/std": 30.73468589782715,
"epoch": 0.5717277486910994,
"fcm_dpo/beta": 0.02284211292862892,
"fcm_dpo/delta": -0.036032721400260925,
"fcm_dpo/margin": 25.53744888305664,
"fcm_dpo/q_t": 0.3772345781326294,
"grad_norm": 41.545528411865234,
"learning_rate": 2.3262175892620062e-07,
"logits/chosen": 1.272435188293457,
"logits/rejected": 1.365143060684204,
"logps/chosen": -284.01873779296875,
"logps/ref_chosen": -284.98846435546875,
"logps/ref_rejected": -273.5077819824219,
"logps/rejected": -298.0755310058594,
"loss": 4.2081,
"margin_dpo/margin_mean": 25.53744888305664,
"margin_dpo/margin_std": 36.59501266479492,
"step": 273
},
{
"KL/chosen_KL_mean": 0.7551822662353516,
"KL/mean": -13.36454963684082,
"KL/rejected_KL_mean": -27.48428726196289,
"KL/std": 32.58087921142578,
"epoch": 0.5738219895287958,
"fcm_dpo/beta": 0.022201525047421455,
"fcm_dpo/delta": -0.09407390654087067,
"fcm_dpo/margin": 28.23946189880371,
"fcm_dpo/q_t": 0.36471086740493774,
"grad_norm": 37.03623580932617,
"learning_rate": 2.3079590480333827e-07,
"logits/chosen": 1.5402204990386963,
"logits/rejected": 1.5999386310577393,
"logps/chosen": -261.9244079589844,
"logps/ref_chosen": -262.6795959472656,
"logps/ref_rejected": -236.13792419433594,
"logps/rejected": -263.62225341796875,
"loss": 4.0284,
"margin_dpo/margin_mean": 28.239459991455078,
"margin_dpo/margin_std": 35.92232131958008,
"step": 274
},
{
"KL/chosen_KL_mean": 2.9862890243530273,
"KL/mean": -12.728757858276367,
"KL/rejected_KL_mean": -28.443809509277344,
"KL/std": 31.11075210571289,
"epoch": 0.5759162303664922,
"fcm_dpo/beta": 0.021001681685447693,
"fcm_dpo/delta": -0.06485743820667267,
"fcm_dpo/margin": 31.430097579956055,
"fcm_dpo/q_t": 0.3577304482460022,
"grad_norm": 36.05902099609375,
"learning_rate": 2.2897108053782e-07,
"logits/chosen": 1.327195167541504,
"logits/rejected": 1.4250489473342896,
"logps/chosen": -243.4132080078125,
"logps/ref_chosen": -246.39950561523438,
"logps/ref_rejected": -247.4836883544922,
"logps/rejected": -275.927490234375,
"loss": 3.8239,
"margin_dpo/margin_mean": 31.430099487304688,
"margin_dpo/margin_std": 34.86501693725586,
"step": 275
},
{
"KL/chosen_KL_mean": 4.6192169189453125,
"KL/mean": -9.579551696777344,
"KL/rejected_KL_mean": -23.778339385986328,
"KL/std": 30.969295501708984,
"epoch": 0.5780104712041885,
"fcm_dpo/beta": 0.0204936396330595,
"fcm_dpo/delta": 0.018703483045101166,
"fcm_dpo/margin": 28.39756202697754,
"fcm_dpo/q_t": 0.37401020526885986,
"grad_norm": 38.20977020263672,
"learning_rate": 2.2714738398943308e-07,
"logits/chosen": 1.6414986848831177,
"logits/rejected": 1.7540438175201416,
"logps/chosen": -284.30084228515625,
"logps/ref_chosen": -288.9200439453125,
"logps/ref_rejected": -291.69305419921875,
"logps/rejected": -315.47137451171875,
"loss": 4.0103,
"margin_dpo/margin_mean": 28.397560119628906,
"margin_dpo/margin_std": 34.412315368652344,
"step": 276
},
{
"KL/chosen_KL_mean": -2.7718963623046875,
"KL/mean": -13.085480690002441,
"KL/rejected_KL_mean": -23.399076461791992,
"KL/std": 30.761445999145508,
"epoch": 0.5801047120418849,
"fcm_dpo/beta": 0.02186032012104988,
"fcm_dpo/delta": 0.10023684799671173,
"fcm_dpo/margin": 20.627185821533203,
"fcm_dpo/q_t": 0.4030647277832031,
"grad_norm": 40.561344146728516,
"learning_rate": 2.2532491295748865e-07,
"logits/chosen": 1.16107177734375,
"logits/rejected": 1.1822755336761475,
"logps/chosen": -259.8568115234375,
"logps/ref_chosen": -257.08489990234375,
"logps/ref_rejected": -251.61532592773438,
"logps/rejected": -275.014404296875,
"loss": 4.6091,
"margin_dpo/margin_mean": 20.62718391418457,
"margin_dpo/margin_std": 37.93412780761719,
"step": 277
},
{
"KL/chosen_KL_mean": -5.834524154663086,
"KL/mean": -15.804929733276367,
"KL/rejected_KL_mean": -25.77533531188965,
"KL/std": 32.02848434448242,
"epoch": 0.5821989528795811,
"fcm_dpo/beta": 0.022726019844412804,
"fcm_dpo/delta": 0.009016614407300949,
"fcm_dpo/margin": 19.940826416015625,
"fcm_dpo/q_t": 0.40428292751312256,
"grad_norm": 43.55221176147461,
"learning_rate": 2.2350376517557726e-07,
"logits/chosen": 1.2224705219268799,
"logits/rejected": 1.1500102281570435,
"logps/chosen": -266.5758972167969,
"logps/ref_chosen": -260.7413635253906,
"logps/ref_rejected": -227.4325408935547,
"logps/rejected": -253.2078857421875,
"loss": 4.7198,
"margin_dpo/margin_mean": 19.940826416015625,
"margin_dpo/margin_std": 39.22321701049805,
"step": 278
},
{
"KL/chosen_KL_mean": -0.06951332092285156,
"KL/mean": -15.247830390930176,
"KL/rejected_KL_mean": -30.4261474609375,
"KL/std": 34.06287384033203,
"epoch": 0.5842931937172775,
"fcm_dpo/beta": 0.02161788009107113,
"fcm_dpo/delta": -0.06343191862106323,
"fcm_dpo/margin": 30.356632232666016,
"fcm_dpo/q_t": 0.3606010377407074,
"grad_norm": 39.08658981323242,
"learning_rate": 2.2168403830632769e-07,
"logits/chosen": 1.1724772453308105,
"logits/rejected": 1.2419006824493408,
"logps/chosen": -303.01702880859375,
"logps/ref_chosen": -302.947509765625,
"logps/ref_rejected": -294.1771545410156,
"logps/rejected": -324.6033020019531,
"loss": 3.9275,
"margin_dpo/margin_mean": 30.35663414001465,
"margin_dpo/margin_std": 36.632137298583984,
"step": 279
},
{
"KL/chosen_KL_mean": -0.2930927276611328,
"KL/mean": -13.874507904052734,
"KL/rejected_KL_mean": -27.45592498779297,
"KL/std": 30.778343200683594,
"epoch": 0.5863874345549738,
"fcm_dpo/beta": 0.02132745087146759,
"fcm_dpo/delta": 0.021288638934493065,
"fcm_dpo/margin": 27.16282844543457,
"fcm_dpo/q_t": 0.37750864028930664,
"grad_norm": 44.36606216430664,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": 1.2394180297851562,
"logits/rejected": 1.3001892566680908,
"logps/chosen": -258.191162109375,
"logps/ref_chosen": -257.8981018066406,
"logps/ref_rejected": -244.9084014892578,
"logps/rejected": -272.36431884765625,
"loss": 4.1394,
"margin_dpo/margin_mean": 27.162826538085938,
"margin_dpo/margin_std": 36.861446380615234,
"step": 280
},
{
"KL/chosen_KL_mean": -6.789348602294922,
"KL/mean": -19.637094497680664,
"KL/rejected_KL_mean": -32.484840393066406,
"KL/std": 33.552734375,
"epoch": 0.5884816753926702,
"fcm_dpo/beta": 0.021715322509407997,
"fcm_dpo/delta": -0.008483527228236198,
"fcm_dpo/margin": 25.695491790771484,
"fcm_dpo/q_t": 0.38245102763175964,
"grad_norm": 43.89140319824219,
"learning_rate": 2.1804923757009882e-07,
"logits/chosen": 1.3251450061798096,
"logits/rejected": 1.377968430519104,
"logps/chosen": -296.9840393066406,
"logps/ref_chosen": -290.1946716308594,
"logps/ref_rejected": -290.98883056640625,
"logps/rejected": -323.47369384765625,
"loss": 4.1787,
"margin_dpo/margin_mean": 25.695493698120117,
"margin_dpo/margin_std": 35.247562408447266,
"step": 281
},
{
"KL/chosen_KL_mean": -4.626255035400391,
"KL/mean": -17.162841796875,
"KL/rejected_KL_mean": -29.699420928955078,
"KL/std": 33.78520965576172,
"epoch": 0.5905759162303665,
"fcm_dpo/beta": 0.02181394025683403,
"fcm_dpo/delta": 0.05608820542693138,
"fcm_dpo/margin": 25.07317352294922,
"fcm_dpo/q_t": 0.3867621123790741,
"grad_norm": 48.58553695678711,
"learning_rate": 2.1623435862645205e-07,
"logits/chosen": 1.5871410369873047,
"logits/rejected": 1.6240686178207397,
"logps/chosen": -313.543701171875,
"logps/ref_chosen": -308.91741943359375,
"logps/ref_rejected": -272.2177734375,
"logps/rejected": -301.91717529296875,
"loss": 4.351,
"margin_dpo/margin_mean": 25.07317352294922,
"margin_dpo/margin_std": 39.92859649658203,
"step": 282
},
{
"KL/chosen_KL_mean": -7.476833343505859,
"KL/mean": -20.217012405395508,
"KL/rejected_KL_mean": -32.95720672607422,
"KL/std": 31.2985782623291,
"epoch": 0.5926701570680628,
"fcm_dpo/beta": 0.02321285754442215,
"fcm_dpo/delta": 0.005616554990410805,
"fcm_dpo/margin": 25.48038673400879,
"fcm_dpo/q_t": 0.3762415945529938,
"grad_norm": 44.006996154785156,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 1.5656588077545166,
"logits/rejected": 1.6084039211273193,
"logps/chosen": -255.4988250732422,
"logps/ref_chosen": -248.02200317382812,
"logps/ref_rejected": -234.78541564941406,
"logps/rejected": -267.7426452636719,
"loss": 4.2182,
"margin_dpo/margin_mean": 25.48038673400879,
"margin_dpo/margin_std": 36.498069763183594,
"step": 283
},
{
"KL/chosen_KL_mean": -7.681129455566406,
"KL/mean": -20.3056697845459,
"KL/rejected_KL_mean": -32.930213928222656,
"KL/std": 34.0436897277832,
"epoch": 0.5947643979057592,
"fcm_dpo/beta": 0.023079728707671165,
"fcm_dpo/delta": 0.015980251133441925,
"fcm_dpo/margin": 25.24908447265625,
"fcm_dpo/q_t": 0.38073068857192993,
"grad_norm": 45.13376235961914,
"learning_rate": 2.1261013021512378e-07,
"logits/chosen": 1.3176496028900146,
"logits/rejected": 1.2986741065979004,
"logps/chosen": -272.1703796386719,
"logps/ref_chosen": -264.4892578125,
"logps/ref_rejected": -255.43092346191406,
"logps/rejected": -288.36114501953125,
"loss": 4.4246,
"margin_dpo/margin_mean": 25.249080657958984,
"margin_dpo/margin_std": 42.03435134887695,
"step": 284
},
{
"KL/chosen_KL_mean": -6.775789260864258,
"KL/mean": -16.943470001220703,
"KL/rejected_KL_mean": -27.11114501953125,
"KL/std": 31.593013763427734,
"epoch": 0.5968586387434555,
"fcm_dpo/beta": 0.023795977234840393,
"fcm_dpo/delta": 0.00033229123800992966,
"fcm_dpo/margin": 20.335365295410156,
"fcm_dpo/q_t": 0.3984742760658264,
"grad_norm": 57.09181594848633,
"learning_rate": 2.1080097510381294e-07,
"logits/chosen": 1.5144256353378296,
"logits/rejected": 1.507455825805664,
"logps/chosen": -308.2445068359375,
"logps/ref_chosen": -301.46868896484375,
"logps/ref_rejected": -274.2380676269531,
"logps/rejected": -301.3492431640625,
"loss": 4.5294,
"margin_dpo/margin_mean": 20.335365295410156,
"margin_dpo/margin_std": 34.23255920410156,
"step": 285
},
{
"KL/chosen_KL_mean": -8.32232666015625,
"KL/mean": -19.552255630493164,
"KL/rejected_KL_mean": -30.782180786132812,
"KL/std": 32.217063903808594,
"epoch": 0.5989528795811518,
"fcm_dpo/beta": 0.023625340312719345,
"fcm_dpo/delta": 0.07382149249315262,
"fcm_dpo/margin": 22.4598445892334,
"fcm_dpo/q_t": 0.3923597037792206,
"grad_norm": 51.8724365234375,
"learning_rate": 2.089939221172446e-07,
"logits/chosen": 1.5711992979049683,
"logits/rejected": 1.6472262144088745,
"logps/chosen": -274.8813781738281,
"logps/ref_chosen": -266.5590515136719,
"logps/ref_rejected": -280.12066650390625,
"logps/rejected": -310.90289306640625,
"loss": 4.5259,
"margin_dpo/margin_mean": 22.4598445892334,
"margin_dpo/margin_std": 40.209190368652344,
"step": 286
},
{
"KL/chosen_KL_mean": -7.606317520141602,
"KL/mean": -19.478515625,
"KL/rejected_KL_mean": -31.35072135925293,
"KL/std": 30.595090866088867,
"epoch": 0.6010471204188481,
"fcm_dpo/beta": 0.024595845490694046,
"fcm_dpo/delta": 0.015402635559439659,
"fcm_dpo/margin": 23.74439811706543,
"fcm_dpo/q_t": 0.37745368480682373,
"grad_norm": 49.05549240112305,
"learning_rate": 2.0718906816218595e-07,
"logits/chosen": 1.2915270328521729,
"logits/rejected": 1.335796594619751,
"logps/chosen": -277.54266357421875,
"logps/ref_chosen": -269.93634033203125,
"logps/ref_rejected": -232.32330322265625,
"logps/rejected": -263.6740417480469,
"loss": 4.3272,
"margin_dpo/margin_mean": 23.744396209716797,
"margin_dpo/margin_std": 37.42576217651367,
"step": 287
},
{
"KL/chosen_KL_mean": -7.191789627075195,
"KL/mean": -18.660404205322266,
"KL/rejected_KL_mean": -30.129026412963867,
"KL/std": 32.191680908203125,
"epoch": 0.6031413612565445,
"fcm_dpo/beta": 0.02573678456246853,
"fcm_dpo/delta": 0.008166726678609848,
"fcm_dpo/margin": 22.937244415283203,
"fcm_dpo/q_t": 0.3816714882850647,
"grad_norm": 52.09629821777344,
"learning_rate": 2.053865100274774e-07,
"logits/chosen": 1.4072273969650269,
"logits/rejected": 1.3486778736114502,
"logps/chosen": -291.2373046875,
"logps/ref_chosen": -284.04547119140625,
"logps/ref_rejected": -261.19378662109375,
"logps/rejected": -291.3227844238281,
"loss": 4.3388,
"margin_dpo/margin_mean": 22.937246322631836,
"margin_dpo/margin_std": 37.41172790527344,
"step": 288
},
{
"KL/chosen_KL_mean": -10.13895034790039,
"KL/mean": -18.886932373046875,
"KL/rejected_KL_mean": -27.63491439819336,
"KL/std": 33.31560134887695,
"epoch": 0.6052356020942409,
"fcm_dpo/beta": 0.025546494871377945,
"fcm_dpo/delta": 0.05048464983701706,
"fcm_dpo/margin": 17.4959774017334,
"fcm_dpo/q_t": 0.40983152389526367,
"grad_norm": 51.257930755615234,
"learning_rate": 2.035863443788411e-07,
"logits/chosen": 1.4110479354858398,
"logits/rejected": 1.36525297164917,
"logps/chosen": -333.4201965332031,
"logps/ref_chosen": -323.28125,
"logps/ref_rejected": -274.2093200683594,
"logps/rejected": -301.84423828125,
"loss": 4.8117,
"margin_dpo/margin_mean": 17.4959774017334,
"margin_dpo/margin_std": 37.651920318603516,
"step": 289
},
{
"KL/chosen_KL_mean": -6.435283660888672,
"KL/mean": -17.43682861328125,
"KL/rejected_KL_mean": -28.43837547302246,
"KL/std": 33.041744232177734,
"epoch": 0.6073298429319371,
"fcm_dpo/beta": 0.025500264018774033,
"fcm_dpo/delta": -0.022538959980010986,
"fcm_dpo/margin": 22.00309944152832,
"fcm_dpo/q_t": 0.38546958565711975,
"grad_norm": 56.410152435302734,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": 1.3780235052108765,
"logits/rejected": 1.3338419198989868,
"logps/chosen": -323.8706970214844,
"logps/ref_chosen": -317.4353942871094,
"logps/ref_rejected": -309.32403564453125,
"logps/rejected": -337.7624206542969,
"loss": 4.38,
"margin_dpo/margin_mean": 22.00309944152832,
"margin_dpo/margin_std": 35.39474868774414,
"step": 290
},
{
"KL/chosen_KL_mean": -5.390897750854492,
"KL/mean": -20.070323944091797,
"KL/rejected_KL_mean": -34.749759674072266,
"KL/std": 34.942203521728516,
"epoch": 0.6094240837696335,
"fcm_dpo/beta": 0.02390497922897339,
"fcm_dpo/delta": -0.11430876702070236,
"fcm_dpo/margin": 29.35886001586914,
"fcm_dpo/q_t": 0.35525354743003845,
"grad_norm": 39.238685607910156,
"learning_rate": 1.9999357655598891e-07,
"logits/chosen": 1.2701756954193115,
"logits/rejected": 1.4179400205612183,
"logps/chosen": -270.8991394042969,
"logps/ref_chosen": -265.50823974609375,
"logps/ref_rejected": -275.1380310058594,
"logps/rejected": -309.8877868652344,
"loss": 3.9931,
"margin_dpo/margin_mean": 29.35886001586914,
"margin_dpo/margin_std": 38.15861511230469,
"step": 291
},
{
"KL/chosen_KL_mean": -2.940328598022461,
"KL/mean": -14.973896980285645,
"KL/rejected_KL_mean": -27.007455825805664,
"KL/std": 32.84165573120117,
"epoch": 0.6115183246073298,
"fcm_dpo/beta": 0.02307257242500782,
"fcm_dpo/delta": -0.007151174359023571,
"fcm_dpo/margin": 24.06713104248047,
"fcm_dpo/q_t": 0.3814610242843628,
"grad_norm": 52.42905044555664,
"learning_rate": 1.9820116705100775e-07,
"logits/chosen": 1.338476538658142,
"logits/rejected": 1.4540705680847168,
"logps/chosen": -252.84165954589844,
"logps/ref_chosen": -249.9013214111328,
"logps/ref_rejected": -274.58245849609375,
"logps/rejected": -301.5899353027344,
"loss": 4.2864,
"margin_dpo/margin_mean": 24.067129135131836,
"margin_dpo/margin_std": 35.604591369628906,
"step": 292
},
{
"KL/chosen_KL_mean": -8.426515579223633,
"KL/mean": -20.50110626220703,
"KL/rejected_KL_mean": -32.57570266723633,
"KL/std": 34.23207092285156,
"epoch": 0.6136125654450262,
"fcm_dpo/beta": 0.023404065519571304,
"fcm_dpo/delta": -0.01296854019165039,
"fcm_dpo/margin": 24.149171829223633,
"fcm_dpo/q_t": 0.38174688816070557,
"grad_norm": 54.628761291503906,
"learning_rate": 1.9641153536023642e-07,
"logits/chosen": 1.5635805130004883,
"logits/rejected": 1.5213494300842285,
"logps/chosen": -304.73248291015625,
"logps/ref_chosen": -296.30596923828125,
"logps/ref_rejected": -268.0640869140625,
"logps/rejected": -300.6397705078125,
"loss": 4.2688,
"margin_dpo/margin_mean": 24.149169921875,
"margin_dpo/margin_std": 36.250484466552734,
"step": 293
},
{
"KL/chosen_KL_mean": -5.804834365844727,
"KL/mean": -16.724685668945312,
"KL/rejected_KL_mean": -27.64453125,
"KL/std": 32.72875213623047,
"epoch": 0.6157068062827226,
"fcm_dpo/beta": 0.023987282067537308,
"fcm_dpo/delta": 0.07841254025697708,
"fcm_dpo/margin": 21.839691162109375,
"fcm_dpo/q_t": 0.3923855125904083,
"grad_norm": 58.72978210449219,
"learning_rate": 1.9462477745619106e-07,
"logits/chosen": 1.5025861263275146,
"logits/rejected": 1.5393353700637817,
"logps/chosen": -239.68780517578125,
"logps/ref_chosen": -233.8829803466797,
"logps/ref_rejected": -224.14334106445312,
"logps/rejected": -251.78787231445312,
"loss": 4.4377,
"margin_dpo/margin_mean": 21.839691162109375,
"margin_dpo/margin_std": 36.5875129699707,
"step": 294
},
{
"KL/chosen_KL_mean": -6.276914596557617,
"KL/mean": -18.98217010498047,
"KL/rejected_KL_mean": -31.687421798706055,
"KL/std": 33.36238479614258,
"epoch": 0.6178010471204188,
"fcm_dpo/beta": 0.02413392812013626,
"fcm_dpo/delta": -0.015403367578983307,
"fcm_dpo/margin": 25.410507202148438,
"fcm_dpo/q_t": 0.37668105959892273,
"grad_norm": 41.531402587890625,
"learning_rate": 1.928409891572757e-07,
"logits/chosen": 1.3584163188934326,
"logits/rejected": 1.348545789718628,
"logps/chosen": -250.38824462890625,
"logps/ref_chosen": -244.111328125,
"logps/ref_rejected": -240.18373107910156,
"logps/rejected": -271.8711242675781,
"loss": 4.28,
"margin_dpo/margin_mean": 25.410507202148438,
"margin_dpo/margin_std": 39.370628356933594,
"step": 295
},
{
"KL/chosen_KL_mean": -4.065727233886719,
"KL/mean": -21.36711311340332,
"KL/rejected_KL_mean": -38.66850662231445,
"KL/std": 35.191795349121094,
"epoch": 0.6198952879581152,
"fcm_dpo/beta": 0.021722082048654556,
"fcm_dpo/delta": -0.21751227974891663,
"fcm_dpo/margin": 34.602783203125,
"fcm_dpo/q_t": 0.340170294046402,
"grad_norm": 43.19810485839844,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 1.4622397422790527,
"logits/rejected": 1.6147329807281494,
"logps/chosen": -288.74627685546875,
"logps/ref_chosen": -284.6805725097656,
"logps/ref_rejected": -282.1923828125,
"logps/rejected": -320.86083984375,
"loss": 3.7668,
"margin_dpo/margin_mean": 34.602779388427734,
"margin_dpo/margin_std": 39.81512451171875,
"step": 296
},
{
"KL/chosen_KL_mean": -5.419075012207031,
"KL/mean": -18.290597915649414,
"KL/rejected_KL_mean": -31.16211700439453,
"KL/std": 34.01791763305664,
"epoch": 0.6219895287958115,
"fcm_dpo/beta": 0.020490456372499466,
"fcm_dpo/delta": 0.07718978822231293,
"fcm_dpo/margin": 25.743038177490234,
"fcm_dpo/q_t": 0.3893577754497528,
"grad_norm": 42.12018585205078,
"learning_rate": 1.8928270384706582e-07,
"logits/chosen": 1.5777825117111206,
"logits/rejected": 1.5727683305740356,
"logps/chosen": -285.647705078125,
"logps/ref_chosen": -280.2286376953125,
"logps/ref_rejected": -280.8172912597656,
"logps/rejected": -311.97943115234375,
"loss": 4.3216,
"margin_dpo/margin_mean": 25.743038177490234,
"margin_dpo/margin_std": 38.29811096191406,
"step": 297
},
{
"KL/chosen_KL_mean": -0.35713958740234375,
"KL/mean": -15.070883750915527,
"KL/rejected_KL_mean": -29.784626007080078,
"KL/std": 34.838706970214844,
"epoch": 0.6240837696335079,
"fcm_dpo/beta": 0.02112782560288906,
"fcm_dpo/delta": -0.07453294098377228,
"fcm_dpo/margin": 29.427486419677734,
"fcm_dpo/q_t": 0.3718750774860382,
"grad_norm": 42.121124267578125,
"learning_rate": 1.875083976558136e-07,
"logits/chosen": 1.0546501874923706,
"logits/rejected": 1.023137092590332,
"logps/chosen": -297.3574523925781,
"logps/ref_chosen": -297.0003356933594,
"logps/ref_rejected": -279.1561279296875,
"logps/rejected": -308.94073486328125,
"loss": 4.1183,
"margin_dpo/margin_mean": 29.4274845123291,
"margin_dpo/margin_std": 40.24848937988281,
"step": 298
},
{
"KL/chosen_KL_mean": -7.39764404296875,
"KL/mean": -19.16098403930664,
"KL/rejected_KL_mean": -30.9243106842041,
"KL/std": 34.038780212402344,
"epoch": 0.6261780104712041,
"fcm_dpo/beta": 0.0208900086581707,
"fcm_dpo/delta": 0.04286790266633034,
"fcm_dpo/margin": 23.52667236328125,
"fcm_dpo/q_t": 0.3937390148639679,
"grad_norm": 47.02155685424805,
"learning_rate": 1.8573744269954297e-07,
"logits/chosen": 1.3947925567626953,
"logits/rejected": 1.4756568670272827,
"logps/chosen": -278.86907958984375,
"logps/ref_chosen": -271.471435546875,
"logps/ref_rejected": -268.0870056152344,
"logps/rejected": -299.0113220214844,
"loss": 4.3195,
"margin_dpo/margin_mean": 23.52667236328125,
"margin_dpo/margin_std": 34.02577209472656,
"step": 299
},
{
"KL/chosen_KL_mean": -6.763256072998047,
"KL/mean": -18.935020446777344,
"KL/rejected_KL_mean": -31.10677719116211,
"KL/std": 32.86475372314453,
"epoch": 0.6282722513089005,
"fcm_dpo/beta": 0.021149903535842896,
"fcm_dpo/delta": 0.032501671463251114,
"fcm_dpo/margin": 24.343521118164062,
"fcm_dpo/q_t": 0.39046284556388855,
"grad_norm": 61.73127365112305,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": 1.5065898895263672,
"logits/rejected": 1.4867640733718872,
"logps/chosen": -313.12115478515625,
"logps/ref_chosen": -306.35784912109375,
"logps/ref_rejected": -268.64776611328125,
"logps/rejected": -299.7545471191406,
"loss": 4.3839,
"margin_dpo/margin_mean": 24.343521118164062,
"margin_dpo/margin_std": 38.673377990722656,
"step": 300
},
{
"KL/chosen_KL_mean": -7.070320129394531,
"KL/mean": -19.201602935791016,
"KL/rejected_KL_mean": -31.3328800201416,
"KL/std": 34.259525299072266,
"epoch": 0.6303664921465969,
"fcm_dpo/beta": 0.022351747378706932,
"fcm_dpo/delta": 0.06047248840332031,
"fcm_dpo/margin": 24.26255226135254,
"fcm_dpo/q_t": 0.387367308139801,
"grad_norm": 47.29561996459961,
"learning_rate": 1.8220596619089573e-07,
"logits/chosen": 1.300453782081604,
"logits/rejected": 1.3295258283615112,
"logps/chosen": -277.06280517578125,
"logps/ref_chosen": -269.9924621582031,
"logps/ref_rejected": -271.3135681152344,
"logps/rejected": -302.6464538574219,
"loss": 4.3893,
"margin_dpo/margin_mean": 24.26255226135254,
"margin_dpo/margin_std": 39.784507751464844,
"step": 301
},
{
"KL/chosen_KL_mean": -5.97479248046875,
"KL/mean": -20.132221221923828,
"KL/rejected_KL_mean": -34.289649963378906,
"KL/std": 33.1053466796875,
"epoch": 0.6324607329842932,
"fcm_dpo/beta": 0.021658364683389664,
"fcm_dpo/delta": -0.06815872341394424,
"fcm_dpo/margin": 28.314838409423828,
"fcm_dpo/q_t": 0.3689638376235962,
"grad_norm": 42.739051818847656,
"learning_rate": 1.8044563402088682e-07,
"logits/chosen": 1.5012693405151367,
"logits/rejected": 1.5727123022079468,
"logps/chosen": -266.43548583984375,
"logps/ref_chosen": -260.460693359375,
"logps/ref_rejected": -254.2005615234375,
"logps/rejected": -288.4902038574219,
"loss": 4.118,
"margin_dpo/margin_mean": 28.31484031677246,
"margin_dpo/margin_std": 38.24860763549805,
"step": 302
},
{
"KL/chosen_KL_mean": -4.667455673217773,
"KL/mean": -17.669416427612305,
"KL/rejected_KL_mean": -30.671361923217773,
"KL/std": 34.337013244628906,
"epoch": 0.6345549738219896,
"fcm_dpo/beta": 0.021459730342030525,
"fcm_dpo/delta": -0.02537161111831665,
"fcm_dpo/margin": 26.00390625,
"fcm_dpo/q_t": 0.3800218105316162,
"grad_norm": 50.90971374511719,
"learning_rate": 1.7868903184043885e-07,
"logits/chosen": 1.248102068901062,
"logits/rejected": 1.2776561975479126,
"logps/chosen": -300.76373291015625,
"logps/ref_chosen": -296.09625244140625,
"logps/ref_rejected": -264.7806701660156,
"logps/rejected": -295.4520263671875,
"loss": 4.3216,
"margin_dpo/margin_mean": 26.003908157348633,
"margin_dpo/margin_std": 39.529457092285156,
"step": 303
},
{
"KL/chosen_KL_mean": -11.185628890991211,
"KL/mean": -23.168046951293945,
"KL/rejected_KL_mean": -35.15047073364258,
"KL/std": 36.77131271362305,
"epoch": 0.6366492146596858,
"fcm_dpo/beta": 0.021864380687475204,
"fcm_dpo/delta": 0.07957626134157181,
"fcm_dpo/margin": 23.964828491210938,
"fcm_dpo/q_t": 0.3897435963153839,
"grad_norm": 56.1390495300293,
"learning_rate": 1.7693625385079574e-07,
"logits/chosen": 1.4579769372940063,
"logits/rejected": 1.455704689025879,
"logps/chosen": -293.03509521484375,
"logps/ref_chosen": -281.8494873046875,
"logps/ref_rejected": -234.3673858642578,
"logps/rejected": -269.5178527832031,
"loss": 4.3434,
"margin_dpo/margin_mean": 23.964828491210938,
"margin_dpo/margin_std": 37.54193878173828,
"step": 304
},
{
"KL/chosen_KL_mean": -1.231353759765625,
"KL/mean": -18.501249313354492,
"KL/rejected_KL_mean": -35.77115249633789,
"KL/std": 35.91838455200195,
"epoch": 0.6387434554973822,
"fcm_dpo/beta": 0.021032236516475677,
"fcm_dpo/delta": -0.13967712223529816,
"fcm_dpo/margin": 34.539798736572266,
"fcm_dpo/q_t": 0.34484899044036865,
"grad_norm": 44.405479431152344,
"learning_rate": 1.7518739404812155e-07,
"logits/chosen": 1.2939680814743042,
"logits/rejected": 1.388938069343567,
"logps/chosen": -249.2741241455078,
"logps/ref_chosen": -248.0427703857422,
"logps/ref_rejected": -263.464599609375,
"logps/rejected": -299.23577880859375,
"loss": 3.7795,
"margin_dpo/margin_mean": 34.53980255126953,
"margin_dpo/margin_std": 38.691078186035156,
"step": 305
},
{
"KL/chosen_KL_mean": -0.8278446197509766,
"KL/mean": -15.201208114624023,
"KL/rejected_KL_mean": -29.57457160949707,
"KL/std": 33.9945068359375,
"epoch": 0.6408376963350786,
"fcm_dpo/beta": 0.02030482515692711,
"fcm_dpo/delta": 0.01582380384206772,
"fcm_dpo/margin": 28.746726989746094,
"fcm_dpo/q_t": 0.37564003467559814,
"grad_norm": 42.810768127441406,
"learning_rate": 1.7344254621846017e-07,
"logits/chosen": 1.4623794555664062,
"logits/rejected": 1.3773338794708252,
"logps/chosen": -312.11376953125,
"logps/ref_chosen": -311.2859191894531,
"logps/ref_rejected": -268.4949645996094,
"logps/rejected": -298.06951904296875,
"loss": 4.142,
"margin_dpo/margin_mean": 28.746726989746094,
"margin_dpo/margin_std": 38.66185760498047,
"step": 306
},
{
"KL/chosen_KL_mean": -5.657934188842773,
"KL/mean": -21.340206146240234,
"KL/rejected_KL_mean": -37.022483825683594,
"KL/std": 36.54871368408203,
"epoch": 0.6429319371727749,
"fcm_dpo/beta": 0.01990652084350586,
"fcm_dpo/delta": -0.09277225285768509,
"fcm_dpo/margin": 31.364559173583984,
"fcm_dpo/q_t": 0.3639976978302002,
"grad_norm": 36.270023345947266,
"learning_rate": 1.717018039327053e-07,
"logits/chosen": 1.384634017944336,
"logits/rejected": 1.3476641178131104,
"logps/chosen": -275.87042236328125,
"logps/ref_chosen": -270.21246337890625,
"logps/ref_rejected": -239.1932830810547,
"logps/rejected": -276.2157287597656,
"loss": 4.0124,
"margin_dpo/margin_mean": 31.36455535888672,
"margin_dpo/margin_std": 38.58887481689453,
"step": 307
},
{
"KL/chosen_KL_mean": -7.683645248413086,
"KL/mean": -19.247112274169922,
"KL/rejected_KL_mean": -30.810577392578125,
"KL/std": 35.05336380004883,
"epoch": 0.6450261780104712,
"fcm_dpo/beta": 0.019985979422926903,
"fcm_dpo/delta": 0.14615751802921295,
"fcm_dpo/margin": 23.126928329467773,
"fcm_dpo/q_t": 0.4005933403968811,
"grad_norm": 38.301239013671875,
"learning_rate": 1.699652605415828e-07,
"logits/chosen": 1.4254227876663208,
"logits/rejected": 1.4125336408615112,
"logps/chosen": -296.20648193359375,
"logps/ref_chosen": -288.5228271484375,
"logps/ref_rejected": -253.90597534179688,
"logps/rejected": -284.7165832519531,
"loss": 4.4693,
"margin_dpo/margin_mean": 23.126930236816406,
"margin_dpo/margin_std": 38.16487121582031,
"step": 308
},
{
"KL/chosen_KL_mean": -7.721963882446289,
"KL/mean": -22.947174072265625,
"KL/rejected_KL_mean": -38.1723747253418,
"KL/std": 35.69746780395508,
"epoch": 0.6471204188481675,
"fcm_dpo/beta": 0.02104841358959675,
"fcm_dpo/delta": -0.04495348036289215,
"fcm_dpo/margin": 30.45040512084961,
"fcm_dpo/q_t": 0.36615273356437683,
"grad_norm": 41.06772994995117,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 1.2092869281768799,
"logits/rejected": 1.278046727180481,
"logps/chosen": -279.61669921875,
"logps/ref_chosen": -271.8947448730469,
"logps/ref_rejected": -260.5709228515625,
"logps/rejected": -298.7432556152344,
"loss": 4.0536,
"margin_dpo/margin_mean": 30.45040512084961,
"margin_dpo/margin_std": 40.20072937011719,
"step": 309
},
{
"KL/chosen_KL_mean": -10.000167846679688,
"KL/mean": -23.129484176635742,
"KL/rejected_KL_mean": -36.25880432128906,
"KL/std": 33.86246871948242,
"epoch": 0.6492146596858639,
"fcm_dpo/beta": 0.0207273680716753,
"fcm_dpo/delta": 0.0043011680245399475,
"fcm_dpo/margin": 26.258644104003906,
"fcm_dpo/q_t": 0.38430681824684143,
"grad_norm": 46.07544708251953,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": 1.467308759689331,
"logits/rejected": 1.573281168937683,
"logps/chosen": -280.1756286621094,
"logps/ref_chosen": -270.17547607421875,
"logps/ref_rejected": -257.6781311035156,
"logps/rejected": -293.9369201660156,
"loss": 4.2085,
"margin_dpo/margin_mean": 26.258644104003906,
"margin_dpo/margin_std": 36.735416412353516,
"step": 310
},
{
"KL/chosen_KL_mean": -9.258365631103516,
"KL/mean": -23.174116134643555,
"KL/rejected_KL_mean": -37.08986282348633,
"KL/std": 35.249542236328125,
"epoch": 0.6513089005235602,
"fcm_dpo/beta": 0.02109854482114315,
"fcm_dpo/delta": 0.011416468769311905,
"fcm_dpo/margin": 27.831506729125977,
"fcm_dpo/q_t": 0.376702219247818,
"grad_norm": 49.128501892089844,
"learning_rate": 1.647817538357072e-07,
"logits/chosen": 1.146061658859253,
"logits/rejected": 1.2810291051864624,
"logps/chosen": -273.0206298828125,
"logps/ref_chosen": -263.76226806640625,
"logps/ref_rejected": -263.8758850097656,
"logps/rejected": -300.9657287597656,
"loss": 4.2775,
"margin_dpo/margin_mean": 27.83150863647461,
"margin_dpo/margin_std": 42.079673767089844,
"step": 311
},
{
"KL/chosen_KL_mean": -8.129560470581055,
"KL/mean": -21.88031005859375,
"KL/rejected_KL_mean": -35.631072998046875,
"KL/std": 33.595943450927734,
"epoch": 0.6534031413612565,
"fcm_dpo/beta": 0.02058618888258934,
"fcm_dpo/delta": 0.034953583031892776,
"fcm_dpo/margin": 27.501510620117188,
"fcm_dpo/q_t": 0.37825676798820496,
"grad_norm": 49.96211624145508,
"learning_rate": 1.6306293495205755e-07,
"logits/chosen": 1.4143263101577759,
"logits/rejected": 1.37110435962677,
"logps/chosen": -282.8019714355469,
"logps/ref_chosen": -274.67242431640625,
"logps/ref_rejected": -242.1553955078125,
"logps/rejected": -277.7864685058594,
"loss": 4.2139,
"margin_dpo/margin_mean": 27.50151252746582,
"margin_dpo/margin_std": 38.7227668762207,
"step": 312
},
{
"KL/chosen_KL_mean": -9.555313110351562,
"KL/mean": -23.033031463623047,
"KL/rejected_KL_mean": -36.5107536315918,
"KL/std": 37.531429290771484,
"epoch": 0.6554973821989529,
"fcm_dpo/beta": 0.021746179088950157,
"fcm_dpo/delta": -0.046679336577653885,
"fcm_dpo/margin": 26.955429077148438,
"fcm_dpo/q_t": 0.3799072802066803,
"grad_norm": 41.02532958984375,
"learning_rate": 1.6134877823936607e-07,
"logits/chosen": 1.40608811378479,
"logits/rejected": 1.4458882808685303,
"logps/chosen": -304.7431640625,
"logps/ref_chosen": -295.1878662109375,
"logps/ref_rejected": -271.3431091308594,
"logps/rejected": -307.8538818359375,
"loss": 4.3841,
"margin_dpo/margin_mean": 26.955429077148438,
"margin_dpo/margin_std": 42.24140930175781,
"step": 313
},
{
"KL/chosen_KL_mean": -7.515888214111328,
"KL/mean": -20.866962432861328,
"KL/rejected_KL_mean": -34.218040466308594,
"KL/std": 35.264766693115234,
"epoch": 0.6575916230366492,
"fcm_dpo/beta": 0.02145400270819664,
"fcm_dpo/delta": 0.0263986699283123,
"fcm_dpo/margin": 26.702152252197266,
"fcm_dpo/q_t": 0.3787587881088257,
"grad_norm": 43.04774856567383,
"learning_rate": 1.5963937562265522e-07,
"logits/chosen": 1.3249597549438477,
"logits/rejected": 1.352759599685669,
"logps/chosen": -297.2837219238281,
"logps/ref_chosen": -289.767822265625,
"logps/ref_rejected": -267.0824279785156,
"logps/rejected": -301.3004455566406,
"loss": 4.2972,
"margin_dpo/margin_mean": 26.702152252197266,
"margin_dpo/margin_std": 40.670169830322266,
"step": 314
},
{
"KL/chosen_KL_mean": -6.553337097167969,
"KL/mean": -21.8732967376709,
"KL/rejected_KL_mean": -37.193267822265625,
"KL/std": 35.742828369140625,
"epoch": 0.6596858638743456,
"fcm_dpo/beta": 0.020826663821935654,
"fcm_dpo/delta": -0.04218269884586334,
"fcm_dpo/margin": 30.639928817749023,
"fcm_dpo/q_t": 0.3655908703804016,
"grad_norm": 39.36012649536133,
"learning_rate": 1.5793481877199943e-07,
"logits/chosen": 1.5957554578781128,
"logits/rejected": 1.5931099653244019,
"logps/chosen": -304.9723815917969,
"logps/ref_chosen": -298.4190673828125,
"logps/ref_rejected": -266.64447021484375,
"logps/rejected": -303.83770751953125,
"loss": 4.0497,
"margin_dpo/margin_mean": 30.63992691040039,
"margin_dpo/margin_std": 40.569557189941406,
"step": 315
},
{
"KL/chosen_KL_mean": -8.842336654663086,
"KL/mean": -25.09406089782715,
"KL/rejected_KL_mean": -41.345787048339844,
"KL/std": 36.73515701293945,
"epoch": 0.6617801047120419,
"fcm_dpo/beta": 0.019872888922691345,
"fcm_dpo/delta": -0.050521984696388245,
"fcm_dpo/margin": 32.50345230102539,
"fcm_dpo/q_t": 0.36559754610061646,
"grad_norm": 41.435325622558594,
"learning_rate": 1.562351990976095e-07,
"logits/chosen": 1.4114371538162231,
"logits/rejected": 1.3870769739151,
"logps/chosen": -311.0722351074219,
"logps/ref_chosen": -302.2298583984375,
"logps/ref_rejected": -272.7162170410156,
"logps/rejected": -314.0619812011719,
"loss": 4.0318,
"margin_dpo/margin_mean": 32.50345230102539,
"margin_dpo/margin_std": 43.02135467529297,
"step": 316
},
{
"KL/chosen_KL_mean": -5.404819488525391,
"KL/mean": -19.728431701660156,
"KL/rejected_KL_mean": -34.052059173583984,
"KL/std": 34.26416015625,
"epoch": 0.6638743455497382,
"fcm_dpo/beta": 0.019800491631031036,
"fcm_dpo/delta": 0.03399087116122246,
"fcm_dpo/margin": 28.647233963012695,
"fcm_dpo/q_t": 0.3793562948703766,
"grad_norm": 45.80251693725586,
"learning_rate": 1.5454060774493065e-07,
"logits/chosen": 1.3171772956848145,
"logits/rejected": 1.286457896232605,
"logps/chosen": -252.25823974609375,
"logps/ref_chosen": -246.85342407226562,
"logps/ref_rejected": -216.5264892578125,
"logps/rejected": -250.5785369873047,
"loss": 4.1583,
"margin_dpo/margin_mean": 28.647232055664062,
"margin_dpo/margin_std": 39.641090393066406,
"step": 317
},
{
"KL/chosen_KL_mean": -7.649684906005859,
"KL/mean": -21.736515045166016,
"KL/rejected_KL_mean": -35.82334899902344,
"KL/std": 35.691532135009766,
"epoch": 0.6659685863874345,
"fcm_dpo/beta": 0.02003423683345318,
"fcm_dpo/delta": 0.03660472482442856,
"fcm_dpo/margin": 28.173673629760742,
"fcm_dpo/q_t": 0.3774641156196594,
"grad_norm": 44.10676956176758,
"learning_rate": 1.5285113558975427e-07,
"logits/chosen": 1.3463513851165771,
"logits/rejected": 1.4206820726394653,
"logps/chosen": -270.29931640625,
"logps/ref_chosen": -262.649658203125,
"logps/ref_rejected": -251.24945068359375,
"logps/rejected": -287.0727844238281,
"loss": 4.1704,
"margin_dpo/margin_mean": 28.17367172241211,
"margin_dpo/margin_std": 38.388057708740234,
"step": 318
},
{
"KL/chosen_KL_mean": -7.615129470825195,
"KL/mean": -22.933155059814453,
"KL/rejected_KL_mean": -38.25117492675781,
"KL/std": 35.312164306640625,
"epoch": 0.6680628272251309,
"fcm_dpo/beta": 0.02006690390408039,
"fcm_dpo/delta": -0.017564807087183,
"fcm_dpo/margin": 30.636045455932617,
"fcm_dpo/q_t": 0.36597752571105957,
"grad_norm": 37.150978088378906,
"learning_rate": 1.5116687323334464e-07,
"logits/chosen": 1.3580387830734253,
"logits/rejected": 1.3618606328964233,
"logps/chosen": -301.00146484375,
"logps/ref_chosen": -293.3863220214844,
"logps/ref_rejected": -238.86026000976562,
"logps/rejected": -277.1114501953125,
"loss": 3.9492,
"margin_dpo/margin_mean": 30.636043548583984,
"margin_dpo/margin_std": 36.14667510986328,
"step": 319
},
{
"KL/chosen_KL_mean": -5.44903564453125,
"KL/mean": -20.269977569580078,
"KL/rejected_KL_mean": -35.090904235839844,
"KL/std": 35.45115280151367,
"epoch": 0.6701570680628273,
"fcm_dpo/beta": 0.0197887159883976,
"fcm_dpo/delta": -0.052189748734235764,
"fcm_dpo/margin": 29.64186668395996,
"fcm_dpo/q_t": 0.37496477365493774,
"grad_norm": 42.01469421386719,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": 1.458266019821167,
"logits/rejected": 1.4709018468856812,
"logps/chosen": -300.95281982421875,
"logps/ref_chosen": -295.5038146972656,
"logps/ref_rejected": -278.524169921875,
"logps/rejected": -313.6151123046875,
"loss": 4.2415,
"margin_dpo/margin_mean": 29.641864776611328,
"margin_dpo/margin_std": 43.217124938964844,
"step": 320
},
{
"KL/chosen_KL_mean": -8.362350463867188,
"KL/mean": -21.00874137878418,
"KL/rejected_KL_mean": -33.65511703491211,
"KL/std": 35.1718635559082,
"epoch": 0.6722513089005235,
"fcm_dpo/beta": 0.020625794306397438,
"fcm_dpo/delta": 0.08006928116083145,
"fcm_dpo/margin": 25.292766571044922,
"fcm_dpo/q_t": 0.3896982669830322,
"grad_norm": 42.11892318725586,
"learning_rate": 1.478143389201113e-07,
"logits/chosen": 1.6907308101654053,
"logits/rejected": 1.6073836088180542,
"logps/chosen": -289.11773681640625,
"logps/ref_chosen": -280.7553405761719,
"logps/ref_rejected": -237.47286987304688,
"logps/rejected": -271.12799072265625,
"loss": 4.3342,
"margin_dpo/margin_mean": 25.292766571044922,
"margin_dpo/margin_std": 39.51212692260742,
"step": 321
},
{
"KL/chosen_KL_mean": -7.008272171020508,
"KL/mean": -22.74313735961914,
"KL/rejected_KL_mean": -38.478004455566406,
"KL/std": 35.99267578125,
"epoch": 0.6743455497382199,
"fcm_dpo/beta": 0.020769845694303513,
"fcm_dpo/delta": -0.05772664397954941,
"fcm_dpo/margin": 31.4697265625,
"fcm_dpo/q_t": 0.3622688949108124,
"grad_norm": 46.31185531616211,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 1.26900053024292,
"logits/rejected": 1.255014181137085,
"logps/chosen": -306.0995788574219,
"logps/ref_chosen": -299.09130859375,
"logps/ref_rejected": -268.9637756347656,
"logps/rejected": -307.4417419433594,
"loss": 3.9975,
"margin_dpo/margin_mean": 31.4697265625,
"margin_dpo/margin_std": 40.3599739074707,
"step": 322
},
{
"KL/chosen_KL_mean": -12.073066711425781,
"KL/mean": -22.89596939086914,
"KL/rejected_KL_mean": -33.7188720703125,
"KL/std": 33.74138641357422,
"epoch": 0.6764397905759162,
"fcm_dpo/beta": 0.020806610584259033,
"fcm_dpo/delta": 0.08319887518882751,
"fcm_dpo/margin": 21.645793914794922,
"fcm_dpo/q_t": 0.40604478120803833,
"grad_norm": 50.52162551879883,
"learning_rate": 1.4448372394055246e-07,
"logits/chosen": 1.5834410190582275,
"logits/rejected": 1.522539496421814,
"logps/chosen": -288.76910400390625,
"logps/ref_chosen": -276.696044921875,
"logps/ref_rejected": -225.78790283203125,
"logps/rejected": -259.50677490234375,
"loss": 4.69,
"margin_dpo/margin_mean": 21.645793914794922,
"margin_dpo/margin_std": 42.407405853271484,
"step": 323
},
{
"KL/chosen_KL_mean": -5.5897064208984375,
"KL/mean": -22.75788688659668,
"KL/rejected_KL_mean": -39.926063537597656,
"KL/std": 35.74304962158203,
"epoch": 0.6785340314136126,
"fcm_dpo/beta": 0.01992855593562126,
"fcm_dpo/delta": -0.09492021054029465,
"fcm_dpo/margin": 34.33635330200195,
"fcm_dpo/q_t": 0.3534626364707947,
"grad_norm": 59.69330978393555,
"learning_rate": 1.428268596492364e-07,
"logits/chosen": 1.3824721574783325,
"logits/rejected": 1.4464294910430908,
"logps/chosen": -236.17471313476562,
"logps/ref_chosen": -230.58499145507812,
"logps/ref_rejected": -229.8074493408203,
"logps/rejected": -269.7335205078125,
"loss": 3.7786,
"margin_dpo/margin_mean": 34.33634948730469,
"margin_dpo/margin_std": 37.06605911254883,
"step": 324
},
{
"KL/chosen_KL_mean": -6.523805618286133,
"KL/mean": -21.514896392822266,
"KL/rejected_KL_mean": -36.50599670410156,
"KL/std": 38.17298889160156,
"epoch": 0.680628272251309,
"fcm_dpo/beta": 0.019193103536963463,
"fcm_dpo/delta": -0.02855522558093071,
"fcm_dpo/margin": 29.982200622558594,
"fcm_dpo/q_t": 0.383538156747818,
"grad_norm": 46.019588470458984,
"learning_rate": 1.4117574272818386e-07,
"logits/chosen": 1.2470699548721313,
"logits/rejected": 1.259156584739685,
"logps/chosen": -277.0798034667969,
"logps/ref_chosen": -270.5559997558594,
"logps/ref_rejected": -266.328369140625,
"logps/rejected": -302.8343505859375,
"loss": 4.3168,
"margin_dpo/margin_mean": 29.982200622558594,
"margin_dpo/margin_std": 47.193885803222656,
"step": 325
},
{
"KL/chosen_KL_mean": -6.474815368652344,
"KL/mean": -20.72551727294922,
"KL/rejected_KL_mean": -34.97622299194336,
"KL/std": 33.760223388671875,
"epoch": 0.6827225130890052,
"fcm_dpo/beta": 0.019825132563710213,
"fcm_dpo/delta": 0.035527441650629044,
"fcm_dpo/margin": 28.501415252685547,
"fcm_dpo/q_t": 0.3799126148223877,
"grad_norm": 61.182281494140625,
"learning_rate": 1.3953046172178413e-07,
"logits/chosen": 1.3199985027313232,
"logits/rejected": 1.396640419960022,
"logps/chosen": -239.71340942382812,
"logps/ref_chosen": -233.2386016845703,
"logps/ref_rejected": -258.154541015625,
"logps/rejected": -293.1307678222656,
"loss": 4.2604,
"margin_dpo/margin_mean": 28.501415252685547,
"margin_dpo/margin_std": 42.30944061279297,
"step": 326
},
{
"KL/chosen_KL_mean": -0.09135627746582031,
"KL/mean": -16.629085540771484,
"KL/rejected_KL_mean": -33.166831970214844,
"KL/std": 35.447776794433594,
"epoch": 0.6848167539267016,
"fcm_dpo/beta": 0.019445110112428665,
"fcm_dpo/delta": -0.048530131578445435,
"fcm_dpo/margin": 33.075462341308594,
"fcm_dpo/q_t": 0.3634200990200043,
"grad_norm": 36.06775665283203,
"learning_rate": 1.3789110486146468e-07,
"logits/chosen": 1.5129821300506592,
"logits/rejected": 1.5304069519042969,
"logps/chosen": -268.6375427246094,
"logps/ref_chosen": -268.5461730957031,
"logps/ref_rejected": -266.5316467285156,
"logps/rejected": -299.698486328125,
"loss": 4.0516,
"margin_dpo/margin_mean": 33.075462341308594,
"margin_dpo/margin_std": 43.234737396240234,
"step": 327
},
{
"KL/chosen_KL_mean": -8.440298080444336,
"KL/mean": -22.394554138183594,
"KL/rejected_KL_mean": -36.34880828857422,
"KL/std": 35.926353454589844,
"epoch": 0.6869109947643979,
"fcm_dpo/beta": 0.019331524148583412,
"fcm_dpo/delta": 0.06380513310432434,
"fcm_dpo/margin": 27.90850830078125,
"fcm_dpo/q_t": 0.38404858112335205,
"grad_norm": 35.696510314941406,
"learning_rate": 1.362577600609588e-07,
"logits/chosen": 1.454308271408081,
"logits/rejected": 1.4678572416305542,
"logps/chosen": -303.31707763671875,
"logps/ref_chosen": -294.8768005371094,
"logps/ref_rejected": -281.09771728515625,
"logps/rejected": -317.446533203125,
"loss": 4.2428,
"margin_dpo/margin_mean": 27.908506393432617,
"margin_dpo/margin_std": 39.95448684692383,
"step": 328
},
{
"KL/chosen_KL_mean": -11.517646789550781,
"KL/mean": -24.1033878326416,
"KL/rejected_KL_mean": -36.68913269042969,
"KL/std": 38.94247817993164,
"epoch": 0.6890052356020943,
"fcm_dpo/beta": 0.020308827981352806,
"fcm_dpo/delta": 0.041645098477602005,
"fcm_dpo/margin": 25.171483993530273,
"fcm_dpo/q_t": 0.3962605595588684,
"grad_norm": 48.052101135253906,
"learning_rate": 1.3463051491159093e-07,
"logits/chosen": 1.7028543949127197,
"logits/rejected": 1.7223467826843262,
"logps/chosen": -323.8154296875,
"logps/ref_chosen": -312.2977600097656,
"logps/ref_rejected": -304.6536560058594,
"logps/rejected": -341.3427429199219,
"loss": 4.5731,
"margin_dpo/margin_mean": 25.171485900878906,
"margin_dpo/margin_std": 45.91475296020508,
"step": 329
},
{
"KL/chosen_KL_mean": -9.744510650634766,
"KL/mean": -21.173860549926758,
"KL/rejected_KL_mean": -32.60321807861328,
"KL/std": 36.04499053955078,
"epoch": 0.6910994764397905,
"fcm_dpo/beta": 0.021884959191083908,
"fcm_dpo/delta": 0.10345837473869324,
"fcm_dpo/margin": 22.858707427978516,
"fcm_dpo/q_t": 0.3985418379306793,
"grad_norm": 49.91687774658203,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": 1.309699535369873,
"logits/rejected": 1.3560664653778076,
"logps/chosen": -303.9486999511719,
"logps/ref_chosen": -294.2042236328125,
"logps/ref_rejected": -296.2102355957031,
"logps/rejected": -328.8134765625,
"loss": 4.5652,
"margin_dpo/margin_mean": 22.858707427978516,
"margin_dpo/margin_std": 42.745452880859375,
"step": 330
},
{
"KL/chosen_KL_mean": -5.785097122192383,
"KL/mean": -20.0535945892334,
"KL/rejected_KL_mean": -34.32209777832031,
"KL/std": 36.29338455200195,
"epoch": 0.6931937172774869,
"fcm_dpo/beta": 0.021633943542838097,
"fcm_dpo/delta": -0.0712338387966156,
"fcm_dpo/margin": 28.536991119384766,
"fcm_dpo/q_t": 0.3712262809276581,
"grad_norm": 46.72981643676758,
"learning_rate": 1.3139467229135998e-07,
"logits/chosen": 1.5535022020339966,
"logits/rejected": 1.564803957939148,
"logps/chosen": -274.0977478027344,
"logps/ref_chosen": -268.3126525878906,
"logps/ref_rejected": -255.5543212890625,
"logps/rejected": -289.87640380859375,
"loss": 4.164,
"margin_dpo/margin_mean": 28.5369930267334,
"margin_dpo/margin_std": 40.84954071044922,
"step": 331
},
{
"KL/chosen_KL_mean": -7.356256484985352,
"KL/mean": -23.277719497680664,
"KL/rejected_KL_mean": -39.19919204711914,
"KL/std": 35.934471130371094,
"epoch": 0.6952879581151833,
"fcm_dpo/beta": 0.020195266231894493,
"fcm_dpo/delta": -0.04919080436229706,
"fcm_dpo/margin": 31.84294319152832,
"fcm_dpo/q_t": 0.3655468225479126,
"grad_norm": 40.15316390991211,
"learning_rate": 1.2978624834891626e-07,
"logits/chosen": 1.53472900390625,
"logits/rejected": 1.5672580003738403,
"logps/chosen": -271.106689453125,
"logps/ref_chosen": -263.75042724609375,
"logps/ref_rejected": -234.5144500732422,
"logps/rejected": -273.713623046875,
"loss": 4.0945,
"margin_dpo/margin_mean": 31.842941284179688,
"margin_dpo/margin_std": 43.65056228637695,
"step": 332
},
{
"KL/chosen_KL_mean": -4.462230682373047,
"KL/mean": -17.031757354736328,
"KL/rejected_KL_mean": -29.601285934448242,
"KL/std": 34.89313888549805,
"epoch": 0.6973821989528796,
"fcm_dpo/beta": 0.02129294164478779,
"fcm_dpo/delta": 0.06581726670265198,
"fcm_dpo/margin": 25.139057159423828,
"fcm_dpo/q_t": 0.3857528865337372,
"grad_norm": 52.96217346191406,
"learning_rate": 1.281842711051438e-07,
"logits/chosen": 1.63423490524292,
"logits/rejected": 1.6159865856170654,
"logps/chosen": -291.2786560058594,
"logps/ref_chosen": -286.81646728515625,
"logps/ref_rejected": -260.4169616699219,
"logps/rejected": -290.0182189941406,
"loss": 4.3227,
"margin_dpo/margin_mean": 25.139057159423828,
"margin_dpo/margin_std": 38.40645980834961,
"step": 333
},
{
"KL/chosen_KL_mean": -3.3436641693115234,
"KL/mean": -18.017654418945312,
"KL/rejected_KL_mean": -32.691654205322266,
"KL/std": 36.4597053527832,
"epoch": 0.6994764397905759,
"fcm_dpo/beta": 0.021236978471279144,
"fcm_dpo/delta": -0.07299379259347916,
"fcm_dpo/margin": 29.347986221313477,
"fcm_dpo/q_t": 0.3713444471359253,
"grad_norm": 48.177833557128906,
"learning_rate": 1.2658882646922033e-07,
"logits/chosen": 1.4887720346450806,
"logits/rejected": 1.4807991981506348,
"logps/chosen": -296.654541015625,
"logps/ref_chosen": -293.3109130859375,
"logps/ref_rejected": -264.1449279785156,
"logps/rejected": -296.8365783691406,
"loss": 4.2622,
"margin_dpo/margin_mean": 29.34798812866211,
"margin_dpo/margin_std": 43.27864456176758,
"step": 334
},
{
"KL/chosen_KL_mean": -11.716907501220703,
"KL/mean": -24.610498428344727,
"KL/rejected_KL_mean": -37.50409698486328,
"KL/std": 35.383934020996094,
"epoch": 0.7015706806282722,
"fcm_dpo/beta": 0.020372921600937843,
"fcm_dpo/delta": 0.014315152540802956,
"fcm_dpo/margin": 25.787187576293945,
"fcm_dpo/q_t": 0.39156395196914673,
"grad_norm": 52.540733337402344,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 1.2842155694961548,
"logits/rejected": 1.3597443103790283,
"logps/chosen": -281.4732666015625,
"logps/ref_chosen": -269.75634765625,
"logps/ref_rejected": -241.08438110351562,
"logps/rejected": -278.5884704589844,
"loss": 4.5266,
"margin_dpo/margin_mean": 25.787187576293945,
"margin_dpo/margin_std": 44.98386764526367,
"step": 335
},
{
"KL/chosen_KL_mean": -6.840425491333008,
"KL/mean": -19.60683822631836,
"KL/rejected_KL_mean": -32.373252868652344,
"KL/std": 35.83504104614258,
"epoch": 0.7036649214659686,
"fcm_dpo/beta": 0.020670827478170395,
"fcm_dpo/delta": 0.019132599234580994,
"fcm_dpo/margin": 25.532833099365234,
"fcm_dpo/q_t": 0.3904283046722412,
"grad_norm": 41.105438232421875,
"learning_rate": 1.2341787690142435e-07,
"logits/chosen": 1.408745527267456,
"logits/rejected": 1.4292922019958496,
"logps/chosen": -244.21221923828125,
"logps/ref_chosen": -237.37179565429688,
"logps/ref_rejected": -253.5463409423828,
"logps/rejected": -285.9195861816406,
"loss": 4.3834,
"margin_dpo/margin_mean": 25.532833099365234,
"margin_dpo/margin_std": 41.5406494140625,
"step": 336
},
{
"KL/chosen_KL_mean": -9.320682525634766,
"KL/mean": -23.700389862060547,
"KL/rejected_KL_mean": -38.08009338378906,
"KL/std": 35.18950653076172,
"epoch": 0.7057591623036649,
"fcm_dpo/beta": 0.020788341760635376,
"fcm_dpo/delta": -0.04489980638027191,
"fcm_dpo/margin": 28.759424209594727,
"fcm_dpo/q_t": 0.3769844174385071,
"grad_norm": 43.92770767211914,
"learning_rate": 1.2184254201795363e-07,
"logits/chosen": 1.1415191888809204,
"logits/rejected": 1.1761301755905151,
"logps/chosen": -270.62835693359375,
"logps/ref_chosen": -261.3076477050781,
"logps/ref_rejected": -252.3170166015625,
"logps/rejected": -290.3971252441406,
"loss": 4.2185,
"margin_dpo/margin_mean": 28.759424209594727,
"margin_dpo/margin_std": 42.25171661376953,
"step": 337
},
{
"KL/chosen_KL_mean": 0.043670654296875,
"KL/mean": -16.341350555419922,
"KL/rejected_KL_mean": -32.72637176513672,
"KL/std": 36.32640075683594,
"epoch": 0.7078534031413612,
"fcm_dpo/beta": 0.019904684275388718,
"fcm_dpo/delta": -0.05567874759435654,
"fcm_dpo/margin": 32.770042419433594,
"fcm_dpo/q_t": 0.36414873600006104,
"grad_norm": 44.52647018432617,
"learning_rate": 1.202740798300168e-07,
"logits/chosen": 1.5777091979980469,
"logits/rejected": 1.5485655069351196,
"logps/chosen": -268.72808837890625,
"logps/ref_chosen": -268.7717590332031,
"logps/ref_rejected": -232.3352813720703,
"logps/rejected": -265.0616455078125,
"loss": 4.0027,
"margin_dpo/margin_mean": 32.77003860473633,
"margin_dpo/margin_std": 43.178646087646484,
"step": 338
},
{
"KL/chosen_KL_mean": -3.705251693725586,
"KL/mean": -20.399065017700195,
"KL/rejected_KL_mean": -37.0928840637207,
"KL/std": 37.70600128173828,
"epoch": 0.7099476439790576,
"fcm_dpo/beta": 0.01890597492456436,
"fcm_dpo/delta": -0.0346045047044754,
"fcm_dpo/margin": 33.387630462646484,
"fcm_dpo/q_t": 0.364393949508667,
"grad_norm": 40.38267135620117,
"learning_rate": 1.1871257444948096e-07,
"logits/chosen": 1.378730297088623,
"logits/rejected": 1.336645245552063,
"logps/chosen": -297.3183898925781,
"logps/ref_chosen": -293.6131286621094,
"logps/ref_rejected": -281.87738037109375,
"logps/rejected": -318.97027587890625,
"loss": 4.0497,
"margin_dpo/margin_mean": 33.387630462646484,
"margin_dpo/margin_std": 44.12839889526367,
"step": 339
},
{
"KL/chosen_KL_mean": -7.231061935424805,
"KL/mean": -21.675649642944336,
"KL/rejected_KL_mean": -36.12025451660156,
"KL/std": 37.74037170410156,
"epoch": 0.7120418848167539,
"fcm_dpo/beta": 0.019060026854276657,
"fcm_dpo/delta": 0.05121763423085213,
"fcm_dpo/margin": 28.889205932617188,
"fcm_dpo/q_t": 0.38518282771110535,
"grad_norm": 41.96944808959961,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": 1.042682409286499,
"logits/rejected": 1.1461615562438965,
"logps/chosen": -258.9881286621094,
"logps/ref_chosen": -251.75704956054688,
"logps/ref_rejected": -255.76441955566406,
"logps/rejected": -291.8846740722656,
"loss": 4.2907,
"margin_dpo/margin_mean": 28.88920783996582,
"margin_dpo/margin_std": 43.76124572753906,
"step": 340
},
{
"KL/chosen_KL_mean": -14.124200820922852,
"KL/mean": -25.866802215576172,
"KL/rejected_KL_mean": -37.60939025878906,
"KL/std": 39.57828903198242,
"epoch": 0.7141361256544503,
"fcm_dpo/beta": 0.020194221287965775,
"fcm_dpo/delta": 0.058979712426662445,
"fcm_dpo/margin": 23.485197067260742,
"fcm_dpo/q_t": 0.40257659554481506,
"grad_norm": 52.897464752197266,
"learning_rate": 1.1561076868822755e-07,
"logits/chosen": 1.408077359199524,
"logits/rejected": 1.5080537796020508,
"logps/chosen": -320.7718200683594,
"logps/ref_chosen": -306.64764404296875,
"logps/ref_rejected": -301.4248352050781,
"logps/rejected": -339.03424072265625,
"loss": 4.6646,
"margin_dpo/margin_mean": 23.485197067260742,
"margin_dpo/margin_std": 45.1025505065918,
"step": 341
},
{
"KL/chosen_KL_mean": -4.896259307861328,
"KL/mean": -19.20393943786621,
"KL/rejected_KL_mean": -33.511627197265625,
"KL/std": 37.52045440673828,
"epoch": 0.7162303664921466,
"fcm_dpo/beta": 0.020969020202755928,
"fcm_dpo/delta": -0.0023633725941181183,
"fcm_dpo/margin": 28.615371704101562,
"fcm_dpo/q_t": 0.3745231032371521,
"grad_norm": 43.30937576293945,
"learning_rate": 1.1407063464793965e-07,
"logits/chosen": 1.527671456336975,
"logits/rejected": 1.570663332939148,
"logps/chosen": -267.565185546875,
"logps/ref_chosen": -262.6689147949219,
"logps/ref_rejected": -260.8409118652344,
"logps/rejected": -294.3525390625,
"loss": 4.1659,
"margin_dpo/margin_mean": 28.615371704101562,
"margin_dpo/margin_std": 40.10309600830078,
"step": 342
},
{
"KL/chosen_KL_mean": -4.125505447387695,
"KL/mean": -17.500391006469727,
"KL/rejected_KL_mean": -30.875286102294922,
"KL/std": 37.22210693359375,
"epoch": 0.7183246073298429,
"fcm_dpo/beta": 0.020785322412848473,
"fcm_dpo/delta": 0.04561643302440643,
"fcm_dpo/margin": 26.749792098999023,
"fcm_dpo/q_t": 0.3863937258720398,
"grad_norm": 49.587791442871094,
"learning_rate": 1.125377900869913e-07,
"logits/chosen": 1.5034934282302856,
"logits/rejected": 1.4729253053665161,
"logps/chosen": -303.75146484375,
"logps/ref_chosen": -299.6259765625,
"logps/ref_rejected": -261.2701416015625,
"logps/rejected": -292.14544677734375,
"loss": 4.3603,
"margin_dpo/margin_mean": 26.749792098999023,
"margin_dpo/margin_std": 42.81586456298828,
"step": 343
},
{
"KL/chosen_KL_mean": -5.473409652709961,
"KL/mean": -20.547622680664062,
"KL/rejected_KL_mean": -35.62184143066406,
"KL/std": 37.833587646484375,
"epoch": 0.7204188481675393,
"fcm_dpo/beta": 0.021515903994441032,
"fcm_dpo/delta": -0.053787894546985626,
"fcm_dpo/margin": 30.14841079711914,
"fcm_dpo/q_t": 0.3678366541862488,
"grad_norm": 40.71405029296875,
"learning_rate": 1.110123172071844e-07,
"logits/chosen": 1.3965474367141724,
"logits/rejected": 1.421886920928955,
"logps/chosen": -296.423828125,
"logps/ref_chosen": -290.950439453125,
"logps/ref_rejected": -255.15374755859375,
"logps/rejected": -290.77557373046875,
"loss": 4.1411,
"margin_dpo/margin_mean": 30.14841079711914,
"margin_dpo/margin_std": 42.23366165161133,
"step": 344
},
{
"KL/chosen_KL_mean": -1.812255859375,
"KL/mean": -15.496662139892578,
"KL/rejected_KL_mean": -29.181068420410156,
"KL/std": 35.286991119384766,
"epoch": 0.7225130890052356,
"fcm_dpo/beta": 0.02078464813530445,
"fcm_dpo/delta": 0.030701331794261932,
"fcm_dpo/margin": 27.368812561035156,
"fcm_dpo/q_t": 0.37889885902404785,
"grad_norm": 40.63283157348633,
"learning_rate": 1.09494297815e-07,
"logits/chosen": 1.4443119764328003,
"logits/rejected": 1.5055794715881348,
"logps/chosen": -275.078125,
"logps/ref_chosen": -273.2658996582031,
"logps/ref_rejected": -263.9664306640625,
"logps/rejected": -293.1474914550781,
"loss": 4.126,
"margin_dpo/margin_mean": 27.368812561035156,
"margin_dpo/margin_std": 36.180908203125,
"step": 345
},
{
"KL/chosen_KL_mean": -9.687026977539062,
"KL/mean": -24.494407653808594,
"KL/rejected_KL_mean": -39.30180358886719,
"KL/std": 36.424896240234375,
"epoch": 0.724607329842932,
"fcm_dpo/beta": 0.020185664296150208,
"fcm_dpo/delta": -0.05673644691705704,
"fcm_dpo/margin": 29.61478042602539,
"fcm_dpo/q_t": 0.3723044991493225,
"grad_norm": 41.669403076171875,
"learning_rate": 1.0798381331721107e-07,
"logits/chosen": 1.423187255859375,
"logits/rejected": 1.4670082330703735,
"logps/chosen": -272.36920166015625,
"logps/ref_chosen": -262.6821594238281,
"logps/ref_rejected": -226.46859741210938,
"logps/rejected": -265.7704162597656,
"loss": 4.1397,
"margin_dpo/margin_mean": 29.614784240722656,
"margin_dpo/margin_std": 38.93079376220703,
"step": 346
},
{
"KL/chosen_KL_mean": -8.110563278198242,
"KL/mean": -20.27287483215332,
"KL/rejected_KL_mean": -32.43518829345703,
"KL/std": 35.65875244140625,
"epoch": 0.7267015706806282,
"fcm_dpo/beta": 0.02065792679786682,
"fcm_dpo/delta": 0.10232961177825928,
"fcm_dpo/margin": 24.324626922607422,
"fcm_dpo/q_t": 0.39234477281570435,
"grad_norm": 47.63564682006836,
"learning_rate": 1.0648094471651722e-07,
"logits/chosen": 1.4029858112335205,
"logits/rejected": 1.4183018207550049,
"logps/chosen": -271.9351806640625,
"logps/ref_chosen": -263.8246154785156,
"logps/ref_rejected": -237.9121551513672,
"logps/rejected": -270.34735107421875,
"loss": 4.4389,
"margin_dpo/margin_mean": 24.324626922607422,
"margin_dpo/margin_std": 40.04618453979492,
"step": 347
},
{
"KL/chosen_KL_mean": -3.905031204223633,
"KL/mean": -15.264150619506836,
"KL/rejected_KL_mean": -26.623271942138672,
"KL/std": 36.13999938964844,
"epoch": 0.7287958115183246,
"fcm_dpo/beta": 0.02141868695616722,
"fcm_dpo/delta": 0.012950234115123749,
"fcm_dpo/margin": 22.718244552612305,
"fcm_dpo/q_t": 0.39967650175094604,
"grad_norm": 42.38212585449219,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 1.3707566261291504,
"logits/rejected": 1.397828459739685,
"logps/chosen": -282.058349609375,
"logps/ref_chosen": -278.1533203125,
"logps/ref_rejected": -263.39691162109375,
"logps/rejected": -290.02020263671875,
"loss": 4.4869,
"margin_dpo/margin_mean": 22.71824073791504,
"margin_dpo/margin_std": 38.25540542602539,
"step": 348
},
{
"KL/chosen_KL_mean": -6.222209930419922,
"KL/mean": -21.949100494384766,
"KL/rejected_KL_mean": -37.67598342895508,
"KL/std": 38.69367218017578,
"epoch": 0.7308900523560209,
"fcm_dpo/beta": 0.020623190328478813,
"fcm_dpo/delta": -0.11218712478876114,
"fcm_dpo/margin": 31.453781127929688,
"fcm_dpo/q_t": 0.3656691014766693,
"grad_norm": 50.22072982788086,
"learning_rate": 1.0349837717080347e-07,
"logits/chosen": 1.647545576095581,
"logits/rejected": 1.678118109703064,
"logps/chosen": -324.6772766113281,
"logps/ref_chosen": -318.455078125,
"logps/ref_rejected": -288.5691833496094,
"logps/rejected": -326.2451171875,
"loss": 4.0761,
"margin_dpo/margin_mean": 31.45378303527832,
"margin_dpo/margin_std": 42.22328186035156,
"step": 349
},
{
"KL/chosen_KL_mean": -8.401691436767578,
"KL/mean": -22.730972290039062,
"KL/rejected_KL_mean": -37.06025695800781,
"KL/std": 38.17472839355469,
"epoch": 0.7329842931937173,
"fcm_dpo/beta": 0.02000611647963524,
"fcm_dpo/delta": -0.04766222834587097,
"fcm_dpo/margin": 28.6585693359375,
"fcm_dpo/q_t": 0.3795121908187866,
"grad_norm": 44.429786682128906,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": 1.4076790809631348,
"logits/rejected": 1.3850817680358887,
"logps/chosen": -292.59088134765625,
"logps/ref_chosen": -284.1891784667969,
"logps/ref_rejected": -247.94589233398438,
"logps/rejected": -285.0061340332031,
"loss": 4.3099,
"margin_dpo/margin_mean": 28.6585693359375,
"margin_dpo/margin_std": 43.448463439941406,
"step": 350
},
{
"KL/chosen_KL_mean": -5.480752944946289,
"KL/mean": -16.321317672729492,
"KL/rejected_KL_mean": -27.161882400512695,
"KL/std": 36.58054733276367,
"epoch": 0.7350785340314137,
"fcm_dpo/beta": 0.020766399800777435,
"fcm_dpo/delta": 0.10397867113351822,
"fcm_dpo/margin": 21.68113136291504,
"fcm_dpo/q_t": 0.4100274443626404,
"grad_norm": 53.084224700927734,
"learning_rate": 1.0054723495346482e-07,
"logits/chosen": 1.6622663736343384,
"logits/rejected": 1.6453075408935547,
"logps/chosen": -308.7888488769531,
"logps/ref_chosen": -303.30810546875,
"logps/ref_rejected": -260.1866455078125,
"logps/rejected": -287.3485107421875,
"loss": 4.7544,
"margin_dpo/margin_mean": 21.681129455566406,
"margin_dpo/margin_std": 44.741676330566406,
"step": 351
},
{
"KL/chosen_KL_mean": -5.815118789672852,
"KL/mean": -21.60936737060547,
"KL/rejected_KL_mean": -37.40361785888672,
"KL/std": 37.99953842163086,
"epoch": 0.7371727748691099,
"fcm_dpo/beta": 0.020391924306750298,
"fcm_dpo/delta": -0.050486717373132706,
"fcm_dpo/margin": 31.588489532470703,
"fcm_dpo/q_t": 0.36532890796661377,
"grad_norm": 39.152435302734375,
"learning_rate": 9.908364643332398e-08,
"logits/chosen": 1.3815817832946777,
"logits/rejected": 1.5531269311904907,
"logps/chosen": -252.62222290039062,
"logps/ref_chosen": -246.80709838867188,
"logps/ref_rejected": -253.13223266601562,
"logps/rejected": -290.5358581542969,
"loss": 4.1302,
"margin_dpo/margin_mean": 31.58849334716797,
"margin_dpo/margin_std": 42.74152755737305,
"step": 352
},
{
"KL/chosen_KL_mean": -6.554887771606445,
"KL/mean": -22.182954788208008,
"KL/rejected_KL_mean": -37.81101989746094,
"KL/std": 37.12267303466797,
"epoch": 0.7392670157068063,
"fcm_dpo/beta": 0.019203264266252518,
"fcm_dpo/delta": -0.060051385313272476,
"fcm_dpo/margin": 31.256126403808594,
"fcm_dpo/q_t": 0.37281614542007446,
"grad_norm": 45.9362678527832,
"learning_rate": 9.76281510992176e-08,
"logits/chosen": 1.3499311208724976,
"logits/rejected": 1.4172637462615967,
"logps/chosen": -268.1563720703125,
"logps/ref_chosen": -261.6014709472656,
"logps/ref_rejected": -262.74761962890625,
"logps/rejected": -300.5586242675781,
"loss": 4.1085,
"margin_dpo/margin_mean": 31.256126403808594,
"margin_dpo/margin_std": 41.039066314697266,
"step": 353
},
{
"KL/chosen_KL_mean": -7.3160295486450195,
"KL/mean": -16.680030822753906,
"KL/rejected_KL_mean": -26.044034957885742,
"KL/std": 37.433170318603516,
"epoch": 0.7413612565445026,
"fcm_dpo/beta": 0.01969905197620392,
"fcm_dpo/delta": 0.08830844610929489,
"fcm_dpo/margin": 18.72800064086914,
"fcm_dpo/q_t": 0.4253513813018799,
"grad_norm": 47.81275177001953,
"learning_rate": 9.618082700494318e-08,
"logits/chosen": 1.524624228477478,
"logits/rejected": 1.634684681892395,
"logps/chosen": -259.4644470214844,
"logps/ref_chosen": -252.14840698242188,
"logps/ref_rejected": -244.1017608642578,
"logps/rejected": -270.1457824707031,
"loss": 4.984,
"margin_dpo/margin_mean": 18.72800064086914,
"margin_dpo/margin_std": 46.1493034362793,
"step": 354
},
{
"KL/chosen_KL_mean": -6.15156364440918,
"KL/mean": -22.054759979248047,
"KL/rejected_KL_mean": -37.95793533325195,
"KL/std": 38.03178787231445,
"epoch": 0.743455497382199,
"fcm_dpo/beta": 0.019542938098311424,
"fcm_dpo/delta": -0.025795362889766693,
"fcm_dpo/margin": 31.806377410888672,
"fcm_dpo/q_t": 0.36854490637779236,
"grad_norm": 38.87739562988281,
"learning_rate": 9.474175176609956e-08,
"logits/chosen": 1.4897949695587158,
"logits/rejected": 1.593730092048645,
"logps/chosen": -292.3140869140625,
"logps/ref_chosen": -286.16253662109375,
"logps/ref_rejected": -272.3275146484375,
"logps/rejected": -310.2854919433594,
"loss": 4.1449,
"margin_dpo/margin_mean": 31.806377410888672,
"margin_dpo/margin_std": 44.38508224487305,
"step": 355
},
{
"KL/chosen_KL_mean": -4.26123046875,
"KL/mean": -16.71814727783203,
"KL/rejected_KL_mean": -29.175060272216797,
"KL/std": 34.76670837402344,
"epoch": 0.7455497382198953,
"fcm_dpo/beta": 0.020574284717440605,
"fcm_dpo/delta": 0.04235248267650604,
"fcm_dpo/margin": 24.913829803466797,
"fcm_dpo/q_t": 0.39130815863609314,
"grad_norm": 41.14774703979492,
"learning_rate": 9.331100255592436e-08,
"logits/chosen": 1.5562423467636108,
"logits/rejected": 1.603265643119812,
"logps/chosen": -202.41400146484375,
"logps/ref_chosen": -198.1527862548828,
"logps/ref_rejected": -209.80569458007812,
"logps/rejected": -238.9807586669922,
"loss": 4.3745,
"margin_dpo/margin_mean": 24.91383171081543,
"margin_dpo/margin_std": 38.90260314941406,
"step": 356
},
{
"KL/chosen_KL_mean": -7.882879257202148,
"KL/mean": -22.27526092529297,
"KL/rejected_KL_mean": -36.66762924194336,
"KL/std": 36.296688079833984,
"epoch": 0.7476439790575916,
"fcm_dpo/beta": 0.02032068371772766,
"fcm_dpo/delta": -0.040481653064489365,
"fcm_dpo/margin": 28.784751892089844,
"fcm_dpo/q_t": 0.3821459114551544,
"grad_norm": 38.99663543701172,
"learning_rate": 9.18886561011557e-08,
"logits/chosen": 1.5026752948760986,
"logits/rejected": 1.5520050525665283,
"logps/chosen": -264.3900451660156,
"logps/ref_chosen": -256.50714111328125,
"logps/ref_rejected": -237.40769958496094,
"logps/rejected": -274.0753479003906,
"loss": 4.3032,
"margin_dpo/margin_mean": 28.784751892089844,
"margin_dpo/margin_std": 44.479881286621094,
"step": 357
},
{
"KL/chosen_KL_mean": -1.4076976776123047,
"KL/mean": -17.4775390625,
"KL/rejected_KL_mean": -33.547386169433594,
"KL/std": 37.1981315612793,
"epoch": 0.749738219895288,
"fcm_dpo/beta": 0.0198576208204031,
"fcm_dpo/delta": -0.04260528087615967,
"fcm_dpo/margin": 32.13968276977539,
"fcm_dpo/q_t": 0.36835816502571106,
"grad_norm": 38.77471160888672,
"learning_rate": 9.047478867791731e-08,
"logits/chosen": 1.4969563484191895,
"logits/rejected": 1.4219486713409424,
"logps/chosen": -293.85498046875,
"logps/ref_chosen": -292.447265625,
"logps/ref_rejected": -255.47872924804688,
"logps/rejected": -289.026123046875,
"loss": 4.17,
"margin_dpo/margin_mean": 32.139686584472656,
"margin_dpo/margin_std": 45.9384765625,
"step": 358
},
{
"KL/chosen_KL_mean": -1.8805828094482422,
"KL/mean": -16.736221313476562,
"KL/rejected_KL_mean": -31.591859817504883,
"KL/std": 36.91073989868164,
"epoch": 0.7518324607329843,
"fcm_dpo/beta": 0.0197370033711195,
"fcm_dpo/delta": 0.012806244194507599,
"fcm_dpo/margin": 29.71127700805664,
"fcm_dpo/q_t": 0.37310200929641724,
"grad_norm": 46.65188217163086,
"learning_rate": 8.906947610762825e-08,
"logits/chosen": 1.6143345832824707,
"logits/rejected": 1.5761593580245972,
"logps/chosen": -295.64056396484375,
"logps/ref_chosen": -293.7599792480469,
"logps/ref_rejected": -259.4142761230469,
"logps/rejected": -291.0061340332031,
"loss": 4.1267,
"margin_dpo/margin_mean": 29.71127700805664,
"margin_dpo/margin_std": 39.605995178222656,
"step": 359
},
{
"KL/chosen_KL_mean": -6.4589996337890625,
"KL/mean": -19.369590759277344,
"KL/rejected_KL_mean": -32.280174255371094,
"KL/std": 36.81121826171875,
"epoch": 0.7539267015706806,
"fcm_dpo/beta": 0.01992819271981716,
"fcm_dpo/delta": 0.03362187370657921,
"fcm_dpo/margin": 25.8211669921875,
"fcm_dpo/q_t": 0.389692485332489,
"grad_norm": 50.656654357910156,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": 1.5480406284332275,
"logits/rejected": 1.4931936264038086,
"logps/chosen": -307.37274169921875,
"logps/ref_chosen": -300.91375732421875,
"logps/ref_rejected": -256.4080810546875,
"logps/rejected": -288.688232421875,
"loss": 4.3402,
"margin_dpo/margin_mean": 25.821170806884766,
"margin_dpo/margin_std": 39.488887786865234,
"step": 360
},
{
"KL/chosen_KL_mean": -0.2387371063232422,
"KL/mean": -17.319400787353516,
"KL/rejected_KL_mean": -34.40007019042969,
"KL/std": 38.262062072753906,
"epoch": 0.7560209424083769,
"fcm_dpo/beta": 0.019601240754127502,
"fcm_dpo/delta": -0.0743849128484726,
"fcm_dpo/margin": 34.16133117675781,
"fcm_dpo/q_t": 0.3612736463546753,
"grad_norm": 38.9226188659668,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.9703270792961121,
"logits/rejected": 1.04444420337677,
"logps/chosen": -256.799072265625,
"logps/ref_chosen": -256.5603332519531,
"logps/ref_rejected": -271.043701171875,
"logps/rejected": -305.4437561035156,
"loss": 4.0002,
"margin_dpo/margin_mean": 34.16133499145508,
"margin_dpo/margin_std": 45.310611724853516,
"step": 361
},
{
"KL/chosen_KL_mean": -2.9542789459228516,
"KL/mean": -16.490875244140625,
"KL/rejected_KL_mean": -30.02747344970703,
"KL/std": 34.13772964477539,
"epoch": 0.7581151832460733,
"fcm_dpo/beta": 0.019504547119140625,
"fcm_dpo/delta": 0.07467053830623627,
"fcm_dpo/margin": 27.073196411132812,
"fcm_dpo/q_t": 0.38740891218185425,
"grad_norm": 51.599544525146484,
"learning_rate": 8.490561882286135e-08,
"logits/chosen": 1.5528841018676758,
"logits/rejected": 1.4546035528182983,
"logps/chosen": -297.65472412109375,
"logps/ref_chosen": -294.700439453125,
"logps/ref_rejected": -239.67922973632812,
"logps/rejected": -269.70672607421875,
"loss": 4.2945,
"margin_dpo/margin_mean": 27.073198318481445,
"margin_dpo/margin_std": 39.47699737548828,
"step": 362
},
{
"KL/chosen_KL_mean": -3.676485061645508,
"KL/mean": -19.368915557861328,
"KL/rejected_KL_mean": -35.06134796142578,
"KL/std": 37.644344329833984,
"epoch": 0.7602094240837697,
"fcm_dpo/beta": 0.019930703565478325,
"fcm_dpo/delta": -0.02877102419734001,
"fcm_dpo/margin": 31.38487434387207,
"fcm_dpo/q_t": 0.36726850271224976,
"grad_norm": 46.586761474609375,
"learning_rate": 8.353527464267104e-08,
"logits/chosen": 1.4351568222045898,
"logits/rejected": 1.4231568574905396,
"logps/chosen": -297.3523254394531,
"logps/ref_chosen": -293.67584228515625,
"logps/ref_rejected": -260.85675048828125,
"logps/rejected": -295.9181213378906,
"loss": 4.0882,
"margin_dpo/margin_mean": 31.38487434387207,
"margin_dpo/margin_std": 42.509033203125,
"step": 363
},
{
"KL/chosen_KL_mean": -14.330312728881836,
"KL/mean": -26.858896255493164,
"KL/rejected_KL_mean": -39.38748550415039,
"KL/std": 37.47501754760742,
"epoch": 0.762303664921466,
"fcm_dpo/beta": 0.019709203392267227,
"fcm_dpo/delta": 0.023300671949982643,
"fcm_dpo/margin": 25.057165145874023,
"fcm_dpo/q_t": 0.3976060748100281,
"grad_norm": 42.29460906982422,
"learning_rate": 8.217385746050742e-08,
"logits/chosen": 1.3992136716842651,
"logits/rejected": 1.460357666015625,
"logps/chosen": -293.2257080078125,
"logps/ref_chosen": -278.8953857421875,
"logps/ref_rejected": -283.8898620605469,
"logps/rejected": -323.2773132324219,
"loss": 4.5456,
"margin_dpo/margin_mean": 25.057165145874023,
"margin_dpo/margin_std": 44.332950592041016,
"step": 364
},
{
"KL/chosen_KL_mean": -4.5511627197265625,
"KL/mean": -19.906902313232422,
"KL/rejected_KL_mean": -35.26264190673828,
"KL/std": 38.66453552246094,
"epoch": 0.7643979057591623,
"fcm_dpo/beta": 0.02006695233285427,
"fcm_dpo/delta": -0.018149226903915405,
"fcm_dpo/margin": 30.71148681640625,
"fcm_dpo/q_t": 0.37343525886535645,
"grad_norm": 61.094390869140625,
"learning_rate": 8.082144028504231e-08,
"logits/chosen": 1.3744169473648071,
"logits/rejected": 1.406067132949829,
"logps/chosen": -270.195068359375,
"logps/ref_chosen": -265.6439208984375,
"logps/ref_rejected": -255.40138244628906,
"logps/rejected": -290.66400146484375,
"loss": 4.1269,
"margin_dpo/margin_mean": 30.71148681640625,
"margin_dpo/margin_std": 43.33749008178711,
"step": 365
},
{
"KL/chosen_KL_mean": -6.680112838745117,
"KL/mean": -22.339298248291016,
"KL/rejected_KL_mean": -37.99848937988281,
"KL/std": 38.10138702392578,
"epoch": 0.7664921465968586,
"fcm_dpo/beta": 0.019372303038835526,
"fcm_dpo/delta": -0.008491618558764458,
"fcm_dpo/margin": 31.318374633789062,
"fcm_dpo/q_t": 0.37076860666275024,
"grad_norm": 36.94200134277344,
"learning_rate": 7.947809564230445e-08,
"logits/chosen": 1.5797892808914185,
"logits/rejected": 1.5851740837097168,
"logps/chosen": -286.90460205078125,
"logps/ref_chosen": -280.2244873046875,
"logps/ref_rejected": -248.72366333007812,
"logps/rejected": -286.72210693359375,
"loss": 4.1119,
"margin_dpo/margin_mean": 31.31837272644043,
"margin_dpo/margin_std": 42.21375274658203,
"step": 366
},
{
"KL/chosen_KL_mean": -4.263193130493164,
"KL/mean": -20.494478225708008,
"KL/rejected_KL_mean": -36.72576904296875,
"KL/std": 36.97572326660156,
"epoch": 0.768586387434555,
"fcm_dpo/beta": 0.019131647422909737,
"fcm_dpo/delta": -0.023691326379776,
"fcm_dpo/margin": 32.46257400512695,
"fcm_dpo/q_t": 0.36672520637512207,
"grad_norm": 44.760868072509766,
"learning_rate": 7.814389557179016e-08,
"logits/chosen": 1.3935260772705078,
"logits/rejected": 1.3774563074111938,
"logps/chosen": -297.1998596191406,
"logps/ref_chosen": -292.9366760253906,
"logps/ref_rejected": -261.88763427734375,
"logps/rejected": -298.6133728027344,
"loss": 3.9551,
"margin_dpo/margin_mean": 32.46257400512695,
"margin_dpo/margin_std": 39.45927810668945,
"step": 367
},
{
"KL/chosen_KL_mean": -0.7215766906738281,
"KL/mean": -19.261903762817383,
"KL/rejected_KL_mean": -37.802223205566406,
"KL/std": 38.32929229736328,
"epoch": 0.7706806282722513,
"fcm_dpo/beta": 0.018251920118927956,
"fcm_dpo/delta": -0.08612095564603806,
"fcm_dpo/margin": 37.08063888549805,
"fcm_dpo/q_t": 0.35398417711257935,
"grad_norm": 38.82743453979492,
"learning_rate": 7.681891162260015e-08,
"logits/chosen": 1.420930027961731,
"logits/rejected": 1.4985603094100952,
"logps/chosen": -285.6273193359375,
"logps/ref_chosen": -284.90576171875,
"logps/ref_rejected": -256.59197998046875,
"logps/rejected": -294.3941955566406,
"loss": 3.8001,
"margin_dpo/margin_mean": 37.08063888549805,
"margin_dpo/margin_std": 40.02778244018555,
"step": 368
},
{
"KL/chosen_KL_mean": -5.350444793701172,
"KL/mean": -18.499868392944336,
"KL/rejected_KL_mean": -31.64929962158203,
"KL/std": 36.598114013671875,
"epoch": 0.7727748691099476,
"fcm_dpo/beta": 0.018250327557325363,
"fcm_dpo/delta": 0.07484041899442673,
"fcm_dpo/margin": 26.298860549926758,
"fcm_dpo/q_t": 0.397874653339386,
"grad_norm": 49.15440368652344,
"learning_rate": 7.550321484960251e-08,
"logits/chosen": 1.309835433959961,
"logits/rejected": 1.3136308193206787,
"logps/chosen": -281.266357421875,
"logps/ref_chosen": -275.91595458984375,
"logps/ref_rejected": -264.79736328125,
"logps/rejected": -296.4466552734375,
"loss": 4.4474,
"margin_dpo/margin_mean": 26.298860549926758,
"margin_dpo/margin_std": 42.30354309082031,
"step": 369
},
{
"KL/chosen_KL_mean": -4.719293594360352,
"KL/mean": -20.991464614868164,
"KL/rejected_KL_mean": -37.263633728027344,
"KL/std": 36.66276550292969,
"epoch": 0.774869109947644,
"fcm_dpo/beta": 0.01837090589106083,
"fcm_dpo/delta": -0.046606432646512985,
"fcm_dpo/margin": 32.544342041015625,
"fcm_dpo/q_t": 0.370233952999115,
"grad_norm": 34.6463623046875,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": 1.2520381212234497,
"logits/rejected": 1.2990646362304688,
"logps/chosen": -249.66131591796875,
"logps/ref_chosen": -244.9420166015625,
"logps/ref_rejected": -235.61978149414062,
"logps/rejected": -272.8834228515625,
"loss": 4.0898,
"margin_dpo/margin_mean": 32.544342041015625,
"margin_dpo/margin_std": 43.14802551269531,
"step": 370
},
{
"KL/chosen_KL_mean": -7.0436248779296875,
"KL/mean": -20.212116241455078,
"KL/rejected_KL_mean": -33.38059997558594,
"KL/std": 37.32386016845703,
"epoch": 0.7769633507853403,
"fcm_dpo/beta": 0.019493352621793747,
"fcm_dpo/delta": 0.08880892395973206,
"fcm_dpo/margin": 26.336973190307617,
"fcm_dpo/q_t": 0.39040279388427734,
"grad_norm": 55.44318771362305,
"learning_rate": 7.289996455765748e-08,
"logits/chosen": 1.2644152641296387,
"logits/rejected": 1.2208240032196045,
"logps/chosen": -296.3934631347656,
"logps/ref_chosen": -289.3498229980469,
"logps/ref_rejected": -243.70205688476562,
"logps/rejected": -277.0826416015625,
"loss": 4.3441,
"margin_dpo/margin_mean": 26.33697509765625,
"margin_dpo/margin_std": 40.09697723388672,
"step": 371
},
{
"KL/chosen_KL_mean": -3.991708755493164,
"KL/mean": -23.316816329956055,
"KL/rejected_KL_mean": -42.64191436767578,
"KL/std": 39.0753173828125,
"epoch": 0.7790575916230367,
"fcm_dpo/beta": 0.018297212198376656,
"fcm_dpo/delta": -0.11904658377170563,
"fcm_dpo/margin": 38.65019607543945,
"fcm_dpo/q_t": 0.3469123840332031,
"grad_norm": 38.49943161010742,
"learning_rate": 7.161255064312283e-08,
"logits/chosen": 1.6757351160049438,
"logits/rejected": 1.6532821655273438,
"logps/chosen": -325.6452331542969,
"logps/ref_chosen": -321.65350341796875,
"logps/ref_rejected": -282.73773193359375,
"logps/rejected": -325.379638671875,
"loss": 3.7615,
"margin_dpo/margin_mean": 38.65019226074219,
"margin_dpo/margin_std": 41.94559097290039,
"step": 372
},
{
"KL/chosen_KL_mean": -4.463361740112305,
"KL/mean": -20.560094833374023,
"KL/rejected_KL_mean": -36.656829833984375,
"KL/std": 39.52842330932617,
"epoch": 0.7811518324607329,
"fcm_dpo/beta": 0.01794876903295517,
"fcm_dpo/delta": 0.0224030502140522,
"fcm_dpo/margin": 32.19346618652344,
"fcm_dpo/q_t": 0.3757041096687317,
"grad_norm": 38.73172378540039,
"learning_rate": 7.033470310611945e-08,
"logits/chosen": 1.400538682937622,
"logits/rejected": 1.3460829257965088,
"logps/chosen": -314.2138366699219,
"logps/ref_chosen": -309.75048828125,
"logps/ref_rejected": -267.41925048828125,
"logps/rejected": -304.0760803222656,
"loss": 4.113,
"margin_dpo/margin_mean": 32.19346618652344,
"margin_dpo/margin_std": 42.02708435058594,
"step": 373
},
{
"KL/chosen_KL_mean": -5.112339019775391,
"KL/mean": -17.980426788330078,
"KL/rejected_KL_mean": -30.84852409362793,
"KL/std": 37.83140182495117,
"epoch": 0.7832460732984293,
"fcm_dpo/beta": 0.019059795886278152,
"fcm_dpo/delta": 0.11390713602304459,
"fcm_dpo/margin": 25.736186981201172,
"fcm_dpo/q_t": 0.39781954884529114,
"grad_norm": 41.872093200683594,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 1.4691405296325684,
"logits/rejected": 1.4243155717849731,
"logps/chosen": -316.1064453125,
"logps/ref_chosen": -310.9941101074219,
"logps/ref_rejected": -279.94915771484375,
"logps/rejected": -310.7976989746094,
"loss": 4.4453,
"margin_dpo/margin_mean": 25.736183166503906,
"margin_dpo/margin_std": 41.93350601196289,
"step": 374
},
{
"KL/chosen_KL_mean": -5.18425178527832,
"KL/mean": -19.81911277770996,
"KL/rejected_KL_mean": -34.45397186279297,
"KL/std": 35.83353805541992,
"epoch": 0.7853403141361257,
"fcm_dpo/beta": 0.020486168563365936,
"fcm_dpo/delta": -0.002023853361606598,
"fcm_dpo/margin": 29.269716262817383,
"fcm_dpo/q_t": 0.3779732584953308,
"grad_norm": 38.118839263916016,
"learning_rate": 6.780798075635675e-08,
"logits/chosen": 1.664709210395813,
"logits/rejected": 1.5848511457443237,
"logps/chosen": -314.3677062988281,
"logps/ref_chosen": -309.18341064453125,
"logps/ref_rejected": -256.9963073730469,
"logps/rejected": -291.45025634765625,
"loss": 4.2414,
"margin_dpo/margin_mean": 29.26971435546875,
"margin_dpo/margin_std": 44.088584899902344,
"step": 375
},
{
"KL/chosen_KL_mean": -10.31795883178711,
"KL/mean": -24.29334831237793,
"KL/rejected_KL_mean": -38.26873779296875,
"KL/std": 38.56963348388672,
"epoch": 0.787434554973822,
"fcm_dpo/beta": 0.019764304161071777,
"fcm_dpo/delta": -0.0034585334360599518,
"fcm_dpo/margin": 27.95077133178711,
"fcm_dpo/q_t": 0.38577285408973694,
"grad_norm": 43.306644439697266,
"learning_rate": 6.655924144404906e-08,
"logits/chosen": 1.6048816442489624,
"logits/rejected": 1.6655864715576172,
"logps/chosen": -292.5931396484375,
"logps/ref_chosen": -282.275146484375,
"logps/ref_rejected": -255.17941284179688,
"logps/rejected": -293.4481506347656,
"loss": 4.3495,
"margin_dpo/margin_mean": 27.950769424438477,
"margin_dpo/margin_std": 43.145328521728516,
"step": 376
},
{
"KL/chosen_KL_mean": -7.029672622680664,
"KL/mean": -20.04590606689453,
"KL/rejected_KL_mean": -33.06214141845703,
"KL/std": 36.110069274902344,
"epoch": 0.7895287958115184,
"fcm_dpo/beta": 0.020612318068742752,
"fcm_dpo/delta": 0.065596804022789,
"fcm_dpo/margin": 26.03246307373047,
"fcm_dpo/q_t": 0.39077773690223694,
"grad_norm": 40.35654067993164,
"learning_rate": 6.532033950290885e-08,
"logits/chosen": 1.4133380651474,
"logits/rejected": 1.431910514831543,
"logps/chosen": -299.9068603515625,
"logps/ref_chosen": -292.8772277832031,
"logps/ref_rejected": -266.78509521484375,
"logps/rejected": -299.84722900390625,
"loss": 4.3946,
"margin_dpo/margin_mean": 26.03246307373047,
"margin_dpo/margin_std": 42.76789855957031,
"step": 377
},
{
"KL/chosen_KL_mean": -7.506763458251953,
"KL/mean": -20.087665557861328,
"KL/rejected_KL_mean": -32.6685791015625,
"KL/std": 38.545738220214844,
"epoch": 0.7916230366492146,
"fcm_dpo/beta": 0.021401086822152138,
"fcm_dpo/delta": 0.00895449798554182,
"fcm_dpo/margin": 25.16181755065918,
"fcm_dpo/q_t": 0.38900816440582275,
"grad_norm": 48.65238571166992,
"learning_rate": 6.409134137148736e-08,
"logits/chosen": 1.5526601076126099,
"logits/rejected": 1.6031973361968994,
"logps/chosen": -280.40045166015625,
"logps/ref_chosen": -272.8937072753906,
"logps/ref_rejected": -293.058349609375,
"logps/rejected": -325.7269287109375,
"loss": 4.3716,
"margin_dpo/margin_mean": 25.16181755065918,
"margin_dpo/margin_std": 40.151939392089844,
"step": 378
},
{
"KL/chosen_KL_mean": -5.767313003540039,
"KL/mean": -19.887914657592773,
"KL/rejected_KL_mean": -34.00851821899414,
"KL/std": 35.92317581176758,
"epoch": 0.793717277486911,
"fcm_dpo/beta": 0.02095981501042843,
"fcm_dpo/delta": 0.0069956425577402115,
"fcm_dpo/margin": 28.241214752197266,
"fcm_dpo/q_t": 0.37797582149505615,
"grad_norm": 52.0435905456543,
"learning_rate": 6.28723129572247e-08,
"logits/chosen": 1.3321895599365234,
"logits/rejected": 1.3269301652908325,
"logps/chosen": -264.9089050292969,
"logps/ref_chosen": -259.1416015625,
"logps/ref_rejected": -229.64996337890625,
"logps/rejected": -263.6584777832031,
"loss": 4.2885,
"margin_dpo/margin_mean": 28.2412166595459,
"margin_dpo/margin_std": 43.94401168823242,
"step": 379
},
{
"KL/chosen_KL_mean": -9.226726531982422,
"KL/mean": -23.88406753540039,
"KL/rejected_KL_mean": -38.541419982910156,
"KL/std": 39.329925537109375,
"epoch": 0.7958115183246073,
"fcm_dpo/beta": 0.020642951130867004,
"fcm_dpo/delta": -0.07479599863290787,
"fcm_dpo/margin": 29.31468963623047,
"fcm_dpo/q_t": 0.3779948353767395,
"grad_norm": 53.2426872253418,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": 1.5265839099884033,
"logits/rejected": 1.4460859298706055,
"logps/chosen": -305.5646667480469,
"logps/ref_chosen": -296.33795166015625,
"logps/ref_rejected": -283.7359924316406,
"logps/rejected": -322.27740478515625,
"loss": 4.2068,
"margin_dpo/margin_mean": 29.31468963623047,
"margin_dpo/margin_std": 43.51792907714844,
"step": 380
},
{
"KL/chosen_KL_mean": -8.992172241210938,
"KL/mean": -23.937061309814453,
"KL/rejected_KL_mean": -38.8819465637207,
"KL/std": 38.82854461669922,
"epoch": 0.7979057591623037,
"fcm_dpo/beta": 0.019958244636654854,
"fcm_dpo/delta": 0.002718057483434677,
"fcm_dpo/margin": 29.889772415161133,
"fcm_dpo/q_t": 0.376090943813324,
"grad_norm": 59.369956970214844,
"learning_rate": 6.046442623320145e-08,
"logits/chosen": 1.263857126235962,
"logits/rejected": 1.29478120803833,
"logps/chosen": -257.96539306640625,
"logps/ref_chosen": -248.97320556640625,
"logps/ref_rejected": -259.5140075683594,
"logps/rejected": -298.3959655761719,
"loss": 4.1502,
"margin_dpo/margin_mean": 29.889774322509766,
"margin_dpo/margin_std": 42.35132598876953,
"step": 381
},
{
"KL/chosen_KL_mean": -1.8784332275390625,
"KL/mean": -19.92194175720215,
"KL/rejected_KL_mean": -37.9654541015625,
"KL/std": 37.75444793701172,
"epoch": 0.8,
"fcm_dpo/beta": 0.019227033481001854,
"fcm_dpo/delta": -0.10280030220746994,
"fcm_dpo/margin": 36.087013244628906,
"fcm_dpo/q_t": 0.35315173864364624,
"grad_norm": 46.13385009765625,
"learning_rate": 5.9275697051098275e-08,
"logits/chosen": 1.3950705528259277,
"logits/rejected": 1.5298354625701904,
"logps/chosen": -282.55072021484375,
"logps/ref_chosen": -280.67230224609375,
"logps/ref_rejected": -272.4359130859375,
"logps/rejected": -310.4013366699219,
"loss": 3.8681,
"margin_dpo/margin_mean": 36.087013244628906,
"margin_dpo/margin_std": 42.34565734863281,
"step": 382
},
{
"KL/chosen_KL_mean": -5.869083404541016,
"KL/mean": -20.48516082763672,
"KL/rejected_KL_mean": -35.10124206542969,
"KL/std": 36.95631790161133,
"epoch": 0.8020942408376963,
"fcm_dpo/beta": 0.018535105511546135,
"fcm_dpo/delta": 0.006197173148393631,
"fcm_dpo/margin": 29.232160568237305,
"fcm_dpo/q_t": 0.38371843099594116,
"grad_norm": 52.805259704589844,
"learning_rate": 5.809719583454414e-08,
"logits/chosen": 0.9901968240737915,
"logits/rejected": 1.0240157842636108,
"logps/chosen": -273.5275573730469,
"logps/ref_chosen": -267.658447265625,
"logps/ref_rejected": -262.9810791015625,
"logps/rejected": -298.0823059082031,
"loss": 4.2197,
"margin_dpo/margin_mean": 29.232160568237305,
"margin_dpo/margin_std": 40.99141311645508,
"step": 383
},
{
"KL/chosen_KL_mean": -7.576007843017578,
"KL/mean": -19.12158203125,
"KL/rejected_KL_mean": -30.667158126831055,
"KL/std": 35.436187744140625,
"epoch": 0.8041884816753927,
"fcm_dpo/beta": 0.02009403146803379,
"fcm_dpo/delta": 0.139180526137352,
"fcm_dpo/margin": 23.091142654418945,
"fcm_dpo/q_t": 0.40392279624938965,
"grad_norm": 56.3973388671875,
"learning_rate": 5.6928985782982524e-08,
"logits/chosen": 1.4375354051589966,
"logits/rejected": 1.6076993942260742,
"logps/chosen": -273.82257080078125,
"logps/ref_chosen": -266.2465515136719,
"logps/ref_rejected": -281.1450500488281,
"logps/rejected": -311.8122253417969,
"loss": 4.614,
"margin_dpo/margin_mean": 23.091140747070312,
"margin_dpo/margin_std": 43.37619400024414,
"step": 384
},
{
"KL/chosen_KL_mean": -6.043983459472656,
"KL/mean": -20.263090133666992,
"KL/rejected_KL_mean": -34.482177734375,
"KL/std": 37.30935287475586,
"epoch": 0.806282722513089,
"fcm_dpo/beta": 0.02078263834118843,
"fcm_dpo/delta": 0.007318824529647827,
"fcm_dpo/margin": 28.43819808959961,
"fcm_dpo/q_t": 0.3783205449581146,
"grad_norm": 44.51754379272461,
"learning_rate": 5.57711295439732e-08,
"logits/chosen": 1.384292721748352,
"logits/rejected": 1.3526872396469116,
"logps/chosen": -280.349853515625,
"logps/ref_chosen": -274.30584716796875,
"logps/ref_rejected": -242.94247436523438,
"logps/rejected": -277.4246520996094,
"loss": 4.2147,
"margin_dpo/margin_mean": 28.43819808959961,
"margin_dpo/margin_std": 41.831268310546875,
"step": 385
},
{
"KL/chosen_KL_mean": -3.1711044311523438,
"KL/mean": -20.52104949951172,
"KL/rejected_KL_mean": -37.871002197265625,
"KL/std": 39.688323974609375,
"epoch": 0.8083769633507853,
"fcm_dpo/beta": 0.019427498802542686,
"fcm_dpo/delta": -0.0865735337138176,
"fcm_dpo/margin": 34.69987869262695,
"fcm_dpo/q_t": 0.3566838204860687,
"grad_norm": 38.12848663330078,
"learning_rate": 5.4623689209832484e-08,
"logits/chosen": 1.7019646167755127,
"logits/rejected": 1.7641416788101196,
"logps/chosen": -312.43756103515625,
"logps/ref_chosen": -309.2664489746094,
"logps/ref_rejected": -298.4182434082031,
"logps/rejected": -336.2892150878906,
"loss": 3.9511,
"margin_dpo/margin_mean": 34.69988250732422,
"margin_dpo/margin_std": 42.274879455566406,
"step": 386
},
{
"KL/chosen_KL_mean": -3.9953250885009766,
"KL/mean": -20.024505615234375,
"KL/rejected_KL_mean": -36.05369567871094,
"KL/std": 38.43357849121094,
"epoch": 0.8104712041884817,
"fcm_dpo/beta": 0.019006196409463882,
"fcm_dpo/delta": -0.06764067709445953,
"fcm_dpo/margin": 32.05836486816406,
"fcm_dpo/q_t": 0.37299811840057373,
"grad_norm": 38.89561462402344,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 1.4557772874832153,
"logits/rejected": 1.4827194213867188,
"logps/chosen": -268.0162353515625,
"logps/ref_chosen": -264.0209045410156,
"logps/ref_rejected": -264.3447570800781,
"logps/rejected": -300.3984375,
"loss": 4.1598,
"margin_dpo/margin_mean": 32.05836486816406,
"margin_dpo/margin_std": 44.533687591552734,
"step": 387
},
{
"KL/chosen_KL_mean": -11.501548767089844,
"KL/mean": -23.445556640625,
"KL/rejected_KL_mean": -35.38957214355469,
"KL/std": 35.58702087402344,
"epoch": 0.812565445026178,
"fcm_dpo/beta": 0.018847376108169556,
"fcm_dpo/delta": 0.03600002080202103,
"fcm_dpo/margin": 23.88802719116211,
"fcm_dpo/q_t": 0.4046207666397095,
"grad_norm": 59.93172073364258,
"learning_rate": 5.2360301829254745e-08,
"logits/chosen": 1.3926148414611816,
"logits/rejected": 1.42686128616333,
"logps/chosen": -274.72039794921875,
"logps/ref_chosen": -263.2188415527344,
"logps/ref_rejected": -236.08546447753906,
"logps/rejected": -271.4750061035156,
"loss": 4.5808,
"margin_dpo/margin_mean": 23.88802719116211,
"margin_dpo/margin_std": 42.49546813964844,
"step": 388
},
{
"KL/chosen_KL_mean": -5.861154556274414,
"KL/mean": -21.513050079345703,
"KL/rejected_KL_mean": -37.16495132446289,
"KL/std": 35.916481018066406,
"epoch": 0.8146596858638744,
"fcm_dpo/beta": 0.018980950117111206,
"fcm_dpo/delta": 0.004103680606931448,
"fcm_dpo/margin": 31.303802490234375,
"fcm_dpo/q_t": 0.37250933051109314,
"grad_norm": 39.85160827636719,
"learning_rate": 5.1244476161413806e-08,
"logits/chosen": 1.433082103729248,
"logits/rejected": 1.3986594676971436,
"logps/chosen": -279.6590270996094,
"logps/ref_chosen": -273.79791259765625,
"logps/ref_rejected": -242.4090118408203,
"logps/rejected": -279.5739440917969,
"loss": 4.114,
"margin_dpo/margin_mean": 31.303802490234375,
"margin_dpo/margin_std": 41.43454360961914,
"step": 389
},
{
"KL/chosen_KL_mean": -5.90693473815918,
"KL/mean": -21.321962356567383,
"KL/rejected_KL_mean": -36.73699188232422,
"KL/std": 35.705467224121094,
"epoch": 0.8167539267015707,
"fcm_dpo/beta": 0.019459852948784828,
"fcm_dpo/delta": -0.0016894396394491196,
"fcm_dpo/margin": 30.830060958862305,
"fcm_dpo/q_t": 0.3736165165901184,
"grad_norm": 46.194740295410156,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": 1.3219387531280518,
"logits/rejected": 1.3668385744094849,
"logps/chosen": -281.22210693359375,
"logps/ref_chosen": -275.3151550292969,
"logps/ref_rejected": -278.1514587402344,
"logps/rejected": -314.88848876953125,
"loss": 4.2208,
"margin_dpo/margin_mean": 30.830059051513672,
"margin_dpo/margin_std": 44.96910858154297,
"step": 390
},
{
"KL/chosen_KL_mean": -10.438993453979492,
"KL/mean": -25.486530303955078,
"KL/rejected_KL_mean": -40.53406524658203,
"KL/std": 38.74292755126953,
"epoch": 0.818848167539267,
"fcm_dpo/beta": 0.01855631358921528,
"fcm_dpo/delta": -0.060978908091783524,
"fcm_dpo/margin": 30.095077514648438,
"fcm_dpo/q_t": 0.3802896738052368,
"grad_norm": 35.211151123046875,
"learning_rate": 4.904486005914027e-08,
"logits/chosen": 1.5178322792053223,
"logits/rejected": 1.4987064599990845,
"logps/chosen": -285.6383361816406,
"logps/ref_chosen": -275.1993408203125,
"logps/ref_rejected": -261.32122802734375,
"logps/rejected": -301.85528564453125,
"loss": 4.1482,
"margin_dpo/margin_mean": 30.095077514648438,
"margin_dpo/margin_std": 39.87967300415039,
"step": 391
},
{
"KL/chosen_KL_mean": -5.631750106811523,
"KL/mean": -25.026447296142578,
"KL/rejected_KL_mean": -44.4211540222168,
"KL/std": 38.97100830078125,
"epoch": 0.8209424083769633,
"fcm_dpo/beta": 0.016711238771677017,
"fcm_dpo/delta": -0.11873180419206619,
"fcm_dpo/margin": 38.78941345214844,
"fcm_dpo/q_t": 0.3590206503868103,
"grad_norm": 31.533557891845703,
"learning_rate": 4.796118758344353e-08,
"logits/chosen": 1.291164755821228,
"logits/rejected": 1.2510210275650024,
"logps/chosen": -305.45953369140625,
"logps/ref_chosen": -299.8277587890625,
"logps/ref_rejected": -251.24798583984375,
"logps/rejected": -295.66912841796875,
"loss": 3.8347,
"margin_dpo/margin_mean": 38.78941345214844,
"margin_dpo/margin_std": 41.62689208984375,
"step": 392
},
{
"KL/chosen_KL_mean": -5.230253219604492,
"KL/mean": -19.421533584594727,
"KL/rejected_KL_mean": -33.612815856933594,
"KL/std": 36.66984558105469,
"epoch": 0.8230366492146597,
"fcm_dpo/beta": 0.01709682308137417,
"fcm_dpo/delta": 0.062248699367046356,
"fcm_dpo/margin": 28.382564544677734,
"fcm_dpo/q_t": 0.3945763111114502,
"grad_norm": 41.22100067138672,
"learning_rate": 4.688834983610082e-08,
"logits/chosen": 1.3987441062927246,
"logits/rejected": 1.3628212213516235,
"logps/chosen": -285.69140625,
"logps/ref_chosen": -280.4610900878906,
"logps/ref_rejected": -228.34823608398438,
"logps/rejected": -261.9610290527344,
"loss": 4.385,
"margin_dpo/margin_mean": 28.382564544677734,
"margin_dpo/margin_std": 43.90695571899414,
"step": 393
},
{
"KL/chosen_KL_mean": -3.360300064086914,
"KL/mean": -16.546672821044922,
"KL/rejected_KL_mean": -29.73303985595703,
"KL/std": 35.980621337890625,
"epoch": 0.8251308900523561,
"fcm_dpo/beta": 0.01769283041357994,
"fcm_dpo/delta": 0.07207678258419037,
"fcm_dpo/margin": 26.37274169921875,
"fcm_dpo/q_t": 0.40133777260780334,
"grad_norm": 36.066593170166016,
"learning_rate": 4.582640435014459e-08,
"logits/chosen": 1.5340707302093506,
"logits/rejected": 1.6380658149719238,
"logps/chosen": -322.96038818359375,
"logps/ref_chosen": -319.60009765625,
"logps/ref_rejected": -316.42822265625,
"logps/rejected": -346.1612548828125,
"loss": 4.413,
"margin_dpo/margin_mean": 26.37274169921875,
"margin_dpo/margin_std": 40.14125061035156,
"step": 394
},
{
"KL/chosen_KL_mean": -7.686824798583984,
"KL/mean": -22.12656021118164,
"KL/rejected_KL_mean": -36.56629943847656,
"KL/std": 36.45963668823242,
"epoch": 0.8272251308900523,
"fcm_dpo/beta": 0.018843896687030792,
"fcm_dpo/delta": 0.05734879896044731,
"fcm_dpo/margin": 28.87946319580078,
"fcm_dpo/q_t": 0.38550975918769836,
"grad_norm": 43.15156173706055,
"learning_rate": 4.477540807448832e-08,
"logits/chosen": 1.1463429927825928,
"logits/rejected": 1.2481884956359863,
"logps/chosen": -268.10467529296875,
"logps/ref_chosen": -260.4178466796875,
"logps/ref_rejected": -270.04498291015625,
"logps/rejected": -306.6112976074219,
"loss": 4.2021,
"margin_dpo/margin_mean": 28.87946319580078,
"margin_dpo/margin_std": 39.44015121459961,
"step": 395
},
{
"KL/chosen_KL_mean": -6.388545989990234,
"KL/mean": -21.564329147338867,
"KL/rejected_KL_mean": -36.740116119384766,
"KL/std": 37.587928771972656,
"epoch": 0.8293193717277487,
"fcm_dpo/beta": 0.019035927951335907,
"fcm_dpo/delta": -0.04120335355401039,
"fcm_dpo/margin": 30.351564407348633,
"fcm_dpo/q_t": 0.3766162097454071,
"grad_norm": 39.945011138916016,
"learning_rate": 4.373541737087263e-08,
"logits/chosen": 1.628139853477478,
"logits/rejected": 1.6172528266906738,
"logps/chosen": -287.0915832519531,
"logps/ref_chosen": -280.7030029296875,
"logps/ref_rejected": -250.76385498046875,
"logps/rejected": -287.50396728515625,
"loss": 4.1762,
"margin_dpo/margin_mean": 30.351564407348633,
"margin_dpo/margin_std": 42.127288818359375,
"step": 396
},
{
"KL/chosen_KL_mean": -4.6651458740234375,
"KL/mean": -19.02102279663086,
"KL/rejected_KL_mean": -33.37690734863281,
"KL/std": 38.191673278808594,
"epoch": 0.831413612565445,
"fcm_dpo/beta": 0.018389301374554634,
"fcm_dpo/delta": -0.04833086580038071,
"fcm_dpo/margin": 28.711769104003906,
"fcm_dpo/q_t": 0.3868543803691864,
"grad_norm": 37.401527404785156,
"learning_rate": 4.270648801084295e-08,
"logits/chosen": 1.2988146543502808,
"logits/rejected": 1.3936009407043457,
"logps/chosen": -305.5516052246094,
"logps/ref_chosen": -300.8864440917969,
"logps/ref_rejected": -288.5682067871094,
"logps/rejected": -321.94512939453125,
"loss": 4.333,
"margin_dpo/margin_mean": 28.711769104003906,
"margin_dpo/margin_std": 42.40093231201172,
"step": 397
},
{
"KL/chosen_KL_mean": -12.974161148071289,
"KL/mean": -24.25926971435547,
"KL/rejected_KL_mean": -35.544368743896484,
"KL/std": 38.720703125,
"epoch": 0.8335078534031414,
"fcm_dpo/beta": 0.018865050747990608,
"fcm_dpo/delta": 0.12429942190647125,
"fcm_dpo/margin": 22.570209503173828,
"fcm_dpo/q_t": 0.41304242610931396,
"grad_norm": 73.18061828613281,
"learning_rate": 4.168867517275806e-08,
"logits/chosen": 1.156326174736023,
"logits/rejected": 1.1686561107635498,
"logps/chosen": -306.2765197753906,
"logps/ref_chosen": -293.3023681640625,
"logps/ref_rejected": -270.0457763671875,
"logps/rejected": -305.59014892578125,
"loss": 4.8132,
"margin_dpo/margin_mean": 22.570213317871094,
"margin_dpo/margin_std": 48.60316467285156,
"step": 398
},
{
"KL/chosen_KL_mean": -8.889374732971191,
"KL/mean": -22.127668380737305,
"KL/rejected_KL_mean": -35.365962982177734,
"KL/std": 39.10053634643555,
"epoch": 0.8356020942408376,
"fcm_dpo/beta": 0.020086301490664482,
"fcm_dpo/delta": 0.07246292382478714,
"fcm_dpo/margin": 26.476594924926758,
"fcm_dpo/q_t": 0.3878498375415802,
"grad_norm": 46.15391540527344,
"learning_rate": 4.0682033438831584e-08,
"logits/chosen": 1.3159246444702148,
"logits/rejected": 1.37637197971344,
"logps/chosen": -292.28863525390625,
"logps/ref_chosen": -283.39923095703125,
"logps/ref_rejected": -264.5092468261719,
"logps/rejected": -299.8752136230469,
"loss": 4.3662,
"margin_dpo/margin_mean": 26.476594924926758,
"margin_dpo/margin_std": 41.488059997558594,
"step": 399
},
{
"KL/chosen_KL_mean": -6.484638214111328,
"KL/mean": -21.498777389526367,
"KL/rejected_KL_mean": -36.51289749145508,
"KL/std": 35.018836975097656,
"epoch": 0.837696335078534,
"fcm_dpo/beta": 0.020130477845668793,
"fcm_dpo/delta": -0.05539043992757797,
"fcm_dpo/margin": 30.028249740600586,
"fcm_dpo/q_t": 0.37194401025772095,
"grad_norm": 43.100982666015625,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 1.3445234298706055,
"logits/rejected": 1.40453040599823,
"logps/chosen": -260.95245361328125,
"logps/ref_chosen": -254.4678192138672,
"logps/ref_rejected": -252.00326538085938,
"logps/rejected": -288.5161437988281,
"loss": 4.1465,
"margin_dpo/margin_mean": 30.028249740600586,
"margin_dpo/margin_std": 40.52297592163086,
"step": 400
},
{
"KL/chosen_KL_mean": -5.151699066162109,
"KL/mean": -23.097524642944336,
"KL/rejected_KL_mean": -41.04334259033203,
"KL/std": 39.70207595825195,
"epoch": 0.8397905759162304,
"fcm_dpo/beta": 0.019138243049383163,
"fcm_dpo/delta": -0.0941125750541687,
"fcm_dpo/margin": 35.891639709472656,
"fcm_dpo/q_t": 0.3543274402618408,
"grad_norm": 55.37236022949219,
"learning_rate": 3.8702478614051345e-08,
"logits/chosen": 1.364419937133789,
"logits/rejected": 1.390592098236084,
"logps/chosen": -315.7512512207031,
"logps/ref_chosen": -310.5995788574219,
"logps/ref_rejected": -292.7754821777344,
"logps/rejected": -333.8188171386719,
"loss": 3.8402,
"margin_dpo/margin_mean": 35.891639709472656,
"margin_dpo/margin_std": 41.956520080566406,
"step": 401
},
{
"KL/chosen_KL_mean": -3.8576431274414062,
"KL/mean": -19.410980224609375,
"KL/rejected_KL_mean": -34.964324951171875,
"KL/std": 38.21479034423828,
"epoch": 0.8418848167539267,
"fcm_dpo/beta": 0.01879115216434002,
"fcm_dpo/delta": 0.014545461162924767,
"fcm_dpo/margin": 31.106664657592773,
"fcm_dpo/q_t": 0.3764471113681793,
"grad_norm": 35.57643127441406,
"learning_rate": 3.772967168071517e-08,
"logits/chosen": 1.4840364456176758,
"logits/rejected": 1.4013035297393799,
"logps/chosen": -303.708984375,
"logps/ref_chosen": -299.8513488769531,
"logps/ref_rejected": -278.5664367675781,
"logps/rejected": -313.53076171875,
"loss": 4.1945,
"margin_dpo/margin_mean": 31.106664657592773,
"margin_dpo/margin_std": 44.161434173583984,
"step": 402
},
{
"KL/chosen_KL_mean": -3.828258514404297,
"KL/mean": -24.283838272094727,
"KL/rejected_KL_mean": -44.73942565917969,
"KL/std": 39.80998992919922,
"epoch": 0.8439790575916231,
"fcm_dpo/beta": 0.017841363325715065,
"fcm_dpo/delta": -0.13944655656814575,
"fcm_dpo/margin": 40.911163330078125,
"fcm_dpo/q_t": 0.3421737253665924,
"grad_norm": 37.96006393432617,
"learning_rate": 3.676824816087978e-08,
"logits/chosen": 1.5281907320022583,
"logits/rejected": 1.5547796487808228,
"logps/chosen": -306.2598876953125,
"logps/ref_chosen": -302.4316101074219,
"logps/ref_rejected": -270.951171875,
"logps/rejected": -315.6905822753906,
"loss": 3.6257,
"margin_dpo/margin_mean": 40.91115951538086,
"margin_dpo/margin_std": 40.92106628417969,
"step": 403
},
{
"KL/chosen_KL_mean": -9.52946662902832,
"KL/mean": -22.904190063476562,
"KL/rejected_KL_mean": -36.278907775878906,
"KL/std": 38.89113235473633,
"epoch": 0.8460732984293193,
"fcm_dpo/beta": 0.01735624298453331,
"fcm_dpo/delta": 0.07848434150218964,
"fcm_dpo/margin": 26.749435424804688,
"fcm_dpo/q_t": 0.4002022445201874,
"grad_norm": 47.30289077758789,
"learning_rate": 3.581825961277074e-08,
"logits/chosen": 1.5369975566864014,
"logits/rejected": 1.5787920951843262,
"logps/chosen": -299.4325256347656,
"logps/ref_chosen": -289.9031066894531,
"logps/ref_rejected": -252.84918212890625,
"logps/rejected": -289.1280822753906,
"loss": 4.5215,
"margin_dpo/margin_mean": 26.749435424804688,
"margin_dpo/margin_std": 44.79439163208008,
"step": 404
},
{
"KL/chosen_KL_mean": -6.036563873291016,
"KL/mean": -22.505189895629883,
"KL/rejected_KL_mean": -38.97380828857422,
"KL/std": 39.04482650756836,
"epoch": 0.8481675392670157,
"fcm_dpo/beta": 0.018190890550613403,
"fcm_dpo/delta": -0.0007007038220763206,
"fcm_dpo/margin": 32.93724822998047,
"fcm_dpo/q_t": 0.3718109130859375,
"grad_norm": 32.787559509277344,
"learning_rate": 3.487975698139084e-08,
"logits/chosen": 1.4501793384552002,
"logits/rejected": 1.6205589771270752,
"logps/chosen": -254.21060180664062,
"logps/ref_chosen": -248.17404174804688,
"logps/ref_rejected": -252.23764038085938,
"logps/rejected": -291.2114562988281,
"loss": 4.0347,
"margin_dpo/margin_mean": 32.93724822998047,
"margin_dpo/margin_std": 41.04658889770508,
"step": 405
},
{
"KL/chosen_KL_mean": -10.721895217895508,
"KL/mean": -23.478830337524414,
"KL/rejected_KL_mean": -36.23577880859375,
"KL/std": 37.815269470214844,
"epoch": 0.8502617801047121,
"fcm_dpo/beta": 0.018215851858258247,
"fcm_dpo/delta": 0.03488771989941597,
"fcm_dpo/margin": 25.51388168334961,
"fcm_dpo/q_t": 0.4009677767753601,
"grad_norm": 48.44990921020508,
"learning_rate": 3.3952790595787986e-08,
"logits/chosen": 1.2857542037963867,
"logits/rejected": 1.3423209190368652,
"logps/chosen": -287.1380310058594,
"logps/ref_chosen": -276.4161376953125,
"logps/ref_rejected": -264.0082702636719,
"logps/rejected": -300.2440185546875,
"loss": 4.5071,
"margin_dpo/margin_mean": 25.513879776000977,
"margin_dpo/margin_std": 43.34572219848633,
"step": 406
},
{
"KL/chosen_KL_mean": -13.202856063842773,
"KL/mean": -28.707468032836914,
"KL/rejected_KL_mean": -44.21208190917969,
"KL/std": 39.666770935058594,
"epoch": 0.8523560209424084,
"fcm_dpo/beta": 0.01855405420064926,
"fcm_dpo/delta": 0.02565898559987545,
"fcm_dpo/margin": 31.009233474731445,
"fcm_dpo/q_t": 0.3793700337409973,
"grad_norm": 40.636619567871094,
"learning_rate": 3.303741016635614e-08,
"logits/chosen": 1.6501725912094116,
"logits/rejected": 1.6834803819656372,
"logps/chosen": -270.64019775390625,
"logps/ref_chosen": -257.4373474121094,
"logps/ref_rejected": -219.66390991210938,
"logps/rejected": -263.8759765625,
"loss": 4.1759,
"margin_dpo/margin_mean": 31.00923728942871,
"margin_dpo/margin_std": 43.9052734375,
"step": 407
},
{
"KL/chosen_KL_mean": -7.150575637817383,
"KL/mean": -22.323060989379883,
"KL/rejected_KL_mean": -37.49555206298828,
"KL/std": 37.51593780517578,
"epoch": 0.8544502617801047,
"fcm_dpo/beta": 0.01921917498111725,
"fcm_dpo/delta": 0.014848966151475906,
"fcm_dpo/margin": 30.344974517822266,
"fcm_dpo/q_t": 0.37587589025497437,
"grad_norm": 41.318260192871094,
"learning_rate": 3.2133664782169944e-08,
"logits/chosen": 1.4881930351257324,
"logits/rejected": 1.6108531951904297,
"logps/chosen": -295.8039245605469,
"logps/ref_chosen": -288.6533508300781,
"logps/ref_rejected": -291.54107666015625,
"logps/rejected": -329.03662109375,
"loss": 4.171,
"margin_dpo/margin_mean": 30.344972610473633,
"margin_dpo/margin_std": 41.09373474121094,
"step": 408
},
{
"KL/chosen_KL_mean": -8.33292007446289,
"KL/mean": -24.719064712524414,
"KL/rejected_KL_mean": -41.1052131652832,
"KL/std": 38.697723388671875,
"epoch": 0.856544502617801,
"fcm_dpo/beta": 0.018651580438017845,
"fcm_dpo/delta": -0.013694172725081444,
"fcm_dpo/margin": 32.77228546142578,
"fcm_dpo/q_t": 0.3731399178504944,
"grad_norm": 43.54973602294922,
"learning_rate": 3.12416029083514e-08,
"logits/chosen": 1.5189157724380493,
"logits/rejected": 1.6089897155761719,
"logps/chosen": -286.0703125,
"logps/ref_chosen": -277.7373962402344,
"logps/ref_rejected": -272.988525390625,
"logps/rejected": -314.0937194824219,
"loss": 4.1718,
"margin_dpo/margin_mean": 32.77228546142578,
"margin_dpo/margin_std": 47.268409729003906,
"step": 409
},
{
"KL/chosen_KL_mean": -6.488674163818359,
"KL/mean": -20.94354820251465,
"KL/rejected_KL_mean": -35.39842224121094,
"KL/std": 38.69110870361328,
"epoch": 0.8586387434554974,
"fcm_dpo/beta": 0.01897762157022953,
"fcm_dpo/delta": -0.00017729774117469788,
"fcm_dpo/margin": 28.909753799438477,
"fcm_dpo/q_t": 0.38531434535980225,
"grad_norm": 39.235260009765625,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": 1.6496421098709106,
"logits/rejected": 1.6191202402114868,
"logps/chosen": -280.70977783203125,
"logps/ref_chosen": -274.2210998535156,
"logps/ref_rejected": -264.33465576171875,
"logps/rejected": -299.73309326171875,
"loss": 4.3552,
"margin_dpo/margin_mean": 28.90974998474121,
"margin_dpo/margin_std": 45.983367919921875,
"step": 410
},
{
"KL/chosen_KL_mean": -5.183387756347656,
"KL/mean": -21.776199340820312,
"KL/rejected_KL_mean": -38.3690071105957,
"KL/std": 39.54701232910156,
"epoch": 0.8607329842931937,
"fcm_dpo/beta": 0.018332835286855698,
"fcm_dpo/delta": -0.0687853991985321,
"fcm_dpo/margin": 33.185630798339844,
"fcm_dpo/q_t": 0.3696379065513611,
"grad_norm": 40.225460052490234,
"learning_rate": 2.9492720416985e-08,
"logits/chosen": 1.442059874534607,
"logits/rejected": 1.4909590482711792,
"logps/chosen": -278.0513916015625,
"logps/ref_chosen": -272.86798095703125,
"logps/ref_rejected": -250.35772705078125,
"logps/rejected": -288.72674560546875,
"loss": 4.0559,
"margin_dpo/margin_mean": 33.185630798339844,
"margin_dpo/margin_std": 41.469581604003906,
"step": 411
},
{
"KL/chosen_KL_mean": -10.896099090576172,
"KL/mean": -25.94062042236328,
"KL/rejected_KL_mean": -40.985145568847656,
"KL/std": 39.2274169921875,
"epoch": 0.86282722513089,
"fcm_dpo/beta": 0.017277266830205917,
"fcm_dpo/delta": -0.023805608972907066,
"fcm_dpo/margin": 30.089038848876953,
"fcm_dpo/q_t": 0.3888762593269348,
"grad_norm": 36.48346710205078,
"learning_rate": 2.863599358669755e-08,
"logits/chosen": 1.4429807662963867,
"logits/rejected": 1.544088363647461,
"logps/chosen": -278.8153076171875,
"logps/ref_chosen": -267.919189453125,
"logps/ref_rejected": -271.73162841796875,
"logps/rejected": -312.7167663574219,
"loss": 4.3299,
"margin_dpo/margin_mean": 30.089040756225586,
"margin_dpo/margin_std": 45.334686279296875,
"step": 412
},
{
"KL/chosen_KL_mean": -7.543354034423828,
"KL/mean": -20.802610397338867,
"KL/rejected_KL_mean": -34.06187057495117,
"KL/std": 37.07807922363281,
"epoch": 0.8649214659685864,
"fcm_dpo/beta": 0.018553953617811203,
"fcm_dpo/delta": 0.11212041229009628,
"fcm_dpo/margin": 26.518516540527344,
"fcm_dpo/q_t": 0.3955715298652649,
"grad_norm": 47.803489685058594,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 1.1849263906478882,
"logits/rejected": 1.2352192401885986,
"logps/chosen": -272.50140380859375,
"logps/ref_chosen": -264.95806884765625,
"logps/ref_rejected": -267.40093994140625,
"logps/rejected": -301.46282958984375,
"loss": 4.3189,
"margin_dpo/margin_mean": 26.518516540527344,
"margin_dpo/margin_std": 39.869110107421875,
"step": 413
},
{
"KL/chosen_KL_mean": -10.233379364013672,
"KL/mean": -25.646608352661133,
"KL/rejected_KL_mean": -41.059837341308594,
"KL/std": 39.12742614746094,
"epoch": 0.8670157068062827,
"fcm_dpo/beta": 0.01886744238436222,
"fcm_dpo/delta": -0.043764229863882065,
"fcm_dpo/margin": 30.826452255249023,
"fcm_dpo/q_t": 0.3794119358062744,
"grad_norm": 66.53843688964844,
"learning_rate": 2.6958198472749717e-08,
"logits/chosen": 1.5521750450134277,
"logits/rejected": 1.6116161346435547,
"logps/chosen": -300.2391357421875,
"logps/ref_chosen": -290.00579833984375,
"logps/ref_rejected": -266.8810119628906,
"logps/rejected": -307.94085693359375,
"loss": 4.2611,
"margin_dpo/margin_mean": 30.82645034790039,
"margin_dpo/margin_std": 46.020320892333984,
"step": 414
},
{
"KL/chosen_KL_mean": -5.656030654907227,
"KL/mean": -23.226139068603516,
"KL/rejected_KL_mean": -40.796260833740234,
"KL/std": 38.863365173339844,
"epoch": 0.8691099476439791,
"fcm_dpo/beta": 0.018389534205198288,
"fcm_dpo/delta": -0.049901485443115234,
"fcm_dpo/margin": 35.140228271484375,
"fcm_dpo/q_t": 0.36399033665657043,
"grad_norm": 39.664424896240234,
"learning_rate": 2.613722016414943e-08,
"logits/chosen": 1.4949822425842285,
"logits/rejected": 1.5791767835617065,
"logps/chosen": -295.467041015625,
"logps/ref_chosen": -289.81103515625,
"logps/ref_rejected": -278.2758483886719,
"logps/rejected": -319.0721435546875,
"loss": 3.9743,
"margin_dpo/margin_mean": 35.140228271484375,
"margin_dpo/margin_std": 44.041748046875,
"step": 415
},
{
"KL/chosen_KL_mean": -4.292598724365234,
"KL/mean": -21.877098083496094,
"KL/rejected_KL_mean": -39.46160125732422,
"KL/std": 38.75770568847656,
"epoch": 0.8712041884816754,
"fcm_dpo/beta": 0.016837069764733315,
"fcm_dpo/delta": -0.04827772453427315,
"fcm_dpo/margin": 35.16900634765625,
"fcm_dpo/q_t": 0.37268009781837463,
"grad_norm": 36.53273391723633,
"learning_rate": 2.5328246937043525e-08,
"logits/chosen": 1.546844482421875,
"logits/rejected": 1.51033616065979,
"logps/chosen": -308.94140625,
"logps/ref_chosen": -304.6488037109375,
"logps/ref_rejected": -264.2703857421875,
"logps/rejected": -303.73199462890625,
"loss": 4.0881,
"margin_dpo/margin_mean": 35.16900634765625,
"margin_dpo/margin_std": 45.07542419433594,
"step": 416
},
{
"KL/chosen_KL_mean": -6.947269439697266,
"KL/mean": -22.328693389892578,
"KL/rejected_KL_mean": -37.71012878417969,
"KL/std": 37.7503662109375,
"epoch": 0.8732984293193717,
"fcm_dpo/beta": 0.01717413403093815,
"fcm_dpo/delta": 0.016851838678121567,
"fcm_dpo/margin": 30.76287078857422,
"fcm_dpo/q_t": 0.3861393928527832,
"grad_norm": 41.19669723510742,
"learning_rate": 2.4531322174210973e-08,
"logits/chosen": 1.467063069343567,
"logits/rejected": 1.5336133241653442,
"logps/chosen": -308.836181640625,
"logps/ref_chosen": -301.8889465332031,
"logps/ref_rejected": -275.0347595214844,
"logps/rejected": -312.744873046875,
"loss": 4.2633,
"margin_dpo/margin_mean": 30.762866973876953,
"margin_dpo/margin_std": 43.28641891479492,
"step": 417
},
{
"KL/chosen_KL_mean": -12.334508895874023,
"KL/mean": -26.311321258544922,
"KL/rejected_KL_mean": -40.288143157958984,
"KL/std": 39.47275161743164,
"epoch": 0.875392670157068,
"fcm_dpo/beta": 0.017377493903040886,
"fcm_dpo/delta": -0.01183069497346878,
"fcm_dpo/margin": 27.953641891479492,
"fcm_dpo/q_t": 0.39519989490509033,
"grad_norm": 50.294490814208984,
"learning_rate": 2.3746488612308295e-08,
"logits/chosen": 1.4132051467895508,
"logits/rejected": 1.4223443269729614,
"logps/chosen": -281.58184814453125,
"logps/ref_chosen": -269.2473449707031,
"logps/ref_rejected": -275.5192565917969,
"logps/rejected": -315.8074035644531,
"loss": 4.3995,
"margin_dpo/margin_mean": 27.953643798828125,
"margin_dpo/margin_std": 42.52241897583008,
"step": 418
},
{
"KL/chosen_KL_mean": -11.854555130004883,
"KL/mean": -30.271154403686523,
"KL/rejected_KL_mean": -48.68775939941406,
"KL/std": 41.765953063964844,
"epoch": 0.8774869109947644,
"fcm_dpo/beta": 0.016464462503790855,
"fcm_dpo/delta": -0.05605008453130722,
"fcm_dpo/margin": 36.83320999145508,
"fcm_dpo/q_t": 0.36880791187286377,
"grad_norm": 34.323795318603516,
"learning_rate": 2.297378833957761e-08,
"logits/chosen": 1.4475170373916626,
"logits/rejected": 1.3895591497421265,
"logps/chosen": -303.01495361328125,
"logps/ref_chosen": -291.1604309082031,
"logps/ref_rejected": -245.18325805664062,
"logps/rejected": -293.8710021972656,
"loss": 4.075,
"margin_dpo/margin_mean": 36.83320999145508,
"margin_dpo/margin_std": 47.66911697387695,
"step": 419
},
{
"KL/chosen_KL_mean": -10.157136917114258,
"KL/mean": -27.602710723876953,
"KL/rejected_KL_mean": -45.048274993896484,
"KL/std": 40.69925308227539,
"epoch": 0.8795811518324608,
"fcm_dpo/beta": 0.016606658697128296,
"fcm_dpo/delta": 0.020674100145697594,
"fcm_dpo/margin": 34.891143798828125,
"fcm_dpo/q_t": 0.37389612197875977,
"grad_norm": 38.67104721069336,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": 1.411975622177124,
"logits/rejected": 1.364816427230835,
"logps/chosen": -268.6269836425781,
"logps/ref_chosen": -258.4698181152344,
"logps/ref_rejected": -242.916015625,
"logps/rejected": -287.96429443359375,
"loss": 4.1057,
"margin_dpo/margin_mean": 34.891143798828125,
"margin_dpo/margin_std": 45.79709243774414,
"step": 420
},
{
"KL/chosen_KL_mean": -3.941652297973633,
"KL/mean": -20.879179000854492,
"KL/rejected_KL_mean": -37.816707611083984,
"KL/std": 37.67058563232422,
"epoch": 0.881675392670157,
"fcm_dpo/beta": 0.01673482172191143,
"fcm_dpo/delta": 0.034025367349386215,
"fcm_dpo/margin": 33.87507247924805,
"fcm_dpo/q_t": 0.37542274594306946,
"grad_norm": 33.99753189086914,
"learning_rate": 2.1464952759020856e-08,
"logits/chosen": 1.2181888818740845,
"logits/rejected": 1.1769070625305176,
"logps/chosen": -294.0824279785156,
"logps/ref_chosen": -290.1407775878906,
"logps/ref_rejected": -239.65676879882812,
"logps/rejected": -277.47344970703125,
"loss": 4.0449,
"margin_dpo/margin_mean": 33.87507247924805,
"margin_dpo/margin_std": 40.482669830322266,
"step": 421
},
{
"KL/chosen_KL_mean": -8.990995407104492,
"KL/mean": -22.929763793945312,
"KL/rejected_KL_mean": -36.8685302734375,
"KL/std": 37.68763732910156,
"epoch": 0.8837696335078534,
"fcm_dpo/beta": 0.018041063100099564,
"fcm_dpo/delta": 0.10089747607707977,
"fcm_dpo/margin": 27.877517700195312,
"fcm_dpo/q_t": 0.392340749502182,
"grad_norm": 46.466827392578125,
"learning_rate": 2.07288983654679e-08,
"logits/chosen": 1.280303716659546,
"logits/rejected": 1.326034426689148,
"logps/chosen": -285.9270324707031,
"logps/ref_chosen": -276.93603515625,
"logps/ref_rejected": -250.398681640625,
"logps/rejected": -287.2672119140625,
"loss": 4.4266,
"margin_dpo/margin_mean": 27.877519607543945,
"margin_dpo/margin_std": 45.0127067565918,
"step": 422
},
{
"KL/chosen_KL_mean": -6.769617080688477,
"KL/mean": -24.917064666748047,
"KL/rejected_KL_mean": -43.06451416015625,
"KL/std": 40.1577262878418,
"epoch": 0.8858638743455497,
"fcm_dpo/beta": 0.018019232898950577,
"fcm_dpo/delta": -0.05869268625974655,
"fcm_dpo/margin": 36.29489517211914,
"fcm_dpo/q_t": 0.3627795875072479,
"grad_norm": 34.83606719970703,
"learning_rate": 2.0005139085293942e-08,
"logits/chosen": 1.4147975444793701,
"logits/rejected": 1.4877537488937378,
"logps/chosen": -292.19970703125,
"logps/ref_chosen": -285.43011474609375,
"logps/ref_rejected": -258.0743408203125,
"logps/rejected": -301.13885498046875,
"loss": 3.9663,
"margin_dpo/margin_mean": 36.294891357421875,
"margin_dpo/margin_std": 45.899147033691406,
"step": 423
},
{
"KL/chosen_KL_mean": -4.655523300170898,
"KL/mean": -21.756614685058594,
"KL/rejected_KL_mean": -38.85771179199219,
"KL/std": 37.40180969238281,
"epoch": 0.8879581151832461,
"fcm_dpo/beta": 0.017788197845220566,
"fcm_dpo/delta": -0.010666869580745697,
"fcm_dpo/margin": 34.20218276977539,
"fcm_dpo/q_t": 0.3682326078414917,
"grad_norm": 37.221805572509766,
"learning_rate": 1.9293713731512673e-08,
"logits/chosen": 1.4349040985107422,
"logits/rejected": 1.4128334522247314,
"logps/chosen": -304.0005187988281,
"logps/ref_chosen": -299.3450012207031,
"logps/ref_rejected": -244.60169982910156,
"logps/rejected": -283.45941162109375,
"loss": 4.0036,
"margin_dpo/margin_mean": 34.202178955078125,
"margin_dpo/margin_std": 41.48379898071289,
"step": 424
},
{
"KL/chosen_KL_mean": -8.957204818725586,
"KL/mean": -23.96082305908203,
"KL/rejected_KL_mean": -38.96443176269531,
"KL/std": 39.549564361572266,
"epoch": 0.8900523560209425,
"fcm_dpo/beta": 0.017491746693849564,
"fcm_dpo/delta": -0.010492067784070969,
"fcm_dpo/margin": 30.007234573364258,
"fcm_dpo/q_t": 0.38694730401039124,
"grad_norm": 35.75682830810547,
"learning_rate": 1.8594660455706763e-08,
"logits/chosen": 1.5854737758636475,
"logits/rejected": 1.6753469705581665,
"logps/chosen": -281.9069519042969,
"logps/ref_chosen": -272.9497375488281,
"logps/ref_rejected": -274.6335754394531,
"logps/rejected": -313.5980224609375,
"loss": 4.2994,
"margin_dpo/margin_mean": 30.007234573364258,
"margin_dpo/margin_std": 42.59223175048828,
"step": 425
},
{
"KL/chosen_KL_mean": -5.635072708129883,
"KL/mean": -22.328262329101562,
"KL/rejected_KL_mean": -39.02144241333008,
"KL/std": 39.1007194519043,
"epoch": 0.8921465968586387,
"fcm_dpo/beta": 0.01741386204957962,
"fcm_dpo/delta": -0.023887064307928085,
"fcm_dpo/margin": 33.386375427246094,
"fcm_dpo/q_t": 0.37471699714660645,
"grad_norm": 38.75654983520508,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 1.4561810493469238,
"logits/rejected": 1.490870475769043,
"logps/chosen": -287.5766296386719,
"logps/ref_chosen": -281.9415588378906,
"logps/ref_rejected": -253.20486450195312,
"logps/rejected": -292.2262878417969,
"loss": 4.0885,
"margin_dpo/margin_mean": 33.386375427246094,
"margin_dpo/margin_std": 43.17636489868164,
"step": 426
},
{
"KL/chosen_KL_mean": -4.920648574829102,
"KL/mean": -26.763418197631836,
"KL/rejected_KL_mean": -48.6061897277832,
"KL/std": 40.561283111572266,
"epoch": 0.8942408376963351,
"fcm_dpo/beta": 0.01672692783176899,
"fcm_dpo/delta": -0.1363137662410736,
"fcm_dpo/margin": 43.68552780151367,
"fcm_dpo/q_t": 0.34173884987831116,
"grad_norm": 33.15166473388672,
"learning_rate": 1.7233819424956247e-08,
"logits/chosen": 1.4568595886230469,
"logits/rejected": 1.4481300115585327,
"logps/chosen": -285.5372619628906,
"logps/ref_chosen": -280.6166076660156,
"logps/ref_rejected": -263.7532958984375,
"logps/rejected": -312.3595275878906,
"loss": 3.6473,
"margin_dpo/margin_mean": 43.68552780151367,
"margin_dpo/margin_std": 42.516300201416016,
"step": 427
},
{
"KL/chosen_KL_mean": -7.886140823364258,
"KL/mean": -28.404512405395508,
"KL/rejected_KL_mean": -48.922889709472656,
"KL/std": 40.87632369995117,
"epoch": 0.8963350785340314,
"fcm_dpo/beta": 0.015319863334298134,
"fcm_dpo/delta": -0.03134823217988014,
"fcm_dpo/margin": 41.036746978759766,
"fcm_dpo/q_t": 0.36050668358802795,
"grad_norm": 36.520240783691406,
"learning_rate": 1.6572104647786245e-08,
"logits/chosen": 1.235212802886963,
"logits/rejected": 1.3253754377365112,
"logps/chosen": -294.0483703613281,
"logps/ref_chosen": -286.1622314453125,
"logps/ref_rejected": -273.3803405761719,
"logps/rejected": -322.30322265625,
"loss": 3.8141,
"margin_dpo/margin_mean": 41.036746978759766,
"margin_dpo/margin_std": 41.604248046875,
"step": 428
},
{
"KL/chosen_KL_mean": -8.069477081298828,
"KL/mean": -24.538747787475586,
"KL/rejected_KL_mean": -41.008018493652344,
"KL/std": 38.877811431884766,
"epoch": 0.8984293193717278,
"fcm_dpo/beta": 0.015415645204484463,
"fcm_dpo/delta": 0.04732373356819153,
"fcm_dpo/margin": 32.93855667114258,
"fcm_dpo/q_t": 0.3912946581840515,
"grad_norm": 29.93174934387207,
"learning_rate": 1.5922907900227017e-08,
"logits/chosen": 1.491369605064392,
"logits/rejected": 1.491215705871582,
"logps/chosen": -274.40606689453125,
"logps/ref_chosen": -266.3365783691406,
"logps/ref_rejected": -255.01956176757812,
"logps/rejected": -296.027587890625,
"loss": 4.3693,
"margin_dpo/margin_mean": 32.93855667114258,
"margin_dpo/margin_std": 50.389549255371094,
"step": 429
},
{
"KL/chosen_KL_mean": -7.625219345092773,
"KL/mean": -22.037118911743164,
"KL/rejected_KL_mean": -36.44902038574219,
"KL/std": 37.850013732910156,
"epoch": 0.900523560209424,
"fcm_dpo/beta": 0.016457416117191315,
"fcm_dpo/delta": 0.06396263837814331,
"fcm_dpo/margin": 28.82379722595215,
"fcm_dpo/q_t": 0.39826422929763794,
"grad_norm": 66.31047821044922,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": 1.4523224830627441,
"logits/rejected": 1.4776817560195923,
"logps/chosen": -289.776611328125,
"logps/ref_chosen": -282.1513977050781,
"logps/ref_rejected": -263.4792175292969,
"logps/rejected": -299.9282531738281,
"loss": 4.4131,
"margin_dpo/margin_mean": 28.823795318603516,
"margin_dpo/margin_std": 44.25785827636719,
"step": 430
},
{
"KL/chosen_KL_mean": -9.346113204956055,
"KL/mean": -23.709529876708984,
"KL/rejected_KL_mean": -38.072940826416016,
"KL/std": 39.246543884277344,
"epoch": 0.9026178010471204,
"fcm_dpo/beta": 0.017523720860481262,
"fcm_dpo/delta": 0.09749704599380493,
"fcm_dpo/margin": 28.726835250854492,
"fcm_dpo/q_t": 0.3940257728099823,
"grad_norm": 41.90385437011719,
"learning_rate": 1.4662207078575684e-08,
"logits/chosen": 1.5903630256652832,
"logits/rejected": 1.750954270362854,
"logps/chosen": -278.0592346191406,
"logps/ref_chosen": -268.713134765625,
"logps/ref_rejected": -266.13446044921875,
"logps/rejected": -304.2074279785156,
"loss": 4.345,
"margin_dpo/margin_mean": 28.726835250854492,
"margin_dpo/margin_std": 44.24424362182617,
"step": 431
},
{
"KL/chosen_KL_mean": -5.156131744384766,
"KL/mean": -21.788583755493164,
"KL/rejected_KL_mean": -38.42102813720703,
"KL/std": 38.79159164428711,
"epoch": 0.9047120418848168,
"fcm_dpo/beta": 0.017326056957244873,
"fcm_dpo/delta": 0.02436162531375885,
"fcm_dpo/margin": 33.264896392822266,
"fcm_dpo/q_t": 0.37820175290107727,
"grad_norm": 36.16777801513672,
"learning_rate": 1.40507706120426e-08,
"logits/chosen": 1.6775932312011719,
"logits/rejected": 1.7190947532653809,
"logps/chosen": -289.1403503417969,
"logps/ref_chosen": -283.9842224121094,
"logps/ref_rejected": -253.9568634033203,
"logps/rejected": -292.3779296875,
"loss": 4.1458,
"margin_dpo/margin_mean": 33.26490020751953,
"margin_dpo/margin_std": 45.536964416503906,
"step": 432
},
{
"KL/chosen_KL_mean": -13.55865478515625,
"KL/mean": -27.595563888549805,
"KL/rejected_KL_mean": -41.632476806640625,
"KL/std": 38.84959411621094,
"epoch": 0.9068062827225131,
"fcm_dpo/beta": 0.01837783120572567,
"fcm_dpo/delta": 0.08972224593162537,
"fcm_dpo/margin": 28.07382583618164,
"fcm_dpo/q_t": 0.38659048080444336,
"grad_norm": 41.5974235534668,
"learning_rate": 1.345198738661285e-08,
"logits/chosen": 1.3054968118667603,
"logits/rejected": 1.3207252025604248,
"logps/chosen": -256.0018310546875,
"logps/ref_chosen": -242.4431915283203,
"logps/ref_rejected": -252.87979125976562,
"logps/rejected": -294.51226806640625,
"loss": 4.3669,
"margin_dpo/margin_mean": 28.073823928833008,
"margin_dpo/margin_std": 43.10464096069336,
"step": 433
},
{
"KL/chosen_KL_mean": -10.705320358276367,
"KL/mean": -25.866989135742188,
"KL/rejected_KL_mean": -41.028648376464844,
"KL/std": 38.6884651184082,
"epoch": 0.9089005235602095,
"fcm_dpo/beta": 0.019349027425050735,
"fcm_dpo/delta": 0.012435603886842728,
"fcm_dpo/margin": 30.323333740234375,
"fcm_dpo/q_t": 0.378314733505249,
"grad_norm": 42.35694885253906,
"learning_rate": 1.2865889513213628e-08,
"logits/chosen": 1.5045528411865234,
"logits/rejected": 1.4233698844909668,
"logps/chosen": -298.989501953125,
"logps/ref_chosen": -288.2841796875,
"logps/ref_rejected": -252.9123992919922,
"logps/rejected": -293.9410400390625,
"loss": 4.2264,
"margin_dpo/margin_mean": 30.323333740234375,
"margin_dpo/margin_std": 44.75054931640625,
"step": 434
},
{
"KL/chosen_KL_mean": -5.947370529174805,
"KL/mean": -20.233585357666016,
"KL/rejected_KL_mean": -34.519805908203125,
"KL/std": 38.588722229003906,
"epoch": 0.9109947643979057,
"fcm_dpo/beta": 0.01993248052895069,
"fcm_dpo/delta": 0.029382452368736267,
"fcm_dpo/margin": 28.572433471679688,
"fcm_dpo/q_t": 0.3792368769645691,
"grad_norm": 58.921566009521484,
"learning_rate": 1.2292508422495157e-08,
"logits/chosen": 1.4650075435638428,
"logits/rejected": 1.547285795211792,
"logps/chosen": -254.9622039794922,
"logps/ref_chosen": -249.01483154296875,
"logps/ref_rejected": -243.81192016601562,
"logps/rejected": -278.3316955566406,
"loss": 4.2578,
"margin_dpo/margin_mean": 28.572433471679688,
"margin_dpo/margin_std": 41.97202682495117,
"step": 435
},
{
"KL/chosen_KL_mean": -10.389013290405273,
"KL/mean": -21.403196334838867,
"KL/rejected_KL_mean": -32.417381286621094,
"KL/std": 40.2339973449707,
"epoch": 0.9130890052356021,
"fcm_dpo/beta": 0.021259302273392677,
"fcm_dpo/delta": 0.1383521556854248,
"fcm_dpo/margin": 22.028366088867188,
"fcm_dpo/q_t": 0.40718600153923035,
"grad_norm": 53.56985855102539,
"learning_rate": 1.1731874863145142e-08,
"logits/chosen": 1.462636113166809,
"logits/rejected": 1.5538408756256104,
"logps/chosen": -323.319091796875,
"logps/ref_chosen": -312.9300537109375,
"logps/ref_rejected": -296.2721862792969,
"logps/rejected": -328.6895751953125,
"loss": 4.6678,
"margin_dpo/margin_mean": 22.028366088867188,
"margin_dpo/margin_std": 43.784568786621094,
"step": 436
},
{
"KL/chosen_KL_mean": -6.537776947021484,
"KL/mean": -24.114627838134766,
"KL/rejected_KL_mean": -41.691471099853516,
"KL/std": 41.110382080078125,
"epoch": 0.9151832460732985,
"fcm_dpo/beta": 0.020916461944580078,
"fcm_dpo/delta": -0.1481207013130188,
"fcm_dpo/margin": 35.153690338134766,
"fcm_dpo/q_t": 0.35055267810821533,
"grad_norm": 39.32798385620117,
"learning_rate": 1.118401890024001e-08,
"logits/chosen": 1.6977591514587402,
"logits/rejected": 1.745481014251709,
"logps/chosen": -274.8516845703125,
"logps/ref_chosen": -268.31390380859375,
"logps/ref_rejected": -268.7603759765625,
"logps/rejected": -310.4518737792969,
"loss": 3.9117,
"margin_dpo/margin_mean": 35.153690338134766,
"margin_dpo/margin_std": 44.421180725097656,
"step": 437
},
{
"KL/chosen_KL_mean": -13.689395904541016,
"KL/mean": -22.959537506103516,
"KL/rejected_KL_mean": -32.22968292236328,
"KL/std": 38.005943298339844,
"epoch": 0.9172774869109948,
"fcm_dpo/beta": 0.02057368867099285,
"fcm_dpo/delta": 0.08883843570947647,
"fcm_dpo/margin": 18.540285110473633,
"fcm_dpo/q_t": 0.42163756489753723,
"grad_norm": 90.26850891113281,
"learning_rate": 1.06489699136324e-08,
"logits/chosen": 1.33683443069458,
"logits/rejected": 1.407370686531067,
"logps/chosen": -268.9212646484375,
"logps/ref_chosen": -255.23190307617188,
"logps/ref_rejected": -241.1186981201172,
"logps/rejected": -273.348388671875,
"loss": 4.9188,
"margin_dpo/margin_mean": 18.540285110473633,
"margin_dpo/margin_std": 43.72941970825195,
"step": 438
},
{
"KL/chosen_KL_mean": -10.832399368286133,
"KL/mean": -25.905786514282227,
"KL/rejected_KL_mean": -40.979164123535156,
"KL/std": 37.94921112060547,
"epoch": 0.9193717277486911,
"fcm_dpo/beta": 0.02108529582619667,
"fcm_dpo/delta": -0.03847494348883629,
"fcm_dpo/margin": 30.146764755249023,
"fcm_dpo/q_t": 0.37095460295677185,
"grad_norm": 40.88623046875,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 1.2603108882904053,
"logits/rejected": 1.2645988464355469,
"logps/chosen": -261.812744140625,
"logps/ref_chosen": -250.9803466796875,
"logps/ref_rejected": -239.9314422607422,
"logps/rejected": -280.9106140136719,
"loss": 4.1643,
"margin_dpo/margin_mean": 30.146760940551758,
"margin_dpo/margin_std": 43.66949462890625,
"step": 439
},
{
"KL/chosen_KL_mean": -13.518768310546875,
"KL/mean": -25.87021827697754,
"KL/rejected_KL_mean": -38.22166442871094,
"KL/std": 38.02260208129883,
"epoch": 0.9214659685863874,
"fcm_dpo/beta": 0.02056608349084854,
"fcm_dpo/delta": -0.03422413766384125,
"fcm_dpo/margin": 24.702899932861328,
"fcm_dpo/q_t": 0.39521169662475586,
"grad_norm": 50.304046630859375,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": 1.4519972801208496,
"logits/rejected": 1.371519684791565,
"logps/chosen": -314.52154541015625,
"logps/ref_chosen": -301.0027770996094,
"logps/ref_rejected": -263.628662109375,
"logps/rejected": -301.850341796875,
"loss": 4.5864,
"margin_dpo/margin_mean": 24.70290184020996,
"margin_dpo/margin_std": 44.9271240234375,
"step": 440
},
{
"KL/chosen_KL_mean": -6.745321273803711,
"KL/mean": -24.43891143798828,
"KL/rejected_KL_mean": -42.132503509521484,
"KL/std": 39.826961517333984,
"epoch": 0.9235602094240838,
"fcm_dpo/beta": 0.019525915384292603,
"fcm_dpo/delta": -0.09813890606164932,
"fcm_dpo/margin": 35.387176513671875,
"fcm_dpo/q_t": 0.35289281606674194,
"grad_norm": 41.78523254394531,
"learning_rate": 9.12094829893642e-09,
"logits/chosen": 1.5125982761383057,
"logits/rejected": 1.5976024866104126,
"logps/chosen": -307.5940856933594,
"logps/ref_chosen": -300.8487854003906,
"logps/ref_rejected": -296.7118835449219,
"logps/rejected": -338.8443908691406,
"loss": 3.8317,
"margin_dpo/margin_mean": 35.38717269897461,
"margin_dpo/margin_std": 38.58313751220703,
"step": 441
},
{
"KL/chosen_KL_mean": -12.945383071899414,
"KL/mean": -26.59016990661621,
"KL/rejected_KL_mean": -40.234962463378906,
"KL/std": 37.29020309448242,
"epoch": 0.9256544502617801,
"fcm_dpo/beta": 0.01888934150338173,
"fcm_dpo/delta": 0.09072023630142212,
"fcm_dpo/margin": 27.289583206176758,
"fcm_dpo/q_t": 0.3936287760734558,
"grad_norm": 54.41708755493164,
"learning_rate": 8.637407257200496e-09,
"logits/chosen": 1.6090409755706787,
"logits/rejected": 1.670328140258789,
"logps/chosen": -284.59381103515625,
"logps/ref_chosen": -271.6484069824219,
"logps/ref_rejected": -238.21197509765625,
"logps/rejected": -278.4469299316406,
"loss": 4.5528,
"margin_dpo/margin_mean": 27.289583206176758,
"margin_dpo/margin_std": 49.01494598388672,
"step": 442
},
{
"KL/chosen_KL_mean": -9.426856994628906,
"KL/mean": -23.713916778564453,
"KL/rejected_KL_mean": -38.0009765625,
"KL/std": 37.76384735107422,
"epoch": 0.9277486910994764,
"fcm_dpo/beta": 0.020260203629732132,
"fcm_dpo/delta": 0.01686916872859001,
"fcm_dpo/margin": 28.57412338256836,
"fcm_dpo/q_t": 0.37880489230155945,
"grad_norm": 45.84555435180664,
"learning_rate": 8.166809758815895e-09,
"logits/chosen": 1.3293415307998657,
"logits/rejected": 1.3950724601745605,
"logps/chosen": -275.723388671875,
"logps/ref_chosen": -266.2965087890625,
"logps/ref_rejected": -263.7326965332031,
"logps/rejected": -301.733642578125,
"loss": 4.2638,
"margin_dpo/margin_mean": 28.574121475219727,
"margin_dpo/margin_std": 41.3916015625,
"step": 443
},
{
"KL/chosen_KL_mean": -10.304304122924805,
"KL/mean": -24.973583221435547,
"KL/rejected_KL_mean": -39.64287185668945,
"KL/std": 39.80085754394531,
"epoch": 0.9298429319371728,
"fcm_dpo/beta": 0.020134340971708298,
"fcm_dpo/delta": 0.008157186210155487,
"fcm_dpo/margin": 29.338579177856445,
"fcm_dpo/q_t": 0.3796614408493042,
"grad_norm": 50.080379486083984,
"learning_rate": 7.709181040498253e-09,
"logits/chosen": 1.194259762763977,
"logits/rejected": 1.1803237199783325,
"logps/chosen": -302.6650390625,
"logps/ref_chosen": -292.36077880859375,
"logps/ref_rejected": -264.35662841796875,
"logps/rejected": -303.99951171875,
"loss": 4.316,
"margin_dpo/margin_mean": 29.338579177856445,
"margin_dpo/margin_std": 46.49433135986328,
"step": 444
},
{
"KL/chosen_KL_mean": -9.584835052490234,
"KL/mean": -22.41143035888672,
"KL/rejected_KL_mean": -35.23802185058594,
"KL/std": 38.90290832519531,
"epoch": 0.9319371727748691,
"fcm_dpo/beta": 0.020845849066972733,
"fcm_dpo/delta": -0.007416628301143646,
"fcm_dpo/margin": 25.65318489074707,
"fcm_dpo/q_t": 0.39004504680633545,
"grad_norm": 45.288604736328125,
"learning_rate": 7.2645456434869965e-09,
"logits/chosen": 1.257015585899353,
"logits/rejected": 1.344290852546692,
"logps/chosen": -256.30462646484375,
"logps/ref_chosen": -246.71978759765625,
"logps/ref_rejected": -261.339599609375,
"logps/rejected": -296.57763671875,
"loss": 4.5136,
"margin_dpo/margin_mean": 25.653186798095703,
"margin_dpo/margin_std": 42.89739227294922,
"step": 445
},
{
"KL/chosen_KL_mean": -5.0773468017578125,
"KL/mean": -21.905426025390625,
"KL/rejected_KL_mean": -38.733516693115234,
"KL/std": 38.742462158203125,
"epoch": 0.9340314136125655,
"fcm_dpo/beta": 0.01960984244942665,
"fcm_dpo/delta": -0.06857600808143616,
"fcm_dpo/margin": 33.65616226196289,
"fcm_dpo/q_t": 0.36124396324157715,
"grad_norm": 61.7397575378418,
"learning_rate": 6.832927412229017e-09,
"logits/chosen": 1.2315505743026733,
"logits/rejected": 1.2126425504684448,
"logps/chosen": -303.6360778808594,
"logps/ref_chosen": -298.5587158203125,
"logps/ref_rejected": -264.5877685546875,
"logps/rejected": -303.3212890625,
"loss": 4.0676,
"margin_dpo/margin_mean": 33.656158447265625,
"margin_dpo/margin_std": 44.581233978271484,
"step": 446
},
{
"KL/chosen_KL_mean": -3.0303268432617188,
"KL/mean": -20.40411949157715,
"KL/rejected_KL_mean": -37.77790832519531,
"KL/std": 36.89284133911133,
"epoch": 0.9361256544502617,
"fcm_dpo/beta": 0.018508557230234146,
"fcm_dpo/delta": -0.0480596087872982,
"fcm_dpo/margin": 34.747581481933594,
"fcm_dpo/q_t": 0.36139318346977234,
"grad_norm": 36.14704895019531,
"learning_rate": 6.414349493100129e-09,
"logits/chosen": 1.2298775911331177,
"logits/rejected": 1.1924989223480225,
"logps/chosen": -252.883544921875,
"logps/ref_chosen": -249.8532257080078,
"logps/ref_rejected": -233.2117156982422,
"logps/rejected": -270.9896240234375,
"loss": 3.8808,
"margin_dpo/margin_mean": 34.747581481933594,
"margin_dpo/margin_std": 39.805580139160156,
"step": 447
},
{
"KL/chosen_KL_mean": -6.709333419799805,
"KL/mean": -21.88568687438965,
"KL/rejected_KL_mean": -37.062042236328125,
"KL/std": 38.11823272705078,
"epoch": 0.9382198952879581,
"fcm_dpo/beta": 0.018123872578144073,
"fcm_dpo/delta": -0.01662895828485489,
"fcm_dpo/margin": 30.352706909179688,
"fcm_dpo/q_t": 0.3823128640651703,
"grad_norm": 52.687721252441406,
"learning_rate": 6.0088343331638756e-09,
"logits/chosen": 1.4540094137191772,
"logits/rejected": 1.6012420654296875,
"logps/chosen": -265.5309753417969,
"logps/ref_chosen": -258.8216247558594,
"logps/ref_rejected": -260.4953308105469,
"logps/rejected": -297.557373046875,
"loss": 4.1937,
"margin_dpo/margin_mean": 30.352706909179688,
"margin_dpo/margin_std": 41.40434265136719,
"step": 448
},
{
"KL/chosen_KL_mean": -5.103841781616211,
"KL/mean": -21.434587478637695,
"KL/rejected_KL_mean": -37.76533508300781,
"KL/std": 37.36093521118164,
"epoch": 0.9403141361256544,
"fcm_dpo/beta": 0.018367934972047806,
"fcm_dpo/delta": -0.001986050046980381,
"fcm_dpo/margin": 32.661495208740234,
"fcm_dpo/q_t": 0.3707224428653717,
"grad_norm": 43.69072723388672,
"learning_rate": 5.616403678967624e-09,
"logits/chosen": 1.5900778770446777,
"logits/rejected": 1.4752076864242554,
"logps/chosen": -329.3203430175781,
"logps/ref_chosen": -324.2165222167969,
"logps/ref_rejected": -237.05453491210938,
"logps/rejected": -274.81988525390625,
"loss": 4.0803,
"margin_dpo/margin_mean": 32.661495208740234,
"margin_dpo/margin_std": 42.267757415771484,
"step": 449
},
{
"KL/chosen_KL_mean": -12.929838180541992,
"KL/mean": -27.109384536743164,
"KL/rejected_KL_mean": -41.28893280029297,
"KL/std": 39.57197570800781,
"epoch": 0.9424083769633508,
"fcm_dpo/beta": 0.01822546496987343,
"fcm_dpo/delta": 0.0388304740190506,
"fcm_dpo/margin": 28.359092712402344,
"fcm_dpo/q_t": 0.38950634002685547,
"grad_norm": 49.931846618652344,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": 1.437721610069275,
"logits/rejected": 1.4123148918151855,
"logps/chosen": -291.3310852050781,
"logps/ref_chosen": -278.4012451171875,
"logps/ref_rejected": -247.72671508789062,
"logps/rejected": -289.0156555175781,
"loss": 4.2995,
"margin_dpo/margin_mean": 28.35909080505371,
"margin_dpo/margin_std": 42.51638412475586,
"step": 450
},
{
"KL/chosen_KL_mean": -7.553045272827148,
"KL/mean": -23.329837799072266,
"KL/rejected_KL_mean": -39.10662841796875,
"KL/std": 38.305355072021484,
"epoch": 0.9445026178010472,
"fcm_dpo/beta": 0.018385058268904686,
"fcm_dpo/delta": -0.04042164981365204,
"fcm_dpo/margin": 31.5535888671875,
"fcm_dpo/q_t": 0.37636423110961914,
"grad_norm": 44.744327545166016,
"learning_rate": 4.8708793644441086e-09,
"logits/chosen": 1.3844701051712036,
"logits/rejected": 1.392881989479065,
"logps/chosen": -299.6476745605469,
"logps/ref_chosen": -292.0946350097656,
"logps/ref_rejected": -252.24237060546875,
"logps/rejected": -291.3489990234375,
"loss": 4.1547,
"margin_dpo/margin_mean": 31.5535888671875,
"margin_dpo/margin_std": 41.81111526489258,
"step": 451
},
{
"KL/chosen_KL_mean": -3.678445816040039,
"KL/mean": -19.315462112426758,
"KL/rejected_KL_mean": -34.952491760253906,
"KL/std": 39.858558654785156,
"epoch": 0.9465968586387434,
"fcm_dpo/beta": 0.01781679131090641,
"fcm_dpo/delta": 0.04480419307947159,
"fcm_dpo/margin": 31.274038314819336,
"fcm_dpo/q_t": 0.38180726766586304,
"grad_norm": 33.700416564941406,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 1.3815429210662842,
"logits/rejected": 1.4320133924484253,
"logps/chosen": -294.28924560546875,
"logps/ref_chosen": -290.61077880859375,
"logps/ref_rejected": -279.8963928222656,
"logps/rejected": -314.8488464355469,
"loss": 4.1625,
"margin_dpo/margin_mean": 31.2740421295166,
"margin_dpo/margin_std": 41.90071487426758,
"step": 452
},
{
"KL/chosen_KL_mean": -8.826772689819336,
"KL/mean": -24.943347930908203,
"KL/rejected_KL_mean": -41.05990982055664,
"KL/std": 39.05281066894531,
"epoch": 0.9486910994764398,
"fcm_dpo/beta": 0.018621232360601425,
"fcm_dpo/delta": -0.0006048642098903656,
"fcm_dpo/margin": 32.233131408691406,
"fcm_dpo/q_t": 0.37227222323417664,
"grad_norm": 40.410343170166016,
"learning_rate": 4.1779364682113794e-09,
"logits/chosen": 1.4764719009399414,
"logits/rejected": 1.4323137998580933,
"logps/chosen": -325.14202880859375,
"logps/ref_chosen": -316.3152770996094,
"logps/ref_rejected": -300.5086975097656,
"logps/rejected": -341.5686340332031,
"loss": 4.062,
"margin_dpo/margin_mean": 32.233131408691406,
"margin_dpo/margin_std": 42.46658706665039,
"step": 453
},
{
"KL/chosen_KL_mean": -7.95329475402832,
"KL/mean": -23.164857864379883,
"KL/rejected_KL_mean": -38.376426696777344,
"KL/std": 40.72514343261719,
"epoch": 0.9507853403141361,
"fcm_dpo/beta": 0.01902606338262558,
"fcm_dpo/delta": 0.0204494409263134,
"fcm_dpo/margin": 30.423128128051758,
"fcm_dpo/q_t": 0.3785085678100586,
"grad_norm": 40.92826843261719,
"learning_rate": 3.851229943335393e-09,
"logits/chosen": 1.430807113647461,
"logits/rejected": 1.5160000324249268,
"logps/chosen": -298.49859619140625,
"logps/ref_chosen": -290.5452575683594,
"logps/ref_rejected": -297.2102355957031,
"logps/rejected": -335.586669921875,
"loss": 4.1744,
"margin_dpo/margin_mean": 30.423131942749023,
"margin_dpo/margin_std": 41.89031982421875,
"step": 454
},
{
"KL/chosen_KL_mean": -5.455175399780273,
"KL/mean": -18.343996047973633,
"KL/rejected_KL_mean": -31.23281478881836,
"KL/std": 37.38507080078125,
"epoch": 0.9528795811518325,
"fcm_dpo/beta": 0.01976756751537323,
"fcm_dpo/delta": 0.09486553072929382,
"fcm_dpo/margin": 25.777637481689453,
"fcm_dpo/q_t": 0.39325541257858276,
"grad_norm": 47.402862548828125,
"learning_rate": 3.5377236299748147e-09,
"logits/chosen": 1.4440865516662598,
"logits/rejected": 1.5377838611602783,
"logps/chosen": -268.8755187988281,
"logps/ref_chosen": -263.4203186035156,
"logps/ref_rejected": -270.30291748046875,
"logps/rejected": -301.53570556640625,
"loss": 4.4293,
"margin_dpo/margin_mean": 25.777637481689453,
"margin_dpo/margin_std": 42.960609436035156,
"step": 455
},
{
"KL/chosen_KL_mean": -11.12274169921875,
"KL/mean": -29.427356719970703,
"KL/rejected_KL_mean": -47.73197937011719,
"KL/std": 42.497840881347656,
"epoch": 0.9549738219895288,
"fcm_dpo/beta": 0.01928105764091015,
"fcm_dpo/delta": -0.11647738516330719,
"fcm_dpo/margin": 36.60923385620117,
"fcm_dpo/q_t": 0.358774334192276,
"grad_norm": 39.07716751098633,
"learning_rate": 3.2374343405217884e-09,
"logits/chosen": 1.2928086519241333,
"logits/rejected": 1.4272217750549316,
"logps/chosen": -323.1601257324219,
"logps/ref_chosen": -312.0373840332031,
"logps/ref_rejected": -290.50946044921875,
"logps/rejected": -338.2414245605469,
"loss": 4.0107,
"margin_dpo/margin_mean": 36.609230041503906,
"margin_dpo/margin_std": 49.75794219970703,
"step": 456
},
{
"KL/chosen_KL_mean": -9.828344345092773,
"KL/mean": -26.873695373535156,
"KL/rejected_KL_mean": -43.919044494628906,
"KL/std": 41.28899002075195,
"epoch": 0.9570680628272251,
"fcm_dpo/beta": 0.01832345686852932,
"fcm_dpo/delta": -0.02747979573905468,
"fcm_dpo/margin": 34.09069061279297,
"fcm_dpo/q_t": 0.3670627176761627,
"grad_norm": 38.195186614990234,
"learning_rate": 2.9503781785795713e-09,
"logits/chosen": 1.3719782829284668,
"logits/rejected": 1.3869181871414185,
"logps/chosen": -307.0003356933594,
"logps/ref_chosen": -297.1719970703125,
"logps/ref_rejected": -269.15557861328125,
"logps/rejected": -313.07464599609375,
"loss": 4.0677,
"margin_dpo/margin_mean": 34.0906867980957,
"margin_dpo/margin_std": 45.770652770996094,
"step": 457
},
{
"KL/chosen_KL_mean": -7.7780609130859375,
"KL/mean": -23.371183395385742,
"KL/rejected_KL_mean": -38.96431350708008,
"KL/std": 41.964454650878906,
"epoch": 0.9591623036649215,
"fcm_dpo/beta": 0.018945634365081787,
"fcm_dpo/delta": 0.0071364520117640495,
"fcm_dpo/margin": 31.186248779296875,
"fcm_dpo/q_t": 0.37805095314979553,
"grad_norm": 43.36493682861328,
"learning_rate": 2.6765705380989432e-09,
"logits/chosen": 1.3832483291625977,
"logits/rejected": 1.2892534732818604,
"logps/chosen": -295.1429748535156,
"logps/ref_chosen": -287.3648986816406,
"logps/ref_rejected": -271.1472473144531,
"logps/rejected": -310.111572265625,
"loss": 4.2142,
"margin_dpo/margin_mean": 31.186248779296875,
"margin_dpo/margin_std": 44.78030776977539,
"step": 458
},
{
"KL/chosen_KL_mean": -9.517560958862305,
"KL/mean": -21.69217300415039,
"KL/rejected_KL_mean": -33.866790771484375,
"KL/std": 38.41093063354492,
"epoch": 0.9612565445026178,
"fcm_dpo/beta": 0.018657265231013298,
"fcm_dpo/delta": 0.007451209239661694,
"fcm_dpo/margin": 24.349231719970703,
"fcm_dpo/q_t": 0.4049905836582184,
"grad_norm": 43.45826721191406,
"learning_rate": 2.416026102552732e-09,
"logits/chosen": 1.2569316625595093,
"logits/rejected": 1.3241119384765625,
"logps/chosen": -298.1236572265625,
"logps/ref_chosen": -288.6061096191406,
"logps/ref_rejected": -285.6366271972656,
"logps/rejected": -319.50341796875,
"loss": 4.6272,
"margin_dpo/margin_mean": 24.349231719970703,
"margin_dpo/margin_std": 44.906341552734375,
"step": 459
},
{
"KL/chosen_KL_mean": -10.26303482055664,
"KL/mean": -22.68872833251953,
"KL/rejected_KL_mean": -35.114418029785156,
"KL/std": 38.66323471069336,
"epoch": 0.9633507853403142,
"fcm_dpo/beta": 0.01909796893596649,
"fcm_dpo/delta": 0.060689061880111694,
"fcm_dpo/margin": 24.85137176513672,
"fcm_dpo/q_t": 0.40231454372406006,
"grad_norm": 62.3796272277832,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": 1.3401086330413818,
"logits/rejected": 1.2732634544372559,
"logps/chosen": -320.5149841308594,
"logps/ref_chosen": -310.251953125,
"logps/ref_rejected": -252.962158203125,
"logps/rejected": -288.0765686035156,
"loss": 4.661,
"margin_dpo/margin_mean": 24.85137176513672,
"margin_dpo/margin_std": 47.31897735595703,
"step": 460
},
{
"KL/chosen_KL_mean": -8.58036994934082,
"KL/mean": -23.64044189453125,
"KL/rejected_KL_mean": -38.70050811767578,
"KL/std": 39.988468170166016,
"epoch": 0.9654450261780104,
"fcm_dpo/beta": 0.01960652321577072,
"fcm_dpo/delta": 0.00908164493739605,
"fcm_dpo/margin": 30.120145797729492,
"fcm_dpo/q_t": 0.3788868486881256,
"grad_norm": 45.759761810302734,
"learning_rate": 1.9347820230782295e-09,
"logits/chosen": 1.1388990879058838,
"logits/rejected": 1.248889446258545,
"logps/chosen": -245.3607940673828,
"logps/ref_chosen": -236.7804412841797,
"logps/ref_rejected": -229.67550659179688,
"logps/rejected": -268.37603759765625,
"loss": 4.2962,
"margin_dpo/margin_mean": 30.120145797729492,
"margin_dpo/margin_std": 47.037715911865234,
"step": 461
},
{
"KL/chosen_KL_mean": -5.170505523681641,
"KL/mean": -23.70345115661621,
"KL/rejected_KL_mean": -42.236385345458984,
"KL/std": 39.78120803833008,
"epoch": 0.9675392670157068,
"fcm_dpo/beta": 0.018582535907626152,
"fcm_dpo/delta": -0.09749210625886917,
"fcm_dpo/margin": 37.065879821777344,
"fcm_dpo/q_t": 0.3565976619720459,
"grad_norm": 50.27894973754883,
"learning_rate": 1.7141081868094209e-09,
"logits/chosen": 1.4427176713943481,
"logits/rejected": 1.3286992311477661,
"logps/chosen": -340.4740295410156,
"logps/ref_chosen": -335.30352783203125,
"logps/ref_rejected": -250.6796875,
"logps/rejected": -292.9160461425781,
"loss": 3.9905,
"margin_dpo/margin_mean": 37.065879821777344,
"margin_dpo/margin_std": 49.16842269897461,
"step": 462
},
{
"KL/chosen_KL_mean": -10.044492721557617,
"KL/mean": -24.297210693359375,
"KL/rejected_KL_mean": -38.5499267578125,
"KL/std": 40.091426849365234,
"epoch": 0.9696335078534032,
"fcm_dpo/beta": 0.018351979553699493,
"fcm_dpo/delta": 0.03311692178249359,
"fcm_dpo/margin": 28.505441665649414,
"fcm_dpo/q_t": 0.38852405548095703,
"grad_norm": 37.71248245239258,
"learning_rate": 1.5067491694100153e-09,
"logits/chosen": 1.7018460035324097,
"logits/rejected": 1.5499235391616821,
"logps/chosen": -298.2647399902344,
"logps/ref_chosen": -288.22021484375,
"logps/ref_rejected": -227.96673583984375,
"logps/rejected": -266.5166320800781,
"loss": 4.3149,
"margin_dpo/margin_mean": 28.505441665649414,
"margin_dpo/margin_std": 42.738094329833984,
"step": 463
},
{
"KL/chosen_KL_mean": -10.695844650268555,
"KL/mean": -25.198429107666016,
"KL/rejected_KL_mean": -39.70100784301758,
"KL/std": 39.06782150268555,
"epoch": 0.9717277486910995,
"fcm_dpo/beta": 0.01894655078649521,
"fcm_dpo/delta": 0.05104460194706917,
"fcm_dpo/margin": 29.005159378051758,
"fcm_dpo/q_t": 0.3809618353843689,
"grad_norm": 35.92738723754883,
"learning_rate": 1.3127160909147672e-09,
"logits/chosen": 1.6909222602844238,
"logits/rejected": 1.7193284034729004,
"logps/chosen": -271.0328063964844,
"logps/ref_chosen": -260.3369140625,
"logps/ref_rejected": -254.2083740234375,
"logps/rejected": -293.9093933105469,
"loss": 4.2979,
"margin_dpo/margin_mean": 29.005159378051758,
"margin_dpo/margin_std": 42.85988998413086,
"step": 464
},
{
"KL/chosen_KL_mean": -2.0730247497558594,
"KL/mean": -20.170202255249023,
"KL/rejected_KL_mean": -38.26738357543945,
"KL/std": 39.66697692871094,
"epoch": 0.9738219895287958,
"fcm_dpo/beta": 0.01791943982243538,
"fcm_dpo/delta": -0.13475660979747772,
"fcm_dpo/margin": 36.19435119628906,
"fcm_dpo/q_t": 0.3630122244358063,
"grad_norm": 38.98749542236328,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 1.6583516597747803,
"logits/rejected": 1.7033283710479736,
"logps/chosen": -285.14251708984375,
"logps/ref_chosen": -283.0694885253906,
"logps/ref_rejected": -289.1370544433594,
"logps/rejected": -327.40447998046875,
"loss": 3.9985,
"margin_dpo/margin_mean": 36.19435119628906,
"margin_dpo/margin_std": 43.46207046508789,
"step": 465
},
{
"KL/chosen_KL_mean": -3.785524368286133,
"KL/mean": -22.948883056640625,
"KL/rejected_KL_mean": -42.112239837646484,
"KL/std": 39.4052848815918,
"epoch": 0.9759162303664921,
"fcm_dpo/beta": 0.01683930680155754,
"fcm_dpo/delta": -0.05012017488479614,
"fcm_dpo/margin": 38.32670593261719,
"fcm_dpo/q_t": 0.3580254316329956,
"grad_norm": 42.31191635131836,
"learning_rate": 9.64668657069706e-10,
"logits/chosen": 1.0171492099761963,
"logits/rejected": 1.056290626525879,
"logps/chosen": -260.6705627441406,
"logps/ref_chosen": -256.8850402832031,
"logps/ref_rejected": -247.0802459716797,
"logps/rejected": -289.1924743652344,
"loss": 3.7963,
"margin_dpo/margin_mean": 38.32670593261719,
"margin_dpo/margin_std": 40.48317337036133,
"step": 466
},
{
"KL/chosen_KL_mean": -10.045860290527344,
"KL/mean": -24.08479118347168,
"KL/rejected_KL_mean": -38.12372589111328,
"KL/std": 39.32081985473633,
"epoch": 0.9780104712041885,
"fcm_dpo/beta": 0.016987893730401993,
"fcm_dpo/delta": 0.006081026047468185,
"fcm_dpo/margin": 28.077857971191406,
"fcm_dpo/q_t": 0.39910078048706055,
"grad_norm": 38.65256118774414,
"learning_rate": 8.106729664475176e-10,
"logits/chosen": 1.1129744052886963,
"logits/rejected": 1.2602795362472534,
"logps/chosen": -267.5699462890625,
"logps/ref_chosen": -257.5240783691406,
"logps/ref_rejected": -274.82122802734375,
"logps/rejected": -312.9449462890625,
"loss": 4.4977,
"margin_dpo/margin_mean": 28.077857971191406,
"margin_dpo/margin_std": 46.522178649902344,
"step": 467
},
{
"KL/chosen_KL_mean": -5.589506149291992,
"KL/mean": -18.8349666595459,
"KL/rejected_KL_mean": -32.0804328918457,
"KL/std": 38.555419921875,
"epoch": 0.9801047120418848,
"fcm_dpo/beta": 0.017922434955835342,
"fcm_dpo/delta": 0.1293361335992813,
"fcm_dpo/margin": 26.490917205810547,
"fcm_dpo/q_t": 0.3997229337692261,
"grad_norm": 36.69874572753906,
"learning_rate": 6.700405431837585e-10,
"logits/chosen": 1.524809718132019,
"logits/rejected": 1.4071067571640015,
"logps/chosen": -317.8492431640625,
"logps/ref_chosen": -312.2597351074219,
"logps/ref_rejected": -259.74383544921875,
"logps/rejected": -291.8242492675781,
"loss": 4.4111,
"margin_dpo/margin_mean": 26.490917205810547,
"margin_dpo/margin_std": 41.01814651489258,
"step": 468
},
{
"KL/chosen_KL_mean": -4.054059982299805,
"KL/mean": -21.49460220336914,
"KL/rejected_KL_mean": -38.935142517089844,
"KL/std": 40.927608489990234,
"epoch": 0.9821989528795811,
"fcm_dpo/beta": 0.018519805744290352,
"fcm_dpo/delta": -0.05003425478935242,
"fcm_dpo/margin": 34.88108444213867,
"fcm_dpo/q_t": 0.363657146692276,
"grad_norm": 39.32550811767578,
"learning_rate": 5.427789289685347e-10,
"logits/chosen": 1.2567800283432007,
"logits/rejected": 1.1510202884674072,
"logps/chosen": -321.0937805175781,
"logps/ref_chosen": -317.03973388671875,
"logps/ref_rejected": -260.12628173828125,
"logps/rejected": -299.0614013671875,
"loss": 4.0756,
"margin_dpo/margin_mean": 34.88108444213867,
"margin_dpo/margin_std": 47.22452926635742,
"step": 469
},
{
"KL/chosen_KL_mean": -2.6450576782226562,
"KL/mean": -20.932540893554688,
"KL/rejected_KL_mean": -39.22002410888672,
"KL/std": 40.9278564453125,
"epoch": 0.9842931937172775,
"fcm_dpo/beta": 0.01805165596306324,
"fcm_dpo/delta": -0.06591328233480453,
"fcm_dpo/margin": 36.574974060058594,
"fcm_dpo/q_t": 0.36200690269470215,
"grad_norm": 36.61351776123047,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": 1.4258358478546143,
"logits/rejected": 1.345468282699585,
"logps/chosen": -307.03900146484375,
"logps/ref_chosen": -304.3939514160156,
"logps/ref_rejected": -257.6587219238281,
"logps/rejected": -296.8786926269531,
"loss": 4.0182,
"margin_dpo/margin_mean": 36.574974060058594,
"margin_dpo/margin_std": 46.32111358642578,
"step": 470
},
{
"KL/chosen_KL_mean": -10.424781799316406,
"KL/mean": -23.33281707763672,
"KL/rejected_KL_mean": -36.240848541259766,
"KL/std": 40.707862854003906,
"epoch": 0.9863874345549738,
"fcm_dpo/beta": 0.01781153865158558,
"fcm_dpo/delta": 0.09338882565498352,
"fcm_dpo/margin": 25.816076278686523,
"fcm_dpo/q_t": 0.4024926722049713,
"grad_norm": 45.583438873291016,
"learning_rate": 3.2839470889836627e-10,
"logits/chosen": 1.3496334552764893,
"logits/rejected": 1.4129079580307007,
"logps/chosen": -295.3686218261719,
"logps/ref_chosen": -284.94384765625,
"logps/ref_rejected": -297.4357604980469,
"logps/rejected": -333.6766357421875,
"loss": 4.463,
"margin_dpo/margin_mean": 25.81607437133789,
"margin_dpo/margin_std": 42.50419235229492,
"step": 471
},
{
"KL/chosen_KL_mean": -4.572975158691406,
"KL/mean": -21.51951026916504,
"KL/rejected_KL_mean": -38.466064453125,
"KL/std": 38.62905502319336,
"epoch": 0.9884816753926702,
"fcm_dpo/beta": 0.018027406185865402,
"fcm_dpo/delta": -0.012763120234012604,
"fcm_dpo/margin": 33.89308547973633,
"fcm_dpo/q_t": 0.37206608057022095,
"grad_norm": 34.75439453125,
"learning_rate": 2.412835998185092e-10,
"logits/chosen": 1.3935006856918335,
"logits/rejected": 1.5130993127822876,
"logps/chosen": -236.92724609375,
"logps/ref_chosen": -232.35427856445312,
"logps/ref_rejected": -245.6057891845703,
"logps/rejected": -284.07183837890625,
"loss": 4.0583,
"margin_dpo/margin_mean": 33.89308547973633,
"margin_dpo/margin_std": 44.567867279052734,
"step": 472
},
{
"KL/chosen_KL_mean": -7.315509796142578,
"KL/mean": -24.49239730834961,
"KL/rejected_KL_mean": -41.669288635253906,
"KL/std": 38.954444885253906,
"epoch": 0.9905759162303664,
"fcm_dpo/beta": 0.018295086920261383,
"fcm_dpo/delta": -0.031036358326673508,
"fcm_dpo/margin": 34.353790283203125,
"fcm_dpo/q_t": 0.36757490038871765,
"grad_norm": 38.30513381958008,
"learning_rate": 1.6756629272085544e-10,
"logits/chosen": 1.2387901544570923,
"logits/rejected": 1.2626218795776367,
"logps/chosen": -285.3223876953125,
"logps/ref_chosen": -278.0068664550781,
"logps/ref_rejected": -257.19256591796875,
"logps/rejected": -298.86187744140625,
"loss": 4.0698,
"margin_dpo/margin_mean": 34.353790283203125,
"margin_dpo/margin_std": 45.340179443359375,
"step": 473
},
{
"KL/chosen_KL_mean": -11.093404769897461,
"KL/mean": -25.184301376342773,
"KL/rejected_KL_mean": -39.27520751953125,
"KL/std": 38.69279479980469,
"epoch": 0.9926701570680628,
"fcm_dpo/beta": 0.017912233248353004,
"fcm_dpo/delta": 0.10232597589492798,
"fcm_dpo/margin": 28.181808471679688,
"fcm_dpo/q_t": 0.390829861164093,
"grad_norm": 54.18507766723633,
"learning_rate": 1.072467408408384e-10,
"logits/chosen": 1.256346344947815,
"logits/rejected": 1.2883105278015137,
"logps/chosen": -291.85089111328125,
"logps/ref_chosen": -280.75750732421875,
"logps/ref_rejected": -264.34295654296875,
"logps/rejected": -303.6181335449219,
"loss": 4.2791,
"margin_dpo/margin_mean": 28.18181037902832,
"margin_dpo/margin_std": 37.79776382446289,
"step": 474
},
{
"KL/chosen_KL_mean": -5.9772186279296875,
"KL/mean": -19.791828155517578,
"KL/rejected_KL_mean": -33.60643005371094,
"KL/std": 37.01370620727539,
"epoch": 0.9947643979057592,
"fcm_dpo/beta": 0.01921505481004715,
"fcm_dpo/delta": -0.009704768657684326,
"fcm_dpo/margin": 27.629199981689453,
"fcm_dpo/q_t": 0.38869670033454895,
"grad_norm": 39.49391174316406,
"learning_rate": 6.032817893297793e-11,
"logits/chosen": 1.2344403266906738,
"logits/rejected": 1.3401638269424438,
"logps/chosen": -247.4485626220703,
"logps/ref_chosen": -241.4713592529297,
"logps/ref_rejected": -244.0782928466797,
"logps/rejected": -277.6847229003906,
"loss": 4.3437,
"margin_dpo/margin_mean": 27.629199981689453,
"margin_dpo/margin_std": 42.206539154052734,
"step": 475
},
{
"KL/chosen_KL_mean": -1.0645999908447266,
"KL/mean": -17.27362060546875,
"KL/rejected_KL_mean": -33.482635498046875,
"KL/std": 37.50159454345703,
"epoch": 0.9968586387434555,
"fcm_dpo/beta": 0.01907847449183464,
"fcm_dpo/delta": -0.02217945083975792,
"fcm_dpo/margin": 32.41803741455078,
"fcm_dpo/q_t": 0.3712438941001892,
"grad_norm": 37.655029296875,
"learning_rate": 2.6813123097352287e-11,
"logits/chosen": 1.4651304483413696,
"logits/rejected": 1.4587892293930054,
"logps/chosen": -319.83892822265625,
"logps/ref_chosen": -318.7742919921875,
"logps/ref_rejected": -293.66595458984375,
"logps/rejected": -327.1485595703125,
"loss": 4.0899,
"margin_dpo/margin_mean": 32.41803741455078,
"margin_dpo/margin_std": 43.326416015625,
"step": 476
},
{
"KL/chosen_KL_mean": -8.864704132080078,
"KL/mean": -25.756881713867188,
"KL/rejected_KL_mean": -42.649051666259766,
"KL/std": 40.76933288574219,
"epoch": 0.9989528795811519,
"fcm_dpo/beta": 0.018941551446914673,
"fcm_dpo/delta": -0.04303491488099098,
"fcm_dpo/margin": 33.78434753417969,
"fcm_dpo/q_t": 0.3677205443382263,
"grad_norm": 38.56641387939453,
"learning_rate": 6.7033706447061635e-12,
"logits/chosen": 1.3105425834655762,
"logits/rejected": 1.332196593284607,
"logps/chosen": -274.8004150390625,
"logps/ref_chosen": -265.9356994628906,
"logps/ref_rejected": -243.6986083984375,
"logps/rejected": -286.34765625,
"loss": 4.1499,
"margin_dpo/margin_mean": 33.78434753417969,
"margin_dpo/margin_std": 47.822933197021484,
"step": 477
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 4.596999753196284,
"train_runtime": 7387.5434,
"train_samples_per_second": 8.275,
"train_steps_per_second": 0.065
}
],
"logging_steps": 1,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink