W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.3-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-ult…/trainer_state.json
ModelHub XC 1d0aa7f57e 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.3-20260428-045924
Source: Original Platform
2026-05-15 20:24:15 +08:00

11057 lines
414 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": -0.05443763732910156,
"KL/mean": -0.04046645760536194,
"KL/rejected_KL_mean": -0.026498794555664062,
"KL/std": 0.47091516852378845,
"epoch": 0.0020942408376963353,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02793481945991516,
"fcm_dpo/q_t": 0.500069797039032,
"grad_norm": 28.59173011779785,
"learning_rate": 0.0,
"logits/chosen": -0.5898098945617676,
"logits/rejected": -0.604260265827179,
"logps/chosen": -275.28570556640625,
"logps/ref_chosen": -275.2312927246094,
"logps/ref_rejected": -222.9380340576172,
"logps/rejected": -222.96453857421875,
"loss": 5.5463,
"margin_dpo/margin_mean": -0.02793477475643158,
"margin_dpo/margin_std": 0.5724214911460876,
"step": 1
},
{
"KL/chosen_KL_mean": 0.04458808898925781,
"KL/mean": 0.0374295711517334,
"KL/rejected_KL_mean": 0.030269622802734375,
"KL/std": 0.49036121368408203,
"epoch": 0.004188481675392671,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014312177896499634,
"fcm_dpo/q_t": 0.4999642074108124,
"grad_norm": 27.88316535949707,
"learning_rate": 1.0416666666666666e-08,
"logits/chosen": -0.6574729681015015,
"logits/rejected": -0.6464410424232483,
"logps/chosen": -264.7165222167969,
"logps/ref_chosen": -264.7611083984375,
"logps/ref_rejected": -242.5597686767578,
"logps/rejected": -242.52951049804688,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.014312252402305603,
"margin_dpo/margin_std": 0.6423971652984619,
"step": 2
},
{
"KL/chosen_KL_mean": 0.01624298095703125,
"KL/mean": -0.02929476648569107,
"KL/rejected_KL_mean": -0.0748291015625,
"KL/std": 0.480156272649765,
"epoch": 0.0062827225130890054,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09107272326946259,
"fcm_dpo/q_t": 0.499772310256958,
"grad_norm": 25.9517879486084,
"learning_rate": 2.083333333333333e-08,
"logits/chosen": -0.6839807033538818,
"logits/rejected": -0.735178530216217,
"logps/chosen": -274.0856628417969,
"logps/ref_chosen": -274.1018981933594,
"logps/ref_rejected": -286.5882568359375,
"logps/rejected": -286.6631164550781,
"loss": 5.5416,
"margin_dpo/margin_mean": 0.09107272326946259,
"margin_dpo/margin_std": 0.6260531544685364,
"step": 3
},
{
"KL/chosen_KL_mean": -0.030345916748046875,
"KL/mean": -0.059315308928489685,
"KL/rejected_KL_mean": -0.08828353881835938,
"KL/std": 0.6341561079025269,
"epoch": 0.008376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05793502926826477,
"fcm_dpo/q_t": 0.4998551607131958,
"grad_norm": 31.600723266601562,
"learning_rate": 3.125e-08,
"logits/chosen": -0.6173092126846313,
"logits/rejected": -0.6115694642066956,
"logps/chosen": -329.8685607910156,
"logps/ref_chosen": -329.8382568359375,
"logps/ref_rejected": -303.2850646972656,
"logps/rejected": -303.3733215332031,
"loss": 5.5429,
"margin_dpo/margin_mean": 0.05793498456478119,
"margin_dpo/margin_std": 0.8304102420806885,
"step": 4
},
{
"KL/chosen_KL_mean": -0.03276824951171875,
"KL/mean": 0.0054293423891067505,
"KL/rejected_KL_mean": 0.04363250732421875,
"KL/std": 0.567704975605011,
"epoch": 0.010471204188481676,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07639628648757935,
"fcm_dpo/q_t": 0.5001909732818604,
"grad_norm": 29.59075927734375,
"learning_rate": 4.166666666666666e-08,
"logits/chosen": -0.57039475440979,
"logits/rejected": -0.5864973068237305,
"logps/chosen": -301.771728515625,
"logps/ref_chosen": -301.7389221191406,
"logps/ref_rejected": -274.7654724121094,
"logps/rejected": -274.72186279296875,
"loss": 5.5483,
"margin_dpo/margin_mean": -0.07639645040035248,
"margin_dpo/margin_std": 0.8294250965118408,
"step": 5
},
{
"KL/chosen_KL_mean": 0.03812408447265625,
"KL/mean": 0.029881909489631653,
"KL/rejected_KL_mean": 0.021627426147460938,
"KL/std": 0.5347965359687805,
"epoch": 0.012565445026178011,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.016495585441589355,
"fcm_dpo/q_t": 0.4999587833881378,
"grad_norm": 28.209911346435547,
"learning_rate": 5.208333333333333e-08,
"logits/chosen": -0.6794368624687195,
"logits/rejected": -0.6423019766807556,
"logps/chosen": -285.65655517578125,
"logps/ref_chosen": -285.6946716308594,
"logps/ref_rejected": -245.8200225830078,
"logps/rejected": -245.79840087890625,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.016496747732162476,
"margin_dpo/margin_std": 0.706045925617218,
"step": 6
},
{
"KL/chosen_KL_mean": -0.04505348205566406,
"KL/mean": -0.026556827127933502,
"KL/rejected_KL_mean": -0.008066177368164062,
"KL/std": 0.46558207273483276,
"epoch": 0.014659685863874346,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.036994144320487976,
"fcm_dpo/q_t": 0.5000925064086914,
"grad_norm": 28.658977508544922,
"learning_rate": 6.25e-08,
"logits/chosen": -0.5635161399841309,
"logits/rejected": -0.5949946045875549,
"logps/chosen": -264.7004699707031,
"logps/ref_chosen": -264.65545654296875,
"logps/ref_rejected": -253.10305786132812,
"logps/rejected": -253.11111450195312,
"loss": 5.5467,
"margin_dpo/margin_mean": -0.036993831396102905,
"margin_dpo/margin_std": 0.6402279734611511,
"step": 7
},
{
"KL/chosen_KL_mean": 0.021518707275390625,
"KL/mean": 0.013283878564834595,
"KL/rejected_KL_mean": 0.0050487518310546875,
"KL/std": 0.5391606092453003,
"epoch": 0.016753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01646500825881958,
"fcm_dpo/q_t": 0.4999588429927826,
"grad_norm": 30.747331619262695,
"learning_rate": 7.291666666666667e-08,
"logits/chosen": -0.6774884462356567,
"logits/rejected": -0.6843160390853882,
"logps/chosen": -354.167236328125,
"logps/ref_chosen": -354.1887512207031,
"logps/ref_rejected": -282.9112243652344,
"logps/rejected": -282.9061584472656,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.016465887427330017,
"margin_dpo/margin_std": 0.6959210634231567,
"step": 8
},
{
"KL/chosen_KL_mean": -0.06751441955566406,
"KL/mean": -0.02276702970266342,
"KL/rejected_KL_mean": 0.02196502685546875,
"KL/std": 0.5027087330818176,
"epoch": 0.018848167539267015,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.08947370946407318,
"fcm_dpo/q_t": 0.5002236366271973,
"grad_norm": 27.86223030090332,
"learning_rate": 8.333333333333333e-08,
"logits/chosen": -0.6372052431106567,
"logits/rejected": -0.6553710103034973,
"logps/chosen": -285.61773681640625,
"logps/ref_chosen": -285.5502014160156,
"logps/ref_rejected": -267.99664306640625,
"logps/rejected": -267.9747009277344,
"loss": 5.5488,
"margin_dpo/margin_mean": -0.08947360515594482,
"margin_dpo/margin_std": 0.6783429980278015,
"step": 9
},
{
"KL/chosen_KL_mean": -0.0031414031982421875,
"KL/mean": -0.010139897465705872,
"KL/rejected_KL_mean": -0.017139434814453125,
"KL/std": 0.4715445637702942,
"epoch": 0.020942408376963352,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014001935720443726,
"fcm_dpo/q_t": 0.49996495246887207,
"grad_norm": 26.741235733032227,
"learning_rate": 9.375e-08,
"logits/chosen": -0.6870542168617249,
"logits/rejected": -0.6816274523735046,
"logps/chosen": -251.91552734375,
"logps/ref_chosen": -251.91238403320312,
"logps/ref_rejected": -226.45260620117188,
"logps/rejected": -226.46974182128906,
"loss": 5.5447,
"margin_dpo/margin_mean": 0.014002710580825806,
"margin_dpo/margin_std": 0.6491466760635376,
"step": 10
},
{
"KL/chosen_KL_mean": 0.036834716796875,
"KL/mean": 0.05126556009054184,
"KL/rejected_KL_mean": 0.06568717956542969,
"KL/std": 0.5492945313453674,
"epoch": 0.023036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02885313332080841,
"fcm_dpo/q_t": 0.5000721216201782,
"grad_norm": 28.992164611816406,
"learning_rate": 1.0416666666666667e-07,
"logits/chosen": -0.5933151841163635,
"logits/rejected": -0.648064374923706,
"logps/chosen": -301.046630859375,
"logps/ref_chosen": -301.08343505859375,
"logps/ref_rejected": -259.546630859375,
"logps/rejected": -259.4809265136719,
"loss": 5.5464,
"margin_dpo/margin_mean": -0.028853505849838257,
"margin_dpo/margin_std": 0.7581360340118408,
"step": 11
},
{
"KL/chosen_KL_mean": 0.10894775390625,
"KL/mean": 0.005401916801929474,
"KL/rejected_KL_mean": -0.09813308715820312,
"KL/std": 0.517360508441925,
"epoch": 0.025130890052356022,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20707155764102936,
"fcm_dpo/q_t": 0.49948233366012573,
"grad_norm": 30.014650344848633,
"learning_rate": 1.1458333333333332e-07,
"logits/chosen": -0.5822965502738953,
"logits/rejected": -0.5422216653823853,
"logps/chosen": -287.43914794921875,
"logps/ref_chosen": -287.548095703125,
"logps/ref_rejected": -277.37945556640625,
"logps/rejected": -277.47760009765625,
"loss": 5.537,
"margin_dpo/margin_mean": 0.20707103610038757,
"margin_dpo/margin_std": 0.7214613556861877,
"step": 12
},
{
"KL/chosen_KL_mean": 0.026136398315429688,
"KL/mean": -0.010055795311927795,
"KL/rejected_KL_mean": -0.04625511169433594,
"KL/std": 0.46862512826919556,
"epoch": 0.027225130890052355,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07239121198654175,
"fcm_dpo/q_t": 0.4998190402984619,
"grad_norm": 27.232789993286133,
"learning_rate": 1.25e-07,
"logits/chosen": -0.6514136791229248,
"logits/rejected": -0.658698558807373,
"logps/chosen": -270.6402893066406,
"logps/ref_chosen": -270.6664123535156,
"logps/ref_rejected": -274.6546936035156,
"logps/rejected": -274.7009582519531,
"loss": 5.5423,
"margin_dpo/margin_mean": 0.07239186763763428,
"margin_dpo/margin_std": 0.6270844340324402,
"step": 13
},
{
"KL/chosen_KL_mean": 0.031122207641601562,
"KL/mean": 0.017020471394062042,
"KL/rejected_KL_mean": 0.002918243408203125,
"KL/std": 0.5255962610244751,
"epoch": 0.02931937172774869,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.028207316994667053,
"fcm_dpo/q_t": 0.4999294877052307,
"grad_norm": 28.295730590820312,
"learning_rate": 1.3541666666666666e-07,
"logits/chosen": -0.620563268661499,
"logits/rejected": -0.6489490866661072,
"logps/chosen": -281.5621032714844,
"logps/ref_chosen": -281.59320068359375,
"logps/ref_rejected": -263.52215576171875,
"logps/rejected": -263.51922607421875,
"loss": 5.5441,
"margin_dpo/margin_mean": 0.028206750750541687,
"margin_dpo/margin_std": 0.6688634753227234,
"step": 14
},
{
"KL/chosen_KL_mean": 0.03985404968261719,
"KL/mean": 0.014157146215438843,
"KL/rejected_KL_mean": -0.011541366577148438,
"KL/std": 0.4595259428024292,
"epoch": 0.031413612565445025,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05139949917793274,
"fcm_dpo/q_t": 0.49987149238586426,
"grad_norm": 30.344572067260742,
"learning_rate": 1.4583333333333335e-07,
"logits/chosen": -0.631891131401062,
"logits/rejected": -0.6428329944610596,
"logps/chosen": -298.41357421875,
"logps/ref_chosen": -298.45343017578125,
"logps/ref_rejected": -227.17832946777344,
"logps/rejected": -227.18988037109375,
"loss": 5.5432,
"margin_dpo/margin_mean": 0.05139845609664917,
"margin_dpo/margin_std": 0.6178590059280396,
"step": 15
},
{
"KL/chosen_KL_mean": 0.07450485229492188,
"KL/mean": 0.021834172308444977,
"KL/rejected_KL_mean": -0.030839920043945312,
"KL/std": 0.539286732673645,
"epoch": 0.033507853403141365,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10533960163593292,
"fcm_dpo/q_t": 0.4997366666793823,
"grad_norm": 30.12870216369629,
"learning_rate": 1.5624999999999999e-07,
"logits/chosen": -0.5921044945716858,
"logits/rejected": -0.588280439376831,
"logps/chosen": -293.8921203613281,
"logps/ref_chosen": -293.96661376953125,
"logps/ref_rejected": -250.78443908691406,
"logps/rejected": -250.81527709960938,
"loss": 5.541,
"margin_dpo/margin_mean": 0.10533848404884338,
"margin_dpo/margin_std": 0.7177047729492188,
"step": 16
},
{
"KL/chosen_KL_mean": 0.03164863586425781,
"KL/mean": 0.015181809663772583,
"KL/rejected_KL_mean": -0.00128173828125,
"KL/std": 0.49001258611679077,
"epoch": 0.0356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03292769193649292,
"fcm_dpo/q_t": 0.4999176859855652,
"grad_norm": 27.68689727783203,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5722857117652893,
"logits/rejected": -0.5986632704734802,
"logps/chosen": -262.36236572265625,
"logps/ref_chosen": -262.39398193359375,
"logps/ref_rejected": -248.500244140625,
"logps/rejected": -248.50152587890625,
"loss": 5.5439,
"margin_dpo/margin_mean": 0.03292873501777649,
"margin_dpo/margin_std": 0.6998825073242188,
"step": 17
},
{
"KL/chosen_KL_mean": 0.013545989990234375,
"KL/mean": -0.026699572801589966,
"KL/rejected_KL_mean": -0.06694602966308594,
"KL/std": 0.5444977879524231,
"epoch": 0.03769633507853403,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08047735691070557,
"fcm_dpo/q_t": 0.49979880452156067,
"grad_norm": 29.722179412841797,
"learning_rate": 1.7708333333333334e-07,
"logits/chosen": -0.6103276014328003,
"logits/rejected": -0.6163787245750427,
"logps/chosen": -293.6956787109375,
"logps/ref_chosen": -293.709228515625,
"logps/ref_rejected": -274.5875244140625,
"logps/rejected": -274.65447998046875,
"loss": 5.542,
"margin_dpo/margin_mean": 0.08047764003276825,
"margin_dpo/margin_std": 0.7309471368789673,
"step": 18
},
{
"KL/chosen_KL_mean": 0.04509544372558594,
"KL/mean": 0.022539354860782623,
"KL/rejected_KL_mean": -9.5367431640625e-06,
"KL/std": 0.48370176553726196,
"epoch": 0.039790575916230364,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04510428011417389,
"fcm_dpo/q_t": 0.49988722801208496,
"grad_norm": 28.11159324645996,
"learning_rate": 1.875e-07,
"logits/chosen": -0.6241991519927979,
"logits/rejected": -0.6202787756919861,
"logps/chosen": -280.2205810546875,
"logps/ref_chosen": -280.26568603515625,
"logps/ref_rejected": -259.9742736816406,
"logps/rejected": -259.9742736816406,
"loss": 5.5434,
"margin_dpo/margin_mean": 0.045103684067726135,
"margin_dpo/margin_std": 0.6318709850311279,
"step": 19
},
{
"KL/chosen_KL_mean": 0.16654586791992188,
"KL/mean": 0.10966788977384567,
"KL/rejected_KL_mean": 0.052791595458984375,
"KL/std": 0.5513770580291748,
"epoch": 0.041884816753926704,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11376507580280304,
"fcm_dpo/q_t": 0.4997155964374542,
"grad_norm": 29.670021057128906,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": -0.6223905086517334,
"logits/rejected": -0.6546608805656433,
"logps/chosen": -303.7288818359375,
"logps/ref_chosen": -303.8954162597656,
"logps/ref_rejected": -260.214599609375,
"logps/rejected": -260.16180419921875,
"loss": 5.5407,
"margin_dpo/margin_mean": 0.11376452445983887,
"margin_dpo/margin_std": 0.7740194797515869,
"step": 20
},
{
"KL/chosen_KL_mean": 0.11112022399902344,
"KL/mean": 0.04954635351896286,
"KL/rejected_KL_mean": -0.01203155517578125,
"KL/std": 0.5387072563171387,
"epoch": 0.04397905759162304,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12316055595874786,
"fcm_dpo/q_t": 0.4996921122074127,
"grad_norm": 35.265602111816406,
"learning_rate": 2.0833333333333333e-07,
"logits/chosen": -0.6434299349784851,
"logits/rejected": -0.6710826754570007,
"logps/chosen": -301.4223937988281,
"logps/ref_chosen": -301.5334777832031,
"logps/ref_rejected": -280.28900146484375,
"logps/rejected": -280.301025390625,
"loss": 5.5403,
"margin_dpo/margin_mean": 0.12316262722015381,
"margin_dpo/margin_std": 0.7420811653137207,
"step": 21
},
{
"KL/chosen_KL_mean": 0.061832427978515625,
"KL/mean": 0.07514448463916779,
"KL/rejected_KL_mean": 0.08844947814941406,
"KL/std": 0.5327686667442322,
"epoch": 0.04607329842931937,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.026612281799316406,
"fcm_dpo/q_t": 0.5000665187835693,
"grad_norm": 25.43146514892578,
"learning_rate": 2.1875e-07,
"logits/chosen": -0.6658810377120972,
"logits/rejected": -0.6679869890213013,
"logps/chosen": -259.93328857421875,
"logps/ref_chosen": -259.9951477050781,
"logps/ref_rejected": -243.0721435546875,
"logps/rejected": -242.98370361328125,
"loss": 5.5463,
"margin_dpo/margin_mean": -0.026612013578414917,
"margin_dpo/margin_std": 0.780571460723877,
"step": 22
},
{
"KL/chosen_KL_mean": 0.0414276123046875,
"KL/mean": 0.010329000651836395,
"KL/rejected_KL_mean": -0.020765304565429688,
"KL/std": 0.5395455956459045,
"epoch": 0.048167539267015703,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06218160688877106,
"fcm_dpo/q_t": 0.49984458088874817,
"grad_norm": 27.855823516845703,
"learning_rate": 2.2916666666666663e-07,
"logits/chosen": -0.6225969791412354,
"logits/rejected": -0.6568140387535095,
"logps/chosen": -282.13934326171875,
"logps/ref_chosen": -282.1807556152344,
"logps/ref_rejected": -265.0758056640625,
"logps/rejected": -265.0965576171875,
"loss": 5.5428,
"margin_dpo/margin_mean": 0.06218172609806061,
"margin_dpo/margin_std": 0.7284041047096252,
"step": 23
},
{
"KL/chosen_KL_mean": 0.22072601318359375,
"KL/mean": 0.10974334180355072,
"KL/rejected_KL_mean": -0.0012378692626953125,
"KL/std": 0.5583473443984985,
"epoch": 0.050261780104712044,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22195783257484436,
"fcm_dpo/q_t": 0.4994451403617859,
"grad_norm": 29.714969635009766,
"learning_rate": 2.3958333333333335e-07,
"logits/chosen": -0.650376558303833,
"logits/rejected": -0.5625022053718567,
"logps/chosen": -300.9588623046875,
"logps/ref_chosen": -301.17962646484375,
"logps/ref_rejected": -302.12786865234375,
"logps/rejected": -302.12908935546875,
"loss": 5.5364,
"margin_dpo/margin_mean": 0.2219579666852951,
"margin_dpo/margin_std": 0.722432017326355,
"step": 24
},
{
"KL/chosen_KL_mean": 0.1817455291748047,
"KL/mean": 0.10572192817926407,
"KL/rejected_KL_mean": 0.029699325561523438,
"KL/std": 0.5168058276176453,
"epoch": 0.05235602094240838,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15205390751361847,
"fcm_dpo/q_t": 0.49961987137794495,
"grad_norm": 26.451797485351562,
"learning_rate": 2.5e-07,
"logits/chosen": -0.5981687307357788,
"logits/rejected": -0.6063342690467834,
"logps/chosen": -246.5647430419922,
"logps/ref_chosen": -246.74649047851562,
"logps/ref_rejected": -235.55638122558594,
"logps/rejected": -235.52667236328125,
"loss": 5.5392,
"margin_dpo/margin_mean": 0.15205399692058563,
"margin_dpo/margin_std": 0.7110834717750549,
"step": 25
},
{
"KL/chosen_KL_mean": 0.23089218139648438,
"KL/mean": 0.15290102362632751,
"KL/rejected_KL_mean": 0.07491302490234375,
"KL/std": 0.5813014507293701,
"epoch": 0.05445026178010471,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1559784710407257,
"fcm_dpo/q_t": 0.4996100664138794,
"grad_norm": 28.624317169189453,
"learning_rate": 2.604166666666667e-07,
"logits/chosen": -0.6492102742195129,
"logits/rejected": -0.6630299687385559,
"logps/chosen": -281.9646911621094,
"logps/ref_chosen": -282.1955871582031,
"logps/ref_rejected": -235.3135528564453,
"logps/rejected": -235.2386474609375,
"loss": 5.539,
"margin_dpo/margin_mean": 0.15597863495349884,
"margin_dpo/margin_std": 0.8204578161239624,
"step": 26
},
{
"KL/chosen_KL_mean": 0.3009986877441406,
"KL/mean": 0.2155596911907196,
"KL/rejected_KL_mean": 0.13011741638183594,
"KL/std": 0.6508041620254517,
"epoch": 0.05654450261780105,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17088854312896729,
"fcm_dpo/q_t": 0.4995727837085724,
"grad_norm": 27.787202835083008,
"learning_rate": 2.708333333333333e-07,
"logits/chosen": -0.6545995473861694,
"logits/rejected": -0.674304723739624,
"logps/chosen": -323.55535888671875,
"logps/ref_chosen": -323.8563537597656,
"logps/ref_rejected": -245.968017578125,
"logps/rejected": -245.83790588378906,
"loss": 5.5384,
"margin_dpo/margin_mean": 0.17088766396045685,
"margin_dpo/margin_std": 0.8634297847747803,
"step": 27
},
{
"KL/chosen_KL_mean": 0.2823295593261719,
"KL/mean": 0.2152925282716751,
"KL/rejected_KL_mean": 0.1482524871826172,
"KL/std": 0.5848507881164551,
"epoch": 0.05863874345549738,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1340872049331665,
"fcm_dpo/q_t": 0.4996648132801056,
"grad_norm": 26.31330680847168,
"learning_rate": 2.8125e-07,
"logits/chosen": -0.6248371005058289,
"logits/rejected": -0.6342379450798035,
"logps/chosen": -247.9644012451172,
"logps/ref_chosen": -248.24673461914062,
"logps/ref_rejected": -240.0382080078125,
"logps/rejected": -239.88995361328125,
"loss": 5.5399,
"margin_dpo/margin_mean": 0.13408710062503815,
"margin_dpo/margin_std": 0.8620131015777588,
"step": 28
},
{
"KL/chosen_KL_mean": 0.3334083557128906,
"KL/mean": 0.26686322689056396,
"KL/rejected_KL_mean": 0.20032119750976562,
"KL/std": 0.648971438407898,
"epoch": 0.060732984293193716,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1330825835466385,
"fcm_dpo/q_t": 0.49966728687286377,
"grad_norm": 29.749004364013672,
"learning_rate": 2.916666666666667e-07,
"logits/chosen": -0.590377688407898,
"logits/rejected": -0.6123830676078796,
"logps/chosen": -317.9230041503906,
"logps/ref_chosen": -318.2564392089844,
"logps/ref_rejected": -286.75848388671875,
"logps/rejected": -286.55816650390625,
"loss": 5.5399,
"margin_dpo/margin_mean": 0.13308259844779968,
"margin_dpo/margin_std": 0.8849895596504211,
"step": 29
},
{
"KL/chosen_KL_mean": 0.3189105987548828,
"KL/mean": 0.20669472217559814,
"KL/rejected_KL_mean": 0.09447669982910156,
"KL/std": 0.6948302388191223,
"epoch": 0.06282722513089005,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22443243861198425,
"fcm_dpo/q_t": 0.49943894147872925,
"grad_norm": 28.941471099853516,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": -0.5937674045562744,
"logits/rejected": -0.6108086705207825,
"logps/chosen": -252.730224609375,
"logps/ref_chosen": -253.0491485595703,
"logps/ref_rejected": -261.30029296875,
"logps/rejected": -261.205810546875,
"loss": 5.5363,
"margin_dpo/margin_mean": 0.22443242371082306,
"margin_dpo/margin_std": 0.9711648225784302,
"step": 30
},
{
"KL/chosen_KL_mean": 0.43929100036621094,
"KL/mean": 0.26924771070480347,
"KL/rejected_KL_mean": 0.09921455383300781,
"KL/std": 0.7348856925964355,
"epoch": 0.06492146596858639,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3400823473930359,
"fcm_dpo/q_t": 0.4991498291492462,
"grad_norm": 24.970046997070312,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": -0.6570526361465454,
"logits/rejected": -0.6919869184494019,
"logps/chosen": -247.71372985839844,
"logps/ref_chosen": -248.15301513671875,
"logps/ref_rejected": -203.17703247070312,
"logps/rejected": -203.0778350830078,
"loss": 5.5317,
"margin_dpo/margin_mean": 0.34008198976516724,
"margin_dpo/margin_std": 0.9680950045585632,
"step": 31
},
{
"KL/chosen_KL_mean": 0.4526844024658203,
"KL/mean": 0.3364194929599762,
"KL/rejected_KL_mean": 0.22015380859375,
"KL/std": 0.7210460901260376,
"epoch": 0.06701570680628273,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23253017663955688,
"fcm_dpo/q_t": 0.4994186758995056,
"grad_norm": 29.655418395996094,
"learning_rate": 3.2291666666666666e-07,
"logits/chosen": -0.6053009033203125,
"logits/rejected": -0.609401524066925,
"logps/chosen": -305.0872497558594,
"logps/ref_chosen": -305.5399475097656,
"logps/ref_rejected": -267.6527099609375,
"logps/rejected": -267.4325256347656,
"loss": 5.536,
"margin_dpo/margin_mean": 0.23253048956394196,
"margin_dpo/margin_std": 0.9508224725723267,
"step": 32
},
{
"KL/chosen_KL_mean": 0.6426219940185547,
"KL/mean": 0.4425170421600342,
"KL/rejected_KL_mean": 0.2424163818359375,
"KL/std": 0.869756281375885,
"epoch": 0.06910994764397906,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4001980721950531,
"fcm_dpo/q_t": 0.49899953603744507,
"grad_norm": 28.3889102935791,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.6407966613769531,
"logits/rejected": -0.6533186435699463,
"logps/chosen": -285.59088134765625,
"logps/ref_chosen": -286.2335205078125,
"logps/ref_rejected": -255.38748168945312,
"logps/rejected": -255.14508056640625,
"loss": 5.5293,
"margin_dpo/margin_mean": 0.40019840002059937,
"margin_dpo/margin_std": 1.0171304941177368,
"step": 33
},
{
"KL/chosen_KL_mean": 0.7163200378417969,
"KL/mean": 0.39098507165908813,
"KL/rejected_KL_mean": 0.06563377380371094,
"KL/std": 0.9822959899902344,
"epoch": 0.0712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6506999135017395,
"fcm_dpo/q_t": 0.4983733296394348,
"grad_norm": 31.151697158813477,
"learning_rate": 3.4375e-07,
"logits/chosen": -0.6305243372917175,
"logits/rejected": -0.6405848860740662,
"logps/chosen": -340.87567138671875,
"logps/ref_chosen": -341.5920104980469,
"logps/ref_rejected": -278.8866882324219,
"logps/rejected": -278.821044921875,
"loss": 5.5193,
"margin_dpo/margin_mean": 0.650700032711029,
"margin_dpo/margin_std": 1.2110459804534912,
"step": 34
},
{
"KL/chosen_KL_mean": 0.695404052734375,
"KL/mean": 0.49731701612472534,
"KL/rejected_KL_mean": 0.299224853515625,
"KL/std": 0.9898061752319336,
"epoch": 0.07329842931937172,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3961779773235321,
"fcm_dpo/q_t": 0.4990096092224121,
"grad_norm": 26.41559600830078,
"learning_rate": 3.541666666666667e-07,
"logits/chosen": -0.6202894449234009,
"logits/rejected": -0.6397201418876648,
"logps/chosen": -264.3841552734375,
"logps/ref_chosen": -265.0795593261719,
"logps/ref_rejected": -264.4876708984375,
"logps/rejected": -264.1884460449219,
"loss": 5.5295,
"margin_dpo/margin_mean": 0.39617788791656494,
"margin_dpo/margin_std": 1.3193421363830566,
"step": 35
},
{
"KL/chosen_KL_mean": 0.8081455230712891,
"KL/mean": 0.492267370223999,
"KL/rejected_KL_mean": 0.17638397216796875,
"KL/std": 1.098358392715454,
"epoch": 0.07539267015706806,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6317663192749023,
"fcm_dpo/q_t": 0.49842071533203125,
"grad_norm": 31.96265411376953,
"learning_rate": 3.645833333333333e-07,
"logits/chosen": -0.6001757383346558,
"logits/rejected": -0.6177087426185608,
"logps/chosen": -296.5179748535156,
"logps/ref_chosen": -297.3261413574219,
"logps/ref_rejected": -282.09515380859375,
"logps/rejected": -281.91876220703125,
"loss": 5.5201,
"margin_dpo/margin_mean": 0.6317658424377441,
"margin_dpo/margin_std": 1.3704296350479126,
"step": 36
},
{
"KL/chosen_KL_mean": 0.6605033874511719,
"KL/mean": 0.4618363380432129,
"KL/rejected_KL_mean": 0.26319122314453125,
"KL/std": 1.0892057418823242,
"epoch": 0.0774869109947644,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3973109722137451,
"fcm_dpo/q_t": 0.49900680780410767,
"grad_norm": 30.963905334472656,
"learning_rate": 3.75e-07,
"logits/chosen": -0.6110119223594666,
"logits/rejected": -0.6263231039047241,
"logps/chosen": -313.37353515625,
"logps/ref_chosen": -314.0340270996094,
"logps/ref_rejected": -299.3437805175781,
"logps/rejected": -299.08056640625,
"loss": 5.5295,
"margin_dpo/margin_mean": 0.39730948209762573,
"margin_dpo/margin_std": 1.4675891399383545,
"step": 37
},
{
"KL/chosen_KL_mean": 1.0695934295654297,
"KL/mean": 0.758275032043457,
"KL/rejected_KL_mean": 0.4469585418701172,
"KL/std": 1.2175971269607544,
"epoch": 0.07958115183246073,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6226468086242676,
"fcm_dpo/q_t": 0.49844348430633545,
"grad_norm": 28.48445701599121,
"learning_rate": 3.8541666666666665e-07,
"logits/chosen": -0.6337790489196777,
"logits/rejected": -0.6456969976425171,
"logps/chosen": -281.4715881347656,
"logps/ref_chosen": -282.54119873046875,
"logps/ref_rejected": -269.7773132324219,
"logps/rejected": -269.330322265625,
"loss": 5.5206,
"margin_dpo/margin_mean": 0.622646689414978,
"margin_dpo/margin_std": 1.480731725692749,
"step": 38
},
{
"KL/chosen_KL_mean": 1.3233833312988281,
"KL/mean": 0.7047661542892456,
"KL/rejected_KL_mean": 0.08614921569824219,
"KL/std": 1.4990904331207275,
"epoch": 0.08167539267015707,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2372335195541382,
"fcm_dpo/q_t": 0.4969073534011841,
"grad_norm": 29.540149688720703,
"learning_rate": 3.958333333333333e-07,
"logits/chosen": -0.6165052056312561,
"logits/rejected": -0.6305378675460815,
"logps/chosen": -275.44952392578125,
"logps/ref_chosen": -276.7729187011719,
"logps/ref_rejected": -249.95889282226562,
"logps/rejected": -249.87274169921875,
"loss": 5.4962,
"margin_dpo/margin_mean": 1.2372331619262695,
"margin_dpo/margin_std": 1.7841914892196655,
"step": 39
},
{
"KL/chosen_KL_mean": 1.01873779296875,
"KL/mean": 0.6700364947319031,
"KL/rejected_KL_mean": 0.32134056091308594,
"KL/std": 1.3435927629470825,
"epoch": 0.08376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6973839998245239,
"fcm_dpo/q_t": 0.4982568323612213,
"grad_norm": 27.431995391845703,
"learning_rate": 4.0625e-07,
"logits/chosen": -0.6209850907325745,
"logits/rejected": -0.6562680006027222,
"logps/chosen": -283.2883605957031,
"logps/ref_chosen": -284.30706787109375,
"logps/ref_rejected": -244.4459991455078,
"logps/rejected": -244.1246337890625,
"loss": 5.5177,
"margin_dpo/margin_mean": 0.6973843574523926,
"margin_dpo/margin_std": 1.787576675415039,
"step": 40
},
{
"KL/chosen_KL_mean": 1.0851669311523438,
"KL/mean": 0.7243558168411255,
"KL/rejected_KL_mean": 0.3635540008544922,
"KL/std": 1.4870356321334839,
"epoch": 0.08586387434554973,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7216100096702576,
"fcm_dpo/q_t": 0.4981961250305176,
"grad_norm": 30.58069610595703,
"learning_rate": 4.1666666666666667e-07,
"logits/chosen": -0.6218698620796204,
"logits/rejected": -0.646290123462677,
"logps/chosen": -292.72998046875,
"logps/ref_chosen": -293.8151550292969,
"logps/ref_rejected": -252.16815185546875,
"logps/rejected": -251.80459594726562,
"loss": 5.5167,
"margin_dpo/margin_mean": 0.7216103672981262,
"margin_dpo/margin_std": 1.7212419509887695,
"step": 41
},
{
"KL/chosen_KL_mean": 1.3099021911621094,
"KL/mean": 0.8193244934082031,
"KL/rejected_KL_mean": 0.3287544250488281,
"KL/std": 1.7772796154022217,
"epoch": 0.08795811518324607,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9811455011367798,
"fcm_dpo/q_t": 0.4975475072860718,
"grad_norm": 27.563005447387695,
"learning_rate": 4.270833333333333e-07,
"logits/chosen": -0.6424188613891602,
"logits/rejected": -0.6603102684020996,
"logps/chosen": -251.45033264160156,
"logps/ref_chosen": -252.76023864746094,
"logps/ref_rejected": -261.0414733886719,
"logps/rejected": -260.7127380371094,
"loss": 5.5066,
"margin_dpo/margin_mean": 0.9811455607414246,
"margin_dpo/margin_std": 2.2941291332244873,
"step": 42
},
{
"KL/chosen_KL_mean": 1.2875423431396484,
"KL/mean": 0.6953762173652649,
"KL/rejected_KL_mean": 0.10320854187011719,
"KL/std": 1.8649598360061646,
"epoch": 0.09005235602094241,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.18433678150177,
"fcm_dpo/q_t": 0.4970394968986511,
"grad_norm": 29.913719177246094,
"learning_rate": 4.375e-07,
"logits/chosen": -0.5954654216766357,
"logits/rejected": -0.611076295375824,
"logps/chosen": -315.5472106933594,
"logps/ref_chosen": -316.8347473144531,
"logps/ref_rejected": -273.7649230957031,
"logps/rejected": -273.66168212890625,
"loss": 5.4985,
"margin_dpo/margin_mean": 1.1843376159667969,
"margin_dpo/margin_std": 2.194983720779419,
"step": 43
},
{
"KL/chosen_KL_mean": 1.4896392822265625,
"KL/mean": 0.7002232670783997,
"KL/rejected_KL_mean": -0.08919525146484375,
"KL/std": 2.1647138595581055,
"epoch": 0.09214659685863874,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.578826665878296,
"fcm_dpo/q_t": 0.49605363607406616,
"grad_norm": 30.989070892333984,
"learning_rate": 4.479166666666667e-07,
"logits/chosen": -0.5941963791847229,
"logits/rejected": -0.5926939845085144,
"logps/chosen": -285.3860168457031,
"logps/ref_chosen": -286.8757019042969,
"logps/ref_rejected": -282.4681396484375,
"logps/rejected": -282.5573425292969,
"loss": 5.4832,
"margin_dpo/margin_mean": 1.5788276195526123,
"margin_dpo/margin_std": 2.984520435333252,
"step": 44
},
{
"KL/chosen_KL_mean": 1.6895427703857422,
"KL/mean": 1.006917119026184,
"KL/rejected_KL_mean": 0.3242931365966797,
"KL/std": 2.26188063621521,
"epoch": 0.09424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3652527332305908,
"fcm_dpo/q_t": 0.49658796191215515,
"grad_norm": 28.858821868896484,
"learning_rate": 4.5833333333333327e-07,
"logits/chosen": -0.6818274259567261,
"logits/rejected": -0.7043554782867432,
"logps/chosen": -322.5738525390625,
"logps/ref_chosen": -324.2633972167969,
"logps/ref_rejected": -293.09466552734375,
"logps/rejected": -292.7703857421875,
"loss": 5.4916,
"margin_dpo/margin_mean": 1.36525297164917,
"margin_dpo/margin_std": 2.8198041915893555,
"step": 45
},
{
"KL/chosen_KL_mean": 1.6807384490966797,
"KL/mean": 0.9394608736038208,
"KL/rejected_KL_mean": 0.19817733764648438,
"KL/std": 2.450871467590332,
"epoch": 0.09633507853403141,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4825652837753296,
"fcm_dpo/q_t": 0.4962944984436035,
"grad_norm": 30.326108932495117,
"learning_rate": 4.6874999999999996e-07,
"logits/chosen": -0.6200026869773865,
"logits/rejected": -0.633066713809967,
"logps/chosen": -296.6549987792969,
"logps/ref_chosen": -298.3357238769531,
"logps/ref_rejected": -267.66204833984375,
"logps/rejected": -267.4638366699219,
"loss": 5.4869,
"margin_dpo/margin_mean": 1.4825655221939087,
"margin_dpo/margin_std": 2.687807321548462,
"step": 46
},
{
"KL/chosen_KL_mean": 1.51885986328125,
"KL/mean": 1.0282460451126099,
"KL/rejected_KL_mean": 0.5376300811767578,
"KL/std": 2.3429102897644043,
"epoch": 0.09842931937172775,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9812304973602295,
"fcm_dpo/q_t": 0.4975472688674927,
"grad_norm": 26.336135864257812,
"learning_rate": 4.791666666666667e-07,
"logits/chosen": -0.5996535420417786,
"logits/rejected": -0.6199891567230225,
"logps/chosen": -261.0480651855469,
"logps/ref_chosen": -262.5669250488281,
"logps/ref_rejected": -258.70989990234375,
"logps/rejected": -258.1722412109375,
"loss": 5.5073,
"margin_dpo/margin_mean": 0.9812304973602295,
"margin_dpo/margin_std": 3.329333782196045,
"step": 47
},
{
"KL/chosen_KL_mean": 1.8202342987060547,
"KL/mean": 1.0875205993652344,
"KL/rejected_KL_mean": 0.3548126220703125,
"KL/std": 2.496361494064331,
"epoch": 0.10052356020942409,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4654216766357422,
"fcm_dpo/q_t": 0.49633827805519104,
"grad_norm": 27.6088924407959,
"learning_rate": 4.895833333333333e-07,
"logits/chosen": -0.6158616542816162,
"logits/rejected": -0.6411494016647339,
"logps/chosen": -267.67303466796875,
"logps/ref_chosen": -269.4932556152344,
"logps/ref_rejected": -241.888916015625,
"logps/rejected": -241.5341033935547,
"loss": 5.4877,
"margin_dpo/margin_mean": 1.465421199798584,
"margin_dpo/margin_std": 2.838312864303589,
"step": 48
},
{
"KL/chosen_KL_mean": 2.259225845336914,
"KL/mean": 1.3766483068466187,
"KL/rejected_KL_mean": 0.4940757751464844,
"KL/std": 2.617175817489624,
"epoch": 0.10261780104712041,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7651441097259521,
"fcm_dpo/q_t": 0.4955885410308838,
"grad_norm": 27.615480422973633,
"learning_rate": 5e-07,
"logits/chosen": -0.6648316383361816,
"logits/rejected": -0.6493117809295654,
"logps/chosen": -255.62527465820312,
"logps/ref_chosen": -257.8844909667969,
"logps/ref_rejected": -256.8912048339844,
"logps/rejected": -256.3971252441406,
"loss": 5.4761,
"margin_dpo/margin_mean": 1.7651444673538208,
"margin_dpo/margin_std": 3.3019487857818604,
"step": 49
},
{
"KL/chosen_KL_mean": 2.061922073364258,
"KL/mean": 1.5143541097640991,
"KL/rejected_KL_mean": 0.9667835235595703,
"KL/std": 3.0319771766662598,
"epoch": 0.10471204188481675,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0951329469680786,
"fcm_dpo/q_t": 0.4972635805606842,
"grad_norm": 28.010818481445312,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": -0.6277458667755127,
"logits/rejected": -0.6493853330612183,
"logps/chosen": -299.56695556640625,
"logps/ref_chosen": -301.62884521484375,
"logps/ref_rejected": -298.2716064453125,
"logps/rejected": -297.3048400878906,
"loss": 5.5031,
"margin_dpo/margin_mean": 1.09513258934021,
"margin_dpo/margin_std": 3.8171825408935547,
"step": 50
},
{
"KL/chosen_KL_mean": 2.1011962890625,
"KL/mean": 1.1179943084716797,
"KL/rejected_KL_mean": 0.1347827911376953,
"KL/std": 3.6277670860290527,
"epoch": 0.1068062827225131,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.966414451599121,
"fcm_dpo/q_t": 0.4950869083404541,
"grad_norm": 29.124319076538086,
"learning_rate": 4.999731868769026e-07,
"logits/chosen": -0.5978573560714722,
"logits/rejected": -0.5924739241600037,
"logps/chosen": -267.27117919921875,
"logps/ref_chosen": -269.37237548828125,
"logps/ref_rejected": -297.0167541503906,
"logps/rejected": -296.8819580078125,
"loss": 5.4694,
"margin_dpo/margin_mean": 1.9664149284362793,
"margin_dpo/margin_std": 4.608149528503418,
"step": 51
},
{
"KL/chosen_KL_mean": 2.7861671447753906,
"KL/mean": 1.2450693845748901,
"KL/rejected_KL_mean": -0.2960319519042969,
"KL/std": 3.817547082901001,
"epoch": 0.10890052356020942,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.0822057723999023,
"fcm_dpo/q_t": 0.49230119585990906,
"grad_norm": 30.227222442626953,
"learning_rate": 4.99939671821067e-07,
"logits/chosen": -0.6642853617668152,
"logits/rejected": -0.6713843941688538,
"logps/chosen": -304.1166076660156,
"logps/ref_chosen": -306.9028015136719,
"logps/ref_rejected": -281.24737548828125,
"logps/rejected": -281.54339599609375,
"loss": 5.4253,
"margin_dpo/margin_mean": 3.082204580307007,
"margin_dpo/margin_std": 4.780548095703125,
"step": 52
},
{
"KL/chosen_KL_mean": 2.8322067260742188,
"KL/mean": 1.739880919456482,
"KL/rejected_KL_mean": 0.647552490234375,
"KL/std": 4.097768306732178,
"epoch": 0.11099476439790576,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.1846604347229004,
"fcm_dpo/q_t": 0.49454259872436523,
"grad_norm": 29.97759246826172,
"learning_rate": 4.998927532591591e-07,
"logits/chosen": -0.6540624499320984,
"logits/rejected": -0.6947383284568787,
"logps/chosen": -283.1437072753906,
"logps/ref_chosen": -285.9759521484375,
"logps/ref_rejected": -273.9073486328125,
"logps/rejected": -273.2597961425781,
"loss": 5.4613,
"margin_dpo/margin_mean": 2.184659481048584,
"margin_dpo/margin_std": 5.051357269287109,
"step": 53
},
{
"KL/chosen_KL_mean": 2.740182876586914,
"KL/mean": 1.7801817655563354,
"KL/rejected_KL_mean": 0.8201732635498047,
"KL/std": 4.306924819946289,
"epoch": 0.1130890052356021,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9200044870376587,
"fcm_dpo/q_t": 0.4952048659324646,
"grad_norm": 26.368751525878906,
"learning_rate": 4.998324337072792e-07,
"logits/chosen": -0.6892499923706055,
"logits/rejected": -0.6974684000015259,
"logps/chosen": -303.7644348144531,
"logps/ref_chosen": -306.504638671875,
"logps/ref_rejected": -272.67431640625,
"logps/rejected": -271.8541564941406,
"loss": 5.4719,
"margin_dpo/margin_mean": 1.9200036525726318,
"margin_dpo/margin_std": 5.399931907653809,
"step": 54
},
{
"KL/chosen_KL_mean": 1.7780208587646484,
"KL/mean": 0.6723177433013916,
"KL/rejected_KL_mean": -0.4333839416503906,
"KL/std": 3.7362210750579834,
"epoch": 0.11518324607329843,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.211399555206299,
"fcm_dpo/q_t": 0.4944767355918884,
"grad_norm": 24.839492797851562,
"learning_rate": 4.997587164001815e-07,
"logits/chosen": -0.6404720544815063,
"logits/rejected": -0.6446375250816345,
"logps/chosen": -220.5521240234375,
"logps/ref_chosen": -222.33013916015625,
"logps/ref_rejected": -206.59571838378906,
"logps/rejected": -207.02911376953125,
"loss": 5.4598,
"margin_dpo/margin_mean": 2.211399555206299,
"margin_dpo/margin_std": 4.992412567138672,
"step": 55
},
{
"KL/chosen_KL_mean": 2.8933238983154297,
"KL/mean": 1.2889918088912964,
"KL/rejected_KL_mean": -0.31532859802246094,
"KL/std": 4.780613422393799,
"epoch": 0.11727748691099477,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.2086493968963623,
"fcm_dpo/q_t": 0.49199068546295166,
"grad_norm": 27.42078399658203,
"learning_rate": 4.996716052911017e-07,
"logits/chosen": -0.6034107804298401,
"logits/rejected": -0.6173827648162842,
"logps/chosen": -247.5848388671875,
"logps/ref_chosen": -250.47816467285156,
"logps/ref_rejected": -228.25848388671875,
"logps/rejected": -228.57382202148438,
"loss": 5.4221,
"margin_dpo/margin_mean": 3.208649158477783,
"margin_dpo/margin_std": 5.916287899017334,
"step": 56
},
{
"KL/chosen_KL_mean": 3.8232650756835938,
"KL/mean": 1.7507920265197754,
"KL/rejected_KL_mean": -0.3216896057128906,
"KL/std": 4.765433311462402,
"epoch": 0.1193717277486911,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.14495849609375,
"fcm_dpo/q_t": 0.48964935541152954,
"grad_norm": 31.133817672729492,
"learning_rate": 4.99571105051544e-07,
"logits/chosen": -0.6981046199798584,
"logits/rejected": -0.668836772441864,
"logps/chosen": -311.2962646484375,
"logps/ref_chosen": -315.1195373535156,
"logps/ref_rejected": -272.755615234375,
"logps/rejected": -273.0773010253906,
"loss": 5.3845,
"margin_dpo/margin_mean": 4.14495849609375,
"margin_dpo/margin_std": 5.633490085601807,
"step": 57
},
{
"KL/chosen_KL_mean": 2.441774368286133,
"KL/mean": 1.288618803024292,
"KL/rejected_KL_mean": 0.13546371459960938,
"KL/std": 4.413588047027588,
"epoch": 0.12146596858638743,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.30631685256958,
"fcm_dpo/q_t": 0.4942378103733063,
"grad_norm": 28.11114501953125,
"learning_rate": 4.994572210710314e-07,
"logits/chosen": -0.6136592030525208,
"logits/rejected": -0.637704610824585,
"logps/chosen": -262.7398986816406,
"logps/ref_chosen": -265.1816711425781,
"logps/ref_rejected": -268.2203369140625,
"logps/rejected": -268.0848388671875,
"loss": 5.457,
"margin_dpo/margin_mean": 2.306316614151001,
"margin_dpo/margin_std": 5.692234992980957,
"step": 58
},
{
"KL/chosen_KL_mean": 2.0896434783935547,
"KL/mean": 1.329612374305725,
"KL/rejected_KL_mean": 0.5695877075195312,
"KL/std": 5.218595504760742,
"epoch": 0.12356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5200635194778442,
"fcm_dpo/q_t": 0.49620577692985535,
"grad_norm": 29.61176109313965,
"learning_rate": 4.993299594568162e-07,
"logits/chosen": -0.6077237725257874,
"logits/rejected": -0.601131796836853,
"logps/chosen": -284.2643127441406,
"logps/ref_chosen": -286.35394287109375,
"logps/ref_rejected": -260.6757507324219,
"logps/rejected": -260.106201171875,
"loss": 5.4902,
"margin_dpo/margin_mean": 1.5200631618499756,
"margin_dpo/margin_std": 6.907461166381836,
"step": 59
},
{
"KL/chosen_KL_mean": 2.9706172943115234,
"KL/mean": 1.2671046257019043,
"KL/rejected_KL_mean": -0.43640899658203125,
"KL/std": 5.212099075317383,
"epoch": 0.1256544502617801,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.407027244567871,
"fcm_dpo/q_t": 0.49149319529533386,
"grad_norm": 27.876686096191406,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": -0.6783978343009949,
"logits/rejected": -0.703014612197876,
"logps/chosen": -255.7779998779297,
"logps/ref_chosen": -258.74859619140625,
"logps/ref_rejected": -255.04893493652344,
"logps/rejected": -255.48533630371094,
"loss": 5.4156,
"margin_dpo/margin_mean": 3.4070284366607666,
"margin_dpo/margin_std": 7.180816650390625,
"step": 60
},
{
"KL/chosen_KL_mean": 3.0242786407470703,
"KL/mean": 1.413300633430481,
"KL/rejected_KL_mean": -0.1976757049560547,
"KL/std": 5.535090923309326,
"epoch": 0.12774869109947645,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.2219600677490234,
"fcm_dpo/q_t": 0.4919591248035431,
"grad_norm": 29.932416915893555,
"learning_rate": 4.990353313429303e-07,
"logits/chosen": -0.6423856616020203,
"logits/rejected": -0.6588336229324341,
"logps/chosen": -275.4436340332031,
"logps/ref_chosen": -278.4678955078125,
"logps/ref_rejected": -252.02720642089844,
"logps/rejected": -252.22488403320312,
"loss": 5.4239,
"margin_dpo/margin_mean": 3.2219598293304443,
"margin_dpo/margin_std": 7.598827838897705,
"step": 61
},
{
"KL/chosen_KL_mean": 4.0137786865234375,
"KL/mean": 2.280641794204712,
"KL/rejected_KL_mean": 0.5475139617919922,
"KL/std": 5.737850189208984,
"epoch": 0.12984293193717278,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.466265916824341,
"fcm_dpo/q_t": 0.4913550615310669,
"grad_norm": 26.708263397216797,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": -0.6178765296936035,
"logits/rejected": -0.6604681611061096,
"logps/chosen": -268.9105224609375,
"logps/ref_chosen": -272.92431640625,
"logps/ref_rejected": -260.7935485839844,
"logps/rejected": -260.2460632324219,
"loss": 5.4135,
"margin_dpo/margin_mean": 3.46626615524292,
"margin_dpo/margin_std": 7.4036478996276855,
"step": 62
},
{
"KL/chosen_KL_mean": 3.3519554138183594,
"KL/mean": 1.6410435438156128,
"KL/rejected_KL_mean": -0.06987190246582031,
"KL/std": 5.6098198890686035,
"epoch": 0.1319371727748691,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.4218201637268066,
"fcm_dpo/q_t": 0.49145957827568054,
"grad_norm": 28.320823669433594,
"learning_rate": 4.986872839090852e-07,
"logits/chosen": -0.6582809686660767,
"logits/rejected": -0.6645787954330444,
"logps/chosen": -273.7370300292969,
"logps/ref_chosen": -277.0889892578125,
"logps/ref_rejected": -273.3413391113281,
"logps/rejected": -273.41119384765625,
"loss": 5.4151,
"margin_dpo/margin_mean": 3.4218192100524902,
"margin_dpo/margin_std": 7.18504524230957,
"step": 63
},
{
"KL/chosen_KL_mean": 4.1611785888671875,
"KL/mean": 2.0322928428649902,
"KL/rejected_KL_mean": -0.09659004211425781,
"KL/std": 6.245217800140381,
"epoch": 0.13403141361256546,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.2577714920043945,
"fcm_dpo/q_t": 0.4893771708011627,
"grad_norm": 28.30405616760254,
"learning_rate": 4.9849325083059e-07,
"logits/chosen": -0.6383385062217712,
"logits/rejected": -0.6341136693954468,
"logps/chosen": -279.6632995605469,
"logps/ref_chosen": -283.8244934082031,
"logps/ref_rejected": -263.29351806640625,
"logps/rejected": -263.39007568359375,
"loss": 5.3841,
"margin_dpo/margin_mean": 4.257771015167236,
"margin_dpo/margin_std": 8.265096664428711,
"step": 64
},
{
"KL/chosen_KL_mean": 3.4199390411376953,
"KL/mean": 1.7757415771484375,
"KL/rejected_KL_mean": 0.1315479278564453,
"KL/std": 5.751835823059082,
"epoch": 0.13612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.2883832454681396,
"fcm_dpo/q_t": 0.4917920231819153,
"grad_norm": 27.830177307128906,
"learning_rate": 4.982858918131906e-07,
"logits/chosen": -0.705287754535675,
"logits/rejected": -0.6732350587844849,
"logps/chosen": -261.45001220703125,
"logps/ref_chosen": -264.8699645996094,
"logps/ref_rejected": -268.5076904296875,
"logps/rejected": -268.37615966796875,
"loss": 5.4205,
"margin_dpo/margin_mean": 3.2883825302124023,
"margin_dpo/margin_std": 7.373363971710205,
"step": 65
},
{
"KL/chosen_KL_mean": 3.138427734375,
"KL/mean": 1.04404616355896,
"KL/rejected_KL_mean": -1.0503253936767578,
"KL/std": 7.417426109313965,
"epoch": 0.1382198952879581,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.188744068145752,
"fcm_dpo/q_t": 0.4895643889904022,
"grad_norm": 27.398483276367188,
"learning_rate": 4.980652179769217e-07,
"logits/chosen": -0.6739647388458252,
"logits/rejected": -0.6936634182929993,
"logps/chosen": -269.7898864746094,
"logps/ref_chosen": -272.9283142089844,
"logps/ref_rejected": -280.94696044921875,
"logps/rejected": -281.9972839355469,
"loss": 5.3898,
"margin_dpo/margin_mean": 4.188744068145752,
"margin_dpo/margin_std": 10.030872344970703,
"step": 66
},
{
"KL/chosen_KL_mean": 4.055376052856445,
"KL/mean": 2.244265079498291,
"KL/rejected_KL_mean": 0.43314552307128906,
"KL/std": 7.153616428375244,
"epoch": 0.14031413612565444,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.6222338676452637,
"fcm_dpo/q_t": 0.4909643232822418,
"grad_norm": 25.641693115234375,
"learning_rate": 4.978312411558517e-07,
"logits/chosen": -0.7017156481742859,
"logits/rejected": -0.7357592582702637,
"logps/chosen": -262.1315612792969,
"logps/ref_chosen": -266.18695068359375,
"logps/ref_rejected": -250.17405700683594,
"logps/rejected": -249.74093627929688,
"loss": 5.41,
"margin_dpo/margin_mean": 3.6222333908081055,
"margin_dpo/margin_std": 8.869529724121094,
"step": 67
},
{
"KL/chosen_KL_mean": 3.0623416900634766,
"KL/mean": 0.7446590065956116,
"KL/rejected_KL_mean": -1.5730171203613281,
"KL/std": 8.063435554504395,
"epoch": 0.1424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.635357856750488,
"fcm_dpo/q_t": 0.48847031593322754,
"grad_norm": 27.99388313293457,
"learning_rate": 4.975839738974473e-07,
"logits/chosen": -0.6817157864570618,
"logits/rejected": -0.6947129368782043,
"logps/chosen": -294.8762512207031,
"logps/ref_chosen": -297.9385986328125,
"logps/ref_rejected": -261.5141296386719,
"logps/rejected": -263.0871276855469,
"loss": 5.374,
"margin_dpo/margin_mean": 4.635357856750488,
"margin_dpo/margin_std": 10.582465171813965,
"step": 68
},
{
"KL/chosen_KL_mean": 3.916715621948242,
"KL/mean": 0.8802259564399719,
"KL/rejected_KL_mean": -2.156261444091797,
"KL/std": 7.877801418304443,
"epoch": 0.14450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.072974681854248,
"fcm_dpo/q_t": 0.4848654866218567,
"grad_norm": 28.498083114624023,
"learning_rate": 4.97323429461901e-07,
"logits/chosen": -0.6778756380081177,
"logits/rejected": -0.7077374458312988,
"logps/chosen": -261.7008056640625,
"logps/ref_chosen": -265.6175231933594,
"logps/ref_rejected": -236.8287353515625,
"logps/rejected": -238.9849853515625,
"loss": 5.3158,
"margin_dpo/margin_mean": 6.0729756355285645,
"margin_dpo/margin_std": 9.63194465637207,
"step": 69
},
{
"KL/chosen_KL_mean": 4.352378845214844,
"KL/mean": 1.3984824419021606,
"KL/rejected_KL_mean": -1.5554084777832031,
"KL/std": 8.641536712646484,
"epoch": 0.14659685863874344,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.907784461975098,
"fcm_dpo/q_t": 0.4852859377861023,
"grad_norm": 28.619237899780273,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": -0.6764180064201355,
"logits/rejected": -0.7098894119262695,
"logps/chosen": -291.87353515625,
"logps/ref_chosen": -296.2259216308594,
"logps/ref_rejected": -254.68496704101562,
"logps/rejected": -256.2403564453125,
"loss": 5.3252,
"margin_dpo/margin_mean": 5.907782554626465,
"margin_dpo/margin_std": 10.884395599365234,
"step": 70
},
{
"KL/chosen_KL_mean": 5.073089599609375,
"KL/mean": 2.5308048725128174,
"KL/rejected_KL_mean": -0.011476516723632812,
"KL/std": 9.272811889648438,
"epoch": 0.1486910994764398,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.084573268890381,
"fcm_dpo/q_t": 0.48732662200927734,
"grad_norm": 28.492225646972656,
"learning_rate": 4.967625656594781e-07,
"logits/chosen": -0.6502672433853149,
"logits/rejected": -0.6391031742095947,
"logps/chosen": -283.8541564941406,
"logps/ref_chosen": -288.92724609375,
"logps/ref_rejected": -278.6405334472656,
"logps/rejected": -278.6519775390625,
"loss": 5.3608,
"margin_dpo/margin_mean": 5.084572792053223,
"margin_dpo/margin_std": 12.225592613220215,
"step": 71
},
{
"KL/chosen_KL_mean": 3.342805862426758,
"KL/mean": 0.7642459869384766,
"KL/rejected_KL_mean": -1.8143062591552734,
"KL/std": 8.467608451843262,
"epoch": 0.15078534031413612,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.157101631164551,
"fcm_dpo/q_t": 0.4871388077735901,
"grad_norm": 28.240171432495117,
"learning_rate": 4.964622763700252e-07,
"logits/chosen": -0.692705512046814,
"logits/rejected": -0.7039174437522888,
"logps/chosen": -233.70248413085938,
"logps/ref_chosen": -237.0452880859375,
"logps/ref_rejected": -252.7946319580078,
"logps/rejected": -254.6089324951172,
"loss": 5.3542,
"margin_dpo/margin_mean": 5.157101631164551,
"margin_dpo/margin_std": 10.475278854370117,
"step": 72
},
{
"KL/chosen_KL_mean": 4.400068283081055,
"KL/mean": 1.9903484582901,
"KL/rejected_KL_mean": -0.41938209533691406,
"KL/std": 8.966333389282227,
"epoch": 0.15287958115183245,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.81945276260376,
"fcm_dpo/q_t": 0.4879787862300873,
"grad_norm": 27.833248138427734,
"learning_rate": 4.961487700566646e-07,
"logits/chosen": -0.6518418192863464,
"logits/rejected": -0.668941855430603,
"logps/chosen": -268.65301513671875,
"logps/ref_chosen": -273.0531005859375,
"logps/ref_rejected": -246.8330841064453,
"logps/rejected": -247.25247192382812,
"loss": 5.3711,
"margin_dpo/margin_mean": 4.81945276260376,
"margin_dpo/margin_std": 12.24702262878418,
"step": 73
},
{
"KL/chosen_KL_mean": 3.4003162384033203,
"KL/mean": 1.1663212776184082,
"KL/rejected_KL_mean": -1.0676746368408203,
"KL/std": 9.370096206665039,
"epoch": 0.1549738219895288,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.4679975509643555,
"fcm_dpo/q_t": 0.48887601494789124,
"grad_norm": 30.320070266723633,
"learning_rate": 4.958220635317885e-07,
"logits/chosen": -0.7227351665496826,
"logits/rejected": -0.7008697986602783,
"logps/chosen": -338.881591796875,
"logps/ref_chosen": -342.2818908691406,
"logps/ref_rejected": -330.0293884277344,
"logps/rejected": -331.0970153808594,
"loss": 5.3832,
"margin_dpo/margin_mean": 4.4679975509643555,
"margin_dpo/margin_std": 11.65971565246582,
"step": 74
},
{
"KL/chosen_KL_mean": 4.72509765625,
"KL/mean": 1.3391616344451904,
"KL/rejected_KL_mean": -2.0467777252197266,
"KL/std": 9.442550659179688,
"epoch": 0.15706806282722513,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.771880149841309,
"fcm_dpo/q_t": 0.4831286668777466,
"grad_norm": 29.469329833984375,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": -0.6357704401016235,
"logits/rejected": -0.6371195316314697,
"logps/chosen": -262.1390380859375,
"logps/ref_chosen": -266.8641662597656,
"logps/ref_rejected": -276.8699951171875,
"logps/rejected": -278.9167785644531,
"loss": 5.2922,
"margin_dpo/margin_mean": 6.771879196166992,
"margin_dpo/margin_std": 10.950206756591797,
"step": 75
},
{
"KL/chosen_KL_mean": 4.118385314941406,
"KL/mean": 0.5083938837051392,
"KL/rejected_KL_mean": -3.101591110229492,
"KL/std": 9.978694915771484,
"epoch": 0.15916230366492146,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.219972610473633,
"fcm_dpo/q_t": 0.48206230998039246,
"grad_norm": 29.70305061340332,
"learning_rate": 4.951291206355559e-07,
"logits/chosen": -0.7215156555175781,
"logits/rejected": -0.72944575548172,
"logps/chosen": -277.05621337890625,
"logps/ref_chosen": -281.174560546875,
"logps/ref_rejected": -263.6067199707031,
"logps/rejected": -266.70831298828125,
"loss": 5.2794,
"margin_dpo/margin_mean": 7.219972610473633,
"margin_dpo/margin_std": 12.839584350585938,
"step": 76
},
{
"KL/chosen_KL_mean": 3.7498321533203125,
"KL/mean": 0.8297483921051025,
"KL/rejected_KL_mean": -2.0903377532958984,
"KL/std": 11.009910583496094,
"epoch": 0.1612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.840177059173584,
"fcm_dpo/q_t": 0.4854981005191803,
"grad_norm": 33.02766418457031,
"learning_rate": 4.947629214246236e-07,
"logits/chosen": -0.5660206079483032,
"logits/rejected": -0.5753567814826965,
"logps/chosen": -302.345458984375,
"logps/ref_chosen": -306.09527587890625,
"logps/ref_rejected": -253.49569702148438,
"logps/rejected": -255.58602905273438,
"loss": 5.3359,
"margin_dpo/margin_mean": 5.840176582336426,
"margin_dpo/margin_std": 13.89721393585205,
"step": 77
},
{
"KL/chosen_KL_mean": 3.9143428802490234,
"KL/mean": -0.4267261326313019,
"KL/rejected_KL_mean": -4.767789840698242,
"KL/std": 10.995488166809082,
"epoch": 0.16335078534031414,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.68213176727295,
"fcm_dpo/q_t": 0.4784572124481201,
"grad_norm": 29.675912857055664,
"learning_rate": 4.943835963210323e-07,
"logits/chosen": -0.6788281202316284,
"logits/rejected": -0.6731718182563782,
"logps/chosen": -252.9879913330078,
"logps/ref_chosen": -256.90234375,
"logps/ref_rejected": -211.57154846191406,
"logps/rejected": -216.33935546875,
"loss": 5.227,
"margin_dpo/margin_mean": 8.68213176727295,
"margin_dpo/margin_std": 14.381519317626953,
"step": 78
},
{
"KL/chosen_KL_mean": 3.0857925415039062,
"KL/mean": -1.0710463523864746,
"KL/rejected_KL_mean": -5.227884292602539,
"KL/std": 11.816179275512695,
"epoch": 0.16544502617801046,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.313674926757812,
"fcm_dpo/q_t": 0.4793742299079895,
"grad_norm": 30.00199317932129,
"learning_rate": 4.939911656668361e-07,
"logits/chosen": -0.6557655334472656,
"logits/rejected": -0.6764158010482788,
"logps/chosen": -263.18780517578125,
"logps/ref_chosen": -266.2735595703125,
"logps/ref_rejected": -251.57257080078125,
"logps/rejected": -256.8004455566406,
"loss": 5.2437,
"margin_dpo/margin_mean": 8.313674926757812,
"margin_dpo/margin_std": 15.012711524963379,
"step": 79
},
{
"KL/chosen_KL_mean": 1.8700141906738281,
"KL/mean": -1.4277009963989258,
"KL/rejected_KL_mean": -4.725408554077148,
"KL/std": 11.388775825500488,
"epoch": 0.16753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.595414161682129,
"fcm_dpo/q_t": 0.48363250494003296,
"grad_norm": 28.94468879699707,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": -0.6733145117759705,
"logits/rejected": -0.7031750082969666,
"logps/chosen": -285.98095703125,
"logps/ref_chosen": -287.8509826660156,
"logps/ref_rejected": -256.0766296386719,
"logps/rejected": -260.802001953125,
"loss": 5.3057,
"margin_dpo/margin_mean": 6.595414638519287,
"margin_dpo/margin_std": 13.258899688720703,
"step": 80
},
{
"KL/chosen_KL_mean": 2.3981895446777344,
"KL/mean": -1.178763747215271,
"KL/rejected_KL_mean": -4.755710601806641,
"KL/std": 12.212273597717285,
"epoch": 0.16963350785340314,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.153894424438477,
"fcm_dpo/q_t": 0.48225945234298706,
"grad_norm": 28.060094833374023,
"learning_rate": 4.93167072587771e-07,
"logits/chosen": -0.6365951299667358,
"logits/rejected": -0.6353590488433838,
"logps/chosen": -266.1250305175781,
"logps/ref_chosen": -268.5232238769531,
"logps/ref_rejected": -237.81137084960938,
"logps/rejected": -242.56707763671875,
"loss": 5.2938,
"margin_dpo/margin_mean": 7.153894901275635,
"margin_dpo/margin_std": 16.709213256835938,
"step": 81
},
{
"KL/chosen_KL_mean": 2.450326919555664,
"KL/mean": -1.5248596668243408,
"KL/rejected_KL_mean": -5.500043869018555,
"KL/std": 12.273836135864258,
"epoch": 0.17172774869109947,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.950379848480225,
"fcm_dpo/q_t": 0.48033344745635986,
"grad_norm": 27.83127784729004,
"learning_rate": 4.92735454356513e-07,
"logits/chosen": -0.719543993473053,
"logits/rejected": -0.7259870171546936,
"logps/chosen": -276.9136047363281,
"logps/ref_chosen": -279.36395263671875,
"logps/ref_rejected": -236.51365661621094,
"logps/rejected": -242.01370239257812,
"loss": 5.2621,
"margin_dpo/margin_mean": 7.950380325317383,
"margin_dpo/margin_std": 15.83857250213623,
"step": 82
},
{
"KL/chosen_KL_mean": 0.28804588317871094,
"KL/mean": -4.03283166885376,
"KL/rejected_KL_mean": -8.35371208190918,
"KL/std": 13.607312202453613,
"epoch": 0.17382198952879582,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.641751289367676,
"fcm_dpo/q_t": 0.47854292392730713,
"grad_norm": 30.7332706451416,
"learning_rate": 4.922908189595017e-07,
"logits/chosen": -0.6931532621383667,
"logits/rejected": -0.6767294406890869,
"logps/chosen": -273.9311828613281,
"logps/ref_chosen": -274.21923828125,
"logps/ref_rejected": -276.2212219238281,
"logps/rejected": -284.5749206542969,
"loss": 5.2434,
"margin_dpo/margin_mean": 8.64175033569336,
"margin_dpo/margin_std": 18.098514556884766,
"step": 83
},
{
"KL/chosen_KL_mean": 0.5773792266845703,
"KL/mean": -3.0747742652893066,
"KL/rejected_KL_mean": -6.726924896240234,
"KL/std": 13.29410171508789,
"epoch": 0.17591623036649215,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.304309368133545,
"fcm_dpo/q_t": 0.4818825423717499,
"grad_norm": 29.943376541137695,
"learning_rate": 4.918331902411841e-07,
"logits/chosen": -0.7306097149848938,
"logits/rejected": -0.7447658181190491,
"logps/chosen": -293.8201904296875,
"logps/ref_chosen": -294.3975524902344,
"logps/ref_rejected": -279.81884765625,
"logps/rejected": -286.5457458496094,
"loss": 5.2894,
"margin_dpo/margin_mean": 7.3043107986450195,
"margin_dpo/margin_std": 16.863800048828125,
"step": 84
},
{
"KL/chosen_KL_mean": -1.4663372039794922,
"KL/mean": -4.343188762664795,
"KL/rejected_KL_mean": -7.220043182373047,
"KL/std": 12.284307479858398,
"epoch": 0.17801047120418848,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.753708362579346,
"fcm_dpo/q_t": 0.4857223331928253,
"grad_norm": 29.357282638549805,
"learning_rate": 4.913625927427995e-07,
"logits/chosen": -0.6696852445602417,
"logits/rejected": -0.6779837608337402,
"logps/chosen": -245.12852478027344,
"logps/ref_chosen": -243.66220092773438,
"logps/ref_rejected": -263.9421691894531,
"logps/rejected": -271.1622314453125,
"loss": 5.345,
"margin_dpo/margin_mean": 5.753708839416504,
"margin_dpo/margin_std": 15.366795539855957,
"step": 85
},
{
"KL/chosen_KL_mean": 1.204885482788086,
"KL/mean": -3.256032705307007,
"KL/rejected_KL_mean": -7.716947555541992,
"KL/std": 13.279273986816406,
"epoch": 0.18010471204188483,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.92184066772461,
"fcm_dpo/q_t": 0.477853924036026,
"grad_norm": 36.05186462402344,
"learning_rate": 4.908790517010636e-07,
"logits/chosen": -0.694229245185852,
"logits/rejected": -0.6882424354553223,
"logps/chosen": -308.2257995605469,
"logps/ref_chosen": -309.4306945800781,
"logps/ref_rejected": -290.91278076171875,
"logps/rejected": -298.6297607421875,
"loss": 5.2279,
"margin_dpo/margin_mean": 8.921839714050293,
"margin_dpo/margin_std": 17.340333938598633,
"step": 86
},
{
"KL/chosen_KL_mean": -0.13620948791503906,
"KL/mean": -5.250948429107666,
"KL/rejected_KL_mean": -10.365686416625977,
"KL/std": 15.311408042907715,
"epoch": 0.18219895287958116,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.22948169708252,
"fcm_dpo/q_t": 0.4747437834739685,
"grad_norm": 29.675899505615234,
"learning_rate": 4.903825930468148e-07,
"logits/chosen": -0.7572658061981201,
"logits/rejected": -0.7517373561859131,
"logps/chosen": -278.1639404296875,
"logps/ref_chosen": -278.0277099609375,
"logps/ref_rejected": -245.70123291015625,
"logps/rejected": -256.06689453125,
"loss": 5.1852,
"margin_dpo/margin_mean": 10.22948169708252,
"margin_dpo/margin_std": 18.946619033813477,
"step": 87
},
{
"KL/chosen_KL_mean": -2.036558151245117,
"KL/mean": -6.464334011077881,
"KL/rejected_KL_mean": -10.892110824584961,
"KL/std": 15.0281400680542,
"epoch": 0.18429319371727748,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.855554580688477,
"fcm_dpo/q_t": 0.4781079888343811,
"grad_norm": 28.795900344848633,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": -0.7639174461364746,
"logits/rejected": -0.7798334956169128,
"logps/chosen": -268.5513610839844,
"logps/ref_chosen": -266.5148010253906,
"logps/ref_rejected": -265.90081787109375,
"logps/rejected": -276.7929382324219,
"loss": 5.2366,
"margin_dpo/margin_mean": 8.855554580688477,
"margin_dpo/margin_std": 19.017532348632812,
"step": 88
},
{
"KL/chosen_KL_mean": 0.0593719482421875,
"KL/mean": -4.611057281494141,
"KL/rejected_KL_mean": -9.281488418579102,
"KL/std": 14.272215843200684,
"epoch": 0.18638743455497384,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.34085750579834,
"fcm_dpo/q_t": 0.4768635332584381,
"grad_norm": 30.523344039916992,
"learning_rate": 4.893510300863676e-07,
"logits/chosen": -0.7424763441085815,
"logits/rejected": -0.7329275608062744,
"logps/chosen": -265.6299133300781,
"logps/ref_chosen": -265.6893005371094,
"logps/ref_rejected": -251.49314880371094,
"logps/rejected": -260.7746276855469,
"loss": 5.2174,
"margin_dpo/margin_mean": 9.340858459472656,
"margin_dpo/margin_std": 18.225364685058594,
"step": 89
},
{
"KL/chosen_KL_mean": -1.1600418090820312,
"KL/mean": -5.250759124755859,
"KL/rejected_KL_mean": -9.341472625732422,
"KL/std": 14.324559211730957,
"epoch": 0.18848167539267016,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.181434631347656,
"fcm_dpo/q_t": 0.47971487045288086,
"grad_norm": 29.79361343383789,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": -0.7358546257019043,
"logits/rejected": -0.7441498637199402,
"logps/chosen": -308.5850524902344,
"logps/ref_chosen": -307.4250183105469,
"logps/ref_rejected": -265.7172546386719,
"logps/rejected": -275.0587158203125,
"loss": 5.2588,
"margin_dpo/margin_mean": 8.181434631347656,
"margin_dpo/margin_std": 17.716218948364258,
"step": 90
},
{
"KL/chosen_KL_mean": -2.122282028198242,
"KL/mean": -6.840453624725342,
"KL/rejected_KL_mean": -11.558622360229492,
"KL/std": 14.71864128112793,
"epoch": 0.1905759162303665,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.436348915100098,
"fcm_dpo/q_t": 0.47672733664512634,
"grad_norm": 32.89813232421875,
"learning_rate": 4.882681251368548e-07,
"logits/chosen": -0.6823728084564209,
"logits/rejected": -0.699794352054596,
"logps/chosen": -237.86328125,
"logps/ref_chosen": -235.74098205566406,
"logps/ref_rejected": -226.6428985595703,
"logps/rejected": -238.20152282714844,
"loss": 5.2185,
"margin_dpo/margin_mean": 9.436347961425781,
"margin_dpo/margin_std": 19.16326141357422,
"step": 91
},
{
"KL/chosen_KL_mean": -2.9682865142822266,
"KL/mean": -8.081915855407715,
"KL/rejected_KL_mean": -13.195533752441406,
"KL/std": 17.40971565246582,
"epoch": 0.19267015706806281,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.227250099182129,
"fcm_dpo/q_t": 0.474727988243103,
"grad_norm": 33.458744049072266,
"learning_rate": 4.877074915775048e-07,
"logits/chosen": -0.7365894317626953,
"logits/rejected": -0.7199798822402954,
"logps/chosen": -286.41583251953125,
"logps/ref_chosen": -283.4475402832031,
"logps/ref_rejected": -273.134033203125,
"logps/rejected": -286.3295593261719,
"loss": 5.194,
"margin_dpo/margin_mean": 10.227250099182129,
"margin_dpo/margin_std": 21.2034912109375,
"step": 92
},
{
"KL/chosen_KL_mean": -2.397554397583008,
"KL/mean": -7.223630905151367,
"KL/rejected_KL_mean": -12.049707412719727,
"KL/std": 15.443262100219727,
"epoch": 0.19476439790575917,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.652153015136719,
"fcm_dpo/q_t": 0.476184606552124,
"grad_norm": 28.543886184692383,
"learning_rate": 4.871341104867864e-07,
"logits/chosen": -0.7294468283653259,
"logits/rejected": -0.7528958916664124,
"logps/chosen": -235.73471069335938,
"logps/ref_chosen": -233.33714294433594,
"logps/ref_rejected": -230.54273986816406,
"logps/rejected": -242.59243774414062,
"loss": 5.2092,
"margin_dpo/margin_mean": 9.652153968811035,
"margin_dpo/margin_std": 19.43905258178711,
"step": 93
},
{
"KL/chosen_KL_mean": -2.360097885131836,
"KL/mean": -7.027770519256592,
"KL/rejected_KL_mean": -11.695440292358398,
"KL/std": 16.456880569458008,
"epoch": 0.1968586387434555,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.33534049987793,
"fcm_dpo/q_t": 0.47690367698669434,
"grad_norm": 31.50308609008789,
"learning_rate": 4.865480126133871e-07,
"logits/chosen": -0.6894798278808594,
"logits/rejected": -0.7111689448356628,
"logps/chosen": -297.01287841796875,
"logps/ref_chosen": -294.6528015136719,
"logps/ref_rejected": -283.657958984375,
"logps/rejected": -295.3534240722656,
"loss": 5.2318,
"margin_dpo/margin_mean": 9.335341453552246,
"margin_dpo/margin_std": 21.897310256958008,
"step": 94
},
{
"KL/chosen_KL_mean": -3.2690467834472656,
"KL/mean": -8.3305082321167,
"KL/rejected_KL_mean": -13.391958236694336,
"KL/std": 18.094928741455078,
"epoch": 0.19895287958115182,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.122918128967285,
"fcm_dpo/q_t": 0.47508278489112854,
"grad_norm": 33.46260452270508,
"learning_rate": 4.859492293879573e-07,
"logits/chosen": -0.7189546227455139,
"logits/rejected": -0.7395079731941223,
"logps/chosen": -314.9387512207031,
"logps/ref_chosen": -311.6697082519531,
"logps/ref_rejected": -262.7471923828125,
"logps/rejected": -276.13916015625,
"loss": 5.2083,
"margin_dpo/margin_mean": 10.122917175292969,
"margin_dpo/margin_std": 22.639379501342773,
"step": 95
},
{
"KL/chosen_KL_mean": -4.606351852416992,
"KL/mean": -9.789222717285156,
"KL/rejected_KL_mean": -14.972097396850586,
"KL/std": 19.352949142456055,
"epoch": 0.20104712041884817,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.365747451782227,
"fcm_dpo/q_t": 0.47450557351112366,
"grad_norm": 33.26982879638672,
"learning_rate": 4.853377929214243e-07,
"logits/chosen": -0.7056884765625,
"logits/rejected": -0.7183771133422852,
"logps/chosen": -287.162353515625,
"logps/ref_chosen": -282.55596923828125,
"logps/ref_rejected": -242.71588134765625,
"logps/rejected": -257.6879577636719,
"loss": 5.1996,
"margin_dpo/margin_mean": 10.365745544433594,
"margin_dpo/margin_std": 23.32305908203125,
"step": 96
},
{
"KL/chosen_KL_mean": -4.363153457641602,
"KL/mean": -10.424278259277344,
"KL/rejected_KL_mean": -16.485395431518555,
"KL/std": 17.310468673706055,
"epoch": 0.2031413612565445,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.122227668762207,
"fcm_dpo/q_t": 0.47013965249061584,
"grad_norm": 32.01287841796875,
"learning_rate": 4.847137360032699e-07,
"logits/chosen": -0.7406127452850342,
"logits/rejected": -0.727538526058197,
"logps/chosen": -307.9409484863281,
"logps/ref_chosen": -303.57781982421875,
"logps/ref_rejected": -264.22491455078125,
"logps/rejected": -280.7103271484375,
"loss": 5.1279,
"margin_dpo/margin_mean": 12.122227668762207,
"margin_dpo/margin_std": 22.35078239440918,
"step": 97
},
{
"KL/chosen_KL_mean": -4.8404998779296875,
"KL/mean": -10.949996948242188,
"KL/rejected_KL_mean": -17.059505462646484,
"KL/std": 18.937652587890625,
"epoch": 0.20523560209424083,
"fcm_dpo/beta": 0.010055022314190865,
"fcm_dpo/delta": 0.013165837153792381,
"fcm_dpo/margin": 12.219008445739746,
"fcm_dpo/q_t": 0.47002190351486206,
"grad_norm": 34.929656982421875,
"learning_rate": 4.84077092099773e-07,
"logits/chosen": -0.7656075954437256,
"logits/rejected": -0.7773485779762268,
"logps/chosen": -291.67083740234375,
"logps/ref_chosen": -286.8303527832031,
"logps/ref_rejected": -278.08331298828125,
"logps/rejected": -295.1427917480469,
"loss": 5.1346,
"margin_dpo/margin_mean": 12.219010353088379,
"margin_dpo/margin_std": 22.74274253845215,
"step": 98
},
{
"KL/chosen_KL_mean": -5.254631042480469,
"KL/mean": -11.531015396118164,
"KL/rejected_KL_mean": -17.80739974975586,
"KL/std": 19.79965591430664,
"epoch": 0.20732984293193718,
"fcm_dpo/beta": 0.010154977440834045,
"fcm_dpo/delta": 0.009678316302597523,
"fcm_dpo/margin": 12.552767753601074,
"fcm_dpo/q_t": 0.4687933325767517,
"grad_norm": 32.95009231567383,
"learning_rate": 4.834278953522137e-07,
"logits/chosen": -0.7380152344703674,
"logits/rejected": -0.7510902285575867,
"logps/chosen": -285.17584228515625,
"logps/ref_chosen": -279.92120361328125,
"logps/ref_rejected": -250.3365478515625,
"logps/rejected": -268.1439514160156,
"loss": 5.1327,
"margin_dpo/margin_mean": 12.552766799926758,
"margin_dpo/margin_std": 27.131689071655273,
"step": 99
},
{
"KL/chosen_KL_mean": -7.756633758544922,
"KL/mean": -14.034221649169922,
"KL/rejected_KL_mean": -20.311803817749023,
"KL/std": 21.95221710205078,
"epoch": 0.2094240837696335,
"fcm_dpo/beta": 0.010277610272169113,
"fcm_dpo/delta": 0.0157307181507349,
"fcm_dpo/margin": 12.555181503295898,
"fcm_dpo/q_t": 0.4685354232788086,
"grad_norm": 36.376800537109375,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": -0.7792176604270935,
"logits/rejected": -0.7921389937400818,
"logps/chosen": -304.5843505859375,
"logps/ref_chosen": -296.8276672363281,
"logps/ref_rejected": -275.56146240234375,
"logps/rejected": -295.8732604980469,
"loss": 5.1137,
"margin_dpo/margin_mean": 12.555181503295898,
"margin_dpo/margin_std": 24.093582153320312,
"step": 100
},
{
"KL/chosen_KL_mean": -5.120674133300781,
"KL/mean": -12.59157943725586,
"KL/rejected_KL_mean": -20.062488555908203,
"KL/std": 22.75184440612793,
"epoch": 0.21151832460732983,
"fcm_dpo/beta": 0.010311111807823181,
"fcm_dpo/delta": -0.0004459689953364432,
"fcm_dpo/margin": 14.941802978515625,
"fcm_dpo/q_t": 0.46235236525535583,
"grad_norm": 33.297786712646484,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": -0.7734586000442505,
"logits/rejected": -0.7872774004936218,
"logps/chosen": -257.8627014160156,
"logps/ref_chosen": -252.74203491210938,
"logps/ref_rejected": -276.4185485839844,
"logps/rejected": -296.4810791015625,
"loss": 5.0315,
"margin_dpo/margin_mean": 14.941804885864258,
"margin_dpo/margin_std": 26.297649383544922,
"step": 101
},
{
"KL/chosen_KL_mean": -5.081005096435547,
"KL/mean": -12.523896217346191,
"KL/rejected_KL_mean": -19.96678352355957,
"KL/std": 19.68603515625,
"epoch": 0.2136125654450262,
"fcm_dpo/beta": 0.01035170815885067,
"fcm_dpo/delta": 0.01568884216248989,
"fcm_dpo/margin": 14.88577938079834,
"fcm_dpo/q_t": 0.4620976448059082,
"grad_norm": 33.22737121582031,
"learning_rate": 4.814053395442932e-07,
"logits/chosen": -0.7597612738609314,
"logits/rejected": -0.7561210989952087,
"logps/chosen": -224.63473510742188,
"logps/ref_chosen": -219.5537109375,
"logps/ref_rejected": -231.90853881835938,
"logps/rejected": -251.8753204345703,
"loss": 5.0247,
"margin_dpo/margin_mean": 14.885781288146973,
"margin_dpo/margin_std": 24.928041458129883,
"step": 102
},
{
"KL/chosen_KL_mean": -4.84321403503418,
"KL/mean": -11.587666511535645,
"KL/rejected_KL_mean": -18.33211898803711,
"KL/std": 21.602670669555664,
"epoch": 0.2157068062827225,
"fcm_dpo/beta": 0.010488416068255901,
"fcm_dpo/delta": 0.01155367773026228,
"fcm_dpo/margin": 13.488908767700195,
"fcm_dpo/q_t": 0.4657100439071655,
"grad_norm": 34.28849411010742,
"learning_rate": 4.807062862684873e-07,
"logits/chosen": -0.7750781774520874,
"logits/rejected": -0.7720986008644104,
"logps/chosen": -264.51824951171875,
"logps/ref_chosen": -259.6750793457031,
"logps/ref_rejected": -278.7400817871094,
"logps/rejected": -297.07220458984375,
"loss": 5.0773,
"margin_dpo/margin_mean": 13.488907814025879,
"margin_dpo/margin_std": 25.666749954223633,
"step": 103
},
{
"KL/chosen_KL_mean": -9.041154861450195,
"KL/mean": -14.049955368041992,
"KL/rejected_KL_mean": -19.05875015258789,
"KL/std": 22.522296905517578,
"epoch": 0.21780104712041884,
"fcm_dpo/beta": 0.010536887682974339,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.017589569091797,
"fcm_dpo/q_t": 0.47418421506881714,
"grad_norm": 34.80829620361328,
"learning_rate": 4.799948609147061e-07,
"logits/chosen": -0.7744643688201904,
"logits/rejected": -0.7815247774124146,
"logps/chosen": -277.01531982421875,
"logps/ref_chosen": -267.9741516113281,
"logps/ref_rejected": -230.5306396484375,
"logps/rejected": -249.58938598632812,
"loss": 5.2141,
"margin_dpo/margin_mean": 10.01759147644043,
"margin_dpo/margin_std": 26.329910278320312,
"step": 104
},
{
"KL/chosen_KL_mean": -5.195383071899414,
"KL/mean": -15.491511344909668,
"KL/rejected_KL_mean": -25.787647247314453,
"KL/std": 25.20000457763672,
"epoch": 0.2198952879581152,
"fcm_dpo/beta": 0.010571734979748726,
"fcm_dpo/delta": 0.00043234386248514056,
"fcm_dpo/margin": 20.592254638671875,
"fcm_dpo/q_t": 0.4467693567276001,
"grad_norm": 36.65842819213867,
"learning_rate": 4.792711016345321e-07,
"logits/chosen": -0.7523764371871948,
"logits/rejected": -0.7615402340888977,
"logps/chosen": -327.4501953125,
"logps/ref_chosen": -322.25482177734375,
"logps/ref_rejected": -279.02978515625,
"logps/rejected": -304.8174133300781,
"loss": 4.8127,
"margin_dpo/margin_mean": 20.592256546020508,
"margin_dpo/margin_std": 27.057451248168945,
"step": 105
},
{
"KL/chosen_KL_mean": -12.456212997436523,
"KL/mean": -18.74678611755371,
"KL/rejected_KL_mean": -25.037351608276367,
"KL/std": 23.551477432250977,
"epoch": 0.22198952879581152,
"fcm_dpo/beta": 0.010547379031777382,
"fcm_dpo/delta": 0.003242878010496497,
"fcm_dpo/margin": 12.581135749816895,
"fcm_dpo/q_t": 0.46765029430389404,
"grad_norm": 39.138153076171875,
"learning_rate": 4.785350472409791e-07,
"logits/chosen": -0.7371970415115356,
"logits/rejected": -0.7723554968833923,
"logps/chosen": -308.614013671875,
"logps/ref_chosen": -296.15777587890625,
"logps/ref_rejected": -266.2691650390625,
"logps/rejected": -291.3065185546875,
"loss": 5.1337,
"margin_dpo/margin_mean": 12.581133842468262,
"margin_dpo/margin_std": 29.417034149169922,
"step": 106
},
{
"KL/chosen_KL_mean": -4.330564498901367,
"KL/mean": -14.176000595092773,
"KL/rejected_KL_mean": -24.021432876586914,
"KL/std": 22.089252471923828,
"epoch": 0.22408376963350785,
"fcm_dpo/beta": 0.01077681127935648,
"fcm_dpo/delta": 0.043501920998096466,
"fcm_dpo/margin": 19.69086265563965,
"fcm_dpo/q_t": 0.4485030770301819,
"grad_norm": 40.70358657836914,
"learning_rate": 4.777867372064105e-07,
"logits/chosen": -0.7831494212150574,
"logits/rejected": -0.7764496207237244,
"logps/chosen": -311.3268737792969,
"logps/ref_chosen": -306.996337890625,
"logps/ref_rejected": -296.79412841796875,
"logps/rejected": -320.8155517578125,
"loss": 4.8351,
"margin_dpo/margin_mean": 19.69086265563965,
"margin_dpo/margin_std": 27.672435760498047,
"step": 107
},
{
"KL/chosen_KL_mean": -5.463014602661133,
"KL/mean": -14.59699535369873,
"KL/rejected_KL_mean": -23.730979919433594,
"KL/std": 25.10581398010254,
"epoch": 0.2261780104712042,
"fcm_dpo/beta": 0.011032754555344582,
"fcm_dpo/delta": 0.01777113415300846,
"fcm_dpo/margin": 18.267986297607422,
"fcm_dpo/q_t": 0.4515891969203949,
"grad_norm": 112.73442077636719,
"learning_rate": 4.770262116604223e-07,
"logits/chosen": -0.7686399817466736,
"logits/rejected": -0.7808905839920044,
"logps/chosen": -300.61566162109375,
"logps/ref_chosen": -295.1526794433594,
"logps/ref_rejected": -235.974853515625,
"logps/rejected": -259.7058410644531,
"loss": 4.9053,
"margin_dpo/margin_mean": 18.267986297607422,
"margin_dpo/margin_std": 30.327621459960938,
"step": 108
},
{
"KL/chosen_KL_mean": -8.268291473388672,
"KL/mean": -18.426597595214844,
"KL/rejected_KL_mean": -28.584897994995117,
"KL/std": 25.283273696899414,
"epoch": 0.22827225130890053,
"fcm_dpo/beta": 0.01130305789411068,
"fcm_dpo/delta": 0.04050566256046295,
"fcm_dpo/margin": 20.31660270690918,
"fcm_dpo/q_t": 0.4453127086162567,
"grad_norm": 42.34718322753906,
"learning_rate": 4.7625351138769166e-07,
"logits/chosen": -0.7919974327087402,
"logits/rejected": -0.7888400554656982,
"logps/chosen": -334.1930847167969,
"logps/ref_chosen": -325.9248046875,
"logps/ref_rejected": -279.15423583984375,
"logps/rejected": -307.7391052246094,
"loss": 4.8109,
"margin_dpo/margin_mean": 20.31660270690918,
"margin_dpo/margin_std": 31.698184967041016,
"step": 109
},
{
"KL/chosen_KL_mean": -8.925228118896484,
"KL/mean": -18.446353912353516,
"KL/rejected_KL_mean": -27.96748161315918,
"KL/std": 27.263511657714844,
"epoch": 0.23036649214659685,
"fcm_dpo/beta": 0.011561272665858269,
"fcm_dpo/delta": 0.012318210676312447,
"fcm_dpo/margin": 19.042247772216797,
"fcm_dpo/q_t": 0.4473344683647156,
"grad_norm": 39.34458923339844,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": -0.8064714670181274,
"logits/rejected": -0.7954632043838501,
"logps/chosen": -283.36444091796875,
"logps/ref_chosen": -274.439208984375,
"logps/ref_rejected": -260.0552062988281,
"logps/rejected": -288.022705078125,
"loss": 4.8673,
"margin_dpo/margin_mean": 19.04224967956543,
"margin_dpo/margin_std": 33.730098724365234,
"step": 110
},
{
"KL/chosen_KL_mean": -9.773136138916016,
"KL/mean": -19.87885093688965,
"KL/rejected_KL_mean": -29.98457908630371,
"KL/std": 27.962778091430664,
"epoch": 0.2324607329842932,
"fcm_dpo/beta": 0.011709115467965603,
"fcm_dpo/delta": 0.020104624330997467,
"fcm_dpo/margin": 20.211442947387695,
"fcm_dpo/q_t": 0.44378963112831116,
"grad_norm": 43.68085861206055,
"learning_rate": 4.7467175306295647e-07,
"logits/chosen": -0.8389703631401062,
"logits/rejected": -0.8186056613922119,
"logps/chosen": -339.00927734375,
"logps/ref_chosen": -329.2361755371094,
"logps/ref_rejected": -287.82830810546875,
"logps/rejected": -317.8128662109375,
"loss": 4.8241,
"margin_dpo/margin_mean": 20.211441040039062,
"margin_dpo/margin_std": 34.38101577758789,
"step": 111
},
{
"KL/chosen_KL_mean": -15.453165054321289,
"KL/mean": -22.01694107055664,
"KL/rejected_KL_mean": -28.580717086791992,
"KL/std": 26.992534637451172,
"epoch": 0.23455497382198953,
"fcm_dpo/beta": 0.011764442548155785,
"fcm_dpo/delta": 0.009524336084723473,
"fcm_dpo/margin": 13.127549171447754,
"fcm_dpo/q_t": 0.4634850323200226,
"grad_norm": 46.13030242919922,
"learning_rate": 4.7386277983585053e-07,
"logits/chosen": -0.7355121970176697,
"logits/rejected": -0.7651410698890686,
"logps/chosen": -272.5125427246094,
"logps/ref_chosen": -257.0593566894531,
"logps/ref_rejected": -272.9595031738281,
"logps/rejected": -301.5401916503906,
"loss": 5.1565,
"margin_dpo/margin_mean": 13.12755012512207,
"margin_dpo/margin_std": 36.32170867919922,
"step": 112
},
{
"KL/chosen_KL_mean": -12.787643432617188,
"KL/mean": -25.045021057128906,
"KL/rejected_KL_mean": -37.302391052246094,
"KL/std": 30.94366455078125,
"epoch": 0.23664921465968586,
"fcm_dpo/beta": 0.011613179929554462,
"fcm_dpo/delta": -0.032145921140909195,
"fcm_dpo/margin": 24.514759063720703,
"fcm_dpo/q_t": 0.4332182705402374,
"grad_norm": 47.04500961303711,
"learning_rate": 4.7304180152725024e-07,
"logits/chosen": -0.8058240413665771,
"logits/rejected": -0.8091129660606384,
"logps/chosen": -298.8293151855469,
"logps/ref_chosen": -286.0416564941406,
"logps/ref_rejected": -270.374267578125,
"logps/rejected": -307.6766357421875,
"loss": 4.6973,
"margin_dpo/margin_mean": 24.514759063720703,
"margin_dpo/margin_std": 39.107757568359375,
"step": 113
},
{
"KL/chosen_KL_mean": -16.7172794342041,
"KL/mean": -23.604433059692383,
"KL/rejected_KL_mean": -30.491592407226562,
"KL/std": 27.874691009521484,
"epoch": 0.2387434554973822,
"fcm_dpo/beta": 0.011623743921518326,
"fcm_dpo/delta": 0.012387579306960106,
"fcm_dpo/margin": 13.774309158325195,
"fcm_dpo/q_t": 0.4614120125770569,
"grad_norm": 45.54480743408203,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": -0.8526460528373718,
"logits/rejected": -0.854054868221283,
"logps/chosen": -276.7257385253906,
"logps/ref_chosen": -260.0084533691406,
"logps/ref_rejected": -246.67190551757812,
"logps/rejected": -277.1634826660156,
"loss": 5.1092,
"margin_dpo/margin_mean": 13.774311065673828,
"margin_dpo/margin_std": 35.25768280029297,
"step": 114
},
{
"KL/chosen_KL_mean": -18.6426944732666,
"KL/mean": -25.6184139251709,
"KL/rejected_KL_mean": -32.594139099121094,
"KL/std": 30.649730682373047,
"epoch": 0.24083769633507854,
"fcm_dpo/beta": 0.01156668271869421,
"fcm_dpo/delta": -0.014433400705456734,
"fcm_dpo/margin": 13.951436042785645,
"fcm_dpo/q_t": 0.462697297334671,
"grad_norm": 50.22829055786133,
"learning_rate": 4.7136400641330245e-07,
"logits/chosen": -0.8506746888160706,
"logits/rejected": -0.8111424446105957,
"logps/chosen": -318.0656433105469,
"logps/ref_chosen": -299.4229736328125,
"logps/ref_rejected": -272.1186828613281,
"logps/rejected": -304.71282958984375,
"loss": 5.1336,
"margin_dpo/margin_mean": 13.951436996459961,
"margin_dpo/margin_std": 38.57262420654297,
"step": 115
},
{
"KL/chosen_KL_mean": -14.825180053710938,
"KL/mean": -23.868736267089844,
"KL/rejected_KL_mean": -32.91230010986328,
"KL/std": 28.654550552368164,
"epoch": 0.24293193717277486,
"fcm_dpo/beta": 0.011589784175157547,
"fcm_dpo/delta": 0.010382231324911118,
"fcm_dpo/margin": 18.087129592895508,
"fcm_dpo/q_t": 0.44975095987319946,
"grad_norm": 43.76200485229492,
"learning_rate": 4.70507279583015e-07,
"logits/chosen": -0.8624769449234009,
"logits/rejected": -0.8245735764503479,
"logps/chosen": -294.089111328125,
"logps/ref_chosen": -279.263916015625,
"logps/ref_rejected": -253.6192169189453,
"logps/rejected": -286.531494140625,
"loss": 4.9196,
"margin_dpo/margin_mean": 18.08713150024414,
"margin_dpo/margin_std": 34.976829528808594,
"step": 116
},
{
"KL/chosen_KL_mean": -18.559932708740234,
"KL/mean": -28.868959426879883,
"KL/rejected_KL_mean": -39.177978515625,
"KL/std": 27.26178741455078,
"epoch": 0.2450261780104712,
"fcm_dpo/beta": 0.011582687497138977,
"fcm_dpo/delta": -0.038109198212623596,
"fcm_dpo/margin": 20.61803436279297,
"fcm_dpo/q_t": 0.44285690784454346,
"grad_norm": 47.86320495605469,
"learning_rate": 4.6963872761652834e-07,
"logits/chosen": -0.8096694350242615,
"logits/rejected": -0.8094600439071655,
"logps/chosen": -277.7848205566406,
"logps/ref_chosen": -259.2248840332031,
"logps/ref_rejected": -229.3042755126953,
"logps/rejected": -268.48223876953125,
"loss": 4.8158,
"margin_dpo/margin_mean": 20.61803436279297,
"margin_dpo/margin_std": 31.60936164855957,
"step": 117
},
{
"KL/chosen_KL_mean": -24.599096298217773,
"KL/mean": -36.10367965698242,
"KL/rejected_KL_mean": -47.60826873779297,
"KL/std": 34.55908203125,
"epoch": 0.24712041884816754,
"fcm_dpo/beta": 0.011433707550168037,
"fcm_dpo/delta": 0.008326723240315914,
"fcm_dpo/margin": 23.00917625427246,
"fcm_dpo/q_t": 0.4388441741466522,
"grad_norm": 50.52167892456055,
"learning_rate": 4.687583970916486e-07,
"logits/chosen": -0.8109974265098572,
"logits/rejected": -0.798851490020752,
"logps/chosen": -291.6697692871094,
"logps/ref_chosen": -267.0707092285156,
"logps/ref_rejected": -272.7322082519531,
"logps/rejected": -320.3404541015625,
"loss": 4.8061,
"margin_dpo/margin_mean": 23.009178161621094,
"margin_dpo/margin_std": 42.8681640625,
"step": 118
},
{
"KL/chosen_KL_mean": -29.91927719116211,
"KL/mean": -38.381263732910156,
"KL/rejected_KL_mean": -46.8432502746582,
"KL/std": 33.76998519897461,
"epoch": 0.24921465968586387,
"fcm_dpo/beta": 0.011522539891302586,
"fcm_dpo/delta": 0.013245711103081703,
"fcm_dpo/margin": 16.92397689819336,
"fcm_dpo/q_t": 0.4548139274120331,
"grad_norm": 51.030372619628906,
"learning_rate": 4.6786633521783005e-07,
"logits/chosen": -0.8936063051223755,
"logits/rejected": -0.8938306570053101,
"logps/chosen": -354.595947265625,
"logps/ref_chosen": -324.6766357421875,
"logps/ref_rejected": -306.0322265625,
"logps/rejected": -352.87548828125,
"loss": 5.0481,
"margin_dpo/margin_mean": 16.923974990844727,
"margin_dpo/margin_std": 41.06504821777344,
"step": 119
},
{
"KL/chosen_KL_mean": -26.801218032836914,
"KL/mean": -36.47585678100586,
"KL/rejected_KL_mean": -46.150489807128906,
"KL/std": 32.804019927978516,
"epoch": 0.2513089005235602,
"fcm_dpo/beta": 0.011345265433192253,
"fcm_dpo/delta": -0.025357699021697044,
"fcm_dpo/margin": 19.34927749633789,
"fcm_dpo/q_t": 0.4487614631652832,
"grad_norm": 46.38136672973633,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": -0.841278076171875,
"logits/rejected": -0.8530943989753723,
"logps/chosen": -342.06298828125,
"logps/ref_chosen": -315.2617492675781,
"logps/ref_rejected": -265.32501220703125,
"logps/rejected": -311.4754943847656,
"loss": 4.9559,
"margin_dpo/margin_mean": 19.349279403686523,
"margin_dpo/margin_std": 40.81332778930664,
"step": 120
},
{
"KL/chosen_KL_mean": -31.404523849487305,
"KL/mean": -38.39643859863281,
"KL/rejected_KL_mean": -45.38835144042969,
"KL/std": 31.411802291870117,
"epoch": 0.2534031413612565,
"fcm_dpo/beta": 0.011385759338736534,
"fcm_dpo/delta": 0.026212375611066818,
"fcm_dpo/margin": 13.983831405639648,
"fcm_dpo/q_t": 0.46326711773872375,
"grad_norm": 62.95341491699219,
"learning_rate": 4.6604720940421207e-07,
"logits/chosen": -0.8449280261993408,
"logits/rejected": -0.8593005537986755,
"logps/chosen": -254.400634765625,
"logps/ref_chosen": -222.99609375,
"logps/ref_rejected": -226.92860412597656,
"logps/rejected": -272.31695556640625,
"loss": 5.144,
"margin_dpo/margin_mean": 13.983829498291016,
"margin_dpo/margin_std": 39.88007354736328,
"step": 121
},
{
"KL/chosen_KL_mean": -31.899211883544922,
"KL/mean": -41.62091064453125,
"KL/rejected_KL_mean": -51.34260177612305,
"KL/std": 36.922428131103516,
"epoch": 0.2554973821989529,
"fcm_dpo/beta": 0.011439606547355652,
"fcm_dpo/delta": -0.004928136244416237,
"fcm_dpo/margin": 19.44339942932129,
"fcm_dpo/q_t": 0.44821009039878845,
"grad_norm": 58.85353469848633,
"learning_rate": 4.651202430186092e-07,
"logits/chosen": -0.9108130931854248,
"logits/rejected": -0.8724952936172485,
"logps/chosen": -307.9255065917969,
"logps/ref_chosen": -276.02630615234375,
"logps/ref_rejected": -277.97418212890625,
"logps/rejected": -329.3167724609375,
"loss": 4.9831,
"margin_dpo/margin_mean": 19.44339942932129,
"margin_dpo/margin_std": 45.87887191772461,
"step": 122
},
{
"KL/chosen_KL_mean": -24.832054138183594,
"KL/mean": -37.42338943481445,
"KL/rejected_KL_mean": -50.01473617553711,
"KL/std": 36.377105712890625,
"epoch": 0.25759162303664923,
"fcm_dpo/beta": 0.011429657228291035,
"fcm_dpo/delta": -0.007541924715042114,
"fcm_dpo/margin": 25.18267059326172,
"fcm_dpo/q_t": 0.4322325587272644,
"grad_norm": 54.087303161621094,
"learning_rate": 4.6418174038722924e-07,
"logits/chosen": -0.8302006125450134,
"logits/rejected": -0.821265697479248,
"logps/chosen": -352.9866943359375,
"logps/ref_chosen": -328.1546325683594,
"logps/ref_rejected": -280.6911315917969,
"logps/rejected": -330.70587158203125,
"loss": 4.7131,
"margin_dpo/margin_mean": 25.18267059326172,
"margin_dpo/margin_std": 41.76557159423828,
"step": 123
},
{
"KL/chosen_KL_mean": -25.639102935791016,
"KL/mean": -36.52279281616211,
"KL/rejected_KL_mean": -47.40650177001953,
"KL/std": 34.944732666015625,
"epoch": 0.25968586387434556,
"fcm_dpo/beta": 0.011149123311042786,
"fcm_dpo/delta": -0.033872563391923904,
"fcm_dpo/margin": 21.76740837097168,
"fcm_dpo/q_t": 0.4419231116771698,
"grad_norm": 55.05583953857422,
"learning_rate": 4.6323175183912023e-07,
"logits/chosen": -0.8557197451591492,
"logits/rejected": -0.8204752802848816,
"logps/chosen": -301.3352966308594,
"logps/ref_chosen": -275.6961975097656,
"logps/ref_rejected": -225.361572265625,
"logps/rejected": -272.76806640625,
"loss": 4.837,
"margin_dpo/margin_mean": 21.767410278320312,
"margin_dpo/margin_std": 38.42668914794922,
"step": 124
},
{
"KL/chosen_KL_mean": -27.239627838134766,
"KL/mean": -37.06490707397461,
"KL/rejected_KL_mean": -46.89017868041992,
"KL/std": 36.27045822143555,
"epoch": 0.2617801047120419,
"fcm_dpo/beta": 0.0110806655138731,
"fcm_dpo/delta": -0.004720758646726608,
"fcm_dpo/margin": 19.650556564331055,
"fcm_dpo/q_t": 0.4501379430294037,
"grad_norm": 55.957820892333984,
"learning_rate": 4.6227032831928483e-07,
"logits/chosen": -0.8200584053993225,
"logits/rejected": -0.7745668888092041,
"logps/chosen": -305.30938720703125,
"logps/ref_chosen": -278.06976318359375,
"logps/ref_rejected": -265.63873291015625,
"logps/rejected": -312.5289001464844,
"loss": 5.0283,
"margin_dpo/margin_mean": 19.650556564331055,
"margin_dpo/margin_std": 49.57725524902344,
"step": 125
},
{
"KL/chosen_KL_mean": -23.706384658813477,
"KL/mean": -34.74204635620117,
"KL/rejected_KL_mean": -45.777713775634766,
"KL/std": 37.79378128051758,
"epoch": 0.2638743455497382,
"fcm_dpo/beta": 0.010993003845214844,
"fcm_dpo/delta": -0.00607214355841279,
"fcm_dpo/margin": 22.071332931518555,
"fcm_dpo/q_t": 0.44283780455589294,
"grad_norm": 53.37355041503906,
"learning_rate": 4.612975213859487e-07,
"logits/chosen": -0.8317501544952393,
"logits/rejected": -0.8419036865234375,
"logps/chosen": -345.10247802734375,
"logps/ref_chosen": -321.3960876464844,
"logps/ref_rejected": -285.37664794921875,
"logps/rejected": -331.15435791015625,
"loss": 4.8702,
"margin_dpo/margin_mean": 22.071331024169922,
"margin_dpo/margin_std": 44.08152770996094,
"step": 126
},
{
"KL/chosen_KL_mean": -22.35991096496582,
"KL/mean": -35.4288330078125,
"KL/rejected_KL_mean": -48.49776077270508,
"KL/std": 39.06257629394531,
"epoch": 0.26596858638743454,
"fcm_dpo/beta": 0.010855698958039284,
"fcm_dpo/delta": -0.05256950855255127,
"fcm_dpo/margin": 26.137863159179688,
"fcm_dpo/q_t": 0.4339562654495239,
"grad_norm": 52.98072814941406,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": -0.9140666127204895,
"logits/rejected": -0.8634395003318787,
"logps/chosen": -328.9186706542969,
"logps/ref_chosen": -306.55877685546875,
"logps/ref_rejected": -274.8651428222656,
"logps/rejected": -323.3628845214844,
"loss": 4.7582,
"margin_dpo/margin_mean": 26.13786506652832,
"margin_dpo/margin_std": 44.73991394042969,
"step": 127
},
{
"KL/chosen_KL_mean": -17.275562286376953,
"KL/mean": -32.32242202758789,
"KL/rejected_KL_mean": -47.3692741394043,
"KL/std": 40.74695587158203,
"epoch": 0.2680628272251309,
"fcm_dpo/beta": 0.010318944230675697,
"fcm_dpo/delta": -0.034354813396930695,
"fcm_dpo/margin": 30.093717575073242,
"fcm_dpo/q_t": 0.427314817905426,
"grad_norm": 48.40740203857422,
"learning_rate": 4.5931796656116837e-07,
"logits/chosen": -0.7917481660842896,
"logits/rejected": -0.7879784107208252,
"logps/chosen": -282.6729431152344,
"logps/ref_chosen": -265.3973693847656,
"logps/ref_rejected": -250.9737548828125,
"logps/rejected": -298.343017578125,
"loss": 4.6545,
"margin_dpo/margin_mean": 30.093721389770508,
"margin_dpo/margin_std": 48.82278060913086,
"step": 128
},
{
"KL/chosen_KL_mean": -18.557172775268555,
"KL/mean": -32.06791305541992,
"KL/rejected_KL_mean": -45.57864761352539,
"KL/std": 38.27027893066406,
"epoch": 0.27015706806282724,
"fcm_dpo/beta": 0.010120774619281292,
"fcm_dpo/delta": -0.0329061821103096,
"fcm_dpo/margin": 27.021467208862305,
"fcm_dpo/q_t": 0.43612736463546753,
"grad_norm": 45.152587890625,
"learning_rate": 4.5831132482724193e-07,
"logits/chosen": -0.8318417072296143,
"logits/rejected": -0.8305982947349548,
"logps/chosen": -321.7156066894531,
"logps/ref_chosen": -303.158447265625,
"logps/ref_rejected": -275.9891052246094,
"logps/rejected": -321.5677490234375,
"loss": 4.7692,
"margin_dpo/margin_mean": 27.021469116210938,
"margin_dpo/margin_std": 47.870628356933594,
"step": 129
},
{
"KL/chosen_KL_mean": -21.258207321166992,
"KL/mean": -34.769718170166016,
"KL/rejected_KL_mean": -48.28123474121094,
"KL/std": 37.819278717041016,
"epoch": 0.27225130890052357,
"fcm_dpo/beta": 0.009886497631669044,
"fcm_dpo/delta": -0.0007312544621527195,
"fcm_dpo/margin": 27.023021697998047,
"fcm_dpo/q_t": 0.4373343586921692,
"grad_norm": 55.255393981933594,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": -0.8017497062683105,
"logits/rejected": -0.8390852808952332,
"logps/chosen": -307.6655578613281,
"logps/ref_chosen": -286.4073486328125,
"logps/ref_rejected": -294.38665771484375,
"logps/rejected": -342.66790771484375,
"loss": 4.7757,
"margin_dpo/margin_mean": 27.02302360534668,
"margin_dpo/margin_std": 48.35560989379883,
"step": 130
},
{
"KL/chosen_KL_mean": -25.688549041748047,
"KL/mean": -36.71128463745117,
"KL/rejected_KL_mean": -47.734012603759766,
"KL/std": 42.36379623413086,
"epoch": 0.2743455497382199,
"fcm_dpo/beta": 0.009975353255867958,
"fcm_dpo/delta": 0.0071073779836297035,
"fcm_dpo/margin": 22.045459747314453,
"fcm_dpo/q_t": 0.44867265224456787,
"grad_norm": 58.347599029541016,
"learning_rate": 4.5626458262912735e-07,
"logits/chosen": -0.8796436190605164,
"logits/rejected": -0.8274049162864685,
"logps/chosen": -337.2536315917969,
"logps/ref_chosen": -311.5650634765625,
"logps/ref_rejected": -291.62432861328125,
"logps/rejected": -339.3583679199219,
"loss": 4.9858,
"margin_dpo/margin_mean": 22.045461654663086,
"margin_dpo/margin_std": 51.318870544433594,
"step": 131
},
{
"KL/chosen_KL_mean": -21.675294876098633,
"KL/mean": -37.08460998535156,
"KL/rejected_KL_mean": -52.493934631347656,
"KL/std": 42.055477142333984,
"epoch": 0.2764397905759162,
"fcm_dpo/beta": 0.009898187592625618,
"fcm_dpo/delta": -0.05365057289600372,
"fcm_dpo/margin": 30.81864356994629,
"fcm_dpo/q_t": 0.4293951690196991,
"grad_norm": 60.90685272216797,
"learning_rate": 4.5522459192551166e-07,
"logits/chosen": -0.8308587074279785,
"logits/rejected": -0.8142504692077637,
"logps/chosen": -291.7571105957031,
"logps/ref_chosen": -270.0818176269531,
"logps/ref_rejected": -284.3084411621094,
"logps/rejected": -336.8023681640625,
"loss": 4.7056,
"margin_dpo/margin_mean": 30.81864356994629,
"margin_dpo/margin_std": 52.920997619628906,
"step": 132
},
{
"KL/chosen_KL_mean": -25.237079620361328,
"KL/mean": -39.2971305847168,
"KL/rejected_KL_mean": -53.35718536376953,
"KL/std": 39.58035659790039,
"epoch": 0.27853403141361255,
"fcm_dpo/beta": 0.00956808403134346,
"fcm_dpo/delta": -0.005704056471586227,
"fcm_dpo/margin": 28.12010383605957,
"fcm_dpo/q_t": 0.43685850501060486,
"grad_norm": 41.762489318847656,
"learning_rate": 4.541735956498554e-07,
"logits/chosen": -0.861563503742218,
"logits/rejected": -0.8600152730941772,
"logps/chosen": -310.85845947265625,
"logps/ref_chosen": -285.6213684082031,
"logps/ref_rejected": -251.19386291503906,
"logps/rejected": -304.551025390625,
"loss": 4.7711,
"margin_dpo/margin_mean": 28.12010383605957,
"margin_dpo/margin_std": 49.382789611816406,
"step": 133
},
{
"KL/chosen_KL_mean": -37.50963592529297,
"KL/mean": -48.0689697265625,
"KL/rejected_KL_mean": -58.62830352783203,
"KL/std": 41.11132049560547,
"epoch": 0.2806282722513089,
"fcm_dpo/beta": 0.009509644471108913,
"fcm_dpo/delta": -0.009044441394507885,
"fcm_dpo/margin": 21.118667602539062,
"fcm_dpo/q_t": 0.45208245515823364,
"grad_norm": 48.476036071777344,
"learning_rate": 4.5311165016389914e-07,
"logits/chosen": -0.8715083599090576,
"logits/rejected": -0.8705750703811646,
"logps/chosen": -356.43048095703125,
"logps/ref_chosen": -318.92083740234375,
"logps/ref_rejected": -293.1894836425781,
"logps/rejected": -351.8177490234375,
"loss": 4.9727,
"margin_dpo/margin_mean": 21.118667602539062,
"margin_dpo/margin_std": 44.15083694458008,
"step": 134
},
{
"KL/chosen_KL_mean": -34.88170623779297,
"KL/mean": -47.295005798339844,
"KL/rejected_KL_mean": -59.70830154418945,
"KL/std": 43.2322998046875,
"epoch": 0.28272251308900526,
"fcm_dpo/beta": 0.009455316700041294,
"fcm_dpo/delta": -0.0223441980779171,
"fcm_dpo/margin": 24.82658576965332,
"fcm_dpo/q_t": 0.44377559423446655,
"grad_norm": 57.29498291015625,
"learning_rate": 4.520388124165564e-07,
"logits/chosen": -0.7520244717597961,
"logits/rejected": -0.7897548079490662,
"logps/chosen": -327.7034606933594,
"logps/ref_chosen": -292.8217468261719,
"logps/ref_rejected": -269.2896728515625,
"logps/rejected": -328.99798583984375,
"loss": 4.8953,
"margin_dpo/margin_mean": 24.826589584350586,
"margin_dpo/margin_std": 48.393245697021484,
"step": 135
},
{
"KL/chosen_KL_mean": -36.76615905761719,
"KL/mean": -50.90818405151367,
"KL/rejected_KL_mean": -65.05021667480469,
"KL/std": 43.206573486328125,
"epoch": 0.2848167539267016,
"fcm_dpo/beta": 0.009310539811849594,
"fcm_dpo/delta": -0.014653807505965233,
"fcm_dpo/margin": 28.2840633392334,
"fcm_dpo/q_t": 0.4387061595916748,
"grad_norm": 64.72450256347656,
"learning_rate": 4.5095513994085974e-07,
"logits/chosen": -0.8104878664016724,
"logits/rejected": -0.8044265508651733,
"logps/chosen": -309.6186828613281,
"logps/ref_chosen": -272.8525390625,
"logps/ref_rejected": -252.68202209472656,
"logps/rejected": -317.73223876953125,
"loss": 4.8238,
"margin_dpo/margin_mean": 28.284059524536133,
"margin_dpo/margin_std": 54.00830841064453,
"step": 136
},
{
"KL/chosen_KL_mean": -41.005741119384766,
"KL/mean": -53.573143005371094,
"KL/rejected_KL_mean": -66.14056396484375,
"KL/std": 45.26162338256836,
"epoch": 0.2869109947643979,
"fcm_dpo/beta": 0.009244083426892757,
"fcm_dpo/delta": -0.0003261703532189131,
"fcm_dpo/margin": 25.134830474853516,
"fcm_dpo/q_t": 0.4460538625717163,
"grad_norm": 55.087921142578125,
"learning_rate": 4.498606908508753e-07,
"logits/chosen": -0.865790843963623,
"logits/rejected": -0.8519459366798401,
"logps/chosen": -341.7579650878906,
"logps/ref_chosen": -300.7522277832031,
"logps/ref_rejected": -286.1935119628906,
"logps/rejected": -352.33404541015625,
"loss": 4.9047,
"margin_dpo/margin_mean": 25.13483428955078,
"margin_dpo/margin_std": 51.73419952392578,
"step": 137
},
{
"KL/chosen_KL_mean": -38.480438232421875,
"KL/mean": -53.19182205200195,
"KL/rejected_KL_mean": -67.90321350097656,
"KL/std": 46.54023361206055,
"epoch": 0.28900523560209423,
"fcm_dpo/beta": 0.009153381921350956,
"fcm_dpo/delta": -0.008525258861482143,
"fcm_dpo/margin": 29.422779083251953,
"fcm_dpo/q_t": 0.43822887539863586,
"grad_norm": 53.85645294189453,
"learning_rate": 4.487555238385862e-07,
"logits/chosen": -0.7926889061927795,
"logits/rejected": -0.7721691727638245,
"logps/chosen": -327.4174499511719,
"logps/ref_chosen": -288.9369812011719,
"logps/ref_rejected": -263.7076416015625,
"logps/rejected": -331.61083984375,
"loss": 4.8456,
"margin_dpo/margin_mean": 29.422779083251953,
"margin_dpo/margin_std": 60.80415725708008,
"step": 138
},
{
"KL/chosen_KL_mean": -35.84889221191406,
"KL/mean": -46.137901306152344,
"KL/rejected_KL_mean": -56.42690658569336,
"KL/std": 41.7497673034668,
"epoch": 0.29109947643979056,
"fcm_dpo/beta": 0.009096279740333557,
"fcm_dpo/delta": -0.008328789845108986,
"fcm_dpo/margin": 20.578014373779297,
"fcm_dpo/q_t": 0.4562779664993286,
"grad_norm": 55.9819450378418,
"learning_rate": 4.476396981707453e-07,
"logits/chosen": -0.8190897703170776,
"logits/rejected": -0.8487043380737305,
"logps/chosen": -305.8932189941406,
"logps/ref_chosen": -270.0443115234375,
"logps/ref_rejected": -267.3226013183594,
"logps/rejected": -323.7494812011719,
"loss": 5.0658,
"margin_dpo/margin_mean": 20.578014373779297,
"margin_dpo/margin_std": 51.444358825683594,
"step": 139
},
{
"KL/chosen_KL_mean": -33.45403289794922,
"KL/mean": -49.221893310546875,
"KL/rejected_KL_mean": -64.98976135253906,
"KL/std": 44.43138885498047,
"epoch": 0.2931937172774869,
"fcm_dpo/beta": 0.008965959772467613,
"fcm_dpo/delta": -0.04622640460729599,
"fcm_dpo/margin": 31.53574562072754,
"fcm_dpo/q_t": 0.4328317940235138,
"grad_norm": 48.354827880859375,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": -0.8959548473358154,
"logits/rejected": -0.861539363861084,
"logps/chosen": -316.40960693359375,
"logps/ref_chosen": -282.9555969238281,
"logps/ref_rejected": -251.17181396484375,
"logps/rejected": -316.1615905761719,
"loss": 4.6938,
"margin_dpo/margin_mean": 31.53574562072754,
"margin_dpo/margin_std": 48.106788635253906,
"step": 140
},
{
"KL/chosen_KL_mean": -32.11634826660156,
"KL/mean": -48.44844055175781,
"KL/rejected_KL_mean": -64.78053283691406,
"KL/std": 45.843807220458984,
"epoch": 0.29528795811518327,
"fcm_dpo/beta": 0.008571082726120949,
"fcm_dpo/delta": -0.03963158652186394,
"fcm_dpo/margin": 32.664180755615234,
"fcm_dpo/q_t": 0.4331643581390381,
"grad_norm": 54.81271743774414,
"learning_rate": 4.453763107901675e-07,
"logits/chosen": -0.8185430765151978,
"logits/rejected": -0.812373697757721,
"logps/chosen": -328.41650390625,
"logps/ref_chosen": -296.3001708984375,
"logps/ref_rejected": -279.8486633300781,
"logps/rejected": -344.6291809082031,
"loss": 4.7447,
"margin_dpo/margin_mean": 32.66417694091797,
"margin_dpo/margin_std": 56.69047546386719,
"step": 141
},
{
"KL/chosen_KL_mean": -33.568763732910156,
"KL/mean": -46.956512451171875,
"KL/rejected_KL_mean": -60.34426498413086,
"KL/std": 46.972068786621094,
"epoch": 0.2973821989528796,
"fcm_dpo/beta": 0.008418058045208454,
"fcm_dpo/delta": -0.010385151952505112,
"fcm_dpo/margin": 26.775487899780273,
"fcm_dpo/q_t": 0.44701626896858215,
"grad_norm": 51.90497970581055,
"learning_rate": 4.4422887045602674e-07,
"logits/chosen": -0.836083173751831,
"logits/rejected": -0.8355280160903931,
"logps/chosen": -334.1346130371094,
"logps/ref_chosen": -300.56585693359375,
"logps/ref_rejected": -231.43316650390625,
"logps/rejected": -291.7774353027344,
"loss": 4.9431,
"margin_dpo/margin_mean": 26.775489807128906,
"margin_dpo/margin_std": 57.83775329589844,
"step": 142
},
{
"KL/chosen_KL_mean": -33.348106384277344,
"KL/mean": -49.812156677246094,
"KL/rejected_KL_mean": -66.27620697021484,
"KL/std": 44.9078483581543,
"epoch": 0.2994764397905759,
"fcm_dpo/beta": 0.008253632113337517,
"fcm_dpo/delta": -0.035776134580373764,
"fcm_dpo/margin": 32.9280891418457,
"fcm_dpo/q_t": 0.4356859028339386,
"grad_norm": 49.289283752441406,
"learning_rate": 4.4307101421701755e-07,
"logits/chosen": -0.8303281664848328,
"logits/rejected": -0.8063556551933289,
"logps/chosen": -330.0804443359375,
"logps/ref_chosen": -296.73236083984375,
"logps/ref_rejected": -266.45257568359375,
"logps/rejected": -332.72882080078125,
"loss": 4.725,
"margin_dpo/margin_mean": 32.9280891418457,
"margin_dpo/margin_std": 52.243892669677734,
"step": 143
},
{
"KL/chosen_KL_mean": -33.928253173828125,
"KL/mean": -47.52806854248047,
"KL/rejected_KL_mean": -61.12789535522461,
"KL/std": 48.23427200317383,
"epoch": 0.30157068062827225,
"fcm_dpo/beta": 0.008003258146345615,
"fcm_dpo/delta": -0.02481032907962799,
"fcm_dpo/margin": 27.19965171813965,
"fcm_dpo/q_t": 0.4491684138774872,
"grad_norm": 45.131038665771484,
"learning_rate": 4.419028041654559e-07,
"logits/chosen": -0.8930144309997559,
"logits/rejected": -0.8774906992912292,
"logps/chosen": -332.772216796875,
"logps/ref_chosen": -298.843994140625,
"logps/ref_rejected": -266.120849609375,
"logps/rejected": -327.24871826171875,
"loss": 4.9486,
"margin_dpo/margin_mean": 27.19965362548828,
"margin_dpo/margin_std": 58.43843078613281,
"step": 144
},
{
"KL/chosen_KL_mean": -30.280269622802734,
"KL/mean": -47.94648742675781,
"KL/rejected_KL_mean": -65.61270141601562,
"KL/std": 44.90657424926758,
"epoch": 0.3036649214659686,
"fcm_dpo/beta": 0.007799885701388121,
"fcm_dpo/delta": -0.04888714477419853,
"fcm_dpo/margin": 35.33243179321289,
"fcm_dpo/q_t": 0.4344967007637024,
"grad_norm": 50.87854766845703,
"learning_rate": 4.4072430294890166e-07,
"logits/chosen": -0.8770700097084045,
"logits/rejected": -0.8764016628265381,
"logps/chosen": -306.03314208984375,
"logps/ref_chosen": -275.7528381347656,
"logps/ref_rejected": -214.74807739257812,
"logps/rejected": -280.36077880859375,
"loss": 4.7152,
"margin_dpo/margin_mean": 35.33243179321289,
"margin_dpo/margin_std": 54.74494171142578,
"step": 145
},
{
"KL/chosen_KL_mean": -40.7142219543457,
"KL/mean": -58.507232666015625,
"KL/rejected_KL_mean": -76.30026245117188,
"KL/std": 50.367061614990234,
"epoch": 0.3057591623036649,
"fcm_dpo/beta": 0.007565884385257959,
"fcm_dpo/delta": -0.025456473231315613,
"fcm_dpo/margin": 35.5860481262207,
"fcm_dpo/q_t": 0.43644025921821594,
"grad_norm": 54.75914764404297,
"learning_rate": 4.395355737667985e-07,
"logits/chosen": -0.8618282675743103,
"logits/rejected": -0.8532693982124329,
"logps/chosen": -317.8124084472656,
"logps/ref_chosen": -277.09820556640625,
"logps/ref_rejected": -265.41046142578125,
"logps/rejected": -341.7107238769531,
"loss": 4.7365,
"margin_dpo/margin_mean": 35.5860481262207,
"margin_dpo/margin_std": 56.042728424072266,
"step": 146
},
{
"KL/chosen_KL_mean": -42.89373779296875,
"KL/mean": -56.810821533203125,
"KL/rejected_KL_mean": -70.7279052734375,
"KL/std": 52.10712432861328,
"epoch": 0.3078534031413613,
"fcm_dpo/beta": 0.007512391544878483,
"fcm_dpo/delta": -0.02998882159590721,
"fcm_dpo/margin": 27.83416748046875,
"fcm_dpo/q_t": 0.4501326084136963,
"grad_norm": 44.88176345825195,
"learning_rate": 4.3833668036708483e-07,
"logits/chosen": -0.8450511693954468,
"logits/rejected": -0.8371117115020752,
"logps/chosen": -334.31231689453125,
"logps/ref_chosen": -291.4185791015625,
"logps/ref_rejected": -253.43051147460938,
"logps/rejected": -324.15838623046875,
"loss": 4.9668,
"margin_dpo/margin_mean": 27.834169387817383,
"margin_dpo/margin_std": 57.25261688232422,
"step": 147
},
{
"KL/chosen_KL_mean": -42.17375564575195,
"KL/mean": -57.056846618652344,
"KL/rejected_KL_mean": -71.93993377685547,
"KL/std": 55.181541442871094,
"epoch": 0.3099476439790576,
"fcm_dpo/beta": 0.007428798824548721,
"fcm_dpo/delta": 0.017847800627350807,
"fcm_dpo/margin": 29.76618003845215,
"fcm_dpo/q_t": 0.4492494761943817,
"grad_norm": 52.519874572753906,
"learning_rate": 4.3712768704277524e-07,
"logits/chosen": -0.9135507345199585,
"logits/rejected": -0.9141757488250732,
"logps/chosen": -278.9222717285156,
"logps/ref_chosen": -236.74850463867188,
"logps/ref_rejected": -231.4674072265625,
"logps/rejected": -303.40728759765625,
"loss": 4.9502,
"margin_dpo/margin_mean": 29.76618003845215,
"margin_dpo/margin_std": 66.35972595214844,
"step": 148
},
{
"KL/chosen_KL_mean": -49.42608642578125,
"KL/mean": -69.45520782470703,
"KL/rejected_KL_mean": -89.48432922363281,
"KL/std": 50.71747970581055,
"epoch": 0.31204188481675393,
"fcm_dpo/beta": 0.007459109649062157,
"fcm_dpo/delta": 0.001031714491546154,
"fcm_dpo/margin": 40.05824661254883,
"fcm_dpo/q_t": 0.429160475730896,
"grad_norm": 54.98072814941406,
"learning_rate": 4.3590865862851263e-07,
"logits/chosen": -0.8939695358276367,
"logits/rejected": -0.8774114847183228,
"logps/chosen": -369.35455322265625,
"logps/ref_chosen": -319.9284973144531,
"logps/ref_rejected": -308.20233154296875,
"logps/rejected": -397.6866760253906,
"loss": 4.6089,
"margin_dpo/margin_mean": 40.05824661254883,
"margin_dpo/margin_std": 55.38935470581055,
"step": 149
},
{
"KL/chosen_KL_mean": -56.57506561279297,
"KL/mean": -71.85128021240234,
"KL/rejected_KL_mean": -87.12750244140625,
"KL/std": 52.93745422363281,
"epoch": 0.31413612565445026,
"fcm_dpo/beta": 0.0074956417083740234,
"fcm_dpo/delta": 0.0003224576357752085,
"fcm_dpo/margin": 30.552452087402344,
"fcm_dpo/q_t": 0.44559797644615173,
"grad_norm": 57.156436920166016,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": -0.860993504524231,
"logits/rejected": -0.8432093262672424,
"logps/chosen": -332.893310546875,
"logps/ref_chosen": -276.3182373046875,
"logps/ref_rejected": -273.02215576171875,
"logps/rejected": -360.1496276855469,
"loss": 4.8725,
"margin_dpo/margin_mean": 30.552452087402344,
"margin_dpo/margin_std": 57.681915283203125,
"step": 150
},
{
"KL/chosen_KL_mean": -54.340576171875,
"KL/mean": -76.96028137207031,
"KL/rejected_KL_mean": -99.57998657226562,
"KL/std": 53.98088455200195,
"epoch": 0.3162303664921466,
"fcm_dpo/beta": 0.007399183232337236,
"fcm_dpo/delta": -0.03600255399942398,
"fcm_dpo/margin": 45.239410400390625,
"fcm_dpo/q_t": 0.42125940322875977,
"grad_norm": 66.83990478515625,
"learning_rate": 4.3344075855595097e-07,
"logits/chosen": -0.8806943297386169,
"logits/rejected": -0.8683636784553528,
"logps/chosen": -351.6534118652344,
"logps/ref_chosen": -297.31280517578125,
"logps/ref_rejected": -266.1003723144531,
"logps/rejected": -365.68035888671875,
"loss": 4.541,
"margin_dpo/margin_mean": 45.239410400390625,
"margin_dpo/margin_std": 62.988826751708984,
"step": 151
},
{
"KL/chosen_KL_mean": -47.37806701660156,
"KL/mean": -67.31233978271484,
"KL/rejected_KL_mean": -87.2466049194336,
"KL/std": 57.862388610839844,
"epoch": 0.3183246073298429,
"fcm_dpo/beta": 0.007059152238070965,
"fcm_dpo/delta": -0.05571059137582779,
"fcm_dpo/margin": 39.8685417175293,
"fcm_dpo/q_t": 0.4344164729118347,
"grad_norm": 50.730186462402344,
"learning_rate": 4.3219201924364323e-07,
"logits/chosen": -0.8954218029975891,
"logits/rejected": -0.8904637098312378,
"logps/chosen": -317.6250915527344,
"logps/ref_chosen": -270.2470397949219,
"logps/ref_rejected": -269.7749328613281,
"logps/rejected": -357.02154541015625,
"loss": 4.7382,
"margin_dpo/margin_mean": 39.86853790283203,
"margin_dpo/margin_std": 67.04737854003906,
"step": 152
},
{
"KL/chosen_KL_mean": -54.68328094482422,
"KL/mean": -82.62890625,
"KL/rejected_KL_mean": -110.57453918457031,
"KL/std": 62.350685119628906,
"epoch": 0.3204188481675393,
"fcm_dpo/beta": 0.006717229261994362,
"fcm_dpo/delta": -0.07843609899282455,
"fcm_dpo/margin": 55.89125442504883,
"fcm_dpo/q_t": 0.41156265139579773,
"grad_norm": 52.00371551513672,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": -0.8805234432220459,
"logits/rejected": -0.8666960597038269,
"logps/chosen": -328.4623107910156,
"logps/ref_chosen": -273.779052734375,
"logps/ref_rejected": -280.9530944824219,
"logps/rejected": -391.5276184082031,
"loss": 4.373,
"margin_dpo/margin_mean": 55.89125442504883,
"margin_dpo/margin_std": 64.85965728759766,
"step": 153
},
{
"KL/chosen_KL_mean": -53.89643859863281,
"KL/mean": -76.14729309082031,
"KL/rejected_KL_mean": -98.39814758300781,
"KL/std": 64.30680084228516,
"epoch": 0.3225130890052356,
"fcm_dpo/beta": 0.00642622634768486,
"fcm_dpo/delta": -0.034452252089977264,
"fcm_dpo/margin": 44.501712799072266,
"fcm_dpo/q_t": 0.43304443359375,
"grad_norm": 47.76982879638672,
"learning_rate": 4.2966529689388064e-07,
"logits/chosen": -0.920985221862793,
"logits/rejected": -0.9008965492248535,
"logps/chosen": -343.7996520996094,
"logps/ref_chosen": -289.9031982421875,
"logps/ref_rejected": -261.5166320800781,
"logps/rejected": -359.9147644042969,
"loss": 4.7259,
"margin_dpo/margin_mean": 44.501712799072266,
"margin_dpo/margin_std": 75.18342590332031,
"step": 154
},
{
"KL/chosen_KL_mean": -59.23719787597656,
"KL/mean": -78.5732650756836,
"KL/rejected_KL_mean": -97.90933227539062,
"KL/std": 61.958526611328125,
"epoch": 0.32460732984293195,
"fcm_dpo/beta": 0.0063635921105742455,
"fcm_dpo/delta": 0.0016274466179311275,
"fcm_dpo/margin": 38.672122955322266,
"fcm_dpo/q_t": 0.4432119131088257,
"grad_norm": 59.706298828125,
"learning_rate": 4.2838744935687716e-07,
"logits/chosen": -0.8491299152374268,
"logits/rejected": -0.8450287580490112,
"logps/chosen": -345.098388671875,
"logps/ref_chosen": -285.8612060546875,
"logps/ref_rejected": -300.1272888183594,
"logps/rejected": -398.03662109375,
"loss": 4.8296,
"margin_dpo/margin_mean": 38.672122955322266,
"margin_dpo/margin_std": 70.07493591308594,
"step": 155
},
{
"KL/chosen_KL_mean": -52.546714782714844,
"KL/mean": -79.25215148925781,
"KL/rejected_KL_mean": -105.95761108398438,
"KL/std": 64.53652954101562,
"epoch": 0.3267015706806283,
"fcm_dpo/beta": 0.006244411692023277,
"fcm_dpo/delta": -0.07311277091503143,
"fcm_dpo/margin": 53.410911560058594,
"fcm_dpo/q_t": 0.42212918400764465,
"grad_norm": 51.060523986816406,
"learning_rate": 4.271000354423425e-07,
"logits/chosen": -0.8737393617630005,
"logits/rejected": -0.8699842095375061,
"logps/chosen": -331.5821533203125,
"logps/ref_chosen": -279.0354919433594,
"logps/ref_rejected": -244.2198486328125,
"logps/rejected": -350.177490234375,
"loss": 4.5922,
"margin_dpo/margin_mean": 53.410911560058594,
"margin_dpo/margin_std": 81.18497467041016,
"step": 156
},
{
"KL/chosen_KL_mean": -54.379493713378906,
"KL/mean": -75.57967376708984,
"KL/rejected_KL_mean": -96.77983856201172,
"KL/std": 63.81241226196289,
"epoch": 0.3287958115183246,
"fcm_dpo/beta": 0.005938378162682056,
"fcm_dpo/delta": -0.021369297057390213,
"fcm_dpo/margin": 42.40034103393555,
"fcm_dpo/q_t": 0.4399886727333069,
"grad_norm": 52.73342514038086,
"learning_rate": 4.258031241903777e-07,
"logits/chosen": -0.9237609505653381,
"logits/rejected": -0.9217565059661865,
"logps/chosen": -325.2098083496094,
"logps/ref_chosen": -270.830322265625,
"logps/ref_rejected": -259.08319091796875,
"logps/rejected": -355.863037109375,
"loss": 4.7865,
"margin_dpo/margin_mean": 42.40034103393555,
"margin_dpo/margin_std": 69.72222900390625,
"step": 157
},
{
"KL/chosen_KL_mean": -52.62913131713867,
"KL/mean": -76.1375732421875,
"KL/rejected_KL_mean": -99.64601135253906,
"KL/std": 64.6611099243164,
"epoch": 0.3308900523560209,
"fcm_dpo/beta": 0.005866985768079758,
"fcm_dpo/delta": -0.02696356363594532,
"fcm_dpo/margin": 47.016876220703125,
"fcm_dpo/q_t": 0.43413034081459045,
"grad_norm": 62.1641731262207,
"learning_rate": 4.2449678515039743e-07,
"logits/chosen": -0.8632103204727173,
"logits/rejected": -0.848564863204956,
"logps/chosen": -342.595458984375,
"logps/ref_chosen": -289.9663391113281,
"logps/ref_rejected": -271.335693359375,
"logps/rejected": -370.9817199707031,
"loss": 4.7019,
"margin_dpo/margin_mean": 47.016876220703125,
"margin_dpo/margin_std": 69.72993469238281,
"step": 158
},
{
"KL/chosen_KL_mean": -62.773353576660156,
"KL/mean": -76.6348876953125,
"KL/rejected_KL_mean": -90.49641418457031,
"KL/std": 60.769474029541016,
"epoch": 0.33298429319371725,
"fcm_dpo/beta": 0.005801479797810316,
"fcm_dpo/delta": 0.011277447454631329,
"fcm_dpo/margin": 27.723072052001953,
"fcm_dpo/q_t": 0.46263986825942993,
"grad_norm": 68.88479614257812,
"learning_rate": 4.2318108837739986e-07,
"logits/chosen": -0.9515292644500732,
"logits/rejected": -0.9028602838516235,
"logps/chosen": -384.1517333984375,
"logps/ref_chosen": -321.37835693359375,
"logps/ref_rejected": -250.45652770996094,
"logps/rejected": -340.95294189453125,
"loss": 5.1459,
"margin_dpo/margin_mean": 27.72307014465332,
"margin_dpo/margin_std": 76.83480072021484,
"step": 159
},
{
"KL/chosen_KL_mean": -53.467838287353516,
"KL/mean": -79.95763397216797,
"KL/rejected_KL_mean": -106.44743347167969,
"KL/std": 64.74884796142578,
"epoch": 0.33507853403141363,
"fcm_dpo/beta": 0.005812094546854496,
"fcm_dpo/delta": -0.008517347276210785,
"fcm_dpo/margin": 52.97959899902344,
"fcm_dpo/q_t": 0.4265155494213104,
"grad_norm": 48.53911209106445,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": -0.8837435245513916,
"logits/rejected": -0.8850059509277344,
"logps/chosen": -329.7513427734375,
"logps/ref_chosen": -276.28350830078125,
"logps/ref_rejected": -262.7477722167969,
"logps/rejected": -369.1951904296875,
"loss": 4.5471,
"margin_dpo/margin_mean": 52.97959899902344,
"margin_dpo/margin_std": 62.66265869140625,
"step": 160
},
{
"KL/chosen_KL_mean": -65.78504943847656,
"KL/mean": -89.10560607910156,
"KL/rejected_KL_mean": -112.4261703491211,
"KL/std": 68.93439483642578,
"epoch": 0.33717277486910996,
"fcm_dpo/beta": 0.005809702444821596,
"fcm_dpo/delta": -0.0074594165198504925,
"fcm_dpo/margin": 46.641117095947266,
"fcm_dpo/q_t": 0.4373480975627899,
"grad_norm": 62.1060905456543,
"learning_rate": 4.2052190435769554e-07,
"logits/chosen": -0.9010162949562073,
"logits/rejected": -0.8825740218162537,
"logps/chosen": -376.27783203125,
"logps/ref_chosen": -310.4927978515625,
"logps/ref_rejected": -250.25347900390625,
"logps/rejected": -362.67962646484375,
"loss": 4.7666,
"margin_dpo/margin_mean": 46.64112091064453,
"margin_dpo/margin_std": 81.07203674316406,
"step": 161
},
{
"KL/chosen_KL_mean": -62.530601501464844,
"KL/mean": -84.44666290283203,
"KL/rejected_KL_mean": -106.36271667480469,
"KL/std": 64.6400375366211,
"epoch": 0.3392670157068063,
"fcm_dpo/beta": 0.005650822538882494,
"fcm_dpo/delta": -0.010540883988142014,
"fcm_dpo/margin": 43.832122802734375,
"fcm_dpo/q_t": 0.44126516580581665,
"grad_norm": 40.41975784301758,
"learning_rate": 4.1917855971495763e-07,
"logits/chosen": -0.8757296204566956,
"logits/rejected": -0.8666384816169739,
"logps/chosen": -358.6410827636719,
"logps/ref_chosen": -296.1105041503906,
"logps/ref_rejected": -253.4247589111328,
"logps/rejected": -359.7874450683594,
"loss": 4.7996,
"margin_dpo/margin_mean": 43.832122802734375,
"margin_dpo/margin_std": 72.63944244384766,
"step": 162
},
{
"KL/chosen_KL_mean": -67.80076599121094,
"KL/mean": -92.6597900390625,
"KL/rejected_KL_mean": -117.51881408691406,
"KL/std": 63.900360107421875,
"epoch": 0.3413612565445026,
"fcm_dpo/beta": 0.005691584199666977,
"fcm_dpo/delta": -0.013408953323960304,
"fcm_dpo/margin": 49.7180290222168,
"fcm_dpo/q_t": 0.43232640624046326,
"grad_norm": 62.017704010009766,
"learning_rate": 4.1782614253949255e-07,
"logits/chosen": -0.9154207110404968,
"logits/rejected": -0.9098785519599915,
"logps/chosen": -361.3006591796875,
"logps/ref_chosen": -293.4999084472656,
"logps/ref_rejected": -266.7116394042969,
"logps/rejected": -384.23046875,
"loss": 4.6703,
"margin_dpo/margin_mean": 49.7180290222168,
"margin_dpo/margin_std": 71.16876983642578,
"step": 163
},
{
"KL/chosen_KL_mean": -80.27838134765625,
"KL/mean": -102.35275268554688,
"KL/rejected_KL_mean": -124.4271469116211,
"KL/std": 70.86463928222656,
"epoch": 0.34345549738219894,
"fcm_dpo/beta": 0.005669012665748596,
"fcm_dpo/delta": -0.007750518620014191,
"fcm_dpo/margin": 44.148780822753906,
"fcm_dpo/q_t": 0.4416029751300812,
"grad_norm": 58.888221740722656,
"learning_rate": 4.164647253573289e-07,
"logits/chosen": -0.8757019639015198,
"logits/rejected": -0.8851001262664795,
"logps/chosen": -347.327880859375,
"logps/ref_chosen": -267.04949951171875,
"logps/ref_rejected": -215.9768829345703,
"logps/rejected": -340.4040222167969,
"loss": 4.8288,
"margin_dpo/margin_mean": 44.148780822753906,
"margin_dpo/margin_std": 81.29022216796875,
"step": 164
},
{
"KL/chosen_KL_mean": -69.51595306396484,
"KL/mean": -85.5526351928711,
"KL/rejected_KL_mean": -101.5893325805664,
"KL/std": 59.26508712768555,
"epoch": 0.34554973821989526,
"fcm_dpo/beta": 0.00558866374194622,
"fcm_dpo/delta": -0.004711986985057592,
"fcm_dpo/margin": 32.07337188720703,
"fcm_dpo/q_t": 0.4569927155971527,
"grad_norm": 62.76590347290039,
"learning_rate": 4.1509438117713863e-07,
"logits/chosen": -0.8961373567581177,
"logits/rejected": -0.8679142594337463,
"logps/chosen": -347.57745361328125,
"logps/ref_chosen": -278.06146240234375,
"logps/ref_rejected": -260.4288635253906,
"logps/rejected": -362.0182189941406,
"loss": 5.0112,
"margin_dpo/margin_mean": 32.07337188720703,
"margin_dpo/margin_std": 66.55110168457031,
"step": 165
},
{
"KL/chosen_KL_mean": -64.54472351074219,
"KL/mean": -85.60026550292969,
"KL/rejected_KL_mean": -106.65580749511719,
"KL/std": 63.61589431762695,
"epoch": 0.34764397905759165,
"fcm_dpo/beta": 0.0055479020811617374,
"fcm_dpo/delta": -0.0022045508958399296,
"fcm_dpo/margin": 42.111087799072266,
"fcm_dpo/q_t": 0.44493845105171204,
"grad_norm": 52.08544921875,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": -0.8519665598869324,
"logits/rejected": -0.8151019215583801,
"logps/chosen": -340.4938049316406,
"logps/ref_chosen": -275.9490661621094,
"logps/ref_rejected": -232.13473510742188,
"logps/rejected": -338.79058837890625,
"loss": 4.8543,
"margin_dpo/margin_mean": 42.111087799072266,
"margin_dpo/margin_std": 77.61107635498047,
"step": 166
},
{
"KL/chosen_KL_mean": -74.52401733398438,
"KL/mean": -98.62837219238281,
"KL/rejected_KL_mean": -122.73274230957031,
"KL/std": 69.09346008300781,
"epoch": 0.34973821989528797,
"fcm_dpo/beta": 0.005625554360449314,
"fcm_dpo/delta": 0.007754523307085037,
"fcm_dpo/margin": 48.208717346191406,
"fcm_dpo/q_t": 0.4354207515716553,
"grad_norm": 49.939430236816406,
"learning_rate": 4.123272062470633e-07,
"logits/chosen": -0.8557583689689636,
"logits/rejected": -0.8407419323921204,
"logps/chosen": -355.075439453125,
"logps/ref_chosen": -280.5514221191406,
"logps/ref_rejected": -255.2896728515625,
"logps/rejected": -378.02239990234375,
"loss": 4.7377,
"margin_dpo/margin_mean": 48.20872116088867,
"margin_dpo/margin_std": 78.89886474609375,
"step": 167
},
{
"KL/chosen_KL_mean": -76.38279724121094,
"KL/mean": -102.09847259521484,
"KL/rejected_KL_mean": -127.81414794921875,
"KL/std": 71.54838562011719,
"epoch": 0.3518324607329843,
"fcm_dpo/beta": 0.005538075231015682,
"fcm_dpo/delta": -0.018495822325348854,
"fcm_dpo/margin": 51.43134689331055,
"fcm_dpo/q_t": 0.4321715533733368,
"grad_norm": 106.25069427490234,
"learning_rate": 4.1093052389237174e-07,
"logits/chosen": -0.8275035619735718,
"logits/rejected": -0.7970058917999268,
"logps/chosen": -392.1810302734375,
"logps/ref_chosen": -315.7982177734375,
"logps/ref_rejected": -291.48406982421875,
"logps/rejected": -419.2982482910156,
"loss": 4.6839,
"margin_dpo/margin_mean": 51.43134689331055,
"margin_dpo/margin_std": 78.42271423339844,
"step": 168
},
{
"KL/chosen_KL_mean": -71.67886352539062,
"KL/mean": -102.86964416503906,
"KL/rejected_KL_mean": -134.06040954589844,
"KL/std": 73.42317962646484,
"epoch": 0.3539267015706806,
"fcm_dpo/beta": 0.00543187977746129,
"fcm_dpo/delta": -0.05743662640452385,
"fcm_dpo/margin": 62.381561279296875,
"fcm_dpo/q_t": 0.41999682784080505,
"grad_norm": 57.55976104736328,
"learning_rate": 4.0952521132208267e-07,
"logits/chosen": -0.823381245136261,
"logits/rejected": -0.8265929818153381,
"logps/chosen": -332.74310302734375,
"logps/ref_chosen": -261.06427001953125,
"logps/ref_rejected": -235.40663146972656,
"logps/rejected": -369.467041015625,
"loss": 4.5049,
"margin_dpo/margin_mean": 62.381561279296875,
"margin_dpo/margin_std": 80.02377319335938,
"step": 169
},
{
"KL/chosen_KL_mean": -71.20687103271484,
"KL/mean": -96.61824798583984,
"KL/rejected_KL_mean": -122.02960205078125,
"KL/std": 77.275146484375,
"epoch": 0.35602094240837695,
"fcm_dpo/beta": 0.005277743563055992,
"fcm_dpo/delta": 0.012714497745037079,
"fcm_dpo/margin": 50.82273864746094,
"fcm_dpo/q_t": 0.4381609857082367,
"grad_norm": 54.236793518066406,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": -0.7995795607566833,
"logits/rejected": -0.7993738055229187,
"logps/chosen": -380.17413330078125,
"logps/ref_chosen": -308.96722412109375,
"logps/ref_rejected": -263.8466796875,
"logps/rejected": -385.87628173828125,
"loss": 4.8003,
"margin_dpo/margin_mean": 50.82273864746094,
"margin_dpo/margin_std": 97.3071517944336,
"step": 170
},
{
"KL/chosen_KL_mean": -58.641273498535156,
"KL/mean": -90.96955871582031,
"KL/rejected_KL_mean": -123.29782104492188,
"KL/std": 73.8992919921875,
"epoch": 0.3581151832460733,
"fcm_dpo/beta": 0.005161117296665907,
"fcm_dpo/delta": -0.0646912157535553,
"fcm_dpo/margin": 64.65653228759766,
"fcm_dpo/q_t": 0.4205283522605896,
"grad_norm": 64.7024154663086,
"learning_rate": 4.0668899744407567e-07,
"logits/chosen": -0.8300844430923462,
"logits/rejected": -0.8436256647109985,
"logps/chosen": -317.5302734375,
"logps/ref_chosen": -258.8890380859375,
"logps/ref_rejected": -262.19140625,
"logps/rejected": -385.4892272949219,
"loss": 4.4977,
"margin_dpo/margin_mean": 64.65653228759766,
"margin_dpo/margin_std": 79.60781860351562,
"step": 171
},
{
"KL/chosen_KL_mean": -66.7177505493164,
"KL/mean": -84.35318756103516,
"KL/rejected_KL_mean": -101.9886474609375,
"KL/std": 65.91461181640625,
"epoch": 0.36020942408376966,
"fcm_dpo/beta": 0.005088986828923225,
"fcm_dpo/delta": 0.008377095684409142,
"fcm_dpo/margin": 35.27089309692383,
"fcm_dpo/q_t": 0.45685499906539917,
"grad_norm": 41.981468200683594,
"learning_rate": 4.0525824823390043e-07,
"logits/chosen": -0.8545408844947815,
"logits/rejected": -0.8687371015548706,
"logps/chosen": -405.74005126953125,
"logps/ref_chosen": -339.0223388671875,
"logps/ref_rejected": -295.78759765625,
"logps/rejected": -397.7762451171875,
"loss": 5.0282,
"margin_dpo/margin_mean": 35.27088928222656,
"margin_dpo/margin_std": 80.5746841430664,
"step": 172
},
{
"KL/chosen_KL_mean": -65.19223022460938,
"KL/mean": -88.45635223388672,
"KL/rejected_KL_mean": -111.72047424316406,
"KL/std": 68.98754119873047,
"epoch": 0.362303664921466,
"fcm_dpo/beta": 0.005017148330807686,
"fcm_dpo/delta": -0.033796440809965134,
"fcm_dpo/margin": 46.52825164794922,
"fcm_dpo/q_t": 0.4449205994606018,
"grad_norm": 60.493263244628906,
"learning_rate": 4.0381917299505686e-07,
"logits/chosen": -0.8733835220336914,
"logits/rejected": -0.8692581057548523,
"logps/chosen": -365.3037109375,
"logps/ref_chosen": -300.1114501953125,
"logps/ref_rejected": -273.78460693359375,
"logps/rejected": -385.505126953125,
"loss": 4.8467,
"margin_dpo/margin_mean": 46.52825164794922,
"margin_dpo/margin_std": 81.14070129394531,
"step": 173
},
{
"KL/chosen_KL_mean": -62.053688049316406,
"KL/mean": -90.67073822021484,
"KL/rejected_KL_mean": -119.28778076171875,
"KL/std": 70.07064819335938,
"epoch": 0.3643979057591623,
"fcm_dpo/beta": 0.004972523078322411,
"fcm_dpo/delta": 0.003636527806520462,
"fcm_dpo/margin": 57.23409652709961,
"fcm_dpo/q_t": 0.431392639875412,
"grad_norm": 71.27803039550781,
"learning_rate": 4.0237184890078243e-07,
"logits/chosen": -0.8437986373901367,
"logits/rejected": -0.8229135870933533,
"logps/chosen": -397.1075744628906,
"logps/ref_chosen": -335.0538635253906,
"logps/ref_rejected": -257.4646911621094,
"logps/rejected": -376.7524719238281,
"loss": 4.6288,
"margin_dpo/margin_mean": 57.234092712402344,
"margin_dpo/margin_std": 74.53538513183594,
"step": 174
},
{
"KL/chosen_KL_mean": -80.63742065429688,
"KL/mean": -106.42805480957031,
"KL/rejected_KL_mean": -132.21868896484375,
"KL/std": 80.53546905517578,
"epoch": 0.36649214659685864,
"fcm_dpo/beta": 0.0049472167156636715,
"fcm_dpo/delta": -0.037538737058639526,
"fcm_dpo/margin": 51.581260681152344,
"fcm_dpo/q_t": 0.44112372398376465,
"grad_norm": 57.902069091796875,
"learning_rate": 4.00916353566676e-07,
"logits/chosen": -0.8539243936538696,
"logits/rejected": -0.8540430665016174,
"logps/chosen": -365.0329895019531,
"logps/ref_chosen": -284.39556884765625,
"logps/ref_rejected": -283.3876647949219,
"logps/rejected": -415.6062927246094,
"loss": 4.8182,
"margin_dpo/margin_mean": 51.58126449584961,
"margin_dpo/margin_std": 90.54949951171875,
"step": 175
},
{
"KL/chosen_KL_mean": -78.06394958496094,
"KL/mean": -98.68791198730469,
"KL/rejected_KL_mean": -119.31187438964844,
"KL/std": 74.86479187011719,
"epoch": 0.36858638743455496,
"fcm_dpo/beta": 0.004791987594217062,
"fcm_dpo/delta": -0.025556650012731552,
"fcm_dpo/margin": 41.2479248046875,
"fcm_dpo/q_t": 0.45347633957862854,
"grad_norm": 61.08745193481445,
"learning_rate": 3.994527650465352e-07,
"logits/chosen": -0.8255434036254883,
"logits/rejected": -0.8293647170066833,
"logps/chosen": -329.8767395019531,
"logps/ref_chosen": -251.81280517578125,
"logps/ref_rejected": -242.05328369140625,
"logps/rejected": -361.36517333984375,
"loss": 5.0155,
"margin_dpo/margin_mean": 41.247928619384766,
"margin_dpo/margin_std": 94.69705200195312,
"step": 176
},
{
"KL/chosen_KL_mean": -86.40694427490234,
"KL/mean": -107.02310180664062,
"KL/rejected_KL_mean": -127.63926696777344,
"KL/std": 72.94503784179688,
"epoch": 0.3706806282722513,
"fcm_dpo/beta": 0.004743529949337244,
"fcm_dpo/delta": -0.0020989251788705587,
"fcm_dpo/margin": 41.23231887817383,
"fcm_dpo/q_t": 0.4538949728012085,
"grad_norm": 60.08957290649414,
"learning_rate": 3.979811618281705e-07,
"logits/chosen": -0.9129557013511658,
"logits/rejected": -0.8802610635757446,
"logps/chosen": -385.0533447265625,
"logps/ref_chosen": -298.6463928222656,
"logps/ref_rejected": -295.66534423828125,
"logps/rejected": -423.3045959472656,
"loss": 5.0155,
"margin_dpo/margin_mean": 41.23231887817383,
"margin_dpo/margin_std": 89.64039611816406,
"step": 177
},
{
"KL/chosen_KL_mean": -74.41709899902344,
"KL/mean": -104.72712707519531,
"KL/rejected_KL_mean": -135.0371551513672,
"KL/std": 80.7784652709961,
"epoch": 0.37277486910994767,
"fcm_dpo/beta": 0.004676537588238716,
"fcm_dpo/delta": -0.01576678641140461,
"fcm_dpo/margin": 60.62005615234375,
"fcm_dpo/q_t": 0.4334092140197754,
"grad_norm": 44.437416076660156,
"learning_rate": 3.9650162282919654e-07,
"logits/chosen": -0.842369019985199,
"logits/rejected": -0.8331227898597717,
"logps/chosen": -360.67474365234375,
"logps/ref_chosen": -286.2576599121094,
"logps/ref_rejected": -243.97491455078125,
"logps/rejected": -379.0120544433594,
"loss": 4.6885,
"margin_dpo/margin_mean": 60.620059967041016,
"margin_dpo/margin_std": 93.60894775390625,
"step": 178
},
{
"KL/chosen_KL_mean": -85.76069641113281,
"KL/mean": -113.523681640625,
"KL/rejected_KL_mean": -141.2866973876953,
"KL/std": 82.93985748291016,
"epoch": 0.374869109947644,
"fcm_dpo/beta": 0.004612220451235771,
"fcm_dpo/delta": -0.01727444678544998,
"fcm_dpo/margin": 55.525997161865234,
"fcm_dpo/q_t": 0.4389217495918274,
"grad_norm": 58.93293380737305,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": -0.8293529748916626,
"logits/rejected": -0.8276381492614746,
"logps/chosen": -345.49774169921875,
"logps/ref_chosen": -259.737060546875,
"logps/ref_rejected": -277.8813171386719,
"logps/rejected": -419.16802978515625,
"loss": 4.7825,
"margin_dpo/margin_mean": 55.525997161865234,
"margin_dpo/margin_std": 91.86540985107422,
"step": 179
},
{
"KL/chosen_KL_mean": -93.309326171875,
"KL/mean": -121.91683959960938,
"KL/rejected_KL_mean": -150.52435302734375,
"KL/std": 79.11772918701172,
"epoch": 0.3769633507853403,
"fcm_dpo/beta": 0.00460518104955554,
"fcm_dpo/delta": 0.0006139341276139021,
"fcm_dpo/margin": 57.21503448486328,
"fcm_dpo/q_t": 0.43886619806289673,
"grad_norm": 78.7430648803711,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": -0.8608454465866089,
"logits/rejected": -0.8818348050117493,
"logps/chosen": -360.61822509765625,
"logps/ref_chosen": -267.30889892578125,
"logps/ref_rejected": -230.4376983642578,
"logps/rejected": -380.9620666503906,
"loss": 4.7406,
"margin_dpo/margin_mean": 57.21503448486328,
"margin_dpo/margin_std": 90.0433578491211,
"step": 180
},
{
"KL/chosen_KL_mean": -109.72969055175781,
"KL/mean": -134.2457275390625,
"KL/rejected_KL_mean": -158.7617645263672,
"KL/std": 81.4984130859375,
"epoch": 0.37905759162303665,
"fcm_dpo/beta": 0.0046082790940999985,
"fcm_dpo/delta": 0.011829550378024578,
"fcm_dpo/margin": 49.0320930480957,
"fcm_dpo/q_t": 0.4466300904750824,
"grad_norm": 73.56534576416016,
"learning_rate": 3.920161866827889e-07,
"logits/chosen": -0.8686573505401611,
"logits/rejected": -0.8654269576072693,
"logps/chosen": -410.2210998535156,
"logps/ref_chosen": -300.49139404296875,
"logps/ref_rejected": -278.98284912109375,
"logps/rejected": -437.74462890625,
"loss": 4.8905,
"margin_dpo/margin_mean": 49.0320930480957,
"margin_dpo/margin_std": 95.10774993896484,
"step": 181
},
{
"KL/chosen_KL_mean": -108.689453125,
"KL/mean": -143.95114135742188,
"KL/rejected_KL_mean": -179.2128143310547,
"KL/std": 81.61646270751953,
"epoch": 0.381151832460733,
"fcm_dpo/beta": 0.0045288666151463985,
"fcm_dpo/delta": -0.06309524923563004,
"fcm_dpo/margin": 70.52334594726562,
"fcm_dpo/q_t": 0.42456480860710144,
"grad_norm": 56.7416877746582,
"learning_rate": 3.90505702185e-07,
"logits/chosen": -0.8651958107948303,
"logits/rejected": -0.8845742344856262,
"logps/chosen": -388.1876220703125,
"logps/ref_chosen": -279.4981689453125,
"logps/ref_rejected": -263.6926574707031,
"logps/rejected": -442.90545654296875,
"loss": 4.5715,
"margin_dpo/margin_mean": 70.52334594726562,
"margin_dpo/margin_std": 96.27135467529297,
"step": 182
},
{
"KL/chosen_KL_mean": -111.27757263183594,
"KL/mean": -141.33160400390625,
"KL/rejected_KL_mean": -171.38565063476562,
"KL/std": 83.98210144042969,
"epoch": 0.3832460732984293,
"fcm_dpo/beta": 0.004440652672201395,
"fcm_dpo/delta": 0.01649186573922634,
"fcm_dpo/margin": 60.108070373535156,
"fcm_dpo/q_t": 0.437813401222229,
"grad_norm": 64.96882629394531,
"learning_rate": 3.889876827928156e-07,
"logits/chosen": -0.9078041911125183,
"logits/rejected": -0.9069268703460693,
"logps/chosen": -382.4832458496094,
"logps/ref_chosen": -271.2057189941406,
"logps/ref_rejected": -243.91549682617188,
"logps/rejected": -415.3011474609375,
"loss": 4.7459,
"margin_dpo/margin_mean": 60.10807418823242,
"margin_dpo/margin_std": 99.40080261230469,
"step": 183
},
{
"KL/chosen_KL_mean": -110.44056701660156,
"KL/mean": -153.08717346191406,
"KL/rejected_KL_mean": -195.73379516601562,
"KL/std": 93.96009063720703,
"epoch": 0.38534031413612563,
"fcm_dpo/beta": 0.004284189082682133,
"fcm_dpo/delta": -0.069338858127594,
"fcm_dpo/margin": 85.2932357788086,
"fcm_dpo/q_t": 0.4131174087524414,
"grad_norm": 68.06148529052734,
"learning_rate": 3.874622099130087e-07,
"logits/chosen": -0.9283655881881714,
"logits/rejected": -0.8976894021034241,
"logps/chosen": -428.8863220214844,
"logps/ref_chosen": -318.4457702636719,
"logps/ref_rejected": -266.640869140625,
"logps/rejected": -462.3746643066406,
"loss": 4.4215,
"margin_dpo/margin_mean": 85.2932357788086,
"margin_dpo/margin_std": 105.1895980834961,
"step": 184
},
{
"KL/chosen_KL_mean": -122.83193969726562,
"KL/mean": -152.273193359375,
"KL/rejected_KL_mean": -181.71441650390625,
"KL/std": 99.07866668701172,
"epoch": 0.387434554973822,
"fcm_dpo/beta": 0.004218521062284708,
"fcm_dpo/delta": 0.006759931333363056,
"fcm_dpo/margin": 58.88247299194336,
"fcm_dpo/q_t": 0.4418259263038635,
"grad_norm": 83.13562774658203,
"learning_rate": 3.859293653520604e-07,
"logits/chosen": -0.9097890853881836,
"logits/rejected": -0.9024062752723694,
"logps/chosen": -397.1407775878906,
"logps/ref_chosen": -274.308837890625,
"logps/ref_rejected": -260.7274169921875,
"logps/rejected": -442.44183349609375,
"loss": 4.8157,
"margin_dpo/margin_mean": 58.88247299194336,
"margin_dpo/margin_std": 104.70620727539062,
"step": 185
},
{
"KL/chosen_KL_mean": -120.67276000976562,
"KL/mean": -149.5914764404297,
"KL/rejected_KL_mean": -178.51016235351562,
"KL/std": 88.66747283935547,
"epoch": 0.38952879581151834,
"fcm_dpo/beta": 0.004291029646992683,
"fcm_dpo/delta": 0.005338459741324186,
"fcm_dpo/margin": 57.8374137878418,
"fcm_dpo/q_t": 0.44115057587623596,
"grad_norm": 60.644100189208984,
"learning_rate": 3.8438923131177237e-07,
"logits/chosen": -0.921852171421051,
"logits/rejected": -0.9182274341583252,
"logps/chosen": -419.6781005859375,
"logps/ref_chosen": -299.00537109375,
"logps/ref_rejected": -274.4014587402344,
"logps/rejected": -452.9115905761719,
"loss": 4.7951,
"margin_dpo/margin_mean": 57.8374137878418,
"margin_dpo/margin_std": 96.82072448730469,
"step": 186
},
{
"KL/chosen_KL_mean": -130.63047790527344,
"KL/mean": -154.88661193847656,
"KL/rejected_KL_mean": -179.14276123046875,
"KL/std": 94.44642639160156,
"epoch": 0.39162303664921466,
"fcm_dpo/beta": 0.0042493510991334915,
"fcm_dpo/delta": -0.010592436417937279,
"fcm_dpo/margin": 48.51228713989258,
"fcm_dpo/q_t": 0.44973719120025635,
"grad_norm": 70.30011749267578,
"learning_rate": 3.828418903848593e-07,
"logits/chosen": -0.8689478039741516,
"logits/rejected": -0.851067066192627,
"logps/chosen": -460.45574951171875,
"logps/ref_chosen": -329.8253173828125,
"logps/ref_rejected": -263.73175048828125,
"logps/rejected": -442.8745422363281,
"loss": 5.0081,
"margin_dpo/margin_mean": 48.51228713989258,
"margin_dpo/margin_std": 113.23208618164062,
"step": 187
},
{
"KL/chosen_KL_mean": -113.98969268798828,
"KL/mean": -141.99594116210938,
"KL/rejected_KL_mean": -170.00218200683594,
"KL/std": 95.48326873779297,
"epoch": 0.393717277486911,
"fcm_dpo/beta": 0.004231072496622801,
"fcm_dpo/delta": -0.0032284194603562355,
"fcm_dpo/margin": 56.012489318847656,
"fcm_dpo/q_t": 0.4451254904270172,
"grad_norm": 73.19304656982422,
"learning_rate": 3.812874255505191e-07,
"logits/chosen": -0.8525975346565247,
"logits/rejected": -0.8392002582550049,
"logps/chosen": -376.9953308105469,
"logps/ref_chosen": -263.005615234375,
"logps/ref_rejected": -247.08668518066406,
"logps/rejected": -417.08880615234375,
"loss": 4.9198,
"margin_dpo/margin_mean": 56.012489318847656,
"margin_dpo/margin_std": 120.3454360961914,
"step": 188
},
{
"KL/chosen_KL_mean": -92.47537231445312,
"KL/mean": -129.1150665283203,
"KL/rejected_KL_mean": -165.75473022460938,
"KL/std": 93.57108306884766,
"epoch": 0.3958115183246073,
"fcm_dpo/beta": 0.004115546587854624,
"fcm_dpo/delta": -0.04511391744017601,
"fcm_dpo/margin": 73.27936553955078,
"fcm_dpo/q_t": 0.4287566840648651,
"grad_norm": 63.68251037597656,
"learning_rate": 3.797259201699833e-07,
"logits/chosen": -0.8904998898506165,
"logits/rejected": -0.8967751860618591,
"logps/chosen": -365.4357604980469,
"logps/ref_chosen": -272.96038818359375,
"logps/ref_rejected": -275.13238525390625,
"logps/rejected": -440.88714599609375,
"loss": 4.6142,
"margin_dpo/margin_mean": 73.27935791015625,
"margin_dpo/margin_std": 102.02257537841797,
"step": 189
},
{
"KL/chosen_KL_mean": -90.63619995117188,
"KL/mean": -123.51648712158203,
"KL/rejected_KL_mean": -156.39678955078125,
"KL/std": 85.31573486328125,
"epoch": 0.39790575916230364,
"fcm_dpo/beta": 0.00409154687076807,
"fcm_dpo/delta": 0.01720324158668518,
"fcm_dpo/margin": 65.7605972290039,
"fcm_dpo/q_t": 0.43600544333457947,
"grad_norm": 63.41102981567383,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": -0.8776698708534241,
"logits/rejected": -0.834459662437439,
"logps/chosen": -348.4337463378906,
"logps/ref_chosen": -257.79754638671875,
"logps/ref_rejected": -225.2164306640625,
"logps/rejected": -381.6131896972656,
"loss": 4.6957,
"margin_dpo/margin_mean": 65.76058959960938,
"margin_dpo/margin_std": 94.38175964355469,
"step": 190
},
{
"KL/chosen_KL_mean": -84.44734954833984,
"KL/mean": -114.27680206298828,
"KL/rejected_KL_mean": -144.1062469482422,
"KL/std": 86.19569396972656,
"epoch": 0.4,
"fcm_dpo/beta": 0.004162714816629887,
"fcm_dpo/delta": 0.016047921031713486,
"fcm_dpo/margin": 59.65888977050781,
"fcm_dpo/q_t": 0.44123804569244385,
"grad_norm": 62.25904083251953,
"learning_rate": 3.765821230985757e-07,
"logits/chosen": -0.9132973551750183,
"logits/rejected": -0.914715588092804,
"logps/chosen": -328.305908203125,
"logps/ref_chosen": -243.8585205078125,
"logps/ref_rejected": -245.12136840820312,
"logps/rejected": -389.22760009765625,
"loss": 4.8031,
"margin_dpo/margin_mean": 59.65888977050781,
"margin_dpo/margin_std": 103.41494750976562,
"step": 191
},
{
"KL/chosen_KL_mean": -92.90057373046875,
"KL/mean": -118.03880310058594,
"KL/rejected_KL_mean": -143.17701721191406,
"KL/std": 79.73211669921875,
"epoch": 0.40209424083769635,
"fcm_dpo/beta": 0.004252881743013859,
"fcm_dpo/delta": 0.03286261856555939,
"fcm_dpo/margin": 50.27644348144531,
"fcm_dpo/q_t": 0.44909244775772095,
"grad_norm": 93.99234008789062,
"learning_rate": 3.75e-07,
"logits/chosen": -0.8387249708175659,
"logits/rejected": -0.8206347227096558,
"logps/chosen": -359.8804931640625,
"logps/ref_chosen": -266.9799499511719,
"logps/ref_rejected": -260.1697082519531,
"logps/rejected": -403.3467102050781,
"loss": 4.8962,
"margin_dpo/margin_mean": 50.27644729614258,
"margin_dpo/margin_std": 90.78395080566406,
"step": 192
},
{
"KL/chosen_KL_mean": -108.05162811279297,
"KL/mean": -137.0741729736328,
"KL/rejected_KL_mean": -166.09671020507812,
"KL/std": 91.33769989013672,
"epoch": 0.4041884816753927,
"fcm_dpo/beta": 0.0042908648028969765,
"fcm_dpo/delta": -0.0064502134919166565,
"fcm_dpo/margin": 58.045074462890625,
"fcm_dpo/q_t": 0.4416932761669159,
"grad_norm": 55.59531021118164,
"learning_rate": 3.734111735307796e-07,
"logits/chosen": -0.8882525563240051,
"logits/rejected": -0.8587220311164856,
"logps/chosen": -388.3048400878906,
"logps/ref_chosen": -280.25323486328125,
"logps/ref_rejected": -291.0348815917969,
"logps/rejected": -457.1316223144531,
"loss": 4.8238,
"margin_dpo/margin_mean": 58.04507827758789,
"margin_dpo/margin_std": 103.87382507324219,
"step": 193
},
{
"KL/chosen_KL_mean": -106.48265075683594,
"KL/mean": -130.47157287597656,
"KL/rejected_KL_mean": -154.46047973632812,
"KL/std": 87.91346740722656,
"epoch": 0.406282722513089,
"fcm_dpo/beta": 0.004275742452591658,
"fcm_dpo/delta": 0.00011287536472082138,
"fcm_dpo/margin": 47.977813720703125,
"fcm_dpo/q_t": 0.45183444023132324,
"grad_norm": 50.88446807861328,
"learning_rate": 3.7181572889485623e-07,
"logits/chosen": -0.8614975214004517,
"logits/rejected": -0.8511518239974976,
"logps/chosen": -394.62213134765625,
"logps/ref_chosen": -288.13946533203125,
"logps/ref_rejected": -251.31529235839844,
"logps/rejected": -405.7757568359375,
"loss": 4.943,
"margin_dpo/margin_mean": 47.977813720703125,
"margin_dpo/margin_std": 95.73290252685547,
"step": 194
},
{
"KL/chosen_KL_mean": -108.96260833740234,
"KL/mean": -132.38340759277344,
"KL/rejected_KL_mean": -155.80416870117188,
"KL/std": 83.44720458984375,
"epoch": 0.4083769633507853,
"fcm_dpo/beta": 0.004298967309296131,
"fcm_dpo/delta": 0.013229867443442345,
"fcm_dpo/margin": 46.84156036376953,
"fcm_dpo/q_t": 0.453215628862381,
"grad_norm": 47.520111083984375,
"learning_rate": 3.7021375165108377e-07,
"logits/chosen": -0.8580233454704285,
"logits/rejected": -0.8627651333808899,
"logps/chosen": -382.9632568359375,
"logps/ref_chosen": -274.0006408691406,
"logps/ref_rejected": -280.22723388671875,
"logps/rejected": -436.0314025878906,
"loss": 4.9577,
"margin_dpo/margin_mean": 46.84156036376953,
"margin_dpo/margin_std": 93.48262023925781,
"step": 195
},
{
"KL/chosen_KL_mean": -93.70986938476562,
"KL/mean": -121.1141357421875,
"KL/rejected_KL_mean": -148.51840209960938,
"KL/std": 85.61984252929688,
"epoch": 0.41047120418848165,
"fcm_dpo/beta": 0.004368783440440893,
"fcm_dpo/delta": 0.010245390236377716,
"fcm_dpo/margin": 54.80853271484375,
"fcm_dpo/q_t": 0.44323647022247314,
"grad_norm": 61.44898223876953,
"learning_rate": 3.6860532770864005e-07,
"logits/chosen": -0.8427191376686096,
"logits/rejected": -0.8498020172119141,
"logps/chosen": -368.6105651855469,
"logps/ref_chosen": -274.90069580078125,
"logps/ref_rejected": -248.7281951904297,
"logps/rejected": -397.2466125488281,
"loss": 4.8345,
"margin_dpo/margin_mean": 54.80853271484375,
"margin_dpo/margin_std": 98.43161010742188,
"step": 196
},
{
"KL/chosen_KL_mean": -102.14750671386719,
"KL/mean": -134.74530029296875,
"KL/rejected_KL_mean": -167.3431396484375,
"KL/std": 87.57936096191406,
"epoch": 0.41256544502617803,
"fcm_dpo/beta": 0.0043055955320596695,
"fcm_dpo/delta": -0.03260016813874245,
"fcm_dpo/margin": 65.19561004638672,
"fcm_dpo/q_t": 0.4334312677383423,
"grad_norm": 73.9769287109375,
"learning_rate": 3.6699054332241985e-07,
"logits/chosen": -0.8774630427360535,
"logits/rejected": -0.8628032207489014,
"logps/chosen": -411.68231201171875,
"logps/ref_chosen": -309.5348205566406,
"logps/ref_rejected": -264.3179931640625,
"logps/rejected": -431.6611022949219,
"loss": 4.6842,
"margin_dpo/margin_mean": 65.19561004638672,
"margin_dpo/margin_std": 94.93258666992188,
"step": 197
},
{
"KL/chosen_KL_mean": -99.85154724121094,
"KL/mean": -134.7466583251953,
"KL/rejected_KL_mean": -169.6417694091797,
"KL/std": 91.86687469482422,
"epoch": 0.41465968586387436,
"fcm_dpo/beta": 0.0042022340930998325,
"fcm_dpo/delta": -0.010694863274693489,
"fcm_dpo/margin": 69.79022979736328,
"fcm_dpo/q_t": 0.4308726191520691,
"grad_norm": 61.50657653808594,
"learning_rate": 3.653694850884091e-07,
"logits/chosen": -0.8791506290435791,
"logits/rejected": -0.8535560369491577,
"logps/chosen": -400.8650207519531,
"logps/ref_chosen": -301.0134582519531,
"logps/ref_rejected": -292.84185791015625,
"logps/rejected": -462.4836120605469,
"loss": 4.6516,
"margin_dpo/margin_mean": 69.79023742675781,
"margin_dpo/margin_std": 103.8303451538086,
"step": 198
},
{
"KL/chosen_KL_mean": -110.65092468261719,
"KL/mean": -140.47079467773438,
"KL/rejected_KL_mean": -170.29063415527344,
"KL/std": 86.93942260742188,
"epoch": 0.4167539267015707,
"fcm_dpo/beta": 0.004240399692207575,
"fcm_dpo/delta": 0.0082355672493577,
"fcm_dpo/margin": 59.63971710205078,
"fcm_dpo/q_t": 0.4405139088630676,
"grad_norm": 103.90625762939453,
"learning_rate": 3.6374223993904124e-07,
"logits/chosen": -0.848025381565094,
"logits/rejected": -0.8029439449310303,
"logps/chosen": -375.2567443847656,
"logps/ref_chosen": -264.6058654785156,
"logps/ref_rejected": -214.9014892578125,
"logps/rejected": -385.1921081542969,
"loss": 4.8064,
"margin_dpo/margin_mean": 59.63971710205078,
"margin_dpo/margin_std": 107.35498046875,
"step": 199
},
{
"KL/chosen_KL_mean": -123.91107940673828,
"KL/mean": -150.3993377685547,
"KL/rejected_KL_mean": -176.88763427734375,
"KL/std": 90.9786376953125,
"epoch": 0.418848167539267,
"fcm_dpo/beta": 0.004204145632684231,
"fcm_dpo/delta": -0.014344342984259129,
"fcm_dpo/margin": 52.97654342651367,
"fcm_dpo/q_t": 0.44768112897872925,
"grad_norm": 70.7446060180664,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": -0.9124985933303833,
"logits/rejected": -0.8872630000114441,
"logps/chosen": -448.0699462890625,
"logps/ref_chosen": -324.1588134765625,
"logps/ref_rejected": -277.80218505859375,
"logps/rejected": -454.6898193359375,
"loss": 4.9611,
"margin_dpo/margin_mean": 52.97654724121094,
"margin_dpo/margin_std": 117.82571411132812,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_KL/chosen_KL_mean": -132.41326904296875,
"eval_KL/mean": -163.7840576171875,
"eval_KL/rejected_KL_mean": -195.15484619140625,
"eval_KL/std": 96.79059600830078,
"eval_fcm_dpo/beta": 0.004193730652332306,
"eval_logits/chosen": -0.8912529349327087,
"eval_logits/rejected": -0.8724632859230042,
"eval_logps/chosen": -420.2400207519531,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -462.0848693847656,
"eval_loss": 0.6005851030349731,
"eval_margin_dpo/margin_mean": 62.7415885925293,
"eval_margin_dpo/margin_std": 107.95939636230469,
"eval_runtime": 78.042,
"eval_samples_per_second": 25.627,
"eval_steps_per_second": 3.203,
"step": 200
},
{
"KL/chosen_KL_mean": -131.6876220703125,
"KL/mean": -163.39187622070312,
"KL/rejected_KL_mean": -195.0961151123047,
"KL/std": 98.21855163574219,
"epoch": 0.42094240837696334,
"fcm_dpo/beta": 0.004204457625746727,
"fcm_dpo/delta": -0.014064101502299309,
"fcm_dpo/margin": 63.40850067138672,
"fcm_dpo/q_t": 0.4373506009578705,
"grad_norm": 70.647216796875,
"learning_rate": 3.604695382782159e-07,
"logits/chosen": -0.8758408427238464,
"logits/rejected": -0.8588773012161255,
"logps/chosen": -403.1833190917969,
"logps/ref_chosen": -271.49566650390625,
"logps/ref_rejected": -245.71414184570312,
"logps/rejected": -440.8102722167969,
"loss": 4.7786,
"margin_dpo/margin_mean": 63.40850830078125,
"margin_dpo/margin_std": 111.03482818603516,
"step": 201
},
{
"KL/chosen_KL_mean": -132.4500732421875,
"KL/mean": -164.19525146484375,
"KL/rejected_KL_mean": -195.94044494628906,
"KL/std": 97.9029769897461,
"epoch": 0.42303664921465967,
"fcm_dpo/beta": 0.004103401210159063,
"fcm_dpo/delta": -0.00629676878452301,
"fcm_dpo/margin": 63.49036407470703,
"fcm_dpo/q_t": 0.4392518997192383,
"grad_norm": 82.53160858154297,
"learning_rate": 3.588242572718162e-07,
"logits/chosen": -0.8929991722106934,
"logits/rejected": -0.8781752586364746,
"logps/chosen": -404.5479736328125,
"logps/ref_chosen": -272.0979309082031,
"logps/ref_rejected": -235.94805908203125,
"logps/rejected": -431.88848876953125,
"loss": 4.8147,
"margin_dpo/margin_mean": 63.49037170410156,
"margin_dpo/margin_std": 119.6777114868164,
"step": 202
},
{
"KL/chosen_KL_mean": -143.78916931152344,
"KL/mean": -167.3878173828125,
"KL/rejected_KL_mean": -190.98648071289062,
"KL/std": 96.77014923095703,
"epoch": 0.42513089005235605,
"fcm_dpo/beta": 0.00414351187646389,
"fcm_dpo/delta": 0.007255503907799721,
"fcm_dpo/margin": 47.19731521606445,
"fcm_dpo/q_t": 0.4533665180206299,
"grad_norm": 82.75169372558594,
"learning_rate": 3.571731403507635e-07,
"logits/chosen": -0.8697388172149658,
"logits/rejected": -0.8752584457397461,
"logps/chosen": -424.01129150390625,
"logps/ref_chosen": -280.2221374511719,
"logps/ref_rejected": -251.79798889160156,
"logps/rejected": -442.78448486328125,
"loss": 4.9882,
"margin_dpo/margin_mean": 47.19731521606445,
"margin_dpo/margin_std": 101.90586853027344,
"step": 203
},
{
"KL/chosen_KL_mean": -135.544921875,
"KL/mean": -169.3643341064453,
"KL/rejected_KL_mean": -203.18374633789062,
"KL/std": 96.953369140625,
"epoch": 0.4272251308900524,
"fcm_dpo/beta": 0.0041145868599414825,
"fcm_dpo/delta": -0.010046249255537987,
"fcm_dpo/margin": 67.6387939453125,
"fcm_dpo/q_t": 0.4341426193714142,
"grad_norm": 76.59571838378906,
"learning_rate": 3.5551627605944746e-07,
"logits/chosen": -0.914879560470581,
"logits/rejected": -0.8851953744888306,
"logps/chosen": -454.3409729003906,
"logps/ref_chosen": -318.7960510253906,
"logps/ref_rejected": -269.69921875,
"logps/rejected": -472.8829650878906,
"loss": 4.7134,
"margin_dpo/margin_mean": 67.6387939453125,
"margin_dpo/margin_std": 109.43064880371094,
"step": 204
},
{
"KL/chosen_KL_mean": -131.20462036132812,
"KL/mean": -172.88485717773438,
"KL/rejected_KL_mean": -214.56509399414062,
"KL/std": 105.96263122558594,
"epoch": 0.4293193717277487,
"fcm_dpo/beta": 0.004047570750117302,
"fcm_dpo/delta": -0.03919146582484245,
"fcm_dpo/margin": 83.3604736328125,
"fcm_dpo/q_t": 0.4227820038795471,
"grad_norm": 58.605960845947266,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": -0.8598192930221558,
"logits/rejected": -0.8276709914207458,
"logps/chosen": -414.9666748046875,
"logps/ref_chosen": -283.7620544433594,
"logps/ref_rejected": -297.69439697265625,
"logps/rejected": -512.259521484375,
"loss": 4.5619,
"margin_dpo/margin_mean": 83.3604736328125,
"margin_dpo/margin_std": 123.52423095703125,
"step": 205
},
{
"KL/chosen_KL_mean": -140.71893310546875,
"KL/mean": -172.0816192626953,
"KL/rejected_KL_mean": -203.44432067871094,
"KL/std": 101.86876678466797,
"epoch": 0.431413612565445,
"fcm_dpo/beta": 0.003931830637156963,
"fcm_dpo/delta": -0.03239187225699425,
"fcm_dpo/margin": 62.725379943847656,
"fcm_dpo/q_t": 0.44201433658599854,
"grad_norm": 59.901485443115234,
"learning_rate": 3.5218566107988867e-07,
"logits/chosen": -0.8909974694252014,
"logits/rejected": -0.9125638008117676,
"logps/chosen": -434.38275146484375,
"logps/ref_chosen": -293.66387939453125,
"logps/ref_rejected": -291.3056640625,
"logps/rejected": -494.75,
"loss": 4.8594,
"margin_dpo/margin_mean": 62.725379943847656,
"margin_dpo/margin_std": 120.58600616455078,
"step": 206
},
{
"KL/chosen_KL_mean": -133.72694396972656,
"KL/mean": -165.00857543945312,
"KL/rejected_KL_mean": -196.29022216796875,
"KL/std": 113.79143524169922,
"epoch": 0.43350785340314135,
"fcm_dpo/beta": 0.0038526002317667007,
"fcm_dpo/delta": -0.010718029923737049,
"fcm_dpo/margin": 62.56328201293945,
"fcm_dpo/q_t": 0.44273653626441956,
"grad_norm": 85.31092834472656,
"learning_rate": 3.505120890024195e-07,
"logits/chosen": -0.8365156650543213,
"logits/rejected": -0.8453404307365417,
"logps/chosen": -404.26202392578125,
"logps/ref_chosen": -270.5350646972656,
"logps/ref_rejected": -278.7747497558594,
"logps/rejected": -475.0649719238281,
"loss": 4.9322,
"margin_dpo/margin_mean": 62.56327438354492,
"margin_dpo/margin_std": 138.38185119628906,
"step": 207
},
{
"KL/chosen_KL_mean": -134.51309204101562,
"KL/mean": -175.3253173828125,
"KL/rejected_KL_mean": -216.13755798339844,
"KL/std": 110.93104553222656,
"epoch": 0.4356020942408377,
"fcm_dpo/beta": 0.0038300526794046164,
"fcm_dpo/delta": -0.026927048340439796,
"fcm_dpo/margin": 81.62447357177734,
"fcm_dpo/q_t": 0.4281120300292969,
"grad_norm": 68.95054626464844,
"learning_rate": 3.4883312676665534e-07,
"logits/chosen": -0.8949654698371887,
"logits/rejected": -0.846905529499054,
"logps/chosen": -414.0958557128906,
"logps/ref_chosen": -279.582763671875,
"logps/ref_rejected": -290.041015625,
"logps/rejected": -506.1785888671875,
"loss": 4.6102,
"margin_dpo/margin_mean": 81.62447357177734,
"margin_dpo/margin_std": 121.49321746826172,
"step": 208
},
{
"KL/chosen_KL_mean": -149.1902313232422,
"KL/mean": -173.98388671875,
"KL/rejected_KL_mean": -198.7775115966797,
"KL/std": 104.27743530273438,
"epoch": 0.437696335078534,
"fcm_dpo/beta": 0.0037748278118669987,
"fcm_dpo/delta": 0.0028195905033499002,
"fcm_dpo/margin": 49.58729553222656,
"fcm_dpo/q_t": 0.4552307724952698,
"grad_norm": 62.14313888549805,
"learning_rate": 3.4714886441024573e-07,
"logits/chosen": -0.8140920996665955,
"logits/rejected": -0.8075211644172668,
"logps/chosen": -468.0627136230469,
"logps/ref_chosen": -318.8725280761719,
"logps/ref_rejected": -270.64324951171875,
"logps/rejected": -469.4207763671875,
"loss": 5.0178,
"margin_dpo/margin_mean": 49.58729553222656,
"margin_dpo/margin_std": 111.15589904785156,
"step": 209
},
{
"KL/chosen_KL_mean": -133.66421508789062,
"KL/mean": -168.675048828125,
"KL/rejected_KL_mean": -203.68588256835938,
"KL/std": 101.76795959472656,
"epoch": 0.4397905759162304,
"fcm_dpo/beta": 0.0037062368355691433,
"fcm_dpo/delta": -0.02896423637866974,
"fcm_dpo/margin": 70.02168273925781,
"fcm_dpo/q_t": 0.43855100870132446,
"grad_norm": 68.48916625976562,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": -0.840144693851471,
"logits/rejected": -0.8310486078262329,
"logps/chosen": -416.80450439453125,
"logps/ref_chosen": -283.14031982421875,
"logps/ref_rejected": -287.2986755371094,
"logps/rejected": -490.98455810546875,
"loss": 4.7707,
"margin_dpo/margin_mean": 70.02169036865234,
"margin_dpo/margin_std": 117.46251678466797,
"step": 210
},
{
"KL/chosen_KL_mean": -119.03889465332031,
"KL/mean": -163.0735626220703,
"KL/rejected_KL_mean": -207.1082000732422,
"KL/std": 99.52161407470703,
"epoch": 0.4418848167539267,
"fcm_dpo/beta": 0.0035809765104204416,
"fcm_dpo/delta": -0.06169634312391281,
"fcm_dpo/margin": 88.06932067871094,
"fcm_dpo/q_t": 0.4240798354148865,
"grad_norm": 62.72855758666992,
"learning_rate": 3.4376480090239047e-07,
"logits/chosen": -0.865166187286377,
"logits/rejected": -0.8419003486633301,
"logps/chosen": -395.4617614746094,
"logps/ref_chosen": -276.4228515625,
"logps/ref_rejected": -252.40603637695312,
"logps/rejected": -459.5142517089844,
"loss": 4.5323,
"margin_dpo/margin_mean": 88.0693130493164,
"margin_dpo/margin_std": 102.9296646118164,
"step": 211
},
{
"KL/chosen_KL_mean": -138.11749267578125,
"KL/mean": -164.1044158935547,
"KL/rejected_KL_mean": -190.09130859375,
"KL/std": 96.616943359375,
"epoch": 0.44397905759162304,
"fcm_dpo/beta": 0.003526175394654274,
"fcm_dpo/delta": 0.015544392168521881,
"fcm_dpo/margin": 51.97382354736328,
"fcm_dpo/q_t": 0.45574235916137695,
"grad_norm": 79.6462173461914,
"learning_rate": 3.4206518122800055e-07,
"logits/chosen": -0.8534685373306274,
"logits/rejected": -0.8499578237533569,
"logps/chosen": -409.82305908203125,
"logps/ref_chosen": -271.7055358886719,
"logps/ref_rejected": -241.18511962890625,
"logps/rejected": -431.2764587402344,
"loss": 5.0072,
"margin_dpo/margin_mean": 51.97381591796875,
"margin_dpo/margin_std": 109.19019317626953,
"step": 212
},
{
"KL/chosen_KL_mean": -145.60855102539062,
"KL/mean": -175.45535278320312,
"KL/rejected_KL_mean": -205.3021697998047,
"KL/std": 116.7935562133789,
"epoch": 0.44607329842931936,
"fcm_dpo/beta": 0.003567378269508481,
"fcm_dpo/delta": 0.0018861833959817886,
"fcm_dpo/margin": 59.69361877441406,
"fcm_dpo/q_t": 0.45073333382606506,
"grad_norm": 71.9309310913086,
"learning_rate": 3.403606243773448e-07,
"logits/chosen": -0.8502581119537354,
"logits/rejected": -0.8634451031684875,
"logps/chosen": -447.90618896484375,
"logps/ref_chosen": -302.2976379394531,
"logps/ref_rejected": -303.6202087402344,
"logps/rejected": -508.9223327636719,
"loss": 4.9608,
"margin_dpo/margin_mean": 59.69361877441406,
"margin_dpo/margin_std": 130.421142578125,
"step": 213
},
{
"KL/chosen_KL_mean": -159.0684814453125,
"KL/mean": -196.1217041015625,
"KL/rejected_KL_mean": -233.17489624023438,
"KL/std": 104.86323547363281,
"epoch": 0.4481675392670157,
"fcm_dpo/beta": 0.0035179981496185064,
"fcm_dpo/delta": -0.0281633660197258,
"fcm_dpo/margin": 74.10640716552734,
"fcm_dpo/q_t": 0.4384738802909851,
"grad_norm": 71.70868682861328,
"learning_rate": 3.3865122176063385e-07,
"logits/chosen": -0.8574197292327881,
"logits/rejected": -0.8577027916908264,
"logps/chosen": -431.20111083984375,
"logps/ref_chosen": -272.13262939453125,
"logps/ref_rejected": -294.82354736328125,
"logps/rejected": -527.9984130859375,
"loss": 4.7398,
"margin_dpo/margin_mean": 74.10640716552734,
"margin_dpo/margin_std": 111.84681701660156,
"step": 214
},
{
"KL/chosen_KL_mean": -164.32371520996094,
"KL/mean": -187.74632263183594,
"KL/rejected_KL_mean": -211.16893005371094,
"KL/std": 117.98490142822266,
"epoch": 0.450261780104712,
"fcm_dpo/beta": 0.0034630722366273403,
"fcm_dpo/delta": -0.017829960212111473,
"fcm_dpo/margin": 46.845191955566406,
"fcm_dpo/q_t": 0.46209806203842163,
"grad_norm": 78.56700897216797,
"learning_rate": 3.3693706504794243e-07,
"logits/chosen": -0.8971854448318481,
"logits/rejected": -0.8750616908073425,
"logps/chosen": -455.70208740234375,
"logps/ref_chosen": -291.3782958984375,
"logps/ref_rejected": -261.05792236328125,
"logps/rejected": -472.22686767578125,
"loss": 5.1106,
"margin_dpo/margin_mean": 46.84518814086914,
"margin_dpo/margin_std": 120.64424133300781,
"step": 215
},
{
"KL/chosen_KL_mean": -151.2208251953125,
"KL/mean": -186.5756378173828,
"KL/rejected_KL_mean": -221.93045043945312,
"KL/std": 109.45191955566406,
"epoch": 0.4523560209424084,
"fcm_dpo/beta": 0.00344485929235816,
"fcm_dpo/delta": -0.002506938762962818,
"fcm_dpo/margin": 70.70963287353516,
"fcm_dpo/q_t": 0.44286611676216125,
"grad_norm": 60.88117980957031,
"learning_rate": 3.3521824616429284e-07,
"logits/chosen": -0.9232491254806519,
"logits/rejected": -0.9159709215164185,
"logps/chosen": -489.72625732421875,
"logps/ref_chosen": -338.50543212890625,
"logps/ref_rejected": -305.76104736328125,
"logps/rejected": -527.6914672851562,
"loss": 4.8403,
"margin_dpo/margin_mean": 70.70964050292969,
"margin_dpo/margin_std": 130.25973510742188,
"step": 216
},
{
"KL/chosen_KL_mean": -157.36825561523438,
"KL/mean": -200.84681701660156,
"KL/rejected_KL_mean": -244.32540893554688,
"KL/std": 115.38238525390625,
"epoch": 0.4544502617801047,
"fcm_dpo/beta": 0.0033965944312512875,
"fcm_dpo/delta": -0.01266030129045248,
"fcm_dpo/margin": 86.9571533203125,
"fcm_dpo/q_t": 0.43058374524116516,
"grad_norm": 45.89756393432617,
"learning_rate": 3.334948572847253e-07,
"logits/chosen": -0.8301356434822083,
"logits/rejected": -0.7868068814277649,
"logps/chosen": -450.91802978515625,
"logps/ref_chosen": -293.5498046875,
"logps/ref_rejected": -256.7830810546875,
"logps/rejected": -501.10845947265625,
"loss": 4.6518,
"margin_dpo/margin_mean": 86.95714569091797,
"margin_dpo/margin_std": 131.319091796875,
"step": 217
},
{
"KL/chosen_KL_mean": -157.33384704589844,
"KL/mean": -196.21994018554688,
"KL/rejected_KL_mean": -235.10601806640625,
"KL/std": 113.00043487548828,
"epoch": 0.45654450261780105,
"fcm_dpo/beta": 0.0033359171357005835,
"fcm_dpo/delta": -0.02211921662092209,
"fcm_dpo/margin": 77.77216339111328,
"fcm_dpo/q_t": 0.43903183937072754,
"grad_norm": 53.51487350463867,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": -0.8625221252441406,
"logits/rejected": -0.8787875175476074,
"logps/chosen": -477.9132080078125,
"logps/ref_chosen": -320.579345703125,
"logps/ref_rejected": -294.0381164550781,
"logps/rejected": -529.1441650390625,
"loss": 4.7594,
"margin_dpo/margin_mean": 77.77216339111328,
"margin_dpo/margin_std": 127.42372131347656,
"step": 218
},
{
"KL/chosen_KL_mean": -159.63987731933594,
"KL/mean": -200.8522491455078,
"KL/rejected_KL_mean": -242.0646209716797,
"KL/std": 109.93661499023438,
"epoch": 0.4586387434554974,
"fcm_dpo/beta": 0.003269998822361231,
"fcm_dpo/delta": -0.035739410668611526,
"fcm_dpo/margin": 82.42474365234375,
"fcm_dpo/q_t": 0.43642672896385193,
"grad_norm": 61.093894958496094,
"learning_rate": 3.300347394584172e-07,
"logits/chosen": -0.8373451828956604,
"logits/rejected": -0.8628327250480652,
"logps/chosen": -428.0585021972656,
"logps/ref_chosen": -268.4186096191406,
"logps/ref_rejected": -265.7808837890625,
"logps/rejected": -507.8454895019531,
"loss": 4.7514,
"margin_dpo/margin_mean": 82.42474365234375,
"margin_dpo/margin_std": 134.4219512939453,
"step": 219
},
{
"KL/chosen_KL_mean": -158.48060607910156,
"KL/mean": -197.44924926757812,
"KL/rejected_KL_mean": -236.41790771484375,
"KL/std": 106.5591049194336,
"epoch": 0.4607329842931937,
"fcm_dpo/beta": 0.003238196484744549,
"fcm_dpo/delta": 0.010954329743981361,
"fcm_dpo/margin": 77.93732452392578,
"fcm_dpo/q_t": 0.439789742231369,
"grad_norm": 55.85917663574219,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": -0.9119465947151184,
"logits/rejected": -0.8783890008926392,
"logps/chosen": -471.3670654296875,
"logps/ref_chosen": -312.8864440917969,
"logps/ref_rejected": -259.5191955566406,
"logps/rejected": -495.9371337890625,
"loss": 4.7505,
"margin_dpo/margin_mean": 77.93733215332031,
"margin_dpo/margin_std": 118.18209838867188,
"step": 220
},
{
"KL/chosen_KL_mean": -172.79757690429688,
"KL/mean": -207.20826721191406,
"KL/rejected_KL_mean": -241.6189422607422,
"KL/std": 113.18028259277344,
"epoch": 0.46282722513089003,
"fcm_dpo/beta": 0.0032698106952011585,
"fcm_dpo/delta": -0.015935488045215607,
"fcm_dpo/margin": 68.82136535644531,
"fcm_dpo/q_t": 0.447135865688324,
"grad_norm": 58.454261779785156,
"learning_rate": 3.265574537815398e-07,
"logits/chosen": -0.8103448152542114,
"logits/rejected": -0.8210662603378296,
"logps/chosen": -473.1233825683594,
"logps/ref_chosen": -300.32586669921875,
"logps/ref_rejected": -286.312255859375,
"logps/rejected": -527.9312133789062,
"loss": 4.9031,
"margin_dpo/margin_mean": 68.82136535644531,
"margin_dpo/margin_std": 134.83969116210938,
"step": 221
},
{
"KL/chosen_KL_mean": -169.22169494628906,
"KL/mean": -202.10203552246094,
"KL/rejected_KL_mean": -234.98236083984375,
"KL/std": 103.63680267333984,
"epoch": 0.4649214659685864,
"fcm_dpo/beta": 0.0032462161034345627,
"fcm_dpo/delta": 0.005263763479888439,
"fcm_dpo/margin": 65.76068115234375,
"fcm_dpo/q_t": 0.448941707611084,
"grad_norm": 52.64696502685547,
"learning_rate": 3.248126059518784e-07,
"logits/chosen": -0.9082949757575989,
"logits/rejected": -0.8724523186683655,
"logps/chosen": -466.3330383300781,
"logps/ref_chosen": -297.1113586425781,
"logps/ref_rejected": -235.53146362304688,
"logps/rejected": -470.5138244628906,
"loss": 4.894,
"margin_dpo/margin_mean": 65.76068115234375,
"margin_dpo/margin_std": 120.36772155761719,
"step": 222
},
{
"KL/chosen_KL_mean": -174.15005493164062,
"KL/mean": -210.77581787109375,
"KL/rejected_KL_mean": -247.4015655517578,
"KL/std": 112.02201843261719,
"epoch": 0.46701570680628274,
"fcm_dpo/beta": 0.0032290732488036156,
"fcm_dpo/delta": -0.0026750918477773666,
"fcm_dpo/margin": 73.25151062011719,
"fcm_dpo/q_t": 0.4433066248893738,
"grad_norm": 51.26232147216797,
"learning_rate": 3.230637461492043e-07,
"logits/chosen": -0.8510481715202332,
"logits/rejected": -0.8167670369148254,
"logps/chosen": -460.56512451171875,
"logps/ref_chosen": -286.41510009765625,
"logps/ref_rejected": -241.1181640625,
"logps/rejected": -488.5197448730469,
"loss": 4.8299,
"margin_dpo/margin_mean": 73.25151062011719,
"margin_dpo/margin_std": 127.65164184570312,
"step": 223
},
{
"KL/chosen_KL_mean": -169.56051635742188,
"KL/mean": -205.94590759277344,
"KL/rejected_KL_mean": -242.33132934570312,
"KL/std": 109.94374084472656,
"epoch": 0.46910994764397906,
"fcm_dpo/beta": 0.0032642222940921783,
"fcm_dpo/delta": 0.0056016454473137856,
"fcm_dpo/margin": 72.77080535888672,
"fcm_dpo/q_t": 0.44394826889038086,
"grad_norm": 64.43927764892578,
"learning_rate": 3.213109681595612e-07,
"logits/chosen": -0.8066267371177673,
"logits/rejected": -0.8236450552940369,
"logps/chosen": -419.0528564453125,
"logps/ref_chosen": -249.49234008789062,
"logps/ref_rejected": -233.10752868652344,
"logps/rejected": -475.4388427734375,
"loss": 4.832,
"margin_dpo/margin_mean": 72.77080535888672,
"margin_dpo/margin_std": 126.9173812866211,
"step": 224
},
{
"KL/chosen_KL_mean": -191.29263305664062,
"KL/mean": -226.8955535888672,
"KL/rejected_KL_mean": -262.4984130859375,
"KL/std": 117.79441833496094,
"epoch": 0.4712041884816754,
"fcm_dpo/beta": 0.0032615442760288715,
"fcm_dpo/delta": 0.009283446706831455,
"fcm_dpo/margin": 71.2057876586914,
"fcm_dpo/q_t": 0.4457131624221802,
"grad_norm": 65.1972885131836,
"learning_rate": 3.1955436597911315e-07,
"logits/chosen": -0.8588619828224182,
"logits/rejected": -0.8319200873374939,
"logps/chosen": -503.1509704589844,
"logps/ref_chosen": -311.8583679199219,
"logps/ref_rejected": -336.8523864746094,
"logps/rejected": -599.3507690429688,
"loss": 4.8845,
"margin_dpo/margin_mean": 71.2057876586914,
"margin_dpo/margin_std": 140.93931579589844,
"step": 225
},
{
"KL/chosen_KL_mean": -175.48651123046875,
"KL/mean": -205.9941864013672,
"KL/rejected_KL_mean": -236.5018768310547,
"KL/std": 112.99928283691406,
"epoch": 0.4732984293193717,
"fcm_dpo/beta": 0.00331744528375566,
"fcm_dpo/delta": 0.015448614954948425,
"fcm_dpo/margin": 61.015350341796875,
"fcm_dpo/q_t": 0.45238372683525085,
"grad_norm": 58.19024658203125,
"learning_rate": 3.1779403380910425e-07,
"logits/chosen": -0.874785304069519,
"logits/rejected": -0.8616780638694763,
"logps/chosen": -427.687744140625,
"logps/ref_chosen": -252.20123291015625,
"logps/ref_rejected": -254.41162109375,
"logps/rejected": -490.9134826660156,
"loss": 4.9792,
"margin_dpo/margin_mean": 61.015350341796875,
"margin_dpo/margin_std": 135.62869262695312,
"step": 226
},
{
"KL/chosen_KL_mean": -170.14938354492188,
"KL/mean": -212.30055236816406,
"KL/rejected_KL_mean": -254.45169067382812,
"KL/std": 110.70320892333984,
"epoch": 0.47539267015706804,
"fcm_dpo/beta": 0.0032987315207719803,
"fcm_dpo/delta": -0.015660453587770462,
"fcm_dpo/margin": 84.30230712890625,
"fcm_dpo/q_t": 0.4337405860424042,
"grad_norm": 55.077152252197266,
"learning_rate": 3.160300660508064e-07,
"logits/chosen": -0.8456053733825684,
"logits/rejected": -0.8431981801986694,
"logps/chosen": -455.4088134765625,
"logps/ref_chosen": -285.25946044921875,
"logps/ref_rejected": -261.3220520019531,
"logps/rejected": -515.773681640625,
"loss": 4.7053,
"margin_dpo/margin_mean": 84.30230712890625,
"margin_dpo/margin_std": 131.59988403320312,
"step": 227
},
{
"KL/chosen_KL_mean": -158.64784240722656,
"KL/mean": -197.88645935058594,
"KL/rejected_KL_mean": -237.12509155273438,
"KL/std": 105.5765380859375,
"epoch": 0.4774869109947644,
"fcm_dpo/beta": 0.003275742521509528,
"fcm_dpo/delta": -0.015454575419425964,
"fcm_dpo/margin": 78.47726440429688,
"fcm_dpo/q_t": 0.438965380191803,
"grad_norm": 57.37860870361328,
"learning_rate": 3.1426255730045695e-07,
"logits/chosen": -0.8636391162872314,
"logits/rejected": -0.8306207656860352,
"logps/chosen": -472.4666442871094,
"logps/ref_chosen": -313.81878662109375,
"logps/ref_rejected": -258.07061767578125,
"logps/rejected": -495.1957092285156,
"loss": 4.7617,
"margin_dpo/margin_mean": 78.47726440429688,
"margin_dpo/margin_std": 124.56838989257812,
"step": 228
},
{
"KL/chosen_KL_mean": -173.101806640625,
"KL/mean": -220.78363037109375,
"KL/rejected_KL_mean": -268.4654235839844,
"KL/std": 115.71029663085938,
"epoch": 0.47958115183246075,
"fcm_dpo/beta": 0.0031459908932447433,
"fcm_dpo/delta": -0.05162987485527992,
"fcm_dpo/margin": 95.36361694335938,
"fcm_dpo/q_t": 0.4281517565250397,
"grad_norm": 42.334407806396484,
"learning_rate": 3.1249160234418644e-07,
"logits/chosen": -0.8365947008132935,
"logits/rejected": -0.8395388126373291,
"logps/chosen": -465.07257080078125,
"logps/ref_chosen": -291.9707946777344,
"logps/ref_rejected": -263.42059326171875,
"logps/rejected": -531.885986328125,
"loss": 4.6133,
"margin_dpo/margin_mean": 95.36361694335938,
"margin_dpo/margin_std": 127.28816223144531,
"step": 229
},
{
"KL/chosen_KL_mean": -157.16651916503906,
"KL/mean": -198.59124755859375,
"KL/rejected_KL_mean": -240.01600646972656,
"KL/std": 123.6738052368164,
"epoch": 0.4816753926701571,
"fcm_dpo/beta": 0.0030816374346613884,
"fcm_dpo/delta": -0.005881241988390684,
"fcm_dpo/margin": 82.84947204589844,
"fcm_dpo/q_t": 0.44025325775146484,
"grad_norm": 49.02077102661133,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": -0.8741714358329773,
"logits/rejected": -0.8733953833580017,
"logps/chosen": -390.4266357421875,
"logps/ref_chosen": -233.2601318359375,
"logps/ref_rejected": -238.922119140625,
"logps/rejected": -478.9381103515625,
"loss": 4.7806,
"margin_dpo/margin_mean": 82.8494644165039,
"margin_dpo/margin_std": 139.558349609375,
"step": 230
},
{
"KL/chosen_KL_mean": -168.19107055664062,
"KL/mean": -202.2855682373047,
"KL/rejected_KL_mean": -236.38006591796875,
"KL/std": 106.24060821533203,
"epoch": 0.4837696335078534,
"fcm_dpo/beta": 0.003070762613788247,
"fcm_dpo/delta": -0.0014391806907951832,
"fcm_dpo/margin": 68.18899536132812,
"fcm_dpo/q_t": 0.4494929909706116,
"grad_norm": 62.14220428466797,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": -0.8697140216827393,
"logits/rejected": -0.8564260005950928,
"logps/chosen": -490.3462829589844,
"logps/ref_chosen": -322.1551818847656,
"logps/ref_rejected": -280.97613525390625,
"logps/rejected": -517.356201171875,
"loss": 4.8992,
"margin_dpo/margin_mean": 68.18899536132812,
"margin_dpo/margin_std": 120.236328125,
"step": 231
},
{
"KL/chosen_KL_mean": -168.53921508789062,
"KL/mean": -201.57940673828125,
"KL/rejected_KL_mean": -234.6195831298828,
"KL/std": 117.83100128173828,
"epoch": 0.48586387434554973,
"fcm_dpo/beta": 0.003069432219490409,
"fcm_dpo/delta": -0.005725067108869553,
"fcm_dpo/margin": 66.08037567138672,
"fcm_dpo/q_t": 0.45206257700920105,
"grad_norm": 58.8627815246582,
"learning_rate": 3.071590108427243e-07,
"logits/chosen": -0.8262802362442017,
"logits/rejected": -0.8069007396697998,
"logps/chosen": -440.28302001953125,
"logps/ref_chosen": -271.7437744140625,
"logps/ref_rejected": -249.94981384277344,
"logps/rejected": -484.56939697265625,
"loss": 4.9796,
"margin_dpo/margin_mean": 66.08036804199219,
"margin_dpo/margin_std": 138.81605529785156,
"step": 232
},
{
"KL/chosen_KL_mean": -159.59194946289062,
"KL/mean": -203.31967163085938,
"KL/rejected_KL_mean": -247.0474090576172,
"KL/std": 114.54499053955078,
"epoch": 0.48795811518324606,
"fcm_dpo/beta": 0.0030200586188584566,
"fcm_dpo/delta": -0.03824126720428467,
"fcm_dpo/margin": 87.45545196533203,
"fcm_dpo/q_t": 0.4372973144054413,
"grad_norm": 45.34437561035156,
"learning_rate": 3.05375222543809e-07,
"logits/chosen": -0.8714592456817627,
"logits/rejected": -0.8599870204925537,
"logps/chosen": -444.93426513671875,
"logps/ref_chosen": -285.3423156738281,
"logps/ref_rejected": -266.34320068359375,
"logps/rejected": -513.3905639648438,
"loss": 4.7357,
"margin_dpo/margin_mean": 87.45545196533203,
"margin_dpo/margin_std": 130.9783935546875,
"step": 233
},
{
"KL/chosen_KL_mean": -161.26751708984375,
"KL/mean": -199.64366149902344,
"KL/rejected_KL_mean": -238.01988220214844,
"KL/std": 113.85478973388672,
"epoch": 0.4900523560209424,
"fcm_dpo/beta": 0.003014399204403162,
"fcm_dpo/delta": 0.028298456221818924,
"fcm_dpo/margin": 76.75235748291016,
"fcm_dpo/q_t": 0.44490310549736023,
"grad_norm": 42.643096923828125,
"learning_rate": 3.035884646397637e-07,
"logits/chosen": -0.8460695743560791,
"logits/rejected": -0.8349598050117493,
"logps/chosen": -456.17327880859375,
"logps/ref_chosen": -294.9057312011719,
"logps/ref_rejected": -299.37054443359375,
"logps/rejected": -537.3904418945312,
"loss": 4.8618,
"margin_dpo/margin_mean": 76.75235748291016,
"margin_dpo/margin_std": 141.786865234375,
"step": 234
},
{
"KL/chosen_KL_mean": -162.90518188476562,
"KL/mean": -201.39187622070312,
"KL/rejected_KL_mean": -239.87855529785156,
"KL/std": 123.73214721679688,
"epoch": 0.49214659685863876,
"fcm_dpo/beta": 0.0030970873776823282,
"fcm_dpo/delta": 0.02609279938042164,
"fcm_dpo/margin": 76.973388671875,
"fcm_dpo/q_t": 0.4432239234447479,
"grad_norm": 47.34357452392578,
"learning_rate": 3.017988329489923e-07,
"logits/chosen": -0.8745311498641968,
"logits/rejected": -0.862747073173523,
"logps/chosen": -452.40277099609375,
"logps/ref_chosen": -289.49755859375,
"logps/ref_rejected": -247.55076599121094,
"logps/rejected": -487.4293212890625,
"loss": 4.8187,
"margin_dpo/margin_mean": 76.973388671875,
"margin_dpo/margin_std": 132.9673309326172,
"step": 235
},
{
"KL/chosen_KL_mean": -153.37669372558594,
"KL/mean": -191.69894409179688,
"KL/rejected_KL_mean": -230.02120971679688,
"KL/std": 117.47241973876953,
"epoch": 0.4942408376963351,
"fcm_dpo/beta": 0.0031566726975142956,
"fcm_dpo/delta": 0.016904598101973534,
"fcm_dpo/margin": 76.64453125,
"fcm_dpo/q_t": 0.4432297945022583,
"grad_norm": 57.95139694213867,
"learning_rate": 3.000064234440111e-07,
"logits/chosen": -0.8885783553123474,
"logits/rejected": -0.8885743618011475,
"logps/chosen": -442.2613525390625,
"logps/ref_chosen": -288.8846435546875,
"logps/ref_rejected": -242.0452880859375,
"logps/rejected": -472.06646728515625,
"loss": 4.8325,
"margin_dpo/margin_mean": 76.64453125,
"margin_dpo/margin_std": 139.19834899902344,
"step": 236
},
{
"KL/chosen_KL_mean": -160.40975952148438,
"KL/mean": -200.35916137695312,
"KL/rejected_KL_mean": -240.30856323242188,
"KL/std": 103.72799682617188,
"epoch": 0.4963350785340314,
"fcm_dpo/beta": 0.003175287041813135,
"fcm_dpo/delta": 0.009177761152386665,
"fcm_dpo/margin": 79.89883422851562,
"fcm_dpo/q_t": 0.4399101138114929,
"grad_norm": 42.846649169921875,
"learning_rate": 2.9821133224630223e-07,
"logits/chosen": -0.8538817167282104,
"logits/rejected": -0.8301167488098145,
"logps/chosen": -425.888427734375,
"logps/ref_chosen": -265.47869873046875,
"logps/ref_rejected": -267.9891357421875,
"logps/rejected": -508.2976989746094,
"loss": 4.7534,
"margin_dpo/margin_mean": 79.8988265991211,
"margin_dpo/margin_std": 124.49240112304688,
"step": 237
},
{
"KL/chosen_KL_mean": -169.00497436523438,
"KL/mean": -208.2441864013672,
"KL/rejected_KL_mean": -247.48336791992188,
"KL/std": 114.40408325195312,
"epoch": 0.49842931937172774,
"fcm_dpo/beta": 0.0031809653155505657,
"fcm_dpo/delta": 0.0017670519882813096,
"fcm_dpo/margin": 78.4783706665039,
"fcm_dpo/q_t": 0.44169679284095764,
"grad_norm": 57.76200485229492,
"learning_rate": 2.964136556211588e-07,
"logits/chosen": -0.862713634967804,
"logits/rejected": -0.8289677500724792,
"logps/chosen": -481.00762939453125,
"logps/ref_chosen": -312.0026550292969,
"logps/ref_rejected": -270.0257263183594,
"logps/rejected": -517.5090942382812,
"loss": 4.7887,
"margin_dpo/margin_mean": 78.4783706665039,
"margin_dpo/margin_std": 131.0048065185547,
"step": 238
},
{
"KL/chosen_KL_mean": -155.6094970703125,
"KL/mean": -191.5946807861328,
"KL/rejected_KL_mean": -227.57989501953125,
"KL/std": 121.70386505126953,
"epoch": 0.5005235602094241,
"fcm_dpo/beta": 0.003189996350556612,
"fcm_dpo/delta": 0.004023873247206211,
"fcm_dpo/margin": 71.97037506103516,
"fcm_dpo/q_t": 0.44634878635406494,
"grad_norm": 68.1906967163086,
"learning_rate": 2.946134899725226e-07,
"logits/chosen": -0.8460709452629089,
"logits/rejected": -0.8845388889312744,
"logps/chosen": -422.7767639160156,
"logps/ref_chosen": -267.167236328125,
"logps/ref_rejected": -275.99468994140625,
"logps/rejected": -503.5745544433594,
"loss": 4.9432,
"margin_dpo/margin_mean": 71.97037506103516,
"margin_dpo/margin_std": 159.57943725585938,
"step": 239
},
{
"KL/chosen_KL_mean": -157.0327606201172,
"KL/mean": -200.23236083984375,
"KL/rejected_KL_mean": -243.43197631835938,
"KL/std": 119.24127960205078,
"epoch": 0.5026178010471204,
"fcm_dpo/beta": 0.0031683642882853746,
"fcm_dpo/delta": -0.00633890088647604,
"fcm_dpo/margin": 86.39923095703125,
"fcm_dpo/q_t": 0.4365653991699219,
"grad_norm": 67.59850311279297,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": -0.9054303765296936,
"logits/rejected": -0.8970333337783813,
"logps/chosen": -443.01239013671875,
"logps/ref_chosen": -285.9796142578125,
"logps/ref_rejected": -256.8258056640625,
"logps/rejected": -500.2577819824219,
"loss": 4.7286,
"margin_dpo/margin_mean": 86.39923095703125,
"margin_dpo/margin_std": 141.60586547851562,
"step": 240
},
{
"KL/chosen_KL_mean": -176.8218231201172,
"KL/mean": -210.4013214111328,
"KL/rejected_KL_mean": -243.9808349609375,
"KL/std": 128.3367919921875,
"epoch": 0.5047120418848168,
"fcm_dpo/beta": 0.0031660483218729496,
"fcm_dpo/delta": 0.00029843486845493317,
"fcm_dpo/margin": 67.15899658203125,
"fcm_dpo/q_t": 0.45050010085105896,
"grad_norm": 50.04656982421875,
"learning_rate": 2.910060778827554e-07,
"logits/chosen": -0.8268923759460449,
"logits/rejected": -0.8014039397239685,
"logps/chosen": -438.33868408203125,
"logps/ref_chosen": -261.516845703125,
"logps/ref_rejected": -250.2250518798828,
"logps/rejected": -494.2059020996094,
"loss": 4.9681,
"margin_dpo/margin_mean": 67.15899658203125,
"margin_dpo/margin_std": 144.54986572265625,
"step": 241
},
{
"KL/chosen_KL_mean": -161.33384704589844,
"KL/mean": -197.4034881591797,
"KL/rejected_KL_mean": -233.47311401367188,
"KL/std": 123.67906188964844,
"epoch": 0.506806282722513,
"fcm_dpo/beta": 0.0031525217927992344,
"fcm_dpo/delta": 0.003016672795638442,
"fcm_dpo/margin": 72.1392822265625,
"fcm_dpo/q_t": 0.44707444310188293,
"grad_norm": 65.69779205322266,
"learning_rate": 2.891990248961871e-07,
"logits/chosen": -0.8855007886886597,
"logits/rejected": -0.8583632707595825,
"logps/chosen": -431.8477783203125,
"logps/ref_chosen": -270.51397705078125,
"logps/ref_rejected": -244.8560791015625,
"logps/rejected": -478.3291931152344,
"loss": 4.8718,
"margin_dpo/margin_mean": 72.1392822265625,
"margin_dpo/margin_std": 136.6427001953125,
"step": 242
},
{
"KL/chosen_KL_mean": -154.03050231933594,
"KL/mean": -196.08612060546875,
"KL/rejected_KL_mean": -238.14178466796875,
"KL/std": 124.8993148803711,
"epoch": 0.5089005235602094,
"fcm_dpo/beta": 0.0031572661828249693,
"fcm_dpo/delta": -0.03359325975179672,
"fcm_dpo/margin": 84.11128234863281,
"fcm_dpo/q_t": 0.4410480856895447,
"grad_norm": 62.51230239868164,
"learning_rate": 2.873898697848762e-07,
"logits/chosen": -0.8862600326538086,
"logits/rejected": -0.8692023754119873,
"logps/chosen": -478.7125549316406,
"logps/ref_chosen": -324.68206787109375,
"logps/ref_rejected": -307.1111755371094,
"logps/rejected": -545.2529296875,
"loss": 4.7993,
"margin_dpo/margin_mean": 84.11128234863281,
"margin_dpo/margin_std": 155.82298278808594,
"step": 243
},
{
"KL/chosen_KL_mean": -163.61180114746094,
"KL/mean": -206.80072021484375,
"KL/rejected_KL_mean": -249.98959350585938,
"KL/std": 115.1811752319336,
"epoch": 0.5109947643979058,
"fcm_dpo/beta": 0.0030796793289482594,
"fcm_dpo/delta": -0.0004585727583616972,
"fcm_dpo/margin": 86.37779235839844,
"fcm_dpo/q_t": 0.43625351786613464,
"grad_norm": 52.980777740478516,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": -0.867234468460083,
"logits/rejected": -0.8243776559829712,
"logps/chosen": -482.591064453125,
"logps/ref_chosen": -318.979248046875,
"logps/ref_rejected": -269.67572021484375,
"logps/rejected": -519.6653442382812,
"loss": 4.7323,
"margin_dpo/margin_mean": 86.37779235839844,
"margin_dpo/margin_std": 129.40206909179688,
"step": 244
},
{
"KL/chosen_KL_mean": -154.42869567871094,
"KL/mean": -198.804931640625,
"KL/rejected_KL_mean": -243.18118286132812,
"KL/std": 110.18680572509766,
"epoch": 0.5130890052356021,
"fcm_dpo/beta": 0.003070926060900092,
"fcm_dpo/delta": -0.018872009590268135,
"fcm_dpo/margin": 88.75247192382812,
"fcm_dpo/q_t": 0.43488627672195435,
"grad_norm": 62.36658477783203,
"learning_rate": 2.837656413735479e-07,
"logits/chosen": -0.8802121877670288,
"logits/rejected": -0.8869256377220154,
"logps/chosen": -449.32672119140625,
"logps/ref_chosen": -294.8980712890625,
"logps/ref_rejected": -239.8111114501953,
"logps/rejected": -482.9922790527344,
"loss": 4.6985,
"margin_dpo/margin_mean": 88.75247192382812,
"margin_dpo/margin_std": 128.5753631591797,
"step": 245
},
{
"KL/chosen_KL_mean": -186.30972290039062,
"KL/mean": -216.44203186035156,
"KL/rejected_KL_mean": -246.57437133789062,
"KL/std": 116.18084716796875,
"epoch": 0.5151832460732985,
"fcm_dpo/beta": 0.003061179770156741,
"fcm_dpo/delta": 0.012664351612329483,
"fcm_dpo/margin": 60.264644622802734,
"fcm_dpo/q_t": 0.457215815782547,
"grad_norm": 64.10836791992188,
"learning_rate": 2.8195076242990116e-07,
"logits/chosen": -0.8674138784408569,
"logits/rejected": -0.8710952401161194,
"logps/chosen": -466.9951171875,
"logps/ref_chosen": -280.6854248046875,
"logps/ref_rejected": -253.65382385253906,
"logps/rejected": -500.2281799316406,
"loss": 5.0715,
"margin_dpo/margin_mean": 60.264644622802734,
"margin_dpo/margin_std": 155.37069702148438,
"step": 246
},
{
"KL/chosen_KL_mean": -174.0077667236328,
"KL/mean": -212.17257690429688,
"KL/rejected_KL_mean": -250.33738708496094,
"KL/std": 119.93651580810547,
"epoch": 0.5172774869109947,
"fcm_dpo/beta": 0.0030726096592843533,
"fcm_dpo/delta": -0.012722790241241455,
"fcm_dpo/margin": 76.32959747314453,
"fcm_dpo/q_t": 0.44508832693099976,
"grad_norm": 56.767269134521484,
"learning_rate": 2.801341700638307e-07,
"logits/chosen": -0.8562105298042297,
"logits/rejected": -0.8530141115188599,
"logps/chosen": -455.1168518066406,
"logps/ref_chosen": -281.1091003417969,
"logps/ref_rejected": -260.3700866699219,
"logps/rejected": -510.7074890136719,
"loss": 4.8493,
"margin_dpo/margin_mean": 76.32959747314453,
"margin_dpo/margin_std": 135.0543212890625,
"step": 247
},
{
"KL/chosen_KL_mean": -178.17813110351562,
"KL/mean": -217.06997680664062,
"KL/rejected_KL_mean": -255.96185302734375,
"KL/std": 118.73770904541016,
"epoch": 0.5193717277486911,
"fcm_dpo/beta": 0.0030005162116140127,
"fcm_dpo/delta": -0.03079632669687271,
"fcm_dpo/margin": 77.78372192382812,
"fcm_dpo/q_t": 0.4442315101623535,
"grad_norm": 67.58417510986328,
"learning_rate": 2.7831596169367227e-07,
"logits/chosen": -0.817348301410675,
"logits/rejected": -0.8214455842971802,
"logps/chosen": -448.49652099609375,
"logps/ref_chosen": -270.318359375,
"logps/ref_rejected": -233.46778869628906,
"logps/rejected": -489.42962646484375,
"loss": 4.8472,
"margin_dpo/margin_mean": 77.78372192382812,
"margin_dpo/margin_std": 135.59178161621094,
"step": 248
},
{
"KL/chosen_KL_mean": -205.08566284179688,
"KL/mean": -236.98524475097656,
"KL/rejected_KL_mean": -268.88482666015625,
"KL/std": 122.88720703125,
"epoch": 0.5214659685863874,
"fcm_dpo/beta": 0.002997219329699874,
"fcm_dpo/delta": 0.01746796816587448,
"fcm_dpo/margin": 63.79915237426758,
"fcm_dpo/q_t": 0.4539327025413513,
"grad_norm": 51.39647674560547,
"learning_rate": 2.7649623482442274e-07,
"logits/chosen": -0.854729175567627,
"logits/rejected": -0.8268671631813049,
"logps/chosen": -480.8945007324219,
"logps/ref_chosen": -275.8088684082031,
"logps/ref_rejected": -243.45138549804688,
"logps/rejected": -512.336181640625,
"loss": 5.0311,
"margin_dpo/margin_mean": 63.79915237426758,
"margin_dpo/margin_std": 153.8164825439453,
"step": 249
},
{
"KL/chosen_KL_mean": -214.7335662841797,
"KL/mean": -255.48167419433594,
"KL/rejected_KL_mean": -296.22979736328125,
"KL/std": 139.7268524169922,
"epoch": 0.5235602094240838,
"fcm_dpo/beta": 0.0029838732443749905,
"fcm_dpo/delta": -0.01467887032777071,
"fcm_dpo/margin": 81.49626159667969,
"fcm_dpo/q_t": 0.443011999130249,
"grad_norm": 68.4245834350586,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": -0.8430991768836975,
"logits/rejected": -0.8344937562942505,
"logps/chosen": -507.2281188964844,
"logps/ref_chosen": -292.4945373535156,
"logps/ref_rejected": -284.2869567871094,
"logps/rejected": -580.5167236328125,
"loss": 4.8745,
"margin_dpo/margin_mean": 81.49626159667969,
"margin_dpo/margin_std": 161.67291259765625,
"step": 250
},
{
"KL/chosen_KL_mean": -203.28543090820312,
"KL/mean": -243.52606201171875,
"KL/rejected_KL_mean": -283.7666931152344,
"KL/std": 141.32611083984375,
"epoch": 0.5256544502617801,
"fcm_dpo/beta": 0.002941292244940996,
"fcm_dpo/delta": -0.046136289834976196,
"fcm_dpo/margin": 80.48124694824219,
"fcm_dpo/q_t": 0.4475208818912506,
"grad_norm": 59.001190185546875,
"learning_rate": 2.7285261601056697e-07,
"logits/chosen": -0.8501484394073486,
"logits/rejected": -0.8327538967132568,
"logps/chosen": -485.0220031738281,
"logps/ref_chosen": -281.736572265625,
"logps/ref_rejected": -255.9419708251953,
"logps/rejected": -539.7086791992188,
"loss": 4.907,
"margin_dpo/margin_mean": 80.48124694824219,
"margin_dpo/margin_std": 165.2854766845703,
"step": 251
},
{
"KL/chosen_KL_mean": -218.41604614257812,
"KL/mean": -259.9915771484375,
"KL/rejected_KL_mean": -301.5671081542969,
"KL/std": 131.10438537597656,
"epoch": 0.5277486910994764,
"fcm_dpo/beta": 0.002828023862093687,
"fcm_dpo/delta": -0.023578261956572533,
"fcm_dpo/margin": 83.15106201171875,
"fcm_dpo/q_t": 0.44388335943222046,
"grad_norm": 65.67149353027344,
"learning_rate": 2.7102891946217994e-07,
"logits/chosen": -0.9110385775566101,
"logits/rejected": -0.884746789932251,
"logps/chosen": -514.3834228515625,
"logps/ref_chosen": -295.9674072265625,
"logps/ref_rejected": -280.111572265625,
"logps/rejected": -581.6787109375,
"loss": 4.8493,
"margin_dpo/margin_mean": 83.15106201171875,
"margin_dpo/margin_std": 149.3827362060547,
"step": 252
},
{
"KL/chosen_KL_mean": -233.10960388183594,
"KL/mean": -268.5941467285156,
"KL/rejected_KL_mean": -304.07867431640625,
"KL/std": 137.26515197753906,
"epoch": 0.5298429319371728,
"fcm_dpo/beta": 0.002824939088895917,
"fcm_dpo/delta": 0.0016382570611312985,
"fcm_dpo/margin": 70.96907043457031,
"fcm_dpo/q_t": 0.45377257466316223,
"grad_norm": 51.97392272949219,
"learning_rate": 2.692040951966617e-07,
"logits/chosen": -0.8730481863021851,
"logits/rejected": -0.8639348745346069,
"logps/chosen": -510.181884765625,
"logps/ref_chosen": -277.072265625,
"logps/ref_rejected": -247.31643676757812,
"logps/rejected": -551.3951416015625,
"loss": 5.025,
"margin_dpo/margin_mean": 70.96907043457031,
"margin_dpo/margin_std": 168.4381103515625,
"step": 253
},
{
"KL/chosen_KL_mean": -216.6527099609375,
"KL/mean": -260.6445007324219,
"KL/rejected_KL_mean": -304.6363220214844,
"KL/std": 141.9154052734375,
"epoch": 0.5319371727748691,
"fcm_dpo/beta": 0.0028039044700562954,
"fcm_dpo/delta": -0.033436983823776245,
"fcm_dpo/margin": 87.98359680175781,
"fcm_dpo/q_t": 0.4428294897079468,
"grad_norm": 53.65641403198242,
"learning_rate": 2.6737824107379947e-07,
"logits/chosen": -0.8167764544487,
"logits/rejected": -0.795870304107666,
"logps/chosen": -486.6005554199219,
"logps/ref_chosen": -269.9478454589844,
"logps/ref_rejected": -249.45005798339844,
"logps/rejected": -554.0863647460938,
"loss": 4.8213,
"margin_dpo/margin_mean": 87.98358917236328,
"margin_dpo/margin_std": 156.28126525878906,
"step": 254
},
{
"KL/chosen_KL_mean": -208.58926391601562,
"KL/mean": -257.9679260253906,
"KL/rejected_KL_mean": -307.34661865234375,
"KL/std": 138.3050994873047,
"epoch": 0.5340314136125655,
"fcm_dpo/beta": 0.0027517471462488174,
"fcm_dpo/delta": 0.006953006610274315,
"fcm_dpo/margin": 98.75733947753906,
"fcm_dpo/q_t": 0.4354252517223358,
"grad_norm": 51.104637145996094,
"learning_rate": 2.655514550086086e-07,
"logits/chosen": -0.8255376219749451,
"logits/rejected": -0.7873697280883789,
"logps/chosen": -515.2445678710938,
"logps/ref_chosen": -306.6552734375,
"logps/ref_rejected": -254.47528076171875,
"logps/rejected": -561.8218383789062,
"loss": 4.7701,
"margin_dpo/margin_mean": 98.75733947753906,
"margin_dpo/margin_std": 176.37081909179688,
"step": 255
},
{
"KL/chosen_KL_mean": -212.90078735351562,
"KL/mean": -253.50555419921875,
"KL/rejected_KL_mean": -294.1103210449219,
"KL/std": 134.76048278808594,
"epoch": 0.5361256544502618,
"fcm_dpo/beta": 0.002711032284423709,
"fcm_dpo/delta": -0.04468809440732002,
"fcm_dpo/margin": 81.20955657958984,
"fcm_dpo/q_t": 0.44711193442344666,
"grad_norm": 60.70124816894531,
"learning_rate": 2.6372383496608186e-07,
"logits/chosen": -0.8649431467056274,
"logits/rejected": -0.8482561707496643,
"logps/chosen": -536.6189575195312,
"logps/ref_chosen": -323.7181701660156,
"logps/ref_rejected": -254.1871337890625,
"logps/rejected": -548.2974853515625,
"loss": 4.9389,
"margin_dpo/margin_mean": 81.20955657958984,
"margin_dpo/margin_std": 163.05526733398438,
"step": 256
},
{
"KL/chosen_KL_mean": -213.99270629882812,
"KL/mean": -261.0357360839844,
"KL/rejected_KL_mean": -308.07879638671875,
"KL/std": 134.3519287109375,
"epoch": 0.5382198952879581,
"fcm_dpo/beta": 0.0026609383057802916,
"fcm_dpo/delta": -0.0028730593621730804,
"fcm_dpo/margin": 94.08607482910156,
"fcm_dpo/q_t": 0.44072961807250977,
"grad_norm": 55.27256774902344,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": -0.8478982448577881,
"logits/rejected": -0.8403754234313965,
"logps/chosen": -481.2048034667969,
"logps/ref_chosen": -267.21209716796875,
"logps/ref_rejected": -249.12579345703125,
"logps/rejected": -557.20458984375,
"loss": 4.7854,
"margin_dpo/margin_mean": 94.08607482910156,
"margin_dpo/margin_std": 158.97830200195312,
"step": 257
},
{
"KL/chosen_KL_mean": -220.2513427734375,
"KL/mean": -272.7309875488281,
"KL/rejected_KL_mean": -325.2106628417969,
"KL/std": 131.34759521484375,
"epoch": 0.5403141361256545,
"fcm_dpo/beta": 0.0025932216085493565,
"fcm_dpo/delta": -0.044448841363191605,
"fcm_dpo/margin": 104.95930480957031,
"fcm_dpo/q_t": 0.43503230810165405,
"grad_norm": 51.729408264160156,
"learning_rate": 2.600664850273538e-07,
"logits/chosen": -0.8749293088912964,
"logits/rejected": -0.8454850316047668,
"logps/chosen": -497.93408203125,
"logps/ref_chosen": -277.6827392578125,
"logps/ref_rejected": -250.73385620117188,
"logps/rejected": -575.9445190429688,
"loss": 4.698,
"margin_dpo/margin_mean": 104.95930480957031,
"margin_dpo/margin_std": 149.8773193359375,
"step": 258
},
{
"KL/chosen_KL_mean": -205.57168579101562,
"KL/mean": -254.19036865234375,
"KL/rejected_KL_mean": -302.80902099609375,
"KL/std": 146.01449584960938,
"epoch": 0.5424083769633508,
"fcm_dpo/beta": 0.0025686267763376236,
"fcm_dpo/delta": -0.010285234078764915,
"fcm_dpo/margin": 97.23735809326172,
"fcm_dpo/q_t": 0.4419627785682678,
"grad_norm": 61.2537956237793,
"learning_rate": 2.582369512637302e-07,
"logits/chosen": -0.8877295255661011,
"logits/rejected": -0.8807848691940308,
"logps/chosen": -500.1816711425781,
"logps/ref_chosen": -294.6099853515625,
"logps/ref_rejected": -272.2725830078125,
"logps/rejected": -575.0816040039062,
"loss": 4.8003,
"margin_dpo/margin_mean": 97.23736572265625,
"margin_dpo/margin_std": 166.01547241210938,
"step": 259
},
{
"KL/chosen_KL_mean": -215.47994995117188,
"KL/mean": -236.25408935546875,
"KL/rejected_KL_mean": -257.0281982421875,
"KL/std": 120.56246948242188,
"epoch": 0.5445026178010471,
"fcm_dpo/beta": 0.0025487085804343224,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.548255920410156,
"fcm_dpo/q_t": 0.4746982753276825,
"grad_norm": 50.7750129699707,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": -0.8832182884216309,
"logits/rejected": -0.8753829002380371,
"logps/chosen": -506.3370666503906,
"logps/ref_chosen": -290.85711669921875,
"logps/ref_rejected": -277.5970153808594,
"logps/rejected": -534.625244140625,
"loss": 5.265,
"margin_dpo/margin_mean": 41.548255920410156,
"margin_dpo/margin_std": 138.51763916015625,
"step": 260
},
{
"KL/chosen_KL_mean": -227.56674194335938,
"KL/mean": -264.64959716796875,
"KL/rejected_KL_mean": -301.732421875,
"KL/std": 133.1595458984375,
"epoch": 0.5465968586387434,
"fcm_dpo/beta": 0.0025418729055672884,
"fcm_dpo/delta": -0.007170156575739384,
"fcm_dpo/margin": 74.16572570800781,
"fcm_dpo/q_t": 0.4553813934326172,
"grad_norm": 65.28130340576172,
"learning_rate": 2.5457665670441937e-07,
"logits/chosen": -0.7683281302452087,
"logits/rejected": -0.7774943113327026,
"logps/chosen": -478.6989440917969,
"logps/ref_chosen": -251.13223266601562,
"logps/ref_rejected": -244.76016235351562,
"logps/rejected": -546.4925537109375,
"loss": 5.0165,
"margin_dpo/margin_mean": 74.16571807861328,
"margin_dpo/margin_std": 169.27870178222656,
"step": 261
},
{
"KL/chosen_KL_mean": -209.81301879882812,
"KL/mean": -261.0389099121094,
"KL/rejected_KL_mean": -312.2647705078125,
"KL/std": 132.10711669921875,
"epoch": 0.5486910994764398,
"fcm_dpo/beta": 0.002556585008278489,
"fcm_dpo/delta": -0.00010207295417785645,
"fcm_dpo/margin": 102.45171356201172,
"fcm_dpo/q_t": 0.4371283948421478,
"grad_norm": 63.33259582519531,
"learning_rate": 2.527460921992209e-07,
"logits/chosen": -0.8112813830375671,
"logits/rejected": -0.7984949350357056,
"logps/chosen": -509.5347900390625,
"logps/ref_chosen": -299.7217712402344,
"logps/ref_rejected": -277.0969543457031,
"logps/rejected": -589.3616943359375,
"loss": 4.7045,
"margin_dpo/margin_mean": 102.45170593261719,
"margin_dpo/margin_std": 142.5891876220703,
"step": 262
},
{
"KL/chosen_KL_mean": -216.5228729248047,
"KL/mean": -258.5859680175781,
"KL/rejected_KL_mean": -300.64898681640625,
"KL/std": 135.47119140625,
"epoch": 0.5507853403141362,
"fcm_dpo/beta": 0.0025327729526907206,
"fcm_dpo/delta": -0.0021185576915740967,
"fcm_dpo/margin": 84.12616729736328,
"fcm_dpo/q_t": 0.4490615129470825,
"grad_norm": 44.847206115722656,
"learning_rate": 2.509153804294318e-07,
"logits/chosen": -0.8098288178443909,
"logits/rejected": -0.7890791296958923,
"logps/chosen": -496.4754333496094,
"logps/ref_chosen": -279.95257568359375,
"logps/ref_rejected": -256.5327453613281,
"logps/rejected": -557.1817626953125,
"loss": 4.9059,
"margin_dpo/margin_mean": 84.12616729736328,
"margin_dpo/margin_std": 155.34945678710938,
"step": 263
},
{
"KL/chosen_KL_mean": -206.41465759277344,
"KL/mean": -260.0697937011719,
"KL/rejected_KL_mean": -313.7249450683594,
"KL/std": 139.62014770507812,
"epoch": 0.5528795811518324,
"fcm_dpo/beta": 0.0024950942024588585,
"fcm_dpo/delta": -0.015230976976454258,
"fcm_dpo/margin": 107.31029510498047,
"fcm_dpo/q_t": 0.43691185116767883,
"grad_norm": 69.77923583984375,
"learning_rate": 2.4908461957056825e-07,
"logits/chosen": -0.8283612728118896,
"logits/rejected": -0.8172589540481567,
"logps/chosen": -466.94976806640625,
"logps/ref_chosen": -260.53509521484375,
"logps/ref_rejected": -255.53799438476562,
"logps/rejected": -569.262939453125,
"loss": 4.7187,
"margin_dpo/margin_mean": 107.31029510498047,
"margin_dpo/margin_std": 161.04441833496094,
"step": 264
},
{
"KL/chosen_KL_mean": -207.57537841796875,
"KL/mean": -265.12884521484375,
"KL/rejected_KL_mean": -322.6822814941406,
"KL/std": 153.11061096191406,
"epoch": 0.5549738219895288,
"fcm_dpo/beta": 0.00249352864921093,
"fcm_dpo/delta": -0.007051775231957436,
"fcm_dpo/margin": 115.10690307617188,
"fcm_dpo/q_t": 0.43265461921691895,
"grad_norm": 40.28909683227539,
"learning_rate": 2.4725390780077905e-07,
"logits/chosen": -0.8889414668083191,
"logits/rejected": -0.8872557878494263,
"logps/chosen": -491.2884216308594,
"logps/ref_chosen": -283.7130432128906,
"logps/ref_rejected": -270.3209533691406,
"logps/rejected": -593.0032348632812,
"loss": 4.6979,
"margin_dpo/margin_mean": 115.10690307617188,
"margin_dpo/margin_std": 186.2714385986328,
"step": 265
},
{
"KL/chosen_KL_mean": -189.83436584472656,
"KL/mean": -245.71548461914062,
"KL/rejected_KL_mean": -301.5965881347656,
"KL/std": 136.78062438964844,
"epoch": 0.5570680628272251,
"fcm_dpo/beta": 0.0025241486728191376,
"fcm_dpo/delta": 0.017888490110635757,
"fcm_dpo/margin": 111.76222229003906,
"fcm_dpo/q_t": 0.4332635700702667,
"grad_norm": 66.26383972167969,
"learning_rate": 2.454233432955807e-07,
"logits/chosen": -0.9080507755279541,
"logits/rejected": -0.8778403401374817,
"logps/chosen": -467.93365478515625,
"logps/ref_chosen": -278.09930419921875,
"logps/ref_rejected": -260.6734619140625,
"logps/rejected": -562.2700805664062,
"loss": 4.6501,
"margin_dpo/margin_mean": 111.76222229003906,
"margin_dpo/margin_std": 153.3962860107422,
"step": 266
},
{
"KL/chosen_KL_mean": -225.291015625,
"KL/mean": -266.1501159667969,
"KL/rejected_KL_mean": -307.0091857910156,
"KL/std": 149.58706665039062,
"epoch": 0.5591623036649215,
"fcm_dpo/beta": 0.002512805862352252,
"fcm_dpo/delta": -0.0011747367680072784,
"fcm_dpo/margin": 81.71817016601562,
"fcm_dpo/q_t": 0.45182037353515625,
"grad_norm": 65.87675476074219,
"learning_rate": 2.435930242225919e-07,
"logits/chosen": -0.8443954586982727,
"logits/rejected": -0.8486557602882385,
"logps/chosen": -505.6242370605469,
"logps/ref_chosen": -280.33319091796875,
"logps/ref_rejected": -247.78099060058594,
"logps/rejected": -554.7901611328125,
"loss": 4.9719,
"margin_dpo/margin_mean": 81.71817016601562,
"margin_dpo/margin_std": 172.19381713867188,
"step": 267
},
{
"KL/chosen_KL_mean": -215.37521362304688,
"KL/mean": -266.24896240234375,
"KL/rejected_KL_mean": -317.1227111816406,
"KL/std": 149.34877014160156,
"epoch": 0.5612565445026177,
"fcm_dpo/beta": 0.002550693228840828,
"fcm_dpo/delta": 0.024130037054419518,
"fcm_dpo/margin": 101.74750518798828,
"fcm_dpo/q_t": 0.43864506483078003,
"grad_norm": 51.989707946777344,
"learning_rate": 2.4176304873626984e-07,
"logits/chosen": -0.8222531080245972,
"logits/rejected": -0.8021284341812134,
"logps/chosen": -519.5538940429688,
"logps/ref_chosen": -304.1787109375,
"logps/ref_rejected": -272.80316162109375,
"logps/rejected": -589.9259033203125,
"loss": 4.7542,
"margin_dpo/margin_mean": 101.74750518798828,
"margin_dpo/margin_std": 164.10125732421875,
"step": 268
},
{
"KL/chosen_KL_mean": -209.81631469726562,
"KL/mean": -247.71542358398438,
"KL/rejected_KL_mean": -285.6145324707031,
"KL/std": 126.85572052001953,
"epoch": 0.5633507853403141,
"fcm_dpo/beta": 0.002604001434519887,
"fcm_dpo/delta": 0.029024910181760788,
"fcm_dpo/margin": 75.79827880859375,
"fcm_dpo/q_t": 0.45281317830085754,
"grad_norm": 49.523555755615234,
"learning_rate": 2.399335149726463e-07,
"logits/chosen": -0.8387216329574585,
"logits/rejected": -0.8304933309555054,
"logps/chosen": -459.6614074707031,
"logps/ref_chosen": -249.84512329101562,
"logps/ref_rejected": -223.37356567382812,
"logps/rejected": -508.9881591796875,
"loss": 4.9555,
"margin_dpo/margin_mean": 75.79827880859375,
"margin_dpo/margin_std": 152.57525634765625,
"step": 269
},
{
"KL/chosen_KL_mean": -222.31195068359375,
"KL/mean": -272.5738525390625,
"KL/rejected_KL_mean": -322.8357238769531,
"KL/std": 154.9764404296875,
"epoch": 0.5654450261780105,
"fcm_dpo/beta": 0.0026149782352149487,
"fcm_dpo/delta": 0.005393762607127428,
"fcm_dpo/margin": 100.5237808227539,
"fcm_dpo/q_t": 0.4390154480934143,
"grad_norm": 45.56135940551758,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": -0.9296514391899109,
"logits/rejected": -0.9286025762557983,
"logps/chosen": -540.8743286132812,
"logps/ref_chosen": -318.5623779296875,
"logps/ref_rejected": -281.1880798339844,
"logps/rejected": -604.0238037109375,
"loss": 4.81,
"margin_dpo/margin_mean": 100.5237808227539,
"margin_dpo/margin_std": 193.27618408203125,
"step": 270
},
{
"KL/chosen_KL_mean": -219.34278869628906,
"KL/mean": -264.65020751953125,
"KL/rejected_KL_mean": -309.9576721191406,
"KL/std": 151.9405517578125,
"epoch": 0.5675392670157068,
"fcm_dpo/beta": 0.002673304872587323,
"fcm_dpo/delta": 0.020023522898554802,
"fcm_dpo/margin": 90.61488342285156,
"fcm_dpo/q_t": 0.4431973099708557,
"grad_norm": 49.953712463378906,
"learning_rate": 2.3627616503391812e-07,
"logits/chosen": -0.7674182057380676,
"logits/rejected": -0.7575105428695679,
"logps/chosen": -503.44744873046875,
"logps/ref_chosen": -284.104736328125,
"logps/ref_rejected": -253.9580535888672,
"logps/rejected": -563.915771484375,
"loss": 4.8207,
"margin_dpo/margin_mean": 90.61488342285156,
"margin_dpo/margin_std": 158.77781677246094,
"step": 271
},
{
"KL/chosen_KL_mean": -213.052734375,
"KL/mean": -261.3451232910156,
"KL/rejected_KL_mean": -309.63751220703125,
"KL/std": 154.0564422607422,
"epoch": 0.5696335078534032,
"fcm_dpo/beta": 0.0026661050505936146,
"fcm_dpo/delta": -0.0036860520485788584,
"fcm_dpo/margin": 96.58477783203125,
"fcm_dpo/q_t": 0.44022059440612793,
"grad_norm": 59.533260345458984,
"learning_rate": 2.344485449913914e-07,
"logits/chosen": -0.86882084608078,
"logits/rejected": -0.8478980660438538,
"logps/chosen": -510.4117431640625,
"logps/ref_chosen": -297.3590087890625,
"logps/ref_rejected": -279.20196533203125,
"logps/rejected": -588.8394775390625,
"loss": 4.8307,
"margin_dpo/margin_mean": 96.58477783203125,
"margin_dpo/margin_std": 188.1719970703125,
"step": 272
},
{
"KL/chosen_KL_mean": -212.71578979492188,
"KL/mean": -263.0025634765625,
"KL/rejected_KL_mean": -313.28936767578125,
"KL/std": 143.3462371826172,
"epoch": 0.5717277486910994,
"fcm_dpo/beta": 0.002671892289072275,
"fcm_dpo/delta": 0.0007058036280795932,
"fcm_dpo/margin": 100.5736083984375,
"fcm_dpo/q_t": 0.43806958198547363,
"grad_norm": 48.56292724609375,
"learning_rate": 2.3262175892620062e-07,
"logits/chosen": -0.8706230521202087,
"logits/rejected": -0.8805627226829529,
"logps/chosen": -505.9215393066406,
"logps/ref_chosen": -293.20574951171875,
"logps/ref_rejected": -274.7646789550781,
"logps/rejected": -588.0540771484375,
"loss": 4.7907,
"margin_dpo/margin_mean": 100.5736083984375,
"margin_dpo/margin_std": 187.0547637939453,
"step": 273
},
{
"KL/chosen_KL_mean": -211.43441772460938,
"KL/mean": -272.50616455078125,
"KL/rejected_KL_mean": -333.57781982421875,
"KL/std": 158.19119262695312,
"epoch": 0.5738219895287958,
"fcm_dpo/beta": 0.00263871718198061,
"fcm_dpo/delta": -0.023334428668022156,
"fcm_dpo/margin": 122.1434326171875,
"fcm_dpo/q_t": 0.4251728653907776,
"grad_norm": 49.472415924072266,
"learning_rate": 2.3079590480333827e-07,
"logits/chosen": -0.8166660070419312,
"logits/rejected": -0.7837158441543579,
"logps/chosen": -481.9930725097656,
"logps/ref_chosen": -270.55865478515625,
"logps/ref_rejected": -239.47048950195312,
"logps/rejected": -573.0482788085938,
"loss": 4.5988,
"margin_dpo/margin_mean": 122.1434326171875,
"margin_dpo/margin_std": 185.6885528564453,
"step": 274
},
{
"KL/chosen_KL_mean": -190.10397338867188,
"KL/mean": -249.68603515625,
"KL/rejected_KL_mean": -309.2680969238281,
"KL/std": 151.7843780517578,
"epoch": 0.5759162303664922,
"fcm_dpo/beta": 0.002591077471151948,
"fcm_dpo/delta": -0.01983138732612133,
"fcm_dpo/margin": 119.16410064697266,
"fcm_dpo/q_t": 0.42760196328163147,
"grad_norm": 40.00221252441406,
"learning_rate": 2.2897108053782e-07,
"logits/chosen": -0.8611962795257568,
"logits/rejected": -0.8414742946624756,
"logps/chosen": -440.4231872558594,
"logps/ref_chosen": -250.31922912597656,
"logps/ref_rejected": -249.3187255859375,
"logps/rejected": -558.5867919921875,
"loss": 4.5833,
"margin_dpo/margin_mean": 119.16410064697266,
"margin_dpo/margin_std": 162.83229064941406,
"step": 275
},
{
"KL/chosen_KL_mean": -208.79034423828125,
"KL/mean": -267.42169189453125,
"KL/rejected_KL_mean": -326.05303955078125,
"KL/std": 164.59161376953125,
"epoch": 0.5780104712041885,
"fcm_dpo/beta": 0.0025207321159541607,
"fcm_dpo/delta": -0.0197414793074131,
"fcm_dpo/margin": 117.26268005371094,
"fcm_dpo/q_t": 0.43078911304473877,
"grad_norm": 56.43882369995117,
"learning_rate": 2.2714738398943308e-07,
"logits/chosen": -0.9205873012542725,
"logits/rejected": -0.8991212844848633,
"logps/chosen": -506.4214172363281,
"logps/ref_chosen": -297.6310729980469,
"logps/ref_rejected": -295.225830078125,
"logps/rejected": -621.2788696289062,
"loss": 4.6872,
"margin_dpo/margin_mean": 117.26268005371094,
"margin_dpo/margin_std": 186.11912536621094,
"step": 276
},
{
"KL/chosen_KL_mean": -236.33641052246094,
"KL/mean": -275.8341369628906,
"KL/rejected_KL_mean": -315.3318786621094,
"KL/std": 163.67776489257812,
"epoch": 0.5801047120418849,
"fcm_dpo/beta": 0.0025246557779610157,
"fcm_dpo/delta": -0.01643836311995983,
"fcm_dpo/margin": 78.99545288085938,
"fcm_dpo/q_t": 0.45376577973365784,
"grad_norm": 67.82545471191406,
"learning_rate": 2.2532491295748865e-07,
"logits/chosen": -0.8489004969596863,
"logits/rejected": -0.848082423210144,
"logps/chosen": -502.6968994140625,
"logps/ref_chosen": -266.3604736328125,
"logps/ref_rejected": -253.36767578125,
"logps/rejected": -568.6995239257812,
"loss": 5.0265,
"margin_dpo/margin_mean": 78.99545288085938,
"margin_dpo/margin_std": 191.4637451171875,
"step": 277
},
{
"KL/chosen_KL_mean": -250.80859375,
"KL/mean": -280.59112548828125,
"KL/rejected_KL_mean": -310.3736572265625,
"KL/std": 171.3376922607422,
"epoch": 0.5821989528795811,
"fcm_dpo/beta": 0.002524363109841943,
"fcm_dpo/delta": 0.014590254053473473,
"fcm_dpo/margin": 59.56508255004883,
"fcm_dpo/q_t": 0.46817678213119507,
"grad_norm": 93.50398254394531,
"learning_rate": 2.2350376517557726e-07,
"logits/chosen": -0.8730792999267578,
"logits/rejected": -0.8339457511901855,
"logps/chosen": -518.2158813476562,
"logps/ref_chosen": -267.40728759765625,
"logps/ref_rejected": -229.5758514404297,
"logps/rejected": -539.949462890625,
"loss": 5.2911,
"margin_dpo/margin_mean": 59.56508255004883,
"margin_dpo/margin_std": 218.88031005859375,
"step": 278
},
{
"KL/chosen_KL_mean": -235.7555389404297,
"KL/mean": -301.6609191894531,
"KL/rejected_KL_mean": -367.5663146972656,
"KL/std": 183.39990234375,
"epoch": 0.5842931937172775,
"fcm_dpo/beta": 0.002474685199558735,
"fcm_dpo/delta": -0.06022359058260918,
"fcm_dpo/margin": 131.81077575683594,
"fcm_dpo/q_t": 0.42557230591773987,
"grad_norm": 52.174964904785156,
"learning_rate": 2.2168403830632769e-07,
"logits/chosen": -0.8034964799880981,
"logits/rejected": -0.7859145998954773,
"logps/chosen": -549.123291015625,
"logps/ref_chosen": -313.3677978515625,
"logps/ref_rejected": -299.1744384765625,
"logps/rejected": -666.7407836914062,
"loss": 4.6083,
"margin_dpo/margin_mean": 131.81077575683594,
"margin_dpo/margin_std": 206.10121154785156,
"step": 279
},
{
"KL/chosen_KL_mean": -223.7259063720703,
"KL/mean": -272.3006286621094,
"KL/rejected_KL_mean": -320.87530517578125,
"KL/std": 148.4961700439453,
"epoch": 0.5863874345549738,
"fcm_dpo/beta": 0.002427027327939868,
"fcm_dpo/delta": 0.006918002385646105,
"fcm_dpo/margin": 97.14940643310547,
"fcm_dpo/q_t": 0.4443333148956299,
"grad_norm": 53.57758331298828,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": -0.8703023791313171,
"logits/rejected": -0.8816097974777222,
"logps/chosen": -489.28173828125,
"logps/ref_chosen": -265.5558166503906,
"logps/ref_rejected": -247.1573944091797,
"logps/rejected": -568.03271484375,
"loss": 4.8549,
"margin_dpo/margin_mean": 97.14940643310547,
"margin_dpo/margin_std": 184.31455993652344,
"step": 280
},
{
"KL/chosen_KL_mean": -253.8318634033203,
"KL/mean": -313.9239501953125,
"KL/rejected_KL_mean": -374.0159912109375,
"KL/std": 170.88157653808594,
"epoch": 0.5884816753926702,
"fcm_dpo/beta": 0.002389302011579275,
"fcm_dpo/delta": -0.018726233392953873,
"fcm_dpo/margin": 120.18415832519531,
"fcm_dpo/q_t": 0.4337007999420166,
"grad_norm": 62.008872985839844,
"learning_rate": 2.1804923757009882e-07,
"logits/chosen": -0.8482406735420227,
"logits/rejected": -0.855226993560791,
"logps/chosen": -549.1314086914062,
"logps/ref_chosen": -295.2995910644531,
"logps/ref_rejected": -293.80877685546875,
"logps/rejected": -667.82470703125,
"loss": 4.6938,
"margin_dpo/margin_mean": 120.18414306640625,
"margin_dpo/margin_std": 185.703369140625,
"step": 281
},
{
"KL/chosen_KL_mean": -258.6143493652344,
"KL/mean": -303.7931213378906,
"KL/rejected_KL_mean": -348.971923828125,
"KL/std": 165.28585815429688,
"epoch": 0.5905759162303665,
"fcm_dpo/beta": 0.002437637187540531,
"fcm_dpo/delta": 0.035667020827531815,
"fcm_dpo/margin": 90.3575668334961,
"fcm_dpo/q_t": 0.4478360116481781,
"grad_norm": 55.04109573364258,
"learning_rate": 2.1623435862645205e-07,
"logits/chosen": -0.8456370234489441,
"logits/rejected": -0.8471148610115051,
"logps/chosen": -577.2515258789062,
"logps/ref_chosen": -318.63714599609375,
"logps/ref_rejected": -273.5943603515625,
"logps/rejected": -622.5662841796875,
"loss": 4.9269,
"margin_dpo/margin_mean": 90.35755920410156,
"margin_dpo/margin_std": 188.9864959716797,
"step": 282
},
{
"KL/chosen_KL_mean": -248.04953002929688,
"KL/mean": -300.7643127441406,
"KL/rejected_KL_mean": -353.47906494140625,
"KL/std": 167.42236328125,
"epoch": 0.5926701570680628,
"fcm_dpo/beta": 0.0024897807743400335,
"fcm_dpo/delta": 0.0018690899014472961,
"fcm_dpo/margin": 105.42955780029297,
"fcm_dpo/q_t": 0.4403661787509918,
"grad_norm": 55.374061584472656,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": -0.8618201017379761,
"logits/rejected": -0.846630871295929,
"logps/chosen": -502.7100830078125,
"logps/ref_chosen": -254.66053771972656,
"logps/ref_rejected": -236.8627166748047,
"logps/rejected": -590.3417358398438,
"loss": 4.8116,
"margin_dpo/margin_mean": 105.42955780029297,
"margin_dpo/margin_std": 198.71615600585938,
"step": 283
},
{
"KL/chosen_KL_mean": -259.0715026855469,
"KL/mean": -313.150390625,
"KL/rejected_KL_mean": -367.229248046875,
"KL/std": 178.3697509765625,
"epoch": 0.5947643979057592,
"fcm_dpo/beta": 0.002422032179310918,
"fcm_dpo/delta": -0.03486540541052818,
"fcm_dpo/margin": 108.15774536132812,
"fcm_dpo/q_t": 0.4399450421333313,
"grad_norm": 78.6396255493164,
"learning_rate": 2.1261013021512378e-07,
"logits/chosen": -0.8256248235702515,
"logits/rejected": -0.7858996987342834,
"logps/chosen": -532.4268188476562,
"logps/ref_chosen": -273.355224609375,
"logps/ref_rejected": -259.84759521484375,
"logps/rejected": -627.0768432617188,
"loss": 4.8311,
"margin_dpo/margin_mean": 108.15774536132812,
"margin_dpo/margin_std": 207.28114318847656,
"step": 284
},
{
"KL/chosen_KL_mean": -270.5787353515625,
"KL/mean": -302.9770202636719,
"KL/rejected_KL_mean": -335.37530517578125,
"KL/std": 163.78836059570312,
"epoch": 0.5968586387434555,
"fcm_dpo/beta": 0.00241269962862134,
"fcm_dpo/delta": -0.005318695679306984,
"fcm_dpo/margin": 64.7966079711914,
"fcm_dpo/q_t": 0.46380117535591125,
"grad_norm": 94.97909545898438,
"learning_rate": 2.1080097510381294e-07,
"logits/chosen": -0.8431457877159119,
"logits/rejected": -0.8374038338661194,
"logps/chosen": -580.3809204101562,
"logps/ref_chosen": -309.8022155761719,
"logps/ref_rejected": -279.11846923828125,
"logps/rejected": -614.4937744140625,
"loss": 5.2171,
"margin_dpo/margin_mean": 64.7966079711914,
"margin_dpo/margin_std": 207.95433044433594,
"step": 285
},
{
"KL/chosen_KL_mean": -250.92564392089844,
"KL/mean": -298.47711181640625,
"KL/rejected_KL_mean": -346.0285339355469,
"KL/std": 179.1441192626953,
"epoch": 0.5989528795811518,
"fcm_dpo/beta": 0.002375759882852435,
"fcm_dpo/delta": -0.007511192001402378,
"fcm_dpo/margin": 95.10287475585938,
"fcm_dpo/q_t": 0.44973820447921753,
"grad_norm": 66.971435546875,
"learning_rate": 2.089939221172446e-07,
"logits/chosen": -0.8156773447990417,
"logits/rejected": -0.8105944991111755,
"logps/chosen": -522.3911743164062,
"logps/ref_chosen": -271.4655456542969,
"logps/ref_rejected": -279.531494140625,
"logps/rejected": -625.5599975585938,
"loss": 4.9477,
"margin_dpo/margin_mean": 95.10287475585938,
"margin_dpo/margin_std": 213.24725341796875,
"step": 286
},
{
"KL/chosen_KL_mean": -257.23956298828125,
"KL/mean": -302.582275390625,
"KL/rejected_KL_mean": -347.92498779296875,
"KL/std": 168.57205200195312,
"epoch": 0.6010471204188481,
"fcm_dpo/beta": 0.0023798570036888123,
"fcm_dpo/delta": 0.008962479420006275,
"fcm_dpo/margin": 90.68539428710938,
"fcm_dpo/q_t": 0.4496259093284607,
"grad_norm": 69.09150695800781,
"learning_rate": 2.0718906816218595e-07,
"logits/chosen": -0.8373988270759583,
"logits/rejected": -0.821157693862915,
"logps/chosen": -534.3328247070312,
"logps/ref_chosen": -277.0932312011719,
"logps/ref_rejected": -233.55599975585938,
"logps/rejected": -581.4810180664062,
"loss": 4.9939,
"margin_dpo/margin_mean": 90.68539428710938,
"margin_dpo/margin_std": 213.72222900390625,
"step": 287
},
{
"KL/chosen_KL_mean": -254.89105224609375,
"KL/mean": -298.76812744140625,
"KL/rejected_KL_mean": -342.64520263671875,
"KL/std": 174.66859436035156,
"epoch": 0.6031413612565445,
"fcm_dpo/beta": 0.00239731278270483,
"fcm_dpo/delta": -0.010840969160199165,
"fcm_dpo/margin": 87.75419616699219,
"fcm_dpo/q_t": 0.45140206813812256,
"grad_norm": 61.62644577026367,
"learning_rate": 2.053865100274774e-07,
"logits/chosen": -0.862617015838623,
"logits/rejected": -0.8665178418159485,
"logps/chosen": -548.0592041015625,
"logps/ref_chosen": -293.1681823730469,
"logps/ref_rejected": -263.4059143066406,
"logps/rejected": -606.0511474609375,
"loss": 4.9783,
"margin_dpo/margin_mean": 87.75419616699219,
"margin_dpo/margin_std": 197.9239959716797,
"step": 288
},
{
"KL/chosen_KL_mean": -280.8144226074219,
"KL/mean": -305.62109375,
"KL/rejected_KL_mean": -330.42779541015625,
"KL/std": 159.7655029296875,
"epoch": 0.6052356020942409,
"fcm_dpo/beta": 0.002391617279499769,
"fcm_dpo/delta": 0.014636196196079254,
"fcm_dpo/margin": 49.613338470458984,
"fcm_dpo/q_t": 0.47206318378448486,
"grad_norm": 76.14956665039062,
"learning_rate": 2.035863443788411e-07,
"logits/chosen": -0.8445371985435486,
"logits/rejected": -0.8282222747802734,
"logps/chosen": -610.7718505859375,
"logps/ref_chosen": -329.9574279785156,
"logps/ref_rejected": -276.7565002441406,
"logps/rejected": -607.1842651367188,
"loss": 5.2943,
"margin_dpo/margin_mean": 49.61334228515625,
"margin_dpo/margin_std": 183.8115234375,
"step": 289
},
{
"KL/chosen_KL_mean": -241.115966796875,
"KL/mean": -295.4197998046875,
"KL/rejected_KL_mean": -349.7236328125,
"KL/std": 173.45321655273438,
"epoch": 0.6073298429319371,
"fcm_dpo/beta": 0.0023513035848736763,
"fcm_dpo/delta": -0.02749420329928398,
"fcm_dpo/margin": 108.60765838623047,
"fcm_dpo/q_t": 0.4406154751777649,
"grad_norm": 70.177490234375,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": -0.8551790714263916,
"logits/rejected": -0.7867915034294128,
"logps/chosen": -565.7849731445312,
"logps/ref_chosen": -324.6690673828125,
"logps/ref_rejected": -311.8439636230469,
"logps/rejected": -661.5675659179688,
"loss": 4.8276,
"margin_dpo/margin_mean": 108.60765838623047,
"margin_dpo/margin_std": 204.76686096191406,
"step": 290
},
{
"KL/chosen_KL_mean": -229.23654174804688,
"KL/mean": -290.4278869628906,
"KL/rejected_KL_mean": -351.6192321777344,
"KL/std": 162.2596435546875,
"epoch": 0.6094240837696335,
"fcm_dpo/beta": 0.002302813110873103,
"fcm_dpo/delta": -0.028997551649808884,
"fcm_dpo/margin": 122.38272857666016,
"fcm_dpo/q_t": 0.43339967727661133,
"grad_norm": 71.22866821289062,
"learning_rate": 1.9999357655598891e-07,
"logits/chosen": -0.832295298576355,
"logits/rejected": -0.8221176862716675,
"logps/chosen": -503.380615234375,
"logps/ref_chosen": -274.1440734863281,
"logps/ref_rejected": -278.07208251953125,
"logps/rejected": -629.6913452148438,
"loss": 4.6779,
"margin_dpo/margin_mean": 122.38272094726562,
"margin_dpo/margin_std": 177.83932495117188,
"step": 291
},
{
"KL/chosen_KL_mean": -220.65379333496094,
"KL/mean": -276.26177978515625,
"KL/rejected_KL_mean": -331.8697204589844,
"KL/std": 173.44187927246094,
"epoch": 0.6115183246073298,
"fcm_dpo/beta": 0.0022653606720268726,
"fcm_dpo/delta": -0.005662755109369755,
"fcm_dpo/margin": 111.21595764160156,
"fcm_dpo/q_t": 0.4417262077331543,
"grad_norm": 73.43364715576172,
"learning_rate": 1.9820116705100775e-07,
"logits/chosen": -0.8171704411506653,
"logits/rejected": -0.8130745887756348,
"logps/chosen": -480.01751708984375,
"logps/ref_chosen": -259.3636779785156,
"logps/ref_rejected": -279.30218505859375,
"logps/rejected": -611.1719970703125,
"loss": 4.8279,
"margin_dpo/margin_mean": 111.21595764160156,
"margin_dpo/margin_std": 205.4509735107422,
"step": 292
},
{
"KL/chosen_KL_mean": -248.36492919921875,
"KL/mean": -304.3353576660156,
"KL/rejected_KL_mean": -360.3057861328125,
"KL/std": 182.17123413085938,
"epoch": 0.6136125654450262,
"fcm_dpo/beta": 0.0022749018389731646,
"fcm_dpo/delta": -0.007784893270581961,
"fcm_dpo/margin": 111.94085693359375,
"fcm_dpo/q_t": 0.4398689568042755,
"grad_norm": 80.19559478759766,
"learning_rate": 1.9641153536023642e-07,
"logits/chosen": -0.9110856652259827,
"logits/rejected": -0.871292233467102,
"logps/chosen": -552.1357421875,
"logps/ref_chosen": -303.77081298828125,
"logps/ref_rejected": -270.07513427734375,
"logps/rejected": -630.3809204101562,
"loss": 4.782,
"margin_dpo/margin_mean": 111.94085693359375,
"margin_dpo/margin_std": 186.9821014404297,
"step": 293
},
{
"KL/chosen_KL_mean": -241.12615966796875,
"KL/mean": -287.281494140625,
"KL/rejected_KL_mean": -333.4368591308594,
"KL/std": 172.89413452148438,
"epoch": 0.6157068062827226,
"fcm_dpo/beta": 0.002252732403576374,
"fcm_dpo/delta": -0.025135207921266556,
"fcm_dpo/margin": 92.31072235107422,
"fcm_dpo/q_t": 0.4520157277584076,
"grad_norm": 46.71921157836914,
"learning_rate": 1.9462477745619106e-07,
"logits/chosen": -0.8031244277954102,
"logits/rejected": -0.8073158264160156,
"logps/chosen": -481.364501953125,
"logps/ref_chosen": -240.23831176757812,
"logps/ref_rejected": -229.187744140625,
"logps/rejected": -562.6246337890625,
"loss": 5.0051,
"margin_dpo/margin_mean": 92.31072235107422,
"margin_dpo/margin_std": 210.6240234375,
"step": 294
},
{
"KL/chosen_KL_mean": -238.98159790039062,
"KL/mean": -284.9803771972656,
"KL/rejected_KL_mean": -330.9790954589844,
"KL/std": 149.19898986816406,
"epoch": 0.6178010471204188,
"fcm_dpo/beta": 0.002246445743367076,
"fcm_dpo/delta": 0.018294546753168106,
"fcm_dpo/margin": 91.99751281738281,
"fcm_dpo/q_t": 0.4525008201599121,
"grad_norm": 48.98105239868164,
"learning_rate": 1.928409891572757e-07,
"logits/chosen": -0.8140034675598145,
"logits/rejected": -0.8149075508117676,
"logps/chosen": -489.99127197265625,
"logps/ref_chosen": -251.00970458984375,
"logps/ref_rejected": -244.15142822265625,
"logps/rejected": -575.1305541992188,
"loss": 4.9526,
"margin_dpo/margin_mean": 91.99751281738281,
"margin_dpo/margin_std": 192.78147888183594,
"step": 295
},
{
"KL/chosen_KL_mean": -246.5187530517578,
"KL/mean": -312.3707580566406,
"KL/rejected_KL_mean": -378.2227478027344,
"KL/std": 179.55752563476562,
"epoch": 0.6198952879581152,
"fcm_dpo/beta": 0.002214499283581972,
"fcm_dpo/delta": -0.0431789830327034,
"fcm_dpo/margin": 131.70401000976562,
"fcm_dpo/q_t": 0.43224072456359863,
"grad_norm": 54.046234130859375,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": -0.8091863989830017,
"logits/rejected": -0.774419903755188,
"logps/chosen": -540.3992919921875,
"logps/ref_chosen": -293.880615234375,
"logps/ref_rejected": -283.4175720214844,
"logps/rejected": -661.640380859375,
"loss": 4.7058,
"margin_dpo/margin_mean": 131.70401000976562,
"margin_dpo/margin_std": 217.57437133789062,
"step": 296
},
{
"KL/chosen_KL_mean": -244.05921936035156,
"KL/mean": -290.41455078125,
"KL/rejected_KL_mean": -336.7698669433594,
"KL/std": 159.8158721923828,
"epoch": 0.6219895287958115,
"fcm_dpo/beta": 0.0021560671739280224,
"fcm_dpo/delta": -0.01855265535414219,
"fcm_dpo/margin": 92.71065521240234,
"fcm_dpo/q_t": 0.45290887355804443,
"grad_norm": 45.659339904785156,
"learning_rate": 1.8928270384706582e-07,
"logits/chosen": -0.8852605819702148,
"logits/rejected": -0.8780304193496704,
"logps/chosen": -533.519287109375,
"logps/ref_chosen": -289.4600830078125,
"logps/ref_rejected": -283.69110107421875,
"logps/rejected": -620.4609375,
"loss": 4.9569,
"margin_dpo/margin_mean": 92.71065521240234,
"margin_dpo/margin_std": 185.48695373535156,
"step": 297
},
{
"KL/chosen_KL_mean": -233.01348876953125,
"KL/mean": -288.38299560546875,
"KL/rejected_KL_mean": -343.7524719238281,
"KL/std": 172.53945922851562,
"epoch": 0.6240837696335079,
"fcm_dpo/beta": 0.002129694912582636,
"fcm_dpo/delta": -0.02039862424135208,
"fcm_dpo/margin": 110.73900604248047,
"fcm_dpo/q_t": 0.4452170133590698,
"grad_norm": 39.55707931518555,
"learning_rate": 1.875083976558136e-07,
"logits/chosen": -0.8103281259536743,
"logits/rejected": -0.7954323887825012,
"logps/chosen": -539.5285034179688,
"logps/ref_chosen": -306.5150146484375,
"logps/ref_rejected": -280.6969909667969,
"logps/rejected": -624.449462890625,
"loss": 4.884,
"margin_dpo/margin_mean": 110.73900604248047,
"margin_dpo/margin_std": 216.20150756835938,
"step": 298
},
{
"KL/chosen_KL_mean": -272.2557373046875,
"KL/mean": -314.5762634277344,
"KL/rejected_KL_mean": -356.8967590332031,
"KL/std": 171.6256103515625,
"epoch": 0.6261780104712041,
"fcm_dpo/beta": 0.0021101348102092743,
"fcm_dpo/delta": -0.007379084825515747,
"fcm_dpo/margin": 84.64105224609375,
"fcm_dpo/q_t": 0.45750677585601807,
"grad_norm": 48.66347122192383,
"learning_rate": 1.8573744269954297e-07,
"logits/chosen": -0.8078640699386597,
"logits/rejected": -0.7908865809440613,
"logps/chosen": -553.6195678710938,
"logps/ref_chosen": -281.36376953125,
"logps/ref_rejected": -270.39508056640625,
"logps/rejected": -627.2918701171875,
"loss": 5.0207,
"margin_dpo/margin_mean": 84.64105224609375,
"margin_dpo/margin_std": 174.2266082763672,
"step": 299
},
{
"KL/chosen_KL_mean": -270.9239501953125,
"KL/mean": -316.153564453125,
"KL/rejected_KL_mean": -361.38323974609375,
"KL/std": 182.1024627685547,
"epoch": 0.6282722513089005,
"fcm_dpo/beta": 0.002133725443854928,
"fcm_dpo/delta": 0.029630564153194427,
"fcm_dpo/margin": 90.45928192138672,
"fcm_dpo/q_t": 0.4544250965118408,
"grad_norm": 60.00415802001953,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": -0.8176003694534302,
"logits/rejected": -0.7979527115821838,
"logps/chosen": -585.7597045898438,
"logps/ref_chosen": -314.83575439453125,
"logps/ref_rejected": -269.1154479980469,
"logps/rejected": -630.4987182617188,
"loss": 5.0057,
"margin_dpo/margin_mean": 90.45928192138672,
"margin_dpo/margin_std": 204.20758056640625,
"step": 300
},
{
"KL/chosen_KL_mean": -255.38441467285156,
"KL/mean": -300.5905456542969,
"KL/rejected_KL_mean": -345.79669189453125,
"KL/std": 168.21604919433594,
"epoch": 0.6303664921465969,
"fcm_dpo/beta": 0.0021687678527086973,
"fcm_dpo/delta": 0.005485248751938343,
"fcm_dpo/margin": 90.41228485107422,
"fcm_dpo/q_t": 0.45402035117149353,
"grad_norm": 65.89996337890625,
"learning_rate": 1.8220596619089573e-07,
"logits/chosen": -0.8691757321357727,
"logits/rejected": -0.8592531681060791,
"logps/chosen": -535.2789306640625,
"logps/ref_chosen": -279.89453125,
"logps/ref_rejected": -271.6694641113281,
"logps/rejected": -617.4661865234375,
"loss": 4.9779,
"margin_dpo/margin_mean": 90.41228485107422,
"margin_dpo/margin_std": 187.416748046875,
"step": 301
},
{
"KL/chosen_KL_mean": -254.57908630371094,
"KL/mean": -314.37725830078125,
"KL/rejected_KL_mean": -374.1754150390625,
"KL/std": 175.4994659423828,
"epoch": 0.6324607329842932,
"fcm_dpo/beta": 0.002159257186576724,
"fcm_dpo/delta": -0.00899820402264595,
"fcm_dpo/margin": 119.59630584716797,
"fcm_dpo/q_t": 0.439103364944458,
"grad_norm": 77.14640808105469,
"learning_rate": 1.8044563402088682e-07,
"logits/chosen": -0.8205274343490601,
"logits/rejected": -0.8089447617530823,
"logps/chosen": -525.910888671875,
"logps/ref_chosen": -271.3318176269531,
"logps/ref_rejected": -256.5587158203125,
"logps/rejected": -630.734130859375,
"loss": 4.7509,
"margin_dpo/margin_mean": 119.59629821777344,
"margin_dpo/margin_std": 186.8571319580078,
"step": 302
},
{
"KL/chosen_KL_mean": -256.7622375488281,
"KL/mean": -310.6941833496094,
"KL/rejected_KL_mean": -364.6260986328125,
"KL/std": 153.02818298339844,
"epoch": 0.6345549738219896,
"fcm_dpo/beta": 0.002148208674043417,
"fcm_dpo/delta": -0.004830796271562576,
"fcm_dpo/margin": 107.8638916015625,
"fcm_dpo/q_t": 0.44455963373184204,
"grad_norm": 52.09537124633789,
"learning_rate": 1.7868903184043885e-07,
"logits/chosen": -0.8106588125228882,
"logits/rejected": -0.7836918830871582,
"logps/chosen": -561.6432495117188,
"logps/ref_chosen": -304.88104248046875,
"logps/ref_rejected": -269.063720703125,
"logps/rejected": -633.6898803710938,
"loss": 4.8106,
"margin_dpo/margin_mean": 107.8638916015625,
"margin_dpo/margin_std": 165.25503540039062,
"step": 303
},
{
"KL/chosen_KL_mean": -276.5693359375,
"KL/mean": -339.6380920410156,
"KL/rejected_KL_mean": -402.7069091796875,
"KL/std": 195.83033752441406,
"epoch": 0.6366492146596858,
"fcm_dpo/beta": 0.0021316506899893284,
"fcm_dpo/delta": 0.0033886905293911695,
"fcm_dpo/margin": 126.1375732421875,
"fcm_dpo/q_t": 0.438024640083313,
"grad_norm": 60.87114334106445,
"learning_rate": 1.7693625385079574e-07,
"logits/chosen": -0.8092816472053528,
"logits/rejected": -0.813895046710968,
"logps/chosen": -567.2802734375,
"logps/ref_chosen": -290.7109680175781,
"logps/ref_rejected": -237.6885986328125,
"logps/rejected": -640.3955688476562,
"loss": 4.757,
"margin_dpo/margin_mean": 126.13758850097656,
"margin_dpo/margin_std": 216.04942321777344,
"step": 304
},
{
"KL/chosen_KL_mean": -237.2429656982422,
"KL/mean": -328.9360046386719,
"KL/rejected_KL_mean": -420.62908935546875,
"KL/std": 190.5575714111328,
"epoch": 0.6387434554973822,
"fcm_dpo/beta": 0.0020590811036527157,
"fcm_dpo/delta": -0.10773821175098419,
"fcm_dpo/margin": 183.38613891601562,
"fcm_dpo/q_t": 0.41207101941108704,
"grad_norm": 81.0551528930664,
"learning_rate": 1.7518739404812155e-07,
"logits/chosen": -0.8565566539764404,
"logits/rejected": -0.8202612996101379,
"logps/chosen": -493.7269592285156,
"logps/ref_chosen": -256.4839782714844,
"logps/ref_rejected": -266.4063415527344,
"logps/rejected": -687.035400390625,
"loss": 4.3971,
"margin_dpo/margin_mean": 183.38613891601562,
"margin_dpo/margin_std": 223.84494018554688,
"step": 305
},
{
"KL/chosen_KL_mean": -277.597900390625,
"KL/mean": -328.07940673828125,
"KL/rejected_KL_mean": -378.5609130859375,
"KL/std": 166.4034423828125,
"epoch": 0.6408376963350786,
"fcm_dpo/beta": 0.001981166424229741,
"fcm_dpo/delta": 0.015996161848306656,
"fcm_dpo/margin": 100.9630355834961,
"fcm_dpo/q_t": 0.45244964957237244,
"grad_norm": 73.6423568725586,
"learning_rate": 1.7344254621846017e-07,
"logits/chosen": -0.8491054177284241,
"logits/rejected": -0.8253453969955444,
"logps/chosen": -598.2471313476562,
"logps/ref_chosen": -320.6492004394531,
"logps/ref_rejected": -273.36773681640625,
"logps/rejected": -651.9286499023438,
"loss": 4.9412,
"margin_dpo/margin_mean": 100.9630355834961,
"margin_dpo/margin_std": 185.91299438476562,
"step": 306
},
{
"KL/chosen_KL_mean": -311.2239074707031,
"KL/mean": -363.1423645019531,
"KL/rejected_KL_mean": -415.060791015625,
"KL/std": 186.05921936035156,
"epoch": 0.6429319371727749,
"fcm_dpo/beta": 0.00200508045963943,
"fcm_dpo/delta": 0.010745943523943424,
"fcm_dpo/margin": 103.8368911743164,
"fcm_dpo/q_t": 0.44979315996170044,
"grad_norm": 51.963165283203125,
"learning_rate": 1.717018039327053e-07,
"logits/chosen": -0.7687966227531433,
"logits/rejected": -0.781086266040802,
"logps/chosen": -590.6781005859375,
"logps/ref_chosen": -279.4541931152344,
"logps/ref_rejected": -240.3796844482422,
"logps/rejected": -655.4404907226562,
"loss": 4.8875,
"margin_dpo/margin_mean": 103.83688354492188,
"margin_dpo/margin_std": 176.36830139160156,
"step": 307
},
{
"KL/chosen_KL_mean": -348.136474609375,
"KL/mean": -388.1585693359375,
"KL/rejected_KL_mean": -428.1806335449219,
"KL/std": 201.38768005371094,
"epoch": 0.6450261780104712,
"fcm_dpo/beta": 0.0020293970592319965,
"fcm_dpo/delta": 0.011533215641975403,
"fcm_dpo/margin": 80.04412841796875,
"fcm_dpo/q_t": 0.46260982751846313,
"grad_norm": 68.01890563964844,
"learning_rate": 1.699652605415828e-07,
"logits/chosen": -0.8337502479553223,
"logits/rejected": -0.84173983335495,
"logps/chosen": -644.73486328125,
"logps/ref_chosen": -296.598388671875,
"logps/ref_rejected": -258.6953430175781,
"logps/rejected": -686.8759765625,
"loss": 5.1313,
"margin_dpo/margin_mean": 80.04411315917969,
"margin_dpo/margin_std": 218.29266357421875,
"step": 308
},
{
"KL/chosen_KL_mean": -329.8775939941406,
"KL/mean": -393.9901123046875,
"KL/rejected_KL_mean": -458.1026916503906,
"KL/std": 194.3191375732422,
"epoch": 0.6471204188481675,
"fcm_dpo/beta": 0.0020433913450688124,
"fcm_dpo/delta": 0.0097114909440279,
"fcm_dpo/margin": 128.22503662109375,
"fcm_dpo/q_t": 0.43997424840927124,
"grad_norm": 47.546016693115234,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": -0.823945164680481,
"logits/rejected": -0.8316885232925415,
"logps/chosen": -611.2657470703125,
"logps/ref_chosen": -281.3881530761719,
"logps/ref_rejected": -262.458740234375,
"logps/rejected": -720.5614013671875,
"loss": 4.7889,
"margin_dpo/margin_mean": 128.2250518798828,
"margin_dpo/margin_std": 234.49032592773438,
"step": 309
},
{
"KL/chosen_KL_mean": -325.3269348144531,
"KL/mean": -389.1058044433594,
"KL/rejected_KL_mean": -452.8846130371094,
"KL/std": 203.00721740722656,
"epoch": 0.6492146596858639,
"fcm_dpo/beta": 0.002034237841144204,
"fcm_dpo/delta": -0.013699397444725037,
"fcm_dpo/margin": 127.55763244628906,
"fcm_dpo/q_t": 0.44047310948371887,
"grad_norm": 61.21415710449219,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": -0.8205724954605103,
"logits/rejected": -0.7908133864402771,
"logps/chosen": -604.5142211914062,
"logps/ref_chosen": -279.1872863769531,
"logps/ref_rejected": -261.8279724121094,
"logps/rejected": -714.7125244140625,
"loss": 4.7957,
"margin_dpo/margin_mean": 127.5576171875,
"margin_dpo/margin_std": 224.91964721679688,
"step": 310
},
{
"KL/chosen_KL_mean": -325.9942321777344,
"KL/mean": -393.7972412109375,
"KL/rejected_KL_mean": -461.60028076171875,
"KL/std": 239.32907104492188,
"epoch": 0.6513089005235602,
"fcm_dpo/beta": 0.0019492747960612178,
"fcm_dpo/delta": -0.049720633774995804,
"fcm_dpo/margin": 135.60604858398438,
"fcm_dpo/q_t": 0.44296321272850037,
"grad_norm": 53.1006965637207,
"learning_rate": 1.647817538357072e-07,
"logits/chosen": -0.806901216506958,
"logits/rejected": -0.7805233001708984,
"logps/chosen": -597.3923950195312,
"logps/ref_chosen": -271.39813232421875,
"logps/ref_rejected": -266.12701416015625,
"logps/rejected": -727.727294921875,
"loss": 4.8429,
"margin_dpo/margin_mean": 135.6060791015625,
"margin_dpo/margin_std": 269.9830627441406,
"step": 311
},
{
"KL/chosen_KL_mean": -323.4100341796875,
"KL/mean": -384.630126953125,
"KL/rejected_KL_mean": -445.8502197265625,
"KL/std": 218.39796447753906,
"epoch": 0.6534031413612565,
"fcm_dpo/beta": 0.0019210537429898977,
"fcm_dpo/delta": -0.01848769001662731,
"fcm_dpo/margin": 122.44014739990234,
"fcm_dpo/q_t": 0.44824573397636414,
"grad_norm": 80.48198699951172,
"learning_rate": 1.6306293495205755e-07,
"logits/chosen": -0.8279159665107727,
"logits/rejected": -0.7945848703384399,
"logps/chosen": -605.7950439453125,
"logps/ref_chosen": -282.3850402832031,
"logps/ref_rejected": -246.35389709472656,
"logps/rejected": -692.2041015625,
"loss": 4.9828,
"margin_dpo/margin_mean": 122.44013977050781,
"margin_dpo/margin_std": 290.7316589355469,
"step": 312
},
{
"KL/chosen_KL_mean": -329.6253662109375,
"KL/mean": -385.30535888671875,
"KL/rejected_KL_mean": -440.98541259765625,
"KL/std": 222.88763427734375,
"epoch": 0.6554973821989529,
"fcm_dpo/beta": 0.0019223717972636223,
"fcm_dpo/delta": -0.002858624327927828,
"fcm_dpo/margin": 111.36000061035156,
"fcm_dpo/q_t": 0.4549371302127838,
"grad_norm": 56.645931243896484,
"learning_rate": 1.6134877823936607e-07,
"logits/chosen": -0.8647723197937012,
"logits/rejected": -0.8453179001808167,
"logps/chosen": -633.2562255859375,
"logps/ref_chosen": -303.630859375,
"logps/ref_rejected": -273.1156921386719,
"logps/rejected": -714.10107421875,
"loss": 5.0238,
"margin_dpo/margin_mean": 111.36000061035156,
"margin_dpo/margin_std": 269.7938537597656,
"step": 313
},
{
"KL/chosen_KL_mean": -324.699462890625,
"KL/mean": -377.6182556152344,
"KL/rejected_KL_mean": -430.53704833984375,
"KL/std": 193.24305725097656,
"epoch": 0.6575916230366492,
"fcm_dpo/beta": 0.0019222039263695478,
"fcm_dpo/delta": 0.01650712452828884,
"fcm_dpo/margin": 105.83755493164062,
"fcm_dpo/q_t": 0.45183467864990234,
"grad_norm": 50.479618072509766,
"learning_rate": 1.5963937562265522e-07,
"logits/chosen": -0.876418948173523,
"logits/rejected": -0.8731269836425781,
"logps/chosen": -627.0037231445312,
"logps/ref_chosen": -302.3042907714844,
"logps/ref_rejected": -273.6416015625,
"logps/rejected": -704.1786499023438,
"loss": 4.9631,
"margin_dpo/margin_mean": 105.83756256103516,
"margin_dpo/margin_std": 222.26353454589844,
"step": 314
},
{
"KL/chosen_KL_mean": -307.293212890625,
"KL/mean": -383.67724609375,
"KL/rejected_KL_mean": -460.061279296875,
"KL/std": 201.8465118408203,
"epoch": 0.6596858638743456,
"fcm_dpo/beta": 0.0019190021557733417,
"fcm_dpo/delta": -0.030157245695590973,
"fcm_dpo/margin": 152.7681121826172,
"fcm_dpo/q_t": 0.4318777322769165,
"grad_norm": 58.37327194213867,
"learning_rate": 1.5793481877199943e-07,
"logits/chosen": -0.8553957939147949,
"logits/rejected": -0.8330000042915344,
"logps/chosen": -610.0224609375,
"logps/ref_chosen": -302.729248046875,
"logps/ref_rejected": -270.26910400390625,
"logps/rejected": -730.3304443359375,
"loss": 4.6619,
"margin_dpo/margin_mean": 152.76809692382812,
"margin_dpo/margin_std": 228.41522216796875,
"step": 315
},
{
"KL/chosen_KL_mean": -319.1265869140625,
"KL/mean": -379.0169372558594,
"KL/rejected_KL_mean": -438.9073486328125,
"KL/std": 191.73924255371094,
"epoch": 0.6617801047120419,
"fcm_dpo/beta": 0.0019111887086182833,
"fcm_dpo/delta": -0.0019163042306900024,
"fcm_dpo/margin": 119.78076171875,
"fcm_dpo/q_t": 0.44601792097091675,
"grad_norm": 51.57410430908203,
"learning_rate": 1.562351990976095e-07,
"logits/chosen": -0.882705807685852,
"logits/rejected": -0.8593080043792725,
"logps/chosen": -629.6972045898438,
"logps/ref_chosen": -310.5706481933594,
"logps/ref_rejected": -272.9354553222656,
"logps/rejected": -711.8428344726562,
"loss": 4.8727,
"margin_dpo/margin_mean": 119.78076934814453,
"margin_dpo/margin_std": 223.42430114746094,
"step": 316
},
{
"KL/chosen_KL_mean": -270.0894775390625,
"KL/mean": -324.0589599609375,
"KL/rejected_KL_mean": -378.0285339355469,
"KL/std": 167.6583251953125,
"epoch": 0.6638743455497382,
"fcm_dpo/beta": 0.0019239266403019428,
"fcm_dpo/delta": 0.03239323943853378,
"fcm_dpo/margin": 107.93901824951172,
"fcm_dpo/q_t": 0.45031195878982544,
"grad_norm": 51.05884552001953,
"learning_rate": 1.5454060774493065e-07,
"logits/chosen": -0.8534022569656372,
"logits/rejected": -0.8216595649719238,
"logps/chosen": -523.9898681640625,
"logps/ref_chosen": -253.90036010742188,
"logps/ref_rejected": -218.74078369140625,
"logps/rejected": -596.769287109375,
"loss": 4.8902,
"margin_dpo/margin_mean": 107.93902587890625,
"margin_dpo/margin_std": 182.3351287841797,
"step": 317
},
{
"KL/chosen_KL_mean": -276.93927001953125,
"KL/mean": -353.7762451171875,
"KL/rejected_KL_mean": -430.61322021484375,
"KL/std": 195.12513732910156,
"epoch": 0.6659685863874345,
"fcm_dpo/beta": 0.0018967122305184603,
"fcm_dpo/delta": -0.02675527334213257,
"fcm_dpo/margin": 153.67393493652344,
"fcm_dpo/q_t": 0.43146854639053345,
"grad_norm": 60.334529876708984,
"learning_rate": 1.5285113558975427e-07,
"logits/chosen": -0.8724962472915649,
"logits/rejected": -0.8433178663253784,
"logps/chosen": -547.7621459960938,
"logps/ref_chosen": -270.8228759765625,
"logps/ref_rejected": -255.30972290039062,
"logps/rejected": -685.9229125976562,
"loss": 4.6496,
"margin_dpo/margin_mean": 153.67393493652344,
"margin_dpo/margin_std": 221.29598999023438,
"step": 318
},
{
"KL/chosen_KL_mean": -292.36883544921875,
"KL/mean": -358.0066833496094,
"KL/rejected_KL_mean": -423.64459228515625,
"KL/std": 185.0775909423828,
"epoch": 0.6680628272251309,
"fcm_dpo/beta": 0.0018959176959469914,
"fcm_dpo/delta": 0.0009724181145429611,
"fcm_dpo/margin": 131.27574157714844,
"fcm_dpo/q_t": 0.4395703971385956,
"grad_norm": 47.7817497253418,
"learning_rate": 1.5116687323334464e-07,
"logits/chosen": -0.8555124998092651,
"logits/rejected": -0.8220670819282532,
"logps/chosen": -593.3716430664062,
"logps/ref_chosen": -301.0028076171875,
"logps/ref_rejected": -242.39002990722656,
"logps/rejected": -666.0346069335938,
"loss": 4.7536,
"margin_dpo/margin_mean": 131.27574157714844,
"margin_dpo/margin_std": 191.61190795898438,
"step": 319
},
{
"KL/chosen_KL_mean": -291.9874267578125,
"KL/mean": -353.7680358886719,
"KL/rejected_KL_mean": -415.5486755371094,
"KL/std": 201.40379333496094,
"epoch": 0.6701570680628273,
"fcm_dpo/beta": 0.0018904343014582992,
"fcm_dpo/delta": -0.003565825056284666,
"fcm_dpo/margin": 123.561279296875,
"fcm_dpo/q_t": 0.44610100984573364,
"grad_norm": 71.67448425292969,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": -0.8239633440971375,
"logits/rejected": -0.8111226558685303,
"logps/chosen": -595.6099853515625,
"logps/ref_chosen": -303.6225891113281,
"logps/ref_rejected": -280.85174560546875,
"logps/rejected": -696.400390625,
"loss": 4.8644,
"margin_dpo/margin_mean": 123.561279296875,
"margin_dpo/margin_std": 236.85836791992188,
"step": 320
},
{
"KL/chosen_KL_mean": -319.300537109375,
"KL/mean": -363.65484619140625,
"KL/rejected_KL_mean": -408.0091857910156,
"KL/std": 196.95912170410156,
"epoch": 0.6722513089005235,
"fcm_dpo/beta": 0.0019133113091811538,
"fcm_dpo/delta": 0.015708304941654205,
"fcm_dpo/margin": 88.7086181640625,
"fcm_dpo/q_t": 0.4608987867832184,
"grad_norm": 51.643287658691406,
"learning_rate": 1.478143389201113e-07,
"logits/chosen": -0.8439365029335022,
"logits/rejected": -0.811326801776886,
"logps/chosen": -608.286376953125,
"logps/ref_chosen": -288.98583984375,
"logps/ref_rejected": -241.1822052001953,
"logps/rejected": -649.1913452148438,
"loss": 5.0942,
"margin_dpo/margin_mean": 88.7086181640625,
"margin_dpo/margin_std": 222.06124877929688,
"step": 321
},
{
"KL/chosen_KL_mean": -333.3482666015625,
"KL/mean": -380.2379455566406,
"KL/rejected_KL_mean": -427.12762451171875,
"KL/std": 206.414306640625,
"epoch": 0.6743455497382199,
"fcm_dpo/beta": 0.001928423997014761,
"fcm_dpo/delta": 0.017177335917949677,
"fcm_dpo/margin": 93.77931213378906,
"fcm_dpo/q_t": 0.4577179551124573,
"grad_norm": 72.54539489746094,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": -0.8990727066993713,
"logits/rejected": -0.865135908126831,
"logps/chosen": -641.8917846679688,
"logps/ref_chosen": -308.54345703125,
"logps/ref_rejected": -269.7995910644531,
"logps/rejected": -696.9271850585938,
"loss": 5.0935,
"margin_dpo/margin_mean": 93.77930450439453,
"margin_dpo/margin_std": 251.90206909179688,
"step": 322
},
{
"KL/chosen_KL_mean": -305.1156311035156,
"KL/mean": -348.0940856933594,
"KL/rejected_KL_mean": -391.07257080078125,
"KL/std": 183.42276000976562,
"epoch": 0.6764397905759162,
"fcm_dpo/beta": 0.0019287847680971026,
"fcm_dpo/delta": -0.01517847552895546,
"fcm_dpo/margin": 85.95698547363281,
"fcm_dpo/q_t": 0.46190139651298523,
"grad_norm": 76.46353149414062,
"learning_rate": 1.4448372394055246e-07,
"logits/chosen": -0.8569475412368774,
"logits/rejected": -0.8476258516311646,
"logps/chosen": -587.6092529296875,
"logps/ref_chosen": -282.49365234375,
"logps/ref_rejected": -227.7105255126953,
"logps/rejected": -618.7831420898438,
"loss": 5.1093,
"margin_dpo/margin_mean": 85.95698547363281,
"margin_dpo/margin_std": 220.92446899414062,
"step": 323
},
{
"KL/chosen_KL_mean": -270.2607421875,
"KL/mean": -339.736328125,
"KL/rejected_KL_mean": -409.2118835449219,
"KL/std": 195.49070739746094,
"epoch": 0.6785340314136126,
"fcm_dpo/beta": 0.0018783288542181253,
"fcm_dpo/delta": -0.012117970734834671,
"fcm_dpo/margin": 138.95114135742188,
"fcm_dpo/q_t": 0.4381677508354187,
"grad_norm": 49.30532455444336,
"learning_rate": 1.428268596492364e-07,
"logits/chosen": -0.7851729989051819,
"logits/rejected": -0.7778810262680054,
"logps/chosen": -509.59912109375,
"logps/ref_chosen": -239.33836364746094,
"logps/ref_rejected": -230.53775024414062,
"logps/rejected": -639.7496337890625,
"loss": 4.7328,
"margin_dpo/margin_mean": 138.95114135742188,
"margin_dpo/margin_std": 205.01913452148438,
"step": 324
},
{
"KL/chosen_KL_mean": -310.2890625,
"KL/mean": -369.69451904296875,
"KL/rejected_KL_mean": -429.0999755859375,
"KL/std": 213.21319580078125,
"epoch": 0.680628272251309,
"fcm_dpo/beta": 0.0018853824585676193,
"fcm_dpo/delta": -0.007832746021449566,
"fcm_dpo/margin": 118.81090545654297,
"fcm_dpo/q_t": 0.4491155743598938,
"grad_norm": 67.0599594116211,
"learning_rate": 1.4117574272818386e-07,
"logits/chosen": -0.7914143204689026,
"logits/rejected": -0.7728296518325806,
"logps/chosen": -590.9180297851562,
"logps/ref_chosen": -280.62896728515625,
"logps/ref_rejected": -270.5085754394531,
"logps/rejected": -699.6085205078125,
"loss": 4.9665,
"margin_dpo/margin_mean": 118.8109130859375,
"margin_dpo/margin_std": 269.921142578125,
"step": 325
},
{
"KL/chosen_KL_mean": -282.23529052734375,
"KL/mean": -345.9592590332031,
"KL/rejected_KL_mean": -409.68316650390625,
"KL/std": 194.88714599609375,
"epoch": 0.6827225130890052,
"fcm_dpo/beta": 0.0019169156439602375,
"fcm_dpo/delta": 0.019461024552583694,
"fcm_dpo/margin": 127.44788360595703,
"fcm_dpo/q_t": 0.44288212060928345,
"grad_norm": 94.46167755126953,
"learning_rate": 1.3953046172178413e-07,
"logits/chosen": -0.9129518866539001,
"logits/rejected": -0.9026789665222168,
"logps/chosen": -523.222412109375,
"logps/ref_chosen": -240.9871368408203,
"logps/ref_rejected": -261.0238342285156,
"logps/rejected": -670.7069702148438,
"loss": 4.8168,
"margin_dpo/margin_mean": 127.44788360595703,
"margin_dpo/margin_std": 225.53358459472656,
"step": 326
},
{
"KL/chosen_KL_mean": -272.2273254394531,
"KL/mean": -343.879150390625,
"KL/rejected_KL_mean": -415.531005859375,
"KL/std": 187.35333251953125,
"epoch": 0.6848167539267016,
"fcm_dpo/beta": 0.0019026086665689945,
"fcm_dpo/delta": -0.026806041598320007,
"fcm_dpo/margin": 143.3036651611328,
"fcm_dpo/q_t": 0.43724074959754944,
"grad_norm": 67.36668395996094,
"learning_rate": 1.3789110486146468e-07,
"logits/chosen": -0.8426380753517151,
"logits/rejected": -0.8181723952293396,
"logps/chosen": -551.747314453125,
"logps/ref_chosen": -279.52001953125,
"logps/ref_rejected": -269.51824951171875,
"logps/rejected": -685.0491943359375,
"loss": 4.7248,
"margin_dpo/margin_mean": 143.3036651611328,
"margin_dpo/margin_std": 221.62916564941406,
"step": 327
},
{
"KL/chosen_KL_mean": -292.79803466796875,
"KL/mean": -357.90460205078125,
"KL/rejected_KL_mean": -423.01116943359375,
"KL/std": 195.24517822265625,
"epoch": 0.6869109947643979,
"fcm_dpo/beta": 0.0018517444841563702,
"fcm_dpo/delta": -0.01919415406882763,
"fcm_dpo/margin": 130.21310424804688,
"fcm_dpo/q_t": 0.4430963397026062,
"grad_norm": 58.3312873840332,
"learning_rate": 1.362577600609588e-07,
"logits/chosen": -0.8256867527961731,
"logits/rejected": -0.8203503489494324,
"logps/chosen": -593.8314819335938,
"logps/ref_chosen": -301.033447265625,
"logps/ref_rejected": -284.2101135253906,
"logps/rejected": -707.2212524414062,
"loss": 4.822,
"margin_dpo/margin_mean": 130.21310424804688,
"margin_dpo/margin_std": 222.81100463867188,
"step": 328
},
{
"KL/chosen_KL_mean": -317.37786865234375,
"KL/mean": -394.31024169921875,
"KL/rejected_KL_mean": -471.24267578125,
"KL/std": 254.76437377929688,
"epoch": 0.6890052356020943,
"fcm_dpo/beta": 0.0018192044226452708,
"fcm_dpo/delta": -0.03148316219449043,
"fcm_dpo/margin": 153.8647918701172,
"fcm_dpo/q_t": 0.43870800733566284,
"grad_norm": 66.76805114746094,
"learning_rate": 1.3463051491159093e-07,
"logits/chosen": -0.8388152122497559,
"logits/rejected": -0.7993863821029663,
"logps/chosen": -637.3667602539062,
"logps/ref_chosen": -319.9888610839844,
"logps/ref_rejected": -307.5588684082031,
"logps/rejected": -778.8015747070312,
"loss": 4.7845,
"margin_dpo/margin_mean": 153.8647918701172,
"margin_dpo/margin_std": 286.5224304199219,
"step": 329
},
{
"KL/chosen_KL_mean": -318.2249755859375,
"KL/mean": -386.0837707519531,
"KL/rejected_KL_mean": -453.9425048828125,
"KL/std": 222.40377807617188,
"epoch": 0.6910994764397905,
"fcm_dpo/beta": 0.0017876912606880069,
"fcm_dpo/delta": -0.022983934730291367,
"fcm_dpo/margin": 135.717529296875,
"fcm_dpo/q_t": 0.444413959980011,
"grad_norm": 64.91725158691406,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": -0.8319533467292786,
"logits/rejected": -0.8325479030609131,
"logps/chosen": -619.3397216796875,
"logps/ref_chosen": -301.11474609375,
"logps/ref_rejected": -299.673095703125,
"logps/rejected": -753.6156005859375,
"loss": 4.8354,
"margin_dpo/margin_mean": 135.71754455566406,
"margin_dpo/margin_std": 240.82728576660156,
"step": 330
},
{
"KL/chosen_KL_mean": -303.1584777832031,
"KL/mean": -362.89227294921875,
"KL/rejected_KL_mean": -422.6261291503906,
"KL/std": 216.93350219726562,
"epoch": 0.6931937172774869,
"fcm_dpo/beta": 0.001791123067960143,
"fcm_dpo/delta": 0.03097469173371792,
"fcm_dpo/margin": 119.4676513671875,
"fcm_dpo/q_t": 0.45106902718544006,
"grad_norm": 50.95379638671875,
"learning_rate": 1.3139467229135998e-07,
"logits/chosen": -0.8472961187362671,
"logits/rejected": -0.8238837718963623,
"logps/chosen": -580.7499389648438,
"logps/ref_chosen": -277.59149169921875,
"logps/ref_rejected": -256.025634765625,
"logps/rejected": -678.6517944335938,
"loss": 4.963,
"margin_dpo/margin_mean": 119.4676513671875,
"margin_dpo/margin_std": 267.2186584472656,
"step": 331
},
{
"KL/chosen_KL_mean": -322.1370849609375,
"KL/mean": -387.9303894042969,
"KL/rejected_KL_mean": -453.72369384765625,
"KL/std": 211.57984924316406,
"epoch": 0.6952879581151833,
"fcm_dpo/beta": 0.001803787425160408,
"fcm_dpo/delta": -0.002744631841778755,
"fcm_dpo/margin": 131.58660888671875,
"fcm_dpo/q_t": 0.44477543234825134,
"grad_norm": 52.000823974609375,
"learning_rate": 1.2978624834891626e-07,
"logits/chosen": -0.834603488445282,
"logits/rejected": -0.8034043908119202,
"logps/chosen": -592.11083984375,
"logps/ref_chosen": -269.97369384765625,
"logps/ref_rejected": -235.03164672851562,
"logps/rejected": -688.75537109375,
"loss": 4.911,
"margin_dpo/margin_mean": 131.58660888671875,
"margin_dpo/margin_std": 275.0579833984375,
"step": 332
},
{
"KL/chosen_KL_mean": -320.29888916015625,
"KL/mean": -371.5259094238281,
"KL/rejected_KL_mean": -422.75299072265625,
"KL/std": 204.25921630859375,
"epoch": 0.6973821989528796,
"fcm_dpo/beta": 0.0018104577902704477,
"fcm_dpo/delta": 0.011496592313051224,
"fcm_dpo/margin": 102.45411682128906,
"fcm_dpo/q_t": 0.4570245146751404,
"grad_norm": 61.41099548339844,
"learning_rate": 1.281842711051438e-07,
"logits/chosen": -0.9206676483154297,
"logits/rejected": -0.8802468776702881,
"logps/chosen": -617.0618896484375,
"logps/ref_chosen": -296.76300048828125,
"logps/ref_rejected": -265.97991943359375,
"logps/rejected": -688.7329711914062,
"loss": 5.0439,
"margin_dpo/margin_mean": 102.45410919189453,
"margin_dpo/margin_std": 249.8954620361328,
"step": 333
},
{
"KL/chosen_KL_mean": -309.74859619140625,
"KL/mean": -369.7897033691406,
"KL/rejected_KL_mean": -429.8308410644531,
"KL/std": 202.07411193847656,
"epoch": 0.6994764397905759,
"fcm_dpo/beta": 0.0018201847560703754,
"fcm_dpo/delta": -0.018839359283447266,
"fcm_dpo/margin": 120.08220672607422,
"fcm_dpo/q_t": 0.44838762283325195,
"grad_norm": 74.61259460449219,
"learning_rate": 1.2658882646922033e-07,
"logits/chosen": -0.8214524984359741,
"logits/rejected": -0.7901473045349121,
"logps/chosen": -610.785400390625,
"logps/ref_chosen": -301.0367431640625,
"logps/ref_rejected": -268.87652587890625,
"logps/rejected": -698.7073364257812,
"loss": 4.9181,
"margin_dpo/margin_mean": 120.08220672607422,
"margin_dpo/margin_std": 236.2801055908203,
"step": 334
},
{
"KL/chosen_KL_mean": -325.9380187988281,
"KL/mean": -387.39630126953125,
"KL/rejected_KL_mean": -448.8546447753906,
"KL/std": 182.33425903320312,
"epoch": 0.7015706806282722,
"fcm_dpo/beta": 0.0017861748347058892,
"fcm_dpo/delta": -0.0105055533349514,
"fcm_dpo/margin": 122.9166259765625,
"fcm_dpo/q_t": 0.44840696454048157,
"grad_norm": 78.46752166748047,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.8165170550346375,
"logits/rejected": -0.8090022802352905,
"logps/chosen": -602.0708618164062,
"logps/ref_chosen": -276.13275146484375,
"logps/ref_rejected": -243.44203186035156,
"logps/rejected": -692.296630859375,
"loss": 4.8941,
"margin_dpo/margin_mean": 122.9166259765625,
"margin_dpo/margin_std": 230.23648071289062,
"step": 335
},
{
"KL/chosen_KL_mean": -330.6165771484375,
"KL/mean": -392.3404846191406,
"KL/rejected_KL_mean": -454.0643310546875,
"KL/std": 247.4468994140625,
"epoch": 0.7036649214659686,
"fcm_dpo/beta": 0.0017896379576995969,
"fcm_dpo/delta": 0.007889148779213428,
"fcm_dpo/margin": 123.44775390625,
"fcm_dpo/q_t": 0.4509285092353821,
"grad_norm": 63.69434356689453,
"learning_rate": 1.2341787690142435e-07,
"logits/chosen": -0.8447047472000122,
"logits/rejected": -0.7878402471542358,
"logps/chosen": -576.8792724609375,
"logps/ref_chosen": -246.2626495361328,
"logps/ref_rejected": -261.0617980957031,
"logps/rejected": -715.126220703125,
"loss": 4.9596,
"margin_dpo/margin_mean": 123.44773864746094,
"margin_dpo/margin_std": 272.46478271484375,
"step": 336
},
{
"KL/chosen_KL_mean": -309.1364440917969,
"KL/mean": -379.6464538574219,
"KL/rejected_KL_mean": -450.1564636230469,
"KL/std": 201.87640380859375,
"epoch": 0.7057591623036649,
"fcm_dpo/beta": 0.0017988713225349784,
"fcm_dpo/delta": -0.01950749382376671,
"fcm_dpo/margin": 141.02003479003906,
"fcm_dpo/q_t": 0.4397309422492981,
"grad_norm": 67.87557220458984,
"learning_rate": 1.2184254201795363e-07,
"logits/chosen": -0.8472563028335571,
"logits/rejected": -0.8143435120582581,
"logps/chosen": -576.1302490234375,
"logps/ref_chosen": -266.9937744140625,
"logps/ref_rejected": -253.015625,
"logps/rejected": -703.172119140625,
"loss": 4.7793,
"margin_dpo/margin_mean": 141.02003479003906,
"margin_dpo/margin_std": 226.84255981445312,
"step": 337
},
{
"KL/chosen_KL_mean": -302.8250427246094,
"KL/mean": -373.4712219238281,
"KL/rejected_KL_mean": -444.117431640625,
"KL/std": 216.23487854003906,
"epoch": 0.7078534031413612,
"fcm_dpo/beta": 0.0017496687360107899,
"fcm_dpo/delta": -0.004362808540463448,
"fcm_dpo/margin": 141.2923583984375,
"fcm_dpo/q_t": 0.4422909617424011,
"grad_norm": 79.64447021484375,
"learning_rate": 1.202740798300168e-07,
"logits/chosen": -0.8437353372573853,
"logits/rejected": -0.8279985785484314,
"logps/chosen": -579.4176025390625,
"logps/ref_chosen": -276.5925598144531,
"logps/ref_rejected": -233.979248046875,
"logps/rejected": -678.0966186523438,
"loss": 4.8058,
"margin_dpo/margin_mean": 141.2923583984375,
"margin_dpo/margin_std": 242.50144958496094,
"step": 338
},
{
"KL/chosen_KL_mean": -316.61181640625,
"KL/mean": -395.79437255859375,
"KL/rejected_KL_mean": -474.9768981933594,
"KL/std": 224.5152587890625,
"epoch": 0.7099476439790576,
"fcm_dpo/beta": 0.0017471557948738337,
"fcm_dpo/delta": -0.03965400159358978,
"fcm_dpo/margin": 158.36505126953125,
"fcm_dpo/q_t": 0.4362345039844513,
"grad_norm": 69.49354553222656,
"learning_rate": 1.1871257444948096e-07,
"logits/chosen": -0.8592104911804199,
"logits/rejected": -0.8398917317390442,
"logps/chosen": -620.1395874023438,
"logps/ref_chosen": -303.5277404785156,
"logps/ref_rejected": -283.11676025390625,
"logps/rejected": -758.09375,
"loss": 4.745,
"margin_dpo/margin_mean": 158.36505126953125,
"margin_dpo/margin_std": 268.96624755859375,
"step": 339
},
{
"KL/chosen_KL_mean": -326.1214599609375,
"KL/mean": -385.92706298828125,
"KL/rejected_KL_mean": -445.7326965332031,
"KL/std": 232.37582397460938,
"epoch": 0.7120418848167539,
"fcm_dpo/beta": 0.0017085066065192223,
"fcm_dpo/delta": 0.008730532601475716,
"fcm_dpo/margin": 119.61127471923828,
"fcm_dpo/q_t": 0.4503565728664398,
"grad_norm": 50.96295928955078,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": -0.8013584613800049,
"logits/rejected": -0.7939292192459106,
"logps/chosen": -587.647216796875,
"logps/ref_chosen": -261.5257568359375,
"logps/ref_rejected": -259.39862060546875,
"logps/rejected": -705.1312866210938,
"loss": 5.0456,
"margin_dpo/margin_mean": 119.61126708984375,
"margin_dpo/margin_std": 292.8406066894531,
"step": 340
},
{
"KL/chosen_KL_mean": -386.3853454589844,
"KL/mean": -446.016357421875,
"KL/rejected_KL_mean": -505.6473693847656,
"KL/std": 270.52874755859375,
"epoch": 0.7141361256544503,
"fcm_dpo/beta": 0.0017137773102149367,
"fcm_dpo/delta": -0.013240095227956772,
"fcm_dpo/margin": 119.26201629638672,
"fcm_dpo/q_t": 0.4538622498512268,
"grad_norm": 76.47930908203125,
"learning_rate": 1.1561076868822755e-07,
"logits/chosen": -0.823747992515564,
"logits/rejected": -0.7963781356811523,
"logps/chosen": -702.2888793945312,
"logps/ref_chosen": -315.903564453125,
"logps/ref_rejected": -308.02392578125,
"logps/rejected": -813.6712646484375,
"loss": 5.0467,
"margin_dpo/margin_mean": 119.26200866699219,
"margin_dpo/margin_std": 304.12493896484375,
"step": 341
},
{
"KL/chosen_KL_mean": -317.0606689453125,
"KL/mean": -373.2706298828125,
"KL/rejected_KL_mean": -429.48052978515625,
"KL/std": 199.8746337890625,
"epoch": 0.7162303664921466,
"fcm_dpo/beta": 0.0017002663807943463,
"fcm_dpo/delta": -0.0062491451390087605,
"fcm_dpo/margin": 112.41980743408203,
"fcm_dpo/q_t": 0.4558378756046295,
"grad_norm": 62.66633987426758,
"learning_rate": 1.1407063464793965e-07,
"logits/chosen": -0.818219780921936,
"logits/rejected": -0.8124086260795593,
"logps/chosen": -586.2393798828125,
"logps/ref_chosen": -269.17864990234375,
"logps/ref_rejected": -260.8977355957031,
"logps/rejected": -690.3782958984375,
"loss": 4.9878,
"margin_dpo/margin_mean": 112.41981506347656,
"margin_dpo/margin_std": 235.89794921875,
"step": 342
},
{
"KL/chosen_KL_mean": -344.4051818847656,
"KL/mean": -385.9061279296875,
"KL/rejected_KL_mean": -427.4070129394531,
"KL/std": 199.83212280273438,
"epoch": 0.7183246073298429,
"fcm_dpo/beta": 0.0016928425757214427,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 83.00184631347656,
"fcm_dpo/q_t": 0.46619269251823425,
"grad_norm": 54.531524658203125,
"learning_rate": 1.125377900869913e-07,
"logits/chosen": -0.8236934542655945,
"logits/rejected": -0.798531711101532,
"logps/chosen": -655.1251831054688,
"logps/ref_chosen": -310.719970703125,
"logps/ref_rejected": -263.5224914550781,
"logps/rejected": -690.9295654296875,
"loss": 5.1421,
"margin_dpo/margin_mean": 83.00184631347656,
"margin_dpo/margin_std": 217.48834228515625,
"step": 343
},
{
"KL/chosen_KL_mean": -331.8240966796875,
"KL/mean": -385.7796630859375,
"KL/rejected_KL_mean": -439.7351989746094,
"KL/std": 196.72119140625,
"epoch": 0.7204188481675393,
"fcm_dpo/beta": 0.0016955842729657888,
"fcm_dpo/delta": 0.00766171608120203,
"fcm_dpo/margin": 107.91111755371094,
"fcm_dpo/q_t": 0.4565431475639343,
"grad_norm": 53.93059539794922,
"learning_rate": 1.110123172071844e-07,
"logits/chosen": -0.8151747584342957,
"logits/rejected": -0.8005221486091614,
"logps/chosen": -633.6240234375,
"logps/ref_chosen": -301.7999267578125,
"logps/ref_rejected": -257.9061584472656,
"logps/rejected": -697.641357421875,
"loss": 5.0036,
"margin_dpo/margin_mean": 107.91112518310547,
"margin_dpo/margin_std": 228.1441650390625,
"step": 344
},
{
"KL/chosen_KL_mean": -310.6817626953125,
"KL/mean": -362.8054504394531,
"KL/rejected_KL_mean": -414.9290771484375,
"KL/std": 194.0869903564453,
"epoch": 0.7225130890052356,
"fcm_dpo/beta": 0.0016927202232182026,
"fcm_dpo/delta": -0.012885820120573044,
"fcm_dpo/margin": 104.247314453125,
"fcm_dpo/q_t": 0.45815154910087585,
"grad_norm": 64.63471984863281,
"learning_rate": 1.09494297815e-07,
"logits/chosen": -0.8192985653877258,
"logits/rejected": -0.8202333450317383,
"logps/chosen": -593.7001953125,
"logps/ref_chosen": -283.0184326171875,
"logps/ref_rejected": -266.8457336425781,
"logps/rejected": -681.7747802734375,
"loss": 5.0197,
"margin_dpo/margin_mean": 104.247314453125,
"margin_dpo/margin_std": 206.515625,
"step": 345
},
{
"KL/chosen_KL_mean": -328.94573974609375,
"KL/mean": -400.6337890625,
"KL/rejected_KL_mean": -472.32177734375,
"KL/std": 245.28079223632812,
"epoch": 0.724607329842932,
"fcm_dpo/beta": 0.0016636288492009044,
"fcm_dpo/delta": -0.02877185493707657,
"fcm_dpo/margin": 143.3760528564453,
"fcm_dpo/q_t": 0.44495314359664917,
"grad_norm": 38.55996322631836,
"learning_rate": 1.0798381331721107e-07,
"logits/chosen": -0.8979187607765198,
"logits/rejected": -0.846237063407898,
"logps/chosen": -597.386962890625,
"logps/ref_chosen": -268.44122314453125,
"logps/ref_rejected": -227.8225860595703,
"logps/rejected": -700.1443481445312,
"loss": 4.8526,
"margin_dpo/margin_mean": 143.3760528564453,
"margin_dpo/margin_std": 257.0928955078125,
"step": 346
},
{
"KL/chosen_KL_mean": -304.1871337890625,
"KL/mean": -360.26837158203125,
"KL/rejected_KL_mean": -416.3497009277344,
"KL/std": 195.80422973632812,
"epoch": 0.7267015706806282,
"fcm_dpo/beta": 0.0016611977480351925,
"fcm_dpo/delta": 0.01643402874469757,
"fcm_dpo/margin": 112.16256713867188,
"fcm_dpo/q_t": 0.4552745223045349,
"grad_norm": 60.81416702270508,
"learning_rate": 1.0648094471651722e-07,
"logits/chosen": -0.745126485824585,
"logits/rejected": -0.7690558433532715,
"logps/chosen": -577.890625,
"logps/ref_chosen": -273.70355224609375,
"logps/ref_rejected": -243.65521240234375,
"logps/rejected": -660.0048217773438,
"loss": 4.9932,
"margin_dpo/margin_mean": 112.16256713867188,
"margin_dpo/margin_std": 225.79522705078125,
"step": 347
},
{
"KL/chosen_KL_mean": -297.6252136230469,
"KL/mean": -350.5988464355469,
"KL/rejected_KL_mean": -403.5724792480469,
"KL/std": 215.3185577392578,
"epoch": 0.7287958115183246,
"fcm_dpo/beta": 0.0016852599801495671,
"fcm_dpo/delta": 0.017294129356741905,
"fcm_dpo/margin": 105.94729614257812,
"fcm_dpo/q_t": 0.45961958169937134,
"grad_norm": 58.32246017456055,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": -0.8524982929229736,
"logits/rejected": -0.8348946571350098,
"logps/chosen": -583.2666015625,
"logps/ref_chosen": -285.64141845703125,
"logps/ref_rejected": -265.6270446777344,
"logps/rejected": -669.1995849609375,
"loss": 5.0505,
"margin_dpo/margin_mean": 105.9472885131836,
"margin_dpo/margin_std": 251.26690673828125,
"step": 348
},
{
"KL/chosen_KL_mean": -297.42645263671875,
"KL/mean": -378.73858642578125,
"KL/rejected_KL_mean": -460.05078125,
"KL/std": 242.02818298339844,
"epoch": 0.7308900523560209,
"fcm_dpo/beta": 0.0016933238366618752,
"fcm_dpo/delta": -0.01814393885433674,
"fcm_dpo/margin": 162.62429809570312,
"fcm_dpo/q_t": 0.43410930037498474,
"grad_norm": 54.12470626831055,
"learning_rate": 1.0349837717080347e-07,
"logits/chosen": -0.8172075152397156,
"logits/rejected": -0.8003545999526978,
"logps/chosen": -625.7439575195312,
"logps/ref_chosen": -328.3175048828125,
"logps/ref_rejected": -292.37872314453125,
"logps/rejected": -752.429443359375,
"loss": 4.7733,
"margin_dpo/margin_mean": 162.62429809570312,
"margin_dpo/margin_std": 289.4478454589844,
"step": 349
},
{
"KL/chosen_KL_mean": -318.63232421875,
"KL/mean": -383.8518981933594,
"KL/rejected_KL_mean": -449.071533203125,
"KL/std": 229.48977661132812,
"epoch": 0.7329842931937173,
"fcm_dpo/beta": 0.0016819657757878304,
"fcm_dpo/delta": 0.02026461437344551,
"fcm_dpo/margin": 130.43919372558594,
"fcm_dpo/q_t": 0.44916805624961853,
"grad_norm": 65.67074584960938,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": -0.7871538996696472,
"logits/rejected": -0.7973438501358032,
"logps/chosen": -611.4369506835938,
"logps/ref_chosen": -292.8046569824219,
"logps/ref_rejected": -250.35504150390625,
"logps/rejected": -699.426513671875,
"loss": 4.9129,
"margin_dpo/margin_mean": 130.439208984375,
"margin_dpo/margin_std": 262.13153076171875,
"step": 350
},
{
"KL/chosen_KL_mean": -307.9429931640625,
"KL/mean": -345.2463073730469,
"KL/rejected_KL_mean": -382.5496520996094,
"KL/std": 196.58447265625,
"epoch": 0.7350785340314137,
"fcm_dpo/beta": 0.0016986449481919408,
"fcm_dpo/delta": 0.0035051475279033184,
"fcm_dpo/margin": 74.606689453125,
"fcm_dpo/q_t": 0.4699043929576874,
"grad_norm": 44.76028060913086,
"learning_rate": 1.0054723495346482e-07,
"logits/chosen": -0.8658809065818787,
"logits/rejected": -0.8433598279953003,
"logps/chosen": -619.83203125,
"logps/ref_chosen": -311.8890380859375,
"logps/ref_rejected": -263.59033203125,
"logps/rejected": -646.1400146484375,
"loss": 5.2382,
"margin_dpo/margin_mean": 74.606689453125,
"margin_dpo/margin_std": 245.6429443359375,
"step": 351
},
{
"KL/chosen_KL_mean": -285.75799560546875,
"KL/mean": -362.57232666015625,
"KL/rejected_KL_mean": -439.3866271972656,
"KL/std": 213.55348205566406,
"epoch": 0.7371727748691099,
"fcm_dpo/beta": 0.001679836306720972,
"fcm_dpo/delta": -0.0261016096919775,
"fcm_dpo/margin": 153.6286163330078,
"fcm_dpo/q_t": 0.4385840892791748,
"grad_norm": 82.70178985595703,
"learning_rate": 9.908364643332398e-08,
"logits/chosen": -0.7824245095252991,
"logits/rejected": -0.7497365474700928,
"logps/chosen": -540.6658935546875,
"logps/ref_chosen": -254.9078826904297,
"logps/ref_rejected": -257.1688232421875,
"logps/rejected": -696.555419921875,
"loss": 4.759,
"margin_dpo/margin_mean": 153.6286163330078,
"margin_dpo/margin_std": 241.4222412109375,
"step": 352
},
{
"KL/chosen_KL_mean": -301.4290771484375,
"KL/mean": -370.8741149902344,
"KL/rejected_KL_mean": -440.3191223144531,
"KL/std": 203.89584350585938,
"epoch": 0.7392670157068063,
"fcm_dpo/beta": 0.0016587604768574238,
"fcm_dpo/delta": -0.01662537083029747,
"fcm_dpo/margin": 138.89007568359375,
"fcm_dpo/q_t": 0.4453839659690857,
"grad_norm": 61.3695182800293,
"learning_rate": 9.76281510992176e-08,
"logits/chosen": -0.7979697585105896,
"logits/rejected": -0.7893396615982056,
"logps/chosen": -571.8051147460938,
"logps/ref_chosen": -270.3760681152344,
"logps/ref_rejected": -264.65234375,
"logps/rejected": -704.9714965820312,
"loss": 4.8318,
"margin_dpo/margin_mean": 138.89007568359375,
"margin_dpo/margin_std": 225.83079528808594,
"step": 353
},
{
"KL/chosen_KL_mean": -303.3616943359375,
"KL/mean": -355.75030517578125,
"KL/rejected_KL_mean": -408.1388854980469,
"KL/std": 229.685302734375,
"epoch": 0.7413612565445026,
"fcm_dpo/beta": 0.0016466137021780014,
"fcm_dpo/delta": 0.0029272071551531553,
"fcm_dpo/margin": 104.77714538574219,
"fcm_dpo/q_t": 0.46023455262184143,
"grad_norm": 51.536128997802734,
"learning_rate": 9.618082700494318e-08,
"logits/chosen": -0.7839207053184509,
"logits/rejected": -0.8151556253433228,
"logps/chosen": -561.01025390625,
"logps/ref_chosen": -257.6485595703125,
"logps/ref_rejected": -246.94203186035156,
"logps/rejected": -655.0808715820312,
"loss": 5.0629,
"margin_dpo/margin_mean": 104.77714538574219,
"margin_dpo/margin_std": 247.20835876464844,
"step": 354
},
{
"KL/chosen_KL_mean": -307.9608154296875,
"KL/mean": -374.2004089355469,
"KL/rejected_KL_mean": -440.4400329589844,
"KL/std": 232.50241088867188,
"epoch": 0.743455497382199,
"fcm_dpo/beta": 0.001645258511416614,
"fcm_dpo/delta": 0.0026130429469048977,
"fcm_dpo/margin": 132.4791717529297,
"fcm_dpo/q_t": 0.44836515188217163,
"grad_norm": 49.24905776977539,
"learning_rate": 9.474175176609956e-08,
"logits/chosen": -0.8556162714958191,
"logits/rejected": -0.8463337421417236,
"logps/chosen": -601.3141479492188,
"logps/ref_chosen": -293.35333251953125,
"logps/ref_rejected": -275.6051940917969,
"logps/rejected": -716.0452270507812,
"loss": 4.9783,
"margin_dpo/margin_mean": 132.4791717529297,
"margin_dpo/margin_std": 294.04217529296875,
"step": 355
},
{
"KL/chosen_KL_mean": -286.2415771484375,
"KL/mean": -336.8193359375,
"KL/rejected_KL_mean": -387.39703369140625,
"KL/std": 191.365234375,
"epoch": 0.7455497382198953,
"fcm_dpo/beta": 0.0016517550684511662,
"fcm_dpo/delta": -0.0037825205363333225,
"fcm_dpo/margin": 101.15542602539062,
"fcm_dpo/q_t": 0.46026772260665894,
"grad_norm": 59.42401123046875,
"learning_rate": 9.331100255592436e-08,
"logits/chosen": -0.7560597658157349,
"logits/rejected": -0.7795335650444031,
"logps/chosen": -490.4970703125,
"logps/ref_chosen": -204.25550842285156,
"logps/ref_rejected": -213.467529296875,
"logps/rejected": -600.864501953125,
"loss": 5.0282,
"margin_dpo/margin_mean": 101.15543365478516,
"margin_dpo/margin_std": 203.6053466796875,
"step": 356
},
{
"KL/chosen_KL_mean": -323.9365539550781,
"KL/mean": -389.21868896484375,
"KL/rejected_KL_mean": -454.5008239746094,
"KL/std": 225.989990234375,
"epoch": 0.7476439790575916,
"fcm_dpo/beta": 0.0016592942411080003,
"fcm_dpo/delta": 0.024898748844861984,
"fcm_dpo/margin": 130.5642547607422,
"fcm_dpo/q_t": 0.4498782753944397,
"grad_norm": 54.635040283203125,
"learning_rate": 9.18886561011557e-08,
"logits/chosen": -0.7446423172950745,
"logits/rejected": -0.7396554946899414,
"logps/chosen": -590.30712890625,
"logps/ref_chosen": -266.3705749511719,
"logps/ref_rejected": -239.04490661621094,
"logps/rejected": -693.5457153320312,
"loss": 4.9304,
"margin_dpo/margin_mean": 130.56427001953125,
"margin_dpo/margin_std": 261.66534423828125,
"step": 357
},
{
"KL/chosen_KL_mean": -291.0216369628906,
"KL/mean": -364.53607177734375,
"KL/rejected_KL_mean": -438.05047607421875,
"KL/std": 207.47088623046875,
"epoch": 0.749738219895288,
"fcm_dpo/beta": 0.0016887328820303082,
"fcm_dpo/delta": 0.008424624800682068,
"fcm_dpo/margin": 147.02883911132812,
"fcm_dpo/q_t": 0.4417012631893158,
"grad_norm": 45.379417419433594,
"learning_rate": 9.047478867791731e-08,
"logits/chosen": -0.840856671333313,
"logits/rejected": -0.8128774166107178,
"logps/chosen": -590.1691284179688,
"logps/ref_chosen": -299.1474609375,
"logps/ref_rejected": -257.2531433105469,
"logps/rejected": -695.3037109375,
"loss": 4.8174,
"margin_dpo/margin_mean": 147.02883911132812,
"margin_dpo/margin_std": 259.8493957519531,
"step": 358
},
{
"KL/chosen_KL_mean": -311.9315490722656,
"KL/mean": -373.98565673828125,
"KL/rejected_KL_mean": -436.0397033691406,
"KL/std": 224.18618774414062,
"epoch": 0.7518324607329843,
"fcm_dpo/beta": 0.0017018289072439075,
"fcm_dpo/delta": 0.012597101740539074,
"fcm_dpo/margin": 124.10816955566406,
"fcm_dpo/q_t": 0.45063310861587524,
"grad_norm": 70.6742935180664,
"learning_rate": 8.906947610762825e-08,
"logits/chosen": -0.8120812177658081,
"logits/rejected": -0.8148164749145508,
"logps/chosen": -614.9293212890625,
"logps/ref_chosen": -302.99786376953125,
"logps/ref_rejected": -260.4137268066406,
"logps/rejected": -696.4534301757812,
"loss": 4.9726,
"margin_dpo/margin_mean": 124.10816192626953,
"margin_dpo/margin_std": 277.532470703125,
"step": 359
},
{
"KL/chosen_KL_mean": -323.221923828125,
"KL/mean": -384.31170654296875,
"KL/rejected_KL_mean": -445.4015808105469,
"KL/std": 263.3553466796875,
"epoch": 0.7539267015706806,
"fcm_dpo/beta": 0.0016766900662332773,
"fcm_dpo/delta": -0.02853110060095787,
"fcm_dpo/margin": 122.17969512939453,
"fcm_dpo/q_t": 0.45524489879608154,
"grad_norm": 63.280357360839844,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": -0.8388208746910095,
"logits/rejected": -0.8119864463806152,
"logps/chosen": -632.8333129882812,
"logps/ref_chosen": -309.6114501953125,
"logps/ref_rejected": -256.64031982421875,
"logps/rejected": -702.0418701171875,
"loss": 5.1195,
"margin_dpo/margin_mean": 122.17970275878906,
"margin_dpo/margin_std": 326.59295654296875,
"step": 360
},
{
"KL/chosen_KL_mean": -273.9930419921875,
"KL/mean": -359.9635314941406,
"KL/rejected_KL_mean": -445.93402099609375,
"KL/std": 247.8087158203125,
"epoch": 0.7560209424083769,
"fcm_dpo/beta": 0.0016680698608979583,
"fcm_dpo/delta": -0.004171290434896946,
"fcm_dpo/margin": 171.94100952148438,
"fcm_dpo/q_t": 0.43602752685546875,
"grad_norm": 57.07918930053711,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": -0.7725020051002502,
"logits/rejected": -0.7491423487663269,
"logps/chosen": -537.3727416992188,
"logps/ref_chosen": -263.3797607421875,
"logps/ref_rejected": -271.18157958984375,
"logps/rejected": -717.1156005859375,
"loss": 4.742,
"margin_dpo/margin_mean": 171.94100952148438,
"margin_dpo/margin_std": 304.07904052734375,
"step": 361
},
{
"KL/chosen_KL_mean": -307.6124267578125,
"KL/mean": -366.4735412597656,
"KL/rejected_KL_mean": -425.3346862792969,
"KL/std": 190.72222900390625,
"epoch": 0.7581151832460733,
"fcm_dpo/beta": 0.0016689738258719444,
"fcm_dpo/delta": 0.010700585320591927,
"fcm_dpo/margin": 117.72223663330078,
"fcm_dpo/q_t": 0.45483019948005676,
"grad_norm": 67.26338195800781,
"learning_rate": 8.490561882286135e-08,
"logits/chosen": -0.8269234895706177,
"logits/rejected": -0.8027861714363098,
"logps/chosen": -610.870849609375,
"logps/ref_chosen": -303.2583923339844,
"logps/ref_rejected": -243.22891235351562,
"logps/rejected": -668.5635375976562,
"loss": 4.949,
"margin_dpo/margin_mean": 117.72223663330078,
"margin_dpo/margin_std": 217.4143829345703,
"step": 362
},
{
"KL/chosen_KL_mean": -318.3824462890625,
"KL/mean": -391.4795837402344,
"KL/rejected_KL_mean": -464.5767822265625,
"KL/std": 235.78480529785156,
"epoch": 0.7602094240837697,
"fcm_dpo/beta": 0.0016669887118041515,
"fcm_dpo/delta": -0.028279222548007965,
"fcm_dpo/margin": 146.19439697265625,
"fcm_dpo/q_t": 0.444366991519928,
"grad_norm": 56.83020782470703,
"learning_rate": 8.353527464267104e-08,
"logits/chosen": -0.8206988573074341,
"logits/rejected": -0.7746375799179077,
"logps/chosen": -621.7296752929688,
"logps/ref_chosen": -303.34722900390625,
"logps/ref_rejected": -262.05419921875,
"logps/rejected": -726.6309814453125,
"loss": 4.8521,
"margin_dpo/margin_mean": 146.19439697265625,
"margin_dpo/margin_std": 274.79852294921875,
"step": 363
},
{
"KL/chosen_KL_mean": -361.5899658203125,
"KL/mean": -448.52484130859375,
"KL/rejected_KL_mean": -535.459716796875,
"KL/std": 327.3614807128906,
"epoch": 0.762303664921466,
"fcm_dpo/beta": 0.0016274906229227781,
"fcm_dpo/delta": -0.05278400331735611,
"fcm_dpo/margin": 173.8697509765625,
"fcm_dpo/q_t": 0.4439254403114319,
"grad_norm": 57.509544372558594,
"learning_rate": 8.217385746050742e-08,
"logits/chosen": -0.7986475825309753,
"logits/rejected": -0.7825050354003906,
"logps/chosen": -647.1337280273438,
"logps/ref_chosen": -285.54376220703125,
"logps/ref_rejected": -284.84619140625,
"logps/rejected": -820.3059692382812,
"loss": 4.8973,
"margin_dpo/margin_mean": 173.86978149414062,
"margin_dpo/margin_std": 399.4956970214844,
"step": 364
},
{
"KL/chosen_KL_mean": -339.6693115234375,
"KL/mean": -399.89129638671875,
"KL/rejected_KL_mean": -460.11328125,
"KL/std": 253.50515747070312,
"epoch": 0.7643979057591623,
"fcm_dpo/beta": 0.0015868466580286622,
"fcm_dpo/delta": 0.014260734431445599,
"fcm_dpo/margin": 120.44401550292969,
"fcm_dpo/q_t": 0.45612239837646484,
"grad_norm": 60.942710876464844,
"learning_rate": 8.082144028504231e-08,
"logits/chosen": -0.8087151050567627,
"logits/rejected": -0.804487943649292,
"logps/chosen": -614.4570922851562,
"logps/ref_chosen": -274.7878112792969,
"logps/ref_rejected": -256.5738220214844,
"logps/rejected": -716.6871337890625,
"loss": 5.0254,
"margin_dpo/margin_mean": 120.44401550292969,
"margin_dpo/margin_std": 279.97271728515625,
"step": 365
},
{
"KL/chosen_KL_mean": -317.1668701171875,
"KL/mean": -382.4986572265625,
"KL/rejected_KL_mean": -447.8304138183594,
"KL/std": 217.1991424560547,
"epoch": 0.7664921465968586,
"fcm_dpo/beta": 0.0016188733279705048,
"fcm_dpo/delta": 0.03934203460812569,
"fcm_dpo/margin": 130.66355895996094,
"fcm_dpo/q_t": 0.45035284757614136,
"grad_norm": 73.36628723144531,
"learning_rate": 7.947809564230445e-08,
"logits/chosen": -0.7664698362350464,
"logits/rejected": -0.7650561928749084,
"logps/chosen": -603.8164672851562,
"logps/ref_chosen": -286.6496276855469,
"logps/ref_rejected": -251.97140502929688,
"logps/rejected": -699.8018188476562,
"loss": 4.9317,
"margin_dpo/margin_mean": 130.66355895996094,
"margin_dpo/margin_std": 264.78948974609375,
"step": 366
},
{
"KL/chosen_KL_mean": -322.634521484375,
"KL/mean": -398.5595703125,
"KL/rejected_KL_mean": -474.48455810546875,
"KL/std": 227.88592529296875,
"epoch": 0.768586387434555,
"fcm_dpo/beta": 0.001617192174308002,
"fcm_dpo/delta": -0.01441990491002798,
"fcm_dpo/margin": 151.8500518798828,
"fcm_dpo/q_t": 0.44398194551467896,
"grad_norm": 52.11629867553711,
"learning_rate": 7.814389557179016e-08,
"logits/chosen": -0.809653639793396,
"logits/rejected": -0.7744249105453491,
"logps/chosen": -624.5795288085938,
"logps/ref_chosen": -301.9449768066406,
"logps/ref_rejected": -265.5677185058594,
"logps/rejected": -740.0523071289062,
"loss": 4.8384,
"margin_dpo/margin_mean": 151.85003662109375,
"margin_dpo/margin_std": 275.8656311035156,
"step": 367
},
{
"KL/chosen_KL_mean": -310.06488037109375,
"KL/mean": -400.94287109375,
"KL/rejected_KL_mean": -491.8209228515625,
"KL/std": 228.97950744628906,
"epoch": 0.7706806282722513,
"fcm_dpo/beta": 0.0015950199449434876,
"fcm_dpo/delta": -0.037169210612773895,
"fcm_dpo/margin": 181.75608825683594,
"fcm_dpo/q_t": 0.43195655941963196,
"grad_norm": 53.12458801269531,
"learning_rate": 7.681891162260015e-08,
"logits/chosen": -0.7793463468551636,
"logits/rejected": -0.77353835105896,
"logps/chosen": -604.6913452148438,
"logps/ref_chosen": -294.62652587890625,
"logps/ref_rejected": -258.7628479003906,
"logps/rejected": -750.583740234375,
"loss": 4.6447,
"margin_dpo/margin_mean": 181.75608825683594,
"margin_dpo/margin_std": 250.93226623535156,
"step": 368
},
{
"KL/chosen_KL_mean": -319.5187683105469,
"KL/mean": -384.4464111328125,
"KL/rejected_KL_mean": -449.3740234375,
"KL/std": 208.54287719726562,
"epoch": 0.7727748691099476,
"fcm_dpo/beta": 0.0015673936577513814,
"fcm_dpo/delta": -0.004795985296368599,
"fcm_dpo/margin": 129.85523986816406,
"fcm_dpo/q_t": 0.45139309763908386,
"grad_norm": 75.6568374633789,
"learning_rate": 7.550321484960251e-08,
"logits/chosen": -0.8449119329452515,
"logits/rejected": -0.8261924982070923,
"logps/chosen": -602.0244750976562,
"logps/ref_chosen": -282.5057373046875,
"logps/ref_rejected": -266.41607666015625,
"logps/rejected": -715.7900390625,
"loss": 4.8941,
"margin_dpo/margin_mean": 129.85523986816406,
"margin_dpo/margin_std": 217.25881958007812,
"step": 369
},
{
"KL/chosen_KL_mean": -300.08746337890625,
"KL/mean": -385.3553771972656,
"KL/rejected_KL_mean": -470.623291015625,
"KL/std": 238.8286590576172,
"epoch": 0.774869109947644,
"fcm_dpo/beta": 0.0015783013077452779,
"fcm_dpo/delta": 0.0026395046152174473,
"fcm_dpo/margin": 170.53579711914062,
"fcm_dpo/q_t": 0.43791812658309937,
"grad_norm": 65.24024200439453,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": -0.8119034767150879,
"logits/rejected": -0.8299652338027954,
"logps/chosen": -551.0939331054688,
"logps/ref_chosen": -251.00640869140625,
"logps/ref_rejected": -238.12542724609375,
"logps/rejected": -708.7487182617188,
"loss": 4.7508,
"margin_dpo/margin_mean": 170.53579711914062,
"margin_dpo/margin_std": 288.934326171875,
"step": 370
},
{
"KL/chosen_KL_mean": -358.78350830078125,
"KL/mean": -403.6153564453125,
"KL/rejected_KL_mean": -448.4471740722656,
"KL/std": 235.52256774902344,
"epoch": 0.7769633507853403,
"fcm_dpo/beta": 0.001578000606968999,
"fcm_dpo/delta": 0.0009506037458777428,
"fcm_dpo/margin": 89.66363525390625,
"fcm_dpo/q_t": 0.4663865566253662,
"grad_norm": 57.798397064208984,
"learning_rate": 7.289996455765748e-08,
"logits/chosen": -0.7915385961532593,
"logits/rejected": -0.762095034122467,
"logps/chosen": -655.442626953125,
"logps/ref_chosen": -296.6591491699219,
"logps/ref_rejected": -251.14675903320312,
"logps/rejected": -699.5939331054688,
"loss": 5.1806,
"margin_dpo/margin_mean": 89.66363525390625,
"margin_dpo/margin_std": 257.93023681640625,
"step": 371
},
{
"KL/chosen_KL_mean": -348.8611145019531,
"KL/mean": -430.9217834472656,
"KL/rejected_KL_mean": -512.982421875,
"KL/std": 264.5611267089844,
"epoch": 0.7790575916230367,
"fcm_dpo/beta": 0.0015493937535211444,
"fcm_dpo/delta": -0.015690520405769348,
"fcm_dpo/margin": 164.12132263183594,
"fcm_dpo/q_t": 0.44282326102256775,
"grad_norm": 55.466941833496094,
"learning_rate": 7.161255064312283e-08,
"logits/chosen": -0.7481983304023743,
"logits/rejected": -0.7245327234268188,
"logps/chosen": -680.2325439453125,
"logps/ref_chosen": -331.3714599609375,
"logps/ref_rejected": -285.56805419921875,
"logps/rejected": -798.5504760742188,
"loss": 4.8437,
"margin_dpo/margin_mean": 164.12132263183594,
"margin_dpo/margin_std": 297.5496826171875,
"step": 372
},
{
"KL/chosen_KL_mean": -335.027099609375,
"KL/mean": -411.2965087890625,
"KL/rejected_KL_mean": -487.56591796875,
"KL/std": 254.25425720214844,
"epoch": 0.7811518324607329,
"fcm_dpo/beta": 0.001557708252221346,
"fcm_dpo/delta": -0.0038446406833827496,
"fcm_dpo/margin": 152.53880310058594,
"fcm_dpo/q_t": 0.44847139716148376,
"grad_norm": 49.32960891723633,
"learning_rate": 7.033470310611945e-08,
"logits/chosen": -0.8375375866889954,
"logits/rejected": -0.7990292310714722,
"logps/chosen": -656.9700927734375,
"logps/ref_chosen": -321.9429931640625,
"logps/ref_rejected": -271.2288513183594,
"logps/rejected": -758.7947998046875,
"loss": 4.9111,
"margin_dpo/margin_mean": 152.53878784179688,
"margin_dpo/margin_std": 324.3055419921875,
"step": 373
},
{
"KL/chosen_KL_mean": -341.29376220703125,
"KL/mean": -412.0097961425781,
"KL/rejected_KL_mean": -482.7259826660156,
"KL/std": 252.2704315185547,
"epoch": 0.7832460732984293,
"fcm_dpo/beta": 0.0015299688093364239,
"fcm_dpo/delta": -0.02197812870144844,
"fcm_dpo/margin": 141.4322052001953,
"fcm_dpo/q_t": 0.44911688566207886,
"grad_norm": 37.8529167175293,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": -0.8442825675010681,
"logits/rejected": -0.8310015797615051,
"logps/chosen": -660.4622802734375,
"logps/ref_chosen": -319.1685485839844,
"logps/ref_rejected": -284.6263732910156,
"logps/rejected": -767.352294921875,
"loss": 4.8947,
"margin_dpo/margin_mean": 141.43222045898438,
"margin_dpo/margin_std": 258.3140563964844,
"step": 374
},
{
"KL/chosen_KL_mean": -364.37353515625,
"KL/mean": -419.4544677734375,
"KL/rejected_KL_mean": -474.5354309082031,
"KL/std": 210.785400390625,
"epoch": 0.7853403141361257,
"fcm_dpo/beta": 0.0015292322495952249,
"fcm_dpo/delta": 0.005789346527308226,
"fcm_dpo/margin": 110.16191864013672,
"fcm_dpo/q_t": 0.4604509472846985,
"grad_norm": 45.025943756103516,
"learning_rate": 6.780798075635675e-08,
"logits/chosen": -0.8373956084251404,
"logits/rejected": -0.814143717288971,
"logps/chosen": -679.2493286132812,
"logps/ref_chosen": -314.87579345703125,
"logps/ref_rejected": -259.1965026855469,
"logps/rejected": -733.73193359375,
"loss": 5.0522,
"margin_dpo/margin_mean": 110.16191864013672,
"margin_dpo/margin_std": 243.85162353515625,
"step": 375
},
{
"KL/chosen_KL_mean": -353.1634216308594,
"KL/mean": -430.255859375,
"KL/rejected_KL_mean": -507.3483581542969,
"KL/std": 252.78001403808594,
"epoch": 0.787434554973822,
"fcm_dpo/beta": 0.001551387831568718,
"fcm_dpo/delta": 0.02010853961110115,
"fcm_dpo/margin": 154.18492126464844,
"fcm_dpo/q_t": 0.44513237476348877,
"grad_norm": 48.4976806640625,
"learning_rate": 6.655924144404906e-08,
"logits/chosen": -0.8034533262252808,
"logits/rejected": -0.7987213134765625,
"logps/chosen": -640.836669921875,
"logps/ref_chosen": -287.6732482910156,
"logps/ref_rejected": -256.6697082519531,
"logps/rejected": -764.01806640625,
"loss": 4.8674,
"margin_dpo/margin_mean": 154.18492126464844,
"margin_dpo/margin_std": 304.42388916015625,
"step": 376
},
{
"KL/chosen_KL_mean": -386.02972412109375,
"KL/mean": -438.955322265625,
"KL/rejected_KL_mean": -491.88092041015625,
"KL/std": 254.73690795898438,
"epoch": 0.7895287958115184,
"fcm_dpo/beta": 0.0015574595890939236,
"fcm_dpo/delta": 0.011809874325990677,
"fcm_dpo/margin": 105.85123443603516,
"fcm_dpo/q_t": 0.46167030930519104,
"grad_norm": 64.08843231201172,
"learning_rate": 6.532033950290885e-08,
"logits/chosen": -0.7984158396720886,
"logits/rejected": -0.7836294174194336,
"logps/chosen": -691.2911987304688,
"logps/ref_chosen": -305.261474609375,
"logps/ref_rejected": -271.8887023925781,
"logps/rejected": -763.7696533203125,
"loss": 5.145,
"margin_dpo/margin_mean": 105.85124206542969,
"margin_dpo/margin_std": 296.171630859375,
"step": 377
},
{
"KL/chosen_KL_mean": -372.6801452636719,
"KL/mean": -446.7356262207031,
"KL/rejected_KL_mean": -520.7911376953125,
"KL/std": 272.634521484375,
"epoch": 0.7916230366492146,
"fcm_dpo/beta": 0.0015734572662040591,
"fcm_dpo/delta": 0.004630648531019688,
"fcm_dpo/margin": 148.11105346679688,
"fcm_dpo/q_t": 0.4467093348503113,
"grad_norm": 57.25436782836914,
"learning_rate": 6.409134137148736e-08,
"logits/chosen": -0.778599739074707,
"logits/rejected": -0.7654741406440735,
"logps/chosen": -654.209716796875,
"logps/ref_chosen": -281.5295715332031,
"logps/ref_rejected": -296.980224609375,
"logps/rejected": -817.7713623046875,
"loss": 4.9086,
"margin_dpo/margin_mean": 148.11105346679688,
"margin_dpo/margin_std": 312.68133544921875,
"step": 378
},
{
"KL/chosen_KL_mean": -342.051513671875,
"KL/mean": -402.2879333496094,
"KL/rejected_KL_mean": -462.52435302734375,
"KL/std": 208.310302734375,
"epoch": 0.793717277486911,
"fcm_dpo/beta": 0.0015739906812086701,
"fcm_dpo/delta": -0.005612190812826157,
"fcm_dpo/margin": 120.47281646728516,
"fcm_dpo/q_t": 0.45524847507476807,
"grad_norm": 55.82075500488281,
"learning_rate": 6.28723129572247e-08,
"logits/chosen": -0.8357870578765869,
"logits/rejected": -0.8165754675865173,
"logps/chosen": -607.13232421875,
"logps/ref_chosen": -265.0807800292969,
"logps/ref_rejected": -230.58932495117188,
"logps/rejected": -693.1136474609375,
"loss": 5.0035,
"margin_dpo/margin_mean": 120.47280883789062,
"margin_dpo/margin_std": 265.7042236328125,
"step": 379
},
{
"KL/chosen_KL_mean": -374.452880859375,
"KL/mean": -457.7342834472656,
"KL/rejected_KL_mean": -541.015625,
"KL/std": 298.7972106933594,
"epoch": 0.7958115183246073,
"fcm_dpo/beta": 0.0015483014285564423,
"fcm_dpo/delta": -0.030254196375608444,
"fcm_dpo/margin": 166.56275939941406,
"fcm_dpo/q_t": 0.4449843466281891,
"grad_norm": 62.38126754760742,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": -0.8070506453514099,
"logits/rejected": -0.7720668911933899,
"logps/chosen": -680.3612670898438,
"logps/ref_chosen": -305.90838623046875,
"logps/ref_rejected": -286.5906677246094,
"logps/rejected": -827.6063842773438,
"loss": 4.9223,
"margin_dpo/margin_mean": 166.56275939941406,
"margin_dpo/margin_std": 373.67486572265625,
"step": 380
},
{
"KL/chosen_KL_mean": -369.968505859375,
"KL/mean": -463.00933837890625,
"KL/rejected_KL_mean": -556.0502319335938,
"KL/std": 326.89111328125,
"epoch": 0.7979057591623037,
"fcm_dpo/beta": 0.0015199973713606596,
"fcm_dpo/delta": -0.028817355632781982,
"fcm_dpo/margin": 186.08172607421875,
"fcm_dpo/q_t": 0.4386086165904999,
"grad_norm": 55.27873992919922,
"learning_rate": 6.046442623320145e-08,
"logits/chosen": -0.7852144837379456,
"logits/rejected": -0.7426830530166626,
"logps/chosen": -622.8392333984375,
"logps/ref_chosen": -252.87066650390625,
"logps/ref_rejected": -261.1927490234375,
"logps/rejected": -817.2429809570312,
"loss": 4.7924,
"margin_dpo/margin_mean": 186.08172607421875,
"margin_dpo/margin_std": 358.6522216796875,
"step": 381
},
{
"KL/chosen_KL_mean": -367.1063537597656,
"KL/mean": -461.32720947265625,
"KL/rejected_KL_mean": -555.5479736328125,
"KL/std": 248.60533142089844,
"epoch": 0.8,
"fcm_dpo/beta": 0.0014805117389187217,
"fcm_dpo/delta": -0.010750288143754005,
"fcm_dpo/margin": 188.44158935546875,
"fcm_dpo/q_t": 0.43490761518478394,
"grad_norm": 65.76939392089844,
"learning_rate": 5.9275697051098275e-08,
"logits/chosen": -0.8315447568893433,
"logits/rejected": -0.8279691934585571,
"logps/chosen": -656.31787109375,
"logps/ref_chosen": -289.2114562988281,
"logps/ref_rejected": -278.45751953125,
"logps/rejected": -834.0054321289062,
"loss": 4.7007,
"margin_dpo/margin_mean": 188.44158935546875,
"margin_dpo/margin_std": 298.6485595703125,
"step": 382
},
{
"KL/chosen_KL_mean": -354.55718994140625,
"KL/mean": -444.1139221191406,
"KL/rejected_KL_mean": -533.6705932617188,
"KL/std": 305.8647766113281,
"epoch": 0.8020942408376963,
"fcm_dpo/beta": 0.0015076538547873497,
"fcm_dpo/delta": -0.013801611959934235,
"fcm_dpo/margin": 179.1134796142578,
"fcm_dpo/q_t": 0.44126763939857483,
"grad_norm": 61.43276596069336,
"learning_rate": 5.809719583454414e-08,
"logits/chosen": -0.7901857495307922,
"logits/rejected": -0.7668880820274353,
"logps/chosen": -628.1880493164062,
"logps/ref_chosen": -273.630859375,
"logps/ref_rejected": -261.44024658203125,
"logps/rejected": -795.1109008789062,
"loss": 4.8441,
"margin_dpo/margin_mean": 179.1134796142578,
"margin_dpo/margin_std": 363.99066162109375,
"step": 383
},
{
"KL/chosen_KL_mean": -368.296630859375,
"KL/mean": -438.9754333496094,
"KL/rejected_KL_mean": -509.65423583984375,
"KL/std": 221.85972595214844,
"epoch": 0.8041884816753927,
"fcm_dpo/beta": 0.001479036407545209,
"fcm_dpo/delta": 0.02152104675769806,
"fcm_dpo/margin": 141.35757446289062,
"fcm_dpo/q_t": 0.450982928276062,
"grad_norm": 50.3799934387207,
"learning_rate": 5.6928985782982524e-08,
"logits/chosen": -0.8385475277900696,
"logits/rejected": -0.8302735686302185,
"logps/chosen": -642.86669921875,
"logps/ref_chosen": -274.5699462890625,
"logps/ref_rejected": -285.8253479003906,
"logps/rejected": -795.4794921875,
"loss": 4.9229,
"margin_dpo/margin_mean": 141.35757446289062,
"margin_dpo/margin_std": 276.80145263671875,
"step": 384
},
{
"KL/chosen_KL_mean": -368.0294494628906,
"KL/mean": -428.7261962890625,
"KL/rejected_KL_mean": -489.4228515625,
"KL/std": 226.50360107421875,
"epoch": 0.806282722513089,
"fcm_dpo/beta": 0.0015037511475384235,
"fcm_dpo/delta": 0.007470243610441685,
"fcm_dpo/margin": 121.39341735839844,
"fcm_dpo/q_t": 0.4575641453266144,
"grad_norm": 60.63737869262695,
"learning_rate": 5.57711295439732e-08,
"logits/chosen": -0.7681106328964233,
"logits/rejected": -0.7596469521522522,
"logps/chosen": -652.1801147460938,
"logps/ref_chosen": -284.150634765625,
"logps/ref_rejected": -244.87921142578125,
"logps/rejected": -734.3021240234375,
"loss": 5.0108,
"margin_dpo/margin_mean": 121.39341735839844,
"margin_dpo/margin_std": 255.0092010498047,
"step": 385
},
{
"KL/chosen_KL_mean": -365.3989562988281,
"KL/mean": -464.4845275878906,
"KL/rejected_KL_mean": -563.5701293945312,
"KL/std": 260.08233642578125,
"epoch": 0.8083769633507853,
"fcm_dpo/beta": 0.001453772303648293,
"fcm_dpo/delta": -0.05015706643462181,
"fcm_dpo/margin": 198.17123413085938,
"fcm_dpo/q_t": 0.4336777925491333,
"grad_norm": 91.76905822753906,
"learning_rate": 5.4623689209832484e-08,
"logits/chosen": -0.7455501556396484,
"logits/rejected": -0.7397761940956116,
"logps/chosen": -685.5752563476562,
"logps/ref_chosen": -320.1762390136719,
"logps/ref_rejected": -302.05023193359375,
"logps/rejected": -865.620361328125,
"loss": 4.6863,
"margin_dpo/margin_mean": 198.17123413085938,
"margin_dpo/margin_std": 294.59130859375,
"step": 386
},
{
"KL/chosen_KL_mean": -407.8135986328125,
"KL/mean": -475.7935485839844,
"KL/rejected_KL_mean": -543.7735595703125,
"KL/std": 272.47930908203125,
"epoch": 0.8104712041884817,
"fcm_dpo/beta": 0.001420615240931511,
"fcm_dpo/delta": -0.033063966780900955,
"fcm_dpo/margin": 135.95997619628906,
"fcm_dpo/q_t": 0.4548041522502899,
"grad_norm": 57.505863189697266,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": -0.7548569440841675,
"logits/rejected": -0.7590454816818237,
"logps/chosen": -680.09375,
"logps/ref_chosen": -272.2801513671875,
"logps/ref_rejected": -265.1615905761719,
"logps/rejected": -808.9351806640625,
"loss": 5.076,
"margin_dpo/margin_mean": 135.95997619628906,
"margin_dpo/margin_std": 337.6916809082031,
"step": 387
},
{
"KL/chosen_KL_mean": -430.6405334472656,
"KL/mean": -481.8836669921875,
"KL/rejected_KL_mean": -533.1268310546875,
"KL/std": 267.2966003417969,
"epoch": 0.812565445026178,
"fcm_dpo/beta": 0.0014033719198778272,
"fcm_dpo/delta": 0.007294456474483013,
"fcm_dpo/margin": 102.4863052368164,
"fcm_dpo/q_t": 0.46697717905044556,
"grad_norm": 49.18757629394531,
"learning_rate": 5.2360301829254745e-08,
"logits/chosen": -0.7436924576759338,
"logits/rejected": -0.7248135209083557,
"logps/chosen": -703.171875,
"logps/ref_chosen": -272.5313415527344,
"logps/ref_rejected": -239.55735778808594,
"logps/rejected": -772.6842041015625,
"loss": 5.2023,
"margin_dpo/margin_mean": 102.4863052368164,
"margin_dpo/margin_std": 319.1353759765625,
"step": 388
},
{
"KL/chosen_KL_mean": -439.1097717285156,
"KL/mean": -521.8563232421875,
"KL/rejected_KL_mean": -604.6029052734375,
"KL/std": 302.55511474609375,
"epoch": 0.8146596858638744,
"fcm_dpo/beta": 0.0013935244642198086,
"fcm_dpo/delta": -0.028961144387722015,
"fcm_dpo/margin": 165.49313354492188,
"fcm_dpo/q_t": 0.44787466526031494,
"grad_norm": 57.25077438354492,
"learning_rate": 5.1244476161413806e-08,
"logits/chosen": -0.7774551510810852,
"logits/rejected": -0.7715262174606323,
"logps/chosen": -720.198974609375,
"logps/ref_chosen": -281.0892639160156,
"logps/ref_rejected": -246.50045776367188,
"logps/rejected": -851.1033935546875,
"loss": 4.9262,
"margin_dpo/margin_mean": 165.49314880371094,
"margin_dpo/margin_std": 342.1744384765625,
"step": 389
},
{
"KL/chosen_KL_mean": -406.08447265625,
"KL/mean": -487.11151123046875,
"KL/rejected_KL_mean": -568.1385498046875,
"KL/std": 261.64813232421875,
"epoch": 0.8167539267015707,
"fcm_dpo/beta": 0.0013833637349307537,
"fcm_dpo/delta": 0.0005270633846521378,
"fcm_dpo/margin": 162.0540771484375,
"fcm_dpo/q_t": 0.4468287527561188,
"grad_norm": 63.353206634521484,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": -0.7989583015441895,
"logits/rejected": -0.8104404807090759,
"logps/chosen": -690.072021484375,
"logps/ref_chosen": -283.98748779296875,
"logps/ref_rejected": -283.465087890625,
"logps/rejected": -851.6036376953125,
"loss": 4.8718,
"margin_dpo/margin_mean": 162.05409240722656,
"margin_dpo/margin_std": 298.63543701171875,
"step": 390
},
{
"KL/chosen_KL_mean": -428.979736328125,
"KL/mean": -515.8250122070312,
"KL/rejected_KL_mean": -602.6703491210938,
"KL/std": 311.1672668457031,
"epoch": 0.818848167539267,
"fcm_dpo/beta": 0.001366346376016736,
"fcm_dpo/delta": -0.023520534858107567,
"fcm_dpo/margin": 173.6905517578125,
"fcm_dpo/q_t": 0.4458865821361542,
"grad_norm": 53.48771667480469,
"learning_rate": 4.904486005914027e-08,
"logits/chosen": -0.7384014129638672,
"logits/rejected": -0.7049020528793335,
"logps/chosen": -712.8411254882812,
"logps/ref_chosen": -283.86138916015625,
"logps/ref_rejected": -263.5093688964844,
"logps/rejected": -866.1796875,
"loss": 4.887,
"margin_dpo/margin_mean": 173.69053649902344,
"margin_dpo/margin_std": 344.82696533203125,
"step": 391
},
{
"KL/chosen_KL_mean": -400.332763671875,
"KL/mean": -495.0168762207031,
"KL/rejected_KL_mean": -589.7010498046875,
"KL/std": 263.923583984375,
"epoch": 0.8209424083769633,
"fcm_dpo/beta": 0.0013336706906557083,
"fcm_dpo/delta": -0.014936976134777069,
"fcm_dpo/margin": 189.36825561523438,
"fcm_dpo/q_t": 0.4395950138568878,
"grad_norm": 65.17903900146484,
"learning_rate": 4.796118758344353e-08,
"logits/chosen": -0.7459010481834412,
"logits/rejected": -0.7519693374633789,
"logps/chosen": -710.40283203125,
"logps/ref_chosen": -310.070068359375,
"logps/ref_rejected": -252.89817810058594,
"logps/rejected": -842.5991821289062,
"loss": 4.779,
"margin_dpo/margin_mean": 189.36825561523438,
"margin_dpo/margin_std": 308.4510803222656,
"step": 392
},
{
"KL/chosen_KL_mean": -422.2491455078125,
"KL/mean": -484.3158874511719,
"KL/rejected_KL_mean": -546.3826293945312,
"KL/std": 261.5521545410156,
"epoch": 0.8230366492146597,
"fcm_dpo/beta": 0.0013305129250511527,
"fcm_dpo/delta": -0.00916867796331644,
"fcm_dpo/margin": 124.13351440429688,
"fcm_dpo/q_t": 0.4618360996246338,
"grad_norm": 50.20282745361328,
"learning_rate": 4.688834983610082e-08,
"logits/chosen": -0.7697139978408813,
"logits/rejected": -0.7453039288520813,
"logps/chosen": -708.9647216796875,
"logps/ref_chosen": -286.7156677246094,
"logps/ref_rejected": -230.00357055664062,
"logps/rejected": -776.38623046875,
"loss": 5.1041,
"margin_dpo/margin_mean": 124.13351440429688,
"margin_dpo/margin_std": 312.4925537109375,
"step": 393
},
{
"KL/chosen_KL_mean": -431.40948486328125,
"KL/mean": -507.337890625,
"KL/rejected_KL_mean": -583.2664184570312,
"KL/std": 355.6380920410156,
"epoch": 0.8251308900523561,
"fcm_dpo/beta": 0.001326891127973795,
"fcm_dpo/delta": -0.0004829922690987587,
"fcm_dpo/margin": 151.85691833496094,
"fcm_dpo/q_t": 0.45768600702285767,
"grad_norm": 45.90632629394531,
"learning_rate": 4.582640435014459e-08,
"logits/chosen": -0.8130242228507996,
"logits/rejected": -0.7946957349777222,
"logps/chosen": -757.4029541015625,
"logps/ref_chosen": -325.9934387207031,
"logps/ref_rejected": -317.42706298828125,
"logps/rejected": -900.6934814453125,
"loss": 5.0776,
"margin_dpo/margin_mean": 151.85691833496094,
"margin_dpo/margin_std": 398.54766845703125,
"step": 394
},
{
"KL/chosen_KL_mean": -396.9059753417969,
"KL/mean": -497.17181396484375,
"KL/rejected_KL_mean": -597.4376220703125,
"KL/std": 304.17828369140625,
"epoch": 0.8272251308900523,
"fcm_dpo/beta": 0.0013342795427888632,
"fcm_dpo/delta": -0.008068141527473927,
"fcm_dpo/margin": 200.5316619873047,
"fcm_dpo/q_t": 0.44144290685653687,
"grad_norm": 54.122161865234375,
"learning_rate": 4.477540807448832e-08,
"logits/chosen": -0.7593112587928772,
"logits/rejected": -0.7634783983230591,
"logps/chosen": -665.8067626953125,
"logps/ref_chosen": -268.90081787109375,
"logps/ref_rejected": -272.85809326171875,
"logps/rejected": -870.2957763671875,
"loss": 4.8032,
"margin_dpo/margin_mean": 200.5316619873047,
"margin_dpo/margin_std": 380.6814880371094,
"step": 395
},
{
"KL/chosen_KL_mean": -439.490478515625,
"KL/mean": -507.13873291015625,
"KL/rejected_KL_mean": -574.7869873046875,
"KL/std": 280.7339782714844,
"epoch": 0.8293193717277487,
"fcm_dpo/beta": 0.0013154197949916124,
"fcm_dpo/delta": -0.006766099948436022,
"fcm_dpo/margin": 135.29653930664062,
"fcm_dpo/q_t": 0.4586159884929657,
"grad_norm": 77.64254760742188,
"learning_rate": 4.373541737087263e-08,
"logits/chosen": -0.7809940576553345,
"logits/rejected": -0.7539770603179932,
"logps/chosen": -730.688720703125,
"logps/ref_chosen": -291.19830322265625,
"logps/ref_rejected": -253.2803955078125,
"logps/rejected": -828.0673828125,
"loss": 5.0548,
"margin_dpo/margin_mean": 135.29653930664062,
"margin_dpo/margin_std": 323.10113525390625,
"step": 396
},
{
"KL/chosen_KL_mean": -422.6109619140625,
"KL/mean": -488.63763427734375,
"KL/rejected_KL_mean": -554.6641845703125,
"KL/std": 269.7872619628906,
"epoch": 0.831413612565445,
"fcm_dpo/beta": 0.0012935028644278646,
"fcm_dpo/delta": -0.021948356181383133,
"fcm_dpo/margin": 132.0531768798828,
"fcm_dpo/q_t": 0.46082544326782227,
"grad_norm": 43.36281204223633,
"learning_rate": 4.270648801084295e-08,
"logits/chosen": -0.784430742263794,
"logits/rejected": -0.7664919495582581,
"logps/chosen": -732.4334106445312,
"logps/ref_chosen": -309.8224182128906,
"logps/ref_rejected": -291.9057922363281,
"logps/rejected": -846.5699462890625,
"loss": 5.0907,
"margin_dpo/margin_mean": 132.05319213867188,
"margin_dpo/margin_std": 318.5513916015625,
"step": 397
},
{
"KL/chosen_KL_mean": -475.15447998046875,
"KL/mean": -527.6256713867188,
"KL/rejected_KL_mean": -580.0968017578125,
"KL/std": 301.6562805175781,
"epoch": 0.8335078534031414,
"fcm_dpo/beta": 0.0012898289132863283,
"fcm_dpo/delta": 0.005256129428744316,
"fcm_dpo/margin": 104.9423599243164,
"fcm_dpo/q_t": 0.4673859477043152,
"grad_norm": 83.37042999267578,
"learning_rate": 4.168867517275806e-08,
"logits/chosen": -0.6738908886909485,
"logits/rejected": -0.711742103099823,
"logps/chosen": -772.968017578125,
"logps/ref_chosen": -297.8135070800781,
"logps/ref_rejected": -270.5025634765625,
"logps/rejected": -850.5994873046875,
"loss": 5.3166,
"margin_dpo/margin_mean": 104.94235229492188,
"margin_dpo/margin_std": 399.0448913574219,
"step": 398
},
{
"KL/chosen_KL_mean": -434.46160888671875,
"KL/mean": -512.9297485351562,
"KL/rejected_KL_mean": -591.3978271484375,
"KL/std": 286.52301025390625,
"epoch": 0.8356020942408376,
"fcm_dpo/beta": 0.0013032241258770227,
"fcm_dpo/delta": 0.01705552078783512,
"fcm_dpo/margin": 156.936279296875,
"fcm_dpo/q_t": 0.4539187550544739,
"grad_norm": 78.72877502441406,
"learning_rate": 4.0682033438831584e-08,
"logits/chosen": -0.7885871529579163,
"logits/rejected": -0.7415661811828613,
"logps/chosen": -727.3084106445312,
"logps/ref_chosen": -292.8467712402344,
"logps/ref_rejected": -268.3638916015625,
"logps/rejected": -859.76171875,
"loss": 4.9874,
"margin_dpo/margin_mean": 156.93626403808594,
"margin_dpo/margin_std": 351.0825500488281,
"step": 399
},
{
"KL/chosen_KL_mean": -427.16387939453125,
"KL/mean": -506.3305969238281,
"KL/rejected_KL_mean": -585.497314453125,
"KL/std": 281.44110107421875,
"epoch": 0.837696335078534,
"fcm_dpo/beta": 0.0013135882327333093,
"fcm_dpo/delta": 0.005816956982016563,
"fcm_dpo/margin": 158.33343505859375,
"fcm_dpo/q_t": 0.4515479803085327,
"grad_norm": 66.58218383789062,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": -0.8234080076217651,
"logits/rejected": -0.8150714039802551,
"logps/chosen": -690.84033203125,
"logps/ref_chosen": -263.6763916015625,
"logps/ref_rejected": -258.67266845703125,
"logps/rejected": -844.1699829101562,
"loss": 4.9672,
"margin_dpo/margin_mean": 158.3334197998047,
"margin_dpo/margin_std": 340.52227783203125,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_KL/chosen_KL_mean": -431.93609619140625,
"eval_KL/mean": -511.6028137207031,
"eval_KL/rejected_KL_mean": -591.2696533203125,
"eval_KL/std": 291.858154296875,
"eval_fcm_dpo/beta": 0.0013183961855247617,
"eval_logits/chosen": -0.7613639235496521,
"eval_logits/rejected": -0.7369002103805542,
"eval_logps/chosen": -719.7628173828125,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -858.19970703125,
"eval_loss": 0.6247319579124451,
"eval_margin_dpo/margin_mean": 159.33351135253906,
"eval_margin_dpo/margin_std": 334.47113037109375,
"eval_runtime": 78.1079,
"eval_samples_per_second": 25.606,
"eval_steps_per_second": 3.201,
"step": 400
},
{
"KL/chosen_KL_mean": -426.7398376464844,
"KL/mean": -502.2171630859375,
"KL/rejected_KL_mean": -577.6944580078125,
"KL/std": 288.7701416015625,
"epoch": 0.8397905759162304,
"fcm_dpo/beta": 0.001324369921348989,
"fcm_dpo/delta": 0.019296318292617798,
"fcm_dpo/margin": 150.95465087890625,
"fcm_dpo/q_t": 0.4532890021800995,
"grad_norm": 71.27273559570312,
"learning_rate": 3.8702478614051345e-08,
"logits/chosen": -0.7519879937171936,
"logits/rejected": -0.7463111877441406,
"logps/chosen": -745.025146484375,
"logps/ref_chosen": -318.2853088378906,
"logps/ref_rejected": -293.75225830078125,
"logps/rejected": -871.4466552734375,
"loss": 4.9883,
"margin_dpo/margin_mean": 150.95465087890625,
"margin_dpo/margin_std": 336.1131896972656,
"step": 401
},
{
"KL/chosen_KL_mean": -430.9375305175781,
"KL/mean": -494.4723205566406,
"KL/rejected_KL_mean": -558.0069580078125,
"KL/std": 295.1917724609375,
"epoch": 0.8418848167539267,
"fcm_dpo/beta": 0.0013450951082631946,
"fcm_dpo/delta": 0.007680067326873541,
"fcm_dpo/margin": 127.0694580078125,
"fcm_dpo/q_t": 0.46227654814720154,
"grad_norm": 67.7490463256836,
"learning_rate": 3.772967168071517e-08,
"logits/chosen": -0.8251144886016846,
"logits/rejected": -0.7937527298927307,
"logps/chosen": -740.3653564453125,
"logps/ref_chosen": -309.4278564453125,
"logps/ref_rejected": -282.0279846191406,
"logps/rejected": -840.0350341796875,
"loss": 5.1424,
"margin_dpo/margin_mean": 127.0694580078125,
"margin_dpo/margin_std": 378.865966796875,
"step": 402
},
{
"KL/chosen_KL_mean": -396.99566650390625,
"KL/mean": -511.38201904296875,
"KL/rejected_KL_mean": -625.768310546875,
"KL/std": 321.3643798828125,
"epoch": 0.8439790575916231,
"fcm_dpo/beta": 0.0013335788389667869,
"fcm_dpo/delta": -0.03909500688314438,
"fcm_dpo/margin": 228.77272033691406,
"fcm_dpo/q_t": 0.4304249584674835,
"grad_norm": 51.24015426635742,
"learning_rate": 3.676824816087978e-08,
"logits/chosen": -0.8037683963775635,
"logits/rejected": -0.7648081183433533,
"logps/chosen": -706.0241088867188,
"logps/ref_chosen": -309.0284729003906,
"logps/ref_rejected": -272.9622497558594,
"logps/rejected": -898.7305908203125,
"loss": 4.6302,
"margin_dpo/margin_mean": 228.772705078125,
"margin_dpo/margin_std": 343.154541015625,
"step": 403
},
{
"KL/chosen_KL_mean": -442.41656494140625,
"KL/mean": -525.9095458984375,
"KL/rejected_KL_mean": -609.4025268554688,
"KL/std": 310.6871032714844,
"epoch": 0.8460732984293193,
"fcm_dpo/beta": 0.001324630226008594,
"fcm_dpo/delta": 0.01602954789996147,
"fcm_dpo/margin": 166.98602294921875,
"fcm_dpo/q_t": 0.449731707572937,
"grad_norm": 60.760292053222656,
"learning_rate": 3.581825961277074e-08,
"logits/chosen": -0.8088729381561279,
"logits/rejected": -0.7796494364738464,
"logps/chosen": -739.7001953125,
"logps/ref_chosen": -297.2837219238281,
"logps/ref_rejected": -256.99041748046875,
"logps/rejected": -866.3929443359375,
"loss": 4.9848,
"margin_dpo/margin_mean": 166.98602294921875,
"margin_dpo/margin_std": 390.0989990234375,
"step": 404
},
{
"KL/chosen_KL_mean": -426.5012512207031,
"KL/mean": -526.6109008789062,
"KL/rejected_KL_mean": -626.7206420898438,
"KL/std": 317.7232971191406,
"epoch": 0.8481675392670157,
"fcm_dpo/beta": 0.0013220272958278656,
"fcm_dpo/delta": -6.324471905827522e-05,
"fcm_dpo/margin": 200.21939086914062,
"fcm_dpo/q_t": 0.43894970417022705,
"grad_norm": 57.350894927978516,
"learning_rate": 3.487975698139084e-08,
"logits/chosen": -0.7303048372268677,
"logits/rejected": -0.7393031120300293,
"logps/chosen": -684.4666137695312,
"logps/ref_chosen": -257.96533203125,
"logps/ref_rejected": -255.811279296875,
"logps/rejected": -882.531982421875,
"loss": 4.784,
"margin_dpo/margin_mean": 200.21939086914062,
"margin_dpo/margin_std": 362.07623291015625,
"step": 405
},
{
"KL/chosen_KL_mean": -448.4387512207031,
"KL/mean": -505.62249755859375,
"KL/rejected_KL_mean": -562.8062133789062,
"KL/std": 272.3173828125,
"epoch": 0.8502617801047121,
"fcm_dpo/beta": 0.0013311943039298058,
"fcm_dpo/delta": 0.01564777083694935,
"fcm_dpo/margin": 114.367431640625,
"fcm_dpo/q_t": 0.4627969264984131,
"grad_norm": 58.29486846923828,
"learning_rate": 3.3952790595787986e-08,
"logits/chosen": -0.7839182615280151,
"logits/rejected": -0.7578977346420288,
"logps/chosen": -733.619873046875,
"logps/ref_chosen": -285.1810607910156,
"logps/ref_rejected": -264.41351318359375,
"logps/rejected": -827.2197265625,
"loss": 5.1225,
"margin_dpo/margin_mean": 114.367431640625,
"margin_dpo/margin_std": 295.78497314453125,
"step": 406
},
{
"KL/chosen_KL_mean": -450.4339599609375,
"KL/mean": -536.018798828125,
"KL/rejected_KL_mean": -621.6035766601562,
"KL/std": 315.64569091796875,
"epoch": 0.8523560209424084,
"fcm_dpo/beta": 0.0013364061014726758,
"fcm_dpo/delta": -0.014766542240977287,
"fcm_dpo/margin": 171.16966247558594,
"fcm_dpo/q_t": 0.4484463036060333,
"grad_norm": 60.3465461730957,
"learning_rate": 3.303741016635614e-08,
"logits/chosen": -0.7542386054992676,
"logits/rejected": -0.7638047933578491,
"logps/chosen": -715.6720581054688,
"logps/ref_chosen": -265.23809814453125,
"logps/ref_rejected": -219.0631561279297,
"logps/rejected": -840.6668090820312,
"loss": 4.9662,
"margin_dpo/margin_mean": 171.16966247558594,
"margin_dpo/margin_std": 389.5382995605469,
"step": 407
},
{
"KL/chosen_KL_mean": -413.54681396484375,
"KL/mean": -505.22528076171875,
"KL/rejected_KL_mean": -596.9036865234375,
"KL/std": 283.1024169921875,
"epoch": 0.8544502617801047,
"fcm_dpo/beta": 0.0013287973124533892,
"fcm_dpo/delta": -0.018995679914951324,
"fcm_dpo/margin": 183.35684204101562,
"fcm_dpo/q_t": 0.4423255920410156,
"grad_norm": 65.34120178222656,
"learning_rate": 3.2133664782169944e-08,
"logits/chosen": -0.7940729856491089,
"logits/rejected": -0.7762957215309143,
"logps/chosen": -710.5194091796875,
"logps/ref_chosen": -296.9726257324219,
"logps/ref_rejected": -295.4786376953125,
"logps/rejected": -892.38232421875,
"loss": 4.8044,
"margin_dpo/margin_mean": 183.35687255859375,
"margin_dpo/margin_std": 300.9643859863281,
"step": 408
},
{
"KL/chosen_KL_mean": -426.305419921875,
"KL/mean": -522.0606689453125,
"KL/rejected_KL_mean": -617.8158569335938,
"KL/std": 355.56585693359375,
"epoch": 0.856544502617801,
"fcm_dpo/beta": 0.001289794105105102,
"fcm_dpo/delta": -0.027987752109766006,
"fcm_dpo/margin": 191.5104522705078,
"fcm_dpo/q_t": 0.4499598741531372,
"grad_norm": 46.15365982055664,
"learning_rate": 3.12416029083514e-08,
"logits/chosen": -0.7818073034286499,
"logits/rejected": -0.761074960231781,
"logps/chosen": -713.684814453125,
"logps/ref_chosen": -287.37933349609375,
"logps/ref_rejected": -275.80291748046875,
"logps/rejected": -893.6187744140625,
"loss": 4.9385,
"margin_dpo/margin_mean": 191.51046752929688,
"margin_dpo/margin_std": 431.0859069824219,
"step": 409
},
{
"KL/chosen_KL_mean": -437.843505859375,
"KL/mean": -509.63983154296875,
"KL/rejected_KL_mean": -581.43603515625,
"KL/std": 298.229736328125,
"epoch": 0.8586387434554974,
"fcm_dpo/beta": 0.001277453382499516,
"fcm_dpo/delta": 0.0034497997257858515,
"fcm_dpo/margin": 143.592529296875,
"fcm_dpo/q_t": 0.45774397253990173,
"grad_norm": 70.2773208618164,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": -0.7765858769416809,
"logits/rejected": -0.7771980166435242,
"logps/chosen": -719.6236572265625,
"logps/ref_chosen": -281.7801818847656,
"logps/ref_rejected": -266.7550354003906,
"logps/rejected": -848.1910400390625,
"loss": 5.0507,
"margin_dpo/margin_mean": 143.592529296875,
"margin_dpo/margin_std": 343.8995056152344,
"step": 410
},
{
"KL/chosen_KL_mean": -411.8667297363281,
"KL/mean": -501.0542907714844,
"KL/rejected_KL_mean": -590.2418823242188,
"KL/std": 304.47467041015625,
"epoch": 0.8607329842931937,
"fcm_dpo/beta": 0.0012669181451201439,
"fcm_dpo/delta": -0.0407031811773777,
"fcm_dpo/margin": 178.37509155273438,
"fcm_dpo/q_t": 0.4484909772872925,
"grad_norm": 55.282936096191406,
"learning_rate": 2.9492720416985e-08,
"logits/chosen": -0.7966690063476562,
"logits/rejected": -0.7400557398796082,
"logps/chosen": -693.4540405273438,
"logps/ref_chosen": -281.5872497558594,
"logps/ref_rejected": -254.78916931152344,
"logps/rejected": -845.031005859375,
"loss": 4.8709,
"margin_dpo/margin_mean": 178.37509155273438,
"margin_dpo/margin_std": 299.1616516113281,
"step": 411
},
{
"KL/chosen_KL_mean": -471.7283630371094,
"KL/mean": -547.4363403320312,
"KL/rejected_KL_mean": -623.1444702148438,
"KL/std": 309.1171569824219,
"epoch": 0.86282722513089,
"fcm_dpo/beta": 0.001206240151077509,
"fcm_dpo/delta": -0.03707820922136307,
"fcm_dpo/margin": 151.41604614257812,
"fcm_dpo/q_t": 0.4574580192565918,
"grad_norm": 91.96940612792969,
"learning_rate": 2.863599358669755e-08,
"logits/chosen": -0.7530825734138489,
"logits/rejected": -0.7635226249694824,
"logps/chosen": -748.2625122070312,
"logps/ref_chosen": -276.5341796875,
"logps/ref_rejected": -273.8751220703125,
"logps/rejected": -897.01953125,
"loss": 5.0485,
"margin_dpo/margin_mean": 151.4160614013672,
"margin_dpo/margin_std": 343.7683410644531,
"step": 412
},
{
"KL/chosen_KL_mean": -426.9269104003906,
"KL/mean": -498.89007568359375,
"KL/rejected_KL_mean": -570.8531494140625,
"KL/std": 293.0097351074219,
"epoch": 0.8649214659685864,
"fcm_dpo/beta": 0.0012071667006239295,
"fcm_dpo/delta": 0.00854087807238102,
"fcm_dpo/margin": 143.92626953125,
"fcm_dpo/q_t": 0.46002674102783203,
"grad_norm": 51.58241653442383,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": -0.778816819190979,
"logits/rejected": -0.7682943344116211,
"logps/chosen": -698.2015380859375,
"logps/ref_chosen": -271.2745666503906,
"logps/ref_rejected": -270.16912841796875,
"logps/rejected": -841.0222778320312,
"loss": 5.0608,
"margin_dpo/margin_mean": 143.92626953125,
"margin_dpo/margin_std": 354.32513427734375,
"step": 413
},
{
"KL/chosen_KL_mean": -444.8292236328125,
"KL/mean": -537.6610107421875,
"KL/rejected_KL_mean": -630.49267578125,
"KL/std": 300.1920166015625,
"epoch": 0.8670157068062827,
"fcm_dpo/beta": 0.0012186833191663027,
"fcm_dpo/delta": 0.010454244911670685,
"fcm_dpo/margin": 185.66348266601562,
"fcm_dpo/q_t": 0.4482007324695587,
"grad_norm": 49.763938903808594,
"learning_rate": 2.6958198472749717e-08,
"logits/chosen": -0.8072595000267029,
"logits/rejected": -0.7890864610671997,
"logps/chosen": -741.9442749023438,
"logps/ref_chosen": -297.11505126953125,
"logps/ref_rejected": -271.7034606933594,
"logps/rejected": -902.1961059570312,
"loss": 4.882,
"margin_dpo/margin_mean": 185.66348266601562,
"margin_dpo/margin_std": 349.4181213378906,
"step": 414
},
{
"KL/chosen_KL_mean": -425.5120849609375,
"KL/mean": -520.7711791992188,
"KL/rejected_KL_mean": -616.0302734375,
"KL/std": 286.868896484375,
"epoch": 0.8691099476439791,
"fcm_dpo/beta": 0.0012264670804142952,
"fcm_dpo/delta": 0.006376807112246752,
"fcm_dpo/margin": 190.51815795898438,
"fcm_dpo/q_t": 0.4448871314525604,
"grad_norm": 91.31519317626953,
"learning_rate": 2.613722016414943e-08,
"logits/chosen": -0.7885741591453552,
"logits/rejected": -0.7686821818351746,
"logps/chosen": -723.204833984375,
"logps/ref_chosen": -297.6926574707031,
"logps/ref_rejected": -279.0503234863281,
"logps/rejected": -895.08056640625,
"loss": 4.8312,
"margin_dpo/margin_mean": 190.51815795898438,
"margin_dpo/margin_std": 332.1375732421875,
"step": 415
},
{
"KL/chosen_KL_mean": -427.74713134765625,
"KL/mean": -514.8232421875,
"KL/rejected_KL_mean": -601.8994750976562,
"KL/std": 273.8400573730469,
"epoch": 0.8712041884816754,
"fcm_dpo/beta": 0.0012284396216273308,
"fcm_dpo/delta": 0.01868375763297081,
"fcm_dpo/margin": 174.15232849121094,
"fcm_dpo/q_t": 0.449332594871521,
"grad_norm": 81.10550689697266,
"learning_rate": 2.5328246937043525e-08,
"logits/chosen": -0.7960155606269836,
"logits/rejected": -0.7860805988311768,
"logps/chosen": -739.5726318359375,
"logps/ref_chosen": -311.8255615234375,
"logps/ref_rejected": -268.6170654296875,
"logps/rejected": -870.5166015625,
"loss": 4.8818,
"margin_dpo/margin_mean": 174.15231323242188,
"margin_dpo/margin_std": 305.1326904296875,
"step": 416
},
{
"KL/chosen_KL_mean": -471.95123291015625,
"KL/mean": -555.9248657226562,
"KL/rejected_KL_mean": -639.8984985351562,
"KL/std": 336.646240234375,
"epoch": 0.8732984293193717,
"fcm_dpo/beta": 0.0012344487477093935,
"fcm_dpo/delta": -0.029500747099518776,
"fcm_dpo/margin": 167.94720458984375,
"fcm_dpo/q_t": 0.45440053939819336,
"grad_norm": 57.704097747802734,
"learning_rate": 2.4531322174210973e-08,
"logits/chosen": -0.7565368413925171,
"logits/rejected": -0.734302282333374,
"logps/chosen": -782.3880615234375,
"logps/ref_chosen": -310.43682861328125,
"logps/ref_rejected": -277.15283203125,
"logps/rejected": -917.05126953125,
"loss": 5.0212,
"margin_dpo/margin_mean": 167.9472198486328,
"margin_dpo/margin_std": 395.55621337890625,
"step": 417
},
{
"KL/chosen_KL_mean": -478.63140869140625,
"KL/mean": -567.9556274414062,
"KL/rejected_KL_mean": -657.2798461914062,
"KL/std": 373.185302734375,
"epoch": 0.875392670157068,
"fcm_dpo/beta": 0.001195128308609128,
"fcm_dpo/delta": -0.022249775007367134,
"fcm_dpo/margin": 178.6484832763672,
"fcm_dpo/q_t": 0.4534074366092682,
"grad_norm": 60.83293151855469,
"learning_rate": 2.3746488612308295e-08,
"logits/chosen": -0.7093433141708374,
"logits/rejected": -0.6751872897148132,
"logps/chosen": -757.1273193359375,
"logps/ref_chosen": -278.49591064453125,
"logps/ref_rejected": -276.56671142578125,
"logps/rejected": -933.8466796875,
"loss": 5.0329,
"margin_dpo/margin_mean": 178.6484832763672,
"margin_dpo/margin_std": 459.3668518066406,
"step": 418
},
{
"KL/chosen_KL_mean": -466.22515869140625,
"KL/mean": -540.742919921875,
"KL/rejected_KL_mean": -615.2605590820312,
"KL/std": 327.9731750488281,
"epoch": 0.8774869109947644,
"fcm_dpo/beta": 0.001201170147396624,
"fcm_dpo/delta": 0.011327693238854408,
"fcm_dpo/margin": 149.03546142578125,
"fcm_dpo/q_t": 0.4571627676486969,
"grad_norm": 66.67911529541016,
"learning_rate": 2.297378833957761e-08,
"logits/chosen": -0.7795159220695496,
"logits/rejected": -0.7457402944564819,
"logps/chosen": -765.1253662109375,
"logps/ref_chosen": -298.9002380371094,
"logps/ref_rejected": -246.1540985107422,
"logps/rejected": -861.4146728515625,
"loss": 5.1194,
"margin_dpo/margin_mean": 149.0354461669922,
"margin_dpo/margin_std": 419.2374572753906,
"step": 419
},
{
"KL/chosen_KL_mean": -451.4075927734375,
"KL/mean": -554.6902465820312,
"KL/rejected_KL_mean": -657.972900390625,
"KL/std": 334.60174560546875,
"epoch": 0.8795811518324608,
"fcm_dpo/beta": 0.0011963802389800549,
"fcm_dpo/delta": -0.025561338290572166,
"fcm_dpo/margin": 206.5653076171875,
"fcm_dpo/q_t": 0.44405436515808105,
"grad_norm": 48.52541732788086,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": -0.7045433521270752,
"logits/rejected": -0.6653756499290466,
"logps/chosen": -715.968505859375,
"logps/ref_chosen": -264.5608825683594,
"logps/ref_rejected": -245.67031860351562,
"logps/rejected": -903.6431884765625,
"loss": 4.8964,
"margin_dpo/margin_mean": 206.5653076171875,
"margin_dpo/margin_std": 432.84625244140625,
"step": 420
},
{
"KL/chosen_KL_mean": -413.5687561035156,
"KL/mean": -502.61492919921875,
"KL/rejected_KL_mean": -591.6611938476562,
"KL/std": 279.97216796875,
"epoch": 0.881675392670157,
"fcm_dpo/beta": 0.001170460251159966,
"fcm_dpo/delta": -0.023245420306921005,
"fcm_dpo/margin": 178.09242248535156,
"fcm_dpo/q_t": 0.44988635182380676,
"grad_norm": 61.88900375366211,
"learning_rate": 2.1464952759020856e-08,
"logits/chosen": -0.7998079657554626,
"logits/rejected": -0.7664088010787964,
"logps/chosen": -711.2737426757812,
"logps/ref_chosen": -297.70501708984375,
"logps/ref_rejected": -243.74771118164062,
"logps/rejected": -835.4088745117188,
"loss": 4.9225,
"margin_dpo/margin_mean": 178.09242248535156,
"margin_dpo/margin_std": 329.73651123046875,
"step": 421
},
{
"KL/chosen_KL_mean": -449.1634826660156,
"KL/mean": -527.736083984375,
"KL/rejected_KL_mean": -606.3087158203125,
"KL/std": 306.22967529296875,
"epoch": 0.8837696335078534,
"fcm_dpo/beta": 0.001158454455435276,
"fcm_dpo/delta": 0.0001966398995136842,
"fcm_dpo/margin": 157.14520263671875,
"fcm_dpo/q_t": 0.4581235349178314,
"grad_norm": 46.273345947265625,
"learning_rate": 2.07288983654679e-08,
"logits/chosen": -0.6590052247047424,
"logits/rejected": -0.6952117085456848,
"logps/chosen": -737.522216796875,
"logps/ref_chosen": -288.3587646484375,
"logps/ref_rejected": -256.4377746582031,
"logps/rejected": -862.7464599609375,
"loss": 5.0234,
"margin_dpo/margin_mean": 157.14520263671875,
"margin_dpo/margin_std": 353.7700500488281,
"step": 422
},
{
"KL/chosen_KL_mean": -467.9991455078125,
"KL/mean": -561.8355712890625,
"KL/rejected_KL_mean": -655.6720581054688,
"KL/std": 315.5198974609375,
"epoch": 0.8858638743455497,
"fcm_dpo/beta": 0.0011576899560168386,
"fcm_dpo/delta": -0.0015649726847186685,
"fcm_dpo/margin": 187.67288208007812,
"fcm_dpo/q_t": 0.4489739239215851,
"grad_norm": 48.21940231323242,
"learning_rate": 2.0005139085293942e-08,
"logits/chosen": -0.7844016551971436,
"logits/rejected": -0.7567226886749268,
"logps/chosen": -764.0061645507812,
"logps/ref_chosen": -296.00701904296875,
"logps/ref_rejected": -261.3480529785156,
"logps/rejected": -917.0200805664062,
"loss": 4.9226,
"margin_dpo/margin_mean": 187.6728973388672,
"margin_dpo/margin_std": 380.403564453125,
"step": 423
},
{
"KL/chosen_KL_mean": -434.7855224609375,
"KL/mean": -512.031494140625,
"KL/rejected_KL_mean": -589.2774658203125,
"KL/std": 287.8274230957031,
"epoch": 0.8879581151832461,
"fcm_dpo/beta": 0.0011533235665410757,
"fcm_dpo/delta": -0.00939516257494688,
"fcm_dpo/margin": 154.49185180664062,
"fcm_dpo/q_t": 0.45800527930259705,
"grad_norm": 76.39921569824219,
"learning_rate": 1.9293713731512673e-08,
"logits/chosen": -0.7860153317451477,
"logits/rejected": -0.76336669921875,
"logps/chosen": -744.2073974609375,
"logps/ref_chosen": -309.421875,
"logps/ref_rejected": -249.14886474609375,
"logps/rejected": -838.4263305664062,
"loss": 5.0235,
"margin_dpo/margin_mean": 154.49185180664062,
"margin_dpo/margin_std": 335.57342529296875,
"step": 424
},
{
"KL/chosen_KL_mean": -412.7100524902344,
"KL/mean": -507.1483154296875,
"KL/rejected_KL_mean": -601.5865478515625,
"KL/std": 304.760986328125,
"epoch": 0.8900523560209425,
"fcm_dpo/beta": 0.0011293534189462662,
"fcm_dpo/delta": -0.029587510973215103,
"fcm_dpo/margin": 188.87649536132812,
"fcm_dpo/q_t": 0.4508245587348938,
"grad_norm": 64.6044692993164,
"learning_rate": 1.8594660455706763e-08,
"logits/chosen": -0.7572231888771057,
"logits/rejected": -0.7515621781349182,
"logps/chosen": -693.2191772460938,
"logps/ref_chosen": -280.50909423828125,
"logps/ref_rejected": -276.8252258300781,
"logps/rejected": -878.4116821289062,
"loss": 4.9217,
"margin_dpo/margin_mean": 188.87649536132812,
"margin_dpo/margin_std": 359.30194091796875,
"step": 425
},
{
"KL/chosen_KL_mean": -442.26385498046875,
"KL/mean": -525.7176513671875,
"KL/rejected_KL_mean": -609.1713256835938,
"KL/std": 298.6646728515625,
"epoch": 0.8921465968586387,
"fcm_dpo/beta": 0.0011355035239830613,
"fcm_dpo/delta": 0.027290547266602516,
"fcm_dpo/margin": 166.90748596191406,
"fcm_dpo/q_t": 0.4546186625957489,
"grad_norm": 55.23300552368164,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": -0.7826669812202454,
"logits/rejected": -0.7525278925895691,
"logps/chosen": -735.049072265625,
"logps/ref_chosen": -292.78521728515625,
"logps/ref_rejected": -255.62698364257812,
"logps/rejected": -864.79833984375,
"loss": 4.9927,
"margin_dpo/margin_mean": 166.90748596191406,
"margin_dpo/margin_std": 347.90826416015625,
"step": 426
},
{
"KL/chosen_KL_mean": -442.0292053222656,
"KL/mean": -572.7102661132812,
"KL/rejected_KL_mean": -703.3912353515625,
"KL/std": 374.0984191894531,
"epoch": 0.8942408376963351,
"fcm_dpo/beta": 0.001141170971095562,
"fcm_dpo/delta": -0.0355389341711998,
"fcm_dpo/margin": 261.36199951171875,
"fcm_dpo/q_t": 0.43112075328826904,
"grad_norm": 64.85372924804688,
"learning_rate": 1.7233819424956247e-08,
"logits/chosen": -0.759992778301239,
"logits/rejected": -0.7070995569229126,
"logps/chosen": -730.7979125976562,
"logps/ref_chosen": -288.7687072753906,
"logps/ref_rejected": -268.4986572265625,
"logps/rejected": -971.889892578125,
"loss": 4.7296,
"margin_dpo/margin_mean": 261.36199951171875,
"margin_dpo/margin_std": 463.7349853515625,
"step": 427
},
{
"KL/chosen_KL_mean": -482.4189453125,
"KL/mean": -601.1357421875,
"KL/rejected_KL_mean": -719.8524169921875,
"KL/std": 355.86456298828125,
"epoch": 0.8963350785340314,
"fcm_dpo/beta": 0.0010970557341352105,
"fcm_dpo/delta": -0.02731183171272278,
"fcm_dpo/margin": 237.43350219726562,
"fcm_dpo/q_t": 0.4391084909439087,
"grad_norm": 63.699703216552734,
"learning_rate": 1.6572104647786245e-08,
"logits/chosen": -0.7069486975669861,
"logits/rejected": -0.7331293225288391,
"logps/chosen": -777.93994140625,
"logps/ref_chosen": -295.5209655761719,
"logps/ref_rejected": -275.71026611328125,
"logps/rejected": -995.5628051757812,
"loss": 4.7554,
"margin_dpo/margin_mean": 237.43350219726562,
"margin_dpo/margin_std": 377.74169921875,
"step": 428
},
{
"KL/chosen_KL_mean": -476.097412109375,
"KL/mean": -550.6402587890625,
"KL/rejected_KL_mean": -625.1830444335938,
"KL/std": 336.6489562988281,
"epoch": 0.8984293193717278,
"fcm_dpo/beta": 0.0011044524144381285,
"fcm_dpo/delta": 0.02802928164601326,
"fcm_dpo/margin": 149.08566284179688,
"fcm_dpo/q_t": 0.46070483326911926,
"grad_norm": 84.84398651123047,
"learning_rate": 1.5922907900227017e-08,
"logits/chosen": -0.7112962603569031,
"logits/rejected": -0.7193475365638733,
"logps/chosen": -750.48974609375,
"logps/ref_chosen": -274.392333984375,
"logps/ref_rejected": -258.574462890625,
"logps/rejected": -883.7575073242188,
"loss": 5.1361,
"margin_dpo/margin_mean": 149.08566284179688,
"margin_dpo/margin_std": 409.7275695800781,
"step": 429
},
{
"KL/chosen_KL_mean": -446.72955322265625,
"KL/mean": -537.5370483398438,
"KL/rejected_KL_mean": -628.344482421875,
"KL/std": 328.6794738769531,
"epoch": 0.900523560209424,
"fcm_dpo/beta": 0.0011219914304092526,
"fcm_dpo/delta": 0.002371245063841343,
"fcm_dpo/margin": 181.61495971679688,
"fcm_dpo/q_t": 0.4525456428527832,
"grad_norm": 79.65233612060547,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": -0.7959480285644531,
"logits/rejected": -0.7541887760162354,
"logps/chosen": -735.4686889648438,
"logps/ref_chosen": -288.7391357421875,
"logps/ref_rejected": -268.6106262207031,
"logps/rejected": -896.955078125,
"loss": 4.9536,
"margin_dpo/margin_mean": 181.6149444580078,
"margin_dpo/margin_std": 366.8076477050781,
"step": 430
},
{
"KL/chosen_KL_mean": -441.72039794921875,
"KL/mean": -512.4733276367188,
"KL/rejected_KL_mean": -583.2261962890625,
"KL/std": 299.0691833496094,
"epoch": 0.9026178010471204,
"fcm_dpo/beta": 0.001115912338718772,
"fcm_dpo/delta": -0.009300373494625092,
"fcm_dpo/margin": 141.50582885742188,
"fcm_dpo/q_t": 0.4632718563079834,
"grad_norm": 49.450313568115234,
"learning_rate": 1.4662207078575684e-08,
"logits/chosen": -0.7692259550094604,
"logits/rejected": -0.7354046702384949,
"logps/chosen": -717.4451904296875,
"logps/ref_chosen": -275.7247314453125,
"logps/ref_rejected": -268.91729736328125,
"logps/rejected": -852.1436157226562,
"loss": 5.1099,
"margin_dpo/margin_mean": 141.50582885742188,
"margin_dpo/margin_std": 358.96746826171875,
"step": 431
},
{
"KL/chosen_KL_mean": -396.63092041015625,
"KL/mean": -489.38275146484375,
"KL/rejected_KL_mean": -582.1345825195312,
"KL/std": 305.49884033203125,
"epoch": 0.9047120418848168,
"fcm_dpo/beta": 0.0011289321118965745,
"fcm_dpo/delta": 0.0328177772462368,
"fcm_dpo/margin": 185.50364685058594,
"fcm_dpo/q_t": 0.45187050104141235,
"grad_norm": 48.00371170043945,
"learning_rate": 1.40507706120426e-08,
"logits/chosen": -0.817834734916687,
"logits/rejected": -0.788792073726654,
"logps/chosen": -688.051025390625,
"logps/ref_chosen": -291.42010498046875,
"logps/ref_rejected": -255.48202514648438,
"logps/rejected": -837.6165771484375,
"loss": 4.9446,
"margin_dpo/margin_mean": 185.50363159179688,
"margin_dpo/margin_std": 378.95550537109375,
"step": 432
},
{
"KL/chosen_KL_mean": -459.69891357421875,
"KL/mean": -535.0228271484375,
"KL/rejected_KL_mean": -610.3468017578125,
"KL/std": 313.0767822265625,
"epoch": 0.9068062827225131,
"fcm_dpo/beta": 0.001144950743764639,
"fcm_dpo/delta": 0.013033966533839703,
"fcm_dpo/margin": 150.6478729248047,
"fcm_dpo/q_t": 0.4605104327201843,
"grad_norm": 76.42973327636719,
"learning_rate": 1.345198738661285e-08,
"logits/chosen": -0.7439653277397156,
"logits/rejected": -0.7333388328552246,
"logps/chosen": -705.92578125,
"logps/ref_chosen": -246.2268829345703,
"logps/ref_rejected": -253.65924072265625,
"logps/rejected": -864.0060424804688,
"loss": 5.0636,
"margin_dpo/margin_mean": 150.64788818359375,
"margin_dpo/margin_std": 367.3023376464844,
"step": 433
},
{
"KL/chosen_KL_mean": -501.3629150390625,
"KL/mean": -568.4837036132812,
"KL/rejected_KL_mean": -635.6045532226562,
"KL/std": 315.385986328125,
"epoch": 0.9089005235602095,
"fcm_dpo/beta": 0.001152915065176785,
"fcm_dpo/delta": 0.004903012420982122,
"fcm_dpo/margin": 134.24163818359375,
"fcm_dpo/q_t": 0.4633429944515228,
"grad_norm": 64.78780364990234,
"learning_rate": 1.2865889513213628e-08,
"logits/chosen": -0.7485510110855103,
"logits/rejected": -0.7544351816177368,
"logps/chosen": -796.82470703125,
"logps/ref_chosen": -295.4618225097656,
"logps/ref_rejected": -256.2254333496094,
"logps/rejected": -891.8299560546875,
"loss": 5.1368,
"margin_dpo/margin_mean": 134.2416229248047,
"margin_dpo/margin_std": 361.8308410644531,
"step": 434
},
{
"KL/chosen_KL_mean": -415.9727783203125,
"KL/mean": -528.1788940429688,
"KL/rejected_KL_mean": -640.385009765625,
"KL/std": 337.5413513183594,
"epoch": 0.9109947643979057,
"fcm_dpo/beta": 0.001140053616836667,
"fcm_dpo/delta": -0.03828362748026848,
"fcm_dpo/margin": 224.41220092773438,
"fcm_dpo/q_t": 0.44446510076522827,
"grad_norm": 68.40642547607422,
"learning_rate": 1.2292508422495157e-08,
"logits/chosen": -0.7327988147735596,
"logits/rejected": -0.7096722722053528,
"logps/chosen": -676.711181640625,
"logps/ref_chosen": -260.7384033203125,
"logps/ref_rejected": -248.5688018798828,
"logps/rejected": -888.953857421875,
"loss": 4.8225,
"margin_dpo/margin_mean": 224.41220092773438,
"margin_dpo/margin_std": 394.1939697265625,
"step": 435
},
{
"KL/chosen_KL_mean": -489.31561279296875,
"KL/mean": -566.24462890625,
"KL/rejected_KL_mean": -643.1736450195312,
"KL/std": 384.1330261230469,
"epoch": 0.9130890052356021,
"fcm_dpo/beta": 0.0011170388897880912,
"fcm_dpo/delta": -0.0015771682374179363,
"fcm_dpo/margin": 153.85800170898438,
"fcm_dpo/q_t": 0.4651756286621094,
"grad_norm": 48.129756927490234,
"learning_rate": 1.1731874863145142e-08,
"logits/chosen": -0.7540550231933594,
"logits/rejected": -0.7378331422805786,
"logps/chosen": -808.63818359375,
"logps/ref_chosen": -319.3224792480469,
"logps/ref_rejected": -299.30322265625,
"logps/rejected": -942.4769287109375,
"loss": 5.141,
"margin_dpo/margin_mean": 153.8579864501953,
"margin_dpo/margin_std": 445.167724609375,
"step": 436
},
{
"KL/chosen_KL_mean": -440.3687438964844,
"KL/mean": -549.68701171875,
"KL/rejected_KL_mean": -659.005126953125,
"KL/std": 355.17633056640625,
"epoch": 0.9151832460732985,
"fcm_dpo/beta": 0.0011156515683978796,
"fcm_dpo/delta": -0.011046146973967552,
"fcm_dpo/margin": 218.63638305664062,
"fcm_dpo/q_t": 0.44490429759025574,
"grad_norm": 69.38978576660156,
"learning_rate": 1.118401890024001e-08,
"logits/chosen": -0.7746818661689758,
"logits/rejected": -0.742755651473999,
"logps/chosen": -719.197509765625,
"logps/ref_chosen": -278.82879638671875,
"logps/ref_rejected": -272.55303955078125,
"logps/rejected": -931.5582275390625,
"loss": 4.8757,
"margin_dpo/margin_mean": 218.6363983154297,
"margin_dpo/margin_std": 423.87152099609375,
"step": 437
},
{
"KL/chosen_KL_mean": -474.75103759765625,
"KL/mean": -517.1573486328125,
"KL/rejected_KL_mean": -559.5636596679688,
"KL/std": 331.350341796875,
"epoch": 0.9172774869109948,
"fcm_dpo/beta": 0.001117004663683474,
"fcm_dpo/delta": 0.006466980557888746,
"fcm_dpo/margin": 84.8126449584961,
"fcm_dpo/q_t": 0.47826677560806274,
"grad_norm": 88.04359436035156,
"learning_rate": 1.06489699136324e-08,
"logits/chosen": -0.7461004257202148,
"logits/rejected": -0.7531372308731079,
"logps/chosen": -734.070068359375,
"logps/ref_chosen": -259.31903076171875,
"logps/ref_rejected": -240.99581909179688,
"logps/rejected": -800.5595092773438,
"loss": 5.3795,
"margin_dpo/margin_mean": 84.81265258789062,
"margin_dpo/margin_std": 385.8534851074219,
"step": 438
},
{
"KL/chosen_KL_mean": -444.52960205078125,
"KL/mean": -533.1126708984375,
"KL/rejected_KL_mean": -621.6957397460938,
"KL/std": 293.2552795410156,
"epoch": 0.9193717277486911,
"fcm_dpo/beta": 0.001108786673285067,
"fcm_dpo/delta": -0.006224591284990311,
"fcm_dpo/margin": 177.16610717773438,
"fcm_dpo/q_t": 0.45308101177215576,
"grad_norm": 51.11709213256836,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": -0.7281197905540466,
"logits/rejected": -0.7132576107978821,
"logps/chosen": -701.6539916992188,
"logps/ref_chosen": -257.1243896484375,
"logps/ref_rejected": -243.20416259765625,
"logps/rejected": -864.89990234375,
"loss": 4.9507,
"margin_dpo/margin_mean": 177.1660919189453,
"margin_dpo/margin_std": 342.137451171875,
"step": 439
},
{
"KL/chosen_KL_mean": -474.2215576171875,
"KL/mean": -542.44384765625,
"KL/rejected_KL_mean": -610.666259765625,
"KL/std": 283.33172607421875,
"epoch": 0.9214659685863874,
"fcm_dpo/beta": 0.0011141917202621698,
"fcm_dpo/delta": 0.01364380493760109,
"fcm_dpo/margin": 136.44468688964844,
"fcm_dpo/q_t": 0.46402812004089355,
"grad_norm": 48.65362548828125,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": -0.8012443780899048,
"logits/rejected": -0.7894343733787537,
"logps/chosen": -781.7530517578125,
"logps/ref_chosen": -307.5315246582031,
"logps/ref_rejected": -264.3540954589844,
"logps/rejected": -875.0203857421875,
"loss": 5.1135,
"margin_dpo/margin_mean": 136.44468688964844,
"margin_dpo/margin_std": 327.4635009765625,
"step": 440
},
{
"KL/chosen_KL_mean": -447.04949951171875,
"KL/mean": -549.7024536132812,
"KL/rejected_KL_mean": -652.3553466796875,
"KL/std": 306.68316650390625,
"epoch": 0.9235602094240838,
"fcm_dpo/beta": 0.001123150810599327,
"fcm_dpo/delta": -0.0012265940895304084,
"fcm_dpo/margin": 205.3058624267578,
"fcm_dpo/q_t": 0.44529205560684204,
"grad_norm": 68.43964385986328,
"learning_rate": 9.12094829893642e-09,
"logits/chosen": -0.7399640679359436,
"logits/rejected": -0.7328054308891296,
"logps/chosen": -757.0314331054688,
"logps/ref_chosen": -309.9819641113281,
"logps/ref_rejected": -297.4968566894531,
"logps/rejected": -949.852294921875,
"loss": 4.8237,
"margin_dpo/margin_mean": 205.30584716796875,
"margin_dpo/margin_std": 332.4583435058594,
"step": 441
},
{
"KL/chosen_KL_mean": -504.6666564941406,
"KL/mean": -573.6967163085938,
"KL/rejected_KL_mean": -642.726806640625,
"KL/std": 340.78564453125,
"epoch": 0.9256544502617801,
"fcm_dpo/beta": 0.0011333015281707048,
"fcm_dpo/delta": 0.019338196143507957,
"fcm_dpo/margin": 138.06015014648438,
"fcm_dpo/q_t": 0.46324992179870605,
"grad_norm": 47.62255096435547,
"learning_rate": 8.637407257200496e-09,
"logits/chosen": -0.8079502582550049,
"logits/rejected": -0.7644075751304626,
"logps/chosen": -783.645751953125,
"logps/ref_chosen": -278.9791564941406,
"logps/ref_rejected": -242.87310791015625,
"logps/rejected": -885.5999755859375,
"loss": 5.1402,
"margin_dpo/margin_mean": 138.06015014648438,
"margin_dpo/margin_std": 388.3710632324219,
"step": 442
},
{
"KL/chosen_KL_mean": -456.517822265625,
"KL/mean": -535.2323608398438,
"KL/rejected_KL_mean": -613.947021484375,
"KL/std": 287.5276794433594,
"epoch": 0.9277486910994764,
"fcm_dpo/beta": 0.0011378020280972123,
"fcm_dpo/delta": -0.008695745840668678,
"fcm_dpo/margin": 157.42919921875,
"fcm_dpo/q_t": 0.45707207918167114,
"grad_norm": 54.09843826293945,
"learning_rate": 8.166809758815895e-09,
"logits/chosen": -0.6914748549461365,
"logits/rejected": -0.7063671946525574,
"logps/chosen": -730.0767822265625,
"logps/ref_chosen": -273.5590515136719,
"logps/ref_rejected": -264.0199279785156,
"logps/rejected": -877.9669799804688,
"loss": 5.0316,
"margin_dpo/margin_mean": 157.42919921875,
"margin_dpo/margin_std": 345.0313415527344,
"step": 443
},
{
"KL/chosen_KL_mean": -447.2994079589844,
"KL/mean": -540.459716796875,
"KL/rejected_KL_mean": -633.6199951171875,
"KL/std": 347.92816162109375,
"epoch": 0.9298429319371728,
"fcm_dpo/beta": 0.0011359085328876972,
"fcm_dpo/delta": 0.008714323863387108,
"fcm_dpo/margin": 186.3206329345703,
"fcm_dpo/q_t": 0.45250868797302246,
"grad_norm": 54.263832092285156,
"learning_rate": 7.709181040498253e-09,
"logits/chosen": -0.7050941586494446,
"logits/rejected": -0.6862058639526367,
"logps/chosen": -745.443603515625,
"logps/ref_chosen": -298.1441955566406,
"logps/ref_rejected": -268.0572814941406,
"logps/rejected": -901.67724609375,
"loss": 4.977,
"margin_dpo/margin_mean": 186.3206329345703,
"margin_dpo/margin_std": 420.0409851074219,
"step": 444
},
{
"KL/chosen_KL_mean": -447.3858642578125,
"KL/mean": -535.5361938476562,
"KL/rejected_KL_mean": -623.6864013671875,
"KL/std": 328.20489501953125,
"epoch": 0.9319371727748691,
"fcm_dpo/beta": 0.0011325681116431952,
"fcm_dpo/delta": -0.020159322768449783,
"fcm_dpo/margin": 176.30056762695312,
"fcm_dpo/q_t": 0.4535037875175476,
"grad_norm": 53.423500061035156,
"learning_rate": 7.2645456434869965e-09,
"logits/chosen": -0.7933753728866577,
"logits/rejected": -0.7875576019287109,
"logps/chosen": -701.9265747070312,
"logps/ref_chosen": -254.54067993164062,
"logps/ref_rejected": -264.2445983886719,
"logps/rejected": -887.9310913085938,
"loss": 5.007,
"margin_dpo/margin_mean": 176.30055236816406,
"margin_dpo/margin_std": 394.967041015625,
"step": 445
},
{
"KL/chosen_KL_mean": -436.194091796875,
"KL/mean": -528.354248046875,
"KL/rejected_KL_mean": -620.514404296875,
"KL/std": 328.3664855957031,
"epoch": 0.9340314136125655,
"fcm_dpo/beta": 0.001101514557376504,
"fcm_dpo/delta": -0.017040304839611053,
"fcm_dpo/margin": 184.32025146484375,
"fcm_dpo/q_t": 0.4546178877353668,
"grad_norm": 41.92047882080078,
"learning_rate": 6.832927412229017e-09,
"logits/chosen": -0.7097505331039429,
"logits/rejected": -0.7096059322357178,
"logps/chosen": -742.9165649414062,
"logps/ref_chosen": -306.72247314453125,
"logps/ref_rejected": -266.3735656738281,
"logps/rejected": -886.8878784179688,
"loss": 4.9714,
"margin_dpo/margin_mean": 184.32025146484375,
"margin_dpo/margin_std": 382.3779602050781,
"step": 446
},
{
"KL/chosen_KL_mean": -450.354736328125,
"KL/mean": -550.6195678710938,
"KL/rejected_KL_mean": -650.8844604492188,
"KL/std": 328.66229248046875,
"epoch": 0.9361256544502617,
"fcm_dpo/beta": 0.0010986090637743473,
"fcm_dpo/delta": -0.006220587063580751,
"fcm_dpo/margin": 200.52969360351562,
"fcm_dpo/q_t": 0.4488736689090729,
"grad_norm": 46.463623046875,
"learning_rate": 6.414349493100129e-09,
"logits/chosen": -0.7155401706695557,
"logits/rejected": -0.7002200484275818,
"logps/chosen": -710.8720703125,
"logps/ref_chosen": -260.51727294921875,
"logps/ref_rejected": -236.47061157226562,
"logps/rejected": -887.3550415039062,
"loss": 4.9114,
"margin_dpo/margin_mean": 200.52969360351562,
"margin_dpo/margin_std": 392.89251708984375,
"step": 447
},
{
"KL/chosen_KL_mean": -461.6020202636719,
"KL/mean": -561.8016967773438,
"KL/rejected_KL_mean": -662.00146484375,
"KL/std": 313.98712158203125,
"epoch": 0.9382198952879581,
"fcm_dpo/beta": 0.001077001099474728,
"fcm_dpo/delta": -0.028215978294610977,
"fcm_dpo/margin": 200.39947509765625,
"fcm_dpo/q_t": 0.4488033652305603,
"grad_norm": 54.6915168762207,
"learning_rate": 6.0088343331638756e-09,
"logits/chosen": -0.7094584107398987,
"logits/rejected": -0.6863497495651245,
"logps/chosen": -730.3890991210938,
"logps/ref_chosen": -268.78704833984375,
"logps/ref_rejected": -262.1703796386719,
"logps/rejected": -924.171875,
"loss": 4.8867,
"margin_dpo/margin_mean": 200.39947509765625,
"margin_dpo/margin_std": 350.0815124511719,
"step": 448
},
{
"KL/chosen_KL_mean": -462.7024230957031,
"KL/mean": -540.6029663085938,
"KL/rejected_KL_mean": -618.5035400390625,
"KL/std": 322.6525573730469,
"epoch": 0.9403141361256544,
"fcm_dpo/beta": 0.0010794418631121516,
"fcm_dpo/delta": 0.01929195038974285,
"fcm_dpo/margin": 155.8011016845703,
"fcm_dpo/q_t": 0.45973268151283264,
"grad_norm": 43.563201904296875,
"learning_rate": 5.616403678967624e-09,
"logits/chosen": -0.835517168045044,
"logits/rejected": -0.8153800368309021,
"logps/chosen": -793.65380859375,
"logps/ref_chosen": -330.9514465332031,
"logps/ref_rejected": -239.76974487304688,
"logps/rejected": -858.2732543945312,
"loss": 5.074,
"margin_dpo/margin_mean": 155.80111694335938,
"margin_dpo/margin_std": 373.4229431152344,
"step": 449
},
{
"KL/chosen_KL_mean": -485.2821044921875,
"KL/mean": -559.709228515625,
"KL/rejected_KL_mean": -634.1363525390625,
"KL/std": 333.56817626953125,
"epoch": 0.9424083769633508,
"fcm_dpo/beta": 0.0010939650237560272,
"fcm_dpo/delta": 0.0062581561505794525,
"fcm_dpo/margin": 148.8542938232422,
"fcm_dpo/q_t": 0.46235862374305725,
"grad_norm": 54.444618225097656,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": -0.7135239243507385,
"logits/rejected": -0.6962549090385437,
"logps/chosen": -769.5475463867188,
"logps/ref_chosen": -284.26544189453125,
"logps/ref_rejected": -250.5401611328125,
"logps/rejected": -884.6766357421875,
"loss": 5.0828,
"margin_dpo/margin_mean": 148.8542938232422,
"margin_dpo/margin_std": 354.3857421875,
"step": 450
},
{
"KL/chosen_KL_mean": -498.91839599609375,
"KL/mean": -550.2883911132812,
"KL/rejected_KL_mean": -601.6583251953125,
"KL/std": 294.4558410644531,
"epoch": 0.9445026178010472,
"fcm_dpo/beta": 0.0010946807451546192,
"fcm_dpo/delta": -0.0018840640550479293,
"fcm_dpo/margin": 102.73995971679688,
"fcm_dpo/q_t": 0.4727487862110138,
"grad_norm": 80.72921752929688,
"learning_rate": 4.8708793644441086e-09,
"logits/chosen": -0.7374417185783386,
"logits/rejected": -0.702163815498352,
"logps/chosen": -801.2393798828125,
"logps/ref_chosen": -302.3209228515625,
"logps/ref_rejected": -254.09747314453125,
"logps/rejected": -855.7559814453125,
"loss": 5.2356,
"margin_dpo/margin_mean": 102.73995971679688,
"margin_dpo/margin_std": 311.53643798828125,
"step": 451
},
{
"KL/chosen_KL_mean": -456.5367126464844,
"KL/mean": -576.4017333984375,
"KL/rejected_KL_mean": -696.2666015625,
"KL/std": 430.700927734375,
"epoch": 0.9465968586387434,
"fcm_dpo/beta": 0.0010616311337798834,
"fcm_dpo/delta": -0.04442259296774864,
"fcm_dpo/margin": 239.7299346923828,
"fcm_dpo/q_t": 0.4475480318069458,
"grad_norm": 47.51118850708008,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": -0.7811554670333862,
"logits/rejected": -0.7392996549606323,
"logps/chosen": -755.9288330078125,
"logps/ref_chosen": -299.39215087890625,
"logps/ref_rejected": -284.3475036621094,
"logps/rejected": -980.6141967773438,
"loss": 4.9112,
"margin_dpo/margin_mean": 239.72991943359375,
"margin_dpo/margin_std": 534.2335205078125,
"step": 452
},
{
"KL/chosen_KL_mean": -476.1728820800781,
"KL/mean": -562.9217529296875,
"KL/rejected_KL_mean": -649.6705322265625,
"KL/std": 316.76861572265625,
"epoch": 0.9486910994764398,
"fcm_dpo/beta": 0.0010666890302672982,
"fcm_dpo/delta": 0.020303022116422653,
"fcm_dpo/margin": 173.4977264404297,
"fcm_dpo/q_t": 0.45698270201683044,
"grad_norm": 51.801551818847656,
"learning_rate": 4.1779364682113794e-09,
"logits/chosen": -0.7304836511611938,
"logits/rejected": -0.7136521339416504,
"logps/chosen": -800.8245239257812,
"logps/ref_chosen": -324.6517028808594,
"logps/ref_rejected": -304.1527099609375,
"logps/rejected": -953.8233032226562,
"loss": 5.0035,
"margin_dpo/margin_mean": 173.4976806640625,
"margin_dpo/margin_std": 376.6050109863281,
"step": 453
},
{
"KL/chosen_KL_mean": -467.48114013671875,
"KL/mean": -588.2965087890625,
"KL/rejected_KL_mean": -709.1119995117188,
"KL/std": 378.76043701171875,
"epoch": 0.9507853403141361,
"fcm_dpo/beta": 0.001057203160598874,
"fcm_dpo/delta": -0.029810752719640732,
"fcm_dpo/margin": 241.630859375,
"fcm_dpo/q_t": 0.44075632095336914,
"grad_norm": 77.30159759521484,
"learning_rate": 3.851229943335393e-09,
"logits/chosen": -0.7590242624282837,
"logits/rejected": -0.750261664390564,
"logps/chosen": -767.0928955078125,
"logps/ref_chosen": -299.6117248535156,
"logps/ref_rejected": -303.74224853515625,
"logps/rejected": -1012.854248046875,
"loss": 4.8069,
"margin_dpo/margin_mean": 241.630859375,
"margin_dpo/margin_std": 433.0992431640625,
"step": 454
},
{
"KL/chosen_KL_mean": -456.4439697265625,
"KL/mean": -515.0791015625,
"KL/rejected_KL_mean": -573.7142944335938,
"KL/std": 320.5881652832031,
"epoch": 0.9528795811518325,
"fcm_dpo/beta": 0.0010561775416135788,
"fcm_dpo/delta": 0.01066396851092577,
"fcm_dpo/margin": 117.27033233642578,
"fcm_dpo/q_t": 0.4721914529800415,
"grad_norm": 46.713623046875,
"learning_rate": 3.5377236299748147e-09,
"logits/chosen": -0.7215044498443604,
"logits/rejected": -0.7258639335632324,
"logps/chosen": -730.0556640625,
"logps/ref_chosen": -273.6116943359375,
"logps/ref_rejected": -274.4293518066406,
"logps/rejected": -848.1436157226562,
"loss": 5.2412,
"margin_dpo/margin_mean": 117.27033233642578,
"margin_dpo/margin_std": 370.8341369628906,
"step": 455
},
{
"KL/chosen_KL_mean": -489.3581848144531,
"KL/mean": -581.25244140625,
"KL/rejected_KL_mean": -673.1466674804688,
"KL/std": 343.29412841796875,
"epoch": 0.9549738219895288,
"fcm_dpo/beta": 0.001065379474312067,
"fcm_dpo/delta": 0.016431938856840134,
"fcm_dpo/margin": 183.78857421875,
"fcm_dpo/q_t": 0.455960214138031,
"grad_norm": 42.45829391479492,
"learning_rate": 3.2374343405217884e-09,
"logits/chosen": -0.6549510955810547,
"logits/rejected": -0.6670107841491699,
"logps/chosen": -811.5300903320312,
"logps/ref_chosen": -322.17193603515625,
"logps/ref_rejected": -294.54461669921875,
"logps/rejected": -967.6912841796875,
"loss": 5.0033,
"margin_dpo/margin_mean": 183.78857421875,
"margin_dpo/margin_std": 419.4525146484375,
"step": 456
},
{
"KL/chosen_KL_mean": -504.164306640625,
"KL/mean": -611.7514038085938,
"KL/rejected_KL_mean": -719.3384399414062,
"KL/std": 379.7641906738281,
"epoch": 0.9570680628272251,
"fcm_dpo/beta": 0.0010634178761392832,
"fcm_dpo/delta": -0.008057055063545704,
"fcm_dpo/margin": 215.17417907714844,
"fcm_dpo/q_t": 0.44393303990364075,
"grad_norm": 74.09278106689453,
"learning_rate": 2.9503781785795713e-09,
"logits/chosen": -0.7260345816612244,
"logits/rejected": -0.722815752029419,
"logps/chosen": -811.9605712890625,
"logps/ref_chosen": -307.7962341308594,
"logps/ref_rejected": -274.5501403808594,
"logps/rejected": -993.888671875,
"loss": 4.9483,
"margin_dpo/margin_mean": 215.17417907714844,
"margin_dpo/margin_std": 461.9056396484375,
"step": 457
},
{
"KL/chosen_KL_mean": -467.6044616699219,
"KL/mean": -549.5364379882812,
"KL/rejected_KL_mean": -631.4684448242188,
"KL/std": 351.8533630371094,
"epoch": 0.9591623036649215,
"fcm_dpo/beta": 0.0010722451843321323,
"fcm_dpo/delta": 0.013735459186136723,
"fcm_dpo/margin": 163.86398315429688,
"fcm_dpo/q_t": 0.45987468957901,
"grad_norm": 42.21095275878906,
"learning_rate": 2.6765705380989432e-09,
"logits/chosen": -0.7559491395950317,
"logits/rejected": -0.7171483635902405,
"logps/chosen": -764.6361694335938,
"logps/ref_chosen": -297.0316467285156,
"logps/ref_rejected": -276.1112365722656,
"logps/rejected": -907.5797119140625,
"loss": 5.0922,
"margin_dpo/margin_mean": 163.86398315429688,
"margin_dpo/margin_std": 412.696044921875,
"step": 458
},
{
"KL/chosen_KL_mean": -458.0815124511719,
"KL/mean": -525.6103515625,
"KL/rejected_KL_mean": -593.1392822265625,
"KL/std": 330.22552490234375,
"epoch": 0.9612565445026178,
"fcm_dpo/beta": 0.0010813046246767044,
"fcm_dpo/delta": 0.006299625150859356,
"fcm_dpo/margin": 135.0577392578125,
"fcm_dpo/q_t": 0.46749192476272583,
"grad_norm": 58.347652435302734,
"learning_rate": 2.416026102552732e-09,
"logits/chosen": -0.8096132874488831,
"logits/rejected": -0.8010511994361877,
"logps/chosen": -751.6068115234375,
"logps/ref_chosen": -293.5252990722656,
"logps/ref_rejected": -289.30126953125,
"logps/rejected": -882.4405517578125,
"loss": 5.1984,
"margin_dpo/margin_mean": 135.05772399902344,
"margin_dpo/margin_std": 416.4771728515625,
"step": 459
},
{
"KL/chosen_KL_mean": -477.0860290527344,
"KL/mean": -545.5380859375,
"KL/rejected_KL_mean": -613.9900512695312,
"KL/std": 299.5758361816406,
"epoch": 0.9633507853403142,
"fcm_dpo/beta": 0.0010851901024580002,
"fcm_dpo/delta": 0.008401179686188698,
"fcm_dpo/margin": 136.9040069580078,
"fcm_dpo/q_t": 0.46470630168914795,
"grad_norm": 72.48368072509766,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": -0.7803556323051453,
"logits/rejected": -0.7745279669761658,
"logps/chosen": -795.866455078125,
"logps/ref_chosen": -318.7803649902344,
"logps/ref_rejected": -258.7906799316406,
"logps/rejected": -872.7807006835938,
"loss": 5.1416,
"margin_dpo/margin_mean": 136.9040069580078,
"margin_dpo/margin_std": 376.03265380859375,
"step": 460
},
{
"KL/chosen_KL_mean": -461.55438232421875,
"KL/mean": -552.3533325195312,
"KL/rejected_KL_mean": -643.15234375,
"KL/std": 352.3241882324219,
"epoch": 0.9654450261780104,
"fcm_dpo/beta": 0.0010989790316671133,
"fcm_dpo/delta": 0.023482128977775574,
"fcm_dpo/margin": 181.59786987304688,
"fcm_dpo/q_t": 0.4545283019542694,
"grad_norm": 63.46205520629883,
"learning_rate": 1.9347820230782295e-09,
"logits/chosen": -0.7056773900985718,
"logits/rejected": -0.7202193140983582,
"logps/chosen": -705.4644165039062,
"logps/ref_chosen": -243.9099884033203,
"logps/ref_rejected": -232.6382293701172,
"logps/rejected": -875.7904663085938,
"loss": 5.0086,
"margin_dpo/margin_mean": 181.59786987304688,
"margin_dpo/margin_std": 421.58538818359375,
"step": 461
},
{
"KL/chosen_KL_mean": -449.98541259765625,
"KL/mean": -550.90478515625,
"KL/rejected_KL_mean": -651.82421875,
"KL/std": 304.707763671875,
"epoch": 0.9675392670157068,
"fcm_dpo/beta": 0.0011157675180584192,
"fcm_dpo/delta": 0.003436810802668333,
"fcm_dpo/margin": 201.8387908935547,
"fcm_dpo/q_t": 0.44782936573028564,
"grad_norm": 72.2857437133789,
"learning_rate": 1.7141081868094209e-09,
"logits/chosen": -0.7916049957275391,
"logits/rejected": -0.7290614247322083,
"logps/chosen": -794.076416015625,
"logps/ref_chosen": -344.09100341796875,
"logps/ref_rejected": -252.45037841796875,
"logps/rejected": -904.274658203125,
"loss": 4.8414,
"margin_dpo/margin_mean": 201.8387908935547,
"margin_dpo/margin_std": 333.3164978027344,
"step": 462
},
{
"KL/chosen_KL_mean": -470.310302734375,
"KL/mean": -540.879638671875,
"KL/rejected_KL_mean": -611.4489135742188,
"KL/std": 348.91265869140625,
"epoch": 0.9696335078534032,
"fcm_dpo/beta": 0.0011070480104535818,
"fcm_dpo/delta": -0.016562294214963913,
"fcm_dpo/margin": 141.1386260986328,
"fcm_dpo/q_t": 0.4637291133403778,
"grad_norm": 66.43771362304688,
"learning_rate": 1.5067491694100153e-09,
"logits/chosen": -0.7600305080413818,
"logits/rejected": -0.7077437043190002,
"logps/chosen": -767.4527587890625,
"logps/ref_chosen": -297.1424560546875,
"logps/ref_rejected": -234.0208282470703,
"logps/rejected": -845.4697875976562,
"loss": 5.174,
"margin_dpo/margin_mean": 141.1386260986328,
"margin_dpo/margin_std": 422.1974182128906,
"step": 463
},
{
"KL/chosen_KL_mean": -452.6793518066406,
"KL/mean": -539.170166015625,
"KL/rejected_KL_mean": -625.6610107421875,
"KL/std": 322.0829162597656,
"epoch": 0.9717277486910995,
"fcm_dpo/beta": 0.0011113593354821205,
"fcm_dpo/delta": 0.016370417550206184,
"fcm_dpo/margin": 172.981689453125,
"fcm_dpo/q_t": 0.4547066390514374,
"grad_norm": 65.45187377929688,
"learning_rate": 1.3127160909147672e-09,
"logits/chosen": -0.7473437190055847,
"logits/rejected": -0.7489007115364075,
"logps/chosen": -718.3900756835938,
"logps/ref_chosen": -265.71075439453125,
"logps/ref_rejected": -256.4108581542969,
"logps/rejected": -882.0718994140625,
"loss": 4.9785,
"margin_dpo/margin_mean": 172.98167419433594,
"margin_dpo/margin_std": 365.59429931640625,
"step": 464
},
{
"KL/chosen_KL_mean": -431.3951416015625,
"KL/mean": -539.25048828125,
"KL/rejected_KL_mean": -647.1058349609375,
"KL/std": 306.6711730957031,
"epoch": 0.9738219895287958,
"fcm_dpo/beta": 0.0010858203750103712,
"fcm_dpo/delta": -0.023821821436285973,
"fcm_dpo/margin": 215.71067810058594,
"fcm_dpo/q_t": 0.4461546540260315,
"grad_norm": 58.74069595336914,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": -0.8197015523910522,
"logits/rejected": -0.7692691683769226,
"logps/chosen": -724.5478515625,
"logps/ref_chosen": -293.1527404785156,
"logps/ref_rejected": -293.70947265625,
"logps/rejected": -940.8153686523438,
"loss": 4.8416,
"margin_dpo/margin_mean": 215.710693359375,
"margin_dpo/margin_std": 361.2246398925781,
"step": 465
},
{
"KL/chosen_KL_mean": -410.092041015625,
"KL/mean": -514.951416015625,
"KL/rejected_KL_mean": -619.8109130859375,
"KL/std": 301.6080322265625,
"epoch": 0.9759162303664921,
"fcm_dpo/beta": 0.0010852470295503736,
"fcm_dpo/delta": -0.006281266920268536,
"fcm_dpo/margin": 209.71878051757812,
"fcm_dpo/q_t": 0.44774800539016724,
"grad_norm": 50.34075927734375,
"learning_rate": 9.64668657069706e-10,
"logits/chosen": -0.7202876210212708,
"logits/rejected": -0.6470254063606262,
"logps/chosen": -671.5696411132812,
"logps/ref_chosen": -261.4775695800781,
"logps/ref_rejected": -248.36282348632812,
"logps/rejected": -868.1737060546875,
"loss": 4.8706,
"margin_dpo/margin_mean": 209.71878051757812,
"margin_dpo/margin_std": 377.7794494628906,
"step": 466
},
{
"KL/chosen_KL_mean": -453.5052795410156,
"KL/mean": -544.7555541992188,
"KL/rejected_KL_mean": -636.0057373046875,
"KL/std": 345.9569396972656,
"epoch": 0.9780104712041885,
"fcm_dpo/beta": 0.0010806998470798135,
"fcm_dpo/delta": -0.003298811614513397,
"fcm_dpo/margin": 182.50048828125,
"fcm_dpo/q_t": 0.4551469683647156,
"grad_norm": 58.7347412109375,
"learning_rate": 8.106729664475176e-10,
"logits/chosen": -0.7169447541236877,
"logits/rejected": -0.6979930400848389,
"logps/chosen": -719.8595581054688,
"logps/ref_chosen": -266.354248046875,
"logps/ref_rejected": -277.76324462890625,
"logps/rejected": -913.7689819335938,
"loss": 5.0016,
"margin_dpo/margin_mean": 182.50045776367188,
"margin_dpo/margin_std": 418.3126220703125,
"step": 467
},
{
"KL/chosen_KL_mean": -473.4375,
"KL/mean": -515.6995849609375,
"KL/rejected_KL_mean": -557.9615478515625,
"KL/std": 294.5906677246094,
"epoch": 0.9801047120418848,
"fcm_dpo/beta": 0.0010830379324033856,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 84.52410888671875,
"fcm_dpo/q_t": 0.47659796476364136,
"grad_norm": 90.00639343261719,
"learning_rate": 6.700405431837585e-10,
"logits/chosen": -0.7935588955879211,
"logits/rejected": -0.7631844282150269,
"logps/chosen": -791.4006958007812,
"logps/ref_chosen": -317.9631652832031,
"logps/ref_rejected": -261.8744201660156,
"logps/rejected": -819.8359985351562,
"loss": 5.3815,
"margin_dpo/margin_mean": 84.52410125732422,
"margin_dpo/margin_std": 381.2652587890625,
"step": 468
},
{
"KL/chosen_KL_mean": -442.52105712890625,
"KL/mean": -534.7864379882812,
"KL/rejected_KL_mean": -627.0517578125,
"KL/std": 324.3078918457031,
"epoch": 0.9821989528795811,
"fcm_dpo/beta": 0.001092868042178452,
"fcm_dpo/delta": 0.0077682118862867355,
"fcm_dpo/margin": 184.53070068359375,
"fcm_dpo/q_t": 0.45302706956863403,
"grad_norm": 45.122615814208984,
"learning_rate": 5.427789289685347e-10,
"logits/chosen": -0.718509316444397,
"logits/rejected": -0.6958325505256653,
"logps/chosen": -767.4078369140625,
"logps/ref_chosen": -324.8868103027344,
"logps/ref_rejected": -264.0421447753906,
"logps/rejected": -891.0939331054688,
"loss": 4.9723,
"margin_dpo/margin_mean": 184.53070068359375,
"margin_dpo/margin_std": 391.1455383300781,
"step": 469
},
{
"KL/chosen_KL_mean": -439.5545654296875,
"KL/mean": -550.50146484375,
"KL/rejected_KL_mean": -661.4483642578125,
"KL/std": 344.94183349609375,
"epoch": 0.9842931937172775,
"fcm_dpo/beta": 0.0010674743680283427,
"fcm_dpo/delta": -0.032233696430921555,
"fcm_dpo/margin": 221.89381408691406,
"fcm_dpo/q_t": 0.44584980607032776,
"grad_norm": 44.609622955322266,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": -0.7209665775299072,
"logits/rejected": -0.710899293422699,
"logps/chosen": -754.2588500976562,
"logps/ref_chosen": -314.7042541503906,
"logps/ref_rejected": -259.2276611328125,
"logps/rejected": -920.6760864257812,
"loss": 4.8792,
"margin_dpo/margin_mean": 221.893798828125,
"margin_dpo/margin_std": 426.10504150390625,
"step": 470
},
{
"KL/chosen_KL_mean": -462.39056396484375,
"KL/mean": -570.186279296875,
"KL/rejected_KL_mean": -677.98193359375,
"KL/std": 388.74066162109375,
"epoch": 0.9863874345549738,
"fcm_dpo/beta": 0.0010593057377263904,
"fcm_dpo/delta": -0.02124343067407608,
"fcm_dpo/margin": 215.5913848876953,
"fcm_dpo/q_t": 0.45041680335998535,
"grad_norm": 76.77764892578125,
"learning_rate": 3.2839470889836627e-10,
"logits/chosen": -0.7802831530570984,
"logits/rejected": -0.7523050308227539,
"logps/chosen": -754.9653930664062,
"logps/ref_chosen": -292.5748291015625,
"logps/ref_rejected": -298.7525329589844,
"logps/rejected": -976.7344970703125,
"loss": 4.9355,
"margin_dpo/margin_mean": 215.5913848876953,
"margin_dpo/margin_std": 460.085693359375,
"step": 471
},
{
"KL/chosen_KL_mean": -429.9429931640625,
"KL/mean": -550.7857666015625,
"KL/rejected_KL_mean": -671.6284790039062,
"KL/std": 356.4366760253906,
"epoch": 0.9884816753926702,
"fcm_dpo/beta": 0.0010364714544266462,
"fcm_dpo/delta": -0.011188654229044914,
"fcm_dpo/margin": 241.68548583984375,
"fcm_dpo/q_t": 0.44299882650375366,
"grad_norm": 62.40871810913086,
"learning_rate": 2.412835998185092e-10,
"logits/chosen": -0.7540468573570251,
"logits/rejected": -0.7562943696975708,
"logps/chosen": -673.3168334960938,
"logps/ref_chosen": -243.37380981445312,
"logps/ref_rejected": -251.12109375,
"logps/rejected": -922.7495727539062,
"loss": 4.8288,
"margin_dpo/margin_mean": 241.68548583984375,
"margin_dpo/margin_std": 443.82684326171875,
"step": 472
},
{
"KL/chosen_KL_mean": -455.9682922363281,
"KL/mean": -553.0786743164062,
"KL/rejected_KL_mean": -650.1890869140625,
"KL/std": 289.64495849609375,
"epoch": 0.9905759162303664,
"fcm_dpo/beta": 0.0010364020708948374,
"fcm_dpo/delta": 0.0068814936093986034,
"fcm_dpo/margin": 194.22076416015625,
"fcm_dpo/q_t": 0.4516716003417969,
"grad_norm": 58.57929992675781,
"learning_rate": 1.6756629272085544e-10,
"logits/chosen": -0.7046954035758972,
"logits/rejected": -0.694038987159729,
"logps/chosen": -742.2969360351562,
"logps/ref_chosen": -286.3286437988281,
"logps/ref_rejected": -258.6535339355469,
"logps/rejected": -908.8426513671875,
"loss": 4.9053,
"margin_dpo/margin_mean": 194.22076416015625,
"margin_dpo/margin_std": 324.9159240722656,
"step": 473
},
{
"KL/chosen_KL_mean": -456.2850341796875,
"KL/mean": -544.32763671875,
"KL/rejected_KL_mean": -632.3701782226562,
"KL/std": 346.0625305175781,
"epoch": 0.9926701570680628,
"fcm_dpo/beta": 0.0010376791469752789,
"fcm_dpo/delta": -0.004413094371557236,
"fcm_dpo/margin": 176.0851287841797,
"fcm_dpo/q_t": 0.45776984095573425,
"grad_norm": 48.36650085449219,
"learning_rate": 1.072467408408384e-10,
"logits/chosen": -0.7669036984443665,
"logits/rejected": -0.7505558729171753,
"logps/chosen": -744.3746337890625,
"logps/ref_chosen": -288.08966064453125,
"logps/ref_rejected": -266.69696044921875,
"logps/rejected": -899.067138671875,
"loss": 5.0173,
"margin_dpo/margin_mean": 176.0851287841797,
"margin_dpo/margin_std": 386.46929931640625,
"step": 474
},
{
"KL/chosen_KL_mean": -405.2410583496094,
"KL/mean": -492.22198486328125,
"KL/rejected_KL_mean": -579.202880859375,
"KL/std": 290.22174072265625,
"epoch": 0.9947643979057592,
"fcm_dpo/beta": 0.0010348742362111807,
"fcm_dpo/delta": -0.004362727515399456,
"fcm_dpo/margin": 173.9619140625,
"fcm_dpo/q_t": 0.4581862986087799,
"grad_norm": 47.80220413208008,
"learning_rate": 6.032817893297793e-11,
"logits/chosen": -0.7423918843269348,
"logits/rejected": -0.7410845756530762,
"logps/chosen": -661.244140625,
"logps/ref_chosen": -256.0030517578125,
"logps/ref_rejected": -244.50660705566406,
"logps/rejected": -823.7095336914062,
"loss": 4.9956,
"margin_dpo/margin_mean": 173.96189880371094,
"margin_dpo/margin_std": 340.6141052246094,
"step": 475
},
{
"KL/chosen_KL_mean": -434.2881774902344,
"KL/mean": -515.4061279296875,
"KL/rejected_KL_mean": -596.5241088867188,
"KL/std": 296.0513610839844,
"epoch": 0.9968586387434555,
"fcm_dpo/beta": 0.0010204007849097252,
"fcm_dpo/delta": -0.0251779668033123,
"fcm_dpo/margin": 162.23599243164062,
"fcm_dpo/q_t": 0.46206921339035034,
"grad_norm": 38.84898376464844,
"learning_rate": 2.6813123097352287e-11,
"logits/chosen": -0.8090430498123169,
"logits/rejected": -0.7528334856033325,
"logps/chosen": -755.7556762695312,
"logps/ref_chosen": -321.467529296875,
"logps/ref_rejected": -295.0592956542969,
"logps/rejected": -891.5834350585938,
"loss": 5.0824,
"margin_dpo/margin_mean": 162.23599243164062,
"margin_dpo/margin_std": 391.11846923828125,
"step": 476
},
{
"KL/chosen_KL_mean": -469.63360595703125,
"KL/mean": -557.083740234375,
"KL/rejected_KL_mean": -644.5338134765625,
"KL/std": 356.2486572265625,
"epoch": 0.9989528795811519,
"fcm_dpo/beta": 0.0010169773595407605,
"fcm_dpo/delta": 0.00865718349814415,
"fcm_dpo/margin": 174.90023803710938,
"fcm_dpo/q_t": 0.4589536190032959,
"grad_norm": 72.80176544189453,
"learning_rate": 6.7033706447061635e-12,
"logits/chosen": -0.6653886437416077,
"logits/rejected": -0.6609338521957397,
"logps/chosen": -746.4275512695312,
"logps/ref_chosen": -276.7939758300781,
"logps/ref_rejected": -244.82919311523438,
"logps/rejected": -889.3629760742188,
"loss": 5.0882,
"margin_dpo/margin_mean": 174.90023803710938,
"margin_dpo/margin_std": 460.7952575683594,
"step": 477
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 5.002585172153369,
"train_runtime": 6534.5428,
"train_samples_per_second": 9.356,
"train_steps_per_second": 0.073
}
],
"logging_steps": 1,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink