jackf857/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-ult…/trainer_state.json
ModelHub XC 6d0881812d 初始化项目，由ModelHub XC社区提供模型 
Model: jackf857/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4
Source: Original Platform
2026-05-26 12:58:23 +08:00

11057 lines
414 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": -0.05443763732910156,
"KL/mean": -0.04046645760536194,
"KL/rejected_KL_mean": -0.026498794555664062,
"KL/std": 0.47091516852378845,
"epoch": 0.0020942408376963353,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02793481945991516,
"fcm_dpo/q_t": 0.500069797039032,
"grad_norm": 28.589929580688477,
"learning_rate": 0.0,
"logits/chosen": -0.5898098945617676,
"logits/rejected": -0.604260265827179,
"logps/chosen": -275.28570556640625,
"logps/ref_chosen": -275.2312927246094,
"logps/ref_rejected": -222.9380340576172,
"logps/rejected": -222.96453857421875,
"loss": 5.5463,
"margin_dpo/margin_mean": -0.02793477475643158,
"margin_dpo/margin_std": 0.5724214911460876,
"step": 1
},
{
"KL/chosen_KL_mean": 0.04458808898925781,
"KL/mean": 0.0374295711517334,
"KL/rejected_KL_mean": 0.030269622802734375,
"KL/std": 0.49036121368408203,
"epoch": 0.004188481675392671,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014312177896499634,
"fcm_dpo/q_t": 0.4999642074108124,
"grad_norm": 27.881349563598633,
"learning_rate": 1.0416666666666666e-08,
"logits/chosen": -0.6574729681015015,
"logits/rejected": -0.6464410424232483,
"logps/chosen": -264.7165222167969,
"logps/ref_chosen": -264.7611083984375,
"logps/ref_rejected": -242.5597686767578,
"logps/rejected": -242.52951049804688,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.014312252402305603,
"margin_dpo/margin_std": 0.6423971652984619,
"step": 2
},
{
"KL/chosen_KL_mean": -0.015283584594726562,
"KL/mean": -0.027470149099826813,
"KL/rejected_KL_mean": -0.03966331481933594,
"KL/std": 0.4722851514816284,
"epoch": 0.0062827225130890054,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02437432110309601,
"fcm_dpo/q_t": 0.49993908405303955,
"grad_norm": 25.772430419921875,
"learning_rate": 2.083333333333333e-08,
"logits/chosen": -0.6839795112609863,
"logits/rejected": -0.7351416349411011,
"logps/chosen": -274.1171875,
"logps/ref_chosen": -274.1018981933594,
"logps/ref_rejected": -286.5882568359375,
"logps/rejected": -286.6279296875,
"loss": 5.5442,
"margin_dpo/margin_mean": 0.024372458457946777,
"margin_dpo/margin_std": 0.6266783475875854,
"step": 3
},
{
"KL/chosen_KL_mean": 0.04073333740234375,
"KL/mean": -0.02466951310634613,
"KL/rejected_KL_mean": -0.0900726318359375,
"KL/std": 0.5628562569618225,
"epoch": 0.008376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13081058859825134,
"fcm_dpo/q_t": 0.49967294931411743,
"grad_norm": 31.76841926574707,
"learning_rate": 3.125e-08,
"logits/chosen": -0.6164096593856812,
"logits/rejected": -0.61078280210495,
"logps/chosen": -329.7975158691406,
"logps/ref_chosen": -329.8382568359375,
"logps/ref_rejected": -303.2850646972656,
"logps/rejected": -303.3750915527344,
"loss": 5.54,
"margin_dpo/margin_mean": 0.1308116316795349,
"margin_dpo/margin_std": 0.7712123990058899,
"step": 4
},
{
"KL/chosen_KL_mean": -0.004119873046875,
"KL/mean": 0.020755663514137268,
"KL/rejected_KL_mean": 0.045635223388671875,
"KL/std": 0.533147931098938,
"epoch": 0.010471204188481676,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04974430799484253,
"fcm_dpo/q_t": 0.5001243352890015,
"grad_norm": 29.5401554107666,
"learning_rate": 4.166666666666666e-08,
"logits/chosen": -0.5749531984329224,
"logits/rejected": -0.5917044281959534,
"logps/chosen": -301.7430419921875,
"logps/ref_chosen": -301.7389221191406,
"logps/ref_rejected": -274.7654724121094,
"logps/rejected": -274.7198181152344,
"loss": 5.5472,
"margin_dpo/margin_mean": -0.049745798110961914,
"margin_dpo/margin_std": 0.8052065968513489,
"step": 5
},
{
"KL/chosen_KL_mean": -0.016300201416015625,
"KL/mean": 0.0033769533038139343,
"KL/rejected_KL_mean": 0.023052215576171875,
"KL/std": 0.5462159514427185,
"epoch": 0.012565445026178011,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03935401141643524,
"fcm_dpo/q_t": 0.5000984072685242,
"grad_norm": 28.32386589050293,
"learning_rate": 5.208333333333333e-08,
"logits/chosen": -0.6760701537132263,
"logits/rejected": -0.6386649012565613,
"logps/chosen": -285.7109680175781,
"logps/ref_chosen": -285.6946716308594,
"logps/ref_rejected": -245.8200225830078,
"logps/rejected": -245.79696655273438,
"loss": 5.5468,
"margin_dpo/margin_mean": -0.03935439884662628,
"margin_dpo/margin_std": 0.7249687314033508,
"step": 6
},
{
"KL/chosen_KL_mean": -0.08362960815429688,
"KL/mean": -0.03842785954475403,
"KL/rejected_KL_mean": 0.006763458251953125,
"KL/std": 0.4993395209312439,
"epoch": 0.014659685863874346,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.09039899706840515,
"fcm_dpo/q_t": 0.5002260208129883,
"grad_norm": 28.561147689819336,
"learning_rate": 6.25e-08,
"logits/chosen": -0.5755224823951721,
"logits/rejected": -0.6081489324569702,
"logps/chosen": -264.7390441894531,
"logps/ref_chosen": -264.65545654296875,
"logps/ref_rejected": -253.10305786132812,
"logps/rejected": -253.09628295898438,
"loss": 5.5488,
"margin_dpo/margin_mean": -0.09039971232414246,
"margin_dpo/margin_std": 0.6376783847808838,
"step": 7
},
{
"KL/chosen_KL_mean": 0.041156768798828125,
"KL/mean": -0.013764090836048126,
"KL/rejected_KL_mean": -0.06868171691894531,
"KL/std": 0.5888418555259705,
"epoch": 0.016753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10983617603778839,
"fcm_dpo/q_t": 0.49972546100616455,
"grad_norm": 30.76198387145996,
"learning_rate": 7.291666666666667e-08,
"logits/chosen": -0.6753714680671692,
"logits/rejected": -0.681854784488678,
"logps/chosen": -354.14764404296875,
"logps/ref_chosen": -354.1887512207031,
"logps/ref_rejected": -282.9112243652344,
"logps/rejected": -282.9798889160156,
"loss": 5.5409,
"margin_dpo/margin_mean": 0.10983628034591675,
"margin_dpo/margin_std": 0.7891420125961304,
"step": 8
},
{
"KL/chosen_KL_mean": 0.0035858154296875,
"KL/mean": 0.005865432322025299,
"KL/rejected_KL_mean": 0.008136749267578125,
"KL/std": 0.5102388858795166,
"epoch": 0.018848167539267015,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.004547819495201111,
"fcm_dpo/q_t": 0.5000113844871521,
"grad_norm": 27.88920021057129,
"learning_rate": 8.333333333333333e-08,
"logits/chosen": -0.6402922868728638,
"logits/rejected": -0.6586679220199585,
"logps/chosen": -285.546630859375,
"logps/ref_chosen": -285.5502014160156,
"logps/ref_rejected": -267.99664306640625,
"logps/rejected": -267.988525390625,
"loss": 5.5454,
"margin_dpo/margin_mean": -0.004548460245132446,
"margin_dpo/margin_std": 0.7229992747306824,
"step": 9
},
{
"KL/chosen_KL_mean": 0.06154441833496094,
"KL/mean": 0.019467219710350037,
"KL/rejected_KL_mean": -0.022617340087890625,
"KL/std": 0.4935854971408844,
"epoch": 0.020942408376963352,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08416536450386047,
"fcm_dpo/q_t": 0.49978959560394287,
"grad_norm": 26.521408081054688,
"learning_rate": 9.375e-08,
"logits/chosen": -0.6825000047683716,
"logits/rejected": -0.6766196489334106,
"logps/chosen": -251.850830078125,
"logps/ref_chosen": -251.91238403320312,
"logps/ref_rejected": -226.45260620117188,
"logps/rejected": -226.4752197265625,
"loss": 5.5419,
"margin_dpo/margin_mean": 0.08416460454463959,
"margin_dpo/margin_std": 0.7020270228385925,
"step": 10
},
{
"KL/chosen_KL_mean": 0.02030181884765625,
"KL/mean": 0.06047465652227402,
"KL/rejected_KL_mean": 0.10064697265625,
"KL/std": 0.5014757513999939,
"epoch": 0.023036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.08033938705921173,
"fcm_dpo/q_t": 0.5002008080482483,
"grad_norm": 28.9848575592041,
"learning_rate": 1.0416666666666667e-07,
"logits/chosen": -0.5878165364265442,
"logits/rejected": -0.6417873501777649,
"logps/chosen": -301.06317138671875,
"logps/ref_chosen": -301.08343505859375,
"logps/ref_rejected": -259.546630859375,
"logps/rejected": -259.44598388671875,
"loss": 5.5484,
"margin_dpo/margin_mean": -0.08033882081508636,
"margin_dpo/margin_std": 0.6635822653770447,
"step": 11
},
{
"KL/chosen_KL_mean": -0.0308685302734375,
"KL/mean": -0.08398044854402542,
"KL/rejected_KL_mean": -0.1370849609375,
"KL/std": 0.5727797746658325,
"epoch": 0.025130890052356022,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1062176376581192,
"fcm_dpo/q_t": 0.4997345209121704,
"grad_norm": 30.093599319458008,
"learning_rate": 1.1458333333333332e-07,
"logits/chosen": -0.5787484645843506,
"logits/rejected": -0.538584291934967,
"logps/chosen": -287.5789794921875,
"logps/ref_chosen": -287.548095703125,
"logps/ref_rejected": -277.37945556640625,
"logps/rejected": -277.51654052734375,
"loss": 5.541,
"margin_dpo/margin_mean": 0.10621786117553711,
"margin_dpo/margin_std": 0.8124967813491821,
"step": 12
},
{
"KL/chosen_KL_mean": 0.04111671447753906,
"KL/mean": -0.03685136139392853,
"KL/rejected_KL_mean": -0.11482620239257812,
"KL/std": 0.49719753861427307,
"epoch": 0.027225130890052355,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15594592690467834,
"fcm_dpo/q_t": 0.49961015582084656,
"grad_norm": 27.183223724365234,
"learning_rate": 1.25e-07,
"logits/chosen": -0.6483482122421265,
"logits/rejected": -0.6553612947463989,
"logps/chosen": -270.62530517578125,
"logps/ref_chosen": -270.6664123535156,
"logps/ref_rejected": -274.6546936035156,
"logps/rejected": -274.76953125,
"loss": 5.539,
"margin_dpo/margin_mean": 0.15594591200351715,
"margin_dpo/margin_std": 0.6842634677886963,
"step": 13
},
{
"KL/chosen_KL_mean": 0.08417701721191406,
"KL/mean": 0.03670407831668854,
"KL/rejected_KL_mean": -0.010768890380859375,
"KL/std": 0.5663655400276184,
"epoch": 0.02931937172774869,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09495711326599121,
"fcm_dpo/q_t": 0.4997625946998596,
"grad_norm": 28.36115264892578,
"learning_rate": 1.3541666666666666e-07,
"logits/chosen": -0.6208177208900452,
"logits/rejected": -0.6492733955383301,
"logps/chosen": -281.5090637207031,
"logps/ref_chosen": -281.59320068359375,
"logps/ref_rejected": -263.52215576171875,
"logps/rejected": -263.532958984375,
"loss": 5.5414,
"margin_dpo/margin_mean": 0.09495654702186584,
"margin_dpo/margin_std": 0.7788461446762085,
"step": 14
},
{
"KL/chosen_KL_mean": -0.000659942626953125,
"KL/mean": 0.006451621651649475,
"KL/rejected_KL_mean": 0.013570785522460938,
"KL/std": 0.5095148086547852,
"epoch": 0.031413612565445025,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.014226287603378296,
"fcm_dpo/q_t": 0.5000355839729309,
"grad_norm": 30.425567626953125,
"learning_rate": 1.4583333333333335e-07,
"logits/chosen": -0.6453888416290283,
"logits/rejected": -0.6586918830871582,
"logps/chosen": -298.4541015625,
"logps/ref_chosen": -298.45343017578125,
"logps/ref_rejected": -227.17832946777344,
"logps/rejected": -227.16476440429688,
"loss": 5.5458,
"margin_dpo/margin_mean": -0.014227613806724548,
"margin_dpo/margin_std": 0.7101153135299683,
"step": 15
},
{
"KL/chosen_KL_mean": 0.062591552734375,
"KL/mean": 0.027167625725269318,
"KL/rejected_KL_mean": -0.008253097534179688,
"KL/std": 0.5141140222549438,
"epoch": 0.033507853403141365,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07084225118160248,
"fcm_dpo/q_t": 0.4998228847980499,
"grad_norm": 30.161983489990234,
"learning_rate": 1.5624999999999999e-07,
"logits/chosen": -0.6102164387702942,
"logits/rejected": -0.60810387134552,
"logps/chosen": -293.9040222167969,
"logps/ref_chosen": -293.96661376953125,
"logps/ref_rejected": -250.78443908691406,
"logps/rejected": -250.79269409179688,
"loss": 5.5424,
"margin_dpo/margin_mean": 0.07084193825721741,
"margin_dpo/margin_std": 0.7234665751457214,
"step": 16
},
{
"KL/chosen_KL_mean": 0.08296775817871094,
"KL/mean": 0.07325167208909988,
"KL/rejected_KL_mean": 0.06353187561035156,
"KL/std": 0.48913800716400146,
"epoch": 0.0356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.019437596201896667,
"fcm_dpo/q_t": 0.49995142221450806,
"grad_norm": 27.688617706298828,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5614761114120483,
"logits/rejected": -0.5870969891548157,
"logps/chosen": -262.3110656738281,
"logps/ref_chosen": -262.39398193359375,
"logps/ref_rejected": -248.500244140625,
"logps/rejected": -248.43670654296875,
"loss": 5.5444,
"margin_dpo/margin_mean": 0.019437864422798157,
"margin_dpo/margin_std": 0.6257155537605286,
"step": 17
},
{
"KL/chosen_KL_mean": -0.00669097900390625,
"KL/mean": -0.022606775164604187,
"KL/rejected_KL_mean": -0.038524627685546875,
"KL/std": 0.49792757630348206,
"epoch": 0.03769633507853403,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03182762861251831,
"fcm_dpo/q_t": 0.49992042779922485,
"grad_norm": 29.69187355041504,
"learning_rate": 1.7708333333333334e-07,
"logits/chosen": -0.6175638437271118,
"logits/rejected": -0.6237700581550598,
"logps/chosen": -293.7159423828125,
"logps/ref_chosen": -293.709228515625,
"logps/ref_rejected": -274.5875244140625,
"logps/rejected": -274.6260681152344,
"loss": 5.544,
"margin_dpo/margin_mean": 0.03182883560657501,
"margin_dpo/margin_std": 0.6748430728912354,
"step": 18
},
{
"KL/chosen_KL_mean": 0.03246307373046875,
"KL/mean": 0.049572668969631195,
"KL/rejected_KL_mean": 0.06668472290039062,
"KL/std": 0.5367411375045776,
"epoch": 0.039790575916230364,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.034227266907691956,
"fcm_dpo/q_t": 0.5000855326652527,
"grad_norm": 28.177858352661133,
"learning_rate": 1.875e-07,
"logits/chosen": -0.6283941268920898,
"logits/rejected": -0.6246621608734131,
"logps/chosen": -280.23321533203125,
"logps/ref_chosen": -280.26568603515625,
"logps/ref_rejected": -259.9742736816406,
"logps/rejected": -259.9075927734375,
"loss": 5.5466,
"margin_dpo/margin_mean": -0.03422766923904419,
"margin_dpo/margin_std": 0.7041956782341003,
"step": 19
},
{
"KL/chosen_KL_mean": 0.17637062072753906,
"KL/mean": 0.1533583253622055,
"KL/rejected_KL_mean": 0.1303424835205078,
"KL/std": 0.54378741979599,
"epoch": 0.041884816753926704,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04602688550949097,
"fcm_dpo/q_t": 0.4998849332332611,
"grad_norm": 29.69119644165039,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": -0.6134122610092163,
"logits/rejected": -0.6440813541412354,
"logps/chosen": -303.71905517578125,
"logps/ref_chosen": -303.8954162597656,
"logps/ref_rejected": -260.214599609375,
"logps/rejected": -260.0842590332031,
"loss": 5.5434,
"margin_dpo/margin_mean": 0.04602590203285217,
"margin_dpo/margin_std": 0.8291985392570496,
"step": 20
},
{
"KL/chosen_KL_mean": 0.09269332885742188,
"KL/mean": 0.03890705853700638,
"KL/rejected_KL_mean": -0.01488494873046875,
"KL/std": 0.5381225347518921,
"epoch": 0.04397905759162304,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10758335888385773,
"fcm_dpo/q_t": 0.49973100423812866,
"grad_norm": 34.960567474365234,
"learning_rate": 2.0833333333333333e-07,
"logits/chosen": -0.6435227394104004,
"logits/rejected": -0.671474277973175,
"logps/chosen": -301.4407958984375,
"logps/ref_chosen": -301.5334777832031,
"logps/ref_rejected": -280.28900146484375,
"logps/rejected": -280.3038635253906,
"loss": 5.5409,
"margin_dpo/margin_mean": 0.10758353769779205,
"margin_dpo/margin_std": 0.7066560387611389,
"step": 21
},
{
"KL/chosen_KL_mean": 0.023479461669921875,
"KL/mean": 0.034437984228134155,
"KL/rejected_KL_mean": 0.04539680480957031,
"KL/std": 0.5244945287704468,
"epoch": 0.04607329842931937,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0219249427318573,
"fcm_dpo/q_t": 0.5000548362731934,
"grad_norm": 25.298784255981445,
"learning_rate": 2.1875e-07,
"logits/chosen": -0.6568877696990967,
"logits/rejected": -0.658561110496521,
"logps/chosen": -259.9716491699219,
"logps/ref_chosen": -259.9951477050781,
"logps/ref_rejected": -243.0721435546875,
"logps/rejected": -243.02676391601562,
"loss": 5.5461,
"margin_dpo/margin_mean": -0.021924346685409546,
"margin_dpo/margin_std": 0.713553249835968,
"step": 22
},
{
"KL/chosen_KL_mean": 0.09302520751953125,
"KL/mean": 0.048005327582359314,
"KL/rejected_KL_mean": 0.0029850006103515625,
"KL/std": 0.5733331441879272,
"epoch": 0.048167539267015703,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09003856778144836,
"fcm_dpo/q_t": 0.4997749328613281,
"grad_norm": 28.17792320251465,
"learning_rate": 2.2916666666666663e-07,
"logits/chosen": -0.6223236918449402,
"logits/rejected": -0.6566183567047119,
"logps/chosen": -282.0877380371094,
"logps/ref_chosen": -282.1807556152344,
"logps/ref_rejected": -265.0758056640625,
"logps/rejected": -265.07281494140625,
"loss": 5.5416,
"margin_dpo/margin_mean": 0.09003835916519165,
"margin_dpo/margin_std": 0.8055158853530884,
"step": 23
},
{
"KL/chosen_KL_mean": 0.18692398071289062,
"KL/mean": 0.0784425437450409,
"KL/rejected_KL_mean": -0.0300445556640625,
"KL/std": 0.5381811857223511,
"epoch": 0.050261780104712044,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21695753931999207,
"fcm_dpo/q_t": 0.49945762753486633,
"grad_norm": 29.881175994873047,
"learning_rate": 2.3958333333333335e-07,
"logits/chosen": -0.6491536498069763,
"logits/rejected": -0.5612991452217102,
"logps/chosen": -300.99267578125,
"logps/ref_chosen": -301.17962646484375,
"logps/ref_rejected": -302.12786865234375,
"logps/rejected": -302.15789794921875,
"loss": 5.5366,
"margin_dpo/margin_mean": 0.21695764362812042,
"margin_dpo/margin_std": 0.681999921798706,
"step": 24
},
{
"KL/chosen_KL_mean": 0.1545238494873047,
"KL/mean": 0.11132223159074783,
"KL/rejected_KL_mean": 0.06811237335205078,
"KL/std": 0.5142117738723755,
"epoch": 0.05235602094240838,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08640654385089874,
"fcm_dpo/q_t": 0.499783992767334,
"grad_norm": 27.276826858520508,
"learning_rate": 2.5e-07,
"logits/chosen": -0.5908087491989136,
"logits/rejected": -0.5986031293869019,
"logps/chosen": -246.59197998046875,
"logps/ref_chosen": -246.74649047851562,
"logps/ref_rejected": -235.55638122558594,
"logps/rejected": -235.48825073242188,
"loss": 5.5418,
"margin_dpo/margin_mean": 0.08640624582767487,
"margin_dpo/margin_std": 0.6666019558906555,
"step": 25
},
{
"KL/chosen_KL_mean": 0.22557830810546875,
"KL/mean": 0.16001597046852112,
"KL/rejected_KL_mean": 0.09445762634277344,
"KL/std": 0.6255167126655579,
"epoch": 0.05445026178010471,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13112373650074005,
"fcm_dpo/q_t": 0.49967217445373535,
"grad_norm": 28.602731704711914,
"learning_rate": 2.604166666666667e-07,
"logits/chosen": -0.646224319934845,
"logits/rejected": -0.6599107384681702,
"logps/chosen": -281.97003173828125,
"logps/ref_chosen": -282.1955871582031,
"logps/ref_rejected": -235.3135528564453,
"logps/rejected": -235.21910095214844,
"loss": 5.54,
"margin_dpo/margin_mean": 0.13112422823905945,
"margin_dpo/margin_std": 0.8185635805130005,
"step": 26
},
{
"KL/chosen_KL_mean": 0.3226490020751953,
"KL/mean": 0.2239154875278473,
"KL/rejected_KL_mean": 0.1251850128173828,
"KL/std": 0.6283989548683167,
"epoch": 0.05654450261780105,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19746924936771393,
"fcm_dpo/q_t": 0.4995063543319702,
"grad_norm": 27.78577423095703,
"learning_rate": 2.708333333333333e-07,
"logits/chosen": -0.6513367891311646,
"logits/rejected": -0.6705080270767212,
"logps/chosen": -323.53369140625,
"logps/ref_chosen": -323.8563537597656,
"logps/ref_rejected": -245.968017578125,
"logps/rejected": -245.84283447265625,
"loss": 5.5374,
"margin_dpo/margin_mean": 0.19746987521648407,
"margin_dpo/margin_std": 0.8737579584121704,
"step": 27
},
{
"KL/chosen_KL_mean": 0.2768516540527344,
"KL/mean": 0.19153092801570892,
"KL/rejected_KL_mean": 0.10620689392089844,
"KL/std": 0.5605363845825195,
"epoch": 0.05863874345549738,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1706477254629135,
"fcm_dpo/q_t": 0.49957337975502014,
"grad_norm": 26.49478530883789,
"learning_rate": 2.8125e-07,
"logits/chosen": -0.6174635887145996,
"logits/rejected": -0.6264654397964478,
"logps/chosen": -247.9698944091797,
"logps/ref_chosen": -248.24673461914062,
"logps/ref_rejected": -240.0382080078125,
"logps/rejected": -239.9320068359375,
"loss": 5.5384,
"margin_dpo/margin_mean": 0.17064793407917023,
"margin_dpo/margin_std": 0.7738831043243408,
"step": 28
},
{
"KL/chosen_KL_mean": 0.29106712341308594,
"KL/mean": 0.19878274202346802,
"KL/rejected_KL_mean": 0.10648536682128906,
"KL/std": 0.5895746350288391,
"epoch": 0.060732984293193716,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18458740413188934,
"fcm_dpo/q_t": 0.4995385706424713,
"grad_norm": 29.785554885864258,
"learning_rate": 2.916666666666667e-07,
"logits/chosen": -0.5842040777206421,
"logits/rejected": -0.6051267385482788,
"logps/chosen": -317.9653625488281,
"logps/ref_chosen": -318.2564392089844,
"logps/ref_rejected": -286.75848388671875,
"logps/rejected": -286.65203857421875,
"loss": 5.5379,
"margin_dpo/margin_mean": 0.18458794057369232,
"margin_dpo/margin_std": 0.7843542098999023,
"step": 29
},
{
"KL/chosen_KL_mean": 0.3651390075683594,
"KL/mean": 0.22121500968933105,
"KL/rejected_KL_mean": 0.07729148864746094,
"KL/std": 0.6174904704093933,
"epoch": 0.06282722513089005,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.28784090280532837,
"fcm_dpo/q_t": 0.4992803931236267,
"grad_norm": 28.858478546142578,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": -0.6003649830818176,
"logits/rejected": -0.6176969408988953,
"logps/chosen": -252.68399047851562,
"logps/ref_chosen": -253.0491485595703,
"logps/ref_rejected": -261.30029296875,
"logps/rejected": -261.2229919433594,
"loss": 5.5337,
"margin_dpo/margin_mean": 0.2878407835960388,
"margin_dpo/margin_std": 0.8439157605171204,
"step": 30
},
{
"KL/chosen_KL_mean": 0.41502952575683594,
"KL/mean": 0.2872907519340515,
"KL/rejected_KL_mean": 0.1595611572265625,
"KL/std": 0.7194770574569702,
"epoch": 0.06492146596858639,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.25546789169311523,
"fcm_dpo/q_t": 0.4993613660335541,
"grad_norm": 25.050519943237305,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": -0.6552233099937439,
"logits/rejected": -0.6901096105575562,
"logps/chosen": -247.73800659179688,
"logps/ref_chosen": -248.15301513671875,
"logps/ref_rejected": -203.17703247070312,
"logps/rejected": -203.01748657226562,
"loss": 5.5351,
"margin_dpo/margin_mean": 0.25546786189079285,
"margin_dpo/margin_std": 0.900585949420929,
"step": 31
},
{
"KL/chosen_KL_mean": 0.45583343505859375,
"KL/mean": 0.34075260162353516,
"KL/rejected_KL_mean": 0.22568130493164062,
"KL/std": 0.723097026348114,
"epoch": 0.06701570680628273,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2301485389471054,
"fcm_dpo/q_t": 0.49942466616630554,
"grad_norm": 29.70162582397461,
"learning_rate": 3.2291666666666666e-07,
"logits/chosen": -0.6078366637229919,
"logits/rejected": -0.6122292876243591,
"logps/chosen": -305.0841369628906,
"logps/ref_chosen": -305.5399475097656,
"logps/ref_rejected": -267.6527099609375,
"logps/rejected": -267.4270324707031,
"loss": 5.5361,
"margin_dpo/margin_mean": 0.23014913499355316,
"margin_dpo/margin_std": 0.9483384490013123,
"step": 32
},
{
"KL/chosen_KL_mean": 0.5750503540039062,
"KL/mean": 0.37175193428993225,
"KL/rejected_KL_mean": 0.16846084594726562,
"KL/std": 0.9059445858001709,
"epoch": 0.06910994764397906,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4065783619880676,
"fcm_dpo/q_t": 0.49898362159729004,
"grad_norm": 28.4556941986084,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.640916109085083,
"logits/rejected": -0.6534519195556641,
"logps/chosen": -285.6584777832031,
"logps/ref_chosen": -286.2335205078125,
"logps/ref_rejected": -255.38748168945312,
"logps/rejected": -255.2190399169922,
"loss": 5.5291,
"margin_dpo/margin_mean": 0.4065782427787781,
"margin_dpo/margin_std": 1.132638931274414,
"step": 33
},
{
"KL/chosen_KL_mean": 0.7782306671142578,
"KL/mean": 0.35225048661231995,
"KL/rejected_KL_mean": -0.07373237609863281,
"KL/std": 1.0198003053665161,
"epoch": 0.0712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8519618511199951,
"fcm_dpo/q_t": 0.49787020683288574,
"grad_norm": 31.113811492919922,
"learning_rate": 3.4375e-07,
"logits/chosen": -0.6230222582817078,
"logits/rejected": -0.6323559284210205,
"logps/chosen": -340.8137512207031,
"logps/ref_chosen": -341.5920104980469,
"logps/ref_rejected": -278.8866882324219,
"logps/rejected": -278.9604187011719,
"loss": 5.5113,
"margin_dpo/margin_mean": 0.8519620299339294,
"margin_dpo/margin_std": 1.140161156654358,
"step": 34
},
{
"KL/chosen_KL_mean": 0.6100578308105469,
"KL/mean": 0.45359697937965393,
"KL/rejected_KL_mean": 0.2971305847167969,
"KL/std": 0.9627859592437744,
"epoch": 0.07329842931937172,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.31292349100112915,
"fcm_dpo/q_t": 0.4992177188396454,
"grad_norm": 26.534446716308594,
"learning_rate": 3.541666666666667e-07,
"logits/chosen": -0.6332972049713135,
"logits/rejected": -0.6537303924560547,
"logps/chosen": -264.469482421875,
"logps/ref_chosen": -265.0795593261719,
"logps/ref_rejected": -264.4876708984375,
"logps/rejected": -264.1905517578125,
"loss": 5.5328,
"margin_dpo/margin_mean": 0.3129233419895172,
"margin_dpo/margin_std": 1.3062032461166382,
"step": 35
},
{
"KL/chosen_KL_mean": 0.8854637145996094,
"KL/mean": 0.5023590326309204,
"KL/rejected_KL_mean": 0.11924552917480469,
"KL/std": 1.133253812789917,
"epoch": 0.07539267015706806,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7662180662155151,
"fcm_dpo/q_t": 0.4980846047401428,
"grad_norm": 31.921070098876953,
"learning_rate": 3.645833333333333e-07,
"logits/chosen": -0.6024245023727417,
"logits/rejected": -0.6200557947158813,
"logps/chosen": -296.440673828125,
"logps/ref_chosen": -297.3261413574219,
"logps/ref_rejected": -282.09515380859375,
"logps/rejected": -281.97589111328125,
"loss": 5.5148,
"margin_dpo/margin_mean": 0.7662180066108704,
"margin_dpo/margin_std": 1.4552934169769287,
"step": 36
},
{
"KL/chosen_KL_mean": 0.7702865600585938,
"KL/mean": 0.4680500328540802,
"KL/rejected_KL_mean": 0.1658344268798828,
"KL/std": 1.1320182085037231,
"epoch": 0.0774869109947644,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6044507026672363,
"fcm_dpo/q_t": 0.4984889626502991,
"grad_norm": 31.38983917236328,
"learning_rate": 3.75e-07,
"logits/chosen": -0.6020391583442688,
"logits/rejected": -0.6165008544921875,
"logps/chosen": -313.26373291015625,
"logps/ref_chosen": -314.0340270996094,
"logps/ref_rejected": -299.3437805175781,
"logps/rejected": -299.1779479980469,
"loss": 5.5213,
"margin_dpo/margin_mean": 0.6044489741325378,
"margin_dpo/margin_std": 1.5211684703826904,
"step": 37
},
{
"KL/chosen_KL_mean": 1.0551738739013672,
"KL/mean": 0.7106471061706543,
"KL/rejected_KL_mean": 0.36612892150878906,
"KL/std": 1.2290730476379395,
"epoch": 0.07958115183246073,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.689054012298584,
"fcm_dpo/q_t": 0.4982774555683136,
"grad_norm": 28.488887786865234,
"learning_rate": 3.8541666666666665e-07,
"logits/chosen": -0.6347458362579346,
"logits/rejected": -0.6467496156692505,
"logps/chosen": -281.48602294921875,
"logps/ref_chosen": -282.54119873046875,
"logps/ref_rejected": -269.7773132324219,
"logps/rejected": -269.4111633300781,
"loss": 5.5179,
"margin_dpo/margin_mean": 0.6890544891357422,
"margin_dpo/margin_std": 1.4637418985366821,
"step": 38
},
{
"KL/chosen_KL_mean": 1.2901058197021484,
"KL/mean": 0.679212212562561,
"KL/rejected_KL_mean": 0.06832122802734375,
"KL/std": 1.531196117401123,
"epoch": 0.08167539267015707,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2217795848846436,
"fcm_dpo/q_t": 0.49694594740867615,
"grad_norm": 29.57316017150879,
"learning_rate": 3.958333333333333e-07,
"logits/chosen": -0.6168429851531982,
"logits/rejected": -0.6308571100234985,
"logps/chosen": -275.48284912109375,
"logps/ref_chosen": -276.7729187011719,
"logps/ref_rejected": -249.95889282226562,
"logps/rejected": -249.89056396484375,
"loss": 5.4968,
"margin_dpo/margin_mean": 1.221779465675354,
"margin_dpo/margin_std": 1.7274365425109863,
"step": 39
},
{
"KL/chosen_KL_mean": 1.0404300689697266,
"KL/mean": 0.7055494785308838,
"KL/rejected_KL_mean": 0.3706798553466797,
"KL/std": 1.3352773189544678,
"epoch": 0.08376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6697390675544739,
"fcm_dpo/q_t": 0.4983258545398712,
"grad_norm": 27.396766662597656,
"learning_rate": 4.0625e-07,
"logits/chosen": -0.6229257583618164,
"logits/rejected": -0.6586186289787292,
"logps/chosen": -283.26666259765625,
"logps/ref_chosen": -284.30706787109375,
"logps/ref_rejected": -244.4459991455078,
"logps/rejected": -244.07530212402344,
"loss": 5.5188,
"margin_dpo/margin_mean": 0.6697392463684082,
"margin_dpo/margin_std": 1.7444779872894287,
"step": 40
},
{
"KL/chosen_KL_mean": 1.0908069610595703,
"KL/mean": 0.7018202543258667,
"KL/rejected_KL_mean": 0.31284141540527344,
"KL/std": 1.4677762985229492,
"epoch": 0.08586387434554973,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7779622077941895,
"fcm_dpo/q_t": 0.49805524945259094,
"grad_norm": 30.210500717163086,
"learning_rate": 4.1666666666666667e-07,
"logits/chosen": -0.6286399364471436,
"logits/rejected": -0.6542515754699707,
"logps/chosen": -292.724365234375,
"logps/ref_chosen": -293.8151550292969,
"logps/ref_rejected": -252.16815185546875,
"logps/rejected": -251.85531616210938,
"loss": 5.5144,
"margin_dpo/margin_mean": 0.7779629826545715,
"margin_dpo/margin_std": 1.7769787311553955,
"step": 41
},
{
"KL/chosen_KL_mean": 1.2728042602539062,
"KL/mean": 0.7940388917922974,
"KL/rejected_KL_mean": 0.31528282165527344,
"KL/std": 1.7132642269134521,
"epoch": 0.08795811518324607,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9575194716453552,
"fcm_dpo/q_t": 0.4976065456867218,
"grad_norm": 27.403196334838867,
"learning_rate": 4.270833333333333e-07,
"logits/chosen": -0.6356105804443359,
"logits/rejected": -0.6535137891769409,
"logps/chosen": -251.4874725341797,
"logps/ref_chosen": -252.76023864746094,
"logps/ref_rejected": -261.0414733886719,
"logps/rejected": -260.7261962890625,
"loss": 5.5075,
"margin_dpo/margin_mean": 0.95751953125,
"margin_dpo/margin_std": 2.236936092376709,
"step": 42
},
{
"KL/chosen_KL_mean": 1.2807998657226562,
"KL/mean": 0.6640318632125854,
"KL/rejected_KL_mean": 0.04726409912109375,
"KL/std": 1.9099464416503906,
"epoch": 0.09005235602094241,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.233534812927246,
"fcm_dpo/q_t": 0.4969165027141571,
"grad_norm": 29.962818145751953,
"learning_rate": 4.375e-07,
"logits/chosen": -0.6014646887779236,
"logits/rejected": -0.6178789138793945,
"logps/chosen": -315.553955078125,
"logps/ref_chosen": -316.8347473144531,
"logps/ref_rejected": -273.7649230957031,
"logps/rejected": -273.7176513671875,
"loss": 5.4966,
"margin_dpo/margin_mean": 1.2335340976715088,
"margin_dpo/margin_std": 2.2836263179779053,
"step": 43
},
{
"KL/chosen_KL_mean": 1.5793323516845703,
"KL/mean": 0.7775524854660034,
"KL/rejected_KL_mean": -0.02422332763671875,
"KL/std": 2.20059871673584,
"epoch": 0.09214659685863874,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6035535335540771,
"fcm_dpo/q_t": 0.49599185585975647,
"grad_norm": 30.827856063842773,
"learning_rate": 4.479166666666667e-07,
"logits/chosen": -0.5976197719573975,
"logits/rejected": -0.5963581800460815,
"logps/chosen": -285.2963562011719,
"logps/ref_chosen": -286.8757019042969,
"logps/ref_rejected": -282.4681396484375,
"logps/rejected": -282.49237060546875,
"loss": 5.4822,
"margin_dpo/margin_mean": 1.6035535335540771,
"margin_dpo/margin_std": 2.972196340560913,
"step": 44
},
{
"KL/chosen_KL_mean": 1.7299308776855469,
"KL/mean": 1.0418126583099365,
"KL/rejected_KL_mean": 0.35369300842285156,
"KL/std": 2.250051975250244,
"epoch": 0.09424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3762341737747192,
"fcm_dpo/q_t": 0.4965605139732361,
"grad_norm": 28.878467559814453,
"learning_rate": 4.5833333333333327e-07,
"logits/chosen": -0.688926100730896,
"logits/rejected": -0.7126062512397766,
"logps/chosen": -322.533447265625,
"logps/ref_chosen": -324.2633972167969,
"logps/ref_rejected": -293.09466552734375,
"logps/rejected": -292.740966796875,
"loss": 5.4912,
"margin_dpo/margin_mean": 1.376233696937561,
"margin_dpo/margin_std": 2.813117742538452,
"step": 45
},
{
"KL/chosen_KL_mean": 1.6946258544921875,
"KL/mean": 0.9494956135749817,
"KL/rejected_KL_mean": 0.20435523986816406,
"KL/std": 2.433075189590454,
"epoch": 0.09633507853403141,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4902722835540771,
"fcm_dpo/q_t": 0.4962751865386963,
"grad_norm": 30.186275482177734,
"learning_rate": 4.6874999999999996e-07,
"logits/chosen": -0.6262366771697998,
"logits/rejected": -0.6397067904472351,
"logps/chosen": -296.64111328125,
"logps/ref_chosen": -298.3357238769531,
"logps/ref_rejected": -267.66204833984375,
"logps/rejected": -267.4576721191406,
"loss": 5.4865,
"margin_dpo/margin_mean": 1.4902722835540771,
"margin_dpo/margin_std": 2.6397438049316406,
"step": 46
},
{
"KL/chosen_KL_mean": 1.5152873992919922,
"KL/mean": 1.0190459489822388,
"KL/rejected_KL_mean": 0.5227985382080078,
"KL/std": 2.292161464691162,
"epoch": 0.09842931937172775,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9924949407577515,
"fcm_dpo/q_t": 0.4975191056728363,
"grad_norm": 26.3062801361084,
"learning_rate": 4.791666666666667e-07,
"logits/chosen": -0.6064034700393677,
"logits/rejected": -0.6271387934684753,
"logps/chosen": -261.0516357421875,
"logps/ref_chosen": -262.5669250488281,
"logps/ref_rejected": -258.70989990234375,
"logps/rejected": -258.18707275390625,
"loss": 5.5068,
"margin_dpo/margin_mean": 0.9924947619438171,
"margin_dpo/margin_std": 3.2504637241363525,
"step": 47
},
{
"KL/chosen_KL_mean": 1.820608139038086,
"KL/mean": 1.0710346698760986,
"KL/rejected_KL_mean": 0.32146644592285156,
"KL/std": 2.579556465148926,
"epoch": 0.10052356020942409,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4991388320922852,
"fcm_dpo/q_t": 0.4962541460990906,
"grad_norm": 27.537498474121094,
"learning_rate": 4.895833333333333e-07,
"logits/chosen": -0.6123904585838318,
"logits/rejected": -0.637353777885437,
"logps/chosen": -267.67266845703125,
"logps/ref_chosen": -269.4932556152344,
"logps/ref_rejected": -241.888916015625,
"logps/rejected": -241.56744384765625,
"loss": 5.4865,
"margin_dpo/margin_mean": 1.4991381168365479,
"margin_dpo/margin_std": 2.9511523246765137,
"step": 48
},
{
"KL/chosen_KL_mean": 2.2462005615234375,
"KL/mean": 1.3681857585906982,
"KL/rejected_KL_mean": 0.49018096923828125,
"KL/std": 2.621547222137451,
"epoch": 0.10261780104712041,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7560153007507324,
"fcm_dpo/q_t": 0.4956113398075104,
"grad_norm": 27.621536254882812,
"learning_rate": 5e-07,
"logits/chosen": -0.664618194103241,
"logits/rejected": -0.6490676999092102,
"logps/chosen": -255.63827514648438,
"logps/ref_chosen": -257.8844909667969,
"logps/ref_rejected": -256.8912048339844,
"logps/rejected": -256.4010009765625,
"loss": 5.4765,
"margin_dpo/margin_mean": 1.7560160160064697,
"margin_dpo/margin_std": 3.3718807697296143,
"step": 49
},
{
"KL/chosen_KL_mean": 1.9965705871582031,
"KL/mean": 1.436202883720398,
"KL/rejected_KL_mean": 0.8758354187011719,
"KL/std": 2.9903759956359863,
"epoch": 0.10471204188481675,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1207308769226074,
"fcm_dpo/q_t": 0.4971994161605835,
"grad_norm": 27.850839614868164,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": -0.6214494109153748,
"logits/rejected": -0.642817497253418,
"logps/chosen": -299.63226318359375,
"logps/ref_chosen": -301.62884521484375,
"logps/ref_rejected": -298.2716064453125,
"logps/rejected": -297.3957824707031,
"loss": 5.5021,
"margin_dpo/margin_mean": 1.1207301616668701,
"margin_dpo/margin_std": 3.682468891143799,
"step": 50
},
{
"KL/chosen_KL_mean": 2.1178417205810547,
"KL/mean": 1.1750433444976807,
"KL/rejected_KL_mean": 0.23224449157714844,
"KL/std": 3.547469139099121,
"epoch": 0.1068062827225131,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8856074810028076,
"fcm_dpo/q_t": 0.49528858065605164,
"grad_norm": 29.11636734008789,
"learning_rate": 4.999731868769026e-07,
"logits/chosen": -0.6039535403251648,
"logits/rejected": -0.5983304381370544,
"logps/chosen": -267.2545166015625,
"logps/ref_chosen": -269.37237548828125,
"logps/ref_rejected": -297.0167541503906,
"logps/rejected": -296.7845153808594,
"loss": 5.4725,
"margin_dpo/margin_mean": 1.8856079578399658,
"margin_dpo/margin_std": 4.448060512542725,
"step": 51
},
{
"KL/chosen_KL_mean": 2.7982959747314453,
"KL/mean": 1.2421739101409912,
"KL/rejected_KL_mean": -0.3139476776123047,
"KL/std": 3.763486862182617,
"epoch": 0.10890052356020942,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.1122491359710693,
"fcm_dpo/q_t": 0.49222618341445923,
"grad_norm": 30.2871150970459,
"learning_rate": 4.99939671821067e-07,
"logits/chosen": -0.6552134156227112,
"logits/rejected": -0.6618373990058899,
"logps/chosen": -304.1044921875,
"logps/ref_chosen": -306.9028015136719,
"logps/ref_rejected": -281.24737548828125,
"logps/rejected": -281.561279296875,
"loss": 5.4241,
"margin_dpo/margin_mean": 3.1122488975524902,
"margin_dpo/margin_std": 4.741540431976318,
"step": 52
},
{
"KL/chosen_KL_mean": 2.7981128692626953,
"KL/mean": 1.7105417251586914,
"KL/rejected_KL_mean": 0.6229763031005859,
"KL/std": 4.075300216674805,
"epoch": 0.11099476439790576,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.1751344203948975,
"fcm_dpo/q_t": 0.49456626176834106,
"grad_norm": 30.046728134155273,
"learning_rate": 4.998927532591591e-07,
"logits/chosen": -0.6532727479934692,
"logits/rejected": -0.6938633322715759,
"logps/chosen": -283.1778564453125,
"logps/ref_chosen": -285.9759521484375,
"logps/ref_rejected": -273.9073486328125,
"logps/rejected": -273.28436279296875,
"loss": 5.4616,
"margin_dpo/margin_mean": 2.17513370513916,
"margin_dpo/margin_std": 5.007012367248535,
"step": 53
},
{
"KL/chosen_KL_mean": 2.679128646850586,
"KL/mean": 1.732741355895996,
"KL/rejected_KL_mean": 0.7863483428955078,
"KL/std": 4.326586723327637,
"epoch": 0.1130890052356021,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8927712440490723,
"fcm_dpo/q_t": 0.4952733814716339,
"grad_norm": 26.467914581298828,
"learning_rate": 4.998324337072792e-07,
"logits/chosen": -0.690001368522644,
"logits/rejected": -0.6982678174972534,
"logps/chosen": -303.8254699707031,
"logps/ref_chosen": -306.504638671875,
"logps/ref_rejected": -272.67431640625,
"logps/rejected": -271.8879699707031,
"loss": 5.473,
"margin_dpo/margin_mean": 1.89277184009552,
"margin_dpo/margin_std": 5.428363800048828,
"step": 54
},
{
"KL/chosen_KL_mean": 1.8250560760498047,
"KL/mean": 0.6974931955337524,
"KL/rejected_KL_mean": -0.43007469177246094,
"KL/std": 3.710113525390625,
"epoch": 0.11518324607329843,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.255131244659424,
"fcm_dpo/q_t": 0.49436742067337036,
"grad_norm": 24.798433303833008,
"learning_rate": 4.997587164001815e-07,
"logits/chosen": -0.6320837736129761,
"logits/rejected": -0.635603129863739,
"logps/chosen": -220.50509643554688,
"logps/ref_chosen": -222.33013916015625,
"logps/ref_rejected": -206.59571838378906,
"logps/rejected": -207.0258026123047,
"loss": 5.458,
"margin_dpo/margin_mean": 2.255131959915161,
"margin_dpo/margin_std": 4.947845458984375,
"step": 55
},
{
"KL/chosen_KL_mean": 2.8548545837402344,
"KL/mean": 1.2973169088363647,
"KL/rejected_KL_mean": -0.26021766662597656,
"KL/std": 4.783367156982422,
"epoch": 0.11727748691099477,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.1150808334350586,
"fcm_dpo/q_t": 0.49222424626350403,
"grad_norm": 27.468914031982422,
"learning_rate": 4.996716052911017e-07,
"logits/chosen": -0.613375723361969,
"logits/rejected": -0.6283761262893677,
"logps/chosen": -247.62330627441406,
"logps/ref_chosen": -250.47816467285156,
"logps/ref_rejected": -228.25848388671875,
"logps/rejected": -228.51870727539062,
"loss": 5.4259,
"margin_dpo/margin_mean": 3.115081310272217,
"margin_dpo/margin_std": 5.996671676635742,
"step": 56
},
{
"KL/chosen_KL_mean": 3.899179458618164,
"KL/mean": 1.7846465110778809,
"KL/rejected_KL_mean": -0.3298816680908203,
"KL/std": 4.845379829406738,
"epoch": 0.1193717277486911,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.229069709777832,
"fcm_dpo/q_t": 0.4894401431083679,
"grad_norm": 30.87196159362793,
"learning_rate": 4.99571105051544e-07,
"logits/chosen": -0.7000694274902344,
"logits/rejected": -0.6709764003753662,
"logps/chosen": -311.2203369140625,
"logps/ref_chosen": -315.1195373535156,
"logps/ref_rejected": -272.755615234375,
"logps/rejected": -273.0854797363281,
"loss": 5.3813,
"margin_dpo/margin_mean": 4.229069709777832,
"margin_dpo/margin_std": 5.674056529998779,
"step": 57
},
{
"KL/chosen_KL_mean": 2.4763240814208984,
"KL/mean": 1.3078148365020752,
"KL/rejected_KL_mean": 0.13930511474609375,
"KL/std": 4.4075093269348145,
"epoch": 0.12146596858638743,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.337027072906494,
"fcm_dpo/q_t": 0.494160920381546,
"grad_norm": 27.59241485595703,
"learning_rate": 4.994572210710314e-07,
"logits/chosen": -0.6156286597251892,
"logits/rejected": -0.6398040652275085,
"logps/chosen": -262.705322265625,
"logps/ref_chosen": -265.1816711425781,
"logps/ref_rejected": -268.2203369140625,
"logps/rejected": -268.0810241699219,
"loss": 5.4557,
"margin_dpo/margin_mean": 2.3370275497436523,
"margin_dpo/margin_std": 5.594066619873047,
"step": 58
},
{
"KL/chosen_KL_mean": 2.1144371032714844,
"KL/mean": 1.332619071006775,
"KL/rejected_KL_mean": 0.5508079528808594,
"KL/std": 5.286384582519531,
"epoch": 0.12356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5636342763900757,
"fcm_dpo/q_t": 0.4960969388484955,
"grad_norm": 30.200071334838867,
"learning_rate": 4.993299594568162e-07,
"logits/chosen": -0.6096245646476746,
"logits/rejected": -0.603198766708374,
"logps/chosen": -284.2395324707031,
"logps/ref_chosen": -286.35394287109375,
"logps/ref_rejected": -260.6757507324219,
"logps/rejected": -260.12493896484375,
"loss": 5.4887,
"margin_dpo/margin_mean": 1.5636343955993652,
"margin_dpo/margin_std": 7.07401180267334,
"step": 59
},
{
"KL/chosen_KL_mean": 2.8281993865966797,
"KL/mean": 1.1985671520233154,
"KL/rejected_KL_mean": -0.43106842041015625,
"KL/std": 5.1828532218933105,
"epoch": 0.1256544502617801,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.2592642307281494,
"fcm_dpo/q_t": 0.49186116456985474,
"grad_norm": 28.115793228149414,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": -0.6829241514205933,
"logits/rejected": -0.7077679634094238,
"logps/chosen": -255.92039489746094,
"logps/ref_chosen": -258.74859619140625,
"logps/ref_rejected": -255.04893493652344,
"logps/rejected": -255.47998046875,
"loss": 5.4214,
"margin_dpo/margin_mean": 3.2592647075653076,
"margin_dpo/margin_std": 7.173513412475586,
"step": 60
},
{
"KL/chosen_KL_mean": 3.054656982421875,
"KL/mean": 1.4622215032577515,
"KL/rejected_KL_mean": -0.13021087646484375,
"KL/std": 5.500098705291748,
"epoch": 0.12774869109947645,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.184875965118408,
"fcm_dpo/q_t": 0.492051899433136,
"grad_norm": 29.790935516357422,
"learning_rate": 4.990353313429303e-07,
"logits/chosen": -0.6545593738555908,
"logits/rejected": -0.6720077395439148,
"logps/chosen": -275.41326904296875,
"logps/ref_chosen": -278.4678955078125,
"logps/ref_rejected": -252.02720642089844,
"logps/rejected": -252.1573944091797,
"loss": 5.4254,
"margin_dpo/margin_mean": 3.184875965118408,
"margin_dpo/margin_std": 7.548828125,
"step": 61
},
{
"KL/chosen_KL_mean": 3.975238800048828,
"KL/mean": 2.29765248298645,
"KL/rejected_KL_mean": 0.6200656890869141,
"KL/std": 5.711119174957275,
"epoch": 0.12984293193717278,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3551716804504395,
"fcm_dpo/q_t": 0.49163132905960083,
"grad_norm": 26.7069091796875,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": -0.6063894629478455,
"logits/rejected": -0.6477152705192566,
"logps/chosen": -268.9490661621094,
"logps/ref_chosen": -272.92431640625,
"logps/ref_rejected": -260.7935485839844,
"logps/rejected": -260.1734924316406,
"loss": 5.4178,
"margin_dpo/margin_mean": 3.3551721572875977,
"margin_dpo/margin_std": 7.388532638549805,
"step": 62
},
{
"KL/chosen_KL_mean": 3.386953353881836,
"KL/mean": 1.6372132301330566,
"KL/rejected_KL_mean": -0.11253166198730469,
"KL/std": 5.6143670082092285,
"epoch": 0.1319371727748691,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.4994750022888184,
"fcm_dpo/q_t": 0.4912656843662262,
"grad_norm": 28.21631622314453,
"learning_rate": 4.986872839090852e-07,
"logits/chosen": -0.6588255167007446,
"logits/rejected": -0.6650886535644531,
"logps/chosen": -273.7020568847656,
"logps/ref_chosen": -277.0889892578125,
"logps/ref_rejected": -273.3413391113281,
"logps/rejected": -273.453857421875,
"loss": 5.4121,
"margin_dpo/margin_mean": 3.4994747638702393,
"margin_dpo/margin_std": 7.25181770324707,
"step": 63
},
{
"KL/chosen_KL_mean": 4.168355941772461,
"KL/mean": 2.0174996852874756,
"KL/rejected_KL_mean": -0.13335037231445312,
"KL/std": 6.28251314163208,
"epoch": 0.13403141361256546,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.301701545715332,
"fcm_dpo/q_t": 0.48926711082458496,
"grad_norm": 28.383853912353516,
"learning_rate": 4.9849325083059e-07,
"logits/chosen": -0.6357841491699219,
"logits/rejected": -0.6312809586524963,
"logps/chosen": -279.6561279296875,
"logps/ref_chosen": -283.8244934082031,
"logps/ref_rejected": -263.29351806640625,
"logps/rejected": -263.4268798828125,
"loss": 5.3824,
"margin_dpo/margin_mean": 4.301700592041016,
"margin_dpo/margin_std": 8.25213623046875,
"step": 64
},
{
"KL/chosen_KL_mean": 3.4636306762695312,
"KL/mean": 1.8112151622772217,
"KL/rejected_KL_mean": 0.15880584716796875,
"KL/std": 5.756163597106934,
"epoch": 0.13612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3048229217529297,
"fcm_dpo/q_t": 0.4917513132095337,
"grad_norm": 27.825502395629883,
"learning_rate": 4.982858918131906e-07,
"logits/chosen": -0.7022321820259094,
"logits/rejected": -0.6702457070350647,
"logps/chosen": -261.40631103515625,
"logps/ref_chosen": -264.8699645996094,
"logps/ref_rejected": -268.5076904296875,
"logps/rejected": -268.348876953125,
"loss": 5.4199,
"margin_dpo/margin_mean": 3.3048219680786133,
"margin_dpo/margin_std": 7.397376537322998,
"step": 65
},
{
"KL/chosen_KL_mean": 3.1544952392578125,
"KL/mean": 1.0874898433685303,
"KL/rejected_KL_mean": -0.9795093536376953,
"KL/std": 7.427680015563965,
"epoch": 0.1382198952879581,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.133997440338135,
"fcm_dpo/q_t": 0.4896990954875946,
"grad_norm": 27.89104461669922,
"learning_rate": 4.980652179769217e-07,
"logits/chosen": -0.6843776702880859,
"logits/rejected": -0.704654335975647,
"logps/chosen": -269.7738342285156,
"logps/ref_chosen": -272.9283142089844,
"logps/ref_rejected": -280.94696044921875,
"logps/rejected": -281.9264831542969,
"loss": 5.3922,
"margin_dpo/margin_mean": 4.133997917175293,
"margin_dpo/margin_std": 10.16850471496582,
"step": 66
},
{
"KL/chosen_KL_mean": 3.9878292083740234,
"KL/mean": 2.169799566268921,
"KL/rejected_KL_mean": 0.3517570495605469,
"KL/std": 7.131447792053223,
"epoch": 0.14031413612565444,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.6360793113708496,
"fcm_dpo/q_t": 0.4909287989139557,
"grad_norm": 25.531761169433594,
"learning_rate": 4.978312411558517e-07,
"logits/chosen": -0.6986119747161865,
"logits/rejected": -0.7322227954864502,
"logps/chosen": -262.1990966796875,
"logps/ref_chosen": -266.18695068359375,
"logps/ref_rejected": -250.17405700683594,
"logps/rejected": -249.82232666015625,
"loss": 5.4094,
"margin_dpo/margin_mean": 3.6360793113708496,
"margin_dpo/margin_std": 8.844635963439941,
"step": 67
},
{
"KL/chosen_KL_mean": 3.0627079010009766,
"KL/mean": 0.719912588596344,
"KL/rejected_KL_mean": -1.6228828430175781,
"KL/std": 8.057466506958008,
"epoch": 0.1424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.685594081878662,
"fcm_dpo/q_t": 0.488344669342041,
"grad_norm": 27.940174102783203,
"learning_rate": 4.975839738974473e-07,
"logits/chosen": -0.6814758777618408,
"logits/rejected": -0.6943728923797607,
"logps/chosen": -294.8758850097656,
"logps/ref_chosen": -297.9385986328125,
"logps/ref_rejected": -261.5141296386719,
"logps/rejected": -263.1369934082031,
"loss": 5.3719,
"margin_dpo/margin_mean": 4.685595512390137,
"margin_dpo/margin_std": 10.544692039489746,
"step": 68
},
{
"KL/chosen_KL_mean": 3.918996810913086,
"KL/mean": 0.8691984415054321,
"KL/rejected_KL_mean": -2.1805973052978516,
"KL/std": 7.882665634155273,
"epoch": 0.14450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.09959602355957,
"fcm_dpo/q_t": 0.48479950428009033,
"grad_norm": 28.539579391479492,
"learning_rate": 4.97323429461901e-07,
"logits/chosen": -0.6883444786071777,
"logits/rejected": -0.719520092010498,
"logps/chosen": -261.69854736328125,
"logps/ref_chosen": -265.6175231933594,
"logps/ref_rejected": -236.8287353515625,
"logps/rejected": -239.00933837890625,
"loss": 5.3149,
"margin_dpo/margin_mean": 6.09959602355957,
"margin_dpo/margin_std": 9.689201354980469,
"step": 69
},
{
"KL/chosen_KL_mean": 4.362461090087891,
"KL/mean": 1.4138339757919312,
"KL/rejected_KL_mean": -1.5347957611083984,
"KL/std": 8.69215202331543,
"epoch": 0.14659685863874344,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.897259712219238,
"fcm_dpo/q_t": 0.48531487584114075,
"grad_norm": 28.618436813354492,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": -0.6700032353401184,
"logits/rejected": -0.7025509476661682,
"logps/chosen": -291.86346435546875,
"logps/ref_chosen": -296.2259216308594,
"logps/ref_rejected": -254.68496704101562,
"logps/rejected": -256.2197570800781,
"loss": 5.3258,
"margin_dpo/margin_mean": 5.897259712219238,
"margin_dpo/margin_std": 10.952218055725098,
"step": 70
},
{
"KL/chosen_KL_mean": 5.089008331298828,
"KL/mean": 2.5183145999908447,
"KL/rejected_KL_mean": -0.05237388610839844,
"KL/std": 9.370198249816895,
"epoch": 0.1486910994764398,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.141389846801758,
"fcm_dpo/q_t": 0.4871872365474701,
"grad_norm": 28.453304290771484,
"learning_rate": 4.967625656594781e-07,
"logits/chosen": -0.6482348442077637,
"logits/rejected": -0.6368006467819214,
"logps/chosen": -283.8382263183594,
"logps/ref_chosen": -288.92724609375,
"logps/ref_rejected": -278.6405334472656,
"logps/rejected": -278.69287109375,
"loss": 5.3587,
"margin_dpo/margin_mean": 5.141388893127441,
"margin_dpo/margin_std": 12.29645824432373,
"step": 71
},
{
"KL/chosen_KL_mean": 3.3922672271728516,
"KL/mean": 0.7981147766113281,
"KL/rejected_KL_mean": -1.7960338592529297,
"KL/std": 8.441047668457031,
"epoch": 0.15078534031413612,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.188295364379883,
"fcm_dpo/q_t": 0.48706212639808655,
"grad_norm": 28.272216796875,
"learning_rate": 4.964622763700252e-07,
"logits/chosen": -0.6958415508270264,
"logits/rejected": -0.7072215676307678,
"logps/chosen": -233.65304565429688,
"logps/ref_chosen": -237.0452880859375,
"logps/ref_rejected": -252.7946319580078,
"logps/rejected": -254.59066772460938,
"loss": 5.3528,
"margin_dpo/margin_mean": 5.188295364379883,
"margin_dpo/margin_std": 10.39709186553955,
"step": 72
},
{
"KL/chosen_KL_mean": 4.309686660766602,
"KL/mean": 1.9369821548461914,
"KL/rejected_KL_mean": -0.43572235107421875,
"KL/std": 8.945561408996582,
"epoch": 0.15287958115183245,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.74540901184082,
"fcm_dpo/q_t": 0.4881640076637268,
"grad_norm": 27.904314041137695,
"learning_rate": 4.961487700566646e-07,
"logits/chosen": -0.6588871479034424,
"logits/rejected": -0.6765344142913818,
"logps/chosen": -268.743408203125,
"logps/ref_chosen": -273.0531005859375,
"logps/ref_rejected": -246.8330841064453,
"logps/rejected": -247.26881408691406,
"loss": 5.3739,
"margin_dpo/margin_mean": 4.745408058166504,
"margin_dpo/margin_std": 12.261763572692871,
"step": 73
},
{
"KL/chosen_KL_mean": 3.3888206481933594,
"KL/mean": 1.1340489387512207,
"KL/rejected_KL_mean": -1.1207256317138672,
"KL/std": 9.382542610168457,
"epoch": 0.1549738219895288,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.5095534324646,
"fcm_dpo/q_t": 0.48877227306365967,
"grad_norm": 30.305757522583008,
"learning_rate": 4.958220635317885e-07,
"logits/chosen": -0.7255840301513672,
"logits/rejected": -0.7040022611618042,
"logps/chosen": -338.8930969238281,
"logps/ref_chosen": -342.2818908691406,
"logps/ref_rejected": -330.0293884277344,
"logps/rejected": -331.15008544921875,
"loss": 5.3817,
"margin_dpo/margin_mean": 4.509552478790283,
"margin_dpo/margin_std": 11.70348834991455,
"step": 74
},
{
"KL/chosen_KL_mean": 4.573873519897461,
"KL/mean": 1.2532798051834106,
"KL/rejected_KL_mean": -2.0673141479492188,
"KL/std": 9.401840209960938,
"epoch": 0.15706806282722513,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.64119291305542,
"fcm_dpo/q_t": 0.4834534525871277,
"grad_norm": 29.507638931274414,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": -0.6492323279380798,
"logits/rejected": -0.6508209109306335,
"logps/chosen": -262.290283203125,
"logps/ref_chosen": -266.8641662597656,
"logps/ref_rejected": -276.8699951171875,
"logps/rejected": -278.93731689453125,
"loss": 5.2973,
"margin_dpo/margin_mean": 6.641193389892578,
"margin_dpo/margin_std": 10.963482856750488,
"step": 75
},
{
"KL/chosen_KL_mean": 4.076141357421875,
"KL/mean": 0.49301546812057495,
"KL/rejected_KL_mean": -3.0901031494140625,
"KL/std": 9.934422492980957,
"epoch": 0.15916230366492146,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.166243553161621,
"fcm_dpo/q_t": 0.48219114542007446,
"grad_norm": 29.160371780395508,
"learning_rate": 4.951291206355559e-07,
"logits/chosen": -0.7237194180488586,
"logits/rejected": -0.731726884841919,
"logps/chosen": -277.09844970703125,
"logps/ref_chosen": -281.174560546875,
"logps/ref_rejected": -263.6067199707031,
"logps/rejected": -266.6968078613281,
"loss": 5.2812,
"margin_dpo/margin_mean": 7.166243553161621,
"margin_dpo/margin_std": 12.741656303405762,
"step": 76
},
{
"KL/chosen_KL_mean": 3.6962413787841797,
"KL/mean": 0.7907732725143433,
"KL/rejected_KL_mean": -2.1146984100341797,
"KL/std": 11.032858848571777,
"epoch": 0.1612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.81093692779541,
"fcm_dpo/q_t": 0.48557212948799133,
"grad_norm": 33.117801666259766,
"learning_rate": 4.947629214246236e-07,
"logits/chosen": -0.5612236857414246,
"logits/rejected": -0.5700373649597168,
"logps/chosen": -302.3990478515625,
"logps/ref_chosen": -306.09527587890625,
"logps/ref_rejected": -253.49569702148438,
"logps/rejected": -255.61038208007812,
"loss": 5.3373,
"margin_dpo/margin_mean": 5.810935974121094,
"margin_dpo/margin_std": 14.009510040283203,
"step": 77
},
{
"KL/chosen_KL_mean": 3.9560623168945312,
"KL/mean": -0.42408621311187744,
"KL/rejected_KL_mean": -4.804229736328125,
"KL/std": 11.058279037475586,
"epoch": 0.16335078534031414,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.760295867919922,
"fcm_dpo/q_t": 0.4782665967941284,
"grad_norm": 29.5765323638916,
"learning_rate": 4.943835963210323e-07,
"logits/chosen": -0.6872852444648743,
"logits/rejected": -0.6824613213539124,
"logps/chosen": -252.9462890625,
"logps/ref_chosen": -256.90234375,
"logps/ref_rejected": -211.57154846191406,
"logps/rejected": -216.37579345703125,
"loss": 5.2243,
"margin_dpo/margin_mean": 8.760295867919922,
"margin_dpo/margin_std": 14.509359359741211,
"step": 78
},
{
"KL/chosen_KL_mean": 3.094745635986328,
"KL/mean": -1.0207563638687134,
"KL/rejected_KL_mean": -5.136257171630859,
"KL/std": 11.751511573791504,
"epoch": 0.16544502617801046,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.231000900268555,
"fcm_dpo/q_t": 0.47957462072372437,
"grad_norm": 29.715789794921875,
"learning_rate": 4.939911656668361e-07,
"logits/chosen": -0.6479459404945374,
"logits/rejected": -0.6681416630744934,
"logps/chosen": -263.1788330078125,
"logps/ref_chosen": -266.2735595703125,
"logps/ref_rejected": -251.57257080078125,
"logps/rejected": -256.7088317871094,
"loss": 5.2467,
"margin_dpo/margin_mean": 8.231000900268555,
"margin_dpo/margin_std": 14.964641571044922,
"step": 79
},
{
"KL/chosen_KL_mean": 1.8658809661865234,
"KL/mean": -1.5003409385681152,
"KL/rejected_KL_mean": -4.866567611694336,
"KL/std": 11.38090991973877,
"epoch": 0.16753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.732439994812012,
"fcm_dpo/q_t": 0.48329317569732666,
"grad_norm": 29.368417739868164,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": -0.6606127619743347,
"logits/rejected": -0.6897295713424683,
"logps/chosen": -285.9851379394531,
"logps/ref_chosen": -287.8509826660156,
"logps/ref_rejected": -256.0766296386719,
"logps/rejected": -260.94317626953125,
"loss": 5.3003,
"margin_dpo/margin_mean": 6.732440948486328,
"margin_dpo/margin_std": 13.253829002380371,
"step": 80
},
{
"KL/chosen_KL_mean": 2.3929805755615234,
"KL/mean": -1.201179027557373,
"KL/rejected_KL_mean": -4.795331954956055,
"KL/std": 12.221700668334961,
"epoch": 0.16963350785340314,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.188300132751465,
"fcm_dpo/q_t": 0.4821750223636627,
"grad_norm": 28.137069702148438,
"learning_rate": 4.93167072587771e-07,
"logits/chosen": -0.639922022819519,
"logits/rejected": -0.6389482021331787,
"logps/chosen": -266.1302490234375,
"logps/ref_chosen": -268.5232238769531,
"logps/ref_rejected": -237.81137084960938,
"logps/rejected": -242.606689453125,
"loss": 5.2926,
"margin_dpo/margin_mean": 7.188301086425781,
"margin_dpo/margin_std": 16.771045684814453,
"step": 81
},
{
"KL/chosen_KL_mean": 2.4245166778564453,
"KL/mean": -1.560113549232483,
"KL/rejected_KL_mean": -5.544746398925781,
"KL/std": 12.169999122619629,
"epoch": 0.17172774869109947,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.9692702293396,
"fcm_dpo/q_t": 0.48028764128685,
"grad_norm": 27.79278564453125,
"learning_rate": 4.92735454356513e-07,
"logits/chosen": -0.7311412692070007,
"logits/rejected": -0.7389302849769592,
"logps/chosen": -276.939453125,
"logps/ref_chosen": -279.36395263671875,
"logps/ref_rejected": -236.51365661621094,
"logps/rejected": -242.0584259033203,
"loss": 5.2612,
"margin_dpo/margin_mean": 7.969269275665283,
"margin_dpo/margin_std": 15.728371620178223,
"step": 82
},
{
"KL/chosen_KL_mean": 0.2986888885498047,
"KL/mean": -4.056522846221924,
"KL/rejected_KL_mean": -8.411733627319336,
"KL/std": 13.574930191040039,
"epoch": 0.17382198952879582,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.710416793823242,
"fcm_dpo/q_t": 0.4783723056316376,
"grad_norm": 30.74686050415039,
"learning_rate": 4.922908189595017e-07,
"logits/chosen": -0.6830888390541077,
"logits/rejected": -0.6660097241401672,
"logps/chosen": -273.9205322265625,
"logps/ref_chosen": -274.21923828125,
"logps/ref_rejected": -276.2212219238281,
"logps/rejected": -284.6329345703125,
"loss": 5.2404,
"margin_dpo/margin_mean": 8.71041488647461,
"margin_dpo/margin_std": 18.00052261352539,
"step": 83
},
{
"KL/chosen_KL_mean": 0.45940589904785156,
"KL/mean": -3.1783106327056885,
"KL/rejected_KL_mean": -6.816030502319336,
"KL/std": 13.303352355957031,
"epoch": 0.17591623036649215,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.275437355041504,
"fcm_dpo/q_t": 0.4819525480270386,
"grad_norm": 29.886608123779297,
"learning_rate": 4.918331902411841e-07,
"logits/chosen": -0.7300011515617371,
"logits/rejected": -0.7440951466560364,
"logps/chosen": -293.9381408691406,
"logps/ref_chosen": -294.3975524902344,
"logps/ref_rejected": -279.81884765625,
"logps/rejected": -286.6348571777344,
"loss": 5.2907,
"margin_dpo/margin_mean": 7.275436878204346,
"margin_dpo/margin_std": 16.904508590698242,
"step": 84
},
{
"KL/chosen_KL_mean": -1.547414779663086,
"KL/mean": -4.410724639892578,
"KL/rejected_KL_mean": -7.274040222167969,
"KL/std": 12.268994331359863,
"epoch": 0.17801047120418848,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.726627349853516,
"fcm_dpo/q_t": 0.4857904314994812,
"grad_norm": 29.356456756591797,
"learning_rate": 4.913625927427995e-07,
"logits/chosen": -0.6729651093482971,
"logits/rejected": -0.681312620639801,
"logps/chosen": -245.20960998535156,
"logps/ref_chosen": -243.66220092773438,
"logps/ref_rejected": -263.9421691894531,
"logps/rejected": -271.2162170410156,
"loss": 5.3462,
"margin_dpo/margin_mean": 5.726626873016357,
"margin_dpo/margin_std": 15.388749122619629,
"step": 85
},
{
"KL/chosen_KL_mean": 1.1150398254394531,
"KL/mean": -3.287755012512207,
"KL/rejected_KL_mean": -7.690544128417969,
"KL/std": 13.255000114440918,
"epoch": 0.18010471204188483,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.805594444274902,
"fcm_dpo/q_t": 0.47813674807548523,
"grad_norm": 35.038631439208984,
"learning_rate": 4.908790517010636e-07,
"logits/chosen": -0.6957335472106934,
"logits/rejected": -0.689771294593811,
"logps/chosen": -308.315673828125,
"logps/ref_chosen": -309.4306945800781,
"logps/ref_rejected": -290.91278076171875,
"logps/rejected": -298.60333251953125,
"loss": 5.2323,
"margin_dpo/margin_mean": 8.805594444274902,
"margin_dpo/margin_std": 17.30478286743164,
"step": 86
},
{
"KL/chosen_KL_mean": -0.07946395874023438,
"KL/mean": -5.250776290893555,
"KL/rejected_KL_mean": -10.42209243774414,
"KL/std": 15.329278945922852,
"epoch": 0.18219895287958116,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.342628479003906,
"fcm_dpo/q_t": 0.47446227073669434,
"grad_norm": 29.616622924804688,
"learning_rate": 4.903825930468148e-07,
"logits/chosen": -0.7527617812156677,
"logits/rejected": -0.7467857599258423,
"logps/chosen": -278.107177734375,
"logps/ref_chosen": -278.0277099609375,
"logps/ref_rejected": -245.70123291015625,
"logps/rejected": -256.1233215332031,
"loss": 5.1807,
"margin_dpo/margin_mean": 10.34262752532959,
"margin_dpo/margin_std": 18.891633987426758,
"step": 87
},
{
"KL/chosen_KL_mean": -2.065469741821289,
"KL/mean": -6.552918910980225,
"KL/rejected_KL_mean": -11.040365219116211,
"KL/std": 15.065473556518555,
"epoch": 0.18429319371727748,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.974905014038086,
"fcm_dpo/q_t": 0.47781187295913696,
"grad_norm": 28.785079956054688,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": -0.7596137523651123,
"logits/rejected": -0.7751888632774353,
"logps/chosen": -268.58026123046875,
"logps/ref_chosen": -266.5148010253906,
"logps/ref_rejected": -265.90081787109375,
"logps/rejected": -276.9411926269531,
"loss": 5.2321,
"margin_dpo/margin_mean": 8.974905014038086,
"margin_dpo/margin_std": 19.049829483032227,
"step": 88
},
{
"KL/chosen_KL_mean": 0.024187088012695312,
"KL/mean": -4.604069709777832,
"KL/rejected_KL_mean": -9.232324600219727,
"KL/std": 14.210840225219727,
"epoch": 0.18638743455497384,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.256507873535156,
"fcm_dpo/q_t": 0.47706693410873413,
"grad_norm": 30.544334411621094,
"learning_rate": 4.893510300863676e-07,
"logits/chosen": -0.7496181726455688,
"logits/rejected": -0.7401238083839417,
"logps/chosen": -265.66510009765625,
"logps/ref_chosen": -265.6893005371094,
"logps/ref_rejected": -251.49314880371094,
"logps/rejected": -260.7254638671875,
"loss": 5.2206,
"margin_dpo/margin_mean": 9.256505966186523,
"margin_dpo/margin_std": 18.1968994140625,
"step": 89
},
{
"KL/chosen_KL_mean": -1.1707649230957031,
"KL/mean": -5.307417869567871,
"KL/rejected_KL_mean": -9.444063186645508,
"KL/std": 14.324629783630371,
"epoch": 0.18848167539267016,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.273297309875488,
"fcm_dpo/q_t": 0.47948795557022095,
"grad_norm": 29.870534896850586,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": -0.7418705224990845,
"logits/rejected": -0.7506077885627747,
"logps/chosen": -308.59576416015625,
"logps/ref_chosen": -307.4250183105469,
"logps/ref_rejected": -265.7172546386719,
"logps/rejected": -275.16131591796875,
"loss": 5.255,
"margin_dpo/margin_mean": 8.273297309875488,
"margin_dpo/margin_std": 17.62283706665039,
"step": 90
},
{
"KL/chosen_KL_mean": -2.0556297302246094,
"KL/mean": -6.845937252044678,
"KL/rejected_KL_mean": -11.636245727539062,
"KL/std": 14.768171310424805,
"epoch": 0.1905759162303665,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.580626487731934,
"fcm_dpo/q_t": 0.4763721525669098,
"grad_norm": 32.8757438659668,
"learning_rate": 4.882681251368548e-07,
"logits/chosen": -0.6754969358444214,
"logits/rejected": -0.6923132538795471,
"logps/chosen": -237.79660034179688,
"logps/ref_chosen": -235.74098205566406,
"logps/ref_rejected": -226.6428985595703,
"logps/rejected": -238.27914428710938,
"loss": 5.213,
"margin_dpo/margin_mean": 9.580625534057617,
"margin_dpo/margin_std": 19.14481544494629,
"step": 91
},
{
"KL/chosen_KL_mean": -3.073078155517578,
"KL/mean": -8.132434844970703,
"KL/rejected_KL_mean": -13.191781997680664,
"KL/std": 17.528228759765625,
"epoch": 0.19267015706806281,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.118701934814453,
"fcm_dpo/q_t": 0.47499901056289673,
"grad_norm": 33.32277297973633,
"learning_rate": 4.877074915775048e-07,
"logits/chosen": -0.7399584054946899,
"logits/rejected": -0.7235689759254456,
"logps/chosen": -286.5206298828125,
"logps/ref_chosen": -283.4475402832031,
"logps/ref_rejected": -273.134033203125,
"logps/rejected": -286.3258056640625,
"loss": 5.1989,
"margin_dpo/margin_mean": 10.118700981140137,
"margin_dpo/margin_std": 21.361736297607422,
"step": 92
},
{
"KL/chosen_KL_mean": -2.37042236328125,
"KL/mean": -7.15866756439209,
"KL/rejected_KL_mean": -11.946908950805664,
"KL/std": 15.452308654785156,
"epoch": 0.19476439790575917,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.576484680175781,
"fcm_dpo/q_t": 0.47637003660202026,
"grad_norm": 28.583599090576172,
"learning_rate": 4.871341104867864e-07,
"logits/chosen": -0.7276648879051208,
"logits/rejected": -0.7507765293121338,
"logps/chosen": -235.70758056640625,
"logps/ref_chosen": -233.33714294433594,
"logps/ref_rejected": -230.54273986816406,
"logps/rejected": -242.48963928222656,
"loss": 5.2119,
"margin_dpo/margin_mean": 9.576486587524414,
"margin_dpo/margin_std": 19.396862030029297,
"step": 93
},
{
"KL/chosen_KL_mean": -2.370729446411133,
"KL/mean": -7.069452285766602,
"KL/rejected_KL_mean": -11.768178939819336,
"KL/std": 16.41796875,
"epoch": 0.1968586387434555,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.397444725036621,
"fcm_dpo/q_t": 0.47674643993377686,
"grad_norm": 31.590362548828125,
"learning_rate": 4.865480126133871e-07,
"logits/chosen": -0.6804681420326233,
"logits/rejected": -0.701062798500061,
"logps/chosen": -297.0235290527344,
"logps/ref_chosen": -294.6528015136719,
"logps/ref_rejected": -283.657958984375,
"logps/rejected": -295.4261779785156,
"loss": 5.229,
"margin_dpo/margin_mean": 9.397444725036621,
"margin_dpo/margin_std": 21.770397186279297,
"step": 94
},
{
"KL/chosen_KL_mean": -3.2703723907470703,
"KL/mean": -8.370147705078125,
"KL/rejected_KL_mean": -13.469928741455078,
"KL/std": 18.059005737304688,
"epoch": 0.19895287958115182,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.199569702148438,
"fcm_dpo/q_t": 0.47488659620285034,
"grad_norm": 33.58381652832031,
"learning_rate": 4.859492293879573e-07,
"logits/chosen": -0.7297713756561279,
"logits/rejected": -0.7513625621795654,
"logps/chosen": -314.9400634765625,
"logps/ref_chosen": -311.6697082519531,
"logps/ref_rejected": -262.7471923828125,
"logps/rejected": -276.2171325683594,
"loss": 5.2047,
"margin_dpo/margin_mean": 10.199568748474121,
"margin_dpo/margin_std": 22.428693771362305,
"step": 95
},
{
"KL/chosen_KL_mean": -4.741254806518555,
"KL/mean": -9.848160743713379,
"KL/rejected_KL_mean": -14.955074310302734,
"KL/std": 19.324003219604492,
"epoch": 0.20104712041884817,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.213811874389648,
"fcm_dpo/q_t": 0.47487515211105347,
"grad_norm": 32.13194274902344,
"learning_rate": 4.853377929214243e-07,
"logits/chosen": -0.7099624276161194,
"logits/rejected": -0.7225924134254456,
"logps/chosen": -287.2972412109375,
"logps/ref_chosen": -282.55596923828125,
"logps/ref_rejected": -242.71588134765625,
"logps/rejected": -257.67095947265625,
"loss": 5.2055,
"margin_dpo/margin_mean": 10.213811874389648,
"margin_dpo/margin_std": 23.361492156982422,
"step": 96
},
{
"KL/chosen_KL_mean": -4.432626724243164,
"KL/mean": -10.405980110168457,
"KL/rejected_KL_mean": -16.379331588745117,
"KL/std": 17.279687881469727,
"epoch": 0.2031413612565445,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 11.946697235107422,
"fcm_dpo/q_t": 0.4705726206302643,
"grad_norm": 32.038726806640625,
"learning_rate": 4.847137360032699e-07,
"logits/chosen": -0.7478024959564209,
"logits/rejected": -0.7351346015930176,
"logps/chosen": -308.0104064941406,
"logps/ref_chosen": -303.57781982421875,
"logps/ref_rejected": -264.22491455078125,
"logps/rejected": -280.6042785644531,
"loss": 5.134,
"margin_dpo/margin_mean": 11.946695327758789,
"margin_dpo/margin_std": 22.253314971923828,
"step": 97
},
{
"KL/chosen_KL_mean": -4.91217041015625,
"KL/mean": -11.056485176086426,
"KL/rejected_KL_mean": -17.200815200805664,
"KL/std": 18.921445846557617,
"epoch": 0.20523560209424083,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.288653373718262,
"fcm_dpo/q_t": 0.46991679072380066,
"grad_norm": 34.590091705322266,
"learning_rate": 4.84077092099773e-07,
"logits/chosen": -0.773546576499939,
"logits/rejected": -0.7858595848083496,
"logps/chosen": -291.7425231933594,
"logps/ref_chosen": -286.8303527832031,
"logps/ref_rejected": -278.08331298828125,
"logps/rejected": -295.28411865234375,
"loss": 5.1334,
"margin_dpo/margin_mean": 12.288653373718262,
"margin_dpo/margin_std": 22.73413848876953,
"step": 98
},
{
"KL/chosen_KL_mean": -5.357412338256836,
"KL/mean": -11.63817310333252,
"KL/rejected_KL_mean": -17.918926239013672,
"KL/std": 19.809310913085938,
"epoch": 0.20732984293193718,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.561515808105469,
"fcm_dpo/q_t": 0.46920517086982727,
"grad_norm": 32.64725112915039,
"learning_rate": 4.834278953522137e-07,
"logits/chosen": -0.7336536645889282,
"logits/rejected": -0.7463732957839966,
"logps/chosen": -285.27862548828125,
"logps/ref_chosen": -279.92120361328125,
"logps/ref_rejected": -250.3365478515625,
"logps/rejected": -268.2554626464844,
"loss": 5.1374,
"margin_dpo/margin_mean": 12.561513900756836,
"margin_dpo/margin_std": 27.08693504333496,
"step": 99
},
{
"KL/chosen_KL_mean": -7.981771469116211,
"KL/mean": -14.22314167022705,
"KL/rejected_KL_mean": -20.464502334594727,
"KL/std": 22.023868560791016,
"epoch": 0.2094240837696335,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.482728958129883,
"fcm_dpo/q_t": 0.46943894028663635,
"grad_norm": 35.665283203125,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": -0.7793423533439636,
"logits/rejected": -0.792355477809906,
"logps/chosen": -304.8094482421875,
"logps/ref_chosen": -296.8276672363281,
"logps/ref_rejected": -275.56146240234375,
"logps/rejected": -296.0259704589844,
"loss": 5.1259,
"margin_dpo/margin_mean": 12.482728958129883,
"margin_dpo/margin_std": 24.164777755737305,
"step": 100
},
{
"KL/chosen_KL_mean": -5.265691757202148,
"KL/mean": -12.787820816040039,
"KL/rejected_KL_mean": -20.309947967529297,
"KL/std": 22.83624267578125,
"epoch": 0.21151832460732983,
"fcm_dpo/beta": 0.010065114125609398,
"fcm_dpo/delta": 0.012955582700669765,
"fcm_dpo/margin": 15.044249534606934,
"fcm_dpo/q_t": 0.46306145191192627,
"grad_norm": 32.623504638671875,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": -0.7783185839653015,
"logits/rejected": -0.7923865914344788,
"logps/chosen": -258.0077209472656,
"logps/ref_chosen": -252.74203491210938,
"logps/ref_rejected": -276.4185485839844,
"logps/rejected": -296.728515625,
"loss": 5.0388,
"margin_dpo/margin_mean": 15.044251441955566,
"margin_dpo/margin_std": 26.482027053833008,
"step": 101
},
{
"KL/chosen_KL_mean": -5.2807159423828125,
"KL/mean": -12.680944442749023,
"KL/rejected_KL_mean": -20.0811710357666,
"KL/std": 19.747940063476562,
"epoch": 0.2136125654450262,
"fcm_dpo/beta": 0.010104183107614517,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 14.800459861755371,
"fcm_dpo/q_t": 0.4631401300430298,
"grad_norm": 32.70088577270508,
"learning_rate": 4.814053395442932e-07,
"logits/chosen": -0.748501718044281,
"logits/rejected": -0.7446244359016418,
"logps/chosen": -224.8344268798828,
"logps/ref_chosen": -219.5537109375,
"logps/ref_rejected": -231.90853881835938,
"logps/rejected": -251.98971557617188,
"loss": 5.0391,
"margin_dpo/margin_mean": 14.800461769104004,
"margin_dpo/margin_std": 24.98674201965332,
"step": 102
},
{
"KL/chosen_KL_mean": -4.796598434448242,
"KL/mean": -11.604303359985352,
"KL/rejected_KL_mean": -18.41200828552246,
"KL/std": 21.641616821289062,
"epoch": 0.2157068062827225,
"fcm_dpo/beta": 0.010104183107614517,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.615408897399902,
"fcm_dpo/q_t": 0.46653062105178833,
"grad_norm": 33.16029357910156,
"learning_rate": 4.807062862684873e-07,
"logits/chosen": -0.7775754332542419,
"logits/rejected": -0.7745975852012634,
"logps/chosen": -264.4716491699219,
"logps/ref_chosen": -259.6750793457031,
"logps/ref_rejected": -278.7400817871094,
"logps/rejected": -297.1520690917969,
"loss": 5.087,
"margin_dpo/margin_mean": 13.615410804748535,
"margin_dpo/margin_std": 25.769948959350586,
"step": 103
},
{
"KL/chosen_KL_mean": -9.229019165039062,
"KL/mean": -14.208805084228516,
"KL/rejected_KL_mean": -19.188581466674805,
"KL/std": 22.482053756713867,
"epoch": 0.21780104712041884,
"fcm_dpo/beta": 0.010104183107614517,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.959566116333008,
"fcm_dpo/q_t": 0.4753492772579193,
"grad_norm": 33.56724166870117,
"learning_rate": 4.799948609147061e-07,
"logits/chosen": -0.7666143178939819,
"logits/rejected": -0.7724973559379578,
"logps/chosen": -277.20318603515625,
"logps/ref_chosen": -267.9741516113281,
"logps/ref_rejected": -230.5306396484375,
"logps/rejected": -249.7191925048828,
"loss": 5.2263,
"margin_dpo/margin_mean": 9.959567070007324,
"margin_dpo/margin_std": 26.30889129638672,
"step": 104
},
{
"KL/chosen_KL_mean": -5.367591857910156,
"KL/mean": -15.618803977966309,
"KL/rejected_KL_mean": -25.870025634765625,
"KL/std": 25.24723243713379,
"epoch": 0.2198952879581152,
"fcm_dpo/beta": 0.010265516117215157,
"fcm_dpo/delta": 0.03177732229232788,
"fcm_dpo/margin": 20.502422332763672,
"fcm_dpo/q_t": 0.44872522354125977,
"grad_norm": 35.49409484863281,
"learning_rate": 4.792711016345321e-07,
"logits/chosen": -0.7507005929946899,
"logits/rejected": -0.7593094706535339,
"logps/chosen": -327.6224365234375,
"logps/ref_chosen": -322.25482177734375,
"logps/ref_rejected": -279.02978515625,
"logps/rejected": -304.8998107910156,
"loss": 4.8339,
"margin_dpo/margin_mean": 20.502424240112305,
"margin_dpo/margin_std": 27.04251480102539,
"step": 105
},
{
"KL/chosen_KL_mean": -12.565797805786133,
"KL/mean": -18.833768844604492,
"KL/rejected_KL_mean": -25.101734161376953,
"KL/std": 23.593149185180664,
"epoch": 0.22198952879581152,
"fcm_dpo/beta": 0.010364342480897903,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.535932540893555,
"fcm_dpo/q_t": 0.46829015016555786,
"grad_norm": 39.71345901489258,
"learning_rate": 4.785350472409791e-07,
"logits/chosen": -0.7475220561027527,
"logits/rejected": -0.7846206426620483,
"logps/chosen": -308.7236022949219,
"logps/ref_chosen": -296.15777587890625,
"logps/ref_rejected": -266.2691650390625,
"logps/rejected": -291.3708801269531,
"loss": 5.141,
"margin_dpo/margin_mean": 12.535932540893555,
"margin_dpo/margin_std": 29.484294891357422,
"step": 106
},
{
"KL/chosen_KL_mean": -4.345104217529297,
"KL/mean": -14.212882995605469,
"KL/rejected_KL_mean": -24.08066749572754,
"KL/std": 22.098743438720703,
"epoch": 0.22408376963350785,
"fcm_dpo/beta": 0.010364342480897903,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.735557556152344,
"fcm_dpo/q_t": 0.45006412267684937,
"grad_norm": 39.31448745727539,
"learning_rate": 4.777867372064105e-07,
"logits/chosen": -0.7761357426643372,
"logits/rejected": -0.7689798474311829,
"logps/chosen": -311.3414001464844,
"logps/ref_chosen": -306.996337890625,
"logps/ref_rejected": -296.79412841796875,
"logps/rejected": -320.8747863769531,
"loss": 4.8556,
"margin_dpo/margin_mean": 19.735557556152344,
"margin_dpo/margin_std": 27.56524658203125,
"step": 107
},
{
"KL/chosen_KL_mean": -5.493007659912109,
"KL/mean": -14.60330867767334,
"KL/rejected_KL_mean": -23.713605880737305,
"KL/std": 25.141176223754883,
"epoch": 0.2261780104712042,
"fcm_dpo/beta": 0.010364342480897903,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.220611572265625,
"fcm_dpo/q_t": 0.45436638593673706,
"grad_norm": 105.63594055175781,
"learning_rate": 4.770262116604223e-07,
"logits/chosen": -0.7658447027206421,
"logits/rejected": -0.7774515151977539,
"logps/chosen": -300.6456604003906,
"logps/ref_chosen": -295.1526794433594,
"logps/ref_rejected": -235.974853515625,
"logps/rejected": -259.6884460449219,
"loss": 4.9372,
"margin_dpo/margin_mean": 18.220611572265625,
"margin_dpo/margin_std": 30.389331817626953,
"step": 108
},
{
"KL/chosen_KL_mean": -8.395013809204102,
"KL/mean": -18.529293060302734,
"KL/rejected_KL_mean": -28.663572311401367,
"KL/std": 25.285425186157227,
"epoch": 0.22827225130890053,
"fcm_dpo/beta": 0.010364342480897903,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.268556594848633,
"fcm_dpo/q_t": 0.4494061768054962,
"grad_norm": 39.464012145996094,
"learning_rate": 4.7625351138769166e-07,
"logits/chosen": -0.7855874300003052,
"logits/rejected": -0.7813213467597961,
"logps/chosen": -334.31982421875,
"logps/ref_chosen": -325.9248046875,
"logps/ref_rejected": -279.15423583984375,
"logps/rejected": -307.81781005859375,
"loss": 4.8598,
"margin_dpo/margin_mean": 20.2685546875,
"margin_dpo/margin_std": 31.691814422607422,
"step": 109
},
{
"KL/chosen_KL_mean": -8.880786895751953,
"KL/mean": -18.487215042114258,
"KL/rejected_KL_mean": -28.093652725219727,
"KL/std": 27.44156837463379,
"epoch": 0.23036649214659685,
"fcm_dpo/beta": 0.010364342480897903,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.212865829467773,
"fcm_dpo/q_t": 0.45193198323249817,
"grad_norm": 35.9247932434082,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": -0.8031843900680542,
"logits/rejected": -0.7917722463607788,
"logps/chosen": -283.32000732421875,
"logps/ref_chosen": -274.439208984375,
"logps/ref_rejected": -260.0552062988281,
"logps/rejected": -288.14886474609375,
"loss": 4.9139,
"margin_dpo/margin_mean": 19.212865829467773,
"margin_dpo/margin_std": 33.88175964355469,
"step": 110
},
{
"KL/chosen_KL_mean": -9.789043426513672,
"KL/mean": -19.974393844604492,
"KL/rejected_KL_mean": -30.159746170043945,
"KL/std": 28.102462768554688,
"epoch": 0.2324607329842932,
"fcm_dpo/beta": 0.010417740792036057,
"fcm_dpo/delta": 0.017835039645433426,
"fcm_dpo/margin": 20.370704650878906,
"fcm_dpo/q_t": 0.4491615891456604,
"grad_norm": 39.79193878173828,
"learning_rate": 4.7467175306295647e-07,
"logits/chosen": -0.8448382616043091,
"logits/rejected": -0.8246790170669556,
"logps/chosen": -339.02520751953125,
"logps/ref_chosen": -329.2361755371094,
"logps/ref_rejected": -287.82830810546875,
"logps/rejected": -317.988037109375,
"loss": 4.8781,
"margin_dpo/margin_mean": 20.370704650878906,
"margin_dpo/margin_std": 34.60567092895508,
"step": 111
},
{
"KL/chosen_KL_mean": -15.530427932739258,
"KL/mean": -22.135120391845703,
"KL/rejected_KL_mean": -28.739818572998047,
"KL/std": 27.021099090576172,
"epoch": 0.23455497382198953,
"fcm_dpo/beta": 0.010554548352956772,
"fcm_dpo/delta": 0.006524841766804457,
"fcm_dpo/margin": 13.209390640258789,
"fcm_dpo/q_t": 0.46666276454925537,
"grad_norm": 41.61723327636719,
"learning_rate": 4.7386277983585053e-07,
"logits/chosen": -0.7402675151824951,
"logits/rejected": -0.7702385783195496,
"logps/chosen": -272.5898132324219,
"logps/ref_chosen": -257.0593566894531,
"logps/ref_rejected": -272.9595031738281,
"logps/rejected": -301.6993103027344,
"loss": 5.1718,
"margin_dpo/margin_mean": 13.209390640258789,
"margin_dpo/margin_std": 36.302940368652344,
"step": 112
},
{
"KL/chosen_KL_mean": -12.931312561035156,
"KL/mean": -25.255403518676758,
"KL/rejected_KL_mean": -37.579498291015625,
"KL/std": 31.106075286865234,
"epoch": 0.23664921465968586,
"fcm_dpo/beta": 0.010605855844914913,
"fcm_dpo/delta": 0.008770487271249294,
"fcm_dpo/margin": 24.648176193237305,
"fcm_dpo/q_t": 0.43847501277923584,
"grad_norm": 42.69111251831055,
"learning_rate": 4.7304180152725024e-07,
"logits/chosen": -0.801536500453949,
"logits/rejected": -0.8040984869003296,
"logps/chosen": -298.97296142578125,
"logps/ref_chosen": -286.0416564941406,
"logps/ref_rejected": -270.374267578125,
"logps/rejected": -307.9537658691406,
"loss": 4.7466,
"margin_dpo/margin_mean": 24.648176193237305,
"margin_dpo/margin_std": 39.36397171020508,
"step": 113
},
{
"KL/chosen_KL_mean": -17.070255279541016,
"KL/mean": -23.929719924926758,
"KL/rejected_KL_mean": -30.78919219970703,
"KL/std": 28.087270736694336,
"epoch": 0.2387434554973822,
"fcm_dpo/beta": 0.01064271479845047,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.718945503234863,
"fcm_dpo/q_t": 0.46457406878471375,
"grad_norm": 42.539310455322266,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": -0.8641890287399292,
"logits/rejected": -0.8660076856613159,
"logps/chosen": -277.0787353515625,
"logps/ref_chosen": -260.0084533691406,
"logps/ref_rejected": -246.67190551757812,
"logps/rejected": -277.4610900878906,
"loss": 5.1358,
"margin_dpo/margin_mean": 13.718944549560547,
"margin_dpo/margin_std": 35.54159927368164,
"step": 114
},
{
"KL/chosen_KL_mean": -18.9876766204834,
"KL/mean": -25.9890079498291,
"KL/rejected_KL_mean": -32.99033737182617,
"KL/std": 30.873493194580078,
"epoch": 0.24083769633507854,
"fcm_dpo/beta": 0.010650977492332458,
"fcm_dpo/delta": 0.0012931342935189605,
"fcm_dpo/margin": 14.002666473388672,
"fcm_dpo/q_t": 0.4652654826641083,
"grad_norm": 46.39145278930664,
"learning_rate": 4.7136400641330245e-07,
"logits/chosen": -0.856718122959137,
"logits/rejected": -0.817272424697876,
"logps/chosen": -318.41064453125,
"logps/ref_chosen": -299.4229736328125,
"logps/ref_rejected": -272.1186828613281,
"logps/rejected": -305.1090393066406,
"loss": 5.1502,
"margin_dpo/margin_mean": 14.002666473388672,
"margin_dpo/margin_std": 38.7987174987793,
"step": 115
},
{
"KL/chosen_KL_mean": -15.55940055847168,
"KL/mean": -24.596479415893555,
"KL/rejected_KL_mean": -33.633567810058594,
"KL/std": 28.916170120239258,
"epoch": 0.24293193717277486,
"fcm_dpo/beta": 0.010653730481863022,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.074167251586914,
"fcm_dpo/q_t": 0.4535205662250519,
"grad_norm": 40.90885925292969,
"learning_rate": 4.70507279583015e-07,
"logits/chosen": -0.8591501116752625,
"logits/rejected": -0.8206615447998047,
"logps/chosen": -294.8233337402344,
"logps/ref_chosen": -279.263916015625,
"logps/ref_rejected": -253.6192169189453,
"logps/rejected": -287.2527770996094,
"loss": 4.9579,
"margin_dpo/margin_mean": 18.074169158935547,
"margin_dpo/margin_std": 35.237911224365234,
"step": 116
},
{
"KL/chosen_KL_mean": -18.99713897705078,
"KL/mean": -29.38507652282715,
"KL/rejected_KL_mean": -39.77302169799805,
"KL/std": 27.603759765625,
"epoch": 0.2450261780104712,
"fcm_dpo/beta": 0.010707004927098751,
"fcm_dpo/delta": 0.0019599078223109245,
"fcm_dpo/margin": 20.775876998901367,
"fcm_dpo/q_t": 0.44658583402633667,
"grad_norm": 51.663761138916016,
"learning_rate": 4.6963872761652834e-07,
"logits/chosen": -0.8078407645225525,
"logits/rejected": -0.8070172071456909,
"logps/chosen": -278.2220153808594,
"logps/ref_chosen": -259.2248840332031,
"logps/ref_rejected": -229.3042755126953,
"logps/rejected": -269.0773010253906,
"loss": 4.8498,
"margin_dpo/margin_mean": 20.775875091552734,
"margin_dpo/margin_std": 31.987648010253906,
"step": 117
},
{
"KL/chosen_KL_mean": -25.057470321655273,
"KL/mean": -36.63970184326172,
"KL/rejected_KL_mean": -48.22193145751953,
"KL/std": 34.87174987792969,
"epoch": 0.24712041884816754,
"fcm_dpo/beta": 0.010724391788244247,
"fcm_dpo/delta": 0.009311456233263016,
"fcm_dpo/margin": 23.16446304321289,
"fcm_dpo/q_t": 0.44190675020217896,
"grad_norm": 48.20987319946289,
"learning_rate": 4.687583970916486e-07,
"logits/chosen": -0.8008370399475098,
"logits/rejected": -0.7879800200462341,
"logps/chosen": -292.128173828125,
"logps/ref_chosen": -267.0707092285156,
"logps/ref_rejected": -272.7322082519531,
"logps/rejected": -320.9541320800781,
"loss": 4.833,
"margin_dpo/margin_mean": 23.164464950561523,
"margin_dpo/margin_std": 43.23383712768555,
"step": 118
},
{
"KL/chosen_KL_mean": -30.545703887939453,
"KL/mean": -39.11827087402344,
"KL/rejected_KL_mean": -47.69084930419922,
"KL/std": 34.09941482543945,
"epoch": 0.24921465968586387,
"fcm_dpo/beta": 0.010751364752650261,
"fcm_dpo/delta": 0.0010554725304245949,
"fcm_dpo/margin": 17.145132064819336,
"fcm_dpo/q_t": 0.45693254470825195,
"grad_norm": 48.39207077026367,
"learning_rate": 4.6786633521783005e-07,
"logits/chosen": -0.8896147012710571,
"logits/rejected": -0.8895549774169922,
"logps/chosen": -355.2223815917969,
"logps/ref_chosen": -324.6766357421875,
"logps/ref_rejected": -306.0322265625,
"logps/rejected": -353.72308349609375,
"loss": 5.0604,
"margin_dpo/margin_mean": 17.145130157470703,
"margin_dpo/margin_std": 41.501827239990234,
"step": 119
},
{
"KL/chosen_KL_mean": -27.356718063354492,
"KL/mean": -37.051090240478516,
"KL/rejected_KL_mean": -46.745460510253906,
"KL/std": 33.13863754272461,
"epoch": 0.2513089005235602,
"fcm_dpo/beta": 0.010827319696545601,
"fcm_dpo/delta": 0.009227736853063107,
"fcm_dpo/margin": 19.38873863220215,
"fcm_dpo/q_t": 0.4509553909301758,
"grad_norm": 44.932960510253906,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": -0.8416849970817566,
"logits/rejected": -0.8531475067138672,
"logps/chosen": -342.6184997558594,
"logps/ref_chosen": -315.2617492675781,
"logps/ref_rejected": -265.32501220703125,
"logps/rejected": -312.0704345703125,
"loss": 4.9718,
"margin_dpo/margin_mean": 19.38874053955078,
"margin_dpo/margin_std": 41.262054443359375,
"step": 120
},
{
"KL/chosen_KL_mean": -31.884214401245117,
"KL/mean": -38.95367431640625,
"KL/rejected_KL_mean": -46.02313995361328,
"KL/std": 31.924612045288086,
"epoch": 0.2534031413612565,
"fcm_dpo/beta": 0.010839032009243965,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 14.138921737670898,
"fcm_dpo/q_t": 0.464445561170578,
"grad_norm": 60.977882385253906,
"learning_rate": 4.6604720940421207e-07,
"logits/chosen": -0.8611897230148315,
"logits/rejected": -0.875971257686615,
"logps/chosen": -254.8802947998047,
"logps/ref_chosen": -222.99609375,
"logps/ref_rejected": -226.92860412597656,
"logps/rejected": -272.95172119140625,
"loss": 5.1531,
"margin_dpo/margin_mean": 14.138921737670898,
"margin_dpo/margin_std": 40.55924606323242,
"step": 121
},
{
"KL/chosen_KL_mean": -32.263099670410156,
"KL/mean": -42.11972427368164,
"KL/rejected_KL_mean": -51.97634506225586,
"KL/std": 37.34040451049805,
"epoch": 0.2554973821989529,
"fcm_dpo/beta": 0.010961100459098816,
"fcm_dpo/delta": 0.01673881895840168,
"fcm_dpo/margin": 19.7132511138916,
"fcm_dpo/q_t": 0.44972339272499084,
"grad_norm": 58.791770935058594,
"learning_rate": 4.651202430186092e-07,
"logits/chosen": -0.9088048934936523,
"logits/rejected": -0.8704884648323059,
"logps/chosen": -308.2893981933594,
"logps/ref_chosen": -276.02630615234375,
"logps/ref_rejected": -277.97418212890625,
"logps/rejected": -329.9505310058594,
"loss": 4.9927,
"margin_dpo/margin_mean": 19.713253021240234,
"margin_dpo/margin_std": 46.560882568359375,
"step": 122
},
{
"KL/chosen_KL_mean": -25.356151580810547,
"KL/mean": -38.11783218383789,
"KL/rejected_KL_mean": -50.8795280456543,
"KL/std": 36.9088249206543,
"epoch": 0.25759162303664923,
"fcm_dpo/beta": 0.011069193482398987,
"fcm_dpo/delta": 0.024628084152936935,
"fcm_dpo/margin": 25.523366928100586,
"fcm_dpo/q_t": 0.4335595369338989,
"grad_norm": 52.54579544067383,
"learning_rate": 4.6418174038722924e-07,
"logits/chosen": -0.827944278717041,
"logits/rejected": -0.818522036075592,
"logps/chosen": -353.5107421875,
"logps/ref_chosen": -328.1546325683594,
"logps/ref_rejected": -280.6911315917969,
"logps/rejected": -331.5706481933594,
"loss": 4.7223,
"margin_dpo/margin_mean": 25.523366928100586,
"margin_dpo/margin_std": 42.302734375,
"step": 123
},
{
"KL/chosen_KL_mean": -26.102346420288086,
"KL/mean": -37.06385040283203,
"KL/rejected_KL_mean": -48.025367736816406,
"KL/std": 35.34791564941406,
"epoch": 0.25968586387434556,
"fcm_dpo/beta": 0.011221332475543022,
"fcm_dpo/delta": 0.0027072872035205364,
"fcm_dpo/margin": 21.92302131652832,
"fcm_dpo/q_t": 0.44157105684280396,
"grad_norm": 55.5904426574707,
"learning_rate": 4.6323175183912023e-07,
"logits/chosen": -0.8537352085113525,
"logits/rejected": -0.818031907081604,
"logps/chosen": -301.7985534667969,
"logps/ref_chosen": -275.6961975097656,
"logps/ref_rejected": -225.361572265625,
"logps/rejected": -273.386962890625,
"loss": 4.8314,
"margin_dpo/margin_mean": 21.923023223876953,
"margin_dpo/margin_std": 38.72266387939453,
"step": 124
},
{
"KL/chosen_KL_mean": -27.758771896362305,
"KL/mean": -37.67707061767578,
"KL/rejected_KL_mean": -47.595367431640625,
"KL/std": 36.744564056396484,
"epoch": 0.2617801047120419,
"fcm_dpo/beta": 0.01130264438688755,
"fcm_dpo/delta": 0.0206233449280262,
"fcm_dpo/margin": 19.836593627929688,
"fcm_dpo/q_t": 0.44899219274520874,
"grad_norm": 58.402217864990234,
"learning_rate": 4.6227032831928483e-07,
"logits/chosen": -0.8152500987052917,
"logits/rejected": -0.7693508863449097,
"logps/chosen": -305.8285217285156,
"logps/ref_chosen": -278.06976318359375,
"logps/ref_rejected": -265.63873291015625,
"logps/rejected": -313.234130859375,
"loss": 5.0234,
"margin_dpo/margin_mean": 19.836593627929688,
"margin_dpo/margin_std": 50.21840286254883,
"step": 125
},
{
"KL/chosen_KL_mean": -23.98495864868164,
"KL/mean": -35.167537689208984,
"KL/rejected_KL_mean": -46.350120544433594,
"KL/std": 38.36277770996094,
"epoch": 0.2638743455497382,
"fcm_dpo/beta": 0.011509076692163944,
"fcm_dpo/delta": 0.01648152619600296,
"fcm_dpo/margin": 22.365156173706055,
"fcm_dpo/q_t": 0.44002822041511536,
"grad_norm": 56.53205871582031,
"learning_rate": 4.612975213859487e-07,
"logits/chosen": -0.8276666402816772,
"logits/rejected": -0.8373324275016785,
"logps/chosen": -345.38104248046875,
"logps/ref_chosen": -321.3960876464844,
"logps/ref_rejected": -285.37664794921875,
"logps/rejected": -331.72674560546875,
"loss": 4.8493,
"margin_dpo/margin_mean": 22.365154266357422,
"margin_dpo/margin_std": 44.759033203125,
"step": 126
},
{
"KL/chosen_KL_mean": -22.506771087646484,
"KL/mean": -35.692604064941406,
"KL/rejected_KL_mean": -48.87844467163086,
"KL/std": 39.546775817871094,
"epoch": 0.26596858638743454,
"fcm_dpo/beta": 0.011503228917717934,
"fcm_dpo/delta": -0.021350204944610596,
"fcm_dpo/margin": 26.37167739868164,
"fcm_dpo/q_t": 0.43013378977775574,
"grad_norm": 56.40062713623047,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": -0.9043377637863159,
"logits/rejected": -0.8537869453430176,
"logps/chosen": -329.0655212402344,
"logps/ref_chosen": -306.55877685546875,
"logps/ref_rejected": -274.8651428222656,
"logps/rejected": -323.74359130859375,
"loss": 4.7248,
"margin_dpo/margin_mean": 26.371681213378906,
"margin_dpo/margin_std": 45.39989471435547,
"step": 127
},
{
"KL/chosen_KL_mean": -17.200231552124023,
"KL/mean": -32.34983825683594,
"KL/rejected_KL_mean": -47.49944305419922,
"KL/std": 41.117652893066406,
"epoch": 0.2680628272251309,
"fcm_dpo/beta": 0.011355388909578323,
"fcm_dpo/delta": 0.0041136001236736774,
"fcm_dpo/margin": 30.299209594726562,
"fcm_dpo/q_t": 0.4208820164203644,
"grad_norm": 53.725135803222656,
"learning_rate": 4.5931796656116837e-07,
"logits/chosen": -0.7902308702468872,
"logits/rejected": -0.7869139909744263,
"logps/chosen": -282.59759521484375,
"logps/ref_chosen": -265.3973693847656,
"logps/ref_rejected": -250.9737548828125,
"logps/rejected": -298.47320556640625,
"loss": 4.5971,
"margin_dpo/margin_mean": 30.299209594726562,
"margin_dpo/margin_std": 49.29853820800781,
"step": 128
},
{
"KL/chosen_KL_mean": -18.242015838623047,
"KL/mean": -31.862220764160156,
"KL/rejected_KL_mean": -45.482425689697266,
"KL/std": 38.571449279785156,
"epoch": 0.27015706806282724,
"fcm_dpo/beta": 0.011395599693059921,
"fcm_dpo/delta": -0.014722984284162521,
"fcm_dpo/margin": 27.24040985107422,
"fcm_dpo/q_t": 0.4288609027862549,
"grad_norm": 56.34138488769531,
"learning_rate": 4.5831132482724193e-07,
"logits/chosen": -0.8255420327186584,
"logits/rejected": -0.8241854310035706,
"logps/chosen": -321.40045166015625,
"logps/ref_chosen": -303.158447265625,
"logps/ref_rejected": -275.9891052246094,
"logps/rejected": -321.47149658203125,
"loss": 4.7074,
"margin_dpo/margin_mean": 27.24040985107422,
"margin_dpo/margin_std": 48.436065673828125,
"step": 129
},
{
"KL/chosen_KL_mean": -20.89402961730957,
"KL/mean": -34.46953201293945,
"KL/rejected_KL_mean": -48.04502487182617,
"KL/std": 38.12776565551758,
"epoch": 0.27225130890052357,
"fcm_dpo/beta": 0.011203363537788391,
"fcm_dpo/delta": -0.00045868102461099625,
"fcm_dpo/margin": 27.150991439819336,
"fcm_dpo/q_t": 0.4299607276916504,
"grad_norm": 62.90643310546875,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": -0.7972059845924377,
"logits/rejected": -0.8352242112159729,
"logps/chosen": -307.3013610839844,
"logps/ref_chosen": -286.4073486328125,
"logps/ref_rejected": -294.38665771484375,
"logps/rejected": -342.4317321777344,
"loss": 4.7155,
"margin_dpo/margin_mean": 27.1509952545166,
"margin_dpo/margin_std": 48.800880432128906,
"step": 130
},
{
"KL/chosen_KL_mean": -25.511310577392578,
"KL/mean": -36.389923095703125,
"KL/rejected_KL_mean": -47.268531799316406,
"KL/std": 42.743492126464844,
"epoch": 0.2743455497382199,
"fcm_dpo/beta": 0.011295843869447708,
"fcm_dpo/delta": 0.008524668402969837,
"fcm_dpo/margin": 21.75722312927246,
"fcm_dpo/q_t": 0.4437254071235657,
"grad_norm": 66.18495178222656,
"learning_rate": 4.5626458262912735e-07,
"logits/chosen": -0.8689754009246826,
"logits/rejected": -0.8165445327758789,
"logps/chosen": -337.0763854980469,
"logps/ref_chosen": -311.5650634765625,
"logps/ref_rejected": -291.62432861328125,
"logps/rejected": -338.89288330078125,
"loss": 4.9757,
"margin_dpo/margin_mean": 21.75722312927246,
"margin_dpo/margin_std": 51.93965148925781,
"step": 131
},
{
"KL/chosen_KL_mean": -21.638566970825195,
"KL/mean": -36.89574432373047,
"KL/rejected_KL_mean": -52.152923583984375,
"KL/std": 42.402366638183594,
"epoch": 0.2764397905759162,
"fcm_dpo/beta": 0.01130670215934515,
"fcm_dpo/delta": -0.03159793093800545,
"fcm_dpo/margin": 30.514366149902344,
"fcm_dpo/q_t": 0.42199385166168213,
"grad_norm": 70.13489532470703,
"learning_rate": 4.5522459192551166e-07,
"logits/chosen": -0.8335902690887451,
"logits/rejected": -0.8173937797546387,
"logps/chosen": -291.72039794921875,
"logps/ref_chosen": -270.0818176269531,
"logps/ref_rejected": -284.3084411621094,
"logps/rejected": -336.46136474609375,
"loss": 4.658,
"margin_dpo/margin_mean": 30.514366149902344,
"margin_dpo/margin_std": 53.337528228759766,
"step": 132
},
{
"KL/chosen_KL_mean": -24.91779327392578,
"KL/mean": -39.1947135925293,
"KL/rejected_KL_mean": -53.47163009643555,
"KL/std": 39.884578704833984,
"epoch": 0.27853403141361255,
"fcm_dpo/beta": 0.011096285656094551,
"fcm_dpo/delta": -0.002665869891643524,
"fcm_dpo/margin": 28.553842544555664,
"fcm_dpo/q_t": 0.42722228169441223,
"grad_norm": 47.607017517089844,
"learning_rate": 4.541735956498554e-07,
"logits/chosen": -0.8624144196510315,
"logits/rejected": -0.8618667721748352,
"logps/chosen": -310.5391540527344,
"logps/ref_chosen": -285.6213684082031,
"logps/ref_rejected": -251.19386291503906,
"logps/rejected": -304.66546630859375,
"loss": 4.6841,
"margin_dpo/margin_mean": 28.553842544555664,
"margin_dpo/margin_std": 49.61962127685547,
"step": 133
},
{
"KL/chosen_KL_mean": -37.28353500366211,
"KL/mean": -47.89252471923828,
"KL/rejected_KL_mean": -58.50152587890625,
"KL/std": 41.15799331665039,
"epoch": 0.2806282722513089,
"fcm_dpo/beta": 0.010984513908624649,
"fcm_dpo/delta": -0.00815630704164505,
"fcm_dpo/margin": 21.21798324584961,
"fcm_dpo/q_t": 0.4452402591705322,
"grad_norm": 54.9819450378418,
"learning_rate": 4.5311165016389914e-07,
"logits/chosen": -0.8601773977279663,
"logits/rejected": -0.8593323230743408,
"logps/chosen": -356.204345703125,
"logps/ref_chosen": -318.92083740234375,
"logps/ref_rejected": -293.1894836425781,
"logps/rejected": -351.69097900390625,
"loss": 4.9206,
"margin_dpo/margin_mean": 21.21798324584961,
"margin_dpo/margin_std": 44.41059875488281,
"step": 134
},
{
"KL/chosen_KL_mean": -34.964073181152344,
"KL/mean": -47.551513671875,
"KL/rejected_KL_mean": -60.138946533203125,
"KL/std": 43.32963943481445,
"epoch": 0.28272251308900526,
"fcm_dpo/beta": 0.011102442629635334,
"fcm_dpo/delta": 0.0017053596675395966,
"fcm_dpo/margin": 25.174869537353516,
"fcm_dpo/q_t": 0.4345686733722687,
"grad_norm": 65.74720001220703,
"learning_rate": 4.520388124165564e-07,
"logits/chosen": -0.7585154175758362,
"logits/rejected": -0.798126757144928,
"logps/chosen": -327.7857971191406,
"logps/ref_chosen": -292.8217468261719,
"logps/ref_rejected": -269.2896728515625,
"logps/rejected": -329.42864990234375,
"loss": 4.8261,
"margin_dpo/margin_mean": 25.174869537353516,
"margin_dpo/margin_std": 48.767791748046875,
"step": 135
},
{
"KL/chosen_KL_mean": -37.39518356323242,
"KL/mean": -51.40061950683594,
"KL/rejected_KL_mean": -65.40606689453125,
"KL/std": 43.074127197265625,
"epoch": 0.2848167539267016,
"fcm_dpo/beta": 0.010944095440208912,
"fcm_dpo/delta": -0.015430422499775887,
"fcm_dpo/margin": 28.01087760925293,
"fcm_dpo/q_t": 0.4303656220436096,
"grad_norm": 75.27836608886719,
"learning_rate": 4.5095513994085974e-07,
"logits/chosen": -0.8118724226951599,
"logits/rejected": -0.806470513343811,
"logps/chosen": -310.2476806640625,
"logps/ref_chosen": -272.8525390625,
"logps/ref_rejected": -252.68202209472656,
"logps/rejected": -318.08807373046875,
"loss": 4.7693,
"margin_dpo/margin_mean": 28.01087760925293,
"margin_dpo/margin_std": 54.0770378112793,
"step": 136
},
{
"KL/chosen_KL_mean": -41.546897888183594,
"KL/mean": -54.02006149291992,
"KL/rejected_KL_mean": -66.49323272705078,
"KL/std": 45.12068176269531,
"epoch": 0.2869109947643979,
"fcm_dpo/beta": 0.010924424976110458,
"fcm_dpo/delta": 0.014189784415066242,
"fcm_dpo/margin": 24.94634437561035,
"fcm_dpo/q_t": 0.43837597966194153,
"grad_norm": 66.66802978515625,
"learning_rate": 4.498606908508753e-07,
"logits/chosen": -0.859586775302887,
"logits/rejected": -0.8458654880523682,
"logps/chosen": -342.2991027832031,
"logps/ref_chosen": -300.7522277832031,
"logps/ref_rejected": -286.1935119628906,
"logps/rejected": -352.686767578125,
"loss": 4.8497,
"margin_dpo/margin_mean": 24.94634437561035,
"margin_dpo/margin_std": 51.542877197265625,
"step": 137
},
{
"KL/chosen_KL_mean": -38.696102142333984,
"KL/mean": -53.19694900512695,
"KL/rejected_KL_mean": -67.69778442382812,
"KL/std": 46.134708404541016,
"epoch": 0.28900523560209423,
"fcm_dpo/beta": 0.011035628616809845,
"fcm_dpo/delta": 0.011557774618268013,
"fcm_dpo/margin": 29.00168228149414,
"fcm_dpo/q_t": 0.42893779277801514,
"grad_norm": 61.13792419433594,
"learning_rate": 4.487555238385862e-07,
"logits/chosen": -0.7798894047737122,
"logits/rejected": -0.7592524290084839,
"logps/chosen": -327.63311767578125,
"logps/ref_chosen": -288.9369812011719,
"logps/ref_rejected": -263.7076416015625,
"logps/rejected": -331.40545654296875,
"loss": 4.7942,
"margin_dpo/margin_mean": 29.00168228149414,
"margin_dpo/margin_std": 60.18000030517578,
"step": 138
},
{
"KL/chosen_KL_mean": -35.5364990234375,
"KL/mean": -45.61208724975586,
"KL/rejected_KL_mean": -55.68767547607422,
"KL/std": 41.366214752197266,
"epoch": 0.29109947643979056,
"fcm_dpo/beta": 0.01104138232767582,
"fcm_dpo/delta": -0.008695616386830807,
"fcm_dpo/margin": 20.151180267333984,
"fcm_dpo/q_t": 0.4494304060935974,
"grad_norm": 69.4866714477539,
"learning_rate": 4.476396981707453e-07,
"logits/chosen": -0.8099366426467896,
"logits/rejected": -0.8398616313934326,
"logps/chosen": -305.580810546875,
"logps/ref_chosen": -270.0443115234375,
"logps/ref_rejected": -267.3226013183594,
"logps/rejected": -323.01025390625,
"loss": 5.0408,
"margin_dpo/margin_mean": 20.151180267333984,
"margin_dpo/margin_std": 51.15916442871094,
"step": 139
},
{
"KL/chosen_KL_mean": -33.201412200927734,
"KL/mean": -48.6481819152832,
"KL/rejected_KL_mean": -64.09495544433594,
"KL/std": 43.8369255065918,
"epoch": 0.2931937172774869,
"fcm_dpo/beta": 0.010931570082902908,
"fcm_dpo/delta": -0.034940119832754135,
"fcm_dpo/margin": 30.893543243408203,
"fcm_dpo/q_t": 0.4216638505458832,
"grad_norm": 62.90419006347656,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": -0.8739014863967896,
"logits/rejected": -0.8388884663581848,
"logps/chosen": -316.156982421875,
"logps/ref_chosen": -282.9555969238281,
"logps/ref_rejected": -251.17181396484375,
"logps/rejected": -315.2667541503906,
"loss": 4.596,
"margin_dpo/margin_mean": 30.893543243408203,
"margin_dpo/margin_std": 47.828704833984375,
"step": 140
},
{
"KL/chosen_KL_mean": -31.26448631286621,
"KL/mean": -47.31875228881836,
"KL/rejected_KL_mean": -63.37302017211914,
"KL/std": 45.12312316894531,
"epoch": 0.29528795811518327,
"fcm_dpo/beta": 0.010553342290222645,
"fcm_dpo/delta": -0.02428608201444149,
"fcm_dpo/margin": 32.10853958129883,
"fcm_dpo/q_t": 0.42113417387008667,
"grad_norm": 68.2318344116211,
"learning_rate": 4.453763107901675e-07,
"logits/chosen": -0.8127657771110535,
"logits/rejected": -0.8073655366897583,
"logps/chosen": -327.56463623046875,
"logps/ref_chosen": -296.3001708984375,
"logps/ref_rejected": -279.8486633300781,
"logps/rejected": -343.2216796875,
"loss": 4.6544,
"margin_dpo/margin_mean": 32.10853958129883,
"margin_dpo/margin_std": 55.96519470214844,
"step": 141
},
{
"KL/chosen_KL_mean": -32.01213836669922,
"KL/mean": -45.19526672363281,
"KL/rejected_KL_mean": -58.378387451171875,
"KL/std": 45.81889343261719,
"epoch": 0.2973821989528796,
"fcm_dpo/beta": 0.01045268028974533,
"fcm_dpo/delta": -0.0036399501841515303,
"fcm_dpo/margin": 26.366239547729492,
"fcm_dpo/q_t": 0.4370165467262268,
"grad_norm": 58.871788024902344,
"learning_rate": 4.4422887045602674e-07,
"logits/chosen": -0.8336638808250427,
"logits/rejected": -0.8351055979728699,
"logps/chosen": -332.5780029296875,
"logps/ref_chosen": -300.56585693359375,
"logps/ref_rejected": -231.43316650390625,
"logps/rejected": -289.8115539550781,
"loss": 4.8816,
"margin_dpo/margin_mean": 26.366239547729492,
"margin_dpo/margin_std": 56.89813995361328,
"step": 142
},
{
"KL/chosen_KL_mean": -31.096927642822266,
"KL/mean": -47.20490264892578,
"KL/rejected_KL_mean": -63.3128776550293,
"KL/std": 43.217559814453125,
"epoch": 0.2994764397905759,
"fcm_dpo/beta": 0.01030801422894001,
"fcm_dpo/delta": -0.026560742408037186,
"fcm_dpo/margin": 32.2159538269043,
"fcm_dpo/q_t": 0.42342373728752136,
"grad_norm": 58.806697845458984,
"learning_rate": 4.4307101421701755e-07,
"logits/chosen": -0.8223780393600464,
"logits/rejected": -0.8012036681175232,
"logps/chosen": -327.82928466796875,
"logps/ref_chosen": -296.73236083984375,
"logps/ref_rejected": -266.45257568359375,
"logps/rejected": -329.7654724121094,
"loss": 4.6057,
"margin_dpo/margin_mean": 32.2159538269043,
"margin_dpo/margin_std": 50.72016143798828,
"step": 143
},
{
"KL/chosen_KL_mean": -31.219467163085938,
"KL/mean": -44.27275848388672,
"KL/rejected_KL_mean": -57.326053619384766,
"KL/std": 46.5449333190918,
"epoch": 0.30157068062827225,
"fcm_dpo/beta": 0.010053442791104317,
"fcm_dpo/delta": -0.015576949343085289,
"fcm_dpo/margin": 26.106597900390625,
"fcm_dpo/q_t": 0.44039374589920044,
"grad_norm": 52.784488677978516,
"learning_rate": 4.419028041654559e-07,
"logits/chosen": -0.8913372755050659,
"logits/rejected": -0.8779230117797852,
"logps/chosen": -330.06341552734375,
"logps/ref_chosen": -298.843994140625,
"logps/ref_rejected": -266.120849609375,
"logps/rejected": -323.4468994140625,
"loss": 4.8861,
"margin_dpo/margin_mean": 26.10660171508789,
"margin_dpo/margin_std": 56.03483963012695,
"step": 144
},
{
"KL/chosen_KL_mean": -27.039196014404297,
"KL/mean": -43.98484420776367,
"KL/rejected_KL_mean": -60.930477142333984,
"KL/std": 42.91300964355469,
"epoch": 0.3036649214659686,
"fcm_dpo/beta": 0.00988059677183628,
"fcm_dpo/delta": -0.03598638251423836,
"fcm_dpo/margin": 33.89128112792969,
"fcm_dpo/q_t": 0.42246097326278687,
"grad_norm": 58.97926712036133,
"learning_rate": 4.4072430294890166e-07,
"logits/chosen": -0.8730674982070923,
"logits/rejected": -0.8762320876121521,
"logps/chosen": -302.79205322265625,
"logps/ref_chosen": -275.7528381347656,
"logps/ref_rejected": -214.74807739257812,
"logps/rejected": -275.6785583496094,
"loss": 4.6006,
"margin_dpo/margin_mean": 33.89128112792969,
"margin_dpo/margin_std": 52.56437683105469,
"step": 145
},
{
"KL/chosen_KL_mean": -36.02202606201172,
"KL/mean": -52.752845764160156,
"KL/rejected_KL_mean": -69.48365783691406,
"KL/std": 47.3927001953125,
"epoch": 0.3057591623036649,
"fcm_dpo/beta": 0.009669817984104156,
"fcm_dpo/delta": -0.026783179491758347,
"fcm_dpo/margin": 33.461639404296875,
"fcm_dpo/q_t": 0.42534470558166504,
"grad_norm": 59.773223876953125,
"learning_rate": 4.395355737667985e-07,
"logits/chosen": -0.8538410067558289,
"logits/rejected": -0.848624587059021,
"logps/chosen": -313.1202392578125,
"logps/ref_chosen": -277.09820556640625,
"logps/ref_rejected": -265.41046142578125,
"logps/rejected": -334.89410400390625,
"loss": 4.6348,
"margin_dpo/margin_mean": 33.461639404296875,
"margin_dpo/margin_std": 52.627410888671875,
"step": 146
},
{
"KL/chosen_KL_mean": -37.466670989990234,
"KL/mean": -50.719993591308594,
"KL/rejected_KL_mean": -63.97330856323242,
"KL/std": 49.072593688964844,
"epoch": 0.3078534031413613,
"fcm_dpo/beta": 0.009586220607161522,
"fcm_dpo/delta": -0.023667726665735245,
"fcm_dpo/margin": 26.50663185119629,
"fcm_dpo/q_t": 0.4408421814441681,
"grad_norm": 54.904624938964844,
"learning_rate": 4.3833668036708483e-07,
"logits/chosen": -0.83968186378479,
"logits/rejected": -0.8350346088409424,
"logps/chosen": -328.88525390625,
"logps/ref_chosen": -291.4185791015625,
"logps/ref_rejected": -253.43051147460938,
"logps/rejected": -317.40380859375,
"loss": 4.9045,
"margin_dpo/margin_mean": 26.506629943847656,
"margin_dpo/margin_std": 54.914512634277344,
"step": 147
},
{
"KL/chosen_KL_mean": -35.33280944824219,
"KL/mean": -49.60417175292969,
"KL/rejected_KL_mean": -63.87554168701172,
"KL/std": 51.94282913208008,
"epoch": 0.3099476439790576,
"fcm_dpo/beta": 0.009600591845810413,
"fcm_dpo/delta": 0.039563555270433426,
"fcm_dpo/margin": 28.542724609375,
"fcm_dpo/q_t": 0.43925410509109497,
"grad_norm": 56.438453674316406,
"learning_rate": 4.3712768704277524e-07,
"logits/chosen": -0.9007651209831238,
"logits/rejected": -0.9030435681343079,
"logps/chosen": -272.0813293457031,
"logps/ref_chosen": -236.74850463867188,
"logps/ref_rejected": -231.4674072265625,
"logps/rejected": -295.3429260253906,
"loss": 4.8703,
"margin_dpo/margin_mean": 28.542724609375,
"margin_dpo/margin_std": 62.54542922973633,
"step": 148
},
{
"KL/chosen_KL_mean": -41.490516662597656,
"KL/mean": -60.27011489868164,
"KL/rejected_KL_mean": -79.04971313476562,
"KL/std": 47.53480911254883,
"epoch": 0.31204188481675393,
"fcm_dpo/beta": 0.009786082431674004,
"fcm_dpo/delta": 0.0004139197990298271,
"fcm_dpo/margin": 37.55918884277344,
"fcm_dpo/q_t": 0.4152625799179077,
"grad_norm": 60.076107025146484,
"learning_rate": 4.3590865862851263e-07,
"logits/chosen": -0.8776776790618896,
"logits/rejected": -0.8625072836875916,
"logps/chosen": -361.41900634765625,
"logps/ref_chosen": -319.9284973144531,
"logps/ref_rejected": -308.20233154296875,
"logps/rejected": -387.2520751953125,
"loss": 4.4703,
"margin_dpo/margin_mean": 37.55918884277344,
"margin_dpo/margin_std": 52.95542526245117,
"step": 149
},
{
"KL/chosen_KL_mean": -47.29694366455078,
"KL/mean": -62.3214225769043,
"KL/rejected_KL_mean": -77.34589385986328,
"KL/std": 50.34033966064453,
"epoch": 0.31413612565445026,
"fcm_dpo/beta": 0.009834048338234425,
"fcm_dpo/delta": 0.013316557742655277,
"fcm_dpo/margin": 30.048938751220703,
"fcm_dpo/q_t": 0.4316878020763397,
"grad_norm": 59.99736022949219,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": -0.8376021981239319,
"logits/rejected": -0.8213092088699341,
"logps/chosen": -323.61517333984375,
"logps/ref_chosen": -276.3182373046875,
"logps/ref_rejected": -273.02215576171875,
"logps/rejected": -350.3680114746094,
"loss": 4.7488,
"margin_dpo/margin_mean": 30.04894256591797,
"margin_dpo/margin_std": 55.58155059814453,
"step": 150
},
{
"KL/chosen_KL_mean": -44.79521560668945,
"KL/mean": -66.85020446777344,
"KL/rejected_KL_mean": -88.90518951416016,
"KL/std": 50.85723876953125,
"epoch": 0.3162303664921466,
"fcm_dpo/beta": 0.009750737808644772,
"fcm_dpo/delta": -0.03168368339538574,
"fcm_dpo/margin": 44.109981536865234,
"fcm_dpo/q_t": 0.4018837809562683,
"grad_norm": 62.32393264770508,
"learning_rate": 4.3344075855595097e-07,
"logits/chosen": -0.8728746771812439,
"logits/rejected": -0.8648634552955627,
"logps/chosen": -342.1080322265625,
"logps/ref_chosen": -297.31280517578125,
"logps/ref_rejected": -266.1003723144531,
"logps/rejected": -355.0055847167969,
"loss": 4.3372,
"margin_dpo/margin_mean": 44.109981536865234,
"margin_dpo/margin_std": 58.83431625366211,
"step": 151
},
{
"KL/chosen_KL_mean": -39.10889434814453,
"KL/mean": -57.962406158447266,
"KL/rejected_KL_mean": -76.81593322753906,
"KL/std": 53.661624908447266,
"epoch": 0.3183246073298429,
"fcm_dpo/beta": 0.009311305359005928,
"fcm_dpo/delta": -0.05312967672944069,
"fcm_dpo/margin": 37.7070198059082,
"fcm_dpo/q_t": 0.421106219291687,
"grad_norm": 54.742496490478516,
"learning_rate": 4.3219201924364323e-07,
"logits/chosen": -0.8810481429100037,
"logits/rejected": -0.8775116205215454,
"logps/chosen": -309.35589599609375,
"logps/ref_chosen": -270.2470397949219,
"logps/ref_rejected": -269.7749328613281,
"logps/rejected": -346.59088134765625,
"loss": 4.6232,
"margin_dpo/margin_mean": 37.7070198059082,
"margin_dpo/margin_std": 62.859439849853516,
"step": 152
},
{
"KL/chosen_KL_mean": -46.92686462402344,
"KL/mean": -73.7924575805664,
"KL/rejected_KL_mean": -100.65806579589844,
"KL/std": 58.3165397644043,
"epoch": 0.3204188481675393,
"fcm_dpo/beta": 0.008776971139013767,
"fcm_dpo/delta": -0.09027046710252762,
"fcm_dpo/margin": 53.731204986572266,
"fcm_dpo/q_t": 0.39192867279052734,
"grad_norm": 56.017066955566406,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": -0.8752989172935486,
"logits/rejected": -0.8623684048652649,
"logps/chosen": -320.7059326171875,
"logps/ref_chosen": -273.779052734375,
"logps/ref_rejected": -280.9530944824219,
"logps/rejected": -381.6112060546875,
"loss": 4.1495,
"margin_dpo/margin_mean": 53.731204986572266,
"margin_dpo/margin_std": 60.12602996826172,
"step": 153
},
{
"KL/chosen_KL_mean": -47.703121185302734,
"KL/mean": -68.70870208740234,
"KL/rejected_KL_mean": -89.71429443359375,
"KL/std": 60.09400177001953,
"epoch": 0.3225130890052356,
"fcm_dpo/beta": 0.0084590008482337,
"fcm_dpo/delta": -0.023284776136279106,
"fcm_dpo/margin": 42.011173248291016,
"fcm_dpo/q_t": 0.41954493522644043,
"grad_norm": 59.05760955810547,
"learning_rate": 4.2966529689388064e-07,
"logits/chosen": -0.9124890565872192,
"logits/rejected": -0.8928197622299194,
"logps/chosen": -337.6063232421875,
"logps/ref_chosen": -289.9031982421875,
"logps/ref_rejected": -261.5166320800781,
"logps/rejected": -351.2309265136719,
"loss": 4.6048,
"margin_dpo/margin_mean": 42.011173248291016,
"margin_dpo/margin_std": 70.03306579589844,
"step": 154
},
{
"KL/chosen_KL_mean": -54.11407470703125,
"KL/mean": -72.34459686279297,
"KL/rejected_KL_mean": -90.57511901855469,
"KL/std": 57.328651428222656,
"epoch": 0.32460732984293195,
"fcm_dpo/beta": 0.008467341773211956,
"fcm_dpo/delta": 0.010241080075502396,
"fcm_dpo/margin": 36.461036682128906,
"fcm_dpo/q_t": 0.4311079978942871,
"grad_norm": 69.09295654296875,
"learning_rate": 4.2838744935687716e-07,
"logits/chosen": -0.8517003655433655,
"logits/rejected": -0.848478376865387,
"logps/chosen": -339.97528076171875,
"logps/ref_chosen": -285.8612060546875,
"logps/ref_rejected": -300.1272888183594,
"logps/rejected": -390.702392578125,
"loss": 4.7183,
"margin_dpo/margin_mean": 36.461036682128906,
"margin_dpo/margin_std": 65.83427429199219,
"step": 155
},
{
"KL/chosen_KL_mean": -50.88059997558594,
"KL/mean": -76.42092895507812,
"KL/rejected_KL_mean": -101.96127319335938,
"KL/std": 61.5806884765625,
"epoch": 0.3267015706806283,
"fcm_dpo/beta": 0.008312368765473366,
"fcm_dpo/delta": -0.07811664044857025,
"fcm_dpo/margin": 51.08067321777344,
"fcm_dpo/q_t": 0.4043055474758148,
"grad_norm": 58.513641357421875,
"learning_rate": 4.271000354423425e-07,
"logits/chosen": -0.8742014765739441,
"logits/rejected": -0.8701291084289551,
"logps/chosen": -329.91607666015625,
"logps/ref_chosen": -279.0354919433594,
"logps/ref_rejected": -244.2198486328125,
"logps/rejected": -346.1811218261719,
"loss": 4.4398,
"margin_dpo/margin_mean": 51.08067321777344,
"margin_dpo/margin_std": 77.06956481933594,
"step": 156
},
{
"KL/chosen_KL_mean": -54.794883728027344,
"KL/mean": -74.77871704101562,
"KL/rejected_KL_mean": -94.76253509521484,
"KL/std": 61.28529357910156,
"epoch": 0.3287958115183246,
"fcm_dpo/beta": 0.007772133685648441,
"fcm_dpo/delta": -0.0433635450899601,
"fcm_dpo/margin": 39.9676513671875,
"fcm_dpo/q_t": 0.4275224804878235,
"grad_norm": 53.20646286010742,
"learning_rate": 4.258031241903777e-07,
"logits/chosen": -0.9390982985496521,
"logits/rejected": -0.937108039855957,
"logps/chosen": -325.62518310546875,
"logps/ref_chosen": -270.830322265625,
"logps/ref_rejected": -259.08319091796875,
"logps/rejected": -353.84576416015625,
"loss": 4.6787,
"margin_dpo/margin_mean": 39.9676513671875,
"margin_dpo/margin_std": 66.26377868652344,
"step": 157
},
{
"KL/chosen_KL_mean": -54.49103546142578,
"KL/mean": -77.38906860351562,
"KL/rejected_KL_mean": -100.28709411621094,
"KL/std": 62.60470199584961,
"epoch": 0.3308900523560209,
"fcm_dpo/beta": 0.007661835290491581,
"fcm_dpo/delta": -0.022122088819742203,
"fcm_dpo/margin": 45.79604721069336,
"fcm_dpo/q_t": 0.4180770218372345,
"grad_norm": 55.99773406982422,
"learning_rate": 4.2449678515039743e-07,
"logits/chosen": -0.8606072068214417,
"logits/rejected": -0.8443244695663452,
"logps/chosen": -344.4573669433594,
"logps/ref_chosen": -289.9663391113281,
"logps/ref_rejected": -271.335693359375,
"logps/rejected": -371.6227722167969,
"loss": 4.5509,
"margin_dpo/margin_mean": 45.796051025390625,
"margin_dpo/margin_std": 67.24652862548828,
"step": 158
},
{
"KL/chosen_KL_mean": -60.751251220703125,
"KL/mean": -74.46627807617188,
"KL/rejected_KL_mean": -88.18130493164062,
"KL/std": 58.07416534423828,
"epoch": 0.33298429319371725,
"fcm_dpo/beta": 0.007585161365568638,
"fcm_dpo/delta": 0.009133652783930302,
"fcm_dpo/margin": 27.430057525634766,
"fcm_dpo/q_t": 0.4531330466270447,
"grad_norm": 74.61146545410156,
"learning_rate": 4.2318108837739986e-07,
"logits/chosen": -0.9520941972732544,
"logits/rejected": -0.9025680422782898,
"logps/chosen": -382.12957763671875,
"logps/ref_chosen": -321.37835693359375,
"logps/ref_rejected": -250.45652770996094,
"logps/rejected": -338.63787841796875,
"loss": 5.0799,
"margin_dpo/margin_mean": 27.430057525634766,
"margin_dpo/margin_std": 72.122802734375,
"step": 159
},
{
"KL/chosen_KL_mean": -50.046653747558594,
"KL/mean": -75.42059326171875,
"KL/rejected_KL_mean": -100.79450988769531,
"KL/std": 61.544471740722656,
"epoch": 0.33507853403141363,
"fcm_dpo/beta": 0.00758227426558733,
"fcm_dpo/delta": -0.000992432702332735,
"fcm_dpo/margin": 50.74785614013672,
"fcm_dpo/q_t": 0.41024836897850037,
"grad_norm": 55.165855407714844,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": -0.8842563629150391,
"logits/rejected": -0.8855242133140564,
"logps/chosen": -326.3301696777344,
"logps/ref_chosen": -276.28350830078125,
"logps/ref_rejected": -262.7477722167969,
"logps/rejected": -363.54229736328125,
"loss": 4.3613,
"margin_dpo/margin_mean": 50.74785614013672,
"margin_dpo/margin_std": 59.35392761230469,
"step": 160
},
{
"KL/chosen_KL_mean": -59.74705123901367,
"KL/mean": -82.03538513183594,
"KL/rejected_KL_mean": -104.32372283935547,
"KL/std": 64.49849700927734,
"epoch": 0.33717277486910996,
"fcm_dpo/beta": 0.00754944933578372,
"fcm_dpo/delta": -0.02097606658935547,
"fcm_dpo/margin": 44.5766716003418,
"fcm_dpo/q_t": 0.42417094111442566,
"grad_norm": 75.2176284790039,
"learning_rate": 4.2052190435769554e-07,
"logits/chosen": -0.8945316076278687,
"logits/rejected": -0.8761196136474609,
"logps/chosen": -370.2398376464844,
"logps/ref_chosen": -310.4927978515625,
"logps/ref_rejected": -250.25347900390625,
"logps/rejected": -354.57720947265625,
"loss": 4.6438,
"margin_dpo/margin_mean": 44.5766716003418,
"margin_dpo/margin_std": 75.5418701171875,
"step": 161
},
{
"KL/chosen_KL_mean": -54.763946533203125,
"KL/mean": -75.6114273071289,
"KL/rejected_KL_mean": -96.45890808105469,
"KL/std": 60.076480865478516,
"epoch": 0.3392670157068063,
"fcm_dpo/beta": 0.00734911672770977,
"fcm_dpo/delta": -0.007708599790930748,
"fcm_dpo/margin": 41.69496154785156,
"fcm_dpo/q_t": 0.42868733406066895,
"grad_norm": 45.79068374633789,
"learning_rate": 4.1917855971495763e-07,
"logits/chosen": -0.8724191784858704,
"logits/rejected": -0.8640581965446472,
"logps/chosen": -350.8743896484375,
"logps/ref_chosen": -296.1105041503906,
"logps/ref_rejected": -253.4247589111328,
"logps/rejected": -349.8836669921875,
"loss": 4.6753,
"margin_dpo/margin_mean": 41.69496536254883,
"margin_dpo/margin_std": 67.41392517089844,
"step": 162
},
{
"KL/chosen_KL_mean": -59.032291412353516,
"KL/mean": -82.56084442138672,
"KL/rejected_KL_mean": -106.08937072753906,
"KL/std": 59.548545837402344,
"epoch": 0.3413612565445026,
"fcm_dpo/beta": 0.00737118162214756,
"fcm_dpo/delta": -0.02219436690211296,
"fcm_dpo/margin": 47.05707550048828,
"fcm_dpo/q_t": 0.4185625910758972,
"grad_norm": 65.94065856933594,
"learning_rate": 4.1782614253949255e-07,
"logits/chosen": -0.9133932590484619,
"logits/rejected": -0.9096333384513855,
"logps/chosen": -352.5321960449219,
"logps/ref_chosen": -293.4999084472656,
"logps/ref_rejected": -266.7116394042969,
"logps/rejected": -372.801025390625,
"loss": 4.5254,
"margin_dpo/margin_mean": 47.05707550048828,
"margin_dpo/margin_std": 65.4988021850586,
"step": 163
},
{
"KL/chosen_KL_mean": -69.27760314941406,
"KL/mean": -90.69017028808594,
"KL/rejected_KL_mean": -112.10274505615234,
"KL/std": 66.29005432128906,
"epoch": 0.34345549738219894,
"fcm_dpo/beta": 0.007323879282921553,
"fcm_dpo/delta": -0.013420582748949528,
"fcm_dpo/margin": 42.82514572143555,
"fcm_dpo/q_t": 0.42865079641342163,
"grad_norm": 72.55785369873047,
"learning_rate": 4.164647253573289e-07,
"logits/chosen": -0.86087965965271,
"logits/rejected": -0.8710927963256836,
"logps/chosen": -336.3271179199219,
"logps/ref_chosen": -267.04949951171875,
"logps/ref_rejected": -215.9768829345703,
"logps/rejected": -328.0796203613281,
"loss": 4.7004,
"margin_dpo/margin_mean": 42.82514572143555,
"margin_dpo/margin_std": 75.33882904052734,
"step": 164
},
{
"KL/chosen_KL_mean": -57.162994384765625,
"KL/mean": -72.81074523925781,
"KL/rejected_KL_mean": -88.45850372314453,
"KL/std": 55.15323257446289,
"epoch": 0.34554973821989526,
"fcm_dpo/beta": 0.007216047961264849,
"fcm_dpo/delta": 0.00300390413030982,
"fcm_dpo/margin": 31.295488357543945,
"fcm_dpo/q_t": 0.4470052123069763,
"grad_norm": 65.5068588256836,
"learning_rate": 4.1509438117713863e-07,
"logits/chosen": -0.8794437646865845,
"logits/rejected": -0.8518754243850708,
"logps/chosen": -335.2244873046875,
"logps/ref_chosen": -278.06146240234375,
"logps/ref_rejected": -260.4288635253906,
"logps/rejected": -348.88739013671875,
"loss": 4.908,
"margin_dpo/margin_mean": 31.295486450195312,
"margin_dpo/margin_std": 61.843997955322266,
"step": 165
},
{
"KL/chosen_KL_mean": -51.96410369873047,
"KL/mean": -71.39328002929688,
"KL/rejected_KL_mean": -90.82244873046875,
"KL/std": 58.12259292602539,
"epoch": 0.34764397905759165,
"fcm_dpo/beta": 0.007127226330339909,
"fcm_dpo/delta": -0.0192001573741436,
"fcm_dpo/margin": 38.85832977294922,
"fcm_dpo/q_t": 0.43620848655700684,
"grad_norm": 58.496612548828125,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": -0.8499168157577515,
"logits/rejected": -0.8151109218597412,
"logps/chosen": -327.91314697265625,
"logps/ref_chosen": -275.9490661621094,
"logps/ref_rejected": -232.13473510742188,
"logps/rejected": -322.9571838378906,
"loss": 4.776,
"margin_dpo/margin_mean": 38.85832977294922,
"margin_dpo/margin_std": 71.73967742919922,
"step": 166
},
{
"KL/chosen_KL_mean": -60.47264099121094,
"KL/mean": -83.17312622070312,
"KL/rejected_KL_mean": -105.87361145019531,
"KL/std": 63.884830474853516,
"epoch": 0.34973821989528797,
"fcm_dpo/beta": 0.007206224836409092,
"fcm_dpo/delta": 0.01624479703605175,
"fcm_dpo/margin": 45.400970458984375,
"fcm_dpo/q_t": 0.4238249361515045,
"grad_norm": 78.14469146728516,
"learning_rate": 4.123272062470633e-07,
"logits/chosen": -0.8364863395690918,
"logits/rejected": -0.8216645121574402,
"logps/chosen": -341.0240478515625,
"logps/ref_chosen": -280.5514221191406,
"logps/ref_rejected": -255.2896728515625,
"logps/rejected": -361.1632995605469,
"loss": 4.6281,
"margin_dpo/margin_mean": 45.400970458984375,
"margin_dpo/margin_std": 73.75985717773438,
"step": 167
},
{
"KL/chosen_KL_mean": -62.252197265625,
"KL/mean": -86.59355163574219,
"KL/rejected_KL_mean": -110.93489837646484,
"KL/std": 66.37821960449219,
"epoch": 0.3518324607329843,
"fcm_dpo/beta": 0.007193275727331638,
"fcm_dpo/delta": -0.006395729724317789,
"fcm_dpo/margin": 48.682701110839844,
"fcm_dpo/q_t": 0.41802850365638733,
"grad_norm": 124.62866973876953,
"learning_rate": 4.1093052389237174e-07,
"logits/chosen": -0.8164465427398682,
"logits/rejected": -0.7877838611602783,
"logps/chosen": -378.0504150390625,
"logps/ref_chosen": -315.7982177734375,
"logps/ref_rejected": -291.48406982421875,
"logps/rejected": -402.4189758300781,
"loss": 4.5506,
"margin_dpo/margin_mean": 48.682701110839844,
"margin_dpo/margin_std": 73.01609802246094,
"step": 168
},
{
"KL/chosen_KL_mean": -57.24802017211914,
"KL/mean": -87.19869232177734,
"KL/rejected_KL_mean": -117.14934539794922,
"KL/std": 67.05406188964844,
"epoch": 0.3539267015706806,
"fcm_dpo/beta": 0.007014545612037182,
"fcm_dpo/delta": -0.061939746141433716,
"fcm_dpo/margin": 59.90133285522461,
"fcm_dpo/q_t": 0.40285056829452515,
"grad_norm": 89.98390197753906,
"learning_rate": 4.0952521132208267e-07,
"logits/chosen": -0.8162607550621033,
"logits/rejected": -0.8252590894699097,
"logps/chosen": -318.3122863769531,
"logps/ref_chosen": -261.06427001953125,
"logps/ref_rejected": -235.40663146972656,
"logps/rejected": -352.55596923828125,
"loss": 4.3106,
"margin_dpo/margin_mean": 59.90133285522461,
"margin_dpo/margin_std": 72.81208801269531,
"step": 169
},
{
"KL/chosen_KL_mean": -56.95838165283203,
"KL/mean": -81.45191955566406,
"KL/rejected_KL_mean": -105.94544982910156,
"KL/std": 70.3679428100586,
"epoch": 0.35602094240837695,
"fcm_dpo/beta": 0.006829574704170227,
"fcm_dpo/delta": 0.006389938294887543,
"fcm_dpo/margin": 48.9870719909668,
"fcm_dpo/q_t": 0.42464104294776917,
"grad_norm": 79.10480499267578,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": -0.7996067404747009,
"logits/rejected": -0.8019770383834839,
"logps/chosen": -365.9256286621094,
"logps/ref_chosen": -308.96722412109375,
"logps/ref_rejected": -263.8466796875,
"logps/rejected": -369.7921142578125,
"loss": 4.6634,
"margin_dpo/margin_mean": 48.98707580566406,
"margin_dpo/margin_std": 88.27198791503906,
"step": 170
},
{
"KL/chosen_KL_mean": -48.412353515625,
"KL/mean": -77.40523529052734,
"KL/rejected_KL_mean": -106.39811706542969,
"KL/std": 66.31096649169922,
"epoch": 0.3581151832460733,
"fcm_dpo/beta": 0.006731792353093624,
"fcm_dpo/delta": -0.03171330690383911,
"fcm_dpo/margin": 57.98576354980469,
"fcm_dpo/q_t": 0.40904250741004944,
"grad_norm": 76.98529052734375,
"learning_rate": 4.0668899744407567e-07,
"logits/chosen": -0.8132805824279785,
"logits/rejected": -0.8281208872795105,
"logps/chosen": -307.3013916015625,
"logps/ref_chosen": -258.8890380859375,
"logps/ref_rejected": -262.19140625,
"logps/rejected": -368.5894775390625,
"loss": 4.3738,
"margin_dpo/margin_mean": 57.98576354980469,
"margin_dpo/margin_std": 71.38015747070312,
"step": 171
},
{
"KL/chosen_KL_mean": -54.201725006103516,
"KL/mean": -70.12786865234375,
"KL/rejected_KL_mean": -86.05400085449219,
"KL/std": 60.457340240478516,
"epoch": 0.36020942408376966,
"fcm_dpo/beta": 0.006770864129066467,
"fcm_dpo/delta": 0.021626032888889313,
"fcm_dpo/margin": 31.85228157043457,
"fcm_dpo/q_t": 0.4491257965564728,
"grad_norm": 90.36054229736328,
"learning_rate": 4.0525824823390043e-07,
"logits/chosen": -0.8342244625091553,
"logits/rejected": -0.850119411945343,
"logps/chosen": -393.22406005859375,
"logps/ref_chosen": -339.0223388671875,
"logps/ref_rejected": -295.78759765625,
"logps/rejected": -381.84161376953125,
"loss": 4.9797,
"margin_dpo/margin_mean": 31.852283477783203,
"margin_dpo/margin_std": 74.22462463378906,
"step": 172
},
{
"KL/chosen_KL_mean": -56.15380096435547,
"KL/mean": -77.50936889648438,
"KL/rejected_KL_mean": -98.86494445800781,
"KL/std": 64.10746765136719,
"epoch": 0.362303664921466,
"fcm_dpo/beta": 0.006724307779222727,
"fcm_dpo/delta": -0.025682823732495308,
"fcm_dpo/margin": 42.711151123046875,
"fcm_dpo/q_t": 0.43435436487197876,
"grad_norm": 104.70634460449219,
"learning_rate": 4.0381917299505686e-07,
"logits/chosen": -0.8539585471153259,
"logits/rejected": -0.8501390814781189,
"logps/chosen": -356.2652282714844,
"logps/ref_chosen": -300.1114501953125,
"logps/ref_rejected": -273.78460693359375,
"logps/rejected": -372.6495666503906,
"loss": 4.7538,
"margin_dpo/margin_mean": 42.711151123046875,
"margin_dpo/margin_std": 76.24950408935547,
"step": 173
},
{
"KL/chosen_KL_mean": -56.64954376220703,
"KL/mean": -83.61141967773438,
"KL/rejected_KL_mean": -110.57327270507812,
"KL/std": 65.21598815917969,
"epoch": 0.3643979057591623,
"fcm_dpo/beta": 0.00670324731618166,
"fcm_dpo/delta": 0.006629655137658119,
"fcm_dpo/margin": 53.923744201660156,
"fcm_dpo/q_t": 0.41451215744018555,
"grad_norm": 70.95905303955078,
"learning_rate": 4.0237184890078243e-07,
"logits/chosen": -0.8421565294265747,
"logits/rejected": -0.8234596848487854,
"logps/chosen": -391.703369140625,
"logps/ref_chosen": -335.0538635253906,
"logps/ref_rejected": -257.4646911621094,
"logps/rejected": -368.0379638671875,
"loss": 4.4474,
"margin_dpo/margin_mean": 53.923744201660156,
"margin_dpo/margin_std": 69.16473388671875,
"step": 174
},
{
"KL/chosen_KL_mean": -73.34457397460938,
"KL/mean": -97.99015808105469,
"KL/rejected_KL_mean": -122.63572692871094,
"KL/std": 74.52360534667969,
"epoch": 0.36649214659685864,
"fcm_dpo/beta": 0.006607607007026672,
"fcm_dpo/delta": -0.04977422207593918,
"fcm_dpo/margin": 49.29114532470703,
"fcm_dpo/q_t": 0.42731624841690063,
"grad_norm": 61.65116882324219,
"learning_rate": 4.00916353566676e-07,
"logits/chosen": -0.8486623764038086,
"logits/rejected": -0.8486083149909973,
"logps/chosen": -357.7401428222656,
"logps/ref_chosen": -284.39556884765625,
"logps/ref_rejected": -283.3876647949219,
"logps/rejected": -406.02337646484375,
"loss": 4.6827,
"margin_dpo/margin_mean": 49.29115295410156,
"margin_dpo/margin_std": 83.50072479248047,
"step": 175
},
{
"KL/chosen_KL_mean": -77.07302856445312,
"KL/mean": -96.13621520996094,
"KL/rejected_KL_mean": -115.19940185546875,
"KL/std": 70.39031219482422,
"epoch": 0.36858638743455496,
"fcm_dpo/beta": 0.006414460018277168,
"fcm_dpo/delta": -0.01322941668331623,
"fcm_dpo/margin": 38.12636184692383,
"fcm_dpo/q_t": 0.44413357973098755,
"grad_norm": 84.40768432617188,
"learning_rate": 3.994527650465352e-07,
"logits/chosen": -0.8279335498809814,
"logits/rejected": -0.8326810002326965,
"logps/chosen": -328.8858337402344,
"logps/ref_chosen": -251.81280517578125,
"logps/ref_rejected": -242.05328369140625,
"logps/rejected": -357.2526550292969,
"loss": 4.9587,
"margin_dpo/margin_mean": 38.12635803222656,
"margin_dpo/margin_std": 88.72412872314453,
"step": 176
},
{
"KL/chosen_KL_mean": -91.10723876953125,
"KL/mean": -110.1352310180664,
"KL/rejected_KL_mean": -129.1632080078125,
"KL/std": 69.8150634765625,
"epoch": 0.3706806282722513,
"fcm_dpo/beta": 0.006443779915571213,
"fcm_dpo/delta": 0.014410565607249737,
"fcm_dpo/margin": 38.05598449707031,
"fcm_dpo/q_t": 0.4435146152973175,
"grad_norm": 86.33255004882812,
"learning_rate": 3.979811618281705e-07,
"logits/chosen": -0.9177388548851013,
"logits/rejected": -0.8854184746742249,
"logps/chosen": -389.7536315917969,
"logps/ref_chosen": -298.6463928222656,
"logps/ref_rejected": -295.66534423828125,
"logps/rejected": -424.8285217285156,
"loss": 4.9676,
"margin_dpo/margin_mean": 38.05598068237305,
"margin_dpo/margin_std": 86.24855041503906,
"step": 177
},
{
"KL/chosen_KL_mean": -78.96491241455078,
"KL/mean": -106.84060668945312,
"KL/rejected_KL_mean": -134.71629333496094,
"KL/std": 75.7911376953125,
"epoch": 0.37277486910994767,
"fcm_dpo/beta": 0.006407511420547962,
"fcm_dpo/delta": -0.014248996041715145,
"fcm_dpo/margin": 55.751380920410156,
"fcm_dpo/q_t": 0.4190220236778259,
"grad_norm": 78.22713470458984,
"learning_rate": 3.9650162282919654e-07,
"logits/chosen": -0.8397620320320129,
"logits/rejected": -0.8298240303993225,
"logps/chosen": -365.22259521484375,
"logps/ref_chosen": -286.2576599121094,
"logps/ref_rejected": -243.97491455078125,
"logps/rejected": -378.69122314453125,
"loss": 4.5792,
"margin_dpo/margin_mean": 55.751380920410156,
"margin_dpo/margin_std": 90.83074951171875,
"step": 178
},
{
"KL/chosen_KL_mean": -84.91183471679688,
"KL/mean": -111.3154067993164,
"KL/rejected_KL_mean": -137.71897888183594,
"KL/std": 76.51798248291016,
"epoch": 0.374869109947644,
"fcm_dpo/beta": 0.006282145623117685,
"fcm_dpo/delta": -0.02392883226275444,
"fcm_dpo/margin": 52.80714797973633,
"fcm_dpo/q_t": 0.42275673151016235,
"grad_norm": 82.36101531982422,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": -0.8264894485473633,
"logits/rejected": -0.825171709060669,
"logps/chosen": -344.64892578125,
"logps/ref_chosen": -259.737060546875,
"logps/ref_rejected": -277.8813171386719,
"logps/rejected": -415.6003112792969,
"loss": 4.6261,
"margin_dpo/margin_mean": 52.80714797973633,
"margin_dpo/margin_std": 83.93762969970703,
"step": 179
},
{
"KL/chosen_KL_mean": -87.58425903320312,
"KL/mean": -113.29141235351562,
"KL/rejected_KL_mean": -138.99856567382812,
"KL/std": 72.59463500976562,
"epoch": 0.3769633507853403,
"fcm_dpo/beta": 0.006279164459556341,
"fcm_dpo/delta": 0.013550683856010437,
"fcm_dpo/margin": 51.414302825927734,
"fcm_dpo/q_t": 0.4277075529098511,
"grad_norm": 102.1092758178711,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": -0.8694858551025391,
"logits/rejected": -0.8932338356971741,
"logps/chosen": -354.8931579589844,
"logps/ref_chosen": -267.30889892578125,
"logps/ref_rejected": -230.4376983642578,
"logps/rejected": -369.436279296875,
"loss": 4.6452,
"margin_dpo/margin_mean": 51.41429901123047,
"margin_dpo/margin_std": 85.73615264892578,
"step": 180
},
{
"KL/chosen_KL_mean": -95.89186096191406,
"KL/mean": -118.81684112548828,
"KL/rejected_KL_mean": -141.7418212890625,
"KL/std": 74.99784088134766,
"epoch": 0.37905759162303665,
"fcm_dpo/beta": 0.006303788162767887,
"fcm_dpo/delta": 0.002721858210861683,
"fcm_dpo/margin": 45.849979400634766,
"fcm_dpo/q_t": 0.43346357345581055,
"grad_norm": 80.63015747070312,
"learning_rate": 3.920161866827889e-07,
"logits/chosen": -0.857856810092926,
"logits/rejected": -0.856174886226654,
"logps/chosen": -396.38323974609375,
"logps/ref_chosen": -300.49139404296875,
"logps/ref_rejected": -278.98284912109375,
"logps/rejected": -420.72467041015625,
"loss": 4.7731,
"margin_dpo/margin_mean": 45.849979400634766,
"margin_dpo/margin_std": 86.88410186767578,
"step": 181
},
{
"KL/chosen_KL_mean": -90.28527069091797,
"KL/mean": -123.19674682617188,
"KL/rejected_KL_mean": -156.1082000732422,
"KL/std": 74.51239013671875,
"epoch": 0.381151832460733,
"fcm_dpo/beta": 0.00616471329703927,
"fcm_dpo/delta": -0.06726302951574326,
"fcm_dpo/margin": 65.82292938232422,
"fcm_dpo/q_t": 0.40653538703918457,
"grad_norm": 70.58544158935547,
"learning_rate": 3.90505702185e-07,
"logits/chosen": -0.8527212142944336,
"logits/rejected": -0.8740633726119995,
"logps/chosen": -369.7834167480469,
"logps/ref_chosen": -279.4981689453125,
"logps/ref_rejected": -263.6926574707031,
"logps/rejected": -419.8008117675781,
"loss": 4.3893,
"margin_dpo/margin_mean": 65.82292938232422,
"margin_dpo/margin_std": 87.4244613647461,
"step": 182
},
{
"KL/chosen_KL_mean": -88.05329895019531,
"KL/mean": -115.47635650634766,
"KL/rejected_KL_mean": -142.8994140625,
"KL/std": 75.31936645507812,
"epoch": 0.3832460732984293,
"fcm_dpo/beta": 0.006069937255233526,
"fcm_dpo/delta": 0.017182346433401108,
"fcm_dpo/margin": 54.846107482910156,
"fcm_dpo/q_t": 0.42420393228530884,
"grad_norm": 76.18987274169922,
"learning_rate": 3.889876827928156e-07,
"logits/chosen": -0.8800171613693237,
"logits/rejected": -0.8790852427482605,
"logps/chosen": -359.259033203125,
"logps/ref_chosen": -271.2057189941406,
"logps/ref_rejected": -243.91549682617188,
"logps/rejected": -386.8149108886719,
"loss": 4.6149,
"margin_dpo/margin_mean": 54.846107482910156,
"margin_dpo/margin_std": 89.88021087646484,
"step": 183
},
{
"KL/chosen_KL_mean": -83.444580078125,
"KL/mean": -122.65955352783203,
"KL/rejected_KL_mean": -161.87454223632812,
"KL/std": 84.31695556640625,
"epoch": 0.38534031413612563,
"fcm_dpo/beta": 0.00580790126696229,
"fcm_dpo/delta": -0.07667214423418045,
"fcm_dpo/margin": 78.4299545288086,
"fcm_dpo/q_t": 0.39444056153297424,
"grad_norm": 72.20956420898438,
"learning_rate": 3.874622099130087e-07,
"logits/chosen": -0.929442822933197,
"logits/rejected": -0.9039252996444702,
"logps/chosen": -401.89031982421875,
"logps/ref_chosen": -318.4457702636719,
"logps/ref_rejected": -266.640869140625,
"logps/rejected": -428.5154113769531,
"loss": 4.2258,
"margin_dpo/margin_mean": 78.4299545288086,
"margin_dpo/margin_std": 94.27364349365234,
"step": 184
},
{
"KL/chosen_KL_mean": -90.38389587402344,
"KL/mean": -119.19790649414062,
"KL/rejected_KL_mean": -148.01194763183594,
"KL/std": 88.37677764892578,
"epoch": 0.387434554973822,
"fcm_dpo/beta": 0.005671571008861065,
"fcm_dpo/delta": -0.00434712041169405,
"fcm_dpo/margin": 57.628047943115234,
"fcm_dpo/q_t": 0.42499980330467224,
"grad_norm": 110.43806457519531,
"learning_rate": 3.859293653520604e-07,
"logits/chosen": -0.8981891870498657,
"logits/rejected": -0.8912757635116577,
"logps/chosen": -364.69268798828125,
"logps/ref_chosen": -274.308837890625,
"logps/ref_rejected": -260.7274169921875,
"logps/rejected": -408.7393493652344,
"loss": 4.63,
"margin_dpo/margin_mean": 57.62804412841797,
"margin_dpo/margin_std": 93.35250091552734,
"step": 185
},
{
"KL/chosen_KL_mean": -89.22308349609375,
"KL/mean": -115.18913269042969,
"KL/rejected_KL_mean": -141.15518188476562,
"KL/std": 79.0693359375,
"epoch": 0.38952879581151834,
"fcm_dpo/beta": 0.005763459485024214,
"fcm_dpo/delta": 0.013718879781663418,
"fcm_dpo/margin": 51.932098388671875,
"fcm_dpo/q_t": 0.4305761456489563,
"grad_norm": 92.03166961669922,
"learning_rate": 3.8438923131177237e-07,
"logits/chosen": -0.9018892049789429,
"logits/rejected": -0.9007343649864197,
"logps/chosen": -388.22845458984375,
"logps/ref_chosen": -299.00537109375,
"logps/ref_rejected": -274.4014587402344,
"logps/rejected": -415.5566101074219,
"loss": 4.6983,
"margin_dpo/margin_mean": 51.932098388671875,
"margin_dpo/margin_std": 86.81627655029297,
"step": 186
},
{
"KL/chosen_KL_mean": -96.13522338867188,
"KL/mean": -119.4954605102539,
"KL/rejected_KL_mean": -142.85569763183594,
"KL/std": 82.50918579101562,
"epoch": 0.39162303664921466,
"fcm_dpo/beta": 0.0057730223052203655,
"fcm_dpo/delta": 0.004285093396902084,
"fcm_dpo/margin": 46.72047805786133,
"fcm_dpo/q_t": 0.43553128838539124,
"grad_norm": 85.7626953125,
"learning_rate": 3.828418903848593e-07,
"logits/chosen": -0.8542200922966003,
"logits/rejected": -0.8379209637641907,
"logps/chosen": -425.96051025390625,
"logps/ref_chosen": -329.8253173828125,
"logps/ref_rejected": -263.73175048828125,
"logps/rejected": -406.58746337890625,
"loss": 4.8707,
"margin_dpo/margin_mean": 46.72047805786133,
"margin_dpo/margin_std": 97.68164825439453,
"step": 187
},
{
"KL/chosen_KL_mean": -85.97469329833984,
"KL/mean": -112.66793823242188,
"KL/rejected_KL_mean": -139.36119079589844,
"KL/std": 84.3029556274414,
"epoch": 0.393717277486911,
"fcm_dpo/beta": 0.0057581509463489056,
"fcm_dpo/delta": -0.01138460636138916,
"fcm_dpo/margin": 53.38649368286133,
"fcm_dpo/q_t": 0.43061012029647827,
"grad_norm": 67.76911926269531,
"learning_rate": 3.812874255505191e-07,
"logits/chosen": -0.8586629629135132,
"logits/rejected": -0.8478258848190308,
"logps/chosen": -348.9803161621094,
"logps/ref_chosen": -263.005615234375,
"logps/ref_rejected": -247.08668518066406,
"logps/rejected": -386.4478454589844,
"loss": 4.7691,
"margin_dpo/margin_mean": 53.38649368286133,
"margin_dpo/margin_std": 103.5580825805664,
"step": 188
},
{
"KL/chosen_KL_mean": -68.06344604492188,
"KL/mean": -101.9673080444336,
"KL/rejected_KL_mean": -135.8711700439453,
"KL/std": 82.10550689697266,
"epoch": 0.3958115183246073,
"fcm_dpo/beta": 0.00560600683093071,
"fcm_dpo/delta": -0.03640016168355942,
"fcm_dpo/margin": 67.8077392578125,
"fcm_dpo/q_t": 0.4118382930755615,
"grad_norm": 69.77622985839844,
"learning_rate": 3.797259201699833e-07,
"logits/chosen": -0.880431592464447,
"logits/rejected": -0.8870152235031128,
"logps/chosen": -341.0238342285156,
"logps/ref_chosen": -272.96038818359375,
"logps/ref_rejected": -275.13238525390625,
"logps/rejected": -411.0035705566406,
"loss": 4.4049,
"margin_dpo/margin_mean": 67.80773162841797,
"margin_dpo/margin_std": 85.12550354003906,
"step": 189
},
{
"KL/chosen_KL_mean": -74.26818084716797,
"KL/mean": -101.46267700195312,
"KL/rejected_KL_mean": -128.65719604492188,
"KL/std": 77.80656433105469,
"epoch": 0.39790575916230364,
"fcm_dpo/beta": 0.005546726752072573,
"fcm_dpo/delta": -0.009571824222803116,
"fcm_dpo/margin": 54.38901138305664,
"fcm_dpo/q_t": 0.4297546446323395,
"grad_norm": 93.16829681396484,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": -0.8819738030433655,
"logits/rejected": -0.841058611869812,
"logps/chosen": -332.06573486328125,
"logps/ref_chosen": -257.79754638671875,
"logps/ref_rejected": -225.2164306640625,
"logps/rejected": -353.8736572265625,
"loss": 4.657,
"margin_dpo/margin_mean": 54.38901138305664,
"margin_dpo/margin_std": 85.22750091552734,
"step": 190
},
{
"KL/chosen_KL_mean": -66.46964263916016,
"KL/mean": -94.91885375976562,
"KL/rejected_KL_mean": -123.3680648803711,
"KL/std": 79.13052368164062,
"epoch": 0.4,
"fcm_dpo/beta": 0.0055416785180568695,
"fcm_dpo/delta": 0.0067793321795761585,
"fcm_dpo/margin": 56.89841842651367,
"fcm_dpo/q_t": 0.4267407953739166,
"grad_norm": 61.709129333496094,
"learning_rate": 3.765821230985757e-07,
"logits/chosen": -0.9132006764411926,
"logits/rejected": -0.9146799445152283,
"logps/chosen": -310.3281555175781,
"logps/ref_chosen": -243.8585205078125,
"logps/ref_rejected": -245.12136840820312,
"logps/rejected": -368.48944091796875,
"loss": 4.6375,
"margin_dpo/margin_mean": 56.89841842651367,
"margin_dpo/margin_std": 91.43920135498047,
"step": 191
},
{
"KL/chosen_KL_mean": -79.08763122558594,
"KL/mean": -101.60113525390625,
"KL/rejected_KL_mean": -124.11463928222656,
"KL/std": 76.82417297363281,
"epoch": 0.40209424083769635,
"fcm_dpo/beta": 0.0056621660478413105,
"fcm_dpo/delta": 0.029165830463171005,
"fcm_dpo/margin": 45.02701187133789,
"fcm_dpo/q_t": 0.44025444984436035,
"grad_norm": 73.44487762451172,
"learning_rate": 3.75e-07,
"logits/chosen": -0.8472084999084473,
"logits/rejected": -0.8306607604026794,
"logps/chosen": -346.06756591796875,
"logps/ref_chosen": -266.9799499511719,
"logps/ref_rejected": -260.1697082519531,
"logps/rejected": -384.2843017578125,
"loss": 4.824,
"margin_dpo/margin_mean": 45.02701187133789,
"margin_dpo/margin_std": 83.66022491455078,
"step": 192
},
{
"KL/chosen_KL_mean": -94.40106964111328,
"KL/mean": -121.47966003417969,
"KL/rejected_KL_mean": -148.55825805664062,
"KL/std": 83.79367065429688,
"epoch": 0.4041884816753927,
"fcm_dpo/beta": 0.005736154038459063,
"fcm_dpo/delta": 0.005536096170544624,
"fcm_dpo/margin": 54.157196044921875,
"fcm_dpo/q_t": 0.42890992760658264,
"grad_norm": 72.95458221435547,
"learning_rate": 3.734111735307796e-07,
"logits/chosen": -0.8859913945198059,
"logits/rejected": -0.8564121723175049,
"logps/chosen": -374.654296875,
"logps/ref_chosen": -280.25323486328125,
"logps/ref_rejected": -291.0348815917969,
"logps/rejected": -439.5931396484375,
"loss": 4.7054,
"margin_dpo/margin_mean": 54.157196044921875,
"margin_dpo/margin_std": 94.62388610839844,
"step": 193
},
{
"KL/chosen_KL_mean": -92.71996307373047,
"KL/mean": -116.64244842529297,
"KL/rejected_KL_mean": -140.56492614746094,
"KL/std": 82.40608978271484,
"epoch": 0.406282722513089,
"fcm_dpo/beta": 0.00563084939494729,
"fcm_dpo/delta": -0.02582578919827938,
"fcm_dpo/margin": 47.844966888427734,
"fcm_dpo/q_t": 0.43761441111564636,
"grad_norm": 99.4243392944336,
"learning_rate": 3.7181572889485623e-07,
"logits/chosen": -0.8625931739807129,
"logits/rejected": -0.8507343530654907,
"logps/chosen": -380.8594055175781,
"logps/ref_chosen": -288.13946533203125,
"logps/ref_rejected": -251.31529235839844,
"logps/rejected": -391.8802185058594,
"loss": 4.7859,
"margin_dpo/margin_mean": 47.844966888427734,
"margin_dpo/margin_std": 85.31089782714844,
"step": 194
},
{
"KL/chosen_KL_mean": -97.91946411132812,
"KL/mean": -119.41505432128906,
"KL/rejected_KL_mean": -140.91062927246094,
"KL/std": 80.02857971191406,
"epoch": 0.4083769633507853,
"fcm_dpo/beta": 0.005613365676254034,
"fcm_dpo/delta": -0.0017420882359147072,
"fcm_dpo/margin": 42.99113464355469,
"fcm_dpo/q_t": 0.44507643580436707,
"grad_norm": 73.5301513671875,
"learning_rate": 3.7021375165108377e-07,
"logits/chosen": -0.8627254366874695,
"logits/rejected": -0.8665878176689148,
"logps/chosen": -371.92010498046875,
"logps/ref_chosen": -274.0006408691406,
"logps/ref_rejected": -280.22723388671875,
"logps/rejected": -421.13787841796875,
"loss": 4.8972,
"margin_dpo/margin_mean": 42.99113464355469,
"margin_dpo/margin_std": 88.47681427001953,
"step": 195
},
{
"KL/chosen_KL_mean": -82.35733032226562,
"KL/mean": -109.65511322021484,
"KL/rejected_KL_mean": -136.95291137695312,
"KL/std": 82.34046936035156,
"epoch": 0.41047120418848165,
"fcm_dpo/beta": 0.005610158666968346,
"fcm_dpo/delta": -0.0006429278291761875,
"fcm_dpo/margin": 54.595584869384766,
"fcm_dpo/q_t": 0.42899465560913086,
"grad_norm": 76.89717864990234,
"learning_rate": 3.6860532770864005e-07,
"logits/chosen": -0.8541459441184998,
"logits/rejected": -0.8584092855453491,
"logps/chosen": -357.25799560546875,
"logps/ref_chosen": -274.90069580078125,
"logps/ref_rejected": -248.7281951904297,
"logps/rejected": -385.6811218261719,
"loss": 4.6883,
"margin_dpo/margin_mean": 54.595584869384766,
"margin_dpo/margin_std": 92.98194122314453,
"step": 196
},
{
"KL/chosen_KL_mean": -92.13257598876953,
"KL/mean": -123.7594985961914,
"KL/rejected_KL_mean": -155.38644409179688,
"KL/std": 82.77664184570312,
"epoch": 0.41256544502617803,
"fcm_dpo/beta": 0.00550233107060194,
"fcm_dpo/delta": -0.03711705654859543,
"fcm_dpo/margin": 63.25384521484375,
"fcm_dpo/q_t": 0.4192579984664917,
"grad_norm": 90.26445770263672,
"learning_rate": 3.6699054332241985e-07,
"logits/chosen": -0.8871943950653076,
"logits/rejected": -0.8706080913543701,
"logps/chosen": -401.66741943359375,
"logps/ref_chosen": -309.5348205566406,
"logps/ref_rejected": -264.3179931640625,
"logps/rejected": -419.7044372558594,
"loss": 4.54,
"margin_dpo/margin_mean": 63.25384521484375,
"margin_dpo/margin_std": 89.81802368164062,
"step": 197
},
{
"KL/chosen_KL_mean": -87.10678100585938,
"KL/mean": -119.78683471679688,
"KL/rejected_KL_mean": -152.46690368652344,
"KL/std": 88.0517807006836,
"epoch": 0.41465968586387436,
"fcm_dpo/beta": 0.005422430112957954,
"fcm_dpo/delta": 0.002035717014223337,
"fcm_dpo/margin": 65.360107421875,
"fcm_dpo/q_t": 0.41849538683891296,
"grad_norm": 61.8067512512207,
"learning_rate": 3.653694850884091e-07,
"logits/chosen": -0.8917360305786133,
"logits/rejected": -0.8658944964408875,
"logps/chosen": -388.1202392578125,
"logps/ref_chosen": -301.0134582519531,
"logps/ref_rejected": -292.84185791015625,
"logps/rejected": -445.3087158203125,
"loss": 4.5293,
"margin_dpo/margin_mean": 65.36009979248047,
"margin_dpo/margin_std": 97.01239013671875,
"step": 198
},
{
"KL/chosen_KL_mean": -97.43031311035156,
"KL/mean": -124.94877624511719,
"KL/rejected_KL_mean": -152.46722412109375,
"KL/std": 83.84455871582031,
"epoch": 0.4167539267015707,
"fcm_dpo/beta": 0.0055153826251626015,
"fcm_dpo/delta": 0.018592506647109985,
"fcm_dpo/margin": 55.036903381347656,
"fcm_dpo/q_t": 0.4302946627140045,
"grad_norm": 96.4948501586914,
"learning_rate": 3.6374223993904124e-07,
"logits/chosen": -0.8654062747955322,
"logits/rejected": -0.8202941417694092,
"logps/chosen": -362.03619384765625,
"logps/ref_chosen": -264.6058654785156,
"logps/ref_rejected": -214.9014892578125,
"logps/rejected": -367.36871337890625,
"loss": 4.6993,
"margin_dpo/margin_mean": 55.03690719604492,
"margin_dpo/margin_std": 95.58619689941406,
"step": 199
},
{
"KL/chosen_KL_mean": -107.61213684082031,
"KL/mean": -131.58660888671875,
"KL/rejected_KL_mean": -155.56106567382812,
"KL/std": 85.42179870605469,
"epoch": 0.418848167539267,
"fcm_dpo/beta": 0.005476394202560186,
"fcm_dpo/delta": -0.020585589110851288,
"fcm_dpo/margin": 47.948944091796875,
"fcm_dpo/q_t": 0.4395952820777893,
"grad_norm": 106.97425842285156,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": -0.9183908700942993,
"logits/rejected": -0.8932676315307617,
"logps/chosen": -431.7709655761719,
"logps/ref_chosen": -324.1588134765625,
"logps/ref_rejected": -277.80218505859375,
"logps/rejected": -433.36328125,
"loss": 4.9173,
"margin_dpo/margin_mean": 47.948944091796875,
"margin_dpo/margin_std": 107.63665008544922,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_KL/chosen_KL_mean": -118.4775161743164,
"eval_KL/mean": -146.10684204101562,
"eval_KL/rejected_KL_mean": -173.73617553710938,
"eval_KL/std": 87.68894958496094,
"eval_fcm_dpo/beta": 0.005431666970252991,
"eval_logits/chosen": -0.9033797979354858,
"eval_logits/rejected": -0.8850231170654297,
"eval_logps/chosen": -406.3043212890625,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -440.666259765625,
"eval_loss": 0.5937886834144592,
"eval_margin_dpo/margin_mean": 55.258663177490234,
"eval_margin_dpo/margin_std": 96.29785919189453,
"eval_runtime": 78.3994,
"eval_samples_per_second": 25.51,
"eval_steps_per_second": 3.189,
"step": 200
},
{
"KL/chosen_KL_mean": -114.04182434082031,
"KL/mean": -142.33677673339844,
"KL/rejected_KL_mean": -170.63168334960938,
"KL/std": 90.65510559082031,
"epoch": 0.42094240837696334,
"fcm_dpo/beta": 0.005450923927128315,
"fcm_dpo/delta": -0.003941504284739494,
"fcm_dpo/margin": 56.58985900878906,
"fcm_dpo/q_t": 0.4291457533836365,
"grad_norm": 69.15496826171875,
"learning_rate": 3.604695382782159e-07,
"logits/chosen": -0.8902816772460938,
"logits/rejected": -0.8751832246780396,
"logps/chosen": -385.53753662109375,
"logps/ref_chosen": -271.49566650390625,
"logps/ref_rejected": -245.71414184570312,
"logps/rejected": -416.3458251953125,
"loss": 4.7244,
"margin_dpo/margin_mean": 56.58985900878906,
"margin_dpo/margin_std": 102.68513488769531,
"step": 201
},
{
"KL/chosen_KL_mean": -119.89775848388672,
"KL/mean": -146.6784210205078,
"KL/rejected_KL_mean": -173.45907592773438,
"KL/std": 88.81134796142578,
"epoch": 0.42303664921465967,
"fcm_dpo/beta": 0.005403751973062754,
"fcm_dpo/delta": 9.850505739450455e-05,
"fcm_dpo/margin": 53.561317443847656,
"fcm_dpo/q_t": 0.43379461765289307,
"grad_norm": 89.58255767822266,
"learning_rate": 3.588242572718162e-07,
"logits/chosen": -0.9021543264389038,
"logits/rejected": -0.8883557915687561,
"logps/chosen": -391.9956970214844,
"logps/ref_chosen": -272.0979309082031,
"logps/ref_rejected": -235.94805908203125,
"logps/rejected": -409.4071350097656,
"loss": 4.7836,
"margin_dpo/margin_mean": 53.561317443847656,
"margin_dpo/margin_std": 102.62480163574219,
"step": 202
},
{
"KL/chosen_KL_mean": -130.4102783203125,
"KL/mean": -151.12998962402344,
"KL/rejected_KL_mean": -171.84970092773438,
"KL/std": 87.56797790527344,
"epoch": 0.42513089005235605,
"fcm_dpo/beta": 0.00546378456056118,
"fcm_dpo/delta": 0.014473012648522854,
"fcm_dpo/margin": 41.439414978027344,
"fcm_dpo/q_t": 0.44730111956596375,
"grad_norm": 102.8392562866211,
"learning_rate": 3.571731403507635e-07,
"logits/chosen": -0.8795283436775208,
"logits/rejected": -0.8848550319671631,
"logps/chosen": -410.6324462890625,
"logps/ref_chosen": -280.2221374511719,
"logps/ref_rejected": -251.79798889160156,
"logps/rejected": -423.6476745605469,
"loss": 4.9634,
"margin_dpo/margin_mean": 41.439414978027344,
"margin_dpo/margin_std": 95.13355255126953,
"step": 203
},
{
"KL/chosen_KL_mean": -132.2213592529297,
"KL/mean": -160.99591064453125,
"KL/rejected_KL_mean": -189.77044677734375,
"KL/std": 87.84689331054688,
"epoch": 0.4272251308900524,
"fcm_dpo/beta": 0.005502759013324976,
"fcm_dpo/delta": 0.007985341362655163,
"fcm_dpo/margin": 57.549095153808594,
"fcm_dpo/q_t": 0.42689085006713867,
"grad_norm": 125.6895751953125,
"learning_rate": 3.5551627605944746e-07,
"logits/chosen": -0.9232524633407593,
"logits/rejected": -0.8936776518821716,
"logps/chosen": -451.0173645019531,
"logps/ref_chosen": -318.7960510253906,
"logps/ref_rejected": -269.69921875,
"logps/rejected": -459.4697265625,
"loss": 4.671,
"margin_dpo/margin_mean": 57.549102783203125,
"margin_dpo/margin_std": 98.45879364013672,
"step": 204
},
{
"KL/chosen_KL_mean": -136.3469696044922,
"KL/mean": -168.44393920898438,
"KL/rejected_KL_mean": -200.54092407226562,
"KL/std": 92.91146850585938,
"epoch": 0.4293193717277487,
"fcm_dpo/beta": 0.005452723708003759,
"fcm_dpo/delta": -0.045753512531518936,
"fcm_dpo/margin": 64.19393920898438,
"fcm_dpo/q_t": 0.421352356672287,
"grad_norm": 128.93753051757812,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": -0.8728401064872742,
"logits/rejected": -0.841010570526123,
"logps/chosen": -420.10906982421875,
"logps/ref_chosen": -283.7620544433594,
"logps/ref_rejected": -297.69439697265625,
"logps/rejected": -498.2353210449219,
"loss": 4.6025,
"margin_dpo/margin_mean": 64.19393920898438,
"margin_dpo/margin_std": 103.74689483642578,
"step": 205
},
{
"KL/chosen_KL_mean": -142.62896728515625,
"KL/mean": -167.98255920410156,
"KL/rejected_KL_mean": -193.33612060546875,
"KL/std": 92.03362274169922,
"epoch": 0.431413612565445,
"fcm_dpo/beta": 0.005307645071297884,
"fcm_dpo/delta": -0.0065862322226166725,
"fcm_dpo/margin": 50.70718002319336,
"fcm_dpo/q_t": 0.43794897198677063,
"grad_norm": 107.9933090209961,
"learning_rate": 3.5218566107988867e-07,
"logits/chosen": -0.895841121673584,
"logits/rejected": -0.9164851903915405,
"logps/chosen": -436.2928466796875,
"logps/ref_chosen": -293.66387939453125,
"logps/ref_rejected": -291.3056640625,
"logps/rejected": -484.6418151855469,
"loss": 4.8515,
"margin_dpo/margin_mean": 50.70718002319336,
"margin_dpo/margin_std": 103.65234375,
"step": 206
},
{
"KL/chosen_KL_mean": -129.6578826904297,
"KL/mean": -155.3979034423828,
"KL/rejected_KL_mean": -181.13790893554688,
"KL/std": 103.08729553222656,
"epoch": 0.43350785340314135,
"fcm_dpo/beta": 0.005296694580465555,
"fcm_dpo/delta": 0.01233250554651022,
"fcm_dpo/margin": 51.480037689208984,
"fcm_dpo/q_t": 0.43691202998161316,
"grad_norm": 71.85640716552734,
"learning_rate": 3.505120890024195e-07,
"logits/chosen": -0.8434122800827026,
"logits/rejected": -0.8525888323783875,
"logps/chosen": -400.1929931640625,
"logps/ref_chosen": -270.5350646972656,
"logps/ref_rejected": -278.7747497558594,
"logps/rejected": -459.91265869140625,
"loss": 4.9192,
"margin_dpo/margin_mean": 51.480037689208984,
"margin_dpo/margin_std": 117.58082580566406,
"step": 207
},
{
"KL/chosen_KL_mean": -137.4970703125,
"KL/mean": -168.57623291015625,
"KL/rejected_KL_mean": -199.65541076660156,
"KL/std": 101.80587005615234,
"epoch": 0.4356020942408377,
"fcm_dpo/beta": 0.0053030322305858135,
"fcm_dpo/delta": -0.015698019415140152,
"fcm_dpo/margin": 62.15835952758789,
"fcm_dpo/q_t": 0.42619529366493225,
"grad_norm": 76.81256103515625,
"learning_rate": 3.4883312676665534e-07,
"logits/chosen": -0.9078364968299866,
"logits/rejected": -0.8600209355354309,
"logps/chosen": -417.0798034667969,
"logps/ref_chosen": -279.582763671875,
"logps/ref_rejected": -290.041015625,
"logps/rejected": -489.6964111328125,
"loss": 4.6342,
"margin_dpo/margin_mean": 62.15835952758789,
"margin_dpo/margin_std": 105.14048767089844,
"step": 208
},
{
"KL/chosen_KL_mean": -139.73585510253906,
"KL/mean": -159.71441650390625,
"KL/rejected_KL_mean": -179.6929931640625,
"KL/std": 94.56495666503906,
"epoch": 0.437696335078534,
"fcm_dpo/beta": 0.005294729024171829,
"fcm_dpo/delta": 0.011330154724419117,
"fcm_dpo/margin": 39.9571533203125,
"fcm_dpo/q_t": 0.4502761662006378,
"grad_norm": 107.32901763916016,
"learning_rate": 3.4714886441024573e-07,
"logits/chosen": -0.8233649730682373,
"logits/rejected": -0.8163270950317383,
"logps/chosen": -458.6083679199219,
"logps/ref_chosen": -318.8725280761719,
"logps/ref_rejected": -270.64324951171875,
"logps/rejected": -450.33624267578125,
"loss": 5.0317,
"margin_dpo/margin_mean": 39.9571533203125,
"margin_dpo/margin_std": 99.94036102294922,
"step": 209
},
{
"KL/chosen_KL_mean": -127.69154357910156,
"KL/mean": -158.0561981201172,
"KL/rejected_KL_mean": -188.4208526611328,
"KL/std": 90.50584411621094,
"epoch": 0.4397905759162304,
"fcm_dpo/beta": 0.0052696773782372475,
"fcm_dpo/delta": -0.023277752101421356,
"fcm_dpo/margin": 60.729339599609375,
"fcm_dpo/q_t": 0.4259607493877411,
"grad_norm": 83.70458984375,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": -0.8556398153305054,
"logits/rejected": -0.8458552360534668,
"logps/chosen": -410.83184814453125,
"logps/ref_chosen": -283.14031982421875,
"logps/ref_rejected": -287.2986755371094,
"logps/rejected": -475.71954345703125,
"loss": 4.6405,
"margin_dpo/margin_mean": 60.729331970214844,
"margin_dpo/margin_std": 97.90538024902344,
"step": 210
},
{
"KL/chosen_KL_mean": -111.53688049316406,
"KL/mean": -149.20333862304688,
"KL/rejected_KL_mean": -186.8698272705078,
"KL/std": 91.30079650878906,
"epoch": 0.4418848167539267,
"fcm_dpo/beta": 0.005068170838057995,
"fcm_dpo/delta": -0.047747448086738586,
"fcm_dpo/margin": 75.33293151855469,
"fcm_dpo/q_t": 0.4104166328907013,
"grad_norm": 83.38289642333984,
"learning_rate": 3.4376480090239047e-07,
"logits/chosen": -0.8847814798355103,
"logits/rejected": -0.8608183860778809,
"logps/chosen": -387.9597473144531,
"logps/ref_chosen": -276.4228515625,
"logps/ref_rejected": -252.40603637695312,
"logps/rejected": -439.2758483886719,
"loss": 4.3908,
"margin_dpo/margin_mean": 75.33293151855469,
"margin_dpo/margin_std": 90.77347564697266,
"step": 211
},
{
"KL/chosen_KL_mean": -120.57503509521484,
"KL/mean": -145.82603454589844,
"KL/rejected_KL_mean": -171.07704162597656,
"KL/std": 89.40132141113281,
"epoch": 0.44397905759162304,
"fcm_dpo/beta": 0.005025006365031004,
"fcm_dpo/delta": 0.008119989186525345,
"fcm_dpo/margin": 50.502010345458984,
"fcm_dpo/q_t": 0.4400903582572937,
"grad_norm": 74.55123138427734,
"learning_rate": 3.4206518122800055e-07,
"logits/chosen": -0.8634406328201294,
"logits/rejected": -0.8582726716995239,
"logps/chosen": -392.28057861328125,
"logps/ref_chosen": -271.7055358886719,
"logps/ref_rejected": -241.18511962890625,
"logps/rejected": -412.26220703125,
"loss": 4.8539,
"margin_dpo/margin_mean": 50.50200653076172,
"margin_dpo/margin_std": 98.8302001953125,
"step": 212
},
{
"KL/chosen_KL_mean": -118.58316802978516,
"KL/mean": -146.1175537109375,
"KL/rejected_KL_mean": -173.6519317626953,
"KL/std": 103.11312103271484,
"epoch": 0.44607329842931936,
"fcm_dpo/beta": 0.0050498321652412415,
"fcm_dpo/delta": -0.005192122422158718,
"fcm_dpo/margin": 55.06875991821289,
"fcm_dpo/q_t": 0.43775975704193115,
"grad_norm": 82.58657836914062,
"learning_rate": 3.403606243773448e-07,
"logits/chosen": -0.8551130294799805,
"logits/rejected": -0.8685761094093323,
"logps/chosen": -420.88079833984375,
"logps/ref_chosen": -302.2976379394531,
"logps/ref_rejected": -303.6202087402344,
"logps/rejected": -477.2721252441406,
"loss": 4.8435,
"margin_dpo/margin_mean": 55.06875991821289,
"margin_dpo/margin_std": 114.744384765625,
"step": 213
},
{
"KL/chosen_KL_mean": -125.1562728881836,
"KL/mean": -158.9164581298828,
"KL/rejected_KL_mean": -192.6766357421875,
"KL/std": 91.4408950805664,
"epoch": 0.4481675392670157,
"fcm_dpo/beta": 0.004997848533093929,
"fcm_dpo/delta": -0.02767416462302208,
"fcm_dpo/margin": 67.52037811279297,
"fcm_dpo/q_t": 0.42145538330078125,
"grad_norm": 90.98759460449219,
"learning_rate": 3.3865122176063385e-07,
"logits/chosen": -0.8687431216239929,
"logits/rejected": -0.8687351942062378,
"logps/chosen": -397.2889099121094,
"logps/ref_chosen": -272.13262939453125,
"logps/ref_rejected": -294.82354736328125,
"logps/rejected": -487.50018310546875,
"loss": 4.535,
"margin_dpo/margin_mean": 67.5203857421875,
"margin_dpo/margin_std": 91.6003189086914,
"step": 214
},
{
"KL/chosen_KL_mean": -128.5072479248047,
"KL/mean": -150.73866271972656,
"KL/rejected_KL_mean": -172.97007751464844,
"KL/std": 100.23798370361328,
"epoch": 0.450261780104712,
"fcm_dpo/beta": 0.004917190410196781,
"fcm_dpo/delta": -0.015194178558886051,
"fcm_dpo/margin": 44.46282958984375,
"fcm_dpo/q_t": 0.45000001788139343,
"grad_norm": 86.00048828125,
"learning_rate": 3.3693706504794243e-07,
"logits/chosen": -0.9164157509803772,
"logits/rejected": -0.8946561813354492,
"logps/chosen": -419.8855285644531,
"logps/ref_chosen": -291.3782958984375,
"logps/ref_rejected": -261.05792236328125,
"logps/rejected": -434.02801513671875,
"loss": 4.9817,
"margin_dpo/margin_mean": 44.46282958984375,
"margin_dpo/margin_std": 101.5790786743164,
"step": 215
},
{
"KL/chosen_KL_mean": -115.26673889160156,
"KL/mean": -144.14666748046875,
"KL/rejected_KL_mean": -173.02658081054688,
"KL/std": 91.47891998291016,
"epoch": 0.4523560209424084,
"fcm_dpo/beta": 0.004956618882715702,
"fcm_dpo/delta": 0.033862657845020294,
"fcm_dpo/margin": 57.759849548339844,
"fcm_dpo/q_t": 0.4338792860507965,
"grad_norm": 67.37702178955078,
"learning_rate": 3.3521824616429284e-07,
"logits/chosen": -0.9285804033279419,
"logits/rejected": -0.9208376407623291,
"logps/chosen": -453.7721862792969,
"logps/ref_chosen": -338.50543212890625,
"logps/ref_rejected": -305.76104736328125,
"logps/rejected": -478.7876281738281,
"loss": 4.7734,
"margin_dpo/margin_mean": 57.759849548339844,
"margin_dpo/margin_std": 107.90360260009766,
"step": 216
},
{
"KL/chosen_KL_mean": -118.00670623779297,
"KL/mean": -152.04202270507812,
"KL/rejected_KL_mean": -186.07737731933594,
"KL/std": 93.6181869506836,
"epoch": 0.4544502617801047,
"fcm_dpo/beta": 0.00493368785828352,
"fcm_dpo/delta": -0.02944992296397686,
"fcm_dpo/margin": 68.07068634033203,
"fcm_dpo/q_t": 0.4224163889884949,
"grad_norm": 73.08317565917969,
"learning_rate": 3.334948572847253e-07,
"logits/chosen": -0.8377935886383057,
"logits/rejected": -0.7964776754379272,
"logps/chosen": -411.5565185546875,
"logps/ref_chosen": -293.5498046875,
"logps/ref_rejected": -256.7830810546875,
"logps/rejected": -442.8605041503906,
"loss": 4.5865,
"margin_dpo/margin_mean": 68.07068634033203,
"margin_dpo/margin_std": 105.03534698486328,
"step": 217
},
{
"KL/chosen_KL_mean": -114.50249481201172,
"KL/mean": -144.8571319580078,
"KL/rejected_KL_mean": -175.2117462158203,
"KL/std": 92.68726348876953,
"epoch": 0.45654450261780105,
"fcm_dpo/beta": 0.0048797354102134705,
"fcm_dpo/delta": -0.008044907823204994,
"fcm_dpo/margin": 60.709266662597656,
"fcm_dpo/q_t": 0.4313804805278778,
"grad_norm": 64.33407592773438,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": -0.8685387372970581,
"logits/rejected": -0.8849347829818726,
"logps/chosen": -435.08184814453125,
"logps/ref_chosen": -320.579345703125,
"logps/ref_rejected": -294.0381164550781,
"logps/rejected": -469.2499084472656,
"loss": 4.6864,
"margin_dpo/margin_mean": 60.70926284790039,
"margin_dpo/margin_std": 99.89984130859375,
"step": 218
},
{
"KL/chosen_KL_mean": -108.34378051757812,
"KL/mean": -139.96334838867188,
"KL/rejected_KL_mean": -171.58291625976562,
"KL/std": 83.42886352539062,
"epoch": 0.4586387434554974,
"fcm_dpo/beta": 0.004817603155970573,
"fcm_dpo/delta": -0.010519558563828468,
"fcm_dpo/margin": 63.2391471862793,
"fcm_dpo/q_t": 0.4293725788593292,
"grad_norm": 86.39237976074219,
"learning_rate": 3.300347394584172e-07,
"logits/chosen": -0.847774863243103,
"logits/rejected": -0.8739175796508789,
"logps/chosen": -376.76239013671875,
"logps/ref_chosen": -268.4186096191406,
"logps/ref_rejected": -265.7808837890625,
"logps/rejected": -437.36376953125,
"loss": 4.6705,
"margin_dpo/margin_mean": 63.2391471862793,
"margin_dpo/margin_std": 100.10897827148438,
"step": 219
},
{
"KL/chosen_KL_mean": -109.47624206542969,
"KL/mean": -141.2556610107422,
"KL/rejected_KL_mean": -173.03509521484375,
"KL/std": 82.5869369506836,
"epoch": 0.4607329842931937,
"fcm_dpo/beta": 0.004888718016445637,
"fcm_dpo/delta": 0.03557638078927994,
"fcm_dpo/margin": 63.5588493347168,
"fcm_dpo/q_t": 0.4272876977920532,
"grad_norm": 79.07698822021484,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": -0.9154024124145508,
"logits/rejected": -0.8836403489112854,
"logps/chosen": -422.3627014160156,
"logps/ref_chosen": -312.8864440917969,
"logps/ref_rejected": -259.5191955566406,
"logps/rejected": -432.5542907714844,
"loss": 4.5985,
"margin_dpo/margin_mean": 63.55884552001953,
"margin_dpo/margin_std": 90.30359649658203,
"step": 220
},
{
"KL/chosen_KL_mean": -113.12982940673828,
"KL/mean": -138.3195037841797,
"KL/rejected_KL_mean": -163.50918579101562,
"KL/std": 86.43636322021484,
"epoch": 0.46282722513089003,
"fcm_dpo/beta": 0.004951215349137783,
"fcm_dpo/delta": -0.004505933728069067,
"fcm_dpo/margin": 50.37934112548828,
"fcm_dpo/q_t": 0.4432408809661865,
"grad_norm": 70.59790802001953,
"learning_rate": 3.265574537815398e-07,
"logits/chosen": -0.8173964023590088,
"logits/rejected": -0.8285130262374878,
"logps/chosen": -413.4556579589844,
"logps/ref_chosen": -300.32586669921875,
"logps/ref_rejected": -286.312255859375,
"logps/rejected": -449.8214111328125,
"loss": 4.8711,
"margin_dpo/margin_mean": 50.37934112548828,
"margin_dpo/margin_std": 101.99186706542969,
"step": 221
},
{
"KL/chosen_KL_mean": -106.68103790283203,
"KL/mean": -134.0705108642578,
"KL/rejected_KL_mean": -161.45997619628906,
"KL/std": 78.9898681640625,
"epoch": 0.4649214659685864,
"fcm_dpo/beta": 0.004972555674612522,
"fcm_dpo/delta": 0.014529617503285408,
"fcm_dpo/margin": 54.77894592285156,
"fcm_dpo/q_t": 0.43578585982322693,
"grad_norm": 83.3537826538086,
"learning_rate": 3.248126059518784e-07,
"logits/chosen": -0.9040961265563965,
"logits/rejected": -0.8702518939971924,
"logps/chosen": -403.79241943359375,
"logps/ref_chosen": -297.1113586425781,
"logps/ref_rejected": -235.53146362304688,
"logps/rejected": -396.991455078125,
"loss": 4.7331,
"margin_dpo/margin_mean": 54.7789421081543,
"margin_dpo/margin_std": 88.59034729003906,
"step": 222
},
{
"KL/chosen_KL_mean": -112.10569763183594,
"KL/mean": -143.2996826171875,
"KL/rejected_KL_mean": -174.49366760253906,
"KL/std": 86.74018096923828,
"epoch": 0.46701570680628274,
"fcm_dpo/beta": 0.005028898827731609,
"fcm_dpo/delta": 0.010601907968521118,
"fcm_dpo/margin": 62.38795852661133,
"fcm_dpo/q_t": 0.4269620180130005,
"grad_norm": 95.17221069335938,
"learning_rate": 3.230637461492043e-07,
"logits/chosen": -0.8466207385063171,
"logits/rejected": -0.8117715716362,
"logps/chosen": -398.52081298828125,
"logps/ref_chosen": -286.41510009765625,
"logps/ref_rejected": -241.1181640625,
"logps/rejected": -415.61181640625,
"loss": 4.6314,
"margin_dpo/margin_mean": 62.38795852661133,
"margin_dpo/margin_std": 97.02510070800781,
"step": 223
},
{
"KL/chosen_KL_mean": -103.6304702758789,
"KL/mean": -138.8751220703125,
"KL/rejected_KL_mean": -174.11976623535156,
"KL/std": 85.77777099609375,
"epoch": 0.46910994764397906,
"fcm_dpo/beta": 0.005118512082844973,
"fcm_dpo/delta": 0.005692493170499802,
"fcm_dpo/margin": 70.48930358886719,
"fcm_dpo/q_t": 0.4164232611656189,
"grad_norm": 69.0962905883789,
"learning_rate": 3.213109681595612e-07,
"logits/chosen": -0.8163408637046814,
"logits/rejected": -0.8316698670387268,
"logps/chosen": -353.1227722167969,
"logps/ref_chosen": -249.49234008789062,
"logps/ref_rejected": -233.10752868652344,
"logps/rejected": -407.2273254394531,
"loss": 4.4561,
"margin_dpo/margin_mean": 70.48930358886719,
"margin_dpo/margin_std": 91.14543151855469,
"step": 224
},
{
"KL/chosen_KL_mean": -123.72078704833984,
"KL/mean": -154.9224395751953,
"KL/rejected_KL_mean": -186.1240997314453,
"KL/std": 93.72308349609375,
"epoch": 0.4712041884816754,
"fcm_dpo/beta": 0.005051084794104099,
"fcm_dpo/delta": 0.011575451120734215,
"fcm_dpo/margin": 62.4033088684082,
"fcm_dpo/q_t": 0.42811909317970276,
"grad_norm": 72.46212005615234,
"learning_rate": 3.1955436597911315e-07,
"logits/chosen": -0.8486655354499817,
"logits/rejected": -0.821964681148529,
"logps/chosen": -435.57916259765625,
"logps/ref_chosen": -311.8583679199219,
"logps/ref_rejected": -336.8523864746094,
"logps/rejected": -522.9765014648438,
"loss": 4.6715,
"margin_dpo/margin_mean": 62.40331268310547,
"margin_dpo/margin_std": 105.1444091796875,
"step": 225
},
{
"KL/chosen_KL_mean": -112.75259399414062,
"KL/mean": -141.30508422851562,
"KL/rejected_KL_mean": -169.85757446289062,
"KL/std": 89.33605194091797,
"epoch": 0.4732984293193717,
"fcm_dpo/beta": 0.005086014047265053,
"fcm_dpo/delta": -0.0021250159479677677,
"fcm_dpo/margin": 57.104957580566406,
"fcm_dpo/q_t": 0.4333423972129822,
"grad_norm": 56.582576751708984,
"learning_rate": 3.1779403380910425e-07,
"logits/chosen": -0.8840992450714111,
"logits/rejected": -0.8711143136024475,
"logps/chosen": -364.9538269042969,
"logps/ref_chosen": -252.20123291015625,
"logps/ref_rejected": -254.41162109375,
"logps/rejected": -424.2691650390625,
"loss": 4.7484,
"margin_dpo/margin_mean": 57.104957580566406,
"margin_dpo/margin_std": 105.84939575195312,
"step": 226
},
{
"KL/chosen_KL_mean": -117.15921020507812,
"KL/mean": -151.5059814453125,
"KL/rejected_KL_mean": -185.85281372070312,
"KL/std": 86.63410949707031,
"epoch": 0.47539267015706804,
"fcm_dpo/beta": 0.0050234198570251465,
"fcm_dpo/delta": -0.021003495901823044,
"fcm_dpo/margin": 68.6935806274414,
"fcm_dpo/q_t": 0.41943684220314026,
"grad_norm": 72.25760650634766,
"learning_rate": 3.160300660508064e-07,
"logits/chosen": -0.8448896408081055,
"logits/rejected": -0.8412713408470154,
"logps/chosen": -402.4186706542969,
"logps/ref_chosen": -285.25946044921875,
"logps/ref_rejected": -261.3220520019531,
"logps/rejected": -447.1748352050781,
"loss": 4.5729,
"margin_dpo/margin_mean": 68.6935806274414,
"margin_dpo/margin_std": 106.60133361816406,
"step": 227
},
{
"KL/chosen_KL_mean": -108.93597412109375,
"KL/mean": -143.87466430664062,
"KL/rejected_KL_mean": -178.81336975097656,
"KL/std": 85.76429748535156,
"epoch": 0.4774869109947644,
"fcm_dpo/beta": 0.005010589491575956,
"fcm_dpo/delta": -0.023567285388708115,
"fcm_dpo/margin": 69.87741088867188,
"fcm_dpo/q_t": 0.4189654588699341,
"grad_norm": 60.705631256103516,
"learning_rate": 3.1426255730045695e-07,
"logits/chosen": -0.8538888692855835,
"logits/rejected": -0.8198533654212952,
"logps/chosen": -422.7547607421875,
"logps/ref_chosen": -313.81878662109375,
"logps/ref_rejected": -258.07061767578125,
"logps/rejected": -436.8840026855469,
"loss": 4.511,
"margin_dpo/margin_mean": 69.87741088867188,
"margin_dpo/margin_std": 96.2317886352539,
"step": 228
},
{
"KL/chosen_KL_mean": -128.9109344482422,
"KL/mean": -168.88983154296875,
"KL/rejected_KL_mean": -208.8687286376953,
"KL/std": 94.05682373046875,
"epoch": 0.47958115183246075,
"fcm_dpo/beta": 0.00481749139726162,
"fcm_dpo/delta": -0.03823835402727127,
"fcm_dpo/margin": 79.95777130126953,
"fcm_dpo/q_t": 0.4103056490421295,
"grad_norm": 84.75572967529297,
"learning_rate": 3.1249160234418644e-07,
"logits/chosen": -0.8427700400352478,
"logits/rejected": -0.8486419916152954,
"logps/chosen": -420.8816833496094,
"logps/ref_chosen": -291.9707946777344,
"logps/ref_rejected": -263.42059326171875,
"logps/rejected": -472.2893371582031,
"loss": 4.4112,
"margin_dpo/margin_mean": 79.95777893066406,
"margin_dpo/margin_std": 104.22063446044922,
"step": 229
},
{
"KL/chosen_KL_mean": -126.17520141601562,
"KL/mean": -160.9757843017578,
"KL/rejected_KL_mean": -195.77639770507812,
"KL/std": 99.210205078125,
"epoch": 0.4816753926701571,
"fcm_dpo/beta": 0.004781956318765879,
"fcm_dpo/delta": 0.01782013103365898,
"fcm_dpo/margin": 69.6011734008789,
"fcm_dpo/q_t": 0.42428159713745117,
"grad_norm": 75.2130355834961,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": -0.8815920948982239,
"logits/rejected": -0.8813921213150024,
"logps/chosen": -359.435302734375,
"logps/ref_chosen": -233.2601318359375,
"logps/ref_rejected": -238.922119140625,
"logps/rejected": -434.698486328125,
"loss": 4.6226,
"margin_dpo/margin_mean": 69.6011734008789,
"margin_dpo/margin_std": 115.32308197021484,
"step": 230
},
{
"KL/chosen_KL_mean": -137.8847198486328,
"KL/mean": -166.61566162109375,
"KL/rejected_KL_mean": -195.3466033935547,
"KL/std": 88.32843780517578,
"epoch": 0.4837696335078534,
"fcm_dpo/beta": 0.004834107588976622,
"fcm_dpo/delta": 0.005063319578766823,
"fcm_dpo/margin": 57.46186828613281,
"fcm_dpo/q_t": 0.4348789155483246,
"grad_norm": 71.28133392333984,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": -0.8586534261703491,
"logits/rejected": -0.8459566235542297,
"logps/chosen": -460.0399169921875,
"logps/ref_chosen": -322.1551818847656,
"logps/ref_rejected": -280.97613525390625,
"logps/rejected": -476.3227844238281,
"loss": 4.7507,
"margin_dpo/margin_mean": 57.461883544921875,
"margin_dpo/margin_std": 99.15318298339844,
"step": 231
},
{
"KL/chosen_KL_mean": -136.840576171875,
"KL/mean": -166.6940155029297,
"KL/rejected_KL_mean": -196.54745483398438,
"KL/std": 97.31487274169922,
"epoch": 0.48586387434554973,
"fcm_dpo/beta": 0.004791594110429287,
"fcm_dpo/delta": -0.025890890508890152,
"fcm_dpo/margin": 59.70685577392578,
"fcm_dpo/q_t": 0.43394574522972107,
"grad_norm": 77.55052947998047,
"learning_rate": 3.071590108427243e-07,
"logits/chosen": -0.833568811416626,
"logits/rejected": -0.8151888251304626,
"logps/chosen": -408.5843505859375,
"logps/ref_chosen": -271.7437744140625,
"logps/ref_rejected": -249.94981384277344,
"logps/rejected": -446.4972839355469,
"loss": 4.7829,
"margin_dpo/margin_mean": 59.70685958862305,
"margin_dpo/margin_std": 108.40725708007812,
"step": 232
},
{
"KL/chosen_KL_mean": -142.8009033203125,
"KL/mean": -177.56996154785156,
"KL/rejected_KL_mean": -212.33900451660156,
"KL/std": 95.32579803466797,
"epoch": 0.48795811518324606,
"fcm_dpo/beta": 0.004678625613451004,
"fcm_dpo/delta": -0.037981029599905014,
"fcm_dpo/margin": 69.5381088256836,
"fcm_dpo/q_t": 0.42466118931770325,
"grad_norm": 63.76099395751953,
"learning_rate": 3.05375222543809e-07,
"logits/chosen": -0.8736680746078491,
"logits/rejected": -0.8643991947174072,
"logps/chosen": -428.14324951171875,
"logps/ref_chosen": -285.3423156738281,
"logps/ref_rejected": -266.34320068359375,
"logps/rejected": -478.68218994140625,
"loss": 4.6045,
"margin_dpo/margin_mean": 69.5381088256836,
"margin_dpo/margin_std": 104.16007232666016,
"step": 233
},
{
"KL/chosen_KL_mean": -138.01681518554688,
"KL/mean": -173.01083374023438,
"KL/rejected_KL_mean": -208.0048370361328,
"KL/std": 98.38790130615234,
"epoch": 0.4900523560209424,
"fcm_dpo/beta": 0.004634845070540905,
"fcm_dpo/delta": 0.012818412855267525,
"fcm_dpo/margin": 69.98805236816406,
"fcm_dpo/q_t": 0.42441120743751526,
"grad_norm": 64.4365005493164,
"learning_rate": 3.035884646397637e-07,
"logits/chosen": -0.8443531394004822,
"logits/rejected": -0.8304785490036011,
"logps/chosen": -432.92254638671875,
"logps/ref_chosen": -294.9057312011719,
"logps/ref_rejected": -299.37054443359375,
"logps/rejected": -507.37542724609375,
"loss": 4.6463,
"margin_dpo/margin_mean": 69.98805236816406,
"margin_dpo/margin_std": 114.38517761230469,
"step": 234
},
{
"KL/chosen_KL_mean": -149.86141967773438,
"KL/mean": -180.7827606201172,
"KL/rejected_KL_mean": -211.7041015625,
"KL/std": 103.66209411621094,
"epoch": 0.49214659685863876,
"fcm_dpo/beta": 0.0047287228517234325,
"fcm_dpo/delta": 0.039121147245168686,
"fcm_dpo/margin": 61.84267044067383,
"fcm_dpo/q_t": 0.4319824278354645,
"grad_norm": 76.6629638671875,
"learning_rate": 3.017988329489923e-07,
"logits/chosen": -0.863947868347168,
"logits/rejected": -0.855870246887207,
"logps/chosen": -439.3590087890625,
"logps/ref_chosen": -289.49755859375,
"logps/ref_rejected": -247.55076599121094,
"logps/rejected": -459.2548522949219,
"loss": 4.7503,
"margin_dpo/margin_mean": 61.84267044067383,
"margin_dpo/margin_std": 115.53366088867188,
"step": 235
},
{
"KL/chosen_KL_mean": -145.44284057617188,
"KL/mean": -175.6923828125,
"KL/rejected_KL_mean": -205.94192504882812,
"KL/std": 100.11052703857422,
"epoch": 0.4942408376963351,
"fcm_dpo/beta": 0.004794456996023655,
"fcm_dpo/delta": -0.02471328340470791,
"fcm_dpo/margin": 60.49909591674805,
"fcm_dpo/q_t": 0.43273258209228516,
"grad_norm": 72.24032592773438,
"learning_rate": 3.000064234440111e-07,
"logits/chosen": -0.8775507807731628,
"logits/rejected": -0.8777625560760498,
"logps/chosen": -434.3274841308594,
"logps/ref_chosen": -288.8846435546875,
"logps/ref_rejected": -242.0452880859375,
"logps/rejected": -447.9872131347656,
"loss": 4.7912,
"margin_dpo/margin_mean": 60.49909210205078,
"margin_dpo/margin_std": 114.68053436279297,
"step": 236
},
{
"KL/chosen_KL_mean": -158.35903930664062,
"KL/mean": -189.57205200195312,
"KL/rejected_KL_mean": -220.7850341796875,
"KL/std": 89.05985260009766,
"epoch": 0.4963350785340314,
"fcm_dpo/beta": 0.004720585886389017,
"fcm_dpo/delta": 0.015051835216581821,
"fcm_dpo/margin": 62.426025390625,
"fcm_dpo/q_t": 0.4318310618400574,
"grad_norm": 90.479248046875,
"learning_rate": 2.9821133224630223e-07,
"logits/chosen": -0.8398151397705078,
"logits/rejected": -0.8189886212348938,
"logps/chosen": -423.8377685546875,
"logps/ref_chosen": -265.47869873046875,
"logps/ref_rejected": -267.9891357421875,
"logps/rejected": -488.774169921875,
"loss": 4.7226,
"margin_dpo/margin_mean": 62.426025390625,
"margin_dpo/margin_std": 109.10443115234375,
"step": 237
},
{
"KL/chosen_KL_mean": -163.52406311035156,
"KL/mean": -198.2044677734375,
"KL/rejected_KL_mean": -232.88482666015625,
"KL/std": 100.38282775878906,
"epoch": 0.49842931937172774,
"fcm_dpo/beta": 0.0047994209453463554,
"fcm_dpo/delta": 0.0028489064425230026,
"fcm_dpo/margin": 69.36079406738281,
"fcm_dpo/q_t": 0.42530369758605957,
"grad_norm": 77.99054718017578,
"learning_rate": 2.964136556211588e-07,
"logits/chosen": -0.8484768867492676,
"logits/rejected": -0.8160721659660339,
"logps/chosen": -475.5267333984375,
"logps/ref_chosen": -312.0026550292969,
"logps/ref_rejected": -270.0257263183594,
"logps/rejected": -502.9105224609375,
"loss": 4.6355,
"margin_dpo/margin_mean": 69.36080169677734,
"margin_dpo/margin_std": 117.8523941040039,
"step": 238
},
{
"KL/chosen_KL_mean": -161.76864624023438,
"KL/mean": -191.57293701171875,
"KL/rejected_KL_mean": -221.37716674804688,
"KL/std": 100.97264862060547,
"epoch": 0.5005235602094241,
"fcm_dpo/beta": 0.004840420559048653,
"fcm_dpo/delta": 0.02600950561463833,
"fcm_dpo/margin": 59.608524322509766,
"fcm_dpo/q_t": 0.4337758719921112,
"grad_norm": 65.61921691894531,
"learning_rate": 2.946134899725226e-07,
"logits/chosen": -0.8408248424530029,
"logits/rejected": -0.880803108215332,
"logps/chosen": -428.93585205078125,
"logps/ref_chosen": -267.167236328125,
"logps/ref_rejected": -275.99468994140625,
"logps/rejected": -497.37188720703125,
"loss": 4.8741,
"margin_dpo/margin_mean": 59.608524322509766,
"margin_dpo/margin_std": 132.66128540039062,
"step": 239
},
{
"KL/chosen_KL_mean": -158.13919067382812,
"KL/mean": -196.2527313232422,
"KL/rejected_KL_mean": -234.36627197265625,
"KL/std": 103.12637329101562,
"epoch": 0.5026178010471204,
"fcm_dpo/beta": 0.0048082321882247925,
"fcm_dpo/delta": -0.015010528266429901,
"fcm_dpo/margin": 76.22711181640625,
"fcm_dpo/q_t": 0.4153556823730469,
"grad_norm": 79.14559173583984,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": -0.9028453230857849,
"logits/rejected": -0.8958457708358765,
"logps/chosen": -444.1188049316406,
"logps/ref_chosen": -285.9796142578125,
"logps/ref_rejected": -256.8258056640625,
"logps/rejected": -491.1921081542969,
"loss": 4.4889,
"margin_dpo/margin_mean": 76.22711181640625,
"margin_dpo/margin_std": 109.41165924072266,
"step": 240
},
{
"KL/chosen_KL_mean": -180.80047607421875,
"KL/mean": -206.89230346679688,
"KL/rejected_KL_mean": -232.984130859375,
"KL/std": 111.80062866210938,
"epoch": 0.5047120418848168,
"fcm_dpo/beta": 0.004808627534657717,
"fcm_dpo/delta": -0.006103941705077887,
"fcm_dpo/margin": 52.18364715576172,
"fcm_dpo/q_t": 0.4433174133300781,
"grad_norm": 88.64216613769531,
"learning_rate": 2.910060778827554e-07,
"logits/chosen": -0.8175089955329895,
"logits/rejected": -0.7948691844940186,
"logps/chosen": -442.31732177734375,
"logps/ref_chosen": -261.516845703125,
"logps/ref_rejected": -250.2250518798828,
"logps/rejected": -483.20916748046875,
"loss": 4.9968,
"margin_dpo/margin_mean": 52.18364715576172,
"margin_dpo/margin_std": 127.83100128173828,
"step": 241
},
{
"KL/chosen_KL_mean": -161.50523376464844,
"KL/mean": -195.77635192871094,
"KL/rejected_KL_mean": -230.0474395751953,
"KL/std": 106.44923400878906,
"epoch": 0.506806282722513,
"fcm_dpo/beta": 0.004742825403809547,
"fcm_dpo/delta": -0.012836568988859653,
"fcm_dpo/margin": 68.5422134399414,
"fcm_dpo/q_t": 0.42594078183174133,
"grad_norm": 88.80677032470703,
"learning_rate": 2.891990248961871e-07,
"logits/chosen": -0.8814034461975098,
"logits/rejected": -0.856747031211853,
"logps/chosen": -432.01922607421875,
"logps/ref_chosen": -270.51397705078125,
"logps/ref_rejected": -244.8560791015625,
"logps/rejected": -474.9035339355469,
"loss": 4.6268,
"margin_dpo/margin_mean": 68.5422134399414,
"margin_dpo/margin_std": 111.54056549072266,
"step": 242
},
{
"KL/chosen_KL_mean": -153.52972412109375,
"KL/mean": -195.6244659423828,
"KL/rejected_KL_mean": -237.71920776367188,
"KL/std": 106.60342407226562,
"epoch": 0.5089005235602094,
"fcm_dpo/beta": 0.0047475434839725494,
"fcm_dpo/delta": -0.034220486879348755,
"fcm_dpo/margin": 84.18949127197266,
"fcm_dpo/q_t": 0.41172146797180176,
"grad_norm": 102.534912109375,
"learning_rate": 2.873898697848762e-07,
"logits/chosen": -0.8814060091972351,
"logits/rejected": -0.8652539849281311,
"logps/chosen": -478.21173095703125,
"logps/ref_chosen": -324.68206787109375,
"logps/ref_rejected": -307.1111755371094,
"logps/rejected": -544.8303833007812,
"loss": 4.4595,
"margin_dpo/margin_mean": 84.18949127197266,
"margin_dpo/margin_std": 128.6507110595703,
"step": 243
},
{
"KL/chosen_KL_mean": -152.66043090820312,
"KL/mean": -192.0305633544922,
"KL/rejected_KL_mean": -231.4007110595703,
"KL/std": 100.23159790039062,
"epoch": 0.5109947643979058,
"fcm_dpo/beta": 0.004584737122058868,
"fcm_dpo/delta": -0.017737431451678276,
"fcm_dpo/margin": 78.74027252197266,
"fcm_dpo/q_t": 0.4154585003852844,
"grad_norm": 77.94464111328125,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": -0.863549530506134,
"logits/rejected": -0.8185126185417175,
"logps/chosen": -471.6396789550781,
"logps/ref_chosen": -318.979248046875,
"logps/ref_rejected": -269.67572021484375,
"logps/rejected": -501.0764465332031,
"loss": 4.5167,
"margin_dpo/margin_mean": 78.74027252197266,
"margin_dpo/margin_std": 112.84671020507812,
"step": 244
},
{
"KL/chosen_KL_mean": -143.22264099121094,
"KL/mean": -182.23419189453125,
"KL/rejected_KL_mean": -221.24574279785156,
"KL/std": 96.40372467041016,
"epoch": 0.5130890052356021,
"fcm_dpo/beta": 0.004495944362133741,
"fcm_dpo/delta": -0.03692924976348877,
"fcm_dpo/margin": 78.02310943603516,
"fcm_dpo/q_t": 0.41797515749931335,
"grad_norm": 59.87691879272461,
"learning_rate": 2.837656413735479e-07,
"logits/chosen": -0.8684712648391724,
"logits/rejected": -0.8744341731071472,
"logps/chosen": -438.1206970214844,
"logps/ref_chosen": -294.8980712890625,
"logps/ref_rejected": -239.8111114501953,
"logps/rejected": -461.0568542480469,
"loss": 4.5402,
"margin_dpo/margin_mean": 78.02310943603516,
"margin_dpo/margin_std": 112.71392822265625,
"step": 245
},
{
"KL/chosen_KL_mean": -157.49560546875,
"KL/mean": -186.96458435058594,
"KL/rejected_KL_mean": -216.43356323242188,
"KL/std": 101.2094497680664,
"epoch": 0.5151832460732985,
"fcm_dpo/beta": 0.004423782229423523,
"fcm_dpo/delta": -0.0013894562143832445,
"fcm_dpo/margin": 58.93793487548828,
"fcm_dpo/q_t": 0.44051581621170044,
"grad_norm": 90.08110809326172,
"learning_rate": 2.8195076242990116e-07,
"logits/chosen": -0.8500803709030151,
"logits/rejected": -0.8542726039886475,
"logps/chosen": -438.1810302734375,
"logps/ref_chosen": -280.6854248046875,
"logps/ref_rejected": -253.65382385253906,
"logps/rejected": -470.0873718261719,
"loss": 4.8966,
"margin_dpo/margin_mean": 58.93793487548828,
"margin_dpo/margin_std": 127.81275177001953,
"step": 246
},
{
"KL/chosen_KL_mean": -140.9234619140625,
"KL/mean": -177.16494750976562,
"KL/rejected_KL_mean": -213.40638732910156,
"KL/std": 101.20616149902344,
"epoch": 0.5172774869109947,
"fcm_dpo/beta": 0.0044204313308000565,
"fcm_dpo/delta": -0.004756400361657143,
"fcm_dpo/margin": 72.48292541503906,
"fcm_dpo/q_t": 0.4261864423751831,
"grad_norm": 62.47346878051758,
"learning_rate": 2.801341700638307e-07,
"logits/chosen": -0.8428160548210144,
"logits/rejected": -0.8401237726211548,
"logps/chosen": -422.0325927734375,
"logps/ref_chosen": -281.1091003417969,
"logps/ref_rejected": -260.3700866699219,
"logps/rejected": -473.7765197753906,
"loss": 4.6164,
"margin_dpo/margin_mean": 72.48292541503906,
"margin_dpo/margin_std": 109.14017486572266,
"step": 247
},
{
"KL/chosen_KL_mean": -138.6498565673828,
"KL/mean": -173.0037384033203,
"KL/rejected_KL_mean": -207.35763549804688,
"KL/std": 101.77959442138672,
"epoch": 0.5193717277486911,
"fcm_dpo/beta": 0.004360491875559092,
"fcm_dpo/delta": -0.01580795831978321,
"fcm_dpo/margin": 68.70777893066406,
"fcm_dpo/q_t": 0.4298118054866791,
"grad_norm": 68.65995788574219,
"learning_rate": 2.7831596169367227e-07,
"logits/chosen": -0.813584566116333,
"logits/rejected": -0.8213151693344116,
"logps/chosen": -408.9682312011719,
"logps/ref_chosen": -270.318359375,
"logps/ref_rejected": -233.46778869628906,
"logps/rejected": -440.825439453125,
"loss": 4.6974,
"margin_dpo/margin_mean": 68.70777893066406,
"margin_dpo/margin_std": 112.36260986328125,
"step": 248
},
{
"KL/chosen_KL_mean": -151.7535400390625,
"KL/mean": -181.5082550048828,
"KL/rejected_KL_mean": -211.26295471191406,
"KL/std": 104.59439849853516,
"epoch": 0.5214659685863874,
"fcm_dpo/beta": 0.004381960723549128,
"fcm_dpo/delta": 0.020416682586073875,
"fcm_dpo/margin": 59.50940704345703,
"fcm_dpo/q_t": 0.4388493001461029,
"grad_norm": 62.078365325927734,
"learning_rate": 2.7649623482442274e-07,
"logits/chosen": -0.8454968333244324,
"logits/rejected": -0.8196409344673157,
"logps/chosen": -427.56243896484375,
"logps/ref_chosen": -275.8088684082031,
"logps/ref_rejected": -243.45138549804688,
"logps/rejected": -454.7143249511719,
"loss": 4.8702,
"margin_dpo/margin_mean": 59.50940704345703,
"margin_dpo/margin_std": 124.977783203125,
"step": 249
},
{
"KL/chosen_KL_mean": -146.8738250732422,
"KL/mean": -185.1446533203125,
"KL/rejected_KL_mean": -223.41546630859375,
"KL/std": 114.59971618652344,
"epoch": 0.5235602094240838,
"fcm_dpo/beta": 0.004331628791987896,
"fcm_dpo/delta": -0.035680629312992096,
"fcm_dpo/margin": 76.54167938232422,
"fcm_dpo/q_t": 0.4232044219970703,
"grad_norm": 52.845611572265625,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": -0.8485902547836304,
"logits/rejected": -0.8440597057342529,
"logps/chosen": -439.36834716796875,
"logps/ref_chosen": -292.4945373535156,
"logps/ref_rejected": -284.2869567871094,
"logps/rejected": -507.70245361328125,
"loss": 4.6384,
"margin_dpo/margin_mean": 76.54167938232422,
"margin_dpo/margin_std": 125.18535614013672,
"step": 250
},
{
"KL/chosen_KL_mean": -128.29568481445312,
"KL/mean": -163.9156951904297,
"KL/rejected_KL_mean": -199.53573608398438,
"KL/std": 107.50858306884766,
"epoch": 0.5256544502617801,
"fcm_dpo/beta": 0.004250232130289078,
"fcm_dpo/delta": -0.027963757514953613,
"fcm_dpo/margin": 71.24006652832031,
"fcm_dpo/q_t": 0.43231555819511414,
"grad_norm": 63.01066970825195,
"learning_rate": 2.7285261601056697e-07,
"logits/chosen": -0.8441932201385498,
"logits/rejected": -0.8291101455688477,
"logps/chosen": -410.03228759765625,
"logps/ref_chosen": -281.736572265625,
"logps/ref_rejected": -255.9419708251953,
"logps/rejected": -455.47772216796875,
"loss": 4.6987,
"margin_dpo/margin_mean": 71.24006652832031,
"margin_dpo/margin_std": 119.5110092163086,
"step": 251
},
{
"KL/chosen_KL_mean": -138.88409423828125,
"KL/mean": -173.60968017578125,
"KL/rejected_KL_mean": -208.3352508544922,
"KL/std": 102.1337661743164,
"epoch": 0.5277486910994764,
"fcm_dpo/beta": 0.004174236208200455,
"fcm_dpo/delta": -0.011869529262185097,
"fcm_dpo/margin": 69.45118713378906,
"fcm_dpo/q_t": 0.43159157037734985,
"grad_norm": 62.90778732299805,
"learning_rate": 2.7102891946217994e-07,
"logits/chosen": -0.8931678533554077,
"logits/rejected": -0.8665968179702759,
"logps/chosen": -434.85150146484375,
"logps/ref_chosen": -295.9674072265625,
"logps/ref_rejected": -280.111572265625,
"logps/rejected": -488.44683837890625,
"loss": 4.7147,
"margin_dpo/margin_mean": 69.45118713378906,
"margin_dpo/margin_std": 117.09326171875,
"step": 252
},
{
"KL/chosen_KL_mean": -144.9630584716797,
"KL/mean": -177.68711853027344,
"KL/rejected_KL_mean": -210.41119384765625,
"KL/std": 112.56256103515625,
"epoch": 0.5298429319371728,
"fcm_dpo/beta": 0.00418475829064846,
"fcm_dpo/delta": 0.008287381380796432,
"fcm_dpo/margin": 65.44812774658203,
"fcm_dpo/q_t": 0.4371293783187866,
"grad_norm": 56.794734954833984,
"learning_rate": 2.692040951966617e-07,
"logits/chosen": -0.8609364628791809,
"logits/rejected": -0.8533194661140442,
"logps/chosen": -422.0353088378906,
"logps/ref_chosen": -277.072265625,
"logps/ref_rejected": -247.31643676757812,
"logps/rejected": -457.7276306152344,
"loss": 4.812,
"margin_dpo/margin_mean": 65.44812774658203,
"margin_dpo/margin_std": 124.55648040771484,
"step": 253
},
{
"KL/chosen_KL_mean": -136.63442993164062,
"KL/mean": -172.9150390625,
"KL/rejected_KL_mean": -209.19570922851562,
"KL/std": 108.1693115234375,
"epoch": 0.5319371727748691,
"fcm_dpo/beta": 0.0041767992079257965,
"fcm_dpo/delta": -0.02163619175553322,
"fcm_dpo/margin": 72.561279296875,
"fcm_dpo/q_t": 0.430123507976532,
"grad_norm": 50.65597152709961,
"learning_rate": 2.6737824107379947e-07,
"logits/chosen": -0.8018378019332886,
"logits/rejected": -0.7838313579559326,
"logps/chosen": -406.5822448730469,
"logps/ref_chosen": -269.9478454589844,
"logps/ref_rejected": -249.45005798339844,
"logps/rejected": -458.645751953125,
"loss": 4.6683,
"margin_dpo/margin_mean": 72.56128692626953,
"margin_dpo/margin_std": 116.72087097167969,
"step": 254
},
{
"KL/chosen_KL_mean": -130.73049926757812,
"KL/mean": -171.3893280029297,
"KL/rejected_KL_mean": -212.04815673828125,
"KL/std": 108.50567626953125,
"epoch": 0.5340314136125655,
"fcm_dpo/beta": 0.004186869598925114,
"fcm_dpo/delta": 0.02620495669543743,
"fcm_dpo/margin": 81.31766510009766,
"fcm_dpo/q_t": 0.42155683040618896,
"grad_norm": 64.20274353027344,
"learning_rate": 2.655514550086086e-07,
"logits/chosen": -0.8257165551185608,
"logits/rejected": -0.792091429233551,
"logps/chosen": -437.38580322265625,
"logps/ref_chosen": -306.6552734375,
"logps/ref_rejected": -254.47528076171875,
"logps/rejected": -466.5234069824219,
"loss": 4.5829,
"margin_dpo/margin_mean": 81.31766510009766,
"margin_dpo/margin_std": 127.40750122070312,
"step": 255
},
{
"KL/chosen_KL_mean": -134.72642517089844,
"KL/mean": -171.73529052734375,
"KL/rejected_KL_mean": -208.74420166015625,
"KL/std": 108.54015350341797,
"epoch": 0.5361256544502618,
"fcm_dpo/beta": 0.004111228045076132,
"fcm_dpo/delta": -0.056310322135686874,
"fcm_dpo/margin": 74.01776123046875,
"fcm_dpo/q_t": 0.4271259903907776,
"grad_norm": 98.39281463623047,
"learning_rate": 2.6372383496608186e-07,
"logits/chosen": -0.85221928358078,
"logits/rejected": -0.8382247686386108,
"logps/chosen": -458.4446105957031,
"logps/ref_chosen": -323.7181701660156,
"logps/ref_rejected": -254.1871337890625,
"logps/rejected": -462.9313049316406,
"loss": 4.7255,
"margin_dpo/margin_mean": 74.01776123046875,
"margin_dpo/margin_std": 125.59674835205078,
"step": 256
},
{
"KL/chosen_KL_mean": -137.28506469726562,
"KL/mean": -176.9401092529297,
"KL/rejected_KL_mean": -216.59512329101562,
"KL/std": 103.74283599853516,
"epoch": 0.5382198952879581,
"fcm_dpo/beta": 0.004046399611979723,
"fcm_dpo/delta": 0.012473970651626587,
"fcm_dpo/margin": 79.31005859375,
"fcm_dpo/q_t": 0.4253496825695038,
"grad_norm": 80.60918426513672,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": -0.8426645994186401,
"logits/rejected": -0.8348857164382935,
"logps/chosen": -404.4971618652344,
"logps/ref_chosen": -267.21209716796875,
"logps/ref_rejected": -249.12579345703125,
"logps/rejected": -465.72088623046875,
"loss": 4.5916,
"margin_dpo/margin_mean": 79.31005859375,
"margin_dpo/margin_std": 119.5707778930664,
"step": 257
},
{
"KL/chosen_KL_mean": -145.31399536132812,
"KL/mean": -187.23464965820312,
"KL/rejected_KL_mean": -229.1552734375,
"KL/std": 105.5355224609375,
"epoch": 0.5403141361256545,
"fcm_dpo/beta": 0.003976684529334307,
"fcm_dpo/delta": -0.035320185124874115,
"fcm_dpo/margin": 83.84129333496094,
"fcm_dpo/q_t": 0.4211466908454895,
"grad_norm": 66.4306411743164,
"learning_rate": 2.600664850273538e-07,
"logits/chosen": -0.8702799677848816,
"logits/rejected": -0.8408257961273193,
"logps/chosen": -422.9967346191406,
"logps/ref_chosen": -277.6827392578125,
"logps/ref_rejected": -250.73385620117188,
"logps/rejected": -479.8891296386719,
"loss": 4.5294,
"margin_dpo/margin_mean": 83.8412857055664,
"margin_dpo/margin_std": 111.361328125,
"step": 258
},
{
"KL/chosen_KL_mean": -138.1536102294922,
"KL/mean": -175.6834716796875,
"KL/rejected_KL_mean": -213.21331787109375,
"KL/std": 108.60619354248047,
"epoch": 0.5424083769633508,
"fcm_dpo/beta": 0.003963233437389135,
"fcm_dpo/delta": 0.0012955646961927414,
"fcm_dpo/margin": 75.05972290039062,
"fcm_dpo/q_t": 0.4318161606788635,
"grad_norm": 80.89569091796875,
"learning_rate": 2.582369512637302e-07,
"logits/chosen": -0.8907499313354492,
"logits/rejected": -0.885222315788269,
"logps/chosen": -432.7635498046875,
"logps/ref_chosen": -294.6099853515625,
"logps/ref_rejected": -272.2725830078125,
"logps/rejected": -485.4858703613281,
"loss": 4.6869,
"margin_dpo/margin_mean": 75.05972290039062,
"margin_dpo/margin_std": 123.2925033569336,
"step": 259
},
{
"KL/chosen_KL_mean": -151.42819213867188,
"KL/mean": -169.2168426513672,
"KL/rejected_KL_mean": -187.00550842285156,
"KL/std": 95.43949127197266,
"epoch": 0.5445026178010471,
"fcm_dpo/beta": 0.003953398671001196,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 35.57732391357422,
"fcm_dpo/q_t": 0.4671649634838104,
"grad_norm": 74.01944732666016,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": -0.8799023628234863,
"logits/rejected": -0.873131513595581,
"logps/chosen": -442.2852783203125,
"logps/ref_chosen": -290.85711669921875,
"logps/ref_rejected": -277.5970153808594,
"logps/rejected": -464.6025390625,
"loss": 5.2063,
"margin_dpo/margin_mean": 35.57732009887695,
"margin_dpo/margin_std": 112.50765228271484,
"step": 260
},
{
"KL/chosen_KL_mean": -163.0951690673828,
"KL/mean": -195.61669921875,
"KL/rejected_KL_mean": -228.1382293701172,
"KL/std": 109.09551239013672,
"epoch": 0.5465968586387434,
"fcm_dpo/beta": 0.003954585175961256,
"fcm_dpo/delta": 0.004022921435534954,
"fcm_dpo/margin": 65.04306030273438,
"fcm_dpo/q_t": 0.44060125946998596,
"grad_norm": 52.7375373840332,
"learning_rate": 2.5457665670441937e-07,
"logits/chosen": -0.778756856918335,
"logits/rejected": -0.787772536277771,
"logps/chosen": -414.2274169921875,
"logps/ref_chosen": -251.13223266601562,
"logps/ref_rejected": -244.76016235351562,
"logps/rejected": -472.89837646484375,
"loss": 4.8579,
"margin_dpo/margin_mean": 65.04306030273438,
"margin_dpo/margin_std": 130.55770874023438,
"step": 261
},
{
"KL/chosen_KL_mean": -157.522216796875,
"KL/mean": -198.93121337890625,
"KL/rejected_KL_mean": -240.34019470214844,
"KL/std": 108.54368591308594,
"epoch": 0.5486910994764398,
"fcm_dpo/beta": 0.003956255502998829,
"fcm_dpo/delta": -0.019527418538928032,
"fcm_dpo/margin": 82.81796264648438,
"fcm_dpo/q_t": 0.4236336648464203,
"grad_norm": 64.078125,
"learning_rate": 2.527460921992209e-07,
"logits/chosen": -0.8164616823196411,
"logits/rejected": -0.8031435012817383,
"logps/chosen": -457.2440185546875,
"logps/ref_chosen": -299.7217712402344,
"logps/ref_rejected": -277.0969543457031,
"logps/rejected": -517.4371948242188,
"loss": 4.5771,
"margin_dpo/margin_mean": 82.81796264648438,
"margin_dpo/margin_std": 120.71702575683594,
"step": 262
},
{
"KL/chosen_KL_mean": -167.35940551757812,
"KL/mean": -204.05819702148438,
"KL/rejected_KL_mean": -240.75698852539062,
"KL/std": 112.38304138183594,
"epoch": 0.5507853403141362,
"fcm_dpo/beta": 0.0039150831289589405,
"fcm_dpo/delta": 0.004613391123712063,
"fcm_dpo/margin": 73.39756774902344,
"fcm_dpo/q_t": 0.4326966404914856,
"grad_norm": 57.92847442626953,
"learning_rate": 2.509153804294318e-07,
"logits/chosen": -0.8228314518928528,
"logits/rejected": -0.8032459020614624,
"logps/chosen": -447.3119812011719,
"logps/ref_chosen": -279.95257568359375,
"logps/ref_rejected": -256.5327453613281,
"logps/rejected": -497.28973388671875,
"loss": 4.7408,
"margin_dpo/margin_mean": 73.39756774902344,
"margin_dpo/margin_std": 127.13219451904297,
"step": 263
},
{
"KL/chosen_KL_mean": -162.80029296875,
"KL/mean": -206.62985229492188,
"KL/rejected_KL_mean": -250.45938110351562,
"KL/std": 113.47503662109375,
"epoch": 0.5528795811518324,
"fcm_dpo/beta": 0.003873462090268731,
"fcm_dpo/delta": -0.020922593772411346,
"fcm_dpo/margin": 87.65908813476562,
"fcm_dpo/q_t": 0.42167770862579346,
"grad_norm": 74.20809936523438,
"learning_rate": 2.4908461957056825e-07,
"logits/chosen": -0.8265196084976196,
"logits/rejected": -0.8174469470977783,
"logps/chosen": -423.3354187011719,
"logps/ref_chosen": -260.53509521484375,
"logps/ref_rejected": -255.53799438476562,
"logps/rejected": -505.9973449707031,
"loss": 4.5581,
"margin_dpo/margin_mean": 87.65908813476562,
"margin_dpo/margin_std": 128.83778381347656,
"step": 264
},
{
"KL/chosen_KL_mean": -166.21475219726562,
"KL/mean": -212.19488525390625,
"KL/rejected_KL_mean": -258.17498779296875,
"KL/std": 123.5406265258789,
"epoch": 0.5549738219895288,
"fcm_dpo/beta": 0.0038373975548893213,
"fcm_dpo/delta": -0.007137385196983814,
"fcm_dpo/margin": 91.96025848388672,
"fcm_dpo/q_t": 0.4191870093345642,
"grad_norm": 64.72450256347656,
"learning_rate": 2.4725390780077905e-07,
"logits/chosen": -0.8913218379020691,
"logits/rejected": -0.8904725909233093,
"logps/chosen": -449.9277648925781,
"logps/ref_chosen": -283.7130432128906,
"logps/ref_rejected": -270.3209533691406,
"logps/rejected": -528.4959716796875,
"loss": 4.5638,
"margin_dpo/margin_mean": 91.96025848388672,
"margin_dpo/margin_std": 144.87387084960938,
"step": 265
},
{
"KL/chosen_KL_mean": -152.7468719482422,
"KL/mean": -199.51541137695312,
"KL/rejected_KL_mean": -246.28395080566406,
"KL/std": 112.15851593017578,
"epoch": 0.5570680628272251,
"fcm_dpo/beta": 0.0038449072744697332,
"fcm_dpo/delta": -0.0055340006947517395,
"fcm_dpo/margin": 93.53707885742188,
"fcm_dpo/q_t": 0.41636204719543457,
"grad_norm": 73.08821868896484,
"learning_rate": 2.454233432955807e-07,
"logits/chosen": -0.9144778847694397,
"logits/rejected": -0.8848609328269958,
"logps/chosen": -430.84613037109375,
"logps/ref_chosen": -278.09930419921875,
"logps/ref_rejected": -260.6734619140625,
"logps/rejected": -506.9573974609375,
"loss": 4.4522,
"margin_dpo/margin_mean": 93.53707885742188,
"margin_dpo/margin_std": 120.03593444824219,
"step": 266
},
{
"KL/chosen_KL_mean": -182.0021514892578,
"KL/mean": -217.25442504882812,
"KL/rejected_KL_mean": -252.5067138671875,
"KL/std": 117.42387390136719,
"epoch": 0.5591623036649215,
"fcm_dpo/beta": 0.003810013411566615,
"fcm_dpo/delta": -0.003673842176795006,
"fcm_dpo/margin": 70.50455474853516,
"fcm_dpo/q_t": 0.4374660849571228,
"grad_norm": 74.9322738647461,
"learning_rate": 2.435930242225919e-07,
"logits/chosen": -0.8645923733711243,
"logits/rejected": -0.8707435131072998,
"logps/chosen": -462.3353271484375,
"logps/ref_chosen": -280.33319091796875,
"logps/ref_rejected": -247.78099060058594,
"logps/rejected": -500.28765869140625,
"loss": 4.8086,
"margin_dpo/margin_mean": 70.50454711914062,
"margin_dpo/margin_std": 132.0673065185547,
"step": 267
},
{
"KL/chosen_KL_mean": -172.97525024414062,
"KL/mean": -217.17364501953125,
"KL/rejected_KL_mean": -261.3720703125,
"KL/std": 120.18418884277344,
"epoch": 0.5612565445026177,
"fcm_dpo/beta": 0.0038873879238963127,
"fcm_dpo/delta": 0.034165654331445694,
"fcm_dpo/margin": 88.39683532714844,
"fcm_dpo/q_t": 0.42039138078689575,
"grad_norm": 72.54732513427734,
"learning_rate": 2.4176304873626984e-07,
"logits/chosen": -0.829031229019165,
"logits/rejected": -0.807607889175415,
"logps/chosen": -477.1539611816406,
"logps/ref_chosen": -304.1787109375,
"logps/ref_rejected": -272.80316162109375,
"logps/rejected": -534.1752319335938,
"loss": 4.5311,
"margin_dpo/margin_mean": 88.39683532714844,
"margin_dpo/margin_std": 126.67704010009766,
"step": 268
},
{
"KL/chosen_KL_mean": -176.60040283203125,
"KL/mean": -205.09878540039062,
"KL/rejected_KL_mean": -233.59716796875,
"KL/std": 104.05487060546875,
"epoch": 0.5633507853403141,
"fcm_dpo/beta": 0.003917751833796501,
"fcm_dpo/delta": 0.0035603800788521767,
"fcm_dpo/margin": 56.99675750732422,
"fcm_dpo/q_t": 0.4470731317996979,
"grad_norm": 71.9229965209961,
"learning_rate": 2.399335149726463e-07,
"logits/chosen": -0.8421822786331177,
"logits/rejected": -0.835168719291687,
"logps/chosen": -426.4455261230469,
"logps/ref_chosen": -249.84512329101562,
"logps/ref_rejected": -223.37356567382812,
"logps/rejected": -456.9707336425781,
"loss": 4.9429,
"margin_dpo/margin_mean": 56.996761322021484,
"margin_dpo/margin_std": 125.362548828125,
"step": 269
},
{
"KL/chosen_KL_mean": -185.8561248779297,
"KL/mean": -224.6792755126953,
"KL/rejected_KL_mean": -263.5024108886719,
"KL/std": 120.34756469726562,
"epoch": 0.5654450261780105,
"fcm_dpo/beta": 0.003944146912544966,
"fcm_dpo/delta": 0.01949712075293064,
"fcm_dpo/margin": 77.64627838134766,
"fcm_dpo/q_t": 0.43020257353782654,
"grad_norm": 67.843505859375,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": -0.934403657913208,
"logits/rejected": -0.9376578330993652,
"logps/chosen": -504.4185485839844,
"logps/ref_chosen": -318.5623779296875,
"logps/ref_rejected": -281.1880798339844,
"logps/rejected": -544.6904296875,
"loss": 4.7418,
"margin_dpo/margin_mean": 77.64627838134766,
"margin_dpo/margin_std": 149.40438842773438,
"step": 270
},
{
"KL/chosen_KL_mean": -181.69509887695312,
"KL/mean": -218.57923889160156,
"KL/rejected_KL_mean": -255.46336364746094,
"KL/std": 120.86827850341797,
"epoch": 0.5675392670157068,
"fcm_dpo/beta": 0.003990632947534323,
"fcm_dpo/delta": 0.0031273523345589638,
"fcm_dpo/margin": 73.76829528808594,
"fcm_dpo/q_t": 0.4324439764022827,
"grad_norm": 58.05919647216797,
"learning_rate": 2.3627616503391812e-07,
"logits/chosen": -0.7795909643173218,
"logits/rejected": -0.7716021537780762,
"logps/chosen": -465.7997741699219,
"logps/ref_chosen": -284.104736328125,
"logps/ref_rejected": -253.9580535888672,
"logps/rejected": -509.42144775390625,
"loss": 4.7087,
"margin_dpo/margin_mean": 73.76829528808594,
"margin_dpo/margin_std": 125.9541015625,
"step": 271
},
{
"KL/chosen_KL_mean": -169.88497924804688,
"KL/mean": -210.8214111328125,
"KL/rejected_KL_mean": -251.75787353515625,
"KL/std": 119.86404418945312,
"epoch": 0.5696335078534032,
"fcm_dpo/beta": 0.0039629037491977215,
"fcm_dpo/delta": -0.0006313333287835121,
"fcm_dpo/margin": 81.87287902832031,
"fcm_dpo/q_t": 0.42572107911109924,
"grad_norm": 86.46068572998047,
"learning_rate": 2.344485449913914e-07,
"logits/chosen": -0.883471667766571,
"logits/rejected": -0.8656666874885559,
"logps/chosen": -467.2440185546875,
"logps/ref_chosen": -297.3590087890625,
"logps/ref_rejected": -279.20196533203125,
"logps/rejected": -530.9597778320312,
"loss": 4.678,
"margin_dpo/margin_mean": 81.87287902832031,
"margin_dpo/margin_std": 145.54736328125,
"step": 272
},
{
"KL/chosen_KL_mean": -174.3992919921875,
"KL/mean": -216.345947265625,
"KL/rejected_KL_mean": -258.2926025390625,
"KL/std": 115.58932495117188,
"epoch": 0.5717277486910994,
"fcm_dpo/beta": 0.003995942883193493,
"fcm_dpo/delta": 0.009679936803877354,
"fcm_dpo/margin": 83.89334106445312,
"fcm_dpo/q_t": 0.4239039421081543,
"grad_norm": 88.02112579345703,
"learning_rate": 2.3262175892620062e-07,
"logits/chosen": -0.8663466572761536,
"logits/rejected": -0.8779019117355347,
"logps/chosen": -467.60504150390625,
"logps/ref_chosen": -293.20574951171875,
"logps/ref_rejected": -274.7646789550781,
"logps/rejected": -533.0572509765625,
"loss": 4.6362,
"margin_dpo/margin_mean": 83.89334869384766,
"margin_dpo/margin_std": 143.1727752685547,
"step": 273
},
{
"KL/chosen_KL_mean": -172.38304138183594,
"KL/mean": -221.9813690185547,
"KL/rejected_KL_mean": -271.5797119140625,
"KL/std": 125.11678314208984,
"epoch": 0.5738219895287958,
"fcm_dpo/beta": 0.0040048351511359215,
"fcm_dpo/delta": 0.0025398635771125555,
"fcm_dpo/margin": 99.1966552734375,
"fcm_dpo/q_t": 0.4098568260669708,
"grad_norm": 81.96195220947266,
"learning_rate": 2.3079590480333827e-07,
"logits/chosen": -0.8212272524833679,
"logits/rejected": -0.78861403465271,
"logps/chosen": -442.94171142578125,
"logps/ref_chosen": -270.55865478515625,
"logps/ref_rejected": -239.47048950195312,
"logps/rejected": -511.0502014160156,
"loss": 4.4372,
"margin_dpo/margin_mean": 99.19667053222656,
"margin_dpo/margin_std": 144.70790100097656,
"step": 274
},
{
"KL/chosen_KL_mean": -157.3403778076172,
"KL/mean": -207.17703247070312,
"KL/rejected_KL_mean": -257.013671875,
"KL/std": 118.32302856445312,
"epoch": 0.5759162303664922,
"fcm_dpo/beta": 0.003956434316933155,
"fcm_dpo/delta": -0.030254080891609192,
"fcm_dpo/margin": 99.67333221435547,
"fcm_dpo/q_t": 0.40914809703826904,
"grad_norm": 58.78046417236328,
"learning_rate": 2.2897108053782e-07,
"logits/chosen": -0.875052809715271,
"logits/rejected": -0.8559980988502502,
"logps/chosen": -407.65960693359375,
"logps/ref_chosen": -250.31922912597656,
"logps/ref_rejected": -249.3187255859375,
"logps/rejected": -506.3324279785156,
"loss": 4.3663,
"margin_dpo/margin_mean": 99.67330932617188,
"margin_dpo/margin_std": 125.57425689697266,
"step": 275
},
{
"KL/chosen_KL_mean": -171.91647338867188,
"KL/mean": -217.97848510742188,
"KL/rejected_KL_mean": -264.04046630859375,
"KL/std": 126.98786926269531,
"epoch": 0.5780104712041885,
"fcm_dpo/beta": 0.0038329358212649822,
"fcm_dpo/delta": -0.035872187465429306,
"fcm_dpo/margin": 92.1240005493164,
"fcm_dpo/q_t": 0.4182504713535309,
"grad_norm": 55.775657653808594,
"learning_rate": 2.2714738398943308e-07,
"logits/chosen": -0.9235668182373047,
"logits/rejected": -0.9017929434776306,
"logps/chosen": -469.54754638671875,
"logps/ref_chosen": -297.6310729980469,
"logps/ref_rejected": -295.225830078125,
"logps/rejected": -559.2662963867188,
"loss": 4.556,
"margin_dpo/margin_mean": 92.1240005493164,
"margin_dpo/margin_std": 140.80569458007812,
"step": 276
},
{
"KL/chosen_KL_mean": -193.89830017089844,
"KL/mean": -225.9439697265625,
"KL/rejected_KL_mean": -257.9896240234375,
"KL/std": 126.7256088256836,
"epoch": 0.5801047120418849,
"fcm_dpo/beta": 0.003798598423600197,
"fcm_dpo/delta": -0.014602910727262497,
"fcm_dpo/margin": 64.09132385253906,
"fcm_dpo/q_t": 0.4443640112876892,
"grad_norm": 64.50138092041016,
"learning_rate": 2.2532491295748865e-07,
"logits/chosen": -0.8662960529327393,
"logits/rejected": -0.8660153746604919,
"logps/chosen": -460.2587890625,
"logps/ref_chosen": -266.3604736328125,
"logps/ref_rejected": -253.36767578125,
"logps/rejected": -511.3572998046875,
"loss": 4.9367,
"margin_dpo/margin_mean": 64.09132385253906,
"margin_dpo/margin_std": 143.8057861328125,
"step": 277
},
{
"KL/chosen_KL_mean": -202.2888641357422,
"KL/mean": -230.39736938476562,
"KL/rejected_KL_mean": -258.5058898925781,
"KL/std": 130.2160186767578,
"epoch": 0.5821989528795811,
"fcm_dpo/beta": 0.0037986235693097115,
"fcm_dpo/delta": 0.011226484552025795,
"fcm_dpo/margin": 56.217010498046875,
"fcm_dpo/q_t": 0.4543144702911377,
"grad_norm": 96.25747680664062,
"learning_rate": 2.2350376517557726e-07,
"logits/chosen": -0.8956958651542664,
"logits/rejected": -0.8570727109909058,
"logps/chosen": -469.6961669921875,
"logps/ref_chosen": -267.40728759765625,
"logps/ref_rejected": -229.5758514404297,
"logps/rejected": -488.08172607421875,
"loss": 5.1072,
"margin_dpo/margin_mean": 56.217010498046875,
"margin_dpo/margin_std": 157.5133514404297,
"step": 278
},
{
"KL/chosen_KL_mean": -185.69024658203125,
"KL/mean": -241.31065368652344,
"KL/rejected_KL_mean": -296.9310607910156,
"KL/std": 138.99148559570312,
"epoch": 0.5842931937172775,
"fcm_dpo/beta": 0.0037318726535886526,
"fcm_dpo/delta": -0.05649794638156891,
"fcm_dpo/margin": 111.24081420898438,
"fcm_dpo/q_t": 0.405769944190979,
"grad_norm": 66.29354858398438,
"learning_rate": 2.2168403830632769e-07,
"logits/chosen": -0.8187253475189209,
"logits/rejected": -0.8022856116294861,
"logps/chosen": -499.05804443359375,
"logps/ref_chosen": -313.3677978515625,
"logps/ref_rejected": -299.1744384765625,
"logps/rejected": -596.10546875,
"loss": 4.3506,
"margin_dpo/margin_mean": 111.24081420898438,
"margin_dpo/margin_std": 146.99954223632812,
"step": 279
},
{
"KL/chosen_KL_mean": -182.4557342529297,
"KL/mean": -221.92530822753906,
"KL/rejected_KL_mean": -261.3948669433594,
"KL/std": 114.89640045166016,
"epoch": 0.5863874345549738,
"fcm_dpo/beta": 0.0036900462582707405,
"fcm_dpo/delta": 0.03023211471736431,
"fcm_dpo/margin": 78.93912506103516,
"fcm_dpo/q_t": 0.43237319588661194,
"grad_norm": 65.89375305175781,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": -0.8743321895599365,
"logits/rejected": -0.8858817219734192,
"logps/chosen": -448.0115661621094,
"logps/ref_chosen": -265.5558166503906,
"logps/ref_rejected": -247.1573944091797,
"logps/rejected": -508.55224609375,
"loss": 4.7364,
"margin_dpo/margin_mean": 78.93912506103516,
"margin_dpo/margin_std": 143.53533935546875,
"step": 280
},
{
"KL/chosen_KL_mean": -207.59323120117188,
"KL/mean": -253.11624145507812,
"KL/rejected_KL_mean": -298.63922119140625,
"KL/std": 127.79008483886719,
"epoch": 0.5884816753926702,
"fcm_dpo/beta": 0.0036901538260281086,
"fcm_dpo/delta": -0.028504468500614166,
"fcm_dpo/margin": 91.0459976196289,
"fcm_dpo/q_t": 0.4240657687187195,
"grad_norm": 67.67650604248047,
"learning_rate": 2.1804923757009882e-07,
"logits/chosen": -0.8543596267700195,
"logits/rejected": -0.8635410070419312,
"logps/chosen": -502.892822265625,
"logps/ref_chosen": -295.2995910644531,
"logps/ref_rejected": -293.80877685546875,
"logps/rejected": -592.447998046875,
"loss": 4.6098,
"margin_dpo/margin_mean": 91.04598236083984,
"margin_dpo/margin_std": 144.34996032714844,
"step": 281
},
{
"KL/chosen_KL_mean": -202.552490234375,
"KL/mean": -242.41110229492188,
"KL/rejected_KL_mean": -282.2697448730469,
"KL/std": 125.7529067993164,
"epoch": 0.5905759162303665,
"fcm_dpo/beta": 0.003655828535556793,
"fcm_dpo/delta": 0.0036571845412254333,
"fcm_dpo/margin": 79.71723937988281,
"fcm_dpo/q_t": 0.43203699588775635,
"grad_norm": 72.81260681152344,
"learning_rate": 2.1623435862645205e-07,
"logits/chosen": -0.8544440269470215,
"logits/rejected": -0.8522729873657227,
"logps/chosen": -521.1896362304688,
"logps/ref_chosen": -318.63714599609375,
"logps/ref_rejected": -273.5943603515625,
"logps/rejected": -555.8641357421875,
"loss": 4.7281,
"margin_dpo/margin_mean": 79.71723937988281,
"margin_dpo/margin_std": 139.59954833984375,
"step": 282
},
{
"KL/chosen_KL_mean": -202.61456298828125,
"KL/mean": -244.5399932861328,
"KL/rejected_KL_mean": -286.4654541015625,
"KL/std": 124.06411743164062,
"epoch": 0.5926701570680628,
"fcm_dpo/beta": 0.0036506266333162785,
"fcm_dpo/delta": -0.0035746542271226645,
"fcm_dpo/margin": 83.85090637207031,
"fcm_dpo/q_t": 0.43061453104019165,
"grad_norm": 57.49915313720703,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": -0.876941442489624,
"logits/rejected": -0.8653864860534668,
"logps/chosen": -457.2751159667969,
"logps/ref_chosen": -254.66053771972656,
"logps/ref_rejected": -236.8627166748047,
"logps/rejected": -523.3281860351562,
"loss": 4.6959,
"margin_dpo/margin_mean": 83.85089111328125,
"margin_dpo/margin_std": 145.12371826171875,
"step": 283
},
{
"KL/chosen_KL_mean": -210.0465545654297,
"KL/mean": -254.60189819335938,
"KL/rejected_KL_mean": -299.1572265625,
"KL/std": 129.64718627929688,
"epoch": 0.5947643979057592,
"fcm_dpo/beta": 0.0035921428352594376,
"fcm_dpo/delta": -0.02668537199497223,
"fcm_dpo/margin": 89.11066436767578,
"fcm_dpo/q_t": 0.42657744884490967,
"grad_norm": 73.3239517211914,
"learning_rate": 2.1261013021512378e-07,
"logits/chosen": -0.8376402258872986,
"logits/rejected": -0.8031928539276123,
"logps/chosen": -483.4017639160156,
"logps/ref_chosen": -273.355224609375,
"logps/ref_rejected": -259.84759521484375,
"logps/rejected": -559.0048217773438,
"loss": 4.6655,
"margin_dpo/margin_mean": 89.11066436767578,
"margin_dpo/margin_std": 149.5060577392578,
"step": 284
},
{
"KL/chosen_KL_mean": -217.2931365966797,
"KL/mean": -246.5795135498047,
"KL/rejected_KL_mean": -275.86590576171875,
"KL/std": 122.36019897460938,
"epoch": 0.5968586387434555,
"fcm_dpo/beta": 0.0035818694159388542,
"fcm_dpo/delta": 0.003046293742954731,
"fcm_dpo/margin": 58.572792053222656,
"fcm_dpo/q_t": 0.4514738619327545,
"grad_norm": 84.60891723632812,
"learning_rate": 2.1080097510381294e-07,
"logits/chosen": -0.8543550372123718,
"logits/rejected": -0.847979724407196,
"logps/chosen": -527.0953369140625,
"logps/ref_chosen": -309.8022155761719,
"logps/ref_rejected": -279.11846923828125,
"logps/rejected": -554.984375,
"loss": 5.0642,
"margin_dpo/margin_mean": 58.572792053222656,
"margin_dpo/margin_std": 150.84938049316406,
"step": 285
},
{
"KL/chosen_KL_mean": -208.0546112060547,
"KL/mean": -246.29867553710938,
"KL/rejected_KL_mean": -284.5427551269531,
"KL/std": 131.73287963867188,
"epoch": 0.5989528795811518,
"fcm_dpo/beta": 0.00358386617153883,
"fcm_dpo/delta": 0.004873398691415787,
"fcm_dpo/margin": 76.48812103271484,
"fcm_dpo/q_t": 0.43887919187545776,
"grad_norm": 92.04210662841797,
"learning_rate": 2.089939221172446e-07,
"logits/chosen": -0.8385077118873596,
"logits/rejected": -0.8326424360275269,
"logps/chosen": -479.5201721191406,
"logps/ref_chosen": -271.4655456542969,
"logps/ref_rejected": -279.531494140625,
"logps/rejected": -564.07421875,
"loss": 4.83,
"margin_dpo/margin_mean": 76.48812103271484,
"margin_dpo/margin_std": 154.2538604736328,
"step": 286
},
{
"KL/chosen_KL_mean": -207.88980102539062,
"KL/mean": -244.87991333007812,
"KL/rejected_KL_mean": -281.87005615234375,
"KL/std": 124.23129272460938,
"epoch": 0.6010471204188481,
"fcm_dpo/beta": 0.003638399764895439,
"fcm_dpo/delta": 0.026291579008102417,
"fcm_dpo/margin": 73.98025512695312,
"fcm_dpo/q_t": 0.4376610815525055,
"grad_norm": 68.3472900390625,
"learning_rate": 2.0718906816218595e-07,
"logits/chosen": -0.8518833518028259,
"logits/rejected": -0.8357435464859009,
"logps/chosen": -484.9830322265625,
"logps/ref_chosen": -277.0932312011719,
"logps/ref_rejected": -233.55599975585938,
"logps/rejected": -515.426025390625,
"loss": 4.859,
"margin_dpo/margin_mean": 73.98026275634766,
"margin_dpo/margin_std": 155.37527465820312,
"step": 287
},
{
"KL/chosen_KL_mean": -199.41354370117188,
"KL/mean": -238.12754821777344,
"KL/rejected_KL_mean": -276.84161376953125,
"KL/std": 129.22042846679688,
"epoch": 0.6031413612565445,
"fcm_dpo/beta": 0.003688263474032283,
"fcm_dpo/delta": -0.005784134846180677,
"fcm_dpo/margin": 77.42807006835938,
"fcm_dpo/q_t": 0.43434223532676697,
"grad_norm": 77.68144226074219,
"learning_rate": 2.053865100274774e-07,
"logits/chosen": -0.8734298944473267,
"logits/rejected": -0.8790357708930969,
"logps/chosen": -492.58172607421875,
"logps/ref_chosen": -293.1681823730469,
"logps/ref_rejected": -263.4059143066406,
"logps/rejected": -540.2474975585938,
"loss": 4.7674,
"margin_dpo/margin_mean": 77.4280776977539,
"margin_dpo/margin_std": 144.46633911132812,
"step": 288
},
{
"KL/chosen_KL_mean": -219.11532592773438,
"KL/mean": -241.2076873779297,
"KL/rejected_KL_mean": -263.3000183105469,
"KL/std": 120.56001281738281,
"epoch": 0.6052356020942409,
"fcm_dpo/beta": 0.00366892758756876,
"fcm_dpo/delta": 0.015712561085820198,
"fcm_dpo/margin": 44.1847038269043,
"fcm_dpo/q_t": 0.46217232942581177,
"grad_norm": 70.22594451904297,
"learning_rate": 2.035863443788411e-07,
"logits/chosen": -0.8561855554580688,
"logits/rejected": -0.8383423089981079,
"logps/chosen": -549.07275390625,
"logps/ref_chosen": -329.9574279785156,
"logps/ref_rejected": -276.7565002441406,
"logps/rejected": -540.0565185546875,
"loss": 5.1915,
"margin_dpo/margin_mean": 44.1847038269043,
"margin_dpo/margin_std": 136.41799926757812,
"step": 289
},
{
"KL/chosen_KL_mean": -193.29393005371094,
"KL/mean": -233.5602569580078,
"KL/rejected_KL_mean": -273.82659912109375,
"KL/std": 127.18586730957031,
"epoch": 0.6073298429319371,
"fcm_dpo/beta": 0.0036051059141755104,
"fcm_dpo/delta": -0.03102608397603035,
"fcm_dpo/margin": 80.5326919555664,
"fcm_dpo/q_t": 0.43328824639320374,
"grad_norm": 100.93278503417969,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": -0.8621543049812317,
"logits/rejected": -0.7959156036376953,
"logps/chosen": -517.9629516601562,
"logps/ref_chosen": -324.6690673828125,
"logps/ref_rejected": -311.8439636230469,
"logps/rejected": -585.6705932617188,
"loss": 4.7497,
"margin_dpo/margin_mean": 80.5326919555664,
"margin_dpo/margin_std": 144.66436767578125,
"step": 290
},
{
"KL/chosen_KL_mean": -185.8102264404297,
"KL/mean": -235.31077575683594,
"KL/rejected_KL_mean": -284.8113098144531,
"KL/std": 127.50725555419922,
"epoch": 0.6094240837696335,
"fcm_dpo/beta": 0.003556034294888377,
"fcm_dpo/delta": -0.027959764003753662,
"fcm_dpo/margin": 99.00109100341797,
"fcm_dpo/q_t": 0.41859322786331177,
"grad_norm": 57.9321174621582,
"learning_rate": 1.9999357655598891e-07,
"logits/chosen": -0.8408191204071045,
"logits/rejected": -0.8317016959190369,
"logps/chosen": -459.95428466796875,
"logps/ref_chosen": -274.1440734863281,
"logps/ref_rejected": -278.07208251953125,
"logps/rejected": -562.8833618164062,
"loss": 4.5248,
"margin_dpo/margin_mean": 99.00109100341797,
"margin_dpo/margin_std": 140.4064178466797,
"step": 291
},
{
"KL/chosen_KL_mean": -173.9268798828125,
"KL/mean": -216.89697265625,
"KL/rejected_KL_mean": -259.8670349121094,
"KL/std": 126.9601821899414,
"epoch": 0.6115183246073298,
"fcm_dpo/beta": 0.0034741119015961885,
"fcm_dpo/delta": -0.024277152493596077,
"fcm_dpo/margin": 85.94014739990234,
"fcm_dpo/q_t": 0.4312286972999573,
"grad_norm": 64.79742431640625,
"learning_rate": 1.9820116705100775e-07,
"logits/chosen": -0.8414206504821777,
"logits/rejected": -0.8373555541038513,
"logps/chosen": -433.2906188964844,
"logps/ref_chosen": -259.3636779785156,
"logps/ref_rejected": -279.30218505859375,
"logps/rejected": -539.1692504882812,
"loss": 4.7164,
"margin_dpo/margin_mean": 85.94014739990234,
"margin_dpo/margin_std": 148.6029815673828,
"step": 292
},
{
"KL/chosen_KL_mean": -196.158203125,
"KL/mean": -242.4661102294922,
"KL/rejected_KL_mean": -288.7740783691406,
"KL/std": 135.72119140625,
"epoch": 0.6136125654450262,
"fcm_dpo/beta": 0.003440189640969038,
"fcm_dpo/delta": -0.006982447579503059,
"fcm_dpo/margin": 92.61585998535156,
"fcm_dpo/q_t": 0.42598748207092285,
"grad_norm": 62.069820404052734,
"learning_rate": 1.9641153536023642e-07,
"logits/chosen": -0.9244207143783569,
"logits/rejected": -0.8833814859390259,
"logps/chosen": -499.92901611328125,
"logps/ref_chosen": -303.77081298828125,
"logps/ref_rejected": -270.07513427734375,
"logps/rejected": -558.8491821289062,
"loss": 4.6122,
"margin_dpo/margin_mean": 92.61585998535156,
"margin_dpo/margin_std": 139.9814910888672,
"step": 293
},
{
"KL/chosen_KL_mean": -186.78228759765625,
"KL/mean": -225.4820556640625,
"KL/rejected_KL_mean": -264.18182373046875,
"KL/std": 131.13441467285156,
"epoch": 0.6157068062827226,
"fcm_dpo/beta": 0.003445146605372429,
"fcm_dpo/delta": -0.002983933314681053,
"fcm_dpo/margin": 77.39950561523438,
"fcm_dpo/q_t": 0.4392993748188019,
"grad_norm": 56.854644775390625,
"learning_rate": 1.9462477745619106e-07,
"logits/chosen": -0.8256325125694275,
"logits/rejected": -0.8303864002227783,
"logps/chosen": -427.02056884765625,
"logps/ref_chosen": -240.23831176757812,
"logps/ref_rejected": -229.187744140625,
"logps/rejected": -493.3695373535156,
"loss": 4.8415,
"margin_dpo/margin_mean": 77.39949798583984,
"margin_dpo/margin_std": 154.39572143554688,
"step": 294
},
{
"KL/chosen_KL_mean": -188.3880615234375,
"KL/mean": -226.49118041992188,
"KL/rejected_KL_mean": -264.59429931640625,
"KL/std": 112.24237060546875,
"epoch": 0.6178010471204188,
"fcm_dpo/beta": 0.003445256035774946,
"fcm_dpo/delta": 0.014424074441194534,
"fcm_dpo/margin": 76.20626068115234,
"fcm_dpo/q_t": 0.4396905303001404,
"grad_norm": 69.74424743652344,
"learning_rate": 1.928409891572757e-07,
"logits/chosen": -0.8317238688468933,
"logits/rejected": -0.8348393440246582,
"logps/chosen": -439.3977966308594,
"logps/ref_chosen": -251.00970458984375,
"logps/ref_rejected": -244.15142822265625,
"logps/rejected": -508.74578857421875,
"loss": 4.7987,
"margin_dpo/margin_mean": 76.20626068115234,
"margin_dpo/margin_std": 140.69558715820312,
"step": 295
},
{
"KL/chosen_KL_mean": -192.59164428710938,
"KL/mean": -244.67086791992188,
"KL/rejected_KL_mean": -296.7500915527344,
"KL/std": 131.72547912597656,
"epoch": 0.6198952879581152,
"fcm_dpo/beta": 0.0034456930588930845,
"fcm_dpo/delta": -0.02413531392812729,
"fcm_dpo/margin": 104.15848541259766,
"fcm_dpo/q_t": 0.417242169380188,
"grad_norm": 68.13108825683594,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": -0.8389942646026611,
"logits/rejected": -0.8055410981178284,
"logps/chosen": -486.47222900390625,
"logps/ref_chosen": -293.880615234375,
"logps/ref_rejected": -283.4175720214844,
"logps/rejected": -580.167724609375,
"loss": 4.539,
"margin_dpo/margin_mean": 104.15848541259766,
"margin_dpo/margin_std": 157.52789306640625,
"step": 296
},
{
"KL/chosen_KL_mean": -189.89254760742188,
"KL/mean": -227.05242919921875,
"KL/rejected_KL_mean": -264.21234130859375,
"KL/std": 118.02452087402344,
"epoch": 0.6219895287958115,
"fcm_dpo/beta": 0.003379724221304059,
"fcm_dpo/delta": -0.012697052210569382,
"fcm_dpo/margin": 74.31977844238281,
"fcm_dpo/q_t": 0.44128429889678955,
"grad_norm": 60.725318908691406,
"learning_rate": 1.8928270384706582e-07,
"logits/chosen": -0.9069033265113831,
"logits/rejected": -0.9000098705291748,
"logps/chosen": -479.35260009765625,
"logps/ref_chosen": -289.4600830078125,
"logps/ref_rejected": -283.69110107421875,
"logps/rejected": -547.9033813476562,
"loss": 4.8268,
"margin_dpo/margin_mean": 74.31977844238281,
"margin_dpo/margin_std": 135.9732666015625,
"step": 297
},
{
"KL/chosen_KL_mean": -180.1241455078125,
"KL/mean": -226.821044921875,
"KL/rejected_KL_mean": -273.5179443359375,
"KL/std": 125.18611907958984,
"epoch": 0.6240837696335079,
"fcm_dpo/beta": 0.0033408179879188538,
"fcm_dpo/delta": -0.020929735153913498,
"fcm_dpo/margin": 93.3937759399414,
"fcm_dpo/q_t": 0.4280543327331543,
"grad_norm": 89.00557708740234,
"learning_rate": 1.875083976558136e-07,
"logits/chosen": -0.8199465870857239,
"logits/rejected": -0.8056501150131226,
"logps/chosen": -486.6391296386719,
"logps/ref_chosen": -306.5150146484375,
"logps/ref_rejected": -280.6969909667969,
"logps/rejected": -554.21484375,
"loss": 4.6812,
"margin_dpo/margin_mean": 93.3937759399414,
"margin_dpo/margin_std": 156.3482666015625,
"step": 298
},
{
"KL/chosen_KL_mean": -210.533447265625,
"KL/mean": -247.99322509765625,
"KL/rejected_KL_mean": -285.4530334472656,
"KL/std": 129.0129852294922,
"epoch": 0.6261780104712041,
"fcm_dpo/beta": 0.0033021531999111176,
"fcm_dpo/delta": -0.0064115277491509914,
"fcm_dpo/margin": 74.91960906982422,
"fcm_dpo/q_t": 0.442142128944397,
"grad_norm": 70.9146957397461,
"learning_rate": 1.8573744269954297e-07,
"logits/chosen": -0.8116376399993896,
"logits/rejected": -0.7935671210289001,
"logps/chosen": -491.8971862792969,
"logps/ref_chosen": -281.36376953125,
"logps/ref_rejected": -270.39508056640625,
"logps/rejected": -555.84814453125,
"loss": 4.8285,
"margin_dpo/margin_mean": 74.91960906982422,
"margin_dpo/margin_std": 130.8252410888672,
"step": 299
},
{
"KL/chosen_KL_mean": -212.4322509765625,
"KL/mean": -248.7073211669922,
"KL/rejected_KL_mean": -284.982421875,
"KL/std": 135.9843292236328,
"epoch": 0.6282722513089005,
"fcm_dpo/beta": 0.00330313085578382,
"fcm_dpo/delta": 0.010395125485956669,
"fcm_dpo/margin": 72.5501480102539,
"fcm_dpo/q_t": 0.4443802237510681,
"grad_norm": 66.76209259033203,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": -0.836089551448822,
"logits/rejected": -0.8139994144439697,
"logps/chosen": -527.26806640625,
"logps/ref_chosen": -314.83575439453125,
"logps/ref_rejected": -269.1154479980469,
"logps/rejected": -554.0978393554688,
"loss": 4.9016,
"margin_dpo/margin_mean": 72.5501480102539,
"margin_dpo/margin_std": 154.16453552246094,
"step": 300
},
{
"KL/chosen_KL_mean": -201.32894897460938,
"KL/mean": -238.62901306152344,
"KL/rejected_KL_mean": -275.9290771484375,
"KL/std": 128.91531372070312,
"epoch": 0.6303664921465969,
"fcm_dpo/beta": 0.003320886753499508,
"fcm_dpo/delta": -0.009585805237293243,
"fcm_dpo/margin": 74.60012817382812,
"fcm_dpo/q_t": 0.4424639344215393,
"grad_norm": 81.5901870727539,
"learning_rate": 1.8220596619089573e-07,
"logits/chosen": -0.8685322999954224,
"logits/rejected": -0.8592681884765625,
"logps/chosen": -481.2235107421875,
"logps/ref_chosen": -279.89453125,
"logps/ref_rejected": -271.6694641113281,
"logps/rejected": -547.5985107421875,
"loss": 4.8592,
"margin_dpo/margin_mean": 74.60012817382812,
"margin_dpo/margin_std": 144.12062072753906,
"step": 301
},
{
"KL/chosen_KL_mean": -197.4744415283203,
"KL/mean": -246.68585205078125,
"KL/rejected_KL_mean": -295.89727783203125,
"KL/std": 135.5497589111328,
"epoch": 0.6324607329842932,
"fcm_dpo/beta": 0.0033070500940084457,
"fcm_dpo/delta": 0.005456558428704739,
"fcm_dpo/margin": 98.42284393310547,
"fcm_dpo/q_t": 0.42443469166755676,
"grad_norm": 70.69303131103516,
"learning_rate": 1.8044563402088682e-07,
"logits/chosen": -0.8222125172615051,
"logits/rejected": -0.8099421262741089,
"logps/chosen": -468.8062744140625,
"logps/ref_chosen": -271.3318176269531,
"logps/ref_rejected": -256.5587158203125,
"logps/rejected": -552.4559936523438,
"loss": 4.5808,
"margin_dpo/margin_mean": 98.42283630371094,
"margin_dpo/margin_std": 144.48379516601562,
"step": 302
},
{
"KL/chosen_KL_mean": -200.82537841796875,
"KL/mean": -243.748291015625,
"KL/rejected_KL_mean": -286.67120361328125,
"KL/std": 122.43389892578125,
"epoch": 0.6345549738219896,
"fcm_dpo/beta": 0.0033165114000439644,
"fcm_dpo/delta": 0.004780970048159361,
"fcm_dpo/margin": 85.84585571289062,
"fcm_dpo/q_t": 0.4328896403312683,
"grad_norm": 66.35929870605469,
"learning_rate": 1.7868903184043885e-07,
"logits/chosen": -0.8214887380599976,
"logits/rejected": -0.7972728610038757,
"logps/chosen": -505.7063903808594,
"logps/ref_chosen": -304.88104248046875,
"logps/ref_rejected": -269.063720703125,
"logps/rejected": -555.7349853515625,
"loss": 4.6998,
"margin_dpo/margin_mean": 85.84585571289062,
"margin_dpo/margin_std": 135.42276000976562,
"step": 303
},
{
"KL/chosen_KL_mean": -215.84197998046875,
"KL/mean": -262.8927001953125,
"KL/rejected_KL_mean": -309.9433898925781,
"KL/std": 149.1090850830078,
"epoch": 0.6366492146596858,
"fcm_dpo/beta": 0.003333859611302614,
"fcm_dpo/delta": 0.005845099687576294,
"fcm_dpo/margin": 94.10143280029297,
"fcm_dpo/q_t": 0.42921072244644165,
"grad_norm": 54.69770812988281,
"learning_rate": 1.7693625385079574e-07,
"logits/chosen": -0.8201622366905212,
"logits/rejected": -0.8260871171951294,
"logps/chosen": -506.55291748046875,
"logps/ref_chosen": -290.7109680175781,
"logps/ref_rejected": -237.6885986328125,
"logps/rejected": -547.6320190429688,
"loss": 4.677,
"margin_dpo/margin_mean": 94.10143280029297,
"margin_dpo/margin_std": 163.25584411621094,
"step": 304
},
{
"KL/chosen_KL_mean": -180.0332489013672,
"KL/mean": -250.61903381347656,
"KL/rejected_KL_mean": -321.204833984375,
"KL/std": 140.91082763671875,
"epoch": 0.6387434554973822,
"fcm_dpo/beta": 0.0031983419321477413,
"fcm_dpo/delta": -0.1049448549747467,
"fcm_dpo/margin": 141.17156982421875,
"fcm_dpo/q_t": 0.3961653709411621,
"grad_norm": 65.096923828125,
"learning_rate": 1.7518739404812155e-07,
"logits/chosen": -0.8557301759719849,
"logits/rejected": -0.8239875435829163,
"logps/chosen": -436.51727294921875,
"logps/ref_chosen": -256.4839782714844,
"logps/ref_rejected": -266.4063415527344,
"logps/rejected": -587.6111450195312,
"loss": 4.2208,
"margin_dpo/margin_mean": 141.17156982421875,
"margin_dpo/margin_std": 163.42698669433594,
"step": 305
},
{
"KL/chosen_KL_mean": -197.95596313476562,
"KL/mean": -238.4625244140625,
"KL/rejected_KL_mean": -278.9691162109375,
"KL/std": 122.23258209228516,
"epoch": 0.6408376963350786,
"fcm_dpo/beta": 0.0031039295718073845,
"fcm_dpo/delta": 0.02336675301194191,
"fcm_dpo/margin": 81.01317596435547,
"fcm_dpo/q_t": 0.4409124553203583,
"grad_norm": 70.48225402832031,
"learning_rate": 1.7344254621846017e-07,
"logits/chosen": -0.8595868945121765,
"logits/rejected": -0.8387668132781982,
"logps/chosen": -518.6051635742188,
"logps/ref_chosen": -320.6492004394531,
"logps/ref_rejected": -273.36773681640625,
"logps/rejected": -552.3368530273438,
"loss": 4.7914,
"margin_dpo/margin_mean": 81.01317596435547,
"margin_dpo/margin_std": 134.45468139648438,
"step": 306
},
{
"KL/chosen_KL_mean": -226.18310546875,
"KL/mean": -269.3814392089844,
"KL/rejected_KL_mean": -312.57977294921875,
"KL/std": 138.75106811523438,
"epoch": 0.6429319371727749,
"fcm_dpo/beta": 0.00315967109054327,
"fcm_dpo/delta": 0.011747002601623535,
"fcm_dpo/margin": 86.39667510986328,
"fcm_dpo/q_t": 0.4350773096084595,
"grad_norm": 107.51629638671875,
"learning_rate": 1.717018039327053e-07,
"logits/chosen": -0.797162652015686,
"logits/rejected": -0.8184522986412048,
"logps/chosen": -505.6372985839844,
"logps/ref_chosen": -279.4541931152344,
"logps/ref_rejected": -240.3796844482422,
"logps/rejected": -552.95947265625,
"loss": 4.7194,
"margin_dpo/margin_mean": 86.39667510986328,
"margin_dpo/margin_std": 132.84042358398438,
"step": 307
},
{
"KL/chosen_KL_mean": -245.1857147216797,
"KL/mean": -275.275146484375,
"KL/rejected_KL_mean": -305.36456298828125,
"KL/std": 142.4264373779297,
"epoch": 0.6450261780104712,
"fcm_dpo/beta": 0.003162857610732317,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 60.1788330078125,
"fcm_dpo/q_t": 0.4557945728302002,
"grad_norm": 66.74758911132812,
"learning_rate": 1.699652605415828e-07,
"logits/chosen": -0.8476360440254211,
"logits/rejected": -0.8570371866226196,
"logps/chosen": -541.7840576171875,
"logps/ref_chosen": -296.598388671875,
"logps/ref_rejected": -258.6953430175781,
"logps/rejected": -564.0598754882812,
"loss": 5.0646,
"margin_dpo/margin_mean": 60.1788330078125,
"margin_dpo/margin_std": 152.62615966796875,
"step": 308
},
{
"KL/chosen_KL_mean": -229.7421112060547,
"KL/mean": -281.80914306640625,
"KL/rejected_KL_mean": -333.8762512207031,
"KL/std": 138.70144653320312,
"epoch": 0.6471204188481675,
"fcm_dpo/beta": 0.0031696951482445,
"fcm_dpo/delta": 0.008564796298742294,
"fcm_dpo/margin": 104.13411712646484,
"fcm_dpo/q_t": 0.4243236780166626,
"grad_norm": 69.46084594726562,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": -0.8503900766372681,
"logits/rejected": -0.8593624234199524,
"logps/chosen": -511.1302185058594,
"logps/ref_chosen": -281.3881530761719,
"logps/ref_rejected": -262.458740234375,
"logps/rejected": -596.3349609375,
"loss": 4.583,
"margin_dpo/margin_mean": 104.13410949707031,
"margin_dpo/margin_std": 161.4528045654297,
"step": 309
},
{
"KL/chosen_KL_mean": -222.09640502929688,
"KL/mean": -270.9717712402344,
"KL/rejected_KL_mean": -319.84710693359375,
"KL/std": 141.6556396484375,
"epoch": 0.6492146596858639,
"fcm_dpo/beta": 0.003191425697878003,
"fcm_dpo/delta": 0.008555184118449688,
"fcm_dpo/margin": 97.75069427490234,
"fcm_dpo/q_t": 0.4272443950176239,
"grad_norm": 97.37775421142578,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": -0.8479313850402832,
"logits/rejected": -0.8224340081214905,
"logps/chosen": -501.2836608886719,
"logps/ref_chosen": -279.1872863769531,
"logps/ref_rejected": -261.8279724121094,
"logps/rejected": -581.675048828125,
"loss": 4.6234,
"margin_dpo/margin_mean": 97.75069427490234,
"margin_dpo/margin_std": 146.51434326171875,
"step": 310
},
{
"KL/chosen_KL_mean": -227.7700958251953,
"KL/mean": -280.1990661621094,
"KL/rejected_KL_mean": -332.6280212402344,
"KL/std": 150.89422607421875,
"epoch": 0.6513089005235602,
"fcm_dpo/beta": 0.0031548929400742054,
"fcm_dpo/delta": -0.02223263308405876,
"fcm_dpo/margin": 104.85791015625,
"fcm_dpo/q_t": 0.42464131116867065,
"grad_norm": 55.97704315185547,
"learning_rate": 1.647817538357072e-07,
"logits/chosen": -0.8462434411048889,
"logits/rejected": -0.8253843784332275,
"logps/chosen": -499.1682434082031,
"logps/ref_chosen": -271.39813232421875,
"logps/ref_rejected": -266.12701416015625,
"logps/rejected": -598.7550659179688,
"loss": 4.6293,
"margin_dpo/margin_mean": 104.85791015625,
"margin_dpo/margin_std": 171.730224609375,
"step": 311
},
{
"KL/chosen_KL_mean": -225.2976837158203,
"KL/mean": -271.5206604003906,
"KL/rejected_KL_mean": -317.74365234375,
"KL/std": 148.50408935546875,
"epoch": 0.6534031413612565,
"fcm_dpo/beta": 0.0031063989736139774,
"fcm_dpo/delta": -0.02017890475690365,
"fcm_dpo/margin": 92.44593811035156,
"fcm_dpo/q_t": 0.4358768165111542,
"grad_norm": 69.38884735107422,
"learning_rate": 1.6306293495205755e-07,
"logits/chosen": -0.8498347997665405,
"logits/rejected": -0.8207939863204956,
"logps/chosen": -507.6827087402344,
"logps/ref_chosen": -282.3850402832031,
"logps/ref_rejected": -246.35389709472656,
"logps/rejected": -564.0975341796875,
"loss": 4.8437,
"margin_dpo/margin_mean": 92.44593048095703,
"margin_dpo/margin_std": 191.8460693359375,
"step": 312
},
{
"KL/chosen_KL_mean": -228.06698608398438,
"KL/mean": -272.4660949707031,
"KL/rejected_KL_mean": -316.86517333984375,
"KL/std": 152.6967315673828,
"epoch": 0.6554973821989529,
"fcm_dpo/beta": 0.0031254414934664965,
"fcm_dpo/delta": -0.0015552723780274391,
"fcm_dpo/margin": 88.7981948852539,
"fcm_dpo/q_t": 0.43964749574661255,
"grad_norm": 53.70850372314453,
"learning_rate": 1.6134877823936607e-07,
"logits/chosen": -0.894835352897644,
"logits/rejected": -0.8788830637931824,
"logps/chosen": -531.6978759765625,
"logps/ref_chosen": -303.630859375,
"logps/ref_rejected": -273.1156921386719,
"logps/rejected": -589.9808349609375,
"loss": 4.844,
"margin_dpo/margin_mean": 88.7981948852539,
"margin_dpo/margin_std": 178.9306640625,
"step": 313
},
{
"KL/chosen_KL_mean": -218.431640625,
"KL/mean": -264.6053466796875,
"KL/rejected_KL_mean": -310.779052734375,
"KL/std": 135.10806274414062,
"epoch": 0.6575916230366492,
"fcm_dpo/beta": 0.0031253828201442957,
"fcm_dpo/delta": 0.024184562265872955,
"fcm_dpo/margin": 92.347412109375,
"fcm_dpo/q_t": 0.43235987424850464,
"grad_norm": 63.3831901550293,
"learning_rate": 1.5963937562265522e-07,
"logits/chosen": -0.8976191282272339,
"logits/rejected": -0.8891264200210571,
"logps/chosen": -520.7359619140625,
"logps/ref_chosen": -302.3042907714844,
"logps/ref_rejected": -273.6416015625,
"logps/rejected": -584.4207153320312,
"loss": 4.7032,
"margin_dpo/margin_mean": 92.34739685058594,
"margin_dpo/margin_std": 150.6728057861328,
"step": 314
},
{
"KL/chosen_KL_mean": -211.95986938476562,
"KL/mean": -268.8974609375,
"KL/rejected_KL_mean": -325.8350524902344,
"KL/std": 141.79452514648438,
"epoch": 0.6596858638743456,
"fcm_dpo/beta": 0.0031580040231347084,
"fcm_dpo/delta": -0.00586454663425684,
"fcm_dpo/margin": 113.87518310546875,
"fcm_dpo/q_t": 0.4168047308921814,
"grad_norm": 71.96776580810547,
"learning_rate": 1.5793481877199943e-07,
"logits/chosen": -0.8908346891403198,
"logits/rejected": -0.8735600113868713,
"logps/chosen": -514.6891479492188,
"logps/ref_chosen": -302.729248046875,
"logps/ref_rejected": -270.26910400390625,
"logps/rejected": -596.1041259765625,
"loss": 4.4891,
"margin_dpo/margin_mean": 113.87518310546875,
"margin_dpo/margin_std": 158.78358459472656,
"step": 315
},
{
"KL/chosen_KL_mean": -222.13168334960938,
"KL/mean": -270.41119384765625,
"KL/rejected_KL_mean": -318.69073486328125,
"KL/std": 140.0904541015625,
"epoch": 0.6617801047120419,
"fcm_dpo/beta": 0.003170891897752881,
"fcm_dpo/delta": 0.0005847662687301636,
"fcm_dpo/margin": 96.55903625488281,
"fcm_dpo/q_t": 0.4291931390762329,
"grad_norm": 62.24413299560547,
"learning_rate": 1.562351990976095e-07,
"logits/chosen": -0.9054620265960693,
"logits/rejected": -0.8874289393424988,
"logps/chosen": -532.702392578125,
"logps/ref_chosen": -310.5706481933594,
"logps/ref_rejected": -272.9354553222656,
"logps/rejected": -591.626220703125,
"loss": 4.6598,
"margin_dpo/margin_mean": 96.55903625488281,
"margin_dpo/margin_std": 156.25230407714844,
"step": 316
},
{
"KL/chosen_KL_mean": -193.6438446044922,
"KL/mean": -238.2861328125,
"KL/rejected_KL_mean": -282.9284362792969,
"KL/std": 125.384033203125,
"epoch": 0.6638743455497382,
"fcm_dpo/beta": 0.0031651100143790245,
"fcm_dpo/delta": 0.015250356867909431,
"fcm_dpo/margin": 89.2845458984375,
"fcm_dpo/q_t": 0.43306419253349304,
"grad_norm": 67.99295043945312,
"learning_rate": 1.5454060774493065e-07,
"logits/chosen": -0.8674439787864685,
"logits/rejected": -0.8366461992263794,
"logps/chosen": -447.5442199707031,
"logps/ref_chosen": -253.90036010742188,
"logps/ref_rejected": -218.74078369140625,
"logps/rejected": -501.669189453125,
"loss": 4.68,
"margin_dpo/margin_mean": 89.2845458984375,
"margin_dpo/margin_std": 131.8083038330078,
"step": 317
},
{
"KL/chosen_KL_mean": -200.83892822265625,
"KL/mean": -259.72735595703125,
"KL/rejected_KL_mean": -318.61578369140625,
"KL/std": 138.72280883789062,
"epoch": 0.6659685863874345,
"fcm_dpo/beta": 0.0031195408664643764,
"fcm_dpo/delta": -0.029568474739789963,
"fcm_dpo/margin": 117.77685546875,
"fcm_dpo/q_t": 0.41461455821990967,
"grad_norm": 64.10457611083984,
"learning_rate": 1.5285113558975427e-07,
"logits/chosen": -0.9008802771568298,
"logits/rejected": -0.8715929388999939,
"logps/chosen": -471.66180419921875,
"logps/ref_chosen": -270.8228759765625,
"logps/ref_rejected": -255.30972290039062,
"logps/rejected": -573.925537109375,
"loss": 4.4471,
"margin_dpo/margin_mean": 117.77685546875,
"margin_dpo/margin_std": 154.478515625,
"step": 318
},
{
"KL/chosen_KL_mean": -211.363037109375,
"KL/mean": -264.632568359375,
"KL/rejected_KL_mean": -317.902099609375,
"KL/std": 134.839599609375,
"epoch": 0.6680628272251309,
"fcm_dpo/beta": 0.0031336878892034292,
"fcm_dpo/delta": 0.01707865111529827,
"fcm_dpo/margin": 106.53904724121094,
"fcm_dpo/q_t": 0.42066890001296997,
"grad_norm": 84.68042755126953,
"learning_rate": 1.5116687323334464e-07,
"logits/chosen": -0.8805448412895203,
"logits/rejected": -0.8484068512916565,
"logps/chosen": -512.3659057617188,
"logps/ref_chosen": -301.0028076171875,
"logps/ref_rejected": -242.39002990722656,
"logps/rejected": -560.2921142578125,
"loss": 4.506,
"margin_dpo/margin_mean": 106.53904724121094,
"margin_dpo/margin_std": 134.68338012695312,
"step": 319
},
{
"KL/chosen_KL_mean": -210.13380432128906,
"KL/mean": -259.4587707519531,
"KL/rejected_KL_mean": -308.78369140625,
"KL/std": 143.7012176513672,
"epoch": 0.6701570680628273,
"fcm_dpo/beta": 0.0031071542762219906,
"fcm_dpo/delta": -0.02263573370873928,
"fcm_dpo/margin": 98.64984893798828,
"fcm_dpo/q_t": 0.4299119710922241,
"grad_norm": 75.7774658203125,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": -0.8548473715782166,
"logits/rejected": -0.8449010848999023,
"logps/chosen": -513.7564086914062,
"logps/ref_chosen": -303.6225891113281,
"logps/ref_rejected": -280.85174560546875,
"logps/rejected": -589.6353759765625,
"loss": 4.6904,
"margin_dpo/margin_mean": 98.64985656738281,
"margin_dpo/margin_std": 171.7944793701172,
"step": 320
},
{
"KL/chosen_KL_mean": -231.89138793945312,
"KL/mean": -268.456787109375,
"KL/rejected_KL_mean": -305.02215576171875,
"KL/std": 141.75418090820312,
"epoch": 0.6722513089005235,
"fcm_dpo/beta": 0.0031160078942775726,
"fcm_dpo/delta": 0.01728174462914467,
"fcm_dpo/margin": 73.13074493408203,
"fcm_dpo/q_t": 0.4479866623878479,
"grad_norm": 60.38873291015625,
"learning_rate": 1.478143389201113e-07,
"logits/chosen": -0.865630567073822,
"logits/rejected": -0.8288436532020569,
"logps/chosen": -520.877197265625,
"logps/ref_chosen": -288.98583984375,
"logps/ref_rejected": -241.1822052001953,
"logps/rejected": -546.204345703125,
"loss": 4.9345,
"margin_dpo/margin_mean": 73.13074493408203,
"margin_dpo/margin_std": 156.70712280273438,
"step": 321
},
{
"KL/chosen_KL_mean": -239.51849365234375,
"KL/mean": -283.0441589355469,
"KL/rejected_KL_mean": -326.56982421875,
"KL/std": 145.20364379882812,
"epoch": 0.6743455497382199,
"fcm_dpo/beta": 0.0031907029915601015,
"fcm_dpo/delta": 0.04024318605661392,
"fcm_dpo/margin": 87.05138397216797,
"fcm_dpo/q_t": 0.4357774257659912,
"grad_norm": 65.08485412597656,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": -0.930508017539978,
"logits/rejected": -0.8933470845222473,
"logps/chosen": -548.0618896484375,
"logps/ref_chosen": -308.54345703125,
"logps/ref_rejected": -269.7995910644531,
"logps/rejected": -596.369384765625,
"loss": 4.8048,
"margin_dpo/margin_mean": 87.05138397216797,
"margin_dpo/margin_std": 171.3778076171875,
"step": 322
},
{
"KL/chosen_KL_mean": -233.23052978515625,
"KL/mean": -264.7674560546875,
"KL/rejected_KL_mean": -296.30438232421875,
"KL/std": 134.792236328125,
"epoch": 0.6764397905759162,
"fcm_dpo/beta": 0.003232902381569147,
"fcm_dpo/delta": -0.0018465555040165782,
"fcm_dpo/margin": 63.07388687133789,
"fcm_dpo/q_t": 0.4540364444255829,
"grad_norm": 92.90402221679688,
"learning_rate": 1.4448372394055246e-07,
"logits/chosen": -0.8878564834594727,
"logits/rejected": -0.8803995847702026,
"logps/chosen": -515.72412109375,
"logps/ref_chosen": -282.49365234375,
"logps/ref_rejected": -227.7105255126953,
"logps/rejected": -524.014892578125,
"loss": 5.0631,
"margin_dpo/margin_mean": 63.073883056640625,
"margin_dpo/margin_std": 164.00965881347656,
"step": 323
},
{
"KL/chosen_KL_mean": -203.592041015625,
"KL/mean": -258.118896484375,
"KL/rejected_KL_mean": -312.64569091796875,
"KL/std": 143.88446044921875,
"epoch": 0.6785340314136126,
"fcm_dpo/beta": 0.0031611351296305656,
"fcm_dpo/delta": -0.020262351259589195,
"fcm_dpo/margin": 109.05362701416016,
"fcm_dpo/q_t": 0.41922521591186523,
"grad_norm": 52.81629943847656,
"learning_rate": 1.428268596492364e-07,
"logits/chosen": -0.8115453124046326,
"logits/rejected": -0.8045988082885742,
"logps/chosen": -442.930419921875,
"logps/ref_chosen": -239.33836364746094,
"logps/ref_rejected": -230.53775024414062,
"logps/rejected": -543.1834716796875,
"loss": 4.5051,
"margin_dpo/margin_mean": 109.05361938476562,
"margin_dpo/margin_std": 144.5257568359375,
"step": 324
},
{
"KL/chosen_KL_mean": -234.3408203125,
"KL/mean": -280.7336120605469,
"KL/rejected_KL_mean": -327.1264343261719,
"KL/std": 150.88209533691406,
"epoch": 0.680628272251309,
"fcm_dpo/beta": 0.003149219322949648,
"fcm_dpo/delta": -0.012197908945381641,
"fcm_dpo/margin": 92.78559875488281,
"fcm_dpo/q_t": 0.4342680275440216,
"grad_norm": 94.48786163330078,
"learning_rate": 1.4117574272818386e-07,
"logits/chosen": -0.8268110156059265,
"logits/rejected": -0.8105596303939819,
"logps/chosen": -514.9697875976562,
"logps/ref_chosen": -280.62896728515625,
"logps/ref_rejected": -270.5085754394531,
"logps/rejected": -597.6349487304688,
"loss": 4.8091,
"margin_dpo/margin_mean": 92.78559875488281,
"margin_dpo/margin_std": 185.6398468017578,
"step": 325
},
{
"KL/chosen_KL_mean": -214.26693725585938,
"KL/mean": -265.3741455078125,
"KL/rejected_KL_mean": -316.4813232421875,
"KL/std": 142.83740234375,
"epoch": 0.6827225130890052,
"fcm_dpo/beta": 0.0032117185182869434,
"fcm_dpo/delta": 0.028104102239012718,
"fcm_dpo/margin": 102.21440124511719,
"fcm_dpo/q_t": 0.4247254431247711,
"grad_norm": 64.72186279296875,
"learning_rate": 1.3953046172178413e-07,
"logits/chosen": -0.945293664932251,
"logits/rejected": -0.9358657598495483,
"logps/chosen": -455.2540588378906,
"logps/ref_chosen": -240.9871368408203,
"logps/ref_rejected": -261.0238342285156,
"logps/rejected": -577.5051879882812,
"loss": 4.6272,
"margin_dpo/margin_mean": 102.21440124511719,
"margin_dpo/margin_std": 167.8172149658203,
"step": 326
},
{
"KL/chosen_KL_mean": -207.91552734375,
"KL/mean": -264.1645812988281,
"KL/rejected_KL_mean": -320.4136657714844,
"KL/std": 134.82386779785156,
"epoch": 0.6848167539267016,
"fcm_dpo/beta": 0.0031878852751106024,
"fcm_dpo/delta": -0.03201831877231598,
"fcm_dpo/margin": 112.49812316894531,
"fcm_dpo/q_t": 0.4176490604877472,
"grad_norm": 94.11993408203125,
"learning_rate": 1.3789110486146468e-07,
"logits/chosen": -0.8883708119392395,
"logits/rejected": -0.8645241856575012,
"logps/chosen": -487.435546875,
"logps/ref_chosen": -279.52001953125,
"logps/ref_rejected": -269.51824951171875,
"logps/rejected": -589.931884765625,
"loss": 4.5059,
"margin_dpo/margin_mean": 112.49813079833984,
"margin_dpo/margin_std": 157.23403930664062,
"step": 327
},
{
"KL/chosen_KL_mean": -229.1092529296875,
"KL/mean": -280.052490234375,
"KL/rejected_KL_mean": -330.9957275390625,
"KL/std": 139.11770629882812,
"epoch": 0.6869109947643979,
"fcm_dpo/beta": 0.003118137363344431,
"fcm_dpo/delta": -0.0110612902790308,
"fcm_dpo/margin": 101.88642883300781,
"fcm_dpo/q_t": 0.42611968517303467,
"grad_norm": 63.17850112915039,
"learning_rate": 1.362577600609588e-07,
"logits/chosen": -0.8533894419670105,
"logits/rejected": -0.8511139154434204,
"logps/chosen": -530.1427612304688,
"logps/ref_chosen": -301.033447265625,
"logps/ref_rejected": -284.2101135253906,
"logps/rejected": -615.205810546875,
"loss": 4.6045,
"margin_dpo/margin_mean": 101.88642883300781,
"margin_dpo/margin_std": 151.49839782714844,
"step": 328
},
{
"KL/chosen_KL_mean": -243.52928161621094,
"KL/mean": -300.4892883300781,
"KL/rejected_KL_mean": -357.4493103027344,
"KL/std": 178.26580810546875,
"epoch": 0.6890052356020943,
"fcm_dpo/beta": 0.0030765547417104244,
"fcm_dpo/delta": -0.023377757519483566,
"fcm_dpo/margin": 113.92003631591797,
"fcm_dpo/q_t": 0.4244258999824524,
"grad_norm": 67.04179382324219,
"learning_rate": 1.3463051491159093e-07,
"logits/chosen": -0.8606047630310059,
"logits/rejected": -0.8284364938735962,
"logps/chosen": -563.5181274414062,
"logps/ref_chosen": -319.9888610839844,
"logps/ref_rejected": -307.5588684082031,
"logps/rejected": -665.0081787109375,
"loss": 4.6515,
"margin_dpo/margin_mean": 113.92002868652344,
"margin_dpo/margin_std": 201.48175048828125,
"step": 329
},
{
"KL/chosen_KL_mean": -238.3928985595703,
"KL/mean": -291.460205078125,
"KL/rejected_KL_mean": -344.5274963378906,
"KL/std": 156.5318603515625,
"epoch": 0.6910994764397905,
"fcm_dpo/beta": 0.0030223994981497526,
"fcm_dpo/delta": -0.023758048191666603,
"fcm_dpo/margin": 106.13461303710938,
"fcm_dpo/q_t": 0.42642486095428467,
"grad_norm": 73.99193572998047,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": -0.857183039188385,
"logits/rejected": -0.8630150556564331,
"logps/chosen": -539.5076904296875,
"logps/ref_chosen": -301.11474609375,
"logps/ref_rejected": -299.673095703125,
"logps/rejected": -644.2005615234375,
"loss": 4.6121,
"margin_dpo/margin_mean": 106.13459777832031,
"margin_dpo/margin_std": 160.67657470703125,
"step": 330
},
{
"KL/chosen_KL_mean": -226.96383666992188,
"KL/mean": -274.009765625,
"KL/rejected_KL_mean": -321.0557556152344,
"KL/std": 153.6221466064453,
"epoch": 0.6931937172774869,
"fcm_dpo/beta": 0.0030192735139280558,
"fcm_dpo/delta": 0.01395251415669918,
"fcm_dpo/margin": 94.09191131591797,
"fcm_dpo/q_t": 0.4358104467391968,
"grad_norm": 80.74335479736328,
"learning_rate": 1.3139467229135998e-07,
"logits/chosen": -0.8754410743713379,
"logits/rejected": -0.8539817333221436,
"logps/chosen": -504.5553283691406,
"logps/ref_chosen": -277.59149169921875,
"logps/ref_rejected": -256.025634765625,
"logps/rejected": -577.0813598632812,
"loss": 4.7924,
"margin_dpo/margin_mean": 94.09191131591797,
"margin_dpo/margin_std": 185.90357971191406,
"step": 331
},
{
"KL/chosen_KL_mean": -240.19212341308594,
"KL/mean": -288.86236572265625,
"KL/rejected_KL_mean": -337.5325927734375,
"KL/std": 150.13291931152344,
"epoch": 0.6952879581151833,
"fcm_dpo/beta": 0.003028081264346838,
"fcm_dpo/delta": -0.003697561100125313,
"fcm_dpo/margin": 97.34046936035156,
"fcm_dpo/q_t": 0.4329964518547058,
"grad_norm": 66.77661895751953,
"learning_rate": 1.2978624834891626e-07,
"logits/chosen": -0.8756115436553955,
"logits/rejected": -0.8486796617507935,
"logps/chosen": -510.16583251953125,
"logps/ref_chosen": -269.97369384765625,
"logps/ref_rejected": -235.03164672851562,
"logps/rejected": -572.564208984375,
"loss": 4.7937,
"margin_dpo/margin_mean": 97.34046936035156,
"margin_dpo/margin_std": 192.24783325195312,
"step": 332
},
{
"KL/chosen_KL_mean": -243.05345153808594,
"KL/mean": -284.6350402832031,
"KL/rejected_KL_mean": -326.21661376953125,
"KL/std": 145.56326293945312,
"epoch": 0.6973821989528796,
"fcm_dpo/beta": 0.0030364254489541054,
"fcm_dpo/delta": 0.02071891352534294,
"fcm_dpo/margin": 83.16316986083984,
"fcm_dpo/q_t": 0.4418914318084717,
"grad_norm": 94.2875747680664,
"learning_rate": 1.281842711051438e-07,
"logits/chosen": -0.9469788074493408,
"logits/rejected": -0.9081500172615051,
"logps/chosen": -539.8164672851562,
"logps/ref_chosen": -296.76300048828125,
"logps/ref_rejected": -265.97991943359375,
"logps/rejected": -592.1965942382812,
"loss": 4.8583,
"margin_dpo/margin_mean": 83.16317749023438,
"margin_dpo/margin_std": 168.8401641845703,
"step": 333
},
{
"KL/chosen_KL_mean": -237.4409942626953,
"KL/mean": -285.7542724609375,
"KL/rejected_KL_mean": -334.0675354003906,
"KL/std": 146.4021453857422,
"epoch": 0.6994764397905759,
"fcm_dpo/beta": 0.0030563974287360907,
"fcm_dpo/delta": -0.020071666687726974,
"fcm_dpo/margin": 96.62651062011719,
"fcm_dpo/q_t": 0.43138545751571655,
"grad_norm": 58.2845573425293,
"learning_rate": 1.2658882646922033e-07,
"logits/chosen": -0.8468398451805115,
"logits/rejected": -0.817967414855957,
"logps/chosen": -538.4777221679688,
"logps/ref_chosen": -301.0367431640625,
"logps/ref_rejected": -268.87652587890625,
"logps/rejected": -602.9440307617188,
"loss": 4.7228,
"margin_dpo/margin_mean": 96.62650299072266,
"margin_dpo/margin_std": 164.7143096923828,
"step": 334
},
{
"KL/chosen_KL_mean": -256.3838806152344,
"KL/mean": -304.9598388671875,
"KL/rejected_KL_mean": -353.535888671875,
"KL/std": 136.20851135253906,
"epoch": 0.7015706806282722,
"fcm_dpo/beta": 0.0029724808409810066,
"fcm_dpo/delta": -0.027509108185768127,
"fcm_dpo/margin": 97.15202331542969,
"fcm_dpo/q_t": 0.43334853649139404,
"grad_norm": 62.15879821777344,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.8522113561630249,
"logits/rejected": -0.8463060259819031,
"logps/chosen": -532.5166625976562,
"logps/ref_chosen": -276.13275146484375,
"logps/ref_rejected": -243.44203186035156,
"logps/rejected": -596.9779052734375,
"loss": 4.7403,
"margin_dpo/margin_mean": 97.15201568603516,
"margin_dpo/margin_std": 172.3384246826172,
"step": 335
},
{
"KL/chosen_KL_mean": -251.550048828125,
"KL/mean": -300.40789794921875,
"KL/rejected_KL_mean": -349.2657165527344,
"KL/std": 172.0433349609375,
"epoch": 0.7036649214659686,
"fcm_dpo/beta": 0.0029860164504498243,
"fcm_dpo/delta": 0.013530546799302101,
"fcm_dpo/margin": 97.7156753540039,
"fcm_dpo/q_t": 0.4363878071308136,
"grad_norm": 74.54422760009766,
"learning_rate": 1.2341787690142435e-07,
"logits/chosen": -0.8671432733535767,
"logits/rejected": -0.8070932030677795,
"logps/chosen": -497.81268310546875,
"logps/ref_chosen": -246.2626495361328,
"logps/ref_rejected": -261.0617980957031,
"logps/rejected": -610.3275756835938,
"loss": 4.7949,
"margin_dpo/margin_mean": 97.7156753540039,
"margin_dpo/margin_std": 191.96246337890625,
"step": 336
},
{
"KL/chosen_KL_mean": -237.27195739746094,
"KL/mean": -291.9794006347656,
"KL/rejected_KL_mean": -346.6868896484375,
"KL/std": 143.3297882080078,
"epoch": 0.7057591623036649,
"fcm_dpo/beta": 0.0029976258520036936,
"fcm_dpo/delta": -0.021170055493712425,
"fcm_dpo/margin": 109.41490936279297,
"fcm_dpo/q_t": 0.42369288206100464,
"grad_norm": 77.50701904296875,
"learning_rate": 1.2184254201795363e-07,
"logits/chosen": -0.8679603338241577,
"logits/rejected": -0.8385904431343079,
"logps/chosen": -504.2657470703125,
"logps/ref_chosen": -266.9937744140625,
"logps/ref_rejected": -253.015625,
"logps/rejected": -599.7025146484375,
"loss": 4.5946,
"margin_dpo/margin_mean": 109.41490936279297,
"margin_dpo/margin_std": 163.00048828125,
"step": 337
},
{
"KL/chosen_KL_mean": -235.5108642578125,
"KL/mean": -288.2745056152344,
"KL/rejected_KL_mean": -341.03814697265625,
"KL/std": 152.8268280029297,
"epoch": 0.7078534031413612,
"fcm_dpo/beta": 0.0029197270050644875,
"fcm_dpo/delta": -0.005821624770760536,
"fcm_dpo/margin": 105.52726745605469,
"fcm_dpo/q_t": 0.4291449189186096,
"grad_norm": 75.93712615966797,
"learning_rate": 1.202740798300168e-07,
"logits/chosen": -0.8869338631629944,
"logits/rejected": -0.8708446025848389,
"logps/chosen": -512.1034545898438,
"logps/ref_chosen": -276.5925598144531,
"logps/ref_rejected": -233.979248046875,
"logps/rejected": -575.017333984375,
"loss": 4.6638,
"margin_dpo/margin_mean": 105.52726745605469,
"margin_dpo/margin_std": 172.4578094482422,
"step": 338
},
{
"KL/chosen_KL_mean": -237.4051513671875,
"KL/mean": -297.82196044921875,
"KL/rejected_KL_mean": -358.23876953125,
"KL/std": 157.0062255859375,
"epoch": 0.7099476439790576,
"fcm_dpo/beta": 0.0029296469874680042,
"fcm_dpo/delta": -0.021757004782557487,
"fcm_dpo/margin": 120.8335952758789,
"fcm_dpo/q_t": 0.41886672377586365,
"grad_norm": 64.22445678710938,
"learning_rate": 1.1871257444948096e-07,
"logits/chosen": -0.9070875644683838,
"logits/rejected": -0.8940801620483398,
"logps/chosen": -540.9329223632812,
"logps/ref_chosen": -303.5277404785156,
"logps/ref_rejected": -283.11676025390625,
"logps/rejected": -641.35546875,
"loss": 4.5489,
"margin_dpo/margin_mean": 120.83360290527344,
"margin_dpo/margin_std": 184.66961669921875,
"step": 339
},
{
"KL/chosen_KL_mean": -246.29815673828125,
"KL/mean": -293.8956604003906,
"KL/rejected_KL_mean": -341.4931640625,
"KL/std": 165.5948486328125,
"epoch": 0.7120418848167539,
"fcm_dpo/beta": 0.0028934727888554335,
"fcm_dpo/delta": 0.01472179964184761,
"fcm_dpo/margin": 95.19498443603516,
"fcm_dpo/q_t": 0.43556487560272217,
"grad_norm": 89.77741241455078,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": -0.8359464406967163,
"logits/rejected": -0.8298177719116211,
"logps/chosen": -507.82391357421875,
"logps/ref_chosen": -261.5257568359375,
"logps/ref_rejected": -259.39862060546875,
"logps/rejected": -600.8917236328125,
"loss": 4.88,
"margin_dpo/margin_mean": 95.19498443603516,
"margin_dpo/margin_std": 201.7469482421875,
"step": 340
},
{
"KL/chosen_KL_mean": -288.7161865234375,
"KL/mean": -333.5431213378906,
"KL/rejected_KL_mean": -378.3700256347656,
"KL/std": 184.27386474609375,
"epoch": 0.7141361256544503,
"fcm_dpo/beta": 0.002914420096203685,
"fcm_dpo/delta": -0.00907097663730383,
"fcm_dpo/margin": 89.65385437011719,
"fcm_dpo/q_t": 0.4414941966533661,
"grad_norm": 75.35196685791016,
"learning_rate": 1.1561076868822755e-07,
"logits/chosen": -0.8593753576278687,
"logits/rejected": -0.8326888680458069,
"logps/chosen": -604.6196899414062,
"logps/ref_chosen": -315.903564453125,
"logps/ref_rejected": -308.02392578125,
"logps/rejected": -686.3939819335938,
"loss": 4.9406,
"margin_dpo/margin_mean": 89.65385437011719,
"margin_dpo/margin_std": 208.85565185546875,
"step": 341
},
{
"KL/chosen_KL_mean": -238.28636169433594,
"KL/mean": -286.4154052734375,
"KL/rejected_KL_mean": -334.54443359375,
"KL/std": 143.09034729003906,
"epoch": 0.7162303664921466,
"fcm_dpo/beta": 0.002888055983930826,
"fcm_dpo/delta": -0.005859066266566515,
"fcm_dpo/margin": 96.25808715820312,
"fcm_dpo/q_t": 0.4352652430534363,
"grad_norm": 82.94450378417969,
"learning_rate": 1.1407063464793965e-07,
"logits/chosen": -0.8524309992790222,
"logits/rejected": -0.8473560810089111,
"logps/chosen": -507.4649963378906,
"logps/ref_chosen": -269.17864990234375,
"logps/ref_rejected": -260.8977355957031,
"logps/rejected": -595.442138671875,
"loss": 4.7214,
"margin_dpo/margin_mean": 96.25808715820312,
"margin_dpo/margin_std": 158.40966796875,
"step": 342
},
{
"KL/chosen_KL_mean": -251.1302032470703,
"KL/mean": -289.5614318847656,
"KL/rejected_KL_mean": -327.99267578125,
"KL/std": 144.47100830078125,
"epoch": 0.7183246073298429,
"fcm_dpo/beta": 0.0028825635090470314,
"fcm_dpo/delta": 0.007316844072192907,
"fcm_dpo/margin": 76.86248779296875,
"fcm_dpo/q_t": 0.4476861357688904,
"grad_norm": 78.4572525024414,
"learning_rate": 1.125377900869913e-07,
"logits/chosen": -0.8445292711257935,
"logits/rejected": -0.8152074813842773,
"logps/chosen": -561.8502197265625,
"logps/ref_chosen": -310.719970703125,
"logps/ref_rejected": -263.5224914550781,
"logps/rejected": -591.51513671875,
"loss": 4.9124,
"margin_dpo/margin_mean": 76.86248779296875,
"margin_dpo/margin_std": 157.17347717285156,
"step": 343
},
{
"KL/chosen_KL_mean": -247.63592529296875,
"KL/mean": -293.60247802734375,
"KL/rejected_KL_mean": -339.5689697265625,
"KL/std": 140.92300415039062,
"epoch": 0.7204188481675393,
"fcm_dpo/beta": 0.0028950828127563,
"fcm_dpo/delta": -0.003131479024887085,
"fcm_dpo/margin": 91.93303680419922,
"fcm_dpo/q_t": 0.4373406767845154,
"grad_norm": 72.97909545898438,
"learning_rate": 1.110123172071844e-07,
"logits/chosen": -0.8511172533035278,
"logits/rejected": -0.8328704237937927,
"logps/chosen": -549.4358520507812,
"logps/ref_chosen": -301.7999267578125,
"logps/ref_rejected": -257.9061584472656,
"logps/rejected": -597.4751586914062,
"loss": 4.7968,
"margin_dpo/margin_mean": 91.93302917480469,
"margin_dpo/margin_std": 167.35589599609375,
"step": 344
},
{
"KL/chosen_KL_mean": -232.2271728515625,
"KL/mean": -273.35272216796875,
"KL/rejected_KL_mean": -314.4782409667969,
"KL/std": 139.15582275390625,
"epoch": 0.7225130890052356,
"fcm_dpo/beta": 0.0028755811508744955,
"fcm_dpo/delta": -0.004357065074145794,
"fcm_dpo/margin": 82.25109100341797,
"fcm_dpo/q_t": 0.444185733795166,
"grad_norm": 76.50686645507812,
"learning_rate": 1.09494297815e-07,
"logits/chosen": -0.8535346984863281,
"logits/rejected": -0.8539408445358276,
"logps/chosen": -515.24560546875,
"logps/ref_chosen": -283.0184326171875,
"logps/ref_rejected": -266.8457336425781,
"logps/rejected": -581.323974609375,
"loss": 4.8509,
"margin_dpo/margin_mean": 82.25108337402344,
"margin_dpo/margin_std": 142.70669555664062,
"step": 345
},
{
"KL/chosen_KL_mean": -243.66827392578125,
"KL/mean": -296.9436950683594,
"KL/rejected_KL_mean": -350.21917724609375,
"KL/std": 160.29945373535156,
"epoch": 0.724607329842932,
"fcm_dpo/beta": 0.0028548564296215773,
"fcm_dpo/delta": -0.013966667465865612,
"fcm_dpo/margin": 106.5509033203125,
"fcm_dpo/q_t": 0.4294491708278656,
"grad_norm": 50.279972076416016,
"learning_rate": 1.0798381331721107e-07,
"logits/chosen": -0.9294825792312622,
"logits/rejected": -0.87758469581604,
"logps/chosen": -512.1094970703125,
"logps/ref_chosen": -268.44122314453125,
"logps/ref_rejected": -227.8225860595703,
"logps/rejected": -578.041748046875,
"loss": 4.666,
"margin_dpo/margin_mean": 106.5509033203125,
"margin_dpo/margin_std": 163.5757293701172,
"step": 346
},
{
"KL/chosen_KL_mean": -224.2758331298828,
"KL/mean": -267.6025390625,
"KL/rejected_KL_mean": -310.92919921875,
"KL/std": 138.48739624023438,
"epoch": 0.7267015706806282,
"fcm_dpo/beta": 0.002854297636076808,
"fcm_dpo/delta": 0.007548038847744465,
"fcm_dpo/margin": 86.65332794189453,
"fcm_dpo/q_t": 0.4415615499019623,
"grad_norm": 83.579833984375,
"learning_rate": 1.0648094471651722e-07,
"logits/chosen": -0.7760555148124695,
"logits/rejected": -0.8024675250053406,
"logps/chosen": -497.9794006347656,
"logps/ref_chosen": -273.70355224609375,
"logps/ref_rejected": -243.65521240234375,
"logps/rejected": -554.5843505859375,
"loss": 4.8312,
"margin_dpo/margin_mean": 86.65332794189453,
"margin_dpo/margin_std": 157.4734344482422,
"step": 347
},
{
"KL/chosen_KL_mean": -220.09567260742188,
"KL/mean": -258.8450622558594,
"KL/rejected_KL_mean": -297.594482421875,
"KL/std": 147.97817993164062,
"epoch": 0.7287958115183246,
"fcm_dpo/beta": 0.0028738253749907017,
"fcm_dpo/delta": 0.0110637117177248,
"fcm_dpo/margin": 77.49883270263672,
"fcm_dpo/q_t": 0.4496265649795532,
"grad_norm": 75.31076049804688,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": -0.892849326133728,
"logits/rejected": -0.8771178722381592,
"logps/chosen": -505.7370910644531,
"logps/ref_chosen": -285.64141845703125,
"logps/ref_rejected": -265.6270446777344,
"logps/rejected": -563.2215576171875,
"loss": 4.9376,
"margin_dpo/margin_mean": 77.49882507324219,
"margin_dpo/margin_std": 166.94793701171875,
"step": 348
},
{
"KL/chosen_KL_mean": -220.64576721191406,
"KL/mean": -279.0647277832031,
"KL/rejected_KL_mean": -337.48370361328125,
"KL/std": 160.11648559570312,
"epoch": 0.7308900523560209,
"fcm_dpo/beta": 0.0028767730109393597,
"fcm_dpo/delta": -0.013212434016168118,
"fcm_dpo/margin": 116.83795928955078,
"fcm_dpo/q_t": 0.42093393206596375,
"grad_norm": 66.55834197998047,
"learning_rate": 1.0349837717080347e-07,
"logits/chosen": -0.8378231525421143,
"logits/rejected": -0.8238610625267029,
"logps/chosen": -548.9632568359375,
"logps/ref_chosen": -328.3175048828125,
"logps/ref_rejected": -292.37872314453125,
"logps/rejected": -629.8624267578125,
"loss": 4.6037,
"margin_dpo/margin_mean": 116.83795928955078,
"margin_dpo/margin_std": 184.72499084472656,
"step": 349
},
{
"KL/chosen_KL_mean": -237.5376434326172,
"KL/mean": -285.01214599609375,
"KL/rejected_KL_mean": -332.4866027832031,
"KL/std": 148.0513916015625,
"epoch": 0.7329842931937173,
"fcm_dpo/beta": 0.0028601842932403088,
"fcm_dpo/delta": 0.004661480896174908,
"fcm_dpo/margin": 94.94898986816406,
"fcm_dpo/q_t": 0.43736475706100464,
"grad_norm": 80.1038818359375,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": -0.8289676904678345,
"logits/rejected": -0.8410451412200928,
"logps/chosen": -530.34228515625,
"logps/ref_chosen": -292.8046569824219,
"logps/ref_rejected": -250.35504150390625,
"logps/rejected": -582.8416748046875,
"loss": 4.7774,
"margin_dpo/margin_mean": 94.94898986816406,
"margin_dpo/margin_std": 173.54653930664062,
"step": 350
},
{
"KL/chosen_KL_mean": -223.345458984375,
"KL/mean": -254.21771240234375,
"KL/rejected_KL_mean": -285.0899658203125,
"KL/std": 138.79812622070312,
"epoch": 0.7350785340314137,
"fcm_dpo/beta": 0.0028854927513748407,
"fcm_dpo/delta": 0.013731059618294239,
"fcm_dpo/margin": 61.744510650634766,
"fcm_dpo/q_t": 0.4585198163986206,
"grad_norm": 70.86201477050781,
"learning_rate": 1.0054723495346482e-07,
"logits/chosen": -0.8867701292037964,
"logits/rejected": -0.8659510612487793,
"logps/chosen": -535.2344970703125,
"logps/ref_chosen": -311.8890380859375,
"logps/ref_rejected": -263.59033203125,
"logps/rejected": -548.6802978515625,
"loss": 5.113,
"margin_dpo/margin_mean": 61.744510650634766,
"margin_dpo/margin_std": 169.5723876953125,
"step": 351
},
{
"KL/chosen_KL_mean": -210.1036376953125,
"KL/mean": -268.3479309082031,
"KL/rejected_KL_mean": -326.5922546386719,
"KL/std": 149.0188446044922,
"epoch": 0.7371727748691099,
"fcm_dpo/beta": 0.0028885467909276485,
"fcm_dpo/delta": -0.00935527216643095,
"fcm_dpo/margin": 116.48863220214844,
"fcm_dpo/q_t": 0.42103269696235657,
"grad_norm": 57.36745834350586,
"learning_rate": 9.908364643332398e-08,
"logits/chosen": -0.8118300437927246,
"logits/rejected": -0.7807732820510864,
"logps/chosen": -465.0115051269531,
"logps/ref_chosen": -254.9078826904297,
"logps/ref_rejected": -257.1688232421875,
"logps/rejected": -583.7610473632812,
"loss": 4.5527,
"margin_dpo/margin_mean": 116.48863983154297,
"margin_dpo/margin_std": 165.00392150878906,
"step": 352
},
{
"KL/chosen_KL_mean": -229.11819458007812,
"KL/mean": -279.8221740722656,
"KL/rejected_KL_mean": -330.526123046875,
"KL/std": 147.36636352539062,
"epoch": 0.7392670157068063,
"fcm_dpo/beta": 0.0028685785364359617,
"fcm_dpo/delta": -0.013993466272950172,
"fcm_dpo/margin": 101.40792846679688,
"fcm_dpo/q_t": 0.4323953688144684,
"grad_norm": 93.38015747070312,
"learning_rate": 9.76281510992176e-08,
"logits/chosen": -0.8352831602096558,
"logits/rejected": -0.8298249840736389,
"logps/chosen": -499.4942626953125,
"logps/ref_chosen": -270.3760681152344,
"logps/ref_rejected": -264.65234375,
"logps/rejected": -595.178466796875,
"loss": 4.7011,
"margin_dpo/margin_mean": 101.40792846679688,
"margin_dpo/margin_std": 164.58172607421875,
"step": 353
},
{
"KL/chosen_KL_mean": -226.212646484375,
"KL/mean": -265.5102233886719,
"KL/rejected_KL_mean": -304.8078308105469,
"KL/std": 154.871826171875,
"epoch": 0.7413612565445026,
"fcm_dpo/beta": 0.002862285589799285,
"fcm_dpo/delta": 0.012168960645794868,
"fcm_dpo/margin": 78.59518432617188,
"fcm_dpo/q_t": 0.4484597146511078,
"grad_norm": 77.23882293701172,
"learning_rate": 9.618082700494318e-08,
"logits/chosen": -0.8314879536628723,
"logits/rejected": -0.8639715909957886,
"logps/chosen": -483.8612365722656,
"logps/ref_chosen": -257.6485595703125,
"logps/ref_rejected": -246.94203186035156,
"logps/rejected": -551.7498779296875,
"loss": 4.9439,
"margin_dpo/margin_mean": 78.59518432617188,
"margin_dpo/margin_std": 171.47744750976562,
"step": 354
},
{
"KL/chosen_KL_mean": -228.62823486328125,
"KL/mean": -280.390625,
"KL/rejected_KL_mean": -332.1530456542969,
"KL/std": 161.08724975585938,
"epoch": 0.743455497382199,
"fcm_dpo/beta": 0.0028833611868321896,
"fcm_dpo/delta": 0.009556437842547894,
"fcm_dpo/margin": 103.52484893798828,
"fcm_dpo/q_t": 0.431077778339386,
"grad_norm": 55.02450942993164,
"learning_rate": 9.474175176609956e-08,
"logits/chosen": -0.8792758584022522,
"logits/rejected": -0.8736676573753357,
"logps/chosen": -521.9815673828125,
"logps/ref_chosen": -293.35333251953125,
"logps/ref_rejected": -275.6051940917969,
"logps/rejected": -607.7582397460938,
"loss": 4.7879,
"margin_dpo/margin_mean": 103.52484893798828,
"margin_dpo/margin_std": 198.6239013671875,
"step": 355
},
{
"KL/chosen_KL_mean": -216.16531372070312,
"KL/mean": -253.62515258789062,
"KL/rejected_KL_mean": -291.08502197265625,
"KL/std": 134.0445556640625,
"epoch": 0.7455497382198953,
"fcm_dpo/beta": 0.0028942120261490345,
"fcm_dpo/delta": -0.0024351300671696663,
"fcm_dpo/margin": 74.91969299316406,
"fcm_dpo/q_t": 0.44854995608329773,
"grad_norm": 71.81739044189453,
"learning_rate": 9.331100255592436e-08,
"logits/chosen": -0.7871492505073547,
"logits/rejected": -0.8126851320266724,
"logps/chosen": -420.4208068847656,
"logps/ref_chosen": -204.25550842285156,
"logps/ref_rejected": -213.467529296875,
"logps/rejected": -504.552490234375,
"loss": 4.9034,
"margin_dpo/margin_mean": 74.91969299316406,
"margin_dpo/margin_std": 139.6249542236328,
"step": 356
},
{
"KL/chosen_KL_mean": -239.94329833984375,
"KL/mean": -289.7812805175781,
"KL/rejected_KL_mean": -339.6192932128906,
"KL/std": 154.73379516601562,
"epoch": 0.7476439790575916,
"fcm_dpo/beta": 0.00291292741894722,
"fcm_dpo/delta": 0.021629903465509415,
"fcm_dpo/margin": 99.67597961425781,
"fcm_dpo/q_t": 0.433463990688324,
"grad_norm": 69.52693939208984,
"learning_rate": 9.18886561011557e-08,
"logits/chosen": -0.7813151478767395,
"logits/rejected": -0.7780415415763855,
"logps/chosen": -506.3138427734375,
"logps/ref_chosen": -266.3705749511719,
"logps/ref_rejected": -239.04490661621094,
"logps/rejected": -578.6642456054688,
"loss": 4.7521,
"margin_dpo/margin_mean": 99.67597961425781,
"margin_dpo/margin_std": 178.00946044921875,
"step": 357
},
{
"KL/chosen_KL_mean": -217.33197021484375,
"KL/mean": -272.29388427734375,
"KL/rejected_KL_mean": -327.2558288574219,
"KL/std": 141.83389282226562,
"epoch": 0.749738219895288,
"fcm_dpo/beta": 0.00295689282938838,
"fcm_dpo/delta": 0.014568077400326729,
"fcm_dpo/margin": 109.92384338378906,
"fcm_dpo/q_t": 0.42453300952911377,
"grad_norm": 83.9714126586914,
"learning_rate": 9.047478867791731e-08,
"logits/chosen": -0.8699600696563721,
"logits/rejected": -0.8453044891357422,
"logps/chosen": -516.4794311523438,
"logps/ref_chosen": -299.1474609375,
"logps/ref_rejected": -257.2531433105469,
"logps/rejected": -584.509033203125,
"loss": 4.6161,
"margin_dpo/margin_mean": 109.92383575439453,
"margin_dpo/margin_std": 175.24615478515625,
"step": 358
},
{
"KL/chosen_KL_mean": -230.4601593017578,
"KL/mean": -278.14129638671875,
"KL/rejected_KL_mean": -325.82244873046875,
"KL/std": 147.65829467773438,
"epoch": 0.7518324607329843,
"fcm_dpo/beta": 0.0029695353005081415,
"fcm_dpo/delta": 0.0013146661221981049,
"fcm_dpo/margin": 95.36227416992188,
"fcm_dpo/q_t": 0.43455877900123596,
"grad_norm": 70.06732177734375,
"learning_rate": 8.906947610762825e-08,
"logits/chosen": -0.8403257131576538,
"logits/rejected": -0.8475351333618164,
"logps/chosen": -533.4580078125,
"logps/ref_chosen": -302.99786376953125,
"logps/ref_rejected": -260.4137268066406,
"logps/rejected": -586.2362060546875,
"loss": 4.7681,
"margin_dpo/margin_mean": 95.36227416992188,
"margin_dpo/margin_std": 175.98709106445312,
"step": 359
},
{
"KL/chosen_KL_mean": -232.94993591308594,
"KL/mean": -278.8365783691406,
"KL/rejected_KL_mean": -324.7232360839844,
"KL/std": 160.8883056640625,
"epoch": 0.7539267015706806,
"fcm_dpo/beta": 0.0029307494405657053,
"fcm_dpo/delta": -0.03134762495756149,
"fcm_dpo/margin": 91.77330780029297,
"fcm_dpo/q_t": 0.43876707553863525,
"grad_norm": 60.748844146728516,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": -0.8615961074829102,
"logits/rejected": -0.8414865136146545,
"logps/chosen": -542.5614013671875,
"logps/ref_chosen": -309.6114501953125,
"logps/ref_rejected": -256.64031982421875,
"logps/rejected": -581.363525390625,
"loss": 4.8976,
"margin_dpo/margin_mean": 91.77330780029297,
"margin_dpo/margin_std": 189.3778076171875,
"step": 360
},
{
"KL/chosen_KL_mean": -196.89012145996094,
"KL/mean": -256.0363464355469,
"KL/rejected_KL_mean": -315.18255615234375,
"KL/std": 154.51834106445312,
"epoch": 0.7560209424083769,
"fcm_dpo/beta": 0.0029320104513317347,
"fcm_dpo/delta": 0.015726841986179352,
"fcm_dpo/margin": 118.29248046875,
"fcm_dpo/q_t": 0.421514093875885,
"grad_norm": 64.89899444580078,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": -0.803794264793396,
"logits/rejected": -0.7853226065635681,
"logps/chosen": -460.2698669433594,
"logps/ref_chosen": -263.3797607421875,
"logps/ref_rejected": -271.18157958984375,
"logps/rejected": -586.3641357421875,
"loss": 4.5926,
"margin_dpo/margin_mean": 118.29248046875,
"margin_dpo/margin_std": 193.5961456298828,
"step": 361
},
{
"KL/chosen_KL_mean": -219.24996948242188,
"KL/mean": -266.0592956542969,
"KL/rejected_KL_mean": -312.86865234375,
"KL/std": 129.30587768554688,
"epoch": 0.7581151832460733,
"fcm_dpo/beta": 0.002948556561022997,
"fcm_dpo/delta": 0.006594155449420214,
"fcm_dpo/margin": 93.61869812011719,
"fcm_dpo/q_t": 0.43606704473495483,
"grad_norm": 72.42217254638672,
"learning_rate": 8.490561882286135e-08,
"logits/chosen": -0.8361051082611084,
"logits/rejected": -0.81890469789505,
"logps/chosen": -522.5083618164062,
"logps/ref_chosen": -303.2583923339844,
"logps/ref_rejected": -243.22891235351562,
"logps/rejected": -556.0975341796875,
"loss": 4.7161,
"margin_dpo/margin_mean": 93.61870574951172,
"margin_dpo/margin_std": 146.4253387451172,
"step": 362
},
{
"KL/chosen_KL_mean": -224.68502807617188,
"KL/mean": -277.4530029296875,
"KL/rejected_KL_mean": -330.220947265625,
"KL/std": 156.96578979492188,
"epoch": 0.7602094240837697,
"fcm_dpo/beta": 0.002929537557065487,
"fcm_dpo/delta": -0.03551473096013069,
"fcm_dpo/margin": 105.53596496582031,
"fcm_dpo/q_t": 0.42976054549217224,
"grad_norm": 71.42375183105469,
"learning_rate": 8.353527464267104e-08,
"logits/chosen": -0.8412438035011292,
"logits/rejected": -0.7956109642982483,
"logps/chosen": -528.0322875976562,
"logps/ref_chosen": -303.34722900390625,
"logps/ref_rejected": -262.05419921875,
"logps/rejected": -592.2752075195312,
"loss": 4.7065,
"margin_dpo/margin_mean": 105.53596496582031,
"margin_dpo/margin_std": 184.98667907714844,
"step": 363
},
{
"KL/chosen_KL_mean": -254.75527954101562,
"KL/mean": -303.8872375488281,
"KL/rejected_KL_mean": -353.0191650390625,
"KL/std": 167.35247802734375,
"epoch": 0.762303664921466,
"fcm_dpo/beta": 0.002902527106925845,
"fcm_dpo/delta": 0.02750171534717083,
"fcm_dpo/margin": 98.26390838623047,
"fcm_dpo/q_t": 0.43715596199035645,
"grad_norm": 70.5536117553711,
"learning_rate": 8.217385746050742e-08,
"logits/chosen": -0.8307126760482788,
"logits/rejected": -0.8298197984695435,
"logps/chosen": -540.2990112304688,
"logps/ref_chosen": -285.54376220703125,
"logps/ref_rejected": -284.84619140625,
"logps/rejected": -637.8654174804688,
"loss": 4.8201,
"margin_dpo/margin_mean": 98.263916015625,
"margin_dpo/margin_std": 200.9420623779297,
"step": 364
},
{
"KL/chosen_KL_mean": -238.25563049316406,
"KL/mean": -282.6103515625,
"KL/rejected_KL_mean": -326.96502685546875,
"KL/std": 162.19873046875,
"epoch": 0.7643979057591623,
"fcm_dpo/beta": 0.002954564755782485,
"fcm_dpo/delta": 0.009645760990679264,
"fcm_dpo/margin": 88.70938110351562,
"fcm_dpo/q_t": 0.44021692872047424,
"grad_norm": 70.69696807861328,
"learning_rate": 8.082144028504231e-08,
"logits/chosen": -0.8417816758155823,
"logits/rejected": -0.8418366312980652,
"logps/chosen": -513.04345703125,
"logps/ref_chosen": -274.7878112792969,
"logps/ref_rejected": -256.5738220214844,
"logps/rejected": -583.5388793945312,
"loss": 4.8457,
"margin_dpo/margin_mean": 88.70938110351562,
"margin_dpo/margin_std": 176.14007568359375,
"step": 365
},
{
"KL/chosen_KL_mean": -222.59596252441406,
"KL/mean": -271.5234680175781,
"KL/rejected_KL_mean": -320.450927734375,
"KL/std": 141.25015258789062,
"epoch": 0.7664921465968586,
"fcm_dpo/beta": 0.0030219045002013445,
"fcm_dpo/delta": 0.037596531212329865,
"fcm_dpo/margin": 97.85496520996094,
"fcm_dpo/q_t": 0.4315846860408783,
"grad_norm": 70.44728088378906,
"learning_rate": 7.947809564230445e-08,
"logits/chosen": -0.7966880798339844,
"logits/rejected": -0.8008946776390076,
"logps/chosen": -509.24560546875,
"logps/ref_chosen": -286.6496276855469,
"logps/ref_rejected": -251.97140502929688,
"logps/rejected": -572.42236328125,
"loss": 4.717,
"margin_dpo/margin_mean": 97.85496520996094,
"margin_dpo/margin_std": 171.53121948242188,
"step": 366
},
{
"KL/chosen_KL_mean": -226.7736358642578,
"KL/mean": -278.9706115722656,
"KL/rejected_KL_mean": -331.16766357421875,
"KL/std": 146.42007446289062,
"epoch": 0.768586387434555,
"fcm_dpo/beta": 0.00305999256670475,
"fcm_dpo/delta": 0.010715863667428493,
"fcm_dpo/margin": 104.39400482177734,
"fcm_dpo/q_t": 0.4274834990501404,
"grad_norm": 82.6015625,
"learning_rate": 7.814389557179016e-08,
"logits/chosen": -0.8351811766624451,
"logits/rejected": -0.8077444434165955,
"logps/chosen": -528.7186279296875,
"logps/ref_chosen": -301.9449768066406,
"logps/ref_rejected": -265.5677185058594,
"logps/rejected": -596.7353515625,
"loss": 4.6422,
"margin_dpo/margin_mean": 104.39400482177734,
"margin_dpo/margin_std": 168.87966918945312,
"step": 367
},
{
"KL/chosen_KL_mean": -209.43209838867188,
"KL/mean": -276.49969482421875,
"KL/rejected_KL_mean": -343.56732177734375,
"KL/std": 147.89605712890625,
"epoch": 0.7706806282722513,
"fcm_dpo/beta": 0.0030316186603158712,
"fcm_dpo/delta": -0.04178418964147568,
"fcm_dpo/margin": 134.13523864746094,
"fcm_dpo/q_t": 0.405577689409256,
"grad_norm": 76.57606506347656,
"learning_rate": 7.681891162260015e-08,
"logits/chosen": -0.7988978028297424,
"logits/rejected": -0.8013542890548706,
"logps/chosen": -504.05865478515625,
"logps/ref_chosen": -294.62652587890625,
"logps/ref_rejected": -258.7628479003906,
"logps/rejected": -602.3301391601562,
"loss": 4.3154,
"margin_dpo/margin_mean": 134.13523864746094,
"margin_dpo/margin_std": 154.99842834472656,
"step": 368
},
{
"KL/chosen_KL_mean": -225.83438110351562,
"KL/mean": -273.8876647949219,
"KL/rejected_KL_mean": -321.9409484863281,
"KL/std": 137.2335205078125,
"epoch": 0.7727748691099476,
"fcm_dpo/beta": 0.0030030980706214905,
"fcm_dpo/delta": 0.01607823558151722,
"fcm_dpo/margin": 96.10658264160156,
"fcm_dpo/q_t": 0.4320639371871948,
"grad_norm": 82.7309341430664,
"learning_rate": 7.550321484960251e-08,
"logits/chosen": -0.8810120820999146,
"logits/rejected": -0.8641123175621033,
"logps/chosen": -508.340087890625,
"logps/ref_chosen": -282.5057373046875,
"logps/ref_rejected": -266.41607666015625,
"logps/rejected": -588.3570556640625,
"loss": 4.6647,
"margin_dpo/margin_mean": 96.10657501220703,
"margin_dpo/margin_std": 144.0449981689453,
"step": 369
},
{
"KL/chosen_KL_mean": -215.18609619140625,
"KL/mean": -274.7394104003906,
"KL/rejected_KL_mean": -334.2928161621094,
"KL/std": 153.53570556640625,
"epoch": 0.774869109947644,
"fcm_dpo/beta": 0.003025064943358302,
"fcm_dpo/delta": -0.0023751449771225452,
"fcm_dpo/margin": 119.10672760009766,
"fcm_dpo/q_t": 0.4178846776485443,
"grad_norm": 62.49055480957031,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": -0.8580411076545715,
"logits/rejected": -0.8811969757080078,
"logps/chosen": -466.1924743652344,
"logps/ref_chosen": -251.00640869140625,
"logps/ref_rejected": -238.12542724609375,
"logps/rejected": -572.418212890625,
"loss": 4.5629,
"margin_dpo/margin_mean": 119.10673522949219,
"margin_dpo/margin_std": 193.12779235839844,
"step": 370
},
{
"KL/chosen_KL_mean": -250.4105682373047,
"KL/mean": -285.2479248046875,
"KL/rejected_KL_mean": -320.0852355957031,
"KL/std": 149.4564666748047,
"epoch": 0.7769633507853403,
"fcm_dpo/beta": 0.003027756232768297,
"fcm_dpo/delta": -0.0017022611573338509,
"fcm_dpo/margin": 69.6746597290039,
"fcm_dpo/q_t": 0.450680136680603,
"grad_norm": 105.20964050292969,
"learning_rate": 7.289996455765748e-08,
"logits/chosen": -0.8298521041870117,
"logits/rejected": -0.8093293309211731,
"logps/chosen": -547.0697021484375,
"logps/ref_chosen": -296.6591491699219,
"logps/ref_rejected": -251.14675903320312,
"logps/rejected": -571.23193359375,
"loss": 4.9843,
"margin_dpo/margin_mean": 69.67466735839844,
"margin_dpo/margin_std": 154.35647583007812,
"step": 371
},
{
"KL/chosen_KL_mean": -240.75746154785156,
"KL/mean": -299.1548156738281,
"KL/rejected_KL_mean": -357.55224609375,
"KL/std": 165.2203826904297,
"epoch": 0.7790575916230367,
"fcm_dpo/beta": 0.0029513647314161062,
"fcm_dpo/delta": -0.024020541459321976,
"fcm_dpo/margin": 116.7947998046875,
"fcm_dpo/q_t": 0.4221827983856201,
"grad_norm": 70.41334533691406,
"learning_rate": 7.161255064312283e-08,
"logits/chosen": -0.7842466235160828,
"logits/rejected": -0.7680008411407471,
"logps/chosen": -572.12890625,
"logps/ref_chosen": -331.3714599609375,
"logps/ref_rejected": -285.56805419921875,
"logps/rejected": -643.1203002929688,
"loss": 4.6121,
"margin_dpo/margin_mean": 116.7947998046875,
"margin_dpo/margin_std": 187.99862670898438,
"step": 372
},
{
"KL/chosen_KL_mean": -226.25857543945312,
"KL/mean": -281.29608154296875,
"KL/rejected_KL_mean": -336.3335876464844,
"KL/std": 142.73548889160156,
"epoch": 0.7811518324607329,
"fcm_dpo/beta": 0.002992741297930479,
"fcm_dpo/delta": 0.03157367929816246,
"fcm_dpo/margin": 110.07496643066406,
"fcm_dpo/q_t": 0.42270153760910034,
"grad_norm": 86.4478988647461,
"learning_rate": 7.033470310611945e-08,
"logits/chosen": -0.8854488730430603,
"logits/rejected": -0.8503902554512024,
"logps/chosen": -548.2015380859375,
"logps/ref_chosen": -321.9429931640625,
"logps/ref_rejected": -271.2288513183594,
"logps/rejected": -607.5624389648438,
"loss": 4.5869,
"margin_dpo/margin_mean": 110.0749740600586,
"margin_dpo/margin_std": 165.2940216064453,
"step": 373
},
{
"KL/chosen_KL_mean": -236.50103759765625,
"KL/mean": -282.9904479980469,
"KL/rejected_KL_mean": -329.4798278808594,
"KL/std": 160.59323120117188,
"epoch": 0.7832460732984293,
"fcm_dpo/beta": 0.0029890371952205896,
"fcm_dpo/delta": -0.015482398681342602,
"fcm_dpo/margin": 92.97879028320312,
"fcm_dpo/q_t": 0.43591830134391785,
"grad_norm": 74.21908569335938,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": -0.8732917904853821,
"logits/rejected": -0.8661702871322632,
"logps/chosen": -555.6695556640625,
"logps/ref_chosen": -319.1685485839844,
"logps/ref_rejected": -284.6263732910156,
"logps/rejected": -614.1061401367188,
"loss": 4.773,
"margin_dpo/margin_mean": 92.97879028320312,
"margin_dpo/margin_std": 169.21636962890625,
"step": 374
},
{
"KL/chosen_KL_mean": -245.4715576171875,
"KL/mean": -289.0046691894531,
"KL/rejected_KL_mean": -332.53778076171875,
"KL/std": 133.46920776367188,
"epoch": 0.7853403141361257,
"fcm_dpo/beta": 0.0029952474869787693,
"fcm_dpo/delta": -0.0004676980897784233,
"fcm_dpo/margin": 87.06620788574219,
"fcm_dpo/q_t": 0.43925830721855164,
"grad_norm": 74.76742553710938,
"learning_rate": 6.780798075635675e-08,
"logits/chosen": -0.8679319620132446,
"logits/rejected": -0.8446568250656128,
"logps/chosen": -560.3473510742188,
"logps/ref_chosen": -314.87579345703125,
"logps/ref_rejected": -259.1965026855469,
"logps/rejected": -591.7342529296875,
"loss": 4.8,
"margin_dpo/margin_mean": 87.06620788574219,
"margin_dpo/margin_std": 155.70870971679688,
"step": 375
},
{
"KL/chosen_KL_mean": -241.67770385742188,
"KL/mean": -294.02313232421875,
"KL/rejected_KL_mean": -346.36859130859375,
"KL/std": 157.73382568359375,
"epoch": 0.787434554973822,
"fcm_dpo/beta": 0.0029938980005681515,
"fcm_dpo/delta": -0.005739666521549225,
"fcm_dpo/margin": 104.69088745117188,
"fcm_dpo/q_t": 0.42898622155189514,
"grad_norm": 100.38687133789062,
"learning_rate": 6.655924144404906e-08,
"logits/chosen": -0.8551640510559082,
"logits/rejected": -0.8558748364448547,
"logps/chosen": -529.3509521484375,
"logps/ref_chosen": -287.6732482910156,
"logps/ref_rejected": -256.6697082519531,
"logps/rejected": -603.0382690429688,
"loss": 4.7103,
"margin_dpo/margin_mean": 104.69087982177734,
"margin_dpo/margin_std": 189.6014404296875,
"step": 376
},
{
"KL/chosen_KL_mean": -258.047119140625,
"KL/mean": -295.9388732910156,
"KL/rejected_KL_mean": -333.83062744140625,
"KL/std": 157.74354553222656,
"epoch": 0.7895287958115184,
"fcm_dpo/beta": 0.00297717796638608,
"fcm_dpo/delta": 0.003270770888775587,
"fcm_dpo/margin": 75.78350830078125,
"fcm_dpo/q_t": 0.4478086233139038,
"grad_norm": 64.10608673095703,
"learning_rate": 6.532033950290885e-08,
"logits/chosen": -0.8300923109054565,
"logits/rejected": -0.8211076855659485,
"logps/chosen": -563.30859375,
"logps/ref_chosen": -305.261474609375,
"logps/ref_rejected": -271.8887023925781,
"logps/rejected": -605.7192993164062,
"loss": 4.9974,
"margin_dpo/margin_mean": 75.78350830078125,
"margin_dpo/margin_std": 182.6964874267578,
"step": 377
},
{
"KL/chosen_KL_mean": -248.6783905029297,
"KL/mean": -296.946533203125,
"KL/rejected_KL_mean": -345.21466064453125,
"KL/std": 163.25067138671875,
"epoch": 0.7916230366492146,
"fcm_dpo/beta": 0.0030068312771618366,
"fcm_dpo/delta": 0.02455754391849041,
"fcm_dpo/margin": 96.5362548828125,
"fcm_dpo/q_t": 0.4341698884963989,
"grad_norm": 74.27600860595703,
"learning_rate": 6.409134137148736e-08,
"logits/chosen": -0.8239084482192993,
"logits/rejected": -0.8121182322502136,
"logps/chosen": -530.2080078125,
"logps/ref_chosen": -281.5295715332031,
"logps/ref_rejected": -296.980224609375,
"logps/rejected": -642.1949462890625,
"loss": 4.7595,
"margin_dpo/margin_mean": 96.5362548828125,
"margin_dpo/margin_std": 179.10707092285156,
"step": 378
},
{
"KL/chosen_KL_mean": -228.6240234375,
"KL/mean": -274.6946105957031,
"KL/rejected_KL_mean": -320.76519775390625,
"KL/std": 132.6265869140625,
"epoch": 0.793717277486911,
"fcm_dpo/beta": 0.0030300889629870653,
"fcm_dpo/delta": -0.00830287579447031,
"fcm_dpo/margin": 92.14117431640625,
"fcm_dpo/q_t": 0.43474820256233215,
"grad_norm": 89.22589874267578,
"learning_rate": 6.28723129572247e-08,
"logits/chosen": -0.8947068452835083,
"logits/rejected": -0.8764788508415222,
"logps/chosen": -493.704833984375,
"logps/ref_chosen": -265.0807800292969,
"logps/ref_rejected": -230.58932495117188,
"logps/rejected": -551.3544921875,
"loss": 4.7549,
"margin_dpo/margin_mean": 92.14117431640625,
"margin_dpo/margin_std": 164.5974578857422,
"step": 379
},
{
"KL/chosen_KL_mean": -243.8111572265625,
"KL/mean": -299.34429931640625,
"KL/rejected_KL_mean": -354.87744140625,
"KL/std": 164.056640625,
"epoch": 0.7958115183246073,
"fcm_dpo/beta": 0.0029976628720760345,
"fcm_dpo/delta": -0.021797433495521545,
"fcm_dpo/margin": 111.06632995605469,
"fcm_dpo/q_t": 0.4255141019821167,
"grad_norm": 91.8367691040039,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": -0.8527557253837585,
"logits/rejected": -0.8262585997581482,
"logps/chosen": -549.7195434570312,
"logps/ref_chosen": -305.90838623046875,
"logps/ref_rejected": -286.5906677246094,
"logps/rejected": -641.4681396484375,
"loss": 4.6676,
"margin_dpo/margin_mean": 111.06632232666016,
"margin_dpo/margin_std": 190.367919921875,
"step": 380
},
{
"KL/chosen_KL_mean": -243.97262573242188,
"KL/mean": -303.4189147949219,
"KL/rejected_KL_mean": -362.86517333984375,
"KL/std": 184.5408477783203,
"epoch": 0.7979057591623037,
"fcm_dpo/beta": 0.002953719347715378,
"fcm_dpo/delta": -0.021564383059740067,
"fcm_dpo/margin": 118.892578125,
"fcm_dpo/q_t": 0.422404944896698,
"grad_norm": 64.48091888427734,
"learning_rate": 6.046442623320145e-08,
"logits/chosen": -0.8266147375106812,
"logits/rejected": -0.7924824357032776,
"logps/chosen": -496.84326171875,
"logps/ref_chosen": -252.87066650390625,
"logps/ref_rejected": -261.1927490234375,
"logps/rejected": -624.0579223632812,
"loss": 4.5793,
"margin_dpo/margin_mean": 118.892578125,
"margin_dpo/margin_std": 191.17279052734375,
"step": 381
},
{
"KL/chosen_KL_mean": -236.02357482910156,
"KL/mean": -297.5960998535156,
"KL/rejected_KL_mean": -359.1686706542969,
"KL/std": 151.47557067871094,
"epoch": 0.8,
"fcm_dpo/beta": 0.0028901936020702124,
"fcm_dpo/delta": -0.010631639510393143,
"fcm_dpo/margin": 123.14509582519531,
"fcm_dpo/q_t": 0.4171634614467621,
"grad_norm": 66.18559265136719,
"learning_rate": 5.9275697051098275e-08,
"logits/chosen": -0.8696933388710022,
"logits/rejected": -0.8662832379341125,
"logps/chosen": -525.2350463867188,
"logps/ref_chosen": -289.2114562988281,
"logps/ref_rejected": -278.45751953125,
"logps/rejected": -637.626220703125,
"loss": 4.4974,
"margin_dpo/margin_mean": 123.14509582519531,
"margin_dpo/margin_std": 173.98348999023438,
"step": 382
},
{
"KL/chosen_KL_mean": -237.31979370117188,
"KL/mean": -290.00091552734375,
"KL/rejected_KL_mean": -342.68206787109375,
"KL/std": 168.33383178710938,
"epoch": 0.8020942408376963,
"fcm_dpo/beta": 0.0028781089931726456,
"fcm_dpo/delta": -0.025008056312799454,
"fcm_dpo/margin": 105.36231994628906,
"fcm_dpo/q_t": 0.43072545528411865,
"grad_norm": 75.10661315917969,
"learning_rate": 5.809719583454414e-08,
"logits/chosen": -0.850218653678894,
"logits/rejected": -0.8339353203773499,
"logps/chosen": -510.9505920410156,
"logps/ref_chosen": -273.630859375,
"logps/ref_rejected": -261.44024658203125,
"logps/rejected": -604.122314453125,
"loss": 4.7315,
"margin_dpo/margin_mean": 105.36231231689453,
"margin_dpo/margin_std": 192.60629272460938,
"step": 383
},
{
"KL/chosen_KL_mean": -241.386474609375,
"KL/mean": -290.0650939941406,
"KL/rejected_KL_mean": -338.7437438964844,
"KL/std": 136.2803192138672,
"epoch": 0.8041884816753927,
"fcm_dpo/beta": 0.0028522873762995005,
"fcm_dpo/delta": 0.014860259369015694,
"fcm_dpo/margin": 97.35729217529297,
"fcm_dpo/q_t": 0.4356921911239624,
"grad_norm": 54.56941223144531,
"learning_rate": 5.6928985782982524e-08,
"logits/chosen": -0.8740838766098022,
"logits/rejected": -0.8694276213645935,
"logps/chosen": -515.9564208984375,
"logps/ref_chosen": -274.5699462890625,
"logps/ref_rejected": -285.8253479003906,
"logps/rejected": -624.569091796875,
"loss": 4.7479,
"margin_dpo/margin_mean": 97.35729217529297,
"margin_dpo/margin_std": 170.97972106933594,
"step": 384
},
{
"KL/chosen_KL_mean": -248.646484375,
"KL/mean": -288.6224365234375,
"KL/rejected_KL_mean": -328.598388671875,
"KL/std": 142.26695251464844,
"epoch": 0.806282722513089,
"fcm_dpo/beta": 0.002904432127252221,
"fcm_dpo/delta": 0.019007494673132896,
"fcm_dpo/margin": 79.95191955566406,
"fcm_dpo/q_t": 0.44628220796585083,
"grad_norm": 70.91700744628906,
"learning_rate": 5.57711295439732e-08,
"logits/chosen": -0.8183778524398804,
"logits/rejected": -0.8135970830917358,
"logps/chosen": -532.797119140625,
"logps/ref_chosen": -284.150634765625,
"logps/ref_rejected": -244.87921142578125,
"logps/rejected": -573.4776000976562,
"loss": 4.8742,
"margin_dpo/margin_mean": 79.95191955566406,
"margin_dpo/margin_std": 152.34213256835938,
"step": 385
},
{
"KL/chosen_KL_mean": -239.798095703125,
"KL/mean": -304.68206787109375,
"KL/rejected_KL_mean": -369.5660095214844,
"KL/std": 161.51425170898438,
"epoch": 0.8083769633507853,
"fcm_dpo/beta": 0.0028078884351998568,
"fcm_dpo/delta": -0.05693187564611435,
"fcm_dpo/margin": 129.7678985595703,
"fcm_dpo/q_t": 0.4165182411670685,
"grad_norm": 78.76203155517578,
"learning_rate": 5.4623689209832484e-08,
"logits/chosen": -0.7965095639228821,
"logits/rejected": -0.7943077087402344,
"logps/chosen": -559.974365234375,
"logps/ref_chosen": -320.1762390136719,
"logps/ref_rejected": -302.05023193359375,
"logps/rejected": -671.6162109375,
"loss": 4.503,
"margin_dpo/margin_mean": 129.7678985595703,
"margin_dpo/margin_std": 177.5083770751953,
"step": 386
},
{
"KL/chosen_KL_mean": -255.77442932128906,
"KL/mean": -303.63458251953125,
"KL/rejected_KL_mean": -351.4947204589844,
"KL/std": 153.2172088623047,
"epoch": 0.8104712041884817,
"fcm_dpo/beta": 0.0027528819628059864,
"fcm_dpo/delta": -0.027687324211001396,
"fcm_dpo/margin": 95.72029113769531,
"fcm_dpo/q_t": 0.43756136298179626,
"grad_norm": 80.19580078125,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": -0.8275306224822998,
"logits/rejected": -0.8312669396400452,
"logps/chosen": -528.0545654296875,
"logps/ref_chosen": -272.2801513671875,
"logps/ref_rejected": -265.1615905761719,
"logps/rejected": -616.6563110351562,
"loss": 4.8415,
"margin_dpo/margin_mean": 95.72029113769531,
"margin_dpo/margin_std": 183.86953735351562,
"step": 387
},
{
"KL/chosen_KL_mean": -273.6151123046875,
"KL/mean": -306.04327392578125,
"KL/rejected_KL_mean": -338.471435546875,
"KL/std": 158.35028076171875,
"epoch": 0.812565445026178,
"fcm_dpo/beta": 0.0026982480194419622,
"fcm_dpo/delta": -0.011122354306280613,
"fcm_dpo/margin": 64.85627746582031,
"fcm_dpo/q_t": 0.45950567722320557,
"grad_norm": 84.51703643798828,
"learning_rate": 5.2360301829254745e-08,
"logits/chosen": -0.8161114454269409,
"logits/rejected": -0.8020353317260742,
"logps/chosen": -546.146484375,
"logps/ref_chosen": -272.5313415527344,
"logps/ref_rejected": -239.55735778808594,
"logps/rejected": -578.02880859375,
"loss": 5.1513,
"margin_dpo/margin_mean": 64.85627746582031,
"margin_dpo/margin_std": 188.13427734375,
"step": 388
},
{
"KL/chosen_KL_mean": -266.1781005859375,
"KL/mean": -321.2748718261719,
"KL/rejected_KL_mean": -376.37164306640625,
"KL/std": 165.8982391357422,
"epoch": 0.8146596858638744,
"fcm_dpo/beta": 0.0026822565123438835,
"fcm_dpo/delta": -0.018625523895025253,
"fcm_dpo/margin": 110.19355010986328,
"fcm_dpo/q_t": 0.43248510360717773,
"grad_norm": 62.38093948364258,
"learning_rate": 5.1244476161413806e-08,
"logits/chosen": -0.8523331880569458,
"logits/rejected": -0.8490720987319946,
"logps/chosen": -547.267333984375,
"logps/ref_chosen": -281.0892639160156,
"logps/ref_rejected": -246.50045776367188,
"logps/rejected": -622.8720703125,
"loss": 4.7258,
"margin_dpo/margin_mean": 110.19355773925781,
"margin_dpo/margin_std": 189.8052978515625,
"step": 389
},
{
"KL/chosen_KL_mean": -259.96453857421875,
"KL/mean": -313.5345458984375,
"KL/rejected_KL_mean": -367.10455322265625,
"KL/std": 155.7751922607422,
"epoch": 0.8167539267015707,
"fcm_dpo/beta": 0.0026583157014101744,
"fcm_dpo/delta": -0.00272680283524096,
"fcm_dpo/margin": 107.14004516601562,
"fcm_dpo/q_t": 0.4331052899360657,
"grad_norm": 59.64759063720703,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": -0.8624210357666016,
"logits/rejected": -0.8728106021881104,
"logps/chosen": -543.9520263671875,
"logps/ref_chosen": -283.98748779296875,
"logps/ref_rejected": -283.465087890625,
"logps/rejected": -650.569580078125,
"loss": 4.7094,
"margin_dpo/margin_mean": 107.14004516601562,
"margin_dpo/margin_std": 176.22068786621094,
"step": 390
},
{
"KL/chosen_KL_mean": -267.4212951660156,
"KL/mean": -325.6793518066406,
"KL/rejected_KL_mean": -383.93743896484375,
"KL/std": 176.0823516845703,
"epoch": 0.818848167539267,
"fcm_dpo/beta": 0.002617758931592107,
"fcm_dpo/delta": -0.02285085991024971,
"fcm_dpo/margin": 116.51616668701172,
"fcm_dpo/q_t": 0.42977476119995117,
"grad_norm": 69.12674713134766,
"learning_rate": 4.904486005914027e-08,
"logits/chosen": -0.8110455274581909,
"logits/rejected": -0.7923341393470764,
"logps/chosen": -551.2826538085938,
"logps/ref_chosen": -283.86138916015625,
"logps/ref_rejected": -263.5093688964844,
"logps/rejected": -647.44677734375,
"loss": 4.6727,
"margin_dpo/margin_mean": 116.51615905761719,
"margin_dpo/margin_std": 188.8794708251953,
"step": 391
},
{
"KL/chosen_KL_mean": -254.72386169433594,
"KL/mean": -318.38873291015625,
"KL/rejected_KL_mean": -382.05364990234375,
"KL/std": 156.4735870361328,
"epoch": 0.8209424083769633,
"fcm_dpo/beta": 0.0025636577047407627,
"fcm_dpo/delta": -0.012513198889791965,
"fcm_dpo/margin": 127.32978820800781,
"fcm_dpo/q_t": 0.42234691977500916,
"grad_norm": 87.62789916992188,
"learning_rate": 4.796118758344353e-08,
"logits/chosen": -0.8127277493476868,
"logits/rejected": -0.8246631026268005,
"logps/chosen": -564.7939453125,
"logps/ref_chosen": -310.070068359375,
"logps/ref_rejected": -252.89817810058594,
"logps/rejected": -634.9518432617188,
"loss": 4.5707,
"margin_dpo/margin_mean": 127.32978820800781,
"margin_dpo/margin_std": 177.67172241210938,
"step": 392
},
{
"KL/chosen_KL_mean": -254.2708740234375,
"KL/mean": -301.04620361328125,
"KL/rejected_KL_mean": -347.821533203125,
"KL/std": 150.13577270507812,
"epoch": 0.8230366492146597,
"fcm_dpo/beta": 0.0025904097128659487,
"fcm_dpo/delta": 0.024187462404370308,
"fcm_dpo/margin": 93.55062103271484,
"fcm_dpo/q_t": 0.4434916079044342,
"grad_norm": 52.19322204589844,
"learning_rate": 4.688834983610082e-08,
"logits/chosen": -0.8437351584434509,
"logits/rejected": -0.8238535523414612,
"logps/chosen": -540.9865112304688,
"logps/ref_chosen": -286.7156677246094,
"logps/ref_rejected": -230.00357055664062,
"logps/rejected": -577.8250732421875,
"loss": 4.8433,
"margin_dpo/margin_mean": 93.55061340332031,
"margin_dpo/margin_std": 174.47572326660156,
"step": 393
},
{
"KL/chosen_KL_mean": -257.57318115234375,
"KL/mean": -304.79718017578125,
"KL/rejected_KL_mean": -352.02117919921875,
"KL/std": 179.62074279785156,
"epoch": 0.8251308900523561,
"fcm_dpo/beta": 0.0026478366926312447,
"fcm_dpo/delta": 0.02115212380886078,
"fcm_dpo/margin": 94.44804382324219,
"fcm_dpo/q_t": 0.444901704788208,
"grad_norm": 57.13371658325195,
"learning_rate": 4.582640435014459e-08,
"logits/chosen": -0.8858999609947205,
"logits/rejected": -0.875397801399231,
"logps/chosen": -583.566650390625,
"logps/ref_chosen": -325.9934387207031,
"logps/ref_rejected": -317.42706298828125,
"logps/rejected": -669.4482421875,
"loss": 4.9197,
"margin_dpo/margin_mean": 94.44804382324219,
"margin_dpo/margin_std": 205.414306640625,
"step": 394
},
{
"KL/chosen_KL_mean": -244.72979736328125,
"KL/mean": -310.8471984863281,
"KL/rejected_KL_mean": -376.9646301269531,
"KL/std": 167.30059814453125,
"epoch": 0.8272251308900523,
"fcm_dpo/beta": 0.002669533481821418,
"fcm_dpo/delta": -0.015710730105638504,
"fcm_dpo/margin": 132.23486328125,
"fcm_dpo/q_t": 0.4210633933544159,
"grad_norm": 63.38975524902344,
"learning_rate": 4.477540807448832e-08,
"logits/chosen": -0.8237651586532593,
"logits/rejected": -0.8312820792198181,
"logps/chosen": -513.630615234375,
"logps/ref_chosen": -268.90081787109375,
"logps/ref_rejected": -272.85809326171875,
"logps/rejected": -649.82275390625,
"loss": 4.5299,
"margin_dpo/margin_mean": 132.23486328125,
"margin_dpo/margin_std": 197.66409301757812,
"step": 395
},
{
"KL/chosen_KL_mean": -267.392333984375,
"KL/mean": -313.00030517578125,
"KL/rejected_KL_mean": -358.6082763671875,
"KL/std": 159.95083618164062,
"epoch": 0.8293193717277487,
"fcm_dpo/beta": 0.002658443059772253,
"fcm_dpo/delta": 0.013091212138533592,
"fcm_dpo/margin": 91.2159194946289,
"fcm_dpo/q_t": 0.443788081407547,
"grad_norm": 103.75393676757812,
"learning_rate": 4.373541737087263e-08,
"logits/chosen": -0.833990752696991,
"logits/rejected": -0.8136544823646545,
"logps/chosen": -558.5906372070312,
"logps/ref_chosen": -291.19830322265625,
"logps/ref_rejected": -253.2803955078125,
"logps/rejected": -611.888671875,
"loss": 4.8872,
"margin_dpo/margin_mean": 91.2159194946289,
"margin_dpo/margin_std": 185.2631072998047,
"step": 396
},
{
"KL/chosen_KL_mean": -252.65269470214844,
"KL/mean": -297.8766174316406,
"KL/rejected_KL_mean": -343.1005859375,
"KL/std": 157.75360107421875,
"epoch": 0.831413612565445,
"fcm_dpo/beta": 0.002626030705869198,
"fcm_dpo/delta": -0.026181943714618683,
"fcm_dpo/margin": 90.44788360595703,
"fcm_dpo/q_t": 0.4456873834133148,
"grad_norm": 70.29791259765625,
"learning_rate": 4.270648801084295e-08,
"logits/chosen": -0.8465428352355957,
"logits/rejected": -0.8269012570381165,
"logps/chosen": -562.47509765625,
"logps/ref_chosen": -309.8224182128906,
"logps/ref_rejected": -291.9057922363281,
"logps/rejected": -635.00634765625,
"loss": 4.9391,
"margin_dpo/margin_mean": 90.44788360595703,
"margin_dpo/margin_std": 190.6138153076172,
"step": 397
},
{
"KL/chosen_KL_mean": -283.8010559082031,
"KL/mean": -324.195068359375,
"KL/rejected_KL_mean": -364.5890197753906,
"KL/std": 173.2881317138672,
"epoch": 0.8335078534031414,
"fcm_dpo/beta": 0.0025994465686380863,
"fcm_dpo/delta": -0.006987897679209709,
"fcm_dpo/margin": 80.7879638671875,
"fcm_dpo/q_t": 0.44956809282302856,
"grad_norm": 79.59461212158203,
"learning_rate": 4.168867517275806e-08,
"logits/chosen": -0.7517906427383423,
"logits/rejected": -0.7874580025672913,
"logps/chosen": -581.6146240234375,
"logps/ref_chosen": -297.8135070800781,
"logps/ref_rejected": -270.5025634765625,
"logps/rejected": -635.0916137695312,
"loss": 5.1179,
"margin_dpo/margin_mean": 80.78797149658203,
"margin_dpo/margin_std": 224.45187377929688,
"step": 398
},
{
"KL/chosen_KL_mean": -264.94573974609375,
"KL/mean": -315.5485534667969,
"KL/rejected_KL_mean": -366.15130615234375,
"KL/std": 161.73268127441406,
"epoch": 0.8356020942408376,
"fcm_dpo/beta": 0.0026253818068653345,
"fcm_dpo/delta": 0.022293759509921074,
"fcm_dpo/margin": 101.20550537109375,
"fcm_dpo/q_t": 0.4391327202320099,
"grad_norm": 73.089111328125,
"learning_rate": 4.0682033438831584e-08,
"logits/chosen": -0.8615554571151733,
"logits/rejected": -0.8228447437286377,
"logps/chosen": -557.7925415039062,
"logps/ref_chosen": -292.8467712402344,
"logps/ref_rejected": -268.3638916015625,
"logps/rejected": -634.51513671875,
"loss": 4.8136,
"margin_dpo/margin_mean": 101.20550537109375,
"margin_dpo/margin_std": 190.82528686523438,
"step": 399
},
{
"KL/chosen_KL_mean": -255.96389770507812,
"KL/mean": -306.29583740234375,
"KL/rejected_KL_mean": -356.6277770996094,
"KL/std": 157.36746215820312,
"epoch": 0.837696335078534,
"fcm_dpo/beta": 0.002646203152835369,
"fcm_dpo/delta": 0.005138233304023743,
"fcm_dpo/margin": 100.66392517089844,
"fcm_dpo/q_t": 0.43769270181655884,
"grad_norm": 51.399471282958984,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": -0.8845930695533752,
"logits/rejected": -0.8838775753974915,
"logps/chosen": -519.6402587890625,
"logps/ref_chosen": -263.6763916015625,
"logps/ref_rejected": -258.67266845703125,
"logps/rejected": -615.3004760742188,
"loss": 4.8145,
"margin_dpo/margin_mean": 100.66392517089844,
"margin_dpo/margin_std": 187.87037658691406,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_KL/chosen_KL_mean": -259.0677185058594,
"eval_KL/mean": -311.7682800292969,
"eval_KL/rejected_KL_mean": -364.4688720703125,
"eval_KL/std": 162.36822509765625,
"eval_fcm_dpo/beta": 0.0026530208997428417,
"eval_logits/chosen": -0.8539494872093201,
"eval_logits/rejected": -0.8364132642745972,
"eval_logps/chosen": -546.8944702148438,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -631.3988647460938,
"eval_loss": 0.5982628464698792,
"eval_margin_dpo/margin_mean": 105.40113830566406,
"eval_margin_dpo/margin_std": 182.74659729003906,
"eval_runtime": 78.7931,
"eval_samples_per_second": 25.383,
"eval_steps_per_second": 3.173,
"step": 400
},
{
"KL/chosen_KL_mean": -256.0751953125,
"KL/mean": -309.08087158203125,
"KL/rejected_KL_mean": -362.0865478515625,
"KL/std": 163.9885711669922,
"epoch": 0.8397905759162304,
"fcm_dpo/beta": 0.002689874265342951,
"fcm_dpo/delta": 0.024891149252653122,
"fcm_dpo/margin": 106.01134490966797,
"fcm_dpo/q_t": 0.4338783919811249,
"grad_norm": 69.87895202636719,
"learning_rate": 3.8702478614051345e-08,
"logits/chosen": -0.83061283826828,
"logits/rejected": -0.8284586668014526,
"logps/chosen": -574.3605346679688,
"logps/ref_chosen": -318.2853088378906,
"logps/ref_rejected": -293.75225830078125,
"logps/rejected": -655.8388061523438,
"loss": 4.7405,
"margin_dpo/margin_mean": 106.01134490966797,
"margin_dpo/margin_std": 185.73927307128906,
"step": 401
},
{
"KL/chosen_KL_mean": -247.47999572753906,
"KL/mean": -295.99700927734375,
"KL/rejected_KL_mean": -344.5140075683594,
"KL/std": 166.7805938720703,
"epoch": 0.8418848167539267,
"fcm_dpo/beta": 0.0027063796296715736,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 97.03401184082031,
"fcm_dpo/q_t": 0.4415395259857178,
"grad_norm": 67.62411499023438,
"learning_rate": 3.772967168071517e-08,
"logits/chosen": -0.8851345181465149,
"logits/rejected": -0.8621891736984253,
"logps/chosen": -556.9078369140625,
"logps/ref_chosen": -309.4278564453125,
"logps/ref_rejected": -282.0279846191406,
"logps/rejected": -626.5419921875,
"loss": 4.8518,
"margin_dpo/margin_mean": 97.03401184082031,
"margin_dpo/margin_std": 202.4774169921875,
"step": 402
},
{
"KL/chosen_KL_mean": -236.76856994628906,
"KL/mean": -308.6864318847656,
"KL/rejected_KL_mean": -380.604248046875,
"KL/std": 171.2498779296875,
"epoch": 0.8439790575916231,
"fcm_dpo/beta": 0.002709330292418599,
"fcm_dpo/delta": -0.025865260511636734,
"fcm_dpo/margin": 143.83566284179688,
"fcm_dpo/q_t": 0.41079238057136536,
"grad_norm": 60.24233627319336,
"learning_rate": 3.676824816087978e-08,
"logits/chosen": -0.8610972166061401,
"logits/rejected": -0.835875928401947,
"logps/chosen": -545.7970581054688,
"logps/ref_chosen": -309.0284729003906,
"logps/ref_rejected": -272.9622497558594,
"logps/rejected": -653.5665283203125,
"loss": 4.3735,
"margin_dpo/margin_mean": 143.83567810058594,
"margin_dpo/margin_std": 179.44992065429688,
"step": 403
},
{
"KL/chosen_KL_mean": -261.0257873535156,
"KL/mean": -315.1200256347656,
"KL/rejected_KL_mean": -369.2142028808594,
"KL/std": 173.9381866455078,
"epoch": 0.8460732984293193,
"fcm_dpo/beta": 0.002677470212802291,
"fcm_dpo/delta": 0.0036769555881619453,
"fcm_dpo/margin": 108.18839263916016,
"fcm_dpo/q_t": 0.4339839220046997,
"grad_norm": 78.65907287597656,
"learning_rate": 3.581825961277074e-08,
"logits/chosen": -0.8783624172210693,
"logits/rejected": -0.8541516661643982,
"logps/chosen": -558.3095703125,
"logps/ref_chosen": -297.2837219238281,
"logps/ref_rejected": -256.99041748046875,
"logps/rejected": -626.20458984375,
"loss": 4.7808,
"margin_dpo/margin_mean": 108.18839263916016,
"margin_dpo/margin_std": 206.70828247070312,
"step": 404
},
{
"KL/chosen_KL_mean": -249.36727905273438,
"KL/mean": -309.5423889160156,
"KL/rejected_KL_mean": -369.7174987792969,
"KL/std": 169.79800415039062,
"epoch": 0.8481675392670157,
"fcm_dpo/beta": 0.002700282959267497,
"fcm_dpo/delta": 0.026766503229737282,
"fcm_dpo/margin": 120.3502197265625,
"fcm_dpo/q_t": 0.42523056268692017,
"grad_norm": 61.90435028076172,
"learning_rate": 3.487975698139084e-08,
"logits/chosen": -0.8048846125602722,
"logits/rejected": -0.8137521147727966,
"logps/chosen": -507.3326110839844,
"logps/ref_chosen": -257.96533203125,
"logps/ref_rejected": -255.811279296875,
"logps/rejected": -625.52880859375,
"loss": 4.6128,
"margin_dpo/margin_mean": 120.3502197265625,
"margin_dpo/margin_std": 191.45315551757812,
"step": 405
},
{
"KL/chosen_KL_mean": -269.34295654296875,
"KL/mean": -309.04095458984375,
"KL/rejected_KL_mean": -348.739013671875,
"KL/std": 154.44068908691406,
"epoch": 0.8502617801047121,
"fcm_dpo/beta": 0.002717172261327505,
"fcm_dpo/delta": 0.003170239506289363,
"fcm_dpo/margin": 79.39606475830078,
"fcm_dpo/q_t": 0.4483673572540283,
"grad_norm": 113.7933349609375,
"learning_rate": 3.3952790595787986e-08,
"logits/chosen": -0.8405861258506775,
"logits/rejected": -0.8171727061271667,
"logps/chosen": -554.5239868164062,
"logps/ref_chosen": -285.1810607910156,
"logps/ref_rejected": -264.41351318359375,
"logps/rejected": -613.1525268554688,
"loss": 4.9484,
"margin_dpo/margin_mean": 79.39606475830078,
"margin_dpo/margin_std": 170.13916015625,
"step": 406
},
{
"KL/chosen_KL_mean": -265.46820068359375,
"KL/mean": -321.43084716796875,
"KL/rejected_KL_mean": -377.3935546875,
"KL/std": 168.72860717773438,
"epoch": 0.8523560209424084,
"fcm_dpo/beta": 0.0027630478143692017,
"fcm_dpo/delta": 0.02725648693740368,
"fcm_dpo/margin": 111.92539978027344,
"fcm_dpo/q_t": 0.4298943281173706,
"grad_norm": 55.85504913330078,
"learning_rate": 3.303741016635614e-08,
"logits/chosen": -0.8258509635925293,
"logits/rejected": -0.8421826362609863,
"logps/chosen": -530.706298828125,
"logps/ref_chosen": -265.23809814453125,
"logps/ref_rejected": -219.0631561279297,
"logps/rejected": -596.4567260742188,
"loss": 4.7059,
"margin_dpo/margin_mean": 111.9253921508789,
"margin_dpo/margin_std": 197.7037353515625,
"step": 407
},
{
"KL/chosen_KL_mean": -247.27427673339844,
"KL/mean": -302.8316345214844,
"KL/rejected_KL_mean": -358.3890686035156,
"KL/std": 157.02703857421875,
"epoch": 0.8544502617801047,
"fcm_dpo/beta": 0.0027716769836843014,
"fcm_dpo/delta": -0.04041987285017967,
"fcm_dpo/margin": 111.11476135253906,
"fcm_dpo/q_t": 0.42840081453323364,
"grad_norm": 79.1960678100586,
"learning_rate": 3.2133664782169944e-08,
"logits/chosen": -0.8630377054214478,
"logits/rejected": -0.8538559079170227,
"logps/chosen": -544.2468872070312,
"logps/ref_chosen": -296.9726257324219,
"logps/ref_rejected": -295.4786376953125,
"logps/rejected": -653.8676147460938,
"loss": 4.6593,
"margin_dpo/margin_mean": 111.11476135253906,
"margin_dpo/margin_std": 172.04241943359375,
"step": 408
},
{
"KL/chosen_KL_mean": -260.24188232421875,
"KL/mean": -322.407958984375,
"KL/rejected_KL_mean": -384.5740051269531,
"KL/std": 207.52333068847656,
"epoch": 0.856544502617801,
"fcm_dpo/beta": 0.002639027778059244,
"fcm_dpo/delta": -0.04701896011829376,
"fcm_dpo/margin": 124.33211517333984,
"fcm_dpo/q_t": 0.4329832196235657,
"grad_norm": 53.74481201171875,
"learning_rate": 3.12416029083514e-08,
"logits/chosen": -0.839493989944458,
"logits/rejected": -0.8232097029685974,
"logps/chosen": -547.6212768554688,
"logps/ref_chosen": -287.37933349609375,
"logps/ref_rejected": -275.80291748046875,
"logps/rejected": -660.3768310546875,
"loss": 4.759,
"margin_dpo/margin_mean": 124.33211517333984,
"margin_dpo/margin_std": 254.77313232421875,
"step": 409
},
{
"KL/chosen_KL_mean": -251.41685485839844,
"KL/mean": -302.3774108886719,
"KL/rejected_KL_mean": -353.33795166015625,
"KL/std": 163.56126403808594,
"epoch": 0.8586387434554974,
"fcm_dpo/beta": 0.0026220958679914474,
"fcm_dpo/delta": 0.018517207354307175,
"fcm_dpo/margin": 101.92111206054688,
"fcm_dpo/q_t": 0.43905141949653625,
"grad_norm": 59.67084884643555,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": -0.8541938662528992,
"logits/rejected": -0.8597331047058105,
"logps/chosen": -533.197021484375,
"logps/ref_chosen": -281.7801818847656,
"logps/ref_rejected": -266.7550354003906,
"logps/rejected": -620.093017578125,
"loss": 4.8163,
"margin_dpo/margin_mean": 101.92111206054688,
"margin_dpo/margin_std": 195.57972717285156,
"step": 410
},
{
"KL/chosen_KL_mean": -244.28140258789062,
"KL/mean": -301.5091857910156,
"KL/rejected_KL_mean": -358.7369079589844,
"KL/std": 168.92539978027344,
"epoch": 0.8607329842931937,
"fcm_dpo/beta": 0.0026748264208436012,
"fcm_dpo/delta": 0.0014014113694429398,
"fcm_dpo/margin": 114.45552825927734,
"fcm_dpo/q_t": 0.4303073585033417,
"grad_norm": 85.47948455810547,
"learning_rate": 2.9492720416985e-08,
"logits/chosen": -0.8583596348762512,
"logits/rejected": -0.8106494545936584,
"logps/chosen": -525.86865234375,
"logps/ref_chosen": -281.5872497558594,
"logps/ref_rejected": -254.78916931152344,
"logps/rejected": -613.526123046875,
"loss": 4.6366,
"margin_dpo/margin_mean": 114.45552825927734,
"margin_dpo/margin_std": 170.4075927734375,
"step": 411
},
{
"KL/chosen_KL_mean": -273.9906921386719,
"KL/mean": -319.8974304199219,
"KL/rejected_KL_mean": -365.8041687011719,
"KL/std": 163.98312377929688,
"epoch": 0.86282722513089,
"fcm_dpo/beta": 0.002576531609520316,
"fcm_dpo/delta": -0.026781653985381126,
"fcm_dpo/margin": 91.8134994506836,
"fcm_dpo/q_t": 0.44552770256996155,
"grad_norm": 70.79202270507812,
"learning_rate": 2.863599358669755e-08,
"logits/chosen": -0.8241173624992371,
"logits/rejected": -0.8364276885986328,
"logps/chosen": -550.5248413085938,
"logps/ref_chosen": -276.5341796875,
"logps/ref_rejected": -273.8751220703125,
"logps/rejected": -639.6793212890625,
"loss": 4.9052,
"margin_dpo/margin_mean": 91.8134994506836,
"margin_dpo/margin_std": 183.9867401123047,
"step": 412
},
{
"KL/chosen_KL_mean": -252.1434783935547,
"KL/mean": -297.56536865234375,
"KL/rejected_KL_mean": -342.98724365234375,
"KL/std": 162.1449737548828,
"epoch": 0.8649214659685864,
"fcm_dpo/beta": 0.0025809926446527243,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.84378051757812,
"fcm_dpo/q_t": 0.44537127017974854,
"grad_norm": 60.38282775878906,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": -0.840187132358551,
"logits/rejected": -0.8357990384101868,
"logps/chosen": -523.4180297851562,
"logps/ref_chosen": -271.2745666503906,
"logps/ref_rejected": -270.16912841796875,
"logps/rejected": -613.1563720703125,
"loss": 4.8861,
"margin_dpo/margin_mean": 90.84378051757812,
"margin_dpo/margin_std": 185.5474395751953,
"step": 413
},
{
"KL/chosen_KL_mean": -254.0613250732422,
"KL/mean": -316.70928955078125,
"KL/rejected_KL_mean": -379.35723876953125,
"KL/std": 163.72683715820312,
"epoch": 0.8670157068062827,
"fcm_dpo/beta": 0.0025707168970257044,
"fcm_dpo/delta": -0.01698092371225357,
"fcm_dpo/margin": 125.29594421386719,
"fcm_dpo/q_t": 0.4260947108268738,
"grad_norm": 62.23833084106445,
"learning_rate": 2.6958198472749717e-08,
"logits/chosen": -0.8793128728866577,
"logits/rejected": -0.872239351272583,
"logps/chosen": -551.1763305664062,
"logps/ref_chosen": -297.11505126953125,
"logps/ref_rejected": -271.7034606933594,
"logps/rejected": -651.060791015625,
"loss": 4.6003,
"margin_dpo/margin_mean": 125.29594421386719,
"margin_dpo/margin_std": 187.06222534179688,
"step": 414
},
{
"KL/chosen_KL_mean": -251.2029266357422,
"KL/mean": -303.4581604003906,
"KL/rejected_KL_mean": -355.71331787109375,
"KL/std": 149.83013916015625,
"epoch": 0.8691099476439791,
"fcm_dpo/beta": 0.0025737180840224028,
"fcm_dpo/delta": 0.01915055885910988,
"fcm_dpo/margin": 104.51042938232422,
"fcm_dpo/q_t": 0.4367385804653168,
"grad_norm": 63.56855773925781,
"learning_rate": 2.613722016414943e-08,
"logits/chosen": -0.871803343296051,
"logits/rejected": -0.8581423163414001,
"logps/chosen": -548.8956298828125,
"logps/ref_chosen": -297.6926574707031,
"logps/ref_rejected": -279.0503234863281,
"logps/rejected": -634.7637329101562,
"loss": 4.7351,
"margin_dpo/margin_mean": 104.51043701171875,
"margin_dpo/margin_std": 170.8563995361328,
"step": 415
},
{
"KL/chosen_KL_mean": -236.1012725830078,
"KL/mean": -297.1469421386719,
"KL/rejected_KL_mean": -358.192626953125,
"KL/std": 151.40481567382812,
"epoch": 0.8712041884816754,
"fcm_dpo/beta": 0.002560949884355068,
"fcm_dpo/delta": -5.1200266170781106e-05,
"fcm_dpo/margin": 122.09134674072266,
"fcm_dpo/q_t": 0.42640596628189087,
"grad_norm": 39.975067138671875,
"learning_rate": 2.5328246937043525e-08,
"logits/chosen": -0.8686580061912537,
"logits/rejected": -0.8653780817985535,
"logps/chosen": -547.9268188476562,
"logps/ref_chosen": -311.8255615234375,
"logps/ref_rejected": -268.6170654296875,
"logps/rejected": -626.8096923828125,
"loss": 4.6017,
"margin_dpo/margin_mean": 122.09133911132812,
"margin_dpo/margin_std": 176.49038696289062,
"step": 416
},
{
"KL/chosen_KL_mean": -256.92279052734375,
"KL/mean": -309.4592590332031,
"KL/rejected_KL_mean": -361.9957275390625,
"KL/std": 169.861083984375,
"epoch": 0.8732984293193717,
"fcm_dpo/beta": 0.0025655170902609825,
"fcm_dpo/delta": -0.02516508474946022,
"fcm_dpo/margin": 105.07295227050781,
"fcm_dpo/q_t": 0.43865251541137695,
"grad_norm": 58.286231994628906,
"learning_rate": 2.4531322174210973e-08,
"logits/chosen": -0.824512779712677,
"logits/rejected": -0.8112680315971375,
"logps/chosen": -567.3595581054688,
"logps/ref_chosen": -310.43682861328125,
"logps/ref_rejected": -277.15283203125,
"logps/rejected": -639.1485595703125,
"loss": 4.8026,
"margin_dpo/margin_mean": 105.07294464111328,
"margin_dpo/margin_std": 191.708740234375,
"step": 417
},
{
"KL/chosen_KL_mean": -271.6235046386719,
"KL/mean": -324.97332763671875,
"KL/rejected_KL_mean": -378.3231506347656,
"KL/std": 181.0856170654297,
"epoch": 0.875392670157068,
"fcm_dpo/beta": 0.0025076887104660273,
"fcm_dpo/delta": -0.01675930619239807,
"fcm_dpo/margin": 106.69969177246094,
"fcm_dpo/q_t": 0.43919453024864197,
"grad_norm": 80.68965911865234,
"learning_rate": 2.3746488612308295e-08,
"logits/chosen": -0.8077826499938965,
"logits/rejected": -0.7858579158782959,
"logps/chosen": -550.1194458007812,
"logps/ref_chosen": -278.49591064453125,
"logps/ref_rejected": -276.56671142578125,
"logps/rejected": -654.889892578125,
"loss": 4.8247,
"margin_dpo/margin_mean": 106.69967651367188,
"margin_dpo/margin_std": 208.5244598388672,
"step": 418
},
{
"KL/chosen_KL_mean": -267.290771484375,
"KL/mean": -320.2489013671875,
"KL/rejected_KL_mean": -373.2070007324219,
"KL/std": 170.7522430419922,
"epoch": 0.8774869109947644,
"fcm_dpo/beta": 0.0025136300828307867,
"fcm_dpo/delta": 0.009540699422359467,
"fcm_dpo/margin": 105.91624450683594,
"fcm_dpo/q_t": 0.4366569519042969,
"grad_norm": 75.40731048583984,
"learning_rate": 2.297378833957761e-08,
"logits/chosen": -0.8613269329071045,
"logits/rejected": -0.8316982388496399,
"logps/chosen": -566.1910400390625,
"logps/ref_chosen": -298.9002380371094,
"logps/ref_rejected": -246.1540985107422,
"logps/rejected": -619.3611450195312,
"loss": 4.8257,
"margin_dpo/margin_mean": 105.91624450683594,
"margin_dpo/margin_std": 208.2015838623047,
"step": 419
},
{
"KL/chosen_KL_mean": -260.46661376953125,
"KL/mean": -325.86968994140625,
"KL/rejected_KL_mean": -391.2727966308594,
"KL/std": 182.0087432861328,
"epoch": 0.8795811518324608,
"fcm_dpo/beta": 0.0025134901516139507,
"fcm_dpo/delta": -0.011981809511780739,
"fcm_dpo/margin": 130.80618286132812,
"fcm_dpo/q_t": 0.4250591993331909,
"grad_norm": 66.69501495361328,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": -0.8003321290016174,
"logits/rejected": -0.771695613861084,
"logps/chosen": -525.0275268554688,
"logps/ref_chosen": -264.5608825683594,
"logps/ref_rejected": -245.67031860351562,
"logps/rejected": -636.9431762695312,
"loss": 4.6755,
"margin_dpo/margin_mean": 130.80618286132812,
"margin_dpo/margin_std": 231.65260314941406,
"step": 420
},
{
"KL/chosen_KL_mean": -239.69073486328125,
"KL/mean": -296.38812255859375,
"KL/rejected_KL_mean": -353.08551025390625,
"KL/std": 154.60948181152344,
"epoch": 0.881675392670157,
"fcm_dpo/beta": 0.002471539657562971,
"fcm_dpo/delta": -0.02126682549715042,
"fcm_dpo/margin": 113.39478302001953,
"fcm_dpo/q_t": 0.4333392381668091,
"grad_norm": 67.4413070678711,
"learning_rate": 2.1464952759020856e-08,
"logits/chosen": -0.8876159191131592,
"logits/rejected": -0.8631241917610168,
"logps/chosen": -537.395751953125,
"logps/ref_chosen": -297.70501708984375,
"logps/ref_rejected": -243.74771118164062,
"logps/rejected": -596.8331909179688,
"loss": 4.7061,
"margin_dpo/margin_mean": 113.394775390625,
"margin_dpo/margin_std": 176.9335479736328,
"step": 421
},
{
"KL/chosen_KL_mean": -253.8042449951172,
"KL/mean": -306.0145263671875,
"KL/rejected_KL_mean": -358.22479248046875,
"KL/std": 167.76966857910156,
"epoch": 0.8837696335078534,
"fcm_dpo/beta": 0.002462256234139204,
"fcm_dpo/delta": 0.015522689558565617,
"fcm_dpo/margin": 104.4205322265625,
"fcm_dpo/q_t": 0.44079747796058655,
"grad_norm": 51.60731887817383,
"learning_rate": 2.07288983654679e-08,
"logits/chosen": -0.7430369257926941,
"logits/rejected": -0.7820625901222229,
"logps/chosen": -542.1630249023438,
"logps/ref_chosen": -288.3587646484375,
"logps/ref_rejected": -256.4377746582031,
"logps/rejected": -614.66259765625,
"loss": 4.805,
"margin_dpo/margin_mean": 104.42054748535156,
"margin_dpo/margin_std": 192.31918334960938,
"step": 422
},
{
"KL/chosen_KL_mean": -258.586181640625,
"KL/mean": -319.3099670410156,
"KL/rejected_KL_mean": -380.03375244140625,
"KL/std": 160.17820739746094,
"epoch": 0.8858638743455497,
"fcm_dpo/beta": 0.0024836647789925337,
"fcm_dpo/delta": 0.003166984301060438,
"fcm_dpo/margin": 121.4476318359375,
"fcm_dpo/q_t": 0.42924964427948,
"grad_norm": 62.990482330322266,
"learning_rate": 2.0005139085293942e-08,
"logits/chosen": -0.8718672394752502,
"logits/rejected": -0.849821925163269,
"logps/chosen": -554.5932006835938,
"logps/ref_chosen": -296.00701904296875,
"logps/ref_rejected": -261.3480529785156,
"logps/rejected": -641.3818359375,
"loss": 4.6526,
"margin_dpo/margin_mean": 121.44761657714844,
"margin_dpo/margin_std": 191.81884765625,
"step": 423
},
{
"KL/chosen_KL_mean": -238.3405303955078,
"KL/mean": -292.3135986328125,
"KL/rejected_KL_mean": -346.28668212890625,
"KL/std": 150.8682098388672,
"epoch": 0.8879581151832461,
"fcm_dpo/beta": 0.0024709829594939947,
"fcm_dpo/delta": -0.0036196252331137657,
"fcm_dpo/margin": 107.9461669921875,
"fcm_dpo/q_t": 0.4369615614414215,
"grad_norm": 80.2655258178711,
"learning_rate": 1.9293713731512673e-08,
"logits/chosen": -0.8483703136444092,
"logits/rejected": -0.8329979777336121,
"logps/chosen": -547.7623901367188,
"logps/ref_chosen": -309.421875,
"logps/ref_rejected": -249.14886474609375,
"logps/rejected": -595.4356079101562,
"loss": 4.734,
"margin_dpo/margin_mean": 107.9461669921875,
"margin_dpo/margin_std": 167.56033325195312,
"step": 424
},
{
"KL/chosen_KL_mean": -247.47341918945312,
"KL/mean": -297.63507080078125,
"KL/rejected_KL_mean": -347.79669189453125,
"KL/std": 160.0552520751953,
"epoch": 0.8900523560209425,
"fcm_dpo/beta": 0.0024773648474365473,
"fcm_dpo/delta": -0.0067837778478860855,
"fcm_dpo/margin": 100.32325744628906,
"fcm_dpo/q_t": 0.44239264726638794,
"grad_norm": 47.60661697387695,
"learning_rate": 1.8594660455706763e-08,
"logits/chosen": -0.8335504531860352,
"logits/rejected": -0.8358312249183655,
"logps/chosen": -527.9825439453125,
"logps/ref_chosen": -280.50909423828125,
"logps/ref_rejected": -276.8252258300781,
"logps/rejected": -624.6218872070312,
"loss": 4.8279,
"margin_dpo/margin_mean": 100.32325744628906,
"margin_dpo/margin_std": 179.0926971435547,
"step": 425
},
{
"KL/chosen_KL_mean": -253.93557739257812,
"KL/mean": -309.0635986328125,
"KL/rejected_KL_mean": -364.19158935546875,
"KL/std": 163.70797729492188,
"epoch": 0.8921465968586387,
"fcm_dpo/beta": 0.002501498442143202,
"fcm_dpo/delta": 0.027417544275522232,
"fcm_dpo/margin": 110.25602722167969,
"fcm_dpo/q_t": 0.4352967441082001,
"grad_norm": 61.759361267089844,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": -0.8617511987686157,
"logits/rejected": -0.8399728536605835,
"logps/chosen": -546.7207641601562,
"logps/ref_chosen": -292.78521728515625,
"logps/ref_rejected": -255.62698364257812,
"logps/rejected": -619.8185424804688,
"loss": 4.733,
"margin_dpo/margin_mean": 110.25602722167969,
"margin_dpo/margin_std": 181.19354248046875,
"step": 426
},
{
"KL/chosen_KL_mean": -242.61668395996094,
"KL/mean": -321.30316162109375,
"KL/rejected_KL_mean": -399.9896240234375,
"KL/std": 182.28509521484375,
"epoch": 0.8942408376963351,
"fcm_dpo/beta": 0.002506428863853216,
"fcm_dpo/delta": -0.04454171657562256,
"fcm_dpo/margin": 157.37294006347656,
"fcm_dpo/q_t": 0.40898868441581726,
"grad_norm": 74.9897232055664,
"learning_rate": 1.7233819424956247e-08,
"logits/chosen": -0.8332107663154602,
"logits/rejected": -0.7955859303474426,
"logps/chosen": -531.3854370117188,
"logps/ref_chosen": -288.7687072753906,
"logps/ref_rejected": -268.4986572265625,
"logps/rejected": -668.48828125,
"loss": 4.4275,
"margin_dpo/margin_mean": 157.37295532226562,
"margin_dpo/margin_std": 218.2144775390625,
"step": 427
},
{
"KL/chosen_KL_mean": -274.7869873046875,
"KL/mean": -343.9637451171875,
"KL/rejected_KL_mean": -413.14044189453125,
"KL/std": 183.8684539794922,
"epoch": 0.8963350785340314,
"fcm_dpo/beta": 0.0024281598161906004,
"fcm_dpo/delta": -0.008501682430505753,
"fcm_dpo/margin": 138.35345458984375,
"fcm_dpo/q_t": 0.42190542817115784,
"grad_norm": 59.844512939453125,
"learning_rate": 1.6572104647786245e-08,
"logits/chosen": -0.791337788105011,
"logits/rejected": -0.8210979700088501,
"logps/chosen": -570.3079833984375,
"logps/ref_chosen": -295.5209655761719,
"logps/ref_rejected": -275.71026611328125,
"logps/rejected": -688.8507080078125,
"loss": 4.5482,
"margin_dpo/margin_mean": 138.35345458984375,
"margin_dpo/margin_std": 195.07057189941406,
"step": 428
},
{
"KL/chosen_KL_mean": -264.47540283203125,
"KL/mean": -318.24371337890625,
"KL/rejected_KL_mean": -372.01202392578125,
"KL/std": 177.7375946044922,
"epoch": 0.8984293193717278,
"fcm_dpo/beta": 0.0024204240180552006,
"fcm_dpo/delta": 0.012468225322663784,
"fcm_dpo/margin": 107.53662109375,
"fcm_dpo/q_t": 0.4388344883918762,
"grad_norm": 59.98493194580078,
"learning_rate": 1.5922907900227017e-08,
"logits/chosen": -0.7992820739746094,
"logits/rejected": -0.8066202402114868,
"logps/chosen": -538.8677368164062,
"logps/ref_chosen": -274.392333984375,
"logps/ref_rejected": -258.574462890625,
"logps/rejected": -630.5865478515625,
"loss": 4.862,
"margin_dpo/margin_mean": 107.53662109375,
"margin_dpo/margin_std": 216.66375732421875,
"step": 429
},
{
"KL/chosen_KL_mean": -253.07960510253906,
"KL/mean": -308.39447021484375,
"KL/rejected_KL_mean": -363.7093505859375,
"KL/std": 166.74659729003906,
"epoch": 0.900523560209424,
"fcm_dpo/beta": 0.0024694507010281086,
"fcm_dpo/delta": 0.014632913284003735,
"fcm_dpo/margin": 110.62974548339844,
"fcm_dpo/q_t": 0.43679171800613403,
"grad_norm": 72.91577911376953,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": -0.8794166445732117,
"logits/rejected": -0.847719132900238,
"logps/chosen": -541.8187255859375,
"logps/ref_chosen": -288.7391357421875,
"logps/ref_rejected": -268.6106262207031,
"logps/rejected": -632.3199462890625,
"loss": 4.756,
"margin_dpo/margin_mean": 110.62974548339844,
"margin_dpo/margin_std": 189.39857482910156,
"step": 430
},
{
"KL/chosen_KL_mean": -250.58224487304688,
"KL/mean": -295.65850830078125,
"KL/rejected_KL_mean": -340.7347106933594,
"KL/std": 156.20620727539062,
"epoch": 0.9026178010471204,
"fcm_dpo/beta": 0.002479498041793704,
"fcm_dpo/delta": -0.0046107154339551926,
"fcm_dpo/margin": 90.15248107910156,
"fcm_dpo/q_t": 0.447639524936676,
"grad_norm": 50.818077087402344,
"learning_rate": 1.4662207078575684e-08,
"logits/chosen": -0.853556215763092,
"logits/rejected": -0.8220837712287903,
"logps/chosen": -526.3070068359375,
"logps/ref_chosen": -275.7247314453125,
"logps/ref_rejected": -268.91729736328125,
"logps/rejected": -609.6519775390625,
"loss": 4.9115,
"margin_dpo/margin_mean": 90.1524658203125,
"margin_dpo/margin_std": 178.93893432617188,
"step": 431
},
{
"KL/chosen_KL_mean": -233.45767211914062,
"KL/mean": -291.3896484375,
"KL/rejected_KL_mean": -349.32171630859375,
"KL/std": 166.04766845703125,
"epoch": 0.9047120418848168,
"fcm_dpo/beta": 0.0024858207907527685,
"fcm_dpo/delta": 0.01790335588157177,
"fcm_dpo/margin": 115.8640365600586,
"fcm_dpo/q_t": 0.4340059757232666,
"grad_norm": 57.34628677368164,
"learning_rate": 1.40507706120426e-08,
"logits/chosen": -0.887722373008728,
"logits/rejected": -0.8687385320663452,
"logps/chosen": -524.8777465820312,
"logps/ref_chosen": -291.42010498046875,
"logps/ref_rejected": -255.48202514648438,
"logps/rejected": -604.8037109375,
"loss": 4.7118,
"margin_dpo/margin_mean": 115.8640365600586,
"margin_dpo/margin_std": 193.5400390625,
"step": 432
},
{
"KL/chosen_KL_mean": -264.11077880859375,
"KL/mean": -312.563720703125,
"KL/rejected_KL_mean": -361.0166931152344,
"KL/std": 158.07310485839844,
"epoch": 0.9068062827225131,
"fcm_dpo/beta": 0.002510627033188939,
"fcm_dpo/delta": 0.007871107198297977,
"fcm_dpo/margin": 96.9058609008789,
"fcm_dpo/q_t": 0.44348740577697754,
"grad_norm": 75.00899505615234,
"learning_rate": 1.345198738661285e-08,
"logits/chosen": -0.8320577144622803,
"logits/rejected": -0.8262537717819214,
"logps/chosen": -510.33770751953125,
"logps/ref_chosen": -246.2268829345703,
"logps/ref_rejected": -253.65924072265625,
"logps/rejected": -614.6759033203125,
"loss": 4.8285,
"margin_dpo/margin_mean": 96.9058609008789,
"margin_dpo/margin_std": 178.91293334960938,
"step": 433
},
{
"KL/chosen_KL_mean": -274.1569519042969,
"KL/mean": -325.5293884277344,
"KL/rejected_KL_mean": -376.90179443359375,
"KL/std": 164.99172973632812,
"epoch": 0.9089005235602095,
"fcm_dpo/beta": 0.002546996111050248,
"fcm_dpo/delta": 0.02380567416548729,
"fcm_dpo/margin": 102.74482727050781,
"fcm_dpo/q_t": 0.43898072838783264,
"grad_norm": 74.7905044555664,
"learning_rate": 1.2865889513213628e-08,
"logits/chosen": -0.8248652219772339,
"logits/rejected": -0.8263947367668152,
"logps/chosen": -569.6187744140625,
"logps/ref_chosen": -295.4618225097656,
"logps/ref_rejected": -256.2254333496094,
"logps/rejected": -633.127197265625,
"loss": 4.7812,
"margin_dpo/margin_mean": 102.74482727050781,
"margin_dpo/margin_std": 179.2010498046875,
"step": 434
},
{
"KL/chosen_KL_mean": -244.2108917236328,
"KL/mean": -307.3104248046875,
"KL/rejected_KL_mean": -370.40997314453125,
"KL/std": 169.79331970214844,
"epoch": 0.9109947643979057,
"fcm_dpo/beta": 0.0025554713793098927,
"fcm_dpo/delta": -0.02224489115178585,
"fcm_dpo/margin": 126.19908142089844,
"fcm_dpo/q_t": 0.42764154076576233,
"grad_norm": 70.11904907226562,
"learning_rate": 1.2292508422495157e-08,
"logits/chosen": -0.8194464445114136,
"logits/rejected": -0.8064465522766113,
"logps/chosen": -504.9492492675781,
"logps/ref_chosen": -260.7384033203125,
"logps/ref_rejected": -248.5688018798828,
"logps/rejected": -618.978759765625,
"loss": 4.6041,
"margin_dpo/margin_mean": 126.19908142089844,
"margin_dpo/margin_std": 186.30056762695312,
"step": 435
},
{
"KL/chosen_KL_mean": -273.79022216796875,
"KL/mean": -318.10125732421875,
"KL/rejected_KL_mean": -362.4123840332031,
"KL/std": 188.65408325195312,
"epoch": 0.9130890052356021,
"fcm_dpo/beta": 0.002511697355657816,
"fcm_dpo/delta": -0.003989637829363346,
"fcm_dpo/margin": 88.62214660644531,
"fcm_dpo/q_t": 0.4518389403820038,
"grad_norm": 63.78618240356445,
"learning_rate": 1.1731874863145142e-08,
"logits/chosen": -0.81931072473526,
"logits/rejected": -0.8128557205200195,
"logps/chosen": -593.1127319335938,
"logps/ref_chosen": -319.3224792480469,
"logps/ref_rejected": -299.30322265625,
"logps/rejected": -661.715576171875,
"loss": 4.9787,
"margin_dpo/margin_mean": 88.62214660644531,
"margin_dpo/margin_std": 211.4658966064453,
"step": 436
},
{
"KL/chosen_KL_mean": -249.11143493652344,
"KL/mean": -312.220703125,
"KL/rejected_KL_mean": -375.3299865722656,
"KL/std": 183.77027893066406,
"epoch": 0.9151832460732985,
"fcm_dpo/beta": 0.0025260450784116983,
"fcm_dpo/delta": 0.0047508468851447105,
"fcm_dpo/margin": 126.21853637695312,
"fcm_dpo/q_t": 0.42712879180908203,
"grad_norm": 65.6551742553711,
"learning_rate": 1.118401890024001e-08,
"logits/chosen": -0.8493269085884094,
"logits/rejected": -0.8305841684341431,
"logps/chosen": -527.940185546875,
"logps/ref_chosen": -278.82879638671875,
"logps/ref_rejected": -272.55303955078125,
"logps/rejected": -647.8829956054688,
"loss": 4.6504,
"margin_dpo/margin_mean": 126.21853637695312,
"margin_dpo/margin_std": 204.38760375976562,
"step": 437
},
{
"KL/chosen_KL_mean": -263.64837646484375,
"KL/mean": -293.3003845214844,
"KL/rejected_KL_mean": -322.9523620605469,
"KL/std": 168.18655395507812,
"epoch": 0.9172774869109948,
"fcm_dpo/beta": 0.0025404624175280333,
"fcm_dpo/delta": 0.007852256298065186,
"fcm_dpo/margin": 59.30400466918945,
"fcm_dpo/q_t": 0.46499937772750854,
"grad_norm": 64.0697021484375,
"learning_rate": 1.06489699136324e-08,
"logits/chosen": -0.8310567736625671,
"logits/rejected": -0.840906023979187,
"logps/chosen": -522.9674682617188,
"logps/ref_chosen": -259.31903076171875,
"logps/ref_rejected": -240.99581909179688,
"logps/rejected": -563.9481811523438,
"loss": 5.2207,
"margin_dpo/margin_mean": 59.303993225097656,
"margin_dpo/margin_std": 191.96351623535156,
"step": 438
},
{
"KL/chosen_KL_mean": -252.53065490722656,
"KL/mean": -308.7279357910156,
"KL/rejected_KL_mean": -364.92523193359375,
"KL/std": 159.78070068359375,
"epoch": 0.9193717277486911,
"fcm_dpo/beta": 0.0025289033073931932,
"fcm_dpo/delta": -0.005491324700415134,
"fcm_dpo/margin": 112.39454650878906,
"fcm_dpo/q_t": 0.4330964684486389,
"grad_norm": 69.76829528808594,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": -0.8097501993179321,
"logits/rejected": -0.8053499460220337,
"logps/chosen": -509.65509033203125,
"logps/ref_chosen": -257.1243896484375,
"logps/ref_rejected": -243.20416259765625,
"logps/rejected": -608.1293334960938,
"loss": 4.6923,
"margin_dpo/margin_mean": 112.39454650878906,
"margin_dpo/margin_std": 177.54962158203125,
"step": 439
},
{
"KL/chosen_KL_mean": -271.7328186035156,
"KL/mean": -313.9856872558594,
"KL/rejected_KL_mean": -356.2385559082031,
"KL/std": 148.13238525390625,
"epoch": 0.9214659685863874,
"fcm_dpo/beta": 0.0025394840631633997,
"fcm_dpo/delta": 0.01721479743719101,
"fcm_dpo/margin": 84.5057601928711,
"fcm_dpo/q_t": 0.4492788314819336,
"grad_norm": 85.56015014648438,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": -0.8681034445762634,
"logits/rejected": -0.8587910532951355,
"logps/chosen": -579.2643432617188,
"logps/ref_chosen": -307.5315246582031,
"logps/ref_rejected": -264.3540954589844,
"logps/rejected": -620.5927124023438,
"loss": 4.9114,
"margin_dpo/margin_mean": 84.5057601928711,
"margin_dpo/margin_std": 158.16221618652344,
"step": 440
},
{
"KL/chosen_KL_mean": -251.61834716796875,
"KL/mean": -314.0332336425781,
"KL/rejected_KL_mean": -376.44805908203125,
"KL/std": 164.15542602539062,
"epoch": 0.9235602094240838,
"fcm_dpo/beta": 0.0025592236779630184,
"fcm_dpo/delta": -0.005991383455693722,
"fcm_dpo/margin": 124.82971954345703,
"fcm_dpo/q_t": 0.425258606672287,
"grad_norm": 64.42757415771484,
"learning_rate": 9.12094829893642e-09,
"logits/chosen": -0.8236322402954102,
"logits/rejected": -0.8174566626548767,
"logps/chosen": -561.6002807617188,
"logps/ref_chosen": -309.9819641113281,
"logps/ref_rejected": -297.4968566894531,
"logps/rejected": -673.9448852539062,
"loss": 4.5766,
"margin_dpo/margin_mean": 124.82971954345703,
"margin_dpo/margin_std": 175.52149963378906,
"step": 441
},
{
"KL/chosen_KL_mean": -281.2906494140625,
"KL/mean": -326.7481994628906,
"KL/rejected_KL_mean": -372.2057189941406,
"KL/std": 168.01153564453125,
"epoch": 0.9256544502617801,
"fcm_dpo/beta": 0.002559695625677705,
"fcm_dpo/delta": 0.007131714373826981,
"fcm_dpo/margin": 90.91503143310547,
"fcm_dpo/q_t": 0.44628316164016724,
"grad_norm": 72.68498229980469,
"learning_rate": 8.637407257200496e-09,
"logits/chosen": -0.8905078768730164,
"logits/rejected": -0.8481354713439941,
"logps/chosen": -560.2698364257812,
"logps/ref_chosen": -278.9791564941406,
"logps/ref_rejected": -242.87310791015625,
"logps/rejected": -615.0787963867188,
"loss": 4.9229,
"margin_dpo/margin_mean": 90.91503143310547,
"margin_dpo/margin_std": 197.5365753173828,
"step": 442
},
{
"KL/chosen_KL_mean": -262.447021484375,
"KL/mean": -311.3345947265625,
"KL/rejected_KL_mean": -360.22210693359375,
"KL/std": 150.85263061523438,
"epoch": 0.9277486910994764,
"fcm_dpo/beta": 0.002550204750150442,
"fcm_dpo/delta": -0.021942952647805214,
"fcm_dpo/margin": 97.77508544921875,
"fcm_dpo/q_t": 0.44047561287879944,
"grad_norm": 80.34606170654297,
"learning_rate": 8.166809758815895e-09,
"logits/chosen": -0.7915633320808411,
"logits/rejected": -0.8125393390655518,
"logps/chosen": -536.006103515625,
"logps/ref_chosen": -273.5590515136719,
"logps/ref_rejected": -264.0199279785156,
"logps/rejected": -624.2420043945312,
"loss": 4.8309,
"margin_dpo/margin_mean": 97.77508544921875,
"margin_dpo/margin_std": 177.55967712402344,
"step": 443
},
{
"KL/chosen_KL_mean": -255.01412963867188,
"KL/mean": -312.3970031738281,
"KL/rejected_KL_mean": -369.7798767089844,
"KL/std": 177.91873168945312,
"epoch": 0.9298429319371728,
"fcm_dpo/beta": 0.002535287756472826,
"fcm_dpo/delta": 0.018113907426595688,
"fcm_dpo/margin": 114.76580047607422,
"fcm_dpo/q_t": 0.43355995416641235,
"grad_norm": 61.830787658691406,
"learning_rate": 7.709181040498253e-09,
"logits/chosen": -0.7957402467727661,
"logits/rejected": -0.7812896370887756,
"logps/chosen": -553.1583251953125,
"logps/ref_chosen": -298.1441955566406,
"logps/ref_rejected": -268.0572814941406,
"logps/rejected": -637.837158203125,
"loss": 4.7328,
"margin_dpo/margin_mean": 114.76579284667969,
"margin_dpo/margin_std": 204.9227294921875,
"step": 444
},
{
"KL/chosen_KL_mean": -255.86961364746094,
"KL/mean": -306.81475830078125,
"KL/rejected_KL_mean": -357.7599182128906,
"KL/std": 171.42166137695312,
"epoch": 0.9319371727748691,
"fcm_dpo/beta": 0.0025360870640724897,
"fcm_dpo/delta": -0.026608020067214966,
"fcm_dpo/margin": 101.89030456542969,
"fcm_dpo/q_t": 0.44091877341270447,
"grad_norm": 75.62554168701172,
"learning_rate": 7.2645456434869965e-09,
"logits/chosen": -0.876228392124176,
"logits/rejected": -0.8805335760116577,
"logps/chosen": -510.4102783203125,
"logps/ref_chosen": -254.54067993164062,
"logps/ref_rejected": -264.2445983886719,
"logps/rejected": -622.0044555664062,
"loss": 4.8553,
"margin_dpo/margin_mean": 101.89029693603516,
"margin_dpo/margin_std": 197.63458251953125,
"step": 445
},
{
"KL/chosen_KL_mean": -250.0303955078125,
"KL/mean": -306.152587890625,
"KL/rejected_KL_mean": -362.2747497558594,
"KL/std": 161.83197021484375,
"epoch": 0.9340314136125655,
"fcm_dpo/beta": 0.002476347843185067,
"fcm_dpo/delta": -0.010473774746060371,
"fcm_dpo/margin": 112.24434661865234,
"fcm_dpo/q_t": 0.4361517131328583,
"grad_norm": 49.15285110473633,
"learning_rate": 6.832927412229017e-09,
"logits/chosen": -0.7973624467849731,
"logits/rejected": -0.8008688688278198,
"logps/chosen": -556.7528686523438,
"logps/ref_chosen": -306.72247314453125,
"logps/ref_rejected": -266.3735656738281,
"logps/rejected": -628.6482543945312,
"loss": 4.7508,
"margin_dpo/margin_mean": 112.24434661865234,
"margin_dpo/margin_std": 195.67247009277344,
"step": 446
},
{
"KL/chosen_KL_mean": -246.97784423828125,
"KL/mean": -308.9893798828125,
"KL/rejected_KL_mean": -371.0009460449219,
"KL/std": 164.9072265625,
"epoch": 0.9361256544502617,
"fcm_dpo/beta": 0.002457038965076208,
"fcm_dpo/delta": -0.005904083605855703,
"fcm_dpo/margin": 124.02310180664062,
"fcm_dpo/q_t": 0.4291154742240906,
"grad_norm": 69.2431640625,
"learning_rate": 6.414349493100129e-09,
"logits/chosen": -0.7988148331642151,
"logits/rejected": -0.7919713258743286,
"logps/chosen": -507.4951171875,
"logps/ref_chosen": -260.51727294921875,
"logps/ref_rejected": -236.47061157226562,
"logps/rejected": -607.4715576171875,
"loss": 4.6405,
"margin_dpo/margin_mean": 124.02310180664062,
"margin_dpo/margin_std": 185.88461303710938,
"step": 447
},
{
"KL/chosen_KL_mean": -260.0938415527344,
"KL/mean": -320.5010070800781,
"KL/rejected_KL_mean": -380.908203125,
"KL/std": 160.903076171875,
"epoch": 0.9382198952879581,
"fcm_dpo/beta": 0.0024337496142834425,
"fcm_dpo/delta": -0.020262887701392174,
"fcm_dpo/margin": 120.81436157226562,
"fcm_dpo/q_t": 0.43088439106941223,
"grad_norm": 60.84456253051758,
"learning_rate": 6.0088343331638756e-09,
"logits/chosen": -0.8034505248069763,
"logits/rejected": -0.7938514351844788,
"logps/chosen": -528.880859375,
"logps/ref_chosen": -268.78704833984375,
"logps/ref_rejected": -262.1703796386719,
"logps/rejected": -643.0785522460938,
"loss": 4.6526,
"margin_dpo/margin_mean": 120.81436157226562,
"margin_dpo/margin_std": 174.54934692382812,
"step": 448
},
{
"KL/chosen_KL_mean": -247.74948120117188,
"KL/mean": -304.65057373046875,
"KL/rejected_KL_mean": -361.5516052246094,
"KL/std": 163.8621826171875,
"epoch": 0.9403141361256544,
"fcm_dpo/beta": 0.002439489820972085,
"fcm_dpo/delta": 0.02136923372745514,
"fcm_dpo/margin": 113.80213928222656,
"fcm_dpo/q_t": 0.4335501194000244,
"grad_norm": 85.0506362915039,
"learning_rate": 5.616403678967624e-09,
"logits/chosen": -0.9094984531402588,
"logits/rejected": -0.8936970829963684,
"logps/chosen": -578.700927734375,
"logps/ref_chosen": -330.9514465332031,
"logps/ref_rejected": -239.76974487304688,
"logps/rejected": -601.3213500976562,
"loss": 4.76,
"margin_dpo/margin_mean": 113.80213928222656,
"margin_dpo/margin_std": 199.1973114013672,
"step": 449
},
{
"KL/chosen_KL_mean": -270.36553955078125,
"KL/mean": -320.8754577636719,
"KL/rejected_KL_mean": -371.3853454589844,
"KL/std": 169.2615203857422,
"epoch": 0.9424083769633508,
"fcm_dpo/beta": 0.0024614077992737293,
"fcm_dpo/delta": -0.004837765824049711,
"fcm_dpo/margin": 101.0197982788086,
"fcm_dpo/q_t": 0.44178810715675354,
"grad_norm": 61.14320373535156,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": -0.8040182590484619,
"logits/rejected": -0.7969501614570618,
"logps/chosen": -554.6309814453125,
"logps/ref_chosen": -284.26544189453125,
"logps/ref_rejected": -250.5401611328125,
"logps/rejected": -621.925537109375,
"loss": 4.8008,
"margin_dpo/margin_mean": 101.0197982788086,
"margin_dpo/margin_std": 170.01528930664062,
"step": 450
},
{
"KL/chosen_KL_mean": -279.3070068359375,
"KL/mean": -320.45257568359375,
"KL/rejected_KL_mean": -361.5981140136719,
"KL/std": 158.63836669921875,
"epoch": 0.9445026178010472,
"fcm_dpo/beta": 0.0024547951761633158,
"fcm_dpo/delta": 0.0002841167151927948,
"fcm_dpo/margin": 82.29108428955078,
"fcm_dpo/q_t": 0.4520818889141083,
"grad_norm": 81.18460845947266,
"learning_rate": 4.8708793644441086e-09,
"logits/chosen": -0.8152300715446472,
"logits/rejected": -0.7828786373138428,
"logps/chosen": -581.6279296875,
"logps/ref_chosen": -302.3209228515625,
"logps/ref_rejected": -254.09747314453125,
"logps/rejected": -615.6956176757812,
"loss": 4.9711,
"margin_dpo/margin_mean": 82.29108428955078,
"margin_dpo/margin_std": 171.24998474121094,
"step": 451
},
{
"KL/chosen_KL_mean": -255.58740234375,
"KL/mean": -323.3721008300781,
"KL/rejected_KL_mean": -391.1568603515625,
"KL/std": 196.24913024902344,
"epoch": 0.9465968586387434,
"fcm_dpo/beta": 0.002398530952632427,
"fcm_dpo/delta": -0.03113538958132267,
"fcm_dpo/margin": 135.5694580078125,
"fcm_dpo/q_t": 0.42863401770591736,
"grad_norm": 50.24382781982422,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": -0.8734545707702637,
"logits/rejected": -0.846143364906311,
"logps/chosen": -554.9795532226562,
"logps/ref_chosen": -299.39215087890625,
"logps/ref_rejected": -284.3475036621094,
"logps/rejected": -675.50439453125,
"loss": 4.6565,
"margin_dpo/margin_mean": 135.5694580078125,
"margin_dpo/margin_std": 229.8317108154297,
"step": 452
},
{
"KL/chosen_KL_mean": -275.34716796875,
"KL/mean": -332.12860107421875,
"KL/rejected_KL_mean": -388.9101257324219,
"KL/std": 167.65248107910156,
"epoch": 0.9486910994764398,
"fcm_dpo/beta": 0.0024174817372113466,
"fcm_dpo/delta": 0.01900402083992958,
"fcm_dpo/margin": 113.56294250488281,
"fcm_dpo/q_t": 0.4362477958202362,
"grad_norm": 65.24600982666016,
"learning_rate": 4.1779364682113794e-09,
"logits/chosen": -0.8175550699234009,
"logits/rejected": -0.8079196214675903,
"logps/chosen": -599.9988403320312,
"logps/ref_chosen": -324.6517028808594,
"logps/ref_rejected": -304.1527099609375,
"logps/rejected": -693.0628051757812,
"loss": 4.7582,
"margin_dpo/margin_mean": 113.56295013427734,
"margin_dpo/margin_std": 204.01402282714844,
"step": 453
},
{
"KL/chosen_KL_mean": -261.29437255859375,
"KL/mean": -329.093994140625,
"KL/rejected_KL_mean": -396.8935546875,
"KL/std": 192.02064514160156,
"epoch": 0.9507853403141361,
"fcm_dpo/beta": 0.002420138567686081,
"fcm_dpo/delta": -0.012020056135952473,
"fcm_dpo/margin": 135.59918212890625,
"fcm_dpo/q_t": 0.42454957962036133,
"grad_norm": 73.93843841552734,
"learning_rate": 3.851229943335393e-09,
"logits/chosen": -0.8420263528823853,
"logits/rejected": -0.8489396572113037,
"logps/chosen": -560.9061279296875,
"logps/ref_chosen": -299.6117248535156,
"logps/ref_rejected": -303.74224853515625,
"logps/rejected": -700.6358032226562,
"loss": 4.6019,
"margin_dpo/margin_mean": 135.59918212890625,
"margin_dpo/margin_std": 211.62002563476562,
"step": 454
},
{
"KL/chosen_KL_mean": -254.47279357910156,
"KL/mean": -297.5650939941406,
"KL/rejected_KL_mean": -340.65740966796875,
"KL/std": 164.42532348632812,
"epoch": 0.9528795811518325,
"fcm_dpo/beta": 0.002410230925306678,
"fcm_dpo/delta": -0.0064119016751646996,
"fcm_dpo/margin": 86.18463897705078,
"fcm_dpo/q_t": 0.45259642601013184,
"grad_norm": 60.235843658447266,
"learning_rate": 3.5377236299748147e-09,
"logits/chosen": -0.8099350333213806,
"logits/rejected": -0.8183305263519287,
"logps/chosen": -528.08447265625,
"logps/ref_chosen": -273.6116943359375,
"logps/ref_rejected": -274.4293518066406,
"logps/rejected": -615.0867919921875,
"loss": 4.9766,
"margin_dpo/margin_mean": 86.18463897705078,
"margin_dpo/margin_std": 188.7678985595703,
"step": 455
},
{
"KL/chosen_KL_mean": -284.4190979003906,
"KL/mean": -342.2475280761719,
"KL/rejected_KL_mean": -400.07598876953125,
"KL/std": 191.12667846679688,
"epoch": 0.9549738219895288,
"fcm_dpo/beta": 0.0024038502015173435,
"fcm_dpo/delta": -0.0050141820684075356,
"fcm_dpo/margin": 115.6568374633789,
"fcm_dpo/q_t": 0.4384552836418152,
"grad_norm": 72.66372680664062,
"learning_rate": 3.2374343405217884e-09,
"logits/chosen": -0.7380684614181519,
"logits/rejected": -0.7515442967414856,
"logps/chosen": -606.591064453125,
"logps/ref_chosen": -322.17193603515625,
"logps/ref_rejected": -294.54461669921875,
"logps/rejected": -694.6205444335938,
"loss": 4.7994,
"margin_dpo/margin_mean": 115.6568374633789,
"margin_dpo/margin_std": 224.9918212890625,
"step": 456
},
{
"KL/chosen_KL_mean": -275.81463623046875,
"KL/mean": -339.070068359375,
"KL/rejected_KL_mean": -402.3255615234375,
"KL/std": 183.56442260742188,
"epoch": 0.9570680628272251,
"fcm_dpo/beta": 0.0023992531932890415,
"fcm_dpo/delta": 0.008432567119598389,
"fcm_dpo/margin": 126.51089477539062,
"fcm_dpo/q_t": 0.4272172451019287,
"grad_norm": 64.16017150878906,
"learning_rate": 2.9503781785795713e-09,
"logits/chosen": -0.8137893080711365,
"logits/rejected": -0.8142549991607666,
"logps/chosen": -583.6109008789062,
"logps/ref_chosen": -307.7962341308594,
"logps/ref_rejected": -274.5501403808594,
"logps/rejected": -676.8756713867188,
"loss": 4.6952,
"margin_dpo/margin_mean": 126.51089477539062,
"margin_dpo/margin_std": 209.9931182861328,
"step": 457
},
{
"KL/chosen_KL_mean": -270.3670349121094,
"KL/mean": -323.4082946777344,
"KL/rejected_KL_mean": -376.44952392578125,
"KL/std": 184.91030883789062,
"epoch": 0.9591623036649215,
"fcm_dpo/beta": 0.002431598724797368,
"fcm_dpo/delta": 0.017081839963793755,
"fcm_dpo/margin": 106.08248901367188,
"fcm_dpo/q_t": 0.4409106373786926,
"grad_norm": 64.75917053222656,
"learning_rate": 2.6765705380989432e-09,
"logits/chosen": -0.8364468812942505,
"logits/rejected": -0.8125419020652771,
"logps/chosen": -567.398681640625,
"logps/ref_chosen": -297.0316467285156,
"logps/ref_rejected": -276.1112365722656,
"logps/rejected": -652.560791015625,
"loss": 4.8427,
"margin_dpo/margin_mean": 106.08248901367188,
"margin_dpo/margin_std": 206.88600158691406,
"step": 458
},
{
"KL/chosen_KL_mean": -255.69676208496094,
"KL/mean": -301.064697265625,
"KL/rejected_KL_mean": -346.4325866699219,
"KL/std": 156.7518768310547,
"epoch": 0.9612565445026178,
"fcm_dpo/beta": 0.002469552680850029,
"fcm_dpo/delta": 0.03057432547211647,
"fcm_dpo/margin": 90.73584747314453,
"fcm_dpo/q_t": 0.44875121116638184,
"grad_norm": 76.80902862548828,
"learning_rate": 2.416026102552732e-09,
"logits/chosen": -0.8981096148490906,
"logits/rejected": -0.8896926641464233,
"logps/chosen": -549.2220458984375,
"logps/ref_chosen": -293.5252990722656,
"logps/ref_rejected": -289.30126953125,
"logps/rejected": -635.7338256835938,
"loss": 4.9246,
"margin_dpo/margin_mean": 90.73584747314453,
"margin_dpo/margin_std": 187.47048950195312,
"step": 459
},
{
"KL/chosen_KL_mean": -263.8416442871094,
"KL/mean": -310.0146789550781,
"KL/rejected_KL_mean": -356.18768310546875,
"KL/std": 150.35638427734375,
"epoch": 0.9633507853403142,
"fcm_dpo/beta": 0.0025106584653258324,
"fcm_dpo/delta": 0.018712403252720833,
"fcm_dpo/margin": 92.34601593017578,
"fcm_dpo/q_t": 0.44546929001808167,
"grad_norm": 55.63397979736328,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": -0.8555883169174194,
"logits/rejected": -0.8532405495643616,
"logps/chosen": -582.6220092773438,
"logps/ref_chosen": -318.7803649902344,
"logps/ref_rejected": -258.7906799316406,
"logps/rejected": -614.9783325195312,
"loss": 4.8808,
"margin_dpo/margin_mean": 92.34600830078125,
"margin_dpo/margin_std": 182.1141357421875,
"step": 460
},
{
"KL/chosen_KL_mean": -259.0890197753906,
"KL/mean": -315.6316833496094,
"KL/rejected_KL_mean": -372.1743469238281,
"KL/std": 172.29824829101562,
"epoch": 0.9654450261780104,
"fcm_dpo/beta": 0.002565775066614151,
"fcm_dpo/delta": 0.022872814908623695,
"fcm_dpo/margin": 113.08531188964844,
"fcm_dpo/q_t": 0.4341578781604767,
"grad_norm": 60.84050750732422,
"learning_rate": 1.9347820230782295e-09,
"logits/chosen": -0.81836998462677,
"logits/rejected": -0.8396799564361572,
"logps/chosen": -502.9990234375,
"logps/ref_chosen": -243.9099884033203,
"logps/ref_rejected": -232.6382293701172,
"logps/rejected": -604.8125610351562,
"loss": 4.7443,
"margin_dpo/margin_mean": 113.08531188964844,
"margin_dpo/margin_std": 204.30162048339844,
"step": 461
},
{
"KL/chosen_KL_mean": -262.39306640625,
"KL/mean": -323.4773864746094,
"KL/rejected_KL_mean": -384.56170654296875,
"KL/std": 163.00491333007812,
"epoch": 0.9675392670157068,
"fcm_dpo/beta": 0.002616667188704014,
"fcm_dpo/delta": 0.022976616397500038,
"fcm_dpo/margin": 122.16867065429688,
"fcm_dpo/q_t": 0.4264695942401886,
"grad_norm": 67.28462219238281,
"learning_rate": 1.7141081868094209e-09,
"logits/chosen": -0.8513904213905334,
"logits/rejected": -0.7988516688346863,
"logps/chosen": -606.4840698242188,
"logps/ref_chosen": -344.09100341796875,
"logps/ref_rejected": -252.45037841796875,
"logps/rejected": -637.0120849609375,
"loss": 4.6085,
"margin_dpo/margin_mean": 122.16866302490234,
"margin_dpo/margin_std": 185.75375366210938,
"step": 462
},
{
"KL/chosen_KL_mean": -260.8435974121094,
"KL/mean": -307.8876037597656,
"KL/rejected_KL_mean": -354.93157958984375,
"KL/std": 176.79953002929688,
"epoch": 0.9696335078534032,
"fcm_dpo/beta": 0.002635349053889513,
"fcm_dpo/delta": 0.00047087250277400017,
"fcm_dpo/margin": 94.08798217773438,
"fcm_dpo/q_t": 0.4436490535736084,
"grad_norm": 87.89910888671875,
"learning_rate": 1.5067491694100153e-09,
"logits/chosen": -0.8419432640075684,
"logits/rejected": -0.7984883189201355,
"logps/chosen": -557.986083984375,
"logps/ref_chosen": -297.1424560546875,
"logps/ref_rejected": -234.0208282470703,
"logps/rejected": -588.9524536132812,
"loss": 4.9332,
"margin_dpo/margin_mean": 94.08797454833984,
"margin_dpo/margin_std": 211.70285034179688,
"step": 463
},
{
"KL/chosen_KL_mean": -269.11505126953125,
"KL/mean": -319.7918701171875,
"KL/rejected_KL_mean": -370.46868896484375,
"KL/std": 167.46054077148438,
"epoch": 0.9717277486910995,
"fcm_dpo/beta": 0.0026447109412401915,
"fcm_dpo/delta": 0.0007200678810477257,
"fcm_dpo/margin": 101.35370635986328,
"fcm_dpo/q_t": 0.43749621510505676,
"grad_norm": 58.30624008178711,
"learning_rate": 1.3127160909147672e-09,
"logits/chosen": -0.8257731199264526,
"logits/rejected": -0.8404132723808289,
"logps/chosen": -534.8258056640625,
"logps/ref_chosen": -265.71075439453125,
"logps/ref_rejected": -256.4108581542969,
"logps/rejected": -626.8795776367188,
"loss": 4.7914,
"margin_dpo/margin_mean": 101.35370635986328,
"margin_dpo/margin_std": 186.4237823486328,
"step": 464
},
{
"KL/chosen_KL_mean": -246.98736572265625,
"KL/mean": -310.69110107421875,
"KL/rejected_KL_mean": -374.39483642578125,
"KL/std": 160.25502014160156,
"epoch": 0.9738219895287958,
"fcm_dpo/beta": 0.002568161580711603,
"fcm_dpo/delta": -0.03626430779695511,
"fcm_dpo/margin": 127.40748596191406,
"fcm_dpo/q_t": 0.42483144998550415,
"grad_norm": 69.23091888427734,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": -0.8961881995201111,
"logits/rejected": -0.8565638065338135,
"logps/chosen": -540.1400756835938,
"logps/ref_chosen": -293.1527404785156,
"logps/ref_rejected": -293.70947265625,
"logps/rejected": -668.1043701171875,
"loss": 4.5974,
"margin_dpo/margin_mean": 127.40748596191406,
"margin_dpo/margin_std": 186.34690856933594,
"step": 465
},
{
"KL/chosen_KL_mean": -234.4508819580078,
"KL/mean": -300.031982421875,
"KL/rejected_KL_mean": -365.6130676269531,
"KL/std": 157.27207946777344,
"epoch": 0.9759162303664921,
"fcm_dpo/beta": 0.0025337152183055878,
"fcm_dpo/delta": -0.0182378850877285,
"fcm_dpo/margin": 131.1621551513672,
"fcm_dpo/q_t": 0.4229571223258972,
"grad_norm": 57.641929626464844,
"learning_rate": 9.64668657069706e-10,
"logits/chosen": -0.8110611438751221,
"logits/rejected": -0.7530339956283569,
"logps/chosen": -495.928466796875,
"logps/ref_chosen": -261.4775695800781,
"logps/ref_rejected": -248.36282348632812,
"logps/rejected": -613.9758911132812,
"loss": 4.5442,
"margin_dpo/margin_mean": 131.1621551513672,
"margin_dpo/margin_std": 181.0853271484375,
"step": 466
},
{
"KL/chosen_KL_mean": -262.05810546875,
"KL/mean": -312.107421875,
"KL/rejected_KL_mean": -362.1567687988281,
"KL/std": 167.3401641845703,
"epoch": 0.9780104712041885,
"fcm_dpo/beta": 0.0024951626546680927,
"fcm_dpo/delta": -0.015738535672426224,
"fcm_dpo/margin": 100.09864807128906,
"fcm_dpo/q_t": 0.4429374635219574,
"grad_norm": 75.20722961425781,
"learning_rate": 8.106729664475176e-10,
"logits/chosen": -0.8004065752029419,
"logits/rejected": -0.7925856709480286,
"logps/chosen": -528.412353515625,
"logps/ref_chosen": -266.354248046875,
"logps/ref_rejected": -277.76324462890625,
"logps/rejected": -639.9200439453125,
"loss": 4.8735,
"margin_dpo/margin_mean": 100.09864807128906,
"margin_dpo/margin_std": 204.54727172851562,
"step": 467
},
{
"KL/chosen_KL_mean": -262.4240417480469,
"KL/mean": -304.2724914550781,
"KL/rejected_KL_mean": -346.12091064453125,
"KL/std": 160.87689208984375,
"epoch": 0.9801047120418848,
"fcm_dpo/beta": 0.0025261654518544674,
"fcm_dpo/delta": 0.0298687182366848,
"fcm_dpo/margin": 83.69690704345703,
"fcm_dpo/q_t": 0.4495968222618103,
"grad_norm": 70.44644165039062,
"learning_rate": 6.700405431837585e-10,
"logits/chosen": -0.8833749294281006,
"logits/rejected": -0.8521139025688171,
"logps/chosen": -580.38720703125,
"logps/ref_chosen": -317.9631652832031,
"logps/ref_rejected": -261.8744201660156,
"logps/rejected": -607.9953002929688,
"loss": 5.0089,
"margin_dpo/margin_mean": 83.69690704345703,
"margin_dpo/margin_std": 198.9070281982422,
"step": 468
},
{
"KL/chosen_KL_mean": -246.26190185546875,
"KL/mean": -308.28466796875,
"KL/rejected_KL_mean": -370.3074951171875,
"KL/std": 168.65597534179688,
"epoch": 0.9821989528795811,
"fcm_dpo/beta": 0.0025742819998413324,
"fcm_dpo/delta": 0.003947127610445023,
"fcm_dpo/margin": 124.04557800292969,
"fcm_dpo/q_t": 0.42625945806503296,
"grad_norm": 52.03805923461914,
"learning_rate": 5.427789289685347e-10,
"logits/chosen": -0.8123364448547363,
"logits/rejected": -0.795385479927063,
"logps/chosen": -571.148681640625,
"logps/ref_chosen": -324.8868103027344,
"logps/ref_rejected": -264.0421447753906,
"logps/rejected": -634.349609375,
"loss": 4.6346,
"margin_dpo/margin_mean": 124.04557800292969,
"margin_dpo/margin_std": 199.66555786132812,
"step": 469
},
{
"KL/chosen_KL_mean": -248.92559814453125,
"KL/mean": -309.4586181640625,
"KL/rejected_KL_mean": -369.9916076660156,
"KL/std": 166.44065856933594,
"epoch": 0.9842931937172775,
"fcm_dpo/beta": 0.0025518168695271015,
"fcm_dpo/delta": -0.006509192753583193,
"fcm_dpo/margin": 121.06607818603516,
"fcm_dpo/q_t": 0.4286976456642151,
"grad_norm": 82.49861145019531,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": -0.8041696548461914,
"logits/rejected": -0.8020331263542175,
"logps/chosen": -563.6297607421875,
"logps/ref_chosen": -314.7042541503906,
"logps/ref_rejected": -259.2276611328125,
"logps/rejected": -629.2192993164062,
"loss": 4.6557,
"margin_dpo/margin_mean": 121.06607818603516,
"margin_dpo/margin_std": 191.92596435546875,
"step": 470
},
{
"KL/chosen_KL_mean": -265.6898498535156,
"KL/mean": -325.8057556152344,
"KL/rejected_KL_mean": -385.921630859375,
"KL/std": 194.99688720703125,
"epoch": 0.9863874345549738,
"fcm_dpo/beta": 0.0025341196451336145,
"fcm_dpo/delta": -0.01957012340426445,
"fcm_dpo/margin": 120.23179626464844,
"fcm_dpo/q_t": 0.43427878618240356,
"grad_norm": 76.36981201171875,
"learning_rate": 3.2839470889836627e-10,
"logits/chosen": -0.8662016987800598,
"logits/rejected": -0.8508539199829102,
"logps/chosen": -558.2646484375,
"logps/ref_chosen": -292.5748291015625,
"logps/ref_rejected": -298.7525329589844,
"logps/rejected": -684.6741333007812,
"loss": 4.7386,
"margin_dpo/margin_mean": 120.23179626464844,
"margin_dpo/margin_std": 221.61729431152344,
"step": 471
},
{
"KL/chosen_KL_mean": -239.04766845703125,
"KL/mean": -306.37890625,
"KL/rejected_KL_mean": -373.710205078125,
"KL/std": 166.82215881347656,
"epoch": 0.9884816753926702,
"fcm_dpo/beta": 0.0024978769943118095,
"fcm_dpo/delta": 0.0006190729327499866,
"fcm_dpo/margin": 134.66250610351562,
"fcm_dpo/q_t": 0.4230276346206665,
"grad_norm": 88.5049057006836,
"learning_rate": 2.412835998185092e-10,
"logits/chosen": -0.8472277522087097,
"logits/rejected": -0.8603392839431763,
"logps/chosen": -482.4215087890625,
"logps/ref_chosen": -243.37380981445312,
"logps/ref_rejected": -251.12109375,
"logps/rejected": -624.831298828125,
"loss": 4.5638,
"margin_dpo/margin_mean": 134.66250610351562,
"margin_dpo/margin_std": 198.72836303710938,
"step": 472
},
{
"KL/chosen_KL_mean": -256.0351867675781,
"KL/mean": -317.4332275390625,
"KL/rejected_KL_mean": -378.8313293457031,
"KL/std": 152.7645263671875,
"epoch": 0.9905759162303664,
"fcm_dpo/beta": 0.0025325005408376455,
"fcm_dpo/delta": 0.027150386944413185,
"fcm_dpo/margin": 122.79612731933594,
"fcm_dpo/q_t": 0.4268743097782135,
"grad_norm": 49.66973114013672,
"learning_rate": 1.6756629272085544e-10,
"logits/chosen": -0.7995487451553345,
"logits/rejected": -0.7962360978126526,
"logps/chosen": -542.36376953125,
"logps/ref_chosen": -286.3286437988281,
"logps/ref_rejected": -258.6535339355469,
"logps/rejected": -637.4849243164062,
"loss": 4.5916,
"margin_dpo/margin_mean": 122.79612731933594,
"margin_dpo/margin_std": 171.07659912109375,
"step": 473
},
{
"KL/chosen_KL_mean": -259.002685546875,
"KL/mean": -313.4521179199219,
"KL/rejected_KL_mean": -367.90155029296875,
"KL/std": 170.04241943359375,
"epoch": 0.9926701570680628,
"fcm_dpo/beta": 0.002547986339777708,
"fcm_dpo/delta": -0.010679654777050018,
"fcm_dpo/margin": 108.89888000488281,
"fcm_dpo/q_t": 0.4356432557106018,
"grad_norm": 75.55198669433594,
"learning_rate": 1.072467408408384e-10,
"logits/chosen": -0.8527939319610596,
"logits/rejected": -0.8474220633506775,
"logps/chosen": -547.0923461914062,
"logps/ref_chosen": -288.08966064453125,
"logps/ref_rejected": -266.69696044921875,
"logps/rejected": -634.5985717773438,
"loss": 4.7386,
"margin_dpo/margin_mean": 108.89888763427734,
"margin_dpo/margin_std": 182.4081573486328,
"step": 474
},
{
"KL/chosen_KL_mean": -241.1262969970703,
"KL/mean": -290.3180236816406,
"KL/rejected_KL_mean": -339.50970458984375,
"KL/std": 152.7301483154297,
"epoch": 0.9947643979057592,
"fcm_dpo/beta": 0.0025214108172804117,
"fcm_dpo/delta": -0.01645851507782936,
"fcm_dpo/margin": 98.38336181640625,
"fcm_dpo/q_t": 0.4433209002017975,
"grad_norm": 57.4051628112793,
"learning_rate": 6.032817893297793e-11,
"logits/chosen": -0.8317741751670837,
"logits/rejected": -0.84046870470047,
"logps/chosen": -497.1293640136719,
"logps/ref_chosen": -256.0030517578125,
"logps/ref_rejected": -244.50660705566406,
"logps/rejected": -584.0162963867188,
"loss": 4.824,
"margin_dpo/margin_mean": 98.38336181640625,
"margin_dpo/margin_std": 176.57955932617188,
"step": 475
},
{
"KL/chosen_KL_mean": -242.7122039794922,
"KL/mean": -295.69720458984375,
"KL/rejected_KL_mean": -348.68218994140625,
"KL/std": 153.21731567382812,
"epoch": 0.9968586387434555,
"fcm_dpo/beta": 0.0024653631262481213,
"fcm_dpo/delta": -0.029824629426002502,
"fcm_dpo/margin": 105.96993255615234,
"fcm_dpo/q_t": 0.44073987007141113,
"grad_norm": 54.92226028442383,
"learning_rate": 2.6813123097352287e-11,
"logits/chosen": -0.8974910378456116,
"logits/rejected": -0.8506391048431396,
"logps/chosen": -564.1797485351562,
"logps/ref_chosen": -321.467529296875,
"logps/ref_rejected": -295.0592956542969,
"logps/rejected": -643.7415161132812,
"loss": 4.8239,
"margin_dpo/margin_mean": 105.96994018554688,
"margin_dpo/margin_std": 196.14195251464844,
"step": 476
},
{
"KL/chosen_KL_mean": -268.98797607421875,
"KL/mean": -318.58404541015625,
"KL/rejected_KL_mean": -368.18011474609375,
"KL/std": 177.54359436035156,
"epoch": 0.9989528795811519,
"fcm_dpo/beta": 0.002445072866976261,
"fcm_dpo/delta": -0.009669403545558453,
"fcm_dpo/margin": 99.1921615600586,
"fcm_dpo/q_t": 0.44362398982048035,
"grad_norm": 69.17218017578125,
"learning_rate": 6.7033706447061635e-12,
"logits/chosen": -0.7706684470176697,
"logits/rejected": -0.7763949632644653,
"logps/chosen": -545.7819213867188,
"logps/ref_chosen": -276.7939758300781,
"logps/ref_rejected": -244.82919311523438,
"logps/rejected": -613.00927734375,
"loss": 4.9399,
"margin_dpo/margin_mean": 99.19215393066406,
"margin_dpo/margin_std": 223.55364990234375,
"step": 477
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 4.88172167052263,
"train_runtime": 7274.147,
"train_samples_per_second": 8.404,
"train_steps_per_second": 0.066
}
],
"logging_steps": 1,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink