W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 89b1c23d07 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3
Source: Original Platform
2026-06-01 07:20:16 +08:00

15247 lines
572 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999244142101285,
"eval_steps": 200,
"global_step": 661,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.02867889404296875,
"KL/mean": 0.029354453086853027,
"KL/rejected_KL_mean": 0.030029296875,
"KL/std": 0.2071000635623932,
"epoch": 0.0015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013532638549804688,
"fcm_dpo/q_t": 0.5000336766242981,
"grad_norm": 28.21967887878418,
"learning_rate": 0.0,
"logits/chosen": 0.13337239623069763,
"logits/rejected": 0.12492949515581131,
"logps/chosen": -64.5841293334961,
"logps/ref_chosen": -64.61280822753906,
"logps/ref_rejected": -64.17195129394531,
"logps/rejected": -64.14192199707031,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0013527870178222656,
"margin_dpo/margin_std": 0.2561596930027008,
"step": 1
},
{
"KL/chosen_KL_mean": -0.00289154052734375,
"KL/mean": -0.021616414189338684,
"KL/rejected_KL_mean": -0.04033660888671875,
"KL/std": 0.19624735414981842,
"epoch": 0.0030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.037450045347213745,
"fcm_dpo/q_t": 0.4990639090538025,
"grad_norm": 27.828706741333008,
"learning_rate": 7.462686567164179e-09,
"logits/chosen": 0.09414851665496826,
"logits/rejected": 0.07363267242908478,
"logps/chosen": -56.101890563964844,
"logps/ref_chosen": -56.0989990234375,
"logps/ref_rejected": -66.59971618652344,
"logps/rejected": -66.64006042480469,
"loss": 1.3828,
"margin_dpo/margin_mean": 0.03744968771934509,
"margin_dpo/margin_std": 0.27811938524246216,
"step": 2
},
{
"KL/chosen_KL_mean": 0.007659912109375,
"KL/mean": 0.032633185386657715,
"KL/rejected_KL_mean": 0.0576019287109375,
"KL/std": 0.2508915066719055,
"epoch": 0.0045351473922902496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04993748664855957,
"fcm_dpo/q_t": 0.501247763633728,
"grad_norm": 31.35702133178711,
"learning_rate": 1.4925373134328357e-08,
"logits/chosen": 0.09931470453739166,
"logits/rejected": 0.06133907288312912,
"logps/chosen": -65.44960021972656,
"logps/ref_chosen": -65.45726013183594,
"logps/ref_rejected": -90.82853698730469,
"logps/rejected": -90.77093505859375,
"loss": 1.3916,
"margin_dpo/margin_mean": -0.04993721842765808,
"margin_dpo/margin_std": 0.33015739917755127,
"step": 3
},
{
"KL/chosen_KL_mean": 0.003841400146484375,
"KL/mean": -0.005494982004165649,
"KL/rejected_KL_mean": -0.014835357666015625,
"KL/std": 0.22913160920143127,
"epoch": 0.006046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01867443323135376,
"fcm_dpo/q_t": 0.49953362345695496,
"grad_norm": 34.08740997314453,
"learning_rate": 2.2388059701492534e-08,
"logits/chosen": 0.1126205176115036,
"logits/rejected": 0.09632235765457153,
"logps/chosen": -76.85633850097656,
"logps/ref_chosen": -76.86018371582031,
"logps/ref_rejected": -79.91523742675781,
"logps/rejected": -79.93006896972656,
"loss": 1.3847,
"margin_dpo/margin_mean": 0.018673986196517944,
"margin_dpo/margin_std": 0.3117932677268982,
"step": 4
},
{
"KL/chosen_KL_mean": -0.014787673950195312,
"KL/mean": -0.016955822706222534,
"KL/rejected_KL_mean": -0.0191192626953125,
"KL/std": 0.222198024392128,
"epoch": 0.007558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.004332542419433594,
"fcm_dpo/q_t": 0.4998916983604431,
"grad_norm": 29.587570190429688,
"learning_rate": 2.9850746268656714e-08,
"logits/chosen": 0.0821952074766159,
"logits/rejected": 0.04323691129684448,
"logps/chosen": -62.98612976074219,
"logps/ref_chosen": -62.97134017944336,
"logps/ref_rejected": -79.9192123413086,
"logps/rejected": -79.9383316040039,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.004333585500717163,
"margin_dpo/margin_std": 0.30388158559799194,
"step": 5
},
{
"KL/chosen_KL_mean": -0.020517349243164062,
"KL/mean": 0.00726713240146637,
"KL/rejected_KL_mean": 0.03505706787109375,
"KL/std": 0.30582496523857117,
"epoch": 0.009070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05557325482368469,
"fcm_dpo/q_t": 0.5013865232467651,
"grad_norm": 29.830081939697266,
"learning_rate": 3.731343283582089e-08,
"logits/chosen": 0.14537031948566437,
"logits/rejected": 0.1057555228471756,
"logps/chosen": -51.327880859375,
"logps/ref_chosen": -51.30736541748047,
"logps/ref_rejected": -82.77239227294922,
"logps/rejected": -82.73733520507812,
"loss": 1.3923,
"margin_dpo/margin_mean": -0.05557364225387573,
"margin_dpo/margin_std": 0.4328301250934601,
"step": 6
},
{
"KL/chosen_KL_mean": 0.017831802368164062,
"KL/mean": 0.02575281262397766,
"KL/rejected_KL_mean": 0.033672332763671875,
"KL/std": 0.20926260948181152,
"epoch": 0.010582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01584029197692871,
"fcm_dpo/q_t": 0.5003958940505981,
"grad_norm": 27.20305633544922,
"learning_rate": 4.477611940298507e-08,
"logits/chosen": 0.007591056637465954,
"logits/rejected": -0.035748470574617386,
"logps/chosen": -51.44158172607422,
"logps/ref_chosen": -51.45941162109375,
"logps/ref_rejected": -66.3828125,
"logps/rejected": -66.34913635253906,
"loss": 1.388,
"margin_dpo/margin_mean": -0.015840649604797363,
"margin_dpo/margin_std": 0.22362905740737915,
"step": 7
},
{
"KL/chosen_KL_mean": 0.002826690673828125,
"KL/mean": -0.014976292848587036,
"KL/rejected_KL_mean": -0.03277397155761719,
"KL/std": 0.2553374171257019,
"epoch": 0.012093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03559964895248413,
"fcm_dpo/q_t": 0.49911004304885864,
"grad_norm": 28.653799057006836,
"learning_rate": 5.223880597014925e-08,
"logits/chosen": 0.06839226186275482,
"logits/rejected": 0.04627063870429993,
"logps/chosen": -62.19471740722656,
"logps/ref_chosen": -62.197547912597656,
"logps/ref_rejected": -74.66180419921875,
"logps/rejected": -74.694580078125,
"loss": 1.383,
"margin_dpo/margin_mean": 0.03559872508049011,
"margin_dpo/margin_std": 0.33308953046798706,
"step": 8
},
{
"KL/chosen_KL_mean": -0.03289794921875,
"KL/mean": -0.015181422233581543,
"KL/rejected_KL_mean": 0.002529144287109375,
"KL/std": 0.23224589228630066,
"epoch": 0.013605442176870748,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03543180227279663,
"fcm_dpo/q_t": 0.5008852481842041,
"grad_norm": 31.527856826782227,
"learning_rate": 5.970149253731343e-08,
"logits/chosen": 0.17893418669700623,
"logits/rejected": 0.1187577098608017,
"logps/chosen": -55.662620544433594,
"logps/ref_chosen": -55.629722595214844,
"logps/ref_rejected": -86.21221923828125,
"logps/rejected": -86.2096939086914,
"loss": 1.3901,
"margin_dpo/margin_mean": -0.035430908203125,
"margin_dpo/margin_std": 0.3422740697860718,
"step": 9
},
{
"KL/chosen_KL_mean": 0.03826141357421875,
"KL/mean": 0.021607831120491028,
"KL/rejected_KL_mean": 0.00495147705078125,
"KL/std": 0.270057737827301,
"epoch": 0.015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03330746293067932,
"fcm_dpo/q_t": 0.49916791915893555,
"grad_norm": 29.675212860107422,
"learning_rate": 6.71641791044776e-08,
"logits/chosen": 0.1317688524723053,
"logits/rejected": 0.10075588524341583,
"logps/chosen": -62.652339935302734,
"logps/ref_chosen": -62.69060134887695,
"logps/ref_rejected": -90.610107421875,
"logps/rejected": -90.60516357421875,
"loss": 1.3834,
"margin_dpo/margin_mean": 0.033307403326034546,
"margin_dpo/margin_std": 0.38862723112106323,
"step": 10
},
{
"KL/chosen_KL_mean": -0.004878997802734375,
"KL/mean": -0.020426854491233826,
"KL/rejected_KL_mean": -0.035980224609375,
"KL/std": 0.1961967498064041,
"epoch": 0.016628873771730914,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.031100362539291382,
"fcm_dpo/q_t": 0.4992234408855438,
"grad_norm": 29.25577163696289,
"learning_rate": 7.462686567164178e-08,
"logits/chosen": 0.11657217144966125,
"logits/rejected": 0.10957114398479462,
"logps/chosen": -65.7719955444336,
"logps/ref_chosen": -65.76712036132812,
"logps/ref_rejected": -72.4764633178711,
"logps/rejected": -72.51244354248047,
"loss": 1.3834,
"margin_dpo/margin_mean": 0.031100690364837646,
"margin_dpo/margin_std": 0.2950153946876526,
"step": 11
},
{
"KL/chosen_KL_mean": -0.008037567138671875,
"KL/mean": -0.016360193490982056,
"KL/rejected_KL_mean": -0.02468109130859375,
"KL/std": 0.21698607504367828,
"epoch": 0.018140589569160998,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.016642391681671143,
"fcm_dpo/q_t": 0.49958401918411255,
"grad_norm": 28.2968692779541,
"learning_rate": 8.208955223880596e-08,
"logits/chosen": 0.04076576605439186,
"logits/rejected": 0.024239014834165573,
"logps/chosen": -60.712928771972656,
"logps/ref_chosen": -60.704891204833984,
"logps/ref_rejected": -69.41564178466797,
"logps/rejected": -69.44032287597656,
"loss": 1.3849,
"margin_dpo/margin_mean": 0.016642063856124878,
"margin_dpo/margin_std": 0.3236123323440552,
"step": 12
},
{
"KL/chosen_KL_mean": -0.025043487548828125,
"KL/mean": 0.008205235004425049,
"KL/rejected_KL_mean": 0.04146575927734375,
"KL/std": 0.23161034286022186,
"epoch": 0.019652305366591082,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06650584936141968,
"fcm_dpo/q_t": 0.5016615986824036,
"grad_norm": 29.276233673095703,
"learning_rate": 8.955223880597014e-08,
"logits/chosen": 0.10737905651330948,
"logits/rejected": 0.04509834945201874,
"logps/chosen": -49.93429946899414,
"logps/ref_chosen": -49.90925598144531,
"logps/ref_rejected": -92.37818145751953,
"logps/rejected": -92.33671569824219,
"loss": 1.3933,
"margin_dpo/margin_mean": -0.06650650501251221,
"margin_dpo/margin_std": 0.34030449390411377,
"step": 13
},
{
"KL/chosen_KL_mean": 0.05045509338378906,
"KL/mean": 0.0025360435247421265,
"KL/rejected_KL_mean": -0.045379638671875,
"KL/std": 0.19650276005268097,
"epoch": 0.021164021164021163,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.095831960439682,
"fcm_dpo/q_t": 0.4976050853729248,
"grad_norm": 29.4014835357666,
"learning_rate": 9.701492537313432e-08,
"logits/chosen": 0.08358614891767502,
"logits/rejected": 0.06596517562866211,
"logps/chosen": -60.568336486816406,
"logps/ref_chosen": -60.61879348754883,
"logps/ref_rejected": -71.79306030273438,
"logps/rejected": -71.83843994140625,
"loss": 1.3769,
"margin_dpo/margin_mean": 0.09583166241645813,
"margin_dpo/margin_std": 0.2745246887207031,
"step": 14
},
{
"KL/chosen_KL_mean": -0.04099273681640625,
"KL/mean": -0.02316710352897644,
"KL/rejected_KL_mean": -0.005344390869140625,
"KL/std": 0.2433144450187683,
"epoch": 0.022675736961451247,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03565239906311035,
"fcm_dpo/q_t": 0.5008911490440369,
"grad_norm": 33.39137649536133,
"learning_rate": 1.044776119402985e-07,
"logits/chosen": 0.06936109066009521,
"logits/rejected": 0.02610601857304573,
"logps/chosen": -63.510528564453125,
"logps/ref_chosen": -63.46953582763672,
"logps/ref_rejected": -88.88951110839844,
"logps/rejected": -88.89485931396484,
"loss": 1.3902,
"margin_dpo/margin_mean": -0.03565263748168945,
"margin_dpo/margin_std": 0.3474429249763489,
"step": 15
},
{
"KL/chosen_KL_mean": -0.024089813232421875,
"KL/mean": -0.025951489806175232,
"KL/rejected_KL_mean": -0.0278167724609375,
"KL/std": 0.20087632536888123,
"epoch": 0.02418745275888133,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.003727465867996216,
"fcm_dpo/q_t": 0.49990740418434143,
"grad_norm": 26.88129997253418,
"learning_rate": 1.1194029850746268e-07,
"logits/chosen": 0.11440499126911163,
"logits/rejected": 0.07738970220088959,
"logps/chosen": -46.55638885498047,
"logps/ref_chosen": -46.53229904174805,
"logps/ref_rejected": -74.27533721923828,
"logps/rejected": -74.30315399169922,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.003727525472640991,
"margin_dpo/margin_std": 0.2645993232727051,
"step": 16
},
{
"KL/chosen_KL_mean": 0.02545928955078125,
"KL/mean": 0.028340786695480347,
"KL/rejected_KL_mean": 0.031219482421875,
"KL/std": 0.23972105979919434,
"epoch": 0.025699168556311415,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.005758881568908691,
"fcm_dpo/q_t": 0.5001437067985535,
"grad_norm": 32.905975341796875,
"learning_rate": 1.1940298507462686e-07,
"logits/chosen": 0.031329765915870667,
"logits/rejected": 0.013444792479276657,
"logps/chosen": -64.05237579345703,
"logps/ref_chosen": -64.07783508300781,
"logps/ref_rejected": -86.40876770019531,
"logps/rejected": -86.37754821777344,
"loss": 1.3871,
"margin_dpo/margin_mean": -0.005759298801422119,
"margin_dpo/margin_std": 0.31801456212997437,
"step": 17
},
{
"KL/chosen_KL_mean": 0.02117156982421875,
"KL/mean": -0.0042324066162109375,
"KL/rejected_KL_mean": -0.02964019775390625,
"KL/std": 0.2313615083694458,
"epoch": 0.027210884353741496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.050807178020477295,
"fcm_dpo/q_t": 0.49873000383377075,
"grad_norm": 27.679706573486328,
"learning_rate": 1.2686567164179106e-07,
"logits/chosen": 0.08802352845668793,
"logits/rejected": 0.042935267090797424,
"logps/chosen": -44.85316467285156,
"logps/ref_chosen": -44.87433624267578,
"logps/ref_rejected": -70.97604370117188,
"logps/rejected": -71.00569152832031,
"loss": 1.3815,
"margin_dpo/margin_mean": 0.05080723762512207,
"margin_dpo/margin_std": 0.3414297103881836,
"step": 18
},
{
"KL/chosen_KL_mean": 0.05913352966308594,
"KL/mean": 0.02635572850704193,
"KL/rejected_KL_mean": -0.006427764892578125,
"KL/std": 0.2230260968208313,
"epoch": 0.02872260015117158,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06556209921836853,
"fcm_dpo/q_t": 0.49836117029190063,
"grad_norm": 30.85642433166504,
"learning_rate": 1.343283582089552e-07,
"logits/chosen": 0.06814460456371307,
"logits/rejected": 0.054654691368341446,
"logps/chosen": -68.10067749023438,
"logps/ref_chosen": -68.1598129272461,
"logps/ref_rejected": -81.17138671875,
"logps/rejected": -81.17781829833984,
"loss": 1.38,
"margin_dpo/margin_mean": 0.06556195020675659,
"margin_dpo/margin_std": 0.2927999496459961,
"step": 19
},
{
"KL/chosen_KL_mean": 0.027414321899414062,
"KL/mean": 0.022625096142292023,
"KL/rejected_KL_mean": 0.0178375244140625,
"KL/std": 0.22135794162750244,
"epoch": 0.030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.009577527642250061,
"fcm_dpo/q_t": 0.4997604191303253,
"grad_norm": 29.147907257080078,
"learning_rate": 1.4179104477611938e-07,
"logits/chosen": 0.12029474228620529,
"logits/rejected": 0.09780436754226685,
"logps/chosen": -53.651145935058594,
"logps/ref_chosen": -53.67856216430664,
"logps/ref_rejected": -74.16911315917969,
"logps/rejected": -74.15127563476562,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.009577080607414246,
"margin_dpo/margin_std": 0.27818232774734497,
"step": 20
},
{
"KL/chosen_KL_mean": -0.01338958740234375,
"KL/mean": -0.00590062141418457,
"KL/rejected_KL_mean": 0.0015869140625,
"KL/std": 0.24868792295455933,
"epoch": 0.031746031746031744,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.014971256256103516,
"fcm_dpo/q_t": 0.5003751516342163,
"grad_norm": 29.285295486450195,
"learning_rate": 1.4925373134328355e-07,
"logits/chosen": 0.12290213257074356,
"logits/rejected": 0.09662627428770065,
"logps/chosen": -64.7149429321289,
"logps/ref_chosen": -64.70155334472656,
"logps/ref_rejected": -81.02095031738281,
"logps/rejected": -81.01936340332031,
"loss": 1.3881,
"margin_dpo/margin_mean": -0.014971524477005005,
"margin_dpo/margin_std": 0.3347770571708679,
"step": 21
},
{
"KL/chosen_KL_mean": -0.007518768310546875,
"KL/mean": -0.034809038043022156,
"KL/rejected_KL_mean": -0.06209564208984375,
"KL/std": 0.20913906395435333,
"epoch": 0.03325774754346183,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05458232760429382,
"fcm_dpo/q_t": 0.4986357092857361,
"grad_norm": 28.887042999267578,
"learning_rate": 1.5671641791044775e-07,
"logits/chosen": 0.02207999862730503,
"logits/rejected": 0.00017184100579470396,
"logps/chosen": -58.04351043701172,
"logps/ref_chosen": -58.03599166870117,
"logps/ref_rejected": -80.72721862792969,
"logps/rejected": -80.78931427001953,
"loss": 1.381,
"margin_dpo/margin_mean": 0.054582446813583374,
"margin_dpo/margin_std": 0.2853773832321167,
"step": 22
},
{
"KL/chosen_KL_mean": 0.012264251708984375,
"KL/mean": 0.009754911065101624,
"KL/rejected_KL_mean": 0.00724029541015625,
"KL/std": 0.26380670070648193,
"epoch": 0.03476946334089191,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.005020111799240112,
"fcm_dpo/q_t": 0.4998743534088135,
"grad_norm": 33.045352935791016,
"learning_rate": 1.6417910447761193e-07,
"logits/chosen": 0.12751388549804688,
"logits/rejected": 0.10264043509960175,
"logps/chosen": -66.34382629394531,
"logps/ref_chosen": -66.35608673095703,
"logps/ref_rejected": -93.02769470214844,
"logps/rejected": -93.02044677734375,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.005019426345825195,
"margin_dpo/margin_std": 0.3555549383163452,
"step": 23
},
{
"KL/chosen_KL_mean": -0.004886627197265625,
"KL/mean": -0.0031652823090553284,
"KL/rejected_KL_mean": -0.0014438629150390625,
"KL/std": 0.2178821861743927,
"epoch": 0.036281179138321996,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.003436669707298279,
"fcm_dpo/q_t": 0.5000854730606079,
"grad_norm": 26.289295196533203,
"learning_rate": 1.716417910447761e-07,
"logits/chosen": 0.15002357959747314,
"logits/rejected": 0.1167023628950119,
"logps/chosen": -54.46612548828125,
"logps/ref_chosen": -54.461238861083984,
"logps/ref_rejected": -68.33817291259766,
"logps/rejected": -68.33961486816406,
"loss": 1.3869,
"margin_dpo/margin_mean": -0.003437221050262451,
"margin_dpo/margin_std": 0.29839447140693665,
"step": 24
},
{
"KL/chosen_KL_mean": -0.03343772888183594,
"KL/mean": -0.024197697639465332,
"KL/rejected_KL_mean": -0.01496124267578125,
"KL/std": 0.23071929812431335,
"epoch": 0.03779289493575208,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01847773790359497,
"fcm_dpo/q_t": 0.5004626512527466,
"grad_norm": 29.7857666015625,
"learning_rate": 1.7910447761194027e-07,
"logits/chosen": 0.11460703611373901,
"logits/rejected": 0.06244586408138275,
"logps/chosen": -60.03764343261719,
"logps/ref_chosen": -60.00420379638672,
"logps/ref_rejected": -90.47376251220703,
"logps/rejected": -90.48872375488281,
"loss": 1.3884,
"margin_dpo/margin_mean": -0.01847824454307556,
"margin_dpo/margin_std": 0.2980327606201172,
"step": 25
},
{
"KL/chosen_KL_mean": -0.00635528564453125,
"KL/mean": -0.007910311222076416,
"KL/rejected_KL_mean": -0.009464263916015625,
"KL/std": 0.1977149397134781,
"epoch": 0.039304610733182165,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.003108978271484375,
"fcm_dpo/q_t": 0.499922513961792,
"grad_norm": 29.830995559692383,
"learning_rate": 1.8656716417910447e-07,
"logits/chosen": 0.0936226099729538,
"logits/rejected": 0.07567183673381805,
"logps/chosen": -56.82550811767578,
"logps/ref_chosen": -56.81915283203125,
"logps/ref_rejected": -77.84333038330078,
"logps/rejected": -77.85279846191406,
"loss": 1.3862,
"margin_dpo/margin_mean": 0.0031086504459381104,
"margin_dpo/margin_std": 0.2862233519554138,
"step": 26
},
{
"KL/chosen_KL_mean": -0.016384124755859375,
"KL/mean": -0.016014084219932556,
"KL/rejected_KL_mean": -0.015649795532226562,
"KL/std": 0.2392708659172058,
"epoch": 0.04081632653061224,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.000739365816116333,
"fcm_dpo/q_t": 0.5000181198120117,
"grad_norm": 28.958324432373047,
"learning_rate": 1.9402985074626865e-07,
"logits/chosen": 0.11648187786340714,
"logits/rejected": 0.09103013575077057,
"logps/chosen": -62.893409729003906,
"logps/ref_chosen": -62.87702560424805,
"logps/ref_rejected": -71.34437561035156,
"logps/rejected": -71.36002349853516,
"loss": 1.3867,
"margin_dpo/margin_mean": -0.0007395446300506592,
"margin_dpo/margin_std": 0.36002424359321594,
"step": 27
},
{
"KL/chosen_KL_mean": -0.012584686279296875,
"KL/mean": -0.004347294569015503,
"KL/rejected_KL_mean": 0.0038890838623046875,
"KL/std": 0.23575863242149353,
"epoch": 0.042328042328042326,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.016472816467285156,
"fcm_dpo/q_t": 0.5004111528396606,
"grad_norm": 27.684032440185547,
"learning_rate": 2.0149253731343282e-07,
"logits/chosen": 0.07097271829843521,
"logits/rejected": 0.06192881613969803,
"logps/chosen": -59.84596252441406,
"logps/ref_chosen": -59.8333740234375,
"logps/ref_rejected": -70.39804077148438,
"logps/rejected": -70.39414978027344,
"loss": 1.3882,
"margin_dpo/margin_mean": -0.016472458839416504,
"margin_dpo/margin_std": 0.3243771195411682,
"step": 28
},
{
"KL/chosen_KL_mean": -0.018489837646484375,
"KL/mean": -0.031423747539520264,
"KL/rejected_KL_mean": -0.04434967041015625,
"KL/std": 0.20472386479377747,
"epoch": 0.04383975812547241,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.025858044624328613,
"fcm_dpo/q_t": 0.4993535876274109,
"grad_norm": 32.789634704589844,
"learning_rate": 2.08955223880597e-07,
"logits/chosen": 0.15825878083705902,
"logits/rejected": 0.1399884819984436,
"logps/chosen": -74.13868713378906,
"logps/ref_chosen": -74.12020111083984,
"logps/ref_rejected": -83.33099365234375,
"logps/rejected": -83.37533569335938,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.025858759880065918,
"margin_dpo/margin_std": 0.2755180895328522,
"step": 29
},
{
"KL/chosen_KL_mean": -0.0454254150390625,
"KL/mean": -0.06478771567344666,
"KL/rejected_KL_mean": -0.0841522216796875,
"KL/std": 0.20571255683898926,
"epoch": 0.045351473922902494,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.038724422454833984,
"fcm_dpo/q_t": 0.49903199076652527,
"grad_norm": 30.31739616394043,
"learning_rate": 2.1641791044776117e-07,
"logits/chosen": 0.11089472472667694,
"logits/rejected": 0.05757853388786316,
"logps/chosen": -50.796714782714844,
"logps/ref_chosen": -50.75128936767578,
"logps/ref_rejected": -89.29063415527344,
"logps/rejected": -89.37478637695312,
"loss": 1.3827,
"margin_dpo/margin_mean": 0.038724422454833984,
"margin_dpo/margin_std": 0.30037564039230347,
"step": 30
},
{
"KL/chosen_KL_mean": 0.008935928344726562,
"KL/mean": -0.04200010001659393,
"KL/rejected_KL_mean": -0.09293365478515625,
"KL/std": 0.27889418601989746,
"epoch": 0.04686318972033258,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10187044739723206,
"fcm_dpo/q_t": 0.4974536597728729,
"grad_norm": 34.08384323120117,
"learning_rate": 2.2388059701492537e-07,
"logits/chosen": 0.111326202750206,
"logits/rejected": 0.0648956447839737,
"logps/chosen": -65.32781982421875,
"logps/ref_chosen": -65.33675384521484,
"logps/ref_rejected": -100.76666259765625,
"logps/rejected": -100.8595962524414,
"loss": 1.3766,
"margin_dpo/margin_mean": 0.10187077522277832,
"margin_dpo/margin_std": 0.39277029037475586,
"step": 31
},
{
"KL/chosen_KL_mean": -0.01952362060546875,
"KL/mean": -0.024612531065940857,
"KL/rejected_KL_mean": -0.029697418212890625,
"KL/std": 0.22407123446464539,
"epoch": 0.04837490551776266,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01017645001411438,
"fcm_dpo/q_t": 0.49974536895751953,
"grad_norm": 30.059810638427734,
"learning_rate": 2.3134328358208954e-07,
"logits/chosen": 0.07288508117198944,
"logits/rejected": 0.06522229313850403,
"logps/chosen": -67.20285034179688,
"logps/ref_chosen": -67.18333435058594,
"logps/ref_rejected": -82.80763244628906,
"logps/rejected": -82.83733367919922,
"loss": 1.3856,
"margin_dpo/margin_mean": 0.010176807641983032,
"margin_dpo/margin_std": 0.31959301233291626,
"step": 32
},
{
"KL/chosen_KL_mean": -0.026327133178710938,
"KL/mean": -0.03532563894987106,
"KL/rejected_KL_mean": -0.044322967529296875,
"KL/std": 0.258139967918396,
"epoch": 0.049886621315192746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.017992421984672546,
"fcm_dpo/q_t": 0.4995502829551697,
"grad_norm": 31.032793045043945,
"learning_rate": 2.388059701492537e-07,
"logits/chosen": 0.0272968877106905,
"logits/rejected": 0.0015150085091590881,
"logps/chosen": -64.06581115722656,
"logps/ref_chosen": -64.03948211669922,
"logps/ref_rejected": -75.68357849121094,
"logps/rejected": -75.7279052734375,
"loss": 1.3848,
"margin_dpo/margin_mean": 0.0179927796125412,
"margin_dpo/margin_std": 0.3556821942329407,
"step": 33
},
{
"KL/chosen_KL_mean": -0.025463104248046875,
"KL/mean": -0.06123238801956177,
"KL/rejected_KL_mean": -0.09700202941894531,
"KL/std": 0.20554988086223602,
"epoch": 0.05139833711262283,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07154315710067749,
"fcm_dpo/q_t": 0.49821144342422485,
"grad_norm": 28.452770233154297,
"learning_rate": 2.4626865671641786e-07,
"logits/chosen": 0.11309738457202911,
"logits/rejected": 0.08256950974464417,
"logps/chosen": -53.68975830078125,
"logps/ref_chosen": -53.6642951965332,
"logps/ref_rejected": -65.77989959716797,
"logps/rejected": -65.87689971923828,
"loss": 1.3793,
"margin_dpo/margin_mean": 0.07154309749603271,
"margin_dpo/margin_std": 0.269156813621521,
"step": 34
},
{
"KL/chosen_KL_mean": -0.09962081909179688,
"KL/mean": -0.09243303537368774,
"KL/rejected_KL_mean": -0.08524322509765625,
"KL/std": 0.23130691051483154,
"epoch": 0.05291005291005291,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01437610387802124,
"fcm_dpo/q_t": 0.5003600120544434,
"grad_norm": 27.835346221923828,
"learning_rate": 2.537313432835821e-07,
"logits/chosen": 0.05176283419132233,
"logits/rejected": 0.029404601082205772,
"logps/chosen": -61.116485595703125,
"logps/ref_chosen": -61.01686096191406,
"logps/ref_rejected": -72.78598022460938,
"logps/rejected": -72.87123107910156,
"loss": 1.388,
"margin_dpo/margin_mean": -0.014376223087310791,
"margin_dpo/margin_std": 0.30166250467300415,
"step": 35
},
{
"KL/chosen_KL_mean": -0.10001564025878906,
"KL/mean": -0.09694293141365051,
"KL/rejected_KL_mean": -0.0938720703125,
"KL/std": 0.2522987723350525,
"epoch": 0.05442176870748299,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.006146609783172607,
"fcm_dpo/q_t": 0.5001544952392578,
"grad_norm": 28.722679138183594,
"learning_rate": 2.611940298507462e-07,
"logits/chosen": 0.08311149477958679,
"logits/rejected": 0.031588129699230194,
"logps/chosen": -50.63737487792969,
"logps/ref_chosen": -50.53736114501953,
"logps/ref_rejected": -78.11678314208984,
"logps/rejected": -78.21065521240234,
"loss": 1.3873,
"margin_dpo/margin_mean": -0.006146460771560669,
"margin_dpo/margin_std": 0.3678821325302124,
"step": 36
},
{
"KL/chosen_KL_mean": -0.05405426025390625,
"KL/mean": -0.0969603955745697,
"KL/rejected_KL_mean": -0.13987350463867188,
"KL/std": 0.2613227665424347,
"epoch": 0.055933484504913075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08581572771072388,
"fcm_dpo/q_t": 0.4978557825088501,
"grad_norm": 37.60136413574219,
"learning_rate": 2.686567164179104e-07,
"logits/chosen": 0.09612583369016647,
"logits/rejected": 0.017332565039396286,
"logps/chosen": -59.608001708984375,
"logps/ref_chosen": -59.55394744873047,
"logps/ref_rejected": -108.27702331542969,
"logps/rejected": -108.41690063476562,
"loss": 1.3781,
"margin_dpo/margin_mean": 0.08581539988517761,
"margin_dpo/margin_std": 0.36218589544296265,
"step": 37
},
{
"KL/chosen_KL_mean": -0.07653617858886719,
"KL/mean": -0.09487441182136536,
"KL/rejected_KL_mean": -0.11320877075195312,
"KL/std": 0.23041307926177979,
"epoch": 0.05744520030234316,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03667086362838745,
"fcm_dpo/q_t": 0.499083936214447,
"grad_norm": 29.755857467651367,
"learning_rate": 2.761194029850746e-07,
"logits/chosen": 0.0761660635471344,
"logits/rejected": 0.0621175691485405,
"logps/chosen": -65.8648910522461,
"logps/ref_chosen": -65.78836059570312,
"logps/ref_rejected": -76.1619873046875,
"logps/rejected": -76.27519226074219,
"loss": 1.3829,
"margin_dpo/margin_mean": 0.03667101263999939,
"margin_dpo/margin_std": 0.30828237533569336,
"step": 38
},
{
"KL/chosen_KL_mean": -0.1410198211669922,
"KL/mean": -0.12570250034332275,
"KL/rejected_KL_mean": -0.110382080078125,
"KL/std": 0.28514018654823303,
"epoch": 0.05895691609977324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.030639827251434326,
"fcm_dpo/q_t": 0.5007643699645996,
"grad_norm": 29.475297927856445,
"learning_rate": 2.8358208955223876e-07,
"logits/chosen": 0.17340002954006195,
"logits/rejected": 0.14621543884277344,
"logps/chosen": -57.31782913208008,
"logps/ref_chosen": -57.17681121826172,
"logps/ref_rejected": -79.486328125,
"logps/rejected": -79.59671020507812,
"loss": 1.3897,
"margin_dpo/margin_mean": -0.03063926100730896,
"margin_dpo/margin_std": 0.3815447986125946,
"step": 39
},
{
"KL/chosen_KL_mean": -0.135894775390625,
"KL/mean": -0.1299644410610199,
"KL/rejected_KL_mean": -0.1240386962890625,
"KL/std": 0.2603451907634735,
"epoch": 0.06046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01186208426952362,
"fcm_dpo/q_t": 0.5002955198287964,
"grad_norm": 31.529159545898438,
"learning_rate": 2.9104477611940296e-07,
"logits/chosen": 0.11634434759616852,
"logits/rejected": 0.06645578145980835,
"logps/chosen": -61.470062255859375,
"logps/ref_chosen": -61.33416748046875,
"logps/ref_rejected": -79.10697174072266,
"logps/rejected": -79.23101043701172,
"loss": 1.3878,
"margin_dpo/margin_mean": -0.011862263083457947,
"margin_dpo/margin_std": 0.34227511286735535,
"step": 40
},
{
"KL/chosen_KL_mean": -0.101654052734375,
"KL/mean": -0.15321721136569977,
"KL/rejected_KL_mean": -0.20477676391601562,
"KL/std": 0.2765595316886902,
"epoch": 0.06198034769463341,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10311892628669739,
"fcm_dpo/q_t": 0.4974229335784912,
"grad_norm": 30.425931930541992,
"learning_rate": 2.985074626865671e-07,
"logits/chosen": 0.06092709302902222,
"logits/rejected": 0.040676962584257126,
"logps/chosen": -67.64838409423828,
"logps/ref_chosen": -67.5467300415039,
"logps/ref_rejected": -83.87788391113281,
"logps/rejected": -84.0826644897461,
"loss": 1.3763,
"margin_dpo/margin_mean": 0.10311943292617798,
"margin_dpo/margin_std": 0.34685975313186646,
"step": 41
},
{
"KL/chosen_KL_mean": -0.1368083953857422,
"KL/mean": -0.14792399108409882,
"KL/rejected_KL_mean": -0.15903854370117188,
"KL/std": 0.2869614362716675,
"epoch": 0.06349206349206349,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.022232264280319214,
"fcm_dpo/q_t": 0.49944519996643066,
"grad_norm": 29.266164779663086,
"learning_rate": 3.059701492537313e-07,
"logits/chosen": 0.07238396257162094,
"logits/rejected": 0.05022376403212547,
"logps/chosen": -61.40166473388672,
"logps/ref_chosen": -61.26485824584961,
"logps/ref_rejected": -76.3629150390625,
"logps/rejected": -76.52195739746094,
"loss": 1.3845,
"margin_dpo/margin_mean": 0.02223220467567444,
"margin_dpo/margin_std": 0.38739311695098877,
"step": 42
},
{
"KL/chosen_KL_mean": -0.08211898803710938,
"KL/mean": -0.10783997178077698,
"KL/rejected_KL_mean": -0.1335601806640625,
"KL/std": 0.28459784388542175,
"epoch": 0.06500377928949358,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.051439881324768066,
"fcm_dpo/q_t": 0.49871477484703064,
"grad_norm": 34.59758758544922,
"learning_rate": 3.134328358208955e-07,
"logits/chosen": 0.08342611789703369,
"logits/rejected": 0.07251065224409103,
"logps/chosen": -71.89114379882812,
"logps/ref_chosen": -71.80902862548828,
"logps/ref_rejected": -81.12464141845703,
"logps/rejected": -81.2582015991211,
"loss": 1.3815,
"margin_dpo/margin_mean": 0.051440536975860596,
"margin_dpo/margin_std": 0.35654395818710327,
"step": 43
},
{
"KL/chosen_KL_mean": -0.14848709106445312,
"KL/mean": -0.1632252335548401,
"KL/rejected_KL_mean": -0.17796707153320312,
"KL/std": 0.3100023865699768,
"epoch": 0.06651549508692366,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.029478788375854492,
"fcm_dpo/q_t": 0.4992641508579254,
"grad_norm": 32.544593811035156,
"learning_rate": 3.2089552238805965e-07,
"logits/chosen": 0.022504672408103943,
"logits/rejected": -0.006912057753652334,
"logps/chosen": -66.69891357421875,
"logps/ref_chosen": -66.55043029785156,
"logps/ref_rejected": -85.06198120117188,
"logps/rejected": -85.23994445800781,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.029478758573532104,
"margin_dpo/margin_std": 0.44489896297454834,
"step": 44
},
{
"KL/chosen_KL_mean": -0.1548919677734375,
"KL/mean": -0.20103472471237183,
"KL/rejected_KL_mean": -0.24718093872070312,
"KL/std": 0.2796245217323303,
"epoch": 0.06802721088435375,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09229524433612823,
"fcm_dpo/q_t": 0.4976927936077118,
"grad_norm": 31.892377853393555,
"learning_rate": 3.2835820895522385e-07,
"logits/chosen": 0.13807475566864014,
"logits/rejected": 0.08398568630218506,
"logps/chosen": -62.398746490478516,
"logps/ref_chosen": -62.24385452270508,
"logps/ref_rejected": -92.96665954589844,
"logps/rejected": -93.2138442993164,
"loss": 1.3774,
"margin_dpo/margin_mean": 0.09229600429534912,
"margin_dpo/margin_std": 0.3383764326572418,
"step": 45
},
{
"KL/chosen_KL_mean": -0.08540916442871094,
"KL/mean": -0.16423317790031433,
"KL/rejected_KL_mean": -0.24305343627929688,
"KL/std": 0.3250676989555359,
"epoch": 0.06953892668178382,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15764117240905762,
"fcm_dpo/q_t": 0.4960617423057556,
"grad_norm": 31.140756607055664,
"learning_rate": 3.3582089552238805e-07,
"logits/chosen": 0.12817460298538208,
"logits/rejected": 0.0819612368941307,
"logps/chosen": -61.584312438964844,
"logps/ref_chosen": -61.498905181884766,
"logps/ref_rejected": -78.91172790527344,
"logps/rejected": -79.15478515625,
"loss": 1.3711,
"margin_dpo/margin_mean": 0.15764087438583374,
"margin_dpo/margin_std": 0.43353208899497986,
"step": 46
},
{
"KL/chosen_KL_mean": -0.10969352722167969,
"KL/mean": -0.1963786482810974,
"KL/rejected_KL_mean": -0.2830657958984375,
"KL/std": 0.2953278422355652,
"epoch": 0.0710506424792139,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17337125539779663,
"fcm_dpo/q_t": 0.4956666827201843,
"grad_norm": 28.431907653808594,
"learning_rate": 3.432835820895522e-07,
"logits/chosen": 0.0386110283434391,
"logits/rejected": -0.0044650724157691,
"logps/chosen": -51.68804168701172,
"logps/ref_chosen": -51.578346252441406,
"logps/ref_rejected": -68.2215576171875,
"logps/rejected": -68.50462341308594,
"loss": 1.3694,
"margin_dpo/margin_mean": 0.17337137460708618,
"margin_dpo/margin_std": 0.39710623025894165,
"step": 47
},
{
"KL/chosen_KL_mean": -0.22099685668945312,
"KL/mean": -0.24017362296581268,
"KL/rejected_KL_mean": -0.2593536376953125,
"KL/std": 0.31508952379226685,
"epoch": 0.07256235827664399,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03835412859916687,
"fcm_dpo/q_t": 0.49904191493988037,
"grad_norm": 26.774696350097656,
"learning_rate": 3.507462686567164e-07,
"logits/chosen": 0.1500009000301361,
"logits/rejected": 0.12023096531629562,
"logps/chosen": -52.0146484375,
"logps/ref_chosen": -51.79365158081055,
"logps/ref_rejected": -64.22503662109375,
"logps/rejected": -64.48439025878906,
"loss": 1.3829,
"margin_dpo/margin_mean": 0.03835412859916687,
"margin_dpo/margin_std": 0.4297327399253845,
"step": 48
},
{
"KL/chosen_KL_mean": -0.18651199340820312,
"KL/mean": -0.25922006368637085,
"KL/rejected_KL_mean": -0.3319263458251953,
"KL/std": 0.33801430463790894,
"epoch": 0.07407407407407407,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1454138606786728,
"fcm_dpo/q_t": 0.49636557698249817,
"grad_norm": 27.162975311279297,
"learning_rate": 3.5820895522388055e-07,
"logits/chosen": 0.026051603257656097,
"logits/rejected": 0.004764057695865631,
"logps/chosen": -58.32111358642578,
"logps/ref_chosen": -58.13460159301758,
"logps/ref_rejected": -64.63206481933594,
"logps/rejected": -64.96399688720703,
"loss": 1.3723,
"margin_dpo/margin_mean": 0.14541450142860413,
"margin_dpo/margin_std": 0.43787360191345215,
"step": 49
},
{
"KL/chosen_KL_mean": -0.30684661865234375,
"KL/mean": -0.3562437891960144,
"KL/rejected_KL_mean": -0.4056396484375,
"KL/std": 0.35121792554855347,
"epoch": 0.07558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09879663586616516,
"fcm_dpo/q_t": 0.497530460357666,
"grad_norm": 28.057424545288086,
"learning_rate": 3.6567164179104475e-07,
"logits/chosen": 0.10507173836231232,
"logits/rejected": 0.07558847963809967,
"logps/chosen": -53.16328430175781,
"logps/ref_chosen": -52.85643768310547,
"logps/ref_rejected": -72.17460632324219,
"logps/rejected": -72.58024597167969,
"loss": 1.3768,
"margin_dpo/margin_mean": 0.09879684448242188,
"margin_dpo/margin_std": 0.38183924555778503,
"step": 50
},
{
"KL/chosen_KL_mean": -0.2800483703613281,
"KL/mean": -0.3949533998966217,
"KL/rejected_KL_mean": -0.5098648071289062,
"KL/std": 0.39111822843551636,
"epoch": 0.07709750566893424,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22981137037277222,
"fcm_dpo/q_t": 0.4942595958709717,
"grad_norm": 31.003662109375,
"learning_rate": 3.7313432835820895e-07,
"logits/chosen": 0.059184394776821136,
"logits/rejected": 0.03226791322231293,
"logps/chosen": -63.936492919921875,
"logps/ref_chosen": -63.65644073486328,
"logps/ref_rejected": -86.13229370117188,
"logps/rejected": -86.64216613769531,
"loss": 1.364,
"margin_dpo/margin_mean": 0.22981131076812744,
"margin_dpo/margin_std": 0.4713187515735626,
"step": 51
},
{
"KL/chosen_KL_mean": -0.3123207092285156,
"KL/mean": -0.42203959822654724,
"KL/rejected_KL_mean": -0.5317573547363281,
"KL/std": 0.42343559861183167,
"epoch": 0.07860922146636433,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21943794190883636,
"fcm_dpo/q_t": 0.49452149868011475,
"grad_norm": 32.59117126464844,
"learning_rate": 3.805970149253731e-07,
"logits/chosen": 0.08497665822505951,
"logits/rejected": 0.03561322018504143,
"logps/chosen": -68.15253448486328,
"logps/ref_chosen": -67.8402099609375,
"logps/ref_rejected": -96.97090911865234,
"logps/rejected": -97.50267028808594,
"loss": 1.3652,
"margin_dpo/margin_mean": 0.21943779289722443,
"margin_dpo/margin_std": 0.5385311841964722,
"step": 52
},
{
"KL/chosen_KL_mean": -0.3397502899169922,
"KL/mean": -0.4259885549545288,
"KL/rejected_KL_mean": -0.5122280120849609,
"KL/std": 0.391630619764328,
"epoch": 0.0801209372637944,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17248192429542542,
"fcm_dpo/q_t": 0.4956892132759094,
"grad_norm": 27.551605224609375,
"learning_rate": 3.880597014925373e-07,
"logits/chosen": 0.10184863209724426,
"logits/rejected": 0.09087856113910675,
"logps/chosen": -57.21788787841797,
"logps/ref_chosen": -56.87813949584961,
"logps/ref_rejected": -60.75569152832031,
"logps/rejected": -61.267921447753906,
"loss": 1.3698,
"margin_dpo/margin_mean": 0.17248201370239258,
"margin_dpo/margin_std": 0.4895680248737335,
"step": 53
},
{
"KL/chosen_KL_mean": -0.39040374755859375,
"KL/mean": -0.47420793771743774,
"KL/rejected_KL_mean": -0.5580081939697266,
"KL/std": 0.3961087465286255,
"epoch": 0.08163265306122448,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.16760483384132385,
"fcm_dpo/q_t": 0.4958149492740631,
"grad_norm": 26.449312210083008,
"learning_rate": 3.9552238805970144e-07,
"logits/chosen": 0.07982660830020905,
"logits/rejected": 0.06419865787029266,
"logps/chosen": -47.657325744628906,
"logps/ref_chosen": -47.26692199707031,
"logps/ref_rejected": -62.19426727294922,
"logps/rejected": -62.75227355957031,
"loss": 1.3703,
"margin_dpo/margin_mean": 0.16760462522506714,
"margin_dpo/margin_std": 0.5214763879776001,
"step": 54
},
{
"KL/chosen_KL_mean": -0.3712482452392578,
"KL/mean": -0.4966890513896942,
"KL/rejected_KL_mean": -0.62213134765625,
"KL/std": 0.4974730610847473,
"epoch": 0.08314436885865457,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2508871555328369,
"fcm_dpo/q_t": 0.493741512298584,
"grad_norm": 31.378108978271484,
"learning_rate": 4.0298507462686564e-07,
"logits/chosen": 0.03751315549015999,
"logits/rejected": -0.03892657905817032,
"logps/chosen": -50.69744110107422,
"logps/ref_chosen": -50.32619094848633,
"logps/ref_rejected": -92.44389343261719,
"logps/rejected": -93.06602478027344,
"loss": 1.3625,
"margin_dpo/margin_mean": 0.2508869171142578,
"margin_dpo/margin_std": 0.6618380546569824,
"step": 55
},
{
"KL/chosen_KL_mean": -0.41105079650878906,
"KL/mean": -0.4723691940307617,
"KL/rejected_KL_mean": -0.5336799621582031,
"KL/std": 0.5030519962310791,
"epoch": 0.08465608465608465,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12263241410255432,
"fcm_dpo/q_t": 0.49693045020103455,
"grad_norm": 27.65058708190918,
"learning_rate": 4.1044776119402984e-07,
"logits/chosen": 0.14740391075611115,
"logits/rejected": 0.12471330165863037,
"logps/chosen": -57.17802429199219,
"logps/ref_chosen": -56.766971588134766,
"logps/ref_rejected": -66.30504608154297,
"logps/rejected": -66.83872985839844,
"loss": 1.3752,
"margin_dpo/margin_mean": 0.12263259291648865,
"margin_dpo/margin_std": 0.6547993421554565,
"step": 56
},
{
"KL/chosen_KL_mean": -0.5095005035400391,
"KL/mean": -0.6674191951751709,
"KL/rejected_KL_mean": -0.8253402709960938,
"KL/std": 0.6238527297973633,
"epoch": 0.08616780045351474,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.315839022397995,
"fcm_dpo/q_t": 0.4921194911003113,
"grad_norm": 30.4954833984375,
"learning_rate": 4.17910447761194e-07,
"logits/chosen": 0.09336856752634048,
"logits/rejected": 0.029725098982453346,
"logps/chosen": -58.277244567871094,
"logps/ref_chosen": -57.76774597167969,
"logps/ref_rejected": -82.75698852539062,
"logps/rejected": -83.58232879638672,
"loss": 1.3562,
"margin_dpo/margin_mean": 0.3158394694328308,
"margin_dpo/margin_std": 0.6920211911201477,
"step": 57
},
{
"KL/chosen_KL_mean": -0.5488929748535156,
"KL/mean": -0.6903192400932312,
"KL/rejected_KL_mean": -0.8317451477050781,
"KL/std": 0.6478506326675415,
"epoch": 0.08767951625094482,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.282858282327652,
"fcm_dpo/q_t": 0.49298739433288574,
"grad_norm": 30.036136627197266,
"learning_rate": 4.253731343283582e-07,
"logits/chosen": 0.0658133402466774,
"logits/rejected": 0.05019402876496315,
"logps/chosen": -73.31298065185547,
"logps/ref_chosen": -72.76408386230469,
"logps/ref_rejected": -84.49275207519531,
"logps/rejected": -85.32449340820312,
"loss": 1.3607,
"margin_dpo/margin_mean": 0.28285834193229675,
"margin_dpo/margin_std": 0.9295713305473328,
"step": 58
},
{
"KL/chosen_KL_mean": -0.5454654693603516,
"KL/mean": -0.6809777021408081,
"KL/rejected_KL_mean": -0.8164939880371094,
"KL/std": 0.6243307590484619,
"epoch": 0.08919123204837491,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.27102532982826233,
"fcm_dpo/q_t": 0.49324700236320496,
"grad_norm": 26.647247314453125,
"learning_rate": 4.3283582089552234e-07,
"logits/chosen": 0.13273830711841583,
"logits/rejected": 0.0665852427482605,
"logps/chosen": -50.366241455078125,
"logps/ref_chosen": -49.820777893066406,
"logps/ref_rejected": -77.14368438720703,
"logps/rejected": -77.96017456054688,
"loss": 1.3612,
"margin_dpo/margin_mean": 0.2710251212120056,
"margin_dpo/margin_std": 0.7743154168128967,
"step": 59
},
{
"KL/chosen_KL_mean": -0.5872421264648438,
"KL/mean": -0.6434541940689087,
"KL/rejected_KL_mean": -0.6996688842773438,
"KL/std": 0.6261066198348999,
"epoch": 0.09070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11242368817329407,
"fcm_dpo/q_t": 0.4971911907196045,
"grad_norm": 29.517229080200195,
"learning_rate": 4.4029850746268654e-07,
"logits/chosen": 0.12953968346118927,
"logits/rejected": 0.12811188399791718,
"logps/chosen": -63.812015533447266,
"logps/ref_chosen": -63.22477340698242,
"logps/ref_rejected": -61.360477447509766,
"logps/rejected": -62.06014633178711,
"loss": 1.3769,
"margin_dpo/margin_mean": 0.11242353916168213,
"margin_dpo/margin_std": 0.850089430809021,
"step": 60
},
{
"KL/chosen_KL_mean": -0.7440834045410156,
"KL/mean": -0.7743111848831177,
"KL/rejected_KL_mean": -0.8045387268066406,
"KL/std": 0.6986474990844727,
"epoch": 0.09221466364323508,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06045445799827576,
"fcm_dpo/q_t": 0.4984866976737976,
"grad_norm": 28.10788917541504,
"learning_rate": 4.4776119402985074e-07,
"logits/chosen": 0.14267820119857788,
"logits/rejected": 0.1098393127322197,
"logps/chosen": -49.76087951660156,
"logps/ref_chosen": -49.01679992675781,
"logps/ref_rejected": -74.90817260742188,
"logps/rejected": -75.71270751953125,
"loss": 1.3827,
"margin_dpo/margin_mean": 0.06045445799827576,
"margin_dpo/margin_std": 0.9751712083816528,
"step": 61
},
{
"KL/chosen_KL_mean": -0.7253894805908203,
"KL/mean": -0.8400178551673889,
"KL/rejected_KL_mean": -0.9546432495117188,
"KL/std": 0.623093843460083,
"epoch": 0.09372637944066516,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.229254812002182,
"fcm_dpo/q_t": 0.4942789077758789,
"grad_norm": 28.548622131347656,
"learning_rate": 4.552238805970149e-07,
"logits/chosen": 0.13225194811820984,
"logits/rejected": 0.09242310374975204,
"logps/chosen": -63.477256774902344,
"logps/ref_chosen": -62.751869201660156,
"logps/ref_rejected": -78.93360900878906,
"logps/rejected": -79.88825225830078,
"loss": 1.3653,
"margin_dpo/margin_mean": 0.22925478219985962,
"margin_dpo/margin_std": 0.8522534370422363,
"step": 62
},
{
"KL/chosen_KL_mean": -0.5486431121826172,
"KL/mean": -0.8511289358139038,
"KL/rejected_KL_mean": -1.1536216735839844,
"KL/std": 0.7331008911132812,
"epoch": 0.09523809523809523,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6049745082855225,
"fcm_dpo/q_t": 0.48493385314941406,
"grad_norm": 31.141754150390625,
"learning_rate": 4.626865671641791e-07,
"logits/chosen": 0.18645809590816498,
"logits/rejected": 0.16159288585186005,
"logps/chosen": -61.06389236450195,
"logps/ref_chosen": -60.51525115966797,
"logps/ref_rejected": -85.11021423339844,
"logps/rejected": -86.26383972167969,
"loss": 1.3289,
"margin_dpo/margin_mean": 0.6049743890762329,
"margin_dpo/margin_std": 0.9378039836883545,
"step": 63
},
{
"KL/chosen_KL_mean": -0.8053035736083984,
"KL/mean": -0.84546959400177,
"KL/rejected_KL_mean": -0.8856353759765625,
"KL/std": 0.6875101327896118,
"epoch": 0.09674981103552532,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08033087849617004,
"fcm_dpo/q_t": 0.49798619747161865,
"grad_norm": 26.139707565307617,
"learning_rate": 4.701492537313433e-07,
"logits/chosen": 0.09058545529842377,
"logits/rejected": 0.06569143384695053,
"logps/chosen": -52.012149810791016,
"logps/ref_chosen": -51.20684814453125,
"logps/ref_rejected": -66.93081665039062,
"logps/rejected": -67.81645202636719,
"loss": 1.38,
"margin_dpo/margin_mean": 0.08033090829849243,
"margin_dpo/margin_std": 0.8200113773345947,
"step": 64
},
{
"KL/chosen_KL_mean": -0.8628063201904297,
"KL/mean": -1.1255594491958618,
"KL/rejected_KL_mean": -1.3883171081542969,
"KL/std": 0.9253497123718262,
"epoch": 0.0982615268329554,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5255085229873657,
"fcm_dpo/q_t": 0.48692017793655396,
"grad_norm": 30.805511474609375,
"learning_rate": 4.776119402985074e-07,
"logits/chosen": 0.22027479112148285,
"logits/rejected": 0.18991133570671082,
"logps/chosen": -68.15150451660156,
"logps/ref_chosen": -67.2886962890625,
"logps/ref_rejected": -74.44281005859375,
"logps/rejected": -75.83112335205078,
"loss": 1.338,
"margin_dpo/margin_mean": 0.5255081057548523,
"margin_dpo/margin_std": 1.190438985824585,
"step": 65
},
{
"KL/chosen_KL_mean": -0.9165191650390625,
"KL/mean": -1.099397897720337,
"KL/rejected_KL_mean": -1.2822761535644531,
"KL/std": 0.8203333616256714,
"epoch": 0.09977324263038549,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3657585382461548,
"fcm_dpo/q_t": 0.4908965229988098,
"grad_norm": 29.26482582092285,
"learning_rate": 4.850746268656717e-07,
"logits/chosen": 0.09936878085136414,
"logits/rejected": 0.07552456855773926,
"logps/chosen": -71.65992736816406,
"logps/ref_chosen": -70.743408203125,
"logps/ref_rejected": -77.26499938964844,
"logps/rejected": -78.54727935791016,
"loss": 1.353,
"margin_dpo/margin_mean": 0.3657584488391876,
"margin_dpo/margin_std": 1.0532437562942505,
"step": 66
},
{
"KL/chosen_KL_mean": -0.7958889007568359,
"KL/mean": -0.9142911434173584,
"KL/rejected_KL_mean": -1.0326957702636719,
"KL/std": 0.7134413123130798,
"epoch": 0.10128495842781557,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23680096864700317,
"fcm_dpo/q_t": 0.49409249424934387,
"grad_norm": 29.31564712524414,
"learning_rate": 4.925373134328357e-07,
"logits/chosen": 0.07396933436393738,
"logits/rejected": 0.01928994059562683,
"logps/chosen": -61.39849090576172,
"logps/ref_chosen": -60.60260009765625,
"logps/ref_rejected": -75.22235870361328,
"logps/rejected": -76.25505065917969,
"loss": 1.3646,
"margin_dpo/margin_mean": 0.23680099844932556,
"margin_dpo/margin_std": 0.8555749654769897,
"step": 67
},
{
"KL/chosen_KL_mean": -1.2452201843261719,
"KL/mean": -1.4087040424346924,
"KL/rejected_KL_mean": -1.5721893310546875,
"KL/std": 0.9630727767944336,
"epoch": 0.10279667422524566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3269740343093872,
"fcm_dpo/q_t": 0.49186715483665466,
"grad_norm": 31.29288101196289,
"learning_rate": 5e-07,
"logits/chosen": 0.08105947077274323,
"logits/rejected": 0.05047762021422386,
"logps/chosen": -78.77359008789062,
"logps/ref_chosen": -77.52836608886719,
"logps/ref_rejected": -93.17778015136719,
"logps/rejected": -94.74996948242188,
"loss": 1.358,
"margin_dpo/margin_mean": 0.3269742727279663,
"margin_dpo/margin_std": 1.267851710319519,
"step": 68
},
{
"KL/chosen_KL_mean": -1.13970947265625,
"KL/mean": -1.466085433959961,
"KL/rejected_KL_mean": -1.7924575805664062,
"KL/std": 1.0306477546691895,
"epoch": 0.10430839002267574,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6527523994445801,
"fcm_dpo/q_t": 0.4837765097618103,
"grad_norm": 30.93454933166504,
"learning_rate": 4.999965034812934e-07,
"logits/chosen": 0.0892995297908783,
"logits/rejected": 0.04694270342588425,
"logps/chosen": -67.082763671875,
"logps/ref_chosen": -65.94305419921875,
"logps/ref_rejected": -89.7735595703125,
"logps/rejected": -91.5660171508789,
"loss": 1.3259,
"margin_dpo/margin_mean": 0.6527519226074219,
"margin_dpo/margin_std": 1.201098918914795,
"step": 69
},
{
"KL/chosen_KL_mean": -1.238992691040039,
"KL/mean": -1.4067550897598267,
"KL/rejected_KL_mean": -1.5745105743408203,
"KL/std": 0.8888180255889893,
"epoch": 0.10582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.33551764488220215,
"fcm_dpo/q_t": 0.49164047837257385,
"grad_norm": 28.678653717041016,
"learning_rate": 4.999860140229787e-07,
"logits/chosen": 0.15405255556106567,
"logits/rejected": 0.13066557049751282,
"logps/chosen": -63.19690704345703,
"logps/ref_chosen": -61.95791244506836,
"logps/ref_rejected": -75.80945587158203,
"logps/rejected": -77.38397216796875,
"loss": 1.3565,
"margin_dpo/margin_mean": 0.3355180025100708,
"margin_dpo/margin_std": 1.1736544370651245,
"step": 70
},
{
"KL/chosen_KL_mean": -1.4223155975341797,
"KL/mean": -1.555302381515503,
"KL/rejected_KL_mean": -1.6882915496826172,
"KL/std": 1.0254878997802734,
"epoch": 0.1073318216175359,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.26597607135772705,
"fcm_dpo/q_t": 0.49335426092147827,
"grad_norm": 28.391786575317383,
"learning_rate": 4.999685319184688e-07,
"logits/chosen": 0.09895138442516327,
"logits/rejected": 0.08333157747983932,
"logps/chosen": -64.7698974609375,
"logps/ref_chosen": -63.34757995605469,
"logps/ref_rejected": -67.49658203125,
"logps/rejected": -69.18487548828125,
"loss": 1.3653,
"margin_dpo/margin_mean": 0.2659757733345032,
"margin_dpo/margin_std": 1.4624427556991577,
"step": 71
},
{
"KL/chosen_KL_mean": -1.1982994079589844,
"KL/mean": -1.54570472240448,
"KL/rejected_KL_mean": -1.8931045532226562,
"KL/std": 1.0516983270645142,
"epoch": 0.10884353741496598,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6948050260543823,
"fcm_dpo/q_t": 0.4827173948287964,
"grad_norm": 30.34983253479004,
"learning_rate": 4.999440576567755e-07,
"logits/chosen": 0.08275315165519714,
"logits/rejected": 0.022004300728440285,
"logps/chosen": -57.05759811401367,
"logps/ref_chosen": -55.85929870605469,
"logps/ref_rejected": -68.45423889160156,
"logps/rejected": -70.34735107421875,
"loss": 1.3223,
"margin_dpo/margin_mean": 0.6948049068450928,
"margin_dpo/margin_std": 1.2801237106323242,
"step": 72
},
{
"KL/chosen_KL_mean": -1.8044281005859375,
"KL/mean": -1.856170892715454,
"KL/rejected_KL_mean": -1.9079151153564453,
"KL/std": 1.1962889432907104,
"epoch": 0.11035525321239607,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10349225997924805,
"fcm_dpo/q_t": 0.49755388498306274,
"grad_norm": 32.140106201171875,
"learning_rate": 4.999125919224965e-07,
"logits/chosen": 0.10440310835838318,
"logits/rejected": 0.09044543653726578,
"logps/chosen": -70.9432373046875,
"logps/ref_chosen": -69.13880920410156,
"logps/ref_rejected": -79.04586791992188,
"logps/rejected": -80.95378112792969,
"loss": 1.3822,
"margin_dpo/margin_mean": 0.10349264740943909,
"margin_dpo/margin_std": 1.502673625946045,
"step": 73
},
{
"KL/chosen_KL_mean": -1.2395572662353516,
"KL/mean": -1.5856974124908447,
"KL/rejected_KL_mean": -1.9318351745605469,
"KL/std": 1.0054885149002075,
"epoch": 0.11186696900982615,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6922777891159058,
"fcm_dpo/q_t": 0.48279356956481934,
"grad_norm": 28.01102638244629,
"learning_rate": 4.998741355957963e-07,
"logits/chosen": 0.13102754950523376,
"logits/rejected": 0.07868199050426483,
"logps/chosen": -51.163291931152344,
"logps/ref_chosen": -49.923736572265625,
"logps/ref_rejected": -81.73213958740234,
"logps/rejected": -83.66397094726562,
"loss": 1.3225,
"margin_dpo/margin_mean": 0.6922776699066162,
"margin_dpo/margin_std": 1.2654963731765747,
"step": 74
},
{
"KL/chosen_KL_mean": -1.4122562408447266,
"KL/mean": -1.7794251441955566,
"KL/rejected_KL_mean": -2.1465892791748047,
"KL/std": 1.0802136659622192,
"epoch": 0.11337868480725624,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.734336793422699,
"fcm_dpo/q_t": 0.4817506670951843,
"grad_norm": 26.342166900634766,
"learning_rate": 4.998286897523808e-07,
"logits/chosen": 0.10357685387134552,
"logits/rejected": 0.07225304841995239,
"logps/chosen": -47.48101043701172,
"logps/ref_chosen": -46.06875228881836,
"logps/ref_rejected": -66.1181411743164,
"logps/rejected": -68.26473236083984,
"loss": 1.3185,
"margin_dpo/margin_mean": 0.7343367338180542,
"margin_dpo/margin_std": 1.30611252784729,
"step": 75
},
{
"KL/chosen_KL_mean": -1.6146888732910156,
"KL/mean": -1.7545742988586426,
"KL/rejected_KL_mean": -1.8944587707519531,
"KL/std": 1.1566370725631714,
"epoch": 0.11489040060468632,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2797727584838867,
"fcm_dpo/q_t": 0.49302801489830017,
"grad_norm": 29.376850128173828,
"learning_rate": 4.997762556634679e-07,
"logits/chosen": 0.13626576960086823,
"logits/rejected": 0.09158352017402649,
"logps/chosen": -55.67743682861328,
"logps/ref_chosen": -54.06275177001953,
"logps/ref_rejected": -74.87464141845703,
"logps/rejected": -76.76910400390625,
"loss": 1.3642,
"margin_dpo/margin_mean": 0.2797728478908539,
"margin_dpo/margin_std": 1.5069594383239746,
"step": 76
},
{
"KL/chosen_KL_mean": -1.6369781494140625,
"KL/mean": -2.009381055831909,
"KL/rejected_KL_mean": -2.3817825317382812,
"KL/std": 1.2048678398132324,
"epoch": 0.1164021164021164,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7448074817657471,
"fcm_dpo/q_t": 0.48149538040161133,
"grad_norm": 29.28656768798828,
"learning_rate": 4.99716834795752e-07,
"logits/chosen": 0.1588551104068756,
"logits/rejected": 0.11727854609489441,
"logps/chosen": -54.713069915771484,
"logps/ref_chosen": -53.07609176635742,
"logps/ref_rejected": -74.45601654052734,
"logps/rejected": -76.83779907226562,
"loss": 1.3183,
"margin_dpo/margin_mean": 0.7448074817657471,
"margin_dpo/margin_std": 1.3909995555877686,
"step": 77
},
{
"KL/chosen_KL_mean": -1.8279800415039062,
"KL/mean": -2.0209426879882812,
"KL/rejected_KL_mean": -2.2139129638671875,
"KL/std": 1.2771286964416504,
"epoch": 0.11791383219954649,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.38592809438705444,
"fcm_dpo/q_t": 0.490395188331604,
"grad_norm": 29.480390548706055,
"learning_rate": 4.996504288113623e-07,
"logits/chosen": 0.10726994276046753,
"logits/rejected": 0.08700601756572723,
"logps/chosen": -69.55340576171875,
"logps/ref_chosen": -67.72541809082031,
"logps/ref_rejected": -79.03926849365234,
"logps/rejected": -81.25318145751953,
"loss": 1.3534,
"margin_dpo/margin_mean": 0.3859279155731201,
"margin_dpo/margin_std": 1.4653525352478027,
"step": 78
},
{
"KL/chosen_KL_mean": -1.7102622985839844,
"KL/mean": -2.2839438915252686,
"KL/rejected_KL_mean": -2.8576278686523438,
"KL/std": 1.5375041961669922,
"epoch": 0.11942554799697656,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1473674774169922,
"fcm_dpo/q_t": 0.4718601703643799,
"grad_norm": 30.79814910888672,
"learning_rate": 4.995770395678171e-07,
"logits/chosen": 0.1881793886423111,
"logits/rejected": 0.12662510573863983,
"logps/chosen": -53.870906829833984,
"logps/ref_chosen": -52.16064453125,
"logps/ref_rejected": -83.31062316894531,
"logps/rejected": -86.16824340820312,
"loss": 1.2847,
"margin_dpo/margin_mean": 1.1473678350448608,
"margin_dpo/margin_std": 1.9917948246002197,
"step": 79
},
{
"KL/chosen_KL_mean": -2.041339874267578,
"KL/mean": -2.3201375007629395,
"KL/rejected_KL_mean": -2.5989341735839844,
"KL/std": 1.3568034172058105,
"epoch": 0.12093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5575928688049316,
"fcm_dpo/q_t": 0.4862174987792969,
"grad_norm": 28.497337341308594,
"learning_rate": 4.994966691179711e-07,
"logits/chosen": 0.14524391293525696,
"logits/rejected": 0.08540673553943634,
"logps/chosen": -63.451900482177734,
"logps/ref_chosen": -61.410560607910156,
"logps/ref_rejected": -78.66004943847656,
"logps/rejected": -81.25898742675781,
"loss": 1.3409,
"margin_dpo/margin_mean": 0.5575932264328003,
"margin_dpo/margin_std": 1.8942968845367432,
"step": 80
},
{
"KL/chosen_KL_mean": -2.136859893798828,
"KL/mean": -2.5757455825805664,
"KL/rejected_KL_mean": -3.0146255493164062,
"KL/std": 1.4461383819580078,
"epoch": 0.12244897959183673,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8777664303779602,
"fcm_dpo/q_t": 0.47836926579475403,
"grad_norm": 28.98223876953125,
"learning_rate": 4.994093197099587e-07,
"logits/chosen": 0.13092893362045288,
"logits/rejected": 0.09700986742973328,
"logps/chosen": -65.94123077392578,
"logps/ref_chosen": -63.80437088012695,
"logps/ref_rejected": -79.3484115600586,
"logps/rejected": -82.363037109375,
"loss": 1.3086,
"margin_dpo/margin_mean": 0.8777658939361572,
"margin_dpo/margin_std": 1.7367991209030151,
"step": 81
},
{
"KL/chosen_KL_mean": -1.7457637786865234,
"KL/mean": -2.286520481109619,
"KL/rejected_KL_mean": -2.8272781372070312,
"KL/std": 1.3924403190612793,
"epoch": 0.12396069538926682,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.081512451171875,
"fcm_dpo/q_t": 0.4731561541557312,
"grad_norm": 26.511188507080078,
"learning_rate": 4.993149937871306e-07,
"logits/chosen": 0.10917188227176666,
"logits/rejected": 0.04636561498045921,
"logps/chosen": -50.563655853271484,
"logps/ref_chosen": -48.817893981933594,
"logps/ref_rejected": -70.31497955322266,
"logps/rejected": -73.14225769042969,
"loss": 1.2874,
"margin_dpo/margin_mean": 1.081512212753296,
"margin_dpo/margin_std": 1.579821228981018,
"step": 82
},
{
"KL/chosen_KL_mean": -2.0326290130615234,
"KL/mean": -2.6406869888305664,
"KL/rejected_KL_mean": -3.248748779296875,
"KL/std": 1.456777811050415,
"epoch": 0.1254724111866969,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2161188125610352,
"fcm_dpo/q_t": 0.4699665307998657,
"grad_norm": 29.091482162475586,
"learning_rate": 4.992136939879856e-07,
"logits/chosen": 0.22049690783023834,
"logits/rejected": 0.16898763179779053,
"logps/chosen": -59.18340301513672,
"logps/ref_chosen": -57.15077209472656,
"logps/ref_rejected": -75.1710205078125,
"logps/rejected": -78.41976928710938,
"loss": 1.2762,
"margin_dpo/margin_mean": 1.216118335723877,
"margin_dpo/margin_std": 1.7663187980651855,
"step": 83
},
{
"KL/chosen_KL_mean": -2.5208396911621094,
"KL/mean": -2.948389768600464,
"KL/rejected_KL_mean": -3.375946044921875,
"KL/std": 1.8345149755477905,
"epoch": 0.12698412698412698,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.855107307434082,
"fcm_dpo/q_t": 0.47885656356811523,
"grad_norm": 30.742326736450195,
"learning_rate": 4.991054231460969e-07,
"logits/chosen": 0.14221562445163727,
"logits/rejected": 0.10221521556377411,
"logps/chosen": -67.29814147949219,
"logps/ref_chosen": -64.77729797363281,
"logps/ref_rejected": -84.71949768066406,
"logps/rejected": -88.09544372558594,
"loss": 1.3119,
"margin_dpo/margin_mean": 0.8551075458526611,
"margin_dpo/margin_std": 1.9248778820037842,
"step": 84
},
{
"KL/chosen_KL_mean": -2.2519493103027344,
"KL/mean": -2.8956236839294434,
"KL/rejected_KL_mean": -3.5392990112304688,
"KL/std": 1.7731411457061768,
"epoch": 0.12849584278155707,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2873495817184448,
"fcm_dpo/q_t": 0.468211829662323,
"grad_norm": 26.718862533569336,
"learning_rate": 4.989901842900325e-07,
"logits/chosen": 0.16839201748371124,
"logits/rejected": 0.12411106377840042,
"logps/chosen": -52.50364685058594,
"logps/ref_chosen": -50.25169372558594,
"logps/ref_rejected": -66.55439758300781,
"logps/rejected": -70.09368896484375,
"loss": 1.272,
"margin_dpo/margin_mean": 1.2873497009277344,
"margin_dpo/margin_std": 1.9850984811782837,
"step": 85
},
{
"KL/chosen_KL_mean": -2.819011688232422,
"KL/mean": -3.3019821643829346,
"KL/rejected_KL_mean": -3.7849502563476562,
"KL/std": 1.8966737985610962,
"epoch": 0.13000755857898716,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9659372568130493,
"fcm_dpo/q_t": 0.4761325418949127,
"grad_norm": 27.210853576660156,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 0.21716086566448212,
"logits/rejected": 0.19776871800422668,
"logps/chosen": -63.54819107055664,
"logps/ref_chosen": -60.72917938232422,
"logps/ref_rejected": -72.30961608886719,
"logps/rejected": -76.09455871582031,
"loss": 1.3025,
"margin_dpo/margin_mean": 0.9659373760223389,
"margin_dpo/margin_std": 2.055220603942871,
"step": 86
},
{
"KL/chosen_KL_mean": -2.9924278259277344,
"KL/mean": -3.5514774322509766,
"KL/rejected_KL_mean": -4.110523223876953,
"KL/std": 2.083639144897461,
"epoch": 0.13151927437641722,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1181044578552246,
"fcm_dpo/q_t": 0.47244584560394287,
"grad_norm": 30.3636474609375,
"learning_rate": 4.987388156241114e-07,
"logits/chosen": 0.1789131462574005,
"logits/rejected": 0.11889512836933136,
"logps/chosen": -68.75039672851562,
"logps/ref_chosen": -65.75796508789062,
"logps/ref_rejected": -84.81159973144531,
"logps/rejected": -88.922119140625,
"loss": 1.2933,
"margin_dpo/margin_mean": 1.1181050539016724,
"margin_dpo/margin_std": 2.491665840148926,
"step": 87
},
{
"KL/chosen_KL_mean": -3.064952850341797,
"KL/mean": -3.4871764183044434,
"KL/rejected_KL_mean": -3.9093971252441406,
"KL/std": 2.3427746295928955,
"epoch": 0.1330309901738473,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.84444260597229,
"fcm_dpo/q_t": 0.4793391227722168,
"grad_norm": 30.429597854614258,
"learning_rate": 4.986026928455767e-07,
"logits/chosen": 0.24011830985546112,
"logits/rejected": 0.21302275359630585,
"logps/chosen": -65.88897705078125,
"logps/ref_chosen": -62.82402801513672,
"logps/ref_rejected": -74.9607162475586,
"logps/rejected": -78.8701171875,
"loss": 1.327,
"margin_dpo/margin_mean": 0.84444260597229,
"margin_dpo/margin_std": 2.9619760513305664,
"step": 88
},
{
"KL/chosen_KL_mean": -2.5539302825927734,
"KL/mean": -3.2578611373901367,
"KL/rejected_KL_mean": -3.9617958068847656,
"KL/std": 2.1723883152008057,
"epoch": 0.1345427059712774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4078590869903564,
"fcm_dpo/q_t": 0.4656612277030945,
"grad_norm": 29.212039947509766,
"learning_rate": 4.984596161153135e-07,
"logits/chosen": 0.24471764266490936,
"logits/rejected": 0.16260842978954315,
"logps/chosen": -43.745365142822266,
"logps/ref_chosen": -41.191436767578125,
"logps/ref_rejected": -85.44769287109375,
"logps/rejected": -89.40949249267578,
"loss": 1.2662,
"margin_dpo/margin_mean": 1.4078595638275146,
"margin_dpo/margin_std": 2.449413776397705,
"step": 89
},
{
"KL/chosen_KL_mean": -3.128904342651367,
"KL/mean": -3.782576560974121,
"KL/rejected_KL_mean": -4.436252593994141,
"KL/std": 2.440657615661621,
"epoch": 0.1360544217687075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3073492050170898,
"fcm_dpo/q_t": 0.4679213762283325,
"grad_norm": 29.046615600585938,
"learning_rate": 4.983095894354857e-07,
"logits/chosen": 0.1570890247821808,
"logits/rejected": 0.10190249979496002,
"logps/chosen": -59.71281433105469,
"logps/ref_chosen": -56.58390808105469,
"logps/ref_rejected": -86.86978149414062,
"logps/rejected": -91.3060302734375,
"loss": 1.276,
"margin_dpo/margin_mean": 1.307349443435669,
"margin_dpo/margin_std": 2.514946460723877,
"step": 90
},
{
"KL/chosen_KL_mean": -3.1567344665527344,
"KL/mean": -3.703597068786621,
"KL/rejected_KL_mean": -4.250457763671875,
"KL/std": 2.2865095138549805,
"epoch": 0.13756613756613756,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0937232971191406,
"fcm_dpo/q_t": 0.47355741262435913,
"grad_norm": 25.90226936340332,
"learning_rate": 4.98152617002662e-07,
"logits/chosen": 0.15374761819839478,
"logits/rejected": 0.11025101691484451,
"logps/chosen": -55.53907775878906,
"logps/ref_chosen": -52.38234329223633,
"logps/ref_rejected": -72.17642211914062,
"logps/rejected": -76.4268798828125,
"loss": 1.2993,
"margin_dpo/margin_mean": 1.0937230587005615,
"margin_dpo/margin_std": 2.792675018310547,
"step": 91
},
{
"KL/chosen_KL_mean": -3.2463722229003906,
"KL/mean": -4.036555290222168,
"KL/rejected_KL_mean": -4.826740264892578,
"KL/std": 2.6802730560302734,
"epoch": 0.13907785336356765,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5803602933883667,
"fcm_dpo/q_t": 0.4618731141090393,
"grad_norm": 27.01513671875,
"learning_rate": 4.979887032076988e-07,
"logits/chosen": 0.22086292505264282,
"logits/rejected": 0.18025824427604675,
"logps/chosen": -56.25507354736328,
"logps/ref_chosen": -53.00870132446289,
"logps/ref_rejected": -79.77812957763672,
"logps/rejected": -84.60487365722656,
"loss": 1.2586,
"margin_dpo/margin_mean": 1.5803604125976562,
"margin_dpo/margin_std": 3.117692470550537,
"step": 92
},
{
"KL/chosen_KL_mean": -3.5561141967773438,
"KL/mean": -4.080088138580322,
"KL/rejected_KL_mean": -4.604063034057617,
"KL/std": 2.6413497924804688,
"epoch": 0.14058956916099774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0479507446289062,
"fcm_dpo/q_t": 0.47497549653053284,
"grad_norm": 24.47339630126953,
"learning_rate": 4.978178526356172e-07,
"logits/chosen": 0.18242627382278442,
"logits/rejected": 0.15430913865566254,
"logps/chosen": -48.463165283203125,
"logps/ref_chosen": -44.90705108642578,
"logps/ref_rejected": -58.7879524230957,
"logps/rejected": -63.39201354980469,
"loss": 1.3097,
"margin_dpo/margin_mean": 1.0479507446289062,
"margin_dpo/margin_std": 3.2303943634033203,
"step": 93
},
{
"KL/chosen_KL_mean": -3.2976722717285156,
"KL/mean": -4.257216453552246,
"KL/rejected_KL_mean": -5.216762542724609,
"KL/std": 3.2533302307128906,
"epoch": 0.1421012849584278,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9190917015075684,
"fcm_dpo/q_t": 0.45400530099868774,
"grad_norm": 27.097318649291992,
"learning_rate": 4.976400700654751e-07,
"logits/chosen": 0.2787092328071594,
"logits/rejected": 0.23703975975513458,
"logps/chosen": -63.235443115234375,
"logps/ref_chosen": -59.93777084350586,
"logps/ref_rejected": -79.3138427734375,
"logps/rejected": -84.53060150146484,
"loss": 1.2433,
"margin_dpo/margin_mean": 1.9190914630889893,
"margin_dpo/margin_std": 4.036993026733398,
"step": 94
},
{
"KL/chosen_KL_mean": -4.280851364135742,
"KL/mean": -5.057154655456543,
"KL/rejected_KL_mean": -5.833454132080078,
"KL/std": 2.690345525741577,
"epoch": 0.1436130007558579,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.552598476409912,
"fcm_dpo/q_t": 0.46246445178985596,
"grad_norm": 29.381898880004883,
"learning_rate": 4.974553604702332e-07,
"logits/chosen": 0.1680109202861786,
"logits/rejected": 0.10245092958211899,
"logps/chosen": -64.4493408203125,
"logps/ref_chosen": -60.168487548828125,
"logps/ref_rejected": -90.73665618896484,
"logps/rejected": -96.57011413574219,
"loss": 1.2654,
"margin_dpo/margin_mean": 1.552598237991333,
"margin_dpo/margin_std": 3.3945610523223877,
"step": 95
},
{
"KL/chosen_KL_mean": -4.168708801269531,
"KL/mean": -5.101164817810059,
"KL/rejected_KL_mean": -6.033618927001953,
"KL/std": 3.3781328201293945,
"epoch": 0.14512471655328799,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8649134635925293,
"fcm_dpo/q_t": 0.4566536247730255,
"grad_norm": 27.34356117248535,
"learning_rate": 4.972637290166157e-07,
"logits/chosen": 0.20887598395347595,
"logits/rejected": 0.16518929600715637,
"logps/chosen": -64.83748626708984,
"logps/ref_chosen": -60.66877746582031,
"logps/ref_rejected": -88.30673217773438,
"logps/rejected": -94.34034729003906,
"loss": 1.2566,
"margin_dpo/margin_mean": 1.8649135828018188,
"margin_dpo/margin_std": 4.4204630851745605,
"step": 96
},
{
"KL/chosen_KL_mean": -5.003776550292969,
"KL/mean": -5.500189304351807,
"KL/rejected_KL_mean": -5.996604919433594,
"KL/std": 3.620288848876953,
"epoch": 0.14663643235071808,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.992835283279419,
"fcm_dpo/q_t": 0.4764229357242584,
"grad_norm": 36.290287017822266,
"learning_rate": 4.970651810649666e-07,
"logits/chosen": 0.15075242519378662,
"logits/rejected": 0.10647913813591003,
"logps/chosen": -70.04789733886719,
"logps/ref_chosen": -65.04412078857422,
"logps/ref_rejected": -78.42092895507812,
"logps/rejected": -84.41753387451172,
"loss": 1.3485,
"margin_dpo/margin_mean": 0.9928349256515503,
"margin_dpo/margin_std": 4.941525459289551,
"step": 97
},
{
"KL/chosen_KL_mean": -4.657588958740234,
"KL/mean": -5.016092300415039,
"KL/rejected_KL_mean": -5.374599456787109,
"KL/std": 3.3176703453063965,
"epoch": 0.14814814814814814,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7170079946517944,
"fcm_dpo/q_t": 0.48207545280456543,
"grad_norm": 30.212032318115234,
"learning_rate": 4.968597221690985e-07,
"logits/chosen": 0.2352330982685089,
"logits/rejected": 0.20778122544288635,
"logps/chosen": -60.16082000732422,
"logps/ref_chosen": -55.503231048583984,
"logps/ref_rejected": -72.81553649902344,
"logps/rejected": -78.19013977050781,
"loss": 1.3422,
"margin_dpo/margin_mean": 0.7170075178146362,
"margin_dpo/margin_std": 3.2238073348999023,
"step": 98
},
{
"KL/chosen_KL_mean": -4.7126617431640625,
"KL/mean": -5.202777862548828,
"KL/rejected_KL_mean": -5.692897796630859,
"KL/std": 3.4897851943969727,
"epoch": 0.14965986394557823,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9802349209785461,
"fcm_dpo/q_t": 0.47680675983428955,
"grad_norm": 32.75977325439453,
"learning_rate": 4.966473580761389e-07,
"logits/chosen": 0.26266586780548096,
"logits/rejected": 0.22229918837547302,
"logps/chosen": -63.288299560546875,
"logps/ref_chosen": -58.57563781738281,
"logps/ref_rejected": -78.693603515625,
"logps/rejected": -84.38650512695312,
"loss": 1.3412,
"margin_dpo/margin_mean": 0.9802342057228088,
"margin_dpo/margin_std": 4.541085720062256,
"step": 99
},
{
"KL/chosen_KL_mean": -5.3377838134765625,
"KL/mean": -6.065300941467285,
"KL/rejected_KL_mean": -6.792819976806641,
"KL/std": 4.165156841278076,
"epoch": 0.15117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.455038070678711,
"fcm_dpo/q_t": 0.4675733149051666,
"grad_norm": 33.132652282714844,
"learning_rate": 4.964280947263676e-07,
"logits/chosen": 0.24497297406196594,
"logits/rejected": 0.23685091733932495,
"logps/chosen": -84.92121887207031,
"logps/ref_chosen": -79.58343505859375,
"logps/ref_rejected": -92.152587890625,
"logps/rejected": -98.94540405273438,
"loss": 1.3314,
"margin_dpo/margin_mean": 1.4550374746322632,
"margin_dpo/margin_std": 5.805309772491455,
"step": 100
},
{
"KL/chosen_KL_mean": -4.292753219604492,
"KL/mean": -5.332766532897949,
"KL/rejected_KL_mean": -6.372776031494141,
"KL/std": 3.6866824626922607,
"epoch": 0.15268329554043839,
"fcm_dpo/beta": 0.10244277864694595,
"fcm_dpo/delta": 0.07949934154748917,
"fcm_dpo/margin": 2.0800251960754395,
"fcm_dpo/q_t": 0.4498664438724518,
"grad_norm": 26.55908203125,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": 0.2531740963459015,
"logits/rejected": 0.21294589340686798,
"logps/chosen": -56.62554168701172,
"logps/ref_chosen": -52.332786560058594,
"logps/ref_rejected": -69.55589294433594,
"logps/rejected": -75.92867279052734,
"loss": 1.2299,
"margin_dpo/margin_mean": 2.0800249576568604,
"margin_dpo/margin_std": 4.159194469451904,
"step": 101
},
{
"KL/chosen_KL_mean": -5.356512069702148,
"KL/mean": -5.766068458557129,
"KL/rejected_KL_mean": -6.175632476806641,
"KL/std": 3.7042946815490723,
"epoch": 0.15419501133786848,
"fcm_dpo/beta": 0.10488555580377579,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8191168308258057,
"fcm_dpo/q_t": 0.4826941192150116,
"grad_norm": 34.83320617675781,
"learning_rate": 4.959688949822748e-07,
"logits/chosen": 0.17034727334976196,
"logits/rejected": 0.13114404678344727,
"logps/chosen": -70.09999084472656,
"logps/ref_chosen": -64.74348449707031,
"logps/ref_rejected": -69.06132507324219,
"logps/rejected": -75.2369613647461,
"loss": 1.38,
"margin_dpo/margin_mean": 0.8191161751747131,
"margin_dpo/margin_std": 5.501483917236328,
"step": 102
},
{
"KL/chosen_KL_mean": -4.785610198974609,
"KL/mean": -5.598519325256348,
"KL/rejected_KL_mean": -6.411426544189453,
"KL/std": 3.8898379802703857,
"epoch": 0.15570672713529857,
"fcm_dpo/beta": 0.10772866755723953,
"fcm_dpo/delta": 0.08799183368682861,
"fcm_dpo/margin": 1.6258132457733154,
"fcm_dpo/q_t": 0.459196537733078,
"grad_norm": 31.35845184326172,
"learning_rate": 4.957289714327572e-07,
"logits/chosen": 0.26058486104011536,
"logits/rejected": 0.22672635316848755,
"logps/chosen": -68.62225341796875,
"logps/ref_chosen": -63.83664321899414,
"logps/ref_rejected": -79.32362365722656,
"logps/rejected": -85.73505401611328,
"loss": 1.2559,
"margin_dpo/margin_mean": 1.6258125305175781,
"margin_dpo/margin_std": 3.6502134799957275,
"step": 103
},
{
"KL/chosen_KL_mean": -4.706455230712891,
"KL/mean": -5.858922004699707,
"KL/rejected_KL_mean": -7.011390686035156,
"KL/std": 4.994261741638184,
"epoch": 0.15721844293272866,
"fcm_dpo/beta": 0.112154021859169,
"fcm_dpo/delta": 0.04702896624803543,
"fcm_dpo/margin": 2.3049371242523193,
"fcm_dpo/q_t": 0.44699156284332275,
"grad_norm": 34.770877838134766,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 0.24318566918373108,
"logits/rejected": 0.1570195108652115,
"logps/chosen": -65.7056655883789,
"logps/ref_chosen": -60.99920654296875,
"logps/ref_rejected": -98.84645080566406,
"logps/rejected": -105.85784912109375,
"loss": 1.2607,
"margin_dpo/margin_mean": 2.304936408996582,
"margin_dpo/margin_std": 6.377119064331055,
"step": 104
},
{
"KL/chosen_KL_mean": -5.352252960205078,
"KL/mean": -6.269779205322266,
"KL/rejected_KL_mean": -7.187309265136719,
"KL/std": 3.987088680267334,
"epoch": 0.15873015873015872,
"fcm_dpo/beta": 0.11373626440763474,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8350532054901123,
"fcm_dpo/q_t": 0.45318853855133057,
"grad_norm": 34.859764099121094,
"learning_rate": 4.952285105344791e-07,
"logits/chosen": 0.20636920630931854,
"logits/rejected": 0.15100428462028503,
"logps/chosen": -76.30252075195312,
"logps/ref_chosen": -70.95027160644531,
"logps/ref_rejected": -87.88340759277344,
"logps/rejected": -95.07072448730469,
"loss": 1.2632,
"margin_dpo/margin_mean": 1.8350532054901123,
"margin_dpo/margin_std": 4.919508457183838,
"step": 105
},
{
"KL/chosen_KL_mean": -5.029964447021484,
"KL/mean": -5.7955522537231445,
"KL/rejected_KL_mean": -6.561138153076172,
"KL/std": 3.89530611038208,
"epoch": 0.1602418745275888,
"fcm_dpo/beta": 0.11373626440763474,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5311670303344727,
"fcm_dpo/q_t": 0.4586039185523987,
"grad_norm": 36.046531677246094,
"learning_rate": 4.949679871846857e-07,
"logits/chosen": 0.24574297666549683,
"logits/rejected": 0.2315644919872284,
"logps/chosen": -67.48929595947266,
"logps/ref_chosen": -62.45933151245117,
"logps/ref_rejected": -67.00595092773438,
"logps/rejected": -73.56708526611328,
"loss": 1.3057,
"margin_dpo/margin_mean": 1.5311667919158936,
"margin_dpo/margin_std": 5.281202793121338,
"step": 106
},
{
"KL/chosen_KL_mean": -6.49530029296875,
"KL/mean": -6.9094672203063965,
"KL/rejected_KL_mean": -7.323631286621094,
"KL/std": 4.887589454650879,
"epoch": 0.1617535903250189,
"fcm_dpo/beta": 0.11373626440763474,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8283355236053467,
"fcm_dpo/q_t": 0.47569483518600464,
"grad_norm": 48.95326232910156,
"learning_rate": 4.947006115536947e-07,
"logits/chosen": 0.17019329965114594,
"logits/rejected": 0.1465989649295807,
"logps/chosen": -82.33326721191406,
"logps/ref_chosen": -75.83796691894531,
"logps/ref_rejected": -87.74038696289062,
"logps/rejected": -95.06401062011719,
"loss": 1.409,
"margin_dpo/margin_mean": 0.828335702419281,
"margin_dpo/margin_std": 6.10739803314209,
"step": 107
},
{
"KL/chosen_KL_mean": -5.463619232177734,
"KL/mean": -6.383543014526367,
"KL/rejected_KL_mean": -7.303462982177734,
"KL/std": 4.103908061981201,
"epoch": 0.16326530612244897,
"fcm_dpo/beta": 0.11373626440763474,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.839846134185791,
"fcm_dpo/q_t": 0.4533940255641937,
"grad_norm": 32.70323944091797,
"learning_rate": 4.944263911205772e-07,
"logits/chosen": 0.19207993149757385,
"logits/rejected": 0.16023018956184387,
"logps/chosen": -73.85685729980469,
"logps/ref_chosen": -68.39323425292969,
"logps/ref_rejected": -83.24267578125,
"logps/rejected": -90.54613494873047,
"loss": 1.2767,
"margin_dpo/margin_mean": 1.8398463726043701,
"margin_dpo/margin_std": 5.364116668701172,
"step": 108
},
{
"KL/chosen_KL_mean": -4.823814392089844,
"KL/mean": -6.310299873352051,
"KL/rejected_KL_mean": -7.796791076660156,
"KL/std": 4.97265100479126,
"epoch": 0.16477702191987906,
"fcm_dpo/beta": 0.11705504357814789,
"fcm_dpo/delta": 0.05780094116926193,
"fcm_dpo/margin": 2.9729766845703125,
"fcm_dpo/q_t": 0.426702082157135,
"grad_norm": 32.38801193237305,
"learning_rate": 4.941453335558681e-07,
"logits/chosen": 0.17428874969482422,
"logits/rejected": 0.12089354544878006,
"logps/chosen": -60.351295471191406,
"logps/ref_chosen": -55.52748107910156,
"logps/ref_rejected": -83.55218505859375,
"logps/rejected": -91.34898376464844,
"loss": 1.1535,
"margin_dpo/margin_mean": 2.9729764461517334,
"margin_dpo/margin_std": 5.388824462890625,
"step": 109
},
{
"KL/chosen_KL_mean": -6.182544708251953,
"KL/mean": -6.312487602233887,
"KL/rejected_KL_mean": -6.442424774169922,
"KL/std": 3.990682363510132,
"epoch": 0.16628873771730915,
"fcm_dpo/beta": 0.1177498996257782,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2598845958709717,
"fcm_dpo/q_t": 0.4898316264152527,
"grad_norm": 53.71846008300781,
"learning_rate": 4.938574467213517e-07,
"logits/chosen": 0.17680279910564423,
"logits/rejected": 0.18614478409290314,
"logps/chosen": -87.34129333496094,
"logps/ref_chosen": -81.15874481201172,
"logps/ref_rejected": -72.56021118164062,
"logps/rejected": -79.00263977050781,
"loss": 1.4483,
"margin_dpo/margin_mean": 0.25988438725471497,
"margin_dpo/margin_std": 5.361874103546143,
"step": 110
},
{
"KL/chosen_KL_mean": -5.583719253540039,
"KL/mean": -6.557001113891602,
"KL/rejected_KL_mean": -7.5302886962890625,
"KL/std": 4.138578414916992,
"epoch": 0.16780045351473924,
"fcm_dpo/beta": 0.1203727126121521,
"fcm_dpo/delta": 0.07264174520969391,
"fcm_dpo/margin": 1.9465702772140503,
"fcm_dpo/q_t": 0.44942060112953186,
"grad_norm": 30.87441062927246,
"learning_rate": 4.935627386698418e-07,
"logits/chosen": 0.2694501280784607,
"logits/rejected": 0.23262867331504822,
"logps/chosen": -57.94270324707031,
"logps/ref_chosen": -52.358985900878906,
"logps/ref_rejected": -77.06150817871094,
"logps/rejected": -84.591796875,
"loss": 1.2512,
"margin_dpo/margin_mean": 1.9465705156326294,
"margin_dpo/margin_std": 5.017286777496338,
"step": 111
},
{
"KL/chosen_KL_mean": -5.246620178222656,
"KL/mean": -6.449799537658691,
"KL/rejected_KL_mean": -7.652973175048828,
"KL/std": 4.185157775878906,
"epoch": 0.1693121693121693,
"fcm_dpo/beta": 0.12910687923431396,
"fcm_dpo/delta": 0.09781448543071747,
"fcm_dpo/margin": 2.4063587188720703,
"fcm_dpo/q_t": 0.4301964044570923,
"grad_norm": 40.442134857177734,
"learning_rate": 4.932612176449559e-07,
"logits/chosen": 0.19383251667022705,
"logits/rejected": 0.12825272977352142,
"logps/chosen": -68.26668548583984,
"logps/ref_chosen": -63.02006530761719,
"logps/ref_rejected": -111.36941528320312,
"logps/rejected": -119.02238464355469,
"loss": 1.1838,
"margin_dpo/margin_mean": 2.4063572883605957,
"margin_dpo/margin_std": 4.644060134887695,
"step": 112
},
{
"KL/chosen_KL_mean": -5.255073547363281,
"KL/mean": -5.934301376342773,
"KL/rejected_KL_mean": -6.613533020019531,
"KL/std": 3.480978488922119,
"epoch": 0.1708238851095994,
"fcm_dpo/beta": 0.13342835009098053,
"fcm_dpo/delta": 0.07491803169250488,
"fcm_dpo/margin": 1.3584556579589844,
"fcm_dpo/q_t": 0.4581720232963562,
"grad_norm": 49.05155944824219,
"learning_rate": 4.929528920808854e-07,
"logits/chosen": 0.19861599802970886,
"logits/rejected": 0.15952877700328827,
"logps/chosen": -61.06273651123047,
"logps/ref_chosen": -55.80766296386719,
"logps/ref_rejected": -69.84014129638672,
"logps/rejected": -76.45367431640625,
"loss": 1.2973,
"margin_dpo/margin_mean": 1.3584556579589844,
"margin_dpo/margin_std": 4.468038558959961,
"step": 113
},
{
"KL/chosen_KL_mean": -4.988502502441406,
"KL/mean": -6.088685035705566,
"KL/rejected_KL_mean": -7.188873291015625,
"KL/std": 3.774844169616699,
"epoch": 0.17233560090702948,
"fcm_dpo/beta": 0.14195749163627625,
"fcm_dpo/delta": 0.09694860130548477,
"fcm_dpo/margin": 2.200364351272583,
"fcm_dpo/q_t": 0.4333820343017578,
"grad_norm": 42.0706901550293,
"learning_rate": 4.92637770602159e-07,
"logits/chosen": 0.24624261260032654,
"logits/rejected": 0.1857389360666275,
"logps/chosen": -71.32127380371094,
"logps/ref_chosen": -66.33277130126953,
"logps/ref_rejected": -71.61489868164062,
"logps/rejected": -78.80377197265625,
"loss": 1.1946,
"margin_dpo/margin_mean": 2.200364828109741,
"margin_dpo/margin_std": 4.570239067077637,
"step": 114
},
{
"KL/chosen_KL_mean": -5.369632720947266,
"KL/mean": -6.250866889953613,
"KL/rejected_KL_mean": -7.132101058959961,
"KL/std": 3.4158787727355957,
"epoch": 0.17384731670445955,
"fcm_dpo/beta": 0.14961527287960052,
"fcm_dpo/delta": 0.056850727647542953,
"fcm_dpo/margin": 1.7624667882919312,
"fcm_dpo/q_t": 0.44253766536712646,
"grad_norm": 42.88827133178711,
"learning_rate": 4.923158620234019e-07,
"logits/chosen": 0.23988944292068481,
"logits/rejected": 0.17627336084842682,
"logps/chosen": -61.11867141723633,
"logps/ref_chosen": -55.74903869628906,
"logps/ref_rejected": -79.59849548339844,
"logps/rejected": -86.73059844970703,
"loss": 1.2282,
"margin_dpo/margin_mean": 1.7624670267105103,
"margin_dpo/margin_std": 4.0995025634765625,
"step": 115
},
{
"KL/chosen_KL_mean": -4.933021545410156,
"KL/mean": -6.309746742248535,
"KL/rejected_KL_mean": -7.6864776611328125,
"KL/std": 3.5162925720214844,
"epoch": 0.17535903250188964,
"fcm_dpo/beta": 0.15041202306747437,
"fcm_dpo/delta": -0.017097879201173782,
"fcm_dpo/margin": 2.7534589767456055,
"fcm_dpo/q_t": 0.4049130082130432,
"grad_norm": 35.12335968017578,
"learning_rate": 4.91987175349089e-07,
"logits/chosen": 0.2185758650302887,
"logits/rejected": 0.15106023848056793,
"logps/chosen": -54.298187255859375,
"logps/ref_chosen": -49.36516571044922,
"logps/ref_rejected": -72.84671020507812,
"logps/rejected": -80.53318786621094,
"loss": 1.0934,
"margin_dpo/margin_mean": 2.7534589767456055,
"margin_dpo/margin_std": 3.808335542678833,
"step": 116
},
{
"KL/chosen_KL_mean": -4.931730270385742,
"KL/mean": -5.858213901519775,
"KL/rejected_KL_mean": -6.784702301025391,
"KL/std": 3.3140063285827637,
"epoch": 0.17687074829931973,
"fcm_dpo/beta": 0.14422379434108734,
"fcm_dpo/delta": -0.04407034069299698,
"fcm_dpo/margin": 1.8529614210128784,
"fcm_dpo/q_t": 0.43684980273246765,
"grad_norm": 40.910606384277344,
"learning_rate": 4.916517197732933e-07,
"logits/chosen": 0.2307669222354889,
"logits/rejected": 0.19402286410331726,
"logps/chosen": -62.64262771606445,
"logps/ref_chosen": -57.710899353027344,
"logps/ref_rejected": -69.77253723144531,
"logps/rejected": -76.55723571777344,
"loss": 1.2279,
"margin_dpo/margin_mean": 1.8529608249664307,
"margin_dpo/margin_std": 4.091181755065918,
"step": 117
},
{
"KL/chosen_KL_mean": -4.755271911621094,
"KL/mean": -5.991068363189697,
"KL/rejected_KL_mean": -7.226863861083984,
"KL/std": 3.593959093093872,
"epoch": 0.17838246409674982,
"fcm_dpo/beta": 0.14586882293224335,
"fcm_dpo/delta": 0.0441410131752491,
"fcm_dpo/margin": 2.471588611602783,
"fcm_dpo/q_t": 0.4175887703895569,
"grad_norm": 38.68694305419922,
"learning_rate": 4.913095046794281e-07,
"logits/chosen": 0.30474239587783813,
"logits/rejected": 0.2626001238822937,
"logps/chosen": -57.23516845703125,
"logps/ref_chosen": -52.479896545410156,
"logps/ref_rejected": -81.359130859375,
"logps/rejected": -88.58599090576172,
"loss": 1.1265,
"margin_dpo/margin_mean": 2.471587896347046,
"margin_dpo/margin_std": 3.647130012512207,
"step": 118
},
{
"KL/chosen_KL_mean": -6.017473220825195,
"KL/mean": -6.920298099517822,
"KL/rejected_KL_mean": -7.823123931884766,
"KL/std": 4.139496803283691,
"epoch": 0.17989417989417988,
"fcm_dpo/beta": 0.14982125163078308,
"fcm_dpo/delta": 0.038419194519519806,
"fcm_dpo/margin": 1.8056480884552002,
"fcm_dpo/q_t": 0.44211965799331665,
"grad_norm": 44.770748138427734,
"learning_rate": 4.909605396399855e-07,
"logits/chosen": 0.20789092779159546,
"logits/rejected": 0.1685742735862732,
"logps/chosen": -67.3751449584961,
"logps/ref_chosen": -61.35767364501953,
"logps/ref_rejected": -75.71510314941406,
"logps/rejected": -83.53822326660156,
"loss": 1.2659,
"margin_dpo/margin_mean": 1.8056479692459106,
"margin_dpo/margin_std": 5.146032333374023,
"step": 119
},
{
"KL/chosen_KL_mean": -4.956226348876953,
"KL/mean": -6.255353927612305,
"KL/rejected_KL_mean": -7.5544891357421875,
"KL/std": 3.479482650756836,
"epoch": 0.18140589569160998,
"fcm_dpo/beta": 0.15262466669082642,
"fcm_dpo/delta": 0.0036748070269823074,
"fcm_dpo/margin": 2.5982601642608643,
"fcm_dpo/q_t": 0.4099358320236206,
"grad_norm": 40.47877502441406,
"learning_rate": 4.906048344162676e-07,
"logits/chosen": 0.19425508379936218,
"logits/rejected": 0.13444660604000092,
"logps/chosen": -64.8637924194336,
"logps/ref_chosen": -59.907569885253906,
"logps/ref_rejected": -79.6910629272461,
"logps/rejected": -87.24555206298828,
"loss": 1.0971,
"margin_dpo/margin_mean": 2.5982608795166016,
"margin_dpo/margin_std": 3.5266871452331543,
"step": 120
},
{
"KL/chosen_KL_mean": -5.358709335327148,
"KL/mean": -6.739768028259277,
"KL/rejected_KL_mean": -8.120830535888672,
"KL/std": 3.996863842010498,
"epoch": 0.18291761148904007,
"fcm_dpo/beta": 0.1501387357711792,
"fcm_dpo/delta": -0.016977794468402863,
"fcm_dpo/margin": 2.7621190547943115,
"fcm_dpo/q_t": 0.4117959439754486,
"grad_norm": 37.1208610534668,
"learning_rate": 4.902423989581143e-07,
"logits/chosen": 0.30443626642227173,
"logits/rejected": 0.2181735634803772,
"logps/chosen": -61.024749755859375,
"logps/ref_chosen": -55.66604232788086,
"logps/ref_rejected": -101.56233978271484,
"logps/rejected": -109.68316650390625,
"loss": 1.1077,
"margin_dpo/margin_mean": 2.7621188163757324,
"margin_dpo/margin_std": 4.442630290985107,
"step": 121
},
{
"KL/chosen_KL_mean": -5.777549743652344,
"KL/mean": -7.061952114105225,
"KL/rejected_KL_mean": -8.346351623535156,
"KL/std": 4.270412445068359,
"epoch": 0.18442932728647016,
"fcm_dpo/beta": 0.15251445770263672,
"fcm_dpo/delta": 0.008429568260908127,
"fcm_dpo/margin": 2.5688040256500244,
"fcm_dpo/q_t": 0.42014241218566895,
"grad_norm": 46.91085433959961,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 0.2145451307296753,
"logits/rejected": 0.1792818009853363,
"logps/chosen": -69.11192321777344,
"logps/ref_chosen": -63.334373474121094,
"logps/ref_rejected": -73.67523193359375,
"logps/rejected": -82.02159118652344,
"loss": 1.1596,
"margin_dpo/margin_mean": 2.568803548812866,
"margin_dpo/margin_std": 5.07568359375,
"step": 122
},
{
"KL/chosen_KL_mean": -5.459968566894531,
"KL/mean": -6.718021392822266,
"KL/rejected_KL_mean": -7.97607421875,
"KL/std": 3.7030413150787354,
"epoch": 0.18594104308390022,
"fcm_dpo/beta": 0.1532004177570343,
"fcm_dpo/delta": 0.01600516401231289,
"fcm_dpo/margin": 2.5161049365997314,
"fcm_dpo/q_t": 0.41378509998321533,
"grad_norm": 38.989585876464844,
"learning_rate": 4.894973780788722e-07,
"logits/chosen": 0.23768547177314758,
"logits/rejected": 0.19618701934814453,
"logps/chosen": -62.35871124267578,
"logps/ref_chosen": -56.89874267578125,
"logps/ref_rejected": -78.97028350830078,
"logps/rejected": -86.94635772705078,
"loss": 1.123,
"margin_dpo/margin_mean": 2.5161049365997314,
"margin_dpo/margin_std": 3.9700427055358887,
"step": 123
},
{
"KL/chosen_KL_mean": -5.782169342041016,
"KL/mean": -7.465737342834473,
"KL/rejected_KL_mean": -9.149307250976562,
"KL/std": 4.042940616607666,
"epoch": 0.1874527588813303,
"fcm_dpo/beta": 0.14460201561450958,
"fcm_dpo/delta": -0.10314086824655533,
"fcm_dpo/margin": 3.367136240005493,
"fcm_dpo/q_t": 0.3885384202003479,
"grad_norm": 36.7084846496582,
"learning_rate": 4.89114813497619e-07,
"logits/chosen": 0.28056734800338745,
"logits/rejected": 0.21720805764198303,
"logps/chosen": -62.89825439453125,
"logps/ref_chosen": -57.116085052490234,
"logps/ref_rejected": -87.93074035644531,
"logps/rejected": -97.08004760742188,
"loss": 1.0417,
"margin_dpo/margin_mean": 3.3671364784240723,
"margin_dpo/margin_std": 4.220187187194824,
"step": 124
},
{
"KL/chosen_KL_mean": -6.223566055297852,
"KL/mean": -7.6594648361206055,
"KL/rejected_KL_mean": -9.095359802246094,
"KL/std": 4.320698261260986,
"epoch": 0.1889644746787604,
"fcm_dpo/beta": 0.1405811309814453,
"fcm_dpo/delta": -0.006451927125453949,
"fcm_dpo/margin": 2.8717947006225586,
"fcm_dpo/q_t": 0.410498708486557,
"grad_norm": 38.985382080078125,
"learning_rate": 4.887255603610184e-07,
"logits/chosen": 0.29075124859809875,
"logits/rejected": 0.22918814420700073,
"logps/chosen": -71.92974853515625,
"logps/ref_chosen": -65.7061767578125,
"logps/ref_rejected": -91.72711944580078,
"logps/rejected": -100.82247924804688,
"loss": 1.1143,
"margin_dpo/margin_mean": 2.8717949390411377,
"margin_dpo/margin_std": 4.4547529220581055,
"step": 125
},
{
"KL/chosen_KL_mean": -5.821783065795898,
"KL/mean": -6.677826404571533,
"KL/rejected_KL_mean": -7.53387451171875,
"KL/std": 4.0686540603637695,
"epoch": 0.19047619047619047,
"fcm_dpo/beta": 0.15313303470611572,
"fcm_dpo/delta": 0.14905323088169098,
"fcm_dpo/margin": 1.7120908498764038,
"fcm_dpo/q_t": 0.4439869523048401,
"grad_norm": 42.53828430175781,
"learning_rate": 4.883296295573176e-07,
"logits/chosen": 0.09436048567295074,
"logits/rejected": 0.08861444890499115,
"logps/chosen": -73.99787139892578,
"logps/ref_chosen": -68.17608642578125,
"logps/ref_rejected": -65.1175537109375,
"logps/rejected": -72.65142822265625,
"loss": 1.2429,
"margin_dpo/margin_mean": 1.7120912075042725,
"margin_dpo/margin_std": 4.371842384338379,
"step": 126
},
{
"KL/chosen_KL_mean": -5.998382568359375,
"KL/mean": -7.5352582931518555,
"KL/rejected_KL_mean": -9.072135925292969,
"KL/std": 3.6161179542541504,
"epoch": 0.19198790627362056,
"fcm_dpo/beta": 0.1484401971101761,
"fcm_dpo/delta": -0.06965281069278717,
"fcm_dpo/margin": 3.0737504959106445,
"fcm_dpo/q_t": 0.3940262198448181,
"grad_norm": 37.3619270324707,
"learning_rate": 4.87927032161552e-07,
"logits/chosen": 0.22243531048297882,
"logits/rejected": 0.1914488971233368,
"logps/chosen": -67.87861633300781,
"logps/ref_chosen": -61.88023376464844,
"logps/ref_rejected": -68.46012878417969,
"logps/rejected": -77.53225708007812,
"loss": 1.0581,
"margin_dpo/margin_mean": 3.0737502574920654,
"margin_dpo/margin_std": 3.76737117767334,
"step": 127
},
{
"KL/chosen_KL_mean": -6.474908828735352,
"KL/mean": -8.036764144897461,
"KL/rejected_KL_mean": -9.598621368408203,
"KL/std": 4.666970252990723,
"epoch": 0.19349962207105065,
"fcm_dpo/beta": 0.14405301213264465,
"fcm_dpo/delta": -0.05977921932935715,
"fcm_dpo/margin": 3.123711109161377,
"fcm_dpo/q_t": 0.4026954770088196,
"grad_norm": 37.78749084472656,
"learning_rate": 4.875177794352363e-07,
"logits/chosen": 0.2339283525943756,
"logits/rejected": 0.17583772540092468,
"logps/chosen": -73.18389129638672,
"logps/ref_chosen": -66.708984375,
"logps/ref_rejected": -94.97969055175781,
"logps/rejected": -104.57830810546875,
"loss": 1.1297,
"margin_dpo/margin_mean": 3.123711585998535,
"margin_dpo/margin_std": 5.568070411682129,
"step": 128
},
{
"KL/chosen_KL_mean": -7.183927536010742,
"KL/mean": -8.192344665527344,
"KL/rejected_KL_mean": -9.200759887695312,
"KL/std": 4.225727558135986,
"epoch": 0.19501133786848074,
"fcm_dpo/beta": 0.15166246891021729,
"fcm_dpo/delta": 0.10337048768997192,
"fcm_dpo/margin": 2.016836404800415,
"fcm_dpo/q_t": 0.43475955724716187,
"grad_norm": 50.885433197021484,
"learning_rate": 4.871018828260491e-07,
"logits/chosen": 0.2141018807888031,
"logits/rejected": 0.20807525515556335,
"logps/chosen": -72.52275085449219,
"logps/ref_chosen": -65.33882904052734,
"logps/ref_rejected": -68.06109619140625,
"logps/rejected": -77.26185607910156,
"loss": 1.2322,
"margin_dpo/margin_mean": 2.0168368816375732,
"margin_dpo/margin_std": 4.955423355102539,
"step": 129
},
{
"KL/chosen_KL_mean": -6.767240524291992,
"KL/mean": -8.247897148132324,
"KL/rejected_KL_mean": -9.72854995727539,
"KL/std": 4.21795129776001,
"epoch": 0.1965230536659108,
"fcm_dpo/beta": 0.1474171131849289,
"fcm_dpo/delta": -0.04654637351632118,
"fcm_dpo/margin": 2.9613113403320312,
"fcm_dpo/q_t": 0.39949163794517517,
"grad_norm": 43.239646911621094,
"learning_rate": 4.866793539675126e-07,
"logits/chosen": 0.1729985773563385,
"logits/rejected": 0.12204181402921677,
"logps/chosen": -65.42798614501953,
"logps/ref_chosen": -58.660743713378906,
"logps/ref_rejected": -79.24510192871094,
"logps/rejected": -88.97364807128906,
"loss": 1.0837,
"margin_dpo/margin_mean": 2.9613113403320312,
"margin_dpo/margin_std": 4.048201084136963,
"step": 130
},
{
"KL/chosen_KL_mean": -6.502410888671875,
"KL/mean": -8.361289978027344,
"KL/rejected_KL_mean": -10.220169067382812,
"KL/std": 4.88187313079834,
"epoch": 0.1980347694633409,
"fcm_dpo/beta": 0.1393418312072754,
"fcm_dpo/delta": -0.14160066843032837,
"fcm_dpo/margin": 3.717766761779785,
"fcm_dpo/q_t": 0.3856920897960663,
"grad_norm": 34.58555603027344,
"learning_rate": 4.86250204678667e-07,
"logits/chosen": 0.19449803233146667,
"logits/rejected": 0.12917035818099976,
"logps/chosen": -59.016944885253906,
"logps/ref_chosen": -52.51453399658203,
"logps/ref_rejected": -85.18299865722656,
"logps/rejected": -95.40316772460938,
"loss": 1.0653,
"margin_dpo/margin_mean": 3.7177672386169434,
"margin_dpo/margin_std": 5.4574785232543945,
"step": 131
},
{
"KL/chosen_KL_mean": -7.104595184326172,
"KL/mean": -8.693756103515625,
"KL/rejected_KL_mean": -10.282918930053711,
"KL/std": 4.658677101135254,
"epoch": 0.19954648526077098,
"fcm_dpo/beta": 0.1348930299282074,
"fcm_dpo/delta": -0.033193182200193405,
"fcm_dpo/margin": 3.1783313751220703,
"fcm_dpo/q_t": 0.4000505208969116,
"grad_norm": 36.00300979614258,
"learning_rate": 4.858144469637408e-07,
"logits/chosen": 0.26692995429039,
"logits/rejected": 0.23427237570285797,
"logps/chosen": -72.78972625732422,
"logps/ref_chosen": -65.68513488769531,
"logps/ref_rejected": -69.54120635986328,
"logps/rejected": -79.82412719726562,
"loss": 1.0949,
"margin_dpo/margin_mean": 3.1783313751220703,
"margin_dpo/margin_std": 4.635673522949219,
"step": 132
},
{
"KL/chosen_KL_mean": -7.397819519042969,
"KL/mean": -8.735570907592773,
"KL/rejected_KL_mean": -10.073318481445312,
"KL/std": 4.610377311706543,
"epoch": 0.20105820105820105,
"fcm_dpo/beta": 0.13705599308013916,
"fcm_dpo/delta": 0.037375159561634064,
"fcm_dpo/margin": 2.675502061843872,
"fcm_dpo/q_t": 0.41829919815063477,
"grad_norm": 41.43574905395508,
"learning_rate": 4.853720930118138e-07,
"logits/chosen": 0.20174159109592438,
"logits/rejected": 0.1917198747396469,
"logps/chosen": -70.99593353271484,
"logps/ref_chosen": -63.598114013671875,
"logps/ref_rejected": -73.72798156738281,
"logps/rejected": -83.80130004882812,
"loss": 1.1554,
"margin_dpo/margin_mean": 2.675502300262451,
"margin_dpo/margin_std": 4.851496696472168,
"step": 133
},
{
"KL/chosen_KL_mean": -6.886777877807617,
"KL/mean": -9.030449867248535,
"KL/rejected_KL_mean": -11.174121856689453,
"KL/std": 5.263517379760742,
"epoch": 0.20256991685563114,
"fcm_dpo/beta": 0.12709717452526093,
"fcm_dpo/delta": -0.17415767908096313,
"fcm_dpo/margin": 4.287341117858887,
"fcm_dpo/q_t": 0.3749292492866516,
"grad_norm": 30.688871383666992,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": 0.29332664608955383,
"logits/rejected": 0.23596280813217163,
"logps/chosen": -60.68135070800781,
"logps/ref_chosen": -53.79457092285156,
"logps/ref_rejected": -74.16741943359375,
"logps/rejected": -85.34153747558594,
"loss": 1.0112,
"margin_dpo/margin_mean": 4.287341594696045,
"margin_dpo/margin_std": 5.231392860412598,
"step": 134
},
{
"KL/chosen_KL_mean": -6.964086532592773,
"KL/mean": -8.33128547668457,
"KL/rejected_KL_mean": -9.698490142822266,
"KL/std": 4.9238457679748535,
"epoch": 0.20408163265306123,
"fcm_dpo/beta": 0.12767961621284485,
"fcm_dpo/delta": 0.05650739371776581,
"fcm_dpo/margin": 2.73439884185791,
"fcm_dpo/q_t": 0.42211827635765076,
"grad_norm": 31.442127227783203,
"learning_rate": 4.844676460754862e-07,
"logits/chosen": 0.22777020931243896,
"logits/rejected": 0.19519740343093872,
"logps/chosen": -56.4051628112793,
"logps/ref_chosen": -49.441078186035156,
"logps/ref_rejected": -65.96878051757812,
"logps/rejected": -75.66726684570312,
"loss": 1.1643,
"margin_dpo/margin_mean": 2.734398603439331,
"margin_dpo/margin_std": 5.071935653686523,
"step": 135
},
{
"KL/chosen_KL_mean": -8.580795288085938,
"KL/mean": -10.33868408203125,
"KL/rejected_KL_mean": -12.096576690673828,
"KL/std": 5.4477667808532715,
"epoch": 0.20559334845049132,
"fcm_dpo/beta": 0.1240832507610321,
"fcm_dpo/delta": -0.043839938938617706,
"fcm_dpo/margin": 3.5157880783081055,
"fcm_dpo/q_t": 0.40707147121429443,
"grad_norm": 37.98371505737305,
"learning_rate": 4.840055783904106e-07,
"logits/chosen": 0.23468932509422302,
"logits/rejected": 0.1596575379371643,
"logps/chosen": -75.34005737304688,
"logps/ref_chosen": -66.75926208496094,
"logps/ref_rejected": -94.61787414550781,
"logps/rejected": -106.71444702148438,
"loss": 1.1626,
"margin_dpo/margin_mean": 3.515789031982422,
"margin_dpo/margin_std": 6.921201705932617,
"step": 136
},
{
"KL/chosen_KL_mean": -8.011341094970703,
"KL/mean": -9.820709228515625,
"KL/rejected_KL_mean": -11.630073547363281,
"KL/std": 5.151564598083496,
"epoch": 0.20710506424792138,
"fcm_dpo/beta": 0.12334209680557251,
"fcm_dpo/delta": -0.05368944630026817,
"fcm_dpo/margin": 3.6187355518341064,
"fcm_dpo/q_t": 0.4003620445728302,
"grad_norm": 30.845598220825195,
"learning_rate": 4.835369650662767e-07,
"logits/chosen": 0.25939062237739563,
"logits/rejected": 0.22949111461639404,
"logps/chosen": -64.79513549804688,
"logps/ref_chosen": -56.78379821777344,
"logps/ref_rejected": -69.89952087402344,
"logps/rejected": -81.52959442138672,
"loss": 1.0943,
"margin_dpo/margin_mean": 3.6187360286712646,
"margin_dpo/margin_std": 5.483453750610352,
"step": 137
},
{
"KL/chosen_KL_mean": -9.439451217651367,
"KL/mean": -10.996476173400879,
"KL/rejected_KL_mean": -12.55350112915039,
"KL/std": 5.402915000915527,
"epoch": 0.20861678004535147,
"fcm_dpo/beta": 0.12223243713378906,
"fcm_dpo/delta": 0.021854639053344727,
"fcm_dpo/margin": 3.1140499114990234,
"fcm_dpo/q_t": 0.4149552583694458,
"grad_norm": 32.88602066040039,
"learning_rate": 4.830618192112065e-07,
"logits/chosen": 0.26229894161224365,
"logits/rejected": 0.22495831549167633,
"logps/chosen": -68.20545959472656,
"logps/ref_chosen": -58.766014099121094,
"logps/ref_rejected": -68.12371826171875,
"logps/rejected": -80.67721557617188,
"loss": 1.1688,
"margin_dpo/margin_mean": 3.1140503883361816,
"margin_dpo/margin_std": 6.036888599395752,
"step": 138
},
{
"KL/chosen_KL_mean": -9.852619171142578,
"KL/mean": -11.74112319946289,
"KL/rejected_KL_mean": -13.629623413085938,
"KL/std": 5.306262969970703,
"epoch": 0.21012849584278157,
"fcm_dpo/beta": 0.12008741497993469,
"fcm_dpo/delta": -0.06189559027552605,
"fcm_dpo/margin": 3.7770047187805176,
"fcm_dpo/q_t": 0.3965948224067688,
"grad_norm": 34.94479751586914,
"learning_rate": 4.825801541160509e-07,
"logits/chosen": 0.21446341276168823,
"logits/rejected": 0.18413154780864716,
"logps/chosen": -81.07820892333984,
"logps/ref_chosen": -71.2255859375,
"logps/ref_rejected": -82.1834716796875,
"logps/rejected": -95.81309509277344,
"loss": 1.0784,
"margin_dpo/margin_mean": 3.7770049571990967,
"margin_dpo/margin_std": 5.333680629730225,
"step": 139
},
{
"KL/chosen_KL_mean": -9.090442657470703,
"KL/mean": -11.611061096191406,
"KL/rejected_KL_mean": -14.13167953491211,
"KL/std": 7.010415077209473,
"epoch": 0.21164021164021163,
"fcm_dpo/beta": 0.1083516851067543,
"fcm_dpo/delta": -0.1772288680076599,
"fcm_dpo/margin": 5.04123592376709,
"fcm_dpo/q_t": 0.3767069876194,
"grad_norm": 32.844200134277344,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 0.2197141796350479,
"logits/rejected": 0.17169150710105896,
"logps/chosen": -72.36810302734375,
"logps/ref_chosen": -63.27766418457031,
"logps/ref_rejected": -83.30647277832031,
"logps/rejected": -97.43815612792969,
"loss": 1.0681,
"margin_dpo/margin_mean": 5.041234970092773,
"margin_dpo/margin_std": 7.689390182495117,
"step": 140
},
{
"KL/chosen_KL_mean": -9.954669952392578,
"KL/mean": -12.317865371704102,
"KL/rejected_KL_mean": -14.681060791015625,
"KL/std": 5.889538764953613,
"epoch": 0.21315192743764172,
"fcm_dpo/beta": 0.09989713877439499,
"fcm_dpo/delta": -0.08965474367141724,
"fcm_dpo/margin": 4.726386070251465,
"fcm_dpo/q_t": 0.3953888714313507,
"grad_norm": 28.874492645263672,
"learning_rate": 4.815973202802966e-07,
"logits/chosen": 0.25134333968162537,
"logits/rejected": 0.20534633100032806,
"logps/chosen": -71.721435546875,
"logps/ref_chosen": -61.76676940917969,
"logps/ref_rejected": -88.60601806640625,
"logps/rejected": -103.28707885742188,
"loss": 1.0964,
"margin_dpo/margin_mean": 4.726386070251465,
"margin_dpo/margin_std": 7.404045104980469,
"step": 141
},
{
"KL/chosen_KL_mean": -9.856754302978516,
"KL/mean": -11.797260284423828,
"KL/rejected_KL_mean": -13.73776626586914,
"KL/std": 5.778664588928223,
"epoch": 0.2146636432350718,
"fcm_dpo/beta": 0.10087822377681732,
"fcm_dpo/delta": 0.009604006074368954,
"fcm_dpo/margin": 3.8810181617736816,
"fcm_dpo/q_t": 0.41101446747779846,
"grad_norm": 27.98453140258789,
"learning_rate": 4.810961790316729e-07,
"logits/chosen": 0.255909264087677,
"logits/rejected": 0.23032045364379883,
"logps/chosen": -75.13153076171875,
"logps/ref_chosen": -65.2747802734375,
"logps/ref_rejected": -81.1378173828125,
"logps/rejected": -94.87557983398438,
"loss": 1.1152,
"margin_dpo/margin_mean": 3.8810176849365234,
"margin_dpo/margin_std": 5.909215927124023,
"step": 142
},
{
"KL/chosen_KL_mean": -10.505781173706055,
"KL/mean": -12.216259002685547,
"KL/rejected_KL_mean": -13.92673110961914,
"KL/std": 6.674481391906738,
"epoch": 0.2161753590325019,
"fcm_dpo/beta": 0.10392451286315918,
"fcm_dpo/delta": 0.04947058483958244,
"fcm_dpo/margin": 3.4209537506103516,
"fcm_dpo/q_t": 0.4193747937679291,
"grad_norm": 37.11439895629883,
"learning_rate": 4.805885735261454e-07,
"logits/chosen": 0.2696152329444885,
"logits/rejected": 0.25197485089302063,
"logps/chosen": -73.12361145019531,
"logps/ref_chosen": -62.617828369140625,
"logps/ref_rejected": -70.39239501953125,
"logps/rejected": -84.31912231445312,
"loss": 1.2105,
"margin_dpo/margin_mean": 3.4209539890289307,
"margin_dpo/margin_std": 7.7674102783203125,
"step": 143
},
{
"KL/chosen_KL_mean": -10.830635070800781,
"KL/mean": -12.981417655944824,
"KL/rejected_KL_mean": -15.132190704345703,
"KL/std": 6.750263214111328,
"epoch": 0.21768707482993196,
"fcm_dpo/beta": 0.1010802835226059,
"fcm_dpo/delta": -0.04181172698736191,
"fcm_dpo/margin": 4.301558971405029,
"fcm_dpo/q_t": 0.40543052554130554,
"grad_norm": 29.94562530517578,
"learning_rate": 4.800745179625307e-07,
"logits/chosen": 0.23211714625358582,
"logits/rejected": 0.20083823800086975,
"logps/chosen": -71.6333236694336,
"logps/ref_chosen": -60.80268859863281,
"logps/ref_rejected": -79.07284545898438,
"logps/rejected": -94.20503234863281,
"loss": 1.1339,
"margin_dpo/margin_mean": 4.301558494567871,
"margin_dpo/margin_std": 7.674080848693848,
"step": 144
},
{
"KL/chosen_KL_mean": -10.08212661743164,
"KL/mean": -12.36398696899414,
"KL/rejected_KL_mean": -14.645843505859375,
"KL/std": 6.890389442443848,
"epoch": 0.21919879062736206,
"fcm_dpo/beta": 0.09865110367536545,
"fcm_dpo/delta": -0.05853622406721115,
"fcm_dpo/margin": 4.563719749450684,
"fcm_dpo/q_t": 0.39978501200675964,
"grad_norm": 32.027320861816406,
"learning_rate": 4.795540267200686e-07,
"logits/chosen": 0.20314472913742065,
"logits/rejected": 0.21950089931488037,
"logps/chosen": -84.69358825683594,
"logps/ref_chosen": -74.61146545410156,
"logps/ref_rejected": -83.24461364746094,
"logps/rejected": -97.89045715332031,
"loss": 1.1284,
"margin_dpo/margin_mean": 4.563719749450684,
"margin_dpo/margin_std": 8.117656707763672,
"step": 145
},
{
"KL/chosen_KL_mean": -9.851871490478516,
"KL/mean": -12.232227325439453,
"KL/rejected_KL_mean": -14.612586975097656,
"KL/std": 6.808660507202148,
"epoch": 0.22071050642479215,
"fcm_dpo/beta": 0.09611174464225769,
"fcm_dpo/delta": -0.06661855429410934,
"fcm_dpo/margin": 4.760718822479248,
"fcm_dpo/q_t": 0.3982863426208496,
"grad_norm": 27.760150909423828,
"learning_rate": 4.790271143580173e-07,
"logits/chosen": 0.22264136373996735,
"logits/rejected": 0.2081959843635559,
"logps/chosen": -67.69285583496094,
"logps/ref_chosen": -57.84098434448242,
"logps/ref_rejected": -67.47422790527344,
"logps/rejected": -82.0868148803711,
"loss": 1.0995,
"margin_dpo/margin_mean": 4.76071834564209,
"margin_dpo/margin_std": 7.512134075164795,
"step": 146
},
{
"KL/chosen_KL_mean": -11.579130172729492,
"KL/mean": -13.471564292907715,
"KL/rejected_KL_mean": -15.364002227783203,
"KL/std": 7.3946404457092285,
"epoch": 0.2222222222222222,
"fcm_dpo/beta": 0.09622933715581894,
"fcm_dpo/delta": 0.040078941732645035,
"fcm_dpo/margin": 3.78487491607666,
"fcm_dpo/q_t": 0.4198254346847534,
"grad_norm": 34.06290817260742,
"learning_rate": 4.784937956152489e-07,
"logits/chosen": 0.24301037192344666,
"logits/rejected": 0.1984240710735321,
"logps/chosen": -78.3926010131836,
"logps/ref_chosen": -66.81346893310547,
"logps/ref_rejected": -81.1796875,
"logps/rejected": -96.54369354248047,
"loss": 1.1853,
"margin_dpo/margin_mean": 3.784874439239502,
"margin_dpo/margin_std": 7.882870674133301,
"step": 147
},
{
"KL/chosen_KL_mean": -10.643348693847656,
"KL/mean": -13.073385238647461,
"KL/rejected_KL_mean": -15.50341796875,
"KL/std": 6.8529863357543945,
"epoch": 0.2237339380196523,
"fcm_dpo/beta": 0.09473659843206406,
"fcm_dpo/delta": -0.07015595585107803,
"fcm_dpo/margin": 4.860075950622559,
"fcm_dpo/q_t": 0.3961476683616638,
"grad_norm": 23.567968368530273,
"learning_rate": 4.779540854098347e-07,
"logits/chosen": 0.36633625626564026,
"logits/rejected": 0.2913614511489868,
"logps/chosen": -59.33110427856445,
"logps/ref_chosen": -48.6877555847168,
"logps/ref_rejected": -67.50503540039062,
"logps/rejected": -83.00845336914062,
"loss": 1.1018,
"margin_dpo/margin_mean": 4.860075950622559,
"margin_dpo/margin_std": 7.616449356079102,
"step": 148
},
{
"KL/chosen_KL_mean": -12.942892074584961,
"KL/mean": -15.660242080688477,
"KL/rejected_KL_mean": -18.37759017944336,
"KL/std": 7.375458717346191,
"epoch": 0.2252456538170824,
"fcm_dpo/beta": 0.08893537521362305,
"fcm_dpo/delta": -0.09736001491546631,
"fcm_dpo/margin": 5.434703350067139,
"fcm_dpo/q_t": 0.3881411850452423,
"grad_norm": 24.1125431060791,
"learning_rate": 4.774079988386296e-07,
"logits/chosen": 0.21307995915412903,
"logits/rejected": 0.16604451835155487,
"logps/chosen": -68.086669921875,
"logps/ref_chosen": -55.143775939941406,
"logps/ref_rejected": -64.79888916015625,
"logps/rejected": -83.17648315429688,
"loss": 1.0482,
"margin_dpo/margin_mean": 5.4347028732299805,
"margin_dpo/margin_std": 6.959212303161621,
"step": 149
},
{
"KL/chosen_KL_mean": -10.891637802124023,
"KL/mean": -14.504773139953613,
"KL/rejected_KL_mean": -18.117908477783203,
"KL/std": 8.100428581237793,
"epoch": 0.22675736961451248,
"fcm_dpo/beta": 0.07920362055301666,
"fcm_dpo/delta": -0.21006450057029724,
"fcm_dpo/margin": 7.226279258728027,
"fcm_dpo/q_t": 0.36848869919776917,
"grad_norm": 21.305072784423828,
"learning_rate": 4.768555511768486e-07,
"logits/chosen": 0.24731308221817017,
"logits/rejected": 0.202421173453331,
"logps/chosen": -78.36238098144531,
"logps/ref_chosen": -67.47074890136719,
"logps/ref_rejected": -89.21170806884766,
"logps/rejected": -107.32962036132812,
"loss": 0.9942,
"margin_dpo/margin_mean": 7.226279258728027,
"margin_dpo/margin_std": 8.23183822631836,
"step": 150
},
{
"KL/chosen_KL_mean": -9.562496185302734,
"KL/mean": -13.620951652526855,
"KL/rejected_KL_mean": -17.679405212402344,
"KL/std": 8.622806549072266,
"epoch": 0.22826908541194255,
"fcm_dpo/beta": 0.07037020474672318,
"fcm_dpo/delta": -0.2068619728088379,
"fcm_dpo/margin": 8.116912841796875,
"fcm_dpo/q_t": 0.3663896322250366,
"grad_norm": 19.14907455444336,
"learning_rate": 4.762967578776406e-07,
"logits/chosen": 0.2455185353755951,
"logits/rejected": 0.1907639503479004,
"logps/chosen": -62.02204132080078,
"logps/ref_chosen": -52.45954132080078,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -96.74241638183594,
"loss": 0.9791,
"margin_dpo/margin_mean": 8.116911888122559,
"margin_dpo/margin_std": 8.724565505981445,
"step": 151
},
{
"KL/chosen_KL_mean": -12.31747817993164,
"KL/mean": -14.943049430847168,
"KL/rejected_KL_mean": -17.568626403808594,
"KL/std": 8.21334171295166,
"epoch": 0.22978080120937264,
"fcm_dpo/beta": 0.07021472603082657,
"fcm_dpo/delta": 0.0327349528670311,
"fcm_dpo/margin": 5.2511467933654785,
"fcm_dpo/q_t": 0.4175514876842499,
"grad_norm": 21.101016998291016,
"learning_rate": 4.757316345716553e-07,
"logits/chosen": 0.3428304195404053,
"logits/rejected": 0.2872798442840576,
"logps/chosen": -68.87130737304688,
"logps/ref_chosen": -56.5538330078125,
"logps/ref_rejected": -76.55074310302734,
"logps/rejected": -94.11936950683594,
"loss": 1.1541,
"margin_dpo/margin_mean": 5.25114631652832,
"margin_dpo/margin_std": 9.311445236206055,
"step": 152
},
{
"KL/chosen_KL_mean": -11.60400390625,
"KL/mean": -14.734918594360352,
"KL/rejected_KL_mean": -17.865833282470703,
"KL/std": 8.638544082641602,
"epoch": 0.23129251700680273,
"fcm_dpo/beta": 0.06840331852436066,
"fcm_dpo/delta": -0.03252699226140976,
"fcm_dpo/margin": 6.261826515197754,
"fcm_dpo/q_t": 0.4008117616176605,
"grad_norm": 20.02750587463379,
"learning_rate": 4.751601970666064e-07,
"logits/chosen": 0.24029231071472168,
"logits/rejected": 0.203395813703537,
"logps/chosen": -79.61090087890625,
"logps/ref_chosen": -68.00689697265625,
"logps/ref_rejected": -74.83482360839844,
"logps/rejected": -92.70065307617188,
"loss": 1.0756,
"margin_dpo/margin_mean": 6.261826515197754,
"margin_dpo/margin_std": 8.131772994995117,
"step": 153
},
{
"KL/chosen_KL_mean": -12.812858581542969,
"KL/mean": -15.129095077514648,
"KL/rejected_KL_mean": -17.445331573486328,
"KL/std": 8.995925903320312,
"epoch": 0.2328042328042328,
"fcm_dpo/beta": 0.0704408809542656,
"fcm_dpo/delta": 0.08183015882968903,
"fcm_dpo/margin": 4.63247537612915,
"fcm_dpo/q_t": 0.42660561203956604,
"grad_norm": 20.98277473449707,
"learning_rate": 4.745824613468292e-07,
"logits/chosen": 0.3132394552230835,
"logits/rejected": 0.3103998899459839,
"logps/chosen": -72.035400390625,
"logps/ref_chosen": -59.222537994384766,
"logps/ref_rejected": -64.19131469726562,
"logps/rejected": -81.63665008544922,
"loss": 1.1868,
"margin_dpo/margin_mean": 4.63247537612915,
"margin_dpo/margin_std": 9.253047943115234,
"step": 154
},
{
"KL/chosen_KL_mean": -13.649024963378906,
"KL/mean": -16.94133949279785,
"KL/rejected_KL_mean": -20.233646392822266,
"KL/std": 8.983402252197266,
"epoch": 0.23431594860166288,
"fcm_dpo/beta": 0.06851907074451447,
"fcm_dpo/delta": -0.06188790500164032,
"fcm_dpo/margin": 6.584621906280518,
"fcm_dpo/q_t": 0.39981919527053833,
"grad_norm": 20.74521255493164,
"learning_rate": 4.7399844357283393e-07,
"logits/chosen": 0.3174290060997009,
"logits/rejected": 0.29693371057510376,
"logps/chosen": -82.10372161865234,
"logps/ref_chosen": -68.45469665527344,
"logps/ref_rejected": -77.91763305664062,
"logps/rejected": -98.15128326416016,
"loss": 1.1242,
"margin_dpo/margin_mean": 6.584621429443359,
"margin_dpo/margin_std": 11.35929012298584,
"step": 155
},
{
"KL/chosen_KL_mean": -13.674962997436523,
"KL/mean": -17.565982818603516,
"KL/rejected_KL_mean": -21.45700454711914,
"KL/std": 10.152885437011719,
"epoch": 0.23582766439909297,
"fcm_dpo/beta": 0.06530015915632248,
"fcm_dpo/delta": -0.12741731107234955,
"fcm_dpo/margin": 7.782038688659668,
"fcm_dpo/q_t": 0.38315021991729736,
"grad_norm": 20.189838409423828,
"learning_rate": 4.7340816008085305e-07,
"logits/chosen": 0.26459452509880066,
"logits/rejected": 0.21891286969184875,
"logps/chosen": -80.94456481933594,
"logps/ref_chosen": -67.26959991455078,
"logps/ref_rejected": -86.95914459228516,
"logps/rejected": -108.41615295410156,
"loss": 1.0288,
"margin_dpo/margin_mean": 7.782039642333984,
"margin_dpo/margin_std": 9.416492462158203,
"step": 156
},
{
"KL/chosen_KL_mean": -12.285833358764648,
"KL/mean": -15.452082633972168,
"KL/rejected_KL_mean": -18.618331909179688,
"KL/std": 9.885778427124023,
"epoch": 0.23733938019652306,
"fcm_dpo/beta": 0.06017298251390457,
"fcm_dpo/delta": -0.09840401262044907,
"fcm_dpo/margin": 6.332496643066406,
"fcm_dpo/q_t": 0.40991830825805664,
"grad_norm": 18.193811416625977,
"learning_rate": 4.728116273823847e-07,
"logits/chosen": 0.2879556715488434,
"logits/rejected": 0.26909253001213074,
"logps/chosen": -67.05870819091797,
"logps/ref_chosen": -54.77287292480469,
"logps/ref_rejected": -63.87866973876953,
"logps/rejected": -82.49700164794922,
"loss": 1.1274,
"margin_dpo/margin_mean": 6.332496166229248,
"margin_dpo/margin_std": 9.572912216186523,
"step": 157
},
{
"KL/chosen_KL_mean": -13.837478637695312,
"KL/mean": -17.254711151123047,
"KL/rejected_KL_mean": -20.671947479248047,
"KL/std": 10.21955680847168,
"epoch": 0.23885109599395313,
"fcm_dpo/beta": 0.060004714876413345,
"fcm_dpo/delta": -0.011527525261044502,
"fcm_dpo/margin": 6.8344645500183105,
"fcm_dpo/q_t": 0.4079589247703552,
"grad_norm": 18.990568161010742,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 0.30475467443466187,
"logits/rejected": 0.266677588224411,
"logps/chosen": -78.76019287109375,
"logps/ref_chosen": -64.92271423339844,
"logps/ref_rejected": -82.23789978027344,
"logps/rejected": -102.90984344482422,
"loss": 1.116,
"margin_dpo/margin_mean": 6.8344645500183105,
"margin_dpo/margin_std": 10.858131408691406,
"step": 158
},
{
"KL/chosen_KL_mean": -14.046161651611328,
"KL/mean": -17.997652053833008,
"KL/rejected_KL_mean": -21.949138641357422,
"KL/std": 11.05732250213623,
"epoch": 0.24036281179138322,
"fcm_dpo/beta": 0.05865692347288132,
"fcm_dpo/delta": -0.07415425777435303,
"fcm_dpo/margin": 7.902973651885986,
"fcm_dpo/q_t": 0.39414629340171814,
"grad_norm": 19.708227157592773,
"learning_rate": 4.715998812855304e-07,
"logits/chosen": 0.33380645513534546,
"logits/rejected": 0.29338324069976807,
"logps/chosen": -71.09315490722656,
"logps/ref_chosen": -57.046993255615234,
"logps/ref_rejected": -73.32441711425781,
"logps/rejected": -95.2735595703125,
"loss": 1.0867,
"margin_dpo/margin_mean": 7.902973175048828,
"margin_dpo/margin_std": 11.675771713256836,
"step": 159
},
{
"KL/chosen_KL_mean": -15.937501907348633,
"KL/mean": -19.319564819335938,
"KL/rejected_KL_mean": -22.701629638671875,
"KL/std": 10.381500244140625,
"epoch": 0.2418745275888133,
"fcm_dpo/beta": 0.05758850276470184,
"fcm_dpo/delta": 0.011783316731452942,
"fcm_dpo/margin": 6.764126777648926,
"fcm_dpo/q_t": 0.4151209890842438,
"grad_norm": 18.15164566040039,
"learning_rate": 4.7098470178228755e-07,
"logits/chosen": 0.19479069113731384,
"logits/rejected": 0.14844146370887756,
"logps/chosen": -65.74441528320312,
"logps/ref_chosen": -49.806915283203125,
"logps/ref_rejected": -68.3370132446289,
"logps/rejected": -91.03864288330078,
"loss": 1.1425,
"margin_dpo/margin_mean": 6.764126300811768,
"margin_dpo/margin_std": 11.992984771728516,
"step": 160
},
{
"KL/chosen_KL_mean": -15.949237823486328,
"KL/mean": -19.665210723876953,
"KL/rejected_KL_mean": -23.381183624267578,
"KL/std": 10.27374267578125,
"epoch": 0.24338624338624337,
"fcm_dpo/beta": 0.05700618401169777,
"fcm_dpo/delta": -0.02708788402378559,
"fcm_dpo/margin": 7.431947708129883,
"fcm_dpo/q_t": 0.4038432836532593,
"grad_norm": 17.391952514648438,
"learning_rate": 4.703633408618955e-07,
"logits/chosen": 0.34192147850990295,
"logits/rejected": 0.3018055856227875,
"logps/chosen": -68.44972229003906,
"logps/ref_chosen": -52.50048828125,
"logps/ref_rejected": -66.04540252685547,
"logps/rejected": -89.42658996582031,
"loss": 1.109,
"margin_dpo/margin_mean": 7.431947708129883,
"margin_dpo/margin_std": 11.519304275512695,
"step": 161
},
{
"KL/chosen_KL_mean": -16.880916595458984,
"KL/mean": -21.874000549316406,
"KL/rejected_KL_mean": -26.867088317871094,
"KL/std": 11.709345817565918,
"epoch": 0.24489795918367346,
"fcm_dpo/beta": 0.052794016897678375,
"fcm_dpo/delta": -0.1518443524837494,
"fcm_dpo/margin": 9.98617172241211,
"fcm_dpo/q_t": 0.376575231552124,
"grad_norm": 18.568201065063477,
"learning_rate": 4.697358159051549e-07,
"logits/chosen": 0.3624524474143982,
"logits/rejected": 0.31142184138298035,
"logps/chosen": -86.35011291503906,
"logps/ref_chosen": -69.46919250488281,
"logps/ref_rejected": -92.00952911376953,
"logps/rejected": -118.87661743164062,
"loss": 1.0213,
"margin_dpo/margin_mean": 9.98617172241211,
"margin_dpo/margin_std": 12.183841705322266,
"step": 162
},
{
"KL/chosen_KL_mean": -14.589275360107422,
"KL/mean": -19.729949951171875,
"KL/rejected_KL_mean": -24.870624542236328,
"KL/std": 11.731868743896484,
"epoch": 0.24640967498110355,
"fcm_dpo/beta": 0.05017222464084625,
"fcm_dpo/delta": -0.13923859596252441,
"fcm_dpo/margin": 10.28134822845459,
"fcm_dpo/q_t": 0.3796467185020447,
"grad_norm": 16.747222900390625,
"learning_rate": 4.691021444652876e-07,
"logits/chosen": 0.30114084482192993,
"logits/rejected": 0.25208836793899536,
"logps/chosen": -65.20310974121094,
"logps/ref_chosen": -50.613834381103516,
"logps/ref_rejected": -74.62033081054688,
"logps/rejected": -99.49095153808594,
"loss": 1.0379,
"margin_dpo/margin_mean": 10.281349182128906,
"margin_dpo/margin_std": 12.120843887329102,
"step": 163
},
{
"KL/chosen_KL_mean": -15.791715621948242,
"KL/mean": -20.810779571533203,
"KL/rejected_KL_mean": -25.829849243164062,
"KL/std": 12.496322631835938,
"epoch": 0.24792139077853365,
"fcm_dpo/beta": 0.04662519693374634,
"fcm_dpo/delta": -0.07952342927455902,
"fcm_dpo/margin": 10.038135528564453,
"fcm_dpo/q_t": 0.3925841450691223,
"grad_norm": 14.853240013122559,
"learning_rate": 4.6846234426744624e-07,
"logits/chosen": 0.29247328639030457,
"logits/rejected": 0.22746919095516205,
"logps/chosen": -70.63983154296875,
"logps/ref_chosen": -54.848114013671875,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -104.89286041259766,
"loss": 1.0734,
"margin_dpo/margin_mean": 10.038135528564453,
"margin_dpo/margin_std": 13.581424713134766,
"step": 164
},
{
"KL/chosen_KL_mean": -17.847217559814453,
"KL/mean": -22.394338607788086,
"KL/rejected_KL_mean": -26.94145965576172,
"KL/std": 12.855323791503906,
"epoch": 0.2494331065759637,
"fcm_dpo/beta": 0.04560963064432144,
"fcm_dpo/delta": -0.01796562224626541,
"fcm_dpo/margin": 9.09423828125,
"fcm_dpo/q_t": 0.404633492231369,
"grad_norm": 15.527134895324707,
"learning_rate": 4.678164332082175e-07,
"logits/chosen": 0.3545019328594208,
"logits/rejected": 0.29826444387435913,
"logps/chosen": -68.93643188476562,
"logps/ref_chosen": -51.089210510253906,
"logps/ref_rejected": -71.23370361328125,
"logps/rejected": -98.17515563964844,
"loss": 1.1163,
"margin_dpo/margin_mean": 9.09423828125,
"margin_dpo/margin_std": 13.98188591003418,
"step": 165
},
{
"KL/chosen_KL_mean": -17.026296615600586,
"KL/mean": -20.65865707397461,
"KL/rejected_KL_mean": -24.29102325439453,
"KL/std": 13.004007339477539,
"epoch": 0.2509448223733938,
"fcm_dpo/beta": 0.04612819850444794,
"fcm_dpo/delta": 0.07232268154621124,
"fcm_dpo/margin": 7.2647247314453125,
"fcm_dpo/q_t": 0.42566025257110596,
"grad_norm": 18.30398941040039,
"learning_rate": 4.6716442935512214e-07,
"logits/chosen": 0.3076387047767639,
"logits/rejected": 0.2137855738401413,
"logps/chosen": -80.21710205078125,
"logps/ref_chosen": -63.19081115722656,
"logps/ref_rejected": -93.8402099609375,
"logps/rejected": -118.13123321533203,
"loss": 1.157,
"margin_dpo/margin_mean": 7.264724254608154,
"margin_dpo/margin_std": 12.455879211425781,
"step": 166
},
{
"KL/chosen_KL_mean": -14.038339614868164,
"KL/mean": -19.74565887451172,
"KL/rejected_KL_mean": -25.452983856201172,
"KL/std": 13.267440795898438,
"epoch": 0.25245653817082386,
"fcm_dpo/beta": 0.04366353154182434,
"fcm_dpo/delta": -0.12011955678462982,
"fcm_dpo/margin": 11.414639472961426,
"fcm_dpo/q_t": 0.3832022547721863,
"grad_norm": 15.69991683959961,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": 0.30109137296676636,
"logits/rejected": 0.2652606666088104,
"logps/chosen": -72.96261596679688,
"logps/ref_chosen": -58.92427062988281,
"logps/ref_rejected": -72.97377014160156,
"logps/rejected": -98.4267578125,
"loss": 1.0246,
"margin_dpo/margin_mean": 11.414640426635742,
"margin_dpo/margin_std": 13.03073501586914,
"step": 167
},
{
"KL/chosen_KL_mean": -16.865724563598633,
"KL/mean": -21.457231521606445,
"KL/rejected_KL_mean": -26.048744201660156,
"KL/std": 14.453012466430664,
"epoch": 0.25396825396825395,
"fcm_dpo/beta": 0.043425120413303375,
"fcm_dpo/delta": 0.0012646140530705452,
"fcm_dpo/margin": 9.183013916015625,
"fcm_dpo/q_t": 0.41033321619033813,
"grad_norm": 16.60943603515625,
"learning_rate": 4.6584221638904767e-07,
"logits/chosen": 0.310530424118042,
"logits/rejected": 0.27260318398475647,
"logps/chosen": -82.51710510253906,
"logps/ref_chosen": -65.65138244628906,
"logps/ref_rejected": -79.71418762207031,
"logps/rejected": -105.76293182373047,
"loss": 1.1069,
"margin_dpo/margin_mean": 9.183013916015625,
"margin_dpo/margin_std": 13.43674087524414,
"step": 168
},
{
"KL/chosen_KL_mean": -13.360231399536133,
"KL/mean": -18.598499298095703,
"KL/rejected_KL_mean": -23.836761474609375,
"KL/std": 14.224285125732422,
"epoch": 0.25547996976568405,
"fcm_dpo/beta": 0.04259002208709717,
"fcm_dpo/delta": -0.0533658042550087,
"fcm_dpo/margin": 10.476530075073242,
"fcm_dpo/q_t": 0.40058034658432007,
"grad_norm": 17.076465606689453,
"learning_rate": 4.651720442612075e-07,
"logits/chosen": 0.38942497968673706,
"logits/rejected": 0.3538346290588379,
"logps/chosen": -74.78610229492188,
"logps/ref_chosen": -61.425865173339844,
"logps/ref_rejected": -76.09590148925781,
"logps/rejected": -99.93266296386719,
"loss": 1.107,
"margin_dpo/margin_mean": 10.476531028747559,
"margin_dpo/margin_std": 16.978721618652344,
"step": 169
},
{
"KL/chosen_KL_mean": -12.65814208984375,
"KL/mean": -17.938709259033203,
"KL/rejected_KL_mean": -23.219276428222656,
"KL/std": 14.04989242553711,
"epoch": 0.25699168556311414,
"fcm_dpo/beta": 0.041654251515865326,
"fcm_dpo/delta": -0.04596564173698425,
"fcm_dpo/margin": 10.561136245727539,
"fcm_dpo/q_t": 0.40057674050331116,
"grad_norm": 14.404908180236816,
"learning_rate": 4.6449585330874425e-07,
"logits/chosen": 0.3427773118019104,
"logits/rejected": 0.34061866998672485,
"logps/chosen": -69.31133270263672,
"logps/ref_chosen": -56.65319061279297,
"logps/ref_rejected": -63.45965576171875,
"logps/rejected": -86.6789321899414,
"loss": 1.1222,
"margin_dpo/margin_mean": 10.561136245727539,
"margin_dpo/margin_std": 17.518104553222656,
"step": 170
},
{
"KL/chosen_KL_mean": -14.985160827636719,
"KL/mean": -20.769207000732422,
"KL/rejected_KL_mean": -26.55325698852539,
"KL/std": 16.237634658813477,
"epoch": 0.2585034013605442,
"fcm_dpo/beta": 0.038949161767959595,
"fcm_dpo/delta": -0.06474211066961288,
"fcm_dpo/margin": 11.56810188293457,
"fcm_dpo/q_t": 0.39703309535980225,
"grad_norm": 14.443869590759277,
"learning_rate": 4.6381366244617224e-07,
"logits/chosen": 0.3847231864929199,
"logits/rejected": 0.3326500654220581,
"logps/chosen": -78.71992492675781,
"logps/ref_chosen": -63.73476028442383,
"logps/ref_rejected": -78.50328063964844,
"logps/rejected": -105.05653381347656,
"loss": 1.1113,
"margin_dpo/margin_mean": 11.56810188293457,
"margin_dpo/margin_std": 18.460580825805664,
"step": 171
},
{
"KL/chosen_KL_mean": -16.810169219970703,
"KL/mean": -22.31606674194336,
"KL/rejected_KL_mean": -27.82196044921875,
"KL/std": 16.634769439697266,
"epoch": 0.2600151171579743,
"fcm_dpo/beta": 0.0396546795964241,
"fcm_dpo/delta": -0.04476618766784668,
"fcm_dpo/margin": 11.011789321899414,
"fcm_dpo/q_t": 0.40239837765693665,
"grad_norm": 16.143247604370117,
"learning_rate": 4.631254907558365e-07,
"logits/chosen": 0.3681456744670868,
"logits/rejected": 0.3081226348876953,
"logps/chosen": -69.01193237304688,
"logps/ref_chosen": -52.201759338378906,
"logps/ref_rejected": -82.85285949707031,
"logps/rejected": -110.67481994628906,
"loss": 1.1085,
"margin_dpo/margin_mean": 11.011789321899414,
"margin_dpo/margin_std": 17.142223358154297,
"step": 172
},
{
"KL/chosen_KL_mean": -15.272012710571289,
"KL/mean": -21.283023834228516,
"KL/rejected_KL_mean": -27.29404067993164,
"KL/std": 17.057445526123047,
"epoch": 0.2615268329554044,
"fcm_dpo/beta": 0.03659302368760109,
"fcm_dpo/delta": -0.052808038890361786,
"fcm_dpo/margin": 12.02202320098877,
"fcm_dpo/q_t": 0.4022437334060669,
"grad_norm": 14.902726173400879,
"learning_rate": 4.624313574873786e-07,
"logits/chosen": 0.39478111267089844,
"logits/rejected": 0.30247700214385986,
"logps/chosen": -70.70674133300781,
"logps/ref_chosen": -55.434722900390625,
"logps/ref_rejected": -77.81967163085938,
"logps/rejected": -105.11371612548828,
"loss": 1.1372,
"margin_dpo/margin_mean": 12.022025108337402,
"margin_dpo/margin_std": 20.99781036376953,
"step": 173
},
{
"KL/chosen_KL_mean": -17.438180923461914,
"KL/mean": -24.091327667236328,
"KL/rejected_KL_mean": -30.744476318359375,
"KL/std": 17.439682006835938,
"epoch": 0.26303854875283444,
"fcm_dpo/beta": 0.03558962047100067,
"fcm_dpo/delta": -0.08698572218418121,
"fcm_dpo/margin": 13.306293487548828,
"fcm_dpo/q_t": 0.3912256360054016,
"grad_norm": 15.202779769897461,
"learning_rate": 4.61731282057198e-07,
"logits/chosen": 0.3748180866241455,
"logits/rejected": 0.305186927318573,
"logps/chosen": -74.61013793945312,
"logps/ref_chosen": -57.17195129394531,
"logps/ref_rejected": -85.47578430175781,
"logps/rejected": -116.22026062011719,
"loss": 1.0715,
"margin_dpo/margin_mean": 13.306294441223145,
"margin_dpo/margin_std": 18.859642028808594,
"step": 174
},
{
"KL/chosen_KL_mean": -14.990503311157227,
"KL/mean": -22.452695846557617,
"KL/rejected_KL_mean": -29.914894104003906,
"KL/std": 17.86532211303711,
"epoch": 0.26455026455026454,
"fcm_dpo/beta": 0.03374076634645462,
"fcm_dpo/delta": -0.12154931575059891,
"fcm_dpo/margin": 14.924393653869629,
"fcm_dpo/q_t": 0.38714203238487244,
"grad_norm": 14.585382461547852,
"learning_rate": 4.6102528404790965e-07,
"logits/chosen": 0.4304041266441345,
"logits/rejected": 0.3987458050251007,
"logps/chosen": -82.6561279296875,
"logps/ref_chosen": -67.6656265258789,
"logps/ref_rejected": -84.36766815185547,
"logps/rejected": -114.28256225585938,
"loss": 1.0598,
"margin_dpo/margin_mean": 14.924394607543945,
"margin_dpo/margin_std": 20.989219665527344,
"step": 175
},
{
"KL/chosen_KL_mean": -19.427631378173828,
"KL/mean": -24.25865936279297,
"KL/rejected_KL_mean": -29.08969497680664,
"KL/std": 19.192419052124023,
"epoch": 0.2660619803476946,
"fcm_dpo/beta": 0.033753879368305206,
"fcm_dpo/delta": 0.08193635195493698,
"fcm_dpo/margin": 9.66206169128418,
"fcm_dpo/q_t": 0.4294675588607788,
"grad_norm": 17.79058265686035,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 0.3631625771522522,
"logits/rejected": 0.33612459897994995,
"logps/chosen": -97.28639221191406,
"logps/ref_chosen": -77.8587646484375,
"logps/ref_rejected": -81.08732604980469,
"logps/rejected": -110.17701721191406,
"loss": 1.2166,
"margin_dpo/margin_mean": 9.66206169128418,
"margin_dpo/margin_std": 21.81412696838379,
"step": 176
},
{
"KL/chosen_KL_mean": -14.337238311767578,
"KL/mean": -23.852760314941406,
"KL/rejected_KL_mean": -33.3682861328125,
"KL/std": 19.595916748046875,
"epoch": 0.2675736961451247,
"fcm_dpo/beta": 0.03083290532231331,
"fcm_dpo/delta": -0.2311885952949524,
"fcm_dpo/margin": 19.031044006347656,
"fcm_dpo/q_t": 0.360975980758667,
"grad_norm": 17.56546974182129,
"learning_rate": 4.5959559945025183e-07,
"logits/chosen": 0.50090491771698,
"logits/rejected": 0.4008740186691284,
"logps/chosen": -69.55763244628906,
"logps/ref_chosen": -55.22039794921875,
"logps/ref_rejected": -92.54973602294922,
"logps/rejected": -125.91802215576172,
"loss": 0.9693,
"margin_dpo/margin_mean": 19.031042098999023,
"margin_dpo/margin_std": 20.025856018066406,
"step": 177
},
{
"KL/chosen_KL_mean": -17.92155647277832,
"KL/mean": -23.787391662597656,
"KL/rejected_KL_mean": -29.653221130371094,
"KL/std": 18.697711944580078,
"epoch": 0.2690854119425548,
"fcm_dpo/beta": 0.02928192727267742,
"fcm_dpo/delta": -0.04489610344171524,
"fcm_dpo/margin": 11.731661796569824,
"fcm_dpo/q_t": 0.4197039306163788,
"grad_norm": 17.016468048095703,
"learning_rate": 4.588719528532341e-07,
"logits/chosen": 0.3574860692024231,
"logits/rejected": 0.3063201308250427,
"logps/chosen": -78.73204803466797,
"logps/ref_chosen": -60.81049346923828,
"logps/ref_rejected": -81.12973022460938,
"logps/rejected": -110.78294372558594,
"loss": 1.1482,
"margin_dpo/margin_mean": 11.731661796569824,
"margin_dpo/margin_std": 18.86032485961914,
"step": 178
},
{
"KL/chosen_KL_mean": -18.39486312866211,
"KL/mean": -25.84204864501953,
"KL/rejected_KL_mean": -33.28923797607422,
"KL/std": 20.058395385742188,
"epoch": 0.2705971277399849,
"fcm_dpo/beta": 0.02877836301922798,
"fcm_dpo/delta": -0.03301222249865532,
"fcm_dpo/margin": 14.894369125366211,
"fcm_dpo/q_t": 0.4027270972728729,
"grad_norm": 14.578384399414062,
"learning_rate": 4.581424636586928e-07,
"logits/chosen": 0.39680802822113037,
"logits/rejected": 0.38149362802505493,
"logps/chosen": -84.06657409667969,
"logps/ref_chosen": -65.67171478271484,
"logps/ref_rejected": -75.32586669921875,
"logps/rejected": -108.61510467529297,
"loss": 1.1094,
"margin_dpo/margin_mean": 14.894369125366211,
"margin_dpo/margin_std": 23.633628845214844,
"step": 179
},
{
"KL/chosen_KL_mean": -14.09145736694336,
"KL/mean": -20.09218978881836,
"KL/rejected_KL_mean": -26.092912673950195,
"KL/std": 19.371274948120117,
"epoch": 0.272108843537415,
"fcm_dpo/beta": 0.029514048248529434,
"fcm_dpo/delta": 0.050767071545124054,
"fcm_dpo/margin": 12.001461029052734,
"fcm_dpo/q_t": 0.4218587279319763,
"grad_norm": 13.016547203063965,
"learning_rate": 4.5740715227200897e-07,
"logits/chosen": 0.24703602492809296,
"logits/rejected": 0.228146493434906,
"logps/chosen": -70.77426147460938,
"logps/ref_chosen": -56.68280792236328,
"logps/ref_rejected": -64.94414520263672,
"logps/rejected": -91.03706359863281,
"loss": 1.1642,
"margin_dpo/margin_mean": 12.001461029052734,
"margin_dpo/margin_std": 22.415281295776367,
"step": 180
},
{
"KL/chosen_KL_mean": -12.105340957641602,
"KL/mean": -20.832984924316406,
"KL/rejected_KL_mean": -29.560623168945312,
"KL/std": 20.69134521484375,
"epoch": 0.273620559334845,
"fcm_dpo/beta": 0.028023969382047653,
"fcm_dpo/delta": -0.10585962235927582,
"fcm_dpo/margin": 17.45528793334961,
"fcm_dpo/q_t": 0.38700011372566223,
"grad_norm": 15.768712997436523,
"learning_rate": 4.566660392614228e-07,
"logits/chosen": 0.42835646867752075,
"logits/rejected": 0.38607752323150635,
"logps/chosen": -72.88139343261719,
"logps/ref_chosen": -60.77604675292969,
"logps/ref_rejected": -83.98361206054688,
"logps/rejected": -113.54423522949219,
"loss": 1.031,
"margin_dpo/margin_mean": 17.45528793334961,
"margin_dpo/margin_std": 20.554719924926758,
"step": 181
},
{
"KL/chosen_KL_mean": -19.097064971923828,
"KL/mean": -28.098819732666016,
"KL/rejected_KL_mean": -37.1005744934082,
"KL/std": 22.797513961791992,
"epoch": 0.2751322751322751,
"fcm_dpo/beta": 0.026667404919862747,
"fcm_dpo/delta": -0.0935642197728157,
"fcm_dpo/margin": 18.003517150878906,
"fcm_dpo/q_t": 0.39077216386795044,
"grad_norm": 16.841510772705078,
"learning_rate": 4.5591914535745817e-07,
"logits/chosen": 0.4146467447280884,
"logits/rejected": 0.32692649960517883,
"logps/chosen": -79.35084533691406,
"logps/ref_chosen": -60.2537841796875,
"logps/ref_rejected": -89.7706298828125,
"logps/rejected": -126.87120056152344,
"loss": 1.071,
"margin_dpo/margin_mean": 18.003517150878906,
"margin_dpo/margin_std": 25.671062469482422,
"step": 182
},
{
"KL/chosen_KL_mean": -23.710100173950195,
"KL/mean": -27.463716506958008,
"KL/rejected_KL_mean": -31.217327117919922,
"KL/std": 21.911300659179688,
"epoch": 0.2766439909297052,
"fcm_dpo/beta": 0.027505898848176003,
"fcm_dpo/delta": 0.07738558948040009,
"fcm_dpo/margin": 7.507230281829834,
"fcm_dpo/q_t": 0.4531930983066559,
"grad_norm": 14.247156143188477,
"learning_rate": 4.551664914523433e-07,
"logits/chosen": 0.40092331171035767,
"logits/rejected": 0.3776063919067383,
"logps/chosen": -85.47151947021484,
"logps/ref_chosen": -61.76142120361328,
"logps/ref_rejected": -72.54627990722656,
"logps/rejected": -103.76361083984375,
"loss": 1.2818,
"margin_dpo/margin_mean": 7.507230281829834,
"margin_dpo/margin_std": 22.548179626464844,
"step": 183
},
{
"KL/chosen_KL_mean": -13.734045028686523,
"KL/mean": -20.633419036865234,
"KL/rejected_KL_mean": -27.532791137695312,
"KL/std": 19.101581573486328,
"epoch": 0.2781557067271353,
"fcm_dpo/beta": 0.02745695784687996,
"fcm_dpo/delta": 0.02308075502514839,
"fcm_dpo/margin": 13.798742294311523,
"fcm_dpo/q_t": 0.4148487448692322,
"grad_norm": 16.434070587158203,
"learning_rate": 4.544080985994258e-07,
"logits/chosen": 0.4927828907966614,
"logits/rejected": 0.42545321583747864,
"logps/chosen": -60.57476806640625,
"logps/ref_chosen": -46.840721130371094,
"logps/ref_rejected": -69.3609390258789,
"logps/rejected": -96.89373016357422,
"loss": 1.1172,
"margin_dpo/margin_mean": 13.798742294311523,
"margin_dpo/margin_std": 20.220142364501953,
"step": 184
},
{
"KL/chosen_KL_mean": -18.83686065673828,
"KL/mean": -27.118553161621094,
"KL/rejected_KL_mean": -35.40023422241211,
"KL/std": 23.780193328857422,
"epoch": 0.2796674225245654,
"fcm_dpo/beta": 0.026584237813949585,
"fcm_dpo/delta": -0.05228573456406593,
"fcm_dpo/margin": 16.563377380371094,
"fcm_dpo/q_t": 0.40145543217658997,
"grad_norm": 14.18293571472168,
"learning_rate": 4.5364398801258394e-07,
"logits/chosen": 0.43396925926208496,
"logits/rejected": 0.38681963086128235,
"logps/chosen": -71.15800476074219,
"logps/ref_chosen": -52.32114028930664,
"logps/ref_rejected": -68.3885726928711,
"logps/rejected": -103.78880310058594,
"loss": 1.1334,
"margin_dpo/margin_mean": 16.56337547302246,
"margin_dpo/margin_std": 28.58876609802246,
"step": 185
},
{
"KL/chosen_KL_mean": -16.70131492614746,
"KL/mean": -25.68690299987793,
"KL/rejected_KL_mean": -34.67249298095703,
"KL/std": 23.681175231933594,
"epoch": 0.2811791383219955,
"fcm_dpo/beta": 0.025996234267950058,
"fcm_dpo/delta": -0.07862785458564758,
"fcm_dpo/margin": 17.971176147460938,
"fcm_dpo/q_t": 0.3956528306007385,
"grad_norm": 17.511899948120117,
"learning_rate": 4.5287418106563354e-07,
"logits/chosen": 0.3419583737850189,
"logits/rejected": 0.299105703830719,
"logps/chosen": -84.12144470214844,
"logps/ref_chosen": -67.42012786865234,
"logps/ref_rejected": -82.50968933105469,
"logps/rejected": -117.18218231201172,
"loss": 1.0992,
"margin_dpo/margin_mean": 17.971176147460938,
"margin_dpo/margin_std": 28.787887573242188,
"step": 186
},
{
"KL/chosen_KL_mean": -22.477245330810547,
"KL/mean": -30.472248077392578,
"KL/rejected_KL_mean": -38.467254638671875,
"KL/std": 25.28232765197754,
"epoch": 0.28269085411942557,
"fcm_dpo/beta": 0.02547437697649002,
"fcm_dpo/delta": -0.008900739252567291,
"fcm_dpo/margin": 15.990015983581543,
"fcm_dpo/q_t": 0.4118683338165283,
"grad_norm": 16.354949951171875,
"learning_rate": 4.520986992917297e-07,
"logits/chosen": 0.4215141534805298,
"logits/rejected": 0.3655751347541809,
"logps/chosen": -98.00273895263672,
"logps/ref_chosen": -75.52549743652344,
"logps/ref_rejected": -94.76289367675781,
"logps/rejected": -133.2301483154297,
"loss": 1.1527,
"margin_dpo/margin_mean": 15.990015983581543,
"margin_dpo/margin_std": 29.899837493896484,
"step": 187
},
{
"KL/chosen_KL_mean": -20.62126922607422,
"KL/mean": -30.150714874267578,
"KL/rejected_KL_mean": -39.68015670776367,
"KL/std": 26.517009735107422,
"epoch": 0.2842025699168556,
"fcm_dpo/beta": 0.024722332134842873,
"fcm_dpo/delta": -0.08292864263057709,
"fcm_dpo/margin": 19.058883666992188,
"fcm_dpo/q_t": 0.3959978222846985,
"grad_norm": 13.136139869689941,
"learning_rate": 4.5131756438276466e-07,
"logits/chosen": 0.42119163274765015,
"logits/rejected": 0.37826281785964966,
"logps/chosen": -92.14459991455078,
"logps/ref_chosen": -71.52333068847656,
"logps/ref_rejected": -78.29949951171875,
"logps/rejected": -117.97966003417969,
"loss": 1.0819,
"margin_dpo/margin_mean": 19.058883666992188,
"margin_dpo/margin_std": 28.6160831451416,
"step": 188
},
{
"KL/chosen_KL_mean": -20.660493850708008,
"KL/mean": -29.015668869018555,
"KL/rejected_KL_mean": -37.37084197998047,
"KL/std": 26.431987762451172,
"epoch": 0.2857142857142857,
"fcm_dpo/beta": 0.023861508816480637,
"fcm_dpo/delta": -0.0019575804471969604,
"fcm_dpo/margin": 16.710351943969727,
"fcm_dpo/q_t": 0.41077572107315063,
"grad_norm": 15.193591117858887,
"learning_rate": 4.5053079818876096e-07,
"logits/chosen": 0.47629261016845703,
"logits/rejected": 0.492626428604126,
"logps/chosen": -92.83676147460938,
"logps/ref_chosen": -72.17626953125,
"logps/ref_rejected": -75.26313781738281,
"logps/rejected": -112.63397216796875,
"loss": 1.1338,
"margin_dpo/margin_mean": 16.71035385131836,
"margin_dpo/margin_std": 27.63460350036621,
"step": 189
},
{
"KL/chosen_KL_mean": -19.903602600097656,
"KL/mean": -32.08153533935547,
"KL/rejected_KL_mean": -44.25946807861328,
"KL/std": 27.69138526916504,
"epoch": 0.2872260015117158,
"fcm_dpo/beta": 0.02243766561150551,
"fcm_dpo/delta": -0.17649930715560913,
"fcm_dpo/margin": 24.355867385864258,
"fcm_dpo/q_t": 0.3752169609069824,
"grad_norm": 19.759334564208984,
"learning_rate": 4.4973842271726024e-07,
"logits/chosen": 0.5364293456077576,
"logits/rejected": 0.3797008693218231,
"logps/chosen": -74.52787780761719,
"logps/ref_chosen": -54.624271392822266,
"logps/ref_rejected": -101.47068786621094,
"logps/rejected": -145.73016357421875,
"loss": 1.0183,
"margin_dpo/margin_mean": 24.355865478515625,
"margin_dpo/margin_std": 30.53769302368164,
"step": 190
},
{
"KL/chosen_KL_mean": -29.63574981689453,
"KL/mean": -39.05760192871094,
"KL/rejected_KL_mean": -48.479454040527344,
"KL/std": 29.330245971679688,
"epoch": 0.2887377173091459,
"fcm_dpo/beta": 0.021781757473945618,
"fcm_dpo/delta": -0.01194111630320549,
"fcm_dpo/margin": 18.843711853027344,
"fcm_dpo/q_t": 0.4067850112915039,
"grad_norm": 15.226569175720215,
"learning_rate": 4.48940460132708e-07,
"logits/chosen": 0.5249963998794556,
"logits/rejected": 0.49379801750183105,
"logps/chosen": -102.56826782226562,
"logps/ref_chosen": -72.93251037597656,
"logps/ref_rejected": -89.95103454589844,
"logps/rejected": -138.43048095703125,
"loss": 1.1252,
"margin_dpo/margin_mean": 18.843713760375977,
"margin_dpo/margin_std": 31.241641998291016,
"step": 191
},
{
"KL/chosen_KL_mean": -33.76609802246094,
"KL/mean": -39.7779426574707,
"KL/rejected_KL_mean": -45.78978729248047,
"KL/std": 30.88973617553711,
"epoch": 0.29024943310657597,
"fcm_dpo/beta": 0.02324771136045456,
"fcm_dpo/delta": 0.1299854815006256,
"fcm_dpo/margin": 12.023693084716797,
"fcm_dpo/q_t": 0.43884414434432983,
"grad_norm": 15.96609878540039,
"learning_rate": 4.481369327558329e-07,
"logits/chosen": 0.5415316820144653,
"logits/rejected": 0.5124877691268921,
"logps/chosen": -87.76721954345703,
"logps/ref_chosen": -54.001121520996094,
"logps/ref_rejected": -63.531551361083984,
"logps/rejected": -109.32133483886719,
"loss": 1.2218,
"margin_dpo/margin_mean": 12.02369213104248,
"margin_dpo/margin_std": 27.488590240478516,
"step": 192
},
{
"KL/chosen_KL_mean": -26.880212783813477,
"KL/mean": -38.29176330566406,
"KL/rejected_KL_mean": -49.70331573486328,
"KL/std": 28.740842819213867,
"epoch": 0.29176114890400606,
"fcm_dpo/beta": 0.02194758504629135,
"fcm_dpo/delta": -0.12258221954107285,
"fcm_dpo/margin": 22.823104858398438,
"fcm_dpo/q_t": 0.38504743576049805,
"grad_norm": 13.015534400939941,
"learning_rate": 4.47327863063023e-07,
"logits/chosen": 0.41978591680526733,
"logits/rejected": 0.39997708797454834,
"logps/chosen": -83.62948608398438,
"logps/ref_chosen": -56.74927520751953,
"logps/ref_rejected": -58.80629348754883,
"logps/rejected": -108.50961303710938,
"loss": 1.0454,
"margin_dpo/margin_mean": 22.82310676574707,
"margin_dpo/margin_std": 29.807849884033203,
"step": 193
},
{
"KL/chosen_KL_mean": -30.15066909790039,
"KL/mean": -39.23329162597656,
"KL/rejected_KL_mean": -48.315921783447266,
"KL/std": 30.352039337158203,
"epoch": 0.29327286470143615,
"fcm_dpo/beta": 0.02143619954586029,
"fcm_dpo/delta": 0.00822427123785019,
"fcm_dpo/margin": 18.165246963500977,
"fcm_dpo/q_t": 0.4135337471961975,
"grad_norm": 17.924869537353516,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 0.526753842830658,
"logits/rejected": 0.4955962300300598,
"logps/chosen": -86.80010986328125,
"logps/ref_chosen": -56.64944076538086,
"logps/ref_rejected": -69.98954772949219,
"logps/rejected": -118.30546569824219,
"loss": 1.1742,
"margin_dpo/margin_mean": 18.165246963500977,
"margin_dpo/margin_std": 35.39740753173828,
"step": 194
},
{
"KL/chosen_KL_mean": -35.72456741333008,
"KL/mean": -48.061038970947266,
"KL/rejected_KL_mean": -60.39750671386719,
"KL/std": 33.23961639404297,
"epoch": 0.2947845804988662,
"fcm_dpo/beta": 0.020506273955106735,
"fcm_dpo/delta": -0.1278218924999237,
"fcm_dpo/margin": 24.672945022583008,
"fcm_dpo/q_t": 0.38505733013153076,
"grad_norm": 16.904264450073242,
"learning_rate": 4.4569318740967043e-07,
"logits/chosen": 0.43730270862579346,
"logits/rejected": 0.44750988483428955,
"logps/chosen": -106.13434600830078,
"logps/ref_chosen": -70.40977478027344,
"logps/ref_rejected": -74.39448547363281,
"logps/rejected": -134.7919921875,
"loss": 1.0634,
"margin_dpo/margin_mean": 24.67294692993164,
"margin_dpo/margin_std": 35.26850128173828,
"step": 195
},
{
"KL/chosen_KL_mean": -36.346405029296875,
"KL/mean": -45.410797119140625,
"KL/rejected_KL_mean": -54.47519302368164,
"KL/std": 34.48020935058594,
"epoch": 0.2962962962962963,
"fcm_dpo/beta": 0.020830024033784866,
"fcm_dpo/delta": 0.02376365289092064,
"fcm_dpo/margin": 18.128787994384766,
"fcm_dpo/q_t": 0.414675235748291,
"grad_norm": 18.091453552246094,
"learning_rate": 4.448676271745197e-07,
"logits/chosen": 0.5153293013572693,
"logits/rejected": 0.46972033381462097,
"logps/chosen": -95.573974609375,
"logps/ref_chosen": -59.227577209472656,
"logps/ref_rejected": -83.54757690429688,
"logps/rejected": -138.02276611328125,
"loss": 1.1365,
"margin_dpo/margin_mean": 18.128787994384766,
"margin_dpo/margin_std": 29.96463966369629,
"step": 196
},
{
"KL/chosen_KL_mean": -32.234886169433594,
"KL/mean": -43.992149353027344,
"KL/rejected_KL_mean": -55.74940872192383,
"KL/std": 32.462039947509766,
"epoch": 0.29780801209372637,
"fcm_dpo/beta": 0.020198438316583633,
"fcm_dpo/delta": -0.08790335804224014,
"fcm_dpo/margin": 23.51451873779297,
"fcm_dpo/q_t": 0.3938829302787781,
"grad_norm": 17.229829788208008,
"learning_rate": 4.440366160729392e-07,
"logits/chosen": 0.593604564666748,
"logits/rejected": 0.5391672849655151,
"logps/chosen": -83.76402282714844,
"logps/ref_chosen": -51.52912902832031,
"logps/ref_rejected": -73.70631408691406,
"logps/rejected": -129.45571899414062,
"loss": 1.1271,
"margin_dpo/margin_mean": 23.51451873779297,
"margin_dpo/margin_std": 40.65414047241211,
"step": 197
},
{
"KL/chosen_KL_mean": -35.11063003540039,
"KL/mean": -47.747779846191406,
"KL/rejected_KL_mean": -60.38494110107422,
"KL/std": 33.210731506347656,
"epoch": 0.29931972789115646,
"fcm_dpo/beta": 0.018990688025951385,
"fcm_dpo/delta": -0.09321387112140656,
"fcm_dpo/margin": 25.274303436279297,
"fcm_dpo/q_t": 0.3890335261821747,
"grad_norm": 15.211456298828125,
"learning_rate": 4.432001773500957e-07,
"logits/chosen": 0.5949510931968689,
"logits/rejected": 0.5511294603347778,
"logps/chosen": -94.893310546875,
"logps/ref_chosen": -59.78268051147461,
"logps/ref_rejected": -72.24533081054688,
"logps/rejected": -132.63026428222656,
"loss": 1.0603,
"margin_dpo/margin_mean": 25.274303436279297,
"margin_dpo/margin_std": 33.733585357666016,
"step": 198
},
{
"KL/chosen_KL_mean": -42.495819091796875,
"KL/mean": -52.347877502441406,
"KL/rejected_KL_mean": -62.19993209838867,
"KL/std": 33.97826385498047,
"epoch": 0.30083144368858655,
"fcm_dpo/beta": 0.0181651059538126,
"fcm_dpo/delta": -0.06439946591854095,
"fcm_dpo/margin": 19.704111099243164,
"fcm_dpo/q_t": 0.41771239042282104,
"grad_norm": 13.64396858215332,
"learning_rate": 4.4235833440297856e-07,
"logits/chosen": 0.5503775477409363,
"logits/rejected": 0.4570651054382324,
"logps/chosen": -98.8825912475586,
"logps/ref_chosen": -56.38677215576172,
"logps/ref_rejected": -74.56779479980469,
"logps/rejected": -136.76773071289062,
"loss": 1.1815,
"margin_dpo/margin_mean": 19.704111099243164,
"margin_dpo/margin_std": 38.327781677246094,
"step": 199
},
{
"KL/chosen_KL_mean": -34.580101013183594,
"KL/mean": -48.731632232666016,
"KL/rejected_KL_mean": -62.88316345214844,
"KL/std": 35.259788513183594,
"epoch": 0.30234315948601664,
"fcm_dpo/beta": 0.016897382214665413,
"fcm_dpo/delta": -0.09347662329673767,
"fcm_dpo/margin": 28.303054809570312,
"fcm_dpo/q_t": 0.39045828580856323,
"grad_norm": 12.926334381103516,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": 0.5941851139068604,
"logits/rejected": 0.512370228767395,
"logps/chosen": -92.40442657470703,
"logps/ref_chosen": -57.82432556152344,
"logps/ref_rejected": -89.28246307373047,
"logps/rejected": -152.16561889648438,
"loss": 1.0802,
"margin_dpo/margin_mean": 28.303054809570312,
"margin_dpo/margin_std": 41.966915130615234,
"step": 200
},
{
"KL/chosen_KL_mean": -47.82463836669922,
"KL/mean": -58.91985321044922,
"KL/rejected_KL_mean": -70.01507568359375,
"KL/std": 37.099159240722656,
"epoch": 0.30385487528344673,
"fcm_dpo/beta": 0.016795823350548744,
"fcm_dpo/delta": 0.029392898082733154,
"fcm_dpo/margin": 22.190425872802734,
"fcm_dpo/q_t": 0.41658544540405273,
"grad_norm": 15.126965522766113,
"learning_rate": 4.4065853017905953e-07,
"logits/chosen": 0.6271176338195801,
"logits/rejected": 0.5756611824035645,
"logps/chosen": -106.82440185546875,
"logps/ref_chosen": -58.999759674072266,
"logps/ref_rejected": -84.67575073242188,
"logps/rejected": -154.69082641601562,
"loss": 1.1508,
"margin_dpo/margin_mean": 22.190425872802734,
"margin_dpo/margin_std": 38.777530670166016,
"step": 201
},
{
"KL/chosen_KL_mean": -40.08701705932617,
"KL/mean": -54.63224792480469,
"KL/rejected_KL_mean": -69.17748260498047,
"KL/std": 35.3343505859375,
"epoch": 0.30536659108087677,
"fcm_dpo/beta": 0.01654578559100628,
"fcm_dpo/delta": -0.09474054723978043,
"fcm_dpo/margin": 29.090471267700195,
"fcm_dpo/q_t": 0.38745981454849243,
"grad_norm": 13.597728729248047,
"learning_rate": 4.3980061644943575e-07,
"logits/chosen": 0.5242820978164673,
"logits/rejected": 0.444463849067688,
"logps/chosen": -87.74766540527344,
"logps/ref_chosen": -47.660648345947266,
"logps/ref_rejected": -73.63249969482422,
"logps/rejected": -142.8099822998047,
"loss": 1.0619,
"margin_dpo/margin_mean": 29.090471267700195,
"margin_dpo/margin_std": 39.005897521972656,
"step": 202
},
{
"KL/chosen_KL_mean": -50.41577911376953,
"KL/mean": -63.223907470703125,
"KL/rejected_KL_mean": -76.03202819824219,
"KL/std": 36.80195999145508,
"epoch": 0.30687830687830686,
"fcm_dpo/beta": 0.016147522255778313,
"fcm_dpo/delta": -0.015619541518390179,
"fcm_dpo/margin": 25.616247177124023,
"fcm_dpo/q_t": 0.4067385792732239,
"grad_norm": 16.166540145874023,
"learning_rate": 4.3893739358856455e-07,
"logits/chosen": 0.5744930505752563,
"logits/rejected": 0.4879806637763977,
"logps/chosen": -112.74131774902344,
"logps/ref_chosen": -62.32553482055664,
"logps/ref_rejected": -99.37226104736328,
"logps/rejected": -175.404296875,
"loss": 1.115,
"margin_dpo/margin_mean": 25.616249084472656,
"margin_dpo/margin_std": 40.74905014038086,
"step": 203
},
{
"KL/chosen_KL_mean": -45.187042236328125,
"KL/mean": -58.972225189208984,
"KL/rejected_KL_mean": -72.75740051269531,
"KL/std": 37.236968994140625,
"epoch": 0.30839002267573695,
"fcm_dpo/beta": 0.015411125496029854,
"fcm_dpo/delta": -0.0340307280421257,
"fcm_dpo/margin": 27.570358276367188,
"fcm_dpo/q_t": 0.40242862701416016,
"grad_norm": 14.934419631958008,
"learning_rate": 4.380688857426449e-07,
"logits/chosen": 0.562065839767456,
"logits/rejected": 0.48749643564224243,
"logps/chosen": -95.81636047363281,
"logps/ref_chosen": -50.62931823730469,
"logps/ref_rejected": -66.60475158691406,
"logps/rejected": -139.36215209960938,
"loss": 1.0957,
"margin_dpo/margin_mean": 27.570356369018555,
"margin_dpo/margin_std": 38.67369079589844,
"step": 204
},
{
"KL/chosen_KL_mean": -52.95893859863281,
"KL/mean": -66.04808044433594,
"KL/rejected_KL_mean": -79.13722229003906,
"KL/std": 39.503875732421875,
"epoch": 0.30990173847316704,
"fcm_dpo/beta": 0.015607166104018688,
"fcm_dpo/delta": -0.009851815178990364,
"fcm_dpo/margin": 26.17829704284668,
"fcm_dpo/q_t": 0.40857863426208496,
"grad_norm": 16.72960090637207,
"learning_rate": 4.3719511720570814e-07,
"logits/chosen": 0.5846419334411621,
"logits/rejected": 0.5103527903556824,
"logps/chosen": -123.31511688232422,
"logps/ref_chosen": -70.3561782836914,
"logps/ref_rejected": -93.39848327636719,
"logps/rejected": -172.53570556640625,
"loss": 1.1293,
"margin_dpo/margin_mean": 26.178298950195312,
"margin_dpo/margin_std": 44.114990234375,
"step": 205
},
{
"KL/chosen_KL_mean": -55.539981842041016,
"KL/mean": -64.34222412109375,
"KL/rejected_KL_mean": -73.14447021484375,
"KL/std": 38.425743103027344,
"epoch": 0.31141345427059713,
"fcm_dpo/beta": 0.015734657645225525,
"fcm_dpo/delta": 0.022806979715824127,
"fcm_dpo/margin": 17.60448455810547,
"fcm_dpo/q_t": 0.4400056302547455,
"grad_norm": 14.786349296569824,
"learning_rate": 4.363161124189387e-07,
"logits/chosen": 0.6014150381088257,
"logits/rejected": 0.5844457149505615,
"logps/chosen": -123.18546295166016,
"logps/ref_chosen": -67.64547729492188,
"logps/ref_rejected": -79.89584350585938,
"logps/rejected": -153.04031372070312,
"loss": 1.2576,
"margin_dpo/margin_mean": 17.6044864654541,
"margin_dpo/margin_std": 47.01768493652344,
"step": 206
},
{
"KL/chosen_KL_mean": -63.995445251464844,
"KL/mean": -76.38787841796875,
"KL/rejected_KL_mean": -88.78030395507812,
"KL/std": 39.89057159423828,
"epoch": 0.3129251700680272,
"fcm_dpo/beta": 0.015928978100419044,
"fcm_dpo/delta": 0.005825323052704334,
"fcm_dpo/margin": 24.784862518310547,
"fcm_dpo/q_t": 0.41365471482276917,
"grad_norm": 13.180200576782227,
"learning_rate": 4.3543189596998986e-07,
"logits/chosen": 0.5777188539505005,
"logits/rejected": 0.5047867894172668,
"logps/chosen": -131.65963745117188,
"logps/ref_chosen": -67.66419219970703,
"logps/ref_rejected": -85.10249328613281,
"logps/rejected": -173.88279724121094,
"loss": 1.1359,
"margin_dpo/margin_mean": 24.784862518310547,
"margin_dpo/margin_std": 42.67274475097656,
"step": 207
},
{
"KL/chosen_KL_mean": -54.84553527832031,
"KL/mean": -61.52077865600586,
"KL/rejected_KL_mean": -68.1960220336914,
"KL/std": 38.0513916015625,
"epoch": 0.3144368858654573,
"fcm_dpo/beta": 0.01662435196340084,
"fcm_dpo/delta": 0.0744815543293953,
"fcm_dpo/margin": 13.350489616394043,
"fcm_dpo/q_t": 0.4523673355579376,
"grad_norm": 21.40291976928711,
"learning_rate": 4.3454249259229664e-07,
"logits/chosen": 0.5781328678131104,
"logits/rejected": 0.5493608713150024,
"logps/chosen": -112.5772476196289,
"logps/ref_chosen": -57.731712341308594,
"logps/ref_rejected": -74.19276428222656,
"logps/rejected": -142.3887939453125,
"loss": 1.3069,
"margin_dpo/margin_mean": 13.350488662719727,
"margin_dpo/margin_std": 44.465492248535156,
"step": 208
},
{
"KL/chosen_KL_mean": -54.61763000488281,
"KL/mean": -70.42623138427734,
"KL/rejected_KL_mean": -86.23484802246094,
"KL/std": 42.82319641113281,
"epoch": 0.31594860166288735,
"fcm_dpo/beta": 0.01570359617471695,
"fcm_dpo/delta": -0.1145251914858818,
"fcm_dpo/margin": 31.617210388183594,
"fcm_dpo/q_t": 0.3877779245376587,
"grad_norm": 15.687070846557617,
"learning_rate": 4.336479271643833e-07,
"logits/chosen": 0.5210272073745728,
"logits/rejected": 0.46536776423454285,
"logps/chosen": -123.16770935058594,
"logps/ref_chosen": -68.55007934570312,
"logps/ref_rejected": -87.90541076660156,
"logps/rejected": -174.1402587890625,
"loss": 1.0937,
"margin_dpo/margin_mean": 31.617210388183594,
"margin_dpo/margin_std": 50.869163513183594,
"step": 209
},
{
"KL/chosen_KL_mean": -53.08406066894531,
"KL/mean": -69.13988494873047,
"KL/rejected_KL_mean": -85.19570922851562,
"KL/std": 43.93097686767578,
"epoch": 0.31746031746031744,
"fcm_dpo/beta": 0.01513850037008524,
"fcm_dpo/delta": -0.10149059444665909,
"fcm_dpo/margin": 32.111637115478516,
"fcm_dpo/q_t": 0.39089614152908325,
"grad_norm": 15.86282730102539,
"learning_rate": 4.327482247091679e-07,
"logits/chosen": 0.6385220289230347,
"logits/rejected": 0.5306141376495361,
"logps/chosen": -110.35234069824219,
"logps/ref_chosen": -57.268272399902344,
"logps/ref_rejected": -85.72807312011719,
"logps/rejected": -170.9237823486328,
"loss": 1.0724,
"margin_dpo/margin_mean": 32.111637115478516,
"margin_dpo/margin_std": 45.491607666015625,
"step": 210
},
{
"KL/chosen_KL_mean": -48.97825622558594,
"KL/mean": -64.92655944824219,
"KL/rejected_KL_mean": -80.8748779296875,
"KL/std": 40.996124267578125,
"epoch": 0.31897203325774753,
"fcm_dpo/beta": 0.014405088499188423,
"fcm_dpo/delta": -0.06979866325855255,
"fcm_dpo/margin": 31.896615982055664,
"fcm_dpo/q_t": 0.39662906527519226,
"grad_norm": 17.855039596557617,
"learning_rate": 4.3184341039326217e-07,
"logits/chosen": 0.6159626245498657,
"logits/rejected": 0.5108903050422668,
"logps/chosen": -102.61896514892578,
"logps/ref_chosen": -53.640708923339844,
"logps/ref_rejected": -93.0387954711914,
"logps/rejected": -173.91366577148438,
"loss": 1.0677,
"margin_dpo/margin_mean": 31.896615982055664,
"margin_dpo/margin_std": 42.558563232421875,
"step": 211
},
{
"KL/chosen_KL_mean": -55.17749786376953,
"KL/mean": -70.99906158447266,
"KL/rejected_KL_mean": -86.82063293457031,
"KL/std": 43.388309478759766,
"epoch": 0.3204837490551776,
"fcm_dpo/beta": 0.013755107298493385,
"fcm_dpo/delta": -0.040556252002716064,
"fcm_dpo/margin": 31.643131256103516,
"fcm_dpo/q_t": 0.3995632529258728,
"grad_norm": 10.621484756469727,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 0.6260430812835693,
"logits/rejected": 0.5469928979873657,
"logps/chosen": -112.54423522949219,
"logps/ref_chosen": -57.36674499511719,
"logps/ref_rejected": -79.89643096923828,
"logps/rejected": -166.71707153320312,
"loss": 1.096,
"margin_dpo/margin_mean": 31.643129348754883,
"margin_dpo/margin_std": 46.78819274902344,
"step": 212
},
{
"KL/chosen_KL_mean": -44.155555725097656,
"KL/mean": -62.20250701904297,
"KL/rejected_KL_mean": -80.24946594238281,
"KL/std": 43.83998107910156,
"epoch": 0.3219954648526077,
"fcm_dpo/beta": 0.013000023551285267,
"fcm_dpo/delta": -0.08304151147603989,
"fcm_dpo/margin": 36.09390640258789,
"fcm_dpo/q_t": 0.3916971981525421,
"grad_norm": 11.67632007598877,
"learning_rate": 4.3001854756006724e-07,
"logits/chosen": 0.5951361656188965,
"logits/rejected": 0.5706344842910767,
"logps/chosen": -109.37667083740234,
"logps/ref_chosen": -65.22111511230469,
"logps/ref_rejected": -80.1810302734375,
"logps/rejected": -160.43048095703125,
"loss": 1.0649,
"margin_dpo/margin_mean": 36.09390640258789,
"margin_dpo/margin_std": 48.79711151123047,
"step": 213
},
{
"KL/chosen_KL_mean": -52.43152618408203,
"KL/mean": -69.8787841796875,
"KL/rejected_KL_mean": -87.3260269165039,
"KL/std": 44.159698486328125,
"epoch": 0.3235071806500378,
"fcm_dpo/beta": 0.012891286984086037,
"fcm_dpo/delta": -0.058638766407966614,
"fcm_dpo/margin": 34.89451217651367,
"fcm_dpo/q_t": 0.39819642901420593,
"grad_norm": 13.59187126159668,
"learning_rate": 4.290985500881143e-07,
"logits/chosen": 0.47652411460876465,
"logits/rejected": 0.46216946840286255,
"logps/chosen": -113.7238540649414,
"logps/ref_chosen": -61.292327880859375,
"logps/ref_rejected": -67.69841003417969,
"logps/rejected": -155.02444458007812,
"loss": 1.0924,
"margin_dpo/margin_mean": 34.89451599121094,
"margin_dpo/margin_std": 51.43964385986328,
"step": 214
},
{
"KL/chosen_KL_mean": -57.65421676635742,
"KL/mean": -75.66905212402344,
"KL/rejected_KL_mean": -93.68387603759766,
"KL/std": 45.626731872558594,
"epoch": 0.3250188964474679,
"fcm_dpo/beta": 0.012519944459199905,
"fcm_dpo/delta": -0.061683304607868195,
"fcm_dpo/margin": 36.02965545654297,
"fcm_dpo/q_t": 0.3975563645362854,
"grad_norm": 12.812163352966309,
"learning_rate": 4.281735428447157e-07,
"logits/chosen": 0.49123620986938477,
"logits/rejected": 0.3736293911933899,
"logps/chosen": -121.52335357666016,
"logps/ref_chosen": -63.869136810302734,
"logps/ref_rejected": -98.7657241821289,
"logps/rejected": -192.44960021972656,
"loss": 1.099,
"margin_dpo/margin_mean": 36.02965545654297,
"margin_dpo/margin_std": 53.505271911621094,
"step": 215
},
{
"KL/chosen_KL_mean": -48.55633544921875,
"KL/mean": -68.02696228027344,
"KL/rejected_KL_mean": -87.49758911132812,
"KL/std": 44.254615783691406,
"epoch": 0.32653061224489793,
"fcm_dpo/beta": 0.011713217943906784,
"fcm_dpo/delta": -0.06535829603672028,
"fcm_dpo/margin": 38.941246032714844,
"fcm_dpo/q_t": 0.3921303153038025,
"grad_norm": 22.394311904907227,
"learning_rate": 4.2724355170431247e-07,
"logits/chosen": 0.5988740921020508,
"logits/rejected": 0.5082088708877563,
"logps/chosen": -116.38128662109375,
"logps/ref_chosen": -67.824951171875,
"logps/ref_rejected": -96.40231323242188,
"logps/rejected": -183.89990234375,
"loss": 1.0456,
"margin_dpo/margin_mean": 38.941246032714844,
"margin_dpo/margin_std": 44.615882873535156,
"step": 216
},
{
"KL/chosen_KL_mean": -52.889949798583984,
"KL/mean": -73.1640625,
"KL/rejected_KL_mean": -93.43817901611328,
"KL/std": 45.83582305908203,
"epoch": 0.328042328042328,
"fcm_dpo/beta": 0.01119733415544033,
"fcm_dpo/delta": -0.06392455846071243,
"fcm_dpo/margin": 40.548240661621094,
"fcm_dpo/q_t": 0.39370107650756836,
"grad_norm": 10.61658000946045,
"learning_rate": 4.26308602680756e-07,
"logits/chosen": 0.5777255892753601,
"logits/rejected": 0.4716952443122864,
"logps/chosen": -113.39494323730469,
"logps/ref_chosen": -60.5049934387207,
"logps/ref_rejected": -84.26618194580078,
"logps/rejected": -177.70436096191406,
"loss": 1.0503,
"margin_dpo/margin_mean": 40.548240661621094,
"margin_dpo/margin_std": 47.78947448730469,
"step": 217
},
{
"KL/chosen_KL_mean": -58.71846008300781,
"KL/mean": -70.38279724121094,
"KL/rejected_KL_mean": -82.04712677001953,
"KL/std": 45.208160400390625,
"epoch": 0.3295540438397581,
"fcm_dpo/beta": 0.011268625035881996,
"fcm_dpo/delta": 0.02180871181190014,
"fcm_dpo/margin": 23.328662872314453,
"fcm_dpo/q_t": 0.4390827715396881,
"grad_norm": 11.695416450500488,
"learning_rate": 4.253687219265803e-07,
"logits/chosen": 0.4690364599227905,
"logits/rejected": 0.472156822681427,
"logps/chosen": -129.31277465820312,
"logps/ref_chosen": -70.59431457519531,
"logps/ref_rejected": -73.89038848876953,
"logps/rejected": -155.93751525878906,
"loss": 1.234,
"margin_dpo/margin_mean": 23.32866096496582,
"margin_dpo/margin_std": 54.49486541748047,
"step": 218
},
{
"KL/chosen_KL_mean": -54.63685989379883,
"KL/mean": -67.47407531738281,
"KL/rejected_KL_mean": -80.31128692626953,
"KL/std": 43.43921661376953,
"epoch": 0.3310657596371882,
"fcm_dpo/beta": 0.011822700500488281,
"fcm_dpo/delta": 0.10650709271430969,
"fcm_dpo/margin": 25.674427032470703,
"fcm_dpo/q_t": 0.4315260052680969,
"grad_norm": 11.611766815185547,
"learning_rate": 4.2442393573227043e-07,
"logits/chosen": 0.5588555335998535,
"logits/rejected": 0.5147054195404053,
"logps/chosen": -115.1278076171875,
"logps/ref_chosen": -60.490943908691406,
"logps/ref_rejected": -75.85001373291016,
"logps/rejected": -156.1613006591797,
"loss": 1.1746,
"margin_dpo/margin_mean": 25.674427032470703,
"margin_dpo/margin_std": 45.024803161621094,
"step": 219
},
{
"KL/chosen_KL_mean": -46.359901428222656,
"KL/mean": -61.93693923950195,
"KL/rejected_KL_mean": -77.51397705078125,
"KL/std": 46.67603302001953,
"epoch": 0.3325774754346183,
"fcm_dpo/beta": 0.012025123462080956,
"fcm_dpo/delta": 0.02687015011906624,
"fcm_dpo/margin": 31.15407943725586,
"fcm_dpo/q_t": 0.4164389967918396,
"grad_norm": 13.288846969604492,
"learning_rate": 4.234742705255272e-07,
"logits/chosen": 0.6493104100227356,
"logits/rejected": 0.5782603025436401,
"logps/chosen": -91.37329864501953,
"logps/ref_chosen": -45.013397216796875,
"logps/ref_rejected": -70.49369812011719,
"logps/rejected": -148.0076904296875,
"loss": 1.1525,
"margin_dpo/margin_mean": 31.154081344604492,
"margin_dpo/margin_std": 54.201744079589844,
"step": 220
},
{
"KL/chosen_KL_mean": -45.98303985595703,
"KL/mean": -63.76835632324219,
"KL/rejected_KL_mean": -81.55365753173828,
"KL/std": 46.344482421875,
"epoch": 0.3340891912320484,
"fcm_dpo/beta": 0.01212066039443016,
"fcm_dpo/delta": -0.03572237864136696,
"fcm_dpo/margin": 35.570621490478516,
"fcm_dpo/q_t": 0.4028802216053009,
"grad_norm": 13.347899436950684,
"learning_rate": 4.22519752870528e-07,
"logits/chosen": 0.577314019203186,
"logits/rejected": 0.494738906621933,
"logps/chosen": -105.0788803100586,
"logps/ref_chosen": -59.09584045410156,
"logps/ref_rejected": -88.64388275146484,
"logps/rejected": -170.19754028320312,
"loss": 1.0937,
"margin_dpo/margin_mean": 35.570621490478516,
"margin_dpo/margin_std": 52.52630615234375,
"step": 221
},
{
"KL/chosen_KL_mean": -46.059627532958984,
"KL/mean": -69.68561553955078,
"KL/rejected_KL_mean": -93.31161499023438,
"KL/std": 52.56859588623047,
"epoch": 0.3356009070294785,
"fcm_dpo/beta": 0.011211092583835125,
"fcm_dpo/delta": -0.15475650131702423,
"fcm_dpo/margin": 47.251983642578125,
"fcm_dpo/q_t": 0.37726062536239624,
"grad_norm": 11.579093933105469,
"learning_rate": 4.2156040946718343e-07,
"logits/chosen": 0.6488215327262878,
"logits/rejected": 0.5492867827415466,
"logps/chosen": -102.05731964111328,
"logps/ref_chosen": -55.9976921081543,
"logps/ref_rejected": -111.94727325439453,
"logps/rejected": -205.25888061523438,
"loss": 1.0177,
"margin_dpo/margin_mean": 47.251983642578125,
"margin_dpo/margin_std": 56.59059143066406,
"step": 222
},
{
"KL/chosen_KL_mean": -55.13484191894531,
"KL/mean": -73.27153015136719,
"KL/rejected_KL_mean": -91.40821838378906,
"KL/std": 49.953399658203125,
"epoch": 0.3371126228269085,
"fcm_dpo/beta": 0.01093536801636219,
"fcm_dpo/delta": 0.003693356178700924,
"fcm_dpo/margin": 36.27337646484375,
"fcm_dpo/q_t": 0.4097467362880707,
"grad_norm": 13.193449974060059,
"learning_rate": 4.2059626715039065e-07,
"logits/chosen": 0.6461388468742371,
"logits/rejected": 0.5862003564834595,
"logps/chosen": -115.02626037597656,
"logps/ref_chosen": -59.891422271728516,
"logps/ref_rejected": -86.28954315185547,
"logps/rejected": -177.69775390625,
"loss": 1.1034,
"margin_dpo/margin_mean": 36.27337646484375,
"margin_dpo/margin_std": 51.730384826660156,
"step": 223
},
{
"KL/chosen_KL_mean": -64.09883117675781,
"KL/mean": -74.00580596923828,
"KL/rejected_KL_mean": -83.91278076171875,
"KL/std": 49.710262298583984,
"epoch": 0.3386243386243386,
"fcm_dpo/beta": 0.011504280380904675,
"fcm_dpo/delta": 0.07703431695699692,
"fcm_dpo/margin": 19.813947677612305,
"fcm_dpo/q_t": 0.449110746383667,
"grad_norm": 16.505056381225586,
"learning_rate": 4.1962735288928304e-07,
"logits/chosen": 0.6655477285385132,
"logits/rejected": 0.6418548822402954,
"logps/chosen": -128.14346313476562,
"logps/ref_chosen": -64.04463195800781,
"logps/ref_rejected": -75.05450439453125,
"logps/rejected": -158.96728515625,
"loss": 1.2611,
"margin_dpo/margin_mean": 19.813949584960938,
"margin_dpo/margin_std": 53.515785217285156,
"step": 224
},
{
"KL/chosen_KL_mean": -58.37687683105469,
"KL/mean": -79.15052795410156,
"KL/rejected_KL_mean": -99.92417907714844,
"KL/std": 54.812477111816406,
"epoch": 0.3401360544217687,
"fcm_dpo/beta": 0.010968768037855625,
"fcm_dpo/delta": -0.06804075092077255,
"fcm_dpo/margin": 41.54730987548828,
"fcm_dpo/q_t": 0.3937387466430664,
"grad_norm": 11.77187728881836,
"learning_rate": 4.186536937864752e-07,
"logits/chosen": 0.606415867805481,
"logits/rejected": 0.4917227327823639,
"logps/chosen": -124.47270202636719,
"logps/ref_chosen": -66.0958251953125,
"logps/ref_rejected": -97.68675231933594,
"logps/rejected": -197.61093139648438,
"loss": 1.0544,
"margin_dpo/margin_mean": 41.54731369018555,
"margin_dpo/margin_std": 49.95249938964844,
"step": 225
},
{
"KL/chosen_KL_mean": -56.503387451171875,
"KL/mean": -72.67691802978516,
"KL/rejected_KL_mean": -88.85043334960938,
"KL/std": 49.93806076049805,
"epoch": 0.3416477702191988,
"fcm_dpo/beta": 0.011389894410967827,
"fcm_dpo/delta": 0.03310755267739296,
"fcm_dpo/margin": 32.347049713134766,
"fcm_dpo/q_t": 0.41660165786743164,
"grad_norm": 13.261937141418457,
"learning_rate": 4.176753170773052e-07,
"logits/chosen": 0.6644654273986816,
"logits/rejected": 0.6175069808959961,
"logps/chosen": -107.92025756835938,
"logps/ref_chosen": -51.4168701171875,
"logps/ref_rejected": -66.30068969726562,
"logps/rejected": -155.151123046875,
"loss": 1.1674,
"margin_dpo/margin_mean": 32.3470458984375,
"margin_dpo/margin_std": 60.95067596435547,
"step": 226
},
{
"KL/chosen_KL_mean": -60.69549560546875,
"KL/mean": -77.11978149414062,
"KL/rejected_KL_mean": -93.5440673828125,
"KL/std": 54.52500915527344,
"epoch": 0.3431594860166289,
"fcm_dpo/beta": 0.01134055107831955,
"fcm_dpo/delta": 0.03080243244767189,
"fcm_dpo/margin": 32.84856414794922,
"fcm_dpo/q_t": 0.41694962978363037,
"grad_norm": 13.040262222290039,
"learning_rate": 4.166922501290729e-07,
"logits/chosen": 0.6646544933319092,
"logits/rejected": 0.6282267570495605,
"logps/chosen": -118.68527221679688,
"logps/ref_chosen": -57.989776611328125,
"logps/ref_rejected": -75.05464172363281,
"logps/rejected": -168.5987091064453,
"loss": 1.1623,
"margin_dpo/margin_mean": 32.84856414794922,
"margin_dpo/margin_std": 61.44432830810547,
"step": 227
},
{
"KL/chosen_KL_mean": -64.08232116699219,
"KL/mean": -83.22323608398438,
"KL/rejected_KL_mean": -102.36415100097656,
"KL/std": 51.782081604003906,
"epoch": 0.34467120181405897,
"fcm_dpo/beta": 0.011403821408748627,
"fcm_dpo/delta": -0.04262223467230797,
"fcm_dpo/margin": 38.281822204589844,
"fcm_dpo/q_t": 0.3999175429344177,
"grad_norm": 15.122424125671387,
"learning_rate": 4.1570452044027405e-07,
"logits/chosen": 0.684807538986206,
"logits/rejected": 0.6012556552886963,
"logps/chosen": -119.64169311523438,
"logps/ref_chosen": -55.55936813354492,
"logps/ref_rejected": -77.02364349365234,
"logps/rejected": -179.38778686523438,
"loss": 1.0908,
"margin_dpo/margin_mean": 38.281822204589844,
"margin_dpo/margin_std": 54.99129867553711,
"step": 228
},
{
"KL/chosen_KL_mean": -54.697181701660156,
"KL/mean": -71.32321166992188,
"KL/rejected_KL_mean": -87.9492416381836,
"KL/std": 50.878578186035156,
"epoch": 0.34618291761148906,
"fcm_dpo/beta": 0.011315664276480675,
"fcm_dpo/delta": 0.02675747312605381,
"fcm_dpo/margin": 33.25206756591797,
"fcm_dpo/q_t": 0.4166818857192993,
"grad_norm": 16.81686019897461,
"learning_rate": 4.147121556398312e-07,
"logits/chosen": 0.7582691311836243,
"logits/rejected": 0.6779066324234009,
"logps/chosen": -105.49183654785156,
"logps/ref_chosen": -50.79466247558594,
"logps/ref_rejected": -78.4474105834961,
"logps/rejected": -166.3966522216797,
"loss": 1.1589,
"margin_dpo/margin_mean": 33.25206756591797,
"margin_dpo/margin_std": 61.93717956542969,
"step": 229
},
{
"KL/chosen_KL_mean": -59.01070022583008,
"KL/mean": -77.44501495361328,
"KL/rejected_KL_mean": -95.87932586669922,
"KL/std": 54.955116271972656,
"epoch": 0.3476946334089191,
"fcm_dpo/beta": 0.011499742977321148,
"fcm_dpo/delta": -0.031111925840377808,
"fcm_dpo/margin": 36.868629455566406,
"fcm_dpo/q_t": 0.403672993183136,
"grad_norm": 13.358758926391602,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 0.6462364196777344,
"logits/rejected": 0.648872971534729,
"logps/chosen": -115.73992919921875,
"logps/ref_chosen": -56.729225158691406,
"logps/ref_rejected": -62.99180603027344,
"logps/rejected": -158.87112426757812,
"loss": 1.1114,
"margin_dpo/margin_mean": 36.86862564086914,
"margin_dpo/margin_std": 56.367958068847656,
"step": 230
},
{
"KL/chosen_KL_mean": -65.8025131225586,
"KL/mean": -93.48104858398438,
"KL/rejected_KL_mean": -121.15958404541016,
"KL/std": 53.62238693237305,
"epoch": 0.3492063492063492,
"fcm_dpo/beta": 0.010107467882335186,
"fcm_dpo/delta": -0.1956898272037506,
"fcm_dpo/margin": 55.35707092285156,
"fcm_dpo/q_t": 0.36493778228759766,
"grad_norm": 12.845860481262207,
"learning_rate": 4.1271363186719835e-07,
"logits/chosen": 0.6005295515060425,
"logits/rejected": 0.5976369976997375,
"logps/chosen": -138.3996124267578,
"logps/ref_chosen": -72.59709930419922,
"logps/ref_rejected": -86.2322998046875,
"logps/rejected": -207.39187622070312,
"loss": 0.9845,
"margin_dpo/margin_mean": 55.35707092285156,
"margin_dpo/margin_std": 58.276145935058594,
"step": 231
},
{
"KL/chosen_KL_mean": -66.58941650390625,
"KL/mean": -85.17105102539062,
"KL/rejected_KL_mean": -103.752685546875,
"KL/std": 59.081756591796875,
"epoch": 0.3507180650037793,
"fcm_dpo/beta": 0.010086476802825928,
"fcm_dpo/delta": 0.028100861236453056,
"fcm_dpo/margin": 37.163265228271484,
"fcm_dpo/q_t": 0.4179150462150574,
"grad_norm": 12.528980255126953,
"learning_rate": 4.1170752879801436e-07,
"logits/chosen": 0.6124403476715088,
"logits/rejected": 0.5794280767440796,
"logps/chosen": -134.70794677734375,
"logps/ref_chosen": -68.1185302734375,
"logps/ref_rejected": -83.79415893554688,
"logps/rejected": -187.54684448242188,
"loss": 1.1554,
"margin_dpo/margin_mean": 37.163265228271484,
"margin_dpo/margin_std": 67.85264587402344,
"step": 232
},
{
"KL/chosen_KL_mean": -76.2654800415039,
"KL/mean": -89.84437561035156,
"KL/rejected_KL_mean": -103.42329406738281,
"KL/std": 53.858177185058594,
"epoch": 0.35222978080120937,
"fcm_dpo/beta": 0.009955944493412971,
"fcm_dpo/delta": -0.02221892587840557,
"fcm_dpo/margin": 27.157814025878906,
"fcm_dpo/q_t": 0.4383409917354584,
"grad_norm": 12.91989803314209,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": 0.6682592630386353,
"logits/rejected": 0.6073845028877258,
"logps/chosen": -131.33563232421875,
"logps/ref_chosen": -55.070152282714844,
"logps/ref_rejected": -66.61845397949219,
"logps/rejected": -170.041748046875,
"loss": 1.2163,
"margin_dpo/margin_mean": 27.157814025878906,
"margin_dpo/margin_std": 56.61811828613281,
"step": 233
},
{
"KL/chosen_KL_mean": -67.56138610839844,
"KL/mean": -80.51182556152344,
"KL/rejected_KL_mean": -93.46226501464844,
"KL/std": 52.28334426879883,
"epoch": 0.35374149659863946,
"fcm_dpo/beta": 0.009842677041888237,
"fcm_dpo/delta": -0.03836081176996231,
"fcm_dpo/margin": 25.900876998901367,
"fcm_dpo/q_t": 0.44125378131866455,
"grad_norm": 18.20611000061035,
"learning_rate": 4.09681781007452e-07,
"logits/chosen": 0.621496319770813,
"logits/rejected": 0.6191729307174683,
"logps/chosen": -123.48728942871094,
"logps/ref_chosen": -55.92589569091797,
"logps/ref_rejected": -51.11608123779297,
"logps/rejected": -144.57833862304688,
"loss": 1.2379,
"margin_dpo/margin_mean": 25.90087890625,
"margin_dpo/margin_std": 57.52040100097656,
"step": 234
},
{
"KL/chosen_KL_mean": -62.01856231689453,
"KL/mean": -85.31228637695312,
"KL/rejected_KL_mean": -108.60600280761719,
"KL/std": 57.6728630065918,
"epoch": 0.35525321239606955,
"fcm_dpo/beta": 0.009540073573589325,
"fcm_dpo/delta": -0.051198702305555344,
"fcm_dpo/margin": 46.587440490722656,
"fcm_dpo/q_t": 0.39602601528167725,
"grad_norm": 14.494193077087402,
"learning_rate": 4.08662192950594e-07,
"logits/chosen": 0.6710847616195679,
"logits/rejected": 0.6589834690093994,
"logps/chosen": -126.55828857421875,
"logps/ref_chosen": -64.53972625732422,
"logps/ref_rejected": -77.69151306152344,
"logps/rejected": -186.29751586914062,
"loss": 1.0562,
"margin_dpo/margin_mean": 46.58743667602539,
"margin_dpo/margin_std": 55.38169860839844,
"step": 235
},
{
"KL/chosen_KL_mean": -89.08969116210938,
"KL/mean": -106.5599365234375,
"KL/rejected_KL_mean": -124.0301742553711,
"KL/std": 55.50977325439453,
"epoch": 0.35676492819349964,
"fcm_dpo/beta": 0.009686114266514778,
"fcm_dpo/delta": 0.06868328154087067,
"fcm_dpo/margin": 34.94047546386719,
"fcm_dpo/q_t": 0.42309021949768066,
"grad_norm": 11.22431468963623,
"learning_rate": 4.076381667711306e-07,
"logits/chosen": 0.6049457788467407,
"logits/rejected": 0.5927116870880127,
"logps/chosen": -160.2444305419922,
"logps/ref_chosen": -71.15473937988281,
"logps/ref_rejected": -84.88541412353516,
"logps/rejected": -208.91558837890625,
"loss": 1.1686,
"margin_dpo/margin_mean": 34.94047546386719,
"margin_dpo/margin_std": 64.02879333496094,
"step": 236
},
{
"KL/chosen_KL_mean": -74.93379211425781,
"KL/mean": -96.43376159667969,
"KL/rejected_KL_mean": -117.93374633789062,
"KL/std": 55.61524963378906,
"epoch": 0.35827664399092973,
"fcm_dpo/beta": 0.009799846448004246,
"fcm_dpo/delta": -0.024707447737455368,
"fcm_dpo/margin": 42.99995422363281,
"fcm_dpo/q_t": 0.402961790561676,
"grad_norm": 15.107486724853516,
"learning_rate": 4.066097311132753e-07,
"logits/chosen": 0.6670963168144226,
"logits/rejected": 0.6589772701263428,
"logps/chosen": -151.0758056640625,
"logps/ref_chosen": -76.14201354980469,
"logps/ref_rejected": -80.88479614257812,
"logps/rejected": -198.81854248046875,
"loss": 1.1068,
"margin_dpo/margin_mean": 42.99995422363281,
"margin_dpo/margin_std": 64.99576568603516,
"step": 237
},
{
"KL/chosen_KL_mean": -66.69758605957031,
"KL/mean": -87.274658203125,
"KL/rejected_KL_mean": -107.85171508789062,
"KL/std": 57.005157470703125,
"epoch": 0.35978835978835977,
"fcm_dpo/beta": 0.009592464193701744,
"fcm_dpo/delta": 0.004950430244207382,
"fcm_dpo/margin": 41.15412902832031,
"fcm_dpo/q_t": 0.4092939496040344,
"grad_norm": 15.823814392089844,
"learning_rate": 4.0557691474458414e-07,
"logits/chosen": 0.6299471855163574,
"logits/rejected": 0.6191462278366089,
"logps/chosen": -135.5824432373047,
"logps/ref_chosen": -68.88484954833984,
"logps/ref_rejected": -75.8946304321289,
"logps/rejected": -183.746337890625,
"loss": 1.1119,
"margin_dpo/margin_mean": 41.15412139892578,
"margin_dpo/margin_std": 60.12267303466797,
"step": 238
},
{
"KL/chosen_KL_mean": -80.4689712524414,
"KL/mean": -102.43445587158203,
"KL/rejected_KL_mean": -124.39994049072266,
"KL/std": 62.442771911621094,
"epoch": 0.36130007558578986,
"fcm_dpo/beta": 0.009715170599520206,
"fcm_dpo/delta": -0.03162968158721924,
"fcm_dpo/margin": 43.93098068237305,
"fcm_dpo/q_t": 0.40267249941825867,
"grad_norm": 18.650232315063477,
"learning_rate": 4.045397465551513e-07,
"logits/chosen": 0.7951195240020752,
"logits/rejected": 0.6444146633148193,
"logps/chosen": -137.2407989501953,
"logps/ref_chosen": -56.771827697753906,
"logps/ref_rejected": -116.23050689697266,
"logps/rejected": -240.6304473876953,
"loss": 1.1093,
"margin_dpo/margin_mean": 43.93098449707031,
"margin_dpo/margin_std": 67.45944213867188,
"step": 239
},
{
"KL/chosen_KL_mean": -78.93304443359375,
"KL/mean": -105.90691375732422,
"KL/rejected_KL_mean": -132.8807830810547,
"KL/std": 62.66727828979492,
"epoch": 0.36281179138321995,
"fcm_dpo/beta": 0.009080219082534313,
"fcm_dpo/delta": -0.10564997047185898,
"fcm_dpo/margin": 53.94773483276367,
"fcm_dpo/q_t": 0.38679730892181396,
"grad_norm": 12.00704574584961,
"learning_rate": 4.0349825555680045e-07,
"logits/chosen": 0.68144690990448,
"logits/rejected": 0.5815201997756958,
"logps/chosen": -132.2871551513672,
"logps/ref_chosen": -53.35411071777344,
"logps/ref_rejected": -80.12019348144531,
"logps/rejected": -213.0009765625,
"loss": 1.0426,
"margin_dpo/margin_mean": 53.947731018066406,
"margin_dpo/margin_std": 67.94525909423828,
"step": 240
},
{
"KL/chosen_KL_mean": -74.51988983154297,
"KL/mean": -92.31866455078125,
"KL/rejected_KL_mean": -110.117431640625,
"KL/std": 59.715797424316406,
"epoch": 0.36432350718065004,
"fcm_dpo/beta": 0.009323697537183762,
"fcm_dpo/delta": 0.07455047965049744,
"fcm_dpo/margin": 35.59754943847656,
"fcm_dpo/q_t": 0.42614781856536865,
"grad_norm": 17.535839080810547,
"learning_rate": 4.0245247088227377e-07,
"logits/chosen": 0.6315206289291382,
"logits/rejected": 0.5956459641456604,
"logps/chosen": -146.41529846191406,
"logps/ref_chosen": -71.89541625976562,
"logps/ref_rejected": -83.03492736816406,
"logps/rejected": -193.15235900878906,
"loss": 1.1761,
"margin_dpo/margin_mean": 35.5975456237793,
"margin_dpo/margin_std": 67.8523941040039,
"step": 241
},
{
"KL/chosen_KL_mean": -76.13401794433594,
"KL/mean": -98.4712905883789,
"KL/rejected_KL_mean": -120.80856323242188,
"KL/std": 61.277618408203125,
"epoch": 0.36583522297808013,
"fcm_dpo/beta": 0.009163142181932926,
"fcm_dpo/delta": -0.012581542134284973,
"fcm_dpo/margin": 44.67455291748047,
"fcm_dpo/q_t": 0.40708714723587036,
"grad_norm": 11.770535469055176,
"learning_rate": 4.0140242178441665e-07,
"logits/chosen": 0.6444741487503052,
"logits/rejected": 0.624873161315918,
"logps/chosen": -134.0614471435547,
"logps/ref_chosen": -57.927433013916016,
"logps/ref_rejected": -67.838623046875,
"logps/rejected": -188.64718627929688,
"loss": 1.1051,
"margin_dpo/margin_mean": 44.67455291748047,
"margin_dpo/margin_std": 65.82492065429688,
"step": 242
},
{
"KL/chosen_KL_mean": -75.29219055175781,
"KL/mean": -95.34475708007812,
"KL/rejected_KL_mean": -115.39732360839844,
"KL/std": 60.416358947753906,
"epoch": 0.3673469387755102,
"fcm_dpo/beta": 0.009415372274816036,
"fcm_dpo/delta": 0.025139713659882545,
"fcm_dpo/margin": 40.10512924194336,
"fcm_dpo/q_t": 0.41358304023742676,
"grad_norm": 16.480270385742188,
"learning_rate": 4.003481376353596e-07,
"logits/chosen": 0.6188076734542847,
"logits/rejected": 0.6329231262207031,
"logps/chosen": -149.56887817382812,
"logps/ref_chosen": -74.27667236328125,
"logps/ref_rejected": -73.24340057373047,
"logps/rejected": -188.64071655273438,
"loss": 1.1272,
"margin_dpo/margin_mean": 40.105133056640625,
"margin_dpo/margin_std": 62.97103500366211,
"step": 243
},
{
"KL/chosen_KL_mean": -68.44050598144531,
"KL/mean": -94.90727996826172,
"KL/rejected_KL_mean": -121.3740463256836,
"KL/std": 60.30860900878906,
"epoch": 0.3688586545729403,
"fcm_dpo/beta": 0.009139430709183216,
"fcm_dpo/delta": -0.09782852977514267,
"fcm_dpo/margin": 52.93354797363281,
"fcm_dpo/q_t": 0.38723719120025635,
"grad_norm": 11.722265243530273,
"learning_rate": 3.9928964792569654e-07,
"logits/chosen": 0.6751892566680908,
"logits/rejected": 0.586534321308136,
"logps/chosen": -121.80440521240234,
"logps/ref_chosen": -53.36390686035156,
"logps/ref_rejected": -71.10276794433594,
"logps/rejected": -192.47682189941406,
"loss": 1.0407,
"margin_dpo/margin_mean": 52.93354797363281,
"margin_dpo/margin_std": 63.211524963378906,
"step": 244
},
{
"KL/chosen_KL_mean": -77.30024719238281,
"KL/mean": -107.91534423828125,
"KL/rejected_KL_mean": -138.53045654296875,
"KL/std": 62.56925964355469,
"epoch": 0.37037037037037035,
"fcm_dpo/beta": 0.008342870511114597,
"fcm_dpo/delta": -0.13193881511688232,
"fcm_dpo/margin": 61.230194091796875,
"fcm_dpo/q_t": 0.37750697135925293,
"grad_norm": 13.34937572479248,
"learning_rate": 3.982269822636601e-07,
"logits/chosen": 0.7413665056228638,
"logits/rejected": 0.7142044305801392,
"logps/chosen": -148.49534606933594,
"logps/ref_chosen": -71.19510650634766,
"logps/ref_rejected": -80.76235961914062,
"logps/rejected": -219.29281616210938,
"loss": 1.0067,
"margin_dpo/margin_mean": 61.230194091796875,
"margin_dpo/margin_std": 64.25137329101562,
"step": 245
},
{
"KL/chosen_KL_mean": -91.80043029785156,
"KL/mean": -113.60960388183594,
"KL/rejected_KL_mean": -135.4187774658203,
"KL/std": 62.362632751464844,
"epoch": 0.37188208616780044,
"fcm_dpo/beta": 0.008444206789135933,
"fcm_dpo/delta": 0.03405901789665222,
"fcm_dpo/margin": 43.61833953857422,
"fcm_dpo/q_t": 0.41692936420440674,
"grad_norm": 12.155149459838867,
"learning_rate": 3.971601703742932e-07,
"logits/chosen": 0.7366214990615845,
"logits/rejected": 0.6703383922576904,
"logps/chosen": -163.42147827148438,
"logps/ref_chosen": -71.62104797363281,
"logps/ref_rejected": -94.03392028808594,
"logps/rejected": -229.45269775390625,
"loss": 1.1571,
"margin_dpo/margin_mean": 43.61833953857422,
"margin_dpo/margin_std": 78.9627914428711,
"step": 246
},
{
"KL/chosen_KL_mean": -94.80854034423828,
"KL/mean": -109.75330352783203,
"KL/rejected_KL_mean": -124.69806671142578,
"KL/std": 63.64940643310547,
"epoch": 0.37339380196523053,
"fcm_dpo/beta": 0.008503669872879982,
"fcm_dpo/delta": 0.047118451446294785,
"fcm_dpo/margin": 29.8895206451416,
"fcm_dpo/q_t": 0.4427517056465149,
"grad_norm": 15.709403991699219,
"learning_rate": 3.960892420986177e-07,
"logits/chosen": 0.6784308552742004,
"logits/rejected": 0.6688522100448608,
"logps/chosen": -174.83108520507812,
"logps/ref_chosen": -80.02254486083984,
"logps/ref_rejected": -89.22705841064453,
"logps/rejected": -213.9251251220703,
"loss": 1.2434,
"margin_dpo/margin_mean": 29.88951873779297,
"margin_dpo/margin_std": 74.12858581542969,
"step": 247
},
{
"KL/chosen_KL_mean": -80.26335144042969,
"KL/mean": -104.7964096069336,
"KL/rejected_KL_mean": -129.3294677734375,
"KL/std": 65.8143310546875,
"epoch": 0.3749055177626606,
"fcm_dpo/beta": 0.008583104237914085,
"fcm_dpo/delta": -0.024302110075950623,
"fcm_dpo/margin": 49.06610107421875,
"fcm_dpo/q_t": 0.4045664370059967,
"grad_norm": 16.8564395904541,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 0.6709645986557007,
"logits/rejected": 0.7285152673721313,
"logps/chosen": -145.6413116455078,
"logps/ref_chosen": -65.37796020507812,
"logps/ref_rejected": -61.365787506103516,
"logps/rejected": -190.69525146484375,
"loss": 1.1079,
"margin_dpo/margin_mean": 49.06610107421875,
"margin_dpo/margin_std": 75.50920104980469,
"step": 248
},
{
"KL/chosen_KL_mean": -101.28555297851562,
"KL/mean": -109.88075256347656,
"KL/rejected_KL_mean": -118.47596740722656,
"KL/std": 61.5130500793457,
"epoch": 0.3764172335600907,
"fcm_dpo/beta": 0.008645986206829548,
"fcm_dpo/delta": 0.05666162446141243,
"fcm_dpo/margin": 17.190412521362305,
"fcm_dpo/q_t": 0.4651719331741333,
"grad_norm": 16.34800910949707,
"learning_rate": 3.9393515632731094e-07,
"logits/chosen": 0.6691962480545044,
"logits/rejected": 0.7120274305343628,
"logps/chosen": -175.8870086669922,
"logps/ref_chosen": -74.60145568847656,
"logps/ref_rejected": -63.79338455200195,
"logps/rejected": -182.26934814453125,
"loss": 1.3502,
"margin_dpo/margin_mean": 17.190414428710938,
"margin_dpo/margin_std": 75.83460998535156,
"step": 249
},
{
"KL/chosen_KL_mean": -88.96881103515625,
"KL/mean": -113.87507629394531,
"KL/rejected_KL_mean": -138.7813720703125,
"KL/std": 69.089599609375,
"epoch": 0.3779289493575208,
"fcm_dpo/beta": 0.008802786469459534,
"fcm_dpo/delta": -0.046542391180992126,
"fcm_dpo/margin": 49.81255340576172,
"fcm_dpo/q_t": 0.39933088421821594,
"grad_norm": 14.993885040283203,
"learning_rate": 3.9285205908608934e-07,
"logits/chosen": 0.7950924634933472,
"logits/rejected": 0.7543875575065613,
"logps/chosen": -150.90701293945312,
"logps/ref_chosen": -61.938209533691406,
"logps/ref_rejected": -72.21602630615234,
"logps/rejected": -210.9973907470703,
"loss": 1.0931,
"margin_dpo/margin_mean": 49.81255340576172,
"margin_dpo/margin_std": 71.54025268554688,
"step": 250
},
{
"KL/chosen_KL_mean": -96.157470703125,
"KL/mean": -113.1865234375,
"KL/rejected_KL_mean": -130.21556091308594,
"KL/std": 60.91024398803711,
"epoch": 0.3794406651549509,
"fcm_dpo/beta": 0.00891035981476307,
"fcm_dpo/delta": 0.10659044981002808,
"fcm_dpo/margin": 34.05808639526367,
"fcm_dpo/q_t": 0.43218693137168884,
"grad_norm": 23.815366744995117,
"learning_rate": 3.9176496596569265e-07,
"logits/chosen": 0.7540764808654785,
"logits/rejected": 0.7111141681671143,
"logps/chosen": -163.01441955566406,
"logps/ref_chosen": -66.85694885253906,
"logps/ref_rejected": -84.83396911621094,
"logps/rejected": -215.04953002929688,
"loss": 1.1989,
"margin_dpo/margin_mean": 34.05808639526367,
"margin_dpo/margin_std": 69.84614562988281,
"step": 251
},
{
"KL/chosen_KL_mean": -90.72431945800781,
"KL/mean": -103.80723571777344,
"KL/rejected_KL_mean": -116.89015197753906,
"KL/std": 63.69288635253906,
"epoch": 0.38095238095238093,
"fcm_dpo/beta": 0.0090938750654459,
"fcm_dpo/delta": -0.007932256907224655,
"fcm_dpo/margin": 26.16583824157715,
"fcm_dpo/q_t": 0.4449775218963623,
"grad_norm": 20.581787109375,
"learning_rate": 3.9067390737445254e-07,
"logits/chosen": 0.6826932430267334,
"logits/rejected": 0.6261128187179565,
"logps/chosen": -146.9482421875,
"logps/ref_chosen": -56.22393035888672,
"logps/ref_rejected": -77.1136245727539,
"logps/rejected": -194.0037841796875,
"loss": 1.2792,
"margin_dpo/margin_mean": 26.16583824157715,
"margin_dpo/margin_std": 74.60096740722656,
"step": 252
},
{
"KL/chosen_KL_mean": -89.88412475585938,
"KL/mean": -108.26229095458984,
"KL/rejected_KL_mean": -126.64045715332031,
"KL/std": 60.83412170410156,
"epoch": 0.382464096749811,
"fcm_dpo/beta": 0.008921799249947071,
"fcm_dpo/delta": -0.027868332341313362,
"fcm_dpo/margin": 36.7563362121582,
"fcm_dpo/q_t": 0.4237293004989624,
"grad_norm": 16.071880340576172,
"learning_rate": 3.8957891383162304e-07,
"logits/chosen": 0.7912516593933105,
"logits/rejected": 0.7534704208374023,
"logps/chosen": -142.09414672851562,
"logps/ref_chosen": -52.21001434326172,
"logps/ref_rejected": -58.75764846801758,
"logps/rejected": -185.39810180664062,
"loss": 1.1652,
"margin_dpo/margin_mean": 36.7563362121582,
"margin_dpo/margin_std": 63.523983001708984,
"step": 253
},
{
"KL/chosen_KL_mean": -98.28326416015625,
"KL/mean": -117.90261840820312,
"KL/rejected_KL_mean": -137.52197265625,
"KL/std": 67.03953552246094,
"epoch": 0.3839758125472411,
"fcm_dpo/beta": 0.009032230824232101,
"fcm_dpo/delta": 0.05078422278165817,
"fcm_dpo/margin": 39.23871612548828,
"fcm_dpo/q_t": 0.42154332995414734,
"grad_norm": 13.182265281677246,
"learning_rate": 3.884800159665276e-07,
"logits/chosen": 0.70362389087677,
"logits/rejected": 0.6483662128448486,
"logps/chosen": -163.91958618164062,
"logps/ref_chosen": -65.63632202148438,
"logps/ref_rejected": -82.34425354003906,
"logps/rejected": -219.86622619628906,
"loss": 1.1509,
"margin_dpo/margin_mean": 39.23871612548828,
"margin_dpo/margin_std": 67.09379577636719,
"step": 254
},
{
"KL/chosen_KL_mean": -87.982666015625,
"KL/mean": -110.94923400878906,
"KL/rejected_KL_mean": -133.91580200195312,
"KL/std": 64.83241271972656,
"epoch": 0.3854875283446712,
"fcm_dpo/beta": 0.009070008993148804,
"fcm_dpo/delta": -0.019309692084789276,
"fcm_dpo/margin": 45.93316650390625,
"fcm_dpo/q_t": 0.4051594138145447,
"grad_norm": 17.457691192626953,
"learning_rate": 3.873772445177015e-07,
"logits/chosen": 0.6465994119644165,
"logits/rejected": 0.6145304441452026,
"logps/chosen": -155.8937530517578,
"logps/ref_chosen": -67.91108703613281,
"logps/ref_rejected": -83.89114379882812,
"logps/rejected": -217.8069610595703,
"loss": 1.1121,
"margin_dpo/margin_mean": 45.933170318603516,
"margin_dpo/margin_std": 72.16119384765625,
"step": 255
},
{
"KL/chosen_KL_mean": -106.05748748779297,
"KL/mean": -128.17782592773438,
"KL/rejected_KL_mean": -150.2981719970703,
"KL/std": 61.828369140625,
"epoch": 0.3869992441421013,
"fcm_dpo/beta": 0.009000243619084358,
"fcm_dpo/delta": 0.0012618564069271088,
"fcm_dpo/margin": 44.24068069458008,
"fcm_dpo/q_t": 0.4116033911705017,
"grad_norm": 15.866179466247559,
"learning_rate": 3.862706303320329e-07,
"logits/chosen": 0.6873655319213867,
"logits/rejected": 0.6170613765716553,
"logps/chosen": -169.55746459960938,
"logps/ref_chosen": -63.49998474121094,
"logps/ref_rejected": -90.77104187011719,
"logps/rejected": -241.0692138671875,
"loss": 1.1498,
"margin_dpo/margin_mean": 44.240684509277344,
"margin_dpo/margin_std": 80.80990600585938,
"step": 256
},
{
"KL/chosen_KL_mean": -99.76148986816406,
"KL/mean": -128.57794189453125,
"KL/rejected_KL_mean": -157.39439392089844,
"KL/std": 69.79254913330078,
"epoch": 0.3885109599395314,
"fcm_dpo/beta": 0.00856287032365799,
"fcm_dpo/delta": -0.11203688383102417,
"fcm_dpo/margin": 57.63291931152344,
"fcm_dpo/q_t": 0.38817721605300903,
"grad_norm": 11.6730318069458,
"learning_rate": 3.851602043638994e-07,
"logits/chosen": 0.6632556915283203,
"logits/rejected": 0.5904806852340698,
"logps/chosen": -170.36215209960938,
"logps/ref_chosen": -70.60064697265625,
"logps/ref_rejected": -108.58313751220703,
"logps/rejected": -265.9775390625,
"loss": 1.0577,
"margin_dpo/margin_mean": 57.63291931152344,
"margin_dpo/margin_std": 79.96395111083984,
"step": 257
},
{
"KL/chosen_KL_mean": -102.39521026611328,
"KL/mean": -123.94535827636719,
"KL/rejected_KL_mean": -145.49549865722656,
"KL/std": 61.29582214355469,
"epoch": 0.3900226757369615,
"fcm_dpo/beta": 0.008610175922513008,
"fcm_dpo/delta": 0.03255663812160492,
"fcm_dpo/margin": 43.10028839111328,
"fcm_dpo/q_t": 0.4138616621494293,
"grad_norm": 14.99991512298584,
"learning_rate": 3.840459976743023e-07,
"logits/chosen": 0.7334133386611938,
"logits/rejected": 0.6779052019119263,
"logps/chosen": -161.64938354492188,
"logps/ref_chosen": -59.25416564941406,
"logps/ref_rejected": -85.58709716796875,
"logps/rejected": -231.0825958251953,
"loss": 1.12,
"margin_dpo/margin_mean": 43.10028839111328,
"margin_dpo/margin_std": 63.61687469482422,
"step": 258
},
{
"KL/chosen_KL_mean": -90.74569702148438,
"KL/mean": -121.82440185546875,
"KL/rejected_KL_mean": -152.9031219482422,
"KL/std": 67.67817687988281,
"epoch": 0.3915343915343915,
"fcm_dpo/beta": 0.008102967403829098,
"fcm_dpo/delta": -0.12510867416858673,
"fcm_dpo/margin": 62.157413482666016,
"fcm_dpo/q_t": 0.3824307322502136,
"grad_norm": 12.970193862915039,
"learning_rate": 3.8292804142999796e-07,
"logits/chosen": 0.6587529182434082,
"logits/rejected": 0.549065113067627,
"logps/chosen": -156.18057250976562,
"logps/ref_chosen": -65.43487548828125,
"logps/ref_rejected": -95.41731262207031,
"logps/rejected": -248.3204345703125,
"loss": 1.0524,
"margin_dpo/margin_mean": 62.157413482666016,
"margin_dpo/margin_std": 83.17296600341797,
"step": 259
},
{
"KL/chosen_KL_mean": -91.60452270507812,
"KL/mean": -115.55876922607422,
"KL/rejected_KL_mean": -139.5130157470703,
"KL/std": 61.73158264160156,
"epoch": 0.3930461073318216,
"fcm_dpo/beta": 0.008017702028155327,
"fcm_dpo/delta": 0.017222033813595772,
"fcm_dpo/margin": 47.908504486083984,
"fcm_dpo/q_t": 0.41276681423187256,
"grad_norm": 18.296205520629883,
"learning_rate": 3.818063669026256e-07,
"logits/chosen": 0.7184348702430725,
"logits/rejected": 0.6148696541786194,
"logps/chosen": -140.69410705566406,
"logps/ref_chosen": -49.08958435058594,
"logps/ref_rejected": -79.01708221435547,
"logps/rejected": -218.53009033203125,
"loss": 1.1336,
"margin_dpo/margin_mean": 47.90850067138672,
"margin_dpo/margin_std": 78.28831481933594,
"step": 260
},
{
"KL/chosen_KL_mean": -91.9481201171875,
"KL/mean": -112.35318756103516,
"KL/rejected_KL_mean": -132.7582550048828,
"KL/std": 60.71597671508789,
"epoch": 0.3945578231292517,
"fcm_dpo/beta": 0.008456382900476456,
"fcm_dpo/delta": 0.05926769599318504,
"fcm_dpo/margin": 40.810142517089844,
"fcm_dpo/q_t": 0.42231160402297974,
"grad_norm": 16.93952178955078,
"learning_rate": 3.806810054678331e-07,
"logits/chosen": 0.5660909414291382,
"logits/rejected": 0.6089234352111816,
"logps/chosen": -162.8205108642578,
"logps/ref_chosen": -70.87239074707031,
"logps/ref_rejected": -65.01522064208984,
"logps/rejected": -197.77346801757812,
"loss": 1.1557,
"margin_dpo/margin_mean": 40.810142517089844,
"margin_dpo/margin_std": 70.20431518554688,
"step": 261
},
{
"KL/chosen_KL_mean": -99.01957702636719,
"KL/mean": -119.05585479736328,
"KL/rejected_KL_mean": -139.09213256835938,
"KL/std": 61.233192443847656,
"epoch": 0.3960695389266818,
"fcm_dpo/beta": 0.008709060028195381,
"fcm_dpo/delta": 0.056011371314525604,
"fcm_dpo/margin": 40.072566986083984,
"fcm_dpo/q_t": 0.42116984724998474,
"grad_norm": 18.52885627746582,
"learning_rate": 3.7955198860439887e-07,
"logits/chosen": 0.7286216020584106,
"logits/rejected": 0.6594671607017517,
"logps/chosen": -166.89019775390625,
"logps/ref_chosen": -67.8706283569336,
"logps/ref_rejected": -88.7205810546875,
"logps/rejected": -227.81271362304688,
"loss": 1.1519,
"margin_dpo/margin_mean": 40.07257080078125,
"margin_dpo/margin_std": 68.45884704589844,
"step": 262
},
{
"KL/chosen_KL_mean": -93.94923400878906,
"KL/mean": -115.14928436279297,
"KL/rejected_KL_mean": -136.34933471679688,
"KL/std": 62.44976806640625,
"epoch": 0.3975812547241119,
"fcm_dpo/beta": 0.008855888620018959,
"fcm_dpo/delta": 0.027387959882616997,
"fcm_dpo/margin": 42.40009307861328,
"fcm_dpo/q_t": 0.41403600573539734,
"grad_norm": 16.228689193725586,
"learning_rate": 3.784193478933516e-07,
"logits/chosen": 0.6711243391036987,
"logits/rejected": 0.5571412444114685,
"logps/chosen": -149.14381408691406,
"logps/ref_chosen": -55.194583892822266,
"logps/ref_rejected": -80.54048156738281,
"logps/rejected": -216.88983154296875,
"loss": 1.1333,
"margin_dpo/margin_mean": 42.40009307861328,
"margin_dpo/margin_std": 68.54437255859375,
"step": 263
},
{
"KL/chosen_KL_mean": -95.62120819091797,
"KL/mean": -118.658203125,
"KL/rejected_KL_mean": -141.6951904296875,
"KL/std": 69.25521087646484,
"epoch": 0.39909297052154197,
"fcm_dpo/beta": 0.00880684144794941,
"fcm_dpo/delta": -0.0066429730504751205,
"fcm_dpo/margin": 46.07396697998047,
"fcm_dpo/q_t": 0.4083606004714966,
"grad_norm": 14.93736743927002,
"learning_rate": 3.7728311501708674e-07,
"logits/chosen": 0.5669023990631104,
"logits/rejected": 0.5313173532485962,
"logps/chosen": -178.79190063476562,
"logps/ref_chosen": -83.17068481445312,
"logps/ref_rejected": -88.33625793457031,
"logps/rejected": -230.0314483642578,
"loss": 1.1267,
"margin_dpo/margin_mean": 46.07396697998047,
"margin_dpo/margin_std": 75.87529754638672,
"step": 264
},
{
"KL/chosen_KL_mean": -91.21208190917969,
"KL/mean": -114.84857177734375,
"KL/rejected_KL_mean": -138.48507690429688,
"KL/std": 64.7674560546875,
"epoch": 0.40060468631897206,
"fcm_dpo/beta": 0.008796760812401772,
"fcm_dpo/delta": -0.018197163939476013,
"fcm_dpo/margin": 47.27299499511719,
"fcm_dpo/q_t": 0.40623968839645386,
"grad_norm": 12.336769104003906,
"learning_rate": 3.7614332175848027e-07,
"logits/chosen": 0.769055962562561,
"logits/rejected": 0.7016372680664062,
"logps/chosen": -142.87493896484375,
"logps/ref_chosen": -51.66284942626953,
"logps/ref_rejected": -67.1720962524414,
"logps/rejected": -205.65716552734375,
"loss": 1.138,
"margin_dpo/margin_mean": 47.27299499511719,
"margin_dpo/margin_std": 82.93624114990234,
"step": 265
},
{
"KL/chosen_KL_mean": -86.67529296875,
"KL/mean": -110.70747375488281,
"KL/rejected_KL_mean": -134.73963928222656,
"KL/std": 65.62783813476562,
"epoch": 0.4021164021164021,
"fcm_dpo/beta": 0.008618027903139591,
"fcm_dpo/delta": -0.016524948179721832,
"fcm_dpo/margin": 48.064353942871094,
"fcm_dpo/q_t": 0.4059688448905945,
"grad_norm": 22.678979873657227,
"learning_rate": 3.75e-07,
"logits/chosen": 0.6709597110748291,
"logits/rejected": 0.5944674015045166,
"logps/chosen": -144.12579345703125,
"logps/ref_chosen": -57.45049285888672,
"logps/ref_rejected": -77.60826110839844,
"logps/rejected": -212.347900390625,
"loss": 1.1195,
"margin_dpo/margin_mean": 48.064353942871094,
"margin_dpo/margin_std": 77.88729858398438,
"step": 266
},
{
"KL/chosen_KL_mean": -79.54592895507812,
"KL/mean": -98.55870056152344,
"KL/rejected_KL_mean": -117.57147216796875,
"KL/std": 63.595970153808594,
"epoch": 0.4036281179138322,
"fcm_dpo/beta": 0.008466208353638649,
"fcm_dpo/delta": -0.06876867264509201,
"fcm_dpo/margin": 38.025535583496094,
"fcm_dpo/q_t": 0.42545759677886963,
"grad_norm": 15.666740417480469,
"learning_rate": 3.738531817228131e-07,
"logits/chosen": 0.7178699970245361,
"logits/rejected": 0.6993035078048706,
"logps/chosen": -134.58128356933594,
"logps/ref_chosen": -55.03535079956055,
"logps/ref_rejected": -66.0953369140625,
"logps/rejected": -183.66680908203125,
"loss": 1.2011,
"margin_dpo/margin_mean": 38.02553939819336,
"margin_dpo/margin_std": 74.72657775878906,
"step": 267
},
{
"KL/chosen_KL_mean": -76.53323364257812,
"KL/mean": -100.77192687988281,
"KL/rejected_KL_mean": -125.0106201171875,
"KL/std": 66.77596282958984,
"epoch": 0.4051398337112623,
"fcm_dpo/beta": 0.008353805169463158,
"fcm_dpo/delta": -0.0063354093581438065,
"fcm_dpo/margin": 48.47738265991211,
"fcm_dpo/q_t": 0.40715789794921875,
"grad_norm": 15.87929916381836,
"learning_rate": 3.7270289900589204e-07,
"logits/chosen": 0.5625072717666626,
"logits/rejected": 0.5523948669433594,
"logps/chosen": -141.60498046875,
"logps/ref_chosen": -65.07174682617188,
"logps/ref_rejected": -71.42485809326172,
"logps/rejected": -196.43548583984375,
"loss": 1.0939,
"margin_dpo/margin_mean": 48.477378845214844,
"margin_dpo/margin_std": 65.1787109375,
"step": 268
},
{
"KL/chosen_KL_mean": -91.21495056152344,
"KL/mean": -115.9734115600586,
"KL/rejected_KL_mean": -140.7318878173828,
"KL/std": 69.93099212646484,
"epoch": 0.40665154950869237,
"fcm_dpo/beta": 0.008321639150381088,
"fcm_dpo/delta": -0.014926902949810028,
"fcm_dpo/margin": 49.516929626464844,
"fcm_dpo/q_t": 0.4059554636478424,
"grad_norm": 14.74932861328125,
"learning_rate": 3.7154918402511714e-07,
"logits/chosen": 0.7675411701202393,
"logits/rejected": 0.7207101583480835,
"logps/chosen": -158.35116577148438,
"logps/ref_chosen": -67.1362075805664,
"logps/ref_rejected": -82.55778503417969,
"logps/rejected": -223.2896728515625,
"loss": 1.0967,
"margin_dpo/margin_mean": 49.51692199707031,
"margin_dpo/margin_std": 69.16948699951172,
"step": 269
},
{
"KL/chosen_KL_mean": -89.67121124267578,
"KL/mean": -112.38426971435547,
"KL/rejected_KL_mean": -135.09732055664062,
"KL/std": 68.95745849609375,
"epoch": 0.40816326530612246,
"fcm_dpo/beta": 0.008279960602521896,
"fcm_dpo/delta": 0.026922225952148438,
"fcm_dpo/margin": 45.42612075805664,
"fcm_dpo/q_t": 0.41440480947494507,
"grad_norm": 16.14615821838379,
"learning_rate": 3.7039206905237656e-07,
"logits/chosen": 0.7438903450965881,
"logits/rejected": 0.6575706005096436,
"logps/chosen": -156.3599090576172,
"logps/ref_chosen": -66.6886978149414,
"logps/ref_rejected": -85.16129302978516,
"logps/rejected": -220.2586212158203,
"loss": 1.1353,
"margin_dpo/margin_mean": 45.426116943359375,
"margin_dpo/margin_std": 75.07006072998047,
"step": 270
},
{
"KL/chosen_KL_mean": -91.91019439697266,
"KL/mean": -108.07470703125,
"KL/rejected_KL_mean": -124.23921966552734,
"KL/std": 74.5631103515625,
"epoch": 0.40967498110355255,
"fcm_dpo/beta": 0.00850964430719614,
"fcm_dpo/delta": 0.0376083105802536,
"fcm_dpo/margin": 32.32902145385742,
"fcm_dpo/q_t": 0.4424757659435272,
"grad_norm": 19.207263946533203,
"learning_rate": 3.692315864546635e-07,
"logits/chosen": 0.7038223743438721,
"logits/rejected": 0.6382116079330444,
"logps/chosen": -164.3177490234375,
"logps/ref_chosen": -72.40754699707031,
"logps/ref_rejected": -92.06311798095703,
"logps/rejected": -216.30233764648438,
"loss": 1.2645,
"margin_dpo/margin_mean": 32.32902145385742,
"margin_dpo/margin_std": 89.17990112304688,
"step": 271
},
{
"KL/chosen_KL_mean": -75.82728576660156,
"KL/mean": -107.36683654785156,
"KL/rejected_KL_mean": -138.9063720703125,
"KL/std": 63.622032165527344,
"epoch": 0.41118669690098264,
"fcm_dpo/beta": 0.008081506937742233,
"fcm_dpo/delta": -0.12915097177028656,
"fcm_dpo/margin": 63.079078674316406,
"fcm_dpo/q_t": 0.3778059780597687,
"grad_norm": 15.495028495788574,
"learning_rate": 3.6806776869317067e-07,
"logits/chosen": 0.7020183801651001,
"logits/rejected": 0.7239981293678284,
"logps/chosen": -142.42868041992188,
"logps/ref_chosen": -66.60140228271484,
"logps/ref_rejected": -67.74340057373047,
"logps/rejected": -206.64976501464844,
"loss": 0.9983,
"margin_dpo/margin_mean": 63.079078674316406,
"margin_dpo/margin_std": 61.04596710205078,
"step": 272
},
{
"KL/chosen_KL_mean": -97.05889892578125,
"KL/mean": -120.1033706665039,
"KL/rejected_KL_mean": -143.1478271484375,
"KL/std": 70.52786254882812,
"epoch": 0.4126984126984127,
"fcm_dpo/beta": 0.007941139861941338,
"fcm_dpo/delta": 0.03800379857420921,
"fcm_dpo/margin": 46.088951110839844,
"fcm_dpo/q_t": 0.4187663197517395,
"grad_norm": 19.712482452392578,
"learning_rate": 3.669006483223828e-07,
"logits/chosen": 0.722759485244751,
"logits/rejected": 0.645821750164032,
"logps/chosen": -154.41378784179688,
"logps/ref_chosen": -57.35487747192383,
"logps/ref_rejected": -84.17168426513672,
"logps/rejected": -227.31951904296875,
"loss": 1.1726,
"margin_dpo/margin_mean": 46.088951110839844,
"margin_dpo/margin_std": 88.80404663085938,
"step": 273
},
{
"KL/chosen_KL_mean": -95.72166442871094,
"KL/mean": -120.80400085449219,
"KL/rejected_KL_mean": -145.88633728027344,
"KL/std": 67.72232055664062,
"epoch": 0.41421012849584277,
"fcm_dpo/beta": 0.008074736222624779,
"fcm_dpo/delta": -0.005904029123485088,
"fcm_dpo/margin": 50.16468048095703,
"fcm_dpo/q_t": 0.4069592356681824,
"grad_norm": 13.46408748626709,
"learning_rate": 3.657302579891656e-07,
"logits/chosen": 0.6265521049499512,
"logits/rejected": 0.609045147895813,
"logps/chosen": -155.3631591796875,
"logps/ref_chosen": -59.64149475097656,
"logps/ref_rejected": -68.29348754882812,
"logps/rejected": -214.17982482910156,
"loss": 1.12,
"margin_dpo/margin_mean": 50.16468048095703,
"margin_dpo/margin_std": 79.97119140625,
"step": 274
},
{
"KL/chosen_KL_mean": -93.24505615234375,
"KL/mean": -117.6572265625,
"KL/rejected_KL_mean": -142.06939697265625,
"KL/std": 64.98899841308594,
"epoch": 0.41572184429327286,
"fcm_dpo/beta": 0.008090265095233917,
"fcm_dpo/delta": 0.005442675203084946,
"fcm_dpo/margin": 48.82433319091797,
"fcm_dpo/q_t": 0.40933775901794434,
"grad_norm": 17.434755325317383,
"learning_rate": 3.645566304318526e-07,
"logits/chosen": 0.692554235458374,
"logits/rejected": 0.6077108383178711,
"logps/chosen": -146.51170349121094,
"logps/ref_chosen": -53.26664352416992,
"logps/ref_rejected": -73.84062194824219,
"logps/rejected": -215.91000366210938,
"loss": 1.0988,
"margin_dpo/margin_mean": 48.82433319091797,
"margin_dpo/margin_std": 66.89923858642578,
"step": 275
},
{
"KL/chosen_KL_mean": -86.27903747558594,
"KL/mean": -110.57057189941406,
"KL/rejected_KL_mean": -134.8621063232422,
"KL/std": 60.111572265625,
"epoch": 0.41723356009070295,
"fcm_dpo/beta": 0.00810546800494194,
"fcm_dpo/delta": 0.006946409121155739,
"fcm_dpo/margin": 48.58306121826172,
"fcm_dpo/q_t": 0.4088294506072998,
"grad_norm": 17.899261474609375,
"learning_rate": 3.633797984793294e-07,
"logits/chosen": 0.6355469226837158,
"logits/rejected": 0.6031860113143921,
"logps/chosen": -139.29983520507812,
"logps/ref_chosen": -53.02079772949219,
"logps/ref_rejected": -61.56678771972656,
"logps/rejected": -196.42889404296875,
"loss": 1.1036,
"margin_dpo/margin_mean": 48.58306121826172,
"margin_dpo/margin_std": 68.3792724609375,
"step": 276
},
{
"KL/chosen_KL_mean": -100.11457824707031,
"KL/mean": -111.95452880859375,
"KL/rejected_KL_mean": -123.79446411132812,
"KL/std": 69.6145248413086,
"epoch": 0.41874527588813304,
"fcm_dpo/beta": 0.008502400480210781,
"fcm_dpo/delta": 0.08324266970157623,
"fcm_dpo/margin": 23.679885864257812,
"fcm_dpo/q_t": 0.4552352726459503,
"grad_norm": 21.53533935546875,
"learning_rate": 3.6219979505011555e-07,
"logits/chosen": 0.743852972984314,
"logits/rejected": 0.7702978253364563,
"logps/chosen": -171.54757690429688,
"logps/ref_chosen": -71.43299102783203,
"logps/ref_rejected": -67.65852355957031,
"logps/rejected": -191.4530029296875,
"loss": 1.3012,
"margin_dpo/margin_mean": 23.679885864257812,
"margin_dpo/margin_std": 78.18290710449219,
"step": 277
},
{
"KL/chosen_KL_mean": -106.93940734863281,
"KL/mean": -130.02792358398438,
"KL/rejected_KL_mean": -153.11642456054688,
"KL/std": 70.84649658203125,
"epoch": 0.42025699168556313,
"fcm_dpo/beta": 0.0086892768740654,
"fcm_dpo/delta": -0.005315154790878296,
"fcm_dpo/margin": 46.17700958251953,
"fcm_dpo/q_t": 0.4069432318210602,
"grad_norm": 23.12691307067871,
"learning_rate": 3.6101665315144353e-07,
"logits/chosen": 0.6467447876930237,
"logits/rejected": 0.5822904109954834,
"logps/chosen": -174.0501708984375,
"logps/ref_chosen": -67.11076354980469,
"logps/ref_rejected": -88.74851989746094,
"logps/rejected": -241.8649444580078,
"loss": 1.1301,
"margin_dpo/margin_mean": 46.17700958251953,
"margin_dpo/margin_std": 73.78169250488281,
"step": 278
},
{
"KL/chosen_KL_mean": -83.75779724121094,
"KL/mean": -115.92961883544922,
"KL/rejected_KL_mean": -148.10145568847656,
"KL/std": 68.06074523925781,
"epoch": 0.4217687074829932,
"fcm_dpo/beta": 0.008041076362133026,
"fcm_dpo/delta": -0.1383289098739624,
"fcm_dpo/margin": 64.34365844726562,
"fcm_dpo/q_t": 0.376503050327301,
"grad_norm": 16.200817108154297,
"learning_rate": 3.5983040587833563e-07,
"logits/chosen": 0.6431260108947754,
"logits/rejected": 0.6045100092887878,
"logps/chosen": -138.25527954101562,
"logps/ref_chosen": -54.49748611450195,
"logps/ref_rejected": -70.42373657226562,
"logps/rejected": -218.52517700195312,
"loss": 1.0068,
"margin_dpo/margin_mean": 64.34365844726562,
"margin_dpo/margin_std": 67.10409545898438,
"step": 279
},
{
"KL/chosen_KL_mean": -89.94000244140625,
"KL/mean": -123.85331726074219,
"KL/rejected_KL_mean": -157.76663208007812,
"KL/std": 71.78616333007812,
"epoch": 0.42328042328042326,
"fcm_dpo/beta": 0.007400077767670155,
"fcm_dpo/delta": -0.12002548575401306,
"fcm_dpo/margin": 67.82662963867188,
"fcm_dpo/q_t": 0.3807293772697449,
"grad_norm": 11.438913345336914,
"learning_rate": 3.586410864126781e-07,
"logits/chosen": 0.7191169261932373,
"logits/rejected": 0.6715154647827148,
"logps/chosen": -150.37281799316406,
"logps/ref_chosen": -60.43281173706055,
"logps/ref_rejected": -78.39051818847656,
"logps/rejected": -236.15716552734375,
"loss": 1.0056,
"margin_dpo/margin_mean": 67.82662963867188,
"margin_dpo/margin_std": 68.43364715576172,
"step": 280
},
{
"KL/chosen_KL_mean": -92.57947540283203,
"KL/mean": -119.57850646972656,
"KL/rejected_KL_mean": -146.57754516601562,
"KL/std": 70.05896759033203,
"epoch": 0.42479213907785335,
"fcm_dpo/beta": 0.007345996797084808,
"fcm_dpo/delta": 0.0029397867619991302,
"fcm_dpo/margin": 53.998077392578125,
"fcm_dpo/q_t": 0.4082247316837311,
"grad_norm": 12.51746940612793,
"learning_rate": 3.574487280222929e-07,
"logits/chosen": 0.6676753163337708,
"logits/rejected": 0.6956182718276978,
"logps/chosen": -152.861572265625,
"logps/ref_chosen": -60.2820930480957,
"logps/ref_rejected": -62.04009246826172,
"logps/rejected": -208.61764526367188,
"loss": 1.1074,
"margin_dpo/margin_mean": 53.998077392578125,
"margin_dpo/margin_std": 77.513427734375,
"step": 281
},
{
"KL/chosen_KL_mean": -101.51087951660156,
"KL/mean": -125.53376770019531,
"KL/rejected_KL_mean": -149.55665588378906,
"KL/std": 70.40267944335938,
"epoch": 0.42630385487528344,
"fcm_dpo/beta": 0.007150539197027683,
"fcm_dpo/delta": -0.056217122822999954,
"fcm_dpo/margin": 48.0457763671875,
"fcm_dpo/q_t": 0.4206792414188385,
"grad_norm": 17.20449447631836,
"learning_rate": 3.562533640600075e-07,
"logits/chosen": 0.5914024114608765,
"logits/rejected": 0.549197256565094,
"logps/chosen": -162.13479614257812,
"logps/ref_chosen": -60.623924255371094,
"logps/ref_rejected": -68.67400360107422,
"logps/rejected": -218.23065185546875,
"loss": 1.1632,
"margin_dpo/margin_mean": 48.045772552490234,
"margin_dpo/margin_std": 82.70968627929688,
"step": 282
},
{
"KL/chosen_KL_mean": -98.51730346679688,
"KL/mean": -124.37229919433594,
"KL/rejected_KL_mean": -150.22727966308594,
"KL/std": 74.53094482421875,
"epoch": 0.42781557067271353,
"fcm_dpo/beta": 0.007138520944863558,
"fcm_dpo/delta": 0.034711360931396484,
"fcm_dpo/margin": 51.70996856689453,
"fcm_dpo/q_t": 0.416533887386322,
"grad_norm": 13.802699089050293,
"learning_rate": 3.550550279627215e-07,
"logits/chosen": 0.6436029672622681,
"logits/rejected": 0.5408251881599426,
"logps/chosen": -166.16505432128906,
"logps/ref_chosen": -67.64775085449219,
"logps/ref_rejected": -99.96835327148438,
"logps/rejected": -250.19564819335938,
"loss": 1.1355,
"margin_dpo/margin_mean": 51.70996856689453,
"margin_dpo/margin_std": 84.02940368652344,
"step": 283
},
{
"KL/chosen_KL_mean": -94.60420227050781,
"KL/mean": -119.55040740966797,
"KL/rejected_KL_mean": -144.49659729003906,
"KL/std": 73.55546569824219,
"epoch": 0.4293272864701436,
"fcm_dpo/beta": 0.007238998077809811,
"fcm_dpo/delta": 0.04319431632757187,
"fcm_dpo/margin": 49.892398834228516,
"fcm_dpo/q_t": 0.41724467277526855,
"grad_norm": 12.608574867248535,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 0.7289709448814392,
"logits/rejected": 0.6605273485183716,
"logps/chosen": -151.57162475585938,
"logps/ref_chosen": -56.96742630004883,
"logps/ref_rejected": -86.36236572265625,
"logps/rejected": -230.85897827148438,
"loss": 1.1231,
"margin_dpo/margin_mean": 49.892398834228516,
"margin_dpo/margin_std": 71.17445373535156,
"step": 284
},
{
"KL/chosen_KL_mean": -108.02169799804688,
"KL/mean": -129.18577575683594,
"KL/rejected_KL_mean": -150.34986877441406,
"KL/std": 66.75422668457031,
"epoch": 0.4308390022675737,
"fcm_dpo/beta": 0.007741519715636969,
"fcm_dpo/delta": 0.07731801271438599,
"fcm_dpo/margin": 42.32817077636719,
"fcm_dpo/q_t": 0.4266880750656128,
"grad_norm": 18.31220245361328,
"learning_rate": 3.5264957352549375e-07,
"logits/chosen": 0.6713681221008301,
"logits/rejected": 0.6448211669921875,
"logps/chosen": -179.67779541015625,
"logps/ref_chosen": -71.65611267089844,
"logps/ref_rejected": -81.63829803466797,
"logps/rejected": -231.9881591796875,
"loss": 1.1678,
"margin_dpo/margin_mean": 42.32816696166992,
"margin_dpo/margin_std": 75.15558624267578,
"step": 285
},
{
"KL/chosen_KL_mean": -103.962890625,
"KL/mean": -138.55560302734375,
"KL/rejected_KL_mean": -173.14833068847656,
"KL/std": 73.11457824707031,
"epoch": 0.4323507180650038,
"fcm_dpo/beta": 0.007291554007679224,
"fcm_dpo/delta": -0.12348268926143646,
"fcm_dpo/margin": 69.1854476928711,
"fcm_dpo/q_t": 0.38189804553985596,
"grad_norm": 12.251094818115234,
"learning_rate": 3.514425224712835e-07,
"logits/chosen": 0.6184149384498596,
"logits/rejected": 0.5270093083381653,
"logps/chosen": -165.04241943359375,
"logps/ref_chosen": -61.07952117919922,
"logps/ref_rejected": -91.28128051757812,
"logps/rejected": -264.42962646484375,
"loss": 1.0169,
"margin_dpo/margin_mean": 69.1854476928711,
"margin_dpo/margin_std": 77.75883483886719,
"step": 286
},
{
"KL/chosen_KL_mean": -86.13887023925781,
"KL/mean": -118.71305847167969,
"KL/rejected_KL_mean": -151.28726196289062,
"KL/std": 72.01513671875,
"epoch": 0.43386243386243384,
"fcm_dpo/beta": 0.006978640798479319,
"fcm_dpo/delta": -0.06320708990097046,
"fcm_dpo/margin": 65.14839172363281,
"fcm_dpo/q_t": 0.3935144543647766,
"grad_norm": 11.138228416442871,
"learning_rate": 3.502326338516534e-07,
"logits/chosen": 0.7138686180114746,
"logits/rejected": 0.6729492545127869,
"logps/chosen": -132.17465209960938,
"logps/ref_chosen": -46.035789489746094,
"logps/ref_rejected": -59.95293426513672,
"logps/rejected": -211.24020385742188,
"loss": 1.0475,
"margin_dpo/margin_mean": 65.14839172363281,
"margin_dpo/margin_std": 75.49003601074219,
"step": 287
},
{
"KL/chosen_KL_mean": -108.60914611816406,
"KL/mean": -133.51132202148438,
"KL/rejected_KL_mean": -158.41348266601562,
"KL/std": 69.94869232177734,
"epoch": 0.43537414965986393,
"fcm_dpo/beta": 0.007091499399393797,
"fcm_dpo/delta": 0.05196710675954819,
"fcm_dpo/margin": 49.80432891845703,
"fcm_dpo/q_t": 0.4193596839904785,
"grad_norm": 13.565356254577637,
"learning_rate": 3.490199415097892e-07,
"logits/chosen": 0.5789628028869629,
"logits/rejected": 0.5190857648849487,
"logps/chosen": -174.0,
"logps/ref_chosen": -65.3908462524414,
"logps/ref_rejected": -88.53607940673828,
"logps/rejected": -246.94955444335938,
"loss": 1.1339,
"margin_dpo/margin_mean": 49.80432891845703,
"margin_dpo/margin_std": 76.44319915771484,
"step": 288
},
{
"KL/chosen_KL_mean": -109.88246154785156,
"KL/mean": -133.86720275878906,
"KL/rejected_KL_mean": -157.85194396972656,
"KL/std": 71.7451171875,
"epoch": 0.436885865457294,
"fcm_dpo/beta": 0.0073424819856882095,
"fcm_dpo/delta": 0.05256052315235138,
"fcm_dpo/margin": 47.9694938659668,
"fcm_dpo/q_t": 0.42131608724594116,
"grad_norm": 11.81661319732666,
"learning_rate": 3.4780447936730247e-07,
"logits/chosen": 0.7805777788162231,
"logits/rejected": 0.7439557313919067,
"logps/chosen": -164.47608947753906,
"logps/ref_chosen": -54.5936279296875,
"logps/ref_rejected": -67.20855712890625,
"logps/rejected": -225.0605010986328,
"loss": 1.1489,
"margin_dpo/margin_mean": 47.9694938659668,
"margin_dpo/margin_std": 80.595703125,
"step": 289
},
{
"KL/chosen_KL_mean": -122.74056243896484,
"KL/mean": -149.30039978027344,
"KL/rejected_KL_mean": -175.86026000976562,
"KL/std": 70.5626220703125,
"epoch": 0.4383975812547241,
"fcm_dpo/beta": 0.007285839878022671,
"fcm_dpo/delta": 0.01345156878232956,
"fcm_dpo/margin": 53.11968994140625,
"fcm_dpo/q_t": 0.4115050137042999,
"grad_norm": 14.872110366821289,
"learning_rate": 3.465862814232821e-07,
"logits/chosen": 0.7720860242843628,
"logits/rejected": 0.694909930229187,
"logps/chosen": -184.12513732910156,
"logps/ref_chosen": -61.38457489013672,
"logps/ref_rejected": -91.92778015136719,
"logps/rejected": -267.78802490234375,
"loss": 1.1188,
"margin_dpo/margin_mean": 53.11968994140625,
"margin_dpo/margin_std": 80.01852416992188,
"step": 290
},
{
"KL/chosen_KL_mean": -111.4900894165039,
"KL/mean": -144.89549255371094,
"KL/rejected_KL_mean": -178.30091857910156,
"KL/std": 73.21499633789062,
"epoch": 0.4399092970521542,
"fcm_dpo/beta": 0.007250026799738407,
"fcm_dpo/delta": -0.10002140700817108,
"fcm_dpo/margin": 66.81082153320312,
"fcm_dpo/q_t": 0.3905301094055176,
"grad_norm": 14.139022827148438,
"learning_rate": 3.4536538175334343e-07,
"logits/chosen": 0.802875280380249,
"logits/rejected": 0.7305212020874023,
"logps/chosen": -162.35311889648438,
"logps/ref_chosen": -50.863037109375,
"logps/ref_rejected": -82.20868682861328,
"logps/rejected": -260.50958251953125,
"loss": 1.0554,
"margin_dpo/margin_mean": 66.8108139038086,
"margin_dpo/margin_std": 86.83169555664062,
"step": 291
},
{
"KL/chosen_KL_mean": -114.57568359375,
"KL/mean": -136.93133544921875,
"KL/rejected_KL_mean": -159.2869873046875,
"KL/std": 76.05390930175781,
"epoch": 0.4414210128495843,
"fcm_dpo/beta": 0.007219928316771984,
"fcm_dpo/delta": 0.08566851168870926,
"fcm_dpo/margin": 44.71131134033203,
"fcm_dpo/q_t": 0.4291321933269501,
"grad_norm": 16.09522247314453,
"learning_rate": 3.4414181450867465e-07,
"logits/chosen": 0.7329747676849365,
"logits/rejected": 0.6827974319458008,
"logps/chosen": -178.924560546875,
"logps/ref_chosen": -64.34888458251953,
"logps/ref_rejected": -72.86434173583984,
"logps/rejected": -232.15133666992188,
"loss": 1.195,
"margin_dpo/margin_mean": 44.71131134033203,
"margin_dpo/margin_std": 93.11201477050781,
"step": 292
},
{
"KL/chosen_KL_mean": -108.29093933105469,
"KL/mean": -143.0504608154297,
"KL/rejected_KL_mean": -177.80999755859375,
"KL/std": 77.0187759399414,
"epoch": 0.4429327286470144,
"fcm_dpo/beta": 0.007083355449140072,
"fcm_dpo/delta": -0.10814375430345535,
"fcm_dpo/margin": 69.51905822753906,
"fcm_dpo/q_t": 0.387387752532959,
"grad_norm": 10.112757682800293,
"learning_rate": 3.4291561391508185e-07,
"logits/chosen": 0.827242374420166,
"logits/rejected": 0.7373911142349243,
"logps/chosen": -163.160400390625,
"logps/ref_chosen": -54.869468688964844,
"logps/ref_rejected": -81.858642578125,
"logps/rejected": -259.66864013671875,
"loss": 1.0591,
"margin_dpo/margin_mean": 69.51905822753906,
"margin_dpo/margin_std": 93.7623291015625,
"step": 293
},
{
"KL/chosen_KL_mean": -117.05465698242188,
"KL/mean": -140.6841583251953,
"KL/rejected_KL_mean": -164.31365966796875,
"KL/std": 78.87942504882812,
"epoch": 0.4444444444444444,
"fcm_dpo/beta": 0.007031150162220001,
"fcm_dpo/delta": 0.07512365281581879,
"fcm_dpo/margin": 47.259010314941406,
"fcm_dpo/q_t": 0.4250057339668274,
"grad_norm": 11.902660369873047,
"learning_rate": 3.4168681427203153e-07,
"logits/chosen": 0.7192553877830505,
"logits/rejected": 0.6725665330886841,
"logps/chosen": -173.72555541992188,
"logps/ref_chosen": -56.670902252197266,
"logps/ref_rejected": -70.32819366455078,
"logps/rejected": -234.64186096191406,
"loss": 1.1411,
"margin_dpo/margin_mean": 47.259010314941406,
"margin_dpo/margin_std": 70.30636596679688,
"step": 294
},
{
"KL/chosen_KL_mean": -119.32664489746094,
"KL/mean": -141.36685180664062,
"KL/rejected_KL_mean": -163.40704345703125,
"KL/std": 73.06025695800781,
"epoch": 0.4459561602418745,
"fcm_dpo/beta": 0.007442053407430649,
"fcm_dpo/delta": 0.07993526011705399,
"fcm_dpo/margin": 44.08039855957031,
"fcm_dpo/q_t": 0.42684632539749146,
"grad_norm": 16.47260856628418,
"learning_rate": 3.4045544995169125e-07,
"logits/chosen": 0.712509036064148,
"logits/rejected": 0.6072986721992493,
"logps/chosen": -169.7275390625,
"logps/ref_chosen": -50.40088653564453,
"logps/ref_rejected": -83.43521881103516,
"logps/rejected": -246.84225463867188,
"loss": 1.1656,
"margin_dpo/margin_mean": 44.08039855957031,
"margin_dpo/margin_std": 78.5947265625,
"step": 295
},
{
"KL/chosen_KL_mean": -124.35566711425781,
"KL/mean": -153.10382080078125,
"KL/rejected_KL_mean": -181.85195922851562,
"KL/std": 80.76585388183594,
"epoch": 0.4474678760393046,
"fcm_dpo/beta": 0.00737424660474062,
"fcm_dpo/delta": -0.02867615595459938,
"fcm_dpo/margin": 57.496299743652344,
"fcm_dpo/q_t": 0.4039427638053894,
"grad_norm": 12.168935775756836,
"learning_rate": 3.392215553979679e-07,
"logits/chosen": 0.6344171762466431,
"logits/rejected": 0.5863820910453796,
"logps/chosen": -193.50601196289062,
"logps/ref_chosen": -69.15034484863281,
"logps/ref_rejected": -89.60166931152344,
"logps/rejected": -271.45361328125,
"loss": 1.1178,
"margin_dpo/margin_mean": 57.496299743652344,
"margin_dpo/margin_std": 93.3548355102539,
"step": 296
},
{
"KL/chosen_KL_mean": -126.96737670898438,
"KL/mean": -155.42323303222656,
"KL/rejected_KL_mean": -183.87908935546875,
"KL/std": 70.00838470458984,
"epoch": 0.4489795918367347,
"fcm_dpo/beta": 0.0074002379551529884,
"fcm_dpo/delta": -0.024462923407554626,
"fcm_dpo/margin": 56.91170883178711,
"fcm_dpo/q_t": 0.4019927382469177,
"grad_norm": 12.931771278381348,
"learning_rate": 3.3798516512554485e-07,
"logits/chosen": 0.6887728571891785,
"logits/rejected": 0.6372237801551819,
"logps/chosen": -184.98367309570312,
"logps/ref_chosen": -58.01630401611328,
"logps/ref_rejected": -69.95780944824219,
"logps/rejected": -253.8369140625,
"loss": 1.0781,
"margin_dpo/margin_mean": 56.911705017089844,
"margin_dpo/margin_std": 72.6642074584961,
"step": 297
},
{
"KL/chosen_KL_mean": -124.369384765625,
"KL/mean": -145.10752868652344,
"KL/rejected_KL_mean": -165.845703125,
"KL/std": 74.36654663085938,
"epoch": 0.4504913076341648,
"fcm_dpo/beta": 0.007596236653625965,
"fcm_dpo/delta": 0.09406104683876038,
"fcm_dpo/margin": 41.47632598876953,
"fcm_dpo/q_t": 0.43052124977111816,
"grad_norm": 13.027968406677246,
"learning_rate": 3.367463137189156e-07,
"logits/chosen": 0.8152356147766113,
"logits/rejected": 0.7590856552124023,
"logps/chosen": -180.5386962890625,
"logps/ref_chosen": -56.1693115234375,
"logps/ref_rejected": -68.55052185058594,
"logps/rejected": -234.39620971679688,
"loss": 1.1905,
"margin_dpo/margin_mean": 41.47632598876953,
"margin_dpo/margin_std": 82.53489685058594,
"step": 298
},
{
"KL/chosen_KL_mean": -128.62130737304688,
"KL/mean": -147.03543090820312,
"KL/rejected_KL_mean": -165.44956970214844,
"KL/std": 71.73980712890625,
"epoch": 0.4520030234315949,
"fcm_dpo/beta": 0.00786143820732832,
"fcm_dpo/delta": 0.026317991316318512,
"fcm_dpo/margin": 36.82827377319336,
"fcm_dpo/q_t": 0.43662887811660767,
"grad_norm": 21.02098274230957,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": 0.631700873374939,
"logits/rejected": 0.5993541479110718,
"logps/chosen": -190.93911743164062,
"logps/ref_chosen": -62.31780242919922,
"logps/ref_rejected": -72.60028839111328,
"logps/rejected": -238.04986572265625,
"loss": 1.2412,
"margin_dpo/margin_mean": 36.828269958496094,
"margin_dpo/margin_std": 91.29080963134766,
"step": 299
},
{
"KL/chosen_KL_mean": -127.76425170898438,
"KL/mean": -151.45286560058594,
"KL/rejected_KL_mean": -175.1414794921875,
"KL/std": 74.65655517578125,
"epoch": 0.45351473922902497,
"fcm_dpo/beta": 0.007995443418622017,
"fcm_dpo/delta": 0.023457201197743416,
"fcm_dpo/margin": 47.377227783203125,
"fcm_dpo/q_t": 0.4160943329334259,
"grad_norm": 15.658246040344238,
"learning_rate": 3.3426136618426043e-07,
"logits/chosen": 0.7396022081375122,
"logits/rejected": 0.6713778972625732,
"logps/chosen": -188.1458282470703,
"logps/ref_chosen": -60.38157653808594,
"logps/ref_rejected": -75.45442199707031,
"logps/rejected": -250.59591674804688,
"loss": 1.152,
"margin_dpo/margin_mean": 47.377227783203125,
"margin_dpo/margin_std": 84.77381896972656,
"step": 300
},
{
"KL/chosen_KL_mean": -127.62648010253906,
"KL/mean": -148.23974609375,
"KL/rejected_KL_mean": -168.85302734375,
"KL/std": 72.34912109375,
"epoch": 0.455026455026455,
"fcm_dpo/beta": 0.00814204104244709,
"fcm_dpo/delta": 0.07148119807243347,
"fcm_dpo/margin": 41.22654724121094,
"fcm_dpo/q_t": 0.426349401473999,
"grad_norm": 13.319229125976562,
"learning_rate": 3.3301533956555885e-07,
"logits/chosen": 0.7542002201080322,
"logits/rejected": 0.7245285511016846,
"logps/chosen": -180.47738647460938,
"logps/ref_chosen": -52.85089111328125,
"logps/ref_rejected": -69.97584533691406,
"logps/rejected": -238.82888793945312,
"loss": 1.1898,
"margin_dpo/margin_mean": 41.22654724121094,
"margin_dpo/margin_std": 84.22346496582031,
"step": 301
},
{
"KL/chosen_KL_mean": -134.59283447265625,
"KL/mean": -150.45022583007812,
"KL/rejected_KL_mean": -166.30764770507812,
"KL/std": 76.41629791259766,
"epoch": 0.4565381708238851,
"fcm_dpo/beta": 0.00883575901389122,
"fcm_dpo/delta": 0.1305047869682312,
"fcm_dpo/margin": 31.714813232421875,
"fcm_dpo/q_t": 0.4405342936515808,
"grad_norm": 17.557220458984375,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 0.5931792259216309,
"logits/rejected": 0.5361766815185547,
"logps/chosen": -201.55934143066406,
"logps/ref_chosen": -66.96650695800781,
"logps/ref_rejected": -88.09510803222656,
"logps/rejected": -254.4027557373047,
"loss": 1.2561,
"margin_dpo/margin_mean": 31.714813232421875,
"margin_dpo/margin_std": 85.427001953125,
"step": 302
},
{
"KL/chosen_KL_mean": -116.82827758789062,
"KL/mean": -148.2735137939453,
"KL/rejected_KL_mean": -179.71875,
"KL/std": 78.29273223876953,
"epoch": 0.4580498866213152,
"fcm_dpo/beta": 0.00839284434914589,
"fcm_dpo/delta": -0.15248456597328186,
"fcm_dpo/margin": 62.89048767089844,
"fcm_dpo/q_t": 0.38311582803726196,
"grad_norm": 13.024530410766602,
"learning_rate": 3.3051635489464793e-07,
"logits/chosen": 0.6601454019546509,
"logits/rejected": 0.5881760120391846,
"logps/chosen": -178.94979858398438,
"logps/ref_chosen": -62.12152862548828,
"logps/ref_rejected": -90.31204223632812,
"logps/rejected": -270.0307922363281,
"loss": 1.0848,
"margin_dpo/margin_mean": 62.89048767089844,
"margin_dpo/margin_std": 98.73794555664062,
"step": 303
},
{
"KL/chosen_KL_mean": -109.92771911621094,
"KL/mean": -142.41702270507812,
"KL/rejected_KL_mean": -174.90631103515625,
"KL/std": 81.21237182617188,
"epoch": 0.4595616024187453,
"fcm_dpo/beta": 0.007727502845227718,
"fcm_dpo/delta": -0.12161770462989807,
"fcm_dpo/margin": 64.97858428955078,
"fcm_dpo/q_t": 0.38245490193367004,
"grad_norm": 12.52614688873291,
"learning_rate": 3.292634667444117e-07,
"logits/chosen": 0.6955829858779907,
"logits/rejected": 0.6446535587310791,
"logps/chosen": -170.622802734375,
"logps/ref_chosen": -60.695091247558594,
"logps/ref_rejected": -78.2525405883789,
"logps/rejected": -253.1588592529297,
"loss": 1.0246,
"margin_dpo/margin_mean": 64.97859191894531,
"margin_dpo/margin_std": 75.37732696533203,
"step": 304
},
{
"KL/chosen_KL_mean": -125.00523376464844,
"KL/mean": -144.45281982421875,
"KL/rejected_KL_mean": -163.90040588378906,
"KL/std": 72.6624755859375,
"epoch": 0.46107331821617537,
"fcm_dpo/beta": 0.0076552326790988445,
"fcm_dpo/delta": 0.004358620848506689,
"fcm_dpo/margin": 38.89517593383789,
"fcm_dpo/q_t": 0.43163472414016724,
"grad_norm": 17.9072208404541,
"learning_rate": 3.280083614246217e-07,
"logits/chosen": 0.6535234451293945,
"logits/rejected": 0.6887332797050476,
"logps/chosen": -197.70437622070312,
"logps/ref_chosen": -72.69914245605469,
"logps/ref_rejected": -65.65670776367188,
"logps/rejected": -229.55712890625,
"loss": 1.2238,
"margin_dpo/margin_mean": 38.89517593383789,
"margin_dpo/margin_std": 89.01888275146484,
"step": 305
},
{
"KL/chosen_KL_mean": -114.75829315185547,
"KL/mean": -139.0925750732422,
"KL/rejected_KL_mean": -163.42686462402344,
"KL/std": 70.30106353759766,
"epoch": 0.46258503401360546,
"fcm_dpo/beta": 0.007610576692968607,
"fcm_dpo/delta": 0.031120628118515015,
"fcm_dpo/margin": 48.66856384277344,
"fcm_dpo/q_t": 0.4150772988796234,
"grad_norm": 11.311169624328613,
"learning_rate": 3.267510740432719e-07,
"logits/chosen": 0.7671083211898804,
"logits/rejected": 0.663329005241394,
"logps/chosen": -168.72882080078125,
"logps/ref_chosen": -53.97052764892578,
"logps/ref_rejected": -71.02423095703125,
"logps/rejected": -234.4510955810547,
"loss": 1.123,
"margin_dpo/margin_mean": 48.66856384277344,
"margin_dpo/margin_std": 69.87991333007812,
"step": 306
},
{
"KL/chosen_KL_mean": -110.68402099609375,
"KL/mean": -122.84514617919922,
"KL/rejected_KL_mean": -135.00628662109375,
"KL/std": 71.01148986816406,
"epoch": 0.46409674981103555,
"fcm_dpo/beta": 0.007799516897648573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 24.322265625,
"fcm_dpo/q_t": 0.45799070596694946,
"grad_norm": 18.40856170654297,
"learning_rate": 3.2549163976939285e-07,
"logits/chosen": 0.7283176183700562,
"logits/rejected": 0.6819779872894287,
"logps/chosen": -168.09713745117188,
"logps/ref_chosen": -57.413108825683594,
"logps/ref_rejected": -68.68010711669922,
"logps/rejected": -203.68638610839844,
"loss": 1.327,
"margin_dpo/margin_mean": 24.322263717651367,
"margin_dpo/margin_std": 91.626708984375,
"step": 307
},
{
"KL/chosen_KL_mean": -111.70747375488281,
"KL/mean": -135.7213592529297,
"KL/rejected_KL_mean": -159.7352294921875,
"KL/std": 73.46157836914062,
"epoch": 0.4656084656084656,
"fcm_dpo/beta": 0.007773532532155514,
"fcm_dpo/delta": 0.02862391620874405,
"fcm_dpo/margin": 48.02775573730469,
"fcm_dpo/q_t": 0.41647428274154663,
"grad_norm": 11.686368942260742,
"learning_rate": 3.2423009383206874e-07,
"logits/chosen": 0.688902735710144,
"logits/rejected": 0.6754565834999084,
"logps/chosen": -178.3062744140625,
"logps/ref_chosen": -66.59879302978516,
"logps/ref_rejected": -74.337158203125,
"logps/rejected": -234.0723876953125,
"loss": 1.1581,
"margin_dpo/margin_mean": 48.02775573730469,
"margin_dpo/margin_std": 87.01692199707031,
"step": 308
},
{
"KL/chosen_KL_mean": -125.97828674316406,
"KL/mean": -151.03265380859375,
"KL/rejected_KL_mean": -176.0870361328125,
"KL/std": 71.87471771240234,
"epoch": 0.4671201814058957,
"fcm_dpo/beta": 0.007821241393685341,
"fcm_dpo/delta": 0.007066238671541214,
"fcm_dpo/margin": 50.10874557495117,
"fcm_dpo/q_t": 0.4118044972419739,
"grad_norm": 13.115415573120117,
"learning_rate": 3.229664715194511e-07,
"logits/chosen": 0.7336107492446899,
"logits/rejected": 0.6807036399841309,
"logps/chosen": -191.37303161621094,
"logps/ref_chosen": -65.39474487304688,
"logps/ref_rejected": -75.70930480957031,
"logps/rejected": -251.7963409423828,
"loss": 1.1404,
"margin_dpo/margin_mean": 50.108741760253906,
"margin_dpo/margin_std": 84.24324035644531,
"step": 309
},
{
"KL/chosen_KL_mean": -134.229248046875,
"KL/mean": -147.745849609375,
"KL/rejected_KL_mean": -161.26243591308594,
"KL/std": 66.22117614746094,
"epoch": 0.46863189720332576,
"fcm_dpo/beta": 0.008301754482090473,
"fcm_dpo/delta": 0.06831841170787811,
"fcm_dpo/margin": 27.033180236816406,
"fcm_dpo/q_t": 0.4524414539337158,
"grad_norm": 15.16258716583252,
"learning_rate": 3.2170080817777257e-07,
"logits/chosen": 0.7237935066223145,
"logits/rejected": 0.7082376480102539,
"logps/chosen": -208.89752197265625,
"logps/ref_chosen": -74.66827392578125,
"logps/ref_rejected": -80.5689697265625,
"logps/rejected": -241.83140563964844,
"loss": 1.2791,
"margin_dpo/margin_mean": 27.033180236816406,
"margin_dpo/margin_std": 80.27223205566406,
"step": 310
},
{
"KL/chosen_KL_mean": -104.70181274414062,
"KL/mean": -132.61630249023438,
"KL/rejected_KL_mean": -160.53082275390625,
"KL/std": 85.0822982788086,
"epoch": 0.47014361300075586,
"fcm_dpo/beta": 0.007963906042277813,
"fcm_dpo/delta": -0.054659221321344376,
"fcm_dpo/margin": 55.829002380371094,
"fcm_dpo/q_t": 0.40370649099349976,
"grad_norm": 13.94991683959961,
"learning_rate": 3.204331392103574e-07,
"logits/chosen": 0.6307729482650757,
"logits/rejected": 0.4885410666465759,
"logps/chosen": -164.43984985351562,
"logps/ref_chosen": -59.738033294677734,
"logps/ref_rejected": -93.60757446289062,
"logps/rejected": -254.13839721679688,
"loss": 1.1068,
"margin_dpo/margin_mean": 55.829002380371094,
"margin_dpo/margin_std": 91.33370971679688,
"step": 311
},
{
"KL/chosen_KL_mean": -108.8864974975586,
"KL/mean": -138.23374938964844,
"KL/rejected_KL_mean": -167.58099365234375,
"KL/std": 72.34786987304688,
"epoch": 0.47165532879818595,
"fcm_dpo/beta": 0.007836501114070415,
"fcm_dpo/delta": -0.06939233839511871,
"fcm_dpo/margin": 58.694496154785156,
"fcm_dpo/q_t": 0.3929941952228546,
"grad_norm": 11.712054252624512,
"learning_rate": 3.1916350007663176e-07,
"logits/chosen": 0.7151386737823486,
"logits/rejected": 0.6277899742126465,
"logps/chosen": -162.70294189453125,
"logps/ref_chosen": -53.816436767578125,
"logps/ref_rejected": -68.6575698852539,
"logps/rejected": -236.23855590820312,
"loss": 1.0515,
"margin_dpo/margin_mean": 58.694496154785156,
"margin_dpo/margin_std": 71.33744812011719,
"step": 312
},
{
"KL/chosen_KL_mean": -114.5650634765625,
"KL/mean": -130.61492919921875,
"KL/rejected_KL_mean": -146.664794921875,
"KL/std": 71.21220397949219,
"epoch": 0.47316704459561604,
"fcm_dpo/beta": 0.007850449532270432,
"fcm_dpo/delta": 0.06122079864144325,
"fcm_dpo/margin": 32.0997314453125,
"fcm_dpo/q_t": 0.4438740611076355,
"grad_norm": 13.880863189697266,
"learning_rate": 3.178919262911314e-07,
"logits/chosen": 0.7805558443069458,
"logits/rejected": 0.7599529027938843,
"logps/chosen": -174.52243041992188,
"logps/ref_chosen": -59.957359313964844,
"logps/ref_rejected": -69.31729888916016,
"logps/rejected": -215.98208618164062,
"loss": 1.2399,
"margin_dpo/margin_mean": 32.0997314453125,
"margin_dpo/margin_std": 78.52413177490234,
"step": 313
},
{
"KL/chosen_KL_mean": -107.243408203125,
"KL/mean": -136.32138061523438,
"KL/rejected_KL_mean": -165.3993682861328,
"KL/std": 67.65750885009766,
"epoch": 0.47467876039304613,
"fcm_dpo/beta": 0.007725432515144348,
"fcm_dpo/delta": -0.05801774933934212,
"fcm_dpo/margin": 58.155967712402344,
"fcm_dpo/q_t": 0.3972868025302887,
"grad_norm": 12.526936531066895,
"learning_rate": 3.166184534225087e-07,
"logits/chosen": 0.6884793043136597,
"logits/rejected": 0.7177518606185913,
"logps/chosen": -177.51156616210938,
"logps/ref_chosen": -70.26815795898438,
"logps/ref_rejected": -69.23971557617188,
"logps/rejected": -234.6390838623047,
"loss": 1.0679,
"margin_dpo/margin_mean": 58.15597152709961,
"margin_dpo/margin_std": 75.71588897705078,
"step": 314
},
{
"KL/chosen_KL_mean": -113.2779541015625,
"KL/mean": -135.77932739257812,
"KL/rejected_KL_mean": -158.28070068359375,
"KL/std": 69.8665771484375,
"epoch": 0.47619047619047616,
"fcm_dpo/beta": 0.007983740419149399,
"fcm_dpo/delta": 0.04358711838722229,
"fcm_dpo/margin": 45.00275802612305,
"fcm_dpo/q_t": 0.41818535327911377,
"grad_norm": 12.93481731414795,
"learning_rate": 3.1534311709253723e-07,
"logits/chosen": 0.6272811889648438,
"logits/rejected": 0.5963453054428101,
"logps/chosen": -181.07264709472656,
"logps/ref_chosen": -67.79469299316406,
"logps/ref_rejected": -74.55148315429688,
"logps/rejected": -232.83218383789062,
"loss": 1.1472,
"margin_dpo/margin_mean": 45.00275802612305,
"margin_dpo/margin_std": 75.59745025634766,
"step": 315
},
{
"KL/chosen_KL_mean": -105.72105407714844,
"KL/mean": -138.439697265625,
"KL/rejected_KL_mean": -171.1583251953125,
"KL/std": 74.41236877441406,
"epoch": 0.47770219198790626,
"fcm_dpo/beta": 0.007744115777313709,
"fcm_dpo/delta": -0.12915290892124176,
"fcm_dpo/margin": 65.43728637695312,
"fcm_dpo/q_t": 0.38332080841064453,
"grad_norm": 13.909781455993652,
"learning_rate": 3.1406595297511564e-07,
"logits/chosen": 0.5902273058891296,
"logits/rejected": 0.45409518480300903,
"logps/chosen": -161.00953674316406,
"logps/ref_chosen": -55.288482666015625,
"logps/ref_rejected": -96.15723419189453,
"logps/rejected": -267.3155517578125,
"loss": 1.0384,
"margin_dpo/margin_mean": 65.43728637695312,
"margin_dpo/margin_std": 76.73016357421875,
"step": 316
},
{
"KL/chosen_KL_mean": -95.98619079589844,
"KL/mean": -126.41956329345703,
"KL/rejected_KL_mean": -156.85293579101562,
"KL/std": 66.47119140625,
"epoch": 0.47921390778533635,
"fcm_dpo/beta": 0.007124145515263081,
"fcm_dpo/delta": -0.04006510600447655,
"fcm_dpo/margin": 60.866737365722656,
"fcm_dpo/q_t": 0.3974044919013977,
"grad_norm": 13.468562126159668,
"learning_rate": 3.1278699679526975e-07,
"logits/chosen": 0.715498685836792,
"logits/rejected": 0.6702585816383362,
"logps/chosen": -150.56756591796875,
"logps/ref_chosen": -54.58137512207031,
"logps/ref_rejected": -72.77232360839844,
"logps/rejected": -229.62525939941406,
"loss": 1.0662,
"margin_dpo/margin_mean": 60.866737365722656,
"margin_dpo/margin_std": 74.26116180419922,
"step": 317
},
{
"KL/chosen_KL_mean": -106.12240600585938,
"KL/mean": -128.9796905517578,
"KL/rejected_KL_mean": -151.8369598388672,
"KL/std": 72.66458129882812,
"epoch": 0.48072562358276644,
"fcm_dpo/beta": 0.007349280174821615,
"fcm_dpo/delta": 0.07138577103614807,
"fcm_dpo/margin": 45.71455764770508,
"fcm_dpo/q_t": 0.42840999364852905,
"grad_norm": 13.374271392822266,
"learning_rate": 3.1150628432815336e-07,
"logits/chosen": 0.6954464912414551,
"logits/rejected": 0.6276102662086487,
"logps/chosen": -159.01065063476562,
"logps/ref_chosen": -52.88822937011719,
"logps/ref_rejected": -80.63988494873047,
"logps/rejected": -232.4768524169922,
"loss": 1.2033,
"margin_dpo/margin_mean": 45.71455764770508,
"margin_dpo/margin_std": 100.32778930664062,
"step": 318
},
{
"KL/chosen_KL_mean": -105.37348937988281,
"KL/mean": -137.54539489746094,
"KL/rejected_KL_mean": -169.71728515625,
"KL/std": 77.01676177978516,
"epoch": 0.48223733938019653,
"fcm_dpo/beta": 0.007210618816316128,
"fcm_dpo/delta": -0.07460720092058182,
"fcm_dpo/margin": 64.34378814697266,
"fcm_dpo/q_t": 0.3947829604148865,
"grad_norm": 10.253146171569824,
"learning_rate": 3.1022385139804707e-07,
"logits/chosen": 0.6536087989807129,
"logits/rejected": 0.6382144689559937,
"logps/chosen": -169.73681640625,
"logps/ref_chosen": -64.36333465576172,
"logps/ref_rejected": -79.47296142578125,
"logps/rejected": -249.19024658203125,
"loss": 1.0683,
"margin_dpo/margin_mean": 64.34378814697266,
"margin_dpo/margin_std": 89.17861938476562,
"step": 319
},
{
"KL/chosen_KL_mean": -96.51083374023438,
"KL/mean": -121.93170166015625,
"KL/rejected_KL_mean": -147.35256958007812,
"KL/std": 78.21331787109375,
"epoch": 0.4837490551776266,
"fcm_dpo/beta": 0.006960249040275812,
"fcm_dpo/delta": -0.08929912000894547,
"fcm_dpo/margin": 50.84172821044922,
"fcm_dpo/q_t": 0.41984930634498596,
"grad_norm": 11.877212524414062,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 0.6033248901367188,
"logits/rejected": 0.5647040605545044,
"logps/chosen": -146.069580078125,
"logps/ref_chosen": -49.558746337890625,
"logps/ref_rejected": -71.23444366455078,
"logps/rejected": -218.58700561523438,
"loss": 1.1531,
"margin_dpo/margin_mean": 50.84172439575195,
"margin_dpo/margin_std": 83.79927062988281,
"step": 320
},
{
"KL/chosen_KL_mean": -100.57034301757812,
"KL/mean": -128.36460876464844,
"KL/rejected_KL_mean": -156.15890502929688,
"KL/std": 66.17784881591797,
"epoch": 0.4852607709750567,
"fcm_dpo/beta": 0.006733224727213383,
"fcm_dpo/delta": 0.02710144966840744,
"fcm_dpo/margin": 55.58854675292969,
"fcm_dpo/q_t": 0.41282814741134644,
"grad_norm": 15.027644157409668,
"learning_rate": 3.0765396768561004e-07,
"logits/chosen": 0.6975396275520325,
"logits/rejected": 0.6782538890838623,
"logps/chosen": -152.65560913085938,
"logps/ref_chosen": -52.08526611328125,
"logps/ref_rejected": -55.58674621582031,
"logps/rejected": -211.74563598632812,
"loss": 1.1195,
"margin_dpo/margin_mean": 55.58854675292969,
"margin_dpo/margin_std": 78.52020263671875,
"step": 321
},
{
"KL/chosen_KL_mean": -112.34437561035156,
"KL/mean": -147.16134643554688,
"KL/rejected_KL_mean": -181.97833251953125,
"KL/std": 75.60289001464844,
"epoch": 0.48677248677248675,
"fcm_dpo/beta": 0.006711442023515701,
"fcm_dpo/delta": -0.07818324863910675,
"fcm_dpo/margin": 69.63396453857422,
"fcm_dpo/q_t": 0.3887701630592346,
"grad_norm": 10.670733451843262,
"learning_rate": 3.063665887884511e-07,
"logits/chosen": 0.7195281982421875,
"logits/rejected": 0.6374897956848145,
"logps/chosen": -159.74847412109375,
"logps/ref_chosen": -47.404109954833984,
"logps/ref_rejected": -73.4260025024414,
"logps/rejected": -255.40432739257812,
"loss": 1.031,
"margin_dpo/margin_mean": 69.63395690917969,
"margin_dpo/margin_std": 72.7001953125,
"step": 322
},
{
"KL/chosen_KL_mean": -116.77687072753906,
"KL/mean": -138.69090270996094,
"KL/rejected_KL_mean": -160.60494995117188,
"KL/std": 72.23223876953125,
"epoch": 0.48828420256991684,
"fcm_dpo/beta": 0.006896693259477615,
"fcm_dpo/delta": 0.10736851394176483,
"fcm_dpo/margin": 43.82807159423828,
"fcm_dpo/q_t": 0.4345787763595581,
"grad_norm": 12.62841510772705,
"learning_rate": 3.0507763319663517e-07,
"logits/chosen": 0.6039708852767944,
"logits/rejected": 0.5336369276046753,
"logps/chosen": -186.78317260742188,
"logps/ref_chosen": -70.00630187988281,
"logps/ref_rejected": -86.96690368652344,
"logps/rejected": -247.57183837890625,
"loss": 1.2031,
"margin_dpo/margin_mean": 43.82807159423828,
"margin_dpo/margin_std": 92.93321228027344,
"step": 323
},
{
"KL/chosen_KL_mean": -102.77244567871094,
"KL/mean": -132.74996948242188,
"KL/rejected_KL_mean": -162.7274932861328,
"KL/std": 84.26118469238281,
"epoch": 0.4897959183673469,
"fcm_dpo/beta": 0.006869921460747719,
"fcm_dpo/delta": -0.014991983771324158,
"fcm_dpo/margin": 59.95502853393555,
"fcm_dpo/q_t": 0.4040718674659729,
"grad_norm": 13.32012939453125,
"learning_rate": 3.0378713696502097e-07,
"logits/chosen": 0.7294822931289673,
"logits/rejected": 0.6708847284317017,
"logps/chosen": -158.66127014160156,
"logps/ref_chosen": -55.88882064819336,
"logps/ref_rejected": -75.23088073730469,
"logps/rejected": -237.9583740234375,
"loss": 1.0831,
"margin_dpo/margin_mean": 59.95503234863281,
"margin_dpo/margin_std": 75.76171875,
"step": 324
},
{
"KL/chosen_KL_mean": -124.10344696044922,
"KL/mean": -154.2080078125,
"KL/rejected_KL_mean": -184.31256103515625,
"KL/std": 82.18153381347656,
"epoch": 0.491307634164777,
"fcm_dpo/beta": 0.006824733689427376,
"fcm_dpo/delta": -0.013523155823349953,
"fcm_dpo/margin": 60.20912170410156,
"fcm_dpo/q_t": 0.4077095687389374,
"grad_norm": 11.118515014648438,
"learning_rate": 3.0249513619156206e-07,
"logits/chosen": 0.6534192562103271,
"logits/rejected": 0.5929680466651917,
"logps/chosen": -188.25045776367188,
"logps/ref_chosen": -64.14701843261719,
"logps/ref_rejected": -79.91143798828125,
"logps/rejected": -264.2239990234375,
"loss": 1.1226,
"margin_dpo/margin_mean": 60.20912170410156,
"margin_dpo/margin_std": 100.43673706054688,
"step": 325
},
{
"KL/chosen_KL_mean": -145.35369873046875,
"KL/mean": -156.6713409423828,
"KL/rejected_KL_mean": -167.98898315429688,
"KL/std": 74.04638671875,
"epoch": 0.4928193499622071,
"fcm_dpo/beta": 0.0068876477889716625,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 22.635299682617188,
"fcm_dpo/q_t": 0.46425288915634155,
"grad_norm": 12.687813758850098,
"learning_rate": 3.012016670162977e-07,
"logits/chosen": 0.6178878545761108,
"logits/rejected": 0.6271042823791504,
"logps/chosen": -220.885009765625,
"logps/ref_chosen": -75.53131103515625,
"logps/ref_rejected": -76.5898666381836,
"logps/rejected": -244.578857421875,
"loss": 1.3161,
"margin_dpo/margin_mean": 22.635299682617188,
"margin_dpo/margin_std": 83.47059631347656,
"step": 326
},
{
"KL/chosen_KL_mean": -128.65380859375,
"KL/mean": -151.80908203125,
"KL/rejected_KL_mean": -174.96434020996094,
"KL/std": 78.66702270507812,
"epoch": 0.4943310657596372,
"fcm_dpo/beta": 0.007172505371272564,
"fcm_dpo/delta": 0.07436676323413849,
"fcm_dpo/margin": 46.310516357421875,
"fcm_dpo/q_t": 0.425973117351532,
"grad_norm": 12.866050720214844,
"learning_rate": 2.99906765620341e-07,
"logits/chosen": 0.5928927063941956,
"logits/rejected": 0.5698734521865845,
"logps/chosen": -197.990966796875,
"logps/ref_chosen": -69.33717346191406,
"logps/ref_rejected": -73.37751770019531,
"logps/rejected": -248.34185791015625,
"loss": 1.1804,
"margin_dpo/margin_mean": 46.310516357421875,
"margin_dpo/margin_std": 90.25209045410156,
"step": 327
},
{
"KL/chosen_KL_mean": -113.55990600585938,
"KL/mean": -142.6298828125,
"KL/rejected_KL_mean": -171.69985961914062,
"KL/std": 80.41529846191406,
"epoch": 0.4958427815570673,
"fcm_dpo/beta": 0.007189217023551464,
"fcm_dpo/delta": -0.02078922465443611,
"fcm_dpo/margin": 58.13993453979492,
"fcm_dpo/q_t": 0.4041920304298401,
"grad_norm": 13.29574966430664,
"learning_rate": 2.9861046822486766e-07,
"logits/chosen": 0.5948680639266968,
"logits/rejected": 0.5532065033912659,
"logps/chosen": -175.26614379882812,
"logps/ref_chosen": -61.70623016357422,
"logps/ref_rejected": -83.73808288574219,
"logps/rejected": -255.43792724609375,
"loss": 1.0833,
"margin_dpo/margin_mean": 58.13993835449219,
"margin_dpo/margin_std": 76.74394226074219,
"step": 328
},
{
"KL/chosen_KL_mean": -127.44103240966797,
"KL/mean": -156.460205078125,
"KL/rejected_KL_mean": -185.47938537597656,
"KL/std": 85.77304077148438,
"epoch": 0.4973544973544973,
"fcm_dpo/beta": 0.007048935629427433,
"fcm_dpo/delta": -0.010564148426055908,
"fcm_dpo/margin": 58.038368225097656,
"fcm_dpo/q_t": 0.40908321738243103,
"grad_norm": 14.72163200378418,
"learning_rate": 2.9731281109010253e-07,
"logits/chosen": 0.7163276672363281,
"logits/rejected": 0.6566041111946106,
"logps/chosen": -191.939453125,
"logps/ref_chosen": -64.4984130859375,
"logps/ref_rejected": -83.6591796875,
"logps/rejected": -269.1385498046875,
"loss": 1.1121,
"margin_dpo/margin_mean": 58.038368225097656,
"margin_dpo/margin_std": 90.64898681640625,
"step": 329
},
{
"KL/chosen_KL_mean": -111.15478515625,
"KL/mean": -139.06466674804688,
"KL/rejected_KL_mean": -166.97454833984375,
"KL/std": 77.32144165039062,
"epoch": 0.4988662131519274,
"fcm_dpo/beta": 0.0071370261721313,
"fcm_dpo/delta": 0.001220574602484703,
"fcm_dpo/margin": 55.819766998291016,
"fcm_dpo/q_t": 0.41049909591674805,
"grad_norm": 12.301667213439941,
"learning_rate": 2.9601383051430505e-07,
"logits/chosen": 0.6728509664535522,
"logits/rejected": 0.6075490713119507,
"logps/chosen": -165.9594268798828,
"logps/ref_chosen": -54.80464172363281,
"logps/ref_rejected": -75.3194351196289,
"logps/rejected": -242.2939910888672,
"loss": 1.1461,
"margin_dpo/margin_mean": 55.81976318359375,
"margin_dpo/margin_std": 99.13479614257812,
"step": 330
},
{
"KL/chosen_KL_mean": -118.90031433105469,
"KL/mean": -155.21522521972656,
"KL/rejected_KL_mean": -191.53013610839844,
"KL/std": 81.10183715820312,
"epoch": 0.5003779289493575,
"fcm_dpo/beta": 0.006745649501681328,
"fcm_dpo/delta": -0.105572909116745,
"fcm_dpo/margin": 72.62981414794922,
"fcm_dpo/q_t": 0.38496410846710205,
"grad_norm": 12.032770156860352,
"learning_rate": 2.947135628327544e-07,
"logits/chosen": 0.7782831192016602,
"logits/rejected": 0.7540770769119263,
"logps/chosen": -178.14288330078125,
"logps/ref_chosen": -59.242584228515625,
"logps/ref_rejected": -69.87483215332031,
"logps/rejected": -261.40496826171875,
"loss": 1.0371,
"margin_dpo/margin_mean": 72.62981414794922,
"margin_dpo/margin_std": 88.2421875,
"step": 331
},
{
"KL/chosen_KL_mean": -124.0113754272461,
"KL/mean": -153.82049560546875,
"KL/rejected_KL_mean": -183.62960815429688,
"KL/std": 78.55375671386719,
"epoch": 0.5018896447467877,
"fcm_dpo/beta": 0.006745035294443369,
"fcm_dpo/delta": -0.00457579642534256,
"fcm_dpo/margin": 59.618255615234375,
"fcm_dpo/q_t": 0.4085049033164978,
"grad_norm": 11.804621696472168,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": 0.6241295337677002,
"logits/rejected": 0.5831087827682495,
"logps/chosen": -191.12112426757812,
"logps/ref_chosen": -67.10975646972656,
"logps/ref_rejected": -77.11839294433594,
"logps/rejected": -260.74798583984375,
"loss": 1.1011,
"margin_dpo/margin_mean": 59.618255615234375,
"margin_dpo/margin_std": 83.41954803466797,
"step": 332
},
{
"KL/chosen_KL_mean": -129.91983032226562,
"KL/mean": -164.92306518554688,
"KL/rejected_KL_mean": -199.92630004882812,
"KL/std": 81.36688232421875,
"epoch": 0.5034013605442177,
"fcm_dpo/beta": 0.0064663151279091835,
"fcm_dpo/delta": -0.060843661427497864,
"fcm_dpo/margin": 70.00645446777344,
"fcm_dpo/q_t": 0.39520421624183655,
"grad_norm": 10.76503849029541,
"learning_rate": 2.921093116725076e-07,
"logits/chosen": 0.6815329790115356,
"logits/rejected": 0.604261040687561,
"logps/chosen": -188.30096435546875,
"logps/ref_chosen": -58.381134033203125,
"logps/ref_rejected": -85.02839660644531,
"logps/rejected": -284.9547119140625,
"loss": 1.0549,
"margin_dpo/margin_mean": 70.00645446777344,
"margin_dpo/margin_std": 85.70564270019531,
"step": 333
},
{
"KL/chosen_KL_mean": -125.50909423828125,
"KL/mean": -150.45120239257812,
"KL/rejected_KL_mean": -175.393310546875,
"KL/std": 80.46173858642578,
"epoch": 0.5049130763416477,
"fcm_dpo/beta": 0.006630011834204197,
"fcm_dpo/delta": 0.07646586000919342,
"fcm_dpo/margin": 49.88423538208008,
"fcm_dpo/q_t": 0.42625001072883606,
"grad_norm": 12.834829330444336,
"learning_rate": 2.9080540104031484e-07,
"logits/chosen": 0.685641348361969,
"logits/rejected": 0.6379861831665039,
"logps/chosen": -192.40109252929688,
"logps/ref_chosen": -66.89199829101562,
"logps/ref_rejected": -91.83695220947266,
"logps/rejected": -267.23028564453125,
"loss": 1.1679,
"margin_dpo/margin_mean": 49.88423156738281,
"margin_dpo/margin_std": 90.79962158203125,
"step": 334
},
{
"KL/chosen_KL_mean": -126.83473205566406,
"KL/mean": -153.43295288085938,
"KL/rejected_KL_mean": -180.03115844726562,
"KL/std": 80.2386474609375,
"epoch": 0.5064247921390779,
"fcm_dpo/beta": 0.00688770180568099,
"fcm_dpo/delta": 0.03577427193522453,
"fcm_dpo/margin": 53.196434020996094,
"fcm_dpo/q_t": 0.41868066787719727,
"grad_norm": 14.178523063659668,
"learning_rate": 2.895003489933375e-07,
"logits/chosen": 0.7071677446365356,
"logits/rejected": 0.6692987084388733,
"logps/chosen": -188.34918212890625,
"logps/ref_chosen": -61.51445770263672,
"logps/ref_rejected": -75.68916320800781,
"logps/rejected": -255.7203369140625,
"loss": 1.1555,
"margin_dpo/margin_mean": 53.196434020996094,
"margin_dpo/margin_std": 95.25994110107422,
"step": 335
},
{
"KL/chosen_KL_mean": -140.4132537841797,
"KL/mean": -169.2587890625,
"KL/rejected_KL_mean": -198.10433959960938,
"KL/std": 85.74678039550781,
"epoch": 0.5079365079365079,
"fcm_dpo/beta": 0.006724413949996233,
"fcm_dpo/delta": 0.010933250188827515,
"fcm_dpo/margin": 57.69108200073242,
"fcm_dpo/q_t": 0.4159916043281555,
"grad_norm": 13.194255828857422,
"learning_rate": 2.8819419203668675e-07,
"logits/chosen": 0.6335151195526123,
"logits/rejected": 0.6091711521148682,
"logps/chosen": -209.26332092285156,
"logps/ref_chosen": -68.85006713867188,
"logps/ref_rejected": -92.99603271484375,
"logps/rejected": -291.100341796875,
"loss": 1.1379,
"margin_dpo/margin_mean": 57.69108581542969,
"margin_dpo/margin_std": 97.63800811767578,
"step": 336
},
{
"KL/chosen_KL_mean": -148.26010131835938,
"KL/mean": -171.10504150390625,
"KL/rejected_KL_mean": -193.94998168945312,
"KL/std": 85.65023803710938,
"epoch": 0.509448223733938,
"fcm_dpo/beta": 0.007140236906707287,
"fcm_dpo/delta": 0.08157092332839966,
"fcm_dpo/margin": 45.68986511230469,
"fcm_dpo/q_t": 0.4279956817626953,
"grad_norm": 11.6563138961792,
"learning_rate": 2.8688696670638053e-07,
"logits/chosen": 0.5941582918167114,
"logits/rejected": 0.5655093193054199,
"logps/chosen": -221.44793701171875,
"logps/ref_chosen": -73.18783569335938,
"logps/ref_rejected": -86.89118957519531,
"logps/rejected": -280.8411865234375,
"loss": 1.1877,
"margin_dpo/margin_mean": 45.68986511230469,
"margin_dpo/margin_std": 92.1810302734375,
"step": 337
},
{
"KL/chosen_KL_mean": -145.2806854248047,
"KL/mean": -168.95672607421875,
"KL/rejected_KL_mean": -192.6327667236328,
"KL/std": 78.41549682617188,
"epoch": 0.5109599395313681,
"fcm_dpo/beta": 0.007375883869826794,
"fcm_dpo/delta": 0.05678681656718254,
"fcm_dpo/margin": 47.35206604003906,
"fcm_dpo/q_t": 0.4219357669353485,
"grad_norm": 11.649815559387207,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 0.6714822053909302,
"logits/rejected": 0.6458303928375244,
"logps/chosen": -209.22030639648438,
"logps/ref_chosen": -63.939613342285156,
"logps/ref_rejected": -75.34243774414062,
"logps/rejected": -267.9752197265625,
"loss": 1.1663,
"margin_dpo/margin_mean": 47.35206604003906,
"margin_dpo/margin_std": 87.88858032226562,
"step": 338
},
{
"KL/chosen_KL_mean": -118.4639892578125,
"KL/mean": -147.70608520507812,
"KL/rejected_KL_mean": -176.9481964111328,
"KL/std": 79.511962890625,
"epoch": 0.5124716553287982,
"fcm_dpo/beta": 0.007303786464035511,
"fcm_dpo/delta": -0.031887464225292206,
"fcm_dpo/margin": 58.484214782714844,
"fcm_dpo/q_t": 0.4025455117225647,
"grad_norm": 14.439207077026367,
"learning_rate": 2.842694572172736e-07,
"logits/chosen": 0.8139640688896179,
"logits/rejected": 0.7292660474777222,
"logps/chosen": -164.01312255859375,
"logps/ref_chosen": -45.54913330078125,
"logps/ref_rejected": -67.0482177734375,
"logps/rejected": -243.9964141845703,
"loss": 1.0965,
"margin_dpo/margin_mean": 58.484214782714844,
"margin_dpo/margin_std": 85.29948425292969,
"step": 339
},
{
"KL/chosen_KL_mean": -133.04364013671875,
"KL/mean": -160.40573120117188,
"KL/rejected_KL_mean": -187.767822265625,
"KL/std": 85.34622192382812,
"epoch": 0.5139833711262283,
"fcm_dpo/beta": 0.007357806898653507,
"fcm_dpo/delta": -0.0032272376120090485,
"fcm_dpo/margin": 54.72416687011719,
"fcm_dpo/q_t": 0.41151127219200134,
"grad_norm": 12.648419380187988,
"learning_rate": 2.8295924627584004e-07,
"logits/chosen": 0.6942380666732788,
"logits/rejected": 0.6745313405990601,
"logps/chosen": -187.04928588867188,
"logps/ref_chosen": -54.00564956665039,
"logps/ref_rejected": -61.314430236816406,
"logps/rejected": -249.08224487304688,
"loss": 1.152,
"margin_dpo/margin_mean": 54.72416687011719,
"margin_dpo/margin_std": 100.8072738647461,
"step": 340
},
{
"KL/chosen_KL_mean": -135.74366760253906,
"KL/mean": -167.9922637939453,
"KL/rejected_KL_mean": -200.2408447265625,
"KL/std": 85.87152099609375,
"epoch": 0.5154950869236583,
"fcm_dpo/beta": 0.006678248755633831,
"fcm_dpo/delta": -0.15111427009105682,
"fcm_dpo/margin": 64.4971923828125,
"fcm_dpo/q_t": 0.39761853218078613,
"grad_norm": 13.96065902709961,
"learning_rate": 2.816481133934373e-07,
"logits/chosen": 0.7185596823692322,
"logits/rejected": 0.6717268824577332,
"logps/chosen": -199.13876342773438,
"logps/ref_chosen": -63.39509582519531,
"logps/ref_rejected": -76.20973205566406,
"logps/rejected": -276.4505920410156,
"loss": 1.0987,
"margin_dpo/margin_mean": 64.4971923828125,
"margin_dpo/margin_std": 91.53335571289062,
"step": 341
},
{
"KL/chosen_KL_mean": -133.71408081054688,
"KL/mean": -164.31256103515625,
"KL/rejected_KL_mean": -194.91104125976562,
"KL/std": 91.55609130859375,
"epoch": 0.5170068027210885,
"fcm_dpo/beta": 0.006609394680708647,
"fcm_dpo/delta": -0.00558522529900074,
"fcm_dpo/margin": 61.19696807861328,
"fcm_dpo/q_t": 0.4092737138271332,
"grad_norm": 10.6194486618042,
"learning_rate": 2.8033609524527046e-07,
"logits/chosen": 0.7518157362937927,
"logits/rejected": 0.7110329270362854,
"logps/chosen": -186.76190185546875,
"logps/ref_chosen": -53.047813415527344,
"logps/ref_rejected": -68.2854232788086,
"logps/rejected": -263.19647216796875,
"loss": 1.1123,
"margin_dpo/margin_mean": 61.19697189331055,
"margin_dpo/margin_std": 93.84374237060547,
"step": 342
},
{
"KL/chosen_KL_mean": -129.63284301757812,
"KL/mean": -152.19546508789062,
"KL/rejected_KL_mean": -174.75808715820312,
"KL/std": 84.81193542480469,
"epoch": 0.5185185185185185,
"fcm_dpo/beta": 0.006589554250240326,
"fcm_dpo/delta": -0.016698164865374565,
"fcm_dpo/margin": 45.125267028808594,
"fcm_dpo/q_t": 0.43187424540519714,
"grad_norm": 10.550444602966309,
"learning_rate": 2.7902322853130753e-07,
"logits/chosen": 0.5749995708465576,
"logits/rejected": 0.5654809474945068,
"logps/chosen": -200.21136474609375,
"logps/ref_chosen": -70.57852935791016,
"logps/ref_rejected": -84.73873901367188,
"logps/rejected": -259.496826171875,
"loss": 1.194,
"margin_dpo/margin_mean": 45.125267028808594,
"margin_dpo/margin_std": 88.6053466796875,
"step": 343
},
{
"KL/chosen_KL_mean": -140.68621826171875,
"KL/mean": -173.6033172607422,
"KL/rejected_KL_mean": -206.52041625976562,
"KL/std": 85.86953735351562,
"epoch": 0.5200302343159486,
"fcm_dpo/beta": 0.006495362147688866,
"fcm_dpo/delta": -0.031697697937488556,
"fcm_dpo/margin": 65.8342056274414,
"fcm_dpo/q_t": 0.403338760137558,
"grad_norm": 12.14207935333252,
"learning_rate": 2.7770954997525274e-07,
"logits/chosen": 0.7135556936264038,
"logits/rejected": 0.6433905959129333,
"logps/chosen": -196.49722290039062,
"logps/ref_chosen": -55.811004638671875,
"logps/ref_rejected": -84.77637481689453,
"logps/rejected": -291.29681396484375,
"loss": 1.1007,
"margin_dpo/margin_mean": 65.8342056274414,
"margin_dpo/margin_std": 99.62645721435547,
"step": 344
},
{
"KL/chosen_KL_mean": -116.20820617675781,
"KL/mean": -145.49966430664062,
"KL/rejected_KL_mean": -174.7911376953125,
"KL/std": 78.99835205078125,
"epoch": 0.5215419501133787,
"fcm_dpo/beta": 0.006551923230290413,
"fcm_dpo/delta": 0.01797226071357727,
"fcm_dpo/margin": 58.582942962646484,
"fcm_dpo/q_t": 0.41312161087989807,
"grad_norm": 11.81797981262207,
"learning_rate": 2.7639509632351927e-07,
"logits/chosen": 0.7470645308494568,
"logits/rejected": 0.7046425342559814,
"logps/chosen": -173.99429321289062,
"logps/ref_chosen": -57.78609848022461,
"logps/ref_rejected": -78.91847229003906,
"logps/rejected": -253.70960998535156,
"loss": 1.1272,
"margin_dpo/margin_mean": 58.58293914794922,
"margin_dpo/margin_std": 93.53245544433594,
"step": 345
},
{
"KL/chosen_KL_mean": -122.06107330322266,
"KL/mean": -158.17715454101562,
"KL/rejected_KL_mean": -194.29324340820312,
"KL/std": 93.81214904785156,
"epoch": 0.5230536659108088,
"fcm_dpo/beta": 0.0063993064686656,
"fcm_dpo/delta": -0.0722820833325386,
"fcm_dpo/margin": 72.23216247558594,
"fcm_dpo/q_t": 0.3927100896835327,
"grad_norm": 13.295743942260742,
"learning_rate": 2.7507990434420123e-07,
"logits/chosen": 0.7267513275146484,
"logits/rejected": 0.6444227695465088,
"logps/chosen": -178.34619140625,
"logps/ref_chosen": -56.285125732421875,
"logps/ref_rejected": -91.15303039550781,
"logps/rejected": -285.4462890625,
"loss": 1.0698,
"margin_dpo/margin_mean": 72.23216247558594,
"margin_dpo/margin_std": 97.0244140625,
"step": 346
},
{
"KL/chosen_KL_mean": -135.72100830078125,
"KL/mean": -162.5164794921875,
"KL/rejected_KL_mean": -189.3119354248047,
"KL/std": 86.25690460205078,
"epoch": 0.5245653817082389,
"fcm_dpo/beta": 0.006409412249922752,
"fcm_dpo/delta": 0.06314625591039658,
"fcm_dpo/margin": 53.5909309387207,
"fcm_dpo/q_t": 0.4225504994392395,
"grad_norm": 12.886883735656738,
"learning_rate": 2.737640108260456e-07,
"logits/chosen": 0.8315824270248413,
"logits/rejected": 0.784061074256897,
"logps/chosen": -189.22055053710938,
"logps/ref_chosen": -53.499542236328125,
"logps/ref_rejected": -72.52565002441406,
"logps/rejected": -261.83758544921875,
"loss": 1.1462,
"margin_dpo/margin_mean": 53.5909309387207,
"margin_dpo/margin_std": 87.14447784423828,
"step": 347
},
{
"KL/chosen_KL_mean": -122.0595932006836,
"KL/mean": -153.3162841796875,
"KL/rejected_KL_mean": -184.57296752929688,
"KL/std": 81.57878112792969,
"epoch": 0.5260770975056689,
"fcm_dpo/beta": 0.006419610697776079,
"fcm_dpo/delta": -0.002443864941596985,
"fcm_dpo/margin": 62.51338195800781,
"fcm_dpo/q_t": 0.41006332635879517,
"grad_norm": 12.771003723144531,
"learning_rate": 2.724474525774229e-07,
"logits/chosen": 0.803961992263794,
"logits/rejected": 0.7731932401657104,
"logps/chosen": -172.846435546875,
"logps/ref_chosen": -50.78684997558594,
"logps/ref_rejected": -68.63732147216797,
"logps/rejected": -253.2102813720703,
"loss": 1.114,
"margin_dpo/margin_mean": 62.51338195800781,
"margin_dpo/margin_std": 96.11529541015625,
"step": 348
},
{
"KL/chosen_KL_mean": -124.03947448730469,
"KL/mean": -155.91339111328125,
"KL/rejected_KL_mean": -187.78729248046875,
"KL/std": 91.26483154296875,
"epoch": 0.527588813303099,
"fcm_dpo/beta": 0.006526275537908077,
"fcm_dpo/delta": -0.019439280033111572,
"fcm_dpo/margin": 63.74781036376953,
"fcm_dpo/q_t": 0.4069031774997711,
"grad_norm": 11.912677764892578,
"learning_rate": 2.711302664252973e-07,
"logits/chosen": 0.7378636598587036,
"logits/rejected": 0.6440693736076355,
"logps/chosen": -177.36448669433594,
"logps/ref_chosen": -53.325008392333984,
"logps/ref_rejected": -83.21236419677734,
"logps/rejected": -270.9996643066406,
"loss": 1.1076,
"margin_dpo/margin_mean": 63.7478141784668,
"margin_dpo/margin_std": 96.80409240722656,
"step": 349
},
{
"KL/chosen_KL_mean": -126.06622314453125,
"KL/mean": -168.70594787597656,
"KL/rejected_KL_mean": -211.34567260742188,
"KL/std": 97.71580505371094,
"epoch": 0.5291005291005291,
"fcm_dpo/beta": 0.006007745862007141,
"fcm_dpo/delta": -0.13404223322868347,
"fcm_dpo/margin": 85.27944946289062,
"fcm_dpo/q_t": 0.3789331316947937,
"grad_norm": 13.077025413513184,
"learning_rate": 2.698124892141971e-07,
"logits/chosen": 0.7190489768981934,
"logits/rejected": 0.6495992541313171,
"logps/chosen": -187.69198608398438,
"logps/ref_chosen": -61.625770568847656,
"logps/ref_rejected": -87.63627624511719,
"logps/rejected": -298.98193359375,
"loss": 1.0091,
"margin_dpo/margin_mean": 85.27944946289062,
"margin_dpo/margin_std": 92.19660186767578,
"step": 350
},
{
"KL/chosen_KL_mean": -122.51304626464844,
"KL/mean": -152.106201171875,
"KL/rejected_KL_mean": -181.69935607910156,
"KL/std": 81.26626586914062,
"epoch": 0.5306122448979592,
"fcm_dpo/beta": 0.006009809672832489,
"fcm_dpo/delta": 0.0496024414896965,
"fcm_dpo/margin": 59.186309814453125,
"fcm_dpo/q_t": 0.4185904562473297,
"grad_norm": 11.393095970153809,
"learning_rate": 2.6849415780518357e-07,
"logits/chosen": 0.6613746881484985,
"logits/rejected": 0.5912742018699646,
"logps/chosen": -178.76937866210938,
"logps/ref_chosen": -56.2563362121582,
"logps/ref_rejected": -79.11589813232422,
"logps/rejected": -260.81524658203125,
"loss": 1.1556,
"margin_dpo/margin_mean": 59.186309814453125,
"margin_dpo/margin_std": 103.63017272949219,
"step": 351
},
{
"KL/chosen_KL_mean": -124.99972534179688,
"KL/mean": -158.94674682617188,
"KL/rejected_KL_mean": -192.89376831054688,
"KL/std": 92.21725463867188,
"epoch": 0.5321239606953893,
"fcm_dpo/beta": 0.006002703681588173,
"fcm_dpo/delta": -0.009663868695497513,
"fcm_dpo/margin": 67.89405822753906,
"fcm_dpo/q_t": 0.40806788206100464,
"grad_norm": 11.126773834228516,
"learning_rate": 2.6717530907482024e-07,
"logits/chosen": 0.7351091504096985,
"logits/rejected": 0.6848005056381226,
"logps/chosen": -188.05166625976562,
"logps/ref_chosen": -63.05195236206055,
"logps/ref_rejected": -85.52035522460938,
"logps/rejected": -278.41412353515625,
"loss": 1.1028,
"margin_dpo/margin_mean": 67.89405822753906,
"margin_dpo/margin_std": 99.7062759399414,
"step": 352
},
{
"KL/chosen_KL_mean": -121.88029479980469,
"KL/mean": -152.44581604003906,
"KL/rejected_KL_mean": -183.01132202148438,
"KL/std": 83.91139221191406,
"epoch": 0.5336356764928194,
"fcm_dpo/beta": 0.006154675967991352,
"fcm_dpo/delta": 0.026616372168064117,
"fcm_dpo/margin": 61.13102340698242,
"fcm_dpo/q_t": 0.4135306775569916,
"grad_norm": 10.312121391296387,
"learning_rate": 2.658559799141411e-07,
"logits/chosen": 0.7169567942619324,
"logits/rejected": 0.7259862422943115,
"logps/chosen": -190.88949584960938,
"logps/ref_chosen": -69.00918579101562,
"logps/ref_rejected": -72.65840148925781,
"logps/rejected": -255.6697235107422,
"loss": 1.1138,
"margin_dpo/margin_mean": 61.131019592285156,
"margin_dpo/margin_std": 87.21917724609375,
"step": 353
},
{
"KL/chosen_KL_mean": -121.11980438232422,
"KL/mean": -157.54824829101562,
"KL/rejected_KL_mean": -193.97665405273438,
"KL/std": 78.74341583251953,
"epoch": 0.5351473922902494,
"fcm_dpo/beta": 0.005997260566800833,
"fcm_dpo/delta": -0.04329075664281845,
"fcm_dpo/margin": 72.85685729980469,
"fcm_dpo/q_t": 0.39849966764450073,
"grad_norm": 12.67549991607666,
"learning_rate": 2.6453620722761895e-07,
"logits/chosen": 0.8099727630615234,
"logits/rejected": 0.6850665807723999,
"logps/chosen": -160.90814208984375,
"logps/ref_chosen": -39.78833770751953,
"logps/ref_rejected": -69.56885528564453,
"logps/rejected": -263.5455322265625,
"loss": 1.0772,
"margin_dpo/margin_mean": 72.85685729980469,
"margin_dpo/margin_std": 97.290283203125,
"step": 354
},
{
"KL/chosen_KL_mean": -125.32882690429688,
"KL/mean": -166.8375244140625,
"KL/rejected_KL_mean": -208.34622192382812,
"KL/std": 94.46980285644531,
"epoch": 0.5366591080876795,
"fcm_dpo/beta": 0.005797875113785267,
"fcm_dpo/delta": -0.09473682194948196,
"fcm_dpo/margin": 83.01737213134766,
"fcm_dpo/q_t": 0.38854479789733887,
"grad_norm": 13.447663307189941,
"learning_rate": 2.632160279321328e-07,
"logits/chosen": 0.7900065183639526,
"logits/rejected": 0.6645567417144775,
"logps/chosen": -171.58421325683594,
"logps/ref_chosen": -46.25537872314453,
"logps/ref_rejected": -78.20236206054688,
"logps/rejected": -286.548583984375,
"loss": 1.0509,
"margin_dpo/margin_mean": 83.01736450195312,
"margin_dpo/margin_std": 104.88021850585938,
"step": 355
},
{
"KL/chosen_KL_mean": -119.88313293457031,
"KL/mean": -152.29454040527344,
"KL/rejected_KL_mean": -184.70596313476562,
"KL/std": 87.65028381347656,
"epoch": 0.5381708238851096,
"fcm_dpo/beta": 0.005688229110091925,
"fcm_dpo/delta": 0.03451262786984444,
"fcm_dpo/margin": 64.82282257080078,
"fcm_dpo/q_t": 0.41692763566970825,
"grad_norm": 11.188521385192871,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 0.7430191040039062,
"logits/rejected": 0.6651791334152222,
"logps/chosen": -167.78929138183594,
"logps/ref_chosen": -47.906158447265625,
"logps/ref_rejected": -74.29397583007812,
"logps/rejected": -258.99993896484375,
"loss": 1.1454,
"margin_dpo/margin_mean": 64.82282257080078,
"margin_dpo/margin_std": 109.8090591430664,
"step": 356
},
{
"KL/chosen_KL_mean": -143.23452758789062,
"KL/mean": -169.093017578125,
"KL/rejected_KL_mean": -194.9515380859375,
"KL/std": 87.9165267944336,
"epoch": 0.5396825396825397,
"fcm_dpo/beta": 0.005629734601825476,
"fcm_dpo/delta": -0.04662557691335678,
"fcm_dpo/margin": 51.71701431274414,
"fcm_dpo/q_t": 0.4305192530155182,
"grad_norm": 11.694966316223145,
"learning_rate": 2.6057459723762076e-07,
"logits/chosen": 0.752831220626831,
"logits/rejected": 0.7355064153671265,
"logps/chosen": -205.86952209472656,
"logps/ref_chosen": -62.63500213623047,
"logps/ref_rejected": -65.11399841308594,
"logps/rejected": -260.0655517578125,
"loss": 1.1955,
"margin_dpo/margin_mean": 51.717018127441406,
"margin_dpo/margin_std": 97.05239868164062,
"step": 357
},
{
"KL/chosen_KL_mean": -142.22438049316406,
"KL/mean": -176.40380859375,
"KL/rejected_KL_mean": -210.58322143554688,
"KL/std": 94.13438415527344,
"epoch": 0.5411942554799698,
"fcm_dpo/beta": 0.00548651535063982,
"fcm_dpo/delta": -0.08703271299600601,
"fcm_dpo/margin": 68.35882568359375,
"fcm_dpo/q_t": 0.41342803835868835,
"grad_norm": 15.619354248046875,
"learning_rate": 2.5925341972508954e-07,
"logits/chosen": 0.6815810203552246,
"logits/rejected": 0.6956534385681152,
"logps/chosen": -209.43399047851562,
"logps/ref_chosen": -67.20960998535156,
"logps/ref_rejected": -69.34715270996094,
"logps/rejected": -279.93035888671875,
"loss": 1.134,
"margin_dpo/margin_mean": 68.35882568359375,
"margin_dpo/margin_std": 105.12598419189453,
"step": 358
},
{
"KL/chosen_KL_mean": -158.29672241210938,
"KL/mean": -176.74636840820312,
"KL/rejected_KL_mean": -195.1959991455078,
"KL/std": 92.31513977050781,
"epoch": 0.5427059712773998,
"fcm_dpo/beta": 0.005430396646261215,
"fcm_dpo/delta": 0.05346956476569176,
"fcm_dpo/margin": 36.89925765991211,
"fcm_dpo/q_t": 0.45445123314857483,
"grad_norm": 13.617073059082031,
"learning_rate": 2.579319833745169e-07,
"logits/chosen": 0.7321456074714661,
"logits/rejected": 0.6983045339584351,
"logps/chosen": -220.822509765625,
"logps/ref_chosen": -62.52578353881836,
"logps/ref_rejected": -76.63114929199219,
"logps/rejected": -271.8271484375,
"loss": 1.2704,
"margin_dpo/margin_mean": 36.89925765991211,
"margin_dpo/margin_std": 100.25183868408203,
"step": 359
},
{
"KL/chosen_KL_mean": -151.61859130859375,
"KL/mean": -184.96478271484375,
"KL/rejected_KL_mean": -218.31097412109375,
"KL/std": 98.37626647949219,
"epoch": 0.54421768707483,
"fcm_dpo/beta": 0.005576558876782656,
"fcm_dpo/delta": 0.031639836728572845,
"fcm_dpo/margin": 66.6923599243164,
"fcm_dpo/q_t": 0.415699303150177,
"grad_norm": 11.561031341552734,
"learning_rate": 2.5661032514931834e-07,
"logits/chosen": 0.6751697063446045,
"logits/rejected": 0.5904595255851746,
"logps/chosen": -215.1063232421875,
"logps/ref_chosen": -63.48772048950195,
"logps/ref_rejected": -90.6891098022461,
"logps/rejected": -309.00006103515625,
"loss": 1.1178,
"margin_dpo/margin_mean": 66.6923599243164,
"margin_dpo/margin_std": 97.05094909667969,
"step": 360
},
{
"KL/chosen_KL_mean": -143.3883056640625,
"KL/mean": -180.1334228515625,
"KL/rejected_KL_mean": -216.8785400390625,
"KL/std": 92.39273071289062,
"epoch": 0.54572940287226,
"fcm_dpo/beta": 0.005615275353193283,
"fcm_dpo/delta": -0.014561126939952374,
"fcm_dpo/margin": 73.49020385742188,
"fcm_dpo/q_t": 0.40417200326919556,
"grad_norm": 10.36706256866455,
"learning_rate": 2.552884820191154e-07,
"logits/chosen": 0.8067169785499573,
"logits/rejected": 0.7659634351730347,
"logps/chosen": -201.30545043945312,
"logps/ref_chosen": -57.917144775390625,
"logps/ref_rejected": -72.39089965820312,
"logps/rejected": -289.2694396972656,
"loss": 1.0856,
"margin_dpo/margin_mean": 73.49020385742188,
"margin_dpo/margin_std": 96.46673583984375,
"step": 361
},
{
"KL/chosen_KL_mean": -144.8900146484375,
"KL/mean": -188.41073608398438,
"KL/rejected_KL_mean": -231.93145751953125,
"KL/std": 106.30439758300781,
"epoch": 0.54724111866969,
"fcm_dpo/beta": 0.005439079366624355,
"fcm_dpo/delta": -0.08564796298742294,
"fcm_dpo/margin": 87.04143524169922,
"fcm_dpo/q_t": 0.39451199769973755,
"grad_norm": 11.566519737243652,
"learning_rate": 2.53966490958702e-07,
"logits/chosen": 0.8194698095321655,
"logits/rejected": 0.7212804555892944,
"logps/chosen": -208.33348083496094,
"logps/ref_chosen": -63.4434700012207,
"logps/ref_rejected": -103.45516967773438,
"logps/rejected": -335.3866271972656,
"loss": 1.0693,
"margin_dpo/margin_mean": 87.04142761230469,
"margin_dpo/margin_std": 124.4408187866211,
"step": 362
},
{
"KL/chosen_KL_mean": -147.78335571289062,
"KL/mean": -186.64004516601562,
"KL/rejected_KL_mean": -225.4967041015625,
"KL/std": 92.55686950683594,
"epoch": 0.5487528344671202,
"fcm_dpo/beta": 0.005266488064080477,
"fcm_dpo/delta": -0.010586944408714771,
"fcm_dpo/margin": 77.71334075927734,
"fcm_dpo/q_t": 0.40681716799736023,
"grad_norm": 13.70695686340332,
"learning_rate": 2.526443889470099e-07,
"logits/chosen": 0.7966769933700562,
"logits/rejected": 0.6717593669891357,
"logps/chosen": -196.43519592285156,
"logps/ref_chosen": -48.65182876586914,
"logps/ref_rejected": -88.65904235839844,
"logps/rejected": -314.15576171875,
"loss": 1.0935,
"margin_dpo/margin_mean": 77.71334075927734,
"margin_dpo/margin_std": 108.69964599609375,
"step": 363
},
{
"KL/chosen_KL_mean": -131.58168029785156,
"KL/mean": -174.65626525878906,
"KL/rejected_KL_mean": -217.7308349609375,
"KL/std": 102.02590942382812,
"epoch": 0.5502645502645502,
"fcm_dpo/beta": 0.0051279449835419655,
"fcm_dpo/delta": -0.04850374162197113,
"fcm_dpo/margin": 86.14913940429688,
"fcm_dpo/q_t": 0.4003210961818695,
"grad_norm": 9.81286334991455,
"learning_rate": 2.513222129660744e-07,
"logits/chosen": 0.6942867040634155,
"logits/rejected": 0.6157143115997314,
"logps/chosen": -189.4527587890625,
"logps/ref_chosen": -57.87107467651367,
"logps/ref_rejected": -80.95503234863281,
"logps/rejected": -298.68585205078125,
"loss": 1.0928,
"margin_dpo/margin_mean": 86.14913940429688,
"margin_dpo/margin_std": 129.07220458984375,
"step": 364
},
{
"KL/chosen_KL_mean": -125.02583312988281,
"KL/mean": -164.640380859375,
"KL/rejected_KL_mean": -204.25491333007812,
"KL/std": 108.65450286865234,
"epoch": 0.5517762660619804,
"fcm_dpo/beta": 0.0050535742193460464,
"fcm_dpo/delta": -0.0012763142585754395,
"fcm_dpo/margin": 79.22908782958984,
"fcm_dpo/q_t": 0.4074895977973938,
"grad_norm": 10.674773216247559,
"learning_rate": 2.5e-07,
"logits/chosen": 0.7654601335525513,
"logits/rejected": 0.7636349201202393,
"logps/chosen": -189.968017578125,
"logps/ref_chosen": -64.94217681884766,
"logps/ref_rejected": -74.8599853515625,
"logps/rejected": -279.1148986816406,
"loss": 1.0785,
"margin_dpo/margin_mean": 79.22908782958984,
"margin_dpo/margin_std": 95.32018280029297,
"step": 365
},
{
"KL/chosen_KL_mean": -124.21640014648438,
"KL/mean": -156.70164489746094,
"KL/rejected_KL_mean": -189.18690490722656,
"KL/std": 95.22203063964844,
"epoch": 0.5532879818594104,
"fcm_dpo/beta": 0.005297908093780279,
"fcm_dpo/delta": 0.06093678995966911,
"fcm_dpo/margin": 64.97052001953125,
"fcm_dpo/q_t": 0.4231628477573395,
"grad_norm": 13.27842903137207,
"learning_rate": 2.486777870339255e-07,
"logits/chosen": 0.7186771631240845,
"logits/rejected": 0.7142035365104675,
"logps/chosen": -179.38238525390625,
"logps/ref_chosen": -55.16598129272461,
"logps/ref_rejected": -65.26121520996094,
"logps/rejected": -254.4481201171875,
"loss": 1.1684,
"margin_dpo/margin_mean": 64.97052001953125,
"margin_dpo/margin_std": 120.9197769165039,
"step": 366
},
{
"KL/chosen_KL_mean": -142.8965606689453,
"KL/mean": -175.44778442382812,
"KL/rejected_KL_mean": -207.9990234375,
"KL/std": 94.457275390625,
"epoch": 0.5547996976568406,
"fcm_dpo/beta": 0.005413766950368881,
"fcm_dpo/delta": 0.05324382334947586,
"fcm_dpo/margin": 65.10247802734375,
"fcm_dpo/q_t": 0.4184330105781555,
"grad_norm": 11.966232299804688,
"learning_rate": 2.4735561105299014e-07,
"logits/chosen": 0.754651665687561,
"logits/rejected": 0.658037543296814,
"logps/chosen": -198.90701293945312,
"logps/ref_chosen": -56.01046371459961,
"logps/ref_rejected": -77.31010437011719,
"logps/rejected": -285.30914306640625,
"loss": 1.1341,
"margin_dpo/margin_mean": 65.10247802734375,
"margin_dpo/margin_std": 99.43478393554688,
"step": 367
},
{
"KL/chosen_KL_mean": -154.50941467285156,
"KL/mean": -183.74929809570312,
"KL/rejected_KL_mean": -212.98915100097656,
"KL/std": 93.31791687011719,
"epoch": 0.5563114134542706,
"fcm_dpo/beta": 0.0056096892803907394,
"fcm_dpo/delta": 0.07988474518060684,
"fcm_dpo/margin": 58.47974395751953,
"fcm_dpo/q_t": 0.42577698826789856,
"grad_norm": 13.028302192687988,
"learning_rate": 2.46033509041298e-07,
"logits/chosen": 0.5896161794662476,
"logits/rejected": 0.592335045337677,
"logps/chosen": -229.3386993408203,
"logps/ref_chosen": -74.82927703857422,
"logps/ref_rejected": -76.11680603027344,
"logps/rejected": -289.10595703125,
"loss": 1.1633,
"margin_dpo/margin_mean": 58.47974395751953,
"margin_dpo/margin_std": 100.59825134277344,
"step": 368
},
{
"KL/chosen_KL_mean": -141.26634216308594,
"KL/mean": -168.32205200195312,
"KL/rejected_KL_mean": -195.3777618408203,
"KL/std": 93.77578735351562,
"epoch": 0.5578231292517006,
"fcm_dpo/beta": 0.0060209897346794605,
"fcm_dpo/delta": 0.0800437182188034,
"fcm_dpo/margin": 54.111427307128906,
"fcm_dpo/q_t": 0.4274851679801941,
"grad_norm": 13.085291862487793,
"learning_rate": 2.447115179808846e-07,
"logits/chosen": 0.7499760389328003,
"logits/rejected": 0.704288125038147,
"logps/chosen": -199.59255981445312,
"logps/ref_chosen": -58.32621765136719,
"logps/ref_rejected": -80.92183685302734,
"logps/rejected": -276.29962158203125,
"loss": 1.1727,
"margin_dpo/margin_mean": 54.111427307128906,
"margin_dpo/margin_std": 99.9386978149414,
"step": 369
},
{
"KL/chosen_KL_mean": -133.82888793945312,
"KL/mean": -173.8616943359375,
"KL/rejected_KL_mean": -213.89450073242188,
"KL/std": 97.72712707519531,
"epoch": 0.5593348450491308,
"fcm_dpo/beta": 0.005765399895608425,
"fcm_dpo/delta": -0.07319603860378265,
"fcm_dpo/margin": 80.06562805175781,
"fcm_dpo/q_t": 0.39234304428100586,
"grad_norm": 12.59056568145752,
"learning_rate": 2.4338967485068164e-07,
"logits/chosen": 0.8812678456306458,
"logits/rejected": 0.8152058124542236,
"logps/chosen": -186.71258544921875,
"logps/ref_chosen": -52.88372039794922,
"logps/ref_rejected": -79.43692016601562,
"logps/rejected": -293.3314208984375,
"loss": 1.0779,
"margin_dpo/margin_mean": 80.06562805175781,
"margin_dpo/margin_std": 113.96977233886719,
"step": 370
},
{
"KL/chosen_KL_mean": -131.85989379882812,
"KL/mean": -166.550048828125,
"KL/rejected_KL_mean": -201.240234375,
"KL/std": 91.61387634277344,
"epoch": 0.5608465608465608,
"fcm_dpo/beta": 0.005842489190399647,
"fcm_dpo/delta": -0.008122660219669342,
"fcm_dpo/margin": 69.38032531738281,
"fcm_dpo/q_t": 0.4071889817714691,
"grad_norm": 15.300116539001465,
"learning_rate": 2.420680166254831e-07,
"logits/chosen": 0.8666425347328186,
"logits/rejected": 0.8389255404472351,
"logps/chosen": -181.0841064453125,
"logps/ref_chosen": -49.224212646484375,
"logps/ref_rejected": -63.348472595214844,
"logps/rejected": -264.58868408203125,
"loss": 1.1081,
"margin_dpo/margin_mean": 69.38032531738281,
"margin_dpo/margin_std": 101.50696563720703,
"step": 371
},
{
"KL/chosen_KL_mean": -139.83580017089844,
"KL/mean": -160.21075439453125,
"KL/rejected_KL_mean": -180.585693359375,
"KL/std": 92.19517517089844,
"epoch": 0.562358276643991,
"fcm_dpo/beta": 0.005775434896349907,
"fcm_dpo/delta": 0.02449907548725605,
"fcm_dpo/margin": 40.7498779296875,
"fcm_dpo/q_t": 0.448069304227829,
"grad_norm": 13.797256469726562,
"learning_rate": 2.4074658027491044e-07,
"logits/chosen": 0.7692815065383911,
"logits/rejected": 0.6794453859329224,
"logps/chosen": -192.1053466796875,
"logps/ref_chosen": -52.269554138183594,
"logps/ref_rejected": -72.99522399902344,
"logps/rejected": -253.58090209960938,
"loss": 1.2809,
"margin_dpo/margin_mean": 40.749881744384766,
"margin_dpo/margin_std": 119.3675537109375,
"step": 372
},
{
"KL/chosen_KL_mean": -158.91616821289062,
"KL/mean": -186.7388916015625,
"KL/rejected_KL_mean": -214.56161499023438,
"KL/std": 99.78253173828125,
"epoch": 0.563869992441421,
"fcm_dpo/beta": 0.006023020949214697,
"fcm_dpo/delta": 0.07183150947093964,
"fcm_dpo/margin": 55.64544677734375,
"fcm_dpo/q_t": 0.42674756050109863,
"grad_norm": 13.5969820022583,
"learning_rate": 2.394254027623792e-07,
"logits/chosen": 0.787378191947937,
"logits/rejected": 0.7218998670578003,
"logps/chosen": -220.0291748046875,
"logps/ref_chosen": -61.112998962402344,
"logps/ref_rejected": -76.24851989746094,
"logps/rejected": -290.81011962890625,
"loss": 1.2072,
"margin_dpo/margin_mean": 55.64544677734375,
"margin_dpo/margin_std": 123.12017822265625,
"step": 373
},
{
"KL/chosen_KL_mean": -134.72891235351562,
"KL/mean": -178.64892578125,
"KL/rejected_KL_mean": -222.56893920898438,
"KL/std": 102.30409240722656,
"epoch": 0.5653817082388511,
"fcm_dpo/beta": 0.005733366124331951,
"fcm_dpo/delta": -0.12269183248281479,
"fcm_dpo/margin": 87.84001159667969,
"fcm_dpo/q_t": 0.38412588834762573,
"grad_norm": 14.229789733886719,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 0.6749398708343506,
"logits/rejected": 0.6806223392486572,
"logps/chosen": -207.3981170654297,
"logps/ref_chosen": -72.66920471191406,
"logps/ref_rejected": -76.83158874511719,
"logps/rejected": -299.4005126953125,
"loss": 1.0639,
"margin_dpo/margin_mean": 87.84001159667969,
"margin_dpo/margin_std": 126.37348175048828,
"step": 374
},
{
"KL/chosen_KL_mean": -133.24624633789062,
"KL/mean": -164.95437622070312,
"KL/rejected_KL_mean": -196.66249084472656,
"KL/std": 110.367431640625,
"epoch": 0.5668934240362812,
"fcm_dpo/beta": 0.005722872447222471,
"fcm_dpo/delta": 0.04165318235754967,
"fcm_dpo/margin": 63.41625213623047,
"fcm_dpo/q_t": 0.4203999638557434,
"grad_norm": 15.683096885681152,
"learning_rate": 2.3678397206786715e-07,
"logits/chosen": 0.7528005838394165,
"logits/rejected": 0.7014273405075073,
"logps/chosen": -190.92953491210938,
"logps/ref_chosen": -57.68330383300781,
"logps/ref_rejected": -79.34097290039062,
"logps/rejected": -276.00347900390625,
"loss": 1.1684,
"margin_dpo/margin_mean": 63.4162483215332,
"margin_dpo/margin_std": 121.38080596923828,
"step": 375
},
{
"KL/chosen_KL_mean": -141.19493103027344,
"KL/mean": -181.21041870117188,
"KL/rejected_KL_mean": -221.2259521484375,
"KL/std": 112.07803344726562,
"epoch": 0.5684051398337112,
"fcm_dpo/beta": 0.005588100291788578,
"fcm_dpo/delta": -0.05586222559213638,
"fcm_dpo/margin": 80.03101348876953,
"fcm_dpo/q_t": 0.4027999937534332,
"grad_norm": 12.794271469116211,
"learning_rate": 2.3546379277238103e-07,
"logits/chosen": 0.8139768838882446,
"logits/rejected": 0.7517552971839905,
"logps/chosen": -192.86898803710938,
"logps/ref_chosen": -51.674072265625,
"logps/ref_rejected": -75.69713592529297,
"logps/rejected": -296.923095703125,
"loss": 1.112,
"margin_dpo/margin_mean": 80.03101348876953,
"margin_dpo/margin_std": 133.2711639404297,
"step": 376
},
{
"KL/chosen_KL_mean": -141.94168090820312,
"KL/mean": -169.37771606445312,
"KL/rejected_KL_mean": -196.8137664794922,
"KL/std": 87.98646545410156,
"epoch": 0.5699168556311414,
"fcm_dpo/beta": 0.005861810874193907,
"fcm_dpo/delta": 0.08542998135089874,
"fcm_dpo/margin": 54.872093200683594,
"fcm_dpo/q_t": 0.42690926790237427,
"grad_norm": 14.541651725769043,
"learning_rate": 2.3414402008585886e-07,
"logits/chosen": 0.7738527059555054,
"logits/rejected": 0.7508142590522766,
"logps/chosen": -188.12020874023438,
"logps/ref_chosen": -46.17853546142578,
"logps/ref_rejected": -57.756500244140625,
"logps/rejected": -254.5702667236328,
"loss": 1.1706,
"margin_dpo/margin_mean": 54.872093200683594,
"margin_dpo/margin_std": 98.96908569335938,
"step": 377
},
{
"KL/chosen_KL_mean": -139.3751678466797,
"KL/mean": -166.8651123046875,
"KL/rejected_KL_mean": -194.35507202148438,
"KL/std": 103.14115142822266,
"epoch": 0.5714285714285714,
"fcm_dpo/beta": 0.006156137678772211,
"fcm_dpo/delta": 0.06586823612451553,
"fcm_dpo/margin": 54.979896545410156,
"fcm_dpo/q_t": 0.4245694875717163,
"grad_norm": 11.343628883361816,
"learning_rate": 2.3282469092517977e-07,
"logits/chosen": 0.8040895462036133,
"logits/rejected": 0.7606953382492065,
"logps/chosen": -198.59405517578125,
"logps/ref_chosen": -59.21887969970703,
"logps/ref_rejected": -71.24818420410156,
"logps/rejected": -265.603271484375,
"loss": 1.1734,
"margin_dpo/margin_mean": 54.97990036010742,
"margin_dpo/margin_std": 101.69361877441406,
"step": 378
},
{
"KL/chosen_KL_mean": -141.87744140625,
"KL/mean": -181.7213897705078,
"KL/rejected_KL_mean": -221.56533813476562,
"KL/std": 107.65896606445312,
"epoch": 0.5729402872260015,
"fcm_dpo/beta": 0.005909925326704979,
"fcm_dpo/delta": -0.08261381089687347,
"fcm_dpo/margin": 79.68791961669922,
"fcm_dpo/q_t": 0.39472633600234985,
"grad_norm": 14.828629493713379,
"learning_rate": 2.3150584219481643e-07,
"logits/chosen": 0.7641682624816895,
"logits/rejected": 0.6950868368148804,
"logps/chosen": -218.19403076171875,
"logps/ref_chosen": -76.31658935546875,
"logps/ref_rejected": -104.26200103759766,
"logps/rejected": -325.82733154296875,
"loss": 1.0766,
"margin_dpo/margin_mean": 79.68791961669922,
"margin_dpo/margin_std": 117.0067138671875,
"step": 379
},
{
"KL/chosen_KL_mean": -118.77130889892578,
"KL/mean": -161.55316162109375,
"KL/rejected_KL_mean": -204.33502197265625,
"KL/std": 95.21235656738281,
"epoch": 0.5744520030234316,
"fcm_dpo/beta": 0.005602892488241196,
"fcm_dpo/delta": -0.09268143028020859,
"fcm_dpo/margin": 85.5636978149414,
"fcm_dpo/q_t": 0.388763964176178,
"grad_norm": 13.451189041137695,
"learning_rate": 2.3018751078580283e-07,
"logits/chosen": 0.7501556873321533,
"logits/rejected": 0.7167741060256958,
"logps/chosen": -180.05447387695312,
"logps/ref_chosen": -61.283164978027344,
"logps/ref_rejected": -72.38892364501953,
"logps/rejected": -276.72393798828125,
"loss": 1.0568,
"margin_dpo/margin_mean": 85.5636978149414,
"margin_dpo/margin_std": 113.08406066894531,
"step": 380
},
{
"KL/chosen_KL_mean": -145.89193725585938,
"KL/mean": -162.467041015625,
"KL/rejected_KL_mean": -179.04214477539062,
"KL/std": 100.26214599609375,
"epoch": 0.5759637188208617,
"fcm_dpo/beta": 0.005632633343338966,
"fcm_dpo/delta": 0.06954426318407059,
"fcm_dpo/margin": 33.150203704833984,
"fcm_dpo/q_t": 0.45884740352630615,
"grad_norm": 11.799565315246582,
"learning_rate": 2.288697335747027e-07,
"logits/chosen": 0.725066065788269,
"logits/rejected": 0.7106317281723022,
"logps/chosen": -204.10592651367188,
"logps/ref_chosen": -58.2139892578125,
"logps/ref_rejected": -60.78669357299805,
"logps/rejected": -239.82882690429688,
"loss": 1.3176,
"margin_dpo/margin_mean": 33.15020751953125,
"margin_dpo/margin_std": 121.16805267333984,
"step": 381
},
{
"KL/chosen_KL_mean": -146.5373077392578,
"KL/mean": -177.61614990234375,
"KL/rejected_KL_mean": -208.69497680664062,
"KL/std": 100.65556335449219,
"epoch": 0.5774754346182918,
"fcm_dpo/beta": 0.005918251350522041,
"fcm_dpo/delta": 0.0343366302549839,
"fcm_dpo/margin": 62.157691955566406,
"fcm_dpo/q_t": 0.4175347685813904,
"grad_norm": 14.061190605163574,
"learning_rate": 2.2755254742257706e-07,
"logits/chosen": 0.7362748384475708,
"logits/rejected": 0.6886606216430664,
"logps/chosen": -208.3626251220703,
"logps/ref_chosen": -61.82532501220703,
"logps/ref_rejected": -83.0452880859375,
"logps/rejected": -291.7402648925781,
"loss": 1.1333,
"margin_dpo/margin_mean": 62.157691955566406,
"margin_dpo/margin_std": 98.62821960449219,
"step": 382
},
{
"KL/chosen_KL_mean": -151.32742309570312,
"KL/mean": -177.73983764648438,
"KL/rejected_KL_mean": -204.15228271484375,
"KL/std": 99.92002868652344,
"epoch": 0.5789871504157218,
"fcm_dpo/beta": 0.006043318659067154,
"fcm_dpo/delta": 0.08946429938077927,
"fcm_dpo/margin": 52.82486343383789,
"fcm_dpo/q_t": 0.43128010630607605,
"grad_norm": 16.85250473022461,
"learning_rate": 2.2623598917395436e-07,
"logits/chosen": 0.6481820344924927,
"logits/rejected": 0.6832894086837769,
"logps/chosen": -231.89068603515625,
"logps/ref_chosen": -80.56326293945312,
"logps/ref_rejected": -74.62922668457031,
"logps/rejected": -278.781494140625,
"loss": 1.2263,
"margin_dpo/margin_mean": 52.824859619140625,
"margin_dpo/margin_std": 125.01660919189453,
"step": 383
},
{
"KL/chosen_KL_mean": -137.10409545898438,
"KL/mean": -170.72140502929688,
"KL/rejected_KL_mean": -204.33868408203125,
"KL/std": 95.64372253417969,
"epoch": 0.5804988662131519,
"fcm_dpo/beta": 0.006174423731863499,
"fcm_dpo/delta": -0.01735561341047287,
"fcm_dpo/margin": 67.234619140625,
"fcm_dpo/q_t": 0.403439462184906,
"grad_norm": 15.312804222106934,
"learning_rate": 2.2492009565579875e-07,
"logits/chosen": 0.7630481123924255,
"logits/rejected": 0.7314999103546143,
"logps/chosen": -202.57925415039062,
"logps/ref_chosen": -65.47514343261719,
"logps/ref_rejected": -79.67378234863281,
"logps/rejected": -284.0124816894531,
"loss": 1.1058,
"margin_dpo/margin_mean": 67.234619140625,
"margin_dpo/margin_std": 101.94595336914062,
"step": 384
},
{
"KL/chosen_KL_mean": -140.1160888671875,
"KL/mean": -182.65090942382812,
"KL/rejected_KL_mean": -225.18572998046875,
"KL/std": 100.97268676757812,
"epoch": 0.582010582010582,
"fcm_dpo/beta": 0.005884962156414986,
"fcm_dpo/delta": -0.11797355115413666,
"fcm_dpo/margin": 85.06967163085938,
"fcm_dpo/q_t": 0.3843558132648468,
"grad_norm": 11.957921028137207,
"learning_rate": 2.2360490367648084e-07,
"logits/chosen": 0.701429009437561,
"logits/rejected": 0.6622118949890137,
"logps/chosen": -206.17259216308594,
"logps/ref_chosen": -66.0565185546875,
"logps/ref_rejected": -86.68023681640625,
"logps/rejected": -311.865966796875,
"loss": 1.0349,
"margin_dpo/margin_mean": 85.06966400146484,
"margin_dpo/margin_std": 103.32591247558594,
"step": 385
},
{
"KL/chosen_KL_mean": -161.50335693359375,
"KL/mean": -189.95013427734375,
"KL/rejected_KL_mean": -218.39691162109375,
"KL/std": 107.01244354248047,
"epoch": 0.5835222978080121,
"fcm_dpo/beta": 0.005871026776731014,
"fcm_dpo/delta": 0.07351464033126831,
"fcm_dpo/margin": 56.8935661315918,
"fcm_dpo/q_t": 0.42427319288253784,
"grad_norm": 13.5010347366333,
"learning_rate": 2.2229045002474724e-07,
"logits/chosen": 0.6531039476394653,
"logits/rejected": 0.6064084768295288,
"logps/chosen": -237.12701416015625,
"logps/ref_chosen": -75.6236572265625,
"logps/ref_rejected": -92.62330627441406,
"logps/rejected": -311.02020263671875,
"loss": 1.1754,
"margin_dpo/margin_mean": 56.89356994628906,
"margin_dpo/margin_std": 107.92727661132812,
"step": 386
},
{
"KL/chosen_KL_mean": -141.63909912109375,
"KL/mean": -181.66258239746094,
"KL/rejected_KL_mean": -221.68606567382812,
"KL/std": 96.4151382446289,
"epoch": 0.5850340136054422,
"fcm_dpo/beta": 0.005757711827754974,
"fcm_dpo/delta": -0.07114600390195847,
"fcm_dpo/margin": 80.04695129394531,
"fcm_dpo/q_t": 0.3940368890762329,
"grad_norm": 13.86131763458252,
"learning_rate": 2.209767714686924e-07,
"logits/chosen": 0.7648110389709473,
"logits/rejected": 0.6623188257217407,
"logps/chosen": -188.86080932617188,
"logps/ref_chosen": -47.22170639038086,
"logps/ref_rejected": -87.338134765625,
"logps/rejected": -309.0242004394531,
"loss": 1.0566,
"margin_dpo/margin_mean": 80.04695129394531,
"margin_dpo/margin_std": 101.74493408203125,
"step": 387
},
{
"KL/chosen_KL_mean": -148.16046142578125,
"KL/mean": -172.68748474121094,
"KL/rejected_KL_mean": -197.21450805664062,
"KL/std": 103.10355377197266,
"epoch": 0.5865457294028723,
"fcm_dpo/beta": 0.0057308124378323555,
"fcm_dpo/delta": 0.003550410969182849,
"fcm_dpo/margin": 49.054039001464844,
"fcm_dpo/q_t": 0.4384087324142456,
"grad_norm": 12.839620590209961,
"learning_rate": 2.1966390475472954e-07,
"logits/chosen": 0.7350201606750488,
"logits/rejected": 0.7327404022216797,
"logps/chosen": -222.73992919921875,
"logps/ref_chosen": -74.5794677734375,
"logps/ref_rejected": -79.92558288574219,
"logps/rejected": -277.14007568359375,
"loss": 1.2403,
"margin_dpo/margin_mean": 49.054039001464844,
"margin_dpo/margin_std": 121.98270416259766,
"step": 388
},
{
"KL/chosen_KL_mean": -138.47047424316406,
"KL/mean": -178.14364624023438,
"KL/rejected_KL_mean": -217.81683349609375,
"KL/std": 97.10962677001953,
"epoch": 0.5880574452003023,
"fcm_dpo/beta": 0.005619604140520096,
"fcm_dpo/delta": -0.05288499593734741,
"fcm_dpo/margin": 79.34635925292969,
"fcm_dpo/q_t": 0.39804351329803467,
"grad_norm": 21.324504852294922,
"learning_rate": 2.1835188660656265e-07,
"logits/chosen": 0.7512508630752563,
"logits/rejected": 0.7208691835403442,
"logps/chosen": -200.0948486328125,
"logps/ref_chosen": -61.624366760253906,
"logps/ref_rejected": -76.50978088378906,
"logps/rejected": -294.32659912109375,
"loss": 1.0743,
"margin_dpo/margin_mean": 79.34635925292969,
"margin_dpo/margin_std": 107.70132446289062,
"step": 389
},
{
"KL/chosen_KL_mean": -123.1337661743164,
"KL/mean": -153.21640014648438,
"KL/rejected_KL_mean": -183.29904174804688,
"KL/std": 92.85203552246094,
"epoch": 0.5895691609977324,
"fcm_dpo/beta": 0.00567449489608407,
"fcm_dpo/delta": 0.06538356095552444,
"fcm_dpo/margin": 60.165260314941406,
"fcm_dpo/q_t": 0.4235107898712158,
"grad_norm": 11.731263160705566,
"learning_rate": 2.170407537241599e-07,
"logits/chosen": 0.8297513723373413,
"logits/rejected": 0.7623568773269653,
"logps/chosen": -169.00563049316406,
"logps/ref_chosen": -45.871864318847656,
"logps/ref_rejected": -61.305999755859375,
"logps/rejected": -244.60504150390625,
"loss": 1.15,
"margin_dpo/margin_mean": 60.16526412963867,
"margin_dpo/margin_std": 99.79165649414062,
"step": 390
},
{
"KL/chosen_KL_mean": -138.01197814941406,
"KL/mean": -175.88485717773438,
"KL/rejected_KL_mean": -213.7576904296875,
"KL/std": 97.07452392578125,
"epoch": 0.5910808767951625,
"fcm_dpo/beta": 0.005652905907481909,
"fcm_dpo/delta": -0.03288257122039795,
"fcm_dpo/margin": 75.74574279785156,
"fcm_dpo/q_t": 0.40263625979423523,
"grad_norm": 12.69221305847168,
"learning_rate": 2.1573054278272636e-07,
"logits/chosen": 0.7633911967277527,
"logits/rejected": 0.7019753456115723,
"logps/chosen": -196.198974609375,
"logps/ref_chosen": -58.18701171875,
"logps/ref_rejected": -83.63442993164062,
"logps/rejected": -297.39215087890625,
"loss": 1.1052,
"margin_dpo/margin_mean": 75.74574279785156,
"margin_dpo/margin_std": 116.3956298828125,
"step": 391
},
{
"KL/chosen_KL_mean": -127.93291473388672,
"KL/mean": -166.52969360351562,
"KL/rejected_KL_mean": -205.12648010253906,
"KL/std": 105.89525604248047,
"epoch": 0.5925925925925926,
"fcm_dpo/beta": 0.005653767846524715,
"fcm_dpo/delta": -0.043347496539354324,
"fcm_dpo/margin": 77.19355773925781,
"fcm_dpo/q_t": 0.4022483229637146,
"grad_norm": 11.86667537689209,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 0.7835770845413208,
"logits/rejected": 0.7315959930419922,
"logps/chosen": -197.6774444580078,
"logps/ref_chosen": -69.7445297241211,
"logps/ref_rejected": -94.05877685546875,
"logps/rejected": -299.18524169921875,
"loss": 1.1083,
"margin_dpo/margin_mean": 77.19355773925781,
"margin_dpo/margin_std": 123.01547241210938,
"step": 392
},
{
"KL/chosen_KL_mean": -142.97479248046875,
"KL/mean": -184.1305694580078,
"KL/rejected_KL_mean": -225.28634643554688,
"KL/std": 102.22776794433594,
"epoch": 0.5941043083900227,
"fcm_dpo/beta": 0.005305946804583073,
"fcm_dpo/delta": -0.04566541686654091,
"fcm_dpo/margin": 82.31155395507812,
"fcm_dpo/q_t": 0.3981964588165283,
"grad_norm": 10.274748802185059,
"learning_rate": 2.131130332936195e-07,
"logits/chosen": 0.7935898303985596,
"logits/rejected": 0.7587308883666992,
"logps/chosen": -195.3096923828125,
"logps/ref_chosen": -52.33489990234375,
"logps/ref_rejected": -74.33809661865234,
"logps/rejected": -299.62445068359375,
"loss": 1.0668,
"margin_dpo/margin_mean": 82.3115463256836,
"margin_dpo/margin_std": 100.75532531738281,
"step": 393
},
{
"KL/chosen_KL_mean": -136.1352996826172,
"KL/mean": -171.53225708007812,
"KL/rejected_KL_mean": -206.92921447753906,
"KL/std": 95.84933471679688,
"epoch": 0.5956160241874527,
"fcm_dpo/beta": 0.005464646033942699,
"fcm_dpo/delta": 0.013672813773155212,
"fcm_dpo/margin": 70.79390716552734,
"fcm_dpo/q_t": 0.41006964445114136,
"grad_norm": 12.026927947998047,
"learning_rate": 2.1180580796331323e-07,
"logits/chosen": 0.7781930565834045,
"logits/rejected": 0.751598596572876,
"logps/chosen": -196.81143188476562,
"logps/ref_chosen": -60.6761360168457,
"logps/ref_rejected": -71.36074829101562,
"logps/rejected": -278.28997802734375,
"loss": 1.1026,
"margin_dpo/margin_mean": 70.79390716552734,
"margin_dpo/margin_std": 94.3880615234375,
"step": 394
},
{
"KL/chosen_KL_mean": -137.5908660888672,
"KL/mean": -171.83412170410156,
"KL/rejected_KL_mean": -206.07737731933594,
"KL/std": 98.70809173583984,
"epoch": 0.5971277399848829,
"fcm_dpo/beta": 0.005507326684892178,
"fcm_dpo/delta": 0.02526915818452835,
"fcm_dpo/margin": 68.48651885986328,
"fcm_dpo/q_t": 0.41558602452278137,
"grad_norm": 11.91160774230957,
"learning_rate": 2.104996510066625e-07,
"logits/chosen": 0.8012561798095703,
"logits/rejected": 0.7142231464385986,
"logps/chosen": -188.1951904296875,
"logps/ref_chosen": -50.60432434082031,
"logps/ref_rejected": -77.08731079101562,
"logps/rejected": -283.1646728515625,
"loss": 1.1236,
"margin_dpo/margin_mean": 68.48651885986328,
"margin_dpo/margin_std": 106.47164916992188,
"step": 395
},
{
"KL/chosen_KL_mean": -133.67884826660156,
"KL/mean": -168.15289306640625,
"KL/rejected_KL_mean": -202.62692260742188,
"KL/std": 106.60101318359375,
"epoch": 0.5986394557823129,
"fcm_dpo/beta": 0.0052660866640508175,
"fcm_dpo/delta": -0.07049454748630524,
"fcm_dpo/margin": 68.94806671142578,
"fcm_dpo/q_t": 0.4145070016384125,
"grad_norm": 10.632168769836426,
"learning_rate": 2.0919459895968517e-07,
"logits/chosen": 0.7917169332504272,
"logits/rejected": 0.7078319787979126,
"logps/chosen": -185.0384521484375,
"logps/ref_chosen": -51.35961151123047,
"logps/ref_rejected": -79.89360046386719,
"logps/rejected": -282.5205078125,
"loss": 1.1199,
"margin_dpo/margin_mean": 68.94806671142578,
"margin_dpo/margin_std": 95.077392578125,
"step": 396
},
{
"KL/chosen_KL_mean": -154.8973388671875,
"KL/mean": -174.40435791015625,
"KL/rejected_KL_mean": -193.911376953125,
"KL/std": 99.12435150146484,
"epoch": 0.600151171579743,
"fcm_dpo/beta": 0.0052660866640508175,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 39.0140266418457,
"fcm_dpo/q_t": 0.45285657048225403,
"grad_norm": 11.291228294372559,
"learning_rate": 2.078906883274924e-07,
"logits/chosen": 0.7051912546157837,
"logits/rejected": 0.6572903990745544,
"logps/chosen": -221.35357666015625,
"logps/ref_chosen": -66.45622253417969,
"logps/ref_rejected": -85.74736785888672,
"logps/rejected": -279.65875244140625,
"loss": 1.2758,
"margin_dpo/margin_mean": 39.01403045654297,
"margin_dpo/margin_std": 111.57695770263672,
"step": 397
},
{
"KL/chosen_KL_mean": -129.05348205566406,
"KL/mean": -174.99749755859375,
"KL/rejected_KL_mean": -220.94149780273438,
"KL/std": 103.12016296386719,
"epoch": 0.6016628873771731,
"fcm_dpo/beta": 0.0050222016870975494,
"fcm_dpo/delta": -0.0740644708275795,
"fcm_dpo/margin": 91.88800048828125,
"fcm_dpo/q_t": 0.39427825808525085,
"grad_norm": 9.868289947509766,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.7551954984664917,
"logits/rejected": 0.6983195543289185,
"logps/chosen": -178.2977294921875,
"logps/ref_chosen": -49.244239807128906,
"logps/ref_rejected": -75.18949127197266,
"logps/rejected": -296.1309814453125,
"loss": 1.0491,
"margin_dpo/margin_mean": 91.88800048828125,
"margin_dpo/margin_std": 112.2702865600586,
"step": 398
},
{
"KL/chosen_KL_mean": -154.25454711914062,
"KL/mean": -207.419921875,
"KL/rejected_KL_mean": -260.58526611328125,
"KL/std": 118.88225555419922,
"epoch": 0.6031746031746031,
"fcm_dpo/beta": 0.004701526835560799,
"fcm_dpo/delta": -0.12128760665655136,
"fcm_dpo/margin": 106.33074188232422,
"fcm_dpo/q_t": 0.3860124349594116,
"grad_norm": 10.134847640991211,
"learning_rate": 2.052864371672457e-07,
"logits/chosen": 0.7207819223403931,
"logits/rejected": 0.591735303401947,
"logps/chosen": -222.56134033203125,
"logps/ref_chosen": -68.30679321289062,
"logps/ref_rejected": -113.2708511352539,
"logps/rejected": -373.85614013671875,
"loss": 1.0369,
"margin_dpo/margin_mean": 106.33074188232422,
"margin_dpo/margin_std": 135.85537719726562,
"step": 399
},
{
"KL/chosen_KL_mean": -171.22833251953125,
"KL/mean": -200.75521850585938,
"KL/rejected_KL_mean": -230.28211975097656,
"KL/std": 106.50505065917969,
"epoch": 0.6046863189720333,
"fcm_dpo/beta": 0.004656361881643534,
"fcm_dpo/delta": -0.00973660871386528,
"fcm_dpo/margin": 59.05378341674805,
"fcm_dpo/q_t": 0.43434858322143555,
"grad_norm": 13.978921890258789,
"learning_rate": 2.0398616948569493e-07,
"logits/chosen": 0.7625016570091248,
"logits/rejected": 0.7079190611839294,
"logps/chosen": -242.85482788085938,
"logps/ref_chosen": -71.62649536132812,
"logps/ref_rejected": -90.98765563964844,
"logps/rejected": -321.269775390625,
"loss": 1.188,
"margin_dpo/margin_mean": 59.05378341674805,
"margin_dpo/margin_std": 100.4428939819336,
"step": 400
},
{
"KL/chosen_KL_mean": -128.79498291015625,
"KL/mean": -171.2772674560547,
"KL/rejected_KL_mean": -213.75953674316406,
"KL/std": 112.90093994140625,
"epoch": 0.6061980347694633,
"fcm_dpo/beta": 0.004625148139894009,
"fcm_dpo/delta": 0.007325233891606331,
"fcm_dpo/margin": 84.96454620361328,
"fcm_dpo/q_t": 0.4098255932331085,
"grad_norm": 7.507087230682373,
"learning_rate": 2.0268718890989752e-07,
"logits/chosen": 0.8112884759902954,
"logits/rejected": 0.7324414253234863,
"logps/chosen": -182.51995849609375,
"logps/ref_chosen": -53.72495651245117,
"logps/ref_rejected": -75.06304931640625,
"logps/rejected": -288.82257080078125,
"loss": 1.0941,
"margin_dpo/margin_mean": 84.96454620361328,
"margin_dpo/margin_std": 110.92296600341797,
"step": 401
},
{
"KL/chosen_KL_mean": -143.20652770996094,
"KL/mean": -174.81996154785156,
"KL/rejected_KL_mean": -206.43341064453125,
"KL/std": 94.28620910644531,
"epoch": 0.6077097505668935,
"fcm_dpo/beta": 0.004691399168223143,
"fcm_dpo/delta": 0.005168864969164133,
"fcm_dpo/margin": 63.226871490478516,
"fcm_dpo/q_t": 0.43117454648017883,
"grad_norm": 11.448514938354492,
"learning_rate": 2.013895317751323e-07,
"logits/chosen": 0.7634487152099609,
"logits/rejected": 0.7546102404594421,
"logps/chosen": -205.0804443359375,
"logps/ref_chosen": -61.873931884765625,
"logps/ref_rejected": -66.15198516845703,
"logps/rejected": -272.58538818359375,
"loss": 1.1843,
"margin_dpo/margin_mean": 63.22686767578125,
"margin_dpo/margin_std": 115.47822570800781,
"step": 402
},
{
"KL/chosen_KL_mean": -149.43191528320312,
"KL/mean": -191.5021209716797,
"KL/rejected_KL_mean": -233.57232666015625,
"KL/std": 111.39325714111328,
"epoch": 0.6092214663643235,
"fcm_dpo/beta": 0.004674067720770836,
"fcm_dpo/delta": 0.007327292114496231,
"fcm_dpo/margin": 84.14042663574219,
"fcm_dpo/q_t": 0.4105861485004425,
"grad_norm": 8.619983673095703,
"learning_rate": 2.0009323437965898e-07,
"logits/chosen": 0.8772906064987183,
"logits/rejected": 0.7967815399169922,
"logps/chosen": -200.75341796875,
"logps/ref_chosen": -51.321502685546875,
"logps/ref_rejected": -86.54010772705078,
"logps/rejected": -320.1124267578125,
"loss": 1.1015,
"margin_dpo/margin_mean": 84.14042663574219,
"margin_dpo/margin_std": 116.0869140625,
"step": 403
},
{
"KL/chosen_KL_mean": -143.56373596191406,
"KL/mean": -187.47364807128906,
"KL/rejected_KL_mean": -231.38356018066406,
"KL/std": 115.00575256347656,
"epoch": 0.6107331821617535,
"fcm_dpo/beta": 0.0046137175522744656,
"fcm_dpo/delta": -0.00824352353811264,
"fcm_dpo/margin": 87.81982421875,
"fcm_dpo/q_t": 0.40820419788360596,
"grad_norm": 12.866887092590332,
"learning_rate": 1.9879833298370237e-07,
"logits/chosen": 0.7568644285202026,
"logits/rejected": 0.6709632873535156,
"logps/chosen": -205.82662963867188,
"logps/ref_chosen": -62.26288604736328,
"logps/ref_rejected": -95.19029998779297,
"logps/rejected": -326.5738525390625,
"loss": 1.1079,
"margin_dpo/margin_mean": 87.81982421875,
"margin_dpo/margin_std": 128.9308624267578,
"step": 404
},
{
"KL/chosen_KL_mean": -144.8743896484375,
"KL/mean": -178.5138397216797,
"KL/rejected_KL_mean": -212.15328979492188,
"KL/std": 99.92108154296875,
"epoch": 0.6122448979591837,
"fcm_dpo/beta": 0.004627191927284002,
"fcm_dpo/delta": -0.022051792591810226,
"fcm_dpo/margin": 67.27889251708984,
"fcm_dpo/q_t": 0.4273110032081604,
"grad_norm": 9.795719146728516,
"learning_rate": 1.975048638084379e-07,
"logits/chosen": 0.8541995286941528,
"logits/rejected": 0.8144976496696472,
"logps/chosen": -195.458740234375,
"logps/ref_chosen": -50.5843391418457,
"logps/ref_rejected": -65.43156433105469,
"logps/rejected": -277.5848388671875,
"loss": 1.1557,
"margin_dpo/margin_mean": 67.27889251708984,
"margin_dpo/margin_std": 100.67938232421875,
"step": 405
},
{
"KL/chosen_KL_mean": -136.11376953125,
"KL/mean": -183.8330078125,
"KL/rejected_KL_mean": -231.55224609375,
"KL/std": 109.36830139160156,
"epoch": 0.6137566137566137,
"fcm_dpo/beta": 0.004584764130413532,
"fcm_dpo/delta": -0.04349678382277489,
"fcm_dpo/margin": 95.4384536743164,
"fcm_dpo/q_t": 0.398017942905426,
"grad_norm": 9.676344871520996,
"learning_rate": 1.9621286303497914e-07,
"logits/chosen": 0.8280668258666992,
"logits/rejected": 0.685680627822876,
"logps/chosen": -185.109375,
"logps/ref_chosen": -48.99560546875,
"logps/ref_rejected": -92.47774505615234,
"logps/rejected": -324.02996826171875,
"loss": 1.0703,
"margin_dpo/margin_mean": 95.43846130371094,
"margin_dpo/margin_std": 120.8671875,
"step": 406
},
{
"KL/chosen_KL_mean": -180.22271728515625,
"KL/mean": -216.6717529296875,
"KL/rejected_KL_mean": -253.1208038330078,
"KL/std": 127.8733139038086,
"epoch": 0.6152683295540439,
"fcm_dpo/beta": 0.004674847237765789,
"fcm_dpo/delta": 0.06505510210990906,
"fcm_dpo/margin": 72.8980941772461,
"fcm_dpo/q_t": 0.4223581552505493,
"grad_norm": 11.925191879272461,
"learning_rate": 1.9492236680336483e-07,
"logits/chosen": 0.6953166723251343,
"logits/rejected": 0.6430000066757202,
"logps/chosen": -269.623291015625,
"logps/ref_chosen": -89.40056610107422,
"logps/ref_rejected": -99.28775024414062,
"logps/rejected": -352.4085693359375,
"loss": 1.1603,
"margin_dpo/margin_mean": 72.89810180664062,
"margin_dpo/margin_std": 128.1045684814453,
"step": 407
},
{
"KL/chosen_KL_mean": -133.22731018066406,
"KL/mean": -183.61326599121094,
"KL/rejected_KL_mean": -233.99923706054688,
"KL/std": 101.46388244628906,
"epoch": 0.6167800453514739,
"fcm_dpo/beta": 0.004601156804710627,
"fcm_dpo/delta": -0.07428466528654099,
"fcm_dpo/margin": 100.77191162109375,
"fcm_dpo/q_t": 0.3899993896484375,
"grad_norm": 9.173323631286621,
"learning_rate": 1.9363341121154895e-07,
"logits/chosen": 0.7866462469100952,
"logits/rejected": 0.7229700088500977,
"logps/chosen": -187.9312286376953,
"logps/ref_chosen": -54.70391845703125,
"logps/ref_rejected": -73.98648834228516,
"logps/rejected": -307.9857177734375,
"loss": 1.0333,
"margin_dpo/margin_mean": 100.77191925048828,
"margin_dpo/margin_std": 104.93372344970703,
"step": 408
},
{
"KL/chosen_KL_mean": -165.34664916992188,
"KL/mean": -187.67367553710938,
"KL/rejected_KL_mean": -210.000732421875,
"KL/std": 87.25897216796875,
"epoch": 0.618291761148904,
"fcm_dpo/beta": 0.004743535537272692,
"fcm_dpo/delta": 0.09411386400461197,
"fcm_dpo/margin": 44.654083251953125,
"fcm_dpo/q_t": 0.45126640796661377,
"grad_norm": 11.334348678588867,
"learning_rate": 1.9234603231438994e-07,
"logits/chosen": 0.798064649105072,
"logits/rejected": 0.807654857635498,
"logps/chosen": -227.46487426757812,
"logps/ref_chosen": -62.11822509765625,
"logps/ref_rejected": -61.933509826660156,
"logps/rejected": -271.9342346191406,
"loss": 1.2371,
"margin_dpo/margin_mean": 44.654083251953125,
"margin_dpo/margin_std": 96.73648071289062,
"step": 409
},
{
"KL/chosen_KL_mean": -156.24102783203125,
"KL/mean": -196.73635864257812,
"KL/rejected_KL_mean": -237.231689453125,
"KL/std": 102.72321319580078,
"epoch": 0.6198034769463341,
"fcm_dpo/beta": 0.0046883877366781235,
"fcm_dpo/delta": 0.020214948803186417,
"fcm_dpo/margin": 80.99066162109375,
"fcm_dpo/q_t": 0.4119493365287781,
"grad_norm": 10.344640731811523,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 0.7926585674285889,
"logits/rejected": 0.7750309705734253,
"logps/chosen": -218.043701171875,
"logps/ref_chosen": -61.80266189575195,
"logps/ref_rejected": -76.60002136230469,
"logps/rejected": -313.83172607421875,
"loss": 1.1039,
"margin_dpo/margin_mean": 80.99066162109375,
"margin_dpo/margin_std": 103.93037414550781,
"step": 410
},
{
"KL/chosen_KL_mean": -162.68948364257812,
"KL/mean": -203.23043823242188,
"KL/rejected_KL_mean": -243.77142333984375,
"KL/std": 108.2699966430664,
"epoch": 0.6213151927437641,
"fcm_dpo/beta": 0.004831541329622269,
"fcm_dpo/delta": 0.009294688701629639,
"fcm_dpo/margin": 81.08193969726562,
"fcm_dpo/q_t": 0.4101915955543518,
"grad_norm": 9.14470386505127,
"learning_rate": 1.8977614860195296e-07,
"logits/chosen": 0.8241767287254333,
"logits/rejected": 0.779667317867279,
"logps/chosen": -217.13487243652344,
"logps/ref_chosen": -54.44539260864258,
"logps/ref_rejected": -74.5650863647461,
"logps/rejected": -318.3365173339844,
"loss": 1.1031,
"margin_dpo/margin_mean": 81.08193969726562,
"margin_dpo/margin_std": 114.11431884765625,
"step": 411
},
{
"KL/chosen_KL_mean": -166.61082458496094,
"KL/mean": -202.292236328125,
"KL/rejected_KL_mean": -237.9736328125,
"KL/std": 91.02214813232422,
"epoch": 0.6228269085411943,
"fcm_dpo/beta": 0.004908284172415733,
"fcm_dpo/delta": 0.05554809793829918,
"fcm_dpo/margin": 71.36279296875,
"fcm_dpo/q_t": 0.41911643743515015,
"grad_norm": 10.918268203735352,
"learning_rate": 1.8849371567184662e-07,
"logits/chosen": 0.8318126797676086,
"logits/rejected": 0.7821172475814819,
"logps/chosen": -221.85891723632812,
"logps/ref_chosen": -55.248085021972656,
"logps/ref_rejected": -68.96623229980469,
"logps/rejected": -306.9398498535156,
"loss": 1.1214,
"margin_dpo/margin_mean": 71.36279296875,
"margin_dpo/margin_std": 96.66009521484375,
"step": 412
},
{
"KL/chosen_KL_mean": -187.35760498046875,
"KL/mean": -221.8975830078125,
"KL/rejected_KL_mean": -256.4375305175781,
"KL/std": 111.54348754882812,
"epoch": 0.6243386243386243,
"fcm_dpo/beta": 0.00513922143727541,
"fcm_dpo/delta": 0.04962531104683876,
"fcm_dpo/margin": 69.0799560546875,
"fcm_dpo/q_t": 0.42187249660491943,
"grad_norm": 13.688892364501953,
"learning_rate": 1.872130032047302e-07,
"logits/chosen": 0.6731942296028137,
"logits/rejected": 0.6520799398422241,
"logps/chosen": -256.0783386230469,
"logps/ref_chosen": -68.72074890136719,
"logps/ref_rejected": -78.76539611816406,
"logps/rejected": -335.20294189453125,
"loss": 1.1735,
"margin_dpo/margin_mean": 69.0799560546875,
"margin_dpo/margin_std": 134.26339721679688,
"step": 413
},
{
"KL/chosen_KL_mean": -164.39401245117188,
"KL/mean": -205.43978881835938,
"KL/rejected_KL_mean": -246.48556518554688,
"KL/std": 119.57435607910156,
"epoch": 0.6258503401360545,
"fcm_dpo/beta": 0.005096019245684147,
"fcm_dpo/delta": -0.020970303565263748,
"fcm_dpo/margin": 82.09153747558594,
"fcm_dpo/q_t": 0.40284112095832825,
"grad_norm": 10.116288185119629,
"learning_rate": 1.8593404702488436e-07,
"logits/chosen": 0.8319721221923828,
"logits/rejected": 0.7853256464004517,
"logps/chosen": -218.5322265625,
"logps/ref_chosen": -54.138214111328125,
"logps/ref_rejected": -74.65741729736328,
"logps/rejected": -321.1429748535156,
"loss": 1.0866,
"margin_dpo/margin_mean": 82.09153747558594,
"margin_dpo/margin_std": 109.47962951660156,
"step": 414
},
{
"KL/chosen_KL_mean": -164.3896484375,
"KL/mean": -195.80303955078125,
"KL/rejected_KL_mean": -227.21646118164062,
"KL/std": 107.44061279296875,
"epoch": 0.6273620559334845,
"fcm_dpo/beta": 0.0052430275827646255,
"fcm_dpo/delta": 0.07850854098796844,
"fcm_dpo/margin": 62.826805114746094,
"fcm_dpo/q_t": 0.4264346957206726,
"grad_norm": 11.805853843688965,
"learning_rate": 1.846568829074628e-07,
"logits/chosen": 0.812616765499115,
"logits/rejected": 0.801384449005127,
"logps/chosen": -220.30819702148438,
"logps/ref_chosen": -55.91856002807617,
"logps/ref_rejected": -61.747703552246094,
"logps/rejected": -288.96417236328125,
"loss": 1.1706,
"margin_dpo/margin_mean": 62.826805114746094,
"margin_dpo/margin_std": 115.88273620605469,
"step": 415
},
{
"KL/chosen_KL_mean": -174.27407836914062,
"KL/mean": -204.4618682861328,
"KL/rejected_KL_mean": -234.649658203125,
"KL/std": 114.86009216308594,
"epoch": 0.6288737717309146,
"fcm_dpo/beta": 0.005179405678063631,
"fcm_dpo/delta": -0.09270616620779037,
"fcm_dpo/margin": 60.37559509277344,
"fcm_dpo/q_t": 0.4285816550254822,
"grad_norm": 12.966651916503906,
"learning_rate": 1.8338154657749128e-07,
"logits/chosen": 0.8164198398590088,
"logits/rejected": 0.7761438488960266,
"logps/chosen": -228.99716186523438,
"logps/ref_chosen": -54.72308349609375,
"logps/ref_rejected": -69.17388916015625,
"logps/rejected": -303.82354736328125,
"loss": 1.1915,
"margin_dpo/margin_mean": 60.37559509277344,
"margin_dpo/margin_std": 108.85138702392578,
"step": 416
},
{
"KL/chosen_KL_mean": -181.67933654785156,
"KL/mean": -218.7706298828125,
"KL/rejected_KL_mean": -255.86190795898438,
"KL/std": 106.55653381347656,
"epoch": 0.6303854875283447,
"fcm_dpo/beta": 0.005155562423169613,
"fcm_dpo/delta": 0.017802808433771133,
"fcm_dpo/margin": 74.18257904052734,
"fcm_dpo/q_t": 0.4107144773006439,
"grad_norm": 12.372243881225586,
"learning_rate": 1.8210807370886849e-07,
"logits/chosen": 0.9078543782234192,
"logits/rejected": 0.8568928837776184,
"logps/chosen": -238.47059631347656,
"logps/ref_chosen": -56.791259765625,
"logps/ref_rejected": -68.7791748046875,
"logps/rejected": -324.64111328125,
"loss": 1.1455,
"margin_dpo/margin_mean": 74.18257904052734,
"margin_dpo/margin_std": 129.5694122314453,
"step": 417
},
{
"KL/chosen_KL_mean": -196.171875,
"KL/mean": -226.3531036376953,
"KL/rejected_KL_mean": -256.53436279296875,
"KL/std": 122.66242980957031,
"epoch": 0.6318972033257747,
"fcm_dpo/beta": 0.004903099965304136,
"fcm_dpo/delta": -0.06217623129487038,
"fcm_dpo/margin": 60.362483978271484,
"fcm_dpo/q_t": 0.4314631521701813,
"grad_norm": 12.916627883911133,
"learning_rate": 1.8083649992336825e-07,
"logits/chosen": 0.8412420153617859,
"logits/rejected": 0.8448699712753296,
"logps/chosen": -265.27984619140625,
"logps/ref_chosen": -69.10798645019531,
"logps/ref_rejected": -75.09132385253906,
"logps/rejected": -331.62567138671875,
"loss": 1.2087,
"margin_dpo/margin_mean": 60.362483978271484,
"margin_dpo/margin_std": 124.63722229003906,
"step": 418
},
{
"KL/chosen_KL_mean": -157.86094665527344,
"KL/mean": -203.56805419921875,
"KL/rejected_KL_mean": -249.27517700195312,
"KL/std": 108.47840881347656,
"epoch": 0.6334089191232048,
"fcm_dpo/beta": 0.004795808810740709,
"fcm_dpo/delta": -0.04438748210668564,
"fcm_dpo/margin": 91.41424560546875,
"fcm_dpo/q_t": 0.40094202756881714,
"grad_norm": 11.021547317504883,
"learning_rate": 1.7956686078964255e-07,
"logits/chosen": 0.7512973546981812,
"logits/rejected": 0.717224657535553,
"logps/chosen": -216.03271484375,
"logps/ref_chosen": -58.1717643737793,
"logps/ref_rejected": -71.67066955566406,
"logps/rejected": -320.94586181640625,
"loss": 1.0782,
"margin_dpo/margin_mean": 91.41425323486328,
"margin_dpo/margin_std": 126.16435241699219,
"step": 419
},
{
"KL/chosen_KL_mean": -189.92071533203125,
"KL/mean": -213.03028869628906,
"KL/rejected_KL_mean": -236.13986206054688,
"KL/std": 111.73857116699219,
"epoch": 0.6349206349206349,
"fcm_dpo/beta": 0.004948785062879324,
"fcm_dpo/delta": 0.05984485149383545,
"fcm_dpo/margin": 46.219139099121094,
"fcm_dpo/q_t": 0.4502519965171814,
"grad_norm": 11.667963981628418,
"learning_rate": 1.782991918222275e-07,
"logits/chosen": 0.8240588903427124,
"logits/rejected": 0.7886266708374023,
"logps/chosen": -246.97422790527344,
"logps/ref_chosen": -57.05351257324219,
"logps/ref_rejected": -62.670982360839844,
"logps/rejected": -298.81085205078125,
"loss": 1.2697,
"margin_dpo/margin_mean": 46.219139099121094,
"margin_dpo/margin_std": 130.39532470703125,
"step": 420
},
{
"KL/chosen_KL_mean": -178.27980041503906,
"KL/mean": -212.07435607910156,
"KL/rejected_KL_mean": -245.86886596679688,
"KL/std": 111.47808837890625,
"epoch": 0.636432350718065,
"fcm_dpo/beta": 0.004891910124570131,
"fcm_dpo/delta": -0.01926536299288273,
"fcm_dpo/margin": 67.58908081054688,
"fcm_dpo/q_t": 0.42542341351509094,
"grad_norm": 12.780967712402344,
"learning_rate": 1.7703352848054887e-07,
"logits/chosen": 0.7950316071510315,
"logits/rejected": 0.7450087070465088,
"logps/chosen": -235.60305786132812,
"logps/ref_chosen": -57.32324981689453,
"logps/ref_rejected": -75.33782958984375,
"logps/rejected": -321.20672607421875,
"loss": 1.2105,
"margin_dpo/margin_mean": 67.5890884399414,
"margin_dpo/margin_std": 150.59071350097656,
"step": 421
},
{
"KL/chosen_KL_mean": -158.81130981445312,
"KL/mean": -200.82315063476562,
"KL/rejected_KL_mean": -242.8350067138672,
"KL/std": 111.04067993164062,
"epoch": 0.6379440665154951,
"fcm_dpo/beta": 0.004911776166409254,
"fcm_dpo/delta": -0.015146823599934578,
"fcm_dpo/margin": 84.02371215820312,
"fcm_dpo/q_t": 0.40485405921936035,
"grad_norm": 12.647407531738281,
"learning_rate": 1.7576990616793137e-07,
"logits/chosen": 0.7745881080627441,
"logits/rejected": 0.775098443031311,
"logps/chosen": -225.86888122558594,
"logps/ref_chosen": -67.05757141113281,
"logps/ref_rejected": -72.12803649902344,
"logps/rejected": -314.9630432128906,
"loss": 1.0853,
"margin_dpo/margin_mean": 84.0237045288086,
"margin_dpo/margin_std": 109.76282501220703,
"step": 422
},
{
"KL/chosen_KL_mean": -155.72247314453125,
"KL/mean": -203.5447998046875,
"KL/rejected_KL_mean": -251.36709594726562,
"KL/std": 113.12423706054688,
"epoch": 0.6394557823129252,
"fcm_dpo/beta": 0.004685261286795139,
"fcm_dpo/delta": -0.05671604722738266,
"fcm_dpo/margin": 95.64463806152344,
"fcm_dpo/q_t": 0.39703303575515747,
"grad_norm": 11.059937477111816,
"learning_rate": 1.745083602306071e-07,
"logits/chosen": 0.8790537118911743,
"logits/rejected": 0.8249520063400269,
"logps/chosen": -209.78414916992188,
"logps/ref_chosen": -54.06167221069336,
"logps/ref_rejected": -76.64092254638672,
"logps/rejected": -328.0080261230469,
"loss": 1.0619,
"margin_dpo/margin_mean": 95.6446304321289,
"margin_dpo/margin_std": 121.1661605834961,
"step": 423
},
{
"KL/chosen_KL_mean": -170.83416748046875,
"KL/mean": -215.72669982910156,
"KL/rejected_KL_mean": -260.6192321777344,
"KL/std": 103.92689514160156,
"epoch": 0.6409674981103552,
"fcm_dpo/beta": 0.004581287503242493,
"fcm_dpo/delta": -0.014987409114837646,
"fcm_dpo/margin": 89.7850570678711,
"fcm_dpo/q_t": 0.40572795271873474,
"grad_norm": 14.250228881835938,
"learning_rate": 1.7324892595672804e-07,
"logits/chosen": 0.7274860143661499,
"logits/rejected": 0.6937150359153748,
"logps/chosen": -224.4430389404297,
"logps/ref_chosen": -53.60887145996094,
"logps/ref_rejected": -79.2139892578125,
"logps/rejected": -339.8332214355469,
"loss": 1.0895,
"margin_dpo/margin_mean": 89.7850570678711,
"margin_dpo/margin_std": 120.62132263183594,
"step": 424
},
{
"KL/chosen_KL_mean": -167.8055419921875,
"KL/mean": -202.077392578125,
"KL/rejected_KL_mean": -236.3492431640625,
"KL/std": 100.29512023925781,
"epoch": 0.6424792139077853,
"fcm_dpo/beta": 0.004820922389626503,
"fcm_dpo/delta": 0.07685823738574982,
"fcm_dpo/margin": 68.54373168945312,
"fcm_dpo/q_t": 0.4257993698120117,
"grad_norm": 11.25399398803711,
"learning_rate": 1.7199163857537824e-07,
"logits/chosen": 0.8642111420631409,
"logits/rejected": 0.843815803527832,
"logps/chosen": -226.22021484375,
"logps/ref_chosen": -58.41468048095703,
"logps/ref_rejected": -66.59054565429688,
"logps/rejected": -302.9397888183594,
"loss": 1.157,
"margin_dpo/margin_mean": 68.54371643066406,
"margin_dpo/margin_std": 115.27527618408203,
"step": 425
},
{
"KL/chosen_KL_mean": -201.40029907226562,
"KL/mean": -221.83224487304688,
"KL/rejected_KL_mean": -242.26416015625,
"KL/std": 111.92045593261719,
"epoch": 0.6439909297052154,
"fcm_dpo/beta": 0.004862668924033642,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 40.86387634277344,
"fcm_dpo/q_t": 0.4547138810157776,
"grad_norm": 15.746712684631348,
"learning_rate": 1.7073653325558828e-07,
"logits/chosen": 0.7816729545593262,
"logits/rejected": 0.7827064990997314,
"logps/chosen": -273.1085205078125,
"logps/ref_chosen": -71.70822143554688,
"logps/ref_rejected": -73.57725524902344,
"logps/rejected": -315.8414306640625,
"loss": 1.3073,
"margin_dpo/margin_mean": 40.86387634277344,
"margin_dpo/margin_std": 142.01010131835938,
"step": 426
},
{
"KL/chosen_KL_mean": -189.36972045898438,
"KL/mean": -227.52943420410156,
"KL/rejected_KL_mean": -265.6891784667969,
"KL/std": 122.76222229003906,
"epoch": 0.6455026455026455,
"fcm_dpo/beta": 0.004909333307296038,
"fcm_dpo/delta": 0.028550054877996445,
"fcm_dpo/margin": 76.31942749023438,
"fcm_dpo/q_t": 0.417738676071167,
"grad_norm": 12.455328941345215,
"learning_rate": 1.6948364510535218e-07,
"logits/chosen": 0.8194973468780518,
"logits/rejected": 0.7630200982093811,
"logps/chosen": -248.0124969482422,
"logps/ref_chosen": -58.64276885986328,
"logps/ref_rejected": -86.25437927246094,
"logps/rejected": -351.94354248046875,
"loss": 1.1438,
"margin_dpo/margin_mean": 76.31942749023438,
"margin_dpo/margin_std": 133.41810607910156,
"step": 427
},
{
"KL/chosen_KL_mean": -184.95486450195312,
"KL/mean": -222.78594970703125,
"KL/rejected_KL_mean": -260.6170349121094,
"KL/std": 124.32109832763672,
"epoch": 0.6470143613000756,
"fcm_dpo/beta": 0.005052429623901844,
"fcm_dpo/delta": 0.018595866858959198,
"fcm_dpo/margin": 75.66215515136719,
"fcm_dpo/q_t": 0.4149612486362457,
"grad_norm": 11.38692569732666,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 0.8025504946708679,
"logits/rejected": 0.7721965909004211,
"logps/chosen": -251.55091857910156,
"logps/ref_chosen": -66.5960464477539,
"logps/ref_rejected": -82.3941650390625,
"logps/rejected": -343.01116943359375,
"loss": 1.1482,
"margin_dpo/margin_mean": 75.66214752197266,
"margin_dpo/margin_std": 133.01632690429688,
"step": 428
},
{
"KL/chosen_KL_mean": -187.84649658203125,
"KL/mean": -216.41619873046875,
"KL/rejected_KL_mean": -244.9859161376953,
"KL/std": 100.31537628173828,
"epoch": 0.6485260770975056,
"fcm_dpo/beta": 0.005307496525347233,
"fcm_dpo/delta": 0.10380184650421143,
"fcm_dpo/margin": 57.139427185058594,
"fcm_dpo/q_t": 0.43025392293930054,
"grad_norm": 12.65349006652832,
"learning_rate": 1.669846604344412e-07,
"logits/chosen": 0.7885690927505493,
"logits/rejected": 0.8087594509124756,
"logps/chosen": -244.856201171875,
"logps/ref_chosen": -57.00970458984375,
"logps/ref_rejected": -59.86549377441406,
"logps/rejected": -304.8514099121094,
"loss": 1.1977,
"margin_dpo/margin_mean": 57.139427185058594,
"margin_dpo/margin_std": 115.14334106445312,
"step": 429
},
{
"KL/chosen_KL_mean": -165.86953735351562,
"KL/mean": -214.21212768554688,
"KL/rejected_KL_mean": -262.5546875,
"KL/std": 101.71453094482422,
"epoch": 0.6500377928949358,
"fcm_dpo/beta": 0.005062808282673359,
"fcm_dpo/delta": -0.1054583340883255,
"fcm_dpo/margin": 96.68514251708984,
"fcm_dpo/q_t": 0.3835219144821167,
"grad_norm": 13.401054382324219,
"learning_rate": 1.6573863381573954e-07,
"logits/chosen": 0.7247724533081055,
"logits/rejected": 0.7243069410324097,
"logps/chosen": -225.4327392578125,
"logps/ref_chosen": -59.563194274902344,
"logps/ref_rejected": -70.52289581298828,
"logps/rejected": -333.0776062011719,
"loss": 1.0434,
"margin_dpo/margin_mean": 96.68515014648438,
"margin_dpo/margin_std": 119.87529754638672,
"step": 430
},
{
"KL/chosen_KL_mean": -164.5930633544922,
"KL/mean": -201.393310546875,
"KL/rejected_KL_mean": -238.1935577392578,
"KL/std": 106.98062896728516,
"epoch": 0.6515495086923658,
"fcm_dpo/beta": 0.0050440579652786255,
"fcm_dpo/delta": 0.032343633472919464,
"fcm_dpo/margin": 73.6004867553711,
"fcm_dpo/q_t": 0.41668662428855896,
"grad_norm": 12.450831413269043,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 0.790619969367981,
"logits/rejected": 0.7420604825019836,
"logps/chosen": -214.79339599609375,
"logps/ref_chosen": -50.20032501220703,
"logps/ref_rejected": -77.81680297851562,
"logps/rejected": -316.0103759765625,
"loss": 1.1385,
"margin_dpo/margin_mean": 73.60047912597656,
"margin_dpo/margin_std": 121.77957916259766,
"step": 431
},
{
"KL/chosen_KL_mean": -174.41119384765625,
"KL/mean": -212.10067749023438,
"KL/rejected_KL_mean": -249.7901611328125,
"KL/std": 106.63932037353516,
"epoch": 0.6530612244897959,
"fcm_dpo/beta": 0.005112434271723032,
"fcm_dpo/delta": 0.016498606652021408,
"fcm_dpo/margin": 75.37895202636719,
"fcm_dpo/q_t": 0.4119753837585449,
"grad_norm": 13.068375587463379,
"learning_rate": 1.632536862810844e-07,
"logits/chosen": 0.8498221039772034,
"logits/rejected": 0.8017172813415527,
"logps/chosen": -236.07394409179688,
"logps/ref_chosen": -61.662757873535156,
"logps/ref_rejected": -83.94496154785156,
"logps/rejected": -333.735107421875,
"loss": 1.1373,
"margin_dpo/margin_mean": 75.37895202636719,
"margin_dpo/margin_std": 127.255615234375,
"step": 432
},
{
"KL/chosen_KL_mean": -171.37161254882812,
"KL/mean": -213.88760375976562,
"KL/rejected_KL_mean": -256.4035949707031,
"KL/std": 99.43716430664062,
"epoch": 0.654572940287226,
"fcm_dpo/beta": 0.00509435310959816,
"fcm_dpo/delta": -0.03820203244686127,
"fcm_dpo/margin": 85.03195190429688,
"fcm_dpo/q_t": 0.3992968797683716,
"grad_norm": 13.277294158935547,
"learning_rate": 1.6201483487445515e-07,
"logits/chosen": 0.8910690546035767,
"logits/rejected": 0.8961876034736633,
"logps/chosen": -235.10079956054688,
"logps/ref_chosen": -63.72917938232422,
"logps/ref_rejected": -65.8391342163086,
"logps/rejected": -322.24273681640625,
"loss": 1.079,
"margin_dpo/margin_mean": 85.03195190429688,
"margin_dpo/margin_std": 113.61282348632812,
"step": 433
},
{
"KL/chosen_KL_mean": -146.40029907226562,
"KL/mean": -194.85731506347656,
"KL/rejected_KL_mean": -243.3143310546875,
"KL/std": 110.77456665039062,
"epoch": 0.656084656084656,
"fcm_dpo/beta": 0.004744514357298613,
"fcm_dpo/delta": -0.07580451667308807,
"fcm_dpo/margin": 96.91404724121094,
"fcm_dpo/q_t": 0.3922083377838135,
"grad_norm": 11.460370063781738,
"learning_rate": 1.6077844460203204e-07,
"logits/chosen": 0.9136182069778442,
"logits/rejected": 0.8583601713180542,
"logps/chosen": -194.3736114501953,
"logps/ref_chosen": -47.97331619262695,
"logps/ref_rejected": -72.51132202148438,
"logps/rejected": -315.8256530761719,
"loss": 1.0742,
"margin_dpo/margin_mean": 96.91405487060547,
"margin_dpo/margin_std": 131.41989135742188,
"step": 434
},
{
"KL/chosen_KL_mean": -177.25384521484375,
"KL/mean": -212.98779296875,
"KL/rejected_KL_mean": -248.72177124023438,
"KL/std": 106.70513916015625,
"epoch": 0.6575963718820862,
"fcm_dpo/beta": 0.004973133094608784,
"fcm_dpo/delta": 0.04807582497596741,
"fcm_dpo/margin": 71.46792602539062,
"fcm_dpo/q_t": 0.4196769893169403,
"grad_norm": 13.298686027526855,
"learning_rate": 1.5954455004830878e-07,
"logits/chosen": 0.9066946506500244,
"logits/rejected": 0.8741989135742188,
"logps/chosen": -234.3140869140625,
"logps/ref_chosen": -57.06024932861328,
"logps/ref_rejected": -71.69146728515625,
"logps/rejected": -320.4132385253906,
"loss": 1.1529,
"margin_dpo/margin_mean": 71.46792602539062,
"margin_dpo/margin_std": 123.43423461914062,
"step": 435
},
{
"KL/chosen_KL_mean": -178.68710327148438,
"KL/mean": -208.48751831054688,
"KL/rejected_KL_mean": -238.28793334960938,
"KL/std": 103.73971557617188,
"epoch": 0.6591080876795162,
"fcm_dpo/beta": 0.005142658948898315,
"fcm_dpo/delta": 0.1033397912979126,
"fcm_dpo/margin": 59.600830078125,
"fcm_dpo/q_t": 0.4327392578125,
"grad_norm": 13.91799545288086,
"learning_rate": 1.5831318572796847e-07,
"logits/chosen": 0.8306369781494141,
"logits/rejected": 0.7831582427024841,
"logps/chosen": -234.84515380859375,
"logps/ref_chosen": -56.158050537109375,
"logps/ref_rejected": -67.63787841796875,
"logps/rejected": -305.9258117675781,
"loss": 1.206,
"margin_dpo/margin_mean": 59.600830078125,
"margin_dpo/margin_std": 127.89569091796875,
"step": 436
},
{
"KL/chosen_KL_mean": -183.9375457763672,
"KL/mean": -221.9730224609375,
"KL/rejected_KL_mean": -260.00848388671875,
"KL/std": 115.573486328125,
"epoch": 0.6606198034769464,
"fcm_dpo/beta": 0.005009764805436134,
"fcm_dpo/delta": -0.082071453332901,
"fcm_dpo/margin": 76.07093811035156,
"fcm_dpo/q_t": 0.41367313265800476,
"grad_norm": 15.81583023071289,
"learning_rate": 1.5708438608491815e-07,
"logits/chosen": 0.8169673085212708,
"logits/rejected": 0.7099220752716064,
"logps/chosen": -240.92333984375,
"logps/ref_chosen": -56.98578643798828,
"logps/ref_rejected": -85.61524963378906,
"logps/rejected": -345.62371826171875,
"loss": 1.1729,
"margin_dpo/margin_mean": 76.07093811035156,
"margin_dpo/margin_std": 148.16189575195312,
"step": 437
},
{
"KL/chosen_KL_mean": -159.0219268798828,
"KL/mean": -206.44488525390625,
"KL/rejected_KL_mean": -253.86782836914062,
"KL/std": 118.10974884033203,
"epoch": 0.6621315192743764,
"fcm_dpo/beta": 0.004896242171525955,
"fcm_dpo/delta": -0.07475844770669937,
"fcm_dpo/margin": 94.84590148925781,
"fcm_dpo/q_t": 0.3970157504081726,
"grad_norm": 12.760135650634766,
"learning_rate": 1.558581854913253e-07,
"logits/chosen": 0.8870106935501099,
"logits/rejected": 0.8304316997528076,
"logps/chosen": -200.29971313476562,
"logps/ref_chosen": -41.27777862548828,
"logps/ref_rejected": -65.33840942382812,
"logps/rejected": -319.20623779296875,
"loss": 1.0676,
"margin_dpo/margin_mean": 94.84590148925781,
"margin_dpo/margin_std": 133.10060119628906,
"step": 438
},
{
"KL/chosen_KL_mean": -186.03314208984375,
"KL/mean": -229.74700927734375,
"KL/rejected_KL_mean": -273.4609069824219,
"KL/std": 121.18110656738281,
"epoch": 0.6636432350718064,
"fcm_dpo/beta": 0.004845023155212402,
"fcm_dpo/delta": -0.03054000437259674,
"fcm_dpo/margin": 87.42778015136719,
"fcm_dpo/q_t": 0.40231576561927795,
"grad_norm": 11.937276840209961,
"learning_rate": 1.5463461824665658e-07,
"logits/chosen": 0.726641833782196,
"logits/rejected": 0.7027250528335571,
"logps/chosen": -267.4508056640625,
"logps/ref_chosen": -81.41764831542969,
"logps/ref_rejected": -94.72309875488281,
"logps/rejected": -368.1839904785156,
"loss": 1.0944,
"margin_dpo/margin_mean": 87.42778015136719,
"margin_dpo/margin_std": 121.04769897460938,
"step": 439
},
{
"KL/chosen_KL_mean": -155.7847900390625,
"KL/mean": -199.2237548828125,
"KL/rejected_KL_mean": -242.66273498535156,
"KL/std": 106.59927368164062,
"epoch": 0.6651549508692366,
"fcm_dpo/beta": 0.00472570164129138,
"fcm_dpo/delta": -0.012669751420617104,
"fcm_dpo/margin": 86.87796020507812,
"fcm_dpo/q_t": 0.40606674551963806,
"grad_norm": 15.137947082519531,
"learning_rate": 1.534137185767178e-07,
"logits/chosen": 0.8190924525260925,
"logits/rejected": 0.7351869344711304,
"logps/chosen": -198.32296752929688,
"logps/ref_chosen": -42.538185119628906,
"logps/ref_rejected": -69.78813934326172,
"logps/rejected": -312.45086669921875,
"loss": 1.0963,
"margin_dpo/margin_mean": 86.87796783447266,
"margin_dpo/margin_std": 120.90702819824219,
"step": 440
},
{
"KL/chosen_KL_mean": -163.71417236328125,
"KL/mean": -209.40249633789062,
"KL/rejected_KL_mean": -255.09083557128906,
"KL/std": 117.68836212158203,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.004568706266582012,
"fcm_dpo/delta": -0.021269341930747032,
"fcm_dpo/margin": 91.37667083740234,
"fcm_dpo/q_t": 0.4020621180534363,
"grad_norm": 12.57343864440918,
"learning_rate": 1.521955206326976e-07,
"logits/chosen": 0.7851335406303406,
"logits/rejected": 0.7083240747451782,
"logps/chosen": -221.30738830566406,
"logps/ref_chosen": -57.593223571777344,
"logps/ref_rejected": -84.82878875732422,
"logps/rejected": -339.91961669921875,
"loss": 1.0718,
"margin_dpo/margin_mean": 91.37667083740234,
"margin_dpo/margin_std": 108.32664489746094,
"step": 441
},
{
"KL/chosen_KL_mean": -194.39462280273438,
"KL/mean": -241.25390625,
"KL/rejected_KL_mean": -288.11322021484375,
"KL/std": 117.97537231445312,
"epoch": 0.6681783824640968,
"fcm_dpo/beta": 0.004501115530729294,
"fcm_dpo/delta": -0.026340365409851074,
"fcm_dpo/margin": 93.71859741210938,
"fcm_dpo/q_t": 0.40267208218574524,
"grad_norm": 14.261401176452637,
"learning_rate": 1.5098005849021078e-07,
"logits/chosen": 0.790300726890564,
"logits/rejected": 0.7574805021286011,
"logps/chosen": -261.8558349609375,
"logps/ref_chosen": -67.46121978759766,
"logps/ref_rejected": -89.0693588256836,
"logps/rejected": -377.18255615234375,
"loss": 1.0786,
"margin_dpo/margin_mean": 93.7186050415039,
"margin_dpo/margin_std": 120.49788665771484,
"step": 442
},
{
"KL/chosen_KL_mean": -160.43263244628906,
"KL/mean": -216.71011352539062,
"KL/rejected_KL_mean": -272.98760986328125,
"KL/std": 124.23033905029297,
"epoch": 0.6696900982615268,
"fcm_dpo/beta": 0.004319561645388603,
"fcm_dpo/delta": -0.10150419175624847,
"fcm_dpo/margin": 112.55494689941406,
"fcm_dpo/q_t": 0.3878478407859802,
"grad_norm": 11.641592025756836,
"learning_rate": 1.4976736614834662e-07,
"logits/chosen": 0.8116977214813232,
"logits/rejected": 0.7602815628051758,
"logps/chosen": -215.22872924804688,
"logps/ref_chosen": -54.79610061645508,
"logps/ref_rejected": -77.80781555175781,
"logps/rejected": -350.79541015625,
"loss": 1.0382,
"margin_dpo/margin_mean": 112.5549545288086,
"margin_dpo/margin_std": 139.6777801513672,
"step": 443
},
{
"KL/chosen_KL_mean": -191.93292236328125,
"KL/mean": -221.1522674560547,
"KL/rejected_KL_mean": -250.37164306640625,
"KL/std": 127.5732421875,
"epoch": 0.671201814058957,
"fcm_dpo/beta": 0.00436423160135746,
"fcm_dpo/delta": 0.03717103973031044,
"fcm_dpo/margin": 58.438720703125,
"fcm_dpo/q_t": 0.4444841146469116,
"grad_norm": 12.10918140411377,
"learning_rate": 1.4855747752871654e-07,
"logits/chosen": 0.8365426063537598,
"logits/rejected": 0.7632496356964111,
"logps/chosen": -250.68197631835938,
"logps/ref_chosen": -58.749061584472656,
"logps/ref_rejected": -86.87396240234375,
"logps/rejected": -337.24560546875,
"loss": 1.2394,
"margin_dpo/margin_mean": 58.438720703125,
"margin_dpo/margin_std": 142.1253662109375,
"step": 444
},
{
"KL/chosen_KL_mean": -183.2960968017578,
"KL/mean": -231.47491455078125,
"KL/rejected_KL_mean": -279.6537170410156,
"KL/std": 110.73534393310547,
"epoch": 0.672713529856387,
"fcm_dpo/beta": 0.004318728111684322,
"fcm_dpo/delta": -0.01852697879076004,
"fcm_dpo/margin": 96.35762023925781,
"fcm_dpo/q_t": 0.4018581509590149,
"grad_norm": 10.044200897216797,
"learning_rate": 1.473504264745062e-07,
"logits/chosen": 0.7895313501358032,
"logits/rejected": 0.7898776531219482,
"logps/chosen": -244.21353149414062,
"logps/ref_chosen": -60.91743850708008,
"logps/ref_rejected": -71.5637435913086,
"logps/rejected": -351.21746826171875,
"loss": 1.0725,
"margin_dpo/margin_mean": 96.35762023925781,
"margin_dpo/margin_std": 115.74655151367188,
"step": 445
},
{
"KL/chosen_KL_mean": -164.505126953125,
"KL/mean": -212.84637451171875,
"KL/rejected_KL_mean": -261.1876525878906,
"KL/std": 113.10897827148438,
"epoch": 0.674225245653817,
"fcm_dpo/beta": 0.004369608126580715,
"fcm_dpo/delta": -0.0294049009680748,
"fcm_dpo/margin": 96.6825180053711,
"fcm_dpo/q_t": 0.4011649489402771,
"grad_norm": 11.11204719543457,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 0.8308959007263184,
"logits/rejected": 0.7669998407363892,
"logps/chosen": -213.3043670654297,
"logps/ref_chosen": -48.79924774169922,
"logps/ref_rejected": -71.8719482421875,
"logps/rejected": -333.0596008300781,
"loss": 1.0718,
"margin_dpo/margin_mean": 96.6825180053711,
"margin_dpo/margin_std": 111.4892349243164,
"step": 446
},
{
"KL/chosen_KL_mean": -160.591796875,
"KL/mean": -217.41326904296875,
"KL/rejected_KL_mean": -274.2347412109375,
"KL/std": 113.17485046386719,
"epoch": 0.6757369614512472,
"fcm_dpo/beta": 0.004061791114509106,
"fcm_dpo/delta": -0.07315800338983536,
"fcm_dpo/margin": 113.64291381835938,
"fcm_dpo/q_t": 0.3898429274559021,
"grad_norm": 9.85269546508789,
"learning_rate": 1.4494497203727843e-07,
"logits/chosen": 0.8119666576385498,
"logits/rejected": 0.7234156131744385,
"logps/chosen": -214.27452087402344,
"logps/ref_chosen": -53.682716369628906,
"logps/ref_rejected": -88.17315673828125,
"logps/rejected": -362.40789794921875,
"loss": 1.0415,
"margin_dpo/margin_mean": 113.6429214477539,
"margin_dpo/margin_std": 126.60255432128906,
"step": 447
},
{
"KL/chosen_KL_mean": -177.05801391601562,
"KL/mean": -221.44229125976562,
"KL/rejected_KL_mean": -265.82659912109375,
"KL/std": 117.35690307617188,
"epoch": 0.6772486772486772,
"fcm_dpo/beta": 0.0041312845423817635,
"fcm_dpo/delta": 0.03732322156429291,
"fcm_dpo/margin": 88.76856231689453,
"fcm_dpo/q_t": 0.41569164395332336,
"grad_norm": 8.617594718933105,
"learning_rate": 1.4374663593999256e-07,
"logits/chosen": 0.8559887409210205,
"logits/rejected": 0.8163323402404785,
"logps/chosen": -230.80926513671875,
"logps/ref_chosen": -53.75125503540039,
"logps/ref_rejected": -77.17623901367188,
"logps/rejected": -343.0028076171875,
"loss": 1.1175,
"margin_dpo/margin_mean": 88.76856231689453,
"margin_dpo/margin_std": 127.51617431640625,
"step": 448
},
{
"KL/chosen_KL_mean": -211.01304626464844,
"KL/mean": -233.7032470703125,
"KL/rejected_KL_mean": -256.3934326171875,
"KL/std": 123.69744873046875,
"epoch": 0.6787603930461074,
"fcm_dpo/beta": 0.004320129752159119,
"fcm_dpo/delta": 0.06683868914842606,
"fcm_dpo/margin": 45.38038635253906,
"fcm_dpo/q_t": 0.45501431822776794,
"grad_norm": 17.40116310119629,
"learning_rate": 1.4255127197770707e-07,
"logits/chosen": 0.6914898157119751,
"logits/rejected": 0.6926765441894531,
"logps/chosen": -286.8404235839844,
"logps/ref_chosen": -75.82737731933594,
"logps/ref_rejected": -82.20687866210938,
"logps/rejected": -338.6003112792969,
"loss": 1.2687,
"margin_dpo/margin_mean": 45.38037872314453,
"margin_dpo/margin_std": 122.3283920288086,
"step": 449
},
{
"KL/chosen_KL_mean": -170.02328491210938,
"KL/mean": -208.6387176513672,
"KL/rejected_KL_mean": -247.25418090820312,
"KL/std": 121.925537109375,
"epoch": 0.6802721088435374,
"fcm_dpo/beta": 0.0044110435992479324,
"fcm_dpo/delta": 0.06613278388977051,
"fcm_dpo/margin": 77.23091888427734,
"fcm_dpo/q_t": 0.4259797930717468,
"grad_norm": 12.112007141113281,
"learning_rate": 1.4135891358732205e-07,
"logits/chosen": 0.9353618621826172,
"logits/rejected": 0.8390960097312927,
"logps/chosen": -217.1389923095703,
"logps/ref_chosen": -47.11572265625,
"logps/ref_rejected": -78.7546615600586,
"logps/rejected": -326.00885009765625,
"loss": 1.1643,
"margin_dpo/margin_mean": 77.23091888427734,
"margin_dpo/margin_std": 139.64755249023438,
"step": 450
},
{
"KL/chosen_KL_mean": -172.9991912841797,
"KL/mean": -207.0963134765625,
"KL/rejected_KL_mean": -241.19345092773438,
"KL/std": 114.673095703125,
"epoch": 0.6817838246409675,
"fcm_dpo/beta": 0.004698569420725107,
"fcm_dpo/delta": 0.08733482658863068,
"fcm_dpo/margin": 68.19424438476562,
"fcm_dpo/q_t": 0.4278087615966797,
"grad_norm": 12.366849899291992,
"learning_rate": 1.4016959412166437e-07,
"logits/chosen": 0.7637462615966797,
"logits/rejected": 0.7285028696060181,
"logps/chosen": -236.34963989257812,
"logps/ref_chosen": -63.350440979003906,
"logps/ref_rejected": -76.28530883789062,
"logps/rejected": -317.478759765625,
"loss": 1.1642,
"margin_dpo/margin_mean": 68.19424438476562,
"margin_dpo/margin_std": 118.42993927001953,
"step": 451
},
{
"KL/chosen_KL_mean": -173.6109161376953,
"KL/mean": -212.7416229248047,
"KL/rejected_KL_mean": -251.87232971191406,
"KL/std": 110.82177734375,
"epoch": 0.6832955404383976,
"fcm_dpo/beta": 0.004780272487550974,
"fcm_dpo/delta": 0.02917136438190937,
"fcm_dpo/margin": 78.26138305664062,
"fcm_dpo/q_t": 0.41561344265937805,
"grad_norm": 13.4918851852417,
"learning_rate": 1.3898334684855645e-07,
"logits/chosen": 0.7915092706680298,
"logits/rejected": 0.723894476890564,
"logps/chosen": -229.19674682617188,
"logps/ref_chosen": -55.58583450317383,
"logps/ref_rejected": -77.68738555908203,
"logps/rejected": -329.5596923828125,
"loss": 1.1423,
"margin_dpo/margin_mean": 78.26138305664062,
"margin_dpo/margin_std": 133.20782470703125,
"step": 452
},
{
"KL/chosen_KL_mean": -175.6853485107422,
"KL/mean": -211.41357421875,
"KL/rejected_KL_mean": -247.1417999267578,
"KL/std": 114.35888671875,
"epoch": 0.6848072562358276,
"fcm_dpo/beta": 0.004915791098028421,
"fcm_dpo/delta": 0.05457682907581329,
"fcm_dpo/margin": 71.45646667480469,
"fcm_dpo/q_t": 0.42137593030929565,
"grad_norm": 15.09415340423584,
"learning_rate": 1.3780020494988445e-07,
"logits/chosen": 0.7863320708274841,
"logits/rejected": 0.772057294845581,
"logps/chosen": -237.4635467529297,
"logps/ref_chosen": -61.778202056884766,
"logps/ref_rejected": -71.51403045654297,
"logps/rejected": -318.65582275390625,
"loss": 1.1581,
"margin_dpo/margin_mean": 71.45646667480469,
"margin_dpo/margin_std": 126.16264343261719,
"step": 453
},
{
"KL/chosen_KL_mean": -158.27056884765625,
"KL/mean": -200.40567016601562,
"KL/rejected_KL_mean": -242.54075622558594,
"KL/std": 111.83387756347656,
"epoch": 0.6863189720332578,
"fcm_dpo/beta": 0.0049323514103889465,
"fcm_dpo/delta": -0.01793980970978737,
"fcm_dpo/margin": 84.27018737792969,
"fcm_dpo/q_t": 0.4046657383441925,
"grad_norm": 11.837895393371582,
"learning_rate": 1.366202015206706e-07,
"logits/chosen": 0.8560349345207214,
"logits/rejected": 0.8259134292602539,
"logps/chosen": -209.86572265625,
"logps/ref_chosen": -51.59515380859375,
"logps/ref_rejected": -63.96732711791992,
"logps/rejected": -306.5080871582031,
"loss": 1.1011,
"margin_dpo/margin_mean": 84.27018737792969,
"margin_dpo/margin_std": 126.40806579589844,
"step": 454
},
{
"KL/chosen_KL_mean": -186.71917724609375,
"KL/mean": -226.54803466796875,
"KL/rejected_KL_mean": -266.37689208984375,
"KL/std": 115.4573745727539,
"epoch": 0.6878306878306878,
"fcm_dpo/beta": 0.004945638123899698,
"fcm_dpo/delta": 0.006844331510365009,
"fcm_dpo/margin": 79.65769958496094,
"fcm_dpo/q_t": 0.41296789050102234,
"grad_norm": 13.294573783874512,
"learning_rate": 1.354433695681474e-07,
"logits/chosen": 0.7359009385108948,
"logits/rejected": 0.717836320400238,
"logps/chosen": -257.3708801269531,
"logps/ref_chosen": -70.65170288085938,
"logps/ref_rejected": -77.44276428222656,
"logps/rejected": -343.81964111328125,
"loss": 1.1178,
"margin_dpo/margin_mean": 79.65769958496094,
"margin_dpo/margin_std": 125.02285766601562,
"step": 455
},
{
"KL/chosen_KL_mean": -177.96502685546875,
"KL/mean": -217.13229370117188,
"KL/rejected_KL_mean": -256.299560546875,
"KL/std": 114.41992950439453,
"epoch": 0.6893424036281179,
"fcm_dpo/beta": 0.0049400972202420235,
"fcm_dpo/delta": 0.014393320307135582,
"fcm_dpo/margin": 78.33451843261719,
"fcm_dpo/q_t": 0.41560494899749756,
"grad_norm": 17.193586349487305,
"learning_rate": 1.3426974201083439e-07,
"logits/chosen": 0.7663091421127319,
"logits/rejected": 0.7156355381011963,
"logps/chosen": -234.36331176757812,
"logps/ref_chosen": -56.398284912109375,
"logps/ref_rejected": -82.61642456054688,
"logps/rejected": -338.91595458984375,
"loss": 1.1366,
"margin_dpo/margin_mean": 78.33451843261719,
"margin_dpo/margin_std": 135.641357421875,
"step": 456
},
{
"KL/chosen_KL_mean": -173.41329956054688,
"KL/mean": -213.361572265625,
"KL/rejected_KL_mean": -253.3098602294922,
"KL/std": 117.31248474121094,
"epoch": 0.690854119425548,
"fcm_dpo/beta": 0.004963235929608345,
"fcm_dpo/delta": 0.003583187237381935,
"fcm_dpo/margin": 79.89659118652344,
"fcm_dpo/q_t": 0.4083296060562134,
"grad_norm": 14.671182632446289,
"learning_rate": 1.3309935167761717e-07,
"logits/chosen": 0.9071280360221863,
"logits/rejected": 0.8378027677536011,
"logps/chosen": -218.1338653564453,
"logps/ref_chosen": -44.72057342529297,
"logps/ref_rejected": -68.1158676147461,
"logps/rejected": -321.42572021484375,
"loss": 1.1027,
"margin_dpo/margin_mean": 79.8965835571289,
"margin_dpo/margin_std": 113.3616943359375,
"step": 457
},
{
"KL/chosen_KL_mean": -168.78573608398438,
"KL/mean": -213.8314208984375,
"KL/rejected_KL_mean": -258.87713623046875,
"KL/std": 119.34392547607422,
"epoch": 0.6923658352229781,
"fcm_dpo/beta": 0.004918850027024746,
"fcm_dpo/delta": -0.049693845212459564,
"fcm_dpo/margin": 90.09138488769531,
"fcm_dpo/q_t": 0.3987731337547302,
"grad_norm": 12.683366775512695,
"learning_rate": 1.3193223130682936e-07,
"logits/chosen": 0.8713840246200562,
"logits/rejected": 0.7701141834259033,
"logps/chosen": -218.7914276123047,
"logps/ref_chosen": -50.00569152832031,
"logps/ref_rejected": -87.50015258789062,
"logps/rejected": -346.37725830078125,
"loss": 1.0915,
"margin_dpo/margin_mean": 90.09138488769531,
"margin_dpo/margin_std": 134.1453399658203,
"step": 458
},
{
"KL/chosen_KL_mean": -167.32614135742188,
"KL/mean": -218.5425262451172,
"KL/rejected_KL_mean": -269.7589111328125,
"KL/std": 142.56143188476562,
"epoch": 0.6938775510204082,
"fcm_dpo/beta": 0.004762580618262291,
"fcm_dpo/delta": -0.10504220426082611,
"fcm_dpo/margin": 102.4327392578125,
"fcm_dpo/q_t": 0.3850099444389343,
"grad_norm": 11.739134788513184,
"learning_rate": 1.3076841354533658e-07,
"logits/chosen": 0.868898868560791,
"logits/rejected": 0.8451403975486755,
"logps/chosen": -232.7041015625,
"logps/ref_chosen": -65.37794494628906,
"logps/ref_rejected": -88.19244384765625,
"logps/rejected": -357.95135498046875,
"loss": 1.0621,
"margin_dpo/margin_mean": 102.43274688720703,
"margin_dpo/margin_std": 134.36404418945312,
"step": 459
},
{
"KL/chosen_KL_mean": -177.44586181640625,
"KL/mean": -228.05621337890625,
"KL/rejected_KL_mean": -278.66656494140625,
"KL/std": 122.83148193359375,
"epoch": 0.6953892668178382,
"fcm_dpo/beta": 0.0044008782133460045,
"fcm_dpo/delta": -0.05401448905467987,
"fcm_dpo/margin": 101.220703125,
"fcm_dpo/q_t": 0.3985062539577484,
"grad_norm": 10.761886596679688,
"learning_rate": 1.2960793094762345e-07,
"logits/chosen": 0.8326910734176636,
"logits/rejected": 0.7360147833824158,
"logps/chosen": -242.00753784179688,
"logps/ref_chosen": -64.5616683959961,
"logps/ref_rejected": -88.67890167236328,
"logps/rejected": -367.345458984375,
"loss": 1.0705,
"margin_dpo/margin_mean": 101.220703125,
"margin_dpo/margin_std": 134.64776611328125,
"step": 460
},
{
"KL/chosen_KL_mean": -146.41845703125,
"KL/mean": -197.54617309570312,
"KL/rejected_KL_mean": -248.6739044189453,
"KL/std": 113.24662780761719,
"epoch": 0.6969009826152683,
"fcm_dpo/beta": 0.004217734094709158,
"fcm_dpo/delta": -0.04221995174884796,
"fcm_dpo/margin": 102.25545501708984,
"fcm_dpo/q_t": 0.3992024064064026,
"grad_norm": 10.860950469970703,
"learning_rate": 1.2845081597488286e-07,
"logits/chosen": 0.897905707359314,
"logits/rejected": 0.8329803943634033,
"logps/chosen": -195.89637756347656,
"logps/ref_chosen": -49.4779167175293,
"logps/ref_rejected": -72.65262603759766,
"logps/rejected": -321.3265380859375,
"loss": 1.0704,
"margin_dpo/margin_mean": 102.25545501708984,
"margin_dpo/margin_std": 122.31402587890625,
"step": 461
},
{
"KL/chosen_KL_mean": -162.05267333984375,
"KL/mean": -212.8834228515625,
"KL/rejected_KL_mean": -263.71417236328125,
"KL/std": 105.76100158691406,
"epoch": 0.6984126984126984,
"fcm_dpo/beta": 0.004214158747345209,
"fcm_dpo/delta": -0.0330844447016716,
"fcm_dpo/margin": 101.66149139404297,
"fcm_dpo/q_t": 0.39864546060562134,
"grad_norm": 10.956618309020996,
"learning_rate": 1.27297100994108e-07,
"logits/chosen": 0.8604691028594971,
"logits/rejected": 0.8234196901321411,
"logps/chosen": -222.54779052734375,
"logps/ref_chosen": -60.4951171875,
"logps/ref_rejected": -74.82136535644531,
"logps/rejected": -338.5355224609375,
"loss": 1.062,
"margin_dpo/margin_mean": 101.66149139404297,
"margin_dpo/margin_std": 117.94256591796875,
"step": 462
},
{
"KL/chosen_KL_mean": -197.11163330078125,
"KL/mean": -225.55178833007812,
"KL/rejected_KL_mean": -253.991943359375,
"KL/std": 109.48463439941406,
"epoch": 0.6999244142101285,
"fcm_dpo/beta": 0.004320027772337198,
"fcm_dpo/delta": 0.039919111877679825,
"fcm_dpo/margin": 56.880313873291016,
"fcm_dpo/q_t": 0.44253993034362793,
"grad_norm": 17.169588088989258,
"learning_rate": 1.2614681827718695e-07,
"logits/chosen": 0.8233051896095276,
"logits/rejected": 0.8192650675773621,
"logps/chosen": -264.7967529296875,
"logps/ref_chosen": -67.68511962890625,
"logps/ref_rejected": -71.32196044921875,
"logps/rejected": -325.31390380859375,
"loss": 1.2257,
"margin_dpo/margin_mean": 56.88031005859375,
"margin_dpo/margin_std": 123.54747009277344,
"step": 463
},
{
"KL/chosen_KL_mean": -176.97918701171875,
"KL/mean": -224.4793701171875,
"KL/rejected_KL_mean": -271.97955322265625,
"KL/std": 127.88569641113281,
"epoch": 0.7014361300075586,
"fcm_dpo/beta": 0.0043390789069235325,
"fcm_dpo/delta": -0.014604151248931885,
"fcm_dpo/margin": 95.00037384033203,
"fcm_dpo/q_t": 0.40515702962875366,
"grad_norm": 10.686640739440918,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 0.8410533666610718,
"logits/rejected": 0.8190803527832031,
"logps/chosen": -236.14483642578125,
"logps/ref_chosen": -59.16564178466797,
"logps/ref_rejected": -69.56146240234375,
"logps/rejected": -341.541015625,
"loss": 1.108,
"margin_dpo/margin_mean": 95.0003662109375,
"margin_dpo/margin_std": 141.3034210205078,
"step": 464
},
{
"KL/chosen_KL_mean": -184.62344360351562,
"KL/mean": -226.74331665039062,
"KL/rejected_KL_mean": -268.8631896972656,
"KL/std": 118.66059875488281,
"epoch": 0.7029478458049887,
"fcm_dpo/beta": 0.004308546893298626,
"fcm_dpo/delta": 0.04102543368935585,
"fcm_dpo/margin": 84.23976135253906,
"fcm_dpo/q_t": 0.4176803529262543,
"grad_norm": 11.88809871673584,
"learning_rate": 1.238566782415197e-07,
"logits/chosen": 0.8855515718460083,
"logits/rejected": 0.836800217628479,
"logps/chosen": -243.13711547851562,
"logps/ref_chosen": -58.513671875,
"logps/ref_rejected": -84.31745910644531,
"logps/rejected": -353.1806640625,
"loss": 1.1367,
"margin_dpo/margin_mean": 84.23976135253906,
"margin_dpo/margin_std": 132.88246154785156,
"step": 465
},
{
"KL/chosen_KL_mean": -203.14230346679688,
"KL/mean": -222.7254638671875,
"KL/rejected_KL_mean": -242.3086395263672,
"KL/std": 127.04100799560547,
"epoch": 0.7044595616024187,
"fcm_dpo/beta": 0.004587305244058371,
"fcm_dpo/delta": 0.07363003492355347,
"fcm_dpo/margin": 39.16636276245117,
"fcm_dpo/q_t": 0.45767632126808167,
"grad_norm": 16.998245239257812,
"learning_rate": 1.2271688498291334e-07,
"logits/chosen": 0.8211065530776978,
"logits/rejected": 0.8255276083946228,
"logps/chosen": -276.4081115722656,
"logps/ref_chosen": -73.26580810546875,
"logps/ref_rejected": -74.83621215820312,
"logps/rejected": -317.14483642578125,
"loss": 1.3027,
"margin_dpo/margin_mean": 39.166358947753906,
"margin_dpo/margin_std": 131.44540405273438,
"step": 466
},
{
"KL/chosen_KL_mean": -176.02005004882812,
"KL/mean": -217.30421447753906,
"KL/rejected_KL_mean": -258.58837890625,
"KL/std": 126.46763610839844,
"epoch": 0.7059712773998488,
"fcm_dpo/beta": 0.004604835994541645,
"fcm_dpo/delta": 0.022210007533431053,
"fcm_dpo/margin": 82.56832885742188,
"fcm_dpo/q_t": 0.41688525676727295,
"grad_norm": 10.904779434204102,
"learning_rate": 1.2158065210664848e-07,
"logits/chosen": 0.8689050674438477,
"logits/rejected": 0.7562901973724365,
"logps/chosen": -223.59951782226562,
"logps/ref_chosen": -47.57947540283203,
"logps/ref_rejected": -78.68522644042969,
"logps/rejected": -337.27362060546875,
"loss": 1.1317,
"margin_dpo/margin_mean": 82.56832885742188,
"margin_dpo/margin_std": 138.4664306640625,
"step": 467
},
{
"KL/chosen_KL_mean": -171.49169921875,
"KL/mean": -218.56295776367188,
"KL/rejected_KL_mean": -265.6342468261719,
"KL/std": 121.87532806396484,
"epoch": 0.7074829931972789,
"fcm_dpo/beta": 0.004571876488626003,
"fcm_dpo/delta": -0.03497558832168579,
"fcm_dpo/margin": 94.14254760742188,
"fcm_dpo/q_t": 0.39979416131973267,
"grad_norm": 13.04074764251709,
"learning_rate": 1.204480113956011e-07,
"logits/chosen": 0.798638105392456,
"logits/rejected": 0.7876749038696289,
"logps/chosen": -235.4194793701172,
"logps/ref_chosen": -63.92778778076172,
"logps/ref_rejected": -76.51626586914062,
"logps/rejected": -342.1505126953125,
"loss": 1.101,
"margin_dpo/margin_mean": 94.14254760742188,
"margin_dpo/margin_std": 142.54898071289062,
"step": 468
},
{
"KL/chosen_KL_mean": -172.53428649902344,
"KL/mean": -215.07339477539062,
"KL/rejected_KL_mean": -257.6125183105469,
"KL/std": 118.53016662597656,
"epoch": 0.708994708994709,
"fcm_dpo/beta": 0.0045113954693078995,
"fcm_dpo/delta": 0.016616512089967728,
"fcm_dpo/margin": 85.07823181152344,
"fcm_dpo/q_t": 0.4123454689979553,
"grad_norm": 13.645792961120605,
"learning_rate": 1.1931899453216697e-07,
"logits/chosen": 0.8709956407546997,
"logits/rejected": 0.855722188949585,
"logps/chosen": -231.59246826171875,
"logps/ref_chosen": -59.05818176269531,
"logps/ref_rejected": -75.67672729492188,
"logps/rejected": -333.28924560546875,
"loss": 1.1028,
"margin_dpo/margin_mean": 85.0782241821289,
"margin_dpo/margin_std": 114.00239562988281,
"step": 469
},
{
"KL/chosen_KL_mean": -160.6546630859375,
"KL/mean": -205.504150390625,
"KL/rejected_KL_mean": -250.3536376953125,
"KL/std": 114.10494995117188,
"epoch": 0.7105064247921391,
"fcm_dpo/beta": 0.004605274647474289,
"fcm_dpo/delta": -0.015301994979381561,
"fcm_dpo/margin": 89.69898986816406,
"fcm_dpo/q_t": 0.4052748680114746,
"grad_norm": 11.65384578704834,
"learning_rate": 1.1819363309737438e-07,
"logits/chosen": 0.8430336713790894,
"logits/rejected": 0.7940839529037476,
"logps/chosen": -208.5220947265625,
"logps/ref_chosen": -47.86743927001953,
"logps/ref_rejected": -65.96859741210938,
"logps/rejected": -316.32220458984375,
"loss": 1.0995,
"margin_dpo/margin_mean": 89.69898986816406,
"margin_dpo/margin_std": 129.3060760498047,
"step": 470
},
{
"KL/chosen_KL_mean": -158.75082397460938,
"KL/mean": -207.02197265625,
"KL/rejected_KL_mean": -255.29310607910156,
"KL/std": 111.86181640625,
"epoch": 0.7120181405895691,
"fcm_dpo/beta": 0.004474523477256298,
"fcm_dpo/delta": -0.03683408349752426,
"fcm_dpo/margin": 96.54228210449219,
"fcm_dpo/q_t": 0.3983868360519409,
"grad_norm": 12.488258361816406,
"learning_rate": 1.1707195857000215e-07,
"logits/chosen": 0.8283665180206299,
"logits/rejected": 0.7850099802017212,
"logps/chosen": -216.52867126464844,
"logps/ref_chosen": -57.777854919433594,
"logps/ref_rejected": -73.81172180175781,
"logps/rejected": -329.1048278808594,
"loss": 1.0719,
"margin_dpo/margin_mean": 96.54228210449219,
"margin_dpo/margin_std": 121.92526245117188,
"step": 471
},
{
"KL/chosen_KL_mean": -159.306884765625,
"KL/mean": -197.11712646484375,
"KL/rejected_KL_mean": -234.9273681640625,
"KL/std": 117.94495391845703,
"epoch": 0.7135298563869993,
"fcm_dpo/beta": 0.004620303399860859,
"fcm_dpo/delta": 0.054885368794202805,
"fcm_dpo/margin": 75.62045288085938,
"fcm_dpo/q_t": 0.4230712652206421,
"grad_norm": 12.124162673950195,
"learning_rate": 1.1595400232569768e-07,
"logits/chosen": 0.8315516710281372,
"logits/rejected": 0.7926985025405884,
"logps/chosen": -215.21556091308594,
"logps/ref_chosen": -55.908668518066406,
"logps/ref_rejected": -74.70294189453125,
"logps/rejected": -309.63031005859375,
"loss": 1.1772,
"margin_dpo/margin_mean": 75.62045288085938,
"margin_dpo/margin_std": 149.9696044921875,
"step": 472
},
{
"KL/chosen_KL_mean": -163.22857666015625,
"KL/mean": -215.5937042236328,
"KL/rejected_KL_mean": -267.95880126953125,
"KL/std": 143.45526123046875,
"epoch": 0.7150415721844293,
"fcm_dpo/beta": 0.004440706688910723,
"fcm_dpo/delta": -0.0759398564696312,
"fcm_dpo/margin": 104.73027038574219,
"fcm_dpo/q_t": 0.3982590436935425,
"grad_norm": 13.992444038391113,
"learning_rate": 1.1483979563610069e-07,
"logits/chosen": 0.9021942615509033,
"logits/rejected": 0.809353232383728,
"logps/chosen": -217.3894500732422,
"logps/ref_chosen": -54.16088104248047,
"logps/ref_rejected": -92.76789855957031,
"logps/rejected": -360.7267150878906,
"loss": 1.096,
"margin_dpo/margin_mean": 104.73027038574219,
"margin_dpo/margin_std": 166.75250244140625,
"step": 473
},
{
"KL/chosen_KL_mean": -165.42330932617188,
"KL/mean": -207.98435974121094,
"KL/rejected_KL_mean": -250.54539489746094,
"KL/std": 122.40956115722656,
"epoch": 0.7165532879818595,
"fcm_dpo/beta": 0.004457107279449701,
"fcm_dpo/delta": 0.02313932217657566,
"fcm_dpo/margin": 85.1220703125,
"fcm_dpo/q_t": 0.41587191820144653,
"grad_norm": 17.439899444580078,
"learning_rate": 1.1372936966796709e-07,
"logits/chosen": 0.9221115112304688,
"logits/rejected": 0.8645721673965454,
"logps/chosen": -212.10902404785156,
"logps/ref_chosen": -46.685707092285156,
"logps/ref_rejected": -71.44731903076172,
"logps/rejected": -321.9927062988281,
"loss": 1.1342,
"margin_dpo/margin_mean": 85.1220703125,
"margin_dpo/margin_std": 141.8304443359375,
"step": 474
},
{
"KL/chosen_KL_mean": -164.73910522460938,
"KL/mean": -222.14610290527344,
"KL/rejected_KL_mean": -279.5531005859375,
"KL/std": 123.50313568115234,
"epoch": 0.7180650037792895,
"fcm_dpo/beta": 0.004225746262818575,
"fcm_dpo/delta": -0.10140831023454666,
"fcm_dpo/margin": 114.81401062011719,
"fcm_dpo/q_t": 0.3880518078804016,
"grad_norm": 8.140756607055664,
"learning_rate": 1.126227554822985e-07,
"logits/chosen": 0.8333104848861694,
"logits/rejected": 0.7912643551826477,
"logps/chosen": -223.22640991210938,
"logps/ref_chosen": -58.4873046875,
"logps/ref_rejected": -87.00187683105469,
"logps/rejected": -366.55499267578125,
"loss": 1.0399,
"margin_dpo/margin_mean": 114.81401062011719,
"margin_dpo/margin_std": 140.57183837890625,
"step": 475
},
{
"KL/chosen_KL_mean": -198.56451416015625,
"KL/mean": -232.54180908203125,
"KL/rejected_KL_mean": -266.51910400390625,
"KL/std": 132.65438842773438,
"epoch": 0.7195767195767195,
"fcm_dpo/beta": 0.004227438010275364,
"fcm_dpo/delta": 0.02281678467988968,
"fcm_dpo/margin": 67.95459747314453,
"fcm_dpo/q_t": 0.43554818630218506,
"grad_norm": 13.2657470703125,
"learning_rate": 1.1151998403347243e-07,
"logits/chosen": 0.7603304386138916,
"logits/rejected": 0.764335036277771,
"logps/chosen": -273.9461364746094,
"logps/ref_chosen": -75.38162231445312,
"logps/ref_rejected": -76.99822235107422,
"logps/rejected": -343.517333984375,
"loss": 1.2024,
"margin_dpo/margin_mean": 67.95459747314453,
"margin_dpo/margin_std": 140.07485961914062,
"step": 476
},
{
"KL/chosen_KL_mean": -194.00865173339844,
"KL/mean": -236.6685791015625,
"KL/rejected_KL_mean": -279.3285217285156,
"KL/std": 133.24874877929688,
"epoch": 0.7210884353741497,
"fcm_dpo/beta": 0.004316464997828007,
"fcm_dpo/delta": 0.035700492560863495,
"fcm_dpo/margin": 85.31986999511719,
"fcm_dpo/q_t": 0.4191203713417053,
"grad_norm": 12.513143539428711,
"learning_rate": 1.1042108616837692e-07,
"logits/chosen": 0.8567191362380981,
"logits/rejected": 0.8166630268096924,
"logps/chosen": -255.08203125,
"logps/ref_chosen": -61.073387145996094,
"logps/ref_rejected": -81.34375,
"logps/rejected": -360.6722717285156,
"loss": 1.1779,
"margin_dpo/margin_mean": 85.31986236572266,
"margin_dpo/margin_std": 170.67071533203125,
"step": 477
},
{
"KL/chosen_KL_mean": -171.2495574951172,
"KL/mean": -202.14273071289062,
"KL/rejected_KL_mean": -233.03590393066406,
"KL/std": 106.54711151123047,
"epoch": 0.7226001511715797,
"fcm_dpo/beta": 0.004573522135615349,
"fcm_dpo/delta": 0.1288546621799469,
"fcm_dpo/margin": 61.78632736206055,
"fcm_dpo/q_t": 0.4368034899234772,
"grad_norm": 14.673408508300781,
"learning_rate": 1.0932609262554746e-07,
"logits/chosen": 0.8044695854187012,
"logits/rejected": 0.8161039352416992,
"logps/chosen": -228.4168701171875,
"logps/ref_chosen": -57.16731643676758,
"logps/ref_rejected": -53.30917739868164,
"logps/rejected": -286.3450927734375,
"loss": 1.2093,
"margin_dpo/margin_mean": 61.78633117675781,
"margin_dpo/margin_std": 129.24046325683594,
"step": 478
},
{
"KL/chosen_KL_mean": -177.13055419921875,
"KL/mean": -204.7392578125,
"KL/rejected_KL_mean": -232.34796142578125,
"KL/std": 110.43617248535156,
"epoch": 0.7241118669690099,
"fcm_dpo/beta": 0.004763299599289894,
"fcm_dpo/delta": 0.022982783615589142,
"fcm_dpo/margin": 55.2174072265625,
"fcm_dpo/q_t": 0.44107237458229065,
"grad_norm": 14.663484573364258,
"learning_rate": 1.0823503403430734e-07,
"logits/chosen": 0.8042382001876831,
"logits/rejected": 0.778433084487915,
"logps/chosen": -236.0438690185547,
"logps/ref_chosen": -58.91331481933594,
"logps/ref_rejected": -63.7403450012207,
"logps/rejected": -296.08831787109375,
"loss": 1.2346,
"margin_dpo/margin_mean": 55.2174072265625,
"margin_dpo/margin_std": 131.55545043945312,
"step": 479
},
{
"KL/chosen_KL_mean": -184.4405975341797,
"KL/mean": -227.0792236328125,
"KL/rejected_KL_mean": -269.71783447265625,
"KL/std": 115.04316711425781,
"epoch": 0.7256235827664399,
"fcm_dpo/beta": 0.004858708009123802,
"fcm_dpo/delta": -0.02168682962656021,
"fcm_dpo/margin": 85.27723693847656,
"fcm_dpo/q_t": 0.40630629658699036,
"grad_norm": 15.03400707244873,
"learning_rate": 1.0714794091391072e-07,
"logits/chosen": 0.7868234515190125,
"logits/rejected": 0.7834052443504333,
"logps/chosen": -247.2412109375,
"logps/ref_chosen": -62.80061340332031,
"logps/ref_rejected": -67.58859252929688,
"logps/rejected": -337.30645751953125,
"loss": 1.1233,
"margin_dpo/margin_mean": 85.27723693847656,
"margin_dpo/margin_std": 134.57638549804688,
"step": 480
},
{
"KL/chosen_KL_mean": -182.68031311035156,
"KL/mean": -222.99514770507812,
"KL/rejected_KL_mean": -263.3099670410156,
"KL/std": 121.83908081054688,
"epoch": 0.72713529856387,
"fcm_dpo/beta": 0.004709561821073294,
"fcm_dpo/delta": 0.022568443790078163,
"fcm_dpo/margin": 80.62967681884766,
"fcm_dpo/q_t": 0.4156482219696045,
"grad_norm": 12.423121452331543,
"learning_rate": 1.0606484367268906e-07,
"logits/chosen": 0.8025602102279663,
"logits/rejected": 0.8090294599533081,
"logps/chosen": -247.966796875,
"logps/ref_chosen": -65.28649139404297,
"logps/ref_rejected": -70.78668212890625,
"logps/rejected": -334.0966491699219,
"loss": 1.132,
"margin_dpo/margin_mean": 80.62968444824219,
"margin_dpo/margin_std": 133.42701721191406,
"step": 481
},
{
"KL/chosen_KL_mean": -201.8643798828125,
"KL/mean": -246.51490783691406,
"KL/rejected_KL_mean": -291.16546630859375,
"KL/std": 135.26222229003906,
"epoch": 0.7286470143613001,
"fcm_dpo/beta": 0.004750003106892109,
"fcm_dpo/delta": -0.02802390605211258,
"fcm_dpo/margin": 89.30106353759766,
"fcm_dpo/q_t": 0.41019535064697266,
"grad_norm": 15.123866081237793,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 0.7807713747024536,
"logits/rejected": 0.6487196087837219,
"logps/chosen": -262.77056884765625,
"logps/ref_chosen": -60.906185150146484,
"logps/ref_rejected": -103.44656372070312,
"logps/rejected": -394.61199951171875,
"loss": 1.1465,
"margin_dpo/margin_mean": 89.30105590820312,
"margin_dpo/margin_std": 167.93521118164062,
"step": 482
},
{
"KL/chosen_KL_mean": -169.6586151123047,
"KL/mean": -220.8936309814453,
"KL/rejected_KL_mean": -272.128662109375,
"KL/std": 117.83233642578125,
"epoch": 0.7301587301587301,
"fcm_dpo/beta": 0.004558105953037739,
"fcm_dpo/delta": -0.07779208570718765,
"fcm_dpo/margin": 102.47002410888672,
"fcm_dpo/q_t": 0.39235061407089233,
"grad_norm": 13.63161849975586,
"learning_rate": 1.0391075790138232e-07,
"logits/chosen": 0.8855106830596924,
"logits/rejected": 0.7963603734970093,
"logps/chosen": -222.85061645507812,
"logps/ref_chosen": -53.192012786865234,
"logps/ref_rejected": -81.83927154541016,
"logps/rejected": -353.9678955078125,
"loss": 1.0606,
"margin_dpo/margin_mean": 102.47001647949219,
"margin_dpo/margin_std": 134.73834228515625,
"step": 483
},
{
"KL/chosen_KL_mean": -176.22354125976562,
"KL/mean": -208.54843139648438,
"KL/rejected_KL_mean": -240.87332153320312,
"KL/std": 116.13772583007812,
"epoch": 0.7316704459561603,
"fcm_dpo/beta": 0.004734124056994915,
"fcm_dpo/delta": 0.10121459513902664,
"fcm_dpo/margin": 64.64977264404297,
"fcm_dpo/q_t": 0.4296380281448364,
"grad_norm": 16.20368003845215,
"learning_rate": 1.0283982962570681e-07,
"logits/chosen": 0.8983777761459351,
"logits/rejected": 0.8746373057365417,
"logps/chosen": -233.9929962158203,
"logps/ref_chosen": -57.76945877075195,
"logps/ref_rejected": -71.6829833984375,
"logps/rejected": -312.5563049316406,
"loss": 1.159,
"margin_dpo/margin_mean": 64.64977264404297,
"margin_dpo/margin_std": 101.06908416748047,
"step": 484
},
{
"KL/chosen_KL_mean": -182.181396484375,
"KL/mean": -218.22222900390625,
"KL/rejected_KL_mean": -254.26309204101562,
"KL/std": 114.957763671875,
"epoch": 0.7331821617535903,
"fcm_dpo/beta": 0.004620219115167856,
"fcm_dpo/delta": -0.04711120203137398,
"fcm_dpo/margin": 72.08170318603516,
"fcm_dpo/q_t": 0.4228460192680359,
"grad_norm": 13.281914710998535,
"learning_rate": 1.0177301773633992e-07,
"logits/chosen": 0.9076398611068726,
"logits/rejected": 0.8928012847900391,
"logps/chosen": -238.81724548339844,
"logps/ref_chosen": -56.63584899902344,
"logps/ref_rejected": -70.85614013671875,
"logps/rejected": -325.1192321777344,
"loss": 1.1514,
"margin_dpo/margin_mean": 72.08171081542969,
"margin_dpo/margin_std": 115.27439880371094,
"step": 485
},
{
"KL/chosen_KL_mean": -207.69192504882812,
"KL/mean": -243.96141052246094,
"KL/rejected_KL_mean": -280.23089599609375,
"KL/std": 137.62863159179688,
"epoch": 0.7346938775510204,
"fcm_dpo/beta": 0.00475216843187809,
"fcm_dpo/delta": 0.0615973100066185,
"fcm_dpo/margin": 72.53897857666016,
"fcm_dpo/q_t": 0.42720019817352295,
"grad_norm": 11.114177703857422,
"learning_rate": 1.007103520743035e-07,
"logits/chosen": 0.8727377653121948,
"logits/rejected": 0.7796794176101685,
"logps/chosen": -264.0389404296875,
"logps/ref_chosen": -56.347023010253906,
"logps/ref_rejected": -85.97221374511719,
"logps/rejected": -366.203125,
"loss": 1.2038,
"margin_dpo/margin_mean": 72.53898620605469,
"margin_dpo/margin_std": 161.3694610595703,
"step": 486
},
{
"KL/chosen_KL_mean": -187.3951416015625,
"KL/mean": -228.60194396972656,
"KL/rejected_KL_mean": -269.80877685546875,
"KL/std": 126.41487121582031,
"epoch": 0.7362055933484505,
"fcm_dpo/beta": 0.004806404933333397,
"fcm_dpo/delta": 0.004405636340379715,
"fcm_dpo/margin": 82.41361999511719,
"fcm_dpo/q_t": 0.4161032438278198,
"grad_norm": 13.856200218200684,
"learning_rate": 9.965186236464046e-08,
"logits/chosen": 0.9156618118286133,
"logits/rejected": 0.8678491115570068,
"logps/chosen": -248.01235961914062,
"logps/ref_chosen": -60.617218017578125,
"logps/ref_rejected": -82.50975036621094,
"logps/rejected": -352.3184814453125,
"loss": 1.1419,
"margin_dpo/margin_mean": 82.41361236572266,
"margin_dpo/margin_std": 151.87709045410156,
"step": 487
},
{
"KL/chosen_KL_mean": -175.20208740234375,
"KL/mean": -221.89056396484375,
"KL/rejected_KL_mean": -268.57904052734375,
"KL/std": 118.11080932617188,
"epoch": 0.7377173091458806,
"fcm_dpo/beta": 0.004773295484483242,
"fcm_dpo/delta": -0.05380316078662872,
"fcm_dpo/margin": 93.37692260742188,
"fcm_dpo/q_t": 0.39976638555526733,
"grad_norm": 14.908041000366211,
"learning_rate": 9.859757821558337e-08,
"logits/chosen": 0.8313355445861816,
"logits/rejected": 0.792324423789978,
"logps/chosen": -238.31114196777344,
"logps/ref_chosen": -63.10905075073242,
"logps/ref_rejected": -82.49348449707031,
"logps/rejected": -351.072509765625,
"loss": 1.0806,
"margin_dpo/margin_mean": 93.3769302368164,
"margin_dpo/margin_std": 130.49966430664062,
"step": 488
},
{
"KL/chosen_KL_mean": -204.91387939453125,
"KL/mean": -233.1622314453125,
"KL/rejected_KL_mean": -261.41058349609375,
"KL/std": 130.28018188476562,
"epoch": 0.7392290249433107,
"fcm_dpo/beta": 0.0049452874809503555,
"fcm_dpo/delta": 0.13130411505699158,
"fcm_dpo/margin": 56.49671936035156,
"fcm_dpo/q_t": 0.4389078915119171,
"grad_norm": 13.527437210083008,
"learning_rate": 9.754752911772615e-08,
"logits/chosen": 0.8165801763534546,
"logits/rejected": 0.780007004737854,
"logps/chosen": -269.90283203125,
"logps/ref_chosen": -64.98896026611328,
"logps/ref_rejected": -84.39607238769531,
"logps/rejected": -345.806640625,
"loss": 1.2361,
"margin_dpo/margin_mean": 56.49671936035156,
"margin_dpo/margin_std": 137.77053833007812,
"step": 489
},
{
"KL/chosen_KL_mean": -184.7069091796875,
"KL/mean": -218.8941192626953,
"KL/rejected_KL_mean": -253.08131408691406,
"KL/std": 124.6688232421875,
"epoch": 0.7407407407407407,
"fcm_dpo/beta": 0.005108346696943045,
"fcm_dpo/delta": 0.05644252896308899,
"fcm_dpo/margin": 68.3744125366211,
"fcm_dpo/q_t": 0.4248546361923218,
"grad_norm": 12.403369903564453,
"learning_rate": 9.650174444319956e-08,
"logits/chosen": 0.8969284892082214,
"logits/rejected": 0.8864274024963379,
"logps/chosen": -246.61566162109375,
"logps/ref_chosen": -61.90874481201172,
"logps/ref_rejected": -70.58566284179688,
"logps/rejected": -323.6669921875,
"loss": 1.2101,
"margin_dpo/margin_mean": 68.3744125366211,
"margin_dpo/margin_std": 154.39389038085938,
"step": 490
},
{
"KL/chosen_KL_mean": -178.02613830566406,
"KL/mean": -212.85516357421875,
"KL/rejected_KL_mean": -247.6842041015625,
"KL/std": 112.87677001953125,
"epoch": 0.7422524565381708,
"fcm_dpo/beta": 0.005029967520385981,
"fcm_dpo/delta": -0.05832719802856445,
"fcm_dpo/margin": 69.65803527832031,
"fcm_dpo/q_t": 0.41990724205970764,
"grad_norm": 10.252985000610352,
"learning_rate": 9.546025344484868e-08,
"logits/chosen": 0.7992293834686279,
"logits/rejected": 0.745780348777771,
"logps/chosen": -233.50184631347656,
"logps/ref_chosen": -55.47570037841797,
"logps/ref_rejected": -78.70318603515625,
"logps/rejected": -326.38739013671875,
"loss": 1.1552,
"margin_dpo/margin_mean": 69.65803527832031,
"margin_dpo/margin_std": 120.66893005371094,
"step": 491
},
{
"KL/chosen_KL_mean": -211.10516357421875,
"KL/mean": -243.54579162597656,
"KL/rejected_KL_mean": -275.98638916015625,
"KL/std": 131.36009216308594,
"epoch": 0.7437641723356009,
"fcm_dpo/beta": 0.005244538653641939,
"fcm_dpo/delta": 0.06441338360309601,
"fcm_dpo/margin": 64.88124084472656,
"fcm_dpo/q_t": 0.4269523024559021,
"grad_norm": 15.789043426513672,
"learning_rate": 9.442308525541589e-08,
"logits/chosen": 0.8516812920570374,
"logits/rejected": 0.7955601215362549,
"logps/chosen": -278.39154052734375,
"logps/ref_chosen": -67.28638458251953,
"logps/ref_rejected": -82.78628540039062,
"logps/rejected": -358.772705078125,
"loss": 1.2185,
"margin_dpo/margin_mean": 64.88124084472656,
"margin_dpo/margin_std": 148.1044921875,
"step": 492
},
{
"KL/chosen_KL_mean": -176.27764892578125,
"KL/mean": -222.05245971679688,
"KL/rejected_KL_mean": -267.8272705078125,
"KL/std": 127.64932250976562,
"epoch": 0.745275888133031,
"fcm_dpo/beta": 0.005175878759473562,
"fcm_dpo/delta": -0.08929616212844849,
"fcm_dpo/margin": 91.54963684082031,
"fcm_dpo/q_t": 0.3954174220561981,
"grad_norm": 15.408149719238281,
"learning_rate": 9.339026888672468e-08,
"logits/chosen": 0.8074703812599182,
"logits/rejected": 0.7508785128593445,
"logps/chosen": -232.20513916015625,
"logps/ref_chosen": -55.92750549316406,
"logps/ref_rejected": -79.12149810791016,
"logps/rejected": -346.9487609863281,
"loss": 1.0922,
"margin_dpo/margin_mean": 91.54963684082031,
"margin_dpo/margin_std": 139.12684631347656,
"step": 493
},
{
"KL/chosen_KL_mean": -181.6630096435547,
"KL/mean": -223.94744873046875,
"KL/rejected_KL_mean": -266.23193359375,
"KL/std": 128.83995056152344,
"epoch": 0.7467876039304611,
"fcm_dpo/beta": 0.004889290779829025,
"fcm_dpo/delta": -0.015794701874256134,
"fcm_dpo/margin": 84.56891632080078,
"fcm_dpo/q_t": 0.4098696708679199,
"grad_norm": 14.83998966217041,
"learning_rate": 9.236183322886945e-08,
"logits/chosen": 0.7640923857688904,
"logits/rejected": 0.7272478342056274,
"logps/chosen": -249.61712646484375,
"logps/ref_chosen": -67.95410919189453,
"logps/ref_rejected": -90.50865173339844,
"logps/rejected": -356.7405700683594,
"loss": 1.1521,
"margin_dpo/margin_mean": 84.56890869140625,
"margin_dpo/margin_std": 161.0829620361328,
"step": 494
},
{
"KL/chosen_KL_mean": -177.00015258789062,
"KL/mean": -209.45477294921875,
"KL/rejected_KL_mean": -241.90940856933594,
"KL/std": 125.46236419677734,
"epoch": 0.7482993197278912,
"fcm_dpo/beta": 0.004900210537016392,
"fcm_dpo/delta": -0.005768109112977982,
"fcm_dpo/margin": 64.90924835205078,
"fcm_dpo/q_t": 0.4349347949028015,
"grad_norm": 16.218822479248047,
"learning_rate": 9.133780704940594e-08,
"logits/chosen": 0.8546461462974548,
"logits/rejected": 0.8084453344345093,
"logps/chosen": -229.6256103515625,
"logps/ref_chosen": -52.62546157836914,
"logps/ref_rejected": -72.06781005859375,
"logps/rejected": -313.97723388671875,
"loss": 1.2211,
"margin_dpo/margin_mean": 64.90924835205078,
"margin_dpo/margin_std": 156.70840454101562,
"step": 495
},
{
"KL/chosen_KL_mean": -191.11280822753906,
"KL/mean": -238.58676147460938,
"KL/rejected_KL_mean": -286.0606994628906,
"KL/std": 138.51930236816406,
"epoch": 0.7498110355253212,
"fcm_dpo/beta": 0.004659120924770832,
"fcm_dpo/delta": -0.054637569934129715,
"fcm_dpo/margin": 94.94786834716797,
"fcm_dpo/q_t": 0.40979450941085815,
"grad_norm": 11.95200252532959,
"learning_rate": 9.031821899254797e-08,
"logits/chosen": 0.8415661454200745,
"logits/rejected": 0.7441533803939819,
"logps/chosen": -248.71014404296875,
"logps/ref_chosen": -57.597320556640625,
"logps/ref_rejected": -94.36127471923828,
"logps/rejected": -380.4219665527344,
"loss": 1.1355,
"margin_dpo/margin_mean": 94.94786834716797,
"margin_dpo/margin_std": 179.43215942382812,
"step": 496
},
{
"KL/chosen_KL_mean": -197.67959594726562,
"KL/mean": -241.09596252441406,
"KL/rejected_KL_mean": -284.5123291015625,
"KL/std": 124.52940368652344,
"epoch": 0.7513227513227513,
"fcm_dpo/beta": 0.004687286913394928,
"fcm_dpo/delta": -0.008316351100802422,
"fcm_dpo/margin": 86.83271789550781,
"fcm_dpo/q_t": 0.406663715839386,
"grad_norm": 13.023819923400879,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 0.853161096572876,
"logits/rejected": 0.8242659568786621,
"logps/chosen": -270.46954345703125,
"logps/ref_chosen": -72.78994750976562,
"logps/ref_rejected": -89.48483276367188,
"logps/rejected": -373.99713134765625,
"loss": 1.116,
"margin_dpo/margin_mean": 86.83271789550781,
"margin_dpo/margin_std": 136.6311492919922,
"step": 497
},
{
"KL/chosen_KL_mean": -175.44488525390625,
"KL/mean": -222.20950317382812,
"KL/rejected_KL_mean": -268.97412109375,
"KL/std": 117.231201171875,
"epoch": 0.7528344671201814,
"fcm_dpo/beta": 0.00461503816768527,
"fcm_dpo/delta": -0.036642152816057205,
"fcm_dpo/margin": 93.52923583984375,
"fcm_dpo/q_t": 0.39883777499198914,
"grad_norm": 13.756954193115234,
"learning_rate": 8.829247120198563e-08,
"logits/chosen": 0.8330202102661133,
"logits/rejected": 0.8214312791824341,
"logps/chosen": -243.81060791015625,
"logps/ref_chosen": -68.36572265625,
"logps/ref_rejected": -71.28846740722656,
"logps/rejected": -340.2626037597656,
"loss": 1.0839,
"margin_dpo/margin_mean": 93.52923583984375,
"margin_dpo/margin_std": 129.33668518066406,
"step": 498
},
{
"KL/chosen_KL_mean": -179.2896728515625,
"KL/mean": -229.2539825439453,
"KL/rejected_KL_mean": -279.2182922363281,
"KL/std": 132.14337158203125,
"epoch": 0.7543461829176115,
"fcm_dpo/beta": 0.004479332827031612,
"fcm_dpo/delta": -0.055337462574243546,
"fcm_dpo/margin": 99.9286117553711,
"fcm_dpo/q_t": 0.40216517448425293,
"grad_norm": 19.308687210083008,
"learning_rate": 8.728636813280163e-08,
"logits/chosen": 0.8368034362792969,
"logits/rejected": 0.7836736440658569,
"logps/chosen": -241.19850158691406,
"logps/ref_chosen": -61.90882873535156,
"logps/ref_rejected": -91.9411392211914,
"logps/rejected": -371.159423828125,
"loss": 1.1267,
"margin_dpo/margin_mean": 99.92861938476562,
"margin_dpo/margin_std": 174.06784057617188,
"step": 499
},
{
"KL/chosen_KL_mean": -188.59982299804688,
"KL/mean": -225.63168334960938,
"KL/rejected_KL_mean": -262.66351318359375,
"KL/std": 115.21099853515625,
"epoch": 0.7558578987150416,
"fcm_dpo/beta": 0.00457107275724411,
"fcm_dpo/delta": 0.06856130063533783,
"fcm_dpo/margin": 74.0636978149414,
"fcm_dpo/q_t": 0.42126649618148804,
"grad_norm": 14.0758695602417,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.8072834014892578,
"logits/rejected": 0.8198223114013672,
"logps/chosen": -258.8256530761719,
"logps/ref_chosen": -70.225830078125,
"logps/ref_rejected": -71.72203063964844,
"logps/rejected": -334.38555908203125,
"loss": 1.1835,
"margin_dpo/margin_mean": 74.0636978149414,
"margin_dpo/margin_std": 146.86541748046875,
"step": 500
},
{
"KL/chosen_KL_mean": -187.39126586914062,
"KL/mean": -222.43211364746094,
"KL/rejected_KL_mean": -257.47296142578125,
"KL/std": 122.28995513916016,
"epoch": 0.7573696145124716,
"fcm_dpo/beta": 0.004816969856619835,
"fcm_dpo/delta": 0.06878426671028137,
"fcm_dpo/margin": 70.08174133300781,
"fcm_dpo/q_t": 0.42239677906036377,
"grad_norm": 11.63576889038086,
"learning_rate": 8.528784436016878e-08,
"logits/chosen": 0.833807110786438,
"logits/rejected": 0.8403280973434448,
"logps/chosen": -251.9900665283203,
"logps/ref_chosen": -64.59880828857422,
"logps/ref_rejected": -70.59329223632812,
"logps/rejected": -328.0662841796875,
"loss": 1.132,
"margin_dpo/margin_mean": 70.08173370361328,
"margin_dpo/margin_std": 99.08871459960938,
"step": 501
},
{
"KL/chosen_KL_mean": -186.89634704589844,
"KL/mean": -225.32411193847656,
"KL/rejected_KL_mean": -263.75189208984375,
"KL/std": 133.69595336914062,
"epoch": 0.7588813303099018,
"fcm_dpo/beta": 0.004946129396557808,
"fcm_dpo/delta": 0.021091684699058533,
"fcm_dpo/margin": 76.85551452636719,
"fcm_dpo/q_t": 0.41427573561668396,
"grad_norm": 16.61263656616211,
"learning_rate": 8.4295479559726e-08,
"logits/chosen": 0.8703242540359497,
"logits/rejected": 0.8291156888008118,
"logps/chosen": -252.36297607421875,
"logps/ref_chosen": -65.46662902832031,
"logps/ref_rejected": -90.22233581542969,
"logps/rejected": -353.9742126464844,
"loss": 1.1302,
"margin_dpo/margin_mean": 76.85551452636719,
"margin_dpo/margin_std": 123.59288024902344,
"step": 502
},
{
"KL/chosen_KL_mean": -165.91510009765625,
"KL/mean": -204.1113739013672,
"KL/rejected_KL_mean": -242.3076171875,
"KL/std": 110.12651062011719,
"epoch": 0.7603930461073318,
"fcm_dpo/beta": 0.004937716759741306,
"fcm_dpo/delta": 0.02568972110748291,
"fcm_dpo/margin": 76.39253997802734,
"fcm_dpo/q_t": 0.41301050782203674,
"grad_norm": 12.290018081665039,
"learning_rate": 8.330774987092712e-08,
"logits/chosen": 0.8349542617797852,
"logits/rejected": 0.8423024415969849,
"logps/chosen": -217.74986267089844,
"logps/ref_chosen": -51.83476257324219,
"logps/ref_rejected": -57.62522506713867,
"logps/rejected": -299.932861328125,
"loss": 1.1346,
"margin_dpo/margin_mean": 76.39253997802734,
"margin_dpo/margin_std": 124.44419860839844,
"step": 503
},
{
"KL/chosen_KL_mean": -174.50894165039062,
"KL/mean": -224.36419677734375,
"KL/rejected_KL_mean": -274.2194519042969,
"KL/std": 112.46851348876953,
"epoch": 0.7619047619047619,
"fcm_dpo/beta": 0.0048708608373999596,
"fcm_dpo/delta": -0.10153862833976746,
"fcm_dpo/margin": 99.71051025390625,
"fcm_dpo/q_t": 0.3862113058567047,
"grad_norm": 11.172093391418457,
"learning_rate": 8.232468292269479e-08,
"logits/chosen": 0.8066504001617432,
"logits/rejected": 0.7978000044822693,
"logps/chosen": -243.16014099121094,
"logps/ref_chosen": -68.65119934082031,
"logps/ref_rejected": -77.91394805908203,
"logps/rejected": -352.1334228515625,
"loss": 1.0413,
"margin_dpo/margin_mean": 99.71051025390625,
"margin_dpo/margin_std": 116.26593017578125,
"step": 504
},
{
"KL/chosen_KL_mean": -184.58160400390625,
"KL/mean": -219.14266967773438,
"KL/rejected_KL_mean": -253.7037353515625,
"KL/std": 134.02723693847656,
"epoch": 0.763416477702192,
"fcm_dpo/beta": 0.004590325523167849,
"fcm_dpo/delta": -0.03364703059196472,
"fcm_dpo/margin": 69.12216186523438,
"fcm_dpo/q_t": 0.4291911721229553,
"grad_norm": 13.124581336975098,
"learning_rate": 8.134630621352483e-08,
"logits/chosen": 0.8404581546783447,
"logits/rejected": 0.8074555397033691,
"logps/chosen": -244.5804443359375,
"logps/ref_chosen": -59.99884796142578,
"logps/ref_rejected": -76.88048553466797,
"logps/rejected": -330.584228515625,
"loss": 1.2016,
"margin_dpo/margin_mean": 69.12216186523438,
"margin_dpo/margin_std": 145.76710510253906,
"step": 505
},
{
"KL/chosen_KL_mean": -185.93222045898438,
"KL/mean": -222.56521606445312,
"KL/rejected_KL_mean": -259.1982116699219,
"KL/std": 118.15262603759766,
"epoch": 0.764928193499622,
"fcm_dpo/beta": 0.00469227135181427,
"fcm_dpo/delta": 0.06280513852834702,
"fcm_dpo/margin": 73.26598358154297,
"fcm_dpo/q_t": 0.42215216159820557,
"grad_norm": 12.451192855834961,
"learning_rate": 8.037264711071698e-08,
"logits/chosen": 0.8122513890266418,
"logits/rejected": 0.8038109540939331,
"logps/chosen": -256.0035400390625,
"logps/ref_chosen": -70.07130432128906,
"logps/ref_rejected": -82.03775024414062,
"logps/rejected": -341.2359619140625,
"loss": 1.1876,
"margin_dpo/margin_mean": 73.2659912109375,
"margin_dpo/margin_std": 149.51535034179688,
"step": 506
},
{
"KL/chosen_KL_mean": -195.21424865722656,
"KL/mean": -238.96438598632812,
"KL/rejected_KL_mean": -282.71453857421875,
"KL/std": 134.91099548339844,
"epoch": 0.7664399092970522,
"fcm_dpo/beta": 0.004667514935135841,
"fcm_dpo/delta": -0.011594630777835846,
"fcm_dpo/margin": 87.50030517578125,
"fcm_dpo/q_t": 0.41374701261520386,
"grad_norm": 13.628171920776367,
"learning_rate": 7.940373284960933e-08,
"logits/chosen": 0.852962076663971,
"logits/rejected": 0.8136993646621704,
"logps/chosen": -267.2212829589844,
"logps/ref_chosen": -72.00703430175781,
"logps/ref_rejected": -93.94987487792969,
"logps/rejected": -376.6644287109375,
"loss": 1.1472,
"margin_dpo/margin_mean": 87.50030517578125,
"margin_dpo/margin_std": 159.53916931152344,
"step": 507
},
{
"KL/chosen_KL_mean": -180.30194091796875,
"KL/mean": -231.80496215820312,
"KL/rejected_KL_mean": -283.3079833984375,
"KL/std": 141.48910522460938,
"epoch": 0.7679516250944822,
"fcm_dpo/beta": 0.004642902873456478,
"fcm_dpo/delta": -0.0925033912062645,
"fcm_dpo/margin": 103.00605773925781,
"fcm_dpo/q_t": 0.3941517472267151,
"grad_norm": 15.925118446350098,
"learning_rate": 7.843959053281663e-08,
"logits/chosen": 0.8314013481140137,
"logits/rejected": 0.7309713959693909,
"logps/chosen": -240.5218505859375,
"logps/ref_chosen": -60.21992492675781,
"logps/ref_rejected": -95.9200668334961,
"logps/rejected": -379.2280578613281,
"loss": 1.0892,
"margin_dpo/margin_mean": 103.00605773925781,
"margin_dpo/margin_std": 155.87930297851562,
"step": 508
},
{
"KL/chosen_KL_mean": -190.84176635742188,
"KL/mean": -227.35723876953125,
"KL/rejected_KL_mean": -263.8727111816406,
"KL/std": 123.76275634765625,
"epoch": 0.7694633408919124,
"fcm_dpo/beta": 0.004593648016452789,
"fcm_dpo/delta": 0.07191221415996552,
"fcm_dpo/margin": 73.03093719482422,
"fcm_dpo/q_t": 0.42444539070129395,
"grad_norm": 15.941633224487305,
"learning_rate": 7.748024712947204e-08,
"logits/chosen": 0.8122048377990723,
"logits/rejected": 0.7954943776130676,
"logps/chosen": -257.1119384765625,
"logps/ref_chosen": -66.27017211914062,
"logps/ref_rejected": -71.73065185546875,
"logps/rejected": -335.60333251953125,
"loss": 1.1895,
"margin_dpo/margin_mean": 73.03093719482422,
"margin_dpo/margin_std": 148.45220947265625,
"step": 509
},
{
"KL/chosen_KL_mean": -189.88351440429688,
"KL/mean": -238.0792999267578,
"KL/rejected_KL_mean": -286.27508544921875,
"KL/std": 139.2322540283203,
"epoch": 0.7709750566893424,
"fcm_dpo/beta": 0.004529876634478569,
"fcm_dpo/delta": -0.04380728304386139,
"fcm_dpo/margin": 96.39158630371094,
"fcm_dpo/q_t": 0.40820297598838806,
"grad_norm": 14.016731262207031,
"learning_rate": 7.652572947447272e-08,
"logits/chosen": 0.9248769283294678,
"logits/rejected": 0.8472373485565186,
"logps/chosen": -243.4283905029297,
"logps/ref_chosen": -53.54487609863281,
"logps/ref_rejected": -91.36648559570312,
"logps/rejected": -377.6415710449219,
"loss": 1.1419,
"margin_dpo/margin_mean": 96.39158630371094,
"margin_dpo/margin_std": 180.8556365966797,
"step": 510
},
{
"KL/chosen_KL_mean": -177.66510009765625,
"KL/mean": -231.65658569335938,
"KL/rejected_KL_mean": -285.6480712890625,
"KL/std": 121.31975555419922,
"epoch": 0.7724867724867724,
"fcm_dpo/beta": 0.004389429464936256,
"fcm_dpo/delta": -0.08617927134037018,
"fcm_dpo/margin": 107.98297119140625,
"fcm_dpo/q_t": 0.38881558179855347,
"grad_norm": 14.997942924499512,
"learning_rate": 7.557606426772961e-08,
"logits/chosen": 0.8680216670036316,
"logits/rejected": 0.8205208778381348,
"logps/chosen": -233.50949096679688,
"logps/ref_chosen": -55.844383239746094,
"logps/ref_rejected": -86.49819946289062,
"logps/rejected": -372.1462707519531,
"loss": 1.0434,
"margin_dpo/margin_mean": 107.98297119140625,
"margin_dpo/margin_std": 127.46925354003906,
"step": 511
},
{
"KL/chosen_KL_mean": -185.30795288085938,
"KL/mean": -217.4866485595703,
"KL/rejected_KL_mean": -249.66534423828125,
"KL/std": 113.47346496582031,
"epoch": 0.7739984882842026,
"fcm_dpo/beta": 0.004385386127978563,
"fcm_dpo/delta": 0.023245226591825485,
"fcm_dpo/margin": 64.3573989868164,
"fcm_dpo/q_t": 0.43406805396080017,
"grad_norm": 19.79949378967285,
"learning_rate": 7.463127807341966e-08,
"logits/chosen": 0.7447441816329956,
"logits/rejected": 0.7479703426361084,
"logps/chosen": -246.96099853515625,
"logps/ref_chosen": -61.653038024902344,
"logps/ref_rejected": -72.83148193359375,
"logps/rejected": -322.496826171875,
"loss": 1.2125,
"margin_dpo/margin_mean": 64.3573989868164,
"margin_dpo/margin_std": 137.24069213867188,
"step": 512
},
{
"KL/chosen_KL_mean": -166.38235473632812,
"KL/mean": -207.5938720703125,
"KL/rejected_KL_mean": -248.80538940429688,
"KL/std": 113.1106948852539,
"epoch": 0.7755102040816326,
"fcm_dpo/beta": 0.00444423221051693,
"fcm_dpo/delta": 0.03789909556508064,
"fcm_dpo/margin": 82.42303466796875,
"fcm_dpo/q_t": 0.4171144366264343,
"grad_norm": 10.346599578857422,
"learning_rate": 7.369139731924401e-08,
"logits/chosen": 0.9161933660507202,
"logits/rejected": 0.8713808655738831,
"logps/chosen": -217.23492431640625,
"logps/ref_chosen": -50.85256576538086,
"logps/ref_rejected": -69.21754455566406,
"logps/rejected": -318.02294921875,
"loss": 1.1225,
"margin_dpo/margin_mean": 82.42303466796875,
"margin_dpo/margin_std": 122.04438018798828,
"step": 513
},
{
"KL/chosen_KL_mean": -187.51318359375,
"KL/mean": -235.22531127929688,
"KL/rejected_KL_mean": -282.93743896484375,
"KL/std": 125.52702331542969,
"epoch": 0.7770219198790628,
"fcm_dpo/beta": 0.004477824550122023,
"fcm_dpo/delta": -0.03195594251155853,
"fcm_dpo/margin": 95.42424011230469,
"fcm_dpo/q_t": 0.40270817279815674,
"grad_norm": 15.198481559753418,
"learning_rate": 7.275644829568747e-08,
"logits/chosen": 0.8606992959976196,
"logits/rejected": 0.837975263595581,
"logps/chosen": -256.89813232421875,
"logps/ref_chosen": -69.38493347167969,
"logps/ref_rejected": -83.32447814941406,
"logps/rejected": -366.26190185546875,
"loss": 1.0998,
"margin_dpo/margin_mean": 95.42424774169922,
"margin_dpo/margin_std": 142.98733520507812,
"step": 514
},
{
"KL/chosen_KL_mean": -194.24365234375,
"KL/mean": -232.44851684570312,
"KL/rejected_KL_mean": -270.65338134765625,
"KL/std": 117.28639221191406,
"epoch": 0.7785336356764928,
"fcm_dpo/beta": 0.004507084377110004,
"fcm_dpo/delta": 0.062156371772289276,
"fcm_dpo/margin": 76.40972900390625,
"fcm_dpo/q_t": 0.4212613105773926,
"grad_norm": 12.018317222595215,
"learning_rate": 7.182645715528435e-08,
"logits/chosen": 0.8571234345436096,
"logits/rejected": 0.7873523235321045,
"logps/chosen": -247.93069458007812,
"logps/ref_chosen": -53.687034606933594,
"logps/ref_rejected": -83.59614562988281,
"logps/rejected": -354.24951171875,
"loss": 1.1574,
"margin_dpo/margin_mean": 76.40972900390625,
"margin_dpo/margin_std": 133.1450653076172,
"step": 515
},
{
"KL/chosen_KL_mean": -168.87564086914062,
"KL/mean": -206.1563262939453,
"KL/rejected_KL_mean": -243.43699645996094,
"KL/std": 115.80433654785156,
"epoch": 0.780045351473923,
"fcm_dpo/beta": 0.004723469726741314,
"fcm_dpo/delta": 0.05215982347726822,
"fcm_dpo/margin": 74.56133270263672,
"fcm_dpo/q_t": 0.41921383142471313,
"grad_norm": 15.109283447265625,
"learning_rate": 7.090144991188568e-08,
"logits/chosen": 0.8263188600540161,
"logits/rejected": 0.7975659370422363,
"logps/chosen": -225.77737426757812,
"logps/ref_chosen": -56.9017219543457,
"logps/ref_rejected": -67.83477783203125,
"logps/rejected": -311.27178955078125,
"loss": 1.1633,
"margin_dpo/margin_mean": 74.56134033203125,
"margin_dpo/margin_std": 136.02789306640625,
"step": 516
},
{
"KL/chosen_KL_mean": -201.57992553710938,
"KL/mean": -228.9405517578125,
"KL/rejected_KL_mean": -256.3011779785156,
"KL/std": 123.93075561523438,
"epoch": 0.781557067271353,
"fcm_dpo/beta": 0.004781331401318312,
"fcm_dpo/delta": 0.04728348180651665,
"fcm_dpo/margin": 54.72124481201172,
"fcm_dpo/q_t": 0.4420250654220581,
"grad_norm": 13.745712280273438,
"learning_rate": 6.998145243993284e-08,
"logits/chosen": 0.8844262361526489,
"logits/rejected": 0.8897521495819092,
"logps/chosen": -263.3550720214844,
"logps/ref_chosen": -61.775142669677734,
"logps/ref_rejected": -62.88270950317383,
"logps/rejected": -319.18389892578125,
"loss": 1.2287,
"margin_dpo/margin_mean": 54.72125244140625,
"margin_dpo/margin_std": 127.82027435302734,
"step": 517
},
{
"KL/chosen_KL_mean": -173.06509399414062,
"KL/mean": -213.97628784179688,
"KL/rejected_KL_mean": -254.88751220703125,
"KL/std": 121.97323608398438,
"epoch": 0.783068783068783,
"fcm_dpo/beta": 0.004860128741711378,
"fcm_dpo/delta": 0.0026370445266366005,
"fcm_dpo/margin": 81.82240295410156,
"fcm_dpo/q_t": 0.41280555725097656,
"grad_norm": 13.351316452026367,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 0.8314996957778931,
"logits/rejected": 0.799196720123291,
"logps/chosen": -235.09033203125,
"logps/ref_chosen": -62.02523422241211,
"logps/ref_rejected": -79.06085205078125,
"logps/rejected": -333.9483642578125,
"loss": 1.13,
"margin_dpo/margin_mean": 81.82240295410156,
"margin_dpo/margin_std": 138.0740966796875,
"step": 518
},
{
"KL/chosen_KL_mean": -204.05413818359375,
"KL/mean": -227.598388671875,
"KL/rejected_KL_mean": -251.1426239013672,
"KL/std": 121.79615020751953,
"epoch": 0.7845804988662132,
"fcm_dpo/beta": 0.00496459798887372,
"fcm_dpo/delta": 0.07236956804990768,
"fcm_dpo/margin": 47.088470458984375,
"fcm_dpo/q_t": 0.44760170578956604,
"grad_norm": 19.102073669433594,
"learning_rate": 6.815658960673781e-08,
"logits/chosen": 0.8614056706428528,
"logits/rejected": 0.8238772749900818,
"logps/chosen": -265.6605224609375,
"logps/ref_chosen": -61.60636901855469,
"logps/ref_rejected": -74.50727844238281,
"logps/rejected": -325.64990234375,
"loss": 1.3117,
"margin_dpo/margin_mean": 47.088478088378906,
"margin_dpo/margin_std": 159.3134765625,
"step": 519
},
{
"KL/chosen_KL_mean": -189.42742919921875,
"KL/mean": -226.57359313964844,
"KL/rejected_KL_mean": -263.71978759765625,
"KL/std": 123.97309112548828,
"epoch": 0.7860922146636432,
"fcm_dpo/beta": 0.005040885414928198,
"fcm_dpo/delta": 0.02684013918042183,
"fcm_dpo/margin": 74.29235076904297,
"fcm_dpo/q_t": 0.4172334671020508,
"grad_norm": 16.04875373840332,
"learning_rate": 6.725177529083209e-08,
"logits/chosen": 0.9040594100952148,
"logits/rejected": 0.8583179116249084,
"logps/chosen": -252.30084228515625,
"logps/ref_chosen": -62.87343215942383,
"logps/ref_rejected": -76.505615234375,
"logps/rejected": -340.2253723144531,
"loss": 1.1434,
"margin_dpo/margin_mean": 74.29235076904297,
"margin_dpo/margin_std": 126.32000732421875,
"step": 520
},
{
"KL/chosen_KL_mean": -187.01785278320312,
"KL/mean": -239.00657653808594,
"KL/rejected_KL_mean": -290.9953308105469,
"KL/std": 124.35540008544922,
"epoch": 0.7876039304610734,
"fcm_dpo/beta": 0.00489568617194891,
"fcm_dpo/delta": -0.1282760053873062,
"fcm_dpo/margin": 103.97747802734375,
"fcm_dpo/q_t": 0.3860055208206177,
"grad_norm": 11.902304649353027,
"learning_rate": 6.63520728356167e-08,
"logits/chosen": 0.7507075071334839,
"logits/rejected": 0.691694974899292,
"logps/chosen": -251.22451782226562,
"logps/ref_chosen": -64.20668029785156,
"logps/ref_rejected": -92.28083038330078,
"logps/rejected": -383.2761535644531,
"loss": 1.0477,
"margin_dpo/margin_mean": 103.97747802734375,
"margin_dpo/margin_std": 142.90042114257812,
"step": 521
},
{
"KL/chosen_KL_mean": -189.99844360351562,
"KL/mean": -219.89682006835938,
"KL/rejected_KL_mean": -249.79522705078125,
"KL/std": 121.7298583984375,
"epoch": 0.7891156462585034,
"fcm_dpo/beta": 0.004985436797142029,
"fcm_dpo/delta": 0.1123264878988266,
"fcm_dpo/margin": 59.796783447265625,
"fcm_dpo/q_t": 0.43511706590652466,
"grad_norm": 14.604763984680176,
"learning_rate": 6.545750740770336e-08,
"logits/chosen": 0.8380166292190552,
"logits/rejected": 0.8317359089851379,
"logps/chosen": -248.3681640625,
"logps/ref_chosen": -58.369720458984375,
"logps/ref_rejected": -68.79248046875,
"logps/rejected": -318.58770751953125,
"loss": 1.2558,
"margin_dpo/margin_mean": 59.796783447265625,
"margin_dpo/margin_std": 157.91360473632812,
"step": 522
},
{
"KL/chosen_KL_mean": -195.73423767089844,
"KL/mean": -234.60708618164062,
"KL/rejected_KL_mean": -273.47991943359375,
"KL/std": 125.57742309570312,
"epoch": 0.7906273620559335,
"fcm_dpo/beta": 0.005116622429341078,
"fcm_dpo/delta": 0.002492782659828663,
"fcm_dpo/margin": 77.7457046508789,
"fcm_dpo/q_t": 0.4087350368499756,
"grad_norm": 18.966442108154297,
"learning_rate": 6.456810403001012e-08,
"logits/chosen": 0.8302779793739319,
"logits/rejected": 0.728390634059906,
"logps/chosen": -261.4474792480469,
"logps/ref_chosen": -65.71324157714844,
"logps/ref_rejected": -91.98896789550781,
"logps/rejected": -365.4689025878906,
"loss": 1.1459,
"margin_dpo/margin_mean": 77.74569702148438,
"margin_dpo/margin_std": 139.64659118652344,
"step": 523
},
{
"KL/chosen_KL_mean": -173.21641540527344,
"KL/mean": -211.4966278076172,
"KL/rejected_KL_mean": -249.77685546875,
"KL/std": 121.24716186523438,
"epoch": 0.7921390778533636,
"fcm_dpo/beta": 0.005164645612239838,
"fcm_dpo/delta": 0.004730384796857834,
"fcm_dpo/margin": 76.56043243408203,
"fcm_dpo/q_t": 0.4102930724620819,
"grad_norm": 15.195032119750977,
"learning_rate": 6.368388758106134e-08,
"logits/chosen": 0.7700381875038147,
"logits/rejected": 0.7530121803283691,
"logps/chosen": -249.56765747070312,
"logps/ref_chosen": -76.35124969482422,
"logps/ref_rejected": -89.96072387695312,
"logps/rejected": -339.7375793457031,
"loss": 1.1218,
"margin_dpo/margin_mean": 76.5604248046875,
"margin_dpo/margin_std": 122.09478759765625,
"step": 524
},
{
"KL/chosen_KL_mean": -193.4317626953125,
"KL/mean": -225.12921142578125,
"KL/rejected_KL_mean": -256.82666015625,
"KL/std": 117.6169204711914,
"epoch": 0.7936507936507936,
"fcm_dpo/beta": 0.0052588703110814095,
"fcm_dpo/delta": 0.07416895776987076,
"fcm_dpo/margin": 63.3948974609375,
"fcm_dpo/q_t": 0.42921310663223267,
"grad_norm": 19.115665435791016,
"learning_rate": 6.280488279429185e-08,
"logits/chosen": 0.7093310952186584,
"logits/rejected": 0.7094483971595764,
"logps/chosen": -268.92755126953125,
"logps/ref_chosen": -75.49578857421875,
"logps/ref_rejected": -84.04852294921875,
"logps/rejected": -340.87518310546875,
"loss": 1.2039,
"margin_dpo/margin_mean": 63.394893646240234,
"margin_dpo/margin_std": 140.00247192382812,
"step": 525
},
{
"KL/chosen_KL_mean": -198.68052673339844,
"KL/mean": -229.0758819580078,
"KL/rejected_KL_mean": -259.47125244140625,
"KL/std": 122.75657653808594,
"epoch": 0.7951625094482238,
"fcm_dpo/beta": 0.005194986704736948,
"fcm_dpo/delta": -0.052471913397312164,
"fcm_dpo/margin": 60.79069900512695,
"fcm_dpo/q_t": 0.42842093110084534,
"grad_norm": 14.487711906433105,
"learning_rate": 6.193111425735515e-08,
"logits/chosen": 0.8522295951843262,
"logits/rejected": 0.7942419648170471,
"logps/chosen": -259.9729309082031,
"logps/ref_chosen": -61.29241943359375,
"logps/ref_rejected": -82.47763061523438,
"logps/rejected": -341.9488525390625,
"loss": 1.1995,
"margin_dpo/margin_mean": 60.79069900512695,
"margin_dpo/margin_std": 125.36583709716797,
"step": 526
},
{
"KL/chosen_KL_mean": -207.10394287109375,
"KL/mean": -233.09361267089844,
"KL/rejected_KL_mean": -259.0832824707031,
"KL/std": 121.70289611816406,
"epoch": 0.7966742252456538,
"fcm_dpo/beta": 0.005217268131673336,
"fcm_dpo/delta": 0.014235904440283775,
"fcm_dpo/margin": 51.979347229003906,
"fcm_dpo/q_t": 0.44098371267318726,
"grad_norm": 15.067662239074707,
"learning_rate": 6.106260641143546e-08,
"logits/chosen": 0.9322826266288757,
"logits/rejected": 0.8576953411102295,
"logps/chosen": -268.5765686035156,
"logps/ref_chosen": -61.472625732421875,
"logps/ref_rejected": -90.52831268310547,
"logps/rejected": -349.6116027832031,
"loss": 1.2546,
"margin_dpo/margin_mean": 51.979347229003906,
"margin_dpo/margin_std": 137.49215698242188,
"step": 527
},
{
"KL/chosen_KL_mean": -183.95230102539062,
"KL/mean": -214.3380584716797,
"KL/rejected_KL_mean": -244.72381591796875,
"KL/std": 114.09736633300781,
"epoch": 0.7981859410430839,
"fcm_dpo/beta": 0.005392709746956825,
"fcm_dpo/delta": 0.08037621527910233,
"fcm_dpo/margin": 60.771522521972656,
"fcm_dpo/q_t": 0.4303974509239197,
"grad_norm": 17.312894821166992,
"learning_rate": 6.019938355056422e-08,
"logits/chosen": 0.8145349621772766,
"logits/rejected": 0.7549326419830322,
"logps/chosen": -242.7443084716797,
"logps/ref_chosen": -58.792015075683594,
"logps/ref_rejected": -71.82516479492188,
"logps/rejected": -316.5489807128906,
"loss": 1.2365,
"margin_dpo/margin_mean": 60.771522521972656,
"margin_dpo/margin_std": 148.80494689941406,
"step": 528
},
{
"KL/chosen_KL_mean": -178.44883728027344,
"KL/mean": -234.34661865234375,
"KL/rejected_KL_mean": -290.2444152832031,
"KL/std": 112.45913696289062,
"epoch": 0.799697656840514,
"fcm_dpo/beta": 0.005035213194787502,
"fcm_dpo/delta": -0.1965881735086441,
"fcm_dpo/margin": 111.79557800292969,
"fcm_dpo/q_t": 0.3671392798423767,
"grad_norm": 15.271794319152832,
"learning_rate": 5.934146982094049e-08,
"logits/chosen": 0.7759064435958862,
"logits/rejected": 0.7353440523147583,
"logps/chosen": -233.51980590820312,
"logps/ref_chosen": -55.070960998535156,
"logps/ref_rejected": -75.44007873535156,
"logps/rejected": -365.68450927734375,
"loss": 0.9889,
"margin_dpo/margin_mean": 111.79557800292969,
"margin_dpo/margin_std": 122.36137390136719,
"step": 529
},
{
"KL/chosen_KL_mean": -176.23171997070312,
"KL/mean": -214.2334747314453,
"KL/rejected_KL_mean": -252.23526000976562,
"KL/std": 122.73760986328125,
"epoch": 0.8012093726379441,
"fcm_dpo/beta": 0.004925016779452562,
"fcm_dpo/delta": 0.028898127377033234,
"fcm_dpo/margin": 76.00357055664062,
"fcm_dpo/q_t": 0.4183453917503357,
"grad_norm": 16.061771392822266,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 0.8739749193191528,
"logits/rejected": 0.836665153503418,
"logps/chosen": -232.97552490234375,
"logps/ref_chosen": -56.743812561035156,
"logps/ref_rejected": -76.6692123413086,
"logps/rejected": -328.90447998046875,
"loss": 1.1416,
"margin_dpo/margin_mean": 76.00357055664062,
"margin_dpo/margin_std": 129.30343627929688,
"step": 530
},
{
"KL/chosen_KL_mean": -176.384521484375,
"KL/mean": -215.5994873046875,
"KL/rejected_KL_mean": -254.81443786621094,
"KL/std": 120.86744689941406,
"epoch": 0.8027210884353742,
"fcm_dpo/beta": 0.0049905176274478436,
"fcm_dpo/delta": 0.009740887209773064,
"fcm_dpo/margin": 78.42990112304688,
"fcm_dpo/q_t": 0.41260451078414917,
"grad_norm": 13.00410270690918,
"learning_rate": 5.7641665597021435e-08,
"logits/chosen": 0.8380810618400574,
"logits/rejected": 0.7674177885055542,
"logps/chosen": -227.5009765625,
"logps/ref_chosen": -51.116455078125,
"logps/ref_rejected": -79.52884674072266,
"logps/rejected": -334.34326171875,
"loss": 1.1312,
"margin_dpo/margin_mean": 78.42990112304688,
"margin_dpo/margin_std": 130.77659606933594,
"step": 531
},
{
"KL/chosen_KL_mean": -206.09271240234375,
"KL/mean": -245.57955932617188,
"KL/rejected_KL_mean": -285.06646728515625,
"KL/std": 119.67216491699219,
"epoch": 0.8042328042328042,
"fcm_dpo/beta": 0.005016631446778774,
"fcm_dpo/delta": 0.004318161401897669,
"fcm_dpo/margin": 78.97373962402344,
"fcm_dpo/q_t": 0.4110661447048187,
"grad_norm": 14.44060230255127,
"learning_rate": 5.679982264990424e-08,
"logits/chosen": 0.8174869418144226,
"logits/rejected": 0.774748682975769,
"logps/chosen": -264.3726501464844,
"logps/ref_chosen": -58.279945373535156,
"logps/ref_rejected": -78.05426788330078,
"logps/rejected": -363.1207275390625,
"loss": 1.1321,
"margin_dpo/margin_mean": 78.97373962402344,
"margin_dpo/margin_std": 132.41571044921875,
"step": 532
},
{
"KL/chosen_KL_mean": -148.63015747070312,
"KL/mean": -195.14443969726562,
"KL/rejected_KL_mean": -241.65870666503906,
"KL/std": 127.76698303222656,
"epoch": 0.8057445200302343,
"fcm_dpo/beta": 0.004902138374745846,
"fcm_dpo/delta": -0.06478457897901535,
"fcm_dpo/margin": 93.02857208251953,
"fcm_dpo/q_t": 0.39741477370262146,
"grad_norm": 14.071173667907715,
"learning_rate": 5.596338392706076e-08,
"logits/chosen": 0.9038737416267395,
"logits/rejected": 0.8549951314926147,
"logps/chosen": -205.04815673828125,
"logps/ref_chosen": -56.41801071166992,
"logps/ref_rejected": -73.89324951171875,
"logps/rejected": -315.55194091796875,
"loss": 1.0814,
"margin_dpo/margin_mean": 93.02857971191406,
"margin_dpo/margin_std": 134.64431762695312,
"step": 533
},
{
"KL/chosen_KL_mean": -185.22732543945312,
"KL/mean": -220.9058837890625,
"KL/rejected_KL_mean": -256.58441162109375,
"KL/std": 115.06756591796875,
"epoch": 0.8072562358276644,
"fcm_dpo/beta": 0.004894358105957508,
"fcm_dpo/delta": 0.0565091148018837,
"fcm_dpo/margin": 71.35710144042969,
"fcm_dpo/q_t": 0.423191636800766,
"grad_norm": 13.81146240234375,
"learning_rate": 5.513237282548033e-08,
"logits/chosen": 0.816210150718689,
"logits/rejected": 0.7883972525596619,
"logps/chosen": -245.97601318359375,
"logps/ref_chosen": -60.748687744140625,
"logps/ref_rejected": -73.8623046875,
"logps/rejected": -330.44671630859375,
"loss": 1.1783,
"margin_dpo/margin_mean": 71.35710144042969,
"margin_dpo/margin_std": 140.3214111328125,
"step": 534
},
{
"KL/chosen_KL_mean": -196.07479858398438,
"KL/mean": -230.17123413085938,
"KL/rejected_KL_mean": -264.26763916015625,
"KL/std": 123.80078887939453,
"epoch": 0.8087679516250945,
"fcm_dpo/beta": 0.005145275965332985,
"fcm_dpo/delta": 0.054210808128118515,
"fcm_dpo/margin": 68.1928482055664,
"fcm_dpo/q_t": 0.4239533543586731,
"grad_norm": 15.887540817260742,
"learning_rate": 5.430681259032957e-08,
"logits/chosen": 0.7688905000686646,
"logits/rejected": 0.7201669812202454,
"logps/chosen": -257.71221923828125,
"logps/ref_chosen": -61.637413024902344,
"logps/ref_rejected": -80.93138885498047,
"logps/rejected": -345.19903564453125,
"loss": 1.1858,
"margin_dpo/margin_mean": 68.1928482055664,
"margin_dpo/margin_std": 138.06951904296875,
"step": 535
},
{
"KL/chosen_KL_mean": -169.49693298339844,
"KL/mean": -222.1195068359375,
"KL/rejected_KL_mean": -274.7420959472656,
"KL/std": 123.30923461914062,
"epoch": 0.8102796674225246,
"fcm_dpo/beta": 0.004823877010494471,
"fcm_dpo/delta": -0.12901368737220764,
"fcm_dpo/margin": 105.24514770507812,
"fcm_dpo/q_t": 0.3823744058609009,
"grad_norm": 10.788253784179688,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 0.8770267963409424,
"logits/rejected": 0.8094578981399536,
"logps/chosen": -221.3859100341797,
"logps/ref_chosen": -51.88897705078125,
"logps/ref_rejected": -73.34864044189453,
"logps/rejected": -348.0907287597656,
"loss": 1.0211,
"margin_dpo/margin_mean": 105.24514770507812,
"margin_dpo/margin_std": 122.67505645751953,
"step": 536
},
{
"KL/chosen_KL_mean": -192.197509765625,
"KL/mean": -238.49530029296875,
"KL/rejected_KL_mean": -284.7930603027344,
"KL/std": 133.47552490234375,
"epoch": 0.8117913832199547,
"fcm_dpo/beta": 0.004660390317440033,
"fcm_dpo/delta": -0.03684820234775543,
"fcm_dpo/margin": 92.59556579589844,
"fcm_dpo/q_t": 0.4039597511291504,
"grad_norm": 13.52630615234375,
"learning_rate": 5.267213693697695e-08,
"logits/chosen": 0.9239789843559265,
"logits/rejected": 0.8453102707862854,
"logps/chosen": -246.44613647460938,
"logps/ref_chosen": -54.248619079589844,
"logps/ref_rejected": -94.94343566894531,
"logps/rejected": -379.73651123046875,
"loss": 1.1087,
"margin_dpo/margin_mean": 92.59556579589844,
"margin_dpo/margin_std": 146.90977478027344,
"step": 537
},
{
"KL/chosen_KL_mean": -195.64366149902344,
"KL/mean": -239.24957275390625,
"KL/rejected_KL_mean": -282.8554992675781,
"KL/std": 131.49520874023438,
"epoch": 0.8133030990173847,
"fcm_dpo/beta": 0.004660367965698242,
"fcm_dpo/delta": -0.00734717445448041,
"fcm_dpo/margin": 87.21183776855469,
"fcm_dpo/q_t": 0.407853901386261,
"grad_norm": 12.92613697052002,
"learning_rate": 5.1863067244167144e-08,
"logits/chosen": 0.8298152685165405,
"logits/rejected": 0.8131669759750366,
"logps/chosen": -265.7371826171875,
"logps/ref_chosen": -70.09353637695312,
"logps/ref_rejected": -79.49833679199219,
"logps/rejected": -362.35382080078125,
"loss": 1.1085,
"margin_dpo/margin_mean": 87.21183776855469,
"margin_dpo/margin_std": 133.052001953125,
"step": 538
},
{
"KL/chosen_KL_mean": -199.86004638671875,
"KL/mean": -234.01394653320312,
"KL/rejected_KL_mean": -268.1678466796875,
"KL/std": 120.2940902709961,
"epoch": 0.8148148148148148,
"fcm_dpo/beta": 0.004803318530321121,
"fcm_dpo/delta": 0.0798788070678711,
"fcm_dpo/margin": 68.30778503417969,
"fcm_dpo/q_t": 0.43010759353637695,
"grad_norm": 14.349104881286621,
"learning_rate": 5.105953986729195e-08,
"logits/chosen": 0.8133977651596069,
"logits/rejected": 0.7414849400520325,
"logps/chosen": -261.791748046875,
"logps/ref_chosen": -61.93169403076172,
"logps/ref_rejected": -84.08946228027344,
"logps/rejected": -352.25732421875,
"loss": 1.1762,
"margin_dpo/margin_mean": 68.30778503417969,
"margin_dpo/margin_std": 132.4424591064453,
"step": 539
},
{
"KL/chosen_KL_mean": -183.73110961914062,
"KL/mean": -237.5785369873047,
"KL/rejected_KL_mean": -291.42596435546875,
"KL/std": 140.208251953125,
"epoch": 0.8163265306122449,
"fcm_dpo/beta": 0.004663660190999508,
"fcm_dpo/delta": -0.11989345401525497,
"fcm_dpo/margin": 107.69487762451172,
"fcm_dpo/q_t": 0.3844473659992218,
"grad_norm": 12.575929641723633,
"learning_rate": 5.026157728273966e-08,
"logits/chosen": 0.8718093633651733,
"logits/rejected": 0.7981296181678772,
"logps/chosen": -246.4353485107422,
"logps/ref_chosen": -62.704254150390625,
"logps/ref_rejected": -95.63597106933594,
"logps/rejected": -387.06195068359375,
"loss": 1.0385,
"margin_dpo/margin_mean": 107.69487762451172,
"margin_dpo/margin_std": 134.9456787109375,
"step": 540
},
{
"KL/chosen_KL_mean": -179.12994384765625,
"KL/mean": -221.66107177734375,
"KL/rejected_KL_mean": -264.1922302246094,
"KL/std": 119.99224853515625,
"epoch": 0.817838246409675,
"fcm_dpo/beta": 0.004524096846580505,
"fcm_dpo/delta": 0.016618637368083,
"fcm_dpo/margin": 85.06229400634766,
"fcm_dpo/q_t": 0.41030341386795044,
"grad_norm": 11.988428115844727,
"learning_rate": 4.9469201811239035e-08,
"logits/chosen": 0.8793637752532959,
"logits/rejected": 0.9092428684234619,
"logps/chosen": -241.61077880859375,
"logps/ref_chosen": -62.48084259033203,
"logps/ref_rejected": -57.55541229248047,
"logps/rejected": -321.74761962890625,
"loss": 1.1052,
"margin_dpo/margin_mean": 85.06229400634766,
"margin_dpo/margin_std": 115.75038146972656,
"step": 541
},
{
"KL/chosen_KL_mean": -155.74496459960938,
"KL/mean": -203.70249938964844,
"KL/rejected_KL_mean": -251.66004943847656,
"KL/std": 112.92686462402344,
"epoch": 0.8193499622071051,
"fcm_dpo/beta": 0.004484962671995163,
"fcm_dpo/delta": -0.03508245199918747,
"fcm_dpo/margin": 95.91510772705078,
"fcm_dpo/q_t": 0.40033650398254395,
"grad_norm": 11.718820571899414,
"learning_rate": 4.868243561723534e-08,
"logits/chosen": 0.9004342555999756,
"logits/rejected": 0.8623563051223755,
"logps/chosen": -205.19984436035156,
"logps/ref_chosen": -49.454891204833984,
"logps/ref_rejected": -65.33275604248047,
"logps/rejected": -316.9927978515625,
"loss": 1.0898,
"margin_dpo/margin_mean": 95.91511535644531,
"margin_dpo/margin_std": 136.8139190673828,
"step": 542
},
{
"KL/chosen_KL_mean": -170.90618896484375,
"KL/mean": -218.6575164794922,
"KL/rejected_KL_mean": -266.4088134765625,
"KL/std": 120.44759368896484,
"epoch": 0.8208616780045351,
"fcm_dpo/beta": 0.0044562616385519505,
"fcm_dpo/delta": -0.029431413859128952,
"fcm_dpo/margin": 95.50263977050781,
"fcm_dpo/q_t": 0.40224945545196533,
"grad_norm": 10.733229637145996,
"learning_rate": 4.790130070827028e-08,
"logits/chosen": 0.8563024997711182,
"logits/rejected": 0.7859128713607788,
"logps/chosen": -222.00704956054688,
"logps/ref_chosen": -51.100860595703125,
"logps/ref_rejected": -76.06130981445312,
"logps/rejected": -342.47015380859375,
"loss": 1.0909,
"margin_dpo/margin_mean": 95.50263977050781,
"margin_dpo/margin_std": 133.79779052734375,
"step": 543
},
{
"KL/chosen_KL_mean": -183.34523010253906,
"KL/mean": -237.28253173828125,
"KL/rejected_KL_mean": -291.21978759765625,
"KL/std": 133.13616943359375,
"epoch": 0.8223733938019653,
"fcm_dpo/beta": 0.004260823130607605,
"fcm_dpo/delta": -0.06947409361600876,
"fcm_dpo/margin": 107.87458038330078,
"fcm_dpo/q_t": 0.39583128690719604,
"grad_norm": 13.217146873474121,
"learning_rate": 4.7125818934366454e-08,
"logits/chosen": 0.8630880117416382,
"logits/rejected": 0.7988213300704956,
"logps/chosen": -243.62246704101562,
"logps/ref_chosen": -60.2772331237793,
"logps/ref_rejected": -88.40553283691406,
"logps/rejected": -379.6253356933594,
"loss": 1.0805,
"margin_dpo/margin_mean": 107.87457275390625,
"margin_dpo/margin_std": 156.6991424560547,
"step": 544
},
{
"KL/chosen_KL_mean": -197.96246337890625,
"KL/mean": -227.96221923828125,
"KL/rejected_KL_mean": -257.9619445800781,
"KL/std": 122.77239990234375,
"epoch": 0.8238851095993953,
"fcm_dpo/beta": 0.0045236810110509396,
"fcm_dpo/delta": 0.13886934518814087,
"fcm_dpo/margin": 59.999481201171875,
"fcm_dpo/q_t": 0.4393478035926819,
"grad_norm": 13.92799186706543,
"learning_rate": 4.635601198741607e-08,
"logits/chosen": 0.8469263315200806,
"logits/rejected": 0.7915807366371155,
"logps/chosen": -259.57769775390625,
"logps/ref_chosen": -61.61524963378906,
"logps/ref_rejected": -78.71266174316406,
"logps/rejected": -336.67462158203125,
"loss": 1.2181,
"margin_dpo/margin_mean": 59.99948501586914,
"margin_dpo/margin_std": 132.14329528808594,
"step": 545
},
{
"KL/chosen_KL_mean": -189.22254943847656,
"KL/mean": -221.83804321289062,
"KL/rejected_KL_mean": -254.45355224609375,
"KL/std": 116.31465911865234,
"epoch": 0.8253968253968254,
"fcm_dpo/beta": 0.004792468622326851,
"fcm_dpo/delta": 0.09647992998361588,
"fcm_dpo/margin": 65.2309799194336,
"fcm_dpo/q_t": 0.4293987452983856,
"grad_norm": 18.117361068725586,
"learning_rate": 4.559190140057428e-08,
"logits/chosen": 0.957390308380127,
"logits/rejected": 0.95533686876297,
"logps/chosen": -248.5358123779297,
"logps/ref_chosen": -59.313262939453125,
"logps/ref_rejected": -64.73631286621094,
"logps/rejected": -319.1898498535156,
"loss": 1.209,
"margin_dpo/margin_mean": 65.23097229003906,
"margin_dpo/margin_std": 142.85653686523438,
"step": 546
},
{
"KL/chosen_KL_mean": -165.288330078125,
"KL/mean": -215.5997772216797,
"KL/rejected_KL_mean": -265.9112243652344,
"KL/std": 121.85418701171875,
"epoch": 0.8269085411942555,
"fcm_dpo/beta": 0.004634900949895382,
"fcm_dpo/delta": -0.07973036170005798,
"fcm_dpo/margin": 100.62291717529297,
"fcm_dpo/q_t": 0.3919479250907898,
"grad_norm": 12.286870002746582,
"learning_rate": 4.483350854765672e-08,
"logits/chosen": 0.7986713647842407,
"logits/rejected": 0.7496638298034668,
"logps/chosen": -220.26507568359375,
"logps/ref_chosen": -54.97674560546875,
"logps/ref_rejected": -75.35922241210938,
"logps/rejected": -341.27044677734375,
"loss": 1.0699,
"margin_dpo/margin_mean": 100.6229248046875,
"margin_dpo/margin_std": 139.28582763671875,
"step": 547
},
{
"KL/chosen_KL_mean": -196.86660766601562,
"KL/mean": -226.04827880859375,
"KL/rejected_KL_mean": -255.2299346923828,
"KL/std": 120.6140365600586,
"epoch": 0.8284202569916855,
"fcm_dpo/beta": 0.004941076040267944,
"fcm_dpo/delta": 0.12047231942415237,
"fcm_dpo/margin": 58.363319396972656,
"fcm_dpo/q_t": 0.43659114837646484,
"grad_norm": 16.297290802001953,
"learning_rate": 4.4080854642541826e-08,
"logits/chosen": 0.8177128434181213,
"logits/rejected": 0.7706683278083801,
"logps/chosen": -260.0772705078125,
"logps/ref_chosen": -63.21067428588867,
"logps/ref_rejected": -81.23347473144531,
"logps/rejected": -336.4634094238281,
"loss": 1.2194,
"margin_dpo/margin_mean": 58.36331558227539,
"margin_dpo/margin_std": 132.25997924804688,
"step": 548
},
{
"KL/chosen_KL_mean": -194.89645385742188,
"KL/mean": -231.360595703125,
"KL/rejected_KL_mean": -267.8247375488281,
"KL/std": 134.2811737060547,
"epoch": 0.8299319727891157,
"fcm_dpo/beta": 0.005007403437048197,
"fcm_dpo/delta": 0.03835400193929672,
"fcm_dpo/margin": 72.92826080322266,
"fcm_dpo/q_t": 0.4223543405532837,
"grad_norm": 18.85927963256836,
"learning_rate": 4.333396073857723e-08,
"logits/chosen": 0.9243029952049255,
"logits/rejected": 0.8670613765716553,
"logps/chosen": -259.16998291015625,
"logps/ref_chosen": -64.27351379394531,
"logps/ref_rejected": -92.31663513183594,
"logps/rejected": -360.141357421875,
"loss": 1.199,
"margin_dpo/margin_mean": 72.92826080322266,
"margin_dpo/margin_std": 159.06471252441406,
"step": 549
},
{
"KL/chosen_KL_mean": -193.56687927246094,
"KL/mean": -220.8571319580078,
"KL/rejected_KL_mean": -248.14736938476562,
"KL/std": 117.43412017822266,
"epoch": 0.8314436885865457,
"fcm_dpo/beta": 0.005119511857628822,
"fcm_dpo/delta": 0.004839625675231218,
"fcm_dpo/margin": 54.58049392700195,
"fcm_dpo/q_t": 0.43776458501815796,
"grad_norm": 15.661591529846191,
"learning_rate": 4.259284772799099e-08,
"logits/chosen": 0.8754514455795288,
"logits/rejected": 0.8566372990608215,
"logps/chosen": -249.7973175048828,
"logps/ref_chosen": -56.230438232421875,
"logps/ref_rejected": -62.59788513183594,
"logps/rejected": -310.7452697753906,
"loss": 1.2314,
"margin_dpo/margin_mean": 54.58049392700195,
"margin_dpo/margin_std": 128.6197509765625,
"step": 550
},
{
"KL/chosen_KL_mean": -201.9175262451172,
"KL/mean": -235.73777770996094,
"KL/rejected_KL_mean": -269.55804443359375,
"KL/std": 127.38159942626953,
"epoch": 0.8329554043839759,
"fcm_dpo/beta": 0.0052591776475310326,
"fcm_dpo/delta": 0.0491047166287899,
"fcm_dpo/margin": 67.64051055908203,
"fcm_dpo/q_t": 0.42484039068222046,
"grad_norm": 14.79029369354248,
"learning_rate": 4.1857536341307176e-08,
"logits/chosen": 0.8869715929031372,
"logits/rejected": 0.8607189655303955,
"logps/chosen": -269.66473388671875,
"logps/ref_chosen": -67.74720764160156,
"logps/ref_rejected": -87.04285430908203,
"logps/rejected": -356.60089111328125,
"loss": 1.1578,
"margin_dpo/margin_mean": 67.64051818847656,
"margin_dpo/margin_std": 126.78274536132812,
"step": 551
},
{
"KL/chosen_KL_mean": -186.17919921875,
"KL/mean": -225.56101989746094,
"KL/rejected_KL_mean": -264.94281005859375,
"KL/std": 130.92601013183594,
"epoch": 0.8344671201814059,
"fcm_dpo/beta": 0.005316844675689936,
"fcm_dpo/delta": -0.023466050624847412,
"fcm_dpo/margin": 78.76361083984375,
"fcm_dpo/q_t": 0.4041834771633148,
"grad_norm": 18.497468948364258,
"learning_rate": 4.112804714676593e-08,
"logits/chosen": 0.851292073726654,
"logits/rejected": 0.8060963153839111,
"logps/chosen": -249.10546875,
"logps/ref_chosen": -62.92625427246094,
"logps/ref_rejected": -82.98365783691406,
"logps/rejected": -347.92645263671875,
"loss": 1.1122,
"margin_dpo/margin_mean": 78.76361083984375,
"margin_dpo/margin_std": 122.3577651977539,
"step": 552
},
{
"KL/chosen_KL_mean": -201.0130615234375,
"KL/mean": -239.95428466796875,
"KL/rejected_KL_mean": -278.8955078125,
"KL/std": 127.23973083496094,
"epoch": 0.8359788359788359,
"fcm_dpo/beta": 0.005192439071834087,
"fcm_dpo/delta": -0.0050092036835849285,
"fcm_dpo/margin": 77.88245391845703,
"fcm_dpo/q_t": 0.41371631622314453,
"grad_norm": 14.231207847595215,
"learning_rate": 4.0404400549748144e-08,
"logits/chosen": 0.838943362236023,
"logits/rejected": 0.7557328939437866,
"logps/chosen": -257.0515441894531,
"logps/ref_chosen": -56.038490295410156,
"logps/ref_rejected": -84.48454284667969,
"logps/rejected": -363.3800354003906,
"loss": 1.158,
"margin_dpo/margin_mean": 77.8824462890625,
"margin_dpo/margin_std": 150.22286987304688,
"step": 553
},
{
"KL/chosen_KL_mean": -182.27084350585938,
"KL/mean": -227.0911865234375,
"KL/rejected_KL_mean": -271.91156005859375,
"KL/std": 118.93246459960938,
"epoch": 0.8374905517762661,
"fcm_dpo/beta": 0.0050364104099571705,
"fcm_dpo/delta": -0.05972806736826897,
"fcm_dpo/margin": 89.64070129394531,
"fcm_dpo/q_t": 0.39440518617630005,
"grad_norm": 14.413543701171875,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 0.8107480406761169,
"logits/rejected": 0.8082457780838013,
"logps/chosen": -246.80142211914062,
"logps/ref_chosen": -64.53059387207031,
"logps/ref_rejected": -71.2155990600586,
"logps/rejected": -343.12713623046875,
"loss": 1.0842,
"margin_dpo/margin_mean": 89.64070129394531,
"margin_dpo/margin_std": 128.74749755859375,
"step": 554
},
{
"KL/chosen_KL_mean": -196.64315795898438,
"KL/mean": -230.64242553710938,
"KL/rejected_KL_mean": -264.6416931152344,
"KL/std": 117.89613342285156,
"epoch": 0.8390022675736961,
"fcm_dpo/beta": 0.005218766164034605,
"fcm_dpo/delta": 0.04598980396986008,
"fcm_dpo/margin": 67.9985580444336,
"fcm_dpo/q_t": 0.4178510904312134,
"grad_norm": 15.634132385253906,
"learning_rate": 3.89747159520904e-08,
"logits/chosen": 0.8837528228759766,
"logits/rejected": 0.8687556982040405,
"logps/chosen": -263.2950744628906,
"logps/ref_chosen": -66.65191650390625,
"logps/ref_rejected": -68.6667251586914,
"logps/rejected": -333.30841064453125,
"loss": 1.1975,
"margin_dpo/margin_mean": 67.9985580444336,
"margin_dpo/margin_std": 140.62289428710938,
"step": 555
},
{
"KL/chosen_KL_mean": -193.26910400390625,
"KL/mean": -228.2431640625,
"KL/rejected_KL_mean": -263.21722412109375,
"KL/std": 122.90667724609375,
"epoch": 0.8405139833711263,
"fcm_dpo/beta": 0.005200220271945,
"fcm_dpo/delta": 0.040658943355083466,
"fcm_dpo/margin": 69.94815063476562,
"fcm_dpo/q_t": 0.42058825492858887,
"grad_norm": 14.569718360900879,
"learning_rate": 3.826871794280192e-08,
"logits/chosen": 0.8939093351364136,
"logits/rejected": 0.8580869436264038,
"logps/chosen": -246.10147094726562,
"logps/ref_chosen": -52.832366943359375,
"logps/ref_rejected": -64.49044036865234,
"logps/rejected": -327.7076721191406,
"loss": 1.173,
"margin_dpo/margin_mean": 69.94815826416016,
"margin_dpo/margin_std": 135.2369842529297,
"step": 556
},
{
"KL/chosen_KL_mean": -191.77627563476562,
"KL/mean": -241.3262939453125,
"KL/rejected_KL_mean": -290.87628173828125,
"KL/std": 122.63204193115234,
"epoch": 0.8420256991685563,
"fcm_dpo/beta": 0.004956225864589214,
"fcm_dpo/delta": -0.10954894870519638,
"fcm_dpo/margin": 99.10000610351562,
"fcm_dpo/q_t": 0.38677504658699036,
"grad_norm": 12.53800106048584,
"learning_rate": 3.756864251262143e-08,
"logits/chosen": 0.9151580929756165,
"logits/rejected": 0.8559887409210205,
"logps/chosen": -246.81227111816406,
"logps/ref_chosen": -55.03598403930664,
"logps/ref_rejected": -75.80644989013672,
"logps/rejected": -366.6827392578125,
"loss": 1.0388,
"margin_dpo/margin_mean": 99.10000610351562,
"margin_dpo/margin_std": 120.31071472167969,
"step": 557
},
{
"KL/chosen_KL_mean": -180.4897003173828,
"KL/mean": -234.2262420654297,
"KL/rejected_KL_mean": -287.9627685546875,
"KL/std": 130.8162078857422,
"epoch": 0.8435374149659864,
"fcm_dpo/beta": 0.00461628008633852,
"fcm_dpo/delta": -0.11640501767396927,
"fcm_dpo/margin": 107.4730453491211,
"fcm_dpo/q_t": 0.38629502058029175,
"grad_norm": 10.66162395477295,
"learning_rate": 3.687450924416341e-08,
"logits/chosen": 0.8702242970466614,
"logits/rejected": 0.8275398015975952,
"logps/chosen": -243.716064453125,
"logps/ref_chosen": -63.226348876953125,
"logps/ref_rejected": -91.46881866455078,
"logps/rejected": -379.43157958984375,
"loss": 1.0436,
"margin_dpo/margin_mean": 107.4730453491211,
"margin_dpo/margin_std": 138.13763427734375,
"step": 558
},
{
"KL/chosen_KL_mean": -184.31173706054688,
"KL/mean": -231.5626678466797,
"KL/rejected_KL_mean": -278.8135986328125,
"KL/std": 130.44276428222656,
"epoch": 0.8450491307634165,
"fcm_dpo/beta": 0.004464943893253803,
"fcm_dpo/delta": -0.027830161154270172,
"fcm_dpo/margin": 94.5018310546875,
"fcm_dpo/q_t": 0.408642053604126,
"grad_norm": 11.145119667053223,
"learning_rate": 3.6186337553827743e-08,
"logits/chosen": 0.8411096334457397,
"logits/rejected": 0.7925565242767334,
"logps/chosen": -245.83338928222656,
"logps/ref_chosen": -61.521644592285156,
"logps/ref_rejected": -82.83859252929688,
"logps/rejected": -361.65216064453125,
"loss": 1.1223,
"margin_dpo/margin_mean": 94.5018310546875,
"margin_dpo/margin_std": 158.6476287841797,
"step": 559
},
{
"KL/chosen_KL_mean": -199.8177490234375,
"KL/mean": -244.57350158691406,
"KL/rejected_KL_mean": -289.3292541503906,
"KL/std": 123.44962310791016,
"epoch": 0.8465608465608465,
"fcm_dpo/beta": 0.004556549713015556,
"fcm_dpo/delta": -0.009785711765289307,
"fcm_dpo/margin": 89.5114974975586,
"fcm_dpo/q_t": 0.4058662950992584,
"grad_norm": 16.259973526000977,
"learning_rate": 3.550414669125573e-08,
"logits/chosen": 0.8841683864593506,
"logits/rejected": 0.8540163040161133,
"logps/chosen": -260.458984375,
"logps/ref_chosen": -60.64122009277344,
"logps/ref_rejected": -78.75474548339844,
"logps/rejected": -368.083984375,
"loss": 1.0881,
"margin_dpo/margin_mean": 89.5114974975586,
"margin_dpo/margin_std": 116.04913330078125,
"step": 560
},
{
"KL/chosen_KL_mean": -187.08404541015625,
"KL/mean": -225.16224670410156,
"KL/rejected_KL_mean": -263.2404479980469,
"KL/std": 119.03514099121094,
"epoch": 0.8480725623582767,
"fcm_dpo/beta": 0.004581731744110584,
"fcm_dpo/delta": 0.05710749328136444,
"fcm_dpo/margin": 76.15641784667969,
"fcm_dpo/q_t": 0.42259570956230164,
"grad_norm": 11.56666374206543,
"learning_rate": 3.482795573879241e-08,
"logits/chosen": 0.8361924886703491,
"logits/rejected": 0.8081060647964478,
"logps/chosen": -249.5826416015625,
"logps/ref_chosen": -62.49859619140625,
"logps/ref_rejected": -78.72064208984375,
"logps/rejected": -341.9610900878906,
"loss": 1.1554,
"margin_dpo/margin_mean": 76.15641784667969,
"margin_dpo/margin_std": 131.91342163085938,
"step": 561
},
{
"KL/chosen_KL_mean": -197.9278564453125,
"KL/mean": -250.18325805664062,
"KL/rejected_KL_mean": -302.4386901855469,
"KL/std": 145.4912109375,
"epoch": 0.8495842781557067,
"fcm_dpo/beta": 0.004399956203997135,
"fcm_dpo/delta": -0.07504507899284363,
"fcm_dpo/margin": 104.51081848144531,
"fcm_dpo/q_t": 0.3958006501197815,
"grad_norm": 14.884467124938965,
"learning_rate": 3.415778361095226e-08,
"logits/chosen": 0.8424015045166016,
"logits/rejected": 0.820836067199707,
"logps/chosen": -272.7095947265625,
"logps/ref_chosen": -74.78173828125,
"logps/ref_rejected": -92.63499450683594,
"logps/rejected": -395.07366943359375,
"loss": 1.0671,
"margin_dpo/margin_mean": 104.51081848144531,
"margin_dpo/margin_std": 139.0022430419922,
"step": 562
},
{
"KL/chosen_KL_mean": -164.1915283203125,
"KL/mean": -207.3955535888672,
"KL/rejected_KL_mean": -250.5995635986328,
"KL/std": 106.54350280761719,
"epoch": 0.8510959939531368,
"fcm_dpo/beta": 0.004485203884541988,
"fcm_dpo/delta": 0.01402625348418951,
"fcm_dpo/margin": 86.40805053710938,
"fcm_dpo/q_t": 0.41050225496292114,
"grad_norm": 18.291183471679688,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.9199409484863281,
"logits/rejected": 0.8749498128890991,
"logps/chosen": -214.39004516601562,
"logps/ref_chosen": -50.19850158691406,
"logps/ref_rejected": -66.76687622070312,
"logps/rejected": -317.366455078125,
"loss": 1.1242,
"margin_dpo/margin_mean": 86.40804290771484,
"margin_dpo/margin_std": 135.92982482910156,
"step": 563
},
{
"KL/chosen_KL_mean": -165.2862091064453,
"KL/mean": -219.43699645996094,
"KL/rejected_KL_mean": -273.5877685546875,
"KL/std": 115.89987182617188,
"epoch": 0.8526077097505669,
"fcm_dpo/beta": 0.004321890417486429,
"fcm_dpo/delta": -0.07944278419017792,
"fcm_dpo/margin": 108.30156707763672,
"fcm_dpo/q_t": 0.3904789388179779,
"grad_norm": 11.61489486694336,
"learning_rate": 3.283557064487785e-08,
"logits/chosen": 0.8549599051475525,
"logits/rejected": 0.8323065042495728,
"logps/chosen": -221.02706909179688,
"logps/ref_chosen": -55.7408447265625,
"logps/ref_rejected": -74.82323455810547,
"logps/rejected": -348.4110107421875,
"loss": 1.0597,
"margin_dpo/margin_mean": 108.30156707763672,
"margin_dpo/margin_std": 141.85757446289062,
"step": 564
},
{
"KL/chosen_KL_mean": -198.53341674804688,
"KL/mean": -242.3411865234375,
"KL/rejected_KL_mean": -286.14898681640625,
"KL/std": 123.26646423339844,
"epoch": 0.854119425547997,
"fcm_dpo/beta": 0.004352842457592487,
"fcm_dpo/delta": 0.02035663276910782,
"fcm_dpo/margin": 87.61554718017578,
"fcm_dpo/q_t": 0.4115191698074341,
"grad_norm": 13.795531272888184,
"learning_rate": 3.218356679178252e-08,
"logits/chosen": 0.886420488357544,
"logits/rejected": 0.8450945615768433,
"logps/chosen": -256.87078857421875,
"logps/ref_chosen": -58.33738327026367,
"logps/ref_rejected": -78.31776428222656,
"logps/rejected": -364.46673583984375,
"loss": 1.1088,
"margin_dpo/margin_mean": 87.61554718017578,
"margin_dpo/margin_std": 122.73006439208984,
"step": 565
},
{
"KL/chosen_KL_mean": -189.62576293945312,
"KL/mean": -228.21856689453125,
"KL/rejected_KL_mean": -266.81134033203125,
"KL/std": 130.59353637695312,
"epoch": 0.8556311413454271,
"fcm_dpo/beta": 0.004506157711148262,
"fcm_dpo/delta": 0.055811285972595215,
"fcm_dpo/margin": 77.18556213378906,
"fcm_dpo/q_t": 0.4241322875022888,
"grad_norm": 12.119365692138672,
"learning_rate": 3.1537655732553764e-08,
"logits/chosen": 0.8291478157043457,
"logits/rejected": 0.8274587392807007,
"logps/chosen": -260.8495178222656,
"logps/ref_chosen": -71.22373962402344,
"logps/ref_rejected": -71.11601257324219,
"logps/rejected": -337.9273681640625,
"loss": 1.1956,
"margin_dpo/margin_mean": 77.18555450439453,
"margin_dpo/margin_std": 164.42337036132812,
"step": 566
},
{
"KL/chosen_KL_mean": -179.55332946777344,
"KL/mean": -228.03167724609375,
"KL/rejected_KL_mean": -276.5100402832031,
"KL/std": 120.30104064941406,
"epoch": 0.8571428571428571,
"fcm_dpo/beta": 0.004341046325862408,
"fcm_dpo/delta": -0.02707264944911003,
"fcm_dpo/margin": 96.95671844482422,
"fcm_dpo/q_t": 0.4018208384513855,
"grad_norm": 10.208983421325684,
"learning_rate": 3.089785553471233e-08,
"logits/chosen": 0.8846025466918945,
"logits/rejected": 0.8115559816360474,
"logps/chosen": -232.22259521484375,
"logps/ref_chosen": -52.669273376464844,
"logps/ref_rejected": -74.34785461425781,
"logps/rejected": -350.85791015625,
"loss": 1.0904,
"margin_dpo/margin_mean": 96.95671081542969,
"margin_dpo/margin_std": 131.9867401123047,
"step": 567
},
{
"KL/chosen_KL_mean": -167.61322021484375,
"KL/mean": -225.8235626220703,
"KL/rejected_KL_mean": -284.0339050292969,
"KL/std": 128.29510498046875,
"epoch": 0.8586545729402872,
"fcm_dpo/beta": 0.0042817965149879456,
"fcm_dpo/delta": -0.1166883111000061,
"fcm_dpo/margin": 116.42066955566406,
"fcm_dpo/q_t": 0.3852314352989197,
"grad_norm": 13.811408042907715,
"learning_rate": 3.026418409484513e-08,
"logits/chosen": 0.8656144738197327,
"logits/rejected": 0.7998012900352478,
"logps/chosen": -219.79122924804688,
"logps/ref_chosen": -52.178001403808594,
"logps/ref_rejected": -85.8277587890625,
"logps/rejected": -369.8616638183594,
"loss": 1.0303,
"margin_dpo/margin_mean": 116.42066955566406,
"margin_dpo/margin_std": 132.6590576171875,
"step": 568
},
{
"KL/chosen_KL_mean": -197.28277587890625,
"KL/mean": -225.33621215820312,
"KL/rejected_KL_mean": -253.38963317871094,
"KL/std": 129.16079711914062,
"epoch": 0.8601662887377173,
"fcm_dpo/beta": 0.004134716931730509,
"fcm_dpo/delta": 0.010899978689849377,
"fcm_dpo/margin": 56.106849670410156,
"fcm_dpo/q_t": 0.4470366835594177,
"grad_norm": 13.65986442565918,
"learning_rate": 2.963665913810451e-08,
"logits/chosen": 0.7979574203491211,
"logits/rejected": 0.7730275392532349,
"logps/chosen": -259.9320373535156,
"logps/ref_chosen": -62.649261474609375,
"logps/ref_rejected": -75.4298324584961,
"logps/rejected": -328.8194580078125,
"loss": 1.2424,
"margin_dpo/margin_mean": 56.106842041015625,
"margin_dpo/margin_std": 131.40196228027344,
"step": 569
},
{
"KL/chosen_KL_mean": -172.97483825683594,
"KL/mean": -227.10557556152344,
"KL/rejected_KL_mean": -281.23626708984375,
"KL/std": 119.91664123535156,
"epoch": 0.8616780045351474,
"fcm_dpo/beta": 0.004098374396562576,
"fcm_dpo/delta": -0.05100153386592865,
"fcm_dpo/margin": 108.261474609375,
"fcm_dpo/q_t": 0.39809536933898926,
"grad_norm": 11.868734359741211,
"learning_rate": 2.9015298217712453e-08,
"logits/chosen": 0.8359570503234863,
"logits/rejected": 0.7631780505180359,
"logps/chosen": -223.01663208007812,
"logps/ref_chosen": -50.04179382324219,
"logps/ref_rejected": -78.27146911621094,
"logps/rejected": -359.50775146484375,
"loss": 1.0673,
"margin_dpo/margin_mean": 108.261474609375,
"margin_dpo/margin_std": 139.59825134277344,
"step": 570
},
{
"KL/chosen_KL_mean": -183.54083251953125,
"KL/mean": -215.38575744628906,
"KL/rejected_KL_mean": -247.2306671142578,
"KL/std": 116.32156372070312,
"epoch": 0.8631897203325775,
"fcm_dpo/beta": 0.004064176231622696,
"fcm_dpo/delta": 0.044348590075969696,
"fcm_dpo/margin": 63.68983840942383,
"fcm_dpo/q_t": 0.44058936834335327,
"grad_norm": 11.032413482666016,
"learning_rate": 2.840011871446962e-08,
"logits/chosen": 0.8487415909767151,
"logits/rejected": 0.8350812196731567,
"logps/chosen": -237.19764709472656,
"logps/ref_chosen": -53.65681457519531,
"logps/ref_rejected": -66.13298034667969,
"logps/rejected": -313.3636474609375,
"loss": 1.2164,
"margin_dpo/margin_mean": 63.689842224121094,
"margin_dpo/margin_std": 133.0120849609375,
"step": 571
},
{
"KL/chosen_KL_mean": -191.580078125,
"KL/mean": -224.85147094726562,
"KL/rejected_KL_mean": -258.12286376953125,
"KL/std": 121.44296264648438,
"epoch": 0.8647014361300076,
"fcm_dpo/beta": 0.004147428087890148,
"fcm_dpo/delta": 0.023455314338207245,
"fcm_dpo/margin": 66.54278564453125,
"fcm_dpo/q_t": 0.4355035126209259,
"grad_norm": 13.11365795135498,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 0.8420843482017517,
"logits/rejected": 0.8802669048309326,
"logps/chosen": -266.39801025390625,
"logps/ref_chosen": -74.81792449951172,
"logps/ref_rejected": -65.88681030273438,
"logps/rejected": -324.0096435546875,
"loss": 1.1947,
"margin_dpo/margin_mean": 66.54278564453125,
"margin_dpo/margin_std": 124.58303833007812,
"step": 572
},
{
"KL/chosen_KL_mean": -204.505126953125,
"KL/mean": -246.52902221679688,
"KL/rejected_KL_mean": -288.55291748046875,
"KL/std": 137.63925170898438,
"epoch": 0.8662131519274376,
"fcm_dpo/beta": 0.004270592704415321,
"fcm_dpo/delta": 0.04590342938899994,
"fcm_dpo/margin": 84.04781341552734,
"fcm_dpo/q_t": 0.4227134585380554,
"grad_norm": 12.49295425415039,
"learning_rate": 2.718837261761528e-08,
"logits/chosen": 0.8661466836929321,
"logits/rejected": 0.8327881097793579,
"logps/chosen": -273.23077392578125,
"logps/ref_chosen": -68.72564697265625,
"logps/ref_rejected": -88.16201782226562,
"logps/rejected": -376.7149658203125,
"loss": 1.1742,
"margin_dpo/margin_mean": 84.04782104492188,
"margin_dpo/margin_std": 166.25271606445312,
"step": 573
},
{
"KL/chosen_KL_mean": -182.09056091308594,
"KL/mean": -235.65748596191406,
"KL/rejected_KL_mean": -289.22442626953125,
"KL/std": 127.6211166381836,
"epoch": 0.8677248677248677,
"fcm_dpo/beta": 0.004246646538376808,
"fcm_dpo/delta": -0.06463881582021713,
"fcm_dpo/margin": 107.13386535644531,
"fcm_dpo/q_t": 0.3947995901107788,
"grad_norm": 10.736037254333496,
"learning_rate": 2.659183991914696e-08,
"logits/chosen": 0.8799293041229248,
"logits/rejected": 0.8327153921127319,
"logps/chosen": -238.40396118164062,
"logps/ref_chosen": -56.31340026855469,
"logps/ref_rejected": -83.91553497314453,
"logps/rejected": -373.13995361328125,
"loss": 1.0468,
"margin_dpo/margin_mean": 107.13386535644531,
"margin_dpo/margin_std": 122.67759704589844,
"step": 574
},
{
"KL/chosen_KL_mean": -188.4454345703125,
"KL/mean": -226.3487091064453,
"KL/rejected_KL_mean": -264.2519836425781,
"KL/std": 127.46905517578125,
"epoch": 0.8692365835222978,
"fcm_dpo/beta": 0.004053239710628986,
"fcm_dpo/delta": -0.031237700954079628,
"fcm_dpo/margin": 75.80654907226562,
"fcm_dpo/q_t": 0.43161332607269287,
"grad_norm": 11.321064949035645,
"learning_rate": 2.600155642716606e-08,
"logits/chosen": 0.8765904903411865,
"logits/rejected": 0.8120791912078857,
"logps/chosen": -253.02957153320312,
"logps/ref_chosen": -64.5841293334961,
"logps/ref_rejected": -93.47034454345703,
"logps/rejected": -357.72235107421875,
"loss": 1.1997,
"margin_dpo/margin_mean": 75.80654907226562,
"margin_dpo/margin_std": 154.2758331298828,
"step": 575
},
{
"KL/chosen_KL_mean": -165.71563720703125,
"KL/mean": -223.3694610595703,
"KL/rejected_KL_mean": -281.02325439453125,
"KL/std": 128.47235107421875,
"epoch": 0.8707482993197279,
"fcm_dpo/beta": 0.0038730669766664505,
"fcm_dpo/delta": -0.05786684527993202,
"fcm_dpo/margin": 115.30767822265625,
"fcm_dpo/q_t": 0.3955625295639038,
"grad_norm": 10.995619773864746,
"learning_rate": 2.5417538653170754e-08,
"logits/chosen": 0.8378143310546875,
"logits/rejected": 0.7612361907958984,
"logps/chosen": -218.99615478515625,
"logps/ref_chosen": -53.28052520751953,
"logps/ref_rejected": -84.2000503540039,
"logps/rejected": -365.22332763671875,
"loss": 1.0621,
"margin_dpo/margin_mean": 115.30767822265625,
"margin_dpo/margin_std": 140.68199157714844,
"step": 576
},
{
"KL/chosen_KL_mean": -185.73255920410156,
"KL/mean": -220.54190063476562,
"KL/rejected_KL_mean": -255.35122680664062,
"KL/std": 123.33712768554688,
"epoch": 0.872260015117158,
"fcm_dpo/beta": 0.004132484085857868,
"fcm_dpo/delta": 0.12298332154750824,
"fcm_dpo/margin": 69.61865234375,
"fcm_dpo/q_t": 0.43443581461906433,
"grad_norm": 12.665448188781738,
"learning_rate": 2.4839802933393607e-08,
"logits/chosen": 0.8501470685005188,
"logits/rejected": 0.8491848707199097,
"logps/chosen": -248.0572509765625,
"logps/ref_chosen": -62.32468795776367,
"logps/ref_rejected": -67.300537109375,
"logps/rejected": -322.6517333984375,
"loss": 1.1952,
"margin_dpo/margin_mean": 69.61865234375,
"margin_dpo/margin_std": 131.3208770751953,
"step": 577
},
{
"KL/chosen_KL_mean": -177.81094360351562,
"KL/mean": -212.05685424804688,
"KL/rejected_KL_mean": -246.3027801513672,
"KL/std": 130.85694885253906,
"epoch": 0.873771730914588,
"fcm_dpo/beta": 0.004423568025231361,
"fcm_dpo/delta": 0.10660433769226074,
"fcm_dpo/margin": 68.4918441772461,
"fcm_dpo/q_t": 0.43448588252067566,
"grad_norm": 13.477178573608398,
"learning_rate": 2.4268365428344733e-08,
"logits/chosen": 0.8857970237731934,
"logits/rejected": 0.8791966438293457,
"logps/chosen": -234.46652221679688,
"logps/ref_chosen": -56.65557861328125,
"logps/ref_rejected": -68.21835327148438,
"logps/rejected": -314.5211181640625,
"loss": 1.21,
"margin_dpo/margin_mean": 68.49185180664062,
"margin_dpo/margin_std": 150.7880096435547,
"step": 578
},
{
"KL/chosen_KL_mean": -182.95477294921875,
"KL/mean": -233.42739868164062,
"KL/rejected_KL_mean": -283.9000244140625,
"KL/std": 118.61689758300781,
"epoch": 0.8752834467120182,
"fcm_dpo/beta": 0.004369863774627447,
"fcm_dpo/delta": -0.04803343117237091,
"fcm_dpo/margin": 100.94525146484375,
"fcm_dpo/q_t": 0.3958134353160858,
"grad_norm": 10.97381591796875,
"learning_rate": 2.3703242122359357e-08,
"logits/chosen": 0.805631697177887,
"logits/rejected": 0.7936950922012329,
"logps/chosen": -239.76443481445312,
"logps/ref_chosen": -56.809661865234375,
"logps/ref_rejected": -68.09613037109375,
"logps/rejected": -351.99615478515625,
"loss": 1.0605,
"margin_dpo/margin_mean": 100.94525146484375,
"margin_dpo/margin_std": 122.35321807861328,
"step": 579
},
{
"KL/chosen_KL_mean": -186.7950439453125,
"KL/mean": -232.87738037109375,
"KL/rejected_KL_mean": -278.959716796875,
"KL/std": 143.08419799804688,
"epoch": 0.8767951625094482,
"fcm_dpo/beta": 0.004418280906975269,
"fcm_dpo/delta": -0.009930390864610672,
"fcm_dpo/margin": 92.1646728515625,
"fcm_dpo/q_t": 0.41188204288482666,
"grad_norm": 13.299028396606445,
"learning_rate": 2.3144448823151392e-08,
"logits/chosen": 0.8283756375312805,
"logits/rejected": 0.7941699624061584,
"logps/chosen": -244.4951629638672,
"logps/ref_chosen": -57.70011520385742,
"logps/ref_rejected": -77.90664672851562,
"logps/rejected": -356.8663635253906,
"loss": 1.1349,
"margin_dpo/margin_mean": 92.16466522216797,
"margin_dpo/margin_std": 160.25291442871094,
"step": 580
},
{
"KL/chosen_KL_mean": -203.46536254882812,
"KL/mean": -243.61395263671875,
"KL/rejected_KL_mean": -283.7625732421875,
"KL/std": 123.7155532836914,
"epoch": 0.8783068783068783,
"fcm_dpo/beta": 0.004408898763358593,
"fcm_dpo/delta": 0.0514557883143425,
"fcm_dpo/margin": 80.29718017578125,
"fcm_dpo/q_t": 0.42192864418029785,
"grad_norm": 11.928445816040039,
"learning_rate": 2.259200116137039e-08,
"logits/chosen": 0.8557339906692505,
"logits/rejected": 0.8073595762252808,
"logps/chosen": -262.7977294921875,
"logps/ref_chosen": -59.332359313964844,
"logps/ref_rejected": -83.64482116699219,
"logps/rejected": -367.4073791503906,
"loss": 1.1666,
"margin_dpo/margin_mean": 80.29718017578125,
"margin_dpo/margin_std": 150.23489379882812,
"step": 581
},
{
"KL/chosen_KL_mean": -186.45236206054688,
"KL/mean": -223.76060485839844,
"KL/rejected_KL_mean": -261.06884765625,
"KL/std": 116.26499938964844,
"epoch": 0.8798185941043084,
"fcm_dpo/beta": 0.004633273929357529,
"fcm_dpo/delta": 0.05958259850740433,
"fcm_dpo/margin": 74.61648559570312,
"fcm_dpo/q_t": 0.423893004655838,
"grad_norm": 11.573064804077148,
"learning_rate": 2.204591459016525e-08,
"logits/chosen": 0.8565012812614441,
"logits/rejected": 0.8867921829223633,
"logps/chosen": -250.61521911621094,
"logps/ref_chosen": -64.16285705566406,
"logps/ref_rejected": -58.632896423339844,
"logps/rejected": -319.70172119140625,
"loss": 1.1645,
"margin_dpo/margin_mean": 74.61648559570312,
"margin_dpo/margin_std": 135.59817504882812,
"step": 582
},
{
"KL/chosen_KL_mean": -183.05465698242188,
"KL/mean": -235.2905731201172,
"KL/rejected_KL_mean": -287.5264892578125,
"KL/std": 138.89993286132812,
"epoch": 0.8813303099017384,
"fcm_dpo/beta": 0.004453700035810471,
"fcm_dpo/delta": -0.07676573097705841,
"fcm_dpo/margin": 104.47183990478516,
"fcm_dpo/q_t": 0.39537179470062256,
"grad_norm": 13.700339317321777,
"learning_rate": 2.1506204384751064e-08,
"logits/chosen": 0.9295327067375183,
"logits/rejected": 0.8535386323928833,
"logps/chosen": -234.9270477294922,
"logps/ref_chosen": -51.87239456176758,
"logps/ref_rejected": -83.86331176757812,
"logps/rejected": -371.3898010253906,
"loss": 1.0891,
"margin_dpo/margin_mean": 104.47184753417969,
"margin_dpo/margin_std": 159.20358276367188,
"step": 583
},
{
"KL/chosen_KL_mean": -168.95220947265625,
"KL/mean": -212.09420776367188,
"KL/rejected_KL_mean": -255.2362060546875,
"KL/std": 120.96197509765625,
"epoch": 0.8828420256991686,
"fcm_dpo/beta": 0.00443212129175663,
"fcm_dpo/delta": 0.019545778632164,
"fcm_dpo/margin": 86.28401184082031,
"fcm_dpo/q_t": 0.4162449240684509,
"grad_norm": 13.947019577026367,
"learning_rate": 2.09728856419826e-08,
"logits/chosen": 0.9570223093032837,
"logits/rejected": 0.8717998266220093,
"logps/chosen": -215.52359008789062,
"logps/ref_chosen": -46.571388244628906,
"logps/ref_rejected": -80.67969512939453,
"logps/rejected": -335.9158935546875,
"loss": 1.1478,
"margin_dpo/margin_mean": 86.28401184082031,
"margin_dpo/margin_std": 154.09945678710938,
"step": 584
},
{
"KL/chosen_KL_mean": -197.41429138183594,
"KL/mean": -228.14865112304688,
"KL/rejected_KL_mean": -258.88299560546875,
"KL/std": 134.65402221679688,
"epoch": 0.8843537414965986,
"fcm_dpo/beta": 0.004502854309976101,
"fcm_dpo/delta": 0.015462912619113922,
"fcm_dpo/margin": 61.46870422363281,
"fcm_dpo/q_t": 0.4372748136520386,
"grad_norm": 13.030404090881348,
"learning_rate": 2.044597327993153e-08,
"logits/chosen": 0.8678371906280518,
"logits/rejected": 0.8205997943878174,
"logps/chosen": -255.538818359375,
"logps/ref_chosen": -58.124534606933594,
"logps/ref_rejected": -79.00538635253906,
"logps/rejected": -337.88836669921875,
"loss": 1.212,
"margin_dpo/margin_mean": 61.46870422363281,
"margin_dpo/margin_std": 131.03814697265625,
"step": 585
},
{
"KL/chosen_KL_mean": -179.77459716796875,
"KL/mean": -218.6102294921875,
"KL/rejected_KL_mean": -257.44586181640625,
"KL/std": 107.11343383789062,
"epoch": 0.8858654572940288,
"fcm_dpo/beta": 0.004600149113684893,
"fcm_dpo/delta": 0.04789983481168747,
"fcm_dpo/margin": 77.6712646484375,
"fcm_dpo/q_t": 0.4177996516227722,
"grad_norm": 11.622013092041016,
"learning_rate": 1.9925482037469187e-08,
"logits/chosen": 0.8768935203552246,
"logits/rejected": 0.8456597328186035,
"logps/chosen": -233.876220703125,
"logps/ref_chosen": -54.10163879394531,
"logps/ref_rejected": -63.72113037109375,
"logps/rejected": -321.1669921875,
"loss": 1.1115,
"margin_dpo/margin_mean": 77.6712646484375,
"margin_dpo/margin_std": 100.46601104736328,
"step": 586
},
{
"KL/chosen_KL_mean": -193.71388244628906,
"KL/mean": -231.7980194091797,
"KL/rejected_KL_mean": -269.88214111328125,
"KL/std": 131.6758270263672,
"epoch": 0.8873771730914588,
"fcm_dpo/beta": 0.004727904684841633,
"fcm_dpo/delta": 0.04477377235889435,
"fcm_dpo/margin": 76.16827392578125,
"fcm_dpo/q_t": 0.41351163387298584,
"grad_norm": 16.236108779907227,
"learning_rate": 1.9411426473854687e-08,
"logits/chosen": 0.9039092063903809,
"logits/rejected": 0.9038503766059875,
"logps/chosen": -257.1310729980469,
"logps/ref_chosen": -63.41719436645508,
"logps/ref_rejected": -63.47003936767578,
"logps/rejected": -333.3522033691406,
"loss": 1.2072,
"margin_dpo/margin_mean": 76.16828155517578,
"margin_dpo/margin_std": 177.19003295898438,
"step": 587
},
{
"KL/chosen_KL_mean": -197.713134765625,
"KL/mean": -240.94564819335938,
"KL/rejected_KL_mean": -284.1781921386719,
"KL/std": 131.87918090820312,
"epoch": 0.8888888888888888,
"fcm_dpo/beta": 0.004706019535660744,
"fcm_dpo/delta": -0.009034395217895508,
"fcm_dpo/margin": 86.46505737304688,
"fcm_dpo/q_t": 0.410349041223526,
"grad_norm": 16.405550003051758,
"learning_rate": 1.890382096832699e-08,
"logits/chosen": 0.9087653756141663,
"logits/rejected": 0.8741371631622314,
"logps/chosen": -259.9141845703125,
"logps/ref_chosen": -62.20103454589844,
"logps/ref_rejected": -82.10249328613281,
"logps/rejected": -366.28070068359375,
"loss": 1.1565,
"margin_dpo/margin_mean": 86.46505737304688,
"margin_dpo/margin_std": 164.84034729003906,
"step": 588
},
{
"KL/chosen_KL_mean": -183.16661071777344,
"KL/mean": -228.384765625,
"KL/rejected_KL_mean": -273.6029357910156,
"KL/std": 123.86094665527344,
"epoch": 0.890400604686319,
"fcm_dpo/beta": 0.0047372374683618546,
"fcm_dpo/delta": -0.033031828701496124,
"fcm_dpo/margin": 90.43630981445312,
"fcm_dpo/q_t": 0.4000184237957001,
"grad_norm": 11.501230239868164,
"learning_rate": 1.840267971970344e-08,
"logits/chosen": 0.8425379991531372,
"logits/rejected": 0.8174772262573242,
"logps/chosen": -239.88021850585938,
"logps/ref_chosen": -56.71361541748047,
"logps/ref_rejected": -76.7366943359375,
"logps/rejected": -350.339599609375,
"loss": 1.0785,
"margin_dpo/margin_mean": 90.43630981445312,
"margin_dpo/margin_std": 118.53308868408203,
"step": 589
},
{
"KL/chosen_KL_mean": -199.31167602539062,
"KL/mean": -239.75074768066406,
"KL/rejected_KL_mean": -280.1898193359375,
"KL/std": 112.27293395996094,
"epoch": 0.891912320483749,
"fcm_dpo/beta": 0.00475288974121213,
"fcm_dpo/delta": 0.016453173011541367,
"fcm_dpo/margin": 80.87815856933594,
"fcm_dpo/q_t": 0.41211652755737305,
"grad_norm": 16.781558990478516,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 0.8056505918502808,
"logits/rejected": 0.7747730016708374,
"logps/chosen": -265.82550048828125,
"logps/ref_chosen": -66.5138168334961,
"logps/ref_rejected": -85.70820617675781,
"logps/rejected": -365.89801025390625,
"loss": 1.1167,
"margin_dpo/margin_mean": 80.87814331054688,
"margin_dpo/margin_std": 119.72245788574219,
"step": 590
},
{
"KL/chosen_KL_mean": -168.68548583984375,
"KL/mean": -223.0603485107422,
"KL/rejected_KL_mean": -277.43524169921875,
"KL/std": 136.9473876953125,
"epoch": 0.8934240362811792,
"fcm_dpo/beta": 0.004390774294734001,
"fcm_dpo/delta": -0.09836189448833466,
"fcm_dpo/margin": 108.74974060058594,
"fcm_dpo/q_t": 0.3937166929244995,
"grad_norm": 13.011321067810059,
"learning_rate": 1.7419845883949098e-08,
"logits/chosen": 0.9268543124198914,
"logits/rejected": 0.8832913637161255,
"logps/chosen": -229.38265991210938,
"logps/ref_chosen": -60.697181701660156,
"logps/ref_rejected": -86.12278747558594,
"logps/rejected": -363.5580139160156,
"loss": 1.0917,
"margin_dpo/margin_mean": 108.7497329711914,
"margin_dpo/margin_std": 168.9647216796875,
"step": 591
},
{
"KL/chosen_KL_mean": -185.19931030273438,
"KL/mean": -225.90817260742188,
"KL/rejected_KL_mean": -266.6170349121094,
"KL/std": 114.66996765136719,
"epoch": 0.8949357520786092,
"fcm_dpo/beta": 0.004453813657164574,
"fcm_dpo/delta": 0.04094696044921875,
"fcm_dpo/margin": 81.41773986816406,
"fcm_dpo/q_t": 0.4200698137283325,
"grad_norm": 14.020329475402832,
"learning_rate": 1.6938180788793556e-08,
"logits/chosen": 0.9028749465942383,
"logits/rejected": 0.8090362548828125,
"logps/chosen": -236.4366455078125,
"logps/ref_chosen": -51.237327575683594,
"logps/ref_rejected": -81.60242462158203,
"logps/rejected": -348.219482421875,
"loss": 1.137,
"margin_dpo/margin_mean": 81.41773986816406,
"margin_dpo/margin_std": 130.4501190185547,
"step": 592
},
{
"KL/chosen_KL_mean": -151.83436584472656,
"KL/mean": -198.61888122558594,
"KL/rejected_KL_mean": -245.40338134765625,
"KL/std": 116.2724380493164,
"epoch": 0.8964474678760394,
"fcm_dpo/beta": 0.004477858543395996,
"fcm_dpo/delta": -0.022135818377137184,
"fcm_dpo/margin": 93.56902313232422,
"fcm_dpo/q_t": 0.40188807249069214,
"grad_norm": 15.795121192932129,
"learning_rate": 1.6463034933723336e-08,
"logits/chosen": 0.8935588598251343,
"logits/rejected": 0.8158241510391235,
"logps/chosen": -193.91436767578125,
"logps/ref_chosen": -42.08000183105469,
"logps/ref_rejected": -68.47499084472656,
"logps/rejected": -313.87835693359375,
"loss": 1.0948,
"margin_dpo/margin_mean": 93.56902313232422,
"margin_dpo/margin_std": 133.15293884277344,
"step": 593
},
{
"KL/chosen_KL_mean": -195.93930053710938,
"KL/mean": -232.91555786132812,
"KL/rejected_KL_mean": -269.891845703125,
"KL/std": 117.66835021972656,
"epoch": 0.8979591836734694,
"fcm_dpo/beta": 0.004622071050107479,
"fcm_dpo/delta": 0.06481346487998962,
"fcm_dpo/margin": 73.95257568359375,
"fcm_dpo/q_t": 0.4218658208847046,
"grad_norm": 12.912012100219727,
"learning_rate": 1.5994421609589385e-08,
"logits/chosen": 0.7965784668922424,
"logits/rejected": 0.7883630990982056,
"logps/chosen": -259.59796142578125,
"logps/ref_chosen": -63.658668518066406,
"logps/ref_rejected": -70.35597229003906,
"logps/rejected": -340.2478332519531,
"loss": 1.1477,
"margin_dpo/margin_mean": 73.95257568359375,
"margin_dpo/margin_std": 120.45957946777344,
"step": 594
},
{
"KL/chosen_KL_mean": -178.886962890625,
"KL/mean": -230.2701416015625,
"KL/rejected_KL_mean": -281.6533203125,
"KL/std": 126.58238220214844,
"epoch": 0.8994708994708994,
"fcm_dpo/beta": 0.004551096819341183,
"fcm_dpo/delta": -0.07861794531345367,
"fcm_dpo/margin": 102.76634216308594,
"fcm_dpo/q_t": 0.39468061923980713,
"grad_norm": 10.457369804382324,
"learning_rate": 1.553235392451377e-08,
"logits/chosen": 0.8931225538253784,
"logits/rejected": 0.8216215372085571,
"logps/chosen": -235.105712890625,
"logps/ref_chosen": -56.21875762939453,
"logps/ref_rejected": -83.95773315429688,
"logps/rejected": -365.6110534667969,
"loss": 1.0852,
"margin_dpo/margin_mean": 102.76634979248047,
"margin_dpo/margin_std": 154.8719940185547,
"step": 595
},
{
"KL/chosen_KL_mean": -209.03253173828125,
"KL/mean": -227.578369140625,
"KL/rejected_KL_mean": -246.12420654296875,
"KL/std": 113.8258056640625,
"epoch": 0.9009826152683296,
"fcm_dpo/beta": 0.004681180696934462,
"fcm_dpo/delta": 0.08331310749053955,
"fcm_dpo/margin": 37.091651916503906,
"fcm_dpo/q_t": 0.4618571698665619,
"grad_norm": 15.247252464294434,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 0.7848924398422241,
"logits/rejected": 0.8063439726829529,
"logps/chosen": -277.513427734375,
"logps/ref_chosen": -68.48088073730469,
"logps/ref_rejected": -61.732967376708984,
"logps/rejected": -307.857177734375,
"loss": 1.2958,
"margin_dpo/margin_mean": 37.09164810180664,
"margin_dpo/margin_std": 118.8961181640625,
"step": 596
},
{
"KL/chosen_KL_mean": -160.05877685546875,
"KL/mean": -198.61622619628906,
"KL/rejected_KL_mean": -237.17367553710938,
"KL/std": 100.68923950195312,
"epoch": 0.9024943310657596,
"fcm_dpo/beta": 0.004797500092536211,
"fcm_dpo/delta": 0.032740212976932526,
"fcm_dpo/margin": 77.11491394042969,
"fcm_dpo/q_t": 0.41552621126174927,
"grad_norm": 11.356983184814453,
"learning_rate": 1.4627906988186111e-08,
"logits/chosen": 0.8747761845588684,
"logits/rejected": 0.8632344007492065,
"logps/chosen": -208.91627502441406,
"logps/ref_chosen": -48.85750961303711,
"logps/ref_rejected": -55.068084716796875,
"logps/rejected": -292.24176025390625,
"loss": 1.1215,
"margin_dpo/margin_mean": 77.11491394042969,
"margin_dpo/margin_std": 113.88622283935547,
"step": 597
},
{
"KL/chosen_KL_mean": -207.90806579589844,
"KL/mean": -234.0879364013672,
"KL/rejected_KL_mean": -260.267822265625,
"KL/std": 122.74574279785156,
"epoch": 0.9040060468631897,
"fcm_dpo/beta": 0.004856814630329609,
"fcm_dpo/delta": 0.056810446083545685,
"fcm_dpo/margin": 52.3597412109375,
"fcm_dpo/q_t": 0.44433027505874634,
"grad_norm": 12.908487319946289,
"learning_rate": 1.4185553036259095e-08,
"logits/chosen": 0.896747350692749,
"logits/rejected": 0.8310115337371826,
"logps/chosen": -266.79522705078125,
"logps/ref_chosen": -58.88715362548828,
"logps/ref_rejected": -81.43145751953125,
"logps/rejected": -341.69927978515625,
"loss": 1.2418,
"margin_dpo/margin_mean": 52.35974884033203,
"margin_dpo/margin_std": 130.03427124023438,
"step": 598
},
{
"KL/chosen_KL_mean": -208.6796875,
"KL/mean": -237.74996948242188,
"KL/rejected_KL_mean": -266.8202819824219,
"KL/std": 116.68116760253906,
"epoch": 0.9055177626606198,
"fcm_dpo/beta": 0.004964320454746485,
"fcm_dpo/delta": 0.016610831022262573,
"fcm_dpo/margin": 58.14057922363281,
"fcm_dpo/q_t": 0.43591034412384033,
"grad_norm": 16.04824447631836,
"learning_rate": 1.3749795321332885e-08,
"logits/chosen": 0.910493016242981,
"logits/rejected": 0.8786226511001587,
"logps/chosen": -266.286865234375,
"logps/ref_chosen": -57.60719299316406,
"logps/ref_rejected": -71.80469512939453,
"logps/rejected": -338.6249694824219,
"loss": 1.2245,
"margin_dpo/margin_mean": 58.14057922363281,
"margin_dpo/margin_std": 135.10581970214844,
"step": 599
},
{
"KL/chosen_KL_mean": -190.2998046875,
"KL/mean": -226.334228515625,
"KL/rejected_KL_mean": -262.36865234375,
"KL/std": 129.5638885498047,
"epoch": 0.9070294784580499,
"fcm_dpo/beta": 0.004913763143122196,
"fcm_dpo/delta": -0.05108189955353737,
"fcm_dpo/margin": 72.0688247680664,
"fcm_dpo/q_t": 0.4227983355522156,
"grad_norm": 16.685800552368164,
"learning_rate": 1.3320646032487393e-08,
"logits/chosen": 0.8785399198532104,
"logits/rejected": 0.8318706750869751,
"logps/chosen": -248.7421112060547,
"logps/ref_chosen": -58.44231414794922,
"logps/ref_rejected": -83.64639282226562,
"logps/rejected": -346.0150146484375,
"loss": 1.1762,
"margin_dpo/margin_mean": 72.0688247680664,
"margin_dpo/margin_std": 140.14089965820312,
"step": 600
},
{
"KL/chosen_KL_mean": -171.17523193359375,
"KL/mean": -220.55459594726562,
"KL/rejected_KL_mean": -269.9339599609375,
"KL/std": 136.19314575195312,
"epoch": 0.90854119425548,
"fcm_dpo/beta": 0.0046309540048241615,
"fcm_dpo/delta": -0.0687466561794281,
"fcm_dpo/margin": 98.75874328613281,
"fcm_dpo/q_t": 0.39738285541534424,
"grad_norm": 11.404427528381348,
"learning_rate": 1.2898117173950868e-08,
"logits/chosen": 0.8668086528778076,
"logits/rejected": 0.8014013171195984,
"logps/chosen": -226.76954650878906,
"logps/ref_chosen": -55.59432601928711,
"logps/ref_rejected": -83.68630981445312,
"logps/rejected": -353.62030029296875,
"loss": 1.0981,
"margin_dpo/margin_mean": 98.75874328613281,
"margin_dpo/margin_std": 154.93240356445312,
"step": 601
},
{
"KL/chosen_KL_mean": -160.80662536621094,
"KL/mean": -204.2389373779297,
"KL/rejected_KL_mean": -247.67123413085938,
"KL/std": 116.13655853271484,
"epoch": 0.91005291005291,
"fcm_dpo/beta": 0.004675906151533127,
"fcm_dpo/delta": -0.007718130946159363,
"fcm_dpo/margin": 86.8646011352539,
"fcm_dpo/q_t": 0.4061970114707947,
"grad_norm": 13.002616882324219,
"learning_rate": 1.2482220564763667e-08,
"logits/chosen": 0.8623030185699463,
"logits/rejected": 0.8387429118156433,
"logps/chosen": -217.15582275390625,
"logps/ref_chosen": -56.349185943603516,
"logps/ref_rejected": -71.9959716796875,
"logps/rejected": -319.6672058105469,
"loss": 1.0989,
"margin_dpo/margin_mean": 86.8646011352539,
"margin_dpo/margin_std": 123.03114318847656,
"step": 602
},
{
"KL/chosen_KL_mean": -177.90234375,
"KL/mean": -222.01742553710938,
"KL/rejected_KL_mean": -266.13250732421875,
"KL/std": 118.25772094726562,
"epoch": 0.9115646258503401,
"fcm_dpo/beta": 0.0045766388066112995,
"fcm_dpo/delta": -0.004777451977133751,
"fcm_dpo/margin": 88.23017883300781,
"fcm_dpo/q_t": 0.40897947549819946,
"grad_norm": 11.585609436035156,
"learning_rate": 1.2072967838448051e-08,
"logits/chosen": 0.8506954908370972,
"logits/rejected": 0.8117414712905884,
"logps/chosen": -231.0707244873047,
"logps/ref_chosen": -53.16838836669922,
"logps/ref_rejected": -73.8604736328125,
"logps/rejected": -339.99298095703125,
"loss": 1.1146,
"margin_dpo/margin_mean": 88.23017883300781,
"margin_dpo/margin_std": 137.55128479003906,
"step": 603
},
{
"KL/chosen_KL_mean": -174.488037109375,
"KL/mean": -210.52664184570312,
"KL/rejected_KL_mean": -246.56524658203125,
"KL/std": 110.95604705810547,
"epoch": 0.9130763416477702,
"fcm_dpo/beta": 0.004770040512084961,
"fcm_dpo/delta": 0.06184397637844086,
"fcm_dpo/margin": 72.07720947265625,
"fcm_dpo/q_t": 0.42401158809661865,
"grad_norm": 13.443038940429688,
"learning_rate": 1.1670370442682459e-08,
"logits/chosen": 0.8126691579818726,
"logits/rejected": 0.8226346969604492,
"logps/chosen": -247.13746643066406,
"logps/ref_chosen": -72.64942169189453,
"logps/ref_rejected": -69.8792724609375,
"logps/rejected": -316.44451904296875,
"loss": 1.1842,
"margin_dpo/margin_mean": 72.07720947265625,
"margin_dpo/margin_std": 146.4234619140625,
"step": 604
},
{
"KL/chosen_KL_mean": -195.28396606445312,
"KL/mean": -235.2552490234375,
"KL/rejected_KL_mean": -275.22650146484375,
"KL/std": 115.78852844238281,
"epoch": 0.9145880574452003,
"fcm_dpo/beta": 0.00478768814355135,
"fcm_dpo/delta": 0.01924796961247921,
"fcm_dpo/margin": 79.94253540039062,
"fcm_dpo/q_t": 0.4138326644897461,
"grad_norm": 14.64280891418457,
"learning_rate": 1.1274439638981532e-08,
"logits/chosen": 0.9054049253463745,
"logits/rejected": 0.8625191450119019,
"logps/chosen": -256.8968200683594,
"logps/ref_chosen": -61.61284637451172,
"logps/ref_rejected": -79.34398651123047,
"logps/rejected": -354.57049560546875,
"loss": 1.142,
"margin_dpo/margin_mean": 79.94253540039062,
"margin_dpo/margin_std": 137.07847595214844,
"step": 605
},
{
"KL/chosen_KL_mean": -170.9767608642578,
"KL/mean": -221.29107666015625,
"KL/rejected_KL_mean": -271.6053466796875,
"KL/std": 122.00587463378906,
"epoch": 0.9160997732426304,
"fcm_dpo/beta": 0.004710886627435684,
"fcm_dpo/delta": -0.08624030649662018,
"fcm_dpo/margin": 100.62860870361328,
"fcm_dpo/q_t": 0.3922450542449951,
"grad_norm": 14.318273544311523,
"learning_rate": 1.0885186502381016e-08,
"logits/chosen": 0.8514528274536133,
"logits/rejected": 0.7999155521392822,
"logps/chosen": -225.4409942626953,
"logps/ref_chosen": -54.46424102783203,
"logps/ref_rejected": -79.62708282470703,
"logps/rejected": -351.2324523925781,
"loss": 1.0643,
"margin_dpo/margin_mean": 100.62861633300781,
"margin_dpo/margin_std": 136.915283203125,
"step": 606
},
{
"KL/chosen_KL_mean": -189.23269653320312,
"KL/mean": -231.669921875,
"KL/rejected_KL_mean": -274.10711669921875,
"KL/std": 117.58612060546875,
"epoch": 0.9176114890400605,
"fcm_dpo/beta": 0.004336424637585878,
"fcm_dpo/delta": -0.09618955850601196,
"fcm_dpo/margin": 84.87444305419922,
"fcm_dpo/q_t": 0.4125783443450928,
"grad_norm": 11.543998718261719,
"learning_rate": 1.0502621921127774e-08,
"logits/chosen": 0.8325244188308716,
"logits/rejected": 0.8169835805892944,
"logps/chosen": -252.09356689453125,
"logps/ref_chosen": -62.86086654663086,
"logps/ref_rejected": -72.5501937866211,
"logps/rejected": -346.6573486328125,
"loss": 1.1358,
"margin_dpo/margin_mean": 84.87443542480469,
"margin_dpo/margin_std": 131.11700439453125,
"step": 607
},
{
"KL/chosen_KL_mean": -191.9585723876953,
"KL/mean": -239.46746826171875,
"KL/rejected_KL_mean": -286.976318359375,
"KL/std": 132.16444396972656,
"epoch": 0.9191232048374905,
"fcm_dpo/beta": 0.00433045718818903,
"fcm_dpo/delta": -0.013165525160729885,
"fcm_dpo/margin": 95.01773834228516,
"fcm_dpo/q_t": 0.4065626263618469,
"grad_norm": 12.041932106018066,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 0.8287484645843506,
"logits/rejected": 0.7658513784408569,
"logps/chosen": -255.13929748535156,
"logps/ref_chosen": -63.18071746826172,
"logps/ref_rejected": -99.15888214111328,
"logps/rejected": -386.13519287109375,
"loss": 1.0962,
"margin_dpo/margin_mean": 95.01774597167969,
"margin_dpo/margin_std": 135.38153076171875,
"step": 608
},
{
"KL/chosen_KL_mean": -171.56765747070312,
"KL/mean": -219.2465057373047,
"KL/rejected_KL_mean": -266.9253845214844,
"KL/std": 113.94578552246094,
"epoch": 0.9206349206349206,
"fcm_dpo/beta": 0.004231642931699753,
"fcm_dpo/delta": -0.005523219704627991,
"fcm_dpo/margin": 95.35771942138672,
"fcm_dpo/q_t": 0.4046892821788788,
"grad_norm": 11.678070068359375,
"learning_rate": 9.757601041885694e-09,
"logits/chosen": 0.9319273829460144,
"logits/rejected": 0.9010778665542603,
"logps/chosen": -220.1908721923828,
"logps/ref_chosen": -48.62322235107422,
"logps/ref_rejected": -68.28271484375,
"logps/rejected": -335.2080993652344,
"loss": 1.0812,
"margin_dpo/margin_mean": 95.35771942138672,
"margin_dpo/margin_std": 111.58070373535156,
"step": 609
},
{
"KL/chosen_KL_mean": -191.14016723632812,
"KL/mean": -233.82876586914062,
"KL/rejected_KL_mean": -276.51739501953125,
"KL/std": 129.33859252929688,
"epoch": 0.9221466364323507,
"fcm_dpo/beta": 0.004307628609240055,
"fcm_dpo/delta": 0.03571552410721779,
"fcm_dpo/margin": 85.37721252441406,
"fcm_dpo/q_t": 0.41646891832351685,
"grad_norm": 13.54400634765625,
"learning_rate": 9.395165583732379e-09,
"logits/chosen": 0.86832594871521,
"logits/rejected": 0.8651347160339355,
"logps/chosen": -263.8052978515625,
"logps/ref_chosen": -72.66513061523438,
"logps/ref_rejected": -87.15310668945312,
"logps/rejected": -363.67047119140625,
"loss": 1.1555,
"margin_dpo/margin_mean": 85.37721252441406,
"margin_dpo/margin_std": 151.74803161621094,
"step": 610
},
{
"KL/chosen_KL_mean": -172.24514770507812,
"KL/mean": -208.3623504638672,
"KL/rejected_KL_mean": -244.47955322265625,
"KL/std": 112.20661926269531,
"epoch": 0.9236583522297808,
"fcm_dpo/beta": 0.004518900997936726,
"fcm_dpo/delta": 0.08180320262908936,
"fcm_dpo/margin": 72.23439025878906,
"fcm_dpo/q_t": 0.4254700541496277,
"grad_norm": 12.482162475585938,
"learning_rate": 9.03946036001449e-09,
"logits/chosen": 0.8952933549880981,
"logits/rejected": 0.8567260503768921,
"logps/chosen": -220.55374145507812,
"logps/ref_chosen": -48.30857849121094,
"logps/ref_rejected": -70.6141128540039,
"logps/rejected": -315.0936584472656,
"loss": 1.149,
"margin_dpo/margin_mean": 72.23438262939453,
"margin_dpo/margin_std": 112.78076171875,
"step": 611
},
{
"KL/chosen_KL_mean": -189.6739044189453,
"KL/mean": -245.22344970703125,
"KL/rejected_KL_mean": -300.77301025390625,
"KL/std": 119.71464538574219,
"epoch": 0.9251700680272109,
"fcm_dpo/beta": 0.004395745228976011,
"fcm_dpo/delta": -0.10357952862977982,
"fcm_dpo/margin": 111.09910583496094,
"fcm_dpo/q_t": 0.3854334354400635,
"grad_norm": 11.041399955749512,
"learning_rate": 8.690495320571839e-09,
"logits/chosen": 0.7935118675231934,
"logits/rejected": 0.735627293586731,
"logps/chosen": -250.90545654296875,
"logps/ref_chosen": -61.23155975341797,
"logps/ref_rejected": -94.37979888916016,
"logps/rejected": -395.15283203125,
"loss": 1.0329,
"margin_dpo/margin_mean": 111.09910583496094,
"margin_dpo/margin_std": 131.01829528808594,
"step": 612
},
{
"KL/chosen_KL_mean": -166.52000427246094,
"KL/mean": -213.49661254882812,
"KL/rejected_KL_mean": -260.47320556640625,
"KL/std": 132.3287811279297,
"epoch": 0.926681783824641,
"fcm_dpo/beta": 0.004340745974332094,
"fcm_dpo/delta": -0.009288674220442772,
"fcm_dpo/margin": 93.95318603515625,
"fcm_dpo/q_t": 0.4047100841999054,
"grad_norm": 10.03150749206543,
"learning_rate": 8.348280226706722e-09,
"logits/chosen": 0.8340524435043335,
"logits/rejected": 0.8326983451843262,
"logps/chosen": -220.50311279296875,
"logps/ref_chosen": -53.98310852050781,
"logps/ref_rejected": -58.32208251953125,
"logps/rejected": -318.7952880859375,
"loss": 1.0976,
"margin_dpo/margin_mean": 93.95318603515625,
"margin_dpo/margin_std": 131.71676635742188,
"step": 613
},
{
"KL/chosen_KL_mean": -189.92941284179688,
"KL/mean": -233.43902587890625,
"KL/rejected_KL_mean": -276.9486389160156,
"KL/std": 110.43850708007812,
"epoch": 0.9281934996220711,
"fcm_dpo/beta": 0.004416479729115963,
"fcm_dpo/delta": 0.014550581574440002,
"fcm_dpo/margin": 87.01921844482422,
"fcm_dpo/q_t": 0.41044163703918457,
"grad_norm": 13.81963062286377,
"learning_rate": 8.012824650910937e-09,
"logits/chosen": 0.8715497851371765,
"logits/rejected": 0.8666037321090698,
"logps/chosen": -250.17245483398438,
"logps/ref_chosen": -60.24303436279297,
"logps/ref_rejected": -72.26258850097656,
"logps/rejected": -349.21124267578125,
"loss": 1.1157,
"margin_dpo/margin_mean": 87.01921844482422,
"margin_dpo/margin_std": 125.52960205078125,
"step": 614
},
{
"KL/chosen_KL_mean": -195.23587036132812,
"KL/mean": -247.610107421875,
"KL/rejected_KL_mean": -299.9843444824219,
"KL/std": 148.3843994140625,
"epoch": 0.9297052154195011,
"fcm_dpo/beta": 0.004148758016526699,
"fcm_dpo/delta": -0.04528382048010826,
"fcm_dpo/margin": 104.74848937988281,
"fcm_dpo/q_t": 0.40149933099746704,
"grad_norm": 11.946541786193848,
"learning_rate": 7.684137976598088e-09,
"logits/chosen": 0.8352917432785034,
"logits/rejected": 0.7855337858200073,
"logps/chosen": -267.3305358886719,
"logps/ref_chosen": -72.09467315673828,
"logps/ref_rejected": -104.02980041503906,
"logps/rejected": -404.01416015625,
"loss": 1.1023,
"margin_dpo/margin_mean": 104.74848937988281,
"margin_dpo/margin_std": 158.98904418945312,
"step": 615
},
{
"KL/chosen_KL_mean": -187.78256225585938,
"KL/mean": -227.31333923339844,
"KL/rejected_KL_mean": -266.8441162109375,
"KL/std": 136.93423461914062,
"epoch": 0.9312169312169312,
"fcm_dpo/beta": 0.004337611608207226,
"fcm_dpo/delta": 0.06366471946239471,
"fcm_dpo/margin": 79.06153869628906,
"fcm_dpo/q_t": 0.4219726324081421,
"grad_norm": 11.41348648071289,
"learning_rate": 7.36222939784098e-09,
"logits/chosen": 0.8961197137832642,
"logits/rejected": 0.8402373194694519,
"logps/chosen": -246.31329345703125,
"logps/ref_chosen": -58.530723571777344,
"logps/ref_rejected": -75.48025512695312,
"logps/rejected": -342.3243713378906,
"loss": 1.1435,
"margin_dpo/margin_mean": 79.06153869628906,
"margin_dpo/margin_std": 126.18814086914062,
"step": 616
},
{
"KL/chosen_KL_mean": -195.61953735351562,
"KL/mean": -232.73971557617188,
"KL/rejected_KL_mean": -269.85992431640625,
"KL/std": 123.03448486328125,
"epoch": 0.9327286470143613,
"fcm_dpo/beta": 0.004304808564484119,
"fcm_dpo/delta": -0.031089257448911667,
"fcm_dpo/margin": 74.24037170410156,
"fcm_dpo/q_t": 0.4274696111679077,
"grad_norm": 15.603584289550781,
"learning_rate": 7.047107919114586e-09,
"logits/chosen": 0.8908584713935852,
"logits/rejected": 0.8490594029426575,
"logps/chosen": -253.2282257080078,
"logps/ref_chosen": -57.608673095703125,
"logps/ref_rejected": -81.22109985351562,
"logps/rejected": -351.0810241699219,
"loss": 1.1746,
"margin_dpo/margin_mean": 74.24037170410156,
"margin_dpo/margin_std": 133.52430725097656,
"step": 617
},
{
"KL/chosen_KL_mean": -182.47540283203125,
"KL/mean": -227.52099609375,
"KL/rejected_KL_mean": -272.5666198730469,
"KL/std": 136.0541229248047,
"epoch": 0.9342403628117913,
"fcm_dpo/beta": 0.004296266473829746,
"fcm_dpo/delta": 0.014270158484578133,
"fcm_dpo/margin": 90.0912094116211,
"fcm_dpo/q_t": 0.4141734838485718,
"grad_norm": 15.708011627197266,
"learning_rate": 6.738782355044048e-09,
"logits/chosen": 0.8595304489135742,
"logits/rejected": 0.7856009006500244,
"logps/chosen": -239.17135620117188,
"logps/ref_chosen": -56.69594192504883,
"logps/ref_rejected": -85.92362976074219,
"logps/rejected": -358.490234375,
"loss": 1.1172,
"margin_dpo/margin_mean": 90.09120178222656,
"margin_dpo/margin_std": 138.923583984375,
"step": 618
},
{
"KL/chosen_KL_mean": -174.67626953125,
"KL/mean": -222.02593994140625,
"KL/rejected_KL_mean": -269.3756408691406,
"KL/std": 125.88426971435547,
"epoch": 0.9357520786092215,
"fcm_dpo/beta": 0.004293524194508791,
"fcm_dpo/delta": -0.007992902770638466,
"fcm_dpo/margin": 94.6993637084961,
"fcm_dpo/q_t": 0.40730637311935425,
"grad_norm": 11.522953987121582,
"learning_rate": 6.437261330158206e-09,
"logits/chosen": 0.9194344282150269,
"logits/rejected": 0.8575088977813721,
"logps/chosen": -228.73468017578125,
"logps/ref_chosen": -54.05841827392578,
"logps/ref_rejected": -83.55493927001953,
"logps/rejected": -352.9305725097656,
"loss": 1.103,
"margin_dpo/margin_mean": 94.6993637084961,
"margin_dpo/margin_std": 138.9624481201172,
"step": 619
},
{
"KL/chosen_KL_mean": -182.78765869140625,
"KL/mean": -217.02841186523438,
"KL/rejected_KL_mean": -251.26914978027344,
"KL/std": 116.54869842529297,
"epoch": 0.9372637944066515,
"fcm_dpo/beta": 0.004260548390448093,
"fcm_dpo/delta": -0.04734001308679581,
"fcm_dpo/margin": 68.48147583007812,
"fcm_dpo/q_t": 0.43087059259414673,
"grad_norm": 12.573121070861816,
"learning_rate": 6.142553278648238e-09,
"logits/chosen": 0.8781119585037231,
"logits/rejected": 0.8820315599441528,
"logps/chosen": -246.15737915039062,
"logps/ref_chosen": -63.36971664428711,
"logps/ref_rejected": -65.68269348144531,
"logps/rejected": -316.95184326171875,
"loss": 1.1914,
"margin_dpo/margin_mean": 68.48147583007812,
"margin_dpo/margin_std": 121.66095733642578,
"step": 620
},
{
"KL/chosen_KL_mean": -175.8046875,
"KL/mean": -221.00616455078125,
"KL/rejected_KL_mean": -266.2076416015625,
"KL/std": 130.67002868652344,
"epoch": 0.9387755102040817,
"fcm_dpo/beta": 0.004282790701836348,
"fcm_dpo/delta": 0.012993309646844864,
"fcm_dpo/margin": 90.40296936035156,
"fcm_dpo/q_t": 0.415774405002594,
"grad_norm": 12.253860473632812,
"learning_rate": 5.854666444131934e-09,
"logits/chosen": 0.9052057862281799,
"logits/rejected": 0.8152543902397156,
"logps/chosen": -228.12591552734375,
"logps/ref_chosen": -52.321224212646484,
"logps/ref_rejected": -88.09001159667969,
"logps/rejected": -354.29766845703125,
"loss": 1.1415,
"margin_dpo/margin_mean": 90.4029769897461,
"margin_dpo/margin_std": 156.630859375,
"step": 621
},
{
"KL/chosen_KL_mean": -193.3104248046875,
"KL/mean": -235.54830932617188,
"KL/rejected_KL_mean": -277.78619384765625,
"KL/std": 133.7432861328125,
"epoch": 0.9402872260015117,
"fcm_dpo/beta": 0.004352467134594917,
"fcm_dpo/delta": 0.034792713820934296,
"fcm_dpo/margin": 84.47576904296875,
"fcm_dpo/q_t": 0.41774168610572815,
"grad_norm": 13.547627449035645,
"learning_rate": 5.573608879422875e-09,
"logits/chosen": 0.8366628885269165,
"logits/rejected": 0.797753095626831,
"logps/chosen": -253.17588806152344,
"logps/ref_chosen": -59.86545944213867,
"logps/ref_rejected": -81.86668395996094,
"logps/rejected": -359.65289306640625,
"loss": 1.1378,
"margin_dpo/margin_mean": 84.47576904296875,
"margin_dpo/margin_std": 138.1892852783203,
"step": 622
},
{
"KL/chosen_KL_mean": -206.13430786132812,
"KL/mean": -251.16110229492188,
"KL/rejected_KL_mean": -296.18792724609375,
"KL/std": 141.82073974609375,
"epoch": 0.9417989417989417,
"fcm_dpo/beta": 0.004277583211660385,
"fcm_dpo/delta": 0.014890506863594055,
"fcm_dpo/margin": 90.05360412597656,
"fcm_dpo/q_t": 0.4128887951374054,
"grad_norm": 12.97014045715332,
"learning_rate": 5.299388446305342e-09,
"logits/chosen": 0.8766790628433228,
"logits/rejected": 0.8372782468795776,
"logps/chosen": -273.50274658203125,
"logps/ref_chosen": -67.36846160888672,
"logps/ref_rejected": -82.02733612060547,
"logps/rejected": -378.21527099609375,
"loss": 1.1335,
"margin_dpo/margin_mean": 90.05360412597656,
"margin_dpo/margin_std": 147.62298583984375,
"step": 623
},
{
"KL/chosen_KL_mean": -175.81561279296875,
"KL/mean": -221.06178283691406,
"KL/rejected_KL_mean": -266.30792236328125,
"KL/std": 129.63758850097656,
"epoch": 0.9433106575963719,
"fcm_dpo/beta": 0.0043399459682404995,
"fcm_dpo/delta": 0.007810791954398155,
"fcm_dpo/margin": 90.49232482910156,
"fcm_dpo/q_t": 0.41192591190338135,
"grad_norm": 14.457840919494629,
"learning_rate": 5.03201281531429e-09,
"logits/chosen": 0.8671579957008362,
"logits/rejected": 0.7905720472335815,
"logps/chosen": -226.8421630859375,
"logps/ref_chosen": -51.02655029296875,
"logps/ref_rejected": -76.49203491210938,
"logps/rejected": -342.79998779296875,
"loss": 1.117,
"margin_dpo/margin_mean": 90.49234008789062,
"margin_dpo/margin_std": 139.15968322753906,
"step": 624
},
{
"KL/chosen_KL_mean": -175.61386108398438,
"KL/mean": -212.56109619140625,
"KL/rejected_KL_mean": -249.50831604003906,
"KL/std": 123.15960693359375,
"epoch": 0.9448223733938019,
"fcm_dpo/beta": 0.004533303901553154,
"fcm_dpo/delta": 0.07208952307701111,
"fcm_dpo/margin": 73.89444732666016,
"fcm_dpo/q_t": 0.427746057510376,
"grad_norm": 12.597397804260254,
"learning_rate": 4.7714894655209174e-09,
"logits/chosen": 0.9103964567184448,
"logits/rejected": 0.836378276348114,
"logps/chosen": -229.82147216796875,
"logps/ref_chosen": -54.20761489868164,
"logps/ref_rejected": -84.93669128417969,
"logps/rejected": -334.44500732421875,
"loss": 1.1816,
"margin_dpo/margin_mean": 73.89444732666016,
"margin_dpo/margin_std": 147.33132934570312,
"step": 625
},
{
"KL/chosen_KL_mean": -165.85633850097656,
"KL/mean": -216.77398681640625,
"KL/rejected_KL_mean": -267.691650390625,
"KL/std": 134.21945190429688,
"epoch": 0.9463340891912321,
"fcm_dpo/beta": 0.004439115524291992,
"fcm_dpo/delta": -0.06053989380598068,
"fcm_dpo/margin": 101.8353271484375,
"fcm_dpo/q_t": 0.40118056535720825,
"grad_norm": 13.96564769744873,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 0.9465058445930481,
"logits/rejected": 0.8354614973068237,
"logps/chosen": -210.91836547851562,
"logps/ref_chosen": -45.06201934814453,
"logps/ref_rejected": -89.66368103027344,
"logps/rejected": -357.3553466796875,
"loss": 1.0962,
"margin_dpo/margin_mean": 101.8353271484375,
"margin_dpo/margin_std": 162.71804809570312,
"step": 626
},
{
"KL/chosen_KL_mean": -190.51968383789062,
"KL/mean": -244.76681518554688,
"KL/rejected_KL_mean": -299.0139465332031,
"KL/std": 139.9709930419922,
"epoch": 0.9478458049886621,
"fcm_dpo/beta": 0.00424289982765913,
"fcm_dpo/delta": -0.07100832462310791,
"fcm_dpo/margin": 108.49424743652344,
"fcm_dpo/q_t": 0.39188235998153687,
"grad_norm": 12.912845611572266,
"learning_rate": 4.271028567242818e-09,
"logits/chosen": 0.8388135433197021,
"logits/rejected": 0.7388994693756104,
"logps/chosen": -249.31072998046875,
"logps/ref_chosen": -58.791053771972656,
"logps/ref_rejected": -94.90802001953125,
"logps/rejected": -393.9219665527344,
"loss": 1.0535,
"margin_dpo/margin_mean": 108.49424743652344,
"margin_dpo/margin_std": 133.7566375732422,
"step": 627
},
{
"KL/chosen_KL_mean": -177.43270874023438,
"KL/mean": -227.32901000976562,
"KL/rejected_KL_mean": -277.22528076171875,
"KL/std": 129.86306762695312,
"epoch": 0.9493575207860923,
"fcm_dpo/beta": 0.004253713879734278,
"fcm_dpo/delta": -0.030077699571847916,
"fcm_dpo/margin": 99.7926025390625,
"fcm_dpo/q_t": 0.4006233811378479,
"grad_norm": 13.415197372436523,
"learning_rate": 4.0311050177251895e-09,
"logits/chosen": 0.8698492646217346,
"logits/rejected": 0.838459849357605,
"logps/chosen": -230.2362823486328,
"logps/ref_chosen": -52.80357360839844,
"logps/ref_rejected": -76.49468994140625,
"logps/rejected": -353.719970703125,
"loss": 1.1146,
"margin_dpo/margin_mean": 99.7926025390625,
"margin_dpo/margin_std": 148.1082763671875,
"step": 628
},
{
"KL/chosen_KL_mean": -189.38441467285156,
"KL/mean": -225.45184326171875,
"KL/rejected_KL_mean": -261.51922607421875,
"KL/std": 118.73931884765625,
"epoch": 0.9508692365835223,
"fcm_dpo/beta": 0.004356414079666138,
"fcm_dpo/delta": 0.09377999603748322,
"fcm_dpo/margin": 72.13484191894531,
"fcm_dpo/q_t": 0.43087315559387207,
"grad_norm": 11.81552505493164,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 0.8537815809249878,
"logits/rejected": 0.857978343963623,
"logps/chosen": -260.1019287109375,
"logps/ref_chosen": -70.71749877929688,
"logps/ref_rejected": -78.96273803710938,
"logps/rejected": -340.48199462890625,
"loss": 1.1703,
"margin_dpo/margin_mean": 72.13484191894531,
"margin_dpo/margin_std": 128.88232421875,
"step": 629
},
{
"KL/chosen_KL_mean": -175.94125366210938,
"KL/mean": -225.40545654296875,
"KL/rejected_KL_mean": -274.8696594238281,
"KL/std": 126.93406677246094,
"epoch": 0.9523809523809523,
"fcm_dpo/beta": 0.004350777715444565,
"fcm_dpo/delta": -0.03495955839753151,
"fcm_dpo/margin": 98.92840576171875,
"fcm_dpo/q_t": 0.4007263779640198,
"grad_norm": 10.979743003845215,
"learning_rate": 3.5719052736323806e-09,
"logits/chosen": 0.8208729028701782,
"logits/rejected": 0.7930917739868164,
"logps/chosen": -232.14266967773438,
"logps/ref_chosen": -56.201412200927734,
"logps/ref_rejected": -74.69807434082031,
"logps/rejected": -349.5677490234375,
"loss": 1.0891,
"margin_dpo/margin_mean": 98.92840576171875,
"margin_dpo/margin_std": 140.695556640625,
"step": 630
},
{
"KL/chosen_KL_mean": -166.6142120361328,
"KL/mean": -226.4366455078125,
"KL/rejected_KL_mean": -286.25909423828125,
"KL/std": 137.6940460205078,
"epoch": 0.9538926681783825,
"fcm_dpo/beta": 0.00401288690045476,
"fcm_dpo/delta": -0.10077112913131714,
"fcm_dpo/margin": 119.64486694335938,
"fcm_dpo/q_t": 0.3887532353401184,
"grad_norm": 11.224316596984863,
"learning_rate": 3.352641923861144e-09,
"logits/chosen": 0.9621773362159729,
"logits/rejected": 0.8688410520553589,
"logps/chosen": -225.434814453125,
"logps/ref_chosen": -58.82059860229492,
"logps/ref_rejected": -96.51437377929688,
"logps/rejected": -382.7734680175781,
"loss": 1.052,
"margin_dpo/margin_mean": 119.64486694335938,
"margin_dpo/margin_std": 151.0715789794922,
"step": 631
},
{
"KL/chosen_KL_mean": -169.34906005859375,
"KL/mean": -222.4647216796875,
"KL/rejected_KL_mean": -275.58038330078125,
"KL/std": 118.47867584228516,
"epoch": 0.9554043839758125,
"fcm_dpo/beta": 0.004019321873784065,
"fcm_dpo/delta": -0.031023576855659485,
"fcm_dpo/margin": 106.2313232421875,
"fcm_dpo/q_t": 0.39956796169281006,
"grad_norm": 11.955537796020508,
"learning_rate": 3.140277830901428e-09,
"logits/chosen": 0.8837977051734924,
"logits/rejected": 0.8737013339996338,
"logps/chosen": -228.13510131835938,
"logps/ref_chosen": -58.786048889160156,
"logps/ref_rejected": -67.21923828125,
"logps/rejected": -342.79962158203125,
"loss": 1.0674,
"margin_dpo/margin_mean": 106.2313232421875,
"margin_dpo/margin_std": 127.79652404785156,
"step": 632
},
{
"KL/chosen_KL_mean": -165.90658569335938,
"KL/mean": -208.74703979492188,
"KL/rejected_KL_mean": -251.58749389648438,
"KL/std": 116.66265869140625,
"epoch": 0.9569160997732427,
"fcm_dpo/beta": 0.004116720985621214,
"fcm_dpo/delta": 0.05110543221235275,
"fcm_dpo/margin": 85.68087768554688,
"fcm_dpo/q_t": 0.42053651809692383,
"grad_norm": 12.592107772827148,
"learning_rate": 2.9348189350335007e-09,
"logits/chosen": 0.8748718500137329,
"logits/rejected": 0.8350610733032227,
"logps/chosen": -218.0367889404297,
"logps/ref_chosen": -52.13019561767578,
"logps/ref_rejected": -67.23016357421875,
"logps/rejected": -318.817626953125,
"loss": 1.1355,
"margin_dpo/margin_mean": 85.68087768554688,
"margin_dpo/margin_std": 131.61073303222656,
"step": 633
},
{
"KL/chosen_KL_mean": -198.66600036621094,
"KL/mean": -213.45736694335938,
"KL/rejected_KL_mean": -228.24871826171875,
"KL/std": 122.29270935058594,
"epoch": 0.9584278155706727,
"fcm_dpo/beta": 0.004098186735063791,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 29.582706451416016,
"fcm_dpo/q_t": 0.47323864698410034,
"grad_norm": 14.698355674743652,
"learning_rate": 2.736270983384276e-09,
"logits/chosen": 0.9170514941215515,
"logits/rejected": 0.9342811703681946,
"logps/chosen": -259.64581298828125,
"logps/ref_chosen": -60.97979736328125,
"logps/ref_rejected": -58.50825119018555,
"logps/rejected": -286.7569580078125,
"loss": 1.3477,
"margin_dpo/margin_mean": 29.582706451416016,
"margin_dpo/margin_std": 139.38327026367188,
"step": 634
},
{
"KL/chosen_KL_mean": -215.3644256591797,
"KL/mean": -250.3736572265625,
"KL/rejected_KL_mean": -285.38287353515625,
"KL/std": 116.93312072753906,
"epoch": 0.9599395313681028,
"fcm_dpo/beta": 0.004337867721915245,
"fcm_dpo/delta": 0.10398625582456589,
"fcm_dpo/margin": 70.0184555053711,
"fcm_dpo/q_t": 0.43173298239707947,
"grad_norm": 11.96898078918457,
"learning_rate": 2.5446395297668287e-09,
"logits/chosen": 0.7966681718826294,
"logits/rejected": 0.7563410997390747,
"logps/chosen": -281.3374328613281,
"logps/ref_chosen": -65.9730224609375,
"logps/ref_rejected": -85.61317443847656,
"logps/rejected": -370.99603271484375,
"loss": 1.1911,
"margin_dpo/margin_mean": 70.0184555053711,
"margin_dpo/margin_std": 137.1169891357422,
"step": 635
},
{
"KL/chosen_KL_mean": -167.98959350585938,
"KL/mean": -215.552490234375,
"KL/rejected_KL_mean": -263.1153564453125,
"KL/std": 120.42471313476562,
"epoch": 0.9614512471655329,
"fcm_dpo/beta": 0.004312177188694477,
"fcm_dpo/delta": -0.011981801129877567,
"fcm_dpo/margin": 95.12577819824219,
"fcm_dpo/q_t": 0.4046896696090698,
"grad_norm": 9.827645301818848,
"learning_rate": 2.359929934524829e-09,
"logits/chosen": 0.8474336862564087,
"logits/rejected": 0.7673357129096985,
"logps/chosen": -217.1297607421875,
"logps/ref_chosen": -49.140167236328125,
"logps/ref_rejected": -81.26971435546875,
"logps/rejected": -344.38507080078125,
"loss": 1.0884,
"margin_dpo/margin_mean": 95.12577819824219,
"margin_dpo/margin_std": 126.56785583496094,
"step": 636
},
{
"KL/chosen_KL_mean": -226.68218994140625,
"KL/mean": -262.2201232910156,
"KL/rejected_KL_mean": -297.758056640625,
"KL/std": 124.49876403808594,
"epoch": 0.9629629629629629,
"fcm_dpo/beta": 0.0045396797358989716,
"fcm_dpo/delta": 0.08410540223121643,
"fcm_dpo/margin": 71.07585144042969,
"fcm_dpo/q_t": 0.4289219081401825,
"grad_norm": 13.477401733398438,
"learning_rate": 2.1821473643827137e-09,
"logits/chosen": 0.8291600942611694,
"logits/rejected": 0.7868282794952393,
"logps/chosen": -300.3787841796875,
"logps/ref_chosen": -73.69658660888672,
"logps/ref_rejected": -83.01487731933594,
"logps/rejected": -380.77294921875,
"loss": 1.1968,
"margin_dpo/margin_mean": 71.07585144042969,
"margin_dpo/margin_std": 150.5892333984375,
"step": 637
},
{
"KL/chosen_KL_mean": -196.17276000976562,
"KL/mean": -238.7854461669922,
"KL/rejected_KL_mean": -281.39813232421875,
"KL/std": 120.97940063476562,
"epoch": 0.9644746787603931,
"fcm_dpo/beta": 0.0045569646172225475,
"fcm_dpo/delta": 0.013032622635364532,
"fcm_dpo/margin": 85.22538757324219,
"fcm_dpo/q_t": 0.4132801294326782,
"grad_norm": 13.217300415039062,
"learning_rate": 2.0112967923011646e-09,
"logits/chosen": 0.8621759414672852,
"logits/rejected": 0.8292132616043091,
"logps/chosen": -258.954345703125,
"logps/ref_chosen": -62.78158187866211,
"logps/ref_rejected": -85.40478515625,
"logps/rejected": -366.80291748046875,
"loss": 1.1234,
"margin_dpo/margin_mean": 85.22538757324219,
"margin_dpo/margin_std": 135.74893188476562,
"step": 638
},
{
"KL/chosen_KL_mean": -175.19105529785156,
"KL/mean": -225.06832885742188,
"KL/rejected_KL_mean": -274.94561767578125,
"KL/std": 131.4615478515625,
"epoch": 0.9659863945578231,
"fcm_dpo/beta": 0.004495006985962391,
"fcm_dpo/delta": -0.05581279098987579,
"fcm_dpo/margin": 99.75454711914062,
"fcm_dpo/q_t": 0.39895889163017273,
"grad_norm": 15.222760200500488,
"learning_rate": 1.847382997337943e-09,
"logits/chosen": 0.8577284812927246,
"logits/rejected": 0.7885833978652954,
"logps/chosen": -228.9576416015625,
"logps/ref_chosen": -53.76658630371094,
"logps/ref_rejected": -72.30009460449219,
"logps/rejected": -347.2456970214844,
"loss": 1.0828,
"margin_dpo/margin_mean": 99.75454711914062,
"margin_dpo/margin_std": 143.67074584960938,
"step": 639
},
{
"KL/chosen_KL_mean": -182.3665771484375,
"KL/mean": -225.64895629882812,
"KL/rejected_KL_mean": -268.9313049316406,
"KL/std": 123.47915649414062,
"epoch": 0.9674981103552532,
"fcm_dpo/beta": 0.004439163953065872,
"fcm_dpo/delta": 0.017474712803959846,
"fcm_dpo/margin": 86.56473541259766,
"fcm_dpo/q_t": 0.4128446578979492,
"grad_norm": 13.212712287902832,
"learning_rate": 1.690410564514244e-09,
"logits/chosen": 0.9096243381500244,
"logits/rejected": 0.8597659468650818,
"logps/chosen": -233.78436279296875,
"logps/ref_chosen": -51.41777801513672,
"logps/ref_rejected": -77.27879333496094,
"logps/rejected": -346.2101135253906,
"loss": 1.1238,
"margin_dpo/margin_mean": 86.56473541259766,
"margin_dpo/margin_std": 135.56790161132812,
"step": 640
},
{
"KL/chosen_KL_mean": -191.44329833984375,
"KL/mean": -231.0288543701172,
"KL/rejected_KL_mean": -270.6143798828125,
"KL/std": 114.08880615234375,
"epoch": 0.9690098261526833,
"fcm_dpo/beta": 0.004546813666820526,
"fcm_dpo/delta": 0.04486197605729103,
"fcm_dpo/margin": 79.17110443115234,
"fcm_dpo/q_t": 0.41722530126571655,
"grad_norm": 13.189962387084961,
"learning_rate": 1.5403838846864692e-09,
"logits/chosen": 0.8492909073829651,
"logits/rejected": 0.8432836532592773,
"logps/chosen": -262.49798583984375,
"logps/ref_chosen": -71.0546646118164,
"logps/ref_rejected": -82.2440185546875,
"logps/rejected": -352.8583984375,
"loss": 1.1164,
"margin_dpo/margin_mean": 79.17110443115234,
"margin_dpo/margin_std": 108.83973693847656,
"step": 641
},
{
"KL/chosen_KL_mean": -200.26123046875,
"KL/mean": -229.521728515625,
"KL/rejected_KL_mean": -258.7822265625,
"KL/std": 112.85456085205078,
"epoch": 0.9705215419501134,
"fcm_dpo/beta": 0.004678209777921438,
"fcm_dpo/delta": 0.02495141327381134,
"fcm_dpo/margin": 58.52101135253906,
"fcm_dpo/q_t": 0.43944495916366577,
"grad_norm": 15.50776481628418,
"learning_rate": 1.3973071544233218e-09,
"logits/chosen": 0.8077861070632935,
"logits/rejected": 0.8297359943389893,
"logps/chosen": -269.19049072265625,
"logps/ref_chosen": -68.92927551269531,
"logps/ref_rejected": -70.85682678222656,
"logps/rejected": -329.6390686035156,
"loss": 1.2261,
"margin_dpo/margin_mean": 58.521018981933594,
"margin_dpo/margin_std": 135.40769958496094,
"step": 642
},
{
"KL/chosen_KL_mean": -182.4813232421875,
"KL/mean": -230.08364868164062,
"KL/rejected_KL_mean": -277.68597412109375,
"KL/std": 124.2804946899414,
"epoch": 0.9720332577475435,
"fcm_dpo/beta": 0.004592161625623703,
"fcm_dpo/delta": -0.04280085489153862,
"fcm_dpo/margin": 95.20464324951172,
"fcm_dpo/q_t": 0.401271790266037,
"grad_norm": 18.248327255249023,
"learning_rate": 1.261184375888541e-09,
"logits/chosen": 0.8441503047943115,
"logits/rejected": 0.778213381767273,
"logps/chosen": -247.79037475585938,
"logps/ref_chosen": -65.30903625488281,
"logps/ref_rejected": -83.61613464355469,
"logps/rejected": -361.3021240234375,
"loss": 1.0961,
"margin_dpo/margin_mean": 95.20463562011719,
"margin_dpo/margin_std": 142.61578369140625,
"step": 643
},
{
"KL/chosen_KL_mean": -173.4718017578125,
"KL/mean": -202.7007293701172,
"KL/rejected_KL_mean": -231.92965698242188,
"KL/std": 106.54061889648438,
"epoch": 0.9735449735449735,
"fcm_dpo/beta": 0.004578079096972942,
"fcm_dpo/delta": 0.013454647734761238,
"fcm_dpo/margin": 58.45787048339844,
"fcm_dpo/q_t": 0.43763917684555054,
"grad_norm": 11.429241180419922,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 0.939828634262085,
"logits/rejected": 0.912724494934082,
"logps/chosen": -224.47439575195312,
"logps/ref_chosen": -51.002601623535156,
"logps/ref_rejected": -64.46372985839844,
"logps/rejected": -296.39337158203125,
"loss": 1.2201,
"margin_dpo/margin_mean": 58.45787048339844,
"margin_dpo/margin_std": 126.97639465332031,
"step": 644
},
{
"KL/chosen_KL_mean": -190.52029418945312,
"KL/mean": -227.52386474609375,
"KL/rejected_KL_mean": -264.5274353027344,
"KL/std": 117.05931091308594,
"epoch": 0.9750566893424036,
"fcm_dpo/beta": 0.004725134931504726,
"fcm_dpo/delta": 0.05598610267043114,
"fcm_dpo/margin": 74.00714111328125,
"fcm_dpo/q_t": 0.4198160469532013,
"grad_norm": 16.172887802124023,
"learning_rate": 1.0098157099674987e-09,
"logits/chosen": 0.868065357208252,
"logits/rejected": 0.8554788827896118,
"logps/chosen": -251.48370361328125,
"logps/ref_chosen": -60.963409423828125,
"logps/ref_rejected": -69.73353576660156,
"logps/rejected": -334.260986328125,
"loss": 1.1443,
"margin_dpo/margin_mean": 74.00714111328125,
"margin_dpo/margin_std": 120.89301300048828,
"step": 645
},
{
"KL/chosen_KL_mean": -205.87545776367188,
"KL/mean": -243.1065673828125,
"KL/rejected_KL_mean": -280.3376770019531,
"KL/std": 141.6141357421875,
"epoch": 0.9765684051398337,
"fcm_dpo/beta": 0.004856896586716175,
"fcm_dpo/delta": 0.04285380616784096,
"fcm_dpo/margin": 74.46222686767578,
"fcm_dpo/q_t": 0.4203829765319824,
"grad_norm": 12.586745262145996,
"learning_rate": 8.945768539031783e-10,
"logits/chosen": 0.8746490478515625,
"logits/rejected": 0.8254890441894531,
"logps/chosen": -268.16552734375,
"logps/ref_chosen": -62.290069580078125,
"logps/ref_rejected": -85.54812622070312,
"logps/rejected": -365.88580322265625,
"loss": 1.1618,
"margin_dpo/margin_mean": 74.46221923828125,
"margin_dpo/margin_std": 138.90460205078125,
"step": 646
},
{
"KL/chosen_KL_mean": -203.10272216796875,
"KL/mean": -261.1683654785156,
"KL/rejected_KL_mean": -319.2340087890625,
"KL/std": 138.07496643066406,
"epoch": 0.9780801209372638,
"fcm_dpo/beta": 0.004566292278468609,
"fcm_dpo/delta": -0.15473519265651703,
"fcm_dpo/margin": 116.13129425048828,
"fcm_dpo/q_t": 0.3759966492652893,
"grad_norm": 14.314106941223145,
"learning_rate": 7.863060120144316e-10,
"logits/chosen": 0.8773380517959595,
"logits/rejected": 0.7988033294677734,
"logps/chosen": -270.61859130859375,
"logps/ref_chosen": -67.515869140625,
"logps/ref_rejected": -101.50871276855469,
"logps/rejected": -420.7427062988281,
"loss": 1.0135,
"margin_dpo/margin_mean": 116.13129425048828,
"margin_dpo/margin_std": 136.10638427734375,
"step": 647
},
{
"KL/chosen_KL_mean": -204.65838623046875,
"KL/mean": -243.85040283203125,
"KL/rejected_KL_mean": -283.04241943359375,
"KL/std": 119.53384399414062,
"epoch": 0.9795918367346939,
"fcm_dpo/beta": 0.004495234228670597,
"fcm_dpo/delta": 0.05285690724849701,
"fcm_dpo/margin": 78.38400268554688,
"fcm_dpo/q_t": 0.422693133354187,
"grad_norm": 12.690834999084473,
"learning_rate": 6.850062128694045e-10,
"logits/chosen": 0.8503772020339966,
"logits/rejected": 0.8010015487670898,
"logps/chosen": -269.25433349609375,
"logps/ref_chosen": -64.59593963623047,
"logps/ref_rejected": -83.384033203125,
"logps/rejected": -366.42645263671875,
"loss": 1.1861,
"margin_dpo/margin_mean": 78.38400268554688,
"margin_dpo/margin_std": 156.04074096679688,
"step": 648
},
{
"KL/chosen_KL_mean": -184.55648803710938,
"KL/mean": -225.07196044921875,
"KL/rejected_KL_mean": -265.5874328613281,
"KL/std": 115.69061279296875,
"epoch": 0.981103552532124,
"fcm_dpo/beta": 0.0046140593476593494,
"fcm_dpo/delta": 0.0292498916387558,
"fcm_dpo/margin": 81.03094482421875,
"fcm_dpo/q_t": 0.4144395589828491,
"grad_norm": 17.35785484313965,
"learning_rate": 5.906802900412788e-10,
"logits/chosen": 0.9027397632598877,
"logits/rejected": 0.850858211517334,
"logps/chosen": -233.8661346435547,
"logps/ref_chosen": -49.30964660644531,
"logps/ref_rejected": -73.73710632324219,
"logps/rejected": -339.32452392578125,
"loss": 1.1432,
"margin_dpo/margin_mean": 81.03094482421875,
"margin_dpo/margin_std": 137.96710205078125,
"step": 649
},
{
"KL/chosen_KL_mean": -178.10488891601562,
"KL/mean": -222.58074951171875,
"KL/rejected_KL_mean": -267.05657958984375,
"KL/std": 120.7571029663086,
"epoch": 0.982615268329554,
"fcm_dpo/beta": 0.004608334973454475,
"fcm_dpo/delta": -0.011857246980071068,
"fcm_dpo/margin": 88.95169067382812,
"fcm_dpo/q_t": 0.40797871351242065,
"grad_norm": 13.712696075439453,
"learning_rate": 5.033308820289184e-10,
"logits/chosen": 0.9457135200500488,
"logits/rejected": 0.896229088306427,
"logps/chosen": -233.16815185546875,
"logps/ref_chosen": -55.06325912475586,
"logps/ref_rejected": -77.39610290527344,
"logps/rejected": -344.45269775390625,
"loss": 1.1262,
"margin_dpo/margin_mean": 88.95169067382812,
"margin_dpo/margin_std": 148.16856384277344,
"step": 650
},
{
"KL/chosen_KL_mean": -176.51058959960938,
"KL/mean": -214.7784881591797,
"KL/rejected_KL_mean": -253.04638671875,
"KL/std": 118.41923522949219,
"epoch": 0.9841269841269841,
"fcm_dpo/beta": 0.004761428106576204,
"fcm_dpo/delta": 0.03895041346549988,
"fcm_dpo/margin": 76.53578186035156,
"fcm_dpo/q_t": 0.4195203185081482,
"grad_norm": 13.958356857299805,
"learning_rate": 4.2296043218295606e-10,
"logits/chosen": 0.9171409606933594,
"logits/rejected": 0.8589807748794556,
"logps/chosen": -230.5757598876953,
"logps/ref_chosen": -54.065162658691406,
"logps/ref_rejected": -77.79080200195312,
"logps/rejected": -330.8371887207031,
"loss": 1.1358,
"margin_dpo/margin_mean": 76.53578186035156,
"margin_dpo/margin_std": 123.71870422363281,
"step": 651
},
{
"KL/chosen_KL_mean": -209.03729248046875,
"KL/mean": -242.5619659423828,
"KL/rejected_KL_mean": -276.086669921875,
"KL/std": 132.26441955566406,
"epoch": 0.9856386999244142,
"fcm_dpo/beta": 0.004862497560679913,
"fcm_dpo/delta": 0.0819351002573967,
"fcm_dpo/margin": 67.04937744140625,
"fcm_dpo/q_t": 0.4319889545440674,
"grad_norm": 16.18764877319336,
"learning_rate": 3.4957118863768176e-10,
"logits/chosen": 0.8910359144210815,
"logits/rejected": 0.8544408082962036,
"logps/chosen": -272.6775817871094,
"logps/ref_chosen": -63.64030456542969,
"logps/ref_rejected": -78.86882019042969,
"logps/rejected": -354.95550537109375,
"loss": 1.2244,
"margin_dpo/margin_mean": 67.04937744140625,
"margin_dpo/margin_std": 160.25506591796875,
"step": 652
},
{
"KL/chosen_KL_mean": -185.3717803955078,
"KL/mean": -227.00796508789062,
"KL/rejected_KL_mean": -268.6441955566406,
"KL/std": 115.03123474121094,
"epoch": 0.9871504157218443,
"fcm_dpo/beta": 0.0050035035237669945,
"fcm_dpo/delta": -0.01987006887793541,
"fcm_dpo/margin": 83.27240753173828,
"fcm_dpo/q_t": 0.4065983295440674,
"grad_norm": 15.927544593811035,
"learning_rate": 2.831652042480093e-10,
"logits/chosen": 0.8695135116577148,
"logits/rejected": 0.8330869674682617,
"logps/chosen": -247.04014587402344,
"logps/ref_chosen": -61.668373107910156,
"logps/ref_rejected": -73.83012390136719,
"logps/rejected": -342.47430419921875,
"loss": 1.1174,
"margin_dpo/margin_mean": 83.27241516113281,
"margin_dpo/margin_std": 133.44725036621094,
"step": 653
},
{
"KL/chosen_KL_mean": -192.90228271484375,
"KL/mean": -233.95965576171875,
"KL/rejected_KL_mean": -275.01702880859375,
"KL/std": 139.22682189941406,
"epoch": 0.9886621315192744,
"fcm_dpo/beta": 0.0046582408249378204,
"fcm_dpo/delta": -0.09315278381109238,
"fcm_dpo/margin": 82.11475372314453,
"fcm_dpo/q_t": 0.41140812635421753,
"grad_norm": 13.904915809631348,
"learning_rate": 2.2374433653205016e-10,
"logits/chosen": 0.8664817810058594,
"logits/rejected": 0.7882392406463623,
"logps/chosen": -250.47055053710938,
"logps/ref_chosen": -57.568267822265625,
"logps/ref_rejected": -87.74789428710938,
"logps/rejected": -362.7649230957031,
"loss": 1.1324,
"margin_dpo/margin_mean": 82.11475372314453,
"margin_dpo/margin_std": 129.8606719970703,
"step": 654
},
{
"KL/chosen_KL_mean": -152.3089599609375,
"KL/mean": -211.10946655273438,
"KL/rejected_KL_mean": -269.90997314453125,
"KL/std": 120.35342407226562,
"epoch": 0.9901738473167044,
"fcm_dpo/beta": 0.004261736758053303,
"fcm_dpo/delta": -0.1305772364139557,
"fcm_dpo/margin": 117.60102081298828,
"fcm_dpo/q_t": 0.38035333156585693,
"grad_norm": 11.05696964263916,
"learning_rate": 1.7131024761923852e-10,
"logits/chosen": 0.8656521439552307,
"logits/rejected": 0.7972902655601501,
"logps/chosen": -204.45611572265625,
"logps/ref_chosen": -52.14714813232422,
"logps/ref_rejected": -80.85014343261719,
"logps/rejected": -350.7601318359375,
"loss": 1.0168,
"margin_dpo/margin_mean": 117.60101318359375,
"margin_dpo/margin_std": 123.00428771972656,
"step": 655
},
{
"KL/chosen_KL_mean": -204.72833251953125,
"KL/mean": -246.2228546142578,
"KL/rejected_KL_mean": -287.7173767089844,
"KL/std": 123.21992492675781,
"epoch": 0.9916855631141346,
"fcm_dpo/beta": 0.004393207840621471,
"fcm_dpo/delta": 0.0396343432366848,
"fcm_dpo/margin": 82.98904418945312,
"fcm_dpo/q_t": 0.4176909923553467,
"grad_norm": 11.27653694152832,
"learning_rate": 1.2586440420372934e-10,
"logits/chosen": 0.8115413188934326,
"logits/rejected": 0.7813726663589478,
"logps/chosen": -277.98504638671875,
"logps/ref_chosen": -73.25672912597656,
"logps/ref_rejected": -85.35127258300781,
"logps/rejected": -373.06866455078125,
"loss": 1.138,
"margin_dpo/margin_mean": 82.98904418945312,
"margin_dpo/margin_std": 135.81607055664062,
"step": 656
},
{
"KL/chosen_KL_mean": -176.87982177734375,
"KL/mean": -229.69300842285156,
"KL/rejected_KL_mean": -282.5062255859375,
"KL/std": 135.0663604736328,
"epoch": 0.9931972789115646,
"fcm_dpo/beta": 0.0043113818392157555,
"fcm_dpo/delta": -0.06402204930782318,
"fcm_dpo/margin": 105.62638854980469,
"fcm_dpo/q_t": 0.3964206874370575,
"grad_norm": 11.030098915100098,
"learning_rate": 8.740807750345913e-11,
"logits/chosen": 0.9459801912307739,
"logits/rejected": 0.8843368291854858,
"logps/chosen": -226.60321044921875,
"logps/ref_chosen": -49.72339630126953,
"logps/ref_rejected": -75.1568603515625,
"logps/rejected": -357.6630859375,
"loss": 1.0776,
"margin_dpo/margin_mean": 105.62638854980469,
"margin_dpo/margin_std": 149.3883056640625,
"step": 657
},
{
"KL/chosen_KL_mean": -190.58401489257812,
"KL/mean": -233.33816528320312,
"KL/rejected_KL_mean": -276.09234619140625,
"KL/std": 143.47869873046875,
"epoch": 0.9947089947089947,
"fcm_dpo/beta": 0.004380069673061371,
"fcm_dpo/delta": 0.025314655154943466,
"fcm_dpo/margin": 85.5083236694336,
"fcm_dpo/q_t": 0.41635990142822266,
"grad_norm": 10.85285758972168,
"learning_rate": 5.594234322453539e-11,
"logits/chosen": 0.8922824859619141,
"logits/rejected": 0.8569316864013672,
"logps/chosen": -253.63035583496094,
"logps/ref_chosen": -63.04634094238281,
"logps/ref_rejected": -83.44963073730469,
"logps/rejected": -359.5419616699219,
"loss": 1.1715,
"margin_dpo/margin_mean": 85.50831604003906,
"margin_dpo/margin_std": 166.18760681152344,
"step": 658
},
{
"KL/chosen_KL_mean": -192.74554443359375,
"KL/mean": -224.75161743164062,
"KL/rejected_KL_mean": -256.7576904296875,
"KL/std": 123.44963073730469,
"epoch": 0.9962207105064248,
"fcm_dpo/beta": 0.004360191989690065,
"fcm_dpo/delta": 0.01944735273718834,
"fcm_dpo/margin": 64.01215362548828,
"fcm_dpo/q_t": 0.4362792372703552,
"grad_norm": 14.053616523742676,
"learning_rate": 3.146808153123293e-11,
"logits/chosen": 0.9274283051490784,
"logits/rejected": 0.8760404586791992,
"logps/chosen": -247.82574462890625,
"logps/ref_chosen": -55.0802001953125,
"logps/ref_rejected": -71.91049194335938,
"logps/rejected": -328.6681823730469,
"loss": 1.2169,
"margin_dpo/margin_mean": 64.01214599609375,
"margin_dpo/margin_std": 140.3470458984375,
"step": 659
},
{
"KL/chosen_KL_mean": -180.63677978515625,
"KL/mean": -235.10302734375,
"KL/rejected_KL_mean": -289.56927490234375,
"KL/std": 133.57896423339844,
"epoch": 0.9977324263038548,
"fcm_dpo/beta": 0.004226707387715578,
"fcm_dpo/delta": -0.07013650238513947,
"fcm_dpo/margin": 108.93248748779297,
"fcm_dpo/q_t": 0.3952319025993347,
"grad_norm": 13.667136192321777,
"learning_rate": 1.3985977021235829e-11,
"logits/chosen": 0.9608404636383057,
"logits/rejected": 0.9127137660980225,
"logps/chosen": -235.16268920898438,
"logps/ref_chosen": -54.525917053222656,
"logps/ref_rejected": -81.23604583740234,
"logps/rejected": -370.8052978515625,
"loss": 1.0608,
"margin_dpo/margin_mean": 108.93248748779297,
"margin_dpo/margin_std": 145.68505859375,
"step": 660
},
{
"KL/chosen_KL_mean": -210.68487548828125,
"KL/mean": -243.6689910888672,
"KL/rejected_KL_mean": -276.653076171875,
"KL/std": 131.54515075683594,
"epoch": 0.999244142101285,
"fcm_dpo/beta": 0.004441337194293737,
"fcm_dpo/delta": 0.1156606376171112,
"fcm_dpo/margin": 65.96818542480469,
"fcm_dpo/q_t": 0.4353044927120209,
"grad_norm": 11.607975959777832,
"learning_rate": 3.4965187065971735e-12,
"logits/chosen": 0.8547237515449524,
"logits/rejected": 0.7953514456748962,
"logps/chosen": -271.0575256347656,
"logps/ref_chosen": -60.37263870239258,
"logps/ref_rejected": -77.42874145507812,
"logps/rejected": -354.08184814453125,
"loss": 1.2196,
"margin_dpo/margin_mean": 65.96818542480469,
"margin_dpo/margin_std": 151.36160278320312,
"step": 661
},
{
"epoch": 0.999244142101285,
"step": 661,
"total_flos": 0.0,
"train_loss": 1.1683954546924078,
"train_runtime": 1648.9587,
"train_samples_per_second": 25.674,
"train_steps_per_second": 0.401
}
],
"logging_steps": 1,
"max_steps": 661,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink