W-61/llama3-hh-harmless-qt045-b0p8-20260429-085449
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
6f346413ec25846e2dcbc3fdab033986f02c3031
llama3-hh-harmless-qt045-b0…/trainer_state.json
ModelHub XC 6f346413ec 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama3-hh-harmless-qt045-b0p8-20260429-085449
Source: Original Platform
2026-06-16 04:42:19 +08:00

15247 lines
572 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999244142101285,
"eval_steps": 200,
"global_step": 661,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.02867889404296875,
"KL/mean": 0.029354453086853027,
"KL/rejected_KL_mean": 0.030029296875,
"KL/std": 0.2071000635623932,
"epoch": 0.0015117157974300832,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013532638549804688,
"fcm_dpo/q_t": 0.5001926422119141,
"grad_norm": 227.88804626464844,
"learning_rate": 0.0,
"logits/chosen": 0.13337239623069763,
"logits/rejected": 0.12492949515581131,
"logps/chosen": -64.5841293334961,
"logps/ref_chosen": -64.61280822753906,
"logps/ref_rejected": -64.17195129394531,
"logps/rejected": -64.14192199707031,
"loss": 1.3978,
"margin_dpo/margin_mean": -0.0013527870178222656,
"margin_dpo/margin_std": 0.2561596930027008,
"step": 1
},
{
"KL/chosen_KL_mean": -0.00289154052734375,
"KL/mean": -0.021616414189338684,
"KL/rejected_KL_mean": -0.04033660888671875,
"KL/std": 0.19624735414981842,
"epoch": 0.0030234315948601664,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.037450045347213745,
"fcm_dpo/q_t": 0.49259763956069946,
"grad_norm": 222.1438751220703,
"learning_rate": 7.462686567164179e-09,
"logits/chosen": 0.09414851665496826,
"logits/rejected": 0.07363267242908478,
"logps/chosen": -56.101890563964844,
"logps/ref_chosen": -56.0989990234375,
"logps/ref_rejected": -66.59971618652344,
"logps/rejected": -66.64006042480469,
"loss": 1.3697,
"margin_dpo/margin_mean": 0.03744968771934509,
"margin_dpo/margin_std": 0.27811938524246216,
"step": 2
},
{
"KL/chosen_KL_mean": 0.030059814453125,
"KL/mean": 0.01856975257396698,
"KL/rejected_KL_mean": 0.00707244873046875,
"KL/std": 0.2663958966732025,
"epoch": 0.0045351473922902496,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02298787236213684,
"fcm_dpo/q_t": 0.4953998029232025,
"grad_norm": 254.62628173828125,
"learning_rate": 1.4925373134328357e-08,
"logits/chosen": 0.0995001345872879,
"logits/rejected": 0.061426181346178055,
"logps/chosen": -65.42720031738281,
"logps/ref_chosen": -65.45726013183594,
"logps/ref_rejected": -90.82853698730469,
"logps/rejected": -90.82145690917969,
"loss": 1.3905,
"margin_dpo/margin_mean": 0.022987276315689087,
"margin_dpo/margin_std": 0.3719334900379181,
"step": 3
},
{
"KL/chosen_KL_mean": 0.008388519287109375,
"KL/mean": 0.007060617208480835,
"KL/rejected_KL_mean": 0.00572967529296875,
"KL/std": 0.22156520187854767,
"epoch": 0.006046863189720333,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.002654552459716797,
"fcm_dpo/q_t": 0.49956855177879333,
"grad_norm": 287.84783935546875,
"learning_rate": 2.2388059701492534e-08,
"logits/chosen": 0.10069665312767029,
"logits/rejected": 0.08469942957162857,
"logps/chosen": -76.85179138183594,
"logps/ref_chosen": -76.86018371582031,
"logps/ref_rejected": -79.91523742675781,
"logps/rejected": -79.90950775146484,
"loss": 1.4036,
"margin_dpo/margin_mean": 0.0026539862155914307,
"margin_dpo/margin_std": 0.34323328733444214,
"step": 4
},
{
"KL/chosen_KL_mean": 0.011861801147460938,
"KL/mean": -0.012050449848175049,
"KL/rejected_KL_mean": -0.035961151123046875,
"KL/std": 0.23195374011993408,
"epoch": 0.007558578987150416,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.047826290130615234,
"fcm_dpo/q_t": 0.49080324172973633,
"grad_norm": 228.13427734375,
"learning_rate": 2.9850746268656714e-08,
"logits/chosen": 0.04918619990348816,
"logits/rejected": 0.011818725615739822,
"logps/chosen": -62.95948028564453,
"logps/ref_chosen": -62.97134017944336,
"logps/ref_rejected": -79.9192123413086,
"logps/rejected": -79.95516967773438,
"loss": 1.3646,
"margin_dpo/margin_mean": 0.04782620072364807,
"margin_dpo/margin_std": 0.315399169921875,
"step": 5
},
{
"KL/chosen_KL_mean": -0.03392982482910156,
"KL/mean": 0.009025231003761292,
"KL/rejected_KL_mean": 0.0519866943359375,
"KL/std": 0.21147847175598145,
"epoch": 0.009070294784580499,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.08590993285179138,
"fcm_dpo/q_t": 0.5169426202774048,
"grad_norm": 252.74085998535156,
"learning_rate": 3.731343283582089e-08,
"logits/chosen": 0.14063377678394318,
"logits/rejected": 0.10133487731218338,
"logps/chosen": -51.34129333496094,
"logps/ref_chosen": -51.30736541748047,
"logps/ref_rejected": -82.77239227294922,
"logps/rejected": -82.72040557861328,
"loss": 1.4724,
"margin_dpo/margin_mean": -0.08591002225875854,
"margin_dpo/margin_std": 0.3187505602836609,
"step": 6
},
{
"KL/chosen_KL_mean": 0.02046966552734375,
"KL/mean": 0.021168455481529236,
"KL/rejected_KL_mean": 0.0218658447265625,
"KL/std": 0.1829671859741211,
"epoch": 0.010582010582010581,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.001397162675857544,
"fcm_dpo/q_t": 0.5002532601356506,
"grad_norm": 221.77197265625,
"learning_rate": 4.477611940298507e-08,
"logits/chosen": 0.03592286631464958,
"logits/rejected": -0.009084239602088928,
"logps/chosen": -51.438941955566406,
"logps/ref_chosen": -51.45941162109375,
"logps/ref_rejected": -66.3828125,
"logps/rejected": -66.36094665527344,
"loss": 1.3963,
"margin_dpo/margin_mean": -0.0013970732688903809,
"margin_dpo/margin_std": 0.23323728144168854,
"step": 7
},
{
"KL/chosen_KL_mean": 0.020849227905273438,
"KL/mean": 0.012436389923095703,
"KL/rejected_KL_mean": 0.0040283203125,
"KL/std": 0.24311554431915283,
"epoch": 0.012093726379440665,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01681619882583618,
"fcm_dpo/q_t": 0.49677836894989014,
"grad_norm": 223.00634765625,
"learning_rate": 5.223880597014925e-08,
"logits/chosen": 0.07211041450500488,
"logits/rejected": 0.04997313767671585,
"logps/chosen": -62.17669677734375,
"logps/ref_chosen": -62.197547912597656,
"logps/ref_rejected": -74.66180419921875,
"logps/rejected": -74.65777587890625,
"loss": 1.3933,
"margin_dpo/margin_mean": 0.016815185546875,
"margin_dpo/margin_std": 0.3559471666812897,
"step": 8
},
{
"KL/chosen_KL_mean": -0.04191398620605469,
"KL/mean": -0.04535558819770813,
"KL/rejected_KL_mean": -0.048801422119140625,
"KL/std": 0.22056418657302856,
"epoch": 0.013605442176870748,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.006889760494232178,
"fcm_dpo/q_t": 0.49866464734077454,
"grad_norm": 253.6171875,
"learning_rate": 5.970149253731343e-08,
"logits/chosen": 0.15722443163394928,
"logits/rejected": 0.09891875833272934,
"logps/chosen": -55.671634674072266,
"logps/ref_chosen": -55.629722595214844,
"logps/ref_rejected": -86.21221923828125,
"logps/rejected": -86.26102447509766,
"loss": 1.3941,
"margin_dpo/margin_mean": 0.006889969110488892,
"margin_dpo/margin_std": 0.2871861457824707,
"step": 9
},
{
"KL/chosen_KL_mean": 0.028337478637695312,
"KL/mean": 0.009996294975280762,
"KL/rejected_KL_mean": -0.008350372314453125,
"KL/std": 0.242633655667305,
"epoch": 0.015117157974300832,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03668475151062012,
"fcm_dpo/q_t": 0.4932301342487335,
"grad_norm": 237.77821350097656,
"learning_rate": 6.71641791044776e-08,
"logits/chosen": 0.14226600527763367,
"logits/rejected": 0.11069996654987335,
"logps/chosen": -62.662261962890625,
"logps/ref_chosen": -62.69060134887695,
"logps/ref_rejected": -90.610107421875,
"logps/rejected": -90.61846160888672,
"loss": 1.3812,
"margin_dpo/margin_mean": 0.036684393882751465,
"margin_dpo/margin_std": 0.3912660777568817,
"step": 10
},
{
"KL/chosen_KL_mean": 0.021038055419921875,
"KL/mean": -0.009770780801773071,
"KL/rejected_KL_mean": -0.04058074951171875,
"KL/std": 0.20581898093223572,
"epoch": 0.016628873771730914,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06162160634994507,
"fcm_dpo/q_t": 0.4876581132411957,
"grad_norm": 225.41688537597656,
"learning_rate": 7.462686567164178e-08,
"logits/chosen": 0.11608986556529999,
"logits/rejected": 0.10907270014286041,
"logps/chosen": -65.74607849121094,
"logps/ref_chosen": -65.76712036132812,
"logps/ref_rejected": -72.4764633178711,
"logps/rejected": -72.51704406738281,
"loss": 1.351,
"margin_dpo/margin_mean": 0.06162214279174805,
"margin_dpo/margin_std": 0.28799864649772644,
"step": 11
},
{
"KL/chosen_KL_mean": -0.010614395141601562,
"KL/mean": 0.00831557810306549,
"KL/rejected_KL_mean": 0.02724456787109375,
"KL/std": 0.21397629380226135,
"epoch": 0.018140589569160998,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03785964846611023,
"fcm_dpo/q_t": 0.5074305534362793,
"grad_norm": 231.9459686279297,
"learning_rate": 8.208955223880596e-08,
"logits/chosen": 0.04229931905865669,
"logits/rejected": 0.02573547512292862,
"logps/chosen": -60.71550750732422,
"logps/ref_chosen": -60.704891204833984,
"logps/ref_rejected": -69.41564178466797,
"logps/rejected": -69.38839721679688,
"loss": 1.4314,
"margin_dpo/margin_mean": -0.03785929083824158,
"margin_dpo/margin_std": 0.3016618490219116,
"step": 12
},
{
"KL/chosen_KL_mean": 0.003421783447265625,
"KL/mean": 0.03152443468570709,
"KL/rejected_KL_mean": 0.05963134765625,
"KL/std": 0.22777202725410461,
"epoch": 0.019652305366591082,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05621209740638733,
"fcm_dpo/q_t": 0.5108703970909119,
"grad_norm": 244.4044952392578,
"learning_rate": 8.955223880597014e-08,
"logits/chosen": 0.1259368658065796,
"logits/rejected": 0.06180703267455101,
"logps/chosen": -49.90583419799805,
"logps/ref_chosen": -49.90925598144531,
"logps/ref_rejected": -92.37818145751953,
"logps/rejected": -92.31855010986328,
"loss": 1.4487,
"margin_dpo/margin_mean": -0.05621263384819031,
"margin_dpo/margin_std": 0.32052451372146606,
"step": 13
},
{
"KL/chosen_KL_mean": 0.05495262145996094,
"KL/mean": 0.012301474809646606,
"KL/rejected_KL_mean": -0.0303497314453125,
"KL/std": 0.2289624810218811,
"epoch": 0.021164021164021163,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08529627323150635,
"fcm_dpo/q_t": 0.4840930998325348,
"grad_norm": 221.74154663085938,
"learning_rate": 9.701492537313432e-08,
"logits/chosen": 0.06417852640151978,
"logits/rejected": 0.04712294787168503,
"logps/chosen": -60.5638427734375,
"logps/ref_chosen": -60.61879348754883,
"logps/ref_rejected": -71.79306030273438,
"logps/rejected": -71.82341003417969,
"loss": 1.3411,
"margin_dpo/margin_mean": 0.08529558777809143,
"margin_dpo/margin_std": 0.37497806549072266,
"step": 14
},
{
"KL/chosen_KL_mean": -0.013608932495117188,
"KL/mean": -0.002418234944343567,
"KL/rejected_KL_mean": 0.008769989013671875,
"KL/std": 0.23164832592010498,
"epoch": 0.022675736961451247,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.022380679845809937,
"fcm_dpo/q_t": 0.504250168800354,
"grad_norm": 275.0318908691406,
"learning_rate": 1.044776119402985e-07,
"logits/chosen": 0.0738568902015686,
"logits/rejected": 0.030333304777741432,
"logps/chosen": -63.48314666748047,
"logps/ref_chosen": -63.46953582763672,
"logps/ref_rejected": -88.88951110839844,
"logps/rejected": -88.88074493408203,
"loss": 1.4208,
"margin_dpo/margin_mean": -0.022380709648132324,
"margin_dpo/margin_std": 0.32323992252349854,
"step": 15
},
{
"KL/chosen_KL_mean": -0.027456283569335938,
"KL/mean": -0.030255019664764404,
"KL/rejected_KL_mean": -0.03305816650390625,
"KL/std": 0.18161174654960632,
"epoch": 0.02418745275888133,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.005598485469818115,
"fcm_dpo/q_t": 0.498818576335907,
"grad_norm": 215.34849548339844,
"learning_rate": 1.1194029850746268e-07,
"logits/chosen": 0.11181557178497314,
"logits/rejected": 0.07493522763252258,
"logps/chosen": -46.55975341796875,
"logps/ref_chosen": -46.53229904174805,
"logps/ref_rejected": -74.27533721923828,
"logps/rejected": -74.30839538574219,
"loss": 1.3913,
"margin_dpo/margin_mean": 0.005598574876785278,
"margin_dpo/margin_std": 0.2444663643836975,
"step": 16
},
{
"KL/chosen_KL_mean": 0.0041522979736328125,
"KL/mean": -0.012048691511154175,
"KL/rejected_KL_mean": -0.0282440185546875,
"KL/std": 0.21299785375595093,
"epoch": 0.025699168556311415,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03239566087722778,
"fcm_dpo/q_t": 0.4937340319156647,
"grad_norm": 251.91502380371094,
"learning_rate": 1.1940298507462686e-07,
"logits/chosen": 0.05359330773353577,
"logits/rejected": 0.03492668643593788,
"logps/chosen": -64.07368469238281,
"logps/ref_chosen": -64.07783508300781,
"logps/ref_rejected": -86.40876770019531,
"logps/rejected": -86.43701171875,
"loss": 1.3716,
"margin_dpo/margin_mean": 0.032395362854003906,
"margin_dpo/margin_std": 0.26138976216316223,
"step": 17
},
{
"KL/chosen_KL_mean": 0.04368019104003906,
"KL/mean": 0.015288189053535461,
"KL/rejected_KL_mean": -0.013103485107421875,
"KL/std": 0.214588925242424,
"epoch": 0.027210884353741496,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.056785255670547485,
"fcm_dpo/q_t": 0.4887694716453552,
"grad_norm": 224.17413330078125,
"learning_rate": 1.2686567164179106e-07,
"logits/chosen": 0.08548756688833237,
"logits/rejected": 0.04056599363684654,
"logps/chosen": -44.830657958984375,
"logps/ref_chosen": -44.87433624267578,
"logps/ref_rejected": -70.97604370117188,
"logps/rejected": -70.98915100097656,
"loss": 1.3526,
"margin_dpo/margin_mean": 0.05678561329841614,
"margin_dpo/margin_std": 0.2663358449935913,
"step": 18
},
{
"KL/chosen_KL_mean": 0.07179832458496094,
"KL/mean": 0.035570770502090454,
"KL/rejected_KL_mean": -0.00066375732421875,
"KL/std": 0.2750711739063263,
"epoch": 0.02872260015117158,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07246798276901245,
"fcm_dpo/q_t": 0.48557358980178833,
"grad_norm": 245.1780242919922,
"learning_rate": 1.343283582089552e-07,
"logits/chosen": 0.09194637835025787,
"logits/rejected": 0.0781373679637909,
"logps/chosen": -68.0880126953125,
"logps/ref_chosen": -68.1598129272461,
"logps/ref_rejected": -81.17138671875,
"logps/rejected": -81.17205810546875,
"loss": 1.3506,
"margin_dpo/margin_mean": 0.07246837019920349,
"margin_dpo/margin_std": 0.36530712246894836,
"step": 19
},
{
"KL/chosen_KL_mean": 0.03515625,
"KL/mean": 0.021619953215122223,
"KL/rejected_KL_mean": 0.008083343505859375,
"KL/std": 0.22413024306297302,
"epoch": 0.030234315948601664,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02707172930240631,
"fcm_dpo/q_t": 0.49454957246780396,
"grad_norm": 237.1182861328125,
"learning_rate": 1.4179104477611938e-07,
"logits/chosen": 0.1436234712600708,
"logits/rejected": 0.12027327716350555,
"logps/chosen": -53.64340591430664,
"logps/ref_chosen": -53.67856216430664,
"logps/ref_rejected": -74.16911315917969,
"logps/rejected": -74.1610336303711,
"loss": 1.3784,
"margin_dpo/margin_mean": 0.027072086930274963,
"margin_dpo/margin_std": 0.29309147596359253,
"step": 20
},
{
"KL/chosen_KL_mean": -0.012136459350585938,
"KL/mean": -0.007877066731452942,
"KL/rejected_KL_mean": -0.003620147705078125,
"KL/std": 0.19681471586227417,
"epoch": 0.031746031746031744,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.008510202169418335,
"fcm_dpo/q_t": 0.5017505884170532,
"grad_norm": 230.82269287109375,
"learning_rate": 1.4925373134328355e-07,
"logits/chosen": 0.12048260867595673,
"logits/rejected": 0.09423836320638657,
"logps/chosen": -64.71369171142578,
"logps/ref_chosen": -64.70155334472656,
"logps/ref_rejected": -81.02095031738281,
"logps/rejected": -81.02456665039062,
"loss": 1.4026,
"margin_dpo/margin_mean": -0.008510619401931763,
"margin_dpo/margin_std": 0.23914138972759247,
"step": 21
},
{
"KL/chosen_KL_mean": 0.006290435791015625,
"KL/mean": -0.014534056186676025,
"KL/rejected_KL_mean": -0.035358428955078125,
"KL/std": 0.23257115483283997,
"epoch": 0.03325774754346183,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.041648805141448975,
"fcm_dpo/q_t": 0.4913468360900879,
"grad_norm": 234.92774963378906,
"learning_rate": 1.5671641791044775e-07,
"logits/chosen": -0.009143848903477192,
"logits/rejected": -0.029366828501224518,
"logps/chosen": -58.029701232910156,
"logps/ref_chosen": -58.03599166870117,
"logps/ref_rejected": -80.72721862792969,
"logps/rejected": -80.7625732421875,
"loss": 1.3693,
"margin_dpo/margin_mean": 0.041648685932159424,
"margin_dpo/margin_std": 0.31927213072776794,
"step": 22
},
{
"KL/chosen_KL_mean": -0.021076202392578125,
"KL/mean": -0.013543367385864258,
"KL/rejected_KL_mean": -0.006008148193359375,
"KL/std": 0.2585999667644501,
"epoch": 0.03476946334089191,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.015064418315887451,
"fcm_dpo/q_t": 0.5026655197143555,
"grad_norm": 280.2370910644531,
"learning_rate": 1.6417910447761193e-07,
"logits/chosen": 0.1390341967344284,
"logits/rejected": 0.11366377770900726,
"logps/chosen": -66.37716674804688,
"logps/ref_chosen": -66.35608673095703,
"logps/ref_rejected": -93.02769470214844,
"logps/rejected": -93.03369903564453,
"loss": 1.4177,
"margin_dpo/margin_mean": -0.015064775943756104,
"margin_dpo/margin_std": 0.3431517481803894,
"step": 23
},
{
"KL/chosen_KL_mean": -0.015094757080078125,
"KL/mean": -0.016147926449775696,
"KL/rejected_KL_mean": -0.017202377319335938,
"KL/std": 0.1953171342611313,
"epoch": 0.036281179138321996,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.002107471227645874,
"fcm_dpo/q_t": 0.49957120418548584,
"grad_norm": 215.9775390625,
"learning_rate": 1.716417910447761e-07,
"logits/chosen": 0.15169034898281097,
"logits/rejected": 0.11822134256362915,
"logps/chosen": -54.47633361816406,
"logps/ref_chosen": -54.461238861083984,
"logps/ref_rejected": -68.33817291259766,
"logps/rejected": -68.35537719726562,
"loss": 1.3942,
"margin_dpo/margin_mean": 0.002107083797454834,
"margin_dpo/margin_std": 0.23902641236782074,
"step": 24
},
{
"KL/chosen_KL_mean": -0.015642166137695312,
"KL/mean": -0.01463077962398529,
"KL/rejected_KL_mean": -0.013622283935546875,
"KL/std": 0.2428501695394516,
"epoch": 0.03779289493575208,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0020219385623931885,
"fcm_dpo/q_t": 0.5008809566497803,
"grad_norm": 233.53453063964844,
"learning_rate": 1.7910447761194027e-07,
"logits/chosen": 0.08950161933898926,
"logits/rejected": 0.038990531116724014,
"logps/chosen": -60.01984405517578,
"logps/ref_chosen": -60.00420379638672,
"logps/ref_rejected": -90.47376251220703,
"logps/rejected": -90.48738098144531,
"loss": 1.4046,
"margin_dpo/margin_mean": -0.0020221471786499023,
"margin_dpo/margin_std": 0.32416456937789917,
"step": 25
},
{
"KL/chosen_KL_mean": -0.008241653442382812,
"KL/mean": -0.013406708836555481,
"KL/rejected_KL_mean": -0.0185699462890625,
"KL/std": 0.22539734840393066,
"epoch": 0.039304610733182165,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.010331660509109497,
"fcm_dpo/q_t": 0.49805325269699097,
"grad_norm": 234.74398803710938,
"learning_rate": 1.8656716417910447e-07,
"logits/chosen": 0.10135327279567719,
"logits/rejected": 0.08315593004226685,
"logps/chosen": -56.827396392822266,
"logps/ref_chosen": -56.81915283203125,
"logps/ref_rejected": -77.84333038330078,
"logps/rejected": -77.86190032958984,
"loss": 1.3934,
"margin_dpo/margin_mean": 0.010331422090530396,
"margin_dpo/margin_std": 0.3104252219200134,
"step": 26
},
{
"KL/chosen_KL_mean": 0.012708663940429688,
"KL/mean": -0.006882116198539734,
"KL/rejected_KL_mean": -0.026475906372070312,
"KL/std": 0.19745078682899475,
"epoch": 0.04081632653061224,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03918078541755676,
"fcm_dpo/q_t": 0.49225401878356934,
"grad_norm": 229.41184997558594,
"learning_rate": 1.9402985074626865e-07,
"logits/chosen": 0.11735519021749496,
"logits/rejected": 0.09195482730865479,
"logps/chosen": -62.86431884765625,
"logps/ref_chosen": -62.87702560424805,
"logps/ref_rejected": -71.34437561035156,
"logps/rejected": -71.370849609375,
"loss": 1.3656,
"margin_dpo/margin_mean": 0.03918081521987915,
"margin_dpo/margin_std": 0.2550206184387207,
"step": 27
},
{
"KL/chosen_KL_mean": -0.04155158996582031,
"KL/mean": -0.02161906659603119,
"KL/rejected_KL_mean": -0.0016841888427734375,
"KL/std": 0.21058428287506104,
"epoch": 0.042328042328042326,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.039867013692855835,
"fcm_dpo/q_t": 0.5077934265136719,
"grad_norm": 230.16476440429688,
"learning_rate": 2.0149253731343282e-07,
"logits/chosen": 0.05627727508544922,
"logits/rejected": 0.04757063090801239,
"logps/chosen": -59.87492752075195,
"logps/ref_chosen": -59.8333740234375,
"logps/ref_rejected": -70.39804077148438,
"logps/rejected": -70.39971923828125,
"loss": 1.4301,
"margin_dpo/margin_mean": -0.03986704349517822,
"margin_dpo/margin_std": 0.26896584033966064,
"step": 28
},
{
"KL/chosen_KL_mean": -0.02333831787109375,
"KL/mean": -0.027756929397583008,
"KL/rejected_KL_mean": -0.03217315673828125,
"KL/std": 0.2341362088918686,
"epoch": 0.04383975812547241,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.008836507797241211,
"fcm_dpo/q_t": 0.4984675645828247,
"grad_norm": 262.4019775390625,
"learning_rate": 2.08955223880597e-07,
"logits/chosen": 0.15859892964363098,
"logits/rejected": 0.1403963267803192,
"logps/chosen": -74.14353942871094,
"logps/ref_chosen": -74.12020111083984,
"logps/ref_rejected": -83.33099365234375,
"logps/rejected": -83.3631591796875,
"loss": 1.3981,
"margin_dpo/margin_mean": 0.008836179971694946,
"margin_dpo/margin_std": 0.3412613868713379,
"step": 29
},
{
"KL/chosen_KL_mean": -0.04350852966308594,
"KL/mean": -0.03481011092662811,
"KL/rejected_KL_mean": -0.026111602783203125,
"KL/std": 0.2395039200782776,
"epoch": 0.045351473922902494,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.017396360635757446,
"fcm_dpo/q_t": 0.5035183429718018,
"grad_norm": 245.30319213867188,
"learning_rate": 2.1641791044776117e-07,
"logits/chosen": 0.12713733315467834,
"logits/rejected": 0.07243612408638,
"logps/chosen": -50.7947998046875,
"logps/ref_chosen": -50.75128936767578,
"logps/ref_rejected": -89.29063415527344,
"logps/rejected": -89.31674194335938,
"loss": 1.4161,
"margin_dpo/margin_mean": -0.01739645004272461,
"margin_dpo/margin_std": 0.3162926435470581,
"step": 30
},
{
"KL/chosen_KL_mean": -0.05584144592285156,
"KL/mean": -0.07723797857761383,
"KL/rejected_KL_mean": -0.0986328125,
"KL/std": 0.25701966881752014,
"epoch": 0.04686318972033258,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.042792826890945435,
"fcm_dpo/q_t": 0.49151384830474854,
"grad_norm": 272.9398193359375,
"learning_rate": 2.2388059701492537e-07,
"logits/chosen": 0.10328017175197601,
"logits/rejected": 0.057278163731098175,
"logps/chosen": -65.39259338378906,
"logps/ref_chosen": -65.33675384521484,
"logps/ref_rejected": -100.76666259765625,
"logps/rejected": -100.86529541015625,
"loss": 1.372,
"margin_dpo/margin_mean": 0.042792946100234985,
"margin_dpo/margin_std": 0.35190892219543457,
"step": 31
},
{
"KL/chosen_KL_mean": -0.003879547119140625,
"KL/mean": -0.011247843503952026,
"KL/rejected_KL_mean": -0.01862335205078125,
"KL/std": 0.23257334530353546,
"epoch": 0.04837490551776266,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01474606990814209,
"fcm_dpo/q_t": 0.4971124231815338,
"grad_norm": 247.50511169433594,
"learning_rate": 2.3134328358208954e-07,
"logits/chosen": 0.09891624748706818,
"logits/rejected": 0.09087042510509491,
"logps/chosen": -67.18721008300781,
"logps/ref_chosen": -67.18333435058594,
"logps/ref_rejected": -82.80763244628906,
"logps/rejected": -82.82626342773438,
"loss": 1.3941,
"margin_dpo/margin_mean": 0.014745950698852539,
"margin_dpo/margin_std": 0.3449150025844574,
"step": 32
},
{
"KL/chosen_KL_mean": 0.008508682250976562,
"KL/mean": -0.023860938847064972,
"KL/rejected_KL_mean": -0.0562286376953125,
"KL/std": 0.24450770020484924,
"epoch": 0.049886621315192746,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06473095715045929,
"fcm_dpo/q_t": 0.4871603548526764,
"grad_norm": 245.69308471679688,
"learning_rate": 2.388059701492537e-07,
"logits/chosen": 0.026495473459362984,
"logits/rejected": 0.0007232502102851868,
"logps/chosen": -64.03097534179688,
"logps/ref_chosen": -64.03948211669922,
"logps/ref_rejected": -75.68357849121094,
"logps/rejected": -75.73980712890625,
"loss": 1.3535,
"margin_dpo/margin_mean": 0.0647314041852951,
"margin_dpo/margin_std": 0.33662211894989014,
"step": 33
},
{
"KL/chosen_KL_mean": -0.03393745422363281,
"KL/mean": -0.05475132167339325,
"KL/rejected_KL_mean": -0.07556533813476562,
"KL/std": 0.21867325901985168,
"epoch": 0.05139833711262283,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04163369536399841,
"fcm_dpo/q_t": 0.491929829120636,
"grad_norm": 226.5720672607422,
"learning_rate": 2.4626865671641786e-07,
"logits/chosen": 0.09200664609670639,
"logits/rejected": 0.062414735555648804,
"logps/chosen": -53.69823455810547,
"logps/ref_chosen": -53.6642951965332,
"logps/ref_rejected": -65.77989959716797,
"logps/rejected": -65.85546875,
"loss": 1.3702,
"margin_dpo/margin_mean": 0.04163375496864319,
"margin_dpo/margin_std": 0.3225608468055725,
"step": 34
},
{
"KL/chosen_KL_mean": -0.06984138488769531,
"KL/mean": -0.05096860229969025,
"KL/rejected_KL_mean": -0.03209686279296875,
"KL/std": 0.22077873349189758,
"epoch": 0.05291005291005291,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03774866461753845,
"fcm_dpo/q_t": 0.5073720216751099,
"grad_norm": 231.8986053466797,
"learning_rate": 2.537313432835821e-07,
"logits/chosen": 0.057314082980155945,
"logits/rejected": 0.03480309993028641,
"logps/chosen": -61.08670425415039,
"logps/ref_chosen": -61.01686096191406,
"logps/ref_rejected": -72.78598022460938,
"logps/rejected": -72.81808471679688,
"loss": 1.4331,
"margin_dpo/margin_mean": -0.037748783826828,
"margin_dpo/margin_std": 0.31947654485702515,
"step": 35
},
{
"KL/chosen_KL_mean": -0.08660125732421875,
"KL/mean": -0.09120562672615051,
"KL/rejected_KL_mean": -0.0958099365234375,
"KL/std": 0.2505699396133423,
"epoch": 0.05442176870748299,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.009209752082824707,
"fcm_dpo/q_t": 0.4983007311820984,
"grad_norm": 234.24310302734375,
"learning_rate": 2.611940298507462e-07,
"logits/chosen": 0.10258600115776062,
"logits/rejected": 0.049621693789958954,
"logps/chosen": -50.62396240234375,
"logps/ref_chosen": -50.53736114501953,
"logps/ref_rejected": -78.11678314208984,
"logps/rejected": -78.21259307861328,
"loss": 1.4034,
"margin_dpo/margin_mean": 0.009210050106048584,
"margin_dpo/margin_std": 0.39130350947380066,
"step": 36
},
{
"KL/chosen_KL_mean": -0.006591796875,
"KL/mean": -0.05672261118888855,
"KL/rejected_KL_mean": -0.10684967041015625,
"KL/std": 0.30893322825431824,
"epoch": 0.055933484504913075,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10025268793106079,
"fcm_dpo/q_t": 0.4804428219795227,
"grad_norm": 294.3005676269531,
"learning_rate": 2.686567164179104e-07,
"logits/chosen": 0.08570870757102966,
"logits/rejected": 0.008101830258965492,
"logps/chosen": -59.56053924560547,
"logps/ref_chosen": -59.55394744873047,
"logps/ref_rejected": -108.27702331542969,
"logps/rejected": -108.38388061523438,
"loss": 1.3395,
"margin_dpo/margin_mean": 0.10025274753570557,
"margin_dpo/margin_std": 0.44817155599594116,
"step": 37
},
{
"KL/chosen_KL_mean": -0.092193603515625,
"KL/mean": -0.0783776044845581,
"KL/rejected_KL_mean": -0.06456565856933594,
"KL/std": 0.25777286291122437,
"epoch": 0.05744520030234316,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.027630925178527832,
"fcm_dpo/q_t": 0.5057640075683594,
"grad_norm": 244.69921875,
"learning_rate": 2.761194029850746e-07,
"logits/chosen": 0.09132996201515198,
"logits/rejected": 0.07707769423723221,
"logps/chosen": -65.88055419921875,
"logps/ref_chosen": -65.78836059570312,
"logps/ref_rejected": -76.1619873046875,
"logps/rejected": -76.22655487060547,
"loss": 1.433,
"margin_dpo/margin_mean": -0.02763056755065918,
"margin_dpo/margin_std": 0.38167810440063477,
"step": 38
},
{
"KL/chosen_KL_mean": -0.08689498901367188,
"KL/mean": -0.12177233397960663,
"KL/rejected_KL_mean": -0.156646728515625,
"KL/std": 0.24981790781021118,
"epoch": 0.05895691609977324,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06974777579307556,
"fcm_dpo/q_t": 0.4862174987792969,
"grad_norm": 229.70477294921875,
"learning_rate": 2.8358208955223876e-07,
"logits/chosen": 0.1752331256866455,
"logits/rejected": 0.14800116419792175,
"logps/chosen": -57.263702392578125,
"logps/ref_chosen": -57.17681121826172,
"logps/ref_rejected": -79.486328125,
"logps/rejected": -79.64297485351562,
"loss": 1.3507,
"margin_dpo/margin_mean": 0.06974801421165466,
"margin_dpo/margin_std": 0.34825581312179565,
"step": 39
},
{
"KL/chosen_KL_mean": -0.08170700073242188,
"KL/mean": -0.12046042084693909,
"KL/rejected_KL_mean": -0.15921783447265625,
"KL/std": 0.2606281042098999,
"epoch": 0.06046863189720333,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07750892639160156,
"fcm_dpo/q_t": 0.4847288429737091,
"grad_norm": 251.29733276367188,
"learning_rate": 2.9104477611940296e-07,
"logits/chosen": 0.10897394269704819,
"logits/rejected": 0.05944906175136566,
"logps/chosen": -61.41587448120117,
"logps/ref_chosen": -61.33416748046875,
"logps/ref_rejected": -79.10697174072266,
"logps/rejected": -79.26618957519531,
"loss": 1.3448,
"margin_dpo/margin_mean": 0.07750925421714783,
"margin_dpo/margin_std": 0.3492031991481781,
"step": 40
},
{
"KL/chosen_KL_mean": -0.14647674560546875,
"KL/mean": -0.15313176810741425,
"KL/rejected_KL_mean": -0.15977859497070312,
"KL/std": 0.29649409651756287,
"epoch": 0.06198034769463341,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01329854130744934,
"fcm_dpo/q_t": 0.4973070025444031,
"grad_norm": 251.9221954345703,
"learning_rate": 2.985074626865671e-07,
"logits/chosen": 0.021239612251520157,
"logits/rejected": 0.0020996499806642532,
"logps/chosen": -67.69320678710938,
"logps/ref_chosen": -67.5467300415039,
"logps/ref_rejected": -83.87788391113281,
"logps/rejected": -84.03766632080078,
"loss": 1.4028,
"margin_dpo/margin_mean": 0.013299375772476196,
"margin_dpo/margin_std": 0.4114866852760315,
"step": 41
},
{
"KL/chosen_KL_mean": -0.09899139404296875,
"KL/mean": -0.12127295881509781,
"KL/rejected_KL_mean": -0.14355850219726562,
"KL/std": 0.26968640089035034,
"epoch": 0.06349206349206349,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04457102715969086,
"fcm_dpo/q_t": 0.49134361743927,
"grad_norm": 235.34556579589844,
"learning_rate": 3.059701492537313e-07,
"logits/chosen": 0.05914067476987839,
"logits/rejected": 0.03732679784297943,
"logps/chosen": -61.36384963989258,
"logps/ref_chosen": -61.26485824584961,
"logps/ref_rejected": -76.3629150390625,
"logps/rejected": -76.5064697265625,
"loss": 1.3742,
"margin_dpo/margin_mean": 0.044570907950401306,
"margin_dpo/margin_std": 0.37643399834632874,
"step": 42
},
{
"KL/chosen_KL_mean": -0.046878814697265625,
"KL/mean": -0.1046941876411438,
"KL/rejected_KL_mean": -0.1625041961669922,
"KL/std": 0.2962798476219177,
"epoch": 0.06500377928949358,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11562466621398926,
"fcm_dpo/q_t": 0.47788119316101074,
"grad_norm": 252.61216735839844,
"learning_rate": 3.134328358208955e-07,
"logits/chosen": 0.09768113493919373,
"logits/rejected": 0.08650224655866623,
"logps/chosen": -71.85591125488281,
"logps/ref_chosen": -71.80902862548828,
"logps/ref_rejected": -81.12464141845703,
"logps/rejected": -81.28714752197266,
"loss": 1.3246,
"margin_dpo/margin_mean": 0.11562475562095642,
"margin_dpo/margin_std": 0.41630876064300537,
"step": 43
},
{
"KL/chosen_KL_mean": -0.16477584838867188,
"KL/mean": -0.17331074178218842,
"KL/rejected_KL_mean": -0.18184661865234375,
"KL/std": 0.3096635341644287,
"epoch": 0.06651549508692366,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.017076164484024048,
"fcm_dpo/q_t": 0.4972341060638428,
"grad_norm": 265.5024719238281,
"learning_rate": 3.2089552238805965e-07,
"logits/chosen": 0.049512311816215515,
"logits/rejected": 0.018965082243084908,
"logps/chosen": -66.7152099609375,
"logps/ref_chosen": -66.55043029785156,
"logps/ref_rejected": -85.06198120117188,
"logps/rejected": -85.24382781982422,
"loss": 1.4008,
"margin_dpo/margin_mean": 0.017076104879379272,
"margin_dpo/margin_std": 0.42464640736579895,
"step": 44
},
{
"KL/chosen_KL_mean": -0.12146759033203125,
"KL/mean": -0.19493669271469116,
"KL/rejected_KL_mean": -0.2684059143066406,
"KL/std": 0.2962506115436554,
"epoch": 0.06802721088435375,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.146940216422081,
"fcm_dpo/q_t": 0.47128647565841675,
"grad_norm": 237.75613403320312,
"learning_rate": 3.2835820895522385e-07,
"logits/chosen": 0.12587401270866394,
"logits/rejected": 0.07270471006631851,
"logps/chosen": -62.36532211303711,
"logps/ref_chosen": -62.24385452270508,
"logps/ref_rejected": -92.96665954589844,
"logps/rejected": -93.23506927490234,
"loss": 1.2958,
"margin_dpo/margin_mean": 0.14694073796272278,
"margin_dpo/margin_std": 0.38624435663223267,
"step": 45
},
{
"KL/chosen_KL_mean": -0.09909439086914062,
"KL/mean": -0.18654456734657288,
"KL/rejected_KL_mean": -0.27399444580078125,
"KL/std": 0.3293907046318054,
"epoch": 0.06953892668178382,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17489582300186157,
"fcm_dpo/q_t": 0.46630242466926575,
"grad_norm": 226.0812530517578,
"learning_rate": 3.3582089552238805e-07,
"logits/chosen": 0.10262426733970642,
"logits/rejected": 0.05781745910644531,
"logps/chosen": -61.597999572753906,
"logps/ref_chosen": -61.498905181884766,
"logps/ref_rejected": -78.91172790527344,
"logps/rejected": -79.18572235107422,
"loss": 1.2817,
"margin_dpo/margin_mean": 0.174896240234375,
"margin_dpo/margin_std": 0.43405789136886597,
"step": 46
},
{
"KL/chosen_KL_mean": -0.14288330078125,
"KL/mean": -0.2334054708480835,
"KL/rejected_KL_mean": -0.3239326477050781,
"KL/std": 0.3144547939300537,
"epoch": 0.0710506424792139,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18104791641235352,
"fcm_dpo/q_t": 0.46470946073532104,
"grad_norm": 209.09397888183594,
"learning_rate": 3.432835820895522e-07,
"logits/chosen": 0.03155403211712837,
"logits/rejected": -0.011172996833920479,
"logps/chosen": -51.721229553222656,
"logps/ref_chosen": -51.578346252441406,
"logps/ref_rejected": -68.2215576171875,
"logps/rejected": -68.54548645019531,
"loss": 1.2728,
"margin_dpo/margin_mean": 0.18104803562164307,
"margin_dpo/margin_std": 0.4067476987838745,
"step": 47
},
{
"KL/chosen_KL_mean": -0.21188735961914062,
"KL/mean": -0.21695484220981598,
"KL/rejected_KL_mean": -0.22202301025390625,
"KL/std": 0.3169354796409607,
"epoch": 0.07256235827664399,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01013365387916565,
"fcm_dpo/q_t": 0.49802806973457336,
"grad_norm": 220.99514770507812,
"learning_rate": 3.507462686567164e-07,
"logits/chosen": 0.12512364983558655,
"logits/rejected": 0.09601491689682007,
"logps/chosen": -52.00553894042969,
"logps/ref_chosen": -51.79365158081055,
"logps/ref_rejected": -64.22503662109375,
"logps/rejected": -64.44705963134766,
"loss": 1.41,
"margin_dpo/margin_mean": 0.010134011507034302,
"margin_dpo/margin_std": 0.4460296332836151,
"step": 48
},
{
"KL/chosen_KL_mean": -0.1875743865966797,
"KL/mean": -0.2489079385995865,
"KL/rejected_KL_mean": -0.3102397918701172,
"KL/std": 0.32727736234664917,
"epoch": 0.07407407407407407,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12266728281974792,
"fcm_dpo/q_t": 0.47613510489463806,
"grad_norm": 208.04432678222656,
"learning_rate": 3.5820895522388055e-07,
"logits/chosen": 0.01999567821621895,
"logits/rejected": -0.0011389795690774918,
"logps/chosen": -58.322174072265625,
"logps/ref_chosen": -58.13460159301758,
"logps/ref_rejected": -64.63206481933594,
"logps/rejected": -64.94230651855469,
"loss": 1.3216,
"margin_dpo/margin_mean": 0.12266790866851807,
"margin_dpo/margin_std": 0.4320908486843109,
"step": 49
},
{
"KL/chosen_KL_mean": -0.23402976989746094,
"KL/mean": -0.3014345169067383,
"KL/rejected_KL_mean": -0.3688392639160156,
"KL/std": 0.3133654296398163,
"epoch": 0.07558578987150416,
"fcm_dpo/beta": 0.800000011920929,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13481035828590393,
"fcm_dpo/q_t": 0.47352075576782227,
"grad_norm": 210.95211791992188,
"learning_rate": 3.6567164179104475e-07,
"logits/chosen": 0.10641828924417496,
"logits/rejected": 0.07685194909572601,
"logps/chosen": -53.09046936035156,
"logps/ref_chosen": -52.85643768310547,
"logps/ref_rejected": -72.17460632324219,
"logps/rejected": -72.54344177246094,
"loss": 1.3002,
"margin_dpo/margin_mean": 0.1348104178905487,
"margin_dpo/margin_std": 0.33946073055267334,
"step": 50
},
{
"KL/chosen_KL_mean": -0.2200794219970703,
"KL/mean": -0.3465917110443115,
"KL/rejected_KL_mean": -0.47310638427734375,
"KL/std": 0.37242260575294495,
"epoch": 0.07709750566893424,
"fcm_dpo/beta": 0.814177393913269,
"fcm_dpo/delta": 0.08783261477947235,
"fcm_dpo/margin": 0.2530253231525421,
"fcm_dpo/q_t": 0.4525066614151001,
"grad_norm": 214.0352325439453,
"learning_rate": 3.7313432835820895e-07,
"logits/chosen": 0.08434007316827774,
"logits/rejected": 0.05659899860620499,
"logps/chosen": -63.876522064208984,
"logps/ref_chosen": -63.65644073486328,
"logps/ref_rejected": -86.13229370117188,
"logps/rejected": -86.60540771484375,
"loss": 1.2332,
"margin_dpo/margin_mean": 0.2530254125595093,
"margin_dpo/margin_std": 0.5112677812576294,
"step": 51
},
{
"KL/chosen_KL_mean": -0.2667884826660156,
"KL/mean": -0.4041314125061035,
"KL/rejected_KL_mean": -0.5414810180664062,
"KL/std": 0.41137245297431946,
"epoch": 0.07860922146636433,
"fcm_dpo/beta": 0.8209208250045776,
"fcm_dpo/delta": 0.08214651048183441,
"fcm_dpo/margin": 0.27468934655189514,
"fcm_dpo/q_t": 0.448085755109787,
"grad_norm": 234.45989990234375,
"learning_rate": 3.805970149253731e-07,
"logits/chosen": 0.06220635771751404,
"logits/rejected": 0.01463498454540968,
"logps/chosen": -68.10700225830078,
"logps/ref_chosen": -67.8402099609375,
"logps/ref_rejected": -96.97090911865234,
"logps/rejected": -97.51239013671875,
"loss": 1.2265,
"margin_dpo/margin_mean": 0.27468955516815186,
"margin_dpo/margin_std": 0.5717782974243164,
"step": 52
},
{
"KL/chosen_KL_mean": -0.3107757568359375,
"KL/mean": -0.3775358200073242,
"KL/rejected_KL_mean": -0.44429969787597656,
"KL/std": 0.35028141736984253,
"epoch": 0.0801209372637944,
"fcm_dpo/beta": 0.8276642560958862,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1335272192955017,
"fcm_dpo/q_t": 0.4731205701828003,
"grad_norm": 214.9593963623047,
"learning_rate": 3.880597014925373e-07,
"logits/chosen": 0.08362244814634323,
"logits/rejected": 0.07295048981904984,
"logps/chosen": -57.18891525268555,
"logps/ref_chosen": -56.87813949584961,
"logps/ref_rejected": -60.75569152832031,
"logps/rejected": -61.19999313354492,
"loss": 1.3097,
"margin_dpo/margin_mean": 0.13352787494659424,
"margin_dpo/margin_std": 0.42652446031570435,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3166675567626953,
"KL/mean": -0.39567673206329346,
"KL/rejected_KL_mean": -0.4746818542480469,
"KL/std": 0.34335705637931824,
"epoch": 0.08163265306122448,
"fcm_dpo/beta": 0.8276642560958862,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1580154001712799,
"fcm_dpo/q_t": 0.46843764185905457,
"grad_norm": 211.7559814453125,
"learning_rate": 3.9552238805970144e-07,
"logits/chosen": 0.03795847296714783,
"logits/rejected": 0.02295723930001259,
"logps/chosen": -47.583587646484375,
"logps/ref_chosen": -47.26692199707031,
"logps/ref_rejected": -62.19426727294922,
"logps/rejected": -62.6689453125,
"loss": 1.2909,
"margin_dpo/margin_mean": 0.1580154299736023,
"margin_dpo/margin_std": 0.42730119824409485,
"step": 54
},
{
"KL/chosen_KL_mean": -0.33272552490234375,
"KL/mean": -0.44846922159194946,
"KL/rejected_KL_mean": -0.5642166137695312,
"KL/std": 0.44844868779182434,
"epoch": 0.08314436885865457,
"fcm_dpo/beta": 0.8333209753036499,
"fcm_dpo/delta": 0.06788266450166702,
"fcm_dpo/margin": 0.23149140179157257,
"fcm_dpo/q_t": 0.4564506411552429,
"grad_norm": 244.24220275878906,
"learning_rate": 4.0298507462686564e-07,
"logits/chosen": 0.04083487018942833,
"logits/rejected": -0.03650583699345589,
"logps/chosen": -50.65891647338867,
"logps/ref_chosen": -50.32619094848633,
"logps/ref_rejected": -92.44389343261719,
"logps/rejected": -93.00810241699219,
"loss": 1.2643,
"margin_dpo/margin_mean": 0.2314915508031845,
"margin_dpo/margin_std": 0.6033967733383179,
"step": 55
},
{
"KL/chosen_KL_mean": -0.3264274597167969,
"KL/mean": -0.41880887746810913,
"KL/rejected_KL_mean": -0.5111846923828125,
"KL/std": 0.39167922735214233,
"epoch": 0.08465608465608465,
"fcm_dpo/beta": 0.8389776945114136,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18476131558418274,
"fcm_dpo/q_t": 0.4626288115978241,
"grad_norm": 222.12582397460938,
"learning_rate": 4.1044776119402984e-07,
"logits/chosen": 0.14603421092033386,
"logits/rejected": 0.12327264994382858,
"logps/chosen": -57.09339904785156,
"logps/ref_chosen": -56.766971588134766,
"logps/ref_rejected": -66.30504608154297,
"logps/rejected": -66.81623077392578,
"loss": 1.2795,
"margin_dpo/margin_mean": 0.18476131558418274,
"margin_dpo/margin_std": 0.4938068389892578,
"step": 56
},
{
"KL/chosen_KL_mean": -0.41528892517089844,
"KL/mean": -0.5447049140930176,
"KL/rejected_KL_mean": -0.6741218566894531,
"KL/std": 0.5022920370101929,
"epoch": 0.08616780045351474,
"fcm_dpo/beta": 0.8477333188056946,
"fcm_dpo/delta": 0.05190989002585411,
"fcm_dpo/margin": 0.2588346600532532,
"fcm_dpo/q_t": 0.44758230447769165,
"grad_norm": 220.42100524902344,
"learning_rate": 4.17910447761194e-07,
"logits/chosen": 0.11368558555841446,
"logits/rejected": 0.04829259589314461,
"logps/chosen": -58.18303298950195,
"logps/ref_chosen": -57.76774597167969,
"logps/ref_rejected": -82.75698852539062,
"logps/rejected": -83.43110656738281,
"loss": 1.2279,
"margin_dpo/margin_mean": 0.2588345408439636,
"margin_dpo/margin_std": 0.5217838287353516,
"step": 57
},
{
"KL/chosen_KL_mean": -0.4838447570800781,
"KL/mean": -0.5685634613037109,
"KL/rejected_KL_mean": -0.6532821655273438,
"KL/std": 0.5231010317802429,
"epoch": 0.08767951625094482,
"fcm_dpo/beta": 0.8556123375892639,
"fcm_dpo/delta": 0.04625631868839264,
"fcm_dpo/margin": 0.169439435005188,
"fcm_dpo/q_t": 0.4711190462112427,
"grad_norm": 258.2833251953125,
"learning_rate": 4.253731343283582e-07,
"logits/chosen": 0.06217523664236069,
"logits/rejected": 0.04635544866323471,
"logps/chosen": -73.24793243408203,
"logps/ref_chosen": -72.76408386230469,
"logps/ref_rejected": -84.49275207519531,
"logps/rejected": -85.14603424072266,
"loss": 1.3503,
"margin_dpo/margin_mean": 0.16943949460983276,
"margin_dpo/margin_std": 0.7654597759246826,
"step": 58
},
{
"KL/chosen_KL_mean": -0.4406089782714844,
"KL/mean": -0.548049807548523,
"KL/rejected_KL_mean": -0.6554946899414062,
"KL/std": 0.47114166617393494,
"epoch": 0.08919123204837491,
"fcm_dpo/beta": 0.8577494025230408,
"fcm_dpo/delta": 0.02491498738527298,
"fcm_dpo/margin": 0.21487921476364136,
"fcm_dpo/q_t": 0.459256112575531,
"grad_norm": 213.29129028320312,
"learning_rate": 4.3283582089552234e-07,
"logits/chosen": 0.113294318318367,
"logits/rejected": 0.048204269260168076,
"logps/chosen": -50.261383056640625,
"logps/ref_chosen": -49.820777893066406,
"logps/ref_rejected": -77.14368438720703,
"logps/rejected": -77.79917907714844,
"loss": 1.2703,
"margin_dpo/margin_mean": 0.21487951278686523,
"margin_dpo/margin_std": 0.5466220378875732,
"step": 59
},
{
"KL/chosen_KL_mean": -0.5373973846435547,
"KL/mean": -0.5357345342636108,
"KL/rejected_KL_mean": -0.5340766906738281,
"KL/std": 0.4177909195423126,
"epoch": 0.09070294784580499,
"fcm_dpo/beta": 0.8598864674568176,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0033222436904907227,
"fcm_dpo/q_t": 0.5001885890960693,
"grad_norm": 281.2565002441406,
"learning_rate": 4.4029850746268654e-07,
"logits/chosen": 0.10876858979463577,
"logits/rejected": 0.10734610259532928,
"logps/chosen": -63.762168884277344,
"logps/ref_chosen": -63.22477340698242,
"logps/ref_rejected": -61.360477447509766,
"logps/rejected": -61.894554138183594,
"loss": 1.4416,
"margin_dpo/margin_mean": -0.0033222734928131104,
"margin_dpo/margin_std": 0.5349164009094238,
"step": 60
},
{
"KL/chosen_KL_mean": -0.5890903472900391,
"KL/mean": -0.6313471794128418,
"KL/rejected_KL_mean": -0.6736068725585938,
"KL/std": 0.5355270504951477,
"epoch": 0.09221466364323508,
"fcm_dpo/beta": 0.8598864674568176,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08451084792613983,
"fcm_dpo/q_t": 0.48391294479370117,
"grad_norm": 264.2339172363281,
"learning_rate": 4.4776119402985074e-07,
"logits/chosen": 0.15465795993804932,
"logits/rejected": 0.12083549797534943,
"logps/chosen": -49.60588836669922,
"logps/ref_chosen": -49.01679992675781,
"logps/ref_rejected": -74.90817260742188,
"logps/rejected": -75.58177185058594,
"loss": 1.4171,
"margin_dpo/margin_mean": 0.08451053500175476,
"margin_dpo/margin_std": 0.7538120746612549,
"step": 61
},
{
"KL/chosen_KL_mean": -0.6008815765380859,
"KL/mean": -0.6999142169952393,
"KL/rejected_KL_mean": -0.7989463806152344,
"KL/std": 0.4923873543739319,
"epoch": 0.09372637944066516,
"fcm_dpo/beta": 0.8598864674568176,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19806843996047974,
"fcm_dpo/q_t": 0.45975828170776367,
"grad_norm": 251.43060302734375,
"learning_rate": 4.552238805970149e-07,
"logits/chosen": 0.10388742387294769,
"logits/rejected": 0.0647771954536438,
"logps/chosen": -63.352752685546875,
"logps/ref_chosen": -62.751869201660156,
"logps/ref_rejected": -78.93360900878906,
"logps/rejected": -79.73255920410156,
"loss": 1.3076,
"margin_dpo/margin_mean": 0.19806808233261108,
"margin_dpo/margin_std": 0.6950019001960754,
"step": 62
},
{
"KL/chosen_KL_mean": -0.43421363830566406,
"KL/mean": -0.6513885855674744,
"KL/rejected_KL_mean": -0.8685646057128906,
"KL/std": 0.46996253728866577,
"epoch": 0.09523809523809523,
"fcm_dpo/beta": 0.8645204305648804,
"fcm_dpo/delta": 0.025358233600854874,
"fcm_dpo/margin": 0.43435075879096985,
"fcm_dpo/q_t": 0.41197603940963745,
"grad_norm": 213.9260711669922,
"learning_rate": 4.626865671641791e-07,
"logits/chosen": 0.155286505818367,
"logits/rejected": 0.13132315874099731,
"logps/chosen": -60.949462890625,
"logps/ref_chosen": -60.51525115966797,
"logps/ref_rejected": -85.11021423339844,
"logps/rejected": -85.97877502441406,
"loss": 1.0906,
"margin_dpo/margin_mean": 0.43434983491897583,
"margin_dpo/margin_std": 0.5073477029800415,
"step": 63
},
{
"KL/chosen_KL_mean": -0.6374263763427734,
"KL/mean": -0.6766533255577087,
"KL/rejected_KL_mean": -0.7158851623535156,
"KL/std": 0.5493475198745728,
"epoch": 0.09674981103552532,
"fcm_dpo/beta": 0.8642585873603821,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0784534215927124,
"fcm_dpo/q_t": 0.48456645011901855,
"grad_norm": 245.12164306640625,
"learning_rate": 4.701492537313433e-07,
"logits/chosen": 0.08370202779769897,
"logits/rejected": 0.058646999299526215,
"logps/chosen": -51.84427261352539,
"logps/ref_chosen": -51.20684814453125,
"logps/ref_rejected": -66.93081665039062,
"logps/rejected": -67.6467056274414,
"loss": 1.3934,
"margin_dpo/margin_mean": 0.07845339179039001,
"margin_dpo/margin_std": 0.647331714630127,
"step": 64
},
{
"KL/chosen_KL_mean": -0.5670604705810547,
"KL/mean": -0.8318616151809692,
"KL/rejected_KL_mean": -1.0966682434082031,
"KL/std": 0.6215205192565918,
"epoch": 0.0982615268329554,
"fcm_dpo/beta": 0.857285737991333,
"fcm_dpo/delta": -0.05657501518726349,
"fcm_dpo/margin": 0.5296034216880798,
"fcm_dpo/q_t": 0.3967716693878174,
"grad_norm": 224.72190856933594,
"learning_rate": 4.776119402985074e-07,
"logits/chosen": 0.20663005113601685,
"logits/rejected": 0.17640256881713867,
"logps/chosen": -67.85575866699219,
"logps/ref_chosen": -67.2886962890625,
"logps/ref_rejected": -74.44281005859375,
"logps/rejected": -75.53947448730469,
"loss": 1.0806,
"margin_dpo/margin_mean": 0.5296029448509216,
"margin_dpo/margin_std": 0.7609937191009521,
"step": 65
},
{
"KL/chosen_KL_mean": -0.6433124542236328,
"KL/mean": -0.7771072387695312,
"KL/rejected_KL_mean": -0.9109039306640625,
"KL/std": 0.5370617508888245,
"epoch": 0.09977324263038549,
"fcm_dpo/beta": 0.8587494492530823,
"fcm_dpo/delta": 0.02460136078298092,
"fcm_dpo/margin": 0.26759013533592224,
"fcm_dpo/q_t": 0.4492912292480469,
"grad_norm": 237.6848602294922,
"learning_rate": 4.850746268656717e-07,
"logits/chosen": 0.08247023820877075,
"logits/rejected": 0.05861452966928482,
"logps/chosen": -71.38672637939453,
"logps/ref_chosen": -70.743408203125,
"logps/ref_rejected": -77.26499938964844,
"logps/rejected": -78.1759033203125,
"loss": 1.2513,
"margin_dpo/margin_mean": 0.26758939027786255,
"margin_dpo/margin_std": 0.6794909238815308,
"step": 66
},
{
"KL/chosen_KL_mean": -0.5358200073242188,
"KL/mean": -0.6512033939361572,
"KL/rejected_KL_mean": -0.7665863037109375,
"KL/std": 0.5264816880226135,
"epoch": 0.10128495842781557,
"fcm_dpo/beta": 0.8738381862640381,
"fcm_dpo/delta": 0.0870901569724083,
"fcm_dpo/margin": 0.23076286911964417,
"fcm_dpo/q_t": 0.45437803864479065,
"grad_norm": 235.6294403076172,
"learning_rate": 4.925373134328357e-07,
"logits/chosen": 0.08094270527362823,
"logits/rejected": 0.024854552000761032,
"logps/chosen": -61.13842010498047,
"logps/ref_chosen": -60.60260009765625,
"logps/ref_rejected": -75.22235870361328,
"logps/rejected": -75.98894500732422,
"loss": 1.2719,
"margin_dpo/margin_mean": 0.23076248168945312,
"margin_dpo/margin_std": 0.6542218923568726,
"step": 67
},
{
"KL/chosen_KL_mean": -0.7412834167480469,
"KL/mean": -0.9074845910072327,
"KL/rejected_KL_mean": -1.0736885070800781,
"KL/std": 0.5885103940963745,
"epoch": 0.10279667422524566,
"fcm_dpo/beta": 0.8849332928657532,
"fcm_dpo/delta": 0.10917352139949799,
"fcm_dpo/margin": 0.33240845799446106,
"fcm_dpo/q_t": 0.4343593120574951,
"grad_norm": 251.3780059814453,
"learning_rate": 5e-07,
"logits/chosen": 0.03282208740711212,
"logits/rejected": 0.003747999668121338,
"logps/chosen": -78.2696533203125,
"logps/ref_chosen": -77.52836608886719,
"logps/ref_rejected": -93.17778015136719,
"logps/rejected": -94.25146484375,
"loss": 1.2013,
"margin_dpo/margin_mean": 0.33240818977355957,
"margin_dpo/margin_std": 0.6967720985412598,
"step": 68
},
{
"KL/chosen_KL_mean": -0.6654434204101562,
"KL/mean": -0.8706564903259277,
"KL/rejected_KL_mean": -1.0758705139160156,
"KL/std": 0.5950401425361633,
"epoch": 0.10430839002267574,
"fcm_dpo/beta": 0.8879094123840332,
"fcm_dpo/delta": -0.05877486243844032,
"fcm_dpo/margin": 0.41042694449424744,
"fcm_dpo/q_t": 0.41809147596359253,
"grad_norm": 220.00698852539062,
"learning_rate": 4.999965034812934e-07,
"logits/chosen": 0.10515225678682327,
"logits/rejected": 0.06099225580692291,
"logps/chosen": -66.6084976196289,
"logps/ref_chosen": -65.94305419921875,
"logps/ref_rejected": -89.7735595703125,
"logps/rejected": -90.84942626953125,
"loss": 1.1336,
"margin_dpo/margin_mean": 0.41042596101760864,
"margin_dpo/margin_std": 0.6377642154693604,
"step": 69
},
{
"KL/chosen_KL_mean": -0.7461910247802734,
"KL/mean": -0.8999744653701782,
"KL/rejected_KL_mean": -1.0537586212158203,
"KL/std": 0.552111804485321,
"epoch": 0.10582010582010581,
"fcm_dpo/beta": 0.8870489597320557,
"fcm_dpo/delta": 0.024626009166240692,
"fcm_dpo/margin": 0.30756843090057373,
"fcm_dpo/q_t": 0.4361230731010437,
"grad_norm": 236.58360290527344,
"learning_rate": 4.999860140229787e-07,
"logits/chosen": 0.09749700129032135,
"logits/rejected": 0.07497746497392654,
"logps/chosen": -62.7041015625,
"logps/ref_chosen": -61.95791244506836,
"logps/ref_rejected": -75.80945587158203,
"logps/rejected": -76.86322021484375,
"loss": 1.2145,
"margin_dpo/margin_mean": 0.3075684607028961,
"margin_dpo/margin_std": 0.6525850296020508,
"step": 70
},
{
"KL/chosen_KL_mean": -0.9178848266601562,
"KL/mean": -0.9946730136871338,
"KL/rejected_KL_mean": -1.0714645385742188,
"KL/std": 0.5646952390670776,
"epoch": 0.1073318216175359,
"fcm_dpo/beta": 0.8870489597320557,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15357764065265656,
"fcm_dpo/q_t": 0.4659923315048218,
"grad_norm": 256.8360595703125,
"learning_rate": 4.999685319184688e-07,
"logits/chosen": 0.07698483020067215,
"logits/rejected": 0.06158116087317467,
"logps/chosen": -64.26546478271484,
"logps/ref_chosen": -63.34757995605469,
"logps/ref_rejected": -67.49658203125,
"logps/rejected": -68.56804656982422,
"loss": 1.3461,
"margin_dpo/margin_mean": 0.15357764065265656,
"margin_dpo/margin_std": 0.698381781578064,
"step": 71
},
{
"KL/chosen_KL_mean": -0.760345458984375,
"KL/mean": -0.9752969145774841,
"KL/rejected_KL_mean": -1.19024658203125,
"KL/std": 0.6159436702728271,
"epoch": 0.10884353741496598,
"fcm_dpo/beta": 0.8868120908737183,
"fcm_dpo/delta": 0.01935591921210289,
"fcm_dpo/margin": 0.4299049377441406,
"fcm_dpo/q_t": 0.4145790934562683,
"grad_norm": 231.55209350585938,
"learning_rate": 4.999440576567755e-07,
"logits/chosen": 0.11963581293821335,
"logits/rejected": 0.05480026826262474,
"logps/chosen": -56.61964416503906,
"logps/ref_chosen": -55.85929870605469,
"logps/ref_rejected": -68.45423889160156,
"logps/rejected": -69.64448547363281,
"loss": 1.1451,
"margin_dpo/margin_mean": 0.4299052357673645,
"margin_dpo/margin_std": 0.7550399899482727,
"step": 72
},
{
"KL/chosen_KL_mean": -1.1089591979980469,
"KL/mean": -1.1802775859832764,
"KL/rejected_KL_mean": -1.2515926361083984,
"KL/std": 0.672644853591919,
"epoch": 0.11035525321239607,
"fcm_dpo/beta": 0.8904895186424255,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1426388919353485,
"fcm_dpo/q_t": 0.4737260341644287,
"grad_norm": 279.4287414550781,
"learning_rate": 4.999125919224965e-07,
"logits/chosen": 0.06624437868595123,
"logits/rejected": 0.05240562930703163,
"logps/chosen": -70.24776458740234,
"logps/ref_chosen": -69.13880920410156,
"logps/ref_rejected": -79.04586791992188,
"logps/rejected": -80.2974624633789,
"loss": 1.3894,
"margin_dpo/margin_mean": 0.14263877272605896,
"margin_dpo/margin_std": 0.8156429529190063,
"step": 73
},
{
"KL/chosen_KL_mean": -0.8509178161621094,
"KL/mean": -1.0642307996749878,
"KL/rejected_KL_mean": -1.2775421142578125,
"KL/std": 0.5651123523712158,
"epoch": 0.11186696900982615,
"fcm_dpo/beta": 0.8989685773849487,
"fcm_dpo/delta": 0.01599665731191635,
"fcm_dpo/margin": 0.4266296327114105,
"fcm_dpo/q_t": 0.4143070578575134,
"grad_norm": 210.08905029296875,
"learning_rate": 4.998741355957963e-07,
"logits/chosen": 0.10601222515106201,
"logits/rejected": 0.054012730717659,
"logps/chosen": -50.774654388427734,
"logps/ref_chosen": -49.923736572265625,
"logps/ref_rejected": -81.73213958740234,
"logps/rejected": -83.00968170166016,
"loss": 1.131,
"margin_dpo/margin_mean": 0.42662960290908813,
"margin_dpo/margin_std": 0.6828247308731079,
"step": 74
},
{
"KL/chosen_KL_mean": -0.9244384765625,
"KL/mean": -1.164058804512024,
"KL/rejected_KL_mean": -1.403676986694336,
"KL/std": 0.6995598077774048,
"epoch": 0.11337868480725624,
"fcm_dpo/beta": 0.8896996974945068,
"fcm_dpo/delta": -0.027554970234632492,
"fcm_dpo/margin": 0.47924092411994934,
"fcm_dpo/q_t": 0.4055970311164856,
"grad_norm": 188.25650024414062,
"learning_rate": 4.998286897523808e-07,
"logits/chosen": 0.09290479868650436,
"logits/rejected": 0.060672298073768616,
"logps/chosen": -46.99319076538086,
"logps/ref_chosen": -46.06875228881836,
"logps/ref_rejected": -66.1181411743164,
"logps/rejected": -67.52182006835938,
"loss": 1.119,
"margin_dpo/margin_mean": 0.4792408347129822,
"margin_dpo/margin_std": 0.7964383363723755,
"step": 75
},
{
"KL/chosen_KL_mean": -0.9558849334716797,
"KL/mean": -1.0949684381484985,
"KL/rejected_KL_mean": -1.2340545654296875,
"KL/std": 0.7243768572807312,
"epoch": 0.11489040060468632,
"fcm_dpo/beta": 0.898980438709259,
"fcm_dpo/delta": 0.05900757759809494,
"fcm_dpo/margin": 0.2781708836555481,
"fcm_dpo/q_t": 0.4475979804992676,
"grad_norm": 247.96389770507812,
"learning_rate": 4.997762556634679e-07,
"logits/chosen": 0.08283071964979172,
"logits/rejected": 0.04031769931316376,
"logps/chosen": -55.01863479614258,
"logps/ref_chosen": -54.06275177001953,
"logps/ref_rejected": -74.87464141845703,
"logps/rejected": -76.10869598388672,
"loss": 1.2726,
"margin_dpo/margin_mean": 0.27817073464393616,
"margin_dpo/margin_std": 0.8085579872131348,
"step": 76
},
{
"KL/chosen_KL_mean": -1.1360054016113281,
"KL/mean": -1.3067750930786133,
"KL/rejected_KL_mean": -1.4775390625,
"KL/std": 0.694945216178894,
"epoch": 0.1164021164021164,
"fcm_dpo/beta": 0.8866174817085266,
"fcm_dpo/delta": -0.06923830509185791,
"fcm_dpo/margin": 0.3415396511554718,
"fcm_dpo/q_t": 0.43092960119247437,
"grad_norm": 241.7581787109375,
"learning_rate": 4.99716834795752e-07,
"logits/chosen": 0.13189122080802917,
"logits/rejected": 0.09041719138622284,
"logps/chosen": -54.21209716796875,
"logps/ref_chosen": -53.07609176635742,
"logps/ref_rejected": -74.45601654052734,
"logps/rejected": -75.93355560302734,
"loss": 1.2132,
"margin_dpo/margin_mean": 0.3415394425392151,
"margin_dpo/margin_std": 0.7136242389678955,
"step": 77
},
{
"KL/chosen_KL_mean": -1.0402603149414062,
"KL/mean": -1.223612904548645,
"KL/rejected_KL_mean": -1.4069671630859375,
"KL/std": 0.6672722697257996,
"epoch": 0.11791383219954649,
"fcm_dpo/beta": 0.8964298963546753,
"fcm_dpo/delta": 0.07370098680257797,
"fcm_dpo/margin": 0.3667004704475403,
"fcm_dpo/q_t": 0.4249332547187805,
"grad_norm": 251.27975463867188,
"learning_rate": 4.996504288113623e-07,
"logits/chosen": 0.07498917728662491,
"logits/rejected": 0.05498968064785004,
"logps/chosen": -68.76568603515625,
"logps/ref_chosen": -67.72541809082031,
"logps/ref_rejected": -79.03926849365234,
"logps/rejected": -80.44623565673828,
"loss": 1.1975,
"margin_dpo/margin_mean": 0.3667002320289612,
"margin_dpo/margin_std": 0.777641773223877,
"step": 78
},
{
"KL/chosen_KL_mean": -1.0738334655761719,
"KL/mean": -1.3531278371810913,
"KL/rejected_KL_mean": -1.6324234008789062,
"KL/std": 0.7191259860992432,
"epoch": 0.11942554799697656,
"fcm_dpo/beta": 0.8858178853988647,
"fcm_dpo/delta": -0.09984079003334045,
"fcm_dpo/margin": 0.5585932731628418,
"fcm_dpo/q_t": 0.3917655944824219,
"grad_norm": 199.2655792236328,
"learning_rate": 4.995770395678171e-07,
"logits/chosen": 0.12931254506111145,
"logits/rejected": 0.0703156366944313,
"logps/chosen": -53.23447799682617,
"logps/ref_chosen": -52.16064453125,
"logps/ref_rejected": -83.31062316894531,
"logps/rejected": -84.94303894042969,
"loss": 1.0631,
"margin_dpo/margin_mean": 0.5585935115814209,
"margin_dpo/margin_std": 0.8100461959838867,
"step": 79
},
{
"KL/chosen_KL_mean": -1.1816082000732422,
"KL/mean": -1.3618828058242798,
"KL/rejected_KL_mean": -1.5421600341796875,
"KL/std": 0.7551975250244141,
"epoch": 0.12093726379440665,
"fcm_dpo/beta": 0.880176305770874,
"fcm_dpo/delta": -0.020655568689107895,
"fcm_dpo/margin": 0.36055511236190796,
"fcm_dpo/q_t": 0.43023842573165894,
"grad_norm": 259.9960021972656,
"learning_rate": 4.994966691179711e-07,
"logits/chosen": 0.1080971509218216,
"logits/rejected": 0.04923234507441521,
"logps/chosen": -62.59217071533203,
"logps/ref_chosen": -61.410560607910156,
"logps/ref_rejected": -78.66004943847656,
"logps/rejected": -80.20220947265625,
"loss": 1.232,
"margin_dpo/margin_mean": 0.3605545461177826,
"margin_dpo/margin_std": 0.8744406700134277,
"step": 80
},
{
"KL/chosen_KL_mean": -1.1950340270996094,
"KL/mean": -1.4731804132461548,
"KL/rejected_KL_mean": -1.7513275146484375,
"KL/std": 0.7835187911987305,
"epoch": 0.12244897959183673,
"fcm_dpo/beta": 0.8621048331260681,
"fcm_dpo/delta": -0.08486048132181168,
"fcm_dpo/margin": 0.5562969446182251,
"fcm_dpo/q_t": 0.3954606056213379,
"grad_norm": 205.29806518554688,
"learning_rate": 4.994093197099587e-07,
"logits/chosen": 0.08074239641427994,
"logits/rejected": 0.047511570155620575,
"logps/chosen": -64.99940490722656,
"logps/ref_chosen": -63.80437088012695,
"logps/ref_rejected": -79.3484115600586,
"logps/rejected": -81.09973907470703,
"loss": 1.0667,
"margin_dpo/margin_mean": 0.5562969446182251,
"margin_dpo/margin_std": 0.7805662155151367,
"step": 81
},
{
"KL/chosen_KL_mean": -1.0413265228271484,
"KL/mean": -1.3763136863708496,
"KL/rejected_KL_mean": -1.7113037109375,
"KL/std": 0.76537024974823,
"epoch": 0.12396069538926682,
"fcm_dpo/beta": 0.8392397165298462,
"fcm_dpo/delta": -0.17262759804725647,
"fcm_dpo/margin": 0.6699746251106262,
"fcm_dpo/q_t": 0.3704856038093567,
"grad_norm": 178.28672790527344,
"learning_rate": 4.993149937871306e-07,
"logits/chosen": 0.0748857855796814,
"logits/rejected": 0.012260101735591888,
"logps/chosen": -49.85921859741211,
"logps/ref_chosen": -48.817893981933594,
"logps/ref_rejected": -70.31497955322266,
"logps/rejected": -72.02628326416016,
"loss": 0.9836,
"margin_dpo/margin_mean": 0.6699748039245605,
"margin_dpo/margin_std": 0.7135200500488281,
"step": 82
},
{
"KL/chosen_KL_mean": -1.1975154876708984,
"KL/mean": -1.4916658401489258,
"KL/rejected_KL_mean": -1.7858200073242188,
"KL/std": 0.7630441784858704,
"epoch": 0.1254724111866969,
"fcm_dpo/beta": 0.8230397701263428,
"fcm_dpo/delta": -0.0885235071182251,
"fcm_dpo/margin": 0.5883083343505859,
"fcm_dpo/q_t": 0.3947487771511078,
"grad_norm": 197.41090393066406,
"learning_rate": 4.992136939879856e-07,
"logits/chosen": 0.14176270365715027,
"logits/rejected": 0.0925317257642746,
"logps/chosen": -58.348289489746094,
"logps/ref_chosen": -57.15077209472656,
"logps/ref_rejected": -75.1710205078125,
"logps/rejected": -76.95684814453125,
"loss": 1.087,
"margin_dpo/margin_mean": 0.5883078575134277,
"margin_dpo/margin_std": 0.9470099210739136,
"step": 83
},
{
"KL/chosen_KL_mean": -1.3953399658203125,
"KL/mean": -1.5767252445220947,
"KL/rejected_KL_mean": -1.75811767578125,
"KL/std": 0.8215476274490356,
"epoch": 0.12698412698412698,
"fcm_dpo/beta": 0.8298979997634888,
"fcm_dpo/delta": 0.10212840139865875,
"fcm_dpo/margin": 0.36277827620506287,
"fcm_dpo/q_t": 0.43051877617836,
"grad_norm": 255.95265197753906,
"learning_rate": 4.991054231460969e-07,
"logits/chosen": 0.13572925329208374,
"logits/rejected": 0.09350337088108063,
"logps/chosen": -66.17263793945312,
"logps/ref_chosen": -64.77729797363281,
"logps/ref_rejected": -84.71949768066406,
"logps/rejected": -86.47761535644531,
"loss": 1.2161,
"margin_dpo/margin_mean": 0.36277878284454346,
"margin_dpo/margin_std": 0.8071293830871582,
"step": 84
},
{
"KL/chosen_KL_mean": -1.1967315673828125,
"KL/mean": -1.5478744506835938,
"KL/rejected_KL_mean": -1.899017333984375,
"KL/std": 0.8741401433944702,
"epoch": 0.12849584278155707,
"fcm_dpo/beta": 0.8177739381790161,
"fcm_dpo/delta": -0.1845196932554245,
"fcm_dpo/margin": 0.702286958694458,
"fcm_dpo/q_t": 0.37274277210235596,
"grad_norm": 199.63287353515625,
"learning_rate": 4.989901842900325e-07,
"logits/chosen": 0.11141739785671234,
"logits/rejected": 0.06853729486465454,
"logps/chosen": -51.44842529296875,
"logps/ref_chosen": -50.25169372558594,
"logps/ref_rejected": -66.55439758300781,
"logps/rejected": -68.45341491699219,
"loss": 1.0332,
"margin_dpo/margin_mean": 0.7022866010665894,
"margin_dpo/margin_std": 0.9415004849433899,
"step": 85
},
{
"KL/chosen_KL_mean": -1.3920631408691406,
"KL/mean": -1.6537511348724365,
"KL/rejected_KL_mean": -1.9154396057128906,
"KL/std": 0.8503645658493042,
"epoch": 0.13000755857898716,
"fcm_dpo/beta": 0.8028476238250732,
"fcm_dpo/delta": -0.021123308688402176,
"fcm_dpo/margin": 0.5233771800994873,
"fcm_dpo/q_t": 0.40762412548065186,
"grad_norm": 181.38487243652344,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 0.11902812123298645,
"logits/rejected": 0.10133795440196991,
"logps/chosen": -62.12124252319336,
"logps/ref_chosen": -60.72917938232422,
"logps/ref_rejected": -72.30961608886719,
"logps/rejected": -74.22505187988281,
"loss": 1.1234,
"margin_dpo/margin_mean": 0.5233776569366455,
"margin_dpo/margin_std": 0.8783669471740723,
"step": 86
},
{
"KL/chosen_KL_mean": -1.4519119262695312,
"KL/mean": -1.7144184112548828,
"KL/rejected_KL_mean": -1.9769172668457031,
"KL/std": 0.960472583770752,
"epoch": 0.13151927437641722,
"fcm_dpo/beta": 0.7950679063796997,
"fcm_dpo/delta": -0.01926865056157112,
"fcm_dpo/margin": 0.5250035524368286,
"fcm_dpo/q_t": 0.4067332148551941,
"grad_norm": 242.1222381591797,
"learning_rate": 4.987388156241114e-07,
"logits/chosen": 0.12725430727005005,
"logits/rejected": 0.06851398944854736,
"logps/chosen": -67.20988464355469,
"logps/ref_chosen": -65.75796508789062,
"logps/ref_rejected": -84.81159973144531,
"logps/rejected": -86.78851318359375,
"loss": 1.2014,
"margin_dpo/margin_mean": 0.5250037908554077,
"margin_dpo/margin_std": 1.1465673446655273,
"step": 87
},
{
"KL/chosen_KL_mean": -1.3908004760742188,
"KL/mean": -1.6730873584747314,
"KL/rejected_KL_mean": -1.9553718566894531,
"KL/std": 0.9087913036346436,
"epoch": 0.1330309901738473,
"fcm_dpo/beta": 0.7887861728668213,
"fcm_dpo/delta": -0.048637814819812775,
"fcm_dpo/margin": 0.5645675659179688,
"fcm_dpo/q_t": 0.40452295541763306,
"grad_norm": 207.7094268798828,
"learning_rate": 4.986026928455767e-07,
"logits/chosen": 0.19777879118919373,
"logits/rejected": 0.17085707187652588,
"logps/chosen": -64.21482849121094,
"logps/ref_chosen": -62.82402801513672,
"logps/ref_rejected": -74.9607162475586,
"logps/rejected": -76.91609191894531,
"loss": 1.1494,
"margin_dpo/margin_mean": 0.564567506313324,
"margin_dpo/margin_std": 1.0587239265441895,
"step": 88
},
{
"KL/chosen_KL_mean": -1.2803001403808594,
"KL/mean": -1.5476927757263184,
"KL/rejected_KL_mean": -1.8150901794433594,
"KL/std": 0.9286909103393555,
"epoch": 0.1345427059712774,
"fcm_dpo/beta": 0.7942764759063721,
"fcm_dpo/delta": -0.02662864699959755,
"fcm_dpo/margin": 0.5347846746444702,
"fcm_dpo/q_t": 0.40659964084625244,
"grad_norm": 206.88941955566406,
"learning_rate": 4.984596161153135e-07,
"logits/chosen": 0.1913776993751526,
"logits/rejected": 0.11048424988985062,
"logps/chosen": -42.47173309326172,
"logps/ref_chosen": -41.191436767578125,
"logps/ref_rejected": -85.44769287109375,
"logps/rejected": -87.26278686523438,
"loss": 1.1613,
"margin_dpo/margin_mean": 0.5347847938537598,
"margin_dpo/margin_std": 1.032776117324829,
"step": 89
},
{
"KL/chosen_KL_mean": -1.4412975311279297,
"KL/mean": -1.6775047779083252,
"KL/rejected_KL_mean": -1.9137153625488281,
"KL/std": 0.9316179752349854,
"epoch": 0.1360544217687075,
"fcm_dpo/beta": 0.7926943898200989,
"fcm_dpo/delta": 0.026345502585172653,
"fcm_dpo/margin": 0.47241735458374023,
"fcm_dpo/q_t": 0.419416606426239,
"grad_norm": 216.96438598632812,
"learning_rate": 4.983095894354857e-07,
"logits/chosen": 0.10932404547929764,
"logits/rejected": 0.054373688995838165,
"logps/chosen": -58.02520751953125,
"logps/ref_chosen": -56.58390808105469,
"logps/ref_rejected": -86.86978149414062,
"logps/rejected": -88.78349304199219,
"loss": 1.2128,
"margin_dpo/margin_mean": 0.47241726517677307,
"margin_dpo/margin_std": 1.1056712865829468,
"step": 90
},
{
"KL/chosen_KL_mean": -1.4198989868164062,
"KL/mean": -1.7759813070297241,
"KL/rejected_KL_mean": -2.132061004638672,
"KL/std": 1.0024120807647705,
"epoch": 0.13756613756613756,
"fcm_dpo/beta": 0.7739899158477783,
"fcm_dpo/delta": -0.15991877019405365,
"fcm_dpo/margin": 0.7121652364730835,
"fcm_dpo/q_t": 0.3804309070110321,
"grad_norm": 168.0975341796875,
"learning_rate": 4.98152617002662e-07,
"logits/chosen": 0.09888456016778946,
"logits/rejected": 0.05619416385889053,
"logps/chosen": -53.802242279052734,
"logps/ref_chosen": -52.38234329223633,
"logps/ref_rejected": -72.17642211914062,
"logps/rejected": -74.30848693847656,
"loss": 1.0561,
"margin_dpo/margin_mean": 0.712165355682373,
"margin_dpo/margin_std": 1.0622575283050537,
"step": 91
},
{
"KL/chosen_KL_mean": -1.5504646301269531,
"KL/mean": -1.844726324081421,
"KL/rejected_KL_mean": -2.138988494873047,
"KL/std": 1.0119301080703735,
"epoch": 0.13907785336356765,
"fcm_dpo/beta": 0.7568857669830322,
"fcm_dpo/delta": -0.1340516060590744,
"fcm_dpo/margin": 0.5885196328163147,
"fcm_dpo/q_t": 0.40348243713378906,
"grad_norm": 174.2653045654297,
"learning_rate": 4.979887032076988e-07,
"logits/chosen": 0.16256186366081238,
"logits/rejected": 0.12284956872463226,
"logps/chosen": -54.559165954589844,
"logps/ref_chosen": -53.00870132446289,
"logps/ref_rejected": -79.77812957763672,
"logps/rejected": -81.9171142578125,
"loss": 1.1573,
"margin_dpo/margin_mean": 0.5885197520256042,
"margin_dpo/margin_std": 1.0937684774398804,
"step": 92
},
{
"KL/chosen_KL_mean": -1.6001701354980469,
"KL/mean": -1.876657485961914,
"KL/rejected_KL_mean": -2.1531448364257812,
"KL/std": 1.0181267261505127,
"epoch": 0.14058956916099774,
"fcm_dpo/beta": 0.745780348777771,
"fcm_dpo/delta": -0.012931982055306435,
"fcm_dpo/margin": 0.5529758930206299,
"fcm_dpo/q_t": 0.41189247369766235,
"grad_norm": 161.0563201904297,
"learning_rate": 4.978178526356172e-07,
"logits/chosen": 0.13192062079906464,
"logits/rejected": 0.10375410318374634,
"logps/chosen": -46.507225036621094,
"logps/ref_chosen": -44.90705108642578,
"logps/ref_rejected": -58.7879524230957,
"logps/rejected": -60.941097259521484,
"loss": 1.164,
"margin_dpo/margin_mean": 0.5529758930206299,
"margin_dpo/margin_std": 1.098515510559082,
"step": 93
},
{
"KL/chosen_KL_mean": -1.3311805725097656,
"KL/mean": -1.6613003015518188,
"KL/rejected_KL_mean": -1.9914207458496094,
"KL/std": 1.0487146377563477,
"epoch": 0.1421012849584278,
"fcm_dpo/beta": 0.7265796661376953,
"fcm_dpo/delta": -0.08760561794042587,
"fcm_dpo/margin": 0.6602369546890259,
"fcm_dpo/q_t": 0.3942943215370178,
"grad_norm": 174.7665557861328,
"learning_rate": 4.976400700654751e-07,
"logits/chosen": 0.19272944331169128,
"logits/rejected": 0.15360750257968903,
"logps/chosen": -61.268951416015625,
"logps/ref_chosen": -59.93777084350586,
"logps/ref_rejected": -79.3138427734375,
"logps/rejected": -81.30525970458984,
"loss": 1.1594,
"margin_dpo/margin_mean": 0.6602364778518677,
"margin_dpo/margin_std": 1.286454677581787,
"step": 94
},
{
"KL/chosen_KL_mean": -1.6789188385009766,
"KL/mean": -2.059368133544922,
"KL/rejected_KL_mean": -2.4398155212402344,
"KL/std": 0.9847538471221924,
"epoch": 0.1436130007558579,
"fcm_dpo/beta": 0.7116259336471558,
"fcm_dpo/delta": -0.15106014907360077,
"fcm_dpo/margin": 0.7608870267868042,
"fcm_dpo/q_t": 0.3815036416053772,
"grad_norm": 173.473388671875,
"learning_rate": 4.974553604702332e-07,
"logits/chosen": 0.10755741596221924,
"logits/rejected": 0.04345201700925827,
"logps/chosen": -61.847408294677734,
"logps/ref_chosen": -60.168487548828125,
"logps/ref_rejected": -90.73665618896484,
"logps/rejected": -93.17646789550781,
"loss": 1.0468,
"margin_dpo/margin_mean": 0.7608871459960938,
"margin_dpo/margin_std": 1.0690686702728271,
"step": 95
},
{
"KL/chosen_KL_mean": -1.5870532989501953,
"KL/mean": -1.9773613214492798,
"KL/rejected_KL_mean": -2.3676681518554688,
"KL/std": 1.0919381380081177,
"epoch": 0.14512471655328799,
"fcm_dpo/beta": 0.6952941417694092,
"fcm_dpo/delta": -0.1507873833179474,
"fcm_dpo/margin": 0.7806140184402466,
"fcm_dpo/q_t": 0.38217777013778687,
"grad_norm": 157.1096649169922,
"learning_rate": 4.972637290166157e-07,
"logits/chosen": 0.14316622912883759,
"logits/rejected": 0.09965945780277252,
"logps/chosen": -62.255828857421875,
"logps/ref_chosen": -60.66877746582031,
"logps/ref_rejected": -88.30673217773438,
"logps/rejected": -90.67440032958984,
"loss": 1.0612,
"margin_dpo/margin_mean": 0.7806137800216675,
"margin_dpo/margin_std": 1.151845932006836,
"step": 96
},
{
"KL/chosen_KL_mean": -1.8969554901123047,
"KL/mean": -2.182953119277954,
"KL/rejected_KL_mean": -2.468952178955078,
"KL/std": 1.0534627437591553,
"epoch": 0.14663643235071808,
"fcm_dpo/beta": 0.681002676486969,
"fcm_dpo/delta": -0.05837059020996094,
"fcm_dpo/margin": 0.5719989538192749,
"fcm_dpo/q_t": 0.42118215560913086,
"grad_norm": 195.87579345703125,
"learning_rate": 4.970651810649666e-07,
"logits/chosen": 0.05669859051704407,
"logits/rejected": 0.01407955028116703,
"logps/chosen": -66.94107818603516,
"logps/ref_chosen": -65.04412078857422,
"logps/ref_rejected": -78.42092895507812,
"logps/rejected": -80.88987731933594,
"loss": 1.1914,
"margin_dpo/margin_mean": 0.5719987154006958,
"margin_dpo/margin_std": 1.2701518535614014,
"step": 97
},
{
"KL/chosen_KL_mean": -1.5694503784179688,
"KL/mean": -1.8198587894439697,
"KL/rejected_KL_mean": -2.070270538330078,
"KL/std": 0.9753029346466064,
"epoch": 0.14814814814814814,
"fcm_dpo/beta": 0.6845871210098267,
"fcm_dpo/delta": 0.05912531912326813,
"fcm_dpo/margin": 0.5008178949356079,
"fcm_dpo/q_t": 0.4241343140602112,
"grad_norm": 181.65858459472656,
"learning_rate": 4.968597221690985e-07,
"logits/chosen": 0.16081318259239197,
"logits/rejected": 0.13379907608032227,
"logps/chosen": -57.07268142700195,
"logps/ref_chosen": -55.503231048583984,
"logps/ref_rejected": -72.81553649902344,
"logps/rejected": -74.88581085205078,
"loss": 1.1758,
"margin_dpo/margin_mean": 0.5008175373077393,
"margin_dpo/margin_std": 0.968307614326477,
"step": 98
},
{
"KL/chosen_KL_mean": -1.6459503173828125,
"KL/mean": -2.0242457389831543,
"KL/rejected_KL_mean": -2.4025421142578125,
"KL/std": 1.1888670921325684,
"epoch": 0.14965986394557823,
"fcm_dpo/beta": 0.6779689788818359,
"fcm_dpo/delta": -0.11892664432525635,
"fcm_dpo/margin": 0.756589412689209,
"fcm_dpo/q_t": 0.3921675980091095,
"grad_norm": 184.27322387695312,
"learning_rate": 4.966473580761389e-07,
"logits/chosen": 0.17038282752037048,
"logits/rejected": 0.1327345073223114,
"logps/chosen": -60.221588134765625,
"logps/ref_chosen": -58.57563781738281,
"logps/ref_rejected": -78.693603515625,
"logps/rejected": -81.09614562988281,
"loss": 1.0894,
"margin_dpo/margin_mean": 0.7565888166427612,
"margin_dpo/margin_std": 1.2397615909576416,
"step": 99
},
{
"KL/chosen_KL_mean": -1.7846717834472656,
"KL/mean": -2.103785991668701,
"KL/rejected_KL_mean": -2.422893524169922,
"KL/std": 1.118520736694336,
"epoch": 0.15117157974300832,
"fcm_dpo/beta": 0.6647679805755615,
"fcm_dpo/delta": -0.12242830544710159,
"fcm_dpo/margin": 0.6382254362106323,
"fcm_dpo/q_t": 0.4146166443824768,
"grad_norm": 202.11663818359375,
"learning_rate": 4.964280947263676e-07,
"logits/chosen": 0.16854572296142578,
"logits/rejected": 0.16087126731872559,
"logps/chosen": -81.36810302734375,
"logps/ref_chosen": -79.58343505859375,
"logps/ref_rejected": -92.152587890625,
"logps/rejected": -94.57547760009766,
"loss": 1.2224,
"margin_dpo/margin_mean": 0.6382259130477905,
"margin_dpo/margin_std": 1.4874173402786255,
"step": 100
},
{
"KL/chosen_KL_mean": -1.5746269226074219,
"KL/mean": -2.0431926250457764,
"KL/rejected_KL_mean": -2.511760711669922,
"KL/std": 1.1946470737457275,
"epoch": 0.15268329554043839,
"fcm_dpo/beta": 0.6401762366294861,
"fcm_dpo/delta": -0.21215790510177612,
"fcm_dpo/margin": 0.9371323585510254,
"fcm_dpo/q_t": 0.3709060847759247,
"grad_norm": 137.63209533691406,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": 0.1399805247783661,
"logits/rejected": 0.10337221622467041,
"logps/chosen": -53.907413482666016,
"logps/ref_chosen": -52.332786560058594,
"logps/ref_rejected": -69.55589294433594,
"logps/rejected": -72.06765747070312,
"loss": 1.0012,
"margin_dpo/margin_mean": 0.9371322393417358,
"margin_dpo/margin_std": 1.1807992458343506,
"step": 101
},
{
"KL/chosen_KL_mean": -1.8736400604248047,
"KL/mean": -2.1846694946289062,
"KL/rejected_KL_mean": -2.4957008361816406,
"KL/std": 1.1980339288711548,
"epoch": 0.15419501133786848,
"fcm_dpo/beta": 0.6317287087440491,
"fcm_dpo/delta": 0.00714368000626564,
"fcm_dpo/margin": 0.6220631003379822,
"fcm_dpo/q_t": 0.4191555976867676,
"grad_norm": 170.46401977539062,
"learning_rate": 4.959688949822748e-07,
"logits/chosen": 0.07324576377868652,
"logits/rejected": 0.03491155803203583,
"logps/chosen": -66.61712646484375,
"logps/ref_chosen": -64.74348449707031,
"logps/ref_rejected": -69.06132507324219,
"logps/rejected": -71.5570297241211,
"loss": 1.2114,
"margin_dpo/margin_mean": 0.6220629215240479,
"margin_dpo/margin_std": 1.4274628162384033,
"step": 102
},
{
"KL/chosen_KL_mean": -1.815877914428711,
"KL/mean": -2.180802822113037,
"KL/rejected_KL_mean": -2.5457305908203125,
"KL/std": 1.2718205451965332,
"epoch": 0.15570672713529857,
"fcm_dpo/beta": 0.6257190108299255,
"fcm_dpo/delta": -0.059353649616241455,
"fcm_dpo/margin": 0.7298542261123657,
"fcm_dpo/q_t": 0.40414753556251526,
"grad_norm": 175.6512908935547,
"learning_rate": 4.957289714327572e-07,
"logits/chosen": 0.1976650059223175,
"logits/rejected": 0.16458025574684143,
"logps/chosen": -65.65251922607422,
"logps/ref_chosen": -63.83664321899414,
"logps/ref_rejected": -79.32362365722656,
"logps/rejected": -81.86935424804688,
"loss": 1.1334,
"margin_dpo/margin_mean": 0.7298538088798523,
"margin_dpo/margin_std": 1.3353081941604614,
"step": 103
},
{
"KL/chosen_KL_mean": -1.8301982879638672,
"KL/mean": -2.1910319328308105,
"KL/rejected_KL_mean": -2.551868438720703,
"KL/std": 1.4269229173660278,
"epoch": 0.15721844293272866,
"fcm_dpo/beta": 0.6197404861450195,
"fcm_dpo/delta": -0.04945854842662811,
"fcm_dpo/margin": 0.7216684222221375,
"fcm_dpo/q_t": 0.4082034230232239,
"grad_norm": 184.15505981445312,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 0.1419924795627594,
"logits/rejected": 0.061123307794332504,
"logps/chosen": -62.82940673828125,
"logps/ref_chosen": -60.99920654296875,
"logps/ref_rejected": -98.84645080566406,
"logps/rejected": -101.39832305908203,
"loss": 1.1558,
"margin_dpo/margin_mean": 0.7216675281524658,
"margin_dpo/margin_std": 1.4065872430801392,
"step": 104
},
{
"KL/chosen_KL_mean": -1.8332481384277344,
"KL/mean": -2.141366958618164,
"KL/rejected_KL_mean": -2.4494895935058594,
"KL/std": 1.2786672115325928,
"epoch": 0.15873015873015872,
"fcm_dpo/beta": 0.6212728023529053,
"fcm_dpo/delta": 0.017269816249608994,
"fcm_dpo/margin": 0.6162393093109131,
"fcm_dpo/q_t": 0.421569287776947,
"grad_norm": 191.83030700683594,
"learning_rate": 4.952285105344791e-07,
"logits/chosen": 0.10652521252632141,
"logits/rejected": 0.054222628474235535,
"logps/chosen": -72.78351593017578,
"logps/ref_chosen": -70.95027160644531,
"logps/ref_rejected": -87.88340759277344,
"logps/rejected": -90.33290100097656,
"loss": 1.2168,
"margin_dpo/margin_mean": 0.616238534450531,
"margin_dpo/margin_std": 1.4349312782287598,
"step": 105
},
{
"KL/chosen_KL_mean": -1.7448806762695312,
"KL/mean": -2.0746073722839355,
"KL/rejected_KL_mean": -2.4043350219726562,
"KL/std": 1.2081918716430664,
"epoch": 0.1602418745275888,
"fcm_dpo/beta": 0.6169089078903198,
"fcm_dpo/delta": -0.007208941504359245,
"fcm_dpo/margin": 0.6594525575637817,
"fcm_dpo/q_t": 0.40902554988861084,
"grad_norm": 175.51333618164062,
"learning_rate": 4.949679871846857e-07,
"logits/chosen": 0.15391142666339874,
"logits/rejected": 0.14048755168914795,
"logps/chosen": -64.20420837402344,
"logps/ref_chosen": -62.45933151245117,
"logps/ref_rejected": -67.00595092773438,
"logps/rejected": -69.4102783203125,
"loss": 1.1851,
"margin_dpo/margin_mean": 0.6594526171684265,
"margin_dpo/margin_std": 1.3870201110839844,
"step": 106
},
{
"KL/chosen_KL_mean": -2.0156211853027344,
"KL/mean": -2.2199954986572266,
"KL/rejected_KL_mean": -2.4243698120117188,
"KL/std": 1.3425004482269287,
"epoch": 0.1617535903250189,
"fcm_dpo/beta": 0.6140162944793701,
"fcm_dpo/delta": -0.03172950819134712,
"fcm_dpo/margin": 0.4087449312210083,
"fcm_dpo/q_t": 0.4492019712924957,
"grad_norm": 217.292724609375,
"learning_rate": 4.947006115536947e-07,
"logits/chosen": 0.09739897400140762,
"logits/rejected": 0.076191246509552,
"logps/chosen": -77.85359191894531,
"logps/ref_chosen": -75.83796691894531,
"logps/ref_rejected": -87.74038696289062,
"logps/rejected": -90.16474914550781,
"loss": 1.3509,
"margin_dpo/margin_mean": 0.4087446928024292,
"margin_dpo/margin_std": 1.4993884563446045,
"step": 107
},
{
"KL/chosen_KL_mean": -1.7861709594726562,
"KL/mean": -2.182220220565796,
"KL/rejected_KL_mean": -2.578266143798828,
"KL/std": 1.1284149885177612,
"epoch": 0.16326530612244897,
"fcm_dpo/beta": 0.6087417602539062,
"fcm_dpo/delta": -0.08634026348590851,
"fcm_dpo/margin": 0.7920923233032227,
"fcm_dpo/q_t": 0.3993995785713196,
"grad_norm": 160.85107421875,
"learning_rate": 4.944263911205772e-07,
"logits/chosen": 0.10616310685873032,
"logits/rejected": 0.07731328904628754,
"logps/chosen": -70.17941284179688,
"logps/ref_chosen": -68.39323425292969,
"logps/ref_rejected": -83.24267578125,
"logps/rejected": -85.82093811035156,
"loss": 1.1408,
"margin_dpo/margin_mean": 0.7920923233032227,
"margin_dpo/margin_std": 1.4701333045959473,
"step": 108
},
{
"KL/chosen_KL_mean": -1.7226943969726562,
"KL/mean": -2.2120048999786377,
"KL/rejected_KL_mean": -2.7013206481933594,
"KL/std": 1.3271257877349854,
"epoch": 0.16477702191987906,
"fcm_dpo/beta": 0.5871719121932983,
"fcm_dpo/delta": -0.1852605789899826,
"fcm_dpo/margin": 0.9786251783370972,
"fcm_dpo/q_t": 0.37816399335861206,
"grad_norm": 132.53904724121094,
"learning_rate": 4.941453335558681e-07,
"logits/chosen": 0.13148732483386993,
"logits/rejected": 0.07897891104221344,
"logps/chosen": -57.25017547607422,
"logps/ref_chosen": -55.52748107910156,
"logps/ref_rejected": -83.55218505859375,
"logps/rejected": -86.25350952148438,
"loss": 1.0096,
"margin_dpo/margin_mean": 0.9786243438720703,
"margin_dpo/margin_std": 1.2952110767364502,
"step": 109
},
{
"KL/chosen_KL_mean": -1.9659843444824219,
"KL/mean": -2.2083446979522705,
"KL/rejected_KL_mean": -2.4506988525390625,
"KL/std": 1.2513947486877441,
"epoch": 0.16628873771730915,
"fcm_dpo/beta": 0.5821672081947327,
"fcm_dpo/delta": 0.01055875513702631,
"fcm_dpo/margin": 0.48471495509147644,
"fcm_dpo/q_t": 0.4386028051376343,
"grad_norm": 182.23204040527344,
"learning_rate": 4.938574467213517e-07,
"logits/chosen": 0.07212984561920166,
"logits/rejected": 0.08014155924320221,
"logps/chosen": -83.12472534179688,
"logps/ref_chosen": -81.15874481201172,
"logps/ref_rejected": -72.56021118164062,
"logps/rejected": -75.01091003417969,
"loss": 1.2787,
"margin_dpo/margin_mean": 0.48471444845199585,
"margin_dpo/margin_std": 1.4037401676177979,
"step": 110
},
{
"KL/chosen_KL_mean": -2.121826171875,
"KL/mean": -2.510913372039795,
"KL/rejected_KL_mean": -2.9000015258789062,
"KL/std": 1.3530070781707764,
"epoch": 0.16780045351473924,
"fcm_dpo/beta": 0.580007791519165,
"fcm_dpo/delta": -0.053815118968486786,
"fcm_dpo/margin": 0.7781772613525391,
"fcm_dpo/q_t": 0.4041319787502289,
"grad_norm": 149.5125732421875,
"learning_rate": 4.935627386698418e-07,
"logits/chosen": 0.21666651964187622,
"logits/rejected": 0.17978689074516296,
"logps/chosen": -54.480812072753906,
"logps/ref_chosen": -52.358985900878906,
"logps/ref_rejected": -77.06150817871094,
"logps/rejected": -79.96150207519531,
"loss": 1.1631,
"margin_dpo/margin_mean": 0.7781772613525391,
"margin_dpo/margin_std": 1.5178613662719727,
"step": 111
},
{
"KL/chosen_KL_mean": -1.7510795593261719,
"KL/mean": -2.2202892303466797,
"KL/rejected_KL_mean": -2.6894989013671875,
"KL/std": 1.298105239868164,
"epoch": 0.1693121693121693,
"fcm_dpo/beta": 0.5631550550460815,
"fcm_dpo/delta": -0.13630488514900208,
"fcm_dpo/margin": 0.9384247660636902,
"fcm_dpo/q_t": 0.3831174969673157,
"grad_norm": 152.3679656982422,
"learning_rate": 4.932612176449559e-07,
"logits/chosen": 0.11729119718074799,
"logits/rejected": 0.055764634162187576,
"logps/chosen": -64.77114868164062,
"logps/ref_chosen": -63.02006530761719,
"logps/ref_rejected": -111.36941528320312,
"logps/rejected": -114.05891418457031,
"loss": 1.0705,
"margin_dpo/margin_mean": 0.9384238719940186,
"margin_dpo/margin_std": 1.4105713367462158,
"step": 112
},
{
"KL/chosen_KL_mean": -2.0300254821777344,
"KL/mean": -2.406491756439209,
"KL/rejected_KL_mean": -2.782958984375,
"KL/std": 1.3018248081207275,
"epoch": 0.1708238851095994,
"fcm_dpo/beta": 0.5636543035507202,
"fcm_dpo/delta": -0.026750415563583374,
"fcm_dpo/margin": 0.7529296278953552,
"fcm_dpo/q_t": 0.4060809910297394,
"grad_norm": 163.25575256347656,
"learning_rate": 4.929528920808854e-07,
"logits/chosen": 0.11804546415805817,
"logits/rejected": 0.08184659481048584,
"logps/chosen": -57.83768844604492,
"logps/ref_chosen": -55.80766296386719,
"logps/ref_rejected": -69.84014129638672,
"logps/rejected": -72.62310028076172,
"loss": 1.168,
"margin_dpo/margin_mean": 0.7529294490814209,
"margin_dpo/margin_std": 1.4496371746063232,
"step": 113
},
{
"KL/chosen_KL_mean": -1.7258930206298828,
"KL/mean": -2.333104133605957,
"KL/rejected_KL_mean": -2.9403228759765625,
"KL/std": 1.4261996746063232,
"epoch": 0.17233560090702948,
"fcm_dpo/beta": 0.5330042243003845,
"fcm_dpo/delta": -0.26693016290664673,
"fcm_dpo/margin": 1.2144203186035156,
"fcm_dpo/q_t": 0.36310064792633057,
"grad_norm": 117.72270965576172,
"learning_rate": 4.92637770602159e-07,
"logits/chosen": 0.15210115909576416,
"logits/rejected": 0.09475834667682648,
"logps/chosen": -68.05867004394531,
"logps/ref_chosen": -66.33277130126953,
"logps/ref_rejected": -71.61489868164062,
"logps/rejected": -74.55522155761719,
"loss": 0.9881,
"margin_dpo/margin_mean": 1.214421033859253,
"margin_dpo/margin_std": 1.5546455383300781,
"step": 114
},
{
"KL/chosen_KL_mean": -2.0225791931152344,
"KL/mean": -2.4677305221557617,
"KL/rejected_KL_mean": -2.9128856658935547,
"KL/std": 1.310913324356079,
"epoch": 0.17384731670445955,
"fcm_dpo/beta": 0.5242152214050293,
"fcm_dpo/delta": -0.06992093473672867,
"fcm_dpo/margin": 0.8903029561042786,
"fcm_dpo/q_t": 0.4018552005290985,
"grad_norm": 139.11448669433594,
"learning_rate": 4.923158620234019e-07,
"logits/chosen": 0.15971511602401733,
"logits/rejected": 0.1008654534816742,
"logps/chosen": -57.7716178894043,
"logps/ref_chosen": -55.74903869628906,
"logps/ref_rejected": -79.59849548339844,
"logps/rejected": -82.51138305664062,
"loss": 1.1056,
"margin_dpo/margin_mean": 0.8903037309646606,
"margin_dpo/margin_std": 1.484168291091919,
"step": 115
},
{
"KL/chosen_KL_mean": -1.9005279541015625,
"KL/mean": -2.3991334438323975,
"KL/rejected_KL_mean": -2.8977394104003906,
"KL/std": 1.3422160148620605,
"epoch": 0.17535903250188964,
"fcm_dpo/beta": 0.5140076875686646,
"fcm_dpo/delta": -0.11848685890436172,
"fcm_dpo/margin": 0.9972133636474609,
"fcm_dpo/q_t": 0.38590121269226074,
"grad_norm": 113.81331634521484,
"learning_rate": 4.91987175349089e-07,
"logits/chosen": 0.16647222638130188,
"logits/rejected": 0.10132342576980591,
"logps/chosen": -51.26569366455078,
"logps/ref_chosen": -49.36516571044922,
"logps/ref_rejected": -72.84671020507812,
"logps/rejected": -75.74444580078125,
"loss": 1.0387,
"margin_dpo/margin_mean": 0.9972136616706848,
"margin_dpo/margin_std": 1.29032564163208,
"step": 116
},
{
"KL/chosen_KL_mean": -1.7605628967285156,
"KL/mean": -2.1923394203186035,
"KL/rejected_KL_mean": -2.6241226196289062,
"KL/std": 1.2839633226394653,
"epoch": 0.17687074829931973,
"fcm_dpo/beta": 0.5017569065093994,
"fcm_dpo/delta": -0.03711225837469101,
"fcm_dpo/margin": 0.8635532855987549,
"fcm_dpo/q_t": 0.40248775482177734,
"grad_norm": 122.98551177978516,
"learning_rate": 4.916517197732933e-07,
"logits/chosen": 0.1693899929523468,
"logits/rejected": 0.13437990844249725,
"logps/chosen": -59.471458435058594,
"logps/ref_chosen": -57.710899353027344,
"logps/ref_rejected": -69.77253723144531,
"logps/rejected": -72.39665985107422,
"loss": 1.1345,
"margin_dpo/margin_mean": 0.8635537624359131,
"margin_dpo/margin_std": 1.4805222749710083,
"step": 117
},
{
"KL/chosen_KL_mean": -1.7426433563232422,
"KL/mean": -2.271818161010742,
"KL/rejected_KL_mean": -2.8009910583496094,
"KL/std": 1.3044204711914062,
"epoch": 0.17838246409674982,
"fcm_dpo/beta": 0.49396204948425293,
"fcm_dpo/delta": -0.13044118881225586,
"fcm_dpo/margin": 1.0583550930023193,
"fcm_dpo/q_t": 0.38405054807662964,
"grad_norm": 121.79106140136719,
"learning_rate": 4.913095046794281e-07,
"logits/chosen": 0.2404821366071701,
"logits/rejected": 0.20023274421691895,
"logps/chosen": -54.22254180908203,
"logps/ref_chosen": -52.479896545410156,
"logps/ref_rejected": -81.359130859375,
"logps/rejected": -84.16011810302734,
"loss": 1.0466,
"margin_dpo/margin_mean": 1.0583544969558716,
"margin_dpo/margin_std": 1.4395395517349243,
"step": 118
},
{
"KL/chosen_KL_mean": -2.245157241821289,
"KL/mean": -2.6533188819885254,
"KL/rejected_KL_mean": -3.061481475830078,
"KL/std": 1.4000425338745117,
"epoch": 0.17989417989417988,
"fcm_dpo/beta": 0.4941544532775879,
"fcm_dpo/delta": -0.0037781037390232086,
"fcm_dpo/margin": 0.816328763961792,
"fcm_dpo/q_t": 0.4127449095249176,
"grad_norm": 130.3661651611328,
"learning_rate": 4.909605396399855e-07,
"logits/chosen": 0.15195196866989136,
"logits/rejected": 0.11524452269077301,
"logps/chosen": -63.60282897949219,
"logps/ref_chosen": -61.35767364501953,
"logps/ref_rejected": -75.71510314941406,
"logps/rejected": -78.77658081054688,
"loss": 1.167,
"margin_dpo/margin_mean": 0.8163291811943054,
"margin_dpo/margin_std": 1.5954315662384033,
"step": 119
},
{
"KL/chosen_KL_mean": -1.8852157592773438,
"KL/mean": -2.4653992652893066,
"KL/rejected_KL_mean": -3.0455856323242188,
"KL/std": 1.3705031871795654,
"epoch": 0.18140589569160998,
"fcm_dpo/beta": 0.4791821837425232,
"fcm_dpo/delta": -0.1657349020242691,
"fcm_dpo/margin": 1.1603673696517944,
"fcm_dpo/q_t": 0.37820184230804443,
"grad_norm": 112.84229278564453,
"learning_rate": 4.906048344162676e-07,
"logits/chosen": 0.1201338917016983,
"logits/rejected": 0.06471075117588043,
"logps/chosen": -61.79278564453125,
"logps/ref_chosen": -59.907569885253906,
"logps/ref_rejected": -79.6910629272461,
"logps/rejected": -82.73664855957031,
"loss": 1.0137,
"margin_dpo/margin_mean": 1.1603679656982422,
"margin_dpo/margin_std": 1.4593796730041504,
"step": 120
},
{
"KL/chosen_KL_mean": -2.1395797729492188,
"KL/mean": -2.5901975631713867,
"KL/rejected_KL_mean": -3.0408248901367188,
"KL/std": 1.428723931312561,
"epoch": 0.18291761148904007,
"fcm_dpo/beta": 0.4728338122367859,
"fcm_dpo/delta": -0.027584142982959747,
"fcm_dpo/margin": 0.9012417793273926,
"fcm_dpo/q_t": 0.40799450874328613,
"grad_norm": 110.91853332519531,
"learning_rate": 4.902423989581143e-07,
"logits/chosen": 0.23473472893238068,
"logits/rejected": 0.15421560406684875,
"logps/chosen": -57.80562210083008,
"logps/ref_chosen": -55.66604232788086,
"logps/ref_rejected": -101.56233978271484,
"logps/rejected": -104.60316467285156,
"loss": 1.1032,
"margin_dpo/margin_mean": 0.9012415409088135,
"margin_dpo/margin_std": 1.432379961013794,
"step": 121
},
{
"KL/chosen_KL_mean": -2.140247344970703,
"KL/mean": -2.774712562561035,
"KL/rejected_KL_mean": -3.409181594848633,
"KL/std": 1.581752061843872,
"epoch": 0.18442932728647016,
"fcm_dpo/beta": 0.46123456954956055,
"fcm_dpo/delta": -0.19642525911331177,
"fcm_dpo/margin": 1.2689313888549805,
"fcm_dpo/q_t": 0.375938355922699,
"grad_norm": 120.42190551757812,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 0.1547389179468155,
"logits/rejected": 0.12131767719984055,
"logps/chosen": -65.47462463378906,
"logps/ref_chosen": -63.334373474121094,
"logps/ref_rejected": -73.67523193359375,
"logps/rejected": -77.08441925048828,
"loss": 1.0168,
"margin_dpo/margin_mean": 1.2689316272735596,
"margin_dpo/margin_std": 1.7266268730163574,
"step": 122
},
{
"KL/chosen_KL_mean": -2.0633163452148438,
"KL/mean": -2.544942617416382,
"KL/rejected_KL_mean": -3.026569366455078,
"KL/std": 1.4022800922393799,
"epoch": 0.18594104308390022,
"fcm_dpo/beta": 0.4563339054584503,
"fcm_dpo/delta": -0.04245033860206604,
"fcm_dpo/margin": 0.963251531124115,
"fcm_dpo/q_t": 0.40118855237960815,
"grad_norm": 114.81712341308594,
"learning_rate": 4.894973780788722e-07,
"logits/chosen": 0.16946694254875183,
"logits/rejected": 0.12972989678382874,
"logps/chosen": -58.962059020996094,
"logps/ref_chosen": -56.89874267578125,
"logps/ref_rejected": -78.97028350830078,
"logps/rejected": -81.99685668945312,
"loss": 1.1173,
"margin_dpo/margin_mean": 0.9632514715194702,
"margin_dpo/margin_std": 1.56075918674469,
"step": 123
},
{
"KL/chosen_KL_mean": -2.179250717163086,
"KL/mean": -2.8869895935058594,
"KL/rejected_KL_mean": -3.5947303771972656,
"KL/std": 1.5565268993377686,
"epoch": 0.1874527588813303,
"fcm_dpo/beta": 0.4349837601184845,
"fcm_dpo/delta": -0.230790913105011,
"fcm_dpo/margin": 1.4154765605926514,
"fcm_dpo/q_t": 0.36387136578559875,
"grad_norm": 95.26594543457031,
"learning_rate": 4.89114813497619e-07,
"logits/chosen": 0.1885579228401184,
"logits/rejected": 0.13248518109321594,
"logps/chosen": -59.29533386230469,
"logps/ref_chosen": -57.116085052490234,
"logps/ref_rejected": -87.93074035644531,
"logps/rejected": -91.52547454833984,
"loss": 0.9767,
"margin_dpo/margin_mean": 1.4154765605926514,
"margin_dpo/margin_std": 1.6168615818023682,
"step": 124
},
{
"KL/chosen_KL_mean": -2.348531723022461,
"KL/mean": -2.918497323989868,
"KL/rejected_KL_mean": -3.4884605407714844,
"KL/std": 1.6976053714752197,
"epoch": 0.1889644746787604,
"fcm_dpo/beta": 0.4234713912010193,
"fcm_dpo/delta": -0.08786194771528244,
"fcm_dpo/margin": 1.1399312019348145,
"fcm_dpo/q_t": 0.39451566338539124,
"grad_norm": 107.3994140625,
"learning_rate": 4.887255603610184e-07,
"logits/chosen": 0.22186070680618286,
"logits/rejected": 0.16541635990142822,
"logps/chosen": -68.0547103881836,
"logps/ref_chosen": -65.7061767578125,
"logps/ref_rejected": -91.72711944580078,
"logps/rejected": -95.215576171875,
"loss": 1.0755,
"margin_dpo/margin_mean": 1.139931559562683,
"margin_dpo/margin_std": 1.6825425624847412,
"step": 125
},
{
"KL/chosen_KL_mean": -2.1544017791748047,
"KL/mean": -2.6223111152648926,
"KL/rejected_KL_mean": -3.090221405029297,
"KL/std": 1.8600356578826904,
"epoch": 0.19047619047619047,
"fcm_dpo/beta": 0.42346400022506714,
"fcm_dpo/delta": 0.003847735933959484,
"fcm_dpo/margin": 0.9358229637145996,
"fcm_dpo/q_t": 0.4186503291130066,
"grad_norm": 101.1116714477539,
"learning_rate": 4.883296295573176e-07,
"logits/chosen": 0.03314465656876564,
"logits/rejected": 0.027043253183364868,
"logps/chosen": -70.33049011230469,
"logps/ref_chosen": -68.17608642578125,
"logps/ref_rejected": -65.1175537109375,
"logps/rejected": -68.20777893066406,
"loss": 1.1791,
"margin_dpo/margin_mean": 0.9358232021331787,
"margin_dpo/margin_std": 2.0149693489074707,
"step": 126
},
{
"KL/chosen_KL_mean": -2.306184768676758,
"KL/mean": -2.985496997833252,
"KL/rejected_KL_mean": -3.6648082733154297,
"KL/std": 1.5123531818389893,
"epoch": 0.19198790627362056,
"fcm_dpo/beta": 0.4105684757232666,
"fcm_dpo/delta": -0.16932585835456848,
"fcm_dpo/margin": 1.3586195707321167,
"fcm_dpo/q_t": 0.37370991706848145,
"grad_norm": 96.787109375,
"learning_rate": 4.87927032161552e-07,
"logits/chosen": 0.12655611336231232,
"logits/rejected": 0.09727788716554642,
"logps/chosen": -64.18641662597656,
"logps/ref_chosen": -61.88023376464844,
"logps/ref_rejected": -68.46012878417969,
"logps/rejected": -72.12493133544922,
"loss": 0.9962,
"margin_dpo/margin_mean": 1.358619213104248,
"margin_dpo/margin_std": 1.515355110168457,
"step": 127
},
{
"KL/chosen_KL_mean": -2.4993038177490234,
"KL/mean": -3.0452373027801514,
"KL/rejected_KL_mean": -3.591175079345703,
"KL/std": 1.7433449029922485,
"epoch": 0.19349962207105065,
"fcm_dpo/beta": 0.4058646857738495,
"fcm_dpo/delta": -0.04547997564077377,
"fcm_dpo/margin": 1.0918666124343872,
"fcm_dpo/q_t": 0.40237781405448914,
"grad_norm": 103.96131896972656,
"learning_rate": 4.875177794352363e-07,
"logits/chosen": 0.1509719043970108,
"logits/rejected": 0.09882430732250214,
"logps/chosen": -69.20828247070312,
"logps/ref_chosen": -66.708984375,
"logps/ref_rejected": -94.97969055175781,
"logps/rejected": -98.57086181640625,
"loss": 1.1333,
"margin_dpo/margin_mean": 1.0918666124343872,
"margin_dpo/margin_std": 1.9513864517211914,
"step": 128
},
{
"KL/chosen_KL_mean": -2.702585220336914,
"KL/mean": -3.182559013366699,
"KL/rejected_KL_mean": -3.66253662109375,
"KL/std": 1.5918266773223877,
"epoch": 0.19501133786848074,
"fcm_dpo/beta": 0.4065204858779907,
"fcm_dpo/delta": 0.01015464123338461,
"fcm_dpo/margin": 0.9599518775939941,
"fcm_dpo/q_t": 0.41556084156036377,
"grad_norm": 111.83089447021484,
"learning_rate": 4.871018828260491e-07,
"logits/chosen": 0.15229831635951996,
"logits/rejected": 0.14520448446273804,
"logps/chosen": -68.04141235351562,
"logps/ref_chosen": -65.33882904052734,
"logps/ref_rejected": -68.06109619140625,
"logps/rejected": -71.7236328125,
"loss": 1.1526,
"margin_dpo/margin_mean": 0.9599519371986389,
"margin_dpo/margin_std": 1.7882498502731323,
"step": 129
},
{
"KL/chosen_KL_mean": -2.546079635620117,
"KL/mean": -3.085463285446167,
"KL/rejected_KL_mean": -3.624847412109375,
"KL/std": 1.5492210388183594,
"epoch": 0.1965230536659108,
"fcm_dpo/beta": 0.40080416202545166,
"fcm_dpo/delta": -0.03621768206357956,
"fcm_dpo/margin": 1.0787646770477295,
"fcm_dpo/q_t": 0.4028658866882324,
"grad_norm": 115.964599609375,
"learning_rate": 4.866793539675126e-07,
"logits/chosen": 0.10994696617126465,
"logits/rejected": 0.06416427344083786,
"logps/chosen": -61.206825256347656,
"logps/ref_chosen": -58.660743713378906,
"logps/ref_rejected": -79.24510192871094,
"logps/rejected": -82.86994934082031,
"loss": 1.0892,
"margin_dpo/margin_mean": 1.0787646770477295,
"margin_dpo/margin_std": 1.4687458276748657,
"step": 130
},
{
"KL/chosen_KL_mean": -2.4539833068847656,
"KL/mean": -3.1280264854431152,
"KL/rejected_KL_mean": -3.8020706176757812,
"KL/std": 1.852320909500122,
"epoch": 0.1980347694633409,
"fcm_dpo/beta": 0.39587312936782837,
"fcm_dpo/delta": -0.14114192128181458,
"fcm_dpo/margin": 1.3480905294418335,
"fcm_dpo/q_t": 0.3839923143386841,
"grad_norm": 94.91565704345703,
"learning_rate": 4.86250204678667e-07,
"logits/chosen": 0.13292667269706726,
"logits/rejected": 0.07401425391435623,
"logps/chosen": -54.96852111816406,
"logps/ref_chosen": -52.51453399658203,
"logps/ref_rejected": -85.18299865722656,
"logps/rejected": -88.98506927490234,
"loss": 1.0648,
"margin_dpo/margin_mean": 1.3480902910232544,
"margin_dpo/margin_std": 2.0145888328552246,
"step": 131
},
{
"KL/chosen_KL_mean": -2.7316837310791016,
"KL/mean": -3.319563150405884,
"KL/rejected_KL_mean": -3.907438278198242,
"KL/std": 1.8400081396102905,
"epoch": 0.19954648526077098,
"fcm_dpo/beta": 0.3890807330608368,
"fcm_dpo/delta": -0.06025748327374458,
"fcm_dpo/margin": 1.175754189491272,
"fcm_dpo/q_t": 0.397086501121521,
"grad_norm": 100.52850341796875,
"learning_rate": 4.858144469637408e-07,
"logits/chosen": 0.21467986702919006,
"logits/rejected": 0.18342456221580505,
"logps/chosen": -68.41681671142578,
"logps/ref_chosen": -65.68513488769531,
"logps/ref_rejected": -69.54120635986328,
"logps/rejected": -73.44864654541016,
"loss": 1.1134,
"margin_dpo/margin_mean": 1.1757543087005615,
"margin_dpo/margin_std": 1.974447250366211,
"step": 132
},
{
"KL/chosen_KL_mean": -2.690654754638672,
"KL/mean": -3.2350802421569824,
"KL/rejected_KL_mean": -3.7795028686523438,
"KL/std": 1.7662453651428223,
"epoch": 0.20105820105820105,
"fcm_dpo/beta": 0.38747304677963257,
"fcm_dpo/delta": -0.022889260202646255,
"fcm_dpo/margin": 1.0888489484786987,
"fcm_dpo/q_t": 0.4062184691429138,
"grad_norm": 104.7711410522461,
"learning_rate": 4.853720930118138e-07,
"logits/chosen": 0.12074915319681168,
"logits/rejected": 0.11150172352790833,
"logps/chosen": -66.28877258300781,
"logps/ref_chosen": -63.598114013671875,
"logps/ref_rejected": -73.72798156738281,
"logps/rejected": -77.50748443603516,
"loss": 1.1298,
"margin_dpo/margin_mean": 1.0888489484786987,
"margin_dpo/margin_std": 1.8826422691345215,
"step": 133
},
{
"KL/chosen_KL_mean": -2.5860939025878906,
"KL/mean": -3.3605504035949707,
"KL/rejected_KL_mean": -4.135005950927734,
"KL/std": 1.967972993850708,
"epoch": 0.20256991685563114,
"fcm_dpo/beta": 0.37378889322280884,
"fcm_dpo/delta": -0.19127684831619263,
"fcm_dpo/margin": 1.5489141941070557,
"fcm_dpo/q_t": 0.3716249465942383,
"grad_norm": 85.89019775390625,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": 0.21915540099143982,
"logits/rejected": 0.16665717959403992,
"logps/chosen": -56.38066482543945,
"logps/ref_chosen": -53.79457092285156,
"logps/ref_rejected": -74.16741943359375,
"logps/rejected": -78.30242919921875,
"loss": 1.0065,
"margin_dpo/margin_mean": 1.5489141941070557,
"margin_dpo/margin_std": 1.9299194812774658,
"step": 134
},
{
"KL/chosen_KL_mean": -2.618410110473633,
"KL/mean": -3.136845111846924,
"KL/rejected_KL_mean": -3.6552886962890625,
"KL/std": 1.9088587760925293,
"epoch": 0.20408163265306123,
"fcm_dpo/beta": 0.3727998733520508,
"fcm_dpo/delta": 0.013980102725327015,
"fcm_dpo/margin": 1.0368762016296387,
"fcm_dpo/q_t": 0.41572168469429016,
"grad_norm": 85.65084838867188,
"learning_rate": 4.844676460754862e-07,
"logits/chosen": 0.12532413005828857,
"logits/rejected": 0.09490326046943665,
"logps/chosen": -52.059486389160156,
"logps/ref_chosen": -49.441078186035156,
"logps/ref_rejected": -65.96878051757812,
"logps/rejected": -69.62406921386719,
"loss": 1.1534,
"margin_dpo/margin_mean": 1.03687584400177,
"margin_dpo/margin_std": 1.9357593059539795,
"step": 135
},
{
"KL/chosen_KL_mean": -3.030406951904297,
"KL/mean": -3.6895689964294434,
"KL/rejected_KL_mean": -4.3487396240234375,
"KL/std": 2.0789108276367188,
"epoch": 0.20559334845049132,
"fcm_dpo/beta": 0.36738646030426025,
"fcm_dpo/delta": -0.0888245701789856,
"fcm_dpo/margin": 1.3183355331420898,
"fcm_dpo/q_t": 0.404574453830719,
"grad_norm": 107.13855743408203,
"learning_rate": 4.840055783904106e-07,
"logits/chosen": 0.13531756401062012,
"logits/rejected": 0.07051342725753784,
"logps/chosen": -69.7896728515625,
"logps/ref_chosen": -66.75926208496094,
"logps/ref_rejected": -94.61787414550781,
"logps/rejected": -98.96661376953125,
"loss": 1.1618,
"margin_dpo/margin_mean": 1.3183361291885376,
"margin_dpo/margin_std": 2.6758766174316406,
"step": 136
},
{
"KL/chosen_KL_mean": -2.863384246826172,
"KL/mean": -3.5742931365966797,
"KL/rejected_KL_mean": -4.2852020263671875,
"KL/std": 1.9335532188415527,
"epoch": 0.20710506424792138,
"fcm_dpo/beta": 0.36090317368507385,
"fcm_dpo/delta": -0.11908543109893799,
"fcm_dpo/margin": 1.4218175411224365,
"fcm_dpo/q_t": 0.38955453038215637,
"grad_norm": 82.42172241210938,
"learning_rate": 4.835369650662767e-07,
"logits/chosen": 0.16094376146793365,
"logits/rejected": 0.13444793224334717,
"logps/chosen": -59.64718246459961,
"logps/ref_chosen": -56.78379821777344,
"logps/ref_rejected": -69.89952087402344,
"logps/rejected": -74.18472290039062,
"loss": 1.0739,
"margin_dpo/margin_mean": 1.4218180179595947,
"margin_dpo/margin_std": 2.185852527618408,
"step": 137
},
{
"KL/chosen_KL_mean": -3.364490509033203,
"KL/mean": -3.902451276779175,
"KL/rejected_KL_mean": -4.440422058105469,
"KL/std": 1.9939281940460205,
"epoch": 0.20861678004535147,
"fcm_dpo/beta": 0.3582695722579956,
"fcm_dpo/delta": 0.015112070366740227,
"fcm_dpo/margin": 1.075927495956421,
"fcm_dpo/q_t": 0.41441237926483154,
"grad_norm": 90.68624877929688,
"learning_rate": 4.830618192112065e-07,
"logits/chosen": 0.16553908586502075,
"logits/rejected": 0.13115090131759644,
"logps/chosen": -62.13050079345703,
"logps/ref_chosen": -58.766014099121094,
"logps/ref_rejected": -68.12371826171875,
"logps/rejected": -72.56414031982422,
"loss": 1.1643,
"margin_dpo/margin_mean": 1.0759272575378418,
"margin_dpo/margin_std": 2.064164161682129,
"step": 138
},
{
"KL/chosen_KL_mean": -3.3314437866210938,
"KL/mean": -4.026371479034424,
"KL/rejected_KL_mean": -4.7212982177734375,
"KL/std": 1.9014110565185547,
"epoch": 0.21012849584278157,
"fcm_dpo/beta": 0.3557409346103668,
"fcm_dpo/delta": -0.0995248481631279,
"fcm_dpo/margin": 1.389854907989502,
"fcm_dpo/q_t": 0.3908173143863678,
"grad_norm": 92.82787322998047,
"learning_rate": 4.825801541160509e-07,
"logits/chosen": 0.12655504047870636,
"logits/rejected": 0.09962915629148483,
"logps/chosen": -74.55703735351562,
"logps/ref_chosen": -71.2255859375,
"logps/ref_rejected": -82.1834716796875,
"logps/rejected": -86.90476989746094,
"loss": 1.0595,
"margin_dpo/margin_mean": 1.389855146408081,
"margin_dpo/margin_std": 1.9059739112854004,
"step": 139
},
{
"KL/chosen_KL_mean": -3.0566234588623047,
"KL/mean": -3.9430952072143555,
"KL/rejected_KL_mean": -4.829566955566406,
"KL/std": 2.5058655738830566,
"epoch": 0.21164021164021163,
"fcm_dpo/beta": 0.3404355049133301,
"fcm_dpo/delta": -0.2167389988899231,
"fcm_dpo/margin": 1.7729389667510986,
"fcm_dpo/q_t": 0.3692883849143982,
"grad_norm": 94.72462463378906,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 0.12459614872932434,
"logits/rejected": 0.08268279582262039,
"logps/chosen": -66.33428955078125,
"logps/ref_chosen": -63.27766418457031,
"logps/ref_rejected": -83.30647277832031,
"logps/rejected": -88.13603973388672,
"loss": 1.0579,
"margin_dpo/margin_mean": 1.77293860912323,
"margin_dpo/margin_std": 2.728281259536743,
"step": 140
},
{
"KL/chosen_KL_mean": -3.254852294921875,
"KL/mean": -4.092084884643555,
"KL/rejected_KL_mean": -4.929317474365234,
"KL/std": 2.1259002685546875,
"epoch": 0.21315192743764172,
"fcm_dpo/beta": 0.32717373967170715,
"fcm_dpo/delta": -0.15788133442401886,
"fcm_dpo/margin": 1.6744616031646729,
"fcm_dpo/q_t": 0.3854616582393646,
"grad_norm": 85.70342254638672,
"learning_rate": 4.815973202802966e-07,
"logits/chosen": 0.16417661309242249,
"logits/rejected": 0.12390686571598053,
"logps/chosen": -65.02162170410156,
"logps/ref_chosen": -61.76676940917969,
"logps/ref_rejected": -88.60601806640625,
"logps/rejected": -93.53533935546875,
"loss": 1.0674,
"margin_dpo/margin_mean": 1.674462080001831,
"margin_dpo/margin_std": 2.571665048599243,
"step": 141
},
{
"KL/chosen_KL_mean": -3.2735595703125,
"KL/mean": -3.8929660320281982,
"KL/rejected_KL_mean": -4.512371063232422,
"KL/std": 2.063791513442993,
"epoch": 0.2146636432350718,
"fcm_dpo/beta": 0.3266686797142029,
"fcm_dpo/delta": -0.004939114674925804,
"fcm_dpo/margin": 1.238810658454895,
"fcm_dpo/q_t": 0.41002559661865234,
"grad_norm": 83.58145904541016,
"learning_rate": 4.810961790316729e-07,
"logits/chosen": 0.16356688737869263,
"logits/rejected": 0.1397327035665512,
"logps/chosen": -68.54833984375,
"logps/ref_chosen": -65.2747802734375,
"logps/ref_rejected": -81.1378173828125,
"logps/rejected": -85.65019226074219,
"loss": 1.125,
"margin_dpo/margin_mean": 1.2388105392456055,
"margin_dpo/margin_std": 2.054414749145508,
"step": 142
},
{
"KL/chosen_KL_mean": -3.3590450286865234,
"KL/mean": -3.998101234436035,
"KL/rejected_KL_mean": -4.637153625488281,
"KL/std": 2.18355655670166,
"epoch": 0.2161753590325019,
"fcm_dpo/beta": 0.3245221972465515,
"fcm_dpo/delta": -0.015544133260846138,
"fcm_dpo/margin": 1.2781095504760742,
"fcm_dpo/q_t": 0.4078383445739746,
"grad_norm": 99.0627670288086,
"learning_rate": 4.805885735261454e-07,
"logits/chosen": 0.1751534640789032,
"logits/rejected": 0.15927816927433014,
"logps/chosen": -65.97686767578125,
"logps/ref_chosen": -62.617828369140625,
"logps/ref_rejected": -70.39239501953125,
"logps/rejected": -75.029541015625,
"loss": 1.1659,
"margin_dpo/margin_mean": 1.2781095504760742,
"margin_dpo/margin_std": 2.5324602127075195,
"step": 143
},
{
"KL/chosen_KL_mean": -3.566843032836914,
"KL/mean": -4.33714485168457,
"KL/rejected_KL_mean": -5.107444763183594,
"KL/std": 2.3009777069091797,
"epoch": 0.21768707482993196,
"fcm_dpo/beta": 0.319795161485672,
"fcm_dpo/delta": -0.09767691791057587,
"fcm_dpo/margin": 1.5406033992767334,
"fcm_dpo/q_t": 0.3934960961341858,
"grad_norm": 84.77015686035156,
"learning_rate": 4.800745179625307e-07,
"logits/chosen": 0.1603230983018875,
"logits/rejected": 0.13270872831344604,
"logps/chosen": -64.3695297241211,
"logps/ref_chosen": -60.80268859863281,
"logps/ref_rejected": -79.07284545898438,
"logps/rejected": -84.18028259277344,
"loss": 1.0946,
"margin_dpo/margin_mean": 1.5406036376953125,
"margin_dpo/margin_std": 2.5048251152038574,
"step": 144
},
{
"KL/chosen_KL_mean": -3.4350738525390625,
"KL/mean": -4.177140235900879,
"KL/rejected_KL_mean": -4.9192047119140625,
"KL/std": 2.394912004470825,
"epoch": 0.21919879062736206,
"fcm_dpo/beta": 0.31537872552871704,
"fcm_dpo/delta": -0.07139455527067184,
"fcm_dpo/margin": 1.484137773513794,
"fcm_dpo/q_t": 0.3996923863887787,
"grad_norm": 93.71647644042969,
"learning_rate": 4.795540267200686e-07,
"logits/chosen": 0.10820844769477844,
"logits/rejected": 0.12509872019290924,
"logps/chosen": -78.04653930664062,
"logps/ref_chosen": -74.61146545410156,
"logps/ref_rejected": -83.24461364746094,
"logps/rejected": -88.163818359375,
"loss": 1.1342,
"margin_dpo/margin_mean": 1.484137773513794,
"margin_dpo/margin_std": 2.7337613105773926,
"step": 145
},
{
"KL/chosen_KL_mean": -3.231609344482422,
"KL/mean": -4.0334978103637695,
"KL/rejected_KL_mean": -4.83538818359375,
"KL/std": 2.3423705101013184,
"epoch": 0.22071050642479215,
"fcm_dpo/beta": 0.3101949691772461,
"fcm_dpo/delta": -0.10246110707521439,
"fcm_dpo/margin": 1.6037830114364624,
"fcm_dpo/q_t": 0.3915921449661255,
"grad_norm": 77.48421478271484,
"learning_rate": 4.790271143580173e-07,
"logits/chosen": 0.1114337369799614,
"logits/rejected": 0.09617681801319122,
"logps/chosen": -61.072593688964844,
"logps/ref_chosen": -57.84098434448242,
"logps/ref_rejected": -67.47422790527344,
"logps/rejected": -72.30961608886719,
"loss": 1.072,
"margin_dpo/margin_mean": 1.6037828922271729,
"margin_dpo/margin_std": 2.388605833053589,
"step": 146
},
{
"KL/chosen_KL_mean": -3.8188533782958984,
"KL/mean": -4.413590908050537,
"KL/rejected_KL_mean": -5.008327484130859,
"KL/std": 2.3607306480407715,
"epoch": 0.2222222222222222,
"fcm_dpo/beta": 0.30978289246559143,
"fcm_dpo/delta": 0.03262418136000633,
"fcm_dpo/margin": 1.189474105834961,
"fcm_dpo/q_t": 0.41801732778549194,
"grad_norm": 101.688720703125,
"learning_rate": 4.784937956152489e-07,
"logits/chosen": 0.106835275888443,
"logits/rejected": 0.06854995340108871,
"logps/chosen": -70.63232421875,
"logps/ref_chosen": -66.81346893310547,
"logps/ref_rejected": -81.1796875,
"logps/rejected": -86.18801879882812,
"loss": 1.2012,
"margin_dpo/margin_mean": 1.1894733905792236,
"margin_dpo/margin_std": 2.6434860229492188,
"step": 147
},
{
"KL/chosen_KL_mean": -3.61895751953125,
"KL/mean": -4.4911603927612305,
"KL/rejected_KL_mean": -5.3633575439453125,
"KL/std": 2.2856435775756836,
"epoch": 0.2237339380196523,
"fcm_dpo/beta": 0.3047756552696228,
"fcm_dpo/delta": -0.13882245123386383,
"fcm_dpo/margin": 1.7444008588790894,
"fcm_dpo/q_t": 0.3859821856021881,
"grad_norm": 66.71563720703125,
"learning_rate": 4.779540854098347e-07,
"logits/chosen": 0.2662171721458435,
"logits/rejected": 0.1974124312400818,
"logps/chosen": -52.30671310424805,
"logps/ref_chosen": -48.6877555847168,
"logps/ref_rejected": -67.50503540039062,
"logps/rejected": -72.86839294433594,
"loss": 1.0672,
"margin_dpo/margin_mean": 1.7444007396697998,
"margin_dpo/margin_std": 2.620556354522705,
"step": 148
},
{
"KL/chosen_KL_mean": -4.425138473510742,
"KL/mean": -5.410511016845703,
"KL/rejected_KL_mean": -6.395885467529297,
"KL/std": 2.864663600921631,
"epoch": 0.2252456538170824,
"fcm_dpo/beta": 0.2930014133453369,
"fcm_dpo/delta": -0.18832086026668549,
"fcm_dpo/margin": 1.9707480669021606,
"fcm_dpo/q_t": 0.37564554810523987,
"grad_norm": 69.97044372558594,
"learning_rate": 4.774079988386296e-07,
"logits/chosen": 0.11522063612937927,
"logits/rejected": 0.07022479176521301,
"logps/chosen": -59.56891632080078,
"logps/ref_chosen": -55.143775939941406,
"logps/ref_rejected": -64.79888916015625,
"logps/rejected": -71.19477844238281,
"loss": 1.0262,
"margin_dpo/margin_mean": 1.9707480669021606,
"margin_dpo/margin_std": 2.7147183418273926,
"step": 149
},
{
"KL/chosen_KL_mean": -3.3575782775878906,
"KL/mean": -4.577151298522949,
"KL/rejected_KL_mean": -5.796714782714844,
"KL/std": 2.655878782272339,
"epoch": 0.22675736961451248,
"fcm_dpo/beta": 0.27746373414993286,
"fcm_dpo/delta": -0.2971411943435669,
"fcm_dpo/margin": 2.4391417503356934,
"fcm_dpo/q_t": 0.3503156304359436,
"grad_norm": 64.59636688232422,
"learning_rate": 4.768555511768486e-07,
"logits/chosen": 0.16118960082530975,
"logits/rejected": 0.12086163461208344,
"logps/chosen": -70.82832336425781,
"logps/ref_chosen": -67.47074890136719,
"logps/ref_rejected": -89.21170806884766,
"logps/rejected": -95.0084228515625,
"loss": 0.9324,
"margin_dpo/margin_mean": 2.4391419887542725,
"margin_dpo/margin_std": 2.5503554344177246,
"step": 150
},
{
"KL/chosen_KL_mean": -3.3143768310546875,
"KL/mean": -4.572887897491455,
"KL/rejected_KL_mean": -5.831398010253906,
"KL/std": 2.6611104011535645,
"epoch": 0.22826908541194255,
"fcm_dpo/beta": 0.2628706097602844,
"fcm_dpo/delta": -0.2797275483608246,
"fcm_dpo/margin": 2.517019748687744,
"fcm_dpo/q_t": 0.35459136962890625,
"grad_norm": 55.8674201965332,
"learning_rate": 4.762967578776406e-07,
"logits/chosen": 0.17514903843402863,
"logits/rejected": 0.12448206543922424,
"logps/chosen": -55.77391815185547,
"logps/ref_chosen": -52.45954132080078,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -84.8944091796875,
"loss": 0.9427,
"margin_dpo/margin_mean": 2.517019271850586,
"margin_dpo/margin_std": 2.736574649810791,
"step": 151
},
{
"KL/chosen_KL_mean": -4.198028564453125,
"KL/mean": -5.243851661682129,
"KL/rejected_KL_mean": -6.289680480957031,
"KL/std": 2.7917838096618652,
"epoch": 0.22978080120937264,
"fcm_dpo/beta": 0.2548731565475464,
"fcm_dpo/delta": -0.14054642617702484,
"fcm_dpo/margin": 2.091653823852539,
"fcm_dpo/q_t": 0.3867358863353729,
"grad_norm": 64.32059478759766,
"learning_rate": 4.757316345716553e-07,
"logits/chosen": 0.24530437588691711,
"logits/rejected": 0.19649431109428406,
"logps/chosen": -60.751861572265625,
"logps/ref_chosen": -56.5538330078125,
"logps/ref_rejected": -76.55074310302734,
"logps/rejected": -82.84042358398438,
"loss": 1.0706,
"margin_dpo/margin_mean": 2.091653823852539,
"margin_dpo/margin_std": 3.169095516204834,
"step": 152
},
{
"KL/chosen_KL_mean": -4.060447692871094,
"KL/mean": -5.112576961517334,
"KL/rejected_KL_mean": -6.164703369140625,
"KL/std": 3.0209126472473145,
"epoch": 0.23129251700680273,
"fcm_dpo/beta": 0.24635225534439087,
"fcm_dpo/delta": -0.12512800097465515,
"fcm_dpo/margin": 2.1042511463165283,
"fcm_dpo/q_t": 0.3836144506931305,
"grad_norm": 59.735877990722656,
"learning_rate": 4.751601970666064e-07,
"logits/chosen": 0.12516067922115326,
"logits/rejected": 0.08992981165647507,
"logps/chosen": -72.06733703613281,
"logps/ref_chosen": -68.00689697265625,
"logps/ref_rejected": -74.83482360839844,
"logps/rejected": -80.99952697753906,
"loss": 1.0276,
"margin_dpo/margin_mean": 2.1042513847351074,
"margin_dpo/margin_std": 2.618156671524048,
"step": 153
},
{
"KL/chosen_KL_mean": -4.809171676635742,
"KL/mean": -5.558682441711426,
"KL/rejected_KL_mean": -6.308197021484375,
"KL/std": 3.005613327026367,
"epoch": 0.2328042328042328,
"fcm_dpo/beta": 0.24646613001823425,
"fcm_dpo/delta": 0.03169224038720131,
"fcm_dpo/margin": 1.4990254640579224,
"fcm_dpo/q_t": 0.41769248247146606,
"grad_norm": 62.221458435058594,
"learning_rate": 4.745824613468292e-07,
"logits/chosen": 0.21865665912628174,
"logits/rejected": 0.21534715592861176,
"logps/chosen": -64.03170776367188,
"logps/ref_chosen": -59.222537994384766,
"logps/ref_rejected": -64.19131469726562,
"logps/rejected": -70.49951171875,
"loss": 1.1987,
"margin_dpo/margin_mean": 1.499024748802185,
"margin_dpo/margin_std": 3.273876905441284,
"step": 154
},
{
"KL/chosen_KL_mean": -4.674365997314453,
"KL/mean": -5.707864761352539,
"KL/rejected_KL_mean": -6.741355895996094,
"KL/std": 2.986532688140869,
"epoch": 0.23431594860166288,
"fcm_dpo/beta": 0.2417684644460678,
"fcm_dpo/delta": -0.1059052050113678,
"fcm_dpo/margin": 2.0669875144958496,
"fcm_dpo/q_t": 0.39424359798431396,
"grad_norm": 64.11404418945312,
"learning_rate": 4.7399844357283393e-07,
"logits/chosen": 0.20878386497497559,
"logits/rejected": 0.18994128704071045,
"logps/chosen": -73.12905883789062,
"logps/ref_chosen": -68.45469665527344,
"logps/ref_rejected": -77.91763305664062,
"logps/rejected": -84.65899658203125,
"loss": 1.1207,
"margin_dpo/margin_mean": 2.0669875144958496,
"margin_dpo/margin_std": 3.6316781044006348,
"step": 155
},
{
"KL/chosen_KL_mean": -4.716386795043945,
"KL/mean": -5.981790542602539,
"KL/rejected_KL_mean": -7.2471923828125,
"KL/std": 3.2690048217773438,
"epoch": 0.23582766439909297,
"fcm_dpo/beta": 0.23468288779258728,
"fcm_dpo/delta": -0.20591211318969727,
"fcm_dpo/margin": 2.5308122634887695,
"fcm_dpo/q_t": 0.3709757328033447,
"grad_norm": 61.8338737487793,
"learning_rate": 4.7340816008085305e-07,
"logits/chosen": 0.17751815915107727,
"logits/rejected": 0.1345776617527008,
"logps/chosen": -71.9859848022461,
"logps/ref_chosen": -67.26959991455078,
"logps/ref_rejected": -86.95914459228516,
"logps/rejected": -94.20633697509766,
"loss": 0.9984,
"margin_dpo/margin_mean": 2.5308117866516113,
"margin_dpo/margin_std": 3.159181594848633,
"step": 156
},
{
"KL/chosen_KL_mean": -4.443971633911133,
"KL/mean": -5.518423080444336,
"KL/rejected_KL_mean": -6.592872619628906,
"KL/std": 3.4100513458251953,
"epoch": 0.23733938019652306,
"fcm_dpo/beta": 0.22627218067646027,
"fcm_dpo/delta": -0.09378941357135773,
"fcm_dpo/margin": 2.1488969326019287,
"fcm_dpo/q_t": 0.3927996754646301,
"grad_norm": 54.28609085083008,
"learning_rate": 4.728116273823847e-07,
"logits/chosen": 0.1749960035085678,
"logits/rejected": 0.155268132686615,
"logps/chosen": -59.21684646606445,
"logps/ref_chosen": -54.77287292480469,
"logps/ref_rejected": -63.87866973876953,
"logps/rejected": -70.47154235839844,
"loss": 1.0755,
"margin_dpo/margin_mean": 2.1488969326019287,
"margin_dpo/margin_std": 3.115206718444824,
"step": 157
},
{
"KL/chosen_KL_mean": -4.889453887939453,
"KL/mean": -5.9344987869262695,
"KL/rejected_KL_mean": -6.9795379638671875,
"KL/std": 3.2228195667266846,
"epoch": 0.23885109599395313,
"fcm_dpo/beta": 0.2247191220521927,
"fcm_dpo/delta": -0.07326777279376984,
"fcm_dpo/margin": 2.0900797843933105,
"fcm_dpo/q_t": 0.3955712914466858,
"grad_norm": 58.8206672668457,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 0.20392277836799622,
"logits/rejected": 0.17039340734481812,
"logps/chosen": -69.81217193603516,
"logps/ref_chosen": -64.92271423339844,
"logps/ref_rejected": -82.23789978027344,
"logps/rejected": -89.21743774414062,
"loss": 1.081,
"margin_dpo/margin_mean": 2.0900797843933105,
"margin_dpo/margin_std": 3.0950093269348145,
"step": 158
},
{
"KL/chosen_KL_mean": -5.048187255859375,
"KL/mean": -6.26618766784668,
"KL/rejected_KL_mean": -7.484188079833984,
"KL/std": 3.5134024620056152,
"epoch": 0.24036281179138322,
"fcm_dpo/beta": 0.22092238068580627,
"fcm_dpo/delta": -0.14598813652992249,
"fcm_dpo/margin": 2.436002254486084,
"fcm_dpo/q_t": 0.3809051811695099,
"grad_norm": 63.20360565185547,
"learning_rate": 4.715998812855304e-07,
"logits/chosen": 0.21805179119110107,
"logits/rejected": 0.18288499116897583,
"logps/chosen": -62.09518051147461,
"logps/ref_chosen": -57.046993255615234,
"logps/ref_rejected": -73.32441711425781,
"logps/rejected": -80.80860900878906,
"loss": 1.069,
"margin_dpo/margin_mean": 2.436002254486084,
"margin_dpo/margin_std": 3.6891605854034424,
"step": 159
},
{
"KL/chosen_KL_mean": -5.758979797363281,
"KL/mean": -6.87229061126709,
"KL/rejected_KL_mean": -7.985603332519531,
"KL/std": 3.3299851417541504,
"epoch": 0.2418745275888133,
"fcm_dpo/beta": 0.2156430035829544,
"fcm_dpo/delta": -0.08412165194749832,
"fcm_dpo/margin": 2.2266221046447754,
"fcm_dpo/q_t": 0.3995361030101776,
"grad_norm": 51.68805694580078,
"learning_rate": 4.7098470178228755e-07,
"logits/chosen": 0.08146971464157104,
"logits/rejected": 0.04228462278842926,
"logps/chosen": -55.565895080566406,
"logps/ref_chosen": -49.806915283203125,
"logps/ref_rejected": -68.3370132446289,
"logps/rejected": -76.32261657714844,
"loss": 1.1127,
"margin_dpo/margin_mean": 2.2266225814819336,
"margin_dpo/margin_std": 3.848104476928711,
"step": 160
},
{
"KL/chosen_KL_mean": -5.726755142211914,
"KL/mean": -6.963866233825684,
"KL/rejected_KL_mean": -8.200981140136719,
"KL/std": 3.537992477416992,
"epoch": 0.24338624338624337,
"fcm_dpo/beta": 0.21053171157836914,
"fcm_dpo/delta": -0.12737557291984558,
"fcm_dpo/margin": 2.4742283821105957,
"fcm_dpo/q_t": 0.3877101540565491,
"grad_norm": 49.041908264160156,
"learning_rate": 4.703633408618955e-07,
"logits/chosen": 0.19561749696731567,
"logits/rejected": 0.15993468463420868,
"logps/chosen": -58.22724151611328,
"logps/ref_chosen": -52.50048828125,
"logps/ref_rejected": -66.04540252685547,
"logps/rejected": -74.24638366699219,
"loss": 1.0729,
"margin_dpo/margin_mean": 2.474228858947754,
"margin_dpo/margin_std": 3.7947888374328613,
"step": 161
},
{
"KL/chosen_KL_mean": -6.315296173095703,
"KL/mean": -8.033920288085938,
"KL/rejected_KL_mean": -9.752544403076172,
"KL/std": 3.9574198722839355,
"epoch": 0.24489795918367346,
"fcm_dpo/beta": 0.19909542798995972,
"fcm_dpo/delta": -0.3059368133544922,
"fcm_dpo/margin": 3.4372496604919434,
"fcm_dpo/q_t": 0.34949296712875366,
"grad_norm": 48.66947555541992,
"learning_rate": 4.697358159051549e-07,
"logits/chosen": 0.2640194296836853,
"logits/rejected": 0.2174208015203476,
"logps/chosen": -75.78448486328125,
"logps/ref_chosen": -69.46919250488281,
"logps/ref_rejected": -92.00952911376953,
"logps/rejected": -101.76206970214844,
"loss": 0.9383,
"margin_dpo/margin_mean": 3.4372501373291016,
"margin_dpo/margin_std": 3.790897846221924,
"step": 162
},
{
"KL/chosen_KL_mean": -5.789758682250977,
"KL/mean": -7.474250793457031,
"KL/rejected_KL_mean": -9.158744812011719,
"KL/std": 3.744152784347534,
"epoch": 0.24640967498110355,
"fcm_dpo/beta": 0.19080322980880737,
"fcm_dpo/delta": -0.25889816880226135,
"fcm_dpo/margin": 3.368985176086426,
"fcm_dpo/q_t": 0.3616185784339905,
"grad_norm": 46.0795783996582,
"learning_rate": 4.691021444652876e-07,
"logits/chosen": 0.18252956867218018,
"logits/rejected": 0.13875460624694824,
"logps/chosen": -56.403594970703125,
"logps/ref_chosen": -50.613834381103516,
"logps/ref_rejected": -74.62033081054688,
"logps/rejected": -83.77906799316406,
"loss": 0.9952,
"margin_dpo/margin_mean": 3.368985652923584,
"margin_dpo/margin_std": 4.1972150802612305,
"step": 163
},
{
"KL/chosen_KL_mean": -6.431758880615234,
"KL/mean": -8.085715293884277,
"KL/rejected_KL_mean": -9.739673614501953,
"KL/std": 4.0724196434021,
"epoch": 0.24792139077853365,
"fcm_dpo/beta": 0.18106049299240112,
"fcm_dpo/delta": -0.21126613020896912,
"fcm_dpo/margin": 3.307917594909668,
"fcm_dpo/q_t": 0.3714461922645569,
"grad_norm": 43.34768295288086,
"learning_rate": 4.6846234426744624e-07,
"logits/chosen": 0.20137447118759155,
"logits/rejected": 0.140909805893898,
"logps/chosen": -61.279869079589844,
"logps/ref_chosen": -54.848114013671875,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -88.80268859863281,
"loss": 1.0192,
"margin_dpo/margin_mean": 3.307917594909668,
"margin_dpo/margin_std": 4.35736608505249,
"step": 164
},
{
"KL/chosen_KL_mean": -7.033287048339844,
"KL/mean": -8.54382610321045,
"KL/rejected_KL_mean": -10.054359436035156,
"KL/std": 4.177250862121582,
"epoch": 0.2494331065759637,
"fcm_dpo/beta": 0.17607228457927704,
"fcm_dpo/delta": -0.13920900225639343,
"fcm_dpo/margin": 3.0210766792297363,
"fcm_dpo/q_t": 0.38089755177497864,
"grad_norm": 43.28285217285156,
"learning_rate": 4.678164332082175e-07,
"logits/chosen": 0.2319449484348297,
"logits/rejected": 0.17967045307159424,
"logps/chosen": -58.12249755859375,
"logps/ref_chosen": -51.089210510253906,
"logps/ref_rejected": -71.23370361328125,
"logps/rejected": -81.28805541992188,
"loss": 1.036,
"margin_dpo/margin_mean": 3.021076202392578,
"margin_dpo/margin_std": 3.8693056106567383,
"step": 165
},
{
"KL/chosen_KL_mean": -6.964670181274414,
"KL/mean": -8.115772247314453,
"KL/rejected_KL_mean": -9.266876220703125,
"KL/std": 3.8802921772003174,
"epoch": 0.2509448223733938,
"fcm_dpo/beta": 0.17331616580486298,
"fcm_dpo/delta": 0.00084679014980793,
"fcm_dpo/margin": 2.3022074699401855,
"fcm_dpo/q_t": 0.41318219900131226,
"grad_norm": 49.405643463134766,
"learning_rate": 4.6716442935512214e-07,
"logits/chosen": 0.20399600267410278,
"logits/rejected": 0.12173682451248169,
"logps/chosen": -70.15547943115234,
"logps/ref_chosen": -63.19081115722656,
"logps/ref_rejected": -93.8402099609375,
"logps/rejected": -103.10708618164062,
"loss": 1.1346,
"margin_dpo/margin_mean": 2.3022077083587646,
"margin_dpo/margin_std": 3.9645309448242188,
"step": 166
},
{
"KL/chosen_KL_mean": -6.476751327514648,
"KL/mean": -8.268972396850586,
"KL/rejected_KL_mean": -10.06119155883789,
"KL/std": 4.168022155761719,
"epoch": 0.25245653817082386,
"fcm_dpo/beta": 0.16734230518341064,
"fcm_dpo/delta": -0.21473875641822815,
"fcm_dpo/margin": 3.584441661834717,
"fcm_dpo/q_t": 0.36651501059532166,
"grad_norm": 38.167747497558594,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": 0.18713980913162231,
"logits/rejected": 0.15212638676166534,
"logps/chosen": -65.4010238647461,
"logps/ref_chosen": -58.92427062988281,
"logps/ref_rejected": -72.97377014160156,
"logps/rejected": -83.03495788574219,
"loss": 0.9828,
"margin_dpo/margin_mean": 3.5844411849975586,
"margin_dpo/margin_std": 4.134008884429932,
"step": 167
},
{
"KL/chosen_KL_mean": -7.856416702270508,
"KL/mean": -9.048755645751953,
"KL/rejected_KL_mean": -10.241092681884766,
"KL/std": 4.334060192108154,
"epoch": 0.25396825396825395,
"fcm_dpo/beta": 0.16658124327659607,
"fcm_dpo/delta": 0.002872538287192583,
"fcm_dpo/margin": 2.3846707344055176,
"fcm_dpo/q_t": 0.41130581498146057,
"grad_norm": 47.059017181396484,
"learning_rate": 4.6584221638904767e-07,
"logits/chosen": 0.18407779932022095,
"logits/rejected": 0.15321126580238342,
"logps/chosen": -73.50779724121094,
"logps/ref_chosen": -65.65138244628906,
"logps/ref_rejected": -79.71418762207031,
"logps/rejected": -89.95527648925781,
"loss": 1.1225,
"margin_dpo/margin_mean": 2.3846707344055176,
"margin_dpo/margin_std": 3.7981090545654297,
"step": 168
},
{
"KL/chosen_KL_mean": -7.007802963256836,
"KL/mean": -8.737764358520508,
"KL/rejected_KL_mean": -10.467723846435547,
"KL/std": 4.646932125091553,
"epoch": 0.25547996976568405,
"fcm_dpo/beta": 0.16238990426063538,
"fcm_dpo/delta": -0.1715552657842636,
"fcm_dpo/margin": 3.4599173069000244,
"fcm_dpo/q_t": 0.38320809602737427,
"grad_norm": 43.243282318115234,
"learning_rate": 4.651720442612075e-07,
"logits/chosen": 0.24398066103458405,
"logits/rejected": 0.2120930552482605,
"logps/chosen": -68.43367004394531,
"logps/ref_chosen": -61.425865173339844,
"logps/ref_rejected": -76.09590148925781,
"logps/rejected": -86.5636215209961,
"loss": 1.0479,
"margin_dpo/margin_mean": 3.4599175453186035,
"margin_dpo/margin_std": 5.212441444396973,
"step": 169
},
{
"KL/chosen_KL_mean": -7.393749237060547,
"KL/mean": -8.941905975341797,
"KL/rejected_KL_mean": -10.490058898925781,
"KL/std": 4.516660213470459,
"epoch": 0.25699168556311414,
"fcm_dpo/beta": 0.15906530618667603,
"fcm_dpo/delta": -0.0971936583518982,
"fcm_dpo/margin": 3.0963125228881836,
"fcm_dpo/q_t": 0.39114242792129517,
"grad_norm": 36.715030670166016,
"learning_rate": 4.6449585330874425e-07,
"logits/chosen": 0.17781506478786469,
"logits/rejected": 0.17536525428295135,
"logps/chosen": -64.04693603515625,
"logps/ref_chosen": -56.65319061279297,
"logps/ref_rejected": -63.45965576171875,
"logps/rejected": -73.94971466064453,
"loss": 1.0986,
"margin_dpo/margin_mean": 3.0963125228881836,
"margin_dpo/margin_std": 5.024144649505615,
"step": 170
},
{
"KL/chosen_KL_mean": -7.825422286987305,
"KL/mean": -9.71345329284668,
"KL/rejected_KL_mean": -11.601486206054688,
"KL/std": 5.133350372314453,
"epoch": 0.2585034013605442,
"fcm_dpo/beta": 0.1521233767271042,
"fcm_dpo/delta": -0.18862421810626984,
"fcm_dpo/margin": 3.776066303253174,
"fcm_dpo/q_t": 0.3753628432750702,
"grad_norm": 41.07695007324219,
"learning_rate": 4.6381366244617224e-07,
"logits/chosen": 0.26963961124420166,
"logits/rejected": 0.22098302841186523,
"logps/chosen": -71.5601806640625,
"logps/ref_chosen": -63.73476028442383,
"logps/ref_rejected": -78.50328063964844,
"logps/rejected": -90.10476684570312,
"loss": 1.0539,
"margin_dpo/margin_mean": 3.776066780090332,
"margin_dpo/margin_std": 5.530969619750977,
"step": 171
},
{
"KL/chosen_KL_mean": -8.639448165893555,
"KL/mean": -10.513274192810059,
"KL/rejected_KL_mean": -12.387104034423828,
"KL/std": 5.03934383392334,
"epoch": 0.2600151171579743,
"fcm_dpo/beta": 0.14950308203697205,
"fcm_dpo/delta": -0.1697678118944168,
"fcm_dpo/margin": 3.7476518154144287,
"fcm_dpo/q_t": 0.3746216893196106,
"grad_norm": 36.18354415893555,
"learning_rate": 4.631254907558365e-07,
"logits/chosen": 0.2803534269332886,
"logits/rejected": 0.22625818848609924,
"logps/chosen": -60.841209411621094,
"logps/ref_chosen": -52.201759338378906,
"logps/ref_rejected": -82.85285949707031,
"logps/rejected": -95.2399673461914,
"loss": 1.029,
"margin_dpo/margin_mean": 3.747652053833008,
"margin_dpo/margin_std": 4.890772819519043,
"step": 172
},
{
"KL/chosen_KL_mean": -8.68513298034668,
"KL/mean": -10.668136596679688,
"KL/rejected_KL_mean": -12.651142120361328,
"KL/std": 5.370039939880371,
"epoch": 0.2615268329554044,
"fcm_dpo/beta": 0.14140120148658752,
"fcm_dpo/delta": -0.1758767068386078,
"fcm_dpo/margin": 3.966012954711914,
"fcm_dpo/q_t": 0.3847277760505676,
"grad_norm": 34.97652053833008,
"learning_rate": 4.624313574873786e-07,
"logits/chosen": 0.2670894265174866,
"logits/rejected": 0.18332575261592865,
"logps/chosen": -64.11985778808594,
"logps/ref_chosen": -55.434722900390625,
"logps/ref_rejected": -77.81967163085938,
"logps/rejected": -90.47081756591797,
"loss": 1.1001,
"margin_dpo/margin_mean": 3.9660134315490723,
"margin_dpo/margin_std": 6.542463302612305,
"step": 173
},
{
"KL/chosen_KL_mean": -9.613245010375977,
"KL/mean": -11.67945671081543,
"KL/rejected_KL_mean": -13.745670318603516,
"KL/std": 5.322442054748535,
"epoch": 0.26303854875283444,
"fcm_dpo/beta": 0.13782568275928497,
"fcm_dpo/delta": -0.18035998940467834,
"fcm_dpo/margin": 4.132425308227539,
"fcm_dpo/q_t": 0.3784136176109314,
"grad_norm": 37.85453796386719,
"learning_rate": 4.61731282057198e-07,
"logits/chosen": 0.24215909838676453,
"logits/rejected": 0.1789240539073944,
"logps/chosen": -66.78520202636719,
"logps/ref_chosen": -57.17195129394531,
"logps/ref_rejected": -85.47578430175781,
"logps/rejected": -99.22145080566406,
"loss": 1.0505,
"margin_dpo/margin_mean": 4.132425308227539,
"margin_dpo/margin_std": 6.126347541809082,
"step": 174
},
{
"KL/chosen_KL_mean": -9.509271621704102,
"KL/mean": -11.811378479003906,
"KL/rejected_KL_mean": -14.11349105834961,
"KL/std": 5.786849021911621,
"epoch": 0.26455026455026454,
"fcm_dpo/beta": 0.13261333107948303,
"fcm_dpo/delta": -0.2239903062582016,
"fcm_dpo/margin": 4.60421895980835,
"fcm_dpo/q_t": 0.372279554605484,
"grad_norm": 35.7861213684082,
"learning_rate": 4.6102528404790965e-07,
"logits/chosen": 0.28869926929473877,
"logits/rejected": 0.2589804530143738,
"logps/chosen": -77.17489624023438,
"logps/ref_chosen": -67.6656265258789,
"logps/ref_rejected": -84.36766815185547,
"logps/rejected": -98.48115539550781,
"loss": 1.032,
"margin_dpo/margin_mean": 4.604219436645508,
"margin_dpo/margin_std": 6.558835029602051,
"step": 175
},
{
"KL/chosen_KL_mean": -10.641273498535156,
"KL/mean": -12.232093811035156,
"KL/rejected_KL_mean": -13.822917938232422,
"KL/std": 6.113104820251465,
"epoch": 0.2660619803476946,
"fcm_dpo/beta": 0.12980622053146362,
"fcm_dpo/delta": -0.014422226697206497,
"fcm_dpo/margin": 3.1816508769989014,
"fcm_dpo/q_t": 0.41411373019218445,
"grad_norm": 42.98408889770508,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 0.21037542819976807,
"logits/rejected": 0.18521608412265778,
"logps/chosen": -88.50003051757812,
"logps/ref_chosen": -77.8587646484375,
"logps/ref_rejected": -81.08732604980469,
"logps/rejected": -94.91024780273438,
"loss": 1.1777,
"margin_dpo/margin_mean": 3.1816506385803223,
"margin_dpo/margin_std": 6.577012062072754,
"step": 176
},
{
"KL/chosen_KL_mean": -9.549625396728516,
"KL/mean": -12.63090991973877,
"KL/rejected_KL_mean": -15.712196350097656,
"KL/std": 6.261933326721191,
"epoch": 0.2675736961451247,
"fcm_dpo/beta": 0.12264996767044067,
"fcm_dpo/delta": -0.3873238265514374,
"fcm_dpo/margin": 6.16256856918335,
"fcm_dpo/q_t": 0.33408263325691223,
"grad_norm": 39.7496452331543,
"learning_rate": 4.5959559945025183e-07,
"logits/chosen": 0.34224826097488403,
"logits/rejected": 0.25038087368011475,
"logps/chosen": -64.77001953125,
"logps/ref_chosen": -55.22039794921875,
"logps/ref_rejected": -92.54973602294922,
"logps/rejected": -108.26193237304688,
"loss": 0.8891,
"margin_dpo/margin_mean": 6.162568092346191,
"margin_dpo/margin_std": 6.180594444274902,
"step": 177
},
{
"KL/chosen_KL_mean": -10.474905014038086,
"KL/mean": -12.309574127197266,
"KL/rejected_KL_mean": -14.144237518310547,
"KL/std": 5.904752731323242,
"epoch": 0.2690854119425548,
"fcm_dpo/beta": 0.11894647032022476,
"fcm_dpo/delta": -0.040279775857925415,
"fcm_dpo/margin": 3.669332981109619,
"fcm_dpo/q_t": 0.40267473459243774,
"grad_norm": 34.56374740600586,
"learning_rate": 4.588719528532341e-07,
"logits/chosen": 0.21595916152000427,
"logits/rejected": 0.16919106245040894,
"logps/chosen": -71.285400390625,
"logps/ref_chosen": -60.81049346923828,
"logps/ref_rejected": -81.12973022460938,
"logps/rejected": -95.27396392822266,
"loss": 1.1084,
"margin_dpo/margin_mean": 3.669332981109619,
"margin_dpo/margin_std": 5.664151191711426,
"step": 178
},
{
"KL/chosen_KL_mean": -11.489044189453125,
"KL/mean": -13.538455963134766,
"KL/rejected_KL_mean": -15.58786392211914,
"KL/std": 5.988779067993164,
"epoch": 0.2705971277399849,
"fcm_dpo/beta": 0.11831910908222198,
"fcm_dpo/delta": -0.08924552798271179,
"fcm_dpo/margin": 4.09881591796875,
"fcm_dpo/q_t": 0.3954525887966156,
"grad_norm": 33.3326301574707,
"learning_rate": 4.581424636586928e-07,
"logits/chosen": 0.2961423993110657,
"logits/rejected": 0.2790898084640503,
"logps/chosen": -77.16075897216797,
"logps/ref_chosen": -65.67171478271484,
"logps/ref_rejected": -75.32586669921875,
"logps/rejected": -90.91372680664062,
"loss": 1.1081,
"margin_dpo/margin_mean": 4.09881591796875,
"margin_dpo/margin_std": 6.943804740905762,
"step": 179
},
{
"KL/chosen_KL_mean": -9.654487609863281,
"KL/mean": -11.612443923950195,
"KL/rejected_KL_mean": -13.570402145385742,
"KL/std": 6.35772705078125,
"epoch": 0.272108843537415,
"fcm_dpo/beta": 0.11695965379476547,
"fcm_dpo/delta": -0.060777340084314346,
"fcm_dpo/margin": 3.9159162044525146,
"fcm_dpo/q_t": 0.4039306044578552,
"grad_norm": 32.40278625488281,
"learning_rate": 4.5740715227200897e-07,
"logits/chosen": 0.12212781608104706,
"logits/rejected": 0.1029723584651947,
"logps/chosen": -66.33729553222656,
"logps/ref_chosen": -56.68280792236328,
"logps/ref_rejected": -64.94414520263672,
"logps/rejected": -78.5145492553711,
"loss": 1.1464,
"margin_dpo/margin_mean": 3.9159162044525146,
"margin_dpo/margin_std": 7.415275573730469,
"step": 180
},
{
"KL/chosen_KL_mean": -9.152425765991211,
"KL/mean": -12.092589378356934,
"KL/rejected_KL_mean": -15.032752990722656,
"KL/std": 7.016723155975342,
"epoch": 0.273620559334845,
"fcm_dpo/beta": 0.11149968206882477,
"fcm_dpo/delta": -0.27399927377700806,
"fcm_dpo/margin": 5.88032341003418,
"fcm_dpo/q_t": 0.3537420630455017,
"grad_norm": 29.230892181396484,
"learning_rate": 4.566660392614228e-07,
"logits/chosen": 0.2703360319137573,
"logits/rejected": 0.232833594083786,
"logps/chosen": -69.92847442626953,
"logps/ref_chosen": -60.77604675292969,
"logps/ref_rejected": -83.98361206054688,
"logps/rejected": -99.01637268066406,
"loss": 0.9367,
"margin_dpo/margin_mean": 5.88032341003418,
"margin_dpo/margin_std": 6.134858131408691,
"step": 181
},
{
"KL/chosen_KL_mean": -10.694526672363281,
"KL/mean": -13.846564292907715,
"KL/rejected_KL_mean": -16.99859619140625,
"KL/std": 7.271864891052246,
"epoch": 0.2751322751322751,
"fcm_dpo/beta": 0.10512416809797287,
"fcm_dpo/delta": -0.28286096453666687,
"fcm_dpo/margin": 6.304077625274658,
"fcm_dpo/q_t": 0.35856950283050537,
"grad_norm": 29.052644729614258,
"learning_rate": 4.5591914535745817e-07,
"logits/chosen": 0.26619184017181396,
"logits/rejected": 0.1893734633922577,
"logps/chosen": -70.94831085205078,
"logps/ref_chosen": -60.2537841796875,
"logps/ref_rejected": -89.7706298828125,
"logps/rejected": -106.76922607421875,
"loss": 0.9904,
"margin_dpo/margin_mean": 6.304078102111816,
"margin_dpo/margin_std": 8.163893699645996,
"step": 182
},
{
"KL/chosen_KL_mean": -12.871854782104492,
"KL/mean": -14.117193222045898,
"KL/rejected_KL_mean": -15.362525939941406,
"KL/std": 7.093344688415527,
"epoch": 0.2766439909297052,
"fcm_dpo/beta": 0.10492784529924393,
"fcm_dpo/delta": 0.043590083718299866,
"fcm_dpo/margin": 2.4906742572784424,
"fcm_dpo/q_t": 0.44075942039489746,
"grad_norm": 31.582111358642578,
"learning_rate": 4.551664914523433e-07,
"logits/chosen": 0.2629430890083313,
"logits/rejected": 0.241647869348526,
"logps/chosen": -74.63327026367188,
"logps/ref_chosen": -61.76142120361328,
"logps/ref_rejected": -72.54627990722656,
"logps/rejected": -87.9088134765625,
"loss": 1.2568,
"margin_dpo/margin_mean": 2.4906740188598633,
"margin_dpo/margin_std": 6.654599666595459,
"step": 183
},
{
"KL/chosen_KL_mean": -9.920059204101562,
"KL/mean": -12.445627212524414,
"KL/rejected_KL_mean": -14.971195220947266,
"KL/std": 6.375822067260742,
"epoch": 0.2781557067271353,
"fcm_dpo/beta": 0.10218354314565659,
"fcm_dpo/delta": -0.12475556880235672,
"fcm_dpo/margin": 5.051133155822754,
"fcm_dpo/q_t": 0.3857799768447876,
"grad_norm": 24.51209259033203,
"learning_rate": 4.544080985994258e-07,
"logits/chosen": 0.33628761768341064,
"logits/rejected": 0.27536916732788086,
"logps/chosen": -56.760780334472656,
"logps/ref_chosen": -46.840721130371094,
"logps/ref_rejected": -69.3609390258789,
"logps/rejected": -84.33213806152344,
"loss": 1.0338,
"margin_dpo/margin_mean": 5.0511322021484375,
"margin_dpo/margin_std": 6.313591957092285,
"step": 184
},
{
"KL/chosen_KL_mean": -11.451126098632812,
"KL/mean": -14.077653884887695,
"KL/rejected_KL_mean": -16.704174041748047,
"KL/std": 7.328970909118652,
"epoch": 0.2796674225245654,
"fcm_dpo/beta": 0.09977151453495026,
"fcm_dpo/delta": -0.1324116587638855,
"fcm_dpo/margin": 5.253050804138184,
"fcm_dpo/q_t": 0.39040905237197876,
"grad_norm": 25.837413787841797,
"learning_rate": 4.5364398801258394e-07,
"logits/chosen": 0.27261149883270264,
"logits/rejected": 0.22698205709457397,
"logps/chosen": -63.77226638793945,
"logps/ref_chosen": -52.32114028930664,
"logps/ref_rejected": -68.3885726928711,
"logps/rejected": -85.09274291992188,
"loss": 1.1098,
"margin_dpo/margin_mean": 5.253050804138184,
"margin_dpo/margin_std": 8.9921293258667,
"step": 185
},
{
"KL/chosen_KL_mean": -10.808832168579102,
"KL/mean": -13.7213773727417,
"KL/rejected_KL_mean": -16.63391876220703,
"KL/std": 7.43798303604126,
"epoch": 0.2811791383219955,
"fcm_dpo/beta": 0.09699708223342896,
"fcm_dpo/delta": -0.1751311719417572,
"fcm_dpo/margin": 5.825077056884766,
"fcm_dpo/q_t": 0.38233405351638794,
"grad_norm": 30.920795440673828,
"learning_rate": 4.5287418106563354e-07,
"logits/chosen": 0.21480430662631989,
"logits/rejected": 0.1741763800382614,
"logps/chosen": -78.22895812988281,
"logps/ref_chosen": -67.42012786865234,
"logps/ref_rejected": -82.50968933105469,
"logps/rejected": -99.14360809326172,
"loss": 1.0727,
"margin_dpo/margin_mean": 5.825077056884766,
"margin_dpo/margin_std": 9.249723434448242,
"step": 186
},
{
"KL/chosen_KL_mean": -12.215843200683594,
"KL/mean": -14.944025993347168,
"KL/rejected_KL_mean": -17.672218322753906,
"KL/std": 7.700148105621338,
"epoch": 0.28269085411942557,
"fcm_dpo/beta": 0.09425411373376846,
"fcm_dpo/delta": -0.12099070847034454,
"fcm_dpo/margin": 5.456380844116211,
"fcm_dpo/q_t": 0.3869907855987549,
"grad_norm": 30.69752311706543,
"learning_rate": 4.520986992917297e-07,
"logits/chosen": 0.26730459928512573,
"logits/rejected": 0.21251502633094788,
"logps/chosen": -87.7413330078125,
"logps/ref_chosen": -75.52549743652344,
"logps/ref_rejected": -94.76289367675781,
"logps/rejected": -112.43511199951172,
"loss": 1.0841,
"margin_dpo/margin_mean": 5.456380844116211,
"margin_dpo/margin_std": 8.528963088989258,
"step": 187
},
{
"KL/chosen_KL_mean": -11.301044464111328,
"KL/mean": -14.108506202697754,
"KL/rejected_KL_mean": -16.915966033935547,
"KL/std": 8.115912437438965,
"epoch": 0.2842025699168556,
"fcm_dpo/beta": 0.0922112762928009,
"fcm_dpo/delta": -0.12419946491718292,
"fcm_dpo/margin": 5.61491584777832,
"fcm_dpo/q_t": 0.3875483572483063,
"grad_norm": 29.468542098999023,
"learning_rate": 4.5131756438276466e-07,
"logits/chosen": 0.293914258480072,
"logits/rejected": 0.25059744715690613,
"logps/chosen": -82.82437133789062,
"logps/ref_chosen": -71.52333068847656,
"logps/ref_rejected": -78.29949951171875,
"logps/rejected": -95.21546936035156,
"loss": 1.0788,
"margin_dpo/margin_mean": 5.61491584777832,
"margin_dpo/margin_std": 8.844915390014648,
"step": 188
},
{
"KL/chosen_KL_mean": -10.672468185424805,
"KL/mean": -13.489643096923828,
"KL/rejected_KL_mean": -16.306812286376953,
"KL/std": 7.987409591674805,
"epoch": 0.2857142857142857,
"fcm_dpo/beta": 0.08913347870111465,
"fcm_dpo/delta": -0.11137335002422333,
"fcm_dpo/margin": 5.6343464851379395,
"fcm_dpo/q_t": 0.38765114545822144,
"grad_norm": 27.814828872680664,
"learning_rate": 4.5053079818876096e-07,
"logits/chosen": 0.3022003769874573,
"logits/rejected": 0.31391239166259766,
"logps/chosen": -82.84873962402344,
"logps/ref_chosen": -72.17626953125,
"logps/ref_rejected": -75.26313781738281,
"logps/rejected": -91.5699462890625,
"loss": 1.0789,
"margin_dpo/margin_mean": 5.634347438812256,
"margin_dpo/margin_std": 8.217448234558105,
"step": 189
},
{
"KL/chosen_KL_mean": -10.488744735717773,
"KL/mean": -14.275890350341797,
"KL/rejected_KL_mean": -18.063034057617188,
"KL/std": 8.285619735717773,
"epoch": 0.2872260015117158,
"fcm_dpo/beta": 0.08610306680202484,
"fcm_dpo/delta": -0.2697725296020508,
"fcm_dpo/margin": 7.574289321899414,
"fcm_dpo/q_t": 0.35756736993789673,
"grad_norm": 27.436847686767578,
"learning_rate": 4.4973842271726024e-07,
"logits/chosen": 0.34051239490509033,
"logits/rejected": 0.20306336879730225,
"logps/chosen": -65.1130142211914,
"logps/ref_chosen": -54.624271392822266,
"logps/ref_rejected": -101.47068786621094,
"logps/rejected": -119.53372192382812,
"loss": 0.9625,
"margin_dpo/margin_mean": 7.574289321899414,
"margin_dpo/margin_std": 8.801969528198242,
"step": 190
},
{
"KL/chosen_KL_mean": -12.875495910644531,
"KL/mean": -15.73067855834961,
"KL/rejected_KL_mean": -18.585865020751953,
"KL/std": 8.43212890625,
"epoch": 0.2887377173091459,
"fcm_dpo/beta": 0.08393500745296478,
"fcm_dpo/delta": -0.08323581516742706,
"fcm_dpo/margin": 5.7103753089904785,
"fcm_dpo/q_t": 0.39323675632476807,
"grad_norm": 28.250213623046875,
"learning_rate": 4.48940460132708e-07,
"logits/chosen": 0.345758318901062,
"logits/rejected": 0.3188805878162384,
"logps/chosen": -85.80801391601562,
"logps/ref_chosen": -72.93251037597656,
"logps/ref_rejected": -89.95103454589844,
"logps/rejected": -108.53689575195312,
"loss": 1.1,
"margin_dpo/margin_mean": 5.71037483215332,
"margin_dpo/margin_std": 9.162508010864258,
"step": 191
},
{
"KL/chosen_KL_mean": -13.147985458374023,
"KL/mean": -15.195459365844727,
"KL/rejected_KL_mean": -17.24292755126953,
"KL/std": 8.910942077636719,
"epoch": 0.29024943310657597,
"fcm_dpo/beta": 0.08443897217512131,
"fcm_dpo/delta": 0.05585295706987381,
"fcm_dpo/margin": 4.094945430755615,
"fcm_dpo/q_t": 0.4225963354110718,
"grad_norm": 22.723339080810547,
"learning_rate": 4.481369327558329e-07,
"logits/chosen": 0.33716925978660583,
"logits/rejected": 0.31166955828666687,
"logps/chosen": -67.14910888671875,
"logps/ref_chosen": -54.001121520996094,
"logps/ref_rejected": -63.531551361083984,
"logps/rejected": -80.77447509765625,
"loss": 1.1793,
"margin_dpo/margin_mean": 4.094945430755615,
"margin_dpo/margin_std": 8.139238357543945,
"step": 192
},
{
"KL/chosen_KL_mean": -11.603879928588867,
"KL/mean": -15.223213195800781,
"KL/rejected_KL_mean": -18.842544555664062,
"KL/std": 8.593984603881836,
"epoch": 0.29176114890400606,
"fcm_dpo/beta": 0.08154302835464478,
"fcm_dpo/delta": -0.2036181539297104,
"fcm_dpo/margin": 7.238663196563721,
"fcm_dpo/q_t": 0.37004029750823975,
"grad_norm": 21.685712814331055,
"learning_rate": 4.47327863063023e-07,
"logits/chosen": 0.25180673599243164,
"logits/rejected": 0.22910341620445251,
"logps/chosen": -68.35315704345703,
"logps/ref_chosen": -56.74927520751953,
"logps/ref_rejected": -58.80629348754883,
"logps/rejected": -77.64883422851562,
"loss": 0.9949,
"margin_dpo/margin_mean": 7.238663196563721,
"margin_dpo/margin_std": 8.828506469726562,
"step": 193
},
{
"KL/chosen_KL_mean": -11.626914978027344,
"KL/mean": -14.529808044433594,
"KL/rejected_KL_mean": -17.43271255493164,
"KL/std": 8.440224647521973,
"epoch": 0.29327286470143615,
"fcm_dpo/beta": 0.07977467030286789,
"fcm_dpo/delta": -0.06739965826272964,
"fcm_dpo/margin": 5.805790901184082,
"fcm_dpo/q_t": 0.39755988121032715,
"grad_norm": 22.833391189575195,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 0.32282212376594543,
"logits/rejected": 0.2938089370727539,
"logps/chosen": -68.27635192871094,
"logps/ref_chosen": -56.64944076538086,
"logps/ref_rejected": -69.98954772949219,
"logps/rejected": -87.42225646972656,
"loss": 1.1424,
"margin_dpo/margin_mean": 5.80579137802124,
"margin_dpo/margin_std": 10.607912063598633,
"step": 194
},
{
"KL/chosen_KL_mean": -13.355850219726562,
"KL/mean": -16.958824157714844,
"KL/rejected_KL_mean": -20.561798095703125,
"KL/std": 9.667953491210938,
"epoch": 0.2947845804988662,
"fcm_dpo/beta": 0.07755885273218155,
"fcm_dpo/delta": -0.16988505423069,
"fcm_dpo/margin": 7.205946922302246,
"fcm_dpo/q_t": 0.37830594182014465,
"grad_norm": 23.50065803527832,
"learning_rate": 4.4569318740967043e-07,
"logits/chosen": 0.2331201732158661,
"logits/rejected": 0.23444901406764984,
"logps/chosen": -83.765625,
"logps/ref_chosen": -70.40977478027344,
"logps/ref_rejected": -74.39448547363281,
"logps/rejected": -94.95628356933594,
"loss": 1.031,
"margin_dpo/margin_mean": 7.205946922302246,
"margin_dpo/margin_std": 9.715158462524414,
"step": 195
},
{
"KL/chosen_KL_mean": -12.422384262084961,
"KL/mean": -15.504018783569336,
"KL/rejected_KL_mean": -18.585647583007812,
"KL/std": 9.727346420288086,
"epoch": 0.2962962962962963,
"fcm_dpo/beta": 0.07681725919246674,
"fcm_dpo/delta": -0.07722026109695435,
"fcm_dpo/margin": 6.163267135620117,
"fcm_dpo/q_t": 0.3931337893009186,
"grad_norm": 21.98168182373047,
"learning_rate": 4.448676271745197e-07,
"logits/chosen": 0.3079131543636322,
"logits/rejected": 0.2679440379142761,
"logps/chosen": -71.64996337890625,
"logps/ref_chosen": -59.227577209472656,
"logps/ref_rejected": -83.54757690429688,
"logps/rejected": -102.13322448730469,
"loss": 1.0837,
"margin_dpo/margin_mean": 6.163267135620117,
"margin_dpo/margin_std": 9.117142677307129,
"step": 196
},
{
"KL/chosen_KL_mean": -11.057069778442383,
"KL/mean": -14.910604476928711,
"KL/rejected_KL_mean": -18.764137268066406,
"KL/std": 9.970842361450195,
"epoch": 0.29780801209372637,
"fcm_dpo/beta": 0.07460330426692963,
"fcm_dpo/delta": -0.1853725016117096,
"fcm_dpo/margin": 7.7070698738098145,
"fcm_dpo/q_t": 0.3767518401145935,
"grad_norm": 22.034433364868164,
"learning_rate": 4.440366160729392e-07,
"logits/chosen": 0.4055364727973938,
"logits/rejected": 0.35669881105422974,
"logps/chosen": -62.58620071411133,
"logps/ref_chosen": -51.52912902832031,
"logps/ref_rejected": -73.70631408691406,
"logps/rejected": -92.47044372558594,
"loss": 1.0843,
"margin_dpo/margin_mean": 7.707070350646973,
"margin_dpo/margin_std": 12.205463409423828,
"step": 197
},
{
"KL/chosen_KL_mean": -11.597648620605469,
"KL/mean": -15.803947448730469,
"KL/rejected_KL_mean": -20.01025390625,
"KL/std": 9.128683090209961,
"epoch": 0.29931972789115646,
"fcm_dpo/beta": 0.07150924205780029,
"fcm_dpo/delta": -0.21394206583499908,
"fcm_dpo/margin": 8.412599563598633,
"fcm_dpo/q_t": 0.3645872473716736,
"grad_norm": 21.250316619873047,
"learning_rate": 4.432001773500957e-07,
"logits/chosen": 0.35114845633506775,
"logits/rejected": 0.3125147223472595,
"logps/chosen": -71.38032531738281,
"logps/ref_chosen": -59.78268051147461,
"logps/ref_rejected": -72.24533081054688,
"logps/rejected": -92.25558471679688,
"loss": 0.9756,
"margin_dpo/margin_mean": 8.412599563598633,
"margin_dpo/margin_std": 9.298688888549805,
"step": 198
},
{
"KL/chosen_KL_mean": -13.459989547729492,
"KL/mean": -16.656028747558594,
"KL/rejected_KL_mean": -19.852069854736328,
"KL/std": 9.94611930847168,
"epoch": 0.30083144368858655,
"fcm_dpo/beta": 0.07043890655040741,
"fcm_dpo/delta": -0.05417756736278534,
"fcm_dpo/margin": 6.392084121704102,
"fcm_dpo/q_t": 0.40116173028945923,
"grad_norm": 22.40580940246582,
"learning_rate": 4.4235833440297856e-07,
"logits/chosen": 0.32230302691459656,
"logits/rejected": 0.23618870973587036,
"logps/chosen": -69.84675598144531,
"logps/ref_chosen": -56.38677215576172,
"logps/ref_rejected": -74.56779479980469,
"logps/rejected": -94.41986846923828,
"loss": 1.1609,
"margin_dpo/margin_mean": 6.392083168029785,
"margin_dpo/margin_std": 12.043109893798828,
"step": 199
},
{
"KL/chosen_KL_mean": -10.720462799072266,
"KL/mean": -15.326833724975586,
"KL/rejected_KL_mean": -19.933212280273438,
"KL/std": 10.655014038085938,
"epoch": 0.30234315948601664,
"fcm_dpo/beta": 0.06677936017513275,
"fcm_dpo/delta": -0.23146937787532806,
"fcm_dpo/margin": 9.212747573852539,
"fcm_dpo/q_t": 0.3676430583000183,
"grad_norm": 20.76715660095215,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": 0.36081990599632263,
"logits/rejected": 0.2913385331630707,
"logps/chosen": -68.54478454589844,
"logps/ref_chosen": -57.82432556152344,
"logps/ref_rejected": -89.28246307373047,
"logps/rejected": -109.2156753540039,
"loss": 1.0252,
"margin_dpo/margin_mean": 9.212747573852539,
"margin_dpo/margin_std": 12.840559005737305,
"step": 200
},
{
"KL/chosen_KL_mean": -14.078449249267578,
"KL/mean": -17.949726104736328,
"KL/rejected_KL_mean": -21.821002960205078,
"KL/std": 11.107925415039062,
"epoch": 0.30385487528344673,
"fcm_dpo/beta": 0.06507912278175354,
"fcm_dpo/delta": -0.11002416908740997,
"fcm_dpo/margin": 7.742550849914551,
"fcm_dpo/q_t": 0.39129719138145447,
"grad_norm": 22.2618465423584,
"learning_rate": 4.4065853017905953e-07,
"logits/chosen": 0.38354283571243286,
"logits/rejected": 0.3394392728805542,
"logps/chosen": -73.07820892333984,
"logps/ref_chosen": -58.999759674072266,
"logps/ref_rejected": -84.67575073242188,
"logps/rejected": -106.49674987792969,
"loss": 1.0826,
"margin_dpo/margin_mean": 7.742550849914551,
"margin_dpo/margin_std": 12.054117202758789,
"step": 201
},
{
"KL/chosen_KL_mean": -11.895166397094727,
"KL/mean": -16.40951156616211,
"KL/rejected_KL_mean": -20.923843383789062,
"KL/std": 10.968514442443848,
"epoch": 0.30536659108087677,
"fcm_dpo/beta": 0.0634693130850792,
"fcm_dpo/delta": -0.1831568330526352,
"fcm_dpo/margin": 9.028682708740234,
"fcm_dpo/q_t": 0.37187156081199646,
"grad_norm": 19.839948654174805,
"learning_rate": 4.3980061644943575e-07,
"logits/chosen": 0.30112141370773315,
"logits/rejected": 0.2322790026664734,
"logps/chosen": -59.555816650390625,
"logps/ref_chosen": -47.660648345947266,
"logps/ref_rejected": -73.63249969482422,
"logps/rejected": -94.55634307861328,
"loss": 1.0252,
"margin_dpo/margin_mean": 9.028682708740234,
"margin_dpo/margin_std": 11.844956398010254,
"step": 202
},
{
"KL/chosen_KL_mean": -13.715679168701172,
"KL/mean": -17.84027862548828,
"KL/rejected_KL_mean": -21.964881896972656,
"KL/std": 11.133407592773438,
"epoch": 0.30687830687830686,
"fcm_dpo/beta": 0.06169985234737396,
"fcm_dpo/delta": -0.11466041207313538,
"fcm_dpo/margin": 8.249202728271484,
"fcm_dpo/q_t": 0.3896998167037964,
"grad_norm": 22.613859176635742,
"learning_rate": 4.3893739358856455e-07,
"logits/chosen": 0.40282106399536133,
"logits/rejected": 0.3293677866458893,
"logps/chosen": -76.04121398925781,
"logps/ref_chosen": -62.32553482055664,
"logps/ref_rejected": -99.37226104736328,
"logps/rejected": -121.33714294433594,
"loss": 1.0681,
"margin_dpo/margin_mean": 8.2492036819458,
"margin_dpo/margin_std": 12.2598876953125,
"step": 203
},
{
"KL/chosen_KL_mean": -13.306692123413086,
"KL/mean": -17.573535919189453,
"KL/rejected_KL_mean": -21.840377807617188,
"KL/std": 11.92041015625,
"epoch": 0.30839002267573695,
"fcm_dpo/beta": 0.05942771956324577,
"fcm_dpo/delta": -0.1170601025223732,
"fcm_dpo/margin": 8.533686637878418,
"fcm_dpo/q_t": 0.38851553201675415,
"grad_norm": 19.43342399597168,
"learning_rate": 4.380688857426449e-07,
"logits/chosen": 0.3297405540943146,
"logits/rejected": 0.2622869312763214,
"logps/chosen": -63.93600845336914,
"logps/ref_chosen": -50.62931823730469,
"logps/ref_rejected": -66.60475158691406,
"logps/rejected": -88.44512939453125,
"loss": 1.0697,
"margin_dpo/margin_mean": 8.533686637878418,
"margin_dpo/margin_std": 12.159065246582031,
"step": 204
},
{
"KL/chosen_KL_mean": -14.52587890625,
"KL/mean": -18.86947250366211,
"KL/rejected_KL_mean": -23.213069915771484,
"KL/std": 11.899272918701172,
"epoch": 0.30990173847316704,
"fcm_dpo/beta": 0.058575842529535294,
"fcm_dpo/delta": -0.11523065716028214,
"fcm_dpo/margin": 8.687185287475586,
"fcm_dpo/q_t": 0.38943153619766235,
"grad_norm": 24.11298179626465,
"learning_rate": 4.3719511720570814e-07,
"logits/chosen": 0.3770410418510437,
"logits/rejected": 0.31624114513397217,
"logps/chosen": -84.8820571899414,
"logps/ref_chosen": -70.3561782836914,
"logps/ref_rejected": -93.39848327636719,
"logps/rejected": -116.6115493774414,
"loss": 1.0883,
"margin_dpo/margin_mean": 8.687185287475586,
"margin_dpo/margin_std": 13.842249870300293,
"step": 205
},
{
"KL/chosen_KL_mean": -15.22751235961914,
"KL/mean": -18.422212600708008,
"KL/rejected_KL_mean": -21.616912841796875,
"KL/std": 12.097978591918945,
"epoch": 0.31141345427059713,
"fcm_dpo/beta": 0.0589301735162735,
"fcm_dpo/delta": 0.023669734597206116,
"fcm_dpo/margin": 6.389399528503418,
"fcm_dpo/q_t": 0.42056867480278015,
"grad_norm": 21.71643829345703,
"learning_rate": 4.363161124189387e-07,
"logits/chosen": 0.35867053270339966,
"logits/rejected": 0.34352797269821167,
"logps/chosen": -82.87299346923828,
"logps/ref_chosen": -67.64547729492188,
"logps/ref_rejected": -79.89584350585938,
"logps/rejected": -101.51275634765625,
"loss": 1.218,
"margin_dpo/margin_mean": 6.389400482177734,
"margin_dpo/margin_std": 14.692683219909668,
"step": 206
},
{
"KL/chosen_KL_mean": -17.164901733398438,
"KL/mean": -21.810579299926758,
"KL/rejected_KL_mean": -26.456253051757812,
"KL/std": 12.925216674804688,
"epoch": 0.3129251700680272,
"fcm_dpo/beta": 0.05732639506459236,
"fcm_dpo/delta": -0.14058543741703033,
"fcm_dpo/margin": 9.291353225708008,
"fcm_dpo/q_t": 0.3860167860984802,
"grad_norm": 19.448396682739258,
"learning_rate": 4.3543189596998986e-07,
"logits/chosen": 0.3271971344947815,
"logits/rejected": 0.26136887073516846,
"logps/chosen": -84.82909393310547,
"logps/ref_chosen": -67.66419219970703,
"logps/ref_rejected": -85.10249328613281,
"logps/rejected": -111.55874633789062,
"loss": 1.0525,
"margin_dpo/margin_mean": 9.291353225708008,
"margin_dpo/margin_std": 13.424016952514648,
"step": 207
},
{
"KL/chosen_KL_mean": -13.356035232543945,
"KL/mean": -16.117992401123047,
"KL/rejected_KL_mean": -18.87995147705078,
"KL/std": 11.92393684387207,
"epoch": 0.3144368858654573,
"fcm_dpo/beta": 0.05777502804994583,
"fcm_dpo/delta": 0.08351733535528183,
"fcm_dpo/margin": 5.5239152908325195,
"fcm_dpo/q_t": 0.4302397668361664,
"grad_norm": 22.242469787597656,
"learning_rate": 4.3454249259229664e-07,
"logits/chosen": 0.3382790684700012,
"logits/rejected": 0.31333252787590027,
"logps/chosen": -71.0877456665039,
"logps/ref_chosen": -57.731712341308594,
"logps/ref_rejected": -74.19276428222656,
"logps/rejected": -93.07271575927734,
"loss": 1.2291,
"margin_dpo/margin_mean": 5.5239152908325195,
"margin_dpo/margin_std": 13.161931991577148,
"step": 208
},
{
"KL/chosen_KL_mean": -13.599538803100586,
"KL/mean": -19.107500076293945,
"KL/rejected_KL_mean": -24.615467071533203,
"KL/std": 13.729157447814941,
"epoch": 0.31594860166288735,
"fcm_dpo/beta": 0.056135572493076324,
"fcm_dpo/delta": -0.23248827457427979,
"fcm_dpo/margin": 11.0159273147583,
"fcm_dpo/q_t": 0.36714643239974976,
"grad_norm": 21.617216110229492,
"learning_rate": 4.336479271643833e-07,
"logits/chosen": 0.318255215883255,
"logits/rejected": 0.263971209526062,
"logps/chosen": -82.14962005615234,
"logps/ref_chosen": -68.55007934570312,
"logps/ref_rejected": -87.90541076660156,
"logps/rejected": -112.52088165283203,
"loss": 1.0376,
"margin_dpo/margin_mean": 11.015928268432617,
"margin_dpo/margin_std": 16.050796508789062,
"step": 209
},
{
"KL/chosen_KL_mean": -13.569158554077148,
"KL/mean": -18.891977310180664,
"KL/rejected_KL_mean": -24.214797973632812,
"KL/std": 14.49488639831543,
"epoch": 0.31746031746031744,
"fcm_dpo/beta": 0.05405519902706146,
"fcm_dpo/delta": -0.1857774555683136,
"fcm_dpo/margin": 10.645635604858398,
"fcm_dpo/q_t": 0.37807339429855347,
"grad_norm": 19.007171630859375,
"learning_rate": 4.327482247091679e-07,
"logits/chosen": 0.43442434072494507,
"logits/rejected": 0.3393166661262512,
"logps/chosen": -70.83743286132812,
"logps/ref_chosen": -57.268272399902344,
"logps/ref_rejected": -85.72807312011719,
"logps/rejected": -109.94287109375,
"loss": 1.0486,
"margin_dpo/margin_mean": 10.645635604858398,
"margin_dpo/margin_std": 15.562570571899414,
"step": 210
},
{
"KL/chosen_KL_mean": -11.815755844116211,
"KL/mean": -16.830108642578125,
"KL/rejected_KL_mean": -21.844467163085938,
"KL/std": 13.361391067504883,
"epoch": 0.31897203325774753,
"fcm_dpo/beta": 0.052563317120075226,
"fcm_dpo/delta": -0.13404600322246552,
"fcm_dpo/margin": 10.028705596923828,
"fcm_dpo/q_t": 0.38719442486763,
"grad_norm": 21.34895896911621,
"learning_rate": 4.3184341039326217e-07,
"logits/chosen": 0.4384046792984009,
"logits/rejected": 0.3472185730934143,
"logps/chosen": -65.45646667480469,
"logps/ref_chosen": -53.640708923339844,
"logps/ref_rejected": -93.0387954711914,
"logps/rejected": -114.88326263427734,
"loss": 1.0594,
"margin_dpo/margin_mean": 10.028705596923828,
"margin_dpo/margin_std": 14.62277603149414,
"step": 211
},
{
"KL/chosen_KL_mean": -14.026079177856445,
"KL/mean": -19.678508758544922,
"KL/rejected_KL_mean": -25.33094024658203,
"KL/std": 14.330770492553711,
"epoch": 0.3204837490551776,
"fcm_dpo/beta": 0.05051539093255997,
"fcm_dpo/delta": -0.18164601922035217,
"fcm_dpo/margin": 11.304863929748535,
"fcm_dpo/q_t": 0.3743218183517456,
"grad_norm": 17.196802139282227,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 0.4241589605808258,
"logits/rejected": 0.35381272435188293,
"logps/chosen": -71.392822265625,
"logps/ref_chosen": -57.36674499511719,
"logps/ref_rejected": -79.89643096923828,
"logps/rejected": -105.22737121582031,
"loss": 1.0368,
"margin_dpo/margin_mean": 11.304863929748535,
"margin_dpo/margin_std": 15.752253532409668,
"step": 212
},
{
"KL/chosen_KL_mean": -9.737762451171875,
"KL/mean": -16.1937255859375,
"KL/rejected_KL_mean": -22.64969253540039,
"KL/std": 14.696576118469238,
"epoch": 0.3219954648526077,
"fcm_dpo/beta": 0.04813341051340103,
"fcm_dpo/delta": -0.23775681853294373,
"fcm_dpo/margin": 12.911933898925781,
"fcm_dpo/q_t": 0.36463862657546997,
"grad_norm": 15.5367431640625,
"learning_rate": 4.3001854756006724e-07,
"logits/chosen": 0.4189993739128113,
"logits/rejected": 0.3951151371002197,
"logps/chosen": -74.95887756347656,
"logps/ref_chosen": -65.22111511230469,
"logps/ref_rejected": -80.1810302734375,
"logps/rejected": -102.83071899414062,
"loss": 1.0028,
"margin_dpo/margin_mean": 12.911933898925781,
"margin_dpo/margin_std": 16.8407039642334,
"step": 213
},
{
"KL/chosen_KL_mean": -11.835357666015625,
"KL/mean": -18.098674774169922,
"KL/rejected_KL_mean": -24.361989974975586,
"KL/std": 14.943780899047852,
"epoch": 0.3235071806500378,
"fcm_dpo/beta": 0.046699561178684235,
"fcm_dpo/delta": -0.19606538116931915,
"fcm_dpo/margin": 12.526634216308594,
"fcm_dpo/q_t": 0.3739134669303894,
"grad_norm": 20.22979164123535,
"learning_rate": 4.290985500881143e-07,
"logits/chosen": 0.27848193049430847,
"logits/rejected": 0.2563505172729492,
"logps/chosen": -73.127685546875,
"logps/ref_chosen": -61.292327880859375,
"logps/ref_rejected": -67.69841003417969,
"logps/rejected": -92.06039428710938,
"loss": 1.0303,
"margin_dpo/margin_mean": 12.52663516998291,
"margin_dpo/margin_std": 17.092254638671875,
"step": 214
},
{
"KL/chosen_KL_mean": -15.029420852661133,
"KL/mean": -21.492494583129883,
"KL/rejected_KL_mean": -27.95557403564453,
"KL/std": 15.688613891601562,
"epoch": 0.3250188964474679,
"fcm_dpo/beta": 0.044977862387895584,
"fcm_dpo/delta": -0.19229058921337128,
"fcm_dpo/margin": 12.926143646240234,
"fcm_dpo/q_t": 0.3759158253669739,
"grad_norm": 17.81093406677246,
"learning_rate": 4.281735428447157e-07,
"logits/chosen": 0.31165915727615356,
"logits/rejected": 0.20928305387496948,
"logps/chosen": -78.8985595703125,
"logps/ref_chosen": -63.869136810302734,
"logps/ref_rejected": -98.7657241821289,
"logps/rejected": -126.72129821777344,
"loss": 1.0403,
"margin_dpo/margin_mean": 12.926143646240234,
"margin_dpo/margin_std": 18.022686004638672,
"step": 215
},
{
"KL/chosen_KL_mean": -13.74349594116211,
"KL/mean": -20.631071090698242,
"KL/rejected_KL_mean": -27.518638610839844,
"KL/std": 16.43567657470703,
"epoch": 0.32653061224489793,
"fcm_dpo/beta": 0.04293996840715408,
"fcm_dpo/delta": -0.20333018898963928,
"fcm_dpo/margin": 13.775140762329102,
"fcm_dpo/q_t": 0.37093037366867065,
"grad_norm": 20.95448112487793,
"learning_rate": 4.2724355170431247e-07,
"logits/chosen": 0.4595262408256531,
"logits/rejected": 0.3734198808670044,
"logps/chosen": -81.56845092773438,
"logps/ref_chosen": -67.824951171875,
"logps/ref_rejected": -96.40231323242188,
"logps/rejected": -123.92095947265625,
"loss": 1.0077,
"margin_dpo/margin_mean": 13.775140762329102,
"margin_dpo/margin_std": 17.89256477355957,
"step": 216
},
{
"KL/chosen_KL_mean": -16.006874084472656,
"KL/mean": -23.1407470703125,
"KL/rejected_KL_mean": -30.27462387084961,
"KL/std": 16.95585823059082,
"epoch": 0.328042328042328,
"fcm_dpo/beta": 0.041124336421489716,
"fcm_dpo/delta": -0.19886408746242523,
"fcm_dpo/margin": 14.267748832702637,
"fcm_dpo/q_t": 0.37181177735328674,
"grad_norm": 15.961468696594238,
"learning_rate": 4.26308602680756e-07,
"logits/chosen": 0.4025682806968689,
"logits/rejected": 0.2978231906890869,
"logps/chosen": -76.51187133789062,
"logps/ref_chosen": -60.5049934387207,
"logps/ref_rejected": -84.26618194580078,
"logps/rejected": -114.54080200195312,
"loss": 1.006,
"margin_dpo/margin_mean": 14.26774787902832,
"margin_dpo/margin_std": 18.178325653076172,
"step": 217
},
{
"KL/chosen_KL_mean": -16.748504638671875,
"KL/mean": -21.318729400634766,
"KL/rejected_KL_mean": -25.88895034790039,
"KL/std": 15.89367389678955,
"epoch": 0.3295540438397581,
"fcm_dpo/beta": 0.0404946506023407,
"fcm_dpo/delta": -0.07148971408605576,
"fcm_dpo/margin": 9.140448570251465,
"fcm_dpo/q_t": 0.41877812147140503,
"grad_norm": 18.13237953186035,
"learning_rate": 4.253687219265803e-07,
"logits/chosen": 0.28222280740737915,
"logits/rejected": 0.2771342396736145,
"logps/chosen": -87.34281921386719,
"logps/ref_chosen": -70.59431457519531,
"logps/ref_rejected": -73.89038848876953,
"logps/rejected": -99.77934265136719,
"loss": 1.2054,
"margin_dpo/margin_mean": 9.140449523925781,
"margin_dpo/margin_std": 19.60466957092285,
"step": 218
},
{
"KL/chosen_KL_mean": -15.531333923339844,
"KL/mean": -20.453414916992188,
"KL/rejected_KL_mean": -25.37550163269043,
"KL/std": 17.02822494506836,
"epoch": 0.3310657596371882,
"fcm_dpo/beta": 0.04007101431488991,
"fcm_dpo/delta": 0.0054306164383888245,
"fcm_dpo/margin": 9.844169616699219,
"fcm_dpo/q_t": 0.412641704082489,
"grad_norm": 17.729270935058594,
"learning_rate": 4.2442393573227043e-07,
"logits/chosen": 0.36880671977996826,
"logits/rejected": 0.32753318548202515,
"logps/chosen": -76.02227783203125,
"logps/ref_chosen": -60.490943908691406,
"logps/ref_rejected": -75.85001373291016,
"logps/rejected": -101.22550964355469,
"loss": 1.1393,
"margin_dpo/margin_mean": 9.844169616699219,
"margin_dpo/margin_std": 17.01150131225586,
"step": 219
},
{
"KL/chosen_KL_mean": -13.573007583618164,
"KL/mean": -19.37187385559082,
"KL/rejected_KL_mean": -25.17074203491211,
"KL/std": 17.334457397460938,
"epoch": 0.3325774754346183,
"fcm_dpo/beta": 0.03964848816394806,
"fcm_dpo/delta": -0.06370130181312561,
"fcm_dpo/margin": 11.597736358642578,
"fcm_dpo/q_t": 0.4014459252357483,
"grad_norm": 14.930502891540527,
"learning_rate": 4.234742705255272e-07,
"logits/chosen": 0.43720513582229614,
"logits/rejected": 0.37640994787216187,
"logps/chosen": -58.58640670776367,
"logps/ref_chosen": -45.013397216796875,
"logps/ref_rejected": -70.49369812011719,
"logps/rejected": -95.66444396972656,
"loss": 1.1213,
"margin_dpo/margin_mean": 11.597736358642578,
"margin_dpo/margin_std": 19.820152282714844,
"step": 220
},
{
"KL/chosen_KL_mean": -13.391084671020508,
"KL/mean": -19.645097732543945,
"KL/rejected_KL_mean": -25.89910888671875,
"KL/std": 17.12027359008789,
"epoch": 0.3340891912320484,
"fcm_dpo/beta": 0.03922563046216965,
"fcm_dpo/delta": -0.09522987902164459,
"fcm_dpo/margin": 12.508028030395508,
"fcm_dpo/q_t": 0.39440637826919556,
"grad_norm": 17.11784553527832,
"learning_rate": 4.22519752870528e-07,
"logits/chosen": 0.4418843388557434,
"logits/rejected": 0.3704308271408081,
"logps/chosen": -72.48692321777344,
"logps/ref_chosen": -59.09584045410156,
"logps/ref_rejected": -88.64388275146484,
"logps/rejected": -114.5429916381836,
"loss": 1.0884,
"margin_dpo/margin_mean": 12.508028030395508,
"margin_dpo/margin_std": 19.78207778930664,
"step": 221
},
{
"KL/chosen_KL_mean": -14.346302032470703,
"KL/mean": -22.558067321777344,
"KL/rejected_KL_mean": -30.76983642578125,
"KL/std": 19.20968246459961,
"epoch": 0.3356009070294785,
"fcm_dpo/beta": 0.037618488073349,
"fcm_dpo/delta": -0.2325230985879898,
"fcm_dpo/margin": 16.42353057861328,
"fcm_dpo/q_t": 0.3641064167022705,
"grad_norm": 17.134920120239258,
"learning_rate": 4.2156040946718343e-07,
"logits/chosen": 0.47954899072647095,
"logits/rejected": 0.39413005113601685,
"logps/chosen": -70.343994140625,
"logps/ref_chosen": -55.9976921081543,
"logps/ref_rejected": -111.94727325439453,
"logps/rejected": -142.71710205078125,
"loss": 1.0009,
"margin_dpo/margin_mean": 16.42353057861328,
"margin_dpo/margin_std": 20.89666748046875,
"step": 222
},
{
"KL/chosen_KL_mean": -16.787445068359375,
"KL/mean": -24.475242614746094,
"KL/rejected_KL_mean": -32.16303634643555,
"KL/std": 19.250900268554688,
"epoch": 0.3371126228269085,
"fcm_dpo/beta": 0.036129191517829895,
"fcm_dpo/delta": -0.1660102903842926,
"fcm_dpo/margin": 15.375591278076172,
"fcm_dpo/q_t": 0.37710410356521606,
"grad_norm": 15.727400779724121,
"learning_rate": 4.2059626715039065e-07,
"logits/chosen": 0.4636209011077881,
"logits/rejected": 0.40776753425598145,
"logps/chosen": -76.67886352539062,
"logps/ref_chosen": -59.891422271728516,
"logps/ref_rejected": -86.28954315185547,
"logps/rejected": -118.45257568359375,
"loss": 1.0087,
"margin_dpo/margin_mean": 15.375591278076172,
"margin_dpo/margin_std": 18.521175384521484,
"step": 223
},
{
"KL/chosen_KL_mean": -19.554534912109375,
"KL/mean": -23.83641815185547,
"KL/rejected_KL_mean": -28.118305206298828,
"KL/std": 18.881946563720703,
"epoch": 0.3386243386243386,
"fcm_dpo/beta": 0.03643026202917099,
"fcm_dpo/delta": 0.0909515768289566,
"fcm_dpo/margin": 8.56376838684082,
"fcm_dpo/q_t": 0.43271827697753906,
"grad_norm": 21.18820571899414,
"learning_rate": 4.1962735288928304e-07,
"logits/chosen": 0.5005279779434204,
"logits/rejected": 0.4795645475387573,
"logps/chosen": -83.59916687011719,
"logps/ref_chosen": -64.04463195800781,
"logps/ref_rejected": -75.05450439453125,
"logps/rejected": -103.17280578613281,
"loss": 1.2153,
"margin_dpo/margin_mean": 8.563769340515137,
"margin_dpo/margin_std": 19.515071868896484,
"step": 224
},
{
"KL/chosen_KL_mean": -17.946279525756836,
"KL/mean": -25.81399154663086,
"KL/rejected_KL_mean": -33.68170166015625,
"KL/std": 21.082653045654297,
"epoch": 0.3401360544217687,
"fcm_dpo/beta": 0.03559402376413345,
"fcm_dpo/delta": -0.17068368196487427,
"fcm_dpo/margin": 15.735418319702148,
"fcm_dpo/q_t": 0.378330260515213,
"grad_norm": 16.837215423583984,
"learning_rate": 4.186536937864752e-07,
"logits/chosen": 0.43999171257019043,
"logits/rejected": 0.32848042249679565,
"logps/chosen": -84.04209899902344,
"logps/ref_chosen": -66.0958251953125,
"logps/ref_rejected": -97.68675231933594,
"logps/rejected": -131.3684539794922,
"loss": 1.0259,
"margin_dpo/margin_mean": 15.735418319702148,
"margin_dpo/margin_std": 20.820514678955078,
"step": 225
},
{
"KL/chosen_KL_mean": -16.996238708496094,
"KL/mean": -23.387149810791016,
"KL/rejected_KL_mean": -29.778060913085938,
"KL/std": 20.35839080810547,
"epoch": 0.3416477702191988,
"fcm_dpo/beta": 0.03531336039304733,
"fcm_dpo/delta": -0.053985681384801865,
"fcm_dpo/margin": 12.781817436218262,
"fcm_dpo/q_t": 0.4018649756908417,
"grad_norm": 15.166661262512207,
"learning_rate": 4.176753170773052e-07,
"logits/chosen": 0.5153093338012695,
"logits/rejected": 0.4674370586872101,
"logps/chosen": -68.4131088256836,
"logps/ref_chosen": -51.4168701171875,
"logps/ref_rejected": -66.30068969726562,
"logps/rejected": -96.07875061035156,
"loss": 1.1434,
"margin_dpo/margin_mean": 12.781817436218262,
"margin_dpo/margin_std": 23.433242797851562,
"step": 226
},
{
"KL/chosen_KL_mean": -18.4608211517334,
"KL/mean": -25.43888282775879,
"KL/rejected_KL_mean": -32.41695022583008,
"KL/std": 21.919437408447266,
"epoch": 0.3431594860166289,
"fcm_dpo/beta": 0.03448785841464996,
"fcm_dpo/delta": -0.08596926182508469,
"fcm_dpo/margin": 13.956132888793945,
"fcm_dpo/q_t": 0.3981607258319855,
"grad_norm": 16.10450553894043,
"learning_rate": 4.166922501290729e-07,
"logits/chosen": 0.5286588668823242,
"logits/rejected": 0.48950350284576416,
"logps/chosen": -76.45059967041016,
"logps/ref_chosen": -57.989776611328125,
"logps/ref_rejected": -75.05464172363281,
"logps/rejected": -107.47159576416016,
"loss": 1.1211,
"margin_dpo/margin_mean": 13.956132888793945,
"margin_dpo/margin_std": 24.33257293701172,
"step": 227
},
{
"KL/chosen_KL_mean": -21.115345001220703,
"KL/mean": -28.109249114990234,
"KL/rejected_KL_mean": -35.1031494140625,
"KL/std": 21.120891571044922,
"epoch": 0.34467120181405897,
"fcm_dpo/beta": 0.034144893288612366,
"fcm_dpo/delta": -0.08144711703062057,
"fcm_dpo/margin": 13.987800598144531,
"fcm_dpo/q_t": 0.39574500918388367,
"grad_norm": 16.343774795532227,
"learning_rate": 4.1570452044027405e-07,
"logits/chosen": 0.5214799642562866,
"logits/rejected": 0.44211211800575256,
"logps/chosen": -76.67471313476562,
"logps/ref_chosen": -55.55936813354492,
"logps/ref_rejected": -77.02364349365234,
"logps/rejected": -112.12679290771484,
"loss": 1.0897,
"margin_dpo/margin_mean": 13.987800598144531,
"margin_dpo/margin_std": 21.647363662719727,
"step": 228
},
{
"KL/chosen_KL_mean": -16.314977645874023,
"KL/mean": -23.184484481811523,
"KL/rejected_KL_mean": -30.05398941040039,
"KL/std": 20.473445892333984,
"epoch": 0.34618291761148906,
"fcm_dpo/beta": 0.03352894261479378,
"fcm_dpo/delta": -0.063721664249897,
"fcm_dpo/margin": 13.739012718200684,
"fcm_dpo/q_t": 0.39857470989227295,
"grad_norm": 29.307889938354492,
"learning_rate": 4.147121556398312e-07,
"logits/chosen": 0.5973633527755737,
"logits/rejected": 0.5300034284591675,
"logps/chosen": -67.1096420288086,
"logps/ref_chosen": -50.79466247558594,
"logps/ref_rejected": -78.4474105834961,
"logps/rejected": -108.50140380859375,
"loss": 1.1379,
"margin_dpo/margin_mean": 13.739013671875,
"margin_dpo/margin_std": 25.073959350585938,
"step": 229
},
{
"KL/chosen_KL_mean": -19.013996124267578,
"KL/mean": -26.8618106842041,
"KL/rejected_KL_mean": -34.70962142944336,
"KL/std": 22.844982147216797,
"epoch": 0.3476946334089191,
"fcm_dpo/beta": 0.033160366117954254,
"fcm_dpo/delta": -0.12775377929210663,
"fcm_dpo/margin": 15.695627212524414,
"fcm_dpo/q_t": 0.38632500171661377,
"grad_norm": 16.520418167114258,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 0.507688045501709,
"logits/rejected": 0.5065501928329468,
"logps/chosen": -75.74322509765625,
"logps/ref_chosen": -56.729225158691406,
"logps/ref_rejected": -62.99180603027344,
"logps/rejected": -97.70143127441406,
"loss": 1.0567,
"margin_dpo/margin_mean": 15.695627212524414,
"margin_dpo/margin_std": 21.76026153564453,
"step": 230
},
{
"KL/chosen_KL_mean": -23.133255004882812,
"KL/mean": -34.50753402709961,
"KL/rejected_KL_mean": -45.88182067871094,
"KL/std": 22.756118774414062,
"epoch": 0.3492063492063492,
"fcm_dpo/beta": 0.03089335560798645,
"fcm_dpo/delta": -0.3283424377441406,
"fcm_dpo/margin": 22.748565673828125,
"fcm_dpo/q_t": 0.3419625461101532,
"grad_norm": 17.398941040039062,
"learning_rate": 4.1271363186719835e-07,
"logits/chosen": 0.4435596466064453,
"logits/rejected": 0.431662917137146,
"logps/chosen": -95.73035430908203,
"logps/ref_chosen": -72.59709930419922,
"logps/ref_rejected": -86.2322998046875,
"logps/rejected": -132.11412048339844,
"loss": 0.9257,
"margin_dpo/margin_mean": 22.748565673828125,
"margin_dpo/margin_std": 23.808895111083984,
"step": 231
},
{
"KL/chosen_KL_mean": -22.9188232421875,
"KL/mean": -30.66337013244629,
"KL/rejected_KL_mean": -38.40791320800781,
"KL/std": 24.664813995361328,
"epoch": 0.3507180650037793,
"fcm_dpo/beta": 0.030204694718122482,
"fcm_dpo/delta": -0.0712435320019722,
"fcm_dpo/margin": 15.489089965820312,
"fcm_dpo/q_t": 0.4011420011520386,
"grad_norm": 16.636199951171875,
"learning_rate": 4.1170752879801436e-07,
"logits/chosen": 0.47813618183135986,
"logits/rejected": 0.4484882950782776,
"logps/chosen": -91.037353515625,
"logps/ref_chosen": -68.1185302734375,
"logps/ref_rejected": -83.79415893554688,
"logps/rejected": -122.20207214355469,
"loss": 1.126,
"margin_dpo/margin_mean": 15.489091873168945,
"margin_dpo/margin_std": 27.5382022857666,
"step": 232
},
{
"KL/chosen_KL_mean": -27.133682250976562,
"KL/mean": -33.432437896728516,
"KL/rejected_KL_mean": -39.73119354248047,
"KL/std": 24.389122009277344,
"epoch": 0.35222978080120937,
"fcm_dpo/beta": 0.029510973021388054,
"fcm_dpo/delta": -0.10193730890750885,
"fcm_dpo/margin": 12.597511291503906,
"fcm_dpo/q_t": 0.41866153478622437,
"grad_norm": 16.13553237915039,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": 0.5277206897735596,
"logits/rejected": 0.469798743724823,
"logps/chosen": -82.2038345336914,
"logps/ref_chosen": -55.070152282714844,
"logps/ref_rejected": -66.61845397949219,
"logps/rejected": -106.34963989257812,
"loss": 1.1697,
"margin_dpo/margin_mean": 12.59751033782959,
"margin_dpo/margin_std": 23.40520668029785,
"step": 233
},
{
"KL/chosen_KL_mean": -25.17361068725586,
"KL/mean": -31.620012283325195,
"KL/rejected_KL_mean": -38.06641387939453,
"KL/std": 23.527820587158203,
"epoch": 0.35374149659863946,
"fcm_dpo/beta": 0.02917061373591423,
"fcm_dpo/delta": -0.11668447405099869,
"fcm_dpo/margin": 12.892807960510254,
"fcm_dpo/q_t": 0.4191032946109772,
"grad_norm": 18.15867042541504,
"learning_rate": 4.09681781007452e-07,
"logits/chosen": 0.433084100484848,
"logits/rejected": 0.4224512577056885,
"logps/chosen": -81.09950256347656,
"logps/ref_chosen": -55.92589569091797,
"logps/ref_rejected": -51.11608123779297,
"logps/rejected": -89.1824951171875,
"loss": 1.1939,
"margin_dpo/margin_mean": 12.892807960510254,
"margin_dpo/margin_std": 25.88437271118164,
"step": 234
},
{
"KL/chosen_KL_mean": -20.61785316467285,
"KL/mean": -31.362199783325195,
"KL/rejected_KL_mean": -42.10654830932617,
"KL/std": 25.210582733154297,
"epoch": 0.35525321239606955,
"fcm_dpo/beta": 0.02799680456519127,
"fcm_dpo/delta": -0.21412935853004456,
"fcm_dpo/margin": 21.488697052001953,
"fcm_dpo/q_t": 0.3646219074726105,
"grad_norm": 15.914239883422852,
"learning_rate": 4.08662192950594e-07,
"logits/chosen": 0.5549330711364746,
"logits/rejected": 0.5380803346633911,
"logps/chosen": -85.15757751464844,
"logps/ref_chosen": -64.53972625732422,
"logps/ref_rejected": -77.69151306152344,
"logps/rejected": -119.79805755615234,
"loss": 0.9811,
"margin_dpo/margin_mean": 21.488697052001953,
"margin_dpo/margin_std": 24.690166473388672,
"step": 235
},
{
"KL/chosen_KL_mean": -33.625633239746094,
"KL/mean": -41.95179748535156,
"KL/rejected_KL_mean": -50.27796936035156,
"KL/std": 25.48017120361328,
"epoch": 0.35676492819349964,
"fcm_dpo/beta": 0.027331937104463577,
"fcm_dpo/delta": -0.05795658379793167,
"fcm_dpo/margin": 16.652341842651367,
"fcm_dpo/q_t": 0.40000301599502563,
"grad_norm": 14.209693908691406,
"learning_rate": 4.076381667711306e-07,
"logits/chosen": 0.49884456396102905,
"logits/rejected": 0.4853667914867401,
"logps/chosen": -104.78036499023438,
"logps/ref_chosen": -71.15473937988281,
"logps/ref_rejected": -84.88541412353516,
"logps/rejected": -135.16339111328125,
"loss": 1.1222,
"margin_dpo/margin_mean": 16.652339935302734,
"margin_dpo/margin_std": 28.560272216796875,
"step": 236
},
{
"KL/chosen_KL_mean": -29.424114227294922,
"KL/mean": -39.12672805786133,
"KL/rejected_KL_mean": -48.829345703125,
"KL/std": 25.990968704223633,
"epoch": 0.35827664399092973,
"fcm_dpo/beta": 0.026831991970539093,
"fcm_dpo/delta": -0.1271677315235138,
"fcm_dpo/margin": 19.405227661132812,
"fcm_dpo/q_t": 0.3854559659957886,
"grad_norm": 16.918258666992188,
"learning_rate": 4.066097311132753e-07,
"logits/chosen": 0.5593961477279663,
"logits/rejected": 0.5475857257843018,
"logps/chosen": -105.56613159179688,
"logps/ref_chosen": -76.14201354980469,
"logps/ref_rejected": -80.88479614257812,
"logps/rejected": -129.71414184570312,
"loss": 1.0732,
"margin_dpo/margin_mean": 19.405229568481445,
"margin_dpo/margin_std": 28.86574363708496,
"step": 237
},
{
"KL/chosen_KL_mean": -27.228557586669922,
"KL/mean": -36.94152069091797,
"KL/rejected_KL_mean": -46.65448760986328,
"KL/std": 27.733543395996094,
"epoch": 0.35978835978835977,
"fcm_dpo/beta": 0.026071514934301376,
"fcm_dpo/delta": -0.1129072904586792,
"fcm_dpo/margin": 19.425922393798828,
"fcm_dpo/q_t": 0.38826340436935425,
"grad_norm": 22.84075927734375,
"learning_rate": 4.0557691474458414e-07,
"logits/chosen": 0.4906197190284729,
"logits/rejected": 0.4794694781303406,
"logps/chosen": -96.1134033203125,
"logps/ref_chosen": -68.88484954833984,
"logps/ref_rejected": -75.8946304321289,
"logps/rejected": -122.54911804199219,
"loss": 1.0671,
"margin_dpo/margin_mean": 19.425922393798828,
"margin_dpo/margin_std": 28.16571807861328,
"step": 238
},
{
"KL/chosen_KL_mean": -33.087646484375,
"KL/mean": -42.84894561767578,
"KL/rejected_KL_mean": -52.61023712158203,
"KL/std": 29.231101989746094,
"epoch": 0.36130007558578986,
"fcm_dpo/beta": 0.025806337594985962,
"fcm_dpo/delta": -0.10971814393997192,
"fcm_dpo/margin": 19.522605895996094,
"fcm_dpo/q_t": 0.3904213309288025,
"grad_norm": 18.117996215820312,
"learning_rate": 4.045397465551513e-07,
"logits/chosen": 0.6238787174224854,
"logits/rejected": 0.49679049849510193,
"logps/chosen": -89.8594741821289,
"logps/ref_chosen": -56.771827697753906,
"logps/ref_rejected": -116.23050689697266,
"logps/rejected": -168.8407440185547,
"loss": 1.0839,
"margin_dpo/margin_mean": 19.522605895996094,
"margin_dpo/margin_std": 29.67517852783203,
"step": 239
},
{
"KL/chosen_KL_mean": -32.0657958984375,
"KL/mean": -44.820091247558594,
"KL/rejected_KL_mean": -57.57439041137695,
"KL/std": 29.670385360717773,
"epoch": 0.36281179138321995,
"fcm_dpo/beta": 0.024577822536230087,
"fcm_dpo/delta": -0.24198389053344727,
"fcm_dpo/margin": 25.508596420288086,
"fcm_dpo/q_t": 0.3613056540489197,
"grad_norm": 13.543078422546387,
"learning_rate": 4.0349825555680045e-07,
"logits/chosen": 0.5578250288963318,
"logits/rejected": 0.46434295177459717,
"logps/chosen": -85.41990661621094,
"logps/ref_chosen": -53.35411071777344,
"logps/ref_rejected": -80.12019348144531,
"logps/rejected": -137.694580078125,
"loss": 0.9745,
"margin_dpo/margin_mean": 25.508594512939453,
"margin_dpo/margin_std": 29.75225830078125,
"step": 240
},
{
"KL/chosen_KL_mean": -32.774986267089844,
"KL/mean": -41.37029266357422,
"KL/rejected_KL_mean": -49.96559143066406,
"KL/std": 28.415935516357422,
"epoch": 0.36432350718065004,
"fcm_dpo/beta": 0.02427198737859726,
"fcm_dpo/delta": -0.018171856179833412,
"fcm_dpo/margin": 17.190608978271484,
"fcm_dpo/q_t": 0.4086453318595886,
"grad_norm": 16.303773880004883,
"learning_rate": 4.0245247088227377e-07,
"logits/chosen": 0.49140608310699463,
"logits/rejected": 0.45661377906799316,
"logps/chosen": -104.67039489746094,
"logps/ref_chosen": -71.89541625976562,
"logps/ref_rejected": -83.03492736816406,
"logps/rejected": -133.00051879882812,
"loss": 1.133,
"margin_dpo/margin_mean": 17.190608978271484,
"margin_dpo/margin_std": 29.775129318237305,
"step": 241
},
{
"KL/chosen_KL_mean": -33.861358642578125,
"KL/mean": -45.696693420410156,
"KL/rejected_KL_mean": -57.53202438354492,
"KL/std": 30.87794303894043,
"epoch": 0.36583522297808013,
"fcm_dpo/beta": 0.023374799638986588,
"fcm_dpo/delta": -0.16532181203365326,
"fcm_dpo/margin": 23.670665740966797,
"fcm_dpo/q_t": 0.37864089012145996,
"grad_norm": 12.985538482666016,
"learning_rate": 4.0140242178441665e-07,
"logits/chosen": 0.5361425876617432,
"logits/rejected": 0.5154822468757629,
"logps/chosen": -91.78878784179688,
"logps/ref_chosen": -57.927433013916016,
"logps/ref_rejected": -67.838623046875,
"logps/rejected": -125.37064361572266,
"loss": 1.0351,
"margin_dpo/margin_mean": 23.670665740966797,
"margin_dpo/margin_std": 31.78559684753418,
"step": 242
},
{
"KL/chosen_KL_mean": -34.01261520385742,
"KL/mean": -44.46428298950195,
"KL/rejected_KL_mean": -54.915950775146484,
"KL/std": 30.95125389099121,
"epoch": 0.3673469387755102,
"fcm_dpo/beta": 0.02309669926762581,
"fcm_dpo/delta": -0.08695002645254135,
"fcm_dpo/margin": 20.903337478637695,
"fcm_dpo/q_t": 0.3931156396865845,
"grad_norm": 17.241031646728516,
"learning_rate": 4.003481376353596e-07,
"logits/chosen": 0.5158106088638306,
"logits/rejected": 0.5203914642333984,
"logps/chosen": -108.28929138183594,
"logps/ref_chosen": -74.27667236328125,
"logps/ref_rejected": -73.24340057373047,
"logps/rejected": -128.1593475341797,
"loss": 1.0789,
"margin_dpo/margin_mean": 20.903337478637695,
"margin_dpo/margin_std": 31.15512466430664,
"step": 243
},
{
"KL/chosen_KL_mean": -34.350364685058594,
"KL/mean": -48.03395080566406,
"KL/rejected_KL_mean": -61.71753692626953,
"KL/std": 30.761280059814453,
"epoch": 0.3688586545729403,
"fcm_dpo/beta": 0.02230009436607361,
"fcm_dpo/delta": -0.22346463799476624,
"fcm_dpo/margin": 27.367176055908203,
"fcm_dpo/q_t": 0.36376476287841797,
"grad_norm": 15.551508903503418,
"learning_rate": 3.9928964792569654e-07,
"logits/chosen": 0.558070957660675,
"logits/rejected": 0.4750991463661194,
"logps/chosen": -87.71426391601562,
"logps/ref_chosen": -53.36390686035156,
"logps/ref_rejected": -71.10276794433594,
"logps/rejected": -132.8203125,
"loss": 0.9702,
"margin_dpo/margin_mean": 27.367176055908203,
"margin_dpo/margin_std": 30.240100860595703,
"step": 244
},
{
"KL/chosen_KL_mean": -36.503814697265625,
"KL/mean": -52.387237548828125,
"KL/rejected_KL_mean": -68.2706527709961,
"KL/std": 32.38478088378906,
"epoch": 0.37037037037037035,
"fcm_dpo/beta": 0.021037843078374863,
"fcm_dpo/delta": -0.28781792521476746,
"fcm_dpo/margin": 31.766828536987305,
"fcm_dpo/q_t": 0.3499138355255127,
"grad_norm": 16.55666160583496,
"learning_rate": 3.982269822636601e-07,
"logits/chosen": 0.618838906288147,
"logits/rejected": 0.5927552580833435,
"logps/chosen": -107.69892120361328,
"logps/ref_chosen": -71.19510650634766,
"logps/ref_rejected": -80.76235961914062,
"logps/rejected": -149.03302001953125,
"loss": 0.9281,
"margin_dpo/margin_mean": 31.766828536987305,
"margin_dpo/margin_std": 32.20833206176758,
"step": 245
},
{
"KL/chosen_KL_mean": -43.60735321044922,
"KL/mean": -56.665950775146484,
"KL/rejected_KL_mean": -69.72454833984375,
"KL/std": 32.904762268066406,
"epoch": 0.37188208616780044,
"fcm_dpo/beta": 0.020430248230695724,
"fcm_dpo/delta": -0.14096316695213318,
"fcm_dpo/margin": 26.117191314697266,
"fcm_dpo/q_t": 0.3837364912033081,
"grad_norm": 17.192018508911133,
"learning_rate": 3.971601703742932e-07,
"logits/chosen": 0.6050068140029907,
"logits/rejected": 0.544990062713623,
"logps/chosen": -115.22840118408203,
"logps/ref_chosen": -71.62104797363281,
"logps/ref_rejected": -94.03392028808594,
"logps/rejected": -163.75848388671875,
"loss": 1.0723,
"margin_dpo/margin_mean": 26.117191314697266,
"margin_dpo/margin_std": 39.31426239013672,
"step": 246
},
{
"KL/chosen_KL_mean": -49.28352355957031,
"KL/mean": -56.45580291748047,
"KL/rejected_KL_mean": -63.628074645996094,
"KL/std": 32.952980041503906,
"epoch": 0.37339380196523053,
"fcm_dpo/beta": 0.02019241452217102,
"fcm_dpo/delta": 0.01191400084644556,
"fcm_dpo/margin": 14.344557762145996,
"fcm_dpo/q_t": 0.43515753746032715,
"grad_norm": 17.813844680786133,
"learning_rate": 3.960892420986177e-07,
"logits/chosen": 0.5864748358726501,
"logits/rejected": 0.5768144130706787,
"logps/chosen": -129.30606079101562,
"logps/ref_chosen": -80.02254486083984,
"logps/ref_rejected": -89.22705841064453,
"logps/rejected": -152.85513305664062,
"loss": 1.2326,
"margin_dpo/margin_mean": 14.344557762145996,
"margin_dpo/margin_std": 34.36824035644531,
"step": 247
},
{
"KL/chosen_KL_mean": -41.71202850341797,
"KL/mean": -55.5155029296875,
"KL/rejected_KL_mean": -69.31898498535156,
"KL/std": 36.72417449951172,
"epoch": 0.3749055177626606,
"fcm_dpo/beta": 0.019840724766254425,
"fcm_dpo/delta": -0.1560136079788208,
"fcm_dpo/margin": 27.606952667236328,
"fcm_dpo/q_t": 0.38317927718162537,
"grad_norm": 14.924769401550293,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 0.5998907089233398,
"logits/rejected": 0.6501777172088623,
"logps/chosen": -107.08998107910156,
"logps/ref_chosen": -65.37796020507812,
"logps/ref_rejected": -61.365787506103516,
"logps/rejected": -130.6847686767578,
"loss": 1.0613,
"margin_dpo/margin_mean": 27.606952667236328,
"margin_dpo/margin_std": 40.85491943359375,
"step": 248
},
{
"KL/chosen_KL_mean": -54.32504653930664,
"KL/mean": -59.762290954589844,
"KL/rejected_KL_mean": -65.19953918457031,
"KL/std": 34.97101593017578,
"epoch": 0.3764172335600907,
"fcm_dpo/beta": 0.019655220210552216,
"fcm_dpo/delta": 0.030433597043156624,
"fcm_dpo/margin": 10.874480247497559,
"fcm_dpo/q_t": 0.45171886682510376,
"grad_norm": 19.399675369262695,
"learning_rate": 3.9393515632731094e-07,
"logits/chosen": 0.5788969993591309,
"logits/rejected": 0.6164053082466125,
"logps/chosen": -128.926513671875,
"logps/ref_chosen": -74.60145568847656,
"logps/ref_rejected": -63.79338455200195,
"logps/rejected": -128.992919921875,
"loss": 1.3182,
"margin_dpo/margin_mean": 10.874479293823242,
"margin_dpo/margin_std": 37.496307373046875,
"step": 249
},
{
"KL/chosen_KL_mean": -46.774871826171875,
"KL/mean": -60.51074981689453,
"KL/rejected_KL_mean": -74.24663543701172,
"KL/std": 36.776817321777344,
"epoch": 0.3779289493575208,
"fcm_dpo/beta": 0.01937510445713997,
"fcm_dpo/delta": -0.1394677758216858,
"fcm_dpo/margin": 27.471759796142578,
"fcm_dpo/q_t": 0.3808504045009613,
"grad_norm": 15.311856269836426,
"learning_rate": 3.9285205908608934e-07,
"logits/chosen": 0.6721572279930115,
"logits/rejected": 0.6293501853942871,
"logps/chosen": -108.71308898925781,
"logps/ref_chosen": -61.938209533691406,
"logps/ref_rejected": -72.21602630615234,
"logps/rejected": -146.46266174316406,
"loss": 1.0463,
"margin_dpo/margin_mean": 27.471759796142578,
"margin_dpo/margin_std": 37.92514419555664,
"step": 250
},
{
"KL/chosen_KL_mean": -52.91298294067383,
"KL/mean": -61.85021209716797,
"KL/rejected_KL_mean": -70.78742980957031,
"KL/std": 34.53257369995117,
"epoch": 0.3794406651549509,
"fcm_dpo/beta": 0.019310234114527702,
"fcm_dpo/delta": 0.05684041231870651,
"fcm_dpo/margin": 17.87444496154785,
"fcm_dpo/q_t": 0.423962265253067,
"grad_norm": 20.963951110839844,
"learning_rate": 3.9176496596569265e-07,
"logits/chosen": 0.6601051688194275,
"logits/rejected": 0.6193727850914001,
"logps/chosen": -119.76992797851562,
"logps/ref_chosen": -66.85694885253906,
"logps/ref_rejected": -84.83396911621094,
"logps/rejected": -155.62139892578125,
"loss": 1.204,
"margin_dpo/margin_mean": 17.87444305419922,
"margin_dpo/margin_std": 39.21337127685547,
"step": 251
},
{
"KL/chosen_KL_mean": -48.95282745361328,
"KL/mean": -57.23577880859375,
"KL/rejected_KL_mean": -65.51873016357422,
"KL/std": 37.31624221801758,
"epoch": 0.38095238095238093,
"fcm_dpo/beta": 0.019211940467357635,
"fcm_dpo/delta": -0.09368051588535309,
"fcm_dpo/margin": 16.565898895263672,
"fcm_dpo/q_t": 0.43007659912109375,
"grad_norm": 25.564207077026367,
"learning_rate": 3.9067390737445254e-07,
"logits/chosen": 0.5914499759674072,
"logits/rejected": 0.5368775129318237,
"logps/chosen": -105.1767578125,
"logps/ref_chosen": -56.22393035888672,
"logps/ref_rejected": -77.1136245727539,
"logps/rejected": -142.63235473632812,
"loss": 1.2506,
"margin_dpo/margin_mean": 16.565898895263672,
"margin_dpo/margin_std": 40.483367919921875,
"step": 252
},
{
"KL/chosen_KL_mean": -50.39899444580078,
"KL/mean": -60.09886932373047,
"KL/rejected_KL_mean": -69.79873657226562,
"KL/std": 36.75677490234375,
"epoch": 0.382464096749811,
"fcm_dpo/beta": 0.018776969984173775,
"fcm_dpo/delta": -0.06744483858346939,
"fcm_dpo/margin": 19.39974594116211,
"fcm_dpo/q_t": 0.4195025563240051,
"grad_norm": 18.586383819580078,
"learning_rate": 3.8957891383162304e-07,
"logits/chosen": 0.6764658093452454,
"logits/rejected": 0.6350239515304565,
"logps/chosen": -102.6090087890625,
"logps/ref_chosen": -52.21001434326172,
"logps/ref_rejected": -58.75764846801758,
"logps/rejected": -128.55638122558594,
"loss": 1.1711,
"margin_dpo/margin_mean": 19.39974594116211,
"margin_dpo/margin_std": 36.59492874145508,
"step": 253
},
{
"KL/chosen_KL_mean": -52.575721740722656,
"KL/mean": -64.28272247314453,
"KL/rejected_KL_mean": -75.98971557617188,
"KL/std": 38.303611755371094,
"epoch": 0.3839758125472411,
"fcm_dpo/beta": 0.018581921234726906,
"fcm_dpo/delta": -0.03734355419874191,
"fcm_dpo/margin": 23.414005279541016,
"fcm_dpo/q_t": 0.4057334065437317,
"grad_norm": 14.820300102233887,
"learning_rate": 3.884800159665276e-07,
"logits/chosen": 0.5761537551879883,
"logits/rejected": 0.5271477699279785,
"logps/chosen": -118.21205139160156,
"logps/ref_chosen": -65.63632202148438,
"logps/ref_rejected": -82.34425354003906,
"logps/rejected": -158.33396911621094,
"loss": 1.1181,
"margin_dpo/margin_mean": 23.414005279541016,
"margin_dpo/margin_std": 38.766605377197266,
"step": 254
},
{
"KL/chosen_KL_mean": -50.35456085205078,
"KL/mean": -63.37836456298828,
"KL/rejected_KL_mean": -76.40216064453125,
"KL/std": 39.2579345703125,
"epoch": 0.3854875283446712,
"fcm_dpo/beta": 0.018372762948274612,
"fcm_dpo/delta": -0.08276001363992691,
"fcm_dpo/margin": 26.04759979248047,
"fcm_dpo/q_t": 0.39628180861473083,
"grad_norm": 22.969327926635742,
"learning_rate": 3.873772445177015e-07,
"logits/chosen": 0.5657912492752075,
"logits/rejected": 0.5377863645553589,
"logps/chosen": -118.26565551757812,
"logps/ref_chosen": -67.91108703613281,
"logps/ref_rejected": -83.89114379882812,
"logps/rejected": -160.29330444335938,
"loss": 1.1063,
"margin_dpo/margin_mean": 26.04759979248047,
"margin_dpo/margin_std": 43.40495300292969,
"step": 255
},
{
"KL/chosen_KL_mean": -57.47937774658203,
"KL/mean": -70.29521179199219,
"KL/rejected_KL_mean": -83.11105346679688,
"KL/std": 35.82374572753906,
"epoch": 0.3869992441421013,
"fcm_dpo/beta": 0.018089592456817627,
"fcm_dpo/delta": -0.06720145046710968,
"fcm_dpo/margin": 25.631671905517578,
"fcm_dpo/q_t": 0.39948275685310364,
"grad_norm": 18.48078155517578,
"learning_rate": 3.862706303320329e-07,
"logits/chosen": 0.644359290599823,
"logits/rejected": 0.5806387662887573,
"logps/chosen": -120.97936248779297,
"logps/ref_chosen": -63.49998474121094,
"logps/ref_rejected": -90.77104187011719,
"logps/rejected": -173.88209533691406,
"loss": 1.1313,
"margin_dpo/margin_mean": 25.631671905517578,
"margin_dpo/margin_std": 45.804534912109375,
"step": 256
},
{
"KL/chosen_KL_mean": -56.7821159362793,
"KL/mean": -71.2591552734375,
"KL/rejected_KL_mean": -85.73617553710938,
"KL/std": 40.295997619628906,
"epoch": 0.3885109599395314,
"fcm_dpo/beta": 0.01773456111550331,
"fcm_dpo/delta": -0.11991943418979645,
"fcm_dpo/margin": 28.954063415527344,
"fcm_dpo/q_t": 0.38973554968833923,
"grad_norm": 16.94999122619629,
"learning_rate": 3.851602043638994e-07,
"logits/chosen": 0.5933499932289124,
"logits/rejected": 0.5298876166343689,
"logps/chosen": -127.38276672363281,
"logps/ref_chosen": -70.60064697265625,
"logps/ref_rejected": -108.58313751220703,
"logps/rejected": -194.31930541992188,
"loss": 1.083,
"margin_dpo/margin_mean": 28.954063415527344,
"margin_dpo/margin_std": 45.69303894042969,
"step": 257
},
{
"KL/chosen_KL_mean": -54.69956970214844,
"KL/mean": -67.32147216796875,
"KL/rejected_KL_mean": -79.943359375,
"KL/std": 35.66672897338867,
"epoch": 0.3900226757369615,
"fcm_dpo/beta": 0.017553571611642838,
"fcm_dpo/delta": -0.04509525001049042,
"fcm_dpo/margin": 25.243791580200195,
"fcm_dpo/q_t": 0.39821261167526245,
"grad_norm": 15.35595703125,
"learning_rate": 3.840459976743023e-07,
"logits/chosen": 0.6394772529602051,
"logits/rejected": 0.5900181531906128,
"logps/chosen": -113.9537353515625,
"logps/ref_chosen": -59.25416564941406,
"logps/ref_rejected": -85.58709716796875,
"logps/rejected": -165.53045654296875,
"loss": 1.0747,
"margin_dpo/margin_mean": 25.243789672851562,
"margin_dpo/margin_std": 34.329689025878906,
"step": 258
},
{
"KL/chosen_KL_mean": -44.934913635253906,
"KL/mean": -64.14384460449219,
"KL/rejected_KL_mean": -83.352783203125,
"KL/std": 40.59214782714844,
"epoch": 0.3915343915343915,
"fcm_dpo/beta": 0.016735419631004333,
"fcm_dpo/delta": -0.2618575394153595,
"fcm_dpo/margin": 38.41786193847656,
"fcm_dpo/q_t": 0.3562944233417511,
"grad_norm": 13.93376636505127,
"learning_rate": 3.8292804142999796e-07,
"logits/chosen": 0.5679116249084473,
"logits/rejected": 0.46820923686027527,
"logps/chosen": -110.36978912353516,
"logps/ref_chosen": -65.43487548828125,
"logps/ref_rejected": -95.41731262207031,
"logps/rejected": -178.77008056640625,
"loss": 0.9763,
"margin_dpo/margin_mean": 38.41786193847656,
"margin_dpo/margin_std": 45.278961181640625,
"step": 259
},
{
"KL/chosen_KL_mean": -45.69921875,
"KL/mean": -60.529823303222656,
"KL/rejected_KL_mean": -75.36042785644531,
"KL/std": 38.62810134887695,
"epoch": 0.3930461073318216,
"fcm_dpo/beta": 0.016313474625349045,
"fcm_dpo/delta": -0.08897658437490463,
"fcm_dpo/margin": 29.661205291748047,
"fcm_dpo/q_t": 0.39434176683425903,
"grad_norm": 14.801098823547363,
"learning_rate": 3.818063669026256e-07,
"logits/chosen": 0.6178157329559326,
"logits/rejected": 0.5333956480026245,
"logps/chosen": -94.78880310058594,
"logps/ref_chosen": -49.08958435058594,
"logps/ref_rejected": -79.01708221435547,
"logps/rejected": -154.37750244140625,
"loss": 1.0974,
"margin_dpo/margin_mean": 29.661205291748047,
"margin_dpo/margin_std": 47.30998229980469,
"step": 260
},
{
"KL/chosen_KL_mean": -45.938331604003906,
"KL/mean": -58.4451904296875,
"KL/rejected_KL_mean": -70.9520492553711,
"KL/std": 37.34497833251953,
"epoch": 0.3945578231292517,
"fcm_dpo/beta": 0.01630301959812641,
"fcm_dpo/delta": -0.008132414892315865,
"fcm_dpo/margin": 25.013710021972656,
"fcm_dpo/q_t": 0.41026413440704346,
"grad_norm": 17.233556747436523,
"learning_rate": 3.806810054678331e-07,
"logits/chosen": 0.4819830656051636,
"logits/rejected": 0.5123116970062256,
"logps/chosen": -116.81072998046875,
"logps/ref_chosen": -70.87239074707031,
"logps/ref_rejected": -65.01522064208984,
"logps/rejected": -135.96726989746094,
"loss": 1.1249,
"margin_dpo/margin_mean": 25.01371192932129,
"margin_dpo/margin_std": 41.439422607421875,
"step": 261
},
{
"KL/chosen_KL_mean": -46.528953552246094,
"KL/mean": -59.45687484741211,
"KL/rejected_KL_mean": -72.3847885131836,
"KL/std": 36.256553649902344,
"epoch": 0.3960695389266818,
"fcm_dpo/beta": 0.01626831665635109,
"fcm_dpo/delta": -0.021578827872872353,
"fcm_dpo/margin": 25.855838775634766,
"fcm_dpo/q_t": 0.4057990312576294,
"grad_norm": 16.480358123779297,
"learning_rate": 3.7955198860439887e-07,
"logits/chosen": 0.6472454071044922,
"logits/rejected": 0.5862551927566528,
"logps/chosen": -114.39958190917969,
"logps/ref_chosen": -67.8706283569336,
"logps/ref_rejected": -88.7205810546875,
"logps/rejected": -161.10537719726562,
"loss": 1.1058,
"margin_dpo/margin_mean": 25.855838775634766,
"margin_dpo/margin_std": 39.506980895996094,
"step": 262
},
{
"KL/chosen_KL_mean": -44.480228424072266,
"KL/mean": -56.57789611816406,
"KL/rejected_KL_mean": -68.67556762695312,
"KL/std": 37.02964782714844,
"epoch": 0.3975812547241119,
"fcm_dpo/beta": 0.016271250322461128,
"fcm_dpo/delta": 0.006439458578824997,
"fcm_dpo/margin": 24.195329666137695,
"fcm_dpo/q_t": 0.41121095418930054,
"grad_norm": 13.955968856811523,
"learning_rate": 3.784193478933516e-07,
"logits/chosen": 0.5845399498939514,
"logits/rejected": 0.47931110858917236,
"logps/chosen": -99.67481231689453,
"logps/ref_chosen": -55.194583892822266,
"logps/ref_rejected": -80.54048156738281,
"logps/rejected": -149.21604919433594,
"loss": 1.133,
"margin_dpo/margin_mean": 24.195329666137695,
"margin_dpo/margin_std": 40.630882263183594,
"step": 263
},
{
"KL/chosen_KL_mean": -47.25312805175781,
"KL/mean": -60.714229583740234,
"KL/rejected_KL_mean": -74.17533111572266,
"KL/std": 38.969966888427734,
"epoch": 0.39909297052154197,
"fcm_dpo/beta": 0.01612680032849312,
"fcm_dpo/delta": -0.03582238778471947,
"fcm_dpo/margin": 26.922195434570312,
"fcm_dpo/q_t": 0.40335631370544434,
"grad_norm": 14.952977180480957,
"learning_rate": 3.7728311501708674e-07,
"logits/chosen": 0.497216135263443,
"logits/rejected": 0.4524659514427185,
"logps/chosen": -130.42381286621094,
"logps/ref_chosen": -83.17068481445312,
"logps/ref_rejected": -88.33625793457031,
"logps/rejected": -162.5115966796875,
"loss": 1.1082,
"margin_dpo/margin_mean": 26.922195434570312,
"margin_dpo/margin_std": 42.735145568847656,
"step": 264
},
{
"KL/chosen_KL_mean": -44.69921875,
"KL/mean": -60.36857223510742,
"KL/rejected_KL_mean": -76.03793334960938,
"KL/std": 41.01289367675781,
"epoch": 0.40060468631897206,
"fcm_dpo/beta": 0.015967700630426407,
"fcm_dpo/delta": -0.10580786317586899,
"fcm_dpo/margin": 31.338699340820312,
"fcm_dpo/q_t": 0.3908243775367737,
"grad_norm": 14.666281700134277,
"learning_rate": 3.7614332175848027e-07,
"logits/chosen": 0.6847161054611206,
"logits/rejected": 0.6193395853042603,
"logps/chosen": -96.36207580566406,
"logps/ref_chosen": -51.66284942626953,
"logps/ref_rejected": -67.1720962524414,
"logps/rejected": -143.21002197265625,
"loss": 1.0966,
"margin_dpo/margin_mean": 31.338699340820312,
"margin_dpo/margin_std": 49.95252227783203,
"step": 265
},
{
"KL/chosen_KL_mean": -43.260528564453125,
"KL/mean": -57.662353515625,
"KL/rejected_KL_mean": -72.06417846679688,
"KL/std": 41.540184020996094,
"epoch": 0.4021164021164021,
"fcm_dpo/beta": 0.015635395422577858,
"fcm_dpo/delta": -0.052870072424411774,
"fcm_dpo/margin": 28.803661346435547,
"fcm_dpo/q_t": 0.40012824535369873,
"grad_norm": 16.151596069335938,
"learning_rate": 3.75e-07,
"logits/chosen": 0.6196706295013428,
"logits/rejected": 0.5463729500770569,
"logps/chosen": -100.71102142333984,
"logps/ref_chosen": -57.45049285888672,
"logps/ref_rejected": -77.60826110839844,
"logps/rejected": -149.6724395751953,
"loss": 1.0976,
"margin_dpo/margin_mean": 28.803661346435547,
"margin_dpo/margin_std": 44.732513427734375,
"step": 266
},
{
"KL/chosen_KL_mean": -37.7889404296875,
"KL/mean": -48.83605194091797,
"KL/rejected_KL_mean": -59.88316345214844,
"KL/std": 37.36830139160156,
"epoch": 0.4036281179138322,
"fcm_dpo/beta": 0.015527920797467232,
"fcm_dpo/delta": -0.05823346599936485,
"fcm_dpo/margin": 22.094219207763672,
"fcm_dpo/q_t": 0.42264825105667114,
"grad_norm": 15.540796279907227,
"learning_rate": 3.738531817228131e-07,
"logits/chosen": 0.631234884262085,
"logits/rejected": 0.6142922639846802,
"logps/chosen": -92.82429504394531,
"logps/ref_chosen": -55.03535079956055,
"logps/ref_rejected": -66.0953369140625,
"logps/rejected": -125.97850036621094,
"loss": 1.1921,
"margin_dpo/margin_mean": 22.094219207763672,
"margin_dpo/margin_std": 44.37456512451172,
"step": 267
},
{
"KL/chosen_KL_mean": -38.225894927978516,
"KL/mean": -52.12761688232422,
"KL/rejected_KL_mean": -66.02934265136719,
"KL/std": 40.468048095703125,
"epoch": 0.4051398337112623,
"fcm_dpo/beta": 0.015403296798467636,
"fcm_dpo/delta": -0.02956201881170273,
"fcm_dpo/margin": 27.803451538085938,
"fcm_dpo/q_t": 0.40386512875556946,
"grad_norm": 13.097114562988281,
"learning_rate": 3.7270289900589204e-07,
"logits/chosen": 0.5190380215644836,
"logits/rejected": 0.5041170120239258,
"logps/chosen": -103.29763793945312,
"logps/ref_chosen": -65.07174682617188,
"logps/ref_rejected": -71.42485809326172,
"logps/rejected": -137.45419311523438,
"loss": 1.0838,
"margin_dpo/margin_mean": 27.803451538085938,
"margin_dpo/margin_std": 38.19640350341797,
"step": 268
},
{
"KL/chosen_KL_mean": -44.171669006347656,
"KL/mean": -60.03845977783203,
"KL/rejected_KL_mean": -75.90525817871094,
"KL/std": 42.52558898925781,
"epoch": 0.40665154950869237,
"fcm_dpo/beta": 0.015258044004440308,
"fcm_dpo/delta": -0.08893659710884094,
"fcm_dpo/margin": 31.73358917236328,
"fcm_dpo/q_t": 0.3918406367301941,
"grad_norm": 13.794229507446289,
"learning_rate": 3.7154918402511714e-07,
"logits/chosen": 0.7249884009361267,
"logits/rejected": 0.6769078969955444,
"logps/chosen": -111.30787658691406,
"logps/ref_chosen": -67.1362075805664,
"logps/ref_rejected": -82.55778503417969,
"logps/rejected": -158.46304321289062,
"loss": 1.0605,
"margin_dpo/margin_mean": 31.73358917236328,
"margin_dpo/margin_std": 42.283042907714844,
"step": 269
},
{
"KL/chosen_KL_mean": -45.161434173583984,
"KL/mean": -57.58110046386719,
"KL/rejected_KL_mean": -70.00077056884766,
"KL/std": 42.186431884765625,
"epoch": 0.40816326530612246,
"fcm_dpo/beta": 0.01507522352039814,
"fcm_dpo/delta": 0.0263163261115551,
"fcm_dpo/margin": 24.839336395263672,
"fcm_dpo/q_t": 0.41556787490844727,
"grad_norm": 14.325583457946777,
"learning_rate": 3.7039206905237656e-07,
"logits/chosen": 0.6480659246444702,
"logits/rejected": 0.5702933073043823,
"logps/chosen": -111.85012817382812,
"logps/ref_chosen": -66.6886978149414,
"logps/ref_rejected": -85.16129302978516,
"logps/rejected": -155.1620635986328,
"loss": 1.1526,
"margin_dpo/margin_mean": 24.839336395263672,
"margin_dpo/margin_std": 44.66209411621094,
"step": 270
},
{
"KL/chosen_KL_mean": -44.642276763916016,
"KL/mean": -55.103538513183594,
"KL/rejected_KL_mean": -65.56480407714844,
"KL/std": 43.059349060058594,
"epoch": 0.40967498110355255,
"fcm_dpo/beta": 0.015149587765336037,
"fcm_dpo/delta": -0.0019481488270685077,
"fcm_dpo/margin": 20.922527313232422,
"fcm_dpo/q_t": 0.43366163969039917,
"grad_norm": 16.50081443786621,
"learning_rate": 3.692315864546635e-07,
"logits/chosen": 0.6500439643859863,
"logits/rejected": 0.5881233215332031,
"logps/chosen": -117.04981994628906,
"logps/ref_chosen": -72.40754699707031,
"logps/ref_rejected": -92.06311798095703,
"logps/rejected": -157.6279296875,
"loss": 1.2306,
"margin_dpo/margin_mean": 20.92252540588379,
"margin_dpo/margin_std": 50.708343505859375,
"step": 271
},
{
"KL/chosen_KL_mean": -36.9219970703125,
"KL/mean": -57.8653564453125,
"KL/rejected_KL_mean": -78.8087158203125,
"KL/std": 41.90574264526367,
"epoch": 0.41118669690098264,
"fcm_dpo/beta": 0.01468550506979227,
"fcm_dpo/delta": -0.22875632345676422,
"fcm_dpo/margin": 41.88671112060547,
"fcm_dpo/q_t": 0.3596448302268982,
"grad_norm": 15.573667526245117,
"learning_rate": 3.6806776869317067e-07,
"logits/chosen": 0.6257216334342957,
"logits/rejected": 0.6441141366958618,
"logps/chosen": -103.52339935302734,
"logps/ref_chosen": -66.60140228271484,
"logps/ref_rejected": -67.74340057373047,
"logps/rejected": -146.5521240234375,
"loss": 0.9435,
"margin_dpo/margin_mean": 41.88671112060547,
"margin_dpo/margin_std": 40.36542510986328,
"step": 272
},
{
"KL/chosen_KL_mean": -53.16020202636719,
"KL/mean": -67.60650634765625,
"KL/rejected_KL_mean": -82.05280303955078,
"KL/std": 44.438331604003906,
"epoch": 0.4126984126984127,
"fcm_dpo/beta": 0.014402521774172783,
"fcm_dpo/delta": -0.016972802579402924,
"fcm_dpo/margin": 28.89260482788086,
"fcm_dpo/q_t": 0.4075608551502228,
"grad_norm": 16.329553604125977,
"learning_rate": 3.669006483223828e-07,
"logits/chosen": 0.6559746265411377,
"logits/rejected": 0.5886766314506531,
"logps/chosen": -110.51507568359375,
"logps/ref_chosen": -57.35487747192383,
"logps/ref_rejected": -84.17168426513672,
"logps/rejected": -166.2244873046875,
"loss": 1.165,
"margin_dpo/margin_mean": 28.892608642578125,
"margin_dpo/margin_std": 56.325111389160156,
"step": 273
},
{
"KL/chosen_KL_mean": -52.543373107910156,
"KL/mean": -68.96923065185547,
"KL/rejected_KL_mean": -85.39509582519531,
"KL/std": 45.451881408691406,
"epoch": 0.41421012849584277,
"fcm_dpo/beta": 0.014286793768405914,
"fcm_dpo/delta": -0.07270047068595886,
"fcm_dpo/margin": 32.851715087890625,
"fcm_dpo/q_t": 0.3971703052520752,
"grad_norm": 13.8760404586792,
"learning_rate": 3.657302579891656e-07,
"logits/chosen": 0.5500935912132263,
"logits/rejected": 0.5341925024986267,
"logps/chosen": -112.18487548828125,
"logps/ref_chosen": -59.64149475097656,
"logps/ref_rejected": -68.29348754882812,
"logps/rejected": -153.68856811523438,
"loss": 1.1093,
"margin_dpo/margin_mean": 32.851715087890625,
"margin_dpo/margin_std": 54.40715789794922,
"step": 274
},
{
"KL/chosen_KL_mean": -52.10219955444336,
"KL/mean": -68.65190887451172,
"KL/rejected_KL_mean": -85.20160675048828,
"KL/std": 44.36628723144531,
"epoch": 0.41572184429327286,
"fcm_dpo/beta": 0.01410981547087431,
"fcm_dpo/delta": -0.070284903049469,
"fcm_dpo/margin": 33.09941482543945,
"fcm_dpo/q_t": 0.394775390625,
"grad_norm": 14.450504302978516,
"learning_rate": 3.645566304318526e-07,
"logits/chosen": 0.6269994974136353,
"logits/rejected": 0.5459173917770386,
"logps/chosen": -105.36884307861328,
"logps/ref_chosen": -53.26664352416992,
"logps/ref_rejected": -73.84062194824219,
"logps/rejected": -159.042236328125,
"loss": 1.0757,
"margin_dpo/margin_mean": 33.09941864013672,
"margin_dpo/margin_std": 47.155517578125,
"step": 275
},
{
"KL/chosen_KL_mean": -49.204444885253906,
"KL/mean": -65.23859405517578,
"KL/rejected_KL_mean": -81.27273559570312,
"KL/std": 42.764747619628906,
"epoch": 0.41723356009070295,
"fcm_dpo/beta": 0.013920679688453674,
"fcm_dpo/delta": -0.048565976321697235,
"fcm_dpo/margin": 32.06829071044922,
"fcm_dpo/q_t": 0.39945292472839355,
"grad_norm": 15.985830307006836,
"learning_rate": 3.633797984793294e-07,
"logits/chosen": 0.5954059362411499,
"logits/rejected": 0.563401460647583,
"logps/chosen": -102.2252426147461,
"logps/ref_chosen": -53.02079772949219,
"logps/ref_rejected": -61.56678771972656,
"logps/rejected": -142.8395233154297,
"loss": 1.0886,
"margin_dpo/margin_mean": 32.06829071044922,
"margin_dpo/margin_std": 46.98406219482422,
"step": 276
},
{
"KL/chosen_KL_mean": -55.67822265625,
"KL/mean": -65.41090393066406,
"KL/rejected_KL_mean": -75.14356994628906,
"KL/std": 43.27152633666992,
"epoch": 0.41874527588813304,
"fcm_dpo/beta": 0.014150941744446754,
"fcm_dpo/delta": 0.12805846333503723,
"fcm_dpo/margin": 19.465354919433594,
"fcm_dpo/q_t": 0.43875253200531006,
"grad_norm": 19.56378746032715,
"learning_rate": 3.6219979505011555e-07,
"logits/chosen": 0.7127367258071899,
"logits/rejected": 0.7402825355529785,
"logps/chosen": -127.11121368408203,
"logps/ref_chosen": -71.43299102783203,
"logps/ref_rejected": -67.65852355957031,
"logps/rejected": -142.80209350585938,
"loss": 1.2434,
"margin_dpo/margin_mean": 19.465354919433594,
"margin_dpo/margin_std": 49.18296432495117,
"step": 277
},
{
"KL/chosen_KL_mean": -60.133182525634766,
"KL/mean": -76.22462463378906,
"KL/rejected_KL_mean": -92.31607055664062,
"KL/std": 48.23322296142578,
"epoch": 0.42025699168556313,
"fcm_dpo/beta": 0.01420608814805746,
"fcm_dpo/delta": -0.06040242686867714,
"fcm_dpo/margin": 32.18287658691406,
"fcm_dpo/q_t": 0.39822348952293396,
"grad_norm": 19.295873641967773,
"learning_rate": 3.6101665315144353e-07,
"logits/chosen": 0.5860699415206909,
"logits/rejected": 0.5357317924499512,
"logps/chosen": -127.24394989013672,
"logps/ref_chosen": -67.11076354980469,
"logps/ref_rejected": -88.74851989746094,
"logps/rejected": -181.06459045410156,
"loss": 1.1052,
"margin_dpo/margin_mean": 32.18288040161133,
"margin_dpo/margin_std": 51.11756134033203,
"step": 278
},
{
"KL/chosen_KL_mean": -45.40765380859375,
"KL/mean": -67.76667022705078,
"KL/rejected_KL_mean": -90.12568664550781,
"KL/std": 47.80771255493164,
"epoch": 0.4217687074829932,
"fcm_dpo/beta": 0.01364688016474247,
"fcm_dpo/delta": -0.2234923541545868,
"fcm_dpo/margin": 44.71803283691406,
"fcm_dpo/q_t": 0.36206403374671936,
"grad_norm": 17.90473175048828,
"learning_rate": 3.5983040587833563e-07,
"logits/chosen": 0.6358820796012878,
"logits/rejected": 0.5994046926498413,
"logps/chosen": -99.90513610839844,
"logps/ref_chosen": -54.49748611450195,
"logps/ref_rejected": -70.42373657226562,
"logps/rejected": -160.54940795898438,
"loss": 0.9606,
"margin_dpo/margin_mean": 44.71803283691406,
"margin_dpo/margin_std": 46.38705062866211,
"step": 279
},
{
"KL/chosen_KL_mean": -50.05751419067383,
"KL/mean": -72.81253051757812,
"KL/rejected_KL_mean": -95.56754302978516,
"KL/std": 50.09235382080078,
"epoch": 0.42328042328042326,
"fcm_dpo/beta": 0.013050587847828865,
"fcm_dpo/delta": -0.20617029070854187,
"fcm_dpo/margin": 45.51002883911133,
"fcm_dpo/q_t": 0.36619704961776733,
"grad_norm": 11.714454650878906,
"learning_rate": 3.586410864126781e-07,
"logits/chosen": 0.6924614906311035,
"logits/rejected": 0.6519962549209595,
"logps/chosen": -110.49032592773438,
"logps/ref_chosen": -60.43281173706055,
"logps/ref_rejected": -78.39051818847656,
"logps/rejected": -173.95806884765625,
"loss": 0.96,
"margin_dpo/margin_mean": 45.510032653808594,
"margin_dpo/margin_std": 46.505958557128906,
"step": 280
},
{
"KL/chosen_KL_mean": -54.632354736328125,
"KL/mean": -75.26278686523438,
"KL/rejected_KL_mean": -95.89321899414062,
"KL/std": 49.39891815185547,
"epoch": 0.42479213907785335,
"fcm_dpo/beta": 0.0126886535435915,
"fcm_dpo/delta": -0.13019640743732452,
"fcm_dpo/margin": 41.260860443115234,
"fcm_dpo/q_t": 0.38291144371032715,
"grad_norm": 13.580436706542969,
"learning_rate": 3.574487280222929e-07,
"logits/chosen": 0.657416582107544,
"logits/rejected": 0.6811779737472534,
"logps/chosen": -114.91444396972656,
"logps/ref_chosen": -60.2820930480957,
"logps/ref_rejected": -62.04009246826172,
"logps/rejected": -157.93331909179688,
"loss": 1.0354,
"margin_dpo/margin_mean": 41.260860443115234,
"margin_dpo/margin_std": 53.31696701049805,
"step": 281
},
{
"KL/chosen_KL_mean": -61.861053466796875,
"KL/mean": -80.40922546386719,
"KL/rejected_KL_mean": -98.9573974609375,
"KL/std": 49.61908721923828,
"epoch": 0.42630385487528344,
"fcm_dpo/beta": 0.012562556192278862,
"fcm_dpo/delta": -0.07028567790985107,
"fcm_dpo/margin": 37.09632873535156,
"fcm_dpo/q_t": 0.3977304995059967,
"grad_norm": 16.906856536865234,
"learning_rate": 3.562533640600075e-07,
"logits/chosen": 0.5947375893592834,
"logits/rejected": 0.5484437942504883,
"logps/chosen": -122.48497772216797,
"logps/ref_chosen": -60.623924255371094,
"logps/ref_rejected": -68.67400360107422,
"logps/rejected": -167.63140869140625,
"loss": 1.101,
"margin_dpo/margin_mean": 37.09632873535156,
"margin_dpo/margin_std": 57.269248962402344,
"step": 282
},
{
"KL/chosen_KL_mean": -62.06941604614258,
"KL/mean": -78.96687316894531,
"KL/rejected_KL_mean": -95.86431884765625,
"KL/std": 48.696449279785156,
"epoch": 0.42781557067271353,
"fcm_dpo/beta": 0.01240278035402298,
"fcm_dpo/delta": -0.02010105364024639,
"fcm_dpo/margin": 33.79491424560547,
"fcm_dpo/q_t": 0.4062108099460602,
"grad_norm": 15.976112365722656,
"learning_rate": 3.550550279627215e-07,
"logits/chosen": 0.651677131652832,
"logits/rejected": 0.5501687526702881,
"logps/chosen": -129.7171630859375,
"logps/ref_chosen": -67.64775085449219,
"logps/ref_rejected": -99.96835327148438,
"logps/rejected": -195.83267211914062,
"loss": 1.1202,
"margin_dpo/margin_mean": 33.79491424560547,
"margin_dpo/margin_std": 55.20383071899414,
"step": 283
},
{
"KL/chosen_KL_mean": -59.57140350341797,
"KL/mean": -77.59307861328125,
"KL/rejected_KL_mean": -95.61476135253906,
"KL/std": 52.5833740234375,
"epoch": 0.4293272864701436,
"fcm_dpo/beta": 0.012229856103658676,
"fcm_dpo/delta": -0.04312637448310852,
"fcm_dpo/margin": 36.043357849121094,
"fcm_dpo/q_t": 0.3998270630836487,
"grad_norm": 13.327878952026367,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 0.6895169019699097,
"logits/rejected": 0.6276400089263916,
"logps/chosen": -116.53883361816406,
"logps/ref_chosen": -56.96742630004883,
"logps/ref_rejected": -86.36236572265625,
"logps/rejected": -181.9771270751953,
"loss": 1.0829,
"margin_dpo/margin_mean": 36.043357849121094,
"margin_dpo/margin_std": 50.488067626953125,
"step": 284
},
{
"KL/chosen_KL_mean": -73.75648498535156,
"KL/mean": -88.84334564208984,
"KL/rejected_KL_mean": -103.93020629882812,
"KL/std": 47.19378662109375,
"epoch": 0.4308390022675737,
"fcm_dpo/beta": 0.01233905553817749,
"fcm_dpo/delta": 0.028463171795010567,
"fcm_dpo/margin": 30.173725128173828,
"fcm_dpo/q_t": 0.4166564345359802,
"grad_norm": 17.621870040893555,
"learning_rate": 3.5264957352549375e-07,
"logits/chosen": 0.6998270750045776,
"logits/rejected": 0.6734578013420105,
"logps/chosen": -145.41259765625,
"logps/ref_chosen": -71.65611267089844,
"logps/ref_rejected": -81.63829803466797,
"logps/rejected": -185.56851196289062,
"loss": 1.1462,
"margin_dpo/margin_mean": 30.173725128173828,
"margin_dpo/margin_std": 52.27867126464844,
"step": 285
},
{
"KL/chosen_KL_mean": -66.48196411132812,
"KL/mean": -90.62456512451172,
"KL/rejected_KL_mean": -114.76716613769531,
"KL/std": 50.86594772338867,
"epoch": 0.4323507180650038,
"fcm_dpo/beta": 0.011955272406339645,
"fcm_dpo/delta": -0.1885601282119751,
"fcm_dpo/margin": 48.285194396972656,
"fcm_dpo/q_t": 0.37013694643974304,
"grad_norm": 13.443557739257812,
"learning_rate": 3.514425224712835e-07,
"logits/chosen": 0.596250057220459,
"logits/rejected": 0.5051765441894531,
"logps/chosen": -127.56149291992188,
"logps/ref_chosen": -61.07952117919922,
"logps/ref_rejected": -91.28128051757812,
"logps/rejected": -206.04844665527344,
"loss": 0.9799,
"margin_dpo/margin_mean": 48.285194396972656,
"margin_dpo/margin_std": 52.92146301269531,
"step": 286
},
{
"KL/chosen_KL_mean": -54.13965606689453,
"KL/mean": -78.59452819824219,
"KL/rejected_KL_mean": -103.04940795898438,
"KL/std": 53.95775604248047,
"epoch": 0.43386243386243384,
"fcm_dpo/beta": 0.011567133478820324,
"fcm_dpo/delta": -0.17555159330368042,
"fcm_dpo/margin": 48.909759521484375,
"fcm_dpo/q_t": 0.37243402004241943,
"grad_norm": 12.431777000427246,
"learning_rate": 3.502326338516534e-07,
"logits/chosen": 0.6803244352340698,
"logits/rejected": 0.6446952819824219,
"logps/chosen": -100.17544555664062,
"logps/ref_chosen": -46.035789489746094,
"logps/ref_rejected": -59.95293426513672,
"logps/rejected": -163.00234985351562,
"loss": 0.9935,
"margin_dpo/margin_mean": 48.909759521484375,
"margin_dpo/margin_std": 55.84917449951172,
"step": 287
},
{
"KL/chosen_KL_mean": -69.79075622558594,
"KL/mean": -87.41785430908203,
"KL/rejected_KL_mean": -105.04496002197266,
"KL/std": 50.810791015625,
"epoch": 0.43537414965986393,
"fcm_dpo/beta": 0.011459792032837868,
"fcm_dpo/delta": -0.004200035706162453,
"fcm_dpo/margin": 35.254207611083984,
"fcm_dpo/q_t": 0.40890318155288696,
"grad_norm": 14.516646385192871,
"learning_rate": 3.490199415097892e-07,
"logits/chosen": 0.5388568639755249,
"logits/rejected": 0.4858013093471527,
"logps/chosen": -135.18161010742188,
"logps/ref_chosen": -65.3908462524414,
"logps/ref_rejected": -88.53607940673828,
"logps/rejected": -193.58103942871094,
"loss": 1.1212,
"margin_dpo/margin_mean": 35.254207611083984,
"margin_dpo/margin_std": 56.70970916748047,
"step": 288
},
{
"KL/chosen_KL_mean": -70.68275451660156,
"KL/mean": -86.9596939086914,
"KL/rejected_KL_mean": -103.23663330078125,
"KL/std": 50.765769958496094,
"epoch": 0.436885865457294,
"fcm_dpo/beta": 0.011538593098521233,
"fcm_dpo/delta": 0.024780981242656708,
"fcm_dpo/margin": 32.55389404296875,
"fcm_dpo/q_t": 0.41757404804229736,
"grad_norm": 17.976184844970703,
"learning_rate": 3.4780447936730247e-07,
"logits/chosen": 0.7239351272583008,
"logits/rejected": 0.6874663829803467,
"logps/chosen": -125.27638244628906,
"logps/ref_chosen": -54.5936279296875,
"logps/ref_rejected": -67.20855712890625,
"logps/rejected": -170.4451904296875,
"loss": 1.1549,
"margin_dpo/margin_mean": 32.55389404296875,
"margin_dpo/margin_std": 58.6180305480957,
"step": 289
},
{
"KL/chosen_KL_mean": -79.00485229492188,
"KL/mean": -99.15615844726562,
"KL/rejected_KL_mean": -119.30744934082031,
"KL/std": 49.97688293457031,
"epoch": 0.4383975812547241,
"fcm_dpo/beta": 0.011349892243742943,
"fcm_dpo/delta": -0.060778290033340454,
"fcm_dpo/margin": 40.3026123046875,
"fcm_dpo/q_t": 0.39717093110084534,
"grad_norm": 16.848310470581055,
"learning_rate": 3.465862814232821e-07,
"logits/chosen": 0.7556173205375671,
"logits/rejected": 0.6843053102493286,
"logps/chosen": -140.38943481445312,
"logps/ref_chosen": -61.38457489013672,
"logps/ref_rejected": -91.92778015136719,
"logps/rejected": -211.2352294921875,
"loss": 1.0929,
"margin_dpo/margin_mean": 40.3026123046875,
"margin_dpo/margin_std": 61.24738311767578,
"step": 290
},
{
"KL/chosen_KL_mean": -75.96649932861328,
"KL/mean": -96.99172973632812,
"KL/rejected_KL_mean": -118.01696014404297,
"KL/std": 49.590797424316406,
"epoch": 0.4399092970521542,
"fcm_dpo/beta": 0.011323593556880951,
"fcm_dpo/delta": -0.08079756796360016,
"fcm_dpo/margin": 42.05046081542969,
"fcm_dpo/q_t": 0.3937586545944214,
"grad_norm": 15.46800422668457,
"learning_rate": 3.4536538175334343e-07,
"logits/chosen": 0.800622284412384,
"logits/rejected": 0.7322646379470825,
"logps/chosen": -126.82952880859375,
"logps/ref_chosen": -50.863037109375,
"logps/ref_rejected": -82.20868682861328,
"logps/rejected": -200.22564697265625,
"loss": 1.0652,
"margin_dpo/margin_mean": 42.05046081542969,
"margin_dpo/margin_std": 56.03511047363281,
"step": 291
},
{
"KL/chosen_KL_mean": -74.34617614746094,
"KL/mean": -91.58586120605469,
"KL/rejected_KL_mean": -108.82554626464844,
"KL/std": 52.798606872558594,
"epoch": 0.4414210128495843,
"fcm_dpo/beta": 0.011205028742551804,
"fcm_dpo/delta": 0.014202935621142387,
"fcm_dpo/margin": 34.47936248779297,
"fcm_dpo/q_t": 0.41324666142463684,
"grad_norm": 15.464279174804688,
"learning_rate": 3.4414181450867465e-07,
"logits/chosen": 0.7104899287223816,
"logits/rejected": 0.662022590637207,
"logps/chosen": -138.695068359375,
"logps/ref_chosen": -64.34888458251953,
"logps/ref_rejected": -72.86434173583984,
"logps/rejected": -181.68988037109375,
"loss": 1.1459,
"margin_dpo/margin_mean": 34.47936248779297,
"margin_dpo/margin_std": 60.98603820800781,
"step": 292
},
{
"KL/chosen_KL_mean": -74.22430419921875,
"KL/mean": -98.60188293457031,
"KL/rejected_KL_mean": -122.9794692993164,
"KL/std": 54.332801818847656,
"epoch": 0.4429327286470144,
"fcm_dpo/beta": 0.011021770536899567,
"fcm_dpo/delta": -0.14493146538734436,
"fcm_dpo/margin": 48.75517272949219,
"fcm_dpo/q_t": 0.3803362250328064,
"grad_norm": 11.889296531677246,
"learning_rate": 3.4291561391508185e-07,
"logits/chosen": 0.8006993532180786,
"logits/rejected": 0.7123322486877441,
"logps/chosen": -129.09376525878906,
"logps/ref_chosen": -54.869468688964844,
"logps/ref_rejected": -81.858642578125,
"logps/rejected": -204.83810424804688,
"loss": 1.0462,
"margin_dpo/margin_mean": 48.75517272949219,
"margin_dpo/margin_std": 66.53107452392578,
"step": 293
},
{
"KL/chosen_KL_mean": -77.36056518554688,
"KL/mean": -93.45787811279297,
"KL/rejected_KL_mean": -109.55517578125,
"KL/std": 55.53190612792969,
"epoch": 0.4444444444444444,
"fcm_dpo/beta": 0.010932950302958488,
"fcm_dpo/delta": 0.04968990758061409,
"fcm_dpo/margin": 32.194602966308594,
"fcm_dpo/q_t": 0.42159321904182434,
"grad_norm": 12.925803184509277,
"learning_rate": 3.4168681427203153e-07,
"logits/chosen": 0.7118106484413147,
"logits/rejected": 0.6668426394462585,
"logps/chosen": -134.03146362304688,
"logps/ref_chosen": -56.670902252197266,
"logps/ref_rejected": -70.32819366455078,
"logps/rejected": -179.88336181640625,
"loss": 1.144,
"margin_dpo/margin_mean": 32.19460678100586,
"margin_dpo/margin_std": 53.10637664794922,
"step": 294
},
{
"KL/chosen_KL_mean": -82.76451110839844,
"KL/mean": -98.13732147216797,
"KL/rejected_KL_mean": -113.5101318359375,
"KL/std": 53.655540466308594,
"epoch": 0.4459561602418745,
"fcm_dpo/beta": 0.01107887364923954,
"fcm_dpo/delta": 0.06150873750448227,
"fcm_dpo/margin": 30.74562644958496,
"fcm_dpo/q_t": 0.42415887117385864,
"grad_norm": 19.125263214111328,
"learning_rate": 3.4045544995169125e-07,
"logits/chosen": 0.7097787857055664,
"logits/rejected": 0.6080629229545593,
"logps/chosen": -133.1654052734375,
"logps/ref_chosen": -50.40088653564453,
"logps/ref_rejected": -83.43521881103516,
"logps/rejected": -196.94534301757812,
"loss": 1.17,
"margin_dpo/margin_mean": 30.745624542236328,
"margin_dpo/margin_std": 57.66575241088867,
"step": 295
},
{
"KL/chosen_KL_mean": -82.25225830078125,
"KL/mean": -101.54887390136719,
"KL/rejected_KL_mean": -120.84550476074219,
"KL/std": 55.17761993408203,
"epoch": 0.4474678760393046,
"fcm_dpo/beta": 0.011037503369152546,
"fcm_dpo/delta": -0.027714837342500687,
"fcm_dpo/margin": 38.59325408935547,
"fcm_dpo/q_t": 0.40471675992012024,
"grad_norm": 13.697413444519043,
"learning_rate": 3.392215553979679e-07,
"logits/chosen": 0.6670588254928589,
"logits/rejected": 0.623749852180481,
"logps/chosen": -151.40260314941406,
"logps/ref_chosen": -69.15034484863281,
"logps/ref_rejected": -89.60166931152344,
"logps/rejected": -210.44717407226562,
"loss": 1.1139,
"margin_dpo/margin_mean": 38.593257904052734,
"margin_dpo/margin_std": 61.52368927001953,
"step": 296
},
{
"KL/chosen_KL_mean": -86.82210540771484,
"KL/mean": -108.19435119628906,
"KL/rejected_KL_mean": -129.56661987304688,
"KL/std": 49.4395751953125,
"epoch": 0.4489795918367347,
"fcm_dpo/beta": 0.011007674038410187,
"fcm_dpo/delta": -0.07413952797651291,
"fcm_dpo/margin": 42.744510650634766,
"fcm_dpo/q_t": 0.3928752541542053,
"grad_norm": 13.25504207611084,
"learning_rate": 3.3798516512554485e-07,
"logits/chosen": 0.7025403380393982,
"logits/rejected": 0.6459665298461914,
"logps/chosen": -144.83840942382812,
"logps/ref_chosen": -58.01630401611328,
"logps/ref_rejected": -69.95780944824219,
"logps/rejected": -199.5244140625,
"loss": 1.0521,
"margin_dpo/margin_mean": 42.744510650634766,
"margin_dpo/margin_std": 53.205196380615234,
"step": 297
},
{
"KL/chosen_KL_mean": -85.28236389160156,
"KL/mean": -101.26274108886719,
"KL/rejected_KL_mean": -117.24310302734375,
"KL/std": 52.560821533203125,
"epoch": 0.4504913076341648,
"fcm_dpo/beta": 0.01099710538983345,
"fcm_dpo/delta": 0.05027089640498161,
"fcm_dpo/margin": 31.960744857788086,
"fcm_dpo/q_t": 0.42255425453186035,
"grad_norm": 13.888681411743164,
"learning_rate": 3.367463137189156e-07,
"logits/chosen": 0.7887052297592163,
"logits/rejected": 0.7301384210586548,
"logps/chosen": -141.45167541503906,
"logps/ref_chosen": -56.1693115234375,
"logps/ref_rejected": -68.55052185058594,
"logps/rejected": -185.7936248779297,
"loss": 1.1817,
"margin_dpo/margin_mean": 31.960742950439453,
"margin_dpo/margin_std": 63.88176345825195,
"step": 298
},
{
"KL/chosen_KL_mean": -83.48313903808594,
"KL/mean": -97.56495666503906,
"KL/rejected_KL_mean": -111.64677429199219,
"KL/std": 51.03688049316406,
"epoch": 0.4520030234315949,
"fcm_dpo/beta": 0.011011083610355854,
"fcm_dpo/delta": -0.004602404776960611,
"fcm_dpo/margin": 28.16363525390625,
"fcm_dpo/q_t": 0.43071186542510986,
"grad_norm": 17.668432235717773,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": 0.6086280345916748,
"logits/rejected": 0.5812957882881165,
"logps/chosen": -145.80093383789062,
"logps/ref_chosen": -62.31780242919922,
"logps/ref_rejected": -72.60028839111328,
"logps/rejected": -184.2470703125,
"loss": 1.2236,
"margin_dpo/margin_mean": 28.16363525390625,
"margin_dpo/margin_std": 64.58142852783203,
"step": 299
},
{
"KL/chosen_KL_mean": -84.61534118652344,
"KL/mean": -102.70570373535156,
"KL/rejected_KL_mean": -120.79605102539062,
"KL/std": 51.72077178955078,
"epoch": 0.45351473922902497,
"fcm_dpo/beta": 0.01102392002940178,
"fcm_dpo/delta": 0.0011731302365660667,
"fcm_dpo/margin": 36.18071746826172,
"fcm_dpo/q_t": 0.4104636311531067,
"grad_norm": 14.278905868530273,
"learning_rate": 3.3426136618426043e-07,
"logits/chosen": 0.7229694724082947,
"logits/rejected": 0.6558288335800171,
"logps/chosen": -144.99691772460938,
"logps/ref_chosen": -60.38157653808594,
"logps/ref_rejected": -75.45442199707031,
"logps/rejected": -196.25047302246094,
"loss": 1.1384,
"margin_dpo/margin_mean": 36.18071746826172,
"margin_dpo/margin_std": 62.46015167236328,
"step": 300
},
{
"KL/chosen_KL_mean": -82.17597198486328,
"KL/mean": -98.3734130859375,
"KL/rejected_KL_mean": -114.57086181640625,
"KL/std": 51.439849853515625,
"epoch": 0.455026455026455,
"fcm_dpo/beta": 0.011054832488298416,
"fcm_dpo/delta": 0.04340054839849472,
"fcm_dpo/margin": 32.39488983154297,
"fcm_dpo/q_t": 0.4193943440914154,
"grad_norm": 13.773794174194336,
"learning_rate": 3.3301533956555885e-07,
"logits/chosen": 0.7545723915100098,
"logits/rejected": 0.7267623543739319,
"logps/chosen": -135.02685546875,
"logps/ref_chosen": -52.85089111328125,
"logps/ref_rejected": -69.97584533691406,
"logps/rejected": -184.5467071533203,
"loss": 1.1641,
"margin_dpo/margin_mean": 32.39488983154297,
"margin_dpo/margin_std": 60.21562194824219,
"step": 301
},
{
"KL/chosen_KL_mean": -86.02842712402344,
"KL/mean": -98.03132629394531,
"KL/rejected_KL_mean": -110.03424072265625,
"KL/std": 51.48082733154297,
"epoch": 0.4565381708238851,
"fcm_dpo/beta": 0.011317036114633083,
"fcm_dpo/delta": 0.13204258680343628,
"fcm_dpo/margin": 24.005821228027344,
"fcm_dpo/q_t": 0.43845057487487793,
"grad_norm": 18.665828704833984,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 0.5853751301765442,
"logits/rejected": 0.5305138230323792,
"logps/chosen": -152.99493408203125,
"logps/ref_chosen": -66.96650695800781,
"logps/ref_rejected": -88.09510803222656,
"logps/rejected": -198.12936401367188,
"loss": 1.2315,
"margin_dpo/margin_mean": 24.005821228027344,
"margin_dpo/margin_std": 57.24117660522461,
"step": 302
},
{
"KL/chosen_KL_mean": -76.08708190917969,
"KL/mean": -98.33299255371094,
"KL/rejected_KL_mean": -120.57888793945312,
"KL/std": 52.373069763183594,
"epoch": 0.4580498866213152,
"fcm_dpo/beta": 0.01123693585395813,
"fcm_dpo/delta": -0.10515578836202621,
"fcm_dpo/margin": 44.49180603027344,
"fcm_dpo/q_t": 0.3883194327354431,
"grad_norm": 12.11741828918457,
"learning_rate": 3.3051635489464793e-07,
"logits/chosen": 0.6791602373123169,
"logits/rejected": 0.6135026216506958,
"logps/chosen": -138.2086181640625,
"logps/ref_chosen": -62.12152862548828,
"logps/ref_rejected": -90.31204223632812,
"logps/rejected": -210.89093017578125,
"loss": 1.0705,
"margin_dpo/margin_mean": 44.49180603027344,
"margin_dpo/margin_std": 64.82666015625,
"step": 303
},
{
"KL/chosen_KL_mean": -65.99530792236328,
"KL/mean": -87.9817886352539,
"KL/rejected_KL_mean": -109.96827697753906,
"KL/std": 54.95054626464844,
"epoch": 0.4595616024187453,
"fcm_dpo/beta": 0.011013105511665344,
"fcm_dpo/delta": -0.08866756409406662,
"fcm_dpo/margin": 43.97296142578125,
"fcm_dpo/q_t": 0.3877463936805725,
"grad_norm": 13.452021598815918,
"learning_rate": 3.292634667444117e-07,
"logits/chosen": 0.6698247790336609,
"logits/rejected": 0.6164925694465637,
"logps/chosen": -126.69039916992188,
"logps/ref_chosen": -60.695091247558594,
"logps/ref_rejected": -78.2525405883789,
"logps/rejected": -188.2208251953125,
"loss": 1.0307,
"margin_dpo/margin_mean": 43.97296142578125,
"margin_dpo/margin_std": 49.85572052001953,
"step": 304
},
{
"KL/chosen_KL_mean": -77.2333984375,
"KL/mean": -93.83995819091797,
"KL/rejected_KL_mean": -110.44651794433594,
"KL/std": 52.10803985595703,
"epoch": 0.46107331821617537,
"fcm_dpo/beta": 0.01098443754017353,
"fcm_dpo/delta": 0.036268450319767,
"fcm_dpo/margin": 33.213104248046875,
"fcm_dpo/q_t": 0.41780638694763184,
"grad_norm": 13.191741943359375,
"learning_rate": 3.280083614246217e-07,
"logits/chosen": 0.6296533942222595,
"logits/rejected": 0.6592621803283691,
"logps/chosen": -149.9325408935547,
"logps/ref_chosen": -72.69914245605469,
"logps/ref_rejected": -65.65670776367188,
"logps/rejected": -176.1032257080078,
"loss": 1.1659,
"margin_dpo/margin_mean": 33.21310806274414,
"margin_dpo/margin_std": 61.98444366455078,
"step": 305
},
{
"KL/chosen_KL_mean": -71.7294921875,
"KL/mean": -89.06037902832031,
"KL/rejected_KL_mean": -106.39128112792969,
"KL/std": 50.25974655151367,
"epoch": 0.46258503401360546,
"fcm_dpo/beta": 0.011017680168151855,
"fcm_dpo/delta": 0.01821252331137657,
"fcm_dpo/margin": 34.661781311035156,
"fcm_dpo/q_t": 0.4125592112541199,
"grad_norm": 13.104958534240723,
"learning_rate": 3.267510740432719e-07,
"logits/chosen": 0.7577117681503296,
"logits/rejected": 0.6426206827163696,
"logps/chosen": -125.70002746582031,
"logps/ref_chosen": -53.97052764892578,
"logps/ref_rejected": -71.02423095703125,
"logps/rejected": -177.41551208496094,
"loss": 1.1106,
"margin_dpo/margin_mean": 34.661781311035156,
"margin_dpo/margin_std": 48.462059020996094,
"step": 306
},
{
"KL/chosen_KL_mean": -67.007568359375,
"KL/mean": -75.14314270019531,
"KL/rejected_KL_mean": -83.2787094116211,
"KL/std": 48.46715545654297,
"epoch": 0.46409674981103555,
"fcm_dpo/beta": 0.011178033426404,
"fcm_dpo/delta": 0.07564892619848251,
"fcm_dpo/margin": 16.271129608154297,
"fcm_dpo/q_t": 0.4605118930339813,
"grad_norm": 17.651695251464844,
"learning_rate": 3.2549163976939285e-07,
"logits/chosen": 0.7322758436203003,
"logits/rejected": 0.6832484602928162,
"logps/chosen": -124.4206771850586,
"logps/ref_chosen": -57.413108825683594,
"logps/ref_rejected": -68.68010711669922,
"logps/rejected": -151.9588165283203,
"loss": 1.3258,
"margin_dpo/margin_mean": 16.271129608154297,
"margin_dpo/margin_std": 61.56073760986328,
"step": 307
},
{
"KL/chosen_KL_mean": -65.22660064697266,
"KL/mean": -81.66844177246094,
"KL/rejected_KL_mean": -98.11026000976562,
"KL/std": 50.401756286621094,
"epoch": 0.4656084656084656,
"fcm_dpo/beta": 0.011273292824625969,
"fcm_dpo/delta": 0.030268091708421707,
"fcm_dpo/margin": 32.883670806884766,
"fcm_dpo/q_t": 0.41592592000961304,
"grad_norm": 11.697486877441406,
"learning_rate": 3.2423009383206874e-07,
"logits/chosen": 0.6642824411392212,
"logits/rejected": 0.6534437537193298,
"logps/chosen": -131.8253936767578,
"logps/ref_chosen": -66.59879302978516,
"logps/ref_rejected": -74.337158203125,
"logps/rejected": -172.44741821289062,
"loss": 1.1495,
"margin_dpo/margin_mean": 32.88367462158203,
"margin_dpo/margin_std": 57.51547622680664,
"step": 308
},
{
"KL/chosen_KL_mean": -75.81356811523438,
"KL/mean": -92.64073181152344,
"KL/rejected_KL_mean": -109.4678955078125,
"KL/std": 48.065574645996094,
"epoch": 0.4671201814058957,
"fcm_dpo/beta": 0.011321078054606915,
"fcm_dpo/delta": 0.019529415294528008,
"fcm_dpo/margin": 33.654327392578125,
"fcm_dpo/q_t": 0.4130924940109253,
"grad_norm": 11.823284149169922,
"learning_rate": 3.229664715194511e-07,
"logits/chosen": 0.7302178144454956,
"logits/rejected": 0.6720554232597351,
"logps/chosen": -141.20831298828125,
"logps/ref_chosen": -65.39474487304688,
"logps/ref_rejected": -75.70930480957031,
"logps/rejected": -185.1772003173828,
"loss": 1.1212,
"margin_dpo/margin_mean": 33.65432357788086,
"margin_dpo/margin_std": 50.88998031616211,
"step": 309
},
{
"KL/chosen_KL_mean": -77.10743713378906,
"KL/mean": -87.3907699584961,
"KL/rejected_KL_mean": -97.67410278320312,
"KL/std": 44.922515869140625,
"epoch": 0.46863189720332576,
"fcm_dpo/beta": 0.011529898270964622,
"fcm_dpo/delta": 0.0674857497215271,
"fcm_dpo/margin": 20.566661834716797,
"fcm_dpo/q_t": 0.44818443059921265,
"grad_norm": 13.843379020690918,
"learning_rate": 3.2170080817777257e-07,
"logits/chosen": 0.7096024751663208,
"logits/rejected": 0.6966167688369751,
"logps/chosen": -151.7757110595703,
"logps/ref_chosen": -74.66827392578125,
"logps/ref_rejected": -80.5689697265625,
"logps/rejected": -178.24307250976562,
"loss": 1.2565,
"margin_dpo/margin_mean": 20.566661834716797,
"margin_dpo/margin_std": 54.395755767822266,
"step": 310
},
{
"KL/chosen_KL_mean": -62.1547737121582,
"KL/mean": -78.7245101928711,
"KL/rejected_KL_mean": -95.29425048828125,
"KL/std": 52.89256286621094,
"epoch": 0.47014361300075586,
"fcm_dpo/beta": 0.011499082669615746,
"fcm_dpo/delta": 0.019153833389282227,
"fcm_dpo/margin": 33.13947677612305,
"fcm_dpo/q_t": 0.41445329785346985,
"grad_norm": 13.526447296142578,
"learning_rate": 3.204331392103574e-07,
"logits/chosen": 0.5896681547164917,
"logits/rejected": 0.4513469934463501,
"logps/chosen": -121.89280700683594,
"logps/ref_chosen": -59.738033294677734,
"logps/ref_rejected": -93.60757446289062,
"logps/rejected": -188.90182495117188,
"loss": 1.1292,
"margin_dpo/margin_mean": 33.13947677612305,
"margin_dpo/margin_std": 52.75567626953125,
"step": 311
},
{
"KL/chosen_KL_mean": -64.8100814819336,
"KL/mean": -86.31684875488281,
"KL/rejected_KL_mean": -107.82361602783203,
"KL/std": 49.82930374145508,
"epoch": 0.47165532879818595,
"fcm_dpo/beta": 0.011444027535617352,
"fcm_dpo/delta": -0.09691999107599258,
"fcm_dpo/margin": 43.01353454589844,
"fcm_dpo/q_t": 0.3865237832069397,
"grad_norm": 13.072991371154785,
"learning_rate": 3.1916350007663176e-07,
"logits/chosen": 0.7030187845230103,
"logits/rejected": 0.6059480905532837,
"logps/chosen": -118.62651824951172,
"logps/ref_chosen": -53.816436767578125,
"logps/ref_rejected": -68.6575698852539,
"logps/rejected": -176.48118591308594,
"loss": 1.0276,
"margin_dpo/margin_mean": 43.01353454589844,
"margin_dpo/margin_std": 49.13404846191406,
"step": 312
},
{
"KL/chosen_KL_mean": -66.22637176513672,
"KL/mean": -77.51238250732422,
"KL/rejected_KL_mean": -88.79839324951172,
"KL/std": 47.805538177490234,
"epoch": 0.47316704459561604,
"fcm_dpo/beta": 0.011595704592764378,
"fcm_dpo/delta": 0.14197511970996857,
"fcm_dpo/margin": 22.5720272064209,
"fcm_dpo/q_t": 0.44253993034362793,
"grad_norm": 12.121759414672852,
"learning_rate": 3.178919262911314e-07,
"logits/chosen": 0.7611916065216064,
"logits/rejected": 0.7401007413864136,
"logps/chosen": -126.18373107910156,
"logps/ref_chosen": -59.957359313964844,
"logps/ref_rejected": -69.31729888916016,
"logps/rejected": -158.11569213867188,
"loss": 1.2306,
"margin_dpo/margin_mean": 22.572025299072266,
"margin_dpo/margin_std": 52.969276428222656,
"step": 313
},
{
"KL/chosen_KL_mean": -63.15602111816406,
"KL/mean": -85.6715316772461,
"KL/rejected_KL_mean": -108.18704223632812,
"KL/std": 50.80717468261719,
"epoch": 0.47467876039304613,
"fcm_dpo/beta": 0.011435450986027718,
"fcm_dpo/delta": -0.12173415720462799,
"fcm_dpo/margin": 45.03102111816406,
"fcm_dpo/q_t": 0.3847067356109619,
"grad_norm": 12.349407196044922,
"learning_rate": 3.166184534225087e-07,
"logits/chosen": 0.6897181272506714,
"logits/rejected": 0.7231118679046631,
"logps/chosen": -133.42417907714844,
"logps/ref_chosen": -70.26815795898438,
"logps/ref_rejected": -69.23971557617188,
"logps/rejected": -177.4267578125,
"loss": 1.0309,
"margin_dpo/margin_mean": 45.03102111816406,
"margin_dpo/margin_std": 55.811279296875,
"step": 314
},
{
"KL/chosen_KL_mean": -69.03983306884766,
"KL/mean": -86.34467315673828,
"KL/rejected_KL_mean": -103.64952087402344,
"KL/std": 49.86646270751953,
"epoch": 0.47619047619047616,
"fcm_dpo/beta": 0.011460809037089348,
"fcm_dpo/delta": 0.0029491260647773743,
"fcm_dpo/margin": 34.609683990478516,
"fcm_dpo/q_t": 0.4089590907096863,
"grad_norm": 12.50733470916748,
"learning_rate": 3.1534311709253723e-07,
"logits/chosen": 0.612759530544281,
"logits/rejected": 0.5756454467773438,
"logps/chosen": -136.83453369140625,
"logps/ref_chosen": -67.79469299316406,
"logps/ref_rejected": -74.55148315429688,
"logps/rejected": -178.20098876953125,
"loss": 1.1097,
"margin_dpo/margin_mean": 34.60968017578125,
"margin_dpo/margin_std": 50.45848846435547,
"step": 315
},
{
"KL/chosen_KL_mean": -64.08145141601562,
"KL/mean": -86.38172149658203,
"KL/rejected_KL_mean": -108.68197631835938,
"KL/std": 51.62786865234375,
"epoch": 0.47770219198790626,
"fcm_dpo/beta": 0.011328795924782753,
"fcm_dpo/delta": -0.1117531955242157,
"fcm_dpo/margin": 44.60052490234375,
"fcm_dpo/q_t": 0.38467687368392944,
"grad_norm": 13.304482460021973,
"learning_rate": 3.1406595297511564e-07,
"logits/chosen": 0.5855288505554199,
"logits/rejected": 0.4543311297893524,
"logps/chosen": -119.36993408203125,
"logps/ref_chosen": -55.288482666015625,
"logps/ref_rejected": -96.15723419189453,
"logps/rejected": -204.83920288085938,
"loss": 1.0241,
"margin_dpo/margin_mean": 44.60052490234375,
"margin_dpo/margin_std": 48.77356719970703,
"step": 316
},
{
"KL/chosen_KL_mean": -61.04408264160156,
"KL/mean": -84.79362487792969,
"KL/rejected_KL_mean": -108.54316711425781,
"KL/std": 49.97541809082031,
"epoch": 0.47921390778533635,
"fcm_dpo/beta": 0.010910360142588615,
"fcm_dpo/delta": -0.12534061074256897,
"fcm_dpo/margin": 47.49908447265625,
"fcm_dpo/q_t": 0.3813457787036896,
"grad_norm": 17.07743263244629,
"learning_rate": 3.1278699679526975e-07,
"logits/chosen": 0.7253998517990112,
"logits/rejected": 0.6797171831130981,
"logps/chosen": -115.62545776367188,
"logps/ref_chosen": -54.58137512207031,
"logps/ref_rejected": -72.77232360839844,
"logps/rejected": -181.31549072265625,
"loss": 1.0162,
"margin_dpo/margin_mean": 47.49908447265625,
"margin_dpo/margin_std": 54.72552490234375,
"step": 317
},
{
"KL/chosen_KL_mean": -70.27217102050781,
"KL/mean": -86.56277465820312,
"KL/rejected_KL_mean": -102.85338592529297,
"KL/std": 53.20188522338867,
"epoch": 0.48072562358276644,
"fcm_dpo/beta": 0.010931117460131645,
"fcm_dpo/delta": 0.04548676684498787,
"fcm_dpo/margin": 32.581199645996094,
"fcm_dpo/q_t": 0.42247825860977173,
"grad_norm": 13.316046714782715,
"learning_rate": 3.1150628432815336e-07,
"logits/chosen": 0.7100570201873779,
"logits/rejected": 0.6408475041389465,
"logps/chosen": -123.16040802001953,
"logps/ref_chosen": -52.88822937011719,
"logps/ref_rejected": -80.63988494873047,
"logps/rejected": -183.49327087402344,
"loss": 1.1896,
"margin_dpo/margin_mean": 32.58120346069336,
"margin_dpo/margin_std": 67.96830749511719,
"step": 318
},
{
"KL/chosen_KL_mean": -66.3150634765625,
"KL/mean": -89.25406646728516,
"KL/rejected_KL_mean": -112.19306945800781,
"KL/std": 54.300323486328125,
"epoch": 0.48223733938019653,
"fcm_dpo/beta": 0.010813157074153423,
"fcm_dpo/delta": -0.10109373182058334,
"fcm_dpo/margin": 45.87800598144531,
"fcm_dpo/q_t": 0.38941460847854614,
"grad_norm": 13.561705589294434,
"learning_rate": 3.1022385139804707e-07,
"logits/chosen": 0.6610653400421143,
"logits/rejected": 0.6455733776092529,
"logps/chosen": -130.67840576171875,
"logps/ref_chosen": -64.36333465576172,
"logps/ref_rejected": -79.47296142578125,
"logps/rejected": -191.666015625,
"loss": 1.0555,
"margin_dpo/margin_mean": 45.87800598144531,
"margin_dpo/margin_std": 62.763153076171875,
"step": 319
},
{
"KL/chosen_KL_mean": -64.1065444946289,
"KL/mean": -82.70232391357422,
"KL/rejected_KL_mean": -101.29811096191406,
"KL/std": 54.51066589355469,
"epoch": 0.4837490551776266,
"fcm_dpo/beta": 0.010627730749547482,
"fcm_dpo/delta": -0.12094675749540329,
"fcm_dpo/margin": 37.19155502319336,
"fcm_dpo/q_t": 0.41220274567604065,
"grad_norm": 13.543227195739746,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 0.5872669219970703,
"logits/rejected": 0.5465952157974243,
"logps/chosen": -113.66529083251953,
"logps/ref_chosen": -49.558746337890625,
"logps/ref_rejected": -71.23444366455078,
"logps/rejected": -172.53256225585938,
"loss": 1.1346,
"margin_dpo/margin_mean": 37.191551208496094,
"margin_dpo/margin_std": 59.02598571777344,
"step": 320
},
{
"KL/chosen_KL_mean": -69.19467163085938,
"KL/mean": -90.484619140625,
"KL/rejected_KL_mean": -111.77458190917969,
"KL/std": 52.70685577392578,
"epoch": 0.4852607709750567,
"fcm_dpo/beta": 0.010346543043851852,
"fcm_dpo/delta": -0.04412151500582695,
"fcm_dpo/margin": 42.57990264892578,
"fcm_dpo/q_t": 0.39923253655433655,
"grad_norm": 19.086502075195312,
"learning_rate": 3.0765396768561004e-07,
"logits/chosen": 0.6853651404380798,
"logits/rejected": 0.6693944931030273,
"logps/chosen": -121.27993774414062,
"logps/ref_chosen": -52.08526611328125,
"logps/ref_rejected": -55.58674621582031,
"logps/rejected": -167.361328125,
"loss": 1.0943,
"margin_dpo/margin_mean": 42.57990264892578,
"margin_dpo/margin_std": 60.928245544433594,
"step": 321
},
{
"KL/chosen_KL_mean": -80.42525482177734,
"KL/mean": -105.82083129882812,
"KL/rejected_KL_mean": -131.21641540527344,
"KL/std": 57.914947509765625,
"epoch": 0.48677248677248675,
"fcm_dpo/beta": 0.01023766677826643,
"fcm_dpo/delta": -0.12638047337532043,
"fcm_dpo/margin": 50.791160583496094,
"fcm_dpo/q_t": 0.3809961676597595,
"grad_norm": 12.457335472106934,
"learning_rate": 3.063665887884511e-07,
"logits/chosen": 0.7201390862464905,
"logits/rejected": 0.6368743777275085,
"logps/chosen": -127.82936096191406,
"logps/ref_chosen": -47.404109954833984,
"logps/ref_rejected": -73.4260025024414,
"logps/rejected": -204.64242553710938,
"loss": 1.0166,
"margin_dpo/margin_mean": 50.791160583496094,
"margin_dpo/margin_std": 58.66703796386719,
"step": 322
},
{
"KL/chosen_KL_mean": -82.94509887695312,
"KL/mean": -99.22185516357422,
"KL/rejected_KL_mean": -115.49859619140625,
"KL/std": 56.19465637207031,
"epoch": 0.48828420256991684,
"fcm_dpo/beta": 0.010281499475240707,
"fcm_dpo/delta": 0.06715258955955505,
"fcm_dpo/margin": 32.553504943847656,
"fcm_dpo/q_t": 0.4272102117538452,
"grad_norm": 13.987832069396973,
"learning_rate": 3.0507763319663517e-07,
"logits/chosen": 0.625502347946167,
"logits/rejected": 0.5450081825256348,
"logps/chosen": -152.95140075683594,
"logps/ref_chosen": -70.00630187988281,
"logps/ref_rejected": -86.96690368652344,
"logps/rejected": -202.4654998779297,
"loss": 1.2019,
"margin_dpo/margin_mean": 32.55350112915039,
"margin_dpo/margin_std": 70.55862426757812,
"step": 323
},
{
"KL/chosen_KL_mean": -70.84347534179688,
"KL/mean": -94.74235534667969,
"KL/rejected_KL_mean": -118.64125061035156,
"KL/std": 60.21238708496094,
"epoch": 0.4897959183673469,
"fcm_dpo/beta": 0.010116002522408962,
"fcm_dpo/delta": -0.08829785138368607,
"fcm_dpo/margin": 47.79777908325195,
"fcm_dpo/q_t": 0.3895995616912842,
"grad_norm": 17.79944610595703,
"learning_rate": 3.0378713696502097e-07,
"logits/chosen": 0.7284420728683472,
"logits/rejected": 0.6720010042190552,
"logps/chosen": -126.73229217529297,
"logps/ref_chosen": -55.88882064819336,
"logps/ref_rejected": -75.23088073730469,
"logps/rejected": -193.87213134765625,
"loss": 1.0384,
"margin_dpo/margin_mean": 47.79777908325195,
"margin_dpo/margin_std": 57.43890380859375,
"step": 324
},
{
"KL/chosen_KL_mean": -89.91795349121094,
"KL/mean": -110.92127990722656,
"KL/rejected_KL_mean": -131.9246063232422,
"KL/std": 56.947425842285156,
"epoch": 0.491307634164777,
"fcm_dpo/beta": 0.010026042349636555,
"fcm_dpo/delta": -0.022590894252061844,
"fcm_dpo/margin": 42.006649017333984,
"fcm_dpo/q_t": 0.4037303328514099,
"grad_norm": 15.0574312210083,
"learning_rate": 3.0249513619156206e-07,
"logits/chosen": 0.7013384699821472,
"logits/rejected": 0.6352590322494507,
"logps/chosen": -154.06497192382812,
"logps/ref_chosen": -64.14701843261719,
"logps/ref_rejected": -79.91143798828125,
"logps/rejected": -211.83602905273438,
"loss": 1.1117,
"margin_dpo/margin_mean": 42.006649017333984,
"margin_dpo/margin_std": 65.85514831542969,
"step": 325
},
{
"KL/chosen_KL_mean": -104.94251251220703,
"KL/mean": -114.92540740966797,
"KL/rejected_KL_mean": -124.9083023071289,
"KL/std": 58.84989929199219,
"epoch": 0.4928193499622071,
"fcm_dpo/beta": 0.010204941034317017,
"fcm_dpo/delta": 0.07210341840982437,
"fcm_dpo/margin": 19.965795516967773,
"fcm_dpo/q_t": 0.454483300447464,
"grad_norm": 14.433613777160645,
"learning_rate": 3.012016670162977e-07,
"logits/chosen": 0.6130670309066772,
"logits/rejected": 0.6195484399795532,
"logps/chosen": -180.47381591796875,
"logps/ref_chosen": -75.53131103515625,
"logps/ref_rejected": -76.5898666381836,
"logps/rejected": -201.4981689453125,
"loss": 1.2969,
"margin_dpo/margin_mean": 19.965797424316406,
"margin_dpo/margin_std": 63.97681427001953,
"step": 326
},
{
"KL/chosen_KL_mean": -98.93629455566406,
"KL/mean": -115.67805480957031,
"KL/rejected_KL_mean": -132.41981506347656,
"KL/std": 60.78108215332031,
"epoch": 0.4943310657596372,
"fcm_dpo/beta": 0.010323995724320412,
"fcm_dpo/delta": 0.055861108005046844,
"fcm_dpo/margin": 33.48351287841797,
"fcm_dpo/q_t": 0.42331814765930176,
"grad_norm": 16.34779930114746,
"learning_rate": 2.99906765620341e-07,
"logits/chosen": 0.5977568030357361,
"logits/rejected": 0.5657069683074951,
"logps/chosen": -168.27346801757812,
"logps/ref_chosen": -69.33717346191406,
"logps/ref_rejected": -73.37751770019531,
"logps/rejected": -205.79733276367188,
"loss": 1.1836,
"margin_dpo/margin_mean": 33.48351287841797,
"margin_dpo/margin_std": 66.8410415649414,
"step": 327
},
{
"KL/chosen_KL_mean": -85.42854309082031,
"KL/mean": -106.07657623291016,
"KL/rejected_KL_mean": -126.72460174560547,
"KL/std": 61.60851287841797,
"epoch": 0.4958427815570673,
"fcm_dpo/beta": 0.01029128022491932,
"fcm_dpo/delta": -0.026095092296600342,
"fcm_dpo/margin": 41.296051025390625,
"fcm_dpo/q_t": 0.4046742916107178,
"grad_norm": 13.140426635742188,
"learning_rate": 2.9861046822486766e-07,
"logits/chosen": 0.6051807999610901,
"logits/rejected": 0.5736863613128662,
"logps/chosen": -147.13478088378906,
"logps/ref_chosen": -61.70623016357422,
"logps/ref_rejected": -83.73808288574219,
"logps/rejected": -210.46267700195312,
"loss": 1.0981,
"margin_dpo/margin_mean": 41.29604721069336,
"margin_dpo/margin_std": 60.828086853027344,
"step": 328
},
{
"KL/chosen_KL_mean": -94.68350982666016,
"KL/mean": -116.0768814086914,
"KL/rejected_KL_mean": -137.47024536132812,
"KL/std": 60.34092330932617,
"epoch": 0.4973544973544973,
"fcm_dpo/beta": 0.010211347602307796,
"fcm_dpo/delta": -0.038584187626838684,
"fcm_dpo/margin": 42.78675079345703,
"fcm_dpo/q_t": 0.4025897979736328,
"grad_norm": 16.2102108001709,
"learning_rate": 2.9731281109010253e-07,
"logits/chosen": 0.7033920288085938,
"logits/rejected": 0.64765465259552,
"logps/chosen": -159.18191528320312,
"logps/ref_chosen": -64.4984130859375,
"logps/ref_rejected": -83.6591796875,
"logps/rejected": -221.12942504882812,
"loss": 1.0893,
"margin_dpo/margin_mean": 42.78675079345703,
"margin_dpo/margin_std": 62.20570755004883,
"step": 329
},
{
"KL/chosen_KL_mean": -80.99395751953125,
"KL/mean": -104.55245971679688,
"KL/rejected_KL_mean": -128.1109619140625,
"KL/std": 60.29164123535156,
"epoch": 0.4988662131519274,
"fcm_dpo/beta": 0.01010905671864748,
"fcm_dpo/delta": -0.08015096932649612,
"fcm_dpo/margin": 47.11699676513672,
"fcm_dpo/q_t": 0.39401495456695557,
"grad_norm": 15.336221694946289,
"learning_rate": 2.9601383051430505e-07,
"logits/chosen": 0.6909410953521729,
"logits/rejected": 0.6223288178443909,
"logps/chosen": -135.79859924316406,
"logps/ref_chosen": -54.80464172363281,
"logps/ref_rejected": -75.3194351196289,
"logps/rejected": -203.43038940429688,
"loss": 1.1018,
"margin_dpo/margin_mean": 47.11699676513672,
"margin_dpo/margin_std": 75.27323913574219,
"step": 330
},
{
"KL/chosen_KL_mean": -88.23260498046875,
"KL/mean": -116.94786071777344,
"KL/rejected_KL_mean": -145.66311645507812,
"KL/std": 63.48583221435547,
"epoch": 0.5003779289493575,
"fcm_dpo/beta": 0.009797169826924801,
"fcm_dpo/delta": -0.17205177247524261,
"fcm_dpo/margin": 57.430519104003906,
"fcm_dpo/q_t": 0.37265199422836304,
"grad_norm": 12.898703575134277,
"learning_rate": 2.947135628327544e-07,
"logits/chosen": 0.8017250299453735,
"logits/rejected": 0.7755333185195923,
"logps/chosen": -147.47518920898438,
"logps/ref_chosen": -59.242584228515625,
"logps/ref_rejected": -69.87483215332031,
"logps/rejected": -215.53794860839844,
"loss": 1.0118,
"margin_dpo/margin_mean": 57.430519104003906,
"margin_dpo/margin_std": 70.61531066894531,
"step": 331
},
{
"KL/chosen_KL_mean": -89.3869857788086,
"KL/mean": -112.95551300048828,
"KL/rejected_KL_mean": -136.5240478515625,
"KL/std": 60.380882263183594,
"epoch": 0.5018896447467877,
"fcm_dpo/beta": 0.009681256487965584,
"fcm_dpo/delta": -0.05997687205672264,
"fcm_dpo/margin": 47.137062072753906,
"fcm_dpo/q_t": 0.3970368206501007,
"grad_norm": 13.673318862915039,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": 0.6384403705596924,
"logits/rejected": 0.5949603319168091,
"logps/chosen": -156.49673461914062,
"logps/ref_chosen": -67.10975646972656,
"logps/ref_rejected": -77.11839294433594,
"logps/rejected": -213.64242553710938,
"loss": 1.0689,
"margin_dpo/margin_mean": 47.13706588745117,
"margin_dpo/margin_std": 61.14323425292969,
"step": 332
},
{
"KL/chosen_KL_mean": -95.55681610107422,
"KL/mean": -120.54524993896484,
"KL/rejected_KL_mean": -145.53369140625,
"KL/std": 60.45354461669922,
"epoch": 0.5034013605442177,
"fcm_dpo/beta": 0.00947808101773262,
"fcm_dpo/delta": -0.07727605849504471,
"fcm_dpo/margin": 49.97686767578125,
"fcm_dpo/q_t": 0.3928810954093933,
"grad_norm": 12.742399215698242,
"learning_rate": 2.921093116725076e-07,
"logits/chosen": 0.6933913826942444,
"logits/rejected": 0.6193612813949585,
"logps/chosen": -153.93795776367188,
"logps/ref_chosen": -58.381134033203125,
"logps/ref_rejected": -85.02839660644531,
"logps/rejected": -230.56207275390625,
"loss": 1.0516,
"margin_dpo/margin_mean": 49.97686767578125,
"margin_dpo/margin_std": 63.448204040527344,
"step": 333
},
{
"KL/chosen_KL_mean": -90.60214233398438,
"KL/mean": -108.82688903808594,
"KL/rejected_KL_mean": -127.05165100097656,
"KL/std": 62.743072509765625,
"epoch": 0.5049130763416477,
"fcm_dpo/beta": 0.00951945036649704,
"fcm_dpo/delta": 0.05474155396223068,
"fcm_dpo/margin": 36.44950485229492,
"fcm_dpo/q_t": 0.422860324382782,
"grad_norm": 12.695359230041504,
"learning_rate": 2.9080540104031484e-07,
"logits/chosen": 0.7170394062995911,
"logits/rejected": 0.6720852851867676,
"logps/chosen": -157.494140625,
"logps/ref_chosen": -66.89199829101562,
"logps/ref_rejected": -91.83695220947266,
"logps/rejected": -218.88861083984375,
"loss": 1.1831,
"margin_dpo/margin_mean": 36.44950485229492,
"margin_dpo/margin_std": 73.3374252319336,
"step": 334
},
{
"KL/chosen_KL_mean": -91.74083709716797,
"KL/mean": -112.25717163085938,
"KL/rejected_KL_mean": -132.77352905273438,
"KL/std": 61.952857971191406,
"epoch": 0.5064247921390779,
"fcm_dpo/beta": 0.009583601728081703,
"fcm_dpo/delta": 0.006512340158224106,
"fcm_dpo/margin": 41.032684326171875,
"fcm_dpo/q_t": 0.41258928179740906,
"grad_norm": 18.258617401123047,
"learning_rate": 2.895003489933375e-07,
"logits/chosen": 0.6742143630981445,
"logits/rejected": 0.6377497315406799,
"logps/chosen": -153.2552947998047,
"logps/ref_chosen": -61.51445770263672,
"logps/ref_rejected": -75.68916320800781,
"logps/rejected": -208.46267700195312,
"loss": 1.137,
"margin_dpo/margin_mean": 41.03268051147461,
"margin_dpo/margin_std": 69.69954681396484,
"step": 335
},
{
"KL/chosen_KL_mean": -101.37139892578125,
"KL/mean": -122.35487365722656,
"KL/rejected_KL_mean": -143.33837890625,
"KL/std": 62.85322570800781,
"epoch": 0.5079365079365079,
"fcm_dpo/beta": 0.009474512189626694,
"fcm_dpo/delta": 0.0014616698026657104,
"fcm_dpo/margin": 41.96696472167969,
"fcm_dpo/q_t": 0.4120573401451111,
"grad_norm": 12.059959411621094,
"learning_rate": 2.8819419203668675e-07,
"logits/chosen": 0.6271833777427673,
"logits/rejected": 0.6057232618331909,
"logps/chosen": -170.22146606445312,
"logps/ref_chosen": -68.85006713867188,
"logps/ref_rejected": -92.99603271484375,
"logps/rejected": -236.3343963623047,
"loss": 1.1268,
"margin_dpo/margin_mean": 41.96696472167969,
"margin_dpo/margin_std": 67.582275390625,
"step": 336
},
{
"KL/chosen_KL_mean": -104.85516357421875,
"KL/mean": -121.59478759765625,
"KL/rejected_KL_mean": -138.33441162109375,
"KL/std": 61.662418365478516,
"epoch": 0.509448223733938,
"fcm_dpo/beta": 0.009663033299148083,
"fcm_dpo/delta": 0.07897443324327469,
"fcm_dpo/margin": 33.47923278808594,
"fcm_dpo/q_t": 0.42648985981941223,
"grad_norm": 12.632766723632812,
"learning_rate": 2.8688696670638053e-07,
"logits/chosen": 0.5796546339988708,
"logits/rejected": 0.5473772287368774,
"logps/chosen": -178.04299926757812,
"logps/ref_chosen": -73.18783569335938,
"logps/ref_rejected": -86.89118957519531,
"logps/rejected": -225.2255859375,
"loss": 1.1782,
"margin_dpo/margin_mean": 33.47923278808594,
"margin_dpo/margin_std": 63.89472961425781,
"step": 337
},
{
"KL/chosen_KL_mean": -100.32296752929688,
"KL/mean": -118.47251892089844,
"KL/rejected_KL_mean": -136.6220703125,
"KL/std": 60.74869155883789,
"epoch": 0.5109599395313681,
"fcm_dpo/beta": 0.009744174778461456,
"fcm_dpo/delta": 0.048005398362874985,
"fcm_dpo/margin": 36.299102783203125,
"fcm_dpo/q_t": 0.42102691531181335,
"grad_norm": 11.634001731872559,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 0.63679039478302,
"logits/rejected": 0.6114366054534912,
"logps/chosen": -164.2625732421875,
"logps/ref_chosen": -63.939613342285156,
"logps/ref_rejected": -75.34243774414062,
"logps/rejected": -211.96450805664062,
"loss": 1.1674,
"margin_dpo/margin_mean": 36.299102783203125,
"margin_dpo/margin_std": 68.36042785644531,
"step": 338
},
{
"KL/chosen_KL_mean": -82.68782043457031,
"KL/mean": -102.33308410644531,
"KL/rejected_KL_mean": -121.97836303710938,
"KL/std": 58.95627212524414,
"epoch": 0.5124716553287982,
"fcm_dpo/beta": 0.009796416386961937,
"fcm_dpo/delta": 0.01567627489566803,
"fcm_dpo/margin": 39.29054260253906,
"fcm_dpo/q_t": 0.41272926330566406,
"grad_norm": 12.906908988952637,
"learning_rate": 2.842694572172736e-07,
"logits/chosen": 0.7947292327880859,
"logits/rejected": 0.7081258296966553,
"logps/chosen": -128.2369384765625,
"logps/ref_chosen": -45.54913330078125,
"logps/ref_rejected": -67.0482177734375,
"logps/rejected": -189.02658081054688,
"loss": 1.1257,
"margin_dpo/margin_mean": 39.29054260253906,
"margin_dpo/margin_std": 61.959800720214844,
"step": 339
},
{
"KL/chosen_KL_mean": -93.25041198730469,
"KL/mean": -114.18386840820312,
"KL/rejected_KL_mean": -135.11732482910156,
"KL/std": 65.45378112792969,
"epoch": 0.5139833711262283,
"fcm_dpo/beta": 0.009844278916716576,
"fcm_dpo/delta": -0.01303141936659813,
"fcm_dpo/margin": 41.866920471191406,
"fcm_dpo/q_t": 0.4100358486175537,
"grad_norm": 12.670487403869629,
"learning_rate": 2.8295924627584004e-07,
"logits/chosen": 0.6533123254776001,
"logits/rejected": 0.6336033344268799,
"logps/chosen": -147.2560577392578,
"logps/ref_chosen": -54.00564956665039,
"logps/ref_rejected": -61.314430236816406,
"logps/rejected": -196.4317626953125,
"loss": 1.1448,
"margin_dpo/margin_mean": 41.866920471191406,
"margin_dpo/margin_std": 75.3356704711914,
"step": 340
},
{
"KL/chosen_KL_mean": -91.20508575439453,
"KL/mean": -115.03326416015625,
"KL/rejected_KL_mean": -138.86141967773438,
"KL/std": 62.98554992675781,
"epoch": 0.5154950869236583,
"fcm_dpo/beta": 0.009496289305388927,
"fcm_dpo/delta": -0.15392111241817474,
"fcm_dpo/margin": 47.65634536743164,
"fcm_dpo/q_t": 0.3971262574195862,
"grad_norm": 13.147841453552246,
"learning_rate": 2.816481133934373e-07,
"logits/chosen": 0.7132373452186584,
"logits/rejected": 0.6640324592590332,
"logps/chosen": -154.60018920898438,
"logps/ref_chosen": -63.39509582519531,
"logps/ref_rejected": -76.20973205566406,
"logps/rejected": -215.0711669921875,
"loss": 1.0885,
"margin_dpo/margin_mean": 47.65634536743164,
"margin_dpo/margin_std": 66.24585723876953,
"step": 341
},
{
"KL/chosen_KL_mean": -90.6255111694336,
"KL/mean": -114.95613098144531,
"KL/rejected_KL_mean": -139.2867431640625,
"KL/std": 65.58506774902344,
"epoch": 0.5170068027210885,
"fcm_dpo/beta": 0.009376653470098972,
"fcm_dpo/delta": -0.05949697643518448,
"fcm_dpo/margin": 48.661231994628906,
"fcm_dpo/q_t": 0.3983476161956787,
"grad_norm": 12.41876220703125,
"learning_rate": 2.8033609524527046e-07,
"logits/chosen": 0.7255429029464722,
"logits/rejected": 0.6859662532806396,
"logps/chosen": -143.67332458496094,
"logps/ref_chosen": -53.047813415527344,
"logps/ref_rejected": -68.2854232788086,
"logps/rejected": -207.57217407226562,
"loss": 1.0775,
"margin_dpo/margin_mean": 48.661231994628906,
"margin_dpo/margin_std": 68.24559020996094,
"step": 342
},
{
"KL/chosen_KL_mean": -84.96591186523438,
"KL/mean": -100.9553451538086,
"KL/rejected_KL_mean": -116.94477081298828,
"KL/std": 61.304954528808594,
"epoch": 0.5185185185185185,
"fcm_dpo/beta": 0.009333048947155476,
"fcm_dpo/delta": -0.027199773117899895,
"fcm_dpo/margin": 31.97886085510254,
"fcm_dpo/q_t": 0.43068015575408936,
"grad_norm": 11.802735328674316,
"learning_rate": 2.7902322853130753e-07,
"logits/chosen": 0.5725841522216797,
"logits/rejected": 0.5661093592643738,
"logps/chosen": -155.54443359375,
"logps/ref_chosen": -70.57852935791016,
"logps/ref_rejected": -84.73873901367188,
"logps/rejected": -201.68350219726562,
"loss": 1.193,
"margin_dpo/margin_mean": 31.97886085510254,
"margin_dpo/margin_std": 61.72254943847656,
"step": 343
},
{
"KL/chosen_KL_mean": -93.80670166015625,
"KL/mean": -117.95794677734375,
"KL/rejected_KL_mean": -142.10919189453125,
"KL/std": 61.9505615234375,
"epoch": 0.5200302343159486,
"fcm_dpo/beta": 0.009240809828042984,
"fcm_dpo/delta": -0.04879575967788696,
"fcm_dpo/margin": 48.302486419677734,
"fcm_dpo/q_t": 0.39905792474746704,
"grad_norm": 13.704462051391602,
"learning_rate": 2.7770954997525274e-07,
"logits/chosen": 0.6961154937744141,
"logits/rejected": 0.6303431987762451,
"logps/chosen": -149.61770629882812,
"logps/ref_chosen": -55.811004638671875,
"logps/ref_rejected": -84.77637481689453,
"logps/rejected": -226.88555908203125,
"loss": 1.0739,
"margin_dpo/margin_mean": 48.302486419677734,
"margin_dpo/margin_std": 65.42170715332031,
"step": 344
},
{
"KL/chosen_KL_mean": -74.2581787109375,
"KL/mean": -94.885986328125,
"KL/rejected_KL_mean": -115.51378631591797,
"KL/std": 55.5611457824707,
"epoch": 0.5215419501133787,
"fcm_dpo/beta": 0.009291011840105057,
"fcm_dpo/delta": 0.01716582290828228,
"fcm_dpo/margin": 41.25560760498047,
"fcm_dpo/q_t": 0.4132624864578247,
"grad_norm": 13.095402717590332,
"learning_rate": 2.7639509632351927e-07,
"logits/chosen": 0.7482544779777527,
"logits/rejected": 0.7023400664329529,
"logps/chosen": -132.04428100585938,
"logps/ref_chosen": -57.78609848022461,
"logps/ref_rejected": -78.91847229003906,
"logps/rejected": -194.4322509765625,
"loss": 1.1274,
"margin_dpo/margin_mean": 41.25560760498047,
"margin_dpo/margin_std": 65.96044921875,
"step": 345
},
{
"KL/chosen_KL_mean": -82.61579132080078,
"KL/mean": -106.85696411132812,
"KL/rejected_KL_mean": -131.09812927246094,
"KL/std": 64.00788116455078,
"epoch": 0.5230536659108088,
"fcm_dpo/beta": 0.009243748150765896,
"fcm_dpo/delta": -0.050568584352731705,
"fcm_dpo/margin": 48.482337951660156,
"fcm_dpo/q_t": 0.39829227328300476,
"grad_norm": 13.68410587310791,
"learning_rate": 2.7507990434420123e-07,
"logits/chosen": 0.7192884087562561,
"logits/rejected": 0.634939968585968,
"logps/chosen": -138.90090942382812,
"logps/ref_chosen": -56.285125732421875,
"logps/ref_rejected": -91.15303039550781,
"logps/rejected": -222.25115966796875,
"loss": 1.0849,
"margin_dpo/margin_mean": 48.482337951660156,
"margin_dpo/margin_std": 68.10702514648438,
"step": 346
},
{
"KL/chosen_KL_mean": -90.36900329589844,
"KL/mean": -108.75548553466797,
"KL/rejected_KL_mean": -127.14196014404297,
"KL/std": 62.274818420410156,
"epoch": 0.5245653817082389,
"fcm_dpo/beta": 0.009267007000744343,
"fcm_dpo/delta": 0.06130155920982361,
"fcm_dpo/margin": 36.77296829223633,
"fcm_dpo/q_t": 0.42412498593330383,
"grad_norm": 15.956027030944824,
"learning_rate": 2.737640108260456e-07,
"logits/chosen": 0.8033642768859863,
"logits/rejected": 0.7525646686553955,
"logps/chosen": -143.86854553222656,
"logps/ref_chosen": -53.499542236328125,
"logps/ref_rejected": -72.52565002441406,
"logps/rejected": -199.66761779785156,
"loss": 1.1617,
"margin_dpo/margin_mean": 36.77296447753906,
"margin_dpo/margin_std": 66.07007598876953,
"step": 347
},
{
"KL/chosen_KL_mean": -80.65147399902344,
"KL/mean": -103.97816467285156,
"KL/rejected_KL_mean": -127.30485534667969,
"KL/std": 60.63064193725586,
"epoch": 0.5260770975056689,
"fcm_dpo/beta": 0.009225473739206791,
"fcm_dpo/delta": -0.03199518471956253,
"fcm_dpo/margin": 46.65338134765625,
"fcm_dpo/q_t": 0.4057735204696655,
"grad_norm": 12.421178817749023,
"learning_rate": 2.724474525774229e-07,
"logits/chosen": 0.7998018264770508,
"logits/rejected": 0.7702861428260803,
"logps/chosen": -131.43832397460938,
"logps/ref_chosen": -50.78684997558594,
"logps/ref_rejected": -68.63732147216797,
"logps/rejected": -195.94216918945312,
"loss": 1.1085,
"margin_dpo/margin_mean": 46.65338134765625,
"margin_dpo/margin_std": 73.99740600585938,
"step": 348
},
{
"KL/chosen_KL_mean": -80.41311645507812,
"KL/mean": -103.65116119384766,
"KL/rejected_KL_mean": -126.88919830322266,
"KL/std": 62.712249755859375,
"epoch": 0.527588813303099,
"fcm_dpo/beta": 0.009232236072421074,
"fcm_dpo/delta": -0.03062255121767521,
"fcm_dpo/margin": 46.47608947753906,
"fcm_dpo/q_t": 0.40419191122055054,
"grad_norm": 13.522537231445312,
"learning_rate": 2.711302664252973e-07,
"logits/chosen": 0.6993681192398071,
"logits/rejected": 0.6088770031929016,
"logps/chosen": -133.73812866210938,
"logps/ref_chosen": -53.325008392333984,
"logps/ref_rejected": -83.21236419677734,
"logps/rejected": -210.1015625,
"loss": 1.0957,
"margin_dpo/margin_mean": 46.47608947753906,
"margin_dpo/margin_std": 67.97545623779297,
"step": 349
},
{
"KL/chosen_KL_mean": -86.1879653930664,
"KL/mean": -114.68946838378906,
"KL/rejected_KL_mean": -143.1909942626953,
"KL/std": 66.76553344726562,
"epoch": 0.5291005291005291,
"fcm_dpo/beta": 0.009002182632684708,
"fcm_dpo/delta": -0.11952169239521027,
"fcm_dpo/margin": 57.00303268432617,
"fcm_dpo/q_t": 0.38292786478996277,
"grad_norm": 15.202804565429688,
"learning_rate": 2.698124892141971e-07,
"logits/chosen": 0.6883647441864014,
"logits/rejected": 0.6057754755020142,
"logps/chosen": -147.81373596191406,
"logps/ref_chosen": -61.625770568847656,
"logps/ref_rejected": -87.63627624511719,
"logps/rejected": -230.8272705078125,
"loss": 1.0247,
"margin_dpo/margin_mean": 57.00303268432617,
"margin_dpo/margin_std": 68.30619049072266,
"step": 350
},
{
"KL/chosen_KL_mean": -81.957275390625,
"KL/mean": -104.61115264892578,
"KL/rejected_KL_mean": -127.26502990722656,
"KL/std": 60.27055740356445,
"epoch": 0.5306122448979592,
"fcm_dpo/beta": 0.008928779512643814,
"fcm_dpo/delta": -0.004926031455397606,
"fcm_dpo/margin": 45.30775451660156,
"fcm_dpo/q_t": 0.4066680669784546,
"grad_norm": 13.314879417419434,
"learning_rate": 2.6849415780518357e-07,
"logits/chosen": 0.6464298963546753,
"logits/rejected": 0.5695576071739197,
"logps/chosen": -138.213623046875,
"logps/ref_chosen": -56.2563362121582,
"logps/ref_rejected": -79.11589813232422,
"logps/rejected": -206.38092041015625,
"loss": 1.1284,
"margin_dpo/margin_mean": 45.30775451660156,
"margin_dpo/margin_std": 74.37785339355469,
"step": 351
},
{
"KL/chosen_KL_mean": -80.91200256347656,
"KL/mean": -105.3866195678711,
"KL/rejected_KL_mean": -129.86123657226562,
"KL/std": 63.22986602783203,
"epoch": 0.5321239606953893,
"fcm_dpo/beta": 0.008886601775884628,
"fcm_dpo/delta": -0.03673375025391579,
"fcm_dpo/margin": 48.94923400878906,
"fcm_dpo/q_t": 0.40143412351608276,
"grad_norm": 12.169652938842773,
"learning_rate": 2.6717530907482024e-07,
"logits/chosen": 0.7215423583984375,
"logits/rejected": 0.667281985282898,
"logps/chosen": -143.96395874023438,
"logps/ref_chosen": -63.05195236206055,
"logps/ref_rejected": -85.52035522460938,
"logps/rejected": -215.381591796875,
"loss": 1.0838,
"margin_dpo/margin_mean": 48.94923400878906,
"margin_dpo/margin_std": 68.64954376220703,
"step": 352
},
{
"KL/chosen_KL_mean": -78.77203369140625,
"KL/mean": -103.06708526611328,
"KL/rejected_KL_mean": -127.36212158203125,
"KL/std": 62.106597900390625,
"epoch": 0.5336356764928194,
"fcm_dpo/beta": 0.008869750425219536,
"fcm_dpo/delta": -0.03245055675506592,
"fcm_dpo/margin": 48.590087890625,
"fcm_dpo/q_t": 0.4017961919307709,
"grad_norm": 11.374676704406738,
"learning_rate": 2.658559799141411e-07,
"logits/chosen": 0.7122618556022644,
"logits/rejected": 0.7177489995956421,
"logps/chosen": -147.78121948242188,
"logps/ref_chosen": -69.00918579101562,
"logps/ref_rejected": -72.65840148925781,
"logps/rejected": -200.02053833007812,
"loss": 1.088,
"margin_dpo/margin_mean": 48.590087890625,
"margin_dpo/margin_std": 68.10250091552734,
"step": 353
},
{
"KL/chosen_KL_mean": -82.37242889404297,
"KL/mean": -108.74432373046875,
"KL/rejected_KL_mean": -135.1162109375,
"KL/std": 61.034080505371094,
"epoch": 0.5351473922902494,
"fcm_dpo/beta": 0.00870590005069971,
"fcm_dpo/delta": -0.06280030310153961,
"fcm_dpo/margin": 52.743797302246094,
"fcm_dpo/q_t": 0.39583975076675415,
"grad_norm": 13.106264114379883,
"learning_rate": 2.6453620722761895e-07,
"logits/chosen": 0.7645365595817566,
"logits/rejected": 0.630828857421875,
"logps/chosen": -122.1607666015625,
"logps/ref_chosen": -39.78833770751953,
"logps/ref_rejected": -69.56885528564453,
"logps/rejected": -204.68507385253906,
"loss": 1.0777,
"margin_dpo/margin_mean": 52.743797302246094,
"margin_dpo/margin_std": 73.79615783691406,
"step": 354
},
{
"KL/chosen_KL_mean": -86.59736633300781,
"KL/mean": -114.18678283691406,
"KL/rejected_KL_mean": -141.77621459960938,
"KL/std": 66.40558624267578,
"epoch": 0.5366591080876795,
"fcm_dpo/beta": 0.008642604574561119,
"fcm_dpo/delta": -0.08068640530109406,
"fcm_dpo/margin": 55.17882537841797,
"fcm_dpo/q_t": 0.3921471834182739,
"grad_norm": 15.02278995513916,
"learning_rate": 2.632160279321328e-07,
"logits/chosen": 0.732662558555603,
"logits/rejected": 0.5977617502212524,
"logps/chosen": -132.85275268554688,
"logps/ref_chosen": -46.25537872314453,
"logps/ref_rejected": -78.20236206054688,
"logps/rejected": -219.9785614013672,
"loss": 1.0723,
"margin_dpo/margin_mean": 55.17882537841797,
"margin_dpo/margin_std": 77.53952026367188,
"step": 355
},
{
"KL/chosen_KL_mean": -81.78256225585938,
"KL/mean": -104.22731018066406,
"KL/rejected_KL_mean": -126.67205810546875,
"KL/std": 64.93657684326172,
"epoch": 0.5381708238851096,
"fcm_dpo/beta": 0.008562305942177773,
"fcm_dpo/delta": 0.015933889895677567,
"fcm_dpo/margin": 44.88949966430664,
"fcm_dpo/q_t": 0.414185106754303,
"grad_norm": 12.148024559020996,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 0.7186048626899719,
"logits/rejected": 0.6374760270118713,
"logps/chosen": -129.688720703125,
"logps/ref_chosen": -47.906158447265625,
"logps/ref_rejected": -74.29397583007812,
"logps/rejected": -200.96603393554688,
"loss": 1.1665,
"margin_dpo/margin_mean": 44.889495849609375,
"margin_dpo/margin_std": 85.74620056152344,
"step": 356
},
{
"KL/chosen_KL_mean": -97.89361572265625,
"KL/mean": -117.85392761230469,
"KL/rejected_KL_mean": -137.81423950195312,
"KL/std": 63.435585021972656,
"epoch": 0.5396825396825397,
"fcm_dpo/beta": 0.008481711149215698,
"fcm_dpo/delta": -0.07354926317930222,
"fcm_dpo/margin": 39.92060852050781,
"fcm_dpo/q_t": 0.42126625776290894,
"grad_norm": 12.551264762878418,
"learning_rate": 2.6057459723762076e-07,
"logits/chosen": 0.6682271957397461,
"logits/rejected": 0.6437931656837463,
"logps/chosen": -160.52862548828125,
"logps/ref_chosen": -62.63500213623047,
"logps/ref_rejected": -65.11399841308594,
"logps/rejected": -202.92823791503906,
"loss": 1.1628,
"margin_dpo/margin_mean": 39.92060852050781,
"margin_dpo/margin_std": 67.5040054321289,
"step": 357
},
{
"KL/chosen_KL_mean": -92.57527160644531,
"KL/mean": -120.09083557128906,
"KL/rejected_KL_mean": -147.60638427734375,
"KL/std": 64.96084594726562,
"epoch": 0.5411942554799698,
"fcm_dpo/beta": 0.008447141386568546,
"fcm_dpo/delta": -0.06835208088159561,
"fcm_dpo/margin": 55.031105041503906,
"fcm_dpo/q_t": 0.39534831047058105,
"grad_norm": 14.741997718811035,
"learning_rate": 2.5925341972508954e-07,
"logits/chosen": 0.6578631401062012,
"logits/rejected": 0.6739555597305298,
"logps/chosen": -159.78488159179688,
"logps/ref_chosen": -67.20960998535156,
"logps/ref_rejected": -69.34715270996094,
"logps/rejected": -216.9535369873047,
"loss": 1.0678,
"margin_dpo/margin_mean": 55.031105041503906,
"margin_dpo/margin_std": 73.77735900878906,
"step": 358
},
{
"KL/chosen_KL_mean": -104.10276794433594,
"KL/mean": -118.08717346191406,
"KL/rejected_KL_mean": -132.07159423828125,
"KL/std": 64.12388610839844,
"epoch": 0.5427059712773998,
"fcm_dpo/beta": 0.008381571620702744,
"fcm_dpo/delta": 0.017919262871146202,
"fcm_dpo/margin": 27.96881866455078,
"fcm_dpo/q_t": 0.4464063048362732,
"grad_norm": 12.74113941192627,
"learning_rate": 2.579319833745169e-07,
"logits/chosen": 0.660454511642456,
"logits/rejected": 0.631699800491333,
"logps/chosen": -166.6285400390625,
"logps/ref_chosen": -62.52578353881836,
"logps/ref_rejected": -76.63114929199219,
"logps/rejected": -208.70272827148438,
"loss": 1.241,
"margin_dpo/margin_mean": 27.968820571899414,
"margin_dpo/margin_std": 64.83367919921875,
"step": 359
},
{
"KL/chosen_KL_mean": -102.29194641113281,
"KL/mean": -125.53007507324219,
"KL/rejected_KL_mean": -148.7681884765625,
"KL/std": 68.32476806640625,
"epoch": 0.54421768707483,
"fcm_dpo/beta": 0.008394850417971611,
"fcm_dpo/delta": 0.010194879956543446,
"fcm_dpo/margin": 46.47624969482422,
"fcm_dpo/q_t": 0.4118611514568329,
"grad_norm": 11.67658519744873,
"learning_rate": 2.5661032514931834e-07,
"logits/chosen": 0.5947822332382202,
"logits/rejected": 0.5024634599685669,
"logps/chosen": -165.7796630859375,
"logps/ref_chosen": -63.48772048950195,
"logps/ref_rejected": -90.6891098022461,
"logps/rejected": -239.45730590820312,
"loss": 1.114,
"margin_dpo/margin_mean": 46.47624969482422,
"margin_dpo/margin_std": 69.5277099609375,
"step": 360
},
{
"KL/chosen_KL_mean": -100.23130798339844,
"KL/mean": -127.86849975585938,
"KL/rejected_KL_mean": -155.50570678710938,
"KL/std": 68.34567260742188,
"epoch": 0.54572940287226,
"fcm_dpo/beta": 0.008355829864740372,
"fcm_dpo/delta": -0.06483438611030579,
"fcm_dpo/margin": 55.27438735961914,
"fcm_dpo/q_t": 0.3937687873840332,
"grad_norm": 11.867284774780273,
"learning_rate": 2.552884820191154e-07,
"logits/chosen": 0.749343752861023,
"logits/rejected": 0.6997284889221191,
"logps/chosen": -158.14845275878906,
"logps/ref_chosen": -57.917144775390625,
"logps/ref_rejected": -72.39089965820312,
"logps/rejected": -227.8966064453125,
"loss": 1.0534,
"margin_dpo/margin_mean": 55.274391174316406,
"margin_dpo/margin_std": 68.07195281982422,
"step": 361
},
{
"KL/chosen_KL_mean": -101.8868408203125,
"KL/mean": -129.72409057617188,
"KL/rejected_KL_mean": -157.5613250732422,
"KL/std": 70.47108459472656,
"epoch": 0.54724111866969,
"fcm_dpo/beta": 0.008269982412457466,
"fcm_dpo/delta": -0.0635605901479721,
"fcm_dpo/margin": 55.67449188232422,
"fcm_dpo/q_t": 0.3973715901374817,
"grad_norm": 13.859137535095215,
"learning_rate": 2.53966490958702e-07,
"logits/chosen": 0.7974711656570435,
"logits/rejected": 0.6812784671783447,
"logps/chosen": -165.330322265625,
"logps/ref_chosen": -63.4434700012207,
"logps/ref_rejected": -103.45516967773438,
"logps/rejected": -261.0164794921875,
"loss": 1.0863,
"margin_dpo/margin_mean": 55.67449188232422,
"margin_dpo/margin_std": 82.16079711914062,
"step": 362
},
{
"KL/chosen_KL_mean": -107.63494873046875,
"KL/mean": -134.76649475097656,
"KL/rejected_KL_mean": -161.89804077148438,
"KL/std": 65.01280975341797,
"epoch": 0.5487528344671202,
"fcm_dpo/beta": 0.008132774382829666,
"fcm_dpo/delta": -0.04336439073085785,
"fcm_dpo/margin": 54.263099670410156,
"fcm_dpo/q_t": 0.39908909797668457,
"grad_norm": 14.706524848937988,
"learning_rate": 2.526443889470099e-07,
"logits/chosen": 0.776969850063324,
"logits/rejected": 0.6381244659423828,
"logps/chosen": -156.28677368164062,
"logps/ref_chosen": -48.65182876586914,
"logps/ref_rejected": -88.65904235839844,
"logps/rejected": -250.55709838867188,
"loss": 1.076,
"margin_dpo/margin_mean": 54.263099670410156,
"margin_dpo/margin_std": 73.57743835449219,
"step": 363
},
{
"KL/chosen_KL_mean": -97.23054504394531,
"KL/mean": -127.17767333984375,
"KL/rejected_KL_mean": -157.1248016357422,
"KL/std": 72.05257415771484,
"epoch": 0.5502645502645502,
"fcm_dpo/beta": 0.008017941378057003,
"fcm_dpo/delta": -0.08441703021526337,
"fcm_dpo/margin": 59.894256591796875,
"fcm_dpo/q_t": 0.39283275604248047,
"grad_norm": 11.617522239685059,
"learning_rate": 2.513222129660744e-07,
"logits/chosen": 0.5970016121864319,
"logits/rejected": 0.5074905157089233,
"logps/chosen": -155.10162353515625,
"logps/ref_chosen": -57.87107467651367,
"logps/ref_rejected": -80.95503234863281,
"logps/rejected": -238.079833984375,
"loss": 1.0812,
"margin_dpo/margin_mean": 59.894256591796875,
"margin_dpo/margin_std": 89.40785217285156,
"step": 364
},
{
"KL/chosen_KL_mean": -87.37548828125,
"KL/mean": -116.02485656738281,
"KL/rejected_KL_mean": -144.67422485351562,
"KL/std": 74.21676635742188,
"epoch": 0.5517762660619804,
"fcm_dpo/beta": 0.007917352020740509,
"fcm_dpo/delta": -0.05630026012659073,
"fcm_dpo/margin": 57.29872131347656,
"fcm_dpo/q_t": 0.39494040608406067,
"grad_norm": 10.989361763000488,
"learning_rate": 2.5e-07,
"logits/chosen": 0.7256494760513306,
"logits/rejected": 0.7213196754455566,
"logps/chosen": -152.31765747070312,
"logps/ref_chosen": -64.94217681884766,
"logps/ref_rejected": -74.8599853515625,
"logps/rejected": -219.53421020507812,
"loss": 1.0465,
"margin_dpo/margin_mean": 57.29872131347656,
"margin_dpo/margin_std": 65.5568618774414,
"step": 365
},
{
"KL/chosen_KL_mean": -89.08964538574219,
"KL/mean": -112.80401611328125,
"KL/rejected_KL_mean": -136.5183868408203,
"KL/std": 66.60395812988281,
"epoch": 0.5532879818594104,
"fcm_dpo/beta": 0.00795934908092022,
"fcm_dpo/delta": 0.02304329350590706,
"fcm_dpo/margin": 47.42875671386719,
"fcm_dpo/q_t": 0.4156304895877838,
"grad_norm": 13.898573875427246,
"learning_rate": 2.486777870339255e-07,
"logits/chosen": 0.6511447429656982,
"logits/rejected": 0.637090802192688,
"logps/chosen": -144.255615234375,
"logps/ref_chosen": -55.16598129272461,
"logps/ref_rejected": -65.26121520996094,
"logps/rejected": -201.77960205078125,
"loss": 1.1556,
"margin_dpo/margin_mean": 47.42875671386719,
"margin_dpo/margin_std": 86.32708740234375,
"step": 366
},
{
"KL/chosen_KL_mean": -99.81094360351562,
"KL/mean": -124.00721740722656,
"KL/rejected_KL_mean": -148.20347595214844,
"KL/std": 67.95364379882812,
"epoch": 0.5547996976568406,
"fcm_dpo/beta": 0.007937667891383171,
"fcm_dpo/delta": 0.01639546826481819,
"fcm_dpo/margin": 48.39253616333008,
"fcm_dpo/q_t": 0.41048091650009155,
"grad_norm": 12.092884063720703,
"learning_rate": 2.4735561105299014e-07,
"logits/chosen": 0.7002275586128235,
"logits/rejected": 0.5934484004974365,
"logps/chosen": -155.8214111328125,
"logps/ref_chosen": -56.01046371459961,
"logps/ref_rejected": -77.31010437011719,
"logps/rejected": -225.51358032226562,
"loss": 1.1224,
"margin_dpo/margin_mean": 48.39253616333008,
"margin_dpo/margin_std": 74.65963745117188,
"step": 367
},
{
"KL/chosen_KL_mean": -109.84168243408203,
"KL/mean": -132.435302734375,
"KL/rejected_KL_mean": -155.0289306640625,
"KL/std": 67.84854125976562,
"epoch": 0.5563114134542706,
"fcm_dpo/beta": 0.00801210105419159,
"fcm_dpo/delta": 0.03938727825880051,
"fcm_dpo/margin": 45.187255859375,
"fcm_dpo/q_t": 0.4165228009223938,
"grad_norm": 13.236560821533203,
"learning_rate": 2.46033509041298e-07,
"logits/chosen": 0.5023385882377625,
"logits/rejected": 0.5030689239501953,
"logps/chosen": -184.67095947265625,
"logps/ref_chosen": -74.82927703857422,
"logps/ref_rejected": -76.11680603027344,
"logps/rejected": -231.14573669433594,
"loss": 1.1377,
"margin_dpo/margin_mean": 45.187252044677734,
"margin_dpo/margin_std": 72.93472290039062,
"step": 368
},
{
"KL/chosen_KL_mean": -103.68905639648438,
"KL/mean": -122.96196746826172,
"KL/rejected_KL_mean": -142.23487854003906,
"KL/std": 68.13346862792969,
"epoch": 0.5578231292517006,
"fcm_dpo/beta": 0.008171428926289082,
"fcm_dpo/delta": 0.08697890490293503,
"fcm_dpo/margin": 38.54582977294922,
"fcm_dpo/q_t": 0.42861396074295044,
"grad_norm": 13.241608619689941,
"learning_rate": 2.447115179808846e-07,
"logits/chosen": 0.7053878307342529,
"logits/rejected": 0.6514875888824463,
"logps/chosen": -162.01527404785156,
"logps/ref_chosen": -58.32621765136719,
"logps/ref_rejected": -80.92183685302734,
"logps/rejected": -223.15672302246094,
"loss": 1.1809,
"margin_dpo/margin_mean": 38.54582977294922,
"margin_dpo/margin_std": 72.81201171875,
"step": 369
},
{
"KL/chosen_KL_mean": -99.04299926757812,
"KL/mean": -126.85870361328125,
"KL/rejected_KL_mean": -154.6743927001953,
"KL/std": 71.19883728027344,
"epoch": 0.5593348450491308,
"fcm_dpo/beta": 0.0080941803753376,
"fcm_dpo/delta": -0.05279029160737991,
"fcm_dpo/margin": 55.63139343261719,
"fcm_dpo/q_t": 0.397558331489563,
"grad_norm": 13.408743858337402,
"learning_rate": 2.4338967485068164e-07,
"logits/chosen": 0.7617638111114502,
"logits/rejected": 0.6938444375991821,
"logps/chosen": -151.92672729492188,
"logps/ref_chosen": -52.88372039794922,
"logps/ref_rejected": -79.43692016601562,
"logps/rejected": -234.11131286621094,
"loss": 1.0897,
"margin_dpo/margin_mean": 55.63139724731445,
"margin_dpo/margin_std": 82.69889831542969,
"step": 370
},
{
"KL/chosen_KL_mean": -99.44721221923828,
"KL/mean": -123.93897247314453,
"KL/rejected_KL_mean": -148.4307403564453,
"KL/std": 69.45941925048828,
"epoch": 0.5608465608465608,
"fcm_dpo/beta": 0.008130359463393688,
"fcm_dpo/delta": 0.0012083090841770172,
"fcm_dpo/margin": 48.98352813720703,
"fcm_dpo/q_t": 0.41003215312957764,
"grad_norm": 15.85348892211914,
"learning_rate": 2.420680166254831e-07,
"logits/chosen": 0.8482241630554199,
"logits/rejected": 0.8140517473220825,
"logps/chosen": -148.67141723632812,
"logps/ref_chosen": -49.224212646484375,
"logps/ref_rejected": -63.348472595214844,
"logps/rejected": -211.77920532226562,
"loss": 1.118,
"margin_dpo/margin_mean": 48.98352813720703,
"margin_dpo/margin_std": 75.29916381835938,
"step": 371
},
{
"KL/chosen_KL_mean": -106.43836975097656,
"KL/mean": -121.54212951660156,
"KL/rejected_KL_mean": -136.64588928222656,
"KL/std": 68.73971557617188,
"epoch": 0.562358276643991,
"fcm_dpo/beta": 0.008100366219878197,
"fcm_dpo/delta": 0.016941992565989494,
"fcm_dpo/margin": 30.207515716552734,
"fcm_dpo/q_t": 0.445268452167511,
"grad_norm": 16.10873031616211,
"learning_rate": 2.4074658027491044e-07,
"logits/chosen": 0.6909885406494141,
"logits/rejected": 0.5951350927352905,
"logps/chosen": -158.70791625976562,
"logps/ref_chosen": -52.269554138183594,
"logps/ref_rejected": -72.99522399902344,
"logps/rejected": -209.64111328125,
"loss": 1.2817,
"margin_dpo/margin_mean": 30.20751190185547,
"margin_dpo/margin_std": 88.66557312011719,
"step": 372
},
{
"KL/chosen_KL_mean": -117.92112731933594,
"KL/mean": -138.2035369873047,
"KL/rejected_KL_mean": -158.48593139648438,
"KL/std": 68.60737609863281,
"epoch": 0.563869992441421,
"fcm_dpo/beta": 0.008211096748709679,
"fcm_dpo/delta": 0.06906390190124512,
"fcm_dpo/margin": 40.564823150634766,
"fcm_dpo/q_t": 0.42571961879730225,
"grad_norm": 13.816263198852539,
"learning_rate": 2.394254027623792e-07,
"logits/chosen": 0.7159205675125122,
"logits/rejected": 0.6442649364471436,
"logps/chosen": -179.03411865234375,
"logps/ref_chosen": -61.112998962402344,
"logps/ref_rejected": -76.24851989746094,
"logps/rejected": -234.73446655273438,
"loss": 1.2069,
"margin_dpo/margin_mean": 40.5648193359375,
"margin_dpo/margin_std": 89.206298828125,
"step": 373
},
{
"KL/chosen_KL_mean": -98.22417449951172,
"KL/mean": -132.6193389892578,
"KL/rejected_KL_mean": -167.01451110839844,
"KL/std": 70.38906860351562,
"epoch": 0.5653817082388511,
"fcm_dpo/beta": 0.008019594475626945,
"fcm_dpo/delta": -0.16084754467010498,
"fcm_dpo/margin": 68.79034423828125,
"fcm_dpo/q_t": 0.3737262487411499,
"grad_norm": 13.687728881835938,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 0.5888317823410034,
"logits/rejected": 0.5906950831413269,
"logps/chosen": -170.89337158203125,
"logps/ref_chosen": -72.66920471191406,
"logps/ref_rejected": -76.83158874511719,
"logps/rejected": -243.84609985351562,
"loss": 1.0015,
"margin_dpo/margin_mean": 68.79034423828125,
"margin_dpo/margin_std": 79.17984008789062,
"step": 374
},
{
"KL/chosen_KL_mean": -96.6202392578125,
"KL/mean": -120.49937438964844,
"KL/rejected_KL_mean": -144.37850952148438,
"KL/std": 73.497802734375,
"epoch": 0.5668934240362812,
"fcm_dpo/beta": 0.007966436445713043,
"fcm_dpo/delta": 0.020184047520160675,
"fcm_dpo/margin": 47.758262634277344,
"fcm_dpo/q_t": 0.41416776180267334,
"grad_norm": 14.614751815795898,
"learning_rate": 2.3678397206786715e-07,
"logits/chosen": 0.7185194492340088,
"logits/rejected": 0.6587230563163757,
"logps/chosen": -154.3035430908203,
"logps/ref_chosen": -57.68330383300781,
"logps/ref_rejected": -79.34097290039062,
"logps/rejected": -223.719482421875,
"loss": 1.1441,
"margin_dpo/margin_mean": 47.758262634277344,
"margin_dpo/margin_std": 82.57972717285156,
"step": 375
},
{
"KL/chosen_KL_mean": -105.34626770019531,
"KL/mean": -134.71389770507812,
"KL/rejected_KL_mean": -164.08154296875,
"KL/std": 73.43299865722656,
"epoch": 0.5684051398337112,
"fcm_dpo/beta": 0.007908320054411888,
"fcm_dpo/delta": -0.06776019185781479,
"fcm_dpo/margin": 58.73528289794922,
"fcm_dpo/q_t": 0.39599794149398804,
"grad_norm": 13.218934059143066,
"learning_rate": 2.3546379277238103e-07,
"logits/chosen": 0.7856276035308838,
"logits/rejected": 0.7111548781394958,
"logps/chosen": -157.0203399658203,
"logps/ref_chosen": -51.674072265625,
"logps/ref_rejected": -75.69713592529297,
"logps/rejected": -239.77867126464844,
"loss": 1.0811,
"margin_dpo/margin_mean": 58.73528289794922,
"margin_dpo/margin_std": 85.76099395751953,
"step": 376
},
{
"KL/chosen_KL_mean": -109.92994689941406,
"KL/mean": -131.07846069335938,
"KL/rejected_KL_mean": -152.22695922851562,
"KL/std": 68.16609191894531,
"epoch": 0.5699168556311414,
"fcm_dpo/beta": 0.008002420887351036,
"fcm_dpo/delta": 0.06295044720172882,
"fcm_dpo/margin": 42.29701232910156,
"fcm_dpo/q_t": 0.42261120676994324,
"grad_norm": 13.345908164978027,
"learning_rate": 2.3414402008585886e-07,
"logits/chosen": 0.7429170608520508,
"logits/rejected": 0.7196171879768372,
"logps/chosen": -156.10848999023438,
"logps/ref_chosen": -46.17853546142578,
"logps/ref_rejected": -57.756500244140625,
"logps/rejected": -209.98345947265625,
"loss": 1.1709,
"margin_dpo/margin_mean": 42.29701232910156,
"margin_dpo/margin_std": 78.35391235351562,
"step": 377
},
{
"KL/chosen_KL_mean": -106.18476867675781,
"KL/mean": -126.3034439086914,
"KL/rejected_KL_mean": -146.422119140625,
"KL/std": 71.95037078857422,
"epoch": 0.5714285714285714,
"fcm_dpo/beta": 0.00811665877699852,
"fcm_dpo/delta": 0.07516461610794067,
"fcm_dpo/margin": 40.23735427856445,
"fcm_dpo/q_t": 0.4256941080093384,
"grad_norm": 12.932868003845215,
"learning_rate": 2.3282469092517977e-07,
"logits/chosen": 0.7587268948554993,
"logits/rejected": 0.7094443440437317,
"logps/chosen": -165.4036407470703,
"logps/ref_chosen": -59.21887969970703,
"logps/ref_rejected": -71.24818420410156,
"logps/rejected": -217.67031860351562,
"loss": 1.1762,
"margin_dpo/margin_mean": 40.23735427856445,
"margin_dpo/margin_std": 74.7750244140625,
"step": 378
},
{
"KL/chosen_KL_mean": -101.88446044921875,
"KL/mean": -128.81649780273438,
"KL/rejected_KL_mean": -155.74853515625,
"KL/std": 71.81948852539062,
"epoch": 0.5729402872260015,
"fcm_dpo/beta": 0.008070360869169235,
"fcm_dpo/delta": -0.03627227246761322,
"fcm_dpo/margin": 53.864105224609375,
"fcm_dpo/q_t": 0.4028571844100952,
"grad_norm": 14.7687406539917,
"learning_rate": 2.3150584219481643e-07,
"logits/chosen": 0.7257020473480225,
"logits/rejected": 0.6491061449050903,
"logps/chosen": -178.2010498046875,
"logps/ref_chosen": -76.31658935546875,
"logps/ref_rejected": -104.26200103759766,
"logps/rejected": -260.01055908203125,
"loss": 1.0954,
"margin_dpo/margin_mean": 53.864105224609375,
"margin_dpo/margin_std": 80.73196411132812,
"step": 379
},
{
"KL/chosen_KL_mean": -89.789794921875,
"KL/mean": -123.39581298828125,
"KL/rejected_KL_mean": -157.00186157226562,
"KL/std": 68.38964080810547,
"epoch": 0.5744520030234316,
"fcm_dpo/beta": 0.007899045944213867,
"fcm_dpo/delta": -0.13812017440795898,
"fcm_dpo/margin": 67.21205139160156,
"fcm_dpo/q_t": 0.37823671102523804,
"grad_norm": 12.306526184082031,
"learning_rate": 2.3018751078580283e-07,
"logits/chosen": 0.7205266952514648,
"logits/rejected": 0.6820650100708008,
"logps/chosen": -151.07296752929688,
"logps/ref_chosen": -61.283164978027344,
"logps/ref_rejected": -72.38892364501953,
"logps/rejected": -229.39077758789062,
"loss": 1.0255,
"margin_dpo/margin_mean": 67.21205139160156,
"margin_dpo/margin_std": 83.0625228881836,
"step": 380
},
{
"KL/chosen_KL_mean": -109.70946502685547,
"KL/mean": -122.92391967773438,
"KL/rejected_KL_mean": -136.13836669921875,
"KL/std": 68.87846374511719,
"epoch": 0.5759637188208617,
"fcm_dpo/beta": 0.007872538641095161,
"fcm_dpo/delta": 0.04893864318728447,
"fcm_dpo/margin": 26.428911209106445,
"fcm_dpo/q_t": 0.4531518816947937,
"grad_norm": 13.719199180603027,
"learning_rate": 2.288697335747027e-07,
"logits/chosen": 0.7018343806266785,
"logits/rejected": 0.6785413026809692,
"logps/chosen": -167.9234619140625,
"logps/ref_chosen": -58.2139892578125,
"logps/ref_rejected": -60.78669357299805,
"logps/rejected": -196.92506408691406,
"loss": 1.2882,
"margin_dpo/margin_mean": 26.428911209106445,
"margin_dpo/margin_std": 81.47897338867188,
"step": 381
},
{
"KL/chosen_KL_mean": -111.03116607666016,
"KL/mean": -133.79153442382812,
"KL/rejected_KL_mean": -156.5518798828125,
"KL/std": 70.28595733642578,
"epoch": 0.5774754346182918,
"fcm_dpo/beta": 0.007991382852196693,
"fcm_dpo/delta": 0.036996498703956604,
"fcm_dpo/margin": 45.52073287963867,
"fcm_dpo/q_t": 0.41654476523399353,
"grad_norm": 13.343153953552246,
"learning_rate": 2.2755254742257706e-07,
"logits/chosen": 0.6953055262565613,
"logits/rejected": 0.641878068447113,
"logps/chosen": -172.8564910888672,
"logps/ref_chosen": -61.82532501220703,
"logps/ref_rejected": -83.0452880859375,
"logps/rejected": -239.59716796875,
"loss": 1.1285,
"margin_dpo/margin_mean": 45.520729064941406,
"margin_dpo/margin_std": 68.6872329711914,
"step": 382
},
{
"KL/chosen_KL_mean": -108.03004455566406,
"KL/mean": -131.80714416503906,
"KL/rejected_KL_mean": -155.58425903320312,
"KL/std": 70.89349365234375,
"epoch": 0.5789871504157218,
"fcm_dpo/beta": 0.00796021893620491,
"fcm_dpo/delta": 0.02198859676718712,
"fcm_dpo/margin": 47.55420684814453,
"fcm_dpo/q_t": 0.41592031717300415,
"grad_norm": 14.067788124084473,
"learning_rate": 2.2623598917395436e-07,
"logits/chosen": 0.6033366918563843,
"logits/rejected": 0.6347865462303162,
"logps/chosen": -188.5933074951172,
"logps/ref_chosen": -80.56326293945312,
"logps/ref_rejected": -74.62922668457031,
"logps/rejected": -230.21347045898438,
"loss": 1.1594,
"margin_dpo/margin_mean": 47.55420684814453,
"margin_dpo/margin_std": 87.8403549194336,
"step": 383
},
{
"KL/chosen_KL_mean": -107.74076843261719,
"KL/mean": -131.1066436767578,
"KL/rejected_KL_mean": -154.47250366210938,
"KL/std": 71.08136749267578,
"epoch": 0.5804988662131519,
"fcm_dpo/beta": 0.008032035082578659,
"fcm_dpo/delta": 0.02561786398291588,
"fcm_dpo/margin": 46.731719970703125,
"fcm_dpo/q_t": 0.41315633058547974,
"grad_norm": 15.100645065307617,
"learning_rate": 2.2492009565579875e-07,
"logits/chosen": 0.7310689687728882,
"logits/rejected": 0.6846098899841309,
"logps/chosen": -173.21591186523438,
"logps/ref_chosen": -65.47514343261719,
"logps/ref_rejected": -79.67378234863281,
"logps/rejected": -234.1462860107422,
"loss": 1.1315,
"margin_dpo/margin_mean": 46.731719970703125,
"margin_dpo/margin_std": 75.48046112060547,
"step": 384
},
{
"KL/chosen_KL_mean": -104.28163146972656,
"KL/mean": -135.58575439453125,
"KL/rejected_KL_mean": -166.88987731933594,
"KL/std": 70.27163696289062,
"epoch": 0.582010582010582,
"fcm_dpo/beta": 0.007950296625494957,
"fcm_dpo/delta": -0.10275811702013016,
"fcm_dpo/margin": 62.60823440551758,
"fcm_dpo/q_t": 0.38700929284095764,
"grad_norm": 13.862860679626465,
"learning_rate": 2.2360490367648084e-07,
"logits/chosen": 0.6451644897460938,
"logits/rejected": 0.6053575277328491,
"logps/chosen": -170.33815002441406,
"logps/ref_chosen": -66.0565185546875,
"logps/ref_rejected": -86.68023681640625,
"logps/rejected": -253.5701141357422,
"loss": 1.0333,
"margin_dpo/margin_mean": 62.60823059082031,
"margin_dpo/margin_std": 75.1285400390625,
"step": 385
},
{
"KL/chosen_KL_mean": -120.93341064453125,
"KL/mean": -140.75448608398438,
"KL/rejected_KL_mean": -160.57554626464844,
"KL/std": 71.16212463378906,
"epoch": 0.5835222978080121,
"fcm_dpo/beta": 0.007975287735462189,
"fcm_dpo/delta": 0.08665543049573898,
"fcm_dpo/margin": 39.64215087890625,
"fcm_dpo/q_t": 0.42656800150871277,
"grad_norm": 13.793825149536133,
"learning_rate": 2.2229045002474724e-07,
"logits/chosen": 0.6204428672790527,
"logits/rejected": 0.5610051155090332,
"logps/chosen": -196.55706787109375,
"logps/ref_chosen": -75.6236572265625,
"logps/ref_rejected": -92.62330627441406,
"logps/rejected": -253.1988525390625,
"loss": 1.1779,
"margin_dpo/margin_mean": 39.64215087890625,
"margin_dpo/margin_std": 74.01336669921875,
"step": 386
},
{
"KL/chosen_KL_mean": -107.25721740722656,
"KL/mean": -137.3277587890625,
"KL/rejected_KL_mean": -167.39833068847656,
"KL/std": 68.76395416259766,
"epoch": 0.5850340136054422,
"fcm_dpo/beta": 0.007923007011413574,
"fcm_dpo/delta": -0.08034680783748627,
"fcm_dpo/margin": 60.14110565185547,
"fcm_dpo/q_t": 0.3910979628562927,
"grad_norm": 13.027965545654297,
"learning_rate": 2.209767714686924e-07,
"logits/chosen": 0.7182176113128662,
"logits/rejected": 0.6100037097930908,
"logps/chosen": -154.47891235351562,
"logps/ref_chosen": -47.22170639038086,
"logps/ref_rejected": -87.338134765625,
"logps/rejected": -254.73646545410156,
"loss": 1.0384,
"margin_dpo/margin_mean": 60.14110565185547,
"margin_dpo/margin_std": 70.9463119506836,
"step": 387
},
{
"KL/chosen_KL_mean": -108.18991088867188,
"KL/mean": -126.91732788085938,
"KL/rejected_KL_mean": -145.64474487304688,
"KL/std": 71.3280258178711,
"epoch": 0.5865457294028723,
"fcm_dpo/beta": 0.007894270122051239,
"fcm_dpo/delta": 0.0012205018429085612,
"fcm_dpo/margin": 37.45484924316406,
"fcm_dpo/q_t": 0.433984637260437,
"grad_norm": 12.99436092376709,
"learning_rate": 2.1966390475472954e-07,
"logits/chosen": 0.7144241333007812,
"logits/rejected": 0.7073640823364258,
"logps/chosen": -182.76937866210938,
"logps/ref_chosen": -74.5794677734375,
"logps/ref_rejected": -79.92558288574219,
"logps/rejected": -225.57034301757812,
"loss": 1.2198,
"margin_dpo/margin_mean": 37.45484924316406,
"margin_dpo/margin_std": 84.88539123535156,
"step": 388
},
{
"KL/chosen_KL_mean": -103.51893615722656,
"KL/mean": -134.0506134033203,
"KL/rejected_KL_mean": -164.582275390625,
"KL/std": 71.60000610351562,
"epoch": 0.5880574452003023,
"fcm_dpo/beta": 0.007809435948729515,
"fcm_dpo/delta": -0.08065281808376312,
"fcm_dpo/margin": 61.06333541870117,
"fcm_dpo/q_t": 0.3916972279548645,
"grad_norm": 27.244335174560547,
"learning_rate": 2.1835188660656265e-07,
"logits/chosen": 0.7373260259628296,
"logits/rejected": 0.699165940284729,
"logps/chosen": -165.143310546875,
"logps/ref_chosen": -61.624366760253906,
"logps/ref_rejected": -76.50978088378906,
"logps/rejected": -241.09207153320312,
"loss": 1.0544,
"margin_dpo/margin_mean": 61.06333541870117,
"margin_dpo/margin_std": 79.44436645507812,
"step": 389
},
{
"KL/chosen_KL_mean": -95.27030944824219,
"KL/mean": -118.47329711914062,
"KL/rejected_KL_mean": -141.67630004882812,
"KL/std": 68.54006958007812,
"epoch": 0.5895691609977324,
"fcm_dpo/beta": 0.007800564169883728,
"fcm_dpo/delta": 0.03944290429353714,
"fcm_dpo/margin": 46.405982971191406,
"fcm_dpo/q_t": 0.4168715476989746,
"grad_norm": 10.817452430725098,
"learning_rate": 2.170407537241599e-07,
"logits/chosen": 0.7971335649490356,
"logits/rejected": 0.7232675552368164,
"logps/chosen": -141.14218139648438,
"logps/ref_chosen": -45.871864318847656,
"logps/ref_rejected": -61.305999755859375,
"logps/rejected": -202.9822998046875,
"loss": 1.1314,
"margin_dpo/margin_mean": 46.405982971191406,
"margin_dpo/margin_std": 71.90489196777344,
"step": 390
},
{
"KL/chosen_KL_mean": -104.57449340820312,
"KL/mean": -132.54464721679688,
"KL/rejected_KL_mean": -160.51480102539062,
"KL/std": 69.6192626953125,
"epoch": 0.5910808767951625,
"fcm_dpo/beta": 0.00775923253968358,
"fcm_dpo/delta": -0.03600364178419113,
"fcm_dpo/margin": 55.940330505371094,
"fcm_dpo/q_t": 0.40101712942123413,
"grad_norm": 12.532876968383789,
"learning_rate": 2.1573054278272636e-07,
"logits/chosen": 0.7184900045394897,
"logits/rejected": 0.6485068798065186,
"logps/chosen": -162.76150512695312,
"logps/ref_chosen": -58.18701171875,
"logps/ref_rejected": -83.63442993164062,
"logps/rejected": -244.14923095703125,
"loss": 1.1048,
"margin_dpo/margin_mean": 55.940330505371094,
"margin_dpo/margin_std": 86.30181884765625,
"step": 391
},
{
"KL/chosen_KL_mean": -91.01192474365234,
"KL/mean": -121.442626953125,
"KL/rejected_KL_mean": -151.8733367919922,
"KL/std": 72.98440551757812,
"epoch": 0.5925925925925926,
"fcm_dpo/beta": 0.007734889164566994,
"fcm_dpo/delta": -0.07455773651599884,
"fcm_dpo/margin": 60.86140441894531,
"fcm_dpo/q_t": 0.3938947319984436,
"grad_norm": 11.02000904083252,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 0.789170503616333,
"logits/rejected": 0.7266790270805359,
"logps/chosen": -160.75645446777344,
"logps/ref_chosen": -69.7445297241211,
"logps/ref_rejected": -94.05877685546875,
"logps/rejected": -245.93211364746094,
"loss": 1.0752,
"margin_dpo/margin_mean": 60.86140441894531,
"margin_dpo/margin_std": 85.86114501953125,
"step": 392
},
{
"KL/chosen_KL_mean": -104.94602966308594,
"KL/mean": -136.30958557128906,
"KL/rejected_KL_mean": -167.67315673828125,
"KL/std": 71.40564727783203,
"epoch": 0.5941043083900227,
"fcm_dpo/beta": 0.007545138709247112,
"fcm_dpo/delta": -0.07741730660200119,
"fcm_dpo/margin": 62.72712707519531,
"fcm_dpo/q_t": 0.3913338780403137,
"grad_norm": 11.602364540100098,
"learning_rate": 2.131130332936195e-07,
"logits/chosen": 0.706555962562561,
"logits/rejected": 0.6680725812911987,
"logps/chosen": -157.28091430664062,
"logps/ref_chosen": -52.33489990234375,
"logps/ref_rejected": -74.33809661865234,
"logps/rejected": -242.01124572753906,
"loss": 1.0423,
"margin_dpo/margin_mean": 62.72712707519531,
"margin_dpo/margin_std": 74.48922729492188,
"step": 393
},
{
"KL/chosen_KL_mean": -99.7177734375,
"KL/mean": -127.16287231445312,
"KL/rejected_KL_mean": -154.60797119140625,
"KL/std": 65.74242401123047,
"epoch": 0.5956160241874527,
"fcm_dpo/beta": 0.007545899134129286,
"fcm_dpo/delta": -0.014940101653337479,
"fcm_dpo/margin": 54.89018249511719,
"fcm_dpo/q_t": 0.4029679596424103,
"grad_norm": 11.847579002380371,
"learning_rate": 2.1180580796331323e-07,
"logits/chosen": 0.7459127306938171,
"logits/rejected": 0.7154402136802673,
"logps/chosen": -160.3939208984375,
"logps/ref_chosen": -60.6761360168457,
"logps/ref_rejected": -71.36074829101562,
"logps/rejected": -225.96871948242188,
"loss": 1.0766,
"margin_dpo/margin_mean": 54.89018630981445,
"margin_dpo/margin_std": 66.64370727539062,
"step": 394
},
{
"KL/chosen_KL_mean": -105.17837524414062,
"KL/mean": -127.70188903808594,
"KL/rejected_KL_mean": -150.22540283203125,
"KL/std": 68.39543151855469,
"epoch": 0.5971277399848829,
"fcm_dpo/beta": 0.007615202572196722,
"fcm_dpo/delta": 0.05839349329471588,
"fcm_dpo/margin": 45.047027587890625,
"fcm_dpo/q_t": 0.42268693447113037,
"grad_norm": 14.568473815917969,
"learning_rate": 2.104996510066625e-07,
"logits/chosen": 0.7183883190155029,
"logits/rejected": 0.61865234375,
"logps/chosen": -155.78269958496094,
"logps/ref_chosen": -50.60432434082031,
"logps/ref_rejected": -77.08731079101562,
"logps/rejected": -227.31271362304688,
"loss": 1.1476,
"margin_dpo/margin_mean": 45.047027587890625,
"margin_dpo/margin_std": 73.49839782714844,
"step": 395
},
{
"KL/chosen_KL_mean": -98.29253387451172,
"KL/mean": -124.56741333007812,
"KL/rejected_KL_mean": -150.84230041503906,
"KL/std": 76.07400512695312,
"epoch": 0.5986394557823129,
"fcm_dpo/beta": 0.007551061920821667,
"fcm_dpo/delta": 0.0021466389298439026,
"fcm_dpo/margin": 52.549766540527344,
"fcm_dpo/q_t": 0.4087793231010437,
"grad_norm": 11.225433349609375,
"learning_rate": 2.0919459895968517e-07,
"logits/chosen": 0.7207078337669373,
"logits/rejected": 0.6202989816665649,
"logps/chosen": -149.6521453857422,
"logps/ref_chosen": -51.35961151123047,
"logps/ref_rejected": -79.89360046386719,
"logps/rejected": -230.73590087890625,
"loss": 1.0952,
"margin_dpo/margin_mean": 52.54976272583008,
"margin_dpo/margin_std": 67.60321044921875,
"step": 396
},
{
"KL/chosen_KL_mean": -113.41151428222656,
"KL/mean": -128.26589965820312,
"KL/rejected_KL_mean": -143.12026977539062,
"KL/std": 69.99290466308594,
"epoch": 0.600151171579743,
"fcm_dpo/beta": 0.0076684970408678055,
"fcm_dpo/delta": 0.07671602815389633,
"fcm_dpo/margin": 29.70874786376953,
"fcm_dpo/q_t": 0.4474959969520569,
"grad_norm": 12.810372352600098,
"learning_rate": 2.078906883274924e-07,
"logits/chosen": 0.6312674283981323,
"logits/rejected": 0.5827088356018066,
"logps/chosen": -179.86773681640625,
"logps/ref_chosen": -66.45622253417969,
"logps/ref_rejected": -85.74736785888672,
"logps/rejected": -228.86764526367188,
"loss": 1.2756,
"margin_dpo/margin_mean": 29.70874786376953,
"margin_dpo/margin_std": 85.7228012084961,
"step": 397
},
{
"KL/chosen_KL_mean": -97.1900634765625,
"KL/mean": -130.2547607421875,
"KL/rejected_KL_mean": -163.3194580078125,
"KL/std": 72.60840606689453,
"epoch": 0.6016628873771731,
"fcm_dpo/beta": 0.0075783152133226395,
"fcm_dpo/delta": -0.1071229875087738,
"fcm_dpo/margin": 66.12939453125,
"fcm_dpo/q_t": 0.385869562625885,
"grad_norm": 11.06219482421875,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.6911704540252686,
"logits/rejected": 0.623024582862854,
"logps/chosen": -146.43431091308594,
"logps/ref_chosen": -49.244239807128906,
"logps/ref_rejected": -75.18949127197266,
"logps/rejected": -238.50894165039062,
"loss": 1.0202,
"margin_dpo/margin_mean": 66.12939453125,
"margin_dpo/margin_std": 74.68193054199219,
"step": 398
},
{
"KL/chosen_KL_mean": -113.19331359863281,
"KL/mean": -149.0525360107422,
"KL/rejected_KL_mean": -184.9117431640625,
"KL/std": 75.94452667236328,
"epoch": 0.6031746031746031,
"fcm_dpo/beta": 0.007383415475487709,
"fcm_dpo/delta": -0.13754862546920776,
"fcm_dpo/margin": 71.71843719482422,
"fcm_dpo/q_t": 0.3801451623439789,
"grad_norm": 13.51389217376709,
"learning_rate": 2.052864371672457e-07,
"logits/chosen": 0.6527610421180725,
"logits/rejected": 0.503684937953949,
"logps/chosen": -181.50010681152344,
"logps/ref_chosen": -68.30679321289062,
"logps/ref_rejected": -113.2708511352539,
"logps/rejected": -298.1826171875,
"loss": 1.0121,
"margin_dpo/margin_mean": 71.71843719482422,
"margin_dpo/margin_std": 83.61109924316406,
"step": 399
},
{
"KL/chosen_KL_mean": -125.81544494628906,
"KL/mean": -147.44232177734375,
"KL/rejected_KL_mean": -169.06918334960938,
"KL/std": 73.22869873046875,
"epoch": 0.6046863189720333,
"fcm_dpo/beta": 0.007310614455491304,
"fcm_dpo/delta": -0.032505691051483154,
"fcm_dpo/margin": 43.25373840332031,
"fcm_dpo/q_t": 0.4267102777957916,
"grad_norm": 16.710817337036133,
"learning_rate": 2.0398616948569493e-07,
"logits/chosen": 0.7390056848526001,
"logits/rejected": 0.6754894256591797,
"logps/chosen": -197.4419403076172,
"logps/ref_chosen": -71.62649536132812,
"logps/ref_rejected": -90.98765563964844,
"logps/rejected": -260.05682373046875,
"loss": 1.1673,
"margin_dpo/margin_mean": 43.253746032714844,
"margin_dpo/margin_std": 72.98222351074219,
"step": 400
},
{
"KL/chosen_KL_mean": -96.62054443359375,
"KL/mean": -127.06369018554688,
"KL/rejected_KL_mean": -157.50680541992188,
"KL/std": 78.07173156738281,
"epoch": 0.6061980347694633,
"fcm_dpo/beta": 0.007243777625262737,
"fcm_dpo/delta": -0.043214187026023865,
"fcm_dpo/margin": 60.88626480102539,
"fcm_dpo/q_t": 0.3989385664463043,
"grad_norm": 9.888081550598145,
"learning_rate": 2.0268718890989752e-07,
"logits/chosen": 0.7789514064788818,
"logits/rejected": 0.675485372543335,
"logps/chosen": -150.3455047607422,
"logps/ref_chosen": -53.72495651245117,
"logps/ref_rejected": -75.06304931640625,
"logps/rejected": -232.56985473632812,
"loss": 1.0602,
"margin_dpo/margin_mean": 60.886268615722656,
"margin_dpo/margin_std": 73.91134643554688,
"step": 401
},
{
"KL/chosen_KL_mean": -105.51597595214844,
"KL/mean": -130.65469360351562,
"KL/rejected_KL_mean": -155.7934112548828,
"KL/std": 69.48705291748047,
"epoch": 0.6077097505668935,
"fcm_dpo/beta": 0.007242328487336636,
"fcm_dpo/delta": 0.036665141582489014,
"fcm_dpo/margin": 50.277435302734375,
"fcm_dpo/q_t": 0.41705572605133057,
"grad_norm": 13.188558578491211,
"learning_rate": 2.013895317751323e-07,
"logits/chosen": 0.7016223073005676,
"logits/rejected": 0.67276930809021,
"logps/chosen": -167.38990783691406,
"logps/ref_chosen": -61.873931884765625,
"logps/ref_rejected": -66.15198516845703,
"logps/rejected": -221.94540405273438,
"loss": 1.1449,
"margin_dpo/margin_mean": 50.27743911743164,
"margin_dpo/margin_std": 82.9628677368164,
"step": 402
},
{
"KL/chosen_KL_mean": -113.32461547851562,
"KL/mean": -143.94859313964844,
"KL/rejected_KL_mean": -174.57257080078125,
"KL/std": 77.32037353515625,
"epoch": 0.6092214663643235,
"fcm_dpo/beta": 0.007255699019879103,
"fcm_dpo/delta": -0.046439509838819504,
"fcm_dpo/margin": 61.24797058105469,
"fcm_dpo/q_t": 0.3994213938713074,
"grad_norm": 11.108885765075684,
"learning_rate": 2.0009323437965898e-07,
"logits/chosen": 0.8246089816093445,
"logits/rejected": 0.7362926006317139,
"logps/chosen": -164.6461181640625,
"logps/ref_chosen": -51.321502685546875,
"logps/ref_rejected": -86.54010772705078,
"logps/rejected": -261.1126708984375,
"loss": 1.0799,
"margin_dpo/margin_mean": 61.24797058105469,
"margin_dpo/margin_std": 84.76484680175781,
"step": 403
},
{
"KL/chosen_KL_mean": -105.19761657714844,
"KL/mean": -136.69189453125,
"KL/rejected_KL_mean": -168.18617248535156,
"KL/std": 78.5311508178711,
"epoch": 0.6107331821617535,
"fcm_dpo/beta": 0.007129160687327385,
"fcm_dpo/delta": -0.052636247128248215,
"fcm_dpo/margin": 62.98854064941406,
"fcm_dpo/q_t": 0.39832448959350586,
"grad_norm": 13.471323013305664,
"learning_rate": 1.9879833298370237e-07,
"logits/chosen": 0.6983689069747925,
"logits/rejected": 0.5986815690994263,
"logps/chosen": -167.46051025390625,
"logps/ref_chosen": -62.26288604736328,
"logps/ref_rejected": -95.19029998779297,
"logps/rejected": -263.37646484375,
"loss": 1.0766,
"margin_dpo/margin_mean": 62.98854064941406,
"margin_dpo/margin_std": 84.63746643066406,
"step": 404
},
{
"KL/chosen_KL_mean": -109.91860961914062,
"KL/mean": -134.79385375976562,
"KL/rejected_KL_mean": -159.66909790039062,
"KL/std": 71.22305297851562,
"epoch": 0.6122448979591837,
"fcm_dpo/beta": 0.007158408872783184,
"fcm_dpo/delta": 0.044883888214826584,
"fcm_dpo/margin": 49.75050354003906,
"fcm_dpo/q_t": 0.4189513325691223,
"grad_norm": 11.214527130126953,
"learning_rate": 1.975048638084379e-07,
"logits/chosen": 0.7808865308761597,
"logits/rejected": 0.7313976287841797,
"logps/chosen": -160.50294494628906,
"logps/ref_chosen": -50.5843391418457,
"logps/ref_rejected": -65.43156433105469,
"logps/rejected": -225.10067749023438,
"loss": 1.1356,
"margin_dpo/margin_mean": 49.7504997253418,
"margin_dpo/margin_std": 75.26756286621094,
"step": 405
},
{
"KL/chosen_KL_mean": -105.98960876464844,
"KL/mean": -138.4895782470703,
"KL/rejected_KL_mean": -170.9895477294922,
"KL/std": 76.79806518554688,
"epoch": 0.6137566137566137,
"fcm_dpo/beta": 0.0071580009534955025,
"fcm_dpo/delta": -0.06840167194604874,
"fcm_dpo/margin": 64.99993896484375,
"fcm_dpo/q_t": 0.39327844977378845,
"grad_norm": 13.47121524810791,
"learning_rate": 1.9621286303497914e-07,
"logits/chosen": 0.7870622873306274,
"logits/rejected": 0.615902841091156,
"logps/chosen": -154.98521423339844,
"logps/ref_chosen": -48.99560546875,
"logps/ref_rejected": -92.47774505615234,
"logps/rejected": -263.46728515625,
"loss": 1.067,
"margin_dpo/margin_mean": 64.99993896484375,
"margin_dpo/margin_std": 87.28829956054688,
"step": 406
},
{
"KL/chosen_KL_mean": -130.16783142089844,
"KL/mean": -155.58700561523438,
"KL/rejected_KL_mean": -181.0061798095703,
"KL/std": 82.6138687133789,
"epoch": 0.6152683295540439,
"fcm_dpo/beta": 0.007182779721915722,
"fcm_dpo/delta": 0.03589393571019173,
"fcm_dpo/margin": 50.83835220336914,
"fcm_dpo/q_t": 0.4170621335506439,
"grad_norm": 13.66480541229248,
"learning_rate": 1.9492236680336483e-07,
"logits/chosen": 0.5793955326080322,
"logits/rejected": 0.5055565237998962,
"logps/chosen": -219.56838989257812,
"logps/ref_chosen": -89.40056610107422,
"logps/ref_rejected": -99.28775024414062,
"logps/rejected": -280.2939453125,
"loss": 1.1435,
"margin_dpo/margin_mean": 50.83835220336914,
"margin_dpo/margin_std": 85.20750427246094,
"step": 407
},
{
"KL/chosen_KL_mean": -99.28434753417969,
"KL/mean": -137.05641174316406,
"KL/rejected_KL_mean": -174.82850646972656,
"KL/std": 74.41302490234375,
"epoch": 0.6167800453514739,
"fcm_dpo/beta": 0.007070041261613369,
"fcm_dpo/delta": -0.14157219231128693,
"fcm_dpo/margin": 75.54414367675781,
"fcm_dpo/q_t": 0.376776784658432,
"grad_norm": 10.324654579162598,
"learning_rate": 1.9363341121154895e-07,
"logits/chosen": 0.7177830934524536,
"logits/rejected": 0.6347646713256836,
"logps/chosen": -153.98825073242188,
"logps/ref_chosen": -54.70391845703125,
"logps/ref_rejected": -73.98648834228516,
"logps/rejected": -248.8149871826172,
"loss": 0.9997,
"margin_dpo/margin_mean": 75.54414367675781,
"margin_dpo/margin_std": 79.42378234863281,
"step": 408
},
{
"KL/chosen_KL_mean": -123.22323608398438,
"KL/mean": -141.80364990234375,
"KL/rejected_KL_mean": -160.38406372070312,
"KL/std": 66.57162475585938,
"epoch": 0.618291761148904,
"fcm_dpo/beta": 0.007112853694707155,
"fcm_dpo/delta": 0.13951367139816284,
"fcm_dpo/margin": 37.16082000732422,
"fcm_dpo/q_t": 0.4397445619106293,
"grad_norm": 12.949170112609863,
"learning_rate": 1.9234603231438994e-07,
"logits/chosen": 0.7330983877182007,
"logits/rejected": 0.7420048713684082,
"logps/chosen": -185.34146118164062,
"logps/ref_chosen": -62.11822509765625,
"logps/ref_rejected": -61.933509826660156,
"logps/rejected": -222.31756591796875,
"loss": 1.2113,
"margin_dpo/margin_mean": 37.16082000732422,
"margin_dpo/margin_std": 76.77306365966797,
"step": 409
},
{
"KL/chosen_KL_mean": -115.03919982910156,
"KL/mean": -145.41026306152344,
"KL/rejected_KL_mean": -175.78134155273438,
"KL/std": 71.03024291992188,
"epoch": 0.6198034769463341,
"fcm_dpo/beta": 0.007094179280102253,
"fcm_dpo/delta": -0.0332571342587471,
"fcm_dpo/margin": 60.74213790893555,
"fcm_dpo/q_t": 0.3995182514190674,
"grad_norm": 11.657567977905273,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 0.7212764620780945,
"logits/rejected": 0.6957427263259888,
"logps/chosen": -176.84185791015625,
"logps/ref_chosen": -61.80266189575195,
"logps/ref_rejected": -76.60002136230469,
"logps/rejected": -252.38134765625,
"loss": 1.0589,
"margin_dpo/margin_mean": 60.74213790893555,
"margin_dpo/margin_std": 67.82354736328125,
"step": 410
},
{
"KL/chosen_KL_mean": -119.87579345703125,
"KL/mean": -149.50830078125,
"KL/rejected_KL_mean": -179.14080810546875,
"KL/std": 79.36740112304688,
"epoch": 0.6213151927437641,
"fcm_dpo/beta": 0.007109199650585651,
"fcm_dpo/delta": -0.022265002131462097,
"fcm_dpo/margin": 59.26503372192383,
"fcm_dpo/q_t": 0.4039009213447571,
"grad_norm": 10.137211799621582,
"learning_rate": 1.8977614860195296e-07,
"logits/chosen": 0.7379822134971619,
"logits/rejected": 0.6752569675445557,
"logps/chosen": -174.32118225097656,
"logps/ref_chosen": -54.44539260864258,
"logps/ref_rejected": -74.5650863647461,
"logps/rejected": -253.70590209960938,
"loss": 1.0944,
"margin_dpo/margin_mean": 59.26503372192383,
"margin_dpo/margin_std": 85.11161804199219,
"step": 411
},
{
"KL/chosen_KL_mean": -125.6766357421875,
"KL/mean": -153.53085327148438,
"KL/rejected_KL_mean": -181.38507080078125,
"KL/std": 70.50398254394531,
"epoch": 0.6228269085411943,
"fcm_dpo/beta": 0.007074539549648762,
"fcm_dpo/delta": 0.005894448608160019,
"fcm_dpo/margin": 55.70844268798828,
"fcm_dpo/q_t": 0.4094070792198181,
"grad_norm": 12.71510124206543,
"learning_rate": 1.8849371567184662e-07,
"logits/chosen": 0.7475090622901917,
"logits/rejected": 0.6767639517784119,
"logps/chosen": -180.92471313476562,
"logps/ref_chosen": -55.248085021972656,
"logps/ref_rejected": -68.96623229980469,
"logps/rejected": -250.35128784179688,
"loss": 1.0995,
"margin_dpo/margin_mean": 55.70844268798828,
"margin_dpo/margin_std": 75.81526184082031,
"step": 412
},
{
"KL/chosen_KL_mean": -137.35931396484375,
"KL/mean": -161.19117736816406,
"KL/rejected_KL_mean": -185.02304077148438,
"KL/std": 74.88512420654297,
"epoch": 0.6243386243386243,
"fcm_dpo/beta": 0.00717338128015399,
"fcm_dpo/delta": 0.0600578673183918,
"fcm_dpo/margin": 47.66373825073242,
"fcm_dpo/q_t": 0.4230996072292328,
"grad_norm": 14.152400970458984,
"learning_rate": 1.872130032047302e-07,
"logits/chosen": 0.5468255877494812,
"logits/rejected": 0.5104795694351196,
"logps/chosen": -206.08004760742188,
"logps/ref_chosen": -68.72074890136719,
"logps/ref_rejected": -78.76539611816406,
"logps/rejected": -263.7884521484375,
"loss": 1.1839,
"margin_dpo/margin_mean": 47.66373825073242,
"margin_dpo/margin_std": 94.72030639648438,
"step": 413
},
{
"KL/chosen_KL_mean": -120.72862243652344,
"KL/mean": -150.89007568359375,
"KL/rejected_KL_mean": -181.05152893066406,
"KL/std": 80.94624328613281,
"epoch": 0.6258503401360545,
"fcm_dpo/beta": 0.007161266636103392,
"fcm_dpo/delta": -0.03342374414205551,
"fcm_dpo/margin": 60.32288360595703,
"fcm_dpo/q_t": 0.4002327024936676,
"grad_norm": 12.253021240234375,
"learning_rate": 1.8593404702488436e-07,
"logits/chosen": 0.7384845018386841,
"logits/rejected": 0.6730071902275085,
"logps/chosen": -174.86683654785156,
"logps/ref_chosen": -54.138214111328125,
"logps/ref_rejected": -74.65741729736328,
"logps/rejected": -255.7089385986328,
"loss": 1.0756,
"margin_dpo/margin_mean": 60.32288360595703,
"margin_dpo/margin_std": 78.53474426269531,
"step": 414
},
{
"KL/chosen_KL_mean": -121.16802978515625,
"KL/mean": -147.07374572753906,
"KL/rejected_KL_mean": -172.97946166992188,
"KL/std": 77.89082336425781,
"epoch": 0.6273620559334845,
"fcm_dpo/beta": 0.007157396525144577,
"fcm_dpo/delta": 0.03026522323489189,
"fcm_dpo/margin": 51.811431884765625,
"fcm_dpo/q_t": 0.4152563512325287,
"grad_norm": 12.313409805297852,
"learning_rate": 1.846568829074628e-07,
"logits/chosen": 0.7715727090835571,
"logits/rejected": 0.7539013624191284,
"logps/chosen": -177.0865936279297,
"logps/ref_chosen": -55.91856002807617,
"logps/ref_rejected": -61.747703552246094,
"logps/rejected": -234.7271728515625,
"loss": 1.1356,
"margin_dpo/margin_mean": 51.811431884765625,
"margin_dpo/margin_std": 84.16819763183594,
"step": 415
},
{
"KL/chosen_KL_mean": -127.87203216552734,
"KL/mean": -150.220947265625,
"KL/rejected_KL_mean": -172.5698699951172,
"KL/std": 80.14581298828125,
"epoch": 0.6288737717309146,
"fcm_dpo/beta": 0.007121403701603413,
"fcm_dpo/delta": -0.08652918040752411,
"fcm_dpo/margin": 44.69782257080078,
"fcm_dpo/q_t": 0.4277215003967285,
"grad_norm": 13.798969268798828,
"learning_rate": 1.8338154657749128e-07,
"logits/chosen": 0.7046973705291748,
"logits/rejected": 0.652430534362793,
"logps/chosen": -182.59512329101562,
"logps/ref_chosen": -54.72308349609375,
"logps/ref_rejected": -69.17388916015625,
"logps/rejected": -241.74375915527344,
"loss": 1.1837,
"margin_dpo/margin_mean": 44.69782257080078,
"margin_dpo/margin_std": 79.43450927734375,
"step": 416
},
{
"KL/chosen_KL_mean": -131.48974609375,
"KL/mean": -161.3472900390625,
"KL/rejected_KL_mean": -191.204833984375,
"KL/std": 75.80394744873047,
"epoch": 0.6303854875283447,
"fcm_dpo/beta": 0.007065145764499903,
"fcm_dpo/delta": -0.023121818900108337,
"fcm_dpo/margin": 59.71507263183594,
"fcm_dpo/q_t": 0.4024716019630432,
"grad_norm": 12.70583438873291,
"learning_rate": 1.8210807370886849e-07,
"logits/chosen": 0.8322412967681885,
"logits/rejected": 0.7600584030151367,
"logps/chosen": -188.281005859375,
"logps/ref_chosen": -56.791259765625,
"logps/ref_rejected": -68.7791748046875,
"logps/rejected": -259.9840087890625,
"loss": 1.1073,
"margin_dpo/margin_mean": 59.71507263183594,
"margin_dpo/margin_std": 91.33202362060547,
"step": 417
},
{
"KL/chosen_KL_mean": -140.48382568359375,
"KL/mean": -163.9290313720703,
"KL/rejected_KL_mean": -187.37423706054688,
"KL/std": 80.66316223144531,
"epoch": 0.6318972033257747,
"fcm_dpo/beta": 0.006957621779292822,
"fcm_dpo/delta": -0.04976249113678932,
"fcm_dpo/margin": 46.89039993286133,
"fcm_dpo/q_t": 0.42588043212890625,
"grad_norm": 13.409867286682129,
"learning_rate": 1.8083649992336825e-07,
"logits/chosen": 0.7194592952728271,
"logits/rejected": 0.7256894111633301,
"logps/chosen": -209.59182739257812,
"logps/ref_chosen": -69.10798645019531,
"logps/ref_rejected": -75.09132385253906,
"logps/rejected": -262.465576171875,
"loss": 1.1682,
"margin_dpo/margin_mean": 46.89039611816406,
"margin_dpo/margin_std": 81.53071594238281,
"step": 418
},
{
"KL/chosen_KL_mean": -112.98391723632812,
"KL/mean": -146.56724548339844,
"KL/rejected_KL_mean": -180.15057373046875,
"KL/std": 78.52078247070312,
"epoch": 0.6334089191232048,
"fcm_dpo/beta": 0.006877239793539047,
"fcm_dpo/delta": -0.06512196362018585,
"fcm_dpo/margin": 67.16665649414062,
"fcm_dpo/q_t": 0.3954851031303406,
"grad_norm": 12.344082832336426,
"learning_rate": 1.7956686078964255e-07,
"logits/chosen": 0.608430027961731,
"logits/rejected": 0.5550130605697632,
"logps/chosen": -171.15567016601562,
"logps/ref_chosen": -58.1717643737793,
"logps/ref_rejected": -71.67066955566406,
"logps/rejected": -251.8212432861328,
"loss": 1.0618,
"margin_dpo/margin_mean": 67.16665649414062,
"margin_dpo/margin_std": 88.19050598144531,
"step": 419
},
{
"KL/chosen_KL_mean": -141.04054260253906,
"KL/mean": -159.03802490234375,
"KL/rejected_KL_mean": -177.03549194335938,
"KL/std": 79.94134521484375,
"epoch": 0.6349206349206349,
"fcm_dpo/beta": 0.006931029260158539,
"fcm_dpo/delta": 0.04597489535808563,
"fcm_dpo/margin": 35.994956970214844,
"fcm_dpo/q_t": 0.44405868649482727,
"grad_norm": 12.823692321777344,
"learning_rate": 1.782991918222275e-07,
"logits/chosen": 0.7288790941238403,
"logits/rejected": 0.6808423399925232,
"logps/chosen": -198.09405517578125,
"logps/ref_chosen": -57.05351257324219,
"logps/ref_rejected": -62.670982360839844,
"logps/rejected": -239.70648193359375,
"loss": 1.2487,
"margin_dpo/margin_mean": 35.994956970214844,
"margin_dpo/margin_std": 91.39073181152344,
"step": 420
},
{
"KL/chosen_KL_mean": -132.10513305664062,
"KL/mean": -157.54034423828125,
"KL/rejected_KL_mean": -182.97557067871094,
"KL/std": 79.04115295410156,
"epoch": 0.636432350718065,
"fcm_dpo/beta": 0.0069469278678298,
"fcm_dpo/delta": 0.0480603352189064,
"fcm_dpo/margin": 50.87043762207031,
"fcm_dpo/q_t": 0.4208451211452484,
"grad_norm": 13.627217292785645,
"learning_rate": 1.7703352848054887e-07,
"logits/chosen": 0.6675734519958496,
"logits/rejected": 0.606522798538208,
"logps/chosen": -189.42837524414062,
"logps/ref_chosen": -57.32324981689453,
"logps/ref_rejected": -75.33782958984375,
"logps/rejected": -258.31341552734375,
"loss": 1.1829,
"margin_dpo/margin_mean": 50.87043380737305,
"margin_dpo/margin_std": 101.04164123535156,
"step": 421
},
{
"KL/chosen_KL_mean": -114.57057189941406,
"KL/mean": -148.39588928222656,
"KL/rejected_KL_mean": -182.22122192382812,
"KL/std": 79.06591796875,
"epoch": 0.6379440665154951,
"fcm_dpo/beta": 0.006941578350961208,
"fcm_dpo/delta": -0.07298602163791656,
"fcm_dpo/margin": 67.65065002441406,
"fcm_dpo/q_t": 0.3924116790294647,
"grad_norm": 14.082544326782227,
"learning_rate": 1.7576990616793137e-07,
"logits/chosen": 0.7062339782714844,
"logits/rejected": 0.6935118436813354,
"logps/chosen": -181.62814331054688,
"logps/ref_chosen": -67.05757141113281,
"logps/ref_rejected": -72.12803649902344,
"logps/rejected": -254.34925842285156,
"loss": 1.0468,
"margin_dpo/margin_mean": 67.65065002441406,
"margin_dpo/margin_std": 81.65357971191406,
"step": 422
},
{
"KL/chosen_KL_mean": -118.56727600097656,
"KL/mean": -152.74942016601562,
"KL/rejected_KL_mean": -186.9315643310547,
"KL/std": 81.0578842163086,
"epoch": 0.6394557823129252,
"fcm_dpo/beta": 0.006814665626734495,
"fcm_dpo/delta": -0.06926769018173218,
"fcm_dpo/margin": 68.36428833007812,
"fcm_dpo/q_t": 0.3939523696899414,
"grad_norm": 11.488704681396484,
"learning_rate": 1.745083602306071e-07,
"logits/chosen": 0.7420529723167419,
"logits/rejected": 0.6691204905509949,
"logps/chosen": -172.6289520263672,
"logps/ref_chosen": -54.06167221069336,
"logps/ref_rejected": -76.64092254638672,
"logps/rejected": -263.572509765625,
"loss": 1.0539,
"margin_dpo/margin_mean": 68.36428833007812,
"margin_dpo/margin_std": 86.33656311035156,
"step": 423
},
{
"KL/chosen_KL_mean": -128.16744995117188,
"KL/mean": -160.90489196777344,
"KL/rejected_KL_mean": -193.64230346679688,
"KL/std": 77.58207702636719,
"epoch": 0.6409674981103552,
"fcm_dpo/beta": 0.006721400655806065,
"fcm_dpo/delta": -0.04278453439474106,
"fcm_dpo/margin": 65.47486114501953,
"fcm_dpo/q_t": 0.3996415138244629,
"grad_norm": 15.941903114318848,
"learning_rate": 1.7324892595672804e-07,
"logits/chosen": 0.6120574474334717,
"logits/rejected": 0.5706311464309692,
"logps/chosen": -181.7763214111328,
"logps/ref_chosen": -53.60887145996094,
"logps/ref_rejected": -79.2139892578125,
"logps/rejected": -272.8563232421875,
"loss": 1.0772,
"margin_dpo/margin_mean": 65.47486114501953,
"margin_dpo/margin_std": 87.71736145019531,
"step": 424
},
{
"KL/chosen_KL_mean": -126.18075561523438,
"KL/mean": -152.52755737304688,
"KL/rejected_KL_mean": -178.87435913085938,
"KL/std": 75.77655029296875,
"epoch": 0.6424792139077853,
"fcm_dpo/beta": 0.0067849173210561275,
"fcm_dpo/delta": 0.044069744646549225,
"fcm_dpo/margin": 52.6936149597168,
"fcm_dpo/q_t": 0.418659508228302,
"grad_norm": 13.16757583618164,
"learning_rate": 1.7199163857537824e-07,
"logits/chosen": 0.7674802541732788,
"logits/rejected": 0.737590491771698,
"logps/chosen": -184.59542846679688,
"logps/ref_chosen": -58.41468048095703,
"logps/ref_rejected": -66.59054565429688,
"logps/rejected": -245.46490478515625,
"loss": 1.1416,
"margin_dpo/margin_mean": 52.6936149597168,
"margin_dpo/margin_std": 86.34124755859375,
"step": 425
},
{
"KL/chosen_KL_mean": -149.9374237060547,
"KL/mean": -166.315673828125,
"KL/rejected_KL_mean": -182.69393920898438,
"KL/std": 78.09856414794922,
"epoch": 0.6439909297052154,
"fcm_dpo/beta": 0.006984601728618145,
"fcm_dpo/delta": 0.17520646750926971,
"fcm_dpo/margin": 32.756500244140625,
"fcm_dpo/q_t": 0.4476398527622223,
"grad_norm": 16.125707626342773,
"learning_rate": 1.7073653325558828e-07,
"logits/chosen": 0.6880191564559937,
"logits/rejected": 0.6956747770309448,
"logps/chosen": -221.64564514160156,
"logps/ref_chosen": -71.70822143554688,
"logps/ref_rejected": -73.57725524902344,
"logps/rejected": -256.27117919921875,
"loss": 1.2788,
"margin_dpo/margin_mean": 32.75650405883789,
"margin_dpo/margin_std": 97.20096588134766,
"step": 426
},
{
"KL/chosen_KL_mean": -140.72433471679688,
"KL/mean": -166.97764587402344,
"KL/rejected_KL_mean": -193.23094177246094,
"KL/std": 83.52520751953125,
"epoch": 0.6455026455026455,
"fcm_dpo/beta": 0.007065876387059689,
"fcm_dpo/delta": 0.030104748904705048,
"fcm_dpo/margin": 52.50662612915039,
"fcm_dpo/q_t": 0.41716307401657104,
"grad_norm": 14.184538841247559,
"learning_rate": 1.6948364510535218e-07,
"logits/chosen": 0.7531858682632446,
"logits/rejected": 0.6878103017807007,
"logps/chosen": -199.36709594726562,
"logps/ref_chosen": -58.64276885986328,
"logps/ref_rejected": -86.25437927246094,
"logps/rejected": -279.4853210449219,
"loss": 1.1533,
"margin_dpo/margin_mean": 52.506629943847656,
"margin_dpo/margin_std": 94.57223510742188,
"step": 427
},
{
"KL/chosen_KL_mean": -133.34292602539062,
"KL/mean": -163.7454376220703,
"KL/rejected_KL_mean": -194.14794921875,
"KL/std": 85.47407531738281,
"epoch": 0.6470143613000756,
"fcm_dpo/beta": 0.007083693519234657,
"fcm_dpo/delta": -0.032318491488695145,
"fcm_dpo/margin": 60.80500793457031,
"fcm_dpo/q_t": 0.40341562032699585,
"grad_norm": 13.646878242492676,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 0.6438695192337036,
"logits/rejected": 0.5988097786903381,
"logps/chosen": -199.93898010253906,
"logps/ref_chosen": -66.5960464477539,
"logps/ref_rejected": -82.3941650390625,
"logps/rejected": -276.5421142578125,
"loss": 1.1006,
"margin_dpo/margin_mean": 60.80500793457031,
"margin_dpo/margin_std": 91.67237854003906,
"step": 428
},
{
"KL/chosen_KL_mean": -137.42991638183594,
"KL/mean": -160.76165771484375,
"KL/rejected_KL_mean": -184.09344482421875,
"KL/std": 77.081298828125,
"epoch": 0.6485260770975056,
"fcm_dpo/beta": 0.007148797623813152,
"fcm_dpo/delta": 0.06791189312934875,
"fcm_dpo/margin": 46.66352844238281,
"fcm_dpo/q_t": 0.42376774549484253,
"grad_norm": 14.72235107421875,
"learning_rate": 1.669846604344412e-07,
"logits/chosen": 0.6531593799591064,
"logits/rejected": 0.6712849140167236,
"logps/chosen": -194.43960571289062,
"logps/ref_chosen": -57.00970458984375,
"logps/ref_rejected": -59.86549377441406,
"logps/rejected": -243.95892333984375,
"loss": 1.1753,
"margin_dpo/margin_mean": 46.66352844238281,
"margin_dpo/margin_std": 87.03643035888672,
"step": 429
},
{
"KL/chosen_KL_mean": -123.23287963867188,
"KL/mean": -161.01443481445312,
"KL/rejected_KL_mean": -198.79598999023438,
"KL/std": 79.27754974365234,
"epoch": 0.6500377928949358,
"fcm_dpo/beta": 0.006995225325226784,
"fcm_dpo/delta": -0.13578736782073975,
"fcm_dpo/margin": 75.56310272216797,
"fcm_dpo/q_t": 0.37848204374313354,
"grad_norm": 12.778807640075684,
"learning_rate": 1.6573863381573954e-07,
"logits/chosen": 0.5895907878875732,
"logits/rejected": 0.5904099345207214,
"logps/chosen": -182.79608154296875,
"logps/ref_chosen": -59.563194274902344,
"logps/ref_rejected": -70.52289581298828,
"logps/rejected": -269.3188781738281,
"loss": 1.0211,
"margin_dpo/margin_mean": 75.56310272216797,
"margin_dpo/margin_std": 90.74784851074219,
"step": 430
},
{
"KL/chosen_KL_mean": -120.74576568603516,
"KL/mean": -147.44810485839844,
"KL/rejected_KL_mean": -174.15042114257812,
"KL/std": 79.4557113647461,
"epoch": 0.6515495086923658,
"fcm_dpo/beta": 0.006972130853682756,
"fcm_dpo/delta": 0.02873518317937851,
"fcm_dpo/margin": 53.404659271240234,
"fcm_dpo/q_t": 0.41524261236190796,
"grad_norm": 12.930608749389648,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 0.666712760925293,
"logits/rejected": 0.6126998662948608,
"logps/chosen": -170.9460906982422,
"logps/ref_chosen": -50.20032501220703,
"logps/ref_rejected": -77.81680297851562,
"logps/rejected": -251.96722412109375,
"loss": 1.1349,
"margin_dpo/margin_mean": 53.404659271240234,
"margin_dpo/margin_std": 86.75981140136719,
"step": 431
},
{
"KL/chosen_KL_mean": -126.7294921875,
"KL/mean": -155.13113403320312,
"KL/rejected_KL_mean": -183.53280639648438,
"KL/std": 76.79438781738281,
"epoch": 0.6530612244897959,
"fcm_dpo/beta": 0.00698945764452219,
"fcm_dpo/delta": 0.003098210785537958,
"fcm_dpo/margin": 56.80329895019531,
"fcm_dpo/q_t": 0.40925368666648865,
"grad_norm": 13.003230094909668,
"learning_rate": 1.632536862810844e-07,
"logits/chosen": 0.697510302066803,
"logits/rejected": 0.6457198858261108,
"logps/chosen": -188.39224243164062,
"logps/ref_chosen": -61.662757873535156,
"logps/ref_rejected": -83.94496154785156,
"logps/rejected": -267.4777526855469,
"loss": 1.1201,
"margin_dpo/margin_mean": 56.80329895019531,
"margin_dpo/margin_std": 89.55986785888672,
"step": 432
},
{
"KL/chosen_KL_mean": -125.5418472290039,
"KL/mean": -160.45352172851562,
"KL/rejected_KL_mean": -195.36521911621094,
"KL/std": 75.46558380126953,
"epoch": 0.654572940287226,
"fcm_dpo/beta": 0.006921480409801006,
"fcm_dpo/delta": -0.08742604404687881,
"fcm_dpo/margin": 69.82337951660156,
"fcm_dpo/q_t": 0.38870155811309814,
"grad_norm": 13.009313583374023,
"learning_rate": 1.6201483487445515e-07,
"logits/chosen": 0.783934473991394,
"logits/rejected": 0.7832895517349243,
"logps/chosen": -189.27102661132812,
"logps/ref_chosen": -63.72917938232422,
"logps/ref_rejected": -65.8391342163086,
"logps/rejected": -261.204345703125,
"loss": 1.0468,
"margin_dpo/margin_mean": 69.82337188720703,
"margin_dpo/margin_std": 86.8525619506836,
"step": 433
},
{
"KL/chosen_KL_mean": -105.35610961914062,
"KL/mean": -143.14828491210938,
"KL/rejected_KL_mean": -180.9404754638672,
"KL/std": 82.13607025146484,
"epoch": 0.656084656084656,
"fcm_dpo/beta": 0.00671165157109499,
"fcm_dpo/delta": -0.11462040990591049,
"fcm_dpo/margin": 75.5843734741211,
"fcm_dpo/q_t": 0.38376089930534363,
"grad_norm": 12.505172729492188,
"learning_rate": 1.6077844460203204e-07,
"logits/chosen": 0.8338220119476318,
"logits/rejected": 0.7672078609466553,
"logps/chosen": -153.3294219970703,
"logps/ref_chosen": -47.97331619262695,
"logps/ref_rejected": -72.51132202148438,
"logps/rejected": -253.45179748535156,
"loss": 1.0492,
"margin_dpo/margin_mean": 75.5843734741211,
"margin_dpo/margin_std": 99.7183609008789,
"step": 434
},
{
"KL/chosen_KL_mean": -129.03106689453125,
"KL/mean": -158.38502502441406,
"KL/rejected_KL_mean": -187.73898315429688,
"KL/std": 78.53376770019531,
"epoch": 0.6575963718820862,
"fcm_dpo/beta": 0.006749986670911312,
"fcm_dpo/delta": 0.003425680100917816,
"fcm_dpo/margin": 58.70793914794922,
"fcm_dpo/q_t": 0.41001203656196594,
"grad_norm": 13.364961624145508,
"learning_rate": 1.5954455004830878e-07,
"logits/chosen": 0.8315505981445312,
"logits/rejected": 0.7905421853065491,
"logps/chosen": -186.09130859375,
"logps/ref_chosen": -57.06024932861328,
"logps/ref_rejected": -71.69146728515625,
"logps/rejected": -259.4304504394531,
"loss": 1.1189,
"margin_dpo/margin_mean": 58.70793533325195,
"margin_dpo/margin_std": 90.6599349975586,
"step": 435
},
{
"KL/chosen_KL_mean": -127.37867736816406,
"KL/mean": -150.99383544921875,
"KL/rejected_KL_mean": -174.60897827148438,
"KL/std": 77.64402770996094,
"epoch": 0.6591080876795162,
"fcm_dpo/beta": 0.006800387986004353,
"fcm_dpo/delta": 0.08145187795162201,
"fcm_dpo/margin": 47.23029708862305,
"fcm_dpo/q_t": 0.4273446202278137,
"grad_norm": 14.902657508850098,
"learning_rate": 1.5831318572796847e-07,
"logits/chosen": 0.721663773059845,
"logits/rejected": 0.6628165245056152,
"logps/chosen": -183.53672790527344,
"logps/ref_chosen": -56.158050537109375,
"logps/ref_rejected": -67.63787841796875,
"logps/rejected": -242.24685668945312,
"loss": 1.192,
"margin_dpo/margin_mean": 47.23029708862305,
"margin_dpo/margin_std": 96.21711730957031,
"step": 436
},
{
"KL/chosen_KL_mean": -134.5484619140625,
"KL/mean": -161.63583374023438,
"KL/rejected_KL_mean": -188.72320556640625,
"KL/std": 81.81932067871094,
"epoch": 0.6606198034769464,
"fcm_dpo/beta": 0.006728970445692539,
"fcm_dpo/delta": -0.07480433583259583,
"fcm_dpo/margin": 54.17472457885742,
"fcm_dpo/q_t": 0.416775107383728,
"grad_norm": 16.643497467041016,
"learning_rate": 1.5708438608491815e-07,
"logits/chosen": 0.714606523513794,
"logits/rejected": 0.5819742679595947,
"logps/chosen": -191.5342559814453,
"logps/ref_chosen": -56.98578643798828,
"logps/ref_rejected": -85.61524963378906,
"logps/rejected": -274.33843994140625,
"loss": 1.172,
"margin_dpo/margin_mean": 54.174720764160156,
"margin_dpo/margin_std": 102.02175903320312,
"step": 437
},
{
"KL/chosen_KL_mean": -115.70340728759766,
"KL/mean": -153.13702392578125,
"KL/rejected_KL_mean": -190.57061767578125,
"KL/std": 85.75027465820312,
"epoch": 0.6621315192743764,
"fcm_dpo/beta": 0.006647449918091297,
"fcm_dpo/delta": -0.10266944766044617,
"fcm_dpo/margin": 74.86722564697266,
"fcm_dpo/q_t": 0.38830190896987915,
"grad_norm": 12.799489974975586,
"learning_rate": 1.558581854913253e-07,
"logits/chosen": 0.7843307852745056,
"logits/rejected": 0.7155150175094604,
"logps/chosen": -156.98118591308594,
"logps/ref_chosen": -41.27777862548828,
"logps/ref_rejected": -65.33840942382812,
"logps/rejected": -255.90904235839844,
"loss": 1.0366,
"margin_dpo/margin_mean": 74.86723327636719,
"margin_dpo/margin_std": 92.356689453125,
"step": 438
},
{
"KL/chosen_KL_mean": -130.4723663330078,
"KL/mean": -161.6959991455078,
"KL/rejected_KL_mean": -192.91961669921875,
"KL/std": 87.09983825683594,
"epoch": 0.6636432350718064,
"fcm_dpo/beta": 0.006615322083234787,
"fcm_dpo/delta": -0.01423458382487297,
"fcm_dpo/margin": 62.4472541809082,
"fcm_dpo/q_t": 0.4059128165245056,
"grad_norm": 13.447953224182129,
"learning_rate": 1.5463461824665658e-07,
"logits/chosen": 0.6172465682029724,
"logits/rejected": 0.5801492929458618,
"logps/chosen": -211.8900146484375,
"logps/ref_chosen": -81.41764831542969,
"logps/ref_rejected": -94.72309875488281,
"logps/rejected": -287.6427001953125,
"loss": 1.0982,
"margin_dpo/margin_mean": 62.44725799560547,
"margin_dpo/margin_std": 88.8106460571289,
"step": 439
},
{
"KL/chosen_KL_mean": -115.12965393066406,
"KL/mean": -147.96469116210938,
"KL/rejected_KL_mean": -180.79971313476562,
"KL/std": 79.61054992675781,
"epoch": 0.6651549508692366,
"fcm_dpo/beta": 0.006573637016117573,
"fcm_dpo/delta": -0.033462464809417725,
"fcm_dpo/margin": 65.67005920410156,
"fcm_dpo/q_t": 0.4016228914260864,
"grad_norm": 18.106090545654297,
"learning_rate": 1.534137185767178e-07,
"logits/chosen": 0.7140671014785767,
"logits/rejected": 0.6113680601119995,
"logps/chosen": -157.6678466796875,
"logps/ref_chosen": -42.538185119628906,
"logps/ref_rejected": -69.78813934326172,
"logps/rejected": -250.58786010742188,
"loss": 1.0962,
"margin_dpo/margin_mean": 65.67005920410156,
"margin_dpo/margin_std": 95.45274353027344,
"step": 440
},
{
"KL/chosen_KL_mean": -113.79468536376953,
"KL/mean": -149.21543884277344,
"KL/rejected_KL_mean": -184.63619995117188,
"KL/std": 82.9810562133789,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.006445100996643305,
"fcm_dpo/delta": -0.05988113582134247,
"fcm_dpo/margin": 70.84149932861328,
"fcm_dpo/q_t": 0.3934841454029083,
"grad_norm": 14.604828834533691,
"learning_rate": 1.521955206326976e-07,
"logits/chosen": 0.7096047401428223,
"logits/rejected": 0.609955906867981,
"logps/chosen": -171.38790893554688,
"logps/ref_chosen": -57.593223571777344,
"logps/ref_rejected": -84.82878875732422,
"logps/rejected": -269.4649963378906,
"loss": 1.0385,
"margin_dpo/margin_mean": 70.84149932861328,
"margin_dpo/margin_std": 75.62371826171875,
"step": 441
},
{
"KL/chosen_KL_mean": -137.17593383789062,
"KL/mean": -171.54620361328125,
"KL/rejected_KL_mean": -205.91647338867188,
"KL/std": 82.21332550048828,
"epoch": 0.6681783824640968,
"fcm_dpo/beta": 0.006400700658559799,
"fcm_dpo/delta": -0.041960593312978745,
"fcm_dpo/margin": 68.74054718017578,
"fcm_dpo/q_t": 0.3983391225337982,
"grad_norm": 14.551726341247559,
"learning_rate": 1.5098005849021078e-07,
"logits/chosen": 0.6707921028137207,
"logits/rejected": 0.6180996894836426,
"logps/chosen": -204.63714599609375,
"logps/ref_chosen": -67.46121978759766,
"logps/ref_rejected": -89.0693588256836,
"logps/rejected": -294.98583984375,
"loss": 1.0658,
"margin_dpo/margin_mean": 68.74055480957031,
"margin_dpo/margin_std": 86.2884292602539,
"step": 442
},
{
"KL/chosen_KL_mean": -113.61299133300781,
"KL/mean": -156.89840698242188,
"KL/rejected_KL_mean": -200.18382263183594,
"KL/std": 87.94349670410156,
"epoch": 0.6696900982615268,
"fcm_dpo/beta": 0.00625761691480875,
"fcm_dpo/delta": -0.14983615279197693,
"fcm_dpo/margin": 86.57083129882812,
"fcm_dpo/q_t": 0.3756743371486664,
"grad_norm": 13.017277717590332,
"learning_rate": 1.4976736614834662e-07,
"logits/chosen": 0.7221077680587769,
"logits/rejected": 0.6516605019569397,
"logps/chosen": -168.40908813476562,
"logps/ref_chosen": -54.79610061645508,
"logps/ref_rejected": -77.80781555175781,
"logps/rejected": -277.99163818359375,
"loss": 1.0034,
"margin_dpo/margin_mean": 86.57083129882812,
"margin_dpo/margin_std": 98.43537902832031,
"step": 443
},
{
"KL/chosen_KL_mean": -141.99095153808594,
"KL/mean": -158.5377960205078,
"KL/rejected_KL_mean": -175.08465576171875,
"KL/std": 85.75154113769531,
"epoch": 0.671201814058957,
"fcm_dpo/beta": 0.0062613519839942455,
"fcm_dpo/delta": 0.04191405326128006,
"fcm_dpo/margin": 33.093685150146484,
"fcm_dpo/q_t": 0.4531075954437256,
"grad_norm": 16.001445770263672,
"learning_rate": 1.4855747752871654e-07,
"logits/chosen": 0.7393509149551392,
"logits/rejected": 0.6406112909317017,
"logps/chosen": -200.74000549316406,
"logps/ref_chosen": -58.749061584472656,
"logps/ref_rejected": -86.87396240234375,
"logps/rejected": -261.9586181640625,
"loss": 1.2785,
"margin_dpo/margin_mean": 33.093685150146484,
"margin_dpo/margin_std": 95.52013397216797,
"step": 444
},
{
"KL/chosen_KL_mean": -132.74156188964844,
"KL/mean": -169.7027130126953,
"KL/rejected_KL_mean": -206.66384887695312,
"KL/std": 83.37004089355469,
"epoch": 0.672713529856387,
"fcm_dpo/beta": 0.006209210492670536,
"fcm_dpo/delta": -0.0618002712726593,
"fcm_dpo/margin": 73.92228698730469,
"fcm_dpo/q_t": 0.39396703243255615,
"grad_norm": 14.041153907775879,
"learning_rate": 1.473504264745062e-07,
"logits/chosen": 0.7065185308456421,
"logits/rejected": 0.6943279504776001,
"logps/chosen": -193.65899658203125,
"logps/ref_chosen": -60.91743850708008,
"logps/ref_rejected": -71.5637435913086,
"logps/rejected": -278.22760009765625,
"loss": 1.0527,
"margin_dpo/margin_mean": 73.92228698730469,
"margin_dpo/margin_std": 90.00228118896484,
"step": 445
},
{
"KL/chosen_KL_mean": -121.05908966064453,
"KL/mean": -158.24090576171875,
"KL/rejected_KL_mean": -195.4227294921875,
"KL/std": 84.6939697265625,
"epoch": 0.674225245653817,
"fcm_dpo/beta": 0.006185232196003199,
"fcm_dpo/delta": -0.06411469727754593,
"fcm_dpo/margin": 74.36363220214844,
"fcm_dpo/q_t": 0.3935438394546509,
"grad_norm": 11.710205078125,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 0.7595170736312866,
"logits/rejected": 0.6753551959991455,
"logps/chosen": -169.85833740234375,
"logps/ref_chosen": -48.79924774169922,
"logps/ref_rejected": -71.8719482421875,
"logps/rejected": -267.294677734375,
"loss": 1.0445,
"margin_dpo/margin_mean": 74.36363220214844,
"margin_dpo/margin_std": 79.353515625,
"step": 446
},
{
"KL/chosen_KL_mean": -118.1234130859375,
"KL/mean": -160.5880126953125,
"KL/rejected_KL_mean": -203.05258178710938,
"KL/std": 83.77147674560547,
"epoch": 0.6757369614512472,
"fcm_dpo/beta": 0.00597979873418808,
"fcm_dpo/delta": -0.11436723172664642,
"fcm_dpo/margin": 84.92918395996094,
"fcm_dpo/q_t": 0.3816147744655609,
"grad_norm": 16.865678787231445,
"learning_rate": 1.4494497203727843e-07,
"logits/chosen": 0.6509027481079102,
"logits/rejected": 0.5459779500961304,
"logps/chosen": -171.80612182617188,
"logps/ref_chosen": -53.682716369628906,
"logps/ref_rejected": -88.17315673828125,
"logps/rejected": -291.22576904296875,
"loss": 1.0253,
"margin_dpo/margin_mean": 84.92918395996094,
"margin_dpo/margin_std": 98.30052947998047,
"step": 447
},
{
"KL/chosen_KL_mean": -126.86201477050781,
"KL/mean": -161.47686767578125,
"KL/rejected_KL_mean": -196.09170532226562,
"KL/std": 84.419921875,
"epoch": 0.6772486772486772,
"fcm_dpo/beta": 0.005955612286925316,
"fcm_dpo/delta": -0.01283574104309082,
"fcm_dpo/margin": 69.22969055175781,
"fcm_dpo/q_t": 0.40422728657722473,
"grad_norm": 10.43131160736084,
"learning_rate": 1.4374663593999256e-07,
"logits/chosen": 0.7159805297851562,
"logits/rejected": 0.6615912318229675,
"logps/chosen": -180.61326599121094,
"logps/ref_chosen": -53.75125503540039,
"logps/ref_rejected": -77.17623901367188,
"logps/rejected": -273.2679443359375,
"loss": 1.0852,
"margin_dpo/margin_mean": 69.22969055175781,
"margin_dpo/margin_std": 91.06256103515625,
"step": 448
},
{
"KL/chosen_KL_mean": -149.3343963623047,
"KL/mean": -166.57122802734375,
"KL/rejected_KL_mean": -183.80804443359375,
"KL/std": 86.64166259765625,
"epoch": 0.6787603930461074,
"fcm_dpo/beta": 0.006043557543307543,
"fcm_dpo/delta": 0.07610173523426056,
"fcm_dpo/margin": 34.47367858886719,
"fcm_dpo/q_t": 0.4520561993122101,
"grad_norm": 18.176597595214844,
"learning_rate": 1.4255127197770707e-07,
"logits/chosen": 0.5756776332855225,
"logits/rejected": 0.5750092267990112,
"logps/chosen": -225.16177368164062,
"logps/ref_chosen": -75.82737731933594,
"logps/ref_rejected": -82.20687866210938,
"logps/rejected": -266.0149230957031,
"loss": 1.2521,
"margin_dpo/margin_mean": 34.47367858886719,
"margin_dpo/margin_std": 84.48080444335938,
"step": 449
},
{
"KL/chosen_KL_mean": -126.5445556640625,
"KL/mean": -153.6673583984375,
"KL/rejected_KL_mean": -180.79017639160156,
"KL/std": 87.68942260742188,
"epoch": 0.6802721088435374,
"fcm_dpo/beta": 0.0060948459431529045,
"fcm_dpo/delta": 0.07181155681610107,
"fcm_dpo/margin": 54.24563217163086,
"fcm_dpo/q_t": 0.42675548791885376,
"grad_norm": 12.587833404541016,
"learning_rate": 1.4135891358732205e-07,
"logits/chosen": 0.7975543141365051,
"logits/rejected": 0.6809457540512085,
"logps/chosen": -173.6602783203125,
"logps/ref_chosen": -47.11572265625,
"logps/ref_rejected": -78.7546615600586,
"logps/rejected": -259.5448303222656,
"loss": 1.1715,
"margin_dpo/margin_mean": 54.245628356933594,
"margin_dpo/margin_std": 100.49533081054688,
"step": 450
},
{
"KL/chosen_KL_mean": -127.44542694091797,
"KL/mean": -152.0277099609375,
"KL/rejected_KL_mean": -176.60995483398438,
"KL/std": 84.69337463378906,
"epoch": 0.6817838246409675,
"fcm_dpo/beta": 0.006232240237295628,
"fcm_dpo/delta": 0.09627757966518402,
"fcm_dpo/margin": 49.16454315185547,
"fcm_dpo/q_t": 0.4297522306442261,
"grad_norm": 12.449745178222656,
"learning_rate": 1.4016959412166437e-07,
"logits/chosen": 0.6532795429229736,
"logits/rejected": 0.6001813411712646,
"logps/chosen": -190.79586791992188,
"logps/ref_chosen": -63.350440979003906,
"logps/ref_rejected": -76.28530883789062,
"logps/rejected": -252.895263671875,
"loss": 1.18,
"margin_dpo/margin_mean": 49.16454315185547,
"margin_dpo/margin_std": 91.25083923339844,
"step": 451
},
{
"KL/chosen_KL_mean": -124.88740539550781,
"KL/mean": -153.61331176757812,
"KL/rejected_KL_mean": -182.33920288085938,
"KL/std": 80.17495727539062,
"epoch": 0.6832955404383976,
"fcm_dpo/beta": 0.006293575279414654,
"fcm_dpo/delta": 0.03980523347854614,
"fcm_dpo/margin": 57.451805114746094,
"fcm_dpo/q_t": 0.41748127341270447,
"grad_norm": 14.384387969970703,
"learning_rate": 1.3898334684855645e-07,
"logits/chosen": 0.6469016075134277,
"logits/rejected": 0.5629381537437439,
"logps/chosen": -180.47323608398438,
"logps/ref_chosen": -55.58583450317383,
"logps/ref_rejected": -77.68738555908203,
"logps/rejected": -260.0265808105469,
"loss": 1.1518,
"margin_dpo/margin_mean": 57.451805114746094,
"margin_dpo/margin_std": 100.0445785522461,
"step": 452
},
{
"KL/chosen_KL_mean": -122.88931274414062,
"KL/mean": -152.79161071777344,
"KL/rejected_KL_mean": -182.69390869140625,
"KL/std": 83.66735076904297,
"epoch": 0.6848072562358276,
"fcm_dpo/beta": 0.006314560305327177,
"fcm_dpo/delta": 0.023236922919750214,
"fcm_dpo/margin": 59.804588317871094,
"fcm_dpo/q_t": 0.4145626425743103,
"grad_norm": 14.594283103942871,
"learning_rate": 1.3780020494988445e-07,
"logits/chosen": 0.6820989847183228,
"logits/rejected": 0.653471052646637,
"logps/chosen": -184.66751098632812,
"logps/ref_chosen": -61.778202056884766,
"logps/ref_rejected": -71.51403045654297,
"logps/rejected": -254.2079315185547,
"loss": 1.1333,
"margin_dpo/margin_mean": 59.804588317871094,
"margin_dpo/margin_std": 97.10567474365234,
"step": 453
},
{
"KL/chosen_KL_mean": -114.92044830322266,
"KL/mean": -149.12408447265625,
"KL/rejected_KL_mean": -183.3277130126953,
"KL/std": 85.0125732421875,
"epoch": 0.6863189720332578,
"fcm_dpo/beta": 0.0062905652448534966,
"fcm_dpo/delta": -0.031771667301654816,
"fcm_dpo/margin": 68.40725708007812,
"fcm_dpo/q_t": 0.4002048969268799,
"grad_norm": 12.348052024841309,
"learning_rate": 1.366202015206706e-07,
"logits/chosen": 0.6867334246635437,
"logits/rejected": 0.6467639207839966,
"logps/chosen": -166.51559448242188,
"logps/ref_chosen": -51.59515380859375,
"logps/ref_rejected": -63.96732711791992,
"logps/rejected": -247.2950439453125,
"loss": 1.0833,
"margin_dpo/margin_mean": 68.40726470947266,
"margin_dpo/margin_std": 94.05560302734375,
"step": 454
},
{
"KL/chosen_KL_mean": -133.7144775390625,
"KL/mean": -165.7753448486328,
"KL/rejected_KL_mean": -197.8362274169922,
"KL/std": 84.17610168457031,
"epoch": 0.6878306878306878,
"fcm_dpo/beta": 0.006278153508901596,
"fcm_dpo/delta": -0.0027168411761522293,
"fcm_dpo/margin": 64.12176513671875,
"fcm_dpo/q_t": 0.40871015191078186,
"grad_norm": 12.936040878295898,
"learning_rate": 1.354433695681474e-07,
"logits/chosen": 0.5652279853820801,
"logits/rejected": 0.5327200293540955,
"logps/chosen": -204.36618041992188,
"logps/ref_chosen": -70.65170288085938,
"logps/ref_rejected": -77.44276428222656,
"logps/rejected": -275.27899169921875,
"loss": 1.1006,
"margin_dpo/margin_mean": 64.12176513671875,
"margin_dpo/margin_std": 91.57505798339844,
"step": 455
},
{
"KL/chosen_KL_mean": -132.67092895507812,
"KL/mean": -161.06515502929688,
"KL/rejected_KL_mean": -189.4593505859375,
"KL/std": 83.19470977783203,
"epoch": 0.6893424036281179,
"fcm_dpo/beta": 0.0063173118978738785,
"fcm_dpo/delta": 0.042804621160030365,
"fcm_dpo/margin": 56.78840637207031,
"fcm_dpo/q_t": 0.418972909450531,
"grad_norm": 15.584725379943848,
"learning_rate": 1.3426974201083439e-07,
"logits/chosen": 0.654152512550354,
"logits/rejected": 0.5844058990478516,
"logps/chosen": -189.0692138671875,
"logps/ref_chosen": -56.398284912109375,
"logps/ref_rejected": -82.61642456054688,
"logps/rejected": -272.0757751464844,
"loss": 1.145,
"margin_dpo/margin_mean": 56.78840637207031,
"margin_dpo/margin_std": 95.3462905883789,
"step": 456
},
{
"KL/chosen_KL_mean": -130.41671752929688,
"KL/mean": -162.0543670654297,
"KL/rejected_KL_mean": -193.69203186035156,
"KL/std": 85.54029083251953,
"epoch": 0.690854119425548,
"fcm_dpo/beta": 0.006323341280221939,
"fcm_dpo/delta": -0.00025469623506069183,
"fcm_dpo/margin": 63.27531051635742,
"fcm_dpo/q_t": 0.40713024139404297,
"grad_norm": 12.248613357543945,
"learning_rate": 1.3309935167761717e-07,
"logits/chosen": 0.8168525099754333,
"logits/rejected": 0.7352825403213501,
"logps/chosen": -175.13729858398438,
"logps/ref_chosen": -44.72057342529297,
"logps/ref_rejected": -68.1158676147461,
"logps/rejected": -261.8078918457031,
"loss": 1.0914,
"margin_dpo/margin_mean": 63.275306701660156,
"margin_dpo/margin_std": 82.94820404052734,
"step": 457
},
{
"KL/chosen_KL_mean": -125.96719360351562,
"KL/mean": -158.32989501953125,
"KL/rejected_KL_mean": -190.6925811767578,
"KL/std": 88.40022277832031,
"epoch": 0.6923658352229781,
"fcm_dpo/beta": 0.006333203986287117,
"fcm_dpo/delta": -0.010343178175389767,
"fcm_dpo/margin": 64.72538757324219,
"fcm_dpo/q_t": 0.4064847230911255,
"grad_norm": 13.382430076599121,
"learning_rate": 1.3193223130682936e-07,
"logits/chosen": 0.7040465474128723,
"logits/rejected": 0.582461953163147,
"logps/chosen": -175.97288513183594,
"logps/ref_chosen": -50.00569152832031,
"logps/ref_rejected": -87.50015258789062,
"logps/rejected": -278.1927490234375,
"loss": 1.1108,
"margin_dpo/margin_mean": 64.72539520263672,
"margin_dpo/margin_std": 99.74910736083984,
"step": 458
},
{
"KL/chosen_KL_mean": -118.89695739746094,
"KL/mean": -159.2664337158203,
"KL/rejected_KL_mean": -199.6359100341797,
"KL/std": 96.21126556396484,
"epoch": 0.6938775510204082,
"fcm_dpo/beta": 0.006271988153457642,
"fcm_dpo/delta": -0.11247433722019196,
"fcm_dpo/margin": 80.73894500732422,
"fcm_dpo/q_t": 0.3832815885543823,
"grad_norm": 11.640632629394531,
"learning_rate": 1.3076841354533658e-07,
"logits/chosen": 0.7114887237548828,
"logits/rejected": 0.6766858100891113,
"logps/chosen": -184.27490234375,
"logps/ref_chosen": -65.37794494628906,
"logps/ref_rejected": -88.19244384765625,
"logps/rejected": -287.828369140625,
"loss": 1.0288,
"margin_dpo/margin_mean": 80.73894500732422,
"margin_dpo/margin_std": 93.04231262207031,
"step": 459
},
{
"KL/chosen_KL_mean": -132.77362060546875,
"KL/mean": -171.4193878173828,
"KL/rejected_KL_mean": -210.06515502929688,
"KL/std": 87.51814270019531,
"epoch": 0.6953892668178382,
"fcm_dpo/beta": 0.0060975514352321625,
"fcm_dpo/delta": -0.07526206970214844,
"fcm_dpo/margin": 77.29153442382812,
"fcm_dpo/q_t": 0.3930175304412842,
"grad_norm": 12.21908187866211,
"learning_rate": 1.2960793094762345e-07,
"logits/chosen": 0.7287610173225403,
"logits/rejected": 0.6012281179428101,
"logps/chosen": -197.33529663085938,
"logps/ref_chosen": -64.5616683959961,
"logps/ref_rejected": -88.67890167236328,
"logps/rejected": -298.7440490722656,
"loss": 1.0488,
"margin_dpo/margin_mean": 77.29153442382812,
"margin_dpo/margin_std": 95.84016418457031,
"step": 460
},
{
"KL/chosen_KL_mean": -109.56300354003906,
"KL/mean": -149.3557586669922,
"KL/rejected_KL_mean": -189.14852905273438,
"KL/std": 81.28533172607422,
"epoch": 0.6969009826152683,
"fcm_dpo/beta": 0.005965542048215866,
"fcm_dpo/delta": -0.08096842467784882,
"fcm_dpo/margin": 79.58551025390625,
"fcm_dpo/q_t": 0.39024484157562256,
"grad_norm": 13.1436185836792,
"learning_rate": 1.2845081597488286e-07,
"logits/chosen": 0.8475281000137329,
"logits/rejected": 0.7559252381324768,
"logps/chosen": -159.04092407226562,
"logps/ref_chosen": -49.4779167175293,
"logps/ref_rejected": -72.65262603759766,
"logps/rejected": -261.8011474609375,
"loss": 1.0434,
"margin_dpo/margin_mean": 79.58551025390625,
"margin_dpo/margin_std": 90.81407165527344,
"step": 461
},
{
"KL/chosen_KL_mean": -121.68338775634766,
"KL/mean": -162.32908630371094,
"KL/rejected_KL_mean": -202.97479248046875,
"KL/std": 83.77405548095703,
"epoch": 0.6984126984126984,
"fcm_dpo/beta": 0.0059011634439229965,
"fcm_dpo/delta": -0.0842406153678894,
"fcm_dpo/margin": 81.2914047241211,
"fcm_dpo/q_t": 0.38803941011428833,
"grad_norm": 12.075309753417969,
"learning_rate": 1.27297100994108e-07,
"logits/chosen": 0.6871299743652344,
"logits/rejected": 0.6317715644836426,
"logps/chosen": -182.17849731445312,
"logps/ref_chosen": -60.4951171875,
"logps/ref_rejected": -74.82136535644531,
"logps/rejected": -277.796142578125,
"loss": 1.0351,
"margin_dpo/margin_mean": 81.2914047241211,
"margin_dpo/margin_std": 93.20660400390625,
"step": 462
},
{
"KL/chosen_KL_mean": -144.84066772460938,
"KL/mean": -170.0337371826172,
"KL/rejected_KL_mean": -195.226806640625,
"KL/std": 81.04251098632812,
"epoch": 0.6999244142101285,
"fcm_dpo/beta": 0.005902908742427826,
"fcm_dpo/delta": 0.006357495207339525,
"fcm_dpo/margin": 50.386138916015625,
"fcm_dpo/q_t": 0.4303584694862366,
"grad_norm": 16.40827751159668,
"learning_rate": 1.2614681827718695e-07,
"logits/chosen": 0.6820461750030518,
"logits/rejected": 0.6828656196594238,
"logps/chosen": -212.52578735351562,
"logps/ref_chosen": -67.68511962890625,
"logps/ref_rejected": -71.32196044921875,
"logps/rejected": -266.54876708984375,
"loss": 1.1765,
"margin_dpo/margin_mean": 50.386138916015625,
"margin_dpo/margin_std": 86.95796966552734,
"step": 463
},
{
"KL/chosen_KL_mean": -132.83026123046875,
"KL/mean": -169.96087646484375,
"KL/rejected_KL_mean": -207.0915069580078,
"KL/std": 88.52447509765625,
"epoch": 0.7014361300075586,
"fcm_dpo/beta": 0.005890951491892338,
"fcm_dpo/delta": -0.03933081775903702,
"fcm_dpo/margin": 74.26124572753906,
"fcm_dpo/q_t": 0.39939481019973755,
"grad_norm": 11.131133079528809,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 0.6883972883224487,
"logits/rejected": 0.6589312553405762,
"logps/chosen": -191.99591064453125,
"logps/ref_chosen": -59.16564178466797,
"logps/ref_rejected": -69.56146240234375,
"logps/rejected": -276.6529541015625,
"loss": 1.0856,
"margin_dpo/margin_mean": 74.26124572753906,
"margin_dpo/margin_std": 102.26775360107422,
"step": 464
},
{
"KL/chosen_KL_mean": -139.8975830078125,
"KL/mean": -171.8345489501953,
"KL/rejected_KL_mean": -203.77151489257812,
"KL/std": 86.05863189697266,
"epoch": 0.7029478458049887,
"fcm_dpo/beta": 0.005867544561624527,
"fcm_dpo/delta": 0.02616678923368454,
"fcm_dpo/margin": 63.87392807006836,
"fcm_dpo/q_t": 0.4142424464225769,
"grad_norm": 13.73096752166748,
"learning_rate": 1.238566782415197e-07,
"logits/chosen": 0.748282790184021,
"logits/rejected": 0.6854946613311768,
"logps/chosen": -198.4112548828125,
"logps/ref_chosen": -58.513671875,
"logps/ref_rejected": -84.31745910644531,
"logps/rejected": -288.0889892578125,
"loss": 1.1276,
"margin_dpo/margin_mean": 63.873931884765625,
"margin_dpo/margin_std": 99.58221435546875,
"step": 465
},
{
"KL/chosen_KL_mean": -151.56088256835938,
"KL/mean": -170.55224609375,
"KL/rejected_KL_mean": -189.5436248779297,
"KL/std": 91.8314208984375,
"epoch": 0.7044595616024187,
"fcm_dpo/beta": 0.005962677299976349,
"fcm_dpo/delta": 0.06353595107793808,
"fcm_dpo/margin": 37.982765197753906,
"fcm_dpo/q_t": 0.4477654695510864,
"grad_norm": 19.188405990600586,
"learning_rate": 1.2271688498291334e-07,
"logits/chosen": 0.7177670001983643,
"logits/rejected": 0.7247218489646912,
"logps/chosen": -224.82669067382812,
"logps/ref_chosen": -73.26580810546875,
"logps/ref_rejected": -74.83621215820312,
"logps/rejected": -264.3798522949219,
"loss": 1.2429,
"margin_dpo/margin_mean": 37.982765197753906,
"margin_dpo/margin_std": 90.40339660644531,
"step": 466
},
{
"KL/chosen_KL_mean": -135.3428497314453,
"KL/mean": -163.8096923828125,
"KL/rejected_KL_mean": -192.2765655517578,
"KL/std": 88.4688720703125,
"epoch": 0.7059712773998488,
"fcm_dpo/beta": 0.006015198305249214,
"fcm_dpo/delta": 0.05957948789000511,
"fcm_dpo/margin": 56.93370819091797,
"fcm_dpo/q_t": 0.421988308429718,
"grad_norm": 12.390352249145508,
"learning_rate": 1.2158065210664848e-07,
"logits/chosen": 0.777511715888977,
"logits/rejected": 0.6185018420219421,
"logps/chosen": -182.92233276367188,
"logps/ref_chosen": -47.57947540283203,
"logps/ref_rejected": -78.68522644042969,
"logps/rejected": -270.9617919921875,
"loss": 1.1391,
"margin_dpo/margin_mean": 56.93370819091797,
"margin_dpo/margin_std": 88.92526245117188,
"step": 467
},
{
"KL/chosen_KL_mean": -127.57308197021484,
"KL/mean": -166.72140502929688,
"KL/rejected_KL_mean": -205.86972045898438,
"KL/std": 87.78189849853516,
"epoch": 0.7074829931972789,
"fcm_dpo/beta": 0.00597839942201972,
"fcm_dpo/delta": -0.07137361168861389,
"fcm_dpo/margin": 78.29663848876953,
"fcm_dpo/q_t": 0.39162519574165344,
"grad_norm": 15.560737609863281,
"learning_rate": 1.204480113956011e-07,
"logits/chosen": 0.6573153734207153,
"logits/rejected": 0.6468690633773804,
"logps/chosen": -191.50086975097656,
"logps/ref_chosen": -63.92778778076172,
"logps/ref_rejected": -76.51626586914062,
"logps/rejected": -282.385986328125,
"loss": 1.0564,
"margin_dpo/margin_mean": 78.29663848876953,
"margin_dpo/margin_std": 99.99575805664062,
"step": 468
},
{
"KL/chosen_KL_mean": -129.8638916015625,
"KL/mean": -166.43460083007812,
"KL/rejected_KL_mean": -203.0052947998047,
"KL/std": 86.45907592773438,
"epoch": 0.708994708994709,
"fcm_dpo/beta": 0.005886958912014961,
"fcm_dpo/delta": -0.032767973840236664,
"fcm_dpo/margin": 73.14139556884766,
"fcm_dpo/q_t": 0.39935657382011414,
"grad_norm": 12.466798782348633,
"learning_rate": 1.1931899453216697e-07,
"logits/chosen": 0.7542613744735718,
"logits/rejected": 0.7423312664031982,
"logps/chosen": -188.92208862304688,
"logps/ref_chosen": -59.05818176269531,
"logps/ref_rejected": -75.67672729492188,
"logps/rejected": -278.6820068359375,
"loss": 1.0575,
"margin_dpo/margin_mean": 73.14139556884766,
"margin_dpo/margin_std": 80.6260757446289,
"step": 469
},
{
"KL/chosen_KL_mean": -125.3199462890625,
"KL/mean": -159.62376403808594,
"KL/rejected_KL_mean": -193.92758178710938,
"KL/std": 85.09135437011719,
"epoch": 0.7105064247921391,
"fcm_dpo/beta": 0.005920417606830597,
"fcm_dpo/delta": -0.006586667150259018,
"fcm_dpo/margin": 68.60760498046875,
"fcm_dpo/q_t": 0.40617385506629944,
"grad_norm": 12.12977123260498,
"learning_rate": 1.1819363309737438e-07,
"logits/chosen": 0.7238911390304565,
"logits/rejected": 0.651435136795044,
"logps/chosen": -173.18739318847656,
"logps/ref_chosen": -47.86743927001953,
"logps/ref_rejected": -65.96859741210938,
"logps/rejected": -259.89617919921875,
"loss": 1.0964,
"margin_dpo/margin_mean": 68.60760498046875,
"margin_dpo/margin_std": 94.92890930175781,
"step": 470
},
{
"KL/chosen_KL_mean": -120.63668823242188,
"KL/mean": -159.82736206054688,
"KL/rejected_KL_mean": -199.01805114746094,
"KL/std": 85.26072692871094,
"epoch": 0.7120181405895691,
"fcm_dpo/beta": 0.005838717333972454,
"fcm_dpo/delta": -0.06058187037706375,
"fcm_dpo/margin": 78.38137817382812,
"fcm_dpo/q_t": 0.39347031712532043,
"grad_norm": 11.796302795410156,
"learning_rate": 1.1707195857000215e-07,
"logits/chosen": 0.7129791975021362,
"logits/rejected": 0.6567329168319702,
"logps/chosen": -178.41453552246094,
"logps/ref_chosen": -57.777854919433594,
"logps/ref_rejected": -73.81172180175781,
"logps/rejected": -272.82977294921875,
"loss": 1.0528,
"margin_dpo/margin_mean": 78.3813705444336,
"margin_dpo/margin_std": 94.56427001953125,
"step": 471
},
{
"KL/chosen_KL_mean": -122.81675720214844,
"KL/mean": -153.1514892578125,
"KL/rejected_KL_mean": -183.48622131347656,
"KL/std": 86.51441192626953,
"epoch": 0.7135298563869993,
"fcm_dpo/beta": 0.005888373125344515,
"fcm_dpo/delta": 0.04398176074028015,
"fcm_dpo/margin": 60.669456481933594,
"fcm_dpo/q_t": 0.4189244508743286,
"grad_norm": 13.525787353515625,
"learning_rate": 1.1595400232569768e-07,
"logits/chosen": 0.7163376808166504,
"logits/rejected": 0.6692396402359009,
"logps/chosen": -178.72543334960938,
"logps/ref_chosen": -55.908668518066406,
"logps/ref_rejected": -74.70294189453125,
"logps/rejected": -258.18914794921875,
"loss": 1.1594,
"margin_dpo/margin_mean": 60.669456481933594,
"margin_dpo/margin_std": 109.81246948242188,
"step": 472
},
{
"KL/chosen_KL_mean": -125.64335632324219,
"KL/mean": -158.94390869140625,
"KL/rejected_KL_mean": -192.24447631835938,
"KL/std": 93.05805206298828,
"epoch": 0.7150415721844293,
"fcm_dpo/beta": 0.005887615494430065,
"fcm_dpo/delta": 0.008200233802199364,
"fcm_dpo/margin": 66.60111236572266,
"fcm_dpo/q_t": 0.4120209813117981,
"grad_norm": 13.417925834655762,
"learning_rate": 1.1483979563610069e-07,
"logits/chosen": 0.7729692459106445,
"logits/rejected": 0.6557892560958862,
"logps/chosen": -179.80422973632812,
"logps/ref_chosen": -54.16088104248047,
"logps/ref_rejected": -92.76789855957031,
"logps/rejected": -285.01239013671875,
"loss": 1.1375,
"margin_dpo/margin_mean": 66.60111999511719,
"margin_dpo/margin_std": 113.83407592773438,
"step": 473
},
{
"KL/chosen_KL_mean": -128.4207763671875,
"KL/mean": -159.42266845703125,
"KL/rejected_KL_mean": -190.424560546875,
"KL/std": 88.38874816894531,
"epoch": 0.7165532879818595,
"fcm_dpo/beta": 0.005925321020185947,
"fcm_dpo/delta": 0.03382481262087822,
"fcm_dpo/margin": 62.00376892089844,
"fcm_dpo/q_t": 0.4172155261039734,
"grad_norm": 16.298526763916016,
"learning_rate": 1.1372936966796709e-07,
"logits/chosen": 0.8211394548416138,
"logits/rejected": 0.7391525506973267,
"logps/chosen": -175.1064910888672,
"logps/ref_chosen": -46.685707092285156,
"logps/ref_rejected": -71.44731903076172,
"logps/rejected": -261.87188720703125,
"loss": 1.1482,
"margin_dpo/margin_mean": 62.00376892089844,
"margin_dpo/margin_std": 107.985595703125,
"step": 474
},
{
"KL/chosen_KL_mean": -122.31303405761719,
"KL/mean": -166.75416564941406,
"KL/rejected_KL_mean": -211.19528198242188,
"KL/std": 90.2840576171875,
"epoch": 0.7180650037792895,
"fcm_dpo/beta": 0.005802489351481199,
"fcm_dpo/delta": -0.12280426919460297,
"fcm_dpo/margin": 88.88225555419922,
"fcm_dpo/q_t": 0.38198548555374146,
"grad_norm": 10.500279426574707,
"learning_rate": 1.126227554822985e-07,
"logits/chosen": 0.707655668258667,
"logits/rejected": 0.6558683514595032,
"logps/chosen": -180.8003387451172,
"logps/ref_chosen": -58.4873046875,
"logps/ref_rejected": -87.00187683105469,
"logps/rejected": -298.1971740722656,
"loss": 1.0124,
"margin_dpo/margin_mean": 88.88224792480469,
"margin_dpo/margin_std": 98.73614501953125,
"step": 475
},
{
"KL/chosen_KL_mean": -147.6066131591797,
"KL/mean": -177.59841918945312,
"KL/rejected_KL_mean": -207.5902099609375,
"KL/std": 92.01683044433594,
"epoch": 0.7195767195767195,
"fcm_dpo/beta": 0.005856232717633247,
"fcm_dpo/delta": 0.049916207790374756,
"fcm_dpo/margin": 59.98360824584961,
"fcm_dpo/q_t": 0.4199449121952057,
"grad_norm": 12.924736976623535,
"learning_rate": 1.1151998403347243e-07,
"logits/chosen": 0.6142306327819824,
"logits/rejected": 0.6142148375511169,
"logps/chosen": -222.98825073242188,
"logps/ref_chosen": -75.38162231445312,
"logps/ref_rejected": -76.99822235107422,
"logps/rejected": -284.58843994140625,
"loss": 1.1471,
"margin_dpo/margin_mean": 59.983612060546875,
"margin_dpo/margin_std": 99.31526947021484,
"step": 476
},
{
"KL/chosen_KL_mean": -151.8356475830078,
"KL/mean": -181.24183654785156,
"KL/rejected_KL_mean": -210.6480255126953,
"KL/std": 91.17938232421875,
"epoch": 0.7210884353741497,
"fcm_dpo/beta": 0.005891036242246628,
"fcm_dpo/delta": 0.055482812225818634,
"fcm_dpo/margin": 58.812374114990234,
"fcm_dpo/q_t": 0.42130500078201294,
"grad_norm": 15.10306453704834,
"learning_rate": 1.1042108616837692e-07,
"logits/chosen": 0.7368456125259399,
"logits/rejected": 0.6839989423751831,
"logps/chosen": -212.90904235839844,
"logps/ref_chosen": -61.073387145996094,
"logps/ref_rejected": -81.34375,
"logps/rejected": -291.99176025390625,
"loss": 1.1882,
"margin_dpo/margin_mean": 58.812374114990234,
"margin_dpo/margin_std": 120.16973876953125,
"step": 477
},
{
"KL/chosen_KL_mean": -133.17063903808594,
"KL/mean": -158.50933837890625,
"KL/rejected_KL_mean": -183.84805297851562,
"KL/std": 84.49418640136719,
"epoch": 0.7226001511715797,
"fcm_dpo/beta": 0.005983785260468721,
"fcm_dpo/delta": 0.09990386664867401,
"fcm_dpo/margin": 50.677398681640625,
"fcm_dpo/q_t": 0.43077266216278076,
"grad_norm": 15.52071475982666,
"learning_rate": 1.0932609262554746e-07,
"logits/chosen": 0.6610161662101746,
"logits/rejected": 0.6723431348800659,
"logps/chosen": -190.33795166015625,
"logps/ref_chosen": -57.16731643676758,
"logps/ref_rejected": -53.30917739868164,
"logps/rejected": -237.1572265625,
"loss": 1.1974,
"margin_dpo/margin_mean": 50.677398681640625,
"margin_dpo/margin_std": 102.94509887695312,
"step": 478
},
{
"KL/chosen_KL_mean": -138.07371520996094,
"KL/mean": -161.25302124023438,
"KL/rejected_KL_mean": -184.43234252929688,
"KL/std": 82.27931213378906,
"epoch": 0.7241118669690099,
"fcm_dpo/beta": 0.006053300108760595,
"fcm_dpo/delta": 0.018644915893673897,
"fcm_dpo/margin": 46.35863494873047,
"fcm_dpo/q_t": 0.4353847801685333,
"grad_norm": 14.631587028503418,
"learning_rate": 1.0823503403430734e-07,
"logits/chosen": 0.6806881427764893,
"logits/rejected": 0.6335880756378174,
"logps/chosen": -196.98703002929688,
"logps/ref_chosen": -58.91331481933594,
"logps/ref_rejected": -63.7403450012207,
"logps/rejected": -248.17269897460938,
"loss": 1.2089,
"margin_dpo/margin_mean": 46.35863494873047,
"margin_dpo/margin_std": 97.01359558105469,
"step": 479
},
{
"KL/chosen_KL_mean": -140.8656768798828,
"KL/mean": -176.2801513671875,
"KL/rejected_KL_mean": -211.69459533691406,
"KL/std": 86.09163665771484,
"epoch": 0.7256235827664399,
"fcm_dpo/beta": 0.006067180074751377,
"fcm_dpo/delta": -0.03184448555111885,
"fcm_dpo/margin": 70.82891082763672,
"fcm_dpo/q_t": 0.4013393521308899,
"grad_norm": 14.73949909210205,
"learning_rate": 1.0714794091391072e-07,
"logits/chosen": 0.6912207007408142,
"logits/rejected": 0.6811619997024536,
"logps/chosen": -203.66629028320312,
"logps/ref_chosen": -62.80061340332031,
"logps/ref_rejected": -67.58859252929688,
"logps/rejected": -279.283203125,
"loss": 1.093,
"margin_dpo/margin_mean": 70.82891082763672,
"margin_dpo/margin_std": 99.73735046386719,
"step": 480
},
{
"KL/chosen_KL_mean": -137.70993041992188,
"KL/mean": -168.67620849609375,
"KL/rejected_KL_mean": -199.64247131347656,
"KL/std": 87.77169036865234,
"epoch": 0.72713529856387,
"fcm_dpo/beta": 0.0060311416164040565,
"fcm_dpo/delta": 0.02750197984278202,
"fcm_dpo/margin": 61.93252944946289,
"fcm_dpo/q_t": 0.4151855707168579,
"grad_norm": 14.199797630310059,
"learning_rate": 1.0606484367268906e-07,
"logits/chosen": 0.6721217036247253,
"logits/rejected": 0.670637845993042,
"logps/chosen": -202.99642944335938,
"logps/ref_chosen": -65.28649139404297,
"logps/ref_rejected": -70.78668212890625,
"logps/rejected": -270.42913818359375,
"loss": 1.1356,
"margin_dpo/margin_mean": 61.932533264160156,
"margin_dpo/margin_std": 101.92668151855469,
"step": 481
},
{
"KL/chosen_KL_mean": -154.27391052246094,
"KL/mean": -185.8995361328125,
"KL/rejected_KL_mean": -217.525146484375,
"KL/std": 90.60935974121094,
"epoch": 0.7286470143613001,
"fcm_dpo/beta": 0.0060750562697649,
"fcm_dpo/delta": 0.016256995499134064,
"fcm_dpo/margin": 63.251243591308594,
"fcm_dpo/q_t": 0.41464337706565857,
"grad_norm": 13.977091789245605,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 0.6302579641342163,
"logits/rejected": 0.475580632686615,
"logps/chosen": -215.18008422851562,
"logps/ref_chosen": -60.906185150146484,
"logps/ref_rejected": -103.44656372070312,
"logps/rejected": -320.9717102050781,
"loss": 1.1538,
"margin_dpo/margin_mean": 63.25123977661133,
"margin_dpo/margin_std": 115.94500732421875,
"step": 482
},
{
"KL/chosen_KL_mean": -131.58995056152344,
"KL/mean": -169.9619598388672,
"KL/rejected_KL_mean": -208.33396911621094,
"KL/std": 85.75035095214844,
"epoch": 0.7301587301587301,
"fcm_dpo/beta": 0.006018957123160362,
"fcm_dpo/delta": -0.06486622989177704,
"fcm_dpo/margin": 76.7440185546875,
"fcm_dpo/q_t": 0.3938300609588623,
"grad_norm": 12.383716583251953,
"learning_rate": 1.0391075790138232e-07,
"logits/chosen": 0.7711484432220459,
"logits/rejected": 0.6594574451446533,
"logps/chosen": -184.78196716308594,
"logps/ref_chosen": -53.192012786865234,
"logps/ref_rejected": -81.83927154541016,
"logps/rejected": -290.1732177734375,
"loss": 1.0571,
"margin_dpo/margin_mean": 76.7440185546875,
"margin_dpo/margin_std": 97.3631591796875,
"step": 483
},
{
"KL/chosen_KL_mean": -135.44894409179688,
"KL/mean": -161.29734802246094,
"KL/rejected_KL_mean": -187.14573669433594,
"KL/std": 82.80963134765625,
"epoch": 0.7316704459561603,
"fcm_dpo/beta": 0.006095028482377529,
"fcm_dpo/delta": 0.08686641603708267,
"fcm_dpo/margin": 51.696807861328125,
"fcm_dpo/q_t": 0.4264023005962372,
"grad_norm": 17.69460678100586,
"learning_rate": 1.0283982962570681e-07,
"logits/chosen": 0.7964112758636475,
"logits/rejected": 0.7605965733528137,
"logps/chosen": -193.21841430664062,
"logps/ref_chosen": -57.76945877075195,
"logps/ref_rejected": -71.6829833984375,
"logps/rejected": -258.8287353515625,
"loss": 1.1474,
"margin_dpo/margin_mean": 51.69680404663086,
"margin_dpo/margin_std": 74.98580169677734,
"step": 484
},
{
"KL/chosen_KL_mean": -139.70346069335938,
"KL/mean": -168.07046508789062,
"KL/rejected_KL_mean": -196.43743896484375,
"KL/std": 87.52056884765625,
"epoch": 0.7331821617535903,
"fcm_dpo/beta": 0.006033752579241991,
"fcm_dpo/delta": -0.05008743703365326,
"fcm_dpo/margin": 56.733978271484375,
"fcm_dpo/q_t": 0.42060738801956177,
"grad_norm": 14.04023265838623,
"learning_rate": 1.0177301773633992e-07,
"logits/chosen": 0.7477602958679199,
"logits/rejected": 0.7239288091659546,
"logps/chosen": -196.33932495117188,
"logps/ref_chosen": -56.63584899902344,
"logps/ref_rejected": -70.85614013671875,
"logps/rejected": -267.2935791015625,
"loss": 1.14,
"margin_dpo/margin_mean": 56.733978271484375,
"margin_dpo/margin_std": 84.94133758544922,
"step": 485
},
{
"KL/chosen_KL_mean": -159.4882354736328,
"KL/mean": -185.74334716796875,
"KL/rejected_KL_mean": -211.99847412109375,
"KL/std": 95.40143585205078,
"epoch": 0.7346938775510204,
"fcm_dpo/beta": 0.006106095388531685,
"fcm_dpo/delta": 0.08202138543128967,
"fcm_dpo/margin": 52.51021957397461,
"fcm_dpo/q_t": 0.42895442247390747,
"grad_norm": 12.479859352111816,
"learning_rate": 1.007103520743035e-07,
"logits/chosen": 0.738491415977478,
"logits/rejected": 0.6127005219459534,
"logps/chosen": -215.83526611328125,
"logps/ref_chosen": -56.347023010253906,
"logps/ref_rejected": -85.97221374511719,
"logps/rejected": -297.9706726074219,
"loss": 1.1945,
"margin_dpo/margin_mean": 52.510215759277344,
"margin_dpo/margin_std": 109.29815673828125,
"step": 486
},
{
"KL/chosen_KL_mean": -143.13406372070312,
"KL/mean": -175.829833984375,
"KL/rejected_KL_mean": -208.525634765625,
"KL/std": 86.87509155273438,
"epoch": 0.7362055933484505,
"fcm_dpo/beta": 0.006127578672021627,
"fcm_dpo/delta": -0.0007367376238107681,
"fcm_dpo/margin": 65.39156341552734,
"fcm_dpo/q_t": 0.41006791591644287,
"grad_norm": 15.440977096557617,
"learning_rate": 9.965186236464046e-08,
"logits/chosen": 0.8319680690765381,
"logits/rejected": 0.7668202519416809,
"logps/chosen": -203.75128173828125,
"logps/ref_chosen": -60.617218017578125,
"logps/ref_rejected": -82.50975036621094,
"logps/rejected": -291.0353698730469,
"loss": 1.1115,
"margin_dpo/margin_mean": 65.39156341552734,
"margin_dpo/margin_std": 100.54965209960938,
"step": 487
},
{
"KL/chosen_KL_mean": -135.84095764160156,
"KL/mean": -170.20909118652344,
"KL/rejected_KL_mean": -204.57723999023438,
"KL/std": 85.58430480957031,
"epoch": 0.7377173091458806,
"fcm_dpo/beta": 0.006132540758699179,
"fcm_dpo/delta": -0.02262810245156288,
"fcm_dpo/margin": 68.73626708984375,
"fcm_dpo/q_t": 0.4044186472892761,
"grad_norm": 17.890012741088867,
"learning_rate": 9.859757821558337e-08,
"logits/chosen": 0.7220809459686279,
"logits/rejected": 0.6579302549362183,
"logps/chosen": -198.95001220703125,
"logps/ref_chosen": -63.10905075073242,
"logps/ref_rejected": -82.49348449707031,
"logps/rejected": -287.0707092285156,
"loss": 1.0908,
"margin_dpo/margin_mean": 68.73626708984375,
"margin_dpo/margin_std": 96.3404312133789,
"step": 488
},
{
"KL/chosen_KL_mean": -156.16629028320312,
"KL/mean": -177.71923828125,
"KL/rejected_KL_mean": -199.2721710205078,
"KL/std": 93.63339233398438,
"epoch": 0.7392290249433107,
"fcm_dpo/beta": 0.006224127020686865,
"fcm_dpo/delta": 0.13544204831123352,
"fcm_dpo/margin": 43.10588455200195,
"fcm_dpo/q_t": 0.43940192461013794,
"grad_norm": 13.054624557495117,
"learning_rate": 9.754752911772615e-08,
"logits/chosen": 0.7028100490570068,
"logits/rejected": 0.6529111862182617,
"logps/chosen": -221.15524291992188,
"logps/ref_chosen": -64.98896026611328,
"logps/ref_rejected": -84.39607238769531,
"logps/rejected": -283.6682434082031,
"loss": 1.2362,
"margin_dpo/margin_mean": 43.10588836669922,
"margin_dpo/margin_std": 104.4052734375,
"step": 489
},
{
"KL/chosen_KL_mean": -140.01063537597656,
"KL/mean": -166.3353271484375,
"KL/rejected_KL_mean": -192.66000366210938,
"KL/std": 93.4408187866211,
"epoch": 0.7407407407407407,
"fcm_dpo/beta": 0.0063074370846152306,
"fcm_dpo/delta": 0.0700986459851265,
"fcm_dpo/margin": 52.64936065673828,
"fcm_dpo/q_t": 0.4261215925216675,
"grad_norm": 12.439244270324707,
"learning_rate": 9.650174444319956e-08,
"logits/chosen": 0.7870948314666748,
"logits/rejected": 0.7623904943466187,
"logps/chosen": -201.91937255859375,
"logps/ref_chosen": -61.90874481201172,
"logps/ref_rejected": -70.58566284179688,
"logps/rejected": -263.24566650390625,
"loss": 1.2173,
"margin_dpo/margin_mean": 52.64936447143555,
"margin_dpo/margin_std": 119.89442443847656,
"step": 490
},
{
"KL/chosen_KL_mean": -136.93490600585938,
"KL/mean": -166.25643920898438,
"KL/rejected_KL_mean": -195.5780029296875,
"KL/std": 87.6930923461914,
"epoch": 0.7422524565381708,
"fcm_dpo/beta": 0.00634703878313303,
"fcm_dpo/delta": 0.02819715440273285,
"fcm_dpo/margin": 58.64308166503906,
"fcm_dpo/q_t": 0.4154743552207947,
"grad_norm": 13.191024780273438,
"learning_rate": 9.546025344484868e-08,
"logits/chosen": 0.6767026782035828,
"logits/rejected": 0.6159626245498657,
"logps/chosen": -192.41061401367188,
"logps/ref_chosen": -55.47570037841797,
"logps/ref_rejected": -78.70318603515625,
"logps/rejected": -274.28118896484375,
"loss": 1.1309,
"margin_dpo/margin_mean": 58.64308166503906,
"margin_dpo/margin_std": 91.34356689453125,
"step": 491
},
{
"KL/chosen_KL_mean": -160.53651428222656,
"KL/mean": -184.10897827148438,
"KL/rejected_KL_mean": -207.68142700195312,
"KL/std": 93.39591979980469,
"epoch": 0.7437641723356009,
"fcm_dpo/beta": 0.006399978883564472,
"fcm_dpo/delta": 0.004464814905077219,
"fcm_dpo/margin": 47.14491271972656,
"fcm_dpo/q_t": 0.4312303066253662,
"grad_norm": 15.08497142791748,
"learning_rate": 9.442308525541589e-08,
"logits/chosen": 0.700499415397644,
"logits/rejected": 0.6257964968681335,
"logps/chosen": -227.82290649414062,
"logps/ref_chosen": -67.28638458251953,
"logps/ref_rejected": -82.78628540039062,
"logps/rejected": -290.46771240234375,
"loss": 1.2184,
"margin_dpo/margin_mean": 47.14491271972656,
"margin_dpo/margin_std": 104.46917724609375,
"step": 492
},
{
"KL/chosen_KL_mean": -132.22528076171875,
"KL/mean": -168.39915466308594,
"KL/rejected_KL_mean": -204.57302856445312,
"KL/std": 91.09408569335938,
"epoch": 0.745275888133031,
"fcm_dpo/beta": 0.006383996456861496,
"fcm_dpo/delta": -0.0653509870171547,
"fcm_dpo/margin": 72.34774017333984,
"fcm_dpo/q_t": 0.3952844738960266,
"grad_norm": 13.52745532989502,
"learning_rate": 9.339026888672468e-08,
"logits/chosen": 0.654663622379303,
"logits/rejected": 0.57381272315979,
"logps/chosen": -188.1527862548828,
"logps/ref_chosen": -55.92750549316406,
"logps/ref_rejected": -79.12149810791016,
"logps/rejected": -283.69451904296875,
"loss": 1.077,
"margin_dpo/margin_mean": 72.34774017333984,
"margin_dpo/margin_std": 100.20462036132812,
"step": 493
},
{
"KL/chosen_KL_mean": -137.5892791748047,
"KL/mean": -167.999755859375,
"KL/rejected_KL_mean": -198.4102783203125,
"KL/std": 92.60104370117188,
"epoch": 0.7467876039304611,
"fcm_dpo/beta": 0.006324524059891701,
"fcm_dpo/delta": 0.01592247188091278,
"fcm_dpo/margin": 60.82097625732422,
"fcm_dpo/q_t": 0.4137570261955261,
"grad_norm": 15.360010147094727,
"learning_rate": 9.236183322886945e-08,
"logits/chosen": 0.6244049072265625,
"logits/rejected": 0.5685616731643677,
"logps/chosen": -205.5433807373047,
"logps/ref_chosen": -67.95410919189453,
"logps/ref_rejected": -90.50865173339844,
"logps/rejected": -288.9189453125,
"loss": 1.1579,
"margin_dpo/margin_mean": 60.82097625732422,
"margin_dpo/margin_std": 113.72390747070312,
"step": 494
},
{
"KL/chosen_KL_mean": -135.71588134765625,
"KL/mean": -160.57827758789062,
"KL/rejected_KL_mean": -185.440673828125,
"KL/std": 90.57270812988281,
"epoch": 0.7482993197278912,
"fcm_dpo/beta": 0.006443200167268515,
"fcm_dpo/delta": 0.08159741759300232,
"fcm_dpo/margin": 49.72478485107422,
"fcm_dpo/q_t": 0.4313252568244934,
"grad_norm": 18.327566146850586,
"learning_rate": 9.133780704940594e-08,
"logits/chosen": 0.7656629681587219,
"logits/rejected": 0.6969238519668579,
"logps/chosen": -188.34133911132812,
"logps/ref_chosen": -52.62546157836914,
"logps/ref_rejected": -72.06781005859375,
"logps/rejected": -257.50848388671875,
"loss": 1.2067,
"margin_dpo/margin_mean": 49.72478485107422,
"margin_dpo/margin_std": 110.06767272949219,
"step": 495
},
{
"KL/chosen_KL_mean": -150.63674926757812,
"KL/mean": -181.3939208984375,
"KL/rejected_KL_mean": -212.15106201171875,
"KL/std": 94.11808013916016,
"epoch": 0.7498110355253212,
"fcm_dpo/beta": 0.00640734750777483,
"fcm_dpo/delta": 0.005259156227111816,
"fcm_dpo/margin": 61.514312744140625,
"fcm_dpo/q_t": 0.4163498878479004,
"grad_norm": 14.123679161071777,
"learning_rate": 9.031821899254797e-08,
"logits/chosen": 0.7562978863716125,
"logits/rejected": 0.6333326697349548,
"logps/chosen": -208.2340850830078,
"logps/ref_chosen": -57.597320556640625,
"logps/ref_rejected": -94.36127471923828,
"logps/rejected": -306.5123291015625,
"loss": 1.1605,
"margin_dpo/margin_mean": 61.514312744140625,
"margin_dpo/margin_std": 118.14901733398438,
"step": 496
},
{
"KL/chosen_KL_mean": -149.52847290039062,
"KL/mean": -184.99334716796875,
"KL/rejected_KL_mean": -220.45826721191406,
"KL/std": 91.86830139160156,
"epoch": 0.7513227513227513,
"fcm_dpo/beta": 0.0063869645819067955,
"fcm_dpo/delta": -0.05569346994161606,
"fcm_dpo/margin": 70.92979431152344,
"fcm_dpo/q_t": 0.39586740732192993,
"grad_norm": 11.90026569366455,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 0.7493371963500977,
"logits/rejected": 0.7143541574478149,
"logps/chosen": -222.3184051513672,
"logps/ref_chosen": -72.78994750976562,
"logps/ref_rejected": -89.48483276367188,
"logps/rejected": -309.943115234375,
"loss": 1.0789,
"margin_dpo/margin_mean": 70.92979431152344,
"margin_dpo/margin_std": 99.6292724609375,
"step": 497
},
{
"KL/chosen_KL_mean": -134.5188751220703,
"KL/mean": -169.0218963623047,
"KL/rejected_KL_mean": -203.52493286132812,
"KL/std": 88.04065704345703,
"epoch": 0.7528344671201814,
"fcm_dpo/beta": 0.006339473649859428,
"fcm_dpo/delta": -0.039198048412799835,
"fcm_dpo/margin": 69.00605010986328,
"fcm_dpo/q_t": 0.39908653497695923,
"grad_norm": 16.51783561706543,
"learning_rate": 8.829247120198563e-08,
"logits/chosen": 0.6983813047409058,
"logits/rejected": 0.6683753728866577,
"logps/chosen": -202.88458251953125,
"logps/ref_chosen": -68.36572265625,
"logps/ref_rejected": -71.28846740722656,
"logps/rejected": -274.81341552734375,
"loss": 1.0777,
"margin_dpo/margin_mean": 69.00605010986328,
"margin_dpo/margin_std": 93.56288146972656,
"step": 498
},
{
"KL/chosen_KL_mean": -133.99395751953125,
"KL/mean": -168.22283935546875,
"KL/rejected_KL_mean": -202.45172119140625,
"KL/std": 91.77056884765625,
"epoch": 0.7543461829176115,
"fcm_dpo/beta": 0.006308514624834061,
"fcm_dpo/delta": -0.03330450877547264,
"fcm_dpo/margin": 68.45777893066406,
"fcm_dpo/q_t": 0.40362346172332764,
"grad_norm": 15.176262855529785,
"learning_rate": 8.728636813280163e-08,
"logits/chosen": 0.7526177167892456,
"logits/rejected": 0.6799595355987549,
"logps/chosen": -195.90277099609375,
"logps/ref_chosen": -61.90882873535156,
"logps/ref_rejected": -91.9411392211914,
"logps/rejected": -294.39288330078125,
"loss": 1.1343,
"margin_dpo/margin_mean": 68.45777893066406,
"margin_dpo/margin_std": 119.55030822753906,
"step": 499
},
{
"KL/chosen_KL_mean": -140.39114379882812,
"KL/mean": -171.11715698242188,
"KL/rejected_KL_mean": -201.8431396484375,
"KL/std": 85.47956085205078,
"epoch": 0.7558578987150416,
"fcm_dpo/beta": 0.006281760521233082,
"fcm_dpo/delta": 0.014384115114808083,
"fcm_dpo/margin": 61.45201110839844,
"fcm_dpo/q_t": 0.41101551055908203,
"grad_norm": 16.758703231811523,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.6605246067047119,
"logits/rejected": 0.6593271493911743,
"logps/chosen": -210.61697387695312,
"logps/ref_chosen": -70.225830078125,
"logps/ref_rejected": -71.72203063964844,
"logps/rejected": -273.565185546875,
"loss": 1.1598,
"margin_dpo/margin_mean": 61.45201110839844,
"margin_dpo/margin_std": 114.59700012207031,
"step": 500
},
{
"KL/chosen_KL_mean": -141.39710998535156,
"KL/mean": -168.8629608154297,
"KL/rejected_KL_mean": -196.3288116455078,
"KL/std": 90.79582214355469,
"epoch": 0.7573696145124716,
"fcm_dpo/beta": 0.006372970528900623,
"fcm_dpo/delta": 0.05140642821788788,
"fcm_dpo/margin": 54.93169403076172,
"fcm_dpo/q_t": 0.41898688673973083,
"grad_norm": 12.851356506347656,
"learning_rate": 8.528784436016878e-08,
"logits/chosen": 0.736147403717041,
"logits/rejected": 0.7371499538421631,
"logps/chosen": -205.99591064453125,
"logps/ref_chosen": -64.59880828857422,
"logps/ref_rejected": -70.59329223632812,
"logps/rejected": -266.922119140625,
"loss": 1.1243,
"margin_dpo/margin_mean": 54.93169403076172,
"margin_dpo/margin_std": 76.56843566894531,
"step": 501
},
{
"KL/chosen_KL_mean": -139.1030731201172,
"KL/mean": -168.59788513183594,
"KL/rejected_KL_mean": -198.09271240234375,
"KL/std": 94.01078796386719,
"epoch": 0.7588813303099018,
"fcm_dpo/beta": 0.006408554967492819,
"fcm_dpo/delta": 0.022638794034719467,
"fcm_dpo/margin": 58.9896354675293,
"fcm_dpo/q_t": 0.413457989692688,
"grad_norm": 14.23745059967041,
"learning_rate": 8.4295479559726e-08,
"logits/chosen": 0.7297828793525696,
"logits/rejected": 0.678575873374939,
"logps/chosen": -204.5697021484375,
"logps/ref_chosen": -65.46662902832031,
"logps/ref_rejected": -90.22233581542969,
"logps/rejected": -288.3150634765625,
"loss": 1.1223,
"margin_dpo/margin_mean": 58.98963165283203,
"margin_dpo/margin_std": 89.94447326660156,
"step": 502
},
{
"KL/chosen_KL_mean": -128.18252563476562,
"KL/mean": -159.30853271484375,
"KL/rejected_KL_mean": -190.4345245361328,
"KL/std": 86.86588287353516,
"epoch": 0.7603930461073318,
"fcm_dpo/beta": 0.006394956260919571,
"fcm_dpo/delta": 0.0019593043252825737,
"fcm_dpo/margin": 62.25199890136719,
"fcm_dpo/q_t": 0.4089614152908325,
"grad_norm": 12.124394416809082,
"learning_rate": 8.330774987092712e-08,
"logits/chosen": 0.7072443962097168,
"logits/rejected": 0.7087694406509399,
"logps/chosen": -180.01727294921875,
"logps/ref_chosen": -51.83476257324219,
"logps/ref_rejected": -57.62522506713867,
"logps/rejected": -248.05975341796875,
"loss": 1.125,
"margin_dpo/margin_mean": 62.25199890136719,
"margin_dpo/margin_std": 99.93350219726562,
"step": 503
},
{
"KL/chosen_KL_mean": -130.51376342773438,
"KL/mean": -171.15078735351562,
"KL/rejected_KL_mean": -211.78778076171875,
"KL/std": 85.13151550292969,
"epoch": 0.7619047619047619,
"fcm_dpo/beta": 0.006323833949863911,
"fcm_dpo/delta": -0.12009334564208984,
"fcm_dpo/margin": 81.2740249633789,
"fcm_dpo/q_t": 0.38102343678474426,
"grad_norm": 15.250235557556152,
"learning_rate": 8.232468292269479e-08,
"logits/chosen": 0.6725870370864868,
"logits/rejected": 0.6501311659812927,
"logps/chosen": -199.1649627685547,
"logps/ref_chosen": -68.65119934082031,
"logps/ref_rejected": -77.91394805908203,
"logps/rejected": -289.70172119140625,
"loss": 1.0115,
"margin_dpo/margin_mean": 81.2740249633789,
"margin_dpo/margin_std": 87.40478515625,
"step": 504
},
{
"KL/chosen_KL_mean": -137.549072265625,
"KL/mean": -164.11795043945312,
"KL/rejected_KL_mean": -190.68679809570312,
"KL/std": 94.96287536621094,
"epoch": 0.763416477702192,
"fcm_dpo/beta": 0.006210042163729668,
"fcm_dpo/delta": -0.03239330276846886,
"fcm_dpo/margin": 53.13771057128906,
"fcm_dpo/q_t": 0.4257761538028717,
"grad_norm": 13.7847318649292,
"learning_rate": 8.134630621352483e-08,
"logits/chosen": 0.7000030279159546,
"logits/rejected": 0.6607384085655212,
"logps/chosen": -197.54794311523438,
"logps/ref_chosen": -59.99884796142578,
"logps/ref_rejected": -76.88048553466797,
"logps/rejected": -267.5672912597656,
"loss": 1.195,
"margin_dpo/margin_mean": 53.13771057128906,
"margin_dpo/margin_std": 108.96891784667969,
"step": 505
},
{
"KL/chosen_KL_mean": -134.461669921875,
"KL/mean": -163.64649963378906,
"KL/rejected_KL_mean": -192.83132934570312,
"KL/std": 85.74610900878906,
"epoch": 0.764928193499622,
"fcm_dpo/beta": 0.006229479797184467,
"fcm_dpo/delta": 0.03772689029574394,
"fcm_dpo/margin": 58.36964416503906,
"fcm_dpo/q_t": 0.416409432888031,
"grad_norm": 15.34953498840332,
"learning_rate": 8.037264711071698e-08,
"logits/chosen": 0.717422604560852,
"logits/rejected": 0.6991676092147827,
"logps/chosen": -204.53297424316406,
"logps/ref_chosen": -70.07130432128906,
"logps/ref_rejected": -82.03775024414062,
"logps/rejected": -274.86907958984375,
"loss": 1.1632,
"margin_dpo/margin_mean": 58.36964416503906,
"margin_dpo/margin_std": 107.82905578613281,
"step": 506
},
{
"KL/chosen_KL_mean": -147.78289794921875,
"KL/mean": -178.89292907714844,
"KL/rejected_KL_mean": -210.00294494628906,
"KL/std": 93.81893920898438,
"epoch": 0.7664399092970522,
"fcm_dpo/beta": 0.006245059426873922,
"fcm_dpo/delta": 0.01166222058236599,
"fcm_dpo/margin": 62.22006607055664,
"fcm_dpo/q_t": 0.4153628349304199,
"grad_norm": 13.224839210510254,
"learning_rate": 7.940373284960933e-08,
"logits/chosen": 0.717066764831543,
"logits/rejected": 0.6627391576766968,
"logps/chosen": -219.78993225097656,
"logps/ref_chosen": -72.00703430175781,
"logps/ref_rejected": -93.94987487792969,
"logps/rejected": -303.95281982421875,
"loss": 1.1506,
"margin_dpo/margin_mean": 62.220069885253906,
"margin_dpo/margin_std": 111.81320190429688,
"step": 507
},
{
"KL/chosen_KL_mean": -135.2474822998047,
"KL/mean": -170.38302612304688,
"KL/rejected_KL_mean": -205.5185546875,
"KL/std": 96.03981018066406,
"epoch": 0.7679516250944822,
"fcm_dpo/beta": 0.006270278245210648,
"fcm_dpo/delta": -0.04299917072057724,
"fcm_dpo/margin": 70.27108764648438,
"fcm_dpo/q_t": 0.4007454514503479,
"grad_norm": 15.34142017364502,
"learning_rate": 7.843959053281663e-08,
"logits/chosen": 0.6530667543411255,
"logits/rejected": 0.5221731662750244,
"logps/chosen": -195.4674072265625,
"logps/ref_chosen": -60.21992492675781,
"logps/ref_rejected": -95.9200668334961,
"logps/rejected": -301.4386291503906,
"loss": 1.0979,
"margin_dpo/margin_mean": 70.27108001708984,
"margin_dpo/margin_std": 104.7369155883789,
"step": 508
},
{
"KL/chosen_KL_mean": -141.84889221191406,
"KL/mean": -171.0345458984375,
"KL/rejected_KL_mean": -200.22021484375,
"KL/std": 88.09879302978516,
"epoch": 0.7694633408919124,
"fcm_dpo/beta": 0.006239317357540131,
"fcm_dpo/delta": 0.03714311867952347,
"fcm_dpo/margin": 58.37133026123047,
"fcm_dpo/q_t": 0.4168880581855774,
"grad_norm": 16.268436431884766,
"learning_rate": 7.748024712947204e-08,
"logits/chosen": 0.6298993825912476,
"logits/rejected": 0.6071436405181885,
"logps/chosen": -208.11904907226562,
"logps/ref_chosen": -66.27017211914062,
"logps/ref_rejected": -71.73065185546875,
"logps/rejected": -271.95086669921875,
"loss": 1.1474,
"margin_dpo/margin_mean": 58.3713264465332,
"margin_dpo/margin_std": 99.90489959716797,
"step": 509
},
{
"KL/chosen_KL_mean": -141.15176391601562,
"KL/mean": -176.81494140625,
"KL/rejected_KL_mean": -212.47811889648438,
"KL/std": 93.73820495605469,
"epoch": 0.7709750566893424,
"fcm_dpo/beta": 0.0061934944242239,
"fcm_dpo/delta": -0.0443672351539135,
"fcm_dpo/margin": 71.32635498046875,
"fcm_dpo/q_t": 0.40235698223114014,
"grad_norm": 14.645244598388672,
"learning_rate": 7.652572947447272e-08,
"logits/chosen": 0.8241918087005615,
"logits/rejected": 0.7173888087272644,
"logps/chosen": -194.6966552734375,
"logps/ref_chosen": -53.54487609863281,
"logps/ref_rejected": -91.36648559570312,
"logps/rejected": -303.8446044921875,
"loss": 1.1148,
"margin_dpo/margin_mean": 71.32635498046875,
"margin_dpo/margin_std": 116.95710754394531,
"step": 510
},
{
"KL/chosen_KL_mean": -134.23875427246094,
"KL/mean": -175.78126525878906,
"KL/rejected_KL_mean": -217.3237762451172,
"KL/std": 88.19770050048828,
"epoch": 0.7724867724867724,
"fcm_dpo/beta": 0.006117081269621849,
"fcm_dpo/delta": -0.11388811469078064,
"fcm_dpo/margin": 83.08502197265625,
"fcm_dpo/q_t": 0.38372671604156494,
"grad_norm": 17.872051239013672,
"learning_rate": 7.557606426772961e-08,
"logits/chosen": 0.7054228186607361,
"logits/rejected": 0.645592212677002,
"logps/chosen": -190.0831298828125,
"logps/ref_chosen": -55.844383239746094,
"logps/ref_rejected": -86.49819946289062,
"logps/rejected": -303.82196044921875,
"loss": 1.031,
"margin_dpo/margin_mean": 83.08502960205078,
"margin_dpo/margin_std": 100.14347076416016,
"step": 511
},
{
"KL/chosen_KL_mean": -136.84991455078125,
"KL/mean": -164.41091918945312,
"KL/rejected_KL_mean": -191.9718780517578,
"KL/std": 83.326904296875,
"epoch": 0.7739984882842026,
"fcm_dpo/beta": 0.006117596291005611,
"fcm_dpo/delta": 0.0650286003947258,
"fcm_dpo/margin": 55.12196350097656,
"fcm_dpo/q_t": 0.4222397208213806,
"grad_norm": 18.95017433166504,
"learning_rate": 7.463127807341966e-08,
"logits/chosen": 0.5895268321037292,
"logits/rejected": 0.5832157135009766,
"logps/chosen": -198.50296020507812,
"logps/ref_chosen": -61.653038024902344,
"logps/ref_rejected": -72.83148193359375,
"logps/rejected": -264.8033752441406,
"loss": 1.1734,
"margin_dpo/margin_mean": 55.1219596862793,
"margin_dpo/margin_std": 103.32221984863281,
"step": 512
},
{
"KL/chosen_KL_mean": -122.05601501464844,
"KL/mean": -156.81182861328125,
"KL/rejected_KL_mean": -191.56765747070312,
"KL/std": 85.8311767578125,
"epoch": 0.7755102040816326,
"fcm_dpo/beta": 0.0061059207655489445,
"fcm_dpo/delta": -0.02579668164253235,
"fcm_dpo/margin": 69.51165771484375,
"fcm_dpo/q_t": 0.40276288986206055,
"grad_norm": 11.792524337768555,
"learning_rate": 7.369139731924401e-08,
"logits/chosen": 0.8557263016700745,
"logits/rejected": 0.7962871789932251,
"logps/chosen": -172.9085693359375,
"logps/ref_chosen": -50.85256576538086,
"logps/ref_rejected": -69.21754455566406,
"logps/rejected": -260.78521728515625,
"loss": 1.0779,
"margin_dpo/margin_mean": 69.51165771484375,
"margin_dpo/margin_std": 90.19849395751953,
"step": 513
},
{
"KL/chosen_KL_mean": -138.94390869140625,
"KL/mean": -176.84555053710938,
"KL/rejected_KL_mean": -214.74722290039062,
"KL/std": 91.81410217285156,
"epoch": 0.7770219198790628,
"fcm_dpo/beta": 0.006081851664930582,
"fcm_dpo/delta": -0.06402106583118439,
"fcm_dpo/margin": 75.80330657958984,
"fcm_dpo/q_t": 0.39448386430740356,
"grad_norm": 15.198996543884277,
"learning_rate": 7.275644829568747e-08,
"logits/chosen": 0.7719430327415466,
"logits/rejected": 0.7340209484100342,
"logps/chosen": -208.32882690429688,
"logps/ref_chosen": -69.38493347167969,
"logps/ref_rejected": -83.32447814941406,
"logps/rejected": -298.0716857910156,
"loss": 1.0747,
"margin_dpo/margin_mean": 75.80330657958984,
"margin_dpo/margin_std": 105.60943603515625,
"step": 514
},
{
"KL/chosen_KL_mean": -147.2511749267578,
"KL/mean": -177.1293487548828,
"KL/rejected_KL_mean": -207.00753784179688,
"KL/std": 88.286865234375,
"epoch": 0.7785336356764928,
"fcm_dpo/beta": 0.006070663221180439,
"fcm_dpo/delta": 0.03865630924701691,
"fcm_dpo/margin": 59.75636291503906,
"fcm_dpo/q_t": 0.4169022738933563,
"grad_norm": 16.843313217163086,
"learning_rate": 7.182645715528435e-08,
"logits/chosen": 0.751872718334198,
"logits/rejected": 0.6661347150802612,
"logps/chosen": -200.93820190429688,
"logps/ref_chosen": -53.687034606933594,
"logps/ref_rejected": -83.59614562988281,
"logps/rejected": -290.6036682128906,
"loss": 1.1496,
"margin_dpo/margin_mean": 59.75636291503906,
"margin_dpo/margin_std": 103.61663818359375,
"step": 515
},
{
"KL/chosen_KL_mean": -125.45993041992188,
"KL/mean": -155.32858276367188,
"KL/rejected_KL_mean": -185.197265625,
"KL/std": 87.19376373291016,
"epoch": 0.780045351473923,
"fcm_dpo/beta": 0.006130651570856571,
"fcm_dpo/delta": 0.03489149734377861,
"fcm_dpo/margin": 59.737335205078125,
"fcm_dpo/q_t": 0.4151182770729065,
"grad_norm": 18.127180099487305,
"learning_rate": 7.090144991188568e-08,
"logits/chosen": 0.7051277160644531,
"logits/rejected": 0.6652114987373352,
"logps/chosen": -182.36166381835938,
"logps/ref_chosen": -56.9017219543457,
"logps/ref_rejected": -67.83477783203125,
"logps/rejected": -253.03204345703125,
"loss": 1.1461,
"margin_dpo/margin_mean": 59.737335205078125,
"margin_dpo/margin_std": 101.89432525634766,
"step": 516
},
{
"KL/chosen_KL_mean": -151.65194702148438,
"KL/mean": -172.27166748046875,
"KL/rejected_KL_mean": -192.89134216308594,
"KL/std": 89.90000915527344,
"epoch": 0.781557067271353,
"fcm_dpo/beta": 0.006148169748485088,
"fcm_dpo/delta": 0.03174401819705963,
"fcm_dpo/margin": 41.239402770996094,
"fcm_dpo/q_t": 0.44275960326194763,
"grad_norm": 16.33505630493164,
"learning_rate": 6.998145243993284e-08,
"logits/chosen": 0.762154221534729,
"logits/rejected": 0.7593005895614624,
"logps/chosen": -213.42709350585938,
"logps/ref_chosen": -61.775142669677734,
"logps/ref_rejected": -62.88270950317383,
"logps/rejected": -255.7740478515625,
"loss": 1.2326,
"margin_dpo/margin_mean": 41.239402770996094,
"margin_dpo/margin_std": 95.73959350585938,
"step": 517
},
{
"KL/chosen_KL_mean": -127.51979064941406,
"KL/mean": -158.21627807617188,
"KL/rejected_KL_mean": -188.91275024414062,
"KL/std": 88.48887634277344,
"epoch": 0.783068783068783,
"fcm_dpo/beta": 0.006185801234096289,
"fcm_dpo/delta": 0.02103758044540882,
"fcm_dpo/margin": 61.39295959472656,
"fcm_dpo/q_t": 0.41513732075691223,
"grad_norm": 13.579456329345703,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 0.7131055593490601,
"logits/rejected": 0.6640417575836182,
"logps/chosen": -189.54502868652344,
"logps/ref_chosen": -62.02523422241211,
"logps/ref_rejected": -79.06085205078125,
"logps/rejected": -267.9736022949219,
"loss": 1.1298,
"margin_dpo/margin_mean": 61.39295959472656,
"margin_dpo/margin_std": 99.5591812133789,
"step": 518
},
{
"KL/chosen_KL_mean": -155.96485900878906,
"KL/mean": -174.09811401367188,
"KL/rejected_KL_mean": -192.23138427734375,
"KL/std": 91.28584289550781,
"epoch": 0.7845804988662132,
"fcm_dpo/beta": 0.006233462132513523,
"fcm_dpo/delta": 0.06380188465118408,
"fcm_dpo/margin": 36.266510009765625,
"fcm_dpo/q_t": 0.4491075873374939,
"grad_norm": 22.287879943847656,
"learning_rate": 6.815658960673781e-08,
"logits/chosen": 0.7426184415817261,
"logits/rejected": 0.6872553825378418,
"logps/chosen": -217.57122802734375,
"logps/ref_chosen": -61.60636901855469,
"logps/ref_rejected": -74.50727844238281,
"logps/rejected": -266.7386474609375,
"loss": 1.3141,
"margin_dpo/margin_mean": 36.26651382446289,
"margin_dpo/margin_std": 123.55844116210938,
"step": 519
},
{
"KL/chosen_KL_mean": -142.34487915039062,
"KL/mean": -168.37210083007812,
"KL/rejected_KL_mean": -194.39932250976562,
"KL/std": 90.46813201904297,
"epoch": 0.7860922146636432,
"fcm_dpo/beta": 0.0062470934353768826,
"fcm_dpo/delta": -0.02087680622935295,
"fcm_dpo/margin": 52.0544548034668,
"fcm_dpo/q_t": 0.4262607991695404,
"grad_norm": 14.57567310333252,
"learning_rate": 6.725177529083209e-08,
"logits/chosen": 0.8132271766662598,
"logits/rejected": 0.7505690455436707,
"logps/chosen": -205.21829223632812,
"logps/ref_chosen": -62.87343215942383,
"logps/ref_rejected": -76.505615234375,
"logps/rejected": -270.9049377441406,
"loss": 1.171,
"margin_dpo/margin_mean": 52.0544548034668,
"margin_dpo/margin_std": 94.20343017578125,
"step": 520
},
{
"KL/chosen_KL_mean": -140.08639526367188,
"KL/mean": -178.951171875,
"KL/rejected_KL_mean": -217.81597900390625,
"KL/std": 88.89877319335938,
"epoch": 0.7876039304610734,
"fcm_dpo/beta": 0.006182870361953974,
"fcm_dpo/delta": -0.08457393944263458,
"fcm_dpo/margin": 77.72958374023438,
"fcm_dpo/q_t": 0.3901776373386383,
"grad_norm": 12.525458335876465,
"learning_rate": 6.63520728356167e-08,
"logits/chosen": 0.5966737866401672,
"logits/rejected": 0.5146248936653137,
"logps/chosen": -204.29306030273438,
"logps/ref_chosen": -64.20668029785156,
"logps/ref_rejected": -92.28083038330078,
"logps/rejected": -310.0968017578125,
"loss": 1.0485,
"margin_dpo/margin_mean": 77.72958374023438,
"margin_dpo/margin_std": 98.73197174072266,
"step": 521
},
{
"KL/chosen_KL_mean": -140.26315307617188,
"KL/mean": -165.13807678222656,
"KL/rejected_KL_mean": -190.0129852294922,
"KL/std": 91.32360076904297,
"epoch": 0.7891156462585034,
"fcm_dpo/beta": 0.006213832646608353,
"fcm_dpo/delta": 0.09386920928955078,
"fcm_dpo/margin": 49.749847412109375,
"fcm_dpo/q_t": 0.4305458962917328,
"grad_norm": 15.26350212097168,
"learning_rate": 6.545750740770336e-08,
"logits/chosen": 0.6940236687660217,
"logits/rejected": 0.6864628791809082,
"logps/chosen": -198.63287353515625,
"logps/ref_chosen": -58.369720458984375,
"logps/ref_rejected": -68.79248046875,
"logps/rejected": -258.80548095703125,
"loss": 1.2264,
"margin_dpo/margin_mean": 49.749847412109375,
"margin_dpo/margin_std": 117.19786071777344,
"step": 522
},
{
"KL/chosen_KL_mean": -145.0025634765625,
"KL/mean": -173.7677459716797,
"KL/rejected_KL_mean": -202.53289794921875,
"KL/std": 89.35675048828125,
"epoch": 0.7906273620559335,
"fcm_dpo/beta": 0.006287074647843838,
"fcm_dpo/delta": 0.03975531458854675,
"fcm_dpo/margin": 57.53034591674805,
"fcm_dpo/q_t": 0.41568297147750854,
"grad_norm": 19.2230224609375,
"learning_rate": 6.456810403001012e-08,
"logits/chosen": 0.7452864050865173,
"logits/rejected": 0.611765444278717,
"logps/chosen": -210.71580505371094,
"logps/ref_chosen": -65.71324157714844,
"logps/ref_rejected": -91.98896789550781,
"logps/rejected": -294.5218811035156,
"loss": 1.1653,
"margin_dpo/margin_mean": 57.53034591674805,
"margin_dpo/margin_std": 107.65438842773438,
"step": 523
},
{
"KL/chosen_KL_mean": -124.53634643554688,
"KL/mean": -154.94583129882812,
"KL/rejected_KL_mean": -185.3553466796875,
"KL/std": 86.52081298828125,
"epoch": 0.7921390778533636,
"fcm_dpo/beta": 0.0063460636883974075,
"fcm_dpo/delta": 0.014257097616791725,
"fcm_dpo/margin": 60.81899642944336,
"fcm_dpo/q_t": 0.41111665964126587,
"grad_norm": 14.744943618774414,
"learning_rate": 6.368388758106134e-08,
"logits/chosen": 0.6547163724899292,
"logits/rejected": 0.6287938356399536,
"logps/chosen": -200.88758850097656,
"logps/ref_chosen": -76.35124969482422,
"logps/ref_rejected": -89.96072387695312,
"logps/rejected": -275.3160705566406,
"loss": 1.12,
"margin_dpo/margin_mean": 60.81899642944336,
"margin_dpo/margin_std": 92.8304443359375,
"step": 524
},
{
"KL/chosen_KL_mean": -139.3240509033203,
"KL/mean": -164.90277099609375,
"KL/rejected_KL_mean": -190.48150634765625,
"KL/std": 87.61563110351562,
"epoch": 0.7936507936507936,
"fcm_dpo/beta": 0.006399834528565407,
"fcm_dpo/delta": 0.07503412663936615,
"fcm_dpo/margin": 51.157466888427734,
"fcm_dpo/q_t": 0.4269210994243622,
"grad_norm": 17.56284523010254,
"learning_rate": 6.280488279429185e-08,
"logits/chosen": 0.5492737889289856,
"logits/rejected": 0.5443263053894043,
"logps/chosen": -214.81982421875,
"logps/ref_chosen": -75.49578857421875,
"logps/ref_rejected": -84.04852294921875,
"logps/rejected": -274.530029296875,
"loss": 1.1899,
"margin_dpo/margin_mean": 51.157466888427734,
"margin_dpo/margin_std": 104.63352966308594,
"step": 525
},
{
"KL/chosen_KL_mean": -149.576416015625,
"KL/mean": -171.56756591796875,
"KL/rejected_KL_mean": -193.5587158203125,
"KL/std": 89.04336547851562,
"epoch": 0.7951625094482238,
"fcm_dpo/beta": 0.006393382325768471,
"fcm_dpo/delta": -0.02260620892047882,
"fcm_dpo/margin": 43.982322692871094,
"fcm_dpo/q_t": 0.4350745379924774,
"grad_norm": 15.18529224395752,
"learning_rate": 6.193111425735515e-08,
"logits/chosen": 0.7070901393890381,
"logits/rejected": 0.6331349015235901,
"logps/chosen": -210.8688201904297,
"logps/ref_chosen": -61.29241943359375,
"logps/ref_rejected": -82.47763061523438,
"logps/rejected": -276.0363464355469,
"loss": 1.2176,
"margin_dpo/margin_mean": 43.982322692871094,
"margin_dpo/margin_std": 94.80473327636719,
"step": 526
},
{
"KL/chosen_KL_mean": -157.97311401367188,
"KL/mean": -177.66783142089844,
"KL/rejected_KL_mean": -197.362548828125,
"KL/std": 89.63215637207031,
"epoch": 0.7966742252456538,
"fcm_dpo/beta": 0.0064140548929572105,
"fcm_dpo/delta": 0.03223041817545891,
"fcm_dpo/margin": 39.389434814453125,
"fcm_dpo/q_t": 0.4437049627304077,
"grad_norm": 15.843182563781738,
"learning_rate": 6.106260641143546e-08,
"logits/chosen": 0.8329297304153442,
"logits/rejected": 0.7425129413604736,
"logps/chosen": -219.44573974609375,
"logps/ref_chosen": -61.472625732421875,
"logps/ref_rejected": -90.52831268310547,
"logps/rejected": -287.890869140625,
"loss": 1.2581,
"margin_dpo/margin_mean": 39.389434814453125,
"margin_dpo/margin_std": 105.0871810913086,
"step": 527
},
{
"KL/chosen_KL_mean": -139.7630157470703,
"KL/mean": -161.988525390625,
"KL/rejected_KL_mean": -184.21405029296875,
"KL/std": 87.73529052734375,
"epoch": 0.7981859410430839,
"fcm_dpo/beta": 0.006528710946440697,
"fcm_dpo/delta": 0.11322879046201706,
"fcm_dpo/margin": 44.4510383605957,
"fcm_dpo/q_t": 0.435050904750824,
"grad_norm": 16.909067153930664,
"learning_rate": 6.019938355056422e-08,
"logits/chosen": 0.6231927871704102,
"logits/rejected": 0.5417755842208862,
"logps/chosen": -198.55502319335938,
"logps/ref_chosen": -58.792015075683594,
"logps/ref_rejected": -71.82516479492188,
"logps/rejected": -256.0392150878906,
"loss": 1.2426,
"margin_dpo/margin_mean": 44.45103454589844,
"margin_dpo/margin_std": 110.95631408691406,
"step": 528
},
{
"KL/chosen_KL_mean": -130.37155151367188,
"KL/mean": -176.52496337890625,
"KL/rejected_KL_mean": -222.67837524414062,
"KL/std": 87.30693054199219,
"epoch": 0.799697656840514,
"fcm_dpo/beta": 0.006379758473485708,
"fcm_dpo/delta": -0.2010403275489807,
"fcm_dpo/margin": 92.30682373046875,
"fcm_dpo/q_t": 0.3642774224281311,
"grad_norm": 16.06795883178711,
"learning_rate": 5.934146982094049e-08,
"logits/chosen": 0.6257309317588806,
"logits/rejected": 0.5704358816146851,
"logps/chosen": -185.4425048828125,
"logps/ref_chosen": -55.070960998535156,
"logps/ref_rejected": -75.44007873535156,
"logps/rejected": -298.1184387207031,
"loss": 0.9655,
"margin_dpo/margin_mean": 92.30682373046875,
"margin_dpo/margin_std": 94.4359130859375,
"step": 529
},
{
"KL/chosen_KL_mean": -134.4344482421875,
"KL/mean": -162.27719116210938,
"KL/rejected_KL_mean": -190.1199493408203,
"KL/std": 89.534912109375,
"epoch": 0.8012093726379441,
"fcm_dpo/beta": 0.0063695237040519714,
"fcm_dpo/delta": 0.046955712139606476,
"fcm_dpo/margin": 55.685489654541016,
"fcm_dpo/q_t": 0.4205434322357178,
"grad_norm": 17.66626739501953,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 0.7525385618209839,
"logits/rejected": 0.7037971019744873,
"logps/chosen": -191.1782684326172,
"logps/ref_chosen": -56.743812561035156,
"logps/ref_rejected": -76.6692123413086,
"logps/rejected": -266.7891540527344,
"loss": 1.1516,
"margin_dpo/margin_mean": 55.685489654541016,
"margin_dpo/margin_std": 95.46461486816406,
"step": 530
},
{
"KL/chosen_KL_mean": -135.20166015625,
"KL/mean": -164.94931030273438,
"KL/rejected_KL_mean": -194.69696044921875,
"KL/std": 89.3927230834961,
"epoch": 0.8027210884353742,
"fcm_dpo/beta": 0.006420046091079712,
"fcm_dpo/delta": 0.01849624700844288,
"fcm_dpo/margin": 59.49530792236328,
"fcm_dpo/q_t": 0.4134773015975952,
"grad_norm": 14.303645133972168,
"learning_rate": 5.7641665597021435e-08,
"logits/chosen": 0.7184991836547852,
"logits/rejected": 0.6369512677192688,
"logps/chosen": -186.318115234375,
"logps/ref_chosen": -51.116455078125,
"logps/ref_rejected": -79.52884674072266,
"logps/rejected": -274.225830078125,
"loss": 1.1288,
"margin_dpo/margin_mean": 59.49530792236328,
"margin_dpo/margin_std": 95.1216812133789,
"step": 531
},
{
"KL/chosen_KL_mean": -155.84034729003906,
"KL/mean": -187.3836669921875,
"KL/rejected_KL_mean": -218.92703247070312,
"KL/std": 87.58135223388672,
"epoch": 0.8042328042328042,
"fcm_dpo/beta": 0.006401236169040203,
"fcm_dpo/delta": -0.003994982689619064,
"fcm_dpo/margin": 63.08666229248047,
"fcm_dpo/q_t": 0.40819212794303894,
"grad_norm": 15.15030574798584,
"learning_rate": 5.679982264990424e-08,
"logits/chosen": 0.6725942492485046,
"logits/rejected": 0.6169898509979248,
"logps/chosen": -214.12030029296875,
"logps/ref_chosen": -58.279945373535156,
"logps/ref_rejected": -78.05426788330078,
"logps/rejected": -296.9812927246094,
"loss": 1.1171,
"margin_dpo/margin_mean": 63.08666229248047,
"margin_dpo/margin_std": 98.99496459960938,
"step": 532
},
{
"KL/chosen_KL_mean": -110.21849060058594,
"KL/mean": -143.2732696533203,
"KL/rejected_KL_mean": -176.32806396484375,
"KL/std": 88.17913818359375,
"epoch": 0.8057445200302343,
"fcm_dpo/beta": 0.0063875531777739525,
"fcm_dpo/delta": -0.02326737344264984,
"fcm_dpo/margin": 66.10958862304688,
"fcm_dpo/q_t": 0.40347611904144287,
"grad_norm": 14.943767547607422,
"learning_rate": 5.596338392706076e-08,
"logits/chosen": 0.7972488403320312,
"logits/rejected": 0.726055383682251,
"logps/chosen": -166.63650512695312,
"logps/ref_chosen": -56.41801071166992,
"logps/ref_rejected": -73.89324951171875,
"logps/rejected": -250.2213134765625,
"loss": 1.0954,
"margin_dpo/margin_mean": 66.10958862304688,
"margin_dpo/margin_std": 94.71406555175781,
"step": 533
},
{
"KL/chosen_KL_mean": -137.1483154296875,
"KL/mean": -167.67965698242188,
"KL/rejected_KL_mean": -198.2110137939453,
"KL/std": 88.89889526367188,
"epoch": 0.8072562358276644,
"fcm_dpo/beta": 0.006359100341796875,
"fcm_dpo/delta": 0.011935360729694366,
"fcm_dpo/margin": 61.062713623046875,
"fcm_dpo/q_t": 0.4135010242462158,
"grad_norm": 13.726229667663574,
"learning_rate": 5.513237282548033e-08,
"logits/chosen": 0.7232074737548828,
"logits/rejected": 0.6840554475784302,
"logps/chosen": -197.89700317382812,
"logps/ref_chosen": -60.748687744140625,
"logps/ref_rejected": -73.8623046875,
"logps/rejected": -272.07330322265625,
"loss": 1.1472,
"margin_dpo/margin_mean": 61.062713623046875,
"margin_dpo/margin_std": 108.4185562133789,
"step": 534
},
{
"KL/chosen_KL_mean": -148.7552032470703,
"KL/mean": -174.89154052734375,
"KL/rejected_KL_mean": -201.0278778076172,
"KL/std": 92.47640991210938,
"epoch": 0.8087679516250945,
"fcm_dpo/beta": 0.0064563388004899025,
"fcm_dpo/delta": 0.06457997858524323,
"fcm_dpo/margin": 52.272666931152344,
"fcm_dpo/q_t": 0.4242613911628723,
"grad_norm": 16.301424026489258,
"learning_rate": 5.430681259032957e-08,
"logits/chosen": 0.6059026718139648,
"logits/rejected": 0.5410973429679871,
"logps/chosen": -210.39260864257812,
"logps/ref_chosen": -61.637413024902344,
"logps/ref_rejected": -80.93138885498047,
"logps/rejected": -281.95928955078125,
"loss": 1.1784,
"margin_dpo/margin_mean": 52.27267074584961,
"margin_dpo/margin_std": 101.03744506835938,
"step": 535
},
{
"KL/chosen_KL_mean": -128.37030029296875,
"KL/mean": -170.2589111328125,
"KL/rejected_KL_mean": -212.14752197265625,
"KL/std": 93.7471694946289,
"epoch": 0.8102796674225246,
"fcm_dpo/beta": 0.006328102201223373,
"fcm_dpo/delta": -0.13771645724773407,
"fcm_dpo/margin": 83.77720642089844,
"fcm_dpo/q_t": 0.37935811281204224,
"grad_norm": 10.940221786499023,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 0.7641968727111816,
"logits/rejected": 0.6712781190872192,
"logps/chosen": -180.25927734375,
"logps/ref_chosen": -51.88897705078125,
"logps/ref_rejected": -73.34864044189453,
"logps/rejected": -285.49615478515625,
"loss": 1.0005,
"margin_dpo/margin_mean": 83.77720642089844,
"margin_dpo/margin_std": 91.10980224609375,
"step": 536
},
{
"KL/chosen_KL_mean": -146.61001586914062,
"KL/mean": -179.15560913085938,
"KL/rejected_KL_mean": -211.70120239257812,
"KL/std": 96.09361267089844,
"epoch": 0.8117913832199547,
"fcm_dpo/beta": 0.006261053029447794,
"fcm_dpo/delta": -0.008215773850679398,
"fcm_dpo/margin": 65.0911865234375,
"fcm_dpo/q_t": 0.40847277641296387,
"grad_norm": 14.004586219787598,
"learning_rate": 5.267213693697695e-08,
"logits/chosen": 0.8198153972625732,
"logits/rejected": 0.7191529273986816,
"logps/chosen": -200.858642578125,
"logps/ref_chosen": -54.248619079589844,
"logps/ref_rejected": -94.94343566894531,
"logps/rejected": -306.6446533203125,
"loss": 1.1182,
"margin_dpo/margin_mean": 65.09120178222656,
"margin_dpo/margin_std": 103.08438110351562,
"step": 537
},
{
"KL/chosen_KL_mean": -142.38180541992188,
"KL/mean": -177.9237060546875,
"KL/rejected_KL_mean": -213.4656219482422,
"KL/std": 92.63967895507812,
"epoch": 0.8133030990173847,
"fcm_dpo/beta": 0.006257187575101852,
"fcm_dpo/delta": -0.04689842462539673,
"fcm_dpo/margin": 71.08383178710938,
"fcm_dpo/q_t": 0.3982582092285156,
"grad_norm": 13.602986335754395,
"learning_rate": 5.1863067244167144e-08,
"logits/chosen": 0.7027615308761597,
"logits/rejected": 0.6736000180244446,
"logps/chosen": -212.475341796875,
"logps/ref_chosen": -70.09353637695312,
"logps/ref_rejected": -79.49833679199219,
"logps/rejected": -292.9639587402344,
"loss": 1.0695,
"margin_dpo/margin_mean": 71.08382415771484,
"margin_dpo/margin_std": 93.18782806396484,
"step": 538
},
{
"KL/chosen_KL_mean": -152.43002319335938,
"KL/mean": -179.57598876953125,
"KL/rejected_KL_mean": -206.72195434570312,
"KL/std": 89.647705078125,
"epoch": 0.8148148148148148,
"fcm_dpo/beta": 0.006267036311328411,
"fcm_dpo/delta": 0.06188402697443962,
"fcm_dpo/margin": 54.29193115234375,
"fcm_dpo/q_t": 0.4240303635597229,
"grad_norm": 15.145447731018066,
"learning_rate": 5.105953986729195e-08,
"logits/chosen": 0.6562758684158325,
"logits/rejected": 0.5707495212554932,
"logps/chosen": -214.36172485351562,
"logps/ref_chosen": -61.93169403076172,
"logps/ref_rejected": -84.08946228027344,
"logps/rejected": -290.8114013671875,
"loss": 1.1588,
"margin_dpo/margin_mean": 54.29193115234375,
"margin_dpo/margin_std": 95.77870178222656,
"step": 539
},
{
"KL/chosen_KL_mean": -137.79824829101562,
"KL/mean": -178.1307373046875,
"KL/rejected_KL_mean": -218.4632110595703,
"KL/std": 99.49797821044922,
"epoch": 0.8163265306122449,
"fcm_dpo/beta": 0.0062315561808645725,
"fcm_dpo/delta": -0.10805132985115051,
"fcm_dpo/margin": 80.66496276855469,
"fcm_dpo/q_t": 0.38502955436706543,
"grad_norm": 12.357481002807617,
"learning_rate": 5.026157728273966e-08,
"logits/chosen": 0.7628463506698608,
"logits/rejected": 0.6582174301147461,
"logps/chosen": -200.50250244140625,
"logps/ref_chosen": -62.704254150390625,
"logps/ref_rejected": -95.63597106933594,
"logps/rejected": -314.09918212890625,
"loss": 1.0282,
"margin_dpo/margin_mean": 80.66496276855469,
"margin_dpo/margin_std": 94.271484375,
"step": 540
},
{
"KL/chosen_KL_mean": -135.28585815429688,
"KL/mean": -170.78558349609375,
"KL/rejected_KL_mean": -206.2853240966797,
"KL/std": 91.22382354736328,
"epoch": 0.817838246409675,
"fcm_dpo/beta": 0.006104937754571438,
"fcm_dpo/delta": -0.0356261283159256,
"fcm_dpo/margin": 70.99945068359375,
"fcm_dpo/q_t": 0.3997143805027008,
"grad_norm": 12.955300331115723,
"learning_rate": 4.9469201811239035e-08,
"logits/chosen": 0.7362730503082275,
"logits/rejected": 0.7617666721343994,
"logps/chosen": -197.76670837402344,
"logps/ref_chosen": -62.48084259033203,
"logps/ref_rejected": -57.55541229248047,
"logps/rejected": -263.8407287597656,
"loss": 1.0741,
"margin_dpo/margin_mean": 70.99945068359375,
"margin_dpo/margin_std": 91.00010681152344,
"step": 541
},
{
"KL/chosen_KL_mean": -118.09880065917969,
"KL/mean": -157.98910522460938,
"KL/rejected_KL_mean": -197.87942504882812,
"KL/std": 88.76216125488281,
"epoch": 0.8193499622071051,
"fcm_dpo/beta": 0.00603675888851285,
"fcm_dpo/delta": -0.08592377603054047,
"fcm_dpo/margin": 79.7806396484375,
"fcm_dpo/q_t": 0.3898007869720459,
"grad_norm": 13.403088569641113,
"learning_rate": 4.868243561723534e-08,
"logits/chosen": 0.7562509775161743,
"logits/rejected": 0.7051761150360107,
"logps/chosen": -167.55368041992188,
"logps/ref_chosen": -49.454891204833984,
"logps/ref_rejected": -65.33275604248047,
"logps/rejected": -263.2121887207031,
"loss": 1.0615,
"margin_dpo/margin_mean": 79.7806396484375,
"margin_dpo/margin_std": 108.47217559814453,
"step": 542
},
{
"KL/chosen_KL_mean": -126.40426635742188,
"KL/mean": -163.99774169921875,
"KL/rejected_KL_mean": -201.59120178222656,
"KL/std": 87.73787689208984,
"epoch": 0.8208616780045351,
"fcm_dpo/beta": 0.00599122978746891,
"fcm_dpo/delta": -0.05288073793053627,
"fcm_dpo/margin": 75.18693542480469,
"fcm_dpo/q_t": 0.3956128656864166,
"grad_norm": 11.480177879333496,
"learning_rate": 4.790130070827028e-08,
"logits/chosen": 0.7083995342254639,
"logits/rejected": 0.617012619972229,
"logps/chosen": -177.505126953125,
"logps/ref_chosen": -51.100860595703125,
"logps/ref_rejected": -76.06130981445312,
"logps/rejected": -277.65252685546875,
"loss": 1.0669,
"margin_dpo/margin_mean": 75.18693542480469,
"margin_dpo/margin_std": 96.68072509765625,
"step": 543
},
{
"KL/chosen_KL_mean": -139.19317626953125,
"KL/mean": -179.93508911132812,
"KL/rejected_KL_mean": -220.67697143554688,
"KL/std": 94.99725341796875,
"epoch": 0.8223733938019653,
"fcm_dpo/beta": 0.005886511877179146,
"fcm_dpo/delta": -0.08362063020467758,
"fcm_dpo/margin": 81.48379516601562,
"fcm_dpo/q_t": 0.39101773500442505,
"grad_norm": 14.965998649597168,
"learning_rate": 4.7125818934366454e-08,
"logits/chosen": 0.6980470418930054,
"logits/rejected": 0.6164635419845581,
"logps/chosen": -199.4704132080078,
"logps/ref_chosen": -60.2772331237793,
"logps/ref_rejected": -88.40553283691406,
"logps/rejected": -309.08251953125,
"loss": 1.0596,
"margin_dpo/margin_mean": 81.48379516601562,
"margin_dpo/margin_std": 109.55349731445312,
"step": 544
},
{
"KL/chosen_KL_mean": -148.39352416992188,
"KL/mean": -173.28126525878906,
"KL/rejected_KL_mean": -198.16900634765625,
"KL/std": 90.24813842773438,
"epoch": 0.8238851095993953,
"fcm_dpo/beta": 0.005952928215265274,
"fcm_dpo/delta": 0.1068287193775177,
"fcm_dpo/margin": 49.775474548339844,
"fcm_dpo/q_t": 0.432598739862442,
"grad_norm": 13.728639602661133,
"learning_rate": 4.635601198741607e-08,
"logits/chosen": 0.6579852104187012,
"logits/rejected": 0.5989206433296204,
"logps/chosen": -210.00877380371094,
"logps/ref_chosen": -61.61524963378906,
"logps/ref_rejected": -78.71266174316406,
"logps/rejected": -276.88165283203125,
"loss": 1.1986,
"margin_dpo/margin_mean": 49.775474548339844,
"margin_dpo/margin_std": 101.11408996582031,
"step": 545
},
{
"KL/chosen_KL_mean": -141.760986328125,
"KL/mean": -169.93963623046875,
"KL/rejected_KL_mean": -198.11827087402344,
"KL/std": 87.8583984375,
"epoch": 0.8253968253968254,
"fcm_dpo/beta": 0.006028347183018923,
"fcm_dpo/delta": 0.06242326647043228,
"fcm_dpo/margin": 56.357269287109375,
"fcm_dpo/q_t": 0.4220507740974426,
"grad_norm": 15.243754386901855,
"learning_rate": 4.559190140057428e-08,
"logits/chosen": 0.7536579370498657,
"logits/rejected": 0.7456855177879333,
"logps/chosen": -201.07424926757812,
"logps/ref_chosen": -59.313262939453125,
"logps/ref_rejected": -64.73631286621094,
"logps/rejected": -262.8545837402344,
"loss": 1.1746,
"margin_dpo/margin_mean": 56.357269287109375,
"margin_dpo/margin_std": 107.15248107910156,
"step": 546
},
{
"KL/chosen_KL_mean": -125.16900634765625,
"KL/mean": -163.8048095703125,
"KL/rejected_KL_mean": -202.44061279296875,
"KL/std": 91.23747253417969,
"epoch": 0.8269085411942555,
"fcm_dpo/beta": 0.005984361283481121,
"fcm_dpo/delta": -0.06575603783130646,
"fcm_dpo/margin": 77.2716064453125,
"fcm_dpo/q_t": 0.39354628324508667,
"grad_norm": 14.384458541870117,
"learning_rate": 4.483350854765672e-08,
"logits/chosen": 0.6743725538253784,
"logits/rejected": 0.6041334271430969,
"logps/chosen": -180.145751953125,
"logps/ref_chosen": -54.97674560546875,
"logps/ref_rejected": -75.35922241210938,
"logps/rejected": -277.7998352050781,
"loss": 1.066,
"margin_dpo/margin_mean": 77.2716064453125,
"margin_dpo/margin_std": 103.70457458496094,
"step": 547
},
{
"KL/chosen_KL_mean": -145.03651428222656,
"KL/mean": -170.27798461914062,
"KL/rejected_KL_mean": -195.51943969726562,
"KL/std": 91.18202209472656,
"epoch": 0.8284202569916855,
"fcm_dpo/beta": 0.006083798129111528,
"fcm_dpo/delta": 0.09534087777137756,
"fcm_dpo/margin": 50.48291778564453,
"fcm_dpo/q_t": 0.4305972754955292,
"grad_norm": 16.008787155151367,
"learning_rate": 4.4080854642541826e-08,
"logits/chosen": 0.6519588232040405,
"logits/rejected": 0.5856359601020813,
"logps/chosen": -208.2471923828125,
"logps/ref_chosen": -63.21067428588867,
"logps/ref_rejected": -81.23347473144531,
"logps/rejected": -276.7529296875,
"loss": 1.1872,
"margin_dpo/margin_mean": 50.48291778564453,
"margin_dpo/margin_std": 97.80047607421875,
"step": 548
},
{
"KL/chosen_KL_mean": -141.17372131347656,
"KL/mean": -170.97410583496094,
"KL/rejected_KL_mean": -200.7744903564453,
"KL/std": 93.58125305175781,
"epoch": 0.8299319727891157,
"fcm_dpo/beta": 0.00611619558185339,
"fcm_dpo/delta": 0.03680401295423508,
"fcm_dpo/margin": 59.60077667236328,
"fcm_dpo/q_t": 0.41750288009643555,
"grad_norm": 15.969023704528809,
"learning_rate": 4.333396073857723e-08,
"logits/chosen": 0.7709946036338806,
"logits/rejected": 0.697953462600708,
"logps/chosen": -205.44723510742188,
"logps/ref_chosen": -64.27351379394531,
"logps/ref_rejected": -92.31663513183594,
"logps/rejected": -293.09112548828125,
"loss": 1.1602,
"margin_dpo/margin_mean": 59.60077667236328,
"margin_dpo/margin_std": 109.434814453125,
"step": 549
},
{
"KL/chosen_KL_mean": -151.7022247314453,
"KL/mean": -171.4727783203125,
"KL/rejected_KL_mean": -191.24331665039062,
"KL/std": 88.01777648925781,
"epoch": 0.8314436885865457,
"fcm_dpo/beta": 0.006176707334816456,
"fcm_dpo/delta": 0.0298943929374218,
"fcm_dpo/margin": 39.54109191894531,
"fcm_dpo/q_t": 0.44400495290756226,
"grad_norm": 16.774959564208984,
"learning_rate": 4.259284772799099e-08,
"logits/chosen": 0.7350375056266785,
"logits/rejected": 0.7036670446395874,
"logps/chosen": -207.9326629638672,
"logps/ref_chosen": -56.230438232421875,
"logps/ref_rejected": -62.59788513183594,
"logps/rejected": -253.84120178222656,
"loss": 1.2431,
"margin_dpo/margin_mean": 39.54109191894531,
"margin_dpo/margin_std": 95.86380004882812,
"step": 550
},
{
"KL/chosen_KL_mean": -151.6258087158203,
"KL/mean": -176.9363555908203,
"KL/rejected_KL_mean": -202.24688720703125,
"KL/std": 93.55429077148438,
"epoch": 0.8329554043839759,
"fcm_dpo/beta": 0.0062708547338843346,
"fcm_dpo/delta": 0.08493860065937042,
"fcm_dpo/margin": 50.62107849121094,
"fcm_dpo/q_t": 0.4290255308151245,
"grad_norm": 14.150769233703613,
"learning_rate": 4.1857536341307176e-08,
"logits/chosen": 0.7145811319351196,
"logits/rejected": 0.6786512136459351,
"logps/chosen": -219.37301635742188,
"logps/ref_chosen": -67.74720764160156,
"logps/ref_rejected": -87.04285430908203,
"logps/rejected": -289.28973388671875,
"loss": 1.1677,
"margin_dpo/margin_mean": 50.62107849121094,
"margin_dpo/margin_std": 90.7059326171875,
"step": 551
},
{
"KL/chosen_KL_mean": -140.75294494628906,
"KL/mean": -171.14222717285156,
"KL/rejected_KL_mean": -201.53152465820312,
"KL/std": 93.18401336669922,
"epoch": 0.8344671201814059,
"fcm_dpo/beta": 0.006325121037662029,
"fcm_dpo/delta": 0.01572517678141594,
"fcm_dpo/margin": 60.77858352661133,
"fcm_dpo/q_t": 0.4094967544078827,
"grad_norm": 15.544822692871094,
"learning_rate": 4.112804714676593e-08,
"logits/chosen": 0.7034376859664917,
"logits/rejected": 0.6460795402526855,
"logps/chosen": -203.67919921875,
"logps/ref_chosen": -62.92625427246094,
"logps/ref_rejected": -82.98365783691406,
"logps/rejected": -284.5151672363281,
"loss": 1.1146,
"margin_dpo/margin_mean": 60.77858352661133,
"margin_dpo/margin_std": 88.78265380859375,
"step": 552
},
{
"KL/chosen_KL_mean": -152.1892547607422,
"KL/mean": -181.77285766601562,
"KL/rejected_KL_mean": -211.35647583007812,
"KL/std": 89.39266967773438,
"epoch": 0.8359788359788359,
"fcm_dpo/beta": 0.006329827010631561,
"fcm_dpo/delta": 0.02645890787243843,
"fcm_dpo/margin": 59.16722106933594,
"fcm_dpo/q_t": 0.4165544807910919,
"grad_norm": 16.954708099365234,
"learning_rate": 4.0404400549748144e-08,
"logits/chosen": 0.6913542747497559,
"logits/rejected": 0.5800439119338989,
"logps/chosen": -208.22775268554688,
"logps/ref_chosen": -56.038490295410156,
"logps/ref_rejected": -84.48454284667969,
"logps/rejected": -295.84100341796875,
"loss": 1.1554,
"margin_dpo/margin_mean": 59.167213439941406,
"margin_dpo/margin_std": 108.44562530517578,
"step": 553
},
{
"KL/chosen_KL_mean": -137.18270874023438,
"KL/mean": -172.22811889648438,
"KL/rejected_KL_mean": -207.2735595703125,
"KL/std": 89.01100158691406,
"epoch": 0.8374905517762661,
"fcm_dpo/beta": 0.006297202780842781,
"fcm_dpo/delta": -0.04327443614602089,
"fcm_dpo/margin": 70.09081268310547,
"fcm_dpo/q_t": 0.39865726232528687,
"grad_norm": 13.708428382873535,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 0.6766440272331238,
"logits/rejected": 0.6580997705459595,
"logps/chosen": -201.71331787109375,
"logps/ref_chosen": -64.53059387207031,
"logps/ref_rejected": -71.2155990600586,
"logps/rejected": -278.4891357421875,
"loss": 1.0844,
"margin_dpo/margin_mean": 70.0908203125,
"margin_dpo/margin_std": 99.07826232910156,
"step": 554
},
{
"KL/chosen_KL_mean": -150.2454833984375,
"KL/mean": -179.2886199951172,
"KL/rejected_KL_mean": -208.33172607421875,
"KL/std": 89.19993591308594,
"epoch": 0.8390022675736961,
"fcm_dpo/beta": 0.00635831244289875,
"fcm_dpo/delta": 0.030418243259191513,
"fcm_dpo/margin": 58.08625030517578,
"fcm_dpo/q_t": 0.4144817590713501,
"grad_norm": 15.325748443603516,
"learning_rate": 3.89747159520904e-08,
"logits/chosen": 0.7089887857437134,
"logits/rejected": 0.6819032430648804,
"logps/chosen": -216.89739990234375,
"logps/ref_chosen": -66.65191650390625,
"logps/ref_rejected": -68.6667251586914,
"logps/rejected": -276.99847412109375,
"loss": 1.168,
"margin_dpo/margin_mean": 58.08625030517578,
"margin_dpo/margin_std": 106.99815368652344,
"step": 555
},
{
"KL/chosen_KL_mean": -150.76223754882812,
"KL/mean": -176.39547729492188,
"KL/rejected_KL_mean": -202.02871704101562,
"KL/std": 93.75662994384766,
"epoch": 0.8405139833711263,
"fcm_dpo/beta": 0.0063774073496460915,
"fcm_dpo/delta": 0.07559022307395935,
"fcm_dpo/margin": 51.26647186279297,
"fcm_dpo/q_t": 0.42761844396591187,
"grad_norm": 13.667985916137695,
"learning_rate": 3.826871794280192e-08,
"logits/chosen": 0.7677052021026611,
"logits/rejected": 0.7135956287384033,
"logps/chosen": -203.5946044921875,
"logps/ref_chosen": -52.832366943359375,
"logps/ref_rejected": -64.49044036865234,
"logps/rejected": -266.5191650390625,
"loss": 1.2009,
"margin_dpo/margin_mean": 51.2664680480957,
"margin_dpo/margin_std": 108.83920288085938,
"step": 556
},
{
"KL/chosen_KL_mean": -147.9571990966797,
"KL/mean": -187.36276245117188,
"KL/rejected_KL_mean": -226.768310546875,
"KL/std": 90.80734252929688,
"epoch": 0.8420256991685563,
"fcm_dpo/beta": 0.006293997168540955,
"fcm_dpo/delta": -0.10179068893194199,
"fcm_dpo/margin": 78.81112670898438,
"fcm_dpo/q_t": 0.3864714503288269,
"grad_norm": 12.005192756652832,
"learning_rate": 3.756864251262143e-08,
"logits/chosen": 0.8267861604690552,
"logits/rejected": 0.7442450523376465,
"logps/chosen": -202.99319458007812,
"logps/ref_chosen": -55.03598403930664,
"logps/ref_rejected": -75.80644989013672,
"logps/rejected": -302.57476806640625,
"loss": 1.0275,
"margin_dpo/margin_mean": 78.81112670898438,
"margin_dpo/margin_std": 90.235107421875,
"step": 557
},
{
"KL/chosen_KL_mean": -135.27023315429688,
"KL/mean": -177.4205322265625,
"KL/rejected_KL_mean": -219.5708465576172,
"KL/std": 97.51036071777344,
"epoch": 0.8435374149659864,
"fcm_dpo/beta": 0.006145142950117588,
"fcm_dpo/delta": -0.12534838914871216,
"fcm_dpo/margin": 84.30059814453125,
"fcm_dpo/q_t": 0.38268476724624634,
"grad_norm": 11.195854187011719,
"learning_rate": 3.687450924416341e-08,
"logits/chosen": 0.762208104133606,
"logits/rejected": 0.704133152961731,
"logps/chosen": -198.49658203125,
"logps/ref_chosen": -63.226348876953125,
"logps/ref_rejected": -91.46881866455078,
"logps/rejected": -311.0396728515625,
"loss": 1.0224,
"margin_dpo/margin_mean": 84.30059814453125,
"margin_dpo/margin_std": 100.35435485839844,
"step": 558
},
{
"KL/chosen_KL_mean": -139.7186279296875,
"KL/mean": -176.25946044921875,
"KL/rejected_KL_mean": -212.80027770996094,
"KL/std": 93.9178695678711,
"epoch": 0.8450491307634165,
"fcm_dpo/beta": 0.006047483533620834,
"fcm_dpo/delta": -0.04520851746201515,
"fcm_dpo/margin": 73.0816650390625,
"fcm_dpo/q_t": 0.4024829566478729,
"grad_norm": 12.266596794128418,
"learning_rate": 3.6186337553827743e-08,
"logits/chosen": 0.6904716491699219,
"logits/rejected": 0.620003342628479,
"logps/chosen": -201.24026489257812,
"logps/ref_chosen": -61.521644592285156,
"logps/ref_rejected": -82.83859252929688,
"logps/rejected": -295.63885498046875,
"loss": 1.0997,
"margin_dpo/margin_mean": 73.0816650390625,
"margin_dpo/margin_std": 111.09150695800781,
"step": 559
},
{
"KL/chosen_KL_mean": -153.00474548339844,
"KL/mean": -185.1983642578125,
"KL/rejected_KL_mean": -217.39199829101562,
"KL/std": 92.76141357421875,
"epoch": 0.8465608465608465,
"fcm_dpo/beta": 0.006110331043601036,
"fcm_dpo/delta": 0.006353672593832016,
"fcm_dpo/margin": 64.38724517822266,
"fcm_dpo/q_t": 0.40973961353302,
"grad_norm": 15.809849739074707,
"learning_rate": 3.550414669125573e-08,
"logits/chosen": 0.7126524448394775,
"logits/rejected": 0.6708424687385559,
"logps/chosen": -213.64596557617188,
"logps/ref_chosen": -60.64122009277344,
"logps/ref_rejected": -78.75474548339844,
"logps/rejected": -296.146728515625,
"loss": 1.1064,
"margin_dpo/margin_mean": 64.38723754882812,
"margin_dpo/margin_std": 90.55433654785156,
"step": 560
},
{
"KL/chosen_KL_mean": -138.85739135742188,
"KL/mean": -170.60830688476562,
"KL/rejected_KL_mean": -202.3592529296875,
"KL/std": 92.10637664794922,
"epoch": 0.8480725623582767,
"fcm_dpo/beta": 0.006088586524128914,
"fcm_dpo/delta": 0.01388754602521658,
"fcm_dpo/margin": 63.501888275146484,
"fcm_dpo/q_t": 0.41299164295196533,
"grad_norm": 13.58033561706543,
"learning_rate": 3.482795573879241e-08,
"logits/chosen": 0.6898171305656433,
"logits/rejected": 0.6543152332305908,
"logps/chosen": -201.35598754882812,
"logps/ref_chosen": -62.49859619140625,
"logps/ref_rejected": -78.72064208984375,
"logps/rejected": -281.07989501953125,
"loss": 1.1245,
"margin_dpo/margin_mean": 63.50188446044922,
"margin_dpo/margin_std": 100.05314636230469,
"step": 561
},
{
"KL/chosen_KL_mean": -147.63851928710938,
"KL/mean": -187.5120086669922,
"KL/rejected_KL_mean": -227.38551330566406,
"KL/std": 100.77043914794922,
"epoch": 0.8495842781557067,
"fcm_dpo/beta": 0.005991585087031126,
"fcm_dpo/delta": -0.08283955603837967,
"fcm_dpo/margin": 79.74699401855469,
"fcm_dpo/q_t": 0.39247214794158936,
"grad_norm": 15.322938919067383,
"learning_rate": 3.415778361095226e-08,
"logits/chosen": 0.7405972480773926,
"logits/rejected": 0.6973186731338501,
"logps/chosen": -222.42025756835938,
"logps/ref_chosen": -74.78173828125,
"logps/ref_rejected": -92.63499450683594,
"logps/rejected": -320.0205078125,
"loss": 1.0535,
"margin_dpo/margin_mean": 79.74699401855469,
"margin_dpo/margin_std": 102.18318176269531,
"step": 562
},
{
"KL/chosen_KL_mean": -125.26051330566406,
"KL/mean": -160.538330078125,
"KL/rejected_KL_mean": -195.816162109375,
"KL/std": 84.16582489013672,
"epoch": 0.8510959939531368,
"fcm_dpo/beta": 0.00598212331533432,
"fcm_dpo/delta": -0.02303909696638584,
"fcm_dpo/margin": 70.55564880371094,
"fcm_dpo/q_t": 0.40320104360580444,
"grad_norm": 18.65880584716797,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.8295519351959229,
"logits/rejected": 0.7749502658843994,
"logps/chosen": -175.45901489257812,
"logps/ref_chosen": -50.19850158691406,
"logps/ref_rejected": -66.76687622070312,
"logps/rejected": -262.5830383300781,
"loss": 1.1073,
"margin_dpo/margin_mean": 70.55564880371094,
"margin_dpo/margin_std": 108.36442565917969,
"step": 563
},
{
"KL/chosen_KL_mean": -127.6347427368164,
"KL/mean": -171.83145141601562,
"KL/rejected_KL_mean": -216.02816772460938,
"KL/std": 91.53362274169922,
"epoch": 0.8526077097505669,
"fcm_dpo/beta": 0.005859338212758303,
"fcm_dpo/delta": -0.12445573508739471,
"fcm_dpo/margin": 88.39341735839844,
"fcm_dpo/q_t": 0.3805384933948517,
"grad_norm": 13.179845809936523,
"learning_rate": 3.283557064487785e-08,
"logits/chosen": 0.7013846039772034,
"logits/rejected": 0.6674783229827881,
"logps/chosen": -183.37557983398438,
"logps/ref_chosen": -55.7408447265625,
"logps/ref_rejected": -74.82323455810547,
"logps/rejected": -290.85137939453125,
"loss": 1.0305,
"margin_dpo/margin_mean": 88.39341735839844,
"margin_dpo/margin_std": 109.67872619628906,
"step": 564
},
{
"KL/chosen_KL_mean": -156.1749725341797,
"KL/mean": -186.40829467773438,
"KL/rejected_KL_mean": -216.64163208007812,
"KL/std": 91.30490112304688,
"epoch": 0.854119425547997,
"fcm_dpo/beta": 0.0058922963216900826,
"fcm_dpo/delta": 0.0446639247238636,
"fcm_dpo/margin": 60.46666717529297,
"fcm_dpo/q_t": 0.4178019165992737,
"grad_norm": 14.750682830810547,
"learning_rate": 3.218356679178252e-08,
"logits/chosen": 0.7720531225204468,
"logits/rejected": 0.7127261161804199,
"logps/chosen": -214.51235961914062,
"logps/ref_chosen": -58.33738327026367,
"logps/ref_rejected": -78.31776428222656,
"logps/rejected": -294.9593811035156,
"loss": 1.1339,
"margin_dpo/margin_mean": 60.46666717529297,
"margin_dpo/margin_std": 92.89695739746094,
"step": 565
},
{
"KL/chosen_KL_mean": -144.64797973632812,
"KL/mean": -173.0709991455078,
"KL/rejected_KL_mean": -201.4940185546875,
"KL/std": 94.91629028320312,
"epoch": 0.8556311413454271,
"fcm_dpo/beta": 0.005958449095487595,
"fcm_dpo/delta": 0.06280165165662766,
"fcm_dpo/margin": 56.84604263305664,
"fcm_dpo/q_t": 0.4245191514492035,
"grad_norm": 16.267759323120117,
"learning_rate": 3.1537655732553764e-08,
"logits/chosen": 0.6981167197227478,
"logits/rejected": 0.6822539567947388,
"logps/chosen": -215.87171936035156,
"logps/ref_chosen": -71.22373962402344,
"logps/ref_rejected": -71.11601257324219,
"logps/rejected": -272.61004638671875,
"loss": 1.197,
"margin_dpo/margin_mean": 56.846046447753906,
"margin_dpo/margin_std": 120.08654022216797,
"step": 566
},
{
"KL/chosen_KL_mean": -138.97036743164062,
"KL/mean": -175.221923828125,
"KL/rejected_KL_mean": -211.4734649658203,
"KL/std": 91.07780456542969,
"epoch": 0.8571428571428571,
"fcm_dpo/beta": 0.0058929030783474445,
"fcm_dpo/delta": -0.029290813952684402,
"fcm_dpo/margin": 72.50309753417969,
"fcm_dpo/q_t": 0.40125784277915955,
"grad_norm": 11.622398376464844,
"learning_rate": 3.089785553471233e-08,
"logits/chosen": 0.7191234827041626,
"logits/rejected": 0.6220579147338867,
"logps/chosen": -191.6396484375,
"logps/ref_chosen": -52.669273376464844,
"logps/ref_rejected": -74.34785461425781,
"logps/rejected": -285.8213195800781,
"loss": 1.0868,
"margin_dpo/margin_mean": 72.50308990478516,
"margin_dpo/margin_std": 98.64824676513672,
"step": 567
},
{
"KL/chosen_KL_mean": -126.99612426757812,
"KL/mean": -171.6866455078125,
"KL/rejected_KL_mean": -216.37718200683594,
"KL/std": 95.61448669433594,
"epoch": 0.8586545729402872,
"fcm_dpo/beta": 0.005839211866259575,
"fcm_dpo/delta": -0.1285656988620758,
"fcm_dpo/margin": 89.38103485107422,
"fcm_dpo/q_t": 0.38138020038604736,
"grad_norm": 15.177702903747559,
"learning_rate": 3.026418409484513e-08,
"logits/chosen": 0.7849606275558472,
"logits/rejected": 0.6944478154182434,
"logps/chosen": -179.17413330078125,
"logps/ref_chosen": -52.178001403808594,
"logps/ref_rejected": -85.8277587890625,
"logps/rejected": -302.2049560546875,
"loss": 1.0092,
"margin_dpo/margin_mean": 89.38103485107422,
"margin_dpo/margin_std": 96.79595947265625,
"step": 568
},
{
"KL/chosen_KL_mean": -148.28836059570312,
"KL/mean": -170.65982055664062,
"KL/rejected_KL_mean": -193.03125,
"KL/std": 93.69496154785156,
"epoch": 0.8601662887377173,
"fcm_dpo/beta": 0.005767214577645063,
"fcm_dpo/delta": -5.881537163077155e-06,
"fcm_dpo/margin": 44.742897033691406,
"fcm_dpo/q_t": 0.4403781294822693,
"grad_norm": 14.973525047302246,
"learning_rate": 2.963665913810451e-08,
"logits/chosen": 0.6696399450302124,
"logits/rejected": 0.6333480477333069,
"logps/chosen": -210.9376220703125,
"logps/ref_chosen": -62.649261474609375,
"logps/ref_rejected": -75.4298324584961,
"logps/rejected": -268.4610900878906,
"loss": 1.2213,
"margin_dpo/margin_mean": 44.742897033691406,
"margin_dpo/margin_std": 95.0347900390625,
"step": 569
},
{
"KL/chosen_KL_mean": -131.74794006347656,
"KL/mean": -175.54510498046875,
"KL/rejected_KL_mean": -219.34226989746094,
"KL/std": 89.6861343383789,
"epoch": 0.8616780045351474,
"fcm_dpo/beta": 0.0057052792981266975,
"fcm_dpo/delta": -0.10496269166469574,
"fcm_dpo/margin": 87.59432220458984,
"fcm_dpo/q_t": 0.38489830493927,
"grad_norm": 12.75351333618164,
"learning_rate": 2.9015298217712453e-08,
"logits/chosen": 0.6851919293403625,
"logits/rejected": 0.5994397401809692,
"logps/chosen": -181.78973388671875,
"logps/ref_chosen": -50.04179382324219,
"logps/ref_rejected": -78.27146911621094,
"logps/rejected": -297.6137390136719,
"loss": 1.0251,
"margin_dpo/margin_mean": 87.59432220458984,
"margin_dpo/margin_std": 99.58367919921875,
"step": 570
},
{
"KL/chosen_KL_mean": -143.87881469726562,
"KL/mean": -168.77749633789062,
"KL/rejected_KL_mean": -193.6761932373047,
"KL/std": 87.45454406738281,
"epoch": 0.8631897203325775,
"fcm_dpo/beta": 0.0056588901206851006,
"fcm_dpo/delta": 0.02028953842818737,
"fcm_dpo/margin": 49.79738235473633,
"fcm_dpo/q_t": 0.4340037703514099,
"grad_norm": 13.22017765045166,
"learning_rate": 2.840011871446962e-08,
"logits/chosen": 0.7288908958435059,
"logits/rejected": 0.6971858143806458,
"logps/chosen": -197.53561401367188,
"logps/ref_chosen": -53.65681457519531,
"logps/ref_rejected": -66.13298034667969,
"logps/rejected": -259.8091735839844,
"loss": 1.2075,
"margin_dpo/margin_mean": 49.79737854003906,
"margin_dpo/margin_std": 101.9254150390625,
"step": 571
},
{
"KL/chosen_KL_mean": -141.73880004882812,
"KL/mean": -170.89016723632812,
"KL/rejected_KL_mean": -200.04156494140625,
"KL/std": 92.01441955566406,
"epoch": 0.8647014361300076,
"fcm_dpo/beta": 0.005748718045651913,
"fcm_dpo/delta": 0.06653580814599991,
"fcm_dpo/margin": 58.30277633666992,
"fcm_dpo/q_t": 0.4222661852836609,
"grad_norm": 13.596394538879395,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 0.6820651292800903,
"logits/rejected": 0.7269065380096436,
"logps/chosen": -216.5567169189453,
"logps/ref_chosen": -74.81792449951172,
"logps/ref_rejected": -65.88681030273438,
"logps/rejected": -265.9283752441406,
"loss": 1.1481,
"margin_dpo/margin_mean": 58.30277633666992,
"margin_dpo/margin_std": 92.760009765625,
"step": 572
},
{
"KL/chosen_KL_mean": -156.14479064941406,
"KL/mean": -187.03018188476562,
"KL/rejected_KL_mean": -217.9155731201172,
"KL/std": 97.7538070678711,
"epoch": 0.8662131519274376,
"fcm_dpo/beta": 0.005783457309007645,
"fcm_dpo/delta": 0.044342391192913055,
"fcm_dpo/margin": 61.77076721191406,
"fcm_dpo/q_t": 0.41982901096343994,
"grad_norm": 14.023571014404297,
"learning_rate": 2.718837261761528e-08,
"logits/chosen": 0.7175908088684082,
"logits/rejected": 0.6679472923278809,
"logps/chosen": -224.87045288085938,
"logps/ref_chosen": -68.72564697265625,
"logps/ref_rejected": -88.16201782226562,
"logps/rejected": -306.07757568359375,
"loss": 1.1661,
"margin_dpo/margin_mean": 61.77076721191406,
"margin_dpo/margin_std": 116.53111267089844,
"step": 573
},
{
"KL/chosen_KL_mean": -139.59814453125,
"KL/mean": -178.59091186523438,
"KL/rejected_KL_mean": -217.58367919921875,
"KL/std": 89.26683044433594,
"epoch": 0.8677248677248677,
"fcm_dpo/beta": 0.0057814596220850945,
"fcm_dpo/delta": -0.05354148894548416,
"fcm_dpo/margin": 77.98553466796875,
"fcm_dpo/q_t": 0.3951573669910431,
"grad_norm": 11.619673728942871,
"learning_rate": 2.659183991914696e-08,
"logits/chosen": 0.7377680540084839,
"logits/rejected": 0.6681383848190308,
"logps/chosen": -195.91152954101562,
"logps/ref_chosen": -56.31340026855469,
"logps/ref_rejected": -83.91553497314453,
"logps/rejected": -301.49920654296875,
"loss": 1.0424,
"margin_dpo/margin_mean": 77.98553466796875,
"margin_dpo/margin_std": 83.24993133544922,
"step": 574
},
{
"KL/chosen_KL_mean": -143.32305908203125,
"KL/mean": -171.73501586914062,
"KL/rejected_KL_mean": -200.14697265625,
"KL/std": 93.6646957397461,
"epoch": 0.8692365835222978,
"fcm_dpo/beta": 0.005692525301128626,
"fcm_dpo/delta": -0.03783988952636719,
"fcm_dpo/margin": 56.823917388916016,
"fcm_dpo/q_t": 0.42775750160217285,
"grad_norm": 13.313933372497559,
"learning_rate": 2.600155642716606e-08,
"logits/chosen": 0.7542673349380493,
"logits/rejected": 0.6671475172042847,
"logps/chosen": -207.9071807861328,
"logps/ref_chosen": -64.5841293334961,
"logps/ref_rejected": -93.47034454345703,
"logps/rejected": -293.6173095703125,
"loss": 1.1958,
"margin_dpo/margin_mean": 56.82392501831055,
"margin_dpo/margin_std": 114.9543228149414,
"step": 575
},
{
"KL/chosen_KL_mean": -126.693359375,
"KL/mean": -166.816162109375,
"KL/rejected_KL_mean": -206.93899536132812,
"KL/std": 92.31230926513672,
"epoch": 0.8707482993197279,
"fcm_dpo/beta": 0.005613856017589569,
"fcm_dpo/delta": -0.05389907583594322,
"fcm_dpo/margin": 80.24562072753906,
"fcm_dpo/q_t": 0.39631304144859314,
"grad_norm": 12.78497314453125,
"learning_rate": 2.5417538653170754e-08,
"logits/chosen": 0.7384647130966187,
"logits/rejected": 0.6245558261871338,
"logps/chosen": -179.973876953125,
"logps/ref_chosen": -53.28052520751953,
"logps/ref_rejected": -84.2000503540039,
"logps/rejected": -291.1390380859375,
"loss": 1.0656,
"margin_dpo/margin_mean": 80.2456283569336,
"margin_dpo/margin_std": 102.20115661621094,
"step": 576
},
{
"KL/chosen_KL_mean": -142.0518798828125,
"KL/mean": -170.05519104003906,
"KL/rejected_KL_mean": -198.05853271484375,
"KL/std": 91.82646179199219,
"epoch": 0.872260015117158,
"fcm_dpo/beta": 0.005698447115719318,
"fcm_dpo/delta": 0.08356067538261414,
"fcm_dpo/margin": 56.00664520263672,
"fcm_dpo/q_t": 0.4262416660785675,
"grad_norm": 12.256202697753906,
"learning_rate": 2.4839802933393607e-08,
"logits/chosen": 0.6879914999008179,
"logits/rejected": 0.6748424768447876,
"logps/chosen": -204.37657165527344,
"logps/ref_chosen": -62.32468795776367,
"logps/ref_rejected": -67.300537109375,
"logps/rejected": -265.35906982421875,
"loss": 1.1764,
"margin_dpo/margin_mean": 56.00664520263672,
"margin_dpo/margin_std": 102.07671356201172,
"step": 577
},
{
"KL/chosen_KL_mean": -135.46859741210938,
"KL/mean": -162.23260498046875,
"KL/rejected_KL_mean": -188.99664306640625,
"KL/std": 93.74736022949219,
"epoch": 0.873771730914588,
"fcm_dpo/beta": 0.005800641141831875,
"fcm_dpo/delta": 0.09243927150964737,
"fcm_dpo/margin": 53.52804946899414,
"fcm_dpo/q_t": 0.4302714169025421,
"grad_norm": 15.295356750488281,
"learning_rate": 2.4268365428344733e-08,
"logits/chosen": 0.7952982187271118,
"logits/rejected": 0.7740713953971863,
"logps/chosen": -192.12417602539062,
"logps/ref_chosen": -56.65557861328125,
"logps/ref_rejected": -68.21835327148438,
"logps/rejected": -257.2149963378906,
"loss": 1.1927,
"margin_dpo/margin_mean": 53.52804946899414,
"margin_dpo/margin_std": 107.76667785644531,
"step": 578
},
{
"KL/chosen_KL_mean": -141.70584106445312,
"KL/mean": -180.70004272460938,
"KL/rejected_KL_mean": -219.6942596435547,
"KL/std": 88.67086029052734,
"epoch": 0.8752834467120182,
"fcm_dpo/beta": 0.005780298262834549,
"fcm_dpo/delta": -0.05328650772571564,
"fcm_dpo/margin": 77.98841094970703,
"fcm_dpo/q_t": 0.39455342292785645,
"grad_norm": 13.555575370788574,
"learning_rate": 2.3703242122359357e-08,
"logits/chosen": 0.6904971599578857,
"logits/rejected": 0.6646016836166382,
"logps/chosen": -198.5155029296875,
"logps/ref_chosen": -56.809661865234375,
"logps/ref_rejected": -68.09613037109375,
"logps/rejected": -287.7904052734375,
"loss": 1.051,
"margin_dpo/margin_mean": 77.98841094970703,
"margin_dpo/margin_std": 91.09934997558594,
"step": 579
},
{
"KL/chosen_KL_mean": -142.43580627441406,
"KL/mean": -174.19207763671875,
"KL/rejected_KL_mean": -205.9483642578125,
"KL/std": 98.74903106689453,
"epoch": 0.8767951625094482,
"fcm_dpo/beta": 0.005820404272526503,
"fcm_dpo/delta": 0.03107970394194126,
"fcm_dpo/margin": 63.51256561279297,
"fcm_dpo/q_t": 0.4168925881385803,
"grad_norm": 13.094014167785645,
"learning_rate": 2.3144448823151392e-08,
"logits/chosen": 0.7184457778930664,
"logits/rejected": 0.6576677560806274,
"logps/chosen": -200.13592529296875,
"logps/ref_chosen": -57.70011520385742,
"logps/ref_rejected": -77.90664672851562,
"logps/rejected": -283.8550109863281,
"loss": 1.152,
"margin_dpo/margin_mean": 63.51256561279297,
"margin_dpo/margin_std": 112.4635009765625,
"step": 580
},
{
"KL/chosen_KL_mean": -154.28033447265625,
"KL/mean": -186.59791564941406,
"KL/rejected_KL_mean": -218.91551208496094,
"KL/std": 90.763427734375,
"epoch": 0.8783068783068783,
"fcm_dpo/beta": 0.005809762980788946,
"fcm_dpo/delta": 0.025293543934822083,
"fcm_dpo/margin": 64.63516998291016,
"fcm_dpo/q_t": 0.4147687554359436,
"grad_norm": 13.4003267288208,
"learning_rate": 2.259200116137039e-08,
"logits/chosen": 0.7498993873596191,
"logits/rejected": 0.6785679459571838,
"logps/chosen": -213.61268615722656,
"logps/ref_chosen": -59.332359313964844,
"logps/ref_rejected": -83.64482116699219,
"logps/rejected": -302.5603332519531,
"loss": 1.1312,
"margin_dpo/margin_mean": 64.63516235351562,
"margin_dpo/margin_std": 103.0958480834961,
"step": 581
},
{
"KL/chosen_KL_mean": -139.30677795410156,
"KL/mean": -170.17782592773438,
"KL/rejected_KL_mean": -201.04888916015625,
"KL/std": 88.71572875976562,
"epoch": 0.8798185941043084,
"fcm_dpo/beta": 0.005882021971046925,
"fcm_dpo/delta": 0.03805776312947273,
"fcm_dpo/margin": 61.742095947265625,
"fcm_dpo/q_t": 0.417174756526947,
"grad_norm": 11.505105018615723,
"learning_rate": 2.204591459016525e-08,
"logits/chosen": 0.7074366807937622,
"logits/rejected": 0.7359520196914673,
"logps/chosen": -203.46963500976562,
"logps/ref_chosen": -64.16285705566406,
"logps/ref_rejected": -58.632896423339844,
"logps/rejected": -259.6817932128906,
"loss": 1.1399,
"margin_dpo/margin_mean": 61.742095947265625,
"margin_dpo/margin_std": 100.97396850585938,
"step": 582
},
{
"KL/chosen_KL_mean": -139.78890991210938,
"KL/mean": -176.47702026367188,
"KL/rejected_KL_mean": -213.1651611328125,
"KL/std": 96.55670166015625,
"epoch": 0.8813303099017384,
"fcm_dpo/beta": 0.005853170529007912,
"fcm_dpo/delta": -0.030815063044428825,
"fcm_dpo/margin": 73.37625122070312,
"fcm_dpo/q_t": 0.40209048986434937,
"grad_norm": 15.238081932067871,
"learning_rate": 2.1506204384751064e-08,
"logits/chosen": 0.7942938804626465,
"logits/rejected": 0.6823156476020813,
"logps/chosen": -191.6613006591797,
"logps/ref_chosen": -51.87239456176758,
"logps/ref_rejected": -83.86331176757812,
"logps/rejected": -297.0284729003906,
"loss": 1.106,
"margin_dpo/margin_mean": 73.37625122070312,
"margin_dpo/margin_std": 113.06787109375,
"step": 583
},
{
"KL/chosen_KL_mean": -128.03213500976562,
"KL/mean": -160.90284729003906,
"KL/rejected_KL_mean": -193.7735595703125,
"KL/std": 90.27666473388672,
"epoch": 0.8828420256991686,
"fcm_dpo/beta": 0.005838742479681969,
"fcm_dpo/delta": 0.01659194752573967,
"fcm_dpo/margin": 65.74142456054688,
"fcm_dpo/q_t": 0.4146254062652588,
"grad_norm": 12.90595531463623,
"learning_rate": 2.09728856419826e-08,
"logits/chosen": 0.8398407697677612,
"logits/rejected": 0.7276151180267334,
"logps/chosen": -174.603515625,
"logps/ref_chosen": -46.571388244628906,
"logps/ref_rejected": -80.67969512939453,
"logps/rejected": -274.4532470703125,
"loss": 1.1483,
"margin_dpo/margin_mean": 65.74142456054688,
"margin_dpo/margin_std": 116.6568832397461,
"step": 584
},
{
"KL/chosen_KL_mean": -149.2040252685547,
"KL/mean": -172.0621795654297,
"KL/rejected_KL_mean": -194.9203338623047,
"KL/std": 97.89974975585938,
"epoch": 0.8843537414965986,
"fcm_dpo/beta": 0.005882401019334793,
"fcm_dpo/delta": 0.030020244419574738,
"fcm_dpo/margin": 45.716304779052734,
"fcm_dpo/q_t": 0.4372592568397522,
"grad_norm": 12.105955123901367,
"learning_rate": 2.044597327993153e-08,
"logits/chosen": 0.6711918711662292,
"logits/rejected": 0.6189085841178894,
"logps/chosen": -207.32855224609375,
"logps/ref_chosen": -58.124534606933594,
"logps/ref_rejected": -79.00538635253906,
"logps/rejected": -273.92572021484375,
"loss": 1.2182,
"margin_dpo/margin_mean": 45.716304779052734,
"margin_dpo/margin_std": 99.43280792236328,
"step": 585
},
{
"KL/chosen_KL_mean": -138.46292114257812,
"KL/mean": -172.1058807373047,
"KL/rejected_KL_mean": -205.7488250732422,
"KL/std": 85.904541015625,
"epoch": 0.8858654572940288,
"fcm_dpo/beta": 0.005887184292078018,
"fcm_dpo/delta": 0.003889678046107292,
"fcm_dpo/margin": 67.28589630126953,
"fcm_dpo/q_t": 0.4077543020248413,
"grad_norm": 15.52278995513916,
"learning_rate": 1.9925482037469187e-08,
"logits/chosen": 0.7429731488227844,
"logits/rejected": 0.6948248147964478,
"logps/chosen": -192.56455993652344,
"logps/ref_chosen": -54.10163879394531,
"logps/ref_rejected": -63.72113037109375,
"logps/rejected": -269.469970703125,
"loss": 1.0794,
"margin_dpo/margin_mean": 67.28589630126953,
"margin_dpo/margin_std": 78.29402160644531,
"step": 586
},
{
"KL/chosen_KL_mean": -142.7909698486328,
"KL/mean": -175.87924194335938,
"KL/rejected_KL_mean": -208.96749877929688,
"KL/std": 96.17425537109375,
"epoch": 0.8873771730914588,
"fcm_dpo/beta": 0.0059089576825499535,
"fcm_dpo/delta": 0.009327705949544907,
"fcm_dpo/margin": 66.17654418945312,
"fcm_dpo/q_t": 0.40794771909713745,
"grad_norm": 14.153724670410156,
"learning_rate": 1.9411426473854687e-08,
"logits/chosen": 0.7969297170639038,
"logits/rejected": 0.7915176153182983,
"logps/chosen": -206.20816040039062,
"logps/ref_chosen": -63.41719436645508,
"logps/ref_rejected": -63.47003936767578,
"logps/rejected": -272.43756103515625,
"loss": 1.16,
"margin_dpo/margin_mean": 66.17654418945312,
"margin_dpo/margin_std": 126.8377685546875,
"step": 587
},
{
"KL/chosen_KL_mean": -144.60537719726562,
"KL/mean": -182.1134796142578,
"KL/rejected_KL_mean": -219.62156677246094,
"KL/std": 95.54666137695312,
"epoch": 0.8888888888888888,
"fcm_dpo/beta": 0.005865715444087982,
"fcm_dpo/delta": -0.042059894651174545,
"fcm_dpo/margin": 75.01617431640625,
"fcm_dpo/q_t": 0.4011402726173401,
"grad_norm": 15.069059371948242,
"learning_rate": 1.890382096832699e-08,
"logits/chosen": 0.7743512392044067,
"logits/rejected": 0.7286670207977295,
"logps/chosen": -206.80642700195312,
"logps/ref_chosen": -62.20103454589844,
"logps/ref_rejected": -82.10249328613281,
"logps/rejected": -301.72406005859375,
"loss": 1.098,
"margin_dpo/margin_mean": 75.01617431640625,
"margin_dpo/margin_std": 114.60283660888672,
"step": 588
},
{
"KL/chosen_KL_mean": -136.7216796875,
"KL/mean": -173.08914184570312,
"KL/rejected_KL_mean": -209.45663452148438,
"KL/std": 92.73431396484375,
"epoch": 0.890400604686319,
"fcm_dpo/beta": 0.00585212605074048,
"fcm_dpo/delta": -0.026804056018590927,
"fcm_dpo/margin": 72.7349624633789,
"fcm_dpo/q_t": 0.4006243050098419,
"grad_norm": 11.532814025878906,
"learning_rate": 1.840267971970344e-08,
"logits/chosen": 0.6853584051132202,
"logits/rejected": 0.6556499600410461,
"logps/chosen": -193.43528747558594,
"logps/ref_chosen": -56.71361541748047,
"logps/ref_rejected": -76.7366943359375,
"logps/rejected": -286.19329833984375,
"loss": 1.0633,
"margin_dpo/margin_mean": 72.73495483398438,
"margin_dpo/margin_std": 84.58289337158203,
"step": 589
},
{
"KL/chosen_KL_mean": -151.93881225585938,
"KL/mean": -186.46914672851562,
"KL/rejected_KL_mean": -220.99948120117188,
"KL/std": 87.15362548828125,
"epoch": 0.891912320483749,
"fcm_dpo/beta": 0.005848293658345938,
"fcm_dpo/delta": -0.004214761778712273,
"fcm_dpo/margin": 69.0606689453125,
"fcm_dpo/q_t": 0.4069703221321106,
"grad_norm": 14.805048942565918,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 0.6506938934326172,
"logits/rejected": 0.617667555809021,
"logps/chosen": -218.45263671875,
"logps/ref_chosen": -66.5138168334961,
"logps/ref_rejected": -85.70820617675781,
"logps/rejected": -306.70770263671875,
"loss": 1.0983,
"margin_dpo/margin_mean": 69.0606689453125,
"margin_dpo/margin_std": 95.96292114257812,
"step": 590
},
{
"KL/chosen_KL_mean": -128.74978637695312,
"KL/mean": -171.23690795898438,
"KL/rejected_KL_mean": -213.7239990234375,
"KL/std": 100.29086303710938,
"epoch": 0.8934240362811792,
"fcm_dpo/beta": 0.005709193646907806,
"fcm_dpo/delta": -0.09125015884637833,
"fcm_dpo/margin": 84.97421264648438,
"fcm_dpo/q_t": 0.39115890860557556,
"grad_norm": 14.836484909057617,
"learning_rate": 1.7419845883949098e-08,
"logits/chosen": 0.8128637075424194,
"logits/rejected": 0.7449182868003845,
"logps/chosen": -189.4469757080078,
"logps/ref_chosen": -60.697181701660156,
"logps/ref_rejected": -86.12278747558594,
"logps/rejected": -299.8468017578125,
"loss": 1.0764,
"margin_dpo/margin_mean": 84.97421264648438,
"margin_dpo/margin_std": 123.05726623535156,
"step": 591
},
{
"KL/chosen_KL_mean": -141.21078491210938,
"KL/mean": -172.82798767089844,
"KL/rejected_KL_mean": -204.4451904296875,
"KL/std": 87.5164794921875,
"epoch": 0.8949357520786092,
"fcm_dpo/beta": 0.005729802884161472,
"fcm_dpo/delta": 0.03880191594362259,
"fcm_dpo/margin": 63.23441696166992,
"fcm_dpo/q_t": 0.4184558689594269,
"grad_norm": 14.47271728515625,
"learning_rate": 1.6938180788793556e-08,
"logits/chosen": 0.7759917974472046,
"logits/rejected": 0.6580488681793213,
"logps/chosen": -192.44810485839844,
"logps/ref_chosen": -51.237327575683594,
"logps/ref_rejected": -81.60242462158203,
"logps/rejected": -286.047607421875,
"loss": 1.1331,
"margin_dpo/margin_mean": 63.23441696166992,
"margin_dpo/margin_std": 97.614013671875,
"step": 592
},
{
"KL/chosen_KL_mean": -117.73814392089844,
"KL/mean": -152.12118530273438,
"KL/rejected_KL_mean": -186.50421142578125,
"KL/std": 89.73828125,
"epoch": 0.8964474678760394,
"fcm_dpo/beta": 0.0057606808841228485,
"fcm_dpo/delta": 0.003943389281630516,
"fcm_dpo/margin": 68.76606750488281,
"fcm_dpo/q_t": 0.40888774394989014,
"grad_norm": 15.012417793273926,
"learning_rate": 1.6463034933723336e-08,
"logits/chosen": 0.7528954744338989,
"logits/rejected": 0.6526628136634827,
"logps/chosen": -159.81814575195312,
"logps/ref_chosen": -42.08000183105469,
"logps/ref_rejected": -68.47499084472656,
"logps/rejected": -254.97918701171875,
"loss": 1.1133,
"margin_dpo/margin_mean": 68.76606750488281,
"margin_dpo/margin_std": 103.5038070678711,
"step": 593
},
{
"KL/chosen_KL_mean": -148.6075439453125,
"KL/mean": -179.48068237304688,
"KL/rejected_KL_mean": -210.35379028320312,
"KL/std": 88.96531677246094,
"epoch": 0.8979591836734694,
"fcm_dpo/beta": 0.005808601155877113,
"fcm_dpo/delta": 0.04289082810282707,
"fcm_dpo/margin": 61.74627685546875,
"fcm_dpo/q_t": 0.4163286089897156,
"grad_norm": 13.334421157836914,
"learning_rate": 1.5994421609589385e-08,
"logits/chosen": 0.6454529762268066,
"logits/rejected": 0.6314476132392883,
"logps/chosen": -212.26620483398438,
"logps/ref_chosen": -63.658668518066406,
"logps/ref_rejected": -70.35597229003906,
"logps/rejected": -280.70977783203125,
"loss": 1.1278,
"margin_dpo/margin_mean": 61.74627685546875,
"margin_dpo/margin_std": 91.94680786132812,
"step": 594
},
{
"KL/chosen_KL_mean": -135.47561645507812,
"KL/mean": -176.2564697265625,
"KL/rejected_KL_mean": -217.03732299804688,
"KL/std": 94.02308654785156,
"epoch": 0.8994708994708994,
"fcm_dpo/beta": 0.0057758791372179985,
"fcm_dpo/delta": -0.07456095516681671,
"fcm_dpo/margin": 81.56169891357422,
"fcm_dpo/q_t": 0.3929978609085083,
"grad_norm": 11.464783668518066,
"learning_rate": 1.553235392451377e-08,
"logits/chosen": 0.8024039268493652,
"logits/rejected": 0.7064827680587769,
"logps/chosen": -191.6943817138672,
"logps/ref_chosen": -56.21875762939453,
"logps/ref_rejected": -83.95773315429688,
"logps/rejected": -300.99505615234375,
"loss": 1.0739,
"margin_dpo/margin_mean": 81.56170654296875,
"margin_dpo/margin_std": 115.7236328125,
"step": 595
},
{
"KL/chosen_KL_mean": -156.56903076171875,
"KL/mean": -174.62710571289062,
"KL/rejected_KL_mean": -192.6851806640625,
"KL/std": 88.60908508300781,
"epoch": 0.9009826152683296,
"fcm_dpo/beta": 0.005814189091324806,
"fcm_dpo/delta": 0.06673035025596619,
"fcm_dpo/margin": 36.11613464355469,
"fcm_dpo/q_t": 0.45263227820396423,
"grad_norm": 12.741703987121582,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 0.6664811372756958,
"logits/rejected": 0.6890226602554321,
"logps/chosen": -225.04991149902344,
"logps/ref_chosen": -68.48088073730469,
"logps/ref_rejected": -61.732967376708984,
"logps/rejected": -254.41815185546875,
"loss": 1.2612,
"margin_dpo/margin_mean": 36.11613464355469,
"margin_dpo/margin_std": 95.08134460449219,
"step": 596
},
{
"KL/chosen_KL_mean": -121.67790222167969,
"KL/mean": -152.9395751953125,
"KL/rejected_KL_mean": -184.2012481689453,
"KL/std": 80.47587585449219,
"epoch": 0.9024943310657596,
"fcm_dpo/beta": 0.005853001959621906,
"fcm_dpo/delta": 0.03525510057806969,
"fcm_dpo/margin": 62.523338317871094,
"fcm_dpo/q_t": 0.4164848327636719,
"grad_norm": 11.317060470581055,
"learning_rate": 1.4627906988186111e-08,
"logits/chosen": 0.7406236529350281,
"logits/rejected": 0.7233434319496155,
"logps/chosen": -170.53541564941406,
"logps/ref_chosen": -48.85750961303711,
"logps/ref_rejected": -55.068084716796875,
"logps/rejected": -239.2693328857422,
"loss": 1.1304,
"margin_dpo/margin_mean": 62.523338317871094,
"margin_dpo/margin_std": 97.37086486816406,
"step": 597
},
{
"KL/chosen_KL_mean": -158.7311553955078,
"KL/mean": -178.59310913085938,
"KL/rejected_KL_mean": -198.45504760742188,
"KL/std": 91.24116516113281,
"epoch": 0.9040060468631897,
"fcm_dpo/beta": 0.005891027860343456,
"fcm_dpo/delta": 0.060597676783800125,
"fcm_dpo/margin": 39.72389221191406,
"fcm_dpo/q_t": 0.4469287395477295,
"grad_norm": 13.371644020080566,
"learning_rate": 1.4185553036259095e-08,
"logits/chosen": 0.7209906578063965,
"logits/rejected": 0.6379245519638062,
"logps/chosen": -217.61831665039062,
"logps/ref_chosen": -58.88715362548828,
"logps/ref_rejected": -81.43145751953125,
"logps/rejected": -279.8865051269531,
"loss": 1.2482,
"margin_dpo/margin_mean": 39.72389221191406,
"margin_dpo/margin_std": 99.56621551513672,
"step": 598
},
{
"KL/chosen_KL_mean": -160.98428344726562,
"KL/mean": -185.35153198242188,
"KL/rejected_KL_mean": -209.71881103515625,
"KL/std": 89.51396179199219,
"epoch": 0.9055177626606198,
"fcm_dpo/beta": 0.006037857383489609,
"fcm_dpo/delta": 0.10858315229415894,
"fcm_dpo/margin": 48.73453903198242,
"fcm_dpo/q_t": 0.4335615038871765,
"grad_norm": 15.323464393615723,
"learning_rate": 1.3749795321332885e-08,
"logits/chosen": 0.8231375217437744,
"logits/rejected": 0.7764627933502197,
"logps/chosen": -218.59146118164062,
"logps/ref_chosen": -57.60719299316406,
"logps/ref_rejected": -71.80469512939453,
"logps/rejected": -281.52349853515625,
"loss": 1.2021,
"margin_dpo/margin_mean": 48.734535217285156,
"margin_dpo/margin_std": 101.18661499023438,
"step": 599
},
{
"KL/chosen_KL_mean": -146.86544799804688,
"KL/mean": -175.1666259765625,
"KL/rejected_KL_mean": -203.4678192138672,
"KL/std": 95.1922378540039,
"epoch": 0.9070294784580499,
"fcm_dpo/beta": 0.0060354797169566154,
"fcm_dpo/delta": -0.03539323806762695,
"fcm_dpo/margin": 56.60239028930664,
"fcm_dpo/q_t": 0.4227682948112488,
"grad_norm": 14.952155113220215,
"learning_rate": 1.3320646032487393e-08,
"logits/chosen": 0.7605217099189758,
"logits/rejected": 0.7032819986343384,
"logps/chosen": -205.30775451660156,
"logps/ref_chosen": -58.44231414794922,
"logps/ref_rejected": -83.64639282226562,
"logps/rejected": -287.11419677734375,
"loss": 1.163,
"margin_dpo/margin_mean": 56.602394104003906,
"margin_dpo/margin_std": 100.25826263427734,
"step": 600
},
{
"KL/chosen_KL_mean": -129.97186279296875,
"KL/mean": -168.6710968017578,
"KL/rejected_KL_mean": -207.37033081054688,
"KL/std": 99.53807067871094,
"epoch": 0.90854119425548,
"fcm_dpo/beta": 0.005935993045568466,
"fcm_dpo/delta": -0.06281746923923492,
"fcm_dpo/margin": 77.39845275878906,
"fcm_dpo/q_t": 0.3965364098548889,
"grad_norm": 11.334195137023926,
"learning_rate": 1.2898117173950868e-08,
"logits/chosen": 0.7119603753089905,
"logits/rejected": 0.632337749004364,
"logps/chosen": -185.56619262695312,
"logps/ref_chosen": -55.59432601928711,
"logps/ref_rejected": -83.68630981445312,
"logps/rejected": -291.056640625,
"loss": 1.0842,
"margin_dpo/margin_mean": 77.39845275878906,
"margin_dpo/margin_std": 113.251953125,
"step": 601
},
{
"KL/chosen_KL_mean": -118.56781768798828,
"KL/mean": -155.1220245361328,
"KL/rejected_KL_mean": -191.67620849609375,
"KL/std": 87.95162963867188,
"epoch": 0.91005291005291,
"fcm_dpo/beta": 0.005926728714257479,
"fcm_dpo/delta": -0.03489077091217041,
"fcm_dpo/margin": 73.1083984375,
"fcm_dpo/q_t": 0.4002050757408142,
"grad_norm": 14.398859024047852,
"learning_rate": 1.2482220564763667e-08,
"logits/chosen": 0.7411153316497803,
"logits/rejected": 0.7103064656257629,
"logps/chosen": -174.91700744628906,
"logps/ref_chosen": -56.349185943603516,
"logps/ref_rejected": -71.9959716796875,
"logps/rejected": -263.67218017578125,
"loss": 1.0744,
"margin_dpo/margin_mean": 73.1083984375,
"margin_dpo/margin_std": 95.10252380371094,
"step": 602
},
{
"KL/chosen_KL_mean": -132.7935333251953,
"KL/mean": -167.63717651367188,
"KL/rejected_KL_mean": -202.48080444335938,
"KL/std": 88.24359130859375,
"epoch": 0.9115646258503401,
"fcm_dpo/beta": 0.005878736265003681,
"fcm_dpo/delta": -0.010163695551455021,
"fcm_dpo/margin": 69.68727111816406,
"fcm_dpo/q_t": 0.40638357400894165,
"grad_norm": 14.447917938232422,
"learning_rate": 1.2072967838448051e-08,
"logits/chosen": 0.6945721507072449,
"logits/rejected": 0.6367731094360352,
"logps/chosen": -185.9619140625,
"logps/ref_chosen": -53.16838836669922,
"logps/ref_rejected": -73.8604736328125,
"logps/rejected": -276.34130859375,
"loss": 1.1049,
"margin_dpo/margin_mean": 69.6872787475586,
"margin_dpo/margin_std": 102.927978515625,
"step": 603
},
{
"KL/chosen_KL_mean": -128.6298828125,
"KL/mean": -159.66763305664062,
"KL/rejected_KL_mean": -190.70538330078125,
"KL/std": 84.08584594726562,
"epoch": 0.9130763416477702,
"fcm_dpo/beta": 0.005923721473664045,
"fcm_dpo/delta": 0.03342335298657417,
"fcm_dpo/margin": 62.07550048828125,
"fcm_dpo/q_t": 0.41685357689857483,
"grad_norm": 14.421860694885254,
"learning_rate": 1.1670370442682459e-08,
"logits/chosen": 0.6826125383377075,
"logits/rejected": 0.6882836818695068,
"logps/chosen": -201.279296875,
"logps/ref_chosen": -72.64942169189453,
"logps/ref_rejected": -69.8792724609375,
"logps/rejected": -260.58465576171875,
"loss": 1.1468,
"margin_dpo/margin_mean": 62.07550048828125,
"margin_dpo/margin_std": 107.28148651123047,
"step": 604
},
{
"KL/chosen_KL_mean": -148.1844940185547,
"KL/mean": -179.252685546875,
"KL/rejected_KL_mean": -210.32086181640625,
"KL/std": 87.4947509765625,
"epoch": 0.9145880574452003,
"fcm_dpo/beta": 0.005941362120211124,
"fcm_dpo/delta": 0.03198657184839249,
"fcm_dpo/margin": 62.1363639831543,
"fcm_dpo/q_t": 0.41510260105133057,
"grad_norm": 13.999643325805664,
"learning_rate": 1.1274439638981532e-08,
"logits/chosen": 0.7563266754150391,
"logits/rejected": 0.6998000144958496,
"logps/chosen": -209.79733276367188,
"logps/ref_chosen": -61.61284637451172,
"logps/ref_rejected": -79.34398651123047,
"logps/rejected": -289.66485595703125,
"loss": 1.1447,
"margin_dpo/margin_mean": 62.13636779785156,
"margin_dpo/margin_std": 105.43777465820312,
"step": 605
},
{
"KL/chosen_KL_mean": -129.15708923339844,
"KL/mean": -167.2366485595703,
"KL/rejected_KL_mean": -205.31622314453125,
"KL/std": 90.20654296875,
"epoch": 0.9160997732426304,
"fcm_dpo/beta": 0.005937991198152304,
"fcm_dpo/delta": -0.054802730679512024,
"fcm_dpo/margin": 76.15914916992188,
"fcm_dpo/q_t": 0.3981695771217346,
"grad_norm": 16.341068267822266,
"learning_rate": 1.0885186502381016e-08,
"logits/chosen": 0.6808478832244873,
"logits/rejected": 0.6129442453384399,
"logps/chosen": -183.62132263183594,
"logps/ref_chosen": -54.46424102783203,
"logps/ref_rejected": -79.62708282470703,
"logps/rejected": -284.94329833984375,
"loss": 1.0782,
"margin_dpo/margin_mean": 76.15914154052734,
"margin_dpo/margin_std": 106.39737701416016,
"step": 606
},
{
"KL/chosen_KL_mean": -144.6417694091797,
"KL/mean": -177.8890380859375,
"KL/rejected_KL_mean": -211.1363067626953,
"KL/std": 89.48819732666016,
"epoch": 0.9176114890400605,
"fcm_dpo/beta": 0.005859079770743847,
"fcm_dpo/delta": 0.009555503726005554,
"fcm_dpo/margin": 66.49452209472656,
"fcm_dpo/q_t": 0.41053706407546997,
"grad_norm": 13.135991096496582,
"learning_rate": 1.0502621921127774e-08,
"logits/chosen": 0.6779167652130127,
"logits/rejected": 0.6527628898620605,
"logps/chosen": -207.5026397705078,
"logps/ref_chosen": -62.86086654663086,
"logps/ref_rejected": -72.5501937866211,
"logps/rejected": -283.6864929199219,
"loss": 1.1264,
"margin_dpo/margin_mean": 66.49452209472656,
"margin_dpo/margin_std": 102.65058135986328,
"step": 607
},
{
"KL/chosen_KL_mean": -146.85772705078125,
"KL/mean": -180.73568725585938,
"KL/rejected_KL_mean": -214.6136474609375,
"KL/std": 93.26339721679688,
"epoch": 0.9191232048374905,
"fcm_dpo/beta": 0.00591567438095808,
"fcm_dpo/delta": -0.0008830418810248375,
"fcm_dpo/margin": 67.75590515136719,
"fcm_dpo/q_t": 0.4082740247249603,
"grad_norm": 13.509528160095215,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 0.6950168013572693,
"logits/rejected": 0.6124898195266724,
"logps/chosen": -210.0384521484375,
"logps/ref_chosen": -63.18071746826172,
"logps/ref_rejected": -99.15888214111328,
"logps/rejected": -313.77252197265625,
"loss": 1.1011,
"margin_dpo/margin_mean": 67.75590515136719,
"margin_dpo/margin_std": 95.83975982666016,
"step": 608
},
{
"KL/chosen_KL_mean": -130.3685302734375,
"KL/mean": -168.13217163085938,
"KL/rejected_KL_mean": -205.89581298828125,
"KL/std": 88.60955810546875,
"epoch": 0.9206349206349206,
"fcm_dpo/beta": 0.005846591666340828,
"fcm_dpo/delta": -0.04399782419204712,
"fcm_dpo/margin": 75.52725982666016,
"fcm_dpo/q_t": 0.39679408073425293,
"grad_norm": 12.15417766571045,
"learning_rate": 9.757601041885694e-09,
"logits/chosen": 0.8001549243927002,
"logits/rejected": 0.7611320614814758,
"logps/chosen": -178.99176025390625,
"logps/ref_chosen": -48.62322235107422,
"logps/ref_rejected": -68.28271484375,
"logps/rejected": -274.17852783203125,
"loss": 1.0622,
"margin_dpo/margin_mean": 75.52726745605469,
"margin_dpo/margin_std": 89.76765441894531,
"step": 609
},
{
"KL/chosen_KL_mean": -141.3332977294922,
"KL/mean": -178.12103271484375,
"KL/rejected_KL_mean": -214.9087677001953,
"KL/std": 94.57626342773438,
"epoch": 0.9221466364323507,
"fcm_dpo/beta": 0.005809293128550053,
"fcm_dpo/delta": -0.029053177684545517,
"fcm_dpo/margin": 73.57546997070312,
"fcm_dpo/q_t": 0.4037247896194458,
"grad_norm": 13.424199104309082,
"learning_rate": 9.395165583732379e-09,
"logits/chosen": 0.7112727165222168,
"logits/rejected": 0.7056193947792053,
"logps/chosen": -213.9984130859375,
"logps/ref_chosen": -72.66513061523438,
"logps/ref_rejected": -87.15310668945312,
"logps/rejected": -302.0618896484375,
"loss": 1.0946,
"margin_dpo/margin_mean": 73.5754623413086,
"margin_dpo/margin_std": 106.85142517089844,
"step": 610
},
{
"KL/chosen_KL_mean": -132.4799346923828,
"KL/mean": -161.47647094726562,
"KL/rejected_KL_mean": -190.4730224609375,
"KL/std": 86.05290222167969,
"epoch": 0.9236583522297808,
"fcm_dpo/beta": 0.005862545222043991,
"fcm_dpo/delta": 0.06215390935540199,
"fcm_dpo/margin": 57.99309158325195,
"fcm_dpo/q_t": 0.4210967719554901,
"grad_norm": 14.677167892456055,
"learning_rate": 9.03946036001449e-09,
"logits/chosen": 0.7628319263458252,
"logits/rejected": 0.7115751504898071,
"logps/chosen": -180.78851318359375,
"logps/ref_chosen": -48.30857849121094,
"logps/ref_rejected": -70.6141128540039,
"logps/rejected": -261.087158203125,
"loss": 1.1421,
"margin_dpo/margin_mean": 57.99309158325195,
"margin_dpo/margin_std": 90.12339782714844,
"step": 611
},
{
"KL/chosen_KL_mean": -143.25289916992188,
"KL/mean": -185.50271606445312,
"KL/rejected_KL_mean": -227.75253295898438,
"KL/std": 91.38240051269531,
"epoch": 0.9251700680272109,
"fcm_dpo/beta": 0.00581524008885026,
"fcm_dpo/delta": -0.09606201201677322,
"fcm_dpo/margin": 84.49961853027344,
"fcm_dpo/q_t": 0.3861920237541199,
"grad_norm": 11.991162300109863,
"learning_rate": 8.690495320571839e-09,
"logits/chosen": 0.6451847553253174,
"logits/rejected": 0.5737862586975098,
"logps/chosen": -204.48446655273438,
"logps/ref_chosen": -61.23155975341797,
"logps/ref_rejected": -94.37979888916016,
"logps/rejected": -322.13232421875,
"loss": 1.0341,
"margin_dpo/margin_mean": 84.49961853027344,
"margin_dpo/margin_std": 100.94599914550781,
"step": 612
},
{
"KL/chosen_KL_mean": -122.99929809570312,
"KL/mean": -161.93942260742188,
"KL/rejected_KL_mean": -200.87953186035156,
"KL/std": 94.48210144042969,
"epoch": 0.926681783824641,
"fcm_dpo/beta": 0.005757839884608984,
"fcm_dpo/delta": -0.05073459818959236,
"fcm_dpo/margin": 77.8802490234375,
"fcm_dpo/q_t": 0.3961522579193115,
"grad_norm": 11.217021942138672,
"learning_rate": 8.348280226706722e-09,
"logits/chosen": 0.6813284754753113,
"logits/rejected": 0.6765438914299011,
"logps/chosen": -176.98240661621094,
"logps/ref_chosen": -53.98310852050781,
"logps/ref_rejected": -58.32208251953125,
"logps/rejected": -259.20159912109375,
"loss": 1.0643,
"margin_dpo/margin_mean": 77.88024139404297,
"margin_dpo/margin_std": 98.7802505493164,
"step": 613
},
{
"KL/chosen_KL_mean": -143.9332275390625,
"KL/mean": -179.5696563720703,
"KL/rejected_KL_mean": -215.20608520507812,
"KL/std": 83.36962890625,
"epoch": 0.9281934996220711,
"fcm_dpo/beta": 0.005744350142776966,
"fcm_dpo/delta": -0.01025397703051567,
"fcm_dpo/margin": 71.2728500366211,
"fcm_dpo/q_t": 0.4045924246311188,
"grad_norm": 15.103445053100586,
"learning_rate": 8.012824650910937e-09,
"logits/chosen": 0.7549277544021606,
"logits/rejected": 0.7466669082641602,
"logps/chosen": -204.17626953125,
"logps/ref_chosen": -60.24303436279297,
"logps/ref_rejected": -72.26258850097656,
"logps/rejected": -287.46868896484375,
"loss": 1.0919,
"margin_dpo/margin_mean": 71.2728500366211,
"margin_dpo/margin_std": 95.47000122070312,
"step": 614
},
{
"KL/chosen_KL_mean": -150.774658203125,
"KL/mean": -186.82733154296875,
"KL/rejected_KL_mean": -222.87998962402344,
"KL/std": 103.77711486816406,
"epoch": 0.9297052154195011,
"fcm_dpo/beta": 0.00567289162427187,
"fcm_dpo/delta": -0.010130487382411957,
"fcm_dpo/margin": 72.10533905029297,
"fcm_dpo/q_t": 0.40692615509033203,
"grad_norm": 12.820241928100586,
"learning_rate": 7.684137976598088e-09,
"logits/chosen": 0.6949265003204346,
"logits/rejected": 0.636421799659729,
"logps/chosen": -222.86932373046875,
"logps/ref_chosen": -72.09467315673828,
"logps/ref_rejected": -104.02980041503906,
"logps/rejected": -326.9097900390625,
"loss": 1.1224,
"margin_dpo/margin_mean": 72.10533905029297,
"margin_dpo/margin_std": 115.87667846679688,
"step": 615
},
{
"KL/chosen_KL_mean": -142.3875732421875,
"KL/mean": -174.14651489257812,
"KL/rejected_KL_mean": -205.90545654296875,
"KL/std": 96.92578125,
"epoch": 0.9312169312169312,
"fcm_dpo/beta": 0.005723685026168823,
"fcm_dpo/delta": 0.03782026842236519,
"fcm_dpo/margin": 63.51789093017578,
"fcm_dpo/q_t": 0.4169592261314392,
"grad_norm": 12.180818557739258,
"learning_rate": 7.36222939784098e-09,
"logits/chosen": 0.7669482231140137,
"logits/rejected": 0.6834902763366699,
"logps/chosen": -200.9182891845703,
"logps/ref_chosen": -58.530723571777344,
"logps/ref_rejected": -75.48025512695312,
"logps/rejected": -281.3857116699219,
"loss": 1.1315,
"margin_dpo/margin_mean": 63.51789093017578,
"margin_dpo/margin_std": 98.78520965576172,
"step": 616
},
{
"KL/chosen_KL_mean": -154.86825561523438,
"KL/mean": -181.0230255126953,
"KL/rejected_KL_mean": -207.17779541015625,
"KL/std": 90.08709716796875,
"epoch": 0.9327286470143613,
"fcm_dpo/beta": 0.005718774627894163,
"fcm_dpo/delta": -0.022925637662410736,
"fcm_dpo/margin": 52.30952835083008,
"fcm_dpo/q_t": 0.43085378408432007,
"grad_norm": 15.74920654296875,
"learning_rate": 7.047107919114586e-09,
"logits/chosen": 0.728320300579071,
"logits/rejected": 0.6747007369995117,
"logps/chosen": -212.47694396972656,
"logps/ref_chosen": -57.608673095703125,
"logps/ref_rejected": -81.22109985351562,
"logps/rejected": -288.3988952636719,
"loss": 1.1848,
"margin_dpo/margin_mean": 52.30952453613281,
"margin_dpo/margin_std": 95.00151062011719,
"step": 617
},
{
"KL/chosen_KL_mean": -136.32061767578125,
"KL/mean": -169.56851196289062,
"KL/rejected_KL_mean": -202.81642150878906,
"KL/std": 97.36782836914062,
"epoch": 0.9342403628117913,
"fcm_dpo/beta": 0.005721730180084705,
"fcm_dpo/delta": 0.02022417262196541,
"fcm_dpo/margin": 66.49579620361328,
"fcm_dpo/q_t": 0.4136330485343933,
"grad_norm": 16.270727157592773,
"learning_rate": 6.738782355044048e-09,
"logits/chosen": 0.6903648376464844,
"logits/rejected": 0.5835120677947998,
"logps/chosen": -193.0165557861328,
"logps/ref_chosen": -56.69594192504883,
"logps/ref_rejected": -85.92362976074219,
"logps/rejected": -288.74005126953125,
"loss": 1.1142,
"margin_dpo/margin_mean": 66.49580383300781,
"margin_dpo/margin_std": 97.39218139648438,
"step": 618
},
{
"KL/chosen_KL_mean": -133.4502410888672,
"KL/mean": -169.5776824951172,
"KL/rejected_KL_mean": -205.70513916015625,
"KL/std": 94.65431213378906,
"epoch": 0.9357520786092215,
"fcm_dpo/beta": 0.005723532289266586,
"fcm_dpo/delta": -0.01417827233672142,
"fcm_dpo/margin": 72.2548828125,
"fcm_dpo/q_t": 0.4046638607978821,
"grad_norm": 12.724713325500488,
"learning_rate": 6.437261330158206e-09,
"logits/chosen": 0.8282185792922974,
"logits/rejected": 0.7467609643936157,
"logps/chosen": -187.5086669921875,
"logps/ref_chosen": -54.05841827392578,
"logps/ref_rejected": -83.55493927001953,
"logps/rejected": -289.26007080078125,
"loss": 1.0989,
"margin_dpo/margin_mean": 72.2548828125,
"margin_dpo/margin_std": 104.76480102539062,
"step": 619
},
{
"KL/chosen_KL_mean": -133.63052368164062,
"KL/mean": -162.80810546875,
"KL/rejected_KL_mean": -191.98568725585938,
"KL/std": 84.86981201171875,
"epoch": 0.9372637944066515,
"fcm_dpo/beta": 0.005691590253263712,
"fcm_dpo/delta": -0.059919971972703934,
"fcm_dpo/margin": 58.35517883300781,
"fcm_dpo/q_t": 0.4229113459587097,
"grad_norm": 13.340130805969238,
"learning_rate": 6.142553278648238e-09,
"logits/chosen": 0.7194168567657471,
"logits/rejected": 0.7200058698654175,
"logps/chosen": -197.000244140625,
"logps/ref_chosen": -63.36971664428711,
"logps/ref_rejected": -65.68269348144531,
"logps/rejected": -257.66839599609375,
"loss": 1.1571,
"margin_dpo/margin_mean": 58.35517883300781,
"margin_dpo/margin_std": 93.1148681640625,
"step": 620
},
{
"KL/chosen_KL_mean": -136.25088500976562,
"KL/mean": -165.6092529296875,
"KL/rejected_KL_mean": -194.9676513671875,
"KL/std": 89.51348876953125,
"epoch": 0.9387755102040817,
"fcm_dpo/beta": 0.005727657116949558,
"fcm_dpo/delta": 0.06560888886451721,
"fcm_dpo/margin": 58.716758728027344,
"fcm_dpo/q_t": 0.4238740801811218,
"grad_norm": 15.22817325592041,
"learning_rate": 5.854666444131934e-09,
"logits/chosen": 0.780386209487915,
"logits/rejected": 0.6619117259979248,
"logps/chosen": -188.57211303710938,
"logps/ref_chosen": -52.321224212646484,
"logps/ref_rejected": -88.09001159667969,
"logps/rejected": -283.0576477050781,
"loss": 1.166,
"margin_dpo/margin_mean": 58.716758728027344,
"margin_dpo/margin_std": 106.28216552734375,
"step": 621
},
{
"KL/chosen_KL_mean": -147.62847900390625,
"KL/mean": -179.83387756347656,
"KL/rejected_KL_mean": -212.03927612304688,
"KL/std": 95.06315612792969,
"epoch": 0.9402872260015117,
"fcm_dpo/beta": 0.0057709356769919395,
"fcm_dpo/delta": 0.029196467250585556,
"fcm_dpo/margin": 64.41080474853516,
"fcm_dpo/q_t": 0.4147273004055023,
"grad_norm": 15.071149826049805,
"learning_rate": 5.573608879422875e-09,
"logits/chosen": 0.7118933796882629,
"logits/rejected": 0.6644724607467651,
"logps/chosen": -207.49392700195312,
"logps/ref_chosen": -59.86545944213867,
"logps/ref_rejected": -81.86668395996094,
"logps/rejected": -293.90594482421875,
"loss": 1.1291,
"margin_dpo/margin_mean": 64.41080474853516,
"margin_dpo/margin_std": 100.54658508300781,
"step": 622
},
{
"KL/chosen_KL_mean": -156.4019775390625,
"KL/mean": -190.06399536132812,
"KL/rejected_KL_mean": -223.72601318359375,
"KL/std": 98.73883056640625,
"epoch": 0.9417989417989417,
"fcm_dpo/beta": 0.00575483962893486,
"fcm_dpo/delta": 0.012818563729524612,
"fcm_dpo/margin": 67.32402801513672,
"fcm_dpo/q_t": 0.41123396158218384,
"grad_norm": 13.648994445800781,
"learning_rate": 5.299388446305342e-09,
"logits/chosen": 0.7300339341163635,
"logits/rejected": 0.6706830263137817,
"logps/chosen": -223.7704315185547,
"logps/ref_chosen": -67.36846160888672,
"logps/ref_rejected": -82.02733612060547,
"logps/rejected": -305.75335693359375,
"loss": 1.1213,
"margin_dpo/margin_mean": 67.32402038574219,
"margin_dpo/margin_std": 103.84030151367188,
"step": 623
},
{
"KL/chosen_KL_mean": -132.14297485351562,
"KL/mean": -169.37939453125,
"KL/rejected_KL_mean": -206.6158447265625,
"KL/std": 96.08959197998047,
"epoch": 0.9433106575963719,
"fcm_dpo/beta": 0.005741535220295191,
"fcm_dpo/delta": -0.029015716165304184,
"fcm_dpo/margin": 74.47286224365234,
"fcm_dpo/q_t": 0.4038606882095337,
"grad_norm": 14.041501998901367,
"learning_rate": 5.03201281531429e-09,
"logits/chosen": 0.7264994382858276,
"logits/rejected": 0.6279963254928589,
"logps/chosen": -183.16952514648438,
"logps/ref_chosen": -51.02655029296875,
"logps/ref_rejected": -76.49203491210938,
"logps/rejected": -283.1078796386719,
"loss": 1.0952,
"margin_dpo/margin_mean": 74.47286987304688,
"margin_dpo/margin_std": 108.83842468261719,
"step": 624
},
{
"KL/chosen_KL_mean": -134.61871337890625,
"KL/mean": -163.3648681640625,
"KL/rejected_KL_mean": -192.1110076904297,
"KL/std": 91.74197387695312,
"epoch": 0.9448223733938019,
"fcm_dpo/beta": 0.005805259104818106,
"fcm_dpo/delta": 0.06854051351547241,
"fcm_dpo/margin": 57.492279052734375,
"fcm_dpo/q_t": 0.42583543062210083,
"grad_norm": 13.817388534545898,
"learning_rate": 4.7714894655209174e-09,
"logits/chosen": 0.8289808034896851,
"logits/rejected": 0.734531044960022,
"logps/chosen": -188.8263397216797,
"logps/ref_chosen": -54.20761489868164,
"logps/ref_rejected": -84.93669128417969,
"logps/rejected": -277.0476989746094,
"loss": 1.1775,
"margin_dpo/margin_mean": 57.492279052734375,
"margin_dpo/margin_std": 111.24788665771484,
"step": 625
},
{
"KL/chosen_KL_mean": -128.06040954589844,
"KL/mean": -166.5856475830078,
"KL/rejected_KL_mean": -205.11090087890625,
"KL/std": 95.63418579101562,
"epoch": 0.9463340891912321,
"fcm_dpo/beta": 0.005776531994342804,
"fcm_dpo/delta": -0.04730883240699768,
"fcm_dpo/margin": 77.05049896240234,
"fcm_dpo/q_t": 0.4014032483100891,
"grad_norm": 13.485941886901855,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 0.8055673837661743,
"logits/rejected": 0.6686593294143677,
"logps/chosen": -173.12242126464844,
"logps/ref_chosen": -45.06201934814453,
"logps/ref_rejected": -89.66368103027344,
"logps/rejected": -294.77459716796875,
"loss": 1.0938,
"margin_dpo/margin_mean": 77.05049133300781,
"margin_dpo/margin_std": 117.03031921386719,
"step": 626
},
{
"KL/chosen_KL_mean": -145.78143310546875,
"KL/mean": -186.88687133789062,
"KL/rejected_KL_mean": -227.99229431152344,
"KL/std": 98.71812438964844,
"epoch": 0.9478458049886621,
"fcm_dpo/beta": 0.005693660117685795,
"fcm_dpo/delta": -0.07180622965097427,
"fcm_dpo/margin": 82.21084594726562,
"fcm_dpo/q_t": 0.3918594717979431,
"grad_norm": 13.955216407775879,
"learning_rate": 4.271028567242818e-09,
"logits/chosen": 0.6949923634529114,
"logits/rejected": 0.5684172511100769,
"logps/chosen": -204.572509765625,
"logps/ref_chosen": -58.791053771972656,
"logps/ref_rejected": -94.90802001953125,
"logps/rejected": -322.90032958984375,
"loss": 1.0605,
"margin_dpo/margin_mean": 82.21084594726562,
"margin_dpo/margin_std": 107.46525573730469,
"step": 627
},
{
"KL/chosen_KL_mean": -131.73631286621094,
"KL/mean": -171.53302001953125,
"KL/rejected_KL_mean": -211.32972717285156,
"KL/std": 94.16317749023438,
"epoch": 0.9493575207860923,
"fcm_dpo/beta": 0.00568841677159071,
"fcm_dpo/delta": -0.05616312474012375,
"fcm_dpo/margin": 79.59342193603516,
"fcm_dpo/q_t": 0.395234078168869,
"grad_norm": 16.013681411743164,
"learning_rate": 4.0311050177251895e-09,
"logits/chosen": 0.7259294390678406,
"logits/rejected": 0.6854862570762634,
"logps/chosen": -184.53988647460938,
"logps/ref_chosen": -52.80357360839844,
"logps/ref_rejected": -76.49468994140625,
"logps/rejected": -287.82440185546875,
"loss": 1.0874,
"margin_dpo/margin_mean": 79.59342193603516,
"margin_dpo/margin_std": 108.70342254638672,
"step": 628
},
{
"KL/chosen_KL_mean": -142.18206787109375,
"KL/mean": -171.57269287109375,
"KL/rejected_KL_mean": -200.96328735351562,
"KL/std": 89.83036041259766,
"epoch": 0.9508692365835223,
"fcm_dpo/beta": 0.0056978208012878895,
"fcm_dpo/delta": 0.06699429452419281,
"fcm_dpo/margin": 58.781219482421875,
"fcm_dpo/q_t": 0.423664927482605,
"grad_norm": 12.265828132629395,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 0.7545243501663208,
"logits/rejected": 0.7487726807594299,
"logps/chosen": -212.89956665039062,
"logps/ref_chosen": -70.71749877929688,
"logps/ref_rejected": -78.96273803710938,
"logps/rejected": -279.926025390625,
"loss": 1.1458,
"margin_dpo/margin_mean": 58.781219482421875,
"margin_dpo/margin_std": 92.58798217773438,
"step": 629
},
{
"KL/chosen_KL_mean": -133.0452880859375,
"KL/mean": -173.24391174316406,
"KL/rejected_KL_mean": -213.44252014160156,
"KL/std": 96.12916564941406,
"epoch": 0.9523809523809523,
"fcm_dpo/beta": 0.00566629134118557,
"fcm_dpo/delta": -0.05818511173129082,
"fcm_dpo/margin": 80.39723205566406,
"fcm_dpo/q_t": 0.3956824839115143,
"grad_norm": 10.79253101348877,
"learning_rate": 3.5719052736323806e-09,
"logits/chosen": 0.6766912937164307,
"logits/rejected": 0.6319398880004883,
"logps/chosen": -189.2467041015625,
"logps/ref_chosen": -56.201412200927734,
"logps/ref_rejected": -74.69807434082031,
"logps/rejected": -288.1405944824219,
"loss": 1.0619,
"margin_dpo/margin_mean": 80.39723205566406,
"margin_dpo/margin_std": 103.09152221679688,
"step": 630
},
{
"KL/chosen_KL_mean": -127.51341247558594,
"KL/mean": -172.18319702148438,
"KL/rejected_KL_mean": -216.85299682617188,
"KL/std": 100.53424072265625,
"epoch": 0.9538926681783825,
"fcm_dpo/beta": 0.00551101379096508,
"fcm_dpo/delta": -0.09908513724803925,
"fcm_dpo/margin": 89.33956146240234,
"fcm_dpo/q_t": 0.3883308172225952,
"grad_norm": 12.608369827270508,
"learning_rate": 3.352641923861144e-09,
"logits/chosen": 0.8288528919219971,
"logits/rejected": 0.710574209690094,
"logps/chosen": -186.33401489257812,
"logps/ref_chosen": -58.82059860229492,
"logps/ref_rejected": -96.51437377929688,
"logps/rejected": -313.36737060546875,
"loss": 1.0462,
"margin_dpo/margin_mean": 89.33956146240234,
"margin_dpo/margin_std": 111.76412963867188,
"step": 631
},
{
"KL/chosen_KL_mean": -130.11866760253906,
"KL/mean": -172.71804809570312,
"KL/rejected_KL_mean": -215.3174285888672,
"KL/std": 90.61511993408203,
"epoch": 0.9554043839758125,
"fcm_dpo/beta": 0.005482650361955166,
"fcm_dpo/delta": -0.07036474347114563,
"fcm_dpo/margin": 85.19876861572266,
"fcm_dpo/q_t": 0.39144212007522583,
"grad_norm": 12.551990509033203,
"learning_rate": 3.140277830901428e-09,
"logits/chosen": 0.7222434282302856,
"logits/rejected": 0.7026859521865845,
"logps/chosen": -188.90472412109375,
"logps/ref_chosen": -58.786048889160156,
"logps/ref_rejected": -67.21923828125,
"logps/rejected": -282.53668212890625,
"loss": 1.0451,
"margin_dpo/margin_mean": 85.19876098632812,
"margin_dpo/margin_std": 100.32020568847656,
"step": 632
},
{
"KL/chosen_KL_mean": -122.58628845214844,
"KL/mean": -155.8394317626953,
"KL/rejected_KL_mean": -189.09255981445312,
"KL/std": 86.15221405029297,
"epoch": 0.9569160997732427,
"fcm_dpo/beta": 0.005492908880114555,
"fcm_dpo/delta": 0.035726308822631836,
"fcm_dpo/margin": 66.50627899169922,
"fcm_dpo/q_t": 0.4165397882461548,
"grad_norm": 12.629836082458496,
"learning_rate": 2.9348189350335007e-09,
"logits/chosen": 0.7262308597564697,
"logits/rejected": 0.6633630990982056,
"logps/chosen": -174.71649169921875,
"logps/ref_chosen": -52.13019561767578,
"logps/ref_rejected": -67.23016357421875,
"logps/rejected": -256.3227233886719,
"loss": 1.1247,
"margin_dpo/margin_mean": 66.50627899169922,
"margin_dpo/margin_std": 98.96324157714844,
"step": 633
},
{
"KL/chosen_KL_mean": -152.16098022460938,
"KL/mean": -165.3771514892578,
"KL/rejected_KL_mean": -178.59332275390625,
"KL/std": 89.72291564941406,
"epoch": 0.9584278155706727,
"fcm_dpo/beta": 0.005486940965056419,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 26.43233871459961,
"fcm_dpo/q_t": 0.4672384262084961,
"grad_norm": 16.054452896118164,
"learning_rate": 2.736270983384276e-09,
"logits/chosen": 0.7948806285858154,
"logits/rejected": 0.8081480264663696,
"logps/chosen": -213.14077758789062,
"logps/ref_chosen": -60.97979736328125,
"logps/ref_rejected": -58.50825119018555,
"logps/rejected": -237.10157775878906,
"loss": 1.3235,
"margin_dpo/margin_mean": 26.43233871459961,
"margin_dpo/margin_std": 103.15001678466797,
"step": 634
},
{
"KL/chosen_KL_mean": -165.2581787109375,
"KL/mean": -191.9426727294922,
"KL/rejected_KL_mean": -218.62716674804688,
"KL/std": 90.02232360839844,
"epoch": 0.9599395313681028,
"fcm_dpo/beta": 0.005586233921349049,
"fcm_dpo/delta": 0.10464800894260406,
"fcm_dpo/margin": 53.3690185546875,
"fcm_dpo/q_t": 0.4323941171169281,
"grad_norm": 13.974189758300781,
"learning_rate": 2.5446395297668287e-09,
"logits/chosen": 0.627932071685791,
"logits/rejected": 0.565004825592041,
"logps/chosen": -231.23118591308594,
"logps/ref_chosen": -65.9730224609375,
"logps/ref_rejected": -85.61317443847656,
"logps/rejected": -304.2403564453125,
"loss": 1.2015,
"margin_dpo/margin_mean": 53.3690185546875,
"margin_dpo/margin_std": 110.18885803222656,
"step": 635
},
{
"KL/chosen_KL_mean": -129.96096801757812,
"KL/mean": -166.38412475585938,
"KL/rejected_KL_mean": -202.80726623535156,
"KL/std": 90.97728729248047,
"epoch": 0.9614512471655329,
"fcm_dpo/beta": 0.005583517253398895,
"fcm_dpo/delta": -0.007160985842347145,
"fcm_dpo/margin": 72.8463134765625,
"fcm_dpo/q_t": 0.40524113178253174,
"grad_norm": 10.883511543273926,
"learning_rate": 2.359929934524829e-09,
"logits/chosen": 0.7227901816368103,
"logits/rejected": 0.6219326257705688,
"logps/chosen": -179.10113525390625,
"logps/ref_chosen": -49.140167236328125,
"logps/ref_rejected": -81.26971435546875,
"logps/rejected": -284.07696533203125,
"loss": 1.0894,
"margin_dpo/margin_mean": 72.8463134765625,
"margin_dpo/margin_std": 96.14746856689453,
"step": 636
},
{
"KL/chosen_KL_mean": -170.8201904296875,
"KL/mean": -197.82333374023438,
"KL/rejected_KL_mean": -224.82647705078125,
"KL/std": 87.77001953125,
"epoch": 0.9629629629629629,
"fcm_dpo/beta": 0.0056931450963020325,
"fcm_dpo/delta": 0.09495221078395844,
"fcm_dpo/margin": 54.00630187988281,
"fcm_dpo/q_t": 0.4308604896068573,
"grad_norm": 15.127143859863281,
"learning_rate": 2.1821473643827137e-09,
"logits/chosen": 0.7148442268371582,
"logits/rejected": 0.645779013633728,
"logps/chosen": -244.51678466796875,
"logps/ref_chosen": -73.69658660888672,
"logps/ref_rejected": -83.01487731933594,
"logps/rejected": -307.84136962890625,
"loss": 1.1975,
"margin_dpo/margin_mean": 54.00630187988281,
"margin_dpo/margin_std": 111.81935119628906,
"step": 637
},
{
"KL/chosen_KL_mean": -149.79718017578125,
"KL/mean": -182.2518768310547,
"KL/rejected_KL_mean": -214.70657348632812,
"KL/std": 90.21208190917969,
"epoch": 0.9644746787603931,
"fcm_dpo/beta": 0.005730494391173124,
"fcm_dpo/delta": 0.029095135629177094,
"fcm_dpo/margin": 64.90939331054688,
"fcm_dpo/q_t": 0.41533297300338745,
"grad_norm": 12.788329124450684,
"learning_rate": 2.0112967923011646e-09,
"logits/chosen": 0.7149261236190796,
"logits/rejected": 0.666912853717804,
"logps/chosen": -212.57876586914062,
"logps/ref_chosen": -62.78158187866211,
"logps/ref_rejected": -85.40478515625,
"logps/rejected": -300.1113586425781,
"loss": 1.1245,
"margin_dpo/margin_mean": 64.90939331054688,
"margin_dpo/margin_std": 99.1895751953125,
"step": 638
},
{
"KL/chosen_KL_mean": -133.94517517089844,
"KL/mean": -172.11093139648438,
"KL/rejected_KL_mean": -210.2766571044922,
"KL/std": 96.70921325683594,
"epoch": 0.9659863945578231,
"fcm_dpo/beta": 0.005708941258490086,
"fcm_dpo/delta": -0.03738473355770111,
"fcm_dpo/margin": 76.33148193359375,
"fcm_dpo/q_t": 0.4016588628292084,
"grad_norm": 13.268625259399414,
"learning_rate": 1.847382997337943e-09,
"logits/chosen": 0.7439556121826172,
"logits/rejected": 0.6408558487892151,
"logps/chosen": -187.71176147460938,
"logps/ref_chosen": -53.76658630371094,
"logps/ref_rejected": -72.30009460449219,
"logps/rejected": -282.5767517089844,
"loss": 1.0834,
"margin_dpo/margin_mean": 76.33148193359375,
"margin_dpo/margin_std": 106.65922546386719,
"step": 639
},
{
"KL/chosen_KL_mean": -139.14096069335938,
"KL/mean": -174.42507934570312,
"KL/rejected_KL_mean": -209.70921325683594,
"KL/std": 93.50321197509766,
"epoch": 0.9674981103552532,
"fcm_dpo/beta": 0.0056776199489831924,
"fcm_dpo/delta": -0.0008220486342906952,
"fcm_dpo/margin": 70.56825256347656,
"fcm_dpo/q_t": 0.40782514214515686,
"grad_norm": 12.540871620178223,
"learning_rate": 1.690410564514244e-09,
"logits/chosen": 0.7819277048110962,
"logits/rejected": 0.716408371925354,
"logps/chosen": -190.55874633789062,
"logps/ref_chosen": -51.41777801513672,
"logps/ref_rejected": -77.27879333496094,
"logps/rejected": -286.9880065917969,
"loss": 1.1047,
"margin_dpo/margin_mean": 70.56825256347656,
"margin_dpo/margin_std": 102.07494354248047,
"step": 640
},
{
"KL/chosen_KL_mean": -147.49749755859375,
"KL/mean": -178.9612274169922,
"KL/rejected_KL_mean": -210.42495727539062,
"KL/std": 88.54032897949219,
"epoch": 0.9690098261526833,
"fcm_dpo/beta": 0.005715455859899521,
"fcm_dpo/delta": 0.04184335470199585,
"fcm_dpo/margin": 62.92748260498047,
"fcm_dpo/q_t": 0.415992796421051,
"grad_norm": 13.414076805114746,
"learning_rate": 1.5403838846864692e-09,
"logits/chosen": 0.7360565066337585,
"logits/rejected": 0.7161175012588501,
"logps/chosen": -218.55215454101562,
"logps/ref_chosen": -71.0546646118164,
"logps/ref_rejected": -82.2440185546875,
"logps/rejected": -292.6689758300781,
"loss": 1.1146,
"margin_dpo/margin_mean": 62.92747497558594,
"margin_dpo/margin_std": 84.7451171875,
"step": 641
},
{
"KL/chosen_KL_mean": -157.2935333251953,
"KL/mean": -180.04251098632812,
"KL/rejected_KL_mean": -202.79150390625,
"KL/std": 85.213134765625,
"epoch": 0.9705215419501134,
"fcm_dpo/beta": 0.005745013244450092,
"fcm_dpo/delta": 0.004635404795408249,
"fcm_dpo/margin": 45.49795913696289,
"fcm_dpo/q_t": 0.4401233196258545,
"grad_norm": 16.21065902709961,
"learning_rate": 1.3973071544233218e-09,
"logits/chosen": 0.684654951095581,
"logits/rejected": 0.7038168907165527,
"logps/chosen": -226.22280883789062,
"logps/ref_chosen": -68.92927551269531,
"logps/ref_rejected": -70.85682678222656,
"logps/rejected": -273.6483154296875,
"loss": 1.2301,
"margin_dpo/margin_mean": 45.49795913696289,
"margin_dpo/margin_std": 103.0499038696289,
"step": 642
},
{
"KL/chosen_KL_mean": -139.9108123779297,
"KL/mean": -175.9429931640625,
"KL/rejected_KL_mean": -211.97520446777344,
"KL/std": 91.34405517578125,
"epoch": 0.9720332577475435,
"fcm_dpo/beta": 0.005725730210542679,
"fcm_dpo/delta": -0.013222461566329002,
"fcm_dpo/margin": 72.06438446044922,
"fcm_dpo/q_t": 0.40598538517951965,
"grad_norm": 19.63475227355957,
"learning_rate": 1.261184375888541e-09,
"logits/chosen": 0.6746104955673218,
"logits/rejected": 0.5852953195571899,
"logps/chosen": -205.2198486328125,
"logps/ref_chosen": -65.30903625488281,
"logps/ref_rejected": -83.61613464355469,
"logps/rejected": -295.5913391113281,
"loss": 1.1065,
"margin_dpo/margin_mean": 72.06439208984375,
"margin_dpo/margin_std": 107.77041625976562,
"step": 643
},
{
"KL/chosen_KL_mean": -131.8123779296875,
"KL/mean": -158.04876708984375,
"KL/rejected_KL_mean": -184.28512573242188,
"KL/std": 84.9591064453125,
"epoch": 0.9735449735449735,
"fcm_dpo/beta": 0.005724203772842884,
"fcm_dpo/delta": -0.009847259148955345,
"fcm_dpo/margin": 52.47273254394531,
"fcm_dpo/q_t": 0.43036067485809326,
"grad_norm": 12.569685935974121,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 0.8391200304031372,
"logits/rejected": 0.805716872215271,
"logps/chosen": -182.8149871826172,
"logps/ref_chosen": -51.002601623535156,
"logps/ref_rejected": -64.46372985839844,
"logps/rejected": -248.7488555908203,
"loss": 1.1991,
"margin_dpo/margin_mean": 52.47273254394531,
"margin_dpo/margin_std": 105.81692504882812,
"step": 644
},
{
"KL/chosen_KL_mean": -141.92608642578125,
"KL/mean": -174.30859375,
"KL/rejected_KL_mean": -206.69107055664062,
"KL/std": 89.70710754394531,
"epoch": 0.9750566893424036,
"fcm_dpo/beta": 0.005741228349506855,
"fcm_dpo/delta": 0.029265832155942917,
"fcm_dpo/margin": 64.7649917602539,
"fcm_dpo/q_t": 0.41344964504241943,
"grad_norm": 14.167495727539062,
"learning_rate": 1.0098157099674987e-09,
"logits/chosen": 0.6873973608016968,
"logits/rejected": 0.6706234216690063,
"logps/chosen": -202.88949584960938,
"logps/ref_chosen": -60.963409423828125,
"logps/ref_rejected": -69.73353576660156,
"logps/rejected": -276.42462158203125,
"loss": 1.112,
"margin_dpo/margin_mean": 64.7649917602539,
"margin_dpo/margin_std": 90.13683319091797,
"step": 645
},
{
"KL/chosen_KL_mean": -156.96002197265625,
"KL/mean": -185.32113647460938,
"KL/rejected_KL_mean": -213.6822509765625,
"KL/std": 99.1893539428711,
"epoch": 0.9765684051398337,
"fcm_dpo/beta": 0.005817126017063856,
"fcm_dpo/delta": 0.07239460945129395,
"fcm_dpo/margin": 56.72221374511719,
"fcm_dpo/q_t": 0.4251546263694763,
"grad_norm": 12.480823516845703,
"learning_rate": 8.945768539031783e-10,
"logits/chosen": 0.718536913394928,
"logits/rejected": 0.6606748104095459,
"logps/chosen": -219.25009155273438,
"logps/ref_chosen": -62.290069580078125,
"logps/ref_rejected": -85.54812622070312,
"logps/rejected": -299.2303771972656,
"loss": 1.1724,
"margin_dpo/margin_mean": 56.72221374511719,
"margin_dpo/margin_std": 105.77682495117188,
"step": 646
},
{
"KL/chosen_KL_mean": -153.3720703125,
"KL/mean": -198.96734619140625,
"KL/rejected_KL_mean": -244.56259155273438,
"KL/std": 99.53343200683594,
"epoch": 0.9780801209372638,
"fcm_dpo/beta": 0.005736473947763443,
"fcm_dpo/delta": -0.12972837686538696,
"fcm_dpo/margin": 91.19052124023438,
"fcm_dpo/q_t": 0.3792613446712494,
"grad_norm": 14.296761512756348,
"learning_rate": 7.863060120144316e-10,
"logits/chosen": 0.7803740501403809,
"logits/rejected": 0.6787436008453369,
"logps/chosen": -220.887939453125,
"logps/ref_chosen": -67.515869140625,
"logps/ref_rejected": -101.50871276855469,
"logps/rejected": -346.0712890625,
"loss": 1.0045,
"margin_dpo/margin_mean": 91.19052124023438,
"margin_dpo/margin_std": 98.2463150024414,
"step": 647
},
{
"KL/chosen_KL_mean": -152.12294006347656,
"KL/mean": -184.05935668945312,
"KL/rejected_KL_mean": -215.99575805664062,
"KL/std": 87.59455871582031,
"epoch": 0.9795918367346939,
"fcm_dpo/beta": 0.005696025677025318,
"fcm_dpo/delta": 0.03737743943929672,
"fcm_dpo/margin": 63.87282943725586,
"fcm_dpo/q_t": 0.4173229932785034,
"grad_norm": 13.838418960571289,
"learning_rate": 6.850062128694045e-10,
"logits/chosen": 0.6736407279968262,
"logits/rejected": 0.6090872287750244,
"logps/chosen": -216.7188720703125,
"logps/ref_chosen": -64.59593963623047,
"logps/ref_rejected": -83.384033203125,
"logps/rejected": -299.37982177734375,
"loss": 1.154,
"margin_dpo/margin_mean": 63.87282943725586,
"margin_dpo/margin_std": 111.77117919921875,
"step": 648
},
{
"KL/chosen_KL_mean": -141.0966339111328,
"KL/mean": -173.88868713378906,
"KL/rejected_KL_mean": -206.6807403564453,
"KL/std": 89.18699645996094,
"epoch": 0.981103552532124,
"fcm_dpo/beta": 0.005725952796638012,
"fcm_dpo/delta": 0.025148997083306313,
"fcm_dpo/margin": 65.5841064453125,
"fcm_dpo/q_t": 0.4139162302017212,
"grad_norm": 17.477102279663086,
"learning_rate": 5.906802900412788e-10,
"logits/chosen": 0.7377203106880188,
"logits/rejected": 0.6763726472854614,
"logps/chosen": -190.40628051757812,
"logps/ref_chosen": -49.30964660644531,
"logps/ref_rejected": -73.73710632324219,
"logps/rejected": -280.4178466796875,
"loss": 1.1397,
"margin_dpo/margin_mean": 65.5841064453125,
"margin_dpo/margin_std": 108.93354034423828,
"step": 649
},
{
"KL/chosen_KL_mean": -135.457763671875,
"KL/mean": -168.88119506835938,
"KL/rejected_KL_mean": -202.3046417236328,
"KL/std": 89.22288513183594,
"epoch": 0.982615268329554,
"fcm_dpo/beta": 0.00575958751142025,
"fcm_dpo/delta": 0.015534860081970692,
"fcm_dpo/margin": 66.84689331054688,
"fcm_dpo/q_t": 0.4122008979320526,
"grad_norm": 12.846329689025879,
"learning_rate": 5.033308820289184e-10,
"logits/chosen": 0.8022534847259521,
"logits/rejected": 0.7341662645339966,
"logps/chosen": -190.52102661132812,
"logps/ref_chosen": -55.06325912475586,
"logps/ref_rejected": -77.39610290527344,
"logps/rejected": -279.70074462890625,
"loss": 1.1343,
"margin_dpo/margin_mean": 66.8469009399414,
"margin_dpo/margin_std": 109.91548156738281,
"step": 650
},
{
"KL/chosen_KL_mean": -136.00405883789062,
"KL/mean": -164.85369873046875,
"KL/rejected_KL_mean": -193.70335388183594,
"KL/std": 90.482421875,
"epoch": 0.9841269841269841,
"fcm_dpo/beta": 0.005842794664204121,
"fcm_dpo/delta": 0.0648837685585022,
"fcm_dpo/margin": 57.69929504394531,
"fcm_dpo/q_t": 0.4242980480194092,
"grad_norm": 12.628108978271484,
"learning_rate": 4.2296043218295606e-10,
"logits/chosen": 0.8145561218261719,
"logits/rejected": 0.7350976467132568,
"logps/chosen": -190.0692138671875,
"logps/ref_chosen": -54.065162658691406,
"logps/ref_rejected": -77.79080200195312,
"logps/rejected": -271.494140625,
"loss": 1.1549,
"margin_dpo/margin_mean": 57.69929122924805,
"margin_dpo/margin_std": 97.94934844970703,
"step": 651
},
{
"KL/chosen_KL_mean": -160.09146118164062,
"KL/mean": -187.93740844726562,
"KL/rejected_KL_mean": -215.7833251953125,
"KL/std": 95.77912139892578,
"epoch": 0.9856386999244142,
"fcm_dpo/beta": 0.005834443029016256,
"fcm_dpo/delta": -0.014574633911252022,
"fcm_dpo/margin": 55.69186019897461,
"fcm_dpo/q_t": 0.427315354347229,
"grad_norm": 14.434667587280273,
"learning_rate": 3.4957118863768176e-10,
"logits/chosen": 0.7214004993438721,
"logits/rejected": 0.670505940914154,
"logps/chosen": -223.7317657470703,
"logps/ref_chosen": -63.64030456542969,
"logps/ref_rejected": -78.86882019042969,
"logps/rejected": -294.65216064453125,
"loss": 1.1923,
"margin_dpo/margin_mean": 55.69186019897461,
"margin_dpo/margin_std": 113.47947692871094,
"step": 652
},
{
"KL/chosen_KL_mean": -143.67312622070312,
"KL/mean": -177.08168029785156,
"KL/rejected_KL_mean": -210.490234375,
"KL/std": 88.8065185546875,
"epoch": 0.9871504157218443,
"fcm_dpo/beta": 0.005856312811374664,
"fcm_dpo/delta": 0.008912090212106705,
"fcm_dpo/margin": 66.8171157836914,
"fcm_dpo/q_t": 0.41040560603141785,
"grad_norm": 14.104089736938477,
"learning_rate": 2.831652042480093e-10,
"logits/chosen": 0.70106041431427,
"logits/rejected": 0.6565027236938477,
"logps/chosen": -205.34149169921875,
"logps/ref_chosen": -61.668373107910156,
"logps/ref_rejected": -73.83012390136719,
"logps/rejected": -284.32037353515625,
"loss": 1.125,
"margin_dpo/margin_mean": 66.8171157836914,
"margin_dpo/margin_std": 106.3432388305664,
"step": 653
},
{
"KL/chosen_KL_mean": -148.42080688476562,
"KL/mean": -177.3148193359375,
"KL/rejected_KL_mean": -206.20880126953125,
"KL/std": 101.04180908203125,
"epoch": 0.9886621315192744,
"fcm_dpo/beta": 0.005777623970061541,
"fcm_dpo/delta": -0.05784344673156738,
"fcm_dpo/margin": 57.78799057006836,
"fcm_dpo/q_t": 0.4234076142311096,
"grad_norm": 13.132534980773926,
"learning_rate": 2.2374433653205016e-10,
"logits/chosen": 0.6848281621932983,
"logits/rejected": 0.582119345664978,
"logps/chosen": -205.98907470703125,
"logps/ref_chosen": -57.568267822265625,
"logps/ref_rejected": -87.74789428710938,
"logps/rejected": -293.9566955566406,
"loss": 1.1595,
"margin_dpo/margin_mean": 57.78799057006836,
"margin_dpo/margin_std": 94.99114990234375,
"step": 654
},
{
"KL/chosen_KL_mean": -116.58991241455078,
"KL/mean": -162.2103271484375,
"KL/rejected_KL_mean": -207.83074951171875,
"KL/std": 91.54662322998047,
"epoch": 0.9901738473167044,
"fcm_dpo/beta": 0.005625207908451557,
"fcm_dpo/delta": -0.12194574624300003,
"fcm_dpo/margin": 91.24083709716797,
"fcm_dpo/q_t": 0.38030263781547546,
"grad_norm": 12.017255783081055,
"learning_rate": 1.7131024761923852e-10,
"logits/chosen": 0.7200064063072205,
"logits/rejected": 0.6282116174697876,
"logps/chosen": -168.737060546875,
"logps/ref_chosen": -52.14714813232422,
"logps/ref_rejected": -80.85014343261719,
"logps/rejected": -288.680908203125,
"loss": 1.0,
"margin_dpo/margin_mean": 91.2408447265625,
"margin_dpo/margin_std": 87.7462387084961,
"step": 655
},
{
"KL/chosen_KL_mean": -151.30734252929688,
"KL/mean": -185.91943359375,
"KL/rejected_KL_mean": -220.5315399169922,
"KL/std": 91.79141235351562,
"epoch": 0.9916855631141346,
"fcm_dpo/beta": 0.005646620877087116,
"fcm_dpo/delta": 0.009487598203122616,
"fcm_dpo/margin": 69.22421264648438,
"fcm_dpo/q_t": 0.4099566340446472,
"grad_norm": 10.540026664733887,
"learning_rate": 1.2586440420372934e-10,
"logits/chosen": 0.6745326519012451,
"logits/rejected": 0.6245888471603394,
"logps/chosen": -224.56405639648438,
"logps/ref_chosen": -73.25672912597656,
"logps/ref_rejected": -85.35127258300781,
"logps/rejected": -305.8828125,
"loss": 1.1119,
"margin_dpo/margin_mean": 69.22420501708984,
"margin_dpo/margin_std": 102.51547241210938,
"step": 656
},
{
"KL/chosen_KL_mean": -137.4180908203125,
"KL/mean": -177.50393676757812,
"KL/rejected_KL_mean": -217.5897979736328,
"KL/std": 101.89553833007812,
"epoch": 0.9931972789115646,
"fcm_dpo/beta": 0.005607600323855877,
"fcm_dpo/delta": -0.051889002323150635,
"fcm_dpo/margin": 80.17169952392578,
"fcm_dpo/q_t": 0.39736613631248474,
"grad_norm": 10.760099411010742,
"learning_rate": 8.740807750345913e-11,
"logits/chosen": 0.857367217540741,
"logits/rejected": 0.7622960209846497,
"logps/chosen": -187.14149475097656,
"logps/ref_chosen": -49.72339630126953,
"logps/ref_rejected": -75.1568603515625,
"logps/rejected": -292.74664306640625,
"loss": 1.0801,
"margin_dpo/margin_mean": 80.17170715332031,
"margin_dpo/margin_std": 112.73518371582031,
"step": 657
},
{
"KL/chosen_KL_mean": -144.88543701171875,
"KL/mean": -176.97201538085938,
"KL/rejected_KL_mean": -209.05856323242188,
"KL/std": 100.89071655273438,
"epoch": 0.9947089947089947,
"fcm_dpo/beta": 0.005655559711158276,
"fcm_dpo/delta": 0.03752633184194565,
"fcm_dpo/margin": 64.17312622070312,
"fcm_dpo/q_t": 0.41788923740386963,
"grad_norm": 11.86836051940918,
"learning_rate": 5.594234322453539e-11,
"logits/chosen": 0.7775052189826965,
"logits/rejected": 0.7297263741493225,
"logps/chosen": -207.93179321289062,
"logps/ref_chosen": -63.04634094238281,
"logps/ref_rejected": -83.44963073730469,
"logps/rejected": -292.5081787109375,
"loss": 1.1648,
"margin_dpo/margin_mean": 64.17313385009766,
"margin_dpo/margin_std": 118.98008728027344,
"step": 658
},
{
"KL/chosen_KL_mean": -149.5328369140625,
"KL/mean": -174.78045654296875,
"KL/rejected_KL_mean": -200.028076171875,
"KL/std": 92.56729125976562,
"epoch": 0.9962207105064248,
"fcm_dpo/beta": 0.00562618812546134,
"fcm_dpo/delta": -0.0059813628904521465,
"fcm_dpo/margin": 50.49524688720703,
"fcm_dpo/q_t": 0.43419986963272095,
"grad_norm": 16.764509201049805,
"learning_rate": 3.146808153123293e-11,
"logits/chosen": 0.8269628286361694,
"logits/rejected": 0.7587199807167053,
"logps/chosen": -204.613037109375,
"logps/ref_chosen": -55.0802001953125,
"logps/ref_rejected": -71.91049194335938,
"logps/rejected": -271.9385681152344,
"loss": 1.2121,
"margin_dpo/margin_mean": 50.49524688720703,
"margin_dpo/margin_std": 106.60450744628906,
"step": 659
},
{
"KL/chosen_KL_mean": -140.72540283203125,
"KL/mean": -180.6097869873047,
"KL/rejected_KL_mean": -220.49417114257812,
"KL/std": 96.89041137695312,
"epoch": 0.9977324263038548,
"fcm_dpo/beta": 0.005590518936514854,
"fcm_dpo/delta": -0.04806827753782272,
"fcm_dpo/margin": 79.76875305175781,
"fcm_dpo/q_t": 0.3983927369117737,
"grad_norm": 12.417089462280273,
"learning_rate": 1.3985977021235829e-11,
"logits/chosen": 0.8573871850967407,
"logits/rejected": 0.7809255123138428,
"logps/chosen": -195.25131225585938,
"logps/ref_chosen": -54.525917053222656,
"logps/ref_rejected": -81.23604583740234,
"logps/rejected": -301.730224609375,
"loss": 1.0645,
"margin_dpo/margin_mean": 79.76876831054688,
"margin_dpo/margin_std": 103.05412292480469,
"step": 660
},
{
"KL/chosen_KL_mean": -161.6375732421875,
"KL/mean": -187.66415405273438,
"KL/rejected_KL_mean": -213.69073486328125,
"KL/std": 99.74840545654297,
"epoch": 0.999244142101285,
"fcm_dpo/beta": 0.005673976615071297,
"fcm_dpo/delta": 0.10753720253705978,
"fcm_dpo/margin": 52.05316162109375,
"fcm_dpo/q_t": 0.4330148696899414,
"grad_norm": 13.184820175170898,
"learning_rate": 3.4965187065971735e-12,
"logits/chosen": 0.6973075866699219,
"logits/rejected": 0.6154038906097412,
"logps/chosen": -222.0102081298828,
"logps/ref_chosen": -60.37263870239258,
"logps/ref_rejected": -77.42874145507812,
"logps/rejected": -291.1195068359375,
"loss": 1.2144,
"margin_dpo/margin_mean": 52.053165435791016,
"margin_dpo/margin_std": 115.87824249267578,
"step": 661
},
{
"epoch": 0.999244142101285,
"step": 661,
"total_flos": 0.0,
"train_loss": 1.1380426484229165,
"train_runtime": 1650.6898,
"train_samples_per_second": 25.647,
"train_steps_per_second": 0.4
}
],
"logging_steps": 1,
"max_steps": 661,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink