W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.6-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC d0e9947175 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.6-20260428-045924
Source: Original Platform
2026-05-20 08:46:52 +08:00

15707 lines
590 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.53045654296875,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.22181701660156,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.005462646484375,
"KL/mean": 0.008471250534057617,
"KL/rejected_KL_mean": 0.011478424072265625,
"KL/std": 0.2345641404390335,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.006016969680786133,
"fcm_dpo/q_t": 0.5001506805419922,
"grad_norm": 71.05252838134766,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.48156821727752686,
"logits/rejected": -0.4421899914741516,
"logps/chosen": -60.97613525390625,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.66111755371094,
"loss": 1.3872,
"margin_dpo/margin_mean": -0.006016522645950317,
"margin_dpo/margin_std": 0.34941720962524414,
"step": 3
},
{
"KL/chosen_KL_mean": -0.03255462646484375,
"KL/mean": 0.011689022183418274,
"KL/rejected_KL_mean": 0.055927276611328125,
"KL/std": 0.2458762526512146,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.08848562836647034,
"fcm_dpo/q_t": 0.5022112131118774,
"grad_norm": 72.06624603271484,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.46879148483276367,
"logits/rejected": -0.4411749243736267,
"logps/chosen": -56.80026626586914,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.5911865234375,
"loss": 1.3954,
"margin_dpo/margin_mean": -0.08848586678504944,
"margin_dpo/margin_std": 0.3339208662509918,
"step": 4
},
{
"KL/chosen_KL_mean": 0.041858673095703125,
"KL/mean": 0.02397400140762329,
"KL/rejected_KL_mean": 0.0060882568359375,
"KL/std": 0.26151031255722046,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.035768210887908936,
"fcm_dpo/q_t": 0.4991059899330139,
"grad_norm": 89.50914764404297,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5147285461425781,
"logits/rejected": -0.471005380153656,
"logps/chosen": -53.8175163269043,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.14309692382812,
"loss": 1.383,
"margin_dpo/margin_mean": 0.03576841950416565,
"margin_dpo/margin_std": 0.3583017587661743,
"step": 5
},
{
"KL/chosen_KL_mean": -0.03546333312988281,
"KL/mean": -0.032052695751190186,
"KL/rejected_KL_mean": -0.0286407470703125,
"KL/std": 0.3007926642894745,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.006822705268859863,
"fcm_dpo/q_t": 0.5001715421676636,
"grad_norm": 91.98088073730469,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5223523378372192,
"logits/rejected": -0.48172158002853394,
"logps/chosen": -63.042945861816406,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.67398834228516,
"loss": 1.3874,
"margin_dpo/margin_mean": -0.006822347640991211,
"margin_dpo/margin_std": 0.4276465177536011,
"step": 6
},
{
"KL/chosen_KL_mean": 0.00897979736328125,
"KL/mean": -0.011391326785087585,
"KL/rejected_KL_mean": -0.03176116943359375,
"KL/std": 0.3294689655303955,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0407429039478302,
"fcm_dpo/q_t": 0.4989830255508423,
"grad_norm": 82.00981903076172,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.5042591691017151,
"logits/rejected": -0.47041141986846924,
"logps/chosen": -57.765838623046875,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.95235443115234,
"loss": 1.3829,
"margin_dpo/margin_mean": 0.04074329137802124,
"margin_dpo/margin_std": 0.5175560712814331,
"step": 7
},
{
"KL/chosen_KL_mean": -0.00151824951171875,
"KL/mean": 0.008417114615440369,
"KL/rejected_KL_mean": 0.0183563232421875,
"KL/std": 0.2529047727584839,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.019877761602401733,
"fcm_dpo/q_t": 0.5004963278770447,
"grad_norm": 78.8486099243164,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.4955812096595764,
"logits/rejected": -0.4683501720428467,
"logps/chosen": -58.717552185058594,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.2930679321289,
"loss": 1.3886,
"margin_dpo/margin_mean": -0.019877701997756958,
"margin_dpo/margin_std": 0.34278473258018494,
"step": 8
},
{
"KL/chosen_KL_mean": 0.009731292724609375,
"KL/mean": 0.02583947777748108,
"KL/rejected_KL_mean": 0.0419464111328125,
"KL/std": 0.3126106262207031,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03221386671066284,
"fcm_dpo/q_t": 0.5008052587509155,
"grad_norm": 85.40772247314453,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.5120226740837097,
"logits/rejected": -0.4663524627685547,
"logps/chosen": -69.85711669921875,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.56071472167969,
"loss": 1.39,
"margin_dpo/margin_mean": -0.032214969396591187,
"margin_dpo/margin_std": 0.4174457788467407,
"step": 9
},
{
"KL/chosen_KL_mean": 0.019561767578125,
"KL/mean": 0.016787193715572357,
"KL/rejected_KL_mean": 0.01401519775390625,
"KL/std": 0.22859910130500793,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0055475980043411255,
"fcm_dpo/q_t": 0.49986132979393005,
"grad_norm": 70.78611755371094,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.4891882538795471,
"logits/rejected": -0.44570988416671753,
"logps/chosen": -48.3381233215332,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.35804748535156,
"loss": 1.386,
"margin_dpo/margin_mean": 0.005548357963562012,
"margin_dpo/margin_std": 0.2920377254486084,
"step": 10
},
{
"KL/chosen_KL_mean": 0.012109756469726562,
"KL/mean": 0.002375006675720215,
"KL/rejected_KL_mean": -0.00736236572265625,
"KL/std": 0.2516428828239441,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.019462406635284424,
"fcm_dpo/q_t": 0.49951374530792236,
"grad_norm": 68.14494323730469,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4490339756011963,
"logits/rejected": -0.4234827160835266,
"logps/chosen": -53.00474548339844,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.78773498535156,
"loss": 1.3847,
"margin_dpo/margin_mean": 0.01946243643760681,
"margin_dpo/margin_std": 0.38668665289878845,
"step": 11
},
{
"KL/chosen_KL_mean": -0.008607864379882812,
"KL/mean": 0.016416624188423157,
"KL/rejected_KL_mean": 0.041446685791015625,
"KL/std": 0.3268115222454071,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05004730820655823,
"fcm_dpo/q_t": 0.5012508630752563,
"grad_norm": 89.79679107666016,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5429999828338623,
"logits/rejected": -0.5069589614868164,
"logps/chosen": -61.81404495239258,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.81682586669922,
"loss": 1.3918,
"margin_dpo/margin_mean": -0.05004820227622986,
"margin_dpo/margin_std": 0.43947750329971313,
"step": 12
},
{
"KL/chosen_KL_mean": 0.026720046997070312,
"KL/mean": 0.002465367317199707,
"KL/rejected_KL_mean": -0.021785736083984375,
"KL/std": 0.26591062545776367,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04851400852203369,
"fcm_dpo/q_t": 0.49878764152526855,
"grad_norm": 78.82386016845703,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.4669307470321655,
"logits/rejected": -0.4377588927745819,
"logps/chosen": -64.23363494873047,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.22486877441406,
"loss": 1.3818,
"margin_dpo/margin_mean": 0.0485135018825531,
"margin_dpo/margin_std": 0.37014269828796387,
"step": 13
},
{
"KL/chosen_KL_mean": -0.019222259521484375,
"KL/mean": -0.014583319425582886,
"KL/rejected_KL_mean": -0.00994873046875,
"KL/std": 0.2809370756149292,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.00926673412322998,
"fcm_dpo/q_t": 0.500230610370636,
"grad_norm": 85.25773620605469,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.43502137064933777,
"logits/rejected": -0.3942975103855133,
"logps/chosen": -58.129432678222656,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.0570297241211,
"loss": 1.3876,
"margin_dpo/margin_mean": -0.00926661491394043,
"margin_dpo/margin_std": 0.39651691913604736,
"step": 14
},
{
"KL/chosen_KL_mean": -0.023839950561523438,
"KL/mean": -0.03170686960220337,
"KL/rejected_KL_mean": -0.039577484130859375,
"KL/std": 0.2464873492717743,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.015737533569335938,
"fcm_dpo/q_t": 0.49960604310035706,
"grad_norm": 64.12519073486328,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.4756002724170685,
"logits/rejected": -0.4559728801250458,
"logps/chosen": -56.99074935913086,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.84822082519531,
"loss": 1.3851,
"margin_dpo/margin_mean": 0.015737563371658325,
"margin_dpo/margin_std": 0.3705330491065979,
"step": 15
},
{
"KL/chosen_KL_mean": -0.018383026123046875,
"KL/mean": -0.03371742367744446,
"KL/rejected_KL_mean": -0.04904937744140625,
"KL/std": 0.2746584713459015,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.030667126178741455,
"fcm_dpo/q_t": 0.4992336332798004,
"grad_norm": 84.03563690185547,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5321472883224487,
"logits/rejected": -0.4919458329677582,
"logps/chosen": -61.758277893066406,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.41851806640625,
"loss": 1.3836,
"margin_dpo/margin_mean": 0.030666619539260864,
"margin_dpo/margin_std": 0.3938596844673157,
"step": 16
},
{
"KL/chosen_KL_mean": -0.004093170166015625,
"KL/mean": -0.00830143690109253,
"KL/rejected_KL_mean": -0.01251220703125,
"KL/std": 0.27522557973861694,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.00841444730758667,
"fcm_dpo/q_t": 0.49978873133659363,
"grad_norm": 79.06344604492188,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.49493852257728577,
"logits/rejected": -0.45694345235824585,
"logps/chosen": -67.71443176269531,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.39116668701172,
"loss": 1.3859,
"margin_dpo/margin_mean": 0.008414477109909058,
"margin_dpo/margin_std": 0.41055363416671753,
"step": 17
},
{
"KL/chosen_KL_mean": -0.032901763916015625,
"KL/mean": -0.02767445147037506,
"KL/rejected_KL_mean": -0.0224456787109375,
"KL/std": 0.2769307494163513,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.010453850030899048,
"fcm_dpo/q_t": 0.5002602934837341,
"grad_norm": 82.40819549560547,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.48359060287475586,
"logits/rejected": -0.427044153213501,
"logps/chosen": -47.77239227294922,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.49473571777344,
"loss": 1.3878,
"margin_dpo/margin_mean": -0.010454446077346802,
"margin_dpo/margin_std": 0.4250386953353882,
"step": 18
},
{
"KL/chosen_KL_mean": 0.036823272705078125,
"KL/mean": -0.018123507499694824,
"KL/rejected_KL_mean": -0.07306671142578125,
"KL/std": 0.252922385931015,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10988301038742065,
"fcm_dpo/q_t": 0.4972538352012634,
"grad_norm": 73.24119567871094,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.47332119941711426,
"logits/rejected": -0.4219907522201538,
"logps/chosen": -70.16853332519531,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.83065795898438,
"loss": 1.3757,
"margin_dpo/margin_mean": 0.10988333821296692,
"margin_dpo/margin_std": 0.3672480881214142,
"step": 19
},
{
"KL/chosen_KL_mean": -0.030652999877929688,
"KL/mean": -0.0650777667760849,
"KL/rejected_KL_mean": -0.0995025634765625,
"KL/std": 0.2643129825592041,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06885161995887756,
"fcm_dpo/q_t": 0.49827927350997925,
"grad_norm": 73.5529556274414,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5481554269790649,
"logits/rejected": -0.4914012551307678,
"logps/chosen": -50.833892822265625,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.9228515625,
"loss": 1.3798,
"margin_dpo/margin_mean": 0.06885164976119995,
"margin_dpo/margin_std": 0.3669779896736145,
"step": 20
},
{
"KL/chosen_KL_mean": 0.0008487701416015625,
"KL/mean": -0.04886701703071594,
"KL/rejected_KL_mean": -0.09858322143554688,
"KL/std": 0.29678937792778015,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09943097829818726,
"fcm_dpo/q_t": 0.49751463532447815,
"grad_norm": 77.5016860961914,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.49147945642471313,
"logits/rejected": -0.4672367572784424,
"logps/chosen": -50.06216812133789,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.96737670898438,
"loss": 1.3767,
"margin_dpo/margin_mean": 0.09943100810050964,
"margin_dpo/margin_std": 0.37303319573402405,
"step": 21
},
{
"KL/chosen_KL_mean": 0.045787811279296875,
"KL/mean": -0.051998257637023926,
"KL/rejected_KL_mean": -0.14978408813476562,
"KL/std": 0.30449533462524414,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1955808401107788,
"fcm_dpo/q_t": 0.4951126277446747,
"grad_norm": 84.59005737304688,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.49251508712768555,
"logits/rejected": -0.44915539026260376,
"logps/chosen": -59.01184844970703,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.65445709228516,
"loss": 1.3673,
"margin_dpo/margin_mean": 0.19558033347129822,
"margin_dpo/margin_std": 0.42738014459609985,
"step": 22
},
{
"KL/chosen_KL_mean": 0.058322906494140625,
"KL/mean": -0.0587589293718338,
"KL/rejected_KL_mean": -0.17584228515625,
"KL/std": 0.31453025341033936,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23416772484779358,
"fcm_dpo/q_t": 0.49415063858032227,
"grad_norm": 80.18136596679688,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.49351799488067627,
"logits/rejected": -0.4712684154510498,
"logps/chosen": -60.019371032714844,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.31539916992188,
"loss": 1.3635,
"margin_dpo/margin_mean": 0.23416772484779358,
"margin_dpo/margin_std": 0.4289299249649048,
"step": 23
},
{
"KL/chosen_KL_mean": 0.018413543701171875,
"KL/mean": -0.09353002905845642,
"KL/rejected_KL_mean": -0.20547866821289062,
"KL/std": 0.3055531978607178,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2238883376121521,
"fcm_dpo/q_t": 0.49440497159957886,
"grad_norm": 82.01287841796875,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5203529596328735,
"logits/rejected": -0.5036816000938416,
"logps/chosen": -44.272621154785156,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.3306884765625,
"loss": 1.3644,
"margin_dpo/margin_mean": 0.22388812899589539,
"margin_dpo/margin_std": 0.3889973759651184,
"step": 24
},
{
"KL/chosen_KL_mean": 0.0025882720947265625,
"KL/mean": -0.12222906947135925,
"KL/rejected_KL_mean": -0.247039794921875,
"KL/std": 0.3658377528190613,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2496315836906433,
"fcm_dpo/q_t": 0.49376434087753296,
"grad_norm": 73.92852020263672,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.4850538969039917,
"logits/rejected": -0.4546140432357788,
"logps/chosen": -52.534461975097656,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.5892333984375,
"loss": 1.362,
"margin_dpo/margin_mean": 0.24963170289993286,
"margin_dpo/margin_std": 0.44056153297424316,
"step": 25
},
{
"KL/chosen_KL_mean": 0.08319664001464844,
"KL/mean": -0.12143772840499878,
"KL/rejected_KL_mean": -0.32607269287109375,
"KL/std": 0.41855013370513916,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.40927183628082275,
"fcm_dpo/q_t": 0.48977863788604736,
"grad_norm": 87.61016082763672,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5254790782928467,
"logits/rejected": -0.4935619533061981,
"logps/chosen": -53.8396110534668,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.685791015625,
"loss": 1.3465,
"margin_dpo/margin_mean": 0.4092727303504944,
"margin_dpo/margin_std": 0.5137361288070679,
"step": 26
},
{
"KL/chosen_KL_mean": 0.1279277801513672,
"KL/mean": -0.09298427402973175,
"KL/rejected_KL_mean": -0.3138999938964844,
"KL/std": 0.4433329701423645,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4418245851993561,
"fcm_dpo/q_t": 0.4889661371707916,
"grad_norm": 93.73269653320312,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5290693640708923,
"logits/rejected": -0.4929000735282898,
"logps/chosen": -42.770599365234375,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.03810119628906,
"loss": 1.3433,
"margin_dpo/margin_mean": 0.4418243169784546,
"margin_dpo/margin_std": 0.5311126708984375,
"step": 27
},
{
"KL/chosen_KL_mean": 0.026353836059570312,
"KL/mean": -0.14182431995868683,
"KL/rejected_KL_mean": -0.3100013732910156,
"KL/std": 0.40777623653411865,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.33634987473487854,
"fcm_dpo/q_t": 0.4915998578071594,
"grad_norm": 75.1212158203125,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5229371786117554,
"logits/rejected": -0.4693087339401245,
"logps/chosen": -60.530147552490234,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.71112060546875,
"loss": 1.3538,
"margin_dpo/margin_mean": 0.33634981513023376,
"margin_dpo/margin_std": 0.5798689126968384,
"step": 28
},
{
"KL/chosen_KL_mean": 0.08238792419433594,
"KL/mean": -0.1901816427707672,
"KL/rejected_KL_mean": -0.4627494812011719,
"KL/std": 0.5029264092445374,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5451391339302063,
"fcm_dpo/q_t": 0.48638832569122314,
"grad_norm": 90.61625671386719,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5645961761474609,
"logits/rejected": -0.5192388296127319,
"logps/chosen": -57.72539520263672,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.85710144042969,
"loss": 1.3334,
"margin_dpo/margin_mean": 0.5451398491859436,
"margin_dpo/margin_std": 0.6029021739959717,
"step": 29
},
{
"KL/chosen_KL_mean": 0.13669967651367188,
"KL/mean": -0.20837418735027313,
"KL/rejected_KL_mean": -0.5534515380859375,
"KL/std": 0.5482817888259888,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6901483535766602,
"fcm_dpo/q_t": 0.48277321457862854,
"grad_norm": 87.55204010009766,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.49230870604515076,
"logits/rejected": -0.4614727795124054,
"logps/chosen": -52.440673828125,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.04265594482422,
"loss": 1.3194,
"margin_dpo/margin_mean": 0.6901484727859497,
"margin_dpo/margin_std": 0.5979875326156616,
"step": 30
},
{
"KL/chosen_KL_mean": 0.09287834167480469,
"KL/mean": -0.14101067185401917,
"KL/rejected_KL_mean": -0.3749046325683594,
"KL/std": 0.5449085235595703,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.46777844429016113,
"fcm_dpo/q_t": 0.48832568526268005,
"grad_norm": 68.1603012084961,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.49934709072113037,
"logits/rejected": -0.4538087844848633,
"logps/chosen": -63.71404266357422,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.26890563964844,
"loss": 1.3411,
"margin_dpo/margin_mean": 0.4677784740924835,
"margin_dpo/margin_std": 0.6523255109786987,
"step": 31
},
{
"KL/chosen_KL_mean": 0.20739173889160156,
"KL/mean": -0.186039000749588,
"KL/rejected_KL_mean": -0.5794715881347656,
"KL/std": 0.7225437164306641,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.786858856678009,
"fcm_dpo/q_t": 0.48040735721588135,
"grad_norm": 81.92281341552734,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.49706003069877625,
"logits/rejected": -0.4546694755554199,
"logps/chosen": -62.532135009765625,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.89697265625,
"loss": 1.3113,
"margin_dpo/margin_mean": 0.7868585586547852,
"margin_dpo/margin_std": 0.9216699600219727,
"step": 32
},
{
"KL/chosen_KL_mean": 0.09073448181152344,
"KL/mean": -0.189952552318573,
"KL/rejected_KL_mean": -0.47064208984375,
"KL/std": 0.574708104133606,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5613725781440735,
"fcm_dpo/q_t": 0.4859866797924042,
"grad_norm": 72.56104278564453,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.48183947801589966,
"logits/rejected": -0.45549333095550537,
"logps/chosen": -53.17023849487305,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.35577392578125,
"loss": 1.332,
"margin_dpo/margin_mean": 0.5613729953765869,
"margin_dpo/margin_std": 0.6500346660614014,
"step": 33
},
{
"KL/chosen_KL_mean": 0.07518196105957031,
"KL/mean": -0.32575562596321106,
"KL/rejected_KL_mean": -0.7266960144042969,
"KL/std": 0.7734376192092896,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8018748760223389,
"fcm_dpo/q_t": 0.4800262153148651,
"grad_norm": 77.36227416992188,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.4970749318599701,
"logits/rejected": -0.47958844900131226,
"logps/chosen": -50.74214553833008,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.64854431152344,
"loss": 1.3101,
"margin_dpo/margin_mean": 0.801874041557312,
"margin_dpo/margin_std": 0.9598724842071533,
"step": 34
},
{
"KL/chosen_KL_mean": 0.1396312713623047,
"KL/mean": -0.4566376209259033,
"KL/rejected_KL_mean": -1.0529060363769531,
"KL/std": 0.9779422283172607,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1925348043441772,
"fcm_dpo/q_t": 0.47035083174705505,
"grad_norm": 82.61671447753906,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5158413648605347,
"logits/rejected": -0.47896769642829895,
"logps/chosen": -50.884857177734375,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.87733459472656,
"loss": 1.2735,
"margin_dpo/margin_mean": 1.1925346851348877,
"margin_dpo/margin_std": 1.0775500535964966,
"step": 35
},
{
"KL/chosen_KL_mean": 0.013763427734375,
"KL/mean": -0.5477051734924316,
"KL/rejected_KL_mean": -1.1091690063476562,
"KL/std": 1.0905871391296387,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1229325532913208,
"fcm_dpo/q_t": 0.472078800201416,
"grad_norm": 72.85025787353516,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5630701184272766,
"logits/rejected": -0.5269886255264282,
"logps/chosen": -51.97772979736328,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.14979553222656,
"loss": 1.281,
"margin_dpo/margin_mean": 1.1229320764541626,
"margin_dpo/margin_std": 1.2045643329620361,
"step": 36
},
{
"KL/chosen_KL_mean": -0.00809478759765625,
"KL/mean": -0.5153483152389526,
"KL/rejected_KL_mean": -1.0226058959960938,
"KL/std": 1.0523146390914917,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.014506220817566,
"fcm_dpo/q_t": 0.4748191237449646,
"grad_norm": 61.76152038574219,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5344172120094299,
"logits/rejected": -0.49267613887786865,
"logps/chosen": -62.81520080566406,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.91767883300781,
"loss": 1.2919,
"margin_dpo/margin_mean": 1.0145056247711182,
"margin_dpo/margin_std": 1.287419080734253,
"step": 37
},
{
"KL/chosen_KL_mean": 0.17576217651367188,
"KL/mean": -0.5427882671356201,
"KL/rejected_KL_mean": -1.2613372802734375,
"KL/std": 1.4168626070022583,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4370999336242676,
"fcm_dpo/q_t": 0.4646208882331848,
"grad_norm": 68.95872497558594,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5328177809715271,
"logits/rejected": -0.5000877976417542,
"logps/chosen": -48.214759826660156,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.17378234863281,
"loss": 1.2556,
"margin_dpo/margin_mean": 1.4370996952056885,
"margin_dpo/margin_std": 1.7722265720367432,
"step": 38
},
{
"KL/chosen_KL_mean": 0.08198738098144531,
"KL/mean": -0.7597079277038574,
"KL/rejected_KL_mean": -1.6013984680175781,
"KL/std": 1.309003472328186,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6833879947662354,
"fcm_dpo/q_t": 0.4582422375679016,
"grad_norm": 72.92730712890625,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.539905846118927,
"logits/rejected": -0.49894577264785767,
"logps/chosen": -50.668487548828125,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.17091369628906,
"loss": 1.2301,
"margin_dpo/margin_mean": 1.683387279510498,
"margin_dpo/margin_std": 1.413730502128601,
"step": 39
},
{
"KL/chosen_KL_mean": 0.24016761779785156,
"KL/mean": -0.545630693435669,
"KL/rejected_KL_mean": -1.3314285278320312,
"KL/std": 1.3814085721969604,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5715957880020142,
"fcm_dpo/q_t": 0.46117883920669556,
"grad_norm": 60.406272888183594,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5031940937042236,
"logits/rejected": -0.47205477952957153,
"logps/chosen": -57.74489974975586,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.63150024414062,
"loss": 1.2421,
"margin_dpo/margin_mean": 1.5715968608856201,
"margin_dpo/margin_std": 1.6131068468093872,
"step": 40
},
{
"KL/chosen_KL_mean": 0.038684844970703125,
"KL/mean": -0.956620454788208,
"KL/rejected_KL_mean": -1.9519233703613281,
"KL/std": 1.903930425643921,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10199292004108429,
"fcm_dpo/delta": 0.1954224854707718,
"fcm_dpo/margin": 1.9906086921691895,
"fcm_dpo/q_t": 0.4510888457298279,
"grad_norm": 69.02500915527344,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5088913440704346,
"logits/rejected": -0.47061896324157715,
"logps/chosen": -62.65713119506836,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.97544860839844,
"loss": 1.2044,
"margin_dpo/margin_mean": 1.9906091690063477,
"margin_dpo/margin_std": 2.014521837234497,
"step": 41
},
{
"KL/chosen_KL_mean": 0.20633697509765625,
"KL/mean": -1.0914945602416992,
"KL/rejected_KL_mean": -2.389324188232422,
"KL/std": 2.13301944732666,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.1092967838048935,
"fcm_dpo/delta": 0.32513314485549927,
"fcm_dpo/margin": 2.5956621170043945,
"fcm_dpo/q_t": 0.4330694079399109,
"grad_norm": 84.8261947631836,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5458312630653381,
"logits/rejected": -0.49941253662109375,
"logps/chosen": -58.76008987426758,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.29769897460938,
"loss": 1.1385,
"margin_dpo/margin_mean": 2.5956621170043945,
"margin_dpo/margin_std": 2.356515407562256,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5495738983154297,
"KL/mean": -0.7069127559661865,
"KL/rejected_KL_mean": -1.9634017944335938,
"KL/std": 1.8634235858917236,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.11624425649642944,
"fcm_dpo/delta": 0.3174755573272705,
"fcm_dpo/margin": 2.5129733085632324,
"fcm_dpo/q_t": 0.4308740496635437,
"grad_norm": 80.33716583251953,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5390653610229492,
"logits/rejected": -0.5141108632087708,
"logps/chosen": -53.606422424316406,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.44358825683594,
"loss": 1.129,
"margin_dpo/margin_mean": 2.5129730701446533,
"margin_dpo/margin_std": 1.9879181385040283,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2190093994140625,
"KL/mean": -1.1946829557418823,
"KL/rejected_KL_mean": -2.608379364013672,
"KL/std": 2.2951462268829346,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.1229436844587326,
"fcm_dpo/delta": 0.2615148425102234,
"fcm_dpo/margin": 2.827385425567627,
"fcm_dpo/q_t": 0.4178880453109741,
"grad_norm": 91.2223129272461,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.44505244493484497,
"logits/rejected": -0.42579957842826843,
"logps/chosen": -49.85948944091797,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.39213562011719,
"loss": 1.0872,
"margin_dpo/margin_mean": 2.827385902404785,
"margin_dpo/margin_std": 2.2556262016296387,
"step": 44
},
{
"KL/chosen_KL_mean": 0.1596202850341797,
"KL/mean": -0.9914059042930603,
"KL/rejected_KL_mean": -2.1424331665039062,
"KL/std": 2.0859012603759766,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.130710631608963,
"fcm_dpo/delta": 0.3077217638492584,
"fcm_dpo/margin": 2.302053928375244,
"fcm_dpo/q_t": 0.43017762899398804,
"grad_norm": 74.77536010742188,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.489723801612854,
"logits/rejected": -0.4772561490535736,
"logps/chosen": -48.25531005859375,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.078857421875,
"loss": 1.1358,
"margin_dpo/margin_mean": 2.302053451538086,
"margin_dpo/margin_std": 2.5962462425231934,
"step": 45
},
{
"KL/chosen_KL_mean": 0.18535804748535156,
"KL/mean": -1.268411636352539,
"KL/rejected_KL_mean": -2.722187042236328,
"KL/std": 2.5633392333984375,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.1369568109512329,
"fcm_dpo/delta": 0.20972974598407745,
"fcm_dpo/margin": 2.9075417518615723,
"fcm_dpo/q_t": 0.4084067940711975,
"grad_norm": 87.37943267822266,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5191621780395508,
"logits/rejected": -0.4675208628177643,
"logps/chosen": -55.81406784057617,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.37477111816406,
"loss": 1.0677,
"margin_dpo/margin_mean": 2.9075417518615723,
"margin_dpo/margin_std": 3.0763049125671387,
"step": 46
},
{
"KL/chosen_KL_mean": 0.3790416717529297,
"KL/mean": -1.0936341285705566,
"KL/rejected_KL_mean": -2.5663070678710938,
"KL/std": 2.5994389057159424,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.1426984965801239,
"fcm_dpo/delta": 0.1863730251789093,
"fcm_dpo/margin": 2.94534969329834,
"fcm_dpo/q_t": 0.4023776650428772,
"grad_norm": 83.57400512695312,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.572807788848877,
"logits/rejected": -0.5201467871665955,
"logps/chosen": -57.547035217285156,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.24551391601562,
"loss": 1.0467,
"margin_dpo/margin_mean": 2.945349931716919,
"margin_dpo/margin_std": 2.759472370147705,
"step": 47
},
{
"KL/chosen_KL_mean": 0.055755615234375,
"KL/mean": -1.5136513710021973,
"KL/rejected_KL_mean": -3.083057403564453,
"KL/std": 2.7071356773376465,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.14606741070747375,
"fcm_dpo/delta": 0.14858855307102203,
"fcm_dpo/margin": 3.1388156414031982,
"fcm_dpo/q_t": 0.39447665214538574,
"grad_norm": 93.96417999267578,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5753400325775146,
"logits/rejected": -0.51603102684021,
"logps/chosen": -57.13231658935547,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.09965515136719,
"loss": 1.03,
"margin_dpo/margin_mean": 3.1388158798217773,
"margin_dpo/margin_std": 2.966205596923828,
"step": 48
},
{
"KL/chosen_KL_mean": 0.3167591094970703,
"KL/mean": -1.5836272239685059,
"KL/rejected_KL_mean": -3.4840126037597656,
"KL/std": 3.3755598068237305,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.14929595589637756,
"fcm_dpo/delta": 0.03374991938471794,
"fcm_dpo/margin": 3.8007659912109375,
"fcm_dpo/q_t": 0.3742704391479492,
"grad_norm": 97.30853271484375,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.558617115020752,
"logits/rejected": -0.5012648105621338,
"logps/chosen": -61.36851119995117,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.2514877319336,
"loss": 0.9725,
"margin_dpo/margin_mean": 3.800764799118042,
"margin_dpo/margin_std": 3.881669759750366,
"step": 49
},
{
"KL/chosen_KL_mean": -0.190521240234375,
"KL/mean": -2.244612216949463,
"KL/rejected_KL_mean": -4.298698425292969,
"KL/std": 3.501471757888794,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.148372083902359,
"fcm_dpo/delta": -0.010192867368459702,
"fcm_dpo/margin": 4.108177185058594,
"fcm_dpo/q_t": 0.36416471004486084,
"grad_norm": 80.24071502685547,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5547747611999512,
"logits/rejected": -0.5192065834999084,
"logps/chosen": -58.91465759277344,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.65684509277344,
"loss": 0.9436,
"margin_dpo/margin_mean": 4.108177185058594,
"margin_dpo/margin_std": 4.009483337402344,
"step": 50
},
{
"KL/chosen_KL_mean": -0.25165367126464844,
"KL/mean": -2.280780553817749,
"KL/rejected_KL_mean": -4.309909820556641,
"KL/std": 4.161914825439453,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.14748243987560272,
"fcm_dpo/delta": 0.0006546974182128906,
"fcm_dpo/margin": 4.058258056640625,
"fcm_dpo/q_t": 0.3747638463973999,
"grad_norm": 65.57543182373047,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.4971970319747925,
"logits/rejected": -0.46293967962265015,
"logps/chosen": -61.62532043457031,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.3119125366211,
"loss": 0.9931,
"margin_dpo/margin_mean": 4.058258056640625,
"margin_dpo/margin_std": 5.099086761474609,
"step": 51
},
{
"KL/chosen_KL_mean": 0.39443016052246094,
"KL/mean": -2.67549467086792,
"KL/rejected_KL_mean": -5.74542236328125,
"KL/std": 4.730856418609619,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.14197938144207,
"fcm_dpo/delta": -0.2979382872581482,
"fcm_dpo/margin": 6.139850616455078,
"fcm_dpo/q_t": 0.31120362877845764,
"grad_norm": 67.91552734375,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5659775733947754,
"logits/rejected": -0.5114268660545349,
"logps/chosen": -51.94292449951172,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.7193374633789,
"loss": 0.7956,
"margin_dpo/margin_mean": 6.139851093292236,
"margin_dpo/margin_std": 5.022448539733887,
"step": 52
},
{
"KL/chosen_KL_mean": -0.21327590942382812,
"KL/mean": -3.2701005935668945,
"KL/rejected_KL_mean": -6.326923370361328,
"KL/std": 5.305433750152588,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.1361648589372635,
"fcm_dpo/delta": -0.2531626522541046,
"fcm_dpo/margin": 6.113650321960449,
"fcm_dpo/q_t": 0.33245524764060974,
"grad_norm": 73.4737777709961,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.624061107635498,
"logits/rejected": -0.602851152420044,
"logps/chosen": -53.52792739868164,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.11051940917969,
"loss": 0.8631,
"margin_dpo/margin_mean": 6.113650321960449,
"margin_dpo/margin_std": 6.09869909286499,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3877124786376953,
"KL/mean": -3.0379018783569336,
"KL/rejected_KL_mean": -5.688087463378906,
"KL/std": 5.028870582580566,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.13124755024909973,
"fcm_dpo/delta": -0.10268934071063995,
"fcm_dpo/margin": 5.300374507904053,
"fcm_dpo/q_t": 0.34891366958618164,
"grad_norm": 85.72852325439453,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5880300402641296,
"logits/rejected": -0.5344229340553284,
"logps/chosen": -51.07637405395508,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.40348052978516,
"loss": 0.8973,
"margin_dpo/margin_mean": 5.300373554229736,
"margin_dpo/margin_std": 5.095399856567383,
"step": 54
},
{
"KL/chosen_KL_mean": -0.8597450256347656,
"KL/mean": -4.082709312438965,
"KL/rejected_KL_mean": -7.305671691894531,
"KL/std": 6.084364891052246,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.1264149248600006,
"fcm_dpo/delta": -0.2333047091960907,
"fcm_dpo/margin": 6.445924758911133,
"fcm_dpo/q_t": 0.3340168595314026,
"grad_norm": 62.58566665649414,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6331645250320435,
"logits/rejected": -0.5696378350257874,
"logps/chosen": -63.474979400634766,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.29917907714844,
"loss": 0.8932,
"margin_dpo/margin_mean": 6.445924282073975,
"margin_dpo/margin_std": 7.250290393829346,
"step": 55
},
{
"KL/chosen_KL_mean": -0.6148452758789062,
"KL/mean": -3.645325183868408,
"KL/rejected_KL_mean": -6.675815582275391,
"KL/std": 5.935892105102539,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.12174921482801437,
"fcm_dpo/delta": -0.14878109097480774,
"fcm_dpo/margin": 6.060969352722168,
"fcm_dpo/q_t": 0.34848877787590027,
"grad_norm": 53.345699310302734,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.6277638077735901,
"logits/rejected": -0.5859877467155457,
"logps/chosen": -58.54757308959961,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -100.85025024414062,
"loss": 0.9333,
"margin_dpo/margin_mean": 6.060969352722168,
"margin_dpo/margin_std": 7.159629821777344,
"step": 56
},
{
"KL/chosen_KL_mean": -0.6768627166748047,
"KL/mean": -4.056164741516113,
"KL/rejected_KL_mean": -7.435459136962891,
"KL/std": 5.846469879150391,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.11769460141658783,
"fcm_dpo/delta": -0.21182167530059814,
"fcm_dpo/margin": 6.7585954666137695,
"fcm_dpo/q_t": 0.3292352259159088,
"grad_norm": 58.79522705078125,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5582081079483032,
"logits/rejected": -0.5293355584144592,
"logps/chosen": -71.17214965820312,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.00091552734375,
"loss": 0.8653,
"margin_dpo/margin_mean": 6.758594512939453,
"margin_dpo/margin_std": 6.491828918457031,
"step": 57
},
{
"KL/chosen_KL_mean": -0.8934497833251953,
"KL/mean": -4.659195899963379,
"KL/rejected_KL_mean": -8.424942016601562,
"KL/std": 6.606472969055176,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.1118813008069992,
"fcm_dpo/delta": -0.26413968205451965,
"fcm_dpo/margin": 7.531491279602051,
"fcm_dpo/q_t": 0.3279980421066284,
"grad_norm": 63.110389709472656,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5848720073699951,
"logits/rejected": -0.5079290270805359,
"logps/chosen": -63.026390075683594,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.042236328125,
"loss": 0.866,
"margin_dpo/margin_mean": 7.531492233276367,
"margin_dpo/margin_std": 7.889359474182129,
"step": 58
},
{
"KL/chosen_KL_mean": -1.3678016662597656,
"KL/mean": -5.400118827819824,
"KL/rejected_KL_mean": -9.432437896728516,
"KL/std": 7.086496353149414,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.10428714752197266,
"fcm_dpo/delta": -0.2687758505344391,
"fcm_dpo/margin": 8.06463623046875,
"fcm_dpo/q_t": 0.32600656151771545,
"grad_norm": 57.96199417114258,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6462990641593933,
"logits/rejected": -0.6072210073471069,
"logps/chosen": -53.30032730102539,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.31764221191406,
"loss": 0.8604,
"margin_dpo/margin_mean": 8.06463623046875,
"margin_dpo/margin_std": 8.19699478149414,
"step": 59
},
{
"KL/chosen_KL_mean": -2.3611888885498047,
"KL/mean": -5.6755170822143555,
"KL/rejected_KL_mean": -8.98984146118164,
"KL/std": 6.482183456420898,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.10224273800849915,
"fcm_dpo/delta": -0.08416806906461716,
"fcm_dpo/margin": 6.6286468505859375,
"fcm_dpo/q_t": 0.3513771891593933,
"grad_norm": 63.67633819580078,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6272934675216675,
"logits/rejected": -0.5685904026031494,
"logps/chosen": -63.303375244140625,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.38323974609375,
"loss": 0.9437,
"margin_dpo/margin_mean": 6.628647327423096,
"margin_dpo/margin_std": 7.337241172790527,
"step": 60
},
{
"KL/chosen_KL_mean": -1.2284622192382812,
"KL/mean": -5.276307106018066,
"KL/rejected_KL_mean": -9.324153900146484,
"KL/std": 8.781950950622559,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.09854944050312042,
"fcm_dpo/delta": -0.21653008460998535,
"fcm_dpo/margin": 8.09569263458252,
"fcm_dpo/q_t": 0.34666797518730164,
"grad_norm": 52.498374938964844,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.6087511777877808,
"logits/rejected": -0.5745435953140259,
"logps/chosen": -61.86198425292969,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.17665100097656,
"loss": 0.932,
"margin_dpo/margin_mean": 8.09569263458252,
"margin_dpo/margin_std": 10.857327461242676,
"step": 61
},
{
"KL/chosen_KL_mean": -1.4789257049560547,
"KL/mean": -4.5049614906311035,
"KL/rejected_KL_mean": -7.530998229980469,
"KL/std": 6.338289260864258,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.09849489480257034,
"fcm_dpo/delta": 0.003545548766851425,
"fcm_dpo/margin": 6.052072525024414,
"fcm_dpo/q_t": 0.3739616572856903,
"grad_norm": 54.56527328491211,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.599959135055542,
"logits/rejected": -0.5654937028884888,
"logps/chosen": -57.62969970703125,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.09719848632812,
"loss": 0.9989,
"margin_dpo/margin_mean": 6.0520734786987305,
"margin_dpo/margin_std": 7.649867057800293,
"step": 62
},
{
"KL/chosen_KL_mean": -2.350595474243164,
"KL/mean": -6.442344665527344,
"KL/rejected_KL_mean": -10.534095764160156,
"KL/std": 8.069721221923828,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.09453290700912476,
"fcm_dpo/delta": -0.1908508688211441,
"fcm_dpo/margin": 8.183499336242676,
"fcm_dpo/q_t": 0.33432888984680176,
"grad_norm": 53.68877029418945,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.5888009667396545,
"logits/rejected": -0.54296875,
"logps/chosen": -75.49798583984375,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -108.14415740966797,
"loss": 0.8904,
"margin_dpo/margin_mean": 8.183499336242676,
"margin_dpo/margin_std": 8.431988716125488,
"step": 63
},
{
"KL/chosen_KL_mean": -0.8246746063232422,
"KL/mean": -5.562028884887695,
"KL/rejected_KL_mean": -10.299385070800781,
"KL/std": 8.699195861816406,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.08936040103435516,
"fcm_dpo/delta": -0.27669695019721985,
"fcm_dpo/margin": 9.47470474243164,
"fcm_dpo/q_t": 0.3278955817222595,
"grad_norm": 48.1386604309082,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.6068239212036133,
"logits/rejected": -0.5758517980575562,
"logps/chosen": -54.823272705078125,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -103.82958984375,
"loss": 0.8827,
"margin_dpo/margin_mean": 9.474705696105957,
"margin_dpo/margin_std": 10.625345230102539,
"step": 64
},
{
"KL/chosen_KL_mean": -2.8620662689208984,
"KL/mean": -7.634676456451416,
"KL/rejected_KL_mean": -12.40728759765625,
"KL/std": 9.632073402404785,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.08559057116508484,
"fcm_dpo/delta": -0.23870998620986938,
"fcm_dpo/margin": 9.545220375061035,
"fcm_dpo/q_t": 0.32906752824783325,
"grad_norm": 48.928260803222656,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6721543073654175,
"logits/rejected": -0.6587865352630615,
"logps/chosen": -67.69806671142578,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -122.35374450683594,
"loss": 0.8965,
"margin_dpo/margin_mean": 9.545220375061035,
"margin_dpo/margin_std": 10.546151161193848,
"step": 65
},
{
"KL/chosen_KL_mean": -2.6382007598876953,
"KL/mean": -7.045764923095703,
"KL/rejected_KL_mean": -11.453327178955078,
"KL/std": 8.719606399536133,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.08258344233036041,
"fcm_dpo/delta": -0.140939399600029,
"fcm_dpo/margin": 8.815117835998535,
"fcm_dpo/q_t": 0.3503197431564331,
"grad_norm": 44.452362060546875,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.643731415271759,
"logits/rejected": -0.6108591556549072,
"logps/chosen": -54.08172607421875,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -87.08963012695312,
"loss": 0.9293,
"margin_dpo/margin_mean": 8.815117835998535,
"margin_dpo/margin_std": 10.418359756469727,
"step": 66
},
{
"KL/chosen_KL_mean": -2.257081985473633,
"KL/mean": -6.745819568634033,
"KL/rejected_KL_mean": -11.234561920166016,
"KL/std": 8.64494514465332,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.08128023147583008,
"fcm_dpo/delta": -0.13966575264930725,
"fcm_dpo/margin": 8.977474212646484,
"fcm_dpo/q_t": 0.34918996691703796,
"grad_norm": 45.32052993774414,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.5956306457519531,
"logits/rejected": -0.5529816150665283,
"logps/chosen": -61.59788513183594,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -84.0218505859375,
"loss": 0.9241,
"margin_dpo/margin_mean": 8.977474212646484,
"margin_dpo/margin_std": 10.422260284423828,
"step": 67
},
{
"KL/chosen_KL_mean": -2.5931625366210938,
"KL/mean": -6.827396869659424,
"KL/rejected_KL_mean": -11.061626434326172,
"KL/std": 7.735515594482422,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.07975387573242188,
"fcm_dpo/delta": -0.0808192640542984,
"fcm_dpo/margin": 8.468461036682129,
"fcm_dpo/q_t": 0.35352686047554016,
"grad_norm": 44.582000732421875,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6226571202278137,
"logits/rejected": -0.561948299407959,
"logps/chosen": -67.79899597167969,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -88.26887512207031,
"loss": 0.9212,
"margin_dpo/margin_mean": 8.468460083007812,
"margin_dpo/margin_std": 8.639623641967773,
"step": 68
},
{
"KL/chosen_KL_mean": -2.993305206298828,
"KL/mean": -8.040552139282227,
"KL/rejected_KL_mean": -13.087799072265625,
"KL/std": 8.982470512390137,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.0765557736158371,
"fcm_dpo/delta": -0.1888705939054489,
"fcm_dpo/margin": 10.094490051269531,
"fcm_dpo/q_t": 0.3356069326400757,
"grad_norm": 46.04783248901367,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6112051010131836,
"logits/rejected": -0.5872414708137512,
"logps/chosen": -62.81254577636719,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -116.4766616821289,
"loss": 0.8677,
"margin_dpo/margin_mean": 10.094490051269531,
"margin_dpo/margin_std": 9.814105987548828,
"step": 69
},
{
"KL/chosen_KL_mean": -4.267280578613281,
"KL/mean": -9.726320266723633,
"KL/rejected_KL_mean": -15.185356140136719,
"KL/std": 10.843002319335938,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.07400006055831909,
"fcm_dpo/delta": -0.22565913200378418,
"fcm_dpo/margin": 10.918074607849121,
"fcm_dpo/q_t": 0.333512544631958,
"grad_norm": 47.22788619995117,
"learning_rate": 5e-07,
"logits/chosen": -0.5941894054412842,
"logits/rejected": -0.5576547384262085,
"logps/chosen": -66.19792175292969,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -106.24613952636719,
"loss": 0.8775,
"margin_dpo/margin_mean": 10.918075561523438,
"margin_dpo/margin_std": 11.484182357788086,
"step": 70
},
{
"KL/chosen_KL_mean": -3.74169921875,
"KL/mean": -9.998859405517578,
"KL/rejected_KL_mean": -16.256023406982422,
"KL/std": 11.078489303588867,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.06955009698867798,
"fcm_dpo/delta": -0.2970452606678009,
"fcm_dpo/margin": 12.51431941986084,
"fcm_dpo/q_t": 0.3210626244544983,
"grad_norm": 43.44234085083008,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.650926947593689,
"logits/rejected": -0.6100718975067139,
"logps/chosen": -65.49203491210938,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -113.59264373779297,
"loss": 0.8321,
"margin_dpo/margin_mean": 12.514318466186523,
"margin_dpo/margin_std": 12.17391586303711,
"step": 71
},
{
"KL/chosen_KL_mean": -4.193330764770508,
"KL/mean": -10.827133178710938,
"KL/rejected_KL_mean": -17.460933685302734,
"KL/std": 11.770431518554688,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.06550109386444092,
"fcm_dpo/delta": -0.29585161805152893,
"fcm_dpo/margin": 13.267606735229492,
"fcm_dpo/q_t": 0.3196568489074707,
"grad_norm": 41.92863845825195,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6538947224617004,
"logits/rejected": -0.6185261011123657,
"logps/chosen": -70.24674224853516,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -112.7479248046875,
"loss": 0.8452,
"margin_dpo/margin_mean": 13.267606735229492,
"margin_dpo/margin_std": 13.469938278198242,
"step": 72
},
{
"KL/chosen_KL_mean": -5.87510871887207,
"KL/mean": -12.412891387939453,
"KL/rejected_KL_mean": -18.95067596435547,
"KL/std": 15.22187614440918,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.06218613684177399,
"fcm_dpo/delta": -0.23374077677726746,
"fcm_dpo/margin": 13.075565338134766,
"fcm_dpo/q_t": 0.34885895252227783,
"grad_norm": 44.58890914916992,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.7102031707763672,
"logits/rejected": -0.6709054708480835,
"logps/chosen": -72.13137817382812,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -109.40681457519531,
"loss": 0.9801,
"margin_dpo/margin_mean": 13.075567245483398,
"margin_dpo/margin_std": 18.78510093688965,
"step": 73
},
{
"KL/chosen_KL_mean": -6.368963241577148,
"KL/mean": -13.39000129699707,
"KL/rejected_KL_mean": -20.41104507446289,
"KL/std": 16.742469787597656,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.05924233794212341,
"fcm_dpo/delta": -0.254236102104187,
"fcm_dpo/margin": 14.042078971862793,
"fcm_dpo/q_t": 0.34124910831451416,
"grad_norm": 45.11907958984375,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6220693588256836,
"logits/rejected": -0.6105754375457764,
"logps/chosen": -59.793846130371094,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -116.35798645019531,
"loss": 0.9438,
"margin_dpo/margin_mean": 14.042078971862793,
"margin_dpo/margin_std": 20.558794021606445,
"step": 74
},
{
"KL/chosen_KL_mean": -6.146207809448242,
"KL/mean": -16.09390640258789,
"KL/rejected_KL_mean": -26.041595458984375,
"KL/std": 17.39598846435547,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.054628752171993256,
"fcm_dpo/delta": -0.5433024764060974,
"fcm_dpo/margin": 19.895397186279297,
"fcm_dpo/q_t": 0.29074186086654663,
"grad_norm": 37.021820068359375,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6639382839202881,
"logits/rejected": -0.6543130278587341,
"logps/chosen": -58.007869720458984,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -137.2955780029297,
"loss": 0.7679,
"margin_dpo/margin_mean": 19.89539337158203,
"margin_dpo/margin_std": 19.379716873168945,
"step": 75
},
{
"KL/chosen_KL_mean": -7.63499641418457,
"KL/mean": -14.161641120910645,
"KL/rejected_KL_mean": -20.688282012939453,
"KL/std": 14.173757553100586,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.052014850080013275,
"fcm_dpo/delta": -0.08468005061149597,
"fcm_dpo/margin": 13.053292274475098,
"fcm_dpo/q_t": 0.35272541642189026,
"grad_norm": 38.93886184692383,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.6819676160812378,
"logits/rejected": -0.6378015279769897,
"logps/chosen": -60.90103530883789,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -98.9049072265625,
"loss": 0.9489,
"margin_dpo/margin_mean": 13.053293228149414,
"margin_dpo/margin_std": 15.290666580200195,
"step": 76
},
{
"KL/chosen_KL_mean": -6.789583206176758,
"KL/mean": -17.20227813720703,
"KL/rejected_KL_mean": -27.614978790283203,
"KL/std": 19.542203903198242,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.04801391810178757,
"fcm_dpo/delta": -0.4500316083431244,
"fcm_dpo/margin": 20.825393676757812,
"fcm_dpo/q_t": 0.2982975244522095,
"grad_norm": 41.4220085144043,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6983308792114258,
"logits/rejected": -0.6599130034446716,
"logps/chosen": -64.88626098632812,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -121.38859558105469,
"loss": 0.8181,
"margin_dpo/margin_mean": 20.825393676757812,
"margin_dpo/margin_std": 21.279712677001953,
"step": 77
},
{
"KL/chosen_KL_mean": -6.869146347045898,
"KL/mean": -14.925251007080078,
"KL/rejected_KL_mean": -22.98135757446289,
"KL/std": 15.62789535522461,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.046391915529966354,
"fcm_dpo/delta": -0.1590861678123474,
"fcm_dpo/margin": 16.11220932006836,
"fcm_dpo/q_t": 0.3436902165412903,
"grad_norm": 37.519630432128906,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.626217782497406,
"logits/rejected": -0.5986917018890381,
"logps/chosen": -62.482933044433594,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -107.91572570800781,
"loss": 0.9114,
"margin_dpo/margin_mean": 16.11220932006836,
"margin_dpo/margin_std": 17.67829132080078,
"step": 78
},
{
"KL/chosen_KL_mean": -7.106161117553711,
"KL/mean": -15.956548690795898,
"KL/rejected_KL_mean": -24.806934356689453,
"KL/std": 18.00418472290039,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.04444153979420662,
"fcm_dpo/delta": -0.20273572206497192,
"fcm_dpo/margin": 17.70077133178711,
"fcm_dpo/q_t": 0.34644412994384766,
"grad_norm": 31.86760711669922,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6549201011657715,
"logits/rejected": -0.6240353584289551,
"logps/chosen": -62.55664825439453,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -112.45449829101562,
"loss": 0.9323,
"margin_dpo/margin_mean": 17.70077133178711,
"margin_dpo/margin_std": 21.962387084960938,
"step": 79
},
{
"KL/chosen_KL_mean": -9.814868927001953,
"KL/mean": -17.918357849121094,
"KL/rejected_KL_mean": -26.021846771240234,
"KL/std": 18.628982543945312,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.04288051277399063,
"fcm_dpo/delta": -0.10531558841466904,
"fcm_dpo/margin": 16.206974029541016,
"fcm_dpo/q_t": 0.3639146089553833,
"grad_norm": 34.424556732177734,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7375781536102295,
"logits/rejected": -0.6945966482162476,
"logps/chosen": -68.33415985107422,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -113.56935119628906,
"loss": 0.9841,
"margin_dpo/margin_mean": 16.206972122192383,
"margin_dpo/margin_std": 21.44426727294922,
"step": 80
},
{
"KL/chosen_KL_mean": -8.757122039794922,
"KL/mean": -19.903606414794922,
"KL/rejected_KL_mean": -31.050094604492188,
"KL/std": 22.16201400756836,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.040648169815540314,
"fcm_dpo/delta": -0.3407384157180786,
"fcm_dpo/margin": 22.292972564697266,
"fcm_dpo/q_t": 0.32367393374443054,
"grad_norm": 36.16831970214844,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7065067887306213,
"logits/rejected": -0.6750401258468628,
"logps/chosen": -75.20599365234375,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -160.71279907226562,
"loss": 0.8724,
"margin_dpo/margin_mean": 22.292970657348633,
"margin_dpo/margin_std": 26.23095703125,
"step": 81
},
{
"KL/chosen_KL_mean": -11.012563705444336,
"KL/mean": -19.953739166259766,
"KL/rejected_KL_mean": -28.894912719726562,
"KL/std": 20.361019134521484,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.040023863315582275,
"fcm_dpo/delta": -0.12649241089820862,
"fcm_dpo/margin": 17.882349014282227,
"fcm_dpo/q_t": 0.3610876798629761,
"grad_norm": 42.08027648925781,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6327459812164307,
"logits/rejected": -0.6187605261802673,
"logps/chosen": -63.24494934082031,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -119.63816833496094,
"loss": 1.0379,
"margin_dpo/margin_mean": 17.882349014282227,
"margin_dpo/margin_std": 27.845230102539062,
"step": 82
},
{
"KL/chosen_KL_mean": -11.224641799926758,
"KL/mean": -21.655147552490234,
"KL/rejected_KL_mean": -32.08565139770508,
"KL/std": 21.140281677246094,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.03790491446852684,
"fcm_dpo/delta": -0.20823806524276733,
"fcm_dpo/margin": 20.861011505126953,
"fcm_dpo/q_t": 0.3407338559627533,
"grad_norm": 33.69918441772461,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6975592374801636,
"logits/rejected": -0.65975022315979,
"logps/chosen": -67.05203247070312,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -135.80154418945312,
"loss": 0.9206,
"margin_dpo/margin_mean": 20.861011505126953,
"margin_dpo/margin_std": 24.786806106567383,
"step": 83
},
{
"KL/chosen_KL_mean": -10.090581893920898,
"KL/mean": -19.29345703125,
"KL/rejected_KL_mean": -28.496337890625,
"KL/std": 18.165950775146484,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.03682379424571991,
"fcm_dpo/delta": -0.08598282188177109,
"fcm_dpo/margin": 18.40575408935547,
"fcm_dpo/q_t": 0.35048460960388184,
"grad_norm": 32.24285125732422,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6537163257598877,
"logits/rejected": -0.6145398616790771,
"logps/chosen": -77.26675415039062,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -115.79493713378906,
"loss": 0.9251,
"margin_dpo/margin_mean": 18.40575408935547,
"margin_dpo/margin_std": 18.526391983032227,
"step": 84
},
{
"KL/chosen_KL_mean": -10.64134407043457,
"KL/mean": -19.9929256439209,
"KL/rejected_KL_mean": -29.344505310058594,
"KL/std": 18.70035171508789,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.03684117645025253,
"fcm_dpo/delta": -0.09671901166439056,
"fcm_dpo/margin": 18.703161239624023,
"fcm_dpo/q_t": 0.35741060972213745,
"grad_norm": 33.3864860534668,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6444880366325378,
"logits/rejected": -0.6126257181167603,
"logps/chosen": -69.04795837402344,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -107.98330688476562,
"loss": 0.961,
"margin_dpo/margin_mean": 18.703161239624023,
"margin_dpo/margin_std": 22.245079040527344,
"step": 85
},
{
"KL/chosen_KL_mean": -13.715383529663086,
"KL/mean": -23.63683319091797,
"KL/rejected_KL_mean": -33.55827713012695,
"KL/std": 24.58064079284668,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.035842716693878174,
"fcm_dpo/delta": -0.11968203634023666,
"fcm_dpo/margin": 19.8429012298584,
"fcm_dpo/q_t": 0.37169545888900757,
"grad_norm": 39.88727569580078,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.7000675201416016,
"logits/rejected": -0.6864509582519531,
"logps/chosen": -69.85284423828125,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -121.679931640625,
"loss": 1.0575,
"margin_dpo/margin_mean": 19.84290313720703,
"margin_dpo/margin_std": 33.51190948486328,
"step": 86
},
{
"KL/chosen_KL_mean": -13.420072555541992,
"KL/mean": -24.801658630371094,
"KL/rejected_KL_mean": -36.183231353759766,
"KL/std": 24.48314094543457,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.03418724611401558,
"fcm_dpo/delta": -0.1955638974905014,
"fcm_dpo/margin": 22.763172149658203,
"fcm_dpo/q_t": 0.3427821099758148,
"grad_norm": 32.262451171875,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.717382550239563,
"logits/rejected": -0.6887001991271973,
"logps/chosen": -69.05616760253906,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -131.65081787109375,
"loss": 0.9558,
"margin_dpo/margin_mean": 22.763172149658203,
"margin_dpo/margin_std": 29.336151123046875,
"step": 87
},
{
"KL/chosen_KL_mean": -16.992156982421875,
"KL/mean": -25.538352966308594,
"KL/rejected_KL_mean": -34.08455276489258,
"KL/std": 25.105436325073242,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.03394782915711403,
"fcm_dpo/delta": 0.02017107605934143,
"fcm_dpo/margin": 17.092395782470703,
"fcm_dpo/q_t": 0.38290032744407654,
"grad_norm": 35.017784118652344,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6727230548858643,
"logits/rejected": -0.6630773544311523,
"logps/chosen": -90.6633071899414,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -140.79306030273438,
"loss": 1.0699,
"margin_dpo/margin_mean": 17.092395782470703,
"margin_dpo/margin_std": 25.96027374267578,
"step": 88
},
{
"KL/chosen_KL_mean": -10.070928573608398,
"KL/mean": -19.611610412597656,
"KL/rejected_KL_mean": -29.15229034423828,
"KL/std": 21.15941619873047,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.03413812071084976,
"fcm_dpo/delta": -0.05540309473872185,
"fcm_dpo/margin": 19.081357955932617,
"fcm_dpo/q_t": 0.3627317547798157,
"grad_norm": 30.411449432373047,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7250231504440308,
"logits/rejected": -0.6905418634414673,
"logps/chosen": -70.69584655761719,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -111.23583984375,
"loss": 0.9665,
"margin_dpo/margin_mean": 19.081357955932617,
"margin_dpo/margin_std": 23.584978103637695,
"step": 89
},
{
"KL/chosen_KL_mean": -13.3211669921875,
"KL/mean": -25.765104293823242,
"KL/rejected_KL_mean": -38.20904541015625,
"KL/std": 28.377769470214844,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.033040180802345276,
"fcm_dpo/delta": -0.24225689470767975,
"fcm_dpo/margin": 24.887874603271484,
"fcm_dpo/q_t": 0.3497656285762787,
"grad_norm": 32.33899688720703,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6367080807685852,
"logits/rejected": -0.6447947025299072,
"logps/chosen": -66.60647583007812,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -149.75375366210938,
"loss": 0.9752,
"margin_dpo/margin_mean": 24.887874603271484,
"margin_dpo/margin_std": 35.06592559814453,
"step": 90
},
{
"KL/chosen_KL_mean": -13.380002975463867,
"KL/mean": -24.273265838623047,
"KL/rejected_KL_mean": -35.16653060913086,
"KL/std": 23.42182159423828,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.031911782920360565,
"fcm_dpo/delta": -0.10237793624401093,
"fcm_dpo/margin": 21.786531448364258,
"fcm_dpo/q_t": 0.36154645681381226,
"grad_norm": 32.23942184448242,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6827590465545654,
"logits/rejected": -0.6465390920639038,
"logps/chosen": -75.1829605102539,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -123.04048156738281,
"loss": 0.9899,
"margin_dpo/margin_mean": 21.786529541015625,
"margin_dpo/margin_std": 29.604764938354492,
"step": 91
},
{
"KL/chosen_KL_mean": -12.197099685668945,
"KL/mean": -22.994224548339844,
"KL/rejected_KL_mean": -33.791351318359375,
"KL/std": 23.79822540283203,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.03140295296907425,
"fcm_dpo/delta": -0.08418678492307663,
"fcm_dpo/margin": 21.594253540039062,
"fcm_dpo/q_t": 0.3629786968231201,
"grad_norm": 29.07423210144043,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.7315187454223633,
"logits/rejected": -0.7085036635398865,
"logps/chosen": -63.837867736816406,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -111.67253112792969,
"loss": 0.9868,
"margin_dpo/margin_mean": 21.594255447387695,
"margin_dpo/margin_std": 28.674663543701172,
"step": 92
},
{
"KL/chosen_KL_mean": -13.579082489013672,
"KL/mean": -24.47039794921875,
"KL/rejected_KL_mean": -35.36171340942383,
"KL/std": 22.706327438354492,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.03072137013077736,
"fcm_dpo/delta": -0.07413841784000397,
"fcm_dpo/margin": 21.782629013061523,
"fcm_dpo/q_t": 0.35937225818634033,
"grad_norm": 29.083242416381836,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7480611801147461,
"logits/rejected": -0.6964054107666016,
"logps/chosen": -66.10832214355469,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -112.5224609375,
"loss": 0.9479,
"margin_dpo/margin_mean": 21.78262710571289,
"margin_dpo/margin_std": 25.83187484741211,
"step": 93
},
{
"KL/chosen_KL_mean": -14.645559310913086,
"KL/mean": -26.840560913085938,
"KL/rejected_KL_mean": -39.03556442260742,
"KL/std": 25.22960090637207,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.0297713503241539,
"fcm_dpo/delta": -0.13794076442718506,
"fcm_dpo/margin": 24.390003204345703,
"fcm_dpo/q_t": 0.34696894884109497,
"grad_norm": 29.175193786621094,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.6889959573745728,
"logits/rejected": -0.6704879999160767,
"logps/chosen": -75.86817169189453,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -138.6345977783203,
"loss": 0.9147,
"margin_dpo/margin_mean": 24.390003204345703,
"margin_dpo/margin_std": 26.14815902709961,
"step": 94
},
{
"KL/chosen_KL_mean": -13.445240020751953,
"KL/mean": -23.329883575439453,
"KL/rejected_KL_mean": -33.21453094482422,
"KL/std": 23.115684509277344,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.02971818298101425,
"fcm_dpo/delta": 0.013067739084362984,
"fcm_dpo/margin": 19.769285202026367,
"fcm_dpo/q_t": 0.3717486560344696,
"grad_norm": 27.53291130065918,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.7019277215003967,
"logits/rejected": -0.670516848564148,
"logps/chosen": -65.96888732910156,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -109.09487915039062,
"loss": 0.9965,
"margin_dpo/margin_mean": 19.769287109375,
"margin_dpo/margin_std": 24.190692901611328,
"step": 95
},
{
"KL/chosen_KL_mean": -12.644975662231445,
"KL/mean": -25.990266799926758,
"KL/rejected_KL_mean": -39.33556365966797,
"KL/std": 27.345867156982422,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.02879452332854271,
"fcm_dpo/delta": -0.18552882969379425,
"fcm_dpo/margin": 26.69058609008789,
"fcm_dpo/q_t": 0.34164929389953613,
"grad_norm": 27.511608123779297,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.6968206167221069,
"logits/rejected": -0.6622928977012634,
"logps/chosen": -74.80194854736328,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -135.93157958984375,
"loss": 0.8982,
"margin_dpo/margin_mean": 26.69058609008789,
"margin_dpo/margin_std": 29.720775604248047,
"step": 96
},
{
"KL/chosen_KL_mean": -13.93777847290039,
"KL/mean": -24.47606086730957,
"KL/rejected_KL_mean": -35.014347076416016,
"KL/std": 22.49617576599121,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.029009651392698288,
"fcm_dpo/delta": -0.014466457068920135,
"fcm_dpo/margin": 21.07657241821289,
"fcm_dpo/q_t": 0.36493146419525146,
"grad_norm": 29.625926971435547,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6640722751617432,
"logits/rejected": -0.6175036430358887,
"logps/chosen": -68.58414459228516,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -111.97909545898438,
"loss": 0.9686,
"margin_dpo/margin_mean": 21.07657241821289,
"margin_dpo/margin_std": 22.399307250976562,
"step": 97
},
{
"KL/chosen_KL_mean": -16.776840209960938,
"KL/mean": -28.678016662597656,
"KL/rejected_KL_mean": -40.579193115234375,
"KL/std": 25.27958106994629,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.02829306200146675,
"fcm_dpo/delta": -0.07892265915870667,
"fcm_dpo/margin": 23.802349090576172,
"fcm_dpo/q_t": 0.3552480638027191,
"grad_norm": 30.168066024780273,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.7093369960784912,
"logits/rejected": -0.6912350654602051,
"logps/chosen": -82.03546905517578,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -137.106689453125,
"loss": 0.9787,
"margin_dpo/margin_mean": 23.802349090576172,
"margin_dpo/margin_std": 29.942073822021484,
"step": 98
},
{
"KL/chosen_KL_mean": -13.65618896484375,
"KL/mean": -27.793987274169922,
"KL/rejected_KL_mean": -41.93178176879883,
"KL/std": 27.672199249267578,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.02736222743988037,
"fcm_dpo/delta": -0.18900561332702637,
"fcm_dpo/margin": 28.275592803955078,
"fcm_dpo/q_t": 0.34526270627975464,
"grad_norm": 26.520252227783203,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6868008375167847,
"logits/rejected": -0.6845619678497314,
"logps/chosen": -59.294673919677734,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -128.36972045898438,
"loss": 0.9118,
"margin_dpo/margin_mean": 28.27559471130371,
"margin_dpo/margin_std": 33.97766876220703,
"step": 99
},
{
"KL/chosen_KL_mean": -15.083608627319336,
"KL/mean": -23.885147094726562,
"KL/rejected_KL_mean": -32.686683654785156,
"KL/std": 22.15597915649414,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.027774441987276077,
"fcm_dpo/delta": 0.11583902686834335,
"fcm_dpo/margin": 17.603078842163086,
"fcm_dpo/q_t": 0.3959774374961853,
"grad_norm": 30.725154876708984,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6721187829971313,
"logits/rejected": -0.6277564167976379,
"logps/chosen": -72.6775894165039,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -106.74689483642578,
"loss": 1.0995,
"margin_dpo/margin_mean": 17.603078842163086,
"margin_dpo/margin_std": 28.475990295410156,
"step": 100
},
{
"KL/chosen_KL_mean": -18.393741607666016,
"KL/mean": -28.85015869140625,
"KL/rejected_KL_mean": -39.30656814575195,
"KL/std": 26.651226043701172,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.027704566717147827,
"fcm_dpo/delta": 0.02101181074976921,
"fcm_dpo/margin": 20.912822723388672,
"fcm_dpo/q_t": 0.3846778869628906,
"grad_norm": 30.959144592285156,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6894014477729797,
"logits/rejected": -0.6551156640052795,
"logps/chosen": -80.04259490966797,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -122.49625396728516,
"loss": 1.0559,
"margin_dpo/margin_mean": 20.912824630737305,
"margin_dpo/margin_std": 32.27412414550781,
"step": 101
},
{
"KL/chosen_KL_mean": -20.296354293823242,
"KL/mean": -28.111732482910156,
"KL/rejected_KL_mean": -35.92710876464844,
"KL/std": 25.370466232299805,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.028836820274591446,
"fcm_dpo/delta": 0.15318328142166138,
"fcm_dpo/margin": 15.63076114654541,
"fcm_dpo/q_t": 0.40718287229537964,
"grad_norm": 36.441226959228516,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.693681538105011,
"logits/rejected": -0.6452783942222595,
"logps/chosen": -84.37522888183594,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -104.11418151855469,
"loss": 1.1497,
"margin_dpo/margin_mean": 15.630760192871094,
"margin_dpo/margin_std": 29.49494171142578,
"step": 102
},
{
"KL/chosen_KL_mean": -14.965106964111328,
"KL/mean": -29.99454116821289,
"KL/rejected_KL_mean": -45.02397537231445,
"KL/std": 27.774456024169922,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.027667928487062454,
"fcm_dpo/delta": -0.2534938454627991,
"fcm_dpo/margin": 30.058866500854492,
"fcm_dpo/q_t": 0.3302461504936218,
"grad_norm": 28.2922306060791,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.7033284902572632,
"logits/rejected": -0.6691305637359619,
"logps/chosen": -76.26438903808594,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -138.5966796875,
"loss": 0.8684,
"margin_dpo/margin_mean": 30.058866500854492,
"margin_dpo/margin_std": 32.256858825683594,
"step": 103
},
{
"KL/chosen_KL_mean": -16.92726707458496,
"KL/mean": -31.169830322265625,
"KL/rejected_KL_mean": -45.412391662597656,
"KL/std": 30.842205047607422,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.02675759233534336,
"fcm_dpo/delta": -0.17523446679115295,
"fcm_dpo/margin": 28.485132217407227,
"fcm_dpo/q_t": 0.351858913898468,
"grad_norm": 28.383094787597656,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.7101203203201294,
"logits/rejected": -0.674048662185669,
"logps/chosen": -71.30004119873047,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -134.97711181640625,
"loss": 0.9505,
"margin_dpo/margin_mean": 28.48513412475586,
"margin_dpo/margin_std": 36.80478286743164,
"step": 104
},
{
"KL/chosen_KL_mean": -15.344198226928711,
"KL/mean": -35.240638732910156,
"KL/rejected_KL_mean": -55.1370849609375,
"KL/std": 30.727638244628906,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.024829303845763206,
"fcm_dpo/delta": -0.430313378572464,
"fcm_dpo/margin": 39.792884826660156,
"fcm_dpo/q_t": 0.29163897037506104,
"grad_norm": 26.41094207763672,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.7010092735290527,
"logits/rejected": -0.6739363074302673,
"logps/chosen": -69.98313903808594,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -153.110595703125,
"loss": 0.7563,
"margin_dpo/margin_mean": 39.792884826660156,
"margin_dpo/margin_std": 33.252655029296875,
"step": 105
},
{
"KL/chosen_KL_mean": -17.2432918548584,
"KL/mean": -29.813838958740234,
"KL/rejected_KL_mean": -42.3843879699707,
"KL/std": 26.08827781677246,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.024186331778764725,
"fcm_dpo/delta": -0.008649196475744247,
"fcm_dpo/margin": 25.14108657836914,
"fcm_dpo/q_t": 0.3670313358306885,
"grad_norm": 25.191970825195312,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.6583723425865173,
"logits/rejected": -0.6192843317985535,
"logps/chosen": -72.07618713378906,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -127.6090087890625,
"loss": 0.9716,
"margin_dpo/margin_mean": 25.14108657836914,
"margin_dpo/margin_std": 28.393421173095703,
"step": 106
},
{
"KL/chosen_KL_mean": -20.843509674072266,
"KL/mean": -34.96947479248047,
"KL/rejected_KL_mean": -49.095436096191406,
"KL/std": 31.35101890563965,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.023814622312784195,
"fcm_dpo/delta": -0.07891468703746796,
"fcm_dpo/margin": 28.251924514770508,
"fcm_dpo/q_t": 0.35584813356399536,
"grad_norm": 26.355724334716797,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.7115845680236816,
"logits/rejected": -0.6630497574806213,
"logps/chosen": -90.55132293701172,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -143.83494567871094,
"loss": 0.9564,
"margin_dpo/margin_mean": 28.25192642211914,
"margin_dpo/margin_std": 32.80548858642578,
"step": 107
},
{
"KL/chosen_KL_mean": -19.167158126831055,
"KL/mean": -35.18861770629883,
"KL/rejected_KL_mean": -51.21007537841797,
"KL/std": 33.77876281738281,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.02327515184879303,
"fcm_dpo/delta": -0.1576060950756073,
"fcm_dpo/margin": 32.04291915893555,
"fcm_dpo/q_t": 0.35573819279670715,
"grad_norm": 26.53054428100586,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.702393651008606,
"logits/rejected": -0.673580527305603,
"logps/chosen": -75.17704772949219,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -147.00608825683594,
"loss": 0.975,
"margin_dpo/margin_mean": 32.04291534423828,
"margin_dpo/margin_std": 43.38271713256836,
"step": 108
},
{
"KL/chosen_KL_mean": -17.375944137573242,
"KL/mean": -34.98335647583008,
"KL/rejected_KL_mean": -52.59076690673828,
"KL/std": 33.18781280517578,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.022326169535517693,
"fcm_dpo/delta": -0.2031365931034088,
"fcm_dpo/margin": 35.21482849121094,
"fcm_dpo/q_t": 0.3348275423049927,
"grad_norm": 26.37941551208496,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.6269769668579102,
"logits/rejected": -0.6036471128463745,
"logps/chosen": -80.26142883300781,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -151.27650451660156,
"loss": 0.8894,
"margin_dpo/margin_mean": 35.21482849121094,
"margin_dpo/margin_std": 37.05272674560547,
"step": 109
},
{
"KL/chosen_KL_mean": -17.29520034790039,
"KL/mean": -33.274478912353516,
"KL/rejected_KL_mean": -49.253761291503906,
"KL/std": 32.64356231689453,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.02154034934937954,
"fcm_dpo/delta": -0.10045865923166275,
"fcm_dpo/margin": 31.958560943603516,
"fcm_dpo/q_t": 0.35664719343185425,
"grad_norm": 24.21642303466797,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.6421069502830505,
"logits/rejected": -0.6008081436157227,
"logps/chosen": -76.04888916015625,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -129.0037841796875,
"loss": 0.9673,
"margin_dpo/margin_mean": 31.958560943603516,
"margin_dpo/margin_std": 38.652034759521484,
"step": 110
},
{
"KL/chosen_KL_mean": -20.667470932006836,
"KL/mean": -38.21528625488281,
"KL/rejected_KL_mean": -55.763092041015625,
"KL/std": 36.594356536865234,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.021102361381053925,
"fcm_dpo/delta": -0.15377238392829895,
"fcm_dpo/margin": 35.09562683105469,
"fcm_dpo/q_t": 0.34861353039741516,
"grad_norm": 26.77528953552246,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.6729174852371216,
"logits/rejected": -0.6483257412910461,
"logps/chosen": -89.29158020019531,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -154.19195556640625,
"loss": 0.9544,
"margin_dpo/margin_mean": 35.09562683105469,
"margin_dpo/margin_std": 43.51227569580078,
"step": 111
},
{
"KL/chosen_KL_mean": -19.272823333740234,
"KL/mean": -30.860448837280273,
"KL/rejected_KL_mean": -42.44807434082031,
"KL/std": 26.632038116455078,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.021312057971954346,
"fcm_dpo/delta": 0.11160361021757126,
"fcm_dpo/margin": 23.175251007080078,
"fcm_dpo/q_t": 0.39253005385398865,
"grad_norm": 24.64253807067871,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.6570972204208374,
"logits/rejected": -0.6123785972595215,
"logps/chosen": -69.5224609375,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -107.22250366210938,
"loss": 1.0555,
"margin_dpo/margin_mean": 23.175251007080078,
"margin_dpo/margin_std": 31.359512329101562,
"step": 112
},
{
"KL/chosen_KL_mean": -23.004501342773438,
"KL/mean": -35.80390167236328,
"KL/rejected_KL_mean": -48.603302001953125,
"KL/std": 26.217470169067383,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.021701056510210037,
"fcm_dpo/delta": 0.047093942761421204,
"fcm_dpo/margin": 25.598800659179688,
"fcm_dpo/q_t": 0.3771820068359375,
"grad_norm": 25.575990676879883,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.6328074336051941,
"logits/rejected": -0.5849325060844421,
"logps/chosen": -89.71745300292969,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -126.57200622558594,
"loss": 0.9954,
"margin_dpo/margin_mean": 25.598800659179688,
"margin_dpo/margin_std": 29.07805633544922,
"step": 113
},
{
"KL/chosen_KL_mean": -21.031354904174805,
"KL/mean": -40.62781524658203,
"KL/rejected_KL_mean": -60.22427749633789,
"KL/std": 40.04113006591797,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.0208163820207119,
"fcm_dpo/delta": -0.23869842290878296,
"fcm_dpo/margin": 39.19293212890625,
"fcm_dpo/q_t": 0.3389107584953308,
"grad_norm": 25.883995056152344,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.6460467576980591,
"logits/rejected": -0.6084377765655518,
"logps/chosen": -78.81643676757812,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -147.3339385986328,
"loss": 0.8997,
"margin_dpo/margin_mean": 39.19293212890625,
"margin_dpo/margin_std": 47.87086486816406,
"step": 114
},
{
"KL/chosen_KL_mean": -28.150527954101562,
"KL/mean": -41.863258361816406,
"KL/rejected_KL_mean": -55.57598876953125,
"KL/std": 37.38142395019531,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.020900558680295944,
"fcm_dpo/delta": 0.028054876253008842,
"fcm_dpo/margin": 27.425457000732422,
"fcm_dpo/q_t": 0.38739147782325745,
"grad_norm": 34.74506378173828,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.6769613027572632,
"logits/rejected": -0.6487143635749817,
"logps/chosen": -93.73316955566406,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -154.14151000976562,
"loss": 1.0936,
"margin_dpo/margin_mean": 27.425457000732422,
"margin_dpo/margin_std": 48.14727783203125,
"step": 115
},
{
"KL/chosen_KL_mean": -21.264305114746094,
"KL/mean": -35.94366455078125,
"KL/rejected_KL_mean": -50.623023986816406,
"KL/std": 31.68724822998047,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.020909134298563004,
"fcm_dpo/delta": -0.015174690634012222,
"fcm_dpo/margin": 29.35871124267578,
"fcm_dpo/q_t": 0.36827123165130615,
"grad_norm": 28.000104904174805,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.6615927815437317,
"logits/rejected": -0.6435688734054565,
"logps/chosen": -72.66461944580078,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -131.14486694335938,
"loss": 0.9682,
"margin_dpo/margin_mean": 29.35871124267578,
"margin_dpo/margin_std": 33.837371826171875,
"step": 116
},
{
"KL/chosen_KL_mean": -26.09354019165039,
"KL/mean": -39.05518341064453,
"KL/rejected_KL_mean": -52.01681900024414,
"KL/std": 33.70083236694336,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.02097507193684578,
"fcm_dpo/delta": 0.059497952461242676,
"fcm_dpo/margin": 25.923280715942383,
"fcm_dpo/q_t": 0.38448774814605713,
"grad_norm": 33.21882247924805,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.6220806837081909,
"logits/rejected": -0.5811136960983276,
"logps/chosen": -95.39195251464844,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -118.6008071899414,
"loss": 1.0458,
"margin_dpo/margin_mean": 25.92327880859375,
"margin_dpo/margin_std": 37.46910858154297,
"step": 117
},
{
"KL/chosen_KL_mean": -22.07351303100586,
"KL/mean": -36.50092315673828,
"KL/rejected_KL_mean": -50.92832946777344,
"KL/std": 30.597415924072266,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.021029407158493996,
"fcm_dpo/delta": -0.007240705192089081,
"fcm_dpo/margin": 28.854822158813477,
"fcm_dpo/q_t": 0.370626300573349,
"grad_norm": 27.60127830505371,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.6917950510978699,
"logits/rejected": -0.6692517995834351,
"logps/chosen": -77.71449279785156,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -126.5973892211914,
"loss": 0.9783,
"margin_dpo/margin_mean": 28.854822158813477,
"margin_dpo/margin_std": 34.29177474975586,
"step": 118
},
{
"KL/chosen_KL_mean": -28.89242172241211,
"KL/mean": -43.309837341308594,
"KL/rejected_KL_mean": -57.72724533081055,
"KL/std": 33.70098114013672,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.020852331072092056,
"fcm_dpo/delta": -0.0024533867835998535,
"fcm_dpo/margin": 28.834829330444336,
"fcm_dpo/q_t": 0.37676042318344116,
"grad_norm": 28.45378875732422,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.6722303628921509,
"logits/rejected": -0.6554035544395447,
"logps/chosen": -102.40261840820312,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -160.70452880859375,
"loss": 1.0194,
"margin_dpo/margin_mean": 28.834827423095703,
"margin_dpo/margin_std": 40.16496658325195,
"step": 119
},
{
"KL/chosen_KL_mean": -30.290496826171875,
"KL/mean": -49.79584884643555,
"KL/rejected_KL_mean": -69.30119323730469,
"KL/std": 42.012657165527344,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.02034316398203373,
"fcm_dpo/delta": -0.21045231819152832,
"fcm_dpo/margin": 39.01069259643555,
"fcm_dpo/q_t": 0.33687207102775574,
"grad_norm": 26.843433380126953,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6685041189193726,
"logits/rejected": -0.6314491033554077,
"logps/chosen": -107.07133483886719,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -177.32493591308594,
"loss": 0.9,
"margin_dpo/margin_mean": 39.01068878173828,
"margin_dpo/margin_std": 43.660438537597656,
"step": 120
},
{
"KL/chosen_KL_mean": -29.122751235961914,
"KL/mean": -50.80091857910156,
"KL/rejected_KL_mean": -72.47908020019531,
"KL/std": 41.654624938964844,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.019261084496974945,
"fcm_dpo/delta": -0.2588822841644287,
"fcm_dpo/margin": 43.356327056884766,
"fcm_dpo/q_t": 0.334938645362854,
"grad_norm": 26.77589988708496,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.6290786266326904,
"logits/rejected": -0.6210762858390808,
"logps/chosen": -90.91264343261719,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -182.47364807128906,
"loss": 0.9071,
"margin_dpo/margin_mean": 43.35633087158203,
"margin_dpo/margin_std": 51.60829162597656,
"step": 121
},
{
"KL/chosen_KL_mean": -26.941118240356445,
"KL/mean": -53.64228057861328,
"KL/rejected_KL_mean": -80.34346008300781,
"KL/std": 47.90687561035156,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.018020082265138626,
"fcm_dpo/delta": -0.40099939703941345,
"fcm_dpo/margin": 53.40232849121094,
"fcm_dpo/q_t": 0.3011833429336548,
"grad_norm": 27.174243927001953,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.6133723258972168,
"logits/rejected": -0.6250221729278564,
"logps/chosen": -73.84333038330078,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -187.05764770507812,
"loss": 0.785,
"margin_dpo/margin_mean": 53.40232849121094,
"margin_dpo/margin_std": 48.34116744995117,
"step": 122
},
{
"KL/chosen_KL_mean": -28.8530216217041,
"KL/mean": -46.64100646972656,
"KL/rejected_KL_mean": -64.42898559570312,
"KL/std": 41.192108154296875,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.01748839020729065,
"fcm_dpo/delta": -0.0243561789393425,
"fcm_dpo/margin": 35.575958251953125,
"fcm_dpo/q_t": 0.37540945410728455,
"grad_norm": 26.237045288085938,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.6377385258674622,
"logits/rejected": -0.6154704093933105,
"logps/chosen": -90.19166564941406,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -152.20437622070312,
"loss": 1.0436,
"margin_dpo/margin_mean": 35.575958251953125,
"margin_dpo/margin_std": 54.298728942871094,
"step": 123
},
{
"KL/chosen_KL_mean": -31.93813705444336,
"KL/mean": -57.24821472167969,
"KL/rejected_KL_mean": -82.55829620361328,
"KL/std": 54.470890045166016,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.01679907739162445,
"fcm_dpo/delta": -0.27494901418685913,
"fcm_dpo/margin": 50.62015151977539,
"fcm_dpo/q_t": 0.3425573408603668,
"grad_norm": 29.16954231262207,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.6193432807922363,
"logits/rejected": -0.6179369688034058,
"logps/chosen": -103.386474609375,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -200.13885498046875,
"loss": 0.9419,
"margin_dpo/margin_mean": 50.62015151977539,
"margin_dpo/margin_std": 68.21569061279297,
"step": 124
},
{
"KL/chosen_KL_mean": -26.163921356201172,
"KL/mean": -49.2783317565918,
"KL/rejected_KL_mean": -72.39274597167969,
"KL/std": 45.21812438964844,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.016263190656900406,
"fcm_dpo/delta": -0.16389188170433044,
"fcm_dpo/margin": 46.228824615478516,
"fcm_dpo/q_t": 0.34715038537979126,
"grad_norm": 23.99207878112793,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.5740602016448975,
"logits/rejected": -0.5690337419509888,
"logps/chosen": -76.30085754394531,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -156.38134765625,
"loss": 0.925,
"margin_dpo/margin_mean": 46.22882080078125,
"margin_dpo/margin_std": 53.9525032043457,
"step": 125
},
{
"KL/chosen_KL_mean": -28.852941513061523,
"KL/mean": -50.07624816894531,
"KL/rejected_KL_mean": -71.29955291748047,
"KL/std": 41.17491912841797,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.01587202399969101,
"fcm_dpo/delta": -0.07921052724123001,
"fcm_dpo/margin": 42.44661331176758,
"fcm_dpo/q_t": 0.35897907614707947,
"grad_norm": 25.790943145751953,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.5833499431610107,
"logits/rejected": -0.5680674314498901,
"logps/chosen": -84.52001190185547,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -169.4293212890625,
"loss": 0.9567,
"margin_dpo/margin_mean": 42.44661331176758,
"margin_dpo/margin_std": 50.80341339111328,
"step": 126
},
{
"KL/chosen_KL_mean": -27.53070831298828,
"KL/mean": -44.140663146972656,
"KL/rejected_KL_mean": -60.750614166259766,
"KL/std": 35.858009338378906,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.01601826772093773,
"fcm_dpo/delta": 0.0715877041220665,
"fcm_dpo/margin": 33.21990203857422,
"fcm_dpo/q_t": 0.3830415606498718,
"grad_norm": 26.187414169311523,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.600145697593689,
"logits/rejected": -0.5759819746017456,
"logps/chosen": -84.08538818359375,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -137.54638671875,
"loss": 1.0353,
"margin_dpo/margin_mean": 33.21990203857422,
"margin_dpo/margin_std": 43.94744873046875,
"step": 127
},
{
"KL/chosen_KL_mean": -30.146015167236328,
"KL/mean": -47.31671905517578,
"KL/rejected_KL_mean": -64.4874267578125,
"KL/std": 43.29245376586914,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.01624487154185772,
"fcm_dpo/delta": 0.04389987140893936,
"fcm_dpo/margin": 34.341407775878906,
"fcm_dpo/q_t": 0.38386672735214233,
"grad_norm": 38.06498718261719,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.6183241605758667,
"logits/rejected": -0.589686393737793,
"logps/chosen": -88.2669677734375,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -140.9263916015625,
"loss": 1.0628,
"margin_dpo/margin_mean": 34.341407775878906,
"margin_dpo/margin_std": 52.57948303222656,
"step": 128
},
{
"KL/chosen_KL_mean": -37.54634094238281,
"KL/mean": -55.68879318237305,
"KL/rejected_KL_mean": -73.83124542236328,
"KL/std": 46.2327880859375,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.01625596359372139,
"fcm_dpo/delta": 0.010706906206905842,
"fcm_dpo/margin": 36.28490447998047,
"fcm_dpo/q_t": 0.3834758400917053,
"grad_norm": 28.041536331176758,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.5989828705787659,
"logits/rejected": -0.581794261932373,
"logps/chosen": -104.46271514892578,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -170.47344970703125,
"loss": 1.0666,
"margin_dpo/margin_mean": 36.28490447998047,
"margin_dpo/margin_std": 59.076148986816406,
"step": 129
},
{
"KL/chosen_KL_mean": -28.067245483398438,
"KL/mean": -55.02955627441406,
"KL/rejected_KL_mean": -81.99185943603516,
"KL/std": 50.90562438964844,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.015807852149009705,
"fcm_dpo/delta": -0.27608606219291687,
"fcm_dpo/margin": 53.92461013793945,
"fcm_dpo/q_t": 0.3301834464073181,
"grad_norm": 22.134960174560547,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.5740154981613159,
"logits/rejected": -0.5684762597084045,
"logps/chosen": -72.73410034179688,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -164.7735137939453,
"loss": 0.8967,
"margin_dpo/margin_mean": 53.92461013793945,
"margin_dpo/margin_std": 59.920475006103516,
"step": 130
},
{
"KL/chosen_KL_mean": -23.585315704345703,
"KL/mean": -49.33613204956055,
"KL/rejected_KL_mean": -75.08694458007812,
"KL/std": 47.14921951293945,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.014925356954336166,
"fcm_dpo/delta": -0.18314488232135773,
"fcm_dpo/margin": 51.501625061035156,
"fcm_dpo/q_t": 0.33481913805007935,
"grad_norm": 26.255428314208984,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.593375563621521,
"logits/rejected": -0.5982913970947266,
"logps/chosen": -68.50990295410156,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -163.5309600830078,
"loss": 0.8857,
"margin_dpo/margin_mean": 51.501625061035156,
"margin_dpo/margin_std": 52.060943603515625,
"step": 131
},
{
"KL/chosen_KL_mean": -32.59339141845703,
"KL/mean": -54.82532501220703,
"KL/rejected_KL_mean": -77.05726623535156,
"KL/std": 48.68935012817383,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.014653876423835754,
"fcm_dpo/delta": -0.05542517825961113,
"fcm_dpo/margin": 44.46385955810547,
"fcm_dpo/q_t": 0.3683886229991913,
"grad_norm": 25.4652156829834,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.6186190843582153,
"logits/rejected": -0.6089913249015808,
"logps/chosen": -91.594482421875,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -164.9494171142578,
"loss": 0.9932,
"margin_dpo/margin_mean": 44.46385955810547,
"margin_dpo/margin_std": 61.32796096801758,
"step": 132
},
{
"KL/chosen_KL_mean": -38.15018844604492,
"KL/mean": -57.84043502807617,
"KL/rejected_KL_mean": -77.53068542480469,
"KL/std": 44.931976318359375,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.01467297226190567,
"fcm_dpo/delta": 0.023506367579102516,
"fcm_dpo/margin": 39.3804931640625,
"fcm_dpo/q_t": 0.38113081455230713,
"grad_norm": 27.35451316833496,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.6102226972579956,
"logits/rejected": -0.5947215557098389,
"logps/chosen": -104.75468444824219,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -173.8642578125,
"loss": 1.0264,
"margin_dpo/margin_mean": 39.3804931640625,
"margin_dpo/margin_std": 55.648597717285156,
"step": 133
},
{
"KL/chosen_KL_mean": -30.243938446044922,
"KL/mean": -51.90043640136719,
"KL/rejected_KL_mean": -73.55693054199219,
"KL/std": 44.689247131347656,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.014689350500702858,
"fcm_dpo/delta": -0.038860246539115906,
"fcm_dpo/margin": 43.31299591064453,
"fcm_dpo/q_t": 0.3648633360862732,
"grad_norm": 23.86644744873047,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.5801087617874146,
"logits/rejected": -0.5580745935440063,
"logps/chosen": -82.31319427490234,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -161.21145629882812,
"loss": 0.9561,
"margin_dpo/margin_mean": 43.31299591064453,
"margin_dpo/margin_std": 49.729827880859375,
"step": 134
},
{
"KL/chosen_KL_mean": -34.838279724121094,
"KL/mean": -63.11912536621094,
"KL/rejected_KL_mean": -91.39996337890625,
"KL/std": 56.09346008300781,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.014142518863081932,
"fcm_dpo/delta": -0.217301607131958,
"fcm_dpo/margin": 56.56169509887695,
"fcm_dpo/q_t": 0.3403737545013428,
"grad_norm": 25.872745513916016,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.5608731508255005,
"logits/rejected": -0.5899795293807983,
"logps/chosen": -85.192138671875,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -207.3797149658203,
"loss": 0.9125,
"margin_dpo/margin_mean": 56.56169891357422,
"margin_dpo/margin_std": 66.48002624511719,
"step": 135
},
{
"KL/chosen_KL_mean": -40.65992736816406,
"KL/mean": -60.256736755371094,
"KL/rejected_KL_mean": -79.85354614257812,
"KL/std": 50.378326416015625,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.013983258977532387,
"fcm_dpo/delta": 0.05406992509961128,
"fcm_dpo/margin": 39.19361877441406,
"fcm_dpo/q_t": 0.3942437171936035,
"grad_norm": 27.28200340270996,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.5846861600875854,
"logits/rejected": -0.5700664520263672,
"logps/chosen": -105.73243713378906,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -176.17477416992188,
"loss": 1.0684,
"margin_dpo/margin_mean": 39.19361877441406,
"margin_dpo/margin_std": 62.954200744628906,
"step": 136
},
{
"KL/chosen_KL_mean": -40.9674072265625,
"KL/mean": -71.33726501464844,
"KL/rejected_KL_mean": -101.70710754394531,
"KL/std": 68.98532104492188,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.013596372678875923,
"fcm_dpo/delta": -0.24686267971992493,
"fcm_dpo/margin": 60.739707946777344,
"fcm_dpo/q_t": 0.3448226749897003,
"grad_norm": 24.049076080322266,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.4880056381225586,
"logits/rejected": -0.5109074115753174,
"logps/chosen": -89.72652435302734,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -215.5708770751953,
"loss": 0.9457,
"margin_dpo/margin_mean": 60.739707946777344,
"margin_dpo/margin_std": 81.2965087890625,
"step": 137
},
{
"KL/chosen_KL_mean": -42.5715446472168,
"KL/mean": -65.39909362792969,
"KL/rejected_KL_mean": -88.22665405273438,
"KL/std": 52.00640106201172,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.013456525281071663,
"fcm_dpo/delta": -0.015391625463962555,
"fcm_dpo/margin": 45.65510559082031,
"fcm_dpo/q_t": 0.36690467596054077,
"grad_norm": 25.93773651123047,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.5287716388702393,
"logits/rejected": -0.5146108269691467,
"logps/chosen": -103.09119415283203,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -181.42359924316406,
"loss": 0.9767,
"margin_dpo/margin_mean": 45.65510559082031,
"margin_dpo/margin_std": 53.583351135253906,
"step": 138
},
{
"KL/chosen_KL_mean": -33.96821594238281,
"KL/mean": -62.38645553588867,
"KL/rejected_KL_mean": -90.8046875,
"KL/std": 50.296024322509766,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.01313326321542263,
"fcm_dpo/delta": -0.15800537168979645,
"fcm_dpo/margin": 56.83647537231445,
"fcm_dpo/q_t": 0.34398433566093445,
"grad_norm": 23.08481788635254,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.5441822409629822,
"logits/rejected": -0.529821515083313,
"logps/chosen": -80.85960388183594,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -170.5326690673828,
"loss": 0.9066,
"margin_dpo/margin_mean": 56.83647537231445,
"margin_dpo/margin_std": 62.40201187133789,
"step": 139
},
{
"KL/chosen_KL_mean": -41.433631896972656,
"KL/mean": -67.50386047363281,
"KL/rejected_KL_mean": -93.5740966796875,
"KL/std": 57.14271545410156,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.012833508662879467,
"fcm_dpo/delta": -0.07425808161497116,
"fcm_dpo/margin": 52.140464782714844,
"fcm_dpo/q_t": 0.36246025562286377,
"grad_norm": 25.392820358276367,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.5448036193847656,
"logits/rejected": -0.528033971786499,
"logps/chosen": -100.40834045410156,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -176.8582000732422,
"loss": 0.9784,
"margin_dpo/margin_mean": 52.14046096801758,
"margin_dpo/margin_std": 67.23936462402344,
"step": 140
},
{
"KL/chosen_KL_mean": -43.79816818237305,
"KL/mean": -68.73210144042969,
"KL/rejected_KL_mean": -93.66603088378906,
"KL/std": 57.7413330078125,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.01274183951318264,
"fcm_dpo/delta": -0.03779372200369835,
"fcm_dpo/margin": 49.86785888671875,
"fcm_dpo/q_t": 0.3641555905342102,
"grad_norm": 28.14995765686035,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.5279260277748108,
"logits/rejected": -0.5093116164207458,
"logps/chosen": -118.87383270263672,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -191.85829162597656,
"loss": 0.9952,
"margin_dpo/margin_mean": 49.867855072021484,
"margin_dpo/margin_std": 65.07711029052734,
"step": 141
},
{
"KL/chosen_KL_mean": -47.75823974609375,
"KL/mean": -76.29313659667969,
"KL/rejected_KL_mean": -104.82803344726562,
"KL/std": 66.47343444824219,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.012545912526547909,
"fcm_dpo/delta": -0.12489978969097137,
"fcm_dpo/margin": 57.069786071777344,
"fcm_dpo/q_t": 0.35846227407455444,
"grad_norm": 28.82069206237793,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.48032519221305847,
"logits/rejected": -0.4738243818283081,
"logps/chosen": -105.78617095947266,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -199.41024780273438,
"loss": 0.9933,
"margin_dpo/margin_mean": 57.069786071777344,
"margin_dpo/margin_std": 77.73387145996094,
"step": 142
},
{
"KL/chosen_KL_mean": -51.977149963378906,
"KL/mean": -70.17994689941406,
"KL/rejected_KL_mean": -88.38274383544922,
"KL/std": 59.076881408691406,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.01261083036661148,
"fcm_dpo/delta": 0.147918239235878,
"fcm_dpo/margin": 36.405601501464844,
"fcm_dpo/q_t": 0.4072745442390442,
"grad_norm": 31.163190841674805,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.4900055229663849,
"logits/rejected": -0.47338736057281494,
"logps/chosen": -109.57360076904297,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -167.38232421875,
"loss": 1.1336,
"margin_dpo/margin_mean": 36.40559768676758,
"margin_dpo/margin_std": 67.0206527709961,
"step": 143
},
{
"KL/chosen_KL_mean": -43.142234802246094,
"KL/mean": -66.69685363769531,
"KL/rejected_KL_mean": -90.25149536132812,
"KL/std": 49.121246337890625,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.012746628373861313,
"fcm_dpo/delta": -0.0005554920062422752,
"fcm_dpo/margin": 47.109256744384766,
"fcm_dpo/q_t": 0.3711130619049072,
"grad_norm": 25.04686164855957,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.49619781970977783,
"logits/rejected": -0.46890437602996826,
"logps/chosen": -103.04859924316406,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -172.2517547607422,
"loss": 0.9885,
"margin_dpo/margin_mean": 47.109256744384766,
"margin_dpo/margin_std": 57.16564178466797,
"step": 144
},
{
"KL/chosen_KL_mean": -44.95766830444336,
"KL/mean": -69.80555725097656,
"KL/rejected_KL_mean": -94.6534423828125,
"KL/std": 52.95355224609375,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.012686775997281075,
"fcm_dpo/delta": -0.03260120004415512,
"fcm_dpo/margin": 49.695777893066406,
"fcm_dpo/q_t": 0.366777628660202,
"grad_norm": 30.272216796875,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.4978819489479065,
"logits/rejected": -0.4692661762237549,
"logps/chosen": -101.55833435058594,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -172.51976013183594,
"loss": 0.9823,
"margin_dpo/margin_mean": 49.695777893066406,
"margin_dpo/margin_std": 61.764617919921875,
"step": 145
},
{
"KL/chosen_KL_mean": -61.40955352783203,
"KL/mean": -81.01863098144531,
"KL/rejected_KL_mean": -100.6277084350586,
"KL/std": 55.258060455322266,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.012825988233089447,
"fcm_dpo/delta": 0.1022133007645607,
"fcm_dpo/margin": 39.218162536621094,
"fcm_dpo/q_t": 0.3929356038570404,
"grad_norm": 38.18339920043945,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.47510290145874023,
"logits/rejected": -0.43386411666870117,
"logps/chosen": -127.41000366210938,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -182.33050537109375,
"loss": 1.104,
"margin_dpo/margin_mean": 39.21815872192383,
"margin_dpo/margin_std": 65.34715270996094,
"step": 146
},
{
"KL/chosen_KL_mean": -44.859989166259766,
"KL/mean": -71.52595520019531,
"KL/rejected_KL_mean": -98.19192504882812,
"KL/std": 57.56193923950195,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.012739801779389381,
"fcm_dpo/delta": -0.08569633960723877,
"fcm_dpo/margin": 53.331939697265625,
"fcm_dpo/q_t": 0.3615683317184448,
"grad_norm": 26.46401023864746,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.5343322157859802,
"logits/rejected": -0.506342887878418,
"logps/chosen": -98.26547241210938,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -169.58255004882812,
"loss": 0.981,
"margin_dpo/margin_mean": 53.33194351196289,
"margin_dpo/margin_std": 70.01872253417969,
"step": 147
},
{
"KL/chosen_KL_mean": -43.625186920166016,
"KL/mean": -65.13818359375,
"KL/rejected_KL_mean": -86.65118408203125,
"KL/std": 54.32254409790039,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.012961819767951965,
"fcm_dpo/delta": 0.041104190051555634,
"fcm_dpo/margin": 43.0260009765625,
"fcm_dpo/q_t": 0.38501986861228943,
"grad_norm": 25.624500274658203,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.576453447341919,
"logits/rejected": -0.5655009746551514,
"logps/chosen": -108.56227111816406,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -189.74502563476562,
"loss": 1.0362,
"margin_dpo/margin_mean": 43.0260009765625,
"margin_dpo/margin_std": 57.388648986816406,
"step": 148
},
{
"KL/chosen_KL_mean": -42.669620513916016,
"KL/mean": -69.8863296508789,
"KL/rejected_KL_mean": -97.10304260253906,
"KL/std": 51.52314758300781,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.012527845799922943,
"fcm_dpo/delta": -0.09101234376430511,
"fcm_dpo/margin": 54.43341827392578,
"fcm_dpo/q_t": 0.3537212610244751,
"grad_norm": 24.742643356323242,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.573143720626831,
"logits/rejected": -0.5531454086303711,
"logps/chosen": -101.14338684082031,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -196.41778564453125,
"loss": 0.9406,
"margin_dpo/margin_mean": 54.43341827392578,
"margin_dpo/margin_std": 59.89306640625,
"step": 149
},
{
"KL/chosen_KL_mean": -38.254905700683594,
"KL/mean": -64.14033508300781,
"KL/rejected_KL_mean": -90.02577209472656,
"KL/std": 60.019920349121094,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.012382544577121735,
"fcm_dpo/delta": -0.04652421921491623,
"fcm_dpo/margin": 51.77086639404297,
"fcm_dpo/q_t": 0.3726668953895569,
"grad_norm": 25.004823684692383,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.5525408983230591,
"logits/rejected": -0.552293062210083,
"logps/chosen": -83.9607162475586,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -173.3733673095703,
"loss": 0.9937,
"margin_dpo/margin_mean": 51.77086639404297,
"margin_dpo/margin_std": 70.93170166015625,
"step": 150
},
{
"KL/chosen_KL_mean": -45.150596618652344,
"KL/mean": -72.27715301513672,
"KL/rejected_KL_mean": -99.40371704101562,
"KL/std": 56.025184631347656,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.01235053688287735,
"fcm_dpo/delta": -0.07504402846097946,
"fcm_dpo/margin": 54.25311279296875,
"fcm_dpo/q_t": 0.3555821180343628,
"grad_norm": 28.739370346069336,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.6167875528335571,
"logits/rejected": -0.5834023952484131,
"logps/chosen": -115.721435546875,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -199.86752319335938,
"loss": 0.9414,
"margin_dpo/margin_mean": 54.253116607666016,
"margin_dpo/margin_std": 60.01045227050781,
"step": 151
},
{
"KL/chosen_KL_mean": -39.41002655029297,
"KL/mean": -70.39037322998047,
"KL/rejected_KL_mean": -101.3707275390625,
"KL/std": 57.81626892089844,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.012059513479471207,
"fcm_dpo/delta": -0.1588035672903061,
"fcm_dpo/margin": 61.960693359375,
"fcm_dpo/q_t": 0.34607693552970886,
"grad_norm": 25.344608306884766,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.5937477350234985,
"logits/rejected": -0.5869547128677368,
"logps/chosen": -99.57441711425781,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -207.5111846923828,
"loss": 0.9203,
"margin_dpo/margin_mean": 61.960697174072266,
"margin_dpo/margin_std": 69.24900817871094,
"step": 152
},
{
"KL/chosen_KL_mean": -39.57324981689453,
"KL/mean": -66.1493148803711,
"KL/rejected_KL_mean": -92.72538757324219,
"KL/std": 60.61945724487305,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.011878933757543564,
"fcm_dpo/delta": -0.03354384005069733,
"fcm_dpo/margin": 53.152137756347656,
"fcm_dpo/q_t": 0.3712490499019623,
"grad_norm": 21.027040481567383,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.5531604290008545,
"logits/rejected": -0.5418503284454346,
"logps/chosen": -95.88851928710938,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -178.3812255859375,
"loss": 1.0002,
"margin_dpo/margin_mean": 53.15213394165039,
"margin_dpo/margin_std": 72.83201599121094,
"step": 153
},
{
"KL/chosen_KL_mean": -47.23925018310547,
"KL/mean": -73.58146667480469,
"KL/rejected_KL_mean": -99.92366027832031,
"KL/std": 61.87311553955078,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.011858733370900154,
"fcm_dpo/delta": -0.027097493410110474,
"fcm_dpo/margin": 52.684425354003906,
"fcm_dpo/q_t": 0.3730657696723938,
"grad_norm": 25.586259841918945,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.5674915313720703,
"logits/rejected": -0.5634927153587341,
"logps/chosen": -109.98181915283203,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -204.16786193847656,
"loss": 1.0311,
"margin_dpo/margin_mean": 52.684425354003906,
"margin_dpo/margin_std": 75.66905212402344,
"step": 154
},
{
"KL/chosen_KL_mean": -42.23951721191406,
"KL/mean": -69.29158020019531,
"KL/rejected_KL_mean": -96.34364318847656,
"KL/std": 58.670745849609375,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.01174941286444664,
"fcm_dpo/delta": -0.038324981927871704,
"fcm_dpo/margin": 54.1041259765625,
"fcm_dpo/q_t": 0.3663828372955322,
"grad_norm": 23.913837432861328,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.5925284624099731,
"logits/rejected": -0.5683289766311646,
"logps/chosen": -102.8927001953125,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -173.83584594726562,
"loss": 0.987,
"margin_dpo/margin_mean": 54.1041259765625,
"margin_dpo/margin_std": 69.77940368652344,
"step": 155
},
{
"KL/chosen_KL_mean": -59.46400833129883,
"KL/mean": -76.07966613769531,
"KL/rejected_KL_mean": -92.6953125,
"KL/std": 58.8329963684082,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.012031801044940948,
"fcm_dpo/delta": 0.20871999859809875,
"fcm_dpo/margin": 33.23129653930664,
"fcm_dpo/q_t": 0.4158884882926941,
"grad_norm": 38.676578521728516,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.5450568199157715,
"logits/rejected": -0.5246114134788513,
"logps/chosen": -128.9558868408203,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -169.86460876464844,
"loss": 1.2218,
"margin_dpo/margin_mean": 33.231300354003906,
"margin_dpo/margin_std": 78.52706909179688,
"step": 156
},
{
"KL/chosen_KL_mean": -54.536277770996094,
"KL/mean": -87.01248168945312,
"KL/rejected_KL_mean": -119.48870849609375,
"KL/std": 64.35616302490234,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.011773204430937767,
"fcm_dpo/delta": -0.18050672113895416,
"fcm_dpo/margin": 64.95243072509766,
"fcm_dpo/q_t": 0.33803147077560425,
"grad_norm": 27.57537078857422,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.5271573662757874,
"logits/rejected": -0.5269450545310974,
"logps/chosen": -115.90470886230469,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -227.13507080078125,
"loss": 0.9174,
"margin_dpo/margin_mean": 64.95243072509766,
"margin_dpo/margin_std": 71.71643829345703,
"step": 157
},
{
"KL/chosen_KL_mean": -52.93199157714844,
"KL/mean": -86.73419189453125,
"KL/rejected_KL_mean": -120.53638458251953,
"KL/std": 80.6708984375,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.011422505602240562,
"fcm_dpo/delta": -0.18662995100021362,
"fcm_dpo/margin": 67.6043930053711,
"fcm_dpo/q_t": 0.3564373850822449,
"grad_norm": 23.633398056030273,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.4981531500816345,
"logits/rejected": -0.5083768367767334,
"logps/chosen": -110.54490661621094,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -234.23106384277344,
"loss": 0.9705,
"margin_dpo/margin_mean": 67.6043930053711,
"margin_dpo/margin_std": 95.4510498046875,
"step": 158
},
{
"KL/chosen_KL_mean": -60.19736862182617,
"KL/mean": -84.65406036376953,
"KL/rejected_KL_mean": -109.11074829101562,
"KL/std": 69.81906127929688,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.011444485746324062,
"fcm_dpo/delta": 0.04176339507102966,
"fcm_dpo/margin": 48.913368225097656,
"fcm_dpo/q_t": 0.3848469853401184,
"grad_norm": 29.66965103149414,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.5534458160400391,
"logits/rejected": -0.5163393020629883,
"logps/chosen": -141.75772094726562,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -198.00946044921875,
"loss": 1.0649,
"margin_dpo/margin_mean": 48.91337203979492,
"margin_dpo/margin_std": 75.9297866821289,
"step": 159
},
{
"KL/chosen_KL_mean": -60.180809020996094,
"KL/mean": -91.90692901611328,
"KL/rejected_KL_mean": -123.63304901123047,
"KL/std": 74.26970672607422,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.011158831417560577,
"fcm_dpo/delta": -0.117271289229393,
"fcm_dpo/margin": 63.452239990234375,
"fcm_dpo/q_t": 0.3646489977836609,
"grad_norm": 25.340007781982422,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.517310380935669,
"logits/rejected": -0.5013754963874817,
"logps/chosen": -125.91168975830078,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -220.85086059570312,
"loss": 1.0045,
"margin_dpo/margin_mean": 63.452239990234375,
"margin_dpo/margin_std": 92.37104797363281,
"step": 160
},
{
"KL/chosen_KL_mean": -52.541419982910156,
"KL/mean": -80.59986877441406,
"KL/rejected_KL_mean": -108.6583251953125,
"KL/std": 60.95550537109375,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.011109799146652222,
"fcm_dpo/delta": -0.024999581277370453,
"fcm_dpo/margin": 56.11690902709961,
"fcm_dpo/q_t": 0.37251612544059753,
"grad_norm": 25.082792282104492,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.5304283499717712,
"logits/rejected": -0.5198980569839478,
"logps/chosen": -104.97789001464844,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -192.08926391601562,
"loss": 1.008,
"margin_dpo/margin_mean": 56.116912841796875,
"margin_dpo/margin_std": 76.4130859375,
"step": 161
},
{
"KL/chosen_KL_mean": -49.14342498779297,
"KL/mean": -78.0987548828125,
"KL/rejected_KL_mean": -107.05409240722656,
"KL/std": 68.20083618164062,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.010901303961873055,
"fcm_dpo/delta": -0.0365702286362648,
"fcm_dpo/margin": 57.910675048828125,
"fcm_dpo/q_t": 0.37162917852401733,
"grad_norm": 23.26062774658203,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.558234453201294,
"logits/rejected": -0.5330092906951904,
"logps/chosen": -111.75401306152344,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -196.44467163085938,
"loss": 1.0083,
"margin_dpo/margin_mean": 57.910675048828125,
"margin_dpo/margin_std": 78.99287414550781,
"step": 162
},
{
"KL/chosen_KL_mean": -54.389244079589844,
"KL/mean": -82.12598419189453,
"KL/rejected_KL_mean": -109.86273193359375,
"KL/std": 63.845420837402344,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.010996324941515923,
"fcm_dpo/delta": -0.010656844824552536,
"fcm_dpo/margin": 55.473487854003906,
"fcm_dpo/q_t": 0.3742431700229645,
"grad_norm": 23.7772159576416,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.5061922073364258,
"logits/rejected": -0.4807218313217163,
"logps/chosen": -109.41087341308594,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -185.28094482421875,
"loss": 1.0072,
"margin_dpo/margin_mean": 55.473487854003906,
"margin_dpo/margin_std": 74.86231231689453,
"step": 163
},
{
"KL/chosen_KL_mean": -50.56852340698242,
"KL/mean": -83.409912109375,
"KL/rejected_KL_mean": -116.25129699707031,
"KL/std": 64.22931671142578,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.010697158053517342,
"fcm_dpo/delta": -0.1133529394865036,
"fcm_dpo/margin": 65.68278503417969,
"fcm_dpo/q_t": 0.3510741591453552,
"grad_norm": 25.257909774780273,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.5137403011322021,
"logits/rejected": -0.49089929461479187,
"logps/chosen": -106.20919036865234,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -195.91592407226562,
"loss": 0.9313,
"margin_dpo/margin_mean": 65.68278503417969,
"margin_dpo/margin_std": 73.34828186035156,
"step": 164
},
{
"KL/chosen_KL_mean": -55.686607360839844,
"KL/mean": -77.25547790527344,
"KL/rejected_KL_mean": -98.82435607910156,
"KL/std": 53.64445877075195,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.010933227837085724,
"fcm_dpo/delta": 0.134904682636261,
"fcm_dpo/margin": 43.13774108886719,
"fcm_dpo/q_t": 0.39930182695388794,
"grad_norm": 41.1522102355957,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.5242682695388794,
"logits/rejected": -0.4860084056854248,
"logps/chosen": -116.99729919433594,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -172.49496459960938,
"loss": 1.0939,
"margin_dpo/margin_mean": 43.13773727416992,
"margin_dpo/margin_std": 67.83528137207031,
"step": 165
},
{
"KL/chosen_KL_mean": -50.98905944824219,
"KL/mean": -89.55670166015625,
"KL/rejected_KL_mean": -128.12435913085938,
"KL/std": 79.5724868774414,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.01057855598628521,
"fcm_dpo/delta": -0.23768538236618042,
"fcm_dpo/margin": 77.13528442382812,
"fcm_dpo/q_t": 0.34610509872436523,
"grad_norm": 22.230655670166016,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.47835230827331543,
"logits/rejected": -0.4880477488040924,
"logps/chosen": -101.97265625,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -226.219482421875,
"loss": 0.9196,
"margin_dpo/margin_mean": 77.13528442382812,
"margin_dpo/margin_std": 98.92354583740234,
"step": 166
},
{
"KL/chosen_KL_mean": -51.534278869628906,
"KL/mean": -92.67387390136719,
"KL/rejected_KL_mean": -133.81346130371094,
"KL/std": 73.3492202758789,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.010119207203388214,
"fcm_dpo/delta": -0.25405317544937134,
"fcm_dpo/margin": 82.27918243408203,
"fcm_dpo/q_t": 0.32515865564346313,
"grad_norm": 22.403348922729492,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.4624894857406616,
"logits/rejected": -0.46389204263687134,
"logps/chosen": -101.9583740234375,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -229.84388732910156,
"loss": 0.8571,
"margin_dpo/margin_mean": 82.27918243408203,
"margin_dpo/margin_std": 81.84159851074219,
"step": 167
},
{
"KL/chosen_KL_mean": -57.26702880859375,
"KL/mean": -88.62904357910156,
"KL/rejected_KL_mean": -119.99104309082031,
"KL/std": 69.72494506835938,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.009935762733221054,
"fcm_dpo/delta": -0.02493857778608799,
"fcm_dpo/margin": 62.724021911621094,
"fcm_dpo/q_t": 0.3682708144187927,
"grad_norm": 24.072589874267578,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.43985286355018616,
"logits/rejected": -0.4221440851688385,
"logps/chosen": -106.7298583984375,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -195.29959106445312,
"loss": 0.9749,
"margin_dpo/margin_mean": 62.72401809692383,
"margin_dpo/margin_std": 75.86625671386719,
"step": 168
},
{
"KL/chosen_KL_mean": -57.34328842163086,
"KL/mean": -85.20939636230469,
"KL/rejected_KL_mean": -113.07550048828125,
"KL/std": 70.55574035644531,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.009921474382281303,
"fcm_dpo/delta": 0.04828646779060364,
"fcm_dpo/margin": 55.73221969604492,
"fcm_dpo/q_t": 0.39203399419784546,
"grad_norm": 27.592586517333984,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.4907569885253906,
"logits/rejected": -0.47010350227355957,
"logps/chosen": -117.146728515625,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -196.42124938964844,
"loss": 1.0869,
"margin_dpo/margin_mean": 55.73221969604492,
"margin_dpo/margin_std": 94.890380859375,
"step": 169
},
{
"KL/chosen_KL_mean": -52.603553771972656,
"KL/mean": -86.20280456542969,
"KL/rejected_KL_mean": -119.80204772949219,
"KL/std": 66.4590835571289,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.009865409694612026,
"fcm_dpo/delta": -0.06916055083274841,
"fcm_dpo/margin": 67.19849395751953,
"fcm_dpo/q_t": 0.35786378383636475,
"grad_norm": 24.34227752685547,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.44147998094558716,
"logits/rejected": -0.42527681589126587,
"logps/chosen": -102.07533264160156,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -195.71939086914062,
"loss": 0.9609,
"margin_dpo/margin_mean": 67.19850158691406,
"margin_dpo/margin_std": 77.97476196289062,
"step": 170
},
{
"KL/chosen_KL_mean": -75.36102294921875,
"KL/mean": -102.78938293457031,
"KL/rejected_KL_mean": -130.21774291992188,
"KL/std": 74.64979553222656,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.009982212446630001,
"fcm_dpo/delta": 0.05546961724758148,
"fcm_dpo/margin": 54.856719970703125,
"fcm_dpo/q_t": 0.39181768894195557,
"grad_norm": 31.270261764526367,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.474338173866272,
"logits/rejected": -0.4445253014564514,
"logps/chosen": -159.86033630371094,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -239.599853515625,
"loss": 1.086,
"margin_dpo/margin_mean": 54.856719970703125,
"margin_dpo/margin_std": 93.97332000732422,
"step": 171
},
{
"KL/chosen_KL_mean": -70.63848876953125,
"KL/mean": -99.10212707519531,
"KL/rejected_KL_mean": -127.56576538085938,
"KL/std": 74.89137268066406,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.010037798434495926,
"fcm_dpo/delta": 0.030263658612966537,
"fcm_dpo/margin": 56.927268981933594,
"fcm_dpo/q_t": 0.38131678104400635,
"grad_norm": 28.78266143798828,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.4877288341522217,
"logits/rejected": -0.4573853611946106,
"logps/chosen": -139.29241943359375,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -213.00244140625,
"loss": 1.0455,
"margin_dpo/margin_mean": 56.927268981933594,
"margin_dpo/margin_std": 84.20733642578125,
"step": 172
},
{
"KL/chosen_KL_mean": -63.48297119140625,
"KL/mean": -92.25379943847656,
"KL/rejected_KL_mean": -121.02462768554688,
"KL/std": 69.28123474121094,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.010144222527742386,
"fcm_dpo/delta": 0.016874615103006363,
"fcm_dpo/margin": 57.541648864746094,
"fcm_dpo/q_t": 0.3754703402519226,
"grad_norm": 27.97823715209961,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.5130044221878052,
"logits/rejected": -0.4780956208705902,
"logps/chosen": -126.53384399414062,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -199.70855712890625,
"loss": 1.0014,
"margin_dpo/margin_mean": 57.541656494140625,
"margin_dpo/margin_std": 72.86973571777344,
"step": 173
},
{
"KL/chosen_KL_mean": -62.479461669921875,
"KL/mean": -94.98529815673828,
"KL/rejected_KL_mean": -127.49114990234375,
"KL/std": 76.12448120117188,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.010098971426486969,
"fcm_dpo/delta": -0.061310555785894394,
"fcm_dpo/margin": 65.01167297363281,
"fcm_dpo/q_t": 0.3680018186569214,
"grad_norm": 32.727054595947266,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.39561927318573,
"logits/rejected": -0.4122796058654785,
"logps/chosen": -115.84243774414062,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -229.40234375,
"loss": 0.9974,
"margin_dpo/margin_mean": 65.01167297363281,
"margin_dpo/margin_std": 87.49495697021484,
"step": 174
},
{
"KL/chosen_KL_mean": -59.786537170410156,
"KL/mean": -104.62965393066406,
"KL/rejected_KL_mean": -149.4727783203125,
"KL/std": 82.91730499267578,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.00956171564757824,
"fcm_dpo/delta": -0.2823520302772522,
"fcm_dpo/margin": 89.68624877929688,
"fcm_dpo/q_t": 0.3200307786464691,
"grad_norm": 28.917924880981445,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.37511971592903137,
"logits/rejected": -0.3597720265388489,
"logps/chosen": -105.20429992675781,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -238.97857666015625,
"loss": 0.8333,
"margin_dpo/margin_mean": 89.68624877929688,
"margin_dpo/margin_std": 85.16973876953125,
"step": 175
},
{
"KL/chosen_KL_mean": -61.96738052368164,
"KL/mean": -98.81568908691406,
"KL/rejected_KL_mean": -135.66400146484375,
"KL/std": 77.03422546386719,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.009335162118077278,
"fcm_dpo/delta": -0.09436807036399841,
"fcm_dpo/margin": 73.69662475585938,
"fcm_dpo/q_t": 0.359141081571579,
"grad_norm": 26.62580108642578,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.3945273756980896,
"logits/rejected": -0.4108158349990845,
"logps/chosen": -112.42021942138672,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -231.22296142578125,
"loss": 0.9499,
"margin_dpo/margin_mean": 73.69662475585938,
"margin_dpo/margin_std": 88.63804626464844,
"step": 176
},
{
"KL/chosen_KL_mean": -70.05691528320312,
"KL/mean": -107.1580810546875,
"KL/rejected_KL_mean": -144.25924682617188,
"KL/std": 84.10592651367188,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.009182149544358253,
"fcm_dpo/delta": -0.0872758999466896,
"fcm_dpo/margin": 74.20231628417969,
"fcm_dpo/q_t": 0.35946571826934814,
"grad_norm": 31.329692840576172,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.4482751488685608,
"logits/rejected": -0.43557900190353394,
"logps/chosen": -131.27337646484375,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -240.1530303955078,
"loss": 0.9515,
"margin_dpo/margin_mean": 74.20231628417969,
"margin_dpo/margin_std": 89.37190246582031,
"step": 177
},
{
"KL/chosen_KL_mean": -79.5255126953125,
"KL/mean": -126.40957641601562,
"KL/rejected_KL_mean": -173.29364013671875,
"KL/std": 102.68285369873047,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.008733097463846207,
"fcm_dpo/delta": -0.2405625581741333,
"fcm_dpo/margin": 93.76814270019531,
"fcm_dpo/q_t": 0.33788061141967773,
"grad_norm": 30.99517250061035,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.3406079411506653,
"logits/rejected": -0.33698827028274536,
"logps/chosen": -137.79029846191406,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -278.65899658203125,
"loss": 0.9021,
"margin_dpo/margin_mean": 93.76814270019531,
"margin_dpo/margin_std": 111.62127685546875,
"step": 178
},
{
"KL/chosen_KL_mean": -78.37075805664062,
"KL/mean": -113.49848175048828,
"KL/rejected_KL_mean": -148.62620544433594,
"KL/std": 84.50762176513672,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.008614875376224518,
"fcm_dpo/delta": -0.006374705582857132,
"fcm_dpo/margin": 70.25543975830078,
"fcm_dpo/q_t": 0.3753349483013153,
"grad_norm": 46.0558967590332,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.3983387351036072,
"logits/rejected": -0.4007897973060608,
"logps/chosen": -139.4290771484375,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -239.15402221679688,
"loss": 1.0429,
"margin_dpo/margin_mean": 70.25544738769531,
"margin_dpo/margin_std": 103.2009506225586,
"step": 179
},
{
"KL/chosen_KL_mean": -64.6424789428711,
"KL/mean": -107.07305908203125,
"KL/rejected_KL_mean": -149.50363159179688,
"KL/std": 76.90628814697266,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.008477364666759968,
"fcm_dpo/delta": -0.12914864718914032,
"fcm_dpo/margin": 84.86116027832031,
"fcm_dpo/q_t": 0.34660762548446655,
"grad_norm": 30.963035583496094,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.4074268341064453,
"logits/rejected": -0.39785900712013245,
"logps/chosen": -118.98519897460938,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -247.71546936035156,
"loss": 0.9246,
"margin_dpo/margin_mean": 84.86116027832031,
"margin_dpo/margin_std": 93.85812377929688,
"step": 180
},
{
"KL/chosen_KL_mean": -54.562355041503906,
"KL/mean": -81.32383728027344,
"KL/rejected_KL_mean": -108.08533477783203,
"KL/std": 67.28744506835938,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.008647149428725243,
"fcm_dpo/delta": 0.14334514737129211,
"fcm_dpo/margin": 53.52298355102539,
"fcm_dpo/q_t": 0.40024834871292114,
"grad_norm": 25.58306884765625,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.4212300181388855,
"logits/rejected": -0.3890141248703003,
"logps/chosen": -109.56281280517578,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -169.7415008544922,
"loss": 1.0924,
"margin_dpo/margin_mean": 53.522987365722656,
"margin_dpo/margin_std": 83.47067260742188,
"step": 181
},
{
"KL/chosen_KL_mean": -52.58146286010742,
"KL/mean": -96.33982849121094,
"KL/rejected_KL_mean": -140.09820556640625,
"KL/std": 83.19712829589844,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.008451974019408226,
"fcm_dpo/delta": -0.15216702222824097,
"fcm_dpo/margin": 87.5167236328125,
"fcm_dpo/q_t": 0.34340500831604004,
"grad_norm": 26.283775329589844,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.3738093972206116,
"logits/rejected": -0.37705719470977783,
"logps/chosen": -93.68931579589844,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -229.61978149414062,
"loss": 0.9288,
"margin_dpo/margin_mean": 87.5167236328125,
"margin_dpo/margin_std": 99.29764556884766,
"step": 182
},
{
"KL/chosen_KL_mean": -77.28178405761719,
"KL/mean": -103.30689239501953,
"KL/rejected_KL_mean": -129.33200073242188,
"KL/std": 65.52974700927734,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.008357454091310501,
"fcm_dpo/delta": -0.04225583001971245,
"fcm_dpo/margin": 52.05022430419922,
"fcm_dpo/q_t": 0.4033220410346985,
"grad_norm": 24.674793243408203,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.4310709238052368,
"logits/rejected": -0.41038137674331665,
"logps/chosen": -134.80633544921875,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -205.3077392578125,
"loss": 1.0949,
"margin_dpo/margin_mean": 52.05022430419922,
"margin_dpo/margin_std": 72.7845458984375,
"step": 183
},
{
"KL/chosen_KL_mean": -62.69811248779297,
"KL/mean": -88.97595977783203,
"KL/rejected_KL_mean": -115.25379943847656,
"KL/std": 63.688819885253906,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.008609607815742493,
"fcm_dpo/delta": 0.1524200290441513,
"fcm_dpo/margin": 52.55569076538086,
"fcm_dpo/q_t": 0.4034864604473114,
"grad_norm": 22.63802719116211,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.4859800934791565,
"logits/rejected": -0.47371089458465576,
"logps/chosen": -121.24307250976562,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -191.8878631591797,
"loss": 1.0882,
"margin_dpo/margin_mean": 52.55569076538086,
"margin_dpo/margin_std": 78.50408935546875,
"step": 184
},
{
"KL/chosen_KL_mean": -69.77159881591797,
"KL/mean": -91.69174194335938,
"KL/rejected_KL_mean": -113.61188507080078,
"KL/std": 73.44252014160156,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.008895106613636017,
"fcm_dpo/delta": 0.21872034668922424,
"fcm_dpo/margin": 43.84027862548828,
"fcm_dpo/q_t": 0.42065635323524475,
"grad_norm": 29.499311447143555,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.44560521841049194,
"logits/rejected": -0.4125337302684784,
"logps/chosen": -131.79745483398438,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -187.37448120117188,
"loss": 1.1605,
"margin_dpo/margin_mean": 43.84028244018555,
"margin_dpo/margin_std": 86.58267211914062,
"step": 185
},
{
"KL/chosen_KL_mean": -65.01848602294922,
"KL/mean": -105.216796875,
"KL/rejected_KL_mean": -145.41510009765625,
"KL/std": 78.28663635253906,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.008828350342810154,
"fcm_dpo/delta": -0.11845053732395172,
"fcm_dpo/margin": 80.3966293334961,
"fcm_dpo/q_t": 0.3488392233848572,
"grad_norm": 45.29069900512695,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.4784323573112488,
"logits/rejected": -0.4480966329574585,
"logps/chosen": -134.3719482421875,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -233.487548828125,
"loss": 0.9155,
"margin_dpo/margin_mean": 80.3966293334961,
"margin_dpo/margin_std": 85.997802734375,
"step": 186
},
{
"KL/chosen_KL_mean": -65.43475341796875,
"KL/mean": -97.90451049804688,
"KL/rejected_KL_mean": -130.374267578125,
"KL/std": 74.2392578125,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.008884964510798454,
"fcm_dpo/delta": 0.023186173290014267,
"fcm_dpo/margin": 64.93951416015625,
"fcm_dpo/q_t": 0.37641221284866333,
"grad_norm": 32.4824333190918,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.4087640047073364,
"logits/rejected": -0.40301185846328735,
"logps/chosen": -118.19121551513672,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -212.34336853027344,
"loss": 1.0072,
"margin_dpo/margin_mean": 64.93951416015625,
"margin_dpo/margin_std": 81.59025573730469,
"step": 187
},
{
"KL/chosen_KL_mean": -60.860313415527344,
"KL/mean": -101.803466796875,
"KL/rejected_KL_mean": -142.7466278076172,
"KL/std": 85.30118560791016,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.00868306215852499,
"fcm_dpo/delta": -0.11944706737995148,
"fcm_dpo/margin": 81.88630676269531,
"fcm_dpo/q_t": 0.3520312011241913,
"grad_norm": 38.476314544677734,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.3950979709625244,
"logits/rejected": -0.3972102999687195,
"logps/chosen": -110.27580261230469,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -232.28704833984375,
"loss": 0.931,
"margin_dpo/margin_mean": 81.88630676269531,
"margin_dpo/margin_std": 93.60944366455078,
"step": 188
},
{
"KL/chosen_KL_mean": -76.00115203857422,
"KL/mean": -110.0859375,
"KL/rejected_KL_mean": -144.17071533203125,
"KL/std": 86.78269958496094,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.008646276779472828,
"fcm_dpo/delta": 0.011217039078474045,
"fcm_dpo/margin": 68.16957092285156,
"fcm_dpo/q_t": 0.3808944523334503,
"grad_norm": 40.60920715332031,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.3444288372993469,
"logits/rejected": -0.31729966402053833,
"logps/chosen": -128.40011596679688,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -216.33807373046875,
"loss": 1.0404,
"margin_dpo/margin_mean": 68.16957092285156,
"margin_dpo/margin_std": 103.68344116210938,
"step": 189
},
{
"KL/chosen_KL_mean": -80.78923034667969,
"KL/mean": -119.83253479003906,
"KL/rejected_KL_mean": -158.87582397460938,
"KL/std": 91.08761596679688,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.00864358339458704,
"fcm_dpo/delta": -0.08245991170406342,
"fcm_dpo/margin": 78.08657836914062,
"fcm_dpo/q_t": 0.363553524017334,
"grad_norm": 29.772865295410156,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.37341126799583435,
"logits/rejected": -0.3552435636520386,
"logps/chosen": -145.4722900390625,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -261.4263610839844,
"loss": 1.0056,
"margin_dpo/margin_mean": 78.08657836914062,
"margin_dpo/margin_std": 105.72311401367188,
"step": 190
},
{
"KL/chosen_KL_mean": -73.4832763671875,
"KL/mean": -131.01290893554688,
"KL/rejected_KL_mean": -188.54254150390625,
"KL/std": 110.68049621582031,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.00801704078912735,
"fcm_dpo/delta": -0.35801875591278076,
"fcm_dpo/margin": 115.05926513671875,
"fcm_dpo/q_t": 0.31671005487442017,
"grad_norm": 24.53072166442871,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.3887477517127991,
"logits/rejected": -0.367370069026947,
"logps/chosen": -142.14215087890625,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -298.68218994140625,
"loss": 0.8322,
"margin_dpo/margin_mean": 115.05926513671875,
"margin_dpo/margin_std": 121.57363891601562,
"step": 191
},
{
"KL/chosen_KL_mean": -96.39784240722656,
"KL/mean": -135.2452850341797,
"KL/rejected_KL_mean": -174.09271240234375,
"KL/std": 93.46896362304688,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.007918072864413261,
"fcm_dpo/delta": -0.016324058175086975,
"fcm_dpo/margin": 77.69488525390625,
"fcm_dpo/q_t": 0.37564581632614136,
"grad_norm": 34.51486587524414,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.3592721223831177,
"logits/rejected": -0.3580414652824402,
"logps/chosen": -166.124755859375,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -277.4140625,
"loss": 1.0261,
"margin_dpo/margin_mean": 77.69488525390625,
"margin_dpo/margin_std": 112.80885314941406,
"step": 192
},
{
"KL/chosen_KL_mean": -96.04309844970703,
"KL/mean": -119.06697845458984,
"KL/rejected_KL_mean": -142.09085083007812,
"KL/std": 81.4603271484375,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.008208666928112507,
"fcm_dpo/delta": 0.22873592376708984,
"fcm_dpo/margin": 46.047760009765625,
"fcm_dpo/q_t": 0.41593098640441895,
"grad_norm": 41.52101135253906,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.3570171892642975,
"logits/rejected": -0.3406020402908325,
"logps/chosen": -156.2335968017578,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -218.4984130859375,
"loss": 1.2039,
"margin_dpo/margin_mean": 46.047752380371094,
"margin_dpo/margin_std": 102.21097564697266,
"step": 193
},
{
"KL/chosen_KL_mean": -60.95546340942383,
"KL/mean": -98.21711730957031,
"KL/rejected_KL_mean": -135.478759765625,
"KL/std": 73.04765319824219,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.008216527290642262,
"fcm_dpo/delta": -0.0133447777479887,
"fcm_dpo/margin": 74.52330017089844,
"fcm_dpo/q_t": 0.3653014004230499,
"grad_norm": 23.641489028930664,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.2709447145462036,
"logits/rejected": -0.25166115164756775,
"logps/chosen": -98.79583740234375,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -196.16354370117188,
"loss": 0.9584,
"margin_dpo/margin_mean": 74.52330017089844,
"margin_dpo/margin_std": 78.96966552734375,
"step": 194
},
{
"KL/chosen_KL_mean": -97.21737670898438,
"KL/mean": -135.9637451171875,
"KL/rejected_KL_mean": -174.71014404296875,
"KL/std": 88.60956573486328,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.008178764954209328,
"fcm_dpo/delta": -0.03618188574910164,
"fcm_dpo/margin": 77.49275207519531,
"fcm_dpo/q_t": 0.364003449678421,
"grad_norm": 39.76923751831055,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.30619484186172485,
"logits/rejected": -0.32382965087890625,
"logps/chosen": -152.10894775390625,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -271.4810791015625,
"loss": 0.9628,
"margin_dpo/margin_mean": 77.49275970458984,
"margin_dpo/margin_std": 89.49713134765625,
"step": 195
},
{
"KL/chosen_KL_mean": -73.69876861572266,
"KL/mean": -117.91693115234375,
"KL/rejected_KL_mean": -162.13510131835938,
"KL/std": 90.82259368896484,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.007992200553417206,
"fcm_dpo/delta": -0.11618456244468689,
"fcm_dpo/margin": 88.43630981445312,
"fcm_dpo/q_t": 0.3538973927497864,
"grad_norm": 26.536027908325195,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.30888891220092773,
"logits/rejected": -0.2882790267467499,
"logps/chosen": -126.94401550292969,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -238.18804931640625,
"loss": 0.9507,
"margin_dpo/margin_mean": 88.43630981445312,
"margin_dpo/margin_std": 104.88288879394531,
"step": 196
},
{
"KL/chosen_KL_mean": -73.21094512939453,
"KL/mean": -107.82876586914062,
"KL/rejected_KL_mean": -142.44659423828125,
"KL/std": 76.45281982421875,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.008061784319579601,
"fcm_dpo/delta": 0.04404710605740547,
"fcm_dpo/margin": 69.23564147949219,
"fcm_dpo/q_t": 0.38000333309173584,
"grad_norm": 36.907161712646484,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.3162170648574829,
"logits/rejected": -0.30719810724258423,
"logps/chosen": -133.63128662109375,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -219.65548706054688,
"loss": 1.0066,
"margin_dpo/margin_mean": 69.23564147949219,
"margin_dpo/margin_std": 85.80416870117188,
"step": 197
},
{
"KL/chosen_KL_mean": -79.67300415039062,
"KL/mean": -121.91914367675781,
"KL/rejected_KL_mean": -164.16526794433594,
"KL/std": 94.43048095703125,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.007981689646840096,
"fcm_dpo/delta": -0.07969578355550766,
"fcm_dpo/margin": 84.49227142333984,
"fcm_dpo/q_t": 0.3630481958389282,
"grad_norm": 34.34934997558594,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.37920045852661133,
"logits/rejected": -0.3809783458709717,
"logps/chosen": -134.70919799804688,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -261.40850830078125,
"loss": 0.9638,
"margin_dpo/margin_mean": 84.49227905273438,
"margin_dpo/margin_std": 108.89409637451172,
"step": 198
},
{
"KL/chosen_KL_mean": -78.15796661376953,
"KL/mean": -118.77691650390625,
"KL/rejected_KL_mean": -159.3958740234375,
"KL/std": 84.8035659790039,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.007828679867088795,
"fcm_dpo/delta": -0.03976024314761162,
"fcm_dpo/margin": 81.23789978027344,
"fcm_dpo/q_t": 0.3655822277069092,
"grad_norm": 36.976749420166016,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.37588804960250854,
"logits/rejected": -0.3604142963886261,
"logps/chosen": -134.98678588867188,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -244.04408264160156,
"loss": 0.9872,
"margin_dpo/margin_mean": 81.23789978027344,
"margin_dpo/margin_std": 101.51104736328125,
"step": 199
},
{
"KL/chosen_KL_mean": -73.245361328125,
"KL/mean": -115.81585693359375,
"KL/rejected_KL_mean": -158.38636779785156,
"KL/std": 91.43511199951172,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.007761807180941105,
"fcm_dpo/delta": -0.06572169065475464,
"fcm_dpo/margin": 85.14102172851562,
"fcm_dpo/q_t": 0.36326026916503906,
"grad_norm": 26.900110244750977,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.31254661083221436,
"logits/rejected": -0.28698211908340454,
"logps/chosen": -126.31241607666016,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -238.99481201171875,
"loss": 0.9835,
"margin_dpo/margin_mean": 85.14102172851562,
"margin_dpo/margin_std": 110.77159118652344,
"step": 200
},
{
"KL/chosen_KL_mean": -75.16486358642578,
"KL/mean": -117.37071228027344,
"KL/rejected_KL_mean": -159.57656860351562,
"KL/std": 93.2561264038086,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.007710574194788933,
"fcm_dpo/delta": -0.05437050759792328,
"fcm_dpo/margin": 84.41168212890625,
"fcm_dpo/q_t": 0.3624987304210663,
"grad_norm": 35.565452575683594,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.3871253728866577,
"logits/rejected": -0.38020139932632446,
"logps/chosen": -150.56707763671875,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -274.384765625,
"loss": 0.9746,
"margin_dpo/margin_mean": 84.41168212890625,
"margin_dpo/margin_std": 104.68900299072266,
"step": 201
},
{
"KL/chosen_KL_mean": -80.17449951171875,
"KL/mean": -109.18447875976562,
"KL/rejected_KL_mean": -138.1944580078125,
"KL/std": 79.37278747558594,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.007871449925005436,
"fcm_dpo/delta": 0.14974863827228546,
"fcm_dpo/margin": 58.01996612548828,
"fcm_dpo/q_t": 0.40400004386901855,
"grad_norm": 25.21083641052246,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.3736286163330078,
"logits/rejected": -0.38233280181884766,
"logps/chosen": -130.27581787109375,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -225.1794891357422,
"loss": 1.1165,
"margin_dpo/margin_mean": 58.01996994018555,
"margin_dpo/margin_std": 99.46681213378906,
"step": 202
},
{
"KL/chosen_KL_mean": -77.28059387207031,
"KL/mean": -113.36930847167969,
"KL/rejected_KL_mean": -149.45802307128906,
"KL/std": 80.0845947265625,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.00793900154531002,
"fcm_dpo/delta": 0.028627078980207443,
"fcm_dpo/margin": 72.17743682861328,
"fcm_dpo/q_t": 0.37443578243255615,
"grad_norm": 31.61592674255371,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.3838493227958679,
"logits/rejected": -0.3709094524383545,
"logps/chosen": -137.89028930664062,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -235.35400390625,
"loss": 0.988,
"margin_dpo/margin_mean": 72.17742919921875,
"margin_dpo/margin_std": 83.06932067871094,
"step": 203
},
{
"KL/chosen_KL_mean": -87.4486083984375,
"KL/mean": -121.0989990234375,
"KL/rejected_KL_mean": -154.7493896484375,
"KL/std": 89.3180160522461,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.008036996237933636,
"fcm_dpo/delta": 0.06246567517518997,
"fcm_dpo/margin": 67.30076599121094,
"fcm_dpo/q_t": 0.38982903957366943,
"grad_norm": 36.763545989990234,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.4234277009963989,
"logits/rejected": -0.37935611605644226,
"logps/chosen": -167.6135711669922,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -242.44529724121094,
"loss": 1.0543,
"margin_dpo/margin_mean": 67.30077362060547,
"margin_dpo/margin_std": 101.38398742675781,
"step": 204
},
{
"KL/chosen_KL_mean": -87.98814392089844,
"KL/mean": -131.1959991455078,
"KL/rejected_KL_mean": -174.40386962890625,
"KL/std": 93.3107681274414,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.007870245724916458,
"fcm_dpo/delta": -0.08928148448467255,
"fcm_dpo/margin": 86.41572570800781,
"fcm_dpo/q_t": 0.3547505736351013,
"grad_norm": 26.64239501953125,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.3537455201148987,
"logits/rejected": -0.32149383425712585,
"logps/chosen": -147.3728790283203,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -259.5289306640625,
"loss": 0.97,
"margin_dpo/margin_mean": 86.41572570800781,
"margin_dpo/margin_std": 102.72451782226562,
"step": 205
},
{
"KL/chosen_KL_mean": -82.00848388671875,
"KL/mean": -128.731689453125,
"KL/rejected_KL_mean": -175.4549102783203,
"KL/std": 89.94499969482422,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.007727504707872868,
"fcm_dpo/delta": -0.13269372284412384,
"fcm_dpo/margin": 93.4464111328125,
"fcm_dpo/q_t": 0.34409236907958984,
"grad_norm": 33.7529411315918,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.2978416681289673,
"logits/rejected": -0.301300585269928,
"logps/chosen": -128.97299194335938,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -274.40838623046875,
"loss": 0.9163,
"margin_dpo/margin_mean": 93.4464111328125,
"margin_dpo/margin_std": 99.09393310546875,
"step": 206
},
{
"KL/chosen_KL_mean": -68.27006530761719,
"KL/mean": -123.47262573242188,
"KL/rejected_KL_mean": -178.6751708984375,
"KL/std": 102.2544937133789,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.007452533580362797,
"fcm_dpo/delta": -0.24242129921913147,
"fcm_dpo/margin": 110.40510559082031,
"fcm_dpo/q_t": 0.3324313163757324,
"grad_norm": 30.658599853515625,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.41197437047958374,
"logits/rejected": -0.38378313183784485,
"logps/chosen": -124.32632446289062,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -263.12298583984375,
"loss": 0.8712,
"margin_dpo/margin_mean": 110.40510559082031,
"margin_dpo/margin_std": 117.73478698730469,
"step": 207
},
{
"KL/chosen_KL_mean": -107.02651977539062,
"KL/mean": -149.1810302734375,
"KL/rejected_KL_mean": -191.33555603027344,
"KL/std": 95.59660339355469,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.007310114800930023,
"fcm_dpo/delta": -0.01752464845776558,
"fcm_dpo/margin": 84.30902862548828,
"fcm_dpo/q_t": 0.3686535358428955,
"grad_norm": 30.297536849975586,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.4279463291168213,
"logits/rejected": -0.4104871153831482,
"logps/chosen": -174.09413146972656,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -285.6224365234375,
"loss": 1.0085,
"margin_dpo/margin_mean": 84.30902099609375,
"margin_dpo/margin_std": 112.68521118164062,
"step": 208
},
{
"KL/chosen_KL_mean": -88.67752838134766,
"KL/mean": -125.71156311035156,
"KL/rejected_KL_mean": -162.74560546875,
"KL/std": 82.72760009765625,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.007376100867986679,
"fcm_dpo/delta": 0.05680035427212715,
"fcm_dpo/margin": 74.06808471679688,
"fcm_dpo/q_t": 0.3826533555984497,
"grad_norm": 33.76628112792969,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.32765740156173706,
"logits/rejected": -0.31723886728286743,
"logps/chosen": -144.85922241210938,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -243.6871337890625,
"loss": 1.0439,
"margin_dpo/margin_mean": 74.06808471679688,
"margin_dpo/margin_std": 102.1053466796875,
"step": 209
},
{
"KL/chosen_KL_mean": -78.84635162353516,
"KL/mean": -120.61445617675781,
"KL/rejected_KL_mean": -162.382568359375,
"KL/std": 84.26818084716797,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.007408691570162773,
"fcm_dpo/delta": -0.02034676820039749,
"fcm_dpo/margin": 83.53620910644531,
"fcm_dpo/q_t": 0.36823898553848267,
"grad_norm": 23.638490676879883,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.37462446093559265,
"logits/rejected": -0.3626801073551178,
"logps/chosen": -125.21817016601562,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -239.06417846679688,
"loss": 0.9779,
"margin_dpo/margin_mean": 83.53620910644531,
"margin_dpo/margin_std": 99.59040832519531,
"step": 210
},
{
"KL/chosen_KL_mean": -111.80021667480469,
"KL/mean": -148.89285278320312,
"KL/rejected_KL_mean": -185.98548889160156,
"KL/std": 93.49342346191406,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.007433430291712284,
"fcm_dpo/delta": 0.05135497450828552,
"fcm_dpo/margin": 74.18527221679688,
"fcm_dpo/q_t": 0.3872218728065491,
"grad_norm": 33.154632568359375,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.3995612859725952,
"logits/rejected": -0.35311049222946167,
"logps/chosen": -190.73257446289062,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -272.80645751953125,
"loss": 1.0473,
"margin_dpo/margin_mean": 74.18527221679688,
"margin_dpo/margin_std": 107.32933044433594,
"step": 211
},
{
"KL/chosen_KL_mean": -91.27392578125,
"KL/mean": -142.9376678466797,
"KL/rejected_KL_mean": -194.60140991210938,
"KL/std": 105.95874786376953,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.0071998536586761475,
"fcm_dpo/delta": -0.1594788283109665,
"fcm_dpo/margin": 103.32747650146484,
"fcm_dpo/q_t": 0.3455570936203003,
"grad_norm": 25.711772918701172,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.3940684497356415,
"logits/rejected": -0.38088148832321167,
"logps/chosen": -149.470947265625,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -297.6592712402344,
"loss": 0.9363,
"margin_dpo/margin_mean": 103.32748413085938,
"margin_dpo/margin_std": 115.93907165527344,
"step": 212
},
{
"KL/chosen_KL_mean": -85.07697296142578,
"KL/mean": -135.42556762695312,
"KL/rejected_KL_mean": -185.774169921875,
"KL/std": 94.00886535644531,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.007048967760056257,
"fcm_dpo/delta": -0.1195511743426323,
"fcm_dpo/margin": 100.69718933105469,
"fcm_dpo/q_t": 0.3497133255004883,
"grad_norm": 25.290956497192383,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.4311680793762207,
"logits/rejected": -0.40246063470840454,
"logps/chosen": -152.58969116210938,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -279.6888732910156,
"loss": 0.932,
"margin_dpo/margin_mean": 100.69718933105469,
"margin_dpo/margin_std": 112.16920471191406,
"step": 213
},
{
"KL/chosen_KL_mean": -79.59456634521484,
"KL/mean": -122.1160888671875,
"KL/rejected_KL_mean": -164.63760375976562,
"KL/std": 92.99964141845703,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.007053151726722717,
"fcm_dpo/delta": 0.0001654941588640213,
"fcm_dpo/margin": 85.04305267333984,
"fcm_dpo/q_t": 0.3740694224834442,
"grad_norm": 25.744068145751953,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.3269927501678467,
"logits/rejected": -0.3326476812362671,
"logps/chosen": -121.19945526123047,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -242.155029296875,
"loss": 1.0027,
"margin_dpo/margin_mean": 85.04305267333984,
"margin_dpo/margin_std": 110.74486541748047,
"step": 214
},
{
"KL/chosen_KL_mean": -87.89337921142578,
"KL/mean": -133.9697265625,
"KL/rejected_KL_mean": -180.04608154296875,
"KL/std": 94.43704986572266,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.00696907751262188,
"fcm_dpo/delta": -0.04566160961985588,
"fcm_dpo/margin": 92.1527099609375,
"fcm_dpo/q_t": 0.3617004156112671,
"grad_norm": 24.02394676208496,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.38371890783309937,
"logits/rejected": -0.37113693356513977,
"logps/chosen": -141.17263793945312,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -270.0107421875,
"loss": 0.952,
"margin_dpo/margin_mean": 92.1527099609375,
"margin_dpo/margin_std": 101.48160552978516,
"step": 215
},
{
"KL/chosen_KL_mean": -90.16885375976562,
"KL/mean": -137.26345825195312,
"KL/rejected_KL_mean": -184.3580322265625,
"KL/std": 98.0830078125,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.006948217749595642,
"fcm_dpo/delta": -0.05834150314331055,
"fcm_dpo/margin": 94.18919372558594,
"fcm_dpo/q_t": 0.3621007204055786,
"grad_norm": 28.196399688720703,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.3626401722431183,
"logits/rejected": -0.3418535888195038,
"logps/chosen": -139.056640625,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -261.5569763183594,
"loss": 0.9808,
"margin_dpo/margin_mean": 94.18919372558594,
"margin_dpo/margin_std": 117.94938659667969,
"step": 216
},
{
"KL/chosen_KL_mean": -85.92221069335938,
"KL/mean": -140.5775909423828,
"KL/rejected_KL_mean": -195.23297119140625,
"KL/std": 98.9375,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.006740123499184847,
"fcm_dpo/delta": -0.14794251322746277,
"fcm_dpo/margin": 109.31074523925781,
"fcm_dpo/q_t": 0.34184032678604126,
"grad_norm": 26.604341506958008,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.38057175278663635,
"logits/rejected": -0.3720359802246094,
"logps/chosen": -135.76751708984375,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -295.311279296875,
"loss": 0.8898,
"margin_dpo/margin_mean": 109.31074523925781,
"margin_dpo/margin_std": 108.24614715576172,
"step": 217
},
{
"KL/chosen_KL_mean": -91.01712036132812,
"KL/mean": -133.67767333984375,
"KL/rejected_KL_mean": -176.33822631835938,
"KL/std": 96.31378173828125,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.006721228361129761,
"fcm_dpo/delta": 0.028160104528069496,
"fcm_dpo/margin": 85.32112121582031,
"fcm_dpo/q_t": 0.3787044286727905,
"grad_norm": 25.266830444335938,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.3943113088607788,
"logits/rejected": -0.3831697702407837,
"logps/chosen": -149.5937957763672,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -264.18463134765625,
"loss": 1.013,
"margin_dpo/margin_mean": 85.32111358642578,
"margin_dpo/margin_std": 111.08908081054688,
"step": 218
},
{
"KL/chosen_KL_mean": -97.29121398925781,
"KL/mean": -134.1545867919922,
"KL/rejected_KL_mean": -171.01795959472656,
"KL/std": 95.68677520751953,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.006872829049825668,
"fcm_dpo/delta": 0.09711559116840363,
"fcm_dpo/margin": 73.72673034667969,
"fcm_dpo/q_t": 0.39309147000312805,
"grad_norm": 32.34254837036133,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.40911221504211426,
"logits/rejected": -0.39490410685539246,
"logps/chosen": -158.3750762939453,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -256.848388671875,
"loss": 1.0774,
"margin_dpo/margin_mean": 73.72672271728516,
"margin_dpo/margin_std": 113.96924591064453,
"step": 219
},
{
"KL/chosen_KL_mean": -108.83876037597656,
"KL/mean": -141.4522705078125,
"KL/rejected_KL_mean": -174.06576538085938,
"KL/std": 83.62852478027344,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.0069946423172950745,
"fcm_dpo/delta": 0.15089839696884155,
"fcm_dpo/margin": 65.22700500488281,
"fcm_dpo/q_t": 0.3969554305076599,
"grad_norm": 29.4495792388916,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.36126798391342163,
"logits/rejected": -0.3342619836330414,
"logps/chosen": -178.8700408935547,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -261.75128173828125,
"loss": 1.0597,
"margin_dpo/margin_mean": 65.22700500488281,
"margin_dpo/margin_std": 81.71757507324219,
"step": 220
},
{
"KL/chosen_KL_mean": -104.93341064453125,
"KL/mean": -168.17288208007812,
"KL/rejected_KL_mean": -231.412353515625,
"KL/std": 114.50070190429688,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.006814577616751194,
"fcm_dpo/delta": -0.2858082056045532,
"fcm_dpo/margin": 126.47894287109375,
"fcm_dpo/q_t": 0.31600338220596313,
"grad_norm": 27.303133010864258,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.3072892427444458,
"logits/rejected": -0.3136233985424042,
"logps/chosen": -157.08811950683594,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -337.8800354003906,
"loss": 0.8278,
"margin_dpo/margin_mean": 126.47894287109375,
"margin_dpo/margin_std": 113.42835235595703,
"step": 221
},
{
"KL/chosen_KL_mean": -111.58712768554688,
"KL/mean": -161.90048217773438,
"KL/rejected_KL_mean": -212.21380615234375,
"KL/std": 104.57847595214844,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.006643012166023254,
"fcm_dpo/delta": -0.07343975454568863,
"fcm_dpo/margin": 100.6266860961914,
"fcm_dpo/q_t": 0.3570266366004944,
"grad_norm": 26.641202926635742,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.3581978380680084,
"logits/rejected": -0.35092827677726746,
"logps/chosen": -172.5582275390625,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -312.2149658203125,
"loss": 0.9491,
"margin_dpo/margin_mean": 100.62669372558594,
"margin_dpo/margin_std": 114.24676513671875,
"step": 222
},
{
"KL/chosen_KL_mean": -124.90408325195312,
"KL/mean": -166.49951171875,
"KL/rejected_KL_mean": -208.09494018554688,
"KL/std": 102.77069091796875,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.006629304029047489,
"fcm_dpo/delta": 0.05136383697390556,
"fcm_dpo/margin": 83.19085693359375,
"fcm_dpo/q_t": 0.3846175968647003,
"grad_norm": 34.42216110229492,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.2642633616924286,
"logits/rejected": -0.2505979537963867,
"logps/chosen": -177.54464721679688,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -290.91998291015625,
"loss": 1.0796,
"margin_dpo/margin_mean": 83.19085693359375,
"margin_dpo/margin_std": 133.55804443359375,
"step": 223
},
{
"KL/chosen_KL_mean": -106.13848876953125,
"KL/mean": -155.70291137695312,
"KL/rejected_KL_mean": -205.267333984375,
"KL/std": 114.56405639648438,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.00653444230556488,
"fcm_dpo/delta": -0.05396012216806412,
"fcm_dpo/margin": 99.12886047363281,
"fcm_dpo/q_t": 0.3666822910308838,
"grad_norm": 32.1656379699707,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.3053729236125946,
"logits/rejected": -0.2811660170555115,
"logps/chosen": -154.73388671875,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -282.3838195800781,
"loss": 0.9971,
"margin_dpo/margin_mean": 99.12886047363281,
"margin_dpo/margin_std": 130.4289093017578,
"step": 224
},
{
"KL/chosen_KL_mean": -123.36929321289062,
"KL/mean": -184.49957275390625,
"KL/rejected_KL_mean": -245.62985229492188,
"KL/std": 116.01712036132812,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.0063905296847224236,
"fcm_dpo/delta": -0.1968863606452942,
"fcm_dpo/margin": 122.26055145263672,
"fcm_dpo/q_t": 0.3378020226955414,
"grad_norm": 26.66506576538086,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.3074514865875244,
"logits/rejected": -0.29252761602401733,
"logps/chosen": -181.3697509765625,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -345.53277587890625,
"loss": 0.9082,
"margin_dpo/margin_mean": 122.26055908203125,
"margin_dpo/margin_std": 137.30938720703125,
"step": 225
},
{
"KL/chosen_KL_mean": -103.83770751953125,
"KL/mean": -152.27734375,
"KL/rejected_KL_mean": -200.7169952392578,
"KL/std": 113.7965087890625,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.006260153837502003,
"fcm_dpo/delta": -0.009051397442817688,
"fcm_dpo/margin": 96.87930297851562,
"fcm_dpo/q_t": 0.3756735324859619,
"grad_norm": 28.65200424194336,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.36528223752975464,
"logits/rejected": -0.319438099861145,
"logps/chosen": -162.73651123046875,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -279.4047546386719,
"loss": 1.0141,
"margin_dpo/margin_mean": 96.8792953491211,
"margin_dpo/margin_std": 130.05320739746094,
"step": 226
},
{
"KL/chosen_KL_mean": -119.4433822631836,
"KL/mean": -179.55564880371094,
"KL/rejected_KL_mean": -239.66790771484375,
"KL/std": 128.799560546875,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.00614282488822937,
"fcm_dpo/delta": -0.15122447907924652,
"fcm_dpo/margin": 120.22454833984375,
"fcm_dpo/q_t": 0.34268540143966675,
"grad_norm": 27.082639694213867,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.3717705011367798,
"logits/rejected": -0.3560107946395874,
"logps/chosen": -178.51556396484375,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -339.0802917480469,
"loss": 0.9195,
"margin_dpo/margin_mean": 120.22454071044922,
"margin_dpo/margin_std": 133.86708068847656,
"step": 227
},
{
"KL/chosen_KL_mean": -118.5246353149414,
"KL/mean": -161.10549926757812,
"KL/rejected_KL_mean": -203.6864013671875,
"KL/std": 101.31814575195312,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.006209590472280979,
"fcm_dpo/delta": 0.07499590516090393,
"fcm_dpo/margin": 85.1617660522461,
"fcm_dpo/q_t": 0.3857218027114868,
"grad_norm": 28.64656639099121,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.341421902179718,
"logits/rejected": -0.3204922080039978,
"logps/chosen": -184.41592407226562,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -294.73516845703125,
"loss": 1.0517,
"margin_dpo/margin_mean": 85.16175842285156,
"margin_dpo/margin_std": 121.01702880859375,
"step": 228
},
{
"KL/chosen_KL_mean": -118.15702819824219,
"KL/mean": -166.1256103515625,
"KL/rejected_KL_mean": -214.09417724609375,
"KL/std": 117.42403411865234,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.0062302835285663605,
"fcm_dpo/delta": 0.0024202782660722733,
"fcm_dpo/margin": 95.9371337890625,
"fcm_dpo/q_t": 0.3707517385482788,
"grad_norm": 37.25086212158203,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.429348349571228,
"logits/rejected": -0.3982895612716675,
"logps/chosen": -188.8634033203125,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -298.62158203125,
"loss": 1.0048,
"margin_dpo/margin_mean": 95.9371337890625,
"margin_dpo/margin_std": 123.97903442382812,
"step": 229
},
{
"KL/chosen_KL_mean": -99.5108413696289,
"KL/mean": -165.11029052734375,
"KL/rejected_KL_mean": -230.70973205566406,
"KL/std": 114.2197265625,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.006020332686603069,
"fcm_dpo/delta": -0.20701435208320618,
"fcm_dpo/margin": 131.19886779785156,
"fcm_dpo/q_t": 0.329476922750473,
"grad_norm": 27.93840789794922,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.30312156677246094,
"logits/rejected": -0.3039769232273102,
"logps/chosen": -138.7928466796875,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -316.3316345214844,
"loss": 0.8676,
"margin_dpo/margin_mean": 131.19886779785156,
"margin_dpo/margin_std": 126.60467529296875,
"step": 230
},
{
"KL/chosen_KL_mean": -106.10425567626953,
"KL/mean": -152.70152282714844,
"KL/rejected_KL_mean": -199.2987823486328,
"KL/std": 99.24888610839844,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.0060048759914934635,
"fcm_dpo/delta": 0.04279271513223648,
"fcm_dpo/margin": 93.19451904296875,
"fcm_dpo/q_t": 0.3795621991157532,
"grad_norm": 25.212106704711914,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.3727571964263916,
"logits/rejected": -0.3431502878665924,
"logps/chosen": -169.38070678710938,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -273.42266845703125,
"loss": 1.0012,
"margin_dpo/margin_mean": 93.19451904296875,
"margin_dpo/margin_std": 112.45889282226562,
"step": 231
},
{
"KL/chosen_KL_mean": -127.84883117675781,
"KL/mean": -166.69406127929688,
"KL/rejected_KL_mean": -205.539306640625,
"KL/std": 115.06063079833984,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.00612263148650527,
"fcm_dpo/delta": 0.13074421882629395,
"fcm_dpo/margin": 77.69046020507812,
"fcm_dpo/q_t": 0.40193843841552734,
"grad_norm": 30.511877059936523,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.36269646883010864,
"logits/rejected": -0.3345995545387268,
"logps/chosen": -198.59759521484375,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -289.516357421875,
"loss": 1.0879,
"margin_dpo/margin_mean": 77.69046020507812,
"margin_dpo/margin_std": 122.96861267089844,
"step": 232
},
{
"KL/chosen_KL_mean": -114.28092193603516,
"KL/mean": -172.80084228515625,
"KL/rejected_KL_mean": -231.32073974609375,
"KL/std": 122.1634292602539,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.00608062744140625,
"fcm_dpo/delta": -0.12016229331493378,
"fcm_dpo/margin": 117.0398178100586,
"fcm_dpo/q_t": 0.35880354046821594,
"grad_norm": 28.278724670410156,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.36172106862068176,
"logits/rejected": -0.35902953147888184,
"logps/chosen": -169.16384887695312,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -338.80084228515625,
"loss": 0.9802,
"margin_dpo/margin_mean": 117.03982543945312,
"margin_dpo/margin_std": 156.60491943359375,
"step": 233
},
{
"KL/chosen_KL_mean": -109.62505340576172,
"KL/mean": -169.53298950195312,
"KL/rejected_KL_mean": -229.44088745117188,
"KL/std": 108.41551208496094,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.005883757025003433,
"fcm_dpo/delta": -0.11559145152568817,
"fcm_dpo/margin": 119.81584167480469,
"fcm_dpo/q_t": 0.3470458388328552,
"grad_norm": 32.83196258544922,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.3249679505825043,
"logits/rejected": -0.34966325759887695,
"logps/chosen": -153.71957397460938,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -329.447509765625,
"loss": 0.9089,
"margin_dpo/margin_mean": 119.81584167480469,
"margin_dpo/margin_std": 118.8182601928711,
"step": 234
},
{
"KL/chosen_KL_mean": -133.02932739257812,
"KL/mean": -171.4790802001953,
"KL/rejected_KL_mean": -209.9288330078125,
"KL/std": 100.35975646972656,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.0060426220297813416,
"fcm_dpo/delta": 0.14138734340667725,
"fcm_dpo/margin": 76.89950561523438,
"fcm_dpo/q_t": 0.3996395468711853,
"grad_norm": 31.084489822387695,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.3844982385635376,
"logits/rejected": -0.3582857847213745,
"logps/chosen": -195.26724243164062,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -300.32391357421875,
"loss": 1.091,
"margin_dpo/margin_mean": 76.89950561523438,
"margin_dpo/margin_std": 118.74656677246094,
"step": 235
},
{
"KL/chosen_KL_mean": -101.2742691040039,
"KL/mean": -160.6046142578125,
"KL/rejected_KL_mean": -219.93495178222656,
"KL/std": 109.5830078125,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.005965487100183964,
"fcm_dpo/delta": -0.11610803753137589,
"fcm_dpo/margin": 118.66070556640625,
"fcm_dpo/q_t": 0.3438200056552887,
"grad_norm": 53.81458282470703,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.34388887882232666,
"logits/rejected": -0.343508780002594,
"logps/chosen": -150.61563110351562,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -323.44659423828125,
"loss": 0.8859,
"margin_dpo/margin_mean": 118.66070556640625,
"margin_dpo/margin_std": 106.65434265136719,
"step": 236
},
{
"KL/chosen_KL_mean": -129.6005096435547,
"KL/mean": -178.44451904296875,
"KL/rejected_KL_mean": -227.28854370117188,
"KL/std": 110.91409301757812,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.005949076730757952,
"fcm_dpo/delta": 0.02000635676085949,
"fcm_dpo/margin": 97.68803405761719,
"fcm_dpo/q_t": 0.375545471906662,
"grad_norm": 31.343692779541016,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.37188029289245605,
"logits/rejected": -0.3660114109516144,
"logps/chosen": -183.7686309814453,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -322.06890869140625,
"loss": 1.0053,
"margin_dpo/margin_mean": 97.68803405761719,
"margin_dpo/margin_std": 124.87786102294922,
"step": 237
},
{
"KL/chosen_KL_mean": -122.53483581542969,
"KL/mean": -170.52203369140625,
"KL/rejected_KL_mean": -218.50924682617188,
"KL/std": 109.5223388671875,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.0059417979791760445,
"fcm_dpo/delta": 0.030809972435235977,
"fcm_dpo/margin": 95.97439575195312,
"fcm_dpo/q_t": 0.38020047545433044,
"grad_norm": 30.291465759277344,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.3097473084926605,
"logits/rejected": -0.32641398906707764,
"logps/chosen": -176.50796508789062,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -307.92718505859375,
"loss": 1.0085,
"margin_dpo/margin_mean": 95.97439575195312,
"margin_dpo/margin_std": 122.57579803466797,
"step": 238
},
{
"KL/chosen_KL_mean": -127.81640625,
"KL/mean": -181.54922485351562,
"KL/rejected_KL_mean": -235.28204345703125,
"KL/std": 105.47319030761719,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.005896158050745726,
"fcm_dpo/delta": -0.03850953280925751,
"fcm_dpo/margin": 107.46563720703125,
"fcm_dpo/q_t": 0.3612305223941803,
"grad_norm": 29.275442123413086,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.37939566373825073,
"logits/rejected": -0.35941123962402344,
"logps/chosen": -185.9142303466797,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -328.875,
"loss": 0.956,
"margin_dpo/margin_mean": 107.46563720703125,
"margin_dpo/margin_std": 112.2184066772461,
"step": 239
},
{
"KL/chosen_KL_mean": -135.52365112304688,
"KL/mean": -184.65708923339844,
"KL/rejected_KL_mean": -233.79051208496094,
"KL/std": 117.26210021972656,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.0059347134083509445,
"fcm_dpo/delta": 0.017533771693706512,
"fcm_dpo/margin": 98.26687622070312,
"fcm_dpo/q_t": 0.375244140625,
"grad_norm": 31.070173263549805,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.39801493287086487,
"logits/rejected": -0.3753182888031006,
"logps/chosen": -196.13815307617188,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -307.9090576171875,
"loss": 1.0254,
"margin_dpo/margin_mean": 98.26687622070312,
"margin_dpo/margin_std": 132.5979461669922,
"step": 240
},
{
"KL/chosen_KL_mean": -112.78053283691406,
"KL/mean": -184.82150268554688,
"KL/rejected_KL_mean": -256.8624572753906,
"KL/std": 130.12432861328125,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0057678730227053165,
"fcm_dpo/delta": -0.25139251351356506,
"fcm_dpo/margin": 144.08193969726562,
"fcm_dpo/q_t": 0.32945752143859863,
"grad_norm": 27.744733810424805,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.3929935097694397,
"logits/rejected": -0.3609086275100708,
"logps/chosen": -178.87159729003906,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -344.92333984375,
"loss": 0.868,
"margin_dpo/margin_mean": 144.08193969726562,
"margin_dpo/margin_std": 149.8402862548828,
"step": 241
},
{
"KL/chosen_KL_mean": -132.4889678955078,
"KL/mean": -183.32427978515625,
"KL/rejected_KL_mean": -234.15956115722656,
"KL/std": 108.88883209228516,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.005721289664506912,
"fcm_dpo/delta": 0.018697984516620636,
"fcm_dpo/margin": 101.67058563232422,
"fcm_dpo/q_t": 0.37682145833969116,
"grad_norm": 31.787302017211914,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.3101498782634735,
"logits/rejected": -0.28199779987335205,
"logps/chosen": -200.3529052734375,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -317.5198974609375,
"loss": 0.9994,
"margin_dpo/margin_mean": 101.67058563232422,
"margin_dpo/margin_std": 127.51887512207031,
"step": 242
},
{
"KL/chosen_KL_mean": -136.32640075683594,
"KL/mean": -199.96463012695312,
"KL/rejected_KL_mean": -263.60284423828125,
"KL/std": 127.57650756835938,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.005583652760833502,
"fcm_dpo/delta": -0.11946261674165726,
"fcm_dpo/margin": 127.27645874023438,
"fcm_dpo/q_t": 0.34753698110580444,
"grad_norm": 28.22019386291504,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.34900963306427,
"logits/rejected": -0.30996784567832947,
"logps/chosen": -199.41064453125,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -339.9384765625,
"loss": 0.9149,
"margin_dpo/margin_mean": 127.27645874023438,
"margin_dpo/margin_std": 135.2490692138672,
"step": 243
},
{
"KL/chosen_KL_mean": -118.91580200195312,
"KL/mean": -182.80093383789062,
"KL/rejected_KL_mean": -246.68606567382812,
"KL/std": 122.1882553100586,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.005457896739244461,
"fcm_dpo/delta": -0.10516402870416641,
"fcm_dpo/margin": 127.77027893066406,
"fcm_dpo/q_t": 0.3478250503540039,
"grad_norm": 28.444786071777344,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.3960545063018799,
"logits/rejected": -0.38508033752441406,
"logps/chosen": -180.05648803710938,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -341.5780029296875,
"loss": 0.9053,
"margin_dpo/margin_mean": 127.77027893066406,
"margin_dpo/margin_std": 123.44183349609375,
"step": 244
},
{
"KL/chosen_KL_mean": -138.50994873046875,
"KL/mean": -192.49716186523438,
"KL/rejected_KL_mean": -246.48435974121094,
"KL/std": 126.490966796875,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.005388730205595493,
"fcm_dpo/delta": 0.016096577048301697,
"fcm_dpo/margin": 107.97443389892578,
"fcm_dpo/q_t": 0.37794744968414307,
"grad_norm": 35.028076171875,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.3952246308326721,
"logits/rejected": -0.3585037589073181,
"logps/chosen": -205.77223205566406,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -334.12445068359375,
"loss": 1.0227,
"margin_dpo/margin_mean": 107.97442626953125,
"margin_dpo/margin_std": 140.54739379882812,
"step": 245
},
{
"KL/chosen_KL_mean": -137.76895141601562,
"KL/mean": -190.95184326171875,
"KL/rejected_KL_mean": -244.13473510742188,
"KL/std": 136.04083251953125,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.005461765453219414,
"fcm_dpo/delta": 0.020096484571695328,
"fcm_dpo/margin": 106.36576843261719,
"fcm_dpo/q_t": 0.38011878728866577,
"grad_norm": 29.635494232177734,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.435746431350708,
"logits/rejected": -0.39667433500289917,
"logps/chosen": -204.46591186523438,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -328.4810791015625,
"loss": 1.0199,
"margin_dpo/margin_mean": 106.36577606201172,
"margin_dpo/margin_std": 147.59591674804688,
"step": 246
},
{
"KL/chosen_KL_mean": -160.16949462890625,
"KL/mean": -236.48556518554688,
"KL/rejected_KL_mean": -312.8016357421875,
"KL/std": 146.5531005859375,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.005285009741783142,
"fcm_dpo/delta": -0.22588995099067688,
"fcm_dpo/margin": 152.63217163085938,
"fcm_dpo/q_t": 0.33097344636917114,
"grad_norm": 28.352855682373047,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.33019566535949707,
"logits/rejected": -0.32581812143325806,
"logps/chosen": -216.77484130859375,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -419.09490966796875,
"loss": 0.8796,
"margin_dpo/margin_mean": 152.63217163085938,
"margin_dpo/margin_std": 158.52627563476562,
"step": 247
},
{
"KL/chosen_KL_mean": -132.25546264648438,
"KL/mean": -195.9093475341797,
"KL/rejected_KL_mean": -259.563232421875,
"KL/std": 114.66520690917969,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.005172984674572945,
"fcm_dpo/delta": -0.06324533373117447,
"fcm_dpo/margin": 127.30779266357422,
"fcm_dpo/q_t": 0.35197725892066956,
"grad_norm": 27.992929458618164,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.2862531840801239,
"logits/rejected": -0.3005514442920685,
"logps/chosen": -176.29867553710938,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -351.42010498046875,
"loss": 0.9061,
"margin_dpo/margin_mean": 127.30780029296875,
"margin_dpo/margin_std": 110.89936065673828,
"step": 248
},
{
"KL/chosen_KL_mean": -166.60195922851562,
"KL/mean": -205.42848205566406,
"KL/rejected_KL_mean": -244.25498962402344,
"KL/std": 119.43587493896484,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.005322196986526251,
"fcm_dpo/delta": 0.19497928023338318,
"fcm_dpo/margin": 77.65301513671875,
"fcm_dpo/q_t": 0.41161924600601196,
"grad_norm": 51.56828689575195,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.3465873897075653,
"logits/rejected": -0.3201597332954407,
"logps/chosen": -229.0443115234375,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -324.7230529785156,
"loss": 1.1397,
"margin_dpo/margin_mean": 77.65301513671875,
"margin_dpo/margin_std": 138.77806091308594,
"step": 249
},
{
"KL/chosen_KL_mean": -138.74188232421875,
"KL/mean": -203.06735229492188,
"KL/rejected_KL_mean": -267.392822265625,
"KL/std": 113.97850799560547,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.00533396378159523,
"fcm_dpo/delta": -0.09259707480669022,
"fcm_dpo/margin": 128.6509246826172,
"fcm_dpo/q_t": 0.3453088402748108,
"grad_norm": 33.41306686401367,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.36225104331970215,
"logits/rejected": -0.30230361223220825,
"logps/chosen": -204.37857055664062,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -341.2646484375,
"loss": 0.8909,
"margin_dpo/margin_mean": 128.65093994140625,
"margin_dpo/margin_std": 108.00267028808594,
"step": 250
},
{
"KL/chosen_KL_mean": -158.38565063476562,
"KL/mean": -204.48248291015625,
"KL/rejected_KL_mean": -250.5792999267578,
"KL/std": 123.70230865478516,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.005347033031284809,
"fcm_dpo/delta": 0.11265549063682556,
"fcm_dpo/margin": 92.19364929199219,
"fcm_dpo/q_t": 0.3936960697174072,
"grad_norm": 37.79524230957031,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3297956585884094,
"logits/rejected": -0.2962522506713867,
"logps/chosen": -215.568359375,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -328.24273681640625,
"loss": 1.0693,
"margin_dpo/margin_mean": 92.19364929199219,
"margin_dpo/margin_std": 132.09817504882812,
"step": 251
},
{
"KL/chosen_KL_mean": -147.82955932617188,
"KL/mean": -209.88397216796875,
"KL/rejected_KL_mean": -271.93841552734375,
"KL/std": 110.409423828125,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0053227427415549755,
"fcm_dpo/delta": -0.06582003831863403,
"fcm_dpo/margin": 124.10884857177734,
"fcm_dpo/q_t": 0.3476342558860779,
"grad_norm": 37.648616790771484,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.400113046169281,
"logits/rejected": -0.35774457454681396,
"logps/chosen": -219.51519775390625,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -356.6964111328125,
"loss": 0.9052,
"margin_dpo/margin_mean": 124.10885620117188,
"margin_dpo/margin_std": 104.44979858398438,
"step": 252
},
{
"KL/chosen_KL_mean": -138.88999938964844,
"KL/mean": -197.119384765625,
"KL/rejected_KL_mean": -255.34872436523438,
"KL/std": 123.3410415649414,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.0053567951545119286,
"fcm_dpo/delta": -0.026659101247787476,
"fcm_dpo/margin": 116.45875549316406,
"fcm_dpo/q_t": 0.363253116607666,
"grad_norm": 27.87826156616211,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.40176594257354736,
"logits/rejected": -0.39333128929138184,
"logps/chosen": -208.02392578125,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -354.05126953125,
"loss": 0.9802,
"margin_dpo/margin_mean": 116.45875549316406,
"margin_dpo/margin_std": 135.98696899414062,
"step": 253
},
{
"KL/chosen_KL_mean": -152.60345458984375,
"KL/mean": -205.70919799804688,
"KL/rejected_KL_mean": -258.81488037109375,
"KL/std": 130.95278930664062,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.005327022634446621,
"fcm_dpo/delta": 0.03627510741353035,
"fcm_dpo/margin": 106.2114486694336,
"fcm_dpo/q_t": 0.38545000553131104,
"grad_norm": 33.61005401611328,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.36970311403274536,
"logits/rejected": -0.34940820932388306,
"logps/chosen": -206.75845336914062,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -349.1225280761719,
"loss": 1.0514,
"margin_dpo/margin_mean": 106.2114486694336,
"margin_dpo/margin_std": 162.00204467773438,
"step": 254
},
{
"KL/chosen_KL_mean": -160.9326629638672,
"KL/mean": -218.49002075195312,
"KL/rejected_KL_mean": -276.04736328125,
"KL/std": 123.29544067382812,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.005308132618665695,
"fcm_dpo/delta": -0.012290194630622864,
"fcm_dpo/margin": 115.11470031738281,
"fcm_dpo/q_t": 0.3695012331008911,
"grad_norm": 33.642147064208984,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.3102189004421234,
"logits/rejected": -0.29994723200798035,
"logps/chosen": -218.0743408203125,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -366.2559509277344,
"loss": 1.0188,
"margin_dpo/margin_mean": 115.11470031738281,
"margin_dpo/margin_std": 155.90023803710938,
"step": 255
},
{
"KL/chosen_KL_mean": -136.15945434570312,
"KL/mean": -198.55532836914062,
"KL/rejected_KL_mean": -260.95123291015625,
"KL/std": 132.63465881347656,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.005278502590954304,
"fcm_dpo/delta": -0.06288222968578339,
"fcm_dpo/margin": 124.79176330566406,
"fcm_dpo/q_t": 0.3590846061706543,
"grad_norm": 27.436307907104492,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.3149402439594269,
"logits/rejected": -0.31964755058288574,
"logps/chosen": -191.32293701171875,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -353.51416015625,
"loss": 0.9552,
"margin_dpo/margin_mean": 124.79176330566406,
"margin_dpo/margin_std": 144.03121948242188,
"step": 256
},
{
"KL/chosen_KL_mean": -134.51638793945312,
"KL/mean": -198.1560821533203,
"KL/rejected_KL_mean": -261.7957763671875,
"KL/std": 134.36761474609375,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.005159501917660236,
"fcm_dpo/delta": -0.06364969164133072,
"fcm_dpo/margin": 127.27938842773438,
"fcm_dpo/q_t": 0.3610895276069641,
"grad_norm": 24.21261215209961,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.3304322063922882,
"logits/rejected": -0.315920352935791,
"logps/chosen": -183.94007873535156,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -341.33367919921875,
"loss": 0.9663,
"margin_dpo/margin_mean": 127.27938079833984,
"margin_dpo/margin_std": 146.71469116210938,
"step": 257
},
{
"KL/chosen_KL_mean": -163.12612915039062,
"KL/mean": -221.1544189453125,
"KL/rejected_KL_mean": -279.1827087402344,
"KL/std": 131.205810546875,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.005203454755246639,
"fcm_dpo/delta": -0.004211133345961571,
"fcm_dpo/margin": 116.05656433105469,
"fcm_dpo/q_t": 0.36778342723846436,
"grad_norm": 32.14767837524414,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.3151804804801941,
"logits/rejected": -0.3099510669708252,
"logps/chosen": -222.51025390625,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -375.17279052734375,
"loss": 1.011,
"margin_dpo/margin_mean": 116.05655670166016,
"margin_dpo/margin_std": 153.79214477539062,
"step": 258
},
{
"KL/chosen_KL_mean": -155.34674072265625,
"KL/mean": -209.0179443359375,
"KL/rejected_KL_mean": -262.689208984375,
"KL/std": 128.54270935058594,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.005203312262892723,
"fcm_dpo/delta": 0.04370558261871338,
"fcm_dpo/margin": 107.34245300292969,
"fcm_dpo/q_t": 0.3811434507369995,
"grad_norm": 33.134132385253906,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.3546004295349121,
"logits/rejected": -0.34873515367507935,
"logps/chosen": -208.17507934570312,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -351.880859375,
"loss": 1.029,
"margin_dpo/margin_mean": 107.34245300292969,
"margin_dpo/margin_std": 143.66549682617188,
"step": 259
},
{
"KL/chosen_KL_mean": -158.1754608154297,
"KL/mean": -223.32553100585938,
"KL/rejected_KL_mean": -288.4755859375,
"KL/std": 132.26785278320312,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.0051955776289105415,
"fcm_dpo/delta": -0.08258084207773209,
"fcm_dpo/margin": 130.30010986328125,
"fcm_dpo/q_t": 0.35594630241394043,
"grad_norm": 36.58110809326172,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.35714462399482727,
"logits/rejected": -0.37431600689888,
"logps/chosen": -205.5931396484375,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -383.56536865234375,
"loss": 0.9306,
"margin_dpo/margin_mean": 130.30010986328125,
"margin_dpo/margin_std": 141.76504516601562,
"step": 260
},
{
"KL/chosen_KL_mean": -157.12448120117188,
"KL/mean": -222.61135864257812,
"KL/rejected_KL_mean": -288.0982666015625,
"KL/std": 143.23435974121094,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.0050858622416853905,
"fcm_dpo/delta": -0.07112769782543182,
"fcm_dpo/margin": 130.97378540039062,
"fcm_dpo/q_t": 0.35856717824935913,
"grad_norm": 27.698970794677734,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.29012176394462585,
"logits/rejected": -0.28605788946151733,
"logps/chosen": -210.15585327148438,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -376.61322021484375,
"loss": 0.9621,
"margin_dpo/margin_mean": 130.97378540039062,
"margin_dpo/margin_std": 156.89776611328125,
"step": 261
},
{
"KL/chosen_KL_mean": -183.9471893310547,
"KL/mean": -237.10520935058594,
"KL/rejected_KL_mean": -290.2632141113281,
"KL/std": 129.7595977783203,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.005139559973031282,
"fcm_dpo/delta": 0.056251220405101776,
"fcm_dpo/margin": 106.31602478027344,
"fcm_dpo/q_t": 0.37805455923080444,
"grad_norm": 34.91385269165039,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.30783891677856445,
"logits/rejected": -0.2801397144794464,
"logps/chosen": -243.56732177734375,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -376.6817626953125,
"loss": 1.006,
"margin_dpo/margin_mean": 106.31602478027344,
"margin_dpo/margin_std": 125.04685974121094,
"step": 262
},
{
"KL/chosen_KL_mean": -161.4795684814453,
"KL/mean": -237.04010009765625,
"KL/rejected_KL_mean": -312.6006164550781,
"KL/std": 147.77047729492188,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.004983935505151749,
"fcm_dpo/delta": -0.1672009825706482,
"fcm_dpo/margin": 151.12103271484375,
"fcm_dpo/q_t": 0.3373766243457794,
"grad_norm": 33.830772399902344,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.32824864983558655,
"logits/rejected": -0.3004014492034912,
"logps/chosen": -220.9005126953125,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -409.45782470703125,
"loss": 0.8908,
"margin_dpo/margin_mean": 151.12103271484375,
"margin_dpo/margin_std": 150.3436737060547,
"step": 263
},
{
"KL/chosen_KL_mean": -168.28366088867188,
"KL/mean": -233.16128540039062,
"KL/rejected_KL_mean": -298.0389404296875,
"KL/std": 135.82254028320312,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.004899903200566769,
"fcm_dpo/delta": -0.039829034358263016,
"fcm_dpo/margin": 129.75526428222656,
"fcm_dpo/q_t": 0.36273515224456787,
"grad_norm": 42.70479965209961,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.3502691388130188,
"logits/rejected": -0.3348994255065918,
"logps/chosen": -231.0057373046875,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -391.8951416015625,
"loss": 0.9644,
"margin_dpo/margin_mean": 129.75526428222656,
"margin_dpo/margin_std": 146.71963500976562,
"step": 264
},
{
"KL/chosen_KL_mean": -192.98976135253906,
"KL/mean": -255.98023986816406,
"KL/rejected_KL_mean": -318.9707336425781,
"KL/std": 158.42835998535156,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.004912365227937698,
"fcm_dpo/delta": -0.02011740952730179,
"fcm_dpo/margin": 125.98097229003906,
"fcm_dpo/q_t": 0.3726310729980469,
"grad_norm": 50.1806640625,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.296209454536438,
"logits/rejected": -0.2759173512458801,
"logps/chosen": -254.96124267578125,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -406.9913330078125,
"loss": 1.0244,
"margin_dpo/margin_mean": 125.98097229003906,
"margin_dpo/margin_std": 181.39669799804688,
"step": 265
},
{
"KL/chosen_KL_mean": -194.6544189453125,
"KL/mean": -251.88507080078125,
"KL/rejected_KL_mean": -309.11572265625,
"KL/std": 138.04885864257812,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.004933970049023628,
"fcm_dpo/delta": 0.037373997271060944,
"fcm_dpo/margin": 114.46128845214844,
"fcm_dpo/q_t": 0.38008320331573486,
"grad_norm": 70.57123565673828,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.3517489731311798,
"logits/rejected": -0.30076098442077637,
"logps/chosen": -261.75408935546875,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -377.08697509765625,
"loss": 1.023,
"margin_dpo/margin_mean": 114.46128845214844,
"margin_dpo/margin_std": 153.71603393554688,
"step": 266
},
{
"KL/chosen_KL_mean": -173.89627075195312,
"KL/mean": -239.25335693359375,
"KL/rejected_KL_mean": -304.6104736328125,
"KL/std": 140.8577117919922,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.004893749952316284,
"fcm_dpo/delta": -0.042935848236083984,
"fcm_dpo/margin": 130.7141876220703,
"fcm_dpo/q_t": 0.3612437844276428,
"grad_norm": 42.1270866394043,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3716611862182617,
"logits/rejected": -0.3366485834121704,
"logps/chosen": -242.8670196533203,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -394.7789001464844,
"loss": 0.9583,
"margin_dpo/margin_mean": 130.7141876220703,
"margin_dpo/margin_std": 149.27880859375,
"step": 267
},
{
"KL/chosen_KL_mean": -187.59814453125,
"KL/mean": -247.2133026123047,
"KL/rejected_KL_mean": -306.8284912109375,
"KL/std": 137.6021728515625,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.0048946812748909,
"fcm_dpo/delta": 0.0171101912856102,
"fcm_dpo/margin": 119.2303466796875,
"fcm_dpo/q_t": 0.37580710649490356,
"grad_norm": 51.12944030761719,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.33014756441116333,
"logits/rejected": -0.3356664776802063,
"logps/chosen": -243.4984588623047,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -408.4761047363281,
"loss": 1.013,
"margin_dpo/margin_mean": 119.23035430908203,
"margin_dpo/margin_std": 158.47544860839844,
"step": 268
},
{
"KL/chosen_KL_mean": -195.63763427734375,
"KL/mean": -270.7357177734375,
"KL/rejected_KL_mean": -345.8337707519531,
"KL/std": 144.40830993652344,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.004824942909181118,
"fcm_dpo/delta": -0.13438718020915985,
"fcm_dpo/margin": 150.1961669921875,
"fcm_dpo/q_t": 0.3482123017311096,
"grad_norm": 49.681697845458984,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.3526710271835327,
"logits/rejected": -0.3315295875072479,
"logps/chosen": -265.67718505859375,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -453.18316650390625,
"loss": 0.9475,
"margin_dpo/margin_mean": 150.1961669921875,
"margin_dpo/margin_std": 180.4862060546875,
"step": 269
},
{
"KL/chosen_KL_mean": -161.33154296875,
"KL/mean": -214.71310424804688,
"KL/rejected_KL_mean": -268.09466552734375,
"KL/std": 127.00556945800781,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.004866244271397591,
"fcm_dpo/delta": 0.08436901122331619,
"fcm_dpo/margin": 106.76312255859375,
"fcm_dpo/q_t": 0.3885307013988495,
"grad_norm": 43.752685546875,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.40687400102615356,
"logits/rejected": -0.39889097213745117,
"logps/chosen": -230.86502075195312,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -378.0233154296875,
"loss": 1.0533,
"margin_dpo/margin_mean": 106.76312255859375,
"margin_dpo/margin_std": 152.41429138183594,
"step": 270
},
{
"KL/chosen_KL_mean": -151.91098022460938,
"KL/mean": -220.47409057617188,
"KL/rejected_KL_mean": -289.0372314453125,
"KL/std": 125.91470336914062,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.004813849925994873,
"fcm_dpo/delta": -0.06460542976856232,
"fcm_dpo/margin": 137.1262664794922,
"fcm_dpo/q_t": 0.35645002126693726,
"grad_norm": 47.08015441894531,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.35235631465911865,
"logits/rejected": -0.3299439549446106,
"logps/chosen": -208.675537109375,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -381.5510559082031,
"loss": 0.928,
"margin_dpo/margin_mean": 137.12625122070312,
"margin_dpo/margin_std": 139.766845703125,
"step": 271
},
{
"KL/chosen_KL_mean": -145.26559448242188,
"KL/mean": -224.08888244628906,
"KL/rejected_KL_mean": -302.9121398925781,
"KL/std": 143.8643341064453,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.0047242045402526855,
"fcm_dpo/delta": -0.15615960955619812,
"fcm_dpo/margin": 157.64654541015625,
"fcm_dpo/q_t": 0.33737969398498535,
"grad_norm": 32.44465637207031,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.2578373849391937,
"logits/rejected": -0.2683930993080139,
"logps/chosen": -194.76275634765625,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -408.4549560546875,
"loss": 0.887,
"margin_dpo/margin_mean": 157.64654541015625,
"margin_dpo/margin_std": 150.65261840820312,
"step": 272
},
{
"KL/chosen_KL_mean": -162.21524047851562,
"KL/mean": -240.81851196289062,
"KL/rejected_KL_mean": -319.42181396484375,
"KL/std": 143.83770751953125,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.004546883516013622,
"fcm_dpo/delta": -0.12521019577980042,
"fcm_dpo/margin": 157.20660400390625,
"fcm_dpo/q_t": 0.3450300097465515,
"grad_norm": 32.86962890625,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.24868756532669067,
"logits/rejected": -0.22487762570381165,
"logps/chosen": -225.1906280517578,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -411.92041015625,
"loss": 0.9037,
"margin_dpo/margin_mean": 157.20660400390625,
"margin_dpo/margin_std": 155.94619750976562,
"step": 273
},
{
"KL/chosen_KL_mean": -196.32150268554688,
"KL/mean": -258.97314453125,
"KL/rejected_KL_mean": -321.6247863769531,
"KL/std": 132.172607421875,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.004573243670165539,
"fcm_dpo/delta": 0.028473064303398132,
"fcm_dpo/margin": 125.30329895019531,
"fcm_dpo/q_t": 0.3751099109649658,
"grad_norm": 30.814266204833984,
"learning_rate": 3.75e-07,
"logits/chosen": -0.20171430706977844,
"logits/rejected": -0.17888331413269043,
"logps/chosen": -251.9892120361328,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -398.9578857421875,
"loss": 0.9973,
"margin_dpo/margin_mean": 125.30329895019531,
"margin_dpo/margin_std": 149.56805419921875,
"step": 274
},
{
"KL/chosen_KL_mean": -160.3232421875,
"KL/mean": -224.93145751953125,
"KL/rejected_KL_mean": -289.5396423339844,
"KL/std": 137.5757293701172,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.004592282697558403,
"fcm_dpo/delta": 0.006736889481544495,
"fcm_dpo/margin": 129.2164306640625,
"fcm_dpo/q_t": 0.3714136481285095,
"grad_norm": 31.915740966796875,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.24127767980098724,
"logits/rejected": -0.24517418444156647,
"logps/chosen": -208.91793823242188,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -382.8433532714844,
"loss": 0.9936,
"margin_dpo/margin_mean": 129.2164306640625,
"margin_dpo/margin_std": 156.31121826171875,
"step": 275
},
{
"KL/chosen_KL_mean": -169.98287963867188,
"KL/mean": -239.55706787109375,
"KL/rejected_KL_mean": -309.13128662109375,
"KL/std": 143.59283447265625,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.004539749585092068,
"fcm_dpo/delta": -0.03416518494486809,
"fcm_dpo/margin": 139.14837646484375,
"fcm_dpo/q_t": 0.3661036193370819,
"grad_norm": 43.11882019042969,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.20103147625923157,
"logits/rejected": -0.16864368319511414,
"logps/chosen": -226.560302734375,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -379.4969482421875,
"loss": 0.9767,
"margin_dpo/margin_mean": 139.14837646484375,
"margin_dpo/margin_std": 169.36935424804688,
"step": 276
},
{
"KL/chosen_KL_mean": -196.20037841796875,
"KL/mean": -265.0767822265625,
"KL/rejected_KL_mean": -333.95318603515625,
"KL/std": 145.370361328125,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.00454034935683012,
"fcm_dpo/delta": -0.027219116687774658,
"fcm_dpo/margin": 137.7528076171875,
"fcm_dpo/q_t": 0.36654841899871826,
"grad_norm": 28.27198600769043,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.24094077944755554,
"logits/rejected": -0.22313499450683594,
"logps/chosen": -252.471923828125,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -426.8344421386719,
"loss": 0.9812,
"margin_dpo/margin_mean": 137.75279235839844,
"margin_dpo/margin_std": 167.303466796875,
"step": 277
},
{
"KL/chosen_KL_mean": -179.59710693359375,
"KL/mean": -258.1764831542969,
"KL/rejected_KL_mean": -336.755859375,
"KL/std": 155.92442321777344,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.0044339424930512905,
"fcm_dpo/delta": -0.10485933721065521,
"fcm_dpo/margin": 157.15870666503906,
"fcm_dpo/q_t": 0.35188618302345276,
"grad_norm": 30.330961227416992,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.22939492762088776,
"logits/rejected": -0.23291751742362976,
"logps/chosen": -232.53904724121094,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -428.0093994140625,
"loss": 0.9259,
"margin_dpo/margin_mean": 157.15870666503906,
"margin_dpo/margin_std": 170.03701782226562,
"step": 278
},
{
"KL/chosen_KL_mean": -210.1209716796875,
"KL/mean": -292.4913635253906,
"KL/rejected_KL_mean": -374.86175537109375,
"KL/std": 166.24716186523438,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.004311530850827694,
"fcm_dpo/delta": -0.12133017182350159,
"fcm_dpo/margin": 164.7407989501953,
"fcm_dpo/q_t": 0.35239866375923157,
"grad_norm": 41.30632781982422,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.15318737924098969,
"logits/rejected": -0.14796026051044464,
"logps/chosen": -258.76226806640625,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -462.71319580078125,
"loss": 0.9524,
"margin_dpo/margin_mean": 164.74078369140625,
"margin_dpo/margin_std": 197.61314392089844,
"step": 279
},
{
"KL/chosen_KL_mean": -207.96261596679688,
"KL/mean": -290.1720275878906,
"KL/rejected_KL_mean": -372.38140869140625,
"KL/std": 149.49325561523438,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.004249866120517254,
"fcm_dpo/delta": -0.10631021857261658,
"fcm_dpo/margin": 164.41880798339844,
"fcm_dpo/q_t": 0.344473659992218,
"grad_norm": 37.26974105834961,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.20403394103050232,
"logits/rejected": -0.20414140820503235,
"logps/chosen": -266.7597351074219,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -471.0002746582031,
"loss": 0.8986,
"margin_dpo/margin_mean": 164.41880798339844,
"margin_dpo/margin_std": 152.754150390625,
"step": 280
},
{
"KL/chosen_KL_mean": -191.81907653808594,
"KL/mean": -266.65350341796875,
"KL/rejected_KL_mean": -341.4879455566406,
"KL/std": 148.75167846679688,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0042279912158846855,
"fcm_dpo/delta": -0.03527888283133507,
"fcm_dpo/margin": 149.66888427734375,
"fcm_dpo/q_t": 0.3564120829105377,
"grad_norm": 25.87759780883789,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.24658602476119995,
"logits/rejected": -0.21498368680477142,
"logps/chosen": -247.30758666992188,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -422.3705139160156,
"loss": 0.9455,
"margin_dpo/margin_mean": 149.66888427734375,
"margin_dpo/margin_std": 152.3994140625,
"step": 281
},
{
"KL/chosen_KL_mean": -211.36709594726562,
"KL/mean": -271.1958312988281,
"KL/rejected_KL_mean": -331.0245666503906,
"KL/std": 149.73577880859375,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.004283766262233257,
"fcm_dpo/delta": 0.09086461365222931,
"fcm_dpo/margin": 119.65745544433594,
"fcm_dpo/q_t": 0.3909056484699249,
"grad_norm": 30.09254264831543,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.3243488073348999,
"logits/rejected": -0.2936389446258545,
"logps/chosen": -284.437255859375,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -426.37554931640625,
"loss": 1.0432,
"margin_dpo/margin_mean": 119.65745544433594,
"margin_dpo/margin_std": 165.5845947265625,
"step": 282
},
{
"KL/chosen_KL_mean": -218.3724822998047,
"KL/mean": -303.41107177734375,
"KL/rejected_KL_mean": -388.4496765136719,
"KL/std": 176.44088745117188,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.00421607680618763,
"fcm_dpo/delta": -0.1259281188249588,
"fcm_dpo/margin": 170.07720947265625,
"fcm_dpo/q_t": 0.3422727584838867,
"grad_norm": 37.20414352416992,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.2760939598083496,
"logits/rejected": -0.2752807140350342,
"logps/chosen": -280.27093505859375,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -485.43621826171875,
"loss": 0.8956,
"margin_dpo/margin_mean": 170.0771942138672,
"margin_dpo/margin_std": 161.67559814453125,
"step": 283
},
{
"KL/chosen_KL_mean": -205.727783203125,
"KL/mean": -286.13623046875,
"KL/rejected_KL_mean": -366.544677734375,
"KL/std": 164.99278259277344,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.004120776429772377,
"fcm_dpo/delta": -0.06740535795688629,
"fcm_dpo/margin": 160.81692504882812,
"fcm_dpo/q_t": 0.354042112827301,
"grad_norm": 34.00525665283203,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.30698153376579285,
"logits/rejected": -0.29135367274284363,
"logps/chosen": -264.163330078125,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -460.013916015625,
"loss": 0.9167,
"margin_dpo/margin_mean": 160.81690979003906,
"margin_dpo/margin_std": 155.95025634765625,
"step": 284
},
{
"KL/chosen_KL_mean": -225.933349609375,
"KL/mean": -297.6268310546875,
"KL/rejected_KL_mean": -369.3203430175781,
"KL/std": 143.97120666503906,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.004147817380726337,
"fcm_dpo/delta": 0.004539430141448975,
"fcm_dpo/margin": 143.38699340820312,
"fcm_dpo/q_t": 0.3656592071056366,
"grad_norm": 33.62853240966797,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.31773972511291504,
"logits/rejected": -0.29628488421440125,
"logps/chosen": -292.16552734375,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -468.4471435546875,
"loss": 0.9642,
"margin_dpo/margin_mean": 143.38699340820312,
"margin_dpo/margin_std": 145.22735595703125,
"step": 285
},
{
"KL/chosen_KL_mean": -223.07327270507812,
"KL/mean": -301.30780029296875,
"KL/rejected_KL_mean": -379.5423583984375,
"KL/std": 160.04953002929688,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.004099573008716106,
"fcm_dpo/delta": -0.04428183659911156,
"fcm_dpo/margin": 156.46908569335938,
"fcm_dpo/q_t": 0.35888007283210754,
"grad_norm": 33.56089401245117,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.33965709805488586,
"logits/rejected": -0.3160383403301239,
"logps/chosen": -296.0242919921875,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -468.13079833984375,
"loss": 0.9458,
"margin_dpo/margin_mean": 156.46908569335938,
"margin_dpo/margin_std": 165.3804473876953,
"step": 286
},
{
"KL/chosen_KL_mean": -212.49472045898438,
"KL/mean": -279.4689025878906,
"KL/rejected_KL_mean": -346.4430847167969,
"KL/std": 141.6124267578125,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.004122943617403507,
"fcm_dpo/delta": 0.050321612507104874,
"fcm_dpo/margin": 133.9483642578125,
"fcm_dpo/q_t": 0.37875235080718994,
"grad_norm": 31.039735794067383,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.27376627922058105,
"logits/rejected": -0.24279624223709106,
"logps/chosen": -274.035888671875,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -424.13916015625,
"loss": 1.0022,
"margin_dpo/margin_mean": 133.9483642578125,
"margin_dpo/margin_std": 158.88485717773438,
"step": 287
},
{
"KL/chosen_KL_mean": -212.31646728515625,
"KL/mean": -291.22772216796875,
"KL/rejected_KL_mean": -370.1389465332031,
"KL/std": 140.23904418945312,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.004055471159517765,
"fcm_dpo/delta": -0.045400090515613556,
"fcm_dpo/margin": 157.82249450683594,
"fcm_dpo/q_t": 0.35538947582244873,
"grad_norm": 33.70338439941406,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.2510570287704468,
"logits/rejected": -0.22578854858875275,
"logps/chosen": -268.9776916503906,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -457.47467041015625,
"loss": 0.9207,
"margin_dpo/margin_mean": 157.82249450683594,
"margin_dpo/margin_std": 135.26321411132812,
"step": 288
},
{
"KL/chosen_KL_mean": -202.70721435546875,
"KL/mean": -292.74169921875,
"KL/rejected_KL_mean": -382.77618408203125,
"KL/std": 160.28533935546875,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.004019693471491337,
"fcm_dpo/delta": -0.13327071070671082,
"fcm_dpo/margin": 180.06893920898438,
"fcm_dpo/q_t": 0.34370124340057373,
"grad_norm": 35.93254470825195,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.22897642850875854,
"logits/rejected": -0.22270119190216064,
"logps/chosen": -247.9376220703125,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -470.4188232421875,
"loss": 0.895,
"margin_dpo/margin_mean": 180.06893920898438,
"margin_dpo/margin_std": 175.3511199951172,
"step": 289
},
{
"KL/chosen_KL_mean": -221.83950805664062,
"KL/mean": -311.77410888671875,
"KL/rejected_KL_mean": -401.7087097167969,
"KL/std": 171.25790405273438,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.0039045047014951706,
"fcm_dpo/delta": -0.11048676073551178,
"fcm_dpo/margin": 179.86923217773438,
"fcm_dpo/q_t": 0.3510878384113312,
"grad_norm": 30.228456497192383,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.23565331101417542,
"logits/rejected": -0.2550272047519684,
"logps/chosen": -277.3110046386719,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -518.4172973632812,
"loss": 0.9275,
"margin_dpo/margin_mean": 179.8692169189453,
"margin_dpo/margin_std": 199.00949096679688,
"step": 290
},
{
"KL/chosen_KL_mean": -193.6474609375,
"KL/mean": -277.68707275390625,
"KL/rejected_KL_mean": -361.7266845703125,
"KL/std": 148.0182342529297,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.0038851494900882244,
"fcm_dpo/delta": -0.05720999464392662,
"fcm_dpo/margin": 168.0792236328125,
"fcm_dpo/q_t": 0.3551742136478424,
"grad_norm": 35.74508285522461,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.2522128224372864,
"logits/rejected": -0.23240351676940918,
"logps/chosen": -256.9078369140625,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -451.0237731933594,
"loss": 0.9366,
"margin_dpo/margin_mean": 168.0792236328125,
"margin_dpo/margin_std": 169.25894165039062,
"step": 291
},
{
"KL/chosen_KL_mean": -221.73016357421875,
"KL/mean": -308.63690185546875,
"KL/rejected_KL_mean": -395.54364013671875,
"KL/std": 176.73696899414062,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0038274754770100117,
"fcm_dpo/delta": -0.07006621360778809,
"fcm_dpo/margin": 173.81346130371094,
"fcm_dpo/q_t": 0.3559858202934265,
"grad_norm": 32.436161041259766,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.2682231068611145,
"logits/rejected": -0.2508416175842285,
"logps/chosen": -275.648681640625,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -485.5050048828125,
"loss": 0.9602,
"margin_dpo/margin_mean": 173.81346130371094,
"margin_dpo/margin_std": 203.7884521484375,
"step": 292
},
{
"KL/chosen_KL_mean": -223.93856811523438,
"KL/mean": -291.9893798828125,
"KL/rejected_KL_mean": -360.0401611328125,
"KL/std": 172.38546752929688,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.003851910587400198,
"fcm_dpo/delta": 0.07950392365455627,
"fcm_dpo/margin": 136.1015625,
"fcm_dpo/q_t": 0.3895997405052185,
"grad_norm": 56.570411682128906,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.252549946308136,
"logits/rejected": -0.2343285083770752,
"logps/chosen": -284.3146057128906,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -437.892578125,
"loss": 1.06,
"margin_dpo/margin_mean": 136.1015625,
"margin_dpo/margin_std": 200.80953979492188,
"step": 293
},
{
"KL/chosen_KL_mean": -209.2860565185547,
"KL/mean": -286.2882080078125,
"KL/rejected_KL_mean": -363.29034423828125,
"KL/std": 161.61154174804688,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.0038465007673949003,
"fcm_dpo/delta": 0.00786316767334938,
"fcm_dpo/margin": 154.00424194335938,
"fcm_dpo/q_t": 0.3744083046913147,
"grad_norm": 28.52789306640625,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.2322460114955902,
"logits/rejected": -0.2340223789215088,
"logps/chosen": -257.37359619140625,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -445.18731689453125,
"loss": 0.9956,
"margin_dpo/margin_mean": 154.00424194335938,
"margin_dpo/margin_std": 191.2259521484375,
"step": 294
},
{
"KL/chosen_KL_mean": -258.8825988769531,
"KL/mean": -339.47515869140625,
"KL/rejected_KL_mean": -420.06768798828125,
"KL/std": 173.15040588378906,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.0038494551554322243,
"fcm_dpo/delta": -0.021829120814800262,
"fcm_dpo/margin": 161.1851043701172,
"fcm_dpo/q_t": 0.3673200011253357,
"grad_norm": 39.77217102050781,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.23391102254390717,
"logits/rejected": -0.23292535543441772,
"logps/chosen": -308.8072509765625,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -507.5240173339844,
"loss": 0.9876,
"margin_dpo/margin_mean": 161.1851043701172,
"margin_dpo/margin_std": 198.60641479492188,
"step": 295
},
{
"KL/chosen_KL_mean": -313.30108642578125,
"KL/mean": -372.1259765625,
"KL/rejected_KL_mean": -430.9508972167969,
"KL/std": 170.87611389160156,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.0038994797505438328,
"fcm_dpo/delta": 0.14787380397319794,
"fcm_dpo/margin": 117.64984893798828,
"fcm_dpo/q_t": 0.4025155305862427,
"grad_norm": 50.97820281982422,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.24989140033721924,
"logits/rejected": -0.22922003269195557,
"logps/chosen": -378.79229736328125,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -524.0399780273438,
"loss": 1.1433,
"margin_dpo/margin_mean": 117.64985656738281,
"margin_dpo/margin_std": 216.04696655273438,
"step": 296
},
{
"KL/chosen_KL_mean": -283.12896728515625,
"KL/mean": -349.2366943359375,
"KL/rejected_KL_mean": -415.3443603515625,
"KL/std": 166.49322509765625,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.003996062092483044,
"fcm_dpo/delta": 0.0757208988070488,
"fcm_dpo/margin": 132.21539306640625,
"fcm_dpo/q_t": 0.3847334086894989,
"grad_norm": 54.83102035522461,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.26566392183303833,
"logits/rejected": -0.26453983783721924,
"logps/chosen": -339.60595703125,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -510.48291015625,
"loss": 1.0296,
"margin_dpo/margin_mean": 132.21539306640625,
"margin_dpo/margin_std": 169.85586547851562,
"step": 297
},
{
"KL/chosen_KL_mean": -281.3887634277344,
"KL/mean": -370.6845703125,
"KL/rejected_KL_mean": -459.9803466796875,
"KL/std": 206.78038024902344,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.003973445855081081,
"fcm_dpo/delta": -0.11797946691513062,
"fcm_dpo/margin": 178.59158325195312,
"fcm_dpo/q_t": 0.3568301498889923,
"grad_norm": 42.96636199951172,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.28307509422302246,
"logits/rejected": -0.29904448986053467,
"logps/chosen": -348.71392822265625,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -576.6425170898438,
"loss": 0.9668,
"margin_dpo/margin_mean": 178.59158325195312,
"margin_dpo/margin_std": 231.4388427734375,
"step": 298
},
{
"KL/chosen_KL_mean": -238.4638671875,
"KL/mean": -317.648681640625,
"KL/rejected_KL_mean": -396.83355712890625,
"KL/std": 160.91534423828125,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.003940481226891279,
"fcm_dpo/delta": -0.026556983590126038,
"fcm_dpo/margin": 158.36965942382812,
"fcm_dpo/q_t": 0.36903831362724304,
"grad_norm": 46.92436599731445,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.2980954051017761,
"logits/rejected": -0.29550328850746155,
"logps/chosen": -287.42596435546875,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -481.1617736816406,
"loss": 0.9887,
"margin_dpo/margin_mean": 158.36965942382812,
"margin_dpo/margin_std": 197.46868896484375,
"step": 299
},
{
"KL/chosen_KL_mean": -289.8644714355469,
"KL/mean": -377.52752685546875,
"KL/rejected_KL_mean": -465.1905822753906,
"KL/std": 186.0885772705078,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.003873254172503948,
"fcm_dpo/delta": -0.08483842015266418,
"fcm_dpo/margin": 175.32608032226562,
"fcm_dpo/q_t": 0.35567396879196167,
"grad_norm": 68.64498901367188,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.33588019013404846,
"logits/rejected": -0.3351149260997772,
"logps/chosen": -348.93817138671875,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -561.1570434570312,
"loss": 0.9796,
"margin_dpo/margin_mean": 175.32608032226562,
"margin_dpo/margin_std": 219.65176391601562,
"step": 300
},
{
"KL/chosen_KL_mean": -232.5543212890625,
"KL/mean": -323.2122802734375,
"KL/rejected_KL_mean": -413.87017822265625,
"KL/std": 169.872314453125,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.003773224074393511,
"fcm_dpo/delta": -0.09107614308595657,
"fcm_dpo/margin": 181.31588745117188,
"fcm_dpo/q_t": 0.35407012701034546,
"grad_norm": 34.75926208496094,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.36688753962516785,
"logits/rejected": -0.35920974612236023,
"logps/chosen": -289.8036804199219,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -506.2237548828125,
"loss": 0.9283,
"margin_dpo/margin_mean": 181.31588745117188,
"margin_dpo/margin_std": 196.02264404296875,
"step": 301
},
{
"KL/chosen_KL_mean": -205.85867309570312,
"KL/mean": -277.554931640625,
"KL/rejected_KL_mean": -349.2511901855469,
"KL/std": 149.74615478515625,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.003789176233112812,
"fcm_dpo/delta": 0.05980518087744713,
"fcm_dpo/margin": 143.39251708984375,
"fcm_dpo/q_t": 0.3816894292831421,
"grad_norm": 42.13399124145508,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.33960092067718506,
"logits/rejected": -0.34297773241996765,
"logps/chosen": -257.0566711425781,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -446.4775390625,
"loss": 1.0113,
"margin_dpo/margin_mean": 143.39251708984375,
"margin_dpo/margin_std": 174.07632446289062,
"step": 302
},
{
"KL/chosen_KL_mean": -203.69264221191406,
"KL/mean": -274.620361328125,
"KL/rejected_KL_mean": -345.54803466796875,
"KL/std": 154.19125366210938,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.003840233199298382,
"fcm_dpo/delta": 0.05843983590602875,
"fcm_dpo/margin": 141.8554229736328,
"fcm_dpo/q_t": 0.38291749358177185,
"grad_norm": 28.319339752197266,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.3495972156524658,
"logits/rejected": -0.32658839225769043,
"logps/chosen": -270.4065856933594,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -432.49346923828125,
"loss": 1.024,
"margin_dpo/margin_mean": 141.8554229736328,
"margin_dpo/margin_std": 182.29901123046875,
"step": 303
},
{
"KL/chosen_KL_mean": -183.831787109375,
"KL/mean": -260.0631408691406,
"KL/rejected_KL_mean": -336.29449462890625,
"KL/std": 149.16224670410156,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.0038858032785356045,
"fcm_dpo/delta": 0.007453545928001404,
"fcm_dpo/margin": 152.46266174316406,
"fcm_dpo/q_t": 0.3638036251068115,
"grad_norm": 38.128318786621094,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.35835734009742737,
"logits/rejected": -0.3397323191165924,
"logps/chosen": -255.78250122070312,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -426.7665100097656,
"loss": 0.9386,
"margin_dpo/margin_mean": 152.46266174316406,
"margin_dpo/margin_std": 127.14396667480469,
"step": 304
},
{
"KL/chosen_KL_mean": -196.5946044921875,
"KL/mean": -266.8968505859375,
"KL/rejected_KL_mean": -337.1990661621094,
"KL/std": 168.6895294189453,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.0038762034382671118,
"fcm_dpo/delta": 0.057699479162693024,
"fcm_dpo/margin": 140.60446166992188,
"fcm_dpo/q_t": 0.38321030139923096,
"grad_norm": 33.18399429321289,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.4135547876358032,
"logits/rejected": -0.39013415575027466,
"logps/chosen": -263.38983154296875,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -429.95367431640625,
"loss": 1.0194,
"margin_dpo/margin_mean": 140.60446166992188,
"margin_dpo/margin_std": 178.82345581054688,
"step": 305
},
{
"KL/chosen_KL_mean": -191.17567443847656,
"KL/mean": -261.2862548828125,
"KL/rejected_KL_mean": -331.3968811035156,
"KL/std": 145.12747192382812,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.003947091288864613,
"fcm_dpo/delta": 0.04913228377699852,
"fcm_dpo/margin": 140.22119140625,
"fcm_dpo/q_t": 0.3768806457519531,
"grad_norm": 36.105377197265625,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.3358730971813202,
"logits/rejected": -0.3046911358833313,
"logps/chosen": -260.85955810546875,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -416.55609130859375,
"loss": 0.9885,
"margin_dpo/margin_mean": 140.22119140625,
"margin_dpo/margin_std": 153.94952392578125,
"step": 306
},
{
"KL/chosen_KL_mean": -188.28726196289062,
"KL/mean": -257.31134033203125,
"KL/rejected_KL_mean": -326.3354187011719,
"KL/std": 135.5795440673828,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.003968643955886364,
"fcm_dpo/delta": 0.05507875978946686,
"fcm_dpo/margin": 138.04812622070312,
"fcm_dpo/q_t": 0.3784691095352173,
"grad_norm": 32.03385543823242,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.32129788398742676,
"logits/rejected": -0.2991190254688263,
"logps/chosen": -258.45269775390625,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -413.3077392578125,
"loss": 1.0056,
"margin_dpo/margin_mean": 138.04812622070312,
"margin_dpo/margin_std": 162.10498046875,
"step": 307
},
{
"KL/chosen_KL_mean": -191.8477020263672,
"KL/mean": -272.7364807128906,
"KL/rejected_KL_mean": -353.625244140625,
"KL/std": 156.09765625,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.003971360623836517,
"fcm_dpo/delta": -0.045369479805231094,
"fcm_dpo/margin": 161.77755737304688,
"fcm_dpo/q_t": 0.3555334508419037,
"grad_norm": 35.78176498413086,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.2915213704109192,
"logits/rejected": -0.27434253692626953,
"logps/chosen": -247.09271240234375,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -432.99749755859375,
"loss": 0.925,
"margin_dpo/margin_mean": 161.77755737304688,
"margin_dpo/margin_std": 149.737060546875,
"step": 308
},
{
"KL/chosen_KL_mean": -196.13375854492188,
"KL/mean": -274.7844543457031,
"KL/rejected_KL_mean": -353.4351501464844,
"KL/std": 161.90078735351562,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.003971335478127003,
"fcm_dpo/delta": -0.027225781232118607,
"fcm_dpo/margin": 157.3013916015625,
"fcm_dpo/q_t": 0.36207816004753113,
"grad_norm": 42.15934371948242,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.24967613816261292,
"logits/rejected": -0.23184144496917725,
"logps/chosen": -245.0928497314453,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -435.77587890625,
"loss": 0.9609,
"margin_dpo/margin_mean": 157.3013916015625,
"margin_dpo/margin_std": 168.75686645507812,
"step": 309
},
{
"KL/chosen_KL_mean": -226.8197021484375,
"KL/mean": -297.0825500488281,
"KL/rejected_KL_mean": -367.34539794921875,
"KL/std": 135.09078979492188,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.003982918336987495,
"fcm_dpo/delta": 0.041870638728141785,
"fcm_dpo/margin": 140.52569580078125,
"fcm_dpo/q_t": 0.37264642119407654,
"grad_norm": 30.770174026489258,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.2896626591682434,
"logits/rejected": -0.269090861082077,
"logps/chosen": -289.5614929199219,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -447.27569580078125,
"loss": 0.9709,
"margin_dpo/margin_mean": 140.5256805419922,
"margin_dpo/margin_std": 134.17430114746094,
"step": 310
},
{
"KL/chosen_KL_mean": -237.55093383789062,
"KL/mean": -323.655029296875,
"KL/rejected_KL_mean": -409.75909423828125,
"KL/std": 188.87551879882812,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.003919628448784351,
"fcm_dpo/delta": -0.08050680160522461,
"fcm_dpo/margin": 172.20819091796875,
"fcm_dpo/q_t": 0.35694873332977295,
"grad_norm": 30.53249168395996,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.2591084837913513,
"logits/rejected": -0.23077183961868286,
"logps/chosen": -290.57891845703125,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -487.19732666015625,
"loss": 0.9463,
"margin_dpo/margin_mean": 172.20819091796875,
"margin_dpo/margin_std": 196.930908203125,
"step": 311
},
{
"KL/chosen_KL_mean": -225.37548828125,
"KL/mean": -306.20111083984375,
"KL/rejected_KL_mean": -387.0267333984375,
"KL/std": 156.8494873046875,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.0039051049388945103,
"fcm_dpo/delta": -0.03364739567041397,
"fcm_dpo/margin": 161.65122985839844,
"fcm_dpo/q_t": 0.3627866506576538,
"grad_norm": 28.854969024658203,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.21233907341957092,
"logits/rejected": -0.1948060244321823,
"logps/chosen": -274.7677001953125,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -462.81951904296875,
"loss": 0.9595,
"margin_dpo/margin_mean": 161.65122985839844,
"margin_dpo/margin_std": 180.9955291748047,
"step": 312
},
{
"KL/chosen_KL_mean": -217.57130432128906,
"KL/mean": -292.1993713378906,
"KL/rejected_KL_mean": -366.8274230957031,
"KL/std": 167.15887451171875,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.003926842473447323,
"fcm_dpo/delta": 0.012592583894729614,
"fcm_dpo/margin": 149.25611877441406,
"fcm_dpo/q_t": 0.37196066975593567,
"grad_norm": 31.07817840576172,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.2286556363105774,
"logits/rejected": -0.21922114491462708,
"logps/chosen": -267.72406005859375,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -453.233642578125,
"loss": 1.0032,
"margin_dpo/margin_mean": 149.25611877441406,
"margin_dpo/margin_std": 180.0426025390625,
"step": 313
},
{
"KL/chosen_KL_mean": -231.25299072265625,
"KL/mean": -303.6961669921875,
"KL/rejected_KL_mean": -376.13934326171875,
"KL/std": 162.85194396972656,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.0038912883028388023,
"fcm_dpo/delta": 0.037969741970300674,
"fcm_dpo/margin": 144.88632202148438,
"fcm_dpo/q_t": 0.3799631595611572,
"grad_norm": 28.8031063079834,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.2824794054031372,
"logits/rejected": -0.2783099412918091,
"logps/chosen": -288.4906005859375,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -473.7358703613281,
"loss": 1.0292,
"margin_dpo/margin_mean": 144.88632202148438,
"margin_dpo/margin_std": 196.08563232421875,
"step": 314
},
{
"KL/chosen_KL_mean": -214.62045288085938,
"KL/mean": -281.68341064453125,
"KL/rejected_KL_mean": -348.7463684082031,
"KL/std": 144.15219116210938,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.003959797788411379,
"fcm_dpo/delta": 0.07279416173696518,
"fcm_dpo/margin": 134.12594604492188,
"fcm_dpo/q_t": 0.3835921585559845,
"grad_norm": 31.880666732788086,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.21176955103874207,
"logits/rejected": -0.2100306749343872,
"logps/chosen": -263.6900329589844,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -434.42724609375,
"loss": 1.0149,
"margin_dpo/margin_mean": 134.12594604492188,
"margin_dpo/margin_std": 161.80035400390625,
"step": 315
},
{
"KL/chosen_KL_mean": -209.0601806640625,
"KL/mean": -299.2228698730469,
"KL/rejected_KL_mean": -389.3855895996094,
"KL/std": 170.9292755126953,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.003884736681357026,
"fcm_dpo/delta": -0.10999269783496857,
"fcm_dpo/margin": 180.32540893554688,
"fcm_dpo/q_t": 0.3494938015937805,
"grad_norm": 28.535741806030273,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.26677030324935913,
"logits/rejected": -0.26802217960357666,
"logps/chosen": -263.3209228515625,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -490.66705322265625,
"loss": 0.9142,
"margin_dpo/margin_mean": 180.32537841796875,
"margin_dpo/margin_std": 184.67803955078125,
"step": 316
},
{
"KL/chosen_KL_mean": -211.87350463867188,
"KL/mean": -312.7148742675781,
"KL/rejected_KL_mean": -413.55621337890625,
"KL/std": 158.79638671875,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.003802879247814417,
"fcm_dpo/delta": -0.18046119809150696,
"fcm_dpo/margin": 201.68270874023438,
"fcm_dpo/q_t": 0.33107978105545044,
"grad_norm": 34.809513092041016,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.2909989356994629,
"logits/rejected": -0.2732127904891968,
"logps/chosen": -267.96771240234375,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -514.2552490234375,
"loss": 0.8564,
"margin_dpo/margin_mean": 201.68270874023438,
"margin_dpo/margin_std": 172.046630859375,
"step": 317
},
{
"KL/chosen_KL_mean": -222.46737670898438,
"KL/mean": -307.3929443359375,
"KL/rejected_KL_mean": -392.31854248046875,
"KL/std": 165.178466796875,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.0037494138814508915,
"fcm_dpo/delta": -0.03965116664767265,
"fcm_dpo/margin": 169.85116577148438,
"fcm_dpo/q_t": 0.3590587377548218,
"grad_norm": 42.407188415527344,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.33724579215049744,
"logits/rejected": -0.29957932233810425,
"logps/chosen": -287.11309814453125,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -475.0827941894531,
"loss": 0.9384,
"margin_dpo/margin_mean": 169.85116577148438,
"margin_dpo/margin_std": 169.72088623046875,
"step": 318
},
{
"KL/chosen_KL_mean": -211.00039672851562,
"KL/mean": -300.62982177734375,
"KL/rejected_KL_mean": -390.25927734375,
"KL/std": 175.908447265625,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0036879051476716995,
"fcm_dpo/delta": -0.06540600210428238,
"fcm_dpo/margin": 179.2588653564453,
"fcm_dpo/q_t": 0.35390669107437134,
"grad_norm": 32.69434356689453,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.3109878599643707,
"logits/rejected": -0.32243919372558594,
"logps/chosen": -260.3841552734375,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -504.165771484375,
"loss": 0.9315,
"margin_dpo/margin_mean": 179.2588653564453,
"margin_dpo/margin_std": 184.39210510253906,
"step": 319
},
{
"KL/chosen_KL_mean": -218.02212524414062,
"KL/mean": -319.87274169921875,
"KL/rejected_KL_mean": -421.723388671875,
"KL/std": 188.8252716064453,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.00358652719296515,
"fcm_dpo/delta": -0.14159169793128967,
"fcm_dpo/margin": 203.7012481689453,
"fcm_dpo/q_t": 0.34120750427246094,
"grad_norm": 39.133209228515625,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.3372478485107422,
"logits/rejected": -0.3382689356803894,
"logps/chosen": -277.5270080566406,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -519.3905639648438,
"loss": 0.885,
"margin_dpo/margin_mean": 203.7012481689453,
"margin_dpo/margin_std": 195.3154754638672,
"step": 320
},
{
"KL/chosen_KL_mean": -267.1432189941406,
"KL/mean": -355.0939636230469,
"KL/rejected_KL_mean": -443.04473876953125,
"KL/std": 195.75750732421875,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.003527258988469839,
"fcm_dpo/delta": -0.023772604763507843,
"fcm_dpo/margin": 175.90151977539062,
"fcm_dpo/q_t": 0.36594638228416443,
"grad_norm": 37.3625602722168,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.24784182012081146,
"logits/rejected": -0.2329789102077484,
"logps/chosen": -328.69189453125,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -534.685791015625,
"loss": 0.9866,
"margin_dpo/margin_mean": 175.90151977539062,
"margin_dpo/margin_std": 210.58360290527344,
"step": 321
},
{
"KL/chosen_KL_mean": -238.1175537109375,
"KL/mean": -333.6112365722656,
"KL/rejected_KL_mean": -429.10498046875,
"KL/std": 175.85693359375,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.0034964876249432564,
"fcm_dpo/delta": -0.0737525001168251,
"fcm_dpo/margin": 190.9874267578125,
"fcm_dpo/q_t": 0.35272741317749023,
"grad_norm": 30.657041549682617,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.25116193294525146,
"logits/rejected": -0.2432917356491089,
"logps/chosen": -295.4075927734375,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -524.8549194335938,
"loss": 0.9219,
"margin_dpo/margin_mean": 190.9874267578125,
"margin_dpo/margin_std": 186.6202392578125,
"step": 322
},
{
"KL/chosen_KL_mean": -254.53573608398438,
"KL/mean": -338.5694580078125,
"KL/rejected_KL_mean": -422.6031494140625,
"KL/std": 179.4539794921875,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.0035015689209103584,
"fcm_dpo/delta": 0.012137187644839287,
"fcm_dpo/margin": 168.06741333007812,
"fcm_dpo/q_t": 0.3708209693431854,
"grad_norm": 45.26333999633789,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.2965390086174011,
"logits/rejected": -0.28871312737464905,
"logps/chosen": -305.76971435546875,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -497.6650695800781,
"loss": 0.9909,
"margin_dpo/margin_mean": 168.06741333007812,
"margin_dpo/margin_std": 194.92874145507812,
"step": 323
},
{
"KL/chosen_KL_mean": -287.0393371582031,
"KL/mean": -375.05023193359375,
"KL/rejected_KL_mean": -463.06109619140625,
"KL/std": 190.91993713378906,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.0035341251641511917,
"fcm_dpo/delta": -0.02536836266517639,
"fcm_dpo/margin": 176.02175903320312,
"fcm_dpo/q_t": 0.36752551794052124,
"grad_norm": 60.69253921508789,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.33062124252319336,
"logits/rejected": -0.30674198269844055,
"logps/chosen": -352.17449951171875,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -549.53857421875,
"loss": 0.9936,
"margin_dpo/margin_mean": 176.02174377441406,
"margin_dpo/margin_std": 213.7113037109375,
"step": 324
},
{
"KL/chosen_KL_mean": -222.19732666015625,
"KL/mean": -292.1839904785156,
"KL/rejected_KL_mean": -362.170654296875,
"KL/std": 164.73373413085938,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.0035609256010502577,
"fcm_dpo/delta": 0.10635978728532791,
"fcm_dpo/margin": 139.97329711914062,
"fcm_dpo/q_t": 0.3865123689174652,
"grad_norm": 40.104759216308594,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.4175834059715271,
"logits/rejected": -0.38348710536956787,
"logps/chosen": -278.41290283203125,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -432.2565612792969,
"loss": 1.0244,
"margin_dpo/margin_mean": 139.9733123779297,
"margin_dpo/margin_std": 153.7855682373047,
"step": 325
},
{
"KL/chosen_KL_mean": -227.307373046875,
"KL/mean": -288.5229797363281,
"KL/rejected_KL_mean": -349.73858642578125,
"KL/std": 146.83538818359375,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.003642299212515354,
"fcm_dpo/delta": 0.16157841682434082,
"fcm_dpo/margin": 122.43121337890625,
"fcm_dpo/q_t": 0.3980899453163147,
"grad_norm": 97.73371887207031,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.43311089277267456,
"logits/rejected": -0.4014735519886017,
"logps/chosen": -300.0323486328125,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -429.58538818359375,
"loss": 1.0483,
"margin_dpo/margin_mean": 122.43121337890625,
"margin_dpo/margin_std": 134.76637268066406,
"step": 326
},
{
"KL/chosen_KL_mean": -191.91265869140625,
"KL/mean": -281.76373291015625,
"KL/rejected_KL_mean": -371.6147766113281,
"KL/std": 153.0264434814453,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.0036406731233000755,
"fcm_dpo/delta": -0.05855613946914673,
"fcm_dpo/margin": 179.7021026611328,
"fcm_dpo/q_t": 0.35203319787979126,
"grad_norm": 76.8426742553711,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.42004019021987915,
"logits/rejected": -0.4010508954524994,
"logps/chosen": -261.0470886230469,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -483.5485534667969,
"loss": 0.9114,
"margin_dpo/margin_mean": 179.70211791992188,
"margin_dpo/margin_std": 158.681396484375,
"step": 327
},
{
"KL/chosen_KL_mean": -220.88621520996094,
"KL/mean": -305.1409912109375,
"KL/rejected_KL_mean": -389.39581298828125,
"KL/std": 167.1742401123047,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0036455639638006687,
"fcm_dpo/delta": -0.0153718963265419,
"fcm_dpo/margin": 168.5095672607422,
"fcm_dpo/q_t": 0.36496418714523315,
"grad_norm": 50.387611389160156,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.36130693554878235,
"logits/rejected": -0.356844037771225,
"logps/chosen": -280.57342529296875,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -480.25079345703125,
"loss": 0.9576,
"margin_dpo/margin_mean": 168.50958251953125,
"margin_dpo/margin_std": 179.7239990234375,
"step": 328
},
{
"KL/chosen_KL_mean": -261.2420654296875,
"KL/mean": -355.39825439453125,
"KL/rejected_KL_mean": -449.554443359375,
"KL/std": 193.32135009765625,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0035698702558875084,
"fcm_dpo/delta": -0.07857734709978104,
"fcm_dpo/margin": 188.3123779296875,
"fcm_dpo/q_t": 0.3539862036705017,
"grad_norm": 41.062164306640625,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.30240148305892944,
"logits/rejected": -0.28401780128479004,
"logps/chosen": -326.48828125,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -550.2521362304688,
"loss": 0.9456,
"margin_dpo/margin_mean": 188.3123779296875,
"margin_dpo/margin_std": 205.26202392578125,
"step": 329
},
{
"KL/chosen_KL_mean": -238.8096923828125,
"KL/mean": -334.1317138671875,
"KL/rejected_KL_mean": -429.4537353515625,
"KL/std": 185.2899932861328,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.0035359621979296207,
"fcm_dpo/delta": -0.07942727208137512,
"fcm_dpo/margin": 190.64404296875,
"fcm_dpo/q_t": 0.35228052735328674,
"grad_norm": 36.57366180419922,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.2722437083721161,
"logits/rejected": -0.2734411656856537,
"logps/chosen": -285.80804443359375,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -516.33056640625,
"loss": 0.9249,
"margin_dpo/margin_mean": 190.64404296875,
"margin_dpo/margin_std": 194.9326171875,
"step": 330
},
{
"KL/chosen_KL_mean": -247.3241424560547,
"KL/mean": -348.72216796875,
"KL/rejected_KL_mean": -450.1202087402344,
"KL/std": 162.57382202148438,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.0034583178348839283,
"fcm_dpo/delta": -0.10915699601173401,
"fcm_dpo/margin": 202.7960662841797,
"fcm_dpo/q_t": 0.3417421579360962,
"grad_norm": 31.007814407348633,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.28995606303215027,
"logits/rejected": -0.2785801291465759,
"logps/chosen": -297.848388671875,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -539.1356201171875,
"loss": 0.8685,
"margin_dpo/margin_mean": 202.79608154296875,
"margin_dpo/margin_std": 159.70310974121094,
"step": 331
},
{
"KL/chosen_KL_mean": -257.9029541015625,
"KL/mean": -328.5245361328125,
"KL/rejected_KL_mean": -399.1461181640625,
"KL/std": 160.31040954589844,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0035032229498028755,
"fcm_dpo/delta": 0.11057021468877792,
"fcm_dpo/margin": 141.2431640625,
"fcm_dpo/q_t": 0.3903195858001709,
"grad_norm": 32.820167541503906,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.2842913269996643,
"logits/rejected": -0.2671872079372406,
"logps/chosen": -307.083251953125,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -475.6312561035156,
"loss": 1.0331,
"margin_dpo/margin_mean": 141.2431640625,
"margin_dpo/margin_std": 171.75302124023438,
"step": 332
},
{
"KL/chosen_KL_mean": -266.2708740234375,
"KL/mean": -362.4892883300781,
"KL/rejected_KL_mean": -458.70770263671875,
"KL/std": 196.3417205810547,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.0034516784362494946,
"fcm_dpo/delta": -0.07118657231330872,
"fcm_dpo/margin": 192.4368438720703,
"fcm_dpo/q_t": 0.3593972325325012,
"grad_norm": 31.546472549438477,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.3376733660697937,
"logits/rejected": -0.3297434449195862,
"logps/chosen": -330.026611328125,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -553.7518310546875,
"loss": 0.9694,
"margin_dpo/margin_mean": 192.4368438720703,
"margin_dpo/margin_std": 227.80015563964844,
"step": 333
},
{
"KL/chosen_KL_mean": -257.32025146484375,
"KL/mean": -342.2994079589844,
"KL/rejected_KL_mean": -427.2785339355469,
"KL/std": 210.2903289794922,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.003475816920399666,
"fcm_dpo/delta": 0.009836681187152863,
"fcm_dpo/margin": 169.95828247070312,
"fcm_dpo/q_t": 0.37404435873031616,
"grad_norm": 32.53291702270508,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.34100693464279175,
"logits/rejected": -0.3352089822292328,
"logps/chosen": -324.29998779296875,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -522.595458984375,
"loss": 1.0197,
"margin_dpo/margin_mean": 169.95828247070312,
"margin_dpo/margin_std": 226.54742431640625,
"step": 334
},
{
"KL/chosen_KL_mean": -277.65484619140625,
"KL/mean": -347.86737060546875,
"KL/rejected_KL_mean": -418.07989501953125,
"KL/std": 192.76744079589844,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.003556094830855727,
"fcm_dpo/delta": 0.10423801094293594,
"fcm_dpo/margin": 140.425048828125,
"fcm_dpo/q_t": 0.3937795162200928,
"grad_norm": 35.099327087402344,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.3291531801223755,
"logits/rejected": -0.30552738904953003,
"logps/chosen": -340.19732666015625,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -505.69757080078125,
"loss": 1.0745,
"margin_dpo/margin_mean": 140.42503356933594,
"margin_dpo/margin_std": 208.7427978515625,
"step": 335
},
{
"KL/chosen_KL_mean": -273.8084716796875,
"KL/mean": -378.71221923828125,
"KL/rejected_KL_mean": -483.6159362792969,
"KL/std": 216.06427001953125,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0034887937363237143,
"fcm_dpo/delta": -0.14215481281280518,
"fcm_dpo/margin": 209.80746459960938,
"fcm_dpo/q_t": 0.3478022813796997,
"grad_norm": 39.19478988647461,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.2858356833457947,
"logits/rejected": -0.2808038294315338,
"logps/chosen": -328.3396301269531,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -588.0201416015625,
"loss": 0.9443,
"margin_dpo/margin_mean": 209.80746459960938,
"margin_dpo/margin_std": 250.23760986328125,
"step": 336
},
{
"KL/chosen_KL_mean": -260.0228271484375,
"KL/mean": -365.30572509765625,
"KL/rejected_KL_mean": -470.5886535644531,
"KL/std": 188.71812438964844,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.0033786194398999214,
"fcm_dpo/delta": -0.12059810012578964,
"fcm_dpo/margin": 210.56581115722656,
"fcm_dpo/q_t": 0.3457186818122864,
"grad_norm": 31.037416458129883,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.32298046350479126,
"logits/rejected": -0.31106552481651306,
"logps/chosen": -325.1514892578125,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -572.315673828125,
"loss": 0.9114,
"margin_dpo/margin_mean": 210.56582641601562,
"margin_dpo/margin_std": 216.82025146484375,
"step": 337
},
{
"KL/chosen_KL_mean": -235.30941772460938,
"KL/mean": -338.76763916015625,
"KL/rejected_KL_mean": -442.22589111328125,
"KL/std": 185.00656127929688,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0033198068849742413,
"fcm_dpo/delta": -0.09342671185731888,
"fcm_dpo/margin": 206.91650390625,
"fcm_dpo/q_t": 0.3477078080177307,
"grad_norm": 24.489032745361328,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.3218623399734497,
"logits/rejected": -0.3046104311943054,
"logps/chosen": -293.73211669921875,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -531.29443359375,
"loss": 0.903,
"margin_dpo/margin_mean": 206.91650390625,
"margin_dpo/margin_std": 193.48056030273438,
"step": 338
},
{
"KL/chosen_KL_mean": -243.41168212890625,
"KL/mean": -341.48370361328125,
"KL/rejected_KL_mean": -439.55572509765625,
"KL/std": 187.0668182373047,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0033112098462879658,
"fcm_dpo/delta": -0.05422411859035492,
"fcm_dpo/margin": 196.14407348632812,
"fcm_dpo/q_t": 0.35589170455932617,
"grad_norm": 30.845415115356445,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.3370082378387451,
"logits/rejected": -0.319965124130249,
"logps/chosen": -303.406982421875,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -543.4666748046875,
"loss": 0.9294,
"margin_dpo/margin_mean": 196.14405822753906,
"margin_dpo/margin_std": 184.70452880859375,
"step": 339
},
{
"KL/chosen_KL_mean": -253.0582733154297,
"KL/mean": -338.5776672363281,
"KL/rejected_KL_mean": -424.0970764160156,
"KL/std": 172.04385375976562,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0032777278684079647,
"fcm_dpo/delta": 0.04154527932405472,
"fcm_dpo/margin": 171.03878784179688,
"fcm_dpo/q_t": 0.3767394423484802,
"grad_norm": 31.456979751586914,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.2925490736961365,
"logits/rejected": -0.2660925090312958,
"logps/chosen": -305.88848876953125,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -497.2043151855469,
"loss": 1.0038,
"margin_dpo/margin_mean": 171.03880310058594,
"margin_dpo/margin_std": 205.11837768554688,
"step": 340
},
{
"KL/chosen_KL_mean": -239.89349365234375,
"KL/mean": -338.16094970703125,
"KL/rejected_KL_mean": -436.4283752441406,
"KL/std": 192.15057373046875,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.0032605028245598078,
"fcm_dpo/delta": -0.044312167912721634,
"fcm_dpo/margin": 196.53485107421875,
"fcm_dpo/q_t": 0.3590165972709656,
"grad_norm": 28.23163414001465,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.3014819920063019,
"logits/rejected": -0.3003576397895813,
"logps/chosen": -287.7933044433594,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -538.23828125,
"loss": 0.9414,
"margin_dpo/margin_mean": 196.53485107421875,
"margin_dpo/margin_std": 200.83834838867188,
"step": 341
},
{
"KL/chosen_KL_mean": -231.43902587890625,
"KL/mean": -320.281494140625,
"KL/rejected_KL_mean": -409.12396240234375,
"KL/std": 177.18858337402344,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.003291256260126829,
"fcm_dpo/delta": 0.015661388635635376,
"fcm_dpo/margin": 177.68490600585938,
"fcm_dpo/q_t": 0.37189486622810364,
"grad_norm": 30.998788833618164,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.3793361783027649,
"logits/rejected": -0.3527992367744446,
"logps/chosen": -303.4356689453125,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -501.7135314941406,
"loss": 0.9814,
"margin_dpo/margin_mean": 177.68490600585938,
"margin_dpo/margin_std": 202.3661346435547,
"step": 342
},
{
"KL/chosen_KL_mean": -216.99691772460938,
"KL/mean": -319.0143737792969,
"KL/rejected_KL_mean": -421.0318603515625,
"KL/std": 176.3646240234375,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.003243247512727976,
"fcm_dpo/delta": -0.06622415781021118,
"fcm_dpo/margin": 204.034912109375,
"fcm_dpo/q_t": 0.34845924377441406,
"grad_norm": 26.33404541015625,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.3428516685962677,
"logits/rejected": -0.33821433782577515,
"logps/chosen": -271.40252685546875,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -532.0732421875,
"loss": 0.8905,
"margin_dpo/margin_mean": 204.034912109375,
"margin_dpo/margin_std": 156.75799560546875,
"step": 343
},
{
"KL/chosen_KL_mean": -223.7577362060547,
"KL/mean": -314.75634765625,
"KL/rejected_KL_mean": -405.75494384765625,
"KL/std": 187.5274658203125,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.0032703643664717674,
"fcm_dpo/delta": 0.0030049607157707214,
"fcm_dpo/margin": 181.99720764160156,
"fcm_dpo/q_t": 0.368649423122406,
"grad_norm": 28.670440673828125,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.3114033341407776,
"logits/rejected": -0.30788958072662354,
"logps/chosen": -277.722412109375,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -496.3782958984375,
"loss": 0.9615,
"margin_dpo/margin_mean": 181.99722290039062,
"margin_dpo/margin_std": 180.90835571289062,
"step": 344
},
{
"KL/chosen_KL_mean": -260.9364929199219,
"KL/mean": -357.65521240234375,
"KL/rejected_KL_mean": -454.3738708496094,
"KL/std": 182.0768280029297,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.003236721269786358,
"fcm_dpo/delta": -0.028028050437569618,
"fcm_dpo/margin": 193.43739318847656,
"fcm_dpo/q_t": 0.36065876483917236,
"grad_norm": 26.28773307800293,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.3865722715854645,
"logits/rejected": -0.3776073455810547,
"logps/chosen": -322.6221923828125,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -553.8642578125,
"loss": 0.9431,
"margin_dpo/margin_mean": 193.4373779296875,
"margin_dpo/margin_std": 194.0311279296875,
"step": 345
},
{
"KL/chosen_KL_mean": -275.28570556640625,
"KL/mean": -367.4283142089844,
"KL/rejected_KL_mean": -459.5709228515625,
"KL/std": 181.28677368164062,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.0032219619024544954,
"fcm_dpo/delta": 0.006633494049310684,
"fcm_dpo/margin": 184.28521728515625,
"fcm_dpo/q_t": 0.36650973558425903,
"grad_norm": 28.077381134033203,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.28504517674446106,
"logits/rejected": -0.26838934421539307,
"logps/chosen": -330.5419616699219,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -536.9862060546875,
"loss": 0.9668,
"margin_dpo/margin_mean": 184.2852325439453,
"margin_dpo/margin_std": 194.44482421875,
"step": 346
},
{
"KL/chosen_KL_mean": -275.24664306640625,
"KL/mean": -368.6918029785156,
"KL/rejected_KL_mean": -462.136962890625,
"KL/std": 192.2908935546875,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.003218740224838257,
"fcm_dpo/delta": -0.0016976585611701012,
"fcm_dpo/margin": 186.89035034179688,
"fcm_dpo/q_t": 0.36761733889579773,
"grad_norm": 33.92915344238281,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.3068666458129883,
"logits/rejected": -0.29829931259155273,
"logps/chosen": -332.8128662109375,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -554.4920654296875,
"loss": 0.9798,
"margin_dpo/margin_mean": 186.89035034179688,
"margin_dpo/margin_std": 215.3690948486328,
"step": 347
},
{
"KL/chosen_KL_mean": -257.12725830078125,
"KL/mean": -338.947509765625,
"KL/rejected_KL_mean": -420.76776123046875,
"KL/std": 172.65524291992188,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.0032317445147782564,
"fcm_dpo/delta": 0.07428452372550964,
"fcm_dpo/margin": 163.6405029296875,
"fcm_dpo/q_t": 0.38319721817970276,
"grad_norm": 32.93280029296875,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.32959896326065063,
"logits/rejected": -0.3305675983428955,
"logps/chosen": -313.4449462890625,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -509.9061279296875,
"loss": 1.0161,
"margin_dpo/margin_mean": 163.6405029296875,
"margin_dpo/margin_std": 189.34259033203125,
"step": 348
},
{
"KL/chosen_KL_mean": -250.6950225830078,
"KL/mean": -338.8772888183594,
"KL/rejected_KL_mean": -427.0595703125,
"KL/std": 188.6269989013672,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.003299848409369588,
"fcm_dpo/delta": 0.018306627869606018,
"fcm_dpo/margin": 176.36451721191406,
"fcm_dpo/q_t": 0.37589964270591736,
"grad_norm": 31.353382110595703,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.3009885549545288,
"logits/rejected": -0.29971545934677124,
"logps/chosen": -308.72052001953125,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -524.564697265625,
"loss": 1.0122,
"margin_dpo/margin_mean": 176.36453247070312,
"margin_dpo/margin_std": 228.71560668945312,
"step": 349
},
{
"KL/chosen_KL_mean": -264.7991943359375,
"KL/mean": -357.25238037109375,
"KL/rejected_KL_mean": -449.7055969238281,
"KL/std": 182.74017333984375,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.003264723112806678,
"fcm_dpo/delta": -0.004455354064702988,
"fcm_dpo/margin": 184.90638732910156,
"fcm_dpo/q_t": 0.36623501777648926,
"grad_norm": 37.09315490722656,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.3165681064128876,
"logits/rejected": -0.3130089044570923,
"logps/chosen": -329.12969970703125,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -539.5772705078125,
"loss": 0.9877,
"margin_dpo/margin_mean": 184.90638732910156,
"margin_dpo/margin_std": 219.01071166992188,
"step": 350
},
{
"KL/chosen_KL_mean": -259.0041809082031,
"KL/mean": -365.97845458984375,
"KL/rejected_KL_mean": -472.9527587890625,
"KL/std": 220.5889892578125,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0032329908572137356,
"fcm_dpo/delta": -0.09850767254829407,
"fcm_dpo/margin": 213.9486083984375,
"fcm_dpo/q_t": 0.35413599014282227,
"grad_norm": 29.696882247924805,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.33023127913475037,
"logits/rejected": -0.32282835245132446,
"logps/chosen": -319.6763000488281,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -574.5181884765625,
"loss": 0.9455,
"margin_dpo/margin_mean": 213.9486083984375,
"margin_dpo/margin_std": 248.4587860107422,
"step": 351
},
{
"KL/chosen_KL_mean": -320.5325012207031,
"KL/mean": -389.07843017578125,
"KL/rejected_KL_mean": -457.62432861328125,
"KL/std": 198.59683227539062,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.003274778835475445,
"fcm_dpo/delta": 0.15839095413684845,
"fcm_dpo/margin": 137.09182739257812,
"fcm_dpo/q_t": 0.40359771251678467,
"grad_norm": 48.864864349365234,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.32900527119636536,
"logits/rejected": -0.2986646890640259,
"logps/chosen": -391.4759521484375,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -534.2662963867188,
"loss": 1.1023,
"margin_dpo/margin_mean": 137.09185791015625,
"margin_dpo/margin_std": 214.62515258789062,
"step": 352
},
{
"KL/chosen_KL_mean": -305.11053466796875,
"KL/mean": -396.4449157714844,
"KL/rejected_KL_mean": -487.7792663574219,
"KL/std": 209.36056518554688,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0032949901651591063,
"fcm_dpo/delta": -0.0030858218669891357,
"fcm_dpo/margin": 182.6687469482422,
"fcm_dpo/q_t": 0.37244531512260437,
"grad_norm": 32.291099548339844,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.3360978960990906,
"logits/rejected": -0.3211211562156677,
"logps/chosen": -363.505859375,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -568.1148071289062,
"loss": 1.0047,
"margin_dpo/margin_mean": 182.6687469482422,
"margin_dpo/margin_std": 233.64218139648438,
"step": 353
},
{
"KL/chosen_KL_mean": -263.1873779296875,
"KL/mean": -363.31756591796875,
"KL/rejected_KL_mean": -463.44775390625,
"KL/std": 189.3446044921875,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.0032818112522363663,
"fcm_dpo/delta": -0.061426229774951935,
"fcm_dpo/margin": 200.26036071777344,
"fcm_dpo/q_t": 0.35361698269844055,
"grad_norm": 31.09943199157715,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.3072470426559448,
"logits/rejected": -0.28870946168899536,
"logps/chosen": -322.9903564453125,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -552.2052612304688,
"loss": 0.9156,
"margin_dpo/margin_mean": 200.26036071777344,
"margin_dpo/margin_std": 186.870361328125,
"step": 354
},
{
"KL/chosen_KL_mean": -260.0174560546875,
"KL/mean": -359.42108154296875,
"KL/rejected_KL_mean": -458.82470703125,
"KL/std": 202.15090942382812,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.003239832818508148,
"fcm_dpo/delta": -0.04776415228843689,
"fcm_dpo/margin": 198.8072509765625,
"fcm_dpo/q_t": 0.3572244644165039,
"grad_norm": 65.14305114746094,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.3648909032344818,
"logits/rejected": -0.3481696844100952,
"logps/chosen": -314.14593505859375,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -541.2307739257812,
"loss": 0.9324,
"margin_dpo/margin_mean": 198.8072509765625,
"margin_dpo/margin_std": 195.83514404296875,
"step": 355
},
{
"KL/chosen_KL_mean": -302.3600158691406,
"KL/mean": -362.93133544921875,
"KL/rejected_KL_mean": -423.502685546875,
"KL/std": 202.94854736328125,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.0032497502397745848,
"fcm_dpo/delta": 0.010602913796901703,
"fcm_dpo/margin": 121.14266967773438,
"fcm_dpo/q_t": 0.4173041582107544,
"grad_norm": 51.611122131347656,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.36508941650390625,
"logits/rejected": -0.3437308669090271,
"logps/chosen": -367.0338134765625,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -499.4019775390625,
"loss": 1.1781,
"margin_dpo/margin_mean": 121.1426773071289,
"margin_dpo/margin_std": 243.44647216796875,
"step": 356
},
{
"KL/chosen_KL_mean": -270.8690185546875,
"KL/mean": -359.669677734375,
"KL/rejected_KL_mean": -448.47027587890625,
"KL/std": 197.69229125976562,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.00324799632653594,
"fcm_dpo/delta": 0.024136211723089218,
"fcm_dpo/margin": 177.60125732421875,
"fcm_dpo/q_t": 0.37204235792160034,
"grad_norm": 43.36853790283203,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.34672385454177856,
"logits/rejected": -0.329520046710968,
"logps/chosen": -323.5948486328125,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -535.3114624023438,
"loss": 0.9803,
"margin_dpo/margin_mean": 177.60125732421875,
"margin_dpo/margin_std": 193.69900512695312,
"step": 357
},
{
"KL/chosen_KL_mean": -237.119140625,
"KL/mean": -316.61358642578125,
"KL/rejected_KL_mean": -396.10797119140625,
"KL/std": 183.86444091796875,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0033094545360654593,
"fcm_dpo/delta": 0.07786554843187332,
"fcm_dpo/margin": 158.9888458251953,
"fcm_dpo/q_t": 0.38529354333877563,
"grad_norm": 41.24159622192383,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.41370850801467896,
"logits/rejected": -0.39462339878082275,
"logps/chosen": -300.3245849609375,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -484.48126220703125,
"loss": 1.0165,
"margin_dpo/margin_mean": 158.98886108398438,
"margin_dpo/margin_std": 194.11642456054688,
"step": 358
},
{
"KL/chosen_KL_mean": -267.33795166015625,
"KL/mean": -363.5581359863281,
"KL/rejected_KL_mean": -459.7783203125,
"KL/std": 186.1640625,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.003295338712632656,
"fcm_dpo/delta": -0.03663625195622444,
"fcm_dpo/margin": 192.44036865234375,
"fcm_dpo/q_t": 0.3640950322151184,
"grad_norm": 44.32215118408203,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.31459060311317444,
"logits/rejected": -0.29139894247055054,
"logps/chosen": -323.70819091796875,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -541.9520263671875,
"loss": 0.9582,
"margin_dpo/margin_mean": 192.44036865234375,
"margin_dpo/margin_std": 213.06277465820312,
"step": 359
},
{
"KL/chosen_KL_mean": -251.32090759277344,
"KL/mean": -326.62860107421875,
"KL/rejected_KL_mean": -401.936279296875,
"KL/std": 158.09228515625,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.003354067215695977,
"fcm_dpo/delta": 0.09945917874574661,
"fcm_dpo/margin": 150.61538696289062,
"fcm_dpo/q_t": 0.3858073949813843,
"grad_norm": 44.88187026977539,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.29673099517822266,
"logits/rejected": -0.2805573344230652,
"logps/chosen": -302.78131103515625,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -471.77520751953125,
"loss": 1.0097,
"margin_dpo/margin_mean": 150.61537170410156,
"margin_dpo/margin_std": 158.94903564453125,
"step": 360
},
{
"KL/chosen_KL_mean": -264.8636474609375,
"KL/mean": -342.61810302734375,
"KL/rejected_KL_mean": -420.37249755859375,
"KL/std": 187.45339965820312,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0034134057350456715,
"fcm_dpo/delta": 0.07235899567604065,
"fcm_dpo/margin": 155.50885009765625,
"fcm_dpo/q_t": 0.38437366485595703,
"grad_norm": 38.67572784423828,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.33011433482170105,
"logits/rejected": -0.3226510286331177,
"logps/chosen": -318.7331848144531,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -511.14178466796875,
"loss": 1.033,
"margin_dpo/margin_mean": 155.5088348388672,
"margin_dpo/margin_std": 203.7720947265625,
"step": 361
},
{
"KL/chosen_KL_mean": -230.20098876953125,
"KL/mean": -345.866455078125,
"KL/rejected_KL_mean": -461.5319519042969,
"KL/std": 199.43051147460938,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.003302750177681446,
"fcm_dpo/delta": -0.17932967841625214,
"fcm_dpo/margin": 231.33099365234375,
"fcm_dpo/q_t": 0.33344629406929016,
"grad_norm": 38.8252067565918,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.35263383388519287,
"logits/rejected": -0.35993361473083496,
"logps/chosen": -288.84002685546875,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -567.1138916015625,
"loss": 0.8578,
"margin_dpo/margin_mean": 231.33099365234375,
"margin_dpo/margin_std": 206.98098754882812,
"step": 362
},
{
"KL/chosen_KL_mean": -207.17837524414062,
"KL/mean": -327.6678466796875,
"KL/rejected_KL_mean": -448.1573486328125,
"KL/std": 196.34474182128906,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0032095485366880894,
"fcm_dpo/delta": -0.18762367963790894,
"fcm_dpo/margin": 240.97897338867188,
"fcm_dpo/q_t": 0.32998448610305786,
"grad_norm": 29.219369888305664,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.3258933126926422,
"logits/rejected": -0.30995145440101624,
"logps/chosen": -251.73675537109375,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -522.852294921875,
"loss": 0.8495,
"margin_dpo/margin_mean": 240.97897338867188,
"margin_dpo/margin_std": 207.41574096679688,
"step": 363
},
{
"KL/chosen_KL_mean": -221.3150634765625,
"KL/mean": -311.22283935546875,
"KL/rejected_KL_mean": -401.130615234375,
"KL/std": 186.37693786621094,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.003198642283678055,
"fcm_dpo/delta": 0.02565312758088112,
"fcm_dpo/margin": 179.81556701660156,
"fcm_dpo/q_t": 0.3736268877983093,
"grad_norm": 30.076086044311523,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.35051482915878296,
"logits/rejected": -0.35855984687805176,
"logps/chosen": -270.2096862792969,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -492.5263671875,
"loss": 0.991,
"margin_dpo/margin_mean": 179.8155517578125,
"margin_dpo/margin_std": 207.90798950195312,
"step": 364
},
{
"KL/chosen_KL_mean": -233.87704467773438,
"KL/mean": -324.1929626464844,
"KL/rejected_KL_mean": -414.5088806152344,
"KL/std": 203.30694580078125,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0031994380988180637,
"fcm_dpo/delta": 0.023408135399222374,
"fcm_dpo/margin": 180.63186645507812,
"fcm_dpo/q_t": 0.37520602345466614,
"grad_norm": 30.558664321899414,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.34205323457717896,
"logits/rejected": -0.34964728355407715,
"logps/chosen": -285.3697814941406,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -507.2105407714844,
"loss": 0.9823,
"margin_dpo/margin_mean": 180.63186645507812,
"margin_dpo/margin_std": 208.0732421875,
"step": 365
},
{
"KL/chosen_KL_mean": -222.40472412109375,
"KL/mean": -314.787109375,
"KL/rejected_KL_mean": -407.1695251464844,
"KL/std": 194.21192932128906,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.003188899951055646,
"fcm_dpo/delta": 0.010924622416496277,
"fcm_dpo/margin": 184.76480102539062,
"fcm_dpo/q_t": 0.37210413813591003,
"grad_norm": 33.09766387939453,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.3388650715351105,
"logits/rejected": -0.35057055950164795,
"logps/chosen": -267.1252746582031,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -490.47991943359375,
"loss": 0.9941,
"margin_dpo/margin_mean": 184.76478576660156,
"margin_dpo/margin_std": 221.24745178222656,
"step": 366
},
{
"KL/chosen_KL_mean": -224.12933349609375,
"KL/mean": -303.8285217285156,
"KL/rejected_KL_mean": -383.5277099609375,
"KL/std": 173.01712036132812,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0032505434937775135,
"fcm_dpo/delta": 0.08631440252065659,
"fcm_dpo/margin": 159.39840698242188,
"fcm_dpo/q_t": 0.3836424946784973,
"grad_norm": 27.49794578552246,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.3360174894332886,
"logits/rejected": -0.3131776452064514,
"logps/chosen": -282.53472900390625,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -460.2790222167969,
"loss": 1.014,
"margin_dpo/margin_mean": 159.39840698242188,
"margin_dpo/margin_std": 182.224609375,
"step": 367
},
{
"KL/chosen_KL_mean": -217.24249267578125,
"KL/mean": -332.43218994140625,
"KL/rejected_KL_mean": -447.62188720703125,
"KL/std": 193.49945068359375,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.003197380807250738,
"fcm_dpo/delta": -0.14732971787452698,
"fcm_dpo/margin": 230.37936401367188,
"fcm_dpo/q_t": 0.3323744535446167,
"grad_norm": 42.817474365234375,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.3793472945690155,
"logits/rejected": -0.38116952776908875,
"logps/chosen": -261.69500732421875,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -546.1771240234375,
"loss": 0.8467,
"margin_dpo/margin_mean": 230.3793487548828,
"margin_dpo/margin_std": 171.62734985351562,
"step": 368
},
{
"KL/chosen_KL_mean": -259.3919982910156,
"KL/mean": -333.142578125,
"KL/rejected_KL_mean": -406.8931579589844,
"KL/std": 190.53173828125,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.003244384191930294,
"fcm_dpo/delta": 0.1265307366847992,
"fcm_dpo/margin": 147.50119018554688,
"fcm_dpo/q_t": 0.3942197859287262,
"grad_norm": 40.823333740234375,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.36486881971359253,
"logits/rejected": -0.3441947400569916,
"logps/chosen": -330.77349853515625,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -498.18896484375,
"loss": 1.0634,
"margin_dpo/margin_mean": 147.50119018554688,
"margin_dpo/margin_std": 202.70083618164062,
"step": 369
},
{
"KL/chosen_KL_mean": -284.43145751953125,
"KL/mean": -346.6457824707031,
"KL/rejected_KL_mean": -408.860107421875,
"KL/std": 187.88595581054688,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.003352168947458267,
"fcm_dpo/delta": 0.1902626007795334,
"fcm_dpo/margin": 124.42868041992188,
"fcm_dpo/q_t": 0.4083307981491089,
"grad_norm": 44.87315368652344,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.3443584144115448,
"logits/rejected": -0.3377593457698822,
"logps/chosen": -356.0389404296875,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -506.1199035644531,
"loss": 1.1158,
"margin_dpo/margin_mean": 124.42868041992188,
"margin_dpo/margin_std": 199.98040771484375,
"step": 370
},
{
"KL/chosen_KL_mean": -271.2171630859375,
"KL/mean": -362.92132568359375,
"KL/rejected_KL_mean": -454.6255187988281,
"KL/std": 199.44146728515625,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0033790075685828924,
"fcm_dpo/delta": -0.0212722048163414,
"fcm_dpo/margin": 183.4083709716797,
"fcm_dpo/q_t": 0.3653963804244995,
"grad_norm": 34.15528869628906,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.3939722180366516,
"logits/rejected": -0.38019344210624695,
"logps/chosen": -340.63165283203125,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -553.7977294921875,
"loss": 0.9792,
"margin_dpo/margin_mean": 183.40835571289062,
"margin_dpo/margin_std": 217.07089233398438,
"step": 371
},
{
"KL/chosen_KL_mean": -263.29425048828125,
"KL/mean": -372.962158203125,
"KL/rejected_KL_mean": -482.63006591796875,
"KL/std": 229.61434936523438,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.0032852408476173878,
"fcm_dpo/delta": -0.13072502613067627,
"fcm_dpo/margin": 219.33583068847656,
"fcm_dpo/q_t": 0.34957951307296753,
"grad_norm": 30.624921798706055,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.35687190294265747,
"logits/rejected": -0.3293916583061218,
"logps/chosen": -325.11224365234375,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -561.1695556640625,
"loss": 0.9314,
"margin_dpo/margin_mean": 219.33583068847656,
"margin_dpo/margin_std": 252.03981018066406,
"step": 372
},
{
"KL/chosen_KL_mean": -284.2254333496094,
"KL/mean": -392.66082763671875,
"KL/rejected_KL_mean": -501.09625244140625,
"KL/std": 223.63372802734375,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.0032459371723234653,
"fcm_dpo/delta": -0.11214579641819,
"fcm_dpo/margin": 216.87083435058594,
"fcm_dpo/q_t": 0.3463241159915924,
"grad_norm": 37.03703689575195,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.33413344621658325,
"logits/rejected": -0.30599355697631836,
"logps/chosen": -348.44256591796875,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -587.055908203125,
"loss": 0.915,
"margin_dpo/margin_mean": 216.87081909179688,
"margin_dpo/margin_std": 215.33447265625,
"step": 373
},
{
"KL/chosen_KL_mean": -254.56488037109375,
"KL/mean": -348.21026611328125,
"KL/rejected_KL_mean": -441.85565185546875,
"KL/std": 231.42327880859375,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.003183186985552311,
"fcm_dpo/delta": 0.0031774193048477173,
"fcm_dpo/margin": 187.29074096679688,
"fcm_dpo/q_t": 0.37585270404815674,
"grad_norm": 40.62038040161133,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.42012012004852295,
"logits/rejected": -0.39598315954208374,
"logps/chosen": -325.215087890625,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -535.495849609375,
"loss": 1.0093,
"margin_dpo/margin_mean": 187.29074096679688,
"margin_dpo/margin_std": 246.7799530029297,
"step": 374
},
{
"KL/chosen_KL_mean": -279.15850830078125,
"KL/mean": -370.8597412109375,
"KL/rejected_KL_mean": -462.5609130859375,
"KL/std": 203.83473205566406,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.003183454740792513,
"fcm_dpo/delta": 0.015603147447109222,
"fcm_dpo/margin": 183.40240478515625,
"fcm_dpo/q_t": 0.3734520375728607,
"grad_norm": 48.257781982421875,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.35360097885131836,
"logits/rejected": -0.35042452812194824,
"logps/chosen": -339.2387390136719,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -551.4992065429688,
"loss": 1.0195,
"margin_dpo/margin_mean": 183.40240478515625,
"margin_dpo/margin_std": 237.6729736328125,
"step": 375
},
{
"KL/chosen_KL_mean": -263.6816101074219,
"KL/mean": -374.9967346191406,
"KL/rejected_KL_mean": -486.3118896484375,
"KL/std": 214.6494140625,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.003158073639497161,
"fcm_dpo/delta": -0.1110667809844017,
"fcm_dpo/margin": 222.63026428222656,
"fcm_dpo/q_t": 0.34953978657722473,
"grad_norm": 33.554718017578125,
"learning_rate": 2.5e-07,
"logits/chosen": -0.35632210969924927,
"logits/rejected": -0.3426344394683838,
"logps/chosen": -326.3419189453125,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -591.8385009765625,
"loss": 0.9354,
"margin_dpo/margin_mean": 222.63027954101562,
"margin_dpo/margin_std": 249.431396484375,
"step": 376
},
{
"KL/chosen_KL_mean": -279.59716796875,
"KL/mean": -383.43109130859375,
"KL/rejected_KL_mean": -487.26507568359375,
"KL/std": 221.8564910888672,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.0031269832979887724,
"fcm_dpo/delta": -0.052792586386203766,
"fcm_dpo/margin": 207.66793823242188,
"fcm_dpo/q_t": 0.358760267496109,
"grad_norm": 32.23828125,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.28237342834472656,
"logits/rejected": -0.276389479637146,
"logps/chosen": -334.07586669921875,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -585.9684448242188,
"loss": 0.9483,
"margin_dpo/margin_mean": 207.66793823242188,
"margin_dpo/margin_std": 225.5984649658203,
"step": 377
},
{
"KL/chosen_KL_mean": -264.40350341796875,
"KL/mean": -379.8725280761719,
"KL/rejected_KL_mean": -495.34149169921875,
"KL/std": 208.23890686035156,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.0030510034412145615,
"fcm_dpo/delta": -0.1129959374666214,
"fcm_dpo/margin": 230.93800354003906,
"fcm_dpo/q_t": 0.3426278829574585,
"grad_norm": 32.43430709838867,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.28662872314453125,
"logits/rejected": -0.29121625423431396,
"logps/chosen": -309.424072265625,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -583.3884887695312,
"loss": 0.8968,
"margin_dpo/margin_mean": 230.93800354003906,
"margin_dpo/margin_std": 214.55853271484375,
"step": 378
},
{
"KL/chosen_KL_mean": -290.95556640625,
"KL/mean": -397.6083679199219,
"KL/rejected_KL_mean": -504.26116943359375,
"KL/std": 205.03558349609375,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.003007414285093546,
"fcm_dpo/delta": -0.04499698057770729,
"fcm_dpo/margin": 213.30560302734375,
"fcm_dpo/q_t": 0.3615230917930603,
"grad_norm": 41.569034576416016,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.2972605526447296,
"logits/rejected": -0.31412017345428467,
"logps/chosen": -344.1376953125,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -618.5613403320312,
"loss": 0.9596,
"margin_dpo/margin_mean": 213.30560302734375,
"margin_dpo/margin_std": 242.21095275878906,
"step": 379
},
{
"KL/chosen_KL_mean": -287.45220947265625,
"KL/mean": -406.9426574707031,
"KL/rejected_KL_mean": -526.4331665039062,
"KL/std": 237.34384155273438,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.0029722503386437893,
"fcm_dpo/delta": -0.11858358979225159,
"fcm_dpo/margin": 238.98095703125,
"fcm_dpo/q_t": 0.344382643699646,
"grad_norm": 33.26060485839844,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.30034565925598145,
"logits/rejected": -0.32280099391937256,
"logps/chosen": -338.80523681640625,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -630.6248779296875,
"loss": 0.9258,
"margin_dpo/margin_mean": 238.98095703125,
"margin_dpo/margin_std": 256.5580139160156,
"step": 380
},
{
"KL/chosen_KL_mean": -285.04913330078125,
"KL/mean": -376.41357421875,
"KL/rejected_KL_mean": -467.7780456542969,
"KL/std": 195.86361694335938,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.002964303595945239,
"fcm_dpo/delta": 0.06171252951025963,
"fcm_dpo/margin": 182.7288818359375,
"fcm_dpo/q_t": 0.3842027187347412,
"grad_norm": 33.78224182128906,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.32100236415863037,
"logits/rejected": -0.29901283979415894,
"logps/chosen": -342.85223388671875,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -546.9974365234375,
"loss": 1.0502,
"margin_dpo/margin_mean": 182.7288818359375,
"margin_dpo/margin_std": 262.4327697753906,
"step": 381
},
{
"KL/chosen_KL_mean": -286.43701171875,
"KL/mean": -387.11102294921875,
"KL/rejected_KL_mean": -487.7850341796875,
"KL/std": 184.3202362060547,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0029781339690089226,
"fcm_dpo/delta": 0.00036939699202775955,
"fcm_dpo/margin": 201.34799194335938,
"fcm_dpo/q_t": 0.3653753697872162,
"grad_norm": 33.5085334777832,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.37374958395957947,
"logits/rejected": -0.3705790638923645,
"logps/chosen": -352.45733642578125,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -598.4951782226562,
"loss": 0.9498,
"margin_dpo/margin_mean": 201.34799194335938,
"margin_dpo/margin_std": 194.20318603515625,
"step": 382
},
{
"KL/chosen_KL_mean": -291.7559814453125,
"KL/mean": -394.878662109375,
"KL/rejected_KL_mean": -498.0013427734375,
"KL/std": 211.57595825195312,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0029702105093747377,
"fcm_dpo/delta": -0.013477612286806107,
"fcm_dpo/margin": 206.245361328125,
"fcm_dpo/q_t": 0.3686668872833252,
"grad_norm": 31.9790096282959,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.3055593967437744,
"logits/rejected": -0.3108689785003662,
"logps/chosen": -342.1474609375,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -591.7172241210938,
"loss": 1.003,
"margin_dpo/margin_mean": 206.245361328125,
"margin_dpo/margin_std": 269.244140625,
"step": 383
},
{
"KL/chosen_KL_mean": -289.405517578125,
"KL/mean": -374.2627868652344,
"KL/rejected_KL_mean": -459.12005615234375,
"KL/std": 191.923095703125,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.003028924809768796,
"fcm_dpo/delta": 0.08948992937803268,
"fcm_dpo/margin": 169.71456909179688,
"fcm_dpo/q_t": 0.38307762145996094,
"grad_norm": 28.06199073791504,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.35736358165740967,
"logits/rejected": -0.3461098074913025,
"logps/chosen": -341.45159912109375,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -544.8809814453125,
"loss": 1.004,
"margin_dpo/margin_mean": 169.71456909179688,
"margin_dpo/margin_std": 176.09442138671875,
"step": 384
},
{
"KL/chosen_KL_mean": -270.73931884765625,
"KL/mean": -371.95343017578125,
"KL/rejected_KL_mean": -473.16754150390625,
"KL/std": 169.21913146972656,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.00302038062363863,
"fcm_dpo/delta": -0.012154972180724144,
"fcm_dpo/margin": 202.42823791503906,
"fcm_dpo/q_t": 0.3613358736038208,
"grad_norm": 29.232404708862305,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.36031654477119446,
"logits/rejected": -0.32724112272262573,
"logps/chosen": -336.2914733886719,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -550.9954833984375,
"loss": 0.9361,
"margin_dpo/margin_mean": 202.42823791503906,
"margin_dpo/margin_std": 184.14300537109375,
"step": 385
},
{
"KL/chosen_KL_mean": -288.425048828125,
"KL/mean": -394.9166259765625,
"KL/rejected_KL_mean": -501.40814208984375,
"KL/std": 207.48939514160156,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.0030043399892747402,
"fcm_dpo/delta": -0.04258216917514801,
"fcm_dpo/margin": 212.98309326171875,
"fcm_dpo/q_t": 0.35987475514411926,
"grad_norm": 28.158893585205078,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.35813117027282715,
"logits/rejected": -0.3487015962600708,
"logps/chosen": -346.64691162109375,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -593.735595703125,
"loss": 0.9509,
"margin_dpo/margin_mean": 212.98309326171875,
"margin_dpo/margin_std": 232.02841186523438,
"step": 386
},
{
"KL/chosen_KL_mean": -312.46868896484375,
"KL/mean": -401.2229919433594,
"KL/rejected_KL_mean": -489.977294921875,
"KL/std": 190.97109985351562,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.0030100648291409016,
"fcm_dpo/delta": 0.06926459819078445,
"fcm_dpo/margin": 177.50863647460938,
"fcm_dpo/q_t": 0.37951982021331787,
"grad_norm": 40.10986328125,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.37982168793678284,
"logits/rejected": -0.3499566912651062,
"logps/chosen": -378.88812255859375,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -582.146484375,
"loss": 0.9967,
"margin_dpo/margin_mean": 177.50863647460938,
"margin_dpo/margin_std": 188.54428100585938,
"step": 387
},
{
"KL/chosen_KL_mean": -294.6017761230469,
"KL/mean": -411.54327392578125,
"KL/rejected_KL_mean": -528.4848022460938,
"KL/std": 224.71279907226562,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.00297818542458117,
"fcm_dpo/delta": -0.10437505692243576,
"fcm_dpo/margin": 233.88296508789062,
"fcm_dpo/q_t": 0.35114359855651855,
"grad_norm": 36.02871322631836,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.3343929648399353,
"logits/rejected": -0.3432833254337311,
"logps/chosen": -344.7312316894531,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -632.9178466796875,
"loss": 0.9262,
"margin_dpo/margin_mean": 233.88296508789062,
"margin_dpo/margin_std": 256.0967712402344,
"step": 388
},
{
"KL/chosen_KL_mean": -325.771484375,
"KL/mean": -428.99578857421875,
"KL/rejected_KL_mean": -532.2200927734375,
"KL/std": 222.25308227539062,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.002964036539196968,
"fcm_dpo/delta": -0.012725681066513062,
"fcm_dpo/margin": 206.44857788085938,
"fcm_dpo/q_t": 0.3668934106826782,
"grad_norm": 34.471981048583984,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.3404477834701538,
"logits/rejected": -0.3211250305175781,
"logps/chosen": -383.6780700683594,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -610.1346435546875,
"loss": 0.9775,
"margin_dpo/margin_mean": 206.44859313964844,
"margin_dpo/margin_std": 240.46298217773438,
"step": 389
},
{
"KL/chosen_KL_mean": -318.89678955078125,
"KL/mean": -418.1431884765625,
"KL/rejected_KL_mean": -517.3895263671875,
"KL/std": 214.55783081054688,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.0029571224004030228,
"fcm_dpo/delta": 0.013474367558956146,
"fcm_dpo/margin": 198.4927978515625,
"fcm_dpo/q_t": 0.3762761950492859,
"grad_norm": 35.749568939208984,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.33758920431137085,
"logits/rejected": -0.3237559497356415,
"logps/chosen": -368.1226806640625,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -602.917724609375,
"loss": 1.0248,
"margin_dpo/margin_mean": 198.4927978515625,
"margin_dpo/margin_std": 276.93096923828125,
"step": 390
},
{
"KL/chosen_KL_mean": -305.608154296875,
"KL/mean": -372.33221435546875,
"KL/rejected_KL_mean": -439.0562744140625,
"KL/std": 204.19851684570312,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0030692159198224545,
"fcm_dpo/delta": 0.1979488730430603,
"fcm_dpo/margin": 133.4481201171875,
"fcm_dpo/q_t": 0.41186365485191345,
"grad_norm": 43.194095611572266,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.35497570037841797,
"logits/rejected": -0.3478752672672272,
"logps/chosen": -369.93780517578125,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -525.79443359375,
"loss": 1.1146,
"margin_dpo/margin_mean": 133.4481201171875,
"margin_dpo/margin_std": 211.03024291992188,
"step": 391
},
{
"KL/chosen_KL_mean": -269.6353454589844,
"KL/mean": -385.9117736816406,
"KL/rejected_KL_mean": -502.1882019042969,
"KL/std": 216.24798583984375,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0030516264960169792,
"fcm_dpo/delta": -0.11790303140878677,
"fcm_dpo/margin": 232.5528564453125,
"fcm_dpo/q_t": 0.3435419201850891,
"grad_norm": 29.419065475463867,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.3578484058380127,
"logits/rejected": -0.3481765389442444,
"logps/chosen": -323.13934326171875,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -604.5340576171875,
"loss": 0.8871,
"margin_dpo/margin_mean": 232.5528564453125,
"margin_dpo/margin_std": 209.47817993164062,
"step": 392
},
{
"KL/chosen_KL_mean": -266.7149658203125,
"KL/mean": -357.5413818359375,
"KL/rejected_KL_mean": -448.36785888671875,
"KL/std": 219.1102294921875,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.00304000498726964,
"fcm_dpo/delta": 0.05059793218970299,
"fcm_dpo/margin": 181.65281677246094,
"fcm_dpo/q_t": 0.38655680418014526,
"grad_norm": 31.98375129699707,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.30934107303619385,
"logits/rejected": -0.3014167845249176,
"logps/chosen": -313.18890380859375,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -520.336669921875,
"loss": 1.0477,
"margin_dpo/margin_mean": 181.65283203125,
"margin_dpo/margin_std": 272.7147216796875,
"step": 393
},
{
"KL/chosen_KL_mean": -298.0819091796875,
"KL/mean": -399.7223205566406,
"KL/rejected_KL_mean": -501.36273193359375,
"KL/std": 228.62066650390625,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0030346158891916275,
"fcm_dpo/delta": -0.018204499036073685,
"fcm_dpo/margin": 203.28082275390625,
"fcm_dpo/q_t": 0.3683304786682129,
"grad_norm": 33.16108703613281,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.3650529682636261,
"logits/rejected": -0.35924166440963745,
"logps/chosen": -350.99346923828125,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -592.1853637695312,
"loss": 0.9828,
"margin_dpo/margin_mean": 203.2808074951172,
"margin_dpo/margin_std": 246.54742431640625,
"step": 394
},
{
"KL/chosen_KL_mean": -299.6165466308594,
"KL/mean": -404.094482421875,
"KL/rejected_KL_mean": -508.57244873046875,
"KL/std": 234.14849853515625,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0030147230718284845,
"fcm_dpo/delta": -0.03235205262899399,
"fcm_dpo/margin": 208.95587158203125,
"fcm_dpo/q_t": 0.36819878220558167,
"grad_norm": 37.2800407409668,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.3679765462875366,
"logits/rejected": -0.3596211075782776,
"logps/chosen": -362.16265869140625,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -592.3550415039062,
"loss": 0.9947,
"margin_dpo/margin_mean": 208.95587158203125,
"margin_dpo/margin_std": 271.637451171875,
"step": 395
},
{
"KL/chosen_KL_mean": -304.382080078125,
"KL/mean": -403.36395263671875,
"KL/rejected_KL_mean": -502.34588623046875,
"KL/std": 222.828857421875,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.0030169161036610603,
"fcm_dpo/delta": 0.002881184220314026,
"fcm_dpo/margin": 197.96380615234375,
"fcm_dpo/q_t": 0.3687392473220825,
"grad_norm": 31.803199768066406,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.36917823553085327,
"logits/rejected": -0.3443046510219574,
"logps/chosen": -373.37799072265625,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -590.9925537109375,
"loss": 0.9648,
"margin_dpo/margin_mean": 197.96380615234375,
"margin_dpo/margin_std": 214.8352813720703,
"step": 396
},
{
"KL/chosen_KL_mean": -284.61212158203125,
"KL/mean": -410.96392822265625,
"KL/rejected_KL_mean": -537.3157348632812,
"KL/std": 225.3281707763672,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.0029368908144533634,
"fcm_dpo/delta": -0.15513816475868225,
"fcm_dpo/margin": 252.70358276367188,
"fcm_dpo/q_t": 0.33645138144493103,
"grad_norm": 36.88515090942383,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.3128030300140381,
"logits/rejected": -0.29988163709640503,
"logps/chosen": -345.8892822265625,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -640.431884765625,
"loss": 0.8754,
"margin_dpo/margin_mean": 252.70358276367188,
"margin_dpo/margin_std": 229.07723999023438,
"step": 397
},
{
"KL/chosen_KL_mean": -316.03265380859375,
"KL/mean": -425.3857421875,
"KL/rejected_KL_mean": -534.7388916015625,
"KL/std": 229.64132690429688,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.0029114214703440666,
"fcm_dpo/delta": -0.03927408903837204,
"fcm_dpo/margin": 218.70614624023438,
"fcm_dpo/q_t": 0.36444124579429626,
"grad_norm": 31.996997833251953,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.4120296239852905,
"logits/rejected": -0.40254727005958557,
"logps/chosen": -384.1842041015625,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -643.262451171875,
"loss": 0.9662,
"margin_dpo/margin_mean": 218.70614624023438,
"margin_dpo/margin_std": 258.6249694824219,
"step": 398
},
{
"KL/chosen_KL_mean": -279.52880859375,
"KL/mean": -384.9920654296875,
"KL/rejected_KL_mean": -490.455322265625,
"KL/std": 210.19830322265625,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0028921724297106266,
"fcm_dpo/delta": -0.010987410321831703,
"fcm_dpo/margin": 210.92648315429688,
"fcm_dpo/q_t": 0.3680976927280426,
"grad_norm": 40.43487548828125,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.34756147861480713,
"logits/rejected": -0.29680293798446655,
"logps/chosen": -340.41864013671875,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -568.4208984375,
"loss": 0.9824,
"margin_dpo/margin_mean": 210.92645263671875,
"margin_dpo/margin_std": 250.62240600585938,
"step": 399
},
{
"KL/chosen_KL_mean": -274.4811096191406,
"KL/mean": -421.08203125,
"KL/rejected_KL_mean": -567.6829223632812,
"KL/std": 229.29571533203125,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.0027863583527505398,
"fcm_dpo/delta": -0.2380165159702301,
"fcm_dpo/margin": 293.2017822265625,
"fcm_dpo/q_t": 0.3205851912498474,
"grad_norm": 29.75553321838379,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.355247437953949,
"logits/rejected": -0.342274010181427,
"logps/chosen": -338.12469482421875,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -672.9356079101562,
"loss": 0.833,
"margin_dpo/margin_mean": 293.2018127441406,
"margin_dpo/margin_std": 252.10061645507812,
"step": 400
},
{
"KL/chosen_KL_mean": -318.316162109375,
"KL/mean": -400.8870849609375,
"KL/rejected_KL_mean": -483.4580383300781,
"KL/std": 223.79833984375,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0028180358931422234,
"fcm_dpo/delta": 0.14141802489757538,
"fcm_dpo/margin": 165.141845703125,
"fcm_dpo/q_t": 0.40394720435142517,
"grad_norm": 37.09299850463867,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.35280734300613403,
"logits/rejected": -0.3450931906700134,
"logps/chosen": -375.47918701171875,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -567.25048828125,
"loss": 1.1331,
"margin_dpo/margin_mean": 165.141845703125,
"margin_dpo/margin_std": 301.13623046875,
"step": 401
},
{
"KL/chosen_KL_mean": -243.015380859375,
"KL/mean": -388.33837890625,
"KL/rejected_KL_mean": -533.661376953125,
"KL/std": 241.9614715576172,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0027553034014999866,
"fcm_dpo/delta": -0.21828174591064453,
"fcm_dpo/margin": 290.64593505859375,
"fcm_dpo/q_t": 0.3237714469432831,
"grad_norm": 28.796403884887695,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.37429875135421753,
"logits/rejected": -0.3534529209136963,
"logps/chosen": -293.7557373046875,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -614.7073974609375,
"loss": 0.8269,
"margin_dpo/margin_mean": 290.64593505859375,
"margin_dpo/margin_std": 238.6392822265625,
"step": 402
},
{
"KL/chosen_KL_mean": -279.77032470703125,
"KL/mean": -396.9857177734375,
"KL/rejected_KL_mean": -514.2011108398438,
"KL/std": 217.40603637695312,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0027013076469302177,
"fcm_dpo/delta": -0.03580711781978607,
"fcm_dpo/margin": 234.43080139160156,
"fcm_dpo/q_t": 0.3589549958705902,
"grad_norm": 42.3283805847168,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.34183019399642944,
"logits/rejected": -0.3340870440006256,
"logps/chosen": -326.91766357421875,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -591.4677734375,
"loss": 0.9378,
"margin_dpo/margin_mean": 234.43080139160156,
"margin_dpo/margin_std": 227.52676391601562,
"step": 403
},
{
"KL/chosen_KL_mean": -298.5272521972656,
"KL/mean": -411.607666015625,
"KL/rejected_KL_mean": -524.6881103515625,
"KL/std": 217.91021728515625,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0026985001750290394,
"fcm_dpo/delta": -0.01097569614648819,
"fcm_dpo/margin": 226.1608428955078,
"fcm_dpo/q_t": 0.3690837621688843,
"grad_norm": 30.959922790527344,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.2883830666542053,
"logits/rejected": -0.2707025408744812,
"logps/chosen": -346.40252685546875,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -601.8431396484375,
"loss": 0.9955,
"margin_dpo/margin_mean": 226.1608428955078,
"margin_dpo/margin_std": 281.708251953125,
"step": 404
},
{
"KL/chosen_KL_mean": -327.94671630859375,
"KL/mean": -425.2786865234375,
"KL/rejected_KL_mean": -522.610595703125,
"KL/std": 234.07749938964844,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.0027550561353564262,
"fcm_dpo/delta": 0.0639815554022789,
"fcm_dpo/margin": 194.66395568847656,
"fcm_dpo/q_t": 0.3866780996322632,
"grad_norm": 42.94267272949219,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.3305842876434326,
"logits/rejected": -0.3149982690811157,
"logps/chosen": -393.109619140625,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -609.7974243164062,
"loss": 1.0558,
"margin_dpo/margin_mean": 194.66395568847656,
"margin_dpo/margin_std": 272.65631103515625,
"step": 405
},
{
"KL/chosen_KL_mean": -310.66717529296875,
"KL/mean": -429.66387939453125,
"KL/rejected_KL_mean": -548.66064453125,
"KL/std": 241.29891967773438,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0027310503646731377,
"fcm_dpo/delta": -0.054131537675857544,
"fcm_dpo/margin": 237.99343872070312,
"fcm_dpo/q_t": 0.360387921333313,
"grad_norm": 36.9166374206543,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.34900373220443726,
"logits/rejected": -0.3453657925128937,
"logps/chosen": -360.407958984375,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -640.7392578125,
"loss": 0.9554,
"margin_dpo/margin_mean": 237.99343872070312,
"margin_dpo/margin_std": 266.49114990234375,
"step": 406
},
{
"KL/chosen_KL_mean": -330.1507568359375,
"KL/mean": -414.1063232421875,
"KL/rejected_KL_mean": -498.0618896484375,
"KL/std": 193.3262481689453,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0027793548069894314,
"fcm_dpo/delta": 0.13783788681030273,
"fcm_dpo/margin": 167.9112091064453,
"fcm_dpo/q_t": 0.396578848361969,
"grad_norm": 41.28742980957031,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.3529721200466156,
"logits/rejected": -0.3218746483325958,
"logps/chosen": -386.4814453125,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -575.573974609375,
"loss": 1.0882,
"margin_dpo/margin_mean": 167.91119384765625,
"margin_dpo/margin_std": 249.52297973632812,
"step": 407
},
{
"KL/chosen_KL_mean": -323.1102294921875,
"KL/mean": -406.231689453125,
"KL/rejected_KL_mean": -489.35308837890625,
"KL/std": 182.276123046875,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.002833049977198243,
"fcm_dpo/delta": 0.13552114367485046,
"fcm_dpo/margin": 166.24288940429688,
"fcm_dpo/q_t": 0.39502090215682983,
"grad_norm": 36.62002182006836,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.34859204292297363,
"logits/rejected": -0.3199809193611145,
"logps/chosen": -392.8995361328125,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -579.4500732421875,
"loss": 1.0511,
"margin_dpo/margin_mean": 166.242919921875,
"margin_dpo/margin_std": 210.2808837890625,
"step": 408
},
{
"KL/chosen_KL_mean": -305.58648681640625,
"KL/mean": -398.7799377441406,
"KL/rejected_KL_mean": -491.973388671875,
"KL/std": 207.736083984375,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0028899768367409706,
"fcm_dpo/delta": 0.06457293033599854,
"fcm_dpo/margin": 186.3869171142578,
"fcm_dpo/q_t": 0.3810780942440033,
"grad_norm": 49.4791145324707,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.34838196635246277,
"logits/rejected": -0.32946842908859253,
"logps/chosen": -372.9039306640625,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -576.877685546875,
"loss": 1.0058,
"margin_dpo/margin_mean": 186.3869171142578,
"margin_dpo/margin_std": 216.09063720703125,
"step": 409
},
{
"KL/chosen_KL_mean": -294.68658447265625,
"KL/mean": -392.9432678222656,
"KL/rejected_KL_mean": -491.199951171875,
"KL/std": 199.1228790283203,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.00290653295814991,
"fcm_dpo/delta": 0.03056664764881134,
"fcm_dpo/margin": 196.51332092285156,
"fcm_dpo/q_t": 0.37201741337776184,
"grad_norm": 36.703369140625,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.38686150312423706,
"logits/rejected": -0.38626179099082947,
"logps/chosen": -346.1519775390625,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -574.39892578125,
"loss": 0.9889,
"margin_dpo/margin_mean": 196.51332092285156,
"margin_dpo/margin_std": 222.33242797851562,
"step": 410
},
{
"KL/chosen_KL_mean": -293.5391845703125,
"KL/mean": -393.26171875,
"KL/rejected_KL_mean": -492.98419189453125,
"KL/std": 214.58355712890625,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0028906632214784622,
"fcm_dpo/delta": 0.022950954735279083,
"fcm_dpo/margin": 199.44500732421875,
"fcm_dpo/q_t": 0.37562650442123413,
"grad_norm": 32.9324836730957,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.348990797996521,
"logits/rejected": -0.3469354510307312,
"logps/chosen": -345.8464660644531,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -573.67919921875,
"loss": 0.9967,
"margin_dpo/margin_mean": 199.44500732421875,
"margin_dpo/margin_std": 233.6770477294922,
"step": 411
},
{
"KL/chosen_KL_mean": -304.8736572265625,
"KL/mean": -400.49853515625,
"KL/rejected_KL_mean": -496.12347412109375,
"KL/std": 204.1537322998047,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0029426063410937786,
"fcm_dpo/delta": 0.03942097723484039,
"fcm_dpo/margin": 191.24978637695312,
"fcm_dpo/q_t": 0.37748444080352783,
"grad_norm": 45.16581344604492,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.40925562381744385,
"logits/rejected": -0.42902207374572754,
"logps/chosen": -358.0177917480469,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -596.1842651367188,
"loss": 1.0085,
"margin_dpo/margin_mean": 191.24978637695312,
"margin_dpo/margin_std": 237.30972290039062,
"step": 412
},
{
"KL/chosen_KL_mean": -310.2420654296875,
"KL/mean": -399.52105712890625,
"KL/rejected_KL_mean": -488.79998779296875,
"KL/std": 192.345947265625,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.002989421132951975,
"fcm_dpo/delta": 0.06934919953346252,
"fcm_dpo/margin": 178.55792236328125,
"fcm_dpo/q_t": 0.379652738571167,
"grad_norm": 55.78431701660156,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.3884110450744629,
"logits/rejected": -0.3837641775608063,
"logps/chosen": -371.82403564453125,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -588.2733764648438,
"loss": 1.0133,
"margin_dpo/margin_mean": 178.55792236328125,
"margin_dpo/margin_std": 211.28070068359375,
"step": 413
},
{
"KL/chosen_KL_mean": -286.4633483886719,
"KL/mean": -401.26605224609375,
"KL/rejected_KL_mean": -516.0687255859375,
"KL/std": 204.48971557617188,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.002969697816297412,
"fcm_dpo/delta": -0.08820728212594986,
"fcm_dpo/margin": 229.60543823242188,
"fcm_dpo/q_t": 0.34863734245300293,
"grad_norm": 46.16475296020508,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.33878421783447266,
"logits/rejected": -0.3258759677410126,
"logps/chosen": -333.0948486328125,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -603.7152709960938,
"loss": 0.9162,
"margin_dpo/margin_mean": 229.60543823242188,
"margin_dpo/margin_std": 222.9532012939453,
"step": 414
},
{
"KL/chosen_KL_mean": -318.5734558105469,
"KL/mean": -402.75079345703125,
"KL/rejected_KL_mean": -486.9281311035156,
"KL/std": 198.31533813476562,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.00297032343223691,
"fcm_dpo/delta": 0.10528141260147095,
"fcm_dpo/margin": 168.35467529296875,
"fcm_dpo/q_t": 0.3894081115722656,
"grad_norm": 39.87880325317383,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.4429052174091339,
"logits/rejected": -0.4342646896839142,
"logps/chosen": -397.1917419433594,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -587.4056396484375,
"loss": 1.0399,
"margin_dpo/margin_mean": 168.35467529296875,
"margin_dpo/margin_std": 213.32687377929688,
"step": 415
},
{
"KL/chosen_KL_mean": -286.0635986328125,
"KL/mean": -412.4378356933594,
"KL/rejected_KL_mean": -538.81201171875,
"KL/std": 217.12852478027344,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0029445146210491657,
"fcm_dpo/delta": -0.15555019676685333,
"fcm_dpo/margin": 252.74844360351562,
"fcm_dpo/q_t": 0.3350893557071686,
"grad_norm": 51.44747543334961,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.3993309438228607,
"logits/rejected": -0.38894549012184143,
"logps/chosen": -344.3427429199219,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -629.3807373046875,
"loss": 0.8637,
"margin_dpo/margin_mean": 252.74844360351562,
"margin_dpo/margin_std": 214.27182006835938,
"step": 416
},
{
"KL/chosen_KL_mean": -298.99530029296875,
"KL/mean": -405.01544189453125,
"KL/rejected_KL_mean": -511.03558349609375,
"KL/std": 194.6809844970703,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.002907459856942296,
"fcm_dpo/delta": -0.017707331106066704,
"fcm_dpo/margin": 212.04025268554688,
"fcm_dpo/q_t": 0.3600369691848755,
"grad_norm": 30.4539852142334,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.3451753258705139,
"logits/rejected": -0.30852943658828735,
"logps/chosen": -349.19403076171875,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -579.1873779296875,
"loss": 0.9308,
"margin_dpo/margin_mean": 212.04025268554688,
"margin_dpo/margin_std": 188.9220733642578,
"step": 417
},
{
"KL/chosen_KL_mean": -299.858154296875,
"KL/mean": -408.159423828125,
"KL/rejected_KL_mean": -516.4607543945312,
"KL/std": 224.7335205078125,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.002907500136643648,
"fcm_dpo/delta": -0.033053018152713776,
"fcm_dpo/margin": 216.60260009765625,
"fcm_dpo/q_t": 0.3656221926212311,
"grad_norm": 26.496732711791992,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.4512847661972046,
"logits/rejected": -0.4162140488624573,
"logps/chosen": -381.83660888671875,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -621.1522216796875,
"loss": 0.9756,
"margin_dpo/margin_mean": 216.6025848388672,
"margin_dpo/margin_std": 254.93975830078125,
"step": 418
},
{
"KL/chosen_KL_mean": -268.255126953125,
"KL/mean": -384.66973876953125,
"KL/rejected_KL_mean": -501.0843200683594,
"KL/std": 204.97784423828125,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.00284761656075716,
"fcm_dpo/delta": -0.06750743836164474,
"fcm_dpo/margin": 232.82919311523438,
"fcm_dpo/q_t": 0.35266152024269104,
"grad_norm": 27.982820510864258,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.3936973810195923,
"logits/rejected": -0.3798670768737793,
"logps/chosen": -321.20379638671875,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -592.6674194335938,
"loss": 0.9238,
"margin_dpo/margin_mean": 232.82919311523438,
"margin_dpo/margin_std": 228.4376220703125,
"step": 419
},
{
"KL/chosen_KL_mean": -312.17315673828125,
"KL/mean": -393.2736511230469,
"KL/rejected_KL_mean": -474.3741455078125,
"KL/std": 201.23605346679688,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.0029041050001978874,
"fcm_dpo/delta": 0.1348034292459488,
"fcm_dpo/margin": 162.2009735107422,
"fcm_dpo/q_t": 0.40072083473205566,
"grad_norm": 47.32814407348633,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.3922487497329712,
"logits/rejected": -0.3409460186958313,
"logps/chosen": -389.94305419921875,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -543.6939697265625,
"loss": 1.0988,
"margin_dpo/margin_mean": 162.20098876953125,
"margin_dpo/margin_std": 264.4722595214844,
"step": 420
},
{
"KL/chosen_KL_mean": -264.4728698730469,
"KL/mean": -347.587646484375,
"KL/rejected_KL_mean": -430.7024230957031,
"KL/std": 191.3187255859375,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0029756648000329733,
"fcm_dpo/delta": 0.11012318730354309,
"fcm_dpo/margin": 166.22955322265625,
"fcm_dpo/q_t": 0.3883700668811798,
"grad_norm": 33.364234924316406,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.42786961793899536,
"logits/rejected": -0.4104306697845459,
"logps/chosen": -318.2387390136719,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -519.98388671875,
"loss": 1.0417,
"margin_dpo/margin_mean": 166.22955322265625,
"margin_dpo/margin_std": 206.43212890625,
"step": 421
},
{
"KL/chosen_KL_mean": -297.6739501953125,
"KL/mean": -390.64801025390625,
"KL/rejected_KL_mean": -483.6220703125,
"KL/std": 201.13197326660156,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.0030023357830941677,
"fcm_dpo/delta": 0.044159695506095886,
"fcm_dpo/margin": 185.9481201171875,
"fcm_dpo/q_t": 0.37614887952804565,
"grad_norm": 32.290706634521484,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.3631212115287781,
"logits/rejected": -0.3462483882904053,
"logps/chosen": -366.3077392578125,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -571.485595703125,
"loss": 1.0118,
"margin_dpo/margin_mean": 185.9481201171875,
"margin_dpo/margin_std": 230.7765655517578,
"step": 422
},
{
"KL/chosen_KL_mean": -271.76458740234375,
"KL/mean": -372.3774719238281,
"KL/rejected_KL_mean": -472.9903564453125,
"KL/std": 189.11013793945312,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.003000294789671898,
"fcm_dpo/delta": -0.004053045064210892,
"fcm_dpo/margin": 201.22573852539062,
"fcm_dpo/q_t": 0.3659422993659973,
"grad_norm": 30.389358520507812,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.40457820892333984,
"logits/rejected": -0.40272462368011475,
"logps/chosen": -326.7548828125,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -559.296875,
"loss": 0.961,
"margin_dpo/margin_mean": 201.22573852539062,
"margin_dpo/margin_std": 212.08755493164062,
"step": 423
},
{
"KL/chosen_KL_mean": -235.85665893554688,
"KL/mean": -333.0506591796875,
"KL/rejected_KL_mean": -430.24462890625,
"KL/std": 193.54437255859375,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0029994507785886526,
"fcm_dpo/delta": 0.017647817730903625,
"fcm_dpo/margin": 194.38795471191406,
"fcm_dpo/q_t": 0.3743167519569397,
"grad_norm": 41.51752853393555,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.4083319902420044,
"logits/rejected": -0.3758869767189026,
"logps/chosen": -291.86859130859375,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -496.72357177734375,
"loss": 0.9936,
"margin_dpo/margin_mean": 194.38795471191406,
"margin_dpo/margin_std": 234.67213439941406,
"step": 424
},
{
"KL/chosen_KL_mean": -257.69268798828125,
"KL/mean": -343.0046081542969,
"KL/rejected_KL_mean": -428.3165283203125,
"KL/std": 176.45599365234375,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.003060657298192382,
"fcm_dpo/delta": 0.08170865476131439,
"fcm_dpo/margin": 170.62384033203125,
"fcm_dpo/q_t": 0.3824508786201477,
"grad_norm": 28.93231773376465,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.3647094964981079,
"logits/rejected": -0.3635455071926117,
"logps/chosen": -304.56170654296875,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -524.241943359375,
"loss": 1.0257,
"margin_dpo/margin_mean": 170.62384033203125,
"margin_dpo/margin_std": 208.47406005859375,
"step": 425
},
{
"KL/chosen_KL_mean": -236.220947265625,
"KL/mean": -339.68017578125,
"KL/rejected_KL_mean": -443.139404296875,
"KL/std": 192.19580078125,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.003069295547902584,
"fcm_dpo/delta": -0.03818836435675621,
"fcm_dpo/margin": 206.91845703125,
"fcm_dpo/q_t": 0.35965073108673096,
"grad_norm": 39.90581512451172,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.40089887380599976,
"logits/rejected": -0.3511797785758972,
"logps/chosen": -312.80450439453125,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -524.4059448242188,
"loss": 0.9459,
"margin_dpo/margin_mean": 206.91844177246094,
"margin_dpo/margin_std": 210.65582275390625,
"step": 426
},
{
"KL/chosen_KL_mean": -233.66932678222656,
"KL/mean": -309.8940124511719,
"KL/rejected_KL_mean": -386.11871337890625,
"KL/std": 164.22198486328125,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.003100508591160178,
"fcm_dpo/delta": 0.13375282287597656,
"fcm_dpo/margin": 152.44937133789062,
"fcm_dpo/q_t": 0.39551717042922974,
"grad_norm": 34.9213981628418,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.4214634299278259,
"logits/rejected": -0.38974082469940186,
"logps/chosen": -298.523193359375,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -464.6847229003906,
"loss": 1.0513,
"margin_dpo/margin_mean": 152.44937133789062,
"margin_dpo/margin_std": 195.4906005859375,
"step": 427
},
{
"KL/chosen_KL_mean": -299.35113525390625,
"KL/mean": -394.12127685546875,
"KL/rejected_KL_mean": -488.891357421875,
"KL/std": 208.70838928222656,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.0031272009946405888,
"fcm_dpo/delta": 0.007727962452918291,
"fcm_dpo/margin": 189.54025268554688,
"fcm_dpo/q_t": 0.37178969383239746,
"grad_norm": 31.63975715637207,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.42497843503952026,
"logits/rejected": -0.4162089228630066,
"logps/chosen": -361.98779296875,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -592.1732177734375,
"loss": 1.0115,
"margin_dpo/margin_mean": 189.54025268554688,
"margin_dpo/margin_std": 247.33074951171875,
"step": 428
},
{
"KL/chosen_KL_mean": -299.4518127441406,
"KL/mean": -382.56231689453125,
"KL/rejected_KL_mean": -465.6728515625,
"KL/std": 182.83213806152344,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0031900014728307724,
"fcm_dpo/delta": 0.07156050205230713,
"fcm_dpo/margin": 166.22103881835938,
"fcm_dpo/q_t": 0.3839982748031616,
"grad_norm": 39.37392044067383,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.4182380437850952,
"logits/rejected": -0.3746296763420105,
"logps/chosen": -380.6858215332031,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -557.4677734375,
"loss": 1.0476,
"margin_dpo/margin_mean": 166.22103881835938,
"margin_dpo/margin_std": 221.43417358398438,
"step": 429
},
{
"KL/chosen_KL_mean": -262.30218505859375,
"KL/mean": -348.0570983886719,
"KL/rejected_KL_mean": -433.81201171875,
"KL/std": 184.96835327148438,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.0032108055893331766,
"fcm_dpo/delta": 0.051723137497901917,
"fcm_dpo/margin": 171.50982666015625,
"fcm_dpo/q_t": 0.37903541326522827,
"grad_norm": 31.567785263061523,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.4530378580093384,
"logits/rejected": -0.44794678688049316,
"logps/chosen": -323.2225036621094,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -538.2348022460938,
"loss": 0.997,
"margin_dpo/margin_mean": 171.50982666015625,
"margin_dpo/margin_std": 195.594970703125,
"step": 430
},
{
"KL/chosen_KL_mean": -259.54559326171875,
"KL/mean": -379.1439514160156,
"KL/rejected_KL_mean": -498.7423095703125,
"KL/std": 206.00767517089844,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0031216712668538094,
"fcm_dpo/delta": -0.15936243534088135,
"fcm_dpo/margin": 239.19674682617188,
"fcm_dpo/q_t": 0.3362034559249878,
"grad_norm": 28.91539192199707,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.3503827452659607,
"logits/rejected": -0.33902430534362793,
"logps/chosen": -316.89434814453125,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -591.58251953125,
"loss": 0.8803,
"margin_dpo/margin_mean": 239.19674682617188,
"margin_dpo/margin_std": 221.66900634765625,
"step": 431
},
{
"KL/chosen_KL_mean": -259.705322265625,
"KL/mean": -368.1300048828125,
"KL/rejected_KL_mean": -476.55462646484375,
"KL/std": 199.1237335205078,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.003099852241575718,
"fcm_dpo/delta": -0.07845187932252884,
"fcm_dpo/margin": 216.84930419921875,
"fcm_dpo/q_t": 0.3529931306838989,
"grad_norm": 30.870288848876953,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.38810741901397705,
"logits/rejected": -0.36052635312080383,
"logps/chosen": -330.7801208496094,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -575.1341552734375,
"loss": 0.9185,
"margin_dpo/margin_mean": 216.84930419921875,
"margin_dpo/margin_std": 207.4125518798828,
"step": 432
},
{
"KL/chosen_KL_mean": -324.0992126464844,
"KL/mean": -403.3609313964844,
"KL/rejected_KL_mean": -482.62261962890625,
"KL/std": 194.7750244140625,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.003114804159849882,
"fcm_dpo/delta": 0.11151393502950668,
"fcm_dpo/margin": 158.52340698242188,
"fcm_dpo/q_t": 0.3896332383155823,
"grad_norm": 38.290496826171875,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.39206206798553467,
"logits/rejected": -0.3715285062789917,
"logps/chosen": -382.3724060058594,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -578.573486328125,
"loss": 1.0665,
"margin_dpo/margin_mean": 158.52340698242188,
"margin_dpo/margin_std": 225.86752319335938,
"step": 433
},
{
"KL/chosen_KL_mean": -286.78277587890625,
"KL/mean": -375.1195983886719,
"KL/rejected_KL_mean": -463.4564208984375,
"KL/std": 197.2329864501953,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0031393172685056925,
"fcm_dpo/delta": 0.04795503616333008,
"fcm_dpo/margin": 176.67364501953125,
"fcm_dpo/q_t": 0.38523241877555847,
"grad_norm": 41.98159408569336,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.38913893699645996,
"logits/rejected": -0.3690014183521271,
"logps/chosen": -348.7564697265625,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -541.9550170898438,
"loss": 1.0229,
"margin_dpo/margin_mean": 176.67364501953125,
"margin_dpo/margin_std": 245.3662872314453,
"step": 434
},
{
"KL/chosen_KL_mean": -255.27166748046875,
"KL/mean": -358.614013671875,
"KL/rejected_KL_mean": -461.95635986328125,
"KL/std": 201.6229705810547,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.003130486700683832,
"fcm_dpo/delta": -0.050365619361400604,
"fcm_dpo/margin": 206.68466186523438,
"fcm_dpo/q_t": 0.3591456711292267,
"grad_norm": 27.236400604248047,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.38453298807144165,
"logits/rejected": -0.3583604097366333,
"logps/chosen": -306.77374267578125,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -549.5232543945312,
"loss": 0.9567,
"margin_dpo/margin_mean": 206.68466186523438,
"margin_dpo/margin_std": 229.32492065429688,
"step": 435
},
{
"KL/chosen_KL_mean": -262.03448486328125,
"KL/mean": -358.820556640625,
"KL/rejected_KL_mean": -455.60662841796875,
"KL/std": 189.44845581054688,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0031167150009423494,
"fcm_dpo/delta": -0.0036548543721437454,
"fcm_dpo/margin": 193.57212829589844,
"fcm_dpo/q_t": 0.368306040763855,
"grad_norm": 38.187164306640625,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.4150596261024475,
"logits/rejected": -0.3662068843841553,
"logps/chosen": -333.4382019042969,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -538.3343505859375,
"loss": 0.9715,
"margin_dpo/margin_mean": 193.5721435546875,
"margin_dpo/margin_std": 217.64614868164062,
"step": 436
},
{
"KL/chosen_KL_mean": -263.16168212890625,
"KL/mean": -355.40069580078125,
"KL/rejected_KL_mean": -447.6397705078125,
"KL/std": 173.9517822265625,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0031388099305331707,
"fcm_dpo/delta": 0.022166123613715172,
"fcm_dpo/margin": 184.47805786132812,
"fcm_dpo/q_t": 0.3715096116065979,
"grad_norm": 27.075496673583984,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.3980615437030792,
"logits/rejected": -0.3590250015258789,
"logps/chosen": -327.90594482421875,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -529.683349609375,
"loss": 0.9796,
"margin_dpo/margin_mean": 184.4780731201172,
"margin_dpo/margin_std": 202.82333374023438,
"step": 437
},
{
"KL/chosen_KL_mean": -266.7298889160156,
"KL/mean": -373.863037109375,
"KL/rejected_KL_mean": -480.99615478515625,
"KL/std": 194.03390502929688,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0030912882648408413,
"fcm_dpo/delta": -0.06760086119174957,
"fcm_dpo/margin": 214.26629638671875,
"fcm_dpo/q_t": 0.35311073064804077,
"grad_norm": 33.12654495239258,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.3885509967803955,
"logits/rejected": -0.35134634375572205,
"logps/chosen": -325.7485656738281,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -564.072998046875,
"loss": 0.9207,
"margin_dpo/margin_mean": 214.26629638671875,
"margin_dpo/margin_std": 205.92266845703125,
"step": 438
},
{
"KL/chosen_KL_mean": -282.52447509765625,
"KL/mean": -370.0906982421875,
"KL/rejected_KL_mean": -457.6568908691406,
"KL/std": 195.64772033691406,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.0030870940536260605,
"fcm_dpo/delta": 0.06057526171207428,
"fcm_dpo/margin": 175.1324462890625,
"fcm_dpo/q_t": 0.38343846797943115,
"grad_norm": 38.511959075927734,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.3765984773635864,
"logits/rejected": -0.37041977047920227,
"logps/chosen": -336.30853271484375,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -541.642333984375,
"loss": 1.0254,
"margin_dpo/margin_mean": 175.1324462890625,
"margin_dpo/margin_std": 213.55517578125,
"step": 439
},
{
"KL/chosen_KL_mean": -296.228759765625,
"KL/mean": -383.74310302734375,
"KL/rejected_KL_mean": -471.25750732421875,
"KL/std": 223.28167724609375,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.003166210139170289,
"fcm_dpo/delta": 0.04812486469745636,
"fcm_dpo/margin": 175.02874755859375,
"fcm_dpo/q_t": 0.3827250599861145,
"grad_norm": 32.48391342163086,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.4486137330532074,
"logits/rejected": -0.42380768060684204,
"logps/chosen": -374.79547119140625,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -567.7552490234375,
"loss": 1.0288,
"margin_dpo/margin_mean": 175.02874755859375,
"margin_dpo/margin_std": 236.19110107421875,
"step": 440
},
{
"KL/chosen_KL_mean": -303.8465576171875,
"KL/mean": -406.1734619140625,
"KL/rejected_KL_mean": -508.50030517578125,
"KL/std": 226.4564208984375,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0031517865136265755,
"fcm_dpo/delta": -0.048205338418483734,
"fcm_dpo/margin": 204.6537628173828,
"fcm_dpo/q_t": 0.3663264811038971,
"grad_norm": 35.5047721862793,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.41966766119003296,
"logits/rejected": -0.400273859500885,
"logps/chosen": -364.6709899902344,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -604.9711303710938,
"loss": 0.9985,
"margin_dpo/margin_mean": 204.6537628173828,
"margin_dpo/margin_std": 272.772705078125,
"step": 441
},
{
"KL/chosen_KL_mean": -265.3717041015625,
"KL/mean": -380.8269348144531,
"KL/rejected_KL_mean": -496.28216552734375,
"KL/std": 217.8755340576172,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.0031001910101622343,
"fcm_dpo/delta": -0.12509563565254211,
"fcm_dpo/margin": 230.91046142578125,
"fcm_dpo/q_t": 0.3464425802230835,
"grad_norm": 29.765480041503906,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.42180293798446655,
"logits/rejected": -0.4058894217014313,
"logps/chosen": -312.3829345703125,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -572.8214111328125,
"loss": 0.9065,
"margin_dpo/margin_mean": 230.91046142578125,
"margin_dpo/margin_std": 230.72811889648438,
"step": 442
},
{
"KL/chosen_KL_mean": -307.4591369628906,
"KL/mean": -382.7317810058594,
"KL/rejected_KL_mean": -458.00439453125,
"KL/std": 204.9183349609375,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0030973325483500957,
"fcm_dpo/delta": 0.14015640318393707,
"fcm_dpo/margin": 150.5452423095703,
"fcm_dpo/q_t": 0.3999614119529724,
"grad_norm": 32.858116149902344,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.38298359513282776,
"logits/rejected": -0.35872089862823486,
"logps/chosen": -378.732177734375,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -544.684326171875,
"loss": 1.1158,
"margin_dpo/margin_mean": 150.5452423095703,
"margin_dpo/margin_std": 252.81396484375,
"step": 443
},
{
"KL/chosen_KL_mean": -296.9382629394531,
"KL/mean": -412.24114990234375,
"KL/rejected_KL_mean": -527.5440673828125,
"KL/std": 230.8134002685547,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0030761491507291794,
"fcm_dpo/delta": -0.1184888705611229,
"fcm_dpo/margin": 230.60581970214844,
"fcm_dpo/q_t": 0.3498353064060211,
"grad_norm": 31.28082847595215,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.3524889647960663,
"logits/rejected": -0.34426116943359375,
"logps/chosen": -354.1519775390625,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -624.7989501953125,
"loss": 0.9321,
"margin_dpo/margin_mean": 230.6058349609375,
"margin_dpo/margin_std": 260.0649719238281,
"step": 444
},
{
"KL/chosen_KL_mean": -276.9281005859375,
"KL/mean": -385.56939697265625,
"KL/rejected_KL_mean": -494.210693359375,
"KL/std": 205.611572265625,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0030374862253665924,
"fcm_dpo/delta": -0.06429732590913773,
"fcm_dpo/margin": 217.28256225585938,
"fcm_dpo/q_t": 0.35918158292770386,
"grad_norm": 34.33201217651367,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.3924905061721802,
"logits/rejected": -0.3715289533138275,
"logps/chosen": -344.2279052734375,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -586.8933715820312,
"loss": 0.9519,
"margin_dpo/margin_mean": 217.28256225585938,
"margin_dpo/margin_std": 243.815673828125,
"step": 445
},
{
"KL/chosen_KL_mean": -247.278076171875,
"KL/mean": -360.55755615234375,
"KL/rejected_KL_mean": -473.83709716796875,
"KL/std": 203.39938354492188,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.003014220856130123,
"fcm_dpo/delta": -0.08959249407052994,
"fcm_dpo/margin": 226.55899047851562,
"fcm_dpo/q_t": 0.34874945878982544,
"grad_norm": 28.63981819152832,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.3813268542289734,
"logits/rejected": -0.3821564316749573,
"logps/chosen": -306.3765563964844,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -575.1012573242188,
"loss": 0.9037,
"margin_dpo/margin_mean": 226.55897521972656,
"margin_dpo/margin_std": 203.93728637695312,
"step": 446
},
{
"KL/chosen_KL_mean": -273.1231384277344,
"KL/mean": -389.95855712890625,
"KL/rejected_KL_mean": -506.7939453125,
"KL/std": 236.97166442871094,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0029223784804344177,
"fcm_dpo/delta": -0.08987779170274734,
"fcm_dpo/margin": 233.67080688476562,
"fcm_dpo/q_t": 0.3536572754383087,
"grad_norm": 31.68606185913086,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.4216350317001343,
"logits/rejected": -0.4224305748939514,
"logps/chosen": -329.198486328125,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -605.4887084960938,
"loss": 0.923,
"margin_dpo/margin_mean": 233.67083740234375,
"margin_dpo/margin_std": 250.795654296875,
"step": 447
},
{
"KL/chosen_KL_mean": -298.9906005859375,
"KL/mean": -421.9812927246094,
"KL/rejected_KL_mean": -544.9719848632812,
"KL/std": 217.21249389648438,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.002884766086935997,
"fcm_dpo/delta": -0.11784309148788452,
"fcm_dpo/margin": 245.98141479492188,
"fcm_dpo/q_t": 0.3457278907299042,
"grad_norm": 45.153812408447266,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.3462919592857361,
"logits/rejected": -0.34202855825424194,
"logps/chosen": -358.99444580078125,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -647.2366333007812,
"loss": 0.8976,
"margin_dpo/margin_mean": 245.98141479492188,
"margin_dpo/margin_std": 235.63455200195312,
"step": 448
},
{
"KL/chosen_KL_mean": -307.90765380859375,
"KL/mean": -421.1962890625,
"KL/rejected_KL_mean": -534.4849243164062,
"KL/std": 244.52383422851562,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.0028343484736979008,
"fcm_dpo/delta": -0.04512559995055199,
"fcm_dpo/margin": 226.5773162841797,
"fcm_dpo/q_t": 0.3684704899787903,
"grad_norm": 30.175477981567383,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.3602468967437744,
"logits/rejected": -0.33677172660827637,
"logps/chosen": -375.43426513671875,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -623.0818481445312,
"loss": 1.0,
"margin_dpo/margin_mean": 226.57730102539062,
"margin_dpo/margin_std": 310.042236328125,
"step": 449
},
{
"KL/chosen_KL_mean": -249.35226440429688,
"KL/mean": -375.23370361328125,
"KL/rejected_KL_mean": -501.11517333984375,
"KL/std": 231.91375732421875,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.0027888580225408077,
"fcm_dpo/delta": -0.10971163213253021,
"fcm_dpo/margin": 251.76284790039062,
"fcm_dpo/q_t": 0.34219545125961304,
"grad_norm": 33.89752197265625,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.3522266447544098,
"logits/rejected": -0.3474314212799072,
"logps/chosen": -306.46038818359375,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -603.8701171875,
"loss": 0.8922,
"margin_dpo/margin_mean": 251.76287841796875,
"margin_dpo/margin_std": 229.1561279296875,
"step": 450
},
{
"KL/chosen_KL_mean": -311.85552978515625,
"KL/mean": -411.384033203125,
"KL/rejected_KL_mean": -510.9126281738281,
"KL/std": 235.77354431152344,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0027988762594759464,
"fcm_dpo/delta": 0.04477785527706146,
"fcm_dpo/margin": 199.05712890625,
"fcm_dpo/q_t": 0.3746575713157654,
"grad_norm": 38.93794631958008,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.33926278352737427,
"logits/rejected": -0.30115002393722534,
"logps/chosen": -370.3243408203125,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -583.842041015625,
"loss": 1.0282,
"margin_dpo/margin_mean": 199.05711364746094,
"margin_dpo/margin_std": 264.158203125,
"step": 451
},
{
"KL/chosen_KL_mean": -231.11196899414062,
"KL/mean": -344.82904052734375,
"KL/rejected_KL_mean": -458.546142578125,
"KL/std": 210.43255615234375,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0027735186740756035,
"fcm_dpo/delta": -0.03301696851849556,
"fcm_dpo/margin": 227.43417358398438,
"fcm_dpo/q_t": 0.3585592806339264,
"grad_norm": 26.01671028137207,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.32991310954093933,
"logits/rejected": -0.3405373692512512,
"logps/chosen": -281.50250244140625,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -556.3175659179688,
"loss": 0.9312,
"margin_dpo/margin_mean": 227.4341583251953,
"margin_dpo/margin_std": 215.60528564453125,
"step": 452
},
{
"KL/chosen_KL_mean": -288.67913818359375,
"KL/mean": -388.9347229003906,
"KL/rejected_KL_mean": -489.1903076171875,
"KL/std": 207.9075469970703,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.002779030241072178,
"fcm_dpo/delta": 0.045001037418842316,
"fcm_dpo/margin": 200.51113891601562,
"fcm_dpo/q_t": 0.3803292214870453,
"grad_norm": 37.94629669189453,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.3870271146297455,
"logits/rejected": -0.3518151044845581,
"logps/chosen": -346.3940124511719,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -571.397705078125,
"loss": 1.0059,
"margin_dpo/margin_mean": 200.51112365722656,
"margin_dpo/margin_std": 247.18231201171875,
"step": 453
},
{
"KL/chosen_KL_mean": -326.3154296875,
"KL/mean": -445.6247253417969,
"KL/rejected_KL_mean": -564.9340209960938,
"KL/std": 241.54116821289062,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.002762860618531704,
"fcm_dpo/delta": -0.0639684721827507,
"fcm_dpo/margin": 238.61856079101562,
"fcm_dpo/q_t": 0.35886693000793457,
"grad_norm": 35.39641189575195,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.3822100758552551,
"logits/rejected": -0.35830071568489075,
"logps/chosen": -387.2611083984375,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -649.884765625,
"loss": 0.948,
"margin_dpo/margin_mean": 238.61856079101562,
"margin_dpo/margin_std": 272.42669677734375,
"step": 454
},
{
"KL/chosen_KL_mean": -301.6221618652344,
"KL/mean": -443.32330322265625,
"KL/rejected_KL_mean": -585.0244750976562,
"KL/std": 262.75738525390625,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0027075689285993576,
"fcm_dpo/delta": -0.18090659379959106,
"fcm_dpo/margin": 283.4023132324219,
"fcm_dpo/q_t": 0.3393672704696655,
"grad_norm": 32.6513671875,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.4092771112918854,
"logits/rejected": -0.4132944941520691,
"logps/chosen": -346.50885009765625,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -700.325927734375,
"loss": 0.8932,
"margin_dpo/margin_mean": 283.4023132324219,
"margin_dpo/margin_std": 293.6563415527344,
"step": 455
},
{
"KL/chosen_KL_mean": -309.55914306640625,
"KL/mean": -443.7057800292969,
"KL/rejected_KL_mean": -577.8524169921875,
"KL/std": 245.8824462890625,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0026249473448842764,
"fcm_dpo/delta": -0.1120506078004837,
"fcm_dpo/margin": 268.29327392578125,
"fcm_dpo/q_t": 0.3443225622177124,
"grad_norm": 25.41524887084961,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.40219759941101074,
"logits/rejected": -0.4123704731464386,
"logps/chosen": -366.59588623046875,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -683.0701904296875,
"loss": 0.8851,
"margin_dpo/margin_mean": 268.29327392578125,
"margin_dpo/margin_std": 238.1885528564453,
"step": 456
},
{
"KL/chosen_KL_mean": -307.5630798339844,
"KL/mean": -438.154296875,
"KL/rejected_KL_mean": -568.7454833984375,
"KL/std": 242.14056396484375,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.002568621654063463,
"fcm_dpo/delta": -0.0764625295996666,
"fcm_dpo/margin": 261.1824645996094,
"fcm_dpo/q_t": 0.3490523099899292,
"grad_norm": 31.589265823364258,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.40978848934173584,
"logits/rejected": -0.38595595955848694,
"logps/chosen": -361.80560302734375,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -653.8551025390625,
"loss": 0.9077,
"margin_dpo/margin_mean": 261.1824645996094,
"margin_dpo/margin_std": 236.63363647460938,
"step": 457
},
{
"KL/chosen_KL_mean": -310.71527099609375,
"KL/mean": -438.909912109375,
"KL/rejected_KL_mean": -567.1046142578125,
"KL/std": 231.90826416015625,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.0025398745201528072,
"fcm_dpo/delta": -0.055026132613420486,
"fcm_dpo/margin": 256.3893127441406,
"fcm_dpo/q_t": 0.35763344168663025,
"grad_norm": 27.47776222229004,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.40042775869369507,
"logits/rejected": -0.38572534918785095,
"logps/chosen": -366.1241455078125,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -664.787841796875,
"loss": 0.9325,
"margin_dpo/margin_mean": 256.3892822265625,
"margin_dpo/margin_std": 266.773681640625,
"step": 458
},
{
"KL/chosen_KL_mean": -338.8822937011719,
"KL/mean": -459.091796875,
"KL/rejected_KL_mean": -579.30126953125,
"KL/std": 242.7357177734375,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.002527649514377117,
"fcm_dpo/delta": -0.008432786911725998,
"fcm_dpo/margin": 240.41900634765625,
"fcm_dpo/q_t": 0.36714914441108704,
"grad_norm": 33.14191818237305,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.40105724334716797,
"logits/rejected": -0.3996381163597107,
"logps/chosen": -385.43975830078125,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -665.4698486328125,
"loss": 0.974,
"margin_dpo/margin_mean": 240.41900634765625,
"margin_dpo/margin_std": 268.98193359375,
"step": 459
},
{
"KL/chosen_KL_mean": -375.49517822265625,
"KL/mean": -500.5682373046875,
"KL/rejected_KL_mean": -625.641357421875,
"KL/std": 234.71273803710938,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0025242562405765057,
"fcm_dpo/delta": -0.03353795409202576,
"fcm_dpo/margin": 250.14614868164062,
"fcm_dpo/q_t": 0.361194372177124,
"grad_norm": 46.20624923706055,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.46263211965560913,
"logits/rejected": -0.4771049916744232,
"logps/chosen": -427.13006591796875,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -729.7606811523438,
"loss": 0.9511,
"margin_dpo/margin_mean": 250.14613342285156,
"margin_dpo/margin_std": 267.33648681640625,
"step": 460
},
{
"KL/chosen_KL_mean": -375.7967529296875,
"KL/mean": -478.203125,
"KL/rejected_KL_mean": -580.6094970703125,
"KL/std": 232.16815185546875,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.002555173821747303,
"fcm_dpo/delta": 0.08051595091819763,
"fcm_dpo/margin": 204.81271362304688,
"fcm_dpo/q_t": 0.38595494627952576,
"grad_norm": 36.09431076049805,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.4403180480003357,
"logits/rejected": -0.43697261810302734,
"logps/chosen": -430.9787292480469,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -667.08642578125,
"loss": 1.0262,
"margin_dpo/margin_mean": 204.81271362304688,
"margin_dpo/margin_std": 259.2124328613281,
"step": 461
},
{
"KL/chosen_KL_mean": -372.1178894042969,
"KL/mean": -458.4106140136719,
"KL/rejected_KL_mean": -544.7033081054688,
"KL/std": 219.6261749267578,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.002613114658743143,
"fcm_dpo/delta": 0.15629872679710388,
"fcm_dpo/margin": 172.58546447753906,
"fcm_dpo/q_t": 0.40055692195892334,
"grad_norm": 46.14814758300781,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.4446120262145996,
"logits/rejected": -0.4250994324684143,
"logps/chosen": -442.0459289550781,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -623.54443359375,
"loss": 1.0714,
"margin_dpo/margin_mean": 172.58544921875,
"margin_dpo/margin_std": 233.74554443359375,
"step": 462
},
{
"KL/chosen_KL_mean": -352.1114807128906,
"KL/mean": -464.76641845703125,
"KL/rejected_KL_mean": -577.42138671875,
"KL/std": 243.61065673828125,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.0026555135846138,
"fcm_dpo/delta": 0.001107487827539444,
"fcm_dpo/margin": 225.30990600585938,
"fcm_dpo/q_t": 0.37362590432167053,
"grad_norm": 50.18327331542969,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.46163278818130493,
"logits/rejected": -0.4520809054374695,
"logps/chosen": -407.3858642578125,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -666.4463500976562,
"loss": 1.006,
"margin_dpo/margin_mean": 225.30990600585938,
"margin_dpo/margin_std": 290.9793701171875,
"step": 463
},
{
"KL/chosen_KL_mean": -330.8517761230469,
"KL/mean": -491.7892150878906,
"KL/rejected_KL_mean": -652.7266235351562,
"KL/std": 277.88262939453125,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0025325994938611984,
"fcm_dpo/delta": -0.23679596185684204,
"fcm_dpo/margin": 321.87481689453125,
"fcm_dpo/q_t": 0.32468390464782715,
"grad_norm": 39.49245834350586,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.4352476894855499,
"logits/rejected": -0.46179062128067017,
"logps/chosen": -381.76409912109375,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -755.2160034179688,
"loss": 0.8473,
"margin_dpo/margin_mean": 321.87481689453125,
"margin_dpo/margin_std": 299.6499328613281,
"step": 464
},
{
"KL/chosen_KL_mean": -314.51800537109375,
"KL/mean": -458.2316589355469,
"KL/rejected_KL_mean": -601.9453125,
"KL/std": 265.89178466796875,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.002483130432665348,
"fcm_dpo/delta": -0.12230255454778671,
"fcm_dpo/margin": 287.42724609375,
"fcm_dpo/q_t": 0.34188520908355713,
"grad_norm": 31.928207397460938,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.4381392002105713,
"logits/rejected": -0.4378528296947479,
"logps/chosen": -374.6348571777344,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -715.891357421875,
"loss": 0.8887,
"margin_dpo/margin_mean": 287.4272766113281,
"margin_dpo/margin_std": 263.517822265625,
"step": 465
},
{
"KL/chosen_KL_mean": -337.96856689453125,
"KL/mean": -450.06103515625,
"KL/rejected_KL_mean": -562.1534423828125,
"KL/std": 233.87637329101562,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.002464518416672945,
"fcm_dpo/delta": 0.05006605386734009,
"fcm_dpo/margin": 224.18490600585938,
"fcm_dpo/q_t": 0.3764589726924896,
"grad_norm": 32.9969482421875,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.43533119559288025,
"logits/rejected": -0.42451655864715576,
"logps/chosen": -390.8894958496094,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -652.4688720703125,
"loss": 1.0007,
"margin_dpo/margin_mean": 224.18490600585938,
"margin_dpo/margin_std": 256.4300842285156,
"step": 466
},
{
"KL/chosen_KL_mean": -396.30499267578125,
"KL/mean": -520.2387084960938,
"KL/rejected_KL_mean": -644.1724853515625,
"KL/std": 268.977294921875,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.002454932779073715,
"fcm_dpo/delta": -0.01099260151386261,
"fcm_dpo/margin": 247.86749267578125,
"fcm_dpo/q_t": 0.37103724479675293,
"grad_norm": 38.6761360168457,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.5167651176452637,
"logits/rejected": -0.5001472234725952,
"logps/chosen": -475.02081298828125,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -747.03271484375,
"loss": 1.0093,
"margin_dpo/margin_mean": 247.86749267578125,
"margin_dpo/margin_std": 319.3533020019531,
"step": 467
},
{
"KL/chosen_KL_mean": -301.0346984863281,
"KL/mean": -432.91510009765625,
"KL/rejected_KL_mean": -564.7955322265625,
"KL/std": 249.4041290283203,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0024472326040267944,
"fcm_dpo/delta": -0.049302905797958374,
"fcm_dpo/margin": 263.7608642578125,
"fcm_dpo/q_t": 0.35723966360092163,
"grad_norm": 31.65911102294922,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.4635354280471802,
"logits/rejected": -0.44282078742980957,
"logps/chosen": -370.9700622558594,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -665.8243408203125,
"loss": 0.9319,
"margin_dpo/margin_mean": 263.7608642578125,
"margin_dpo/margin_std": 261.97076416015625,
"step": 468
},
{
"KL/chosen_KL_mean": -349.6076965332031,
"KL/mean": -485.7550964355469,
"KL/rejected_KL_mean": -621.9024658203125,
"KL/std": 244.61253356933594,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.002414276357740164,
"fcm_dpo/delta": -0.06267710030078888,
"fcm_dpo/margin": 272.2947998046875,
"fcm_dpo/q_t": 0.3552666902542114,
"grad_norm": 31.369281768798828,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.48086249828338623,
"logits/rejected": -0.45679014921188354,
"logps/chosen": -417.7323913574219,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -726.6889038085938,
"loss": 0.9262,
"margin_dpo/margin_mean": 272.2947998046875,
"margin_dpo/margin_std": 267.436767578125,
"step": 469
},
{
"KL/chosen_KL_mean": -272.64764404296875,
"KL/mean": -404.01336669921875,
"KL/rejected_KL_mean": -535.379150390625,
"KL/std": 244.73040771484375,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0024104006588459015,
"fcm_dpo/delta": -0.035525791347026825,
"fcm_dpo/margin": 262.7314453125,
"fcm_dpo/q_t": 0.3573177754878998,
"grad_norm": 26.009780883789062,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.41702497005462646,
"logits/rejected": -0.4054919481277466,
"logps/chosen": -316.4395751953125,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -618.0819702148438,
"loss": 0.9396,
"margin_dpo/margin_mean": 262.7314453125,
"margin_dpo/margin_std": 261.20184326171875,
"step": 470
},
{
"KL/chosen_KL_mean": -321.0496826171875,
"KL/mean": -434.74432373046875,
"KL/rejected_KL_mean": -548.4390258789062,
"KL/std": 247.41632080078125,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.002411898225545883,
"fcm_dpo/delta": 0.05429071933031082,
"fcm_dpo/margin": 227.38934326171875,
"fcm_dpo/q_t": 0.37768125534057617,
"grad_norm": 27.615331649780273,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.4710885286331177,
"logits/rejected": -0.4560639262199402,
"logps/chosen": -384.38922119140625,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -632.0494995117188,
"loss": 0.9908,
"margin_dpo/margin_mean": 227.3893280029297,
"margin_dpo/margin_std": 244.27651977539062,
"step": 471
},
{
"KL/chosen_KL_mean": -326.57818603515625,
"KL/mean": -463.6128234863281,
"KL/rejected_KL_mean": -600.6474609375,
"KL/std": 302.3685302734375,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.002408525673672557,
"fcm_dpo/delta": -0.06433549523353577,
"fcm_dpo/margin": 274.06927490234375,
"fcm_dpo/q_t": 0.3624575734138489,
"grad_norm": 29.133203506469727,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.5421698093414307,
"logits/rejected": -0.5046987533569336,
"logps/chosen": -410.2442626953125,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -717.856689453125,
"loss": 0.956,
"margin_dpo/margin_mean": 274.06927490234375,
"margin_dpo/margin_std": 325.52716064453125,
"step": 472
},
{
"KL/chosen_KL_mean": -359.8109130859375,
"KL/mean": -454.48809814453125,
"KL/rejected_KL_mean": -549.1652221679688,
"KL/std": 257.3474426269531,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0024484009481966496,
"fcm_dpo/delta": 0.14316929876804352,
"fcm_dpo/margin": 189.35433959960938,
"fcm_dpo/q_t": 0.40205615758895874,
"grad_norm": 42.2054557800293,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.43200650811195374,
"logits/rejected": -0.40144896507263184,
"logps/chosen": -423.307861328125,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -630.311767578125,
"loss": 1.1002,
"margin_dpo/margin_mean": 189.3543243408203,
"margin_dpo/margin_std": 315.0071105957031,
"step": 473
},
{
"KL/chosen_KL_mean": -312.0838623046875,
"KL/mean": -415.65679931640625,
"KL/rejected_KL_mean": -519.2298583984375,
"KL/std": 238.22402954101562,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.002514604479074478,
"fcm_dpo/delta": 0.08228041976690292,
"fcm_dpo/margin": 207.14599609375,
"fcm_dpo/q_t": 0.38554972410202026,
"grad_norm": 36.657867431640625,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.49141281843185425,
"logits/rejected": -0.4804675579071045,
"logps/chosen": -364.69580078125,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -609.3102416992188,
"loss": 1.0459,
"margin_dpo/margin_mean": 207.14599609375,
"margin_dpo/margin_std": 280.27203369140625,
"step": 474
},
{
"KL/chosen_KL_mean": -251.16018676757812,
"KL/mean": -391.962646484375,
"KL/rejected_KL_mean": -532.76513671875,
"KL/std": 241.2380828857422,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.002469835337251425,
"fcm_dpo/delta": -0.1028013825416565,
"fcm_dpo/margin": 281.60498046875,
"fcm_dpo/q_t": 0.3463265895843506,
"grad_norm": 26.425996780395508,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.4599395990371704,
"logits/rejected": -0.4634561538696289,
"logps/chosen": -293.6553955078125,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -622.828125,
"loss": 0.897,
"margin_dpo/margin_mean": 281.60498046875,
"margin_dpo/margin_std": 263.2021484375,
"step": 475
},
{
"KL/chosen_KL_mean": -274.4346923828125,
"KL/mean": -389.6119384765625,
"KL/rejected_KL_mean": -504.7891845703125,
"KL/std": 222.2323760986328,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.0024726144038140774,
"fcm_dpo/delta": 0.03223652392625809,
"fcm_dpo/margin": 230.35450744628906,
"fcm_dpo/q_t": 0.3728393316268921,
"grad_norm": 33.15345764160156,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.4732276201248169,
"logits/rejected": -0.46229255199432373,
"logps/chosen": -317.3840637207031,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -578.4993896484375,
"loss": 0.9757,
"margin_dpo/margin_mean": 230.35450744628906,
"margin_dpo/margin_std": 242.6463623046875,
"step": 476
},
{
"KL/chosen_KL_mean": -305.323974609375,
"KL/mean": -411.8943176269531,
"KL/rejected_KL_mean": -518.4646606445312,
"KL/std": 232.3471221923828,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0025053019635379314,
"fcm_dpo/delta": 0.06956921517848969,
"fcm_dpo/margin": 213.14068603515625,
"fcm_dpo/q_t": 0.38124802708625793,
"grad_norm": 29.20063018798828,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.5332554578781128,
"logits/rejected": -0.4827369749546051,
"logps/chosen": -376.0965881347656,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -594.60205078125,
"loss": 1.0239,
"margin_dpo/margin_mean": 213.14068603515625,
"margin_dpo/margin_std": 269.6601257324219,
"step": 477
},
{
"KL/chosen_KL_mean": -252.85372924804688,
"KL/mean": -384.605224609375,
"KL/rejected_KL_mean": -516.356689453125,
"KL/std": 223.99530029296875,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.002497827634215355,
"fcm_dpo/delta": -0.06240279972553253,
"fcm_dpo/margin": 263.5030517578125,
"fcm_dpo/q_t": 0.35455381870269775,
"grad_norm": 28.433320999145508,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.44590333104133606,
"logits/rejected": -0.44256502389907837,
"logps/chosen": -294.29425048828125,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -601.7186889648438,
"loss": 0.9139,
"margin_dpo/margin_mean": 263.5030212402344,
"margin_dpo/margin_std": 242.99679565429688,
"step": 478
},
{
"KL/chosen_KL_mean": -343.3636169433594,
"KL/mean": -451.3651428222656,
"KL/rejected_KL_mean": -559.3666381835938,
"KL/std": 231.16734313964844,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.0025271642953157425,
"fcm_dpo/delta": 0.054066017270088196,
"fcm_dpo/margin": 216.00302124023438,
"fcm_dpo/q_t": 0.3788665533065796,
"grad_norm": 43.08273696899414,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.4322773218154907,
"logits/rejected": -0.43192189931869507,
"logps/chosen": -397.27154541015625,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -654.4829711914062,
"loss": 1.0206,
"margin_dpo/margin_mean": 216.0030059814453,
"margin_dpo/margin_std": 259.35797119140625,
"step": 479
},
{
"KL/chosen_KL_mean": -366.31488037109375,
"KL/mean": -458.1357421875,
"KL/rejected_KL_mean": -549.9566040039062,
"KL/std": 225.42266845703125,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.00257721240632236,
"fcm_dpo/delta": 0.13034236431121826,
"fcm_dpo/margin": 183.6417236328125,
"fcm_dpo/q_t": 0.39666956663131714,
"grad_norm": 46.63106155395508,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.4494960606098175,
"logits/rejected": -0.41708821058273315,
"logps/chosen": -424.9975891113281,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -632.8890991210938,
"loss": 1.0817,
"margin_dpo/margin_mean": 183.64173889160156,
"margin_dpo/margin_std": 264.16253662109375,
"step": 480
},
{
"KL/chosen_KL_mean": -304.6341552734375,
"KL/mean": -430.632568359375,
"KL/rejected_KL_mean": -556.6309814453125,
"KL/std": 250.36875915527344,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.0025500431656837463,
"fcm_dpo/delta": -0.04568130895495415,
"fcm_dpo/margin": 251.99685668945312,
"fcm_dpo/q_t": 0.35894322395324707,
"grad_norm": 33.32291793823242,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.4825635552406311,
"logits/rejected": -0.47148293256759644,
"logps/chosen": -359.59844970703125,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -649.0513916015625,
"loss": 0.9448,
"margin_dpo/margin_mean": 251.99684143066406,
"margin_dpo/margin_std": 268.0703430175781,
"step": 481
},
{
"KL/chosen_KL_mean": -344.23883056640625,
"KL/mean": -428.9852600097656,
"KL/rejected_KL_mean": -513.731689453125,
"KL/std": 227.03057861328125,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0026108575984835625,
"fcm_dpo/delta": 0.1648440957069397,
"fcm_dpo/margin": 169.49282836914062,
"fcm_dpo/q_t": 0.4027259945869446,
"grad_norm": 44.0857048034668,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.48988986015319824,
"logits/rejected": -0.4721633791923523,
"logps/chosen": -411.79229736328125,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -601.3212280273438,
"loss": 1.0977,
"margin_dpo/margin_mean": 169.4928436279297,
"margin_dpo/margin_std": 263.2955627441406,
"step": 482
},
{
"KL/chosen_KL_mean": -308.8171691894531,
"KL/mean": -452.93719482421875,
"KL/rejected_KL_mean": -597.0572509765625,
"KL/std": 259.18682861328125,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0025678810197860003,
"fcm_dpo/delta": -0.15200494229793549,
"fcm_dpo/margin": 288.2401123046875,
"fcm_dpo/q_t": 0.34249088168144226,
"grad_norm": 34.45748519897461,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.4860680103302002,
"logits/rejected": -0.4675144553184509,
"logps/chosen": -372.14697265625,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -692.84423828125,
"loss": 0.8947,
"margin_dpo/margin_mean": 288.2401123046875,
"margin_dpo/margin_std": 294.35589599609375,
"step": 483
},
{
"KL/chosen_KL_mean": -277.099609375,
"KL/mean": -404.1110534667969,
"KL/rejected_KL_mean": -531.1224975585938,
"KL/std": 245.48541259765625,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0025537596084177494,
"fcm_dpo/delta": -0.052785273641347885,
"fcm_dpo/margin": 254.02294921875,
"fcm_dpo/q_t": 0.35719069838523865,
"grad_norm": 28.33159828186035,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.47132617235183716,
"logits/rejected": -0.454983651638031,
"logps/chosen": -336.23773193359375,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -615.4939575195312,
"loss": 0.9269,
"margin_dpo/margin_mean": 254.02294921875,
"margin_dpo/margin_std": 243.93063354492188,
"step": 484
},
{
"KL/chosen_KL_mean": -277.1954345703125,
"KL/mean": -397.7764892578125,
"KL/rejected_KL_mean": -518.3575439453125,
"KL/std": 239.49002075195312,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0025283764116466045,
"fcm_dpo/delta": -0.010414174757897854,
"fcm_dpo/margin": 241.1621551513672,
"fcm_dpo/q_t": 0.3665305972099304,
"grad_norm": 31.031030654907227,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.4665423631668091,
"logits/rejected": -0.4636436104774475,
"logps/chosen": -336.04498291015625,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -621.7216796875,
"loss": 0.9684,
"margin_dpo/margin_mean": 241.16217041015625,
"margin_dpo/margin_std": 270.1797790527344,
"step": 485
},
{
"KL/chosen_KL_mean": -317.493408203125,
"KL/mean": -441.400634765625,
"KL/rejected_KL_mean": -565.307861328125,
"KL/std": 242.16842651367188,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.002518579363822937,
"fcm_dpo/delta": -0.025778962299227715,
"fcm_dpo/margin": 247.81446838378906,
"fcm_dpo/q_t": 0.36535248160362244,
"grad_norm": 46.50453186035156,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -0.4674904942512512,
"logits/rejected": -0.45089191198349,
"logps/chosen": -372.7530517578125,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -657.4472045898438,
"loss": 0.9862,
"margin_dpo/margin_mean": 247.81446838378906,
"margin_dpo/margin_std": 304.4325866699219,
"step": 486
},
{
"KL/chosen_KL_mean": -293.8622741699219,
"KL/mean": -404.47393798828125,
"KL/rejected_KL_mean": -515.0855712890625,
"KL/std": 202.39385986328125,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.002519285771995783,
"fcm_dpo/delta": 0.04513642191886902,
"fcm_dpo/margin": 221.22329711914062,
"fcm_dpo/q_t": 0.3729146122932434,
"grad_norm": 34.71440887451172,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -0.47890907526016235,
"logits/rejected": -0.46555450558662415,
"logps/chosen": -346.92559814453125,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -607.50439453125,
"loss": 0.9782,
"margin_dpo/margin_mean": 221.22329711914062,
"margin_dpo/margin_std": 222.7053680419922,
"step": 487
},
{
"KL/chosen_KL_mean": -245.27224731445312,
"KL/mean": -361.5653991699219,
"KL/rejected_KL_mean": -477.85858154296875,
"KL/std": 222.239990234375,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.002545600291341543,
"fcm_dpo/delta": 0.008101914077997208,
"fcm_dpo/margin": 232.5863037109375,
"fcm_dpo/q_t": 0.36505213379859924,
"grad_norm": 33.56699752807617,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.4596076011657715,
"logits/rejected": -0.44325512647628784,
"logps/chosen": -297.50042724609375,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -561.8651123046875,
"loss": 0.9426,
"margin_dpo/margin_mean": 232.5863037109375,
"margin_dpo/margin_std": 205.98390197753906,
"step": 488
},
{
"KL/chosen_KL_mean": -248.72581481933594,
"KL/mean": -353.61651611328125,
"KL/rejected_KL_mean": -458.5072326660156,
"KL/std": 214.0831756591797,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0025621079839766026,
"fcm_dpo/delta": 0.06606802344322205,
"fcm_dpo/margin": 209.7814178466797,
"fcm_dpo/q_t": 0.38119935989379883,
"grad_norm": 34.845924377441406,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.504858136177063,
"logits/rejected": -0.4839511215686798,
"logps/chosen": -304.7154541015625,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -537.9053955078125,
"loss": 0.9932,
"margin_dpo/margin_mean": 209.7814178466797,
"margin_dpo/margin_std": 227.97195434570312,
"step": 489
},
{
"KL/chosen_KL_mean": -266.3229675292969,
"KL/mean": -383.3382568359375,
"KL/rejected_KL_mean": -500.35357666015625,
"KL/std": 253.58834838867188,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.002577013336122036,
"fcm_dpo/delta": -0.003465762361884117,
"fcm_dpo/margin": 234.0305938720703,
"fcm_dpo/q_t": 0.3708733320236206,
"grad_norm": 42.293663024902344,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.50019770860672,
"logits/rejected": -0.5143351554870605,
"logps/chosen": -318.6893615722656,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -610.7626342773438,
"loss": 0.993,
"margin_dpo/margin_mean": 234.03060913085938,
"margin_dpo/margin_std": 290.219970703125,
"step": 490
},
{
"KL/chosen_KL_mean": -288.1597900390625,
"KL/mean": -374.07470703125,
"KL/rejected_KL_mean": -459.9895935058594,
"KL/std": 217.8290252685547,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.00262988219037652,
"fcm_dpo/delta": 0.15506739914417267,
"fcm_dpo/margin": 171.82980346679688,
"fcm_dpo/q_t": 0.4017429053783417,
"grad_norm": 37.90342712402344,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.4776982069015503,
"logits/rejected": -0.46193790435791016,
"logps/chosen": -348.27606201171875,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -533.2623901367188,
"loss": 1.083,
"margin_dpo/margin_mean": 171.82980346679688,
"margin_dpo/margin_std": 252.08087158203125,
"step": 491
},
{
"KL/chosen_KL_mean": -295.93548583984375,
"KL/mean": -366.8896179199219,
"KL/rejected_KL_mean": -437.84375,
"KL/std": 210.89358520507812,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0027370997704565525,
"fcm_dpo/delta": 0.2202422320842743,
"fcm_dpo/margin": 141.90826416015625,
"fcm_dpo/q_t": 0.41631877422332764,
"grad_norm": 39.937416076660156,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -0.48065948486328125,
"logits/rejected": -0.4587652087211609,
"logps/chosen": -349.9296569824219,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -510.50335693359375,
"loss": 1.1551,
"margin_dpo/margin_mean": 141.90826416015625,
"margin_dpo/margin_std": 264.263671875,
"step": 492
},
{
"KL/chosen_KL_mean": -300.294189453125,
"KL/mean": -384.89044189453125,
"KL/rejected_KL_mean": -469.4866638183594,
"KL/std": 208.40113830566406,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0028138277120888233,
"fcm_dpo/delta": 0.13032102584838867,
"fcm_dpo/margin": 169.19253540039062,
"fcm_dpo/q_t": 0.39434120059013367,
"grad_norm": 42.05187225341797,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -0.5109956860542297,
"logits/rejected": -0.47627633810043335,
"logps/chosen": -375.7914123535156,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -556.8096923828125,
"loss": 1.0487,
"margin_dpo/margin_mean": 169.19253540039062,
"margin_dpo/margin_std": 213.05795288085938,
"step": 493
},
{
"KL/chosen_KL_mean": -242.52565002441406,
"KL/mean": -370.0452880859375,
"KL/rejected_KL_mean": -497.5649108886719,
"KL/std": 219.48666381835938,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.0027876575477421284,
"fcm_dpo/delta": -0.11972147226333618,
"fcm_dpo/margin": 255.0392608642578,
"fcm_dpo/q_t": 0.3406761884689331,
"grad_norm": 35.56473159790039,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.41800960898399353,
"logits/rejected": -0.430025577545166,
"logps/chosen": -283.8849182128906,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -583.65625,
"loss": 0.8836,
"margin_dpo/margin_mean": 255.03924560546875,
"margin_dpo/margin_std": 221.43124389648438,
"step": 494
},
{
"KL/chosen_KL_mean": -277.42822265625,
"KL/mean": -397.37249755859375,
"KL/rejected_KL_mean": -517.3167724609375,
"KL/std": 218.87979125976562,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.002760577481240034,
"fcm_dpo/delta": -0.06677626073360443,
"fcm_dpo/margin": 239.88856506347656,
"fcm_dpo/q_t": 0.35618913173675537,
"grad_norm": 33.82542037963867,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -0.47961604595184326,
"logits/rejected": -0.47053205966949463,
"logps/chosen": -340.96331787109375,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -608.7412109375,
"loss": 0.9378,
"margin_dpo/margin_mean": 239.8885498046875,
"margin_dpo/margin_std": 254.16818237304688,
"step": 495
},
{
"KL/chosen_KL_mean": -317.9449462890625,
"KL/mean": -406.6758728027344,
"KL/rejected_KL_mean": -495.4067687988281,
"KL/std": 182.09466552734375,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0027801187243312597,
"fcm_dpo/delta": 0.1122952476143837,
"fcm_dpo/margin": 177.4618377685547,
"fcm_dpo/q_t": 0.3890904188156128,
"grad_norm": 33.107242584228516,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.4817647933959961,
"logits/rejected": -0.4576947093009949,
"logps/chosen": -390.536865234375,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -579.736083984375,
"loss": 1.0262,
"margin_dpo/margin_mean": 177.4618377685547,
"margin_dpo/margin_std": 200.71798706054688,
"step": 496
},
{
"KL/chosen_KL_mean": -311.9381103515625,
"KL/mean": -392.1148986816406,
"KL/rejected_KL_mean": -472.2917175292969,
"KL/std": 212.1793670654297,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0028411592356860638,
"fcm_dpo/delta": 0.15116159617900848,
"fcm_dpo/margin": 160.35362243652344,
"fcm_dpo/q_t": 0.39820799231529236,
"grad_norm": 29.663463592529297,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.4611259400844574,
"logits/rejected": -0.4396577477455139,
"logps/chosen": -370.5320739746094,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -548.580078125,
"loss": 1.0669,
"margin_dpo/margin_mean": 160.35362243652344,
"margin_dpo/margin_std": 202.749267578125,
"step": 497
},
{
"KL/chosen_KL_mean": -301.0348815917969,
"KL/mean": -402.95806884765625,
"KL/rejected_KL_mean": -504.88128662109375,
"KL/std": 215.5568084716797,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0028933845460414886,
"fcm_dpo/delta": 0.010811783373355865,
"fcm_dpo/margin": 203.84640502929688,
"fcm_dpo/q_t": 0.37201425433158875,
"grad_norm": 32.140716552734375,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.4592130780220032,
"logits/rejected": -0.43541595339775085,
"logps/chosen": -372.24053955078125,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -588.83935546875,
"loss": 1.0088,
"margin_dpo/margin_mean": 203.84640502929688,
"margin_dpo/margin_std": 263.30987548828125,
"step": 498
},
{
"KL/chosen_KL_mean": -287.6316223144531,
"KL/mean": -413.111328125,
"KL/rejected_KL_mean": -538.591064453125,
"KL/std": 235.47779846191406,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.002833035308867693,
"fcm_dpo/delta": -0.12003572285175323,
"fcm_dpo/margin": 250.95947265625,
"fcm_dpo/q_t": 0.34488415718078613,
"grad_norm": 36.83552932739258,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.42466431856155396,
"logits/rejected": -0.4308118224143982,
"logps/chosen": -338.8868103027344,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -639.6697998046875,
"loss": 0.9298,
"margin_dpo/margin_mean": 250.95945739746094,
"margin_dpo/margin_std": 273.89031982421875,
"step": 499
},
{
"KL/chosen_KL_mean": -340.96600341796875,
"KL/mean": -451.2560729980469,
"KL/rejected_KL_mean": -561.546142578125,
"KL/std": 223.45965576171875,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.002813946921378374,
"fcm_dpo/delta": -0.02208024077117443,
"fcm_dpo/margin": 220.58013916015625,
"fcm_dpo/q_t": 0.3649978041648865,
"grad_norm": 56.1822509765625,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.43707555532455444,
"logits/rejected": -0.44277822971343994,
"logps/chosen": -397.99346923828125,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -655.4803466796875,
"loss": 0.9684,
"margin_dpo/margin_mean": 220.5801239013672,
"margin_dpo/margin_std": 253.31436157226562,
"step": 500
},
{
"KL/chosen_KL_mean": -291.3111572265625,
"KL/mean": -410.5042419433594,
"KL/rejected_KL_mean": -529.6973266601562,
"KL/std": 228.95303344726562,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.002791839651763439,
"fcm_dpo/delta": -0.07020300626754761,
"fcm_dpo/margin": 238.38613891601562,
"fcm_dpo/q_t": 0.3549428880214691,
"grad_norm": 32.68425369262695,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.43849706649780273,
"logits/rejected": -0.4090450704097748,
"logps/chosen": -345.6706848144531,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -609.85400390625,
"loss": 0.9455,
"margin_dpo/margin_mean": 238.38613891601562,
"margin_dpo/margin_std": 262.63385009765625,
"step": 501
},
{
"KL/chosen_KL_mean": -274.57452392578125,
"KL/mean": -394.942138671875,
"KL/rejected_KL_mean": -515.309814453125,
"KL/std": 202.78167724609375,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.002760091330856085,
"fcm_dpo/delta": -0.06932684034109116,
"fcm_dpo/margin": 240.73524475097656,
"fcm_dpo/q_t": 0.3472452759742737,
"grad_norm": 29.198741912841797,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.4376978874206543,
"logits/rejected": -0.4115495979785919,
"logps/chosen": -342.175048828125,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -598.258544921875,
"loss": 0.8875,
"margin_dpo/margin_mean": 240.7352294921875,
"margin_dpo/margin_std": 177.9047393798828,
"step": 502
},
{
"KL/chosen_KL_mean": -285.70050048828125,
"KL/mean": -398.41900634765625,
"KL/rejected_KL_mean": -511.1374816894531,
"KL/std": 207.94387817382812,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.002717811381444335,
"fcm_dpo/delta": -0.013907298445701599,
"fcm_dpo/margin": 225.43701171875,
"fcm_dpo/q_t": 0.3645442724227905,
"grad_norm": 29.358478546142578,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.39860522747039795,
"logits/rejected": -0.37364786863327026,
"logps/chosen": -340.7789001464844,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -593.6429443359375,
"loss": 0.9465,
"margin_dpo/margin_mean": 225.43701171875,
"margin_dpo/margin_std": 225.81784057617188,
"step": 503
},
{
"KL/chosen_KL_mean": -313.34429931640625,
"KL/mean": -450.6795654296875,
"KL/rejected_KL_mean": -588.0148315429688,
"KL/std": 243.04483032226562,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.002665948588401079,
"fcm_dpo/delta": -0.14294353127479553,
"fcm_dpo/margin": 274.6705322265625,
"fcm_dpo/q_t": 0.33880650997161865,
"grad_norm": 29.31102180480957,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.43111157417297363,
"logits/rejected": -0.435757040977478,
"logps/chosen": -373.31005859375,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -691.7769775390625,
"loss": 0.8836,
"margin_dpo/margin_mean": 274.6705322265625,
"margin_dpo/margin_std": 249.06939697265625,
"step": 504
},
{
"KL/chosen_KL_mean": -360.6900634765625,
"KL/mean": -438.03375244140625,
"KL/rejected_KL_mean": -515.37744140625,
"KL/std": 232.5611572265625,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0027289404533803463,
"fcm_dpo/delta": 0.18531014025211334,
"fcm_dpo/margin": 154.68734741210938,
"fcm_dpo/q_t": 0.4102995991706848,
"grad_norm": 58.3673095703125,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.49714380502700806,
"logits/rejected": -0.467399001121521,
"logps/chosen": -436.844970703125,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -603.9627685546875,
"loss": 1.1349,
"margin_dpo/margin_mean": 154.68736267089844,
"margin_dpo/margin_std": 274.23516845703125,
"step": 505
},
{
"KL/chosen_KL_mean": -291.33978271484375,
"KL/mean": -413.8785400390625,
"KL/rejected_KL_mean": -536.4172973632812,
"KL/std": 230.0699462890625,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.002709229476749897,
"fcm_dpo/delta": -0.06928421556949615,
"fcm_dpo/margin": 245.0775146484375,
"fcm_dpo/q_t": 0.35389435291290283,
"grad_norm": 25.556169509887695,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.45745474100112915,
"logits/rejected": -0.45172441005706787,
"logps/chosen": -340.30029296875,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -614.8323974609375,
"loss": 0.9291,
"margin_dpo/margin_mean": 245.07752990722656,
"margin_dpo/margin_std": 248.67483520507812,
"step": 506
},
{
"KL/chosen_KL_mean": -349.04791259765625,
"KL/mean": -449.2943420410156,
"KL/rejected_KL_mean": -549.540771484375,
"KL/std": 254.82516479492188,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0027359488885849714,
"fcm_dpo/delta": 0.05440139025449753,
"fcm_dpo/margin": 200.49285888671875,
"fcm_dpo/q_t": 0.38290736079216003,
"grad_norm": 30.00538444519043,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.4429423213005066,
"logits/rejected": -0.42831623554229736,
"logps/chosen": -415.84942626953125,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -644.9136962890625,
"loss": 1.0194,
"margin_dpo/margin_mean": 200.4928741455078,
"margin_dpo/margin_std": 262.6377868652344,
"step": 507
},
{
"KL/chosen_KL_mean": -337.65087890625,
"KL/mean": -433.72088623046875,
"KL/rejected_KL_mean": -529.7908935546875,
"KL/std": 238.59854125976562,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.002776654437184334,
"fcm_dpo/delta": 0.06997797638177872,
"fcm_dpo/margin": 192.13999938964844,
"fcm_dpo/q_t": 0.3879029154777527,
"grad_norm": 36.94618225097656,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.45221245288848877,
"logits/rejected": -0.4418308436870575,
"logps/chosen": -408.954345703125,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -625.41845703125,
"loss": 1.0489,
"margin_dpo/margin_mean": 192.13999938964844,
"margin_dpo/margin_std": 276.2540588378906,
"step": 508
},
{
"KL/chosen_KL_mean": -299.462158203125,
"KL/mean": -394.63677978515625,
"KL/rejected_KL_mean": -489.8114013671875,
"KL/std": 206.77728271484375,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0028194680344313383,
"fcm_dpo/delta": 0.06628626585006714,
"fcm_dpo/margin": 190.34921264648438,
"fcm_dpo/q_t": 0.38451287150382996,
"grad_norm": 40.84049987792969,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.466485857963562,
"logits/rejected": -0.45346006751060486,
"logps/chosen": -363.2811279296875,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -573.06787109375,
"loss": 1.0245,
"margin_dpo/margin_mean": 190.34921264648438,
"margin_dpo/margin_std": 241.7765655517578,
"step": 509
},
{
"KL/chosen_KL_mean": -346.3660583496094,
"KL/mean": -464.6279296875,
"KL/rejected_KL_mean": -582.8897705078125,
"KL/std": 221.30184936523438,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0027909083291888237,
"fcm_dpo/delta": -0.06436065584421158,
"fcm_dpo/margin": 236.52374267578125,
"fcm_dpo/q_t": 0.3517419695854187,
"grad_norm": 31.882673263549805,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.41880273818969727,
"logits/rejected": -0.42115575075149536,
"logps/chosen": -398.2445068359375,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -685.6549072265625,
"loss": 0.9168,
"margin_dpo/margin_mean": 236.5237274169922,
"margin_dpo/margin_std": 220.63653564453125,
"step": 510
},
{
"KL/chosen_KL_mean": -322.9490051269531,
"KL/mean": -444.62188720703125,
"KL/rejected_KL_mean": -566.2947998046875,
"KL/std": 252.72100830078125,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.002760619157925248,
"fcm_dpo/delta": -0.0769667848944664,
"fcm_dpo/margin": 243.34571838378906,
"fcm_dpo/q_t": 0.3572511672973633,
"grad_norm": 28.490880966186523,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.43943625688552856,
"logits/rejected": -0.4329058825969696,
"logps/chosen": -383.1871337890625,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -659.1515502929688,
"loss": 0.9666,
"margin_dpo/margin_mean": 243.34573364257812,
"margin_dpo/margin_std": 298.26580810546875,
"step": 511
},
{
"KL/chosen_KL_mean": -298.0306091308594,
"KL/mean": -426.70806884765625,
"KL/rejected_KL_mean": -555.385498046875,
"KL/std": 233.50778198242188,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.0026991183403879404,
"fcm_dpo/delta": -0.10165773332118988,
"fcm_dpo/margin": 257.3548583984375,
"fcm_dpo/q_t": 0.3462957739830017,
"grad_norm": 30.536540985107422,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.42960307002067566,
"logits/rejected": -0.4208429753780365,
"logps/chosen": -352.93609619140625,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -637.2613525390625,
"loss": 0.8971,
"margin_dpo/margin_mean": 257.3548583984375,
"margin_dpo/margin_std": 237.53274536132812,
"step": 512
},
{
"KL/chosen_KL_mean": -347.88763427734375,
"KL/mean": -433.85980224609375,
"KL/rejected_KL_mean": -519.8319702148438,
"KL/std": 213.32374572753906,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.002720455639064312,
"fcm_dpo/delta": 0.13866117596626282,
"fcm_dpo/margin": 171.9443359375,
"fcm_dpo/q_t": 0.397952675819397,
"grad_norm": 44.469078063964844,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.41821664571762085,
"logits/rejected": -0.386923611164093,
"logps/chosen": -412.8040771484375,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -595.8944091796875,
"loss": 1.0921,
"margin_dpo/margin_mean": 171.9443359375,
"margin_dpo/margin_std": 260.04180908203125,
"step": 513
},
{
"KL/chosen_KL_mean": -329.38861083984375,
"KL/mean": -426.50634765625,
"KL/rejected_KL_mean": -523.6240844726562,
"KL/std": 215.20596313476562,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.002786934142932296,
"fcm_dpo/delta": 0.06197813153266907,
"fcm_dpo/margin": 194.23550415039062,
"fcm_dpo/q_t": 0.38390105962753296,
"grad_norm": 45.516685485839844,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.4705209732055664,
"logits/rejected": -0.45037761330604553,
"logps/chosen": -403.6181945800781,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -602.5696411132812,
"loss": 1.031,
"margin_dpo/margin_mean": 194.23550415039062,
"margin_dpo/margin_std": 259.800537109375,
"step": 514
},
{
"KL/chosen_KL_mean": -293.9521484375,
"KL/mean": -381.98175048828125,
"KL/rejected_KL_mean": -470.01129150390625,
"KL/std": 201.84432983398438,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0028303172439336777,
"fcm_dpo/delta": 0.10716740787029266,
"fcm_dpo/margin": 176.05917358398438,
"fcm_dpo/q_t": 0.3887324929237366,
"grad_norm": 40.49409484863281,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.4513707756996155,
"logits/rejected": -0.4542236328125,
"logps/chosen": -344.35369873046875,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -557.1090087890625,
"loss": 1.0506,
"margin_dpo/margin_mean": 176.05917358398438,
"margin_dpo/margin_std": 234.74807739257812,
"step": 515
},
{
"KL/chosen_KL_mean": -298.52911376953125,
"KL/mean": -423.83404541015625,
"KL/rejected_KL_mean": -549.1390380859375,
"KL/std": 235.95242309570312,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.002816826105117798,
"fcm_dpo/delta": -0.1138298362493515,
"fcm_dpo/margin": 250.60986328125,
"fcm_dpo/q_t": 0.344161719083786,
"grad_norm": 35.118865966796875,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.42747095227241516,
"logits/rejected": -0.4437045454978943,
"logps/chosen": -363.1786804199219,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -660.8613891601562,
"loss": 0.8957,
"margin_dpo/margin_mean": 250.60986328125,
"margin_dpo/margin_std": 234.30316162109375,
"step": 516
},
{
"KL/chosen_KL_mean": -316.02764892578125,
"KL/mean": -439.80255126953125,
"KL/rejected_KL_mean": -563.5775146484375,
"KL/std": 237.56033325195312,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.00274248281493783,
"fcm_dpo/delta": -0.08545368909835815,
"fcm_dpo/margin": 247.54986572265625,
"fcm_dpo/q_t": 0.3531512916088104,
"grad_norm": 29.865224838256836,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.4610327482223511,
"logits/rejected": -0.4480746388435364,
"logps/chosen": -376.941162109375,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -652.6605834960938,
"loss": 0.9248,
"margin_dpo/margin_mean": 247.54986572265625,
"margin_dpo/margin_std": 259.0785827636719,
"step": 517
},
{
"KL/chosen_KL_mean": -315.388427734375,
"KL/mean": -421.05133056640625,
"KL/rejected_KL_mean": -526.7142333984375,
"KL/std": 214.97793579101562,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.0027445396408438683,
"fcm_dpo/delta": 0.021165400743484497,
"fcm_dpo/margin": 211.32577514648438,
"fcm_dpo/q_t": 0.37446266412734985,
"grad_norm": 35.96814727783203,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.45537883043289185,
"logits/rejected": -0.4558147192001343,
"logps/chosen": -372.8443603515625,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -612.0269165039062,
"loss": 1.009,
"margin_dpo/margin_mean": 211.32577514648438,
"margin_dpo/margin_std": 269.41748046875,
"step": 518
},
{
"KL/chosen_KL_mean": -297.58599853515625,
"KL/mean": -387.8175354003906,
"KL/rejected_KL_mean": -478.049072265625,
"KL/std": 207.9893798828125,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0028028993401676416,
"fcm_dpo/delta": 0.09876328706741333,
"fcm_dpo/margin": 180.46304321289062,
"fcm_dpo/q_t": 0.38994652032852173,
"grad_norm": 35.74894714355469,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.47397053241729736,
"logits/rejected": -0.45668813586235046,
"logps/chosen": -371.64935302734375,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -582.4932250976562,
"loss": 1.0451,
"margin_dpo/margin_mean": 180.46304321289062,
"margin_dpo/margin_std": 242.0874786376953,
"step": 519
},
{
"KL/chosen_KL_mean": -296.8106689453125,
"KL/mean": -397.0625915527344,
"KL/rejected_KL_mean": -497.31451416015625,
"KL/std": 205.43948364257812,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.002808785066008568,
"fcm_dpo/delta": 0.038528621196746826,
"fcm_dpo/margin": 200.5038604736328,
"fcm_dpo/q_t": 0.3768748641014099,
"grad_norm": 29.33172035217285,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.45127061009407043,
"logits/rejected": -0.44956347346305847,
"logps/chosen": -367.1104736328125,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -597.2958374023438,
"loss": 0.989,
"margin_dpo/margin_mean": 200.50387573242188,
"margin_dpo/margin_std": 218.33010864257812,
"step": 520
},
{
"KL/chosen_KL_mean": -276.12445068359375,
"KL/mean": -374.5679626464844,
"KL/rejected_KL_mean": -473.0114440917969,
"KL/std": 203.98727416992188,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.002870450960472226,
"fcm_dpo/delta": 0.03510994464159012,
"fcm_dpo/margin": 196.88702392578125,
"fcm_dpo/q_t": 0.37647244334220886,
"grad_norm": 33.32754135131836,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.41739487648010254,
"logits/rejected": -0.40427398681640625,
"logps/chosen": -334.2673645019531,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -556.2920532226562,
"loss": 0.9956,
"margin_dpo/margin_mean": 196.8870391845703,
"margin_dpo/margin_std": 223.54173278808594,
"step": 521
},
{
"KL/chosen_KL_mean": -303.0511169433594,
"KL/mean": -392.3768310546875,
"KL/rejected_KL_mean": -481.7025146484375,
"KL/std": 206.94976806640625,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.002895869081839919,
"fcm_dpo/delta": 0.08676917850971222,
"fcm_dpo/margin": 178.65142822265625,
"fcm_dpo/q_t": 0.384182333946228,
"grad_norm": 44.6025390625,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.4320841431617737,
"logits/rejected": -0.42646926641464233,
"logps/chosen": -349.5987854003906,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -547.7164306640625,
"loss": 1.0266,
"margin_dpo/margin_mean": 178.65142822265625,
"margin_dpo/margin_std": 218.95957946777344,
"step": 522
},
{
"KL/chosen_KL_mean": -290.66778564453125,
"KL/mean": -387.527587890625,
"KL/rejected_KL_mean": -484.3873291015625,
"KL/std": 212.25582885742188,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.002916134661063552,
"fcm_dpo/delta": 0.037197425961494446,
"fcm_dpo/margin": 193.71954345703125,
"fcm_dpo/q_t": 0.37321603298187256,
"grad_norm": 42.96333694458008,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.4802251160144806,
"logits/rejected": -0.4583859443664551,
"logps/chosen": -352.4374084472656,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -568.1487426757812,
"loss": 0.9951,
"margin_dpo/margin_mean": 193.71954345703125,
"margin_dpo/margin_std": 222.89088439941406,
"step": 523
},
{
"KL/chosen_KL_mean": -288.0035095214844,
"KL/mean": -390.158935546875,
"KL/rejected_KL_mean": -492.3143005371094,
"KL/std": 211.02407836914062,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0029141202103346586,
"fcm_dpo/delta": 0.004657458513975143,
"fcm_dpo/margin": 204.31082153320312,
"fcm_dpo/q_t": 0.3655627369880676,
"grad_norm": 42.43870162963867,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.502049446105957,
"logits/rejected": -0.47493791580200195,
"logps/chosen": -366.0755615234375,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -573.6163330078125,
"loss": 0.9649,
"margin_dpo/margin_mean": 204.31082153320312,
"margin_dpo/margin_std": 210.40423583984375,
"step": 524
},
{
"KL/chosen_KL_mean": -284.7969970703125,
"KL/mean": -407.74005126953125,
"KL/rejected_KL_mean": -530.68310546875,
"KL/std": 220.3665008544922,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0028642993420362473,
"fcm_dpo/delta": -0.11306525766849518,
"fcm_dpo/margin": 245.88609313964844,
"fcm_dpo/q_t": 0.34323346614837646,
"grad_norm": 41.25353240966797,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.40292418003082275,
"logits/rejected": -0.41175270080566406,
"logps/chosen": -335.6248779296875,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -630.7360229492188,
"loss": 0.8877,
"margin_dpo/margin_mean": 245.88609313964844,
"margin_dpo/margin_std": 217.70860290527344,
"step": 525
},
{
"KL/chosen_KL_mean": -289.91961669921875,
"KL/mean": -410.85736083984375,
"KL/rejected_KL_mean": -531.795166015625,
"KL/std": 211.06703186035156,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.002838113345205784,
"fcm_dpo/delta": -0.09304739534854889,
"fcm_dpo/margin": 241.87554931640625,
"fcm_dpo/q_t": 0.348857581615448,
"grad_norm": 34.808563232421875,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.4654667377471924,
"logits/rejected": -0.4409719705581665,
"logps/chosen": -353.08685302734375,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -618.1044921875,
"loss": 0.9109,
"margin_dpo/margin_mean": 241.8755340576172,
"margin_dpo/margin_std": 232.07830810546875,
"step": 526
},
{
"KL/chosen_KL_mean": -316.68927001953125,
"KL/mean": -410.96649169921875,
"KL/rejected_KL_mean": -505.24371337890625,
"KL/std": 212.4687957763672,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.002828553318977356,
"fcm_dpo/delta": 0.07045119255781174,
"fcm_dpo/margin": 188.55447387695312,
"fcm_dpo/q_t": 0.3805032968521118,
"grad_norm": 31.366683959960938,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -0.5071585178375244,
"logits/rejected": -0.47814828157424927,
"logps/chosen": -387.8179626464844,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -583.5863037109375,
"loss": 0.998,
"margin_dpo/margin_mean": 188.55447387695312,
"margin_dpo/margin_std": 203.21487426757812,
"step": 527
},
{
"KL/chosen_KL_mean": -316.0058288574219,
"KL/mean": -420.6962890625,
"KL/rejected_KL_mean": -525.3867797851562,
"KL/std": 227.39761352539062,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.002842032117769122,
"fcm_dpo/delta": 0.005128853023052216,
"fcm_dpo/margin": 209.38092041015625,
"fcm_dpo/q_t": 0.37372392416000366,
"grad_norm": 33.0492057800293,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -0.4732360243797302,
"logits/rejected": -0.45158904790878296,
"logps/chosen": -384.09527587890625,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -619.296875,
"loss": 0.9983,
"margin_dpo/margin_mean": 209.38092041015625,
"margin_dpo/margin_std": 265.46966552734375,
"step": 528
},
{
"KL/chosen_KL_mean": -359.05316162109375,
"KL/mean": -439.93634033203125,
"KL/rejected_KL_mean": -520.8194580078125,
"KL/std": 194.19403076171875,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.002907057758420706,
"fcm_dpo/delta": 0.13619258999824524,
"fcm_dpo/margin": 161.7663116455078,
"fcm_dpo/q_t": 0.39629998803138733,
"grad_norm": 40.91783905029297,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -0.4297773540019989,
"logits/rejected": -0.42900708317756653,
"logps/chosen": -414.62811279296875,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -610.028564453125,
"loss": 1.079,
"margin_dpo/margin_mean": 161.7663116455078,
"margin_dpo/margin_std": 232.21859741210938,
"step": 529
},
{
"KL/chosen_KL_mean": -292.5477600097656,
"KL/mean": -419.95404052734375,
"KL/rejected_KL_mean": -547.3603515625,
"KL/std": 228.95162963867188,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.0028622723184525967,
"fcm_dpo/delta": -0.1399245709180832,
"fcm_dpo/margin": 254.81259155273438,
"fcm_dpo/q_t": 0.3427494466304779,
"grad_norm": 34.072994232177734,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.40677356719970703,
"logits/rejected": -0.3933334946632385,
"logps/chosen": -340.149169921875,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -634.6448974609375,
"loss": 0.9234,
"margin_dpo/margin_mean": 254.81259155273438,
"margin_dpo/margin_std": 280.197021484375,
"step": 530
},
{
"KL/chosen_KL_mean": -326.4650573730469,
"KL/mean": -437.4271240234375,
"KL/rejected_KL_mean": -548.38916015625,
"KL/std": 252.55072021484375,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.002833522856235504,
"fcm_dpo/delta": -0.03077915497124195,
"fcm_dpo/margin": 221.92410278320312,
"fcm_dpo/q_t": 0.3626970946788788,
"grad_norm": 34.40644454956055,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -0.4946574866771698,
"logits/rejected": -0.4679161310195923,
"logps/chosen": -381.71112060546875,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -618.9951171875,
"loss": 0.9551,
"margin_dpo/margin_mean": 221.92410278320312,
"margin_dpo/margin_std": 242.02804565429688,
"step": 531
},
{
"KL/chosen_KL_mean": -344.74114990234375,
"KL/mean": -460.53485107421875,
"KL/rejected_KL_mean": -576.32861328125,
"KL/std": 234.98336791992188,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0027756746858358383,
"fcm_dpo/delta": -0.04943156987428665,
"fcm_dpo/margin": 231.58746337890625,
"fcm_dpo/q_t": 0.3634790778160095,
"grad_norm": 49.158451080322266,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -0.4730091691017151,
"logits/rejected": -0.44550079107284546,
"logps/chosen": -415.02716064453125,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -662.919921875,
"loss": 0.9827,
"margin_dpo/margin_mean": 231.58743286132812,
"margin_dpo/margin_std": 276.4494934082031,
"step": 532
},
{
"KL/chosen_KL_mean": -334.2520751953125,
"KL/mean": -424.07659912109375,
"KL/rejected_KL_mean": -513.901123046875,
"KL/std": 211.8524627685547,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.002862041350454092,
"fcm_dpo/delta": 0.08851660788059235,
"fcm_dpo/margin": 179.64907836914062,
"fcm_dpo/q_t": 0.39045393466949463,
"grad_norm": 45.82151412963867,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -0.4523179531097412,
"logits/rejected": -0.42436474561691284,
"logps/chosen": -399.107177734375,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -590.4891967773438,
"loss": 1.0632,
"margin_dpo/margin_mean": 179.64906311035156,
"margin_dpo/margin_std": 257.60186767578125,
"step": 533
},
{
"KL/chosen_KL_mean": -337.57904052734375,
"KL/mean": -437.3919982910156,
"KL/rejected_KL_mean": -537.2049560546875,
"KL/std": 220.24609375,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0028645102865993977,
"fcm_dpo/delta": 0.02988392300903797,
"fcm_dpo/margin": 199.62591552734375,
"fcm_dpo/q_t": 0.37313705682754517,
"grad_norm": 34.530181884765625,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -0.48053455352783203,
"logits/rejected": -0.46491605043411255,
"logps/chosen": -397.69842529296875,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -615.7484130859375,
"loss": 0.9965,
"margin_dpo/margin_mean": 199.6259002685547,
"margin_dpo/margin_std": 237.0495147705078,
"step": 534
},
{
"KL/chosen_KL_mean": -296.08233642578125,
"KL/mean": -421.2332763671875,
"KL/rejected_KL_mean": -546.38427734375,
"KL/std": 227.76918029785156,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.002839513588696718,
"fcm_dpo/delta": -0.11940048635005951,
"fcm_dpo/margin": 250.3018798828125,
"fcm_dpo/q_t": 0.34363484382629395,
"grad_norm": 35.80217742919922,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.471211314201355,
"logits/rejected": -0.4542166590690613,
"logps/chosen": -350.41259765625,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -642.69189453125,
"loss": 0.8927,
"margin_dpo/margin_mean": 250.3018798828125,
"margin_dpo/margin_std": 227.12936401367188,
"step": 535
},
{
"KL/chosen_KL_mean": -281.02056884765625,
"KL/mean": -427.8605651855469,
"KL/rejected_KL_mean": -574.7006225585938,
"KL/std": 274.92578125,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.002715503564104438,
"fcm_dpo/delta": -0.21452924609184265,
"fcm_dpo/margin": 293.6800231933594,
"fcm_dpo/q_t": 0.33177119493484497,
"grad_norm": 28.878692626953125,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.44789987802505493,
"logits/rejected": -0.4465147852897644,
"logps/chosen": -328.10107421875,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -663.7984619140625,
"loss": 0.8609,
"margin_dpo/margin_mean": 293.67999267578125,
"margin_dpo/margin_std": 296.1116027832031,
"step": 536
},
{
"KL/chosen_KL_mean": -280.5416259765625,
"KL/mean": -395.1883544921875,
"KL/rejected_KL_mean": -509.8350830078125,
"KL/std": 230.98880004882812,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.0026884444523602724,
"fcm_dpo/delta": -0.01781396195292473,
"fcm_dpo/margin": 229.29345703125,
"fcm_dpo/q_t": 0.36612236499786377,
"grad_norm": 40.47601318359375,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.43334293365478516,
"logits/rejected": -0.40216875076293945,
"logps/chosen": -338.2890930175781,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -580.2734375,
"loss": 0.9769,
"margin_dpo/margin_mean": 229.29345703125,
"margin_dpo/margin_std": 266.6603698730469,
"step": 537
},
{
"KL/chosen_KL_mean": -341.8367004394531,
"KL/mean": -462.9151916503906,
"KL/rejected_KL_mean": -583.99365234375,
"KL/std": 224.29754638671875,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.0026520469691604376,
"fcm_dpo/delta": -0.045254189521074295,
"fcm_dpo/margin": 242.15695190429688,
"fcm_dpo/q_t": 0.3585195243358612,
"grad_norm": 33.3912239074707,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.519140899181366,
"logits/rejected": -0.4938586950302124,
"logps/chosen": -408.2526550292969,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -668.2218017578125,
"loss": 0.9382,
"margin_dpo/margin_mean": 242.15695190429688,
"margin_dpo/margin_std": 247.3631591796875,
"step": 538
},
{
"KL/chosen_KL_mean": -324.30743408203125,
"KL/mean": -425.1382751464844,
"KL/rejected_KL_mean": -525.9691162109375,
"KL/std": 213.65945434570312,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.0026718254666775465,
"fcm_dpo/delta": 0.06473114341497421,
"fcm_dpo/margin": 201.66162109375,
"fcm_dpo/q_t": 0.38126450777053833,
"grad_norm": 30.309885025024414,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.48138299584388733,
"logits/rejected": -0.45210009813308716,
"logps/chosen": -382.80029296875,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -617.822998046875,
"loss": 1.0144,
"margin_dpo/margin_mean": 201.66162109375,
"margin_dpo/margin_std": 243.572265625,
"step": 539
},
{
"KL/chosen_KL_mean": -303.02752685546875,
"KL/mean": -448.418701171875,
"KL/rejected_KL_mean": -593.8099365234375,
"KL/std": 267.3698425292969,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.002628666814416647,
"fcm_dpo/delta": -0.17760419845581055,
"fcm_dpo/margin": 290.78240966796875,
"fcm_dpo/q_t": 0.3372151851654053,
"grad_norm": 28.53801727294922,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.49332183599472046,
"logits/rejected": -0.4891811013221741,
"logps/chosen": -366.510009765625,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -710.2398681640625,
"loss": 0.8887,
"margin_dpo/margin_mean": 290.7823791503906,
"margin_dpo/margin_std": 288.1028747558594,
"step": 540
},
{
"KL/chosen_KL_mean": -356.8083190917969,
"KL/mean": -450.08843994140625,
"KL/rejected_KL_mean": -543.3685302734375,
"KL/std": 227.94580078125,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0026146559976041317,
"fcm_dpo/delta": 0.11744622886180878,
"fcm_dpo/margin": 186.5601806640625,
"fcm_dpo/q_t": 0.3958682715892792,
"grad_norm": 32.374595642089844,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -0.5324292182922363,
"logits/rejected": -0.5199625492095947,
"logps/chosen": -435.08868408203125,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -646.6422119140625,
"loss": 1.0663,
"margin_dpo/margin_mean": 186.56019592285156,
"margin_dpo/margin_std": 262.82135009765625,
"step": 541
},
{
"KL/chosen_KL_mean": -335.47198486328125,
"KL/mean": -455.1080322265625,
"KL/rejected_KL_mean": -574.7440185546875,
"KL/std": 240.4376220703125,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.002625656547024846,
"fcm_dpo/delta": -0.031004559248685837,
"fcm_dpo/margin": 239.2720489501953,
"fcm_dpo/q_t": 0.3629918694496155,
"grad_norm": 37.954280853271484,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -0.46517491340637207,
"logits/rejected": -0.4631243944168091,
"logps/chosen": -392.95697021484375,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -671.2191162109375,
"loss": 0.9667,
"margin_dpo/margin_mean": 239.2720489501953,
"margin_dpo/margin_std": 271.30889892578125,
"step": 542
},
{
"KL/chosen_KL_mean": -337.7369689941406,
"KL/mean": -426.249755859375,
"KL/rejected_KL_mean": -514.7625732421875,
"KL/std": 247.21511840820312,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0026882379315793514,
"fcm_dpo/delta": 0.13031548261642456,
"fcm_dpo/margin": 177.025634765625,
"fcm_dpo/q_t": 0.4005751609802246,
"grad_norm": 35.18562698364258,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.4823335111141205,
"logits/rejected": -0.48822733759880066,
"logps/chosen": -398.3544921875,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -620.361572265625,
"loss": 1.0926,
"margin_dpo/margin_mean": 177.025634765625,
"margin_dpo/margin_std": 283.3887939453125,
"step": 543
},
{
"KL/chosen_KL_mean": -365.1501159667969,
"KL/mean": -494.2093505859375,
"KL/rejected_KL_mean": -623.2685546875,
"KL/std": 252.28506469726562,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.002666377928107977,
"fcm_dpo/delta": -0.0948493480682373,
"fcm_dpo/margin": 258.1184387207031,
"fcm_dpo/q_t": 0.3496937155723572,
"grad_norm": 38.14399337768555,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.45028936862945557,
"logits/rejected": -0.44636812806129456,
"logps/chosen": -424.79241943359375,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -724.2232666015625,
"loss": 0.9063,
"margin_dpo/margin_mean": 258.1184387207031,
"margin_dpo/margin_std": 256.18896484375,
"step": 544
},
{
"KL/chosen_KL_mean": -342.6295166015625,
"KL/mean": -466.5543518066406,
"KL/rejected_KL_mean": -590.479248046875,
"KL/std": 237.80712890625,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.002638470381498337,
"fcm_dpo/delta": -0.05769067257642746,
"fcm_dpo/margin": 247.84971618652344,
"fcm_dpo/q_t": 0.35607659816741943,
"grad_norm": 31.097354888916016,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.43896418809890747,
"logits/rejected": -0.4325362741947174,
"logps/chosen": -410.278076171875,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -686.38720703125,
"loss": 0.9377,
"margin_dpo/margin_mean": 247.84971618652344,
"margin_dpo/margin_std": 256.32757568359375,
"step": 545
},
{
"KL/chosen_KL_mean": -313.9615478515625,
"KL/mean": -419.17413330078125,
"KL/rejected_KL_mean": -524.38671875,
"KL/std": 213.50949096679688,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0026380978524684906,
"fcm_dpo/delta": 0.04752631112933159,
"fcm_dpo/margin": 210.4251708984375,
"fcm_dpo/q_t": 0.3729926347732544,
"grad_norm": 34.724266052246094,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.46538007259368896,
"logits/rejected": -0.44924771785736084,
"logps/chosen": -364.7057800292969,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -606.2529296875,
"loss": 0.9841,
"margin_dpo/margin_mean": 210.4251708984375,
"margin_dpo/margin_std": 219.16452026367188,
"step": 546
},
{
"KL/chosen_KL_mean": -357.4351806640625,
"KL/mean": -464.5871887207031,
"KL/rejected_KL_mean": -571.7392578125,
"KL/std": 242.331787109375,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.0026768981479108334,
"fcm_dpo/delta": 0.02700256183743477,
"fcm_dpo/margin": 214.3040771484375,
"fcm_dpo/q_t": 0.3751223683357239,
"grad_norm": 49.115257263183594,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.47335153818130493,
"logits/rejected": -0.4591940641403198,
"logps/chosen": -431.12286376953125,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -662.5006103515625,
"loss": 1.0146,
"margin_dpo/margin_mean": 214.3040771484375,
"margin_dpo/margin_std": 275.0681457519531,
"step": 547
},
{
"KL/chosen_KL_mean": -354.8642578125,
"KL/mean": -463.935791015625,
"KL/rejected_KL_mean": -573.0072631835938,
"KL/std": 257.4043273925781,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.002671770518645644,
"fcm_dpo/delta": 0.018233004957437515,
"fcm_dpo/margin": 218.14297485351562,
"fcm_dpo/q_t": 0.37668776512145996,
"grad_norm": 36.122379302978516,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -0.522218644618988,
"logits/rejected": -0.5057340264320374,
"logps/chosen": -420.110595703125,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -667.1253662109375,
"loss": 1.0056,
"margin_dpo/margin_mean": 218.14297485351562,
"margin_dpo/margin_std": 287.7431640625,
"step": 548
},
{
"KL/chosen_KL_mean": -346.433837890625,
"KL/mean": -441.5001525878906,
"KL/rejected_KL_mean": -536.5664672851562,
"KL/std": 212.34481811523438,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0027136695571243763,
"fcm_dpo/delta": 0.08842714130878448,
"fcm_dpo/margin": 190.13262939453125,
"fcm_dpo/q_t": 0.3879546821117401,
"grad_norm": 33.019325256347656,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -0.4459270238876343,
"logits/rejected": -0.42524421215057373,
"logps/chosen": -395.64617919921875,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -610.476806640625,
"loss": 1.0368,
"margin_dpo/margin_mean": 190.13262939453125,
"margin_dpo/margin_std": 251.23681640625,
"step": 549
},
{
"KL/chosen_KL_mean": -333.859619140625,
"KL/mean": -428.53021240234375,
"KL/rejected_KL_mean": -523.2007446289062,
"KL/std": 218.2965545654297,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0027449466288089752,
"fcm_dpo/delta": 0.08466745913028717,
"fcm_dpo/margin": 189.3411865234375,
"fcm_dpo/q_t": 0.38216400146484375,
"grad_norm": 37.457462310791016,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.44693848490715027,
"logits/rejected": -0.45208030939102173,
"logps/chosen": -390.66656494140625,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -618.3265380859375,
"loss": 1.0069,
"margin_dpo/margin_mean": 189.3411865234375,
"margin_dpo/margin_std": 203.54380798339844,
"step": 550
},
{
"KL/chosen_KL_mean": -325.7191162109375,
"KL/mean": -468.3428955078125,
"KL/rejected_KL_mean": -610.9666137695312,
"KL/std": 246.4642333984375,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.0026862614322453737,
"fcm_dpo/delta": -0.18087095022201538,
"fcm_dpo/margin": 285.24749755859375,
"fcm_dpo/q_t": 0.32994920015335083,
"grad_norm": 34.547569274902344,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.44945597648620605,
"logits/rejected": -0.4558793902397156,
"logps/chosen": -384.8254699707031,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -722.639404296875,
"loss": 0.8563,
"margin_dpo/margin_mean": 285.24749755859375,
"margin_dpo/margin_std": 245.61746215820312,
"step": 551
},
{
"KL/chosen_KL_mean": -311.13787841796875,
"KL/mean": -471.29107666015625,
"KL/rejected_KL_mean": -631.4442138671875,
"KL/std": 286.0534362792969,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.002548352349549532,
"fcm_dpo/delta": -0.24064813554286957,
"fcm_dpo/margin": 320.3064270019531,
"fcm_dpo/q_t": 0.3275793790817261,
"grad_norm": 33.82465744018555,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.4366614818572998,
"logits/rejected": -0.42071181535720825,
"logps/chosen": -373.4924621582031,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -736.00634765625,
"loss": 0.8538,
"margin_dpo/margin_mean": 320.3064270019531,
"margin_dpo/margin_std": 304.65704345703125,
"step": 552
},
{
"KL/chosen_KL_mean": -332.5157470703125,
"KL/mean": -434.1606750488281,
"KL/rejected_KL_mean": -535.8056030273438,
"KL/std": 224.3307647705078,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0025649270974099636,
"fcm_dpo/delta": 0.08287201821804047,
"fcm_dpo/margin": 203.2898712158203,
"fcm_dpo/q_t": 0.3870941400527954,
"grad_norm": 31.14446449279785,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.4741405248641968,
"logits/rejected": -0.4594133794307709,
"logps/chosen": -400.7745361328125,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -633.9027099609375,
"loss": 1.0291,
"margin_dpo/margin_mean": 203.28988647460938,
"margin_dpo/margin_std": 258.1274108886719,
"step": 553
},
{
"KL/chosen_KL_mean": -338.2411804199219,
"KL/mean": -473.3432922363281,
"KL/rejected_KL_mean": -608.4454345703125,
"KL/std": 250.04498291015625,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0025644726119935513,
"fcm_dpo/delta": -0.10000116378068924,
"fcm_dpo/margin": 270.2042236328125,
"fcm_dpo/q_t": 0.34848809242248535,
"grad_norm": 29.957738876342773,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.4561349153518677,
"logits/rejected": -0.41868773102760315,
"logps/chosen": -406.1888427734375,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -698.2281494140625,
"loss": 0.9195,
"margin_dpo/margin_mean": 270.2042541503906,
"margin_dpo/margin_std": 274.03692626953125,
"step": 554
},
{
"KL/chosen_KL_mean": -340.6181640625,
"KL/mean": -475.83172607421875,
"KL/rejected_KL_mean": -611.0452880859375,
"KL/std": 260.68096923828125,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.0024976124987006187,
"fcm_dpo/delta": -0.08133400231599808,
"fcm_dpo/margin": 270.4271240234375,
"fcm_dpo/q_t": 0.3569917678833008,
"grad_norm": 33.5825080871582,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -0.51308274269104,
"logits/rejected": -0.5208144187927246,
"logps/chosen": -393.94866943359375,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -719.524658203125,
"loss": 0.9454,
"margin_dpo/margin_mean": 270.4271240234375,
"margin_dpo/margin_std": 309.9132080078125,
"step": 555
},
{
"KL/chosen_KL_mean": -341.1217041015625,
"KL/mean": -437.8756103515625,
"KL/rejected_KL_mean": -534.6295166015625,
"KL/std": 227.84573364257812,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.0025373874232172966,
"fcm_dpo/delta": 0.11464043706655502,
"fcm_dpo/margin": 193.50778198242188,
"fcm_dpo/q_t": 0.3930724859237671,
"grad_norm": 31.107650756835938,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -0.4337393045425415,
"logits/rejected": -0.41474783420562744,
"logps/chosen": -399.76617431640625,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -635.969970703125,
"loss": 1.0448,
"margin_dpo/margin_mean": 193.5077667236328,
"margin_dpo/margin_std": 251.46420288085938,
"step": 556
},
{
"KL/chosen_KL_mean": -355.62567138671875,
"KL/mean": -473.21173095703125,
"KL/rejected_KL_mean": -590.7977905273438,
"KL/std": 259.9843444824219,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.0025444929488003254,
"fcm_dpo/delta": 0.001470029354095459,
"fcm_dpo/margin": 235.17210388183594,
"fcm_dpo/q_t": 0.37023717164993286,
"grad_norm": 38.73280334472656,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -0.4727283716201782,
"logits/rejected": -0.4733373522758484,
"logps/chosen": -423.46630859375,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -700.7374267578125,
"loss": 0.983,
"margin_dpo/margin_mean": 235.172119140625,
"margin_dpo/margin_std": 274.781494140625,
"step": 557
},
{
"KL/chosen_KL_mean": -320.19195556640625,
"KL/mean": -482.168701171875,
"KL/rejected_KL_mean": -644.1454467773438,
"KL/std": 270.105712890625,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0024629058316349983,
"fcm_dpo/delta": -0.21627020835876465,
"fcm_dpo/margin": 323.9534606933594,
"fcm_dpo/q_t": 0.32557377219200134,
"grad_norm": 30.430641174316406,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -0.4561081528663635,
"logits/rejected": -0.4390890598297119,
"logps/chosen": -382.5602111816406,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -746.3064575195312,
"loss": 0.8549,
"margin_dpo/margin_mean": 323.95343017578125,
"margin_dpo/margin_std": 296.66021728515625,
"step": 558
},
{
"KL/chosen_KL_mean": -365.3654479980469,
"KL/mean": -485.8707580566406,
"KL/rejected_KL_mean": -606.3760986328125,
"KL/std": 251.71078491210938,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.002446204423904419,
"fcm_dpo/delta": 0.011072604916989803,
"fcm_dpo/margin": 241.01065063476562,
"fcm_dpo/q_t": 0.36853617429733276,
"grad_norm": 33.76393127441406,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -0.508272647857666,
"logits/rejected": -0.47824934124946594,
"logps/chosen": -426.1177673339844,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -699.818359375,
"loss": 0.9705,
"margin_dpo/margin_mean": 241.01063537597656,
"margin_dpo/margin_std": 258.4521789550781,
"step": 559
},
{
"KL/chosen_KL_mean": -318.3931884765625,
"KL/mean": -409.64385986328125,
"KL/rejected_KL_mean": -500.89453125,
"KL/std": 203.69400024414062,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.0024966727942228317,
"fcm_dpo/delta": 0.15152274072170258,
"fcm_dpo/margin": 182.50132751464844,
"fcm_dpo/q_t": 0.395315945148468,
"grad_norm": 34.535247802734375,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.4614247679710388,
"logits/rejected": -0.4287029504776001,
"logps/chosen": -376.49700927734375,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -580.8857421875,
"loss": 1.0348,
"margin_dpo/margin_mean": 182.50132751464844,
"margin_dpo/margin_std": 186.62606811523438,
"step": 560
},
{
"KL/chosen_KL_mean": -373.19287109375,
"KL/mean": -466.95599365234375,
"KL/rejected_KL_mean": -560.7191162109375,
"KL/std": 222.2974853515625,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0025822059251368046,
"fcm_dpo/delta": 0.12085901200771332,
"fcm_dpo/margin": 187.52621459960938,
"fcm_dpo/q_t": 0.39121192693710327,
"grad_norm": 34.659786224365234,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -0.4320663511753082,
"logits/rejected": -0.43511590361595154,
"logps/chosen": -424.10577392578125,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -659.7876586914062,
"loss": 1.0383,
"margin_dpo/margin_mean": 187.52621459960938,
"margin_dpo/margin_std": 227.34872436523438,
"step": 561
},
{
"KL/chosen_KL_mean": -322.2523193359375,
"KL/mean": -456.97711181640625,
"KL/rejected_KL_mean": -591.701904296875,
"KL/std": 254.9300994873047,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0025373073294758797,
"fcm_dpo/delta": -0.09119724482297897,
"fcm_dpo/margin": 269.4495544433594,
"fcm_dpo/q_t": 0.3511344790458679,
"grad_norm": 41.71552276611328,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -0.4211577773094177,
"logits/rejected": -0.40038222074508667,
"logps/chosen": -381.71673583984375,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -688.2445678710938,
"loss": 0.9109,
"margin_dpo/margin_mean": 269.4495544433594,
"margin_dpo/margin_std": 266.8976135253906,
"step": 562
},
{
"KL/chosen_KL_mean": -366.6177978515625,
"KL/mean": -460.3875732421875,
"KL/rejected_KL_mean": -554.1573486328125,
"KL/std": 227.03662109375,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.002605203539133072,
"fcm_dpo/delta": 0.11501321196556091,
"fcm_dpo/margin": 187.53958129882812,
"fcm_dpo/q_t": 0.39102718234062195,
"grad_norm": 39.29094314575195,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -0.4008295238018036,
"logits/rejected": -0.3941683769226074,
"logps/chosen": -427.2259826660156,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -648.72509765625,
"loss": 1.067,
"margin_dpo/margin_mean": 187.53958129882812,
"margin_dpo/margin_std": 258.1802673339844,
"step": 563
},
{
"KL/chosen_KL_mean": -342.66595458984375,
"KL/mean": -459.0194091796875,
"KL/rejected_KL_mean": -575.3728637695312,
"KL/std": 230.55918884277344,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0026013613678514957,
"fcm_dpo/delta": -0.005724776536226273,
"fcm_dpo/margin": 232.7069091796875,
"fcm_dpo/q_t": 0.36109423637390137,
"grad_norm": 29.79682731628418,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -0.5122097730636597,
"logits/rejected": -0.47511038184165955,
"logps/chosen": -416.934326171875,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -668.6110229492188,
"loss": 0.928,
"margin_dpo/margin_mean": 232.7069091796875,
"margin_dpo/margin_std": 193.72259521484375,
"step": 564
},
{
"KL/chosen_KL_mean": -361.0953063964844,
"KL/mean": -480.45318603515625,
"KL/rejected_KL_mean": -599.81103515625,
"KL/std": 264.8818664550781,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0025825374759733677,
"fcm_dpo/delta": -0.017826005816459656,
"fcm_dpo/margin": 238.71572875976562,
"fcm_dpo/q_t": 0.37151873111724854,
"grad_norm": 35.74713134765625,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -0.49406102299690247,
"logits/rejected": -0.4785243272781372,
"logps/chosen": -430.115234375,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -685.5899658203125,
"loss": 0.9968,
"margin_dpo/margin_mean": 238.71572875976562,
"margin_dpo/margin_std": 309.55841064453125,
"step": 565
},
{
"KL/chosen_KL_mean": -349.69854736328125,
"KL/mean": -471.5858154296875,
"KL/rejected_KL_mean": -593.4730834960938,
"KL/std": 269.077880859375,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.002597364829853177,
"fcm_dpo/delta": -0.03674561530351639,
"fcm_dpo/margin": 243.7745361328125,
"fcm_dpo/q_t": 0.3658466637134552,
"grad_norm": 31.682903289794922,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -0.4594685137271881,
"logits/rejected": -0.461688756942749,
"logps/chosen": -416.24383544921875,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -697.3424072265625,
"loss": 0.999,
"margin_dpo/margin_mean": 243.77455139160156,
"margin_dpo/margin_std": 307.5116882324219,
"step": 566
},
{
"KL/chosen_KL_mean": -343.28936767578125,
"KL/mean": -456.17919921875,
"KL/rejected_KL_mean": -569.069091796875,
"KL/std": 219.7535400390625,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.0025747488252818584,
"fcm_dpo/delta": 0.0198295209556818,
"fcm_dpo/margin": 225.7797088623047,
"fcm_dpo/q_t": 0.3708723783493042,
"grad_norm": 38.66469955444336,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -0.47161629796028137,
"logits/rejected": -0.46085941791534424,
"logps/chosen": -396.14764404296875,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -654.4400634765625,
"loss": 0.9802,
"margin_dpo/margin_mean": 225.77969360351562,
"margin_dpo/margin_std": 250.3170623779297,
"step": 567
},
{
"KL/chosen_KL_mean": -327.84808349609375,
"KL/mean": -460.6175842285156,
"KL/rejected_KL_mean": -593.3870849609375,
"KL/std": 236.972412109375,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.002540252171456814,
"fcm_dpo/delta": -0.08043041080236435,
"fcm_dpo/margin": 265.5389709472656,
"fcm_dpo/q_t": 0.34851759672164917,
"grad_norm": 28.92422866821289,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -0.4522515535354614,
"logits/rejected": -0.4544093608856201,
"logps/chosen": -373.04046630859375,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -682.4794921875,
"loss": 0.9085,
"margin_dpo/margin_mean": 265.5389709472656,
"margin_dpo/margin_std": 243.24615478515625,
"step": 568
},
{
"KL/chosen_KL_mean": -382.8591003417969,
"KL/mean": -483.44317626953125,
"KL/rejected_KL_mean": -584.0272827148438,
"KL/std": 253.66726684570312,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0025851712562143803,
"fcm_dpo/delta": 0.08309976756572723,
"fcm_dpo/margin": 201.1681671142578,
"fcm_dpo/q_t": 0.38607293367385864,
"grad_norm": 41.039772033691406,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -0.4414858818054199,
"logits/rejected": -0.4261478781700134,
"logps/chosen": -446.57965087890625,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -663.1304931640625,
"loss": 1.0442,
"margin_dpo/margin_mean": 201.1681671142578,
"margin_dpo/margin_std": 268.3058166503906,
"step": 569
},
{
"KL/chosen_KL_mean": -344.12200927734375,
"KL/mean": -465.589599609375,
"KL/rejected_KL_mean": -587.0572509765625,
"KL/std": 237.9881134033203,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0025622902903705835,
"fcm_dpo/delta": -0.024312295019626617,
"fcm_dpo/margin": 242.93524169921875,
"fcm_dpo/q_t": 0.36168766021728516,
"grad_norm": 27.275033950805664,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -0.4918212294578552,
"logits/rejected": -0.4713660478591919,
"logps/chosen": -405.73651123046875,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -669.1990966796875,
"loss": 0.942,
"margin_dpo/margin_mean": 242.93527221679688,
"margin_dpo/margin_std": 238.26451110839844,
"step": 570
},
{
"KL/chosen_KL_mean": -344.68499755859375,
"KL/mean": -476.9786071777344,
"KL/rejected_KL_mean": -609.272216796875,
"KL/std": 251.47467041015625,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.002527458593249321,
"fcm_dpo/delta": -0.07453502714633942,
"fcm_dpo/margin": 264.587158203125,
"fcm_dpo/q_t": 0.3549273610115051,
"grad_norm": 37.5210075378418,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -0.4558071494102478,
"logits/rejected": -0.4565303921699524,
"logps/chosen": -397.73907470703125,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -700.6090087890625,
"loss": 0.9521,
"margin_dpo/margin_mean": 264.587158203125,
"margin_dpo/margin_std": 300.0632019042969,
"step": 571
},
{
"KL/chosen_KL_mean": -380.70806884765625,
"KL/mean": -502.6474609375,
"KL/rejected_KL_mean": -624.5869140625,
"KL/std": 286.9884948730469,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.0025160368531942368,
"fcm_dpo/delta": -0.014791365712881088,
"fcm_dpo/margin": 243.87879943847656,
"fcm_dpo/q_t": 0.37445491552352905,
"grad_norm": 33.4094123840332,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -0.4040576219558716,
"logits/rejected": -0.3991444408893585,
"logps/chosen": -429.1673583984375,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -708.1439208984375,
"loss": 1.0136,
"margin_dpo/margin_mean": 243.87879943847656,
"margin_dpo/margin_std": 338.6883239746094,
"step": 572
},
{
"KL/chosen_KL_mean": -358.3140563964844,
"KL/mean": -474.03253173828125,
"KL/rejected_KL_mean": -589.7509765625,
"KL/std": 236.28445434570312,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0025425199419260025,
"fcm_dpo/delta": 0.011455178260803223,
"fcm_dpo/margin": 231.4369354248047,
"fcm_dpo/q_t": 0.36963897943496704,
"grad_norm": 34.65782165527344,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -0.46382439136505127,
"logits/rejected": -0.44606298208236694,
"logps/chosen": -409.93670654296875,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -675.0759887695312,
"loss": 0.9739,
"margin_dpo/margin_mean": 231.4369354248047,
"margin_dpo/margin_std": 248.86544799804688,
"step": 573
},
{
"KL/chosen_KL_mean": -347.3360595703125,
"KL/mean": -479.1291809082031,
"KL/rejected_KL_mean": -610.9222412109375,
"KL/std": 240.4398956298828,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.002510279882699251,
"fcm_dpo/delta": -0.06601053476333618,
"fcm_dpo/margin": 263.58624267578125,
"fcm_dpo/q_t": 0.35497695207595825,
"grad_norm": 28.976726531982422,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -0.5355557799339294,
"logits/rejected": -0.5372272729873657,
"logps/chosen": -398.3805236816406,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -703.7286376953125,
"loss": 0.9362,
"margin_dpo/margin_mean": 263.5862121582031,
"margin_dpo/margin_std": 275.7353515625,
"step": 574
},
{
"KL/chosen_KL_mean": -390.5448303222656,
"KL/mean": -500.8680114746094,
"KL/rejected_KL_mean": -611.191162109375,
"KL/std": 261.77294921875,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.002488768193870783,
"fcm_dpo/delta": 0.05210364609956741,
"fcm_dpo/margin": 220.6463623046875,
"fcm_dpo/q_t": 0.38091862201690674,
"grad_norm": 36.14547348022461,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.4399763345718384,
"logits/rejected": -0.4319089651107788,
"logps/chosen": -462.3349609375,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -706.577392578125,
"loss": 1.0573,
"margin_dpo/margin_mean": 220.6463623046875,
"margin_dpo/margin_std": 306.8372802734375,
"step": 575
},
{
"KL/chosen_KL_mean": -356.26641845703125,
"KL/mean": -462.8583984375,
"KL/rejected_KL_mean": -569.4503784179688,
"KL/std": 242.19851684570312,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.0025273626670241356,
"fcm_dpo/delta": 0.06407326459884644,
"fcm_dpo/margin": 213.18392944335938,
"fcm_dpo/q_t": 0.3830186724662781,
"grad_norm": 33.95637893676758,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -0.49390989542007446,
"logits/rejected": -0.5026179552078247,
"logps/chosen": -410.5294189453125,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -670.2046508789062,
"loss": 1.0416,
"margin_dpo/margin_mean": 213.18392944335938,
"margin_dpo/margin_std": 285.66363525390625,
"step": 576
},
{
"KL/chosen_KL_mean": -321.35223388671875,
"KL/mean": -442.01336669921875,
"KL/rejected_KL_mean": -562.6744995117188,
"KL/std": 237.555908203125,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.002548405434936285,
"fcm_dpo/delta": -0.015967031940817833,
"fcm_dpo/margin": 241.322265625,
"fcm_dpo/q_t": 0.36109089851379395,
"grad_norm": 31.1567440032959,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -0.47257912158966064,
"logits/rejected": -0.44047021865844727,
"logps/chosen": -383.2619323730469,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -646.7452392578125,
"loss": 0.9502,
"margin_dpo/margin_mean": 241.322265625,
"margin_dpo/margin_std": 234.94622802734375,
"step": 577
},
{
"KL/chosen_KL_mean": -319.2591247558594,
"KL/mean": -443.0003967285156,
"KL/rejected_KL_mean": -566.7416381835938,
"KL/std": 241.314453125,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.0025177106726914644,
"fcm_dpo/delta": -0.0259145125746727,
"fcm_dpo/margin": 247.4824981689453,
"fcm_dpo/q_t": 0.35859817266464233,
"grad_norm": 30.354848861694336,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.46608057618141174,
"logits/rejected": -0.46288034319877625,
"logps/chosen": -368.5228271484375,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -650.1778564453125,
"loss": 0.9297,
"margin_dpo/margin_mean": 247.4824981689453,
"margin_dpo/margin_std": 217.65599060058594,
"step": 578
},
{
"KL/chosen_KL_mean": -338.1419982910156,
"KL/mean": -440.29400634765625,
"KL/rejected_KL_mean": -542.446044921875,
"KL/std": 235.01004028320312,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.002561987843364477,
"fcm_dpo/delta": 0.0808427631855011,
"fcm_dpo/margin": 204.3040771484375,
"fcm_dpo/q_t": 0.3831888437271118,
"grad_norm": 33.36688232421875,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.4724149703979492,
"logits/rejected": -0.46301060914993286,
"logps/chosen": -397.02783203125,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -637.233642578125,
"loss": 1.0232,
"margin_dpo/margin_mean": 204.3040771484375,
"margin_dpo/margin_std": 250.47772216796875,
"step": 579
},
{
"KL/chosen_KL_mean": -271.50054931640625,
"KL/mean": -407.8179931640625,
"KL/rejected_KL_mean": -544.135498046875,
"KL/std": 254.89610290527344,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0025263652205467224,
"fcm_dpo/delta": -0.09617502242326736,
"fcm_dpo/margin": 272.63494873046875,
"fcm_dpo/q_t": 0.35097259283065796,
"grad_norm": 25.02663803100586,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.44938066601753235,
"logits/rejected": -0.4464479088783264,
"logps/chosen": -320.2073974609375,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -625.8938598632812,
"loss": 0.926,
"margin_dpo/margin_mean": 272.63494873046875,
"margin_dpo/margin_std": 293.03460693359375,
"step": 580
},
{
"KL/chosen_KL_mean": -364.4176025390625,
"KL/mean": -461.87799072265625,
"KL/rejected_KL_mean": -559.33837890625,
"KL/std": 235.9238739013672,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0025644898414611816,
"fcm_dpo/delta": 0.10541276633739471,
"fcm_dpo/margin": 194.92074584960938,
"fcm_dpo/q_t": 0.3889443576335907,
"grad_norm": 33.31412887573242,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -0.5112712979316711,
"logits/rejected": -0.5016433596611023,
"logps/chosen": -426.63995361328125,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -651.0740356445312,
"loss": 1.0392,
"margin_dpo/margin_mean": 194.92074584960938,
"margin_dpo/margin_std": 246.52740478515625,
"step": 581
},
{
"KL/chosen_KL_mean": -306.1158142089844,
"KL/mean": -439.4873352050781,
"KL/rejected_KL_mean": -572.85888671875,
"KL/std": 247.38870239257812,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0025679708924144506,
"fcm_dpo/delta": -0.09219370782375336,
"fcm_dpo/margin": 266.7430419921875,
"fcm_dpo/q_t": 0.3502892255783081,
"grad_norm": 34.65846633911133,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -0.45925596356391907,
"logits/rejected": -0.4669743776321411,
"logps/chosen": -364.344482421875,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -682.928466796875,
"loss": 0.9376,
"margin_dpo/margin_mean": 266.7430419921875,
"margin_dpo/margin_std": 279.8547668457031,
"step": 582
},
{
"KL/chosen_KL_mean": -350.44207763671875,
"KL/mean": -447.0964660644531,
"KL/rejected_KL_mean": -543.7508544921875,
"KL/std": 237.7489013671875,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0025529933627694845,
"fcm_dpo/delta": 0.11152487248182297,
"fcm_dpo/margin": 193.30885314941406,
"fcm_dpo/q_t": 0.39500147104263306,
"grad_norm": 35.25895309448242,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.46326497197151184,
"logits/rejected": -0.4501265287399292,
"logps/chosen": -407.304931640625,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -632.15478515625,
"loss": 1.0701,
"margin_dpo/margin_mean": 193.30885314941406,
"margin_dpo/margin_std": 279.12738037109375,
"step": 583
},
{
"KL/chosen_KL_mean": -298.342529296875,
"KL/mean": -445.67437744140625,
"KL/rejected_KL_mean": -593.0062255859375,
"KL/std": 244.7476043701172,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0025257887318730354,
"fcm_dpo/delta": -0.1561448872089386,
"fcm_dpo/margin": 294.6636962890625,
"fcm_dpo/q_t": 0.3356255292892456,
"grad_norm": 29.968629837036133,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -0.4433717727661133,
"logits/rejected": -0.430498331785202,
"logps/chosen": -355.2431945800781,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -690.6422729492188,
"loss": 0.8593,
"margin_dpo/margin_mean": 294.6636962890625,
"margin_dpo/margin_std": 250.9517822265625,
"step": 584
},
{
"KL/chosen_KL_mean": -357.19744873046875,
"KL/mean": -474.2387390136719,
"KL/rejected_KL_mean": -591.280029296875,
"KL/std": 245.98831176757812,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0025128626730293036,
"fcm_dpo/delta": 0.012520798482000828,
"fcm_dpo/margin": 234.08258056640625,
"fcm_dpo/q_t": 0.3720248341560364,
"grad_norm": 38.159202575683594,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -0.493826687335968,
"logits/rejected": -0.48273766040802,
"logps/chosen": -418.17144775390625,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -675.4495849609375,
"loss": 0.9903,
"margin_dpo/margin_mean": 234.0825653076172,
"margin_dpo/margin_std": 269.5777587890625,
"step": 585
},
{
"KL/chosen_KL_mean": -357.32025146484375,
"KL/mean": -457.6875,
"KL/rejected_KL_mean": -558.0547485351562,
"KL/std": 230.83807373046875,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0025516143068671227,
"fcm_dpo/delta": 0.09249043464660645,
"fcm_dpo/margin": 200.73446655273438,
"fcm_dpo/q_t": 0.388335645198822,
"grad_norm": 32.51168441772461,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -0.46170011162757874,
"logits/rejected": -0.4636007845401764,
"logps/chosen": -414.1758728027344,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -649.8573608398438,
"loss": 1.0476,
"margin_dpo/margin_mean": 200.73448181152344,
"margin_dpo/margin_std": 273.436767578125,
"step": 586
},
{
"KL/chosen_KL_mean": -276.0436096191406,
"KL/mean": -390.320068359375,
"KL/rejected_KL_mean": -504.5965270996094,
"KL/std": 260.1427307128906,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0025694689247757196,
"fcm_dpo/delta": 0.013513934798538685,
"fcm_dpo/margin": 228.55291748046875,
"fcm_dpo/q_t": 0.3707025647163391,
"grad_norm": 28.48300552368164,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -0.5027564764022827,
"logits/rejected": -0.5109248161315918,
"logps/chosen": -320.7352294921875,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -587.2203979492188,
"loss": 0.9846,
"margin_dpo/margin_mean": 228.5529327392578,
"margin_dpo/margin_std": 264.0683288574219,
"step": 587
},
{
"KL/chosen_KL_mean": -341.34332275390625,
"KL/mean": -453.5398864746094,
"KL/rejected_KL_mean": -565.7364501953125,
"KL/std": 241.7613067626953,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.002566031413152814,
"fcm_dpo/delta": 0.025277376174926758,
"fcm_dpo/margin": 224.39308166503906,
"fcm_dpo/q_t": 0.37462854385375977,
"grad_norm": 32.38241195678711,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -0.5227458477020264,
"logits/rejected": -0.5319730043411255,
"logps/chosen": -391.6383056640625,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -673.1063232421875,
"loss": 0.993,
"margin_dpo/margin_mean": 224.39308166503906,
"margin_dpo/margin_std": 263.5064697265625,
"step": 588
},
{
"KL/chosen_KL_mean": -322.74249267578125,
"KL/mean": -471.30023193359375,
"KL/rejected_KL_mean": -619.8580322265625,
"KL/std": 273.79931640625,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0025234550703316927,
"fcm_dpo/delta": -0.16177603602409363,
"fcm_dpo/margin": 297.1155090332031,
"fcm_dpo/q_t": 0.33872103691101074,
"grad_norm": 26.75434112548828,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -0.46640634536743164,
"logits/rejected": -0.45287472009658813,
"logps/chosen": -382.67242431640625,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -731.513427734375,
"loss": 0.8868,
"margin_dpo/margin_mean": 297.1155090332031,
"margin_dpo/margin_std": 290.9627380371094,
"step": 589
},
{
"KL/chosen_KL_mean": -307.5690002441406,
"KL/mean": -440.033203125,
"KL/rejected_KL_mean": -572.4974365234375,
"KL/std": 248.03265380859375,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.002473164349794388,
"fcm_dpo/delta": -0.05929313227534294,
"fcm_dpo/margin": 264.92840576171875,
"fcm_dpo/q_t": 0.3544720709323883,
"grad_norm": 31.879913330078125,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -0.5063580274581909,
"logits/rejected": -0.5088971257209778,
"logps/chosen": -363.3787841796875,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -678.5602416992188,
"loss": 0.9271,
"margin_dpo/margin_mean": 264.92840576171875,
"margin_dpo/margin_std": 261.29254150390625,
"step": 590
},
{
"KL/chosen_KL_mean": -318.635986328125,
"KL/mean": -444.52264404296875,
"KL/rejected_KL_mean": -570.4093017578125,
"KL/std": 231.72718811035156,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0024611647240817547,
"fcm_dpo/delta": -0.02098141796886921,
"fcm_dpo/margin": 251.77334594726562,
"fcm_dpo/q_t": 0.3605474829673767,
"grad_norm": 30.629581451416016,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -0.49752044677734375,
"logits/rejected": -0.46622711420059204,
"logps/chosen": -374.8765869140625,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -654.1956176757812,
"loss": 0.9423,
"margin_dpo/margin_mean": 251.7733612060547,
"margin_dpo/margin_std": 246.964599609375,
"step": 591
},
{
"KL/chosen_KL_mean": -354.6044921875,
"KL/mean": -467.9051513671875,
"KL/rejected_KL_mean": -581.2058715820312,
"KL/std": 252.9598388671875,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0024665952660143375,
"fcm_dpo/delta": 0.04335062950849533,
"fcm_dpo/margin": 226.60137939453125,
"fcm_dpo/q_t": 0.37739434838294983,
"grad_norm": 32.139060974121094,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -0.42924797534942627,
"logits/rejected": -0.41100186109542847,
"logps/chosen": -402.54473876953125,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -656.938720703125,
"loss": 1.0539,
"margin_dpo/margin_mean": 226.6013946533203,
"margin_dpo/margin_std": 328.1856689453125,
"step": 592
},
{
"KL/chosen_KL_mean": -309.3241882324219,
"KL/mean": -396.719482421875,
"KL/rejected_KL_mean": -484.1147766113281,
"KL/std": 230.8219757080078,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0025403089821338654,
"fcm_dpo/delta": 0.16337433457374573,
"fcm_dpo/margin": 174.79055786132812,
"fcm_dpo/q_t": 0.4015771746635437,
"grad_norm": 33.59453582763672,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.4543134272098541,
"logits/rejected": -0.43490511178970337,
"logps/chosen": -358.01495361328125,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -545.0227661132812,
"loss": 1.0885,
"margin_dpo/margin_mean": 174.79054260253906,
"margin_dpo/margin_std": 252.38653564453125,
"step": 593
},
{
"KL/chosen_KL_mean": -299.691650390625,
"KL/mean": -416.93017578125,
"KL/rejected_KL_mean": -534.168701171875,
"KL/std": 269.12261962890625,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0025944348890334368,
"fcm_dpo/delta": -0.012455061078071594,
"fcm_dpo/margin": 234.47711181640625,
"fcm_dpo/q_t": 0.36959269642829895,
"grad_norm": 31.555274963378906,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -0.5038938522338867,
"logits/rejected": -0.4825102686882019,
"logps/chosen": -354.62652587890625,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -620.2684326171875,
"loss": 1.002,
"margin_dpo/margin_mean": 234.47711181640625,
"margin_dpo/margin_std": 283.622802734375,
"step": 594
},
{
"KL/chosen_KL_mean": -308.0903015136719,
"KL/mean": -418.8306579589844,
"KL/rejected_KL_mean": -529.571044921875,
"KL/std": 230.4139404296875,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0025640493258833885,
"fcm_dpo/delta": 0.03399317339062691,
"fcm_dpo/margin": 221.480712890625,
"fcm_dpo/q_t": 0.37698057293891907,
"grad_norm": 25.988157272338867,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -0.4751130938529968,
"logits/rejected": -0.4569569230079651,
"logps/chosen": -357.5107421875,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -610.1983642578125,
"loss": 0.9881,
"margin_dpo/margin_mean": 221.480712890625,
"margin_dpo/margin_std": 258.1260070800781,
"step": 595
},
{
"KL/chosen_KL_mean": -316.2357177734375,
"KL/mean": -413.747314453125,
"KL/rejected_KL_mean": -511.2588806152344,
"KL/std": 236.9102783203125,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.002618124010041356,
"fcm_dpo/delta": 0.09384307265281677,
"fcm_dpo/margin": 195.02316284179688,
"fcm_dpo/q_t": 0.3880764842033386,
"grad_norm": 33.03876495361328,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -0.47426873445510864,
"logits/rejected": -0.44393420219421387,
"logps/chosen": -376.02740478515625,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -591.3499755859375,
"loss": 1.0532,
"margin_dpo/margin_mean": 195.02317810058594,
"margin_dpo/margin_std": 271.96826171875,
"step": 596
},
{
"KL/chosen_KL_mean": -319.32891845703125,
"KL/mean": -450.066162109375,
"KL/rejected_KL_mean": -580.803466796875,
"KL/std": 264.26776123046875,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.002567911520600319,
"fcm_dpo/delta": -0.07938927412033081,
"fcm_dpo/margin": 261.4744873046875,
"fcm_dpo/q_t": 0.3549570143222809,
"grad_norm": 34.755714416503906,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.48420071601867676,
"logits/rejected": -0.4970840811729431,
"logps/chosen": -376.5897216796875,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -681.4971923828125,
"loss": 0.9385,
"margin_dpo/margin_mean": 261.4744873046875,
"margin_dpo/margin_std": 277.08123779296875,
"step": 597
},
{
"KL/chosen_KL_mean": -335.350830078125,
"KL/mean": -449.6180419921875,
"KL/rejected_KL_mean": -563.88525390625,
"KL/std": 240.95411682128906,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.002562467474490404,
"fcm_dpo/delta": 0.013387367129325867,
"fcm_dpo/margin": 228.53445434570312,
"fcm_dpo/q_t": 0.3685457706451416,
"grad_norm": 36.268619537353516,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -0.47771960496902466,
"logits/rejected": -0.46360164880752563,
"logps/chosen": -387.86932373046875,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -653.3291015625,
"loss": 0.9969,
"margin_dpo/margin_mean": 228.53448486328125,
"margin_dpo/margin_std": 261.8272705078125,
"step": 598
},
{
"KL/chosen_KL_mean": -303.513916015625,
"KL/mean": -407.4415283203125,
"KL/rejected_KL_mean": -511.3691711425781,
"KL/std": 229.56861877441406,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0026012700982391834,
"fcm_dpo/delta": 0.06252237409353256,
"fcm_dpo/margin": 207.85528564453125,
"fcm_dpo/q_t": 0.3765287399291992,
"grad_norm": 30.653757095336914,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.4435848593711853,
"logits/rejected": -0.4369887113571167,
"logps/chosen": -353.31658935546875,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -594.34765625,
"loss": 0.9913,
"margin_dpo/margin_mean": 207.85528564453125,
"margin_dpo/margin_std": 213.85324096679688,
"step": 599
},
{
"KL/chosen_KL_mean": -352.6791076660156,
"KL/mean": -463.65447998046875,
"KL/rejected_KL_mean": -574.6298828125,
"KL/std": 233.15611267089844,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.002624728251248598,
"fcm_dpo/delta": 0.01848490536212921,
"fcm_dpo/margin": 221.95077514648438,
"fcm_dpo/q_t": 0.3731113076210022,
"grad_norm": 39.98947525024414,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -0.5003535747528076,
"logits/rejected": -0.48408395051956177,
"logps/chosen": -419.1139831542969,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -660.0863647460938,
"loss": 1.0024,
"margin_dpo/margin_mean": 221.95077514648438,
"margin_dpo/margin_std": 274.4659423828125,
"step": 600
},
{
"KL/chosen_KL_mean": -363.517578125,
"KL/mean": -482.94793701171875,
"KL/rejected_KL_mean": -602.3783569335938,
"KL/std": 264.3414306640625,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.002612743526697159,
"fcm_dpo/delta": -0.026022687554359436,
"fcm_dpo/margin": 238.86073303222656,
"fcm_dpo/q_t": 0.36359232664108276,
"grad_norm": 36.40314865112305,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -0.49644967913627625,
"logits/rejected": -0.48629212379455566,
"logps/chosen": -422.65118408203125,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -697.0692749023438,
"loss": 0.9685,
"margin_dpo/margin_mean": 238.86074829101562,
"margin_dpo/margin_std": 273.97564697265625,
"step": 601
},
{
"KL/chosen_KL_mean": -264.76422119140625,
"KL/mean": -402.4107666015625,
"KL/rejected_KL_mean": -540.0573120117188,
"KL/std": 240.82546997070312,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0025822389870882034,
"fcm_dpo/delta": -0.11920210719108582,
"fcm_dpo/margin": 275.2930908203125,
"fcm_dpo/q_t": 0.34339991211891174,
"grad_norm": 33.56421661376953,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -0.5091708898544312,
"logits/rejected": -0.5036940574645996,
"logps/chosen": -313.35772705078125,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -627.725830078125,
"loss": 0.8889,
"margin_dpo/margin_mean": 275.2930908203125,
"margin_dpo/margin_std": 253.09454345703125,
"step": 602
},
{
"KL/chosen_KL_mean": -302.5773620605469,
"KL/mean": -423.4753723144531,
"KL/rejected_KL_mean": -544.3734130859375,
"KL/std": 249.7156524658203,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0025528722908347845,
"fcm_dpo/delta": -0.018420854583382607,
"fcm_dpo/margin": 241.7960205078125,
"fcm_dpo/q_t": 0.36729931831359863,
"grad_norm": 40.8131103515625,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -0.514338493347168,
"logits/rejected": -0.4906577467918396,
"logps/chosen": -372.9919738769531,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -644.698974609375,
"loss": 0.9773,
"margin_dpo/margin_mean": 241.7960205078125,
"margin_dpo/margin_std": 286.18743896484375,
"step": 603
},
{
"KL/chosen_KL_mean": -280.10443115234375,
"KL/mean": -420.9599609375,
"KL/rejected_KL_mean": -561.8154907226562,
"KL/std": 244.97970581054688,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0024972236715257168,
"fcm_dpo/delta": -0.11190646886825562,
"fcm_dpo/margin": 281.7110595703125,
"fcm_dpo/q_t": 0.34456294775009155,
"grad_norm": 29.565893173217773,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -0.47602221369743347,
"logits/rejected": -0.47603946924209595,
"logps/chosen": -326.5625,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -653.6699829101562,
"loss": 0.9118,
"margin_dpo/margin_mean": 281.7110595703125,
"margin_dpo/margin_std": 280.1474304199219,
"step": 604
},
{
"KL/chosen_KL_mean": -337.17523193359375,
"KL/mean": -451.63116455078125,
"KL/rejected_KL_mean": -566.0870361328125,
"KL/std": 239.23190307617188,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0024927170015871525,
"fcm_dpo/delta": 0.03097914718091488,
"fcm_dpo/margin": 228.91180419921875,
"fcm_dpo/q_t": 0.378231942653656,
"grad_norm": 31.081411361694336,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -0.4843239188194275,
"logits/rejected": -0.4782636761665344,
"logps/chosen": -403.4245910644531,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -668.3920288085938,
"loss": 0.9991,
"margin_dpo/margin_mean": 228.9118194580078,
"margin_dpo/margin_std": 287.41900634765625,
"step": 605
},
{
"KL/chosen_KL_mean": -309.5989685058594,
"KL/mean": -431.1405029296875,
"KL/rejected_KL_mean": -552.6820068359375,
"KL/std": 232.6980743408203,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.002521621063351631,
"fcm_dpo/delta": -0.015105456113815308,
"fcm_dpo/margin": 243.0830535888672,
"fcm_dpo/q_t": 0.36578673124313354,
"grad_norm": 35.92807388305664,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -0.43920066952705383,
"logits/rejected": -0.43256157636642456,
"logps/chosen": -364.4180908203125,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -651.053466796875,
"loss": 0.961,
"margin_dpo/margin_mean": 243.0830535888672,
"margin_dpo/margin_std": 257.806396484375,
"step": 606
},
{
"KL/chosen_KL_mean": -323.2801818847656,
"KL/mean": -426.964111328125,
"KL/rejected_KL_mean": -530.6480102539062,
"KL/std": 230.68255615234375,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.002536382991820574,
"fcm_dpo/delta": 0.0776420533657074,
"fcm_dpo/margin": 207.36782836914062,
"fcm_dpo/q_t": 0.38285157084465027,
"grad_norm": 26.080936431884766,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.5036798715591431,
"logits/rejected": -0.4835021495819092,
"logps/chosen": -381.3642272949219,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -610.4251098632812,
"loss": 1.0117,
"margin_dpo/margin_mean": 207.36782836914062,
"margin_dpo/margin_std": 236.584716796875,
"step": 607
},
{
"KL/chosen_KL_mean": -307.5203857421875,
"KL/mean": -422.93292236328125,
"KL/rejected_KL_mean": -538.345458984375,
"KL/std": 223.73690795898438,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.0025482624769210815,
"fcm_dpo/delta": 0.012401629239320755,
"fcm_dpo/margin": 230.82510375976562,
"fcm_dpo/q_t": 0.3668464422225952,
"grad_norm": 29.168594360351562,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -0.5181676149368286,
"logits/rejected": -0.5025119185447693,
"logps/chosen": -364.97119140625,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -633.118896484375,
"loss": 0.9524,
"margin_dpo/margin_mean": 230.82510375976562,
"margin_dpo/margin_std": 218.07489013671875,
"step": 608
},
{
"KL/chosen_KL_mean": -299.547119140625,
"KL/mean": -432.0242919921875,
"KL/rejected_KL_mean": -564.5014038085938,
"KL/std": 261.5111999511719,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.002518679480999708,
"fcm_dpo/delta": -0.07211297005414963,
"fcm_dpo/margin": 264.9543151855469,
"fcm_dpo/q_t": 0.35380125045776367,
"grad_norm": 40.434810638427734,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -0.5193660855293274,
"logits/rejected": -0.5068392753601074,
"logps/chosen": -358.35247802734375,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -653.3173828125,
"loss": 0.9252,
"margin_dpo/margin_mean": 264.9543151855469,
"margin_dpo/margin_std": 276.5964660644531,
"step": 609
},
{
"KL/chosen_KL_mean": -305.972412109375,
"KL/mean": -410.21514892578125,
"KL/rejected_KL_mean": -514.4578857421875,
"KL/std": 217.17178344726562,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0025379827711731195,
"fcm_dpo/delta": 0.07474474608898163,
"fcm_dpo/margin": 208.48550415039062,
"fcm_dpo/q_t": 0.38216960430145264,
"grad_norm": 37.960548400878906,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.5026014447212219,
"logits/rejected": -0.4690030813217163,
"logps/chosen": -371.6674499511719,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -597.86328125,
"loss": 1.0134,
"margin_dpo/margin_mean": 208.48550415039062,
"margin_dpo/margin_std": 242.06829833984375,
"step": 610
},
{
"KL/chosen_KL_mean": -315.580810546875,
"KL/mean": -443.51141357421875,
"KL/rejected_KL_mean": -571.4420166015625,
"KL/std": 241.46829223632812,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0025365548208355904,
"fcm_dpo/delta": -0.052558302879333496,
"fcm_dpo/margin": 255.86117553710938,
"fcm_dpo/q_t": 0.3573808968067169,
"grad_norm": 31.380815505981445,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -0.4880908727645874,
"logits/rejected": -0.48622244596481323,
"logps/chosen": -368.1802978515625,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -657.7730102539062,
"loss": 0.9398,
"margin_dpo/margin_mean": 255.86119079589844,
"margin_dpo/margin_std": 263.89599609375,
"step": 611
},
{
"KL/chosen_KL_mean": -319.124267578125,
"KL/mean": -454.0286865234375,
"KL/rejected_KL_mean": -588.9331665039062,
"KL/std": 225.37466430664062,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.002494979649782181,
"fcm_dpo/delta": -0.07837517559528351,
"fcm_dpo/margin": 269.80889892578125,
"fcm_dpo/q_t": 0.3489903211593628,
"grad_norm": 27.15785026550293,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -0.47580569982528687,
"logits/rejected": -0.45688319206237793,
"logps/chosen": -378.447998046875,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -677.2455444335938,
"loss": 0.9,
"margin_dpo/margin_mean": 269.8088684082031,
"margin_dpo/margin_std": 236.9198455810547,
"step": 612
},
{
"KL/chosen_KL_mean": -303.61602783203125,
"KL/mean": -429.251220703125,
"KL/rejected_KL_mean": -554.886474609375,
"KL/std": 233.84408569335938,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0024748805444687605,
"fcm_dpo/delta": -0.023326223716139793,
"fcm_dpo/margin": 251.2704315185547,
"fcm_dpo/q_t": 0.3593218922615051,
"grad_norm": 34.23261642456055,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -0.5309430360794067,
"logits/rejected": -0.5473231077194214,
"logps/chosen": -363.3459777832031,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -659.9940185546875,
"loss": 0.9241,
"margin_dpo/margin_mean": 251.27044677734375,
"margin_dpo/margin_std": 221.08746337890625,
"step": 613
},
{
"KL/chosen_KL_mean": -311.302001953125,
"KL/mean": -442.5368347167969,
"KL/rejected_KL_mean": -573.7716064453125,
"KL/std": 234.4269561767578,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.002445939928293228,
"fcm_dpo/delta": -0.04508571699261665,
"fcm_dpo/margin": 262.4696044921875,
"fcm_dpo/q_t": 0.35646164417266846,
"grad_norm": 32.860660552978516,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -0.4055151343345642,
"logits/rejected": -0.43201106786727905,
"logps/chosen": -364.2409973144531,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -678.4509887695312,
"loss": 0.9359,
"margin_dpo/margin_mean": 262.4696044921875,
"margin_dpo/margin_std": 260.8283386230469,
"step": 614
},
{
"KL/chosen_KL_mean": -312.30572509765625,
"KL/mean": -415.72662353515625,
"KL/rejected_KL_mean": -519.1474609375,
"KL/std": 242.51028442382812,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.002488232683390379,
"fcm_dpo/delta": 0.0893133282661438,
"fcm_dpo/margin": 206.841796875,
"fcm_dpo/q_t": 0.38539278507232666,
"grad_norm": 29.573793411254883,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -0.5157092809677124,
"logits/rejected": -0.5105499029159546,
"logps/chosen": -378.12298583984375,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -614.3250122070312,
"loss": 1.0145,
"margin_dpo/margin_mean": 206.841796875,
"margin_dpo/margin_std": 235.7509765625,
"step": 615
},
{
"KL/chosen_KL_mean": -376.611083984375,
"KL/mean": -460.47747802734375,
"KL/rejected_KL_mean": -544.3438720703125,
"KL/std": 205.46182250976562,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.002566794864833355,
"fcm_dpo/delta": 0.17601539194583893,
"fcm_dpo/margin": 167.73281860351562,
"fcm_dpo/q_t": 0.4059697985649109,
"grad_norm": 45.11885070800781,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -0.486763060092926,
"logits/rejected": -0.45515745878219604,
"logps/chosen": -441.74395751953125,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -619.04443359375,
"loss": 1.0983,
"margin_dpo/margin_mean": 167.73281860351562,
"margin_dpo/margin_std": 251.27783203125,
"step": 616
},
{
"KL/chosen_KL_mean": -341.61639404296875,
"KL/mean": -425.75830078125,
"KL/rejected_KL_mean": -509.90020751953125,
"KL/std": 205.00189208984375,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.002652436029165983,
"fcm_dpo/delta": 0.15979546308517456,
"fcm_dpo/margin": 168.28378295898438,
"fcm_dpo/q_t": 0.4026752710342407,
"grad_norm": 41.452919006347656,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.4813739061355591,
"logits/rejected": -0.4518812298774719,
"logps/chosen": -404.6219482421875,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -574.13427734375,
"loss": 1.0786,
"margin_dpo/margin_mean": 168.28378295898438,
"margin_dpo/margin_std": 237.3399200439453,
"step": 617
},
{
"KL/chosen_KL_mean": -353.5629577636719,
"KL/mean": -474.975830078125,
"KL/rejected_KL_mean": -596.388671875,
"KL/std": 232.87615966796875,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0026595869567245245,
"fcm_dpo/delta": -0.04942517355084419,
"fcm_dpo/margin": 242.8257598876953,
"fcm_dpo/q_t": 0.35790830850601196,
"grad_norm": 40.33106231689453,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -0.48407021164894104,
"logits/rejected": -0.4712235927581787,
"logps/chosen": -420.664306640625,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -688.5421142578125,
"loss": 0.9545,
"margin_dpo/margin_mean": 242.8257598876953,
"margin_dpo/margin_std": 262.332763671875,
"step": 618
},
{
"KL/chosen_KL_mean": -330.8542785644531,
"KL/mean": -433.07806396484375,
"KL/rejected_KL_mean": -535.3017578125,
"KL/std": 234.81472778320312,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0026678997091948986,
"fcm_dpo/delta": 0.0573110356926918,
"fcm_dpo/margin": 204.447509765625,
"fcm_dpo/q_t": 0.38442230224609375,
"grad_norm": 39.504234313964844,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -0.4488782286643982,
"logits/rejected": -0.4363011121749878,
"logps/chosen": -386.83251953125,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -628.4871826171875,
"loss": 1.0494,
"margin_dpo/margin_mean": 204.447509765625,
"margin_dpo/margin_std": 292.4669189453125,
"step": 619
},
{
"KL/chosen_KL_mean": -317.79608154296875,
"KL/mean": -416.914306640625,
"KL/rejected_KL_mean": -516.032470703125,
"KL/std": 214.69119262695312,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.002682661172002554,
"fcm_dpo/delta": 0.07187919318675995,
"fcm_dpo/margin": 198.23635864257812,
"fcm_dpo/q_t": 0.3857539892196655,
"grad_norm": 40.9443359375,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -0.4630637764930725,
"logits/rejected": -0.4404851794242859,
"logps/chosen": -377.5935974121094,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -594.4432373046875,
"loss": 1.0315,
"margin_dpo/margin_mean": 198.23635864257812,
"margin_dpo/margin_std": 261.8980712890625,
"step": 620
},
{
"KL/chosen_KL_mean": -303.6402587890625,
"KL/mean": -444.4114990234375,
"KL/rejected_KL_mean": -585.1827392578125,
"KL/std": 245.6479949951172,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.002659841440618038,
"fcm_dpo/delta": -0.16074436902999878,
"fcm_dpo/margin": 281.54254150390625,
"fcm_dpo/q_t": 0.335953950881958,
"grad_norm": 37.37346649169922,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -0.45611095428466797,
"logits/rejected": -0.45652633905410767,
"logps/chosen": -357.5740051269531,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -673.55224609375,
"loss": 0.8806,
"margin_dpo/margin_mean": 281.54254150390625,
"margin_dpo/margin_std": 262.3354187011719,
"step": 621
},
{
"KL/chosen_KL_mean": -313.4846496582031,
"KL/mean": -430.1204528808594,
"KL/rejected_KL_mean": -546.7562255859375,
"KL/std": 217.1461181640625,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.002598411403596401,
"fcm_dpo/delta": -0.0076771080493927,
"fcm_dpo/margin": 233.2716064453125,
"fcm_dpo/q_t": 0.3677935004234314,
"grad_norm": 32.72286605834961,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.44756630063056946,
"logits/rejected": -0.42837661504745483,
"logps/chosen": -373.7704772949219,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -632.2750244140625,
"loss": 0.9662,
"margin_dpo/margin_mean": 233.2716064453125,
"margin_dpo/margin_std": 255.68695068359375,
"step": 622
},
{
"KL/chosen_KL_mean": -345.10662841796875,
"KL/mean": -437.4360656738281,
"KL/rejected_KL_mean": -529.7655639648438,
"KL/std": 223.7052001953125,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0026566032320261,
"fcm_dpo/delta": 0.11522351950407028,
"fcm_dpo/margin": 184.65890502929688,
"fcm_dpo/q_t": 0.39521849155426025,
"grad_norm": 39.45439147949219,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -0.46171167492866516,
"logits/rejected": -0.4402577877044678,
"logps/chosen": -409.2635803222656,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -614.8486328125,
"loss": 1.0632,
"margin_dpo/margin_mean": 184.65890502929688,
"margin_dpo/margin_std": 267.49810791015625,
"step": 623
},
{
"KL/chosen_KL_mean": -351.6897888183594,
"KL/mean": -479.20550537109375,
"KL/rejected_KL_mean": -606.72119140625,
"KL/std": 249.18637084960938,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0026464269030839205,
"fcm_dpo/delta": -0.08032120764255524,
"fcm_dpo/margin": 255.03140258789062,
"fcm_dpo/q_t": 0.35118943452835083,
"grad_norm": 35.59111404418945,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -0.534546971321106,
"logits/rejected": -0.5034977197647095,
"logps/chosen": -423.6084289550781,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -703.853271484375,
"loss": 0.9326,
"margin_dpo/margin_mean": 255.0314178466797,
"margin_dpo/margin_std": 265.92669677734375,
"step": 624
},
{
"KL/chosen_KL_mean": -311.61895751953125,
"KL/mean": -431.4346923828125,
"KL/rejected_KL_mean": -551.25048828125,
"KL/std": 238.28775024414062,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0026184874586760998,
"fcm_dpo/delta": -0.029427634552121162,
"fcm_dpo/margin": 239.63156127929688,
"fcm_dpo/q_t": 0.3632763624191284,
"grad_norm": 40.88634490966797,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -0.464103639125824,
"logits/rejected": -0.4457782506942749,
"logps/chosen": -369.96099853515625,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -637.3408203125,
"loss": 0.9452,
"margin_dpo/margin_mean": 239.63156127929688,
"margin_dpo/margin_std": 252.7567901611328,
"step": 625
},
{
"KL/chosen_KL_mean": -350.8088684082031,
"KL/mean": -444.5627136230469,
"KL/rejected_KL_mean": -538.3165283203125,
"KL/std": 248.09658813476562,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0026665350887924433,
"fcm_dpo/delta": 0.10491127520799637,
"fcm_dpo/margin": 187.50767517089844,
"fcm_dpo/q_t": 0.3913063108921051,
"grad_norm": 33.02543640136719,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.46444064378738403,
"logits/rejected": -0.4442490339279175,
"logps/chosen": -425.9214782714844,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -637.5052490234375,
"loss": 1.0422,
"margin_dpo/margin_mean": 187.5076904296875,
"margin_dpo/margin_std": 240.2395477294922,
"step": 626
},
{
"KL/chosen_KL_mean": -274.5546875,
"KL/mean": -411.0032958984375,
"KL/rejected_KL_mean": -547.451904296875,
"KL/std": 277.46063232421875,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0026282784529030323,
"fcm_dpo/delta": -0.1261902004480362,
"fcm_dpo/margin": 272.897216796875,
"fcm_dpo/q_t": 0.3550766706466675,
"grad_norm": 33.26662063598633,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.4557987153530121,
"logits/rejected": -0.4810847043991089,
"logps/chosen": -322.2978515625,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -654.2064208984375,
"loss": 0.9497,
"margin_dpo/margin_mean": 272.8972473144531,
"margin_dpo/margin_std": 340.04736328125,
"step": 627
},
{
"KL/chosen_KL_mean": -328.25946044921875,
"KL/mean": -462.94989013671875,
"KL/rejected_KL_mean": -597.6402587890625,
"KL/std": 246.57537841796875,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0025587864220142365,
"fcm_dpo/delta": -0.09681230783462524,
"fcm_dpo/margin": 269.380859375,
"fcm_dpo/q_t": 0.3493661880493164,
"grad_norm": 29.698474884033203,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -0.47059351205825806,
"logits/rejected": -0.4568195343017578,
"logps/chosen": -388.4423828125,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -699.1949462890625,
"loss": 0.9173,
"margin_dpo/margin_mean": 269.380859375,
"margin_dpo/margin_std": 271.6669921875,
"step": 628
},
{
"KL/chosen_KL_mean": -338.8393859863281,
"KL/mean": -451.6732177734375,
"KL/rejected_KL_mean": -564.5070190429688,
"KL/std": 230.832275390625,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0025612092576920986,
"fcm_dpo/delta": 0.02331921085715294,
"fcm_dpo/margin": 225.66766357421875,
"fcm_dpo/q_t": 0.3691335916519165,
"grad_norm": 36.18876647949219,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -0.5002964735031128,
"logits/rejected": -0.48990294337272644,
"logps/chosen": -403.05291748046875,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -656.1607055664062,
"loss": 0.957,
"margin_dpo/margin_mean": 225.6676483154297,
"margin_dpo/margin_std": 211.6520538330078,
"step": 629
},
{
"KL/chosen_KL_mean": -297.7701416015625,
"KL/mean": -384.1854248046875,
"KL/rejected_KL_mean": -470.6007385253906,
"KL/std": 226.5153045654297,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.002636598190292716,
"fcm_dpo/delta": 0.1505165994167328,
"fcm_dpo/margin": 172.83059692382812,
"fcm_dpo/q_t": 0.40159186720848083,
"grad_norm": 34.727291107177734,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -0.5076784491539001,
"logits/rejected": -0.4807358384132385,
"logps/chosen": -357.0611267089844,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -554.1990356445312,
"loss": 1.0729,
"margin_dpo/margin_mean": 172.83059692382812,
"margin_dpo/margin_std": 244.20230102539062,
"step": 630
},
{
"KL/chosen_KL_mean": -350.65191650390625,
"KL/mean": -450.458984375,
"KL/rejected_KL_mean": -550.2659912109375,
"KL/std": 240.3749237060547,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.0026824441738426685,
"fcm_dpo/delta": 0.06787368655204773,
"fcm_dpo/margin": 199.61410522460938,
"fcm_dpo/q_t": 0.3842621147632599,
"grad_norm": 42.071720123291016,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -0.4504157304763794,
"logits/rejected": -0.4283216595649719,
"logps/chosen": -410.10552978515625,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -631.2175903320312,
"loss": 1.0539,
"margin_dpo/margin_mean": 199.61410522460938,
"margin_dpo/margin_std": 285.14569091796875,
"step": 631
},
{
"KL/chosen_KL_mean": -338.2213134765625,
"KL/mean": -441.7378234863281,
"KL/rejected_KL_mean": -545.25439453125,
"KL/std": 214.22528076171875,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0026929371524602175,
"fcm_dpo/delta": 0.04487365856766701,
"fcm_dpo/margin": 207.03306579589844,
"fcm_dpo/q_t": 0.3782033920288086,
"grad_norm": 40.30519104003906,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -0.4482494592666626,
"logits/rejected": -0.4239126741886139,
"logps/chosen": -399.5728454589844,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -631.41455078125,
"loss": 1.0363,
"margin_dpo/margin_mean": 207.03306579589844,
"margin_dpo/margin_std": 282.43548583984375,
"step": 632
},
{
"KL/chosen_KL_mean": -347.100341796875,
"KL/mean": -431.4462890625,
"KL/rejected_KL_mean": -515.792236328125,
"KL/std": 216.15719604492188,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.0027849669568240643,
"fcm_dpo/delta": 0.13436569273471832,
"fcm_dpo/margin": 168.69190979003906,
"fcm_dpo/q_t": 0.39775997400283813,
"grad_norm": 48.632198333740234,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -0.5110678672790527,
"logits/rejected": -0.5130935311317444,
"logps/chosen": -404.37847900390625,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -607.3761596679688,
"loss": 1.0859,
"margin_dpo/margin_mean": 168.69189453125,
"margin_dpo/margin_std": 250.77316284179688,
"step": 633
},
{
"KL/chosen_KL_mean": -364.3929443359375,
"KL/mean": -457.45916748046875,
"KL/rejected_KL_mean": -550.525390625,
"KL/std": 240.02708435058594,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0028042125049978495,
"fcm_dpo/delta": 0.08229043334722519,
"fcm_dpo/margin": 186.1324920654297,
"fcm_dpo/q_t": 0.38886773586273193,
"grad_norm": 37.74142837524414,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.45846250653266907,
"logits/rejected": -0.4469302296638489,
"logps/chosen": -431.01190185546875,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -657.6510620117188,
"loss": 1.0443,
"margin_dpo/margin_mean": 186.1324920654297,
"margin_dpo/margin_std": 252.10240173339844,
"step": 634
},
{
"KL/chosen_KL_mean": -313.33526611328125,
"KL/mean": -410.5537109375,
"KL/rejected_KL_mean": -507.77215576171875,
"KL/std": 231.60560607910156,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0028456407599151134,
"fcm_dpo/delta": 0.04946630820631981,
"fcm_dpo/margin": 194.43692016601562,
"fcm_dpo/q_t": 0.3800857961177826,
"grad_norm": 35.309593200683594,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -0.5347930192947388,
"logits/rejected": -0.4934510588645935,
"logps/chosen": -387.290771484375,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -590.2725830078125,
"loss": 1.0013,
"margin_dpo/margin_mean": 194.43692016601562,
"margin_dpo/margin_std": 231.68768310546875,
"step": 635
},
{
"KL/chosen_KL_mean": -308.8725280761719,
"KL/mean": -420.11651611328125,
"KL/rejected_KL_mean": -531.3604736328125,
"KL/std": 237.08258056640625,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0028612546157091856,
"fcm_dpo/delta": -0.03991235792636871,
"fcm_dpo/margin": 222.4879608154297,
"fcm_dpo/q_t": 0.36007869243621826,
"grad_norm": 38.703651428222656,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.497921884059906,
"logits/rejected": -0.4824295938014984,
"logps/chosen": -368.5014343261719,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -613.3392944335938,
"loss": 0.9412,
"margin_dpo/margin_mean": 222.48794555664062,
"margin_dpo/margin_std": 227.94198608398438,
"step": 636
},
{
"KL/chosen_KL_mean": -324.9109802246094,
"KL/mean": -444.62640380859375,
"KL/rejected_KL_mean": -564.341796875,
"KL/std": 242.96307373046875,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.002797934925183654,
"fcm_dpo/delta": -0.07512859255075455,
"fcm_dpo/margin": 239.4308624267578,
"fcm_dpo/q_t": 0.35456109046936035,
"grad_norm": 40.22451400756836,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -0.4504227638244629,
"logits/rejected": -0.4486101269721985,
"logps/chosen": -374.56365966796875,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -662.7469482421875,
"loss": 0.9293,
"margin_dpo/margin_mean": 239.4308624267578,
"margin_dpo/margin_std": 249.63214111328125,
"step": 637
},
{
"KL/chosen_KL_mean": -299.7750244140625,
"KL/mean": -414.29718017578125,
"KL/rejected_KL_mean": -528.8193359375,
"KL/std": 222.72183227539062,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0027988585643470287,
"fcm_dpo/delta": -0.04517444223165512,
"fcm_dpo/margin": 229.0443878173828,
"fcm_dpo/q_t": 0.3562184274196625,
"grad_norm": 36.91585159301758,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.4906739294528961,
"logits/rejected": -0.46897947788238525,
"logps/chosen": -357.931640625,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -608.120849609375,
"loss": 0.9205,
"margin_dpo/margin_mean": 229.0443878173828,
"margin_dpo/margin_std": 197.80955505371094,
"step": 638
},
{
"KL/chosen_KL_mean": -362.7268981933594,
"KL/mean": -446.0164489746094,
"KL/rejected_KL_mean": -529.3060302734375,
"KL/std": 205.30084228515625,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.002831963822245598,
"fcm_dpo/delta": 0.13393458724021912,
"fcm_dpo/margin": 166.5791015625,
"fcm_dpo/q_t": 0.39667850732803345,
"grad_norm": 50.02350997924805,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.467229962348938,
"logits/rejected": -0.4253367781639099,
"logps/chosen": -435.05010986328125,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -603.5809326171875,
"loss": 1.0812,
"margin_dpo/margin_mean": 166.5791015625,
"margin_dpo/margin_std": 246.52850341796875,
"step": 639
},
{
"KL/chosen_KL_mean": -327.3379821777344,
"KL/mean": -466.7060546875,
"KL/rejected_KL_mean": -606.07421875,
"KL/std": 246.59332275390625,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0027808882296085358,
"fcm_dpo/delta": -0.1894734799861908,
"fcm_dpo/margin": 278.7362060546875,
"fcm_dpo/q_t": 0.33695322275161743,
"grad_norm": 32.66841506958008,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.47765612602233887,
"logits/rejected": -0.46742579340934753,
"logps/chosen": -383.47235107421875,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -714.67431640625,
"loss": 0.8832,
"margin_dpo/margin_mean": 278.7362060546875,
"margin_dpo/margin_std": 286.120361328125,
"step": 640
},
{
"KL/chosen_KL_mean": -366.77716064453125,
"KL/mean": -467.1834716796875,
"KL/rejected_KL_mean": -567.5897827148438,
"KL/std": 221.5579071044922,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0027435789816081524,
"fcm_dpo/delta": 0.051650192588567734,
"fcm_dpo/margin": 200.81259155273438,
"fcm_dpo/q_t": 0.3824378252029419,
"grad_norm": 34.99888229370117,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -0.46247628331184387,
"logits/rejected": -0.44688767194747925,
"logps/chosen": -431.7740478515625,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -654.5821533203125,
"loss": 1.0317,
"margin_dpo/margin_mean": 200.81259155273438,
"margin_dpo/margin_std": 275.9678955078125,
"step": 641
},
{
"KL/chosen_KL_mean": -340.67962646484375,
"KL/mean": -458.9723815917969,
"KL/rejected_KL_mean": -577.2650756835938,
"KL/std": 265.81060791015625,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0027677242178469896,
"fcm_dpo/delta": -0.06007951498031616,
"fcm_dpo/margin": 236.58541870117188,
"fcm_dpo/q_t": 0.3671950697898865,
"grad_norm": 46.96950149536133,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.4605233669281006,
"logits/rejected": -0.4568091630935669,
"logps/chosen": -406.368896484375,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -687.5071411132812,
"loss": 0.9927,
"margin_dpo/margin_mean": 236.58541870117188,
"margin_dpo/margin_std": 314.0240478515625,
"step": 642
},
{
"KL/chosen_KL_mean": -316.3322448730469,
"KL/mean": -423.9166259765625,
"KL/rejected_KL_mean": -531.5009765625,
"KL/std": 239.37664794921875,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0027169701643288136,
"fcm_dpo/delta": 0.015240654349327087,
"fcm_dpo/margin": 215.16873168945312,
"fcm_dpo/q_t": 0.37040460109710693,
"grad_norm": 35.88221740722656,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -0.49901092052459717,
"logits/rejected": -0.4943028688430786,
"logps/chosen": -368.2822265625,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -618.9692993164062,
"loss": 0.9796,
"margin_dpo/margin_mean": 215.16873168945312,
"margin_dpo/margin_std": 232.90634155273438,
"step": 643
},
{
"KL/chosen_KL_mean": -336.2987365722656,
"KL/mean": -431.2506103515625,
"KL/rejected_KL_mean": -526.2025146484375,
"KL/std": 266.94891357421875,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.002768642734736204,
"fcm_dpo/delta": 0.07841017842292786,
"fcm_dpo/margin": 189.90380859375,
"fcm_dpo/q_t": 0.38879114389419556,
"grad_norm": 47.067527770996094,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -0.4899185299873352,
"logits/rejected": -0.47964680194854736,
"logps/chosen": -395.31640625,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -613.3392333984375,
"loss": 1.0682,
"margin_dpo/margin_mean": 189.90380859375,
"margin_dpo/margin_std": 284.33966064453125,
"step": 644
},
{
"KL/chosen_KL_mean": -333.89794921875,
"KL/mean": -412.976806640625,
"KL/rejected_KL_mean": -492.0556945800781,
"KL/std": 230.67254638671875,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.002834226470440626,
"fcm_dpo/delta": 0.15911011397838593,
"fcm_dpo/margin": 158.15777587890625,
"fcm_dpo/q_t": 0.40882962942123413,
"grad_norm": 46.451168060302734,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.4995105266571045,
"logits/rejected": -0.49413156509399414,
"logps/chosen": -389.7739562988281,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -589.8364868164062,
"loss": 1.1329,
"margin_dpo/margin_mean": 158.15777587890625,
"margin_dpo/margin_std": 284.70428466796875,
"step": 645
},
{
"KL/chosen_KL_mean": -315.4554138183594,
"KL/mean": -415.31024169921875,
"KL/rejected_KL_mean": -515.1650390625,
"KL/std": 191.0074920654297,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.002871452597901225,
"fcm_dpo/delta": 0.027814757078886032,
"fcm_dpo/margin": 199.70956420898438,
"fcm_dpo/q_t": 0.3724924325942993,
"grad_norm": 38.34192657470703,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -0.4435100555419922,
"logits/rejected": -0.4152856469154358,
"logps/chosen": -376.731201171875,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -592.6708374023438,
"loss": 0.98,
"margin_dpo/margin_mean": 199.70956420898438,
"margin_dpo/margin_std": 215.77481079101562,
"step": 646
},
{
"KL/chosen_KL_mean": -301.41009521484375,
"KL/mean": -401.6798095703125,
"KL/rejected_KL_mean": -501.9494934082031,
"KL/std": 237.82244873046875,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.002904860069975257,
"fcm_dpo/delta": 0.01835210807621479,
"fcm_dpo/margin": 200.53939819335938,
"fcm_dpo/q_t": 0.3729194402694702,
"grad_norm": 35.68404769897461,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.47512757778167725,
"logits/rejected": -0.4787760376930237,
"logps/chosen": -356.26251220703125,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -595.4688720703125,
"loss": 1.0031,
"margin_dpo/margin_mean": 200.53939819335938,
"margin_dpo/margin_std": 246.927001953125,
"step": 647
},
{
"KL/chosen_KL_mean": -315.9765625,
"KL/mean": -466.2679748535156,
"KL/rejected_KL_mean": -616.559326171875,
"KL/std": 285.6835632324219,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.002791747683659196,
"fcm_dpo/delta": -0.2609490752220154,
"fcm_dpo/margin": 300.58282470703125,
"fcm_dpo/q_t": 0.3320544958114624,
"grad_norm": 31.358680725097656,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -0.4896717071533203,
"logits/rejected": -0.5023912191390991,
"logps/chosen": -370.1480407714844,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -715.2721557617188,
"loss": 0.8925,
"margin_dpo/margin_mean": 300.58282470703125,
"margin_dpo/margin_std": 347.4580993652344,
"step": 648
},
{
"KL/chosen_KL_mean": -329.440673828125,
"KL/mean": -412.6784973144531,
"KL/rejected_KL_mean": -495.9163513183594,
"KL/std": 219.95831298828125,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0028212075121700764,
"fcm_dpo/delta": 0.13575418293476105,
"fcm_dpo/margin": 166.47567749023438,
"fcm_dpo/q_t": 0.40223830938339233,
"grad_norm": 33.24458312988281,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -0.520743191242218,
"logits/rejected": -0.49551695585250854,
"logps/chosen": -391.9210205078125,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -575.9935302734375,
"loss": 1.0923,
"margin_dpo/margin_mean": 166.4756622314453,
"margin_dpo/margin_std": 261.10113525390625,
"step": 649
},
{
"KL/chosen_KL_mean": -333.716796875,
"KL/mean": -459.0279541015625,
"KL/rejected_KL_mean": -584.339111328125,
"KL/std": 258.6231384277344,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.0027777771465480328,
"fcm_dpo/delta": -0.1037888154387474,
"fcm_dpo/margin": 250.62229919433594,
"fcm_dpo/q_t": 0.3517780601978302,
"grad_norm": 35.88649368286133,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -0.4750511646270752,
"logits/rejected": -0.47141119837760925,
"logps/chosen": -389.80963134765625,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -682.6039428710938,
"loss": 0.9468,
"margin_dpo/margin_mean": 250.62229919433594,
"margin_dpo/margin_std": 290.90777587890625,
"step": 650
},
{
"KL/chosen_KL_mean": -270.616455078125,
"KL/mean": -400.36761474609375,
"KL/rejected_KL_mean": -530.1187744140625,
"KL/std": 247.16122436523438,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0027151613030582666,
"fcm_dpo/delta": -0.11311930418014526,
"fcm_dpo/margin": 259.50225830078125,
"fcm_dpo/q_t": 0.34443673491477966,
"grad_norm": 35.49654769897461,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -0.5052357912063599,
"logits/rejected": -0.5239521265029907,
"logps/chosen": -314.04193115234375,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -630.07666015625,
"loss": 0.8938,
"margin_dpo/margin_mean": 259.50225830078125,
"margin_dpo/margin_std": 238.87933349609375,
"step": 651
},
{
"KL/chosen_KL_mean": -297.96221923828125,
"KL/mean": -413.6481018066406,
"KL/rejected_KL_mean": -529.333984375,
"KL/std": 246.03994750976562,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.002703704871237278,
"fcm_dpo/delta": -0.027318572625517845,
"fcm_dpo/margin": 231.37179565429688,
"fcm_dpo/q_t": 0.36686524748802185,
"grad_norm": 35.50530242919922,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -0.5126116275787354,
"logits/rejected": -0.5054411292076111,
"logps/chosen": -360.5390319824219,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -641.101806640625,
"loss": 0.9724,
"margin_dpo/margin_mean": 231.3717803955078,
"margin_dpo/margin_std": 275.05950927734375,
"step": 652
},
{
"KL/chosen_KL_mean": -347.24725341796875,
"KL/mean": -469.0255432128906,
"KL/rejected_KL_mean": -590.8038330078125,
"KL/std": 263.53521728515625,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.002674461342394352,
"fcm_dpo/delta": -0.05492768809199333,
"fcm_dpo/margin": 243.5565185546875,
"fcm_dpo/q_t": 0.3606552183628082,
"grad_norm": 40.63291549682617,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -0.48466378450393677,
"logits/rejected": -0.48202192783355713,
"logps/chosen": -408.3602294921875,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -694.0534057617188,
"loss": 0.9561,
"margin_dpo/margin_mean": 243.5565185546875,
"margin_dpo/margin_std": 279.782470703125,
"step": 653
},
{
"KL/chosen_KL_mean": -346.2580261230469,
"KL/mean": -453.7851257324219,
"KL/rejected_KL_mean": -561.3121948242188,
"KL/std": 231.9652099609375,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.002668055472895503,
"fcm_dpo/delta": 0.02775077149271965,
"fcm_dpo/margin": 215.05416870117188,
"fcm_dpo/q_t": 0.378677099943161,
"grad_norm": 36.322471618652344,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -0.4930952191352844,
"logits/rejected": -0.48199236392974854,
"logps/chosen": -407.9861755371094,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -660.0859985351562,
"loss": 1.014,
"margin_dpo/margin_mean": 215.05416870117188,
"margin_dpo/margin_std": 285.11566162109375,
"step": 654
},
{
"KL/chosen_KL_mean": -317.988525390625,
"KL/mean": -428.18328857421875,
"KL/rejected_KL_mean": -538.3779907226562,
"KL/std": 246.63006591796875,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0026736059226095676,
"fcm_dpo/delta": 0.011207222938537598,
"fcm_dpo/margin": 220.38946533203125,
"fcm_dpo/q_t": 0.37764763832092285,
"grad_norm": 30.54075050354004,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -0.4403926730155945,
"logits/rejected": -0.4524065852165222,
"logps/chosen": -367.56536865234375,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -636.6697998046875,
"loss": 1.0179,
"margin_dpo/margin_mean": 220.38946533203125,
"margin_dpo/margin_std": 299.84930419921875,
"step": 655
},
{
"KL/chosen_KL_mean": -313.58734130859375,
"KL/mean": -475.5693359375,
"KL/rejected_KL_mean": -637.55126953125,
"KL/std": 272.347412109375,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.002589811570942402,
"fcm_dpo/delta": -0.2604025602340698,
"fcm_dpo/margin": 323.96392822265625,
"fcm_dpo/q_t": 0.3177107572555542,
"grad_norm": 44.00099182128906,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.42910003662109375,
"logits/rejected": -0.43964993953704834,
"logps/chosen": -366.13677978515625,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -751.2259521484375,
"loss": 0.8143,
"margin_dpo/margin_mean": 323.96392822265625,
"margin_dpo/margin_std": 268.235595703125,
"step": 656
},
{
"KL/chosen_KL_mean": -295.01385498046875,
"KL/mean": -443.39251708984375,
"KL/rejected_KL_mean": -591.771240234375,
"KL/std": 277.7879333496094,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0024842522107064724,
"fcm_dpo/delta": -0.1489279419183731,
"fcm_dpo/margin": 296.7573547363281,
"fcm_dpo/q_t": 0.3390940725803375,
"grad_norm": 33.499046325683594,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -0.48829033970832825,
"logits/rejected": -0.49025315046310425,
"logps/chosen": -341.7143859863281,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -689.6861572265625,
"loss": 0.8733,
"margin_dpo/margin_mean": 296.75738525390625,
"margin_dpo/margin_std": 272.75311279296875,
"step": 657
},
{
"KL/chosen_KL_mean": -333.5435791015625,
"KL/mean": -449.63934326171875,
"KL/rejected_KL_mean": -565.735107421875,
"KL/std": 214.7669677734375,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0024820156395435333,
"fcm_dpo/delta": 0.02515328861773014,
"fcm_dpo/margin": 232.19149780273438,
"fcm_dpo/q_t": 0.36943238973617554,
"grad_norm": 32.54161834716797,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -0.4681131839752197,
"logits/rejected": -0.4607064723968506,
"logps/chosen": -394.5018005371094,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -661.674560546875,
"loss": 0.9622,
"margin_dpo/margin_mean": 232.19149780273438,
"margin_dpo/margin_std": 226.57803344726562,
"step": 658
},
{
"KL/chosen_KL_mean": -307.7506408691406,
"KL/mean": -428.04095458984375,
"KL/rejected_KL_mean": -548.3312377929688,
"KL/std": 247.4335479736328,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0024785269051790237,
"fcm_dpo/delta": 0.0036827735602855682,
"fcm_dpo/margin": 240.5806121826172,
"fcm_dpo/q_t": 0.3688896596431732,
"grad_norm": 35.441017150878906,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.47357141971588135,
"logits/rejected": -0.4331105947494507,
"logps/chosen": -384.4936218261719,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -635.8021850585938,
"loss": 0.9655,
"margin_dpo/margin_mean": 240.5806121826172,
"margin_dpo/margin_std": 256.9029541015625,
"step": 659
},
{
"KL/chosen_KL_mean": -314.87939453125,
"KL/mean": -450.32110595703125,
"KL/rejected_KL_mean": -585.7628173828125,
"KL/std": 231.671875,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0024635442532598972,
"fcm_dpo/delta": -0.07210573554039001,
"fcm_dpo/margin": 270.88348388671875,
"fcm_dpo/q_t": 0.35125380754470825,
"grad_norm": 27.654815673828125,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -0.4474917948246002,
"logits/rejected": -0.43462279438972473,
"logps/chosen": -373.92724609375,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -661.722900390625,
"loss": 0.9063,
"margin_dpo/margin_mean": 270.8834533691406,
"margin_dpo/margin_std": 248.21163940429688,
"step": 660
},
{
"KL/chosen_KL_mean": -288.40045166015625,
"KL/mean": -414.3653564453125,
"KL/rejected_KL_mean": -540.3302612304688,
"KL/std": 260.1834716796875,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.002462734468281269,
"fcm_dpo/delta": -0.0227794349193573,
"fcm_dpo/margin": 251.9298095703125,
"fcm_dpo/q_t": 0.3650128245353699,
"grad_norm": 34.58781433105469,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -0.4789769649505615,
"logits/rejected": -0.4543425440788269,
"logps/chosen": -339.07440185546875,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -626.3359375,
"loss": 0.9637,
"margin_dpo/margin_mean": 251.9298095703125,
"margin_dpo/margin_std": 281.8808898925781,
"step": 661
},
{
"KL/chosen_KL_mean": -339.27935791015625,
"KL/mean": -443.5726318359375,
"KL/rejected_KL_mean": -547.8658447265625,
"KL/std": 226.29525756835938,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0024745557457208633,
"fcm_dpo/delta": 0.08837015926837921,
"fcm_dpo/margin": 208.58648681640625,
"fcm_dpo/q_t": 0.38610321283340454,
"grad_norm": 28.420780181884766,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -0.466716468334198,
"logits/rejected": -0.44000500440597534,
"logps/chosen": -408.5404052734375,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -636.9217529296875,
"loss": 1.0372,
"margin_dpo/margin_mean": 208.58651733398438,
"margin_dpo/margin_std": 270.3607177734375,
"step": 662
},
{
"KL/chosen_KL_mean": -309.8995361328125,
"KL/mean": -431.86376953125,
"KL/rejected_KL_mean": -553.8280029296875,
"KL/std": 261.62890625,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.002479162998497486,
"fcm_dpo/delta": -0.0051012299954891205,
"fcm_dpo/margin": 243.92845153808594,
"fcm_dpo/q_t": 0.3681510388851166,
"grad_norm": 28.337770462036133,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -0.5133693218231201,
"logits/rejected": -0.509850025177002,
"logps/chosen": -374.7784423828125,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -667.7532958984375,
"loss": 0.9718,
"margin_dpo/margin_mean": 243.92843627929688,
"margin_dpo/margin_std": 279.33349609375,
"step": 663
},
{
"KL/chosen_KL_mean": -334.58544921875,
"KL/mean": -474.548095703125,
"KL/rejected_KL_mean": -614.5107421875,
"KL/std": 266.8731384277344,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.002444789744913578,
"fcm_dpo/delta": -0.09078652411699295,
"fcm_dpo/margin": 279.92535400390625,
"fcm_dpo/q_t": 0.3520078659057617,
"grad_norm": 33.492576599121094,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -0.4867273271083832,
"logits/rejected": -0.482978880405426,
"logps/chosen": -395.4739074707031,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -720.032470703125,
"loss": 0.942,
"margin_dpo/margin_mean": 279.92535400390625,
"margin_dpo/margin_std": 317.3948059082031,
"step": 664
},
{
"KL/chosen_KL_mean": -311.75750732421875,
"KL/mean": -434.8410339355469,
"KL/rejected_KL_mean": -557.9244995117188,
"KL/std": 222.35098266601562,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0024345549754798412,
"fcm_dpo/delta": 0.0006977906450629234,
"fcm_dpo/margin": 246.16700744628906,
"fcm_dpo/q_t": 0.3660983443260193,
"grad_norm": 29.08194923400879,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -0.45567527413368225,
"logits/rejected": -0.438798725605011,
"logps/chosen": -372.3216552734375,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -642.7333374023438,
"loss": 0.9555,
"margin_dpo/margin_mean": 246.16700744628906,
"margin_dpo/margin_std": 245.62155151367188,
"step": 665
},
{
"KL/chosen_KL_mean": -320.36114501953125,
"KL/mean": -439.16192626953125,
"KL/rejected_KL_mean": -557.9627075195312,
"KL/std": 232.1220703125,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0024367435835301876,
"fcm_dpo/delta": 0.0221006628125906,
"fcm_dpo/margin": 237.60159301757812,
"fcm_dpo/q_t": 0.3684921860694885,
"grad_norm": 31.1075382232666,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.44975703954696655,
"logits/rejected": -0.4331943690776825,
"logps/chosen": -384.7810974121094,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -653.8543701171875,
"loss": 0.9679,
"margin_dpo/margin_mean": 237.60159301757812,
"margin_dpo/margin_std": 237.33157348632812,
"step": 666
},
{
"KL/chosen_KL_mean": -336.4880065917969,
"KL/mean": -456.1553039550781,
"KL/rejected_KL_mean": -575.822509765625,
"KL/std": 231.285400390625,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.0024537737481296062,
"fcm_dpo/delta": 0.013521149754524231,
"fcm_dpo/margin": 239.3345489501953,
"fcm_dpo/q_t": 0.36680376529693604,
"grad_norm": 33.70623779296875,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.46697884798049927,
"logits/rejected": -0.43444469571113586,
"logps/chosen": -405.7650451660156,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -663.6580200195312,
"loss": 0.9517,
"margin_dpo/margin_mean": 239.33453369140625,
"margin_dpo/margin_std": 224.86497497558594,
"step": 667
},
{
"KL/chosen_KL_mean": -347.86700439453125,
"KL/mean": -456.05706787109375,
"KL/rejected_KL_mean": -564.2471313476562,
"KL/std": 243.4611053466797,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0024927272461354733,
"fcm_dpo/delta": 0.06305651366710663,
"fcm_dpo/margin": 216.380126953125,
"fcm_dpo/q_t": 0.38239577412605286,
"grad_norm": 47.08059310913086,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -0.5003365874290466,
"logits/rejected": -0.4901411533355713,
"logps/chosen": -420.47100830078125,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -667.9862060546875,
"loss": 1.0232,
"margin_dpo/margin_mean": 216.380126953125,
"margin_dpo/margin_std": 274.2818908691406,
"step": 668
},
{
"KL/chosen_KL_mean": -281.59539794921875,
"KL/mean": -414.58941650390625,
"KL/rejected_KL_mean": -547.58349609375,
"KL/std": 242.7149658203125,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0024631847627460957,
"fcm_dpo/delta": -0.05909912288188934,
"fcm_dpo/margin": 265.98809814453125,
"fcm_dpo/q_t": 0.3539808988571167,
"grad_norm": 29.60248565673828,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -0.44813352823257446,
"logits/rejected": -0.42705005407333374,
"logps/chosen": -327.7117919921875,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -625.5078125,
"loss": 0.934,
"margin_dpo/margin_mean": 265.98809814453125,
"margin_dpo/margin_std": 273.38037109375,
"step": 669
},
{
"KL/chosen_KL_mean": -273.85888671875,
"KL/mean": -416.1429443359375,
"KL/rejected_KL_mean": -558.427001953125,
"KL/std": 248.38107299804688,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.002432377077639103,
"fcm_dpo/delta": -0.09899632632732391,
"fcm_dpo/margin": 284.568115234375,
"fcm_dpo/q_t": 0.34792906045913696,
"grad_norm": 28.793840408325195,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.4715420603752136,
"logits/rejected": -0.46402472257614136,
"logps/chosen": -336.20465087890625,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -655.3675537109375,
"loss": 0.9092,
"margin_dpo/margin_mean": 284.568115234375,
"margin_dpo/margin_std": 276.55712890625,
"step": 670
},
{
"KL/chosen_KL_mean": -332.1285400390625,
"KL/mean": -450.88861083984375,
"KL/rejected_KL_mean": -569.648681640625,
"KL/std": 220.6668701171875,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.002431043889373541,
"fcm_dpo/delta": 0.023196790367364883,
"fcm_dpo/margin": 237.52011108398438,
"fcm_dpo/q_t": 0.3722766637802124,
"grad_norm": 30.9722957611084,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -0.4862244129180908,
"logits/rejected": -0.48458704352378845,
"logps/chosen": -380.1286315917969,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -653.4679565429688,
"loss": 0.9869,
"margin_dpo/margin_mean": 237.52011108398438,
"margin_dpo/margin_std": 266.9666748046875,
"step": 671
},
{
"KL/chosen_KL_mean": -356.54400634765625,
"KL/mean": -466.35333251953125,
"KL/rejected_KL_mean": -576.1627197265625,
"KL/std": 238.53648376464844,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.002446242608129978,
"fcm_dpo/delta": 0.06626123189926147,
"fcm_dpo/margin": 219.61865234375,
"fcm_dpo/q_t": 0.3823355436325073,
"grad_norm": 31.01943016052246,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -0.5086591243743896,
"logits/rejected": -0.48945343494415283,
"logps/chosen": -415.1273193359375,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -669.3028564453125,
"loss": 1.0246,
"margin_dpo/margin_mean": 219.61865234375,
"margin_dpo/margin_std": 275.8586120605469,
"step": 672
},
{
"KL/chosen_KL_mean": -307.94329833984375,
"KL/mean": -441.18487548828125,
"KL/rejected_KL_mean": -574.4263916015625,
"KL/std": 226.81732177734375,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.002432117471471429,
"fcm_dpo/delta": -0.0515076145529747,
"fcm_dpo/margin": 266.48309326171875,
"fcm_dpo/q_t": 0.35203078389167786,
"grad_norm": 32.885894775390625,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -0.4631466865539551,
"logits/rejected": -0.45304206013679504,
"logps/chosen": -354.66650390625,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -659.72265625,
"loss": 0.9038,
"margin_dpo/margin_mean": 266.48309326171875,
"margin_dpo/margin_std": 219.18524169921875,
"step": 673
},
{
"KL/chosen_KL_mean": -288.16949462890625,
"KL/mean": -424.24566650390625,
"KL/rejected_KL_mean": -560.3218383789062,
"KL/std": 238.37387084960938,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.002409696113318205,
"fcm_dpo/delta": -0.05967801436781883,
"fcm_dpo/margin": 272.1523742675781,
"fcm_dpo/q_t": 0.3544433116912842,
"grad_norm": 29.36166000366211,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.43240487575531006,
"logits/rejected": -0.4209243655204773,
"logps/chosen": -333.614990234375,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -630.3677978515625,
"loss": 0.9176,
"margin_dpo/margin_mean": 272.1523742675781,
"margin_dpo/margin_std": 258.2077331542969,
"step": 674
},
{
"KL/chosen_KL_mean": -291.1572265625,
"KL/mean": -433.95159912109375,
"KL/rejected_KL_mean": -576.7459716796875,
"KL/std": 261.02081298828125,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.00236148526892066,
"fcm_dpo/delta": -0.0803893506526947,
"fcm_dpo/margin": 285.58868408203125,
"fcm_dpo/q_t": 0.3520466685295105,
"grad_norm": 28.751583099365234,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -0.47011053562164307,
"logits/rejected": -0.4603818655014038,
"logps/chosen": -335.3335266113281,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -650.837890625,
"loss": 0.9139,
"margin_dpo/margin_mean": 285.58868408203125,
"margin_dpo/margin_std": 278.301513671875,
"step": 675
},
{
"KL/chosen_KL_mean": -335.21527099609375,
"KL/mean": -460.12060546875,
"KL/rejected_KL_mean": -585.02587890625,
"KL/std": 246.58184814453125,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0023564891889691353,
"fcm_dpo/delta": 0.011823825538158417,
"fcm_dpo/margin": 249.81060791015625,
"fcm_dpo/q_t": 0.3675922751426697,
"grad_norm": 34.25426483154297,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -0.47875088453292847,
"logits/rejected": -0.45923903584480286,
"logps/chosen": -406.6138000488281,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -673.3846435546875,
"loss": 0.9602,
"margin_dpo/margin_mean": 249.81060791015625,
"margin_dpo/margin_std": 246.6466064453125,
"step": 676
},
{
"KL/chosen_KL_mean": -324.4154357910156,
"KL/mean": -446.2125244140625,
"KL/rejected_KL_mean": -568.0095825195312,
"KL/std": 212.74632263183594,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0023787831887602806,
"fcm_dpo/delta": 0.021667450666427612,
"fcm_dpo/margin": 243.5941619873047,
"fcm_dpo/q_t": 0.36730748414993286,
"grad_norm": 27.077714920043945,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -0.45189139246940613,
"logits/rejected": -0.4171984791755676,
"logps/chosen": -380.94287109375,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -646.2361450195312,
"loss": 0.9503,
"margin_dpo/margin_mean": 243.5941619873047,
"margin_dpo/margin_std": 215.42355346679688,
"step": 677
},
{
"KL/chosen_KL_mean": -292.2994384765625,
"KL/mean": -423.4198303222656,
"KL/rejected_KL_mean": -554.5401611328125,
"KL/std": 253.46011352539062,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.00237276591360569,
"fcm_dpo/delta": -0.023745864629745483,
"fcm_dpo/margin": 262.2407531738281,
"fcm_dpo/q_t": 0.3624275028705597,
"grad_norm": 27.38273048400879,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -0.4344780743122101,
"logits/rejected": -0.4261288046836853,
"logps/chosen": -338.4339294433594,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -635.1448364257812,
"loss": 0.955,
"margin_dpo/margin_mean": 262.2407531738281,
"margin_dpo/margin_std": 278.6129150390625,
"step": 678
},
{
"KL/chosen_KL_mean": -319.44049072265625,
"KL/mean": -427.00982666015625,
"KL/rejected_KL_mean": -534.5791015625,
"KL/std": 217.1231689453125,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0023964918218553066,
"fcm_dpo/delta": 0.08901788294315338,
"fcm_dpo/margin": 215.13864135742188,
"fcm_dpo/q_t": 0.386159747838974,
"grad_norm": 28.638713836669922,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -0.46735358238220215,
"logits/rejected": -0.44047194719314575,
"logps/chosen": -369.73541259765625,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -611.17724609375,
"loss": 1.0224,
"margin_dpo/margin_mean": 215.13864135742188,
"margin_dpo/margin_std": 260.9317626953125,
"step": 679
},
{
"KL/chosen_KL_mean": -323.8060302734375,
"KL/mean": -451.7135009765625,
"KL/rejected_KL_mean": -579.6209716796875,
"KL/std": 261.1182556152344,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.002396587748080492,
"fcm_dpo/delta": -0.014102071523666382,
"fcm_dpo/margin": 255.81492614746094,
"fcm_dpo/q_t": 0.3657163381576538,
"grad_norm": 27.492860794067383,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -0.4733310341835022,
"logits/rejected": -0.4655664563179016,
"logps/chosen": -400.72174072265625,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -692.0057373046875,
"loss": 0.9722,
"margin_dpo/margin_mean": 255.81491088867188,
"margin_dpo/margin_std": 294.9805908203125,
"step": 680
},
{
"KL/chosen_KL_mean": -335.392578125,
"KL/mean": -447.66070556640625,
"KL/rejected_KL_mean": -559.9288330078125,
"KL/std": 238.89901733398438,
"epoch": 1.0,
"fcm_dpo/beta": 0.002440456300973892,
"fcm_dpo/delta": 0.053125154227018356,
"fcm_dpo/margin": 224.5362091064453,
"fcm_dpo/q_t": 0.3793099522590637,
"grad_norm": 34.65813446044922,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -0.4965662956237793,
"logits/rejected": -0.489308625459671,
"logps/chosen": -396.349853515625,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -648.48681640625,
"loss": 1.0067,
"margin_dpo/margin_mean": 224.5362091064453,
"margin_dpo/margin_std": 255.47976684570312,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 0.9981107428210423,
"train_runtime": 1728.4897,
"train_samples_per_second": 25.223,
"train_steps_per_second": 0.394
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink