W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC b7a1db9fbc 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5
Source: Original Platform
2026-06-01 07:20:25 +08:00

15247 lines
572 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999244142101285,
"eval_steps": 200,
"global_step": 661,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.02867889404296875,
"KL/mean": 0.029354453086853027,
"KL/rejected_KL_mean": 0.030029296875,
"KL/std": 0.2071000635623932,
"epoch": 0.0015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013532638549804688,
"fcm_dpo/q_t": 0.5000336766242981,
"grad_norm": 28.22030258178711,
"learning_rate": 0.0,
"logits/chosen": 0.13337239623069763,
"logits/rejected": 0.12492949515581131,
"logps/chosen": -64.5841293334961,
"logps/ref_chosen": -64.61280822753906,
"logps/ref_rejected": -64.17195129394531,
"logps/rejected": -64.14192199707031,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0013527870178222656,
"margin_dpo/margin_std": 0.2561596930027008,
"step": 1
},
{
"KL/chosen_KL_mean": -0.00289154052734375,
"KL/mean": -0.021616414189338684,
"KL/rejected_KL_mean": -0.04033660888671875,
"KL/std": 0.19624735414981842,
"epoch": 0.0030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.037450045347213745,
"fcm_dpo/q_t": 0.4990639090538025,
"grad_norm": 27.827123641967773,
"learning_rate": 7.462686567164179e-09,
"logits/chosen": 0.09414851665496826,
"logits/rejected": 0.07363267242908478,
"logps/chosen": -56.101890563964844,
"logps/ref_chosen": -56.0989990234375,
"logps/ref_rejected": -66.59971618652344,
"logps/rejected": -66.64006042480469,
"loss": 1.3828,
"margin_dpo/margin_mean": 0.03744968771934509,
"margin_dpo/margin_std": 0.27811938524246216,
"step": 2
},
{
"KL/chosen_KL_mean": 0.02416229248046875,
"KL/mean": 0.028771430253982544,
"KL/rejected_KL_mean": 0.033382415771484375,
"KL/std": 0.23812416195869446,
"epoch": 0.0045351473922902496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.009217977523803711,
"fcm_dpo/q_t": 0.5002306699752808,
"grad_norm": 31.222238540649414,
"learning_rate": 1.4925373134328357e-08,
"logits/chosen": 0.09942537546157837,
"logits/rejected": 0.06147611141204834,
"logps/chosen": -65.43309783935547,
"logps/ref_chosen": -65.45726013183594,
"logps/ref_rejected": -90.82853698730469,
"logps/rejected": -90.79515075683594,
"loss": 1.3874,
"margin_dpo/margin_mean": -0.00921773910522461,
"margin_dpo/margin_std": 0.303035706281662,
"step": 3
},
{
"KL/chosen_KL_mean": 0.028865814208984375,
"KL/mean": 0.006366938352584839,
"KL/rejected_KL_mean": -0.0161285400390625,
"KL/std": 0.23680132627487183,
"epoch": 0.006046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04498940706253052,
"fcm_dpo/q_t": 0.49887585639953613,
"grad_norm": 34.199424743652344,
"learning_rate": 2.2388059701492534e-08,
"logits/chosen": 0.11272860318422318,
"logits/rejected": 0.0964307188987732,
"logps/chosen": -76.83131408691406,
"logps/ref_chosen": -76.86018371582031,
"logps/ref_rejected": -79.91523742675781,
"logps/rejected": -79.93136596679688,
"loss": 1.3821,
"margin_dpo/margin_mean": 0.04498884081840515,
"margin_dpo/margin_std": 0.3451710343360901,
"step": 4
},
{
"KL/chosen_KL_mean": -0.006206512451171875,
"KL/mean": -0.012510225176811218,
"KL/rejected_KL_mean": -0.018810272216796875,
"KL/std": 0.2228856384754181,
"epoch": 0.007558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.012603014707565308,
"fcm_dpo/q_t": 0.4996851086616516,
"grad_norm": 29.621192932128906,
"learning_rate": 2.9850746268656714e-08,
"logits/chosen": 0.08257527649402618,
"logits/rejected": 0.043656595051288605,
"logps/chosen": -62.97754669189453,
"logps/ref_chosen": -62.97134017944336,
"logps/ref_rejected": -79.9192123413086,
"logps/rejected": -79.93801879882812,
"loss": 1.3853,
"margin_dpo/margin_mean": 0.012603580951690674,
"margin_dpo/margin_std": 0.31694692373275757,
"step": 5
},
{
"KL/chosen_KL_mean": -0.02547454833984375,
"KL/mean": -0.015959739685058594,
"KL/rejected_KL_mean": -0.0064449310302734375,
"KL/std": 0.24643856287002563,
"epoch": 0.009070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.019029557704925537,
"fcm_dpo/q_t": 0.5004755854606628,
"grad_norm": 29.573598861694336,
"learning_rate": 3.731343283582089e-08,
"logits/chosen": 0.13114216923713684,
"logits/rejected": 0.09240274131298065,
"logps/chosen": -51.33283996582031,
"logps/ref_chosen": -51.30736541748047,
"logps/ref_rejected": -82.77239227294922,
"logps/rejected": -82.77883911132812,
"loss": 1.3885,
"margin_dpo/margin_mean": -0.019030064344406128,
"margin_dpo/margin_std": 0.326235294342041,
"step": 6
},
{
"KL/chosen_KL_mean": -0.0041332244873046875,
"KL/mean": 0.01974809169769287,
"KL/rejected_KL_mean": 0.043628692626953125,
"KL/std": 0.22289568185806274,
"epoch": 0.010582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04776144027709961,
"fcm_dpo/q_t": 0.5011939406394958,
"grad_norm": 27.26719856262207,
"learning_rate": 4.477611940298507e-08,
"logits/chosen": 0.019673092290759087,
"logits/rejected": -0.02439572475850582,
"logps/chosen": -51.46354675292969,
"logps/ref_chosen": -51.45941162109375,
"logps/ref_rejected": -66.3828125,
"logps/rejected": -66.33917999267578,
"loss": 1.3913,
"margin_dpo/margin_mean": -0.047761619091033936,
"margin_dpo/margin_std": 0.2788490951061249,
"step": 7
},
{
"KL/chosen_KL_mean": 0.0001125335693359375,
"KL/mean": -0.004021286964416504,
"KL/rejected_KL_mean": -0.008153915405273438,
"KL/std": 0.22656387090682983,
"epoch": 0.012093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.008267223834991455,
"fcm_dpo/q_t": 0.4997938871383667,
"grad_norm": 28.687152862548828,
"learning_rate": 5.223880597014925e-08,
"logits/chosen": 0.09223589301109314,
"logits/rejected": 0.06963017582893372,
"logps/chosen": -62.19743347167969,
"logps/ref_chosen": -62.197547912597656,
"logps/ref_rejected": -74.66180419921875,
"logps/rejected": -74.66996002197266,
"loss": 1.3858,
"margin_dpo/margin_mean": 0.008266836404800415,
"margin_dpo/margin_std": 0.33951717615127563,
"step": 8
},
{
"KL/chosen_KL_mean": -0.017620086669921875,
"KL/mean": -0.03803481161594391,
"KL/rejected_KL_mean": -0.05844879150390625,
"KL/std": 0.24063098430633545,
"epoch": 0.013605442176870748,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04083278775215149,
"fcm_dpo/q_t": 0.4989795386791229,
"grad_norm": 31.502504348754883,
"learning_rate": 5.970149253731343e-08,
"logits/chosen": 0.18066352605819702,
"logits/rejected": 0.12050867080688477,
"logps/chosen": -55.6473388671875,
"logps/ref_chosen": -55.629722595214844,
"logps/ref_rejected": -86.21221923828125,
"logps/rejected": -86.27067565917969,
"loss": 1.3825,
"margin_dpo/margin_mean": 0.040832966566085815,
"margin_dpo/margin_std": 0.31956762075424194,
"step": 9
},
{
"KL/chosen_KL_mean": 0.003444671630859375,
"KL/mean": 0.024548858404159546,
"KL/rejected_KL_mean": 0.045650482177734375,
"KL/std": 0.27166080474853516,
"epoch": 0.015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04221159219741821,
"fcm_dpo/q_t": 0.5010551810264587,
"grad_norm": 29.839445114135742,
"learning_rate": 6.71641791044776e-08,
"logits/chosen": 0.14076688885688782,
"logits/rejected": 0.10929006338119507,
"logps/chosen": -62.687156677246094,
"logps/ref_chosen": -62.69060134887695,
"logps/ref_rejected": -90.610107421875,
"logps/rejected": -90.56446075439453,
"loss": 1.3909,
"margin_dpo/margin_mean": -0.04221177101135254,
"margin_dpo/margin_std": 0.3878580927848816,
"step": 10
},
{
"KL/chosen_KL_mean": 0.0004329681396484375,
"KL/mean": -0.0009850859642028809,
"KL/rejected_KL_mean": -0.00240325927734375,
"KL/std": 0.224456787109375,
"epoch": 0.016628873771730914,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0028373003005981445,
"fcm_dpo/q_t": 0.4999290108680725,
"grad_norm": 29.369600296020508,
"learning_rate": 7.462686567164178e-08,
"logits/chosen": 0.13556398451328278,
"logits/rejected": 0.12829625606536865,
"logps/chosen": -65.76668548583984,
"logps/ref_chosen": -65.76712036132812,
"logps/ref_rejected": -72.4764633178711,
"logps/rejected": -72.47886657714844,
"loss": 1.3862,
"margin_dpo/margin_mean": 0.002837926149368286,
"margin_dpo/margin_std": 0.2853427529335022,
"step": 11
},
{
"KL/chosen_KL_mean": -0.0237274169921875,
"KL/mean": -0.03283458948135376,
"KL/rejected_KL_mean": -0.04194450378417969,
"KL/std": 0.22529613971710205,
"epoch": 0.018140589569160998,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.018218934535980225,
"fcm_dpo/q_t": 0.4995444715023041,
"grad_norm": 28.207260131835938,
"learning_rate": 8.208955223880596e-08,
"logits/chosen": 0.017520083114504814,
"logits/rejected": 0.0015218239277601242,
"logps/chosen": -60.72861862182617,
"logps/ref_chosen": -60.704891204833984,
"logps/ref_rejected": -69.41564178466797,
"logps/rejected": -69.45758819580078,
"loss": 1.3848,
"margin_dpo/margin_mean": 0.018218576908111572,
"margin_dpo/margin_std": 0.3317432403564453,
"step": 12
},
{
"KL/chosen_KL_mean": 0.013547897338867188,
"KL/mean": 0.008224859833717346,
"KL/rejected_KL_mean": 0.002910614013671875,
"KL/std": 0.22614026069641113,
"epoch": 0.019652305366591082,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01064404845237732,
"fcm_dpo/q_t": 0.4997338652610779,
"grad_norm": 29.149938583374023,
"learning_rate": 8.955223880597014e-08,
"logits/chosen": 0.1031138002872467,
"logits/rejected": 0.041284747421741486,
"logps/chosen": -49.89570617675781,
"logps/ref_chosen": -49.90925598144531,
"logps/ref_rejected": -92.37818145751953,
"logps/rejected": -92.37527465820312,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.010643362998962402,
"margin_dpo/margin_std": 0.32827091217041016,
"step": 13
},
{
"KL/chosen_KL_mean": 0.05055809020996094,
"KL/mean": 0.008084923028945923,
"KL/rejected_KL_mean": -0.034389495849609375,
"KL/std": 0.2188139706850052,
"epoch": 0.021164021164021163,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08494538068771362,
"fcm_dpo/q_t": 0.4978775382041931,
"grad_norm": 29.400705337524414,
"learning_rate": 9.701492537313432e-08,
"logits/chosen": 0.10135047137737274,
"logits/rejected": 0.08315829187631607,
"logps/chosen": -60.5682373046875,
"logps/ref_chosen": -60.61879348754883,
"logps/ref_rejected": -71.79306030273438,
"logps/rejected": -71.82745361328125,
"loss": 1.3781,
"margin_dpo/margin_mean": 0.08494532108306885,
"margin_dpo/margin_std": 0.3461207151412964,
"step": 14
},
{
"KL/chosen_KL_mean": -0.011671066284179688,
"KL/mean": -0.011060148477554321,
"KL/rejected_KL_mean": -0.010448455810546875,
"KL/std": 0.21656280755996704,
"epoch": 0.022675736961451247,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0012268424034118652,
"fcm_dpo/q_t": 0.5000307559967041,
"grad_norm": 33.25237274169922,
"learning_rate": 1.044776119402985e-07,
"logits/chosen": 0.08315175771713257,
"logits/rejected": 0.03908975049853325,
"logps/chosen": -63.48120880126953,
"logps/ref_chosen": -63.46953582763672,
"logps/ref_rejected": -88.88951110839844,
"logps/rejected": -88.89996337890625,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0012272894382476807,
"margin_dpo/margin_std": 0.27439266443252563,
"step": 15
},
{
"KL/chosen_KL_mean": 0.01654815673828125,
"KL/mean": -0.007980212569236755,
"KL/rejected_KL_mean": -0.0325164794921875,
"KL/std": 0.16992035508155823,
"epoch": 0.02418745275888133,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04905608296394348,
"fcm_dpo/q_t": 0.4987736940383911,
"grad_norm": 26.78843879699707,
"learning_rate": 1.1194029850746268e-07,
"logits/chosen": 0.11605009436607361,
"logits/rejected": 0.07887766510248184,
"logps/chosen": -46.515750885009766,
"logps/ref_chosen": -46.53229904174805,
"logps/ref_rejected": -74.27533721923828,
"logps/rejected": -74.30785369873047,
"loss": 1.3816,
"margin_dpo/margin_mean": 0.049056172370910645,
"margin_dpo/margin_std": 0.253105491399765,
"step": 16
},
{
"KL/chosen_KL_mean": 0.04642677307128906,
"KL/mean": -0.011630088090896606,
"KL/rejected_KL_mean": -0.06968307495117188,
"KL/std": 0.2327060103416443,
"epoch": 0.025699168556311415,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11611384153366089,
"fcm_dpo/q_t": 0.4970986545085907,
"grad_norm": 32.61309814453125,
"learning_rate": 1.1940298507462686e-07,
"logits/chosen": 0.07137341052293777,
"logits/rejected": 0.052080146968364716,
"logps/chosen": -64.03140258789062,
"logps/ref_chosen": -64.07783508300781,
"logps/ref_rejected": -86.40876770019531,
"logps/rejected": -86.47845458984375,
"loss": 1.375,
"margin_dpo/margin_mean": 0.11611288785934448,
"margin_dpo/margin_std": 0.324762225151062,
"step": 17
},
{
"KL/chosen_KL_mean": 0.026746749877929688,
"KL/mean": -0.001529976725578308,
"KL/rejected_KL_mean": -0.029804229736328125,
"KL/std": 0.21335983276367188,
"epoch": 0.027210884353741496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05655255913734436,
"fcm_dpo/q_t": 0.498586505651474,
"grad_norm": 27.79302215576172,
"learning_rate": 1.2686567164179106e-07,
"logits/chosen": 0.09060671925544739,
"logits/rejected": 0.04522589594125748,
"logps/chosen": -44.84758758544922,
"logps/ref_chosen": -44.87433624267578,
"logps/ref_rejected": -70.97604370117188,
"logps/rejected": -71.00585174560547,
"loss": 1.3808,
"margin_dpo/margin_mean": 0.05655243992805481,
"margin_dpo/margin_std": 0.26377081871032715,
"step": 18
},
{
"KL/chosen_KL_mean": 0.028326034545898438,
"KL/mean": -0.006911836564540863,
"KL/rejected_KL_mean": -0.042148590087890625,
"KL/std": 0.2588089108467102,
"epoch": 0.02872260015117158,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07047758996486664,
"fcm_dpo/q_t": 0.4982374310493469,
"grad_norm": 30.838409423828125,
"learning_rate": 1.343283582089552e-07,
"logits/chosen": 0.07254800200462341,
"logits/rejected": 0.05900269374251366,
"logps/chosen": -68.13148498535156,
"logps/ref_chosen": -68.1598129272461,
"logps/ref_rejected": -81.17138671875,
"logps/rejected": -81.21353912353516,
"loss": 1.3796,
"margin_dpo/margin_mean": 0.0704774409532547,
"margin_dpo/margin_std": 0.34581679105758667,
"step": 19
},
{
"KL/chosen_KL_mean": 0.00238800048828125,
"KL/mean": -0.0007894262671470642,
"KL/rejected_KL_mean": -0.003963470458984375,
"KL/std": 0.2168654203414917,
"epoch": 0.030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.006352677941322327,
"fcm_dpo/q_t": 0.499840646982193,
"grad_norm": 29.309476852416992,
"learning_rate": 1.4179104477611938e-07,
"logits/chosen": 0.13272634148597717,
"logits/rejected": 0.10981430113315582,
"logps/chosen": -53.67617416381836,
"logps/ref_chosen": -53.67856216430664,
"logps/ref_rejected": -74.16911315917969,
"logps/rejected": -74.17308044433594,
"loss": 1.3859,
"margin_dpo/margin_mean": 0.0063527971506118774,
"margin_dpo/margin_std": 0.3347752094268799,
"step": 20
},
{
"KL/chosen_KL_mean": -0.01772308349609375,
"KL/mean": 0.0006627514958381653,
"KL/rejected_KL_mean": 0.01905059814453125,
"KL/std": 0.2528798580169678,
"epoch": 0.031746031746031744,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03677301108837128,
"fcm_dpo/q_t": 0.5009191036224365,
"grad_norm": 29.429983139038086,
"learning_rate": 1.4925373134328355e-07,
"logits/chosen": 0.13152486085891724,
"logits/rejected": 0.10482591390609741,
"logps/chosen": -64.71927642822266,
"logps/ref_chosen": -64.70155334472656,
"logps/ref_rejected": -81.02095031738281,
"logps/rejected": -81.00189208984375,
"loss": 1.3903,
"margin_dpo/margin_mean": -0.03677418828010559,
"margin_dpo/margin_std": 0.3447566628456116,
"step": 21
},
{
"KL/chosen_KL_mean": -0.02715301513671875,
"KL/mean": -0.05641151964664459,
"KL/rejected_KL_mean": -0.08567047119140625,
"KL/std": 0.2099946141242981,
"epoch": 0.03325774754346183,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05851736664772034,
"fcm_dpo/q_t": 0.49853768944740295,
"grad_norm": 28.81707763671875,
"learning_rate": 1.5671641791044775e-07,
"logits/chosen": 0.01210310310125351,
"logits/rejected": -0.009254289790987968,
"logps/chosen": -58.06314468383789,
"logps/ref_chosen": -58.03599166870117,
"logps/ref_rejected": -80.72721862792969,
"logps/rejected": -80.8128890991211,
"loss": 1.3807,
"margin_dpo/margin_mean": 0.05851736664772034,
"margin_dpo/margin_std": 0.29362723231315613,
"step": 22
},
{
"KL/chosen_KL_mean": 0.003162384033203125,
"KL/mean": -0.0018716305494308472,
"KL/rejected_KL_mean": -0.006908416748046875,
"KL/std": 0.2396647185087204,
"epoch": 0.03476946334089191,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.010072559118270874,
"fcm_dpo/q_t": 0.4997485876083374,
"grad_norm": 32.89107131958008,
"learning_rate": 1.6417910447761193e-07,
"logits/chosen": 0.14491821825504303,
"logits/rejected": 0.11915301531553268,
"logps/chosen": -66.35292053222656,
"logps/ref_chosen": -66.35608673095703,
"logps/ref_rejected": -93.02769470214844,
"logps/rejected": -93.03459930419922,
"loss": 1.3856,
"margin_dpo/margin_mean": 0.010072082281112671,
"margin_dpo/margin_std": 0.3504166007041931,
"step": 23
},
{
"KL/chosen_KL_mean": -0.030969619750976562,
"KL/mean": -0.0021247342228889465,
"KL/rejected_KL_mean": 0.026720046997070312,
"KL/std": 0.1989734172821045,
"epoch": 0.036281179138321996,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.057684049010276794,
"fcm_dpo/q_t": 0.5014417767524719,
"grad_norm": 26.36972999572754,
"learning_rate": 1.716417910447761e-07,
"logits/chosen": 0.11135051399469376,
"logits/rejected": 0.0792173445224762,
"logps/chosen": -54.492210388183594,
"logps/ref_chosen": -54.461238861083984,
"logps/ref_rejected": -68.33817291259766,
"logps/rejected": -68.31145477294922,
"loss": 1.3922,
"margin_dpo/margin_mean": -0.057683974504470825,
"margin_dpo/margin_std": 0.26602524518966675,
"step": 24
},
{
"KL/chosen_KL_mean": -0.037654876708984375,
"KL/mean": -0.04850602149963379,
"KL/rejected_KL_mean": -0.059356689453125,
"KL/std": 0.23352733254432678,
"epoch": 0.03779289493575208,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02170264720916748,
"fcm_dpo/q_t": 0.49945753812789917,
"grad_norm": 29.69145965576172,
"learning_rate": 1.7910447761194027e-07,
"logits/chosen": 0.12192443013191223,
"logits/rejected": 0.0692099779844284,
"logps/chosen": -60.0418586730957,
"logps/ref_chosen": -60.00420379638672,
"logps/ref_rejected": -90.47376251220703,
"logps/rejected": -90.53311920166016,
"loss": 1.3844,
"margin_dpo/margin_mean": 0.021703064441680908,
"margin_dpo/margin_std": 0.30193835496902466,
"step": 25
},
{
"KL/chosen_KL_mean": -0.04012489318847656,
"KL/mean": -0.026201292872428894,
"KL/rejected_KL_mean": -0.012279510498046875,
"KL/std": 0.23137395083904266,
"epoch": 0.039304610733182165,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.027844339609146118,
"fcm_dpo/q_t": 0.5006958246231079,
"grad_norm": 29.982654571533203,
"learning_rate": 1.8656716417910447e-07,
"logits/chosen": 0.12360776960849762,
"logits/rejected": 0.10471150279045105,
"logps/chosen": -56.85927963256836,
"logps/ref_chosen": -56.81915283203125,
"logps/ref_rejected": -77.84333038330078,
"logps/rejected": -77.85560607910156,
"loss": 1.3894,
"margin_dpo/margin_mean": -0.02784484624862671,
"margin_dpo/margin_std": 0.35270577669143677,
"step": 26
},
{
"KL/chosen_KL_mean": -0.0006389617919921875,
"KL/mean": -2.32011079788208e-05,
"KL/rejected_KL_mean": 0.000591278076171875,
"KL/std": 0.22139030694961548,
"epoch": 0.04081632653061224,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0012349188327789307,
"fcm_dpo/q_t": 0.5000306963920593,
"grad_norm": 28.985897064208984,
"learning_rate": 1.9402985074626865e-07,
"logits/chosen": 0.11917729675769806,
"logits/rejected": 0.09365066885948181,
"logps/chosen": -62.877662658691406,
"logps/ref_chosen": -62.87702560424805,
"logps/ref_rejected": -71.34437561035156,
"logps/rejected": -71.34378051757812,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0012350082397460938,
"margin_dpo/margin_std": 0.27922385931015015,
"step": 27
},
{
"KL/chosen_KL_mean": -0.015308380126953125,
"KL/mean": -0.01000969111919403,
"KL/rejected_KL_mean": -0.0047149658203125,
"KL/std": 0.21849636733531952,
"epoch": 0.042328042328042326,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.010593622922897339,
"fcm_dpo/q_t": 0.500264585018158,
"grad_norm": 27.6954345703125,
"learning_rate": 2.0149253731343282e-07,
"logits/chosen": 0.07589665800333023,
"logits/rejected": 0.06671995669603348,
"logps/chosen": -59.84868621826172,
"logps/ref_chosen": -59.8333740234375,
"logps/ref_rejected": -70.39804077148438,
"logps/rejected": -70.40275573730469,
"loss": 1.3876,
"margin_dpo/margin_mean": -0.010593622922897339,
"margin_dpo/margin_std": 0.30503910779953003,
"step": 28
},
{
"KL/chosen_KL_mean": -0.04323577880859375,
"KL/mean": -0.0652160719037056,
"KL/rejected_KL_mean": -0.08718490600585938,
"KL/std": 0.2304714471101761,
"epoch": 0.04383975812547241,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04395388066768646,
"fcm_dpo/q_t": 0.4989010691642761,
"grad_norm": 32.57505416870117,
"learning_rate": 2.08955223880597e-07,
"logits/chosen": 0.15698179602622986,
"logits/rejected": 0.13876661658287048,
"logps/chosen": -74.16343688964844,
"logps/ref_chosen": -74.12020111083984,
"logps/ref_rejected": -83.33099365234375,
"logps/rejected": -83.41817474365234,
"loss": 1.3822,
"margin_dpo/margin_mean": 0.04395401477813721,
"margin_dpo/margin_std": 0.35602355003356934,
"step": 29
},
{
"KL/chosen_KL_mean": -0.008335113525390625,
"KL/mean": -0.02375747263431549,
"KL/rejected_KL_mean": -0.03917694091796875,
"KL/std": 0.22164683043956757,
"epoch": 0.045351473922902494,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03084227442741394,
"fcm_dpo/q_t": 0.4992305040359497,
"grad_norm": 30.354887008666992,
"learning_rate": 2.1641791044776117e-07,
"logits/chosen": 0.11798453330993652,
"logits/rejected": 0.06400823593139648,
"logps/chosen": -50.75962448120117,
"logps/ref_chosen": -50.75128936767578,
"logps/ref_rejected": -89.29063415527344,
"logps/rejected": -89.3298110961914,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.030841588973999023,
"margin_dpo/margin_std": 0.33075201511383057,
"step": 30
},
{
"KL/chosen_KL_mean": -0.010515213012695312,
"KL/mean": -0.03813314437866211,
"KL/rejected_KL_mean": -0.0657501220703125,
"KL/std": 0.25133079290390015,
"epoch": 0.04686318972033258,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05523216724395752,
"fcm_dpo/q_t": 0.49862000346183777,
"grad_norm": 34.2333869934082,
"learning_rate": 2.2388059701492537e-07,
"logits/chosen": 0.11453694105148315,
"logits/rejected": 0.06778549402952194,
"logps/chosen": -65.3472671508789,
"logps/ref_chosen": -65.33675384521484,
"logps/ref_rejected": -100.76666259765625,
"logps/rejected": -100.83241271972656,
"loss": 1.3811,
"margin_dpo/margin_mean": 0.055232226848602295,
"margin_dpo/margin_std": 0.3529159724712372,
"step": 31
},
{
"KL/chosen_KL_mean": -0.051006317138671875,
"KL/mean": -0.03746132552623749,
"KL/rejected_KL_mean": -0.02391815185546875,
"KL/std": 0.24133098125457764,
"epoch": 0.04837490551776266,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.027087301015853882,
"fcm_dpo/q_t": 0.5006769895553589,
"grad_norm": 30.098434448242188,
"learning_rate": 2.3134328358208954e-07,
"logits/chosen": 0.08350340276956558,
"logits/rejected": 0.07563582062721252,
"logps/chosen": -67.23433685302734,
"logps/ref_chosen": -67.18333435058594,
"logps/ref_rejected": -82.80763244628906,
"logps/rejected": -82.83155822753906,
"loss": 1.3893,
"margin_dpo/margin_mean": -0.027087002992630005,
"margin_dpo/margin_std": 0.3504248857498169,
"step": 32
},
{
"KL/chosen_KL_mean": -0.03555488586425781,
"KL/mean": -0.0537736639380455,
"KL/rejected_KL_mean": -0.071990966796875,
"KL/std": 0.21045207977294922,
"epoch": 0.049886621315192746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03643660247325897,
"fcm_dpo/q_t": 0.4990893006324768,
"grad_norm": 31.021913528442383,
"learning_rate": 2.388059701492537e-07,
"logits/chosen": 0.042121004313230515,
"logits/rejected": 0.01561102643609047,
"logps/chosen": -64.07503509521484,
"logps/ref_chosen": -64.03948211669922,
"logps/ref_rejected": -75.68357849121094,
"logps/rejected": -75.75556945800781,
"loss": 1.3829,
"margin_dpo/margin_mean": 0.036436840891838074,
"margin_dpo/margin_std": 0.2771301865577698,
"step": 33
},
{
"KL/chosen_KL_mean": -0.03723907470703125,
"KL/mean": -0.061888277530670166,
"KL/rejected_KL_mean": -0.08653831481933594,
"KL/std": 0.23711252212524414,
"epoch": 0.05139833711262283,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04930323362350464,
"fcm_dpo/q_t": 0.498767614364624,
"grad_norm": 28.449756622314453,
"learning_rate": 2.4626865671641786e-07,
"logits/chosen": 0.10028792172670364,
"logits/rejected": 0.07043236494064331,
"logps/chosen": -53.701534271240234,
"logps/ref_chosen": -53.6642951965332,
"logps/ref_rejected": -65.77989959716797,
"logps/rejected": -65.86643981933594,
"loss": 1.3817,
"margin_dpo/margin_mean": 0.04930317401885986,
"margin_dpo/margin_std": 0.34076127409935,
"step": 34
},
{
"KL/chosen_KL_mean": -0.09243202209472656,
"KL/mean": -0.09407998621463776,
"KL/rejected_KL_mean": -0.09572601318359375,
"KL/std": 0.2501530349254608,
"epoch": 0.05291005291005291,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.003294020891189575,
"fcm_dpo/q_t": 0.49991774559020996,
"grad_norm": 27.86534309387207,
"learning_rate": 2.537313432835821e-07,
"logits/chosen": 0.05358413979411125,
"logits/rejected": 0.031220555305480957,
"logps/chosen": -61.10929489135742,
"logps/ref_chosen": -61.01686096191406,
"logps/ref_rejected": -72.78598022460938,
"logps/rejected": -72.8817138671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 0.0032941102981567383,
"margin_dpo/margin_std": 0.36158987879753113,
"step": 35
},
{
"KL/chosen_KL_mean": -0.07643318176269531,
"KL/mean": -0.08448834717273712,
"KL/rejected_KL_mean": -0.09254074096679688,
"KL/std": 0.2451927661895752,
"epoch": 0.05442176870748299,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01610603928565979,
"fcm_dpo/q_t": 0.4995979070663452,
"grad_norm": 28.73918342590332,
"learning_rate": 2.611940298507462e-07,
"logits/chosen": 0.09999187290668488,
"logits/rejected": 0.04726378619670868,
"logps/chosen": -50.613792419433594,
"logps/ref_chosen": -50.53736114501953,
"logps/ref_rejected": -78.11678314208984,
"logps/rejected": -78.20932006835938,
"loss": 1.385,
"margin_dpo/margin_mean": 0.01610630750656128,
"margin_dpo/margin_std": 0.35909271240234375,
"step": 36
},
{
"KL/chosen_KL_mean": -0.008823394775390625,
"KL/mean": -0.0899229645729065,
"KL/rejected_KL_mean": -0.17102432250976562,
"KL/std": 0.2953256070613861,
"epoch": 0.055933484504913075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.16220009326934814,
"fcm_dpo/q_t": 0.49594852328300476,
"grad_norm": 37.488372802734375,
"learning_rate": 2.686567164179104e-07,
"logits/chosen": 0.09731757640838623,
"logits/rejected": 0.0182504802942276,
"logps/chosen": -59.56277084350586,
"logps/ref_chosen": -59.55394744873047,
"logps/ref_rejected": -108.27702331542969,
"logps/rejected": -108.44805145263672,
"loss": 1.3706,
"margin_dpo/margin_mean": 0.1622002124786377,
"margin_dpo/margin_std": 0.436340868473053,
"step": 37
},
{
"KL/chosen_KL_mean": -0.06951332092285156,
"KL/mean": -0.07658199965953827,
"KL/rejected_KL_mean": -0.08365058898925781,
"KL/std": 0.28319302201271057,
"epoch": 0.05744520030234316,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01413390040397644,
"fcm_dpo/q_t": 0.49964767694473267,
"grad_norm": 29.705196380615234,
"learning_rate": 2.761194029850746e-07,
"logits/chosen": 0.06554099917411804,
"logits/rejected": 0.05162451043725014,
"logps/chosen": -65.85787200927734,
"logps/ref_chosen": -65.78836059570312,
"logps/ref_rejected": -76.1619873046875,
"logps/rejected": -76.24563598632812,
"loss": 1.3853,
"margin_dpo/margin_mean": 0.014134079217910767,
"margin_dpo/margin_std": 0.4092469811439514,
"step": 38
},
{
"KL/chosen_KL_mean": -0.09235954284667969,
"KL/mean": -0.12527738511562347,
"KL/rejected_KL_mean": -0.15819931030273438,
"KL/std": 0.21432055532932281,
"epoch": 0.05895691609977324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06583818793296814,
"fcm_dpo/q_t": 0.4983542561531067,
"grad_norm": 29.24547576904297,
"learning_rate": 2.8358208955223876e-07,
"logits/chosen": 0.15803761780261993,
"logits/rejected": 0.1312660574913025,
"logps/chosen": -57.269168853759766,
"logps/ref_chosen": -57.17681121826172,
"logps/ref_rejected": -79.486328125,
"logps/rejected": -79.64453125,
"loss": 1.3799,
"margin_dpo/margin_mean": 0.0658378005027771,
"margin_dpo/margin_std": 0.28493544459342957,
"step": 39
},
{
"KL/chosen_KL_mean": -0.0874176025390625,
"KL/mean": -0.1004403755068779,
"KL/rejected_KL_mean": -0.11346435546875,
"KL/std": 0.251811683177948,
"epoch": 0.06046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02604334056377411,
"fcm_dpo/q_t": 0.4993489384651184,
"grad_norm": 31.375804901123047,
"learning_rate": 2.9104477611940296e-07,
"logits/chosen": 0.09894993901252747,
"logits/rejected": 0.04999292641878128,
"logps/chosen": -61.42158508300781,
"logps/ref_chosen": -61.33416748046875,
"logps/ref_rejected": -79.10697174072266,
"logps/rejected": -79.2204360961914,
"loss": 1.384,
"margin_dpo/margin_mean": 0.026043608784675598,
"margin_dpo/margin_std": 0.3506956100463867,
"step": 40
},
{
"KL/chosen_KL_mean": -0.10820770263671875,
"KL/mean": -0.1342216283082962,
"KL/rejected_KL_mean": -0.16022872924804688,
"KL/std": 0.24819834530353546,
"epoch": 0.06198034769463341,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05201807618141174,
"fcm_dpo/q_t": 0.49869972467422485,
"grad_norm": 30.542781829833984,
"learning_rate": 2.985074626865671e-07,
"logits/chosen": 0.05286967754364014,
"logits/rejected": 0.03276796638965607,
"logps/chosen": -67.65493774414062,
"logps/ref_chosen": -67.5467300415039,
"logps/ref_rejected": -83.87788391113281,
"logps/rejected": -84.03811645507812,
"loss": 1.3814,
"margin_dpo/margin_mean": 0.05201798677444458,
"margin_dpo/margin_std": 0.33188194036483765,
"step": 41
},
{
"KL/chosen_KL_mean": -0.108795166015625,
"KL/mean": -0.12974503636360168,
"KL/rejected_KL_mean": -0.15069580078125,
"KL/std": 0.30932196974754333,
"epoch": 0.06349206349206349,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0419042706489563,
"fcm_dpo/q_t": 0.49895286560058594,
"grad_norm": 29.216102600097656,
"learning_rate": 3.059701492537313e-07,
"logits/chosen": 0.03016223944723606,
"logits/rejected": 0.00932481698691845,
"logps/chosen": -61.373653411865234,
"logps/ref_chosen": -61.26485824584961,
"logps/ref_rejected": -76.3629150390625,
"logps/rejected": -76.51361083984375,
"loss": 1.3826,
"margin_dpo/margin_mean": 0.04190385341644287,
"margin_dpo/margin_std": 0.43035653233528137,
"step": 42
},
{
"KL/chosen_KL_mean": -0.09076690673828125,
"KL/mean": -0.135238379240036,
"KL/rejected_KL_mean": -0.17971420288085938,
"KL/std": 0.2912582755088806,
"epoch": 0.06500377928949358,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08895289897918701,
"fcm_dpo/q_t": 0.49777668714523315,
"grad_norm": 34.40074157714844,
"learning_rate": 3.134328358208955e-07,
"logits/chosen": 0.09243658185005188,
"logits/rejected": 0.081363745033741,
"logps/chosen": -71.89979553222656,
"logps/ref_chosen": -71.80902862548828,
"logps/ref_rejected": -81.12464141845703,
"logps/rejected": -81.30435180664062,
"loss": 1.3778,
"margin_dpo/margin_mean": 0.08895328640937805,
"margin_dpo/margin_std": 0.39062684774398804,
"step": 43
},
{
"KL/chosen_KL_mean": -0.16825103759765625,
"KL/mean": -0.17432071268558502,
"KL/rejected_KL_mean": -0.180389404296875,
"KL/std": 0.29811811447143555,
"epoch": 0.06651549508692366,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.012138456106185913,
"fcm_dpo/q_t": 0.4996975064277649,
"grad_norm": 32.57433319091797,
"learning_rate": 3.2089552238805965e-07,
"logits/chosen": 0.04532858729362488,
"logits/rejected": 0.014902985654771328,
"logps/chosen": -66.71868133544922,
"logps/ref_chosen": -66.55043029785156,
"logps/ref_rejected": -85.06198120117188,
"logps/rejected": -85.24237060546875,
"loss": 1.3856,
"margin_dpo/margin_mean": 0.012138217687606812,
"margin_dpo/margin_std": 0.445947527885437,
"step": 44
},
{
"KL/chosen_KL_mean": -0.12631607055664062,
"KL/mean": -0.1697479486465454,
"KL/rejected_KL_mean": -0.2131805419921875,
"KL/std": 0.2817876935005188,
"epoch": 0.06802721088435375,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08686672151088715,
"fcm_dpo/q_t": 0.49782922863960266,
"grad_norm": 31.86672592163086,
"learning_rate": 3.2835820895522385e-07,
"logits/chosen": 0.10761643946170807,
"logits/rejected": 0.05567549169063568,
"logps/chosen": -62.37017059326172,
"logps/ref_chosen": -62.24385452270508,
"logps/ref_rejected": -92.96665954589844,
"logps/rejected": -93.17984008789062,
"loss": 1.378,
"margin_dpo/margin_mean": 0.08686742186546326,
"margin_dpo/margin_std": 0.36542221903800964,
"step": 45
},
{
"KL/chosen_KL_mean": -0.10385322570800781,
"KL/mean": -0.17652657628059387,
"KL/rejected_KL_mean": -0.24920272827148438,
"KL/std": 0.34992483258247375,
"epoch": 0.06953892668178382,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14534282684326172,
"fcm_dpo/q_t": 0.4963688254356384,
"grad_norm": 31.092803955078125,
"learning_rate": 3.3582089552238805e-07,
"logits/chosen": 0.10892710834741592,
"logits/rejected": 0.06376945227384567,
"logps/chosen": -61.602760314941406,
"logps/ref_chosen": -61.498905181884766,
"logps/ref_rejected": -78.91172790527344,
"logps/rejected": -79.16093444824219,
"loss": 1.3723,
"margin_dpo/margin_mean": 0.1453423798084259,
"margin_dpo/margin_std": 0.44881075620651245,
"step": 46
},
{
"KL/chosen_KL_mean": -0.12169647216796875,
"KL/mean": -0.2103060632944107,
"KL/rejected_KL_mean": -0.2989158630371094,
"KL/std": 0.2522382140159607,
"epoch": 0.0710506424792139,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17722073197364807,
"fcm_dpo/q_t": 0.49557074904441833,
"grad_norm": 28.476625442504883,
"learning_rate": 3.432835820895522e-07,
"logits/chosen": 0.033945608884096146,
"logits/rejected": -0.008820239454507828,
"logps/chosen": -51.700042724609375,
"logps/ref_chosen": -51.578346252441406,
"logps/ref_rejected": -68.2215576171875,
"logps/rejected": -68.52047729492188,
"loss": 1.3689,
"margin_dpo/margin_mean": 0.17722094058990479,
"margin_dpo/margin_std": 0.31175607442855835,
"step": 47
},
{
"KL/chosen_KL_mean": -0.21759986877441406,
"KL/mean": -0.234956756234169,
"KL/rejected_KL_mean": -0.25231361389160156,
"KL/std": 0.3202669024467468,
"epoch": 0.07256235827664399,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.034711629152297974,
"fcm_dpo/q_t": 0.49913373589515686,
"grad_norm": 26.849285125732422,
"learning_rate": 3.507462686567164e-07,
"logits/chosen": 0.1537940800189972,
"logits/rejected": 0.12394692003726959,
"logps/chosen": -52.011253356933594,
"logps/ref_chosen": -51.79365158081055,
"logps/ref_rejected": -64.22503662109375,
"logps/rejected": -64.47735595703125,
"loss": 1.3833,
"margin_dpo/margin_mean": 0.034712135791778564,
"margin_dpo/margin_std": 0.4351728558540344,
"step": 48
},
{
"KL/chosen_KL_mean": -0.19132232666015625,
"KL/mean": -0.24339309334754944,
"KL/rejected_KL_mean": -0.29546356201171875,
"KL/std": 0.35654354095458984,
"epoch": 0.07407407407407407,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10414212942123413,
"fcm_dpo/q_t": 0.49739664793014526,
"grad_norm": 27.24091911315918,
"learning_rate": 3.5820895522388055e-07,
"logits/chosen": 0.01570693962275982,
"logits/rejected": -0.00523674301803112,
"logps/chosen": -58.325923919677734,
"logps/ref_chosen": -58.13460159301758,
"logps/ref_rejected": -64.63206481933594,
"logps/rejected": -64.92753601074219,
"loss": 1.3766,
"margin_dpo/margin_mean": 0.10414272546768188,
"margin_dpo/margin_std": 0.5124611854553223,
"step": 49
},
{
"KL/chosen_KL_mean": -0.2684955596923828,
"KL/mean": -0.32318902015686035,
"KL/rejected_KL_mean": -0.3778800964355469,
"KL/std": 0.35013025999069214,
"epoch": 0.07558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10938182473182678,
"fcm_dpo/q_t": 0.4972665309906006,
"grad_norm": 28.025882720947266,
"learning_rate": 3.6567164179104475e-07,
"logits/chosen": 0.11795582622289658,
"logits/rejected": 0.08789660781621933,
"logps/chosen": -53.124935150146484,
"logps/ref_chosen": -52.85643768310547,
"logps/ref_rejected": -72.17460632324219,
"logps/rejected": -72.55248260498047,
"loss": 1.3758,
"margin_dpo/margin_mean": 0.10938182473182678,
"margin_dpo/margin_std": 0.4207468628883362,
"step": 50
},
{
"KL/chosen_KL_mean": -0.25263214111328125,
"KL/mean": -0.3954651355743408,
"KL/rejected_KL_mean": -0.5383033752441406,
"KL/std": 0.3872417211532593,
"epoch": 0.07709750566893424,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2856694459915161,
"fcm_dpo/q_t": 0.4928652048110962,
"grad_norm": 30.83257293701172,
"learning_rate": 3.7313432835820895e-07,
"logits/chosen": 0.11152546107769012,
"logits/rejected": 0.08300299197435379,
"logps/chosen": -63.90907287597656,
"logps/ref_chosen": -63.65644073486328,
"logps/ref_rejected": -86.13229370117188,
"logps/rejected": -86.67060089111328,
"loss": 1.3585,
"margin_dpo/margin_mean": 0.28566935658454895,
"margin_dpo/margin_std": 0.4884032607078552,
"step": 51
},
{
"KL/chosen_KL_mean": -0.3018016815185547,
"KL/mean": -0.4102283716201782,
"KL/rejected_KL_mean": -0.5186576843261719,
"KL/std": 0.4544343650341034,
"epoch": 0.07860922146636433,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21685557067394257,
"fcm_dpo/q_t": 0.494590163230896,
"grad_norm": 32.574974060058594,
"learning_rate": 3.805970149253731e-07,
"logits/chosen": 0.07611118257045746,
"logits/rejected": 0.027582643553614616,
"logps/chosen": -68.14201354980469,
"logps/ref_chosen": -67.8402099609375,
"logps/ref_rejected": -96.97090911865234,
"logps/rejected": -97.48956298828125,
"loss": 1.3657,
"margin_dpo/margin_mean": 0.21685530245304108,
"margin_dpo/margin_std": 0.604822039604187,
"step": 52
},
{
"KL/chosen_KL_mean": -0.3174476623535156,
"KL/mean": -0.4145428538322449,
"KL/rejected_KL_mean": -0.5116405487060547,
"KL/std": 0.40250444412231445,
"epoch": 0.0801209372637944,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19419114291667938,
"fcm_dpo/q_t": 0.49514681100845337,
"grad_norm": 27.431989669799805,
"learning_rate": 3.880597014925373e-07,
"logits/chosen": 0.09542200714349747,
"logits/rejected": 0.08459050208330154,
"logps/chosen": -57.195587158203125,
"logps/ref_chosen": -56.87813949584961,
"logps/ref_rejected": -60.75569152832031,
"logps/rejected": -61.267333984375,
"loss": 1.3677,
"margin_dpo/margin_mean": 0.1941910982131958,
"margin_dpo/margin_std": 0.5309941172599792,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3586311340332031,
"KL/mean": -0.45110976696014404,
"KL/rejected_KL_mean": -0.5435848236083984,
"KL/std": 0.38474512100219727,
"epoch": 0.08163265306122448,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18495365977287292,
"fcm_dpo/q_t": 0.4953802227973938,
"grad_norm": 26.548255920410156,
"learning_rate": 3.9552238805970144e-07,
"logits/chosen": 0.06870827078819275,
"logits/rejected": 0.05323575437068939,
"logps/chosen": -47.62554931640625,
"logps/ref_chosen": -47.26692199707031,
"logps/ref_rejected": -62.19426727294922,
"logps/rejected": -62.737850189208984,
"loss": 1.3685,
"margin_dpo/margin_mean": 0.18495362997055054,
"margin_dpo/margin_std": 0.4835350811481476,
"step": 54
},
{
"KL/chosen_KL_mean": -0.4072132110595703,
"KL/mean": -0.5418218374252319,
"KL/rejected_KL_mean": -0.6764335632324219,
"KL/std": 0.5083746910095215,
"epoch": 0.08314436885865457,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2692183256149292,
"fcm_dpo/q_t": 0.49328285455703735,
"grad_norm": 31.305131912231445,
"learning_rate": 4.0298507462686564e-07,
"logits/chosen": 0.05396968498826027,
"logits/rejected": -0.02452990412712097,
"logps/chosen": -50.73340606689453,
"logps/ref_chosen": -50.32619094848633,
"logps/ref_rejected": -92.44389343261719,
"logps/rejected": -93.12032318115234,
"loss": 1.3607,
"margin_dpo/margin_mean": 0.26921796798706055,
"margin_dpo/margin_std": 0.654507577419281,
"step": 55
},
{
"KL/chosen_KL_mean": -0.3615074157714844,
"KL/mean": -0.4682575464248657,
"KL/rejected_KL_mean": -0.5750026702880859,
"KL/std": 0.52659672498703,
"epoch": 0.08465608465608465,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21349409222602844,
"fcm_dpo/q_t": 0.4946634769439697,
"grad_norm": 27.466205596923828,
"learning_rate": 4.1044776119402984e-07,
"logits/chosen": 0.12170642614364624,
"logits/rejected": 0.09938912093639374,
"logps/chosen": -57.12847900390625,
"logps/ref_chosen": -56.766971588134766,
"logps/ref_rejected": -66.30504608154297,
"logps/rejected": -66.88005065917969,
"loss": 1.3663,
"margin_dpo/margin_mean": 0.21349415183067322,
"margin_dpo/margin_std": 0.6816796064376831,
"step": 56
},
{
"KL/chosen_KL_mean": -0.5067310333251953,
"KL/mean": -0.6605712175369263,
"KL/rejected_KL_mean": -0.8144111633300781,
"KL/std": 0.5967442393302917,
"epoch": 0.08616780045351474,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3076814115047455,
"fcm_dpo/q_t": 0.4923190772533417,
"grad_norm": 30.64410972595215,
"learning_rate": 4.17910447761194e-07,
"logits/chosen": 0.10049479454755783,
"logits/rejected": 0.03633933141827583,
"logps/chosen": -58.27447509765625,
"logps/ref_chosen": -57.76774597167969,
"logps/ref_rejected": -82.75698852539062,
"logps/rejected": -83.57139587402344,
"loss": 1.357,
"margin_dpo/margin_mean": 0.3076811134815216,
"margin_dpo/margin_std": 0.6833426356315613,
"step": 57
},
{
"KL/chosen_KL_mean": -0.5886154174804688,
"KL/mean": -0.6974492073059082,
"KL/rejected_KL_mean": -0.8062820434570312,
"KL/std": 0.6851860284805298,
"epoch": 0.08767951625094482,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21767285466194153,
"fcm_dpo/q_t": 0.4946228861808777,
"grad_norm": 30.1894588470459,
"learning_rate": 4.253731343283582e-07,
"logits/chosen": 0.04159823805093765,
"logits/rejected": 0.026745518669486046,
"logps/chosen": -73.35270690917969,
"logps/ref_chosen": -72.76408386230469,
"logps/ref_rejected": -84.49275207519531,
"logps/rejected": -85.29903411865234,
"loss": 1.3674,
"margin_dpo/margin_mean": 0.2176724672317505,
"margin_dpo/margin_std": 0.980886697769165,
"step": 58
},
{
"KL/chosen_KL_mean": -0.5488433837890625,
"KL/mean": -0.6876722574234009,
"KL/rejected_KL_mean": -0.8265056610107422,
"KL/std": 0.6126680374145508,
"epoch": 0.08919123204837491,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2776602506637573,
"fcm_dpo/q_t": 0.4930819869041443,
"grad_norm": 26.609149932861328,
"learning_rate": 4.3283582089552234e-07,
"logits/chosen": 0.13217273354530334,
"logits/rejected": 0.0661400556564331,
"logps/chosen": -50.36962127685547,
"logps/ref_chosen": -49.820777893066406,
"logps/ref_rejected": -77.14368438720703,
"logps/rejected": -77.9701919555664,
"loss": 1.3605,
"margin_dpo/margin_mean": 0.2776605784893036,
"margin_dpo/margin_std": 0.7779598832130432,
"step": 59
},
{
"KL/chosen_KL_mean": -0.6309585571289062,
"KL/mean": -0.6522994041442871,
"KL/rejected_KL_mean": -0.6736431121826172,
"KL/std": 0.6409569978713989,
"epoch": 0.09070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04268404841423035,
"fcm_dpo/q_t": 0.49893319606781006,
"grad_norm": 29.606698989868164,
"learning_rate": 4.4029850746268654e-07,
"logits/chosen": 0.14147618412971497,
"logits/rejected": 0.14005886018276215,
"logps/chosen": -63.85573196411133,
"logps/ref_chosen": -63.22477340698242,
"logps/ref_rejected": -61.360477447509766,
"logps/rejected": -62.03411865234375,
"loss": 1.3837,
"margin_dpo/margin_mean": 0.04268389940261841,
"margin_dpo/margin_std": 0.8073387145996094,
"step": 60
},
{
"KL/chosen_KL_mean": -0.7385311126708984,
"KL/mean": -0.7797211408615112,
"KL/rejected_KL_mean": -0.8209114074707031,
"KL/std": 0.724621057510376,
"epoch": 0.09221466364323508,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.082376629114151,
"fcm_dpo/q_t": 0.49793773889541626,
"grad_norm": 28.157575607299805,
"learning_rate": 4.4776119402985074e-07,
"logits/chosen": 0.1449713408946991,
"logits/rejected": 0.1120869517326355,
"logps/chosen": -49.75532913208008,
"logps/ref_chosen": -49.01679992675781,
"logps/ref_rejected": -74.90817260742188,
"logps/rejected": -75.72908020019531,
"loss": 1.3806,
"margin_dpo/margin_mean": 0.08237650990486145,
"margin_dpo/margin_std": 0.9990288019180298,
"step": 61
},
{
"KL/chosen_KL_mean": -0.75982666015625,
"KL/mean": -0.855846107006073,
"KL/rejected_KL_mean": -0.9518661499023438,
"KL/std": 0.6547791361808777,
"epoch": 0.09372637944066516,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19204318523406982,
"fcm_dpo/q_t": 0.4952077269554138,
"grad_norm": 28.638532638549805,
"learning_rate": 4.552238805970149e-07,
"logits/chosen": 0.0987754687666893,
"logits/rejected": 0.06032714992761612,
"logps/chosen": -63.511695861816406,
"logps/ref_chosen": -62.751869201660156,
"logps/ref_rejected": -78.93360900878906,
"logps/rejected": -79.8854751586914,
"loss": 1.3693,
"margin_dpo/margin_mean": 0.19204312562942505,
"margin_dpo/margin_std": 0.923231840133667,
"step": 62
},
{
"KL/chosen_KL_mean": -0.5642757415771484,
"KL/mean": -0.8418329954147339,
"KL/rejected_KL_mean": -1.1193962097167969,
"KL/std": 0.6890658140182495,
"epoch": 0.09523809523809523,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5551224946975708,
"fcm_dpo/q_t": 0.48615702986717224,
"grad_norm": 31.38890838623047,
"learning_rate": 4.626865671641791e-07,
"logits/chosen": 0.20929303765296936,
"logits/rejected": 0.18352961540222168,
"logps/chosen": -61.079524993896484,
"logps/ref_chosen": -60.51525115966797,
"logps/ref_rejected": -85.11021423339844,
"logps/rejected": -86.2296142578125,
"loss": 1.3334,
"margin_dpo/margin_mean": 0.5551224946975708,
"margin_dpo/margin_std": 0.8649230003356934,
"step": 63
},
{
"KL/chosen_KL_mean": -0.8340473175048828,
"KL/mean": -0.874204695224762,
"KL/rejected_KL_mean": -0.9143638610839844,
"KL/std": 0.7256166934967041,
"epoch": 0.09674981103552532,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08031821250915527,
"fcm_dpo/q_t": 0.4979880452156067,
"grad_norm": 26.190099716186523,
"learning_rate": 4.701492537313433e-07,
"logits/chosen": 0.0871717631816864,
"logits/rejected": 0.062399592250585556,
"logps/chosen": -52.0408935546875,
"logps/ref_chosen": -51.20684814453125,
"logps/ref_rejected": -66.93081665039062,
"logps/rejected": -67.84518432617188,
"loss": 1.3803,
"margin_dpo/margin_mean": 0.08031848073005676,
"margin_dpo/margin_std": 0.8882924318313599,
"step": 64
},
{
"KL/chosen_KL_mean": -0.8912067413330078,
"KL/mean": -1.1177582740783691,
"KL/rejected_KL_mean": -1.3443145751953125,
"KL/std": 0.8927726149559021,
"epoch": 0.0982615268329554,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.45310866832733154,
"fcm_dpo/q_t": 0.4887198805809021,
"grad_norm": 30.947498321533203,
"learning_rate": 4.776119402985074e-07,
"logits/chosen": 0.20429706573486328,
"logits/rejected": 0.1743599772453308,
"logps/chosen": -68.17990112304688,
"logps/ref_chosen": -67.2886962890625,
"logps/ref_rejected": -74.44281005859375,
"logps/rejected": -75.78712463378906,
"loss": 1.3447,
"margin_dpo/margin_mean": 0.4531082510948181,
"margin_dpo/margin_std": 1.1238809823989868,
"step": 65
},
{
"KL/chosen_KL_mean": -0.9344940185546875,
"KL/mean": -1.0839293003082275,
"KL/rejected_KL_mean": -1.2333641052246094,
"KL/std": 0.8180510997772217,
"epoch": 0.09977324263038549,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.298869788646698,
"fcm_dpo/q_t": 0.49254971742630005,
"grad_norm": 29.427141189575195,
"learning_rate": 4.850746268656717e-07,
"logits/chosen": 0.07761037349700928,
"logits/rejected": 0.05442436411976814,
"logps/chosen": -71.67790222167969,
"logps/ref_chosen": -70.743408203125,
"logps/ref_rejected": -77.26499938964844,
"logps/rejected": -78.49836730957031,
"loss": 1.3598,
"margin_dpo/margin_mean": 0.29886963963508606,
"margin_dpo/margin_std": 1.1081165075302124,
"step": 66
},
{
"KL/chosen_KL_mean": -0.74542236328125,
"KL/mean": -0.9272383451461792,
"KL/rejected_KL_mean": -1.1090545654296875,
"KL/std": 0.750342845916748,
"epoch": 0.10128495842781557,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.36363154649734497,
"fcm_dpo/q_t": 0.49093663692474365,
"grad_norm": 28.98794937133789,
"learning_rate": 4.925373134328357e-07,
"logits/chosen": 0.09056040644645691,
"logits/rejected": 0.03486517444252968,
"logps/chosen": -61.3480224609375,
"logps/ref_chosen": -60.60260009765625,
"logps/ref_rejected": -75.22235870361328,
"logps/rejected": -76.33141326904297,
"loss": 1.3523,
"margin_dpo/margin_mean": 0.3636314272880554,
"margin_dpo/margin_std": 0.9072555303573608,
"step": 67
},
{
"KL/chosen_KL_mean": -1.2406959533691406,
"KL/mean": -1.4078385829925537,
"KL/rejected_KL_mean": -1.574981689453125,
"KL/std": 0.9226593971252441,
"epoch": 0.10279667422524566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.33428969979286194,
"fcm_dpo/q_t": 0.4916718304157257,
"grad_norm": 31.385425567626953,
"learning_rate": 5e-07,
"logits/chosen": 0.06587998569011688,
"logits/rejected": 0.035971544682979584,
"logps/chosen": -78.76905822753906,
"logps/ref_chosen": -77.52836608886719,
"logps/ref_rejected": -93.17778015136719,
"logps/rejected": -94.75276184082031,
"loss": 1.3566,
"margin_dpo/margin_mean": 0.3342894911766052,
"margin_dpo/margin_std": 1.142065167427063,
"step": 68
},
{
"KL/chosen_KL_mean": -1.1218318939208984,
"KL/mean": -1.4224199056625366,
"KL/rejected_KL_mean": -1.7230072021484375,
"KL/std": 0.9665278196334839,
"epoch": 0.10430839002267574,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6011764407157898,
"fcm_dpo/q_t": 0.48505643010139465,
"grad_norm": 30.984333038330078,
"learning_rate": 4.999965034812934e-07,
"logits/chosen": 0.10411858558654785,
"logits/rejected": 0.06089819222688675,
"logps/chosen": -67.06488037109375,
"logps/ref_chosen": -65.94305419921875,
"logps/ref_rejected": -89.7735595703125,
"logps/rejected": -91.49656677246094,
"loss": 1.3307,
"margin_dpo/margin_mean": 0.6011757850646973,
"margin_dpo/margin_std": 1.1689398288726807,
"step": 69
},
{
"KL/chosen_KL_mean": -1.2116470336914062,
"KL/mean": -1.4070619344711304,
"KL/rejected_KL_mean": -1.6024742126464844,
"KL/std": 0.9462293386459351,
"epoch": 0.10582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.39082786440849304,
"fcm_dpo/q_t": 0.49027496576309204,
"grad_norm": 28.671159744262695,
"learning_rate": 4.999860140229787e-07,
"logits/chosen": 0.14850440621376038,
"logits/rejected": 0.12535987794399261,
"logps/chosen": -63.169559478759766,
"logps/ref_chosen": -61.95791244506836,
"logps/ref_rejected": -75.80945587158203,
"logps/rejected": -77.41193389892578,
"loss": 1.3516,
"margin_dpo/margin_mean": 0.3908279538154602,
"margin_dpo/margin_std": 1.268478274345398,
"step": 70
},
{
"KL/chosen_KL_mean": -1.4452323913574219,
"KL/mean": -1.5527398586273193,
"KL/rejected_KL_mean": -1.6602516174316406,
"KL/std": 1.0212029218673706,
"epoch": 0.1073318216175359,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2150188833475113,
"fcm_dpo/q_t": 0.4946098327636719,
"grad_norm": 28.61311912536621,
"learning_rate": 4.999685319184688e-07,
"logits/chosen": 0.09887754917144775,
"logits/rejected": 0.08335565775632858,
"logps/chosen": -64.79280853271484,
"logps/ref_chosen": -63.34757995605469,
"logps/ref_rejected": -67.49658203125,
"logps/rejected": -69.15682983398438,
"loss": 1.3706,
"margin_dpo/margin_mean": 0.21501855552196503,
"margin_dpo/margin_std": 1.4932568073272705,
"step": 71
},
{
"KL/chosen_KL_mean": -1.1772842407226562,
"KL/mean": -1.5442196130752563,
"KL/rejected_KL_mean": -1.9111518859863281,
"KL/std": 1.0610476732254028,
"epoch": 0.10884353741496598,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7338677644729614,
"fcm_dpo/q_t": 0.4817340075969696,
"grad_norm": 30.302104949951172,
"learning_rate": 4.999440576567755e-07,
"logits/chosen": 0.10478617250919342,
"logits/rejected": 0.04250642657279968,
"logps/chosen": -57.036582946777344,
"logps/ref_chosen": -55.85929870605469,
"logps/ref_rejected": -68.45423889160156,
"logps/rejected": -70.36539459228516,
"loss": 1.3182,
"margin_dpo/margin_mean": 0.7338675260543823,
"margin_dpo/margin_std": 1.2317919731140137,
"step": 72
},
{
"KL/chosen_KL_mean": -1.8173637390136719,
"KL/mean": -1.8717069625854492,
"KL/rejected_KL_mean": -1.9260501861572266,
"KL/std": 1.2456495761871338,
"epoch": 0.11035525321239607,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10868409276008606,
"fcm_dpo/q_t": 0.4974502623081207,
"grad_norm": 32.11731719970703,
"learning_rate": 4.999125919224965e-07,
"logits/chosen": 0.09151546657085419,
"logits/rejected": 0.07785354554653168,
"logps/chosen": -70.95616912841797,
"logps/ref_chosen": -69.13880920410156,
"logps/ref_rejected": -79.04586791992188,
"logps/rejected": -80.971923828125,
"loss": 1.3824,
"margin_dpo/margin_mean": 0.10868486762046814,
"margin_dpo/margin_std": 1.5893125534057617,
"step": 73
},
{
"KL/chosen_KL_mean": -1.2636852264404297,
"KL/mean": -1.5985901355743408,
"KL/rejected_KL_mean": -1.9334907531738281,
"KL/std": 1.0060906410217285,
"epoch": 0.11186696900982615,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6698073744773865,
"fcm_dpo/q_t": 0.48337024450302124,
"grad_norm": 27.95173454284668,
"learning_rate": 4.998741355957963e-07,
"logits/chosen": 0.14399270713329315,
"logits/rejected": 0.0904250368475914,
"logps/chosen": -51.18742370605469,
"logps/ref_chosen": -49.923736572265625,
"logps/ref_rejected": -81.73213958740234,
"logps/rejected": -83.66563415527344,
"loss": 1.3251,
"margin_dpo/margin_mean": 0.6698077321052551,
"margin_dpo/margin_std": 1.332854986190796,
"step": 74
},
{
"KL/chosen_KL_mean": -1.3782119750976562,
"KL/mean": -1.7573962211608887,
"KL/rejected_KL_mean": -2.1365795135498047,
"KL/std": 1.0745444297790527,
"epoch": 0.11337868480725624,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7583687901496887,
"fcm_dpo/q_t": 0.4811515808105469,
"grad_norm": 26.23224449157715,
"learning_rate": 4.998286897523808e-07,
"logits/chosen": 0.1194392591714859,
"logits/rejected": 0.08680184185504913,
"logps/chosen": -47.446964263916016,
"logps/ref_chosen": -46.06875228881836,
"logps/ref_rejected": -66.1181411743164,
"logps/rejected": -68.25472259521484,
"loss": 1.3163,
"margin_dpo/margin_mean": 0.7583684921264648,
"margin_dpo/margin_std": 1.3214094638824463,
"step": 75
},
{
"KL/chosen_KL_mean": -1.6166152954101562,
"KL/mean": -1.7738971710205078,
"KL/rejected_KL_mean": -1.931182861328125,
"KL/std": 1.227593183517456,
"epoch": 0.11489040060468632,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3145681321620941,
"fcm_dpo/q_t": 0.49217700958251953,
"grad_norm": 29.346458435058594,
"learning_rate": 4.997762556634679e-07,
"logits/chosen": 0.11621889472007751,
"logits/rejected": 0.0730433315038681,
"logps/chosen": -55.67936706542969,
"logps/ref_chosen": -54.06275177001953,
"logps/ref_rejected": -74.87464141845703,
"logps/rejected": -76.80582427978516,
"loss": 1.3614,
"margin_dpo/margin_mean": 0.3145679235458374,
"margin_dpo/margin_std": 1.596081256866455,
"step": 76
},
{
"KL/chosen_KL_mean": -1.6418571472167969,
"KL/mean": -2.0117712020874023,
"KL/rejected_KL_mean": -2.381683349609375,
"KL/std": 1.2005259990692139,
"epoch": 0.1164021164021164,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7398277521133423,
"fcm_dpo/q_t": 0.48161664605140686,
"grad_norm": 29.373077392578125,
"learning_rate": 4.99716834795752e-07,
"logits/chosen": 0.1848347783088684,
"logits/rejected": 0.14166870713233948,
"logps/chosen": -54.71794891357422,
"logps/ref_chosen": -53.07609176635742,
"logps/ref_rejected": -74.45601654052734,
"logps/rejected": -76.83769989013672,
"loss": 1.3187,
"margin_dpo/margin_mean": 0.7398276329040527,
"margin_dpo/margin_std": 1.3978897333145142,
"step": 77
},
{
"KL/chosen_KL_mean": -1.8243026733398438,
"KL/mean": -2.026995897293091,
"KL/rejected_KL_mean": -2.2296905517578125,
"KL/std": 1.23481285572052,
"epoch": 0.11791383219954649,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.40538251399993896,
"fcm_dpo/q_t": 0.4899021089076996,
"grad_norm": 29.415245056152344,
"learning_rate": 4.996504288113623e-07,
"logits/chosen": 0.09819696843624115,
"logits/rejected": 0.07809644937515259,
"logps/chosen": -69.54972839355469,
"logps/ref_chosen": -67.72541809082031,
"logps/ref_rejected": -79.03926849365234,
"logps/rejected": -81.26895904541016,
"loss": 1.3508,
"margin_dpo/margin_mean": 0.40538227558135986,
"margin_dpo/margin_std": 1.3632533550262451,
"step": 78
},
{
"KL/chosen_KL_mean": -1.7598819732666016,
"KL/mean": -2.3264784812927246,
"KL/rejected_KL_mean": -2.893077850341797,
"KL/std": 1.5687503814697266,
"epoch": 0.11942554799697656,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.133197546005249,
"fcm_dpo/q_t": 0.4722020924091339,
"grad_norm": 30.811988830566406,
"learning_rate": 4.995770395678171e-07,
"logits/chosen": 0.16424202919006348,
"logits/rejected": 0.10485352575778961,
"logps/chosen": -53.92052459716797,
"logps/ref_chosen": -52.16064453125,
"logps/ref_rejected": -83.31062316894531,
"logps/rejected": -86.20369720458984,
"loss": 1.2864,
"margin_dpo/margin_mean": 1.133197546005249,
"margin_dpo/margin_std": 2.019831895828247,
"step": 79
},
{
"KL/chosen_KL_mean": -2.059539794921875,
"KL/mean": -2.3286733627319336,
"KL/rejected_KL_mean": -2.5978050231933594,
"KL/std": 1.3776694536209106,
"epoch": 0.12093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5382720828056335,
"fcm_dpo/q_t": 0.48672568798065186,
"grad_norm": 28.456384658813477,
"learning_rate": 4.994966691179711e-07,
"logits/chosen": 0.16742676496505737,
"logits/rejected": 0.10579954087734222,
"logps/chosen": -63.47010040283203,
"logps/ref_chosen": -61.410560607910156,
"logps/ref_rejected": -78.66004943847656,
"logps/rejected": -81.25785827636719,
"loss": 1.3428,
"margin_dpo/margin_mean": 0.5382719039916992,
"margin_dpo/margin_std": 1.8884304761886597,
"step": 80
},
{
"KL/chosen_KL_mean": -2.1660118103027344,
"KL/mean": -2.568941831588745,
"KL/rejected_KL_mean": -2.9718704223632812,
"KL/std": 1.4467761516571045,
"epoch": 0.12244897959183673,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8058617115020752,
"fcm_dpo/q_t": 0.4801437556743622,
"grad_norm": 29.072032928466797,
"learning_rate": 4.994093197099587e-07,
"logits/chosen": 0.13002906739711761,
"logits/rejected": 0.09610335528850555,
"logps/chosen": -65.97038269042969,
"logps/ref_chosen": -63.80437088012695,
"logps/ref_rejected": -79.3484115600586,
"logps/rejected": -82.32028198242188,
"loss": 1.3155,
"margin_dpo/margin_mean": 0.8058614730834961,
"margin_dpo/margin_std": 1.7406330108642578,
"step": 81
},
{
"KL/chosen_KL_mean": -1.7479782104492188,
"KL/mean": -2.3002214431762695,
"KL/rejected_KL_mean": -2.852466583251953,
"KL/std": 1.379155158996582,
"epoch": 0.12396069538926682,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1044880151748657,
"fcm_dpo/q_t": 0.47257328033447266,
"grad_norm": 26.456167221069336,
"learning_rate": 4.993149937871306e-07,
"logits/chosen": 0.11616471409797668,
"logits/rejected": 0.05289188772439957,
"logps/chosen": -50.56587219238281,
"logps/ref_chosen": -48.817893981933594,
"logps/ref_rejected": -70.31497955322266,
"logps/rejected": -73.16744995117188,
"loss": 1.2853,
"margin_dpo/margin_mean": 1.1044878959655762,
"margin_dpo/margin_std": 1.5824530124664307,
"step": 82
},
{
"KL/chosen_KL_mean": -2.026599884033203,
"KL/mean": -2.6412153244018555,
"KL/rejected_KL_mean": -3.2558326721191406,
"KL/std": 1.5034358501434326,
"epoch": 0.1254724111866969,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.229237675666809,
"fcm_dpo/q_t": 0.4697047173976898,
"grad_norm": 28.895191192626953,
"learning_rate": 4.992136939879856e-07,
"logits/chosen": 0.20435649156570435,
"logits/rejected": 0.1537850946187973,
"logps/chosen": -59.17737579345703,
"logps/ref_chosen": -57.15077209472656,
"logps/ref_rejected": -75.1710205078125,
"logps/rejected": -78.4268569946289,
"loss": 1.2759,
"margin_dpo/margin_mean": 1.2292377948760986,
"margin_dpo/margin_std": 1.876763105392456,
"step": 83
},
{
"KL/chosen_KL_mean": -2.5615406036376953,
"KL/mean": -2.99251127243042,
"KL/rejected_KL_mean": -3.4234886169433594,
"KL/std": 1.8197745084762573,
"epoch": 0.12698412698412698,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8619481325149536,
"fcm_dpo/q_t": 0.4786767363548279,
"grad_norm": 30.56605339050293,
"learning_rate": 4.991054231460969e-07,
"logits/chosen": 0.1459577977657318,
"logits/rejected": 0.10570729523897171,
"logps/chosen": -67.3388442993164,
"logps/ref_chosen": -64.77729797363281,
"logps/ref_rejected": -84.71949768066406,
"logps/rejected": -88.14299011230469,
"loss": 1.3109,
"margin_dpo/margin_mean": 0.8619476556777954,
"margin_dpo/margin_std": 1.8914613723754883,
"step": 84
},
{
"KL/chosen_KL_mean": -2.258808135986328,
"KL/mean": -2.887326955795288,
"KL/rejected_KL_mean": -3.5158462524414062,
"KL/std": 1.792755365371704,
"epoch": 0.12849584278155707,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2570416927337646,
"fcm_dpo/q_t": 0.46897968649864197,
"grad_norm": 26.82134246826172,
"learning_rate": 4.989901842900325e-07,
"logits/chosen": 0.17407938838005066,
"logits/rejected": 0.12974150478839874,
"logps/chosen": -52.51050567626953,
"logps/ref_chosen": -50.25169372558594,
"logps/ref_rejected": -66.55439758300781,
"logps/rejected": -70.07023620605469,
"loss": 1.2753,
"margin_dpo/margin_mean": 1.2570418119430542,
"margin_dpo/margin_std": 2.020524501800537,
"step": 85
},
{
"KL/chosen_KL_mean": -2.8597545623779297,
"KL/mean": -3.328057289123535,
"KL/rejected_KL_mean": -3.7963600158691406,
"KL/std": 1.9231126308441162,
"epoch": 0.13000755857898716,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9366029500961304,
"fcm_dpo/q_t": 0.4768456518650055,
"grad_norm": 27.291244506835938,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 0.1812715232372284,
"logits/rejected": 0.16291239857673645,
"logps/chosen": -63.58893585205078,
"logps/ref_chosen": -60.72917938232422,
"logps/ref_rejected": -72.30961608886719,
"logps/rejected": -76.10597229003906,
"loss": 1.3055,
"margin_dpo/margin_mean": 0.936603307723999,
"margin_dpo/margin_std": 2.06821346282959,
"step": 86
},
{
"KL/chosen_KL_mean": -3.0067214965820312,
"KL/mean": -3.5237975120544434,
"KL/rejected_KL_mean": -4.040866851806641,
"KL/std": 2.07621431350708,
"epoch": 0.13151927437641722,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.034153699874878,
"fcm_dpo/q_t": 0.4745134115219116,
"grad_norm": 30.608993530273438,
"learning_rate": 4.987388156241114e-07,
"logits/chosen": 0.16269248723983765,
"logits/rejected": 0.10423875600099564,
"logps/chosen": -68.76469421386719,
"logps/ref_chosen": -65.75796508789062,
"logps/ref_rejected": -84.81159973144531,
"logps/rejected": -88.85246276855469,
"loss": 1.3017,
"margin_dpo/margin_mean": 1.0341542959213257,
"margin_dpo/margin_std": 2.5299935340881348,
"step": 87
},
{
"KL/chosen_KL_mean": -3.0388126373291016,
"KL/mean": -3.4782047271728516,
"KL/rejected_KL_mean": -3.917591094970703,
"KL/std": 2.3414268493652344,
"epoch": 0.1330309901738473,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8787797689437866,
"fcm_dpo/q_t": 0.4786525368690491,
"grad_norm": 30.143423080444336,
"learning_rate": 4.986026928455767e-07,
"logits/chosen": 0.23746399581432343,
"logits/rejected": 0.21029482781887054,
"logps/chosen": -65.86283874511719,
"logps/ref_chosen": -62.82402801513672,
"logps/ref_rejected": -74.9607162475586,
"logps/rejected": -78.87831115722656,
"loss": 1.3231,
"margin_dpo/margin_mean": 0.8787797689437866,
"margin_dpo/margin_std": 2.9315881729125977,
"step": 88
},
{
"KL/chosen_KL_mean": -2.5165977478027344,
"KL/mean": -3.214366912841797,
"KL/rejected_KL_mean": -3.9121322631835938,
"KL/std": 2.134425640106201,
"epoch": 0.1345427059712774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3955327272415161,
"fcm_dpo/q_t": 0.4659721851348877,
"grad_norm": 29.33747100830078,
"learning_rate": 4.984596161153135e-07,
"logits/chosen": 0.23213061690330505,
"logits/rejected": 0.15128636360168457,
"logps/chosen": -43.708030700683594,
"logps/ref_chosen": -41.191436767578125,
"logps/ref_rejected": -85.44769287109375,
"logps/rejected": -89.35983276367188,
"loss": 1.268,
"margin_dpo/margin_mean": 1.3955327272415161,
"margin_dpo/margin_std": 2.4990651607513428,
"step": 89
},
{
"KL/chosen_KL_mean": -3.1267757415771484,
"KL/mean": -3.7660017013549805,
"KL/rejected_KL_mean": -4.405231475830078,
"KL/std": 2.458479404449463,
"epoch": 0.1360544217687075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.278447151184082,
"fcm_dpo/q_t": 0.4686064124107361,
"grad_norm": 29.23085594177246,
"learning_rate": 4.983095894354857e-07,
"logits/chosen": 0.13286244869232178,
"logits/rejected": 0.0797010213136673,
"logps/chosen": -59.71068572998047,
"logps/ref_chosen": -56.58390808105469,
"logps/ref_rejected": -86.86978149414062,
"logps/rejected": -91.27500915527344,
"loss": 1.2785,
"margin_dpo/margin_mean": 1.2784464359283447,
"margin_dpo/margin_std": 2.485658645629883,
"step": 90
},
{
"KL/chosen_KL_mean": -3.142923355102539,
"KL/mean": -3.6927480697631836,
"KL/rejected_KL_mean": -4.242572784423828,
"KL/std": 2.266369342803955,
"epoch": 0.13756613756613756,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0996489524841309,
"fcm_dpo/q_t": 0.4733736217021942,
"grad_norm": 25.786420822143555,
"learning_rate": 4.98152617002662e-07,
"logits/chosen": 0.17510226368904114,
"logits/rejected": 0.13035190105438232,
"logps/chosen": -55.5252685546875,
"logps/ref_chosen": -52.38234329223633,
"logps/ref_rejected": -72.17642211914062,
"logps/rejected": -76.41899108886719,
"loss": 1.2985,
"margin_dpo/margin_mean": 1.09964919090271,
"margin_dpo/margin_std": 2.7757925987243652,
"step": 91
},
{
"KL/chosen_KL_mean": -3.198556900024414,
"KL/mean": -4.033766746520996,
"KL/rejected_KL_mean": -4.868980407714844,
"KL/std": 2.7098605632781982,
"epoch": 0.13907785336356765,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.670418381690979,
"fcm_dpo/q_t": 0.45981651544570923,
"grad_norm": 26.77798080444336,
"learning_rate": 4.979887032076988e-07,
"logits/chosen": 0.21588903665542603,
"logits/rejected": 0.17576783895492554,
"logps/chosen": -56.20726013183594,
"logps/ref_chosen": -53.00870132446289,
"logps/ref_rejected": -79.77812957763672,
"logps/rejected": -84.64710998535156,
"loss": 1.2519,
"margin_dpo/margin_mean": 1.6704179048538208,
"margin_dpo/margin_std": 3.2294511795043945,
"step": 92
},
{
"KL/chosen_KL_mean": -3.550262451171875,
"KL/mean": -4.06852912902832,
"KL/rejected_KL_mean": -4.586795806884766,
"KL/std": 2.6479063034057617,
"epoch": 0.14058956916099774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0365362167358398,
"fcm_dpo/q_t": 0.47524404525756836,
"grad_norm": 24.42961311340332,
"learning_rate": 4.978178526356172e-07,
"logits/chosen": 0.21999691426753998,
"logits/rejected": 0.18996566534042358,
"logps/chosen": -48.457313537597656,
"logps/ref_chosen": -44.90705108642578,
"logps/ref_rejected": -58.7879524230957,
"logps/rejected": -63.37474822998047,
"loss": 1.3107,
"margin_dpo/margin_mean": 1.0365365743637085,
"margin_dpo/margin_std": 3.226975679397583,
"step": 93
},
{
"KL/chosen_KL_mean": -3.274158477783203,
"KL/mean": -4.244356632232666,
"KL/rejected_KL_mean": -5.214557647705078,
"KL/std": 3.2750868797302246,
"epoch": 0.1421012849584278,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9403965473175049,
"fcm_dpo/q_t": 0.45354098081588745,
"grad_norm": 27.23829460144043,
"learning_rate": 4.976400700654751e-07,
"logits/chosen": 0.2586434483528137,
"logits/rejected": 0.21814142167568207,
"logps/chosen": -63.21192932128906,
"logps/ref_chosen": -59.93777084350586,
"logps/ref_rejected": -79.3138427734375,
"logps/rejected": -84.52839660644531,
"loss": 1.2412,
"margin_dpo/margin_mean": 1.9403966665267944,
"margin_dpo/margin_std": 4.026358604431152,
"step": 94
},
{
"KL/chosen_KL_mean": -4.264810562133789,
"KL/mean": -5.062840461730957,
"KL/rejected_KL_mean": -5.860870361328125,
"KL/std": 2.7003493309020996,
"epoch": 0.1436130007558579,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5960580110549927,
"fcm_dpo/q_t": 0.46138566732406616,
"grad_norm": 29.303754806518555,
"learning_rate": 4.974553604702332e-07,
"logits/chosen": 0.16401472687721252,
"logits/rejected": 0.09887714684009552,
"logps/chosen": -64.43330383300781,
"logps/ref_chosen": -60.168487548828125,
"logps/ref_rejected": -90.73665618896484,
"logps/rejected": -96.59752655029297,
"loss": 1.2612,
"margin_dpo/margin_mean": 1.5960575342178345,
"margin_dpo/margin_std": 3.384384870529175,
"step": 95
},
{
"KL/chosen_KL_mean": -4.164896011352539,
"KL/mean": -5.096510887145996,
"KL/rejected_KL_mean": -6.028129577636719,
"KL/std": 3.407775402069092,
"epoch": 0.14512471655328799,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.863236904144287,
"fcm_dpo/q_t": 0.45675230026245117,
"grad_norm": 27.36403465270996,
"learning_rate": 4.972637290166157e-07,
"logits/chosen": 0.2020627111196518,
"logits/rejected": 0.158769428730011,
"logps/chosen": -64.83367156982422,
"logps/ref_chosen": -60.66877746582031,
"logps/ref_rejected": -88.30673217773438,
"logps/rejected": -94.3348617553711,
"loss": 1.2579,
"margin_dpo/margin_mean": 1.863236427307129,
"margin_dpo/margin_std": 4.4731011390686035,
"step": 96
},
{
"KL/chosen_KL_mean": -5.011560440063477,
"KL/mean": -5.50346565246582,
"KL/rejected_KL_mean": -5.995372772216797,
"KL/std": 3.6453664302825928,
"epoch": 0.14663643235071808,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9838161468505859,
"fcm_dpo/q_t": 0.476479172706604,
"grad_norm": 36.27960968017578,
"learning_rate": 4.970651810649666e-07,
"logits/chosen": 0.14650177955627441,
"logits/rejected": 0.10253134369850159,
"logps/chosen": -70.05567932128906,
"logps/ref_chosen": -65.04412078857422,
"logps/ref_rejected": -78.42092895507812,
"logps/rejected": -84.41630554199219,
"loss": 1.3507,
"margin_dpo/margin_mean": 0.9838162660598755,
"margin_dpo/margin_std": 5.006588935852051,
"step": 97
},
{
"KL/chosen_KL_mean": -4.654573440551758,
"KL/mean": -4.997580528259277,
"KL/rejected_KL_mean": -5.3405914306640625,
"KL/std": 3.2552270889282227,
"epoch": 0.14814814814814814,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6860167980194092,
"fcm_dpo/q_t": 0.4829104542732239,
"grad_norm": 30.320602416992188,
"learning_rate": 4.968597221690985e-07,
"logits/chosen": 0.252997487783432,
"logits/rejected": 0.2249354124069214,
"logps/chosen": -60.157806396484375,
"logps/ref_chosen": -55.503231048583984,
"logps/ref_rejected": -72.81553649902344,
"logps/rejected": -78.1561279296875,
"loss": 1.3452,
"margin_dpo/margin_mean": 0.6860172748565674,
"margin_dpo/margin_std": 3.23012113571167,
"step": 98
},
{
"KL/chosen_KL_mean": -4.6743316650390625,
"KL/mean": -5.189509391784668,
"KL/rejected_KL_mean": -5.704692840576172,
"KL/std": 3.4976234436035156,
"epoch": 0.14965986394557823,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0303622484207153,
"fcm_dpo/q_t": 0.47562655806541443,
"grad_norm": 32.572608947753906,
"learning_rate": 4.966473580761389e-07,
"logits/chosen": 0.24607810378074646,
"logits/rejected": 0.2065376341342926,
"logps/chosen": -63.249969482421875,
"logps/ref_chosen": -58.57563781738281,
"logps/ref_rejected": -78.693603515625,
"logps/rejected": -84.39830017089844,
"loss": 1.3361,
"margin_dpo/margin_mean": 1.0303621292114258,
"margin_dpo/margin_std": 4.529152870178223,
"step": 99
},
{
"KL/chosen_KL_mean": -5.290992736816406,
"KL/mean": -6.019989967346191,
"KL/rejected_KL_mean": -6.748981475830078,
"KL/std": 4.169232368469238,
"epoch": 0.15117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4579877853393555,
"fcm_dpo/q_t": 0.46763741970062256,
"grad_norm": 33.273681640625,
"learning_rate": 4.964280947263676e-07,
"logits/chosen": 0.20944450795650482,
"logits/rejected": 0.20199376344680786,
"logps/chosen": -84.87442779541016,
"logps/ref_chosen": -79.58343505859375,
"logps/ref_rejected": -92.152587890625,
"logps/rejected": -98.90156555175781,
"loss": 1.332,
"margin_dpo/margin_mean": 1.4579877853393555,
"margin_dpo/margin_std": 5.84016752243042,
"step": 100
},
{
"KL/chosen_KL_mean": -4.271394729614258,
"KL/mean": -5.309800148010254,
"KL/rejected_KL_mean": -6.34820556640625,
"KL/std": 3.68625545501709,
"epoch": 0.15268329554043839,
"fcm_dpo/beta": 0.1042826771736145,
"fcm_dpo/delta": 0.08218219131231308,
"fcm_dpo/margin": 2.076812267303467,
"fcm_dpo/q_t": 0.4498174786567688,
"grad_norm": 27.07867431640625,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": 0.23947615921497345,
"logits/rejected": 0.19994020462036133,
"logps/chosen": -56.60417938232422,
"logps/ref_chosen": -52.332786560058594,
"logps/ref_rejected": -69.55589294433594,
"logps/rejected": -75.90409851074219,
"loss": 1.2283,
"margin_dpo/margin_mean": 2.076812267303467,
"margin_dpo/margin_std": 4.143991470336914,
"step": 101
},
{
"KL/chosen_KL_mean": -5.339498519897461,
"KL/mean": -5.742995262145996,
"KL/rejected_KL_mean": -6.146495819091797,
"KL/std": 3.6950936317443848,
"epoch": 0.15419501133786848,
"fcm_dpo/beta": 0.10856536030769348,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8069917559623718,
"fcm_dpo/q_t": 0.4828066825866699,
"grad_norm": 36.31706619262695,
"learning_rate": 4.959688949822748e-07,
"logits/chosen": 0.1819576621055603,
"logits/rejected": 0.1423925906419754,
"logps/chosen": -70.08297729492188,
"logps/ref_chosen": -64.74348449707031,
"logps/ref_rejected": -69.06132507324219,
"logps/rejected": -75.20782470703125,
"loss": 1.3832,
"margin_dpo/margin_mean": 0.8069913387298584,
"margin_dpo/margin_std": 5.488136291503906,
"step": 102
},
{
"KL/chosen_KL_mean": -4.825580596923828,
"KL/mean": -5.593038558959961,
"KL/rejected_KL_mean": -6.360500335693359,
"KL/std": 3.912938117980957,
"epoch": 0.15570672713529857,
"fcm_dpo/beta": 0.11347013711929321,
"fcm_dpo/delta": 0.08650438487529755,
"fcm_dpo/margin": 1.5349235534667969,
"fcm_dpo/q_t": 0.46038591861724854,
"grad_norm": 33.3403434753418,
"learning_rate": 4.957289714327572e-07,
"logits/chosen": 0.2608705461025238,
"logits/rejected": 0.22703281044960022,
"logps/chosen": -68.66222381591797,
"logps/ref_chosen": -63.83664321899414,
"logps/ref_rejected": -79.32362365722656,
"logps/rejected": -85.68412780761719,
"loss": 1.2615,
"margin_dpo/margin_mean": 1.5349233150482178,
"margin_dpo/margin_std": 3.726921558380127,
"step": 103
},
{
"KL/chosen_KL_mean": -4.706798553466797,
"KL/mean": -5.820075035095215,
"KL/rejected_KL_mean": -6.933357238769531,
"KL/std": 4.9595770835876465,
"epoch": 0.15721844293272866,
"fcm_dpo/beta": 0.13176845014095306,
"fcm_dpo/delta": 0.12116898596286774,
"fcm_dpo/margin": 2.2265567779541016,
"fcm_dpo/q_t": 0.4444318413734436,
"grad_norm": 41.81438446044922,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 0.24606768786907196,
"logits/rejected": 0.15957264602184296,
"logps/chosen": -65.70600891113281,
"logps/ref_chosen": -60.99920654296875,
"logps/ref_rejected": -98.84645080566406,
"logps/rejected": -105.77981567382812,
"loss": 1.2672,
"margin_dpo/margin_mean": 2.2265567779541016,
"margin_dpo/margin_std": 6.326627731323242,
"step": 104
},
{
"KL/chosen_KL_mean": -5.340984344482422,
"KL/mean": -6.228916168212891,
"KL/rejected_KL_mean": -7.116844177246094,
"KL/std": 3.9524288177490234,
"epoch": 0.15873015873015872,
"fcm_dpo/beta": 0.13953134417533875,
"fcm_dpo/delta": 0.0847325474023819,
"fcm_dpo/margin": 1.7758561372756958,
"fcm_dpo/q_t": 0.44833284616470337,
"grad_norm": 43.252464294433594,
"learning_rate": 4.952285105344791e-07,
"logits/chosen": 0.1813499629497528,
"logits/rejected": 0.12760114669799805,
"logps/chosen": -76.29125213623047,
"logps/ref_chosen": -70.95027160644531,
"logps/ref_rejected": -87.88340759277344,
"logps/rejected": -95.00025939941406,
"loss": 1.259,
"margin_dpo/margin_mean": 1.7758560180664062,
"margin_dpo/margin_std": 4.853283882141113,
"step": 105
},
{
"KL/chosen_KL_mean": -4.989650726318359,
"KL/mean": -5.752998352050781,
"KL/rejected_KL_mean": -6.5163421630859375,
"KL/std": 3.8832058906555176,
"epoch": 0.1602418745275888,
"fcm_dpo/beta": 0.16595645248889923,
"fcm_dpo/delta": 0.16586124897003174,
"fcm_dpo/margin": 1.5266880989074707,
"fcm_dpo/q_t": 0.44716203212738037,
"grad_norm": 55.961551666259766,
"learning_rate": 4.949679871846857e-07,
"logits/chosen": 0.256155788898468,
"logits/rejected": 0.24164044857025146,
"logps/chosen": -67.44898223876953,
"logps/ref_chosen": -62.45933151245117,
"logps/ref_rejected": -67.00595092773438,
"logps/rejected": -73.52229309082031,
"loss": 1.3149,
"margin_dpo/margin_mean": 1.5266880989074707,
"margin_dpo/margin_std": 5.232348442077637,
"step": 106
},
{
"KL/chosen_KL_mean": -6.383148193359375,
"KL/mean": -6.799151420593262,
"KL/rejected_KL_mean": -7.21514892578125,
"KL/std": 4.821557998657227,
"epoch": 0.1617535903250189,
"fcm_dpo/beta": 0.1813592165708542,
"fcm_dpo/delta": 0.05486012250185013,
"fcm_dpo/margin": 0.832004725933075,
"fcm_dpo/q_t": 0.4626944661140442,
"grad_norm": 88.15316009521484,
"learning_rate": 4.947006115536947e-07,
"logits/chosen": 0.14859333634376526,
"logits/rejected": 0.1259216070175171,
"logps/chosen": -82.22111511230469,
"logps/ref_chosen": -75.83796691894531,
"logps/ref_rejected": -87.74038696289062,
"logps/rejected": -94.95553588867188,
"loss": 1.5013,
"margin_dpo/margin_mean": 0.8320047855377197,
"margin_dpo/margin_std": 6.031615257263184,
"step": 107
},
{
"KL/chosen_KL_mean": -5.376251220703125,
"KL/mean": -6.28802490234375,
"KL/rejected_KL_mean": -7.199794769287109,
"KL/std": 3.9641804695129395,
"epoch": 0.16326530612244897,
"fcm_dpo/beta": 0.19046062231063843,
"fcm_dpo/delta": 0.0635533481836319,
"fcm_dpo/margin": 1.823544979095459,
"fcm_dpo/q_t": 0.432595431804657,
"grad_norm": 58.52869415283203,
"learning_rate": 4.944263911205772e-07,
"logits/chosen": 0.20340561866760254,
"logits/rejected": 0.17100761830806732,
"logps/chosen": -73.76948547363281,
"logps/ref_chosen": -68.39323425292969,
"logps/ref_rejected": -83.24267578125,
"logps/rejected": -90.44246673583984,
"loss": 1.2828,
"margin_dpo/margin_mean": 1.8235454559326172,
"margin_dpo/margin_std": 5.186200141906738,
"step": 108
},
{
"KL/chosen_KL_mean": -4.721492767333984,
"KL/mean": -6.179901123046875,
"KL/rejected_KL_mean": -7.638313293457031,
"KL/std": 4.794259071350098,
"epoch": 0.16477702191987906,
"fcm_dpo/beta": 0.17355400323867798,
"fcm_dpo/delta": -0.14321784675121307,
"fcm_dpo/margin": 2.916821241378784,
"fcm_dpo/q_t": 0.3970474600791931,
"grad_norm": 45.6633415222168,
"learning_rate": 4.941453335558681e-07,
"logits/chosen": 0.2231045961380005,
"logits/rejected": 0.16527457535266876,
"logps/chosen": -60.24897384643555,
"logps/ref_chosen": -55.52748107910156,
"logps/ref_rejected": -83.55218505859375,
"logps/rejected": -91.19050598144531,
"loss": 1.0975,
"margin_dpo/margin_mean": 2.916820526123047,
"margin_dpo/margin_std": 5.183098793029785,
"step": 109
},
{
"KL/chosen_KL_mean": -5.953372955322266,
"KL/mean": -6.1177167892456055,
"KL/rejected_KL_mean": -6.282051086425781,
"KL/std": 3.8472280502319336,
"epoch": 0.16628873771730915,
"fcm_dpo/beta": 0.16747182607650757,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3286726772785187,
"fcm_dpo/q_t": 0.4821946620941162,
"grad_norm": 79.94758605957031,
"learning_rate": 4.938574467213517e-07,
"logits/chosen": 0.1649562120437622,
"logits/rejected": 0.17404744029045105,
"logps/chosen": -87.11212158203125,
"logps/ref_chosen": -81.15874481201172,
"logps/ref_rejected": -72.56021118164062,
"logps/rejected": -78.8422622680664,
"loss": 1.4935,
"margin_dpo/margin_mean": 0.32867202162742615,
"margin_dpo/margin_std": 5.137603759765625,
"step": 110
},
{
"KL/chosen_KL_mean": -5.388044357299805,
"KL/mean": -6.336487770080566,
"KL/rejected_KL_mean": -7.284934997558594,
"KL/std": 3.9769458770751953,
"epoch": 0.16780045351473924,
"fcm_dpo/beta": 0.17862893640995026,
"fcm_dpo/delta": 0.07212649285793304,
"fcm_dpo/margin": 1.8968864679336548,
"fcm_dpo/q_t": 0.43422025442123413,
"grad_norm": 47.36850357055664,
"learning_rate": 4.935627386698418e-07,
"logits/chosen": 0.2890339493751526,
"logits/rejected": 0.25080791115760803,
"logps/chosen": -57.74702835083008,
"logps/ref_chosen": -52.358985900878906,
"logps/ref_rejected": -77.06150817871094,
"logps/rejected": -84.346435546875,
"loss": 1.2395,
"margin_dpo/margin_mean": 1.8968862295150757,
"margin_dpo/margin_std": 4.821080207824707,
"step": 111
},
{
"KL/chosen_KL_mean": -4.940637588500977,
"KL/mean": -6.110255241394043,
"KL/rejected_KL_mean": -7.279872894287109,
"KL/std": 3.8712010383605957,
"epoch": 0.1693121693121693,
"fcm_dpo/beta": 0.17799095809459686,
"fcm_dpo/delta": -0.020651305094361305,
"fcm_dpo/margin": 2.3392438888549805,
"fcm_dpo/q_t": 0.40773606300354004,
"grad_norm": 54.7916259765625,
"learning_rate": 4.932612176449559e-07,
"logits/chosen": 0.1744939088821411,
"logits/rejected": 0.11035922169685364,
"logps/chosen": -67.96070098876953,
"logps/ref_chosen": -63.02006530761719,
"logps/ref_rejected": -111.36941528320312,
"logps/rejected": -118.6492919921875,
"loss": 1.1484,
"margin_dpo/margin_mean": 2.3392434120178223,
"margin_dpo/margin_std": 4.352907657623291,
"step": 112
},
{
"KL/chosen_KL_mean": -4.952281951904297,
"KL/mean": -5.632550239562988,
"KL/rejected_KL_mean": -6.312816619873047,
"KL/std": 3.2599143981933594,
"epoch": 0.1708238851095994,
"fcm_dpo/beta": 0.1789640188217163,
"fcm_dpo/delta": 0.029570240527391434,
"fcm_dpo/margin": 1.3605376482009888,
"fcm_dpo/q_t": 0.4449806213378906,
"grad_norm": 65.98124694824219,
"learning_rate": 4.929528920808854e-07,
"logits/chosen": 0.2306734025478363,
"logits/rejected": 0.18952682614326477,
"logps/chosen": -60.759944915771484,
"logps/ref_chosen": -55.80766296386719,
"logps/ref_rejected": -69.84014129638672,
"logps/rejected": -76.1529541015625,
"loss": 1.2834,
"margin_dpo/margin_mean": 1.3605376482009888,
"margin_dpo/margin_std": 4.0923542976379395,
"step": 113
},
{
"KL/chosen_KL_mean": -4.631011962890625,
"KL/mean": -5.725174427032471,
"KL/rejected_KL_mean": -6.819339752197266,
"KL/std": 3.478522539138794,
"epoch": 0.17233560090702948,
"fcm_dpo/beta": 0.1806703805923462,
"fcm_dpo/delta": 0.005448013544082642,
"fcm_dpo/margin": 2.188323497772217,
"fcm_dpo/q_t": 0.41690295934677124,
"grad_norm": 52.3661994934082,
"learning_rate": 4.92637770602159e-07,
"logits/chosen": 0.2191499024629593,
"logits/rejected": 0.15943503379821777,
"logps/chosen": -70.96378326416016,
"logps/ref_chosen": -66.33277130126953,
"logps/ref_rejected": -71.61489868164062,
"logps/rejected": -78.43423461914062,
"loss": 1.1574,
"margin_dpo/margin_mean": 2.1883232593536377,
"margin_dpo/margin_std": 4.21035623550415,
"step": 114
},
{
"KL/chosen_KL_mean": -4.939592361450195,
"KL/mean": -5.821917533874512,
"KL/rejected_KL_mean": -6.704244613647461,
"KL/std": 3.1544241905212402,
"epoch": 0.17384731670445955,
"fcm_dpo/beta": 0.19181868433952332,
"fcm_dpo/delta": 0.07429927587509155,
"fcm_dpo/margin": 1.7646498680114746,
"fcm_dpo/q_t": 0.42928892374038696,
"grad_norm": 54.519893646240234,
"learning_rate": 4.923158620234019e-07,
"logits/chosen": 0.2349502593278885,
"logits/rejected": 0.1712397038936615,
"logps/chosen": -60.688629150390625,
"logps/ref_chosen": -55.74903869628906,
"logps/ref_rejected": -79.59849548339844,
"logps/rejected": -86.30274200439453,
"loss": 1.1963,
"margin_dpo/margin_mean": 1.7646501064300537,
"margin_dpo/margin_std": 3.7572741508483887,
"step": 115
},
{
"KL/chosen_KL_mean": -4.6262359619140625,
"KL/mean": -5.866840362548828,
"KL/rejected_KL_mean": -7.107444763183594,
"KL/std": 3.217970848083496,
"epoch": 0.17535903250188964,
"fcm_dpo/beta": 0.18887044489383698,
"fcm_dpo/delta": -0.08994907140731812,
"fcm_dpo/margin": 2.481212615966797,
"fcm_dpo/q_t": 0.3914183974266052,
"grad_norm": 42.89326095581055,
"learning_rate": 4.91987175349089e-07,
"logits/chosen": 0.2203093022108078,
"logits/rejected": 0.15200431644916534,
"logps/chosen": -53.99140167236328,
"logps/ref_chosen": -49.36516571044922,
"logps/ref_rejected": -72.84671020507812,
"logps/rejected": -79.95414733886719,
"loss": 1.0722,
"margin_dpo/margin_mean": 2.481212615966797,
"margin_dpo/margin_std": 3.4258229732513428,
"step": 116
},
{
"KL/chosen_KL_mean": -4.495201110839844,
"KL/mean": -5.37297248840332,
"KL/rejected_KL_mean": -6.250743865966797,
"KL/std": 2.998063325881958,
"epoch": 0.17687074829931973,
"fcm_dpo/beta": 0.16782402992248535,
"fcm_dpo/delta": -0.06874273717403412,
"fcm_dpo/margin": 1.755537509918213,
"fcm_dpo/q_t": 0.4289935231208801,
"grad_norm": 46.18778991699219,
"learning_rate": 4.916517197732933e-07,
"logits/chosen": 0.2181738018989563,
"logits/rejected": 0.18157288432121277,
"logps/chosen": -62.20610046386719,
"logps/ref_chosen": -57.710899353027344,
"logps/ref_rejected": -69.77253723144531,
"logps/rejected": -76.02328491210938,
"loss": 1.2104,
"margin_dpo/margin_mean": 1.7555381059646606,
"margin_dpo/margin_std": 3.6577625274658203,
"step": 117
},
{
"KL/chosen_KL_mean": -4.316884994506836,
"KL/mean": -5.443312644958496,
"KL/rejected_KL_mean": -6.569736480712891,
"KL/std": 3.2011799812316895,
"epoch": 0.17838246409674982,
"fcm_dpo/beta": 0.1683710217475891,
"fcm_dpo/delta": 0.02489166334271431,
"fcm_dpo/margin": 2.252856731414795,
"fcm_dpo/q_t": 0.4135647118091583,
"grad_norm": 44.422664642333984,
"learning_rate": 4.913095046794281e-07,
"logits/chosen": 0.26736220717430115,
"logits/rejected": 0.22663536667823792,
"logps/chosen": -56.796783447265625,
"logps/ref_chosen": -52.479896545410156,
"logps/ref_rejected": -81.359130859375,
"logps/rejected": -87.92886352539062,
"loss": 1.1169,
"margin_dpo/margin_mean": 2.252856492996216,
"margin_dpo/margin_std": 3.2952795028686523,
"step": 118
},
{
"KL/chosen_KL_mean": -5.435281753540039,
"KL/mean": -6.261282920837402,
"KL/rejected_KL_mean": -7.087284088134766,
"KL/std": 3.5549156665802,
"epoch": 0.17989417989417988,
"fcm_dpo/beta": 0.1748862862586975,
"fcm_dpo/delta": 0.03239409253001213,
"fcm_dpo/margin": 1.6520025730133057,
"fcm_dpo/q_t": 0.4375898838043213,
"grad_norm": 50.825462341308594,
"learning_rate": 4.909605396399855e-07,
"logits/chosen": 0.17994274199008942,
"logits/rejected": 0.14161953330039978,
"logps/chosen": -66.79295349121094,
"logps/ref_chosen": -61.35767364501953,
"logps/ref_rejected": -75.71510314941406,
"logps/rejected": -82.80238342285156,
"loss": 1.2475,
"margin_dpo/margin_mean": 1.6520024538040161,
"margin_dpo/margin_std": 4.303752899169922,
"step": 119
},
{
"KL/chosen_KL_mean": -4.451520919799805,
"KL/mean": -5.66746711730957,
"KL/rejected_KL_mean": -6.883419036865234,
"KL/std": 3.1244192123413086,
"epoch": 0.18140589569160998,
"fcm_dpo/beta": 0.17477713525295258,
"fcm_dpo/delta": -0.03177126869559288,
"fcm_dpo/margin": 2.4318950176239014,
"fcm_dpo/q_t": 0.4027881622314453,
"grad_norm": 45.37185287475586,
"learning_rate": 4.906048344162676e-07,
"logits/chosen": 0.18768131732940674,
"logits/rejected": 0.12764175236225128,
"logps/chosen": -64.35908508300781,
"logps/ref_chosen": -59.907569885253906,
"logps/ref_rejected": -79.6910629272461,
"logps/rejected": -86.57447814941406,
"loss": 1.0784,
"margin_dpo/margin_mean": 2.4318952560424805,
"margin_dpo/margin_std": 3.188009262084961,
"step": 120
},
{
"KL/chosen_KL_mean": -4.891788482666016,
"KL/mean": -6.08499002456665,
"KL/rejected_KL_mean": -7.278194427490234,
"KL/std": 3.485968589782715,
"epoch": 0.18291761148904007,
"fcm_dpo/beta": 0.1698891520500183,
"fcm_dpo/delta": -0.0071594808250665665,
"fcm_dpo/margin": 2.3864026069641113,
"fcm_dpo/q_t": 0.4125250279903412,
"grad_norm": 41.51447677612305,
"learning_rate": 4.902423989581143e-07,
"logits/chosen": 0.2931978404521942,
"logits/rejected": 0.20720380544662476,
"logps/chosen": -60.557830810546875,
"logps/ref_chosen": -55.66604232788086,
"logps/ref_rejected": -101.56233978271484,
"logps/rejected": -108.84053039550781,
"loss": 1.1083,
"margin_dpo/margin_mean": 2.3864026069641113,
"margin_dpo/margin_std": 3.758132219314575,
"step": 121
},
{
"KL/chosen_KL_mean": -5.182319641113281,
"KL/mean": -6.3793487548828125,
"KL/rejected_KL_mean": -7.576377868652344,
"KL/std": 3.738175868988037,
"epoch": 0.18442932728647016,
"fcm_dpo/beta": 0.17170721292495728,
"fcm_dpo/delta": -0.014732498675584793,
"fcm_dpo/margin": 2.394054889678955,
"fcm_dpo/q_t": 0.414374977350235,
"grad_norm": 51.34184646606445,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 0.18007788062095642,
"logits/rejected": 0.14575596153736115,
"logps/chosen": -68.51669311523438,
"logps/ref_chosen": -63.334373474121094,
"logps/ref_rejected": -73.67523193359375,
"logps/rejected": -81.25161743164062,
"loss": 1.14,
"margin_dpo/margin_mean": 2.394054412841797,
"margin_dpo/margin_std": 4.409090995788574,
"step": 122
},
{
"KL/chosen_KL_mean": -4.814731597900391,
"KL/mean": -5.987967014312744,
"KL/rejected_KL_mean": -7.161205291748047,
"KL/std": 3.269620180130005,
"epoch": 0.18594104308390022,
"fcm_dpo/beta": 0.17096179723739624,
"fcm_dpo/delta": -0.0026538409292697906,
"fcm_dpo/margin": 2.3464741706848145,
"fcm_dpo/q_t": 0.4092286229133606,
"grad_norm": 42.88996124267578,
"learning_rate": 4.894973780788722e-07,
"logits/chosen": 0.2510654628276825,
"logits/rejected": 0.20839804410934448,
"logps/chosen": -61.71347427368164,
"logps/ref_chosen": -56.89874267578125,
"logps/ref_rejected": -78.97028350830078,
"logps/rejected": -86.13148498535156,
"loss": 1.1104,
"margin_dpo/margin_mean": 2.3464739322662354,
"margin_dpo/margin_std": 3.5289008617401123,
"step": 123
},
{
"KL/chosen_KL_mean": -5.191936492919922,
"KL/mean": -6.721776962280273,
"KL/rejected_KL_mean": -8.251617431640625,
"KL/std": 3.5442142486572266,
"epoch": 0.1874527588813303,
"fcm_dpo/beta": 0.15416131913661957,
"fcm_dpo/delta": -0.09668783843517303,
"fcm_dpo/margin": 3.059680938720703,
"fcm_dpo/q_t": 0.3888171315193176,
"grad_norm": 37.990482330322266,
"learning_rate": 4.89114813497619e-07,
"logits/chosen": 0.24223476648330688,
"logits/rejected": 0.18118014931678772,
"logps/chosen": -62.308021545410156,
"logps/ref_chosen": -57.116085052490234,
"logps/ref_rejected": -87.93074035644531,
"logps/rejected": -96.18235778808594,
"loss": 1.0411,
"margin_dpo/margin_mean": 3.0596816539764404,
"margin_dpo/margin_std": 3.643080949783325,
"step": 124
},
{
"KL/chosen_KL_mean": -5.501195907592773,
"KL/mean": -6.80970573425293,
"KL/rejected_KL_mean": -8.118209838867188,
"KL/std": 3.810020923614502,
"epoch": 0.1889644746787604,
"fcm_dpo/beta": 0.14969037473201752,
"fcm_dpo/delta": 0.007746942341327667,
"fcm_dpo/margin": 2.6170156002044678,
"fcm_dpo/q_t": 0.4116974472999573,
"grad_norm": 41.211639404296875,
"learning_rate": 4.887255603610184e-07,
"logits/chosen": 0.26171737909317017,
"logits/rejected": 0.201465904712677,
"logps/chosen": -71.2073745727539,
"logps/ref_chosen": -65.7061767578125,
"logps/ref_rejected": -91.72711944580078,
"logps/rejected": -99.84532928466797,
"loss": 1.112,
"margin_dpo/margin_mean": 2.6170156002044678,
"margin_dpo/margin_std": 3.844024181365967,
"step": 125
},
{
"KL/chosen_KL_mean": -5.178997039794922,
"KL/mean": -5.999068260192871,
"KL/rejected_KL_mean": -6.819145202636719,
"KL/std": 3.7093849182128906,
"epoch": 0.19047619047619047,
"fcm_dpo/beta": 0.17089751362800598,
"fcm_dpo/delta": 0.1385306864976883,
"fcm_dpo/margin": 1.640149474143982,
"fcm_dpo/q_t": 0.4417935013771057,
"grad_norm": 46.21920394897461,
"learning_rate": 4.883296295573176e-07,
"logits/chosen": 0.07007776945829391,
"logits/rejected": 0.06446836143732071,
"logps/chosen": -73.35508728027344,
"logps/ref_chosen": -68.17608642578125,
"logps/ref_rejected": -65.1175537109375,
"logps/rejected": -71.93669891357422,
"loss": 1.2297,
"margin_dpo/margin_mean": 1.640149712562561,
"margin_dpo/margin_std": 3.9861550331115723,
"step": 126
},
{
"KL/chosen_KL_mean": -5.413948059082031,
"KL/mean": -6.823336601257324,
"KL/rejected_KL_mean": -8.232730865478516,
"KL/std": 3.28928804397583,
"epoch": 0.19198790627362056,
"fcm_dpo/beta": 0.160674050450325,
"fcm_dpo/delta": -0.0780230164527893,
"fcm_dpo/margin": 2.818779945373535,
"fcm_dpo/q_t": 0.39240193367004395,
"grad_norm": 40.320648193359375,
"learning_rate": 4.87927032161552e-07,
"logits/chosen": 0.18640990555286407,
"logits/rejected": 0.15599671006202698,
"logps/chosen": -67.29418182373047,
"logps/ref_chosen": -61.88023376464844,
"logps/ref_rejected": -68.46012878417969,
"logps/rejected": -76.69285583496094,
"loss": 1.0581,
"margin_dpo/margin_mean": 2.818779945373535,
"margin_dpo/margin_std": 3.420637607574463,
"step": 127
},
{
"KL/chosen_KL_mean": -5.826803207397461,
"KL/mean": -7.201224327087402,
"KL/rejected_KL_mean": -8.575653076171875,
"KL/std": 4.162779808044434,
"epoch": 0.19349962207105065,
"fcm_dpo/beta": 0.15589803457260132,
"fcm_dpo/delta": -0.0385875478386879,
"fcm_dpo/margin": 2.7488479614257812,
"fcm_dpo/q_t": 0.40568995475769043,
"grad_norm": 41.49650192260742,
"learning_rate": 4.875177794352363e-07,
"logits/chosen": 0.2261795848608017,
"logits/rejected": 0.16798809170722961,
"logps/chosen": -72.53578186035156,
"logps/ref_chosen": -66.708984375,
"logps/ref_rejected": -94.97969055175781,
"logps/rejected": -103.55534362792969,
"loss": 1.1343,
"margin_dpo/margin_mean": 2.748847484588623,
"margin_dpo/margin_std": 4.877597332000732,
"step": 128
},
{
"KL/chosen_KL_mean": -6.429250717163086,
"KL/mean": -7.367158889770508,
"KL/rejected_KL_mean": -8.305068969726562,
"KL/std": 3.734823226928711,
"epoch": 0.19501133786848074,
"fcm_dpo/beta": 0.1696051061153412,
"fcm_dpo/delta": 0.09658706188201904,
"fcm_dpo/margin": 1.8758163452148438,
"fcm_dpo/q_t": 0.43266767263412476,
"grad_norm": 54.703407287597656,
"learning_rate": 4.871018828260491e-07,
"logits/chosen": 0.1982039511203766,
"logits/rejected": 0.19206386804580688,
"logps/chosen": -71.76808166503906,
"logps/ref_chosen": -65.33882904052734,
"logps/ref_rejected": -68.06109619140625,
"logps/rejected": -76.36616516113281,
"loss": 1.2189,
"margin_dpo/margin_mean": 1.8758164644241333,
"margin_dpo/margin_std": 4.385315895080566,
"step": 129
},
{
"KL/chosen_KL_mean": -6.053129196166992,
"KL/mean": -7.374553680419922,
"KL/rejected_KL_mean": -8.695980072021484,
"KL/std": 3.675654411315918,
"epoch": 0.1965230536659108,
"fcm_dpo/beta": 0.16109490394592285,
"fcm_dpo/delta": -0.04224683344364166,
"fcm_dpo/margin": 2.6428451538085938,
"fcm_dpo/q_t": 0.4000886082649231,
"grad_norm": 46.596336364746094,
"learning_rate": 4.866793539675126e-07,
"logits/chosen": 0.17449048161506653,
"logits/rejected": 0.1231340616941452,
"logps/chosen": -64.71387481689453,
"logps/ref_chosen": -58.660743713378906,
"logps/ref_rejected": -79.24510192871094,
"logps/rejected": -87.94108581542969,
"loss": 1.0862,
"margin_dpo/margin_mean": 2.642845630645752,
"margin_dpo/margin_std": 3.5206475257873535,
"step": 130
},
{
"KL/chosen_KL_mean": -5.886774063110352,
"KL/mean": -7.548995018005371,
"KL/rejected_KL_mean": -9.21121597290039,
"KL/std": 4.397624969482422,
"epoch": 0.1980347694633409,
"fcm_dpo/beta": 0.1490076184272766,
"fcm_dpo/delta": -0.1296783685684204,
"fcm_dpo/margin": 3.324448347091675,
"fcm_dpo/q_t": 0.3871789574623108,
"grad_norm": 37.18637466430664,
"learning_rate": 4.86250204678667e-07,
"logits/chosen": 0.17878660559654236,
"logits/rejected": 0.11429747194051743,
"logps/chosen": -58.401309967041016,
"logps/ref_chosen": -52.51453399658203,
"logps/ref_rejected": -85.18299865722656,
"logps/rejected": -94.39421081542969,
"loss": 1.0717,
"margin_dpo/margin_mean": 3.324448347091675,
"margin_dpo/margin_std": 4.856185436248779,
"step": 131
},
{
"KL/chosen_KL_mean": -6.430377960205078,
"KL/mean": -7.8555006980896,
"KL/rejected_KL_mean": -9.28062629699707,
"KL/std": 4.197918891906738,
"epoch": 0.19954648526077098,
"fcm_dpo/beta": 0.14397326111793518,
"fcm_dpo/delta": -0.013092327862977982,
"fcm_dpo/margin": 2.8502440452575684,
"fcm_dpo/q_t": 0.40407687425613403,
"grad_norm": 38.280391693115234,
"learning_rate": 4.858144469637408e-07,
"logits/chosen": 0.258601576089859,
"logits/rejected": 0.225815549492836,
"logps/chosen": -72.11550903320312,
"logps/ref_chosen": -65.68513488769531,
"logps/ref_rejected": -69.54120635986328,
"logps/rejected": -78.82183837890625,
"loss": 1.1043,
"margin_dpo/margin_mean": 2.8502445220947266,
"margin_dpo/margin_std": 4.208562850952148,
"step": 132
},
{
"KL/chosen_KL_mean": -6.6004486083984375,
"KL/mean": -7.834145545959473,
"KL/rejected_KL_mean": -9.06783676147461,
"KL/std": 4.116161346435547,
"epoch": 0.20105820105820105,
"fcm_dpo/beta": 0.14853733777999878,
"fcm_dpo/delta": 0.040870919823646545,
"fcm_dpo/margin": 2.4673891067504883,
"fcm_dpo/q_t": 0.4182808995246887,
"grad_norm": 43.48795700073242,
"learning_rate": 4.853720930118138e-07,
"logits/chosen": 0.17309647798538208,
"logits/rejected": 0.1633840650320053,
"logps/chosen": -70.19856262207031,
"logps/ref_chosen": -63.598114013671875,
"logps/ref_rejected": -73.72798156738281,
"logps/rejected": -82.79582214355469,
"loss": 1.1485,
"margin_dpo/margin_mean": 2.4673893451690674,
"margin_dpo/margin_std": 4.310044288635254,
"step": 133
},
{
"KL/chosen_KL_mean": -6.207492828369141,
"KL/mean": -8.14185905456543,
"KL/rejected_KL_mean": -10.076229095458984,
"KL/std": 4.746306419372559,
"epoch": 0.20256991685563114,
"fcm_dpo/beta": 0.13268497586250305,
"fcm_dpo/delta": -0.15481051802635193,
"fcm_dpo/margin": 3.868732452392578,
"fcm_dpo/q_t": 0.37830227613449097,
"grad_norm": 32.207279205322266,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": 0.3129193186759949,
"logits/rejected": 0.25384044647216797,
"logps/chosen": -60.0020637512207,
"logps/ref_chosen": -53.79457092285156,
"logps/ref_rejected": -74.16741943359375,
"logps/rejected": -84.24365234375,
"loss": 1.0239,
"margin_dpo/margin_mean": 3.8687334060668945,
"margin_dpo/margin_std": 4.69981050491333,
"step": 134
},
{
"KL/chosen_KL_mean": -6.265291213989258,
"KL/mean": -7.531993389129639,
"KL/rejected_KL_mean": -8.798698425292969,
"KL/std": 4.477133750915527,
"epoch": 0.20408163265306123,
"fcm_dpo/beta": 0.1358022391796112,
"fcm_dpo/delta": 0.06674190610647202,
"fcm_dpo/margin": 2.5334067344665527,
"fcm_dpo/q_t": 0.4239792227745056,
"grad_norm": 32.82770538330078,
"learning_rate": 4.844676460754862e-07,
"logits/chosen": 0.20795100927352905,
"logits/rejected": 0.17573975026607513,
"logps/chosen": -55.70636749267578,
"logps/ref_chosen": -49.441078186035156,
"logps/ref_rejected": -65.96878051757812,
"logps/rejected": -74.7674789428711,
"loss": 1.1624,
"margin_dpo/margin_mean": 2.5334067344665527,
"margin_dpo/margin_std": 4.60736083984375,
"step": 135
},
{
"KL/chosen_KL_mean": -7.658565521240234,
"KL/mean": -9.265897750854492,
"KL/rejected_KL_mean": -10.87322998046875,
"KL/std": 4.929014682769775,
"epoch": 0.20559334845049132,
"fcm_dpo/beta": 0.131425142288208,
"fcm_dpo/delta": -0.031049178913235664,
"fcm_dpo/margin": 3.214667797088623,
"fcm_dpo/q_t": 0.4090140461921692,
"grad_norm": 39.72063064575195,
"learning_rate": 4.840055783904106e-07,
"logits/chosen": 0.20339098572731018,
"logits/rejected": 0.1307443231344223,
"logps/chosen": -74.41783142089844,
"logps/ref_chosen": -66.75926208496094,
"logps/ref_rejected": -94.61787414550781,
"logps/rejected": -105.49110412597656,
"loss": 1.1609,
"margin_dpo/margin_mean": 3.2146685123443604,
"margin_dpo/margin_std": 6.277226448059082,
"step": 136
},
{
"KL/chosen_KL_mean": -7.224740982055664,
"KL/mean": -8.883138656616211,
"KL/rejected_KL_mean": -10.54153823852539,
"KL/std": 4.694730758666992,
"epoch": 0.20710506424792138,
"fcm_dpo/beta": 0.13092255592346191,
"fcm_dpo/delta": -0.04417610913515091,
"fcm_dpo/margin": 3.3167953491210938,
"fcm_dpo/q_t": 0.40223658084869385,
"grad_norm": 32.55488967895508,
"learning_rate": 4.835369650662767e-07,
"logits/chosen": 0.2396807074546814,
"logits/rejected": 0.21028423309326172,
"logps/chosen": -64.00853729248047,
"logps/ref_chosen": -56.78379821777344,
"logps/ref_rejected": -69.89952087402344,
"logps/rejected": -80.44105529785156,
"loss": 1.1002,
"margin_dpo/margin_mean": 3.316795825958252,
"margin_dpo/margin_std": 5.056219100952148,
"step": 137
},
{
"KL/chosen_KL_mean": -8.548992156982422,
"KL/mean": -9.971826553344727,
"KL/rejected_KL_mean": -11.394657135009766,
"KL/std": 4.893838882446289,
"epoch": 0.20861678004535147,
"fcm_dpo/beta": 0.1305425614118576,
"fcm_dpo/delta": 0.035015083849430084,
"fcm_dpo/margin": 2.845663547515869,
"fcm_dpo/q_t": 0.41715729236602783,
"grad_norm": 34.6871223449707,
"learning_rate": 4.830618192112065e-07,
"logits/chosen": 0.2283031940460205,
"logits/rejected": 0.191786989569664,
"logps/chosen": -67.31500244140625,
"logps/ref_chosen": -58.766014099121094,
"logps/ref_rejected": -68.12371826171875,
"logps/rejected": -79.51837158203125,
"loss": 1.1657,
"margin_dpo/margin_mean": 2.845663070678711,
"margin_dpo/margin_std": 5.385343074798584,
"step": 138
},
{
"KL/chosen_KL_mean": -8.900955200195312,
"KL/mean": -10.671952247619629,
"KL/rejected_KL_mean": -12.442947387695312,
"KL/std": 4.920912742614746,
"epoch": 0.21012849584278157,
"fcm_dpo/beta": 0.12715420126914978,
"fcm_dpo/delta": -0.06484213471412659,
"fcm_dpo/margin": 3.5419931411743164,
"fcm_dpo/q_t": 0.39593103528022766,
"grad_norm": 36.7552490234375,
"learning_rate": 4.825801541160509e-07,
"logits/chosen": 0.19569626450538635,
"logits/rejected": 0.16585242748260498,
"logps/chosen": -80.12654113769531,
"logps/ref_chosen": -71.2255859375,
"logps/ref_rejected": -82.1834716796875,
"logps/rejected": -94.62641906738281,
"loss": 1.0784,
"margin_dpo/margin_mean": 3.5419931411743164,
"margin_dpo/margin_std": 4.931921005249023,
"step": 139
},
{
"KL/chosen_KL_mean": -8.247709274291992,
"KL/mean": -10.586565017700195,
"KL/rejected_KL_mean": -12.925422668457031,
"KL/std": 6.458744049072266,
"epoch": 0.21164021164021163,
"fcm_dpo/beta": 0.10884112864732742,
"fcm_dpo/delta": -0.15088868141174316,
"fcm_dpo/margin": 4.67771053314209,
"fcm_dpo/q_t": 0.3807474970817566,
"grad_norm": 32.42084884643555,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 0.19952046871185303,
"logits/rejected": 0.1522863507270813,
"logps/chosen": -71.52537536621094,
"logps/ref_chosen": -63.27766418457031,
"logps/ref_rejected": -83.30647277832031,
"logps/rejected": -96.23189544677734,
"loss": 1.0738,
"margin_dpo/margin_mean": 4.67771053314209,
"margin_dpo/margin_std": 7.042229652404785,
"step": 140
},
{
"KL/chosen_KL_mean": -9.049182891845703,
"KL/mean": -11.24361515045166,
"KL/rejected_KL_mean": -13.43804931640625,
"KL/std": 5.461617469787598,
"epoch": 0.21315192743764172,
"fcm_dpo/beta": 0.0988968014717102,
"fcm_dpo/delta": -0.05077613145112991,
"fcm_dpo/margin": 4.38886022567749,
"fcm_dpo/q_t": 0.4017443358898163,
"grad_norm": 28.450496673583984,
"learning_rate": 4.815973202802966e-07,
"logits/chosen": 0.21967005729675293,
"logits/rejected": 0.17520186305046082,
"logps/chosen": -70.81594848632812,
"logps/ref_chosen": -61.76676940917969,
"logps/ref_rejected": -88.60601806640625,
"logps/rejected": -102.0440673828125,
"loss": 1.1074,
"margin_dpo/margin_mean": 4.388860702514648,
"margin_dpo/margin_std": 6.8387346267700195,
"step": 141
},
{
"KL/chosen_KL_mean": -9.000873565673828,
"KL/mean": -10.799939155578613,
"KL/rejected_KL_mean": -12.599006652832031,
"KL/std": 5.402791976928711,
"epoch": 0.2146636432350718,
"fcm_dpo/beta": 0.10335898399353027,
"fcm_dpo/delta": 0.03455838933587074,
"fcm_dpo/margin": 3.598137855529785,
"fcm_dpo/q_t": 0.41603314876556396,
"grad_norm": 28.805465698242188,
"learning_rate": 4.810961790316729e-07,
"logits/chosen": 0.2335534393787384,
"logits/rejected": 0.20831874012947083,
"logps/chosen": -74.27565002441406,
"logps/ref_chosen": -65.2747802734375,
"logps/ref_rejected": -81.1378173828125,
"logps/rejected": -93.73682403564453,
"loss": 1.1255,
"margin_dpo/margin_mean": 3.598137378692627,
"margin_dpo/margin_std": 5.554536819458008,
"step": 142
},
{
"KL/chosen_KL_mean": -9.662801742553711,
"KL/mean": -11.23399543762207,
"KL/rejected_KL_mean": -12.805187225341797,
"KL/std": 6.142293930053711,
"epoch": 0.2161753590325019,
"fcm_dpo/beta": 0.11018843948841095,
"fcm_dpo/delta": 0.06442005187273026,
"fcm_dpo/margin": 3.1423885822296143,
"fcm_dpo/q_t": 0.42225682735443115,
"grad_norm": 38.34531784057617,
"learning_rate": 4.805885735261454e-07,
"logits/chosen": 0.26950305700302124,
"logits/rejected": 0.25179946422576904,
"logps/chosen": -72.28062438964844,
"logps/ref_chosen": -62.617828369140625,
"logps/ref_rejected": -70.39239501953125,
"logps/rejected": -83.19757843017578,
"loss": 1.2132,
"margin_dpo/margin_mean": 3.1423885822296143,
"margin_dpo/margin_std": 7.202130317687988,
"step": 143
},
{
"KL/chosen_KL_mean": -10.057125091552734,
"KL/mean": -12.049142837524414,
"KL/rejected_KL_mean": -14.041160583496094,
"KL/std": 6.3006911277771,
"epoch": 0.21768707482993196,
"fcm_dpo/beta": 0.10638520121574402,
"fcm_dpo/delta": -0.033301614224910736,
"fcm_dpo/margin": 3.9840335845947266,
"fcm_dpo/q_t": 0.40695470571517944,
"grad_norm": 31.39975357055664,
"learning_rate": 4.800745179625307e-07,
"logits/chosen": 0.24360644817352295,
"logits/rejected": 0.21164511144161224,
"logps/chosen": -70.85981750488281,
"logps/ref_chosen": -60.80268859863281,
"logps/ref_rejected": -79.07284545898438,
"logps/rejected": -93.11399841308594,
"loss": 1.1387,
"margin_dpo/margin_mean": 3.9840338230133057,
"margin_dpo/margin_std": 7.172069549560547,
"step": 144
},
{
"KL/chosen_KL_mean": -9.3580322265625,
"KL/mean": -11.521739959716797,
"KL/rejected_KL_mean": -13.685443878173828,
"KL/std": 6.398914813995361,
"epoch": 0.21919879062736206,
"fcm_dpo/beta": 0.1024373397231102,
"fcm_dpo/delta": -0.05667082592844963,
"fcm_dpo/margin": 4.327418804168701,
"fcm_dpo/q_t": 0.39943933486938477,
"grad_norm": 32.25492858886719,
"learning_rate": 4.795540267200686e-07,
"logits/chosen": 0.20689266920089722,
"logits/rejected": 0.22335760295391083,
"logps/chosen": -83.96949768066406,
"logps/ref_chosen": -74.61146545410156,
"logps/ref_rejected": -83.24461364746094,
"logps/rejected": -96.9300537109375,
"loss": 1.1228,
"margin_dpo/margin_mean": 4.327418327331543,
"margin_dpo/margin_std": 7.483129501342773,
"step": 145
},
{
"KL/chosen_KL_mean": -9.144096374511719,
"KL/mean": -11.36795425415039,
"KL/rejected_KL_mean": -13.591815948486328,
"KL/std": 6.335058212280273,
"epoch": 0.22071050642479215,
"fcm_dpo/beta": 0.09927242994308472,
"fcm_dpo/delta": -0.05333557724952698,
"fcm_dpo/margin": 4.447718143463135,
"fcm_dpo/q_t": 0.40064430236816406,
"grad_norm": 28.61855697631836,
"learning_rate": 4.790271143580173e-07,
"logits/chosen": 0.1993100643157959,
"logits/rejected": 0.1848219335079193,
"logps/chosen": -66.98507690429688,
"logps/ref_chosen": -57.84098434448242,
"logps/ref_rejected": -67.47422790527344,
"logps/rejected": -81.0660400390625,
"loss": 1.1045,
"margin_dpo/margin_mean": 4.44771671295166,
"margin_dpo/margin_std": 7.0095109939575195,
"step": 146
},
{
"KL/chosen_KL_mean": -10.900079727172852,
"KL/mean": -12.673417091369629,
"KL/rejected_KL_mean": -14.446762084960938,
"KL/std": 6.889484405517578,
"epoch": 0.2222222222222222,
"fcm_dpo/beta": 0.1005827784538269,
"fcm_dpo/delta": 0.05248674005270004,
"fcm_dpo/margin": 3.546682834625244,
"fcm_dpo/q_t": 0.42228808999061584,
"grad_norm": 35.238861083984375,
"learning_rate": 4.784937956152489e-07,
"logits/chosen": 0.22990134358406067,
"logits/rejected": 0.18562506139278412,
"logps/chosen": -77.71354675292969,
"logps/ref_chosen": -66.81346893310547,
"logps/ref_rejected": -81.1796875,
"logps/rejected": -95.62644958496094,
"loss": 1.1886,
"margin_dpo/margin_mean": 3.546682119369507,
"margin_dpo/margin_std": 7.444207191467285,
"step": 147
},
{
"KL/chosen_KL_mean": -10.144840240478516,
"KL/mean": -12.388810157775879,
"KL/rejected_KL_mean": -14.632774353027344,
"KL/std": 6.4042558670043945,
"epoch": 0.2237339380196523,
"fcm_dpo/beta": 0.09929439425468445,
"fcm_dpo/delta": -0.05904500186443329,
"fcm_dpo/margin": 4.487936496734619,
"fcm_dpo/q_t": 0.39827683568000793,
"grad_norm": 24.603193283081055,
"learning_rate": 4.779540854098347e-07,
"logits/chosen": 0.36354687809944153,
"logits/rejected": 0.28833717107772827,
"logps/chosen": -58.83259582519531,
"logps/ref_chosen": -48.6877555847168,
"logps/ref_rejected": -67.50503540039062,
"logps/rejected": -82.13780975341797,
"loss": 1.1109,
"margin_dpo/margin_mean": 4.487936973571777,
"margin_dpo/margin_std": 7.146442413330078,
"step": 148
},
{
"KL/chosen_KL_mean": -12.343606948852539,
"KL/mean": -14.90990161895752,
"KL/rejected_KL_mean": -17.4761962890625,
"KL/std": 6.945236682891846,
"epoch": 0.2252456538170824,
"fcm_dpo/beta": 0.09050105512142181,
"fcm_dpo/delta": -0.08389414846897125,
"fcm_dpo/margin": 5.13259744644165,
"fcm_dpo/q_t": 0.3906511962413788,
"grad_norm": 24.449920654296875,
"learning_rate": 4.774079988386296e-07,
"logits/chosen": 0.20603233575820923,
"logits/rejected": 0.15894901752471924,
"logps/chosen": -67.48738098144531,
"logps/ref_chosen": -55.143775939941406,
"logps/ref_rejected": -64.79888916015625,
"logps/rejected": -82.27508544921875,
"loss": 1.0558,
"margin_dpo/margin_mean": 5.13259744644165,
"margin_dpo/margin_std": 6.536444664001465,
"step": 149
},
{
"KL/chosen_KL_mean": -10.346208572387695,
"KL/mean": -13.742513656616211,
"KL/rejected_KL_mean": -17.138816833496094,
"KL/std": 7.590294361114502,
"epoch": 0.22675736961451248,
"fcm_dpo/beta": 0.07703472673892975,
"fcm_dpo/delta": -0.17041468620300293,
"fcm_dpo/margin": 6.792609214782715,
"fcm_dpo/q_t": 0.3750220239162445,
"grad_norm": 20.74353790283203,
"learning_rate": 4.768555511768486e-07,
"logits/chosen": 0.220647931098938,
"logits/rejected": 0.17675264179706573,
"logps/chosen": -77.81695556640625,
"logps/ref_chosen": -67.47074890136719,
"logps/ref_rejected": -89.21170806884766,
"logps/rejected": -106.35052490234375,
"loss": 1.0144,
"margin_dpo/margin_mean": 6.792610168457031,
"margin_dpo/margin_std": 7.658425331115723,
"step": 150
},
{
"KL/chosen_KL_mean": -9.255523681640625,
"KL/mean": -13.078845977783203,
"KL/rejected_KL_mean": -16.90216827392578,
"KL/std": 8.135717391967773,
"epoch": 0.22826908541194255,
"fcm_dpo/beta": 0.066739022731781,
"fcm_dpo/delta": -0.14863666892051697,
"fcm_dpo/margin": 7.646646022796631,
"fcm_dpo/q_t": 0.3767693042755127,
"grad_norm": 18.29514503479004,
"learning_rate": 4.762967578776406e-07,
"logits/chosen": 0.23800373077392578,
"logits/rejected": 0.18349212408065796,
"logps/chosen": -61.715065002441406,
"logps/ref_chosen": -52.45954132080078,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -95.96517944335938,
"loss": 1.0081,
"margin_dpo/margin_mean": 7.646645545959473,
"margin_dpo/margin_std": 8.172331809997559,
"step": 151
},
{
"KL/chosen_KL_mean": -11.936054229736328,
"KL/mean": -14.415749549865723,
"KL/rejected_KL_mean": -16.89544677734375,
"KL/std": 7.793455123901367,
"epoch": 0.22978080120937264,
"fcm_dpo/beta": 0.06914802640676498,
"fcm_dpo/delta": 0.0631265938282013,
"fcm_dpo/margin": 4.959397792816162,
"fcm_dpo/q_t": 0.4234861433506012,
"grad_norm": 20.811508178710938,
"learning_rate": 4.757316345716553e-07,
"logits/chosen": 0.30365419387817383,
"logits/rejected": 0.2504726052284241,
"logps/chosen": -68.48988342285156,
"logps/ref_chosen": -56.5538330078125,
"logps/ref_rejected": -76.55074310302734,
"logps/rejected": -93.4461898803711,
"loss": 1.1642,
"margin_dpo/margin_mean": 4.959397315979004,
"margin_dpo/margin_std": 8.868165969848633,
"step": 152
},
{
"KL/chosen_KL_mean": -11.271255493164062,
"KL/mean": -14.22636604309082,
"KL/rejected_KL_mean": -17.181472778320312,
"KL/std": 8.153881072998047,
"epoch": 0.23129251700680273,
"fcm_dpo/beta": 0.06809602677822113,
"fcm_dpo/delta": -0.0030866768211126328,
"fcm_dpo/margin": 5.910217761993408,
"fcm_dpo/q_t": 0.4064687490463257,
"grad_norm": 19.777393341064453,
"learning_rate": 4.751601970666064e-07,
"logits/chosen": 0.23187056183815002,
"logits/rejected": 0.19487443566322327,
"logps/chosen": -79.27815246582031,
"logps/ref_chosen": -68.00689697265625,
"logps/ref_rejected": -74.83482360839844,
"logps/rejected": -92.01629638671875,
"loss": 1.0864,
"margin_dpo/margin_mean": 5.910217761993408,
"margin_dpo/margin_std": 7.565615653991699,
"step": 153
},
{
"KL/chosen_KL_mean": -12.515201568603516,
"KL/mean": -14.71187973022461,
"KL/rejected_KL_mean": -16.90856170654297,
"KL/std": 8.529696464538574,
"epoch": 0.2328042328042328,
"fcm_dpo/beta": 0.07292280346155167,
"fcm_dpo/delta": 0.0948781967163086,
"fcm_dpo/margin": 4.3933634757995605,
"fcm_dpo/q_t": 0.4292556047439575,
"grad_norm": 21.461366653442383,
"learning_rate": 4.745824613468292e-07,
"logits/chosen": 0.3159400224685669,
"logits/rejected": 0.31303977966308594,
"logps/chosen": -71.73773956298828,
"logps/ref_chosen": -59.222537994384766,
"logps/ref_rejected": -64.19131469726562,
"logps/rejected": -81.09988403320312,
"loss": 1.1898,
"margin_dpo/margin_mean": 4.393362998962402,
"margin_dpo/margin_std": 8.842275619506836,
"step": 154
},
{
"KL/chosen_KL_mean": -13.313087463378906,
"KL/mean": -16.432861328125,
"KL/rejected_KL_mean": -19.552631378173828,
"KL/std": 8.535072326660156,
"epoch": 0.23431594860166288,
"fcm_dpo/beta": 0.0700855702161789,
"fcm_dpo/delta": -0.05278134346008301,
"fcm_dpo/margin": 6.239541053771973,
"fcm_dpo/q_t": 0.40147268772125244,
"grad_norm": 20.99993133544922,
"learning_rate": 4.7399844357283393e-07,
"logits/chosen": 0.2972029447555542,
"logits/rejected": 0.2771180272102356,
"logps/chosen": -81.76778411865234,
"logps/ref_chosen": -68.45469665527344,
"logps/ref_rejected": -77.91763305664062,
"logps/rejected": -97.47026824951172,
"loss": 1.1297,
"margin_dpo/margin_mean": 6.239541053771973,
"margin_dpo/margin_std": 10.832403182983398,
"step": 155
},
{
"KL/chosen_KL_mean": -13.374895095825195,
"KL/mean": -17.081748962402344,
"KL/rejected_KL_mean": -20.788604736328125,
"KL/std": 9.638997077941895,
"epoch": 0.23582766439909297,
"fcm_dpo/beta": 0.06551361083984375,
"fcm_dpo/delta": -0.11318676173686981,
"fcm_dpo/margin": 7.4137067794799805,
"fcm_dpo/q_t": 0.3857467770576477,
"grad_norm": 20.287338256835938,
"learning_rate": 4.7340816008085305e-07,
"logits/chosen": 0.2682151794433594,
"logits/rejected": 0.22211289405822754,
"logps/chosen": -80.64449310302734,
"logps/ref_chosen": -67.26959991455078,
"logps/ref_rejected": -86.95914459228516,
"logps/rejected": -107.74774932861328,
"loss": 1.0398,
"margin_dpo/margin_mean": 7.4137067794799805,
"margin_dpo/margin_std": 8.992185592651367,
"step": 156
},
{
"KL/chosen_KL_mean": -12.044370651245117,
"KL/mean": -15.103776931762695,
"KL/rejected_KL_mean": -18.16318130493164,
"KL/std": 9.422752380371094,
"epoch": 0.23733938019652306,
"fcm_dpo/beta": 0.0583278089761734,
"fcm_dpo/delta": -0.08355621248483658,
"fcm_dpo/margin": 6.118807792663574,
"fcm_dpo/q_t": 0.41334038972854614,
"grad_norm": 17.29412078857422,
"learning_rate": 4.728116273823847e-07,
"logits/chosen": 0.2556699812412262,
"logits/rejected": 0.2370312660932541,
"logps/chosen": -66.81724548339844,
"logps/ref_chosen": -54.77287292480469,
"logps/ref_rejected": -63.87866973876953,
"logps/rejected": -82.04185485839844,
"loss": 1.1341,
"margin_dpo/margin_mean": 6.118807792663574,
"margin_dpo/margin_std": 9.063301086425781,
"step": 157
},
{
"KL/chosen_KL_mean": -13.51652717590332,
"KL/mean": -16.806453704833984,
"KL/rejected_KL_mean": -20.096378326416016,
"KL/std": 9.68644905090332,
"epoch": 0.23885109599395313,
"fcm_dpo/beta": 0.05906832590699196,
"fcm_dpo/delta": 0.014011925086379051,
"fcm_dpo/margin": 6.579849720001221,
"fcm_dpo/q_t": 0.4122830927371979,
"grad_norm": 18.49722671508789,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 0.32583776116371155,
"logits/rejected": 0.28690358996391296,
"logps/chosen": -78.43924713134766,
"logps/ref_chosen": -64.92271423339844,
"logps/ref_rejected": -82.23789978027344,
"logps/rejected": -102.33427429199219,
"loss": 1.1197,
"margin_dpo/margin_mean": 6.579849720001221,
"margin_dpo/margin_std": 10.205474853515625,
"step": 158
},
{
"KL/chosen_KL_mean": -13.864181518554688,
"KL/mean": -17.635059356689453,
"KL/rejected_KL_mean": -21.40593719482422,
"KL/std": 10.578052520751953,
"epoch": 0.24036281179138322,
"fcm_dpo/beta": 0.058339424431324005,
"fcm_dpo/delta": -0.05248873680830002,
"fcm_dpo/margin": 7.541755199432373,
"fcm_dpo/q_t": 0.39800336956977844,
"grad_norm": 19.51651954650879,
"learning_rate": 4.715998812855304e-07,
"logits/chosen": 0.3037192225456238,
"logits/rejected": 0.26449885964393616,
"logps/chosen": -70.91117858886719,
"logps/ref_chosen": -57.046993255615234,
"logps/ref_rejected": -73.32441711425781,
"logps/rejected": -94.73035430908203,
"loss": 1.097,
"margin_dpo/margin_mean": 7.541754722595215,
"margin_dpo/margin_std": 11.181962966918945,
"step": 159
},
{
"KL/chosen_KL_mean": -15.68055534362793,
"KL/mean": -18.908214569091797,
"KL/rejected_KL_mean": -22.135875701904297,
"KL/std": 9.930036544799805,
"epoch": 0.2418745275888133,
"fcm_dpo/beta": 0.05775478482246399,
"fcm_dpo/delta": 0.03315534442663193,
"fcm_dpo/margin": 6.455317497253418,
"fcm_dpo/q_t": 0.4187636971473694,
"grad_norm": 17.91891098022461,
"learning_rate": 4.7098470178228755e-07,
"logits/chosen": 0.20492243766784668,
"logits/rejected": 0.1574675440788269,
"logps/chosen": -65.48747253417969,
"logps/ref_chosen": -49.806915283203125,
"logps/ref_rejected": -68.3370132446289,
"logps/rejected": -90.47288513183594,
"loss": 1.1475,
"margin_dpo/margin_mean": 6.455317497253418,
"margin_dpo/margin_std": 11.438055038452148,
"step": 160
},
{
"KL/chosen_KL_mean": -15.670604705810547,
"KL/mean": -19.274505615234375,
"KL/rejected_KL_mean": -22.878414154052734,
"KL/std": 9.855175971984863,
"epoch": 0.24338624338624337,
"fcm_dpo/beta": 0.05751575529575348,
"fcm_dpo/delta": -0.01837945356965065,
"fcm_dpo/margin": 7.207810878753662,
"fcm_dpo/q_t": 0.40520739555358887,
"grad_norm": 17.27115249633789,
"learning_rate": 4.703633408618955e-07,
"logits/chosen": 0.34232738614082336,
"logits/rejected": 0.302024781703949,
"logps/chosen": -68.17109680175781,
"logps/ref_chosen": -52.50048828125,
"logps/ref_rejected": -66.04540252685547,
"logps/rejected": -88.92381286621094,
"loss": 1.1097,
"margin_dpo/margin_mean": 7.20781135559082,
"margin_dpo/margin_std": 10.996399879455566,
"step": 161
},
{
"KL/chosen_KL_mean": -16.62833023071289,
"KL/mean": -21.415283203125,
"KL/rejected_KL_mean": -26.202239990234375,
"KL/std": 11.221863746643066,
"epoch": 0.24489795918367346,
"fcm_dpo/beta": 0.05156968533992767,
"fcm_dpo/delta": -0.1260819137096405,
"fcm_dpo/margin": 9.573906898498535,
"fcm_dpo/q_t": 0.38108617067337036,
"grad_norm": 17.9483585357666,
"learning_rate": 4.697358159051549e-07,
"logits/chosen": 0.37953609228134155,
"logits/rejected": 0.3276635408401489,
"logps/chosen": -86.09751892089844,
"logps/ref_chosen": -69.46919250488281,
"logps/ref_rejected": -92.00952911376953,
"logps/rejected": -118.2117691040039,
"loss": 1.0347,
"margin_dpo/margin_mean": 9.573907852172852,
"margin_dpo/margin_std": 11.599828720092773,
"step": 162
},
{
"KL/chosen_KL_mean": -14.487728118896484,
"KL/mean": -19.402671813964844,
"KL/rejected_KL_mean": -24.317615509033203,
"KL/std": 11.253974914550781,
"epoch": 0.24640967498110355,
"fcm_dpo/beta": 0.048833370208740234,
"fcm_dpo/delta": -0.11060789227485657,
"fcm_dpo/margin": 9.829888343811035,
"fcm_dpo/q_t": 0.38493412733078003,
"grad_norm": 16.407407760620117,
"learning_rate": 4.691021444652876e-07,
"logits/chosen": 0.29747360944747925,
"logits/rejected": 0.24846696853637695,
"logps/chosen": -65.1015625,
"logps/ref_chosen": -50.613834381103516,
"logps/ref_rejected": -74.62033081054688,
"logps/rejected": -98.93794250488281,
"loss": 1.0578,
"margin_dpo/margin_mean": 9.829889297485352,
"margin_dpo/margin_std": 11.631856918334961,
"step": 163
},
{
"KL/chosen_KL_mean": -15.621034622192383,
"KL/mean": -20.437870025634766,
"KL/rejected_KL_mean": -25.25470733642578,
"KL/std": 12.038859367370605,
"epoch": 0.24792139077853365,
"fcm_dpo/beta": 0.044747721403837204,
"fcm_dpo/delta": -0.04132752865552902,
"fcm_dpo/margin": 9.633673667907715,
"fcm_dpo/q_t": 0.39967209100723267,
"grad_norm": 14.316239356994629,
"learning_rate": 4.6846234426744624e-07,
"logits/chosen": 0.28174513578414917,
"logits/rejected": 0.21716471016407013,
"logps/chosen": -70.46914672851562,
"logps/ref_chosen": -54.848114013671875,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -104.31771850585938,
"loss": 1.0909,
"margin_dpo/margin_mean": 9.633672714233398,
"margin_dpo/margin_std": 13.165080070495605,
"step": 164
},
{
"KL/chosen_KL_mean": -17.53729248046875,
"KL/mean": -22.0023193359375,
"KL/rejected_KL_mean": -26.467342376708984,
"KL/std": 12.443459510803223,
"epoch": 0.2494331065759637,
"fcm_dpo/beta": 0.04426497220993042,
"fcm_dpo/delta": 0.0033883340656757355,
"fcm_dpo/margin": 8.9300537109375,
"fcm_dpo/q_t": 0.4085206389427185,
"grad_norm": 14.935009002685547,
"learning_rate": 4.678164332082175e-07,
"logits/chosen": 0.35460126399993896,
"logits/rejected": 0.29810017347335815,
"logps/chosen": -68.62650299072266,
"logps/ref_chosen": -51.089210510253906,
"logps/ref_rejected": -71.23370361328125,
"logps/rejected": -97.70104217529297,
"loss": 1.1201,
"margin_dpo/margin_mean": 8.930052757263184,
"margin_dpo/margin_std": 13.425127029418945,
"step": 165
},
{
"KL/chosen_KL_mean": -16.783632278442383,
"KL/mean": -20.313011169433594,
"KL/rejected_KL_mean": -23.842388153076172,
"KL/std": 12.600139617919922,
"epoch": 0.2509448223733938,
"fcm_dpo/beta": 0.046010538935661316,
"fcm_dpo/delta": 0.09013964980840683,
"fcm_dpo/margin": 7.058765411376953,
"fcm_dpo/q_t": 0.4290624260902405,
"grad_norm": 18.119962692260742,
"learning_rate": 4.6716442935512214e-07,
"logits/chosen": 0.3217995762825012,
"logits/rejected": 0.2264447808265686,
"logps/chosen": -79.97444152832031,
"logps/ref_chosen": -63.19081115722656,
"logps/ref_rejected": -93.8402099609375,
"logps/rejected": -117.68260192871094,
"loss": 1.1621,
"margin_dpo/margin_mean": 7.058765411376953,
"margin_dpo/margin_std": 12.138998985290527,
"step": 166
},
{
"KL/chosen_KL_mean": -13.947896957397461,
"KL/mean": -19.46517562866211,
"KL/rejected_KL_mean": -24.98244857788086,
"KL/std": 12.885375022888184,
"epoch": 0.25245653817082386,
"fcm_dpo/beta": 0.04268675297498703,
"fcm_dpo/delta": -0.10074665397405624,
"fcm_dpo/margin": 11.034552574157715,
"fcm_dpo/q_t": 0.38690847158432007,
"grad_norm": 15.102579116821289,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": 0.25539228320121765,
"logits/rejected": 0.22091025114059448,
"logps/chosen": -72.8721694946289,
"logps/ref_chosen": -58.92427062988281,
"logps/ref_rejected": -72.97377014160156,
"logps/rejected": -97.95622253417969,
"loss": 1.039,
"margin_dpo/margin_mean": 11.034552574157715,
"margin_dpo/margin_std": 12.66136360168457,
"step": 167
},
{
"KL/chosen_KL_mean": -16.616640090942383,
"KL/mean": -21.07298469543457,
"KL/rejected_KL_mean": -25.529327392578125,
"KL/std": 14.074921607971191,
"epoch": 0.25396825396825395,
"fcm_dpo/beta": 0.04318401217460632,
"fcm_dpo/delta": 0.01869083382189274,
"fcm_dpo/margin": 8.912687301635742,
"fcm_dpo/q_t": 0.41359812021255493,
"grad_norm": 16.315370559692383,
"learning_rate": 4.6584221638904767e-07,
"logits/chosen": 0.28571587800979614,
"logits/rejected": 0.24888527393341064,
"logps/chosen": -82.26802062988281,
"logps/ref_chosen": -65.65138244628906,
"logps/ref_rejected": -79.71418762207031,
"logps/rejected": -105.24351501464844,
"loss": 1.1132,
"margin_dpo/margin_mean": 8.912687301635742,
"margin_dpo/margin_std": 13.038930892944336,
"step": 168
},
{
"KL/chosen_KL_mean": -13.18602180480957,
"KL/mean": -18.36703109741211,
"KL/rejected_KL_mean": -23.54804229736328,
"KL/std": 13.908943176269531,
"epoch": 0.25547996976568405,
"fcm_dpo/beta": 0.04207727313041687,
"fcm_dpo/delta": -0.04623054713010788,
"fcm_dpo/margin": 10.362018585205078,
"fcm_dpo/q_t": 0.4016047418117523,
"grad_norm": 16.790719985961914,
"learning_rate": 4.651720442612075e-07,
"logits/chosen": 0.36561042070388794,
"logits/rejected": 0.33055049180984497,
"logps/chosen": -74.61189270019531,
"logps/ref_chosen": -61.425865173339844,
"logps/ref_rejected": -76.09590148925781,
"logps/rejected": -99.64393615722656,
"loss": 1.1074,
"margin_dpo/margin_mean": 10.362018585205078,
"margin_dpo/margin_std": 16.55835723876953,
"step": 169
},
{
"KL/chosen_KL_mean": -12.52975082397461,
"KL/mean": -17.670759201049805,
"KL/rejected_KL_mean": -22.811765670776367,
"KL/std": 13.753107070922852,
"epoch": 0.25699168556311414,
"fcm_dpo/beta": 0.041177112609148026,
"fcm_dpo/delta": -0.02979334443807602,
"fcm_dpo/margin": 10.28201675415039,
"fcm_dpo/q_t": 0.4035836458206177,
"grad_norm": 14.086734771728516,
"learning_rate": 4.6449585330874425e-07,
"logits/chosen": 0.3422015607357025,
"logits/rejected": 0.3399706482887268,
"logps/chosen": -69.18293762207031,
"logps/ref_chosen": -56.65319061279297,
"logps/ref_rejected": -63.45965576171875,
"logps/rejected": -86.27142333984375,
"loss": 1.1271,
"margin_dpo/margin_mean": 10.28201675415039,
"margin_dpo/margin_std": 17.14507293701172,
"step": 170
},
{
"KL/chosen_KL_mean": -14.82802963256836,
"KL/mean": -20.42302703857422,
"KL/rejected_KL_mean": -26.018028259277344,
"KL/std": 15.890604019165039,
"epoch": 0.2585034013605442,
"fcm_dpo/beta": 0.03773067146539688,
"fcm_dpo/delta": -0.037932828068733215,
"fcm_dpo/margin": 11.190006256103516,
"fcm_dpo/q_t": 0.4020886719226837,
"grad_norm": 13.957395553588867,
"learning_rate": 4.6381366244617224e-07,
"logits/chosen": 0.4011567234992981,
"logits/rejected": 0.3479583263397217,
"logps/chosen": -78.56278991699219,
"logps/ref_chosen": -63.73476028442383,
"logps/ref_rejected": -78.50328063964844,
"logps/rejected": -104.52130126953125,
"loss": 1.1247,
"margin_dpo/margin_mean": 11.1900053024292,
"margin_dpo/margin_std": 18.119037628173828,
"step": 171
},
{
"KL/chosen_KL_mean": -16.471878051757812,
"KL/mean": -21.892181396484375,
"KL/rejected_KL_mean": -27.312484741210938,
"KL/std": 16.332107543945312,
"epoch": 0.2600151171579743,
"fcm_dpo/beta": 0.03937235847115517,
"fcm_dpo/delta": -0.03921569883823395,
"fcm_dpo/margin": 10.840606689453125,
"fcm_dpo/q_t": 0.4032423496246338,
"grad_norm": 15.800121307373047,
"learning_rate": 4.631254907558365e-07,
"logits/chosen": 0.41245076060295105,
"logits/rejected": 0.3497459888458252,
"logps/chosen": -68.67363739013672,
"logps/ref_chosen": -52.201759338378906,
"logps/ref_rejected": -82.85285949707031,
"logps/rejected": -110.16534423828125,
"loss": 1.1128,
"margin_dpo/margin_mean": 10.840605735778809,
"margin_dpo/margin_std": 16.780426025390625,
"step": 172
},
{
"KL/chosen_KL_mean": -15.000802993774414,
"KL/mean": -20.906818389892578,
"KL/rejected_KL_mean": -26.81283187866211,
"KL/std": 16.735857009887695,
"epoch": 0.2615268329554044,
"fcm_dpo/beta": 0.03506167232990265,
"fcm_dpo/delta": -0.028556950390338898,
"fcm_dpo/margin": 11.812032699584961,
"fcm_dpo/q_t": 0.40625429153442383,
"grad_norm": 14.270763397216797,
"learning_rate": 4.624313574873786e-07,
"logits/chosen": 0.392696350812912,
"logits/rejected": 0.3005034327507019,
"logps/chosen": -70.43553161621094,
"logps/ref_chosen": -55.434722900390625,
"logps/ref_rejected": -77.81967163085938,
"logps/rejected": -104.63250732421875,
"loss": 1.1447,
"margin_dpo/margin_mean": 11.812033653259277,
"margin_dpo/margin_std": 20.58139419555664,
"step": 173
},
{
"KL/chosen_KL_mean": -17.099306106567383,
"KL/mean": -23.63693618774414,
"KL/rejected_KL_mean": -30.17456817626953,
"KL/std": 17.15781593322754,
"epoch": 0.26303854875283444,
"fcm_dpo/beta": 0.034231267869472504,
"fcm_dpo/delta": -0.0633653849363327,
"fcm_dpo/margin": 13.075263023376465,
"fcm_dpo/q_t": 0.3955170512199402,
"grad_norm": 14.480879783630371,
"learning_rate": 4.61731282057198e-07,
"logits/chosen": 0.402925968170166,
"logits/rejected": 0.33117085695266724,
"logps/chosen": -74.27125549316406,
"logps/ref_chosen": -57.17195129394531,
"logps/ref_rejected": -85.47578430175781,
"logps/rejected": -115.65034484863281,
"loss": 1.0822,
"margin_dpo/margin_mean": 13.075262069702148,
"margin_dpo/margin_std": 18.593975067138672,
"step": 174
},
{
"KL/chosen_KL_mean": -14.67591667175293,
"KL/mean": -22.01688003540039,
"KL/rejected_KL_mean": -29.35784912109375,
"KL/std": 17.625797271728516,
"epoch": 0.26455026455026454,
"fcm_dpo/beta": 0.032175906002521515,
"fcm_dpo/delta": -0.09433892369270325,
"fcm_dpo/margin": 14.681928634643555,
"fcm_dpo/q_t": 0.39160484075546265,
"grad_norm": 13.93406867980957,
"learning_rate": 4.6102528404790965e-07,
"logits/chosen": 0.38863056898117065,
"logits/rejected": 0.3585554361343384,
"logps/chosen": -82.34154510498047,
"logps/ref_chosen": -67.6656265258789,
"logps/ref_rejected": -84.36766815185547,
"logps/rejected": -113.72551727294922,
"loss": 1.071,
"margin_dpo/margin_mean": 14.681928634643555,
"margin_dpo/margin_std": 20.618019104003906,
"step": 175
},
{
"KL/chosen_KL_mean": -19.09964370727539,
"KL/mean": -23.8125,
"KL/rejected_KL_mean": -28.52536392211914,
"KL/std": 18.95433807373047,
"epoch": 0.2660619803476946,
"fcm_dpo/beta": 0.031624685972929,
"fcm_dpo/delta": 0.017538614571094513,
"fcm_dpo/margin": 9.425718307495117,
"fcm_dpo/q_t": 0.4338553547859192,
"grad_norm": 16.63987159729004,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 0.34582072496414185,
"logits/rejected": 0.318925678730011,
"logps/chosen": -96.95840454101562,
"logps/ref_chosen": -77.8587646484375,
"logps/ref_rejected": -81.08732604980469,
"logps/rejected": -109.61268615722656,
"loss": 1.2205,
"margin_dpo/margin_mean": 9.425718307495117,
"margin_dpo/margin_std": 21.521808624267578,
"step": 176
},
{
"KL/chosen_KL_mean": -14.09356689453125,
"KL/mean": -23.430145263671875,
"KL/rejected_KL_mean": -32.766727447509766,
"KL/std": 19.354829788208008,
"epoch": 0.2675736961451247,
"fcm_dpo/beta": 0.02757161483168602,
"fcm_dpo/delta": -0.16072139143943787,
"fcm_dpo/margin": 18.673160552978516,
"fcm_dpo/q_t": 0.37375608086586,
"grad_norm": 15.990117073059082,
"learning_rate": 4.5959559945025183e-07,
"logits/chosen": 0.48053431510925293,
"logits/rejected": 0.3819437026977539,
"logps/chosen": -69.31396484375,
"logps/ref_chosen": -55.22039794921875,
"logps/ref_rejected": -92.54973602294922,
"logps/rejected": -125.31646728515625,
"loss": 1.0054,
"margin_dpo/margin_mean": 18.673158645629883,
"margin_dpo/margin_std": 19.863622665405273,
"step": 177
},
{
"KL/chosen_KL_mean": -17.601396560668945,
"KL/mean": -23.378013610839844,
"KL/rejected_KL_mean": -29.15462875366211,
"KL/std": 18.475444793701172,
"epoch": 0.2690854119425548,
"fcm_dpo/beta": 0.02648773603141308,
"fcm_dpo/delta": -0.016534805297851562,
"fcm_dpo/margin": 11.553230285644531,
"fcm_dpo/q_t": 0.42779386043548584,
"grad_norm": 15.744258880615234,
"learning_rate": 4.588719528532341e-07,
"logits/chosen": 0.3623722791671753,
"logits/rejected": 0.3107893764972687,
"logps/chosen": -78.4118881225586,
"logps/ref_chosen": -60.81049346923828,
"logps/ref_rejected": -81.12973022460938,
"logps/rejected": -110.28435516357422,
"loss": 1.1656,
"margin_dpo/margin_mean": 11.553229331970215,
"margin_dpo/margin_std": 18.718524932861328,
"step": 178
},
{
"KL/chosen_KL_mean": -18.142658233642578,
"KL/mean": -25.477218627929688,
"KL/rejected_KL_mean": -32.81178283691406,
"KL/std": 19.89893913269043,
"epoch": 0.2705971277399849,
"fcm_dpo/beta": 0.02664315700531006,
"fcm_dpo/delta": 0.011396395973861217,
"fcm_dpo/margin": 14.669126510620117,
"fcm_dpo/q_t": 0.41117656230926514,
"grad_norm": 13.520777702331543,
"learning_rate": 4.581424636586928e-07,
"logits/chosen": 0.4051831066608429,
"logits/rejected": 0.38962996006011963,
"logps/chosen": -83.81437683105469,
"logps/ref_chosen": -65.67171478271484,
"logps/ref_rejected": -75.32586669921875,
"logps/rejected": -108.13764953613281,
"loss": 1.1254,
"margin_dpo/margin_mean": 14.669126510620117,
"margin_dpo/margin_std": 23.581867218017578,
"step": 179
},
{
"KL/chosen_KL_mean": -13.859926223754883,
"KL/mean": -19.828092575073242,
"KL/rejected_KL_mean": -25.796255111694336,
"KL/std": 19.319965362548828,
"epoch": 0.272108843537415,
"fcm_dpo/beta": 0.02849998138844967,
"fcm_dpo/delta": 0.07087843120098114,
"fcm_dpo/margin": 11.936328887939453,
"fcm_dpo/q_t": 0.42550933361053467,
"grad_norm": 12.52182388305664,
"learning_rate": 4.5740715227200897e-07,
"logits/chosen": 0.27053821086883545,
"logits/rejected": 0.2509544789791107,
"logps/chosen": -70.54273223876953,
"logps/ref_chosen": -56.68280792236328,
"logps/ref_rejected": -64.94414520263672,
"logps/rejected": -90.74040222167969,
"loss": 1.1686,
"margin_dpo/margin_mean": 11.936328887939453,
"margin_dpo/margin_std": 22.309284210205078,
"step": 180
},
{
"KL/chosen_KL_mean": -11.871850967407227,
"KL/mean": -20.5865478515625,
"KL/rejected_KL_mean": -29.301239013671875,
"KL/std": 20.67347526550293,
"epoch": 0.273620559334845,
"fcm_dpo/beta": 0.026625443249940872,
"fcm_dpo/delta": -0.08586108684539795,
"fcm_dpo/margin": 17.429393768310547,
"fcm_dpo/q_t": 0.3906833529472351,
"grad_norm": 15.263794898986816,
"learning_rate": 4.566660392614228e-07,
"logits/chosen": 0.4320031404495239,
"logits/rejected": 0.38959449529647827,
"logps/chosen": -72.64790344238281,
"logps/ref_chosen": -60.77604675292969,
"logps/ref_rejected": -83.98361206054688,
"logps/rejected": -113.28485107421875,
"loss": 1.0434,
"margin_dpo/margin_mean": 17.429393768310547,
"margin_dpo/margin_std": 20.527790069580078,
"step": 181
},
{
"KL/chosen_KL_mean": -18.988346099853516,
"KL/mean": -27.982421875,
"KL/rejected_KL_mean": -36.97650146484375,
"KL/std": 22.851451873779297,
"epoch": 0.2751322751322751,
"fcm_dpo/beta": 0.025137335062026978,
"fcm_dpo/delta": -0.06744768470525742,
"fcm_dpo/margin": 17.988162994384766,
"fcm_dpo/q_t": 0.39542728662490845,
"grad_norm": 16.23828125,
"learning_rate": 4.5591914535745817e-07,
"logits/chosen": 0.39073818922042847,
"logits/rejected": 0.3045911192893982,
"logps/chosen": -79.24212646484375,
"logps/ref_chosen": -60.2537841796875,
"logps/ref_rejected": -89.7706298828125,
"logps/rejected": -126.74713134765625,
"loss": 1.0819,
"margin_dpo/margin_mean": 17.988162994384766,
"margin_dpo/margin_std": 25.733863830566406,
"step": 182
},
{
"KL/chosen_KL_mean": -23.74131202697754,
"KL/mean": -27.4285888671875,
"KL/rejected_KL_mean": -31.115867614746094,
"KL/std": 22.018535614013672,
"epoch": 0.2766439909297052,
"fcm_dpo/beta": 0.026868050917983055,
"fcm_dpo/delta": 0.0852898508310318,
"fcm_dpo/margin": 7.374558448791504,
"fcm_dpo/q_t": 0.45595434308052063,
"grad_norm": 14.200604438781738,
"learning_rate": 4.551664914523433e-07,
"logits/chosen": 0.3912127912044525,
"logits/rejected": 0.368286669254303,
"logps/chosen": -85.50273132324219,
"logps/ref_chosen": -61.76142120361328,
"logps/ref_rejected": -72.54627990722656,
"logps/rejected": -103.66215515136719,
"loss": 1.2869,
"margin_dpo/margin_mean": 7.374558925628662,
"margin_dpo/margin_std": 22.744346618652344,
"step": 183
},
{
"KL/chosen_KL_mean": -13.689094543457031,
"KL/mean": -20.588485717773438,
"KL/rejected_KL_mean": -27.48788070678711,
"KL/std": 19.24828338623047,
"epoch": 0.2781557067271353,
"fcm_dpo/beta": 0.026989229023456573,
"fcm_dpo/delta": 0.032808832824230194,
"fcm_dpo/margin": 13.798786163330078,
"fcm_dpo/q_t": 0.4167749881744385,
"grad_norm": 16.775178909301758,
"learning_rate": 4.544080985994258e-07,
"logits/chosen": 0.5024610757827759,
"logits/rejected": 0.43457919359207153,
"logps/chosen": -60.529815673828125,
"logps/ref_chosen": -46.840721130371094,
"logps/ref_rejected": -69.3609390258789,
"logps/rejected": -96.84881591796875,
"loss": 1.1215,
"margin_dpo/margin_mean": 13.798786163330078,
"margin_dpo/margin_std": 20.367998123168945,
"step": 184
},
{
"KL/chosen_KL_mean": -18.885101318359375,
"KL/mean": -27.216163635253906,
"KL/rejected_KL_mean": -35.547218322753906,
"KL/std": 23.988807678222656,
"epoch": 0.2796674225245654,
"fcm_dpo/beta": 0.025701653212308884,
"fcm_dpo/delta": -0.04676172509789467,
"fcm_dpo/margin": 16.66212272644043,
"fcm_dpo/q_t": 0.4024374485015869,
"grad_norm": 14.15645980834961,
"learning_rate": 4.5364398801258394e-07,
"logits/chosen": 0.4315561056137085,
"logits/rejected": 0.38441091775894165,
"logps/chosen": -71.20623779296875,
"logps/ref_chosen": -52.32114028930664,
"logps/ref_rejected": -68.3885726928711,
"logps/rejected": -103.935791015625,
"loss": 1.1383,
"margin_dpo/margin_mean": 16.66212272644043,
"margin_dpo/margin_std": 28.837909698486328,
"step": 185
},
{
"KL/chosen_KL_mean": -16.665929794311523,
"KL/mean": -25.744937896728516,
"KL/rejected_KL_mean": -34.823944091796875,
"KL/std": 23.925195693969727,
"epoch": 0.2811791383219955,
"fcm_dpo/beta": 0.02497974783182144,
"fcm_dpo/delta": -0.07023215293884277,
"fcm_dpo/margin": 18.158008575439453,
"fcm_dpo/q_t": 0.39706262946128845,
"grad_norm": 17.287595748901367,
"learning_rate": 4.5287418106563354e-07,
"logits/chosen": 0.3560236692428589,
"logits/rejected": 0.312433660030365,
"logps/chosen": -84.0860595703125,
"logps/ref_chosen": -67.42012786865234,
"logps/ref_rejected": -82.50968933105469,
"logps/rejected": -117.33363342285156,
"loss": 1.104,
"margin_dpo/margin_mean": 18.158010482788086,
"margin_dpo/margin_std": 29.182445526123047,
"step": 186
},
{
"KL/chosen_KL_mean": -22.537792205810547,
"KL/mean": -30.65770721435547,
"KL/rejected_KL_mean": -38.77762985229492,
"KL/std": 25.62303924560547,
"epoch": 0.28269085411942557,
"fcm_dpo/beta": 0.02443239837884903,
"fcm_dpo/delta": 0.0032668951898813248,
"fcm_dpo/margin": 16.239839553833008,
"fcm_dpo/q_t": 0.4135947823524475,
"grad_norm": 15.863929748535156,
"learning_rate": 4.520986992917297e-07,
"logits/chosen": 0.41961655020713806,
"logits/rejected": 0.363817036151886,
"logps/chosen": -98.06328582763672,
"logps/ref_chosen": -75.52549743652344,
"logps/ref_rejected": -94.76289367675781,
"logps/rejected": -133.54052734375,
"loss": 1.1545,
"margin_dpo/margin_mean": 16.239839553833008,
"margin_dpo/margin_std": 30.24182891845703,
"step": 187
},
{
"KL/chosen_KL_mean": -20.841873168945312,
"KL/mean": -30.49065399169922,
"KL/rejected_KL_mean": -40.139442443847656,
"KL/std": 26.88828468322754,
"epoch": 0.2842025699168556,
"fcm_dpo/beta": 0.023507175967097282,
"fcm_dpo/delta": -0.06960050761699677,
"fcm_dpo/margin": 19.297565460205078,
"fcm_dpo/q_t": 0.39809972047805786,
"grad_norm": 12.509038925170898,
"learning_rate": 4.5131756438276466e-07,
"logits/chosen": 0.4281846880912781,
"logits/rejected": 0.384998619556427,
"logps/chosen": -92.36520385742188,
"logps/ref_chosen": -71.52333068847656,
"logps/ref_rejected": -78.29949951171875,
"logps/rejected": -118.4389419555664,
"loss": 1.0881,
"margin_dpo/margin_mean": 19.29756736755371,
"margin_dpo/margin_std": 28.976045608520508,
"step": 188
},
{
"KL/chosen_KL_mean": -20.803133010864258,
"KL/mean": -29.196277618408203,
"KL/rejected_KL_mean": -37.58942413330078,
"KL/std": 26.68855094909668,
"epoch": 0.2857142857142857,
"fcm_dpo/beta": 0.02266472578048706,
"fcm_dpo/delta": 0.0194806307554245,
"fcm_dpo/margin": 16.786296844482422,
"fcm_dpo/q_t": 0.4146800935268402,
"grad_norm": 14.953192710876465,
"learning_rate": 4.5053079818876096e-07,
"logits/chosen": 0.4511597156524658,
"logits/rejected": 0.46751925349235535,
"logps/chosen": -92.97940063476562,
"logps/ref_chosen": -72.17626953125,
"logps/ref_rejected": -75.26313781738281,
"logps/rejected": -112.85255432128906,
"loss": 1.1438,
"margin_dpo/margin_mean": 16.786296844482422,
"margin_dpo/margin_std": 27.98157501220703,
"step": 189
},
{
"KL/chosen_KL_mean": -19.941322326660156,
"KL/mean": -32.42082977294922,
"KL/rejected_KL_mean": -44.90033721923828,
"KL/std": 28.068832397460938,
"epoch": 0.2872260015117158,
"fcm_dpo/beta": 0.02070463076233864,
"fcm_dpo/delta": -0.16096393764019012,
"fcm_dpo/margin": 24.95901870727539,
"fcm_dpo/q_t": 0.37751439213752747,
"grad_norm": 18.85645866394043,
"learning_rate": 4.4973842271726024e-07,
"logits/chosen": 0.5245158672332764,
"logits/rejected": 0.36903607845306396,
"logps/chosen": -74.56559753417969,
"logps/ref_chosen": -54.624271392822266,
"logps/ref_rejected": -101.47068786621094,
"logps/rejected": -146.37103271484375,
"loss": 1.0289,
"margin_dpo/margin_mean": 24.95901870727539,
"margin_dpo/margin_std": 31.038402557373047,
"step": 190
},
{
"KL/chosen_KL_mean": -29.806278228759766,
"KL/mean": -39.38099670410156,
"KL/rejected_KL_mean": -48.95571517944336,
"KL/std": 29.71709442138672,
"epoch": 0.2887377173091459,
"fcm_dpo/beta": 0.020269766449928284,
"fcm_dpo/delta": 0.014703525230288506,
"fcm_dpo/margin": 19.14944076538086,
"fcm_dpo/q_t": 0.41183042526245117,
"grad_norm": 14.513212203979492,
"learning_rate": 4.48940460132708e-07,
"logits/chosen": 0.5108237266540527,
"logits/rejected": 0.4802900552749634,
"logps/chosen": -102.7387924194336,
"logps/ref_chosen": -72.93251037597656,
"logps/ref_rejected": -89.95103454589844,
"logps/rejected": -138.90675354003906,
"loss": 1.1321,
"margin_dpo/margin_mean": 19.14944076538086,
"margin_dpo/margin_std": 31.697952270507812,
"step": 191
},
{
"KL/chosen_KL_mean": -34.189476013183594,
"KL/mean": -40.280208587646484,
"KL/rejected_KL_mean": -46.37093734741211,
"KL/std": 31.34961700439453,
"epoch": 0.29024943310657597,
"fcm_dpo/beta": 0.02095700055360794,
"fcm_dpo/delta": 0.056858569383621216,
"fcm_dpo/margin": 12.181463241577148,
"fcm_dpo/q_t": 0.44299930334091187,
"grad_norm": 14.316689491271973,
"learning_rate": 4.481369327558329e-07,
"logits/chosen": 0.5008162260055542,
"logits/rejected": 0.47292008996009827,
"logps/chosen": -88.19059753417969,
"logps/ref_chosen": -54.001121520996094,
"logps/ref_rejected": -63.531551361083984,
"logps/rejected": -109.9024887084961,
"loss": 1.2285,
"margin_dpo/margin_mean": 12.181463241577148,
"margin_dpo/margin_std": 28.081724166870117,
"step": 192
},
{
"KL/chosen_KL_mean": -27.229509353637695,
"KL/mean": -38.814231872558594,
"KL/rejected_KL_mean": -50.398956298828125,
"KL/std": 29.14958953857422,
"epoch": 0.29176114890400606,
"fcm_dpo/beta": 0.01980617642402649,
"fcm_dpo/delta": -0.08162431418895721,
"fcm_dpo/margin": 23.16944122314453,
"fcm_dpo/q_t": 0.39218446612358093,
"grad_norm": 12.384896278381348,
"learning_rate": 4.47327863063023e-07,
"logits/chosen": 0.46182727813720703,
"logits/rejected": 0.4414292573928833,
"logps/chosen": -83.97879028320312,
"logps/ref_chosen": -56.74927520751953,
"logps/ref_rejected": -58.80629348754883,
"logps/rejected": -109.20524597167969,
"loss": 1.0637,
"margin_dpo/margin_mean": 23.16944122314453,
"margin_dpo/margin_std": 30.30309295654297,
"step": 193
},
{
"KL/chosen_KL_mean": -30.627708435058594,
"KL/mean": -39.823089599609375,
"KL/rejected_KL_mean": -49.018470764160156,
"KL/std": 30.80691146850586,
"epoch": 0.29327286470143615,
"fcm_dpo/beta": 0.019777745008468628,
"fcm_dpo/delta": 0.04038870707154274,
"fcm_dpo/margin": 18.390762329101562,
"fcm_dpo/q_t": 0.4192665219306946,
"grad_norm": 17.1464786529541,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 0.5140712857246399,
"logits/rejected": 0.48340654373168945,
"logps/chosen": -87.27714538574219,
"logps/ref_chosen": -56.64944076538086,
"logps/ref_rejected": -69.98954772949219,
"logps/rejected": -119.00801086425781,
"loss": 1.181,
"margin_dpo/margin_mean": 18.390764236450195,
"margin_dpo/margin_std": 36.06867980957031,
"step": 194
},
{
"KL/chosen_KL_mean": -36.239933013916016,
"KL/mean": -48.81350326538086,
"KL/rejected_KL_mean": -61.38707733154297,
"KL/std": 33.777400970458984,
"epoch": 0.2947845804988662,
"fcm_dpo/beta": 0.018797680735588074,
"fcm_dpo/delta": -0.10017755627632141,
"fcm_dpo/margin": 25.147144317626953,
"fcm_dpo/q_t": 0.38978683948516846,
"grad_norm": 15.896224021911621,
"learning_rate": 4.4569318740967043e-07,
"logits/chosen": 0.4080420136451721,
"logits/rejected": 0.41872015595436096,
"logps/chosen": -106.64971160888672,
"logps/ref_chosen": -70.40977478027344,
"logps/ref_rejected": -74.39448547363281,
"logps/rejected": -135.78155517578125,
"loss": 1.0756,
"margin_dpo/margin_mean": 25.147144317626953,
"margin_dpo/margin_std": 36.095008850097656,
"step": 195
},
{
"KL/chosen_KL_mean": -37.003265380859375,
"KL/mean": -46.09539794921875,
"KL/rejected_KL_mean": -55.18753433227539,
"KL/std": 35.14692687988281,
"epoch": 0.2962962962962963,
"fcm_dpo/beta": 0.01977820321917534,
"fcm_dpo/delta": 0.046010956168174744,
"fcm_dpo/margin": 18.18426513671875,
"fcm_dpo/q_t": 0.41924434900283813,
"grad_norm": 17.794841766357422,
"learning_rate": 4.448676271745197e-07,
"logits/chosen": 0.5215525031089783,
"logits/rejected": 0.4756397604942322,
"logps/chosen": -96.23084259033203,
"logps/ref_chosen": -59.227577209472656,
"logps/ref_rejected": -83.54757690429688,
"logps/rejected": -138.735107421875,
"loss": 1.1459,
"margin_dpo/margin_mean": 18.18426513671875,
"margin_dpo/margin_std": 30.57408905029297,
"step": 196
},
{
"KL/chosen_KL_mean": -32.982200622558594,
"KL/mean": -44.936031341552734,
"KL/rejected_KL_mean": -56.88986587524414,
"KL/std": 33.14744567871094,
"epoch": 0.29780801209372637,
"fcm_dpo/beta": 0.019095640629529953,
"fcm_dpo/delta": -0.07451824843883514,
"fcm_dpo/margin": 23.90766143798828,
"fcm_dpo/q_t": 0.39633709192276,
"grad_norm": 16.9836483001709,
"learning_rate": 4.440366160729392e-07,
"logits/chosen": 0.6078462600708008,
"logits/rejected": 0.5525588989257812,
"logps/chosen": -84.5113296508789,
"logps/ref_chosen": -51.52912902832031,
"logps/ref_rejected": -73.70631408691406,
"logps/rejected": -130.59617614746094,
"loss": 1.134,
"margin_dpo/margin_mean": 23.90765953063965,
"margin_dpo/margin_std": 41.63103485107422,
"step": 197
},
{
"KL/chosen_KL_mean": -35.99111557006836,
"KL/mean": -48.80699920654297,
"KL/rejected_KL_mean": -61.62289047241211,
"KL/std": 33.919517517089844,
"epoch": 0.29931972789115646,
"fcm_dpo/beta": 0.017618469893932343,
"fcm_dpo/delta": -0.06671290844678879,
"fcm_dpo/margin": 25.63176727294922,
"fcm_dpo/q_t": 0.3941599726676941,
"grad_norm": 14.554643630981445,
"learning_rate": 4.432001773500957e-07,
"logits/chosen": 0.5819200873374939,
"logits/rejected": 0.5387852191925049,
"logps/chosen": -95.77379608154297,
"logps/ref_chosen": -59.78268051147461,
"logps/ref_rejected": -72.24533081054688,
"logps/rejected": -133.86822509765625,
"loss": 1.0748,
"margin_dpo/margin_mean": 25.63176727294922,
"margin_dpo/margin_std": 34.73223114013672,
"step": 198
},
{
"KL/chosen_KL_mean": -43.49638366699219,
"KL/mean": -53.40158462524414,
"KL/rejected_KL_mean": -63.306785583496094,
"KL/std": 34.76414108276367,
"epoch": 0.30083144368858655,
"fcm_dpo/beta": 0.01662040874361992,
"fcm_dpo/delta": -0.04701909050345421,
"fcm_dpo/margin": 19.810405731201172,
"fcm_dpo/q_t": 0.4231904149055481,
"grad_norm": 12.835299491882324,
"learning_rate": 4.4235833440297856e-07,
"logits/chosen": 0.529554009437561,
"logits/rejected": 0.437679648399353,
"logps/chosen": -99.88314819335938,
"logps/ref_chosen": -56.38677215576172,
"logps/ref_rejected": -74.56779479980469,
"logps/rejected": -137.8745880126953,
"loss": 1.1928,
"margin_dpo/margin_mean": 19.810405731201172,
"margin_dpo/margin_std": 39.37937927246094,
"step": 199
},
{
"KL/chosen_KL_mean": -35.22080993652344,
"KL/mean": -49.63976287841797,
"KL/rejected_KL_mean": -64.05872344970703,
"KL/std": 36.015777587890625,
"epoch": 0.30234315948601664,
"fcm_dpo/beta": 0.015300557017326355,
"fcm_dpo/delta": -0.05560305714607239,
"fcm_dpo/margin": 28.837909698486328,
"fcm_dpo/q_t": 0.397291362285614,
"grad_norm": 12.404858589172363,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": 0.5855056047439575,
"logits/rejected": 0.5044532418251038,
"logps/chosen": -93.04513549804688,
"logps/ref_chosen": -57.82432556152344,
"logps/ref_rejected": -89.28246307373047,
"logps/rejected": -153.3411865234375,
"loss": 1.0938,
"margin_dpo/margin_mean": 28.837909698486328,
"margin_dpo/margin_std": 42.91114044189453,
"step": 200
},
{
"KL/chosen_KL_mean": -48.78852081298828,
"KL/mean": -60.213199615478516,
"KL/rejected_KL_mean": -71.63787841796875,
"KL/std": 37.877784729003906,
"epoch": 0.30385487528344673,
"fcm_dpo/beta": 0.015576442703604698,
"fcm_dpo/delta": 0.052298612892627716,
"fcm_dpo/margin": 22.84935760498047,
"fcm_dpo/q_t": 0.42080944776535034,
"grad_norm": 14.842419624328613,
"learning_rate": 4.4065853017905953e-07,
"logits/chosen": 0.6083788871765137,
"logits/rejected": 0.5580486059188843,
"logps/chosen": -107.78828430175781,
"logps/ref_chosen": -58.999759674072266,
"logps/ref_rejected": -84.67575073242188,
"logps/rejected": -156.31362915039062,
"loss": 1.157,
"margin_dpo/margin_mean": 22.84935760498047,
"margin_dpo/margin_std": 39.762603759765625,
"step": 201
},
{
"KL/chosen_KL_mean": -40.75391387939453,
"KL/mean": -55.758758544921875,
"KL/rejected_KL_mean": -70.76358795166016,
"KL/std": 36.27582550048828,
"epoch": 0.30536659108087677,
"fcm_dpo/beta": 0.015416000038385391,
"fcm_dpo/delta": -0.08111350238323212,
"fcm_dpo/margin": 30.009674072265625,
"fcm_dpo/q_t": 0.39005911350250244,
"grad_norm": 13.22191047668457,
"learning_rate": 4.3980061644943575e-07,
"logits/chosen": 0.5110350847244263,
"logits/rejected": 0.4323844313621521,
"logps/chosen": -88.41456604003906,
"logps/ref_chosen": -47.660648345947266,
"logps/ref_rejected": -73.63249969482422,
"logps/rejected": -144.39608764648438,
"loss": 1.0704,
"margin_dpo/margin_mean": 30.009674072265625,
"margin_dpo/margin_std": 40.26255798339844,
"step": 202
},
{
"KL/chosen_KL_mean": -51.74449157714844,
"KL/mean": -64.84964752197266,
"KL/rejected_KL_mean": -77.95479583740234,
"KL/std": 37.77312469482422,
"epoch": 0.30687830687830686,
"fcm_dpo/beta": 0.01509188860654831,
"fcm_dpo/delta": 0.005205655470490456,
"fcm_dpo/margin": 26.210308074951172,
"fcm_dpo/q_t": 0.4106394946575165,
"grad_norm": 15.703864097595215,
"learning_rate": 4.3893739358856455e-07,
"logits/chosen": 0.5600358843803406,
"logits/rejected": 0.47498154640197754,
"logps/chosen": -114.07002258300781,
"logps/ref_chosen": -62.32553482055664,
"logps/ref_rejected": -99.37226104736328,
"logps/rejected": -177.32705688476562,
"loss": 1.1234,
"margin_dpo/margin_mean": 26.21030616760254,
"margin_dpo/margin_std": 42.12183380126953,
"step": 203
},
{
"KL/chosen_KL_mean": -46.200294494628906,
"KL/mean": -60.380897521972656,
"KL/rejected_KL_mean": -74.56149291992188,
"KL/std": 38.21998596191406,
"epoch": 0.30839002267573695,
"fcm_dpo/beta": 0.01425536535680294,
"fcm_dpo/delta": -0.013648569583892822,
"fcm_dpo/margin": 28.361196517944336,
"fcm_dpo/q_t": 0.40619611740112305,
"grad_norm": 14.456401824951172,
"learning_rate": 4.380688857426449e-07,
"logits/chosen": 0.552179217338562,
"logits/rejected": 0.4786151647567749,
"logps/chosen": -96.8296127319336,
"logps/ref_chosen": -50.62931823730469,
"logps/ref_rejected": -66.60475158691406,
"logps/rejected": -141.16624450683594,
"loss": 1.1059,
"margin_dpo/margin_mean": 28.36119842529297,
"margin_dpo/margin_std": 39.7681770324707,
"step": 204
},
{
"KL/chosen_KL_mean": -54.09760665893555,
"KL/mean": -67.47254943847656,
"KL/rejected_KL_mean": -80.84750366210938,
"KL/std": 40.442012786865234,
"epoch": 0.30990173847316704,
"fcm_dpo/beta": 0.01479810755699873,
"fcm_dpo/delta": 0.0051469397731125355,
"fcm_dpo/margin": 26.74989128112793,
"fcm_dpo/q_t": 0.4112912714481354,
"grad_norm": 16.738981246948242,
"learning_rate": 4.3719511720570814e-07,
"logits/chosen": 0.6002988815307617,
"logits/rejected": 0.5250147581100464,
"logps/chosen": -124.45378112792969,
"logps/ref_chosen": -70.3561782836914,
"logps/ref_rejected": -93.39848327636719,
"logps/rejected": -174.2459716796875,
"loss": 1.1334,
"margin_dpo/margin_mean": 26.749893188476562,
"margin_dpo/margin_std": 45.088653564453125,
"step": 205
},
{
"KL/chosen_KL_mean": -56.70134735107422,
"KL/mean": -65.66264343261719,
"KL/rejected_KL_mean": -74.62395477294922,
"KL/std": 39.392059326171875,
"epoch": 0.31141345427059713,
"fcm_dpo/beta": 0.015088488347828388,
"fcm_dpo/delta": 0.029385201632976532,
"fcm_dpo/margin": 17.922607421875,
"fcm_dpo/q_t": 0.4415813684463501,
"grad_norm": 14.734772682189941,
"learning_rate": 4.363161124189387e-07,
"logits/chosen": 0.6411951780319214,
"logits/rejected": 0.6234967708587646,
"logps/chosen": -124.34683227539062,
"logps/ref_chosen": -67.64547729492188,
"logps/ref_rejected": -79.89584350585938,
"logps/rejected": -154.51980590820312,
"loss": 1.2593,
"margin_dpo/margin_mean": 17.922609329223633,
"margin_dpo/margin_std": 48.28623580932617,
"step": 206
},
{
"KL/chosen_KL_mean": -65.2525863647461,
"KL/mean": -77.85491943359375,
"KL/rejected_KL_mean": -90.45724487304688,
"KL/std": 40.893856048583984,
"epoch": 0.3129251700680272,
"fcm_dpo/beta": 0.015518700703978539,
"fcm_dpo/delta": 0.010801279917359352,
"fcm_dpo/margin": 25.20465087890625,
"fcm_dpo/q_t": 0.4146854877471924,
"grad_norm": 13.349529266357422,
"learning_rate": 4.3543189596998986e-07,
"logits/chosen": 0.5421028137207031,
"logits/rejected": 0.4715673327445984,
"logps/chosen": -132.91677856445312,
"logps/ref_chosen": -67.66419219970703,
"logps/ref_rejected": -85.10249328613281,
"logps/rejected": -175.55972290039062,
"loss": 1.1394,
"margin_dpo/margin_mean": 25.20465087890625,
"margin_dpo/margin_std": 43.88922119140625,
"step": 207
},
{
"KL/chosen_KL_mean": -55.959251403808594,
"KL/mean": -62.835411071777344,
"KL/rejected_KL_mean": -69.71156311035156,
"KL/std": 39.03756332397461,
"epoch": 0.3144368858654573,
"fcm_dpo/beta": 0.016686484217643738,
"fcm_dpo/delta": 0.07528108358383179,
"fcm_dpo/margin": 13.752311706542969,
"fcm_dpo/q_t": 0.4518064856529236,
"grad_norm": 22.71687126159668,
"learning_rate": 4.3454249259229664e-07,
"logits/chosen": 0.6001783609390259,
"logits/rejected": 0.5711269974708557,
"logps/chosen": -113.69096374511719,
"logps/ref_chosen": -57.731712341308594,
"logps/ref_rejected": -74.19276428222656,
"logps/rejected": -143.90432739257812,
"loss": 1.3074,
"margin_dpo/margin_mean": 13.752311706542969,
"margin_dpo/margin_std": 45.60139846801758,
"step": 208
},
{
"KL/chosen_KL_mean": -55.48735809326172,
"KL/mean": -71.57313537597656,
"KL/rejected_KL_mean": -87.65892028808594,
"KL/std": 43.753570556640625,
"epoch": 0.31594860166288735,
"fcm_dpo/beta": 0.015102138742804527,
"fcm_dpo/delta": -0.11657831072807312,
"fcm_dpo/margin": 32.17155838012695,
"fcm_dpo/q_t": 0.3876298666000366,
"grad_norm": 15.37290096282959,
"learning_rate": 4.336479271643833e-07,
"logits/chosen": 0.5027754306793213,
"logits/rejected": 0.4479963183403015,
"logps/chosen": -124.03743743896484,
"logps/ref_chosen": -68.55007934570312,
"logps/ref_rejected": -87.90541076660156,
"logps/rejected": -175.5643310546875,
"loss": 1.0976,
"margin_dpo/margin_mean": 32.17155838012695,
"margin_dpo/margin_std": 51.997520446777344,
"step": 209
},
{
"KL/chosen_KL_mean": -53.723114013671875,
"KL/mean": -70.12528228759766,
"KL/rejected_KL_mean": -86.52745056152344,
"KL/std": 44.81327819824219,
"epoch": 0.31746031746031744,
"fcm_dpo/beta": 0.014353602193295956,
"fcm_dpo/delta": -0.0943194180727005,
"fcm_dpo/margin": 32.804325103759766,
"fcm_dpo/q_t": 0.39221110939979553,
"grad_norm": 16.092060089111328,
"learning_rate": 4.327482247091679e-07,
"logits/chosen": 0.6441676616668701,
"logits/rejected": 0.5358040928840637,
"logps/chosen": -110.99138641357422,
"logps/ref_chosen": -57.268272399902344,
"logps/ref_rejected": -85.72807312011719,
"logps/rejected": -172.25552368164062,
"loss": 1.0813,
"margin_dpo/margin_mean": 32.80432891845703,
"margin_dpo/margin_std": 46.69495391845703,
"step": 210
},
{
"KL/chosen_KL_mean": -49.60154724121094,
"KL/mean": -65.75942993164062,
"KL/rejected_KL_mean": -81.91730499267578,
"KL/std": 42.09577941894531,
"epoch": 0.31897203325774753,
"fcm_dpo/beta": 0.013455729931592941,
"fcm_dpo/delta": -0.046873997896909714,
"fcm_dpo/margin": 32.31575393676758,
"fcm_dpo/q_t": 0.40088406205177307,
"grad_norm": 17.506433486938477,
"learning_rate": 4.3184341039326217e-07,
"logits/chosen": 0.6236763000488281,
"logits/rejected": 0.517917275428772,
"logps/chosen": -103.24225616455078,
"logps/ref_chosen": -53.640708923339844,
"logps/ref_rejected": -93.0387954711914,
"logps/rejected": -174.9561004638672,
"loss": 1.0823,
"margin_dpo/margin_mean": 32.31575012207031,
"margin_dpo/margin_std": 43.71531677246094,
"step": 211
},
{
"KL/chosen_KL_mean": -55.83790588378906,
"KL/mean": -72.02022552490234,
"KL/rejected_KL_mean": -88.2025375366211,
"KL/std": 44.30055236816406,
"epoch": 0.3204837490551776,
"fcm_dpo/beta": 0.012746179476380348,
"fcm_dpo/delta": -0.015783989802002907,
"fcm_dpo/margin": 32.36463165283203,
"fcm_dpo/q_t": 0.4043083190917969,
"grad_norm": 10.171900749206543,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 0.6819473505020142,
"logits/rejected": 0.5994547605514526,
"logps/chosen": -113.20465087890625,
"logps/ref_chosen": -57.36674499511719,
"logps/ref_rejected": -79.89643096923828,
"logps/rejected": -168.09896850585938,
"loss": 1.105,
"margin_dpo/margin_mean": 32.36463165283203,
"margin_dpo/margin_std": 47.924339294433594,
"step": 212
},
{
"KL/chosen_KL_mean": -44.525672912597656,
"KL/mean": -63.037479400634766,
"KL/rejected_KL_mean": -81.54928588867188,
"KL/std": 44.70442199707031,
"epoch": 0.3219954648526077,
"fcm_dpo/beta": 0.011922631412744522,
"fcm_dpo/delta": -0.05658069625496864,
"fcm_dpo/margin": 37.023616790771484,
"fcm_dpo/q_t": 0.396555632352829,
"grad_norm": 11.255715370178223,
"learning_rate": 4.3001854756006724e-07,
"logits/chosen": 0.6020195484161377,
"logits/rejected": 0.5774927139282227,
"logps/chosen": -109.74678802490234,
"logps/ref_chosen": -65.22111511230469,
"logps/ref_rejected": -80.1810302734375,
"logps/rejected": -161.73031616210938,
"loss": 1.0767,
"margin_dpo/margin_mean": 37.02362060546875,
"margin_dpo/margin_std": 50.05058288574219,
"step": 213
},
{
"KL/chosen_KL_mean": -52.843170166015625,
"KL/mean": -70.69467163085938,
"KL/rejected_KL_mean": -88.5461654663086,
"KL/std": 45.061424255371094,
"epoch": 0.3235071806500378,
"fcm_dpo/beta": 0.012001337483525276,
"fcm_dpo/delta": -0.0386454313993454,
"fcm_dpo/margin": 35.7030029296875,
"fcm_dpo/q_t": 0.4016464650630951,
"grad_norm": 13.126012802124023,
"learning_rate": 4.290985500881143e-07,
"logits/chosen": 0.45403045415878296,
"logits/rejected": 0.4404328465461731,
"logps/chosen": -114.135498046875,
"logps/ref_chosen": -61.292327880859375,
"logps/ref_rejected": -67.69841003417969,
"logps/rejected": -156.24456787109375,
"loss": 1.101,
"margin_dpo/margin_mean": 35.7030029296875,
"margin_dpo/margin_std": 52.66297149658203,
"step": 214
},
{
"KL/chosen_KL_mean": -58.013763427734375,
"KL/mean": -76.46696472167969,
"KL/rejected_KL_mean": -94.920166015625,
"KL/std": 46.602779388427734,
"epoch": 0.3250188964474679,
"fcm_dpo/beta": 0.011644062586128712,
"fcm_dpo/delta": -0.04298893362283707,
"fcm_dpo/margin": 36.906394958496094,
"fcm_dpo/q_t": 0.4008602797985077,
"grad_norm": 12.266192436218262,
"learning_rate": 4.281735428447157e-07,
"logits/chosen": 0.5056573748588562,
"logits/rejected": 0.38637077808380127,
"logps/chosen": -121.88290405273438,
"logps/ref_chosen": -63.869136810302734,
"logps/ref_rejected": -98.7657241821289,
"logps/rejected": -193.68588256835938,
"loss": 1.1084,
"margin_dpo/margin_mean": 36.90639114379883,
"margin_dpo/margin_std": 54.872398376464844,
"step": 215
},
{
"KL/chosen_KL_mean": -49.014102935791016,
"KL/mean": -68.88029479980469,
"KL/rejected_KL_mean": -88.74649047851562,
"KL/std": 45.22344207763672,
"epoch": 0.32653061224489793,
"fcm_dpo/beta": 0.010737748816609383,
"fcm_dpo/delta": -0.03410545364022255,
"fcm_dpo/margin": 39.732391357421875,
"fcm_dpo/q_t": 0.3984612226486206,
"grad_norm": 21.626602172851562,
"learning_rate": 4.2724355170431247e-07,
"logits/chosen": 0.6049730181694031,
"logits/rejected": 0.5137053728103638,
"logps/chosen": -116.83905792236328,
"logps/ref_chosen": -67.824951171875,
"logps/ref_rejected": -96.40231323242188,
"logps/rejected": -185.1488037109375,
"loss": 1.0616,
"margin_dpo/margin_mean": 39.732391357421875,
"margin_dpo/margin_std": 45.669105529785156,
"step": 216
},
{
"KL/chosen_KL_mean": -53.43660354614258,
"KL/mean": -74.12617492675781,
"KL/rejected_KL_mean": -94.81575775146484,
"KL/std": 46.69729995727539,
"epoch": 0.328042328042328,
"fcm_dpo/beta": 0.010264039039611816,
"fcm_dpo/delta": -0.03311503306031227,
"fcm_dpo/margin": 41.379146575927734,
"fcm_dpo/q_t": 0.3996722102165222,
"grad_norm": 10.103537559509277,
"learning_rate": 4.26308602680756e-07,
"logits/chosen": 0.5727427005767822,
"logits/rejected": 0.4670327305793762,
"logps/chosen": -113.94159698486328,
"logps/ref_chosen": -60.5049934387207,
"logps/ref_rejected": -84.26618194580078,
"logps/rejected": -179.08193969726562,
"loss": 1.0655,
"margin_dpo/margin_mean": 41.379146575927734,
"margin_dpo/margin_std": 48.67888641357422,
"step": 217
},
{
"KL/chosen_KL_mean": -59.745277404785156,
"KL/mean": -71.58668518066406,
"KL/rejected_KL_mean": -83.42808532714844,
"KL/std": 46.134681701660156,
"epoch": 0.3295540438397581,
"fcm_dpo/beta": 0.010512400418519974,
"fcm_dpo/delta": 0.033613234758377075,
"fcm_dpo/margin": 23.68280601501465,
"fcm_dpo/q_t": 0.44248485565185547,
"grad_norm": 11.078608512878418,
"learning_rate": 4.253687219265803e-07,
"logits/chosen": 0.46339693665504456,
"logits/rejected": 0.4667159914970398,
"logps/chosen": -130.339599609375,
"logps/ref_chosen": -70.59431457519531,
"logps/ref_rejected": -73.89038848876953,
"logps/rejected": -157.3184814453125,
"loss": 1.2384,
"margin_dpo/margin_mean": 23.68280601501465,
"margin_dpo/margin_std": 55.93402099609375,
"step": 218
},
{
"KL/chosen_KL_mean": -55.8802490234375,
"KL/mean": -68.90615844726562,
"KL/rejected_KL_mean": -81.93206787109375,
"KL/std": 44.350650787353516,
"epoch": 0.3310657596371882,
"fcm_dpo/beta": 0.011553551070392132,
"fcm_dpo/delta": 0.11722569167613983,
"fcm_dpo/margin": 26.05181884765625,
"fcm_dpo/q_t": 0.43380120396614075,
"grad_norm": 12.027776718139648,
"learning_rate": 4.2442393573227043e-07,
"logits/chosen": 0.5445624589920044,
"logits/rejected": 0.5009948015213013,
"logps/chosen": -116.3711929321289,
"logps/ref_chosen": -60.490943908691406,
"logps/ref_rejected": -75.85001373291016,
"logps/rejected": -157.78207397460938,
"loss": 1.1777,
"margin_dpo/margin_mean": 26.05181884765625,
"margin_dpo/margin_std": 46.083717346191406,
"step": 219
},
{
"KL/chosen_KL_mean": -47.478050231933594,
"KL/mean": -63.36997985839844,
"KL/rejected_KL_mean": -79.26191711425781,
"KL/std": 47.789817810058594,
"epoch": 0.3325774754346183,
"fcm_dpo/beta": 0.011883954517543316,
"fcm_dpo/delta": 0.024295464158058167,
"fcm_dpo/margin": 31.783857345581055,
"fcm_dpo/q_t": 0.4159383475780487,
"grad_norm": 13.699325561523438,
"learning_rate": 4.234742705255272e-07,
"logits/chosen": 0.6039453148841858,
"logits/rejected": 0.5356512069702148,
"logps/chosen": -92.491455078125,
"logps/ref_chosen": -45.013397216796875,
"logps/ref_rejected": -70.49369812011719,
"logps/rejected": -149.755615234375,
"loss": 1.1543,
"margin_dpo/margin_mean": 31.783859252929688,
"margin_dpo/margin_std": 55.53855514526367,
"step": 220
},
{
"KL/chosen_KL_mean": -47.119110107421875,
"KL/mean": -65.26559448242188,
"KL/rejected_KL_mean": -83.41207885742188,
"KL/std": 47.285316467285156,
"epoch": 0.3340891912320484,
"fcm_dpo/beta": 0.011956527829170227,
"fcm_dpo/delta": -0.04328383877873421,
"fcm_dpo/margin": 36.29296112060547,
"fcm_dpo/q_t": 0.40149760246276855,
"grad_norm": 13.390650749206543,
"learning_rate": 4.22519752870528e-07,
"logits/chosen": 0.5664784908294678,
"logits/rejected": 0.4844658374786377,
"logps/chosen": -106.21495056152344,
"logps/ref_chosen": -59.09584045410156,
"logps/ref_rejected": -88.64388275146484,
"logps/rejected": -172.05596923828125,
"loss": 1.0937,
"margin_dpo/margin_mean": 36.29296112060547,
"margin_dpo/margin_std": 53.84539031982422,
"step": 221
},
{
"KL/chosen_KL_mean": -47.29530334472656,
"KL/mean": -71.27445983886719,
"KL/rejected_KL_mean": -95.25361633300781,
"KL/std": 53.57499694824219,
"epoch": 0.3356009070294785,
"fcm_dpo/beta": 0.010547778569161892,
"fcm_dpo/delta": -0.14348104596138,
"fcm_dpo/margin": 47.95830535888672,
"fcm_dpo/q_t": 0.3792991638183594,
"grad_norm": 11.575348854064941,
"learning_rate": 4.2156040946718343e-07,
"logits/chosen": 0.6499016284942627,
"logits/rejected": 0.5500251054763794,
"logps/chosen": -103.29299926757812,
"logps/ref_chosen": -55.9976921081543,
"logps/ref_rejected": -111.94727325439453,
"logps/rejected": -207.20089721679688,
"loss": 1.0293,
"margin_dpo/margin_mean": 47.95830535888672,
"margin_dpo/margin_std": 57.93938446044922,
"step": 222
},
{
"KL/chosen_KL_mean": -56.65371322631836,
"KL/mean": -75.22976684570312,
"KL/rejected_KL_mean": -93.80583190917969,
"KL/std": 51.016075134277344,
"epoch": 0.3371126228269085,
"fcm_dpo/beta": 0.010310441255569458,
"fcm_dpo/delta": 0.02092297561466694,
"fcm_dpo/margin": 37.15211486816406,
"fcm_dpo/q_t": 0.4130101799964905,
"grad_norm": 12.59165096282959,
"learning_rate": 4.2059626715039065e-07,
"logits/chosen": 0.6236449480056763,
"logits/rejected": 0.5647650361061096,
"logps/chosen": -116.54513549804688,
"logps/ref_chosen": -59.891422271728516,
"logps/ref_rejected": -86.28954315185547,
"logps/rejected": -180.09536743164062,
"loss": 1.1099,
"margin_dpo/margin_mean": 37.1521110534668,
"margin_dpo/margin_std": 52.98957061767578,
"step": 223
},
{
"KL/chosen_KL_mean": -65.9832763671875,
"KL/mean": -76.13775634765625,
"KL/rejected_KL_mean": -86.292236328125,
"KL/std": 50.740623474121094,
"epoch": 0.3386243386243386,
"fcm_dpo/beta": 0.011297405697405338,
"fcm_dpo/delta": 0.07967288047075272,
"fcm_dpo/margin": 20.30895233154297,
"fcm_dpo/q_t": 0.44954442977905273,
"grad_norm": 17.632829666137695,
"learning_rate": 4.1962735288928304e-07,
"logits/chosen": 0.6315662264823914,
"logits/rejected": 0.6085413098335266,
"logps/chosen": -130.02792358398438,
"logps/ref_chosen": -64.04463195800781,
"logps/ref_rejected": -75.05450439453125,
"logps/rejected": -161.34674072265625,
"loss": 1.26,
"margin_dpo/margin_mean": 20.3089542388916,
"margin_dpo/margin_std": 54.537742614746094,
"step": 224
},
{
"KL/chosen_KL_mean": -59.97877502441406,
"KL/mean": -81.12055206298828,
"KL/rejected_KL_mean": -102.2623291015625,
"KL/std": 55.81074905395508,
"epoch": 0.3401360544217687,
"fcm_dpo/beta": 0.010476674884557724,
"fcm_dpo/delta": -0.061587173491716385,
"fcm_dpo/margin": 42.28356170654297,
"fcm_dpo/q_t": 0.3950624465942383,
"grad_norm": 12.086466789245605,
"learning_rate": 4.186536937864752e-07,
"logits/chosen": 0.6357635259628296,
"logits/rejected": 0.5179706811904907,
"logps/chosen": -126.07460021972656,
"logps/ref_chosen": -66.0958251953125,
"logps/ref_rejected": -97.68675231933594,
"logps/rejected": -199.94908142089844,
"loss": 1.0622,
"margin_dpo/margin_mean": 42.28356170654297,
"margin_dpo/margin_std": 51.27521896362305,
"step": 225
},
{
"KL/chosen_KL_mean": -58.04826354980469,
"KL/mean": -74.63319396972656,
"KL/rejected_KL_mean": -91.21812438964844,
"KL/std": 50.970638275146484,
"epoch": 0.3416477702191988,
"fcm_dpo/beta": 0.011192118749022484,
"fcm_dpo/delta": 0.030396804213523865,
"fcm_dpo/margin": 33.16985321044922,
"fcm_dpo/q_t": 0.4161601662635803,
"grad_norm": 13.262188911437988,
"learning_rate": 4.176753170773052e-07,
"logits/chosen": 0.672427773475647,
"logits/rejected": 0.6251287460327148,
"logps/chosen": -109.46513366699219,
"logps/ref_chosen": -51.4168701171875,
"logps/ref_rejected": -66.30068969726562,
"logps/rejected": -157.518798828125,
"loss": 1.1669,
"margin_dpo/margin_mean": 33.16985321044922,
"margin_dpo/margin_std": 62.296966552734375,
"step": 226
},
{
"KL/chosen_KL_mean": -62.35593795776367,
"KL/mean": -79.04609680175781,
"KL/rejected_KL_mean": -95.73623657226562,
"KL/std": 55.52360153198242,
"epoch": 0.3431594860166289,
"fcm_dpo/beta": 0.011107344180345535,
"fcm_dpo/delta": 0.03568103536963463,
"fcm_dpo/margin": 33.38031005859375,
"fcm_dpo/q_t": 0.4179414212703705,
"grad_norm": 13.083468437194824,
"learning_rate": 4.166922501290729e-07,
"logits/chosen": 0.6802507638931274,
"logits/rejected": 0.6432693004608154,
"logps/chosen": -120.34571838378906,
"logps/ref_chosen": -57.989776611328125,
"logps/ref_rejected": -75.05464172363281,
"logps/rejected": -170.7908935546875,
"loss": 1.1644,
"margin_dpo/margin_mean": 33.38031005859375,
"margin_dpo/margin_std": 62.746315002441406,
"step": 227
},
{
"KL/chosen_KL_mean": -65.89985656738281,
"KL/mean": -85.34629821777344,
"KL/rejected_KL_mean": -104.79273223876953,
"KL/std": 52.757869720458984,
"epoch": 0.34467120181405897,
"fcm_dpo/beta": 0.011195195838809013,
"fcm_dpo/delta": -0.04603715240955353,
"fcm_dpo/margin": 38.89287567138672,
"fcm_dpo/q_t": 0.399330198764801,
"grad_norm": 14.76215648651123,
"learning_rate": 4.1570452044027405e-07,
"logits/chosen": 0.6884451508522034,
"logits/rejected": 0.6041937470436096,
"logps/chosen": -121.45922088623047,
"logps/ref_chosen": -55.55936813354492,
"logps/ref_rejected": -77.02364349365234,
"logps/rejected": -181.81637573242188,
"loss": 1.093,
"margin_dpo/margin_mean": 38.89287567138672,
"margin_dpo/margin_std": 56.33568572998047,
"step": 228
},
{
"KL/chosen_KL_mean": -56.05231475830078,
"KL/mean": -73.21758270263672,
"KL/rejected_KL_mean": -90.38285064697266,
"KL/std": 51.87107849121094,
"epoch": 0.34618291761148906,
"fcm_dpo/beta": 0.011033633723855019,
"fcm_dpo/delta": 0.02618766576051712,
"fcm_dpo/margin": 34.330543518066406,
"fcm_dpo/q_t": 0.4163823127746582,
"grad_norm": 16.91952133178711,
"learning_rate": 4.147121556398312e-07,
"logits/chosen": 0.7776898741722107,
"logits/rejected": 0.6957346200942993,
"logps/chosen": -106.84697723388672,
"logps/ref_chosen": -50.79466247558594,
"logps/ref_rejected": -78.4474105834961,
"logps/rejected": -168.83026123046875,
"loss": 1.1559,
"margin_dpo/margin_mean": 34.330543518066406,
"margin_dpo/margin_std": 63.17416763305664,
"step": 229
},
{
"KL/chosen_KL_mean": -60.39435577392578,
"KL/mean": -79.12918090820312,
"KL/rejected_KL_mean": -97.86400604248047,
"KL/std": 55.7061653137207,
"epoch": 0.3476946334089191,
"fcm_dpo/beta": 0.011317807249724865,
"fcm_dpo/delta": -0.03716056048870087,
"fcm_dpo/margin": 37.46965408325195,
"fcm_dpo/q_t": 0.4025961756706238,
"grad_norm": 13.502501487731934,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 0.6553382873535156,
"logits/rejected": 0.658027172088623,
"logps/chosen": -117.12358093261719,
"logps/ref_chosen": -56.729225158691406,
"logps/ref_rejected": -62.99180603027344,
"logps/rejected": -160.85580444335938,
"loss": 1.1148,
"margin_dpo/margin_mean": 37.46965408325195,
"margin_dpo/margin_std": 57.32815170288086,
"step": 230
},
{
"KL/chosen_KL_mean": -68.11759185791016,
"KL/mean": -95.83013916015625,
"KL/rejected_KL_mean": -123.5427017211914,
"KL/std": 54.258819580078125,
"epoch": 0.3492063492063492,
"fcm_dpo/beta": 0.009298204444348812,
"fcm_dpo/delta": -0.16286741197109222,
"fcm_dpo/margin": 55.42510986328125,
"fcm_dpo/q_t": 0.3714408874511719,
"grad_norm": 12.732369422912598,
"learning_rate": 4.1271363186719835e-07,
"logits/chosen": 0.5794389247894287,
"logits/rejected": 0.5769394636154175,
"logps/chosen": -140.71469116210938,
"logps/ref_chosen": -72.59709930419922,
"logps/ref_rejected": -86.2322998046875,
"logps/rejected": -209.77499389648438,
"loss": 1.0075,
"margin_dpo/margin_mean": 55.42510986328125,
"margin_dpo/margin_std": 59.240150451660156,
"step": 231
},
{
"KL/chosen_KL_mean": -68.00968933105469,
"KL/mean": -86.72840881347656,
"KL/rejected_KL_mean": -105.44712829589844,
"KL/std": 59.867862701416016,
"epoch": 0.3507180650037793,
"fcm_dpo/beta": 0.00953882560133934,
"fcm_dpo/delta": 0.051407087594270706,
"fcm_dpo/margin": 37.43744659423828,
"fcm_dpo/q_t": 0.4222131371498108,
"grad_norm": 12.651902198791504,
"learning_rate": 4.1170752879801436e-07,
"logits/chosen": 0.6084052324295044,
"logits/rejected": 0.5753206014633179,
"logps/chosen": -136.1282196044922,
"logps/ref_chosen": -68.1185302734375,
"logps/ref_rejected": -83.79415893554688,
"logps/rejected": -189.24130249023438,
"loss": 1.1621,
"margin_dpo/margin_mean": 37.43744659423828,
"margin_dpo/margin_std": 68.7430191040039,
"step": 232
},
{
"KL/chosen_KL_mean": -77.7924575805664,
"KL/mean": -91.54695129394531,
"KL/rejected_KL_mean": -105.30143737792969,
"KL/std": 54.55568313598633,
"epoch": 0.35222978080120937,
"fcm_dpo/beta": 0.00945155881345272,
"fcm_dpo/delta": -0.014413751661777496,
"fcm_dpo/margin": 27.508983612060547,
"fcm_dpo/q_t": 0.4402283728122711,
"grad_norm": 12.321041107177734,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": 0.7221860289573669,
"logits/rejected": 0.6576840877532959,
"logps/chosen": -132.86260986328125,
"logps/ref_chosen": -55.070152282714844,
"logps/ref_rejected": -66.61845397949219,
"logps/rejected": -171.91989135742188,
"loss": 1.2193,
"margin_dpo/margin_mean": 27.508983612060547,
"margin_dpo/margin_std": 57.265926361083984,
"step": 233
},
{
"KL/chosen_KL_mean": -68.60944366455078,
"KL/mean": -81.6121597290039,
"KL/rejected_KL_mean": -94.61487579345703,
"KL/std": 52.89654541015625,
"epoch": 0.35374149659863946,
"fcm_dpo/beta": 0.009320363402366638,
"fcm_dpo/delta": -0.028154423460364342,
"fcm_dpo/margin": 26.00543975830078,
"fcm_dpo/q_t": 0.44363802671432495,
"grad_norm": 18.08864974975586,
"learning_rate": 4.09681781007452e-07,
"logits/chosen": 0.606266975402832,
"logits/rejected": 0.6039628982543945,
"logps/chosen": -124.53533935546875,
"logps/ref_chosen": -55.92589569091797,
"logps/ref_rejected": -51.11608123779297,
"logps/rejected": -145.73095703125,
"loss": 1.2432,
"margin_dpo/margin_mean": 26.00543975830078,
"margin_dpo/margin_std": 58.44346618652344,
"step": 234
},
{
"KL/chosen_KL_mean": -62.99231719970703,
"KL/mean": -86.39044189453125,
"KL/rejected_KL_mean": -109.78857421875,
"KL/std": 58.375823974609375,
"epoch": 0.35525321239606955,
"fcm_dpo/beta": 0.009040637873113155,
"fcm_dpo/delta": -0.02921903133392334,
"fcm_dpo/margin": 46.7962646484375,
"fcm_dpo/q_t": 0.4005424380302429,
"grad_norm": 13.630228996276855,
"learning_rate": 4.08662192950594e-07,
"logits/chosen": 0.6960441470146179,
"logits/rejected": 0.6833631992340088,
"logps/chosen": -127.53204345703125,
"logps/ref_chosen": -64.53972625732422,
"logps/ref_rejected": -77.69151306152344,
"logps/rejected": -187.48007202148438,
"loss": 1.0687,
"margin_dpo/margin_mean": 46.7962646484375,
"margin_dpo/margin_std": 56.326026916503906,
"step": 235
},
{
"KL/chosen_KL_mean": -89.99750518798828,
"KL/mean": -107.64643859863281,
"KL/rejected_KL_mean": -125.29537200927734,
"KL/std": 56.224552154541016,
"epoch": 0.35676492819349964,
"fcm_dpo/beta": 0.009423347190022469,
"fcm_dpo/delta": 0.08110842108726501,
"fcm_dpo/margin": 35.29786682128906,
"fcm_dpo/q_t": 0.42564913630485535,
"grad_norm": 11.200446128845215,
"learning_rate": 4.076381667711306e-07,
"logits/chosen": 0.6303587555885315,
"logits/rejected": 0.6174304485321045,
"logps/chosen": -161.15225219726562,
"logps/ref_chosen": -71.15473937988281,
"logps/ref_rejected": -84.88541412353516,
"logps/rejected": -210.1807861328125,
"loss": 1.1709,
"margin_dpo/margin_mean": 35.29786682128906,
"margin_dpo/margin_std": 64.950927734375,
"step": 236
},
{
"KL/chosen_KL_mean": -75.23811340332031,
"KL/mean": -97.05203247070312,
"KL/rejected_KL_mean": -118.8659439086914,
"KL/std": 56.13910675048828,
"epoch": 0.35827664399092973,
"fcm_dpo/beta": 0.009628495201468468,
"fcm_dpo/delta": -0.02585686929523945,
"fcm_dpo/margin": 43.6278190612793,
"fcm_dpo/q_t": 0.4026941657066345,
"grad_norm": 14.894103050231934,
"learning_rate": 4.066097311132753e-07,
"logits/chosen": 0.6865165829658508,
"logits/rejected": 0.6780955791473389,
"logps/chosen": -151.380126953125,
"logps/ref_chosen": -76.14201354980469,
"logps/ref_rejected": -80.88479614257812,
"logps/rejected": -199.750732421875,
"loss": 1.1072,
"margin_dpo/margin_mean": 43.62782287597656,
"margin_dpo/margin_std": 65.629150390625,
"step": 237
},
{
"KL/chosen_KL_mean": -66.66596984863281,
"KL/mean": -87.54930877685547,
"KL/rejected_KL_mean": -108.43265533447266,
"KL/std": 57.61941909790039,
"epoch": 0.35978835978835977,
"fcm_dpo/beta": 0.00932412687689066,
"fcm_dpo/delta": 0.011540941894054413,
"fcm_dpo/margin": 41.76669692993164,
"fcm_dpo/q_t": 0.4105595350265503,
"grad_norm": 15.398371696472168,
"learning_rate": 4.0557691474458414e-07,
"logits/chosen": 0.6271650791168213,
"logits/rejected": 0.6165032386779785,
"logps/chosen": -135.55081176757812,
"logps/ref_chosen": -68.88484954833984,
"logps/ref_rejected": -75.8946304321289,
"logps/rejected": -184.32728576660156,
"loss": 1.1145,
"margin_dpo/margin_mean": 41.766693115234375,
"margin_dpo/margin_std": 60.80207061767578,
"step": 238
},
{
"KL/chosen_KL_mean": -80.04875183105469,
"KL/mean": -102.63557434082031,
"KL/rejected_KL_mean": -125.22238159179688,
"KL/std": 63.35125732421875,
"epoch": 0.36130007558578986,
"fcm_dpo/beta": 0.009503120556473732,
"fcm_dpo/delta": -0.0388723723590374,
"fcm_dpo/margin": 45.17362976074219,
"fcm_dpo/q_t": 0.40100669860839844,
"grad_norm": 18.990455627441406,
"learning_rate": 4.045397465551513e-07,
"logits/chosen": 0.753252387046814,
"logits/rejected": 0.6065235733985901,
"logps/chosen": -136.82058715820312,
"logps/ref_chosen": -56.771827697753906,
"logps/ref_rejected": -116.23050689697266,
"logps/rejected": -241.452880859375,
"loss": 1.106,
"margin_dpo/margin_mean": 45.17362976074219,
"margin_dpo/margin_std": 67.95465850830078,
"step": 239
},
{
"KL/chosen_KL_mean": -79.03121948242188,
"KL/mean": -106.46686553955078,
"KL/rejected_KL_mean": -133.90252685546875,
"KL/std": 63.37854766845703,
"epoch": 0.36281179138321995,
"fcm_dpo/beta": 0.00855420995503664,
"fcm_dpo/delta": -0.09123983979225159,
"fcm_dpo/margin": 54.871299743652344,
"fcm_dpo/q_t": 0.38927310705184937,
"grad_norm": 11.540243148803711,
"learning_rate": 4.0349825555680045e-07,
"logits/chosen": 0.6777857542037964,
"logits/rejected": 0.5775108337402344,
"logps/chosen": -132.3853302001953,
"logps/ref_chosen": -53.35411071777344,
"logps/ref_rejected": -80.12019348144531,
"logps/rejected": -214.02272033691406,
"loss": 1.0507,
"margin_dpo/margin_mean": 54.871299743652344,
"margin_dpo/margin_std": 68.62873840332031,
"step": 240
},
{
"KL/chosen_KL_mean": -74.54257202148438,
"KL/mean": -92.24656677246094,
"KL/rejected_KL_mean": -109.95057678222656,
"KL/std": 60.10264587402344,
"epoch": 0.36432350718065004,
"fcm_dpo/beta": 0.009079969488084316,
"fcm_dpo/delta": 0.0913933590054512,
"fcm_dpo/margin": 35.40801239013672,
"fcm_dpo/q_t": 0.42948025465011597,
"grad_norm": 18.661176681518555,
"learning_rate": 4.0245247088227377e-07,
"logits/chosen": 0.6268007159233093,
"logits/rejected": 0.5906528234481812,
"logps/chosen": -146.43798828125,
"logps/ref_chosen": -71.89541625976562,
"logps/ref_rejected": -83.03492736816406,
"logps/rejected": -192.98550415039062,
"loss": 1.1824,
"margin_dpo/margin_mean": 35.40801239013672,
"margin_dpo/margin_std": 68.74810791015625,
"step": 241
},
{
"KL/chosen_KL_mean": -76.36582946777344,
"KL/mean": -98.97297668457031,
"KL/rejected_KL_mean": -121.58012390136719,
"KL/std": 61.90785598754883,
"epoch": 0.36583522297808013,
"fcm_dpo/beta": 0.00890383217483759,
"fcm_dpo/delta": -0.006449941545724869,
"fcm_dpo/margin": 45.21430206298828,
"fcm_dpo/q_t": 0.40826159715652466,
"grad_norm": 11.749044418334961,
"learning_rate": 4.0140242178441665e-07,
"logits/chosen": 0.6703622341156006,
"logits/rejected": 0.6499079465866089,
"logps/chosen": -134.2932586669922,
"logps/ref_chosen": -57.927433013916016,
"logps/ref_rejected": -67.838623046875,
"logps/rejected": -189.4187469482422,
"loss": 1.1081,
"margin_dpo/margin_mean": 45.21430206298828,
"margin_dpo/margin_std": 66.5533218383789,
"step": 242
},
{
"KL/chosen_KL_mean": -75.65873718261719,
"KL/mean": -95.70713806152344,
"KL/rejected_KL_mean": -115.75553894042969,
"KL/std": 61.08662414550781,
"epoch": 0.3673469387755102,
"fcm_dpo/beta": 0.009353440254926682,
"fcm_dpo/delta": 0.03057572804391384,
"fcm_dpo/margin": 40.096805572509766,
"fcm_dpo/q_t": 0.41490113735198975,
"grad_norm": 16.917295455932617,
"learning_rate": 4.003481376353596e-07,
"logits/chosen": 0.6367839574813843,
"logits/rejected": 0.6507890820503235,
"logps/chosen": -149.93540954589844,
"logps/ref_chosen": -74.27667236328125,
"logps/ref_rejected": -73.24340057373047,
"logps/rejected": -188.99893188476562,
"loss": 1.1307,
"margin_dpo/margin_mean": 40.0968017578125,
"margin_dpo/margin_std": 63.99544906616211,
"step": 243
},
{
"KL/chosen_KL_mean": -68.39207458496094,
"KL/mean": -95.18928527832031,
"KL/rejected_KL_mean": -121.98651123046875,
"KL/std": 61.04261779785156,
"epoch": 0.3688586545729403,
"fcm_dpo/beta": 0.008910516276955605,
"fcm_dpo/delta": -0.10152090340852737,
"fcm_dpo/margin": 53.59442138671875,
"fcm_dpo/q_t": 0.38667529821395874,
"grad_norm": 11.596921920776367,
"learning_rate": 3.9928964792569654e-07,
"logits/chosen": 0.66209876537323,
"logits/rejected": 0.5738247632980347,
"logps/chosen": -121.7559814453125,
"logps/ref_chosen": -53.36390686035156,
"logps/ref_rejected": -71.10276794433594,
"logps/rejected": -193.0892791748047,
"loss": 1.0465,
"margin_dpo/margin_mean": 53.59442138671875,
"margin_dpo/margin_std": 64.69428253173828,
"step": 244
},
{
"KL/chosen_KL_mean": -77.90817260742188,
"KL/mean": -108.65547943115234,
"KL/rejected_KL_mean": -139.40280151367188,
"KL/std": 63.35409164428711,
"epoch": 0.37037037037037035,
"fcm_dpo/beta": 0.007787792012095451,
"fcm_dpo/delta": -0.10556904971599579,
"fcm_dpo/margin": 61.49463653564453,
"fcm_dpo/q_t": 0.3828907907009125,
"grad_norm": 12.549822807312012,
"learning_rate": 3.982269822636601e-07,
"logits/chosen": 0.6861937046051025,
"logits/rejected": 0.660259485244751,
"logps/chosen": -149.103271484375,
"logps/ref_chosen": -71.19510650634766,
"logps/ref_rejected": -80.76235961914062,
"logps/rejected": -220.1651611328125,
"loss": 1.0244,
"margin_dpo/margin_mean": 61.49463653564453,
"margin_dpo/margin_std": 65.15496063232422,
"step": 245
},
{
"KL/chosen_KL_mean": -92.9063720703125,
"KL/mean": -114.83541870117188,
"KL/rejected_KL_mean": -136.76446533203125,
"KL/std": 63.131744384765625,
"epoch": 0.37188208616780044,
"fcm_dpo/beta": 0.0081062326207757,
"fcm_dpo/delta": 0.050879672169685364,
"fcm_dpo/margin": 43.858089447021484,
"fcm_dpo/q_t": 0.42029231786727905,
"grad_norm": 11.98619556427002,
"learning_rate": 3.971601703742932e-07,
"logits/chosen": 0.7157027721405029,
"logits/rejected": 0.6501913070678711,
"logps/chosen": -164.5274200439453,
"logps/ref_chosen": -71.62104797363281,
"logps/ref_rejected": -94.03392028808594,
"logps/rejected": -230.7983856201172,
"loss": 1.1625,
"margin_dpo/margin_mean": 43.858089447021484,
"margin_dpo/margin_std": 80.09846496582031,
"step": 246
},
{
"KL/chosen_KL_mean": -96.20511627197266,
"KL/mean": -111.12162780761719,
"KL/rejected_KL_mean": -126.03813171386719,
"KL/std": 64.35607147216797,
"epoch": 0.37339380196523053,
"fcm_dpo/beta": 0.008263746276497841,
"fcm_dpo/delta": 0.05345098674297333,
"fcm_dpo/margin": 29.833005905151367,
"fcm_dpo/q_t": 0.4449678957462311,
"grad_norm": 15.636235237121582,
"learning_rate": 3.960892420986177e-07,
"logits/chosen": 0.7111119031906128,
"logits/rejected": 0.7009575366973877,
"logps/chosen": -176.2276611328125,
"logps/ref_chosen": -80.02254486083984,
"logps/ref_rejected": -89.22705841064453,
"logps/rejected": -215.26519775390625,
"loss": 1.2462,
"margin_dpo/margin_mean": 29.8330078125,
"margin_dpo/margin_std": 75.14302825927734,
"step": 247
},
{
"KL/chosen_KL_mean": -81.6597900390625,
"KL/mean": -106.406494140625,
"KL/rejected_KL_mean": -131.1531982421875,
"KL/std": 66.31968688964844,
"epoch": 0.3749055177626606,
"fcm_dpo/beta": 0.008432338945567608,
"fcm_dpo/delta": -0.02212671749293804,
"fcm_dpo/margin": 49.493408203125,
"fcm_dpo/q_t": 0.4049510359764099,
"grad_norm": 17.001266479492188,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 0.6953562498092651,
"logits/rejected": 0.7540360689163208,
"logps/chosen": -147.03775024414062,
"logps/ref_chosen": -65.37796020507812,
"logps/ref_rejected": -61.365787506103516,
"logps/rejected": -192.51898193359375,
"loss": 1.1098,
"margin_dpo/margin_mean": 49.493408203125,
"margin_dpo/margin_std": 76.3125,
"step": 248
},
{
"KL/chosen_KL_mean": -103.32197570800781,
"KL/mean": -111.87638854980469,
"KL/rejected_KL_mean": -120.43080139160156,
"KL/std": 62.08591079711914,
"epoch": 0.3764172335600907,
"fcm_dpo/beta": 0.008548267185688019,
"fcm_dpo/delta": 0.058366335928440094,
"fcm_dpo/margin": 17.108829498291016,
"fcm_dpo/q_t": 0.46602004766464233,
"grad_norm": 15.90197467803955,
"learning_rate": 3.9393515632731094e-07,
"logits/chosen": 0.6849374771118164,
"logits/rejected": 0.7285729646682739,
"logps/chosen": -177.92343139648438,
"logps/ref_chosen": -74.60145568847656,
"logps/ref_rejected": -63.79338455200195,
"logps/rejected": -184.22418212890625,
"loss": 1.3511,
"margin_dpo/margin_mean": 17.108827590942383,
"margin_dpo/margin_std": 76.58978271484375,
"step": 249
},
{
"KL/chosen_KL_mean": -90.82321166992188,
"KL/mean": -116.01731872558594,
"KL/rejected_KL_mean": -141.21141052246094,
"KL/std": 69.73461151123047,
"epoch": 0.3779289493575208,
"fcm_dpo/beta": 0.008761906996369362,
"fcm_dpo/delta": -0.056977760046720505,
"fcm_dpo/margin": 50.38819122314453,
"fcm_dpo/q_t": 0.3973717987537384,
"grad_norm": 15.959137916564941,
"learning_rate": 3.9285205908608934e-07,
"logits/chosen": 0.7684859037399292,
"logits/rejected": 0.7283642888069153,
"logps/chosen": -152.7614288330078,
"logps/ref_chosen": -61.938209533691406,
"logps/ref_rejected": -72.21602630615234,
"logps/rejected": -213.42742919921875,
"loss": 1.0926,
"margin_dpo/margin_mean": 50.38819122314453,
"margin_dpo/margin_std": 71.78437805175781,
"step": 250
},
{
"KL/chosen_KL_mean": -98.59217834472656,
"KL/mean": -115.84170532226562,
"KL/rejected_KL_mean": -133.0912322998047,
"KL/std": 61.36365509033203,
"epoch": 0.3794406651549509,
"fcm_dpo/beta": 0.008924310095608234,
"fcm_dpo/delta": 0.10951556265354156,
"fcm_dpo/margin": 34.499053955078125,
"fcm_dpo/q_t": 0.4327406883239746,
"grad_norm": 23.957935333251953,
"learning_rate": 3.9176496596569265e-07,
"logits/chosen": 0.7150743007659912,
"logits/rejected": 0.6731328964233398,
"logps/chosen": -165.44912719726562,
"logps/ref_chosen": -66.85694885253906,
"logps/ref_rejected": -84.83396911621094,
"logps/rejected": -217.92520141601562,
"loss": 1.1986,
"margin_dpo/margin_mean": 34.499053955078125,
"margin_dpo/margin_std": 70.83842468261719,
"step": 251
},
{
"KL/chosen_KL_mean": -93.67896270751953,
"KL/mean": -106.87741088867188,
"KL/rejected_KL_mean": -120.07587432861328,
"KL/std": 63.966712951660156,
"epoch": 0.38095238095238093,
"fcm_dpo/beta": 0.009195334278047085,
"fcm_dpo/delta": -0.016725320369005203,
"fcm_dpo/margin": 26.396902084350586,
"fcm_dpo/q_t": 0.4438965320587158,
"grad_norm": 21.82607078552246,
"learning_rate": 3.9067390737445254e-07,
"logits/chosen": 0.6887344121932983,
"logits/rejected": 0.6316691040992737,
"logps/chosen": -149.90289306640625,
"logps/ref_chosen": -56.22393035888672,
"logps/ref_rejected": -77.1136245727539,
"logps/rejected": -197.1894989013672,
"loss": 1.2812,
"margin_dpo/margin_mean": 26.396900177001953,
"margin_dpo/margin_std": 75.36380767822266,
"step": 252
},
{
"KL/chosen_KL_mean": -93.51123046875,
"KL/mean": -111.86512756347656,
"KL/rejected_KL_mean": -130.21900939941406,
"KL/std": 61.31932067871094,
"epoch": 0.382464096749811,
"fcm_dpo/beta": 0.008835416287183762,
"fcm_dpo/delta": -0.03153040632605553,
"fcm_dpo/margin": 36.70779800415039,
"fcm_dpo/q_t": 0.4239751100540161,
"grad_norm": 16.764225006103516,
"learning_rate": 3.8957891383162304e-07,
"logits/chosen": 0.7815088033676147,
"logits/rejected": 0.7435786724090576,
"logps/chosen": -145.7212371826172,
"logps/ref_chosen": -52.21001434326172,
"logps/ref_rejected": -58.75764846801758,
"logps/rejected": -188.97665405273438,
"loss": 1.1675,
"margin_dpo/margin_mean": 36.70779800415039,
"margin_dpo/margin_std": 63.84772491455078,
"step": 253
},
{
"KL/chosen_KL_mean": -101.70146179199219,
"KL/mean": -121.41376495361328,
"KL/rejected_KL_mean": -141.1260528564453,
"KL/std": 67.48922729492188,
"epoch": 0.3839758125472411,
"fcm_dpo/beta": 0.009037662297487259,
"fcm_dpo/delta": 0.052855074405670166,
"fcm_dpo/margin": 39.424591064453125,
"fcm_dpo/q_t": 0.42191147804260254,
"grad_norm": 13.55975341796875,
"learning_rate": 3.884800159665276e-07,
"logits/chosen": 0.6931102275848389,
"logits/rejected": 0.6379691958427429,
"logps/chosen": -167.33779907226562,
"logps/ref_chosen": -65.63632202148438,
"logps/ref_rejected": -82.34425354003906,
"logps/rejected": -223.47030639648438,
"loss": 1.1501,
"margin_dpo/margin_mean": 39.424591064453125,
"margin_dpo/margin_std": 67.18295288085938,
"step": 254
},
{
"KL/chosen_KL_mean": -92.79487609863281,
"KL/mean": -115.30006408691406,
"KL/rejected_KL_mean": -137.80523681640625,
"KL/std": 64.47406005859375,
"epoch": 0.3854875283446712,
"fcm_dpo/beta": 0.009151535108685493,
"fcm_dpo/delta": -0.015306653454899788,
"fcm_dpo/margin": 45.01036834716797,
"fcm_dpo/q_t": 0.4061132073402405,
"grad_norm": 19.183948516845703,
"learning_rate": 3.873772445177015e-07,
"logits/chosen": 0.6413042545318604,
"logits/rejected": 0.6087009906768799,
"logps/chosen": -160.70596313476562,
"logps/ref_chosen": -67.91108703613281,
"logps/ref_rejected": -83.89114379882812,
"logps/rejected": -221.69639587402344,
"loss": 1.1178,
"margin_dpo/margin_mean": 45.01036834716797,
"margin_dpo/margin_std": 72.30084228515625,
"step": 255
},
{
"KL/chosen_KL_mean": -110.49552917480469,
"KL/mean": -132.25213623046875,
"KL/rejected_KL_mean": -154.00872802734375,
"KL/std": 61.51966094970703,
"epoch": 0.3869992441421013,
"fcm_dpo/beta": 0.009058517403900623,
"fcm_dpo/delta": 0.006048116832971573,
"fcm_dpo/margin": 43.51321029663086,
"fcm_dpo/q_t": 0.4129956364631653,
"grad_norm": 16.235132217407227,
"learning_rate": 3.862706303320329e-07,
"logits/chosen": 0.7141859531402588,
"logits/rejected": 0.6413957476615906,
"logps/chosen": -173.99551391601562,
"logps/ref_chosen": -63.49998474121094,
"logps/ref_rejected": -90.77104187011719,
"logps/rejected": -244.77978515625,
"loss": 1.1552,
"margin_dpo/margin_mean": 43.513206481933594,
"margin_dpo/margin_std": 81.13447570800781,
"step": 256
},
{
"KL/chosen_KL_mean": -102.68710327148438,
"KL/mean": -131.61746215820312,
"KL/rejected_KL_mean": -160.54782104492188,
"KL/std": 69.95053100585938,
"epoch": 0.3885109599395314,
"fcm_dpo/beta": 0.008318210951983929,
"fcm_dpo/delta": -0.11226293444633484,
"fcm_dpo/margin": 57.86071014404297,
"fcm_dpo/q_t": 0.3880605101585388,
"grad_norm": 11.517922401428223,
"learning_rate": 3.851602043638994e-07,
"logits/chosen": 0.6858439445495605,
"logits/rejected": 0.6109893321990967,
"logps/chosen": -173.28775024414062,
"logps/ref_chosen": -70.60064697265625,
"logps/ref_rejected": -108.58313751220703,
"logps/rejected": -269.1309509277344,
"loss": 1.062,
"margin_dpo/margin_mean": 57.86071014404297,
"margin_dpo/margin_std": 80.06947326660156,
"step": 257
},
{
"KL/chosen_KL_mean": -106.1284408569336,
"KL/mean": -127.41146087646484,
"KL/rejected_KL_mean": -148.6944580078125,
"KL/std": 61.25123596191406,
"epoch": 0.3900226757369615,
"fcm_dpo/beta": 0.008503508754074574,
"fcm_dpo/delta": 0.046553075313568115,
"fcm_dpo/margin": 42.5660285949707,
"fcm_dpo/q_t": 0.4167933762073517,
"grad_norm": 14.567258834838867,
"learning_rate": 3.840459976743023e-07,
"logits/chosen": 0.7221835255622864,
"logits/rejected": 0.6666927337646484,
"logps/chosen": -165.38259887695312,
"logps/ref_chosen": -59.25416564941406,
"logps/ref_rejected": -85.58709716796875,
"logps/rejected": -234.28155517578125,
"loss": 1.1265,
"margin_dpo/margin_mean": 42.56603240966797,
"margin_dpo/margin_std": 63.77388000488281,
"step": 258
},
{
"KL/chosen_KL_mean": -93.2396240234375,
"KL/mean": -124.42048645019531,
"KL/rejected_KL_mean": -155.60134887695312,
"KL/std": 67.78335571289062,
"epoch": 0.3915343915343915,
"fcm_dpo/beta": 0.007766705006361008,
"fcm_dpo/delta": -0.11655537784099579,
"fcm_dpo/margin": 62.36172103881836,
"fcm_dpo/q_t": 0.3838648796081543,
"grad_norm": 12.29951286315918,
"learning_rate": 3.8292804142999796e-07,
"logits/chosen": 0.6192357540130615,
"logits/rejected": 0.5121684074401855,
"logps/chosen": -158.67449951171875,
"logps/ref_chosen": -65.43487548828125,
"logps/ref_rejected": -95.41731262207031,
"logps/rejected": -251.01866149902344,
"loss": 1.0588,
"margin_dpo/margin_mean": 62.36172103881836,
"margin_dpo/margin_std": 83.14129638671875,
"step": 259
},
{
"KL/chosen_KL_mean": -93.89341735839844,
"KL/mean": -117.76322937011719,
"KL/rejected_KL_mean": -141.633056640625,
"KL/std": 62.05198669433594,
"epoch": 0.3930461073318216,
"fcm_dpo/beta": 0.00776186678558588,
"fcm_dpo/delta": 0.03546956554055214,
"fcm_dpo/margin": 47.73963928222656,
"fcm_dpo/q_t": 0.4163569509983063,
"grad_norm": 17.107166290283203,
"learning_rate": 3.818063669026256e-07,
"logits/chosen": 0.7102089524269104,
"logits/rejected": 0.6064608097076416,
"logps/chosen": -142.98300170898438,
"logps/ref_chosen": -49.08958435058594,
"logps/ref_rejected": -79.01708221435547,
"logps/rejected": -220.65013122558594,
"loss": 1.1402,
"margin_dpo/margin_mean": 47.73963928222656,
"margin_dpo/margin_std": 78.65926361083984,
"step": 260
},
{
"KL/chosen_KL_mean": -93.87600708007812,
"KL/mean": -114.30181884765625,
"KL/rejected_KL_mean": -134.7276611328125,
"KL/std": 60.913230895996094,
"epoch": 0.3945578231292517,
"fcm_dpo/beta": 0.008519364520907402,
"fcm_dpo/delta": 0.058973412960767746,
"fcm_dpo/margin": 40.85166931152344,
"fcm_dpo/q_t": 0.422269344329834,
"grad_norm": 16.87601661682129,
"learning_rate": 3.806810054678331e-07,
"logits/chosen": 0.5581406354904175,
"logits/rejected": 0.6009823679924011,
"logps/chosen": -164.74839782714844,
"logps/ref_chosen": -70.87239074707031,
"logps/ref_rejected": -65.01522064208984,
"logps/rejected": -199.74288940429688,
"loss": 1.1551,
"margin_dpo/margin_mean": 40.85166931152344,
"margin_dpo/margin_std": 70.33143615722656,
"step": 261
},
{
"KL/chosen_KL_mean": -101.00912475585938,
"KL/mean": -121.16682434082031,
"KL/rejected_KL_mean": -141.32452392578125,
"KL/std": 61.25640869140625,
"epoch": 0.3960695389266818,
"fcm_dpo/beta": 0.008892569690942764,
"fcm_dpo/delta": 0.048768118023872375,
"fcm_dpo/margin": 40.31542205810547,
"fcm_dpo/q_t": 0.41975611448287964,
"grad_norm": 19.799497604370117,
"learning_rate": 3.7955198860439887e-07,
"logits/chosen": 0.7563103437423706,
"logits/rejected": 0.684789776802063,
"logps/chosen": -168.87974548339844,
"logps/ref_chosen": -67.8706283569336,
"logps/ref_rejected": -88.7205810546875,
"logps/rejected": -230.04510498046875,
"loss": 1.148,
"margin_dpo/margin_mean": 40.31542205810547,
"margin_dpo/margin_std": 68.66963195800781,
"step": 262
},
{
"KL/chosen_KL_mean": -95.22956085205078,
"KL/mean": -116.42835998535156,
"KL/rejected_KL_mean": -137.62718200683594,
"KL/std": 62.601688385009766,
"epoch": 0.3975812547241119,
"fcm_dpo/beta": 0.00906696729362011,
"fcm_dpo/delta": 0.01888187974691391,
"fcm_dpo/margin": 42.397621154785156,
"fcm_dpo/q_t": 0.41246914863586426,
"grad_norm": 16.852615356445312,
"learning_rate": 3.784193478933516e-07,
"logits/chosen": 0.6627376675605774,
"logits/rejected": 0.5485380291938782,
"logps/chosen": -150.42413330078125,
"logps/ref_chosen": -55.194583892822266,
"logps/ref_rejected": -80.54048156738281,
"logps/rejected": -218.16766357421875,
"loss": 1.1307,
"margin_dpo/margin_mean": 42.397621154785156,
"margin_dpo/margin_std": 68.78619384765625,
"step": 263
},
{
"KL/chosen_KL_mean": -97.0634765625,
"KL/mean": -120.03509521484375,
"KL/rejected_KL_mean": -143.00674438476562,
"KL/std": 69.64917755126953,
"epoch": 0.39909297052154197,
"fcm_dpo/beta": 0.008921649307012558,
"fcm_dpo/delta": -0.012593775056302547,
"fcm_dpo/margin": 45.94325637817383,
"fcm_dpo/q_t": 0.40747833251953125,
"grad_norm": 15.435047149658203,
"learning_rate": 3.7728311501708674e-07,
"logits/chosen": 0.5868898630142212,
"logits/rejected": 0.5499594211578369,
"logps/chosen": -180.23416137695312,
"logps/ref_chosen": -83.17068481445312,
"logps/ref_rejected": -88.33625793457031,
"logps/rejected": -231.34298706054688,
"loss": 1.1283,
"margin_dpo/margin_mean": 45.94325256347656,
"margin_dpo/margin_std": 76.74105072021484,
"step": 264
},
{
"KL/chosen_KL_mean": -92.12577819824219,
"KL/mean": -115.92205810546875,
"KL/rejected_KL_mean": -139.7183380126953,
"KL/std": 64.89045715332031,
"epoch": 0.40060468631897206,
"fcm_dpo/beta": 0.008840169757604599,
"fcm_dpo/delta": -0.02635415829718113,
"fcm_dpo/margin": 47.59254837036133,
"fcm_dpo/q_t": 0.404721736907959,
"grad_norm": 12.38251781463623,
"learning_rate": 3.7614332175848027e-07,
"logits/chosen": 0.7509992122650146,
"logits/rejected": 0.6839336156845093,
"logps/chosen": -143.78863525390625,
"logps/ref_chosen": -51.66284942626953,
"logps/ref_rejected": -67.1720962524414,
"logps/rejected": -206.89044189453125,
"loss": 1.1367,
"margin_dpo/margin_mean": 47.59254455566406,
"margin_dpo/margin_std": 83.42626953125,
"step": 265
},
{
"KL/chosen_KL_mean": -87.49989318847656,
"KL/mean": -111.67654418945312,
"KL/rejected_KL_mean": -135.8531951904297,
"KL/std": 65.83747863769531,
"epoch": 0.4021164021164021,
"fcm_dpo/beta": 0.008526175282895565,
"fcm_dpo/delta": -0.015927668660879135,
"fcm_dpo/margin": 48.353309631347656,
"fcm_dpo/q_t": 0.40604427456855774,
"grad_norm": 22.090471267700195,
"learning_rate": 3.75e-07,
"logits/chosen": 0.6847240924835205,
"logits/rejected": 0.6065027713775635,
"logps/chosen": -144.9503936767578,
"logps/ref_chosen": -57.45049285888672,
"logps/ref_rejected": -77.60826110839844,
"logps/rejected": -213.46145629882812,
"loss": 1.1181,
"margin_dpo/margin_mean": 48.353309631347656,
"margin_dpo/margin_std": 77.61994934082031,
"step": 266
},
{
"KL/chosen_KL_mean": -80.65776824951172,
"KL/mean": -99.42727661132812,
"KL/rejected_KL_mean": -118.19677734375,
"KL/std": 63.5875358581543,
"epoch": 0.4036281179138322,
"fcm_dpo/beta": 0.008297070860862732,
"fcm_dpo/delta": -0.06384800374507904,
"fcm_dpo/margin": 37.53900909423828,
"fcm_dpo/q_t": 0.4270240068435669,
"grad_norm": 15.486305236816406,
"learning_rate": 3.738531817228131e-07,
"logits/chosen": 0.7166332006454468,
"logits/rejected": 0.697598934173584,
"logps/chosen": -135.693115234375,
"logps/ref_chosen": -55.03535079956055,
"logps/ref_rejected": -66.0953369140625,
"logps/rejected": -184.2921142578125,
"loss": 1.2079,
"margin_dpo/margin_mean": 37.539005279541016,
"margin_dpo/margin_std": 74.69694519042969,
"step": 267
},
{
"KL/chosen_KL_mean": -77.2738037109375,
"KL/mean": -101.48638153076172,
"KL/rejected_KL_mean": -125.69896697998047,
"KL/std": 66.79031372070312,
"epoch": 0.4051398337112623,
"fcm_dpo/beta": 0.008188547566533089,
"fcm_dpo/delta": 0.0029786527156829834,
"fcm_dpo/margin": 48.42516326904297,
"fcm_dpo/q_t": 0.4089292287826538,
"grad_norm": 14.64191722869873,
"learning_rate": 3.7270289900589204e-07,
"logits/chosen": 0.5708452463150024,
"logits/rejected": 0.5601274371147156,
"logps/chosen": -142.34555053710938,
"logps/ref_chosen": -65.07174682617188,
"logps/ref_rejected": -71.42485809326172,
"logps/rejected": -197.1238250732422,
"loss": 1.0983,
"margin_dpo/margin_mean": 48.42516326904297,
"margin_dpo/margin_std": 65.15290832519531,
"step": 268
},
{
"KL/chosen_KL_mean": -91.49454498291016,
"KL/mean": -116.61368560791016,
"KL/rejected_KL_mean": -141.73281860351562,
"KL/std": 70.12186431884766,
"epoch": 0.40665154950869237,
"fcm_dpo/beta": 0.008165009319782257,
"fcm_dpo/delta": -0.015037402510643005,
"fcm_dpo/margin": 50.23826599121094,
"fcm_dpo/q_t": 0.40580272674560547,
"grad_norm": 14.160943031311035,
"learning_rate": 3.7154918402511714e-07,
"logits/chosen": 0.7557035088539124,
"logits/rejected": 0.7087898254394531,
"logps/chosen": -158.63075256347656,
"logps/ref_chosen": -67.1362075805664,
"logps/ref_rejected": -82.55778503417969,
"logps/rejected": -224.2906036376953,
"loss": 1.0967,
"margin_dpo/margin_mean": 50.2382698059082,
"margin_dpo/margin_std": 69.37471771240234,
"step": 269
},
{
"KL/chosen_KL_mean": -90.2132568359375,
"KL/mean": -112.82662200927734,
"KL/rejected_KL_mean": -135.4399871826172,
"KL/std": 68.86807250976562,
"epoch": 0.40816326530612246,
"fcm_dpo/beta": 0.00815459992736578,
"fcm_dpo/delta": 0.0382687933743,
"fcm_dpo/margin": 45.226722717285156,
"fcm_dpo/q_t": 0.41671714186668396,
"grad_norm": 15.69405460357666,
"learning_rate": 3.7039206905237656e-07,
"logits/chosen": 0.7264769673347473,
"logits/rejected": 0.6401715278625488,
"logps/chosen": -156.90194702148438,
"logps/ref_chosen": -66.6886978149414,
"logps/ref_rejected": -85.16129302978516,
"logps/rejected": -220.60128784179688,
"loss": 1.1389,
"margin_dpo/margin_mean": 45.22671890258789,
"margin_dpo/margin_std": 75.06895446777344,
"step": 270
},
{
"KL/chosen_KL_mean": -92.40959167480469,
"KL/mean": -108.70755004882812,
"KL/rejected_KL_mean": -125.00550842285156,
"KL/std": 74.48558044433594,
"epoch": 0.40967498110355255,
"fcm_dpo/beta": 0.008555498905479908,
"fcm_dpo/delta": 0.03689521178603172,
"fcm_dpo/margin": 32.59592056274414,
"fcm_dpo/q_t": 0.44212037324905396,
"grad_norm": 18.4699649810791,
"learning_rate": 3.692315864546635e-07,
"logits/chosen": 0.7211564779281616,
"logits/rejected": 0.6540436744689941,
"logps/chosen": -164.817138671875,
"logps/ref_chosen": -72.40754699707031,
"logps/ref_rejected": -92.06311798095703,
"logps/rejected": -217.06863403320312,
"loss": 1.2611,
"margin_dpo/margin_mean": 32.595916748046875,
"margin_dpo/margin_std": 88.71417236328125,
"step": 271
},
{
"KL/chosen_KL_mean": -76.29147338867188,
"KL/mean": -107.88369750976562,
"KL/rejected_KL_mean": -139.47592163085938,
"KL/std": 63.384490966796875,
"epoch": 0.41118669690098264,
"fcm_dpo/beta": 0.007848805747926235,
"fcm_dpo/delta": -0.127114936709404,
"fcm_dpo/margin": 63.18445587158203,
"fcm_dpo/q_t": 0.3781195282936096,
"grad_norm": 15.043636322021484,
"learning_rate": 3.6806776869317067e-07,
"logits/chosen": 0.6831435561180115,
"logits/rejected": 0.7051393985748291,
"logps/chosen": -142.8928680419922,
"logps/ref_chosen": -66.60140228271484,
"logps/ref_rejected": -67.74340057373047,
"logps/rejected": -207.2193145751953,
"loss": 1.0052,
"margin_dpo/margin_mean": 63.184452056884766,
"margin_dpo/margin_std": 60.51321029663086,
"step": 272
},
{
"KL/chosen_KL_mean": -97.41845703125,
"KL/mean": -120.54800415039062,
"KL/rejected_KL_mean": -143.67755126953125,
"KL/std": 70.76217651367188,
"epoch": 0.4126984126984127,
"fcm_dpo/beta": 0.007727320306003094,
"fcm_dpo/delta": 0.05174464359879494,
"fcm_dpo/margin": 46.25908279418945,
"fcm_dpo/q_t": 0.42130160331726074,
"grad_norm": 19.00057601928711,
"learning_rate": 3.669006483223828e-07,
"logits/chosen": 0.6799752712249756,
"logits/rejected": 0.6046786904335022,
"logps/chosen": -154.77334594726562,
"logps/ref_chosen": -57.35487747192383,
"logps/ref_rejected": -84.17168426513672,
"logps/rejected": -227.84922790527344,
"loss": 1.1757,
"margin_dpo/margin_mean": 46.25908279418945,
"margin_dpo/margin_std": 89.17124938964844,
"step": 273
},
{
"KL/chosen_KL_mean": -96.37403106689453,
"KL/mean": -121.2509765625,
"KL/rejected_KL_mean": -146.1279296875,
"KL/std": 67.85552978515625,
"epoch": 0.41421012849584277,
"fcm_dpo/beta": 0.008015267550945282,
"fcm_dpo/delta": 0.001110810786485672,
"fcm_dpo/margin": 49.753902435302734,
"fcm_dpo/q_t": 0.40851789712905884,
"grad_norm": 13.290173530578613,
"learning_rate": 3.657302579891656e-07,
"logits/chosen": 0.5827606320381165,
"logits/rejected": 0.5658946633338928,
"logps/chosen": -156.01553344726562,
"logps/ref_chosen": -59.64149475097656,
"logps/ref_rejected": -68.29348754882812,
"logps/rejected": -214.42141723632812,
"loss": 1.1243,
"margin_dpo/margin_mean": 49.75389862060547,
"margin_dpo/margin_std": 80.14501190185547,
"step": 274
},
{
"KL/chosen_KL_mean": -93.57954406738281,
"KL/mean": -118.13485717773438,
"KL/rejected_KL_mean": -142.69017028808594,
"KL/std": 64.97964477539062,
"epoch": 0.41572184429327286,
"fcm_dpo/beta": 0.008048377931118011,
"fcm_dpo/delta": 0.005478540435433388,
"fcm_dpo/margin": 49.110626220703125,
"fcm_dpo/q_t": 0.4092429578304291,
"grad_norm": 16.686729431152344,
"learning_rate": 3.645566304318526e-07,
"logits/chosen": 0.666012167930603,
"logits/rejected": 0.5821614265441895,
"logps/chosen": -146.84619140625,
"logps/ref_chosen": -53.26664352416992,
"logps/ref_rejected": -73.84062194824219,
"logps/rejected": -216.53079223632812,
"loss": 1.0982,
"margin_dpo/margin_mean": 49.110626220703125,
"margin_dpo/margin_std": 66.99764251708984,
"step": 275
},
{
"KL/chosen_KL_mean": -87.14787292480469,
"KL/mean": -111.3336181640625,
"KL/rejected_KL_mean": -135.51934814453125,
"KL/std": 59.877315521240234,
"epoch": 0.41723356009070295,
"fcm_dpo/beta": 0.008093775250017643,
"fcm_dpo/delta": 0.010346898809075356,
"fcm_dpo/margin": 48.37148666381836,
"fcm_dpo/q_t": 0.40958449244499207,
"grad_norm": 18.336421966552734,
"learning_rate": 3.633797984793294e-07,
"logits/chosen": 0.6678320169448853,
"logits/rejected": 0.6339179277420044,
"logps/chosen": -140.16867065429688,
"logps/ref_chosen": -53.02079772949219,
"logps/ref_rejected": -61.56678771972656,
"logps/rejected": -197.08615112304688,
"loss": 1.1051,
"margin_dpo/margin_mean": 48.37148666381836,
"margin_dpo/margin_std": 68.28596496582031,
"step": 276
},
{
"KL/chosen_KL_mean": -101.01702880859375,
"KL/mean": -112.96041870117188,
"KL/rejected_KL_mean": -124.90379333496094,
"KL/std": 69.72208404541016,
"epoch": 0.41874527588813304,
"fcm_dpo/beta": 0.008755484595894814,
"fcm_dpo/delta": 0.08153992146253586,
"fcm_dpo/margin": 23.88676643371582,
"fcm_dpo/q_t": 0.45458197593688965,
"grad_norm": 21.524913787841797,
"learning_rate": 3.6219979505011555e-07,
"logits/chosen": 0.7365692853927612,
"logits/rejected": 0.7629575133323669,
"logps/chosen": -172.45001220703125,
"logps/ref_chosen": -71.43299102783203,
"logps/ref_rejected": -67.65852355957031,
"logps/rejected": -192.56231689453125,
"loss": 1.2996,
"margin_dpo/margin_mean": 23.886764526367188,
"margin_dpo/margin_std": 77.9295654296875,
"step": 277
},
{
"KL/chosen_KL_mean": -107.86801147460938,
"KL/mean": -131.03553771972656,
"KL/rejected_KL_mean": -154.20306396484375,
"KL/std": 71.07804870605469,
"epoch": 0.42025699168556313,
"fcm_dpo/beta": 0.008978527039289474,
"fcm_dpo/delta": -0.027673780918121338,
"fcm_dpo/margin": 46.335052490234375,
"fcm_dpo/q_t": 0.4027869999408722,
"grad_norm": 23.630117416381836,
"learning_rate": 3.6101665315144353e-07,
"logits/chosen": 0.651962399482727,
"logits/rejected": 0.5864707827568054,
"logps/chosen": -174.97877502441406,
"logps/ref_chosen": -67.11076354980469,
"logps/ref_rejected": -88.74851989746094,
"logps/rejected": -242.9515838623047,
"loss": 1.1303,
"margin_dpo/margin_mean": 46.33504867553711,
"margin_dpo/margin_std": 74.0306396484375,
"step": 278
},
{
"KL/chosen_KL_mean": -84.87873840332031,
"KL/mean": -117.05733489990234,
"KL/rejected_KL_mean": -149.23593139648438,
"KL/std": 68.21269226074219,
"epoch": 0.4217687074829932,
"fcm_dpo/beta": 0.007804807275533676,
"fcm_dpo/delta": -0.13575728237628937,
"fcm_dpo/margin": 64.35718536376953,
"fcm_dpo/q_t": 0.3770613670349121,
"grad_norm": 15.381987571716309,
"learning_rate": 3.5983040587833563e-07,
"logits/chosen": 0.6404179334640503,
"logits/rejected": 0.6012611389160156,
"logps/chosen": -139.376220703125,
"logps/ref_chosen": -54.49748611450195,
"logps/ref_rejected": -70.42373657226562,
"logps/rejected": -219.65966796875,
"loss": 1.0157,
"margin_dpo/margin_mean": 64.35718536376953,
"margin_dpo/margin_std": 67.33639526367188,
"step": 279
},
{
"KL/chosen_KL_mean": -91.69462585449219,
"KL/mean": -125.66950225830078,
"KL/rejected_KL_mean": -159.64437866210938,
"KL/std": 71.94490051269531,
"epoch": 0.42328042328042326,
"fcm_dpo/beta": 0.006942619103938341,
"fcm_dpo/delta": -0.09404292702674866,
"fcm_dpo/margin": 67.94975280761719,
"fcm_dpo/q_t": 0.385819673538208,
"grad_norm": 10.739294052124023,
"learning_rate": 3.586410864126781e-07,
"logits/chosen": 0.7250418663024902,
"logits/rejected": 0.6764532327651978,
"logps/chosen": -152.12744140625,
"logps/ref_chosen": -60.43281173706055,
"logps/ref_rejected": -78.39051818847656,
"logps/rejected": -238.034912109375,
"loss": 1.0215,
"margin_dpo/margin_mean": 67.94975280761719,
"margin_dpo/margin_std": 68.12228393554688,
"step": 280
},
{
"KL/chosen_KL_mean": -94.15901184082031,
"KL/mean": -121.11312866210938,
"KL/rejected_KL_mean": -148.06723022460938,
"KL/std": 70.25721740722656,
"epoch": 0.42479213907785335,
"fcm_dpo/beta": 0.007024562917649746,
"fcm_dpo/delta": 0.02409074455499649,
"fcm_dpo/margin": 53.908233642578125,
"fcm_dpo/q_t": 0.4124564230442047,
"grad_norm": 12.674901008605957,
"learning_rate": 3.574487280222929e-07,
"logits/chosen": 0.6601166725158691,
"logits/rejected": 0.6880499124526978,
"logps/chosen": -154.44110107421875,
"logps/ref_chosen": -60.2820930480957,
"logps/ref_rejected": -62.04009246826172,
"logps/rejected": -210.10733032226562,
"loss": 1.1157,
"margin_dpo/margin_mean": 53.90823745727539,
"margin_dpo/margin_std": 77.37689208984375,
"step": 281
},
{
"KL/chosen_KL_mean": -102.38758087158203,
"KL/mean": -126.72209167480469,
"KL/rejected_KL_mean": -151.05661010742188,
"KL/std": 70.62893676757812,
"epoch": 0.42630385487528344,
"fcm_dpo/beta": 0.006782787851989269,
"fcm_dpo/delta": -0.04430830478668213,
"fcm_dpo/margin": 48.669010162353516,
"fcm_dpo/q_t": 0.4228168725967407,
"grad_norm": 15.216257095336914,
"learning_rate": 3.562533640600075e-07,
"logits/chosen": 0.6253547668457031,
"logits/rejected": 0.5807002782821655,
"logps/chosen": -163.01150512695312,
"logps/ref_chosen": -60.623924255371094,
"logps/ref_rejected": -68.67400360107422,
"logps/rejected": -219.73060607910156,
"loss": 1.1665,
"margin_dpo/margin_mean": 48.66900634765625,
"margin_dpo/margin_std": 82.96221923828125,
"step": 282
},
{
"KL/chosen_KL_mean": -99.55526733398438,
"KL/mean": -125.99640655517578,
"KL/rejected_KL_mean": -152.4375457763672,
"KL/std": 74.96566772460938,
"epoch": 0.42781557067271353,
"fcm_dpo/beta": 0.006863541901111603,
"fcm_dpo/delta": 0.04517936706542969,
"fcm_dpo/margin": 52.88227844238281,
"fcm_dpo/q_t": 0.41826343536376953,
"grad_norm": 13.062129974365234,
"learning_rate": 3.550550279627215e-07,
"logits/chosen": 0.6494210362434387,
"logits/rejected": 0.5451687574386597,
"logps/chosen": -167.20301818847656,
"logps/ref_chosen": -67.64775085449219,
"logps/ref_rejected": -99.96835327148438,
"logps/rejected": -252.40589904785156,
"loss": 1.1343,
"margin_dpo/margin_mean": 52.88227081298828,
"margin_dpo/margin_std": 83.99928283691406,
"step": 283
},
{
"KL/chosen_KL_mean": -95.5714111328125,
"KL/mean": -120.84217071533203,
"KL/rejected_KL_mean": -146.11294555664062,
"KL/std": 73.72892761230469,
"epoch": 0.4293272864701436,
"fcm_dpo/beta": 0.007086652331054211,
"fcm_dpo/delta": 0.050613950937986374,
"fcm_dpo/margin": 50.541542053222656,
"fcm_dpo/q_t": 0.41856300830841064,
"grad_norm": 12.229044914245605,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 0.7038174867630005,
"logits/rejected": 0.6362642049789429,
"logps/chosen": -152.53883361816406,
"logps/ref_chosen": -56.96742630004883,
"logps/ref_rejected": -86.36236572265625,
"logps/rejected": -232.47531127929688,
"loss": 1.124,
"margin_dpo/margin_mean": 50.541542053222656,
"margin_dpo/margin_std": 71.2654037475586,
"step": 284
},
{
"KL/chosen_KL_mean": -109.09988403320312,
"KL/mean": -130.37918090820312,
"KL/rejected_KL_mean": -151.65847778320312,
"KL/std": 67.08362579345703,
"epoch": 0.4308390022675737,
"fcm_dpo/beta": 0.007905875332653522,
"fcm_dpo/delta": 0.07085709273815155,
"fcm_dpo/margin": 42.5586051940918,
"fcm_dpo/q_t": 0.4255504310131073,
"grad_norm": 19.009510040283203,
"learning_rate": 3.5264957352549375e-07,
"logits/chosen": 0.6826480031013489,
"logits/rejected": 0.6553780436515808,
"logps/chosen": -180.7559814453125,
"logps/ref_chosen": -71.65611267089844,
"logps/ref_rejected": -81.63829803466797,
"logps/rejected": -233.29678344726562,
"loss": 1.1658,
"margin_dpo/margin_mean": 42.55860900878906,
"margin_dpo/margin_std": 75.93214416503906,
"step": 285
},
{
"KL/chosen_KL_mean": -104.54328918457031,
"KL/mean": -139.43801879882812,
"KL/rejected_KL_mean": -174.332763671875,
"KL/std": 73.45622253417969,
"epoch": 0.4323507180650038,
"fcm_dpo/beta": 0.007104329764842987,
"fcm_dpo/delta": -0.12842434644699097,
"fcm_dpo/margin": 69.78947448730469,
"fcm_dpo/q_t": 0.380862295627594,
"grad_norm": 11.564325332641602,
"learning_rate": 3.514425224712835e-07,
"logits/chosen": 0.6346836090087891,
"logits/rejected": 0.5405333042144775,
"logps/chosen": -165.622802734375,
"logps/ref_chosen": -61.07952117919922,
"logps/ref_rejected": -91.28128051757812,
"logps/rejected": -265.6140441894531,
"loss": 1.0207,
"margin_dpo/margin_mean": 69.78947448730469,
"margin_dpo/margin_std": 78.1060791015625,
"step": 286
},
{
"KL/chosen_KL_mean": -86.993896484375,
"KL/mean": -119.77565002441406,
"KL/rejected_KL_mean": -152.55740356445312,
"KL/std": 72.32278442382812,
"epoch": 0.43386243386243384,
"fcm_dpo/beta": 0.00666564516723156,
"fcm_dpo/delta": -0.04734319448471069,
"fcm_dpo/margin": 65.56352233886719,
"fcm_dpo/q_t": 0.3966023027896881,
"grad_norm": 10.716714859008789,
"learning_rate": 3.502326338516534e-07,
"logits/chosen": 0.7223142385482788,
"logits/rejected": 0.6803905963897705,
"logps/chosen": -133.02969360351562,
"logps/ref_chosen": -46.035789489746094,
"logps/ref_rejected": -59.95293426513672,
"logps/rejected": -212.51034545898438,
"loss": 1.0565,
"margin_dpo/margin_mean": 65.56352233886719,
"margin_dpo/margin_std": 75.86518859863281,
"step": 287
},
{
"KL/chosen_KL_mean": -109.24652099609375,
"KL/mean": -134.2801055908203,
"KL/rejected_KL_mean": -159.31369018554688,
"KL/std": 70.14065551757812,
"epoch": 0.43537414965986393,
"fcm_dpo/beta": 0.006941050291061401,
"fcm_dpo/delta": 0.06258052587509155,
"fcm_dpo/margin": 50.067169189453125,
"fcm_dpo/q_t": 0.42138350009918213,
"grad_norm": 13.64344310760498,
"learning_rate": 3.490199415097892e-07,
"logits/chosen": 0.544763445854187,
"logits/rejected": 0.48616740107536316,
"logps/chosen": -174.63735961914062,
"logps/ref_chosen": -65.3908462524414,
"logps/ref_rejected": -88.53607940673828,
"logps/rejected": -247.84976196289062,
"loss": 1.135,
"margin_dpo/margin_mean": 50.067169189453125,
"margin_dpo/margin_std": 76.1004867553711,
"step": 288
},
{
"KL/chosen_KL_mean": -110.31666564941406,
"KL/mean": -134.42742919921875,
"KL/rejected_KL_mean": -158.5382080078125,
"KL/std": 71.93513488769531,
"epoch": 0.436885865457294,
"fcm_dpo/beta": 0.007375278510153294,
"fcm_dpo/delta": 0.05219840630888939,
"fcm_dpo/margin": 48.22156524658203,
"fcm_dpo/q_t": 0.42122745513916016,
"grad_norm": 12.05275821685791,
"learning_rate": 3.4780447936730247e-07,
"logits/chosen": 0.7490577101707458,
"logits/rejected": 0.7129071950912476,
"logps/chosen": -164.9102783203125,
"logps/ref_chosen": -54.5936279296875,
"logps/ref_rejected": -67.20855712890625,
"logps/rejected": -225.74676513671875,
"loss": 1.1474,
"margin_dpo/margin_mean": 48.22156524658203,
"margin_dpo/margin_std": 80.87078857421875,
"step": 289
},
{
"KL/chosen_KL_mean": -123.2154541015625,
"KL/mean": -150.1029815673828,
"KL/rejected_KL_mean": -176.99050903320312,
"KL/std": 71.00227355957031,
"epoch": 0.4383975812547241,
"fcm_dpo/beta": 0.007250770926475525,
"fcm_dpo/delta": 0.010142248123884201,
"fcm_dpo/margin": 53.775054931640625,
"fcm_dpo/q_t": 0.41072556376457214,
"grad_norm": 14.948026657104492,
"learning_rate": 3.465862814232821e-07,
"logits/chosen": 0.7731099128723145,
"logits/rejected": 0.6953067779541016,
"logps/chosen": -184.60003662109375,
"logps/ref_chosen": -61.38457489013672,
"logps/ref_rejected": -91.92778015136719,
"logps/rejected": -268.91827392578125,
"loss": 1.1165,
"margin_dpo/margin_mean": 53.775054931640625,
"margin_dpo/margin_std": 80.15385437011719,
"step": 290
},
{
"KL/chosen_KL_mean": -111.75973510742188,
"KL/mean": -145.4049530029297,
"KL/rejected_KL_mean": -179.0501708984375,
"KL/std": 73.63685607910156,
"epoch": 0.4399092970521542,
"fcm_dpo/beta": 0.0071503822691738605,
"fcm_dpo/delta": -0.10867234319448471,
"fcm_dpo/margin": 67.2904281616211,
"fcm_dpo/q_t": 0.38891780376434326,
"grad_norm": 14.165699005126953,
"learning_rate": 3.4536538175334343e-07,
"logits/chosen": 0.8502572774887085,
"logits/rejected": 0.7742403745651245,
"logps/chosen": -162.62277221679688,
"logps/ref_chosen": -50.863037109375,
"logps/ref_rejected": -82.20868682861328,
"logps/rejected": -261.25885009765625,
"loss": 1.059,
"margin_dpo/margin_mean": 67.2904281616211,
"margin_dpo/margin_std": 87.29713439941406,
"step": 291
},
{
"KL/chosen_KL_mean": -114.97377014160156,
"KL/mean": -137.51162719726562,
"KL/rejected_KL_mean": -160.04949951171875,
"KL/std": 76.66767883300781,
"epoch": 0.4414210128495843,
"fcm_dpo/beta": 0.007128318306058645,
"fcm_dpo/delta": 0.0940467119216919,
"fcm_dpo/margin": 45.07574462890625,
"fcm_dpo/q_t": 0.43068501353263855,
"grad_norm": 15.699105262756348,
"learning_rate": 3.4414181450867465e-07,
"logits/chosen": 0.6994675397872925,
"logits/rejected": 0.6498559713363647,
"logps/chosen": -179.32266235351562,
"logps/ref_chosen": -64.34888458251953,
"logps/ref_rejected": -72.86434173583984,
"logps/rejected": -232.91384887695312,
"loss": 1.1944,
"margin_dpo/margin_mean": 45.075748443603516,
"margin_dpo/margin_std": 93.48008728027344,
"step": 292
},
{
"KL/chosen_KL_mean": -109.03053283691406,
"KL/mean": -144.01535034179688,
"KL/rejected_KL_mean": -179.00018310546875,
"KL/std": 77.45966339111328,
"epoch": 0.4429327286470144,
"fcm_dpo/beta": 0.006905644200742245,
"fcm_dpo/delta": -0.10910916328430176,
"fcm_dpo/margin": 69.96965789794922,
"fcm_dpo/q_t": 0.3872179388999939,
"grad_norm": 10.040387153625488,
"learning_rate": 3.4291561391508185e-07,
"logits/chosen": 0.7848567962646484,
"logits/rejected": 0.6968656778335571,
"logps/chosen": -163.89999389648438,
"logps/ref_chosen": -54.869468688964844,
"logps/ref_rejected": -81.858642578125,
"logps/rejected": -260.85882568359375,
"loss": 1.0643,
"margin_dpo/margin_mean": 69.96965789794922,
"margin_dpo/margin_std": 94.42764282226562,
"step": 293
},
{
"KL/chosen_KL_mean": -117.87823486328125,
"KL/mean": -141.62295532226562,
"KL/rejected_KL_mean": -165.36767578125,
"KL/std": 79.39311981201172,
"epoch": 0.4444444444444444,
"fcm_dpo/beta": 0.006892555393278599,
"fcm_dpo/delta": 0.08704549074172974,
"fcm_dpo/margin": 47.48944091796875,
"fcm_dpo/q_t": 0.4273802638053894,
"grad_norm": 11.643595695495605,
"learning_rate": 3.4168681427203153e-07,
"logits/chosen": 0.7372294068336487,
"logits/rejected": 0.6890612840652466,
"logps/chosen": -174.54913330078125,
"logps/ref_chosen": -56.670902252197266,
"logps/ref_rejected": -70.32819366455078,
"logps/rejected": -235.69586181640625,
"loss": 1.1445,
"margin_dpo/margin_mean": 47.48944091796875,
"margin_dpo/margin_std": 70.49929809570312,
"step": 294
},
{
"KL/chosen_KL_mean": -120.67965698242188,
"KL/mean": -142.65975952148438,
"KL/rejected_KL_mean": -164.63983154296875,
"KL/std": 73.50402069091797,
"epoch": 0.4459561602418745,
"fcm_dpo/beta": 0.007594604976475239,
"fcm_dpo/delta": 0.07951751351356506,
"fcm_dpo/margin": 43.960166931152344,
"fcm_dpo/q_t": 0.4269672632217407,
"grad_norm": 17.615432739257812,
"learning_rate": 3.4045544995169125e-07,
"logits/chosen": 0.7029507160186768,
"logits/rejected": 0.5973961353302002,
"logps/chosen": -171.08055114746094,
"logps/ref_chosen": -50.40088653564453,
"logps/ref_rejected": -83.43521881103516,
"logps/rejected": -248.07505798339844,
"loss": 1.1654,
"margin_dpo/margin_mean": 43.960166931152344,
"margin_dpo/margin_std": 79.29472351074219,
"step": 295
},
{
"KL/chosen_KL_mean": -125.65110778808594,
"KL/mean": -154.50802612304688,
"KL/rejected_KL_mean": -183.36492919921875,
"KL/std": 81.21266174316406,
"epoch": 0.4474678760393046,
"fcm_dpo/beta": 0.007408149540424347,
"fcm_dpo/delta": -0.03780694305896759,
"fcm_dpo/margin": 57.71380615234375,
"fcm_dpo/q_t": 0.40250879526138306,
"grad_norm": 12.49802017211914,
"learning_rate": 3.392215553979679e-07,
"logits/chosen": 0.6703150868415833,
"logits/rejected": 0.6198123693466187,
"logps/chosen": -194.80145263671875,
"logps/ref_chosen": -69.15034484863281,
"logps/ref_rejected": -89.60166931152344,
"logps/rejected": -272.96661376953125,
"loss": 1.1177,
"margin_dpo/margin_mean": 57.71380615234375,
"margin_dpo/margin_std": 94.09014892578125,
"step": 296
},
{
"KL/chosen_KL_mean": -128.17721557617188,
"KL/mean": -156.7912139892578,
"KL/rejected_KL_mean": -185.40521240234375,
"KL/std": 70.26692962646484,
"epoch": 0.4489795918367347,
"fcm_dpo/beta": 0.0074169086292386055,
"fcm_dpo/delta": -0.031429722905159,
"fcm_dpo/margin": 57.227989196777344,
"fcm_dpo/q_t": 0.40057173371315,
"grad_norm": 13.817586898803711,
"learning_rate": 3.3798516512554485e-07,
"logits/chosen": 0.7207061052322388,
"logits/rejected": 0.6670393943786621,
"logps/chosen": -186.1935272216797,
"logps/ref_chosen": -58.01630401611328,
"logps/ref_rejected": -69.95780944824219,
"logps/rejected": -255.36300659179688,
"loss": 1.0764,
"margin_dpo/margin_mean": 57.22799301147461,
"margin_dpo/margin_std": 72.8988037109375,
"step": 297
},
{
"KL/chosen_KL_mean": -125.60884094238281,
"KL/mean": -146.45750427246094,
"KL/rejected_KL_mean": -167.30616760253906,
"KL/std": 74.95790100097656,
"epoch": 0.4504913076341648,
"fcm_dpo/beta": 0.007724473252892494,
"fcm_dpo/delta": 0.0932346060872078,
"fcm_dpo/margin": 41.697330474853516,
"fcm_dpo/q_t": 0.4304448068141937,
"grad_norm": 13.414716720581055,
"learning_rate": 3.367463137189156e-07,
"logits/chosen": 0.8223170042037964,
"logits/rejected": 0.765424370765686,
"logps/chosen": -181.7781524658203,
"logps/ref_chosen": -56.1693115234375,
"logps/ref_rejected": -68.55052185058594,
"logps/rejected": -235.856689453125,
"loss": 1.1899,
"margin_dpo/margin_mean": 41.697330474853516,
"margin_dpo/margin_std": 83.2937240600586,
"step": 298
},
{
"KL/chosen_KL_mean": -129.3072509765625,
"KL/mean": -147.94827270507812,
"KL/rejected_KL_mean": -166.58929443359375,
"KL/std": 71.92723846435547,
"epoch": 0.4520030234315949,
"fcm_dpo/beta": 0.008116363547742367,
"fcm_dpo/delta": 0.019263437017798424,
"fcm_dpo/margin": 37.28206253051758,
"fcm_dpo/q_t": 0.43438130617141724,
"grad_norm": 21.00090217590332,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": 0.5886402726173401,
"logits/rejected": 0.5575330257415771,
"logps/chosen": -191.6250457763672,
"logps/ref_chosen": -62.31780242919922,
"logps/ref_rejected": -72.60028839111328,
"logps/rejected": -239.18959045410156,
"loss": 1.2391,
"margin_dpo/margin_mean": 37.282066345214844,
"margin_dpo/margin_std": 91.93014526367188,
"step": 299
},
{
"KL/chosen_KL_mean": -128.71337890625,
"KL/mean": -152.656982421875,
"KL/rejected_KL_mean": -176.60055541992188,
"KL/std": 75.02449035644531,
"epoch": 0.45351473922902497,
"fcm_dpo/beta": 0.008240276016294956,
"fcm_dpo/delta": 0.006145725026726723,
"fcm_dpo/margin": 47.887184143066406,
"fcm_dpo/q_t": 0.4129873514175415,
"grad_norm": 16.049564361572266,
"learning_rate": 3.3426136618426043e-07,
"logits/chosen": 0.7201621532440186,
"logits/rejected": 0.6523061990737915,
"logps/chosen": -189.09495544433594,
"logps/ref_chosen": -60.38157653808594,
"logps/ref_rejected": -75.45442199707031,
"logps/rejected": -252.0549774169922,
"loss": 1.147,
"margin_dpo/margin_mean": 47.887184143066406,
"margin_dpo/margin_std": 85.44697570800781,
"step": 300
},
{
"KL/chosen_KL_mean": -128.576171875,
"KL/mean": -149.49501037597656,
"KL/rejected_KL_mean": -170.41384887695312,
"KL/std": 72.66934204101562,
"epoch": 0.455026455026455,
"fcm_dpo/beta": 0.008374731056392193,
"fcm_dpo/delta": 0.05971474200487137,
"fcm_dpo/margin": 41.83769989013672,
"fcm_dpo/q_t": 0.424157977104187,
"grad_norm": 14.166158676147461,
"learning_rate": 3.3301533956555885e-07,
"logits/chosen": 0.7449129819869995,
"logits/rejected": 0.7153865694999695,
"logps/chosen": -181.42706298828125,
"logps/ref_chosen": -52.85089111328125,
"logps/ref_rejected": -69.97584533691406,
"logps/rejected": -240.38970947265625,
"loss": 1.1846,
"margin_dpo/margin_mean": 41.83769989013672,
"margin_dpo/margin_std": 84.38682556152344,
"step": 301
},
{
"KL/chosen_KL_mean": -135.41697692871094,
"KL/mean": -151.56887817382812,
"KL/rejected_KL_mean": -167.72076416015625,
"KL/std": 76.77266693115234,
"epoch": 0.4565381708238851,
"fcm_dpo/beta": 0.009450605139136314,
"fcm_dpo/delta": 0.11110319197177887,
"fcm_dpo/margin": 32.30377960205078,
"fcm_dpo/q_t": 0.4369601309299469,
"grad_norm": 19.52178382873535,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 0.5722560882568359,
"logits/rejected": 0.5158857107162476,
"logps/chosen": -202.38348388671875,
"logps/ref_chosen": -66.96650695800781,
"logps/ref_rejected": -88.09510803222656,
"logps/rejected": -255.81588745117188,
"loss": 1.2531,
"margin_dpo/margin_mean": 32.30377960205078,
"margin_dpo/margin_std": 85.83658599853516,
"step": 302
},
{
"KL/chosen_KL_mean": -117.91580200195312,
"KL/mean": -149.74221801757812,
"KL/rejected_KL_mean": -181.56866455078125,
"KL/std": 78.72908020019531,
"epoch": 0.4580498866213152,
"fcm_dpo/beta": 0.008384518325328827,
"fcm_dpo/delta": -0.18558809161186218,
"fcm_dpo/margin": 63.652870178222656,
"fcm_dpo/q_t": 0.3783756494522095,
"grad_norm": 12.682239532470703,
"learning_rate": 3.3051635489464793e-07,
"logits/chosen": 0.6628040671348572,
"logits/rejected": 0.5899385213851929,
"logps/chosen": -180.03732299804688,
"logps/ref_chosen": -62.12152862548828,
"logps/ref_rejected": -90.31204223632812,
"logps/rejected": -271.8807067871094,
"loss": 1.0824,
"margin_dpo/margin_mean": 63.652870178222656,
"margin_dpo/margin_std": 99.21585083007812,
"step": 303
},
{
"KL/chosen_KL_mean": -110.49132537841797,
"KL/mean": -143.3325653076172,
"KL/rejected_KL_mean": -176.17379760742188,
"KL/std": 81.67277526855469,
"epoch": 0.4595616024187453,
"fcm_dpo/beta": 0.007327304221689701,
"fcm_dpo/delta": -0.10975846648216248,
"fcm_dpo/margin": 65.6824722290039,
"fcm_dpo/q_t": 0.3843334913253784,
"grad_norm": 11.644968032836914,
"learning_rate": 3.292634667444117e-07,
"logits/chosen": 0.6995347738265991,
"logits/rejected": 0.6476801633834839,
"logps/chosen": -171.18641662597656,
"logps/ref_chosen": -60.695091247558594,
"logps/ref_rejected": -78.2525405883789,
"logps/rejected": -254.4263458251953,
"loss": 1.0324,
"margin_dpo/margin_mean": 65.68247985839844,
"margin_dpo/margin_std": 74.8603515625,
"step": 304
},
{
"KL/chosen_KL_mean": -126.179931640625,
"KL/mean": -145.57142639160156,
"KL/rejected_KL_mean": -164.96292114257812,
"KL/std": 73.00833892822266,
"epoch": 0.46107331821617537,
"fcm_dpo/beta": 0.007321414537727833,
"fcm_dpo/delta": 0.015304024331271648,
"fcm_dpo/margin": 38.782997131347656,
"fcm_dpo/q_t": 0.43489426374435425,
"grad_norm": 16.657503128051758,
"learning_rate": 3.280083614246217e-07,
"logits/chosen": 0.6606528759002686,
"logits/rejected": 0.696006178855896,
"logps/chosen": -198.87905883789062,
"logps/ref_chosen": -72.69914245605469,
"logps/ref_rejected": -65.65670776367188,
"logps/rejected": -230.61962890625,
"loss": 1.2284,
"margin_dpo/margin_mean": 38.78300094604492,
"margin_dpo/margin_std": 89.8697280883789,
"step": 305
},
{
"KL/chosen_KL_mean": -115.84672546386719,
"KL/mean": -140.39031982421875,
"KL/rejected_KL_mean": -164.9339141845703,
"KL/std": 70.68470001220703,
"epoch": 0.46258503401360546,
"fcm_dpo/beta": 0.007341842632740736,
"fcm_dpo/delta": 0.045422591269016266,
"fcm_dpo/margin": 49.087181091308594,
"fcm_dpo/q_t": 0.41786324977874756,
"grad_norm": 11.100536346435547,
"learning_rate": 3.267510740432719e-07,
"logits/chosen": 0.7595999836921692,
"logits/rejected": 0.6553710699081421,
"logps/chosen": -169.8172607421875,
"logps/ref_chosen": -53.97052764892578,
"logps/ref_rejected": -71.02423095703125,
"logps/rejected": -235.95814514160156,
"loss": 1.1295,
"margin_dpo/margin_mean": 49.087181091308594,
"margin_dpo/margin_std": 70.42362976074219,
"step": 306
},
{
"KL/chosen_KL_mean": -111.7839126586914,
"KL/mean": -123.70463562011719,
"KL/rejected_KL_mean": -135.62535095214844,
"KL/std": 71.19670867919922,
"epoch": 0.46409674981103555,
"fcm_dpo/beta": 0.007661640644073486,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 23.841440200805664,
"fcm_dpo/q_t": 0.45939934253692627,
"grad_norm": 18.099401473999023,
"learning_rate": 3.2549163976939285e-07,
"logits/chosen": 0.7359182834625244,
"logits/rejected": 0.6884441375732422,
"logps/chosen": -169.197021484375,
"logps/ref_chosen": -57.413108825683594,
"logps/ref_rejected": -68.68010711669922,
"logps/rejected": -204.30545043945312,
"loss": 1.3311,
"margin_dpo/margin_mean": 23.84143829345703,
"margin_dpo/margin_std": 92.32785034179688,
"step": 307
},
{
"KL/chosen_KL_mean": -112.34965515136719,
"KL/mean": -136.45751953125,
"KL/rejected_KL_mean": -160.56536865234375,
"KL/std": 73.75102233886719,
"epoch": 0.4656084656084656,
"fcm_dpo/beta": 0.007660580798983574,
"fcm_dpo/delta": 0.03539668023586273,
"fcm_dpo/margin": 48.2157096862793,
"fcm_dpo/q_t": 0.41769713163375854,
"grad_norm": 11.193486213684082,
"learning_rate": 3.2423009383206874e-07,
"logits/chosen": 0.7087165117263794,
"logits/rejected": 0.694772481918335,
"logps/chosen": -178.94845581054688,
"logps/ref_chosen": -66.59879302978516,
"logps/ref_rejected": -74.337158203125,
"logps/rejected": -234.90252685546875,
"loss": 1.1604,
"margin_dpo/margin_mean": 48.21570587158203,
"margin_dpo/margin_std": 87.3203125,
"step": 308
},
{
"KL/chosen_KL_mean": -126.59266662597656,
"KL/mean": -151.807861328125,
"KL/rejected_KL_mean": -177.0230712890625,
"KL/std": 72.11328125,
"epoch": 0.4671201814058957,
"fcm_dpo/beta": 0.007747044786810875,
"fcm_dpo/delta": 0.007666148245334625,
"fcm_dpo/margin": 50.430397033691406,
"fcm_dpo/q_t": 0.4117814600467682,
"grad_norm": 13.232572555541992,
"learning_rate": 3.229664715194511e-07,
"logits/chosen": 0.7303632497787476,
"logits/rejected": 0.6770374774932861,
"logps/chosen": -191.98741149902344,
"logps/ref_chosen": -65.39474487304688,
"logps/ref_rejected": -75.70930480957031,
"logps/rejected": -252.7323760986328,
"loss": 1.142,
"margin_dpo/margin_mean": 50.43040466308594,
"margin_dpo/margin_std": 84.50872039794922,
"step": 309
},
{
"KL/chosen_KL_mean": -134.52862548828125,
"KL/mean": -148.09298706054688,
"KL/rejected_KL_mean": -161.65733337402344,
"KL/std": 66.22860717773438,
"epoch": 0.46863189720332576,
"fcm_dpo/beta": 0.00854377169162035,
"fcm_dpo/delta": 0.06591358780860901,
"fcm_dpo/margin": 27.128713607788086,
"fcm_dpo/q_t": 0.4517921209335327,
"grad_norm": 15.922846794128418,
"learning_rate": 3.2170080817777257e-07,
"logits/chosen": 0.7066018581390381,
"logits/rejected": 0.6913357377052307,
"logps/chosen": -209.1968994140625,
"logps/ref_chosen": -74.66827392578125,
"logps/ref_rejected": -80.5689697265625,
"logps/rejected": -242.22630310058594,
"loss": 1.2784,
"margin_dpo/margin_mean": 27.128713607788086,
"margin_dpo/margin_std": 80.27430725097656,
"step": 310
},
{
"KL/chosen_KL_mean": -105.25132751464844,
"KL/mean": -133.28875732421875,
"KL/rejected_KL_mean": -161.326171875,
"KL/std": 85.32421875,
"epoch": 0.47014361300075586,
"fcm_dpo/beta": 0.007903890684247017,
"fcm_dpo/delta": -0.0625324696302414,
"fcm_dpo/margin": 56.074851989746094,
"fcm_dpo/q_t": 0.4021851122379303,
"grad_norm": 14.648249626159668,
"learning_rate": 3.204331392103574e-07,
"logits/chosen": 0.6372594237327576,
"logits/rejected": 0.492361843585968,
"logps/chosen": -164.98934936523438,
"logps/ref_chosen": -59.738033294677734,
"logps/ref_rejected": -93.60757446289062,
"logps/rejected": -254.93374633789062,
"loss": 1.1085,
"margin_dpo/margin_mean": 56.074851989746094,
"margin_dpo/margin_std": 91.86743927001953,
"step": 311
},
{
"KL/chosen_KL_mean": -108.83519744873047,
"KL/mean": -138.187255859375,
"KL/rejected_KL_mean": -167.539306640625,
"KL/std": 72.46369934082031,
"epoch": 0.47165532879818595,
"fcm_dpo/beta": 0.007707584649324417,
"fcm_dpo/delta": -0.06755498051643372,
"fcm_dpo/margin": 58.70410919189453,
"fcm_dpo/q_t": 0.3933003842830658,
"grad_norm": 11.331823348999023,
"learning_rate": 3.1916350007663176e-07,
"logits/chosen": 0.7237418293952942,
"logits/rejected": 0.6343299150466919,
"logps/chosen": -162.65164184570312,
"logps/ref_chosen": -53.816436767578125,
"logps/ref_rejected": -68.6575698852539,
"logps/rejected": -236.19688415527344,
"loss": 1.0551,
"margin_dpo/margin_mean": 58.70411682128906,
"margin_dpo/margin_std": 71.16246032714844,
"step": 312
},
{
"KL/chosen_KL_mean": -114.29727935791016,
"KL/mean": -130.198974609375,
"KL/rejected_KL_mean": -146.10064697265625,
"KL/std": 70.88896942138672,
"epoch": 0.47316704459561604,
"fcm_dpo/beta": 0.0077678910456597805,
"fcm_dpo/delta": 0.06859148293733597,
"fcm_dpo/margin": 31.803382873535156,
"fcm_dpo/q_t": 0.4456871747970581,
"grad_norm": 13.516242980957031,
"learning_rate": 3.178919262911314e-07,
"logits/chosen": 0.7776950597763062,
"logits/rejected": 0.75665283203125,
"logps/chosen": -174.254638671875,
"logps/ref_chosen": -59.957359313964844,
"logps/ref_rejected": -69.31729888916016,
"logps/rejected": -215.41795349121094,
"loss": 1.2415,
"margin_dpo/margin_mean": 31.803382873535156,
"margin_dpo/margin_std": 78.07221984863281,
"step": 313
},
{
"KL/chosen_KL_mean": -107.2558822631836,
"KL/mean": -136.37551879882812,
"KL/rejected_KL_mean": -165.4951171875,
"KL/std": 67.8990478515625,
"epoch": 0.47467876039304613,
"fcm_dpo/beta": 0.007589129731059074,
"fcm_dpo/delta": -0.05609302595257759,
"fcm_dpo/margin": 58.239227294921875,
"fcm_dpo/q_t": 0.397655725479126,
"grad_norm": 12.090660095214844,
"learning_rate": 3.166184534225087e-07,
"logits/chosen": 0.6843063831329346,
"logits/rejected": 0.7137875556945801,
"logps/chosen": -177.5240478515625,
"logps/ref_chosen": -70.26815795898438,
"logps/ref_rejected": -69.23971557617188,
"logps/rejected": -234.73483276367188,
"loss": 1.072,
"margin_dpo/margin_mean": 58.239227294921875,
"margin_dpo/margin_std": 76.19454956054688,
"step": 314
},
{
"KL/chosen_KL_mean": -112.75736236572266,
"KL/mean": -135.3028564453125,
"KL/rejected_KL_mean": -157.84835815429688,
"KL/std": 70.18635559082031,
"epoch": 0.47619047619047616,
"fcm_dpo/beta": 0.008021621033549309,
"fcm_dpo/delta": 0.043402545154094696,
"fcm_dpo/margin": 45.090999603271484,
"fcm_dpo/q_t": 0.4180910587310791,
"grad_norm": 12.942291259765625,
"learning_rate": 3.1534311709253723e-07,
"logits/chosen": 0.622521162033081,
"logits/rejected": 0.5911740064620972,
"logps/chosen": -180.55206298828125,
"logps/ref_chosen": -67.79469299316406,
"logps/ref_rejected": -74.55148315429688,
"logps/rejected": -232.39984130859375,
"loss": 1.1453,
"margin_dpo/margin_mean": 45.09100341796875,
"margin_dpo/margin_std": 75.40611267089844,
"step": 315
},
{
"KL/chosen_KL_mean": -104.73755645751953,
"KL/mean": -137.7486572265625,
"KL/rejected_KL_mean": -170.75973510742188,
"KL/std": 74.51022338867188,
"epoch": 0.47770219198790626,
"fcm_dpo/beta": 0.0075963689014315605,
"fcm_dpo/delta": -0.1409759819507599,
"fcm_dpo/margin": 66.0221939086914,
"fcm_dpo/q_t": 0.3811737895011902,
"grad_norm": 13.7676362991333,
"learning_rate": 3.1406595297511564e-07,
"logits/chosen": 0.5896261930465698,
"logits/rejected": 0.4515262842178345,
"logps/chosen": -160.02603149414062,
"logps/ref_chosen": -55.288482666015625,
"logps/ref_rejected": -96.15723419189453,
"logps/rejected": -266.9169921875,
"loss": 1.0447,
"margin_dpo/margin_mean": 66.0221939086914,
"margin_dpo/margin_std": 76.85704040527344,
"step": 316
},
{
"KL/chosen_KL_mean": -96.02757263183594,
"KL/mean": -126.60447692871094,
"KL/rejected_KL_mean": -157.18138122558594,
"KL/std": 66.47987365722656,
"epoch": 0.47921390778533635,
"fcm_dpo/beta": 0.006697298958897591,
"fcm_dpo/delta": -0.01382247731089592,
"fcm_dpo/margin": 61.15378189086914,
"fcm_dpo/q_t": 0.4028528034687042,
"grad_norm": 13.007543563842773,
"learning_rate": 3.1278699679526975e-07,
"logits/chosen": 0.7125881910324097,
"logits/rejected": 0.6669232845306396,
"logps/chosen": -150.60894775390625,
"logps/ref_chosen": -54.58137512207031,
"logps/ref_rejected": -72.77232360839844,
"logps/rejected": -229.95370483398438,
"loss": 1.0782,
"margin_dpo/margin_mean": 61.153785705566406,
"margin_dpo/margin_std": 74.21299743652344,
"step": 317
},
{
"KL/chosen_KL_mean": -105.79743957519531,
"KL/mean": -128.85116577148438,
"KL/rejected_KL_mean": -151.90487670898438,
"KL/std": 72.4944076538086,
"epoch": 0.48072562358276644,
"fcm_dpo/beta": 0.007166634313762188,
"fcm_dpo/delta": 0.08371928334236145,
"fcm_dpo/margin": 46.10744857788086,
"fcm_dpo/q_t": 0.43039560317993164,
"grad_norm": 13.263943672180176,
"learning_rate": 3.1150628432815336e-07,
"logits/chosen": 0.7046504020690918,
"logits/rejected": 0.635469913482666,
"logps/chosen": -158.6856689453125,
"logps/ref_chosen": -52.88822937011719,
"logps/ref_rejected": -80.63988494873047,
"logps/rejected": -232.54476928710938,
"loss": 1.2019,
"margin_dpo/margin_mean": 46.107452392578125,
"margin_dpo/margin_std": 100.03981018066406,
"step": 318
},
{
"KL/chosen_KL_mean": -104.99301147460938,
"KL/mean": -137.19601440429688,
"KL/rejected_KL_mean": -169.3990478515625,
"KL/std": 77.18682861328125,
"epoch": 0.48223733938019653,
"fcm_dpo/beta": 0.006997345481067896,
"fcm_dpo/delta": -0.0660954937338829,
"fcm_dpo/margin": 64.40603637695312,
"fcm_dpo/q_t": 0.39640793204307556,
"grad_norm": 10.057544708251953,
"learning_rate": 3.1022385139804707e-07,
"logits/chosen": 0.6656143665313721,
"logits/rejected": 0.6494824290275574,
"logps/chosen": -169.35633850097656,
"logps/ref_chosen": -64.36333465576172,
"logps/ref_rejected": -79.47296142578125,
"logps/rejected": -248.8719940185547,
"loss": 1.0744,
"margin_dpo/margin_mean": 64.40603637695312,
"margin_dpo/margin_std": 89.59465026855469,
"step": 319
},
{
"KL/chosen_KL_mean": -95.88545227050781,
"KL/mean": -121.54087829589844,
"KL/rejected_KL_mean": -147.19630432128906,
"KL/std": 78.3597412109375,
"epoch": 0.4837490551776266,
"fcm_dpo/beta": 0.00663637975230813,
"fcm_dpo/delta": -0.08432682603597641,
"fcm_dpo/margin": 51.31085968017578,
"fcm_dpo/q_t": 0.421201229095459,
"grad_norm": 11.44666576385498,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 0.5955374240875244,
"logits/rejected": 0.556653618812561,
"logps/chosen": -145.44419860839844,
"logps/ref_chosen": -49.558746337890625,
"logps/ref_rejected": -71.23444366455078,
"logps/rejected": -218.43075561523438,
"loss": 1.1595,
"margin_dpo/margin_mean": 51.310855865478516,
"margin_dpo/margin_std": 83.80187225341797,
"step": 320
},
{
"KL/chosen_KL_mean": -100.45033264160156,
"KL/mean": -128.28823852539062,
"KL/rejected_KL_mean": -156.1261444091797,
"KL/std": 66.45053100585938,
"epoch": 0.4852607709750567,
"fcm_dpo/beta": 0.006418926641345024,
"fcm_dpo/delta": 0.049936145544052124,
"fcm_dpo/margin": 55.67579650878906,
"fcm_dpo/q_t": 0.4175720810890198,
"grad_norm": 14.7537202835083,
"learning_rate": 3.0765396768561004e-07,
"logits/chosen": 0.682221531867981,
"logits/rejected": 0.6629788875579834,
"logps/chosen": -152.53561401367188,
"logps/ref_chosen": -52.08526611328125,
"logps/ref_rejected": -55.58674621582031,
"logps/rejected": -211.712890625,
"loss": 1.1299,
"margin_dpo/margin_mean": 55.67579650878906,
"margin_dpo/margin_std": 79.13897705078125,
"step": 321
},
{
"KL/chosen_KL_mean": -112.3656234741211,
"KL/mean": -147.36282348632812,
"KL/rejected_KL_mean": -182.36004638671875,
"KL/std": 75.82512664794922,
"epoch": 0.48677248677248675,
"fcm_dpo/beta": 0.00645124027505517,
"fcm_dpo/delta": -0.0664437934756279,
"fcm_dpo/margin": 69.99443054199219,
"fcm_dpo/q_t": 0.3910978436470032,
"grad_norm": 10.60876178741455,
"learning_rate": 3.063665887884511e-07,
"logits/chosen": 0.7542526721954346,
"logits/rejected": 0.6689043045043945,
"logps/chosen": -159.7697296142578,
"logps/ref_chosen": -47.404109954833984,
"logps/ref_rejected": -73.4260025024414,
"logps/rejected": -255.7860565185547,
"loss": 1.0393,
"margin_dpo/margin_mean": 69.99443054199219,
"margin_dpo/margin_std": 72.6507797241211,
"step": 322
},
{
"KL/chosen_KL_mean": -116.67129516601562,
"KL/mean": -138.6676025390625,
"KL/rejected_KL_mean": -160.66392517089844,
"KL/std": 72.51184844970703,
"epoch": 0.48828420256991684,
"fcm_dpo/beta": 0.006838475354015827,
"fcm_dpo/delta": 0.11610043048858643,
"fcm_dpo/margin": 43.99262619018555,
"fcm_dpo/q_t": 0.43621429800987244,
"grad_norm": 12.70161247253418,
"learning_rate": 3.0507763319663517e-07,
"logits/chosen": 0.5943468809127808,
"logits/rejected": 0.5237321853637695,
"logps/chosen": -186.67758178710938,
"logps/ref_chosen": -70.00630187988281,
"logps/ref_rejected": -86.96690368652344,
"logps/rejected": -247.63082885742188,
"loss": 1.2035,
"margin_dpo/margin_mean": 43.99262619018555,
"margin_dpo/margin_std": 93.22569274902344,
"step": 323
},
{
"KL/chosen_KL_mean": -103.11143493652344,
"KL/mean": -133.23056030273438,
"KL/rejected_KL_mean": -163.34967041015625,
"KL/std": 84.75403594970703,
"epoch": 0.4897959183673469,
"fcm_dpo/beta": 0.006788309663534164,
"fcm_dpo/delta": -0.014123659580945969,
"fcm_dpo/margin": 60.23822021484375,
"fcm_dpo/q_t": 0.404258668422699,
"grad_norm": 13.756731033325195,
"learning_rate": 3.0378713696502097e-07,
"logits/chosen": 0.7186671495437622,
"logits/rejected": 0.6600125432014465,
"logps/chosen": -159.00025939941406,
"logps/ref_chosen": -55.88882064819336,
"logps/ref_rejected": -75.23088073730469,
"logps/rejected": -238.58055114746094,
"loss": 1.0856,
"margin_dpo/margin_mean": 60.23822021484375,
"margin_dpo/margin_std": 76.23624420166016,
"step": 324
},
{
"KL/chosen_KL_mean": -124.19041442871094,
"KL/mean": -154.54751586914062,
"KL/rejected_KL_mean": -184.90463256835938,
"KL/std": 82.73353576660156,
"epoch": 0.491307634164777,
"fcm_dpo/beta": 0.006723565980792046,
"fcm_dpo/delta": -0.012475315481424332,
"fcm_dpo/margin": 60.7142219543457,
"fcm_dpo/q_t": 0.4078647792339325,
"grad_norm": 11.16752815246582,
"learning_rate": 3.0249513619156206e-07,
"logits/chosen": 0.6296889781951904,
"logits/rejected": 0.5697454214096069,
"logps/chosen": -188.33743286132812,
"logps/ref_chosen": -64.14701843261719,
"logps/ref_rejected": -79.91143798828125,
"logps/rejected": -264.8160705566406,
"loss": 1.1234,
"margin_dpo/margin_mean": 60.71421813964844,
"margin_dpo/margin_std": 101.00370788574219,
"step": 325
},
{
"KL/chosen_KL_mean": -145.95529174804688,
"KL/mean": -157.24642944335938,
"KL/rejected_KL_mean": -168.53758239746094,
"KL/std": 74.45347595214844,
"epoch": 0.4928193499622071,
"fcm_dpo/beta": 0.006838089786469936,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 22.582300186157227,
"fcm_dpo/q_t": 0.464555561542511,
"grad_norm": 12.780586242675781,
"learning_rate": 3.012016670162977e-07,
"logits/chosen": 0.5949761271476746,
"logits/rejected": 0.6042267084121704,
"logps/chosen": -221.48660278320312,
"logps/ref_chosen": -75.53131103515625,
"logps/ref_rejected": -76.5898666381836,
"logps/rejected": -245.12744140625,
"loss": 1.3171,
"margin_dpo/margin_mean": 22.582300186157227,
"margin_dpo/margin_std": 83.8711166381836,
"step": 326
},
{
"KL/chosen_KL_mean": -128.95672607421875,
"KL/mean": -152.0322265625,
"KL/rejected_KL_mean": -175.10769653320312,
"KL/std": 79.25118255615234,
"epoch": 0.4943310657596372,
"fcm_dpo/beta": 0.007321531884372234,
"fcm_dpo/delta": 0.07252618670463562,
"fcm_dpo/margin": 46.15097427368164,
"fcm_dpo/q_t": 0.425706684589386,
"grad_norm": 13.338642120361328,
"learning_rate": 2.99906765620341e-07,
"logits/chosen": 0.5909023880958557,
"logits/rejected": 0.5673669576644897,
"logps/chosen": -198.29388427734375,
"logps/ref_chosen": -69.33717346191406,
"logps/ref_rejected": -73.37751770019531,
"logps/rejected": -248.48521423339844,
"loss": 1.1822,
"margin_dpo/margin_mean": 46.150970458984375,
"margin_dpo/margin_std": 91.36515045166016,
"step": 327
},
{
"KL/chosen_KL_mean": -113.59530639648438,
"KL/mean": -142.98179626464844,
"KL/rejected_KL_mean": -172.3682861328125,
"KL/std": 80.62422943115234,
"epoch": 0.4958427815570673,
"fcm_dpo/beta": 0.007260303478688002,
"fcm_dpo/delta": -0.034179363399744034,
"fcm_dpo/margin": 58.77297592163086,
"fcm_dpo/q_t": 0.4015209674835205,
"grad_norm": 13.58584213256836,
"learning_rate": 2.9861046822486766e-07,
"logits/chosen": 0.5832690000534058,
"logits/rejected": 0.5415433049201965,
"logps/chosen": -175.30154418945312,
"logps/ref_chosen": -61.70623016357422,
"logps/ref_rejected": -83.73808288574219,
"logps/rejected": -256.1063537597656,
"loss": 1.0779,
"margin_dpo/margin_mean": 58.77297592163086,
"margin_dpo/margin_std": 76.87384033203125,
"step": 328
},
{
"KL/chosen_KL_mean": -127.56649780273438,
"KL/mean": -156.63983154296875,
"KL/rejected_KL_mean": -185.71316528320312,
"KL/std": 86.29991912841797,
"epoch": 0.4973544973544973,
"fcm_dpo/beta": 0.0070040179416537285,
"fcm_dpo/delta": -0.00944619532674551,
"fcm_dpo/margin": 58.14664840698242,
"fcm_dpo/q_t": 0.4094126522541046,
"grad_norm": 14.41002082824707,
"learning_rate": 2.9731281109010253e-07,
"logits/chosen": 0.7076858282089233,
"logits/rejected": 0.6475342512130737,
"logps/chosen": -192.06491088867188,
"logps/ref_chosen": -64.4984130859375,
"logps/ref_rejected": -83.6591796875,
"logps/rejected": -269.3723449707031,
"loss": 1.1137,
"margin_dpo/margin_mean": 58.14665222167969,
"margin_dpo/margin_std": 91.29844665527344,
"step": 329
},
{
"KL/chosen_KL_mean": -111.404541015625,
"KL/mean": -139.49217224121094,
"KL/rejected_KL_mean": -167.57980346679688,
"KL/std": 77.82200622558594,
"epoch": 0.4988662131519274,
"fcm_dpo/beta": 0.007140832487493753,
"fcm_dpo/delta": -0.002552524209022522,
"fcm_dpo/margin": 56.17527770996094,
"fcm_dpo/q_t": 0.40969616174697876,
"grad_norm": 12.402542114257812,
"learning_rate": 2.9601383051430505e-07,
"logits/chosen": 0.672360360622406,
"logits/rejected": 0.6062643527984619,
"logps/chosen": -166.2091827392578,
"logps/ref_chosen": -54.80464172363281,
"logps/ref_rejected": -75.3194351196289,
"logps/rejected": -242.89923095703125,
"loss": 1.1446,
"margin_dpo/margin_mean": 56.17527389526367,
"margin_dpo/margin_std": 99.22282409667969,
"step": 330
},
{
"KL/chosen_KL_mean": -119.64215087890625,
"KL/mean": -156.07215881347656,
"KL/rejected_KL_mean": -192.50216674804688,
"KL/std": 81.64392852783203,
"epoch": 0.5003779289493575,
"fcm_dpo/beta": 0.0065118009224534035,
"fcm_dpo/delta": -0.09822378307580948,
"fcm_dpo/margin": 72.86000061035156,
"fcm_dpo/q_t": 0.3864233195781708,
"grad_norm": 11.534334182739258,
"learning_rate": 2.947135628327544e-07,
"logits/chosen": 0.76860511302948,
"logits/rejected": 0.7442618608474731,
"logps/chosen": -178.88473510742188,
"logps/ref_chosen": -59.242584228515625,
"logps/ref_rejected": -69.87483215332031,
"logps/rejected": -262.37701416015625,
"loss": 1.0453,
"margin_dpo/margin_mean": 72.86000061035156,
"margin_dpo/margin_std": 89.18681335449219,
"step": 331
},
{
"KL/chosen_KL_mean": -124.42134857177734,
"KL/mean": -154.72769165039062,
"KL/rejected_KL_mean": -185.03404235839844,
"KL/std": 78.88076782226562,
"epoch": 0.5018896447467877,
"fcm_dpo/beta": 0.00656371284276247,
"fcm_dpo/delta": -0.0017648041248321533,
"fcm_dpo/margin": 60.61271286010742,
"fcm_dpo/q_t": 0.40883493423461914,
"grad_norm": 11.724095344543457,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": 0.6075745820999146,
"logits/rejected": 0.566921591758728,
"logps/chosen": -191.53109741210938,
"logps/ref_chosen": -67.10975646972656,
"logps/ref_rejected": -77.11839294433594,
"logps/rejected": -262.1524353027344,
"loss": 1.1018,
"margin_dpo/margin_mean": 60.61271286010742,
"margin_dpo/margin_std": 83.28633117675781,
"step": 332
},
{
"KL/chosen_KL_mean": -130.77462768554688,
"KL/mean": -165.58177185058594,
"KL/rejected_KL_mean": -200.388916015625,
"KL/std": 81.94561767578125,
"epoch": 0.5034013605442177,
"fcm_dpo/beta": 0.006204267032444477,
"fcm_dpo/delta": -0.04060637205839157,
"fcm_dpo/margin": 69.61427307128906,
"fcm_dpo/q_t": 0.39935895800590515,
"grad_norm": 10.982404708862305,
"learning_rate": 2.921093116725076e-07,
"logits/chosen": 0.6704716682434082,
"logits/rejected": 0.5928934812545776,
"logps/chosen": -189.15576171875,
"logps/ref_chosen": -58.381134033203125,
"logps/ref_rejected": -85.02839660644531,
"logps/rejected": -285.41729736328125,
"loss": 1.0682,
"margin_dpo/margin_mean": 69.61427307128906,
"margin_dpo/margin_std": 87.08734893798828,
"step": 333
},
{
"KL/chosen_KL_mean": -126.27867126464844,
"KL/mean": -151.51400756835938,
"KL/rejected_KL_mean": -176.7493438720703,
"KL/std": 80.90272521972656,
"epoch": 0.5049130763416477,
"fcm_dpo/beta": 0.0065368469804525375,
"fcm_dpo/delta": 0.08284910768270493,
"fcm_dpo/margin": 50.47067642211914,
"fcm_dpo/q_t": 0.4275355339050293,
"grad_norm": 12.728032112121582,
"learning_rate": 2.9080540104031484e-07,
"logits/chosen": 0.683144211769104,
"logits/rejected": 0.63508141040802,
"logps/chosen": -193.17068481445312,
"logps/ref_chosen": -66.89199829101562,
"logps/ref_rejected": -91.83695220947266,
"logps/rejected": -268.5863037109375,
"loss": 1.1678,
"margin_dpo/margin_mean": 50.470672607421875,
"margin_dpo/margin_std": 91.65339660644531,
"step": 334
},
{
"KL/chosen_KL_mean": -127.95242309570312,
"KL/mean": -154.70330810546875,
"KL/rejected_KL_mean": -181.45420837402344,
"KL/std": 80.90696716308594,
"epoch": 0.5064247921390779,
"fcm_dpo/beta": 0.0069529893808066845,
"fcm_dpo/delta": 0.030666418373584747,
"fcm_dpo/margin": 53.501800537109375,
"fcm_dpo/q_t": 0.41777414083480835,
"grad_norm": 14.211898803710938,
"learning_rate": 2.895003489933375e-07,
"logits/chosen": 0.6847304105758667,
"logits/rejected": 0.6470900774002075,
"logps/chosen": -189.4668731689453,
"logps/ref_chosen": -61.51445770263672,
"logps/ref_rejected": -75.68916320800781,
"logps/rejected": -257.14337158203125,
"loss": 1.1548,
"margin_dpo/margin_mean": 53.501800537109375,
"margin_dpo/margin_std": 95.9657974243164,
"step": 335
},
{
"KL/chosen_KL_mean": -141.7051239013672,
"KL/mean": -170.696044921875,
"KL/rejected_KL_mean": -199.68698120117188,
"KL/std": 86.33305358886719,
"epoch": 0.5079365079365079,
"fcm_dpo/beta": 0.006664481945335865,
"fcm_dpo/delta": 0.011959366500377655,
"fcm_dpo/margin": 57.98182678222656,
"fcm_dpo/q_t": 0.4162399470806122,
"grad_norm": 13.38348388671875,
"learning_rate": 2.8819419203668675e-07,
"logits/chosen": 0.624375581741333,
"logits/rejected": 0.5998581051826477,
"logps/chosen": -210.55519104003906,
"logps/ref_chosen": -68.85006713867188,
"logps/ref_rejected": -92.99603271484375,
"logps/rejected": -292.6829833984375,
"loss": 1.1397,
"margin_dpo/margin_mean": 57.98182678222656,
"margin_dpo/margin_std": 98.35008239746094,
"step": 336
},
{
"KL/chosen_KL_mean": -149.2178955078125,
"KL/mean": -172.33078002929688,
"KL/rejected_KL_mean": -195.44371032714844,
"KL/std": 86.13853454589844,
"epoch": 0.509448223733938,
"fcm_dpo/beta": 0.0073309652507305145,
"fcm_dpo/delta": 0.07309623807668686,
"fcm_dpo/margin": 46.225830078125,
"fcm_dpo/q_t": 0.4263220429420471,
"grad_norm": 12.12402629852295,
"learning_rate": 2.8688696670638053e-07,
"logits/chosen": 0.5780133605003357,
"logits/rejected": 0.5493387579917908,
"logps/chosen": -222.40573120117188,
"logps/ref_chosen": -73.18783569335938,
"logps/ref_rejected": -86.89118957519531,
"logps/rejected": -282.33489990234375,
"loss": 1.1838,
"margin_dpo/margin_mean": 46.225830078125,
"margin_dpo/margin_std": 92.9611587524414,
"step": 337
},
{
"KL/chosen_KL_mean": -146.11094665527344,
"KL/mean": -169.81564331054688,
"KL/rejected_KL_mean": -193.5203399658203,
"KL/std": 78.87342071533203,
"epoch": 0.5109599395313681,
"fcm_dpo/beta": 0.00764493178576231,
"fcm_dpo/delta": 0.045983582735061646,
"fcm_dpo/margin": 47.40937805175781,
"fcm_dpo/q_t": 0.41997748613357544,
"grad_norm": 12.370586395263672,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 0.6543248891830444,
"logits/rejected": 0.6287999153137207,
"logps/chosen": -210.05056762695312,
"logps/ref_chosen": -63.939613342285156,
"logps/ref_rejected": -75.34243774414062,
"logps/rejected": -268.86279296875,
"loss": 1.1633,
"margin_dpo/margin_mean": 47.40937805175781,
"margin_dpo/margin_std": 88.205810546875,
"step": 338
},
{
"KL/chosen_KL_mean": -118.83457946777344,
"KL/mean": -148.2288818359375,
"KL/rejected_KL_mean": -177.6231689453125,
"KL/std": 79.79850006103516,
"epoch": 0.5124716553287982,
"fcm_dpo/beta": 0.007407545577734709,
"fcm_dpo/delta": -0.04732739180326462,
"fcm_dpo/margin": 58.788604736328125,
"fcm_dpo/q_t": 0.3996833562850952,
"grad_norm": 15.090683937072754,
"learning_rate": 2.842694572172736e-07,
"logits/chosen": 0.8103833198547363,
"logits/rejected": 0.7248860001564026,
"logps/chosen": -164.38369750976562,
"logps/ref_chosen": -45.54913330078125,
"logps/ref_rejected": -67.0482177734375,
"logps/rejected": -244.67138671875,
"loss": 1.0939,
"margin_dpo/margin_mean": 58.78860092163086,
"margin_dpo/margin_std": 85.93904113769531,
"step": 339
},
{
"KL/chosen_KL_mean": -133.77890014648438,
"KL/mean": -161.17971801757812,
"KL/rejected_KL_mean": -188.5805206298828,
"KL/std": 85.82127380371094,
"epoch": 0.5139833711262283,
"fcm_dpo/beta": 0.0074435994029045105,
"fcm_dpo/delta": -0.010237021371722221,
"fcm_dpo/margin": 54.8016242980957,
"fcm_dpo/q_t": 0.41037893295288086,
"grad_norm": 12.742267608642578,
"learning_rate": 2.8295924627584004e-07,
"logits/chosen": 0.652264416217804,
"logits/rejected": 0.6331235766410828,
"logps/chosen": -187.7845458984375,
"logps/ref_chosen": -54.00564956665039,
"logps/ref_rejected": -61.314430236816406,
"logps/rejected": -249.89495849609375,
"loss": 1.1512,
"margin_dpo/margin_mean": 54.80162811279297,
"margin_dpo/margin_std": 101.09391021728516,
"step": 340
},
{
"KL/chosen_KL_mean": -136.74429321289062,
"KL/mean": -168.65585327148438,
"KL/rejected_KL_mean": -200.56744384765625,
"KL/std": 86.67827606201172,
"epoch": 0.5154950869236583,
"fcm_dpo/beta": 0.006320003420114517,
"fcm_dpo/delta": -0.15153226256370544,
"fcm_dpo/margin": 63.823143005371094,
"fcm_dpo/q_t": 0.3994399309158325,
"grad_norm": 13.01077651977539,
"learning_rate": 2.816481133934373e-07,
"logits/chosen": 0.6961194276809692,
"logits/rejected": 0.6493462324142456,
"logps/chosen": -200.13938903808594,
"logps/ref_chosen": -63.39509582519531,
"logps/ref_rejected": -76.20973205566406,
"logps/rejected": -276.77716064453125,
"loss": 1.1119,
"margin_dpo/margin_mean": 63.823143005371094,
"margin_dpo/margin_std": 92.03482055664062,
"step": 341
},
{
"KL/chosen_KL_mean": -134.47647094726562,
"KL/mean": -165.2379150390625,
"KL/rejected_KL_mean": -195.99932861328125,
"KL/std": 91.90939331054688,
"epoch": 0.5170068027210885,
"fcm_dpo/beta": 0.006310966797173023,
"fcm_dpo/delta": 0.0141034796833992,
"fcm_dpo/margin": 61.52287292480469,
"fcm_dpo/q_t": 0.4127877652645111,
"grad_norm": 10.182599067687988,
"learning_rate": 2.8033609524527046e-07,
"logits/chosen": 0.7835279107093811,
"logits/rejected": 0.7403960227966309,
"logps/chosen": -187.5242919921875,
"logps/ref_chosen": -53.047813415527344,
"logps/ref_rejected": -68.2854232788086,
"logps/rejected": -264.2847595214844,
"loss": 1.1187,
"margin_dpo/margin_mean": 61.52287292480469,
"margin_dpo/margin_std": 94.30167388916016,
"step": 342
},
{
"KL/chosen_KL_mean": -130.44664001464844,
"KL/mean": -152.9931640625,
"KL/rejected_KL_mean": -175.53968811035156,
"KL/std": 85.21757507324219,
"epoch": 0.5185185185185185,
"fcm_dpo/beta": 0.006350641138851643,
"fcm_dpo/delta": -0.009267449378967285,
"fcm_dpo/margin": 45.093055725097656,
"fcm_dpo/q_t": 0.43408840894699097,
"grad_norm": 10.364089965820312,
"learning_rate": 2.7902322853130753e-07,
"logits/chosen": 0.5760016441345215,
"logits/rejected": 0.5660780668258667,
"logps/chosen": -201.02517700195312,
"logps/ref_chosen": -70.57852935791016,
"logps/ref_rejected": -84.73873901367188,
"logps/rejected": -260.2784423828125,
"loss": 1.1991,
"margin_dpo/margin_mean": 45.09305191040039,
"margin_dpo/margin_std": 89.2558364868164,
"step": 343
},
{
"KL/chosen_KL_mean": -141.1691436767578,
"KL/mean": -174.16143798828125,
"KL/rejected_KL_mean": -207.15371704101562,
"KL/std": 86.44344329833984,
"epoch": 0.5200302343159486,
"fcm_dpo/beta": 0.0062735131941735744,
"fcm_dpo/delta": -0.017590418457984924,
"fcm_dpo/margin": 65.98458862304688,
"fcm_dpo/q_t": 0.40609464049339294,
"grad_norm": 11.856791496276855,
"learning_rate": 2.7770954997525274e-07,
"logits/chosen": 0.7121565937995911,
"logits/rejected": 0.6411057710647583,
"logps/chosen": -196.9801483154297,
"logps/ref_chosen": -55.811004638671875,
"logps/ref_rejected": -84.77637481689453,
"logps/rejected": -291.93011474609375,
"loss": 1.1069,
"margin_dpo/margin_mean": 65.98458862304688,
"margin_dpo/margin_std": 100.44625854492188,
"step": 344
},
{
"KL/chosen_KL_mean": -116.9457778930664,
"KL/mean": -146.05355834960938,
"KL/rejected_KL_mean": -175.1613311767578,
"KL/std": 79.33267211914062,
"epoch": 0.5215419501133787,
"fcm_dpo/beta": 0.0064442697912454605,
"fcm_dpo/delta": 0.029752016067504883,
"fcm_dpo/margin": 58.21555709838867,
"fcm_dpo/q_t": 0.41542765498161316,
"grad_norm": 11.773383140563965,
"learning_rate": 2.7639509632351927e-07,
"logits/chosen": 0.7386205792427063,
"logits/rejected": 0.6958231329917908,
"logps/chosen": -174.73187255859375,
"logps/ref_chosen": -57.78609848022461,
"logps/ref_rejected": -78.91847229003906,
"logps/rejected": -254.07980346679688,
"loss": 1.1319,
"margin_dpo/margin_mean": 58.21556091308594,
"margin_dpo/margin_std": 93.69883728027344,
"step": 345
},
{
"KL/chosen_KL_mean": -122.50251007080078,
"KL/mean": -158.81753540039062,
"KL/rejected_KL_mean": -195.132568359375,
"KL/std": 94.08961486816406,
"epoch": 0.5230536659108088,
"fcm_dpo/beta": 0.0062287356704473495,
"fcm_dpo/delta": -0.06783194094896317,
"fcm_dpo/margin": 72.63008117675781,
"fcm_dpo/q_t": 0.39354556798934937,
"grad_norm": 13.235642433166504,
"learning_rate": 2.7507990434420123e-07,
"logits/chosen": 0.7180252075195312,
"logits/rejected": 0.6350604295730591,
"logps/chosen": -178.78762817382812,
"logps/ref_chosen": -56.285125732421875,
"logps/ref_rejected": -91.15303039550781,
"logps/rejected": -286.2856140136719,
"loss": 1.074,
"margin_dpo/margin_mean": 72.63007354736328,
"margin_dpo/margin_std": 97.58467102050781,
"step": 346
},
{
"KL/chosen_KL_mean": -135.82313537597656,
"KL/mean": -162.78857421875,
"KL/rejected_KL_mean": -189.75399780273438,
"KL/std": 86.55868530273438,
"epoch": 0.5245653817082389,
"fcm_dpo/beta": 0.0063002025708556175,
"fcm_dpo/delta": 0.07281588017940521,
"fcm_dpo/margin": 53.93086242675781,
"fcm_dpo/q_t": 0.4243871867656708,
"grad_norm": 12.957653999328613,
"learning_rate": 2.737640108260456e-07,
"logits/chosen": 0.8214094638824463,
"logits/rejected": 0.7735979557037354,
"logps/chosen": -189.3226776123047,
"logps/ref_chosen": -53.499542236328125,
"logps/ref_rejected": -72.52565002441406,
"logps/rejected": -262.2796630859375,
"loss": 1.1479,
"margin_dpo/margin_mean": 53.93086242675781,
"margin_dpo/margin_std": 87.56758117675781,
"step": 347
},
{
"KL/chosen_KL_mean": -122.32320404052734,
"KL/mean": -153.75466918945312,
"KL/rejected_KL_mean": -185.1861572265625,
"KL/std": 81.93644714355469,
"epoch": 0.5260770975056689,
"fcm_dpo/beta": 0.006337115541100502,
"fcm_dpo/delta": 0.0002983175218105316,
"fcm_dpo/margin": 62.86295700073242,
"fcm_dpo/q_t": 0.4105902910232544,
"grad_norm": 12.72779655456543,
"learning_rate": 2.724474525774229e-07,
"logits/chosen": 0.8158016204833984,
"logits/rejected": 0.7839329242706299,
"logps/chosen": -173.11004638671875,
"logps/ref_chosen": -50.78684997558594,
"logps/ref_rejected": -68.63732147216797,
"logps/rejected": -253.823486328125,
"loss": 1.1154,
"margin_dpo/margin_mean": 62.862953186035156,
"margin_dpo/margin_std": 96.59371948242188,
"step": 348
},
{
"KL/chosen_KL_mean": -123.90093994140625,
"KL/mean": -156.0067138671875,
"KL/rejected_KL_mean": -188.11248779296875,
"KL/std": 91.27865600585938,
"epoch": 0.527588813303099,
"fcm_dpo/beta": 0.0065030986443161964,
"fcm_dpo/delta": -0.024084649980068207,
"fcm_dpo/margin": 64.21153259277344,
"fcm_dpo/q_t": 0.40591293573379517,
"grad_norm": 12.217432022094727,
"learning_rate": 2.711302664252973e-07,
"logits/chosen": 0.7418991923332214,
"logits/rejected": 0.646633505821228,
"logps/chosen": -177.2259521484375,
"logps/ref_chosen": -53.325008392333984,
"logps/ref_rejected": -83.21236419677734,
"logps/rejected": -271.3248291015625,
"loss": 1.1069,
"margin_dpo/margin_mean": 64.21154022216797,
"margin_dpo/margin_std": 96.88829040527344,
"step": 349
},
{
"KL/chosen_KL_mean": -126.6453857421875,
"KL/mean": -169.14663696289062,
"KL/rejected_KL_mean": -211.64791870117188,
"KL/std": 97.974609375,
"epoch": 0.5291005291005291,
"fcm_dpo/beta": 0.005718736909329891,
"fcm_dpo/delta": -0.1165238469839096,
"fcm_dpo/margin": 85.00253295898438,
"fcm_dpo/q_t": 0.3823647201061249,
"grad_norm": 12.687152862548828,
"learning_rate": 2.698124892141971e-07,
"logits/chosen": 0.694975733757019,
"logits/rejected": 0.6256704330444336,
"logps/chosen": -188.27114868164062,
"logps/ref_chosen": -61.625770568847656,
"logps/ref_rejected": -87.63627624511719,
"logps/rejected": -299.2841796875,
"loss": 1.0231,
"margin_dpo/margin_mean": 85.00252532958984,
"margin_dpo/margin_std": 92.37911224365234,
"step": 350
},
{
"KL/chosen_KL_mean": -123.06855773925781,
"KL/mean": -152.65802001953125,
"KL/rejected_KL_mean": -182.24749755859375,
"KL/std": 81.3293228149414,
"epoch": 0.5306122448979592,
"fcm_dpo/beta": 0.005834533832967281,
"fcm_dpo/delta": 0.06635048985481262,
"fcm_dpo/margin": 59.17893600463867,
"fcm_dpo/q_t": 0.42190346121788025,
"grad_norm": 10.825620651245117,
"learning_rate": 2.6849415780518357e-07,
"logits/chosen": 0.6644502878189087,
"logits/rejected": 0.5926836133003235,
"logps/chosen": -179.32489013671875,
"logps/ref_chosen": -56.2563362121582,
"logps/ref_rejected": -79.11589813232422,
"logps/rejected": -261.3634033203125,
"loss": 1.1597,
"margin_dpo/margin_mean": 59.17893981933594,
"margin_dpo/margin_std": 103.34938049316406,
"step": 351
},
{
"KL/chosen_KL_mean": -125.21533203125,
"KL/mean": -159.1262969970703,
"KL/rejected_KL_mean": -193.03726196289062,
"KL/std": 92.70458984375,
"epoch": 0.5321239606953893,
"fcm_dpo/beta": 0.005886302795261145,
"fcm_dpo/delta": -0.00063290074467659,
"fcm_dpo/margin": 67.82192993164062,
"fcm_dpo/q_t": 0.40983670949935913,
"grad_norm": 11.221183776855469,
"learning_rate": 2.6717530907482024e-07,
"logits/chosen": 0.7265364527702332,
"logits/rejected": 0.6756460666656494,
"logps/chosen": -188.26727294921875,
"logps/ref_chosen": -63.05195236206055,
"logps/ref_rejected": -85.52035522460938,
"logps/rejected": -278.5576171875,
"loss": 1.1079,
"margin_dpo/margin_mean": 67.82193756103516,
"margin_dpo/margin_std": 100.30946350097656,
"step": 352
},
{
"KL/chosen_KL_mean": -122.37887573242188,
"KL/mean": -153.0131072998047,
"KL/rejected_KL_mean": -183.64732360839844,
"KL/std": 84.29683685302734,
"epoch": 0.5336356764928194,
"fcm_dpo/beta": 0.006157569587230682,
"fcm_dpo/delta": 0.027647539973258972,
"fcm_dpo/margin": 61.26842498779297,
"fcm_dpo/q_t": 0.4137980043888092,
"grad_norm": 10.43450927734375,
"learning_rate": 2.658559799141411e-07,
"logits/chosen": 0.7220193147659302,
"logits/rejected": 0.7308777570724487,
"logps/chosen": -191.3880615234375,
"logps/ref_chosen": -69.00918579101562,
"logps/ref_rejected": -72.65840148925781,
"logps/rejected": -256.30572509765625,
"loss": 1.114,
"margin_dpo/margin_mean": 61.2684326171875,
"margin_dpo/margin_std": 87.84406280517578,
"step": 353
},
{
"KL/chosen_KL_mean": -120.68923950195312,
"KL/mean": -157.22747802734375,
"KL/rejected_KL_mean": -193.76571655273438,
"KL/std": 78.87259674072266,
"epoch": 0.5351473922902494,
"fcm_dpo/beta": 0.0058960379101336,
"fcm_dpo/delta": -0.0409696064889431,
"fcm_dpo/margin": 73.07649230957031,
"fcm_dpo/q_t": 0.3988759219646454,
"grad_norm": 12.436847686767578,
"learning_rate": 2.6453620722761895e-07,
"logits/chosen": 0.8180617094039917,
"logits/rejected": 0.6904994249343872,
"logps/chosen": -160.47756958007812,
"logps/ref_chosen": -39.78833770751953,
"logps/ref_rejected": -69.56885528564453,
"logps/rejected": -263.3345947265625,
"loss": 1.0792,
"margin_dpo/margin_mean": 73.07649230957031,
"margin_dpo/margin_std": 97.00477600097656,
"step": 354
},
{
"KL/chosen_KL_mean": -125.70584106445312,
"KL/mean": -166.8114013671875,
"KL/rejected_KL_mean": -207.91696166992188,
"KL/std": 94.3542251586914,
"epoch": 0.5366591080876795,
"fcm_dpo/beta": 0.005632805172353983,
"fcm_dpo/delta": -0.0817156583070755,
"fcm_dpo/margin": 82.21111297607422,
"fcm_dpo/q_t": 0.3910810351371765,
"grad_norm": 13.240791320800781,
"learning_rate": 2.632160279321328e-07,
"logits/chosen": 0.7661707401275635,
"logits/rejected": 0.6408806443214417,
"logps/chosen": -171.9612274169922,
"logps/ref_chosen": -46.25537872314453,
"logps/ref_rejected": -78.20236206054688,
"logps/rejected": -286.11932373046875,
"loss": 1.061,
"margin_dpo/margin_mean": 82.21110534667969,
"margin_dpo/margin_std": 104.74549102783203,
"step": 355
},
{
"KL/chosen_KL_mean": -120.32354736328125,
"KL/mean": -152.593505859375,
"KL/rejected_KL_mean": -184.8634796142578,
"KL/std": 87.85360717773438,
"epoch": 0.5381708238851096,
"fcm_dpo/beta": 0.00554704200476408,
"fcm_dpo/delta": 0.050387270748615265,
"fcm_dpo/margin": 64.53993225097656,
"fcm_dpo/q_t": 0.4200926423072815,
"grad_norm": 10.86155891418457,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 0.7344239950180054,
"logits/rejected": 0.6559726595878601,
"logps/chosen": -168.22970581054688,
"logps/ref_chosen": -47.906158447265625,
"logps/ref_rejected": -74.29397583007812,
"logps/rejected": -259.1574401855469,
"loss": 1.1515,
"margin_dpo/margin_mean": 64.53993225097656,
"margin_dpo/margin_std": 110.30122375488281,
"step": 356
},
{
"KL/chosen_KL_mean": -143.30943298339844,
"KL/mean": -168.89112854003906,
"KL/rejected_KL_mean": -194.47283935546875,
"KL/std": 87.86856079101562,
"epoch": 0.5396825396825397,
"fcm_dpo/beta": 0.005487706046551466,
"fcm_dpo/delta": -0.04394787177443504,
"fcm_dpo/margin": 51.163387298583984,
"fcm_dpo/q_t": 0.4319250285625458,
"grad_norm": 11.494694709777832,
"learning_rate": 2.6057459723762076e-07,
"logits/chosen": 0.7188966274261475,
"logits/rejected": 0.7018017768859863,
"logps/chosen": -205.94442749023438,
"logps/ref_chosen": -62.63500213623047,
"logps/ref_rejected": -65.11399841308594,
"logps/rejected": -259.5868225097656,
"loss": 1.2007,
"margin_dpo/margin_mean": 51.16339111328125,
"margin_dpo/margin_std": 96.90858459472656,
"step": 357
},
{
"KL/chosen_KL_mean": -141.5384063720703,
"KL/mean": -176.0528564453125,
"KL/rejected_KL_mean": -210.56729125976562,
"KL/std": 94.40840148925781,
"epoch": 0.5411942554799698,
"fcm_dpo/beta": 0.00527994055300951,
"fcm_dpo/delta": -0.07874049991369247,
"fcm_dpo/margin": 69.02886962890625,
"fcm_dpo/q_t": 0.41425493359565735,
"grad_norm": 14.46906852722168,
"learning_rate": 2.5925341972508954e-07,
"logits/chosen": 0.7024484872817993,
"logits/rejected": 0.7164376974105835,
"logps/chosen": -208.74801635742188,
"logps/ref_chosen": -67.20960998535156,
"logps/ref_rejected": -69.34715270996094,
"logps/rejected": -279.9144287109375,
"loss": 1.1371,
"margin_dpo/margin_mean": 69.02886962890625,
"margin_dpo/margin_std": 104.65023040771484,
"step": 358
},
{
"KL/chosen_KL_mean": -158.43338012695312,
"KL/mean": -176.73294067382812,
"KL/rejected_KL_mean": -195.03248596191406,
"KL/std": 92.38772583007812,
"epoch": 0.5427059712773998,
"fcm_dpo/beta": 0.0052369264885783195,
"fcm_dpo/delta": 0.06293975561857224,
"fcm_dpo/margin": 36.59911346435547,
"fcm_dpo/q_t": 0.4568448066711426,
"grad_norm": 13.132709503173828,
"learning_rate": 2.579319833745169e-07,
"logits/chosen": 0.703708291053772,
"logits/rejected": 0.6700050830841064,
"logps/chosen": -220.95916748046875,
"logps/ref_chosen": -62.52578353881836,
"logps/ref_rejected": -76.63114929199219,
"logps/rejected": -271.66363525390625,
"loss": 1.2728,
"margin_dpo/margin_mean": 36.59911346435547,
"margin_dpo/margin_std": 100.44338989257812,
"step": 359
},
{
"KL/chosen_KL_mean": -151.61996459960938,
"KL/mean": -185.01283264160156,
"KL/rejected_KL_mean": -218.40570068359375,
"KL/std": 98.46240234375,
"epoch": 0.54421768707483,
"fcm_dpo/beta": 0.005525579676032066,
"fcm_dpo/delta": 0.03805600479245186,
"fcm_dpo/margin": 66.78570556640625,
"fcm_dpo/q_t": 0.41693851351737976,
"grad_norm": 11.473810195922852,
"learning_rate": 2.5661032514931834e-07,
"logits/chosen": 0.6791380643844604,
"logits/rejected": 0.5924465656280518,
"logps/chosen": -215.10769653320312,
"logps/ref_chosen": -63.48772048950195,
"logps/ref_rejected": -90.6891098022461,
"logps/rejected": -309.09478759765625,
"loss": 1.1193,
"margin_dpo/margin_mean": 66.78571319580078,
"margin_dpo/margin_std": 97.07440185546875,
"step": 360
},
{
"KL/chosen_KL_mean": -142.7432861328125,
"KL/mean": -179.56118774414062,
"KL/rejected_KL_mean": -216.37908935546875,
"KL/std": 92.99285888671875,
"epoch": 0.54572940287226,
"fcm_dpo/beta": 0.005594981834292412,
"fcm_dpo/delta": -0.015271172858774662,
"fcm_dpo/margin": 73.63580322265625,
"fcm_dpo/q_t": 0.4040573239326477,
"grad_norm": 10.441182136535645,
"learning_rate": 2.552884820191154e-07,
"logits/chosen": 0.7554844617843628,
"logits/rejected": 0.7160251140594482,
"logps/chosen": -200.66043090820312,
"logps/ref_chosen": -57.917144775390625,
"logps/ref_rejected": -72.39089965820312,
"logps/rejected": -288.7699890136719,
"loss": 1.087,
"margin_dpo/margin_mean": 73.63580322265625,
"margin_dpo/margin_std": 97.2933349609375,
"step": 361
},
{
"KL/chosen_KL_mean": -144.31117248535156,
"KL/mean": -188.12466430664062,
"KL/rejected_KL_mean": -231.93817138671875,
"KL/std": 106.7471923828125,
"epoch": 0.54724111866969,
"fcm_dpo/beta": 0.005310633219778538,
"fcm_dpo/delta": -0.08533253520727158,
"fcm_dpo/margin": 87.62699890136719,
"fcm_dpo/q_t": 0.3944483995437622,
"grad_norm": 11.381577491760254,
"learning_rate": 2.53966490958702e-07,
"logits/chosen": 0.7894976139068604,
"logits/rejected": 0.692185640335083,
"logps/chosen": -207.754638671875,
"logps/ref_chosen": -63.4434700012207,
"logps/ref_rejected": -103.45516967773438,
"logps/rejected": -335.3933410644531,
"loss": 1.0723,
"margin_dpo/margin_mean": 87.62699890136719,
"margin_dpo/margin_std": 124.65602111816406,
"step": 362
},
{
"KL/chosen_KL_mean": -148.01797485351562,
"KL/mean": -187.11468505859375,
"KL/rejected_KL_mean": -226.21136474609375,
"KL/std": 92.74317932128906,
"epoch": 0.5487528344671202,
"fcm_dpo/beta": 0.0050812652334570885,
"fcm_dpo/delta": 0.0033413858618587255,
"fcm_dpo/margin": 78.19340515136719,
"fcm_dpo/q_t": 0.4094967842102051,
"grad_norm": 13.362162590026855,
"learning_rate": 2.526443889470099e-07,
"logits/chosen": 0.793647289276123,
"logits/rejected": 0.6671258211135864,
"logps/chosen": -196.6697998046875,
"logps/ref_chosen": -48.65182876586914,
"logps/ref_rejected": -88.65904235839844,
"logps/rejected": -314.87042236328125,
"loss": 1.0991,
"margin_dpo/margin_mean": 78.19339752197266,
"margin_dpo/margin_std": 109.21980285644531,
"step": 363
},
{
"KL/chosen_KL_mean": -131.89614868164062,
"KL/mean": -175.02871704101562,
"KL/rejected_KL_mean": -218.16127014160156,
"KL/std": 102.58001708984375,
"epoch": 0.5502645502645502,
"fcm_dpo/beta": 0.004936812911182642,
"fcm_dpo/delta": -0.033310696482658386,
"fcm_dpo/margin": 86.26512908935547,
"fcm_dpo/q_t": 0.40305817127227783,
"grad_norm": 9.358033180236816,
"learning_rate": 2.513222129660744e-07,
"logits/chosen": 0.698688268661499,
"logits/rejected": 0.6179659366607666,
"logps/chosen": -189.76722717285156,
"logps/ref_chosen": -57.87107467651367,
"logps/ref_rejected": -80.95503234863281,
"logps/rejected": -299.1163024902344,
"loss": 1.0997,
"margin_dpo/margin_mean": 86.26512145996094,
"margin_dpo/margin_std": 129.88619995117188,
"step": 364
},
{
"KL/chosen_KL_mean": -125.53724670410156,
"KL/mean": -165.1311798095703,
"KL/rejected_KL_mean": -204.72509765625,
"KL/std": 109.31954956054688,
"epoch": 0.5517762660619804,
"fcm_dpo/beta": 0.004892812110483646,
"fcm_dpo/delta": 0.014496054500341415,
"fcm_dpo/margin": 79.18785095214844,
"fcm_dpo/q_t": 0.41075021028518677,
"grad_norm": 10.14580249786377,
"learning_rate": 2.5e-07,
"logits/chosen": 0.745829701423645,
"logits/rejected": 0.7439219951629639,
"logps/chosen": -190.47943115234375,
"logps/ref_chosen": -64.94217681884766,
"logps/ref_rejected": -74.8599853515625,
"logps/rejected": -279.5850830078125,
"loss": 1.0869,
"margin_dpo/margin_mean": 79.18785095214844,
"margin_dpo/margin_std": 95.73055267333984,
"step": 365
},
{
"KL/chosen_KL_mean": -124.95919799804688,
"KL/mean": -157.53627014160156,
"KL/rejected_KL_mean": -190.11334228515625,
"KL/std": 95.9143295288086,
"epoch": 0.5532879818594104,
"fcm_dpo/beta": 0.005323389545083046,
"fcm_dpo/delta": 0.06161736696958542,
"fcm_dpo/margin": 65.1541519165039,
"fcm_dpo/q_t": 0.4233441948890686,
"grad_norm": 13.165140151977539,
"learning_rate": 2.486777870339255e-07,
"logits/chosen": 0.6843109130859375,
"logits/rejected": 0.6801989078521729,
"logps/chosen": -180.12518310546875,
"logps/ref_chosen": -55.16598129272461,
"logps/ref_rejected": -65.26121520996094,
"logps/rejected": -255.3745574951172,
"loss": 1.1685,
"margin_dpo/margin_mean": 65.1541519165039,
"margin_dpo/margin_std": 121.73723602294922,
"step": 366
},
{
"KL/chosen_KL_mean": -143.6685791015625,
"KL/mean": -176.48419189453125,
"KL/rejected_KL_mean": -209.2998046875,
"KL/std": 95.0160903930664,
"epoch": 0.5547996976568406,
"fcm_dpo/beta": 0.005490441806614399,
"fcm_dpo/delta": 0.048465244472026825,
"fcm_dpo/margin": 65.6312255859375,
"fcm_dpo/q_t": 0.4174492359161377,
"grad_norm": 12.372570991516113,
"learning_rate": 2.4735561105299014e-07,
"logits/chosen": 0.7515787482261658,
"logits/rejected": 0.6537929773330688,
"logps/chosen": -199.67904663085938,
"logps/ref_chosen": -56.01046371459961,
"logps/ref_rejected": -77.31010437011719,
"logps/rejected": -286.60992431640625,
"loss": 1.1306,
"margin_dpo/margin_mean": 65.6312255859375,
"margin_dpo/margin_std": 100.20790100097656,
"step": 367
},
{
"KL/chosen_KL_mean": -156.02073669433594,
"KL/mean": -185.53570556640625,
"KL/rejected_KL_mean": -215.05062866210938,
"KL/std": 93.59857177734375,
"epoch": 0.5563114134542706,
"fcm_dpo/beta": 0.005780104547739029,
"fcm_dpo/delta": 0.07085883617401123,
"fcm_dpo/margin": 59.02989959716797,
"fcm_dpo/q_t": 0.4238590598106384,
"grad_norm": 13.36587142944336,
"learning_rate": 2.46033509041298e-07,
"logits/chosen": 0.5935603380203247,
"logits/rejected": 0.5959989428520203,
"logps/chosen": -230.8500213623047,
"logps/ref_chosen": -74.82927703857422,
"logps/ref_rejected": -76.11680603027344,
"logps/rejected": -291.16741943359375,
"loss": 1.1574,
"margin_dpo/margin_mean": 59.02989959716797,
"margin_dpo/margin_std": 100.53662109375,
"step": 368
},
{
"KL/chosen_KL_mean": -142.60076904296875,
"KL/mean": -169.75457763671875,
"KL/rejected_KL_mean": -196.90835571289062,
"KL/std": 94.11172485351562,
"epoch": 0.5578231292517006,
"fcm_dpo/beta": 0.006405709311366081,
"fcm_dpo/delta": 0.058929868042469025,
"fcm_dpo/margin": 54.307586669921875,
"fcm_dpo/q_t": 0.42363277077674866,
"grad_norm": 13.711212158203125,
"learning_rate": 2.447115179808846e-07,
"logits/chosen": 0.7638058662414551,
"logits/rejected": 0.7166075110435486,
"logps/chosen": -200.927001953125,
"logps/ref_chosen": -58.32621765136719,
"logps/ref_rejected": -80.92183685302734,
"logps/rejected": -277.8302001953125,
"loss": 1.1666,
"margin_dpo/margin_mean": 54.307586669921875,
"margin_dpo/margin_std": 100.56336975097656,
"step": 369
},
{
"KL/chosen_KL_mean": -135.7542724609375,
"KL/mean": -175.9358673095703,
"KL/rejected_KL_mean": -216.11746215820312,
"KL/std": 98.13227844238281,
"epoch": 0.5593348450491308,
"fcm_dpo/beta": 0.005805108696222305,
"fcm_dpo/delta": -0.09162604808807373,
"fcm_dpo/margin": 80.36318969726562,
"fcm_dpo/q_t": 0.38908275961875916,
"grad_norm": 12.54875659942627,
"learning_rate": 2.4338967485068164e-07,
"logits/chosen": 0.854151725769043,
"logits/rejected": 0.7885351777076721,
"logps/chosen": -188.6379852294922,
"logps/ref_chosen": -52.88372039794922,
"logps/ref_rejected": -79.43692016601562,
"logps/rejected": -295.55438232421875,
"loss": 1.076,
"margin_dpo/margin_mean": 80.36318969726562,
"margin_dpo/margin_std": 114.33674621582031,
"step": 370
},
{
"KL/chosen_KL_mean": -134.52255249023438,
"KL/mean": -168.52369689941406,
"KL/rejected_KL_mean": -202.5248565673828,
"KL/std": 92.02737426757812,
"epoch": 0.5608465608465608,
"fcm_dpo/beta": 0.00593863520771265,
"fcm_dpo/delta": -0.008677050471305847,
"fcm_dpo/margin": 68.00228881835938,
"fcm_dpo/q_t": 0.4075172543525696,
"grad_norm": 16.5703182220459,
"learning_rate": 2.420680166254831e-07,
"logits/chosen": 0.8438094258308411,
"logits/rejected": 0.8158732652664185,
"logps/chosen": -183.74676513671875,
"logps/ref_chosen": -49.224212646484375,
"logps/ref_rejected": -63.348472595214844,
"logps/rejected": -265.8733215332031,
"loss": 1.1141,
"margin_dpo/margin_mean": 68.00228118896484,
"margin_dpo/margin_std": 102.23562622070312,
"step": 371
},
{
"KL/chosen_KL_mean": -142.50531005859375,
"KL/mean": -162.75030517578125,
"KL/rejected_KL_mean": -182.99530029296875,
"KL/std": 92.52728271484375,
"epoch": 0.562358276643991,
"fcm_dpo/beta": 0.005816627759486437,
"fcm_dpo/delta": 0.02477392926812172,
"fcm_dpo/margin": 40.48997497558594,
"fcm_dpo/q_t": 0.44827184081077576,
"grad_norm": 14.055017471313477,
"learning_rate": 2.4074658027491044e-07,
"logits/chosen": 0.7998120188713074,
"logits/rejected": 0.7064263820648193,
"logps/chosen": -194.77487182617188,
"logps/ref_chosen": -52.269554138183594,
"logps/ref_rejected": -72.99522399902344,
"logps/rejected": -255.99050903320312,
"loss": 1.2831,
"margin_dpo/margin_mean": 40.48997497558594,
"margin_dpo/margin_std": 119.91127014160156,
"step": 372
},
{
"KL/chosen_KL_mean": -160.78884887695312,
"KL/mean": -188.75387573242188,
"KL/rejected_KL_mean": -216.71890258789062,
"KL/std": 100.0822982788086,
"epoch": 0.563869992441421,
"fcm_dpo/beta": 0.006202812306582928,
"fcm_dpo/delta": 0.06371521204710007,
"fcm_dpo/margin": 55.93007278442383,
"fcm_dpo/q_t": 0.4251997470855713,
"grad_norm": 13.777511596679688,
"learning_rate": 2.394254027623792e-07,
"logits/chosen": 0.7657662034034729,
"logits/rejected": 0.7004691362380981,
"logps/chosen": -221.90184020996094,
"logps/ref_chosen": -61.112998962402344,
"logps/ref_rejected": -76.24851989746094,
"logps/rejected": -292.9674072265625,
"loss": 1.2048,
"margin_dpo/margin_mean": 55.93006896972656,
"margin_dpo/margin_std": 123.24330139160156,
"step": 373
},
{
"KL/chosen_KL_mean": -137.24179077148438,
"KL/mean": -181.21661376953125,
"KL/rejected_KL_mean": -225.19142150878906,
"KL/std": 102.51628112792969,
"epoch": 0.5653817082388511,
"fcm_dpo/beta": 0.005645962432026863,
"fcm_dpo/delta": -0.13082769513130188,
"fcm_dpo/margin": 87.94963073730469,
"fcm_dpo/q_t": 0.3829165995121002,
"grad_norm": 13.66327953338623,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 0.6615780591964722,
"logits/rejected": 0.6669573783874512,
"logps/chosen": -209.91099548339844,
"logps/ref_chosen": -72.66920471191406,
"logps/ref_rejected": -76.83158874511719,
"logps/rejected": -302.02301025390625,
"loss": 1.0671,
"margin_dpo/margin_mean": 87.94963073730469,
"margin_dpo/margin_std": 126.89126586914062,
"step": 374
},
{
"KL/chosen_KL_mean": -135.42027282714844,
"KL/mean": -167.710693359375,
"KL/rejected_KL_mean": -200.0010986328125,
"KL/std": 111.11065673828125,
"epoch": 0.5668934240362812,
"fcm_dpo/beta": 0.005644750315696001,
"fcm_dpo/delta": 0.043448496609926224,
"fcm_dpo/margin": 64.58082580566406,
"fcm_dpo/q_t": 0.42045366764068604,
"grad_norm": 15.049660682678223,
"learning_rate": 2.3678397206786715e-07,
"logits/chosen": 0.76460200548172,
"logits/rejected": 0.7118662595748901,
"logps/chosen": -193.10357666015625,
"logps/ref_chosen": -57.68330383300781,
"logps/ref_rejected": -79.34097290039062,
"logps/rejected": -279.3420715332031,
"loss": 1.1656,
"margin_dpo/margin_mean": 64.58082580566406,
"margin_dpo/margin_std": 122.09146118164062,
"step": 375
},
{
"KL/chosen_KL_mean": -143.47830200195312,
"KL/mean": -183.77133178710938,
"KL/rejected_KL_mean": -224.06434631347656,
"KL/std": 112.19872283935547,
"epoch": 0.5684051398337112,
"fcm_dpo/beta": 0.0054377606138587,
"fcm_dpo/delta": -0.0515349879860878,
"fcm_dpo/margin": 80.58604431152344,
"fcm_dpo/q_t": 0.4034051299095154,
"grad_norm": 12.587312698364258,
"learning_rate": 2.3546379277238103e-07,
"logits/chosen": 0.8178266286849976,
"logits/rejected": 0.754265308380127,
"logps/chosen": -195.15237426757812,
"logps/ref_chosen": -51.674072265625,
"logps/ref_rejected": -75.69713592529297,
"logps/rejected": -299.761474609375,
"loss": 1.1133,
"margin_dpo/margin_mean": 80.58604431152344,
"margin_dpo/margin_std": 133.2933349609375,
"step": 376
},
{
"KL/chosen_KL_mean": -144.62603759765625,
"KL/mean": -172.3679962158203,
"KL/rejected_KL_mean": -200.10995483398438,
"KL/std": 88.5212631225586,
"epoch": 0.5699168556311414,
"fcm_dpo/beta": 0.005908847786486149,
"fcm_dpo/delta": 0.08338885009288788,
"fcm_dpo/margin": 55.483924865722656,
"fcm_dpo/q_t": 0.4264184236526489,
"grad_norm": 13.982069969177246,
"learning_rate": 2.3414402008585886e-07,
"logits/chosen": 0.7420355081558228,
"logits/rejected": 0.7195169925689697,
"logps/chosen": -190.8045654296875,
"logps/ref_chosen": -46.17853546142578,
"logps/ref_rejected": -57.756500244140625,
"logps/rejected": -257.866455078125,
"loss": 1.1684,
"margin_dpo/margin_mean": 55.48392868041992,
"margin_dpo/margin_std": 100.01844787597656,
"step": 377
},
{
"KL/chosen_KL_mean": -142.30294799804688,
"KL/mean": -169.65269470214844,
"KL/rejected_KL_mean": -197.00244140625,
"KL/std": 103.02887725830078,
"epoch": 0.5714285714285714,
"fcm_dpo/beta": 0.006364922039210796,
"fcm_dpo/delta": 0.057820893824100494,
"fcm_dpo/margin": 54.6994743347168,
"fcm_dpo/q_t": 0.4231072664260864,
"grad_norm": 11.828926086425781,
"learning_rate": 2.3282469092517977e-07,
"logits/chosen": 0.8158556222915649,
"logits/rejected": 0.7710492610931396,
"logps/chosen": -201.52182006835938,
"logps/ref_chosen": -59.21887969970703,
"logps/ref_rejected": -71.24818420410156,
"logps/rejected": -268.2506103515625,
"loss": 1.1709,
"margin_dpo/margin_mean": 54.69947814941406,
"margin_dpo/margin_std": 101.46234130859375,
"step": 378
},
{
"KL/chosen_KL_mean": -144.59205627441406,
"KL/mean": -184.87893676757812,
"KL/rejected_KL_mean": -225.16583251953125,
"KL/std": 108.66886138916016,
"epoch": 0.5729402872260015,
"fcm_dpo/beta": 0.0058668795973062515,
"fcm_dpo/delta": -0.09539224952459335,
"fcm_dpo/margin": 80.57376098632812,
"fcm_dpo/q_t": 0.3924974203109741,
"grad_norm": 14.25523567199707,
"learning_rate": 2.3150584219481643e-07,
"logits/chosen": 0.7613098621368408,
"logits/rejected": 0.6914204955101013,
"logps/chosen": -220.9086456298828,
"logps/ref_chosen": -76.31658935546875,
"logps/ref_rejected": -104.26200103759766,
"logps/rejected": -329.4278259277344,
"loss": 1.0754,
"margin_dpo/margin_mean": 80.57376098632812,
"margin_dpo/margin_std": 117.86245727539062,
"step": 379
},
{
"KL/chosen_KL_mean": -121.277099609375,
"KL/mean": -164.54278564453125,
"KL/rejected_KL_mean": -207.80848693847656,
"KL/std": 95.66365051269531,
"epoch": 0.5744520030234316,
"fcm_dpo/beta": 0.005375675857067108,
"fcm_dpo/delta": -0.08500650525093079,
"fcm_dpo/margin": 86.53138732910156,
"fcm_dpo/q_t": 0.38998135924339294,
"grad_norm": 12.326929092407227,
"learning_rate": 2.3018751078580283e-07,
"logits/chosen": 0.7833170890808105,
"logits/rejected": 0.7477537393569946,
"logps/chosen": -182.56027221679688,
"logps/ref_chosen": -61.283164978027344,
"logps/ref_rejected": -72.38892364501953,
"logps/rejected": -280.1973876953125,
"loss": 1.0612,
"margin_dpo/margin_mean": 86.5313949584961,
"margin_dpo/margin_std": 113.20811462402344,
"step": 380
},
{
"KL/chosen_KL_mean": -149.11305236816406,
"KL/mean": -165.54931640625,
"KL/rejected_KL_mean": -181.98558044433594,
"KL/std": 100.11683654785156,
"epoch": 0.5759637188208617,
"fcm_dpo/beta": 0.0054644690826535225,
"fcm_dpo/delta": 0.07773777097463608,
"fcm_dpo/margin": 32.87253189086914,
"fcm_dpo/q_t": 0.4607895016670227,
"grad_norm": 12.004046440124512,
"learning_rate": 2.288697335747027e-07,
"logits/chosen": 0.7426190376281738,
"logits/rejected": 0.7275179624557495,
"logps/chosen": -207.32704162597656,
"logps/ref_chosen": -58.2139892578125,
"logps/ref_rejected": -60.78669357299805,
"logps/rejected": -242.77227783203125,
"loss": 1.3172,
"margin_dpo/margin_mean": 32.872535705566406,
"margin_dpo/margin_std": 121.07296752929688,
"step": 381
},
{
"KL/chosen_KL_mean": -149.33331298828125,
"KL/mean": -180.80596923828125,
"KL/rejected_KL_mean": -212.27862548828125,
"KL/std": 100.90338134765625,
"epoch": 0.5774754346182918,
"fcm_dpo/beta": 0.005938488524407148,
"fcm_dpo/delta": 0.02896289899945259,
"fcm_dpo/margin": 62.9453125,
"fcm_dpo/q_t": 0.41630202531814575,
"grad_norm": 14.264102935791016,
"learning_rate": 2.2755254742257706e-07,
"logits/chosen": 0.7569193243980408,
"logits/rejected": 0.7074561715126038,
"logps/chosen": -211.15863037109375,
"logps/ref_chosen": -61.82532501220703,
"logps/ref_rejected": -83.0452880859375,
"logps/rejected": -295.32391357421875,
"loss": 1.1309,
"margin_dpo/margin_mean": 62.9453125,
"margin_dpo/margin_std": 99.38772583007812,
"step": 382
},
{
"KL/chosen_KL_mean": -153.87950134277344,
"KL/mean": -180.53121948242188,
"KL/rejected_KL_mean": -207.18296813964844,
"KL/std": 100.1922607421875,
"epoch": 0.5789871504157218,
"fcm_dpo/beta": 0.006136381067335606,
"fcm_dpo/delta": 0.08745638281106949,
"fcm_dpo/margin": 53.30347442626953,
"fcm_dpo/q_t": 0.4308212995529175,
"grad_norm": 16.983381271362305,
"learning_rate": 2.2623598917395436e-07,
"logits/chosen": 0.6574628353118896,
"logits/rejected": 0.6930520534515381,
"logps/chosen": -234.4427490234375,
"logps/ref_chosen": -80.56326293945312,
"logps/ref_rejected": -74.62922668457031,
"logps/rejected": -281.81219482421875,
"loss": 1.2261,
"margin_dpo/margin_mean": 53.30347442626953,
"margin_dpo/margin_std": 126.07339477539062,
"step": 383
},
{
"KL/chosen_KL_mean": -140.81640625,
"KL/mean": -174.13668823242188,
"KL/rejected_KL_mean": -207.45697021484375,
"KL/std": 96.11679077148438,
"epoch": 0.5804988662131519,
"fcm_dpo/beta": 0.006302967667579651,
"fcm_dpo/delta": -0.025382589548826218,
"fcm_dpo/margin": 66.64056396484375,
"fcm_dpo/q_t": 0.4019022583961487,
"grad_norm": 16.735795974731445,
"learning_rate": 2.2492009565579875e-07,
"logits/chosen": 0.777447521686554,
"logits/rejected": 0.7444401979446411,
"logps/chosen": -206.2915496826172,
"logps/ref_chosen": -65.47514343261719,
"logps/ref_rejected": -79.67378234863281,
"logps/rejected": -287.1307373046875,
"loss": 1.1074,
"margin_dpo/margin_mean": 66.64056396484375,
"margin_dpo/margin_std": 102.94216918945312,
"step": 384
},
{
"KL/chosen_KL_mean": -142.68655395507812,
"KL/mean": -185.57028198242188,
"KL/rejected_KL_mean": -228.4540252685547,
"KL/std": 101.47017669677734,
"epoch": 0.582010582010582,
"fcm_dpo/beta": 0.005779832135885954,
"fcm_dpo/delta": -0.12640322744846344,
"fcm_dpo/margin": 85.7674789428711,
"fcm_dpo/q_t": 0.38280969858169556,
"grad_norm": 12.516278266906738,
"learning_rate": 2.2360490367648084e-07,
"logits/chosen": 0.6712303161621094,
"logits/rejected": 0.6324940919876099,
"logps/chosen": -208.74307250976562,
"logps/ref_chosen": -66.0565185546875,
"logps/ref_rejected": -86.68023681640625,
"logps/rejected": -315.13427734375,
"loss": 1.038,
"margin_dpo/margin_mean": 85.7674789428711,
"margin_dpo/margin_std": 104.35530090332031,
"step": 385
},
{
"KL/chosen_KL_mean": -165.44119262695312,
"KL/mean": -193.70352172851562,
"KL/rejected_KL_mean": -221.9658203125,
"KL/std": 107.32279968261719,
"epoch": 0.5835222978080121,
"fcm_dpo/beta": 0.0058027347549796104,
"fcm_dpo/delta": 0.08648616075515747,
"fcm_dpo/margin": 56.52465057373047,
"fcm_dpo/q_t": 0.42700403928756714,
"grad_norm": 13.528538703918457,
"learning_rate": 2.2229045002474724e-07,
"logits/chosen": 0.6971240043640137,
"logits/rejected": 0.6468753814697266,
"logps/chosen": -241.06484985351562,
"logps/ref_chosen": -75.6236572265625,
"logps/ref_rejected": -92.62330627441406,
"logps/rejected": -314.5891418457031,
"loss": 1.1801,
"margin_dpo/margin_mean": 56.52465057373047,
"margin_dpo/margin_std": 108.72563171386719,
"step": 386
},
{
"KL/chosen_KL_mean": -144.2526397705078,
"KL/mean": -184.9021759033203,
"KL/rejected_KL_mean": -225.5517120361328,
"KL/std": 96.83357238769531,
"epoch": 0.5850340136054422,
"fcm_dpo/beta": 0.005610157269984484,
"fcm_dpo/delta": -0.07386106252670288,
"fcm_dpo/margin": 81.29905700683594,
"fcm_dpo/q_t": 0.3932124376296997,
"grad_norm": 14.18612003326416,
"learning_rate": 2.209767714686924e-07,
"logits/chosen": 0.7909537553787231,
"logits/rejected": 0.6846780776977539,
"logps/chosen": -191.47434997558594,
"logps/ref_chosen": -47.22170639038086,
"logps/ref_rejected": -87.338134765625,
"logps/rejected": -312.88983154296875,
"loss": 1.0565,
"margin_dpo/margin_mean": 81.29905700683594,
"margin_dpo/margin_std": 101.80584716796875,
"step": 387
},
{
"KL/chosen_KL_mean": -150.70068359375,
"KL/mean": -175.45492553710938,
"KL/rejected_KL_mean": -200.20916748046875,
"KL/std": 103.48775482177734,
"epoch": 0.5865457294028723,
"fcm_dpo/beta": 0.005595776252448559,
"fcm_dpo/delta": 0.008372929878532887,
"fcm_dpo/margin": 49.50847244262695,
"fcm_dpo/q_t": 0.43925899267196655,
"grad_norm": 12.53011417388916,
"learning_rate": 2.1966390475472954e-07,
"logits/chosen": 0.7262797355651855,
"logits/rejected": 0.7237277626991272,
"logps/chosen": -225.2801513671875,
"logps/ref_chosen": -74.5794677734375,
"logps/ref_rejected": -79.92558288574219,
"logps/rejected": -280.134765625,
"loss": 1.2396,
"margin_dpo/margin_mean": 49.50847244262695,
"margin_dpo/margin_std": 122.52720642089844,
"step": 388
},
{
"KL/chosen_KL_mean": -141.07130432128906,
"KL/mean": -181.15814208984375,
"KL/rejected_KL_mean": -221.24497985839844,
"KL/std": 97.66059112548828,
"epoch": 0.5880574452003023,
"fcm_dpo/beta": 0.005440755747258663,
"fcm_dpo/delta": -0.04618635028600693,
"fcm_dpo/margin": 80.17366027832031,
"fcm_dpo/q_t": 0.39910855889320374,
"grad_norm": 18.9296932220459,
"learning_rate": 2.1835188660656265e-07,
"logits/chosen": 0.7523651719093323,
"logits/rejected": 0.7214106917381287,
"logps/chosen": -202.6956787109375,
"logps/ref_chosen": -61.624366760253906,
"logps/ref_rejected": -76.50978088378906,
"logps/rejected": -297.7547607421875,
"loss": 1.077,
"margin_dpo/margin_mean": 80.17366027832031,
"margin_dpo/margin_std": 107.77185821533203,
"step": 389
},
{
"KL/chosen_KL_mean": -125.87141418457031,
"KL/mean": -156.2007293701172,
"KL/rejected_KL_mean": -186.530029296875,
"KL/std": 93.57325744628906,
"epoch": 0.5895691609977324,
"fcm_dpo/beta": 0.005581664387136698,
"fcm_dpo/delta": 0.07422532886266708,
"fcm_dpo/margin": 60.65862274169922,
"fcm_dpo/q_t": 0.42511534690856934,
"grad_norm": 11.519083023071289,
"learning_rate": 2.170407537241599e-07,
"logits/chosen": 0.8347935676574707,
"logits/rejected": 0.7659469246864319,
"logps/chosen": -171.7432861328125,
"logps/ref_chosen": -45.871864318847656,
"logps/ref_rejected": -61.305999755859375,
"logps/rejected": -247.83602905273438,
"loss": 1.1511,
"margin_dpo/margin_mean": 60.658626556396484,
"margin_dpo/margin_std": 100.20620727539062,
"step": 390
},
{
"KL/chosen_KL_mean": -141.71585083007812,
"KL/mean": -179.6748046875,
"KL/rejected_KL_mean": -217.63375854492188,
"KL/std": 97.35914611816406,
"epoch": 0.5910808767951625,
"fcm_dpo/beta": 0.00556203443557024,
"fcm_dpo/delta": -0.02935415506362915,
"fcm_dpo/margin": 75.91790771484375,
"fcm_dpo/q_t": 0.4031655788421631,
"grad_norm": 12.19450569152832,
"learning_rate": 2.1573054278272636e-07,
"logits/chosen": 0.7507619857788086,
"logits/rejected": 0.6888763308525085,
"logps/chosen": -199.90286254882812,
"logps/ref_chosen": -58.18701171875,
"logps/ref_rejected": -83.63442993164062,
"logps/rejected": -301.2681884765625,
"loss": 1.1082,
"margin_dpo/margin_mean": 75.91790771484375,
"margin_dpo/margin_std": 116.9903793334961,
"step": 391
},
{
"KL/chosen_KL_mean": -130.40484619140625,
"KL/mean": -169.21307373046875,
"KL/rejected_KL_mean": -208.0213165283203,
"KL/std": 106.27191925048828,
"epoch": 0.5925925925925926,
"fcm_dpo/beta": 0.005578967742621899,
"fcm_dpo/delta": -0.04480854421854019,
"fcm_dpo/margin": 77.61647033691406,
"fcm_dpo/q_t": 0.4019958972930908,
"grad_norm": 12.193196296691895,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 0.8328498601913452,
"logits/rejected": 0.7778192162513733,
"logps/chosen": -200.1493682861328,
"logps/ref_chosen": -69.7445297241211,
"logps/ref_rejected": -94.05877685546875,
"logps/rejected": -302.080078125,
"loss": 1.1104,
"margin_dpo/margin_mean": 77.61647033691406,
"margin_dpo/margin_std": 123.52142333984375,
"step": 392
},
{
"KL/chosen_KL_mean": -145.96200561523438,
"KL/mean": -187.2437286376953,
"KL/rejected_KL_mean": -228.52545166015625,
"KL/std": 102.85442352294922,
"epoch": 0.5941043083900227,
"fcm_dpo/beta": 0.005069409962743521,
"fcm_dpo/delta": -0.02902321144938469,
"fcm_dpo/margin": 82.56344604492188,
"fcm_dpo/q_t": 0.40151870250701904,
"grad_norm": 10.115791320800781,
"learning_rate": 2.131130332936195e-07,
"logits/chosen": 0.7979670763015747,
"logits/rejected": 0.7620906829833984,
"logps/chosen": -198.29690551757812,
"logps/ref_chosen": -52.33489990234375,
"logps/ref_rejected": -74.33809661865234,
"logps/rejected": -302.8635559082031,
"loss": 1.0764,
"margin_dpo/margin_mean": 82.56344604492188,
"margin_dpo/margin_std": 100.86908721923828,
"step": 393
},
{
"KL/chosen_KL_mean": -139.4735870361328,
"KL/mean": -175.12521362304688,
"KL/rejected_KL_mean": -210.77684020996094,
"KL/std": 96.35044860839844,
"epoch": 0.5956160241874527,
"fcm_dpo/beta": 0.005368366837501526,
"fcm_dpo/delta": 0.019182713702321053,
"fcm_dpo/margin": 71.30326080322266,
"fcm_dpo/q_t": 0.41120874881744385,
"grad_norm": 11.45832347869873,
"learning_rate": 2.1180580796331323e-07,
"logits/chosen": 0.7915951013565063,
"logits/rejected": 0.7640597224235535,
"logps/chosen": -200.14971923828125,
"logps/ref_chosen": -60.6761360168457,
"logps/ref_rejected": -71.36074829101562,
"logps/rejected": -282.1375732421875,
"loss": 1.1046,
"margin_dpo/margin_mean": 71.30325317382812,
"margin_dpo/margin_std": 95.12825012207031,
"step": 394
},
{
"KL/chosen_KL_mean": -140.49183654785156,
"KL/mean": -175.02513122558594,
"KL/rejected_KL_mean": -209.55844116210938,
"KL/std": 99.29302978515625,
"epoch": 0.5971277399848829,
"fcm_dpo/beta": 0.0054627228528261185,
"fcm_dpo/delta": 0.026971735060214996,
"fcm_dpo/margin": 69.06660461425781,
"fcm_dpo/q_t": 0.4158324599266052,
"grad_norm": 12.025148391723633,
"learning_rate": 2.104996510066625e-07,
"logits/chosen": 0.8082689046859741,
"logits/rejected": 0.7192596197128296,
"logps/chosen": -191.09616088867188,
"logps/ref_chosen": -50.60432434082031,
"logps/ref_rejected": -77.08731079101562,
"logps/rejected": -286.645751953125,
"loss": 1.1223,
"margin_dpo/margin_mean": 69.06660461425781,
"margin_dpo/margin_std": 106.44404602050781,
"step": 395
},
{
"KL/chosen_KL_mean": -136.07615661621094,
"KL/mean": -170.98126220703125,
"KL/rejected_KL_mean": -205.88636779785156,
"KL/std": 107.50096130371094,
"epoch": 0.5986394557823129,
"fcm_dpo/beta": 0.005069066770374775,
"fcm_dpo/delta": -0.06986290216445923,
"fcm_dpo/margin": 69.81021118164062,
"fcm_dpo/q_t": 0.41486620903015137,
"grad_norm": 10.510855674743652,
"learning_rate": 2.0919459895968517e-07,
"logits/chosen": 0.7915246486663818,
"logits/rejected": 0.7065383195877075,
"logps/chosen": -187.43576049804688,
"logps/ref_chosen": -51.35961151123047,
"logps/ref_rejected": -79.89360046386719,
"logps/rejected": -285.77996826171875,
"loss": 1.1229,
"margin_dpo/margin_mean": 69.81021118164062,
"margin_dpo/margin_std": 94.96488952636719,
"step": 396
},
{
"KL/chosen_KL_mean": -157.47796630859375,
"KL/mean": -177.4036102294922,
"KL/rejected_KL_mean": -197.32923889160156,
"KL/std": 99.39573669433594,
"epoch": 0.600151171579743,
"fcm_dpo/beta": 0.005069066770374775,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 39.851280212402344,
"fcm_dpo/q_t": 0.4533647298812866,
"grad_norm": 11.057273864746094,
"learning_rate": 2.078906883274924e-07,
"logits/chosen": 0.7363402247428894,
"logits/rejected": 0.6861610412597656,
"logps/chosen": -223.9342041015625,
"logps/ref_chosen": -66.45622253417969,
"logps/ref_rejected": -85.74736785888672,
"logps/rejected": -283.07659912109375,
"loss": 1.2732,
"margin_dpo/margin_mean": 39.851280212402344,
"margin_dpo/margin_std": 111.93646240234375,
"step": 397
},
{
"KL/chosen_KL_mean": -131.88458251953125,
"KL/mean": -177.76821899414062,
"KL/rejected_KL_mean": -223.65185546875,
"KL/std": 103.11227416992188,
"epoch": 0.6016628873771731,
"fcm_dpo/beta": 0.00475926510989666,
"fcm_dpo/delta": -0.05145275592803955,
"fcm_dpo/margin": 91.76725006103516,
"fcm_dpo/q_t": 0.3985298275947571,
"grad_norm": 9.738566398620605,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.7597224712371826,
"logits/rejected": 0.701553225517273,
"logps/chosen": -181.1288299560547,
"logps/ref_chosen": -49.244239807128906,
"logps/ref_rejected": -75.18949127197266,
"logps/rejected": -298.8413391113281,
"loss": 1.0615,
"margin_dpo/margin_mean": 91.76725006103516,
"margin_dpo/margin_std": 111.49891662597656,
"step": 398
},
{
"KL/chosen_KL_mean": -156.69384765625,
"KL/mean": -210.0038604736328,
"KL/rejected_KL_mean": -263.3138732910156,
"KL/std": 118.74224853515625,
"epoch": 0.6031746031746031,
"fcm_dpo/beta": 0.004380588419735432,
"fcm_dpo/delta": -0.09351267665624619,
"fcm_dpo/margin": 106.62001037597656,
"fcm_dpo/q_t": 0.3906853199005127,
"grad_norm": 9.792754173278809,
"learning_rate": 2.052864371672457e-07,
"logits/chosen": 0.7528105974197388,
"logits/rejected": 0.6174201965332031,
"logps/chosen": -225.00064086914062,
"logps/ref_chosen": -68.30679321289062,
"logps/ref_rejected": -113.2708511352539,
"logps/rejected": -376.584716796875,
"loss": 1.0513,
"margin_dpo/margin_mean": 106.62001037597656,
"margin_dpo/margin_std": 135.46578979492188,
"step": 399
},
{
"KL/chosen_KL_mean": -173.648681640625,
"KL/mean": -203.43667602539062,
"KL/rejected_KL_mean": -233.22470092773438,
"KL/std": 106.72013854980469,
"epoch": 0.6046863189720333,
"fcm_dpo/beta": 0.004395931493490934,
"fcm_dpo/delta": 0.002488115569576621,
"fcm_dpo/margin": 59.576026916503906,
"fcm_dpo/q_t": 0.4374983310699463,
"grad_norm": 13.163859367370605,
"learning_rate": 2.0398616948569493e-07,
"logits/chosen": 0.7592464685440063,
"logits/rejected": 0.703935980796814,
"logps/chosen": -245.27517700195312,
"logps/ref_chosen": -71.62649536132812,
"logps/ref_rejected": -90.98765563964844,
"logps/rejected": -324.21234130859375,
"loss": 1.1938,
"margin_dpo/margin_mean": 59.57603073120117,
"margin_dpo/margin_std": 100.98155212402344,
"step": 400
},
{
"KL/chosen_KL_mean": -131.31045532226562,
"KL/mean": -173.59060668945312,
"KL/rejected_KL_mean": -215.87075805664062,
"KL/std": 113.15923309326172,
"epoch": 0.6061980347694633,
"fcm_dpo/beta": 0.004426930099725723,
"fcm_dpo/delta": 0.030951913446187973,
"fcm_dpo/margin": 84.5603256225586,
"fcm_dpo/q_t": 0.4145042300224304,
"grad_norm": 7.4008402824401855,
"learning_rate": 2.0268718890989752e-07,
"logits/chosen": 0.7799571752548218,
"logits/rejected": 0.7018869519233704,
"logps/chosen": -185.035400390625,
"logps/ref_chosen": -53.72495651245117,
"logps/ref_rejected": -75.06304931640625,
"logps/rejected": -290.9338073730469,
"loss": 1.1048,
"margin_dpo/margin_mean": 84.5603256225586,
"margin_dpo/margin_std": 110.60939025878906,
"step": 401
},
{
"KL/chosen_KL_mean": -145.2092742919922,
"KL/mean": -177.13595581054688,
"KL/rejected_KL_mean": -209.06265258789062,
"KL/std": 94.68934631347656,
"epoch": 0.6077097505668935,
"fcm_dpo/beta": 0.00457565626129508,
"fcm_dpo/delta": 0.009118745103478432,
"fcm_dpo/margin": 63.85340118408203,
"fcm_dpo/q_t": 0.4322190284729004,
"grad_norm": 11.485504150390625,
"learning_rate": 2.013895317751323e-07,
"logits/chosen": 0.7673013210296631,
"logits/rejected": 0.7580888271331787,
"logps/chosen": -207.08319091796875,
"logps/ref_chosen": -61.873931884765625,
"logps/ref_rejected": -66.15198516845703,
"logps/rejected": -275.21466064453125,
"loss": 1.1854,
"margin_dpo/margin_mean": 63.85340118408203,
"margin_dpo/margin_std": 115.97992706298828,
"step": 402
},
{
"KL/chosen_KL_mean": -151.53750610351562,
"KL/mean": -193.7296142578125,
"KL/rejected_KL_mean": -235.92172241210938,
"KL/std": 111.84996032714844,
"epoch": 0.6092214663643235,
"fcm_dpo/beta": 0.004578583873808384,
"fcm_dpo/delta": 0.01650870591402054,
"fcm_dpo/margin": 84.38421630859375,
"fcm_dpo/q_t": 0.412356436252594,
"grad_norm": 8.309311866760254,
"learning_rate": 2.0009323437965898e-07,
"logits/chosen": 0.882099986076355,
"logits/rejected": 0.7999458312988281,
"logps/chosen": -202.8590087890625,
"logps/ref_chosen": -51.321502685546875,
"logps/ref_rejected": -86.54010772705078,
"logps/rejected": -322.46185302734375,
"loss": 1.1045,
"margin_dpo/margin_mean": 84.38421630859375,
"margin_dpo/margin_std": 116.1924819946289,
"step": 403
},
{
"KL/chosen_KL_mean": -145.2956085205078,
"KL/mean": -189.438720703125,
"KL/rejected_KL_mean": -233.58184814453125,
"KL/std": 115.39171600341797,
"epoch": 0.6107331821617535,
"fcm_dpo/beta": 0.004511928651481867,
"fcm_dpo/delta": -0.0019165575504302979,
"fcm_dpo/margin": 88.28622436523438,
"fcm_dpo/q_t": 0.40923064947128296,
"grad_norm": 12.346050262451172,
"learning_rate": 1.9879833298370237e-07,
"logits/chosen": 0.7575935125350952,
"logits/rejected": 0.6699234247207642,
"logps/chosen": -207.55850219726562,
"logps/ref_chosen": -62.26288604736328,
"logps/ref_rejected": -95.19029998779297,
"logps/rejected": -328.77215576171875,
"loss": 1.1111,
"margin_dpo/margin_mean": 88.2862319946289,
"margin_dpo/margin_std": 128.89202880859375,
"step": 404
},
{
"KL/chosen_KL_mean": -147.33062744140625,
"KL/mean": -181.20425415039062,
"KL/rejected_KL_mean": -215.077880859375,
"KL/std": 100.42963409423828,
"epoch": 0.6122448979591837,
"fcm_dpo/beta": 0.004548364318907261,
"fcm_dpo/delta": -0.02057464048266411,
"fcm_dpo/margin": 67.74724578857422,
"fcm_dpo/q_t": 0.4274998903274536,
"grad_norm": 9.59747314453125,
"learning_rate": 1.975048638084379e-07,
"logits/chosen": 0.868112325668335,
"logits/rejected": 0.8267897963523865,
"logps/chosen": -197.9149627685547,
"logps/ref_chosen": -50.5843391418457,
"logps/ref_rejected": -65.43156433105469,
"logps/rejected": -280.50946044921875,
"loss": 1.1562,
"margin_dpo/margin_mean": 67.74724578857422,
"margin_dpo/margin_std": 100.54032897949219,
"step": 405
},
{
"KL/chosen_KL_mean": -138.15542602539062,
"KL/mean": -185.86033630371094,
"KL/rejected_KL_mean": -233.56524658203125,
"KL/std": 109.98925018310547,
"epoch": 0.6137566137566137,
"fcm_dpo/beta": 0.004500585608184338,
"fcm_dpo/delta": -0.03803994506597519,
"fcm_dpo/margin": 95.40980529785156,
"fcm_dpo/q_t": 0.3991668224334717,
"grad_norm": 9.847400665283203,
"learning_rate": 1.9621286303497914e-07,
"logits/chosen": 0.8270584940910339,
"logits/rejected": 0.6823967099189758,
"logps/chosen": -187.15103149414062,
"logps/ref_chosen": -48.99560546875,
"logps/ref_rejected": -92.47774505615234,
"logps/rejected": -326.04296875,
"loss": 1.0758,
"margin_dpo/margin_mean": 95.40980529785156,
"margin_dpo/margin_std": 121.5294189453125,
"step": 406
},
{
"KL/chosen_KL_mean": -182.1513671875,
"KL/mean": -219.0489501953125,
"KL/rejected_KL_mean": -255.94656372070312,
"KL/std": 128.31683349609375,
"epoch": 0.6152683295540439,
"fcm_dpo/beta": 0.004660026635974646,
"fcm_dpo/delta": 0.06589356809854507,
"fcm_dpo/margin": 73.79518127441406,
"fcm_dpo/q_t": 0.4223455786705017,
"grad_norm": 11.783304214477539,
"learning_rate": 1.9492236680336483e-07,
"logits/chosen": 0.7038700580596924,
"logits/rejected": 0.6494981050491333,
"logps/chosen": -271.55194091796875,
"logps/ref_chosen": -89.40056610107422,
"logps/ref_rejected": -99.28775024414062,
"logps/rejected": -355.23431396484375,
"loss": 1.1571,
"margin_dpo/margin_mean": 73.7951889038086,
"margin_dpo/margin_std": 128.01223754882812,
"step": 407
},
{
"KL/chosen_KL_mean": -134.6771697998047,
"KL/mean": -185.23574829101562,
"KL/rejected_KL_mean": -235.79437255859375,
"KL/std": 102.01171112060547,
"epoch": 0.6167800453514739,
"fcm_dpo/beta": 0.004532967694103718,
"fcm_dpo/delta": -0.076311394572258,
"fcm_dpo/margin": 101.1171875,
"fcm_dpo/q_t": 0.38965433835983276,
"grad_norm": 9.231172561645508,
"learning_rate": 1.9363341121154895e-07,
"logits/chosen": 0.7848796844482422,
"logits/rejected": 0.7198034524917603,
"logps/chosen": -189.38107299804688,
"logps/ref_chosen": -54.70391845703125,
"logps/ref_rejected": -73.98648834228516,
"logps/rejected": -309.7808532714844,
"loss": 1.0377,
"margin_dpo/margin_mean": 101.11719512939453,
"margin_dpo/margin_std": 105.65650939941406,
"step": 408
},
{
"KL/chosen_KL_mean": -167.1765594482422,
"KL/mean": -189.65097045898438,
"KL/rejected_KL_mean": -212.1253662109375,
"KL/std": 87.67469787597656,
"epoch": 0.618291761148904,
"fcm_dpo/beta": 0.004333215299993753,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 44.94880676269531,
"fcm_dpo/q_t": 0.4536062777042389,
"grad_norm": 10.822347640991211,
"learning_rate": 1.9234603231438994e-07,
"logits/chosen": 0.7813749313354492,
"logits/rejected": 0.791049599647522,
"logps/chosen": -229.29478454589844,
"logps/ref_chosen": -62.11822509765625,
"logps/ref_rejected": -61.933509826660156,
"logps/rejected": -274.05889892578125,
"loss": 1.2448,
"margin_dpo/margin_mean": 44.94880676269531,
"margin_dpo/margin_std": 97.49263763427734,
"step": 409
},
{
"KL/chosen_KL_mean": -157.98037719726562,
"KL/mean": -198.59495544433594,
"KL/rejected_KL_mean": -239.2095184326172,
"KL/std": 103.38927459716797,
"epoch": 0.6198034769463341,
"fcm_dpo/beta": 0.004367862828075886,
"fcm_dpo/delta": 0.05229746922850609,
"fcm_dpo/margin": 81.22914123535156,
"fcm_dpo/q_t": 0.41858938336372375,
"grad_norm": 9.7050199508667,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 0.7918972373008728,
"logits/rejected": 0.7738727927207947,
"logps/chosen": -219.78305053710938,
"logps/ref_chosen": -61.80266189575195,
"logps/ref_rejected": -76.60002136230469,
"logps/rejected": -315.8095397949219,
"loss": 1.1201,
"margin_dpo/margin_mean": 81.22914123535156,
"margin_dpo/margin_std": 105.19297790527344,
"step": 410
},
{
"KL/chosen_KL_mean": -164.25729370117188,
"KL/mean": -204.526611328125,
"KL/rejected_KL_mean": -244.7958984375,
"KL/std": 108.70731353759766,
"epoch": 0.6213151927437641,
"fcm_dpo/beta": 0.004682523664087057,
"fcm_dpo/delta": 0.027940556406974792,
"fcm_dpo/margin": 80.53861999511719,
"fcm_dpo/q_t": 0.4140225946903229,
"grad_norm": 9.122180938720703,
"learning_rate": 1.8977614860195296e-07,
"logits/chosen": 0.8158813118934631,
"logits/rejected": 0.7707309722900391,
"logps/chosen": -218.70269775390625,
"logps/ref_chosen": -54.44539260864258,
"logps/ref_rejected": -74.5650863647461,
"logps/rejected": -319.3609924316406,
"loss": 1.1114,
"margin_dpo/margin_mean": 80.53861999511719,
"margin_dpo/margin_std": 114.59844970703125,
"step": 411
},
{
"KL/chosen_KL_mean": -167.36849975585938,
"KL/mean": -203.03668212890625,
"KL/rejected_KL_mean": -238.70486450195312,
"KL/std": 91.77555847167969,
"epoch": 0.6228269085411943,
"fcm_dpo/beta": 0.004863695241510868,
"fcm_dpo/delta": 0.06431097537279129,
"fcm_dpo/margin": 71.33634185791016,
"fcm_dpo/q_t": 0.42097651958465576,
"grad_norm": 10.8851957321167,
"learning_rate": 1.8849371567184662e-07,
"logits/chosen": 0.8179016709327698,
"logits/rejected": 0.7678436040878296,
"logps/chosen": -222.6165771484375,
"logps/ref_chosen": -55.248085021972656,
"logps/ref_rejected": -68.96623229980469,
"logps/rejected": -307.67108154296875,
"loss": 1.124,
"margin_dpo/margin_mean": 71.33634185791016,
"margin_dpo/margin_std": 97.02810668945312,
"step": 412
},
{
"KL/chosen_KL_mean": -188.40090942382812,
"KL/mean": -223.15206909179688,
"KL/rejected_KL_mean": -257.90325927734375,
"KL/std": 112.07737731933594,
"epoch": 0.6243386243386243,
"fcm_dpo/beta": 0.005236283876001835,
"fcm_dpo/delta": 0.04264630749821663,
"fcm_dpo/margin": 69.50237274169922,
"fcm_dpo/q_t": 0.4206687808036804,
"grad_norm": 14.27915096282959,
"learning_rate": 1.872130032047302e-07,
"logits/chosen": 0.6741722822189331,
"logits/rejected": 0.6519167423248291,
"logps/chosen": -257.12164306640625,
"logps/ref_chosen": -68.72074890136719,
"logps/ref_rejected": -78.76539611816406,
"logps/rejected": -336.6686706542969,
"loss": 1.1709,
"margin_dpo/margin_mean": 69.50237274169922,
"margin_dpo/margin_std": 134.83111572265625,
"step": 413
},
{
"KL/chosen_KL_mean": -165.21878051757812,
"KL/mean": -206.52334594726562,
"KL/rejected_KL_mean": -247.8279266357422,
"KL/std": 119.95925903320312,
"epoch": 0.6258503401360545,
"fcm_dpo/beta": 0.005115990526974201,
"fcm_dpo/delta": -0.028705045580863953,
"fcm_dpo/margin": 82.6091537475586,
"fcm_dpo/q_t": 0.4012109339237213,
"grad_norm": 10.268512725830078,
"learning_rate": 1.8593404702488436e-07,
"logits/chosen": 0.8100833892822266,
"logits/rejected": 0.7635753750801086,
"logps/chosen": -219.35699462890625,
"logps/ref_chosen": -54.138214111328125,
"logps/ref_rejected": -74.65741729736328,
"logps/rejected": -322.4853515625,
"loss": 1.0844,
"margin_dpo/margin_mean": 82.6091537475586,
"margin_dpo/margin_std": 109.7034912109375,
"step": 414
},
{
"KL/chosen_KL_mean": -165.16592407226562,
"KL/mean": -196.70492553710938,
"KL/rejected_KL_mean": -228.24395751953125,
"KL/std": 107.95329284667969,
"epoch": 0.6273620559334845,
"fcm_dpo/beta": 0.005346394143998623,
"fcm_dpo/delta": 0.07570715248584747,
"fcm_dpo/margin": 63.078006744384766,
"fcm_dpo/q_t": 0.4259011745452881,
"grad_norm": 12.313481330871582,
"learning_rate": 1.846568829074628e-07,
"logits/chosen": 0.8486864566802979,
"logits/rejected": 0.8356572985649109,
"logps/chosen": -221.08448791503906,
"logps/ref_chosen": -55.91856002807617,
"logps/ref_rejected": -61.747703552246094,
"logps/rejected": -289.99163818359375,
"loss": 1.1686,
"margin_dpo/margin_mean": 63.078006744384766,
"margin_dpo/margin_std": 116.46731567382812,
"step": 415
},
{
"KL/chosen_KL_mean": -174.42445373535156,
"KL/mean": -204.85211181640625,
"KL/rejected_KL_mean": -235.27975463867188,
"KL/std": 115.35769653320312,
"epoch": 0.6288737717309146,
"fcm_dpo/beta": 0.005210232920944691,
"fcm_dpo/delta": -0.10238505899906158,
"fcm_dpo/margin": 60.85530090332031,
"fcm_dpo/q_t": 0.4263876676559448,
"grad_norm": 13.751988410949707,
"learning_rate": 1.8338154657749128e-07,
"logits/chosen": 0.8052107095718384,
"logits/rejected": 0.7643084526062012,
"logps/chosen": -229.1475372314453,
"logps/ref_chosen": -54.72308349609375,
"logps/ref_rejected": -69.17388916015625,
"logps/rejected": -304.4536437988281,
"loss": 1.1932,
"margin_dpo/margin_mean": 60.85530090332031,
"margin_dpo/margin_std": 109.29032897949219,
"step": 416
},
{
"KL/chosen_KL_mean": -182.31307983398438,
"KL/mean": -219.28237915039062,
"KL/rejected_KL_mean": -256.2516784667969,
"KL/std": 106.955810546875,
"epoch": 0.6303854875283447,
"fcm_dpo/beta": 0.005158806219696999,
"fcm_dpo/delta": 0.018698185682296753,
"fcm_dpo/margin": 73.93860626220703,
"fcm_dpo/q_t": 0.4111923575401306,
"grad_norm": 12.965594291687012,
"learning_rate": 1.8210807370886849e-07,
"logits/chosen": 0.9041584134101868,
"logits/rejected": 0.8518546223640442,
"logps/chosen": -239.10433959960938,
"logps/ref_chosen": -56.791259765625,
"logps/ref_rejected": -68.7791748046875,
"logps/rejected": -325.0308532714844,
"loss": 1.1487,
"margin_dpo/margin_mean": 73.93860626220703,
"margin_dpo/margin_std": 130.03167724609375,
"step": 417
},
{
"KL/chosen_KL_mean": -196.0471954345703,
"KL/mean": -226.67172241210938,
"KL/rejected_KL_mean": -257.2962646484375,
"KL/std": 123.40777587890625,
"epoch": 0.6318972033257747,
"fcm_dpo/beta": 0.004728157073259354,
"fcm_dpo/delta": -0.06328493356704712,
"fcm_dpo/margin": 61.249061584472656,
"fcm_dpo/q_t": 0.43131130933761597,
"grad_norm": 12.652046203613281,
"learning_rate": 1.8083649992336825e-07,
"logits/chosen": 0.8404183387756348,
"logits/rejected": 0.8438146114349365,
"logps/chosen": -265.1551818847656,
"logps/ref_chosen": -69.10798645019531,
"logps/ref_rejected": -75.09132385253906,
"logps/rejected": -332.3875732421875,
"loss": 1.2083,
"margin_dpo/margin_mean": 61.249053955078125,
"margin_dpo/margin_std": 124.80625915527344,
"step": 418
},
{
"KL/chosen_KL_mean": -157.66766357421875,
"KL/mean": -203.49771118164062,
"KL/rejected_KL_mean": -249.32774353027344,
"KL/std": 108.96971130371094,
"epoch": 0.6334089191232048,
"fcm_dpo/beta": 0.004607297480106354,
"fcm_dpo/delta": -0.02855597995221615,
"fcm_dpo/margin": 91.66007995605469,
"fcm_dpo/q_t": 0.40371498465538025,
"grad_norm": 10.530730247497559,
"learning_rate": 1.7956686078964255e-07,
"logits/chosen": 0.743537962436676,
"logits/rejected": 0.7084311246871948,
"logps/chosen": -215.8394317626953,
"logps/ref_chosen": -58.1717643737793,
"logps/ref_rejected": -71.67066955566406,
"logps/rejected": -320.9984130859375,
"loss": 1.0856,
"margin_dpo/margin_mean": 91.66007995605469,
"margin_dpo/margin_std": 126.45355224609375,
"step": 419
},
{
"KL/chosen_KL_mean": -190.76968383789062,
"KL/mean": -213.75607299804688,
"KL/rejected_KL_mean": -236.74246215820312,
"KL/std": 112.41749572753906,
"epoch": 0.6349206349206349,
"fcm_dpo/beta": 0.0049021318554878235,
"fcm_dpo/delta": 0.06469061225652695,
"fcm_dpo/margin": 45.97278594970703,
"fcm_dpo/q_t": 0.45175454020500183,
"grad_norm": 11.888525009155273,
"learning_rate": 1.782991918222275e-07,
"logits/chosen": 0.8070861101150513,
"logits/rejected": 0.7711919546127319,
"logps/chosen": -247.8231964111328,
"logps/ref_chosen": -57.05351257324219,
"logps/ref_rejected": -62.670982360839844,
"logps/rejected": -299.4134521484375,
"loss": 1.2724,
"margin_dpo/margin_mean": 45.97278594970703,
"margin_dpo/margin_std": 131.39532470703125,
"step": 420
},
{
"KL/chosen_KL_mean": -177.93927001953125,
"KL/mean": -212.15362548828125,
"KL/rejected_KL_mean": -246.36798095703125,
"KL/std": 111.99041748046875,
"epoch": 0.636432350718065,
"fcm_dpo/beta": 0.00480629364028573,
"fcm_dpo/delta": -0.01974395476281643,
"fcm_dpo/margin": 68.42868041992188,
"fcm_dpo/q_t": 0.4252956509590149,
"grad_norm": 12.691519737243652,
"learning_rate": 1.7703352848054887e-07,
"logits/chosen": 0.7821997404098511,
"logits/rejected": 0.7315264344215393,
"logps/chosen": -235.2625274658203,
"logps/ref_chosen": -57.32324981689453,
"logps/ref_rejected": -75.33782958984375,
"logps/rejected": -321.705810546875,
"loss": 1.2077,
"margin_dpo/margin_mean": 68.42868041992188,
"margin_dpo/margin_std": 150.48504638671875,
"step": 421
},
{
"KL/chosen_KL_mean": -159.26858520507812,
"KL/mean": -201.4471435546875,
"KL/rejected_KL_mean": -243.6256866455078,
"KL/std": 111.61254119873047,
"epoch": 0.6379440665154951,
"fcm_dpo/beta": 0.004849913530051708,
"fcm_dpo/delta": -0.012699028477072716,
"fcm_dpo/margin": 84.35710906982422,
"fcm_dpo/q_t": 0.4054141044616699,
"grad_norm": 12.056890487670898,
"learning_rate": 1.7576990616793137e-07,
"logits/chosen": 0.7797449827194214,
"logits/rejected": 0.7793635129928589,
"logps/chosen": -226.326171875,
"logps/ref_chosen": -67.05757141113281,
"logps/ref_rejected": -72.12803649902344,
"logps/rejected": -315.75372314453125,
"loss": 1.0885,
"margin_dpo/margin_mean": 84.35710906982422,
"margin_dpo/margin_std": 111.04570770263672,
"step": 422
},
{
"KL/chosen_KL_mean": -155.98098754882812,
"KL/mean": -204.0811767578125,
"KL/rejected_KL_mean": -252.18136596679688,
"KL/std": 113.7120132446289,
"epoch": 0.6394557823129252,
"fcm_dpo/beta": 0.004519253503531218,
"fcm_dpo/delta": -0.04623626172542572,
"fcm_dpo/margin": 96.20036315917969,
"fcm_dpo/q_t": 0.39895230531692505,
"grad_norm": 11.132472038269043,
"learning_rate": 1.745083602306071e-07,
"logits/chosen": 0.8631881475448608,
"logits/rejected": 0.8088440895080566,
"logps/chosen": -210.04266357421875,
"logps/ref_chosen": -54.06167221069336,
"logps/ref_rejected": -76.64092254638672,
"logps/rejected": -328.8222961425781,
"loss": 1.0682,
"margin_dpo/margin_mean": 96.20036315917969,
"margin_dpo/margin_std": 121.70199584960938,
"step": 423
},
{
"KL/chosen_KL_mean": -170.24893188476562,
"KL/mean": -215.80943298339844,
"KL/rejected_KL_mean": -261.36993408203125,
"KL/std": 104.85726928710938,
"epoch": 0.6409674981103552,
"fcm_dpo/beta": 0.004399476572871208,
"fcm_dpo/delta": -0.004416920244693756,
"fcm_dpo/margin": 91.12101745605469,
"fcm_dpo/q_t": 0.40782856941223145,
"grad_norm": 13.965441703796387,
"learning_rate": 1.7324892595672804e-07,
"logits/chosen": 0.7467511892318726,
"logits/rejected": 0.7114660739898682,
"logps/chosen": -223.85780334472656,
"logps/ref_chosen": -53.60887145996094,
"logps/ref_rejected": -79.2139892578125,
"logps/rejected": -340.58392333984375,
"loss": 1.0948,
"margin_dpo/margin_mean": 91.12101745605469,
"margin_dpo/margin_std": 122.10706329345703,
"step": 424
},
{
"KL/chosen_KL_mean": -168.24703979492188,
"KL/mean": -202.56069946289062,
"KL/rejected_KL_mean": -236.87437438964844,
"KL/std": 100.82597351074219,
"epoch": 0.6424792139077853,
"fcm_dpo/beta": 0.004823003895580769,
"fcm_dpo/delta": 0.08188430219888687,
"fcm_dpo/margin": 68.62734985351562,
"fcm_dpo/q_t": 0.42684072256088257,
"grad_norm": 10.987018585205078,
"learning_rate": 1.7199163857537824e-07,
"logits/chosen": 0.8584731817245483,
"logits/rejected": 0.8374955654144287,
"logps/chosen": -226.66171264648438,
"logps/ref_chosen": -58.41468048095703,
"logps/ref_rejected": -66.59054565429688,
"logps/rejected": -303.46490478515625,
"loss": 1.1573,
"margin_dpo/margin_mean": 68.62734985351562,
"margin_dpo/margin_std": 115.73839569091797,
"step": 425
},
{
"KL/chosen_KL_mean": -202.30221557617188,
"KL/mean": -222.89907836914062,
"KL/rejected_KL_mean": -243.49595642089844,
"KL/std": 112.37277221679688,
"epoch": 0.6439909297052154,
"fcm_dpo/beta": 0.004896195139735937,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.19374084472656,
"fcm_dpo/q_t": 0.45397740602493286,
"grad_norm": 16.15119171142578,
"learning_rate": 1.7073653325558828e-07,
"logits/chosen": 0.78400719165802,
"logits/rejected": 0.7848511934280396,
"logps/chosen": -274.01043701171875,
"logps/ref_chosen": -71.70822143554688,
"logps/ref_rejected": -73.57725524902344,
"logps/rejected": -317.0732116699219,
"loss": 1.3074,
"margin_dpo/margin_mean": 41.19373321533203,
"margin_dpo/margin_std": 142.9339599609375,
"step": 426
},
{
"KL/chosen_KL_mean": -190.27249145507812,
"KL/mean": -228.701171875,
"KL/rejected_KL_mean": -267.1298522949219,
"KL/std": 124.13754272460938,
"epoch": 0.6455026455026455,
"fcm_dpo/beta": 0.004953271709382534,
"fcm_dpo/delta": 0.02383248880505562,
"fcm_dpo/margin": 76.85736083984375,
"fcm_dpo/q_t": 0.4170149266719818,
"grad_norm": 12.632922172546387,
"learning_rate": 1.6948364510535218e-07,
"logits/chosen": 0.8083971738815308,
"logits/rejected": 0.7516036033630371,
"logps/chosen": -248.91525268554688,
"logps/ref_chosen": -58.64276885986328,
"logps/ref_rejected": -86.25437927246094,
"logps/rejected": -353.38421630859375,
"loss": 1.1428,
"margin_dpo/margin_mean": 76.85736083984375,
"margin_dpo/margin_std": 134.9580078125,
"step": 427
},
{
"KL/chosen_KL_mean": -185.6763916015625,
"KL/mean": -223.56564331054688,
"KL/rejected_KL_mean": -261.45489501953125,
"KL/std": 125.037353515625,
"epoch": 0.6470143613000756,
"fcm_dpo/beta": 0.005163976922631264,
"fcm_dpo/delta": 0.008169233798980713,
"fcm_dpo/margin": 75.77848815917969,
"fcm_dpo/q_t": 0.4132087230682373,
"grad_norm": 11.813305854797363,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 0.766205906867981,
"logits/rejected": 0.736466646194458,
"logps/chosen": -252.27244567871094,
"logps/ref_chosen": -66.5960464477539,
"logps/ref_rejected": -82.3941650390625,
"logps/rejected": -343.84906005859375,
"loss": 1.1464,
"margin_dpo/margin_mean": 75.77848052978516,
"margin_dpo/margin_std": 133.88519287109375,
"step": 428
},
{
"KL/chosen_KL_mean": -188.0496368408203,
"KL/mean": -216.96371459960938,
"KL/rejected_KL_mean": -245.87782287597656,
"KL/std": 101.20547485351562,
"epoch": 0.6485260770975056,
"fcm_dpo/beta": 0.005533907096832991,
"fcm_dpo/delta": 0.09032148122787476,
"fcm_dpo/margin": 57.82816696166992,
"fcm_dpo/q_t": 0.4274323582649231,
"grad_norm": 13.291508674621582,
"learning_rate": 1.669846604344412e-07,
"logits/chosen": 0.8079313635826111,
"logits/rejected": 0.8284199237823486,
"logps/chosen": -245.059326171875,
"logps/ref_chosen": -57.00970458984375,
"logps/ref_rejected": -59.86549377441406,
"logps/rejected": -305.7433166503906,
"loss": 1.1943,
"margin_dpo/margin_mean": 57.828163146972656,
"margin_dpo/margin_std": 117.090576171875,
"step": 429
},
{
"KL/chosen_KL_mean": -166.90045166015625,
"KL/mean": -215.31846618652344,
"KL/rejected_KL_mean": -263.73651123046875,
"KL/std": 102.39797973632812,
"epoch": 0.6500377928949358,
"fcm_dpo/beta": 0.005032334476709366,
"fcm_dpo/delta": -0.11732571572065353,
"fcm_dpo/margin": 96.83605194091797,
"fcm_dpo/q_t": 0.38132691383361816,
"grad_norm": 13.358454704284668,
"learning_rate": 1.6573863381573954e-07,
"logits/chosen": 0.724281370639801,
"logits/rejected": 0.7237146496772766,
"logps/chosen": -226.46363830566406,
"logps/ref_chosen": -59.563194274902344,
"logps/ref_rejected": -70.52289581298828,
"logps/rejected": -334.2593994140625,
"loss": 1.0456,
"margin_dpo/margin_mean": 96.8360595703125,
"margin_dpo/margin_std": 120.72003936767578,
"step": 430
},
{
"KL/chosen_KL_mean": -165.01373291015625,
"KL/mean": -202.18789672851562,
"KL/rejected_KL_mean": -239.36204528808594,
"KL/std": 107.67405700683594,
"epoch": 0.6515495086923658,
"fcm_dpo/beta": 0.005004248581826687,
"fcm_dpo/delta": 0.034310389310121536,
"fcm_dpo/margin": 74.34831237792969,
"fcm_dpo/q_t": 0.41700905561447144,
"grad_norm": 12.656464576721191,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 0.7730939388275146,
"logits/rejected": 0.724521815776825,
"logps/chosen": -215.21405029296875,
"logps/ref_chosen": -50.20032501220703,
"logps/ref_rejected": -77.81680297851562,
"logps/rejected": -317.1788330078125,
"loss": 1.1371,
"margin_dpo/margin_mean": 74.34829711914062,
"margin_dpo/margin_std": 122.35536193847656,
"step": 431
},
{
"KL/chosen_KL_mean": -174.32611083984375,
"KL/mean": -212.357421875,
"KL/rejected_KL_mean": -250.38873291015625,
"KL/std": 107.8907241821289,
"epoch": 0.6530612244897959,
"fcm_dpo/beta": 0.005113082937896252,
"fcm_dpo/delta": 0.013686059042811394,
"fcm_dpo/margin": 76.06259155273438,
"fcm_dpo/q_t": 0.4114718437194824,
"grad_norm": 13.092035293579102,
"learning_rate": 1.632536862810844e-07,
"logits/chosen": 0.8555986881256104,
"logits/rejected": 0.8065499067306519,
"logps/chosen": -235.98887634277344,
"logps/ref_chosen": -61.662757873535156,
"logps/ref_rejected": -83.94496154785156,
"logps/rejected": -334.33367919921875,
"loss": 1.1371,
"margin_dpo/margin_mean": 76.06259155273438,
"margin_dpo/margin_std": 129.0126190185547,
"step": 432
},
{
"KL/chosen_KL_mean": -171.813232421875,
"KL/mean": -214.63470458984375,
"KL/rejected_KL_mean": -257.4562072753906,
"KL/std": 100.15315246582031,
"epoch": 0.654572940287226,
"fcm_dpo/beta": 0.005063719116151333,
"fcm_dpo/delta": -0.04309210181236267,
"fcm_dpo/margin": 85.64299774169922,
"fcm_dpo/q_t": 0.39825403690338135,
"grad_norm": 13.218973159790039,
"learning_rate": 1.6201483487445515e-07,
"logits/chosen": 0.8633350133895874,
"logits/rejected": 0.8686789870262146,
"logps/chosen": -235.5424041748047,
"logps/ref_chosen": -63.72917938232422,
"logps/ref_rejected": -65.8391342163086,
"logps/rejected": -323.29534912109375,
"loss": 1.078,
"margin_dpo/margin_mean": 85.64300537109375,
"margin_dpo/margin_std": 113.72454071044922,
"step": 433
},
{
"KL/chosen_KL_mean": -146.77532958984375,
"KL/mean": -195.40478515625,
"KL/rejected_KL_mean": -244.0342559814453,
"KL/std": 111.6845703125,
"epoch": 0.656084656084656,
"fcm_dpo/beta": 0.004525492433458567,
"fcm_dpo/delta": -0.06290946900844574,
"fcm_dpo/margin": 97.25892639160156,
"fcm_dpo/q_t": 0.394904762506485,
"grad_norm": 11.273813247680664,
"learning_rate": 1.6077844460203204e-07,
"logits/chosen": 0.9331784248352051,
"logits/rejected": 0.8754293322563171,
"logps/chosen": -194.74864196777344,
"logps/ref_chosen": -47.97331619262695,
"logps/ref_rejected": -72.51132202148438,
"logps/rejected": -316.54559326171875,
"loss": 1.0843,
"margin_dpo/margin_mean": 97.25892639160156,
"margin_dpo/margin_std": 132.83673095703125,
"step": 434
},
{
"KL/chosen_KL_mean": -178.30308532714844,
"KL/mean": -213.94564819335938,
"KL/rejected_KL_mean": -249.58822631835938,
"KL/std": 107.2773208618164,
"epoch": 0.6575963718820862,
"fcm_dpo/beta": 0.004943176172673702,
"fcm_dpo/delta": 0.054163962602615356,
"fcm_dpo/margin": 71.28514099121094,
"fcm_dpo/q_t": 0.4210607409477234,
"grad_norm": 13.454949378967285,
"learning_rate": 1.5954455004830878e-07,
"logits/chosen": 0.8959484696388245,
"logits/rejected": 0.8630224466323853,
"logps/chosen": -235.3633270263672,
"logps/ref_chosen": -57.06024932861328,
"logps/ref_rejected": -71.69146728515625,
"logps/rejected": -321.2796936035156,
"loss": 1.1558,
"margin_dpo/margin_mean": 71.28514099121094,
"margin_dpo/margin_std": 124.44223022460938,
"step": 435
},
{
"KL/chosen_KL_mean": -178.91229248046875,
"KL/mean": -209.08297729492188,
"KL/rejected_KL_mean": -239.253662109375,
"KL/std": 104.4656982421875,
"epoch": 0.6591080876795162,
"fcm_dpo/beta": 0.005222709849476814,
"fcm_dpo/delta": 0.10130466520786285,
"fcm_dpo/margin": 60.341373443603516,
"fcm_dpo/q_t": 0.43232089281082153,
"grad_norm": 14.217085838317871,
"learning_rate": 1.5831318572796847e-07,
"logits/chosen": 0.8186235427856445,
"logits/rejected": 0.7705166339874268,
"logps/chosen": -235.07034301757812,
"logps/ref_chosen": -56.158050537109375,
"logps/ref_rejected": -67.63787841796875,
"logps/rejected": -306.89154052734375,
"loss": 1.2038,
"margin_dpo/margin_mean": 60.341373443603516,
"margin_dpo/margin_std": 128.90338134765625,
"step": 436
},
{
"KL/chosen_KL_mean": -184.0316619873047,
"KL/mean": -222.54013061523438,
"KL/rejected_KL_mean": -261.0486145019531,
"KL/std": 116.43235778808594,
"epoch": 0.6606198034769464,
"fcm_dpo/beta": 0.004938242956995964,
"fcm_dpo/delta": -0.09390990436077118,
"fcm_dpo/margin": 77.01693725585938,
"fcm_dpo/q_t": 0.4115540087223053,
"grad_norm": 15.683857917785645,
"learning_rate": 1.5708438608491815e-07,
"logits/chosen": 0.8214137554168701,
"logits/rejected": 0.7116411924362183,
"logps/chosen": -241.0174560546875,
"logps/ref_chosen": -56.98578643798828,
"logps/ref_rejected": -85.61524963378906,
"logps/rejected": -346.66387939453125,
"loss": 1.1721,
"margin_dpo/margin_mean": 77.01693725585938,
"margin_dpo/margin_std": 149.37376403808594,
"step": 437
},
{
"KL/chosen_KL_mean": -158.45684814453125,
"KL/mean": -206.56228637695312,
"KL/rejected_KL_mean": -254.66769409179688,
"KL/std": 119.14268493652344,
"epoch": 0.6621315192743764,
"fcm_dpo/beta": 0.004753970541059971,
"fcm_dpo/delta": -0.07428638637065887,
"fcm_dpo/margin": 96.21085357666016,
"fcm_dpo/q_t": 0.39683008193969727,
"grad_norm": 12.5221529006958,
"learning_rate": 1.558581854913253e-07,
"logits/chosen": 0.8680797219276428,
"logits/rejected": 0.8115641474723816,
"logps/chosen": -199.73463439941406,
"logps/ref_chosen": -41.27777862548828,
"logps/ref_rejected": -65.33840942382812,
"logps/rejected": -320.006103515625,
"loss": 1.0695,
"margin_dpo/margin_mean": 96.21085357666016,
"margin_dpo/margin_std": 133.3427276611328,
"step": 438
},
{
"KL/chosen_KL_mean": -186.97760009765625,
"KL/mean": -230.8473358154297,
"KL/rejected_KL_mean": -274.71710205078125,
"KL/std": 122.46157836914062,
"epoch": 0.6636432350718064,
"fcm_dpo/beta": 0.004707343876361847,
"fcm_dpo/delta": -0.02342548966407776,
"fcm_dpo/margin": 87.739501953125,
"fcm_dpo/q_t": 0.4038691222667694,
"grad_norm": 11.63642406463623,
"learning_rate": 1.5463461824665658e-07,
"logits/chosen": 0.7208713293075562,
"logits/rejected": 0.6962095499038696,
"logps/chosen": -268.395263671875,
"logps/ref_chosen": -81.41764831542969,
"logps/ref_rejected": -94.72309875488281,
"logps/rejected": -369.440185546875,
"loss": 1.1026,
"margin_dpo/margin_mean": 87.739501953125,
"margin_dpo/margin_std": 122.69038391113281,
"step": 439
},
{
"KL/chosen_KL_mean": -155.75674438476562,
"KL/mean": -199.6446075439453,
"KL/rejected_KL_mean": -243.532470703125,
"KL/std": 107.34757232666016,
"epoch": 0.6651549508692366,
"fcm_dpo/beta": 0.004554657265543938,
"fcm_dpo/delta": -0.00109158456325531,
"fcm_dpo/margin": 87.77574157714844,
"fcm_dpo/q_t": 0.4082149267196655,
"grad_norm": 15.26758861541748,
"learning_rate": 1.534137185767178e-07,
"logits/chosen": 0.836760401725769,
"logits/rejected": 0.7494317293167114,
"logps/chosen": -198.294921875,
"logps/ref_chosen": -42.538185119628906,
"logps/ref_rejected": -69.78813934326172,
"logps/rejected": -313.32061767578125,
"loss": 1.1007,
"margin_dpo/margin_mean": 87.77574157714844,
"margin_dpo/margin_std": 121.38520812988281,
"step": 440
},
{
"KL/chosen_KL_mean": -164.15768432617188,
"KL/mean": -210.33230590820312,
"KL/rejected_KL_mean": -256.50689697265625,
"KL/std": 118.62611389160156,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.004367801360785961,
"fcm_dpo/delta": -0.006083235144615173,
"fcm_dpo/margin": 92.34922790527344,
"fcm_dpo/q_t": 0.405091255903244,
"grad_norm": 11.903091430664062,
"learning_rate": 1.521955206326976e-07,
"logits/chosen": 0.7793187499046326,
"logits/rejected": 0.7015305161476135,
"logps/chosen": -221.7509002685547,
"logps/ref_chosen": -57.593223571777344,
"logps/ref_rejected": -84.82878875732422,
"logps/rejected": -341.335693359375,
"loss": 1.0786,
"margin_dpo/margin_mean": 92.34922790527344,
"margin_dpo/margin_std": 108.52428436279297,
"step": 441
},
{
"KL/chosen_KL_mean": -195.50466918945312,
"KL/mean": -242.55752563476562,
"KL/rejected_KL_mean": -289.61041259765625,
"KL/std": 119.05079650878906,
"epoch": 0.6681783824640968,
"fcm_dpo/beta": 0.004326670430600643,
"fcm_dpo/delta": -0.010936200618743896,
"fcm_dpo/margin": 94.10575866699219,
"fcm_dpo/q_t": 0.4057489037513733,
"grad_norm": 14.003538131713867,
"learning_rate": 1.5098005849021078e-07,
"logits/chosen": 0.8129810094833374,
"logits/rejected": 0.7782344818115234,
"logps/chosen": -262.96588134765625,
"logps/ref_chosen": -67.46121978759766,
"logps/ref_rejected": -89.0693588256836,
"logps/rejected": -378.67974853515625,
"loss": 1.0866,
"margin_dpo/margin_mean": 94.10575866699219,
"margin_dpo/margin_std": 121.53174591064453,
"step": 442
},
{
"KL/chosen_KL_mean": -161.1103973388672,
"KL/mean": -217.9424285888672,
"KL/rejected_KL_mean": -274.77447509765625,
"KL/std": 124.97842407226562,
"epoch": 0.6696900982615268,
"fcm_dpo/beta": 0.004096883349120617,
"fcm_dpo/delta": -0.0867290198802948,
"fcm_dpo/margin": 113.66409301757812,
"fcm_dpo/q_t": 0.3905232548713684,
"grad_norm": 11.163167953491211,
"learning_rate": 1.4976736614834662e-07,
"logits/chosen": 0.8272267580032349,
"logits/rejected": 0.7736169099807739,
"logps/chosen": -215.906494140625,
"logps/ref_chosen": -54.79610061645508,
"logps/ref_rejected": -77.80781555175781,
"logps/rejected": -352.5823059082031,
"loss": 1.0474,
"margin_dpo/margin_mean": 113.66409301757812,
"margin_dpo/margin_std": 140.31976318359375,
"step": 443
},
{
"KL/chosen_KL_mean": -192.77120971679688,
"KL/mean": -222.12258911132812,
"KL/rejected_KL_mean": -251.47393798828125,
"KL/std": 128.48406982421875,
"epoch": 0.671201814058957,
"fcm_dpo/beta": 0.0042202952317893505,
"fcm_dpo/delta": 0.045447517186403275,
"fcm_dpo/margin": 58.70273208618164,
"fcm_dpo/q_t": 0.44652265310287476,
"grad_norm": 12.165427207946777,
"learning_rate": 1.4855747752871654e-07,
"logits/chosen": 0.8557889461517334,
"logits/rejected": 0.779559850692749,
"logps/chosen": -251.520263671875,
"logps/ref_chosen": -58.749061584472656,
"logps/ref_rejected": -86.87396240234375,
"logps/rejected": -338.347900390625,
"loss": 1.241,
"margin_dpo/margin_mean": 58.702728271484375,
"margin_dpo/margin_std": 142.91485595703125,
"step": 444
},
{
"KL/chosen_KL_mean": -184.20005798339844,
"KL/mean": -232.47119140625,
"KL/rejected_KL_mean": -280.7423095703125,
"KL/std": 111.41131591796875,
"epoch": 0.672713529856387,
"fcm_dpo/beta": 0.004190181382000446,
"fcm_dpo/delta": -0.005736490711569786,
"fcm_dpo/margin": 96.54226684570312,
"fcm_dpo/q_t": 0.4045560956001282,
"grad_norm": 9.579313278198242,
"learning_rate": 1.473504264745062e-07,
"logits/chosen": 0.7703770399093628,
"logits/rejected": 0.7707059383392334,
"logps/chosen": -245.11749267578125,
"logps/ref_chosen": -60.91743850708008,
"logps/ref_rejected": -71.5637435913086,
"logps/rejected": -352.30609130859375,
"loss": 1.079,
"margin_dpo/margin_mean": 96.54226684570312,
"margin_dpo/margin_std": 116.30671691894531,
"step": 445
},
{
"KL/chosen_KL_mean": -165.2137451171875,
"KL/mean": -213.72683715820312,
"KL/rejected_KL_mean": -262.23992919921875,
"KL/std": 113.98602294921875,
"epoch": 0.674225245653817,
"fcm_dpo/beta": 0.004296318627893925,
"fcm_dpo/delta": -0.028332628309726715,
"fcm_dpo/margin": 97.02619171142578,
"fcm_dpo/q_t": 0.4015599489212036,
"grad_norm": 11.402931213378906,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 0.8551122546195984,
"logits/rejected": 0.7882475852966309,
"logps/chosen": -214.0129852294922,
"logps/ref_chosen": -48.79924774169922,
"logps/ref_rejected": -71.8719482421875,
"logps/rejected": -334.11187744140625,
"loss": 1.0773,
"margin_dpo/margin_mean": 97.02619171142578,
"margin_dpo/margin_std": 112.55924987792969,
"step": 446
},
{
"KL/chosen_KL_mean": -161.5308837890625,
"KL/mean": -218.6871337890625,
"KL/rejected_KL_mean": -275.8433837890625,
"KL/std": 113.90840148925781,
"epoch": 0.6757369614512472,
"fcm_dpo/beta": 0.0038491457235068083,
"fcm_dpo/delta": -0.053994160145521164,
"fcm_dpo/margin": 114.31251525878906,
"fcm_dpo/q_t": 0.3937652111053467,
"grad_norm": 9.615116119384766,
"learning_rate": 1.4494497203727843e-07,
"logits/chosen": 0.7905747294425964,
"logits/rejected": 0.7021323442459106,
"logps/chosen": -215.21359252929688,
"logps/ref_chosen": -53.682716369628906,
"logps/ref_rejected": -88.17315673828125,
"logps/rejected": -364.01654052734375,
"loss": 1.0531,
"margin_dpo/margin_mean": 114.31251525878906,
"margin_dpo/margin_std": 127.45619201660156,
"step": 447
},
{
"KL/chosen_KL_mean": -178.0913848876953,
"KL/mean": -222.7434539794922,
"KL/rejected_KL_mean": -267.3955383300781,
"KL/std": 118.10449981689453,
"epoch": 0.6772486772486772,
"fcm_dpo/beta": 0.004021032713353634,
"fcm_dpo/delta": 0.049615129828453064,
"fcm_dpo/margin": 89.30415344238281,
"fcm_dpo/q_t": 0.4181307256221771,
"grad_norm": 8.591902732849121,
"learning_rate": 1.4374663593999256e-07,
"logits/chosen": 0.8444277048110962,
"logits/rejected": 0.8045238256454468,
"logps/chosen": -231.84263610839844,
"logps/ref_chosen": -53.75125503540039,
"logps/ref_rejected": -77.17623901367188,
"logps/rejected": -344.57177734375,
"loss": 1.1212,
"margin_dpo/margin_mean": 89.30415344238281,
"margin_dpo/margin_std": 128.15696716308594,
"step": 448
},
{
"KL/chosen_KL_mean": -212.66348266601562,
"KL/mean": -235.39556884765625,
"KL/rejected_KL_mean": -258.1276550292969,
"KL/std": 124.3266372680664,
"epoch": 0.6787603930461074,
"fcm_dpo/beta": 0.0043518864549696445,
"fcm_dpo/delta": 0.06915242969989777,
"fcm_dpo/margin": 45.464168548583984,
"fcm_dpo/q_t": 0.4554653763771057,
"grad_norm": 18.149805068969727,
"learning_rate": 1.4255127197770707e-07,
"logits/chosen": 0.6996891498565674,
"logits/rejected": 0.7006729245185852,
"logps/chosen": -288.4908447265625,
"logps/ref_chosen": -75.82737731933594,
"logps/ref_rejected": -82.20687866210938,
"logps/rejected": -340.33453369140625,
"loss": 1.2685,
"margin_dpo/margin_mean": 45.464168548583984,
"margin_dpo/margin_std": 122.72837829589844,
"step": 449
},
{
"KL/chosen_KL_mean": -170.85467529296875,
"KL/mean": -209.89654541015625,
"KL/rejected_KL_mean": -248.93841552734375,
"KL/std": 123.30638122558594,
"epoch": 0.6802721088435374,
"fcm_dpo/beta": 0.0044828737154603004,
"fcm_dpo/delta": 0.060484807938337326,
"fcm_dpo/margin": 78.08375549316406,
"fcm_dpo/q_t": 0.42500025033950806,
"grad_norm": 12.813301086425781,
"learning_rate": 1.4135891358732205e-07,
"logits/chosen": 0.9147292375564575,
"logits/rejected": 0.8186343908309937,
"logps/chosen": -217.97039794921875,
"logps/ref_chosen": -47.11572265625,
"logps/ref_rejected": -78.7546615600586,
"logps/rejected": -327.6930847167969,
"loss": 1.1614,
"margin_dpo/margin_mean": 78.08374786376953,
"margin_dpo/margin_std": 141.2745361328125,
"step": 450
},
{
"KL/chosen_KL_mean": -174.08990478515625,
"KL/mean": -208.5985107421875,
"KL/rejected_KL_mean": -243.10711669921875,
"KL/std": 115.6333236694336,
"epoch": 0.6817838246409675,
"fcm_dpo/beta": 0.004921277053654194,
"fcm_dpo/delta": 0.07145331054925919,
"fcm_dpo/margin": 69.0172119140625,
"fcm_dpo/q_t": 0.4246895909309387,
"grad_norm": 13.280653953552246,
"learning_rate": 1.4016959412166437e-07,
"logits/chosen": 0.7936139106750488,
"logits/rejected": 0.7563437223434448,
"logps/chosen": -237.44033813476562,
"logps/ref_chosen": -63.350440979003906,
"logps/ref_rejected": -76.28530883789062,
"logps/rejected": -319.3924255371094,
"loss": 1.1558,
"margin_dpo/margin_mean": 69.0172119140625,
"margin_dpo/margin_std": 119.37599182128906,
"step": 451
},
{
"KL/chosen_KL_mean": -174.51087951660156,
"KL/mean": -214.18081665039062,
"KL/rejected_KL_mean": -253.8507843017578,
"KL/std": 111.76406860351562,
"epoch": 0.6832955404383976,
"fcm_dpo/beta": 0.004958437755703926,
"fcm_dpo/delta": 0.008032577112317085,
"fcm_dpo/margin": 79.33988952636719,
"fcm_dpo/q_t": 0.4116581082344055,
"grad_norm": 13.893336296081543,
"learning_rate": 1.3898334684855645e-07,
"logits/chosen": 0.8049842119216919,
"logits/rejected": 0.7350988388061523,
"logps/chosen": -230.09671020507812,
"logps/ref_chosen": -55.58583450317383,
"logps/ref_rejected": -77.68738555908203,
"logps/rejected": -331.53814697265625,
"loss": 1.1346,
"margin_dpo/margin_mean": 79.33989715576172,
"margin_dpo/margin_std": 134.18746948242188,
"step": 452
},
{
"KL/chosen_KL_mean": -176.7831268310547,
"KL/mean": -212.78689575195312,
"KL/rejected_KL_mean": -248.79066467285156,
"KL/std": 115.48379516601562,
"epoch": 0.6848072562358276,
"fcm_dpo/beta": 0.0051093436777591705,
"fcm_dpo/delta": 0.03934450447559357,
"fcm_dpo/margin": 72.00753784179688,
"fcm_dpo/q_t": 0.418560266494751,
"grad_norm": 16.030561447143555,
"learning_rate": 1.3780020494988445e-07,
"logits/chosen": 0.7819995880126953,
"logits/rejected": 0.7671021223068237,
"logps/chosen": -238.5613250732422,
"logps/ref_chosen": -61.778202056884766,
"logps/ref_rejected": -71.51403045654297,
"logps/rejected": -320.3046875,
"loss": 1.1529,
"margin_dpo/margin_mean": 72.00753784179688,
"margin_dpo/margin_std": 127.00164794921875,
"step": 453
},
{
"KL/chosen_KL_mean": -159.35975646972656,
"KL/mean": -201.83590698242188,
"KL/rejected_KL_mean": -244.31204223632812,
"KL/std": 112.93978881835938,
"epoch": 0.6863189720332578,
"fcm_dpo/beta": 0.005041994154453278,
"fcm_dpo/delta": -0.03642263263463974,
"fcm_dpo/margin": 84.95228576660156,
"fcm_dpo/q_t": 0.40106698870658875,
"grad_norm": 11.94836139678955,
"learning_rate": 1.366202015206706e-07,
"logits/chosen": 0.8330647945404053,
"logits/rejected": 0.80307936668396,
"logps/chosen": -210.9549102783203,
"logps/ref_chosen": -51.59515380859375,
"logps/ref_rejected": -63.96732711791992,
"logps/rejected": -308.27935791015625,
"loss": 1.0955,
"margin_dpo/margin_mean": 84.95228576660156,
"margin_dpo/margin_std": 127.11490631103516,
"step": 454
},
{
"KL/chosen_KL_mean": -188.39620971679688,
"KL/mean": -228.36636352539062,
"KL/rejected_KL_mean": -268.3365173339844,
"KL/std": 116.3778076171875,
"epoch": 0.6878306878306878,
"fcm_dpo/beta": 0.005019802134484053,
"fcm_dpo/delta": -0.001610086765140295,
"fcm_dpo/margin": 79.9402847290039,
"fcm_dpo/q_t": 0.4116509258747101,
"grad_norm": 14.03894329071045,
"learning_rate": 1.354433695681474e-07,
"logits/chosen": 0.7549115419387817,
"logits/rejected": 0.7353414297103882,
"logps/chosen": -259.04791259765625,
"logps/ref_chosen": -70.65170288085938,
"logps/ref_rejected": -77.44276428222656,
"logps/rejected": -345.779296875,
"loss": 1.1156,
"margin_dpo/margin_mean": 79.9402847290039,
"margin_dpo/margin_std": 126.06718444824219,
"step": 455
},
{
"KL/chosen_KL_mean": -179.34715270996094,
"KL/mean": -218.92030334472656,
"KL/rejected_KL_mean": -258.49346923828125,
"KL/std": 115.43203735351562,
"epoch": 0.6893424036281179,
"fcm_dpo/beta": 0.004967734683305025,
"fcm_dpo/delta": 0.00764651782810688,
"fcm_dpo/margin": 79.14630126953125,
"fcm_dpo/q_t": 0.41450706124305725,
"grad_norm": 17.639022827148438,
"learning_rate": 1.3426974201083439e-07,
"logits/chosen": 0.7384557723999023,
"logits/rejected": 0.6885507106781006,
"logps/chosen": -235.7454376220703,
"logps/ref_chosen": -56.398284912109375,
"logps/ref_rejected": -82.61642456054688,
"logps/rejected": -341.10986328125,
"loss": 1.1349,
"margin_dpo/margin_mean": 79.14630126953125,
"margin_dpo/margin_std": 137.09796142578125,
"step": 456
},
{
"KL/chosen_KL_mean": -175.0521240234375,
"KL/mean": -215.41928100585938,
"KL/rejected_KL_mean": -255.78640747070312,
"KL/std": 118.11895751953125,
"epoch": 0.690854119425548,
"fcm_dpo/beta": 0.004974281415343285,
"fcm_dpo/delta": -0.0028288550674915314,
"fcm_dpo/margin": 80.73428344726562,
"fcm_dpo/q_t": 0.40718016028404236,
"grad_norm": 14.620522499084473,
"learning_rate": 1.3309935167761717e-07,
"logits/chosen": 0.8967911005020142,
"logits/rejected": 0.8270587921142578,
"logps/chosen": -219.772705078125,
"logps/ref_chosen": -44.72057342529297,
"logps/ref_rejected": -68.1158676147461,
"logps/rejected": -323.90228271484375,
"loss": 1.0996,
"margin_dpo/margin_mean": 80.73428344726562,
"margin_dpo/margin_std": 113.923583984375,
"step": 457
},
{
"KL/chosen_KL_mean": -170.12730407714844,
"KL/mean": -215.46165466308594,
"KL/rejected_KL_mean": -260.7959899902344,
"KL/std": 120.08485412597656,
"epoch": 0.6923658352229781,
"fcm_dpo/beta": 0.004880758002400398,
"fcm_dpo/delta": -0.05446285754442215,
"fcm_dpo/margin": 90.668701171875,
"fcm_dpo/q_t": 0.3978913426399231,
"grad_norm": 12.83903980255127,
"learning_rate": 1.3193223130682936e-07,
"logits/chosen": 0.8211711645126343,
"logits/rejected": 0.7230532765388489,
"logps/chosen": -220.13299560546875,
"logps/ref_chosen": -50.00569152832031,
"logps/ref_rejected": -87.50015258789062,
"logps/rejected": -348.296142578125,
"loss": 1.0924,
"margin_dpo/margin_mean": 90.668701171875,
"margin_dpo/margin_std": 135.3630828857422,
"step": 458
},
{
"KL/chosen_KL_mean": -168.60906982421875,
"KL/mean": -220.11251831054688,
"KL/rejected_KL_mean": -271.615966796875,
"KL/std": 143.58975219726562,
"epoch": 0.6938775510204082,
"fcm_dpo/beta": 0.004629744682461023,
"fcm_dpo/delta": -0.10486065596342087,
"fcm_dpo/margin": 103.00687408447266,
"fcm_dpo/q_t": 0.3852209448814392,
"grad_norm": 11.949894905090332,
"learning_rate": 1.3076841354533658e-07,
"logits/chosen": 0.8680670261383057,
"logits/rejected": 0.8436014652252197,
"logps/chosen": -233.9870147705078,
"logps/ref_chosen": -65.37794494628906,
"logps/ref_rejected": -88.19244384765625,
"logps/rejected": -359.80841064453125,
"loss": 1.0705,
"margin_dpo/margin_mean": 103.00686645507812,
"margin_dpo/margin_std": 135.36453247070312,
"step": 459
},
{
"KL/chosen_KL_mean": -178.06622314453125,
"KL/mean": -229.13934326171875,
"KL/rejected_KL_mean": -280.21246337890625,
"KL/std": 123.35194396972656,
"epoch": 0.6953892668178382,
"fcm_dpo/beta": 0.004125220701098442,
"fcm_dpo/delta": -0.02931346371769905,
"fcm_dpo/margin": 102.146240234375,
"fcm_dpo/q_t": 0.4028850197792053,
"grad_norm": 10.467647552490234,
"learning_rate": 1.2960793094762345e-07,
"logits/chosen": 0.8451239466667175,
"logits/rejected": 0.7452915906906128,
"logps/chosen": -242.62789916992188,
"logps/ref_chosen": -64.5616683959961,
"logps/ref_rejected": -88.67890167236328,
"logps/rejected": -368.891357421875,
"loss": 1.0814,
"margin_dpo/margin_mean": 102.14624786376953,
"margin_dpo/margin_std": 135.01199340820312,
"step": 460
},
{
"KL/chosen_KL_mean": -147.82064819335938,
"KL/mean": -199.00198364257812,
"KL/rejected_KL_mean": -250.18333435058594,
"KL/std": 114.1442642211914,
"epoch": 0.6969009826152683,
"fcm_dpo/beta": 0.003954247571527958,
"fcm_dpo/delta": -0.015024319291114807,
"fcm_dpo/margin": 102.36270141601562,
"fcm_dpo/q_t": 0.40458911657333374,
"grad_norm": 10.667238235473633,
"learning_rate": 1.2845081597488286e-07,
"logits/chosen": 0.9264631867408752,
"logits/rejected": 0.8578430414199829,
"logps/chosen": -197.29855346679688,
"logps/ref_chosen": -49.4779167175293,
"logps/ref_rejected": -72.65262603759766,
"logps/rejected": -322.8359680175781,
"loss": 1.0864,
"margin_dpo/margin_mean": 102.36270141601562,
"margin_dpo/margin_std": 123.31861114501953,
"step": 461
},
{
"KL/chosen_KL_mean": -162.76797485351562,
"KL/mean": -213.944580078125,
"KL/rejected_KL_mean": -265.1211853027344,
"KL/std": 106.37005615234375,
"epoch": 0.6984126984126984,
"fcm_dpo/beta": 0.004021008498966694,
"fcm_dpo/delta": -0.015222661197185516,
"fcm_dpo/margin": 102.35319519042969,
"fcm_dpo/q_t": 0.402321994304657,
"grad_norm": 10.825149536132812,
"learning_rate": 1.27297100994108e-07,
"logits/chosen": 0.8083179593086243,
"logits/rejected": 0.7733680009841919,
"logps/chosen": -223.26309204101562,
"logps/ref_chosen": -60.4951171875,
"logps/ref_rejected": -74.82136535644531,
"logps/rejected": -339.94256591796875,
"loss": 1.071,
"margin_dpo/margin_mean": 102.35319519042969,
"margin_dpo/margin_std": 118.42103576660156,
"step": 462
},
{
"KL/chosen_KL_mean": -198.61944580078125,
"KL/mean": -227.19964599609375,
"KL/rejected_KL_mean": -255.77984619140625,
"KL/std": 110.61781311035156,
"epoch": 0.6999244142101285,
"fcm_dpo/beta": 0.004229356534779072,
"fcm_dpo/delta": 0.04422418028116226,
"fcm_dpo/margin": 57.160400390625,
"fcm_dpo/q_t": 0.444000780582428,
"grad_norm": 16.760709762573242,
"learning_rate": 1.2614681827718695e-07,
"logits/chosen": 0.8112308979034424,
"logits/rejected": 0.807045578956604,
"logps/chosen": -266.3045654296875,
"logps/ref_chosen": -67.68511962890625,
"logps/ref_rejected": -71.32196044921875,
"logps/rejected": -327.101806640625,
"loss": 1.2275,
"margin_dpo/margin_mean": 57.160404205322266,
"margin_dpo/margin_std": 124.53732299804688,
"step": 463
},
{
"KL/chosen_KL_mean": -178.37449645996094,
"KL/mean": -226.03128051757812,
"KL/rejected_KL_mean": -273.68804931640625,
"KL/std": 128.52586364746094,
"epoch": 0.7014361300075586,
"fcm_dpo/beta": 0.004276517778635025,
"fcm_dpo/delta": -0.010937310755252838,
"fcm_dpo/margin": 95.31355285644531,
"fcm_dpo/q_t": 0.40591591596603394,
"grad_norm": 10.75070571899414,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 0.8277842998504639,
"logits/rejected": 0.8054889440536499,
"logps/chosen": -237.54013061523438,
"logps/ref_chosen": -59.16564178466797,
"logps/ref_rejected": -69.56146240234375,
"logps/rejected": -343.24951171875,
"loss": 1.1112,
"margin_dpo/margin_mean": 95.31355285644531,
"margin_dpo/margin_std": 142.11141967773438,
"step": 464
},
{
"KL/chosen_KL_mean": -185.91439819335938,
"KL/mean": -228.27606201171875,
"KL/rejected_KL_mean": -270.63775634765625,
"KL/std": 119.63814544677734,
"epoch": 0.7029478458049887,
"fcm_dpo/beta": 0.0042515965178608894,
"fcm_dpo/delta": 0.047829769551754,
"fcm_dpo/margin": 84.72332000732422,
"fcm_dpo/q_t": 0.41904404759407043,
"grad_norm": 11.757485389709473,
"learning_rate": 1.238566782415197e-07,
"logits/chosen": 0.8830685615539551,
"logits/rejected": 0.8336352109909058,
"logps/chosen": -244.42807006835938,
"logps/ref_chosen": -58.513671875,
"logps/ref_rejected": -84.31745910644531,
"logps/rejected": -354.9552001953125,
"loss": 1.1376,
"margin_dpo/margin_mean": 84.72331237792969,
"margin_dpo/margin_std": 133.2949981689453,
"step": 465
},
{
"KL/chosen_KL_mean": -204.39483642578125,
"KL/mean": -224.01669311523438,
"KL/rejected_KL_mean": -243.63856506347656,
"KL/std": 127.39469146728516,
"epoch": 0.7044595616024187,
"fcm_dpo/beta": 0.004723524209111929,
"fcm_dpo/delta": 0.07361281663179398,
"fcm_dpo/margin": 39.24374008178711,
"fcm_dpo/q_t": 0.4571523070335388,
"grad_norm": 17.618196487426758,
"learning_rate": 1.2271688498291334e-07,
"logits/chosen": 0.8117011189460754,
"logits/rejected": 0.8161472082138062,
"logps/chosen": -277.66064453125,
"logps/ref_chosen": -73.26580810546875,
"logps/ref_rejected": -74.83621215820312,
"logps/rejected": -318.47479248046875,
"loss": 1.3039,
"margin_dpo/margin_mean": 39.24374008178711,
"margin_dpo/margin_std": 132.60060119628906,
"step": 466
},
{
"KL/chosen_KL_mean": -177.04547119140625,
"KL/mean": -218.3562774658203,
"KL/rejected_KL_mean": -259.6671142578125,
"KL/std": 126.77050018310547,
"epoch": 0.7059712773998488,
"fcm_dpo/beta": 0.004716761410236359,
"fcm_dpo/delta": 0.012419500388205051,
"fcm_dpo/margin": 82.62162780761719,
"fcm_dpo/q_t": 0.4150483012199402,
"grad_norm": 11.141740798950195,
"learning_rate": 1.2158065210664848e-07,
"logits/chosen": 0.872531533241272,
"logits/rejected": 0.7571277022361755,
"logps/chosen": -224.62493896484375,
"logps/ref_chosen": -47.57947540283203,
"logps/ref_rejected": -78.68522644042969,
"logps/rejected": -338.3523254394531,
"loss": 1.1276,
"margin_dpo/margin_mean": 82.62162780761719,
"margin_dpo/margin_std": 138.0587158203125,
"step": 467
},
{
"KL/chosen_KL_mean": -173.1776123046875,
"KL/mean": -220.29592895507812,
"KL/rejected_KL_mean": -267.41424560546875,
"KL/std": 122.3834228515625,
"epoch": 0.7074829931972789,
"fcm_dpo/beta": 0.004614764358848333,
"fcm_dpo/delta": -0.044785935431718826,
"fcm_dpo/margin": 94.23664855957031,
"fcm_dpo/q_t": 0.39811086654663086,
"grad_norm": 13.203627586364746,
"learning_rate": 1.204480113956011e-07,
"logits/chosen": 0.8035502433776855,
"logits/rejected": 0.7918655872344971,
"logps/chosen": -237.10537719726562,
"logps/ref_chosen": -63.92778778076172,
"logps/ref_rejected": -76.51626586914062,
"logps/rejected": -343.9305114746094,
"loss": 1.1,
"margin_dpo/margin_mean": 94.23664855957031,
"margin_dpo/margin_std": 143.19485473632812,
"step": 468
},
{
"KL/chosen_KL_mean": -173.43836975097656,
"KL/mean": -216.24713134765625,
"KL/rejected_KL_mean": -259.055908203125,
"KL/std": 119.78315734863281,
"epoch": 0.708994708994709,
"fcm_dpo/beta": 0.004494948778301477,
"fcm_dpo/delta": 0.015453502535820007,
"fcm_dpo/margin": 85.61750793457031,
"fcm_dpo/q_t": 0.4122866988182068,
"grad_norm": 13.859041213989258,
"learning_rate": 1.1931899453216697e-07,
"logits/chosen": 0.8977210521697998,
"logits/rejected": 0.8815995454788208,
"logps/chosen": -232.49655151367188,
"logps/ref_chosen": -59.05818176269531,
"logps/ref_rejected": -75.67672729492188,
"logps/rejected": -334.73260498046875,
"loss": 1.104,
"margin_dpo/margin_mean": 85.61751556396484,
"margin_dpo/margin_std": 115.52572631835938,
"step": 469
},
{
"KL/chosen_KL_mean": -161.53884887695312,
"KL/mean": -206.596435546875,
"KL/rejected_KL_mean": -251.65402221679688,
"KL/std": 114.833740234375,
"epoch": 0.7105064247921391,
"fcm_dpo/beta": 0.004632364027202129,
"fcm_dpo/delta": -0.022567128762602806,
"fcm_dpo/margin": 90.11515808105469,
"fcm_dpo/q_t": 0.4039248526096344,
"grad_norm": 11.321891784667969,
"learning_rate": 1.1819363309737438e-07,
"logits/chosen": 0.8473949432373047,
"logits/rejected": 0.7967231273651123,
"logps/chosen": -209.40628051757812,
"logps/ref_chosen": -47.86743927001953,
"logps/ref_rejected": -65.96859741210938,
"logps/rejected": -317.62261962890625,
"loss": 1.0983,
"margin_dpo/margin_mean": 90.11515808105469,
"margin_dpo/margin_std": 130.05743408203125,
"step": 470
},
{
"KL/chosen_KL_mean": -158.923828125,
"KL/mean": -207.42820739746094,
"KL/rejected_KL_mean": -255.9325714111328,
"KL/std": 111.91524505615234,
"epoch": 0.7120181405895691,
"fcm_dpo/beta": 0.004406137391924858,
"fcm_dpo/delta": -0.03516135737299919,
"fcm_dpo/margin": 97.00875854492188,
"fcm_dpo/q_t": 0.3985697329044342,
"grad_norm": 12.091167449951172,
"learning_rate": 1.1707195857000215e-07,
"logits/chosen": 0.8162208795547485,
"logits/rejected": 0.772452712059021,
"logps/chosen": -216.70169067382812,
"logps/ref_chosen": -57.777854919433594,
"logps/ref_rejected": -73.81172180175781,
"logps/rejected": -329.7442932128906,
"loss": 1.073,
"margin_dpo/margin_mean": 97.00875854492188,
"margin_dpo/margin_std": 121.57257080078125,
"step": 471
},
{
"KL/chosen_KL_mean": -159.90924072265625,
"KL/mean": -197.77468872070312,
"KL/rejected_KL_mean": -235.64015197753906,
"KL/std": 118.32003784179688,
"epoch": 0.7135298563869993,
"fcm_dpo/beta": 0.004661578685045242,
"fcm_dpo/delta": 0.05358702316880226,
"fcm_dpo/margin": 75.73089599609375,
"fcm_dpo/q_t": 0.4227460026741028,
"grad_norm": 12.47477912902832,
"learning_rate": 1.1595400232569768e-07,
"logits/chosen": 0.8483719825744629,
"logits/rejected": 0.8078272342681885,
"logps/chosen": -215.81790161132812,
"logps/ref_chosen": -55.908668518066406,
"logps/ref_rejected": -74.70294189453125,
"logps/rejected": -310.34307861328125,
"loss": 1.1769,
"margin_dpo/margin_mean": 75.73089599609375,
"margin_dpo/margin_std": 150.27847290039062,
"step": 472
},
{
"KL/chosen_KL_mean": -164.11631774902344,
"KL/mean": -217.00555419921875,
"KL/rejected_KL_mean": -269.894775390625,
"KL/std": 144.6307373046875,
"epoch": 0.7150415721844293,
"fcm_dpo/beta": 0.004342665895819664,
"fcm_dpo/delta": -0.07786207646131516,
"fcm_dpo/margin": 105.77847290039062,
"fcm_dpo/q_t": 0.3978637456893921,
"grad_norm": 13.936591148376465,
"learning_rate": 1.1483979563610069e-07,
"logits/chosen": 0.8966566324234009,
"logits/rejected": 0.8024401068687439,
"logps/chosen": -218.27719116210938,
"logps/ref_chosen": -54.16088104248047,
"logps/ref_rejected": -92.76789855957031,
"logps/rejected": -362.66265869140625,
"loss": 1.0969,
"margin_dpo/margin_mean": 105.7784652709961,
"margin_dpo/margin_std": 167.810302734375,
"step": 473
},
{
"KL/chosen_KL_mean": -165.61419677734375,
"KL/mean": -208.4927520751953,
"KL/rejected_KL_mean": -251.37130737304688,
"KL/std": 123.07339477539062,
"epoch": 0.7165532879818595,
"fcm_dpo/beta": 0.004392385482788086,
"fcm_dpo/delta": 0.028430059552192688,
"fcm_dpo/margin": 85.75711822509766,
"fcm_dpo/q_t": 0.41677016019821167,
"grad_norm": 14.764763832092285,
"learning_rate": 1.1372936966796709e-07,
"logits/chosen": 0.8863992094993591,
"logits/rejected": 0.8298487663269043,
"logps/chosen": -212.29989624023438,
"logps/ref_chosen": -46.685707092285156,
"logps/ref_rejected": -71.44731903076172,
"logps/rejected": -322.818603515625,
"loss": 1.1344,
"margin_dpo/margin_mean": 85.75711822509766,
"margin_dpo/margin_std": 142.21832275390625,
"step": 474
},
{
"KL/chosen_KL_mean": -165.1353302001953,
"KL/mean": -222.8227081298828,
"KL/rejected_KL_mean": -280.51007080078125,
"KL/std": 124.3143539428711,
"epoch": 0.7180650037792895,
"fcm_dpo/beta": 0.0040479195304214954,
"fcm_dpo/delta": -0.09077291190624237,
"fcm_dpo/margin": 115.37474060058594,
"fcm_dpo/q_t": 0.38997000455856323,
"grad_norm": 7.723613739013672,
"learning_rate": 1.126227554822985e-07,
"logits/chosen": 0.8512206077575684,
"logits/rejected": 0.8066825866699219,
"logps/chosen": -223.6226348876953,
"logps/ref_chosen": -58.4873046875,
"logps/ref_rejected": -87.00187683105469,
"logps/rejected": -367.511962890625,
"loss": 1.0486,
"margin_dpo/margin_mean": 115.37474060058594,
"margin_dpo/margin_std": 141.55166625976562,
"step": 475
},
{
"KL/chosen_KL_mean": -200.2625732421875,
"KL/mean": -234.19955444335938,
"KL/rejected_KL_mean": -268.13653564453125,
"KL/std": 133.18336486816406,
"epoch": 0.7195767195767195,
"fcm_dpo/beta": 0.004081051331013441,
"fcm_dpo/delta": 0.029893899336457253,
"fcm_dpo/margin": 67.8740005493164,
"fcm_dpo/q_t": 0.4379327893257141,
"grad_norm": 12.724250793457031,
"learning_rate": 1.1151998403347243e-07,
"logits/chosen": 0.7401478290557861,
"logits/rejected": 0.744006872177124,
"logps/chosen": -275.6441955566406,
"logps/ref_chosen": -75.38162231445312,
"logps/ref_rejected": -76.99822235107422,
"logps/rejected": -345.134765625,
"loss": 1.2061,
"margin_dpo/margin_mean": 67.87399291992188,
"margin_dpo/margin_std": 140.630615234375,
"step": 476
},
{
"KL/chosen_KL_mean": -194.9725799560547,
"KL/mean": -238.0779266357422,
"KL/rejected_KL_mean": -281.1832580566406,
"KL/std": 134.39028930664062,
"epoch": 0.7210884353741497,
"fcm_dpo/beta": 0.004254788625985384,
"fcm_dpo/delta": 0.04072454944252968,
"fcm_dpo/margin": 86.21067810058594,
"fcm_dpo/q_t": 0.41996684670448303,
"grad_norm": 12.63784122467041,
"learning_rate": 1.1042108616837692e-07,
"logits/chosen": 0.8575061559677124,
"logits/rejected": 0.8164290189743042,
"logps/chosen": -256.04595947265625,
"logps/ref_chosen": -61.073387145996094,
"logps/ref_rejected": -81.34375,
"logps/rejected": -362.5270080566406,
"loss": 1.1771,
"margin_dpo/margin_mean": 86.21067810058594,
"margin_dpo/margin_std": 171.40589904785156,
"step": 477
},
{
"KL/chosen_KL_mean": -172.208984375,
"KL/mean": -203.42808532714844,
"KL/rejected_KL_mean": -234.64720153808594,
"KL/std": 107.170166015625,
"epoch": 0.7226001511715797,
"fcm_dpo/beta": 0.004686059430241585,
"fcm_dpo/delta": 0.12666893005371094,
"fcm_dpo/margin": 62.438201904296875,
"fcm_dpo/q_t": 0.4363289177417755,
"grad_norm": 15.702838897705078,
"learning_rate": 1.0932609262554746e-07,
"logits/chosen": 0.7970494031906128,
"logits/rejected": 0.8083937168121338,
"logps/chosen": -229.3762969970703,
"logps/ref_chosen": -57.16731643676758,
"logps/ref_rejected": -53.30917739868164,
"logps/rejected": -287.95635986328125,
"loss": 1.2065,
"margin_dpo/margin_mean": 62.438201904296875,
"margin_dpo/margin_std": 129.8063507080078,
"step": 478
},
{
"KL/chosen_KL_mean": -178.65333557128906,
"KL/mean": -206.3043212890625,
"KL/rejected_KL_mean": -233.95529174804688,
"KL/std": 111.01861572265625,
"epoch": 0.7241118669690099,
"fcm_dpo/beta": 0.004972537979483604,
"fcm_dpo/delta": 0.015346960164606571,
"fcm_dpo/margin": 55.30194091796875,
"fcm_dpo/q_t": 0.43899282813072205,
"grad_norm": 15.627010345458984,
"learning_rate": 1.0823503403430734e-07,
"logits/chosen": 0.8058916926383972,
"logits/rejected": 0.7788633108139038,
"logps/chosen": -237.566650390625,
"logps/ref_chosen": -58.91331481933594,
"logps/ref_rejected": -63.7403450012207,
"logps/rejected": -297.69561767578125,
"loss": 1.2337,
"margin_dpo/margin_mean": 55.30194091796875,
"margin_dpo/margin_std": 132.65174865722656,
"step": 479
},
{
"KL/chosen_KL_mean": -186.09510803222656,
"KL/mean": -228.88623046875,
"KL/rejected_KL_mean": -271.6773376464844,
"KL/std": 116.18428039550781,
"epoch": 0.7256235827664399,
"fcm_dpo/beta": 0.005056783556938171,
"fcm_dpo/delta": -0.051207661628723145,
"fcm_dpo/margin": 85.58222961425781,
"fcm_dpo/q_t": 0.4012352526187897,
"grad_norm": 15.959285736083984,
"learning_rate": 1.0714794091391072e-07,
"logits/chosen": 0.7871379852294922,
"logits/rejected": 0.783062219619751,
"logps/chosen": -248.89572143554688,
"logps/ref_chosen": -62.80061340332031,
"logps/ref_rejected": -67.58859252929688,
"logps/rejected": -339.26593017578125,
"loss": 1.1235,
"margin_dpo/margin_mean": 85.58222961425781,
"margin_dpo/margin_std": 135.55453491210938,
"step": 480
},
{
"KL/chosen_KL_mean": -183.6075439453125,
"KL/mean": -224.15196228027344,
"KL/rejected_KL_mean": -264.6963806152344,
"KL/std": 122.73133087158203,
"epoch": 0.72713529856387,
"fcm_dpo/beta": 0.00472680851817131,
"fcm_dpo/delta": 0.020053943619132042,
"fcm_dpo/margin": 81.0888442993164,
"fcm_dpo/q_t": 0.41520315408706665,
"grad_norm": 12.672243118286133,
"learning_rate": 1.0606484367268906e-07,
"logits/chosen": 0.7809611558914185,
"logits/rejected": 0.7874898314476013,
"logps/chosen": -248.89404296875,
"logps/ref_chosen": -65.28649139404297,
"logps/ref_rejected": -70.78668212890625,
"logps/rejected": -335.4830627441406,
"loss": 1.1305,
"margin_dpo/margin_mean": 81.0888442993164,
"margin_dpo/margin_std": 134.0117645263672,
"step": 481
},
{
"KL/chosen_KL_mean": -202.88668823242188,
"KL/mean": -247.89273071289062,
"KL/rejected_KL_mean": -292.89874267578125,
"KL/std": 136.02828979492188,
"epoch": 0.7286470143613001,
"fcm_dpo/beta": 0.004762349650263786,
"fcm_dpo/delta": -0.03688174486160278,
"fcm_dpo/margin": 90.01205444335938,
"fcm_dpo/q_t": 0.4086783528327942,
"grad_norm": 15.459307670593262,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 0.7737709879875183,
"logits/rejected": 0.640209436416626,
"logps/chosen": -263.7928771972656,
"logps/ref_chosen": -60.906185150146484,
"logps/ref_rejected": -103.44656372070312,
"logps/rejected": -396.3453369140625,
"loss": 1.1449,
"margin_dpo/margin_mean": 90.01204681396484,
"margin_dpo/margin_std": 167.98159790039062,
"step": 482
},
{
"KL/chosen_KL_mean": -170.41964721679688,
"KL/mean": -221.99755859375,
"KL/rejected_KL_mean": -273.57550048828125,
"KL/std": 118.67953491210938,
"epoch": 0.7301587301587301,
"fcm_dpo/beta": 0.004441915079951286,
"fcm_dpo/delta": -0.07519707083702087,
"fcm_dpo/margin": 103.15583801269531,
"fcm_dpo/q_t": 0.3927938640117645,
"grad_norm": 13.764213562011719,
"learning_rate": 1.0391075790138232e-07,
"logits/chosen": 0.9073851704597473,
"logits/rejected": 0.8138213157653809,
"logps/chosen": -223.61166381835938,
"logps/ref_chosen": -53.192012786865234,
"logps/ref_rejected": -81.83927154541016,
"logps/rejected": -355.4147644042969,
"loss": 1.0646,
"margin_dpo/margin_mean": 103.15583801269531,
"margin_dpo/margin_std": 135.55938720703125,
"step": 483
},
{
"KL/chosen_KL_mean": -176.34378051757812,
"KL/mean": -209.01864624023438,
"KL/rejected_KL_mean": -241.69349670410156,
"KL/std": 116.74180603027344,
"epoch": 0.7316704459561603,
"fcm_dpo/beta": 0.004746517166495323,
"fcm_dpo/delta": 0.10179544240236282,
"fcm_dpo/margin": 65.34970092773438,
"fcm_dpo/q_t": 0.4296528100967407,
"grad_norm": 16.811372756958008,
"learning_rate": 1.0283982962570681e-07,
"logits/chosen": 0.8936479687690735,
"logits/rejected": 0.8692023754119873,
"logps/chosen": -234.11325073242188,
"logps/ref_chosen": -57.76945877075195,
"logps/ref_rejected": -71.6829833984375,
"logps/rejected": -313.37646484375,
"loss": 1.1563,
"margin_dpo/margin_mean": 65.34970092773438,
"margin_dpo/margin_std": 101.48129272460938,
"step": 484
},
{
"KL/chosen_KL_mean": -182.8577117919922,
"KL/mean": -219.1405029296875,
"KL/rejected_KL_mean": -255.42327880859375,
"KL/std": 115.46862030029297,
"epoch": 0.7331821617535903,
"fcm_dpo/beta": 0.004539255518466234,
"fcm_dpo/delta": -0.049730200320482254,
"fcm_dpo/margin": 72.56556701660156,
"fcm_dpo/q_t": 0.4225718379020691,
"grad_norm": 13.539854049682617,
"learning_rate": 1.0177301773633992e-07,
"logits/chosen": 0.8523825407028198,
"logits/rejected": 0.838424801826477,
"logps/chosen": -239.49356079101562,
"logps/ref_chosen": -56.63584899902344,
"logps/ref_rejected": -70.85614013671875,
"logps/rejected": -326.2794189453125,
"loss": 1.153,
"margin_dpo/margin_mean": 72.56556701660156,
"margin_dpo/margin_std": 115.97235107421875,
"step": 485
},
{
"KL/chosen_KL_mean": -208.73068237304688,
"KL/mean": -245.28561401367188,
"KL/rejected_KL_mean": -281.84051513671875,
"KL/std": 138.67807006835938,
"epoch": 0.7346938775510204,
"fcm_dpo/beta": 0.004761700518429279,
"fcm_dpo/delta": 0.06264105439186096,
"fcm_dpo/margin": 73.10984802246094,
"fcm_dpo/q_t": 0.4274640679359436,
"grad_norm": 11.167536735534668,
"learning_rate": 1.007103520743035e-07,
"logits/chosen": 0.883628249168396,
"logits/rejected": 0.7876710295677185,
"logps/chosen": -265.07769775390625,
"logps/ref_chosen": -56.347023010253906,
"logps/ref_rejected": -85.97221374511719,
"logps/rejected": -367.812744140625,
"loss": 1.204,
"margin_dpo/margin_mean": 73.10984802246094,
"margin_dpo/margin_std": 163.16970825195312,
"step": 486
},
{
"KL/chosen_KL_mean": -188.4417724609375,
"KL/mean": -229.6407470703125,
"KL/rejected_KL_mean": -270.8397216796875,
"KL/std": 127.54862213134766,
"epoch": 0.7362055933484505,
"fcm_dpo/beta": 0.00484395632520318,
"fcm_dpo/delta": 0.0010717334225773811,
"fcm_dpo/margin": 82.39796447753906,
"fcm_dpo/q_t": 0.4158893823623657,
"grad_norm": 14.043888092041016,
"learning_rate": 9.965186236464046e-08,
"logits/chosen": 0.9236916303634644,
"logits/rejected": 0.8743011355400085,
"logps/chosen": -249.05899047851562,
"logps/ref_chosen": -60.617218017578125,
"logps/ref_rejected": -82.50975036621094,
"logps/rejected": -353.3494873046875,
"loss": 1.1434,
"margin_dpo/margin_mean": 82.39796447753906,
"margin_dpo/margin_std": 153.81063842773438,
"step": 487
},
{
"KL/chosen_KL_mean": -175.81893920898438,
"KL/mean": -222.56472778320312,
"KL/rejected_KL_mean": -269.31048583984375,
"KL/std": 118.71575164794922,
"epoch": 0.7377173091458806,
"fcm_dpo/beta": 0.00476812943816185,
"fcm_dpo/delta": -0.06080658361315727,
"fcm_dpo/margin": 93.4915771484375,
"fcm_dpo/q_t": 0.3985556662082672,
"grad_norm": 15.597735404968262,
"learning_rate": 9.859757821558337e-08,
"logits/chosen": 0.8415173888206482,
"logits/rejected": 0.8006629347801208,
"logps/chosen": -238.927978515625,
"logps/ref_chosen": -63.10905075073242,
"logps/ref_rejected": -82.49348449707031,
"logps/rejected": -351.8039855957031,
"loss": 1.0822,
"margin_dpo/margin_mean": 93.49156951904297,
"margin_dpo/margin_std": 130.8084716796875,
"step": 488
},
{
"KL/chosen_KL_mean": -205.70184326171875,
"KL/mean": -234.09732055664062,
"KL/rejected_KL_mean": -262.4927673339844,
"KL/std": 130.96896362304688,
"epoch": 0.7392290249433107,
"fcm_dpo/beta": 0.005044779274612665,
"fcm_dpo/delta": 0.13149571418762207,
"fcm_dpo/margin": 56.7909049987793,
"fcm_dpo/q_t": 0.4390004277229309,
"grad_norm": 13.851383209228516,
"learning_rate": 9.754752911772615e-08,
"logits/chosen": 0.8348220586776733,
"logits/rejected": 0.7962938547134399,
"logps/chosen": -270.6907958984375,
"logps/ref_chosen": -64.98896026611328,
"logps/ref_rejected": -84.39607238769531,
"logps/rejected": -346.88885498046875,
"loss": 1.235,
"margin_dpo/margin_mean": 56.79090881347656,
"margin_dpo/margin_std": 138.2364501953125,
"step": 489
},
{
"KL/chosen_KL_mean": -185.4239501953125,
"KL/mean": -219.5350341796875,
"KL/rejected_KL_mean": -253.6461181640625,
"KL/std": 125.25768280029297,
"epoch": 0.7407407407407407,
"fcm_dpo/beta": 0.005278023891150951,
"fcm_dpo/delta": 0.047999098896980286,
"fcm_dpo/margin": 68.22216796875,
"fcm_dpo/q_t": 0.42334112524986267,
"grad_norm": 12.744791984558105,
"learning_rate": 9.650174444319956e-08,
"logits/chosen": 0.8877488374710083,
"logits/rejected": 0.8769302368164062,
"logps/chosen": -247.33270263671875,
"logps/ref_chosen": -61.90874481201172,
"logps/ref_rejected": -70.58566284179688,
"logps/rejected": -324.2317810058594,
"loss": 1.2111,
"margin_dpo/margin_mean": 68.22216796875,
"margin_dpo/margin_std": 154.54383850097656,
"step": 490
},
{
"KL/chosen_KL_mean": -178.17283630371094,
"KL/mean": -213.24208068847656,
"KL/rejected_KL_mean": -248.31134033203125,
"KL/std": 113.33309936523438,
"epoch": 0.7422524565381708,
"fcm_dpo/beta": 0.005064789205789566,
"fcm_dpo/delta": -0.07285159081220627,
"fcm_dpo/margin": 70.13851165771484,
"fcm_dpo/q_t": 0.41720882058143616,
"grad_norm": 10.481760025024414,
"learning_rate": 9.546025344484868e-08,
"logits/chosen": 0.8015914559364319,
"logits/rejected": 0.7469338774681091,
"logps/chosen": -233.64852905273438,
"logps/ref_chosen": -55.47570037841797,
"logps/ref_rejected": -78.70318603515625,
"logps/rejected": -327.0145263671875,
"loss": 1.1531,
"margin_dpo/margin_mean": 70.13851165771484,
"margin_dpo/margin_std": 121.06025695800781,
"step": 491
},
{
"KL/chosen_KL_mean": -211.74505615234375,
"KL/mean": -244.18946838378906,
"KL/rejected_KL_mean": -276.6338806152344,
"KL/std": 131.94345092773438,
"epoch": 0.7437641723356009,
"fcm_dpo/beta": 0.005409009754657745,
"fcm_dpo/delta": 0.05510532110929489,
"fcm_dpo/margin": 64.88883209228516,
"fcm_dpo/q_t": 0.4252876937389374,
"grad_norm": 16.77579116821289,
"learning_rate": 9.442308525541589e-08,
"logits/chosen": 0.87005615234375,
"logits/rejected": 0.8110706806182861,
"logps/chosen": -279.03143310546875,
"logps/ref_chosen": -67.28638458251953,
"logps/ref_rejected": -82.78628540039062,
"logps/rejected": -359.420166015625,
"loss": 1.221,
"margin_dpo/margin_mean": 64.88883209228516,
"margin_dpo/margin_std": 148.9833984375,
"step": 492
},
{
"KL/chosen_KL_mean": -176.25186157226562,
"KL/mean": -222.16436767578125,
"KL/rejected_KL_mean": -268.0768737792969,
"KL/std": 127.96397399902344,
"epoch": 0.745275888133031,
"fcm_dpo/beta": 0.005210395436733961,
"fcm_dpo/delta": -0.10840007662773132,
"fcm_dpo/margin": 91.82500457763672,
"fcm_dpo/q_t": 0.3924015760421753,
"grad_norm": 17.075223922729492,
"learning_rate": 9.339026888672468e-08,
"logits/chosen": 0.7961744666099548,
"logits/rejected": 0.7387325763702393,
"logps/chosen": -232.17938232421875,
"logps/ref_chosen": -55.92750549316406,
"logps/ref_rejected": -79.12149810791016,
"logps/rejected": -347.1983642578125,
"loss": 1.0952,
"margin_dpo/margin_mean": 91.82499694824219,
"margin_dpo/margin_std": 140.10226440429688,
"step": 493
},
{
"KL/chosen_KL_mean": -182.12063598632812,
"KL/mean": -224.42507934570312,
"KL/rejected_KL_mean": -266.72955322265625,
"KL/std": 128.66799926757812,
"epoch": 0.7467876039304611,
"fcm_dpo/beta": 0.00474274717271328,
"fcm_dpo/delta": -0.0021576210856437683,
"fcm_dpo/margin": 84.60891723632812,
"fcm_dpo/q_t": 0.4120602011680603,
"grad_norm": 14.730215072631836,
"learning_rate": 9.236183322886945e-08,
"logits/chosen": 0.7500890493392944,
"logits/rejected": 0.7123274207115173,
"logps/chosen": -250.07473754882812,
"logps/ref_chosen": -67.95410919189453,
"logps/ref_rejected": -90.50865173339844,
"logps/rejected": -357.23822021484375,
"loss": 1.1551,
"margin_dpo/margin_mean": 84.60891723632812,
"margin_dpo/margin_std": 161.02288818359375,
"step": 494
},
{
"KL/chosen_KL_mean": -177.19039916992188,
"KL/mean": -209.7973175048828,
"KL/rejected_KL_mean": -242.4042205810547,
"KL/std": 126.00395965576172,
"epoch": 0.7482993197278912,
"fcm_dpo/beta": 0.004787042737007141,
"fcm_dpo/delta": -0.0018955229315906763,
"fcm_dpo/margin": 65.21380615234375,
"fcm_dpo/q_t": 0.43578070402145386,
"grad_norm": 14.910195350646973,
"learning_rate": 9.133780704940594e-08,
"logits/chosen": 0.8767759799957275,
"logits/rejected": 0.8284322023391724,
"logps/chosen": -229.8158721923828,
"logps/ref_chosen": -52.62546157836914,
"logps/ref_rejected": -72.06781005859375,
"logps/rejected": -314.4720458984375,
"loss": 1.2215,
"margin_dpo/margin_mean": 65.21380615234375,
"margin_dpo/margin_std": 157.19482421875,
"step": 495
},
{
"KL/chosen_KL_mean": -191.9990234375,
"KL/mean": -239.45957946777344,
"KL/rejected_KL_mean": -286.9201354980469,
"KL/std": 139.683349609375,
"epoch": 0.7498110355253212,
"fcm_dpo/beta": 0.004459347575902939,
"fcm_dpo/delta": -0.03923889249563217,
"fcm_dpo/margin": 94.92111206054688,
"fcm_dpo/q_t": 0.41244807839393616,
"grad_norm": 11.721460342407227,
"learning_rate": 9.031821899254797e-08,
"logits/chosen": 0.8311753869056702,
"logits/rejected": 0.7325785160064697,
"logps/chosen": -249.59634399414062,
"logps/ref_chosen": -57.597320556640625,
"logps/ref_rejected": -94.36127471923828,
"logps/rejected": -381.28143310546875,
"loss": 1.1432,
"margin_dpo/margin_mean": 94.92111206054688,
"margin_dpo/margin_std": 180.92669677734375,
"step": 496
},
{
"KL/chosen_KL_mean": -197.59085083007812,
"KL/mean": -241.4178009033203,
"KL/rejected_KL_mean": -285.2447509765625,
"KL/std": 125.32099914550781,
"epoch": 0.7513227513227513,
"fcm_dpo/beta": 0.004555117338895798,
"fcm_dpo/delta": 0.0004740636795759201,
"fcm_dpo/margin": 87.65388488769531,
"fcm_dpo/q_t": 0.4084148406982422,
"grad_norm": 13.068496704101562,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 0.8443832993507385,
"logits/rejected": 0.8150862455368042,
"logps/chosen": -270.38079833984375,
"logps/ref_chosen": -72.78994750976562,
"logps/ref_rejected": -89.48483276367188,
"logps/rejected": -374.7295837402344,
"loss": 1.1185,
"margin_dpo/margin_mean": 87.65388488769531,
"margin_dpo/margin_std": 137.76541137695312,
"step": 497
},
{
"KL/chosen_KL_mean": -175.70782470703125,
"KL/mean": -222.53150939941406,
"KL/rejected_KL_mean": -269.3551940917969,
"KL/std": 117.85083770751953,
"epoch": 0.7528344671201814,
"fcm_dpo/beta": 0.00448596803471446,
"fcm_dpo/delta": -0.025814387947320938,
"fcm_dpo/margin": 93.6473388671875,
"fcm_dpo/q_t": 0.401056706905365,
"grad_norm": 13.499424934387207,
"learning_rate": 8.829247120198563e-08,
"logits/chosen": 0.8166778087615967,
"logits/rejected": 0.8045326471328735,
"logps/chosen": -244.07354736328125,
"logps/ref_chosen": -68.36572265625,
"logps/ref_rejected": -71.28846740722656,
"logps/rejected": -340.6436767578125,
"loss": 1.0898,
"margin_dpo/margin_mean": 93.6473388671875,
"margin_dpo/margin_std": 130.13490295410156,
"step": 498
},
{
"KL/chosen_KL_mean": -179.5928497314453,
"KL/mean": -229.11141967773438,
"KL/rejected_KL_mean": -278.6299743652344,
"KL/std": 132.99771118164062,
"epoch": 0.7543461829176115,
"fcm_dpo/beta": 0.004335935227572918,
"fcm_dpo/delta": -0.03787044808268547,
"fcm_dpo/margin": 99.03713989257812,
"fcm_dpo/q_t": 0.40562453866004944,
"grad_norm": 19.231489181518555,
"learning_rate": 8.728636813280163e-08,
"logits/chosen": 0.8330011367797852,
"logits/rejected": 0.7787662148475647,
"logps/chosen": -241.50167846679688,
"logps/ref_chosen": -61.90882873535156,
"logps/ref_rejected": -91.9411392211914,
"logps/rejected": -370.57110595703125,
"loss": 1.1363,
"margin_dpo/margin_mean": 99.0371322631836,
"margin_dpo/margin_std": 176.20849609375,
"step": 499
},
{
"KL/chosen_KL_mean": -188.66636657714844,
"KL/mean": -225.65838623046875,
"KL/rejected_KL_mean": -262.6504211425781,
"KL/std": 115.54472351074219,
"epoch": 0.7558578987150416,
"fcm_dpo/beta": 0.004538041539490223,
"fcm_dpo/delta": 0.07754088193178177,
"fcm_dpo/margin": 73.98403930664062,
"fcm_dpo/q_t": 0.42312851548194885,
"grad_norm": 14.178508758544922,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.8060486316680908,
"logits/rejected": 0.8183088302612305,
"logps/chosen": -258.8922119140625,
"logps/ref_chosen": -70.225830078125,
"logps/ref_rejected": -71.72203063964844,
"logps/rejected": -334.3724365234375,
"loss": 1.1855,
"margin_dpo/margin_mean": 73.98403930664062,
"margin_dpo/margin_std": 147.24203491210938,
"step": 500
},
{
"KL/chosen_KL_mean": -187.6682891845703,
"KL/mean": -222.70504760742188,
"KL/rejected_KL_mean": -257.7417907714844,
"KL/std": 122.72930908203125,
"epoch": 0.7573696145124716,
"fcm_dpo/beta": 0.004941802471876144,
"fcm_dpo/delta": 0.06365126371383667,
"fcm_dpo/margin": 70.07351684570312,
"fcm_dpo/q_t": 0.42135995626449585,
"grad_norm": 12.289847373962402,
"learning_rate": 8.528784436016878e-08,
"logits/chosen": 0.831879198551178,
"logits/rejected": 0.8381158113479614,
"logps/chosen": -252.26708984375,
"logps/ref_chosen": -64.59880828857422,
"logps/ref_rejected": -70.59329223632812,
"logps/rejected": -328.3350830078125,
"loss": 1.1276,
"margin_dpo/margin_mean": 70.07351684570312,
"margin_dpo/margin_std": 99.20466613769531,
"step": 501
},
{
"KL/chosen_KL_mean": -187.70001220703125,
"KL/mean": -225.89833068847656,
"KL/rejected_KL_mean": -264.09661865234375,
"KL/std": 134.3505096435547,
"epoch": 0.7588813303099018,
"fcm_dpo/beta": 0.005120974965393543,
"fcm_dpo/delta": 0.008247282356023788,
"fcm_dpo/margin": 76.39662170410156,
"fcm_dpo/q_t": 0.412162721157074,
"grad_norm": 16.75588607788086,
"learning_rate": 8.4295479559726e-08,
"logits/chosen": 0.8271726369857788,
"logits/rejected": 0.7866770029067993,
"logps/chosen": -253.16665649414062,
"logps/ref_chosen": -65.46662902832031,
"logps/ref_rejected": -90.22233581542969,
"logps/rejected": -354.3189697265625,
"loss": 1.129,
"margin_dpo/margin_mean": 76.39662170410156,
"margin_dpo/margin_std": 124.2121353149414,
"step": 502
},
{
"KL/chosen_KL_mean": -165.67684936523438,
"KL/mean": -204.37872314453125,
"KL/rejected_KL_mean": -243.0806121826172,
"KL/std": 110.60697174072266,
"epoch": 0.7603930461073318,
"fcm_dpo/beta": 0.00502370111644268,
"fcm_dpo/delta": 0.013673887588083744,
"fcm_dpo/margin": 77.40376281738281,
"fcm_dpo/q_t": 0.4104214310646057,
"grad_norm": 12.808483123779297,
"learning_rate": 8.330774987092712e-08,
"logits/chosen": 0.7944152355194092,
"logits/rejected": 0.8020203709602356,
"logps/chosen": -217.51161193847656,
"logps/ref_chosen": -51.83476257324219,
"logps/ref_rejected": -57.62522506713867,
"logps/rejected": -300.7058410644531,
"loss": 1.1297,
"margin_dpo/margin_mean": 77.40376281738281,
"margin_dpo/margin_std": 125.51486206054688,
"step": 503
},
{
"KL/chosen_KL_mean": -174.6210174560547,
"KL/mean": -224.70675659179688,
"KL/rejected_KL_mean": -274.79248046875,
"KL/std": 113.20222473144531,
"epoch": 0.7619047619047619,
"fcm_dpo/beta": 0.004858715459704399,
"fcm_dpo/delta": -0.11634810268878937,
"fcm_dpo/margin": 100.17145538330078,
"fcm_dpo/q_t": 0.38356009125709534,
"grad_norm": 11.490617752075195,
"learning_rate": 8.232468292269479e-08,
"logits/chosen": 0.7988426089286804,
"logits/rejected": 0.7896198034286499,
"logps/chosen": -243.272216796875,
"logps/ref_chosen": -68.65119934082031,
"logps/ref_rejected": -77.91394805908203,
"logps/rejected": -352.7064208984375,
"loss": 1.0441,
"margin_dpo/margin_mean": 100.17145538330078,
"margin_dpo/margin_std": 116.76637268066406,
"step": 504
},
{
"KL/chosen_KL_mean": -184.5635986328125,
"KL/mean": -219.55801391601562,
"KL/rejected_KL_mean": -254.55239868164062,
"KL/std": 134.74453735351562,
"epoch": 0.763416477702192,
"fcm_dpo/beta": 0.004393025301396847,
"fcm_dpo/delta": -0.02926192618906498,
"fcm_dpo/margin": 69.98880004882812,
"fcm_dpo/q_t": 0.4304890036582947,
"grad_norm": 11.797785758972168,
"learning_rate": 8.134630621352483e-08,
"logits/chosen": 0.8228546380996704,
"logits/rejected": 0.7897151708602905,
"logps/chosen": -244.5624542236328,
"logps/ref_chosen": -59.99884796142578,
"logps/ref_rejected": -76.88048553466797,
"logps/rejected": -331.4328918457031,
"loss": 1.2023,
"margin_dpo/margin_mean": 69.98880004882812,
"margin_dpo/margin_std": 146.3544921875,
"step": 505
},
{
"KL/chosen_KL_mean": -185.9619140625,
"KL/mean": -222.78070068359375,
"KL/rejected_KL_mean": -259.5994873046875,
"KL/std": 118.3785171508789,
"epoch": 0.764928193499622,
"fcm_dpo/beta": 0.004599967040121555,
"fcm_dpo/delta": 0.07407009601593018,
"fcm_dpo/margin": 73.63758850097656,
"fcm_dpo/q_t": 0.42431166768074036,
"grad_norm": 12.498722076416016,
"learning_rate": 8.037264711071698e-08,
"logits/chosen": 0.7942636609077454,
"logits/rejected": 0.785301685333252,
"logps/chosen": -256.033203125,
"logps/ref_chosen": -70.07130432128906,
"logps/ref_rejected": -82.03775024414062,
"logps/rejected": -341.63726806640625,
"loss": 1.1882,
"margin_dpo/margin_mean": 73.63758850097656,
"margin_dpo/margin_std": 149.9478302001953,
"step": 506
},
{
"KL/chosen_KL_mean": -195.21092224121094,
"KL/mean": -239.38064575195312,
"KL/rejected_KL_mean": -283.55035400390625,
"KL/std": 135.94154357910156,
"epoch": 0.7664399092970522,
"fcm_dpo/beta": 0.004573674872517586,
"fcm_dpo/delta": -0.008770979940891266,
"fcm_dpo/margin": 88.33943176269531,
"fcm_dpo/q_t": 0.4142111539840698,
"grad_norm": 13.64824390411377,
"learning_rate": 7.940373284960933e-08,
"logits/chosen": 0.7974521517753601,
"logits/rejected": 0.7599077224731445,
"logps/chosen": -267.21795654296875,
"logps/ref_chosen": -72.00703430175781,
"logps/ref_rejected": -93.94987487792969,
"logps/rejected": -377.500244140625,
"loss": 1.1482,
"margin_dpo/margin_mean": 88.33943176269531,
"margin_dpo/margin_std": 161.13748168945312,
"step": 507
},
{
"KL/chosen_KL_mean": -179.69149780273438,
"KL/mean": -231.722900390625,
"KL/rejected_KL_mean": -283.7542724609375,
"KL/std": 142.07757568359375,
"epoch": 0.7679516250944822,
"fcm_dpo/beta": 0.004537220112979412,
"fcm_dpo/delta": -0.09652116149663925,
"fcm_dpo/margin": 104.06279754638672,
"fcm_dpo/q_t": 0.3932833671569824,
"grad_norm": 14.554759979248047,
"learning_rate": 7.843959053281663e-08,
"logits/chosen": 0.8035924434661865,
"logits/rejected": 0.7036359310150146,
"logps/chosen": -239.9114227294922,
"logps/ref_chosen": -60.21992492675781,
"logps/ref_rejected": -95.9200668334961,
"logps/rejected": -379.67437744140625,
"loss": 1.09,
"margin_dpo/margin_mean": 104.06279754638672,
"margin_dpo/margin_std": 155.7027130126953,
"step": 508
},
{
"KL/chosen_KL_mean": -190.60467529296875,
"KL/mean": -227.2447967529297,
"KL/rejected_KL_mean": -263.8849182128906,
"KL/std": 124.68811798095703,
"epoch": 0.7694633408919124,
"fcm_dpo/beta": 0.004491401370614767,
"fcm_dpo/delta": 0.08523663878440857,
"fcm_dpo/margin": 73.28022003173828,
"fcm_dpo/q_t": 0.4270988702774048,
"grad_norm": 15.783040046691895,
"learning_rate": 7.748024712947204e-08,
"logits/chosen": 0.7905877232551575,
"logits/rejected": 0.7737974524497986,
"logps/chosen": -256.8748474121094,
"logps/ref_chosen": -66.27017211914062,
"logps/ref_rejected": -71.73065185546875,
"logps/rejected": -335.61553955078125,
"loss": 1.1929,
"margin_dpo/margin_mean": 73.28022766113281,
"margin_dpo/margin_std": 149.970458984375,
"step": 509
},
{
"KL/chosen_KL_mean": -189.751708984375,
"KL/mean": -238.32249450683594,
"KL/rejected_KL_mean": -286.8932800292969,
"KL/std": 140.2471466064453,
"epoch": 0.7709750566893424,
"fcm_dpo/beta": 0.004407564178109169,
"fcm_dpo/delta": -0.03904171660542488,
"fcm_dpo/margin": 97.1415786743164,
"fcm_dpo/q_t": 0.40892326831817627,
"grad_norm": 13.543128967285156,
"learning_rate": 7.652572947447272e-08,
"logits/chosen": 0.8949325084686279,
"logits/rejected": 0.8182798624038696,
"logps/chosen": -243.2965850830078,
"logps/ref_chosen": -53.54487609863281,
"logps/ref_rejected": -91.36648559570312,
"logps/rejected": -378.259765625,
"loss": 1.1437,
"margin_dpo/margin_mean": 97.14158630371094,
"margin_dpo/margin_std": 181.95016479492188,
"step": 510
},
{
"KL/chosen_KL_mean": -178.1725616455078,
"KL/mean": -232.1974334716797,
"KL/rejected_KL_mean": -286.2223205566406,
"KL/std": 122.31033325195312,
"epoch": 0.7724867724867724,
"fcm_dpo/beta": 0.0042327032424509525,
"fcm_dpo/delta": -0.07427525520324707,
"fcm_dpo/margin": 108.04974365234375,
"fcm_dpo/q_t": 0.3912004828453064,
"grad_norm": 15.069428443908691,
"learning_rate": 7.557606426772961e-08,
"logits/chosen": 0.8457983136177063,
"logits/rejected": 0.7985345125198364,
"logps/chosen": -234.01693725585938,
"logps/ref_chosen": -55.844383239746094,
"logps/ref_rejected": -86.49819946289062,
"logps/rejected": -372.72052001953125,
"loss": 1.0521,
"margin_dpo/margin_mean": 108.04974365234375,
"margin_dpo/margin_std": 128.53379821777344,
"step": 511
},
{
"KL/chosen_KL_mean": -185.82992553710938,
"KL/mean": -217.70220947265625,
"KL/rejected_KL_mean": -249.574462890625,
"KL/std": 114.0875015258789,
"epoch": 0.7739984882842026,
"fcm_dpo/beta": 0.004275324288755655,
"fcm_dpo/delta": 0.03214414417743683,
"fcm_dpo/margin": 63.744571685791016,
"fcm_dpo/q_t": 0.43680599331855774,
"grad_norm": 19.423717498779297,
"learning_rate": 7.463127807341966e-08,
"logits/chosen": 0.7606216669082642,
"logits/rejected": 0.7632473111152649,
"logps/chosen": -247.4829559326172,
"logps/ref_chosen": -61.653038024902344,
"logps/ref_rejected": -72.83148193359375,
"logps/rejected": -322.40594482421875,
"loss": 1.2179,
"margin_dpo/margin_mean": 63.74457550048828,
"margin_dpo/margin_std": 138.28744506835938,
"step": 512
},
{
"KL/chosen_KL_mean": -166.70736694335938,
"KL/mean": -207.901123046875,
"KL/rejected_KL_mean": -249.09486389160156,
"KL/std": 113.70684814453125,
"epoch": 0.7755102040816326,
"fcm_dpo/beta": 0.004403136670589447,
"fcm_dpo/delta": 0.04559480771422386,
"fcm_dpo/margin": 82.38749694824219,
"fcm_dpo/q_t": 0.4186474084854126,
"grad_norm": 10.126408576965332,
"learning_rate": 7.369139731924401e-08,
"logits/chosen": 0.9330278635025024,
"logits/rejected": 0.8865524530410767,
"logps/chosen": -217.5599365234375,
"logps/ref_chosen": -50.85256576538086,
"logps/ref_rejected": -69.21754455566406,
"logps/rejected": -318.3124084472656,
"loss": 1.1249,
"margin_dpo/margin_mean": 82.38749694824219,
"margin_dpo/margin_std": 122.80973052978516,
"step": 513
},
{
"KL/chosen_KL_mean": -187.76589965820312,
"KL/mean": -235.5794677734375,
"KL/rejected_KL_mean": -283.39300537109375,
"KL/std": 126.61970520019531,
"epoch": 0.7770219198790628,
"fcm_dpo/beta": 0.004458842799067497,
"fcm_dpo/delta": -0.034807316958904266,
"fcm_dpo/margin": 95.62709045410156,
"fcm_dpo/q_t": 0.4023844003677368,
"grad_norm": 15.569659233093262,
"learning_rate": 7.275644829568747e-08,
"logits/chosen": 0.8444833755493164,
"logits/rejected": 0.8213080763816833,
"logps/chosen": -257.1508483886719,
"logps/ref_chosen": -69.38493347167969,
"logps/ref_rejected": -83.32447814941406,
"logps/rejected": -366.71746826171875,
"loss": 1.1018,
"margin_dpo/margin_mean": 95.62709045410156,
"margin_dpo/margin_std": 144.15618896484375,
"step": 514
},
{
"KL/chosen_KL_mean": -194.33404541015625,
"KL/mean": -232.53646850585938,
"KL/rejected_KL_mean": -270.7389221191406,
"KL/std": 117.86701965332031,
"epoch": 0.7785336356764928,
"fcm_dpo/beta": 0.004516595974564552,
"fcm_dpo/delta": 0.06656854599714279,
"fcm_dpo/margin": 76.40487670898438,
"fcm_dpo/q_t": 0.4221508502960205,
"grad_norm": 12.275152206420898,
"learning_rate": 7.182645715528435e-08,
"logits/chosen": 0.8466659784317017,
"logits/rejected": 0.7759132981300354,
"logps/chosen": -248.02108764648438,
"logps/ref_chosen": -53.687034606933594,
"logps/ref_rejected": -83.59614562988281,
"logps/rejected": -354.3350524902344,
"loss": 1.1581,
"margin_dpo/margin_mean": 76.40487670898438,
"margin_dpo/margin_std": 133.73483276367188,
"step": 515
},
{
"KL/chosen_KL_mean": -168.5338897705078,
"KL/mean": -205.44277954101562,
"KL/rejected_KL_mean": -242.3516845703125,
"KL/std": 116.71525573730469,
"epoch": 0.780045351473923,
"fcm_dpo/beta": 0.0048707169480621815,
"fcm_dpo/delta": 0.04681536927819252,
"fcm_dpo/margin": 73.81780242919922,
"fcm_dpo/q_t": 0.41837072372436523,
"grad_norm": 15.747931480407715,
"learning_rate": 7.090144991188568e-08,
"logits/chosen": 0.8174859285354614,
"logits/rejected": 0.7878178358078003,
"logps/chosen": -225.43560791015625,
"logps/ref_chosen": -56.9017219543457,
"logps/ref_rejected": -67.83477783203125,
"logps/rejected": -310.18646240234375,
"loss": 1.1655,
"margin_dpo/margin_mean": 73.81779479980469,
"margin_dpo/margin_std": 137.54354858398438,
"step": 516
},
{
"KL/chosen_KL_mean": -201.6470489501953,
"KL/mean": -229.30369567871094,
"KL/rejected_KL_mean": -256.9603271484375,
"KL/std": 125.04222106933594,
"epoch": 0.781557067271353,
"fcm_dpo/beta": 0.004945510067045689,
"fcm_dpo/delta": 0.042446672916412354,
"fcm_dpo/margin": 55.31329345703125,
"fcm_dpo/q_t": 0.44020599126815796,
"grad_norm": 14.6834077835083,
"learning_rate": 6.998145243993284e-08,
"logits/chosen": 0.909896969795227,
"logits/rejected": 0.9149903059005737,
"logps/chosen": -263.42218017578125,
"logps/ref_chosen": -61.775142669677734,
"logps/ref_rejected": -62.88270950317383,
"logps/rejected": -319.8430480957031,
"loss": 1.225,
"margin_dpo/margin_mean": 55.31329345703125,
"margin_dpo/margin_std": 128.85104370117188,
"step": 517
},
{
"KL/chosen_KL_mean": -173.17193603515625,
"KL/mean": -214.0082550048828,
"KL/rejected_KL_mean": -254.8445587158203,
"KL/std": 122.31529235839844,
"epoch": 0.783068783068783,
"fcm_dpo/beta": 0.005010384600609541,
"fcm_dpo/delta": -0.011576948687434196,
"fcm_dpo/margin": 81.67262268066406,
"fcm_dpo/q_t": 0.4104600250720978,
"grad_norm": 13.930161476135254,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 0.8191814422607422,
"logits/rejected": 0.7862158417701721,
"logps/chosen": -235.19717407226562,
"logps/ref_chosen": -62.02523422241211,
"logps/ref_rejected": -79.06085205078125,
"logps/rejected": -333.9053955078125,
"loss": 1.1273,
"margin_dpo/margin_mean": 81.67261505126953,
"margin_dpo/margin_std": 138.4313201904297,
"step": 518
},
{
"KL/chosen_KL_mean": -204.07974243164062,
"KL/mean": -227.8219451904297,
"KL/rejected_KL_mean": -251.5641326904297,
"KL/std": 122.6117172241211,
"epoch": 0.7845804988662132,
"fcm_dpo/beta": 0.005166331306099892,
"fcm_dpo/delta": 0.067388154566288,
"fcm_dpo/margin": 47.48439025878906,
"fcm_dpo/q_t": 0.4460979700088501,
"grad_norm": 20.920434951782227,
"learning_rate": 6.815658960673781e-08,
"logits/chosen": 0.8258580565452576,
"logits/rejected": 0.7891712188720703,
"logps/chosen": -265.68609619140625,
"logps/ref_chosen": -61.60636901855469,
"logps/ref_rejected": -74.50727844238281,
"logps/rejected": -326.0714111328125,
"loss": 1.314,
"margin_dpo/margin_mean": 47.48439025878906,
"margin_dpo/margin_std": 160.26510620117188,
"step": 519
},
{
"KL/chosen_KL_mean": -189.1623992919922,
"KL/mean": -226.52426147460938,
"KL/rejected_KL_mean": -263.8861083984375,
"KL/std": 124.83399963378906,
"epoch": 0.7860922146636432,
"fcm_dpo/beta": 0.005199735518544912,
"fcm_dpo/delta": 0.009324587881565094,
"fcm_dpo/margin": 74.72369384765625,
"fcm_dpo/q_t": 0.4140581488609314,
"grad_norm": 17.282926559448242,
"learning_rate": 6.725177529083209e-08,
"logits/chosen": 0.8755450248718262,
"logits/rejected": 0.8296347856521606,
"logps/chosen": -252.03582763671875,
"logps/ref_chosen": -62.87343215942383,
"logps/ref_rejected": -76.505615234375,
"logps/rejected": -340.3917236328125,
"loss": 1.141,
"margin_dpo/margin_mean": 74.72369384765625,
"margin_dpo/margin_std": 127.34550476074219,
"step": 520
},
{
"KL/chosen_KL_mean": -186.72055053710938,
"KL/mean": -238.96612548828125,
"KL/rejected_KL_mean": -291.2117004394531,
"KL/std": 125.33415222167969,
"epoch": 0.7876039304610734,
"fcm_dpo/beta": 0.0048751672729849815,
"fcm_dpo/delta": -0.14638295769691467,
"fcm_dpo/margin": 104.49115753173828,
"fcm_dpo/q_t": 0.3831254839897156,
"grad_norm": 12.223865509033203,
"learning_rate": 6.63520728356167e-08,
"logits/chosen": 0.748036801815033,
"logits/rejected": 0.6873229742050171,
"logps/chosen": -250.92721557617188,
"logps/ref_chosen": -64.20668029785156,
"logps/ref_rejected": -92.28083038330078,
"logps/rejected": -383.4925231933594,
"loss": 1.0487,
"margin_dpo/margin_mean": 104.49115753173828,
"margin_dpo/margin_std": 144.06787109375,
"step": 521
},
{
"KL/chosen_KL_mean": -190.2344970703125,
"KL/mean": -220.25045776367188,
"KL/rejected_KL_mean": -250.2664337158203,
"KL/std": 122.27368927001953,
"epoch": 0.7891156462585034,
"fcm_dpo/beta": 0.005026691593229771,
"fcm_dpo/delta": 0.11638805270195007,
"fcm_dpo/margin": 60.03194046020508,
"fcm_dpo/q_t": 0.43588894605636597,
"grad_norm": 15.411672592163086,
"learning_rate": 6.545750740770336e-08,
"logits/chosen": 0.8304263353347778,
"logits/rejected": 0.823676586151123,
"logps/chosen": -248.60421752929688,
"logps/ref_chosen": -58.369720458984375,
"logps/ref_rejected": -68.79248046875,
"logps/rejected": -319.05889892578125,
"loss": 1.2574,
"margin_dpo/margin_mean": 60.03194046020508,
"margin_dpo/margin_std": 158.7082061767578,
"step": 522
},
{
"KL/chosen_KL_mean": -195.03762817382812,
"KL/mean": -234.07977294921875,
"KL/rejected_KL_mean": -273.12188720703125,
"KL/std": 126.46722412109375,
"epoch": 0.7906273620559335,
"fcm_dpo/beta": 0.005208607763051987,
"fcm_dpo/delta": -0.0084223672747612,
"fcm_dpo/margin": 78.08426666259766,
"fcm_dpo/q_t": 0.40672677755355835,
"grad_norm": 18.978595733642578,
"learning_rate": 6.456810403001012e-08,
"logits/chosen": 0.8376122713088989,
"logits/rejected": 0.7323886156082153,
"logps/chosen": -260.7508544921875,
"logps/ref_chosen": -65.71324157714844,
"logps/ref_rejected": -91.98896789550781,
"logps/rejected": -365.11083984375,
"loss": 1.1444,
"margin_dpo/margin_mean": 78.08426666259766,
"margin_dpo/margin_std": 140.59805297851562,
"step": 523
},
{
"KL/chosen_KL_mean": -172.65541076660156,
"KL/mean": -211.43576049804688,
"KL/rejected_KL_mean": -250.216064453125,
"KL/std": 122.14218139648438,
"epoch": 0.7921390778533636,
"fcm_dpo/beta": 0.00523480586707592,
"fcm_dpo/delta": -0.008444221690297127,
"fcm_dpo/margin": 77.56066131591797,
"fcm_dpo/q_t": 0.407756507396698,
"grad_norm": 15.92771053314209,
"learning_rate": 6.368388758106134e-08,
"logits/chosen": 0.751681387424469,
"logits/rejected": 0.7346346974372864,
"logps/chosen": -249.00665283203125,
"logps/ref_chosen": -76.35124969482422,
"logps/ref_rejected": -89.96072387695312,
"logps/rejected": -340.17681884765625,
"loss": 1.1162,
"margin_dpo/margin_mean": 77.56065368652344,
"margin_dpo/margin_std": 122.21437072753906,
"step": 524
},
{
"KL/chosen_KL_mean": -193.3937225341797,
"KL/mean": -225.09326171875,
"KL/rejected_KL_mean": -256.79278564453125,
"KL/std": 118.25725555419922,
"epoch": 0.7936507936507936,
"fcm_dpo/beta": 0.005362164229154587,
"fcm_dpo/delta": 0.07258353382349014,
"fcm_dpo/margin": 63.3990478515625,
"fcm_dpo/q_t": 0.42902350425720215,
"grad_norm": 20.461183547973633,
"learning_rate": 6.280488279429185e-08,
"logits/chosen": 0.6896591186523438,
"logits/rejected": 0.6898474097251892,
"logps/chosen": -268.8895263671875,
"logps/ref_chosen": -75.49578857421875,
"logps/ref_rejected": -84.04852294921875,
"logps/rejected": -340.84130859375,
"loss": 1.2048,
"margin_dpo/margin_mean": 63.3990478515625,
"margin_dpo/margin_std": 140.93850708007812,
"step": 525
},
{
"KL/chosen_KL_mean": -198.66851806640625,
"KL/mean": -229.0810546875,
"KL/rejected_KL_mean": -259.49359130859375,
"KL/std": 123.21441650390625,
"epoch": 0.7951625094482238,
"fcm_dpo/beta": 0.005208217538893223,
"fcm_dpo/delta": -0.06009836867451668,
"fcm_dpo/margin": 60.825103759765625,
"fcm_dpo/q_t": 0.4270055294036865,
"grad_norm": 14.760220527648926,
"learning_rate": 6.193111425735515e-08,
"logits/chosen": 0.8644962310791016,
"logits/rejected": 0.8045707941055298,
"logps/chosen": -259.9609375,
"logps/ref_chosen": -61.29241943359375,
"logps/ref_rejected": -82.47763061523438,
"logps/rejected": -341.9712219238281,
"loss": 1.2005,
"margin_dpo/margin_mean": 60.825103759765625,
"margin_dpo/margin_std": 126.21774291992188,
"step": 526
},
{
"KL/chosen_KL_mean": -207.2027587890625,
"KL/mean": -233.5564727783203,
"KL/rejected_KL_mean": -259.9101867675781,
"KL/std": 122.05842590332031,
"epoch": 0.7966742252456538,
"fcm_dpo/beta": 0.005240567959845066,
"fcm_dpo/delta": 0.012346339412033558,
"fcm_dpo/margin": 52.707401275634766,
"fcm_dpo/q_t": 0.44000500440597534,
"grad_norm": 15.85174560546875,
"learning_rate": 6.106260641143546e-08,
"logits/chosen": 0.8949396014213562,
"logits/rejected": 0.8214942812919617,
"logps/chosen": -268.6753845214844,
"logps/ref_chosen": -61.472625732421875,
"logps/ref_rejected": -90.52831268310547,
"logps/rejected": -350.4385070800781,
"loss": 1.2519,
"margin_dpo/margin_mean": 52.70740509033203,
"margin_dpo/margin_std": 137.97315979003906,
"step": 527
},
{
"KL/chosen_KL_mean": -183.61734008789062,
"KL/mean": -214.14938354492188,
"KL/rejected_KL_mean": -244.68141174316406,
"KL/std": 114.77609252929688,
"epoch": 0.7981859410430839,
"fcm_dpo/beta": 0.005515486933290958,
"fcm_dpo/delta": 0.07630209624767303,
"fcm_dpo/margin": 61.0640869140625,
"fcm_dpo/q_t": 0.4299367070198059,
"grad_norm": 17.928499221801758,
"learning_rate": 6.019938355056422e-08,
"logits/chosen": 0.767498254776001,
"logits/rejected": 0.709592878818512,
"logps/chosen": -242.4093475341797,
"logps/ref_chosen": -58.792015075683594,
"logps/ref_rejected": -71.82516479492188,
"logps/rejected": -316.506591796875,
"loss": 1.237,
"margin_dpo/margin_mean": 61.0640869140625,
"margin_dpo/margin_std": 150.18453979492188,
"step": 528
},
{
"KL/chosen_KL_mean": -178.7403564453125,
"KL/mean": -234.70297241210938,
"KL/rejected_KL_mean": -290.6656494140625,
"KL/std": 112.90904235839844,
"epoch": 0.799697656840514,
"fcm_dpo/beta": 0.004876470193266869,
"fcm_dpo/delta": -0.2033100128173828,
"fcm_dpo/margin": 111.92525482177734,
"fcm_dpo/q_t": 0.366091251373291,
"grad_norm": 15.41568660736084,
"learning_rate": 5.934146982094049e-08,
"logits/chosen": 0.7990100383758545,
"logits/rejected": 0.7555786371231079,
"logps/chosen": -233.81130981445312,
"logps/ref_chosen": -55.070960998535156,
"logps/ref_rejected": -75.44007873535156,
"logps/rejected": -366.105712890625,
"loss": 0.9968,
"margin_dpo/margin_mean": 111.92524719238281,
"margin_dpo/margin_std": 122.16747283935547,
"step": 529
},
{
"KL/chosen_KL_mean": -176.22964477539062,
"KL/mean": -214.50759887695312,
"KL/rejected_KL_mean": -252.78553771972656,
"KL/std": 123.36885070800781,
"epoch": 0.8012093726379441,
"fcm_dpo/beta": 0.004759990144520998,
"fcm_dpo/delta": 0.04360903054475784,
"fcm_dpo/margin": 76.55589294433594,
"fcm_dpo/q_t": 0.4207877516746521,
"grad_norm": 16.144392013549805,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 0.8252302408218384,
"logits/rejected": 0.78911954164505,
"logps/chosen": -232.97344970703125,
"logps/ref_chosen": -56.743812561035156,
"logps/ref_rejected": -76.6692123413086,
"logps/rejected": -329.4547424316406,
"loss": 1.1438,
"margin_dpo/margin_mean": 76.55589294433594,
"margin_dpo/margin_std": 129.4632568359375,
"step": 530
},
{
"KL/chosen_KL_mean": -176.19247436523438,
"KL/mean": -215.78732299804688,
"KL/rejected_KL_mean": -255.38214111328125,
"KL/std": 121.45870971679688,
"epoch": 0.8027210884353742,
"fcm_dpo/beta": 0.004897120874375105,
"fcm_dpo/delta": 0.015145424753427505,
"fcm_dpo/margin": 79.18968200683594,
"fcm_dpo/q_t": 0.41359657049179077,
"grad_norm": 13.057550430297852,
"learning_rate": 5.7641665597021435e-08,
"logits/chosen": 0.8119915723800659,
"logits/rejected": 0.7415198087692261,
"logps/chosen": -227.30892944335938,
"logps/ref_chosen": -51.116455078125,
"logps/ref_rejected": -79.52884674072266,
"logps/rejected": -334.9110107421875,
"loss": 1.1318,
"margin_dpo/margin_mean": 79.18968200683594,
"margin_dpo/margin_std": 131.63864135742188,
"step": 531
},
{
"KL/chosen_KL_mean": -206.1256103515625,
"KL/mean": -245.78433227539062,
"KL/rejected_KL_mean": -285.44305419921875,
"KL/std": 119.93473815917969,
"epoch": 0.8042328042328042,
"fcm_dpo/beta": 0.004965721629559994,
"fcm_dpo/delta": 0.007580554112792015,
"fcm_dpo/margin": 79.31744384765625,
"fcm_dpo/q_t": 0.4115769863128662,
"grad_norm": 14.420048713684082,
"learning_rate": 5.679982264990424e-08,
"logits/chosen": 0.7928520441055298,
"logits/rejected": 0.7500756978988647,
"logps/chosen": -264.4055480957031,
"logps/ref_chosen": -58.279945373535156,
"logps/ref_rejected": -78.05426788330078,
"logps/rejected": -363.497314453125,
"loss": 1.1333,
"margin_dpo/margin_mean": 79.31744384765625,
"margin_dpo/margin_std": 133.19078063964844,
"step": 532
},
{
"KL/chosen_KL_mean": -148.75473022460938,
"KL/mean": -195.34390258789062,
"KL/rejected_KL_mean": -241.93309020996094,
"KL/std": 129.021240234375,
"epoch": 0.8057445200302343,
"fcm_dpo/beta": 0.004798324778676033,
"fcm_dpo/delta": -0.06049473583698273,
"fcm_dpo/margin": 93.17835998535156,
"fcm_dpo/q_t": 0.39857304096221924,
"grad_norm": 14.216915130615234,
"learning_rate": 5.596338392706076e-08,
"logits/chosen": 0.9161920547485352,
"logits/rejected": 0.865228533744812,
"logps/chosen": -205.17274475097656,
"logps/ref_chosen": -56.41801071166992,
"logps/ref_rejected": -73.89324951171875,
"logps/rejected": -315.82635498046875,
"loss": 1.0874,
"margin_dpo/margin_mean": 93.17835998535156,
"margin_dpo/margin_std": 137.08950805664062,
"step": 533
},
{
"KL/chosen_KL_mean": -185.18804931640625,
"KL/mean": -221.02896118164062,
"KL/rejected_KL_mean": -256.86981201171875,
"KL/std": 115.55854034423828,
"epoch": 0.8072562358276644,
"fcm_dpo/beta": 0.004827641882002354,
"fcm_dpo/delta": 0.06521468609571457,
"fcm_dpo/margin": 71.6817855834961,
"fcm_dpo/q_t": 0.42473551630973816,
"grad_norm": 13.707343101501465,
"learning_rate": 5.513237282548033e-08,
"logits/chosen": 0.7912318706512451,
"logits/rejected": 0.7636306285858154,
"logps/chosen": -245.93673706054688,
"logps/ref_chosen": -60.748687744140625,
"logps/ref_rejected": -73.8623046875,
"logps/rejected": -330.73211669921875,
"loss": 1.1807,
"margin_dpo/margin_mean": 71.68177795410156,
"margin_dpo/margin_std": 141.45501708984375,
"step": 534
},
{
"KL/chosen_KL_mean": -195.66860961914062,
"KL/mean": -230.18612670898438,
"KL/rejected_KL_mean": -264.70361328125,
"KL/std": 125.1104507446289,
"epoch": 0.8087679516250945,
"fcm_dpo/beta": 0.005231472663581371,
"fcm_dpo/delta": 0.045737870037555695,
"fcm_dpo/margin": 69.0350112915039,
"fcm_dpo/q_t": 0.4225993752479553,
"grad_norm": 16.398719787597656,
"learning_rate": 5.430681259032957e-08,
"logits/chosen": 0.7519247531890869,
"logits/rejected": 0.7029865980148315,
"logps/chosen": -257.3060302734375,
"logps/ref_chosen": -61.637413024902344,
"logps/ref_rejected": -80.93138885498047,
"logps/rejected": -345.635009765625,
"loss": 1.1829,
"margin_dpo/margin_mean": 69.0350112915039,
"margin_dpo/margin_std": 139.1922607421875,
"step": 535
},
{
"KL/chosen_KL_mean": -169.75575256347656,
"KL/mean": -222.75271606445312,
"KL/rejected_KL_mean": -275.74969482421875,
"KL/std": 124.12066650390625,
"epoch": 0.8102796674225246,
"fcm_dpo/beta": 0.00466238334774971,
"fcm_dpo/delta": -0.1296248584985733,
"fcm_dpo/margin": 105.99392700195312,
"fcm_dpo/q_t": 0.3823320269584656,
"grad_norm": 10.686798095703125,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 0.8615167140960693,
"logits/rejected": 0.7933706045150757,
"logps/chosen": -221.6447296142578,
"logps/ref_chosen": -51.88897705078125,
"logps/ref_rejected": -73.34864044189453,
"logps/rejected": -349.09832763671875,
"loss": 1.0272,
"margin_dpo/margin_mean": 105.99392700195312,
"margin_dpo/margin_std": 123.41902160644531,
"step": 536
},
{
"KL/chosen_KL_mean": -192.2476043701172,
"KL/mean": -238.9248046875,
"KL/rejected_KL_mean": -285.60198974609375,
"KL/std": 134.6256866455078,
"epoch": 0.8117913832199547,
"fcm_dpo/beta": 0.004458250477910042,
"fcm_dpo/delta": -0.021435163915157318,
"fcm_dpo/margin": 93.35438537597656,
"fcm_dpo/q_t": 0.4066166579723358,
"grad_norm": 13.311497688293457,
"learning_rate": 5.267213693697695e-08,
"logits/chosen": 0.889856219291687,
"logits/rejected": 0.8121524453163147,
"logps/chosen": -246.4962158203125,
"logps/ref_chosen": -54.248619079589844,
"logps/ref_rejected": -94.94343566894531,
"logps/rejected": -380.5454406738281,
"loss": 1.1145,
"margin_dpo/margin_mean": 93.35438537597656,
"margin_dpo/margin_std": 148.28921508789062,
"step": 537
},
{
"KL/chosen_KL_mean": -196.03402709960938,
"KL/mean": -239.52902221679688,
"KL/rejected_KL_mean": -283.0240173339844,
"KL/std": 132.6907958984375,
"epoch": 0.8133030990173847,
"fcm_dpo/beta": 0.004526033066213131,
"fcm_dpo/delta": 0.00775851309299469,
"fcm_dpo/margin": 86.99000549316406,
"fcm_dpo/q_t": 0.41095876693725586,
"grad_norm": 12.53797721862793,
"learning_rate": 5.1863067244167144e-08,
"logits/chosen": 0.8347610831260681,
"logits/rejected": 0.8167983293533325,
"logps/chosen": -266.1275634765625,
"logps/ref_chosen": -70.09353637695312,
"logps/ref_rejected": -79.49833679199219,
"logps/rejected": -362.5223388671875,
"loss": 1.1161,
"margin_dpo/margin_mean": 86.99000549316406,
"margin_dpo/margin_std": 134.7762451171875,
"step": 538
},
{
"KL/chosen_KL_mean": -199.68386840820312,
"KL/mean": -234.05479431152344,
"KL/rejected_KL_mean": -268.42572021484375,
"KL/std": 121.34420776367188,
"epoch": 0.8148148148148148,
"fcm_dpo/beta": 0.004799469839781523,
"fcm_dpo/delta": 0.084069162607193,
"fcm_dpo/margin": 68.74186706542969,
"fcm_dpo/q_t": 0.4309224486351013,
"grad_norm": 14.797113418579102,
"learning_rate": 5.105953986729195e-08,
"logits/chosen": 0.7780267596244812,
"logits/rejected": 0.7070616483688354,
"logps/chosen": -261.61553955078125,
"logps/ref_chosen": -61.93169403076172,
"logps/ref_rejected": -84.08946228027344,
"logps/rejected": -352.51519775390625,
"loss": 1.1768,
"margin_dpo/margin_mean": 68.74186706542969,
"margin_dpo/margin_std": 133.76632690429688,
"step": 539
},
{
"KL/chosen_KL_mean": -183.94613647460938,
"KL/mean": -238.01919555664062,
"KL/rejected_KL_mean": -292.09228515625,
"KL/std": 140.93951416015625,
"epoch": 0.8163265306122449,
"fcm_dpo/beta": 0.004561346955597401,
"fcm_dpo/delta": -0.1230163723230362,
"fcm_dpo/margin": 108.14613342285156,
"fcm_dpo/q_t": 0.3840181827545166,
"grad_norm": 12.530052185058594,
"learning_rate": 5.026157728273966e-08,
"logits/chosen": 0.8541229367256165,
"logits/rejected": 0.7800333499908447,
"logps/chosen": -246.650390625,
"logps/ref_chosen": -62.704254150390625,
"logps/ref_rejected": -95.63597106933594,
"logps/rejected": -387.7282409667969,
"loss": 1.0443,
"margin_dpo/margin_mean": 108.14613342285156,
"margin_dpo/margin_std": 136.21054077148438,
"step": 540
},
{
"KL/chosen_KL_mean": -178.83322143554688,
"KL/mean": -221.52305603027344,
"KL/rejected_KL_mean": -264.212890625,
"KL/std": 120.34114837646484,
"epoch": 0.817838246409675,
"fcm_dpo/beta": 0.004383554682135582,
"fcm_dpo/delta": 0.031111612915992737,
"fcm_dpo/margin": 85.37966918945312,
"fcm_dpo/q_t": 0.41337648034095764,
"grad_norm": 11.763753890991211,
"learning_rate": 4.9469201811239035e-08,
"logits/chosen": 0.8471169471740723,
"logits/rejected": 0.8764835596084595,
"logps/chosen": -241.31405639648438,
"logps/ref_chosen": -62.48084259033203,
"logps/ref_rejected": -57.55541229248047,
"logps/rejected": -321.768310546875,
"loss": 1.1119,
"margin_dpo/margin_mean": 85.37966918945312,
"margin_dpo/margin_std": 117.1261978149414,
"step": 541
},
{
"KL/chosen_KL_mean": -156.00730895996094,
"KL/mean": -203.59226989746094,
"KL/rejected_KL_mean": -251.17723083496094,
"KL/std": 113.75357055664062,
"epoch": 0.8193499622071051,
"fcm_dpo/beta": 0.00437610037624836,
"fcm_dpo/delta": -0.02143116295337677,
"fcm_dpo/margin": 95.16992950439453,
"fcm_dpo/q_t": 0.4032912254333496,
"grad_norm": 12.278472900390625,
"learning_rate": 4.868243561723534e-08,
"logits/chosen": 0.8880411386489868,
"logits/rejected": 0.8495569229125977,
"logps/chosen": -205.46218872070312,
"logps/ref_chosen": -49.454891204833984,
"logps/ref_rejected": -65.33275604248047,
"logps/rejected": -316.510009765625,
"loss": 1.0985,
"margin_dpo/margin_mean": 95.16992950439453,
"margin_dpo/margin_std": 138.26119995117188,
"step": 542
},
{
"KL/chosen_KL_mean": -170.74090576171875,
"KL/mean": -218.884033203125,
"KL/rejected_KL_mean": -267.02716064453125,
"KL/std": 121.83112335205078,
"epoch": 0.8208616780045351,
"fcm_dpo/beta": 0.004359533078968525,
"fcm_dpo/delta": -0.025253912433981895,
"fcm_dpo/margin": 96.28624725341797,
"fcm_dpo/q_t": 0.40311557054519653,
"grad_norm": 10.389751434326172,
"learning_rate": 4.790130070827028e-08,
"logits/chosen": 0.8509294986724854,
"logits/rejected": 0.7792444825172424,
"logps/chosen": -221.84176635742188,
"logps/ref_chosen": -51.100860595703125,
"logps/ref_rejected": -76.06130981445312,
"logps/rejected": -343.0884704589844,
"loss": 1.093,
"margin_dpo/margin_mean": 96.2862548828125,
"margin_dpo/margin_std": 134.68243408203125,
"step": 543
},
{
"KL/chosen_KL_mean": -182.93453979492188,
"KL/mean": -237.28018188476562,
"KL/rejected_KL_mean": -291.62579345703125,
"KL/std": 133.7188720703125,
"epoch": 0.8223733938019653,
"fcm_dpo/beta": 0.004085186868906021,
"fcm_dpo/delta": -0.0571129247546196,
"fcm_dpo/margin": 108.6912841796875,
"fcm_dpo/q_t": 0.3980041742324829,
"grad_norm": 13.596083641052246,
"learning_rate": 4.7125818934366454e-08,
"logits/chosen": 0.8448378443717957,
"logits/rejected": 0.7801798582077026,
"logps/chosen": -243.21177673339844,
"logps/ref_chosen": -60.2772331237793,
"logps/ref_rejected": -88.40553283691406,
"logps/rejected": -380.0313720703125,
"loss": 1.0862,
"margin_dpo/margin_mean": 108.6912841796875,
"margin_dpo/margin_std": 157.70164489746094,
"step": 544
},
{
"KL/chosen_KL_mean": -198.3109130859375,
"KL/mean": -228.24383544921875,
"KL/rejected_KL_mean": -258.1767883300781,
"KL/std": 123.42090606689453,
"epoch": 0.8238851095993953,
"fcm_dpo/beta": 0.004550996236503124,
"fcm_dpo/delta": 0.1455247700214386,
"fcm_dpo/margin": 59.86585998535156,
"fcm_dpo/q_t": 0.44071030616760254,
"grad_norm": 14.440302848815918,
"learning_rate": 4.635601198741607e-08,
"logits/chosen": 0.8114050030708313,
"logits/rejected": 0.7570880055427551,
"logps/chosen": -259.9261474609375,
"logps/ref_chosen": -61.61524963378906,
"logps/ref_rejected": -78.71266174316406,
"logps/rejected": -336.8894348144531,
"loss": 1.2182,
"margin_dpo/margin_mean": 59.86585998535156,
"margin_dpo/margin_std": 132.5935821533203,
"step": 545
},
{
"KL/chosen_KL_mean": -189.25192260742188,
"KL/mean": -222.0496826171875,
"KL/rejected_KL_mean": -254.84744262695312,
"KL/std": 116.97176361083984,
"epoch": 0.8253968253968254,
"fcm_dpo/beta": 0.0049902526661753654,
"fcm_dpo/delta": 0.08677574247121811,
"fcm_dpo/margin": 65.59550476074219,
"fcm_dpo/q_t": 0.4274669885635376,
"grad_norm": 19.589603424072266,
"learning_rate": 4.559190140057428e-08,
"logits/chosen": 0.9217461347579956,
"logits/rejected": 0.9196021556854248,
"logps/chosen": -248.565185546875,
"logps/ref_chosen": -59.313262939453125,
"logps/ref_rejected": -64.73631286621094,
"logps/rejected": -319.583740234375,
"loss": 1.2077,
"margin_dpo/margin_mean": 65.59550476074219,
"margin_dpo/margin_std": 144.1741180419922,
"step": 546
},
{
"KL/chosen_KL_mean": -165.40274047851562,
"KL/mean": -215.89859008789062,
"KL/rejected_KL_mean": -266.3944396972656,
"KL/std": 122.97581481933594,
"epoch": 0.8269085411942555,
"fcm_dpo/beta": 0.004632370080798864,
"fcm_dpo/delta": -0.09499009698629379,
"fcm_dpo/margin": 100.99168395996094,
"fcm_dpo/q_t": 0.389495313167572,
"grad_norm": 12.160846710205078,
"learning_rate": 4.483350854765672e-08,
"logits/chosen": 0.7838672995567322,
"logits/rejected": 0.7346804141998291,
"logps/chosen": -220.37948608398438,
"logps/ref_chosen": -54.97674560546875,
"logps/ref_rejected": -75.35922241210938,
"logps/rejected": -341.753662109375,
"loss": 1.0705,
"margin_dpo/margin_mean": 100.99168395996094,
"margin_dpo/margin_std": 140.6912841796875,
"step": 547
},
{
"KL/chosen_KL_mean": -196.61190795898438,
"KL/mean": -225.8825225830078,
"KL/rejected_KL_mean": -255.1531524658203,
"KL/std": 121.33120727539062,
"epoch": 0.8284202569916855,
"fcm_dpo/beta": 0.005131861194968224,
"fcm_dpo/delta": 0.11382049322128296,
"fcm_dpo/margin": 58.541263580322266,
"fcm_dpo/q_t": 0.43540245294570923,
"grad_norm": 17.081180572509766,
"learning_rate": 4.4080854642541826e-08,
"logits/chosen": 0.7621930837631226,
"logits/rejected": 0.7174137830734253,
"logps/chosen": -259.82257080078125,
"logps/ref_chosen": -63.21067428588867,
"logps/ref_rejected": -81.23347473144531,
"logps/rejected": -336.3866271972656,
"loss": 1.2171,
"margin_dpo/margin_mean": 58.54126739501953,
"margin_dpo/margin_std": 132.77597045898438,
"step": 548
},
{
"KL/chosen_KL_mean": -194.5125732421875,
"KL/mean": -231.58607482910156,
"KL/rejected_KL_mean": -268.6595458984375,
"KL/std": 135.0965576171875,
"epoch": 0.8299319727891157,
"fcm_dpo/beta": 0.005150582641363144,
"fcm_dpo/delta": 0.02017725259065628,
"fcm_dpo/margin": 74.14700317382812,
"fcm_dpo/q_t": 0.4193522334098816,
"grad_norm": 20.222238540649414,
"learning_rate": 4.333396073857723e-08,
"logits/chosen": 0.9132688045501709,
"logits/rejected": 0.8558034896850586,
"logps/chosen": -258.78607177734375,
"logps/ref_chosen": -64.27351379394531,
"logps/ref_rejected": -92.31663513183594,
"logps/rejected": -360.9761962890625,
"loss": 1.195,
"margin_dpo/margin_mean": 74.14698791503906,
"margin_dpo/margin_std": 160.41629028320312,
"step": 549
},
{
"KL/chosen_KL_mean": -193.46360778808594,
"KL/mean": -221.16964721679688,
"KL/rejected_KL_mean": -248.87567138671875,
"KL/std": 118.34536743164062,
"epoch": 0.8314436885865457,
"fcm_dpo/beta": 0.0052723027765750885,
"fcm_dpo/delta": -0.006274649873375893,
"fcm_dpo/margin": 55.412071228027344,
"fcm_dpo/q_t": 0.4349931478500366,
"grad_norm": 15.994507789611816,
"learning_rate": 4.259284772799099e-08,
"logits/chosen": 0.8577879667282104,
"logits/rejected": 0.8388744592666626,
"logps/chosen": -249.6940460205078,
"logps/ref_chosen": -56.230438232421875,
"logps/ref_rejected": -62.59788513183594,
"logps/rejected": -311.47357177734375,
"loss": 1.2279,
"margin_dpo/margin_mean": 55.41207504272461,
"margin_dpo/margin_std": 129.4113006591797,
"step": 550
},
{
"KL/chosen_KL_mean": -202.26455688476562,
"KL/mean": -236.17559814453125,
"KL/rejected_KL_mean": -270.086669921875,
"KL/std": 128.12054443359375,
"epoch": 0.8329554043839759,
"fcm_dpo/beta": 0.005462060682475567,
"fcm_dpo/delta": 0.03551365062594414,
"fcm_dpo/margin": 67.82211303710938,
"fcm_dpo/q_t": 0.4226077198982239,
"grad_norm": 15.54771614074707,
"learning_rate": 4.1857536341307176e-08,
"logits/chosen": 0.857745885848999,
"logits/rejected": 0.8316745758056641,
"logps/chosen": -270.01177978515625,
"logps/ref_chosen": -67.74720764160156,
"logps/ref_rejected": -87.04285430908203,
"logps/rejected": -357.1295166015625,
"loss": 1.1536,
"margin_dpo/margin_mean": 67.82211303710938,
"margin_dpo/margin_std": 127.63706970214844,
"step": 551
},
{
"KL/chosen_KL_mean": -185.70755004882812,
"KL/mean": -225.22647094726562,
"KL/rejected_KL_mean": -264.74542236328125,
"KL/std": 131.25448608398438,
"epoch": 0.8344671201814059,
"fcm_dpo/beta": 0.005472094286233187,
"fcm_dpo/delta": -0.044923026114702225,
"fcm_dpo/margin": 79.03784942626953,
"fcm_dpo/q_t": 0.40045636892318726,
"grad_norm": 20.222753524780273,
"learning_rate": 4.112804714676593e-08,
"logits/chosen": 0.8084758520126343,
"logits/rejected": 0.7642360329627991,
"logps/chosen": -248.63381958007812,
"logps/ref_chosen": -62.92625427246094,
"logps/ref_rejected": -82.98365783691406,
"logps/rejected": -347.72906494140625,
"loss": 1.1101,
"margin_dpo/margin_mean": 79.037841796875,
"margin_dpo/margin_std": 123.42013549804688,
"step": 552
},
{
"KL/chosen_KL_mean": -201.88070678710938,
"KL/mean": -240.699462890625,
"KL/rejected_KL_mean": -279.5182189941406,
"KL/std": 127.67655944824219,
"epoch": 0.8359788359788359,
"fcm_dpo/beta": 0.005206805653870106,
"fcm_dpo/delta": -0.005316682159900665,
"fcm_dpo/margin": 77.63748168945312,
"fcm_dpo/q_t": 0.4139271080493927,
"grad_norm": 14.778176307678223,
"learning_rate": 4.0404400549748144e-08,
"logits/chosen": 0.8216131329536438,
"logits/rejected": 0.7376811504364014,
"logps/chosen": -257.919189453125,
"logps/ref_chosen": -56.038490295410156,
"logps/ref_rejected": -84.48454284667969,
"logps/rejected": -364.00274658203125,
"loss": 1.1602,
"margin_dpo/margin_mean": 77.63748168945312,
"margin_dpo/margin_std": 151.02700805664062,
"step": 553
},
{
"KL/chosen_KL_mean": -181.76394653320312,
"KL/mean": -227.24557495117188,
"KL/rejected_KL_mean": -272.7271728515625,
"KL/std": 120.14936065673828,
"epoch": 0.8374905517762661,
"fcm_dpo/beta": 0.004935280419886112,
"fcm_dpo/delta": -0.06360197812318802,
"fcm_dpo/margin": 90.9632568359375,
"fcm_dpo/q_t": 0.3936072587966919,
"grad_norm": 14.283166885375977,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 0.7989473342895508,
"logits/rejected": 0.7961627244949341,
"logps/chosen": -246.2945556640625,
"logps/ref_chosen": -64.53059387207031,
"logps/ref_rejected": -71.2155990600586,
"logps/rejected": -343.94281005859375,
"loss": 1.0841,
"margin_dpo/margin_mean": 90.9632568359375,
"margin_dpo/margin_std": 130.098876953125,
"step": 554
},
{
"KL/chosen_KL_mean": -197.27484130859375,
"KL/mean": -231.05706787109375,
"KL/rejected_KL_mean": -264.83929443359375,
"KL/std": 118.86752319335938,
"epoch": 0.8390022675736961,
"fcm_dpo/beta": 0.004804479889571667,
"fcm_dpo/delta": -0.02939688041806221,
"fcm_dpo/margin": 67.564453125,
"fcm_dpo/q_t": 0.4228440821170807,
"grad_norm": 14.867000579833984,
"learning_rate": 3.89747159520904e-08,
"logits/chosen": 0.8469399213790894,
"logits/rejected": 0.8320943117141724,
"logps/chosen": -263.9267578125,
"logps/ref_chosen": -66.65191650390625,
"logps/ref_rejected": -68.6667251586914,
"logps/rejected": -333.5060119628906,
"loss": 1.2053,
"margin_dpo/margin_mean": 67.564453125,
"margin_dpo/margin_std": 141.87362670898438,
"step": 555
},
{
"KL/chosen_KL_mean": -193.35748291015625,
"KL/mean": -228.408935546875,
"KL/rejected_KL_mean": -263.46038818359375,
"KL/std": 123.95942687988281,
"epoch": 0.8405139833711263,
"fcm_dpo/beta": 0.004928875714540482,
"fcm_dpo/delta": 0.06609125435352325,
"fcm_dpo/margin": 70.10289001464844,
"fcm_dpo/q_t": 0.4252849817276001,
"grad_norm": 13.865388870239258,
"learning_rate": 3.826871794280192e-08,
"logits/chosen": 0.8748590350151062,
"logits/rejected": 0.8386209607124329,
"logps/chosen": -246.18984985351562,
"logps/ref_chosen": -52.832366943359375,
"logps/ref_rejected": -64.49044036865234,
"logps/rejected": -327.9508056640625,
"loss": 1.1806,
"margin_dpo/margin_mean": 70.10289764404297,
"margin_dpo/margin_std": 136.9292755126953,
"step": 556
},
{
"KL/chosen_KL_mean": -192.00454711914062,
"KL/mean": -241.59774780273438,
"KL/rejected_KL_mean": -291.19091796875,
"KL/std": 123.56182861328125,
"epoch": 0.8420256991685563,
"fcm_dpo/beta": 0.004650956951081753,
"fcm_dpo/delta": -0.08415950834751129,
"fcm_dpo/margin": 99.18640899658203,
"fcm_dpo/q_t": 0.3913089632987976,
"grad_norm": 11.975605964660645,
"learning_rate": 3.756864251262143e-08,
"logits/chosen": 0.9212883710861206,
"logits/rejected": 0.8599925637245178,
"logps/chosen": -247.04052734375,
"logps/ref_chosen": -55.03598403930664,
"logps/ref_rejected": -75.80644989013672,
"logps/rejected": -366.99737548828125,
"loss": 1.0543,
"margin_dpo/margin_mean": 99.18640899658203,
"margin_dpo/margin_std": 121.38153076171875,
"step": 557
},
{
"KL/chosen_KL_mean": -181.1847686767578,
"KL/mean": -234.77792358398438,
"KL/rejected_KL_mean": -288.3710632324219,
"KL/std": 131.50604248046875,
"epoch": 0.8435374149659864,
"fcm_dpo/beta": 0.004277751315385103,
"fcm_dpo/delta": -0.08086490631103516,
"fcm_dpo/margin": 107.18629455566406,
"fcm_dpo/q_t": 0.39286717772483826,
"grad_norm": 10.80577564239502,
"learning_rate": 3.687450924416341e-08,
"logits/chosen": 0.8621159195899963,
"logits/rejected": 0.8185891509056091,
"logps/chosen": -244.41111755371094,
"logps/ref_chosen": -63.226348876953125,
"logps/ref_rejected": -91.46881866455078,
"logps/rejected": -379.8398742675781,
"loss": 1.0614,
"margin_dpo/margin_mean": 107.18629455566406,
"margin_dpo/margin_std": 138.95535278320312,
"step": 558
},
{
"KL/chosen_KL_mean": -184.31796264648438,
"KL/mean": -231.82647705078125,
"KL/rejected_KL_mean": -279.3349609375,
"KL/std": 131.37059020996094,
"epoch": 0.8450491307634165,
"fcm_dpo/beta": 0.0041785649955272675,
"fcm_dpo/delta": 0.0005370378494262695,
"fcm_dpo/margin": 95.01700592041016,
"fcm_dpo/q_t": 0.4134005606174469,
"grad_norm": 10.951775550842285,
"learning_rate": 3.6186337553827743e-08,
"logits/chosen": 0.8456987738609314,
"logits/rejected": 0.7955515384674072,
"logps/chosen": -245.839599609375,
"logps/ref_chosen": -61.521644592285156,
"logps/ref_rejected": -82.83859252929688,
"logps/rejected": -362.1735534667969,
"loss": 1.1311,
"margin_dpo/margin_mean": 95.01700592041016,
"margin_dpo/margin_std": 159.78627014160156,
"step": 559
},
{
"KL/chosen_KL_mean": -200.00225830078125,
"KL/mean": -245.2239990234375,
"KL/rejected_KL_mean": -290.44573974609375,
"KL/std": 124.63345336914062,
"epoch": 0.8465608465608465,
"fcm_dpo/beta": 0.004382754676043987,
"fcm_dpo/delta": 0.002800390124320984,
"fcm_dpo/margin": 90.44346618652344,
"fcm_dpo/q_t": 0.4082622528076172,
"grad_norm": 15.912379264831543,
"learning_rate": 3.550414669125573e-08,
"logits/chosen": 0.8386662006378174,
"logits/rejected": 0.8095963001251221,
"logps/chosen": -260.64349365234375,
"logps/ref_chosen": -60.64122009277344,
"logps/ref_rejected": -78.75474548339844,
"logps/rejected": -369.20050048828125,
"loss": 1.0935,
"margin_dpo/margin_mean": 90.4434814453125,
"margin_dpo/margin_std": 116.68689727783203,
"step": 560
},
{
"KL/chosen_KL_mean": -186.88943481445312,
"KL/mean": -225.20635986328125,
"KL/rejected_KL_mean": -263.52325439453125,
"KL/std": 120.019287109375,
"epoch": 0.8480725623582767,
"fcm_dpo/beta": 0.004479461815208197,
"fcm_dpo/delta": 0.0687173530459404,
"fcm_dpo/margin": 76.63383483886719,
"fcm_dpo/q_t": 0.42475682497024536,
"grad_norm": 11.804110527038574,
"learning_rate": 3.482795573879241e-08,
"logits/chosen": 0.7993382215499878,
"logits/rejected": 0.7716021537780762,
"logps/chosen": -249.38803100585938,
"logps/ref_chosen": -62.49859619140625,
"logps/ref_rejected": -78.72064208984375,
"logps/rejected": -342.243896484375,
"loss": 1.1584,
"margin_dpo/margin_mean": 76.63383483886719,
"margin_dpo/margin_std": 132.6756591796875,
"step": 561
},
{
"KL/chosen_KL_mean": -198.23243713378906,
"KL/mean": -250.70225524902344,
"KL/rejected_KL_mean": -303.17205810546875,
"KL/std": 146.47659301757812,
"epoch": 0.8495842781557067,
"fcm_dpo/beta": 0.00422039907425642,
"fcm_dpo/delta": -0.06546664237976074,
"fcm_dpo/margin": 104.93963623046875,
"fcm_dpo/q_t": 0.3977462649345398,
"grad_norm": 15.125443458557129,
"learning_rate": 3.415778361095226e-08,
"logits/chosen": 0.8647781610488892,
"logits/rejected": 0.8419674038887024,
"logps/chosen": -273.01416015625,
"logps/ref_chosen": -74.78173828125,
"logps/ref_rejected": -92.63499450683594,
"logps/rejected": -395.80706787109375,
"loss": 1.0757,
"margin_dpo/margin_mean": 104.93963623046875,
"margin_dpo/margin_std": 140.125244140625,
"step": 562
},
{
"KL/chosen_KL_mean": -163.90536499023438,
"KL/mean": -207.61956787109375,
"KL/rejected_KL_mean": -251.3337860107422,
"KL/std": 107.05549621582031,
"epoch": 0.8510959939531368,
"fcm_dpo/beta": 0.004395781550556421,
"fcm_dpo/delta": 0.019247282296419144,
"fcm_dpo/margin": 87.4284439086914,
"fcm_dpo/q_t": 0.41145721077919006,
"grad_norm": 18.718957901000977,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.9080963730812073,
"logits/rejected": 0.862755537033081,
"logps/chosen": -214.10385131835938,
"logps/ref_chosen": -50.19850158691406,
"logps/ref_rejected": -66.76687622070312,
"logps/rejected": -318.10064697265625,
"loss": 1.124,
"margin_dpo/margin_mean": 87.4284439086914,
"margin_dpo/margin_std": 136.39706420898438,
"step": 563
},
{
"KL/chosen_KL_mean": -165.5902099609375,
"KL/mean": -220.1693115234375,
"KL/rejected_KL_mean": -274.7484130859375,
"KL/std": 116.85183715820312,
"epoch": 0.8526077097505669,
"fcm_dpo/beta": 0.004158479161560535,
"fcm_dpo/delta": -0.07036039233207703,
"fcm_dpo/margin": 109.1581802368164,
"fcm_dpo/q_t": 0.39226311445236206,
"grad_norm": 11.553539276123047,
"learning_rate": 3.283557064487785e-08,
"logits/chosen": 0.8010308146476746,
"logits/rejected": 0.7797284126281738,
"logps/chosen": -221.33106994628906,
"logps/ref_chosen": -55.7408447265625,
"logps/ref_rejected": -74.82323455810547,
"logps/rejected": -349.5716552734375,
"loss": 1.0657,
"margin_dpo/margin_mean": 109.1581802368164,
"margin_dpo/margin_std": 142.97979736328125,
"step": 564
},
{
"KL/chosen_KL_mean": -198.5447998046875,
"KL/mean": -242.15261840820312,
"KL/rejected_KL_mean": -285.76043701171875,
"KL/std": 124.1990966796875,
"epoch": 0.854119425547997,
"fcm_dpo/beta": 0.004263964481651783,
"fcm_dpo/delta": 0.033044081181287766,
"fcm_dpo/margin": 87.21564483642578,
"fcm_dpo/q_t": 0.41419297456741333,
"grad_norm": 13.659796714782715,
"learning_rate": 3.218356679178252e-08,
"logits/chosen": 0.8753294944763184,
"logits/rejected": 0.833126962184906,
"logps/chosen": -256.8821716308594,
"logps/ref_chosen": -58.33738327026367,
"logps/ref_rejected": -78.31776428222656,
"logps/rejected": -364.07818603515625,
"loss": 1.1152,
"margin_dpo/margin_mean": 87.21563720703125,
"margin_dpo/margin_std": 123.91053009033203,
"step": 565
},
{
"KL/chosen_KL_mean": -190.3877716064453,
"KL/mean": -228.84381103515625,
"KL/rejected_KL_mean": -267.29986572265625,
"KL/std": 131.32406616210938,
"epoch": 0.8556311413454271,
"fcm_dpo/beta": 0.00454290583729744,
"fcm_dpo/delta": 0.05613939091563225,
"fcm_dpo/margin": 76.91210174560547,
"fcm_dpo/q_t": 0.42436790466308594,
"grad_norm": 12.740806579589844,
"learning_rate": 3.1537655732553764e-08,
"logits/chosen": 0.8327311277389526,
"logits/rejected": 0.830025851726532,
"logps/chosen": -261.61151123046875,
"logps/ref_chosen": -71.22373962402344,
"logps/ref_rejected": -71.11601257324219,
"logps/rejected": -338.4158935546875,
"loss": 1.1979,
"margin_dpo/margin_mean": 76.912109375,
"margin_dpo/margin_std": 165.12557983398438,
"step": 566
},
{
"KL/chosen_KL_mean": -179.38357543945312,
"KL/mean": -228.2648468017578,
"KL/rejected_KL_mean": -277.1461181640625,
"KL/std": 120.70500946044922,
"epoch": 0.8571428571428571,
"fcm_dpo/beta": 0.004256717395037413,
"fcm_dpo/delta": -0.02629101276397705,
"fcm_dpo/margin": 97.76256561279297,
"fcm_dpo/q_t": 0.4018706977367401,
"grad_norm": 10.185369491577148,
"learning_rate": 3.089785553471233e-08,
"logits/chosen": 0.8462409973144531,
"logits/rejected": 0.7748077511787415,
"logps/chosen": -232.05284118652344,
"logps/ref_chosen": -52.669273376464844,
"logps/ref_rejected": -74.34785461425781,
"logps/rejected": -351.4939880371094,
"loss": 1.093,
"margin_dpo/margin_mean": 97.76256561279297,
"margin_dpo/margin_std": 132.65936279296875,
"step": 567
},
{
"KL/chosen_KL_mean": -167.775634765625,
"KL/mean": -226.44500732421875,
"KL/rejected_KL_mean": -285.1143798828125,
"KL/std": 128.91400146484375,
"epoch": 0.8586545729402872,
"fcm_dpo/beta": 0.004162575583904982,
"fcm_dpo/delta": -0.11834442615509033,
"fcm_dpo/margin": 117.33872985839844,
"fcm_dpo/q_t": 0.3848383128643036,
"grad_norm": 13.649627685546875,
"learning_rate": 3.026418409484513e-08,
"logits/chosen": 0.8517801761627197,
"logits/rejected": 0.7852275371551514,
"logps/chosen": -219.95364379882812,
"logps/ref_chosen": -52.178001403808594,
"logps/ref_rejected": -85.8277587890625,
"logps/rejected": -370.942138671875,
"loss": 1.0358,
"margin_dpo/margin_mean": 117.3387222290039,
"margin_dpo/margin_std": 133.35511779785156,
"step": 568
},
{
"KL/chosen_KL_mean": -196.99539184570312,
"KL/mean": -225.05218505859375,
"KL/rejected_KL_mean": -253.1090087890625,
"KL/std": 130.43826293945312,
"epoch": 0.8601662887377173,
"fcm_dpo/beta": 0.0039661955088377,
"fcm_dpo/delta": 0.02197522297501564,
"fcm_dpo/margin": 56.11360549926758,
"fcm_dpo/q_t": 0.4494395852088928,
"grad_norm": 13.287290573120117,
"learning_rate": 2.963665913810451e-08,
"logits/chosen": 0.7764110565185547,
"logits/rejected": 0.7512776851654053,
"logps/chosen": -259.6446533203125,
"logps/ref_chosen": -62.649261474609375,
"logps/ref_rejected": -75.4298324584961,
"logps/rejected": -328.538818359375,
"loss": 1.2453,
"margin_dpo/margin_mean": 56.11360549926758,
"margin_dpo/margin_std": 131.75128173828125,
"step": 569
},
{
"KL/chosen_KL_mean": -173.2542266845703,
"KL/mean": -227.6485595703125,
"KL/rejected_KL_mean": -282.04290771484375,
"KL/std": 120.7261962890625,
"epoch": 0.8616780045351474,
"fcm_dpo/beta": 0.003945560660213232,
"fcm_dpo/delta": -0.03856857866048813,
"fcm_dpo/margin": 108.78868103027344,
"fcm_dpo/q_t": 0.40049558877944946,
"grad_norm": 11.730306625366211,
"learning_rate": 2.9015298217712453e-08,
"logits/chosen": 0.8236995935440063,
"logits/rejected": 0.7503319978713989,
"logps/chosen": -223.2960205078125,
"logps/ref_chosen": -50.04179382324219,
"logps/ref_rejected": -78.27146911621094,
"logps/rejected": -360.3143615722656,
"loss": 1.0746,
"margin_dpo/margin_mean": 108.7886734008789,
"margin_dpo/margin_std": 139.9506378173828,
"step": 570
},
{
"KL/chosen_KL_mean": -183.8597412109375,
"KL/mean": -215.78030395507812,
"KL/rejected_KL_mean": -247.7008514404297,
"KL/std": 117.1596908569336,
"epoch": 0.8631897203325775,
"fcm_dpo/beta": 0.003919776063412428,
"fcm_dpo/delta": 0.05289241671562195,
"fcm_dpo/margin": 63.841087341308594,
"fcm_dpo/q_t": 0.44315797090530396,
"grad_norm": 10.650339126586914,
"learning_rate": 2.840011871446962e-08,
"logits/chosen": 0.8218934535980225,
"logits/rejected": 0.8082360625267029,
"logps/chosen": -237.51657104492188,
"logps/ref_chosen": -53.65681457519531,
"logps/ref_rejected": -66.13298034667969,
"logps/rejected": -313.8338317871094,
"loss": 1.2213,
"margin_dpo/margin_mean": 63.841087341308594,
"margin_dpo/margin_std": 135.2704315185547,
"step": 571
},
{
"KL/chosen_KL_mean": -192.3538818359375,
"KL/mean": -225.6499786376953,
"KL/rejected_KL_mean": -258.9460754394531,
"KL/std": 122.84170532226562,
"epoch": 0.8647014361300076,
"fcm_dpo/beta": 0.0040804725140333176,
"fcm_dpo/delta": 0.02796788141131401,
"fcm_dpo/margin": 66.59219360351562,
"fcm_dpo/q_t": 0.43682506680488586,
"grad_norm": 12.89725399017334,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 0.824752926826477,
"logits/rejected": 0.863124668598175,
"logps/chosen": -267.17181396484375,
"logps/ref_chosen": -74.81792449951172,
"logps/ref_rejected": -65.88681030273438,
"logps/rejected": -324.8328857421875,
"loss": 1.1976,
"margin_dpo/margin_mean": 66.59219360351562,
"margin_dpo/margin_std": 126.39369201660156,
"step": 572
},
{
"KL/chosen_KL_mean": -205.2719268798828,
"KL/mean": -247.1676788330078,
"KL/rejected_KL_mean": -289.06341552734375,
"KL/std": 138.2989501953125,
"epoch": 0.8662131519274376,
"fcm_dpo/beta": 0.0043049827218055725,
"fcm_dpo/delta": 0.0475587323307991,
"fcm_dpo/margin": 83.79151916503906,
"fcm_dpo/q_t": 0.423142671585083,
"grad_norm": 12.913630485534668,
"learning_rate": 2.718837261761528e-08,
"logits/chosen": 0.838950514793396,
"logits/rejected": 0.8056519031524658,
"logps/chosen": -273.99755859375,
"logps/ref_chosen": -68.72564697265625,
"logps/ref_rejected": -88.16201782226562,
"logps/rejected": -377.2254638671875,
"loss": 1.1755,
"margin_dpo/margin_mean": 83.79151916503906,
"margin_dpo/margin_std": 166.9637451171875,
"step": 573
},
{
"KL/chosen_KL_mean": -182.779296875,
"KL/mean": -236.46792602539062,
"KL/rejected_KL_mean": -290.156494140625,
"KL/std": 128.50814819335938,
"epoch": 0.8677248677248677,
"fcm_dpo/beta": 0.004241702146828175,
"fcm_dpo/delta": -0.07349122315645218,
"fcm_dpo/margin": 107.37722778320312,
"fcm_dpo/q_t": 0.3932126462459564,
"grad_norm": 11.176490783691406,
"learning_rate": 2.659183991914696e-08,
"logits/chosen": 0.8853078484535217,
"logits/rejected": 0.8360748887062073,
"logps/chosen": -239.09271240234375,
"logps/ref_chosen": -56.31340026855469,
"logps/ref_rejected": -83.91553497314453,
"logps/rejected": -374.0720520019531,
"loss": 1.0484,
"margin_dpo/margin_mean": 107.37722778320312,
"margin_dpo/margin_std": 123.63481140136719,
"step": 574
},
{
"KL/chosen_KL_mean": -188.70425415039062,
"KL/mean": -226.75677490234375,
"KL/rejected_KL_mean": -264.809326171875,
"KL/std": 128.05438232421875,
"epoch": 0.8692365835222978,
"fcm_dpo/beta": 0.003928046207875013,
"fcm_dpo/delta": -0.026024334132671356,
"fcm_dpo/margin": 76.10505676269531,
"fcm_dpo/q_t": 0.4327458143234253,
"grad_norm": 11.480947494506836,
"learning_rate": 2.600155642716606e-08,
"logits/chosen": 0.8551039099693298,
"logits/rejected": 0.7906278371810913,
"logps/chosen": -253.2883758544922,
"logps/ref_chosen": -64.5841293334961,
"logps/ref_rejected": -93.47034454345703,
"logps/rejected": -358.2796630859375,
"loss": 1.2022,
"margin_dpo/margin_mean": 76.10505676269531,
"margin_dpo/margin_std": 155.01919555664062,
"step": 575
},
{
"KL/chosen_KL_mean": -165.97982788085938,
"KL/mean": -223.73573303222656,
"KL/rejected_KL_mean": -281.49163818359375,
"KL/std": 128.92742919921875,
"epoch": 0.8707482993197279,
"fcm_dpo/beta": 0.0036893547512590885,
"fcm_dpo/delta": -0.03991346433758736,
"fcm_dpo/margin": 115.5118179321289,
"fcm_dpo/q_t": 0.39912840723991394,
"grad_norm": 10.718217849731445,
"learning_rate": 2.5417538653170754e-08,
"logits/chosen": 0.8531392216682434,
"logits/rejected": 0.7734321355819702,
"logps/chosen": -219.26034545898438,
"logps/ref_chosen": -53.28052520751953,
"logps/ref_rejected": -84.2000503540039,
"logps/rejected": -365.69171142578125,
"loss": 1.0733,
"margin_dpo/margin_mean": 115.5118179321289,
"margin_dpo/margin_std": 141.59327697753906,
"step": 576
},
{
"KL/chosen_KL_mean": -186.1846923828125,
"KL/mean": -221.00384521484375,
"KL/rejected_KL_mean": -255.822998046875,
"KL/std": 124.04796600341797,
"epoch": 0.872260015117158,
"fcm_dpo/beta": 0.0041536302305758,
"fcm_dpo/delta": 0.12937919795513153,
"fcm_dpo/margin": 69.6383056640625,
"fcm_dpo/q_t": 0.43577146530151367,
"grad_norm": 13.241288185119629,
"learning_rate": 2.4839802933393607e-08,
"logits/chosen": 0.8255484104156494,
"logits/rejected": 0.8245355486869812,
"logps/chosen": -248.50938415527344,
"logps/ref_chosen": -62.32468795776367,
"logps/ref_rejected": -67.300537109375,
"logps/rejected": -323.12353515625,
"loss": 1.1965,
"margin_dpo/margin_mean": 69.6383056640625,
"margin_dpo/margin_std": 131.79922485351562,
"step": 577
},
{
"KL/chosen_KL_mean": -177.88160705566406,
"KL/mean": -212.32455444335938,
"KL/rejected_KL_mean": -246.7675018310547,
"KL/std": 131.18441772460938,
"epoch": 0.873771730914588,
"fcm_dpo/beta": 0.004623678512871265,
"fcm_dpo/delta": 0.09660382568836212,
"fcm_dpo/margin": 68.88590240478516,
"fcm_dpo/q_t": 0.4327906668186188,
"grad_norm": 14.404627799987793,
"learning_rate": 2.4268365428344733e-08,
"logits/chosen": 0.8864824771881104,
"logits/rejected": 0.8790782690048218,
"logps/chosen": -234.5371856689453,
"logps/ref_chosen": -56.65557861328125,
"logps/ref_rejected": -68.21835327148438,
"logps/rejected": -314.98583984375,
"loss": 1.2075,
"margin_dpo/margin_mean": 68.88590240478516,
"margin_dpo/margin_std": 151.61154174804688,
"step": 578
},
{
"KL/chosen_KL_mean": -182.9407501220703,
"KL/mean": -233.8430938720703,
"KL/rejected_KL_mean": -284.74542236328125,
"KL/std": 119.40805053710938,
"epoch": 0.8752834467120182,
"fcm_dpo/beta": 0.004438809119164944,
"fcm_dpo/delta": -0.06895958632230759,
"fcm_dpo/margin": 101.8046875,
"fcm_dpo/q_t": 0.3916490375995636,
"grad_norm": 11.517224311828613,
"learning_rate": 2.3703242122359357e-08,
"logits/chosen": 0.7991275191307068,
"logits/rejected": 0.7866017818450928,
"logps/chosen": -239.7504119873047,
"logps/ref_chosen": -56.809661865234375,
"logps/ref_rejected": -68.09613037109375,
"logps/rejected": -352.841552734375,
"loss": 1.0552,
"margin_dpo/margin_mean": 101.8046875,
"margin_dpo/margin_std": 123.41009521484375,
"step": 579
},
{
"KL/chosen_KL_mean": -186.92544555664062,
"KL/mean": -233.40525817871094,
"KL/rejected_KL_mean": -279.88507080078125,
"KL/std": 144.23040771484375,
"epoch": 0.8767951625094482,
"fcm_dpo/beta": 0.004471241496503353,
"fcm_dpo/delta": -0.022698376327753067,
"fcm_dpo/margin": 92.95962524414062,
"fcm_dpo/q_t": 0.40974441170692444,
"grad_norm": 14.038193702697754,
"learning_rate": 2.3144448823151392e-08,
"logits/chosen": 0.8470299243927002,
"logits/rejected": 0.8106606602668762,
"logps/chosen": -244.6255645751953,
"logps/ref_chosen": -57.70011520385742,
"logps/ref_rejected": -77.90664672851562,
"logps/rejected": -357.7917175292969,
"loss": 1.1332,
"margin_dpo/margin_mean": 92.95962524414062,
"margin_dpo/margin_std": 161.46102905273438,
"step": 580
},
{
"KL/chosen_KL_mean": -203.38998413085938,
"KL/mean": -243.64874267578125,
"KL/rejected_KL_mean": -283.90753173828125,
"KL/std": 124.84356689453125,
"epoch": 0.8783068783068783,
"fcm_dpo/beta": 0.004432911053299904,
"fcm_dpo/delta": 0.05242428183555603,
"fcm_dpo/margin": 80.51752471923828,
"fcm_dpo/q_t": 0.4223071336746216,
"grad_norm": 12.285096168518066,
"learning_rate": 2.259200116137039e-08,
"logits/chosen": 0.8738093376159668,
"logits/rejected": 0.8231921792030334,
"logps/chosen": -262.72235107421875,
"logps/ref_chosen": -59.332359313964844,
"logps/ref_rejected": -83.64482116699219,
"logps/rejected": -367.5523376464844,
"loss": 1.168,
"margin_dpo/margin_mean": 80.51752471923828,
"margin_dpo/margin_std": 151.82763671875,
"step": 581
},
{
"KL/chosen_KL_mean": -186.85243225097656,
"KL/mean": -224.37152099609375,
"KL/rejected_KL_mean": -261.890625,
"KL/std": 117.22724151611328,
"epoch": 0.8798185941043084,
"fcm_dpo/beta": 0.004786387085914612,
"fcm_dpo/delta": 0.047857534140348434,
"fcm_dpo/margin": 75.03822326660156,
"fcm_dpo/q_t": 0.4219922423362732,
"grad_norm": 12.014780044555664,
"learning_rate": 2.204591459016525e-08,
"logits/chosen": 0.8366580605506897,
"logits/rejected": 0.8666825294494629,
"logps/chosen": -251.01528930664062,
"logps/ref_chosen": -64.16285705566406,
"logps/ref_rejected": -58.632896423339844,
"logps/rejected": -320.5235595703125,
"loss": 1.1613,
"margin_dpo/margin_mean": 75.03822326660156,
"margin_dpo/margin_std": 137.0283966064453,
"step": 582
},
{
"KL/chosen_KL_mean": -183.95529174804688,
"KL/mean": -235.9126739501953,
"KL/rejected_KL_mean": -287.87005615234375,
"KL/std": 138.58448791503906,
"epoch": 0.8813303099017384,
"fcm_dpo/beta": 0.004435844719409943,
"fcm_dpo/delta": -0.08139852434396744,
"fcm_dpo/margin": 103.91476440429688,
"fcm_dpo/q_t": 0.3945728540420532,
"grad_norm": 13.930092811584473,
"learning_rate": 2.1506204384751064e-08,
"logits/chosen": 0.9213510751724243,
"logits/rejected": 0.8438308238983154,
"logps/chosen": -235.82769775390625,
"logps/ref_chosen": -51.87239456176758,
"logps/ref_rejected": -83.86331176757812,
"logps/rejected": -371.7333679199219,
"loss": 1.0912,
"margin_dpo/margin_mean": 103.91476440429688,
"margin_dpo/margin_std": 158.84449768066406,
"step": 583
},
{
"KL/chosen_KL_mean": -168.98516845703125,
"KL/mean": -212.2644805908203,
"KL/rejected_KL_mean": -255.54379272460938,
"KL/std": 121.94096374511719,
"epoch": 0.8828420256991686,
"fcm_dpo/beta": 0.004407728090882301,
"fcm_dpo/delta": 0.02222277782857418,
"fcm_dpo/margin": 86.5586166381836,
"fcm_dpo/q_t": 0.4167934060096741,
"grad_norm": 14.670916557312012,
"learning_rate": 2.09728856419826e-08,
"logits/chosen": 0.9364026784896851,
"logits/rejected": 0.8511247634887695,
"logps/chosen": -215.55654907226562,
"logps/ref_chosen": -46.571388244628906,
"logps/ref_rejected": -80.67969512939453,
"logps/rejected": -336.2234802246094,
"loss": 1.1498,
"margin_dpo/margin_mean": 86.5586166381836,
"margin_dpo/margin_std": 155.74969482421875,
"step": 584
},
{
"KL/chosen_KL_mean": -198.3673095703125,
"KL/mean": -229.12481689453125,
"KL/rejected_KL_mean": -259.88232421875,
"KL/std": 135.75291442871094,
"epoch": 0.8843537414965986,
"fcm_dpo/beta": 0.004529936239123344,
"fcm_dpo/delta": 0.014046979136765003,
"fcm_dpo/margin": 61.515010833740234,
"fcm_dpo/q_t": 0.43704885244369507,
"grad_norm": 13.787379264831543,
"learning_rate": 2.044597327993153e-08,
"logits/chosen": 0.8455356359481812,
"logits/rejected": 0.7982293367385864,
"logps/chosen": -256.4918518066406,
"logps/ref_chosen": -58.124534606933594,
"logps/ref_rejected": -79.00538635253906,
"logps/rejected": -338.8876953125,
"loss": 1.2115,
"margin_dpo/margin_mean": 61.5150146484375,
"margin_dpo/margin_std": 131.28334045410156,
"step": 585
},
{
"KL/chosen_KL_mean": -179.8886260986328,
"KL/mean": -218.91390991210938,
"KL/rejected_KL_mean": -257.939208984375,
"KL/std": 107.77200317382812,
"epoch": 0.8858654572940288,
"fcm_dpo/beta": 0.004673574585467577,
"fcm_dpo/delta": 0.04316072538495064,
"fcm_dpo/margin": 78.0505599975586,
"fcm_dpo/q_t": 0.4169018268585205,
"grad_norm": 11.847397804260254,
"learning_rate": 1.9925482037469187e-08,
"logits/chosen": 0.8758537769317627,
"logits/rejected": 0.8435441255569458,
"logps/chosen": -233.99026489257812,
"logps/ref_chosen": -54.10163879394531,
"logps/ref_rejected": -63.72113037109375,
"logps/rejected": -321.66033935546875,
"loss": 1.108,
"margin_dpo/margin_mean": 78.0505599975586,
"margin_dpo/margin_std": 101.43817138671875,
"step": 586
},
{
"KL/chosen_KL_mean": -194.5895538330078,
"KL/mean": -232.21359252929688,
"KL/rejected_KL_mean": -269.837646484375,
"KL/std": 132.40415954589844,
"epoch": 0.8873771730914588,
"fcm_dpo/beta": 0.0048727430403232574,
"fcm_dpo/delta": 0.040885716676712036,
"fcm_dpo/margin": 75.24810791015625,
"fcm_dpo/q_t": 0.41261202096939087,
"grad_norm": 17.464710235595703,
"learning_rate": 1.9411426473854687e-08,
"logits/chosen": 0.9062135815620422,
"logits/rejected": 0.9054749011993408,
"logps/chosen": -258.0067443847656,
"logps/ref_chosen": -63.41719436645508,
"logps/ref_rejected": -63.47003936767578,
"logps/rejected": -333.30767822265625,
"loss": 1.2118,
"margin_dpo/margin_mean": 75.24810791015625,
"margin_dpo/margin_std": 178.36077880859375,
"step": 587
},
{
"KL/chosen_KL_mean": -198.12240600585938,
"KL/mean": -241.58908081054688,
"KL/rejected_KL_mean": -285.05572509765625,
"KL/std": 132.34690856933594,
"epoch": 0.8888888888888888,
"fcm_dpo/beta": 0.004769774153828621,
"fcm_dpo/delta": -0.021612241864204407,
"fcm_dpo/margin": 86.93331909179688,
"fcm_dpo/q_t": 0.4080020487308502,
"grad_norm": 18.997106552124023,
"learning_rate": 1.890382096832699e-08,
"logits/chosen": 0.9018181562423706,
"logits/rejected": 0.8665111064910889,
"logps/chosen": -260.32342529296875,
"logps/ref_chosen": -62.20103454589844,
"logps/ref_rejected": -82.10249328613281,
"logps/rejected": -367.1582336425781,
"loss": 1.1557,
"margin_dpo/margin_mean": 86.93331146240234,
"margin_dpo/margin_std": 165.61778259277344,
"step": 588
},
{
"KL/chosen_KL_mean": -183.2729034423828,
"KL/mean": -228.86248779296875,
"KL/rejected_KL_mean": -274.45208740234375,
"KL/std": 124.7643051147461,
"epoch": 0.890400604686319,
"fcm_dpo/beta": 0.004779135808348656,
"fcm_dpo/delta": -0.04623799026012421,
"fcm_dpo/margin": 91.17916870117188,
"fcm_dpo/q_t": 0.39753201603889465,
"grad_norm": 11.72799015045166,
"learning_rate": 1.840267971970344e-08,
"logits/chosen": 0.8504548668861389,
"logits/rejected": 0.8245229721069336,
"logps/chosen": -239.98651123046875,
"logps/ref_chosen": -56.71361541748047,
"logps/ref_rejected": -76.7366943359375,
"logps/rejected": -351.1887512207031,
"loss": 1.0765,
"margin_dpo/margin_mean": 91.17916870117188,
"margin_dpo/margin_std": 120.0302734375,
"step": 589
},
{
"KL/chosen_KL_mean": -199.25860595703125,
"KL/mean": -239.832275390625,
"KL/rejected_KL_mean": -280.40594482421875,
"KL/std": 113.57868957519531,
"epoch": 0.891912320483749,
"fcm_dpo/beta": 0.00477856770157814,
"fcm_dpo/delta": 0.012983638793230057,
"fcm_dpo/margin": 81.1473388671875,
"fcm_dpo/q_t": 0.4115731716156006,
"grad_norm": 17.637571334838867,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 0.7954062819480896,
"logits/rejected": 0.7640959620475769,
"logps/chosen": -265.7724304199219,
"logps/ref_chosen": -66.5138168334961,
"logps/ref_rejected": -85.70820617675781,
"logps/rejected": -366.1141357421875,
"loss": 1.1162,
"margin_dpo/margin_mean": 81.1473388671875,
"margin_dpo/margin_std": 120.72219848632812,
"step": 590
},
{
"KL/chosen_KL_mean": -169.27191162109375,
"KL/mean": -223.61370849609375,
"KL/rejected_KL_mean": -277.95550537109375,
"KL/std": 137.65127563476562,
"epoch": 0.8934240362811792,
"fcm_dpo/beta": 0.004203197546303272,
"fcm_dpo/delta": -0.08832882344722748,
"fcm_dpo/margin": 108.68359375,
"fcm_dpo/q_t": 0.39565524458885193,
"grad_norm": 12.918521881103516,
"learning_rate": 1.7419845883949098e-08,
"logits/chosen": 0.9233525991439819,
"logits/rejected": 0.8787599802017212,
"logps/chosen": -229.96908569335938,
"logps/ref_chosen": -60.697181701660156,
"logps/ref_rejected": -86.12278747558594,
"logps/rejected": -364.07830810546875,
"loss": 1.1002,
"margin_dpo/margin_mean": 108.68359375,
"margin_dpo/margin_std": 169.64715576171875,
"step": 591
},
{
"KL/chosen_KL_mean": -185.65309143066406,
"KL/mean": -226.5780029296875,
"KL/rejected_KL_mean": -267.5029296875,
"KL/std": 115.59219360351562,
"epoch": 0.8949357520786092,
"fcm_dpo/beta": 0.004358598962426186,
"fcm_dpo/delta": 0.05125339329242706,
"fcm_dpo/margin": 81.84980773925781,
"fcm_dpo/q_t": 0.42204612493515015,
"grad_norm": 13.795918464660645,
"learning_rate": 1.6938180788793556e-08,
"logits/chosen": 0.8985134363174438,
"logits/rejected": 0.8028838634490967,
"logps/chosen": -236.89041137695312,
"logps/ref_chosen": -51.237327575683594,
"logps/ref_rejected": -81.60242462158203,
"logps/rejected": -349.1053466796875,
"loss": 1.1411,
"margin_dpo/margin_mean": 81.84980010986328,
"margin_dpo/margin_std": 132.00807189941406,
"step": 592
},
{
"KL/chosen_KL_mean": -152.5126953125,
"KL/mean": -199.21270751953125,
"KL/rejected_KL_mean": -245.91268920898438,
"KL/std": 117.2364501953125,
"epoch": 0.8964474678760394,
"fcm_dpo/beta": 0.004420733079314232,
"fcm_dpo/delta": -0.017015092074871063,
"fcm_dpo/margin": 93.39999389648438,
"fcm_dpo/q_t": 0.40301674604415894,
"grad_norm": 15.768633842468262,
"learning_rate": 1.6463034933723336e-08,
"logits/chosen": 0.8698031902313232,
"logits/rejected": 0.7922099232673645,
"logps/chosen": -194.59271240234375,
"logps/ref_chosen": -42.08000183105469,
"logps/ref_rejected": -68.47499084472656,
"logps/rejected": -314.3876953125,
"loss": 1.0992,
"margin_dpo/margin_mean": 93.39999389648438,
"margin_dpo/margin_std": 134.39059448242188,
"step": 593
},
{
"KL/chosen_KL_mean": -196.26040649414062,
"KL/mean": -233.34219360351562,
"KL/rejected_KL_mean": -270.42401123046875,
"KL/std": 118.11385345458984,
"epoch": 0.8979591836734694,
"fcm_dpo/beta": 0.004671279340982437,
"fcm_dpo/delta": 0.06467173248529434,
"fcm_dpo/margin": 74.16361999511719,
"fcm_dpo/q_t": 0.42175400257110596,
"grad_norm": 13.117592811584473,
"learning_rate": 1.5994421609589385e-08,
"logits/chosen": 0.8108823299407959,
"logits/rejected": 0.8017544746398926,
"logps/chosen": -259.9190673828125,
"logps/ref_chosen": -63.658668518066406,
"logps/ref_rejected": -70.35597229003906,
"logps/rejected": -340.77996826171875,
"loss": 1.1458,
"margin_dpo/margin_mean": 74.16361999511719,
"margin_dpo/margin_std": 120.93412780761719,
"step": 594
},
{
"KL/chosen_KL_mean": -179.6163330078125,
"KL/mean": -230.7767791748047,
"KL/rejected_KL_mean": -281.937255859375,
"KL/std": 127.30757141113281,
"epoch": 0.8994708994708994,
"fcm_dpo/beta": 0.004537756554782391,
"fcm_dpo/delta": -0.08347880840301514,
"fcm_dpo/margin": 102.32089233398438,
"fcm_dpo/q_t": 0.3941081464290619,
"grad_norm": 10.797310829162598,
"learning_rate": 1.553235392451377e-08,
"logits/chosen": 0.892518162727356,
"logits/rejected": 0.8190580606460571,
"logps/chosen": -235.8350830078125,
"logps/ref_chosen": -56.21875762939453,
"logps/ref_rejected": -83.95773315429688,
"logps/rejected": -365.8949890136719,
"loss": 1.09,
"margin_dpo/margin_mean": 102.32089233398438,
"margin_dpo/margin_std": 156.1725616455078,
"step": 595
},
{
"KL/chosen_KL_mean": -209.66781616210938,
"KL/mean": -228.1407470703125,
"KL/rejected_KL_mean": -246.61367797851562,
"KL/std": 114.44010925292969,
"epoch": 0.9009826152683296,
"fcm_dpo/beta": 0.004751176573336124,
"fcm_dpo/delta": 0.08440345525741577,
"fcm_dpo/margin": 36.94586181640625,
"fcm_dpo/q_t": 0.46237796545028687,
"grad_norm": 15.789997100830078,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 0.7922879457473755,
"logits/rejected": 0.8138083219528198,
"logps/chosen": -278.148681640625,
"logps/ref_chosen": -68.48088073730469,
"logps/ref_rejected": -61.732967376708984,
"logps/rejected": -308.34661865234375,
"loss": 1.2976,
"margin_dpo/margin_mean": 36.945858001708984,
"margin_dpo/margin_std": 119.91374206542969,
"step": 596
},
{
"KL/chosen_KL_mean": -160.72445678710938,
"KL/mean": -199.198486328125,
"KL/rejected_KL_mean": -237.67250061035156,
"KL/std": 101.29132080078125,
"epoch": 0.9024943310657596,
"fcm_dpo/beta": 0.004925125278532505,
"fcm_dpo/delta": 0.023953083902597427,
"fcm_dpo/margin": 76.94803619384766,
"fcm_dpo/q_t": 0.41399139165878296,
"grad_norm": 11.786072731018066,
"learning_rate": 1.4627906988186111e-08,
"logits/chosen": 0.8552061319351196,
"logits/rejected": 0.8432618975639343,
"logps/chosen": -209.58197021484375,
"logps/ref_chosen": -48.85750961303711,
"logps/ref_rejected": -55.068084716796875,
"logps/rejected": -292.7406005859375,
"loss": 1.1202,
"margin_dpo/margin_mean": 76.94802856445312,
"margin_dpo/margin_std": 115.46106719970703,
"step": 597
},
{
"KL/chosen_KL_mean": -208.59335327148438,
"KL/mean": -235.1393280029297,
"KL/rejected_KL_mean": -261.685302734375,
"KL/std": 123.59971618652344,
"epoch": 0.9040060468631897,
"fcm_dpo/beta": 0.004996468313038349,
"fcm_dpo/delta": 0.052093617618083954,
"fcm_dpo/margin": 53.09196853637695,
"fcm_dpo/q_t": 0.4427511692047119,
"grad_norm": 13.579572677612305,
"learning_rate": 1.4185553036259095e-08,
"logits/chosen": 0.856338620185852,
"logits/rejected": 0.7920749187469482,
"logps/chosen": -267.4804992675781,
"logps/ref_chosen": -58.88715362548828,
"logps/ref_rejected": -81.43145751953125,
"logps/rejected": -343.11676025390625,
"loss": 1.2382,
"margin_dpo/margin_mean": 53.09196472167969,
"margin_dpo/margin_std": 130.64337158203125,
"step": 598
},
{
"KL/chosen_KL_mean": -209.20242309570312,
"KL/mean": -238.07550048828125,
"KL/rejected_KL_mean": -266.94854736328125,
"KL/std": 117.22828674316406,
"epoch": 0.9055177626606198,
"fcm_dpo/beta": 0.005158504005521536,
"fcm_dpo/delta": 0.012377151288092136,
"fcm_dpo/margin": 57.74616241455078,
"fcm_dpo/q_t": 0.43443089723587036,
"grad_norm": 17.706785202026367,
"learning_rate": 1.3749795321332885e-08,
"logits/chosen": 0.9016659259796143,
"logits/rejected": 0.869124174118042,
"logps/chosen": -266.80963134765625,
"logps/ref_chosen": -57.60719299316406,
"logps/ref_rejected": -71.80469512939453,
"logps/rejected": -338.75323486328125,
"loss": 1.2257,
"margin_dpo/margin_mean": 57.74616241455078,
"margin_dpo/margin_std": 135.96075439453125,
"step": 599
},
{
"KL/chosen_KL_mean": -190.96435546875,
"KL/mean": -227.06735229492188,
"KL/rejected_KL_mean": -263.1703186035156,
"KL/std": 130.8008575439453,
"epoch": 0.9070294784580499,
"fcm_dpo/beta": 0.005027216859161854,
"fcm_dpo/delta": -0.06495367735624313,
"fcm_dpo/margin": 72.2059555053711,
"fcm_dpo/q_t": 0.4207063317298889,
"grad_norm": 18.09197998046875,
"learning_rate": 1.3320646032487393e-08,
"logits/chosen": 0.8972000479698181,
"logits/rejected": 0.8484115600585938,
"logps/chosen": -249.40667724609375,
"logps/ref_chosen": -58.44231414794922,
"logps/ref_rejected": -83.64639282226562,
"logps/rejected": -346.81671142578125,
"loss": 1.1778,
"margin_dpo/margin_mean": 72.2059555053711,
"margin_dpo/margin_std": 141.4042205810547,
"step": 600
},
{
"KL/chosen_KL_mean": -171.49896240234375,
"KL/mean": -221.0233154296875,
"KL/rejected_KL_mean": -270.5477294921875,
"KL/std": 137.1717529296875,
"epoch": 0.90854119425548,
"fcm_dpo/beta": 0.004522332921624184,
"fcm_dpo/delta": -0.06632857024669647,
"fcm_dpo/margin": 99.04876708984375,
"fcm_dpo/q_t": 0.3979038596153259,
"grad_norm": 11.509042739868164,
"learning_rate": 1.2898117173950868e-08,
"logits/chosen": 0.8533494472503662,
"logits/rejected": 0.7871492505073547,
"logps/chosen": -227.09327697753906,
"logps/ref_chosen": -55.59432601928711,
"logps/ref_rejected": -83.68630981445312,
"logps/rejected": -354.2340393066406,
"loss": 1.1023,
"margin_dpo/margin_mean": 99.04876708984375,
"margin_dpo/margin_std": 156.123291015625,
"step": 601
},
{
"KL/chosen_KL_mean": -161.41921997070312,
"KL/mean": -205.10360717773438,
"KL/rejected_KL_mean": -248.7879638671875,
"KL/std": 117.01727294921875,
"epoch": 0.91005291005291,
"fcm_dpo/beta": 0.00461677648127079,
"fcm_dpo/delta": -0.00556892529129982,
"fcm_dpo/margin": 87.36872863769531,
"fcm_dpo/q_t": 0.4065745770931244,
"grad_norm": 13.123578071594238,
"learning_rate": 1.2482220564763667e-08,
"logits/chosen": 0.8886693716049194,
"logits/rejected": 0.8638613224029541,
"logps/chosen": -217.76841735839844,
"logps/ref_chosen": -56.349185943603516,
"logps/ref_rejected": -71.9959716796875,
"logps/rejected": -320.783935546875,
"loss": 1.1009,
"margin_dpo/margin_mean": 87.36872863769531,
"margin_dpo/margin_std": 123.98216247558594,
"step": 602
},
{
"KL/chosen_KL_mean": -178.2960205078125,
"KL/mean": -222.55340576171875,
"KL/rejected_KL_mean": -266.810791015625,
"KL/std": 118.53578186035156,
"epoch": 0.9115646258503401,
"fcm_dpo/beta": 0.004479277413338423,
"fcm_dpo/delta": 0.00364796444773674,
"fcm_dpo/margin": 88.51475524902344,
"fcm_dpo/q_t": 0.41039514541625977,
"grad_norm": 11.956192970275879,
"learning_rate": 1.2072967838448051e-08,
"logits/chosen": 0.8445330858230591,
"logits/rejected": 0.8046483993530273,
"logps/chosen": -231.46441650390625,
"logps/ref_chosen": -53.16838836669922,
"logps/ref_rejected": -73.8604736328125,
"logps/rejected": -340.6712646484375,
"loss": 1.1182,
"margin_dpo/margin_mean": 88.51475524902344,
"margin_dpo/margin_std": 137.9913787841797,
"step": 603
},
{
"KL/chosen_KL_mean": -175.42764282226562,
"KL/mean": -211.24090576171875,
"KL/rejected_KL_mean": -247.05416870117188,
"KL/std": 111.7984848022461,
"epoch": 0.9130763416477702,
"fcm_dpo/beta": 0.004825977608561516,
"fcm_dpo/delta": 0.0639285072684288,
"fcm_dpo/margin": 71.62651062011719,
"fcm_dpo/q_t": 0.4246947765350342,
"grad_norm": 13.927318572998047,
"learning_rate": 1.1670370442682459e-08,
"logits/chosen": 0.8277266025543213,
"logits/rejected": 0.8371890783309937,
"logps/chosen": -248.0770721435547,
"logps/ref_chosen": -72.64942169189453,
"logps/ref_rejected": -69.8792724609375,
"logps/rejected": -316.9334411621094,
"loss": 1.1873,
"margin_dpo/margin_mean": 71.62651062011719,
"margin_dpo/margin_std": 147.93710327148438,
"step": 604
},
{
"KL/chosen_KL_mean": -195.2484588623047,
"KL/mean": -235.98434448242188,
"KL/rejected_KL_mean": -276.7202453613281,
"KL/std": 116.7174301147461,
"epoch": 0.9145880574452003,
"fcm_dpo/beta": 0.0048104869201779366,
"fcm_dpo/delta": 0.009344127029180527,
"fcm_dpo/margin": 81.47175598144531,
"fcm_dpo/q_t": 0.41186654567718506,
"grad_norm": 15.041853904724121,
"learning_rate": 1.1274439638981532e-08,
"logits/chosen": 0.9091506004333496,
"logits/rejected": 0.8648948669433594,
"logps/chosen": -256.8612976074219,
"logps/ref_chosen": -61.61284637451172,
"logps/ref_rejected": -79.34398651123047,
"logps/rejected": -356.064208984375,
"loss": 1.1374,
"margin_dpo/margin_mean": 81.47175598144531,
"margin_dpo/margin_std": 138.40643310546875,
"step": 605
},
{
"KL/chosen_KL_mean": -171.4249725341797,
"KL/mean": -221.96023559570312,
"KL/rejected_KL_mean": -272.4954833984375,
"KL/std": 122.88982391357422,
"epoch": 0.9160997732426304,
"fcm_dpo/beta": 0.004657290410250425,
"fcm_dpo/delta": -0.09221285581588745,
"fcm_dpo/margin": 101.07051086425781,
"fcm_dpo/q_t": 0.3912612795829773,
"grad_norm": 14.664449691772461,
"learning_rate": 1.0885186502381016e-08,
"logits/chosen": 0.8313355445861816,
"logits/rejected": 0.7794297933578491,
"logps/chosen": -225.8892059326172,
"logps/ref_chosen": -54.46424102783203,
"logps/ref_rejected": -79.62708282470703,
"logps/rejected": -352.12255859375,
"loss": 1.0671,
"margin_dpo/margin_mean": 101.07051086425781,
"margin_dpo/margin_std": 137.8955078125,
"step": 606
},
{
"KL/chosen_KL_mean": -189.69970703125,
"KL/mean": -232.14910888671875,
"KL/rejected_KL_mean": -274.5985107421875,
"KL/std": 117.85604858398438,
"epoch": 0.9176114890400605,
"fcm_dpo/beta": 0.0040831370279192924,
"fcm_dpo/delta": -0.0878070592880249,
"fcm_dpo/margin": 84.89881896972656,
"fcm_dpo/q_t": 0.41531041264533997,
"grad_norm": 10.932661056518555,
"learning_rate": 1.0502621921127774e-08,
"logits/chosen": 0.8385425806045532,
"logits/rejected": 0.8218169212341309,
"logps/chosen": -252.56057739257812,
"logps/ref_chosen": -62.86086654663086,
"logps/ref_rejected": -72.5501937866211,
"logps/rejected": -347.1487121582031,
"loss": 1.1455,
"margin_dpo/margin_mean": 84.89881896972656,
"margin_dpo/margin_std": 132.09307861328125,
"step": 607
},
{
"KL/chosen_KL_mean": -192.0813446044922,
"KL/mean": -239.98777770996094,
"KL/rejected_KL_mean": -287.8941955566406,
"KL/std": 133.00662231445312,
"epoch": 0.9191232048374905,
"fcm_dpo/beta": 0.004131226800382137,
"fcm_dpo/delta": 0.004865831695497036,
"fcm_dpo/margin": 95.81285858154297,
"fcm_dpo/q_t": 0.4098547101020813,
"grad_norm": 11.770204544067383,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 0.8204215168952942,
"logits/rejected": 0.756758451461792,
"logps/chosen": -255.26206970214844,
"logps/ref_chosen": -63.18071746826172,
"logps/ref_rejected": -99.15888214111328,
"logps/rejected": -387.0531005859375,
"loss": 1.1027,
"margin_dpo/margin_mean": 95.81285095214844,
"margin_dpo/margin_std": 135.89065551757812,
"step": 608
},
{
"KL/chosen_KL_mean": -171.8243408203125,
"KL/mean": -219.75531005859375,
"KL/rejected_KL_mean": -267.686279296875,
"KL/std": 114.76544952392578,
"epoch": 0.9206349206349206,
"fcm_dpo/beta": 0.0040517146699130535,
"fcm_dpo/delta": 0.01250215619802475,
"fcm_dpo/margin": 95.86190795898438,
"fcm_dpo/q_t": 0.40848666429519653,
"grad_norm": 11.491697311401367,
"learning_rate": 9.757601041885694e-09,
"logits/chosen": 0.9150640368461609,
"logits/rejected": 0.8839371800422668,
"logps/chosen": -220.44757080078125,
"logps/ref_chosen": -48.62322235107422,
"logps/ref_rejected": -68.28271484375,
"logps/rejected": -335.968994140625,
"loss": 1.0908,
"margin_dpo/margin_mean": 95.8619155883789,
"margin_dpo/margin_std": 112.57235717773438,
"step": 609
},
{
"KL/chosen_KL_mean": -191.83123779296875,
"KL/mean": -234.95062255859375,
"KL/rejected_KL_mean": -278.0699768066406,
"KL/std": 130.08535766601562,
"epoch": 0.9221466364323507,
"fcm_dpo/beta": 0.004214122891426086,
"fcm_dpo/delta": 0.04403061419725418,
"fcm_dpo/margin": 86.23873138427734,
"fcm_dpo/q_t": 0.41813790798187256,
"grad_norm": 13.131653785705566,
"learning_rate": 9.395165583732379e-09,
"logits/chosen": 0.8536557555198669,
"logits/rejected": 0.8505412340164185,
"logps/chosen": -264.4963684082031,
"logps/ref_chosen": -72.66513061523438,
"logps/ref_rejected": -87.15310668945312,
"logps/rejected": -365.22308349609375,
"loss": 1.1557,
"margin_dpo/margin_mean": 86.23873138427734,
"margin_dpo/margin_std": 151.37991333007812,
"step": 610
},
{
"KL/chosen_KL_mean": -172.49746704101562,
"KL/mean": -208.9520721435547,
"KL/rejected_KL_mean": -245.40664672851562,
"KL/std": 112.63088989257812,
"epoch": 0.9236583522297808,
"fcm_dpo/beta": 0.004568580538034439,
"fcm_dpo/delta": 0.08063990622758865,
"fcm_dpo/margin": 72.90918731689453,
"fcm_dpo/q_t": 0.4252277612686157,
"grad_norm": 12.904760360717773,
"learning_rate": 9.03946036001449e-09,
"logits/chosen": 0.9144739508628845,
"logits/rejected": 0.8739849925041199,
"logps/chosen": -220.80606079101562,
"logps/ref_chosen": -48.30857849121094,
"logps/ref_rejected": -70.6141128540039,
"logps/rejected": -316.020751953125,
"loss": 1.1462,
"margin_dpo/margin_mean": 72.90919494628906,
"margin_dpo/margin_std": 113.95925903320312,
"step": 611
},
{
"KL/chosen_KL_mean": -189.24465942382812,
"KL/mean": -245.61618041992188,
"KL/rejected_KL_mean": -301.9877014160156,
"KL/std": 120.45782470703125,
"epoch": 0.9251700680272109,
"fcm_dpo/beta": 0.004316345788538456,
"fcm_dpo/delta": -0.11510652303695679,
"fcm_dpo/margin": 112.74305725097656,
"fcm_dpo/q_t": 0.3831419050693512,
"grad_norm": 11.220308303833008,
"learning_rate": 8.690495320571839e-09,
"logits/chosen": 0.7996529936790466,
"logits/rejected": 0.7398805618286133,
"logps/chosen": -250.47622680664062,
"logps/ref_chosen": -61.23155975341797,
"logps/ref_rejected": -94.37979888916016,
"logps/rejected": -396.36749267578125,
"loss": 1.0325,
"margin_dpo/margin_mean": 112.74304962158203,
"margin_dpo/margin_std": 131.59378051757812,
"step": 612
},
{
"KL/chosen_KL_mean": -167.00177001953125,
"KL/mean": -214.05615234375,
"KL/rejected_KL_mean": -261.11053466796875,
"KL/std": 132.7792510986328,
"epoch": 0.926681783824641,
"fcm_dpo/beta": 0.004251118283718824,
"fcm_dpo/delta": -0.0009491164237260818,
"fcm_dpo/margin": 94.1087646484375,
"fcm_dpo/q_t": 0.4064924120903015,
"grad_norm": 10.253765106201172,
"learning_rate": 8.348280226706722e-09,
"logits/chosen": 0.8172672390937805,
"logits/rejected": 0.8157082796096802,
"logps/chosen": -220.98487854003906,
"logps/ref_chosen": -53.98310852050781,
"logps/ref_rejected": -58.32208251953125,
"logps/rejected": -319.4326171875,
"loss": 1.1017,
"margin_dpo/margin_mean": 94.1087646484375,
"margin_dpo/margin_std": 132.22064208984375,
"step": 613
},
{
"KL/chosen_KL_mean": -189.78424072265625,
"KL/mean": -234.03488159179688,
"KL/rejected_KL_mean": -278.2854919433594,
"KL/std": 111.17167663574219,
"epoch": 0.9281934996220711,
"fcm_dpo/beta": 0.004374989774078131,
"fcm_dpo/delta": 0.009936317801475525,
"fcm_dpo/margin": 88.50125122070312,
"fcm_dpo/q_t": 0.40952184796333313,
"grad_norm": 13.926337242126465,
"learning_rate": 8.012824650910937e-09,
"logits/chosen": 0.8528718948364258,
"logits/rejected": 0.848002016544342,
"logps/chosen": -250.02728271484375,
"logps/ref_chosen": -60.24303436279297,
"logps/ref_rejected": -72.26258850097656,
"logps/rejected": -350.548095703125,
"loss": 1.1145,
"margin_dpo/margin_mean": 88.50125885009766,
"margin_dpo/margin_std": 126.64340209960938,
"step": 614
},
{
"KL/chosen_KL_mean": -195.78854370117188,
"KL/mean": -248.58724975585938,
"KL/rejected_KL_mean": -301.38592529296875,
"KL/std": 149.01504516601562,
"epoch": 0.9297052154195011,
"fcm_dpo/beta": 0.0039672525599598885,
"fcm_dpo/delta": -0.03305935114622116,
"fcm_dpo/margin": 105.5973892211914,
"fcm_dpo/q_t": 0.4037814438343048,
"grad_norm": 11.752750396728516,
"learning_rate": 7.684137976598088e-09,
"logits/chosen": 0.8259449005126953,
"logits/rejected": 0.775432825088501,
"logps/chosen": -267.8832092285156,
"logps/ref_chosen": -72.09467315673828,
"logps/ref_rejected": -104.02980041503906,
"logps/rejected": -405.41571044921875,
"loss": 1.1085,
"margin_dpo/margin_mean": 105.59738159179688,
"margin_dpo/margin_std": 159.66317749023438,
"step": 615
},
{
"KL/chosen_KL_mean": -188.25526428222656,
"KL/mean": -227.6978759765625,
"KL/rejected_KL_mean": -267.1404724121094,
"KL/std": 137.1472930908203,
"epoch": 0.9312169312169312,
"fcm_dpo/beta": 0.00431628804653883,
"fcm_dpo/delta": 0.0716884434223175,
"fcm_dpo/margin": 78.88520812988281,
"fcm_dpo/q_t": 0.42356163263320923,
"grad_norm": 11.668061256408691,
"learning_rate": 7.36222939784098e-09,
"logits/chosen": 0.8835878968238831,
"logits/rejected": 0.8267635107040405,
"logps/chosen": -246.78598022460938,
"logps/ref_chosen": -58.530723571777344,
"logps/ref_rejected": -75.48025512695312,
"logps/rejected": -342.6207275390625,
"loss": 1.1449,
"margin_dpo/margin_mean": 78.88520812988281,
"margin_dpo/margin_std": 126.27625274658203,
"step": 616
},
{
"KL/chosen_KL_mean": -196.17718505859375,
"KL/mean": -233.40261840820312,
"KL/rejected_KL_mean": -270.6280517578125,
"KL/std": 124.0031967163086,
"epoch": 0.9327286470143613,
"fcm_dpo/beta": 0.0042549436911940575,
"fcm_dpo/delta": -0.03333226591348648,
"fcm_dpo/margin": 74.45088195800781,
"fcm_dpo/q_t": 0.4275980591773987,
"grad_norm": 15.777302742004395,
"learning_rate": 7.047107919114586e-09,
"logits/chosen": 0.8467553853988647,
"logits/rejected": 0.8062554597854614,
"logps/chosen": -253.78585815429688,
"logps/ref_chosen": -57.608673095703125,
"logps/ref_rejected": -81.22109985351562,
"logps/rejected": -351.8491516113281,
"loss": 1.1771,
"margin_dpo/margin_mean": 74.45087432861328,
"margin_dpo/margin_std": 135.0968475341797,
"step": 617
},
{
"KL/chosen_KL_mean": -183.11309814453125,
"KL/mean": -228.11277770996094,
"KL/rejected_KL_mean": -273.1124572753906,
"KL/std": 136.9832763671875,
"epoch": 0.9342403628117913,
"fcm_dpo/beta": 0.0042600324377417564,
"fcm_dpo/delta": 0.019995192065835,
"fcm_dpo/margin": 89.99934387207031,
"fcm_dpo/q_t": 0.4153464436531067,
"grad_norm": 16.51608657836914,
"learning_rate": 6.738782355044048e-09,
"logits/chosen": 0.8677645921707153,
"logits/rejected": 0.7911471128463745,
"logps/chosen": -239.80905151367188,
"logps/ref_chosen": -56.69594192504883,
"logps/ref_rejected": -85.92362976074219,
"logps/rejected": -359.03607177734375,
"loss": 1.1202,
"margin_dpo/margin_mean": 89.99934387207031,
"margin_dpo/margin_std": 139.83460998535156,
"step": 618
},
{
"KL/chosen_KL_mean": -174.80682373046875,
"KL/mean": -222.34530639648438,
"KL/rejected_KL_mean": -269.8837890625,
"KL/std": 126.5418701171875,
"epoch": 0.9357520786092215,
"fcm_dpo/beta": 0.004261237569153309,
"fcm_dpo/delta": -0.0073447078466415405,
"fcm_dpo/margin": 95.07696533203125,
"fcm_dpo/q_t": 0.40738731622695923,
"grad_norm": 11.838799476623535,
"learning_rate": 6.437261330158206e-09,
"logits/chosen": 0.9209197759628296,
"logits/rejected": 0.8575242757797241,
"logps/chosen": -228.865234375,
"logps/ref_chosen": -54.05841827392578,
"logps/ref_rejected": -83.55493927001953,
"logps/rejected": -353.438720703125,
"loss": 1.103,
"margin_dpo/margin_mean": 95.07696533203125,
"margin_dpo/margin_std": 138.84814453125,
"step": 619
},
{
"KL/chosen_KL_mean": -183.2464599609375,
"KL/mean": -217.73406982421875,
"KL/rejected_KL_mean": -252.22164916992188,
"KL/std": 117.54043579101562,
"epoch": 0.9372637944066515,
"fcm_dpo/beta": 0.004206355661153793,
"fcm_dpo/delta": -0.04886791855096817,
"fcm_dpo/margin": 68.97517395019531,
"fcm_dpo/q_t": 0.4305534362792969,
"grad_norm": 12.827408790588379,
"learning_rate": 6.142553278648238e-09,
"logits/chosen": 0.8646059036254883,
"logits/rejected": 0.8682705163955688,
"logps/chosen": -246.61618041992188,
"logps/ref_chosen": -63.36971664428711,
"logps/ref_rejected": -65.68269348144531,
"logps/rejected": -317.9043273925781,
"loss": 1.194,
"margin_dpo/margin_mean": 68.97517395019531,
"margin_dpo/margin_std": 122.68234252929688,
"step": 620
},
{
"KL/chosen_KL_mean": -176.3372344970703,
"KL/mean": -221.6025848388672,
"KL/rejected_KL_mean": -266.867919921875,
"KL/std": 131.4630584716797,
"epoch": 0.9387755102040817,
"fcm_dpo/beta": 0.004246983677148819,
"fcm_dpo/delta": 0.016644608229398727,
"fcm_dpo/margin": 90.53071594238281,
"fcm_dpo/q_t": 0.4165423810482025,
"grad_norm": 12.35801887512207,
"learning_rate": 5.854666444131934e-09,
"logits/chosen": 0.8844429850578308,
"logits/rejected": 0.7948249578475952,
"logps/chosen": -228.658447265625,
"logps/ref_chosen": -52.321224212646484,
"logps/ref_rejected": -88.09001159667969,
"logps/rejected": -354.95794677734375,
"loss": 1.1435,
"margin_dpo/margin_mean": 90.53071594238281,
"margin_dpo/margin_std": 158.0870361328125,
"step": 621
},
{
"KL/chosen_KL_mean": -194.33999633789062,
"KL/mean": -236.56668090820312,
"KL/rejected_KL_mean": -278.7933349609375,
"KL/std": 134.55697631835938,
"epoch": 0.9402872260015117,
"fcm_dpo/beta": 0.004374613985419273,
"fcm_dpo/delta": 0.034500494599342346,
"fcm_dpo/margin": 84.45333862304688,
"fcm_dpo/q_t": 0.41781359910964966,
"grad_norm": 14.24582576751709,
"learning_rate": 5.573608879422875e-09,
"logits/chosen": 0.8124125003814697,
"logits/rejected": 0.7737932205200195,
"logps/chosen": -254.20545959472656,
"logps/ref_chosen": -59.86545944213867,
"logps/ref_rejected": -81.86668395996094,
"logps/rejected": -360.6600341796875,
"loss": 1.1388,
"margin_dpo/margin_mean": 84.45333862304688,
"margin_dpo/margin_std": 139.4012451171875,
"step": 622
},
{
"KL/chosen_KL_mean": -206.7667236328125,
"KL/mean": -251.3427276611328,
"KL/rejected_KL_mean": -295.9187316894531,
"KL/std": 141.90228271484375,
"epoch": 0.9417989417989417,
"fcm_dpo/beta": 0.004265275783836842,
"fcm_dpo/delta": 0.02134121209383011,
"fcm_dpo/margin": 89.15202331542969,
"fcm_dpo/q_t": 0.41413989663124084,
"grad_norm": 13.33872127532959,
"learning_rate": 5.299388446305342e-09,
"logits/chosen": 0.8636614680290222,
"logits/rejected": 0.8233832120895386,
"logps/chosen": -274.13519287109375,
"logps/ref_chosen": -67.36846160888672,
"logps/ref_rejected": -82.02733612060547,
"logps/rejected": -377.9460754394531,
"loss": 1.1384,
"margin_dpo/margin_mean": 89.15202331542969,
"margin_dpo/margin_std": 147.57623291015625,
"step": 623
},
{
"KL/chosen_KL_mean": -176.5563201904297,
"KL/mean": -221.74380493164062,
"KL/rejected_KL_mean": -266.9313049316406,
"KL/std": 130.1385498046875,
"epoch": 0.9433106575963719,
"fcm_dpo/beta": 0.004365057684481144,
"fcm_dpo/delta": 0.005921470001339912,
"fcm_dpo/margin": 90.37495422363281,
"fcm_dpo/q_t": 0.41168880462646484,
"grad_norm": 15.640230178833008,
"learning_rate": 5.03201281531429e-09,
"logits/chosen": 0.8767238855361938,
"logits/rejected": 0.7973580360412598,
"logps/chosen": -227.58287048339844,
"logps/ref_chosen": -51.02655029296875,
"logps/ref_rejected": -76.49203491210938,
"logps/rejected": -343.42333984375,
"loss": 1.1173,
"margin_dpo/margin_mean": 90.37496185302734,
"margin_dpo/margin_std": 139.69216918945312,
"step": 624
},
{
"KL/chosen_KL_mean": -175.95306396484375,
"KL/mean": -212.92311096191406,
"KL/rejected_KL_mean": -249.89315795898438,
"KL/std": 124.17606353759766,
"epoch": 0.9448223733938019,
"fcm_dpo/beta": 0.00467075128108263,
"fcm_dpo/delta": 0.06579786539077759,
"fcm_dpo/margin": 73.94007873535156,
"fcm_dpo/q_t": 0.4268485903739929,
"grad_norm": 13.2764310836792,
"learning_rate": 4.7714894655209174e-09,
"logits/chosen": 0.9225313663482666,
"logits/rejected": 0.8460739850997925,
"logps/chosen": -230.1606903076172,
"logps/ref_chosen": -54.20761489868164,
"logps/ref_rejected": -84.93669128417969,
"logps/rejected": -334.829833984375,
"loss": 1.1812,
"margin_dpo/margin_mean": 73.94007873535156,
"margin_dpo/margin_std": 149.19314575195312,
"step": 625
},
{
"KL/chosen_KL_mean": -166.25491333007812,
"KL/mean": -217.4595184326172,
"KL/rejected_KL_mean": -268.66412353515625,
"KL/std": 134.90798950195312,
"epoch": 0.9463340891912321,
"fcm_dpo/beta": 0.004452715627849102,
"fcm_dpo/delta": -0.07368428260087967,
"fcm_dpo/margin": 102.40919494628906,
"fcm_dpo/q_t": 0.3990449607372284,
"grad_norm": 14.270200729370117,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 0.941727876663208,
"logits/rejected": 0.8290482759475708,
"logps/chosen": -211.3169403076172,
"logps/ref_chosen": -45.06201934814453,
"logps/ref_rejected": -89.66368103027344,
"logps/rejected": -358.3277893066406,
"loss": 1.0958,
"margin_dpo/margin_mean": 102.40919494628906,
"margin_dpo/margin_std": 164.34068298339844,
"step": 626
},
{
"KL/chosen_KL_mean": -190.93905639648438,
"KL/mean": -245.65646362304688,
"KL/rejected_KL_mean": -300.3738708496094,
"KL/std": 140.89553833007812,
"epoch": 0.9478458049886621,
"fcm_dpo/beta": 0.004127138294279575,
"fcm_dpo/delta": -0.0688527449965477,
"fcm_dpo/margin": 109.43482208251953,
"fcm_dpo/q_t": 0.3921513855457306,
"grad_norm": 11.92895221710205,
"learning_rate": 4.271028567242818e-09,
"logits/chosen": 0.8075392246246338,
"logits/rejected": 0.7088248133659363,
"logps/chosen": -249.7301025390625,
"logps/ref_chosen": -58.791053771972656,
"logps/ref_rejected": -94.90802001953125,
"logps/rejected": -395.2818908691406,
"loss": 1.0566,
"margin_dpo/margin_mean": 109.43482208251953,
"margin_dpo/margin_std": 133.880126953125,
"step": 627
},
{
"KL/chosen_KL_mean": -176.84475708007812,
"KL/mean": -227.326416015625,
"KL/rejected_KL_mean": -277.8080749511719,
"KL/std": 130.1131591796875,
"epoch": 0.9493575207860923,
"fcm_dpo/beta": 0.0041605280712246895,
"fcm_dpo/delta": -0.029174987226724625,
"fcm_dpo/margin": 100.96332550048828,
"fcm_dpo/q_t": 0.4007333219051361,
"grad_norm": 13.848322868347168,
"learning_rate": 4.0311050177251895e-09,
"logits/chosen": 0.8667212128639221,
"logits/rejected": 0.8345490097999573,
"logps/chosen": -229.64833068847656,
"logps/ref_chosen": -52.80357360839844,
"logps/ref_rejected": -76.49468994140625,
"logps/rejected": -354.3027648925781,
"loss": 1.1171,
"margin_dpo/margin_mean": 100.96331787109375,
"margin_dpo/margin_std": 148.15411376953125,
"step": 628
},
{
"KL/chosen_KL_mean": -190.57797241210938,
"KL/mean": -226.43057250976562,
"KL/rejected_KL_mean": -262.28314208984375,
"KL/std": 119.66000366210938,
"epoch": 0.9508692365835223,
"fcm_dpo/beta": 0.004353722557425499,
"fcm_dpo/delta": 0.10218150913715363,
"fcm_dpo/margin": 71.70518493652344,
"fcm_dpo/q_t": 0.43269526958465576,
"grad_norm": 12.406708717346191,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 0.8476235270500183,
"logits/rejected": 0.8516208529472351,
"logps/chosen": -261.29547119140625,
"logps/ref_chosen": -70.71749877929688,
"logps/ref_rejected": -78.96273803710938,
"logps/rejected": -341.24591064453125,
"loss": 1.174,
"margin_dpo/margin_mean": 71.70518493652344,
"margin_dpo/margin_std": 130.66549682617188,
"step": 629
},
{
"KL/chosen_KL_mean": -175.83795166015625,
"KL/mean": -226.13722229003906,
"KL/rejected_KL_mean": -276.4364929199219,
"KL/std": 127.85799407958984,
"epoch": 0.9523809523809523,
"fcm_dpo/beta": 0.0043222480453550816,
"fcm_dpo/delta": -0.04444940388202667,
"fcm_dpo/margin": 100.59855651855469,
"fcm_dpo/q_t": 0.3986924886703491,
"grad_norm": 11.189255714416504,
"learning_rate": 3.5719052736323806e-09,
"logits/chosen": 0.8098629713058472,
"logits/rejected": 0.7818838357925415,
"logps/chosen": -232.0393524169922,
"logps/ref_chosen": -56.201412200927734,
"logps/ref_rejected": -74.69807434082031,
"logps/rejected": -351.13458251953125,
"loss": 1.0859,
"margin_dpo/margin_mean": 100.59855651855469,
"margin_dpo/margin_std": 141.52880859375,
"step": 630
},
{
"KL/chosen_KL_mean": -167.0948028564453,
"KL/mean": -227.14959716796875,
"KL/rejected_KL_mean": -287.20440673828125,
"KL/std": 138.20928955078125,
"epoch": 0.9538926681783825,
"fcm_dpo/beta": 0.0038028829731047153,
"fcm_dpo/delta": -0.08643369376659393,
"fcm_dpo/margin": 120.109619140625,
"fcm_dpo/q_t": 0.3914579153060913,
"grad_norm": 11.006467819213867,
"learning_rate": 3.352641923861144e-09,
"logits/chosen": 0.9120550155639648,
"logits/rejected": 0.8212620615959167,
"logps/chosen": -225.9154052734375,
"logps/ref_chosen": -58.82059860229492,
"logps/ref_rejected": -96.51437377929688,
"logps/rejected": -383.71881103515625,
"loss": 1.0626,
"margin_dpo/margin_mean": 120.109619140625,
"margin_dpo/margin_std": 151.4070281982422,
"step": 631
},
{
"KL/chosen_KL_mean": -169.6227264404297,
"KL/mean": -222.89845275878906,
"KL/rejected_KL_mean": -276.1741638183594,
"KL/std": 119.03538513183594,
"epoch": 0.9554043839758125,
"fcm_dpo/beta": 0.0038762884214520454,
"fcm_dpo/delta": -0.016792653128504753,
"fcm_dpo/margin": 106.55143737792969,
"fcm_dpo/q_t": 0.40256255865097046,
"grad_norm": 12.226197242736816,
"learning_rate": 3.140277830901428e-09,
"logits/chosen": 0.8677657842636108,
"logits/rejected": 0.8573723435401917,
"logps/chosen": -228.40878295898438,
"logps/ref_chosen": -58.786048889160156,
"logps/ref_rejected": -67.21923828125,
"logps/rejected": -343.3934020996094,
"loss": 1.0755,
"margin_dpo/margin_mean": 106.55143737792969,
"margin_dpo/margin_std": 128.8999786376953,
"step": 632
},
{
"KL/chosen_KL_mean": -166.22283935546875,
"KL/mean": -209.18299865722656,
"KL/rejected_KL_mean": -252.14317321777344,
"KL/std": 116.75459289550781,
"epoch": 0.9569160997732427,
"fcm_dpo/beta": 0.004068909212946892,
"fcm_dpo/delta": 0.05746911093592644,
"fcm_dpo/margin": 85.92031860351562,
"fcm_dpo/q_t": 0.42180460691452026,
"grad_norm": 12.814493179321289,
"learning_rate": 2.9348189350335007e-09,
"logits/chosen": 0.8555501699447632,
"logits/rejected": 0.8156156539916992,
"logps/chosen": -218.35304260253906,
"logps/ref_chosen": -52.13019561767578,
"logps/ref_rejected": -67.23016357421875,
"logps/rejected": -319.3733215332031,
"loss": 1.1366,
"margin_dpo/margin_mean": 85.92031860351562,
"margin_dpo/margin_std": 131.57418823242188,
"step": 633
},
{
"KL/chosen_KL_mean": -199.20892333984375,
"KL/mean": -214.2444610595703,
"KL/rejected_KL_mean": -229.27999877929688,
"KL/std": 122.4713363647461,
"epoch": 0.9584278155706727,
"fcm_dpo/beta": 0.0040403553284704685,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 30.071069717407227,
"fcm_dpo/q_t": 0.4731084406375885,
"grad_norm": 14.749358177185059,
"learning_rate": 2.736270983384276e-09,
"logits/chosen": 0.917083203792572,
"logits/rejected": 0.9344348907470703,
"logps/chosen": -260.188720703125,
"logps/ref_chosen": -60.97979736328125,
"logps/ref_rejected": -58.50825119018555,
"logps/rejected": -287.7882385253906,
"loss": 1.3462,
"margin_dpo/margin_mean": 30.07107162475586,
"margin_dpo/margin_std": 140.06642150878906,
"step": 634
},
{
"KL/chosen_KL_mean": -216.4016876220703,
"KL/mean": -251.16989135742188,
"KL/rejected_KL_mean": -285.9380798339844,
"KL/std": 117.64241027832031,
"epoch": 0.9599395313681028,
"fcm_dpo/beta": 0.0044487882405519485,
"fcm_dpo/delta": 0.10310401022434235,
"fcm_dpo/margin": 69.5363998413086,
"fcm_dpo/q_t": 0.43169939517974854,
"grad_norm": 12.148490905761719,
"learning_rate": 2.5446395297668287e-09,
"logits/chosen": 0.7875484228134155,
"logits/rejected": 0.7464572787284851,
"logps/chosen": -282.37469482421875,
"logps/ref_chosen": -65.9730224609375,
"logps/ref_rejected": -85.61317443847656,
"logps/rejected": -371.55126953125,
"loss": 1.1925,
"margin_dpo/margin_mean": 69.5363998413086,
"margin_dpo/margin_std": 138.6806640625,
"step": 635
},
{
"KL/chosen_KL_mean": -168.04318237304688,
"KL/mean": -216.12437438964844,
"KL/rejected_KL_mean": -264.20556640625,
"KL/std": 121.20172882080078,
"epoch": 0.9614512471655329,
"fcm_dpo/beta": 0.0043456945568323135,
"fcm_dpo/delta": -0.02371588721871376,
"fcm_dpo/margin": 96.16236877441406,
"fcm_dpo/q_t": 0.40238264203071594,
"grad_norm": 9.903681755065918,
"learning_rate": 2.359929934524829e-09,
"logits/chosen": 0.885582685470581,
"logits/rejected": 0.8005257844924927,
"logps/chosen": -217.18336486816406,
"logps/ref_chosen": -49.140167236328125,
"logps/ref_rejected": -81.26971435546875,
"logps/rejected": -345.47528076171875,
"loss": 1.0856,
"margin_dpo/margin_mean": 96.16236877441406,
"margin_dpo/margin_std": 128.5138702392578,
"step": 636
},
{
"KL/chosen_KL_mean": -227.61679077148438,
"KL/mean": -262.9964599609375,
"KL/rejected_KL_mean": -298.3760986328125,
"KL/std": 125.6603775024414,
"epoch": 0.9629629629629629,
"fcm_dpo/beta": 0.004716330207884312,
"fcm_dpo/delta": 0.07630355656147003,
"fcm_dpo/margin": 70.7593002319336,
"fcm_dpo/q_t": 0.42755448818206787,
"grad_norm": 14.731769561767578,
"learning_rate": 2.1821473643827137e-09,
"logits/chosen": 0.8204032182693481,
"logits/rejected": 0.7771281599998474,
"logps/chosen": -301.3133850097656,
"logps/ref_chosen": -73.69658660888672,
"logps/ref_rejected": -83.01487731933594,
"logps/rejected": -381.3909912109375,
"loss": 1.1985,
"margin_dpo/margin_mean": 70.7593002319336,
"margin_dpo/margin_std": 152.50254821777344,
"step": 637
},
{
"KL/chosen_KL_mean": -196.76817321777344,
"KL/mean": -239.29661560058594,
"KL/rejected_KL_mean": -281.8250732421875,
"KL/std": 121.583740234375,
"epoch": 0.9644746787603931,
"fcm_dpo/beta": 0.004684511572122574,
"fcm_dpo/delta": 0.0015891995280981064,
"fcm_dpo/margin": 85.056884765625,
"fcm_dpo/q_t": 0.4112497568130493,
"grad_norm": 13.911808967590332,
"learning_rate": 2.0112967923011646e-09,
"logits/chosen": 0.8388936519622803,
"logits/rejected": 0.8055061101913452,
"logps/chosen": -259.54974365234375,
"logps/ref_chosen": -62.78158187866211,
"logps/ref_rejected": -85.40478515625,
"logps/rejected": -367.2298583984375,
"loss": 1.12,
"margin_dpo/margin_mean": 85.05690002441406,
"margin_dpo/margin_std": 135.8700714111328,
"step": 638
},
{
"KL/chosen_KL_mean": -174.95285034179688,
"KL/mean": -225.14132690429688,
"KL/rejected_KL_mean": -275.3298034667969,
"KL/std": 131.77854919433594,
"epoch": 0.9659863945578231,
"fcm_dpo/beta": 0.004515929147601128,
"fcm_dpo/delta": -0.06874261051416397,
"fcm_dpo/margin": 100.37693786621094,
"fcm_dpo/q_t": 0.3964504301548004,
"grad_norm": 15.656349182128906,
"learning_rate": 1.847382997337943e-09,
"logits/chosen": 0.8310115337371826,
"logits/rejected": 0.7619854807853699,
"logps/chosen": -228.7194366455078,
"logps/ref_chosen": -53.76658630371094,
"logps/ref_rejected": -72.30009460449219,
"logps/rejected": -347.6298828125,
"loss": 1.0799,
"margin_dpo/margin_mean": 100.37693786621094,
"margin_dpo/margin_std": 143.62814331054688,
"step": 639
},
{
"KL/chosen_KL_mean": -183.02471923828125,
"KL/mean": -226.49563598632812,
"KL/rejected_KL_mean": -269.966552734375,
"KL/std": 124.19609069824219,
"epoch": 0.9674981103552532,
"fcm_dpo/beta": 0.004421434365212917,
"fcm_dpo/delta": 0.018865177407860756,
"fcm_dpo/margin": 86.94181823730469,
"fcm_dpo/q_t": 0.413200318813324,
"grad_norm": 13.584686279296875,
"learning_rate": 1.690410564514244e-09,
"logits/chosen": 0.9080848693847656,
"logits/rejected": 0.8571711778640747,
"logps/chosen": -234.4425048828125,
"logps/ref_chosen": -51.41777801513672,
"logps/ref_rejected": -77.27879333496094,
"logps/rejected": -347.245361328125,
"loss": 1.1242,
"margin_dpo/margin_mean": 86.94182586669922,
"margin_dpo/margin_std": 136.48196411132812,
"step": 640
},
{
"KL/chosen_KL_mean": -191.9429931640625,
"KL/mean": -231.77511596679688,
"KL/rejected_KL_mean": -271.60723876953125,
"KL/std": 114.53807067871094,
"epoch": 0.9690098261526833,
"fcm_dpo/beta": 0.004588712938129902,
"fcm_dpo/delta": 0.04207627847790718,
"fcm_dpo/margin": 79.66427612304688,
"fcm_dpo/q_t": 0.416546106338501,
"grad_norm": 13.669086456298828,
"learning_rate": 1.5403838846864692e-09,
"logits/chosen": 0.8353901505470276,
"logits/rejected": 0.8291752338409424,
"logps/chosen": -262.9976501464844,
"logps/ref_chosen": -71.0546646118164,
"logps/ref_rejected": -82.2440185546875,
"logps/rejected": -353.85125732421875,
"loss": 1.1129,
"margin_dpo/margin_mean": 79.66426849365234,
"margin_dpo/margin_std": 108.70352172851562,
"step": 641
},
{
"KL/chosen_KL_mean": -200.2201690673828,
"KL/mean": -229.66928100585938,
"KL/rejected_KL_mean": -259.118408203125,
"KL/std": 112.77413940429688,
"epoch": 0.9705215419501134,
"fcm_dpo/beta": 0.004786022938787937,
"fcm_dpo/delta": 0.019887451082468033,
"fcm_dpo/margin": 58.898216247558594,
"fcm_dpo/q_t": 0.43794071674346924,
"grad_norm": 16.15554428100586,
"learning_rate": 1.3973071544233218e-09,
"logits/chosen": 0.785338282585144,
"logits/rejected": 0.8072504997253418,
"logps/chosen": -269.1494445800781,
"logps/ref_chosen": -68.92927551269531,
"logps/ref_rejected": -70.85682678222656,
"logps/rejected": -329.9752197265625,
"loss": 1.2236,
"margin_dpo/margin_mean": 58.898216247558594,
"margin_dpo/margin_std": 135.51150512695312,
"step": 642
},
{
"KL/chosen_KL_mean": -182.64639282226562,
"KL/mean": -230.23406982421875,
"KL/rejected_KL_mean": -277.82177734375,
"KL/std": 124.7582015991211,
"epoch": 0.9720332577475435,
"fcm_dpo/beta": 0.004608517978340387,
"fcm_dpo/delta": -0.04959561303257942,
"fcm_dpo/margin": 95.17536926269531,
"fcm_dpo/q_t": 0.40014487504959106,
"grad_norm": 18.453994750976562,
"learning_rate": 1.261184375888541e-09,
"logits/chosen": 0.8454148769378662,
"logits/rejected": 0.7771642208099365,
"logps/chosen": -247.95541381835938,
"logps/ref_chosen": -65.30903625488281,
"logps/ref_rejected": -83.61613464355469,
"logps/rejected": -361.43792724609375,
"loss": 1.0966,
"margin_dpo/margin_mean": 95.17538452148438,
"margin_dpo/margin_std": 143.47547912597656,
"step": 643
},
{
"KL/chosen_KL_mean": -173.703125,
"KL/mean": -203.21925354003906,
"KL/rejected_KL_mean": -232.7353973388672,
"KL/std": 107.22509765625,
"epoch": 0.9735449735449735,
"fcm_dpo/beta": 0.0045800781808793545,
"fcm_dpo/delta": 0.01119247917085886,
"fcm_dpo/margin": 59.03228759765625,
"fcm_dpo/q_t": 0.43719398975372314,
"grad_norm": 12.038887977600098,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 0.9204496145248413,
"logits/rejected": 0.8933486342430115,
"logps/chosen": -224.70571899414062,
"logps/ref_chosen": -51.002601623535156,
"logps/ref_rejected": -64.46372985839844,
"logps/rejected": -297.1991271972656,
"loss": 1.2189,
"margin_dpo/margin_mean": 59.032283782958984,
"margin_dpo/margin_std": 127.94892883300781,
"step": 644
},
{
"KL/chosen_KL_mean": -190.76394653320312,
"KL/mean": -227.95298767089844,
"KL/rejected_KL_mean": -265.14202880859375,
"KL/std": 117.67291259765625,
"epoch": 0.9750566893424036,
"fcm_dpo/beta": 0.004811743274331093,
"fcm_dpo/delta": 0.05072278529405594,
"fcm_dpo/margin": 74.37810516357422,
"fcm_dpo/q_t": 0.41881194710731506,
"grad_norm": 16.8271541595459,
"learning_rate": 1.0098157099674987e-09,
"logits/chosen": 0.831100344657898,
"logits/rejected": 0.8188281059265137,
"logps/chosen": -251.72735595703125,
"logps/ref_chosen": -60.963409423828125,
"logps/ref_rejected": -69.73353576660156,
"logps/rejected": -334.8755798339844,
"loss": 1.1414,
"margin_dpo/margin_mean": 74.37811279296875,
"margin_dpo/margin_std": 121.78832244873047,
"step": 645
},
{
"KL/chosen_KL_mean": -207.34078979492188,
"KL/mean": -244.42474365234375,
"KL/rejected_KL_mean": -281.5086975097656,
"KL/std": 142.02871704101562,
"epoch": 0.9765684051398337,
"fcm_dpo/beta": 0.004996123258024454,
"fcm_dpo/delta": 0.03588254749774933,
"fcm_dpo/margin": 74.16790771484375,
"fcm_dpo/q_t": 0.4193630516529083,
"grad_norm": 13.366632461547852,
"learning_rate": 8.945768539031783e-10,
"logits/chosen": 0.8661332130432129,
"logits/rejected": 0.8158410787582397,
"logps/chosen": -269.630859375,
"logps/ref_chosen": -62.290069580078125,
"logps/ref_rejected": -85.54812622070312,
"logps/rejected": -367.05682373046875,
"loss": 1.1618,
"margin_dpo/margin_mean": 74.16790771484375,
"margin_dpo/margin_std": 139.99423217773438,
"step": 646
},
{
"KL/chosen_KL_mean": -203.80487060546875,
"KL/mean": -262.35650634765625,
"KL/rejected_KL_mean": -320.9080810546875,
"KL/std": 138.29147338867188,
"epoch": 0.9780801209372638,
"fcm_dpo/beta": 0.004455167334526777,
"fcm_dpo/delta": -0.16519597172737122,
"fcm_dpo/margin": 117.10321044921875,
"fcm_dpo/q_t": 0.37394261360168457,
"grad_norm": 14.46107006072998,
"learning_rate": 7.863060120144316e-10,
"logits/chosen": 0.8677129745483398,
"logits/rejected": 0.7881165742874146,
"logps/chosen": -271.32073974609375,
"logps/ref_chosen": -67.515869140625,
"logps/ref_rejected": -101.50871276855469,
"logps/rejected": -422.41680908203125,
"loss": 1.016,
"margin_dpo/margin_mean": 117.10321044921875,
"margin_dpo/margin_std": 135.55255126953125,
"step": 647
},
{
"KL/chosen_KL_mean": -205.17529296875,
"KL/mean": -244.43492126464844,
"KL/rejected_KL_mean": -283.6945495605469,
"KL/std": 120.4975357055664,
"epoch": 0.9795918367346939,
"fcm_dpo/beta": 0.0043856678530573845,
"fcm_dpo/delta": 0.06681666523218155,
"fcm_dpo/margin": 78.51927185058594,
"fcm_dpo/q_t": 0.4253219962120056,
"grad_norm": 12.654325485229492,
"learning_rate": 6.850062128694045e-10,
"logits/chosen": 0.8319413661956787,
"logits/rejected": 0.7821545004844666,
"logps/chosen": -269.771240234375,
"logps/ref_chosen": -64.59593963623047,
"logps/ref_rejected": -83.384033203125,
"logps/rejected": -367.07861328125,
"loss": 1.189,
"margin_dpo/margin_mean": 78.51927185058594,
"margin_dpo/margin_std": 157.42373657226562,
"step": 648
},
{
"KL/chosen_KL_mean": -184.55213928222656,
"KL/mean": -225.38185119628906,
"KL/rejected_KL_mean": -266.2115783691406,
"KL/std": 116.33067321777344,
"epoch": 0.981103552532124,
"fcm_dpo/beta": 0.0045933909714221954,
"fcm_dpo/delta": 0.030288267880678177,
"fcm_dpo/margin": 81.659423828125,
"fcm_dpo/q_t": 0.41458964347839355,
"grad_norm": 17.702682495117188,
"learning_rate": 5.906802900412788e-10,
"logits/chosen": 0.8983275890350342,
"logits/rejected": 0.8456778526306152,
"logps/chosen": -233.86178588867188,
"logps/ref_chosen": -49.30964660644531,
"logps/ref_rejected": -73.73710632324219,
"logps/rejected": -339.94866943359375,
"loss": 1.1419,
"margin_dpo/margin_mean": 81.659423828125,
"margin_dpo/margin_std": 137.75778198242188,
"step": 649
},
{
"KL/chosen_KL_mean": -179.00872802734375,
"KL/mean": -223.29705810546875,
"KL/rejected_KL_mean": -267.58538818359375,
"KL/std": 121.32177734375,
"epoch": 0.982615268329554,
"fcm_dpo/beta": 0.0045938557013869286,
"fcm_dpo/delta": -0.009712686762213707,
"fcm_dpo/margin": 88.57667541503906,
"fcm_dpo/q_t": 0.408522367477417,
"grad_norm": 14.306544303894043,
"learning_rate": 5.033308820289184e-10,
"logits/chosen": 0.912330150604248,
"logits/rejected": 0.8635398149490356,
"logps/chosen": -234.07199096679688,
"logps/ref_chosen": -55.06325912475586,
"logps/ref_rejected": -77.39610290527344,
"logps/rejected": -344.98150634765625,
"loss": 1.1298,
"margin_dpo/margin_mean": 88.57667541503906,
"margin_dpo/margin_std": 149.25350952148438,
"step": 650
},
{
"KL/chosen_KL_mean": -176.9525146484375,
"KL/mean": -215.21630859375,
"KL/rejected_KL_mean": -253.48007202148438,
"KL/std": 118.9674301147461,
"epoch": 0.9841269841269841,
"fcm_dpo/beta": 0.004845967050641775,
"fcm_dpo/delta": 0.03382531553506851,
"fcm_dpo/margin": 76.52754211425781,
"fcm_dpo/q_t": 0.4186941683292389,
"grad_norm": 13.464821815490723,
"learning_rate": 4.2296043218295606e-10,
"logits/chosen": 0.8908314108848572,
"logits/rejected": 0.8330064415931702,
"logps/chosen": -231.01768493652344,
"logps/ref_chosen": -54.065162658691406,
"logps/ref_rejected": -77.79080200195312,
"logps/rejected": -331.2708740234375,
"loss": 1.1333,
"margin_dpo/margin_mean": 76.52754974365234,
"margin_dpo/margin_std": 123.83651733398438,
"step": 651
},
{
"KL/chosen_KL_mean": -209.56051635742188,
"KL/mean": -243.226806640625,
"KL/rejected_KL_mean": -276.89312744140625,
"KL/std": 132.98809814453125,
"epoch": 0.9856386999244142,
"fcm_dpo/beta": 0.004979660268872976,
"fcm_dpo/delta": 0.07733257114887238,
"fcm_dpo/margin": 67.33261108398438,
"fcm_dpo/q_t": 0.4313938617706299,
"grad_norm": 16.682828903198242,
"learning_rate": 3.4957118863768176e-10,
"logits/chosen": 0.8507139682769775,
"logits/rejected": 0.8146477937698364,
"logps/chosen": -273.2008056640625,
"logps/ref_chosen": -63.64030456542969,
"logps/ref_rejected": -78.86882019042969,
"logps/rejected": -355.761962890625,
"loss": 1.2273,
"margin_dpo/margin_mean": 67.33261108398438,
"margin_dpo/margin_std": 161.7130889892578,
"step": 652
},
{
"KL/chosen_KL_mean": -185.29013061523438,
"KL/mean": -227.1661376953125,
"KL/rejected_KL_mean": -269.04217529296875,
"KL/std": 115.51123046875,
"epoch": 0.9871504157218443,
"fcm_dpo/beta": 0.005149069242179394,
"fcm_dpo/delta": -0.04090055823326111,
"fcm_dpo/margin": 83.75203704833984,
"fcm_dpo/q_t": 0.4029849171638489,
"grad_norm": 16.87303352355957,
"learning_rate": 2.831652042480093e-10,
"logits/chosen": 0.8277177810668945,
"logits/rejected": 0.7922142148017883,
"logps/chosen": -246.95849609375,
"logps/ref_chosen": -61.668373107910156,
"logps/ref_rejected": -73.83012390136719,
"logps/rejected": -342.872314453125,
"loss": 1.1128,
"margin_dpo/margin_mean": 83.75202941894531,
"margin_dpo/margin_std": 134.10134887695312,
"step": 653
},
{
"KL/chosen_KL_mean": -193.46804809570312,
"KL/mean": -234.66799926757812,
"KL/rejected_KL_mean": -275.8679504394531,
"KL/std": 139.91705322265625,
"epoch": 0.9886621315192744,
"fcm_dpo/beta": 0.004517903085798025,
"fcm_dpo/delta": -0.09830736368894577,
"fcm_dpo/margin": 82.39989471435547,
"fcm_dpo/q_t": 0.4111466705799103,
"grad_norm": 13.85952377319336,
"learning_rate": 2.2374433653205016e-10,
"logits/chosen": 0.8260544538497925,
"logits/rejected": 0.7493513226509094,
"logps/chosen": -251.03631591796875,
"logps/ref_chosen": -57.568267822265625,
"logps/ref_rejected": -87.74789428710938,
"logps/rejected": -363.6158447265625,
"loss": 1.1364,
"margin_dpo/margin_mean": 82.39990234375,
"margin_dpo/margin_std": 130.38128662109375,
"step": 654
},
{
"KL/chosen_KL_mean": -152.67041015625,
"KL/mean": -211.55709838867188,
"KL/rejected_KL_mean": -270.4437561035156,
"KL/std": 120.74604797363281,
"epoch": 0.9901738473167044,
"fcm_dpo/beta": 0.0039674765430390835,
"fcm_dpo/delta": -0.10667331516742706,
"fcm_dpo/margin": 117.77336120605469,
"fcm_dpo/q_t": 0.3849404454231262,
"grad_norm": 10.699645042419434,
"learning_rate": 1.7131024761923852e-10,
"logits/chosen": 0.8190682530403137,
"logits/rejected": 0.7526806592941284,
"logps/chosen": -204.81756591796875,
"logps/ref_chosen": -52.14714813232422,
"logps/ref_rejected": -80.85014343261719,
"logps/rejected": -351.29388427734375,
"loss": 1.0359,
"margin_dpo/margin_mean": 117.77336120605469,
"margin_dpo/margin_std": 123.65576171875,
"step": 655
},
{
"KL/chosen_KL_mean": -205.341796875,
"KL/mean": -246.5940704345703,
"KL/rejected_KL_mean": -287.84637451171875,
"KL/std": 123.82130432128906,
"epoch": 0.9916855631141346,
"fcm_dpo/beta": 0.004260948859155178,
"fcm_dpo/delta": 0.05830314755439758,
"fcm_dpo/margin": 82.5046157836914,
"fcm_dpo/q_t": 0.42144423723220825,
"grad_norm": 11.265812873840332,
"learning_rate": 1.2586440420372934e-10,
"logits/chosen": 0.7942889928817749,
"logits/rejected": 0.7635731101036072,
"logps/chosen": -278.5985107421875,
"logps/ref_chosen": -73.25672912597656,
"logps/ref_rejected": -85.35127258300781,
"logps/rejected": -373.1976318359375,
"loss": 1.1449,
"margin_dpo/margin_mean": 82.50460815429688,
"margin_dpo/margin_std": 136.50827026367188,
"step": 656
},
{
"KL/chosen_KL_mean": -177.6498565673828,
"KL/mean": -230.7796173095703,
"KL/rejected_KL_mean": -283.90936279296875,
"KL/std": 136.02264404296875,
"epoch": 0.9931972789115646,
"fcm_dpo/beta": 0.004170597996562719,
"fcm_dpo/delta": -0.055298324674367905,
"fcm_dpo/margin": 106.25950622558594,
"fcm_dpo/q_t": 0.3979775309562683,
"grad_norm": 10.715606689453125,
"learning_rate": 8.740807750345913e-11,
"logits/chosen": 0.9494369029998779,
"logits/rejected": 0.8862440586090088,
"logps/chosen": -227.37326049804688,
"logps/ref_chosen": -49.72339630126953,
"logps/ref_rejected": -75.1568603515625,
"logps/rejected": -359.06622314453125,
"loss": 1.0828,
"margin_dpo/margin_mean": 106.25951385498047,
"margin_dpo/margin_std": 150.11639404296875,
"step": 657
},
{
"KL/chosen_KL_mean": -190.97988891601562,
"KL/mean": -234.16209411621094,
"KL/rejected_KL_mean": -277.34429931640625,
"KL/std": 144.20794677734375,
"epoch": 0.9947089947089947,
"fcm_dpo/beta": 0.00430716760456562,
"fcm_dpo/delta": 0.027860961854457855,
"fcm_dpo/margin": 86.36438751220703,
"fcm_dpo/q_t": 0.4168539345264435,
"grad_norm": 10.777701377868652,
"learning_rate": 5.594234322453539e-11,
"logits/chosen": 0.8744544982910156,
"logits/rejected": 0.8389475345611572,
"logps/chosen": -254.02622985839844,
"logps/ref_chosen": -63.04634094238281,
"logps/ref_rejected": -83.44963073730469,
"logps/rejected": -360.7939147949219,
"loss": 1.1717,
"margin_dpo/margin_mean": 86.3643798828125,
"margin_dpo/margin_std": 166.66415405273438,
"step": 658
},
{
"KL/chosen_KL_mean": -192.81484985351562,
"KL/mean": -224.96310424804688,
"KL/rejected_KL_mean": -257.111328125,
"KL/std": 123.95234680175781,
"epoch": 0.9962207105064248,
"fcm_dpo/beta": 0.004288731142878532,
"fcm_dpo/delta": 0.0237504281103611,
"fcm_dpo/margin": 64.29650115966797,
"fcm_dpo/q_t": 0.4373660981655121,
"grad_norm": 13.88487720489502,
"learning_rate": 3.146808153123293e-11,
"logits/chosen": 0.9137390851974487,
"logits/rejected": 0.8617255687713623,
"logps/chosen": -247.89505004882812,
"logps/ref_chosen": -55.0802001953125,
"logps/ref_rejected": -71.91049194335938,
"logps/rejected": -329.0218505859375,
"loss": 1.2184,
"margin_dpo/margin_mean": 64.2965087890625,
"margin_dpo/margin_std": 141.48397827148438,
"step": 659
},
{
"KL/chosen_KL_mean": -181.04736328125,
"KL/mean": -235.65093994140625,
"KL/rejected_KL_mean": -290.2545166015625,
"KL/std": 134.4461669921875,
"epoch": 0.9977324263038548,
"fcm_dpo/beta": 0.004096397198736668,
"fcm_dpo/delta": -0.06114449352025986,
"fcm_dpo/margin": 109.20713806152344,
"fcm_dpo/q_t": 0.3970072269439697,
"grad_norm": 12.943835258483887,
"learning_rate": 1.3985977021235829e-11,
"logits/chosen": 0.947117269039154,
"logits/rejected": 0.8982895612716675,
"logps/chosen": -235.57327270507812,
"logps/ref_chosen": -54.525917053222656,
"logps/ref_rejected": -81.23604583740234,
"logps/rejected": -371.49053955078125,
"loss": 1.0676,
"margin_dpo/margin_mean": 109.20713806152344,
"margin_dpo/margin_std": 146.55783081054688,
"step": 660
},
{
"KL/chosen_KL_mean": -211.49526977539062,
"KL/mean": -244.50070190429688,
"KL/rejected_KL_mean": -277.506103515625,
"KL/std": 132.15182495117188,
"epoch": 0.999244142101285,
"fcm_dpo/beta": 0.004478364251554012,
"fcm_dpo/delta": 0.11909957975149155,
"fcm_dpo/margin": 66.0108413696289,
"fcm_dpo/q_t": 0.4358712136745453,
"grad_norm": 12.0364990234375,
"learning_rate": 3.4965187065971735e-12,
"logits/chosen": 0.8396966457366943,
"logits/rejected": 0.7795485258102417,
"logps/chosen": -271.867919921875,
"logps/ref_chosen": -60.37263870239258,
"logps/ref_rejected": -77.42874145507812,
"logps/rejected": -354.9348449707031,
"loss": 1.2199,
"margin_dpo/margin_mean": 66.0108413696289,
"margin_dpo/margin_std": 152.02621459960938,
"step": 661
},
{
"epoch": 0.999244142101285,
"step": 661,
"total_flos": 0.0,
"train_loss": 1.1707610893718414,
"train_runtime": 1649.6542,
"train_samples_per_second": 25.664,
"train_steps_per_second": 0.401
}
],
"logging_steps": 1,
"max_steps": 661,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink