W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.45-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC c2464a44b2 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.45-20260428-045924
Source: Original Platform
2026-05-26 11:34:17 +08:00

15707 lines
590 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.5197982788086,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.19226837158203,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": -0.009714126586914062,
"KL/mean": 0.018521636724472046,
"KL/rejected_KL_mean": 0.046756744384765625,
"KL/std": 0.22970828413963318,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0564723014831543,
"fcm_dpo/q_t": 0.5014121532440186,
"grad_norm": 71.04644775390625,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.4816778302192688,
"logits/rejected": -0.44217073917388916,
"logps/chosen": -60.991310119628906,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.62583923339844,
"loss": 1.3922,
"margin_dpo/margin_mean": -0.056471794843673706,
"margin_dpo/margin_std": 0.3306810259819031,
"step": 3
},
{
"KL/chosen_KL_mean": -0.058868408203125,
"KL/mean": -0.012493699789047241,
"KL/rejected_KL_mean": 0.03388214111328125,
"KL/std": 0.24051886796951294,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.09274768829345703,
"fcm_dpo/q_t": 0.5023181438446045,
"grad_norm": 72.18009948730469,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.4687211513519287,
"logits/rejected": -0.4410913288593292,
"logps/chosen": -56.82658004760742,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.61322784423828,
"loss": 1.3959,
"margin_dpo/margin_mean": -0.09274822473526001,
"margin_dpo/margin_std": 0.31391647458076477,
"step": 4
},
{
"KL/chosen_KL_mean": 0.03784751892089844,
"KL/mean": 0.027686715126037598,
"KL/rejected_KL_mean": 0.01752471923828125,
"KL/std": 0.22657622396945953,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.020323216915130615,
"fcm_dpo/q_t": 0.4994918406009674,
"grad_norm": 89.44688415527344,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.514456033706665,
"logits/rejected": -0.4707483947277069,
"logps/chosen": -53.82152557373047,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.13166046142578,
"loss": 1.3845,
"margin_dpo/margin_mean": 0.02032339572906494,
"margin_dpo/margin_std": 0.3100990056991577,
"step": 5
},
{
"KL/chosen_KL_mean": -0.022737503051757812,
"KL/mean": -0.03838449716567993,
"KL/rejected_KL_mean": -0.054027557373046875,
"KL/std": 0.2597954571247101,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.031293392181396484,
"fcm_dpo/q_t": 0.4992179870605469,
"grad_norm": 91.83596801757812,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5073306560516357,
"logits/rejected": -0.4651479721069336,
"logps/chosen": -63.03022003173828,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.69937133789062,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.03129345178604126,
"margin_dpo/margin_std": 0.3678780198097229,
"step": 6
},
{
"KL/chosen_KL_mean": 0.01824188232421875,
"KL/mean": 0.009548112750053406,
"KL/rejected_KL_mean": 0.000858306884765625,
"KL/std": 0.30541443824768066,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01738789677619934,
"fcm_dpo/q_t": 0.499565452337265,
"grad_norm": 82.313232421875,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.5054666996002197,
"logits/rejected": -0.47153782844543457,
"logps/chosen": -57.75657653808594,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.91973876953125,
"loss": 1.3851,
"margin_dpo/margin_mean": 0.01738855242729187,
"margin_dpo/margin_std": 0.47851502895355225,
"step": 7
},
{
"KL/chosen_KL_mean": 0.03425025939941406,
"KL/mean": 0.02390824258327484,
"KL/rejected_KL_mean": 0.013568878173828125,
"KL/std": 0.24170076847076416,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.020681053400039673,
"fcm_dpo/q_t": 0.49948275089263916,
"grad_norm": 78.34900665283203,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.5002312064170837,
"logits/rejected": -0.4734194874763489,
"logps/chosen": -58.681785583496094,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.2978515625,
"loss": 1.3845,
"margin_dpo/margin_mean": 0.020681530237197876,
"margin_dpo/margin_std": 0.34713995456695557,
"step": 8
},
{
"KL/chosen_KL_mean": 0.06855201721191406,
"KL/mean": 0.037594810128211975,
"KL/rejected_KL_mean": 0.0066375732421875,
"KL/std": 0.27004629373550415,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06191793084144592,
"fcm_dpo/q_t": 0.49845263361930847,
"grad_norm": 84.64756774902344,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.48879092931747437,
"logits/rejected": -0.4417349100112915,
"logps/chosen": -69.79829406738281,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.59602355957031,
"loss": 1.3805,
"margin_dpo/margin_mean": 0.06191718578338623,
"margin_dpo/margin_std": 0.39811158180236816,
"step": 9
},
{
"KL/chosen_KL_mean": 0.0026702880859375,
"KL/mean": -0.0006842687726020813,
"KL/rejected_KL_mean": -0.00403594970703125,
"KL/std": 0.271045058965683,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.006704941391944885,
"fcm_dpo/q_t": 0.499832421541214,
"grad_norm": 70.90953063964844,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.49560266733169556,
"logits/rejected": -0.4524238705635071,
"logps/chosen": -48.35501480102539,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.3760986328125,
"loss": 1.386,
"margin_dpo/margin_mean": 0.0067050158977508545,
"margin_dpo/margin_std": 0.3649532198905945,
"step": 10
},
{
"KL/chosen_KL_mean": -0.0072174072265625,
"KL/mean": -0.0030882954597473145,
"KL/rejected_KL_mean": 0.00103759765625,
"KL/std": 0.24918919801712036,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.008259892463684082,
"fcm_dpo/q_t": 0.5002060532569885,
"grad_norm": 68.35945129394531,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4571724534034729,
"logits/rejected": -0.43201056122779846,
"logps/chosen": -53.024070739746094,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.77934265136719,
"loss": 1.3875,
"margin_dpo/margin_mean": -0.008260130882263184,
"margin_dpo/margin_std": 0.372198224067688,
"step": 11
},
{
"KL/chosen_KL_mean": -0.07953453063964844,
"KL/mean": -0.05770687013864517,
"KL/rejected_KL_mean": -0.035869598388671875,
"KL/std": 0.2934768795967102,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04365710914134979,
"fcm_dpo/q_t": 0.5010908842086792,
"grad_norm": 99.78919982910156,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5305306911468506,
"logits/rejected": -0.4941375255584717,
"logps/chosen": -61.884971618652344,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.8941421508789,
"loss": 1.3911,
"margin_dpo/margin_mean": -0.04365821182727814,
"margin_dpo/margin_std": 0.4039604663848877,
"step": 12
},
{
"KL/chosen_KL_mean": 0.021123886108398438,
"KL/mean": -0.0034440606832504272,
"KL/rejected_KL_mean": -0.028011322021484375,
"KL/std": 0.25260454416275024,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04913899302482605,
"fcm_dpo/q_t": 0.49877145886421204,
"grad_norm": 79.35763549804688,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.46979302167892456,
"logits/rejected": -0.44078677892684937,
"logps/chosen": -64.23922729492188,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.23109436035156,
"loss": 1.3817,
"margin_dpo/margin_mean": 0.04913872480392456,
"margin_dpo/margin_std": 0.3550482392311096,
"step": 13
},
{
"KL/chosen_KL_mean": -0.015453338623046875,
"KL/mean": 0.0030228644609451294,
"KL/rejected_KL_mean": 0.021495819091796875,
"KL/std": 0.28745049238204956,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03694823384284973,
"fcm_dpo/q_t": 0.5009225010871887,
"grad_norm": 85.36051177978516,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.4957757592201233,
"logits/rejected": -0.4571382701396942,
"logps/chosen": -58.12566375732422,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.02558898925781,
"loss": 1.3904,
"margin_dpo/margin_mean": -0.036947935819625854,
"margin_dpo/margin_std": 0.405885249376297,
"step": 14
},
{
"KL/chosen_KL_mean": -0.0232086181640625,
"KL/mean": -0.03160741925239563,
"KL/rejected_KL_mean": -0.040012359619140625,
"KL/std": 0.21973538398742676,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01680755615234375,
"fcm_dpo/q_t": 0.4995792806148529,
"grad_norm": 64.38938903808594,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.4843750596046448,
"logits/rejected": -0.46534284949302673,
"logps/chosen": -56.99011993408203,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.8486557006836,
"loss": 1.3849,
"margin_dpo/margin_mean": 0.016807973384857178,
"margin_dpo/margin_std": 0.3134637475013733,
"step": 15
},
{
"KL/chosen_KL_mean": -0.000408172607421875,
"KL/mean": -0.03445975482463837,
"KL/rejected_KL_mean": -0.06850433349609375,
"KL/std": 0.27974092960357666,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06809529662132263,
"fcm_dpo/q_t": 0.4982984960079193,
"grad_norm": 84.10033416748047,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5534201860427856,
"logits/rejected": -0.5151947140693665,
"logps/chosen": -61.74030303955078,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.43797302246094,
"loss": 1.3799,
"margin_dpo/margin_mean": 0.0680951476097107,
"margin_dpo/margin_std": 0.403577983379364,
"step": 16
},
{
"KL/chosen_KL_mean": -0.020050048828125,
"KL/mean": -0.03625258803367615,
"KL/rejected_KL_mean": -0.052455902099609375,
"KL/std": 0.25603610277175903,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03240293264389038,
"fcm_dpo/q_t": 0.49919015169143677,
"grad_norm": 78.74256134033203,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.5004028081893921,
"logits/rejected": -0.4630909860134125,
"logps/chosen": -67.73038482666016,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.43110656738281,
"loss": 1.3834,
"margin_dpo/margin_mean": 0.032403409481048584,
"margin_dpo/margin_std": 0.3761865496635437,
"step": 17
},
{
"KL/chosen_KL_mean": 0.03632545471191406,
"KL/mean": -0.003826543688774109,
"KL/rejected_KL_mean": -0.043979644775390625,
"KL/std": 0.21307751536369324,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08030399680137634,
"fcm_dpo/q_t": 0.4979925751686096,
"grad_norm": 82.01667785644531,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.5047913789749146,
"logits/rejected": -0.4507453441619873,
"logps/chosen": -47.703163146972656,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.51626586914062,
"loss": 1.3785,
"margin_dpo/margin_mean": 0.08030354976654053,
"margin_dpo/margin_std": 0.275020033121109,
"step": 18
},
{
"KL/chosen_KL_mean": -0.029048919677734375,
"KL/mean": -0.06294414401054382,
"KL/rejected_KL_mean": -0.09683609008789062,
"KL/std": 0.2524881362915039,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06778490543365479,
"fcm_dpo/q_t": 0.4983060359954834,
"grad_norm": 73.40299224853516,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.49845147132873535,
"logits/rejected": -0.44924086332321167,
"logps/chosen": -70.23440551757812,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.85443115234375,
"loss": 1.3798,
"margin_dpo/margin_mean": 0.06778547167778015,
"margin_dpo/margin_std": 0.31719499826431274,
"step": 19
},
{
"KL/chosen_KL_mean": -0.0074100494384765625,
"KL/mean": -0.033930569887161255,
"KL/rejected_KL_mean": -0.06044769287109375,
"KL/std": 0.2493729591369629,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.053041279315948486,
"fcm_dpo/q_t": 0.4986746311187744,
"grad_norm": 74.1423110961914,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5496178865432739,
"logits/rejected": -0.49305465817451477,
"logps/chosen": -50.81065368652344,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.88379669189453,
"loss": 1.3812,
"margin_dpo/margin_mean": 0.05304113030433655,
"margin_dpo/margin_std": 0.3071237802505493,
"step": 20
},
{
"KL/chosen_KL_mean": -0.015211105346679688,
"KL/mean": -0.050164662301540375,
"KL/rejected_KL_mean": -0.08511734008789062,
"KL/std": 0.30007779598236084,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06991060078144073,
"fcm_dpo/q_t": 0.4982530474662781,
"grad_norm": 77.39631652832031,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.5219053030014038,
"logits/rejected": -0.500105082988739,
"logps/chosen": -50.07822799682617,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.95391082763672,
"loss": 1.3797,
"margin_dpo/margin_mean": 0.06991110742092133,
"margin_dpo/margin_std": 0.38004350662231445,
"step": 21
},
{
"KL/chosen_KL_mean": 0.012784957885742188,
"KL/mean": -0.08520452678203583,
"KL/rejected_KL_mean": -0.18319320678710938,
"KL/std": 0.290061354637146,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1959802210330963,
"fcm_dpo/q_t": 0.49510306119918823,
"grad_norm": 84.29964447021484,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.471519410610199,
"logits/rejected": -0.4271562695503235,
"logps/chosen": -59.04485321044922,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.6878662109375,
"loss": 1.3672,
"margin_dpo/margin_mean": 0.19597965478897095,
"margin_dpo/margin_std": 0.4104743003845215,
"step": 22
},
{
"KL/chosen_KL_mean": 0.030836105346679688,
"KL/mean": -0.05835852026939392,
"KL/rejected_KL_mean": -0.147552490234375,
"KL/std": 0.29798072576522827,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17839175462722778,
"fcm_dpo/q_t": 0.4955432415008545,
"grad_norm": 80.91433715820312,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.4988241195678711,
"logits/rejected": -0.47680526971817017,
"logps/chosen": -60.04685974121094,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.287109375,
"loss": 1.369,
"margin_dpo/margin_mean": 0.17839175462722778,
"margin_dpo/margin_std": 0.42720067501068115,
"step": 23
},
{
"KL/chosen_KL_mean": 0.004024505615234375,
"KL/mean": -0.11158287525177002,
"KL/rejected_KL_mean": -0.22719573974609375,
"KL/std": 0.2980898320674896,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23121654987335205,
"fcm_dpo/q_t": 0.4942222833633423,
"grad_norm": 80.61768341064453,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5090104341506958,
"logits/rejected": -0.492495596408844,
"logps/chosen": -44.287010192871094,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.35240936279297,
"loss": 1.3638,
"margin_dpo/margin_mean": 0.2312171459197998,
"margin_dpo/margin_std": 0.42055225372314453,
"step": 24
},
{
"KL/chosen_KL_mean": 0.027956008911132812,
"KL/mean": -0.08138252794742584,
"KL/rejected_KL_mean": -0.1907196044921875,
"KL/std": 0.3556647300720215,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21867385506629944,
"fcm_dpo/q_t": 0.4945368468761444,
"grad_norm": 73.873046875,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.4853622019290924,
"logits/rejected": -0.45499587059020996,
"logps/chosen": -52.50909423828125,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.53291320800781,
"loss": 1.365,
"margin_dpo/margin_mean": 0.21867355704307556,
"margin_dpo/margin_std": 0.4231797456741333,
"step": 25
},
{
"KL/chosen_KL_mean": 0.0839385986328125,
"KL/mean": -0.11762094497680664,
"KL/rejected_KL_mean": -0.3191795349121094,
"KL/std": 0.41716688871383667,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.40312010049819946,
"fcm_dpo/q_t": 0.4899322986602783,
"grad_norm": 87.6069564819336,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5463200807571411,
"logits/rejected": -0.5151112079620361,
"logps/chosen": -53.8388671875,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.67889404296875,
"loss": 1.3471,
"margin_dpo/margin_mean": 0.4031205177307129,
"margin_dpo/margin_std": 0.5176035165786743,
"step": 26
},
{
"KL/chosen_KL_mean": 0.12568092346191406,
"KL/mean": -0.11782613396644592,
"KL/rejected_KL_mean": -0.3613319396972656,
"KL/std": 0.4365026354789734,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.48701417446136475,
"fcm_dpo/q_t": 0.4878373146057129,
"grad_norm": 93.9299545288086,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5031396150588989,
"logits/rejected": -0.4665963351726532,
"logps/chosen": -42.772850036621094,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.08553314208984,
"loss": 1.3389,
"margin_dpo/margin_mean": 0.48701420426368713,
"margin_dpo/margin_std": 0.5155599117279053,
"step": 27
},
{
"KL/chosen_KL_mean": 0.016630172729492188,
"KL/mean": -0.12676912546157837,
"KL/rejected_KL_mean": -0.27016448974609375,
"KL/std": 0.4014029800891876,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.28679442405700684,
"fcm_dpo/q_t": 0.4928370714187622,
"grad_norm": 75.26355743408203,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5167675614356995,
"logits/rejected": -0.4624510407447815,
"logps/chosen": -60.53987121582031,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.67127990722656,
"loss": 1.3586,
"margin_dpo/margin_mean": 0.286795049905777,
"margin_dpo/margin_std": 0.571891188621521,
"step": 28
},
{
"KL/chosen_KL_mean": 0.08650970458984375,
"KL/mean": -0.1822274923324585,
"KL/rejected_KL_mean": -0.4509620666503906,
"KL/std": 0.4731426239013672,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5374736189842224,
"fcm_dpo/q_t": 0.4865817129611969,
"grad_norm": 90.55262756347656,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5708495378494263,
"logits/rejected": -0.5259160995483398,
"logps/chosen": -57.721275329589844,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.8453140258789,
"loss": 1.3341,
"margin_dpo/margin_mean": 0.5374747514724731,
"margin_dpo/margin_std": 0.5670518279075623,
"step": 29
},
{
"KL/chosen_KL_mean": 0.13196372985839844,
"KL/mean": -0.16928796470165253,
"KL/rejected_KL_mean": -0.47054290771484375,
"KL/std": 0.5440194606781006,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6025052070617676,
"fcm_dpo/q_t": 0.4849626421928406,
"grad_norm": 87.79325103759766,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.49273908138275146,
"logits/rejected": -0.4620092809200287,
"logps/chosen": -52.44540786743164,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -98.95974731445312,
"loss": 1.328,
"margin_dpo/margin_mean": 0.6025055646896362,
"margin_dpo/margin_std": 0.6316369771957397,
"step": 30
},
{
"KL/chosen_KL_mean": 0.1329364776611328,
"KL/mean": -0.11238974332809448,
"KL/rejected_KL_mean": -0.35771942138671875,
"KL/std": 0.5357916355133057,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4906558394432068,
"fcm_dpo/q_t": 0.4877535104751587,
"grad_norm": 68.40831756591797,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.5103215575218201,
"logits/rejected": -0.4660513401031494,
"logps/chosen": -63.67398452758789,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.25172424316406,
"loss": 1.3389,
"margin_dpo/margin_mean": 0.4906559884548187,
"margin_dpo/margin_std": 0.6376643180847168,
"step": 31
},
{
"KL/chosen_KL_mean": 0.14924049377441406,
"KL/mean": -0.1921585500240326,
"KL/rejected_KL_mean": -0.5335617065429688,
"KL/std": 0.6711945533752441,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6827967762947083,
"fcm_dpo/q_t": 0.4829793870449066,
"grad_norm": 84.73145294189453,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5275529026985168,
"logits/rejected": -0.48725956678390503,
"logps/chosen": -62.59028625488281,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.85105895996094,
"loss": 1.3208,
"margin_dpo/margin_mean": 0.6827970743179321,
"margin_dpo/margin_std": 0.8087345361709595,
"step": 32
},
{
"KL/chosen_KL_mean": 0.111785888671875,
"KL/mean": -0.1832684874534607,
"KL/rejected_KL_mean": -0.4783210754394531,
"KL/std": 0.5401982069015503,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5901058912277222,
"fcm_dpo/q_t": 0.48526614904403687,
"grad_norm": 71.8738784790039,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.4825378954410553,
"logits/rejected": -0.4562189280986786,
"logps/chosen": -53.14918518066406,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.36345672607422,
"loss": 1.3291,
"margin_dpo/margin_mean": 0.5901052951812744,
"margin_dpo/margin_std": 0.6080037355422974,
"step": 33
},
{
"KL/chosen_KL_mean": 0.06308174133300781,
"KL/mean": -0.3366324305534363,
"KL/rejected_KL_mean": -0.7363510131835938,
"KL/std": 0.7346658706665039,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7994296550750732,
"fcm_dpo/q_t": 0.48007193207740784,
"grad_norm": 77.36997985839844,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.48960667848587036,
"logits/rejected": -0.47221672534942627,
"logps/chosen": -50.75424575805664,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.65819549560547,
"loss": 1.3099,
"margin_dpo/margin_mean": 0.7994290590286255,
"margin_dpo/margin_std": 0.8769392371177673,
"step": 34
},
{
"KL/chosen_KL_mean": 0.14878082275390625,
"KL/mean": -0.46657490730285645,
"KL/rejected_KL_mean": -1.0819282531738281,
"KL/std": 1.0033079385757446,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2307109832763672,
"fcm_dpo/q_t": 0.46942776441574097,
"grad_norm": 82.35343170166016,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5199334025382996,
"logits/rejected": -0.4829786419868469,
"logps/chosen": -50.875709533691406,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.90635681152344,
"loss": 1.2702,
"margin_dpo/margin_mean": 1.2307111024856567,
"margin_dpo/margin_std": 1.1161994934082031,
"step": 35
},
{
"KL/chosen_KL_mean": 0.03397941589355469,
"KL/mean": -0.5253908038139343,
"KL/rejected_KL_mean": -1.0847587585449219,
"KL/std": 1.0640511512756348,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1187388896942139,
"fcm_dpo/q_t": 0.4721803665161133,
"grad_norm": 73.1759262084961,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5688107013702393,
"logits/rejected": -0.5328692197799683,
"logps/chosen": -51.95751190185547,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.12538146972656,
"loss": 1.2815,
"margin_dpo/margin_mean": 1.1187382936477661,
"margin_dpo/margin_std": 1.2321219444274902,
"step": 36
},
{
"KL/chosen_KL_mean": 0.016134262084960938,
"KL/mean": -0.5008864402770996,
"KL/rejected_KL_mean": -1.017913818359375,
"KL/std": 1.063030481338501,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.034048080444336,
"fcm_dpo/q_t": 0.47436150908470154,
"grad_norm": 61.269187927246094,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5045929551124573,
"logits/rejected": -0.46050429344177246,
"logps/chosen": -62.79097366333008,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.9129867553711,
"loss": 1.2905,
"margin_dpo/margin_mean": 1.0340485572814941,
"margin_dpo/margin_std": 1.369009017944336,
"step": 37
},
{
"KL/chosen_KL_mean": 0.1357593536376953,
"KL/mean": -0.5494659543037415,
"KL/rejected_KL_mean": -1.2346916198730469,
"KL/std": 1.3799314498901367,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3704538345336914,
"fcm_dpo/q_t": 0.4661816358566284,
"grad_norm": 70.13111877441406,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5132662057876587,
"logits/rejected": -0.4805130362510681,
"logps/chosen": -48.2547607421875,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.14713287353516,
"loss": 1.2609,
"margin_dpo/margin_mean": 1.3704535961151123,
"margin_dpo/margin_std": 1.6687695980072021,
"step": 38
},
{
"KL/chosen_KL_mean": 0.08719444274902344,
"KL/mean": -0.7247500419616699,
"KL/rejected_KL_mean": -1.5366897583007812,
"KL/std": 1.296190619468689,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.623884916305542,
"fcm_dpo/q_t": 0.4597076177597046,
"grad_norm": 73.18695068359375,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5521361827850342,
"logits/rejected": -0.5118452310562134,
"logps/chosen": -50.66327667236328,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.106201171875,
"loss": 1.2356,
"margin_dpo/margin_mean": 1.6238853931427002,
"margin_dpo/margin_std": 1.410783290863037,
"step": 39
},
{
"KL/chosen_KL_mean": 0.20708274841308594,
"KL/mean": -0.5767695903778076,
"KL/rejected_KL_mean": -1.360626220703125,
"KL/std": 1.399301290512085,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5677101612091064,
"fcm_dpo/q_t": 0.46130138635635376,
"grad_norm": 60.95301055908203,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5224695205688477,
"logits/rejected": -0.4926735758781433,
"logps/chosen": -57.777984619140625,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.66069793701172,
"loss": 1.2429,
"margin_dpo/margin_mean": 1.5677103996276855,
"margin_dpo/margin_std": 1.659773349761963,
"step": 40
},
{
"KL/chosen_KL_mean": 0.018999099731445312,
"KL/mean": -0.9265443086624146,
"KL/rejected_KL_mean": -1.8720817565917969,
"KL/std": 1.8526782989501953,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10123378038406372,
"fcm_dpo/delta": 0.12188068777322769,
"fcm_dpo/margin": 1.8910868167877197,
"fcm_dpo/q_t": 0.45350411534309387,
"grad_norm": 69.15343475341797,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5473026037216187,
"logits/rejected": -0.511269211769104,
"logps/chosen": -62.67681884765625,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.89561462402344,
"loss": 1.2142,
"margin_dpo/margin_mean": 1.891087293624878,
"margin_dpo/margin_std": 1.9742361307144165,
"step": 41
},
{
"KL/chosen_KL_mean": 0.19712257385253906,
"KL/mean": -1.0590769052505493,
"KL/rejected_KL_mean": -2.3152732849121094,
"KL/std": 2.1316654682159424,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.10550861060619354,
"fcm_dpo/delta": 0.18977521359920502,
"fcm_dpo/margin": 2.5123987197875977,
"fcm_dpo/q_t": 0.43665236234664917,
"grad_norm": 82.9195556640625,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.529656708240509,
"logits/rejected": -0.4830622673034668,
"logps/chosen": -58.76930236816406,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.22364807128906,
"loss": 1.1548,
"margin_dpo/margin_mean": 2.5123987197875977,
"margin_dpo/margin_std": 2.446010112762451,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5524749755859375,
"KL/mean": -0.7236171960830688,
"KL/rejected_KL_mean": -1.9997138977050781,
"KL/std": 1.8800958395004272,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.10917122662067413,
"fcm_dpo/delta": 0.17643879354000092,
"fcm_dpo/margin": 2.5521881580352783,
"fcm_dpo/q_t": 0.43298065662384033,
"grad_norm": 76.43048858642578,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5261872410774231,
"logits/rejected": -0.5006349086761475,
"logps/chosen": -53.60352325439453,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.47990417480469,
"loss": 1.139,
"margin_dpo/margin_mean": 2.552187919616699,
"margin_dpo/margin_std": 2.015552043914795,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2166748046875,
"KL/mean": -1.1926087141036987,
"KL/rejected_KL_mean": -2.6018943786621094,
"KL/std": 2.293412208557129,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.11247065663337708,
"fcm_dpo/delta": 0.13742657005786896,
"fcm_dpo/margin": 2.8185644149780273,
"fcm_dpo/q_t": 0.42392122745513916,
"grad_norm": 85.97471618652344,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4837245047092438,
"logits/rejected": -0.46352899074554443,
"logps/chosen": -49.86182403564453,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.38565063476562,
"loss": 1.1097,
"margin_dpo/margin_mean": 2.8185648918151855,
"margin_dpo/margin_std": 2.2489266395568848,
"step": 44
},
{
"KL/chosen_KL_mean": 0.1769847869873047,
"KL/mean": -0.9922209978103638,
"KL/rejected_KL_mean": -2.1614303588867188,
"KL/std": 2.0988166332244873,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.11665238440036774,
"fcm_dpo/delta": 0.18180739879608154,
"fcm_dpo/margin": 2.3384156227111816,
"fcm_dpo/q_t": 0.43547263741493225,
"grad_norm": 69.66962432861328,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.5075412392616272,
"logits/rejected": -0.49554643034935,
"logps/chosen": -48.237945556640625,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.09785461425781,
"loss": 1.1544,
"margin_dpo/margin_mean": 2.3384151458740234,
"margin_dpo/margin_std": 2.5854876041412354,
"step": 45
},
{
"KL/chosen_KL_mean": 0.19272613525390625,
"KL/mean": -1.2884098291397095,
"KL/rejected_KL_mean": -2.7695465087890625,
"KL/std": 2.6169562339782715,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.11946526169776917,
"fcm_dpo/delta": 0.09943334758281708,
"fcm_dpo/margin": 2.9622695446014404,
"fcm_dpo/q_t": 0.41688013076782227,
"grad_norm": 80.77259826660156,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5466458201408386,
"logits/rejected": -0.4965612292289734,
"logps/chosen": -55.80670166015625,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.42213439941406,
"loss": 1.0964,
"margin_dpo/margin_mean": 2.9622700214385986,
"margin_dpo/margin_std": 3.1480636596679688,
"step": 46
},
{
"KL/chosen_KL_mean": 0.40157127380371094,
"KL/mean": -1.0910553932189941,
"KL/rejected_KL_mean": -2.5836868286132812,
"KL/std": 2.669870615005493,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.12188410758972168,
"fcm_dpo/delta": 0.08888313174247742,
"fcm_dpo/margin": 2.9852540493011475,
"fcm_dpo/q_t": 0.4136195778846741,
"grad_norm": 75.20211791992188,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5581070184707642,
"logits/rejected": -0.5044229030609131,
"logps/chosen": -57.524505615234375,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.26289367675781,
"loss": 1.0836,
"margin_dpo/margin_mean": 2.9852542877197266,
"margin_dpo/margin_std": 2.7970516681671143,
"step": 47
},
{
"KL/chosen_KL_mean": 0.03553581237792969,
"KL/mean": -1.5421160459518433,
"KL/rejected_KL_mean": -3.1197662353515625,
"KL/std": 2.6868507862091064,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.12283409386873245,
"fcm_dpo/delta": 0.06493905931711197,
"fcm_dpo/margin": 3.1553053855895996,
"fcm_dpo/q_t": 0.40832480788230896,
"grad_norm": 82.24751281738281,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5874247550964355,
"logits/rejected": -0.5290216207504272,
"logps/chosen": -57.15253829956055,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.13636779785156,
"loss": 1.0721,
"margin_dpo/margin_mean": 3.1553053855895996,
"margin_dpo/margin_std": 2.956268548965454,
"step": 48
},
{
"KL/chosen_KL_mean": 0.28813743591308594,
"KL/mean": -1.608811855316162,
"KL/rejected_KL_mean": -3.505756378173828,
"KL/std": 3.38679838180542,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.12366083264350891,
"fcm_dpo/delta": -0.020250260829925537,
"fcm_dpo/margin": 3.7938899993896484,
"fcm_dpo/q_t": 0.39228618144989014,
"grad_norm": 74.37108612060547,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5616977214813232,
"logits/rejected": -0.5045787692070007,
"logps/chosen": -61.397132873535156,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.27323150634766,
"loss": 1.026,
"margin_dpo/margin_mean": 3.7938907146453857,
"margin_dpo/margin_std": 3.9254069328308105,
"step": 49
},
{
"KL/chosen_KL_mean": -0.1705455780029297,
"KL/mean": -2.235653877258301,
"KL/rejected_KL_mean": -4.300762176513672,
"KL/std": 3.464420795440674,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.12194898724555969,
"fcm_dpo/delta": -0.05662181228399277,
"fcm_dpo/margin": 4.130216598510742,
"fcm_dpo/q_t": 0.3830483853816986,
"grad_norm": 71.89964294433594,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5394672155380249,
"logits/rejected": -0.5033368468284607,
"logps/chosen": -58.894683837890625,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.65890502929688,
"loss": 0.9966,
"margin_dpo/margin_mean": 4.130216598510742,
"margin_dpo/margin_std": 3.9122300148010254,
"step": 50
},
{
"KL/chosen_KL_mean": -0.28783416748046875,
"KL/mean": -2.3152730464935303,
"KL/rejected_KL_mean": -4.342716217041016,
"KL/std": 4.196344375610352,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.12034690380096436,
"fcm_dpo/delta": -0.04077950119972229,
"fcm_dpo/margin": 4.054882526397705,
"fcm_dpo/q_t": 0.39256805181503296,
"grad_norm": 58.6294059753418,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5217244625091553,
"logits/rejected": -0.4886956810951233,
"logps/chosen": -61.661502838134766,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.34471893310547,
"loss": 1.0407,
"margin_dpo/margin_mean": 4.054882526397705,
"margin_dpo/margin_std": 5.119638442993164,
"step": 51
},
{
"KL/chosen_KL_mean": 0.3047008514404297,
"KL/mean": -2.708411693572998,
"KL/rejected_KL_mean": -5.721527099609375,
"KL/std": 4.686271667480469,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.11606203764677048,
"fcm_dpo/delta": -0.26816853880882263,
"fcm_dpo/margin": 6.0262274742126465,
"fcm_dpo/q_t": 0.3416193127632141,
"grad_norm": 63.627662658691406,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5571841597557068,
"logits/rejected": -0.5018373131752014,
"logps/chosen": -52.03265380859375,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.69544219970703,
"loss": 0.8787,
"margin_dpo/margin_mean": 6.0262274742126465,
"margin_dpo/margin_std": 5.079197883605957,
"step": 52
},
{
"KL/chosen_KL_mean": -0.18621826171875,
"KL/mean": -3.28546404838562,
"KL/rejected_KL_mean": -6.384708404541016,
"KL/std": 5.369787216186523,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.11117444187402725,
"fcm_dpo/delta": -0.256150484085083,
"fcm_dpo/margin": 6.198496341705322,
"fcm_dpo/q_t": 0.3534468710422516,
"grad_norm": 62.074974060058594,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.5862455368041992,
"logits/rejected": -0.5638996958732605,
"logps/chosen": -53.50086975097656,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.16830444335938,
"loss": 0.9201,
"margin_dpo/margin_mean": 6.198496341705322,
"margin_dpo/margin_std": 6.157903671264648,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3846149444580078,
"KL/mean": -3.0846762657165527,
"KL/rejected_KL_mean": -5.784732818603516,
"KL/std": 5.093508720397949,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.10680510848760605,
"fcm_dpo/delta": -0.13433100283145905,
"fcm_dpo/margin": 5.400116920471191,
"fcm_dpo/q_t": 0.3691354990005493,
"grad_norm": 58.43413543701172,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5928705334663391,
"logits/rejected": -0.5393671989440918,
"logps/chosen": -51.07327651977539,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.5001220703125,
"loss": 0.9555,
"margin_dpo/margin_mean": 5.400116920471191,
"margin_dpo/margin_std": 5.129217624664307,
"step": 54
},
{
"KL/chosen_KL_mean": -0.8974971771240234,
"KL/mean": -4.153885841369629,
"KL/rejected_KL_mean": -7.410274505615234,
"KL/std": 6.167769432067871,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.10262539237737656,
"fcm_dpo/delta": -0.233448326587677,
"fcm_dpo/margin": 6.512773513793945,
"fcm_dpo/q_t": 0.3559892475605011,
"grad_norm": 54.6057014465332,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6421641111373901,
"logits/rejected": -0.5792768001556396,
"logps/chosen": -63.512733459472656,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.40377807617188,
"loss": 0.9461,
"margin_dpo/margin_mean": 6.512773036956787,
"margin_dpo/margin_std": 7.380474090576172,
"step": 55
},
{
"KL/chosen_KL_mean": -0.6917400360107422,
"KL/mean": -3.759640693664551,
"KL/rejected_KL_mean": -6.827548980712891,
"KL/std": 6.048251628875732,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.09863682836294174,
"fcm_dpo/delta": -0.1651272475719452,
"fcm_dpo/margin": 6.135809898376465,
"fcm_dpo/q_t": 0.3689035177230835,
"grad_norm": 47.54218673706055,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.6309115886688232,
"logits/rejected": -0.5893651247024536,
"logps/chosen": -58.62446594238281,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -101.00198364257812,
"loss": 0.9837,
"margin_dpo/margin_mean": 6.135810375213623,
"margin_dpo/margin_std": 7.349212646484375,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7053565979003906,
"KL/mean": -4.171047210693359,
"KL/rejected_KL_mean": -7.636737823486328,
"KL/std": 5.979583263397217,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.09502957761287689,
"fcm_dpo/delta": -0.2228081077337265,
"fcm_dpo/margin": 6.931379795074463,
"fcm_dpo/q_t": 0.3515072464942932,
"grad_norm": 52.40454864501953,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5713903903961182,
"logits/rejected": -0.5428633689880371,
"logps/chosen": -71.20063781738281,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.20219421386719,
"loss": 0.9213,
"margin_dpo/margin_mean": 6.931380271911621,
"margin_dpo/margin_std": 6.604933738708496,
"step": 57
},
{
"KL/chosen_KL_mean": -1.0504646301269531,
"KL/mean": -4.870463848114014,
"KL/rejected_KL_mean": -8.690460205078125,
"KL/std": 6.776705741882324,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.09036161750555038,
"fcm_dpo/delta": -0.2574799656867981,
"fcm_dpo/margin": 7.639996528625488,
"fcm_dpo/q_t": 0.35100215673446655,
"grad_norm": 55.40933609008789,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.6046279668807983,
"logits/rejected": -0.5296686291694641,
"logps/chosen": -63.18340301513672,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.30775451660156,
"loss": 0.9253,
"margin_dpo/margin_mean": 7.639996528625488,
"margin_dpo/margin_std": 8.138544082641602,
"step": 58
},
{
"KL/chosen_KL_mean": -1.4362564086914062,
"KL/mean": -5.643172264099121,
"KL/rejected_KL_mean": -9.850086212158203,
"KL/std": 7.354021072387695,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.08438800275325775,
"fcm_dpo/delta": -0.28398486971855164,
"fcm_dpo/margin": 8.41383171081543,
"fcm_dpo/q_t": 0.34511446952819824,
"grad_norm": 51.24748611450195,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6187716126441956,
"logits/rejected": -0.5780496597290039,
"logps/chosen": -53.36878204345703,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.73529052734375,
"loss": 0.9054,
"margin_dpo/margin_mean": 8.41383171081543,
"margin_dpo/margin_std": 8.447948455810547,
"step": 59
},
{
"KL/chosen_KL_mean": -2.6461715698242188,
"KL/mean": -5.977200508117676,
"KL/rejected_KL_mean": -9.308223724365234,
"KL/std": 6.706561088562012,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.08230020850896835,
"fcm_dpo/delta": -0.10461775958538055,
"fcm_dpo/margin": 6.662050247192383,
"fcm_dpo/q_t": 0.37456339597702026,
"grad_norm": 55.224952697753906,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6233890652656555,
"logits/rejected": -0.5643144249916077,
"logps/chosen": -63.58835983276367,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.70162200927734,
"loss": 1.0012,
"margin_dpo/margin_mean": 6.662050247192383,
"margin_dpo/margin_std": 7.595391273498535,
"step": 60
},
{
"KL/chosen_KL_mean": -1.484731674194336,
"KL/mean": -5.6159772872924805,
"KL/rejected_KL_mean": -9.747222900390625,
"KL/std": 9.038053512573242,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.07915720343589783,
"fcm_dpo/delta": -0.21973448991775513,
"fcm_dpo/margin": 8.262494087219238,
"fcm_dpo/q_t": 0.3663337528705597,
"grad_norm": 46.47762680053711,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.5932574272155762,
"logits/rejected": -0.5587595701217651,
"logps/chosen": -62.118255615234375,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.59971618652344,
"loss": 0.9817,
"margin_dpo/margin_mean": 8.262494087219238,
"margin_dpo/margin_std": 11.187477111816406,
"step": 61
},
{
"KL/chosen_KL_mean": -1.6137275695800781,
"KL/mean": -4.718844413757324,
"KL/rejected_KL_mean": -7.8239593505859375,
"KL/std": 6.609654903411865,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.07849176973104477,
"fcm_dpo/delta": -0.03957677260041237,
"fcm_dpo/margin": 6.210230827331543,
"fcm_dpo/q_t": 0.39217427372932434,
"grad_norm": 46.09128189086914,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.594412624835968,
"logits/rejected": -0.5592746734619141,
"logps/chosen": -57.76449966430664,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.39015197753906,
"loss": 1.0485,
"margin_dpo/margin_mean": 6.210231781005859,
"margin_dpo/margin_std": 8.076946258544922,
"step": 62
},
{
"KL/chosen_KL_mean": -2.6496753692626953,
"KL/mean": -6.814581871032715,
"KL/rejected_KL_mean": -10.979488372802734,
"KL/std": 8.399147987365723,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.07532910257577896,
"fcm_dpo/delta": -0.19143244624137878,
"fcm_dpo/margin": 8.329812049865723,
"fcm_dpo/q_t": 0.3595030903816223,
"grad_norm": 47.35303497314453,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.5923917889595032,
"logits/rejected": -0.547005295753479,
"logps/chosen": -75.79707336425781,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -108.58955383300781,
"loss": 0.9544,
"margin_dpo/margin_mean": 8.329812049865723,
"margin_dpo/margin_std": 8.884681701660156,
"step": 63
},
{
"KL/chosen_KL_mean": -1.0408763885498047,
"KL/mean": -5.9636712074279785,
"KL/rejected_KL_mean": -10.886466979980469,
"KL/std": 9.097991943359375,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.07129538059234619,
"fcm_dpo/delta": -0.27623987197875977,
"fcm_dpo/margin": 9.845589637756348,
"fcm_dpo/q_t": 0.3495906591415405,
"grad_norm": 42.23509979248047,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.5778393745422363,
"logits/rejected": -0.5453320741653442,
"logps/chosen": -55.03947448730469,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.41667175292969,
"loss": 0.9342,
"margin_dpo/margin_mean": 9.845589637756348,
"margin_dpo/margin_std": 11.189323425292969,
"step": 64
},
{
"KL/chosen_KL_mean": -3.26495361328125,
"KL/mean": -8.230804443359375,
"KL/rejected_KL_mean": -13.1966552734375,
"KL/std": 10.192216873168945,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.06810398399829865,
"fcm_dpo/delta": -0.24493469297885895,
"fcm_dpo/margin": 9.931705474853516,
"fcm_dpo/q_t": 0.35102152824401855,
"grad_norm": 43.12425231933594,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6511636972427368,
"logits/rejected": -0.6380265951156616,
"logps/chosen": -68.1009521484375,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -123.14311218261719,
"loss": 0.947,
"margin_dpo/margin_mean": 9.931705474853516,
"margin_dpo/margin_std": 11.15705680847168,
"step": 65
},
{
"KL/chosen_KL_mean": -3.0033321380615234,
"KL/mean": -7.569043159484863,
"KL/rejected_KL_mean": -12.134750366210938,
"KL/std": 9.202418327331543,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.06553401052951813,
"fcm_dpo/delta": -0.16035820543766022,
"fcm_dpo/margin": 9.131417274475098,
"fcm_dpo/q_t": 0.3708842694759369,
"grad_norm": 38.7624626159668,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6315481066703796,
"logits/rejected": -0.5981906652450562,
"logps/chosen": -54.44685745239258,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -87.77104949951172,
"loss": 0.9843,
"margin_dpo/margin_mean": 9.131417274475098,
"margin_dpo/margin_std": 11.071283340454102,
"step": 66
},
{
"KL/chosen_KL_mean": -2.688854217529297,
"KL/mean": -7.336255073547363,
"KL/rejected_KL_mean": -11.983657836914062,
"KL/std": 9.25648307800293,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.06420481204986572,
"fcm_dpo/delta": -0.1558014154434204,
"fcm_dpo/margin": 9.294797897338867,
"fcm_dpo/q_t": 0.3707802891731262,
"grad_norm": 39.4743766784668,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.5986242294311523,
"logits/rejected": -0.555691123008728,
"logps/chosen": -62.029659271240234,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -84.77094268798828,
"loss": 0.9841,
"margin_dpo/margin_mean": 9.294797897338867,
"margin_dpo/margin_std": 11.213580131530762,
"step": 67
},
{
"KL/chosen_KL_mean": -3.099040985107422,
"KL/mean": -7.559505462646484,
"KL/rejected_KL_mean": -12.019973754882812,
"KL/std": 8.27375602722168,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.06258167326450348,
"fcm_dpo/delta": -0.11453147232532501,
"fcm_dpo/margin": 8.920928955078125,
"fcm_dpo/q_t": 0.37400543689727783,
"grad_norm": 38.21786880493164,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6311640739440918,
"logits/rejected": -0.5698702335357666,
"logps/chosen": -68.30487060546875,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -89.22721862792969,
"loss": 0.9784,
"margin_dpo/margin_mean": 8.920928955078125,
"margin_dpo/margin_std": 9.243380546569824,
"step": 68
},
{
"KL/chosen_KL_mean": -3.6259117126464844,
"KL/mean": -9.136199951171875,
"KL/rejected_KL_mean": -14.646488189697266,
"KL/std": 9.907639503479004,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.059743233025074005,
"fcm_dpo/delta": -0.2245938926935196,
"fcm_dpo/margin": 11.020575523376465,
"fcm_dpo/q_t": 0.35451948642730713,
"grad_norm": 40.647071838378906,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.599290668964386,
"logits/rejected": -0.5760542154312134,
"logps/chosen": -63.445152282714844,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -118.03535461425781,
"loss": 0.9216,
"margin_dpo/margin_mean": 11.020574569702148,
"margin_dpo/margin_std": 10.930428504943848,
"step": 69
},
{
"KL/chosen_KL_mean": -5.381891250610352,
"KL/mean": -11.343828201293945,
"KL/rejected_KL_mean": -17.305763244628906,
"KL/std": 12.163260459899902,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.0573650524020195,
"fcm_dpo/delta": -0.2505870461463928,
"fcm_dpo/margin": 11.923868179321289,
"fcm_dpo/q_t": 0.352365642786026,
"grad_norm": 40.70524597167969,
"learning_rate": 5e-07,
"logits/chosen": -0.6251968741416931,
"logits/rejected": -0.5907549858093262,
"logps/chosen": -67.31253051757812,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -108.36654663085938,
"loss": 0.9317,
"margin_dpo/margin_mean": 11.923868179321289,
"margin_dpo/margin_std": 12.913337707519531,
"step": 70
},
{
"KL/chosen_KL_mean": -5.122894287109375,
"KL/mean": -12.12697982788086,
"KL/rejected_KL_mean": -19.13106918334961,
"KL/std": 12.782737731933594,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.05358006805181503,
"fcm_dpo/delta": -0.3263747990131378,
"fcm_dpo/margin": 14.008173942565918,
"fcm_dpo/q_t": 0.3399330973625183,
"grad_norm": 37.48530578613281,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.658132791519165,
"logits/rejected": -0.6188231706619263,
"logps/chosen": -66.87322998046875,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -116.46768951416016,
"loss": 0.8843,
"margin_dpo/margin_mean": 14.008173942565918,
"margin_dpo/margin_std": 13.910791397094727,
"step": 71
},
{
"KL/chosen_KL_mean": -5.92857551574707,
"KL/mean": -13.34725570678711,
"KL/rejected_KL_mean": -20.765933990478516,
"KL/std": 13.781370162963867,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.05028136447072029,
"fcm_dpo/delta": -0.3209035396575928,
"fcm_dpo/margin": 14.837360382080078,
"fcm_dpo/q_t": 0.33984917402267456,
"grad_norm": 37.00834274291992,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6663771271705627,
"logits/rejected": -0.6306756138801575,
"logps/chosen": -71.98198699951172,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -116.05291748046875,
"loss": 0.9045,
"margin_dpo/margin_mean": 14.837360382080078,
"margin_dpo/margin_std": 16.077857971191406,
"step": 72
},
{
"KL/chosen_KL_mean": -7.825841903686523,
"KL/mean": -15.346532821655273,
"KL/rejected_KL_mean": -22.867225646972656,
"KL/std": 18.097063064575195,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.04731083661317825,
"fcm_dpo/delta": -0.2835136353969574,
"fcm_dpo/margin": 15.04138469696045,
"fcm_dpo/q_t": 0.36181771755218506,
"grad_norm": 38.30211639404297,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.6687195301055908,
"logits/rejected": -0.6266169548034668,
"logps/chosen": -74.08211517333984,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -113.3233642578125,
"loss": 1.0145,
"margin_dpo/margin_mean": 15.041385650634766,
"margin_dpo/margin_std": 22.485240936279297,
"step": 73
},
{
"KL/chosen_KL_mean": -8.45234489440918,
"KL/mean": -16.59984588623047,
"KL/rejected_KL_mean": -24.74734115600586,
"KL/std": 20.056711196899414,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.04465902969241142,
"fcm_dpo/delta": -0.30072933435440063,
"fcm_dpo/margin": 16.294998168945312,
"fcm_dpo/q_t": 0.3568170368671417,
"grad_norm": 40.10905075073242,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.648106575012207,
"logits/rejected": -0.6394679546356201,
"logps/chosen": -61.877227783203125,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -120.69427490234375,
"loss": 0.9829,
"margin_dpo/margin_mean": 16.294998168945312,
"margin_dpo/margin_std": 24.927589416503906,
"step": 74
},
{
"KL/chosen_KL_mean": -8.210355758666992,
"KL/mean": -19.646503448486328,
"KL/rejected_KL_mean": -31.08264923095703,
"KL/std": 20.41864013671875,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.04106065630912781,
"fcm_dpo/delta": -0.5371048450469971,
"fcm_dpo/margin": 22.872297286987305,
"fcm_dpo/q_t": 0.3111739158630371,
"grad_norm": 32.669776916503906,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6666178703308105,
"logits/rejected": -0.6588006019592285,
"logps/chosen": -60.072017669677734,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -142.33663940429688,
"loss": 0.8227,
"margin_dpo/margin_mean": 22.872297286987305,
"margin_dpo/margin_std": 23.01834487915039,
"step": 75
},
{
"KL/chosen_KL_mean": -10.067512512207031,
"KL/mean": -17.29259490966797,
"KL/rejected_KL_mean": -24.517669677734375,
"KL/std": 16.541284561157227,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.03894951939582825,
"fcm_dpo/delta": -0.11947859078645706,
"fcm_dpo/margin": 14.450164794921875,
"fcm_dpo/q_t": 0.37349021434783936,
"grad_norm": 34.09317398071289,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.7095220685005188,
"logits/rejected": -0.6687110662460327,
"logps/chosen": -63.33354949951172,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -102.73429870605469,
"loss": 1.0071,
"margin_dpo/margin_mean": 14.450165748596191,
"margin_dpo/margin_std": 17.991954803466797,
"step": 76
},
{
"KL/chosen_KL_mean": -8.949975967407227,
"KL/mean": -20.767536163330078,
"KL/rejected_KL_mean": -32.58510208129883,
"KL/std": 22.64974594116211,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.03596869111061096,
"fcm_dpo/delta": -0.44180840253829956,
"fcm_dpo/margin": 23.635120391845703,
"fcm_dpo/q_t": 0.3196060359477997,
"grad_norm": 35.29338836669922,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6948204040527344,
"logits/rejected": -0.6578306555747986,
"logps/chosen": -67.0466537475586,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -126.35871887207031,
"loss": 0.8707,
"margin_dpo/margin_mean": 23.635120391845703,
"margin_dpo/margin_std": 24.971656799316406,
"step": 77
},
{
"KL/chosen_KL_mean": -8.99593734741211,
"KL/mean": -17.899391174316406,
"KL/rejected_KL_mean": -26.802845001220703,
"KL/std": 18.244056701660156,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.03460092842578888,
"fcm_dpo/delta": -0.17669706046581268,
"fcm_dpo/margin": 17.806903839111328,
"fcm_dpo/q_t": 0.3664587140083313,
"grad_norm": 31.800643920898438,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6439747214317322,
"logits/rejected": -0.6189943552017212,
"logps/chosen": -64.60972595214844,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -111.73721313476562,
"loss": 0.9754,
"margin_dpo/margin_mean": 17.806903839111328,
"margin_dpo/margin_std": 20.876663208007812,
"step": 78
},
{
"KL/chosen_KL_mean": -9.265439987182617,
"KL/mean": -19.035083770751953,
"KL/rejected_KL_mean": -28.804729461669922,
"KL/std": 20.695537567138672,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.03311431035399437,
"fcm_dpo/delta": -0.21076560020446777,
"fcm_dpo/margin": 19.53928565979004,
"fcm_dpo/q_t": 0.36637139320373535,
"grad_norm": 27.853378295898438,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.646593451499939,
"logits/rejected": -0.6158708333969116,
"logps/chosen": -64.71592712402344,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -116.4522933959961,
"loss": 0.9865,
"margin_dpo/margin_mean": 19.539283752441406,
"margin_dpo/margin_std": 25.31096649169922,
"step": 79
},
{
"KL/chosen_KL_mean": -12.27981185913086,
"KL/mean": -21.402843475341797,
"KL/rejected_KL_mean": -30.52587127685547,
"KL/std": 21.86465072631836,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.031817443668842316,
"fcm_dpo/delta": -0.14128068089485168,
"fcm_dpo/margin": 18.24605941772461,
"fcm_dpo/q_t": 0.38100287318229675,
"grad_norm": 29.229244232177734,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7156493663787842,
"logits/rejected": -0.6729994416236877,
"logps/chosen": -70.79910278320312,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -118.07337188720703,
"loss": 1.0304,
"margin_dpo/margin_mean": 18.246057510375977,
"margin_dpo/margin_std": 25.342288970947266,
"step": 80
},
{
"KL/chosen_KL_mean": -11.352493286132812,
"KL/mean": -23.801851272583008,
"KL/rejected_KL_mean": -36.25121307373047,
"KL/std": 25.796279907226562,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.03014964610338211,
"fcm_dpo/delta": -0.32812806963920593,
"fcm_dpo/margin": 24.898719787597656,
"fcm_dpo/q_t": 0.3463107943534851,
"grad_norm": 30.754024505615234,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7107023000717163,
"logits/rejected": -0.6817086935043335,
"logps/chosen": -77.80136108398438,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -165.91390991210938,
"loss": 0.9324,
"margin_dpo/margin_mean": 24.898719787597656,
"margin_dpo/margin_std": 30.64748764038086,
"step": 81
},
{
"KL/chosen_KL_mean": -13.689064025878906,
"KL/mean": -23.61670684814453,
"KL/rejected_KL_mean": -33.544349670410156,
"KL/std": 23.599985122680664,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.02952047996222973,
"fcm_dpo/delta": -0.14547684788703918,
"fcm_dpo/margin": 19.855289459228516,
"fcm_dpo/q_t": 0.38032636046409607,
"grad_norm": 34.90581512451172,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6471748352050781,
"logits/rejected": -0.6358625888824463,
"logps/chosen": -65.92144775390625,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -124.28759765625,
"loss": 1.0777,
"margin_dpo/margin_mean": 19.855289459228516,
"margin_dpo/margin_std": 32.484397888183594,
"step": 82
},
{
"KL/chosen_KL_mean": -13.971277236938477,
"KL/mean": -25.6943359375,
"KL/rejected_KL_mean": -37.41740036010742,
"KL/std": 24.552268981933594,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.02794184908270836,
"fcm_dpo/delta": -0.22061091661453247,
"fcm_dpo/margin": 23.446125030517578,
"fcm_dpo/q_t": 0.36134088039398193,
"grad_norm": 29.178665161132812,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6698837876319885,
"logits/rejected": -0.6350330114364624,
"logps/chosen": -69.79866027832031,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -141.13330078125,
"loss": 0.9742,
"margin_dpo/margin_mean": 23.446123123168945,
"margin_dpo/margin_std": 29.056337356567383,
"step": 83
},
{
"KL/chosen_KL_mean": -12.916067123413086,
"KL/mean": -22.892362594604492,
"KL/rejected_KL_mean": -32.868656158447266,
"KL/std": 20.869712829589844,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.027103561908006668,
"fcm_dpo/delta": -0.09814468771219254,
"fcm_dpo/margin": 19.952594757080078,
"fcm_dpo/q_t": 0.37633585929870605,
"grad_norm": 27.472469329833984,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6623414754867554,
"logits/rejected": -0.6236976385116577,
"logps/chosen": -80.09223937988281,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -120.1672592163086,
"loss": 0.9969,
"margin_dpo/margin_mean": 19.952594757080078,
"margin_dpo/margin_std": 21.364559173583984,
"step": 84
},
{
"KL/chosen_KL_mean": -13.362470626831055,
"KL/mean": -23.700489044189453,
"KL/rejected_KL_mean": -34.038509368896484,
"KL/std": 21.551101684570312,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.026955682784318924,
"fcm_dpo/delta": -0.11413806676864624,
"fcm_dpo/margin": 20.676042556762695,
"fcm_dpo/q_t": 0.37861472368240356,
"grad_norm": 27.54196548461914,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6713258028030396,
"logits/rejected": -0.6413929462432861,
"logps/chosen": -71.76908874511719,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -112.67730712890625,
"loss": 1.0181,
"margin_dpo/margin_mean": 20.676042556762695,
"margin_dpo/margin_std": 25.745641708374023,
"step": 85
},
{
"KL/chosen_KL_mean": -17.15639305114746,
"KL/mean": -28.15932846069336,
"KL/rejected_KL_mean": -39.162261962890625,
"KL/std": 28.993762969970703,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.02617364190518856,
"fcm_dpo/delta": -0.13349324464797974,
"fcm_dpo/margin": 22.005878448486328,
"fcm_dpo/q_t": 0.3906324803829193,
"grad_norm": 41.52824020385742,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6889638304710388,
"logits/rejected": -0.676927924156189,
"logps/chosen": -73.29385375976562,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -127.2839126586914,
"loss": 1.1033,
"margin_dpo/margin_mean": 22.005874633789062,
"margin_dpo/margin_std": 39.980369567871094,
"step": 86
},
{
"KL/chosen_KL_mean": -16.871139526367188,
"KL/mean": -29.736377716064453,
"KL/rejected_KL_mean": -42.60161209106445,
"KL/std": 28.82452964782715,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.02494674362242222,
"fcm_dpo/delta": -0.2069552093744278,
"fcm_dpo/margin": 25.73047637939453,
"fcm_dpo/q_t": 0.3635217845439911,
"grad_norm": 27.312652587890625,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.7120091915130615,
"logits/rejected": -0.6871023774147034,
"logps/chosen": -72.50723266601562,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -138.06918334960938,
"loss": 1.0072,
"margin_dpo/margin_mean": 25.7304744720459,
"margin_dpo/margin_std": 34.88175964355469,
"step": 87
},
{
"KL/chosen_KL_mean": -21.348556518554688,
"KL/mean": -31.000858306884766,
"KL/rejected_KL_mean": -40.65315628051758,
"KL/std": 29.925575256347656,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.02458009123802185,
"fcm_dpo/delta": -0.026618685573339462,
"fcm_dpo/margin": 19.304603576660156,
"fcm_dpo/q_t": 0.40038639307022095,
"grad_norm": 28.89271354675293,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6645029783248901,
"logits/rejected": -0.6577020287513733,
"logps/chosen": -95.01970672607422,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -147.36166381835938,
"loss": 1.1144,
"margin_dpo/margin_mean": 19.30460548400879,
"margin_dpo/margin_std": 31.463354110717773,
"step": 88
},
{
"KL/chosen_KL_mean": -12.971105575561523,
"KL/mean": -23.80228614807129,
"KL/rejected_KL_mean": -34.63346862792969,
"KL/std": 25.211496353149414,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.024498917162418365,
"fcm_dpo/delta": -0.08528416603803635,
"fcm_dpo/margin": 21.6623592376709,
"fcm_dpo/q_t": 0.3838860094547272,
"grad_norm": 25.59996795654297,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7227055430412292,
"logits/rejected": -0.6903188228607178,
"logps/chosen": -73.59602355957031,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -116.7170181274414,
"loss": 1.029,
"margin_dpo/margin_mean": 21.662357330322266,
"margin_dpo/margin_std": 28.733211517333984,
"step": 89
},
{
"KL/chosen_KL_mean": -16.759502410888672,
"KL/mean": -31.41651153564453,
"KL/rejected_KL_mean": -46.07352066040039,
"KL/std": 34.36729431152344,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.02360946498811245,
"fcm_dpo/delta": -0.25962328910827637,
"fcm_dpo/margin": 29.314010620117188,
"fcm_dpo/q_t": 0.3660517930984497,
"grad_norm": 26.869991302490234,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6465960144996643,
"logits/rejected": -0.6594746112823486,
"logps/chosen": -70.04481506347656,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -157.61822509765625,
"loss": 1.0136,
"margin_dpo/margin_mean": 29.314010620117188,
"margin_dpo/margin_std": 42.58348083496094,
"step": 90
},
{
"KL/chosen_KL_mean": -17.327058792114258,
"KL/mean": -29.477928161621094,
"KL/rejected_KL_mean": -41.6287956237793,
"KL/std": 27.533260345458984,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.022767014801502228,
"fcm_dpo/delta": -0.10921620577573776,
"fcm_dpo/margin": 24.301740646362305,
"fcm_dpo/q_t": 0.3839709460735321,
"grad_norm": 26.525238037109375,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6998708844184875,
"logits/rejected": -0.6679236888885498,
"logps/chosen": -79.13002014160156,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -129.50274658203125,
"loss": 1.049,
"margin_dpo/margin_mean": 24.301738739013672,
"margin_dpo/margin_std": 35.562110900878906,
"step": 91
},
{
"KL/chosen_KL_mean": -15.585250854492188,
"KL/mean": -27.81812858581543,
"KL/rejected_KL_mean": -40.05100631713867,
"KL/std": 28.193967819213867,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.0223391130566597,
"fcm_dpo/delta": -0.10221868008375168,
"fcm_dpo/margin": 24.46575164794922,
"fcm_dpo/q_t": 0.3836935758590698,
"grad_norm": 24.357402801513672,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.7078873515129089,
"logits/rejected": -0.6847358345985413,
"logps/chosen": -67.22602081298828,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -117.93218994140625,
"loss": 1.0396,
"margin_dpo/margin_mean": 24.46575164794922,
"margin_dpo/margin_std": 34.062713623046875,
"step": 92
},
{
"KL/chosen_KL_mean": -17.586727142333984,
"KL/mean": -29.87631607055664,
"KL/rejected_KL_mean": -42.16590118408203,
"KL/std": 26.71668243408203,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.021821390837430954,
"fcm_dpo/delta": -0.0911676287651062,
"fcm_dpo/margin": 24.579177856445312,
"fcm_dpo/q_t": 0.38265323638916016,
"grad_norm": 24.912883758544922,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7338370084762573,
"logits/rejected": -0.6846834421157837,
"logps/chosen": -70.115966796875,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -119.32664489746094,
"loss": 1.015,
"margin_dpo/margin_mean": 24.57917594909668,
"margin_dpo/margin_std": 30.87820816040039,
"step": 93
},
{
"KL/chosen_KL_mean": -18.805757522583008,
"KL/mean": -32.98207092285156,
"KL/rejected_KL_mean": -47.158382415771484,
"KL/std": 30.474504470825195,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.021062176674604416,
"fcm_dpo/delta": -0.1582311987876892,
"fcm_dpo/margin": 28.35263442993164,
"fcm_dpo/q_t": 0.3686581254005432,
"grad_norm": 24.569002151489258,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.6481120586395264,
"logits/rejected": -0.6321585178375244,
"logps/chosen": -80.02836608886719,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -146.75741577148438,
"loss": 0.9756,
"margin_dpo/margin_mean": 28.35263442993164,
"margin_dpo/margin_std": 31.874601364135742,
"step": 94
},
{
"KL/chosen_KL_mean": -17.881004333496094,
"KL/mean": -29.28811264038086,
"KL/rejected_KL_mean": -40.695228576660156,
"KL/std": 28.964645385742188,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.020857345312833786,
"fcm_dpo/delta": -0.02749776653945446,
"fcm_dpo/margin": 22.814218521118164,
"fcm_dpo/q_t": 0.3923851251602173,
"grad_norm": 23.259233474731445,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.6979963779449463,
"logits/rejected": -0.6698317527770996,
"logps/chosen": -70.40464782714844,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -116.57557678222656,
"loss": 1.0577,
"margin_dpo/margin_mean": 22.81421661376953,
"margin_dpo/margin_std": 30.32460594177246,
"step": 95
},
{
"KL/chosen_KL_mean": -17.336549758911133,
"KL/mean": -32.80950927734375,
"KL/rejected_KL_mean": -48.282470703125,
"KL/std": 33.05143737792969,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.02017274498939514,
"fcm_dpo/delta": -0.18848399817943573,
"fcm_dpo/margin": 30.945920944213867,
"fcm_dpo/q_t": 0.365802526473999,
"grad_norm": 24.085433959960938,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.7053977251052856,
"logits/rejected": -0.6761200428009033,
"logps/chosen": -79.49352264404297,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -144.87847900390625,
"loss": 0.967,
"margin_dpo/margin_mean": 30.945919036865234,
"margin_dpo/margin_std": 36.34577560424805,
"step": 96
},
{
"KL/chosen_KL_mean": -18.55299186706543,
"KL/mean": -30.541301727294922,
"KL/rejected_KL_mean": -42.52961349487305,
"KL/std": 27.393390655517578,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.020200129598379135,
"fcm_dpo/delta": -0.03740522265434265,
"fcm_dpo/margin": 23.976627349853516,
"fcm_dpo/q_t": 0.3891039192676544,
"grad_norm": 24.69647789001465,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6304230690002441,
"logits/rejected": -0.5861343145370483,
"logps/chosen": -73.19935607910156,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -119.49436950683594,
"loss": 1.0367,
"margin_dpo/margin_mean": 23.97662925720215,
"margin_dpo/margin_std": 27.26099967956543,
"step": 97
},
{
"KL/chosen_KL_mean": -22.866867065429688,
"KL/mean": -36.578529357910156,
"KL/rejected_KL_mean": -50.29020690917969,
"KL/std": 31.22027587890625,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.01966441236436367,
"fcm_dpo/delta": -0.09459578990936279,
"fcm_dpo/margin": 27.423343658447266,
"fcm_dpo/q_t": 0.37884292006492615,
"grad_norm": 25.7805233001709,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.6840416193008423,
"logits/rejected": -0.6680232286453247,
"logps/chosen": -88.12549591064453,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -146.81768798828125,
"loss": 1.0413,
"margin_dpo/margin_mean": 27.423343658447266,
"margin_dpo/margin_std": 37.329193115234375,
"step": 98
},
{
"KL/chosen_KL_mean": -18.138586044311523,
"KL/mean": -34.79328918457031,
"KL/rejected_KL_mean": -51.44799041748047,
"KL/std": 33.86750411987305,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.01898646354675293,
"fcm_dpo/delta": -0.1955394297838211,
"fcm_dpo/margin": 33.30940246582031,
"fcm_dpo/q_t": 0.36740559339523315,
"grad_norm": 22.372419357299805,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.656816840171814,
"logits/rejected": -0.6575205326080322,
"logps/chosen": -63.777069091796875,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -137.88592529296875,
"loss": 0.9756,
"margin_dpo/margin_mean": 33.30940246582031,
"margin_dpo/margin_std": 42.03821563720703,
"step": 99
},
{
"KL/chosen_KL_mean": -20.17072868347168,
"KL/mean": -29.92642593383789,
"KL/rejected_KL_mean": -39.6821174621582,
"KL/std": 27.05113983154297,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.019159015268087387,
"fcm_dpo/delta": 0.0783969983458519,
"fcm_dpo/margin": 19.511388778686523,
"fcm_dpo/q_t": 0.41671568155288696,
"grad_norm": 25.289337158203125,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6635388135910034,
"logits/rejected": -0.6208417415618896,
"logps/chosen": -77.76470947265625,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -113.74232482910156,
"loss": 1.155,
"margin_dpo/margin_mean": 19.511388778686523,
"margin_dpo/margin_std": 34.8048210144043,
"step": 100
},
{
"KL/chosen_KL_mean": -24.82254409790039,
"KL/mean": -36.69569396972656,
"KL/rejected_KL_mean": -48.568843841552734,
"KL/std": 33.368690490722656,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.01902778260409832,
"fcm_dpo/delta": -0.0026979856193065643,
"fcm_dpo/margin": 23.746295928955078,
"fcm_dpo/q_t": 0.4053969383239746,
"grad_norm": 25.20921516418457,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6735844612121582,
"logits/rejected": -0.6423518657684326,
"logps/chosen": -86.47139739990234,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -131.75852966308594,
"loss": 1.116,
"margin_dpo/margin_mean": 23.746295928955078,
"margin_dpo/margin_std": 40.306617736816406,
"step": 101
},
{
"KL/chosen_KL_mean": -26.947221755981445,
"KL/mean": -35.72987747192383,
"KL/rejected_KL_mean": -44.512535095214844,
"KL/std": 31.34524917602539,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.019104380160570145,
"fcm_dpo/delta": -0.019020358100533485,
"fcm_dpo/margin": 17.565311431884766,
"fcm_dpo/q_t": 0.4265472888946533,
"grad_norm": 29.04865837097168,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.6863038539886475,
"logits/rejected": -0.6410502195358276,
"logps/chosen": -91.0261001586914,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -112.69961547851562,
"loss": 1.1984,
"margin_dpo/margin_mean": 17.565311431884766,
"margin_dpo/margin_std": 36.97379684448242,
"step": 102
},
{
"KL/chosen_KL_mean": -21.086196899414062,
"KL/mean": -38.403202056884766,
"KL/rejected_KL_mean": -55.7202033996582,
"KL/std": 34.654380798339844,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.018466107547283173,
"fcm_dpo/delta": -0.20299461483955383,
"fcm_dpo/margin": 34.63400650024414,
"fcm_dpo/q_t": 0.3635409474372864,
"grad_norm": 24.053295135498047,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6788144111633301,
"logits/rejected": -0.6451829671859741,
"logps/chosen": -82.3854751586914,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -149.29290771484375,
"loss": 0.965,
"margin_dpo/margin_mean": 34.63400650024414,
"margin_dpo/margin_std": 41.585411071777344,
"step": 103
},
{
"KL/chosen_KL_mean": -23.070478439331055,
"KL/mean": -39.50807571411133,
"KL/rejected_KL_mean": -55.94567108154297,
"KL/std": 38.31330871582031,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.01794702559709549,
"fcm_dpo/delta": -0.14857444167137146,
"fcm_dpo/margin": 32.87519836425781,
"fcm_dpo/q_t": 0.37896206974983215,
"grad_norm": 23.492591857910156,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.692119836807251,
"logits/rejected": -0.6608579158782959,
"logps/chosen": -77.44325256347656,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -145.51039123535156,
"loss": 1.0276,
"margin_dpo/margin_mean": 32.87519454956055,
"margin_dpo/margin_std": 46.670677185058594,
"step": 104
},
{
"KL/chosen_KL_mean": -21.43729591369629,
"KL/mean": -44.80780792236328,
"KL/rejected_KL_mean": -68.1783218383789,
"KL/std": 37.48930740356445,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.016852889209985733,
"fcm_dpo/delta": -0.3663092255592346,
"fcm_dpo/margin": 46.741024017333984,
"fcm_dpo/q_t": 0.3251432180404663,
"grad_norm": 23.383228302001953,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.6901790499687195,
"logits/rejected": -0.6680899858474731,
"logps/chosen": -76.07624053955078,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -166.15184020996094,
"loss": 0.8472,
"margin_dpo/margin_mean": 46.741024017333984,
"margin_dpo/margin_std": 41.229366302490234,
"step": 105
},
{
"KL/chosen_KL_mean": -24.016298294067383,
"KL/mean": -38.636592864990234,
"KL/rejected_KL_mean": -53.25688934326172,
"KL/std": 32.439598083496094,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.016409728676080704,
"fcm_dpo/delta": -0.03146550431847572,
"fcm_dpo/margin": 29.24058723449707,
"fcm_dpo/q_t": 0.3909507989883423,
"grad_norm": 23.02033233642578,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.6366252899169922,
"logits/rejected": -0.6048162579536438,
"logps/chosen": -78.84919738769531,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -138.48150634765625,
"loss": 1.0408,
"margin_dpo/margin_mean": 29.24058723449707,
"margin_dpo/margin_std": 35.340091705322266,
"step": 106
},
{
"KL/chosen_KL_mean": -28.679424285888672,
"KL/mean": -45.09428405761719,
"KL/rejected_KL_mean": -61.50914001464844,
"KL/std": 39.42516326904297,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.016135554760694504,
"fcm_dpo/delta": -0.084980808198452,
"fcm_dpo/margin": 32.82972717285156,
"fcm_dpo/q_t": 0.3815375864505768,
"grad_norm": 22.050548553466797,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.671083390712738,
"logits/rejected": -0.6238175630569458,
"logps/chosen": -98.38723754882812,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -156.2486572265625,
"loss": 1.0287,
"margin_dpo/margin_mean": 32.82972717285156,
"margin_dpo/margin_std": 41.5906982421875,
"step": 107
},
{
"KL/chosen_KL_mean": -25.770402908325195,
"KL/mean": -45.36616516113281,
"KL/rejected_KL_mean": -64.9619369506836,
"KL/std": 43.32636642456055,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.015721427276730537,
"fcm_dpo/delta": -0.17703697085380554,
"fcm_dpo/margin": 39.1915283203125,
"fcm_dpo/q_t": 0.3744971752166748,
"grad_norm": 22.43386459350586,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.6669068336486816,
"logits/rejected": -0.6445153951644897,
"logps/chosen": -81.78028869628906,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -160.75794982910156,
"loss": 1.0232,
"margin_dpo/margin_mean": 39.1915283203125,
"margin_dpo/margin_std": 55.913177490234375,
"step": 108
},
{
"KL/chosen_KL_mean": -23.891096115112305,
"KL/mean": -44.87135314941406,
"KL/rejected_KL_mean": -65.85160827636719,
"KL/std": 41.50886535644531,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.015076124109327793,
"fcm_dpo/delta": -0.19590097665786743,
"fcm_dpo/margin": 41.96051788330078,
"fcm_dpo/q_t": 0.3611322045326233,
"grad_norm": 23.130586624145508,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.6083803176879883,
"logits/rejected": -0.5868882536888123,
"logps/chosen": -86.77658081054688,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -164.537353515625,
"loss": 0.9673,
"margin_dpo/margin_mean": 41.96051788330078,
"margin_dpo/margin_std": 48.454368591308594,
"step": 109
},
{
"KL/chosen_KL_mean": -24.484817504882812,
"KL/mean": -43.37567901611328,
"KL/rejected_KL_mean": -62.26654815673828,
"KL/std": 41.369407653808594,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.014548167586326599,
"fcm_dpo/delta": -0.11019857972860336,
"fcm_dpo/margin": 37.78173065185547,
"fcm_dpo/q_t": 0.38073986768722534,
"grad_norm": 20.18339729309082,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.6229938268661499,
"logits/rejected": -0.5868622064590454,
"logps/chosen": -83.2385025024414,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -142.01657104492188,
"loss": 1.0352,
"margin_dpo/margin_mean": 37.78173065185547,
"margin_dpo/margin_std": 49.614540100097656,
"step": 110
},
{
"KL/chosen_KL_mean": -28.117158889770508,
"KL/mean": -49.45366668701172,
"KL/rejected_KL_mean": -70.79017639160156,
"KL/std": 46.68696594238281,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.014208821579813957,
"fcm_dpo/delta": -0.16805897653102875,
"fcm_dpo/margin": 42.67301940917969,
"fcm_dpo/q_t": 0.3700113892555237,
"grad_norm": 22.384057998657227,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.6600044965744019,
"logits/rejected": -0.6378269195556641,
"logps/chosen": -96.74126434326172,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -169.2190399169922,
"loss": 1.0153,
"margin_dpo/margin_mean": 42.67301940917969,
"margin_dpo/margin_std": 57.135040283203125,
"step": 111
},
{
"KL/chosen_KL_mean": -25.888656616210938,
"KL/mean": -39.50413131713867,
"KL/rejected_KL_mean": -53.119606018066406,
"KL/std": 32.67273712158203,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.01424017921090126,
"fcm_dpo/delta": 0.06461413204669952,
"fcm_dpo/margin": 27.23094940185547,
"fcm_dpo/q_t": 0.41214606165885925,
"grad_norm": 20.815696716308594,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.6311874985694885,
"logits/rejected": -0.5913982391357422,
"logps/chosen": -76.13829803466797,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -117.89403533935547,
"loss": 1.1106,
"margin_dpo/margin_mean": 27.23094940185547,
"margin_dpo/margin_std": 39.252777099609375,
"step": 112
},
{
"KL/chosen_KL_mean": -32.33839416503906,
"KL/mean": -47.10094451904297,
"KL/rejected_KL_mean": -61.863487243652344,
"KL/std": 33.731163024902344,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.014409849420189857,
"fcm_dpo/delta": 0.02563050016760826,
"fcm_dpo/margin": 29.525096893310547,
"fcm_dpo/q_t": 0.4029633700847626,
"grad_norm": 21.0351505279541,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.5959609746932983,
"logits/rejected": -0.5505021810531616,
"logps/chosen": -99.05134582519531,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -139.83218383789062,
"loss": 1.0727,
"margin_dpo/margin_mean": 29.525096893310547,
"margin_dpo/margin_std": 37.40425109863281,
"step": 113
},
{
"KL/chosen_KL_mean": -29.493261337280273,
"KL/mean": -52.80317306518555,
"KL/rejected_KL_mean": -76.11308288574219,
"KL/std": 50.801734924316406,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.013878151774406433,
"fcm_dpo/delta": -0.21314632892608643,
"fcm_dpo/margin": 46.619815826416016,
"fcm_dpo/q_t": 0.36664098501205444,
"grad_norm": 22.067481994628906,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.624281644821167,
"logits/rejected": -0.5919479131698608,
"logps/chosen": -87.2783432006836,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -163.22274780273438,
"loss": 0.9781,
"margin_dpo/margin_mean": 46.61981201171875,
"margin_dpo/margin_std": 61.55342483520508,
"step": 114
},
{
"KL/chosen_KL_mean": -38.72612380981445,
"KL/mean": -54.643898010253906,
"KL/rejected_KL_mean": -70.56167602539062,
"KL/std": 48.15009307861328,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.013882124796509743,
"fcm_dpo/delta": 0.00821240060031414,
"fcm_dpo/margin": 31.83555030822754,
"fcm_dpo/q_t": 0.408986359834671,
"grad_norm": 27.751873016357422,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.6329885721206665,
"logits/rejected": -0.6110441088676453,
"logps/chosen": -104.30876159667969,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -169.127197265625,
"loss": 1.1543,
"margin_dpo/margin_mean": 31.835548400878906,
"margin_dpo/margin_std": 63.04792022705078,
"step": 115
},
{
"KL/chosen_KL_mean": -28.92960548400879,
"KL/mean": -46.47405242919922,
"KL/rejected_KL_mean": -64.01849365234375,
"KL/std": 40.528472900390625,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.013833288103342056,
"fcm_dpo/delta": -0.03747085481882095,
"fcm_dpo/margin": 35.08888626098633,
"fcm_dpo/q_t": 0.39196979999542236,
"grad_norm": 22.5377254486084,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.6246926784515381,
"logits/rejected": -0.6135732531547546,
"logps/chosen": -80.32991790771484,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -144.54034423828125,
"loss": 1.0413,
"margin_dpo/margin_mean": 35.08888626098633,
"margin_dpo/margin_std": 43.86783218383789,
"step": 116
},
{
"KL/chosen_KL_mean": -36.64705276489258,
"KL/mean": -51.39534378051758,
"KL/rejected_KL_mean": -66.14363098144531,
"KL/std": 43.24048614501953,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.013833757489919662,
"fcm_dpo/delta": 0.04373066499829292,
"fcm_dpo/margin": 29.496578216552734,
"fcm_dpo/q_t": 0.40985727310180664,
"grad_norm": 30.1611328125,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.550119161605835,
"logits/rejected": -0.507682204246521,
"logps/chosen": -105.94546508789062,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -132.72763061523438,
"loss": 1.1218,
"margin_dpo/margin_mean": 29.496578216552734,
"margin_dpo/margin_std": 48.89860916137695,
"step": 117
},
{
"KL/chosen_KL_mean": -29.291458129882812,
"KL/mean": -46.83741760253906,
"KL/rejected_KL_mean": -64.38337707519531,
"KL/std": 39.068748474121094,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.013796643353998661,
"fcm_dpo/delta": -0.035874899476766586,
"fcm_dpo/margin": 35.0919189453125,
"fcm_dpo/q_t": 0.392580509185791,
"grad_norm": 21.916383743286133,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.655544638633728,
"logits/rejected": -0.6371685266494751,
"logps/chosen": -84.93243408203125,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -140.05242919921875,
"loss": 1.0436,
"margin_dpo/margin_mean": 35.0919189453125,
"margin_dpo/margin_std": 44.61895751953125,
"step": 118
},
{
"KL/chosen_KL_mean": -40.089935302734375,
"KL/mean": -57.477745056152344,
"KL/rejected_KL_mean": -74.86555480957031,
"KL/std": 43.875762939453125,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.013616050593554974,
"fcm_dpo/delta": -0.025850612670183182,
"fcm_dpo/margin": 34.7756233215332,
"fcm_dpo/q_t": 0.3989412188529968,
"grad_norm": 23.656314849853516,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.6302667260169983,
"logits/rejected": -0.6170446872711182,
"logps/chosen": -113.60012817382812,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -177.84283447265625,
"loss": 1.0851,
"margin_dpo/margin_mean": 34.7756233215332,
"margin_dpo/margin_std": 53.304588317871094,
"step": 119
},
{
"KL/chosen_KL_mean": -41.21659469604492,
"KL/mean": -65.18665313720703,
"KL/rejected_KL_mean": -89.15672302246094,
"KL/std": 54.75802230834961,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.013283832930028439,
"fcm_dpo/delta": -0.1996966004371643,
"fcm_dpo/margin": 47.94011688232422,
"fcm_dpo/q_t": 0.3626715838909149,
"grad_norm": 23.151220321655273,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6102467775344849,
"logits/rejected": -0.5748085975646973,
"logps/chosen": -117.99742889404297,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -197.1804656982422,
"loss": 0.9707,
"margin_dpo/margin_mean": 47.94011688232422,
"margin_dpo/margin_std": 57.357337951660156,
"step": 120
},
{
"KL/chosen_KL_mean": -40.084228515625,
"KL/mean": -66.49555969238281,
"KL/rejected_KL_mean": -92.90689086914062,
"KL/std": 53.775028228759766,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.012637857347726822,
"fcm_dpo/delta": -0.23500870168209076,
"fcm_dpo/margin": 52.822654724121094,
"fcm_dpo/q_t": 0.35974642634391785,
"grad_norm": 25.187885284423828,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.5842880010604858,
"logits/rejected": -0.5794540643692017,
"logps/chosen": -101.87413024902344,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -202.90145874023438,
"loss": 0.9767,
"margin_dpo/margin_mean": 52.822654724121094,
"margin_dpo/margin_std": 67.26119995117188,
"step": 121
},
{
"KL/chosen_KL_mean": -36.656639099121094,
"KL/mean": -69.3306655883789,
"KL/rejected_KL_mean": -102.00469970703125,
"KL/std": 61.23943328857422,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.011914310976862907,
"fcm_dpo/delta": -0.3561996519565582,
"fcm_dpo/margin": 65.34806060791016,
"fcm_dpo/q_t": 0.3318769931793213,
"grad_norm": 24.457799911499023,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.5659698843955994,
"logits/rejected": -0.581313967704773,
"logps/chosen": -83.55885314941406,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -208.71888732910156,
"loss": 0.8701,
"margin_dpo/margin_mean": 65.34805297851562,
"margin_dpo/margin_std": 63.61327362060547,
"step": 122
},
{
"KL/chosen_KL_mean": -39.99665069580078,
"KL/mean": -61.610984802246094,
"KL/rejected_KL_mean": -83.22531127929688,
"KL/std": 52.34938430786133,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.011532999575138092,
"fcm_dpo/delta": -0.05208038166165352,
"fcm_dpo/margin": 43.228668212890625,
"fcm_dpo/q_t": 0.3948003053665161,
"grad_norm": 21.98365592956543,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.574738085269928,
"logits/rejected": -0.556842565536499,
"logps/chosen": -101.33528900146484,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -171.00070190429688,
"loss": 1.092,
"margin_dpo/margin_mean": 43.228668212890625,
"margin_dpo/margin_std": 69.88395690917969,
"step": 123
},
{
"KL/chosen_KL_mean": -45.20293426513672,
"KL/mean": -76.88467407226562,
"KL/rejected_KL_mean": -108.56641387939453,
"KL/std": 70.95067596435547,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.011075865477323532,
"fcm_dpo/delta": -0.2719235420227051,
"fcm_dpo/margin": 63.36347198486328,
"fcm_dpo/q_t": 0.36204931139945984,
"grad_norm": 23.336181640625,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.5531664490699768,
"logits/rejected": -0.5544106960296631,
"logps/chosen": -116.6512680053711,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -226.14697265625,
"loss": 0.9951,
"margin_dpo/margin_mean": 63.36347198486328,
"margin_dpo/margin_std": 89.0818099975586,
"step": 124
},
{
"KL/chosen_KL_mean": -37.31425476074219,
"KL/mean": -65.44857788085938,
"KL/rejected_KL_mean": -93.58290100097656,
"KL/std": 59.623497009277344,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.010716570541262627,
"fcm_dpo/delta": -0.16258446872234344,
"fcm_dpo/margin": 56.268646240234375,
"fcm_dpo/q_t": 0.37255388498306274,
"grad_norm": 20.094148635864258,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.513544499874115,
"logits/rejected": -0.5137706995010376,
"logps/chosen": -87.4511947631836,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -177.57150268554688,
"loss": 1.0046,
"margin_dpo/margin_mean": 56.26864242553711,
"margin_dpo/margin_std": 73.41596984863281,
"step": 125
},
{
"KL/chosen_KL_mean": -40.404869079589844,
"KL/mean": -66.45927429199219,
"KL/rejected_KL_mean": -92.51365661621094,
"KL/std": 53.786094665527344,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.010430238209664822,
"fcm_dpo/delta": -0.09915146231651306,
"fcm_dpo/margin": 52.10879898071289,
"fcm_dpo/q_t": 0.3821665048599243,
"grad_norm": 21.80858039855957,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.545072078704834,
"logits/rejected": -0.5383429527282715,
"logps/chosen": -96.07194519042969,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -190.64341735839844,
"loss": 1.0239,
"margin_dpo/margin_mean": 52.10879898071289,
"margin_dpo/margin_std": 67.88806915283203,
"step": 126
},
{
"KL/chosen_KL_mean": -39.16326904296875,
"KL/mean": -58.659423828125,
"KL/rejected_KL_mean": -78.15557861328125,
"KL/std": 47.13392639160156,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.010470317676663399,
"fcm_dpo/delta": 0.043486855924129486,
"fcm_dpo/margin": 38.9923095703125,
"fcm_dpo/q_t": 0.406957745552063,
"grad_norm": 21.821304321289062,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.5330368280410767,
"logits/rejected": -0.5118523240089417,
"logps/chosen": -95.71794891357422,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -154.95135498046875,
"loss": 1.1078,
"margin_dpo/margin_mean": 38.9923095703125,
"margin_dpo/margin_std": 58.90929412841797,
"step": 127
},
{
"KL/chosen_KL_mean": -42.080467224121094,
"KL/mean": -62.39935302734375,
"KL/rejected_KL_mean": -82.71824645996094,
"KL/std": 55.34471130371094,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.010566205717623234,
"fcm_dpo/delta": 0.02107478305697441,
"fcm_dpo/margin": 40.63777160644531,
"fcm_dpo/q_t": 0.4061205983161926,
"grad_norm": 30.90243148803711,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.5452651977539062,
"logits/rejected": -0.5237852334976196,
"logps/chosen": -100.20142364501953,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -159.15721130371094,
"loss": 1.1231,
"margin_dpo/margin_mean": 40.637779235839844,
"margin_dpo/margin_std": 68.70756530761719,
"step": 128
},
{
"KL/chosen_KL_mean": -50.868858337402344,
"KL/mean": -72.29450988769531,
"KL/rejected_KL_mean": -93.72015380859375,
"KL/std": 60.7120361328125,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.010539564304053783,
"fcm_dpo/delta": -0.0017273910343647003,
"fcm_dpo/margin": 42.851295471191406,
"fcm_dpo/q_t": 0.40687721967697144,
"grad_norm": 22.193269729614258,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.5510047674179077,
"logits/rejected": -0.53895103931427,
"logps/chosen": -117.78523254394531,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -190.36236572265625,
"loss": 1.1293,
"margin_dpo/margin_mean": 42.85129928588867,
"margin_dpo/margin_std": 78.5627212524414,
"step": 129
},
{
"KL/chosen_KL_mean": -37.846351623535156,
"KL/mean": -69.90611267089844,
"KL/rejected_KL_mean": -101.96586608886719,
"KL/std": 63.394935607910156,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.010300353169441223,
"fcm_dpo/delta": -0.22541257739067078,
"fcm_dpo/margin": 64.11952209472656,
"fcm_dpo/q_t": 0.3606530427932739,
"grad_norm": 21.44586753845215,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.5086920261383057,
"logits/rejected": -0.5030478239059448,
"logps/chosen": -82.51319885253906,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -184.74752807617188,
"loss": 0.9727,
"margin_dpo/margin_mean": 64.11951446533203,
"margin_dpo/margin_std": 76.44377136230469,
"step": 130
},
{
"KL/chosen_KL_mean": -33.34912872314453,
"KL/mean": -65.05560302734375,
"KL/rejected_KL_mean": -96.7620849609375,
"KL/std": 61.83128356933594,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.009780805557966232,
"fcm_dpo/delta": -0.1818436086177826,
"fcm_dpo/margin": 63.41294860839844,
"fcm_dpo/q_t": 0.36090847849845886,
"grad_norm": 28.479732513427734,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.5144104957580566,
"logits/rejected": -0.5210058689117432,
"logps/chosen": -78.27371978759766,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -185.20608520507812,
"loss": 0.9571,
"margin_dpo/margin_mean": 63.41294860839844,
"margin_dpo/margin_std": 67.7553939819336,
"step": 131
},
{
"KL/chosen_KL_mean": -44.536895751953125,
"KL/mean": -71.02592468261719,
"KL/rejected_KL_mean": -97.51495361328125,
"KL/std": 62.367454528808594,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.00959692895412445,
"fcm_dpo/delta": -0.061767518520355225,
"fcm_dpo/margin": 52.97805404663086,
"fcm_dpo/q_t": 0.39321261644363403,
"grad_norm": 21.923797607421875,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.53538578748703,
"logits/rejected": -0.5309890508651733,
"logps/chosen": -103.53797912597656,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -185.4071044921875,
"loss": 1.0649,
"margin_dpo/margin_mean": 52.978057861328125,
"margin_dpo/margin_std": 80.51556396484375,
"step": 132
},
{
"KL/chosen_KL_mean": -52.77239990234375,
"KL/mean": -76.22706604003906,
"KL/rejected_KL_mean": -99.68174743652344,
"KL/std": 56.50593185424805,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.009568007662892342,
"fcm_dpo/delta": 0.0011416934430599213,
"fcm_dpo/margin": 46.909339904785156,
"fcm_dpo/q_t": 0.40291664004325867,
"grad_norm": 27.488916397094727,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.549635648727417,
"logits/rejected": -0.5361599922180176,
"logps/chosen": -119.37689971923828,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -196.0153045654297,
"loss": 1.0885,
"margin_dpo/margin_mean": 46.909339904785156,
"margin_dpo/margin_std": 71.18951416015625,
"step": 133
},
{
"KL/chosen_KL_mean": -41.90770721435547,
"KL/mean": -66.91332244873047,
"KL/rejected_KL_mean": -91.91893768310547,
"KL/std": 55.141109466552734,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.009577769786119461,
"fcm_dpo/delta": -0.030592577531933784,
"fcm_dpo/margin": 50.01123046875,
"fcm_dpo/q_t": 0.39385026693344116,
"grad_norm": 20.826383590698242,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.49232274293899536,
"logits/rejected": -0.4757571816444397,
"logps/chosen": -93.97695922851562,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -179.57345581054688,
"loss": 1.0433,
"margin_dpo/margin_mean": 50.01123046875,
"margin_dpo/margin_std": 63.30561828613281,
"step": 134
},
{
"KL/chosen_KL_mean": -46.68389892578125,
"KL/mean": -82.07420349121094,
"KL/rejected_KL_mean": -117.46450805664062,
"KL/std": 72.77279663085938,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.009226776659488678,
"fcm_dpo/delta": -0.21729746460914612,
"fcm_dpo/margin": 70.78060913085938,
"fcm_dpo/q_t": 0.362811803817749,
"grad_norm": 23.35005760192871,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.5128329396247864,
"logits/rejected": -0.5463232398033142,
"logps/chosen": -97.03776550292969,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -233.44424438476562,
"loss": 0.9735,
"margin_dpo/margin_mean": 70.78060913085938,
"margin_dpo/margin_std": 87.45381164550781,
"step": 135
},
{
"KL/chosen_KL_mean": -55.22693634033203,
"KL/mean": -79.05845642089844,
"KL/rejected_KL_mean": -102.88998413085938,
"KL/std": 65.8864517211914,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.009088754653930664,
"fcm_dpo/delta": 0.016819309443235397,
"fcm_dpo/margin": 47.663047790527344,
"fcm_dpo/q_t": 0.4126465916633606,
"grad_norm": 21.6840877532959,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.5195415616035461,
"logits/rejected": -0.510654091835022,
"logps/chosen": -120.29945373535156,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -199.21121215820312,
"loss": 1.124,
"margin_dpo/margin_mean": 47.663047790527344,
"margin_dpo/margin_std": 83.49158477783203,
"step": 136
},
{
"KL/chosen_KL_mean": -53.765201568603516,
"KL/mean": -90.72117614746094,
"KL/rejected_KL_mean": -127.67715454101562,
"KL/std": 87.34907531738281,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.00885041244328022,
"fcm_dpo/delta": -0.21880337595939636,
"fcm_dpo/margin": 73.91194152832031,
"fcm_dpo/q_t": 0.3690844178199768,
"grad_norm": 19.313182830810547,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.4476552903652191,
"logits/rejected": -0.4745860695838928,
"logps/chosen": -102.52432250976562,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -241.54092407226562,
"loss": 1.006,
"margin_dpo/margin_mean": 73.91194152832031,
"margin_dpo/margin_std": 104.03363037109375,
"step": 137
},
{
"KL/chosen_KL_mean": -55.631492614746094,
"KL/mean": -83.80963134765625,
"KL/rejected_KL_mean": -111.98777770996094,
"KL/std": 66.54393005371094,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.008722840808331966,
"fcm_dpo/delta": -0.04372715577483177,
"fcm_dpo/margin": 56.356292724609375,
"fcm_dpo/q_t": 0.3889402449131012,
"grad_norm": 22.730175018310547,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.4589083194732666,
"logits/rejected": -0.4466325640678406,
"logps/chosen": -116.15113830566406,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -205.18472290039062,
"loss": 1.0345,
"margin_dpo/margin_mean": 56.35628890991211,
"margin_dpo/margin_std": 68.2387466430664,
"step": 138
},
{
"KL/chosen_KL_mean": -46.2087516784668,
"KL/mean": -80.24526977539062,
"KL/rejected_KL_mean": -114.28178405761719,
"KL/std": 63.59034729003906,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.008530115708708763,
"fcm_dpo/delta": -0.13851620256900787,
"fcm_dpo/margin": 68.07303619384766,
"fcm_dpo/q_t": 0.3731107711791992,
"grad_norm": 18.695363998413086,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.4649140238761902,
"logits/rejected": -0.454061895608902,
"logps/chosen": -93.10013580322266,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -194.009765625,
"loss": 0.9904,
"margin_dpo/margin_mean": 68.07304382324219,
"margin_dpo/margin_std": 81.53546905517578,
"step": 139
},
{
"KL/chosen_KL_mean": -56.09660339355469,
"KL/mean": -87.04917907714844,
"KL/rejected_KL_mean": -118.00175476074219,
"KL/std": 71.19284057617188,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.008354827761650085,
"fcm_dpo/delta": -0.07093564420938492,
"fcm_dpo/margin": 61.90514373779297,
"fcm_dpo/q_t": 0.3871724605560303,
"grad_norm": 21.784423828125,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.4650196135044098,
"logits/rejected": -0.4507848024368286,
"logps/chosen": -115.07131958007812,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -201.28585815429688,
"loss": 1.0449,
"margin_dpo/margin_mean": 61.90514373779297,
"margin_dpo/margin_std": 84.87958526611328,
"step": 140
},
{
"KL/chosen_KL_mean": -62.21798324584961,
"KL/mean": -91.52658081054688,
"KL/rejected_KL_mean": -120.83517456054688,
"KL/std": 76.01405334472656,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.00829094648361206,
"fcm_dpo/delta": -0.037813104689121246,
"fcm_dpo/margin": 58.61717987060547,
"fcm_dpo/q_t": 0.3922034502029419,
"grad_norm": 26.592355728149414,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.451934278011322,
"logits/rejected": -0.4369876980781555,
"logps/chosen": -137.29364013671875,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -219.02743530273438,
"loss": 1.0748,
"margin_dpo/margin_mean": 58.61717987060547,
"margin_dpo/margin_std": 87.23815155029297,
"step": 141
},
{
"KL/chosen_KL_mean": -64.0453109741211,
"KL/mean": -97.54293823242188,
"KL/rejected_KL_mean": -131.04055786132812,
"KL/std": 84.23410034179688,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.008179203607141972,
"fcm_dpo/delta": -0.10360373556613922,
"fcm_dpo/margin": 66.99524688720703,
"fcm_dpo/q_t": 0.38415253162384033,
"grad_norm": 26.743053436279297,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.45145779848098755,
"logits/rejected": -0.44982147216796875,
"logps/chosen": -122.0732421875,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -225.62278747558594,
"loss": 1.0591,
"margin_dpo/margin_mean": 66.99524688720703,
"margin_dpo/margin_std": 98.7759780883789,
"step": 142
},
{
"KL/chosen_KL_mean": -68.70494842529297,
"KL/mean": -90.10963439941406,
"KL/rejected_KL_mean": -111.51432037353516,
"KL/std": 74.79816436767578,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.008188603445887566,
"fcm_dpo/delta": 0.1031288281083107,
"fcm_dpo/margin": 42.80937576293945,
"fcm_dpo/q_t": 0.4261817932128906,
"grad_norm": 25.709033966064453,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.42726534605026245,
"logits/rejected": -0.4184831976890564,
"logps/chosen": -126.30139923095703,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -190.51388549804688,
"loss": 1.1816,
"margin_dpo/margin_mean": 42.80937194824219,
"margin_dpo/margin_std": 86.87847900390625,
"step": 143
},
{
"KL/chosen_KL_mean": -59.455833435058594,
"KL/mean": -85.97761535644531,
"KL/rejected_KL_mean": -112.4993896484375,
"KL/std": 62.42079162597656,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.008272461593151093,
"fcm_dpo/delta": 0.011701909825205803,
"fcm_dpo/margin": 53.04355239868164,
"fcm_dpo/q_t": 0.4019629955291748,
"grad_norm": 21.54189682006836,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.44170573353767395,
"logits/rejected": -0.42558401823043823,
"logps/chosen": -119.36219787597656,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -194.49964904785156,
"loss": 1.0796,
"margin_dpo/margin_mean": 53.043548583984375,
"margin_dpo/margin_std": 73.45362854003906,
"step": 144
},
{
"KL/chosen_KL_mean": -57.76536178588867,
"KL/mean": -86.40901947021484,
"KL/rejected_KL_mean": -115.05267333984375,
"KL/std": 63.70155334472656,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.00824956875294447,
"fcm_dpo/delta": -0.0237555094063282,
"fcm_dpo/margin": 57.287315368652344,
"fcm_dpo/q_t": 0.3944385349750519,
"grad_norm": 24.161712646484375,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.4552566111087799,
"logits/rejected": -0.4353902339935303,
"logps/chosen": -114.36602783203125,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -192.9189910888672,
"loss": 1.0585,
"margin_dpo/margin_mean": 57.287315368652344,
"margin_dpo/margin_std": 76.47909545898438,
"step": 145
},
{
"KL/chosen_KL_mean": -79.07931518554688,
"KL/mean": -101.17206573486328,
"KL/rejected_KL_mean": -123.26481628417969,
"KL/std": 68.17335510253906,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.00832764245569706,
"fcm_dpo/delta": 0.08524463325738907,
"fcm_dpo/margin": 44.185508728027344,
"fcm_dpo/q_t": 0.4177442193031311,
"grad_norm": 29.69412612915039,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.45252707600593567,
"logits/rejected": -0.42418697476387024,
"logps/chosen": -145.07977294921875,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -204.96762084960938,
"loss": 1.1619,
"margin_dpo/margin_mean": 44.185508728027344,
"margin_dpo/margin_std": 81.69918060302734,
"step": 146
},
{
"KL/chosen_KL_mean": -57.46442413330078,
"KL/mean": -87.44345092773438,
"KL/rejected_KL_mean": -117.42247009277344,
"KL/std": 69.19750213623047,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.008307845331728458,
"fcm_dpo/delta": -0.05084875971078873,
"fcm_dpo/margin": 59.958045959472656,
"fcm_dpo/q_t": 0.3916861414909363,
"grad_norm": 21.09212875366211,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.4309794306755066,
"logits/rejected": -0.4036545753479004,
"logps/chosen": -110.86991119384766,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -188.81307983398438,
"loss": 1.0599,
"margin_dpo/margin_mean": 59.958045959472656,
"margin_dpo/margin_std": 85.19206237792969,
"step": 147
},
{
"KL/chosen_KL_mean": -57.138404846191406,
"KL/mean": -82.37816619873047,
"KL/rejected_KL_mean": -107.61793518066406,
"KL/std": 68.84814453125,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.008421191945672035,
"fcm_dpo/delta": 0.023522578179836273,
"fcm_dpo/margin": 50.47953414916992,
"fcm_dpo/q_t": 0.40811100602149963,
"grad_norm": 20.28190803527832,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.503899097442627,
"logits/rejected": -0.4953872859477997,
"logps/chosen": -122.07548522949219,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -210.71177673339844,
"loss": 1.101,
"margin_dpo/margin_mean": 50.47953796386719,
"margin_dpo/margin_std": 72.59159088134766,
"step": 148
},
{
"KL/chosen_KL_mean": -55.076839447021484,
"KL/mean": -86.57733917236328,
"KL/rejected_KL_mean": -118.07785034179688,
"KL/std": 62.685951232910156,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.00818311795592308,
"fcm_dpo/delta": -0.07131742686033249,
"fcm_dpo/margin": 63.00099182128906,
"fcm_dpo/q_t": 0.3839060366153717,
"grad_norm": 19.841922760009766,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.518465518951416,
"logits/rejected": -0.5024739503860474,
"logps/chosen": -113.55059814453125,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -217.39259338378906,
"loss": 1.0235,
"margin_dpo/margin_mean": 63.00099563598633,
"margin_dpo/margin_std": 73.64586639404297,
"step": 149
},
{
"KL/chosen_KL_mean": -48.64092254638672,
"KL/mean": -79.2264404296875,
"KL/rejected_KL_mean": -109.81196594238281,
"KL/std": 73.13420104980469,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.00810123048722744,
"fcm_dpo/delta": -0.04957358539104462,
"fcm_dpo/margin": 61.171043395996094,
"fcm_dpo/q_t": 0.3953554034233093,
"grad_norm": 19.716720581054688,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.46694451570510864,
"logits/rejected": -0.46912962198257446,
"logps/chosen": -94.34673309326172,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -193.15956115722656,
"loss": 1.0563,
"margin_dpo/margin_mean": 61.17103958129883,
"margin_dpo/margin_std": 87.25103759765625,
"step": 150
},
{
"KL/chosen_KL_mean": -60.15590286254883,
"KL/mean": -91.48491668701172,
"KL/rejected_KL_mean": -122.81391906738281,
"KL/std": 69.31367492675781,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.008071625605225563,
"fcm_dpo/delta": -0.058733321726322174,
"fcm_dpo/margin": 62.65802001953125,
"fcm_dpo/q_t": 0.38598495721817017,
"grad_norm": 23.326515197753906,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.5557031631469727,
"logits/rejected": -0.5267741680145264,
"logps/chosen": -130.72674560546875,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -223.27774047851562,
"loss": 1.0271,
"margin_dpo/margin_mean": 62.65802001953125,
"margin_dpo/margin_std": 75.5926742553711,
"step": 151
},
{
"KL/chosen_KL_mean": -52.698150634765625,
"KL/mean": -90.16242980957031,
"KL/rejected_KL_mean": -127.62669372558594,
"KL/std": 71.81861877441406,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.007893717847764492,
"fcm_dpo/delta": -0.15009991824626923,
"fcm_dpo/margin": 74.92855072021484,
"fcm_dpo/q_t": 0.37131232023239136,
"grad_norm": 21.844554901123047,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.5226190090179443,
"logits/rejected": -0.5203348398208618,
"logps/chosen": -112.86253356933594,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -233.76715087890625,
"loss": 0.9881,
"margin_dpo/margin_mean": 74.92855072021484,
"margin_dpo/margin_std": 87.656494140625,
"step": 152
},
{
"KL/chosen_KL_mean": -53.36126708984375,
"KL/mean": -85.40214538574219,
"KL/rejected_KL_mean": -117.44302368164062,
"KL/std": 77.24756622314453,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.0077679818496108055,
"fcm_dpo/delta": -0.050270069390535355,
"fcm_dpo/margin": 64.08175659179688,
"fcm_dpo/q_t": 0.3939558267593384,
"grad_norm": 16.88256072998047,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.504237174987793,
"logits/rejected": -0.4979296326637268,
"logps/chosen": -109.67654418945312,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -203.09884643554688,
"loss": 1.0632,
"margin_dpo/margin_mean": 64.08175659179688,
"margin_dpo/margin_std": 94.27104187011719,
"step": 153
},
{
"KL/chosen_KL_mean": -63.94844055175781,
"KL/mean": -95.09716796875,
"KL/rejected_KL_mean": -126.24589538574219,
"KL/std": 78.61471557617188,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.007730868645012379,
"fcm_dpo/delta": -0.033514946699142456,
"fcm_dpo/margin": 62.29746627807617,
"fcm_dpo/q_t": 0.3967205882072449,
"grad_norm": 20.78262710571289,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.49675068259239197,
"logits/rejected": -0.49814850091934204,
"logps/chosen": -126.69100952148438,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -230.49009704589844,
"loss": 1.095,
"margin_dpo/margin_mean": 62.29746627807617,
"margin_dpo/margin_std": 99.08077239990234,
"step": 154
},
{
"KL/chosen_KL_mean": -58.69048309326172,
"KL/mean": -90.29657745361328,
"KL/rejected_KL_mean": -121.90267944335938,
"KL/std": 73.09174346923828,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.007663263939321041,
"fcm_dpo/delta": -0.03624590486288071,
"fcm_dpo/margin": 63.21219253540039,
"fcm_dpo/q_t": 0.393093466758728,
"grad_norm": 20.922475814819336,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.46945518255233765,
"logits/rejected": -0.446572870016098,
"logps/chosen": -119.34366607666016,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -199.39488220214844,
"loss": 1.0631,
"margin_dpo/margin_mean": 63.21219253540039,
"margin_dpo/margin_std": 89.31080627441406,
"step": 155
},
{
"KL/chosen_KL_mean": -82.07888793945312,
"KL/mean": -100.7304458618164,
"KL/rejected_KL_mean": -119.38200378417969,
"KL/std": 76.68929290771484,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.007798383012413979,
"fcm_dpo/delta": 0.16412018239498138,
"fcm_dpo/margin": 37.30310821533203,
"fcm_dpo/q_t": 0.4359789192676544,
"grad_norm": 33.119998931884766,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.43547916412353516,
"logits/rejected": -0.4247688949108124,
"logps/chosen": -151.57077026367188,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -196.55130004882812,
"loss": 1.259,
"margin_dpo/margin_mean": 37.30310821533203,
"margin_dpo/margin_std": 101.03076934814453,
"step": 156
},
{
"KL/chosen_KL_mean": -74.54649353027344,
"KL/mean": -113.61559295654297,
"KL/rejected_KL_mean": -152.6846923828125,
"KL/std": 83.09423065185547,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.007648843340575695,
"fcm_dpo/delta": -0.15882518887519836,
"fcm_dpo/margin": 78.13821411132812,
"fcm_dpo/q_t": 0.3660867214202881,
"grad_norm": 25.022706985473633,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.41863518953323364,
"logits/rejected": -0.42397525906562805,
"logps/chosen": -135.91493225097656,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -260.3310546875,
"loss": 0.9962,
"margin_dpo/margin_mean": 78.13821411132812,
"margin_dpo/margin_std": 93.55778503417969,
"step": 157
},
{
"KL/chosen_KL_mean": -75.25330352783203,
"KL/mean": -116.46820831298828,
"KL/rejected_KL_mean": -157.68312072753906,
"KL/std": 103.57212829589844,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.007441770285367966,
"fcm_dpo/delta": -0.1740458905696869,
"fcm_dpo/margin": 82.4298095703125,
"fcm_dpo/q_t": 0.3776199221611023,
"grad_norm": 23.29196548461914,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.3899335265159607,
"logits/rejected": -0.40577736496925354,
"logps/chosen": -132.86622619628906,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -271.3778076171875,
"loss": 1.0341,
"margin_dpo/margin_mean": 82.4298095703125,
"margin_dpo/margin_std": 123.92778015136719,
"step": 158
},
{
"KL/chosen_KL_mean": -83.34770202636719,
"KL/mean": -111.96831512451172,
"KL/rejected_KL_mean": -140.58892822265625,
"KL/std": 89.91984558105469,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.0074341027066111565,
"fcm_dpo/delta": 0.025001313537359238,
"fcm_dpo/margin": 57.2412223815918,
"fcm_dpo/q_t": 0.4071953594684601,
"grad_norm": 25.42589569091797,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.3967503309249878,
"logits/rejected": -0.3607752323150635,
"logps/chosen": -164.90805053710938,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -229.48764038085938,
"loss": 1.126,
"margin_dpo/margin_mean": 57.2412223815918,
"margin_dpo/margin_std": 97.32987213134766,
"step": 159
},
{
"KL/chosen_KL_mean": -85.25990295410156,
"KL/mean": -123.54020690917969,
"KL/rejected_KL_mean": -161.82049560546875,
"KL/std": 96.52642059326172,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.007252626121044159,
"fcm_dpo/delta": -0.11243004351854324,
"fcm_dpo/margin": 76.56060791015625,
"fcm_dpo/q_t": 0.3867749571800232,
"grad_norm": 28.038400650024414,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.40755778551101685,
"logits/rejected": -0.3971432149410248,
"logps/chosen": -150.99078369140625,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -259.038330078125,
"loss": 1.0673,
"margin_dpo/margin_mean": 76.56060791015625,
"margin_dpo/margin_std": 120.45240783691406,
"step": 160
},
{
"KL/chosen_KL_mean": -72.6231460571289,
"KL/mean": -106.6167984008789,
"KL/rejected_KL_mean": -140.61044311523438,
"KL/std": 77.12852478027344,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.00720019917935133,
"fcm_dpo/delta": -0.04155520722270012,
"fcm_dpo/margin": 67.9873046875,
"fcm_dpo/q_t": 0.39423298835754395,
"grad_norm": 22.304636001586914,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.39059293270111084,
"logits/rejected": -0.3855125904083252,
"logps/chosen": -125.05961608886719,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -224.04139709472656,
"loss": 1.066,
"margin_dpo/margin_mean": 67.9873046875,
"margin_dpo/margin_std": 97.89100646972656,
"step": 161
},
{
"KL/chosen_KL_mean": -69.07589721679688,
"KL/mean": -103.18696594238281,
"KL/rejected_KL_mean": -137.29803466796875,
"KL/std": 85.45391845703125,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.0070790136232972145,
"fcm_dpo/delta": -0.0367625430226326,
"fcm_dpo/margin": 68.22212982177734,
"fcm_dpo/q_t": 0.39606010913848877,
"grad_norm": 23.20815658569336,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.4767564535140991,
"logits/rejected": -0.4587327837944031,
"logps/chosen": -131.68649291992188,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -226.68861389160156,
"loss": 1.0772,
"margin_dpo/margin_mean": 68.22212219238281,
"margin_dpo/margin_std": 100.68720245361328,
"step": 162
},
{
"KL/chosen_KL_mean": -76.93033599853516,
"KL/mean": -109.9669189453125,
"KL/rejected_KL_mean": -143.0034942626953,
"KL/std": 84.1370849609375,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.007111528422683477,
"fcm_dpo/delta": -0.02085772156715393,
"fcm_dpo/margin": 66.07315826416016,
"fcm_dpo/q_t": 0.398965060710907,
"grad_norm": 21.569955825805664,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.4115701913833618,
"logits/rejected": -0.3913278877735138,
"logps/chosen": -131.95196533203125,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -218.42172241210938,
"loss": 1.0824,
"margin_dpo/margin_mean": 66.07315826416016,
"margin_dpo/margin_std": 100.27389526367188,
"step": 163
},
{
"KL/chosen_KL_mean": -69.56802368164062,
"KL/mean": -109.19029235839844,
"KL/rejected_KL_mean": -148.81256103515625,
"KL/std": 82.19638061523438,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.006935762241482735,
"fcm_dpo/delta": -0.10748938471078873,
"fcm_dpo/margin": 79.24454498291016,
"fcm_dpo/q_t": 0.3782011866569519,
"grad_norm": 21.600542068481445,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.4188717305660248,
"logits/rejected": -0.40261319279670715,
"logps/chosen": -125.20869445800781,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -228.47720336914062,
"loss": 1.0077,
"margin_dpo/margin_mean": 79.24454498291016,
"margin_dpo/margin_std": 94.94876098632812,
"step": 164
},
{
"KL/chosen_KL_mean": -75.52891540527344,
"KL/mean": -101.27418518066406,
"KL/rejected_KL_mean": -127.01945495605469,
"KL/std": 68.8742446899414,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.00702784676104784,
"fcm_dpo/delta": 0.0915396437048912,
"fcm_dpo/margin": 51.49053192138672,
"fcm_dpo/q_t": 0.41989004611968994,
"grad_norm": 28.339448928833008,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.4348529577255249,
"logits/rejected": -0.4037681818008423,
"logps/chosen": -136.839599609375,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -200.6900634765625,
"loss": 1.1525,
"margin_dpo/margin_mean": 51.49053192138672,
"margin_dpo/margin_std": 90.902587890625,
"step": 165
},
{
"KL/chosen_KL_mean": -67.45067596435547,
"KL/mean": -114.44827270507812,
"KL/rejected_KL_mean": -161.4458770751953,
"KL/std": 100.94361114501953,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.006829878315329552,
"fcm_dpo/delta": -0.2068059891462326,
"fcm_dpo/margin": 93.99519348144531,
"fcm_dpo/q_t": 0.3710244297981262,
"grad_norm": 18.604509353637695,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.3967953324317932,
"logits/rejected": -0.4072214663028717,
"logps/chosen": -118.43428039550781,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -259.541015625,
"loss": 0.9903,
"margin_dpo/margin_mean": 93.99519348144531,
"margin_dpo/margin_std": 126.79255676269531,
"step": 166
},
{
"KL/chosen_KL_mean": -68.31707000732422,
"KL/mean": -118.00259399414062,
"KL/rejected_KL_mean": -167.6881103515625,
"KL/std": 92.47897338867188,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.006573098246008158,
"fcm_dpo/delta": -0.21753078699111938,
"fcm_dpo/margin": 99.37103271484375,
"fcm_dpo/q_t": 0.3561442792415619,
"grad_norm": 21.904842376708984,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.3720051348209381,
"logits/rejected": -0.37385571002960205,
"logps/chosen": -118.74116516113281,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -263.7185363769531,
"loss": 0.94,
"margin_dpo/margin_mean": 99.37103271484375,
"margin_dpo/margin_std": 105.42371368408203,
"step": 167
},
{
"KL/chosen_KL_mean": -75.26981353759766,
"KL/mean": -112.29487609863281,
"KL/rejected_KL_mean": -149.31991577148438,
"KL/std": 87.42570495605469,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.006460403557866812,
"fcm_dpo/delta": -0.029986891895532608,
"fcm_dpo/margin": 74.05010986328125,
"fcm_dpo/q_t": 0.3942164182662964,
"grad_norm": 20.635385513305664,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.3801596760749817,
"logits/rejected": -0.36851075291633606,
"logps/chosen": -124.73263549804688,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -224.62847900390625,
"loss": 1.0505,
"margin_dpo/margin_mean": 74.05010223388672,
"margin_dpo/margin_std": 96.77279663085938,
"step": 168
},
{
"KL/chosen_KL_mean": -77.44679260253906,
"KL/mean": -110.11390686035156,
"KL/rejected_KL_mean": -142.78103637695312,
"KL/std": 87.40264892578125,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.006434156559407711,
"fcm_dpo/delta": 0.029472671449184418,
"fcm_dpo/margin": 65.33423614501953,
"fcm_dpo/q_t": 0.4121074974536896,
"grad_norm": 23.882349014282227,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.3922446370124817,
"logits/rejected": -0.37415844202041626,
"logps/chosen": -137.250244140625,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -226.1267852783203,
"loss": 1.1416,
"margin_dpo/margin_mean": 65.33423614501953,
"margin_dpo/margin_std": 119.43335723876953,
"step": 169
},
{
"KL/chosen_KL_mean": -69.79130554199219,
"KL/mean": -108.45357513427734,
"KL/rejected_KL_mean": -147.1158447265625,
"KL/std": 82.628173828125,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.006413028109818697,
"fcm_dpo/delta": -0.049491968005895615,
"fcm_dpo/margin": 77.32452392578125,
"fcm_dpo/q_t": 0.38836145401000977,
"grad_norm": 19.305862426757812,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.35024935007095337,
"logits/rejected": -0.3379005193710327,
"logps/chosen": -119.26307678222656,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -223.03317260742188,
"loss": 1.0442,
"margin_dpo/margin_mean": 77.32452392578125,
"margin_dpo/margin_std": 98.25650024414062,
"step": 170
},
{
"KL/chosen_KL_mean": -100.06165313720703,
"KL/mean": -131.94424438476562,
"KL/rejected_KL_mean": -163.8268280029297,
"KL/std": 95.37467956542969,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.006473253481090069,
"fcm_dpo/delta": 0.038848720490932465,
"fcm_dpo/margin": 63.765167236328125,
"fcm_dpo/q_t": 0.41463029384613037,
"grad_norm": 26.568737030029297,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.3878970742225647,
"logits/rejected": -0.36740678548812866,
"logps/chosen": -184.56097412109375,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -273.20892333984375,
"loss": 1.1467,
"margin_dpo/margin_mean": 63.76517105102539,
"margin_dpo/margin_std": 121.90130615234375,
"step": 171
},
{
"KL/chosen_KL_mean": -90.25933837890625,
"KL/mean": -123.08937072753906,
"KL/rejected_KL_mean": -155.91940307617188,
"KL/std": 93.85934448242188,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.006503078620880842,
"fcm_dpo/delta": 0.02402586117386818,
"fcm_dpo/margin": 65.66006469726562,
"fcm_dpo/q_t": 0.4063677191734314,
"grad_norm": 22.084125518798828,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.4236146807670593,
"logits/rejected": -0.4051532745361328,
"logps/chosen": -158.9132537841797,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -241.3560791015625,
"loss": 1.1088,
"margin_dpo/margin_mean": 65.66006469726562,
"margin_dpo/margin_std": 105.15103149414062,
"step": 172
},
{
"KL/chosen_KL_mean": -82.02645874023438,
"KL/mean": -114.88996887207031,
"KL/rejected_KL_mean": -147.7534637451172,
"KL/std": 86.57609558105469,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.006563690025359392,
"fcm_dpo/delta": 0.01906364969909191,
"fcm_dpo/margin": 65.72700500488281,
"fcm_dpo/q_t": 0.403574675321579,
"grad_norm": 22.174354553222656,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.43103909492492676,
"logits/rejected": -0.4012182354927063,
"logps/chosen": -145.07733154296875,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -226.4373779296875,
"loss": 1.0803,
"margin_dpo/margin_mean": 65.72700500488281,
"margin_dpo/margin_std": 90.32469940185547,
"step": 173
},
{
"KL/chosen_KL_mean": -78.2431411743164,
"KL/mean": -117.16438293457031,
"KL/rejected_KL_mean": -156.0856170654297,
"KL/std": 92.78438568115234,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.006528258323669434,
"fcm_dpo/delta": -0.06176135316491127,
"fcm_dpo/margin": 77.84245300292969,
"fcm_dpo/q_t": 0.39108026027679443,
"grad_norm": 25.343835830688477,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.37583017349243164,
"logits/rejected": -0.392827570438385,
"logps/chosen": -131.60610961914062,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -257.996826171875,
"loss": 1.0577,
"margin_dpo/margin_mean": 77.84245300292969,
"margin_dpo/margin_std": 109.80511474609375,
"step": 174
},
{
"KL/chosen_KL_mean": -72.41232299804688,
"KL/mean": -125.403564453125,
"KL/rejected_KL_mean": -178.39483642578125,
"KL/std": 99.21268463134766,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.0062413765117526054,
"fcm_dpo/delta": -0.2268737256526947,
"fcm_dpo/margin": 105.98250579833984,
"fcm_dpo/q_t": 0.3524921238422394,
"grad_norm": 27.583271026611328,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.3298878073692322,
"logits/rejected": -0.31552278995513916,
"logps/chosen": -117.830078125,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -267.900634765625,
"loss": 0.92,
"margin_dpo/margin_mean": 105.98250579833984,
"margin_dpo/margin_std": 103.4354476928711,
"step": 175
},
{
"KL/chosen_KL_mean": -77.36492919921875,
"KL/mean": -121.19436645507812,
"KL/rejected_KL_mean": -165.0238037109375,
"KL/std": 96.19127655029297,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.0061079468578100204,
"fcm_dpo/delta": -0.09015016257762909,
"fcm_dpo/margin": 87.65887451171875,
"fcm_dpo/q_t": 0.3845323622226715,
"grad_norm": 21.98095703125,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.3371141850948334,
"logits/rejected": -0.3543722629547119,
"logps/chosen": -127.8177719116211,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -260.582763671875,
"loss": 1.0211,
"margin_dpo/margin_mean": 87.65887451171875,
"margin_dpo/margin_std": 111.42243957519531,
"step": 176
},
{
"KL/chosen_KL_mean": -88.33280944824219,
"KL/mean": -131.93548583984375,
"KL/rejected_KL_mean": -175.5381622314453,
"KL/std": 103.59379577636719,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.006017684005200863,
"fcm_dpo/delta": -0.07888535410165787,
"fcm_dpo/margin": 87.20537567138672,
"fcm_dpo/q_t": 0.38545751571655273,
"grad_norm": 30.604167938232422,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.3933263123035431,
"logits/rejected": -0.38398683071136475,
"logps/chosen": -149.5492706298828,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -271.43194580078125,
"loss": 1.023,
"margin_dpo/margin_mean": 87.20537567138672,
"margin_dpo/margin_std": 109.85467529296875,
"step": 177
},
{
"KL/chosen_KL_mean": -97.83988952636719,
"KL/mean": -152.79367065429688,
"KL/rejected_KL_mean": -207.74742126464844,
"KL/std": 123.23829650878906,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.005772947799414396,
"fcm_dpo/delta": -0.19834773242473602,
"fcm_dpo/margin": 109.90754699707031,
"fcm_dpo/q_t": 0.36565423011779785,
"grad_norm": 29.574838638305664,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.2817849814891815,
"logits/rejected": -0.27476948499679565,
"logps/chosen": -156.10467529296875,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -313.1127624511719,
"loss": 0.9742,
"margin_dpo/margin_mean": 109.90754699707031,
"margin_dpo/margin_std": 135.3180694580078,
"step": 178
},
{
"KL/chosen_KL_mean": -98.74020385742188,
"KL/mean": -138.61154174804688,
"KL/rejected_KL_mean": -178.48287963867188,
"KL/std": 104.14112854003906,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.005706791765987873,
"fcm_dpo/delta": -0.0058643147349357605,
"fcm_dpo/margin": 79.74267578125,
"fcm_dpo/q_t": 0.40119341015815735,
"grad_norm": 28.889633178710938,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.29684174060821533,
"logits/rejected": -0.3017182946205139,
"logps/chosen": -159.79852294921875,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -269.0107116699219,
"loss": 1.106,
"margin_dpo/margin_mean": 79.74266815185547,
"margin_dpo/margin_std": 128.66012573242188,
"step": 179
},
{
"KL/chosen_KL_mean": -83.98764038085938,
"KL/mean": -133.84719848632812,
"KL/rejected_KL_mean": -183.70677185058594,
"KL/std": 93.82740783691406,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.005624772980809212,
"fcm_dpo/delta": -0.11774921417236328,
"fcm_dpo/margin": 99.7191390991211,
"fcm_dpo/q_t": 0.37429293990135193,
"grad_norm": 22.949077606201172,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.30718743801116943,
"logits/rejected": -0.30064964294433594,
"logps/chosen": -138.33035278320312,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -281.9186096191406,
"loss": 0.9971,
"margin_dpo/margin_mean": 99.71913146972656,
"margin_dpo/margin_std": 116.27046203613281,
"step": 180
},
{
"KL/chosen_KL_mean": -72.35747528076172,
"KL/mean": -103.51927947998047,
"KL/rejected_KL_mean": -134.68109130859375,
"KL/std": 84.68464660644531,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.00569215789437294,
"fcm_dpo/delta": 0.09854313731193542,
"fcm_dpo/margin": 62.3236083984375,
"fcm_dpo/q_t": 0.42022329568862915,
"grad_norm": 18.50898551940918,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.36501559615135193,
"logits/rejected": -0.3385394513607025,
"logps/chosen": -127.3579330444336,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -196.33724975585938,
"loss": 1.1477,
"margin_dpo/margin_mean": 62.3236083984375,
"margin_dpo/margin_std": 106.25810241699219,
"step": 181
},
{
"KL/chosen_KL_mean": -68.99199676513672,
"KL/mean": -122.10658264160156,
"KL/rejected_KL_mean": -175.22119140625,
"KL/std": 102.834228515625,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.0055610742419958115,
"fcm_dpo/delta": -0.15064096450805664,
"fcm_dpo/margin": 106.22918701171875,
"fcm_dpo/q_t": 0.3680379390716553,
"grad_norm": 18.80778694152832,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.3319093883037567,
"logits/rejected": -0.339343786239624,
"logps/chosen": -110.09984588623047,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -264.7427673339844,
"loss": 0.9876,
"margin_dpo/margin_mean": 106.22918701171875,
"margin_dpo/margin_std": 123.63685607910156,
"step": 182
},
{
"KL/chosen_KL_mean": -101.37919616699219,
"KL/mean": -131.22171020507812,
"KL/rejected_KL_mean": -161.064208984375,
"KL/std": 83.54005432128906,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.00549449073150754,
"fcm_dpo/delta": -0.04330216720700264,
"fcm_dpo/margin": 59.685020446777344,
"fcm_dpo/q_t": 0.4243183732032776,
"grad_norm": 19.258472442626953,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.3618839979171753,
"logits/rejected": -0.3487667441368103,
"logps/chosen": -158.9037628173828,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -237.03994750976562,
"loss": 1.1554,
"margin_dpo/margin_mean": 59.685020446777344,
"margin_dpo/margin_std": 92.6783447265625,
"step": 183
},
{
"KL/chosen_KL_mean": -84.08784484863281,
"KL/mean": -114.76734161376953,
"KL/rejected_KL_mean": -145.44683837890625,
"KL/std": 80.12207794189453,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.005615667439997196,
"fcm_dpo/delta": 0.1079062819480896,
"fcm_dpo/margin": 61.358985900878906,
"fcm_dpo/q_t": 0.42317280173301697,
"grad_norm": 18.5307559967041,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.3942424952983856,
"logits/rejected": -0.3832680881023407,
"logps/chosen": -142.63279724121094,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -222.08090209960938,
"loss": 1.1465,
"margin_dpo/margin_mean": 61.35899353027344,
"margin_dpo/margin_std": 100.00926208496094,
"step": 184
},
{
"KL/chosen_KL_mean": -91.45793151855469,
"KL/mean": -116.36428833007812,
"KL/rejected_KL_mean": -141.2706298828125,
"KL/std": 91.72756958007812,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.005753664765506983,
"fcm_dpo/delta": 0.16831731796264648,
"fcm_dpo/margin": 49.812705993652344,
"fcm_dpo/q_t": 0.4390159845352173,
"grad_norm": 19.70173454284668,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.36604273319244385,
"logits/rejected": -0.3395982086658478,
"logps/chosen": -153.48377990722656,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -215.03323364257812,
"loss": 1.2088,
"margin_dpo/margin_mean": 49.812705993652344,
"margin_dpo/margin_std": 109.287841796875,
"step": 185
},
{
"KL/chosen_KL_mean": -84.84208679199219,
"KL/mean": -131.60130310058594,
"KL/rejected_KL_mean": -178.36053466796875,
"KL/std": 94.9348373413086,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.005720070563256741,
"fcm_dpo/delta": -0.08993598818778992,
"fcm_dpo/margin": 93.5184326171875,
"fcm_dpo/q_t": 0.3803972601890564,
"grad_norm": 31.597518920898438,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.3996261954307556,
"logits/rejected": -0.3734767436981201,
"logps/chosen": -154.195556640625,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -266.4329833984375,
"loss": 1.0042,
"margin_dpo/margin_mean": 93.5184326171875,
"margin_dpo/margin_std": 107.01985168457031,
"step": 186
},
{
"KL/chosen_KL_mean": -82.51023864746094,
"KL/mean": -120.74517059326172,
"KL/rejected_KL_mean": -158.9801025390625,
"KL/std": 90.9475326538086,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.005740244407206774,
"fcm_dpo/delta": 0.011016838252544403,
"fcm_dpo/margin": 76.46986389160156,
"fcm_dpo/q_t": 0.40099987387657166,
"grad_norm": 26.53521156311035,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.3313896059989929,
"logits/rejected": -0.324165940284729,
"logps/chosen": -135.26669311523438,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -240.94920349121094,
"loss": 1.0715,
"margin_dpo/margin_mean": 76.46986389160156,
"margin_dpo/margin_std": 99.63009643554688,
"step": 187
},
{
"KL/chosen_KL_mean": -77.40263366699219,
"KL/mean": -124.41639709472656,
"KL/rejected_KL_mean": -171.43016052246094,
"KL/std": 102.34568786621094,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.005648997146636248,
"fcm_dpo/delta": -0.08564944565296173,
"fcm_dpo/margin": 94.02752685546875,
"fcm_dpo/q_t": 0.38270336389541626,
"grad_norm": 32.77531814575195,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.349833607673645,
"logits/rejected": -0.35466983914375305,
"logps/chosen": -126.818115234375,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -260.9705810546875,
"loss": 1.0127,
"margin_dpo/margin_mean": 94.02752685546875,
"margin_dpo/margin_std": 112.28752899169922,
"step": 188
},
{
"KL/chosen_KL_mean": -93.02162170410156,
"KL/mean": -131.88063049316406,
"KL/rejected_KL_mean": -170.73960876464844,
"KL/std": 103.43923950195312,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.005634644068777561,
"fcm_dpo/delta": 0.012608291581273079,
"fcm_dpo/margin": 77.7179946899414,
"fcm_dpo/q_t": 0.4059969186782837,
"grad_norm": 30.910058975219727,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.3398626148700714,
"logits/rejected": -0.31965237855911255,
"logps/chosen": -145.4205780029297,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -242.90695190429688,
"loss": 1.1041,
"margin_dpo/margin_mean": 77.7179946899414,
"margin_dpo/margin_std": 126.54793548583984,
"step": 189
},
{
"KL/chosen_KL_mean": -98.95657348632812,
"KL/mean": -145.08016967773438,
"KL/rejected_KL_mean": -191.20376586914062,
"KL/std": 110.704345703125,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.005625591613352299,
"fcm_dpo/delta": -0.07391072809696198,
"fcm_dpo/margin": 92.24720001220703,
"fcm_dpo/q_t": 0.3878136873245239,
"grad_norm": 21.0270938873291,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.37190186977386475,
"logits/rejected": -0.3592497706413269,
"logps/chosen": -163.63963317871094,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -293.7543029785156,
"loss": 1.057,
"margin_dpo/margin_mean": 92.24720001220703,
"margin_dpo/margin_std": 128.497314453125,
"step": 190
},
{
"KL/chosen_KL_mean": -91.65495300292969,
"KL/mean": -158.4246063232422,
"KL/rejected_KL_mean": -225.19424438476562,
"KL/std": 129.930908203125,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.0053010242991149426,
"fcm_dpo/delta": -0.27925872802734375,
"fcm_dpo/margin": 133.53929138183594,
"fcm_dpo/q_t": 0.3480866551399231,
"grad_norm": 22.660518646240234,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.3247271776199341,
"logits/rejected": -0.30059993267059326,
"logps/chosen": -160.31382751464844,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -335.33392333984375,
"loss": 0.9148,
"margin_dpo/margin_mean": 133.539306640625,
"margin_dpo/margin_std": 142.86781311035156,
"step": 191
},
{
"KL/chosen_KL_mean": -118.78849029541016,
"KL/mean": -162.5628662109375,
"KL/rejected_KL_mean": -206.33724975585938,
"KL/std": 111.55377197265625,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.005254846997559071,
"fcm_dpo/delta": -0.010667569935321808,
"fcm_dpo/margin": 87.54873657226562,
"fcm_dpo/q_t": 0.40085333585739136,
"grad_norm": 27.49567413330078,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.3013686537742615,
"logits/rejected": -0.29982978105545044,
"logps/chosen": -188.51541137695312,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -309.6585693359375,
"loss": 1.0927,
"margin_dpo/margin_mean": 87.54873657226562,
"margin_dpo/margin_std": 137.13919067382812,
"step": 192
},
{
"KL/chosen_KL_mean": -118.83079528808594,
"KL/mean": -144.50320434570312,
"KL/rejected_KL_mean": -170.17559814453125,
"KL/std": 101.15321350097656,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.005260747391730547,
"fcm_dpo/delta": 0.044322919100522995,
"fcm_dpo/margin": 51.34478759765625,
"fcm_dpo/q_t": 0.4367770552635193,
"grad_norm": 30.344806671142578,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.3281133472919464,
"logits/rejected": -0.31660524010658264,
"logps/chosen": -179.02130126953125,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -246.58316040039062,
"loss": 1.2429,
"margin_dpo/margin_mean": 51.34478759765625,
"margin_dpo/margin_std": 128.936767578125,
"step": 193
},
{
"KL/chosen_KL_mean": -73.99462890625,
"KL/mean": -115.9382553100586,
"KL/rejected_KL_mean": -157.88189697265625,
"KL/std": 85.7735595703125,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.005274644121527672,
"fcm_dpo/delta": 0.007718952372670174,
"fcm_dpo/margin": 83.88723754882812,
"fcm_dpo/q_t": 0.3977963924407959,
"grad_norm": 20.127643585205078,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.2561291456222534,
"logits/rejected": -0.2391519397497177,
"logps/chosen": -111.83500671386719,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -218.56666564941406,
"loss": 1.0502,
"margin_dpo/margin_mean": 83.88723754882812,
"margin_dpo/margin_std": 94.03021240234375,
"step": 194
},
{
"KL/chosen_KL_mean": -117.90105438232422,
"KL/mean": -163.21768188476562,
"KL/rejected_KL_mean": -208.5343017578125,
"KL/std": 107.3478012084961,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.005267483647912741,
"fcm_dpo/delta": -0.028798166662454605,
"fcm_dpo/margin": 90.63323974609375,
"fcm_dpo/q_t": 0.39235901832580566,
"grad_norm": 20.91324806213379,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.2966734766960144,
"logits/rejected": -0.3150033950805664,
"logps/chosen": -172.79263305664062,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -305.30523681640625,
"loss": 1.0421,
"margin_dpo/margin_mean": 90.63323974609375,
"margin_dpo/margin_std": 111.12643432617188,
"step": 195
},
{
"KL/chosen_KL_mean": -90.92343139648438,
"KL/mean": -142.19207763671875,
"KL/rejected_KL_mean": -193.46075439453125,
"KL/std": 109.14540100097656,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.005165431182831526,
"fcm_dpo/delta": -0.08529461920261383,
"fcm_dpo/margin": 102.53730773925781,
"fcm_dpo/q_t": 0.3841804265975952,
"grad_norm": 22.65111541748047,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.33719146251678467,
"logits/rejected": -0.3222602605819702,
"logps/chosen": -144.16867065429688,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -269.5137023925781,
"loss": 1.0311,
"margin_dpo/margin_mean": 102.53730773925781,
"margin_dpo/margin_std": 130.28256225585938,
"step": 196
},
{
"KL/chosen_KL_mean": -93.05845642089844,
"KL/mean": -132.66305541992188,
"KL/rejected_KL_mean": -172.2676544189453,
"KL/std": 93.76297760009766,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.0052191768772900105,
"fcm_dpo/delta": 0.03782667964696884,
"fcm_dpo/margin": 79.20919799804688,
"fcm_dpo/q_t": 0.40683090686798096,
"grad_norm": 20.258697509765625,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.29354849457740784,
"logits/rejected": -0.28677839040756226,
"logps/chosen": -153.47879028320312,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -249.4765625,
"loss": 1.0854,
"margin_dpo/margin_mean": 79.20919799804688,
"margin_dpo/margin_std": 105.698486328125,
"step": 197
},
{
"KL/chosen_KL_mean": -105.25746154785156,
"KL/mean": -155.54010009765625,
"KL/rejected_KL_mean": -205.82276916503906,
"KL/std": 117.5333023071289,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.005160834174603224,
"fcm_dpo/delta": -0.07272866368293762,
"fcm_dpo/margin": 100.56529235839844,
"fcm_dpo/q_t": 0.38785430788993835,
"grad_norm": 21.33174705505371,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.33081430196762085,
"logits/rejected": -0.33057862520217896,
"logps/chosen": -160.2936553955078,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -303.0660400390625,
"loss": 1.034,
"margin_dpo/margin_mean": 100.56529235839844,
"margin_dpo/margin_std": 134.0467529296875,
"step": 198
},
{
"KL/chosen_KL_mean": -100.41415405273438,
"KL/mean": -149.92404174804688,
"KL/rejected_KL_mean": -199.4339141845703,
"KL/std": 106.2638168334961,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.005063729360699654,
"fcm_dpo/delta": -0.05523253232240677,
"fcm_dpo/margin": 99.01976776123047,
"fcm_dpo/q_t": 0.3882749080657959,
"grad_norm": 23.801433563232422,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.3572986125946045,
"logits/rejected": -0.3407086133956909,
"logps/chosen": -157.24298095703125,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -284.0821228027344,
"loss": 1.0418,
"margin_dpo/margin_mean": 99.019775390625,
"margin_dpo/margin_std": 126.37538146972656,
"step": 199
},
{
"KL/chosen_KL_mean": -98.62019348144531,
"KL/mean": -148.50802612304688,
"KL/rejected_KL_mean": -198.39585876464844,
"KL/std": 113.95591735839844,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.005019994918256998,
"fcm_dpo/delta": -0.054083775728940964,
"fcm_dpo/margin": 99.77567291259766,
"fcm_dpo/q_t": 0.39153775572776794,
"grad_norm": 25.75031089782715,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.2872212529182434,
"logits/rejected": -0.26524609327316284,
"logps/chosen": -151.687255859375,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -279.0043029785156,
"loss": 1.0616,
"margin_dpo/margin_mean": 99.77567291259766,
"margin_dpo/margin_std": 143.67160034179688,
"step": 200
},
{
"KL/chosen_KL_mean": -102.265380859375,
"KL/mean": -151.52593994140625,
"KL/rejected_KL_mean": -200.78646850585938,
"KL/std": 118.35769653320312,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.005003670696169138,
"fcm_dpo/delta": -0.04519995301961899,
"fcm_dpo/margin": 98.52108764648438,
"fcm_dpo/q_t": 0.39059755206108093,
"grad_norm": 22.498605728149414,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.37770992517471313,
"logits/rejected": -0.37379318475723267,
"logps/chosen": -177.6676025390625,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -315.5946960449219,
"loss": 1.0502,
"margin_dpo/margin_mean": 98.52108764648438,
"margin_dpo/margin_std": 131.62591552734375,
"step": 201
},
{
"KL/chosen_KL_mean": -105.03387451171875,
"KL/mean": -139.44313049316406,
"KL/rejected_KL_mean": -173.85238647460938,
"KL/std": 100.22007751464844,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.005063983611762524,
"fcm_dpo/delta": 0.10505226254463196,
"fcm_dpo/margin": 68.8185043334961,
"fcm_dpo/q_t": 0.42318177223205566,
"grad_norm": 24.158180236816406,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.30475035309791565,
"logits/rejected": -0.316967636346817,
"logps/chosen": -155.13519287109375,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -260.83740234375,
"loss": 1.164,
"margin_dpo/margin_mean": 68.8185043334961,
"margin_dpo/margin_std": 127.62640380859375,
"step": 202
},
{
"KL/chosen_KL_mean": -102.41499328613281,
"KL/mean": -145.3014678955078,
"KL/rejected_KL_mean": -188.18792724609375,
"KL/std": 102.64788818359375,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.005089880898594856,
"fcm_dpo/delta": 0.014019053429365158,
"fcm_dpo/margin": 85.77294921875,
"fcm_dpo/q_t": 0.40114203095436096,
"grad_norm": 22.973190307617188,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.38558489084243774,
"logits/rejected": -0.3768942356109619,
"logps/chosen": -163.02468872070312,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -274.08392333984375,
"loss": 1.0694,
"margin_dpo/margin_mean": 85.77295684814453,
"margin_dpo/margin_std": 111.16223907470703,
"step": 203
},
{
"KL/chosen_KL_mean": -115.02670288085938,
"KL/mean": -154.1737060546875,
"KL/rejected_KL_mean": -193.32069396972656,
"KL/std": 111.26290893554688,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.0051376353949308395,
"fcm_dpo/delta": 0.04976356029510498,
"fcm_dpo/margin": 78.29398345947266,
"fcm_dpo/q_t": 0.41275104880332947,
"grad_norm": 28.295520782470703,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.4326149821281433,
"logits/rejected": -0.39165282249450684,
"logps/chosen": -195.191650390625,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -281.0166015625,
"loss": 1.1225,
"margin_dpo/margin_mean": 78.29398345947266,
"margin_dpo/margin_std": 128.27748107910156,
"step": 204
},
{
"KL/chosen_KL_mean": -111.0508041381836,
"KL/mean": -162.05941772460938,
"KL/rejected_KL_mean": -213.06805419921875,
"KL/std": 115.12797546386719,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.0050528934225440025,
"fcm_dpo/delta": -0.07124295830726624,
"fcm_dpo/margin": 102.01722717285156,
"fcm_dpo/q_t": 0.3834433853626251,
"grad_norm": 22.90087890625,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.3363794982433319,
"logits/rejected": -0.3039674162864685,
"logps/chosen": -170.435546875,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -298.193115234375,
"loss": 1.0345,
"margin_dpo/margin_mean": 102.01722717285156,
"margin_dpo/margin_std": 125.01140594482422,
"step": 205
},
{
"KL/chosen_KL_mean": -104.03678894042969,
"KL/mean": -159.7008056640625,
"KL/rejected_KL_mean": -215.36485290527344,
"KL/std": 109.63488006591797,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.004975572694092989,
"fcm_dpo/delta": -0.11080212146043777,
"fcm_dpo/margin": 111.32804870605469,
"fcm_dpo/q_t": 0.3731870651245117,
"grad_norm": 27.668210983276367,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.2990955710411072,
"logits/rejected": -0.30272185802459717,
"logps/chosen": -151.00128173828125,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -314.318359375,
"loss": 0.988,
"margin_dpo/margin_mean": 111.32805633544922,
"margin_dpo/margin_std": 118.75079345703125,
"step": 206
},
{
"KL/chosen_KL_mean": -94.94453430175781,
"KL/mean": -160.35214233398438,
"KL/rejected_KL_mean": -225.759765625,
"KL/std": 125.71635437011719,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.004834486171603203,
"fcm_dpo/delta": -0.19447346031665802,
"fcm_dpo/margin": 130.81524658203125,
"fcm_dpo/q_t": 0.36242467164993286,
"grad_norm": 23.659076690673828,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.3879826068878174,
"logits/rejected": -0.363485187292099,
"logps/chosen": -151.00079345703125,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -310.20758056640625,
"loss": 0.9535,
"margin_dpo/margin_mean": 130.81524658203125,
"margin_dpo/margin_std": 144.30435180664062,
"step": 207
},
{
"KL/chosen_KL_mean": -147.72682189941406,
"KL/mean": -199.30636596679688,
"KL/rejected_KL_mean": -250.88592529296875,
"KL/std": 121.20966339111328,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.004738894756883383,
"fcm_dpo/delta": -0.041033219546079636,
"fcm_dpo/margin": 103.15910339355469,
"fcm_dpo/q_t": 0.39097169041633606,
"grad_norm": 27.04127311706543,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.38049787282943726,
"logits/rejected": -0.36199867725372314,
"logps/chosen": -214.79443359375,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -345.1728210449219,
"loss": 1.064,
"margin_dpo/margin_mean": 103.15910339355469,
"margin_dpo/margin_std": 145.6819305419922,
"step": 208
},
{
"KL/chosen_KL_mean": -124.6459732055664,
"KL/mean": -169.93202209472656,
"KL/rejected_KL_mean": -215.21807861328125,
"KL/std": 109.89362335205078,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.004754163324832916,
"fcm_dpo/delta": 0.020273543894290924,
"fcm_dpo/margin": 90.57209777832031,
"fcm_dpo/q_t": 0.4042934477329254,
"grad_norm": 31.34199333190918,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.28706932067871094,
"logits/rejected": -0.2759585976600647,
"logps/chosen": -180.82766723632812,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -296.15960693359375,
"loss": 1.1062,
"margin_dpo/margin_mean": 90.57209777832031,
"margin_dpo/margin_std": 139.96685791015625,
"step": 209
},
{
"KL/chosen_KL_mean": -113.70458984375,
"KL/mean": -165.95252990722656,
"KL/rejected_KL_mean": -218.20050048828125,
"KL/std": 113.39631652832031,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.004746555350720882,
"fcm_dpo/delta": -0.04863199591636658,
"fcm_dpo/margin": 104.49591064453125,
"fcm_dpo/q_t": 0.3909037411212921,
"grad_norm": 26.605981826782227,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.2829633355140686,
"logits/rejected": -0.2737138271331787,
"logps/chosen": -160.076416015625,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -294.8821105957031,
"loss": 1.0495,
"margin_dpo/margin_mean": 104.49591064453125,
"margin_dpo/margin_std": 138.57675170898438,
"step": 210
},
{
"KL/chosen_KL_mean": -160.13168334960938,
"KL/mean": -204.631591796875,
"KL/rejected_KL_mean": -249.13148498535156,
"KL/std": 128.61248779296875,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.004733526147902012,
"fcm_dpo/delta": 0.029977422207593918,
"fcm_dpo/margin": 88.99981689453125,
"fcm_dpo/q_t": 0.4108871817588806,
"grad_norm": 35.8740119934082,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.32857006788253784,
"logits/rejected": -0.286856472492218,
"logps/chosen": -239.0640411376953,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -335.95245361328125,
"loss": 1.1259,
"margin_dpo/margin_mean": 88.99981689453125,
"margin_dpo/margin_std": 151.01441955566406,
"step": 211
},
{
"KL/chosen_KL_mean": -130.05645751953125,
"KL/mean": -193.37249755859375,
"KL/rejected_KL_mean": -256.68853759765625,
"KL/std": 137.6314697265625,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.004602103494107723,
"fcm_dpo/delta": -0.14359265565872192,
"fcm_dpo/margin": 126.632080078125,
"fcm_dpo/q_t": 0.3717968463897705,
"grad_norm": 25.104671478271484,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.33368033170700073,
"logits/rejected": -0.3230438530445099,
"logps/chosen": -188.25347900390625,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -359.74639892578125,
"loss": 1.006,
"margin_dpo/margin_mean": 126.632080078125,
"margin_dpo/margin_std": 152.7139892578125,
"step": 212
},
{
"KL/chosen_KL_mean": -121.16593933105469,
"KL/mean": -180.69183349609375,
"KL/rejected_KL_mean": -240.21774291992188,
"KL/std": 118.9180908203125,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.004524152260273695,
"fcm_dpo/delta": -0.09434568881988525,
"fcm_dpo/margin": 119.05180358886719,
"fcm_dpo/q_t": 0.37940219044685364,
"grad_norm": 27.022523880004883,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.33751168847084045,
"logits/rejected": -0.3087567687034607,
"logps/chosen": -188.67864990234375,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -334.1324462890625,
"loss": 1.0147,
"margin_dpo/margin_mean": 119.05180358886719,
"margin_dpo/margin_std": 143.37750244140625,
"step": 213
},
{
"KL/chosen_KL_mean": -103.91690063476562,
"KL/mean": -158.3183135986328,
"KL/rejected_KL_mean": -212.7197265625,
"KL/std": 117.29820251464844,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.004492715001106262,
"fcm_dpo/delta": -0.040815435349941254,
"fcm_dpo/margin": 108.8028335571289,
"fcm_dpo/q_t": 0.39092785120010376,
"grad_norm": 24.806739807128906,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.29573267698287964,
"logits/rejected": -0.3004228174686432,
"logps/chosen": -145.52178955078125,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -290.2371520996094,
"loss": 1.0412,
"margin_dpo/margin_mean": 108.80284118652344,
"margin_dpo/margin_std": 136.7408905029297,
"step": 214
},
{
"KL/chosen_KL_mean": -119.432861328125,
"KL/mean": -176.01162719726562,
"KL/rejected_KL_mean": -232.59039306640625,
"KL/std": 120.8812255859375,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.004435136914253235,
"fcm_dpo/delta": -0.054898492991924286,
"fcm_dpo/margin": 113.15753173828125,
"fcm_dpo/q_t": 0.38608264923095703,
"grad_norm": 31.865793228149414,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.34302496910095215,
"logits/rejected": -0.3348790407180786,
"logps/chosen": -172.71212768554688,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -322.5550537109375,
"loss": 1.019,
"margin_dpo/margin_mean": 113.15753173828125,
"margin_dpo/margin_std": 126.95989227294922,
"step": 215
},
{
"KL/chosen_KL_mean": -123.26240539550781,
"KL/mean": -179.19107055664062,
"KL/rejected_KL_mean": -235.1197509765625,
"KL/std": 124.5248794555664,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.0044202255085110664,
"fcm_dpo/delta": -0.04685918614268303,
"fcm_dpo/margin": 111.85734558105469,
"fcm_dpo/q_t": 0.3906000554561615,
"grad_norm": 24.205480575561523,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.3160155117511749,
"logits/rejected": -0.2979496121406555,
"logps/chosen": -172.15020751953125,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -312.31866455078125,
"loss": 1.0553,
"margin_dpo/margin_mean": 111.85734558105469,
"margin_dpo/margin_std": 152.19393920898438,
"step": 216
},
{
"KL/chosen_KL_mean": -121.02989196777344,
"KL/mean": -186.89378356933594,
"KL/rejected_KL_mean": -252.75765991210938,
"KL/std": 125.570068359375,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.004304962232708931,
"fcm_dpo/delta": -0.12440802156925201,
"fcm_dpo/margin": 131.72776794433594,
"fcm_dpo/q_t": 0.37273240089416504,
"grad_norm": 22.334640502929688,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.2926168739795685,
"logits/rejected": -0.28581422567367554,
"logps/chosen": -170.8751983642578,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -352.83599853515625,
"loss": 0.9772,
"margin_dpo/margin_mean": 131.72776794433594,
"margin_dpo/margin_std": 139.87136840820312,
"step": 217
},
{
"KL/chosen_KL_mean": -130.25543212890625,
"KL/mean": -180.17367553710938,
"KL/rejected_KL_mean": -230.09188842773438,
"KL/std": 122.58163452148438,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.004292862489819527,
"fcm_dpo/delta": 0.022361639887094498,
"fcm_dpo/margin": 99.83648681640625,
"fcm_dpo/q_t": 0.4043183922767639,
"grad_norm": 24.542783737182617,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.326630175113678,
"logits/rejected": -0.31950443983078003,
"logps/chosen": -188.8321075439453,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -317.93829345703125,
"loss": 1.0891,
"margin_dpo/margin_mean": 99.83648681640625,
"margin_dpo/margin_std": 141.80709838867188,
"step": 218
},
{
"KL/chosen_KL_mean": -140.5905303955078,
"KL/mean": -186.1055145263672,
"KL/rejected_KL_mean": -231.62049865722656,
"KL/std": 130.56130981445312,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.004359761718660593,
"fcm_dpo/delta": 0.05462624877691269,
"fcm_dpo/margin": 91.02996826171875,
"fcm_dpo/q_t": 0.4125881791114807,
"grad_norm": 29.01929473876953,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.31307315826416016,
"logits/rejected": -0.3005494773387909,
"logps/chosen": -201.6743927001953,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -317.450927734375,
"loss": 1.1381,
"margin_dpo/margin_mean": 91.02996826171875,
"margin_dpo/margin_std": 158.283203125,
"step": 219
},
{
"KL/chosen_KL_mean": -158.72708129882812,
"KL/mean": -196.89773559570312,
"KL/rejected_KL_mean": -235.0684051513672,
"KL/std": 110.52084350585938,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.004418938420712948,
"fcm_dpo/delta": 0.11673985421657562,
"fcm_dpo/margin": 76.34133911132812,
"fcm_dpo/q_t": 0.42132964730262756,
"grad_norm": 28.393917083740234,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.31454047560691833,
"logits/rejected": -0.28938183188438416,
"logps/chosen": -228.75836181640625,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -322.75390625,
"loss": 1.1342,
"margin_dpo/margin_mean": 76.34133911132812,
"margin_dpo/margin_std": 110.11967468261719,
"step": 220
},
{
"KL/chosen_KL_mean": -139.7552490234375,
"KL/mean": -218.57884216308594,
"KL/rejected_KL_mean": -297.4024353027344,
"KL/std": 141.19781494140625,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.004313099198043346,
"fcm_dpo/delta": -0.24606135487556458,
"fcm_dpo/margin": 157.64718627929688,
"fcm_dpo/q_t": 0.34645724296569824,
"grad_norm": 34.04795837402344,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.2851540446281433,
"logits/rejected": -0.29184699058532715,
"logps/chosen": -191.9099578857422,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -403.8701171875,
"loss": 0.9044,
"margin_dpo/margin_mean": 157.64718627929688,
"margin_dpo/margin_std": 141.8089141845703,
"step": 221
},
{
"KL/chosen_KL_mean": -146.63458251953125,
"KL/mean": -207.00054931640625,
"KL/rejected_KL_mean": -267.36651611328125,
"KL/std": 130.62632751464844,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.0042197853326797485,
"fcm_dpo/delta": -0.06277885288000107,
"fcm_dpo/margin": 120.73193359375,
"fcm_dpo/q_t": 0.3854852318763733,
"grad_norm": 23.816076278686523,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.32067981362342834,
"logits/rejected": -0.3180358409881592,
"logps/chosen": -207.60568237304688,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -367.36767578125,
"loss": 1.0261,
"margin_dpo/margin_mean": 120.73193359375,
"margin_dpo/margin_std": 144.6348876953125,
"step": 222
},
{
"KL/chosen_KL_mean": -155.57662963867188,
"KL/mean": -207.48294067382812,
"KL/rejected_KL_mean": -259.3892822265625,
"KL/std": 130.69064331054688,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.004191592335700989,
"fcm_dpo/delta": 0.015519605949521065,
"fcm_dpo/margin": 103.81266784667969,
"fcm_dpo/q_t": 0.4033808708190918,
"grad_norm": 26.01241111755371,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.2557436525821686,
"logits/rejected": -0.24912384152412415,
"logps/chosen": -208.21719360351562,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -342.21429443359375,
"loss": 1.1072,
"margin_dpo/margin_mean": 103.81266021728516,
"margin_dpo/margin_std": 165.55490112304688,
"step": 223
},
{
"KL/chosen_KL_mean": -132.38632202148438,
"KL/mean": -192.0895233154297,
"KL/rejected_KL_mean": -251.792724609375,
"KL/std": 143.71157836914062,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.004137513227760792,
"fcm_dpo/delta": -0.04802892357110977,
"fcm_dpo/margin": 119.40638732910156,
"fcm_dpo/q_t": 0.39233386516571045,
"grad_norm": 25.351940155029297,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.27835261821746826,
"logits/rejected": -0.25782424211502075,
"logps/chosen": -180.98175048828125,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -328.9092102050781,
"loss": 1.0566,
"margin_dpo/margin_mean": 119.40638732910156,
"margin_dpo/margin_std": 163.24822998046875,
"step": 224
},
{
"KL/chosen_KL_mean": -151.82391357421875,
"KL/mean": -222.4290771484375,
"KL/rejected_KL_mean": -293.03424072265625,
"KL/std": 141.11753845214844,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.004073306918144226,
"fcm_dpo/delta": -0.13306143879890442,
"fcm_dpo/margin": 141.21035766601562,
"fcm_dpo/q_t": 0.3728272318840027,
"grad_norm": 27.907018661499023,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.2939089238643646,
"logits/rejected": -0.28350159525871277,
"logps/chosen": -209.82437133789062,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -392.9371643066406,
"loss": 0.9974,
"margin_dpo/margin_mean": 141.21034240722656,
"margin_dpo/margin_std": 170.23565673828125,
"step": 225
},
{
"KL/chosen_KL_mean": -130.0558319091797,
"KL/mean": -187.48414611816406,
"KL/rejected_KL_mean": -244.91246032714844,
"KL/std": 141.0475616455078,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.004016010090708733,
"fcm_dpo/delta": -0.012848969548940659,
"fcm_dpo/margin": 114.85662841796875,
"fcm_dpo/q_t": 0.399183452129364,
"grad_norm": 31.283985137939453,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.35832834243774414,
"logits/rejected": -0.3170637786388397,
"logps/chosen": -188.9546356201172,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -323.6002197265625,
"loss": 1.0724,
"margin_dpo/margin_mean": 114.85662841796875,
"margin_dpo/margin_std": 159.86416625976562,
"step": 226
},
{
"KL/chosen_KL_mean": -151.1775360107422,
"KL/mean": -221.81304931640625,
"KL/rejected_KL_mean": -292.44854736328125,
"KL/std": 158.22885131835938,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.003954638727009296,
"fcm_dpo/delta": -0.11598802357912064,
"fcm_dpo/margin": 141.27099609375,
"fcm_dpo/q_t": 0.37532055377960205,
"grad_norm": 23.948640823364258,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.3440447747707367,
"logits/rejected": -0.3300914764404297,
"logps/chosen": -210.24972534179688,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -391.86090087890625,
"loss": 0.9984,
"margin_dpo/margin_mean": 141.27101135253906,
"margin_dpo/margin_std": 165.87399291992188,
"step": 227
},
{
"KL/chosen_KL_mean": -157.38519287109375,
"KL/mean": -206.89305114746094,
"KL/rejected_KL_mean": -256.40093994140625,
"KL/std": 129.40603637695312,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.003987784497439861,
"fcm_dpo/delta": 0.05728516727685928,
"fcm_dpo/margin": 99.0157470703125,
"fcm_dpo/q_t": 0.4110216498374939,
"grad_norm": 26.616201400756836,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.334661602973938,
"logits/rejected": -0.31732168793678284,
"logps/chosen": -223.27647399902344,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -347.4496765136719,
"loss": 1.1221,
"margin_dpo/margin_mean": 99.0157470703125,
"margin_dpo/margin_std": 157.97543334960938,
"step": 228
},
{
"KL/chosen_KL_mean": -156.45465087890625,
"KL/mean": -210.65725708007812,
"KL/rejected_KL_mean": -264.85986328125,
"KL/std": 146.39035034179688,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.004005359020084143,
"fcm_dpo/delta": 0.016502387821674347,
"fcm_dpo/margin": 108.40519714355469,
"fcm_dpo/q_t": 0.40141230821609497,
"grad_norm": 31.62211036682129,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.43813785910606384,
"logits/rejected": -0.4117690920829773,
"logps/chosen": -227.16104125976562,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -349.38726806640625,
"loss": 1.0863,
"margin_dpo/margin_mean": 108.40519714355469,
"margin_dpo/margin_std": 154.72164916992188,
"step": 229
},
{
"KL/chosen_KL_mean": -128.100830078125,
"KL/mean": -208.39639282226562,
"KL/rejected_KL_mean": -288.69195556640625,
"KL/std": 143.41812133789062,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.0038900193758308887,
"fcm_dpo/delta": -0.1867695450782776,
"fcm_dpo/margin": 160.59112548828125,
"fcm_dpo/q_t": 0.35838133096694946,
"grad_norm": 25.45254898071289,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.2791915535926819,
"logits/rejected": -0.28188395500183105,
"logps/chosen": -167.38284301757812,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -374.3138427734375,
"loss": 0.942,
"margin_dpo/margin_mean": 160.59112548828125,
"margin_dpo/margin_std": 159.54147338867188,
"step": 230
},
{
"KL/chosen_KL_mean": -142.98562622070312,
"KL/mean": -199.78729248046875,
"KL/rejected_KL_mean": -256.5889587402344,
"KL/std": 126.15511322021484,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.0038650333881378174,
"fcm_dpo/delta": 0.011422183364629745,
"fcm_dpo/margin": 113.60334777832031,
"fcm_dpo/q_t": 0.4017235040664673,
"grad_norm": 25.279603958129883,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.36105793714523315,
"logits/rejected": -0.331612229347229,
"logps/chosen": -206.26206970214844,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -330.7128601074219,
"loss": 1.065,
"margin_dpo/margin_mean": 113.60334777832031,
"margin_dpo/margin_std": 144.85211181640625,
"step": 231
},
{
"KL/chosen_KL_mean": -171.13592529296875,
"KL/mean": -216.0245361328125,
"KL/rejected_KL_mean": -260.9131774902344,
"KL/std": 143.8835906982422,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.003919616807252169,
"fcm_dpo/delta": 0.10181022435426712,
"fcm_dpo/margin": 89.77725219726562,
"fcm_dpo/q_t": 0.42291808128356934,
"grad_norm": 26.38113021850586,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.39240676164627075,
"logits/rejected": -0.36876434087753296,
"logps/chosen": -241.88467407226562,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -344.8902587890625,
"loss": 1.1423,
"margin_dpo/margin_mean": 89.77725219726562,
"margin_dpo/margin_std": 148.82798767089844,
"step": 232
},
{
"KL/chosen_KL_mean": -155.85494995117188,
"KL/mean": -224.17303466796875,
"KL/rejected_KL_mean": -292.4911193847656,
"KL/std": 150.75767517089844,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.003898974508047104,
"fcm_dpo/delta": -0.08736483752727509,
"fcm_dpo/margin": 136.63619995117188,
"fcm_dpo/q_t": 0.3861563205718994,
"grad_norm": 24.671648025512695,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.35894495248794556,
"logits/rejected": -0.3622890114784241,
"logps/chosen": -210.73788452148438,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -399.97119140625,
"loss": 1.0459,
"margin_dpo/margin_mean": 136.63619995117188,
"margin_dpo/margin_std": 192.1343536376953,
"step": 233
},
{
"KL/chosen_KL_mean": -145.80184936523438,
"KL/mean": -217.7124786376953,
"KL/rejected_KL_mean": -289.62310791015625,
"KL/std": 132.61508178710938,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.003794679883867502,
"fcm_dpo/delta": -0.1029752865433693,
"fcm_dpo/margin": 143.82122802734375,
"fcm_dpo/q_t": 0.3759935796260834,
"grad_norm": 38.8779296875,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.33827099204063416,
"logits/rejected": -0.3627021312713623,
"logps/chosen": -189.89637756347656,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -389.6297302246094,
"loss": 0.9867,
"margin_dpo/margin_mean": 143.82122802734375,
"margin_dpo/margin_std": 147.23887634277344,
"step": 234
},
{
"KL/chosen_KL_mean": -175.931396484375,
"KL/mean": -221.4193878173828,
"KL/rejected_KL_mean": -266.9073486328125,
"KL/std": 124.45712280273438,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.003867072518914938,
"fcm_dpo/delta": 0.10145650804042816,
"fcm_dpo/margin": 90.97599792480469,
"fcm_dpo/q_t": 0.4202185869216919,
"grad_norm": 24.789424896240234,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.39839839935302734,
"logits/rejected": -0.3760639429092407,
"logps/chosen": -238.1693115234375,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -357.30242919921875,
"loss": 1.1413,
"margin_dpo/margin_mean": 90.97599792480469,
"margin_dpo/margin_std": 147.0844268798828,
"step": 235
},
{
"KL/chosen_KL_mean": -126.4316635131836,
"KL/mean": -199.6041259765625,
"KL/rejected_KL_mean": -272.77655029296875,
"KL/std": 136.3174591064453,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.0038139871321618557,
"fcm_dpo/delta": -0.11453250050544739,
"fcm_dpo/margin": 146.34490966796875,
"fcm_dpo/q_t": 0.37070026993751526,
"grad_norm": 42.87751007080078,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.3801974058151245,
"logits/rejected": -0.38002660870552063,
"logps/chosen": -175.7730255126953,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -376.2882080078125,
"loss": 0.9602,
"margin_dpo/margin_mean": 146.34490966796875,
"margin_dpo/margin_std": 129.97216796875,
"step": 236
},
{
"KL/chosen_KL_mean": -162.65077209472656,
"KL/mean": -220.71786499023438,
"KL/rejected_KL_mean": -278.78497314453125,
"KL/std": 137.26528930664062,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.003799371188506484,
"fcm_dpo/delta": 0.009143916890025139,
"fcm_dpo/margin": 116.13417053222656,
"fcm_dpo/q_t": 0.400818407535553,
"grad_norm": 24.1297550201416,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.3950151801109314,
"logits/rejected": -0.38823604583740234,
"logps/chosen": -216.8188934326172,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -373.5653076171875,
"loss": 1.0747,
"margin_dpo/margin_mean": 116.13417053222656,
"margin_dpo/margin_std": 158.19332885742188,
"step": 237
},
{
"KL/chosen_KL_mean": -147.71888732910156,
"KL/mean": -203.96322631835938,
"KL/rejected_KL_mean": -260.20758056640625,
"KL/std": 133.24746704101562,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.0037925869692116976,
"fcm_dpo/delta": 0.024032268673181534,
"fcm_dpo/margin": 112.48866271972656,
"fcm_dpo/q_t": 0.40515145659446716,
"grad_norm": 27.994230270385742,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.3679651618003845,
"logits/rejected": -0.3837740421295166,
"logps/chosen": -201.6920166015625,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -349.62548828125,
"loss": 1.0773,
"margin_dpo/margin_mean": 112.48866271972656,
"margin_dpo/margin_std": 148.3927001953125,
"step": 238
},
{
"KL/chosen_KL_mean": -157.4866943359375,
"KL/mean": -220.90524291992188,
"KL/rejected_KL_mean": -284.32379150390625,
"KL/std": 127.85582733154297,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.0037702773697674274,
"fcm_dpo/delta": -0.031009189784526825,
"fcm_dpo/margin": 126.83708953857422,
"fcm_dpo/q_t": 0.3901780843734741,
"grad_norm": 36.62656784057617,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.37180736660957336,
"logits/rejected": -0.35609129071235657,
"logps/chosen": -215.58453369140625,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -377.916748046875,
"loss": 1.0339,
"margin_dpo/margin_mean": 126.83708953857422,
"margin_dpo/margin_std": 139.82359313964844,
"step": 239
},
{
"KL/chosen_KL_mean": -176.02816772460938,
"KL/mean": -231.24081420898438,
"KL/rejected_KL_mean": -286.4534912109375,
"KL/std": 144.84393310546875,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.0037977946922183037,
"fcm_dpo/delta": 0.03184373676776886,
"fcm_dpo/margin": 110.42532348632812,
"fcm_dpo/q_t": 0.40580376982688904,
"grad_norm": 30.680309295654297,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.3963586986064911,
"logits/rejected": -0.3742924928665161,
"logps/chosen": -236.64263916015625,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -360.572021484375,
"loss": 1.1079,
"margin_dpo/margin_mean": 110.42530822753906,
"margin_dpo/margin_std": 169.20822143554688,
"step": 240
},
{
"KL/chosen_KL_mean": -157.76492309570312,
"KL/mean": -244.28536987304688,
"KL/rejected_KL_mean": -330.80584716796875,
"KL/std": 164.7505340576172,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0037105297669768333,
"fcm_dpo/delta": -0.20491331815719604,
"fcm_dpo/margin": 173.0408935546875,
"fcm_dpo/q_t": 0.36018630862236023,
"grad_norm": 25.899518966674805,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.40121355652809143,
"logits/rejected": -0.374218225479126,
"logps/chosen": -223.85598754882812,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -418.8667297363281,
"loss": 0.9517,
"margin_dpo/margin_mean": 173.0408935546875,
"margin_dpo/margin_std": 188.6272735595703,
"step": 241
},
{
"KL/chosen_KL_mean": -182.60260009765625,
"KL/mean": -244.80023193359375,
"KL/rejected_KL_mean": -306.9978942871094,
"KL/std": 141.99755859375,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.0036771988961845636,
"fcm_dpo/delta": -0.008242711424827576,
"fcm_dpo/margin": 124.39527893066406,
"fcm_dpo/q_t": 0.3988867402076721,
"grad_norm": 33.00807189941406,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.34588557481765747,
"logits/rejected": -0.3205920457839966,
"logps/chosen": -250.46652221679688,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -390.35821533203125,
"loss": 1.0644,
"margin_dpo/margin_mean": 124.39527893066406,
"margin_dpo/margin_std": 165.65414428710938,
"step": 242
},
{
"KL/chosen_KL_mean": -189.4199981689453,
"KL/mean": -265.73345947265625,
"KL/rejected_KL_mean": -342.04693603515625,
"KL/std": 162.802001953125,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.0035950704477727413,
"fcm_dpo/delta": -0.1046651154756546,
"fcm_dpo/margin": 152.62692260742188,
"fcm_dpo/q_t": 0.3775637149810791,
"grad_norm": 23.36472511291504,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.35052040219306946,
"logits/rejected": -0.3172234892845154,
"logps/chosen": -252.50424194335938,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -418.382568359375,
"loss": 1.002,
"margin_dpo/margin_mean": 152.62692260742188,
"margin_dpo/margin_std": 179.21963500976562,
"step": 243
},
{
"KL/chosen_KL_mean": -165.89877319335938,
"KL/mean": -243.54913330078125,
"KL/rejected_KL_mean": -321.19952392578125,
"KL/std": 156.23619079589844,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.0035196980461478233,
"fcm_dpo/delta": -0.10252860188484192,
"fcm_dpo/margin": 155.30075073242188,
"fcm_dpo/q_t": 0.37579482793807983,
"grad_norm": 31.815914154052734,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.4422183632850647,
"logits/rejected": -0.4340020716190338,
"logps/chosen": -227.03945922851562,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -416.0914306640625,
"loss": 0.9835,
"margin_dpo/margin_mean": 155.30075073242188,
"margin_dpo/margin_std": 158.2154541015625,
"step": 244
},
{
"KL/chosen_KL_mean": -185.9759521484375,
"KL/mean": -247.7681884765625,
"KL/rejected_KL_mean": -309.5604248046875,
"KL/std": 153.73590087890625,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.0034865711349993944,
"fcm_dpo/delta": 0.01841205358505249,
"fcm_dpo/margin": 123.58447265625,
"fcm_dpo/q_t": 0.4031563997268677,
"grad_norm": 30.421016693115234,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.3945258855819702,
"logits/rejected": -0.3583070635795593,
"logps/chosen": -253.23825073242188,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -397.2005310058594,
"loss": 1.0885,
"margin_dpo/margin_mean": 123.58447265625,
"margin_dpo/margin_std": 169.17431640625,
"step": 245
},
{
"KL/chosen_KL_mean": -179.17410278320312,
"KL/mean": -242.2460479736328,
"KL/rejected_KL_mean": -305.3179626464844,
"KL/std": 167.1992950439453,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.003517426550388336,
"fcm_dpo/delta": 0.006501421332359314,
"fcm_dpo/margin": 126.14387512207031,
"fcm_dpo/q_t": 0.40243223309516907,
"grad_norm": 23.121171951293945,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.45255112648010254,
"logits/rejected": -0.41291046142578125,
"logps/chosen": -245.87107849121094,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -389.664306640625,
"loss": 1.0763,
"margin_dpo/margin_mean": 126.14387512207031,
"margin_dpo/margin_std": 177.28350830078125,
"step": 246
},
{
"KL/chosen_KL_mean": -204.04139709472656,
"KL/mean": -289.42462158203125,
"KL/rejected_KL_mean": -374.80780029296875,
"KL/std": 170.71331787109375,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.0034450972452759743,
"fcm_dpo/delta": -0.14722290635108948,
"fcm_dpo/margin": 170.76641845703125,
"fcm_dpo/q_t": 0.36744260787963867,
"grad_norm": 27.307777404785156,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.38776254653930664,
"logits/rejected": -0.38508284091949463,
"logps/chosen": -260.646728515625,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -481.10107421875,
"loss": 0.971,
"margin_dpo/margin_mean": 170.76641845703125,
"margin_dpo/margin_std": 182.0000762939453,
"step": 247
},
{
"KL/chosen_KL_mean": -174.81797790527344,
"KL/mean": -248.60604858398438,
"KL/rejected_KL_mean": -322.3941345214844,
"KL/std": 136.95773315429688,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.003385141957551241,
"fcm_dpo/delta": -0.052636757493019104,
"fcm_dpo/margin": 147.57614135742188,
"fcm_dpo/q_t": 0.382654070854187,
"grad_norm": 25.895183563232422,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.38079047203063965,
"logits/rejected": -0.39816635847091675,
"logps/chosen": -218.86119079589844,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -414.2510070800781,
"loss": 0.9945,
"margin_dpo/margin_mean": 147.57614135742188,
"margin_dpo/margin_std": 130.29660034179688,
"step": 248
},
{
"KL/chosen_KL_mean": -225.7408905029297,
"KL/mean": -265.8685302734375,
"KL/rejected_KL_mean": -305.9962158203125,
"KL/std": 147.00021362304688,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.0034770106431096792,
"fcm_dpo/delta": 0.17593783140182495,
"fcm_dpo/margin": 80.25532531738281,
"fcm_dpo/q_t": 0.43721505999565125,
"grad_norm": 48.399410247802734,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.43213483691215515,
"logits/rejected": -0.40453487634658813,
"logps/chosen": -288.1832275390625,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -386.46429443359375,
"loss": 1.2089,
"margin_dpo/margin_mean": 80.25532531738281,
"margin_dpo/margin_std": 169.0428466796875,
"step": 249
},
{
"KL/chosen_KL_mean": -195.34091186523438,
"KL/mean": -271.7189025878906,
"KL/rejected_KL_mean": -348.096923828125,
"KL/std": 147.0026092529297,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.0034801422152668238,
"fcm_dpo/delta": -0.08620613068342209,
"fcm_dpo/margin": 152.75599670410156,
"fcm_dpo/q_t": 0.3768426179885864,
"grad_norm": 45.44241714477539,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.43205228447914124,
"logits/rejected": -0.3733093738555908,
"logps/chosen": -260.97760009765625,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -421.96875,
"loss": 0.9819,
"margin_dpo/margin_mean": 152.75599670410156,
"margin_dpo/margin_std": 142.21054077148438,
"step": 250
},
{
"KL/chosen_KL_mean": -213.68890380859375,
"KL/mean": -265.701904296875,
"KL/rejected_KL_mean": -317.71490478515625,
"KL/std": 156.36135864257812,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.0034832912497222424,
"fcm_dpo/delta": 0.09098668396472931,
"fcm_dpo/margin": 104.02600860595703,
"fcm_dpo/q_t": 0.4189932644367218,
"grad_norm": 30.63431739807129,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3991888761520386,
"logits/rejected": -0.3627336621284485,
"logps/chosen": -270.8716125488281,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -395.37835693359375,
"loss": 1.1403,
"margin_dpo/margin_mean": 104.02600860595703,
"margin_dpo/margin_std": 167.71755981445312,
"step": 251
},
{
"KL/chosen_KL_mean": -201.41134643554688,
"KL/mean": -272.4978332519531,
"KL/rejected_KL_mean": -343.5843505859375,
"KL/std": 137.3273162841797,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0034778579138219357,
"fcm_dpo/delta": -0.047106366604566574,
"fcm_dpo/margin": 142.17300415039062,
"fcm_dpo/q_t": 0.38277339935302734,
"grad_norm": 26.57343101501465,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.4910251498222351,
"logits/rejected": -0.4520424008369446,
"logps/chosen": -273.09698486328125,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -428.34234619140625,
"loss": 1.0023,
"margin_dpo/margin_mean": 142.17300415039062,
"margin_dpo/margin_std": 131.0556640625,
"step": 252
},
{
"KL/chosen_KL_mean": -171.62469482421875,
"KL/mean": -237.47177124023438,
"KL/rejected_KL_mean": -303.31884765625,
"KL/std": 145.25961303710938,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.0034945448860526085,
"fcm_dpo/delta": -0.011328611522912979,
"fcm_dpo/margin": 131.69415283203125,
"fcm_dpo/q_t": 0.39354246854782104,
"grad_norm": 30.445125579833984,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.4647923707962036,
"logits/rejected": -0.45453959703445435,
"logps/chosen": -240.75863647460938,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -402.0213928222656,
"loss": 1.0526,
"margin_dpo/margin_mean": 131.69415283203125,
"margin_dpo/margin_std": 157.84063720703125,
"step": 253
},
{
"KL/chosen_KL_mean": -175.22341918945312,
"KL/mean": -234.58761596679688,
"KL/rejected_KL_mean": -293.9517822265625,
"KL/std": 153.671142578125,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.0034880484454333782,
"fcm_dpo/delta": 0.03743617236614227,
"fcm_dpo/margin": 118.72837829589844,
"fcm_dpo/q_t": 0.4104461371898651,
"grad_norm": 24.665857315063477,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.4090794026851654,
"logits/rejected": -0.3888247609138489,
"logps/chosen": -229.37841796875,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -384.2594299316406,
"loss": 1.1125,
"margin_dpo/margin_mean": 118.72837829589844,
"margin_dpo/margin_std": 191.54190063476562,
"step": 254
},
{
"KL/chosen_KL_mean": -180.15988159179688,
"KL/mean": -243.97674560546875,
"KL/rejected_KL_mean": -307.79364013671875,
"KL/std": 137.28518676757812,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.0034879068844020367,
"fcm_dpo/delta": 0.004787445068359375,
"fcm_dpo/margin": 127.63375091552734,
"fcm_dpo/q_t": 0.3985871970653534,
"grad_norm": 22.473079681396484,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.3677319884300232,
"logits/rejected": -0.35488447546958923,
"logps/chosen": -237.30154418945312,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -398.0022277832031,
"loss": 1.0758,
"margin_dpo/margin_mean": 127.63375091552734,
"margin_dpo/margin_std": 172.2808074951172,
"step": 255
},
{
"KL/chosen_KL_mean": -152.079345703125,
"KL/mean": -220.30516052246094,
"KL/rejected_KL_mean": -288.53094482421875,
"KL/std": 147.42538452148438,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.003487388137727976,
"fcm_dpo/delta": -0.02715311199426651,
"fcm_dpo/margin": 136.45159912109375,
"fcm_dpo/q_t": 0.3921736478805542,
"grad_norm": 25.29527473449707,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.358262300491333,
"logits/rejected": -0.36426687240600586,
"logps/chosen": -207.2428436279297,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -381.0938720703125,
"loss": 1.0442,
"margin_dpo/margin_mean": 136.45159912109375,
"margin_dpo/margin_std": 168.40225219726562,
"step": 256
},
{
"KL/chosen_KL_mean": -150.44158935546875,
"KL/mean": -220.73284912109375,
"KL/rejected_KL_mean": -291.0240783691406,
"KL/std": 151.136474609375,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.003438519313931465,
"fcm_dpo/delta": -0.03647083416581154,
"fcm_dpo/margin": 140.58251953125,
"fcm_dpo/q_t": 0.3912065625190735,
"grad_norm": 26.34146499633789,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.36043840646743774,
"logits/rejected": -0.3441726565361023,
"logps/chosen": -199.86529541015625,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -370.56201171875,
"loss": 1.043,
"margin_dpo/margin_mean": 140.58251953125,
"margin_dpo/margin_std": 168.63809204101562,
"step": 257
},
{
"KL/chosen_KL_mean": -197.21380615234375,
"KL/mean": -265.21221923828125,
"KL/rejected_KL_mean": -333.2106628417969,
"KL/std": 157.57400512695312,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.003452843287959695,
"fcm_dpo/delta": -0.02056654915213585,
"fcm_dpo/margin": 135.99685668945312,
"fcm_dpo/q_t": 0.39261341094970703,
"grad_norm": 26.73494529724121,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.34393584728240967,
"logits/rejected": -0.3391512632369995,
"logps/chosen": -256.5979309082031,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -429.20074462890625,
"loss": 1.0654,
"margin_dpo/margin_mean": 135.9968719482422,
"margin_dpo/margin_std": 185.92190551757812,
"step": 258
},
{
"KL/chosen_KL_mean": -186.67657470703125,
"KL/mean": -245.43338012695312,
"KL/rejected_KL_mean": -304.190185546875,
"KL/std": 151.03363037109375,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.0034547275863587856,
"fcm_dpo/delta": 0.04579560458660126,
"fcm_dpo/margin": 117.51361083984375,
"fcm_dpo/q_t": 0.4087771773338318,
"grad_norm": 26.122661590576172,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.4058660864830017,
"logits/rejected": -0.4022112488746643,
"logps/chosen": -239.50491333007812,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -393.3818359375,
"loss": 1.1018,
"margin_dpo/margin_mean": 117.51361083984375,
"margin_dpo/margin_std": 169.58209228515625,
"step": 259
},
{
"KL/chosen_KL_mean": -197.8670654296875,
"KL/mean": -275.69793701171875,
"KL/rejected_KL_mean": -353.52880859375,
"KL/std": 160.15545654296875,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.003441175911575556,
"fcm_dpo/delta": -0.09045177698135376,
"fcm_dpo/margin": 155.6616973876953,
"fcm_dpo/q_t": 0.3801797330379486,
"grad_norm": 28.81717872619629,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.3697262406349182,
"logits/rejected": -0.3860480487346649,
"logps/chosen": -245.2847442626953,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -448.61859130859375,
"loss": 0.9959,
"margin_dpo/margin_mean": 155.6616973876953,
"margin_dpo/margin_std": 170.5771942138672,
"step": 260
},
{
"KL/chosen_KL_mean": -203.37054443359375,
"KL/mean": -278.1752624511719,
"KL/rejected_KL_mean": -352.9800109863281,
"KL/std": 175.67840576171875,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.003373971674591303,
"fcm_dpo/delta": -0.05800588056445122,
"fcm_dpo/margin": 149.6094970703125,
"fcm_dpo/q_t": 0.3877621293067932,
"grad_norm": 22.53188705444336,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.3110651969909668,
"logits/rejected": -0.3086443543434143,
"logps/chosen": -256.40191650390625,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -441.4949645996094,
"loss": 1.0402,
"margin_dpo/margin_mean": 149.6094970703125,
"margin_dpo/margin_std": 194.22335815429688,
"step": 261
},
{
"KL/chosen_KL_mean": -238.98031616210938,
"KL/mean": -299.66339111328125,
"KL/rejected_KL_mean": -360.346435546875,
"KL/std": 159.51092529296875,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.003405648982152343,
"fcm_dpo/delta": 0.037785448133945465,
"fcm_dpo/margin": 121.36612701416016,
"fcm_dpo/q_t": 0.4051642119884491,
"grad_norm": 28.0867919921875,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.31964707374572754,
"logits/rejected": -0.2917061746120453,
"logps/chosen": -298.6004638671875,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -446.76495361328125,
"loss": 1.0826,
"margin_dpo/margin_mean": 121.36612701416016,
"margin_dpo/margin_std": 158.1082000732422,
"step": 262
},
{
"KL/chosen_KL_mean": -213.3681182861328,
"KL/mean": -300.8740234375,
"KL/rejected_KL_mean": -388.3798828125,
"KL/std": 187.53240966796875,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.0033140634186565876,
"fcm_dpo/delta": -0.13908036053180695,
"fcm_dpo/margin": 175.0117645263672,
"fcm_dpo/q_t": 0.3701428174972534,
"grad_norm": 23.549612045288086,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.3330538868904114,
"logits/rejected": -0.3084709942340851,
"logps/chosen": -272.7890625,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -485.23712158203125,
"loss": 0.9859,
"margin_dpo/margin_mean": 175.0117645263672,
"margin_dpo/margin_std": 198.38455200195312,
"step": 263
},
{
"KL/chosen_KL_mean": -223.22073364257812,
"KL/mean": -297.41522216796875,
"KL/rejected_KL_mean": -371.6096496582031,
"KL/std": 168.37533569335938,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.0032626159954816103,
"fcm_dpo/delta": -0.03732195869088173,
"fcm_dpo/margin": 148.388916015625,
"fcm_dpo/q_t": 0.39182916283607483,
"grad_norm": 26.550914764404297,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.38302069902420044,
"logits/rejected": -0.37319496273994446,
"logps/chosen": -285.94281005859375,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -465.46588134765625,
"loss": 1.0515,
"margin_dpo/margin_mean": 148.388916015625,
"margin_dpo/margin_std": 190.17330932617188,
"step": 264
},
{
"KL/chosen_KL_mean": -246.03799438476562,
"KL/mean": -317.5897216796875,
"KL/rejected_KL_mean": -389.1414489746094,
"KL/std": 192.48684692382812,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.003271650057286024,
"fcm_dpo/delta": -0.01907782442867756,
"fcm_dpo/margin": 143.10342407226562,
"fcm_dpo/q_t": 0.3984997272491455,
"grad_norm": 30.72321319580078,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.33858364820480347,
"logits/rejected": -0.3238202929496765,
"logps/chosen": -308.00946044921875,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -477.16204833984375,
"loss": 1.0942,
"margin_dpo/margin_mean": 143.10342407226562,
"margin_dpo/margin_std": 228.45199584960938,
"step": 265
},
{
"KL/chosen_KL_mean": -244.74679565429688,
"KL/mean": -304.958740234375,
"KL/rejected_KL_mean": -365.17071533203125,
"KL/std": 162.22457885742188,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0032967233564704657,
"fcm_dpo/delta": 0.055163055658340454,
"fcm_dpo/margin": 120.42388916015625,
"fcm_dpo/q_t": 0.41121941804885864,
"grad_norm": 51.763267517089844,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.3775002956390381,
"logits/rejected": -0.33649495244026184,
"logps/chosen": -311.8464660644531,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -433.1419372558594,
"loss": 1.1123,
"margin_dpo/margin_mean": 120.42388916015625,
"margin_dpo/margin_std": 183.71939086914062,
"step": 266
},
{
"KL/chosen_KL_mean": -217.10748291015625,
"KL/mean": -288.5802001953125,
"KL/rejected_KL_mean": -360.0528869628906,
"KL/std": 174.79310607910156,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.0032819192856550217,
"fcm_dpo/delta": -0.02046758309006691,
"fcm_dpo/margin": 142.94540405273438,
"fcm_dpo/q_t": 0.39428484439849854,
"grad_norm": 41.82976531982422,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3888688385486603,
"logits/rejected": -0.3568268418312073,
"logps/chosen": -286.0782470703125,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -450.2213134765625,
"loss": 1.0632,
"margin_dpo/margin_mean": 142.94540405273438,
"margin_dpo/margin_std": 193.6571044921875,
"step": 267
},
{
"KL/chosen_KL_mean": -226.6894073486328,
"KL/mean": -294.6236267089844,
"KL/rejected_KL_mean": -362.557861328125,
"KL/std": 167.1015625,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.0032822336070239544,
"fcm_dpo/delta": 0.0040205977857112885,
"fcm_dpo/margin": 135.8684539794922,
"fcm_dpo/q_t": 0.40099048614501953,
"grad_norm": 38.428096771240234,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.3436671793460846,
"logits/rejected": -0.3497740626335144,
"logps/chosen": -282.5897216796875,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -464.20550537109375,
"loss": 1.0847,
"margin_dpo/margin_mean": 135.8684539794922,
"margin_dpo/margin_std": 199.06204223632812,
"step": 268
},
{
"KL/chosen_KL_mean": -241.2147216796875,
"KL/mean": -326.4340515136719,
"KL/rejected_KL_mean": -411.65338134765625,
"KL/std": 169.1653594970703,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.0032417180482298136,
"fcm_dpo/delta": -0.108495332300663,
"fcm_dpo/margin": 170.43869018554688,
"fcm_dpo/q_t": 0.3768116235733032,
"grad_norm": 27.486454010009766,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.3986158072948456,
"logits/rejected": -0.37994933128356934,
"logps/chosen": -311.2542724609375,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -519.0028076171875,
"loss": 1.018,
"margin_dpo/margin_mean": 170.43869018554688,
"margin_dpo/margin_std": 214.85861206054688,
"step": 269
},
{
"KL/chosen_KL_mean": -214.1920166015625,
"KL/mean": -277.78076171875,
"KL/rejected_KL_mean": -341.36956787109375,
"KL/std": 151.48452758789062,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.0032509169541299343,
"fcm_dpo/delta": 0.037847500294446945,
"fcm_dpo/margin": 127.17756652832031,
"fcm_dpo/q_t": 0.40676695108413696,
"grad_norm": 33.916015625,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.4463101029396057,
"logits/rejected": -0.44212576746940613,
"logps/chosen": -283.7254943847656,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -451.2982177734375,
"loss": 1.1022,
"margin_dpo/margin_mean": 127.17755126953125,
"margin_dpo/margin_std": 188.5105438232422,
"step": 270
},
{
"KL/chosen_KL_mean": -199.34915161132812,
"KL/mean": -278.3819885253906,
"KL/rejected_KL_mean": -357.4148254394531,
"KL/std": 152.5343017578125,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.0032084607519209385,
"fcm_dpo/delta": -0.06059388071298599,
"fcm_dpo/margin": 158.065673828125,
"fcm_dpo/q_t": 0.38492733240127563,
"grad_norm": 27.501827239990234,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.36121702194213867,
"logits/rejected": -0.3439444303512573,
"logps/chosen": -256.11370849609375,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -449.92864990234375,
"loss": 1.0118,
"margin_dpo/margin_mean": 158.06568908691406,
"margin_dpo/margin_std": 172.20005798339844,
"step": 271
},
{
"KL/chosen_KL_mean": -184.9806671142578,
"KL/mean": -277.00921630859375,
"KL/rejected_KL_mean": -369.03778076171875,
"KL/std": 169.01669311523438,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.003157893195748329,
"fcm_dpo/delta": -0.13913074135780334,
"fcm_dpo/margin": 184.05712890625,
"fcm_dpo/q_t": 0.3664304316043854,
"grad_norm": 45.42236328125,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.31142860651016235,
"logits/rejected": -0.3235180079936981,
"logps/chosen": -234.4778289794922,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -474.58056640625,
"loss": 0.9588,
"margin_dpo/margin_mean": 184.05712890625,
"margin_dpo/margin_std": 174.16859436035156,
"step": 272
},
{
"KL/chosen_KL_mean": -210.1597900390625,
"KL/mean": -300.50250244140625,
"KL/rejected_KL_mean": -390.84515380859375,
"KL/std": 166.0848846435547,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.003058584639802575,
"fcm_dpo/delta": -0.10935865342617035,
"fcm_dpo/margin": 180.68539428710938,
"fcm_dpo/q_t": 0.3739517629146576,
"grad_norm": 30.045597076416016,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.3383510410785675,
"logits/rejected": -0.3233621120452881,
"logps/chosen": -273.13519287109375,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -483.34375,
"loss": 0.9825,
"margin_dpo/margin_mean": 180.68539428710938,
"margin_dpo/margin_std": 186.4752960205078,
"step": 273
},
{
"KL/chosen_KL_mean": -256.2316589355469,
"KL/mean": -327.0481872558594,
"KL/rejected_KL_mean": -397.86468505859375,
"KL/std": 157.5728759765625,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.0030697216279804707,
"fcm_dpo/delta": 0.01566966250538826,
"fcm_dpo/margin": 141.63302612304688,
"fcm_dpo/q_t": 0.40060126781463623,
"grad_norm": 35.71431350708008,
"learning_rate": 3.75e-07,
"logits/chosen": -0.317413866519928,
"logits/rejected": -0.30049943923950195,
"logps/chosen": -311.8993835449219,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -475.1977844238281,
"loss": 1.067,
"margin_dpo/margin_mean": 141.63304138183594,
"margin_dpo/margin_std": 176.70895385742188,
"step": 274
},
{
"KL/chosen_KL_mean": -195.29779052734375,
"KL/mean": -267.0739440917969,
"KL/rejected_KL_mean": -338.85009765625,
"KL/std": 158.72325134277344,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.003075392683967948,
"fcm_dpo/delta": 0.008720289915800095,
"fcm_dpo/margin": 143.55230712890625,
"fcm_dpo/q_t": 0.39867332577705383,
"grad_norm": 28.375545501708984,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.3294273614883423,
"logits/rejected": -0.3342640697956085,
"logps/chosen": -243.89248657226562,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -432.15380859375,
"loss": 1.0615,
"margin_dpo/margin_mean": 143.55230712890625,
"margin_dpo/margin_std": 175.27035522460938,
"step": 275
},
{
"KL/chosen_KL_mean": -210.61480712890625,
"KL/mean": -287.1915283203125,
"KL/rejected_KL_mean": -363.7682800292969,
"KL/std": 161.99810791015625,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.003052051179111004,
"fcm_dpo/delta": -0.01845492050051689,
"fcm_dpo/margin": 153.15347290039062,
"fcm_dpo/q_t": 0.39402562379837036,
"grad_norm": 37.641090393066406,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.3240549564361572,
"logits/rejected": -0.29721617698669434,
"logps/chosen": -267.19219970703125,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -434.13397216796875,
"loss": 1.0475,
"margin_dpo/margin_mean": 153.1534881591797,
"margin_dpo/margin_std": 187.49969482421875,
"step": 276
},
{
"KL/chosen_KL_mean": -226.70089721679688,
"KL/mean": -302.93359375,
"KL/rejected_KL_mean": -379.166259765625,
"KL/std": 161.0653076171875,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.0030543706379830837,
"fcm_dpo/delta": -0.016483565792441368,
"fcm_dpo/margin": 152.46539306640625,
"fcm_dpo/q_t": 0.39358431100845337,
"grad_norm": 27.86901092529297,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.3397817611694336,
"logits/rejected": -0.32286733388900757,
"logps/chosen": -282.97247314453125,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -472.04754638671875,
"loss": 1.049,
"margin_dpo/margin_mean": 152.46539306640625,
"margin_dpo/margin_std": 185.9540557861328,
"step": 277
},
{
"KL/chosen_KL_mean": -201.0662078857422,
"KL/mean": -289.1669921875,
"KL/rejected_KL_mean": -377.267822265625,
"KL/std": 173.0821533203125,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.0030011532362550497,
"fcm_dpo/delta": -0.08356461673974991,
"fcm_dpo/margin": 176.2015838623047,
"fcm_dpo/q_t": 0.38064807653427124,
"grad_norm": 24.03802490234375,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.3009544014930725,
"logits/rejected": -0.30515652894973755,
"logps/chosen": -254.00814819335938,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -468.5213928222656,
"loss": 1.0017,
"margin_dpo/margin_mean": 176.2015838623047,
"margin_dpo/margin_std": 193.50991821289062,
"step": 278
},
{
"KL/chosen_KL_mean": -227.3751220703125,
"KL/mean": -315.3472595214844,
"KL/rejected_KL_mean": -403.3194274902344,
"KL/std": 180.30804443359375,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.002944803796708584,
"fcm_dpo/delta": -0.07315056025981903,
"fcm_dpo/margin": 175.94432067871094,
"fcm_dpo/q_t": 0.3842847943305969,
"grad_norm": 28.466480255126953,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.2807590961456299,
"logits/rejected": -0.278242826461792,
"logps/chosen": -276.01641845703125,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -491.1708679199219,
"loss": 1.0292,
"margin_dpo/margin_mean": 175.94430541992188,
"margin_dpo/margin_std": 218.19247436523438,
"step": 279
},
{
"KL/chosen_KL_mean": -223.5345458984375,
"KL/mean": -311.9327392578125,
"KL/rejected_KL_mean": -400.33087158203125,
"KL/std": 166.27877807617188,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.00291811628267169,
"fcm_dpo/delta": -0.06963366270065308,
"fcm_dpo/margin": 176.79629516601562,
"fcm_dpo/q_t": 0.3800342082977295,
"grad_norm": 34.31902313232422,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.3296852707862854,
"logits/rejected": -0.33157432079315186,
"logps/chosen": -282.3316650390625,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -498.94970703125,
"loss": 0.9995,
"margin_dpo/margin_mean": 176.79629516601562,
"margin_dpo/margin_std": 179.59506225585938,
"step": 280
},
{
"KL/chosen_KL_mean": -202.8314666748047,
"KL/mean": -283.94891357421875,
"KL/rejected_KL_mean": -365.06640625,
"KL/std": 163.90113830566406,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0029120631515979767,
"fcm_dpo/delta": -0.023735320195555687,
"fcm_dpo/margin": 162.2349395751953,
"fcm_dpo/q_t": 0.38894733786582947,
"grad_norm": 22.588340759277344,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.36799153685569763,
"logits/rejected": -0.3402552008628845,
"logps/chosen": -258.3199768066406,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -445.948974609375,
"loss": 1.0321,
"margin_dpo/margin_mean": 162.23492431640625,
"margin_dpo/margin_std": 176.25933837890625,
"step": 281
},
{
"KL/chosen_KL_mean": -234.31324768066406,
"KL/mean": -297.2037353515625,
"KL/rejected_KL_mean": -360.09423828125,
"KL/std": 174.16539001464844,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.0029452815651893616,
"fcm_dpo/delta": 0.08194846659898758,
"fcm_dpo/margin": 125.781005859375,
"fcm_dpo/q_t": 0.41833823919296265,
"grad_norm": 31.682376861572266,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.41140317916870117,
"logits/rejected": -0.3855805993080139,
"logps/chosen": -307.3833923339844,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -455.44525146484375,
"loss": 1.1245,
"margin_dpo/margin_mean": 125.781005859375,
"margin_dpo/margin_std": 198.57614135742188,
"step": 282
},
{
"KL/chosen_KL_mean": -242.9020233154297,
"KL/mean": -337.974853515625,
"KL/rejected_KL_mean": -433.0476989746094,
"KL/std": 197.83787536621094,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.0029110005125403404,
"fcm_dpo/delta": -0.10949087888002396,
"fcm_dpo/margin": 190.14569091796875,
"fcm_dpo/q_t": 0.37221860885620117,
"grad_norm": 29.892505645751953,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.339688241481781,
"logits/rejected": -0.3391629457473755,
"logps/chosen": -304.80047607421875,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -530.0342407226562,
"loss": 0.9779,
"margin_dpo/margin_mean": 190.14569091796875,
"margin_dpo/margin_std": 186.28033447265625,
"step": 283
},
{
"KL/chosen_KL_mean": -236.25758361816406,
"KL/mean": -328.17041015625,
"KL/rejected_KL_mean": -420.08319091796875,
"KL/std": 193.72735595703125,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.002844389993697405,
"fcm_dpo/delta": -0.07708093523979187,
"fcm_dpo/margin": 183.82562255859375,
"fcm_dpo/q_t": 0.3810979723930359,
"grad_norm": 27.589719772338867,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.40438514947891235,
"logits/rejected": -0.39065611362457275,
"logps/chosen": -294.693115234375,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -513.552490234375,
"loss": 0.9975,
"margin_dpo/margin_mean": 183.82562255859375,
"margin_dpo/margin_std": 193.533447265625,
"step": 284
},
{
"KL/chosen_KL_mean": -280.03094482421875,
"KL/mean": -361.1250305175781,
"KL/rejected_KL_mean": -442.2191467285156,
"KL/std": 169.0643310546875,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.0028481141198426485,
"fcm_dpo/delta": -0.013057064265012741,
"fcm_dpo/margin": 162.18820190429688,
"fcm_dpo/q_t": 0.3921695351600647,
"grad_norm": 26.22045135498047,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.3738415241241455,
"logits/rejected": -0.3561505079269409,
"logps/chosen": -346.26312255859375,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -541.345947265625,
"loss": 1.043,
"margin_dpo/margin_mean": 162.18821716308594,
"margin_dpo/margin_std": 182.13633728027344,
"step": 285
},
{
"KL/chosen_KL_mean": -290.49481201171875,
"KL/mean": -380.4010009765625,
"KL/rejected_KL_mean": -470.30718994140625,
"KL/std": 196.43020629882812,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.002812919206917286,
"fcm_dpo/delta": -0.058750685304403305,
"fcm_dpo/margin": 179.81234741210938,
"fcm_dpo/q_t": 0.38490667939186096,
"grad_norm": 30.215322494506836,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.37286484241485596,
"logits/rejected": -0.354168564081192,
"logps/chosen": -363.44580078125,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -558.8956298828125,
"loss": 1.0278,
"margin_dpo/margin_mean": 179.81234741210938,
"margin_dpo/margin_std": 214.6378173828125,
"step": 286
},
{
"KL/chosen_KL_mean": -271.04339599609375,
"KL/mean": -347.6123352050781,
"KL/rejected_KL_mean": -424.1812744140625,
"KL/std": 178.92794799804688,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.002809434197843075,
"fcm_dpo/delta": 0.020603910088539124,
"fcm_dpo/margin": 153.13784790039062,
"fcm_dpo/q_t": 0.40201419591903687,
"grad_norm": 26.337621688842773,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.3177964985370636,
"logits/rejected": -0.28946465253829956,
"logps/chosen": -332.5845642089844,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -501.8773498535156,
"loss": 1.0687,
"margin_dpo/margin_mean": 153.1378631591797,
"margin_dpo/margin_std": 192.58251953125,
"step": 287
},
{
"KL/chosen_KL_mean": -267.706298828125,
"KL/mean": -359.65765380859375,
"KL/rejected_KL_mean": -451.6090087890625,
"KL/std": 176.45693969726562,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.002759437309578061,
"fcm_dpo/delta": -0.06256371736526489,
"fcm_dpo/margin": 183.90274047851562,
"fcm_dpo/q_t": 0.38231146335601807,
"grad_norm": 24.027070999145508,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.3309929370880127,
"logits/rejected": -0.31245338916778564,
"logps/chosen": -324.3675537109375,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -538.9447021484375,
"loss": 1.0039,
"margin_dpo/margin_mean": 183.90274047851562,
"margin_dpo/margin_std": 178.67431640625,
"step": 288
},
{
"KL/chosen_KL_mean": -256.2225036621094,
"KL/mean": -358.1459045410156,
"KL/rejected_KL_mean": -460.0693359375,
"KL/std": 192.28744506835938,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.002735921647399664,
"fcm_dpo/delta": -0.11391443759202957,
"fcm_dpo/margin": 203.8468017578125,
"fcm_dpo/q_t": 0.3737669587135315,
"grad_norm": 33.68012619018555,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.27321916818618774,
"logits/rejected": -0.2723183333873749,
"logps/chosen": -301.4529113769531,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -547.7120361328125,
"loss": 0.9817,
"margin_dpo/margin_mean": 203.8468017578125,
"margin_dpo/margin_std": 211.8802032470703,
"step": 289
},
{
"KL/chosen_KL_mean": -269.2728271484375,
"KL/mean": -369.11712646484375,
"KL/rejected_KL_mean": -468.96142578125,
"KL/std": 200.68923950195312,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.0026709740050137043,
"fcm_dpo/delta": -0.08831434696912766,
"fcm_dpo/margin": 199.6885986328125,
"fcm_dpo/q_t": 0.3807799816131592,
"grad_norm": 24.526521682739258,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.28396081924438477,
"logits/rejected": -0.3037373423576355,
"logps/chosen": -324.74432373046875,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -585.6699829101562,
"loss": 1.0115,
"margin_dpo/margin_mean": 199.68861389160156,
"margin_dpo/margin_std": 236.9954376220703,
"step": 290
},
{
"KL/chosen_KL_mean": -227.63702392578125,
"KL/mean": -321.8421630859375,
"KL/rejected_KL_mean": -416.04730224609375,
"KL/std": 173.67169189453125,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.002654793206602335,
"fcm_dpo/delta": -0.052990131080150604,
"fcm_dpo/margin": 188.41024780273438,
"fcm_dpo/q_t": 0.38466930389404297,
"grad_norm": 25.388965606689453,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.3481978476047516,
"logits/rejected": -0.3348592519760132,
"logps/chosen": -290.89739990234375,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -505.3443603515625,
"loss": 1.0182,
"margin_dpo/margin_mean": 188.41024780273438,
"margin_dpo/margin_std": 205.23170471191406,
"step": 291
},
{
"KL/chosen_KL_mean": -248.2539520263672,
"KL/mean": -343.5125427246094,
"KL/rejected_KL_mean": -438.7711181640625,
"KL/std": 202.5133056640625,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0026254309341311455,
"fcm_dpo/delta": -0.052929122000932693,
"fcm_dpo/margin": 190.51718139648438,
"fcm_dpo/q_t": 0.3859769105911255,
"grad_norm": 24.36165428161621,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.32978010177612305,
"logits/rejected": -0.3151566982269287,
"logps/chosen": -302.1724853515625,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -528.7324829101562,
"loss": 1.0359,
"margin_dpo/margin_mean": 190.51715087890625,
"margin_dpo/margin_std": 235.57249450683594,
"step": 292
},
{
"KL/chosen_KL_mean": -238.14736938476562,
"KL/mean": -313.1924743652344,
"KL/rejected_KL_mean": -388.237548828125,
"KL/std": 196.93975830078125,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.0026358608156442642,
"fcm_dpo/delta": 0.05641322582960129,
"fcm_dpo/margin": 150.09017944335938,
"fcm_dpo/q_t": 0.411357581615448,
"grad_norm": 45.91537094116211,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.3480873107910156,
"logits/rejected": -0.336029052734375,
"logps/chosen": -298.5234069824219,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -466.09002685546875,
"loss": 1.11,
"margin_dpo/margin_mean": 150.09017944335938,
"margin_dpo/margin_std": 224.6016845703125,
"step": 293
},
{
"KL/chosen_KL_mean": -212.2786865234375,
"KL/mean": -297.28173828125,
"KL/rejected_KL_mean": -382.2847900390625,
"KL/std": 179.94796752929688,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.002632549963891506,
"fcm_dpo/delta": 0.0024539902806282043,
"fcm_dpo/margin": 170.00611877441406,
"fcm_dpo/q_t": 0.39919567108154297,
"grad_norm": 30.026025772094727,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.3344029486179352,
"logits/rejected": -0.3364221453666687,
"logps/chosen": -260.3662109375,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -464.1817932128906,
"loss": 1.0643,
"margin_dpo/margin_mean": 170.00611877441406,
"margin_dpo/margin_std": 218.8177490234375,
"step": 294
},
{
"KL/chosen_KL_mean": -257.17437744140625,
"KL/mean": -346.5780029296875,
"KL/rejected_KL_mean": -435.98162841796875,
"KL/std": 194.82772827148438,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.0026338594034314156,
"fcm_dpo/delta": -0.021993350237607956,
"fcm_dpo/margin": 178.8072509765625,
"fcm_dpo/q_t": 0.3934444487094879,
"grad_norm": 30.588729858398438,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.3404906392097473,
"logits/rejected": -0.34150180220603943,
"logps/chosen": -307.0990295410156,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -523.43798828125,
"loss": 1.0529,
"margin_dpo/margin_mean": 178.80726623535156,
"margin_dpo/margin_std": 226.52297973632812,
"step": 295
},
{
"KL/chosen_KL_mean": -312.295654296875,
"KL/mean": -379.9571533203125,
"KL/rejected_KL_mean": -447.61859130859375,
"KL/std": 192.12496948242188,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.0026506248395889997,
"fcm_dpo/delta": 0.09461938589811325,
"fcm_dpo/margin": 135.32293701171875,
"fcm_dpo/q_t": 0.41918545961380005,
"grad_norm": 41.94928741455078,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.39753079414367676,
"logits/rejected": -0.37970608472824097,
"logps/chosen": -377.7868957519531,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -540.7076416015625,
"loss": 1.1598,
"margin_dpo/margin_mean": 135.32293701171875,
"margin_dpo/margin_std": 244.38113403320312,
"step": 296
},
{
"KL/chosen_KL_mean": -284.9562683105469,
"KL/mean": -362.5632019042969,
"KL/rejected_KL_mean": -440.17010498046875,
"KL/std": 185.80307006835938,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.0026883059181272984,
"fcm_dpo/delta": 0.03417515754699707,
"fcm_dpo/margin": 155.21383666992188,
"fcm_dpo/q_t": 0.403796911239624,
"grad_norm": 30.55962371826172,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.3520761728286743,
"logits/rejected": -0.3518163561820984,
"logps/chosen": -341.4332275390625,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -535.3086547851562,
"loss": 1.0703,
"margin_dpo/margin_mean": 155.21383666992188,
"margin_dpo/margin_std": 185.96414184570312,
"step": 297
},
{
"KL/chosen_KL_mean": -313.055419921875,
"KL/mean": -412.99261474609375,
"KL/rejected_KL_mean": -512.9298095703125,
"KL/std": 245.9295654296875,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.0026719551533460617,
"fcm_dpo/delta": -0.08878251910209656,
"fcm_dpo/margin": 199.8743896484375,
"fcm_dpo/q_t": 0.38617774844169617,
"grad_norm": 27.358156204223633,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.41723042726516724,
"logits/rejected": -0.43608519434928894,
"logps/chosen": -380.38055419921875,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -629.5919799804688,
"loss": 1.0429,
"margin_dpo/margin_mean": 199.8743896484375,
"margin_dpo/margin_std": 282.69732666015625,
"step": 298
},
{
"KL/chosen_KL_mean": -258.68292236328125,
"KL/mean": -347.59979248046875,
"KL/rejected_KL_mean": -436.51666259765625,
"KL/std": 190.63723754882812,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.002657189266756177,
"fcm_dpo/delta": -0.02443765103816986,
"fcm_dpo/margin": 177.833740234375,
"fcm_dpo/q_t": 0.395521342754364,
"grad_norm": 39.517574310302734,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.3908317983150482,
"logits/rejected": -0.3915952742099762,
"logps/chosen": -307.64501953125,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -520.8449096679688,
"loss": 1.0625,
"margin_dpo/margin_mean": 177.83375549316406,
"margin_dpo/margin_std": 237.70399475097656,
"step": 299
},
{
"KL/chosen_KL_mean": -322.5706787109375,
"KL/mean": -428.1722106933594,
"KL/rejected_KL_mean": -533.7738037109375,
"KL/std": 225.27737426757812,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.002602781169116497,
"fcm_dpo/delta": -0.10538507252931595,
"fcm_dpo/margin": 211.20306396484375,
"fcm_dpo/q_t": 0.37716758251190186,
"grad_norm": 39.216007232666016,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.3786313533782959,
"logits/rejected": -0.38016241788864136,
"logps/chosen": -381.6443786621094,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -629.740234375,
"loss": 1.0203,
"margin_dpo/margin_mean": 211.20306396484375,
"margin_dpo/margin_std": 264.9863586425781,
"step": 300
},
{
"KL/chosen_KL_mean": -260.64385986328125,
"KL/mean": -361.54193115234375,
"KL/rejected_KL_mean": -462.44000244140625,
"KL/std": 204.75218200683594,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.002545170485973358,
"fcm_dpo/delta": -0.06761518120765686,
"fcm_dpo/margin": 201.79612731933594,
"fcm_dpo/q_t": 0.38585570454597473,
"grad_norm": 25.292051315307617,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.3724789023399353,
"logits/rejected": -0.3688158392906189,
"logps/chosen": -317.89324951171875,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -554.7935791015625,
"loss": 1.0217,
"margin_dpo/margin_mean": 201.796142578125,
"margin_dpo/margin_std": 242.0333251953125,
"step": 301
},
{
"KL/chosen_KL_mean": -214.75692749023438,
"KL/mean": -296.821533203125,
"KL/rejected_KL_mean": -378.88616943359375,
"KL/std": 178.21484375,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.0025499570183455944,
"fcm_dpo/delta": 0.0327281579375267,
"fcm_dpo/margin": 164.1292266845703,
"fcm_dpo/q_t": 0.404863178730011,
"grad_norm": 42.442935943603516,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.3850117623806,
"logits/rejected": -0.39180663228034973,
"logps/chosen": -265.9549255371094,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -476.1125183105469,
"loss": 1.0753,
"margin_dpo/margin_mean": 164.12924194335938,
"margin_dpo/margin_std": 206.58688354492188,
"step": 302
},
{
"KL/chosen_KL_mean": -237.27133178710938,
"KL/mean": -314.0264892578125,
"KL/rejected_KL_mean": -390.7816162109375,
"KL/std": 187.21142578125,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.002579642226919532,
"fcm_dpo/delta": 0.05625869706273079,
"fcm_dpo/margin": 153.51028442382812,
"fcm_dpo/q_t": 0.41086673736572266,
"grad_norm": 34.44538879394531,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.39609721302986145,
"logits/rejected": -0.37636637687683105,
"logps/chosen": -303.98529052734375,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -477.72705078125,
"loss": 1.1086,
"margin_dpo/margin_mean": 153.51028442382812,
"margin_dpo/margin_std": 225.28631591796875,
"step": 303
},
{
"KL/chosen_KL_mean": -219.05499267578125,
"KL/mean": -307.2504577636719,
"KL/rejected_KL_mean": -395.4459228515625,
"KL/std": 178.584716796875,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.002598279621452093,
"fcm_dpo/delta": -0.008973244577646255,
"fcm_dpo/margin": 176.39096069335938,
"fcm_dpo/q_t": 0.39162832498550415,
"grad_norm": 46.01495361328125,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.4321674704551697,
"logits/rejected": -0.42111736536026,
"logps/chosen": -291.00567626953125,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -485.91796875,
"loss": 1.0203,
"margin_dpo/margin_mean": 176.39096069335938,
"margin_dpo/margin_std": 155.45849609375,
"step": 304
},
{
"KL/chosen_KL_mean": -252.63079833984375,
"KL/mean": -331.663818359375,
"KL/rejected_KL_mean": -410.69683837890625,
"KL/std": 212.29205322265625,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.0025871852412819862,
"fcm_dpo/delta": 0.042250074446201324,
"fcm_dpo/margin": 158.06600952148438,
"fcm_dpo/q_t": 0.40933629870414734,
"grad_norm": 27.672773361206055,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.4089544415473938,
"logits/rejected": -0.38962921500205994,
"logps/chosen": -319.426025390625,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -503.451416015625,
"loss": 1.1033,
"margin_dpo/margin_mean": 158.06602478027344,
"margin_dpo/margin_std": 231.3526611328125,
"step": 305
},
{
"KL/chosen_KL_mean": -250.9173583984375,
"KL/mean": -331.4638366699219,
"KL/rejected_KL_mean": -412.01031494140625,
"KL/std": 187.9032745361328,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.0026251128874719143,
"fcm_dpo/delta": 0.028183521702885628,
"fcm_dpo/margin": 161.09298706054688,
"fcm_dpo/q_t": 0.4037204384803772,
"grad_norm": 26.20707893371582,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.3975362181663513,
"logits/rejected": -0.3718588948249817,
"logps/chosen": -320.60125732421875,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -497.16949462890625,
"loss": 1.0712,
"margin_dpo/margin_mean": 161.09298706054688,
"margin_dpo/margin_std": 201.70741271972656,
"step": 306
},
{
"KL/chosen_KL_mean": -238.816650390625,
"KL/mean": -321.26611328125,
"KL/rejected_KL_mean": -403.7156066894531,
"KL/std": 177.045166015625,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.00262629147619009,
"fcm_dpo/delta": 0.01766267418861389,
"fcm_dpo/margin": 164.89895629882812,
"fcm_dpo/q_t": 0.4013682007789612,
"grad_norm": 25.059446334838867,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.3578367233276367,
"logits/rejected": -0.33675235509872437,
"logps/chosen": -308.9820556640625,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -490.68792724609375,
"loss": 1.074,
"margin_dpo/margin_mean": 164.89895629882812,
"margin_dpo/margin_std": 216.65493774414062,
"step": 307
},
{
"KL/chosen_KL_mean": -241.0372314453125,
"KL/mean": -333.34759521484375,
"KL/rejected_KL_mean": -425.657958984375,
"KL/std": 201.32351684570312,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.002621263265609741,
"fcm_dpo/delta": -0.03565025329589844,
"fcm_dpo/margin": 184.62071228027344,
"fcm_dpo/q_t": 0.38778913021087646,
"grad_norm": 33.72212600708008,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.3401590585708618,
"logits/rejected": -0.3254951238632202,
"logps/chosen": -296.2822265625,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -505.03021240234375,
"loss": 1.02,
"margin_dpo/margin_mean": 184.62069702148438,
"margin_dpo/margin_std": 192.87948608398438,
"step": 308
},
{
"KL/chosen_KL_mean": -237.29989624023438,
"KL/mean": -329.2324523925781,
"KL/rejected_KL_mean": -421.16497802734375,
"KL/std": 205.91294860839844,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.0026156248059123755,
"fcm_dpo/delta": -0.032885223627090454,
"fcm_dpo/margin": 183.86509704589844,
"fcm_dpo/q_t": 0.38867393136024475,
"grad_norm": 55.76332092285156,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.3209107220172882,
"logits/rejected": -0.31004735827445984,
"logps/chosen": -286.25897216796875,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -503.5057067871094,
"loss": 1.0419,
"margin_dpo/margin_mean": 183.8651123046875,
"margin_dpo/margin_std": 219.06765747070312,
"step": 309
},
{
"KL/chosen_KL_mean": -272.84228515625,
"KL/mean": -357.896484375,
"KL/rejected_KL_mean": -442.95068359375,
"KL/std": 177.7974853515625,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.0026099509559571743,
"fcm_dpo/delta": 0.005964267998933792,
"fcm_dpo/margin": 170.10842895507812,
"fcm_dpo/q_t": 0.39623942971229553,
"grad_norm": 28.89272117614746,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.3595883846282959,
"logits/rejected": -0.34507110714912415,
"logps/chosen": -335.58404541015625,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -522.8809814453125,
"loss": 1.041,
"margin_dpo/margin_mean": 170.10842895507812,
"margin_dpo/margin_std": 173.22080993652344,
"step": 310
},
{
"KL/chosen_KL_mean": -283.4051513671875,
"KL/mean": -380.39300537109375,
"KL/rejected_KL_mean": -477.3808898925781,
"KL/std": 230.86802673339844,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.002579387277364731,
"fcm_dpo/delta": -0.053004972636699677,
"fcm_dpo/margin": 193.97573852539062,
"fcm_dpo/q_t": 0.3883788287639618,
"grad_norm": 25.943056106567383,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.3639160990715027,
"logits/rejected": -0.3407902717590332,
"logps/chosen": -336.43310546875,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -554.819091796875,
"loss": 1.0342,
"margin_dpo/margin_mean": 193.97573852539062,
"margin_dpo/margin_std": 241.01882934570312,
"step": 311
},
{
"KL/chosen_KL_mean": -259.5381164550781,
"KL/mean": -349.5057678222656,
"KL/rejected_KL_mean": -439.4734191894531,
"KL/std": 199.23733520507812,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.002578413113951683,
"fcm_dpo/delta": -0.014939859509468079,
"fcm_dpo/margin": 179.93531799316406,
"fcm_dpo/q_t": 0.3965616822242737,
"grad_norm": 24.46059799194336,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.2949761152267456,
"logits/rejected": -0.28196972608566284,
"logps/chosen": -308.9303283691406,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -515.2662353515625,
"loss": 1.0557,
"margin_dpo/margin_mean": 179.935302734375,
"margin_dpo/margin_std": 232.431640625,
"step": 312
},
{
"KL/chosen_KL_mean": -237.4639434814453,
"KL/mean": -327.5711669921875,
"KL/rejected_KL_mean": -417.6783752441406,
"KL/std": 207.86187744140625,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.0025785909965634346,
"fcm_dpo/delta": -0.016460783779621124,
"fcm_dpo/margin": 180.21444702148438,
"fcm_dpo/q_t": 0.39441442489624023,
"grad_norm": 25.501195907592773,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.3515121340751648,
"logits/rejected": -0.346757173538208,
"logps/chosen": -287.61669921875,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -504.0845947265625,
"loss": 1.0561,
"margin_dpo/margin_mean": 180.2144317626953,
"margin_dpo/margin_std": 224.6845703125,
"step": 313
},
{
"KL/chosen_KL_mean": -255.17724609375,
"KL/mean": -338.23541259765625,
"KL/rejected_KL_mean": -421.29351806640625,
"KL/std": 197.79852294921875,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.002554558450356126,
"fcm_dpo/delta": 0.026423994451761246,
"fcm_dpo/margin": 166.11630249023438,
"fcm_dpo/q_t": 0.40657833218574524,
"grad_norm": 22.802820205688477,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.40561333298683167,
"logits/rejected": -0.4046463966369629,
"logps/chosen": -312.4148254394531,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -518.8900756835938,
"loss": 1.1018,
"margin_dpo/margin_mean": 166.11630249023438,
"margin_dpo/margin_std": 251.25624084472656,
"step": 314
},
{
"KL/chosen_KL_mean": -228.4068603515625,
"KL/mean": -305.869873046875,
"KL/rejected_KL_mean": -383.33282470703125,
"KL/std": 177.62628173828125,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.0025859931483864784,
"fcm_dpo/delta": 0.05145182088017464,
"fcm_dpo/margin": 154.92596435546875,
"fcm_dpo/q_t": 0.40844735503196716,
"grad_norm": 25.262500762939453,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.3275286555290222,
"logits/rejected": -0.32907772064208984,
"logps/chosen": -277.4764404296875,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -469.01373291015625,
"loss": 1.0856,
"margin_dpo/margin_mean": 154.92596435546875,
"margin_dpo/margin_std": 197.055908203125,
"step": 315
},
{
"KL/chosen_KL_mean": -234.32891845703125,
"KL/mean": -344.46405029296875,
"KL/rejected_KL_mean": -454.59918212890625,
"KL/std": 213.09478759765625,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.0025378912687301636,
"fcm_dpo/delta": -0.116811104118824,
"fcm_dpo/margin": 220.27024841308594,
"fcm_dpo/q_t": 0.37471282482147217,
"grad_norm": 28.47589111328125,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.3635621666908264,
"logits/rejected": -0.3676885962486267,
"logps/chosen": -288.58966064453125,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -555.880615234375,
"loss": 0.9835,
"margin_dpo/margin_mean": 220.270263671875,
"margin_dpo/margin_std": 234.39602661132812,
"step": 316
},
{
"KL/chosen_KL_mean": -246.70562744140625,
"KL/mean": -367.3144836425781,
"KL/rejected_KL_mean": -487.92333984375,
"KL/std": 197.05282592773438,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.002488123718649149,
"fcm_dpo/delta": -0.15946120023727417,
"fcm_dpo/margin": 241.21768188476562,
"fcm_dpo/q_t": 0.3624528646469116,
"grad_norm": 30.520000457763672,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.3962244987487793,
"logits/rejected": -0.3859291672706604,
"logps/chosen": -302.79986572265625,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -588.6224365234375,
"loss": 0.9439,
"margin_dpo/margin_mean": 241.21768188476562,
"margin_dpo/margin_std": 217.16314697265625,
"step": 317
},
{
"KL/chosen_KL_mean": -272.97113037109375,
"KL/mean": -367.22979736328125,
"KL/rejected_KL_mean": -461.48846435546875,
"KL/std": 211.57278442382812,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.002460531424731016,
"fcm_dpo/delta": -0.01471693441271782,
"fcm_dpo/margin": 188.51731872558594,
"fcm_dpo/q_t": 0.3947198987007141,
"grad_norm": 28.010128021240234,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.41236060857772827,
"logits/rejected": -0.3805282711982727,
"logps/chosen": -337.6168212890625,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -544.252685546875,
"loss": 1.0477,
"margin_dpo/margin_mean": 188.51731872558594,
"margin_dpo/margin_std": 228.11260986328125,
"step": 318
},
{
"KL/chosen_KL_mean": -256.7373046875,
"KL/mean": -366.04266357421875,
"KL/rejected_KL_mean": -475.34796142578125,
"KL/std": 226.67886352539062,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0024200198240578175,
"fcm_dpo/delta": -0.08345872163772583,
"fcm_dpo/margin": 218.61068725585938,
"fcm_dpo/q_t": 0.3794768452644348,
"grad_norm": 22.835662841796875,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.38075074553489685,
"logits/rejected": -0.39493638277053833,
"logps/chosen": -306.1210632324219,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -589.2545166015625,
"loss": 1.0039,
"margin_dpo/margin_mean": 218.61068725585938,
"margin_dpo/margin_std": 243.4561767578125,
"step": 319
},
{
"KL/chosen_KL_mean": -276.7099609375,
"KL/mean": -395.4337158203125,
"KL/rejected_KL_mean": -514.157470703125,
"KL/std": 241.7675323486328,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.0023585117887705564,
"fcm_dpo/delta": -0.11735754460096359,
"fcm_dpo/margin": 237.447509765625,
"fcm_dpo/q_t": 0.3738830089569092,
"grad_norm": 30.08669662475586,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.37936246395111084,
"logits/rejected": -0.3851373493671417,
"logps/chosen": -336.21484375,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -611.8246459960938,
"loss": 0.985,
"margin_dpo/margin_mean": 237.447509765625,
"margin_dpo/margin_std": 257.3951416015625,
"step": 320
},
{
"KL/chosen_KL_mean": -339.2561950683594,
"KL/mean": -442.3502502441406,
"KL/rejected_KL_mean": -545.4443359375,
"KL/std": 248.72198486328125,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.0023283420596271753,
"fcm_dpo/delta": -0.03253670781850815,
"fcm_dpo/margin": 206.18814086914062,
"fcm_dpo/q_t": 0.3924645781517029,
"grad_norm": 26.111909866333008,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.3646908402442932,
"logits/rejected": -0.3575018644332886,
"logps/chosen": -400.80487060546875,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -637.0853271484375,
"loss": 1.05,
"margin_dpo/margin_mean": 206.18814086914062,
"margin_dpo/margin_std": 260.78741455078125,
"step": 321
},
{
"KL/chosen_KL_mean": -301.8092346191406,
"KL/mean": -417.0679931640625,
"KL/rejected_KL_mean": -532.32666015625,
"KL/std": 228.69329833984375,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.0023004289250820875,
"fcm_dpo/delta": -0.08538933843374252,
"fcm_dpo/margin": 230.51748657226562,
"fcm_dpo/q_t": 0.37849855422973633,
"grad_norm": 25.973785400390625,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.36909303069114685,
"logits/rejected": -0.3671499490737915,
"logps/chosen": -359.0992736816406,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -628.07666015625,
"loss": 0.9968,
"margin_dpo/margin_mean": 230.51748657226562,
"margin_dpo/margin_std": 240.91160583496094,
"step": 322
},
{
"KL/chosen_KL_mean": -326.64044189453125,
"KL/mean": -419.115966796875,
"KL/rejected_KL_mean": -511.5914306640625,
"KL/std": 224.9309539794922,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.002301469910889864,
"fcm_dpo/delta": 0.02534468285739422,
"fcm_dpo/margin": 184.95101928710938,
"fcm_dpo/q_t": 0.40258121490478516,
"grad_norm": 52.07743453979492,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.3876006603240967,
"logits/rejected": -0.3855016827583313,
"logps/chosen": -377.8744201660156,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -586.6533813476562,
"loss": 1.0811,
"margin_dpo/margin_mean": 184.95101928710938,
"margin_dpo/margin_std": 240.82904052734375,
"step": 323
},
{
"KL/chosen_KL_mean": -370.4443054199219,
"KL/mean": -462.32537841796875,
"KL/rejected_KL_mean": -554.2064208984375,
"KL/std": 239.63571166992188,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.00233688042499125,
"fcm_dpo/delta": 0.019912488758563995,
"fcm_dpo/margin": 183.76214599609375,
"fcm_dpo/q_t": 0.40560734272003174,
"grad_norm": 59.34591293334961,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.388489305973053,
"logits/rejected": -0.37319934368133545,
"logps/chosen": -435.5794677734375,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -640.6839599609375,
"loss": 1.1079,
"margin_dpo/margin_mean": 183.7621307373047,
"margin_dpo/margin_std": 277.62188720703125,
"step": 324
},
{
"KL/chosen_KL_mean": -291.468994140625,
"KL/mean": -369.7003173828125,
"KL/rejected_KL_mean": -447.9316711425781,
"KL/std": 212.40463256835938,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.0023523285053670406,
"fcm_dpo/delta": 0.08494192361831665,
"fcm_dpo/margin": 156.462646484375,
"fcm_dpo/q_t": 0.4155137538909912,
"grad_norm": 34.735572814941406,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.45538169145584106,
"logits/rejected": -0.43598616123199463,
"logps/chosen": -347.6846008300781,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -518.017578125,
"loss": 1.1154,
"margin_dpo/margin_mean": 156.462646484375,
"margin_dpo/margin_std": 214.5522003173828,
"step": 325
},
{
"KL/chosen_KL_mean": -323.7759094238281,
"KL/mean": -402.790283203125,
"KL/rejected_KL_mean": -481.8047180175781,
"KL/std": 199.88583374023438,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0023813103325664997,
"fcm_dpo/delta": 0.0766344889998436,
"fcm_dpo/margin": 158.02880859375,
"fcm_dpo/q_t": 0.41171562671661377,
"grad_norm": 37.10205841064453,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.44069403409957886,
"logits/rejected": -0.41960227489471436,
"logps/chosen": -396.5008544921875,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -561.6514892578125,
"loss": 1.0953,
"margin_dpo/margin_mean": 158.02882385253906,
"margin_dpo/margin_std": 187.57601928710938,
"step": 326
},
{
"KL/chosen_KL_mean": -274.60150146484375,
"KL/mean": -390.11627197265625,
"KL/rejected_KL_mean": -505.631103515625,
"KL/std": 213.35443115234375,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.002351765986531973,
"fcm_dpo/delta": -0.09938237071037292,
"fcm_dpo/margin": 231.02955627441406,
"fcm_dpo/q_t": 0.37418001890182495,
"grad_norm": 40.187255859375,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.41924217343330383,
"logits/rejected": -0.4057329595088959,
"logps/chosen": -343.7359313964844,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -617.5648193359375,
"loss": 0.9821,
"margin_dpo/margin_mean": 231.02955627441406,
"margin_dpo/margin_std": 227.44580078125,
"step": 327
},
{
"KL/chosen_KL_mean": -281.99481201171875,
"KL/mean": -383.1610412597656,
"KL/rejected_KL_mean": -484.3272705078125,
"KL/std": 223.35235595703125,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0023423531092703342,
"fcm_dpo/delta": -0.025119595229625702,
"fcm_dpo/margin": 202.3324432373047,
"fcm_dpo/q_t": 0.3928408622741699,
"grad_norm": 27.627058029174805,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.421650767326355,
"logits/rejected": -0.4237726926803589,
"logps/chosen": -341.6820068359375,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -575.1822509765625,
"loss": 1.0416,
"margin_dpo/margin_mean": 202.3324432373047,
"margin_dpo/margin_std": 245.05563354492188,
"step": 328
},
{
"KL/chosen_KL_mean": -304.212646484375,
"KL/mean": -413.9533996582031,
"KL/rejected_KL_mean": -523.694091796875,
"KL/std": 236.1326904296875,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.002305768197402358,
"fcm_dpo/delta": -0.05977274850010872,
"fcm_dpo/margin": 219.48150634765625,
"fcm_dpo/q_t": 0.3840027451515198,
"grad_norm": 29.397104263305664,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.3993944525718689,
"logits/rejected": -0.38911157846450806,
"logps/chosen": -369.4588317871094,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -624.391845703125,
"loss": 1.0218,
"margin_dpo/margin_mean": 219.48150634765625,
"margin_dpo/margin_std": 246.65150451660156,
"step": 329
},
{
"KL/chosen_KL_mean": -262.68377685546875,
"KL/mean": -371.68048095703125,
"KL/rejected_KL_mean": -480.67718505859375,
"KL/std": 223.04965209960938,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.0022905156947672367,
"fcm_dpo/delta": -0.05193692445755005,
"fcm_dpo/margin": 217.99343872070312,
"fcm_dpo/q_t": 0.3855849504470825,
"grad_norm": 28.491689682006836,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.3724977970123291,
"logits/rejected": -0.37324658036231995,
"logps/chosen": -309.6820983886719,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -567.5540771484375,
"loss": 1.0158,
"margin_dpo/margin_mean": 217.99343872070312,
"margin_dpo/margin_std": 237.47122192382812,
"step": 330
},
{
"KL/chosen_KL_mean": -280.93597412109375,
"KL/mean": -389.1291809082031,
"KL/rejected_KL_mean": -497.3223571777344,
"KL/std": 182.53598022460938,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.0022698137909173965,
"fcm_dpo/delta": -0.04330473765730858,
"fcm_dpo/margin": 216.3863525390625,
"fcm_dpo/q_t": 0.38430529832839966,
"grad_norm": 37.197208404541016,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.4366343021392822,
"logits/rejected": -0.43291670083999634,
"logps/chosen": -331.4602355957031,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -586.3377685546875,
"loss": 0.9921,
"margin_dpo/margin_mean": 216.3863525390625,
"margin_dpo/margin_std": 179.05184936523438,
"step": 331
},
{
"KL/chosen_KL_mean": -282.11297607421875,
"KL/mean": -359.55120849609375,
"KL/rejected_KL_mean": -436.9894714355469,
"KL/std": 190.00502014160156,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0023019383661448956,
"fcm_dpo/delta": 0.09676185250282288,
"fcm_dpo/margin": 154.87649536132812,
"fcm_dpo/q_t": 0.41818666458129883,
"grad_norm": 24.03437614440918,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.41719603538513184,
"logits/rejected": -0.3998468518257141,
"logps/chosen": -331.29327392578125,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -513.474609375,
"loss": 1.1126,
"margin_dpo/margin_mean": 154.87652587890625,
"margin_dpo/margin_std": 204.28192138671875,
"step": 332
},
{
"KL/chosen_KL_mean": -301.60906982421875,
"KL/mean": -403.718505859375,
"KL/rejected_KL_mean": -505.8279113769531,
"KL/std": 225.57460021972656,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.002289321506395936,
"fcm_dpo/delta": -0.0192737840116024,
"fcm_dpo/margin": 204.2188262939453,
"fcm_dpo/q_t": 0.39486679434776306,
"grad_norm": 23.60999870300293,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.4344450831413269,
"logits/rejected": -0.430309921503067,
"logps/chosen": -365.36480712890625,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -600.8720092773438,
"loss": 1.0626,
"margin_dpo/margin_mean": 204.21881103515625,
"margin_dpo/margin_std": 267.87591552734375,
"step": 333
},
{
"KL/chosen_KL_mean": -290.12060546875,
"KL/mean": -382.6822509765625,
"KL/rejected_KL_mean": -475.24395751953125,
"KL/std": 240.4530029296875,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.002307912800461054,
"fcm_dpo/delta": 0.02377365157008171,
"fcm_dpo/margin": 185.12339782714844,
"fcm_dpo/q_t": 0.4034510850906372,
"grad_norm": 25.177146911621094,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.4144849479198456,
"logits/rejected": -0.4086567163467407,
"logps/chosen": -357.100341796875,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -570.5609130859375,
"loss": 1.0911,
"margin_dpo/margin_mean": 185.12338256835938,
"margin_dpo/margin_std": 263.03802490234375,
"step": 334
},
{
"KL/chosen_KL_mean": -316.4975891113281,
"KL/mean": -389.90167236328125,
"KL/rejected_KL_mean": -463.3057556152344,
"KL/std": 223.39114379882812,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.002362326718866825,
"fcm_dpo/delta": 0.10562027990818024,
"fcm_dpo/margin": 146.80816650390625,
"fcm_dpo/q_t": 0.4218912720680237,
"grad_norm": 29.343273162841797,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.47399646043777466,
"logits/rejected": -0.45253986120224,
"logps/chosen": -379.0400695800781,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -550.9234619140625,
"loss": 1.1476,
"margin_dpo/margin_mean": 146.80816650390625,
"margin_dpo/margin_std": 240.48916625976562,
"step": 335
},
{
"KL/chosen_KL_mean": -313.12445068359375,
"KL/mean": -429.3009948730469,
"KL/rejected_KL_mean": -545.4775390625,
"KL/std": 254.45855712890625,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0023357835598289967,
"fcm_dpo/delta": -0.0979611948132515,
"fcm_dpo/margin": 232.35311889648438,
"fcm_dpo/q_t": 0.3800773024559021,
"grad_norm": 26.39600372314453,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.4094962477684021,
"logits/rejected": -0.40867048501968384,
"logps/chosen": -367.65557861328125,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -649.8818359375,
"loss": 1.0219,
"margin_dpo/margin_mean": 232.35311889648438,
"margin_dpo/margin_std": 291.4840393066406,
"step": 336
},
{
"KL/chosen_KL_mean": -287.0484619140625,
"KL/mean": -399.60888671875,
"KL/rejected_KL_mean": -512.1693115234375,
"KL/std": 210.44012451171875,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.00228647468611598,
"fcm_dpo/delta": -0.06854301691055298,
"fcm_dpo/margin": 225.120849609375,
"fcm_dpo/q_t": 0.3825306296348572,
"grad_norm": 34.630104064941406,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.4810912609100342,
"logits/rejected": -0.4703620672225952,
"logps/chosen": -352.17718505859375,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -613.8963623046875,
"loss": 1.0091,
"margin_dpo/margin_mean": 225.12086486816406,
"margin_dpo/margin_std": 246.9684600830078,
"step": 337
},
{
"KL/chosen_KL_mean": -268.0345153808594,
"KL/mean": -379.52264404296875,
"KL/rejected_KL_mean": -491.0107421875,
"KL/std": 213.11172485351562,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0022620288655161858,
"fcm_dpo/delta": -0.05736400932073593,
"fcm_dpo/margin": 222.97622680664062,
"fcm_dpo/q_t": 0.3833646774291992,
"grad_norm": 30.86869239807129,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.488203763961792,
"logits/rejected": -0.4801350235939026,
"logps/chosen": -326.45721435546875,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -580.0792846679688,
"loss": 1.0026,
"margin_dpo/margin_mean": 222.97622680664062,
"margin_dpo/margin_std": 222.72213745117188,
"step": 338
},
{
"KL/chosen_KL_mean": -279.5220031738281,
"KL/mean": -387.8486022949219,
"KL/rejected_KL_mean": -496.1752014160156,
"KL/std": 218.14901733398438,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.002262189518660307,
"fcm_dpo/delta": -0.043202970176935196,
"fcm_dpo/margin": 216.65322875976562,
"fcm_dpo/q_t": 0.3870697021484375,
"grad_norm": 34.2067756652832,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.5117071270942688,
"logits/rejected": -0.4946392774581909,
"logps/chosen": -339.517333984375,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -600.0861206054688,
"loss": 1.0183,
"margin_dpo/margin_mean": 216.65322875976562,
"margin_dpo/margin_std": 218.69485473632812,
"step": 339
},
{
"KL/chosen_KL_mean": -300.64361572265625,
"KL/mean": -397.70538330078125,
"KL/rejected_KL_mean": -494.76715087890625,
"KL/std": 210.3126220703125,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0022354573011398315,
"fcm_dpo/delta": 0.016527773812413216,
"fcm_dpo/margin": 194.12351989746094,
"fcm_dpo/q_t": 0.4011934995651245,
"grad_norm": 35.80244827270508,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.39741045236587524,
"logits/rejected": -0.37222498655319214,
"logps/chosen": -353.473876953125,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -567.8743896484375,
"loss": 1.0726,
"margin_dpo/margin_mean": 194.12351989746094,
"margin_dpo/margin_std": 251.18772888183594,
"step": 340
},
{
"KL/chosen_KL_mean": -299.45147705078125,
"KL/mean": -409.642822265625,
"KL/rejected_KL_mean": -519.8341064453125,
"KL/std": 234.6087646484375,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.002222803421318531,
"fcm_dpo/delta": -0.04243752360343933,
"fcm_dpo/margin": 220.3826446533203,
"fcm_dpo/q_t": 0.3885568678379059,
"grad_norm": 25.23012351989746,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.4419187605381012,
"logits/rejected": -0.44217973947525024,
"logps/chosen": -347.35125732421875,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -621.6439819335938,
"loss": 1.0256,
"margin_dpo/margin_mean": 220.38262939453125,
"margin_dpo/margin_std": 244.93939208984375,
"step": 341
},
{
"KL/chosen_KL_mean": -293.6312255859375,
"KL/mean": -392.4619140625,
"KL/rejected_KL_mean": -491.2926025390625,
"KL/std": 217.22531127929688,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.002238193526864052,
"fcm_dpo/delta": 0.007614880800247192,
"fcm_dpo/margin": 197.66143798828125,
"fcm_dpo/q_t": 0.39952483773231506,
"grad_norm": 28.198055267333984,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.4391222596168518,
"logits/rejected": -0.41000527143478394,
"logps/chosen": -365.62786865234375,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -583.8822021484375,
"loss": 1.0648,
"margin_dpo/margin_mean": 197.66143798828125,
"margin_dpo/margin_std": 251.34584045410156,
"step": 342
},
{
"KL/chosen_KL_mean": -283.8646240234375,
"KL/mean": -400.5185241699219,
"KL/rejected_KL_mean": -517.1724243164062,
"KL/std": 220.9855499267578,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.002206906210631132,
"fcm_dpo/delta": -0.06844501197338104,
"fcm_dpo/margin": 233.30776977539062,
"fcm_dpo/q_t": 0.3789847493171692,
"grad_norm": 29.322237014770508,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.46948331594467163,
"logits/rejected": -0.4704735279083252,
"logps/chosen": -338.270263671875,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -628.2138671875,
"loss": 0.9819,
"margin_dpo/margin_mean": 233.3077850341797,
"margin_dpo/margin_std": 195.7198944091797,
"step": 343
},
{
"KL/chosen_KL_mean": -286.81103515625,
"KL/mean": -388.08660888671875,
"KL/rejected_KL_mean": -489.36212158203125,
"KL/std": 237.59225463867188,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.002221263013780117,
"fcm_dpo/delta": -0.0018584132194519043,
"fcm_dpo/margin": 202.5511474609375,
"fcm_dpo/q_t": 0.3995116651058197,
"grad_norm": 28.450302124023438,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.4501603841781616,
"logits/rejected": -0.4517921507358551,
"logps/chosen": -340.77569580078125,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -579.9854736328125,
"loss": 1.0596,
"margin_dpo/margin_mean": 202.5511474609375,
"margin_dpo/margin_std": 244.07476806640625,
"step": 344
},
{
"KL/chosen_KL_mean": -332.5111083984375,
"KL/mean": -445.0814208984375,
"KL/rejected_KL_mean": -557.6517333984375,
"KL/std": 230.50149536132812,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.0021899822168052197,
"fcm_dpo/delta": -0.04530047997832298,
"fcm_dpo/margin": 225.14065551757812,
"fcm_dpo/q_t": 0.3868619203567505,
"grad_norm": 26.73749351501465,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.4524056017398834,
"logits/rejected": -0.4489898085594177,
"logps/chosen": -394.19677734375,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -657.1421508789062,
"loss": 1.0222,
"margin_dpo/margin_mean": 225.1406707763672,
"margin_dpo/margin_std": 250.29025268554688,
"step": 345
},
{
"KL/chosen_KL_mean": -332.1961669921875,
"KL/mean": -438.26373291015625,
"KL/rejected_KL_mean": -544.331298828125,
"KL/std": 220.63192749023438,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.002174857072532177,
"fcm_dpo/delta": -0.011919819749891758,
"fcm_dpo/margin": 212.1351318359375,
"fcm_dpo/q_t": 0.39296603202819824,
"grad_norm": 26.011234283447266,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.4090457260608673,
"logits/rejected": -0.39724189043045044,
"logps/chosen": -387.45245361328125,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -621.74658203125,
"loss": 1.043,
"margin_dpo/margin_mean": 212.13511657714844,
"margin_dpo/margin_std": 244.46681213378906,
"step": 346
},
{
"KL/chosen_KL_mean": -324.60968017578125,
"KL/mean": -432.421142578125,
"KL/rejected_KL_mean": -540.2326049804688,
"KL/std": 235.4697265625,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.00216767773963511,
"fcm_dpo/delta": -0.018262427300214767,
"fcm_dpo/margin": 215.62295532226562,
"fcm_dpo/q_t": 0.39333486557006836,
"grad_norm": 24.214601516723633,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.41619834303855896,
"logits/rejected": -0.4093385934829712,
"logps/chosen": -382.1759033203125,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -632.5877075195312,
"loss": 1.0451,
"margin_dpo/margin_mean": 215.62295532226562,
"margin_dpo/margin_std": 258.7292785644531,
"step": 347
},
{
"KL/chosen_KL_mean": -298.324951171875,
"KL/mean": -387.9532470703125,
"KL/rejected_KL_mean": -477.5815734863281,
"KL/std": 203.1543426513672,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.002172997919842601,
"fcm_dpo/delta": 0.06251075863838196,
"fcm_dpo/margin": 179.25662231445312,
"fcm_dpo/q_t": 0.4109954237937927,
"grad_norm": 21.876827239990234,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.45160365104675293,
"logits/rejected": -0.4534376859664917,
"logps/chosen": -354.64263916015625,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -566.719970703125,
"loss": 1.097,
"margin_dpo/margin_mean": 179.25662231445312,
"margin_dpo/margin_std": 231.77615356445312,
"step": 348
},
{
"KL/chosen_KL_mean": -278.4696960449219,
"KL/mean": -378.2044372558594,
"KL/rejected_KL_mean": -477.939208984375,
"KL/std": 224.55136108398438,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.0022066906094551086,
"fcm_dpo/delta": 0.009832058101892471,
"fcm_dpo/margin": 199.469482421875,
"fcm_dpo/q_t": 0.4018779993057251,
"grad_norm": 22.135345458984375,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.407615602016449,
"logits/rejected": -0.40762487053871155,
"logps/chosen": -336.4952087402344,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -575.4443359375,
"loss": 1.0817,
"margin_dpo/margin_mean": 199.469482421875,
"margin_dpo/margin_std": 278.9986877441406,
"step": 349
},
{
"KL/chosen_KL_mean": -294.96185302734375,
"KL/mean": -393.88372802734375,
"KL/rejected_KL_mean": -492.8055725097656,
"KL/std": 212.79945373535156,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.002194004599004984,
"fcm_dpo/delta": 0.016355086117982864,
"fcm_dpo/margin": 197.8437042236328,
"fcm_dpo/q_t": 0.39984023571014404,
"grad_norm": 30.247133255004883,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.44759875535964966,
"logits/rejected": -0.44960999488830566,
"logps/chosen": -359.2923583984375,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -582.67724609375,
"loss": 1.0783,
"margin_dpo/margin_mean": 197.84371948242188,
"margin_dpo/margin_std": 263.1554870605469,
"step": 350
},
{
"KL/chosen_KL_mean": -276.9792175292969,
"KL/mean": -397.68896484375,
"KL/rejected_KL_mean": -518.3987426757812,
"KL/std": 254.1345672607422,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0021747946739196777,
"fcm_dpo/delta": -0.07944449782371521,
"fcm_dpo/margin": 241.41952514648438,
"fcm_dpo/q_t": 0.3832271695137024,
"grad_norm": 25.02871322631836,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.45677679777145386,
"logits/rejected": -0.4540591537952423,
"logps/chosen": -337.6513366699219,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -619.9641723632812,
"loss": 1.0186,
"margin_dpo/margin_mean": 241.41952514648438,
"margin_dpo/margin_std": 290.705810546875,
"step": 351
},
{
"KL/chosen_KL_mean": -333.67742919921875,
"KL/mean": -409.6393127441406,
"KL/rejected_KL_mean": -485.60113525390625,
"KL/std": 232.55503845214844,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.0022003580816090107,
"fcm_dpo/delta": 0.11985920369625092,
"fcm_dpo/margin": 151.92373657226562,
"fcm_dpo/q_t": 0.4246724247932434,
"grad_norm": 32.05812454223633,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.47153419256210327,
"logits/rejected": -0.44236433506011963,
"logps/chosen": -404.62091064453125,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -562.2431640625,
"loss": 1.1565,
"margin_dpo/margin_mean": 151.92373657226562,
"margin_dpo/margin_std": 255.46420288085938,
"step": 352
},
{
"KL/chosen_KL_mean": -318.5333557128906,
"KL/mean": -418.0197448730469,
"KL/rejected_KL_mean": -517.506103515625,
"KL/std": 240.97393798828125,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0022136140614748,
"fcm_dpo/delta": 0.009388644248247147,
"fcm_dpo/margin": 198.9727783203125,
"fcm_dpo/q_t": 0.40121889114379883,
"grad_norm": 37.71311569213867,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.4609450697898865,
"logits/rejected": -0.4458503723144531,
"logps/chosen": -376.9287109375,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -597.8416748046875,
"loss": 1.0762,
"margin_dpo/margin_mean": 198.9727783203125,
"margin_dpo/margin_std": 265.66168212890625,
"step": 353
},
{
"KL/chosen_KL_mean": -269.13421630859375,
"KL/mean": -382.9037170410156,
"KL/rejected_KL_mean": -496.6732177734375,
"KL/std": 222.33949279785156,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.0022065872326493263,
"fcm_dpo/delta": -0.05492671579122543,
"fcm_dpo/margin": 227.53900146484375,
"fcm_dpo/q_t": 0.3830799460411072,
"grad_norm": 24.187728881835938,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.47964227199554443,
"logits/rejected": -0.46824321150779724,
"logps/chosen": -328.93719482421875,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -585.4307250976562,
"loss": 0.9969,
"margin_dpo/margin_mean": 227.53900146484375,
"margin_dpo/margin_std": 212.8988037109375,
"step": 354
},
{
"KL/chosen_KL_mean": -279.54412841796875,
"KL/mean": -391.5539855957031,
"KL/rejected_KL_mean": -503.5638427734375,
"KL/std": 231.0259552001953,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.0021842471323907375,
"fcm_dpo/delta": -0.04154280200600624,
"fcm_dpo/margin": 224.0196990966797,
"fcm_dpo/q_t": 0.3863520622253418,
"grad_norm": 32.41206741333008,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.49721992015838623,
"logits/rejected": -0.4830342233181,
"logps/chosen": -333.6726379394531,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -585.9699096679688,
"loss": 1.0133,
"margin_dpo/margin_mean": 224.0196990966797,
"margin_dpo/margin_std": 226.89224243164062,
"step": 355
},
{
"KL/chosen_KL_mean": -339.30712890625,
"KL/mean": -404.0550537109375,
"KL/rejected_KL_mean": -468.80303955078125,
"KL/std": 238.53335571289062,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.0021884175948798656,
"fcm_dpo/delta": 0.017231887206435204,
"fcm_dpo/margin": 129.49594116210938,
"fcm_dpo/q_t": 0.4372093677520752,
"grad_norm": 44.52901840209961,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.4759572446346283,
"logits/rejected": -0.45413556694984436,
"logps/chosen": -403.9809265136719,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -544.7022705078125,
"loss": 1.225,
"margin_dpo/margin_mean": 129.49594116210938,
"margin_dpo/margin_std": 297.17681884765625,
"step": 356
},
{
"KL/chosen_KL_mean": -308.063720703125,
"KL/mean": -408.1578369140625,
"KL/rejected_KL_mean": -508.251953125,
"KL/std": 230.31048583984375,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.0021870676428079605,
"fcm_dpo/delta": 0.012423541396856308,
"fcm_dpo/margin": 200.188232421875,
"fcm_dpo/q_t": 0.3986167311668396,
"grad_norm": 29.87175941467285,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.45963430404663086,
"logits/rejected": -0.4457460045814514,
"logps/chosen": -360.7895202636719,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -595.0931396484375,
"loss": 1.0577,
"margin_dpo/margin_mean": 200.188232421875,
"margin_dpo/margin_std": 232.631103515625,
"step": 357
},
{
"KL/chosen_KL_mean": -274.03955078125,
"KL/mean": -369.8802795410156,
"KL/rejected_KL_mean": -465.72100830078125,
"KL/std": 225.74777221679688,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0022067471873015165,
"fcm_dpo/delta": 0.028202099725604057,
"fcm_dpo/margin": 191.68145751953125,
"fcm_dpo/q_t": 0.40436553955078125,
"grad_norm": 24.91404914855957,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.4887428879737854,
"logits/rejected": -0.4698476195335388,
"logps/chosen": -337.2449951171875,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -554.0943603515625,
"loss": 1.0753,
"margin_dpo/margin_mean": 191.68145751953125,
"margin_dpo/margin_std": 246.89959716796875,
"step": 358
},
{
"KL/chosen_KL_mean": -309.8017578125,
"KL/mean": -419.9251708984375,
"KL/rejected_KL_mean": -530.0485229492188,
"KL/std": 227.0589599609375,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0021956057753413916,
"fcm_dpo/delta": -0.03536238521337509,
"fcm_dpo/margin": 220.24673461914062,
"fcm_dpo/q_t": 0.3920891284942627,
"grad_norm": 33.248435974121094,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.42775458097457886,
"logits/rejected": -0.4127110540866852,
"logps/chosen": -366.1719970703125,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -612.2222900390625,
"loss": 1.0457,
"margin_dpo/margin_mean": 220.24673461914062,
"margin_dpo/margin_std": 276.7038269042969,
"step": 359
},
{
"KL/chosen_KL_mean": -297.59710693359375,
"KL/mean": -383.20404052734375,
"KL/rejected_KL_mean": -468.81103515625,
"KL/std": 191.41368103027344,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.002220964292064309,
"fcm_dpo/delta": 0.07238072901964188,
"fcm_dpo/margin": 171.2139129638672,
"fcm_dpo/q_t": 0.4112092852592468,
"grad_norm": 36.38239288330078,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.42573845386505127,
"logits/rejected": -0.41087961196899414,
"logps/chosen": -349.0574951171875,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -538.6499633789062,
"loss": 1.0867,
"margin_dpo/margin_mean": 171.21392822265625,
"margin_dpo/margin_std": 194.836669921875,
"step": 360
},
{
"KL/chosen_KL_mean": -317.767578125,
"KL/mean": -402.918701171875,
"KL/rejected_KL_mean": -488.06982421875,
"KL/std": 225.28887939453125,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.002254966413602233,
"fcm_dpo/delta": 0.06830620765686035,
"fcm_dpo/margin": 170.30221557617188,
"fcm_dpo/q_t": 0.4124668836593628,
"grad_norm": 31.59840965270996,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.4371631145477295,
"logits/rejected": -0.4323720335960388,
"logps/chosen": -371.6371154785156,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -578.839111328125,
"loss": 1.1114,
"margin_dpo/margin_mean": 170.30221557617188,
"margin_dpo/margin_std": 246.90036010742188,
"step": 361
},
{
"KL/chosen_KL_mean": -281.20916748046875,
"KL/mean": -412.2677001953125,
"KL/rejected_KL_mean": -543.3262939453125,
"KL/std": 234.77249145507812,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.0022046894300729036,
"fcm_dpo/delta": -0.13639265298843384,
"fcm_dpo/margin": 262.11712646484375,
"fcm_dpo/q_t": 0.3685632348060608,
"grad_norm": 30.286354064941406,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.4322547912597656,
"logits/rejected": -0.4399781823158264,
"logps/chosen": -339.84820556640625,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -648.908203125,
"loss": 0.9576,
"margin_dpo/margin_mean": 262.11712646484375,
"margin_dpo/margin_std": 252.21163940429688,
"step": 362
},
{
"KL/chosen_KL_mean": -252.32077026367188,
"KL/mean": -389.19268798828125,
"KL/rejected_KL_mean": -526.0645751953125,
"KL/std": 239.5186767578125,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.002155223861336708,
"fcm_dpo/delta": -0.1484936624765396,
"fcm_dpo/margin": 273.74383544921875,
"fcm_dpo/q_t": 0.3658607602119446,
"grad_norm": 25.104822158813477,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.40804392099380493,
"logits/rejected": -0.3971943259239197,
"logps/chosen": -296.879150390625,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -600.759521484375,
"loss": 0.9522,
"margin_dpo/margin_mean": 273.74383544921875,
"margin_dpo/margin_std": 261.37506103515625,
"step": 363
},
{
"KL/chosen_KL_mean": -273.26348876953125,
"KL/mean": -377.99298095703125,
"KL/rejected_KL_mean": -482.72247314453125,
"KL/std": 227.8748779296875,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.0021402244456112385,
"fcm_dpo/delta": 0.001525435596704483,
"fcm_dpo/margin": 209.45901489257812,
"fcm_dpo/q_t": 0.39739012718200684,
"grad_norm": 27.63004493713379,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.4067942500114441,
"logits/rejected": -0.4162289500236511,
"logps/chosen": -322.1580810546875,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -574.1182250976562,
"loss": 1.0601,
"margin_dpo/margin_mean": 209.45901489257812,
"margin_dpo/margin_std": 260.34423828125,
"step": 364
},
{
"KL/chosen_KL_mean": -270.8097229003906,
"KL/mean": -377.0350646972656,
"KL/rejected_KL_mean": -483.26043701171875,
"KL/std": 243.41281127929688,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0021314772311598063,
"fcm_dpo/delta": -0.002969269407913089,
"fcm_dpo/margin": 212.45069885253906,
"fcm_dpo/q_t": 0.39748814702033997,
"grad_norm": 25.45064926147461,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.409412145614624,
"logits/rejected": -0.41792500019073486,
"logps/chosen": -322.3024597167969,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -575.9620971679688,
"loss": 1.0483,
"margin_dpo/margin_mean": 212.45069885253906,
"margin_dpo/margin_std": 249.87734985351562,
"step": 365
},
{
"KL/chosen_KL_mean": -247.15567016601562,
"KL/mean": -350.6108093261719,
"KL/rejected_KL_mean": -454.06597900390625,
"KL/std": 221.3289794921875,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.002125158440321684,
"fcm_dpo/delta": 0.010381799191236496,
"fcm_dpo/margin": 206.91030883789062,
"fcm_dpo/q_t": 0.39979618787765503,
"grad_norm": 21.667766571044922,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.3939347565174103,
"logits/rejected": -0.4089328944683075,
"logps/chosen": -291.876220703125,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -537.3764038085938,
"loss": 1.0681,
"margin_dpo/margin_mean": 206.91029357910156,
"margin_dpo/margin_std": 262.265380859375,
"step": 366
},
{
"KL/chosen_KL_mean": -249.67767333984375,
"KL/mean": -338.1475830078125,
"KL/rejected_KL_mean": -426.61749267578125,
"KL/std": 202.29286193847656,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0021613952703773975,
"fcm_dpo/delta": 0.07011134922504425,
"fcm_dpo/margin": 176.93978881835938,
"fcm_dpo/q_t": 0.41084012389183044,
"grad_norm": 21.764284133911133,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.42215287685394287,
"logits/rejected": -0.40269413590431213,
"logps/chosen": -308.0830993652344,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -503.3688049316406,
"loss": 1.0935,
"margin_dpo/margin_mean": 176.93978881835938,
"margin_dpo/margin_std": 219.58963012695312,
"step": 367
},
{
"KL/chosen_KL_mean": -239.2371826171875,
"KL/mean": -372.4246520996094,
"KL/rejected_KL_mean": -505.61212158203125,
"KL/std": 228.0447998046875,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.0021273712627589703,
"fcm_dpo/delta": -0.12365108728408813,
"fcm_dpo/margin": 266.37493896484375,
"fcm_dpo/q_t": 0.3667055070400238,
"grad_norm": 39.34254455566406,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.43125176429748535,
"logits/rejected": -0.43804439902305603,
"logps/chosen": -283.689697265625,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -604.1673583984375,
"loss": 0.9439,
"margin_dpo/margin_mean": 266.37493896484375,
"margin_dpo/margin_std": 209.77322387695312,
"step": 368
},
{
"KL/chosen_KL_mean": -302.921875,
"KL/mean": -376.767578125,
"KL/rejected_KL_mean": -450.61334228515625,
"KL/std": 229.33493041992188,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.0021634360309690237,
"fcm_dpo/delta": 0.13406780362129211,
"fcm_dpo/margin": 147.69146728515625,
"fcm_dpo/q_t": 0.42671999335289,
"grad_norm": 37.03165817260742,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.41576188802719116,
"logits/rejected": -0.3948121666908264,
"logps/chosen": -374.3033447265625,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -541.9091796875,
"loss": 1.1684,
"margin_dpo/margin_mean": 147.69146728515625,
"margin_dpo/margin_std": 262.02093505859375,
"step": 369
},
{
"KL/chosen_KL_mean": -312.0323486328125,
"KL/mean": -386.12933349609375,
"KL/rejected_KL_mean": -460.2262878417969,
"KL/std": 229.80404663085938,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.002216800581663847,
"fcm_dpo/delta": 0.12519779801368713,
"fcm_dpo/margin": 148.19395446777344,
"fcm_dpo/q_t": 0.4249410033226013,
"grad_norm": 27.90708351135254,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.45437344908714294,
"logits/rejected": -0.44695669412612915,
"logps/chosen": -383.63983154296875,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -557.486083984375,
"loss": 1.1573,
"margin_dpo/margin_mean": 148.19395446777344,
"margin_dpo/margin_std": 248.8985595703125,
"step": 370
},
{
"KL/chosen_KL_mean": -301.58782958984375,
"KL/mean": -401.96405029296875,
"KL/rejected_KL_mean": -502.3403015136719,
"KL/std": 236.34710693359375,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.002231742488220334,
"fcm_dpo/delta": 0.0019375644624233246,
"fcm_dpo/margin": 200.75245666503906,
"fcm_dpo/q_t": 0.3994460999965668,
"grad_norm": 31.773405075073242,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.45085662603378296,
"logits/rejected": -0.4364727735519409,
"logps/chosen": -371.0023193359375,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -601.512451171875,
"loss": 1.074,
"margin_dpo/margin_mean": 200.75245666503906,
"margin_dpo/margin_std": 272.80889892578125,
"step": 371
},
{
"KL/chosen_KL_mean": -282.4768371582031,
"KL/mean": -405.1539306640625,
"KL/rejected_KL_mean": -527.8310546875,
"KL/std": 271.397216796875,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.0021908096969127655,
"fcm_dpo/delta": -0.0928315594792366,
"fcm_dpo/margin": 245.35423278808594,
"fcm_dpo/q_t": 0.38182735443115234,
"grad_norm": 26.00803565979004,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.4409677982330322,
"logits/rejected": -0.41506385803222656,
"logps/chosen": -344.2948303222656,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -606.3705444335938,
"loss": 1.0188,
"margin_dpo/margin_mean": 245.35423278808594,
"margin_dpo/margin_std": 307.55609130859375,
"step": 372
},
{
"KL/chosen_KL_mean": -310.77227783203125,
"KL/mean": -425.3634033203125,
"KL/rejected_KL_mean": -539.9545288085938,
"KL/std": 261.83868408203125,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.0021818915847688913,
"fcm_dpo/delta": -0.05314254388213158,
"fcm_dpo/margin": 229.18228149414062,
"fcm_dpo/q_t": 0.3849130868911743,
"grad_norm": 34.27317428588867,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.4386979639530182,
"logits/rejected": -0.41286277770996094,
"logps/chosen": -374.9893798828125,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -625.9141845703125,
"loss": 1.02,
"margin_dpo/margin_mean": 229.18228149414062,
"margin_dpo/margin_std": 245.1495361328125,
"step": 373
},
{
"KL/chosen_KL_mean": -279.8177490234375,
"KL/mean": -383.7864074707031,
"KL/rejected_KL_mean": -487.7550354003906,
"KL/std": 281.73785400390625,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.0021521011367440224,
"fcm_dpo/delta": 0.002089455723762512,
"fcm_dpo/margin": 207.93724060058594,
"fcm_dpo/q_t": 0.40315020084381104,
"grad_norm": 25.19005012512207,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.5260878801345825,
"logits/rejected": -0.5048750638961792,
"logps/chosen": -350.46795654296875,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -581.3952026367188,
"loss": 1.0926,
"margin_dpo/margin_mean": 207.937255859375,
"margin_dpo/margin_std": 318.99566650390625,
"step": 374
},
{
"KL/chosen_KL_mean": -299.0926208496094,
"KL/mean": -396.4559326171875,
"KL/rejected_KL_mean": -493.8193054199219,
"KL/std": 231.56265258789062,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.0021589891985058784,
"fcm_dpo/delta": 0.03020719811320305,
"fcm_dpo/margin": 194.72665405273438,
"fcm_dpo/q_t": 0.40428146719932556,
"grad_norm": 35.277286529541016,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.46823275089263916,
"logits/rejected": -0.46644824743270874,
"logps/chosen": -359.1728515625,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -582.757568359375,
"loss": 1.1003,
"margin_dpo/margin_mean": 194.72665405273438,
"margin_dpo/margin_std": 283.26324462890625,
"step": 375
},
{
"KL/chosen_KL_mean": -286.27801513671875,
"KL/mean": -411.0360107421875,
"KL/rejected_KL_mean": -535.7940063476562,
"KL/std": 255.5110626220703,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0021486622281372547,
"fcm_dpo/delta": -0.09095398336648941,
"fcm_dpo/margin": 249.51597595214844,
"fcm_dpo/q_t": 0.37961071729660034,
"grad_norm": 34.17658996582031,
"learning_rate": 2.5e-07,
"logits/chosen": -0.45805996656417847,
"logits/rejected": -0.44645756483078003,
"logps/chosen": -348.9383239746094,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -641.320556640625,
"loss": 1.0161,
"margin_dpo/margin_mean": 249.51597595214844,
"margin_dpo/margin_std": 301.43994140625,
"step": 376
},
{
"KL/chosen_KL_mean": -297.97998046875,
"KL/mean": -417.2173156738281,
"KL/rejected_KL_mean": -536.4546508789062,
"KL/std": 261.6809997558594,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.0021219179034233093,
"fcm_dpo/delta": -0.058967623859643936,
"fcm_dpo/margin": 238.47463989257812,
"fcm_dpo/q_t": 0.3851660192012787,
"grad_norm": 27.344511032104492,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.41780710220336914,
"logits/rejected": -0.4187828600406647,
"logps/chosen": -352.458740234375,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -635.157958984375,
"loss": 1.0225,
"margin_dpo/margin_mean": 238.47463989257812,
"margin_dpo/margin_std": 277.2920837402344,
"step": 377
},
{
"KL/chosen_KL_mean": -281.651611328125,
"KL/mean": -413.4957580566406,
"KL/rejected_KL_mean": -545.3399047851562,
"KL/std": 253.0030975341797,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.0020764602813869715,
"fcm_dpo/delta": -0.10342134535312653,
"fcm_dpo/margin": 263.68829345703125,
"fcm_dpo/q_t": 0.37369853258132935,
"grad_norm": 26.619735717773438,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.4513610005378723,
"logits/rejected": -0.46656692028045654,
"logps/chosen": -326.6721496582031,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -633.3868408203125,
"loss": 0.985,
"margin_dpo/margin_mean": 263.68829345703125,
"margin_dpo/margin_std": 271.5170593261719,
"step": 378
},
{
"KL/chosen_KL_mean": -328.8233947753906,
"KL/mean": -453.4273376464844,
"KL/rejected_KL_mean": -578.03125,
"KL/std": 253.23651123046875,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.0020405231043696404,
"fcm_dpo/delta": -0.06235864385962486,
"fcm_dpo/margin": 249.2078857421875,
"fcm_dpo/q_t": 0.38584357500076294,
"grad_norm": 39.30223846435547,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.4471816420555115,
"logits/rejected": -0.47077521681785583,
"logps/chosen": -382.0054931640625,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -692.3314208984375,
"loss": 1.0263,
"margin_dpo/margin_mean": 249.2078857421875,
"margin_dpo/margin_std": 299.44866943359375,
"step": 379
},
{
"KL/chosen_KL_mean": -328.95916748046875,
"KL/mean": -467.37396240234375,
"KL/rejected_KL_mean": -605.788818359375,
"KL/std": 285.58038330078125,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.002012487966567278,
"fcm_dpo/delta": -0.11330637335777283,
"fcm_dpo/margin": 276.8296203613281,
"fcm_dpo/q_t": 0.37213844060897827,
"grad_norm": 28.289297103881836,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.43490737676620483,
"logits/rejected": -0.46167534589767456,
"logps/chosen": -380.31219482421875,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -709.98046875,
"loss": 0.9934,
"margin_dpo/margin_mean": 276.8296203613281,
"margin_dpo/margin_std": 309.166015625,
"step": 380
},
{
"KL/chosen_KL_mean": -342.4608459472656,
"KL/mean": -439.27081298828125,
"KL/rejected_KL_mean": -536.080810546875,
"KL/std": 238.89328002929688,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.0020117685198783875,
"fcm_dpo/delta": 0.06297034025192261,
"fcm_dpo/margin": 193.61993408203125,
"fcm_dpo/q_t": 0.4127262830734253,
"grad_norm": 32.157344818115234,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.47354856133461,
"logits/rejected": -0.45360469818115234,
"logps/chosen": -400.263916015625,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -615.3001708984375,
"loss": 1.1312,
"margin_dpo/margin_mean": 193.6199188232422,
"margin_dpo/margin_std": 321.12579345703125,
"step": 381
},
{
"KL/chosen_KL_mean": -339.0706481933594,
"KL/mean": -457.1148376464844,
"KL/rejected_KL_mean": -575.1590576171875,
"KL/std": 229.2410430908203,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0020139641128480434,
"fcm_dpo/delta": -0.026759367436170578,
"fcm_dpo/margin": 236.08834838867188,
"fcm_dpo/q_t": 0.3900807499885559,
"grad_norm": 31.58717918395996,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.48019176721572876,
"logits/rejected": -0.4856945872306824,
"logps/chosen": -405.0909423828125,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -685.8692016601562,
"loss": 1.0245,
"margin_dpo/margin_mean": 236.08834838867188,
"margin_dpo/margin_std": 247.97897338867188,
"step": 382
},
{
"KL/chosen_KL_mean": -347.0646667480469,
"KL/mean": -464.14361572265625,
"KL/rejected_KL_mean": -581.2225341796875,
"KL/std": 260.5420227050781,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0020031521562486887,
"fcm_dpo/delta": -0.020062297582626343,
"fcm_dpo/margin": 234.15789794921875,
"fcm_dpo/q_t": 0.3943726420402527,
"grad_norm": 30.257585525512695,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.4433164596557617,
"logits/rejected": -0.45195838809013367,
"logps/chosen": -397.4561462402344,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -674.9384765625,
"loss": 1.0747,
"margin_dpo/margin_mean": 234.1578826904297,
"margin_dpo/margin_std": 336.49066162109375,
"step": 383
},
{
"KL/chosen_KL_mean": -350.1620178222656,
"KL/mean": -443.7109375,
"KL/rejected_KL_mean": -537.2598876953125,
"KL/std": 237.53985595703125,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.002035179641097784,
"fcm_dpo/delta": 0.07113596796989441,
"fcm_dpo/margin": 187.097900390625,
"fcm_dpo/q_t": 0.4109281003475189,
"grad_norm": 36.85161209106445,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.5195727944374084,
"logits/rejected": -0.5128994584083557,
"logps/chosen": -402.2081298828125,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -623.020751953125,
"loss": 1.091,
"margin_dpo/margin_mean": 187.097900390625,
"margin_dpo/margin_std": 220.66925048828125,
"step": 384
},
{
"KL/chosen_KL_mean": -319.3924865722656,
"KL/mean": -434.7906494140625,
"KL/rejected_KL_mean": -550.188720703125,
"KL/std": 206.83387756347656,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0020276131108403206,
"fcm_dpo/delta": -0.01884370669722557,
"fcm_dpo/margin": 230.79629516601562,
"fcm_dpo/q_t": 0.39033573865890503,
"grad_norm": 41.22309112548828,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.49404364824295044,
"logits/rejected": -0.46726518869400024,
"logps/chosen": -384.94464111328125,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -628.0166625976562,
"loss": 1.0214,
"margin_dpo/margin_mean": 230.79629516601562,
"margin_dpo/margin_std": 225.4544677734375,
"step": 385
},
{
"KL/chosen_KL_mean": -332.60528564453125,
"KL/mean": -456.33349609375,
"KL/rejected_KL_mean": -580.061767578125,
"KL/std": 254.662353515625,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.002012161072343588,
"fcm_dpo/delta": -0.05040276050567627,
"fcm_dpo/margin": 247.45645141601562,
"fcm_dpo/q_t": 0.38694748282432556,
"grad_norm": 24.407316207885742,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.4635084867477417,
"logits/rejected": -0.45638737082481384,
"logps/chosen": -390.8271179199219,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -672.38916015625,
"loss": 1.0257,
"margin_dpo/margin_mean": 247.45645141601562,
"margin_dpo/margin_std": 289.11572265625,
"step": 386
},
{
"KL/chosen_KL_mean": -351.8877868652344,
"KL/mean": -453.7193603515625,
"KL/rejected_KL_mean": -555.5509033203125,
"KL/std": 235.2284698486328,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.002009193878620863,
"fcm_dpo/delta": 0.0423722118139267,
"fcm_dpo/margin": 203.66311645507812,
"fcm_dpo/q_t": 0.4049086570739746,
"grad_norm": 39.93413162231445,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.505128026008606,
"logits/rejected": -0.48430800437927246,
"logps/chosen": -418.3072509765625,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -647.7200927734375,
"loss": 1.0739,
"margin_dpo/margin_mean": 203.66311645507812,
"margin_dpo/margin_std": 238.2667694091797,
"step": 387
},
{
"KL/chosen_KL_mean": -334.48602294921875,
"KL/mean": -472.7179260253906,
"KL/rejected_KL_mean": -610.9498291015625,
"KL/std": 275.4636535644531,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.0019852365367114544,
"fcm_dpo/delta": -0.10500699281692505,
"fcm_dpo/margin": 276.4637756347656,
"fcm_dpo/q_t": 0.37743353843688965,
"grad_norm": 28.62255859375,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.4363114833831787,
"logits/rejected": -0.44703811407089233,
"logps/chosen": -384.615478515625,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -715.3829345703125,
"loss": 0.9961,
"margin_dpo/margin_mean": 276.4637756347656,
"margin_dpo/margin_std": 314.50836181640625,
"step": 388
},
{
"KL/chosen_KL_mean": -364.5044860839844,
"KL/mean": -476.4181213378906,
"KL/rejected_KL_mean": -588.331787109375,
"KL/std": 266.85693359375,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.0019789470825344324,
"fcm_dpo/delta": 0.007349299266934395,
"fcm_dpo/margin": 223.8272705078125,
"fcm_dpo/q_t": 0.39921024441719055,
"grad_norm": 27.75473976135254,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.4892725348472595,
"logits/rejected": -0.4785168170928955,
"logps/chosen": -422.41107177734375,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -666.246337890625,
"loss": 1.0679,
"margin_dpo/margin_mean": 223.8272705078125,
"margin_dpo/margin_std": 290.217529296875,
"step": 389
},
{
"KL/chosen_KL_mean": -362.6640625,
"KL/mean": -475.35125732421875,
"KL/rejected_KL_mean": -588.0384521484375,
"KL/std": 259.41107177734375,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.001976389205083251,
"fcm_dpo/delta": 0.0044990889728069305,
"fcm_dpo/margin": 225.3743896484375,
"fcm_dpo/q_t": 0.4021126627922058,
"grad_norm": 26.84067153930664,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.43252575397491455,
"logits/rejected": -0.42253702878952026,
"logps/chosen": -411.88995361328125,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -673.566650390625,
"loss": 1.0843,
"margin_dpo/margin_mean": 225.3743896484375,
"margin_dpo/margin_std": 330.4794921875,
"step": 390
},
{
"KL/chosen_KL_mean": -352.0406494140625,
"KL/mean": -421.3489685058594,
"KL/rejected_KL_mean": -490.6573486328125,
"KL/std": 240.73985290527344,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.00203782063908875,
"fcm_dpo/delta": 0.17224514484405518,
"fcm_dpo/margin": 138.6166534423828,
"fcm_dpo/q_t": 0.4359980821609497,
"grad_norm": 41.918697357177734,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.45235902070999146,
"logits/rejected": -0.441203236579895,
"logps/chosen": -416.37030029296875,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -577.3955078125,
"loss": 1.1896,
"margin_dpo/margin_mean": 138.61666870117188,
"margin_dpo/margin_std": 255.517333984375,
"step": 391
},
{
"KL/chosen_KL_mean": -301.568359375,
"KL/mean": -434.53594970703125,
"KL/rejected_KL_mean": -567.5035400390625,
"KL/std": 257.3258972167969,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0020327502861618996,
"fcm_dpo/delta": -0.09568814933300018,
"fcm_dpo/margin": 265.9351806640625,
"fcm_dpo/q_t": 0.37543368339538574,
"grad_norm": 28.45415496826172,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.44816461205482483,
"logits/rejected": -0.4401182532310486,
"logps/chosen": -355.07232666015625,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -669.849365234375,
"loss": 0.9758,
"margin_dpo/margin_mean": 265.9352111816406,
"margin_dpo/margin_std": 246.14971923828125,
"step": 392
},
{
"KL/chosen_KL_mean": -301.06414794921875,
"KL/mean": -402.669921875,
"KL/rejected_KL_mean": -504.27569580078125,
"KL/std": 253.20211791992188,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0020245746709406376,
"fcm_dpo/delta": 0.04025638476014137,
"fcm_dpo/margin": 203.21151733398438,
"fcm_dpo/q_t": 0.410128653049469,
"grad_norm": 25.076148986816406,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.4646386504173279,
"logits/rejected": -0.4622265696525574,
"logps/chosen": -347.5380859375,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -576.2445678710938,
"loss": 1.1072,
"margin_dpo/margin_mean": 203.21151733398438,
"margin_dpo/margin_std": 318.36810302734375,
"step": 393
},
{
"KL/chosen_KL_mean": -334.5086364746094,
"KL/mean": -448.73779296875,
"KL/rejected_KL_mean": -562.9669799804688,
"KL/std": 270.51513671875,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0020245497580617666,
"fcm_dpo/delta": -0.013191215693950653,
"fcm_dpo/margin": 228.45831298828125,
"fcm_dpo/q_t": 0.3956291675567627,
"grad_norm": 24.535133361816406,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.4499407708644867,
"logits/rejected": -0.44685885310173035,
"logps/chosen": -387.4201965332031,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -653.7896118164062,
"loss": 1.0496,
"margin_dpo/margin_mean": 228.45831298828125,
"margin_dpo/margin_std": 280.7309875488281,
"step": 394
},
{
"KL/chosen_KL_mean": -333.3252258300781,
"KL/mean": -451.8715515136719,
"KL/rejected_KL_mean": -570.4178466796875,
"KL/std": 266.8553466796875,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0020118574611842632,
"fcm_dpo/delta": -0.02858392894268036,
"fcm_dpo/margin": 237.0926055908203,
"fcm_dpo/q_t": 0.3931659460067749,
"grad_norm": 25.89862632751465,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.4901076555252075,
"logits/rejected": -0.4859440326690674,
"logps/chosen": -395.871337890625,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -654.200439453125,
"loss": 1.0512,
"margin_dpo/margin_mean": 237.0926055908203,
"margin_dpo/margin_std": 305.77239990234375,
"step": 395
},
{
"KL/chosen_KL_mean": -341.327880859375,
"KL/mean": -450.71270751953125,
"KL/rejected_KL_mean": -560.0974731445312,
"KL/std": 258.5706787109375,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.002014473080635071,
"fcm_dpo/delta": 0.009694435633718967,
"fcm_dpo/margin": 218.7696075439453,
"fcm_dpo/q_t": 0.39814984798431396,
"grad_norm": 28.5257511138916,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.48850005865097046,
"logits/rejected": -0.4679301977157593,
"logps/chosen": -410.3238220214844,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -648.744140625,
"loss": 1.0442,
"margin_dpo/margin_mean": 218.7696075439453,
"margin_dpo/margin_std": 234.68031311035156,
"step": 396
},
{
"KL/chosen_KL_mean": -318.0323486328125,
"KL/mean": -456.5221862792969,
"KL/rejected_KL_mean": -595.0120849609375,
"KL/std": 256.3633117675781,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.001978658139705658,
"fcm_dpo/delta": -0.10457342863082886,
"fcm_dpo/margin": 276.9797058105469,
"fcm_dpo/q_t": 0.37371158599853516,
"grad_norm": 37.82392120361328,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.44460928440093994,
"logits/rejected": -0.44621679186820984,
"logps/chosen": -379.30950927734375,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -698.128173828125,
"loss": 0.9756,
"margin_dpo/margin_mean": 276.9797058105469,
"margin_dpo/margin_std": 266.53509521484375,
"step": 397
},
{
"KL/chosen_KL_mean": -352.5345458984375,
"KL/mean": -476.9874267578125,
"KL/rejected_KL_mean": -601.4403686523438,
"KL/std": 269.32940673828125,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.001963344868272543,
"fcm_dpo/delta": -0.040712494403123856,
"fcm_dpo/margin": 248.90582275390625,
"fcm_dpo/q_t": 0.39044615626335144,
"grad_norm": 25.253372192382812,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.5356911420822144,
"logits/rejected": -0.5335468649864197,
"logps/chosen": -420.68609619140625,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -709.9639892578125,
"loss": 1.0363,
"margin_dpo/margin_mean": 248.90582275390625,
"margin_dpo/margin_std": 306.1085205078125,
"step": 398
},
{
"KL/chosen_KL_mean": -309.63555908203125,
"KL/mean": -423.51873779296875,
"KL/rejected_KL_mean": -537.40185546875,
"KL/std": 241.08370971679688,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.001956004649400711,
"fcm_dpo/delta": 0.004525328055024147,
"fcm_dpo/margin": 227.76632690429688,
"fcm_dpo/q_t": 0.39886805415153503,
"grad_norm": 25.981426239013672,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.5114948749542236,
"logits/rejected": -0.47601059079170227,
"logps/chosen": -370.5253601074219,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -615.367431640625,
"loss": 1.0688,
"margin_dpo/margin_mean": 227.76632690429688,
"margin_dpo/margin_std": 297.8195495605469,
"step": 399
},
{
"KL/chosen_KL_mean": -291.9248046875,
"KL/mean": -454.869384765625,
"KL/rejected_KL_mean": -617.8140258789062,
"KL/std": 256.5586242675781,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.001904614269733429,
"fcm_dpo/delta": -0.18265554308891296,
"fcm_dpo/margin": 325.8891906738281,
"fcm_dpo/q_t": 0.3558167815208435,
"grad_norm": 23.150972366333008,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.47274285554885864,
"logits/rejected": -0.4700891375541687,
"logps/chosen": -355.5683898925781,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -723.0667114257812,
"loss": 0.9217,
"margin_dpo/margin_mean": 325.88922119140625,
"margin_dpo/margin_std": 275.33416748046875,
"step": 400
},
{
"KL/chosen_KL_mean": -341.201171875,
"KL/mean": -432.50018310546875,
"KL/rejected_KL_mean": -523.7991943359375,
"KL/std": 262.24749755859375,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0019152223831042647,
"fcm_dpo/delta": 0.10403911024332047,
"fcm_dpo/margin": 182.59799194335938,
"fcm_dpo/q_t": 0.42341041564941406,
"grad_norm": 29.061887741088867,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.45198753476142883,
"logits/rejected": -0.4431244134902954,
"logps/chosen": -398.36419677734375,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -607.5916748046875,
"loss": 1.1722,
"margin_dpo/margin_mean": 182.59799194335938,
"margin_dpo/margin_std": 349.26129150390625,
"step": 401
},
{
"KL/chosen_KL_mean": -253.4839630126953,
"KL/mean": -422.302001953125,
"KL/rejected_KL_mean": -591.1199951171875,
"KL/std": 288.586669921875,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0018758836667984724,
"fcm_dpo/delta": -0.1958317756652832,
"fcm_dpo/margin": 337.63604736328125,
"fcm_dpo/q_t": 0.35503554344177246,
"grad_norm": 27.98488998413086,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.5198030471801758,
"logits/rejected": -0.5098150372505188,
"logps/chosen": -304.2243347167969,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -672.1660766601562,
"loss": 0.9146,
"margin_dpo/margin_mean": 337.63604736328125,
"margin_dpo/margin_std": 291.2978515625,
"step": 402
},
{
"KL/chosen_KL_mean": -288.88800048828125,
"KL/mean": -415.9601745605469,
"KL/rejected_KL_mean": -543.0323486328125,
"KL/std": 270.83282470703125,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0018494720570743084,
"fcm_dpo/delta": -0.021076738834381104,
"fcm_dpo/margin": 254.14431762695312,
"fcm_dpo/q_t": 0.39227643609046936,
"grad_norm": 25.534555435180664,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.479247510433197,
"logits/rejected": -0.4792447090148926,
"logps/chosen": -336.0353088378906,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -620.2989501953125,
"loss": 1.0356,
"margin_dpo/margin_mean": 254.14431762695312,
"margin_dpo/margin_std": 284.37744140625,
"step": 403
},
{
"KL/chosen_KL_mean": -319.4507751464844,
"KL/mean": -445.5923156738281,
"KL/rejected_KL_mean": -571.73388671875,
"KL/std": 257.37884521484375,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0018426838796585798,
"fcm_dpo/delta": -0.01567152328789234,
"fcm_dpo/margin": 252.2830810546875,
"fcm_dpo/q_t": 0.39486825466156006,
"grad_norm": 41.37036895751953,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.48405522108078003,
"logits/rejected": -0.4754765033721924,
"logps/chosen": -367.3260498046875,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -648.888916015625,
"loss": 1.0625,
"margin_dpo/margin_mean": 252.2830810546875,
"margin_dpo/margin_std": 332.94580078125,
"step": 404
},
{
"KL/chosen_KL_mean": -357.9908447265625,
"KL/mean": -459.80157470703125,
"KL/rejected_KL_mean": -561.6121826171875,
"KL/std": 276.78570556640625,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.0018360248068347573,
"fcm_dpo/delta": -0.04308407008647919,
"fcm_dpo/margin": 203.621337890625,
"fcm_dpo/q_t": 0.4165513515472412,
"grad_norm": 33.43486022949219,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.49097639322280884,
"logits/rejected": -0.48623454570770264,
"logps/chosen": -423.15374755859375,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -648.7990112304688,
"loss": 1.1423,
"margin_dpo/margin_mean": 203.62135314941406,
"margin_dpo/margin_std": 338.03448486328125,
"step": 405
},
{
"KL/chosen_KL_mean": -317.1373291015625,
"KL/mean": -450.2159423828125,
"KL/rejected_KL_mean": -583.2945556640625,
"KL/std": 284.7554931640625,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0018263730453327298,
"fcm_dpo/delta": -0.03822207450866699,
"fcm_dpo/margin": 266.1572570800781,
"fcm_dpo/q_t": 0.3904591500759125,
"grad_norm": 24.831371307373047,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.4690974950790405,
"logits/rejected": -0.47915327548980713,
"logps/chosen": -366.8781433105469,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -675.3732299804688,
"loss": 1.0355,
"margin_dpo/margin_mean": 266.1572570800781,
"margin_dpo/margin_std": 316.8359375,
"step": 406
},
{
"KL/chosen_KL_mean": -341.89794921875,
"KL/mean": -428.014892578125,
"KL/rejected_KL_mean": -514.1318359375,
"KL/std": 227.71868896484375,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0018160839099436998,
"fcm_dpo/delta": 0.01041320152580738,
"fcm_dpo/margin": 172.23391723632812,
"fcm_dpo/q_t": 0.4270077347755432,
"grad_norm": 31.205970764160156,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.5169594287872314,
"logits/rejected": -0.4908292889595032,
"logps/chosen": -398.2286376953125,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -591.6439208984375,
"loss": 1.1713,
"margin_dpo/margin_mean": 172.23390197753906,
"margin_dpo/margin_std": 299.04534912109375,
"step": 407
},
{
"KL/chosen_KL_mean": -337.2755432128906,
"KL/mean": -435.49871826171875,
"KL/rejected_KL_mean": -533.7218627929688,
"KL/std": 222.86465454101562,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.001842319150455296,
"fcm_dpo/delta": 0.09141945093870163,
"fcm_dpo/margin": 196.44630432128906,
"fcm_dpo/q_t": 0.41571488976478577,
"grad_norm": 28.42995834350586,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.5169058442115784,
"logits/rejected": -0.49273163080215454,
"logps/chosen": -407.0648498535156,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -623.8187866210938,
"loss": 1.1053,
"margin_dpo/margin_mean": 196.44630432128906,
"margin_dpo/margin_std": 245.20440673828125,
"step": 408
},
{
"KL/chosen_KL_mean": -327.26873779296875,
"KL/mean": -422.0277099609375,
"KL/rejected_KL_mean": -516.7866821289062,
"KL/std": 237.47921752929688,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.00188281643204391,
"fcm_dpo/delta": 0.09627067297697067,
"fcm_dpo/margin": 189.51795959472656,
"fcm_dpo/q_t": 0.4176041781902313,
"grad_norm": 33.84221267700195,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.4907482862472534,
"logits/rejected": -0.47613096237182617,
"logps/chosen": -394.586181640625,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -601.69091796875,
"loss": 1.1211,
"margin_dpo/margin_mean": 189.51795959472656,
"margin_dpo/margin_std": 266.58636474609375,
"step": 409
},
{
"KL/chosen_KL_mean": -303.18829345703125,
"KL/mean": -414.0080261230469,
"KL/rejected_KL_mean": -524.8277587890625,
"KL/std": 235.29071044921875,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.0018944379407912493,
"fcm_dpo/delta": 0.031448617577552795,
"fcm_dpo/margin": 221.63943481445312,
"fcm_dpo/q_t": 0.40194326639175415,
"grad_norm": 26.67073631286621,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.5039969086647034,
"logits/rejected": -0.5042929649353027,
"logps/chosen": -354.6536865234375,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -608.0267333984375,
"loss": 1.0691,
"margin_dpo/margin_mean": 221.63943481445312,
"margin_dpo/margin_std": 264.4658508300781,
"step": 410
},
{
"KL/chosen_KL_mean": -318.7891845703125,
"KL/mean": -429.96795654296875,
"KL/rejected_KL_mean": -541.1467895507812,
"KL/std": 254.40216064453125,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0018927098717540503,
"fcm_dpo/delta": 0.029360853135585785,
"fcm_dpo/margin": 222.35760498046875,
"fcm_dpo/q_t": 0.40499693155288696,
"grad_norm": 27.52345848083496,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.512640118598938,
"logits/rejected": -0.5237692594528198,
"logps/chosen": -371.096435546875,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -621.8417358398438,
"loss": 1.0792,
"margin_dpo/margin_mean": 222.3576202392578,
"margin_dpo/margin_std": 281.51507568359375,
"step": 411
},
{
"KL/chosen_KL_mean": -321.4520263671875,
"KL/mean": -433.51104736328125,
"KL/rejected_KL_mean": -545.570068359375,
"KL/std": 247.77584838867188,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0019169066799804568,
"fcm_dpo/delta": 0.02129899151623249,
"fcm_dpo/margin": 224.1180419921875,
"fcm_dpo/q_t": 0.4017683267593384,
"grad_norm": 37.43172836303711,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.5207057595252991,
"logits/rejected": -0.5486534833908081,
"logps/chosen": -374.5961608886719,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -645.630859375,
"loss": 1.0727,
"margin_dpo/margin_mean": 224.1180419921875,
"margin_dpo/margin_std": 288.02618408203125,
"step": 412
},
{
"KL/chosen_KL_mean": -341.4588928222656,
"KL/mean": -449.88580322265625,
"KL/rejected_KL_mean": -558.312744140625,
"KL/std": 245.0023193359375,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.001933481078594923,
"fcm_dpo/delta": 0.03179698809981346,
"fcm_dpo/margin": 216.85382080078125,
"fcm_dpo/q_t": 0.40238937735557556,
"grad_norm": 32.526119232177734,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.48236048221588135,
"logits/rejected": -0.4867170751094818,
"logps/chosen": -403.0408630371094,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -657.7861328125,
"loss": 1.0776,
"margin_dpo/margin_mean": 216.85382080078125,
"margin_dpo/margin_std": 277.03717041015625,
"step": 413
},
{
"KL/chosen_KL_mean": -309.00274658203125,
"KL/mean": -434.9940185546875,
"KL/rejected_KL_mean": -560.9852905273438,
"KL/std": 244.6328125,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.0019339543068781495,
"fcm_dpo/delta": -0.04007185250520706,
"fcm_dpo/margin": 251.98251342773438,
"fcm_dpo/q_t": 0.3875179588794708,
"grad_norm": 37.2226676940918,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.4675019681453705,
"logits/rejected": -0.46069198846817017,
"logps/chosen": -355.6342468261719,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -648.6318359375,
"loss": 1.0291,
"margin_dpo/margin_mean": 251.98251342773438,
"margin_dpo/margin_std": 279.03228759765625,
"step": 414
},
{
"KL/chosen_KL_mean": -354.3892822265625,
"KL/mean": -446.54925537109375,
"KL/rejected_KL_mean": -538.7092895507812,
"KL/std": 241.19760131835938,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0019371465314179659,
"fcm_dpo/delta": 0.09648571163415909,
"fcm_dpo/margin": 184.32000732421875,
"fcm_dpo/q_t": 0.4172021448612213,
"grad_norm": 31.669076919555664,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.50961834192276,
"logits/rejected": -0.5050987005233765,
"logps/chosen": -433.007568359375,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -639.186767578125,
"loss": 1.1235,
"margin_dpo/margin_mean": 184.32000732421875,
"margin_dpo/margin_std": 263.8765869140625,
"step": 415
},
{
"KL/chosen_KL_mean": -323.3999938964844,
"KL/mean": -470.9483642578125,
"KL/rejected_KL_mean": -618.4967651367188,
"KL/std": 268.1399230957031,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0019243636634200811,
"fcm_dpo/delta": -0.1247919350862503,
"fcm_dpo/margin": 295.0968017578125,
"fcm_dpo/q_t": 0.3685443699359894,
"grad_norm": 30.524866104125977,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.4989939332008362,
"logits/rejected": -0.500846266746521,
"logps/chosen": -381.67913818359375,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -709.0654907226562,
"loss": 0.9556,
"margin_dpo/margin_mean": 295.0968017578125,
"margin_dpo/margin_std": 258.8311767578125,
"step": 416
},
{
"KL/chosen_KL_mean": -328.4429931640625,
"KL/mean": -450.8934326171875,
"KL/rejected_KL_mean": -573.3438720703125,
"KL/std": 248.7044677734375,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.0019041774794459343,
"fcm_dpo/delta": -0.017197906970977783,
"fcm_dpo/margin": 244.90090942382812,
"fcm_dpo/q_t": 0.39115339517593384,
"grad_norm": 34.585113525390625,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.4820740520954132,
"logits/rejected": -0.45922285318374634,
"logps/chosen": -378.6417236328125,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -641.4957275390625,
"loss": 1.0242,
"margin_dpo/margin_mean": 244.90090942382812,
"margin_dpo/margin_std": 243.80502319335938,
"step": 417
},
{
"KL/chosen_KL_mean": -347.431640625,
"KL/mean": -476.9691467285156,
"KL/rejected_KL_mean": -606.5067138671875,
"KL/std": 282.0134582519531,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.0018994242418557405,
"fcm_dpo/delta": -0.04503042995929718,
"fcm_dpo/margin": 259.0750732421875,
"fcm_dpo/q_t": 0.3910468518733978,
"grad_norm": 22.76815414428711,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.5683892965316772,
"logits/rejected": -0.5466711521148682,
"logps/chosen": -429.41009521484375,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -711.1981811523438,
"loss": 1.0455,
"margin_dpo/margin_mean": 259.0750732421875,
"margin_dpo/margin_std": 329.2814025878906,
"step": 418
},
{
"KL/chosen_KL_mean": -326.120361328125,
"KL/mean": -465.67791748046875,
"KL/rejected_KL_mean": -605.2354736328125,
"KL/std": 252.4942626953125,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.0018608008977025747,
"fcm_dpo/delta": -0.07318148016929626,
"fcm_dpo/margin": 279.1151428222656,
"fcm_dpo/q_t": 0.3800005316734314,
"grad_norm": 28.3684139251709,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.4971669018268585,
"logits/rejected": -0.494901180267334,
"logps/chosen": -379.0690002441406,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -696.818603515625,
"loss": 1.0015,
"margin_dpo/margin_mean": 279.11517333984375,
"margin_dpo/margin_std": 291.6534729003906,
"step": 419
},
{
"KL/chosen_KL_mean": -420.7146301269531,
"KL/mean": -511.06640625,
"KL/rejected_KL_mean": -601.4180908203125,
"KL/std": 267.8034362792969,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.0018872196087613702,
"fcm_dpo/delta": 0.1126866489648819,
"fcm_dpo/margin": 180.70352172851562,
"fcm_dpo/q_t": 0.4257734715938568,
"grad_norm": 59.49646759033203,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.5531851649284363,
"logits/rejected": -0.5215734839439392,
"logps/chosen": -498.48455810546875,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -670.7379760742188,
"loss": 1.178,
"margin_dpo/margin_mean": 180.70352172851562,
"margin_dpo/margin_std": 357.1778259277344,
"step": 420
},
{
"KL/chosen_KL_mean": -343.73675537109375,
"KL/mean": -445.46746826171875,
"KL/rejected_KL_mean": -547.1982421875,
"KL/std": 263.5028381347656,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0019216202199459076,
"fcm_dpo/delta": 0.060789939016103745,
"fcm_dpo/margin": 203.46148681640625,
"fcm_dpo/q_t": 0.40861809253692627,
"grad_norm": 28.76307487487793,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.5395439863204956,
"logits/rejected": -0.528742790222168,
"logps/chosen": -397.50262451171875,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -636.4796752929688,
"loss": 1.1064,
"margin_dpo/margin_mean": 203.46148681640625,
"margin_dpo/margin_std": 282.8474426269531,
"step": 421
},
{
"KL/chosen_KL_mean": -387.14404296875,
"KL/mean": -500.13055419921875,
"KL/rejected_KL_mean": -613.1170654296875,
"KL/std": 266.84881591796875,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.00192258192691952,
"fcm_dpo/delta": 0.016255537047982216,
"fcm_dpo/margin": 225.97299194335938,
"fcm_dpo/q_t": 0.40042802691459656,
"grad_norm": 34.73599624633789,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.554384171962738,
"logits/rejected": -0.5551073551177979,
"logps/chosen": -455.77783203125,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -700.9805908203125,
"loss": 1.0796,
"margin_dpo/margin_mean": 225.97296142578125,
"margin_dpo/margin_std": 308.2744140625,
"step": 422
},
{
"KL/chosen_KL_mean": -360.21820068359375,
"KL/mean": -487.47760009765625,
"KL/rejected_KL_mean": -614.7369995117188,
"KL/std": 256.1720275878906,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.0019127356354147196,
"fcm_dpo/delta": -0.038765739649534225,
"fcm_dpo/margin": 254.51885986328125,
"fcm_dpo/q_t": 0.38787776231765747,
"grad_norm": 30.29926300048828,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.5579213500022888,
"logits/rejected": -0.5651543140411377,
"logps/chosen": -415.20849609375,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -701.0435791015625,
"loss": 1.0245,
"margin_dpo/margin_mean": 254.5188751220703,
"margin_dpo/margin_std": 281.6831970214844,
"step": 423
},
{
"KL/chosen_KL_mean": -322.7941589355469,
"KL/mean": -440.9080810546875,
"KL/rejected_KL_mean": -559.02197265625,
"KL/std": 260.0172424316406,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0019028335809707642,
"fcm_dpo/delta": 0.00016645342111587524,
"fcm_dpo/margin": 236.227783203125,
"fcm_dpo/q_t": 0.39966845512390137,
"grad_norm": 32.32646179199219,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.5459982752799988,
"logits/rejected": -0.5231644511222839,
"logps/chosen": -378.80609130859375,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -625.5009765625,
"loss": 1.0689,
"margin_dpo/margin_mean": 236.227783203125,
"margin_dpo/margin_std": 316.46826171875,
"step": 424
},
{
"KL/chosen_KL_mean": -347.72894287109375,
"KL/mean": -442.76678466796875,
"KL/rejected_KL_mean": -537.8046875,
"KL/std": 238.0361785888672,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.0019408970838412642,
"fcm_dpo/delta": 0.08372058719396591,
"fcm_dpo/margin": 190.07571411132812,
"fcm_dpo/q_t": 0.4140620231628418,
"grad_norm": 27.985706329345703,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.5209067463874817,
"logits/rejected": -0.5194746255874634,
"logps/chosen": -394.59796142578125,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -633.7301025390625,
"loss": 1.1256,
"margin_dpo/margin_mean": 190.07571411132812,
"margin_dpo/margin_std": 287.202392578125,
"step": 425
},
{
"KL/chosen_KL_mean": -310.969970703125,
"KL/mean": -432.341552734375,
"KL/rejected_KL_mean": -553.713134765625,
"KL/std": 245.95452880859375,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0019467303063720465,
"fcm_dpo/delta": -0.024154789745807648,
"fcm_dpo/margin": 242.74319458007812,
"fcm_dpo/q_t": 0.3915029764175415,
"grad_norm": 32.165985107421875,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.5235848426818848,
"logits/rejected": -0.4914889931678772,
"logps/chosen": -387.55352783203125,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -634.979736328125,
"loss": 1.0396,
"margin_dpo/margin_mean": 242.74319458007812,
"margin_dpo/margin_std": 277.2171630859375,
"step": 426
},
{
"KL/chosen_KL_mean": -299.4535827636719,
"KL/mean": -382.96673583984375,
"KL/rejected_KL_mean": -466.47991943359375,
"KL/std": 207.34808349609375,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.00196866225451231,
"fcm_dpo/delta": 0.12538892030715942,
"fcm_dpo/margin": 167.02633666992188,
"fcm_dpo/q_t": 0.4241478443145752,
"grad_norm": 25.871044158935547,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.544752299785614,
"logits/rejected": -0.5285120010375977,
"logps/chosen": -364.3074645996094,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -545.0458984375,
"loss": 1.14,
"margin_dpo/margin_mean": 167.02633666992188,
"margin_dpo/margin_std": 245.90792846679688,
"step": 427
},
{
"KL/chosen_KL_mean": -375.49188232421875,
"KL/mean": -489.534423828125,
"KL/rejected_KL_mean": -603.576904296875,
"KL/std": 270.32257080078125,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.0019773703534156084,
"fcm_dpo/delta": -0.001193435862660408,
"fcm_dpo/margin": 228.0850372314453,
"fcm_dpo/q_t": 0.39808985590934753,
"grad_norm": 33.139686584472656,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.5463074445724487,
"logits/rejected": -0.5484579205513,
"logps/chosen": -438.1285400390625,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -706.8587646484375,
"loss": 1.0805,
"margin_dpo/margin_mean": 228.08505249023438,
"margin_dpo/margin_std": 324.0859069824219,
"step": 428
},
{
"KL/chosen_KL_mean": -365.306640625,
"KL/mean": -460.9208984375,
"KL/rejected_KL_mean": -556.53515625,
"KL/std": 241.172607421875,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.002017046557739377,
"fcm_dpo/delta": 0.06516958028078079,
"fcm_dpo/margin": 191.228515625,
"fcm_dpo/q_t": 0.4123389720916748,
"grad_norm": 34.08958435058594,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.5600682497024536,
"logits/rejected": -0.5419148206710815,
"logps/chosen": -446.5406494140625,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -648.330078125,
"loss": 1.1252,
"margin_dpo/margin_mean": 191.228515625,
"margin_dpo/margin_std": 289.6811828613281,
"step": 429
},
{
"KL/chosen_KL_mean": -306.03424072265625,
"KL/mean": -404.863037109375,
"KL/rejected_KL_mean": -503.69183349609375,
"KL/std": 227.00172424316406,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.0020285071805119514,
"fcm_dpo/delta": 0.05085544288158417,
"fcm_dpo/margin": 197.65756225585938,
"fcm_dpo/q_t": 0.4082902669906616,
"grad_norm": 29.763206481933594,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.5655794143676758,
"logits/rejected": -0.5617367625236511,
"logps/chosen": -366.95458984375,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -608.1146240234375,
"loss": 1.0833,
"margin_dpo/margin_mean": 197.65757751464844,
"margin_dpo/margin_std": 247.31101989746094,
"step": 430
},
{
"KL/chosen_KL_mean": -292.8988952636719,
"KL/mean": -434.15472412109375,
"KL/rejected_KL_mean": -575.41064453125,
"KL/std": 256.44805908203125,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0019896533340215683,
"fcm_dpo/delta": -0.11906355619430542,
"fcm_dpo/margin": 282.5116882324219,
"fcm_dpo/q_t": 0.37088543176651,
"grad_norm": 27.917299270629883,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.4833042621612549,
"logits/rejected": -0.48560959100723267,
"logps/chosen": -350.24761962890625,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -668.2508544921875,
"loss": 0.9744,
"margin_dpo/margin_mean": 282.5116882324219,
"margin_dpo/margin_std": 277.9963684082031,
"step": 431
},
{
"KL/chosen_KL_mean": -300.67681884765625,
"KL/mean": -429.38427734375,
"KL/rejected_KL_mean": -558.0917358398438,
"KL/std": 243.09274291992188,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.0019767279736697674,
"fcm_dpo/delta": -0.0625704973936081,
"fcm_dpo/margin": 257.4149169921875,
"fcm_dpo/q_t": 0.38336220383644104,
"grad_norm": 39.286922454833984,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.4827229380607605,
"logits/rejected": -0.4681396484375,
"logps/chosen": -371.7516174316406,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -656.6712646484375,
"loss": 1.0035,
"margin_dpo/margin_mean": 257.4149169921875,
"margin_dpo/margin_std": 251.6300048828125,
"step": 432
},
{
"KL/chosen_KL_mean": -386.28424072265625,
"KL/mean": -480.7707824707031,
"KL/rejected_KL_mean": -575.25732421875,
"KL/std": 247.94790649414062,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0019827033393085003,
"fcm_dpo/delta": 0.07806547731161118,
"fcm_dpo/margin": 188.97311401367188,
"fcm_dpo/q_t": 0.41317322850227356,
"grad_norm": 32.629791259765625,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.5527180433273315,
"logits/rejected": -0.53752601146698,
"logps/chosen": -444.55743408203125,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -671.208251953125,
"loss": 1.1252,
"margin_dpo/margin_mean": 188.97311401367188,
"margin_dpo/margin_std": 289.7763671875,
"step": 433
},
{
"KL/chosen_KL_mean": -345.12957763671875,
"KL/mean": -454.7913818359375,
"KL/rejected_KL_mean": -564.453125,
"KL/std": 267.315673828125,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0019846938084810972,
"fcm_dpo/delta": 0.015174292027950287,
"fcm_dpo/margin": 219.32354736328125,
"fcm_dpo/q_t": 0.407636821269989,
"grad_norm": 24.881118774414062,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.549630880355835,
"logits/rejected": -0.5502759218215942,
"logps/chosen": -407.1033020019531,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -642.9517822265625,
"loss": 1.0916,
"margin_dpo/margin_mean": 219.32354736328125,
"margin_dpo/margin_std": 339.36029052734375,
"step": 434
},
{
"KL/chosen_KL_mean": -320.1326904296875,
"KL/mean": -441.7854919433594,
"KL/rejected_KL_mean": -563.4382934570312,
"KL/std": 257.5054931640625,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.0019807624630630016,
"fcm_dpo/delta": -0.03362615779042244,
"fcm_dpo/margin": 243.30557250976562,
"fcm_dpo/q_t": 0.390411376953125,
"grad_norm": 30.694725036621094,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.5910142660140991,
"logits/rejected": -0.5799299478530884,
"logps/chosen": -371.634765625,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -651.0051879882812,
"loss": 1.0414,
"margin_dpo/margin_mean": 243.30557250976562,
"margin_dpo/margin_std": 290.63226318359375,
"step": 435
},
{
"KL/chosen_KL_mean": -340.8654479980469,
"KL/mean": -449.2633056640625,
"KL/rejected_KL_mean": -557.6612548828125,
"KL/std": 235.69265747070312,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0019800812005996704,
"fcm_dpo/delta": 0.02152244932949543,
"fcm_dpo/margin": 216.79571533203125,
"fcm_dpo/q_t": 0.4021452069282532,
"grad_norm": 45.93106460571289,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.5817707180976868,
"logits/rejected": -0.5527193546295166,
"logps/chosen": -412.2691650390625,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -640.3889770507812,
"loss": 1.0708,
"margin_dpo/margin_mean": 216.79571533203125,
"margin_dpo/margin_std": 274.1136779785156,
"step": 436
},
{
"KL/chosen_KL_mean": -344.4331359863281,
"KL/mean": -455.29022216796875,
"KL/rejected_KL_mean": -566.1473388671875,
"KL/std": 226.3272705078125,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0019896202720701694,
"fcm_dpo/delta": 0.009276720695197582,
"fcm_dpo/margin": 221.71414184570312,
"fcm_dpo/q_t": 0.39847952127456665,
"grad_norm": 27.636913299560547,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.5687664747238159,
"logits/rejected": -0.5476101636886597,
"logps/chosen": -409.1773986816406,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -648.1908569335938,
"loss": 1.0609,
"margin_dpo/margin_mean": 221.714111328125,
"margin_dpo/margin_std": 271.2551574707031,
"step": 437
},
{
"KL/chosen_KL_mean": -358.9455261230469,
"KL/mean": -482.2747802734375,
"KL/rejected_KL_mean": -605.60400390625,
"KL/std": 256.87603759765625,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0019732650835067034,
"fcm_dpo/delta": -0.039032407104969025,
"fcm_dpo/margin": 246.65850830078125,
"fcm_dpo/q_t": 0.3885589838027954,
"grad_norm": 39.22119140625,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.5658591985702515,
"logits/rejected": -0.5471130609512329,
"logps/chosen": -417.9642028808594,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -688.6808471679688,
"loss": 1.0287,
"margin_dpo/margin_mean": 246.65850830078125,
"margin_dpo/margin_std": 279.75701904296875,
"step": 438
},
{
"KL/chosen_KL_mean": -371.96002197265625,
"KL/mean": -471.221923828125,
"KL/rejected_KL_mean": -570.48388671875,
"KL/std": 252.96397399902344,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.0019477336900308728,
"fcm_dpo/delta": -0.07375746965408325,
"fcm_dpo/margin": 198.52389526367188,
"fcm_dpo/q_t": 0.4123408794403076,
"grad_norm": 32.960567474365234,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.5992504358291626,
"logits/rejected": -0.6015431880950928,
"logps/chosen": -425.74407958984375,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -654.4693603515625,
"loss": 1.1156,
"margin_dpo/margin_mean": 198.52391052246094,
"margin_dpo/margin_std": 282.75445556640625,
"step": 439
},
{
"KL/chosen_KL_mean": -395.271240234375,
"KL/mean": -509.26043701171875,
"KL/rejected_KL_mean": -623.2496337890625,
"KL/std": 295.75421142578125,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.0019553429447114468,
"fcm_dpo/delta": 0.004188535735011101,
"fcm_dpo/margin": 227.97836303710938,
"fcm_dpo/q_t": 0.40090298652648926,
"grad_norm": 33.02730941772461,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.610379695892334,
"logits/rejected": -0.6039736270904541,
"logps/chosen": -473.8379821777344,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -719.7474365234375,
"loss": 1.07,
"margin_dpo/margin_mean": 227.97836303710938,
"margin_dpo/margin_std": 305.73785400390625,
"step": 440
},
{
"KL/chosen_KL_mean": -428.8697509765625,
"KL/mean": -545.2886352539062,
"KL/rejected_KL_mean": -661.70751953125,
"KL/std": 302.4857482910156,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.001954545732587576,
"fcm_dpo/delta": -0.0055627599358558655,
"fcm_dpo/margin": 232.83779907226562,
"fcm_dpo/q_t": 0.40071725845336914,
"grad_norm": 33.655216217041016,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.6163830757141113,
"logits/rejected": -0.6027711033821106,
"logps/chosen": -489.6941833496094,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -758.1783447265625,
"loss": 1.0982,
"margin_dpo/margin_mean": 232.83778381347656,
"margin_dpo/margin_std": 361.769775390625,
"step": 441
},
{
"KL/chosen_KL_mean": -376.24456787109375,
"KL/mean": -508.6070251464844,
"KL/rejected_KL_mean": -640.969482421875,
"KL/std": 298.88250732421875,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.0019426376093178988,
"fcm_dpo/delta": -0.06845666468143463,
"fcm_dpo/margin": 264.7249755859375,
"fcm_dpo/q_t": 0.3868436813354492,
"grad_norm": 34.53351974487305,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.5752331018447876,
"logits/rejected": -0.5666894912719727,
"logps/chosen": -423.2557678222656,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -717.5087890625,
"loss": 1.0305,
"margin_dpo/margin_mean": 264.7249755859375,
"margin_dpo/margin_std": 324.138427734375,
"step": 442
},
{
"KL/chosen_KL_mean": -424.26495361328125,
"KL/mean": -506.6785583496094,
"KL/rejected_KL_mean": -589.0921020507812,
"KL/std": 271.61700439453125,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0019203823758289218,
"fcm_dpo/delta": -0.0008800772484391928,
"fcm_dpo/margin": 164.82717895507812,
"fcm_dpo/q_t": 0.42879998683929443,
"grad_norm": 39.51858901977539,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.5896974802017212,
"logits/rejected": -0.5681110620498657,
"logps/chosen": -495.5379638671875,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -675.7720947265625,
"loss": 1.1907,
"margin_dpo/margin_mean": 164.82717895507812,
"margin_dpo/margin_std": 329.18048095703125,
"step": 443
},
{
"KL/chosen_KL_mean": -414.8670654296875,
"KL/mean": -552.189208984375,
"KL/rejected_KL_mean": -689.5113525390625,
"KL/std": 302.2316589355469,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.001895219786092639,
"fcm_dpo/delta": -0.07462961971759796,
"fcm_dpo/margin": 274.64422607421875,
"fcm_dpo/q_t": 0.3836691081523895,
"grad_norm": 32.75905990600586,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.5605142116546631,
"logits/rejected": -0.5698819756507874,
"logps/chosen": -472.0807800292969,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -786.7662353515625,
"loss": 1.0206,
"margin_dpo/margin_mean": 274.6442565917969,
"margin_dpo/margin_std": 333.3004150390625,
"step": 444
},
{
"KL/chosen_KL_mean": -366.47662353515625,
"KL/mean": -495.1947326660156,
"KL/rejected_KL_mean": -623.912841796875,
"KL/std": 253.20156860351562,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0018830732442438602,
"fcm_dpo/delta": -0.036526892334222794,
"fcm_dpo/margin": 257.4361877441406,
"fcm_dpo/q_t": 0.39043277502059937,
"grad_norm": 34.652286529541016,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.5867752432823181,
"logits/rejected": -0.5800052285194397,
"logps/chosen": -433.77642822265625,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -716.595458984375,
"loss": 1.0375,
"margin_dpo/margin_mean": 257.4361572265625,
"margin_dpo/margin_std": 305.46954345703125,
"step": 445
},
{
"KL/chosen_KL_mean": -326.2323913574219,
"KL/mean": -462.51434326171875,
"KL/rejected_KL_mean": -598.79638671875,
"KL/std": 260.0819396972656,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.0018716863123700023,
"fcm_dpo/delta": -0.06374150514602661,
"fcm_dpo/margin": 272.56396484375,
"fcm_dpo/q_t": 0.3816561698913574,
"grad_norm": 33.55470275878906,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.5853652954101562,
"logits/rejected": -0.5997810363769531,
"logps/chosen": -385.33087158203125,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -700.060546875,
"loss": 0.9998,
"margin_dpo/margin_mean": 272.56396484375,
"margin_dpo/margin_std": 263.48199462890625,
"step": 446
},
{
"KL/chosen_KL_mean": -344.34954833984375,
"KL/mean": -484.68585205078125,
"KL/rejected_KL_mean": -625.0221557617188,
"KL/std": 302.6340637207031,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0018296856433153152,
"fcm_dpo/delta": -0.06744357943534851,
"fcm_dpo/margin": 280.672607421875,
"fcm_dpo/q_t": 0.38556453585624695,
"grad_norm": 44.91852569580078,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.6326247453689575,
"logits/rejected": -0.6444536447525024,
"logps/chosen": -400.42486572265625,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -723.7169189453125,
"loss": 1.0182,
"margin_dpo/margin_mean": 280.672607421875,
"margin_dpo/margin_std": 336.85003662109375,
"step": 447
},
{
"KL/chosen_KL_mean": -371.67254638671875,
"KL/mean": -509.5328369140625,
"KL/rejected_KL_mean": -647.3931884765625,
"KL/std": 269.0086975097656,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.001821416663005948,
"fcm_dpo/delta": -0.05504203215241432,
"fcm_dpo/margin": 275.7206726074219,
"fcm_dpo/q_t": 0.3859594464302063,
"grad_norm": 46.23940658569336,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.5190054178237915,
"logits/rejected": -0.526978075504303,
"logps/chosen": -431.6763916015625,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -749.6578369140625,
"loss": 1.02,
"margin_dpo/margin_mean": 275.720703125,
"margin_dpo/margin_std": 309.9884033203125,
"step": 448
},
{
"KL/chosen_KL_mean": -379.67486572265625,
"KL/mean": -511.91595458984375,
"KL/rejected_KL_mean": -644.1571044921875,
"KL/std": 313.6270751953125,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.0017995501402765512,
"fcm_dpo/delta": -0.02732807956635952,
"fcm_dpo/margin": 264.48223876953125,
"fcm_dpo/q_t": 0.3959568738937378,
"grad_norm": 28.76017951965332,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.5502740144729614,
"logits/rejected": -0.5482885837554932,
"logps/chosen": -447.20147705078125,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -732.7540283203125,
"loss": 1.0654,
"margin_dpo/margin_mean": 264.48223876953125,
"margin_dpo/margin_std": 377.0546875,
"step": 449
},
{
"KL/chosen_KL_mean": -308.00689697265625,
"KL/mean": -457.80841064453125,
"KL/rejected_KL_mean": -607.6099853515625,
"KL/std": 286.56097412109375,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.0017785783857107162,
"fcm_dpo/delta": -0.08745589852333069,
"fcm_dpo/margin": 299.60308837890625,
"fcm_dpo/q_t": 0.376192569732666,
"grad_norm": 43.1098747253418,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.569983959197998,
"logits/rejected": -0.5806500315666199,
"logps/chosen": -365.114990234375,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -710.3648681640625,
"loss": 0.9844,
"margin_dpo/margin_mean": 299.6030578613281,
"margin_dpo/margin_std": 291.56182861328125,
"step": 450
},
{
"KL/chosen_KL_mean": -399.1632080078125,
"KL/mean": -514.521484375,
"KL/rejected_KL_mean": -629.8797607421875,
"KL/std": 308.2806396484375,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0017831064760684967,
"fcm_dpo/delta": 0.040064115077257156,
"fcm_dpo/margin": 230.71658325195312,
"fcm_dpo/q_t": 0.4046586751937866,
"grad_norm": 30.686725616455078,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.6119334697723389,
"logits/rejected": -0.5944463014602661,
"logps/chosen": -457.63201904296875,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -702.8092041015625,
"loss": 1.1071,
"margin_dpo/margin_mean": 230.71658325195312,
"margin_dpo/margin_std": 350.39141845703125,
"step": 451
},
{
"KL/chosen_KL_mean": -281.5623779296875,
"KL/mean": -419.28631591796875,
"KL/rejected_KL_mean": -557.01025390625,
"KL/std": 256.2310791015625,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0017701241886243224,
"fcm_dpo/delta": -0.039577219635248184,
"fcm_dpo/margin": 275.44793701171875,
"fcm_dpo/q_t": 0.3859231472015381,
"grad_norm": 22.7626895904541,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.5375150442123413,
"logits/rejected": -0.5545802116394043,
"logps/chosen": -331.95294189453125,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -654.7816772460938,
"loss": 1.0059,
"margin_dpo/margin_mean": 275.4479064941406,
"margin_dpo/margin_std": 258.84332275390625,
"step": 452
},
{
"KL/chosen_KL_mean": -345.3682556152344,
"KL/mean": -464.7120361328125,
"KL/rejected_KL_mean": -584.0557861328125,
"KL/std": 255.49658203125,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0017713573761284351,
"fcm_dpo/delta": 0.028329838067293167,
"fcm_dpo/margin": 238.6875,
"fcm_dpo/q_t": 0.40426090359687805,
"grad_norm": 28.043825149536133,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.5721216201782227,
"logits/rejected": -0.553996205329895,
"logps/chosen": -403.0831298828125,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -666.26318359375,
"loss": 1.0727,
"margin_dpo/margin_mean": 238.68748474121094,
"margin_dpo/margin_std": 301.5249328613281,
"step": 453
},
{
"KL/chosen_KL_mean": -398.77349853515625,
"KL/mean": -537.2942504882812,
"KL/rejected_KL_mean": -675.8150024414062,
"KL/std": 296.11334228515625,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0017631438095122576,
"fcm_dpo/delta": -0.0406990572810173,
"fcm_dpo/margin": 277.04150390625,
"fcm_dpo/q_t": 0.38935303688049316,
"grad_norm": 35.47222900390625,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.6266130208969116,
"logits/rejected": -0.6151348352432251,
"logps/chosen": -459.7191467285156,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -760.7657470703125,
"loss": 1.0307,
"margin_dpo/margin_mean": 277.04150390625,
"margin_dpo/margin_std": 327.983154296875,
"step": 454
},
{
"KL/chosen_KL_mean": -345.7542724609375,
"KL/mean": -505.41656494140625,
"KL/rejected_KL_mean": -665.078857421875,
"KL/std": 305.7221374511719,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0017463660333305597,
"fcm_dpo/delta": -0.11417567729949951,
"fcm_dpo/margin": 319.32452392578125,
"fcm_dpo/q_t": 0.3756142258644104,
"grad_norm": 33.242618560791016,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.5865689516067505,
"logits/rejected": -0.6037914156913757,
"logps/chosen": -390.6409912109375,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -780.3803100585938,
"loss": 0.9926,
"margin_dpo/margin_mean": 319.32452392578125,
"margin_dpo/margin_std": 348.2029724121094,
"step": 455
},
{
"KL/chosen_KL_mean": -358.7083740234375,
"KL/mean": -513.3322143554688,
"KL/rejected_KL_mean": -667.9560546875,
"KL/std": 299.8691711425781,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.001704810420051217,
"fcm_dpo/delta": -0.08146022260189056,
"fcm_dpo/margin": 309.2476806640625,
"fcm_dpo/q_t": 0.3785049617290497,
"grad_norm": 22.669376373291016,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.587792158126831,
"logits/rejected": -0.6062419414520264,
"logps/chosen": -415.74517822265625,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -773.1738891601562,
"loss": 0.9816,
"margin_dpo/margin_mean": 309.2476806640625,
"margin_dpo/margin_std": 289.26556396484375,
"step": 456
},
{
"KL/chosen_KL_mean": -343.7048645019531,
"KL/mean": -496.25714111328125,
"KL/rejected_KL_mean": -648.8094482421875,
"KL/std": 288.51763916015625,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.0016754315001890063,
"fcm_dpo/delta": -0.0648340955376625,
"fcm_dpo/margin": 305.10455322265625,
"fcm_dpo/q_t": 0.3803033232688904,
"grad_norm": 34.253662109375,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.5840927362442017,
"logits/rejected": -0.5754066705703735,
"logps/chosen": -397.9473876953125,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -733.9189453125,
"loss": 0.9921,
"margin_dpo/margin_mean": 305.1045227050781,
"margin_dpo/margin_std": 281.67279052734375,
"step": 457
},
{
"KL/chosen_KL_mean": -343.4403076171875,
"KL/mean": -487.31805419921875,
"KL/rejected_KL_mean": -631.19580078125,
"KL/std": 269.67620849609375,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.001662727678194642,
"fcm_dpo/delta": -0.030117183923721313,
"fcm_dpo/margin": 287.7554931640625,
"fcm_dpo/q_t": 0.3899010121822357,
"grad_norm": 26.614864349365234,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.5804847478866577,
"logits/rejected": -0.5738258361816406,
"logps/chosen": -398.84918212890625,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -728.8790283203125,
"loss": 1.025,
"margin_dpo/margin_mean": 287.7554931640625,
"margin_dpo/margin_std": 308.74249267578125,
"step": 458
},
{
"KL/chosen_KL_mean": -373.6165466308594,
"KL/mean": -522.7772827148438,
"KL/rejected_KL_mean": -671.93798828125,
"KL/std": 304.5191955566406,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0016487675020471215,
"fcm_dpo/delta": -0.04434160515666008,
"fcm_dpo/margin": 298.32147216796875,
"fcm_dpo/q_t": 0.38747796416282654,
"grad_norm": 26.16275405883789,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.6179863214492798,
"logits/rejected": -0.6297715902328491,
"logps/chosen": -420.17401123046875,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -758.1065673828125,
"loss": 1.0262,
"margin_dpo/margin_mean": 298.32147216796875,
"margin_dpo/margin_std": 336.20526123046875,
"step": 459
},
{
"KL/chosen_KL_mean": -432.34326171875,
"KL/mean": -589.347412109375,
"KL/rejected_KL_mean": -746.3515625,
"KL/std": 301.2634582519531,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.001635140273720026,
"fcm_dpo/delta": -0.06683729588985443,
"fcm_dpo/margin": 314.00830078125,
"fcm_dpo/q_t": 0.38239821791648865,
"grad_norm": 37.637847900390625,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.6397815942764282,
"logits/rejected": -0.6632001399993896,
"logps/chosen": -483.9781799316406,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -850.470947265625,
"loss": 1.0048,
"margin_dpo/margin_mean": 314.00830078125,
"margin_dpo/margin_std": 332.333984375,
"step": 460
},
{
"KL/chosen_KL_mean": -436.39190673828125,
"KL/mean": -561.193115234375,
"KL/rejected_KL_mean": -685.9942626953125,
"KL/std": 296.3171691894531,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0016416897997260094,
"fcm_dpo/delta": 0.041765857487916946,
"fcm_dpo/margin": 249.60235595703125,
"fcm_dpo/q_t": 0.40686777234077454,
"grad_norm": 29.86077880859375,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.6551775932312012,
"logits/rejected": -0.6613701581954956,
"logps/chosen": -491.5738525390625,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -772.47119140625,
"loss": 1.0875,
"margin_dpo/margin_mean": 249.60235595703125,
"margin_dpo/margin_std": 334.3131408691406,
"step": 461
},
{
"KL/chosen_KL_mean": -455.2949523925781,
"KL/mean": -557.1524658203125,
"KL/rejected_KL_mean": -659.0100708007812,
"KL/std": 303.5262451171875,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.00166890409309417,
"fcm_dpo/delta": 0.11394853889942169,
"fcm_dpo/margin": 203.7151336669922,
"fcm_dpo/q_t": 0.42322608828544617,
"grad_norm": 42.896141052246094,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.7339116334915161,
"logits/rejected": -0.7233344912528992,
"logps/chosen": -525.2229614257812,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -737.8511962890625,
"loss": 1.147,
"margin_dpo/margin_mean": 203.7151336669922,
"margin_dpo/margin_std": 331.66717529296875,
"step": 462
},
{
"KL/chosen_KL_mean": -450.4989013671875,
"KL/mean": -588.5634765625,
"KL/rejected_KL_mean": -726.6280517578125,
"KL/std": 329.5040283203125,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.0016832472756505013,
"fcm_dpo/delta": -0.015869732946157455,
"fcm_dpo/margin": 276.129150390625,
"fcm_dpo/q_t": 0.396880567073822,
"grad_norm": 38.95415115356445,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.7291879057884216,
"logits/rejected": -0.7324599623680115,
"logps/chosen": -505.7732849121094,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -815.6530151367188,
"loss": 1.0696,
"margin_dpo/margin_mean": 276.129150390625,
"margin_dpo/margin_std": 379.4897155761719,
"step": 463
},
{
"KL/chosen_KL_mean": -442.44622802734375,
"KL/mean": -652.0371704101562,
"KL/rejected_KL_mean": -861.628173828125,
"KL/std": 379.09393310546875,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0016052606515586376,
"fcm_dpo/delta": -0.24131107330322266,
"fcm_dpo/margin": 419.1819152832031,
"fcm_dpo/q_t": 0.3485097885131836,
"grad_norm": 27.98969841003418,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.6850185394287109,
"logits/rejected": -0.7362366914749146,
"logps/chosen": -493.3585205078125,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -964.1175537109375,
"loss": 0.9105,
"margin_dpo/margin_mean": 419.18194580078125,
"margin_dpo/margin_std": 395.11285400390625,
"step": 464
},
{
"KL/chosen_KL_mean": -431.66802978515625,
"KL/mean": -607.3240966796875,
"KL/rejected_KL_mean": -782.980224609375,
"KL/std": 374.92095947265625,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0015757776563987136,
"fcm_dpo/delta": -0.109546959400177,
"fcm_dpo/margin": 351.31219482421875,
"fcm_dpo/q_t": 0.373515784740448,
"grad_norm": 42.90937423706055,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.6763527393341064,
"logits/rejected": -0.6878402233123779,
"logps/chosen": -491.78485107421875,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -896.92626953125,
"loss": 0.9949,
"margin_dpo/margin_mean": 351.31219482421875,
"margin_dpo/margin_std": 390.9532775878906,
"step": 465
},
{
"KL/chosen_KL_mean": -476.77154541015625,
"KL/mean": -619.39599609375,
"KL/rejected_KL_mean": -762.0203857421875,
"KL/std": 340.4292297363281,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0015551161486655474,
"fcm_dpo/delta": 0.00630560889840126,
"fcm_dpo/margin": 285.2489013671875,
"fcm_dpo/q_t": 0.3987009823322296,
"grad_norm": 34.12393569946289,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.7104131579399109,
"logits/rejected": -0.7117249965667725,
"logps/chosen": -529.6925048828125,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -852.3358764648438,
"loss": 1.0748,
"margin_dpo/margin_mean": 285.2489013671875,
"margin_dpo/margin_std": 380.52740478515625,
"step": 466
},
{
"KL/chosen_KL_mean": -609.2482299804688,
"KL/mean": -759.8226928710938,
"KL/rejected_KL_mean": -910.3971557617188,
"KL/std": 434.1422119140625,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.001543498132377863,
"fcm_dpo/delta": -0.01767650991678238,
"fcm_dpo/margin": 301.14898681640625,
"fcm_dpo/q_t": 0.40035611391067505,
"grad_norm": 57.92776870727539,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.7824667692184448,
"logits/rejected": -0.7698106169700623,
"logps/chosen": -687.9640502929688,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -1013.25732421875,
"loss": 1.131,
"margin_dpo/margin_mean": 301.14898681640625,
"margin_dpo/margin_std": 524.8055419921875,
"step": 467
},
{
"KL/chosen_KL_mean": -472.4308166503906,
"KL/mean": -655.0126953125,
"KL/rejected_KL_mean": -837.5946044921875,
"KL/std": 395.85833740234375,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0015256652841344476,
"fcm_dpo/delta": -0.11409050226211548,
"fcm_dpo/margin": 365.1637878417969,
"fcm_dpo/q_t": 0.3762136697769165,
"grad_norm": 44.05681228637695,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.774760365486145,
"logits/rejected": -0.7754192352294922,
"logps/chosen": -542.3662109375,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -938.6234130859375,
"loss": 1.003,
"margin_dpo/margin_mean": 365.16375732421875,
"margin_dpo/margin_std": 439.28802490234375,
"step": 468
},
{
"KL/chosen_KL_mean": -497.517578125,
"KL/mean": -682.90234375,
"KL/rejected_KL_mean": -868.287109375,
"KL/std": 357.67254638671875,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.001489096088334918,
"fcm_dpo/delta": -0.10935281962156296,
"fcm_dpo/margin": 370.7695007324219,
"fcm_dpo/q_t": 0.3750303089618683,
"grad_norm": 42.75261688232422,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.7901207208633423,
"logits/rejected": -0.7956848740577698,
"logps/chosen": -565.6422729492188,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -973.073486328125,
"loss": 0.9883,
"margin_dpo/margin_mean": 370.76947021484375,
"margin_dpo/margin_std": 395.64892578125,
"step": 469
},
{
"KL/chosen_KL_mean": -409.7502746582031,
"KL/mean": -582.3035888671875,
"KL/rejected_KL_mean": -754.8568115234375,
"KL/std": 381.45758056640625,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0014777039177715778,
"fcm_dpo/delta": -0.06313945353031158,
"fcm_dpo/margin": 345.1065673828125,
"fcm_dpo/q_t": 0.38330626487731934,
"grad_norm": 28.172273635864258,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.7635469436645508,
"logits/rejected": -0.77159583568573,
"logps/chosen": -453.54217529296875,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -837.5596923828125,
"loss": 1.0253,
"margin_dpo/margin_mean": 345.1065673828125,
"margin_dpo/margin_std": 412.84716796875,
"step": 470
},
{
"KL/chosen_KL_mean": -509.9143981933594,
"KL/mean": -653.645751953125,
"KL/rejected_KL_mean": -797.3770751953125,
"KL/std": 380.884033203125,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0014690503012388945,
"fcm_dpo/delta": 0.028516050428152084,
"fcm_dpo/margin": 287.4627380371094,
"fcm_dpo/q_t": 0.4048519730567932,
"grad_norm": 34.24898147583008,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.8426990509033203,
"logits/rejected": -0.8456913232803345,
"logps/chosen": -573.25390625,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -880.987548828125,
"loss": 1.0843,
"margin_dpo/margin_mean": 287.46270751953125,
"margin_dpo/margin_std": 388.9373779296875,
"step": 471
},
{
"KL/chosen_KL_mean": -516.6842041015625,
"KL/mean": -687.0798950195312,
"KL/rejected_KL_mean": -857.4755859375,
"KL/std": 442.78399658203125,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.0014645196497440338,
"fcm_dpo/delta": -0.051864590495824814,
"fcm_dpo/margin": 340.7913513183594,
"fcm_dpo/q_t": 0.392802894115448,
"grad_norm": 33.31867599487305,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.8658884763717651,
"logits/rejected": -0.8394947648048401,
"logps/chosen": -600.350341796875,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -974.684814453125,
"loss": 1.0571,
"margin_dpo/margin_mean": 340.79132080078125,
"margin_dpo/margin_std": 477.96142578125,
"step": 472
},
{
"KL/chosen_KL_mean": -618.8231201171875,
"KL/mean": -719.730712890625,
"KL/rejected_KL_mean": -820.6383056640625,
"KL/std": 455.2254943847656,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0014934407081454992,
"fcm_dpo/delta": 0.15348844230175018,
"fcm_dpo/margin": 201.815185546875,
"fcm_dpo/q_t": 0.43637731671333313,
"grad_norm": 90.4559555053711,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.8223817348480225,
"logits/rejected": -0.790869951248169,
"logps/chosen": -682.320068359375,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -901.784912109375,
"loss": 1.2702,
"margin_dpo/margin_mean": 201.815185546875,
"margin_dpo/margin_std": 588.429443359375,
"step": 473
},
{
"KL/chosen_KL_mean": -513.7476196289062,
"KL/mean": -648.7586669921875,
"KL/rejected_KL_mean": -783.769775390625,
"KL/std": 405.5135498046875,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.0015309953596442938,
"fcm_dpo/delta": 0.036234356462955475,
"fcm_dpo/margin": 270.0220947265625,
"fcm_dpo/q_t": 0.4074624478816986,
"grad_norm": 41.096923828125,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.8282029628753662,
"logits/rejected": -0.8300504684448242,
"logps/chosen": -566.359619140625,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -873.8501586914062,
"loss": 1.1394,
"margin_dpo/margin_mean": 270.0220947265625,
"margin_dpo/margin_std": 470.0948791503906,
"step": 474
},
{
"KL/chosen_KL_mean": -387.83551025390625,
"KL/mean": -572.85498046875,
"KL/rejected_KL_mean": -757.87451171875,
"KL/std": 353.4816589355469,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.001494031399488449,
"fcm_dpo/delta": -0.10902509838342667,
"fcm_dpo/margin": 370.0389404296875,
"fcm_dpo/q_t": 0.37435996532440186,
"grad_norm": 37.775211334228516,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.8475956916809082,
"logits/rejected": -0.8703779578208923,
"logps/chosen": -430.3307189941406,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -847.9374389648438,
"loss": 0.9778,
"margin_dpo/margin_mean": 370.0389404296875,
"margin_dpo/margin_std": 375.77508544921875,
"step": 475
},
{
"KL/chosen_KL_mean": -462.84210205078125,
"KL/mean": -608.64599609375,
"KL/rejected_KL_mean": -754.4498291015625,
"KL/std": 369.49456787109375,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.001488366862758994,
"fcm_dpo/delta": 0.016667019575834274,
"fcm_dpo/margin": 291.60772705078125,
"fcm_dpo/q_t": 0.4024167060852051,
"grad_norm": 39.59403610229492,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.8512526154518127,
"logits/rejected": -0.8595808148384094,
"logps/chosen": -505.79150390625,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -828.1600341796875,
"loss": 1.0732,
"margin_dpo/margin_mean": 291.60772705078125,
"margin_dpo/margin_std": 385.09527587890625,
"step": 476
},
{
"KL/chosen_KL_mean": -478.7229919433594,
"KL/mean": -620.529296875,
"KL/rejected_KL_mean": -762.3355712890625,
"KL/std": 378.3432312011719,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0015028079506009817,
"fcm_dpo/delta": 0.024538554251194,
"fcm_dpo/margin": 283.6126403808594,
"fcm_dpo/q_t": 0.40365082025527954,
"grad_norm": 31.907373428344727,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.8746368288993835,
"logits/rejected": -0.8382138013839722,
"logps/chosen": -549.49560546875,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -838.4730224609375,
"loss": 1.1097,
"margin_dpo/margin_mean": 283.6126403808594,
"margin_dpo/margin_std": 449.53643798828125,
"step": 477
},
{
"KL/chosen_KL_mean": -380.40606689453125,
"KL/mean": -544.5728149414062,
"KL/rejected_KL_mean": -708.739501953125,
"KL/std": 327.50653076171875,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.001494677970185876,
"fcm_dpo/delta": -0.04291198402643204,
"fcm_dpo/margin": 328.3334655761719,
"fcm_dpo/q_t": 0.38815736770629883,
"grad_norm": 32.1937141418457,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.7859635353088379,
"logits/rejected": -0.8034379482269287,
"logps/chosen": -421.84661865234375,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -794.1015014648438,
"loss": 1.0287,
"margin_dpo/margin_mean": 328.3334655761719,
"margin_dpo/margin_std": 377.14154052734375,
"step": 478
},
{
"KL/chosen_KL_mean": -488.9976501464844,
"KL/mean": -642.759033203125,
"KL/rejected_KL_mean": -796.5204467773438,
"KL/std": 384.7127380371094,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.001495036412961781,
"fcm_dpo/delta": -0.011177890002727509,
"fcm_dpo/margin": 307.5227966308594,
"fcm_dpo/q_t": 0.39697474241256714,
"grad_norm": 30.52669334411621,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.8063873052597046,
"logits/rejected": -0.8286073207855225,
"logps/chosen": -542.9055786132812,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -891.6367797851562,
"loss": 1.0735,
"margin_dpo/margin_mean": 307.5227966308594,
"margin_dpo/margin_std": 426.5589904785156,
"step": 479
},
{
"KL/chosen_KL_mean": -568.2318725585938,
"KL/mean": -696.13330078125,
"KL/rejected_KL_mean": -824.03466796875,
"KL/std": 368.392333984375,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0014745690859854221,
"fcm_dpo/delta": -0.06113547831773758,
"fcm_dpo/margin": 255.80279541015625,
"fcm_dpo/q_t": 0.4146702289581299,
"grad_norm": 50.518821716308594,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.842763364315033,
"logits/rejected": -0.8381805419921875,
"logps/chosen": -626.91455078125,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -906.9671630859375,
"loss": 1.1417,
"margin_dpo/margin_mean": 255.8028106689453,
"margin_dpo/margin_std": 410.9605712890625,
"step": 480
},
{
"KL/chosen_KL_mean": -473.3878173828125,
"KL/mean": -666.9186401367188,
"KL/rejected_KL_mean": -860.449462890625,
"KL/std": 400.59881591796875,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.0014397315680980682,
"fcm_dpo/delta": -0.11371327191591263,
"fcm_dpo/margin": 387.06158447265625,
"fcm_dpo/q_t": 0.3753812313079834,
"grad_norm": 39.827056884765625,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.8515434265136719,
"logits/rejected": -0.8767064809799194,
"logps/chosen": -528.3521118164062,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -952.869873046875,
"loss": 0.9919,
"margin_dpo/margin_mean": 387.06158447265625,
"margin_dpo/margin_std": 437.6363830566406,
"step": 481
},
{
"KL/chosen_KL_mean": -537.002197265625,
"KL/mean": -625.9591064453125,
"KL/rejected_KL_mean": -714.9160766601562,
"KL/std": 389.1396484375,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0014753930736333132,
"fcm_dpo/delta": 0.19242650270462036,
"fcm_dpo/margin": 177.91390991210938,
"fcm_dpo/q_t": 0.4377623200416565,
"grad_norm": 50.80120849609375,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.7530587315559387,
"logits/rejected": -0.7404736280441284,
"logps/chosen": -604.5556640625,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -802.505615234375,
"loss": 1.2491,
"margin_dpo/margin_mean": 177.91390991210938,
"margin_dpo/margin_std": 463.9508361816406,
"step": 482
},
{
"KL/chosen_KL_mean": -468.40838623046875,
"KL/mean": -670.406494140625,
"KL/rejected_KL_mean": -872.4046630859375,
"KL/std": 407.040283203125,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0014546338934451342,
"fcm_dpo/delta": -0.1465245485305786,
"fcm_dpo/margin": 403.99627685546875,
"fcm_dpo/q_t": 0.37036433815956116,
"grad_norm": 31.81561851501465,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.8379670977592468,
"logits/rejected": -0.8484210968017578,
"logps/chosen": -531.7381591796875,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -968.191650390625,
"loss": 0.9783,
"margin_dpo/margin_mean": 403.99627685546875,
"margin_dpo/margin_std": 463.239013671875,
"step": 483
},
{
"KL/chosen_KL_mean": -399.396484375,
"KL/mean": -573.8797607421875,
"KL/rejected_KL_mean": -748.3630981445312,
"KL/std": 383.2296142578125,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0014413539320230484,
"fcm_dpo/delta": -0.05605652928352356,
"fcm_dpo/margin": 348.9666748046875,
"fcm_dpo/q_t": 0.3857998847961426,
"grad_norm": 47.86911392211914,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.825036883354187,
"logits/rejected": -0.8444268703460693,
"logps/chosen": -458.53460693359375,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -832.7345581054688,
"loss": 1.0165,
"margin_dpo/margin_mean": 348.9666442871094,
"margin_dpo/margin_std": 382.70794677734375,
"step": 484
},
{
"KL/chosen_KL_mean": -446.0076904296875,
"KL/mean": -606.5994873046875,
"KL/rejected_KL_mean": -767.1912841796875,
"KL/std": 386.23712158203125,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0014315908774733543,
"fcm_dpo/delta": -0.01041480153799057,
"fcm_dpo/margin": 321.18359375,
"fcm_dpo/q_t": 0.39765143394470215,
"grad_norm": 30.10974884033203,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.7779227495193481,
"logits/rejected": -0.7912708520889282,
"logps/chosen": -504.85723876953125,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -870.555419921875,
"loss": 1.0713,
"margin_dpo/margin_mean": 321.1835632324219,
"margin_dpo/margin_std": 446.01361083984375,
"step": 485
},
{
"KL/chosen_KL_mean": -487.64141845703125,
"KL/mean": -678.8277587890625,
"KL/rejected_KL_mean": -870.01416015625,
"KL/std": 434.8795471191406,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.0014062260743230581,
"fcm_dpo/delta": -0.09276005625724792,
"fcm_dpo/margin": 382.3727722167969,
"fcm_dpo/q_t": 0.38320037722587585,
"grad_norm": 37.55424118041992,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -0.8121265172958374,
"logits/rejected": -0.8253448009490967,
"logps/chosen": -542.901123046875,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -962.153564453125,
"loss": 1.0407,
"margin_dpo/margin_mean": 382.3727722167969,
"margin_dpo/margin_std": 533.3743286132812,
"step": 486
},
{
"KL/chosen_KL_mean": -539.01416015625,
"KL/mean": -692.552001953125,
"KL/rejected_KL_mean": -846.08984375,
"KL/std": 414.926025390625,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0014086742885410786,
"fcm_dpo/delta": 0.017959173768758774,
"fcm_dpo/margin": 307.0757751464844,
"fcm_dpo/q_t": 0.40038296580314636,
"grad_norm": 47.89488220214844,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -0.8488699197769165,
"logits/rejected": -0.8536651730537415,
"logps/chosen": -592.077392578125,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -938.5086669921875,
"loss": 1.0884,
"margin_dpo/margin_mean": 307.0757751464844,
"margin_dpo/margin_std": 442.1873779296875,
"step": 487
},
{
"KL/chosen_KL_mean": -430.9812927246094,
"KL/mean": -593.21337890625,
"KL/rejected_KL_mean": -755.4454345703125,
"KL/std": 377.4382629394531,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0014093462377786636,
"fcm_dpo/delta": -0.00790850818157196,
"fcm_dpo/margin": 324.46417236328125,
"fcm_dpo/q_t": 0.3960450291633606,
"grad_norm": 43.94591522216797,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.7722993493080139,
"logits/rejected": -0.7786195278167725,
"logps/chosen": -483.2094421386719,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -839.4520263671875,
"loss": 1.0493,
"margin_dpo/margin_mean": 324.4642028808594,
"margin_dpo/margin_std": 387.154541015625,
"step": 488
},
{
"KL/chosen_KL_mean": -415.774658203125,
"KL/mean": -566.9913940429688,
"KL/rejected_KL_mean": -718.2081298828125,
"KL/std": 378.1129455566406,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0014117740793153644,
"fcm_dpo/delta": 0.023877691477537155,
"fcm_dpo/margin": 302.43341064453125,
"fcm_dpo/q_t": 0.40576857328414917,
"grad_norm": 32.45310974121094,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.8120991587638855,
"logits/rejected": -0.8098138570785522,
"logps/chosen": -471.7642822265625,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -797.606201171875,
"loss": 1.0815,
"margin_dpo/margin_mean": 302.43341064453125,
"margin_dpo/margin_std": 418.1960754394531,
"step": 489
},
{
"KL/chosen_KL_mean": -514.631591796875,
"KL/mean": -681.1277465820312,
"KL/rejected_KL_mean": -847.6239013671875,
"KL/std": 470.99456787109375,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.0014093939680606127,
"fcm_dpo/delta": -0.0203517135232687,
"fcm_dpo/margin": 332.9923095703125,
"fcm_dpo/q_t": 0.39851754903793335,
"grad_norm": 56.83975601196289,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.8639624118804932,
"logits/rejected": -0.9009851217269897,
"logps/chosen": -566.998046875,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -958.032958984375,
"loss": 1.1059,
"margin_dpo/margin_mean": 332.9923095703125,
"margin_dpo/margin_std": 549.37158203125,
"step": 490
},
{
"KL/chosen_KL_mean": -476.43017578125,
"KL/mean": -588.9735107421875,
"KL/rejected_KL_mean": -701.5167236328125,
"KL/std": 392.32257080078125,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.001434988807886839,
"fcm_dpo/delta": 0.13088186085224152,
"fcm_dpo/margin": 225.08653259277344,
"fcm_dpo/q_t": 0.42855405807495117,
"grad_norm": 35.58346176147461,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.8441819548606873,
"logits/rejected": -0.8365911841392517,
"logps/chosen": -536.5464477539062,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -774.78955078125,
"loss": 1.1687,
"margin_dpo/margin_mean": 225.08653259277344,
"margin_dpo/margin_std": 409.4179382324219,
"step": 491
},
{
"KL/chosen_KL_mean": -505.2855224609375,
"KL/mean": -620.657958984375,
"KL/rejected_KL_mean": -736.030517578125,
"KL/std": 418.7660827636719,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0014617950655519962,
"fcm_dpo/delta": 0.11679119616746902,
"fcm_dpo/margin": 230.74496459960938,
"fcm_dpo/q_t": 0.42761561274528503,
"grad_norm": 37.419185638427734,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -0.9038804769515991,
"logits/rejected": -0.8865909576416016,
"logps/chosen": -559.2796630859375,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -808.6901245117188,
"loss": 1.1954,
"margin_dpo/margin_mean": 230.74497985839844,
"margin_dpo/margin_std": 487.385498046875,
"step": 492
},
{
"KL/chosen_KL_mean": -544.4580688476562,
"KL/mean": -659.949951171875,
"KL/rejected_KL_mean": -775.44189453125,
"KL/std": 443.6405944824219,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0014702860498800874,
"fcm_dpo/delta": -0.01809088885784149,
"fcm_dpo/margin": 230.98385620117188,
"fcm_dpo/q_t": 0.42237934470176697,
"grad_norm": 41.59807586669922,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -0.9416635036468506,
"logits/rejected": -0.9134702682495117,
"logps/chosen": -619.955322265625,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -862.764892578125,
"loss": 1.1716,
"margin_dpo/margin_mean": 230.98385620117188,
"margin_dpo/margin_std": 431.32763671875,
"step": 493
},
{
"KL/chosen_KL_mean": -424.497802734375,
"KL/mean": -618.718017578125,
"KL/rejected_KL_mean": -812.938232421875,
"KL/std": 400.7784423828125,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.0014452817849814892,
"fcm_dpo/delta": -0.1179928109049797,
"fcm_dpo/margin": 388.44036865234375,
"fcm_dpo/q_t": 0.3718945384025574,
"grad_norm": 39.06217575073242,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.7399489879608154,
"logits/rejected": -0.7753201127052307,
"logps/chosen": -465.8570556640625,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -899.029541015625,
"loss": 0.9768,
"margin_dpo/margin_mean": 388.44036865234375,
"margin_dpo/margin_std": 396.9564208984375,
"step": 494
},
{
"KL/chosen_KL_mean": -481.8299865722656,
"KL/mean": -655.73876953125,
"KL/rejected_KL_mean": -829.6475830078125,
"KL/std": 413.8068542480469,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.00142820761539042,
"fcm_dpo/delta": -0.049179498106241226,
"fcm_dpo/margin": 347.817626953125,
"fcm_dpo/q_t": 0.39178192615509033,
"grad_norm": 40.44029235839844,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -0.91786789894104,
"logits/rejected": -0.9246433973312378,
"logps/chosen": -545.3650512695312,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -921.072021484375,
"loss": 1.0539,
"margin_dpo/margin_mean": 347.817626953125,
"margin_dpo/margin_std": 487.64837646484375,
"step": 495
},
{
"KL/chosen_KL_mean": -596.630859375,
"KL/mean": -693.388916015625,
"KL/rejected_KL_mean": -790.1470336914062,
"KL/std": 331.32080078125,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.001459787250496447,
"fcm_dpo/delta": 0.17228274047374725,
"fcm_dpo/margin": 193.51617431640625,
"fcm_dpo/q_t": 0.4351438879966736,
"grad_norm": 62.60634994506836,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.8866853713989258,
"logits/rejected": -0.8603767156600952,
"logps/chosen": -669.2227783203125,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -874.4763793945312,
"loss": 1.1956,
"margin_dpo/margin_mean": 193.5161895751953,
"margin_dpo/margin_std": 373.43487548828125,
"step": 496
},
{
"KL/chosen_KL_mean": -574.8228759765625,
"KL/mean": -666.93798828125,
"KL/rejected_KL_mean": -759.0531005859375,
"KL/std": 406.2542724609375,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.001477421261370182,
"fcm_dpo/delta": 0.019095903262495995,
"fcm_dpo/margin": 184.23023986816406,
"fcm_dpo/q_t": 0.4375380277633667,
"grad_norm": 46.24803924560547,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.8670381307601929,
"logits/rejected": -0.8514336943626404,
"logps/chosen": -633.4168701171875,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -835.3414916992188,
"loss": 1.2152,
"margin_dpo/margin_mean": 184.23023986816406,
"margin_dpo/margin_std": 388.11932373046875,
"step": 497
},
{
"KL/chosen_KL_mean": -540.2715454101562,
"KL/mean": -685.60107421875,
"KL/rejected_KL_mean": -830.9306640625,
"KL/std": 424.14251708984375,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0014816848561167717,
"fcm_dpo/delta": 0.020207226276397705,
"fcm_dpo/margin": 290.6590881347656,
"fcm_dpo/q_t": 0.40341994166374207,
"grad_norm": 44.21152877807617,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.8600029945373535,
"logits/rejected": -0.8464577198028564,
"logps/chosen": -611.4771728515625,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -914.888671875,
"loss": 1.1199,
"margin_dpo/margin_mean": 290.6590576171875,
"margin_dpo/margin_std": 487.44805908203125,
"step": 498
},
{
"KL/chosen_KL_mean": -472.27783203125,
"KL/mean": -655.0006103515625,
"KL/rejected_KL_mean": -837.723388671875,
"KL/std": 415.3236999511719,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0014626685297116637,
"fcm_dpo/delta": -0.08938349783420563,
"fcm_dpo/margin": 365.4455261230469,
"fcm_dpo/q_t": 0.3801768124103546,
"grad_norm": 38.70993423461914,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.8755702972412109,
"logits/rejected": -0.9045356512069702,
"logps/chosen": -523.5330200195312,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -938.8020629882812,
"loss": 1.0234,
"margin_dpo/margin_mean": 365.4455261230469,
"margin_dpo/margin_std": 463.28094482421875,
"step": 499
},
{
"KL/chosen_KL_mean": -572.775634765625,
"KL/mean": -727.460205078125,
"KL/rejected_KL_mean": -882.1448974609375,
"KL/std": 375.6351318359375,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0014550636988133192,
"fcm_dpo/delta": -0.00019872747361660004,
"fcm_dpo/margin": 309.3692932128906,
"fcm_dpo/q_t": 0.39942067861557007,
"grad_norm": 42.67442321777344,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.8324778079986572,
"logits/rejected": -0.8552345037460327,
"logps/chosen": -629.8030395507812,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -976.0791015625,
"loss": 1.091,
"margin_dpo/margin_mean": 309.36932373046875,
"margin_dpo/margin_std": 463.14117431640625,
"step": 500
},
{
"KL/chosen_KL_mean": -506.867431640625,
"KL/mean": -670.6456909179688,
"KL/rejected_KL_mean": -834.4239501953125,
"KL/std": 402.8489990234375,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.00145415507722646,
"fcm_dpo/delta": -0.027667174115777016,
"fcm_dpo/margin": 327.5564880371094,
"fcm_dpo/q_t": 0.39601457118988037,
"grad_norm": 43.718650817871094,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.8393124341964722,
"logits/rejected": -0.8210893273353577,
"logps/chosen": -561.2269287109375,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -914.5806274414062,
"loss": 1.0852,
"margin_dpo/margin_mean": 327.55645751953125,
"margin_dpo/margin_std": 501.5951232910156,
"step": 501
},
{
"KL/chosen_KL_mean": -455.6566162109375,
"KL/mean": -623.0474243164062,
"KL/rejected_KL_mean": -790.4382934570312,
"KL/std": 357.35693359375,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0014448589645326138,
"fcm_dpo/delta": -0.035458121448755264,
"fcm_dpo/margin": 334.7816467285156,
"fcm_dpo/q_t": 0.3888877034187317,
"grad_norm": 36.984375,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.7750924825668335,
"logits/rejected": -0.7572276592254639,
"logps/chosen": -523.2571411132812,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -873.3870849609375,
"loss": 1.021,
"margin_dpo/margin_mean": 334.7816467285156,
"margin_dpo/margin_std": 354.950927734375,
"step": 502
},
{
"KL/chosen_KL_mean": -465.2955322265625,
"KL/mean": -621.4876098632812,
"KL/rejected_KL_mean": -777.6796875,
"KL/std": 354.7469177246094,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.0014357357285916805,
"fcm_dpo/delta": 0.0014605261385440826,
"fcm_dpo/margin": 312.3841552734375,
"fcm_dpo/q_t": 0.39940670132637024,
"grad_norm": 31.054977416992188,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.7540430426597595,
"logits/rejected": -0.7392014861106873,
"logps/chosen": -520.3739624023438,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -860.1851196289062,
"loss": 1.0571,
"margin_dpo/margin_mean": 312.3841552734375,
"margin_dpo/margin_std": 388.606201171875,
"step": 503
},
{
"KL/chosen_KL_mean": -503.01611328125,
"KL/mean": -684.0391845703125,
"KL/rejected_KL_mean": -865.0621948242188,
"KL/std": 420.90887451171875,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.0014258314622566104,
"fcm_dpo/delta": -0.06977090984582901,
"fcm_dpo/margin": 362.0460510253906,
"fcm_dpo/q_t": 0.38591358065605164,
"grad_norm": 38.77161407470703,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.8095864057540894,
"logits/rejected": -0.8270820379257202,
"logps/chosen": -562.9818115234375,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -968.8243408203125,
"loss": 1.036,
"margin_dpo/margin_mean": 362.0460205078125,
"margin_dpo/margin_std": 463.34405517578125,
"step": 504
},
{
"KL/chosen_KL_mean": -570.5095825195312,
"KL/mean": -671.4177856445312,
"KL/rejected_KL_mean": -772.3260498046875,
"KL/std": 422.823486328125,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0014556320384144783,
"fcm_dpo/delta": 0.16071611642837524,
"fcm_dpo/margin": 201.81646728515625,
"fcm_dpo/q_t": 0.43397390842437744,
"grad_norm": 46.75086975097656,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.825531005859375,
"logits/rejected": -0.7959909439086914,
"logps/chosen": -646.6644897460938,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -860.911376953125,
"loss": 1.1987,
"margin_dpo/margin_mean": 201.81646728515625,
"margin_dpo/margin_std": 405.55169677734375,
"step": 505
},
{
"KL/chosen_KL_mean": -451.24114990234375,
"KL/mean": -619.97607421875,
"KL/rejected_KL_mean": -788.7110595703125,
"KL/std": 395.23736572265625,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0014494328061118722,
"fcm_dpo/delta": -0.0418471023440361,
"fcm_dpo/margin": 337.46990966796875,
"fcm_dpo/q_t": 0.3904111683368683,
"grad_norm": 32.250953674316406,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.8289726376533508,
"logits/rejected": -0.8355446457862854,
"logps/chosen": -500.20166015625,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -867.1260986328125,
"loss": 1.044,
"margin_dpo/margin_mean": 337.46990966796875,
"margin_dpo/margin_std": 427.08843994140625,
"step": 506
},
{
"KL/chosen_KL_mean": -512.9838256835938,
"KL/mean": -654.876953125,
"KL/rejected_KL_mean": -796.7701416015625,
"KL/std": 450.25811767578125,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.001459820894524455,
"fcm_dpo/delta": 0.037282489240169525,
"fcm_dpo/margin": 283.78631591796875,
"fcm_dpo/q_t": 0.41154006123542786,
"grad_norm": 34.5527229309082,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.8142247200012207,
"logits/rejected": -0.8100612163543701,
"logps/chosen": -579.7853393554688,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -892.14306640625,
"loss": 1.1213,
"margin_dpo/margin_mean": 283.7863464355469,
"margin_dpo/margin_std": 480.62255859375,
"step": 507
},
{
"KL/chosen_KL_mean": -530.6473999023438,
"KL/mean": -646.10302734375,
"KL/rejected_KL_mean": -761.5587158203125,
"KL/std": 424.214111328125,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0014920630492269993,
"fcm_dpo/delta": 0.10881674289703369,
"fcm_dpo/margin": 230.91127014160156,
"fcm_dpo/q_t": 0.4257569909095764,
"grad_norm": 52.76668930053711,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.816558837890625,
"logits/rejected": -0.8101228475570679,
"logps/chosen": -601.9508666992188,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -857.186279296875,
"loss": 1.1953,
"margin_dpo/margin_mean": 230.91127014160156,
"margin_dpo/margin_std": 495.2095947265625,
"step": 508
},
{
"KL/chosen_KL_mean": -414.64959716796875,
"KL/mean": -541.17529296875,
"KL/rejected_KL_mean": -667.700927734375,
"KL/std": 303.1500244140625,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0015182525385171175,
"fcm_dpo/delta": 0.06775674968957901,
"fcm_dpo/margin": 253.0513458251953,
"fcm_dpo/q_t": 0.4133894145488739,
"grad_norm": 35.25603103637695,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.7462732791900635,
"logits/rejected": -0.7376848459243774,
"logps/chosen": -478.46856689453125,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -750.9573974609375,
"loss": 1.1125,
"margin_dpo/margin_mean": 253.05133056640625,
"margin_dpo/margin_std": 366.0683288574219,
"step": 509
},
{
"KL/chosen_KL_mean": -472.19097900390625,
"KL/mean": -642.7222900390625,
"KL/rejected_KL_mean": -813.2535400390625,
"KL/std": 352.47137451171875,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0015035050455480814,
"fcm_dpo/delta": -0.06613890826702118,
"fcm_dpo/margin": 341.0625305175781,
"fcm_dpo/q_t": 0.3812809884548187,
"grad_norm": 32.88304138183594,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.724981963634491,
"logits/rejected": -0.7406862378120422,
"logps/chosen": -524.0694580078125,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -916.0186767578125,
"loss": 1.0011,
"margin_dpo/margin_mean": 341.0625305175781,
"margin_dpo/margin_std": 347.5075378417969,
"step": 510
},
{
"KL/chosen_KL_mean": -436.04022216796875,
"KL/mean": -575.18603515625,
"KL/rejected_KL_mean": -714.331787109375,
"KL/std": 371.4091491699219,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.001506120664998889,
"fcm_dpo/delta": 0.032133154571056366,
"fcm_dpo/margin": 278.29156494140625,
"fcm_dpo/q_t": 0.4063592255115509,
"grad_norm": 38.73003387451172,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.7694292068481445,
"logits/rejected": -0.7684683799743652,
"logps/chosen": -496.2783203125,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -807.1885986328125,
"loss": 1.1117,
"margin_dpo/margin_mean": 278.29156494140625,
"margin_dpo/margin_std": 445.8023681640625,
"step": 511
},
{
"KL/chosen_KL_mean": -349.73565673828125,
"KL/mean": -525.7312622070312,
"KL/rejected_KL_mean": -701.726806640625,
"KL/std": 337.34075927734375,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.0014879079535603523,
"fcm_dpo/delta": -0.07789607346057892,
"fcm_dpo/margin": 351.9912109375,
"fcm_dpo/q_t": 0.37877798080444336,
"grad_norm": 44.94166946411133,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.7359640002250671,
"logits/rejected": -0.7423201203346252,
"logps/chosen": -404.64117431640625,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -783.6027221679688,
"loss": 0.9899,
"margin_dpo/margin_mean": 351.9912109375,
"margin_dpo/margin_std": 344.17669677734375,
"step": 512
},
{
"KL/chosen_KL_mean": -427.519287109375,
"KL/mean": -548.4598388671875,
"KL/rejected_KL_mean": -669.4005126953125,
"KL/std": 306.87774658203125,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.0014939235989004374,
"fcm_dpo/delta": 0.09168398380279541,
"fcm_dpo/margin": 241.88121032714844,
"fcm_dpo/q_t": 0.4173031449317932,
"grad_norm": 37.183837890625,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.7703028917312622,
"logits/rejected": -0.7496137619018555,
"logps/chosen": -492.43572998046875,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -745.4629516601562,
"loss": 1.1291,
"margin_dpo/margin_mean": 241.88119506835938,
"margin_dpo/margin_std": 356.2818908691406,
"step": 513
},
{
"KL/chosen_KL_mean": -438.71282958984375,
"KL/mean": -566.1483154296875,
"KL/rejected_KL_mean": -693.583740234375,
"KL/std": 342.07403564453125,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0015267003327608109,
"fcm_dpo/delta": 0.06326548755168915,
"fcm_dpo/margin": 254.87098693847656,
"fcm_dpo/q_t": 0.41354596614837646,
"grad_norm": 29.647640228271484,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.7364473342895508,
"logits/rejected": -0.7220122814178467,
"logps/chosen": -512.9423828125,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -772.529296875,
"loss": 1.1138,
"margin_dpo/margin_mean": 254.87098693847656,
"margin_dpo/margin_std": 387.6467590332031,
"step": 514
},
{
"KL/chosen_KL_mean": -369.8049011230469,
"KL/mean": -478.7362060546875,
"KL/rejected_KL_mean": -587.6676025390625,
"KL/std": 298.692138671875,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0015549686504527926,
"fcm_dpo/delta": 0.11522873491048813,
"fcm_dpo/margin": 217.8626708984375,
"fcm_dpo/q_t": 0.4216243028640747,
"grad_norm": 44.44442367553711,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.716699481010437,
"logits/rejected": -0.7241976261138916,
"logps/chosen": -420.20648193359375,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -674.7653198242188,
"loss": 1.1426,
"margin_dpo/margin_mean": 217.8626708984375,
"margin_dpo/margin_std": 344.031494140625,
"step": 515
},
{
"KL/chosen_KL_mean": -394.34375,
"KL/mean": -553.999267578125,
"KL/rejected_KL_mean": -713.6547241210938,
"KL/std": 348.014892578125,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.001558106392621994,
"fcm_dpo/delta": -0.050014954060316086,
"fcm_dpo/margin": 319.31097412109375,
"fcm_dpo/q_t": 0.3847929537296295,
"grad_norm": 32.48160934448242,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.7692265510559082,
"logits/rejected": -0.7910833358764648,
"logps/chosen": -458.9933166503906,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -825.3770751953125,
"loss": 1.0142,
"margin_dpo/margin_mean": 319.31097412109375,
"margin_dpo/margin_std": 336.8834228515625,
"step": 516
},
{
"KL/chosen_KL_mean": -414.7637939453125,
"KL/mean": -587.2954711914062,
"KL/rejected_KL_mean": -759.8271484375,
"KL/std": 365.62969970703125,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0015280582010746002,
"fcm_dpo/delta": -0.08185283839702606,
"fcm_dpo/margin": 345.0633850097656,
"fcm_dpo/q_t": 0.3820088505744934,
"grad_norm": 34.728092193603516,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.7479615211486816,
"logits/rejected": -0.7408775091171265,
"logps/chosen": -475.6773681640625,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -848.9102783203125,
"loss": 1.0071,
"margin_dpo/margin_mean": 345.0633544921875,
"margin_dpo/margin_std": 395.7159423828125,
"step": 517
},
{
"KL/chosen_KL_mean": -405.2890625,
"KL/mean": -543.8110961914062,
"KL/rejected_KL_mean": -682.3330688476562,
"KL/std": 322.93963623046875,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.001526032341644168,
"fcm_dpo/delta": 0.02827056683599949,
"fcm_dpo/margin": 277.0439453125,
"fcm_dpo/q_t": 0.40260082483291626,
"grad_norm": 40.5467529296875,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.7425769567489624,
"logits/rejected": -0.752970814704895,
"logps/chosen": -462.7449951171875,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -767.645751953125,
"loss": 1.0789,
"margin_dpo/margin_mean": 277.0439453125,
"margin_dpo/margin_std": 359.08953857421875,
"step": 518
},
{
"KL/chosen_KL_mean": -415.0616760253906,
"KL/mean": -519.9083251953125,
"KL/rejected_KL_mean": -624.7550048828125,
"KL/std": 290.7900695800781,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0015673746820539236,
"fcm_dpo/delta": 0.1248587965965271,
"fcm_dpo/margin": 209.69326782226562,
"fcm_dpo/q_t": 0.42542368173599243,
"grad_norm": 40.930904388427734,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.7818655967712402,
"logits/rejected": -0.7639808058738708,
"logps/chosen": -489.125,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -729.1990966796875,
"loss": 1.1625,
"margin_dpo/margin_mean": 209.69326782226562,
"margin_dpo/margin_std": 365.93060302734375,
"step": 519
},
{
"KL/chosen_KL_mean": -439.8741455078125,
"KL/mean": -570.5068969726562,
"KL/rejected_KL_mean": -701.1396484375,
"KL/std": 334.30096435546875,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.0015773207414895296,
"fcm_dpo/delta": 0.03948332369327545,
"fcm_dpo/margin": 261.2655029296875,
"fcm_dpo/q_t": 0.4074580669403076,
"grad_norm": 35.82596206665039,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.6950489282608032,
"logits/rejected": -0.695152759552002,
"logps/chosen": -510.1739501953125,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -801.1209716796875,
"loss": 1.0935,
"margin_dpo/margin_mean": 261.2655029296875,
"margin_dpo/margin_std": 358.3149108886719,
"step": 520
},
{
"KL/chosen_KL_mean": -390.12164306640625,
"KL/mean": -535.8663330078125,
"KL/rejected_KL_mean": -681.611083984375,
"KL/std": 346.71759033203125,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.001592535525560379,
"fcm_dpo/delta": -0.015837617218494415,
"fcm_dpo/margin": 291.48944091796875,
"fcm_dpo/q_t": 0.3943830728530884,
"grad_norm": 36.59835433959961,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.7629398107528687,
"logits/rejected": -0.7609637379646301,
"logps/chosen": -448.2645568847656,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -764.8916625976562,
"loss": 1.0487,
"margin_dpo/margin_mean": 291.48944091796875,
"margin_dpo/margin_std": 345.49517822265625,
"step": 521
},
{
"KL/chosen_KL_mean": -454.544677734375,
"KL/mean": -588.1632080078125,
"KL/rejected_KL_mean": -721.7817993164062,
"KL/std": 362.4481201171875,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.0015862009022384882,
"fcm_dpo/delta": 0.02726486325263977,
"fcm_dpo/margin": 267.2371520996094,
"fcm_dpo/q_t": 0.4044070243835449,
"grad_norm": 43.29872131347656,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.805392324924469,
"logits/rejected": -0.808029055595398,
"logps/chosen": -501.09234619140625,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -787.795654296875,
"loss": 1.0949,
"margin_dpo/margin_mean": 267.2371520996094,
"margin_dpo/margin_std": 387.8713684082031,
"step": 522
},
{
"KL/chosen_KL_mean": -484.6162109375,
"KL/mean": -624.419677734375,
"KL/rejected_KL_mean": -764.2230834960938,
"KL/std": 422.138671875,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0015884996391832829,
"fcm_dpo/delta": 0.006105409003794193,
"fcm_dpo/margin": 279.60687255859375,
"fcm_dpo/q_t": 0.4030250906944275,
"grad_norm": 40.565879821777344,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.8060259819030762,
"logits/rejected": -0.8016799688339233,
"logps/chosen": -546.3858032226562,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -847.9844970703125,
"loss": 1.1083,
"margin_dpo/margin_mean": 279.6069030761719,
"margin_dpo/margin_std": 456.46990966796875,
"step": 523
},
{
"KL/chosen_KL_mean": -479.27288818359375,
"KL/mean": -626.4615478515625,
"KL/rejected_KL_mean": -773.6500854492188,
"KL/std": 402.63641357421875,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0015760126989334822,
"fcm_dpo/delta": -0.01566839963197708,
"fcm_dpo/margin": 294.37713623046875,
"fcm_dpo/q_t": 0.3938806653022766,
"grad_norm": 39.6019172668457,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.8418277502059937,
"logits/rejected": -0.8301990628242493,
"logps/chosen": -557.344970703125,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -854.9520874023438,
"loss": 1.0749,
"margin_dpo/margin_mean": 294.37713623046875,
"margin_dpo/margin_std": 403.175048828125,
"step": 524
},
{
"KL/chosen_KL_mean": -466.339111328125,
"KL/mean": -654.3277587890625,
"KL/rejected_KL_mean": -842.3164672851562,
"KL/std": 413.3827209472656,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.001546194078400731,
"fcm_dpo/delta": -0.14039839804172516,
"fcm_dpo/margin": 375.97735595703125,
"fcm_dpo/q_t": 0.371675968170166,
"grad_norm": 39.6491584777832,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.7878221273422241,
"logits/rejected": -0.8154387474060059,
"logps/chosen": -517.1669921875,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -942.369384765625,
"loss": 0.9844,
"margin_dpo/margin_mean": 375.97735595703125,
"margin_dpo/margin_std": 432.4591064453125,
"step": 525
},
{
"KL/chosen_KL_mean": -510.3265075683594,
"KL/mean": -695.115966796875,
"KL/rejected_KL_mean": -879.9052734375,
"KL/std": 410.5618896484375,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.001521292608231306,
"fcm_dpo/delta": -0.118798166513443,
"fcm_dpo/margin": 369.578857421875,
"fcm_dpo/q_t": 0.3733553886413574,
"grad_norm": 34.82008361816406,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.8415229320526123,
"logits/rejected": -0.8387963175773621,
"logps/chosen": -573.4937744140625,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -966.2146606445312,
"loss": 1.0076,
"margin_dpo/margin_mean": 369.5788269042969,
"margin_dpo/margin_std": 444.198486328125,
"step": 526
},
{
"KL/chosen_KL_mean": -529.979248046875,
"KL/mean": -658.5853271484375,
"KL/rejected_KL_mean": -787.1915283203125,
"KL/std": 419.4071044921875,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.00151540944352746,
"fcm_dpo/delta": 0.06268136203289032,
"fcm_dpo/margin": 257.2122802734375,
"fcm_dpo/q_t": 0.41041019558906555,
"grad_norm": 34.486732482910156,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -0.9882876873016357,
"logits/rejected": -0.9622901678085327,
"logps/chosen": -601.10791015625,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -865.5340576171875,
"loss": 1.1056,
"margin_dpo/margin_mean": 257.2122802734375,
"margin_dpo/margin_std": 361.535888671875,
"step": 527
},
{
"KL/chosen_KL_mean": -559.691162109375,
"KL/mean": -719.7154541015625,
"KL/rejected_KL_mean": -879.73974609375,
"KL/std": 414.49090576171875,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0015139649622142315,
"fcm_dpo/delta": -0.0363934151828289,
"fcm_dpo/margin": 320.04864501953125,
"fcm_dpo/q_t": 0.3962225615978241,
"grad_norm": 43.06204605102539,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -0.9302852153778076,
"logits/rejected": -0.9159493446350098,
"logps/chosen": -627.7805786132812,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -973.6497802734375,
"loss": 1.0788,
"margin_dpo/margin_mean": 320.04864501953125,
"margin_dpo/margin_std": 486.42547607421875,
"step": 528
},
{
"KL/chosen_KL_mean": -652.5494995117188,
"KL/mean": -759.435546875,
"KL/rejected_KL_mean": -866.321533203125,
"KL/std": 369.5655517578125,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.0015353154158219695,
"fcm_dpo/delta": 0.12608586251735687,
"fcm_dpo/margin": 213.7720947265625,
"fcm_dpo/q_t": 0.4263901710510254,
"grad_norm": 58.701969146728516,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -0.9356893301010132,
"logits/rejected": -0.9435217380523682,
"logps/chosen": -708.1244506835938,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -955.5306396484375,
"loss": 1.2075,
"margin_dpo/margin_mean": 213.7720947265625,
"margin_dpo/margin_std": 462.04559326171875,
"step": 529
},
{
"KL/chosen_KL_mean": -566.465576171875,
"KL/mean": -738.3973999023438,
"KL/rejected_KL_mean": -910.3292236328125,
"KL/std": 460.990478515625,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.0015337592922151089,
"fcm_dpo/delta": -0.08168676495552063,
"fcm_dpo/margin": 343.86358642578125,
"fcm_dpo/q_t": 0.3879829943180084,
"grad_norm": 61.309452056884766,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.8594100475311279,
"logits/rejected": -0.853233814239502,
"logps/chosen": -614.0670166015625,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -997.6137084960938,
"loss": 1.0842,
"margin_dpo/margin_mean": 343.86358642578125,
"margin_dpo/margin_std": 559.4108276367188,
"step": 530
},
{
"KL/chosen_KL_mean": -629.9609375,
"KL/mean": -774.3426513671875,
"KL/rejected_KL_mean": -918.724365234375,
"KL/std": 479.8919677734375,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.001531377900391817,
"fcm_dpo/delta": 0.008073193021118641,
"fcm_dpo/margin": 288.76336669921875,
"fcm_dpo/q_t": 0.40305599570274353,
"grad_norm": 46.5568733215332,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -0.9584941864013672,
"logits/rejected": -0.9381182193756104,
"logps/chosen": -685.20703125,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -989.330322265625,
"loss": 1.1007,
"margin_dpo/margin_mean": 288.7633972167969,
"margin_dpo/margin_std": 451.8643798828125,
"step": 531
},
{
"KL/chosen_KL_mean": -607.1270751953125,
"KL/mean": -774.4474487304688,
"KL/rejected_KL_mean": -941.767822265625,
"KL/std": 463.4101867675781,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0015030648792162538,
"fcm_dpo/delta": -0.05784127488732338,
"fcm_dpo/margin": 334.64068603515625,
"fcm_dpo/q_t": 0.39187368750572205,
"grad_norm": 56.40995788574219,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -0.9158481359481812,
"logits/rejected": -0.8925724029541016,
"logps/chosen": -677.4130859375,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1028.359130859375,
"loss": 1.0945,
"margin_dpo/margin_mean": 334.64068603515625,
"margin_dpo/margin_std": 533.584716796875,
"step": 532
},
{
"KL/chosen_KL_mean": -560.2667236328125,
"KL/mean": -678.7752685546875,
"KL/rejected_KL_mean": -797.2838134765625,
"KL/std": 397.15875244140625,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0015424289740622044,
"fcm_dpo/delta": 0.08588970452547073,
"fcm_dpo/margin": 237.0171661376953,
"fcm_dpo/q_t": 0.419960618019104,
"grad_norm": 48.91042709350586,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -0.9363778829574585,
"logits/rejected": -0.9149080514907837,
"logps/chosen": -625.121826171875,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -873.8719482421875,
"loss": 1.1801,
"margin_dpo/margin_mean": 237.01715087890625,
"margin_dpo/margin_std": 479.1162109375,
"step": 533
},
{
"KL/chosen_KL_mean": -579.787841796875,
"KL/mean": -736.6604614257812,
"KL/rejected_KL_mean": -893.5330200195312,
"KL/std": 427.12310791015625,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0015210118144750595,
"fcm_dpo/delta": -0.0296638160943985,
"fcm_dpo/margin": 313.74517822265625,
"fcm_dpo/q_t": 0.3943442106246948,
"grad_norm": 33.77534484863281,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -0.9673094749450684,
"logits/rejected": -0.9635472297668457,
"logps/chosen": -639.9072265625,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -972.0764770507812,
"loss": 1.0675,
"margin_dpo/margin_mean": 313.74517822265625,
"margin_dpo/margin_std": 436.4530334472656,
"step": 534
},
{
"KL/chosen_KL_mean": -489.6236267089844,
"KL/mean": -671.65185546875,
"KL/rejected_KL_mean": -853.6800537109375,
"KL/std": 420.79559326171875,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.0015158955939114094,
"fcm_dpo/delta": -0.10817711055278778,
"fcm_dpo/margin": 364.056396484375,
"fcm_dpo/q_t": 0.37864556908607483,
"grad_norm": 30.668663024902344,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.8600409030914307,
"logits/rejected": -0.855995774269104,
"logps/chosen": -543.953857421875,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -949.9876708984375,
"loss": 1.0123,
"margin_dpo/margin_mean": 364.056396484375,
"margin_dpo/margin_std": 439.95440673828125,
"step": 535
},
{
"KL/chosen_KL_mean": -434.6279296875,
"KL/mean": -647.1360473632812,
"KL/rejected_KL_mean": -859.6441650390625,
"KL/std": 475.992431640625,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.001455649035051465,
"fcm_dpo/delta": -0.17993277311325073,
"fcm_dpo/margin": 425.0162048339844,
"fcm_dpo/q_t": 0.3700369596481323,
"grad_norm": 36.75252914428711,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.8524526357650757,
"logits/rejected": -0.8777526021003723,
"logps/chosen": -481.7084655761719,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -948.7420043945312,
"loss": 0.9893,
"margin_dpo/margin_mean": 425.0162048339844,
"margin_dpo/margin_std": 565.5301513671875,
"step": 536
},
{
"KL/chosen_KL_mean": -446.46673583984375,
"KL/mean": -590.6380615234375,
"KL/rejected_KL_mean": -734.809326171875,
"KL/std": 377.60614013671875,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.001456779194995761,
"fcm_dpo/delta": 0.03063591942191124,
"fcm_dpo/margin": 288.34259033203125,
"fcm_dpo/q_t": 0.40409615635871887,
"grad_norm": 41.455623626708984,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.8700658679008484,
"logits/rejected": -0.845777153968811,
"logps/chosen": -504.2142028808594,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -805.2477416992188,
"loss": 1.1113,
"margin_dpo/margin_mean": 288.34259033203125,
"margin_dpo/margin_std": 448.64703369140625,
"step": 537
},
{
"KL/chosen_KL_mean": -551.7672119140625,
"KL/mean": -713.7427978515625,
"KL/rejected_KL_mean": -875.7183837890625,
"KL/std": 390.9005126953125,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.0014447167050093412,
"fcm_dpo/delta": -0.01904422789812088,
"fcm_dpo/margin": 323.95123291015625,
"fcm_dpo/q_t": 0.3952096700668335,
"grad_norm": 37.458518981933594,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.859600305557251,
"logits/rejected": -0.8362915515899658,
"logps/chosen": -618.18310546875,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -959.9464721679688,
"loss": 1.0501,
"margin_dpo/margin_mean": 323.95123291015625,
"margin_dpo/margin_std": 409.5311279296875,
"step": 538
},
{
"KL/chosen_KL_mean": -473.65972900390625,
"KL/mean": -622.8203735351562,
"KL/rejected_KL_mean": -771.98095703125,
"KL/std": 333.13140869140625,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.0014522131532430649,
"fcm_dpo/delta": 0.017473286017775536,
"fcm_dpo/margin": 298.3211975097656,
"fcm_dpo/q_t": 0.401156485080719,
"grad_norm": 32.28611373901367,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.8876945376396179,
"logits/rejected": -0.8707320690155029,
"logps/chosen": -532.152587890625,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -863.8348999023438,
"loss": 1.0662,
"margin_dpo/margin_mean": 298.3211975097656,
"margin_dpo/margin_std": 367.8041076660156,
"step": 539
},
{
"KL/chosen_KL_mean": -474.45867919921875,
"KL/mean": -679.921142578125,
"KL/rejected_KL_mean": -885.3836669921875,
"KL/std": 437.00653076171875,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.0014261383330449462,
"fcm_dpo/delta": -0.14425253868103027,
"fcm_dpo/margin": 410.92498779296875,
"fcm_dpo/q_t": 0.37306225299835205,
"grad_norm": 47.68659210205078,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.8324606418609619,
"logits/rejected": -0.8481217622756958,
"logps/chosen": -537.9412231445312,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1001.8136596679688,
"loss": 0.998,
"margin_dpo/margin_mean": 410.92498779296875,
"margin_dpo/margin_std": 496.6783447265625,
"step": 540
},
{
"KL/chosen_KL_mean": -559.8260498046875,
"KL/mean": -674.4676513671875,
"KL/rejected_KL_mean": -789.1092529296875,
"KL/std": 374.70294189453125,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0014067405136302114,
"fcm_dpo/delta": -0.012565853074193,
"fcm_dpo/margin": 229.2831573486328,
"fcm_dpo/q_t": 0.427273690700531,
"grad_norm": 42.75546646118164,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -0.937226414680481,
"logits/rejected": -0.9252729415893555,
"logps/chosen": -638.1063842773438,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -892.3829345703125,
"loss": 1.1859,
"margin_dpo/margin_mean": 229.28314208984375,
"margin_dpo/margin_std": 446.5557556152344,
"step": 541
},
{
"KL/chosen_KL_mean": -491.94915771484375,
"KL/mean": -692.276611328125,
"KL/rejected_KL_mean": -892.6041259765625,
"KL/std": 434.2323913574219,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.001373377162963152,
"fcm_dpo/delta": -0.10823685675859451,
"fcm_dpo/margin": 400.65496826171875,
"fcm_dpo/q_t": 0.37876492738723755,
"grad_norm": 37.0699462890625,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -0.9341820478439331,
"logits/rejected": -0.9597252011299133,
"logps/chosen": -549.4341430664062,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -989.0792236328125,
"loss": 1.0183,
"margin_dpo/margin_mean": 400.65496826171875,
"margin_dpo/margin_std": 501.817138671875,
"step": 542
},
{
"KL/chosen_KL_mean": -553.80517578125,
"KL/mean": -692.8021240234375,
"KL/rejected_KL_mean": -831.799072265625,
"KL/std": 508.77581787109375,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0013881053309887648,
"fcm_dpo/delta": 0.06674302369356155,
"fcm_dpo/margin": 277.9939880371094,
"fcm_dpo/q_t": 0.4248654842376709,
"grad_norm": 33.460391998291016,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.8752850890159607,
"logits/rejected": -0.8996802568435669,
"logps/chosen": -614.4226684570312,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -937.3980712890625,
"loss": 1.1719,
"margin_dpo/margin_mean": 277.9939880371094,
"margin_dpo/margin_std": 586.6079711914062,
"step": 543
},
{
"KL/chosen_KL_mean": -558.5087890625,
"KL/mean": -727.3214111328125,
"KL/rejected_KL_mean": -896.134033203125,
"KL/std": 419.15606689453125,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0013904159422963858,
"fcm_dpo/delta": -0.0204063281416893,
"fcm_dpo/margin": 337.62530517578125,
"fcm_dpo/q_t": 0.3948058485984802,
"grad_norm": 34.55818557739258,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.8828062415122986,
"logits/rejected": -0.8915610313415527,
"logps/chosen": -618.1510620117188,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -997.0887451171875,
"loss": 1.0625,
"margin_dpo/margin_mean": 337.62530517578125,
"margin_dpo/margin_std": 459.617431640625,
"step": 544
},
{
"KL/chosen_KL_mean": -547.4271850585938,
"KL/mean": -733.0721435546875,
"KL/rejected_KL_mean": -918.7172241210938,
"KL/std": 412.1142883300781,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0013767255004495382,
"fcm_dpo/delta": -0.0644376277923584,
"fcm_dpo/margin": 371.29010009765625,
"fcm_dpo/q_t": 0.38572588562965393,
"grad_norm": 41.80028533935547,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.8214889764785767,
"logits/rejected": -0.8422670960426331,
"logps/chosen": -615.0758056640625,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1014.625244140625,
"loss": 1.0307,
"margin_dpo/margin_mean": 371.29010009765625,
"margin_dpo/margin_std": 462.669189453125,
"step": 545
},
{
"KL/chosen_KL_mean": -489.4063720703125,
"KL/mean": -630.6460571289062,
"KL/rejected_KL_mean": -771.8857421875,
"KL/std": 367.344970703125,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0013837218284606934,
"fcm_dpo/delta": 0.06155392527580261,
"fcm_dpo/margin": 282.47943115234375,
"fcm_dpo/q_t": 0.40864139795303345,
"grad_norm": 30.46135711669922,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.8541392087936401,
"logits/rejected": -0.8365869522094727,
"logps/chosen": -540.150634765625,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -853.751953125,
"loss": 1.0977,
"margin_dpo/margin_mean": 282.4794006347656,
"margin_dpo/margin_std": 375.0701904296875,
"step": 546
},
{
"KL/chosen_KL_mean": -507.4529724121094,
"KL/mean": -669.9639282226562,
"KL/rejected_KL_mean": -832.4749145507812,
"KL/std": 411.0699462890625,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.001391751691699028,
"fcm_dpo/delta": -0.002595599740743637,
"fcm_dpo/margin": 325.02191162109375,
"fcm_dpo/q_t": 0.39831429719924927,
"grad_norm": 56.123722076416016,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.8213566541671753,
"logits/rejected": -0.8242645263671875,
"logps/chosen": -581.1406860351562,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -923.2362670898438,
"loss": 1.072,
"margin_dpo/margin_mean": 325.0219421386719,
"margin_dpo/margin_std": 445.09332275390625,
"step": 547
},
{
"KL/chosen_KL_mean": -518.0079345703125,
"KL/mean": -678.672119140625,
"KL/rejected_KL_mean": -839.3363037109375,
"KL/std": 424.4984130859375,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0013872598065063357,
"fcm_dpo/delta": 0.0044105639681220055,
"fcm_dpo/margin": 321.3283996582031,
"fcm_dpo/q_t": 0.4029526114463806,
"grad_norm": 34.89374542236328,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -0.8904018402099609,
"logits/rejected": -0.8851450681686401,
"logps/chosen": -583.2542724609375,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -933.4544067382812,
"loss": 1.076,
"margin_dpo/margin_mean": 321.3283996582031,
"margin_dpo/margin_std": 461.08441162109375,
"step": 548
},
{
"KL/chosen_KL_mean": -546.9786376953125,
"KL/mean": -668.66064453125,
"KL/rejected_KL_mean": -790.3426513671875,
"KL/std": 357.53765869140625,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0014170631766319275,
"fcm_dpo/delta": 0.10819605737924576,
"fcm_dpo/margin": 243.364013671875,
"fcm_dpo/q_t": 0.4238278865814209,
"grad_norm": 43.048282623291016,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -0.8407612442970276,
"logits/rejected": -0.8215111494064331,
"logps/chosen": -596.1910400390625,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -864.2529907226562,
"loss": 1.1562,
"margin_dpo/margin_mean": 243.364013671875,
"margin_dpo/margin_std": 427.2247619628906,
"step": 549
},
{
"KL/chosen_KL_mean": -551.3169555664062,
"KL/mean": -692.5491943359375,
"KL/rejected_KL_mean": -833.7813720703125,
"KL/std": 400.9772033691406,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0014286930672824383,
"fcm_dpo/delta": 0.04842350259423256,
"fcm_dpo/margin": 282.4644470214844,
"fcm_dpo/q_t": 0.4092787206172943,
"grad_norm": 44.95556640625,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.8258721828460693,
"logits/rejected": -0.8465061187744141,
"logps/chosen": -608.1239013671875,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -928.9072265625,
"loss": 1.1081,
"margin_dpo/margin_mean": 282.4644470214844,
"margin_dpo/margin_std": 430.794921875,
"step": 550
},
{
"KL/chosen_KL_mean": -480.2443542480469,
"KL/mean": -703.80859375,
"KL/rejected_KL_mean": -927.3729248046875,
"KL/std": 418.818115234375,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.001391774625517428,
"fcm_dpo/delta": -0.18421801924705505,
"fcm_dpo/margin": 447.1285095214844,
"fcm_dpo/q_t": 0.3581318259239197,
"grad_norm": 61.52654266357422,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.8109757304191589,
"logits/rejected": -0.8539941310882568,
"logps/chosen": -539.3507080078125,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1039.045654296875,
"loss": 0.9306,
"margin_dpo/margin_mean": 447.1285095214844,
"margin_dpo/margin_std": 412.7304382324219,
"step": 551
},
{
"KL/chosen_KL_mean": -472.6966857910156,
"KL/mean": -719.04345703125,
"KL/rejected_KL_mean": -965.3901977539062,
"KL/std": 499.03668212890625,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.0013249278999865055,
"fcm_dpo/delta": -0.2206534445285797,
"fcm_dpo/margin": 492.69354248046875,
"fcm_dpo/q_t": 0.35978394746780396,
"grad_norm": 48.82020568847656,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.8634598255157471,
"logits/rejected": -0.885054886341095,
"logps/chosen": -535.05126953125,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1069.952392578125,
"loss": 0.9524,
"margin_dpo/margin_mean": 492.69354248046875,
"margin_dpo/margin_std": 561.2657470703125,
"step": 552
},
{
"KL/chosen_KL_mean": -526.5339965820312,
"KL/mean": -661.3082275390625,
"KL/rejected_KL_mean": -796.0825805664062,
"KL/std": 403.27972412109375,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0013364567421376705,
"fcm_dpo/delta": 0.09321236610412598,
"fcm_dpo/margin": 269.548583984375,
"fcm_dpo/q_t": 0.42117840051651,
"grad_norm": 31.599390029907227,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.8489539623260498,
"logits/rejected": -0.8407363891601562,
"logps/chosen": -594.7928466796875,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -894.1797485351562,
"loss": 1.1387,
"margin_dpo/margin_mean": 269.548583984375,
"margin_dpo/margin_std": 446.8441162109375,
"step": 553
},
{
"KL/chosen_KL_mean": -540.2756958007812,
"KL/mean": -729.8994750976562,
"KL/rejected_KL_mean": -919.523193359375,
"KL/std": 443.01739501953125,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0013425007928162813,
"fcm_dpo/delta": -0.0627017393708229,
"fcm_dpo/margin": 379.24749755859375,
"fcm_dpo/q_t": 0.3905681073665619,
"grad_norm": 32.90158462524414,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.8220857381820679,
"logits/rejected": -0.7953811883926392,
"logps/chosen": -608.223388671875,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1009.305908203125,
"loss": 1.065,
"margin_dpo/margin_mean": 379.24749755859375,
"margin_dpo/margin_std": 553.5936889648438,
"step": 554
},
{
"KL/chosen_KL_mean": -558.227783203125,
"KL/mean": -759.487548828125,
"KL/rejected_KL_mean": -960.7473754882812,
"KL/std": 469.44622802734375,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.001314850407652557,
"fcm_dpo/delta": -0.0836186334490776,
"fcm_dpo/margin": 402.51959228515625,
"fcm_dpo/q_t": 0.38830822706222534,
"grad_norm": 29.105838775634766,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -0.8994369506835938,
"logits/rejected": -0.9377848505973816,
"logps/chosen": -611.5582885742188,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1069.226806640625,
"loss": 1.0599,
"margin_dpo/margin_mean": 402.5195617675781,
"margin_dpo/margin_std": 603.406494140625,
"step": 555
},
{
"KL/chosen_KL_mean": -527.7468872070312,
"KL/mean": -654.3409423828125,
"KL/rejected_KL_mean": -780.9349365234375,
"KL/std": 365.3113708496094,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.0013305556494742632,
"fcm_dpo/delta": 0.11709566414356232,
"fcm_dpo/margin": 253.18809509277344,
"fcm_dpo/q_t": 0.42365801334381104,
"grad_norm": 41.89741134643555,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -0.8416695594787598,
"logits/rejected": -0.8288569450378418,
"logps/chosen": -586.391357421875,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -882.2753295898438,
"loss": 1.1364,
"margin_dpo/margin_mean": 253.18807983398438,
"margin_dpo/margin_std": 379.06951904296875,
"step": 556
},
{
"KL/chosen_KL_mean": -560.56640625,
"KL/mean": -744.7011108398438,
"KL/rejected_KL_mean": -928.8358154296875,
"KL/std": 482.0714111328125,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.0013270850758999586,
"fcm_dpo/delta": -0.04098203778266907,
"fcm_dpo/margin": 368.2693176269531,
"fcm_dpo/q_t": 0.394328236579895,
"grad_norm": 52.759002685546875,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -0.8616045117378235,
"logits/rejected": -0.8892296552658081,
"logps/chosen": -628.4071044921875,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1038.775390625,
"loss": 1.078,
"margin_dpo/margin_mean": 368.26934814453125,
"margin_dpo/margin_std": 555.1060791015625,
"step": 557
},
{
"KL/chosen_KL_mean": -500.85626220703125,
"KL/mean": -749.5455932617188,
"KL/rejected_KL_mean": -998.2349853515625,
"KL/std": 460.37445068359375,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0012872398365288973,
"fcm_dpo/delta": -0.20348095893859863,
"fcm_dpo/margin": 497.3787536621094,
"fcm_dpo/q_t": 0.3545846939086914,
"grad_norm": 32.1386833190918,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -0.934282660484314,
"logits/rejected": -0.9463480114936829,
"logps/chosen": -563.2244873046875,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1100.39599609375,
"loss": 0.9448,
"margin_dpo/margin_mean": 497.3787536621094,
"margin_dpo/margin_std": 516.7579956054688,
"step": 558
},
{
"KL/chosen_KL_mean": -570.8775634765625,
"KL/mean": -772.5048828125,
"KL/rejected_KL_mean": -974.1322631835938,
"KL/std": 453.21221923828125,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.0012640644563362002,
"fcm_dpo/delta": -0.06291316449642181,
"fcm_dpo/margin": 403.254638671875,
"fcm_dpo/q_t": 0.38447457551956177,
"grad_norm": 36.51702880859375,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -0.9993324875831604,
"logits/rejected": -0.9968659281730652,
"logps/chosen": -631.6298828125,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1067.574462890625,
"loss": 1.0214,
"margin_dpo/margin_mean": 403.254638671875,
"margin_dpo/margin_std": 473.37640380859375,
"step": 559
},
{
"KL/chosen_KL_mean": -540.346923828125,
"KL/mean": -687.1452026367188,
"KL/rejected_KL_mean": -833.9434814453125,
"KL/std": 408.98394775390625,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.0012668755371123552,
"fcm_dpo/delta": 0.08083228766918182,
"fcm_dpo/margin": 293.5965881347656,
"fcm_dpo/q_t": 0.415485143661499,
"grad_norm": 34.44981002807617,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.8077666759490967,
"logits/rejected": -0.783920168876648,
"logps/chosen": -598.45068359375,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -913.9346923828125,
"loss": 1.1252,
"margin_dpo/margin_mean": 293.5965881347656,
"margin_dpo/margin_std": 440.9327392578125,
"step": 560
},
{
"KL/chosen_KL_mean": -604.890869140625,
"KL/mean": -737.5593872070312,
"KL/rejected_KL_mean": -870.2279052734375,
"KL/std": 406.80145263671875,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0013053957372903824,
"fcm_dpo/delta": 0.10657086968421936,
"fcm_dpo/margin": 265.33697509765625,
"fcm_dpo/q_t": 0.41968947649002075,
"grad_norm": 41.92377471923828,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -0.9422174692153931,
"logits/rejected": -0.9543702602386475,
"logps/chosen": -655.8037719726562,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -969.2964477539062,
"loss": 1.1646,
"margin_dpo/margin_mean": 265.33697509765625,
"margin_dpo/margin_std": 489.222412109375,
"step": 561
},
{
"KL/chosen_KL_mean": -577.7340698242188,
"KL/mean": -753.0718994140625,
"KL/rejected_KL_mean": -928.40966796875,
"KL/std": 466.2320556640625,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0012981008039787412,
"fcm_dpo/delta": -0.006240431219339371,
"fcm_dpo/margin": 350.67572021484375,
"fcm_dpo/q_t": 0.39807677268981934,
"grad_norm": 43.02619552612305,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -0.8793528079986572,
"logits/rejected": -0.8840149641036987,
"logps/chosen": -637.198486328125,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1024.952392578125,
"loss": 1.0778,
"margin_dpo/margin_mean": 350.6756896972656,
"margin_dpo/margin_std": 499.5946350097656,
"step": 562
},
{
"KL/chosen_KL_mean": -621.6233520507812,
"KL/mean": -756.9385986328125,
"KL/rejected_KL_mean": -892.2537841796875,
"KL/std": 477.40576171875,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.001299937255680561,
"fcm_dpo/delta": -0.04279060661792755,
"fcm_dpo/margin": 270.6304931640625,
"fcm_dpo/q_t": 0.41562989354133606,
"grad_norm": 61.1856575012207,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -0.8713696599006653,
"logits/rejected": -0.8709218502044678,
"logps/chosen": -682.2315673828125,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -986.821533203125,
"loss": 1.1986,
"margin_dpo/margin_mean": 270.6304931640625,
"margin_dpo/margin_std": 564.2909545898438,
"step": 563
},
{
"KL/chosen_KL_mean": -546.1058959960938,
"KL/mean": -717.114501953125,
"KL/rejected_KL_mean": -888.1231079101562,
"KL/std": 416.9074401855469,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0012988729868084192,
"fcm_dpo/delta": 0.0058783069252967834,
"fcm_dpo/margin": 342.0172119140625,
"fcm_dpo/q_t": 0.3986850678920746,
"grad_norm": 41.15483474731445,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -0.9513506889343262,
"logits/rejected": -0.9380670785903931,
"logps/chosen": -620.374267578125,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -981.3612670898438,
"loss": 1.0557,
"margin_dpo/margin_mean": 342.0171813964844,
"margin_dpo/margin_std": 402.26812744140625,
"step": 564
},
{
"KL/chosen_KL_mean": -569.7815551757812,
"KL/mean": -757.853271484375,
"KL/rejected_KL_mean": -945.9249267578125,
"KL/std": 480.99176025390625,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0012876607943326235,
"fcm_dpo/delta": -0.03614979237318039,
"fcm_dpo/margin": 376.14337158203125,
"fcm_dpo/q_t": 0.3959125280380249,
"grad_norm": 36.2006721496582,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -0.8705281615257263,
"logits/rejected": -0.8715401887893677,
"logps/chosen": -638.801513671875,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1031.703857421875,
"loss": 1.0783,
"margin_dpo/margin_mean": 376.14337158203125,
"margin_dpo/margin_std": 571.8416748046875,
"step": 565
},
{
"KL/chosen_KL_mean": -550.79443359375,
"KL/mean": -754.5787963867188,
"KL/rejected_KL_mean": -958.3630981445312,
"KL/std": 515.3678588867188,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0012800227850675583,
"fcm_dpo/delta": -0.07595265656709671,
"fcm_dpo/margin": 407.5686950683594,
"fcm_dpo/q_t": 0.3866634964942932,
"grad_norm": 34.75884246826172,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -0.8881973028182983,
"logits/rejected": -0.908134400844574,
"logps/chosen": -617.3397827148438,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1062.232421875,
"loss": 1.067,
"margin_dpo/margin_mean": 407.56866455078125,
"margin_dpo/margin_std": 599.103271484375,
"step": 566
},
{
"KL/chosen_KL_mean": -554.0453491210938,
"KL/mean": -714.4046630859375,
"KL/rejected_KL_mean": -874.7639770507812,
"KL/std": 369.9399108886719,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.001275490503758192,
"fcm_dpo/delta": 0.042663056403398514,
"fcm_dpo/margin": 320.71868896484375,
"fcm_dpo/q_t": 0.4055943787097931,
"grad_norm": 32.36269760131836,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -0.9792404174804688,
"logits/rejected": -0.988882303237915,
"logps/chosen": -606.9036254882812,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -960.1349487304688,
"loss": 1.0835,
"margin_dpo/margin_mean": 320.71868896484375,
"margin_dpo/margin_std": 416.3003845214844,
"step": 567
},
{
"KL/chosen_KL_mean": -536.5615234375,
"KL/mean": -743.4915771484375,
"KL/rejected_KL_mean": -950.421630859375,
"KL/std": 419.36846923828125,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.001262638601474464,
"fcm_dpo/delta": -0.07662282139062881,
"fcm_dpo/margin": 413.8601989746094,
"fcm_dpo/q_t": 0.3797609806060791,
"grad_norm": 38.35757064819336,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -0.8925222158432007,
"logits/rejected": -0.9186618328094482,
"logps/chosen": -581.75390625,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1039.5140380859375,
"loss": 1.0,
"margin_dpo/margin_mean": 413.86016845703125,
"margin_dpo/margin_std": 437.47808837890625,
"step": 568
},
{
"KL/chosen_KL_mean": -592.8074951171875,
"KL/mean": -766.2435913085938,
"KL/rejected_KL_mean": -939.6796875,
"KL/std": 510.0214538574219,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.001268592313863337,
"fcm_dpo/delta": 0.009497091174125671,
"fcm_dpo/margin": 346.8720703125,
"fcm_dpo/q_t": 0.40101489424705505,
"grad_norm": 50.98748016357422,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -0.9049087166786194,
"logits/rejected": -0.907079815864563,
"logps/chosen": -656.528076171875,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1018.782958984375,
"loss": 1.1008,
"margin_dpo/margin_mean": 346.8721008300781,
"margin_dpo/margin_std": 524.1836547851562,
"step": 569
},
{
"KL/chosen_KL_mean": -560.402099609375,
"KL/mean": -718.6865234375,
"KL/rejected_KL_mean": -876.970947265625,
"KL/std": 420.0425720214844,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0012691834708675742,
"fcm_dpo/delta": 0.05026433989405632,
"fcm_dpo/margin": 316.56884765625,
"fcm_dpo/q_t": 0.40913310647010803,
"grad_norm": 30.43377685546875,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -0.9766222238540649,
"logits/rejected": -0.968289852142334,
"logps/chosen": -622.0166625976562,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -959.11279296875,
"loss": 1.098,
"margin_dpo/margin_mean": 316.56884765625,
"margin_dpo/margin_std": 439.4469909667969,
"step": 570
},
{
"KL/chosen_KL_mean": -550.743896484375,
"KL/mean": -744.7813720703125,
"KL/rejected_KL_mean": -938.81884765625,
"KL/std": 436.6773986816406,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0012616091407835484,
"fcm_dpo/delta": -0.042000800371170044,
"fcm_dpo/margin": 388.07489013671875,
"fcm_dpo/q_t": 0.3914453387260437,
"grad_norm": 35.876590728759766,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -0.8716817498207092,
"logits/rejected": -0.8817984461784363,
"logps/chosen": -603.7979736328125,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1030.1556396484375,
"loss": 1.0597,
"margin_dpo/margin_mean": 388.0748596191406,
"margin_dpo/margin_std": 536.5525512695312,
"step": 571
},
{
"KL/chosen_KL_mean": -602.5777587890625,
"KL/mean": -799.7861938476562,
"KL/rejected_KL_mean": -996.99462890625,
"KL/std": 500.5901184082031,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.0012506109196692705,
"fcm_dpo/delta": -0.045844513922929764,
"fcm_dpo/margin": 394.4167785644531,
"fcm_dpo/q_t": 0.3941086232662201,
"grad_norm": 31.384313583374023,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -0.9135201573371887,
"logits/rejected": -0.9341009259223938,
"logps/chosen": -651.037109375,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1080.5516357421875,
"loss": 1.074,
"margin_dpo/margin_mean": 394.416748046875,
"margin_dpo/margin_std": 598.5731811523438,
"step": 572
},
{
"KL/chosen_KL_mean": -587.642578125,
"KL/mean": -758.319580078125,
"KL/rejected_KL_mean": -928.9966430664062,
"KL/std": 452.03662109375,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0012581718619912863,
"fcm_dpo/delta": 0.02130233310163021,
"fcm_dpo/margin": 341.35406494140625,
"fcm_dpo/q_t": 0.4040926694869995,
"grad_norm": 29.551095962524414,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -0.9765233993530273,
"logits/rejected": -0.9776440858840942,
"logps/chosen": -639.26513671875,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1014.3216552734375,
"loss": 1.0751,
"margin_dpo/margin_mean": 341.35406494140625,
"margin_dpo/margin_std": 455.6972961425781,
"step": 573
},
{
"KL/chosen_KL_mean": -532.7286376953125,
"KL/mean": -718.6502685546875,
"KL/rejected_KL_mean": -904.57177734375,
"KL/std": 414.60821533203125,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0012568001402541995,
"fcm_dpo/delta": -0.018277421593666077,
"fcm_dpo/margin": 371.84307861328125,
"fcm_dpo/q_t": 0.3949261009693146,
"grad_norm": 34.49797058105469,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -0.9791123867034912,
"logits/rejected": -1.0038515329360962,
"logps/chosen": -583.7731323242188,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -997.378173828125,
"loss": 1.0554,
"margin_dpo/margin_mean": 371.84307861328125,
"margin_dpo/margin_std": 477.19677734375,
"step": 574
},
{
"KL/chosen_KL_mean": -602.52099609375,
"KL/mean": -776.2808837890625,
"KL/rejected_KL_mean": -950.040771484375,
"KL/std": 425.5224304199219,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.001243308070115745,
"fcm_dpo/delta": 0.01694381982088089,
"fcm_dpo/margin": 347.519775390625,
"fcm_dpo/q_t": 0.4027356505393982,
"grad_norm": 36.27347183227539,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.8385648727416992,
"logits/rejected": -0.8408699631690979,
"logps/chosen": -674.3111572265625,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1045.427001953125,
"loss": 1.1118,
"margin_dpo/margin_mean": 347.519775390625,
"margin_dpo/margin_std": 533.6824951171875,
"step": 575
},
{
"KL/chosen_KL_mean": -582.2991943359375,
"KL/mean": -714.9830322265625,
"KL/rejected_KL_mean": -847.6668090820312,
"KL/std": 401.23394775390625,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.001272881869226694,
"fcm_dpo/delta": 0.1162538081407547,
"fcm_dpo/margin": 265.3677062988281,
"fcm_dpo/q_t": 0.4249209761619568,
"grad_norm": 36.19147491455078,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -0.9529926776885986,
"logits/rejected": -0.9747903347015381,
"logps/chosen": -636.5621337890625,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -948.4210815429688,
"loss": 1.1681,
"margin_dpo/margin_mean": 265.3677062988281,
"margin_dpo/margin_std": 483.37457275390625,
"step": 576
},
{
"KL/chosen_KL_mean": -515.85498046875,
"KL/mean": -682.475830078125,
"KL/rejected_KL_mean": -849.0966186523438,
"KL/std": 433.720947265625,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.001288012252189219,
"fcm_dpo/delta": 0.021687885746359825,
"fcm_dpo/margin": 333.24163818359375,
"fcm_dpo/q_t": 0.40454742312431335,
"grad_norm": 30.929237365722656,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -0.9174989461898804,
"logits/rejected": -0.9000959396362305,
"logps/chosen": -577.7646484375,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -933.1673583984375,
"loss": 1.0953,
"margin_dpo/margin_mean": 333.24163818359375,
"margin_dpo/margin_std": 466.4169006347656,
"step": 577
},
{
"KL/chosen_KL_mean": -510.0753173828125,
"KL/mean": -700.24609375,
"KL/rejected_KL_mean": -890.4168701171875,
"KL/std": 436.525146484375,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.0012763895792886615,
"fcm_dpo/delta": -0.038308389484882355,
"fcm_dpo/margin": 380.341552734375,
"fcm_dpo/q_t": 0.39006108045578003,
"grad_norm": 34.35224914550781,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.8245141506195068,
"logits/rejected": -0.8499029874801636,
"logps/chosen": -559.3389892578125,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -973.8531494140625,
"loss": 1.0352,
"margin_dpo/margin_mean": 380.341552734375,
"margin_dpo/margin_std": 445.07977294921875,
"step": 578
},
{
"KL/chosen_KL_mean": -582.0475463867188,
"KL/mean": -706.8804321289062,
"KL/rejected_KL_mean": -831.7132568359375,
"KL/std": 444.97283935546875,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.001303818542510271,
"fcm_dpo/delta": 0.12885679304599762,
"fcm_dpo/margin": 249.66574096679688,
"fcm_dpo/q_t": 0.4256833791732788,
"grad_norm": 54.624481201171875,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.8717827796936035,
"logits/rejected": -0.8690820932388306,
"logps/chosen": -640.933349609375,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -926.5009155273438,
"loss": 1.178,
"margin_dpo/margin_mean": 249.66574096679688,
"margin_dpo/margin_std": 475.2186279296875,
"step": 579
},
{
"KL/chosen_KL_mean": -424.3729248046875,
"KL/mean": -635.8189697265625,
"KL/rejected_KL_mean": -847.26513671875,
"KL/std": 469.45257568359375,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0012895048130303621,
"fcm_dpo/delta": -0.10174834728240967,
"fcm_dpo/margin": 422.89215087890625,
"fcm_dpo/q_t": 0.3806644380092621,
"grad_norm": 30.684194564819336,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.8051278591156006,
"logits/rejected": -0.8295614123344421,
"logps/chosen": -473.07977294921875,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -929.0235595703125,
"loss": 1.0126,
"margin_dpo/margin_mean": 422.89215087890625,
"margin_dpo/margin_std": 529.8594360351562,
"step": 580
},
{
"KL/chosen_KL_mean": -568.9508056640625,
"KL/mean": -720.18798828125,
"KL/rejected_KL_mean": -871.4251708984375,
"KL/std": 439.51373291015625,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0013018709141761065,
"fcm_dpo/delta": 0.058401986956596375,
"fcm_dpo/margin": 302.474365234375,
"fcm_dpo/q_t": 0.41300997138023376,
"grad_norm": 31.58887481689453,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -0.9429810047149658,
"logits/rejected": -0.9472246170043945,
"logps/chosen": -631.173095703125,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -963.1608276367188,
"loss": 1.1176,
"margin_dpo/margin_mean": 302.474365234375,
"margin_dpo/margin_std": 481.599853515625,
"step": 581
},
{
"KL/chosen_KL_mean": -494.30291748046875,
"KL/mean": -672.12841796875,
"KL/rejected_KL_mean": -849.9539184570312,
"KL/std": 394.76776123046875,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.001313531189225614,
"fcm_dpo/delta": -0.0199383944272995,
"fcm_dpo/margin": 355.65106201171875,
"fcm_dpo/q_t": 0.39279705286026,
"grad_norm": 40.20054244995117,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -0.8778091669082642,
"logits/rejected": -0.8947253227233887,
"logps/chosen": -552.5315551757812,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -960.0234985351562,
"loss": 1.0566,
"margin_dpo/margin_mean": 355.65106201171875,
"margin_dpo/margin_std": 427.9756164550781,
"step": 582
},
{
"KL/chosen_KL_mean": -576.75927734375,
"KL/mean": -685.605224609375,
"KL/rejected_KL_mean": -794.4511108398438,
"KL/std": 418.72412109375,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0012986322399228811,
"fcm_dpo/delta": -0.0004855971201322973,
"fcm_dpo/margin": 217.69180297851562,
"fcm_dpo/q_t": 0.43750786781311035,
"grad_norm": 56.99325180053711,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.8520580530166626,
"logits/rejected": -0.8441060781478882,
"logps/chosen": -633.6221923828125,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -882.8550415039062,
"loss": 1.2263,
"margin_dpo/margin_mean": 217.69180297851562,
"margin_dpo/margin_std": 497.66668701171875,
"step": 583
},
{
"KL/chosen_KL_mean": -477.69525146484375,
"KL/mean": -696.723876953125,
"KL/rejected_KL_mean": -915.7525634765625,
"KL/std": 448.5584716796875,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0012766908621415496,
"fcm_dpo/delta": -0.11572134494781494,
"fcm_dpo/margin": 438.05731201171875,
"fcm_dpo/q_t": 0.3751903176307678,
"grad_norm": 29.102798461914062,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -0.8786138296127319,
"logits/rejected": -0.8900209665298462,
"logps/chosen": -534.595947265625,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1013.3885498046875,
"loss": 0.9859,
"margin_dpo/margin_mean": 438.05731201171875,
"margin_dpo/margin_std": 475.702880859375,
"step": 584
},
{
"KL/chosen_KL_mean": -591.5244140625,
"KL/mean": -777.2154541015625,
"KL/rejected_KL_mean": -962.9066162109375,
"KL/std": 488.06884765625,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0012623495422303677,
"fcm_dpo/delta": -0.019886359572410583,
"fcm_dpo/margin": 371.38226318359375,
"fcm_dpo/q_t": 0.39668798446655273,
"grad_norm": 32.74428176879883,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -0.936137318611145,
"logits/rejected": -0.9553546905517578,
"logps/chosen": -652.4983520507812,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1047.076171875,
"loss": 1.0889,
"margin_dpo/margin_mean": 371.38226318359375,
"margin_dpo/margin_std": 573.6552734375,
"step": 585
},
{
"KL/chosen_KL_mean": -594.9310302734375,
"KL/mean": -728.787841796875,
"KL/rejected_KL_mean": -862.6447143554688,
"KL/std": 421.7008056640625,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0012830430641770363,
"fcm_dpo/delta": 0.11041760444641113,
"fcm_dpo/margin": 267.71368408203125,
"fcm_dpo/q_t": 0.4225107431411743,
"grad_norm": 33.24190902709961,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -0.9146158695220947,
"logits/rejected": -0.9288753271102905,
"logps/chosen": -651.78662109375,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -954.4473266601562,
"loss": 1.1626,
"margin_dpo/margin_mean": 267.71368408203125,
"margin_dpo/margin_std": 484.7624816894531,
"step": 586
},
{
"KL/chosen_KL_mean": -430.05645751953125,
"KL/mean": -608.1459350585938,
"KL/rejected_KL_mean": -786.2353515625,
"KL/std": 451.8103332519531,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0012925666524097323,
"fcm_dpo/delta": -0.010944999754428864,
"fcm_dpo/margin": 356.178955078125,
"fcm_dpo/q_t": 0.3954969048500061,
"grad_norm": 31.56755256652832,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -0.9272117614746094,
"logits/rejected": -0.9497717022895813,
"logps/chosen": -474.748046875,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -868.8592529296875,
"loss": 1.0577,
"margin_dpo/margin_mean": 356.178955078125,
"margin_dpo/margin_std": 455.32611083984375,
"step": 587
},
{
"KL/chosen_KL_mean": -548.4373779296875,
"KL/mean": -706.7396240234375,
"KL/rejected_KL_mean": -865.0418701171875,
"KL/std": 395.88555908203125,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0012917927233502269,
"fcm_dpo/delta": 0.04255141317844391,
"fcm_dpo/margin": 316.6044616699219,
"fcm_dpo/q_t": 0.40650418400764465,
"grad_norm": 28.8896427154541,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -0.9216235876083374,
"logits/rejected": -0.9430979490280151,
"logps/chosen": -598.7322998046875,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -972.4117431640625,
"loss": 1.0905,
"margin_dpo/margin_mean": 316.6044921875,
"margin_dpo/margin_std": 424.40228271484375,
"step": 588
},
{
"KL/chosen_KL_mean": -545.638671875,
"KL/mean": -745.33154296875,
"KL/rejected_KL_mean": -945.0244140625,
"KL/std": 435.88623046875,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0012890798971056938,
"fcm_dpo/delta": -0.06830345839262009,
"fcm_dpo/margin": 399.3857116699219,
"fcm_dpo/q_t": 0.3833511471748352,
"grad_norm": 31.347322463989258,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -0.8932241201400757,
"logits/rejected": -0.8985201120376587,
"logps/chosen": -605.568603515625,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1056.6796875,
"loss": 1.0145,
"margin_dpo/margin_mean": 399.3857421875,
"margin_dpo/margin_std": 455.48895263671875,
"step": 589
},
{
"KL/chosen_KL_mean": -489.78375244140625,
"KL/mean": -678.56396484375,
"KL/rejected_KL_mean": -867.3441772460938,
"KL/std": 415.4945983886719,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.0012754870112985373,
"fcm_dpo/delta": -0.03320199251174927,
"fcm_dpo/margin": 377.56036376953125,
"fcm_dpo/q_t": 0.3908342719078064,
"grad_norm": 33.08230972290039,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -0.8838923573493958,
"logits/rejected": -0.8997941017150879,
"logps/chosen": -545.5935668945312,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -973.406982421875,
"loss": 1.0352,
"margin_dpo/margin_mean": 377.5603332519531,
"margin_dpo/margin_std": 445.62188720703125,
"step": 590
},
{
"KL/chosen_KL_mean": -525.9390869140625,
"KL/mean": -702.4129638671875,
"KL/rejected_KL_mean": -878.88671875,
"KL/std": 408.47216796875,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.001272842986509204,
"fcm_dpo/delta": 0.0007620565593242645,
"fcm_dpo/margin": 352.9476623535156,
"fcm_dpo/q_t": 0.39712613821029663,
"grad_norm": 35.617069244384766,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -0.9034570455551147,
"logits/rejected": -0.8874039053916931,
"logps/chosen": -582.1796875,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -962.673095703125,
"loss": 1.0647,
"margin_dpo/margin_mean": 352.9476623535156,
"margin_dpo/margin_std": 454.56365966796875,
"step": 591
},
{
"KL/chosen_KL_mean": -567.338623046875,
"KL/mean": -723.809814453125,
"KL/rejected_KL_mean": -880.2809448242188,
"KL/std": 425.2896423339844,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0012821652926504612,
"fcm_dpo/delta": 0.050824809819459915,
"fcm_dpo/margin": 312.9423522949219,
"fcm_dpo/q_t": 0.41050198674201965,
"grad_norm": 32.38739776611328,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -0.9035841226577759,
"logits/rejected": -0.8997513055801392,
"logps/chosen": -615.2788696289062,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -956.0137939453125,
"loss": 1.1585,
"margin_dpo/margin_mean": 312.94232177734375,
"margin_dpo/margin_std": 582.2163696289062,
"step": 592
},
{
"KL/chosen_KL_mean": -525.773681640625,
"KL/mean": -653.7761840820312,
"KL/rejected_KL_mean": -781.7787475585938,
"KL/std": 448.0130310058594,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0013066052924841642,
"fcm_dpo/delta": 0.1196751743555069,
"fcm_dpo/margin": 256.0050964355469,
"fcm_dpo/q_t": 0.4265802502632141,
"grad_norm": 46.937931060791016,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.8258799314498901,
"logits/rejected": -0.8155261278152466,
"logps/chosen": -574.4644165039062,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -842.686767578125,
"loss": 1.1792,
"margin_dpo/margin_mean": 256.0050964355469,
"margin_dpo/margin_std": 498.3818359375,
"step": 593
},
{
"KL/chosen_KL_mean": -498.7892761230469,
"KL/mean": -662.3046264648438,
"KL/rejected_KL_mean": -825.8199462890625,
"KL/std": 448.3605651855469,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0013319381978362799,
"fcm_dpo/delta": 0.013642996549606323,
"fcm_dpo/margin": 327.03070068359375,
"fcm_dpo/q_t": 0.4028928875923157,
"grad_norm": 40.40144729614258,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -0.8832203149795532,
"logits/rejected": -0.8690311312675476,
"logps/chosen": -553.72412109375,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -911.919677734375,
"loss": 1.0955,
"margin_dpo/margin_mean": 327.03070068359375,
"margin_dpo/margin_std": 473.59136962890625,
"step": 594
},
{
"KL/chosen_KL_mean": -497.3454895019531,
"KL/mean": -668.4541015625,
"KL/rejected_KL_mean": -839.5626831054688,
"KL/std": 416.5923156738281,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0013188087614253163,
"fcm_dpo/delta": -0.001495562493801117,
"fcm_dpo/margin": 342.2171630859375,
"fcm_dpo/q_t": 0.40133988857269287,
"grad_norm": 41.03437042236328,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -0.8576099872589111,
"logits/rejected": -0.8629014492034912,
"logps/chosen": -546.765869140625,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -920.1900024414062,
"loss": 1.0676,
"margin_dpo/margin_mean": 342.2171936035156,
"margin_dpo/margin_std": 466.0132751464844,
"step": 595
},
{
"KL/chosen_KL_mean": -563.5756225585938,
"KL/mean": -703.8986206054688,
"KL/rejected_KL_mean": -844.2215576171875,
"KL/std": 430.84686279296875,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.001339010545052588,
"fcm_dpo/delta": 0.07695081830024719,
"fcm_dpo/margin": 280.64593505859375,
"fcm_dpo/q_t": 0.4185601472854614,
"grad_norm": 49.19902801513672,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -0.8751660585403442,
"logits/rejected": -0.8460420370101929,
"logps/chosen": -623.3673095703125,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -924.3126831054688,
"loss": 1.1472,
"margin_dpo/margin_mean": 280.6459045410156,
"margin_dpo/margin_std": 503.51751708984375,
"step": 596
},
{
"KL/chosen_KL_mean": -540.3389282226562,
"KL/mean": -752.1220703125,
"KL/rejected_KL_mean": -963.9052124023438,
"KL/std": 479.4707336425781,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.0013066258979961276,
"fcm_dpo/delta": -0.11277859658002853,
"fcm_dpo/margin": 423.5662536621094,
"fcm_dpo/q_t": 0.3791978359222412,
"grad_norm": 32.8747673034668,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.8611253499984741,
"logits/rejected": -0.8989757299423218,
"logps/chosen": -597.5997314453125,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1064.598876953125,
"loss": 1.0194,
"margin_dpo/margin_mean": 423.56622314453125,
"margin_dpo/margin_std": 527.2634887695312,
"step": 597
},
{
"KL/chosen_KL_mean": -550.748046875,
"KL/mean": -720.5945434570312,
"KL/rejected_KL_mean": -890.441162109375,
"KL/std": 418.1409912109375,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0013012320268899202,
"fcm_dpo/delta": 0.0065561458468437195,
"fcm_dpo/margin": 339.69317626953125,
"fcm_dpo/q_t": 0.39901459217071533,
"grad_norm": 52.765960693359375,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -0.9263842701911926,
"logits/rejected": -0.9303134679794312,
"logps/chosen": -603.2664794921875,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -979.885009765625,
"loss": 1.0824,
"margin_dpo/margin_mean": 339.69317626953125,
"margin_dpo/margin_std": 458.0146179199219,
"step": 598
},
{
"KL/chosen_KL_mean": -541.9983520507812,
"KL/mean": -689.47216796875,
"KL/rejected_KL_mean": -836.946044921875,
"KL/std": 402.40704345703125,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0013150456361472607,
"fcm_dpo/delta": 0.06361641734838486,
"fcm_dpo/margin": 294.94769287109375,
"fcm_dpo/q_t": 0.4082716703414917,
"grad_norm": 37.7735595703125,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.8490918874740601,
"logits/rejected": -0.8530697822570801,
"logps/chosen": -591.801025390625,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -919.924560546875,
"loss": 1.1011,
"margin_dpo/margin_mean": 294.94769287109375,
"margin_dpo/margin_std": 372.10333251953125,
"step": 599
},
{
"KL/chosen_KL_mean": -582.5357666015625,
"KL/mean": -719.8656616210938,
"KL/rejected_KL_mean": -857.1956787109375,
"KL/std": 403.26904296875,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0013454308500513434,
"fcm_dpo/delta": 0.08363349735736847,
"fcm_dpo/margin": 274.6598815917969,
"fcm_dpo/q_t": 0.417849063873291,
"grad_norm": 35.837806701660156,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -0.9528838396072388,
"logits/rejected": -0.9467679262161255,
"logps/chosen": -648.9705810546875,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -942.652099609375,
"loss": 1.1447,
"margin_dpo/margin_mean": 274.6598815917969,
"margin_dpo/margin_std": 474.7832946777344,
"step": 600
},
{
"KL/chosen_KL_mean": -604.0224609375,
"KL/mean": -778.8104858398438,
"KL/rejected_KL_mean": -953.5985717773438,
"KL/std": 454.6282958984375,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.001345046446658671,
"fcm_dpo/delta": -0.021434810012578964,
"fcm_dpo/margin": 349.5760803222656,
"fcm_dpo/q_t": 0.39219212532043457,
"grad_norm": 34.02439880371094,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -0.9269833564758301,
"logits/rejected": -0.9384532570838928,
"logps/chosen": -663.1560668945312,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1048.28955078125,
"loss": 1.0524,
"margin_dpo/margin_mean": 349.57611083984375,
"margin_dpo/margin_std": 442.3649597167969,
"step": 601
},
{
"KL/chosen_KL_mean": -419.7701416015625,
"KL/mean": -627.7533569335938,
"KL/rejected_KL_mean": -835.736572265625,
"KL/std": 411.73822021484375,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0013339366996660829,
"fcm_dpo/delta": -0.11114930361509323,
"fcm_dpo/margin": 415.96649169921875,
"fcm_dpo/q_t": 0.3747555613517761,
"grad_norm": 50.596588134765625,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -0.9215075969696045,
"logits/rejected": -0.944821834564209,
"logps/chosen": -468.3636474609375,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -923.4051513671875,
"loss": 0.9849,
"margin_dpo/margin_mean": 415.96649169921875,
"margin_dpo/margin_std": 435.55535888671875,
"step": 602
},
{
"KL/chosen_KL_mean": -496.24456787109375,
"KL/mean": -692.5504150390625,
"KL/rejected_KL_mean": -888.8563232421875,
"KL/std": 452.21258544921875,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0013055261224508286,
"fcm_dpo/delta": -0.0659245103597641,
"fcm_dpo/margin": 392.6116943359375,
"fcm_dpo/q_t": 0.3891829252243042,
"grad_norm": 44.00050354003906,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -0.9159995317459106,
"logits/rejected": -0.9068554639816284,
"logps/chosen": -566.6591796875,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -989.181884765625,
"loss": 1.0493,
"margin_dpo/margin_mean": 392.6116943359375,
"margin_dpo/margin_std": 547.1536865234375,
"step": 603
},
{
"KL/chosen_KL_mean": -512.2185668945312,
"KL/mean": -721.4580078125,
"KL/rejected_KL_mean": -930.697509765625,
"KL/std": 442.514404296875,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0012818844988942146,
"fcm_dpo/delta": -0.09144748747348785,
"fcm_dpo/margin": 418.4788818359375,
"fcm_dpo/q_t": 0.38152605295181274,
"grad_norm": 31.77489471435547,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -0.8875434398651123,
"logits/rejected": -0.9096465110778809,
"logps/chosen": -558.6766357421875,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1022.552001953125,
"loss": 1.021,
"margin_dpo/margin_mean": 418.4788818359375,
"margin_dpo/margin_std": 527.2413940429688,
"step": 604
},
{
"KL/chosen_KL_mean": -536.88916015625,
"KL/mean": -706.5093994140625,
"KL/rejected_KL_mean": -876.129638671875,
"KL/std": 427.982666015625,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0012768646702170372,
"fcm_dpo/delta": 0.017488297075033188,
"fcm_dpo/margin": 339.24053955078125,
"fcm_dpo/q_t": 0.404773086309433,
"grad_norm": 37.503013610839844,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -0.8663319945335388,
"logits/rejected": -0.8798903226852417,
"logps/chosen": -603.1384887695312,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -978.4346313476562,
"loss": 1.0879,
"margin_dpo/margin_mean": 339.24053955078125,
"margin_dpo/margin_std": 500.1103515625,
"step": 605
},
{
"KL/chosen_KL_mean": -531.7333984375,
"KL/mean": -710.1092529296875,
"KL/rejected_KL_mean": -888.4852294921875,
"KL/std": 433.4575500488281,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0012862420408055186,
"fcm_dpo/delta": -0.009775873273611069,
"fcm_dpo/margin": 356.7518005371094,
"fcm_dpo/q_t": 0.39728018641471863,
"grad_norm": 26.138517379760742,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -0.8830243349075317,
"logits/rejected": -0.8961066603660583,
"logps/chosen": -586.552490234375,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -986.856689453125,
"loss": 1.0611,
"margin_dpo/margin_mean": 356.7518005371094,
"margin_dpo/margin_std": 462.89691162109375,
"step": 606
},
{
"KL/chosen_KL_mean": -550.5501708984375,
"KL/mean": -696.69384765625,
"KL/rejected_KL_mean": -842.837646484375,
"KL/std": 417.550537109375,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0012980522587895393,
"fcm_dpo/delta": 0.07278678566217422,
"fcm_dpo/margin": 292.28741455078125,
"fcm_dpo/q_t": 0.4148247241973877,
"grad_norm": 28.818361282348633,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.8506742715835571,
"logits/rejected": -0.8361988067626953,
"logps/chosen": -608.6342163085938,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -922.61474609375,
"loss": 1.1233,
"margin_dpo/margin_mean": 292.2874450683594,
"margin_dpo/margin_std": 453.874755859375,
"step": 607
},
{
"KL/chosen_KL_mean": -506.8470153808594,
"KL/mean": -678.7813720703125,
"KL/rejected_KL_mean": -850.7156982421875,
"KL/std": 392.28564453125,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.0013015069998800755,
"fcm_dpo/delta": 0.0024357233196496964,
"fcm_dpo/margin": 343.86871337890625,
"fcm_dpo/q_t": 0.39717209339141846,
"grad_norm": 34.781368255615234,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -0.9160112142562866,
"logits/rejected": -0.9214470386505127,
"logps/chosen": -564.2978515625,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -945.4890747070312,
"loss": 1.0498,
"margin_dpo/margin_mean": 343.8686828613281,
"margin_dpo/margin_std": 397.05908203125,
"step": 608
},
{
"KL/chosen_KL_mean": -532.5485229492188,
"KL/mean": -741.901611328125,
"KL/rejected_KL_mean": -951.254638671875,
"KL/std": 516.5100708007812,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.0012833920773118734,
"fcm_dpo/delta": -0.09222295880317688,
"fcm_dpo/margin": 418.7061462402344,
"fcm_dpo/q_t": 0.3859896659851074,
"grad_norm": 31.00870704650879,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -0.9779306650161743,
"logits/rejected": -0.9924079179763794,
"logps/chosen": -591.3538818359375,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1040.0706787109375,
"loss": 1.0368,
"margin_dpo/margin_mean": 418.70611572265625,
"margin_dpo/margin_std": 598.0675048828125,
"step": 609
},
{
"KL/chosen_KL_mean": -518.366455078125,
"KL/mean": -659.4246826171875,
"KL/rejected_KL_mean": -800.4828491210938,
"KL/std": 397.7653503417969,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0012927600182592869,
"fcm_dpo/delta": 0.08842732757329941,
"fcm_dpo/margin": 282.1164245605469,
"fcm_dpo/q_t": 0.4162673056125641,
"grad_norm": 38.39983367919922,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.8724183440208435,
"logits/rejected": -0.8451335430145264,
"logps/chosen": -584.0615234375,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -883.8882446289062,
"loss": 1.1323,
"margin_dpo/margin_mean": 282.11639404296875,
"margin_dpo/margin_std": 448.6925048828125,
"step": 610
},
{
"KL/chosen_KL_mean": -544.4277954101562,
"KL/mean": -774.05908203125,
"KL/rejected_KL_mean": -1003.6903076171875,
"KL/std": 526.8722534179688,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0012711257440969348,
"fcm_dpo/delta": -0.1420023888349533,
"fcm_dpo/margin": 459.2625732421875,
"fcm_dpo/q_t": 0.3745759129524231,
"grad_norm": 33.196842193603516,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -0.934251070022583,
"logits/rejected": -0.9714481830596924,
"logps/chosen": -597.0272216796875,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1090.021240234375,
"loss": 1.0071,
"margin_dpo/margin_mean": 459.2625732421875,
"margin_dpo/margin_std": 592.6350708007812,
"step": 611
},
{
"KL/chosen_KL_mean": -576.2154541015625,
"KL/mean": -766.717529296875,
"KL/rejected_KL_mean": -957.2196655273438,
"KL/std": 433.15924072265625,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0012588687241077423,
"fcm_dpo/delta": -0.03120381385087967,
"fcm_dpo/margin": 381.0041809082031,
"fcm_dpo/q_t": 0.3909718096256256,
"grad_norm": 33.79661178588867,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -0.9247915744781494,
"logits/rejected": -0.9296138286590576,
"logps/chosen": -635.5391845703125,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1045.531982421875,
"loss": 1.042,
"margin_dpo/margin_mean": 381.00421142578125,
"margin_dpo/margin_std": 465.9805908203125,
"step": 612
},
{
"KL/chosen_KL_mean": -513.846923828125,
"KL/mean": -734.3082275390625,
"KL/rejected_KL_mean": -954.7694091796875,
"KL/std": 482.4210205078125,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0012329479213804007,
"fcm_dpo/delta": -0.09923183172941208,
"fcm_dpo/margin": 440.9225158691406,
"fcm_dpo/q_t": 0.37773290276527405,
"grad_norm": 37.04204177856445,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -0.9263854026794434,
"logits/rejected": -0.9770371317863464,
"logps/chosen": -573.576904296875,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1059.876953125,
"loss": 0.9984,
"margin_dpo/margin_mean": 440.9224853515625,
"margin_dpo/margin_std": 497.67266845703125,
"step": 613
},
{
"KL/chosen_KL_mean": -550.5446166992188,
"KL/mean": -752.9097290039062,
"KL/rejected_KL_mean": -955.2749633789062,
"KL/std": 451.09417724609375,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0012219803174957633,
"fcm_dpo/delta": -0.04686766862869263,
"fcm_dpo/margin": 404.73028564453125,
"fcm_dpo/q_t": 0.3896007239818573,
"grad_norm": 47.749725341796875,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -0.8647533655166626,
"logits/rejected": -0.9272447824478149,
"logps/chosen": -603.483642578125,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1059.954345703125,
"loss": 1.0469,
"margin_dpo/margin_mean": 404.73028564453125,
"margin_dpo/margin_std": 529.9136962890625,
"step": 614
},
{
"KL/chosen_KL_mean": -545.2410888671875,
"KL/mean": -722.5862426757812,
"KL/rejected_KL_mean": -899.931396484375,
"KL/std": 499.9029541015625,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.001225535525009036,
"fcm_dpo/delta": 0.015523616224527359,
"fcm_dpo/margin": 354.69036865234375,
"fcm_dpo/q_t": 0.40307652950286865,
"grad_norm": 34.70551681518555,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -0.8659655451774597,
"logits/rejected": -0.8805777430534363,
"logps/chosen": -611.058349609375,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -995.1089477539062,
"loss": 1.0965,
"margin_dpo/margin_mean": 354.69036865234375,
"margin_dpo/margin_std": 541.21923828125,
"step": 615
},
{
"KL/chosen_KL_mean": -641.4904174804688,
"KL/mean": -785.576416015625,
"KL/rejected_KL_mean": -929.6624145507812,
"KL/std": 414.92694091796875,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0012396399397403002,
"fcm_dpo/delta": 0.09611323475837708,
"fcm_dpo/margin": 288.1719970703125,
"fcm_dpo/q_t": 0.4201861619949341,
"grad_norm": 30.241304397583008,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -0.9229423999786377,
"logits/rejected": -0.9040850400924683,
"logps/chosen": -706.623291015625,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1004.3629150390625,
"loss": 1.1428,
"margin_dpo/margin_mean": 288.1719970703125,
"margin_dpo/margin_std": 478.29656982421875,
"step": 616
},
{
"KL/chosen_KL_mean": -588.5724487304688,
"KL/mean": -692.660400390625,
"KL/rejected_KL_mean": -796.7484130859375,
"KL/std": 374.8060302734375,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.001253496389836073,
"fcm_dpo/delta": 0.07038042694330215,
"fcm_dpo/margin": 208.1759490966797,
"fcm_dpo/q_t": 0.4403090476989746,
"grad_norm": 39.535221099853516,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.8760578632354736,
"logits/rejected": -0.8447480201721191,
"logps/chosen": -651.5780029296875,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -860.9824829101562,
"loss": 1.2101,
"margin_dpo/margin_mean": 208.17596435546875,
"margin_dpo/margin_std": 424.15655517578125,
"step": 617
},
{
"KL/chosen_KL_mean": -610.806640625,
"KL/mean": -831.383544921875,
"KL/rejected_KL_mean": -1051.9605712890625,
"KL/std": 508.52911376953125,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0012494558468461037,
"fcm_dpo/delta": -0.10715337097644806,
"fcm_dpo/margin": 441.1539611816406,
"fcm_dpo/q_t": 0.38160985708236694,
"grad_norm": 42.04853439331055,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -0.9283947348594666,
"logits/rejected": -0.9563186168670654,
"logps/chosen": -677.907958984375,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1144.114013671875,
"loss": 1.0438,
"margin_dpo/margin_mean": 441.15399169921875,
"margin_dpo/margin_std": 628.577880859375,
"step": 618
},
{
"KL/chosen_KL_mean": -599.4898071289062,
"KL/mean": -771.0576171875,
"KL/rejected_KL_mean": -942.6253662109375,
"KL/std": 476.91485595703125,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0012401568237692118,
"fcm_dpo/delta": 0.02554713748395443,
"fcm_dpo/margin": 343.1356201171875,
"fcm_dpo/q_t": 0.411257266998291,
"grad_norm": 43.26385498046875,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -0.911241352558136,
"logits/rejected": -0.922271728515625,
"logps/chosen": -655.468017578125,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1035.810791015625,
"loss": 1.1354,
"margin_dpo/margin_mean": 343.1356201171875,
"margin_dpo/margin_std": 618.1133422851562,
"step": 619
},
{
"KL/chosen_KL_mean": -556.5227661132812,
"KL/mean": -716.720703125,
"KL/rejected_KL_mean": -876.918701171875,
"KL/std": 417.01019287109375,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0012462441809475422,
"fcm_dpo/delta": 0.05265815928578377,
"fcm_dpo/margin": 320.3960266113281,
"fcm_dpo/q_t": 0.4107811450958252,
"grad_norm": 32.7905158996582,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -0.9634062051773071,
"logits/rejected": -0.9648847579956055,
"logps/chosen": -616.3202514648438,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -955.3294677734375,
"loss": 1.1067,
"margin_dpo/margin_mean": 320.39599609375,
"margin_dpo/margin_std": 471.8426513671875,
"step": 620
},
{
"KL/chosen_KL_mean": -537.1493530273438,
"KL/mean": -776.6170654296875,
"KL/rejected_KL_mean": -1016.0848388671875,
"KL/std": 520.2274169921875,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.0012356394436210394,
"fcm_dpo/delta": -0.15062512457370758,
"fcm_dpo/margin": 478.9354248046875,
"fcm_dpo/q_t": 0.36890918016433716,
"grad_norm": 55.19932174682617,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -0.9478855729103088,
"logits/rejected": -0.9874153137207031,
"logps/chosen": -591.0831298828125,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1104.454345703125,
"loss": 0.9921,
"margin_dpo/margin_mean": 478.93548583984375,
"margin_dpo/margin_std": 571.7491455078125,
"step": 621
},
{
"KL/chosen_KL_mean": -530.1005249023438,
"KL/mean": -701.41552734375,
"KL/rejected_KL_mean": -872.7305908203125,
"KL/std": 392.04901123046875,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0012181454803794622,
"fcm_dpo/delta": 0.03374656289815903,
"fcm_dpo/margin": 342.63006591796875,
"fcm_dpo/q_t": 0.40700438618659973,
"grad_norm": 32.37852096557617,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.8444244861602783,
"logits/rejected": -0.8345458507537842,
"logps/chosen": -590.3863525390625,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -958.2493896484375,
"loss": 1.0933,
"margin_dpo/margin_mean": 342.6300354003906,
"margin_dpo/margin_std": 495.0185546875,
"step": 622
},
{
"KL/chosen_KL_mean": -592.4083251953125,
"KL/mean": -726.1845092773438,
"KL/rejected_KL_mean": -859.960693359375,
"KL/std": 408.55181884765625,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.001248663291335106,
"fcm_dpo/delta": 0.11987321078777313,
"fcm_dpo/margin": 267.5523681640625,
"fcm_dpo/q_t": 0.4267767071723938,
"grad_norm": 42.03673553466797,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -0.9228973388671875,
"logits/rejected": -0.9195783138275146,
"logps/chosen": -656.5652465820312,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -945.043701171875,
"loss": 1.1644,
"margin_dpo/margin_mean": 267.5523681640625,
"margin_dpo/margin_std": 492.16070556640625,
"step": 623
},
{
"KL/chosen_KL_mean": -580.66943359375,
"KL/mean": -783.0391235351562,
"KL/rejected_KL_mean": -985.4088134765625,
"KL/std": 437.61358642578125,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.001246345229446888,
"fcm_dpo/delta": -0.05730106681585312,
"fcm_dpo/margin": 404.7393798828125,
"fcm_dpo/q_t": 0.3833310604095459,
"grad_norm": 38.60550308227539,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -0.9752143621444702,
"logits/rejected": -0.9746447801589966,
"logps/chosen": -652.588134765625,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1082.540771484375,
"loss": 1.0216,
"margin_dpo/margin_mean": 404.7393493652344,
"margin_dpo/margin_std": 462.248779296875,
"step": 624
},
{
"KL/chosen_KL_mean": -558.2151489257812,
"KL/mean": -771.4315185546875,
"KL/rejected_KL_mean": -984.64794921875,
"KL/std": 474.3367919921875,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0012284404365345836,
"fcm_dpo/delta": -0.0778719037771225,
"fcm_dpo/margin": 426.43280029296875,
"fcm_dpo/q_t": 0.381242036819458,
"grad_norm": 58.66389846801758,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -0.879206657409668,
"logits/rejected": -0.8958455324172974,
"logps/chosen": -616.5572509765625,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1070.73828125,
"loss": 0.9944,
"margin_dpo/margin_mean": 426.43280029296875,
"margin_dpo/margin_std": 444.8097229003906,
"step": 625
},
{
"KL/chosen_KL_mean": -676.9705810546875,
"KL/mean": -801.7283935546875,
"KL/rejected_KL_mean": -926.4862060546875,
"KL/std": 520.6180419921875,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.001250275643542409,
"fcm_dpo/delta": 0.14224281907081604,
"fcm_dpo/margin": 249.51565551757812,
"fcm_dpo/q_t": 0.4286021590232849,
"grad_norm": 42.33663558959961,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.8796349167823792,
"logits/rejected": -0.8687018156051636,
"logps/chosen": -752.0831298828125,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1025.6749267578125,
"loss": 1.1997,
"margin_dpo/margin_mean": 249.51565551757812,
"margin_dpo/margin_std": 512.447998046875,
"step": 626
},
{
"KL/chosen_KL_mean": -472.2640686035156,
"KL/mean": -725.2449340820312,
"KL/rejected_KL_mean": -978.2257080078125,
"KL/std": 566.2703247070312,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0012319653760641813,
"fcm_dpo/delta": -0.1845388561487198,
"fcm_dpo/margin": 505.96173095703125,
"fcm_dpo/q_t": 0.3714005947113037,
"grad_norm": 25.161611557006836,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.8306639194488525,
"logits/rejected": -0.8944810032844543,
"logps/chosen": -520.0072021484375,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1084.980224609375,
"loss": 0.9902,
"margin_dpo/margin_mean": 505.96173095703125,
"margin_dpo/margin_std": 648.461181640625,
"step": 627
},
{
"KL/chosen_KL_mean": -601.7943115234375,
"KL/mean": -791.7022705078125,
"KL/rejected_KL_mean": -981.6102294921875,
"KL/std": 455.6260986328125,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0012090300442650914,
"fcm_dpo/delta": -0.009682614356279373,
"fcm_dpo/margin": 379.8159484863281,
"fcm_dpo/q_t": 0.3963756263256073,
"grad_norm": 35.05025863647461,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -0.9712698459625244,
"logits/rejected": -0.9801019430160522,
"logps/chosen": -661.9771728515625,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1083.1649169921875,
"loss": 1.0631,
"margin_dpo/margin_mean": 379.81591796875,
"margin_dpo/margin_std": 499.5301513671875,
"step": 628
},
{
"KL/chosen_KL_mean": -594.9044189453125,
"KL/mean": -773.1024169921875,
"KL/rejected_KL_mean": -951.3004150390625,
"KL/std": 454.1546936035156,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.001212790608406067,
"fcm_dpo/delta": 0.01857149973511696,
"fcm_dpo/margin": 356.3959655761719,
"fcm_dpo/q_t": 0.4004923105239868,
"grad_norm": 38.04690170288086,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -0.9257842302322388,
"logits/rejected": -0.9372028708457947,
"logps/chosen": -659.117919921875,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1042.9541015625,
"loss": 1.0681,
"margin_dpo/margin_mean": 356.3959655761719,
"margin_dpo/margin_std": 443.0477600097656,
"step": 629
},
{
"KL/chosen_KL_mean": -544.481689453125,
"KL/mean": -651.665283203125,
"KL/rejected_KL_mean": -758.8487548828125,
"KL/std": 456.8427429199219,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0012220152420923114,
"fcm_dpo/delta": 0.0643744021654129,
"fcm_dpo/margin": 214.36705017089844,
"fcm_dpo/q_t": 0.44493043422698975,
"grad_norm": 52.316890716552734,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -0.9155005216598511,
"logits/rejected": -0.889801561832428,
"logps/chosen": -603.772705078125,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -842.4470825195312,
"loss": 1.2379,
"margin_dpo/margin_mean": 214.36705017089844,
"margin_dpo/margin_std": 532.4544677734375,
"step": 630
},
{
"KL/chosen_KL_mean": -656.3375244140625,
"KL/mean": -813.904541015625,
"KL/rejected_KL_mean": -971.4715576171875,
"KL/std": 471.61029052734375,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.0012422431027516723,
"fcm_dpo/delta": 0.06083694100379944,
"fcm_dpo/margin": 315.13397216796875,
"fcm_dpo/q_t": 0.41228383779525757,
"grad_norm": 37.45056915283203,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -0.9725683927536011,
"logits/rejected": -0.9720879197120667,
"logps/chosen": -715.7911376953125,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1052.423095703125,
"loss": 1.1406,
"margin_dpo/margin_mean": 315.1339416503906,
"margin_dpo/margin_std": 551.551513671875,
"step": 631
},
{
"KL/chosen_KL_mean": -576.4053344726562,
"KL/mean": -749.2000732421875,
"KL/rejected_KL_mean": -921.9948120117188,
"KL/std": 414.60711669921875,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0012430009664967656,
"fcm_dpo/delta": 0.020996015518903732,
"fcm_dpo/margin": 345.5894775390625,
"fcm_dpo/q_t": 0.40269041061401367,
"grad_norm": 38.403629302978516,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -0.9375253915786743,
"logits/rejected": -0.9352479577064514,
"logps/chosen": -637.7568969726562,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1008.1549682617188,
"loss": 1.0947,
"margin_dpo/margin_mean": 345.5894775390625,
"margin_dpo/margin_std": 501.6351318359375,
"step": 632
},
{
"KL/chosen_KL_mean": -614.3057861328125,
"KL/mean": -746.3451538085938,
"KL/rejected_KL_mean": -878.384521484375,
"KL/std": 419.60113525390625,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.001277608796954155,
"fcm_dpo/delta": 0.11562201380729675,
"fcm_dpo/margin": 264.0787353515625,
"fcm_dpo/q_t": 0.4232080578804016,
"grad_norm": 41.69476318359375,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -0.9602603912353516,
"logits/rejected": -0.9682069420814514,
"logps/chosen": -671.5839233398438,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -969.968505859375,
"loss": 1.1685,
"margin_dpo/margin_mean": 264.0787353515625,
"margin_dpo/margin_std": 484.4879150390625,
"step": 633
},
{
"KL/chosen_KL_mean": -703.184326171875,
"KL/mean": -820.115478515625,
"KL/rejected_KL_mean": -937.046630859375,
"KL/std": 514.7780151367188,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0013053391594439745,
"fcm_dpo/delta": 0.14952440559864044,
"fcm_dpo/margin": 233.86233520507812,
"fcm_dpo/q_t": 0.4344010651111603,
"grad_norm": 45.4162712097168,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.852074146270752,
"logits/rejected": -0.8421380519866943,
"logps/chosen": -769.80322265625,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1044.1722412109375,
"loss": 1.2035,
"margin_dpo/margin_mean": 233.86233520507812,
"margin_dpo/margin_std": 505.3544921875,
"step": 634
},
{
"KL/chosen_KL_mean": -600.64111328125,
"KL/mean": -754.6363525390625,
"KL/rejected_KL_mean": -908.6317138671875,
"KL/std": 523.8653564453125,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0013240852858871222,
"fcm_dpo/delta": 0.043988071382045746,
"fcm_dpo/margin": 307.99053955078125,
"fcm_dpo/q_t": 0.41226810216903687,
"grad_norm": 45.625946044921875,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -0.9453951120376587,
"logits/rejected": -0.9212517738342285,
"logps/chosen": -674.5966796875,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -991.1321411132812,
"loss": 1.122,
"margin_dpo/margin_mean": 307.99053955078125,
"margin_dpo/margin_std": 517.03857421875,
"step": 635
},
{
"KL/chosen_KL_mean": -575.39306640625,
"KL/mean": -755.2526245117188,
"KL/rejected_KL_mean": -935.1121826171875,
"KL/std": 504.8092346191406,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0013328897766768932,
"fcm_dpo/delta": -0.03171185031533241,
"fcm_dpo/margin": 359.71905517578125,
"fcm_dpo/q_t": 0.3958790898323059,
"grad_norm": 34.01151657104492,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.8697236776351929,
"logits/rejected": -0.8689707517623901,
"logps/chosen": -635.02197265625,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1017.0910034179688,
"loss": 1.0553,
"margin_dpo/margin_mean": 359.71905517578125,
"margin_dpo/margin_std": 498.491455078125,
"step": 636
},
{
"KL/chosen_KL_mean": -581.6488037109375,
"KL/mean": -774.9420166015625,
"KL/rejected_KL_mean": -968.2352294921875,
"KL/std": 499.24310302734375,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0013042533537372947,
"fcm_dpo/delta": -0.05777234956622124,
"fcm_dpo/margin": 386.58642578125,
"fcm_dpo/q_t": 0.38885462284088135,
"grad_norm": 30.27159881591797,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -0.9543923139572144,
"logits/rejected": -0.989661455154419,
"logps/chosen": -631.3014526367188,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1066.640380859375,
"loss": 1.0433,
"margin_dpo/margin_mean": 386.5863952636719,
"margin_dpo/margin_std": 510.89727783203125,
"step": 637
},
{
"KL/chosen_KL_mean": -564.4012451171875,
"KL/mean": -726.737548828125,
"KL/rejected_KL_mean": -889.0739135742188,
"KL/std": 475.2950439453125,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0013248416362330317,
"fcm_dpo/delta": 0.017911911010742188,
"fcm_dpo/margin": 324.6727294921875,
"fcm_dpo/q_t": 0.39857015013694763,
"grad_norm": 40.99110412597656,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.8504060506820679,
"logits/rejected": -0.8326265811920166,
"logps/chosen": -622.557861328125,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -968.3754272460938,
"loss": 1.1229,
"margin_dpo/margin_mean": 324.6727294921875,
"margin_dpo/margin_std": 536.3638916015625,
"step": 638
},
{
"KL/chosen_KL_mean": -756.171875,
"KL/mean": -841.8418579101562,
"KL/rejected_KL_mean": -927.5118408203125,
"KL/std": 439.4996337890625,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0013258617836982012,
"fcm_dpo/delta": 0.11202803254127502,
"fcm_dpo/margin": 171.33995056152344,
"fcm_dpo/q_t": 0.4497663080692291,
"grad_norm": 92.11475372314453,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.8558847904205322,
"logits/rejected": -0.8006620407104492,
"logps/chosen": -828.4951171875,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1001.7867431640625,
"loss": 1.2864,
"margin_dpo/margin_mean": 171.3399658203125,
"margin_dpo/margin_std": 525.029541015625,
"step": 639
},
{
"KL/chosen_KL_mean": -601.2142333984375,
"KL/mean": -849.1571044921875,
"KL/rejected_KL_mean": -1097.10009765625,
"KL/std": 536.4822998046875,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0013078118208795786,
"fcm_dpo/delta": -0.21178483963012695,
"fcm_dpo/margin": 495.8858337402344,
"fcm_dpo/q_t": 0.3637746274471283,
"grad_norm": 36.71278381347656,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.8685541749000549,
"logits/rejected": -0.8943710923194885,
"logps/chosen": -657.3485717773438,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1205.7001953125,
"loss": 0.9841,
"margin_dpo/margin_mean": 495.8858337402344,
"margin_dpo/margin_std": 631.3609008789062,
"step": 640
},
{
"KL/chosen_KL_mean": -672.4144287109375,
"KL/mean": -824.10107421875,
"KL/rejected_KL_mean": -975.7877197265625,
"KL/std": 447.71893310546875,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0012949431547895074,
"fcm_dpo/delta": 0.05952838063240051,
"fcm_dpo/margin": 303.37322998046875,
"fcm_dpo/q_t": 0.41488122940063477,
"grad_norm": 38.896240234375,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -0.936628520488739,
"logits/rejected": -0.9431087970733643,
"logps/chosen": -737.411376953125,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1062.780029296875,
"loss": 1.1532,
"margin_dpo/margin_mean": 303.37322998046875,
"margin_dpo/margin_std": 572.5907592773438,
"step": 641
},
{
"KL/chosen_KL_mean": -648.189453125,
"KL/mean": -826.8723754882812,
"KL/rejected_KL_mean": -1005.5552368164062,
"KL/std": 566.3262939453125,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0013033505529165268,
"fcm_dpo/delta": -0.01691451296210289,
"fcm_dpo/margin": 357.3657531738281,
"fcm_dpo/q_t": 0.40872862935066223,
"grad_norm": 40.749515533447266,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.8487492203712463,
"logits/rejected": -0.8746271133422852,
"logps/chosen": -713.8787231445312,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1115.79736328125,
"loss": 1.1299,
"margin_dpo/margin_mean": 357.36578369140625,
"margin_dpo/margin_std": 689.962646484375,
"step": 642
},
{
"KL/chosen_KL_mean": -570.0316162109375,
"KL/mean": -730.2611694335938,
"KL/rejected_KL_mean": -890.4908447265625,
"KL/std": 434.6820068359375,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0012935586273670197,
"fcm_dpo/delta": 0.036149993538856506,
"fcm_dpo/margin": 320.4592590332031,
"fcm_dpo/q_t": 0.4059324860572815,
"grad_norm": 48.46284484863281,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -0.9710277318954468,
"logits/rejected": -0.9704718589782715,
"logps/chosen": -621.9815673828125,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -977.9591674804688,
"loss": 1.0894,
"margin_dpo/margin_mean": 320.459228515625,
"margin_dpo/margin_std": 425.56072998046875,
"step": 643
},
{
"KL/chosen_KL_mean": -573.2258911132812,
"KL/mean": -717.8189697265625,
"KL/rejected_KL_mean": -862.4119873046875,
"KL/std": 502.07171630859375,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0013184216804802418,
"fcm_dpo/delta": 0.07150032371282578,
"fcm_dpo/margin": 289.1861572265625,
"fcm_dpo/q_t": 0.4177606701850891,
"grad_norm": 49.20467758178711,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -0.8860732316970825,
"logits/rejected": -0.8877465128898621,
"logps/chosen": -632.2435302734375,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -949.5487060546875,
"loss": 1.1697,
"margin_dpo/margin_mean": 289.1861267089844,
"margin_dpo/margin_std": 570.093505859375,
"step": 644
},
{
"KL/chosen_KL_mean": -601.1654052734375,
"KL/mean": -690.848876953125,
"KL/rejected_KL_mean": -780.5322265625,
"KL/std": 411.4150390625,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.001336642075330019,
"fcm_dpo/delta": 0.04631023108959198,
"fcm_dpo/margin": 179.36688232421875,
"fcm_dpo/q_t": 0.44901636242866516,
"grad_norm": 76.54983520507812,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.8880874514579773,
"logits/rejected": -0.8746832609176636,
"logps/chosen": -657.0413818359375,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -878.3131103515625,
"loss": 1.2902,
"margin_dpo/margin_mean": 179.36685180664062,
"margin_dpo/margin_std": 539.417236328125,
"step": 645
},
{
"KL/chosen_KL_mean": -561.1621704101562,
"KL/mean": -693.0960693359375,
"KL/rejected_KL_mean": -825.0301513671875,
"KL/std": 372.6351013183594,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0013508081901818514,
"fcm_dpo/delta": 0.09708744287490845,
"fcm_dpo/margin": 263.867919921875,
"fcm_dpo/q_t": 0.41975581645965576,
"grad_norm": 37.58830261230469,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -0.9159163236618042,
"logits/rejected": -0.8985805511474609,
"logps/chosen": -622.43798828125,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -902.535888671875,
"loss": 1.1401,
"margin_dpo/margin_mean": 263.867919921875,
"margin_dpo/margin_std": 423.1612548828125,
"step": 646
},
{
"KL/chosen_KL_mean": -518.5579833984375,
"KL/mean": -685.169677734375,
"KL/rejected_KL_mean": -851.7813720703125,
"KL/std": 454.782470703125,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0013628093292936683,
"fcm_dpo/delta": -0.004321117885410786,
"fcm_dpo/margin": 333.2234802246094,
"fcm_dpo/q_t": 0.3974537253379822,
"grad_norm": 30.513471603393555,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.841652512550354,
"logits/rejected": -0.865833044052124,
"logps/chosen": -573.410400390625,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -945.30078125,
"loss": 1.0628,
"margin_dpo/margin_mean": 333.2234802246094,
"margin_dpo/margin_std": 430.2264404296875,
"step": 647
},
{
"KL/chosen_KL_mean": -529.4920654296875,
"KL/mean": -776.0848388671875,
"KL/rejected_KL_mean": -1022.6776733398438,
"KL/std": 523.10302734375,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.0013180954847484827,
"fcm_dpo/delta": -0.21416297554969788,
"fcm_dpo/margin": 493.1856689453125,
"fcm_dpo/q_t": 0.3626644015312195,
"grad_norm": 38.04413604736328,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -0.9507880210876465,
"logits/rejected": -0.9994143843650818,
"logps/chosen": -583.6634521484375,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1121.3905029296875,
"loss": 0.9686,
"margin_dpo/margin_mean": 493.1856689453125,
"margin_dpo/margin_std": 615.1771240234375,
"step": 648
},
{
"KL/chosen_KL_mean": -556.886962890625,
"KL/mean": -677.7119750976562,
"KL/rejected_KL_mean": -798.5369873046875,
"KL/std": 411.56622314453125,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0013347992207854986,
"fcm_dpo/delta": 0.13102683424949646,
"fcm_dpo/margin": 241.6501007080078,
"fcm_dpo/q_t": 0.43245118856430054,
"grad_norm": 34.881595611572266,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -0.9540548920631409,
"logits/rejected": -0.9418928027153015,
"logps/chosen": -619.3673095703125,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -878.6141967773438,
"loss": 1.1995,
"margin_dpo/margin_mean": 241.65011596679688,
"margin_dpo/margin_std": 513.494873046875,
"step": 649
},
{
"KL/chosen_KL_mean": -591.3101196289062,
"KL/mean": -781.4625244140625,
"KL/rejected_KL_mean": -971.6149291992188,
"KL/std": 496.02606201171875,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.001323535107076168,
"fcm_dpo/delta": -0.05659695714712143,
"fcm_dpo/margin": 380.3048095703125,
"fcm_dpo/q_t": 0.3918585777282715,
"grad_norm": 32.53947830200195,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -0.9155662655830383,
"logits/rejected": -0.9315616488456726,
"logps/chosen": -647.4029541015625,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1069.8797607421875,
"loss": 1.074,
"margin_dpo/margin_mean": 380.3048095703125,
"margin_dpo/margin_std": 578.767578125,
"step": 650
},
{
"KL/chosen_KL_mean": -417.0550842285156,
"KL/mean": -646.5875244140625,
"KL/rejected_KL_mean": -876.1199951171875,
"KL/std": 477.76422119140625,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0012921846937388182,
"fcm_dpo/delta": -0.1525677740573883,
"fcm_dpo/margin": 459.06494140625,
"fcm_dpo/q_t": 0.3671586513519287,
"grad_norm": 39.78868865966797,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -0.957332968711853,
"logits/rejected": -1.0047751665115356,
"logps/chosen": -460.48052978515625,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -976.077880859375,
"loss": 0.9626,
"margin_dpo/margin_mean": 459.06494140625,
"margin_dpo/margin_std": 480.53533935546875,
"step": 651
},
{
"KL/chosen_KL_mean": -518.1369018554688,
"KL/mean": -699.426513671875,
"KL/rejected_KL_mean": -880.716064453125,
"KL/std": 489.3262023925781,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0012830387568101287,
"fcm_dpo/delta": -0.01596066914498806,
"fcm_dpo/margin": 362.57916259765625,
"fcm_dpo/q_t": 0.39959916472435,
"grad_norm": 35.97466278076172,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -0.9103597402572632,
"logits/rejected": -0.9288771152496338,
"logps/chosen": -580.7137451171875,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -992.48388671875,
"loss": 1.0846,
"margin_dpo/margin_mean": 362.57916259765625,
"margin_dpo/margin_std": 556.9105834960938,
"step": 652
},
{
"KL/chosen_KL_mean": -623.66650390625,
"KL/mean": -787.0184936523438,
"KL/rejected_KL_mean": -950.3704833984375,
"KL/std": 511.86151123046875,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0012876938562840223,
"fcm_dpo/delta": 0.030522624030709267,
"fcm_dpo/margin": 326.70391845703125,
"fcm_dpo/q_t": 0.4086763262748718,
"grad_norm": 36.10502243041992,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -0.9751501083374023,
"logits/rejected": -0.9935754537582397,
"logps/chosen": -684.779541015625,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1053.6201171875,
"loss": 1.116,
"margin_dpo/margin_mean": 326.7038879394531,
"margin_dpo/margin_std": 535.5750122070312,
"step": 653
},
{
"KL/chosen_KL_mean": -577.2074584960938,
"KL/mean": -730.6589965820312,
"KL/rejected_KL_mean": -884.1104736328125,
"KL/std": 413.8174133300781,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.0012976478319615126,
"fcm_dpo/delta": 0.05392075330018997,
"fcm_dpo/margin": 306.90301513671875,
"fcm_dpo/q_t": 0.4124031662940979,
"grad_norm": 32.3806266784668,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -0.9655961394309998,
"logits/rejected": -0.9741114377975464,
"logps/chosen": -638.9356079101562,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -982.88427734375,
"loss": 1.1149,
"margin_dpo/margin_mean": 306.90301513671875,
"margin_dpo/margin_std": 477.42364501953125,
"step": 654
},
{
"KL/chosen_KL_mean": -528.3974609375,
"KL/mean": -698.2972412109375,
"KL/rejected_KL_mean": -868.197021484375,
"KL/std": 432.225830078125,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.001303337630815804,
"fcm_dpo/delta": 0.007457331754267216,
"fcm_dpo/margin": 339.799560546875,
"fcm_dpo/q_t": 0.4031388759613037,
"grad_norm": 38.2427864074707,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -0.9318946599960327,
"logits/rejected": -0.9627610445022583,
"logps/chosen": -577.9742431640625,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -966.4888305664062,
"loss": 1.0911,
"margin_dpo/margin_mean": 339.799560546875,
"margin_dpo/margin_std": 511.25555419921875,
"step": 655
},
{
"KL/chosen_KL_mean": -561.493408203125,
"KL/mean": -830.2171630859375,
"KL/rejected_KL_mean": -1098.9407958984375,
"KL/std": 532.9222412109375,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0012617453467100859,
"fcm_dpo/delta": -0.24415498971939087,
"fcm_dpo/margin": 537.447509765625,
"fcm_dpo/q_t": 0.3515738248825073,
"grad_norm": 71.94105529785156,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.852911651134491,
"logits/rejected": -0.9044617414474487,
"logps/chosen": -614.0428466796875,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1212.615478515625,
"loss": 0.9165,
"margin_dpo/margin_mean": 537.4474487304688,
"margin_dpo/margin_std": 541.6453857421875,
"step": 656
},
{
"KL/chosen_KL_mean": -531.96435546875,
"KL/mean": -757.275146484375,
"KL/rejected_KL_mean": -982.5860595703125,
"KL/std": 519.469482421875,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0012197259347885847,
"fcm_dpo/delta": -0.10577631741762161,
"fcm_dpo/margin": 450.6217041015625,
"fcm_dpo/q_t": 0.3783363699913025,
"grad_norm": 34.344337463378906,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -0.9864233136177063,
"logits/rejected": -1.019301414489746,
"logps/chosen": -578.6649169921875,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1080.5009765625,
"loss": 1.0025,
"margin_dpo/margin_mean": 450.6216735839844,
"margin_dpo/margin_std": 536.0845947265625,
"step": 657
},
{
"KL/chosen_KL_mean": -574.1098022460938,
"KL/mean": -751.432373046875,
"KL/rejected_KL_mean": -928.7550048828125,
"KL/std": 408.3322448730469,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0012142022605985403,
"fcm_dpo/delta": 0.019960545003414154,
"fcm_dpo/margin": 354.645263671875,
"fcm_dpo/q_t": 0.4018761217594147,
"grad_norm": 34.58537292480469,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -0.9124845266342163,
"logits/rejected": -0.9352097511291504,
"logps/chosen": -635.0679931640625,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1024.694580078125,
"loss": 1.076,
"margin_dpo/margin_mean": 354.645263671875,
"margin_dpo/margin_std": 462.6511535644531,
"step": 658
},
{
"KL/chosen_KL_mean": -514.0313110351562,
"KL/mean": -695.9007568359375,
"KL/rejected_KL_mean": -877.7700805664062,
"KL/std": 428.44195556640625,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0012178844772279263,
"fcm_dpo/delta": 0.0071207694709300995,
"fcm_dpo/margin": 363.7388000488281,
"fcm_dpo/q_t": 0.4005226492881775,
"grad_norm": 28.91624641418457,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.8975902795791626,
"logits/rejected": -0.8781511783599854,
"logps/chosen": -590.7742919921875,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -965.2410278320312,
"loss": 1.0658,
"margin_dpo/margin_mean": 363.73876953125,
"margin_dpo/margin_std": 468.4670104980469,
"step": 659
},
{
"KL/chosen_KL_mean": -553.8775024414062,
"KL/mean": -770.415283203125,
"KL/rejected_KL_mean": -986.953125,
"KL/std": 480.31988525390625,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0012099661398679018,
"fcm_dpo/delta": -0.07803238183259964,
"fcm_dpo/margin": 433.0755920410156,
"fcm_dpo/q_t": 0.3820180296897888,
"grad_norm": 49.984649658203125,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -0.9479080438613892,
"logits/rejected": -0.9651952981948853,
"logps/chosen": -612.9254150390625,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1062.9130859375,
"loss": 1.0167,
"margin_dpo/margin_mean": 433.0755615234375,
"margin_dpo/margin_std": 514.4667358398438,
"step": 660
},
{
"KL/chosen_KL_mean": -509.1634216308594,
"KL/mean": -698.8631591796875,
"KL/rejected_KL_mean": -888.5628662109375,
"KL/std": 509.0642395019531,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.0012084790505468845,
"fcm_dpo/delta": -0.009469591081142426,
"fcm_dpo/margin": 379.3994140625,
"fcm_dpo/q_t": 0.3975834846496582,
"grad_norm": 42.885379791259766,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -0.9597800970077515,
"logits/rejected": -0.9635380506515503,
"logps/chosen": -559.83740234375,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -974.5685424804688,
"loss": 1.0607,
"margin_dpo/margin_mean": 379.3994140625,
"margin_dpo/margin_std": 500.22235107421875,
"step": 661
},
{
"KL/chosen_KL_mean": -577.4758911132812,
"KL/mean": -740.566650390625,
"KL/rejected_KL_mean": -903.6572875976562,
"KL/std": 462.82403564453125,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.001211101422086358,
"fcm_dpo/delta": 0.05724027380347252,
"fcm_dpo/margin": 326.181396484375,
"fcm_dpo/q_t": 0.41278716921806335,
"grad_norm": 32.96103286743164,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -0.8911588191986084,
"logits/rejected": -0.887296736240387,
"logps/chosen": -646.7369384765625,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -992.7132568359375,
"loss": 1.1349,
"margin_dpo/margin_mean": 326.181396484375,
"margin_dpo/margin_std": 560.8839721679688,
"step": 662
},
{
"KL/chosen_KL_mean": -556.5789794921875,
"KL/mean": -736.2896118164062,
"KL/rejected_KL_mean": -916.000244140625,
"KL/std": 514.8147583007812,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.001216050237417221,
"fcm_dpo/delta": 0.013502318412065506,
"fcm_dpo/margin": 359.4212951660156,
"fcm_dpo/q_t": 0.4045909643173218,
"grad_norm": 31.383352279663086,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -0.9621092677116394,
"logits/rejected": -0.9762744903564453,
"logps/chosen": -621.4578857421875,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1029.9256591796875,
"loss": 1.1097,
"margin_dpo/margin_mean": 359.4212951660156,
"margin_dpo/margin_std": 600.01806640625,
"step": 663
},
{
"KL/chosen_KL_mean": -570.5477294921875,
"KL/mean": -781.1873779296875,
"KL/rejected_KL_mean": -991.8270263671875,
"KL/std": 502.1649169921875,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.0012026217300444841,
"fcm_dpo/delta": -0.060244910418987274,
"fcm_dpo/margin": 421.27935791015625,
"fcm_dpo/q_t": 0.38703733682632446,
"grad_norm": 32.28097915649414,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -0.9074277877807617,
"logits/rejected": -0.931912899017334,
"logps/chosen": -631.4361572265625,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1097.3487548828125,
"loss": 1.035,
"margin_dpo/margin_mean": 421.27935791015625,
"margin_dpo/margin_std": 532.16455078125,
"step": 664
},
{
"KL/chosen_KL_mean": -507.58367919921875,
"KL/mean": -684.880126953125,
"KL/rejected_KL_mean": -862.1764526367188,
"KL/std": 417.1563720703125,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0012078677536919713,
"fcm_dpo/delta": 0.02266955003142357,
"fcm_dpo/margin": 354.5927734375,
"fcm_dpo/q_t": 0.4022970199584961,
"grad_norm": 43.988059997558594,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.0135409832000732,
"logits/rejected": -1.0313185453414917,
"logps/chosen": -568.1478271484375,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -946.9852905273438,
"loss": 1.0642,
"margin_dpo/margin_mean": 354.5927734375,
"margin_dpo/margin_std": 423.1929626464844,
"step": 665
},
{
"KL/chosen_KL_mean": -559.8313598632812,
"KL/mean": -742.6654052734375,
"KL/rejected_KL_mean": -925.49951171875,
"KL/std": 443.823486328125,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0012086930219084024,
"fcm_dpo/delta": 0.008344907313585281,
"fcm_dpo/margin": 365.66802978515625,
"fcm_dpo/q_t": 0.4003828167915344,
"grad_norm": 30.839902877807617,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.8351026773452759,
"logits/rejected": -0.8394874334335327,
"logps/chosen": -624.2513427734375,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1021.39111328125,
"loss": 1.0731,
"margin_dpo/margin_mean": 365.66802978515625,
"margin_dpo/margin_std": 484.2020568847656,
"step": 666
},
{
"KL/chosen_KL_mean": -613.4276123046875,
"KL/mean": -823.474365234375,
"KL/rejected_KL_mean": -1033.5211181640625,
"KL/std": 469.56622314453125,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.001200200291350484,
"fcm_dpo/delta": -0.05712345987558365,
"fcm_dpo/margin": 420.093505859375,
"fcm_dpo/q_t": 0.38684695959091187,
"grad_norm": 43.96973419189453,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.8881455659866333,
"logits/rejected": -0.8855566382408142,
"logps/chosen": -682.7046508789062,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1121.3565673828125,
"loss": 1.0312,
"margin_dpo/margin_mean": 420.093505859375,
"margin_dpo/margin_std": 518.9374389648438,
"step": 667
},
{
"KL/chosen_KL_mean": -647.3582763671875,
"KL/mean": -796.2598876953125,
"KL/rejected_KL_mean": -945.1614990234375,
"KL/std": 497.18328857421875,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.001220060046762228,
"fcm_dpo/delta": 0.08885399252176285,
"fcm_dpo/margin": 297.80328369140625,
"fcm_dpo/q_t": 0.420962393283844,
"grad_norm": 41.373592376708984,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -0.9342516660690308,
"logits/rejected": -0.938195526599884,
"logps/chosen": -719.9622802734375,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1048.9005126953125,
"loss": 1.1663,
"margin_dpo/margin_mean": 297.80328369140625,
"margin_dpo/margin_std": 567.736083984375,
"step": 668
},
{
"KL/chosen_KL_mean": -506.7996826171875,
"KL/mean": -717.8054809570312,
"KL/rejected_KL_mean": -928.811279296875,
"KL/std": 482.9616394042969,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0012088122311979532,
"fcm_dpo/delta": -0.06332193315029144,
"fcm_dpo/margin": 422.0116271972656,
"fcm_dpo/q_t": 0.3849877417087555,
"grad_norm": 31.68146514892578,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -0.8937242031097412,
"logits/rejected": -0.8956891894340515,
"logps/chosen": -552.9160766601562,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1006.735595703125,
"loss": 1.0352,
"margin_dpo/margin_mean": 422.0115966796875,
"margin_dpo/margin_std": 537.2428588867188,
"step": 669
},
{
"KL/chosen_KL_mean": -483.7650146484375,
"KL/mean": -699.0269775390625,
"KL/rejected_KL_mean": -914.2888793945312,
"KL/std": 446.49737548828125,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.0011966878082603216,
"fcm_dpo/delta": -0.06883969902992249,
"fcm_dpo/margin": 430.52386474609375,
"fcm_dpo/q_t": 0.38446512818336487,
"grad_norm": 30.194900512695312,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.8538155555725098,
"logits/rejected": -0.8683494329452515,
"logps/chosen": -546.1107788085938,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1011.2294311523438,
"loss": 1.0326,
"margin_dpo/margin_mean": 430.52386474609375,
"margin_dpo/margin_std": 539.012451171875,
"step": 670
},
{
"KL/chosen_KL_mean": -600.821533203125,
"KL/mean": -776.72021484375,
"KL/rejected_KL_mean": -952.618896484375,
"KL/std": 440.49432373046875,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0011965722078457475,
"fcm_dpo/delta": 0.02998637594282627,
"fcm_dpo/margin": 351.7973327636719,
"fcm_dpo/q_t": 0.4032197594642639,
"grad_norm": 28.47294807434082,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -0.939659059047699,
"logits/rejected": -0.9617972373962402,
"logps/chosen": -648.8216552734375,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1036.438232421875,
"loss": 1.0956,
"margin_dpo/margin_mean": 351.7973327636719,
"margin_dpo/margin_std": 507.2611083984375,
"step": 671
},
{
"KL/chosen_KL_mean": -638.2493896484375,
"KL/mean": -821.84765625,
"KL/rejected_KL_mean": -1005.44580078125,
"KL/std": 517.1207275390625,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0011946873273700476,
"fcm_dpo/delta": 0.011807839386165142,
"fcm_dpo/margin": 367.1964111328125,
"fcm_dpo/q_t": 0.40388986468315125,
"grad_norm": 41.209190368652344,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -0.9921166896820068,
"logits/rejected": -0.9929635524749756,
"logps/chosen": -696.8327026367188,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1098.5859375,
"loss": 1.1038,
"margin_dpo/margin_mean": 367.1964111328125,
"margin_dpo/margin_std": 573.3291015625,
"step": 672
},
{
"KL/chosen_KL_mean": -594.8367919921875,
"KL/mean": -771.89111328125,
"KL/rejected_KL_mean": -948.9454345703125,
"KL/std": 466.8351745605469,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0012018701527267694,
"fcm_dpo/delta": 0.02548855170607567,
"fcm_dpo/margin": 354.10858154296875,
"fcm_dpo/q_t": 0.4053458571434021,
"grad_norm": 41.65024948120117,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -0.9323441982269287,
"logits/rejected": -0.9339570999145508,
"logps/chosen": -641.5599365234375,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1034.24169921875,
"loss": 1.0899,
"margin_dpo/margin_mean": 354.10858154296875,
"margin_dpo/margin_std": 522.732421875,
"step": 673
},
{
"KL/chosen_KL_mean": -497.871337890625,
"KL/mean": -712.6817626953125,
"KL/rejected_KL_mean": -927.4921875,
"KL/std": 467.4238586425781,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0011941473931074142,
"fcm_dpo/delta": -0.06639358401298523,
"fcm_dpo/margin": 429.620849609375,
"fcm_dpo/q_t": 0.3882029950618744,
"grad_norm": 29.358726501464844,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.8644385933876038,
"logits/rejected": -0.879269003868103,
"logps/chosen": -543.31689453125,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -997.5380859375,
"loss": 1.0309,
"margin_dpo/margin_mean": 429.620849609375,
"margin_dpo/margin_std": 553.901611328125,
"step": 674
},
{
"KL/chosen_KL_mean": -564.3474731445312,
"KL/mean": -789.6226196289062,
"KL/rejected_KL_mean": -1014.897705078125,
"KL/std": 509.0472412109375,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.0011646319180727005,
"fcm_dpo/delta": -0.08057983219623566,
"fcm_dpo/margin": 450.55023193359375,
"fcm_dpo/q_t": 0.3845682144165039,
"grad_norm": 28.507211685180664,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -0.9137386083602905,
"logits/rejected": -0.92195725440979,
"logps/chosen": -608.5238037109375,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1088.98974609375,
"loss": 1.0194,
"margin_dpo/margin_mean": 450.55023193359375,
"margin_dpo/margin_std": 533.7015380859375,
"step": 675
},
{
"KL/chosen_KL_mean": -601.19189453125,
"KL/mean": -811.8079833984375,
"KL/rejected_KL_mean": -1022.424072265625,
"KL/std": 475.8211364746094,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0011619501747190952,
"fcm_dpo/delta": -0.041463352739810944,
"fcm_dpo/margin": 421.23211669921875,
"fcm_dpo/q_t": 0.3885056972503662,
"grad_norm": 27.30488395690918,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -0.8971314430236816,
"logits/rejected": -0.9086474180221558,
"logps/chosen": -672.5903930664062,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1110.7828369140625,
"loss": 1.0226,
"margin_dpo/margin_mean": 421.23211669921875,
"margin_dpo/margin_std": 460.779296875,
"step": 676
},
{
"KL/chosen_KL_mean": -617.59375,
"KL/mean": -808.9059448242188,
"KL/rejected_KL_mean": -1000.2181396484375,
"KL/std": 447.6944580078125,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0011601599399000406,
"fcm_dpo/delta": 0.006374634802341461,
"fcm_dpo/margin": 382.6244201660156,
"fcm_dpo/q_t": 0.40171724557876587,
"grad_norm": 29.162525177001953,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -0.965959906578064,
"logits/rejected": -0.9512898325920105,
"logps/chosen": -674.1211547851562,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1078.4447021484375,
"loss": 1.0718,
"margin_dpo/margin_mean": 382.6243896484375,
"margin_dpo/margin_std": 521.0079345703125,
"step": 677
},
{
"KL/chosen_KL_mean": -509.26019287109375,
"KL/mean": -732.9322509765625,
"KL/rejected_KL_mean": -956.6041259765625,
"KL/std": 503.4005126953125,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.0011527533642947674,
"fcm_dpo/delta": -0.06928122043609619,
"fcm_dpo/margin": 447.3440246582031,
"fcm_dpo/q_t": 0.3845905065536499,
"grad_norm": 39.06260299682617,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -0.9112190008163452,
"logits/rejected": -0.9373574256896973,
"logps/chosen": -555.3946533203125,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1037.208740234375,
"loss": 1.0295,
"margin_dpo/margin_mean": 447.343994140625,
"margin_dpo/margin_std": 562.4586181640625,
"step": 678
},
{
"KL/chosen_KL_mean": -587.34375,
"KL/mean": -751.5213012695312,
"KL/rejected_KL_mean": -915.6988525390625,
"KL/std": 422.2051086425781,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0011560373241081834,
"fcm_dpo/delta": 0.0732286274433136,
"fcm_dpo/margin": 328.3551025390625,
"fcm_dpo/q_t": 0.4141232371330261,
"grad_norm": 31.085031509399414,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -0.9332907199859619,
"logits/rejected": -0.9196882843971252,
"logps/chosen": -637.638671875,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -992.2969970703125,
"loss": 1.1126,
"margin_dpo/margin_mean": 328.3551025390625,
"margin_dpo/margin_std": 476.9893798828125,
"step": 679
},
{
"KL/chosen_KL_mean": -586.6705322265625,
"KL/mean": -796.8468017578125,
"KL/rejected_KL_mean": -1007.023193359375,
"KL/std": 541.9744873046875,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.001149947289377451,
"fcm_dpo/delta": -0.035634323954582214,
"fcm_dpo/margin": 420.35260009765625,
"fcm_dpo/q_t": 0.3903007507324219,
"grad_norm": 38.45374298095703,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -0.9182882905006409,
"logits/rejected": -0.9428098201751709,
"logps/chosen": -663.5863037109375,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1119.407958984375,
"loss": 1.0688,
"margin_dpo/margin_mean": 420.35260009765625,
"margin_dpo/margin_std": 602.3566284179688,
"step": 680
},
{
"KL/chosen_KL_mean": -585.1146240234375,
"KL/mean": -759.0559692382812,
"KL/rejected_KL_mean": -932.997314453125,
"KL/std": 455.07745361328125,
"epoch": 1.0,
"fcm_dpo/beta": 0.0011691637337207794,
"fcm_dpo/delta": 0.043633054941892624,
"fcm_dpo/margin": 347.8826904296875,
"fcm_dpo/q_t": 0.4084250032901764,
"grad_norm": 33.54962158203125,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -0.9781302213668823,
"logits/rejected": -0.9905676245689392,
"logps/chosen": -646.0718994140625,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1021.5552978515625,
"loss": 1.1007,
"margin_dpo/margin_mean": 347.8826904296875,
"margin_dpo/margin_std": 481.5751953125,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.0758972272998961,
"train_runtime": 1729.0144,
"train_samples_per_second": 25.216,
"train_steps_per_second": 0.394
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink