{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.00527191162109375, "KL/mean": 0.016706019639968872, "KL/rejected_KL_mean": 0.028141021728515625, "KL/std": 0.272699236869812, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02287006378173828, "fcm_dpo/q_t": 0.5005706548690796, "grad_norm": 83.5197982788086, "learning_rate": 0.0, "logits/chosen": -0.4974287748336792, "logits/rejected": -0.43299180269241333, "logps/chosen": -50.1435661315918, "logps/ref_chosen": -50.14883804321289, "logps/ref_rejected": -74.1280517578125, "logps/rejected": -74.09991455078125, "loss": 1.389, "margin_dpo/margin_mean": -0.02287048101425171, "margin_dpo/margin_std": 0.41920793056488037, "step": 1 }, { "KL/chosen_KL_mean": -0.03498649597167969, "KL/mean": -0.00212840735912323, "KL/rejected_KL_mean": 0.030735015869140625, "KL/std": 0.24797174334526062, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06572261452674866, "fcm_dpo/q_t": 0.5016425848007202, "grad_norm": 72.19226837158203, "learning_rate": 7.246376811594203e-09, "logits/chosen": -0.49536412954330444, "logits/rejected": -0.4594460427761078, "logps/chosen": -52.65568923950195, "logps/ref_chosen": -52.620704650878906, "logps/ref_rejected": -75.30413818359375, "logps/rejected": -75.27340698242188, "loss": 1.3932, "margin_dpo/margin_mean": -0.06572240591049194, "margin_dpo/margin_std": 0.35048407316207886, "step": 2 }, { "KL/chosen_KL_mean": -0.009714126586914062, "KL/mean": 0.018521636724472046, "KL/rejected_KL_mean": 0.046756744384765625, "KL/std": 0.22970828413963318, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0564723014831543, "fcm_dpo/q_t": 0.5014121532440186, "grad_norm": 71.04644775390625, "learning_rate": 1.4492753623188406e-08, "logits/chosen": -0.4816778302192688, "logits/rejected": -0.44217073917388916, "logps/chosen": -60.991310119628906, "logps/ref_chosen": -60.981597900390625, "logps/ref_rejected": -68.67259216308594, "logps/rejected": -68.62583923339844, "loss": 1.3922, "margin_dpo/margin_mean": -0.056471794843673706, "margin_dpo/margin_std": 0.3306810259819031, "step": 3 }, { "KL/chosen_KL_mean": -0.058868408203125, "KL/mean": -0.012493699789047241, "KL/rejected_KL_mean": 0.03388214111328125, "KL/std": 0.24051886796951294, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09274768829345703, "fcm_dpo/q_t": 0.5023181438446045, "grad_norm": 72.18009948730469, "learning_rate": 2.1739130434782606e-08, "logits/chosen": -0.4687211513519287, "logits/rejected": -0.4410913288593292, "logps/chosen": -56.82658004760742, "logps/ref_chosen": -56.7677116394043, "logps/ref_rejected": -86.64710998535156, "logps/rejected": -86.61322784423828, "loss": 1.3959, "margin_dpo/margin_mean": -0.09274822473526001, "margin_dpo/margin_std": 0.31391647458076477, "step": 4 }, { "KL/chosen_KL_mean": 0.03784751892089844, "KL/mean": 0.027686715126037598, "KL/rejected_KL_mean": 0.01752471923828125, "KL/std": 0.22657622396945953, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.020323216915130615, "fcm_dpo/q_t": 0.4994918406009674, "grad_norm": 89.44688415527344, "learning_rate": 2.898550724637681e-08, "logits/chosen": -0.514456033706665, "logits/rejected": -0.4707483947277069, "logps/chosen": -53.82152557373047, "logps/ref_chosen": -53.859375, "logps/ref_rejected": -84.14918518066406, "logps/rejected": -84.13166046142578, "loss": 1.3845, "margin_dpo/margin_mean": 0.02032339572906494, "margin_dpo/margin_std": 0.3100990056991577, "step": 5 }, { "KL/chosen_KL_mean": -0.022737503051757812, "KL/mean": -0.03838449716567993, "KL/rejected_KL_mean": -0.054027557373046875, "KL/std": 0.2597954571247101, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.031293392181396484, "fcm_dpo/q_t": 0.4992179870605469, "grad_norm": 91.83596801757812, "learning_rate": 3.6231884057971014e-08, "logits/chosen": -0.5073306560516357, "logits/rejected": -0.4651479721069336, "logps/chosen": -63.03022003173828, "logps/ref_chosen": -63.007484436035156, "logps/ref_rejected": -92.64534759521484, "logps/rejected": -92.69937133789062, "loss": 1.3835, "margin_dpo/margin_mean": 0.03129345178604126, "margin_dpo/margin_std": 0.3678780198097229, "step": 6 }, { "KL/chosen_KL_mean": 0.01824188232421875, "KL/mean": 0.009548112750053406, "KL/rejected_KL_mean": 0.000858306884765625, "KL/std": 0.30541443824768066, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01738789677619934, "fcm_dpo/q_t": 0.499565452337265, "grad_norm": 82.313232421875, "learning_rate": 4.347826086956521e-08, "logits/chosen": -0.5054666996002197, "logits/rejected": -0.47153782844543457, "logps/chosen": -57.75657653808594, "logps/ref_chosen": -57.774818420410156, "logps/ref_rejected": -103.92059326171875, "logps/rejected": -103.91973876953125, "loss": 1.3851, "margin_dpo/margin_mean": 0.01738855242729187, "margin_dpo/margin_std": 0.47851502895355225, "step": 7 }, { "KL/chosen_KL_mean": 0.03425025939941406, "KL/mean": 0.02390824258327484, "KL/rejected_KL_mean": 0.013568878173828125, "KL/std": 0.24170076847076416, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.020681053400039673, "fcm_dpo/q_t": 0.49948275089263916, "grad_norm": 78.34900665283203, "learning_rate": 5.0724637681159424e-08, "logits/chosen": -0.5002312064170837, "logits/rejected": -0.4734194874763489, "logps/chosen": -58.681785583496094, "logps/ref_chosen": -58.716033935546875, "logps/ref_rejected": -79.3114242553711, "logps/rejected": -79.2978515625, "loss": 1.3845, "margin_dpo/margin_mean": 0.020681530237197876, "margin_dpo/margin_std": 0.34713995456695557, "step": 8 }, { "KL/chosen_KL_mean": 0.06855201721191406, "KL/mean": 0.037594810128211975, "KL/rejected_KL_mean": 0.0066375732421875, "KL/std": 0.27004629373550415, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06191793084144592, "fcm_dpo/q_t": 0.49845263361930847, "grad_norm": 84.64756774902344, "learning_rate": 5.797101449275362e-08, "logits/chosen": -0.48879092931747437, "logits/rejected": -0.4417349100112915, "logps/chosen": -69.79829406738281, "logps/ref_chosen": -69.8668441772461, "logps/ref_rejected": -99.6026611328125, "logps/rejected": -99.59602355957031, "loss": 1.3805, "margin_dpo/margin_mean": 0.06191718578338623, "margin_dpo/margin_std": 0.39811158180236816, "step": 9 }, { "KL/chosen_KL_mean": 0.0026702880859375, "KL/mean": -0.0006842687726020813, "KL/rejected_KL_mean": -0.00403594970703125, "KL/std": 0.271045058965683, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.006704941391944885, "fcm_dpo/q_t": 0.499832421541214, "grad_norm": 70.90953063964844, "learning_rate": 6.521739130434782e-08, "logits/chosen": -0.49560266733169556, "logits/rejected": -0.4524238705635071, "logps/chosen": -48.35501480102539, "logps/ref_chosen": -48.35768508911133, "logps/ref_rejected": -80.37206268310547, "logps/rejected": -80.3760986328125, "loss": 1.386, "margin_dpo/margin_mean": 0.0067050158977508545, "margin_dpo/margin_std": 0.3649532198905945, "step": 10 }, { "KL/chosen_KL_mean": -0.0072174072265625, "KL/mean": -0.0030882954597473145, "KL/rejected_KL_mean": 0.00103759765625, "KL/std": 0.24918919801712036, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.008259892463684082, "fcm_dpo/q_t": 0.5002060532569885, "grad_norm": 68.35945129394531, "learning_rate": 7.246376811594203e-08, "logits/chosen": -0.4571724534034729, "logits/rejected": -0.43201056122779846, "logps/chosen": -53.024070739746094, "logps/ref_chosen": -53.01685333251953, "logps/ref_rejected": -87.78038024902344, "logps/rejected": -87.77934265136719, "loss": 1.3875, "margin_dpo/margin_mean": -0.008260130882263184, "margin_dpo/margin_std": 0.372198224067688, "step": 11 }, { "KL/chosen_KL_mean": -0.07953453063964844, "KL/mean": -0.05770687013864517, "KL/rejected_KL_mean": -0.035869598388671875, "KL/std": 0.2934768795967102, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04365710914134979, "fcm_dpo/q_t": 0.5010908842086792, "grad_norm": 99.78919982910156, "learning_rate": 7.971014492753623e-08, "logits/chosen": -0.5305306911468506, "logits/rejected": -0.4941375255584717, "logps/chosen": -61.884971618652344, "logps/ref_chosen": -61.80543518066406, "logps/ref_rejected": -104.8582763671875, "logps/rejected": -104.8941421508789, "loss": 1.3911, "margin_dpo/margin_mean": -0.04365821182727814, "margin_dpo/margin_std": 0.4039604663848877, "step": 12 }, { "KL/chosen_KL_mean": 0.021123886108398438, "KL/mean": -0.0034440606832504272, "KL/rejected_KL_mean": -0.028011322021484375, "KL/std": 0.25260454416275024, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04913899302482605, "fcm_dpo/q_t": 0.49877145886421204, "grad_norm": 79.35763549804688, "learning_rate": 8.695652173913042e-08, "logits/chosen": -0.46979302167892456, "logits/rejected": -0.44078677892684937, "logps/chosen": -64.23922729492188, "logps/ref_chosen": -64.2603530883789, "logps/ref_rejected": -87.20307922363281, "logps/rejected": -87.23109436035156, "loss": 1.3817, "margin_dpo/margin_mean": 0.04913872480392456, "margin_dpo/margin_std": 0.3550482392311096, "step": 13 }, { "KL/chosen_KL_mean": -0.015453338623046875, "KL/mean": 0.0030228644609451294, "KL/rejected_KL_mean": 0.021495819091796875, "KL/std": 0.28745049238204956, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03694823384284973, "fcm_dpo/q_t": 0.5009225010871887, "grad_norm": 85.36051177978516, "learning_rate": 9.420289855072464e-08, "logits/chosen": -0.4957757592201233, "logits/rejected": -0.4571382701396942, "logps/chosen": -58.12566375732422, "logps/ref_chosen": -58.11021041870117, "logps/ref_rejected": -104.04708099365234, "logps/rejected": -104.02558898925781, "loss": 1.3904, "margin_dpo/margin_mean": -0.036947935819625854, "margin_dpo/margin_std": 0.405885249376297, "step": 14 }, { "KL/chosen_KL_mean": -0.0232086181640625, "KL/mean": -0.03160741925239563, "KL/rejected_KL_mean": -0.040012359619140625, "KL/std": 0.21973538398742676, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01680755615234375, "fcm_dpo/q_t": 0.4995792806148529, "grad_norm": 64.38938903808594, "learning_rate": 1.0144927536231885e-07, "logits/chosen": -0.4843750596046448, "logits/rejected": -0.46534284949302673, "logps/chosen": -56.99011993408203, "logps/ref_chosen": -56.96691131591797, "logps/ref_rejected": -80.80863952636719, "logps/rejected": -80.8486557006836, "loss": 1.3849, "margin_dpo/margin_mean": 0.016807973384857178, "margin_dpo/margin_std": 0.3134637475013733, "step": 15 }, { "KL/chosen_KL_mean": -0.000408172607421875, "KL/mean": -0.03445975482463837, "KL/rejected_KL_mean": -0.06850433349609375, "KL/std": 0.27974092960357666, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06809529662132263, "fcm_dpo/q_t": 0.4982984960079193, "grad_norm": 84.10033416748047, "learning_rate": 1.0869565217391303e-07, "logits/chosen": -0.5534201860427856, "logits/rejected": -0.5151947140693665, "logps/chosen": -61.74030303955078, "logps/ref_chosen": -61.739891052246094, "logps/ref_rejected": -84.36947631835938, "logps/rejected": -84.43797302246094, "loss": 1.3799, "margin_dpo/margin_mean": 0.0680951476097107, "margin_dpo/margin_std": 0.403577983379364, "step": 16 }, { "KL/chosen_KL_mean": -0.020050048828125, "KL/mean": -0.03625258803367615, "KL/rejected_KL_mean": -0.052455902099609375, "KL/std": 0.25603610277175903, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03240293264389038, "fcm_dpo/q_t": 0.49919015169143677, "grad_norm": 78.74256134033203, "learning_rate": 1.1594202898550725e-07, "logits/chosen": -0.5004028081893921, "logits/rejected": -0.4630909860134125, "logps/chosen": -67.73038482666016, "logps/ref_chosen": -67.71033477783203, "logps/ref_rejected": -85.37865447998047, "logps/rejected": -85.43110656738281, "loss": 1.3834, "margin_dpo/margin_mean": 0.032403409481048584, "margin_dpo/margin_std": 0.3761865496635437, "step": 17 }, { "KL/chosen_KL_mean": 0.03632545471191406, "KL/mean": -0.003826543688774109, "KL/rejected_KL_mean": -0.043979644775390625, "KL/std": 0.21307751536369324, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08030399680137634, "fcm_dpo/q_t": 0.4979925751686096, "grad_norm": 82.01667785644531, "learning_rate": 1.2318840579710146e-07, "logits/chosen": -0.5047913789749146, "logits/rejected": -0.4507453441619873, "logps/chosen": -47.703163146972656, "logps/ref_chosen": -47.7394905090332, "logps/ref_rejected": -75.4722900390625, "logps/rejected": -75.51626586914062, "loss": 1.3785, "margin_dpo/margin_mean": 0.08030354976654053, "margin_dpo/margin_std": 0.275020033121109, "step": 18 }, { "KL/chosen_KL_mean": -0.029048919677734375, "KL/mean": -0.06294414401054382, "KL/rejected_KL_mean": -0.09683609008789062, "KL/std": 0.2524881362915039, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06778490543365479, "fcm_dpo/q_t": 0.4983060359954834, "grad_norm": 73.40299224853516, "learning_rate": 1.3043478260869563e-07, "logits/chosen": -0.49845147132873535, "logits/rejected": -0.44924086332321167, "logps/chosen": -70.23440551757812, "logps/ref_chosen": -70.20536041259766, "logps/ref_rejected": -89.7575912475586, "logps/rejected": -89.85443115234375, "loss": 1.3798, "margin_dpo/margin_mean": 0.06778547167778015, "margin_dpo/margin_std": 0.31719499826431274, "step": 19 }, { "KL/chosen_KL_mean": -0.0074100494384765625, "KL/mean": -0.033930569887161255, "KL/rejected_KL_mean": -0.06044769287109375, "KL/std": 0.2493729591369629, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.053041279315948486, "fcm_dpo/q_t": 0.4986746311187744, "grad_norm": 74.1423110961914, "learning_rate": 1.3768115942028986e-07, "logits/chosen": -0.5496178865432739, "logits/rejected": -0.49305465817451477, "logps/chosen": -50.81065368652344, "logps/ref_chosen": -50.80324172973633, "logps/ref_rejected": -78.82334899902344, "logps/rejected": -78.88379669189453, "loss": 1.3812, "margin_dpo/margin_mean": 0.05304113030433655, "margin_dpo/margin_std": 0.3071237802505493, "step": 20 }, { "KL/chosen_KL_mean": -0.015211105346679688, "KL/mean": -0.050164662301540375, "KL/rejected_KL_mean": -0.08511734008789062, "KL/std": 0.30007779598236084, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06991060078144073, "fcm_dpo/q_t": 0.4982530474662781, "grad_norm": 77.39631652832031, "learning_rate": 1.4492753623188405e-07, "logits/chosen": -0.5219053030014038, "logits/rejected": -0.500105082988739, "logps/chosen": -50.07822799682617, "logps/ref_chosen": -50.063018798828125, "logps/ref_rejected": -77.86878967285156, "logps/rejected": -77.95391082763672, "loss": 1.3797, "margin_dpo/margin_mean": 0.06991110742092133, "margin_dpo/margin_std": 0.38004350662231445, "step": 21 }, { "KL/chosen_KL_mean": 0.012784957885742188, "KL/mean": -0.08520452678203583, "KL/rejected_KL_mean": -0.18319320678710938, "KL/std": 0.290061354637146, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1959802210330963, "fcm_dpo/q_t": 0.49510306119918823, "grad_norm": 84.29964447021484, "learning_rate": 1.5217391304347825e-07, "logits/chosen": -0.471519410610199, "logits/rejected": -0.4271562695503235, "logps/chosen": -59.04485321044922, "logps/ref_chosen": -59.05763626098633, "logps/ref_rejected": -97.50466918945312, "logps/rejected": -97.6878662109375, "loss": 1.3672, "margin_dpo/margin_mean": 0.19597965478897095, "margin_dpo/margin_std": 0.4104743003845215, "step": 22 }, { "KL/chosen_KL_mean": 0.030836105346679688, "KL/mean": -0.05835852026939392, "KL/rejected_KL_mean": -0.147552490234375, "KL/std": 0.29798072576522827, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17839175462722778, "fcm_dpo/q_t": 0.4955432415008545, "grad_norm": 80.91433715820312, "learning_rate": 1.5942028985507245e-07, "logits/chosen": -0.4988241195678711, "logits/rejected": -0.47680526971817017, "logps/chosen": -60.04685974121094, "logps/ref_chosen": -60.07769775390625, "logps/ref_rejected": -81.13955688476562, "logps/rejected": -81.287109375, "loss": 1.369, "margin_dpo/margin_mean": 0.17839175462722778, "margin_dpo/margin_std": 0.42720067501068115, "step": 23 }, { "KL/chosen_KL_mean": 0.004024505615234375, "KL/mean": -0.11158287525177002, "KL/rejected_KL_mean": -0.22719573974609375, "KL/std": 0.2980898320674896, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23121654987335205, "fcm_dpo/q_t": 0.4942222833633423, "grad_norm": 80.61768341064453, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5090104341506958, "logits/rejected": -0.492495596408844, "logps/chosen": -44.287010192871094, "logps/ref_chosen": -44.29103469848633, "logps/ref_rejected": -99.12521362304688, "logps/rejected": -99.35240936279297, "loss": 1.3638, "margin_dpo/margin_mean": 0.2312171459197998, "margin_dpo/margin_std": 0.42055225372314453, "step": 24 }, { "KL/chosen_KL_mean": 0.027956008911132812, "KL/mean": -0.08138252794742584, "KL/rejected_KL_mean": -0.1907196044921875, "KL/std": 0.3556647300720215, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21867385506629944, "fcm_dpo/q_t": 0.4945368468761444, "grad_norm": 73.873046875, "learning_rate": 1.7391304347826085e-07, "logits/chosen": -0.4853622019290924, "logits/rejected": -0.45499587059020996, "logps/chosen": -52.50909423828125, "logps/ref_chosen": -52.537052154541016, "logps/ref_rejected": -89.34219360351562, "logps/rejected": -89.53291320800781, "loss": 1.365, "margin_dpo/margin_mean": 0.21867355704307556, "margin_dpo/margin_std": 0.4231797456741333, "step": 25 }, { "KL/chosen_KL_mean": 0.0839385986328125, "KL/mean": -0.11762094497680664, "KL/rejected_KL_mean": -0.3191795349121094, "KL/std": 0.41716688871383667, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40312010049819946, "fcm_dpo/q_t": 0.4899322986602783, "grad_norm": 87.6069564819336, "learning_rate": 1.8115942028985507e-07, "logits/chosen": -0.5463200807571411, "logits/rejected": -0.5151112079620361, "logps/chosen": -53.8388671875, "logps/ref_chosen": -53.92280578613281, "logps/ref_rejected": -103.35971069335938, "logps/rejected": -103.67889404296875, "loss": 1.3471, "margin_dpo/margin_mean": 0.4031205177307129, "margin_dpo/margin_std": 0.5176035165786743, "step": 26 }, { "KL/chosen_KL_mean": 0.12568092346191406, "KL/mean": -0.11782613396644592, "KL/rejected_KL_mean": -0.3613319396972656, "KL/std": 0.4365026354789734, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.48701417446136475, "fcm_dpo/q_t": 0.4878373146057129, "grad_norm": 93.9299545288086, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.5031396150588989, "logits/rejected": -0.4665963351726532, "logps/chosen": -42.772850036621094, "logps/ref_chosen": -42.898529052734375, "logps/ref_rejected": -98.72419738769531, "logps/rejected": -99.08553314208984, "loss": 1.3389, "margin_dpo/margin_mean": 0.48701420426368713, "margin_dpo/margin_std": 0.5155599117279053, "step": 27 }, { "KL/chosen_KL_mean": 0.016630172729492188, "KL/mean": -0.12676912546157837, "KL/rejected_KL_mean": -0.27016448974609375, "KL/std": 0.4014029800891876, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.28679442405700684, "fcm_dpo/q_t": 0.4928370714187622, "grad_norm": 75.26355743408203, "learning_rate": 1.9565217391304347e-07, "logits/chosen": -0.5167675614356995, "logits/rejected": -0.4624510407447815, "logps/chosen": -60.53987121582031, "logps/ref_chosen": -60.55650329589844, "logps/ref_rejected": -91.40111541748047, "logps/rejected": -91.67127990722656, "loss": 1.3586, "margin_dpo/margin_mean": 0.286795049905777, "margin_dpo/margin_std": 0.571891188621521, "step": 28 }, { "KL/chosen_KL_mean": 0.08650970458984375, "KL/mean": -0.1822274923324585, "KL/rejected_KL_mean": -0.4509620666503906, "KL/std": 0.4731426239013672, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5374736189842224, "fcm_dpo/q_t": 0.4865817129611969, "grad_norm": 90.55262756347656, "learning_rate": 2.028985507246377e-07, "logits/chosen": -0.5708495378494263, "logits/rejected": -0.5259160995483398, "logps/chosen": -57.721275329589844, "logps/ref_chosen": -57.80778503417969, "logps/ref_rejected": -97.39434814453125, "logps/rejected": -97.8453140258789, "loss": 1.3341, "margin_dpo/margin_mean": 0.5374747514724731, "margin_dpo/margin_std": 0.5670518279075623, "step": 29 }, { "KL/chosen_KL_mean": 0.13196372985839844, "KL/mean": -0.16928796470165253, "KL/rejected_KL_mean": -0.47054290771484375, "KL/std": 0.5440194606781006, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6025052070617676, "fcm_dpo/q_t": 0.4849626421928406, "grad_norm": 87.79325103759766, "learning_rate": 2.1014492753623187e-07, "logits/chosen": -0.49273908138275146, "logits/rejected": -0.4620092809200287, "logps/chosen": -52.44540786743164, "logps/ref_chosen": -52.577369689941406, "logps/ref_rejected": -98.48920440673828, "logps/rejected": -98.95974731445312, "loss": 1.328, "margin_dpo/margin_mean": 0.6025055646896362, "margin_dpo/margin_std": 0.6316369771957397, "step": 30 }, { "KL/chosen_KL_mean": 0.1329364776611328, "KL/mean": -0.11238974332809448, "KL/rejected_KL_mean": -0.35771942138671875, "KL/std": 0.5357916355133057, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4906558394432068, "fcm_dpo/q_t": 0.4877535104751587, "grad_norm": 68.40831756591797, "learning_rate": 2.1739130434782607e-07, "logits/chosen": -0.5103215575218201, "logits/rejected": -0.4660513401031494, "logps/chosen": -63.67398452758789, "logps/ref_chosen": -63.806922912597656, "logps/ref_rejected": -72.89400482177734, "logps/rejected": -73.25172424316406, "loss": 1.3389, "margin_dpo/margin_mean": 0.4906559884548187, "margin_dpo/margin_std": 0.6376643180847168, "step": 31 }, { "KL/chosen_KL_mean": 0.14924049377441406, "KL/mean": -0.1921585500240326, "KL/rejected_KL_mean": -0.5335617065429688, "KL/std": 0.6711945533752441, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6827967762947083, "fcm_dpo/q_t": 0.4829793870449066, "grad_norm": 84.73145294189453, "learning_rate": 2.2463768115942027e-07, "logits/chosen": -0.5275529026985168, "logits/rejected": -0.48725956678390503, "logps/chosen": -62.59028625488281, "logps/ref_chosen": -62.739524841308594, "logps/ref_rejected": -89.3175048828125, "logps/rejected": -89.85105895996094, "loss": 1.3208, "margin_dpo/margin_mean": 0.6827970743179321, "margin_dpo/margin_std": 0.8087345361709595, "step": 32 }, { "KL/chosen_KL_mean": 0.111785888671875, "KL/mean": -0.1832684874534607, "KL/rejected_KL_mean": -0.4783210754394531, "KL/std": 0.5401982069015503, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5901058912277222, "fcm_dpo/q_t": 0.48526614904403687, "grad_norm": 71.8738784790039, "learning_rate": 2.318840579710145e-07, "logits/chosen": -0.4825378954410553, "logits/rejected": -0.4562189280986786, "logps/chosen": -53.14918518066406, "logps/ref_chosen": -53.26097106933594, "logps/ref_rejected": -87.8851318359375, "logps/rejected": -88.36345672607422, "loss": 1.3291, "margin_dpo/margin_mean": 0.5901052951812744, "margin_dpo/margin_std": 0.6080037355422974, "step": 33 }, { "KL/chosen_KL_mean": 0.06308174133300781, "KL/mean": -0.3366324305534363, "KL/rejected_KL_mean": -0.7363510131835938, "KL/std": 0.7346658706665039, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7994296550750732, "fcm_dpo/q_t": 0.48007193207740784, "grad_norm": 77.36997985839844, "learning_rate": 2.391304347826087e-07, "logits/chosen": -0.48960667848587036, "logits/rejected": -0.47221672534942627, "logps/chosen": -50.75424575805664, "logps/ref_chosen": -50.81732940673828, "logps/ref_rejected": -101.92184448242188, "logps/rejected": -102.65819549560547, "loss": 1.3099, "margin_dpo/margin_mean": 0.7994290590286255, "margin_dpo/margin_std": 0.8769392371177673, "step": 34 }, { "KL/chosen_KL_mean": 0.14878082275390625, "KL/mean": -0.46657490730285645, "KL/rejected_KL_mean": -1.0819282531738281, "KL/std": 1.0033079385757446, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2307109832763672, "fcm_dpo/q_t": 0.46942776441574097, "grad_norm": 82.35343170166016, "learning_rate": 2.463768115942029e-07, "logits/chosen": -0.5199334025382996, "logits/rejected": -0.4829786419868469, "logps/chosen": -50.875709533691406, "logps/ref_chosen": -51.02449035644531, "logps/ref_rejected": -106.82443237304688, "logps/rejected": -107.90635681152344, "loss": 1.2702, "margin_dpo/margin_mean": 1.2307111024856567, "margin_dpo/margin_std": 1.1161994934082031, "step": 35 }, { "KL/chosen_KL_mean": 0.03397941589355469, "KL/mean": -0.5253908038139343, "KL/rejected_KL_mean": -1.0847587585449219, "KL/std": 1.0640511512756348, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1187388896942139, "fcm_dpo/q_t": 0.4721803665161133, "grad_norm": 73.1759262084961, "learning_rate": 2.536231884057971e-07, "logits/chosen": -0.5688107013702393, "logits/rejected": -0.5328692197799683, "logps/chosen": -51.95751190185547, "logps/ref_chosen": -51.991493225097656, "logps/ref_rejected": -86.0406265258789, "logps/rejected": -87.12538146972656, "loss": 1.2815, "margin_dpo/margin_mean": 1.1187382936477661, "margin_dpo/margin_std": 1.2321219444274902, "step": 36 }, { "KL/chosen_KL_mean": 0.016134262084960938, "KL/mean": -0.5008864402770996, "KL/rejected_KL_mean": -1.017913818359375, "KL/std": 1.063030481338501, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.034048080444336, "fcm_dpo/q_t": 0.47436150908470154, "grad_norm": 61.269187927246094, "learning_rate": 2.6086956521739126e-07, "logits/chosen": -0.5045929551124573, "logits/rejected": -0.46050429344177246, "logps/chosen": -62.79097366333008, "logps/ref_chosen": -62.807106018066406, "logps/ref_rejected": -77.89507293701172, "logps/rejected": -78.9129867553711, "loss": 1.2905, "margin_dpo/margin_mean": 1.0340485572814941, "margin_dpo/margin_std": 1.369009017944336, "step": 37 }, { "KL/chosen_KL_mean": 0.1357593536376953, "KL/mean": -0.5494659543037415, "KL/rejected_KL_mean": -1.2346916198730469, "KL/std": 1.3799314498901367, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3704538345336914, "fcm_dpo/q_t": 0.4661816358566284, "grad_norm": 70.13111877441406, "learning_rate": 2.681159420289855e-07, "logits/chosen": -0.5132662057876587, "logits/rejected": -0.4805130362510681, "logps/chosen": -48.2547607421875, "logps/ref_chosen": -48.39051818847656, "logps/ref_rejected": -97.91244506835938, "logps/rejected": -99.14713287353516, "loss": 1.2609, "margin_dpo/margin_mean": 1.3704535961151123, "margin_dpo/margin_std": 1.6687695980072021, "step": 38 }, { "KL/chosen_KL_mean": 0.08719444274902344, "KL/mean": -0.7247500419616699, "KL/rejected_KL_mean": -1.5366897583007812, "KL/std": 1.296190619468689, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.623884916305542, "fcm_dpo/q_t": 0.4597076177597046, "grad_norm": 73.18695068359375, "learning_rate": 2.753623188405797e-07, "logits/chosen": -0.5521361827850342, "logits/rejected": -0.5118452310562134, "logps/chosen": -50.66327667236328, "logps/ref_chosen": -50.75047302246094, "logps/ref_rejected": -78.56951141357422, "logps/rejected": -80.106201171875, "loss": 1.2356, "margin_dpo/margin_mean": 1.6238853931427002, "margin_dpo/margin_std": 1.410783290863037, "step": 39 }, { "KL/chosen_KL_mean": 0.20708274841308594, "KL/mean": -0.5767695903778076, "KL/rejected_KL_mean": -1.360626220703125, "KL/std": 1.399301290512085, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5677101612091064, "fcm_dpo/q_t": 0.46130138635635376, "grad_norm": 60.95301055908203, "learning_rate": 2.8260869565217386e-07, "logits/chosen": -0.5224695205688477, "logits/rejected": -0.4926735758781433, "logps/chosen": -57.777984619140625, "logps/ref_chosen": -57.985069274902344, "logps/ref_rejected": -74.3000717163086, "logps/rejected": -75.66069793701172, "loss": 1.2429, "margin_dpo/margin_mean": 1.5677103996276855, "margin_dpo/margin_std": 1.659773349761963, "step": 40 }, { "KL/chosen_KL_mean": 0.018999099731445312, "KL/mean": -0.9265443086624146, "KL/rejected_KL_mean": -1.8720817565917969, "KL/std": 1.8526782989501953, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10123378038406372, "fcm_dpo/delta": 0.12188068777322769, "fcm_dpo/margin": 1.8910868167877197, "fcm_dpo/q_t": 0.45350411534309387, "grad_norm": 69.15343475341797, "learning_rate": 2.898550724637681e-07, "logits/chosen": -0.5473026037216187, "logits/rejected": -0.511269211769104, "logps/chosen": -62.67681884765625, "logps/ref_chosen": -62.69581604003906, "logps/ref_rejected": -97.02352905273438, "logps/rejected": -98.89561462402344, "loss": 1.2142, "margin_dpo/margin_mean": 1.891087293624878, "margin_dpo/margin_std": 1.9742361307144165, "step": 41 }, { "KL/chosen_KL_mean": 0.19712257385253906, "KL/mean": -1.0590769052505493, "KL/rejected_KL_mean": -2.3152732849121094, "KL/std": 2.1316654682159424, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10550861060619354, "fcm_dpo/delta": 0.18977521359920502, "fcm_dpo/margin": 2.5123987197875977, "fcm_dpo/q_t": 0.43665236234664917, "grad_norm": 82.9195556640625, "learning_rate": 2.971014492753623e-07, "logits/chosen": -0.529656708240509, "logits/rejected": -0.4830622673034668, "logps/chosen": -58.76930236816406, "logps/ref_chosen": -58.966426849365234, "logps/ref_rejected": -109.90837097167969, "logps/rejected": -112.22364807128906, "loss": 1.1548, "margin_dpo/margin_mean": 2.5123987197875977, "margin_dpo/margin_std": 2.446010112762451, "step": 42 }, { "KL/chosen_KL_mean": 0.5524749755859375, "KL/mean": -0.7236171960830688, "KL/rejected_KL_mean": -1.9997138977050781, "KL/std": 1.8800958395004272, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10917122662067413, "fcm_dpo/delta": 0.17643879354000092, "fcm_dpo/margin": 2.5521881580352783, "fcm_dpo/q_t": 0.43298065662384033, "grad_norm": 76.43048858642578, "learning_rate": 3.043478260869565e-07, "logits/chosen": -0.5261872410774231, "logits/rejected": -0.5006349086761475, "logps/chosen": -53.60352325439453, "logps/ref_chosen": -54.15599822998047, "logps/ref_rejected": -96.48019409179688, "logps/rejected": -98.47990417480469, "loss": 1.139, "margin_dpo/margin_mean": 2.552187919616699, "margin_dpo/margin_std": 2.015552043914795, "step": 43 }, { "KL/chosen_KL_mean": 0.2166748046875, "KL/mean": -1.1926087141036987, "KL/rejected_KL_mean": -2.6018943786621094, "KL/std": 2.293412208557129, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.11247065663337708, "fcm_dpo/delta": 0.13742657005786896, "fcm_dpo/margin": 2.8185644149780273, "fcm_dpo/q_t": 0.42392122745513916, "grad_norm": 85.97471618652344, "learning_rate": 3.115942028985507e-07, "logits/chosen": -0.4837245047092438, "logits/rejected": -0.46352899074554443, "logps/chosen": -49.86182403564453, "logps/ref_chosen": -50.07849884033203, "logps/ref_rejected": -108.78376007080078, "logps/rejected": -111.38565063476562, "loss": 1.1097, "margin_dpo/margin_mean": 2.8185648918151855, "margin_dpo/margin_std": 2.2489266395568848, "step": 44 }, { "KL/chosen_KL_mean": 0.1769847869873047, "KL/mean": -0.9922209978103638, "KL/rejected_KL_mean": -2.1614303588867188, "KL/std": 2.0988166332244873, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.11665238440036774, "fcm_dpo/delta": 0.18180739879608154, "fcm_dpo/margin": 2.3384156227111816, "fcm_dpo/q_t": 0.43547263741493225, "grad_norm": 69.66962432861328, "learning_rate": 3.188405797101449e-07, "logits/chosen": -0.5075412392616272, "logits/rejected": -0.49554643034935, "logps/chosen": -48.237945556640625, "logps/ref_chosen": -48.4149284362793, "logps/ref_rejected": -77.93643188476562, "logps/rejected": -80.09785461425781, "loss": 1.1544, "margin_dpo/margin_mean": 2.3384151458740234, "margin_dpo/margin_std": 2.5854876041412354, "step": 45 }, { "KL/chosen_KL_mean": 0.19272613525390625, "KL/mean": -1.2884098291397095, "KL/rejected_KL_mean": -2.7695465087890625, "KL/std": 2.6169562339782715, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.11946526169776917, "fcm_dpo/delta": 0.09943334758281708, "fcm_dpo/margin": 2.9622695446014404, "fcm_dpo/q_t": 0.41688013076782227, "grad_norm": 80.77259826660156, "learning_rate": 3.260869565217391e-07, "logits/chosen": -0.5466458201408386, "logits/rejected": -0.4965612292289734, "logps/chosen": -55.80670166015625, "logps/ref_chosen": -55.999427795410156, "logps/ref_rejected": -95.652587890625, "logps/rejected": -98.42213439941406, "loss": 1.0964, "margin_dpo/margin_mean": 2.9622700214385986, "margin_dpo/margin_std": 3.1480636596679688, "step": 46 }, { "KL/chosen_KL_mean": 0.40157127380371094, "KL/mean": -1.0910553932189941, "KL/rejected_KL_mean": -2.5836868286132812, "KL/std": 2.669870615005493, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.12188410758972168, "fcm_dpo/delta": 0.08888313174247742, "fcm_dpo/margin": 2.9852540493011475, "fcm_dpo/q_t": 0.4136195778846741, "grad_norm": 75.20211791992188, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.5581070184707642, "logits/rejected": -0.5044229030609131, "logps/chosen": -57.524505615234375, "logps/ref_chosen": -57.92607879638672, "logps/ref_rejected": -94.67920684814453, "logps/rejected": -97.26289367675781, "loss": 1.0836, "margin_dpo/margin_mean": 2.9852542877197266, "margin_dpo/margin_std": 2.7970516681671143, "step": 47 }, { "KL/chosen_KL_mean": 0.03553581237792969, "KL/mean": -1.5421160459518433, "KL/rejected_KL_mean": -3.1197662353515625, "KL/std": 2.6868507862091064, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.12283409386873245, "fcm_dpo/delta": 0.06493905931711197, "fcm_dpo/margin": 3.1553053855895996, "fcm_dpo/q_t": 0.40832480788230896, "grad_norm": 82.24751281738281, "learning_rate": 3.4057971014492755e-07, "logits/chosen": -0.5874247550964355, "logits/rejected": -0.5290216207504272, "logps/chosen": -57.15253829956055, "logps/ref_chosen": -57.188072204589844, "logps/ref_rejected": -88.0166015625, "logps/rejected": -91.13636779785156, "loss": 1.0721, "margin_dpo/margin_mean": 3.1553053855895996, "margin_dpo/margin_std": 2.956268548965454, "step": 48 }, { "KL/chosen_KL_mean": 0.28813743591308594, "KL/mean": -1.608811855316162, "KL/rejected_KL_mean": -3.505756378173828, "KL/std": 3.38679838180542, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.12366083264350891, "fcm_dpo/delta": -0.020250260829925537, "fcm_dpo/margin": 3.7938899993896484, "fcm_dpo/q_t": 0.39228618144989014, "grad_norm": 74.37108612060547, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.5616977214813232, "logits/rejected": -0.5045787692070007, "logps/chosen": -61.397132873535156, "logps/ref_chosen": -61.685272216796875, "logps/ref_rejected": -83.76747131347656, "logps/rejected": -87.27323150634766, "loss": 1.026, "margin_dpo/margin_mean": 3.7938907146453857, "margin_dpo/margin_std": 3.9254069328308105, "step": 49 }, { "KL/chosen_KL_mean": -0.1705455780029297, "KL/mean": -2.235653877258301, "KL/rejected_KL_mean": -4.300762176513672, "KL/std": 3.464420795440674, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.12194898724555969, "fcm_dpo/delta": -0.05662181228399277, "fcm_dpo/margin": 4.130216598510742, "fcm_dpo/q_t": 0.3830483853816986, "grad_norm": 71.89964294433594, "learning_rate": 3.5507246376811595e-07, "logits/chosen": -0.5394672155380249, "logits/rejected": -0.5033368468284607, "logps/chosen": -58.894683837890625, "logps/ref_chosen": -58.72413635253906, "logps/ref_rejected": -96.35814666748047, "logps/rejected": -100.65890502929688, "loss": 0.9966, "margin_dpo/margin_mean": 4.130216598510742, "margin_dpo/margin_std": 3.9122300148010254, "step": 50 }, { "KL/chosen_KL_mean": -0.28783416748046875, "KL/mean": -2.3152730464935303, "KL/rejected_KL_mean": -4.342716217041016, "KL/std": 4.196344375610352, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.12034690380096436, "fcm_dpo/delta": -0.04077950119972229, "fcm_dpo/margin": 4.054882526397705, "fcm_dpo/q_t": 0.39256805181503296, "grad_norm": 58.6294059753418, "learning_rate": 3.6231884057971015e-07, "logits/chosen": -0.5217244625091553, "logits/rejected": -0.4886956810951233, "logps/chosen": -61.661502838134766, "logps/ref_chosen": -61.3736686706543, "logps/ref_rejected": -76.00199890136719, "logps/rejected": -80.34471893310547, "loss": 1.0407, "margin_dpo/margin_mean": 4.054882526397705, "margin_dpo/margin_std": 5.119638442993164, "step": 51 }, { "KL/chosen_KL_mean": 0.3047008514404297, "KL/mean": -2.708411693572998, "KL/rejected_KL_mean": -5.721527099609375, "KL/std": 4.686271667480469, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.11606203764677048, "fcm_dpo/delta": -0.26816853880882263, "fcm_dpo/margin": 6.0262274742126465, "fcm_dpo/q_t": 0.3416193127632141, "grad_norm": 63.627662658691406, "learning_rate": 3.695652173913043e-07, "logits/chosen": -0.5571841597557068, "logits/rejected": -0.5018373131752014, "logps/chosen": -52.03265380859375, "logps/ref_chosen": -52.33735656738281, "logps/ref_rejected": -79.97391510009766, "logps/rejected": -85.69544219970703, "loss": 0.8787, "margin_dpo/margin_mean": 6.0262274742126465, "margin_dpo/margin_std": 5.079197883605957, "step": 52 }, { "KL/chosen_KL_mean": -0.18621826171875, "KL/mean": -3.28546404838562, "KL/rejected_KL_mean": -6.384708404541016, "KL/std": 5.369787216186523, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.11117444187402725, "fcm_dpo/delta": -0.256150484085083, "fcm_dpo/margin": 6.198496341705322, "fcm_dpo/q_t": 0.3534468710422516, "grad_norm": 62.074974060058594, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.5862455368041992, "logits/rejected": -0.5638996958732605, "logps/chosen": -53.50086975097656, "logps/ref_chosen": -53.31465148925781, "logps/ref_rejected": -91.78359985351562, "logps/rejected": -98.16830444335938, "loss": 0.9201, "margin_dpo/margin_mean": 6.198496341705322, "margin_dpo/margin_std": 6.157903671264648, "step": 53 }, { "KL/chosen_KL_mean": -0.3846149444580078, "KL/mean": -3.0846762657165527, "KL/rejected_KL_mean": -5.784732818603516, "KL/std": 5.093508720397949, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10680510848760605, "fcm_dpo/delta": -0.13433100283145905, "fcm_dpo/margin": 5.400116920471191, "fcm_dpo/q_t": 0.3691354990005493, "grad_norm": 58.43413543701172, "learning_rate": 3.8405797101449274e-07, "logits/chosen": -0.5928705334663391, "logits/rejected": -0.5393671989440918, "logps/chosen": -51.07327651977539, "logps/ref_chosen": -50.68865966796875, "logps/ref_rejected": -91.71539306640625, "logps/rejected": -97.5001220703125, "loss": 0.9555, "margin_dpo/margin_mean": 5.400116920471191, "margin_dpo/margin_std": 5.129217624664307, "step": 54 }, { "KL/chosen_KL_mean": -0.8974971771240234, "KL/mean": -4.153885841369629, "KL/rejected_KL_mean": -7.410274505615234, "KL/std": 6.167769432067871, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10262539237737656, "fcm_dpo/delta": -0.233448326587677, "fcm_dpo/margin": 6.512773513793945, "fcm_dpo/q_t": 0.3559892475605011, "grad_norm": 54.6057014465332, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.6421641111373901, "logits/rejected": -0.5792768001556396, "logps/chosen": -63.512733459472656, "logps/ref_chosen": -62.615234375, "logps/ref_rejected": -88.99349975585938, "logps/rejected": -96.40377807617188, "loss": 0.9461, "margin_dpo/margin_mean": 6.512773036956787, "margin_dpo/margin_std": 7.380474090576172, "step": 55 }, { "KL/chosen_KL_mean": -0.6917400360107422, "KL/mean": -3.759640693664551, "KL/rejected_KL_mean": -6.827548980712891, "KL/std": 6.048251628875732, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.09863682836294174, "fcm_dpo/delta": -0.1651272475719452, "fcm_dpo/margin": 6.135809898376465, "fcm_dpo/q_t": 0.3689035177230835, "grad_norm": 47.54218673706055, "learning_rate": 3.9855072463768114e-07, "logits/chosen": -0.6309115886688232, "logits/rejected": -0.5893651247024536, "logps/chosen": -58.62446594238281, "logps/ref_chosen": -57.9327278137207, "logps/ref_rejected": -94.1744384765625, "logps/rejected": -101.00198364257812, "loss": 0.9837, "margin_dpo/margin_mean": 6.135810375213623, "margin_dpo/margin_std": 7.349212646484375, "step": 56 }, { "KL/chosen_KL_mean": -0.7053565979003906, "KL/mean": -4.171047210693359, "KL/rejected_KL_mean": -7.636737823486328, "KL/std": 5.979583263397217, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.09502957761287689, "fcm_dpo/delta": -0.2228081077337265, "fcm_dpo/margin": 6.931379795074463, "fcm_dpo/q_t": 0.3515072464942932, "grad_norm": 52.40454864501953, "learning_rate": 4.057971014492754e-07, "logits/chosen": -0.5713903903961182, "logits/rejected": -0.5428633689880371, "logps/chosen": -71.20063781738281, "logps/ref_chosen": -70.49528503417969, "logps/ref_rejected": -95.56546020507812, "logps/rejected": -103.20219421386719, "loss": 0.9213, "margin_dpo/margin_mean": 6.931380271911621, "margin_dpo/margin_std": 6.604933738708496, "step": 57 }, { "KL/chosen_KL_mean": -1.0504646301269531, "KL/mean": -4.870463848114014, "KL/rejected_KL_mean": -8.690460205078125, "KL/std": 6.776705741882324, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.09036161750555038, "fcm_dpo/delta": -0.2574799656867981, "fcm_dpo/margin": 7.639996528625488, "fcm_dpo/q_t": 0.35100215673446655, "grad_norm": 55.40933609008789, "learning_rate": 4.1304347826086954e-07, "logits/chosen": -0.6046279668807983, "logits/rejected": -0.5296686291694641, "logps/chosen": -63.18340301513672, "logps/ref_chosen": -62.13294219970703, "logps/ref_rejected": -84.61729431152344, "logps/rejected": -93.30775451660156, "loss": 0.9253, "margin_dpo/margin_mean": 7.639996528625488, "margin_dpo/margin_std": 8.138544082641602, "step": 58 }, { "KL/chosen_KL_mean": -1.4362564086914062, "KL/mean": -5.643172264099121, "KL/rejected_KL_mean": -9.850086212158203, "KL/std": 7.354021072387695, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.08438800275325775, "fcm_dpo/delta": -0.28398486971855164, "fcm_dpo/margin": 8.41383171081543, "fcm_dpo/q_t": 0.34511446952819824, "grad_norm": 51.24748611450195, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.6187716126441956, "logits/rejected": -0.5780496597290039, "logps/chosen": -53.36878204345703, "logps/ref_chosen": -51.932525634765625, "logps/ref_rejected": -88.88520050048828, "logps/rejected": -98.73529052734375, "loss": 0.9054, "margin_dpo/margin_mean": 8.41383171081543, "margin_dpo/margin_std": 8.447948455810547, "step": 59 }, { "KL/chosen_KL_mean": -2.6461715698242188, "KL/mean": -5.977200508117676, "KL/rejected_KL_mean": -9.308223724365234, "KL/std": 6.706561088562012, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.08230020850896835, "fcm_dpo/delta": -0.10461775958538055, "fcm_dpo/margin": 6.662050247192383, "fcm_dpo/q_t": 0.37456339597702026, "grad_norm": 55.224952697753906, "learning_rate": 4.2753623188405794e-07, "logits/chosen": -0.6233890652656555, "logits/rejected": -0.5643144249916077, "logps/chosen": -63.58835983276367, "logps/ref_chosen": -60.94218826293945, "logps/ref_rejected": -85.39340209960938, "logps/rejected": -94.70162200927734, "loss": 1.0012, "margin_dpo/margin_mean": 6.662050247192383, "margin_dpo/margin_std": 7.595391273498535, "step": 60 }, { "KL/chosen_KL_mean": -1.484731674194336, "KL/mean": -5.6159772872924805, "KL/rejected_KL_mean": -9.747222900390625, "KL/std": 9.038053512573242, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.07915720343589783, "fcm_dpo/delta": -0.21973448991775513, "fcm_dpo/margin": 8.262494087219238, "fcm_dpo/q_t": 0.3663337528705597, "grad_norm": 46.47762680053711, "learning_rate": 4.3478260869565214e-07, "logits/chosen": -0.5932574272155762, "logits/rejected": -0.5587595701217651, "logps/chosen": -62.118255615234375, "logps/ref_chosen": -60.633522033691406, "logps/ref_rejected": -89.85249328613281, "logps/rejected": -99.59971618652344, "loss": 0.9817, "margin_dpo/margin_mean": 8.262494087219238, "margin_dpo/margin_std": 11.187477111816406, "step": 61 }, { "KL/chosen_KL_mean": -1.6137275695800781, "KL/mean": -4.718844413757324, "KL/rejected_KL_mean": -7.8239593505859375, "KL/std": 6.609654903411865, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.07849176973104477, "fcm_dpo/delta": -0.03957677260041237, "fcm_dpo/margin": 6.210230827331543, "fcm_dpo/q_t": 0.39217427372932434, "grad_norm": 46.09128189086914, "learning_rate": 4.420289855072464e-07, "logits/chosen": -0.594412624835968, "logits/rejected": -0.5592746734619141, "logps/chosen": -57.76449966430664, "logps/ref_chosen": -56.15077209472656, "logps/ref_rejected": -75.56619262695312, "logps/rejected": -83.39015197753906, "loss": 1.0485, "margin_dpo/margin_mean": 6.210231781005859, "margin_dpo/margin_std": 8.076946258544922, "step": 62 }, { "KL/chosen_KL_mean": -2.6496753692626953, "KL/mean": -6.814581871032715, "KL/rejected_KL_mean": -10.979488372802734, "KL/std": 8.399147987365723, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.07532910257577896, "fcm_dpo/delta": -0.19143244624137878, "fcm_dpo/margin": 8.329812049865723, "fcm_dpo/q_t": 0.3595030903816223, "grad_norm": 47.35303497314453, "learning_rate": 4.4927536231884053e-07, "logits/chosen": -0.5923917889595032, "logits/rejected": -0.547005295753479, "logps/chosen": -75.79707336425781, "logps/ref_chosen": -73.14739227294922, "logps/ref_rejected": -97.61006164550781, "logps/rejected": -108.58955383300781, "loss": 0.9544, "margin_dpo/margin_mean": 8.329812049865723, "margin_dpo/margin_std": 8.884681701660156, "step": 63 }, { "KL/chosen_KL_mean": -1.0408763885498047, "KL/mean": -5.9636712074279785, "KL/rejected_KL_mean": -10.886466979980469, "KL/std": 9.097991943359375, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.07129538059234619, "fcm_dpo/delta": -0.27623987197875977, "fcm_dpo/margin": 9.845589637756348, "fcm_dpo/q_t": 0.3495906591415405, "grad_norm": 42.23509979248047, "learning_rate": 4.5652173913043473e-07, "logits/chosen": -0.5778393745422363, "logits/rejected": -0.5453320741653442, "logps/chosen": -55.03947448730469, "logps/ref_chosen": -53.998600006103516, "logps/ref_rejected": -93.53019714355469, "logps/rejected": -104.41667175292969, "loss": 0.9342, "margin_dpo/margin_mean": 9.845589637756348, "margin_dpo/margin_std": 11.189323425292969, "step": 64 }, { "KL/chosen_KL_mean": -3.26495361328125, "KL/mean": -8.230804443359375, "KL/rejected_KL_mean": -13.1966552734375, "KL/std": 10.192216873168945, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.06810398399829865, "fcm_dpo/delta": -0.24493469297885895, "fcm_dpo/margin": 9.931705474853516, "fcm_dpo/q_t": 0.35102152824401855, "grad_norm": 43.12425231933594, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.6511636972427368, "logits/rejected": -0.6380265951156616, "logps/chosen": -68.1009521484375, "logps/ref_chosen": -64.83599853515625, "logps/ref_rejected": -109.94645690917969, "logps/rejected": -123.14311218261719, "loss": 0.947, "margin_dpo/margin_mean": 9.931705474853516, "margin_dpo/margin_std": 11.15705680847168, "step": 65 }, { "KL/chosen_KL_mean": -3.0033321380615234, "KL/mean": -7.569043159484863, "KL/rejected_KL_mean": -12.134750366210938, "KL/std": 9.202418327331543, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.06553401052951813, "fcm_dpo/delta": -0.16035820543766022, "fcm_dpo/margin": 9.131417274475098, "fcm_dpo/q_t": 0.3708842694759369, "grad_norm": 38.7624626159668, "learning_rate": 4.7101449275362313e-07, "logits/chosen": -0.6315481066703796, "logits/rejected": -0.5981906652450562, "logps/chosen": -54.44685745239258, "logps/ref_chosen": -51.44352722167969, "logps/ref_rejected": -75.63629913330078, "logps/rejected": -87.77104949951172, "loss": 0.9843, "margin_dpo/margin_mean": 9.131417274475098, "margin_dpo/margin_std": 11.071283340454102, "step": 66 }, { "KL/chosen_KL_mean": -2.688854217529297, "KL/mean": -7.336255073547363, "KL/rejected_KL_mean": -11.983657836914062, "KL/std": 9.25648307800293, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.06420481204986572, "fcm_dpo/delta": -0.1558014154434204, "fcm_dpo/margin": 9.294797897338867, "fcm_dpo/q_t": 0.3707802891731262, "grad_norm": 39.4743766784668, "learning_rate": 4.782608695652174e-07, "logits/chosen": -0.5986242294311523, "logits/rejected": -0.555691123008728, "logps/chosen": -62.029659271240234, "logps/ref_chosen": -59.34080505371094, "logps/ref_rejected": -72.78728485107422, "logps/rejected": -84.77094268798828, "loss": 0.9841, "margin_dpo/margin_mean": 9.294797897338867, "margin_dpo/margin_std": 11.213580131530762, "step": 67 }, { "KL/chosen_KL_mean": -3.099040985107422, "KL/mean": -7.559505462646484, "KL/rejected_KL_mean": -12.019973754882812, "KL/std": 8.27375602722168, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.06258167326450348, "fcm_dpo/delta": -0.11453147232532501, "fcm_dpo/margin": 8.920928955078125, "fcm_dpo/q_t": 0.37400543689727783, "grad_norm": 38.21786880493164, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.6311640739440918, "logits/rejected": -0.5698702335357666, "logps/chosen": -68.30487060546875, "logps/ref_chosen": -65.2058334350586, "logps/ref_rejected": -77.20724487304688, "logps/rejected": -89.22721862792969, "loss": 0.9784, "margin_dpo/margin_mean": 8.920928955078125, "margin_dpo/margin_std": 9.243380546569824, "step": 68 }, { "KL/chosen_KL_mean": -3.6259117126464844, "KL/mean": -9.136199951171875, "KL/rejected_KL_mean": -14.646488189697266, "KL/std": 9.907639503479004, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.059743233025074005, "fcm_dpo/delta": -0.2245938926935196, "fcm_dpo/margin": 11.020575523376465, "fcm_dpo/q_t": 0.35451948642730713, "grad_norm": 40.647071838378906, "learning_rate": 4.927536231884058e-07, "logits/chosen": -0.599290668964386, "logits/rejected": -0.5760542154312134, "logps/chosen": -63.445152282714844, "logps/ref_chosen": -59.81924057006836, "logps/ref_rejected": -103.38886260986328, "logps/rejected": -118.03535461425781, "loss": 0.9216, "margin_dpo/margin_mean": 11.020574569702148, "margin_dpo/margin_std": 10.930428504943848, "step": 69 }, { "KL/chosen_KL_mean": -5.381891250610352, "KL/mean": -11.343828201293945, "KL/rejected_KL_mean": -17.305763244628906, "KL/std": 12.163260459899902, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.0573650524020195, "fcm_dpo/delta": -0.2505870461463928, "fcm_dpo/margin": 11.923868179321289, "fcm_dpo/q_t": 0.352365642786026, "grad_norm": 40.70524597167969, "learning_rate": 5e-07, "logits/chosen": -0.6251968741416931, "logits/rejected": -0.5907549858093262, "logps/chosen": -67.31253051757812, "logps/ref_chosen": -61.930641174316406, "logps/ref_rejected": -91.06078338623047, "logps/rejected": -108.36654663085938, "loss": 0.9317, "margin_dpo/margin_mean": 11.923868179321289, "margin_dpo/margin_std": 12.913337707519531, "step": 70 }, { "KL/chosen_KL_mean": -5.122894287109375, "KL/mean": -12.12697982788086, "KL/rejected_KL_mean": -19.13106918334961, "KL/std": 12.782737731933594, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.05358006805181503, "fcm_dpo/delta": -0.3263747990131378, "fcm_dpo/margin": 14.008173942565918, "fcm_dpo/q_t": 0.3399330973625183, "grad_norm": 37.48530578613281, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.658132791519165, "logits/rejected": -0.6188231706619263, "logps/chosen": -66.87322998046875, "logps/ref_chosen": -61.750335693359375, "logps/ref_rejected": -97.33662414550781, "logps/rejected": -116.46768951416016, "loss": 0.8843, "margin_dpo/margin_mean": 14.008173942565918, "margin_dpo/margin_std": 13.910791397094727, "step": 71 }, { "KL/chosen_KL_mean": -5.92857551574707, "KL/mean": -13.34725570678711, "KL/rejected_KL_mean": -20.765933990478516, "KL/std": 13.781370162963867, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.05028136447072029, "fcm_dpo/delta": -0.3209035396575928, "fcm_dpo/margin": 14.837360382080078, "fcm_dpo/q_t": 0.33984917402267456, "grad_norm": 37.00834274291992, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.6663771271705627, "logits/rejected": -0.6306756138801575, "logps/chosen": -71.98198699951172, "logps/ref_chosen": -66.05341339111328, "logps/ref_rejected": -95.2869873046875, "logps/rejected": -116.05291748046875, "loss": 0.9045, "margin_dpo/margin_mean": 14.837360382080078, "margin_dpo/margin_std": 16.077857971191406, "step": 72 }, { "KL/chosen_KL_mean": -7.825841903686523, "KL/mean": -15.346532821655273, "KL/rejected_KL_mean": -22.867225646972656, "KL/std": 18.097063064575195, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.04731083661317825, "fcm_dpo/delta": -0.2835136353969574, "fcm_dpo/margin": 15.04138469696045, "fcm_dpo/q_t": 0.36181771755218506, "grad_norm": 38.30211639404297, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.6687195301055908, "logits/rejected": -0.6266169548034668, "logps/chosen": -74.08211517333984, "logps/ref_chosen": -66.25627136230469, "logps/ref_rejected": -90.45613098144531, "logps/rejected": -113.3233642578125, "loss": 1.0145, "margin_dpo/margin_mean": 15.041385650634766, "margin_dpo/margin_std": 22.485240936279297, "step": 73 }, { "KL/chosen_KL_mean": -8.45234489440918, "KL/mean": -16.59984588623047, "KL/rejected_KL_mean": -24.74734115600586, "KL/std": 20.056711196899414, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.04465902969241142, "fcm_dpo/delta": -0.30072933435440063, "fcm_dpo/margin": 16.294998168945312, "fcm_dpo/q_t": 0.3568170368671417, "grad_norm": 40.10905075073242, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.648106575012207, "logits/rejected": -0.6394679546356201, "logps/chosen": -61.877227783203125, "logps/ref_chosen": -53.42488098144531, "logps/ref_rejected": -95.94693756103516, "logps/rejected": -120.69427490234375, "loss": 0.9829, "margin_dpo/margin_mean": 16.294998168945312, "margin_dpo/margin_std": 24.927589416503906, "step": 74 }, { "KL/chosen_KL_mean": -8.210355758666992, "KL/mean": -19.646503448486328, "KL/rejected_KL_mean": -31.08264923095703, "KL/std": 20.41864013671875, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.04106065630912781, "fcm_dpo/delta": -0.5371048450469971, "fcm_dpo/margin": 22.872297286987305, "fcm_dpo/q_t": 0.3111739158630371, "grad_norm": 32.669776916503906, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.6666178703308105, "logits/rejected": -0.6588006019592285, "logps/chosen": -60.072017669677734, "logps/ref_chosen": -51.861663818359375, "logps/ref_rejected": -111.25398254394531, "logps/rejected": -142.33663940429688, "loss": 0.8227, "margin_dpo/margin_mean": 22.872297286987305, "margin_dpo/margin_std": 23.01834487915039, "step": 75 }, { "KL/chosen_KL_mean": -10.067512512207031, "KL/mean": -17.29259490966797, "KL/rejected_KL_mean": -24.517669677734375, "KL/std": 16.541284561157227, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.03894951939582825, "fcm_dpo/delta": -0.11947859078645706, "fcm_dpo/margin": 14.450164794921875, "fcm_dpo/q_t": 0.37349021434783936, "grad_norm": 34.09317398071289, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.7095220685005188, "logits/rejected": -0.6687110662460327, "logps/chosen": -63.33354949951172, "logps/ref_chosen": -53.26603698730469, "logps/ref_rejected": -78.21662902832031, "logps/rejected": -102.73429870605469, "loss": 1.0071, "margin_dpo/margin_mean": 14.450165748596191, "margin_dpo/margin_std": 17.991954803466797, "step": 76 }, { "KL/chosen_KL_mean": -8.949975967407227, "KL/mean": -20.767536163330078, "KL/rejected_KL_mean": -32.58510208129883, "KL/std": 22.64974594116211, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.03596869111061096, "fcm_dpo/delta": -0.44180840253829956, "fcm_dpo/margin": 23.635120391845703, "fcm_dpo/q_t": 0.3196060359477997, "grad_norm": 35.29338836669922, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.6948204040527344, "logits/rejected": -0.6578306555747986, "logps/chosen": -67.0466537475586, "logps/ref_chosen": -58.0966796875, "logps/ref_rejected": -93.77361297607422, "logps/rejected": -126.35871887207031, "loss": 0.8707, "margin_dpo/margin_mean": 23.635120391845703, "margin_dpo/margin_std": 24.971656799316406, "step": 77 }, { "KL/chosen_KL_mean": -8.99593734741211, "KL/mean": -17.899391174316406, "KL/rejected_KL_mean": -26.802845001220703, "KL/std": 18.244056701660156, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.03460092842578888, "fcm_dpo/delta": -0.17669706046581268, "fcm_dpo/margin": 17.806903839111328, "fcm_dpo/q_t": 0.3664587140083313, "grad_norm": 31.800643920898438, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.6439747214317322, "logits/rejected": -0.6189943552017212, "logps/chosen": -64.60972595214844, "logps/ref_chosen": -55.61378479003906, "logps/ref_rejected": -84.93436431884766, "logps/rejected": -111.73721313476562, "loss": 0.9754, "margin_dpo/margin_mean": 17.806903839111328, "margin_dpo/margin_std": 20.876663208007812, "step": 78 }, { "KL/chosen_KL_mean": -9.265439987182617, "KL/mean": -19.035083770751953, "KL/rejected_KL_mean": -28.804729461669922, "KL/std": 20.695537567138672, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.03311431035399437, "fcm_dpo/delta": -0.21076560020446777, "fcm_dpo/margin": 19.53928565979004, "fcm_dpo/q_t": 0.36637139320373535, "grad_norm": 27.853378295898438, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.646593451499939, "logits/rejected": -0.6158708333969116, "logps/chosen": -64.71592712402344, "logps/ref_chosen": -55.45048522949219, "logps/ref_rejected": -87.64756774902344, "logps/rejected": -116.4522933959961, "loss": 0.9865, "margin_dpo/margin_mean": 19.539283752441406, "margin_dpo/margin_std": 25.31096649169922, "step": 79 }, { "KL/chosen_KL_mean": -12.27981185913086, "KL/mean": -21.402843475341797, "KL/rejected_KL_mean": -30.52587127685547, "KL/std": 21.86465072631836, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.031817443668842316, "fcm_dpo/delta": -0.14128068089485168, "fcm_dpo/margin": 18.24605941772461, "fcm_dpo/q_t": 0.38100287318229675, "grad_norm": 29.229244232177734, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7156493663787842, "logits/rejected": -0.6729994416236877, "logps/chosen": -70.79910278320312, "logps/ref_chosen": -58.519290924072266, "logps/ref_rejected": -87.54750061035156, "logps/rejected": -118.07337188720703, "loss": 1.0304, "margin_dpo/margin_mean": 18.246057510375977, "margin_dpo/margin_std": 25.342288970947266, "step": 80 }, { "KL/chosen_KL_mean": -11.352493286132812, "KL/mean": -23.801851272583008, "KL/rejected_KL_mean": -36.25121307373047, "KL/std": 25.796279907226562, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.03014964610338211, "fcm_dpo/delta": -0.32812806963920593, "fcm_dpo/margin": 24.898719787597656, "fcm_dpo/q_t": 0.3463107943534851, "grad_norm": 30.754024505615234, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.7107023000717163, "logits/rejected": -0.6817086935043335, "logps/chosen": -77.80136108398438, "logps/ref_chosen": -66.44886779785156, "logps/ref_rejected": -129.66270446777344, "logps/rejected": -165.91390991210938, "loss": 0.9324, "margin_dpo/margin_mean": 24.898719787597656, "margin_dpo/margin_std": 30.64748764038086, "step": 81 }, { "KL/chosen_KL_mean": -13.689064025878906, "KL/mean": -23.61670684814453, "KL/rejected_KL_mean": -33.544349670410156, "KL/std": 23.599985122680664, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.02952047996222973, "fcm_dpo/delta": -0.14547684788703918, "fcm_dpo/margin": 19.855289459228516, "fcm_dpo/q_t": 0.38032636046409607, "grad_norm": 34.90581512451172, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.6471748352050781, "logits/rejected": -0.6358625888824463, "logps/chosen": -65.92144775390625, "logps/ref_chosen": -52.232383728027344, "logps/ref_rejected": -90.74325561523438, "logps/rejected": -124.28759765625, "loss": 1.0777, "margin_dpo/margin_mean": 19.855289459228516, "margin_dpo/margin_std": 32.484397888183594, "step": 82 }, { "KL/chosen_KL_mean": -13.971277236938477, "KL/mean": -25.6943359375, "KL/rejected_KL_mean": -37.41740036010742, "KL/std": 24.552268981933594, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.02794184908270836, "fcm_dpo/delta": -0.22061091661453247, "fcm_dpo/margin": 23.446125030517578, "fcm_dpo/q_t": 0.36134088039398193, "grad_norm": 29.178665161132812, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6698837876319885, "logits/rejected": -0.6350330114364624, "logps/chosen": -69.79866027832031, "logps/ref_chosen": -55.82738494873047, "logps/ref_rejected": -103.71589660644531, "logps/rejected": -141.13330078125, "loss": 0.9742, "margin_dpo/margin_mean": 23.446123123168945, "margin_dpo/margin_std": 29.056337356567383, "step": 83 }, { "KL/chosen_KL_mean": -12.916067123413086, "KL/mean": -22.892362594604492, "KL/rejected_KL_mean": -32.868656158447266, "KL/std": 20.869712829589844, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.027103561908006668, "fcm_dpo/delta": -0.09814468771219254, "fcm_dpo/margin": 19.952594757080078, "fcm_dpo/q_t": 0.37633585929870605, "grad_norm": 27.472469329833984, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.6623414754867554, "logits/rejected": -0.6236976385116577, "logps/chosen": -80.09223937988281, "logps/ref_chosen": -67.1761703491211, "logps/ref_rejected": -87.29859924316406, "logps/rejected": -120.1672592163086, "loss": 0.9969, "margin_dpo/margin_mean": 19.952594757080078, "margin_dpo/margin_std": 21.364559173583984, "step": 84 }, { "KL/chosen_KL_mean": -13.362470626831055, "KL/mean": -23.700489044189453, "KL/rejected_KL_mean": -34.038509368896484, "KL/std": 21.551101684570312, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.026955682784318924, "fcm_dpo/delta": -0.11413806676864624, "fcm_dpo/margin": 20.676042556762695, "fcm_dpo/q_t": 0.37861472368240356, "grad_norm": 27.54196548461914, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.6713258028030396, "logits/rejected": -0.6413929462432861, "logps/chosen": -71.76908874511719, "logps/ref_chosen": -58.4066162109375, "logps/ref_rejected": -78.63880157470703, "logps/rejected": -112.67730712890625, "loss": 1.0181, "margin_dpo/margin_mean": 20.676042556762695, "margin_dpo/margin_std": 25.745641708374023, "step": 85 }, { "KL/chosen_KL_mean": -17.15639305114746, "KL/mean": -28.15932846069336, "KL/rejected_KL_mean": -39.162261962890625, "KL/std": 28.993762969970703, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.02617364190518856, "fcm_dpo/delta": -0.13349324464797974, "fcm_dpo/margin": 22.005878448486328, "fcm_dpo/q_t": 0.3906324803829193, "grad_norm": 41.52824020385742, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6889638304710388, "logits/rejected": -0.676927924156189, "logps/chosen": -73.29385375976562, "logps/ref_chosen": -56.13746643066406, "logps/ref_rejected": -88.12165069580078, "logps/rejected": -127.2839126586914, "loss": 1.1033, "margin_dpo/margin_mean": 22.005874633789062, "margin_dpo/margin_std": 39.980369567871094, "step": 86 }, { "KL/chosen_KL_mean": -16.871139526367188, "KL/mean": -29.736377716064453, "KL/rejected_KL_mean": -42.60161209106445, "KL/std": 28.82452964782715, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.02494674362242222, "fcm_dpo/delta": -0.2069552093744278, "fcm_dpo/margin": 25.73047637939453, "fcm_dpo/q_t": 0.3635217845439911, "grad_norm": 27.312652587890625, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.7120091915130615, "logits/rejected": -0.6871023774147034, "logps/chosen": -72.50723266601562, "logps/ref_chosen": -55.63609313964844, "logps/ref_rejected": -95.46757507324219, "logps/rejected": -138.06918334960938, "loss": 1.0072, "margin_dpo/margin_mean": 25.7304744720459, "margin_dpo/margin_std": 34.88175964355469, "step": 87 }, { "KL/chosen_KL_mean": -21.348556518554688, "KL/mean": -31.000858306884766, "KL/rejected_KL_mean": -40.65315628051758, "KL/std": 29.925575256347656, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.02458009123802185, "fcm_dpo/delta": -0.026618685573339462, "fcm_dpo/margin": 19.304603576660156, "fcm_dpo/q_t": 0.40038639307022095, "grad_norm": 28.89271354675293, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.6645029783248901, "logits/rejected": -0.6577020287513733, "logps/chosen": -95.01970672607422, "logps/ref_chosen": -73.67115020751953, "logps/ref_rejected": -106.70849609375, "logps/rejected": -147.36166381835938, "loss": 1.1144, "margin_dpo/margin_mean": 19.30460548400879, "margin_dpo/margin_std": 31.463354110717773, "step": 88 }, { "KL/chosen_KL_mean": -12.971105575561523, "KL/mean": -23.80228614807129, "KL/rejected_KL_mean": -34.63346862792969, "KL/std": 25.211496353149414, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.024498917162418365, "fcm_dpo/delta": -0.08528416603803635, "fcm_dpo/margin": 21.6623592376709, "fcm_dpo/q_t": 0.3838860094547272, "grad_norm": 25.59996795654297, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7227055430412292, "logits/rejected": -0.6903188228607178, "logps/chosen": -73.59602355957031, "logps/ref_chosen": -60.624916076660156, "logps/ref_rejected": -82.08354949951172, "logps/rejected": -116.7170181274414, "loss": 1.029, "margin_dpo/margin_mean": 21.662357330322266, "margin_dpo/margin_std": 28.733211517333984, "step": 89 }, { "KL/chosen_KL_mean": -16.759502410888672, "KL/mean": -31.41651153564453, "KL/rejected_KL_mean": -46.07352066040039, "KL/std": 34.36729431152344, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.02360946498811245, "fcm_dpo/delta": -0.25962328910827637, "fcm_dpo/margin": 29.314010620117188, "fcm_dpo/q_t": 0.3660517930984497, "grad_norm": 26.869991302490234, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6465960144996643, "logits/rejected": -0.6594746112823486, "logps/chosen": -70.04481506347656, "logps/ref_chosen": -53.285308837890625, "logps/ref_rejected": -111.54470825195312, "logps/rejected": -157.61822509765625, "loss": 1.0136, "margin_dpo/margin_mean": 29.314010620117188, "margin_dpo/margin_std": 42.58348083496094, "step": 90 }, { "KL/chosen_KL_mean": -17.327058792114258, "KL/mean": -29.477928161621094, "KL/rejected_KL_mean": -41.6287956237793, "KL/std": 27.533260345458984, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.022767014801502228, "fcm_dpo/delta": -0.10921620577573776, "fcm_dpo/margin": 24.301740646362305, "fcm_dpo/q_t": 0.3839709460735321, "grad_norm": 26.525238037109375, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6998708844184875, "logits/rejected": -0.6679236888885498, "logps/chosen": -79.13002014160156, "logps/ref_chosen": -61.802955627441406, "logps/ref_rejected": -87.87395477294922, "logps/rejected": -129.50274658203125, "loss": 1.049, "margin_dpo/margin_mean": 24.301738739013672, "margin_dpo/margin_std": 35.562110900878906, "step": 91 }, { "KL/chosen_KL_mean": -15.585250854492188, "KL/mean": -27.81812858581543, "KL/rejected_KL_mean": -40.05100631713867, "KL/std": 28.193967819213867, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.0223391130566597, "fcm_dpo/delta": -0.10221868008375168, "fcm_dpo/margin": 24.46575164794922, "fcm_dpo/q_t": 0.3836935758590698, "grad_norm": 24.357402801513672, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.7078873515129089, "logits/rejected": -0.6847358345985413, "logps/chosen": -67.22602081298828, "logps/ref_chosen": -51.640769958496094, "logps/ref_rejected": -77.88117980957031, "logps/rejected": -117.93218994140625, "loss": 1.0396, "margin_dpo/margin_mean": 24.46575164794922, "margin_dpo/margin_std": 34.062713623046875, "step": 92 }, { "KL/chosen_KL_mean": -17.586727142333984, "KL/mean": -29.87631607055664, "KL/rejected_KL_mean": -42.16590118408203, "KL/std": 26.71668243408203, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.021821390837430954, "fcm_dpo/delta": -0.0911676287651062, "fcm_dpo/margin": 24.579177856445312, "fcm_dpo/q_t": 0.38265323638916016, "grad_norm": 24.912883758544922, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.7338370084762573, "logits/rejected": -0.6846834421157837, "logps/chosen": -70.115966796875, "logps/ref_chosen": -52.529239654541016, "logps/ref_rejected": -77.16075134277344, "logps/rejected": -119.32664489746094, "loss": 1.015, "margin_dpo/margin_mean": 24.57917594909668, "margin_dpo/margin_std": 30.87820816040039, "step": 93 }, { "KL/chosen_KL_mean": -18.805757522583008, "KL/mean": -32.98207092285156, "KL/rejected_KL_mean": -47.158382415771484, "KL/std": 30.474504470825195, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.021062176674604416, "fcm_dpo/delta": -0.1582311987876892, "fcm_dpo/margin": 28.35263442993164, "fcm_dpo/q_t": 0.3686581254005432, "grad_norm": 24.569002151489258, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.6481120586395264, "logits/rejected": -0.6321585178375244, "logps/chosen": -80.02836608886719, "logps/ref_chosen": -61.22261047363281, "logps/ref_rejected": -99.59902954101562, "logps/rejected": -146.75741577148438, "loss": 0.9756, "margin_dpo/margin_mean": 28.35263442993164, "margin_dpo/margin_std": 31.874601364135742, "step": 94 }, { "KL/chosen_KL_mean": -17.881004333496094, "KL/mean": -29.28811264038086, "KL/rejected_KL_mean": -40.695228576660156, "KL/std": 28.964645385742188, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.020857345312833786, "fcm_dpo/delta": -0.02749776653945446, "fcm_dpo/margin": 22.814218521118164, "fcm_dpo/q_t": 0.3923851251602173, "grad_norm": 23.259233474731445, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.6979963779449463, "logits/rejected": -0.6698317527770996, "logps/chosen": -70.40464782714844, "logps/ref_chosen": -52.523643493652344, "logps/ref_rejected": -75.8803482055664, "logps/rejected": -116.57557678222656, "loss": 1.0577, "margin_dpo/margin_mean": 22.81421661376953, "margin_dpo/margin_std": 30.32460594177246, "step": 95 }, { "KL/chosen_KL_mean": -17.336549758911133, "KL/mean": -32.80950927734375, "KL/rejected_KL_mean": -48.282470703125, "KL/std": 33.05143737792969, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.02017274498939514, "fcm_dpo/delta": -0.18848399817943573, "fcm_dpo/margin": 30.945920944213867, "fcm_dpo/q_t": 0.365802526473999, "grad_norm": 24.085433959960938, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7053977251052856, "logits/rejected": -0.6761200428009033, "logps/chosen": -79.49352264404297, "logps/ref_chosen": -62.15697479248047, "logps/ref_rejected": -96.59601593017578, "logps/rejected": -144.87847900390625, "loss": 0.967, "margin_dpo/margin_mean": 30.945919036865234, "margin_dpo/margin_std": 36.34577560424805, "step": 96 }, { "KL/chosen_KL_mean": -18.55299186706543, "KL/mean": -30.541301727294922, "KL/rejected_KL_mean": -42.52961349487305, "KL/std": 27.393390655517578, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.020200129598379135, "fcm_dpo/delta": -0.03740522265434265, "fcm_dpo/margin": 23.976627349853516, "fcm_dpo/q_t": 0.3891039192676544, "grad_norm": 24.69647789001465, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.6304230690002441, "logits/rejected": -0.5861343145370483, "logps/chosen": -73.19935607910156, "logps/ref_chosen": -54.646366119384766, "logps/ref_rejected": -76.96475219726562, "logps/rejected": -119.49436950683594, "loss": 1.0367, "margin_dpo/margin_mean": 23.97662925720215, "margin_dpo/margin_std": 27.26099967956543, "step": 97 }, { "KL/chosen_KL_mean": -22.866867065429688, "KL/mean": -36.578529357910156, "KL/rejected_KL_mean": -50.29020690917969, "KL/std": 31.22027587890625, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.01966441236436367, "fcm_dpo/delta": -0.09459578990936279, "fcm_dpo/margin": 27.423343658447266, "fcm_dpo/q_t": 0.37884292006492615, "grad_norm": 25.7805233001709, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6840416193008423, "logits/rejected": -0.6680232286453247, "logps/chosen": -88.12549591064453, "logps/ref_chosen": -65.25862884521484, "logps/ref_rejected": -96.5274887084961, "logps/rejected": -146.81768798828125, "loss": 1.0413, "margin_dpo/margin_mean": 27.423343658447266, "margin_dpo/margin_std": 37.329193115234375, "step": 98 }, { "KL/chosen_KL_mean": -18.138586044311523, "KL/mean": -34.79328918457031, "KL/rejected_KL_mean": -51.44799041748047, "KL/std": 33.86750411987305, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.01898646354675293, "fcm_dpo/delta": -0.1955394297838211, "fcm_dpo/margin": 33.30940246582031, "fcm_dpo/q_t": 0.36740559339523315, "grad_norm": 22.372419357299805, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.656816840171814, "logits/rejected": -0.6575205326080322, "logps/chosen": -63.777069091796875, "logps/ref_chosen": -45.638484954833984, "logps/ref_rejected": -86.43793487548828, "logps/rejected": -137.88592529296875, "loss": 0.9756, "margin_dpo/margin_mean": 33.30940246582031, "margin_dpo/margin_std": 42.03821563720703, "step": 99 }, { "KL/chosen_KL_mean": -20.17072868347168, "KL/mean": -29.92642593383789, "KL/rejected_KL_mean": -39.6821174621582, "KL/std": 27.05113983154297, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.019159015268087387, "fcm_dpo/delta": 0.0783969983458519, "fcm_dpo/margin": 19.511388778686523, "fcm_dpo/q_t": 0.41671568155288696, "grad_norm": 25.289337158203125, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6635388135910034, "logits/rejected": -0.6208417415618896, "logps/chosen": -77.76470947265625, "logps/ref_chosen": -57.59397888183594, "logps/ref_rejected": -74.06021118164062, "logps/rejected": -113.74232482910156, "loss": 1.155, "margin_dpo/margin_mean": 19.511388778686523, "margin_dpo/margin_std": 34.8048210144043, "step": 100 }, { "KL/chosen_KL_mean": -24.82254409790039, "KL/mean": -36.69569396972656, "KL/rejected_KL_mean": -48.568843841552734, "KL/std": 33.368690490722656, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.01902778260409832, "fcm_dpo/delta": -0.0026979856193065643, "fcm_dpo/margin": 23.746295928955078, "fcm_dpo/q_t": 0.4053969383239746, "grad_norm": 25.20921516418457, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.6735844612121582, "logits/rejected": -0.6423518657684326, "logps/chosen": -86.47139739990234, "logps/ref_chosen": -61.64885330200195, "logps/ref_rejected": -83.18968200683594, "logps/rejected": -131.75852966308594, "loss": 1.116, "margin_dpo/margin_mean": 23.746295928955078, "margin_dpo/margin_std": 40.306617736816406, "step": 101 }, { "KL/chosen_KL_mean": -26.947221755981445, "KL/mean": -35.72987747192383, "KL/rejected_KL_mean": -44.512535095214844, "KL/std": 31.34524917602539, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.019104380160570145, "fcm_dpo/delta": -0.019020358100533485, "fcm_dpo/margin": 17.565311431884766, "fcm_dpo/q_t": 0.4265472888946533, "grad_norm": 29.04865837097168, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.6863038539886475, "logits/rejected": -0.6410502195358276, "logps/chosen": -91.0261001586914, "logps/ref_chosen": -64.0788803100586, "logps/ref_rejected": -68.18707275390625, "logps/rejected": -112.69961547851562, "loss": 1.1984, "margin_dpo/margin_mean": 17.565311431884766, "margin_dpo/margin_std": 36.97379684448242, "step": 102 }, { "KL/chosen_KL_mean": -21.086196899414062, "KL/mean": -38.403202056884766, "KL/rejected_KL_mean": -55.7202033996582, "KL/std": 34.654380798339844, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.018466107547283173, "fcm_dpo/delta": -0.20299461483955383, "fcm_dpo/margin": 34.63400650024414, "fcm_dpo/q_t": 0.3635409474372864, "grad_norm": 24.053295135498047, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6788144111633301, "logits/rejected": -0.6451829671859741, "logps/chosen": -82.3854751586914, "logps/ref_chosen": -61.299278259277344, "logps/ref_rejected": -93.57270812988281, "logps/rejected": -149.29290771484375, "loss": 0.965, "margin_dpo/margin_mean": 34.63400650024414, "margin_dpo/margin_std": 41.585411071777344, "step": 103 }, { "KL/chosen_KL_mean": -23.070478439331055, "KL/mean": -39.50807571411133, "KL/rejected_KL_mean": -55.94567108154297, "KL/std": 38.31330871582031, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.01794702559709549, "fcm_dpo/delta": -0.14857444167137146, "fcm_dpo/margin": 32.87519836425781, "fcm_dpo/q_t": 0.37896206974983215, "grad_norm": 23.492591857910156, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.692119836807251, "logits/rejected": -0.6608579158782959, "logps/chosen": -77.44325256347656, "logps/ref_chosen": -54.372772216796875, "logps/ref_rejected": -89.5647201538086, "logps/rejected": -145.51039123535156, "loss": 1.0276, "margin_dpo/margin_mean": 32.87519454956055, "margin_dpo/margin_std": 46.670677185058594, "step": 104 }, { "KL/chosen_KL_mean": -21.43729591369629, "KL/mean": -44.80780792236328, "KL/rejected_KL_mean": -68.1783218383789, "KL/std": 37.48930740356445, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.016852889209985733, "fcm_dpo/delta": -0.3663092255592346, "fcm_dpo/margin": 46.741024017333984, "fcm_dpo/q_t": 0.3251432180404663, "grad_norm": 23.383228302001953, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.6901790499687195, "logits/rejected": -0.6680899858474731, "logps/chosen": -76.07624053955078, "logps/ref_chosen": -54.638946533203125, "logps/ref_rejected": -97.97351837158203, "logps/rejected": -166.15184020996094, "loss": 0.8472, "margin_dpo/margin_mean": 46.741024017333984, "margin_dpo/margin_std": 41.229366302490234, "step": 105 }, { "KL/chosen_KL_mean": -24.016298294067383, "KL/mean": -38.636592864990234, "KL/rejected_KL_mean": -53.25688934326172, "KL/std": 32.439598083496094, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.016409728676080704, "fcm_dpo/delta": -0.03146550431847572, "fcm_dpo/margin": 29.24058723449707, "fcm_dpo/q_t": 0.3909507989883423, "grad_norm": 23.02033233642578, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.6366252899169922, "logits/rejected": -0.6048162579536438, "logps/chosen": -78.84919738769531, "logps/ref_chosen": -54.83289337158203, "logps/ref_rejected": -85.22461700439453, "logps/rejected": -138.48150634765625, "loss": 1.0408, "margin_dpo/margin_mean": 29.24058723449707, "margin_dpo/margin_std": 35.340091705322266, "step": 106 }, { "KL/chosen_KL_mean": -28.679424285888672, "KL/mean": -45.09428405761719, "KL/rejected_KL_mean": -61.50914001464844, "KL/std": 39.42516326904297, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.016135554760694504, "fcm_dpo/delta": -0.084980808198452, "fcm_dpo/margin": 32.82972717285156, "fcm_dpo/q_t": 0.3815375864505768, "grad_norm": 22.050548553466797, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.671083390712738, "logits/rejected": -0.6238175630569458, "logps/chosen": -98.38723754882812, "logps/ref_chosen": -69.70780944824219, "logps/ref_rejected": -94.73950958251953, "logps/rejected": -156.2486572265625, "loss": 1.0287, "margin_dpo/margin_mean": 32.82972717285156, "margin_dpo/margin_std": 41.5906982421875, "step": 107 }, { "KL/chosen_KL_mean": -25.770402908325195, "KL/mean": -45.36616516113281, "KL/rejected_KL_mean": -64.9619369506836, "KL/std": 43.32636642456055, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.015721427276730537, "fcm_dpo/delta": -0.17703697085380554, "fcm_dpo/margin": 39.1915283203125, "fcm_dpo/q_t": 0.3744971752166748, "grad_norm": 22.43386459350586, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.6669068336486816, "logits/rejected": -0.6445153951644897, "logps/chosen": -81.78028869628906, "logps/ref_chosen": -56.0098876953125, "logps/ref_rejected": -95.79601287841797, "logps/rejected": -160.75794982910156, "loss": 1.0232, "margin_dpo/margin_mean": 39.1915283203125, "margin_dpo/margin_std": 55.913177490234375, "step": 108 }, { "KL/chosen_KL_mean": -23.891096115112305, "KL/mean": -44.87135314941406, "KL/rejected_KL_mean": -65.85160827636719, "KL/std": 41.50886535644531, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.015076124109327793, "fcm_dpo/delta": -0.19590097665786743, "fcm_dpo/margin": 41.96051788330078, "fcm_dpo/q_t": 0.3611322045326233, "grad_norm": 23.130586624145508, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.6083803176879883, "logits/rejected": -0.5868882536888123, "logps/chosen": -86.77658081054688, "logps/ref_chosen": -62.88549041748047, "logps/ref_rejected": -98.68573760986328, "logps/rejected": -164.537353515625, "loss": 0.9673, "margin_dpo/margin_mean": 41.96051788330078, "margin_dpo/margin_std": 48.454368591308594, "step": 109 }, { "KL/chosen_KL_mean": -24.484817504882812, "KL/mean": -43.37567901611328, "KL/rejected_KL_mean": -62.26654815673828, "KL/std": 41.369407653808594, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.014548167586326599, "fcm_dpo/delta": -0.11019857972860336, "fcm_dpo/margin": 37.78173065185547, "fcm_dpo/q_t": 0.38073986768722534, "grad_norm": 20.18339729309082, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.6229938268661499, "logits/rejected": -0.5868622064590454, "logps/chosen": -83.2385025024414, "logps/ref_chosen": -58.753684997558594, "logps/ref_rejected": -79.75001525878906, "logps/rejected": -142.01657104492188, "loss": 1.0352, "margin_dpo/margin_mean": 37.78173065185547, "margin_dpo/margin_std": 49.614540100097656, "step": 110 }, { "KL/chosen_KL_mean": -28.117158889770508, "KL/mean": -49.45366668701172, "KL/rejected_KL_mean": -70.79017639160156, "KL/std": 46.68696594238281, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.014208821579813957, "fcm_dpo/delta": -0.16805897653102875, "fcm_dpo/margin": 42.67301940917969, "fcm_dpo/q_t": 0.3700113892555237, "grad_norm": 22.384057998657227, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.6600044965744019, "logits/rejected": -0.6378269195556641, "logps/chosen": -96.74126434326172, "logps/ref_chosen": -68.62410736083984, "logps/ref_rejected": -98.42886352539062, "logps/rejected": -169.2190399169922, "loss": 1.0153, "margin_dpo/margin_mean": 42.67301940917969, "margin_dpo/margin_std": 57.135040283203125, "step": 111 }, { "KL/chosen_KL_mean": -25.888656616210938, "KL/mean": -39.50413131713867, "KL/rejected_KL_mean": -53.119606018066406, "KL/std": 32.67273712158203, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.01424017921090126, "fcm_dpo/delta": 0.06461413204669952, "fcm_dpo/margin": 27.23094940185547, "fcm_dpo/q_t": 0.41214606165885925, "grad_norm": 20.815696716308594, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.6311874985694885, "logits/rejected": -0.5913982391357422, "logps/chosen": -76.13829803466797, "logps/ref_chosen": -50.24964141845703, "logps/ref_rejected": -64.77442932128906, "logps/rejected": -117.89403533935547, "loss": 1.1106, "margin_dpo/margin_mean": 27.23094940185547, "margin_dpo/margin_std": 39.252777099609375, "step": 112 }, { "KL/chosen_KL_mean": -32.33839416503906, "KL/mean": -47.10094451904297, "KL/rejected_KL_mean": -61.863487243652344, "KL/std": 33.731163024902344, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.014409849420189857, "fcm_dpo/delta": 0.02563050016760826, "fcm_dpo/margin": 29.525096893310547, "fcm_dpo/q_t": 0.4029633700847626, "grad_norm": 21.0351505279541, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.5959609746932983, "logits/rejected": -0.5505021810531616, "logps/chosen": -99.05134582519531, "logps/ref_chosen": -66.71295166015625, "logps/ref_rejected": -77.96870422363281, "logps/rejected": -139.83218383789062, "loss": 1.0727, "margin_dpo/margin_mean": 29.525096893310547, "margin_dpo/margin_std": 37.40425109863281, "step": 113 }, { "KL/chosen_KL_mean": -29.493261337280273, "KL/mean": -52.80317306518555, "KL/rejected_KL_mean": -76.11308288574219, "KL/std": 50.801734924316406, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.013878151774406433, "fcm_dpo/delta": -0.21314632892608643, "fcm_dpo/margin": 46.619815826416016, "fcm_dpo/q_t": 0.36664098501205444, "grad_norm": 22.067481994628906, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.624281644821167, "logits/rejected": -0.5919479131698608, "logps/chosen": -87.2783432006836, "logps/ref_chosen": -57.78507995605469, "logps/ref_rejected": -87.10966491699219, "logps/rejected": -163.22274780273438, "loss": 0.9781, "margin_dpo/margin_mean": 46.61981201171875, "margin_dpo/margin_std": 61.55342483520508, "step": 114 }, { "KL/chosen_KL_mean": -38.72612380981445, "KL/mean": -54.643898010253906, "KL/rejected_KL_mean": -70.56167602539062, "KL/std": 48.15009307861328, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.013882124796509743, "fcm_dpo/delta": 0.00821240060031414, "fcm_dpo/margin": 31.83555030822754, "fcm_dpo/q_t": 0.408986359834671, "grad_norm": 27.751873016357422, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.6329885721206665, "logits/rejected": -0.6110441088676453, "logps/chosen": -104.30876159667969, "logps/ref_chosen": -65.5826416015625, "logps/ref_rejected": -98.56552124023438, "logps/rejected": -169.127197265625, "loss": 1.1543, "margin_dpo/margin_mean": 31.835548400878906, "margin_dpo/margin_std": 63.04792022705078, "step": 115 }, { "KL/chosen_KL_mean": -28.92960548400879, "KL/mean": -46.47405242919922, "KL/rejected_KL_mean": -64.01849365234375, "KL/std": 40.528472900390625, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.013833288103342056, "fcm_dpo/delta": -0.03747085481882095, "fcm_dpo/margin": 35.08888626098633, "fcm_dpo/q_t": 0.39196979999542236, "grad_norm": 22.5377254486084, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.6246926784515381, "logits/rejected": -0.6135732531547546, "logps/chosen": -80.32991790771484, "logps/ref_chosen": -51.40031433105469, "logps/ref_rejected": -80.5218505859375, "logps/rejected": -144.54034423828125, "loss": 1.0413, "margin_dpo/margin_mean": 35.08888626098633, "margin_dpo/margin_std": 43.86783218383789, "step": 116 }, { "KL/chosen_KL_mean": -36.64705276489258, "KL/mean": -51.39534378051758, "KL/rejected_KL_mean": -66.14363098144531, "KL/std": 43.24048614501953, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.013833757489919662, "fcm_dpo/delta": 0.04373066499829292, "fcm_dpo/margin": 29.496578216552734, "fcm_dpo/q_t": 0.40985727310180664, "grad_norm": 30.1611328125, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.550119161605835, "logits/rejected": -0.507682204246521, "logps/chosen": -105.94546508789062, "logps/ref_chosen": -69.29840850830078, "logps/ref_rejected": -66.583984375, "logps/rejected": -132.72763061523438, "loss": 1.1218, "margin_dpo/margin_mean": 29.496578216552734, "margin_dpo/margin_std": 48.89860916137695, "step": 117 }, { "KL/chosen_KL_mean": -29.291458129882812, "KL/mean": -46.83741760253906, "KL/rejected_KL_mean": -64.38337707519531, "KL/std": 39.068748474121094, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.013796643353998661, "fcm_dpo/delta": -0.035874899476766586, "fcm_dpo/margin": 35.0919189453125, "fcm_dpo/q_t": 0.392580509185791, "grad_norm": 21.916383743286133, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.655544638633728, "logits/rejected": -0.6371685266494751, "logps/chosen": -84.93243408203125, "logps/ref_chosen": -55.6409797668457, "logps/ref_rejected": -75.66905975341797, "logps/rejected": -140.05242919921875, "loss": 1.0436, "margin_dpo/margin_mean": 35.0919189453125, "margin_dpo/margin_std": 44.61895751953125, "step": 118 }, { "KL/chosen_KL_mean": -40.089935302734375, "KL/mean": -57.477745056152344, "KL/rejected_KL_mean": -74.86555480957031, "KL/std": 43.875762939453125, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.013616050593554974, "fcm_dpo/delta": -0.025850612670183182, "fcm_dpo/margin": 34.7756233215332, "fcm_dpo/q_t": 0.3989412188529968, "grad_norm": 23.656314849853516, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.6302667260169983, "logits/rejected": -0.6170446872711182, "logps/chosen": -113.60012817382812, "logps/ref_chosen": -73.51019287109375, "logps/ref_rejected": -102.977294921875, "logps/rejected": -177.84283447265625, "loss": 1.0851, "margin_dpo/margin_mean": 34.7756233215332, "margin_dpo/margin_std": 53.304588317871094, "step": 119 }, { "KL/chosen_KL_mean": -41.21659469604492, "KL/mean": -65.18665313720703, "KL/rejected_KL_mean": -89.15672302246094, "KL/std": 54.75802230834961, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.013283832930028439, "fcm_dpo/delta": -0.1996966004371643, "fcm_dpo/margin": 47.94011688232422, "fcm_dpo/q_t": 0.3626715838909149, "grad_norm": 23.151220321655273, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.6102467775344849, "logits/rejected": -0.5748085975646973, "logps/chosen": -117.99742889404297, "logps/ref_chosen": -76.78083801269531, "logps/ref_rejected": -108.02374267578125, "logps/rejected": -197.1804656982422, "loss": 0.9707, "margin_dpo/margin_mean": 47.94011688232422, "margin_dpo/margin_std": 57.357337951660156, "step": 120 }, { "KL/chosen_KL_mean": -40.084228515625, "KL/mean": -66.49555969238281, "KL/rejected_KL_mean": -92.90689086914062, "KL/std": 53.775028228759766, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.012637857347726822, "fcm_dpo/delta": -0.23500870168209076, "fcm_dpo/margin": 52.822654724121094, "fcm_dpo/q_t": 0.35974642634391785, "grad_norm": 25.187885284423828, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.5842880010604858, "logits/rejected": -0.5794540643692017, "logps/chosen": -101.87413024902344, "logps/ref_chosen": -61.789894104003906, "logps/ref_rejected": -109.99456787109375, "logps/rejected": -202.90145874023438, "loss": 0.9767, "margin_dpo/margin_mean": 52.822654724121094, "margin_dpo/margin_std": 67.26119995117188, "step": 121 }, { "KL/chosen_KL_mean": -36.656639099121094, "KL/mean": -69.3306655883789, "KL/rejected_KL_mean": -102.00469970703125, "KL/std": 61.23943328857422, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.011914310976862907, "fcm_dpo/delta": -0.3561996519565582, "fcm_dpo/margin": 65.34806060791016, "fcm_dpo/q_t": 0.3318769931793213, "grad_norm": 24.457799911499023, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.5659698843955994, "logits/rejected": -0.581313967704773, "logps/chosen": -83.55885314941406, "logps/ref_chosen": -46.9022102355957, "logps/ref_rejected": -106.71418762207031, "logps/rejected": -208.71888732910156, "loss": 0.8701, "margin_dpo/margin_mean": 65.34805297851562, "margin_dpo/margin_std": 63.61327362060547, "step": 122 }, { "KL/chosen_KL_mean": -39.99665069580078, "KL/mean": -61.610984802246094, "KL/rejected_KL_mean": -83.22531127929688, "KL/std": 52.34938430786133, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.011532999575138092, "fcm_dpo/delta": -0.05208038166165352, "fcm_dpo/margin": 43.228668212890625, "fcm_dpo/q_t": 0.3948003053665161, "grad_norm": 21.98365592956543, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.574738085269928, "logits/rejected": -0.556842565536499, "logps/chosen": -101.33528900146484, "logps/ref_chosen": -61.33863830566406, "logps/ref_rejected": -87.775390625, "logps/rejected": -171.00070190429688, "loss": 1.092, "margin_dpo/margin_mean": 43.228668212890625, "margin_dpo/margin_std": 69.88395690917969, "step": 123 }, { "KL/chosen_KL_mean": -45.20293426513672, "KL/mean": -76.88467407226562, "KL/rejected_KL_mean": -108.56641387939453, "KL/std": 70.95067596435547, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.011075865477323532, "fcm_dpo/delta": -0.2719235420227051, "fcm_dpo/margin": 63.36347198486328, "fcm_dpo/q_t": 0.36204931139945984, "grad_norm": 23.336181640625, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.5531664490699768, "logits/rejected": -0.5544106960296631, "logps/chosen": -116.6512680053711, "logps/ref_chosen": -71.44833374023438, "logps/ref_rejected": -117.58056640625, "logps/rejected": -226.14697265625, "loss": 0.9951, "margin_dpo/margin_mean": 63.36347198486328, "margin_dpo/margin_std": 89.0818099975586, "step": 124 }, { "KL/chosen_KL_mean": -37.31425476074219, "KL/mean": -65.44857788085938, "KL/rejected_KL_mean": -93.58290100097656, "KL/std": 59.623497009277344, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.010716570541262627, "fcm_dpo/delta": -0.16258446872234344, "fcm_dpo/margin": 56.268646240234375, "fcm_dpo/q_t": 0.37255388498306274, "grad_norm": 20.094148635864258, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.513544499874115, "logits/rejected": -0.5137706995010376, "logps/chosen": -87.4511947631836, "logps/ref_chosen": -50.136940002441406, "logps/ref_rejected": -83.98861694335938, "logps/rejected": -177.57150268554688, "loss": 1.0046, "margin_dpo/margin_mean": 56.26864242553711, "margin_dpo/margin_std": 73.41596984863281, "step": 125 }, { "KL/chosen_KL_mean": -40.404869079589844, "KL/mean": -66.45927429199219, "KL/rejected_KL_mean": -92.51365661621094, "KL/std": 53.786094665527344, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.010430238209664822, "fcm_dpo/delta": -0.09915146231651306, "fcm_dpo/margin": 52.10879898071289, "fcm_dpo/q_t": 0.3821665048599243, "grad_norm": 21.80858039855957, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.545072078704834, "logits/rejected": -0.5383429527282715, "logps/chosen": -96.07194519042969, "logps/ref_chosen": -55.66706848144531, "logps/ref_rejected": -98.1297607421875, "logps/rejected": -190.64341735839844, "loss": 1.0239, "margin_dpo/margin_mean": 52.10879898071289, "margin_dpo/margin_std": 67.88806915283203, "step": 126 }, { "KL/chosen_KL_mean": -39.16326904296875, "KL/mean": -58.659423828125, "KL/rejected_KL_mean": -78.15557861328125, "KL/std": 47.13392639160156, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.010470317676663399, "fcm_dpo/delta": 0.043486855924129486, "fcm_dpo/margin": 38.9923095703125, "fcm_dpo/q_t": 0.406957745552063, "grad_norm": 21.821304321289062, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.5330368280410767, "logits/rejected": -0.5118523240089417, "logps/chosen": -95.71794891357422, "logps/ref_chosen": -56.55467987060547, "logps/ref_rejected": -76.7957763671875, "logps/rejected": -154.95135498046875, "loss": 1.1078, "margin_dpo/margin_mean": 38.9923095703125, "margin_dpo/margin_std": 58.90929412841797, "step": 127 }, { "KL/chosen_KL_mean": -42.080467224121094, "KL/mean": -62.39935302734375, "KL/rejected_KL_mean": -82.71824645996094, "KL/std": 55.34471130371094, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.010566205717623234, "fcm_dpo/delta": 0.02107478305697441, "fcm_dpo/margin": 40.63777160644531, "fcm_dpo/q_t": 0.4061205983161926, "grad_norm": 30.90243148803711, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.5452651977539062, "logits/rejected": -0.5237852334976196, "logps/chosen": -100.20142364501953, "logps/ref_chosen": -58.12095642089844, "logps/ref_rejected": -76.43896484375, "logps/rejected": -159.15721130371094, "loss": 1.1231, "margin_dpo/margin_mean": 40.637779235839844, "margin_dpo/margin_std": 68.70756530761719, "step": 128 }, { "KL/chosen_KL_mean": -50.868858337402344, "KL/mean": -72.29450988769531, "KL/rejected_KL_mean": -93.72015380859375, "KL/std": 60.7120361328125, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.010539564304053783, "fcm_dpo/delta": -0.0017273910343647003, "fcm_dpo/margin": 42.851295471191406, "fcm_dpo/q_t": 0.40687721967697144, "grad_norm": 22.193269729614258, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.5510047674179077, "logits/rejected": -0.53895103931427, "logps/chosen": -117.78523254394531, "logps/ref_chosen": -66.91637420654297, "logps/ref_rejected": -96.6422119140625, "logps/rejected": -190.36236572265625, "loss": 1.1293, "margin_dpo/margin_mean": 42.85129928588867, "margin_dpo/margin_std": 78.5627212524414, "step": 129 }, { "KL/chosen_KL_mean": -37.846351623535156, "KL/mean": -69.90611267089844, "KL/rejected_KL_mean": -101.96586608886719, "KL/std": 63.394935607910156, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.010300353169441223, "fcm_dpo/delta": -0.22541257739067078, "fcm_dpo/margin": 64.11952209472656, "fcm_dpo/q_t": 0.3606530427932739, "grad_norm": 21.44586753845215, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.5086920261383057, "logits/rejected": -0.5030478239059448, "logps/chosen": -82.51319885253906, "logps/ref_chosen": -44.66685104370117, "logps/ref_rejected": -82.78165435791016, "logps/rejected": -184.74752807617188, "loss": 0.9727, "margin_dpo/margin_mean": 64.11951446533203, "margin_dpo/margin_std": 76.44377136230469, "step": 130 }, { "KL/chosen_KL_mean": -33.34912872314453, "KL/mean": -65.05560302734375, "KL/rejected_KL_mean": -96.7620849609375, "KL/std": 61.83128356933594, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.009780805557966232, "fcm_dpo/delta": -0.1818436086177826, "fcm_dpo/margin": 63.41294860839844, "fcm_dpo/q_t": 0.36090847849845886, "grad_norm": 28.479732513427734, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.5144104957580566, "logits/rejected": -0.5210058689117432, "logps/chosen": -78.27371978759766, "logps/ref_chosen": -44.924591064453125, "logps/ref_rejected": -88.44401550292969, "logps/rejected": -185.20608520507812, "loss": 0.9571, "margin_dpo/margin_mean": 63.41294860839844, "margin_dpo/margin_std": 67.7553939819336, "step": 131 }, { "KL/chosen_KL_mean": -44.536895751953125, "KL/mean": -71.02592468261719, "KL/rejected_KL_mean": -97.51495361328125, "KL/std": 62.367454528808594, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.00959692895412445, "fcm_dpo/delta": -0.061767518520355225, "fcm_dpo/margin": 52.97805404663086, "fcm_dpo/q_t": 0.39321261644363403, "grad_norm": 21.923797607421875, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.53538578748703, "logits/rejected": -0.5309890508651733, "logps/chosen": -103.53797912597656, "logps/ref_chosen": -59.00108337402344, "logps/ref_rejected": -87.89215087890625, "logps/rejected": -185.4071044921875, "loss": 1.0649, "margin_dpo/margin_mean": 52.978057861328125, "margin_dpo/margin_std": 80.51556396484375, "step": 132 }, { "KL/chosen_KL_mean": -52.77239990234375, "KL/mean": -76.22706604003906, "KL/rejected_KL_mean": -99.68174743652344, "KL/std": 56.50593185424805, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.009568007662892342, "fcm_dpo/delta": 0.0011416934430599213, "fcm_dpo/margin": 46.909339904785156, "fcm_dpo/q_t": 0.40291664004325867, "grad_norm": 27.488916397094727, "learning_rate": 4.870401618977415e-07, "logits/chosen": -0.549635648727417, "logits/rejected": -0.5361599922180176, "logps/chosen": -119.37689971923828, "logps/ref_chosen": -66.60449981689453, "logps/ref_rejected": -96.33355712890625, "logps/rejected": -196.0153045654297, "loss": 1.0885, "margin_dpo/margin_mean": 46.909339904785156, "margin_dpo/margin_std": 71.18951416015625, "step": 133 }, { "KL/chosen_KL_mean": -41.90770721435547, "KL/mean": -66.91332244873047, "KL/rejected_KL_mean": -91.91893768310547, "KL/std": 55.141109466552734, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.009577769786119461, "fcm_dpo/delta": -0.030592577531933784, "fcm_dpo/margin": 50.01123046875, "fcm_dpo/q_t": 0.39385026693344116, "grad_norm": 20.826383590698242, "learning_rate": 4.866292092063986e-07, "logits/chosen": -0.49232274293899536, "logits/rejected": -0.4757571816444397, "logps/chosen": -93.97695922851562, "logps/ref_chosen": -52.06925582885742, "logps/ref_rejected": -87.6545181274414, "logps/rejected": -179.57345581054688, "loss": 1.0433, "margin_dpo/margin_mean": 50.01123046875, "margin_dpo/margin_std": 63.30561828613281, "step": 134 }, { "KL/chosen_KL_mean": -46.68389892578125, "KL/mean": -82.07420349121094, "KL/rejected_KL_mean": -117.46450805664062, "KL/std": 72.77279663085938, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.009226776659488678, "fcm_dpo/delta": -0.21729746460914612, "fcm_dpo/margin": 70.78060913085938, "fcm_dpo/q_t": 0.362811803817749, "grad_norm": 23.35005760192871, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.5128329396247864, "logits/rejected": -0.5463232398033142, "logps/chosen": -97.03776550292969, "logps/ref_chosen": -50.353858947753906, "logps/ref_rejected": -115.97975158691406, "logps/rejected": -233.44424438476562, "loss": 0.9735, "margin_dpo/margin_mean": 70.78060913085938, "margin_dpo/margin_std": 87.45381164550781, "step": 135 }, { "KL/chosen_KL_mean": -55.22693634033203, "KL/mean": -79.05845642089844, "KL/rejected_KL_mean": -102.88998413085938, "KL/std": 65.8864517211914, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.009088754653930664, "fcm_dpo/delta": 0.016819309443235397, "fcm_dpo/margin": 47.663047790527344, "fcm_dpo/q_t": 0.4126465916633606, "grad_norm": 21.6840877532959, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.5195415616035461, "logits/rejected": -0.510654091835022, "logps/chosen": -120.29945373535156, "logps/ref_chosen": -65.072509765625, "logps/ref_rejected": -96.32122802734375, "logps/rejected": -199.21121215820312, "loss": 1.124, "margin_dpo/margin_mean": 47.663047790527344, "margin_dpo/margin_std": 83.49158477783203, "step": 136 }, { "KL/chosen_KL_mean": -53.765201568603516, "KL/mean": -90.72117614746094, "KL/rejected_KL_mean": -127.67715454101562, "KL/std": 87.34907531738281, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.00885041244328022, "fcm_dpo/delta": -0.21880337595939636, "fcm_dpo/margin": 73.91194152832031, "fcm_dpo/q_t": 0.3690844178199768, "grad_norm": 19.313182830810547, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.4476552903652191, "logits/rejected": -0.4745860695838928, "logps/chosen": -102.52432250976562, "logps/ref_chosen": -48.759117126464844, "logps/ref_rejected": -113.86376953125, "logps/rejected": -241.54092407226562, "loss": 1.006, "margin_dpo/margin_mean": 73.91194152832031, "margin_dpo/margin_std": 104.03363037109375, "step": 137 }, { "KL/chosen_KL_mean": -55.631492614746094, "KL/mean": -83.80963134765625, "KL/rejected_KL_mean": -111.98777770996094, "KL/std": 66.54393005371094, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.008722840808331966, "fcm_dpo/delta": -0.04372715577483177, "fcm_dpo/margin": 56.356292724609375, "fcm_dpo/q_t": 0.3889402449131012, "grad_norm": 22.730175018310547, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.4589083194732666, "logits/rejected": -0.4466325640678406, "logps/chosen": -116.15113830566406, "logps/ref_chosen": -60.519649505615234, "logps/ref_rejected": -93.19694519042969, "logps/rejected": -205.18472290039062, "loss": 1.0345, "margin_dpo/margin_mean": 56.35628890991211, "margin_dpo/margin_std": 68.2387466430664, "step": 138 }, { "KL/chosen_KL_mean": -46.2087516784668, "KL/mean": -80.24526977539062, "KL/rejected_KL_mean": -114.28178405761719, "KL/std": 63.59034729003906, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.008530115708708763, "fcm_dpo/delta": -0.13851620256900787, "fcm_dpo/margin": 68.07303619384766, "fcm_dpo/q_t": 0.3731107711791992, "grad_norm": 18.695363998413086, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.4649140238761902, "logits/rejected": -0.454061895608902, "logps/chosen": -93.10013580322266, "logps/ref_chosen": -46.89138412475586, "logps/ref_rejected": -79.72798156738281, "logps/rejected": -194.009765625, "loss": 0.9904, "margin_dpo/margin_mean": 68.07304382324219, "margin_dpo/margin_std": 81.53546905517578, "step": 139 }, { "KL/chosen_KL_mean": -56.09660339355469, "KL/mean": -87.04917907714844, "KL/rejected_KL_mean": -118.00175476074219, "KL/std": 71.19284057617188, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.008354827761650085, "fcm_dpo/delta": -0.07093564420938492, "fcm_dpo/margin": 61.90514373779297, "fcm_dpo/q_t": 0.3871724605560303, "grad_norm": 21.784423828125, "learning_rate": 4.840329401637809e-07, "logits/chosen": -0.4650196135044098, "logits/rejected": -0.4507848024368286, "logps/chosen": -115.07131958007812, "logps/ref_chosen": -58.97471618652344, "logps/ref_rejected": -83.28410339355469, "logps/rejected": -201.28585815429688, "loss": 1.0449, "margin_dpo/margin_mean": 61.90514373779297, "margin_dpo/margin_std": 84.87958526611328, "step": 140 }, { "KL/chosen_KL_mean": -62.21798324584961, "KL/mean": -91.52658081054688, "KL/rejected_KL_mean": -120.83517456054688, "KL/std": 76.01405334472656, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.00829094648361206, "fcm_dpo/delta": -0.037813104689121246, "fcm_dpo/margin": 58.61717987060547, "fcm_dpo/q_t": 0.3922034502029419, "grad_norm": 26.592355728149414, "learning_rate": 4.83578576263792e-07, "logits/chosen": -0.451934278011322, "logits/rejected": -0.4369876980781555, "logps/chosen": -137.29364013671875, "logps/ref_chosen": -75.07566833496094, "logps/ref_rejected": -98.1922607421875, "logps/rejected": -219.02743530273438, "loss": 1.0748, "margin_dpo/margin_mean": 58.61717987060547, "margin_dpo/margin_std": 87.23815155029297, "step": 141 }, { "KL/chosen_KL_mean": -64.0453109741211, "KL/mean": -97.54293823242188, "KL/rejected_KL_mean": -131.04055786132812, "KL/std": 84.23410034179688, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.008179203607141972, "fcm_dpo/delta": -0.10360373556613922, "fcm_dpo/margin": 66.99524688720703, "fcm_dpo/q_t": 0.38415253162384033, "grad_norm": 26.743053436279297, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.45145779848098755, "logits/rejected": -0.44982147216796875, "logps/chosen": -122.0732421875, "logps/ref_chosen": -58.027931213378906, "logps/ref_rejected": -94.58222961425781, "logps/rejected": -225.62278747558594, "loss": 1.0591, "margin_dpo/margin_mean": 66.99524688720703, "margin_dpo/margin_std": 98.7759780883789, "step": 142 }, { "KL/chosen_KL_mean": -68.70494842529297, "KL/mean": -90.10963439941406, "KL/rejected_KL_mean": -111.51432037353516, "KL/std": 74.79816436767578, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.008188603445887566, "fcm_dpo/delta": 0.1031288281083107, "fcm_dpo/margin": 42.80937576293945, "fcm_dpo/q_t": 0.4261817932128906, "grad_norm": 25.709033966064453, "learning_rate": 4.826513955607734e-07, "logits/chosen": -0.42726534605026245, "logits/rejected": -0.4184831976890564, "logps/chosen": -126.30139923095703, "logps/ref_chosen": -57.59645080566406, "logps/ref_rejected": -78.99957275390625, "logps/rejected": -190.51388549804688, "loss": 1.1816, "margin_dpo/margin_mean": 42.80937194824219, "margin_dpo/margin_std": 86.87847900390625, "step": 143 }, { "KL/chosen_KL_mean": -59.455833435058594, "KL/mean": -85.97761535644531, "KL/rejected_KL_mean": -112.4993896484375, "KL/std": 62.42079162597656, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.008272461593151093, "fcm_dpo/delta": 0.011701909825205803, "fcm_dpo/margin": 53.04355239868164, "fcm_dpo/q_t": 0.4019629955291748, "grad_norm": 21.54189682006836, "learning_rate": 4.821786031898176e-07, "logits/chosen": -0.44170573353767395, "logits/rejected": -0.42558401823043823, "logps/chosen": -119.36219787597656, "logps/ref_chosen": -59.90636444091797, "logps/ref_rejected": -82.00025939941406, "logps/rejected": -194.49964904785156, "loss": 1.0796, "margin_dpo/margin_mean": 53.043548583984375, "margin_dpo/margin_std": 73.45362854003906, "step": 144 }, { "KL/chosen_KL_mean": -57.76536178588867, "KL/mean": -86.40901947021484, "KL/rejected_KL_mean": -115.05267333984375, "KL/std": 63.70155334472656, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.00824956875294447, "fcm_dpo/delta": -0.0237555094063282, "fcm_dpo/margin": 57.287315368652344, "fcm_dpo/q_t": 0.3944385349750519, "grad_norm": 24.161712646484375, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.4552566111087799, "logits/rejected": -0.4353902339935303, "logps/chosen": -114.36602783203125, "logps/ref_chosen": -56.60066604614258, "logps/ref_rejected": -77.86631774902344, "logps/rejected": -192.9189910888672, "loss": 1.0585, "margin_dpo/margin_mean": 57.287315368652344, "margin_dpo/margin_std": 76.47909545898438, "step": 145 }, { "KL/chosen_KL_mean": -79.07931518554688, "KL/mean": -101.17206573486328, "KL/rejected_KL_mean": -123.26481628417969, "KL/std": 68.17335510253906, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.00832764245569706, "fcm_dpo/delta": 0.08524463325738907, "fcm_dpo/margin": 44.185508728027344, "fcm_dpo/q_t": 0.4177442193031311, "grad_norm": 29.69412612915039, "learning_rate": 4.812146767012779e-07, "logits/chosen": -0.45252707600593567, "logits/rejected": -0.42418697476387024, "logps/chosen": -145.07977294921875, "logps/ref_chosen": -66.00045013427734, "logps/ref_rejected": -81.70278930664062, "logps/rejected": -204.96762084960938, "loss": 1.1619, "margin_dpo/margin_mean": 44.185508728027344, "margin_dpo/margin_std": 81.69918060302734, "step": 146 }, { "KL/chosen_KL_mean": -57.46442413330078, "KL/mean": -87.44345092773438, "KL/rejected_KL_mean": -117.42247009277344, "KL/std": 69.19750213623047, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.008307845331728458, "fcm_dpo/delta": -0.05084875971078873, "fcm_dpo/margin": 59.958045959472656, "fcm_dpo/q_t": 0.3916861414909363, "grad_norm": 21.09212875366211, "learning_rate": 4.807235679840536e-07, "logits/chosen": -0.4309794306755066, "logits/rejected": -0.4036545753479004, "logps/chosen": -110.86991119384766, "logps/ref_chosen": -53.405487060546875, "logps/ref_rejected": -71.39060974121094, "logps/rejected": -188.81307983398438, "loss": 1.0599, "margin_dpo/margin_mean": 59.958045959472656, "margin_dpo/margin_std": 85.19206237792969, "step": 147 }, { "KL/chosen_KL_mean": -57.138404846191406, "KL/mean": -82.37816619873047, "KL/rejected_KL_mean": -107.61793518066406, "KL/std": 68.84814453125, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.008421191945672035, "fcm_dpo/delta": 0.023522578179836273, "fcm_dpo/margin": 50.47953414916992, "fcm_dpo/q_t": 0.40811100602149963, "grad_norm": 20.28190803527832, "learning_rate": 4.802263794862384e-07, "logits/chosen": -0.503899097442627, "logits/rejected": -0.4953872859477997, "logps/chosen": -122.07548522949219, "logps/ref_chosen": -64.93708038330078, "logps/ref_rejected": -103.09384155273438, "logps/rejected": -210.71177673339844, "loss": 1.101, "margin_dpo/margin_mean": 50.47953796386719, "margin_dpo/margin_std": 72.59159088134766, "step": 148 }, { "KL/chosen_KL_mean": -55.076839447021484, "KL/mean": -86.57733917236328, "KL/rejected_KL_mean": -118.07785034179688, "KL/std": 62.685951232910156, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.00818311795592308, "fcm_dpo/delta": -0.07131742686033249, "fcm_dpo/margin": 63.00099182128906, "fcm_dpo/q_t": 0.3839060366153717, "grad_norm": 19.841922760009766, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.518465518951416, "logits/rejected": -0.5024739503860474, "logps/chosen": -113.55059814453125, "logps/ref_chosen": -58.47376251220703, "logps/ref_rejected": -99.31474304199219, "logps/rejected": -217.39259338378906, "loss": 1.0235, "margin_dpo/margin_mean": 63.00099563598633, "margin_dpo/margin_std": 73.64586639404297, "step": 149 }, { "KL/chosen_KL_mean": -48.64092254638672, "KL/mean": -79.2264404296875, "KL/rejected_KL_mean": -109.81196594238281, "KL/std": 73.13420104980469, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.00810123048722744, "fcm_dpo/delta": -0.04957358539104462, "fcm_dpo/margin": 61.171043395996094, "fcm_dpo/q_t": 0.3953554034233093, "grad_norm": 19.716720581054688, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.46694451570510864, "logits/rejected": -0.46912962198257446, "logps/chosen": -94.34673309326172, "logps/ref_chosen": -45.705810546875, "logps/ref_rejected": -83.34759521484375, "logps/rejected": -193.15956115722656, "loss": 1.0563, "margin_dpo/margin_mean": 61.17103958129883, "margin_dpo/margin_std": 87.25103759765625, "step": 150 }, { "KL/chosen_KL_mean": -60.15590286254883, "KL/mean": -91.48491668701172, "KL/rejected_KL_mean": -122.81391906738281, "KL/std": 69.31367492675781, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.008071625605225563, "fcm_dpo/delta": -0.058733321726322174, "fcm_dpo/margin": 62.65802001953125, "fcm_dpo/q_t": 0.38598495721817017, "grad_norm": 23.326515197753906, "learning_rate": 4.786984671220053e-07, "logits/chosen": -0.5557031631469727, "logits/rejected": -0.5267741680145264, "logps/chosen": -130.72674560546875, "logps/ref_chosen": -70.57083129882812, "logps/ref_rejected": -100.46382141113281, "logps/rejected": -223.27774047851562, "loss": 1.0271, "margin_dpo/margin_mean": 62.65802001953125, "margin_dpo/margin_std": 75.5926742553711, "step": 151 }, { "KL/chosen_KL_mean": -52.698150634765625, "KL/mean": -90.16242980957031, "KL/rejected_KL_mean": -127.62669372558594, "KL/std": 71.81861877441406, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.007893717847764492, "fcm_dpo/delta": -0.15009991824626923, "fcm_dpo/margin": 74.92855072021484, "fcm_dpo/q_t": 0.37131232023239136, "grad_norm": 21.844554901123047, "learning_rate": 4.78177092112495e-07, "logits/chosen": -0.5226190090179443, "logits/rejected": -0.5203348398208618, "logps/chosen": -112.86253356933594, "logps/ref_chosen": -60.16438674926758, "logps/ref_rejected": -106.14045715332031, "logps/rejected": -233.76715087890625, "loss": 0.9881, "margin_dpo/margin_mean": 74.92855072021484, "margin_dpo/margin_std": 87.656494140625, "step": 152 }, { "KL/chosen_KL_mean": -53.36126708984375, "KL/mean": -85.40214538574219, "KL/rejected_KL_mean": -117.44302368164062, "KL/std": 77.24756622314453, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.0077679818496108055, "fcm_dpo/delta": -0.050270069390535355, "fcm_dpo/margin": 64.08175659179688, "fcm_dpo/q_t": 0.3939558267593384, "grad_norm": 16.88256072998047, "learning_rate": 4.776497044244016e-07, "logits/chosen": -0.504237174987793, "logits/rejected": -0.4979296326637268, "logps/chosen": -109.67654418945312, "logps/ref_chosen": -56.315277099609375, "logps/ref_rejected": -85.65583801269531, "logps/rejected": -203.09884643554688, "loss": 1.0632, "margin_dpo/margin_mean": 64.08175659179688, "margin_dpo/margin_std": 94.27104187011719, "step": 153 }, { "KL/chosen_KL_mean": -63.94844055175781, "KL/mean": -95.09716796875, "KL/rejected_KL_mean": -126.24589538574219, "KL/std": 78.61471557617188, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.007730868645012379, "fcm_dpo/delta": -0.033514946699142456, "fcm_dpo/margin": 62.29746627807617, "fcm_dpo/q_t": 0.3967205882072449, "grad_norm": 20.78262710571289, "learning_rate": 4.771163179548808e-07, "logits/chosen": -0.49675068259239197, "logits/rejected": -0.49814850091934204, "logps/chosen": -126.69100952148438, "logps/ref_chosen": -62.74256896972656, "logps/ref_rejected": -104.24420166015625, "logps/rejected": -230.49009704589844, "loss": 1.095, "margin_dpo/margin_mean": 62.29746627807617, "margin_dpo/margin_std": 99.08077239990234, "step": 154 }, { "KL/chosen_KL_mean": -58.69048309326172, "KL/mean": -90.29657745361328, "KL/rejected_KL_mean": -121.90267944335938, "KL/std": 73.09174346923828, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.007663263939321041, "fcm_dpo/delta": -0.03624590486288071, "fcm_dpo/margin": 63.21219253540039, "fcm_dpo/q_t": 0.393093466758728, "grad_norm": 20.922475814819336, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.46945518255233765, "logits/rejected": -0.446572870016098, "logps/chosen": -119.34366607666016, "logps/ref_chosen": -60.65318298339844, "logps/ref_rejected": -77.49220275878906, "logps/rejected": -199.39488220214844, "loss": 1.0631, "margin_dpo/margin_mean": 63.21219253540039, "margin_dpo/margin_std": 89.31080627441406, "step": 155 }, { "KL/chosen_KL_mean": -82.07888793945312, "KL/mean": -100.7304458618164, "KL/rejected_KL_mean": -119.38200378417969, "KL/std": 76.68929290771484, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.007798383012413979, "fcm_dpo/delta": 0.16412018239498138, "fcm_dpo/margin": 37.30310821533203, "fcm_dpo/q_t": 0.4359789192676544, "grad_norm": 33.119998931884766, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -0.43547916412353516, "logits/rejected": -0.4247688949108124, "logps/chosen": -151.57077026367188, "logps/ref_chosen": -69.49188232421875, "logps/ref_rejected": -77.16929626464844, "logps/rejected": -196.55130004882812, "loss": 1.259, "margin_dpo/margin_mean": 37.30310821533203, "margin_dpo/margin_std": 101.03076934814453, "step": 156 }, { "KL/chosen_KL_mean": -74.54649353027344, "KL/mean": -113.61559295654297, "KL/rejected_KL_mean": -152.6846923828125, "KL/std": 83.09423065185547, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.007648843340575695, "fcm_dpo/delta": -0.15882518887519836, "fcm_dpo/margin": 78.13821411132812, "fcm_dpo/q_t": 0.3660867214202881, "grad_norm": 25.022706985473633, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -0.41863518953323364, "logits/rejected": -0.42397525906562805, "logps/chosen": -135.91493225097656, "logps/ref_chosen": -61.368438720703125, "logps/ref_rejected": -107.64636993408203, "logps/rejected": -260.3310546875, "loss": 0.9962, "margin_dpo/margin_mean": 78.13821411132812, "margin_dpo/margin_std": 93.55778503417969, "step": 157 }, { "KL/chosen_KL_mean": -75.25330352783203, "KL/mean": -116.46820831298828, "KL/rejected_KL_mean": -157.68312072753906, "KL/std": 103.57212829589844, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.007441770285367966, "fcm_dpo/delta": -0.1740458905696869, "fcm_dpo/margin": 82.4298095703125, "fcm_dpo/q_t": 0.3776199221611023, "grad_norm": 23.29196548461914, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -0.3899335265159607, "logits/rejected": -0.40577736496925354, "logps/chosen": -132.86622619628906, "logps/ref_chosen": -57.612918853759766, "logps/ref_rejected": -113.6946792602539, "logps/rejected": -271.3778076171875, "loss": 1.0341, "margin_dpo/margin_mean": 82.4298095703125, "margin_dpo/margin_std": 123.92778015136719, "step": 158 }, { "KL/chosen_KL_mean": -83.34770202636719, "KL/mean": -111.96831512451172, "KL/rejected_KL_mean": -140.58892822265625, "KL/std": 89.91984558105469, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.0074341027066111565, "fcm_dpo/delta": 0.025001313537359238, "fcm_dpo/margin": 57.2412223815918, "fcm_dpo/q_t": 0.4071953594684601, "grad_norm": 25.42589569091797, "learning_rate": 4.743599013306165e-07, "logits/chosen": -0.3967503309249878, "logits/rejected": -0.3607752323150635, "logps/chosen": -164.90805053710938, "logps/ref_chosen": -81.56034851074219, "logps/ref_rejected": -88.89871215820312, "logps/rejected": -229.48764038085938, "loss": 1.126, "margin_dpo/margin_mean": 57.2412223815918, "margin_dpo/margin_std": 97.32987213134766, "step": 159 }, { "KL/chosen_KL_mean": -85.25990295410156, "KL/mean": -123.54020690917969, "KL/rejected_KL_mean": -161.82049560546875, "KL/std": 96.52642059326172, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.007252626121044159, "fcm_dpo/delta": -0.11243004351854324, "fcm_dpo/margin": 76.56060791015625, "fcm_dpo/q_t": 0.3867749571800232, "grad_norm": 28.038400650024414, "learning_rate": 4.737908228387656e-07, "logits/chosen": -0.40755778551101685, "logits/rejected": -0.3971432149410248, "logps/chosen": -150.99078369140625, "logps/ref_chosen": -65.73088073730469, "logps/ref_rejected": -97.21781921386719, "logps/rejected": -259.038330078125, "loss": 1.0673, "margin_dpo/margin_mean": 76.56060791015625, "margin_dpo/margin_std": 120.45240783691406, "step": 160 }, { "KL/chosen_KL_mean": -72.6231460571289, "KL/mean": -106.6167984008789, "KL/rejected_KL_mean": -140.61044311523438, "KL/std": 77.12852478027344, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.00720019917935133, "fcm_dpo/delta": -0.04155520722270012, "fcm_dpo/margin": 67.9873046875, "fcm_dpo/q_t": 0.39423298835754395, "grad_norm": 22.304636001586914, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -0.39059293270111084, "logits/rejected": -0.3855125904083252, "logps/chosen": -125.05961608886719, "logps/ref_chosen": -52.43647003173828, "logps/ref_rejected": -83.43095397949219, "logps/rejected": -224.04139709472656, "loss": 1.066, "margin_dpo/margin_mean": 67.9873046875, "margin_dpo/margin_std": 97.89100646972656, "step": 161 }, { "KL/chosen_KL_mean": -69.07589721679688, "KL/mean": -103.18696594238281, "KL/rejected_KL_mean": -137.29803466796875, "KL/std": 85.45391845703125, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.0070790136232972145, "fcm_dpo/delta": -0.0367625430226326, "fcm_dpo/margin": 68.22212982177734, "fcm_dpo/q_t": 0.39606010913848877, "grad_norm": 23.20815658569336, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.4767564535140991, "logits/rejected": -0.4587327837944031, "logps/chosen": -131.68649291992188, "logps/ref_chosen": -62.6105842590332, "logps/ref_rejected": -89.39057922363281, "logps/rejected": -226.68861389160156, "loss": 1.0772, "margin_dpo/margin_mean": 68.22212219238281, "margin_dpo/margin_std": 100.68720245361328, "step": 162 }, { "KL/chosen_KL_mean": -76.93033599853516, "KL/mean": -109.9669189453125, "KL/rejected_KL_mean": -143.0034942626953, "KL/std": 84.1370849609375, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.007111528422683477, "fcm_dpo/delta": -0.02085772156715393, "fcm_dpo/margin": 66.07315826416016, "fcm_dpo/q_t": 0.398965060710907, "grad_norm": 21.569955825805664, "learning_rate": 4.720482655449212e-07, "logits/chosen": -0.4115701913833618, "logits/rejected": -0.3913278877735138, "logps/chosen": -131.95196533203125, "logps/ref_chosen": -55.021629333496094, "logps/ref_rejected": -75.418212890625, "logps/rejected": -218.42172241210938, "loss": 1.0824, "margin_dpo/margin_mean": 66.07315826416016, "margin_dpo/margin_std": 100.27389526367188, "step": 163 }, { "KL/chosen_KL_mean": -69.56802368164062, "KL/mean": -109.19029235839844, "KL/rejected_KL_mean": -148.81256103515625, "KL/std": 82.19638061523438, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.006935762241482735, "fcm_dpo/delta": -0.10748938471078873, "fcm_dpo/margin": 79.24454498291016, "fcm_dpo/q_t": 0.3782011866569519, "grad_norm": 21.600542068481445, "learning_rate": 4.714556901942599e-07, "logits/chosen": -0.4188717305660248, "logits/rejected": -0.40261319279670715, "logps/chosen": -125.20869445800781, "logps/ref_chosen": -55.64066696166992, "logps/ref_rejected": -79.66463470458984, "logps/rejected": -228.47720336914062, "loss": 1.0077, "margin_dpo/margin_mean": 79.24454498291016, "margin_dpo/margin_std": 94.94876098632812, "step": 164 }, { "KL/chosen_KL_mean": -75.52891540527344, "KL/mean": -101.27418518066406, "KL/rejected_KL_mean": -127.01945495605469, "KL/std": 68.8742446899414, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.00702784676104784, "fcm_dpo/delta": 0.0915396437048912, "fcm_dpo/margin": 51.49053192138672, "fcm_dpo/q_t": 0.41989004611968994, "grad_norm": 28.339448928833008, "learning_rate": 4.708572792802069e-07, "logits/chosen": -0.4348529577255249, "logits/rejected": -0.4037681818008423, "logps/chosen": -136.839599609375, "logps/ref_chosen": -61.310691833496094, "logps/ref_rejected": -73.67060852050781, "logps/rejected": -200.6900634765625, "loss": 1.1525, "margin_dpo/margin_mean": 51.49053192138672, "margin_dpo/margin_std": 90.902587890625, "step": 165 }, { "KL/chosen_KL_mean": -67.45067596435547, "KL/mean": -114.44827270507812, "KL/rejected_KL_mean": -161.4458770751953, "KL/std": 100.94361114501953, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.006829878315329552, "fcm_dpo/delta": -0.2068059891462326, "fcm_dpo/margin": 93.99519348144531, "fcm_dpo/q_t": 0.3710244297981262, "grad_norm": 18.604509353637695, "learning_rate": 4.702530485714461e-07, "logits/chosen": -0.3967953324317932, "logits/rejected": -0.4072214663028717, "logps/chosen": -118.43428039550781, "logps/ref_chosen": -50.98360061645508, "logps/ref_rejected": -98.09512329101562, "logps/rejected": -259.541015625, "loss": 0.9903, "margin_dpo/margin_mean": 93.99519348144531, "margin_dpo/margin_std": 126.79255676269531, "step": 166 }, { "KL/chosen_KL_mean": -68.31707000732422, "KL/mean": -118.00259399414062, "KL/rejected_KL_mean": -167.6881103515625, "KL/std": 92.47897338867188, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.006573098246008158, "fcm_dpo/delta": -0.21753078699111938, "fcm_dpo/margin": 99.37103271484375, "fcm_dpo/q_t": 0.3561442792415619, "grad_norm": 21.904842376708984, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.3720051348209381, "logits/rejected": -0.37385571002960205, "logps/chosen": -118.74116516113281, "logps/ref_chosen": -50.424095153808594, "logps/ref_rejected": -96.03042602539062, "logps/rejected": -263.7185363769531, "loss": 0.94, "margin_dpo/margin_mean": 99.37103271484375, "margin_dpo/margin_std": 105.42371368408203, "step": 167 }, { "KL/chosen_KL_mean": -75.26981353759766, "KL/mean": -112.29487609863281, "KL/rejected_KL_mean": -149.31991577148438, "KL/std": 87.42570495605469, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.006460403557866812, "fcm_dpo/delta": -0.029986891895532608, "fcm_dpo/margin": 74.05010986328125, "fcm_dpo/q_t": 0.3942164182662964, "grad_norm": 20.635385513305664, "learning_rate": 4.690271916109034e-07, "logits/chosen": -0.3801596760749817, "logits/rejected": -0.36851075291633606, "logps/chosen": -124.73263549804688, "logps/ref_chosen": -49.462825775146484, "logps/ref_rejected": -75.30855560302734, "logps/rejected": -224.62847900390625, "loss": 1.0505, "margin_dpo/margin_mean": 74.05010223388672, "margin_dpo/margin_std": 96.77279663085938, "step": 168 }, { "KL/chosen_KL_mean": -77.44679260253906, "KL/mean": -110.11390686035156, "KL/rejected_KL_mean": -142.78103637695312, "KL/std": 87.40264892578125, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.006434156559407711, "fcm_dpo/delta": 0.029472671449184418, "fcm_dpo/margin": 65.33423614501953, "fcm_dpo/q_t": 0.4121074974536896, "grad_norm": 23.882349014282227, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -0.3922446370124817, "logits/rejected": -0.37415844202041626, "logps/chosen": -137.250244140625, "logps/ref_chosen": -59.803443908691406, "logps/ref_rejected": -83.34574890136719, "logps/rejected": -226.1267852783203, "loss": 1.1416, "margin_dpo/margin_mean": 65.33423614501953, "margin_dpo/margin_std": 119.43335723876953, "step": 169 }, { "KL/chosen_KL_mean": -69.79130554199219, "KL/mean": -108.45357513427734, "KL/rejected_KL_mean": -147.1158447265625, "KL/std": 82.628173828125, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.006413028109818697, "fcm_dpo/delta": -0.049491968005895615, "fcm_dpo/margin": 77.32452392578125, "fcm_dpo/q_t": 0.38836145401000977, "grad_norm": 19.305862426757812, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -0.35024935007095337, "logits/rejected": -0.3379005193710327, "logps/chosen": -119.26307678222656, "logps/ref_chosen": -49.471771240234375, "logps/ref_rejected": -75.91734313964844, "logps/rejected": -223.03317260742188, "loss": 1.0442, "margin_dpo/margin_mean": 77.32452392578125, "margin_dpo/margin_std": 98.25650024414062, "step": 170 }, { "KL/chosen_KL_mean": -100.06165313720703, "KL/mean": -131.94424438476562, "KL/rejected_KL_mean": -163.8268280029297, "KL/std": 95.37467956542969, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.006473253481090069, "fcm_dpo/delta": 0.038848720490932465, "fcm_dpo/margin": 63.765167236328125, "fcm_dpo/q_t": 0.41463029384613037, "grad_norm": 26.568737030029297, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -0.3878970742225647, "logits/rejected": -0.36740678548812866, "logps/chosen": -184.56097412109375, "logps/ref_chosen": -84.49931335449219, "logps/ref_rejected": -109.38209533691406, "logps/rejected": -273.20892333984375, "loss": 1.1467, "margin_dpo/margin_mean": 63.76517105102539, "margin_dpo/margin_std": 121.90130615234375, "step": 171 }, { "KL/chosen_KL_mean": -90.25933837890625, "KL/mean": -123.08937072753906, "KL/rejected_KL_mean": -155.91940307617188, "KL/std": 93.85934448242188, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.006503078620880842, "fcm_dpo/delta": 0.02402586117386818, "fcm_dpo/margin": 65.66006469726562, "fcm_dpo/q_t": 0.4063677191734314, "grad_norm": 22.084125518798828, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -0.4236146807670593, "logits/rejected": -0.4051532745361328, "logps/chosen": -158.9132537841797, "logps/ref_chosen": -68.65391540527344, "logps/ref_rejected": -85.43667602539062, "logps/rejected": -241.3560791015625, "loss": 1.1088, "margin_dpo/margin_mean": 65.66006469726562, "margin_dpo/margin_std": 105.15103149414062, "step": 172 }, { "KL/chosen_KL_mean": -82.02645874023438, "KL/mean": -114.88996887207031, "KL/rejected_KL_mean": -147.7534637451172, "KL/std": 86.57609558105469, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.006563690025359392, "fcm_dpo/delta": 0.01906364969909191, "fcm_dpo/margin": 65.72700500488281, "fcm_dpo/q_t": 0.403574675321579, "grad_norm": 22.174354553222656, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -0.43103909492492676, "logits/rejected": -0.4012182354927063, "logps/chosen": -145.07733154296875, "logps/ref_chosen": -63.050880432128906, "logps/ref_rejected": -78.68392181396484, "logps/rejected": -226.4373779296875, "loss": 1.0803, "margin_dpo/margin_mean": 65.72700500488281, "margin_dpo/margin_std": 90.32469940185547, "step": 173 }, { "KL/chosen_KL_mean": -78.2431411743164, "KL/mean": -117.16438293457031, "KL/rejected_KL_mean": -156.0856170654297, "KL/std": 92.78438568115234, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.006528258323669434, "fcm_dpo/delta": -0.06176135316491127, "fcm_dpo/margin": 77.84245300292969, "fcm_dpo/q_t": 0.39108026027679443, "grad_norm": 25.343835830688477, "learning_rate": 4.652116329460919e-07, "logits/chosen": -0.37583017349243164, "logits/rejected": -0.392827570438385, "logps/chosen": -131.60610961914062, "logps/ref_chosen": -53.36296844482422, "logps/ref_rejected": -101.91120910644531, "logps/rejected": -257.996826171875, "loss": 1.0577, "margin_dpo/margin_mean": 77.84245300292969, "margin_dpo/margin_std": 109.80511474609375, "step": 174 }, { "KL/chosen_KL_mean": -72.41232299804688, "KL/mean": -125.403564453125, "KL/rejected_KL_mean": -178.39483642578125, "KL/std": 99.21268463134766, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.0062413765117526054, "fcm_dpo/delta": -0.2268737256526947, "fcm_dpo/margin": 105.98250579833984, "fcm_dpo/q_t": 0.3524921238422394, "grad_norm": 27.583271026611328, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.3298878073692322, "logits/rejected": -0.31552278995513916, "logps/chosen": -117.830078125, "logps/ref_chosen": -45.417762756347656, "logps/ref_rejected": -89.50579833984375, "logps/rejected": -267.900634765625, "loss": 0.92, "margin_dpo/margin_mean": 105.98250579833984, "margin_dpo/margin_std": 103.4354476928711, "step": 175 }, { "KL/chosen_KL_mean": -77.36492919921875, "KL/mean": -121.19436645507812, "KL/rejected_KL_mean": -165.0238037109375, "KL/std": 96.19127655029297, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.0061079468578100204, "fcm_dpo/delta": -0.09015016257762909, "fcm_dpo/margin": 87.65887451171875, "fcm_dpo/q_t": 0.3845323622226715, "grad_norm": 21.98095703125, "learning_rate": 4.638942309888058e-07, "logits/chosen": -0.3371141850948334, "logits/rejected": -0.3543722629547119, "logps/chosen": -127.8177719116211, "logps/ref_chosen": -50.452842712402344, "logps/ref_rejected": -95.5589599609375, "logps/rejected": -260.582763671875, "loss": 1.0211, "margin_dpo/margin_mean": 87.65887451171875, "margin_dpo/margin_std": 111.42243957519531, "step": 176 }, { "KL/chosen_KL_mean": -88.33280944824219, "KL/mean": -131.93548583984375, "KL/rejected_KL_mean": -175.5381622314453, "KL/std": 103.59379577636719, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.006017684005200863, "fcm_dpo/delta": -0.07888535410165787, "fcm_dpo/margin": 87.20537567138672, "fcm_dpo/q_t": 0.38545751571655273, "grad_norm": 30.604167938232422, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -0.3933263123035431, "logits/rejected": -0.38398683071136475, "logps/chosen": -149.5492706298828, "logps/ref_chosen": -61.216468811035156, "logps/ref_rejected": -95.89378356933594, "logps/rejected": -271.43194580078125, "loss": 1.023, "margin_dpo/margin_mean": 87.20537567138672, "margin_dpo/margin_std": 109.85467529296875, "step": 177 }, { "KL/chosen_KL_mean": -97.83988952636719, "KL/mean": -152.79367065429688, "KL/rejected_KL_mean": -207.74742126464844, "KL/std": 123.23829650878906, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.005772947799414396, "fcm_dpo/delta": -0.19834773242473602, "fcm_dpo/margin": 109.90754699707031, "fcm_dpo/q_t": 0.36565423011779785, "grad_norm": 29.574838638305664, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -0.2817849814891815, "logits/rejected": -0.27476948499679565, "logps/chosen": -156.10467529296875, "logps/ref_chosen": -58.26478958129883, "logps/ref_rejected": -105.3653335571289, "logps/rejected": -313.1127624511719, "loss": 0.9742, "margin_dpo/margin_mean": 109.90754699707031, "margin_dpo/margin_std": 135.3180694580078, "step": 178 }, { "KL/chosen_KL_mean": -98.74020385742188, "KL/mean": -138.61154174804688, "KL/rejected_KL_mean": -178.48287963867188, "KL/std": 104.14112854003906, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.005706791765987873, "fcm_dpo/delta": -0.0058643147349357605, "fcm_dpo/margin": 79.74267578125, "fcm_dpo/q_t": 0.40119341015815735, "grad_norm": 28.889633178710938, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -0.29684174060821533, "logits/rejected": -0.3017182946205139, "logps/chosen": -159.79852294921875, "logps/ref_chosen": -61.05832290649414, "logps/ref_rejected": -90.52782440185547, "logps/rejected": -269.0107116699219, "loss": 1.106, "margin_dpo/margin_mean": 79.74266815185547, "margin_dpo/margin_std": 128.66012573242188, "step": 179 }, { "KL/chosen_KL_mean": -83.98764038085938, "KL/mean": -133.84719848632812, "KL/rejected_KL_mean": -183.70677185058594, "KL/std": 93.82740783691406, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.005624772980809212, "fcm_dpo/delta": -0.11774921417236328, "fcm_dpo/margin": 99.7191390991211, "fcm_dpo/q_t": 0.37429293990135193, "grad_norm": 22.949077606201172, "learning_rate": 4.611919330113591e-07, "logits/chosen": -0.30718743801116943, "logits/rejected": -0.30064964294433594, "logps/chosen": -138.33035278320312, "logps/ref_chosen": -54.34272003173828, "logps/ref_rejected": -98.21183776855469, "logps/rejected": -281.9186096191406, "loss": 0.9971, "margin_dpo/margin_mean": 99.71913146972656, "margin_dpo/margin_std": 116.27046203613281, "step": 180 }, { "KL/chosen_KL_mean": -72.35747528076172, "KL/mean": -103.51927947998047, "KL/rejected_KL_mean": -134.68109130859375, "KL/std": 84.68464660644531, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.00569215789437294, "fcm_dpo/delta": 0.09854313731193542, "fcm_dpo/margin": 62.3236083984375, "fcm_dpo/q_t": 0.42022329568862915, "grad_norm": 18.50898551940918, "learning_rate": 4.605024008834863e-07, "logits/chosen": -0.36501559615135193, "logits/rejected": -0.3385394513607025, "logps/chosen": -127.3579330444336, "logps/ref_chosen": -55.000457763671875, "logps/ref_rejected": -61.656166076660156, "logps/rejected": -196.33724975585938, "loss": 1.1477, "margin_dpo/margin_mean": 62.3236083984375, "margin_dpo/margin_std": 106.25810241699219, "step": 181 }, { "KL/chosen_KL_mean": -68.99199676513672, "KL/mean": -122.10658264160156, "KL/rejected_KL_mean": -175.22119140625, "KL/std": 102.834228515625, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.0055610742419958115, "fcm_dpo/delta": -0.15064096450805664, "fcm_dpo/margin": 106.22918701171875, "fcm_dpo/q_t": 0.3680379390716553, "grad_norm": 18.80778694152832, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.3319093883037567, "logits/rejected": -0.339343786239624, "logps/chosen": -110.09984588623047, "logps/ref_chosen": -41.107852935791016, "logps/ref_rejected": -89.5215835571289, "logps/rejected": -264.7427673339844, "loss": 0.9876, "margin_dpo/margin_mean": 106.22918701171875, "margin_dpo/margin_std": 123.63685607910156, "step": 182 }, { "KL/chosen_KL_mean": -101.37919616699219, "KL/mean": -131.22171020507812, "KL/rejected_KL_mean": -161.064208984375, "KL/std": 83.54005432128906, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.00549449073150754, "fcm_dpo/delta": -0.04330216720700264, "fcm_dpo/margin": 59.685020446777344, "fcm_dpo/q_t": 0.4243183732032776, "grad_norm": 19.258472442626953, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -0.3618839979171753, "logits/rejected": -0.3487667441368103, "logps/chosen": -158.9037628173828, "logps/ref_chosen": -57.52456283569336, "logps/ref_rejected": -75.97572326660156, "logps/rejected": -237.03994750976562, "loss": 1.1554, "margin_dpo/margin_mean": 59.685020446777344, "margin_dpo/margin_std": 92.6783447265625, "step": 183 }, { "KL/chosen_KL_mean": -84.08784484863281, "KL/mean": -114.76734161376953, "KL/rejected_KL_mean": -145.44683837890625, "KL/std": 80.12207794189453, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.005615667439997196, "fcm_dpo/delta": 0.1079062819480896, "fcm_dpo/margin": 61.358985900878906, "fcm_dpo/q_t": 0.42317280173301697, "grad_norm": 18.5307559967041, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -0.3942424952983856, "logits/rejected": -0.3832680881023407, "logps/chosen": -142.63279724121094, "logps/ref_chosen": -58.544952392578125, "logps/ref_rejected": -76.63406372070312, "logps/rejected": -222.08090209960938, "loss": 1.1465, "margin_dpo/margin_mean": 61.35899353027344, "margin_dpo/margin_std": 100.00926208496094, "step": 184 }, { "KL/chosen_KL_mean": -91.45793151855469, "KL/mean": -116.36428833007812, "KL/rejected_KL_mean": -141.2706298828125, "KL/std": 91.72756958007812, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.005753664765506983, "fcm_dpo/delta": 0.16831731796264648, "fcm_dpo/margin": 49.812705993652344, "fcm_dpo/q_t": 0.4390159845352173, "grad_norm": 19.70173454284668, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -0.36604273319244385, "logits/rejected": -0.3395982086658478, "logps/chosen": -153.48377990722656, "logps/ref_chosen": -62.025848388671875, "logps/ref_rejected": -73.7625961303711, "logps/rejected": -215.03323364257812, "loss": 1.2088, "margin_dpo/margin_mean": 49.812705993652344, "margin_dpo/margin_std": 109.287841796875, "step": 185 }, { "KL/chosen_KL_mean": -84.84208679199219, "KL/mean": -131.60130310058594, "KL/rejected_KL_mean": -178.36053466796875, "KL/std": 94.9348373413086, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.005720070563256741, "fcm_dpo/delta": -0.08993598818778992, "fcm_dpo/margin": 93.5184326171875, "fcm_dpo/q_t": 0.3803972601890564, "grad_norm": 31.597518920898438, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -0.3996261954307556, "logits/rejected": -0.3734767436981201, "logps/chosen": -154.195556640625, "logps/ref_chosen": -69.35346984863281, "logps/ref_rejected": -88.07244873046875, "logps/rejected": -266.4329833984375, "loss": 1.0042, "margin_dpo/margin_mean": 93.5184326171875, "margin_dpo/margin_std": 107.01985168457031, "step": 186 }, { "KL/chosen_KL_mean": -82.51023864746094, "KL/mean": -120.74517059326172, "KL/rejected_KL_mean": -158.9801025390625, "KL/std": 90.9475326538086, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.005740244407206774, "fcm_dpo/delta": 0.011016838252544403, "fcm_dpo/margin": 76.46986389160156, "fcm_dpo/q_t": 0.40099987387657166, "grad_norm": 26.53521156311035, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -0.3313896059989929, "logits/rejected": -0.324165940284729, "logps/chosen": -135.26669311523438, "logps/ref_chosen": -52.7564582824707, "logps/ref_rejected": -81.96910095214844, "logps/rejected": -240.94920349121094, "loss": 1.0715, "margin_dpo/margin_mean": 76.46986389160156, "margin_dpo/margin_std": 99.63009643554688, "step": 187 }, { "KL/chosen_KL_mean": -77.40263366699219, "KL/mean": -124.41639709472656, "KL/rejected_KL_mean": -171.43016052246094, "KL/std": 102.34568786621094, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.005648997146636248, "fcm_dpo/delta": -0.08564944565296173, "fcm_dpo/margin": 94.02752685546875, "fcm_dpo/q_t": 0.38270336389541626, "grad_norm": 32.77531814575195, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -0.349833607673645, "logits/rejected": -0.35466983914375305, "logps/chosen": -126.818115234375, "logps/ref_chosen": -49.415489196777344, "logps/ref_rejected": -89.54043579101562, "logps/rejected": -260.9705810546875, "loss": 1.0127, "margin_dpo/margin_mean": 94.02752685546875, "margin_dpo/margin_std": 112.28752899169922, "step": 188 }, { "KL/chosen_KL_mean": -93.02162170410156, "KL/mean": -131.88063049316406, "KL/rejected_KL_mean": -170.73960876464844, "KL/std": 103.43923950195312, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.005634644068777561, "fcm_dpo/delta": 0.012608291581273079, "fcm_dpo/margin": 77.7179946899414, "fcm_dpo/q_t": 0.4059969186782837, "grad_norm": 30.910058975219727, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -0.3398626148700714, "logits/rejected": -0.31965237855911255, "logps/chosen": -145.4205780029297, "logps/ref_chosen": -52.39896011352539, "logps/ref_rejected": -72.16735076904297, "logps/rejected": -242.90695190429688, "loss": 1.1041, "margin_dpo/margin_mean": 77.7179946899414, "margin_dpo/margin_std": 126.54793548583984, "step": 189 }, { "KL/chosen_KL_mean": -98.95657348632812, "KL/mean": -145.08016967773438, "KL/rejected_KL_mean": -191.20376586914062, "KL/std": 110.704345703125, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.005625591613352299, "fcm_dpo/delta": -0.07391072809696198, "fcm_dpo/margin": 92.24720001220703, "fcm_dpo/q_t": 0.3878136873245239, "grad_norm": 21.0270938873291, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -0.37190186977386475, "logits/rejected": -0.3592497706413269, "logps/chosen": -163.63963317871094, "logps/ref_chosen": -64.68305969238281, "logps/ref_rejected": -102.55052185058594, "logps/rejected": -293.7543029785156, "loss": 1.057, "margin_dpo/margin_mean": 92.24720001220703, "margin_dpo/margin_std": 128.497314453125, "step": 190 }, { "KL/chosen_KL_mean": -91.65495300292969, "KL/mean": -158.4246063232422, "KL/rejected_KL_mean": -225.19424438476562, "KL/std": 129.930908203125, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.0053010242991149426, "fcm_dpo/delta": -0.27925872802734375, "fcm_dpo/margin": 133.53929138183594, "fcm_dpo/q_t": 0.3480866551399231, "grad_norm": 22.660518646240234, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -0.3247271776199341, "logits/rejected": -0.30059993267059326, "logps/chosen": -160.31382751464844, "logps/ref_chosen": -68.65887451171875, "logps/ref_rejected": -110.1396713256836, "logps/rejected": -335.33392333984375, "loss": 0.9148, "margin_dpo/margin_mean": 133.539306640625, "margin_dpo/margin_std": 142.86781311035156, "step": 191 }, { "KL/chosen_KL_mean": -118.78849029541016, "KL/mean": -162.5628662109375, "KL/rejected_KL_mean": -206.33724975585938, "KL/std": 111.55377197265625, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.005254846997559071, "fcm_dpo/delta": -0.010667569935321808, "fcm_dpo/margin": 87.54873657226562, "fcm_dpo/q_t": 0.40085333585739136, "grad_norm": 27.49567413330078, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -0.3013686537742615, "logits/rejected": -0.29982978105545044, "logps/chosen": -188.51541137695312, "logps/ref_chosen": -69.72691345214844, "logps/ref_rejected": -103.32135009765625, "logps/rejected": -309.6585693359375, "loss": 1.0927, "margin_dpo/margin_mean": 87.54873657226562, "margin_dpo/margin_std": 137.13919067382812, "step": 192 }, { "KL/chosen_KL_mean": -118.83079528808594, "KL/mean": -144.50320434570312, "KL/rejected_KL_mean": -170.17559814453125, "KL/std": 101.15321350097656, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.005260747391730547, "fcm_dpo/delta": 0.044322919100522995, "fcm_dpo/margin": 51.34478759765625, "fcm_dpo/q_t": 0.4367770552635193, "grad_norm": 30.344806671142578, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -0.3281133472919464, "logits/rejected": -0.31660524010658264, "logps/chosen": -179.02130126953125, "logps/ref_chosen": -60.19049835205078, "logps/ref_rejected": -76.40755462646484, "logps/rejected": -246.58316040039062, "loss": 1.2429, "margin_dpo/margin_mean": 51.34478759765625, "margin_dpo/margin_std": 128.936767578125, "step": 193 }, { "KL/chosen_KL_mean": -73.99462890625, "KL/mean": -115.9382553100586, "KL/rejected_KL_mean": -157.88189697265625, "KL/std": 85.7735595703125, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.005274644121527672, "fcm_dpo/delta": 0.007718952372670174, "fcm_dpo/margin": 83.88723754882812, "fcm_dpo/q_t": 0.3977963924407959, "grad_norm": 20.127643585205078, "learning_rate": 4.510405240853854e-07, "logits/chosen": -0.2561291456222534, "logits/rejected": -0.2391519397497177, "logps/chosen": -111.83500671386719, "logps/ref_chosen": -37.84037399291992, "logps/ref_rejected": -60.684783935546875, "logps/rejected": -218.56666564941406, "loss": 1.0502, "margin_dpo/margin_mean": 83.88723754882812, "margin_dpo/margin_std": 94.03021240234375, "step": 194 }, { "KL/chosen_KL_mean": -117.90105438232422, "KL/mean": -163.21768188476562, "KL/rejected_KL_mean": -208.5343017578125, "KL/std": 107.3478012084961, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.005267483647912741, "fcm_dpo/delta": -0.028798166662454605, "fcm_dpo/margin": 90.63323974609375, "fcm_dpo/q_t": 0.39235901832580566, "grad_norm": 20.91324806213379, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -0.2966734766960144, "logits/rejected": -0.3150033950805664, "logps/chosen": -172.79263305664062, "logps/ref_chosen": -54.891571044921875, "logps/ref_rejected": -96.77095794677734, "logps/rejected": -305.30523681640625, "loss": 1.0421, "margin_dpo/margin_mean": 90.63323974609375, "margin_dpo/margin_std": 111.12643432617188, "step": 195 }, { "KL/chosen_KL_mean": -90.92343139648438, "KL/mean": -142.19207763671875, "KL/rejected_KL_mean": -193.46075439453125, "KL/std": 109.14540100097656, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.005165431182831526, "fcm_dpo/delta": -0.08529461920261383, "fcm_dpo/margin": 102.53730773925781, "fcm_dpo/q_t": 0.3841804265975952, "grad_norm": 22.65111541748047, "learning_rate": 4.495043068200599e-07, "logits/chosen": -0.33719146251678467, "logits/rejected": -0.3222602605819702, "logps/chosen": -144.16867065429688, "logps/ref_chosen": -53.245243072509766, "logps/ref_rejected": -76.05294799804688, "logps/rejected": -269.5137023925781, "loss": 1.0311, "margin_dpo/margin_mean": 102.53730773925781, "margin_dpo/margin_std": 130.28256225585938, "step": 196 }, { "KL/chosen_KL_mean": -93.05845642089844, "KL/mean": -132.66305541992188, "KL/rejected_KL_mean": -172.2676544189453, "KL/std": 93.76297760009766, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.0052191768772900105, "fcm_dpo/delta": 0.03782667964696884, "fcm_dpo/margin": 79.20919799804688, "fcm_dpo/q_t": 0.40683090686798096, "grad_norm": 20.258697509765625, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -0.29354849457740784, "logits/rejected": -0.28677839040756226, "logps/chosen": -153.47879028320312, "logps/ref_chosen": -60.42033386230469, "logps/ref_rejected": -77.20890808105469, "logps/rejected": -249.4765625, "loss": 1.0854, "margin_dpo/margin_mean": 79.20919799804688, "margin_dpo/margin_std": 105.698486328125, "step": 197 }, { "KL/chosen_KL_mean": -105.25746154785156, "KL/mean": -155.54010009765625, "KL/rejected_KL_mean": -205.82276916503906, "KL/std": 117.5333023071289, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.005160834174603224, "fcm_dpo/delta": -0.07272866368293762, "fcm_dpo/margin": 100.56529235839844, "fcm_dpo/q_t": 0.38785430788993835, "grad_norm": 21.33174705505371, "learning_rate": 4.479470611971645e-07, "logits/chosen": -0.33081430196762085, "logits/rejected": -0.33057862520217896, "logps/chosen": -160.2936553955078, "logps/ref_chosen": -55.03618621826172, "logps/ref_rejected": -97.24325561523438, "logps/rejected": -303.0660400390625, "loss": 1.034, "margin_dpo/margin_mean": 100.56529235839844, "margin_dpo/margin_std": 134.0467529296875, "step": 198 }, { "KL/chosen_KL_mean": -100.41415405273438, "KL/mean": -149.92404174804688, "KL/rejected_KL_mean": -199.4339141845703, "KL/std": 106.2638168334961, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.005063729360699654, "fcm_dpo/delta": -0.05523253232240677, "fcm_dpo/margin": 99.01976776123047, "fcm_dpo/q_t": 0.3882749080657959, "grad_norm": 23.801433563232422, "learning_rate": 4.471606039587695e-07, "logits/chosen": -0.3572986125946045, "logits/rejected": -0.3407086133956909, "logps/chosen": -157.24298095703125, "logps/ref_chosen": -56.828826904296875, "logps/ref_rejected": -84.64820861816406, "logps/rejected": -284.0821228027344, "loss": 1.0418, "margin_dpo/margin_mean": 99.019775390625, "margin_dpo/margin_std": 126.37538146972656, "step": 199 }, { "KL/chosen_KL_mean": -98.62019348144531, "KL/mean": -148.50802612304688, "KL/rejected_KL_mean": -198.39585876464844, "KL/std": 113.95591735839844, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.005019994918256998, "fcm_dpo/delta": -0.054083775728940964, "fcm_dpo/margin": 99.77567291259766, "fcm_dpo/q_t": 0.39153775572776794, "grad_norm": 25.75031089782715, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -0.2872212529182434, "logits/rejected": -0.26524609327316284, "logps/chosen": -151.687255859375, "logps/ref_chosen": -53.06706237792969, "logps/ref_rejected": -80.60843658447266, "logps/rejected": -279.0043029785156, "loss": 1.0616, "margin_dpo/margin_mean": 99.77567291259766, "margin_dpo/margin_std": 143.67160034179688, "step": 200 }, { "KL/chosen_KL_mean": -102.265380859375, "KL/mean": -151.52593994140625, "KL/rejected_KL_mean": -200.78646850585938, "KL/std": 118.35769653320312, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.005003670696169138, "fcm_dpo/delta": -0.04519995301961899, "fcm_dpo/margin": 98.52108764648438, "fcm_dpo/q_t": 0.39059755206108093, "grad_norm": 22.498605728149414, "learning_rate": 4.455721242469372e-07, "logits/chosen": -0.37770992517471313, "logits/rejected": -0.37379318475723267, "logps/chosen": -177.6676025390625, "logps/ref_chosen": -75.4022216796875, "logps/ref_rejected": -114.80821990966797, "logps/rejected": -315.5946960449219, "loss": 1.0502, "margin_dpo/margin_mean": 98.52108764648438, "margin_dpo/margin_std": 131.62591552734375, "step": 201 }, { "KL/chosen_KL_mean": -105.03387451171875, "KL/mean": -139.44313049316406, "KL/rejected_KL_mean": -173.85238647460938, "KL/std": 100.22007751464844, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.005063983611762524, "fcm_dpo/delta": 0.10505226254463196, "fcm_dpo/margin": 68.8185043334961, "fcm_dpo/q_t": 0.42318177223205566, "grad_norm": 24.158180236816406, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -0.30475035309791565, "logits/rejected": -0.316967636346817, "logps/chosen": -155.13519287109375, "logps/ref_chosen": -50.101318359375, "logps/ref_rejected": -86.98503112792969, "logps/rejected": -260.83740234375, "loss": 1.164, "margin_dpo/margin_mean": 68.8185043334961, "margin_dpo/margin_std": 127.62640380859375, "step": 202 }, { "KL/chosen_KL_mean": -102.41499328613281, "KL/mean": -145.3014678955078, "KL/rejected_KL_mean": -188.18792724609375, "KL/std": 102.64788818359375, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.005089880898594856, "fcm_dpo/delta": 0.014019053429365158, "fcm_dpo/margin": 85.77294921875, "fcm_dpo/q_t": 0.40114203095436096, "grad_norm": 22.973190307617188, "learning_rate": 4.439630306414758e-07, "logits/chosen": -0.38558489084243774, "logits/rejected": -0.3768942356109619, "logps/chosen": -163.02468872070312, "logps/ref_chosen": -60.60969543457031, "logps/ref_rejected": -85.89596557617188, "logps/rejected": -274.08392333984375, "loss": 1.0694, "margin_dpo/margin_mean": 85.77295684814453, "margin_dpo/margin_std": 111.16223907470703, "step": 203 }, { "KL/chosen_KL_mean": -115.02670288085938, "KL/mean": -154.1737060546875, "KL/rejected_KL_mean": -193.32069396972656, "KL/std": 111.26290893554688, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.0051376353949308395, "fcm_dpo/delta": 0.04976356029510498, "fcm_dpo/margin": 78.29398345947266, "fcm_dpo/q_t": 0.41275104880332947, "grad_norm": 28.295520782470703, "learning_rate": 4.431508065452897e-07, "logits/chosen": -0.4326149821281433, "logits/rejected": -0.39165282249450684, "logps/chosen": -195.191650390625, "logps/ref_chosen": -80.16496276855469, "logps/ref_rejected": -87.69590759277344, "logps/rejected": -281.0166015625, "loss": 1.1225, "margin_dpo/margin_mean": 78.29398345947266, "margin_dpo/margin_std": 128.27748107910156, "step": 204 }, { "KL/chosen_KL_mean": -111.0508041381836, "KL/mean": -162.05941772460938, "KL/rejected_KL_mean": -213.06805419921875, "KL/std": 115.12797546386719, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.0050528934225440025, "fcm_dpo/delta": -0.07124295830726624, "fcm_dpo/margin": 102.01722717285156, "fcm_dpo/q_t": 0.3834433853626251, "grad_norm": 22.90087890625, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -0.3363794982433319, "logits/rejected": -0.3039674162864685, "logps/chosen": -170.435546875, "logps/ref_chosen": -59.384735107421875, "logps/ref_rejected": -85.12505340576172, "logps/rejected": -298.193115234375, "loss": 1.0345, "margin_dpo/margin_mean": 102.01722717285156, "margin_dpo/margin_std": 125.01140594482422, "step": 205 }, { "KL/chosen_KL_mean": -104.03678894042969, "KL/mean": -159.7008056640625, "KL/rejected_KL_mean": -215.36485290527344, "KL/std": 109.63488006591797, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.004975572694092989, "fcm_dpo/delta": -0.11080212146043777, "fcm_dpo/margin": 111.32804870605469, "fcm_dpo/q_t": 0.3731870651245117, "grad_norm": 27.668210983276367, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.2990955710411072, "logits/rejected": -0.30272185802459717, "logps/chosen": -151.00128173828125, "logps/ref_chosen": -46.964500427246094, "logps/ref_rejected": -98.9534912109375, "logps/rejected": -314.318359375, "loss": 0.988, "margin_dpo/margin_mean": 111.32805633544922, "margin_dpo/margin_std": 118.75079345703125, "step": 206 }, { "KL/chosen_KL_mean": -94.94453430175781, "KL/mean": -160.35214233398438, "KL/rejected_KL_mean": -225.759765625, "KL/std": 125.71635437011719, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.004834486171603203, "fcm_dpo/delta": -0.19447346031665802, "fcm_dpo/margin": 130.81524658203125, "fcm_dpo/q_t": 0.36242467164993286, "grad_norm": 23.659076690673828, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -0.3879826068878174, "logits/rejected": -0.363485187292099, "logps/chosen": -151.00079345703125, "logps/ref_chosen": -56.05625915527344, "logps/ref_rejected": -84.44779968261719, "logps/rejected": -310.20758056640625, "loss": 0.9535, "margin_dpo/margin_mean": 130.81524658203125, "margin_dpo/margin_std": 144.30435180664062, "step": 207 }, { "KL/chosen_KL_mean": -147.72682189941406, "KL/mean": -199.30636596679688, "KL/rejected_KL_mean": -250.88592529296875, "KL/std": 121.20966339111328, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.004738894756883383, "fcm_dpo/delta": -0.041033219546079636, "fcm_dpo/margin": 103.15910339355469, "fcm_dpo/q_t": 0.39097169041633606, "grad_norm": 27.04127311706543, "learning_rate": 4.398512291636768e-07, "logits/chosen": -0.38049787282943726, "logits/rejected": -0.36199867725372314, "logps/chosen": -214.79443359375, "logps/ref_chosen": -67.06761169433594, "logps/ref_rejected": -94.28689575195312, "logps/rejected": -345.1728210449219, "loss": 1.064, "margin_dpo/margin_mean": 103.15910339355469, "margin_dpo/margin_std": 145.6819305419922, "step": 208 }, { "KL/chosen_KL_mean": -124.6459732055664, "KL/mean": -169.93202209472656, "KL/rejected_KL_mean": -215.21807861328125, "KL/std": 109.89362335205078, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.004754163324832916, "fcm_dpo/delta": 0.020273543894290924, "fcm_dpo/margin": 90.57209777832031, "fcm_dpo/q_t": 0.4042934477329254, "grad_norm": 31.34199333190918, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -0.28706932067871094, "logits/rejected": -0.2759585976600647, "logps/chosen": -180.82766723632812, "logps/ref_chosen": -56.18169403076172, "logps/ref_rejected": -80.94152069091797, "logps/rejected": -296.15960693359375, "loss": 1.1062, "margin_dpo/margin_mean": 90.57209777832031, "margin_dpo/margin_std": 139.96685791015625, "step": 209 }, { "KL/chosen_KL_mean": -113.70458984375, "KL/mean": -165.95252990722656, "KL/rejected_KL_mean": -218.20050048828125, "KL/std": 113.39631652832031, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.004746555350720882, "fcm_dpo/delta": -0.04863199591636658, "fcm_dpo/margin": 104.49591064453125, "fcm_dpo/q_t": 0.3909037411212921, "grad_norm": 26.605981826782227, "learning_rate": 4.381713366536311e-07, "logits/chosen": -0.2829633355140686, "logits/rejected": -0.2737138271331787, "logps/chosen": -160.076416015625, "logps/ref_chosen": -46.371822357177734, "logps/ref_rejected": -76.68162536621094, "logps/rejected": -294.8821105957031, "loss": 1.0495, "margin_dpo/margin_mean": 104.49591064453125, "margin_dpo/margin_std": 138.57675170898438, "step": 210 }, { "KL/chosen_KL_mean": -160.13168334960938, "KL/mean": -204.631591796875, "KL/rejected_KL_mean": -249.13148498535156, "KL/std": 128.61248779296875, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.004733526147902012, "fcm_dpo/delta": 0.029977422207593918, "fcm_dpo/margin": 88.99981689453125, "fcm_dpo/q_t": 0.4108871817588806, "grad_norm": 35.8740119934082, "learning_rate": 4.373239415645323e-07, "logits/chosen": -0.32857006788253784, "logits/rejected": -0.286856472492218, "logps/chosen": -239.0640411376953, "logps/ref_chosen": -78.93235778808594, "logps/ref_rejected": -86.82098388671875, "logps/rejected": -335.95245361328125, "loss": 1.1259, "margin_dpo/margin_mean": 88.99981689453125, "margin_dpo/margin_std": 151.01441955566406, "step": 211 }, { "KL/chosen_KL_mean": -130.05645751953125, "KL/mean": -193.37249755859375, "KL/rejected_KL_mean": -256.68853759765625, "KL/std": 137.6314697265625, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.004602103494107723, "fcm_dpo/delta": -0.14359265565872192, "fcm_dpo/margin": 126.632080078125, "fcm_dpo/q_t": 0.3717968463897705, "grad_norm": 25.104671478271484, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -0.33368033170700073, "logits/rejected": -0.3230438530445099, "logps/chosen": -188.25347900390625, "logps/ref_chosen": -58.19701385498047, "logps/ref_rejected": -103.05785369873047, "logps/rejected": -359.74639892578125, "loss": 1.006, "margin_dpo/margin_mean": 126.632080078125, "margin_dpo/margin_std": 152.7139892578125, "step": 212 }, { "KL/chosen_KL_mean": -121.16593933105469, "KL/mean": -180.69183349609375, "KL/rejected_KL_mean": -240.21774291992188, "KL/std": 118.9180908203125, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.004524152260273695, "fcm_dpo/delta": -0.09434568881988525, "fcm_dpo/margin": 119.05180358886719, "fcm_dpo/q_t": 0.37940219044685364, "grad_norm": 27.022523880004883, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -0.33751168847084045, "logits/rejected": -0.3087567687034607, "logps/chosen": -188.67864990234375, "logps/ref_chosen": -67.51271057128906, "logps/ref_rejected": -93.91471862792969, "logps/rejected": -334.1324462890625, "loss": 1.0147, "margin_dpo/margin_mean": 119.05180358886719, "margin_dpo/margin_std": 143.37750244140625, "step": 213 }, { "KL/chosen_KL_mean": -103.91690063476562, "KL/mean": -158.3183135986328, "KL/rejected_KL_mean": -212.7197265625, "KL/std": 117.29820251464844, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.004492715001106262, "fcm_dpo/delta": -0.040815435349941254, "fcm_dpo/margin": 108.8028335571289, "fcm_dpo/q_t": 0.39092785120010376, "grad_norm": 24.806739807128906, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -0.29573267698287964, "logits/rejected": -0.3004228174686432, "logps/chosen": -145.52178955078125, "logps/ref_chosen": -41.604888916015625, "logps/ref_rejected": -77.51741027832031, "logps/rejected": -290.2371520996094, "loss": 1.0412, "margin_dpo/margin_mean": 108.80284118652344, "margin_dpo/margin_std": 136.7408905029297, "step": 214 }, { "KL/chosen_KL_mean": -119.432861328125, "KL/mean": -176.01162719726562, "KL/rejected_KL_mean": -232.59039306640625, "KL/std": 120.8812255859375, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.004435136914253235, "fcm_dpo/delta": -0.054898492991924286, "fcm_dpo/margin": 113.15753173828125, "fcm_dpo/q_t": 0.38608264923095703, "grad_norm": 31.865793228149414, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -0.34302496910095215, "logits/rejected": -0.3348790407180786, "logps/chosen": -172.71212768554688, "logps/ref_chosen": -53.279266357421875, "logps/ref_rejected": -89.96464538574219, "logps/rejected": -322.5550537109375, "loss": 1.019, "margin_dpo/margin_mean": 113.15753173828125, "margin_dpo/margin_std": 126.95989227294922, "step": 215 }, { "KL/chosen_KL_mean": -123.26240539550781, "KL/mean": -179.19107055664062, "KL/rejected_KL_mean": -235.1197509765625, "KL/std": 124.5248794555664, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.0044202255085110664, "fcm_dpo/delta": -0.04685918614268303, "fcm_dpo/margin": 111.85734558105469, "fcm_dpo/q_t": 0.3906000554561615, "grad_norm": 24.205480575561523, "learning_rate": 4.330133748510036e-07, "logits/chosen": -0.3160155117511749, "logits/rejected": -0.2979496121406555, "logps/chosen": -172.15020751953125, "logps/ref_chosen": -48.887794494628906, "logps/ref_rejected": -77.19892883300781, "logps/rejected": -312.31866455078125, "loss": 1.0553, "margin_dpo/margin_mean": 111.85734558105469, "margin_dpo/margin_std": 152.19393920898438, "step": 216 }, { "KL/chosen_KL_mean": -121.02989196777344, "KL/mean": -186.89378356933594, "KL/rejected_KL_mean": -252.75765991210938, "KL/std": 125.570068359375, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.004304962232708931, "fcm_dpo/delta": -0.12440802156925201, "fcm_dpo/margin": 131.72776794433594, "fcm_dpo/q_t": 0.37273240089416504, "grad_norm": 22.334640502929688, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -0.2926168739795685, "logits/rejected": -0.28581422567367554, "logps/chosen": -170.8751983642578, "logps/ref_chosen": -49.845306396484375, "logps/ref_rejected": -100.07832336425781, "logps/rejected": -352.83599853515625, "loss": 0.9772, "margin_dpo/margin_mean": 131.72776794433594, "margin_dpo/margin_std": 139.87136840820312, "step": 217 }, { "KL/chosen_KL_mean": -130.25543212890625, "KL/mean": -180.17367553710938, "KL/rejected_KL_mean": -230.09188842773438, "KL/std": 122.58163452148438, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.004292862489819527, "fcm_dpo/delta": 0.022361639887094498, "fcm_dpo/margin": 99.83648681640625, "fcm_dpo/q_t": 0.4043183922767639, "grad_norm": 24.542783737182617, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -0.326630175113678, "logits/rejected": -0.31950443983078003, "logps/chosen": -188.8321075439453, "logps/ref_chosen": -58.576683044433594, "logps/ref_rejected": -87.84639739990234, "logps/rejected": -317.93829345703125, "loss": 1.0891, "margin_dpo/margin_mean": 99.83648681640625, "margin_dpo/margin_std": 141.80709838867188, "step": 218 }, { "KL/chosen_KL_mean": -140.5905303955078, "KL/mean": -186.1055145263672, "KL/rejected_KL_mean": -231.62049865722656, "KL/std": 130.56130981445312, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.004359761718660593, "fcm_dpo/delta": 0.05462624877691269, "fcm_dpo/margin": 91.02996826171875, "fcm_dpo/q_t": 0.4125881791114807, "grad_norm": 29.01929473876953, "learning_rate": 4.303689819449636e-07, "logits/chosen": -0.31307315826416016, "logits/rejected": -0.3005494773387909, "logps/chosen": -201.6743927001953, "logps/ref_chosen": -61.083858489990234, "logps/ref_rejected": -85.83042907714844, "logps/rejected": -317.450927734375, "loss": 1.1381, "margin_dpo/margin_mean": 91.02996826171875, "margin_dpo/margin_std": 158.283203125, "step": 219 }, { "KL/chosen_KL_mean": -158.72708129882812, "KL/mean": -196.89773559570312, "KL/rejected_KL_mean": -235.0684051513672, "KL/std": 110.52084350585938, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.004418938420712948, "fcm_dpo/delta": 0.11673985421657562, "fcm_dpo/margin": 76.34133911132812, "fcm_dpo/q_t": 0.42132964730262756, "grad_norm": 28.393917083740234, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -0.31454047560691833, "logits/rejected": -0.28938183188438416, "logps/chosen": -228.75836181640625, "logps/ref_chosen": -70.03128051757812, "logps/ref_rejected": -87.68551635742188, "logps/rejected": -322.75390625, "loss": 1.1342, "margin_dpo/margin_mean": 76.34133911132812, "margin_dpo/margin_std": 110.11967468261719, "step": 220 }, { "KL/chosen_KL_mean": -139.7552490234375, "KL/mean": -218.57884216308594, "KL/rejected_KL_mean": -297.4024353027344, "KL/std": 141.19781494140625, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.004313099198043346, "fcm_dpo/delta": -0.24606135487556458, "fcm_dpo/margin": 157.64718627929688, "fcm_dpo/q_t": 0.34645724296569824, "grad_norm": 34.04795837402344, "learning_rate": 4.285822501755485e-07, "logits/chosen": -0.2851540446281433, "logits/rejected": -0.29184699058532715, "logps/chosen": -191.9099578857422, "logps/ref_chosen": -52.15470886230469, "logps/ref_rejected": -106.46768188476562, "logps/rejected": -403.8701171875, "loss": 0.9044, "margin_dpo/margin_mean": 157.64718627929688, "margin_dpo/margin_std": 141.8089141845703, "step": 221 }, { "KL/chosen_KL_mean": -146.63458251953125, "KL/mean": -207.00054931640625, "KL/rejected_KL_mean": -267.36651611328125, "KL/std": 130.62632751464844, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.0042197853326797485, "fcm_dpo/delta": -0.06277885288000107, "fcm_dpo/margin": 120.73193359375, "fcm_dpo/q_t": 0.3854852318763733, "grad_norm": 23.816076278686523, "learning_rate": 4.276818137766118e-07, "logits/chosen": -0.32067981362342834, "logits/rejected": -0.3180358409881592, "logps/chosen": -207.60568237304688, "logps/ref_chosen": -60.971099853515625, "logps/ref_rejected": -100.00115203857422, "logps/rejected": -367.36767578125, "loss": 1.0261, "margin_dpo/margin_mean": 120.73193359375, "margin_dpo/margin_std": 144.6348876953125, "step": 222 }, { "KL/chosen_KL_mean": -155.57662963867188, "KL/mean": -207.48294067382812, "KL/rejected_KL_mean": -259.3892822265625, "KL/std": 130.69064331054688, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.004191592335700989, "fcm_dpo/delta": 0.015519605949521065, "fcm_dpo/margin": 103.81266784667969, "fcm_dpo/q_t": 0.4033808708190918, "grad_norm": 26.01241111755371, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -0.2557436525821686, "logits/rejected": -0.24912384152412415, "logps/chosen": -208.21719360351562, "logps/ref_chosen": -52.64057540893555, "logps/ref_rejected": -82.82502746582031, "logps/rejected": -342.21429443359375, "loss": 1.1072, "margin_dpo/margin_mean": 103.81266021728516, "margin_dpo/margin_std": 165.55490112304688, "step": 223 }, { "KL/chosen_KL_mean": -132.38632202148438, "KL/mean": -192.0895233154297, "KL/rejected_KL_mean": -251.792724609375, "KL/std": 143.71157836914062, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.004137513227760792, "fcm_dpo/delta": -0.04802892357110977, "fcm_dpo/margin": 119.40638732910156, "fcm_dpo/q_t": 0.39233386516571045, "grad_norm": 25.351940155029297, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -0.27835261821746826, "logits/rejected": -0.25782424211502075, "logps/chosen": -180.98175048828125, "logps/ref_chosen": -48.59541320800781, "logps/ref_rejected": -77.11648559570312, "logps/rejected": -328.9092102050781, "loss": 1.0566, "margin_dpo/margin_mean": 119.40638732910156, "margin_dpo/margin_std": 163.24822998046875, "step": 224 }, { "KL/chosen_KL_mean": -151.82391357421875, "KL/mean": -222.4290771484375, "KL/rejected_KL_mean": -293.03424072265625, "KL/std": 141.11753845214844, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.004073306918144226, "fcm_dpo/delta": -0.13306143879890442, "fcm_dpo/margin": 141.21035766601562, "fcm_dpo/q_t": 0.3728272318840027, "grad_norm": 27.907018661499023, "learning_rate": 4.249525076191759e-07, "logits/chosen": -0.2939089238643646, "logits/rejected": -0.28350159525871277, "logps/chosen": -209.82437133789062, "logps/ref_chosen": -58.000465393066406, "logps/ref_rejected": -99.90291595458984, "logps/rejected": -392.9371643066406, "loss": 0.9974, "margin_dpo/margin_mean": 141.21034240722656, "margin_dpo/margin_std": 170.23565673828125, "step": 225 }, { "KL/chosen_KL_mean": -130.0558319091797, "KL/mean": -187.48414611816406, "KL/rejected_KL_mean": -244.91246032714844, "KL/std": 141.0475616455078, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.004016010090708733, "fcm_dpo/delta": -0.012848969548940659, "fcm_dpo/margin": 114.85662841796875, "fcm_dpo/q_t": 0.399183452129364, "grad_norm": 31.283985137939453, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -0.35832834243774414, "logits/rejected": -0.3170637786388397, "logps/chosen": -188.9546356201172, "logps/ref_chosen": -58.898799896240234, "logps/ref_rejected": -78.68775939941406, "logps/rejected": -323.6002197265625, "loss": 1.0724, "margin_dpo/margin_mean": 114.85662841796875, "margin_dpo/margin_std": 159.86416625976562, "step": 226 }, { "KL/chosen_KL_mean": -151.1775360107422, "KL/mean": -221.81304931640625, "KL/rejected_KL_mean": -292.44854736328125, "KL/std": 158.22885131835938, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.003954638727009296, "fcm_dpo/delta": -0.11598802357912064, "fcm_dpo/margin": 141.27099609375, "fcm_dpo/q_t": 0.37532055377960205, "grad_norm": 23.948640823364258, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -0.3440447747707367, "logits/rejected": -0.3300914764404297, "logps/chosen": -210.24972534179688, "logps/ref_chosen": -59.072181701660156, "logps/ref_rejected": -99.41236877441406, "logps/rejected": -391.86090087890625, "loss": 0.9984, "margin_dpo/margin_mean": 141.27101135253906, "margin_dpo/margin_std": 165.87399291992188, "step": 227 }, { "KL/chosen_KL_mean": -157.38519287109375, "KL/mean": -206.89305114746094, "KL/rejected_KL_mean": -256.40093994140625, "KL/std": 129.40603637695312, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.003987784497439861, "fcm_dpo/delta": 0.05728516727685928, "fcm_dpo/margin": 99.0157470703125, "fcm_dpo/q_t": 0.4110216498374939, "grad_norm": 26.616201400756836, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -0.334661602973938, "logits/rejected": -0.31732168793678284, "logps/chosen": -223.27647399902344, "logps/ref_chosen": -65.89128875732422, "logps/ref_rejected": -91.04875183105469, "logps/rejected": -347.4496765136719, "loss": 1.1221, "margin_dpo/margin_mean": 99.0157470703125, "margin_dpo/margin_std": 157.97543334960938, "step": 228 }, { "KL/chosen_KL_mean": -156.45465087890625, "KL/mean": -210.65725708007812, "KL/rejected_KL_mean": -264.85986328125, "KL/std": 146.39035034179688, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.004005359020084143, "fcm_dpo/delta": 0.016502387821674347, "fcm_dpo/margin": 108.40519714355469, "fcm_dpo/q_t": 0.40141230821609497, "grad_norm": 31.62211036682129, "learning_rate": 4.212490049118951e-07, "logits/chosen": -0.43813785910606384, "logits/rejected": -0.4117690920829773, "logps/chosen": -227.16104125976562, "logps/ref_chosen": -70.70637512207031, "logps/ref_rejected": -84.52741241455078, "logps/rejected": -349.38726806640625, "loss": 1.0863, "margin_dpo/margin_mean": 108.40519714355469, "margin_dpo/margin_std": 154.72164916992188, "step": 229 }, { "KL/chosen_KL_mean": -128.100830078125, "KL/mean": -208.39639282226562, "KL/rejected_KL_mean": -288.69195556640625, "KL/std": 143.41812133789062, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.0038900193758308887, "fcm_dpo/delta": -0.1867695450782776, "fcm_dpo/margin": 160.59112548828125, "fcm_dpo/q_t": 0.35838133096694946, "grad_norm": 25.45254898071289, "learning_rate": 4.203117865141635e-07, "logits/chosen": -0.2791915535926819, "logits/rejected": -0.28188395500183105, "logps/chosen": -167.38284301757812, "logps/ref_chosen": -39.282005310058594, "logps/ref_rejected": -85.62191009521484, "logps/rejected": -374.3138427734375, "loss": 0.942, "margin_dpo/margin_mean": 160.59112548828125, "margin_dpo/margin_std": 159.54147338867188, "step": 230 }, { "KL/chosen_KL_mean": -142.98562622070312, "KL/mean": -199.78729248046875, "KL/rejected_KL_mean": -256.5889587402344, "KL/std": 126.15511322021484, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.0038650333881378174, "fcm_dpo/delta": 0.011422183364629745, "fcm_dpo/margin": 113.60334777832031, "fcm_dpo/q_t": 0.4017235040664673, "grad_norm": 25.279603958129883, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -0.36105793714523315, "logits/rejected": -0.331612229347229, "logps/chosen": -206.26206970214844, "logps/ref_chosen": -63.27644348144531, "logps/ref_rejected": -74.1239013671875, "logps/rejected": -330.7128601074219, "loss": 1.065, "margin_dpo/margin_mean": 113.60334777832031, "margin_dpo/margin_std": 144.85211181640625, "step": 231 }, { "KL/chosen_KL_mean": -171.13592529296875, "KL/mean": -216.0245361328125, "KL/rejected_KL_mean": -260.9131774902344, "KL/std": 143.8835906982422, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.003919616807252169, "fcm_dpo/delta": 0.10181022435426712, "fcm_dpo/margin": 89.77725219726562, "fcm_dpo/q_t": 0.42291808128356934, "grad_norm": 26.38113021850586, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -0.39240676164627075, "logits/rejected": -0.36876434087753296, "logps/chosen": -241.88467407226562, "logps/ref_chosen": -70.74876403808594, "logps/ref_rejected": -83.97706604003906, "logps/rejected": -344.8902587890625, "loss": 1.1423, "margin_dpo/margin_mean": 89.77725219726562, "margin_dpo/margin_std": 148.82798767089844, "step": 232 }, { "KL/chosen_KL_mean": -155.85494995117188, "KL/mean": -224.17303466796875, "KL/rejected_KL_mean": -292.4911193847656, "KL/std": 150.75767517089844, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.003898974508047104, "fcm_dpo/delta": -0.08736483752727509, "fcm_dpo/margin": 136.63619995117188, "fcm_dpo/q_t": 0.3861563205718994, "grad_norm": 24.671648025512695, "learning_rate": 4.174733034541245e-07, "logits/chosen": -0.35894495248794556, "logits/rejected": -0.3622890114784241, "logps/chosen": -210.73788452148438, "logps/ref_chosen": -54.8829345703125, "logps/ref_rejected": -107.4800796508789, "logps/rejected": -399.97119140625, "loss": 1.0459, "margin_dpo/margin_mean": 136.63619995117188, "margin_dpo/margin_std": 192.1343536376953, "step": 233 }, { "KL/chosen_KL_mean": -145.80184936523438, "KL/mean": -217.7124786376953, "KL/rejected_KL_mean": -289.62310791015625, "KL/std": 132.61508178710938, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.003794679883867502, "fcm_dpo/delta": -0.1029752865433693, "fcm_dpo/margin": 143.82122802734375, "fcm_dpo/q_t": 0.3759935796260834, "grad_norm": 38.8779296875, "learning_rate": 4.165182829193126e-07, "logits/chosen": -0.33827099204063416, "logits/rejected": -0.3627021312713623, "logps/chosen": -189.89637756347656, "logps/ref_chosen": -44.094520568847656, "logps/ref_rejected": -100.00663757324219, "logps/rejected": -389.6297302246094, "loss": 0.9867, "margin_dpo/margin_mean": 143.82122802734375, "margin_dpo/margin_std": 147.23887634277344, "step": 234 }, { "KL/chosen_KL_mean": -175.931396484375, "KL/mean": -221.4193878173828, "KL/rejected_KL_mean": -266.9073486328125, "KL/std": 124.45712280273438, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.003867072518914938, "fcm_dpo/delta": 0.10145650804042816, "fcm_dpo/margin": 90.97599792480469, "fcm_dpo/q_t": 0.4202185869216919, "grad_norm": 24.789424896240234, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -0.39839839935302734, "logits/rejected": -0.3760639429092407, "logps/chosen": -238.1693115234375, "logps/ref_chosen": -62.237911224365234, "logps/ref_rejected": -90.39506530761719, "logps/rejected": -357.30242919921875, "loss": 1.1413, "margin_dpo/margin_mean": 90.97599792480469, "margin_dpo/margin_std": 147.0844268798828, "step": 235 }, { "KL/chosen_KL_mean": -126.4316635131836, "KL/mean": -199.6041259765625, "KL/rejected_KL_mean": -272.77655029296875, "KL/std": 136.3174591064453, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.0038139871321618557, "fcm_dpo/delta": -0.11453250050544739, "fcm_dpo/margin": 146.34490966796875, "fcm_dpo/q_t": 0.37070026993751526, "grad_norm": 42.87751007080078, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -0.3801974058151245, "logits/rejected": -0.38002660870552063, "logps/chosen": -175.7730255126953, "logps/ref_chosen": -49.34136199951172, "logps/ref_rejected": -103.51162719726562, "logps/rejected": -376.2882080078125, "loss": 0.9602, "margin_dpo/margin_mean": 146.34490966796875, "margin_dpo/margin_std": 129.97216796875, "step": 236 }, { "KL/chosen_KL_mean": -162.65077209472656, "KL/mean": -220.71786499023438, "KL/rejected_KL_mean": -278.78497314453125, "KL/std": 137.26528930664062, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.003799371188506484, "fcm_dpo/delta": 0.009143916890025139, "fcm_dpo/margin": 116.13417053222656, "fcm_dpo/q_t": 0.400818407535553, "grad_norm": 24.1297550201416, "learning_rate": 4.136269950853473e-07, "logits/chosen": -0.3950151801109314, "logits/rejected": -0.38823604583740234, "logps/chosen": -216.8188934326172, "logps/ref_chosen": -54.168121337890625, "logps/ref_rejected": -94.78036499023438, "logps/rejected": -373.5653076171875, "loss": 1.0747, "margin_dpo/margin_mean": 116.13417053222656, "margin_dpo/margin_std": 158.19332885742188, "step": 237 }, { "KL/chosen_KL_mean": -147.71888732910156, "KL/mean": -203.96322631835938, "KL/rejected_KL_mean": -260.20758056640625, "KL/std": 133.24746704101562, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.0037925869692116976, "fcm_dpo/delta": 0.024032268673181534, "fcm_dpo/margin": 112.48866271972656, "fcm_dpo/q_t": 0.40515145659446716, "grad_norm": 27.994230270385742, "learning_rate": 4.126545750510605e-07, "logits/chosen": -0.3679651618003845, "logits/rejected": -0.3837740421295166, "logps/chosen": -201.6920166015625, "logps/ref_chosen": -53.973121643066406, "logps/ref_rejected": -89.41795349121094, "logps/rejected": -349.62548828125, "loss": 1.0773, "margin_dpo/margin_mean": 112.48866271972656, "margin_dpo/margin_std": 148.3927001953125, "step": 238 }, { "KL/chosen_KL_mean": -157.4866943359375, "KL/mean": -220.90524291992188, "KL/rejected_KL_mean": -284.32379150390625, "KL/std": 127.85582733154297, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.0037702773697674274, "fcm_dpo/delta": -0.031009189784526825, "fcm_dpo/margin": 126.83708953857422, "fcm_dpo/q_t": 0.3901780843734741, "grad_norm": 36.62656784057617, "learning_rate": 4.116778689174514e-07, "logits/chosen": -0.37180736660957336, "logits/rejected": -0.35609129071235657, "logps/chosen": -215.58453369140625, "logps/ref_chosen": -58.09782409667969, "logps/ref_rejected": -93.59294128417969, "logps/rejected": -377.916748046875, "loss": 1.0339, "margin_dpo/margin_mean": 126.83708953857422, "margin_dpo/margin_std": 139.82359313964844, "step": 239 }, { "KL/chosen_KL_mean": -176.02816772460938, "KL/mean": -231.24081420898438, "KL/rejected_KL_mean": -286.4534912109375, "KL/std": 144.84393310546875, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.0037977946922183037, "fcm_dpo/delta": 0.03184373676776886, "fcm_dpo/margin": 110.42532348632812, "fcm_dpo/q_t": 0.40580376982688904, "grad_norm": 30.680309295654297, "learning_rate": 4.106969024216348e-07, "logits/chosen": -0.3963586986064911, "logits/rejected": -0.3742924928665161, "logps/chosen": -236.64263916015625, "logps/ref_chosen": -60.6144905090332, "logps/ref_rejected": -74.1185302734375, "logps/rejected": -360.572021484375, "loss": 1.1079, "margin_dpo/margin_mean": 110.42530822753906, "margin_dpo/margin_std": 169.20822143554688, "step": 240 }, { "KL/chosen_KL_mean": -157.76492309570312, "KL/mean": -244.28536987304688, "KL/rejected_KL_mean": -330.80584716796875, "KL/std": 164.7505340576172, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.0037105297669768333, "fcm_dpo/delta": -0.20491331815719604, "fcm_dpo/margin": 173.0408935546875, "fcm_dpo/q_t": 0.36018630862236023, "grad_norm": 25.899518966674805, "learning_rate": 4.097117014129903e-07, "logits/chosen": -0.40121355652809143, "logits/rejected": -0.374218225479126, "logps/chosen": -223.85598754882812, "logps/ref_chosen": -66.091064453125, "logps/ref_rejected": -88.06088256835938, "logps/rejected": -418.8667297363281, "loss": 0.9517, "margin_dpo/margin_mean": 173.0408935546875, "margin_dpo/margin_std": 188.6272735595703, "step": 241 }, { "KL/chosen_KL_mean": -182.60260009765625, "KL/mean": -244.80023193359375, "KL/rejected_KL_mean": -306.9978942871094, "KL/std": 141.99755859375, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.0036771988961845636, "fcm_dpo/delta": -0.008242711424827576, "fcm_dpo/margin": 124.39527893066406, "fcm_dpo/q_t": 0.3988867402076721, "grad_norm": 33.00807189941406, "learning_rate": 4.087222918524807e-07, "logits/chosen": -0.34588557481765747, "logits/rejected": -0.3205920457839966, "logps/chosen": -250.46652221679688, "logps/ref_chosen": -67.86392974853516, "logps/ref_rejected": -83.36033630371094, "logps/rejected": -390.35821533203125, "loss": 1.0644, "margin_dpo/margin_mean": 124.39527893066406, "margin_dpo/margin_std": 165.65414428710938, "step": 242 }, { "KL/chosen_KL_mean": -189.4199981689453, "KL/mean": -265.73345947265625, "KL/rejected_KL_mean": -342.04693603515625, "KL/std": 162.802001953125, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.0035950704477727413, "fcm_dpo/delta": -0.1046651154756546, "fcm_dpo/margin": 152.62692260742188, "fcm_dpo/q_t": 0.3775637149810791, "grad_norm": 23.36472511291504, "learning_rate": 4.07728699811968e-07, "logits/chosen": -0.35052040219306946, "logits/rejected": -0.3172234892845154, "logps/chosen": -252.50424194335938, "logps/ref_chosen": -63.0842399597168, "logps/ref_rejected": -76.33563232421875, "logps/rejected": -418.382568359375, "loss": 1.002, "margin_dpo/margin_mean": 152.62692260742188, "margin_dpo/margin_std": 179.21963500976562, "step": 243 }, { "KL/chosen_KL_mean": -165.89877319335938, "KL/mean": -243.54913330078125, "KL/rejected_KL_mean": -321.19952392578125, "KL/std": 156.23619079589844, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.0035196980461478233, "fcm_dpo/delta": -0.10252860188484192, "fcm_dpo/margin": 155.30075073242188, "fcm_dpo/q_t": 0.37579482793807983, "grad_norm": 31.815914154052734, "learning_rate": 4.067309514735267e-07, "logits/chosen": -0.4422183632850647, "logits/rejected": -0.4340020716190338, "logps/chosen": -227.03945922851562, "logps/ref_chosen": -61.140689849853516, "logps/ref_rejected": -94.89193725585938, "logps/rejected": -416.0914306640625, "loss": 0.9835, "margin_dpo/margin_mean": 155.30075073242188, "margin_dpo/margin_std": 158.2154541015625, "step": 244 }, { "KL/chosen_KL_mean": -185.9759521484375, "KL/mean": -247.7681884765625, "KL/rejected_KL_mean": -309.5604248046875, "KL/std": 153.73590087890625, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.0034865711349993944, "fcm_dpo/delta": 0.01841205358505249, "fcm_dpo/margin": 123.58447265625, "fcm_dpo/q_t": 0.4031563997268677, "grad_norm": 30.421016693115234, "learning_rate": 4.057290731287531e-07, "logits/chosen": -0.3945258855819702, "logits/rejected": -0.3583070635795593, "logps/chosen": -253.23825073242188, "logps/ref_chosen": -67.26228332519531, "logps/ref_rejected": -87.64010620117188, "logps/rejected": -397.2005310058594, "loss": 1.0885, "margin_dpo/margin_mean": 123.58447265625, "margin_dpo/margin_std": 169.17431640625, "step": 245 }, { "KL/chosen_KL_mean": -179.17410278320312, "KL/mean": -242.2460479736328, "KL/rejected_KL_mean": -305.3179626464844, "KL/std": 167.1992950439453, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.003517426550388336, "fcm_dpo/delta": 0.006501421332359314, "fcm_dpo/margin": 126.14387512207031, "fcm_dpo/q_t": 0.40243223309516907, "grad_norm": 23.121171951293945, "learning_rate": 4.047230911780736e-07, "logits/chosen": -0.45255112648010254, "logits/rejected": -0.41291046142578125, "logps/chosen": -245.87107849121094, "logps/ref_chosen": -66.69696807861328, "logps/ref_rejected": -84.34634399414062, "logps/rejected": -389.664306640625, "loss": 1.0763, "margin_dpo/margin_mean": 126.14387512207031, "margin_dpo/margin_std": 177.28350830078125, "step": 246 }, { "KL/chosen_KL_mean": -204.04139709472656, "KL/mean": -289.42462158203125, "KL/rejected_KL_mean": -374.80780029296875, "KL/std": 170.71331787109375, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.0034450972452759743, "fcm_dpo/delta": -0.14722290635108948, "fcm_dpo/margin": 170.76641845703125, "fcm_dpo/q_t": 0.36744260787963867, "grad_norm": 27.307777404785156, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -0.38776254653930664, "logits/rejected": -0.38508284091949463, "logps/chosen": -260.646728515625, "logps/ref_chosen": -56.6053466796875, "logps/ref_rejected": -106.29326629638672, "logps/rejected": -481.10107421875, "loss": 0.971, "margin_dpo/margin_mean": 170.76641845703125, "margin_dpo/margin_std": 182.0000762939453, "step": 247 }, { "KL/chosen_KL_mean": -174.81797790527344, "KL/mean": -248.60604858398438, "KL/rejected_KL_mean": -322.3941345214844, "KL/std": 136.95773315429688, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.003385141957551241, "fcm_dpo/delta": -0.052636757493019104, "fcm_dpo/margin": 147.57614135742188, "fcm_dpo/q_t": 0.382654070854187, "grad_norm": 25.895183563232422, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -0.38079047203063965, "logits/rejected": -0.39816635847091675, "logps/chosen": -218.86119079589844, "logps/ref_chosen": -44.043216705322266, "logps/ref_rejected": -91.85687255859375, "logps/rejected": -414.2510070800781, "loss": 0.9945, "margin_dpo/margin_mean": 147.57614135742188, "margin_dpo/margin_std": 130.29660034179688, "step": 248 }, { "KL/chosen_KL_mean": -225.7408905029297, "KL/mean": -265.8685302734375, "KL/rejected_KL_mean": -305.9962158203125, "KL/std": 147.00021362304688, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.0034770106431096792, "fcm_dpo/delta": 0.17593783140182495, "fcm_dpo/margin": 80.25532531738281, "fcm_dpo/q_t": 0.43721505999565125, "grad_norm": 48.399410247802734, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -0.43213483691215515, "logits/rejected": -0.40453487634658813, "logps/chosen": -288.1832275390625, "logps/ref_chosen": -62.442352294921875, "logps/ref_rejected": -80.46806335449219, "logps/rejected": -386.46429443359375, "loss": 1.2089, "margin_dpo/margin_mean": 80.25532531738281, "margin_dpo/margin_std": 169.0428466796875, "step": 249 }, { "KL/chosen_KL_mean": -195.34091186523438, "KL/mean": -271.7189025878906, "KL/rejected_KL_mean": -348.096923828125, "KL/std": 147.0026092529297, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.0034801422152668238, "fcm_dpo/delta": -0.08620613068342209, "fcm_dpo/margin": 152.75599670410156, "fcm_dpo/q_t": 0.3768426179885864, "grad_norm": 45.44241714477539, "learning_rate": 4.006586590948141e-07, "logits/chosen": -0.43205228447914124, "logits/rejected": -0.3733093738555908, "logps/chosen": -260.97760009765625, "logps/ref_chosen": -65.63668823242188, "logps/ref_rejected": -73.87184143066406, "logps/rejected": -421.96875, "loss": 0.9819, "margin_dpo/margin_mean": 152.75599670410156, "margin_dpo/margin_std": 142.21054077148438, "step": 250 }, { "KL/chosen_KL_mean": -213.68890380859375, "KL/mean": -265.701904296875, "KL/rejected_KL_mean": -317.71490478515625, "KL/std": 156.36135864257812, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.0034832912497222424, "fcm_dpo/delta": 0.09098668396472931, "fcm_dpo/margin": 104.02600860595703, "fcm_dpo/q_t": 0.4189932644367218, "grad_norm": 30.63431739807129, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -0.3991888761520386, "logits/rejected": -0.3627336621284485, "logps/chosen": -270.8716125488281, "logps/ref_chosen": -57.182716369628906, "logps/ref_rejected": -77.66343688964844, "logps/rejected": -395.37835693359375, "loss": 1.1403, "margin_dpo/margin_mean": 104.02600860595703, "margin_dpo/margin_std": 167.71755981445312, "step": 251 }, { "KL/chosen_KL_mean": -201.41134643554688, "KL/mean": -272.4978332519531, "KL/rejected_KL_mean": -343.5843505859375, "KL/std": 137.3273162841797, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.0034778579138219357, "fcm_dpo/delta": -0.047106366604566574, "fcm_dpo/margin": 142.17300415039062, "fcm_dpo/q_t": 0.38277339935302734, "grad_norm": 26.57343101501465, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -0.4910251498222351, "logits/rejected": -0.4520424008369446, "logps/chosen": -273.09698486328125, "logps/ref_chosen": -71.68563842773438, "logps/ref_rejected": -84.75799560546875, "logps/rejected": -428.34234619140625, "loss": 1.0023, "margin_dpo/margin_mean": 142.17300415039062, "margin_dpo/margin_std": 131.0556640625, "step": 252 }, { "KL/chosen_KL_mean": -171.62469482421875, "KL/mean": -237.47177124023438, "KL/rejected_KL_mean": -303.31884765625, "KL/std": 145.25961303710938, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.0034945448860526085, "fcm_dpo/delta": -0.011328611522912979, "fcm_dpo/margin": 131.69415283203125, "fcm_dpo/q_t": 0.39354246854782104, "grad_norm": 30.445125579833984, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -0.4647923707962036, "logits/rejected": -0.45453959703445435, "logps/chosen": -240.75863647460938, "logps/ref_chosen": -69.1339340209961, "logps/ref_rejected": -98.70252990722656, "logps/rejected": -402.0213928222656, "loss": 1.0526, "margin_dpo/margin_mean": 131.69415283203125, "margin_dpo/margin_std": 157.84063720703125, "step": 253 }, { "KL/chosen_KL_mean": -175.22341918945312, "KL/mean": -234.58761596679688, "KL/rejected_KL_mean": -293.9517822265625, "KL/std": 153.671142578125, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.0034880484454333782, "fcm_dpo/delta": 0.03743617236614227, "fcm_dpo/margin": 118.72837829589844, "fcm_dpo/q_t": 0.4104461371898651, "grad_norm": 24.665857315063477, "learning_rate": 3.965307091713037e-07, "logits/chosen": -0.4090794026851654, "logits/rejected": -0.3888247609138489, "logps/chosen": -229.37841796875, "logps/ref_chosen": -54.154998779296875, "logps/ref_rejected": -90.30764770507812, "logps/rejected": -384.2594299316406, "loss": 1.1125, "margin_dpo/margin_mean": 118.72837829589844, "margin_dpo/margin_std": 191.54190063476562, "step": 254 }, { "KL/chosen_KL_mean": -180.15988159179688, "KL/mean": -243.97674560546875, "KL/rejected_KL_mean": -307.79364013671875, "KL/std": 137.28518676757812, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.0034879068844020367, "fcm_dpo/delta": 0.004787445068359375, "fcm_dpo/margin": 127.63375091552734, "fcm_dpo/q_t": 0.3985871970653534, "grad_norm": 22.473079681396484, "learning_rate": 3.954890003969163e-07, "logits/chosen": -0.3677319884300232, "logits/rejected": -0.35488447546958923, "logps/chosen": -237.30154418945312, "logps/ref_chosen": -57.14167022705078, "logps/ref_rejected": -90.2085952758789, "logps/rejected": -398.0022277832031, "loss": 1.0758, "margin_dpo/margin_mean": 127.63375091552734, "margin_dpo/margin_std": 172.2808074951172, "step": 255 }, { "KL/chosen_KL_mean": -152.079345703125, "KL/mean": -220.30516052246094, "KL/rejected_KL_mean": -288.53094482421875, "KL/std": 147.42538452148438, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.003487388137727976, "fcm_dpo/delta": -0.02715311199426651, "fcm_dpo/margin": 136.45159912109375, "fcm_dpo/q_t": 0.3921736478805542, "grad_norm": 25.29527473449707, "learning_rate": 3.944434578520628e-07, "logits/chosen": -0.358262300491333, "logits/rejected": -0.36426687240600586, "logps/chosen": -207.2428436279297, "logps/ref_chosen": -55.163490295410156, "logps/ref_rejected": -92.56291961669922, "logps/rejected": -381.0938720703125, "loss": 1.0442, "margin_dpo/margin_mean": 136.45159912109375, "margin_dpo/margin_std": 168.40225219726562, "step": 256 }, { "KL/chosen_KL_mean": -150.44158935546875, "KL/mean": -220.73284912109375, "KL/rejected_KL_mean": -291.0240783691406, "KL/std": 151.136474609375, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.003438519313931465, "fcm_dpo/delta": -0.03647083416581154, "fcm_dpo/margin": 140.58251953125, "fcm_dpo/q_t": 0.3912065625190735, "grad_norm": 26.34146499633789, "learning_rate": 3.933941090877615e-07, "logits/chosen": -0.36043840646743774, "logits/rejected": -0.3441726565361023, "logps/chosen": -199.86529541015625, "logps/ref_chosen": -49.42369842529297, "logps/ref_rejected": -79.53791809082031, "logps/rejected": -370.56201171875, "loss": 1.043, "margin_dpo/margin_mean": 140.58251953125, "margin_dpo/margin_std": 168.63809204101562, "step": 257 }, { "KL/chosen_KL_mean": -197.21380615234375, "KL/mean": -265.21221923828125, "KL/rejected_KL_mean": -333.2106628417969, "KL/std": 157.57400512695312, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.003452843287959695, "fcm_dpo/delta": -0.02056654915213585, "fcm_dpo/margin": 135.99685668945312, "fcm_dpo/q_t": 0.39261341094970703, "grad_norm": 26.73494529724121, "learning_rate": 3.923409817553284e-07, "logits/chosen": -0.34393584728240967, "logits/rejected": -0.3391512632369995, "logps/chosen": -256.5979309082031, "logps/ref_chosen": -59.384124755859375, "logps/ref_rejected": -95.99010467529297, "logps/rejected": -429.20074462890625, "loss": 1.0654, "margin_dpo/margin_mean": 135.9968719482422, "margin_dpo/margin_std": 185.92190551757812, "step": 258 }, { "KL/chosen_KL_mean": -186.67657470703125, "KL/mean": -245.43338012695312, "KL/rejected_KL_mean": -304.190185546875, "KL/std": 151.03363037109375, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.0034547275863587856, "fcm_dpo/delta": 0.04579560458660126, "fcm_dpo/margin": 117.51361083984375, "fcm_dpo/q_t": 0.4087771773338318, "grad_norm": 26.122661590576172, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -0.4058660864830017, "logits/rejected": -0.4022112488746643, "logps/chosen": -239.50491333007812, "logps/ref_chosen": -52.828346252441406, "logps/ref_rejected": -89.191650390625, "logps/rejected": -393.3818359375, "loss": 1.1018, "margin_dpo/margin_mean": 117.51361083984375, "margin_dpo/margin_std": 169.58209228515625, "step": 259 }, { "KL/chosen_KL_mean": -197.8670654296875, "KL/mean": -275.69793701171875, "KL/rejected_KL_mean": -353.52880859375, "KL/std": 160.15545654296875, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.003441175911575556, "fcm_dpo/delta": -0.09045177698135376, "fcm_dpo/margin": 155.6616973876953, "fcm_dpo/q_t": 0.3801797330379486, "grad_norm": 28.81717872619629, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -0.3697262406349182, "logits/rejected": -0.3860480487346649, "logps/chosen": -245.2847442626953, "logps/ref_chosen": -47.41767501831055, "logps/ref_rejected": -95.08978271484375, "logps/rejected": -448.61859130859375, "loss": 0.9959, "margin_dpo/margin_mean": 155.6616973876953, "margin_dpo/margin_std": 170.5771942138672, "step": 260 }, { "KL/chosen_KL_mean": -203.37054443359375, "KL/mean": -278.1752624511719, "KL/rejected_KL_mean": -352.9800109863281, "KL/std": 175.67840576171875, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.003373971674591303, "fcm_dpo/delta": -0.05800588056445122, "fcm_dpo/margin": 149.6094970703125, "fcm_dpo/q_t": 0.3877621293067932, "grad_norm": 22.53188705444336, "learning_rate": 3.891592063515376e-07, "logits/chosen": -0.3110651969909668, "logits/rejected": -0.3086443543434143, "logps/chosen": -256.40191650390625, "logps/ref_chosen": -53.03137969970703, "logps/ref_rejected": -88.51494598388672, "logps/rejected": -441.4949645996094, "loss": 1.0402, "margin_dpo/margin_mean": 149.6094970703125, "margin_dpo/margin_std": 194.22335815429688, "step": 261 }, { "KL/chosen_KL_mean": -238.98031616210938, "KL/mean": -299.66339111328125, "KL/rejected_KL_mean": -360.346435546875, "KL/std": 159.51092529296875, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.003405648982152343, "fcm_dpo/delta": 0.037785448133945465, "fcm_dpo/margin": 121.36612701416016, "fcm_dpo/q_t": 0.4051642119884491, "grad_norm": 28.0867919921875, "learning_rate": 3.880912432401264e-07, "logits/chosen": -0.31964707374572754, "logits/rejected": -0.2917061746120453, "logps/chosen": -298.6004638671875, "logps/ref_chosen": -59.620140075683594, "logps/ref_rejected": -86.41853332519531, "logps/rejected": -446.76495361328125, "loss": 1.0826, "margin_dpo/margin_mean": 121.36612701416016, "margin_dpo/margin_std": 158.1082000732422, "step": 262 }, { "KL/chosen_KL_mean": -213.3681182861328, "KL/mean": -300.8740234375, "KL/rejected_KL_mean": -388.3798828125, "KL/std": 187.53240966796875, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.0033140634186565876, "fcm_dpo/delta": -0.13908036053180695, "fcm_dpo/margin": 175.0117645263672, "fcm_dpo/q_t": 0.3701428174972534, "grad_norm": 23.549612045288086, "learning_rate": 3.870196412960302e-07, "logits/chosen": -0.3330538868904114, "logits/rejected": -0.3084709942340851, "logps/chosen": -272.7890625, "logps/ref_chosen": -59.42094421386719, "logps/ref_rejected": -96.85720825195312, "logps/rejected": -485.23712158203125, "loss": 0.9859, "margin_dpo/margin_mean": 175.0117645263672, "margin_dpo/margin_std": 198.38455200195312, "step": 263 }, { "KL/chosen_KL_mean": -223.22073364257812, "KL/mean": -297.41522216796875, "KL/rejected_KL_mean": -371.6096496582031, "KL/std": 168.37533569335938, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.0032626159954816103, "fcm_dpo/delta": -0.03732195869088173, "fcm_dpo/margin": 148.388916015625, "fcm_dpo/q_t": 0.39182916283607483, "grad_norm": 26.550914764404297, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -0.38302069902420044, "logits/rejected": -0.37319496273994446, "logps/chosen": -285.94281005859375, "logps/ref_chosen": -62.722084045410156, "logps/ref_rejected": -93.85620880126953, "logps/rejected": -465.46588134765625, "loss": 1.0515, "margin_dpo/margin_mean": 148.388916015625, "margin_dpo/margin_std": 190.17330932617188, "step": 264 }, { "KL/chosen_KL_mean": -246.03799438476562, "KL/mean": -317.5897216796875, "KL/rejected_KL_mean": -389.1414489746094, "KL/std": 192.48684692382812, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.003271650057286024, "fcm_dpo/delta": -0.01907782442867756, "fcm_dpo/margin": 143.10342407226562, "fcm_dpo/q_t": 0.3984997272491455, "grad_norm": 30.72321319580078, "learning_rate": 3.848656339557562e-07, "logits/chosen": -0.33858364820480347, "logits/rejected": -0.3238202929496765, "logps/chosen": -308.00946044921875, "logps/ref_chosen": -61.971466064453125, "logps/ref_rejected": -88.02059936523438, "logps/rejected": -477.16204833984375, "loss": 1.0942, "margin_dpo/margin_mean": 143.10342407226562, "margin_dpo/margin_std": 228.45199584960938, "step": 265 }, { "KL/chosen_KL_mean": -244.74679565429688, "KL/mean": -304.958740234375, "KL/rejected_KL_mean": -365.17071533203125, "KL/std": 162.22457885742188, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0032967233564704657, "fcm_dpo/delta": 0.055163055658340454, "fcm_dpo/margin": 120.42388916015625, "fcm_dpo/q_t": 0.41121941804885864, "grad_norm": 51.763267517089844, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -0.3775002956390381, "logits/rejected": -0.33649495244026184, "logps/chosen": -311.8464660644531, "logps/ref_chosen": -67.09967041015625, "logps/ref_rejected": -67.97122192382812, "logps/rejected": -433.1419372558594, "loss": 1.1123, "margin_dpo/margin_mean": 120.42388916015625, "margin_dpo/margin_std": 183.71939086914062, "step": 266 }, { "KL/chosen_KL_mean": -217.10748291015625, "KL/mean": -288.5802001953125, "KL/rejected_KL_mean": -360.0528869628906, "KL/std": 174.79310607910156, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.0032819192856550217, "fcm_dpo/delta": -0.02046758309006691, "fcm_dpo/margin": 142.94540405273438, "fcm_dpo/q_t": 0.39428484439849854, "grad_norm": 41.82976531982422, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -0.3888688385486603, "logits/rejected": -0.3568268418312073, "logps/chosen": -286.0782470703125, "logps/ref_chosen": -68.97075653076172, "logps/ref_rejected": -90.16844940185547, "logps/rejected": -450.2213134765625, "loss": 1.0632, "margin_dpo/margin_mean": 142.94540405273438, "margin_dpo/margin_std": 193.6571044921875, "step": 267 }, { "KL/chosen_KL_mean": -226.6894073486328, "KL/mean": -294.6236267089844, "KL/rejected_KL_mean": -362.557861328125, "KL/std": 167.1015625, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0032822336070239544, "fcm_dpo/delta": 0.0040205977857112885, "fcm_dpo/margin": 135.8684539794922, "fcm_dpo/q_t": 0.40099048614501953, "grad_norm": 38.428096771240234, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -0.3436671793460846, "logits/rejected": -0.3497740626335144, "logps/chosen": -282.5897216796875, "logps/ref_chosen": -55.90031051635742, "logps/ref_rejected": -101.64763641357422, "logps/rejected": -464.20550537109375, "loss": 1.0847, "margin_dpo/margin_mean": 135.8684539794922, "margin_dpo/margin_std": 199.06204223632812, "step": 268 }, { "KL/chosen_KL_mean": -241.2147216796875, "KL/mean": -326.4340515136719, "KL/rejected_KL_mean": -411.65338134765625, "KL/std": 169.1653594970703, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.0032417180482298136, "fcm_dpo/delta": -0.108495332300663, "fcm_dpo/margin": 170.43869018554688, "fcm_dpo/q_t": 0.3768116235733032, "grad_norm": 27.486454010009766, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -0.3986158072948456, "logits/rejected": -0.37994933128356934, "logps/chosen": -311.2542724609375, "logps/ref_chosen": -70.03955841064453, "logps/ref_rejected": -107.34937286376953, "logps/rejected": -519.0028076171875, "loss": 1.018, "margin_dpo/margin_mean": 170.43869018554688, "margin_dpo/margin_std": 214.85861206054688, "step": 269 }, { "KL/chosen_KL_mean": -214.1920166015625, "KL/mean": -277.78076171875, "KL/rejected_KL_mean": -341.36956787109375, "KL/std": 151.48452758789062, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.0032509169541299343, "fcm_dpo/delta": 0.037847500294446945, "fcm_dpo/margin": 127.17756652832031, "fcm_dpo/q_t": 0.40676695108413696, "grad_norm": 33.916015625, "learning_rate": 3.794189242333106e-07, "logits/chosen": -0.4463101029396057, "logits/rejected": -0.44212576746940613, "logps/chosen": -283.7254943847656, "logps/ref_chosen": -69.53347778320312, "logps/ref_rejected": -109.92864990234375, "logps/rejected": -451.2982177734375, "loss": 1.1022, "margin_dpo/margin_mean": 127.17755126953125, "margin_dpo/margin_std": 188.5105438232422, "step": 270 }, { "KL/chosen_KL_mean": -199.34915161132812, "KL/mean": -278.3819885253906, "KL/rejected_KL_mean": -357.4148254394531, "KL/std": 152.5343017578125, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.0032084607519209385, "fcm_dpo/delta": -0.06059388071298599, "fcm_dpo/margin": 158.065673828125, "fcm_dpo/q_t": 0.38492733240127563, "grad_norm": 27.501827239990234, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -0.36121702194213867, "logits/rejected": -0.3439444303512573, "logps/chosen": -256.11370849609375, "logps/ref_chosen": -56.76456832885742, "logps/ref_rejected": -92.51383972167969, "logps/rejected": -449.92864990234375, "loss": 1.0118, "margin_dpo/margin_mean": 158.06568908691406, "margin_dpo/margin_std": 172.20005798339844, "step": 271 }, { "KL/chosen_KL_mean": -184.9806671142578, "KL/mean": -277.00921630859375, "KL/rejected_KL_mean": -369.03778076171875, "KL/std": 169.01669311523438, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.003157893195748329, "fcm_dpo/delta": -0.13913074135780334, "fcm_dpo/margin": 184.05712890625, "fcm_dpo/q_t": 0.3664304316043854, "grad_norm": 45.42236328125, "learning_rate": 3.772161666010912e-07, "logits/chosen": -0.31142860651016235, "logits/rejected": -0.3235180079936981, "logps/chosen": -234.4778289794922, "logps/ref_chosen": -49.497154235839844, "logps/ref_rejected": -105.54279327392578, "logps/rejected": -474.58056640625, "loss": 0.9588, "margin_dpo/margin_mean": 184.05712890625, "margin_dpo/margin_std": 174.16859436035156, "step": 272 }, { "KL/chosen_KL_mean": -210.1597900390625, "KL/mean": -300.50250244140625, "KL/rejected_KL_mean": -390.84515380859375, "KL/std": 166.0848846435547, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.003058584639802575, "fcm_dpo/delta": -0.10935865342617035, "fcm_dpo/margin": 180.68539428710938, "fcm_dpo/q_t": 0.3739517629146576, "grad_norm": 30.045597076416016, "learning_rate": 3.761097448550755e-07, "logits/chosen": -0.3383510410785675, "logits/rejected": -0.3233621120452881, "logps/chosen": -273.13519287109375, "logps/ref_chosen": -62.97539520263672, "logps/ref_rejected": -92.49858093261719, "logps/rejected": -483.34375, "loss": 0.9825, "margin_dpo/margin_mean": 180.68539428710938, "margin_dpo/margin_std": 186.4752960205078, "step": 273 }, { "KL/chosen_KL_mean": -256.2316589355469, "KL/mean": -327.0481872558594, "KL/rejected_KL_mean": -397.86468505859375, "KL/std": 157.5728759765625, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.0030697216279804707, "fcm_dpo/delta": 0.01566966250538826, "fcm_dpo/margin": 141.63302612304688, "fcm_dpo/q_t": 0.40060126781463623, "grad_norm": 35.71431350708008, "learning_rate": 3.75e-07, "logits/chosen": -0.317413866519928, "logits/rejected": -0.30049943923950195, "logps/chosen": -311.8993835449219, "logps/ref_chosen": -55.66770935058594, "logps/ref_rejected": -77.33308410644531, "logps/rejected": -475.1977844238281, "loss": 1.067, "margin_dpo/margin_mean": 141.63304138183594, "margin_dpo/margin_std": 176.70895385742188, "step": 274 }, { "KL/chosen_KL_mean": -195.29779052734375, "KL/mean": -267.0739440917969, "KL/rejected_KL_mean": -338.85009765625, "KL/std": 158.72325134277344, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.003075392683967948, "fcm_dpo/delta": 0.008720289915800095, "fcm_dpo/margin": 143.55230712890625, "fcm_dpo/q_t": 0.39867332577705383, "grad_norm": 28.375545501708984, "learning_rate": 3.738869612786737e-07, "logits/chosen": -0.3294273614883423, "logits/rejected": -0.3342640697956085, "logps/chosen": -243.89248657226562, "logps/ref_chosen": -48.594703674316406, "logps/ref_rejected": -93.30369567871094, "logps/rejected": -432.15380859375, "loss": 1.0615, "margin_dpo/margin_mean": 143.55230712890625, "margin_dpo/margin_std": 175.27035522460938, "step": 275 }, { "KL/chosen_KL_mean": -210.61480712890625, "KL/mean": -287.1915283203125, "KL/rejected_KL_mean": -363.7682800292969, "KL/std": 161.99810791015625, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.003052051179111004, "fcm_dpo/delta": -0.01845492050051689, "fcm_dpo/margin": 153.15347290039062, "fcm_dpo/q_t": 0.39402562379837036, "grad_norm": 37.641090393066406, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -0.3240549564361572, "logits/rejected": -0.29721617698669434, "logps/chosen": -267.19219970703125, "logps/ref_chosen": -56.57740783691406, "logps/ref_rejected": -70.36566925048828, "logps/rejected": -434.13397216796875, "loss": 1.0475, "margin_dpo/margin_mean": 153.1534881591797, "margin_dpo/margin_std": 187.49969482421875, "step": 276 }, { "KL/chosen_KL_mean": -226.70089721679688, "KL/mean": -302.93359375, "KL/rejected_KL_mean": -379.166259765625, "KL/std": 161.0653076171875, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.0030543706379830837, "fcm_dpo/delta": -0.016483565792441368, "fcm_dpo/margin": 152.46539306640625, "fcm_dpo/q_t": 0.39358431100845337, "grad_norm": 27.86901092529297, "learning_rate": 3.71651119641714e-07, "logits/chosen": -0.3397817611694336, "logits/rejected": -0.32286733388900757, "logps/chosen": -282.97247314453125, "logps/ref_chosen": -56.27156066894531, "logps/ref_rejected": -92.88127136230469, "logps/rejected": -472.04754638671875, "loss": 1.049, "margin_dpo/margin_mean": 152.46539306640625, "margin_dpo/margin_std": 185.9540557861328, "step": 277 }, { "KL/chosen_KL_mean": -201.0662078857422, "KL/mean": -289.1669921875, "KL/rejected_KL_mean": -377.267822265625, "KL/std": 173.0821533203125, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.0030011532362550497, "fcm_dpo/delta": -0.08356461673974991, "fcm_dpo/margin": 176.2015838623047, "fcm_dpo/q_t": 0.38064807653427124, "grad_norm": 24.03802490234375, "learning_rate": 3.705283756425872e-07, "logits/chosen": -0.3009544014930725, "logits/rejected": -0.30515652894973755, "logps/chosen": -254.00814819335938, "logps/ref_chosen": -52.94194030761719, "logps/ref_rejected": -91.25357818603516, "logps/rejected": -468.5213928222656, "loss": 1.0017, "margin_dpo/margin_mean": 176.2015838623047, "margin_dpo/margin_std": 193.50991821289062, "step": 278 }, { "KL/chosen_KL_mean": -227.3751220703125, "KL/mean": -315.3472595214844, "KL/rejected_KL_mean": -403.3194274902344, "KL/std": 180.30804443359375, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.002944803796708584, "fcm_dpo/delta": -0.07315056025981903, "fcm_dpo/margin": 175.94432067871094, "fcm_dpo/q_t": 0.3842847943305969, "grad_norm": 28.466480255126953, "learning_rate": 3.6940245560867e-07, "logits/chosen": -0.2807590961456299, "logits/rejected": -0.278242826461792, "logps/chosen": -276.01641845703125, "logps/ref_chosen": -48.641319274902344, "logps/ref_rejected": -87.8514404296875, "logps/rejected": -491.1708679199219, "loss": 1.0292, "margin_dpo/margin_mean": 175.94430541992188, "margin_dpo/margin_std": 218.19247436523438, "step": 279 }, { "KL/chosen_KL_mean": -223.5345458984375, "KL/mean": -311.9327392578125, "KL/rejected_KL_mean": -400.33087158203125, "KL/std": 166.27877807617188, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.00291811628267169, "fcm_dpo/delta": -0.06963366270065308, "fcm_dpo/margin": 176.79629516601562, "fcm_dpo/q_t": 0.3800342082977295, "grad_norm": 34.31902313232422, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -0.3296852707862854, "logits/rejected": -0.33157432079315186, "logps/chosen": -282.3316650390625, "logps/ref_chosen": -58.797122955322266, "logps/ref_rejected": -98.61885070800781, "logps/rejected": -498.94970703125, "loss": 0.9995, "margin_dpo/margin_mean": 176.79629516601562, "margin_dpo/margin_std": 179.59506225585938, "step": 280 }, { "KL/chosen_KL_mean": -202.8314666748047, "KL/mean": -283.94891357421875, "KL/rejected_KL_mean": -365.06640625, "KL/std": 163.90113830566406, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0029120631515979767, "fcm_dpo/delta": -0.023735320195555687, "fcm_dpo/margin": 162.2349395751953, "fcm_dpo/q_t": 0.38894733786582947, "grad_norm": 22.588340759277344, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -0.36799153685569763, "logits/rejected": -0.3402552008628845, "logps/chosen": -258.3199768066406, "logps/ref_chosen": -55.488521575927734, "logps/ref_rejected": -80.88258361816406, "logps/rejected": -445.948974609375, "loss": 1.0321, "margin_dpo/margin_mean": 162.23492431640625, "margin_dpo/margin_std": 176.25933837890625, "step": 281 }, { "KL/chosen_KL_mean": -234.31324768066406, "KL/mean": -297.2037353515625, "KL/rejected_KL_mean": -360.09423828125, "KL/std": 174.16539001464844, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0029452815651893616, "fcm_dpo/delta": 0.08194846659898758, "fcm_dpo/margin": 125.781005859375, "fcm_dpo/q_t": 0.41833823919296265, "grad_norm": 31.682376861572266, "learning_rate": 3.660059364023408e-07, "logits/chosen": -0.41140317916870117, "logits/rejected": -0.3855805993080139, "logps/chosen": -307.3833923339844, "logps/ref_chosen": -73.07014465332031, "logps/ref_rejected": -95.35098266601562, "logps/rejected": -455.44525146484375, "loss": 1.1245, "margin_dpo/margin_mean": 125.781005859375, "margin_dpo/margin_std": 198.57614135742188, "step": 282 }, { "KL/chosen_KL_mean": -242.9020233154297, "KL/mean": -337.974853515625, "KL/rejected_KL_mean": -433.0476989746094, "KL/std": 197.83787536621094, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0029110005125403404, "fcm_dpo/delta": -0.10949087888002396, "fcm_dpo/margin": 190.14569091796875, "fcm_dpo/q_t": 0.37221860885620117, "grad_norm": 29.892505645751953, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -0.339688241481781, "logits/rejected": -0.3391629457473755, "logps/chosen": -304.80047607421875, "logps/ref_chosen": -61.89844512939453, "logps/ref_rejected": -96.98655700683594, "logps/rejected": -530.0342407226562, "loss": 0.9779, "margin_dpo/margin_mean": 190.14569091796875, "margin_dpo/margin_std": 186.28033447265625, "step": 283 }, { "KL/chosen_KL_mean": -236.25758361816406, "KL/mean": -328.17041015625, "KL/rejected_KL_mean": -420.08319091796875, "KL/std": 193.72735595703125, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.002844389993697405, "fcm_dpo/delta": -0.07708093523979187, "fcm_dpo/margin": 183.82562255859375, "fcm_dpo/q_t": 0.3810979723930359, "grad_norm": 27.589719772338867, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -0.40438514947891235, "logits/rejected": -0.39065611362457275, "logps/chosen": -294.693115234375, "logps/ref_chosen": -58.4355354309082, "logps/ref_rejected": -93.46926879882812, "logps/rejected": -513.552490234375, "loss": 0.9975, "margin_dpo/margin_mean": 183.82562255859375, "margin_dpo/margin_std": 193.533447265625, "step": 284 }, { "KL/chosen_KL_mean": -280.03094482421875, "KL/mean": -361.1250305175781, "KL/rejected_KL_mean": -442.2191467285156, "KL/std": 169.0643310546875, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.0028481141198426485, "fcm_dpo/delta": -0.013057064265012741, "fcm_dpo/margin": 162.18820190429688, "fcm_dpo/q_t": 0.3921695351600647, "grad_norm": 26.22045135498047, "learning_rate": 3.625819059005228e-07, "logits/chosen": -0.3738415241241455, "logits/rejected": -0.3561505079269409, "logps/chosen": -346.26312255859375, "logps/ref_chosen": -66.23219299316406, "logps/ref_rejected": -99.1268310546875, "logps/rejected": -541.345947265625, "loss": 1.043, "margin_dpo/margin_mean": 162.18821716308594, "margin_dpo/margin_std": 182.13633728027344, "step": 285 }, { "KL/chosen_KL_mean": -290.49481201171875, "KL/mean": -380.4010009765625, "KL/rejected_KL_mean": -470.30718994140625, "KL/std": 196.43020629882812, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.002812919206917286, "fcm_dpo/delta": -0.058750685304403305, "fcm_dpo/margin": 179.81234741210938, "fcm_dpo/q_t": 0.38490667939186096, "grad_norm": 30.215322494506836, "learning_rate": 3.614345889441346e-07, "logits/chosen": -0.37286484241485596, "logits/rejected": -0.354168564081192, "logps/chosen": -363.44580078125, "logps/ref_chosen": -72.95100402832031, "logps/ref_rejected": -88.58845520019531, "logps/rejected": -558.8956298828125, "loss": 1.0278, "margin_dpo/margin_mean": 179.81234741210938, "margin_dpo/margin_std": 214.6378173828125, "step": 286 }, { "KL/chosen_KL_mean": -271.04339599609375, "KL/mean": -347.6123352050781, "KL/rejected_KL_mean": -424.1812744140625, "KL/std": 178.92794799804688, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.002809434197843075, "fcm_dpo/delta": 0.020603910088539124, "fcm_dpo/margin": 153.13784790039062, "fcm_dpo/q_t": 0.40201419591903687, "grad_norm": 26.337621688842773, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -0.3177964985370636, "logits/rejected": -0.28946465253829956, "logps/chosen": -332.5845642089844, "logps/ref_chosen": -61.54115295410156, "logps/ref_rejected": -77.69607543945312, "logps/rejected": -501.8773498535156, "loss": 1.0687, "margin_dpo/margin_mean": 153.1378631591797, "margin_dpo/margin_std": 192.58251953125, "step": 287 }, { "KL/chosen_KL_mean": -267.706298828125, "KL/mean": -359.65765380859375, "KL/rejected_KL_mean": -451.6090087890625, "KL/std": 176.45693969726562, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.002759437309578061, "fcm_dpo/delta": -0.06256371736526489, "fcm_dpo/margin": 183.90274047851562, "fcm_dpo/q_t": 0.38231146335601807, "grad_norm": 24.027070999145508, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -0.3309929370880127, "logits/rejected": -0.31245338916778564, "logps/chosen": -324.3675537109375, "logps/ref_chosen": -56.661224365234375, "logps/ref_rejected": -87.33570098876953, "logps/rejected": -538.9447021484375, "loss": 1.0039, "margin_dpo/margin_mean": 183.90274047851562, "margin_dpo/margin_std": 178.67431640625, "step": 288 }, { "KL/chosen_KL_mean": -256.2225036621094, "KL/mean": -358.1459045410156, "KL/rejected_KL_mean": -460.0693359375, "KL/std": 192.28744506835938, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.002735921647399664, "fcm_dpo/delta": -0.11391443759202957, "fcm_dpo/margin": 203.8468017578125, "fcm_dpo/q_t": 0.3737669587135315, "grad_norm": 33.68012619018555, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -0.27321916818618774, "logits/rejected": -0.2723183333873749, "logps/chosen": -301.4529113769531, "logps/ref_chosen": -45.23039245605469, "logps/ref_rejected": -87.64266967773438, "logps/rejected": -547.7120361328125, "loss": 0.9817, "margin_dpo/margin_mean": 203.8468017578125, "margin_dpo/margin_std": 211.8802032470703, "step": 289 }, { "KL/chosen_KL_mean": -269.2728271484375, "KL/mean": -369.11712646484375, "KL/rejected_KL_mean": -468.96142578125, "KL/std": 200.68923950195312, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.0026709740050137043, "fcm_dpo/delta": -0.08831434696912766, "fcm_dpo/margin": 199.6885986328125, "fcm_dpo/q_t": 0.3807799816131592, "grad_norm": 24.526521682739258, "learning_rate": 3.568162605525952e-07, "logits/chosen": -0.28396081924438477, "logits/rejected": -0.3037373423576355, "logps/chosen": -324.74432373046875, "logps/ref_chosen": -55.47149658203125, "logps/ref_rejected": -116.70857238769531, "logps/rejected": -585.6699829101562, "loss": 1.0115, "margin_dpo/margin_mean": 199.68861389160156, "margin_dpo/margin_std": 236.9954376220703, "step": 290 }, { "KL/chosen_KL_mean": -227.63702392578125, "KL/mean": -321.8421630859375, "KL/rejected_KL_mean": -416.04730224609375, "KL/std": 173.67169189453125, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.002654793206602335, "fcm_dpo/delta": -0.052990131080150604, "fcm_dpo/margin": 188.41024780273438, "fcm_dpo/q_t": 0.38466930389404297, "grad_norm": 25.388965606689453, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -0.3481978476047516, "logits/rejected": -0.3348592519760132, "logps/chosen": -290.89739990234375, "logps/ref_chosen": -63.26036834716797, "logps/ref_rejected": -89.29708862304688, "logps/rejected": -505.3443603515625, "loss": 1.0182, "margin_dpo/margin_mean": 188.41024780273438, "margin_dpo/margin_std": 205.23170471191406, "step": 291 }, { "KL/chosen_KL_mean": -248.2539520263672, "KL/mean": -343.5125427246094, "KL/rejected_KL_mean": -438.7711181640625, "KL/std": 202.5133056640625, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0026254309341311455, "fcm_dpo/delta": -0.052929122000932693, "fcm_dpo/margin": 190.51718139648438, "fcm_dpo/q_t": 0.3859769105911255, "grad_norm": 24.36165428161621, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -0.32978010177612305, "logits/rejected": -0.3151566982269287, "logps/chosen": -302.1724853515625, "logps/ref_chosen": -53.91852951049805, "logps/ref_rejected": -89.96138000488281, "logps/rejected": -528.7324829101562, "loss": 1.0359, "margin_dpo/margin_mean": 190.51715087890625, "margin_dpo/margin_std": 235.57249450683594, "step": 292 }, { "KL/chosen_KL_mean": -238.14736938476562, "KL/mean": -313.1924743652344, "KL/rejected_KL_mean": -388.237548828125, "KL/std": 196.93975830078125, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.0026358608156442642, "fcm_dpo/delta": 0.05641322582960129, "fcm_dpo/margin": 150.09017944335938, "fcm_dpo/q_t": 0.411357581615448, "grad_norm": 45.91537094116211, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -0.3480873107910156, "logits/rejected": -0.336029052734375, "logps/chosen": -298.5234069824219, "logps/ref_chosen": -60.376033782958984, "logps/ref_rejected": -77.85244750976562, "logps/rejected": -466.09002685546875, "loss": 1.11, "margin_dpo/margin_mean": 150.09017944335938, "margin_dpo/margin_std": 224.6016845703125, "step": 293 }, { "KL/chosen_KL_mean": -212.2786865234375, "KL/mean": -297.28173828125, "KL/rejected_KL_mean": -382.2847900390625, "KL/std": 179.94796752929688, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.002632549963891506, "fcm_dpo/delta": 0.0024539902806282043, "fcm_dpo/margin": 170.00611877441406, "fcm_dpo/q_t": 0.39919567108154297, "grad_norm": 30.026025772094727, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -0.3344029486179352, "logits/rejected": -0.3364221453666687, "logps/chosen": -260.3662109375, "logps/ref_chosen": -48.0875358581543, "logps/ref_rejected": -81.89698791503906, "logps/rejected": -464.1817932128906, "loss": 1.0643, "margin_dpo/margin_mean": 170.00611877441406, "margin_dpo/margin_std": 218.8177490234375, "step": 294 }, { "KL/chosen_KL_mean": -257.17437744140625, "KL/mean": -346.5780029296875, "KL/rejected_KL_mean": -435.98162841796875, "KL/std": 194.82772827148438, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.0026338594034314156, "fcm_dpo/delta": -0.021993350237607956, "fcm_dpo/margin": 178.8072509765625, "fcm_dpo/q_t": 0.3934444487094879, "grad_norm": 30.588729858398438, "learning_rate": 3.509802512179737e-07, "logits/chosen": -0.3404906392097473, "logits/rejected": -0.34150180220603943, "logps/chosen": -307.0990295410156, "logps/ref_chosen": -49.92467498779297, "logps/ref_rejected": -87.45632934570312, "logps/rejected": -523.43798828125, "loss": 1.0529, "margin_dpo/margin_mean": 178.80726623535156, "margin_dpo/margin_std": 226.52297973632812, "step": 295 }, { "KL/chosen_KL_mean": -312.295654296875, "KL/mean": -379.9571533203125, "KL/rejected_KL_mean": -447.61859130859375, "KL/std": 192.12496948242188, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.0026506248395889997, "fcm_dpo/delta": 0.09461938589811325, "fcm_dpo/margin": 135.32293701171875, "fcm_dpo/q_t": 0.41918545961380005, "grad_norm": 41.94928741455078, "learning_rate": 3.498049431928577e-07, "logits/chosen": -0.39753079414367676, "logits/rejected": -0.37970608472824097, "logps/chosen": -377.7868957519531, "logps/ref_chosen": -65.49124145507812, "logps/ref_rejected": -93.08908081054688, "logps/rejected": -540.7076416015625, "loss": 1.1598, "margin_dpo/margin_mean": 135.32293701171875, "margin_dpo/margin_std": 244.38113403320312, "step": 296 }, { "KL/chosen_KL_mean": -284.9562683105469, "KL/mean": -362.5632019042969, "KL/rejected_KL_mean": -440.17010498046875, "KL/std": 185.80307006835938, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.0026883059181272984, "fcm_dpo/delta": 0.03417515754699707, "fcm_dpo/margin": 155.21383666992188, "fcm_dpo/q_t": 0.403796911239624, "grad_norm": 30.55962371826172, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.3520761728286743, "logits/rejected": -0.3518163561820984, "logps/chosen": -341.4332275390625, "logps/ref_chosen": -56.476951599121094, "logps/ref_rejected": -95.1385498046875, "logps/rejected": -535.3086547851562, "loss": 1.0703, "margin_dpo/margin_mean": 155.21383666992188, "margin_dpo/margin_std": 185.96414184570312, "step": 297 }, { "KL/chosen_KL_mean": -313.055419921875, "KL/mean": -412.99261474609375, "KL/rejected_KL_mean": -512.9298095703125, "KL/std": 245.9295654296875, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0026719551533460617, "fcm_dpo/delta": -0.08878251910209656, "fcm_dpo/margin": 199.8743896484375, "fcm_dpo/q_t": 0.38617774844169617, "grad_norm": 27.358156204223633, "learning_rate": 3.474464683231698e-07, "logits/chosen": -0.41723042726516724, "logits/rejected": -0.43608519434928894, "logps/chosen": -380.38055419921875, "logps/ref_chosen": -67.32516479492188, "logps/ref_rejected": -116.66217041015625, "logps/rejected": -629.5919799804688, "loss": 1.0429, "margin_dpo/margin_mean": 199.8743896484375, "margin_dpo/margin_std": 282.69732666015625, "step": 298 }, { "KL/chosen_KL_mean": -258.68292236328125, "KL/mean": -347.59979248046875, "KL/rejected_KL_mean": -436.51666259765625, "KL/std": 190.63723754882812, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.002657189266756177, "fcm_dpo/delta": -0.02443765103816986, "fcm_dpo/margin": 177.833740234375, "fcm_dpo/q_t": 0.395521342754364, "grad_norm": 39.517574310302734, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.3908317983150482, "logits/rejected": -0.3915952742099762, "logps/chosen": -307.64501953125, "logps/ref_chosen": -48.96209716796875, "logps/ref_rejected": -84.32823944091797, "logps/rejected": -520.8449096679688, "loss": 1.0625, "margin_dpo/margin_mean": 177.83375549316406, "margin_dpo/margin_std": 237.70399475097656, "step": 299 }, { "KL/chosen_KL_mean": -322.5706787109375, "KL/mean": -428.1722106933594, "KL/rejected_KL_mean": -533.7738037109375, "KL/std": 225.27737426757812, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.002602781169116497, "fcm_dpo/delta": -0.10538507252931595, "fcm_dpo/margin": 211.20306396484375, "fcm_dpo/q_t": 0.37716758251190186, "grad_norm": 39.216007232666016, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -0.3786313533782959, "logits/rejected": -0.38016241788864136, "logps/chosen": -381.6443786621094, "logps/ref_chosen": -59.07371139526367, "logps/ref_rejected": -95.9664535522461, "logps/rejected": -629.740234375, "loss": 1.0203, "margin_dpo/margin_mean": 211.20306396484375, "margin_dpo/margin_std": 264.9863586425781, "step": 300 }, { "KL/chosen_KL_mean": -260.64385986328125, "KL/mean": -361.54193115234375, "KL/rejected_KL_mean": -462.44000244140625, "KL/std": 204.75218200683594, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.002545170485973358, "fcm_dpo/delta": -0.06761518120765686, "fcm_dpo/margin": 201.79612731933594, "fcm_dpo/q_t": 0.38585570454597473, "grad_norm": 25.292051315307617, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -0.3724789023399353, "logits/rejected": -0.3688158392906189, "logps/chosen": -317.89324951171875, "logps/ref_chosen": -57.249366760253906, "logps/ref_rejected": -92.35354614257812, "logps/rejected": -554.7935791015625, "loss": 1.0217, "margin_dpo/margin_mean": 201.796142578125, "margin_dpo/margin_std": 242.0333251953125, "step": 301 }, { "KL/chosen_KL_mean": -214.75692749023438, "KL/mean": -296.821533203125, "KL/rejected_KL_mean": -378.88616943359375, "KL/std": 178.21484375, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0025499570183455944, "fcm_dpo/delta": 0.0327281579375267, "fcm_dpo/margin": 164.1292266845703, "fcm_dpo/q_t": 0.404863178730011, "grad_norm": 42.442935943603516, "learning_rate": 3.426989547989902e-07, "logits/chosen": -0.3850117623806, "logits/rejected": -0.39180663228034973, "logps/chosen": -265.9549255371094, "logps/ref_chosen": -51.197994232177734, "logps/ref_rejected": -97.22636413574219, "logps/rejected": -476.1125183105469, "loss": 1.0753, "margin_dpo/margin_mean": 164.12924194335938, "margin_dpo/margin_std": 206.58688354492188, "step": 302 }, { "KL/chosen_KL_mean": -237.27133178710938, "KL/mean": -314.0264892578125, "KL/rejected_KL_mean": -390.7816162109375, "KL/std": 187.21142578125, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.002579642226919532, "fcm_dpo/delta": 0.05625869706273079, "fcm_dpo/margin": 153.51028442382812, "fcm_dpo/q_t": 0.41086673736572266, "grad_norm": 34.44538879394531, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -0.39609721302986145, "logits/rejected": -0.37636637687683105, "logps/chosen": -303.98529052734375, "logps/ref_chosen": -66.71394348144531, "logps/ref_rejected": -86.94542694091797, "logps/rejected": -477.72705078125, "loss": 1.1086, "margin_dpo/margin_mean": 153.51028442382812, "margin_dpo/margin_std": 225.28631591796875, "step": 303 }, { "KL/chosen_KL_mean": -219.05499267578125, "KL/mean": -307.2504577636719, "KL/rejected_KL_mean": -395.4459228515625, "KL/std": 178.584716796875, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.002598279621452093, "fcm_dpo/delta": -0.008973244577646255, "fcm_dpo/margin": 176.39096069335938, "fcm_dpo/q_t": 0.39162832498550415, "grad_norm": 46.01495361328125, "learning_rate": 3.403104165467883e-07, "logits/chosen": -0.4321674704551697, "logits/rejected": -0.42111736536026, "logps/chosen": -291.00567626953125, "logps/ref_chosen": -71.95069885253906, "logps/ref_rejected": -90.47203063964844, "logps/rejected": -485.91796875, "loss": 1.0203, "margin_dpo/margin_mean": 176.39096069335938, "margin_dpo/margin_std": 155.45849609375, "step": 304 }, { "KL/chosen_KL_mean": -252.63079833984375, "KL/mean": -331.663818359375, "KL/rejected_KL_mean": -410.69683837890625, "KL/std": 212.29205322265625, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.0025871852412819862, "fcm_dpo/delta": 0.042250074446201324, "fcm_dpo/margin": 158.06600952148438, "fcm_dpo/q_t": 0.40933629870414734, "grad_norm": 27.672773361206055, "learning_rate": 3.391125620245535e-07, "logits/chosen": -0.4089544415473938, "logits/rejected": -0.38962921500205994, "logps/chosen": -319.426025390625, "logps/ref_chosen": -66.79523468017578, "logps/ref_rejected": -92.75459289550781, "logps/rejected": -503.451416015625, "loss": 1.1033, "margin_dpo/margin_mean": 158.06602478027344, "margin_dpo/margin_std": 231.3526611328125, "step": 305 }, { "KL/chosen_KL_mean": -250.9173583984375, "KL/mean": -331.4638366699219, "KL/rejected_KL_mean": -412.01031494140625, "KL/std": 187.9032745361328, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.0026251128874719143, "fcm_dpo/delta": 0.028183521702885628, "fcm_dpo/margin": 161.09298706054688, "fcm_dpo/q_t": 0.4037204384803772, "grad_norm": 26.20707893371582, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -0.3975362181663513, "logits/rejected": -0.3718588948249817, "logps/chosen": -320.60125732421875, "logps/ref_chosen": -69.68389892578125, "logps/ref_rejected": -85.15919494628906, "logps/rejected": -497.16949462890625, "loss": 1.0712, "margin_dpo/margin_mean": 161.09298706054688, "margin_dpo/margin_std": 201.70741271972656, "step": 306 }, { "KL/chosen_KL_mean": -238.816650390625, "KL/mean": -321.26611328125, "KL/rejected_KL_mean": -403.7156066894531, "KL/std": 177.045166015625, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.00262629147619009, "fcm_dpo/delta": 0.01766267418861389, "fcm_dpo/margin": 164.89895629882812, "fcm_dpo/q_t": 0.4013682007789612, "grad_norm": 25.059446334838867, "learning_rate": 3.367098400098881e-07, "logits/chosen": -0.3578367233276367, "logits/rejected": -0.33675235509872437, "logps/chosen": -308.9820556640625, "logps/ref_chosen": -70.16542053222656, "logps/ref_rejected": -86.97230529785156, "logps/rejected": -490.68792724609375, "loss": 1.074, "margin_dpo/margin_mean": 164.89895629882812, "margin_dpo/margin_std": 216.65493774414062, "step": 307 }, { "KL/chosen_KL_mean": -241.0372314453125, "KL/mean": -333.34759521484375, "KL/rejected_KL_mean": -425.657958984375, "KL/std": 201.32351684570312, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.002621263265609741, "fcm_dpo/delta": -0.03565025329589844, "fcm_dpo/margin": 184.62071228027344, "fcm_dpo/q_t": 0.38778913021087646, "grad_norm": 33.72212600708008, "learning_rate": 3.355050358314172e-07, "logits/chosen": -0.3401590585708618, "logits/rejected": -0.3254951238632202, "logps/chosen": -296.2822265625, "logps/ref_chosen": -55.2449951171875, "logps/ref_rejected": -79.37226104736328, "logps/rejected": -505.03021240234375, "loss": 1.02, "margin_dpo/margin_mean": 184.62069702148438, "margin_dpo/margin_std": 192.87948608398438, "step": 308 }, { "KL/chosen_KL_mean": -237.29989624023438, "KL/mean": -329.2324523925781, "KL/rejected_KL_mean": -421.16497802734375, "KL/std": 205.91294860839844, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0026156248059123755, "fcm_dpo/delta": -0.032885223627090454, "fcm_dpo/margin": 183.86509704589844, "fcm_dpo/q_t": 0.38867393136024475, "grad_norm": 55.76332092285156, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -0.3209107220172882, "logits/rejected": -0.31004735827445984, "logps/chosen": -286.25897216796875, "logps/ref_chosen": -48.959083557128906, "logps/ref_rejected": -82.34072875976562, "logps/rejected": -503.5057067871094, "loss": 1.0419, "margin_dpo/margin_mean": 183.8651123046875, "margin_dpo/margin_std": 219.06765747070312, "step": 309 }, { "KL/chosen_KL_mean": -272.84228515625, "KL/mean": -357.896484375, "KL/rejected_KL_mean": -442.95068359375, "KL/std": 177.7974853515625, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.0026099509559571743, "fcm_dpo/delta": 0.005964267998933792, "fcm_dpo/margin": 170.10842895507812, "fcm_dpo/q_t": 0.39623942971229553, "grad_norm": 28.89272117614746, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -0.3595883846282959, "logits/rejected": -0.34507110714912415, "logps/chosen": -335.58404541015625, "logps/ref_chosen": -62.74177932739258, "logps/ref_rejected": -79.9302978515625, "logps/rejected": -522.8809814453125, "loss": 1.041, "margin_dpo/margin_mean": 170.10842895507812, "margin_dpo/margin_std": 173.22080993652344, "step": 310 }, { "KL/chosen_KL_mean": -283.4051513671875, "KL/mean": -380.39300537109375, "KL/rejected_KL_mean": -477.3808898925781, "KL/std": 230.86802673339844, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.002579387277364731, "fcm_dpo/delta": -0.053004972636699677, "fcm_dpo/margin": 193.97573852539062, "fcm_dpo/q_t": 0.3883788287639618, "grad_norm": 25.943056106567383, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -0.3639160990715027, "logits/rejected": -0.3407902717590332, "logps/chosen": -336.43310546875, "logps/ref_chosen": -53.02798080444336, "logps/ref_rejected": -77.43820190429688, "logps/rejected": -554.819091796875, "loss": 1.0342, "margin_dpo/margin_mean": 193.97573852539062, "margin_dpo/margin_std": 241.01882934570312, "step": 311 }, { "KL/chosen_KL_mean": -259.5381164550781, "KL/mean": -349.5057678222656, "KL/rejected_KL_mean": -439.4734191894531, "KL/std": 199.23733520507812, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.002578413113951683, "fcm_dpo/delta": -0.014939859509468079, "fcm_dpo/margin": 179.93531799316406, "fcm_dpo/q_t": 0.3965616822242737, "grad_norm": 24.46059799194336, "learning_rate": 3.306636061080487e-07, "logits/chosen": -0.2949761152267456, "logits/rejected": -0.28196972608566284, "logps/chosen": -308.9303283691406, "logps/ref_chosen": -49.39221954345703, "logps/ref_rejected": -75.79280853271484, "logps/rejected": -515.2662353515625, "loss": 1.0557, "margin_dpo/margin_mean": 179.935302734375, "margin_dpo/margin_std": 232.431640625, "step": 312 }, { "KL/chosen_KL_mean": -237.4639434814453, "KL/mean": -327.5711669921875, "KL/rejected_KL_mean": -417.6783752441406, "KL/std": 207.86187744140625, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0025785909965634346, "fcm_dpo/delta": -0.016460783779621124, "fcm_dpo/margin": 180.21444702148438, "fcm_dpo/q_t": 0.39441442489624023, "grad_norm": 25.501195907592773, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -0.3515121340751648, "logits/rejected": -0.346757173538208, "logps/chosen": -287.61669921875, "logps/ref_chosen": -50.152740478515625, "logps/ref_rejected": -86.40620422363281, "logps/rejected": -504.0845947265625, "loss": 1.0561, "margin_dpo/margin_mean": 180.2144317626953, "margin_dpo/margin_std": 224.6845703125, "step": 313 }, { "KL/chosen_KL_mean": -255.17724609375, "KL/mean": -338.23541259765625, "KL/rejected_KL_mean": -421.29351806640625, "KL/std": 197.79852294921875, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.002554558450356126, "fcm_dpo/delta": 0.026423994451761246, "fcm_dpo/margin": 166.11630249023438, "fcm_dpo/q_t": 0.40657833218574524, "grad_norm": 22.802820205688477, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -0.40561333298683167, "logits/rejected": -0.4046463966369629, "logps/chosen": -312.4148254394531, "logps/ref_chosen": -57.237579345703125, "logps/ref_rejected": -97.5965347290039, "logps/rejected": -518.8900756835938, "loss": 1.1018, "margin_dpo/margin_mean": 166.11630249023438, "margin_dpo/margin_std": 251.25624084472656, "step": 314 }, { "KL/chosen_KL_mean": -228.4068603515625, "KL/mean": -305.869873046875, "KL/rejected_KL_mean": -383.33282470703125, "KL/std": 177.62628173828125, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.0025859931483864784, "fcm_dpo/delta": 0.05145182088017464, "fcm_dpo/margin": 154.92596435546875, "fcm_dpo/q_t": 0.40844735503196716, "grad_norm": 25.262500762939453, "learning_rate": 3.270101039870797e-07, "logits/chosen": -0.3275286555290222, "logits/rejected": -0.32907772064208984, "logps/chosen": -277.4764404296875, "logps/ref_chosen": -49.06958770751953, "logps/ref_rejected": -85.68087768554688, "logps/rejected": -469.01373291015625, "loss": 1.0856, "margin_dpo/margin_mean": 154.92596435546875, "margin_dpo/margin_std": 197.055908203125, "step": 315 }, { "KL/chosen_KL_mean": -234.32891845703125, "KL/mean": -344.46405029296875, "KL/rejected_KL_mean": -454.59918212890625, "KL/std": 213.09478759765625, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.0025378912687301636, "fcm_dpo/delta": -0.116811104118824, "fcm_dpo/margin": 220.27024841308594, "fcm_dpo/q_t": 0.37471282482147217, "grad_norm": 28.47589111328125, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -0.3635621666908264, "logits/rejected": -0.3676885962486267, "logps/chosen": -288.58966064453125, "logps/ref_chosen": -54.26074981689453, "logps/ref_rejected": -101.2814712524414, "logps/rejected": -555.880615234375, "loss": 0.9835, "margin_dpo/margin_mean": 220.270263671875, "margin_dpo/margin_std": 234.39602661132812, "step": 316 }, { "KL/chosen_KL_mean": -246.70562744140625, "KL/mean": -367.3144836425781, "KL/rejected_KL_mean": -487.92333984375, "KL/std": 197.05282592773438, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.002488123718649149, "fcm_dpo/delta": -0.15946120023727417, "fcm_dpo/margin": 241.21768188476562, "fcm_dpo/q_t": 0.3624528646469116, "grad_norm": 30.520000457763672, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -0.3962244987487793, "logits/rejected": -0.3859291672706604, "logps/chosen": -302.79986572265625, "logps/ref_chosen": -56.094207763671875, "logps/ref_rejected": -100.69905090332031, "logps/rejected": -588.6224365234375, "loss": 0.9439, "margin_dpo/margin_mean": 241.21768188476562, "margin_dpo/margin_std": 217.16314697265625, "step": 317 }, { "KL/chosen_KL_mean": -272.97113037109375, "KL/mean": -367.22979736328125, "KL/rejected_KL_mean": -461.48846435546875, "KL/std": 211.57278442382812, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.002460531424731016, "fcm_dpo/delta": -0.01471693441271782, "fcm_dpo/margin": 188.51731872558594, "fcm_dpo/q_t": 0.3947198987007141, "grad_norm": 28.010128021240234, "learning_rate": 3.233383385962115e-07, "logits/chosen": -0.41236060857772827, "logits/rejected": -0.3805282711982727, "logps/chosen": -337.6168212890625, "logps/ref_chosen": -64.64569854736328, "logps/ref_rejected": -82.76425170898438, "logps/rejected": -544.252685546875, "loss": 1.0477, "margin_dpo/margin_mean": 188.51731872558594, "margin_dpo/margin_std": 228.11260986328125, "step": 318 }, { "KL/chosen_KL_mean": -256.7373046875, "KL/mean": -366.04266357421875, "KL/rejected_KL_mean": -475.34796142578125, "KL/std": 226.67886352539062, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.0024200198240578175, "fcm_dpo/delta": -0.08345872163772583, "fcm_dpo/margin": 218.61068725585938, "fcm_dpo/q_t": 0.3794768452644348, "grad_norm": 22.835662841796875, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -0.38075074553489685, "logits/rejected": -0.39493638277053833, "logps/chosen": -306.1210632324219, "logps/ref_chosen": -49.383758544921875, "logps/ref_rejected": -113.90650939941406, "logps/rejected": -589.2545166015625, "loss": 1.0039, "margin_dpo/margin_mean": 218.61068725585938, "margin_dpo/margin_std": 243.4561767578125, "step": 319 }, { "KL/chosen_KL_mean": -276.7099609375, "KL/mean": -395.4337158203125, "KL/rejected_KL_mean": -514.157470703125, "KL/std": 241.7675323486328, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0023585117887705564, "fcm_dpo/delta": -0.11735754460096359, "fcm_dpo/margin": 237.447509765625, "fcm_dpo/q_t": 0.3738830089569092, "grad_norm": 30.08669662475586, "learning_rate": 3.208807785813777e-07, "logits/chosen": -0.37936246395111084, "logits/rejected": -0.3851373493671417, "logps/chosen": -336.21484375, "logps/ref_chosen": -59.50489044189453, "logps/ref_rejected": -97.66717529296875, "logps/rejected": -611.8246459960938, "loss": 0.985, "margin_dpo/margin_mean": 237.447509765625, "margin_dpo/margin_std": 257.3951416015625, "step": 320 }, { "KL/chosen_KL_mean": -339.2561950683594, "KL/mean": -442.3502502441406, "KL/rejected_KL_mean": -545.4443359375, "KL/std": 248.72198486328125, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.0023283420596271753, "fcm_dpo/delta": -0.03253670781850815, "fcm_dpo/margin": 206.18814086914062, "fcm_dpo/q_t": 0.3924645781517029, "grad_norm": 26.111909866333008, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -0.3646908402442932, "logits/rejected": -0.3575018644332886, "logps/chosen": -400.80487060546875, "logps/ref_chosen": -61.548683166503906, "logps/ref_rejected": -91.64103698730469, "logps/rejected": -637.0853271484375, "loss": 1.05, "margin_dpo/margin_mean": 206.18814086914062, "margin_dpo/margin_std": 260.78741455078125, "step": 321 }, { "KL/chosen_KL_mean": -301.8092346191406, "KL/mean": -417.0679931640625, "KL/rejected_KL_mean": -532.32666015625, "KL/std": 228.69329833984375, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0023004289250820875, "fcm_dpo/delta": -0.08538933843374252, "fcm_dpo/margin": 230.51748657226562, "fcm_dpo/q_t": 0.37849855422973633, "grad_norm": 25.973785400390625, "learning_rate": 3.184157475180207e-07, "logits/chosen": -0.36909303069114685, "logits/rejected": -0.3671499490737915, "logps/chosen": -359.0992736816406, "logps/ref_chosen": -57.29003143310547, "logps/ref_rejected": -95.74992370605469, "logps/rejected": -628.07666015625, "loss": 0.9968, "margin_dpo/margin_mean": 230.51748657226562, "margin_dpo/margin_std": 240.91160583496094, "step": 322 }, { "KL/chosen_KL_mean": -326.64044189453125, "KL/mean": -419.115966796875, "KL/rejected_KL_mean": -511.5914306640625, "KL/std": 224.9309539794922, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.002301469910889864, "fcm_dpo/delta": 0.02534468285739422, "fcm_dpo/margin": 184.95101928710938, "fcm_dpo/q_t": 0.40258121490478516, "grad_norm": 52.07743453979492, "learning_rate": 3.171805115074251e-07, "logits/chosen": -0.3876006603240967, "logits/rejected": -0.3855016827583313, "logps/chosen": -377.8744201660156, "logps/ref_chosen": -51.23395919799805, "logps/ref_rejected": -75.06192016601562, "logps/rejected": -586.6533813476562, "loss": 1.0811, "margin_dpo/margin_mean": 184.95101928710938, "margin_dpo/margin_std": 240.82904052734375, "step": 323 }, { "KL/chosen_KL_mean": -370.4443054199219, "KL/mean": -462.32537841796875, "KL/rejected_KL_mean": -554.2064208984375, "KL/std": 239.63571166992188, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.00233688042499125, "fcm_dpo/delta": 0.019912488758563995, "fcm_dpo/margin": 183.76214599609375, "fcm_dpo/q_t": 0.40560734272003174, "grad_norm": 59.34591293334961, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -0.388489305973053, "logits/rejected": -0.37319934368133545, "logps/chosen": -435.5794677734375, "logps/ref_chosen": -65.13516998291016, "logps/ref_rejected": -86.47750854492188, "logps/rejected": -640.6839599609375, "loss": 1.1079, "margin_dpo/margin_mean": 183.7621307373047, "margin_dpo/margin_std": 277.62188720703125, "step": 324 }, { "KL/chosen_KL_mean": -291.468994140625, "KL/mean": -369.7003173828125, "KL/rejected_KL_mean": -447.9316711425781, "KL/std": 212.40463256835938, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.0023523285053670406, "fcm_dpo/delta": 0.08494192361831665, "fcm_dpo/margin": 156.462646484375, "fcm_dpo/q_t": 0.4155137538909912, "grad_norm": 34.735572814941406, "learning_rate": 3.147047612756302e-07, "logits/chosen": -0.45538169145584106, "logits/rejected": -0.43598616123199463, "logps/chosen": -347.6846008300781, "logps/ref_chosen": -56.215599060058594, "logps/ref_rejected": -70.08592987060547, "logps/rejected": -518.017578125, "loss": 1.1154, "margin_dpo/margin_mean": 156.462646484375, "margin_dpo/margin_std": 214.5522003173828, "step": 325 }, { "KL/chosen_KL_mean": -323.7759094238281, "KL/mean": -402.790283203125, "KL/rejected_KL_mean": -481.8047180175781, "KL/std": 199.88583374023438, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0023813103325664997, "fcm_dpo/delta": 0.0766344889998436, "fcm_dpo/margin": 158.02880859375, "fcm_dpo/q_t": 0.41171562671661377, "grad_norm": 37.10205841064453, "learning_rate": 3.134643122927519e-07, "logits/chosen": -0.44069403409957886, "logits/rejected": -0.41960227489471436, "logps/chosen": -396.5008544921875, "logps/ref_chosen": -72.72496032714844, "logps/ref_rejected": -79.8467788696289, "logps/rejected": -561.6514892578125, "loss": 1.0953, "margin_dpo/margin_mean": 158.02882385253906, "margin_dpo/margin_std": 187.57601928710938, "step": 326 }, { "KL/chosen_KL_mean": -274.60150146484375, "KL/mean": -390.11627197265625, "KL/rejected_KL_mean": -505.631103515625, "KL/std": 213.35443115234375, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.002351765986531973, "fcm_dpo/delta": -0.09938237071037292, "fcm_dpo/margin": 231.02955627441406, "fcm_dpo/q_t": 0.37418001890182495, "grad_norm": 40.187255859375, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -0.41924217343330383, "logits/rejected": -0.4057329595088959, "logps/chosen": -343.7359313964844, "logps/ref_chosen": -69.13441467285156, "logps/ref_rejected": -111.93377685546875, "logps/rejected": -617.5648193359375, "loss": 0.9821, "margin_dpo/margin_mean": 231.02955627441406, "margin_dpo/margin_std": 227.44580078125, "step": 327 }, { "KL/chosen_KL_mean": -281.99481201171875, "KL/mean": -383.1610412597656, "KL/rejected_KL_mean": -484.3272705078125, "KL/std": 223.35235595703125, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0023423531092703342, "fcm_dpo/delta": -0.025119595229625702, "fcm_dpo/margin": 202.3324432373047, "fcm_dpo/q_t": 0.3928408622741699, "grad_norm": 27.627058029174805, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -0.421650767326355, "logits/rejected": -0.4237726926803589, "logps/chosen": -341.6820068359375, "logps/ref_chosen": -59.68719482421875, "logps/ref_rejected": -90.85499572753906, "logps/rejected": -575.1822509765625, "loss": 1.0416, "margin_dpo/margin_mean": 202.3324432373047, "margin_dpo/margin_std": 245.05563354492188, "step": 328 }, { "KL/chosen_KL_mean": -304.212646484375, "KL/mean": -413.9533996582031, "KL/rejected_KL_mean": -523.694091796875, "KL/std": 236.1326904296875, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.002305768197402358, "fcm_dpo/delta": -0.05977274850010872, "fcm_dpo/margin": 219.48150634765625, "fcm_dpo/q_t": 0.3840027451515198, "grad_norm": 29.397104263305664, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -0.3993944525718689, "logits/rejected": -0.38911157846450806, "logps/chosen": -369.4588317871094, "logps/ref_chosen": -65.2461929321289, "logps/ref_rejected": -100.69770812988281, "logps/rejected": -624.391845703125, "loss": 1.0218, "margin_dpo/margin_mean": 219.48150634765625, "margin_dpo/margin_std": 246.65150451660156, "step": 329 }, { "KL/chosen_KL_mean": -262.68377685546875, "KL/mean": -371.68048095703125, "KL/rejected_KL_mean": -480.67718505859375, "KL/std": 223.04965209960938, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.0022905156947672367, "fcm_dpo/delta": -0.05193692445755005, "fcm_dpo/margin": 217.99343872070312, "fcm_dpo/q_t": 0.3855849504470825, "grad_norm": 28.491689682006836, "learning_rate": 3.084861204504122e-07, "logits/chosen": -0.3724977970123291, "logits/rejected": -0.37324658036231995, "logps/chosen": -309.6820983886719, "logps/ref_chosen": -46.998348236083984, "logps/ref_rejected": -86.87684631347656, "logps/rejected": -567.5540771484375, "loss": 1.0158, "margin_dpo/margin_mean": 217.99343872070312, "margin_dpo/margin_std": 237.47122192382812, "step": 330 }, { "KL/chosen_KL_mean": -280.93597412109375, "KL/mean": -389.1291809082031, "KL/rejected_KL_mean": -497.3223571777344, "KL/std": 182.53598022460938, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0022698137909173965, "fcm_dpo/delta": -0.04330473765730858, "fcm_dpo/margin": 216.3863525390625, "fcm_dpo/q_t": 0.38430529832839966, "grad_norm": 37.197208404541016, "learning_rate": 3.072376374875335e-07, "logits/chosen": -0.4366343021392822, "logits/rejected": -0.43291670083999634, "logps/chosen": -331.4602355957031, "logps/ref_chosen": -50.52424621582031, "logps/ref_rejected": -89.01544189453125, "logps/rejected": -586.3377685546875, "loss": 0.9921, "margin_dpo/margin_mean": 216.3863525390625, "margin_dpo/margin_std": 179.05184936523438, "step": 331 }, { "KL/chosen_KL_mean": -282.11297607421875, "KL/mean": -359.55120849609375, "KL/rejected_KL_mean": -436.9894714355469, "KL/std": 190.00502014160156, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.0023019383661448956, "fcm_dpo/delta": 0.09676185250282288, "fcm_dpo/margin": 154.87649536132812, "fcm_dpo/q_t": 0.41818666458129883, "grad_norm": 24.03437614440918, "learning_rate": 3.059876462596758e-07, "logits/chosen": -0.41719603538513184, "logits/rejected": -0.3998468518257141, "logps/chosen": -331.29327392578125, "logps/ref_chosen": -49.18028259277344, "logps/ref_rejected": -76.48515319824219, "logps/rejected": -513.474609375, "loss": 1.1126, "margin_dpo/margin_mean": 154.87652587890625, "margin_dpo/margin_std": 204.28192138671875, "step": 332 }, { "KL/chosen_KL_mean": -301.60906982421875, "KL/mean": -403.718505859375, "KL/rejected_KL_mean": -505.8279113769531, "KL/std": 225.57460021972656, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.002289321506395936, "fcm_dpo/delta": -0.0192737840116024, "fcm_dpo/margin": 204.2188262939453, "fcm_dpo/q_t": 0.39486679434776306, "grad_norm": 23.60999870300293, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -0.4344450831413269, "logits/rejected": -0.430309921503067, "logps/chosen": -365.36480712890625, "logps/ref_chosen": -63.75574493408203, "logps/ref_rejected": -95.04411315917969, "logps/rejected": -600.8720092773438, "loss": 1.0626, "margin_dpo/margin_mean": 204.21881103515625, "margin_dpo/margin_std": 267.87591552734375, "step": 333 }, { "KL/chosen_KL_mean": -290.12060546875, "KL/mean": -382.6822509765625, "KL/rejected_KL_mean": -475.24395751953125, "KL/std": 240.4530029296875, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.002307912800461054, "fcm_dpo/delta": 0.02377365157008171, "fcm_dpo/margin": 185.12339782714844, "fcm_dpo/q_t": 0.4034510850906372, "grad_norm": 25.177146911621094, "learning_rate": 3.034832708016243e-07, "logits/chosen": -0.4144849479198456, "logits/rejected": -0.4086567163467407, "logps/chosen": -357.100341796875, "logps/ref_chosen": -66.97975158691406, "logps/ref_rejected": -95.31692504882812, "logps/rejected": -570.5609130859375, "loss": 1.0911, "margin_dpo/margin_mean": 185.12338256835938, "margin_dpo/margin_std": 263.03802490234375, "step": 334 }, { "KL/chosen_KL_mean": -316.4975891113281, "KL/mean": -389.90167236328125, "KL/rejected_KL_mean": -463.3057556152344, "KL/std": 223.39114379882812, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.002362326718866825, "fcm_dpo/delta": 0.10562027990818024, "fcm_dpo/margin": 146.80816650390625, "fcm_dpo/q_t": 0.4218912720680237, "grad_norm": 29.343273162841797, "learning_rate": 3.022289525640531e-07, "logits/chosen": -0.47399646043777466, "logits/rejected": -0.45253986120224, "logps/chosen": -379.0400695800781, "logps/ref_chosen": -62.54248046875, "logps/ref_rejected": -87.61770629882812, "logps/rejected": -550.9234619140625, "loss": 1.1476, "margin_dpo/margin_mean": 146.80816650390625, "margin_dpo/margin_std": 240.48916625976562, "step": 335 }, { "KL/chosen_KL_mean": -313.12445068359375, "KL/mean": -429.3009948730469, "KL/rejected_KL_mean": -545.4775390625, "KL/std": 254.45855712890625, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.0023357835598289967, "fcm_dpo/delta": -0.0979611948132515, "fcm_dpo/margin": 232.35311889648438, "fcm_dpo/q_t": 0.3800773024559021, "grad_norm": 26.39600372314453, "learning_rate": 3.009732580450086e-07, "logits/chosen": -0.4094962477684021, "logits/rejected": -0.40867048501968384, "logps/chosen": -367.65557861328125, "logps/ref_chosen": -54.53115463256836, "logps/ref_rejected": -104.40424346923828, "logps/rejected": -649.8818359375, "loss": 1.0219, "margin_dpo/margin_mean": 232.35311889648438, "margin_dpo/margin_std": 291.4840393066406, "step": 336 }, { "KL/chosen_KL_mean": -287.0484619140625, "KL/mean": -399.60888671875, "KL/rejected_KL_mean": -512.1693115234375, "KL/std": 210.44012451171875, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.00228647468611598, "fcm_dpo/delta": -0.06854301691055298, "fcm_dpo/margin": 225.120849609375, "fcm_dpo/q_t": 0.3825306296348572, "grad_norm": 34.630104064941406, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -0.4810912609100342, "logits/rejected": -0.4703620672225952, "logps/chosen": -352.17718505859375, "logps/ref_chosen": -65.12869262695312, "logps/ref_rejected": -101.72701263427734, "logps/rejected": -613.8963623046875, "loss": 1.0091, "margin_dpo/margin_mean": 225.12086486816406, "margin_dpo/margin_std": 246.9684600830078, "step": 337 }, { "KL/chosen_KL_mean": -268.0345153808594, "KL/mean": -379.52264404296875, "KL/rejected_KL_mean": -491.0107421875, "KL/std": 213.11172485351562, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.0022620288655161858, "fcm_dpo/delta": -0.05736400932073593, "fcm_dpo/margin": 222.97622680664062, "fcm_dpo/q_t": 0.3833646774291992, "grad_norm": 30.86869239807129, "learning_rate": 2.984578725527675e-07, "logits/chosen": -0.488203763961792, "logits/rejected": -0.4801350235939026, "logps/chosen": -326.45721435546875, "logps/ref_chosen": -58.422706604003906, "logps/ref_rejected": -89.06854248046875, "logps/rejected": -580.0792846679688, "loss": 1.0026, "margin_dpo/margin_mean": 222.97622680664062, "margin_dpo/margin_std": 222.72213745117188, "step": 338 }, { "KL/chosen_KL_mean": -279.5220031738281, "KL/mean": -387.8486022949219, "KL/rejected_KL_mean": -496.1752014160156, "KL/std": 218.14901733398438, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.002262189518660307, "fcm_dpo/delta": -0.043202970176935196, "fcm_dpo/margin": 216.65322875976562, "fcm_dpo/q_t": 0.3870697021484375, "grad_norm": 34.2067756652832, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -0.5117071270942688, "logits/rejected": -0.4946392774581909, "logps/chosen": -339.517333984375, "logps/ref_chosen": -59.99531555175781, "logps/ref_rejected": -103.9109115600586, "logps/rejected": -600.0861206054688, "loss": 1.0183, "margin_dpo/margin_mean": 216.65322875976562, "margin_dpo/margin_std": 218.69485473632812, "step": 339 }, { "KL/chosen_KL_mean": -300.64361572265625, "KL/mean": -397.70538330078125, "KL/rejected_KL_mean": -494.76715087890625, "KL/std": 210.3126220703125, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0022354573011398315, "fcm_dpo/delta": 0.016527773812413216, "fcm_dpo/margin": 194.12351989746094, "fcm_dpo/q_t": 0.4011934995651245, "grad_norm": 35.80244827270508, "learning_rate": 2.959373794541426e-07, "logits/chosen": -0.39741045236587524, "logits/rejected": -0.37222498655319214, "logps/chosen": -353.473876953125, "logps/ref_chosen": -52.83022689819336, "logps/ref_rejected": -73.10723114013672, "logps/rejected": -567.8743896484375, "loss": 1.0726, "margin_dpo/margin_mean": 194.12351989746094, "margin_dpo/margin_std": 251.18772888183594, "step": 340 }, { "KL/chosen_KL_mean": -299.45147705078125, "KL/mean": -409.642822265625, "KL/rejected_KL_mean": -519.8341064453125, "KL/std": 234.6087646484375, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.002222803421318531, "fcm_dpo/delta": -0.04243752360343933, "fcm_dpo/margin": 220.3826446533203, "fcm_dpo/q_t": 0.3885568678379059, "grad_norm": 25.23012351989746, "learning_rate": 2.946753005532965e-07, "logits/chosen": -0.4419187605381012, "logits/rejected": -0.44217973947525024, "logps/chosen": -347.35125732421875, "logps/ref_chosen": -47.899803161621094, "logps/ref_rejected": -101.80987548828125, "logps/rejected": -621.6439819335938, "loss": 1.0256, "margin_dpo/margin_mean": 220.38262939453125, "margin_dpo/margin_std": 244.93939208984375, "step": 341 }, { "KL/chosen_KL_mean": -293.6312255859375, "KL/mean": -392.4619140625, "KL/rejected_KL_mean": -491.2926025390625, "KL/std": 217.22531127929688, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.002238193526864052, "fcm_dpo/delta": 0.007614880800247192, "fcm_dpo/margin": 197.66143798828125, "fcm_dpo/q_t": 0.39952483773231506, "grad_norm": 28.198055267333984, "learning_rate": 2.934120444167326e-07, "logits/chosen": -0.4391222596168518, "logits/rejected": -0.41000527143478394, "logps/chosen": -365.62786865234375, "logps/ref_chosen": -71.99664306640625, "logps/ref_rejected": -92.58959197998047, "logps/rejected": -583.8822021484375, "loss": 1.0648, "margin_dpo/margin_mean": 197.66143798828125, "margin_dpo/margin_std": 251.34584045410156, "step": 342 }, { "KL/chosen_KL_mean": -283.8646240234375, "KL/mean": -400.5185241699219, "KL/rejected_KL_mean": -517.1724243164062, "KL/std": 220.9855499267578, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.002206906210631132, "fcm_dpo/delta": -0.06844501197338104, "fcm_dpo/margin": 233.30776977539062, "fcm_dpo/q_t": 0.3789847493171692, "grad_norm": 29.322237014770508, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -0.46948331594467163, "logits/rejected": -0.4704735279083252, "logps/chosen": -338.270263671875, "logps/ref_chosen": -54.405616760253906, "logps/ref_rejected": -111.04142761230469, "logps/rejected": -628.2138671875, "loss": 0.9819, "margin_dpo/margin_mean": 233.3077850341797, "margin_dpo/margin_std": 195.7198944091797, "step": 343 }, { "KL/chosen_KL_mean": -286.81103515625, "KL/mean": -388.08660888671875, "KL/rejected_KL_mean": -489.36212158203125, "KL/std": 237.59225463867188, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.002221263013780117, "fcm_dpo/delta": -0.0018584132194519043, "fcm_dpo/margin": 202.5511474609375, "fcm_dpo/q_t": 0.3995116651058197, "grad_norm": 28.450302124023438, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -0.4501603841781616, "logits/rejected": -0.4517921507358551, "logps/chosen": -340.77569580078125, "logps/ref_chosen": -53.96466827392578, "logps/ref_rejected": -90.62336730957031, "logps/rejected": -579.9854736328125, "loss": 1.0596, "margin_dpo/margin_mean": 202.5511474609375, "margin_dpo/margin_std": 244.07476806640625, "step": 344 }, { "KL/chosen_KL_mean": -332.5111083984375, "KL/mean": -445.0814208984375, "KL/rejected_KL_mean": -557.6517333984375, "KL/std": 230.50149536132812, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.0021899822168052197, "fcm_dpo/delta": -0.04530047997832298, "fcm_dpo/margin": 225.14065551757812, "fcm_dpo/q_t": 0.3868619203567505, "grad_norm": 26.73749351501465, "learning_rate": 2.896155456223163e-07, "logits/chosen": -0.4524056017398834, "logits/rejected": -0.4489898085594177, "logps/chosen": -394.19677734375, "logps/ref_chosen": -61.685699462890625, "logps/ref_rejected": -99.49041748046875, "logps/rejected": -657.1421508789062, "loss": 1.0222, "margin_dpo/margin_mean": 225.1406707763672, "margin_dpo/margin_std": 250.29025268554688, "step": 345 }, { "KL/chosen_KL_mean": -332.1961669921875, "KL/mean": -438.26373291015625, "KL/rejected_KL_mean": -544.331298828125, "KL/std": 220.63192749023438, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.002174857072532177, "fcm_dpo/delta": -0.011919819749891758, "fcm_dpo/margin": 212.1351318359375, "fcm_dpo/q_t": 0.39296603202819824, "grad_norm": 26.011234283447266, "learning_rate": 2.883479137196714e-07, "logits/chosen": -0.4090457260608673, "logits/rejected": -0.39724189043045044, "logps/chosen": -387.45245361328125, "logps/ref_chosen": -55.256263732910156, "logps/ref_rejected": -77.41532135009766, "logps/rejected": -621.74658203125, "loss": 1.043, "margin_dpo/margin_mean": 212.13511657714844, "margin_dpo/margin_std": 244.46681213378906, "step": 346 }, { "KL/chosen_KL_mean": -324.60968017578125, "KL/mean": -432.421142578125, "KL/rejected_KL_mean": -540.2326049804688, "KL/std": 235.4697265625, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.00216767773963511, "fcm_dpo/delta": -0.018262427300214767, "fcm_dpo/margin": 215.62295532226562, "fcm_dpo/q_t": 0.39333486557006836, "grad_norm": 24.214601516723633, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -0.41619834303855896, "logits/rejected": -0.4093385934829712, "logps/chosen": -382.1759033203125, "logps/ref_chosen": -57.56623840332031, "logps/ref_rejected": -92.35509490966797, "logps/rejected": -632.5877075195312, "loss": 1.0451, "margin_dpo/margin_mean": 215.62295532226562, "margin_dpo/margin_std": 258.7292785644531, "step": 347 }, { "KL/chosen_KL_mean": -298.324951171875, "KL/mean": -387.9532470703125, "KL/rejected_KL_mean": -477.5815734863281, "KL/std": 203.1543426513672, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.002172997919842601, "fcm_dpo/delta": 0.06251075863838196, "fcm_dpo/margin": 179.25662231445312, "fcm_dpo/q_t": 0.4109954237937927, "grad_norm": 21.876827239990234, "learning_rate": 2.858096518347179e-07, "logits/chosen": -0.45160365104675293, "logits/rejected": -0.4534376859664917, "logps/chosen": -354.64263916015625, "logps/ref_chosen": -56.31770324707031, "logps/ref_rejected": -89.13836669921875, "logps/rejected": -566.719970703125, "loss": 1.097, "margin_dpo/margin_mean": 179.25662231445312, "margin_dpo/margin_std": 231.77615356445312, "step": 348 }, { "KL/chosen_KL_mean": -278.4696960449219, "KL/mean": -378.2044372558594, "KL/rejected_KL_mean": -477.939208984375, "KL/std": 224.55136108398438, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.0022066906094551086, "fcm_dpo/delta": 0.009832058101892471, "fcm_dpo/margin": 199.469482421875, "fcm_dpo/q_t": 0.4018779993057251, "grad_norm": 22.135345458984375, "learning_rate": 2.845390887379706e-07, "logits/chosen": -0.407615602016449, "logits/rejected": -0.40762487053871155, "logps/chosen": -336.4952087402344, "logps/ref_chosen": -58.025516510009766, "logps/ref_rejected": -97.50515747070312, "logps/rejected": -575.4443359375, "loss": 1.0817, "margin_dpo/margin_mean": 199.469482421875, "margin_dpo/margin_std": 278.9986877441406, "step": 349 }, { "KL/chosen_KL_mean": -294.96185302734375, "KL/mean": -393.88372802734375, "KL/rejected_KL_mean": -492.8055725097656, "KL/std": 212.79945373535156, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.002194004599004984, "fcm_dpo/delta": 0.016355086117982864, "fcm_dpo/margin": 197.8437042236328, "fcm_dpo/q_t": 0.39984023571014404, "grad_norm": 30.247133255004883, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -0.44759875535964966, "logits/rejected": -0.44960999488830566, "logps/chosen": -359.2923583984375, "logps/ref_chosen": -64.33049011230469, "logps/ref_rejected": -89.87164306640625, "logps/rejected": -582.67724609375, "loss": 1.0783, "margin_dpo/margin_mean": 197.84371948242188, "margin_dpo/margin_std": 263.1554870605469, "step": 350 }, { "KL/chosen_KL_mean": -276.9792175292969, "KL/mean": -397.68896484375, "KL/rejected_KL_mean": -518.3987426757812, "KL/std": 254.1345672607422, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0021747946739196777, "fcm_dpo/delta": -0.07944449782371521, "fcm_dpo/margin": 241.41952514648438, "fcm_dpo/q_t": 0.3832271695137024, "grad_norm": 25.02871322631836, "learning_rate": 2.819952656376487e-07, "logits/chosen": -0.45677679777145386, "logits/rejected": -0.4540591537952423, "logps/chosen": -337.6513366699219, "logps/ref_chosen": -60.6721305847168, "logps/ref_rejected": -101.5654296875, "logps/rejected": -619.9641723632812, "loss": 1.0186, "margin_dpo/margin_mean": 241.41952514648438, "margin_dpo/margin_std": 290.705810546875, "step": 351 }, { "KL/chosen_KL_mean": -333.67742919921875, "KL/mean": -409.6393127441406, "KL/rejected_KL_mean": -485.60113525390625, "KL/std": 232.55503845214844, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.0022003580816090107, "fcm_dpo/delta": 0.11985920369625092, "fcm_dpo/margin": 151.92373657226562, "fcm_dpo/q_t": 0.4246724247932434, "grad_norm": 32.05812454223633, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -0.47153419256210327, "logits/rejected": -0.44236433506011963, "logps/chosen": -404.62091064453125, "logps/ref_chosen": -70.9434585571289, "logps/ref_rejected": -76.6419677734375, "logps/rejected": -562.2431640625, "loss": 1.1565, "margin_dpo/margin_mean": 151.92373657226562, "margin_dpo/margin_std": 255.46420288085938, "step": 352 }, { "KL/chosen_KL_mean": -318.5333557128906, "KL/mean": -418.0197448730469, "KL/rejected_KL_mean": -517.506103515625, "KL/std": 240.97393798828125, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.0022136140614748, "fcm_dpo/delta": 0.009388644248247147, "fcm_dpo/margin": 198.9727783203125, "fcm_dpo/q_t": 0.40121889114379883, "grad_norm": 37.71311569213867, "learning_rate": 2.794480701395219e-07, "logits/chosen": -0.4609450697898865, "logits/rejected": -0.4458503723144531, "logps/chosen": -376.9287109375, "logps/ref_chosen": -58.39533996582031, "logps/ref_rejected": -80.33553314208984, "logps/rejected": -597.8416748046875, "loss": 1.0762, "margin_dpo/margin_mean": 198.9727783203125, "margin_dpo/margin_std": 265.66168212890625, "step": 353 }, { "KL/chosen_KL_mean": -269.13421630859375, "KL/mean": -382.9037170410156, "KL/rejected_KL_mean": -496.6732177734375, "KL/std": 222.33949279785156, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.0022065872326493263, "fcm_dpo/delta": -0.05492671579122543, "fcm_dpo/margin": 227.53900146484375, "fcm_dpo/q_t": 0.3830799460411072, "grad_norm": 24.187728881835938, "learning_rate": 2.781732916288303e-07, "logits/chosen": -0.47964227199554443, "logits/rejected": -0.46824321150779724, "logps/chosen": -328.93719482421875, "logps/ref_chosen": -59.80299377441406, "logps/ref_rejected": -88.75750732421875, "logps/rejected": -585.4307250976562, "loss": 0.9969, "margin_dpo/margin_mean": 227.53900146484375, "margin_dpo/margin_std": 212.8988037109375, "step": 354 }, { "KL/chosen_KL_mean": -279.54412841796875, "KL/mean": -391.5539855957031, "KL/rejected_KL_mean": -503.5638427734375, "KL/std": 231.0259552001953, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0021842471323907375, "fcm_dpo/delta": -0.04154280200600624, "fcm_dpo/margin": 224.0196990966797, "fcm_dpo/q_t": 0.3863520622253418, "grad_norm": 32.41206741333008, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -0.49721992015838623, "logits/rejected": -0.4830342233181, "logps/chosen": -333.6726379394531, "logps/ref_chosen": -54.12849807739258, "logps/ref_rejected": -82.40606689453125, "logps/rejected": -585.9699096679688, "loss": 1.0133, "margin_dpo/margin_mean": 224.0196990966797, "margin_dpo/margin_std": 226.89224243164062, "step": 355 }, { "KL/chosen_KL_mean": -339.30712890625, "KL/mean": -404.0550537109375, "KL/rejected_KL_mean": -468.80303955078125, "KL/std": 238.53335571289062, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.0021884175948798656, "fcm_dpo/delta": 0.017231887206435204, "fcm_dpo/margin": 129.49594116210938, "fcm_dpo/q_t": 0.4372093677520752, "grad_norm": 44.52901840209961, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -0.4759572446346283, "logits/rejected": -0.45413556694984436, "logps/chosen": -403.9809265136719, "logps/ref_chosen": -64.6738052368164, "logps/ref_rejected": -75.89926147460938, "logps/rejected": -544.7022705078125, "loss": 1.225, "margin_dpo/margin_mean": 129.49594116210938, "margin_dpo/margin_std": 297.17681884765625, "step": 356 }, { "KL/chosen_KL_mean": -308.063720703125, "KL/mean": -408.1578369140625, "KL/rejected_KL_mean": -508.251953125, "KL/std": 230.31048583984375, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0021870676428079605, "fcm_dpo/delta": 0.012423541396856308, "fcm_dpo/margin": 200.188232421875, "fcm_dpo/q_t": 0.3986167311668396, "grad_norm": 29.87175941467285, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -0.45963430404663086, "logits/rejected": -0.4457460045814514, "logps/chosen": -360.7895202636719, "logps/ref_chosen": -52.725799560546875, "logps/ref_rejected": -86.84115600585938, "logps/rejected": -595.0931396484375, "loss": 1.0577, "margin_dpo/margin_mean": 200.188232421875, "margin_dpo/margin_std": 232.631103515625, "step": 357 }, { "KL/chosen_KL_mean": -274.03955078125, "KL/mean": -369.8802795410156, "KL/rejected_KL_mean": -465.72100830078125, "KL/std": 225.74777221679688, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0022067471873015165, "fcm_dpo/delta": 0.028202099725604057, "fcm_dpo/margin": 191.68145751953125, "fcm_dpo/q_t": 0.40436553955078125, "grad_norm": 24.91404914855957, "learning_rate": 2.730670898658255e-07, "logits/chosen": -0.4887428879737854, "logits/rejected": -0.4698476195335388, "logps/chosen": -337.2449951171875, "logps/ref_chosen": -63.20543670654297, "logps/ref_rejected": -88.373291015625, "logps/rejected": -554.0943603515625, "loss": 1.0753, "margin_dpo/margin_mean": 191.68145751953125, "margin_dpo/margin_std": 246.89959716796875, "step": 358 }, { "KL/chosen_KL_mean": -309.8017578125, "KL/mean": -419.9251708984375, "KL/rejected_KL_mean": -530.0485229492188, "KL/std": 227.0589599609375, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0021956057753413916, "fcm_dpo/delta": -0.03536238521337509, "fcm_dpo/margin": 220.24673461914062, "fcm_dpo/q_t": 0.3920891284942627, "grad_norm": 33.248435974121094, "learning_rate": 2.717889356869146e-07, "logits/chosen": -0.42775458097457886, "logits/rejected": -0.4127110540866852, "logps/chosen": -366.1719970703125, "logps/ref_chosen": -56.370216369628906, "logps/ref_rejected": -82.17375183105469, "logps/rejected": -612.2222900390625, "loss": 1.0457, "margin_dpo/margin_mean": 220.24673461914062, "margin_dpo/margin_std": 276.7038269042969, "step": 359 }, { "KL/chosen_KL_mean": -297.59710693359375, "KL/mean": -383.20404052734375, "KL/rejected_KL_mean": -468.81103515625, "KL/std": 191.41368103027344, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.002220964292064309, "fcm_dpo/delta": 0.07238072901964188, "fcm_dpo/margin": 171.2139129638672, "fcm_dpo/q_t": 0.4112092852592468, "grad_norm": 36.38239288330078, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -0.42573845386505127, "logits/rejected": -0.41087961196899414, "logps/chosen": -349.0574951171875, "logps/ref_chosen": -51.460384368896484, "logps/ref_rejected": -69.83892059326172, "logps/rejected": -538.6499633789062, "loss": 1.0867, "margin_dpo/margin_mean": 171.21392822265625, "margin_dpo/margin_std": 194.836669921875, "step": 360 }, { "KL/chosen_KL_mean": -317.767578125, "KL/mean": -402.918701171875, "KL/rejected_KL_mean": -488.06982421875, "KL/std": 225.28887939453125, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.002254966413602233, "fcm_dpo/delta": 0.06830620765686035, "fcm_dpo/margin": 170.30221557617188, "fcm_dpo/q_t": 0.4124668836593628, "grad_norm": 31.59840965270996, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -0.4371631145477295, "logits/rejected": -0.4323720335960388, "logps/chosen": -371.6371154785156, "logps/ref_chosen": -53.86951446533203, "logps/ref_rejected": -90.7692642211914, "logps/rejected": -578.839111328125, "loss": 1.1114, "margin_dpo/margin_mean": 170.30221557617188, "margin_dpo/margin_std": 246.90036010742188, "step": 361 }, { "KL/chosen_KL_mean": -281.20916748046875, "KL/mean": -412.2677001953125, "KL/rejected_KL_mean": -543.3262939453125, "KL/std": 234.77249145507812, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0022046894300729036, "fcm_dpo/delta": -0.13639265298843384, "fcm_dpo/margin": 262.11712646484375, "fcm_dpo/q_t": 0.3685632348060608, "grad_norm": 30.286354064941406, "learning_rate": 2.679511629948319e-07, "logits/chosen": -0.4322547912597656, "logits/rejected": -0.4399781823158264, "logps/chosen": -339.84820556640625, "logps/ref_chosen": -58.639060974121094, "logps/ref_rejected": -105.58195495605469, "logps/rejected": -648.908203125, "loss": 0.9576, "margin_dpo/margin_mean": 262.11712646484375, "margin_dpo/margin_std": 252.21163940429688, "step": 362 }, { "KL/chosen_KL_mean": -252.32077026367188, "KL/mean": -389.19268798828125, "KL/rejected_KL_mean": -526.0645751953125, "KL/std": 239.5186767578125, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.002155223861336708, "fcm_dpo/delta": -0.1484936624765396, "fcm_dpo/margin": 273.74383544921875, "fcm_dpo/q_t": 0.3658607602119446, "grad_norm": 25.104822158813477, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -0.40804392099380493, "logits/rejected": -0.3971943259239197, "logps/chosen": -296.879150390625, "logps/ref_chosen": -44.558380126953125, "logps/ref_rejected": -74.69496154785156, "logps/rejected": -600.759521484375, "loss": 0.9522, "margin_dpo/margin_mean": 273.74383544921875, "margin_dpo/margin_std": 261.37506103515625, "step": 363 }, { "KL/chosen_KL_mean": -273.26348876953125, "KL/mean": -377.99298095703125, "KL/rejected_KL_mean": -482.72247314453125, "KL/std": 227.8748779296875, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0021402244456112385, "fcm_dpo/delta": 0.001525435596704483, "fcm_dpo/margin": 209.45901489257812, "fcm_dpo/q_t": 0.39739012718200684, "grad_norm": 27.63004493713379, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -0.4067942500114441, "logits/rejected": -0.4162289500236511, "logps/chosen": -322.1580810546875, "logps/ref_chosen": -48.894622802734375, "logps/ref_rejected": -91.395751953125, "logps/rejected": -574.1182250976562, "loss": 1.0601, "margin_dpo/margin_mean": 209.45901489257812, "margin_dpo/margin_std": 260.34423828125, "step": 364 }, { "KL/chosen_KL_mean": -270.8097229003906, "KL/mean": -377.0350646972656, "KL/rejected_KL_mean": -483.26043701171875, "KL/std": 243.41281127929688, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0021314772311598063, "fcm_dpo/delta": -0.002969269407913089, "fcm_dpo/margin": 212.45069885253906, "fcm_dpo/q_t": 0.39748814702033997, "grad_norm": 25.45064926147461, "learning_rate": 2.641091331089811e-07, "logits/chosen": -0.409412145614624, "logits/rejected": -0.41792500019073486, "logps/chosen": -322.3024597167969, "logps/ref_chosen": -51.49274444580078, "logps/ref_rejected": -92.70166778564453, "logps/rejected": -575.9620971679688, "loss": 1.0483, "margin_dpo/margin_mean": 212.45069885253906, "margin_dpo/margin_std": 249.87734985351562, "step": 365 }, { "KL/chosen_KL_mean": -247.15567016601562, "KL/mean": -350.6108093261719, "KL/rejected_KL_mean": -454.06597900390625, "KL/std": 221.3289794921875, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.002125158440321684, "fcm_dpo/delta": 0.010381799191236496, "fcm_dpo/margin": 206.91030883789062, "fcm_dpo/q_t": 0.39979618787765503, "grad_norm": 21.667766571044922, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -0.3939347565174103, "logits/rejected": -0.4089328944683075, "logps/chosen": -291.876220703125, "logps/ref_chosen": -44.7205696105957, "logps/ref_rejected": -83.31040954589844, "logps/rejected": -537.3764038085938, "loss": 1.0681, "margin_dpo/margin_mean": 206.91029357910156, "margin_dpo/margin_std": 262.265380859375, "step": 366 }, { "KL/chosen_KL_mean": -249.67767333984375, "KL/mean": -338.1475830078125, "KL/rejected_KL_mean": -426.61749267578125, "KL/std": 202.29286193847656, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0021613952703773975, "fcm_dpo/delta": 0.07011134922504425, "fcm_dpo/margin": 176.93978881835938, "fcm_dpo/q_t": 0.41084012389183044, "grad_norm": 21.764284133911133, "learning_rate": 2.615458646614349e-07, "logits/chosen": -0.42215287685394287, "logits/rejected": -0.40269413590431213, "logps/chosen": -308.0830993652344, "logps/ref_chosen": -58.405418395996094, "logps/ref_rejected": -76.75132751464844, "logps/rejected": -503.3688049316406, "loss": 1.0935, "margin_dpo/margin_mean": 176.93978881835938, "margin_dpo/margin_std": 219.58963012695312, "step": 367 }, { "KL/chosen_KL_mean": -239.2371826171875, "KL/mean": -372.4246520996094, "KL/rejected_KL_mean": -505.61212158203125, "KL/std": 228.0447998046875, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0021273712627589703, "fcm_dpo/delta": -0.12365108728408813, "fcm_dpo/margin": 266.37493896484375, "fcm_dpo/q_t": 0.3667055070400238, "grad_norm": 39.34254455566406, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -0.43125176429748535, "logits/rejected": -0.43804439902305603, "logps/chosen": -283.689697265625, "logps/ref_chosen": -44.452518463134766, "logps/ref_rejected": -98.55526733398438, "logps/rejected": -604.1673583984375, "loss": 0.9439, "margin_dpo/margin_mean": 266.37493896484375, "margin_dpo/margin_std": 209.77322387695312, "step": 368 }, { "KL/chosen_KL_mean": -302.921875, "KL/mean": -376.767578125, "KL/rejected_KL_mean": -450.61334228515625, "KL/std": 229.33493041992188, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0021634360309690237, "fcm_dpo/delta": 0.13406780362129211, "fcm_dpo/margin": 147.69146728515625, "fcm_dpo/q_t": 0.42671999335289, "grad_norm": 37.03165817260742, "learning_rate": 2.589813792448196e-07, "logits/chosen": -0.41576188802719116, "logits/rejected": -0.3948121666908264, "logps/chosen": -374.3033447265625, "logps/ref_chosen": -71.38150024414062, "logps/ref_rejected": -91.29582214355469, "logps/rejected": -541.9091796875, "loss": 1.1684, "margin_dpo/margin_mean": 147.69146728515625, "margin_dpo/margin_std": 262.02093505859375, "step": 369 }, { "KL/chosen_KL_mean": -312.0323486328125, "KL/mean": -386.12933349609375, "KL/rejected_KL_mean": -460.2262878417969, "KL/std": 229.80404663085938, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.002216800581663847, "fcm_dpo/delta": 0.12519779801368713, "fcm_dpo/margin": 148.19395446777344, "fcm_dpo/q_t": 0.4249410033226013, "grad_norm": 27.90708351135254, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -0.45437344908714294, "logits/rejected": -0.44695669412612915, "logps/chosen": -383.63983154296875, "logps/ref_chosen": -71.60749816894531, "logps/ref_rejected": -97.25978088378906, "logps/rejected": -557.486083984375, "loss": 1.1573, "margin_dpo/margin_mean": 148.19395446777344, "margin_dpo/margin_std": 248.8985595703125, "step": 370 }, { "KL/chosen_KL_mean": -301.58782958984375, "KL/mean": -401.96405029296875, "KL/rejected_KL_mean": -502.3403015136719, "KL/std": 236.34710693359375, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.002231742488220334, "fcm_dpo/delta": 0.0019375644624233246, "fcm_dpo/margin": 200.75245666503906, "fcm_dpo/q_t": 0.3994460999965668, "grad_norm": 31.773405075073242, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -0.45085662603378296, "logits/rejected": -0.4364727735519409, "logps/chosen": -371.0023193359375, "logps/ref_chosen": -69.41448974609375, "logps/ref_rejected": -99.17217254638672, "logps/rejected": -601.512451171875, "loss": 1.074, "margin_dpo/margin_mean": 200.75245666503906, "margin_dpo/margin_std": 272.80889892578125, "step": 371 }, { "KL/chosen_KL_mean": -282.4768371582031, "KL/mean": -405.1539306640625, "KL/rejected_KL_mean": -527.8310546875, "KL/std": 271.397216796875, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.0021908096969127655, "fcm_dpo/delta": -0.0928315594792366, "fcm_dpo/margin": 245.35423278808594, "fcm_dpo/q_t": 0.38182735443115234, "grad_norm": 26.00803565979004, "learning_rate": 2.551329606220976e-07, "logits/chosen": -0.4409677982330322, "logits/rejected": -0.41506385803222656, "logps/chosen": -344.2948303222656, "logps/ref_chosen": -61.8179931640625, "logps/ref_rejected": -78.53948974609375, "logps/rejected": -606.3705444335938, "loss": 1.0188, "margin_dpo/margin_mean": 245.35423278808594, "margin_dpo/margin_std": 307.55609130859375, "step": 372 }, { "KL/chosen_KL_mean": -310.77227783203125, "KL/mean": -425.3634033203125, "KL/rejected_KL_mean": -539.9545288085938, "KL/std": 261.83868408203125, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.0021818915847688913, "fcm_dpo/delta": -0.05314254388213158, "fcm_dpo/margin": 229.18228149414062, "fcm_dpo/q_t": 0.3849130868911743, "grad_norm": 34.27317428588867, "learning_rate": 2.538498388222517e-07, "logits/chosen": -0.4386979639530182, "logits/rejected": -0.41286277770996094, "logps/chosen": -374.9893798828125, "logps/ref_chosen": -64.21713256835938, "logps/ref_rejected": -85.95960998535156, "logps/rejected": -625.9141845703125, "loss": 1.02, "margin_dpo/margin_mean": 229.18228149414062, "margin_dpo/margin_std": 245.1495361328125, "step": 373 }, { "KL/chosen_KL_mean": -279.8177490234375, "KL/mean": -383.7864074707031, "KL/rejected_KL_mean": -487.7550354003906, "KL/std": 281.73785400390625, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.0021521011367440224, "fcm_dpo/delta": 0.002089455723762512, "fcm_dpo/margin": 207.93724060058594, "fcm_dpo/q_t": 0.40315020084381104, "grad_norm": 25.19005012512207, "learning_rate": 2.525666155755725e-07, "logits/chosen": -0.5260878801345825, "logits/rejected": -0.5048750638961792, "logps/chosen": -350.46795654296875, "logps/ref_chosen": -70.65018463134766, "logps/ref_rejected": -93.64016723632812, "logps/rejected": -581.3952026367188, "loss": 1.0926, "margin_dpo/margin_mean": 207.937255859375, "margin_dpo/margin_std": 318.99566650390625, "step": 374 }, { "KL/chosen_KL_mean": -299.0926208496094, "KL/mean": -396.4559326171875, "KL/rejected_KL_mean": -493.8193054199219, "KL/std": 231.56265258789062, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0021589891985058784, "fcm_dpo/delta": 0.03020719811320305, "fcm_dpo/margin": 194.72665405273438, "fcm_dpo/q_t": 0.40428146719932556, "grad_norm": 35.277286529541016, "learning_rate": 2.512833246961859e-07, "logits/chosen": -0.46823275089263916, "logits/rejected": -0.46644824743270874, "logps/chosen": -359.1728515625, "logps/ref_chosen": -60.080223083496094, "logps/ref_rejected": -88.93830871582031, "logps/rejected": -582.757568359375, "loss": 1.1003, "margin_dpo/margin_mean": 194.72665405273438, "margin_dpo/margin_std": 283.26324462890625, "step": 375 }, { "KL/chosen_KL_mean": -286.27801513671875, "KL/mean": -411.0360107421875, "KL/rejected_KL_mean": -535.7940063476562, "KL/std": 255.5110626220703, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0021486622281372547, "fcm_dpo/delta": -0.09095398336648941, "fcm_dpo/margin": 249.51597595214844, "fcm_dpo/q_t": 0.37961071729660034, "grad_norm": 34.17658996582031, "learning_rate": 2.5e-07, "logits/chosen": -0.45805996656417847, "logits/rejected": -0.44645756483078003, "logps/chosen": -348.9383239746094, "logps/ref_chosen": -62.660308837890625, "logps/ref_rejected": -105.52660369873047, "logps/rejected": -641.320556640625, "loss": 1.0161, "margin_dpo/margin_mean": 249.51597595214844, "margin_dpo/margin_std": 301.43994140625, "step": 376 }, { "KL/chosen_KL_mean": -297.97998046875, "KL/mean": -417.2173156738281, "KL/rejected_KL_mean": -536.4546508789062, "KL/std": 261.6809997558594, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.0021219179034233093, "fcm_dpo/delta": -0.058967623859643936, "fcm_dpo/margin": 238.47463989257812, "fcm_dpo/q_t": 0.3851660192012787, "grad_norm": 27.344511032104492, "learning_rate": 2.487166753038141e-07, "logits/chosen": -0.41780710220336914, "logits/rejected": -0.4187828600406647, "logps/chosen": -352.458740234375, "logps/ref_chosen": -54.478736877441406, "logps/ref_rejected": -98.70335388183594, "logps/rejected": -635.157958984375, "loss": 1.0225, "margin_dpo/margin_mean": 238.47463989257812, "margin_dpo/margin_std": 277.2920837402344, "step": 377 }, { "KL/chosen_KL_mean": -281.651611328125, "KL/mean": -413.4957580566406, "KL/rejected_KL_mean": -545.3399047851562, "KL/std": 253.0030975341797, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.0020764602813869715, "fcm_dpo/delta": -0.10342134535312653, "fcm_dpo/margin": 263.68829345703125, "fcm_dpo/q_t": 0.37369853258132935, "grad_norm": 26.619735717773438, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -0.4513610005378723, "logits/rejected": -0.46656692028045654, "logps/chosen": -326.6721496582031, "logps/ref_chosen": -45.02053451538086, "logps/ref_rejected": -88.0469741821289, "logps/rejected": -633.3868408203125, "loss": 0.985, "margin_dpo/margin_mean": 263.68829345703125, "margin_dpo/margin_std": 271.5170593261719, "step": 378 }, { "KL/chosen_KL_mean": -328.8233947753906, "KL/mean": -453.4273376464844, "KL/rejected_KL_mean": -578.03125, "KL/std": 253.23651123046875, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.0020405231043696404, "fcm_dpo/delta": -0.06235864385962486, "fcm_dpo/margin": 249.2078857421875, "fcm_dpo/q_t": 0.38584357500076294, "grad_norm": 39.30223846435547, "learning_rate": 2.461501611777483e-07, "logits/chosen": -0.4471816420555115, "logits/rejected": -0.47077521681785583, "logps/chosen": -382.0054931640625, "logps/ref_chosen": -53.182098388671875, "logps/ref_rejected": -114.3001708984375, "logps/rejected": -692.3314208984375, "loss": 1.0263, "margin_dpo/margin_mean": 249.2078857421875, "margin_dpo/margin_std": 299.44866943359375, "step": 379 }, { "KL/chosen_KL_mean": -328.95916748046875, "KL/mean": -467.37396240234375, "KL/rejected_KL_mean": -605.788818359375, "KL/std": 285.58038330078125, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.002012487966567278, "fcm_dpo/delta": -0.11330637335777283, "fcm_dpo/margin": 276.8296203613281, "fcm_dpo/q_t": 0.37213844060897827, "grad_norm": 28.289297103881836, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -0.43490737676620483, "logits/rejected": -0.46167534589767456, "logps/chosen": -380.31219482421875, "logps/ref_chosen": -51.3530387878418, "logps/ref_rejected": -104.19169616699219, "logps/rejected": -709.98046875, "loss": 0.9934, "margin_dpo/margin_mean": 276.8296203613281, "margin_dpo/margin_std": 309.166015625, "step": 380 }, { "KL/chosen_KL_mean": -342.4608459472656, "KL/mean": -439.27081298828125, "KL/rejected_KL_mean": -536.080810546875, "KL/std": 238.89328002929688, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.0020117685198783875, "fcm_dpo/delta": 0.06297034025192261, "fcm_dpo/margin": 193.61993408203125, "fcm_dpo/q_t": 0.4127262830734253, "grad_norm": 32.157344818115234, "learning_rate": 2.435840528363426e-07, "logits/chosen": -0.47354856133461, "logits/rejected": -0.45360469818115234, "logps/chosen": -400.263916015625, "logps/ref_chosen": -57.80306625366211, "logps/ref_rejected": -79.21940612792969, "logps/rejected": -615.3001708984375, "loss": 1.1312, "margin_dpo/margin_mean": 193.6199188232422, "margin_dpo/margin_std": 321.12579345703125, "step": 381 }, { "KL/chosen_KL_mean": -339.0706481933594, "KL/mean": -457.1148376464844, "KL/rejected_KL_mean": -575.1590576171875, "KL/std": 229.2410430908203, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0020139641128480434, "fcm_dpo/delta": -0.026759367436170578, "fcm_dpo/margin": 236.08834838867188, "fcm_dpo/q_t": 0.3900807499885559, "grad_norm": 31.58717918395996, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -0.48019176721572876, "logits/rejected": -0.4856945872306824, "logps/chosen": -405.0909423828125, "logps/ref_chosen": -66.02030181884766, "logps/ref_rejected": -110.71016693115234, "logps/rejected": -685.8692016601562, "loss": 1.0245, "margin_dpo/margin_mean": 236.08834838867188, "margin_dpo/margin_std": 247.97897338867188, "step": 382 }, { "KL/chosen_KL_mean": -347.0646667480469, "KL/mean": -464.14361572265625, "KL/rejected_KL_mean": -581.2225341796875, "KL/std": 260.5420227050781, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0020031521562486887, "fcm_dpo/delta": -0.020062297582626343, "fcm_dpo/margin": 234.15789794921875, "fcm_dpo/q_t": 0.3943726420402527, "grad_norm": 30.257585525512695, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -0.4433164596557617, "logits/rejected": -0.45195838809013367, "logps/chosen": -397.4561462402344, "logps/ref_chosen": -50.39148712158203, "logps/ref_rejected": -93.71589660644531, "logps/rejected": -674.9384765625, "loss": 1.0747, "margin_dpo/margin_mean": 234.1578826904297, "margin_dpo/margin_std": 336.49066162109375, "step": 383 }, { "KL/chosen_KL_mean": -350.1620178222656, "KL/mean": -443.7109375, "KL/rejected_KL_mean": -537.2598876953125, "KL/std": 237.53985595703125, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.002035179641097784, "fcm_dpo/delta": 0.07113596796989441, "fcm_dpo/margin": 187.097900390625, "fcm_dpo/q_t": 0.4109281003475189, "grad_norm": 36.85161209106445, "learning_rate": 2.397362428170992e-07, "logits/chosen": -0.5195727944374084, "logits/rejected": -0.5128994584083557, "logps/chosen": -402.2081298828125, "logps/ref_chosen": -52.046104431152344, "logps/ref_rejected": -85.76089477539062, "logps/rejected": -623.020751953125, "loss": 1.091, "margin_dpo/margin_mean": 187.097900390625, "margin_dpo/margin_std": 220.66925048828125, "step": 384 }, { "KL/chosen_KL_mean": -319.3924865722656, "KL/mean": -434.7906494140625, "KL/rejected_KL_mean": -550.188720703125, "KL/std": 206.83387756347656, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.0020276131108403206, "fcm_dpo/delta": -0.01884370669722557, "fcm_dpo/margin": 230.79629516601562, "fcm_dpo/q_t": 0.39033573865890503, "grad_norm": 41.22309112548828, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -0.49404364824295044, "logits/rejected": -0.46726518869400024, "logps/chosen": -384.94464111328125, "logps/ref_chosen": -65.55215454101562, "logps/ref_rejected": -77.82792663574219, "logps/rejected": -628.0166625976562, "loss": 1.0214, "margin_dpo/margin_mean": 230.79629516601562, "margin_dpo/margin_std": 225.4544677734375, "step": 385 }, { "KL/chosen_KL_mean": -332.60528564453125, "KL/mean": -456.33349609375, "KL/rejected_KL_mean": -580.061767578125, "KL/std": 254.662353515625, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.002012161072343588, "fcm_dpo/delta": -0.05040276050567627, "fcm_dpo/margin": 247.45645141601562, "fcm_dpo/q_t": 0.38694748282432556, "grad_norm": 24.407316207885742, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -0.4635084867477417, "logits/rejected": -0.45638737082481384, "logps/chosen": -390.8271179199219, "logps/ref_chosen": -58.22185516357422, "logps/ref_rejected": -92.32742309570312, "logps/rejected": -672.38916015625, "loss": 1.0257, "margin_dpo/margin_mean": 247.45645141601562, "margin_dpo/margin_std": 289.11572265625, "step": 386 }, { "KL/chosen_KL_mean": -351.8877868652344, "KL/mean": -453.7193603515625, "KL/rejected_KL_mean": -555.5509033203125, "KL/std": 235.2284698486328, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.002009193878620863, "fcm_dpo/delta": 0.0423722118139267, "fcm_dpo/margin": 203.66311645507812, "fcm_dpo/q_t": 0.4049086570739746, "grad_norm": 39.93413162231445, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -0.505128026008606, "logits/rejected": -0.48430800437927246, "logps/chosen": -418.3072509765625, "logps/ref_chosen": -66.41944885253906, "logps/ref_rejected": -92.16915893554688, "logps/rejected": -647.7200927734375, "loss": 1.0739, "margin_dpo/margin_mean": 203.66311645507812, "margin_dpo/margin_std": 238.2667694091797, "step": 387 }, { "KL/chosen_KL_mean": -334.48602294921875, "KL/mean": -472.7179260253906, "KL/rejected_KL_mean": -610.9498291015625, "KL/std": 275.4636535644531, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.0019852365367114544, "fcm_dpo/delta": -0.10500699281692505, "fcm_dpo/margin": 276.4637756347656, "fcm_dpo/q_t": 0.37743353843688965, "grad_norm": 28.62255859375, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -0.4363114833831787, "logits/rejected": -0.44703811407089233, "logps/chosen": -384.615478515625, "logps/ref_chosen": -50.129459381103516, "logps/ref_rejected": -104.43305969238281, "logps/rejected": -715.3829345703125, "loss": 0.9961, "margin_dpo/margin_mean": 276.4637756347656, "margin_dpo/margin_std": 314.50836181640625, "step": 388 }, { "KL/chosen_KL_mean": -364.5044860839844, "KL/mean": -476.4181213378906, "KL/rejected_KL_mean": -588.331787109375, "KL/std": 266.85693359375, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.0019789470825344324, "fcm_dpo/delta": 0.007349299266934395, "fcm_dpo/margin": 223.8272705078125, "fcm_dpo/q_t": 0.39921024441719055, "grad_norm": 27.75473976135254, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -0.4892725348472595, "logits/rejected": -0.4785168170928955, "logps/chosen": -422.41107177734375, "logps/ref_chosen": -57.906593322753906, "logps/ref_rejected": -77.91454315185547, "logps/rejected": -666.246337890625, "loss": 1.0679, "margin_dpo/margin_mean": 223.8272705078125, "margin_dpo/margin_std": 290.217529296875, "step": 389 }, { "KL/chosen_KL_mean": -362.6640625, "KL/mean": -475.35125732421875, "KL/rejected_KL_mean": -588.0384521484375, "KL/std": 259.41107177734375, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.001976389205083251, "fcm_dpo/delta": 0.0044990889728069305, "fcm_dpo/margin": 225.3743896484375, "fcm_dpo/q_t": 0.4021126627922058, "grad_norm": 26.84067153930664, "learning_rate": 2.320488370051681e-07, "logits/chosen": -0.43252575397491455, "logits/rejected": -0.42253702878952026, "logps/chosen": -411.88995361328125, "logps/ref_chosen": -49.22591781616211, "logps/ref_rejected": -85.5281982421875, "logps/rejected": -673.566650390625, "loss": 1.0843, "margin_dpo/margin_mean": 225.3743896484375, "margin_dpo/margin_std": 330.4794921875, "step": 390 }, { "KL/chosen_KL_mean": -352.0406494140625, "KL/mean": -421.3489685058594, "KL/rejected_KL_mean": -490.6573486328125, "KL/std": 240.73985290527344, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.00203782063908875, "fcm_dpo/delta": 0.17224514484405518, "fcm_dpo/margin": 138.6166534423828, "fcm_dpo/q_t": 0.4359980821609497, "grad_norm": 41.918697357177734, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -0.45235902070999146, "logits/rejected": -0.441203236579895, "logps/chosen": -416.37030029296875, "logps/ref_chosen": -64.32965087890625, "logps/ref_rejected": -86.73820495605469, "logps/rejected": -577.3955078125, "loss": 1.1896, "margin_dpo/margin_mean": 138.61666870117188, "margin_dpo/margin_std": 255.517333984375, "step": 391 }, { "KL/chosen_KL_mean": -301.568359375, "KL/mean": -434.53594970703125, "KL/rejected_KL_mean": -567.5035400390625, "KL/std": 257.3258972167969, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0020327502861618996, "fcm_dpo/delta": -0.09568814933300018, "fcm_dpo/margin": 265.9351806640625, "fcm_dpo/q_t": 0.37543368339538574, "grad_norm": 28.45415496826172, "learning_rate": 2.294897926507156e-07, "logits/chosen": -0.44816461205482483, "logits/rejected": -0.4401182532310486, "logps/chosen": -355.07232666015625, "logps/ref_chosen": -53.50397872924805, "logps/ref_rejected": -102.34584045410156, "logps/rejected": -669.849365234375, "loss": 0.9758, "margin_dpo/margin_mean": 265.9352111816406, "margin_dpo/margin_std": 246.14971923828125, "step": 392 }, { "KL/chosen_KL_mean": -301.06414794921875, "KL/mean": -402.669921875, "KL/rejected_KL_mean": -504.27569580078125, "KL/std": 253.20211791992188, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0020245746709406376, "fcm_dpo/delta": 0.04025638476014137, "fcm_dpo/margin": 203.21151733398438, "fcm_dpo/q_t": 0.410128653049469, "grad_norm": 25.076148986816406, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -0.4646386504173279, "logits/rejected": -0.4622265696525574, "logps/chosen": -347.5380859375, "logps/ref_chosen": -46.473915100097656, "logps/ref_rejected": -71.96885681152344, "logps/rejected": -576.2445678710938, "loss": 1.1072, "margin_dpo/margin_mean": 203.21151733398438, "margin_dpo/margin_std": 318.36810302734375, "step": 393 }, { "KL/chosen_KL_mean": -334.5086364746094, "KL/mean": -448.73779296875, "KL/rejected_KL_mean": -562.9669799804688, "KL/std": 270.51513671875, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.0020245497580617666, "fcm_dpo/delta": -0.013191215693950653, "fcm_dpo/margin": 228.45831298828125, "fcm_dpo/q_t": 0.3956291675567627, "grad_norm": 24.535133361816406, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -0.4499407708644867, "logits/rejected": -0.44685885310173035, "logps/chosen": -387.4201965332031, "logps/ref_chosen": -52.91154861450195, "logps/ref_rejected": -90.8226318359375, "logps/rejected": -653.7896118164062, "loss": 1.0496, "margin_dpo/margin_mean": 228.45831298828125, "margin_dpo/margin_std": 280.7309875488281, "step": 394 }, { "KL/chosen_KL_mean": -333.3252258300781, "KL/mean": -451.8715515136719, "KL/rejected_KL_mean": -570.4178466796875, "KL/std": 266.8553466796875, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.0020118574611842632, "fcm_dpo/delta": -0.02858392894268036, "fcm_dpo/margin": 237.0926055908203, "fcm_dpo/q_t": 0.3931659460067749, "grad_norm": 25.89862632751465, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -0.4901076555252075, "logits/rejected": -0.4859440326690674, "logps/chosen": -395.871337890625, "logps/ref_chosen": -62.546112060546875, "logps/ref_rejected": -83.78262329101562, "logps/rejected": -654.200439453125, "loss": 1.0512, "margin_dpo/margin_mean": 237.0926055908203, "margin_dpo/margin_std": 305.77239990234375, "step": 395 }, { "KL/chosen_KL_mean": -341.327880859375, "KL/mean": -450.71270751953125, "KL/rejected_KL_mean": -560.0974731445312, "KL/std": 258.5706787109375, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.002014473080635071, "fcm_dpo/delta": 0.009694435633718967, "fcm_dpo/margin": 218.7696075439453, "fcm_dpo/q_t": 0.39814984798431396, "grad_norm": 28.5257511138916, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -0.48850005865097046, "logits/rejected": -0.4679301977157593, "logps/chosen": -410.3238220214844, "logps/ref_chosen": -68.99594116210938, "logps/ref_rejected": -88.64665985107422, "logps/rejected": -648.744140625, "loss": 1.0442, "margin_dpo/margin_mean": 218.7696075439453, "margin_dpo/margin_std": 234.68031311035156, "step": 396 }, { "KL/chosen_KL_mean": -318.0323486328125, "KL/mean": -456.5221862792969, "KL/rejected_KL_mean": -595.0120849609375, "KL/std": 256.3633117675781, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.001978658139705658, "fcm_dpo/delta": -0.10457342863082886, "fcm_dpo/margin": 276.9797058105469, "fcm_dpo/q_t": 0.37371158599853516, "grad_norm": 37.82392120361328, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -0.44460928440093994, "logits/rejected": -0.44621679186820984, "logps/chosen": -379.30950927734375, "logps/ref_chosen": -61.27716827392578, "logps/ref_rejected": -103.11612701416016, "logps/rejected": -698.128173828125, "loss": 0.9756, "margin_dpo/margin_mean": 276.9797058105469, "margin_dpo/margin_std": 266.53509521484375, "step": 397 }, { "KL/chosen_KL_mean": -352.5345458984375, "KL/mean": -476.9874267578125, "KL/rejected_KL_mean": -601.4403686523438, "KL/std": 269.32940673828125, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.001963344868272543, "fcm_dpo/delta": -0.040712494403123856, "fcm_dpo/margin": 248.90582275390625, "fcm_dpo/q_t": 0.39044615626335144, "grad_norm": 25.253372192382812, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -0.5356911420822144, "logits/rejected": -0.5335468649864197, "logps/chosen": -420.68609619140625, "logps/ref_chosen": -68.15155029296875, "logps/ref_rejected": -108.52360534667969, "logps/rejected": -709.9639892578125, "loss": 1.0363, "margin_dpo/margin_mean": 248.90582275390625, "margin_dpo/margin_std": 306.1085205078125, "step": 398 }, { "KL/chosen_KL_mean": -309.63555908203125, "KL/mean": -423.51873779296875, "KL/rejected_KL_mean": -537.40185546875, "KL/std": 241.08370971679688, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.001956004649400711, "fcm_dpo/delta": 0.004525328055024147, "fcm_dpo/margin": 227.76632690429688, "fcm_dpo/q_t": 0.39886805415153503, "grad_norm": 25.981426239013672, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -0.5114948749542236, "logits/rejected": -0.47601059079170227, "logps/chosen": -370.5253601074219, "logps/ref_chosen": -60.889801025390625, "logps/ref_rejected": -77.965576171875, "logps/rejected": -615.367431640625, "loss": 1.0688, "margin_dpo/margin_mean": 227.76632690429688, "margin_dpo/margin_std": 297.8195495605469, "step": 399 }, { "KL/chosen_KL_mean": -291.9248046875, "KL/mean": -454.869384765625, "KL/rejected_KL_mean": -617.8140258789062, "KL/std": 256.5586242675781, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.001904614269733429, "fcm_dpo/delta": -0.18265554308891296, "fcm_dpo/margin": 325.8891906738281, "fcm_dpo/q_t": 0.3558167815208435, "grad_norm": 23.150972366333008, "learning_rate": 2.192779273338215e-07, "logits/chosen": -0.47274285554885864, "logits/rejected": -0.4700891375541687, "logps/chosen": -355.5683898925781, "logps/ref_chosen": -63.64359664916992, "logps/ref_rejected": -105.252685546875, "logps/rejected": -723.0667114257812, "loss": 0.9217, "margin_dpo/margin_mean": 325.88922119140625, "margin_dpo/margin_std": 275.33416748046875, "step": 400 }, { "KL/chosen_KL_mean": -341.201171875, "KL/mean": -432.50018310546875, "KL/rejected_KL_mean": -523.7991943359375, "KL/std": 262.24749755859375, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0019152223831042647, "fcm_dpo/delta": 0.10403911024332047, "fcm_dpo/margin": 182.59799194335938, "fcm_dpo/q_t": 0.42341041564941406, "grad_norm": 29.061887741088867, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -0.45198753476142883, "logits/rejected": -0.4431244134902954, "logps/chosen": -398.36419677734375, "logps/ref_chosen": -57.16303253173828, "logps/ref_rejected": -83.79249572753906, "logps/rejected": -607.5916748046875, "loss": 1.1722, "margin_dpo/margin_mean": 182.59799194335938, "margin_dpo/margin_std": 349.26129150390625, "step": 401 }, { "KL/chosen_KL_mean": -253.4839630126953, "KL/mean": -422.302001953125, "KL/rejected_KL_mean": -591.1199951171875, "KL/std": 288.586669921875, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0018758836667984724, "fcm_dpo/delta": -0.1958317756652832, "fcm_dpo/margin": 337.63604736328125, "fcm_dpo/q_t": 0.35503554344177246, "grad_norm": 27.98488998413086, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -0.5198030471801758, "logits/rejected": -0.5098150372505188, "logps/chosen": -304.2243347167969, "logps/ref_chosen": -50.74037170410156, "logps/ref_rejected": -81.0460433959961, "logps/rejected": -672.1660766601562, "loss": 0.9146, "margin_dpo/margin_mean": 337.63604736328125, "margin_dpo/margin_std": 291.2978515625, "step": 402 }, { "KL/chosen_KL_mean": -288.88800048828125, "KL/mean": -415.9601745605469, "KL/rejected_KL_mean": -543.0323486328125, "KL/std": 270.83282470703125, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0018494720570743084, "fcm_dpo/delta": -0.021076738834381104, "fcm_dpo/margin": 254.14431762695312, "fcm_dpo/q_t": 0.39227643609046936, "grad_norm": 25.534555435180664, "learning_rate": 2.154609112620295e-07, "logits/chosen": -0.479247510433197, "logits/rejected": -0.4792447090148926, "logps/chosen": -336.0353088378906, "logps/ref_chosen": -47.14731216430664, "logps/ref_rejected": -77.2666015625, "logps/rejected": -620.2989501953125, "loss": 1.0356, "margin_dpo/margin_mean": 254.14431762695312, "margin_dpo/margin_std": 284.37744140625, "step": 403 }, { "KL/chosen_KL_mean": -319.4507751464844, "KL/mean": -445.5923156738281, "KL/rejected_KL_mean": -571.73388671875, "KL/std": 257.37884521484375, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0018426838796585798, "fcm_dpo/delta": -0.01567152328789234, "fcm_dpo/margin": 252.2830810546875, "fcm_dpo/q_t": 0.39486825466156006, "grad_norm": 41.37036895751953, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -0.48405522108078003, "logits/rejected": -0.4754765033721924, "logps/chosen": -367.3260498046875, "logps/ref_chosen": -47.875274658203125, "logps/ref_rejected": -77.15499877929688, "logps/rejected": -648.888916015625, "loss": 1.0625, "margin_dpo/margin_mean": 252.2830810546875, "margin_dpo/margin_std": 332.94580078125, "step": 404 }, { "KL/chosen_KL_mean": -357.9908447265625, "KL/mean": -459.80157470703125, "KL/rejected_KL_mean": -561.6121826171875, "KL/std": 276.78570556640625, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0018360248068347573, "fcm_dpo/delta": -0.04308407008647919, "fcm_dpo/margin": 203.621337890625, "fcm_dpo/q_t": 0.4165513515472412, "grad_norm": 33.43486022949219, "learning_rate": 2.129207286861638e-07, "logits/chosen": -0.49097639322280884, "logits/rejected": -0.48623454570770264, "logps/chosen": -423.15374755859375, "logps/ref_chosen": -65.16290283203125, "logps/ref_rejected": -87.18678283691406, "logps/rejected": -648.7990112304688, "loss": 1.1423, "margin_dpo/margin_mean": 203.62135314941406, "margin_dpo/margin_std": 338.03448486328125, "step": 405 }, { "KL/chosen_KL_mean": -317.1373291015625, "KL/mean": -450.2159423828125, "KL/rejected_KL_mean": -583.2945556640625, "KL/std": 284.7554931640625, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0018263730453327298, "fcm_dpo/delta": -0.03822207450866699, "fcm_dpo/margin": 266.1572570800781, "fcm_dpo/q_t": 0.3904591500759125, "grad_norm": 24.831371307373047, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -0.4690974950790405, "logits/rejected": -0.47915327548980713, "logps/chosen": -366.8781433105469, "logps/ref_chosen": -49.740814208984375, "logps/ref_rejected": -92.07862854003906, "logps/rejected": -675.3732299804688, "loss": 1.0355, "margin_dpo/margin_mean": 266.1572570800781, "margin_dpo/margin_std": 316.8359375, "step": 406 }, { "KL/chosen_KL_mean": -341.89794921875, "KL/mean": -428.014892578125, "KL/rejected_KL_mean": -514.1318359375, "KL/std": 227.71868896484375, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0018160839099436998, "fcm_dpo/delta": 0.01041320152580738, "fcm_dpo/margin": 172.23391723632812, "fcm_dpo/q_t": 0.4270077347755432, "grad_norm": 31.205970764160156, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -0.5169594287872314, "logits/rejected": -0.4908292889595032, "logps/chosen": -398.2286376953125, "logps/ref_chosen": -56.33069610595703, "logps/ref_rejected": -77.51209259033203, "logps/rejected": -591.6439208984375, "loss": 1.1713, "margin_dpo/margin_mean": 172.23390197753906, "margin_dpo/margin_std": 299.04534912109375, "step": 407 }, { "KL/chosen_KL_mean": -337.2755432128906, "KL/mean": -435.49871826171875, "KL/rejected_KL_mean": -533.7218627929688, "KL/std": 222.86465454101562, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.001842319150455296, "fcm_dpo/delta": 0.09141945093870163, "fcm_dpo/margin": 196.44630432128906, "fcm_dpo/q_t": 0.41571488976478577, "grad_norm": 28.42995834350586, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -0.5169058442115784, "logits/rejected": -0.49273163080215454, "logps/chosen": -407.0648498535156, "logps/ref_chosen": -69.789306640625, "logps/ref_rejected": -90.09693908691406, "logps/rejected": -623.8187866210938, "loss": 1.1053, "margin_dpo/margin_mean": 196.44630432128906, "margin_dpo/margin_std": 245.20440673828125, "step": 408 }, { "KL/chosen_KL_mean": -327.26873779296875, "KL/mean": -422.0277099609375, "KL/rejected_KL_mean": -516.7866821289062, "KL/std": 237.47921752929688, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.00188281643204391, "fcm_dpo/delta": 0.09627067297697067, "fcm_dpo/margin": 189.51795959472656, "fcm_dpo/q_t": 0.4176041781902313, "grad_norm": 33.84221267700195, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -0.4907482862472534, "logits/rejected": -0.47613096237182617, "logps/chosen": -394.586181640625, "logps/ref_chosen": -67.31744384765625, "logps/ref_rejected": -84.904296875, "logps/rejected": -601.69091796875, "loss": 1.1211, "margin_dpo/margin_mean": 189.51795959472656, "margin_dpo/margin_std": 266.58636474609375, "step": 409 }, { "KL/chosen_KL_mean": -303.18829345703125, "KL/mean": -414.0080261230469, "KL/rejected_KL_mean": -524.8277587890625, "KL/std": 235.29071044921875, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0018944379407912493, "fcm_dpo/delta": 0.031448617577552795, "fcm_dpo/margin": 221.63943481445312, "fcm_dpo/q_t": 0.40194326639175415, "grad_norm": 26.67073631286621, "learning_rate": 2.065879555832674e-07, "logits/chosen": -0.5039969086647034, "logits/rejected": -0.5042929649353027, "logps/chosen": -354.6536865234375, "logps/ref_chosen": -51.465354919433594, "logps/ref_rejected": -83.198974609375, "logps/rejected": -608.0267333984375, "loss": 1.0691, "margin_dpo/margin_mean": 221.63943481445312, "margin_dpo/margin_std": 264.4658508300781, "step": 410 }, { "KL/chosen_KL_mean": -318.7891845703125, "KL/mean": -429.96795654296875, "KL/rejected_KL_mean": -541.1467895507812, "KL/std": 254.40216064453125, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0018927098717540503, "fcm_dpo/delta": 0.029360853135585785, "fcm_dpo/margin": 222.35760498046875, "fcm_dpo/q_t": 0.40499693155288696, "grad_norm": 27.52345848083496, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -0.512640118598938, "logits/rejected": -0.5237692594528198, "logps/chosen": -371.096435546875, "logps/ref_chosen": -52.30727005004883, "logps/ref_rejected": -80.69495391845703, "logps/rejected": -621.8417358398438, "loss": 1.0792, "margin_dpo/margin_mean": 222.3576202392578, "margin_dpo/margin_std": 281.51507568359375, "step": 411 }, { "KL/chosen_KL_mean": -321.4520263671875, "KL/mean": -433.51104736328125, "KL/rejected_KL_mean": -545.570068359375, "KL/std": 247.77584838867188, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0019169066799804568, "fcm_dpo/delta": 0.02129899151623249, "fcm_dpo/margin": 224.1180419921875, "fcm_dpo/q_t": 0.4017683267593384, "grad_norm": 37.43172836303711, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -0.5207057595252991, "logits/rejected": -0.5486534833908081, "logps/chosen": -374.5961608886719, "logps/ref_chosen": -53.144126892089844, "logps/ref_rejected": -100.0608139038086, "logps/rejected": -645.630859375, "loss": 1.0727, "margin_dpo/margin_mean": 224.1180419921875, "margin_dpo/margin_std": 288.02618408203125, "step": 412 }, { "KL/chosen_KL_mean": -341.4588928222656, "KL/mean": -449.88580322265625, "KL/rejected_KL_mean": -558.312744140625, "KL/std": 245.0023193359375, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.001933481078594923, "fcm_dpo/delta": 0.03179698809981346, "fcm_dpo/margin": 216.85382080078125, "fcm_dpo/q_t": 0.40238937735557556, "grad_norm": 32.526119232177734, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -0.48236048221588135, "logits/rejected": -0.4867170751094818, "logps/chosen": -403.0408630371094, "logps/ref_chosen": -61.58196258544922, "logps/ref_rejected": -99.47340393066406, "logps/rejected": -657.7861328125, "loss": 1.0776, "margin_dpo/margin_mean": 216.85382080078125, "margin_dpo/margin_std": 277.03717041015625, "step": 413 }, { "KL/chosen_KL_mean": -309.00274658203125, "KL/mean": -434.9940185546875, "KL/rejected_KL_mean": -560.9852905273438, "KL/std": 244.6328125, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0019339543068781495, "fcm_dpo/delta": -0.04007185250520706, "fcm_dpo/margin": 251.98251342773438, "fcm_dpo/q_t": 0.3875179588794708, "grad_norm": 37.2226676940918, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -0.4675019681453705, "logits/rejected": -0.46069198846817017, "logps/chosen": -355.6342468261719, "logps/ref_chosen": -46.63148498535156, "logps/ref_rejected": -87.64653015136719, "logps/rejected": -648.6318359375, "loss": 1.0291, "margin_dpo/margin_mean": 251.98251342773438, "margin_dpo/margin_std": 279.03228759765625, "step": 414 }, { "KL/chosen_KL_mean": -354.3892822265625, "KL/mean": -446.54925537109375, "KL/rejected_KL_mean": -538.7092895507812, "KL/std": 241.19760131835938, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0019371465314179659, "fcm_dpo/delta": 0.09648571163415909, "fcm_dpo/margin": 184.32000732421875, "fcm_dpo/q_t": 0.4172021448612213, "grad_norm": 31.669076919555664, "learning_rate": 2.002837796667909e-07, "logits/chosen": -0.50961834192276, "logits/rejected": -0.5050987005233765, "logps/chosen": -433.007568359375, "logps/ref_chosen": -78.6182861328125, "logps/ref_rejected": -100.47752380371094, "logps/rejected": -639.186767578125, "loss": 1.1235, "margin_dpo/margin_mean": 184.32000732421875, "margin_dpo/margin_std": 263.8765869140625, "step": 415 }, { "KL/chosen_KL_mean": -323.3999938964844, "KL/mean": -470.9483642578125, "KL/rejected_KL_mean": -618.4967651367188, "KL/std": 268.1399230957031, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0019243636634200811, "fcm_dpo/delta": -0.1247919350862503, "fcm_dpo/margin": 295.0968017578125, "fcm_dpo/q_t": 0.3685443699359894, "grad_norm": 30.524866104125977, "learning_rate": 1.990267419549914e-07, "logits/chosen": -0.4989939332008362, "logits/rejected": -0.500846266746521, "logps/chosen": -381.67913818359375, "logps/ref_chosen": -58.27912521362305, "logps/ref_rejected": -90.56871795654297, "logps/rejected": -709.0654907226562, "loss": 0.9556, "margin_dpo/margin_mean": 295.0968017578125, "margin_dpo/margin_std": 258.8311767578125, "step": 416 }, { "KL/chosen_KL_mean": -328.4429931640625, "KL/mean": -450.8934326171875, "KL/rejected_KL_mean": -573.3438720703125, "KL/std": 248.7044677734375, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0019041774794459343, "fcm_dpo/delta": -0.017197906970977783, "fcm_dpo/margin": 244.90090942382812, "fcm_dpo/q_t": 0.39115339517593384, "grad_norm": 34.585113525390625, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -0.4820740520954132, "logits/rejected": -0.45922285318374634, "logps/chosen": -378.6417236328125, "logps/ref_chosen": -50.1987190246582, "logps/ref_rejected": -68.15184020996094, "logps/rejected": -641.4957275390625, "loss": 1.0242, "margin_dpo/margin_mean": 244.90090942382812, "margin_dpo/margin_std": 243.80502319335938, "step": 417 }, { "KL/chosen_KL_mean": -347.431640625, "KL/mean": -476.9691467285156, "KL/rejected_KL_mean": -606.5067138671875, "KL/std": 282.0134582519531, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0018994242418557405, "fcm_dpo/delta": -0.04503042995929718, "fcm_dpo/margin": 259.0750732421875, "fcm_dpo/q_t": 0.3910468518733978, "grad_norm": 22.76815414428711, "learning_rate": 1.965167291983757e-07, "logits/chosen": -0.5683892965316772, "logits/rejected": -0.5466711521148682, "logps/chosen": -429.41009521484375, "logps/ref_chosen": -81.97846984863281, "logps/ref_rejected": -104.69148254394531, "logps/rejected": -711.1981811523438, "loss": 1.0455, "margin_dpo/margin_mean": 259.0750732421875, "margin_dpo/margin_std": 329.2814025878906, "step": 418 }, { "KL/chosen_KL_mean": -326.120361328125, "KL/mean": -465.67791748046875, "KL/rejected_KL_mean": -605.2354736328125, "KL/std": 252.4942626953125, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0018608008977025747, "fcm_dpo/delta": -0.07318148016929626, "fcm_dpo/margin": 279.1151428222656, "fcm_dpo/q_t": 0.3800005316734314, "grad_norm": 28.3684139251709, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -0.4971669018268585, "logits/rejected": -0.494901180267334, "logps/chosen": -379.0690002441406, "logps/ref_chosen": -52.948646545410156, "logps/ref_rejected": -91.58309936523438, "logps/rejected": -696.818603515625, "loss": 1.0015, "margin_dpo/margin_mean": 279.11517333984375, "margin_dpo/margin_std": 291.6534729003906, "step": 419 }, { "KL/chosen_KL_mean": -420.7146301269531, "KL/mean": -511.06640625, "KL/rejected_KL_mean": -601.4180908203125, "KL/std": 267.8034362792969, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0018872196087613702, "fcm_dpo/delta": 0.1126866489648819, "fcm_dpo/margin": 180.70352172851562, "fcm_dpo/q_t": 0.4257734715938568, "grad_norm": 59.49646759033203, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -0.5531851649284363, "logits/rejected": -0.5215734839439392, "logps/chosen": -498.48455810546875, "logps/ref_chosen": -77.7699203491211, "logps/ref_rejected": -69.31985473632812, "logps/rejected": -670.7379760742188, "loss": 1.178, "margin_dpo/margin_mean": 180.70352172851562, "margin_dpo/margin_std": 357.1778259277344, "step": 420 }, { "KL/chosen_KL_mean": -343.73675537109375, "KL/mean": -445.46746826171875, "KL/rejected_KL_mean": -547.1982421875, "KL/std": 263.5028381347656, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0019216202199459076, "fcm_dpo/delta": 0.060789939016103745, "fcm_dpo/margin": 203.46148681640625, "fcm_dpo/q_t": 0.40861809253692627, "grad_norm": 28.76307487487793, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -0.5395439863204956, "logits/rejected": -0.528742790222168, "logps/chosen": -397.50262451171875, "logps/ref_chosen": -53.765865325927734, "logps/ref_rejected": -89.28144836425781, "logps/rejected": -636.4796752929688, "loss": 1.1064, "margin_dpo/margin_mean": 203.46148681640625, "margin_dpo/margin_std": 282.8474426269531, "step": 421 }, { "KL/chosen_KL_mean": -387.14404296875, "KL/mean": -500.13055419921875, "KL/rejected_KL_mean": -613.1170654296875, "KL/std": 266.84881591796875, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.00192258192691952, "fcm_dpo/delta": 0.016255537047982216, "fcm_dpo/margin": 225.97299194335938, "fcm_dpo/q_t": 0.40042802691459656, "grad_norm": 34.73599624633789, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -0.554384171962738, "logits/rejected": -0.5551073551177979, "logps/chosen": -455.77783203125, "logps/ref_chosen": -68.6337661743164, "logps/ref_rejected": -87.86351013183594, "logps/rejected": -700.9805908203125, "loss": 1.0796, "margin_dpo/margin_mean": 225.97296142578125, "margin_dpo/margin_std": 308.2744140625, "step": 422 }, { "KL/chosen_KL_mean": -360.21820068359375, "KL/mean": -487.47760009765625, "KL/rejected_KL_mean": -614.7369995117188, "KL/std": 256.1720275878906, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0019127356354147196, "fcm_dpo/delta": -0.038765739649534225, "fcm_dpo/margin": 254.51885986328125, "fcm_dpo/q_t": 0.38787776231765747, "grad_norm": 30.29926300048828, "learning_rate": 1.902669377503756e-07, "logits/chosen": -0.5579213500022888, "logits/rejected": -0.5651543140411377, "logps/chosen": -415.20849609375, "logps/ref_chosen": -54.99030303955078, "logps/ref_rejected": -86.30654907226562, "logps/rejected": -701.0435791015625, "loss": 1.0245, "margin_dpo/margin_mean": 254.5188751220703, "margin_dpo/margin_std": 281.6831970214844, "step": 423 }, { "KL/chosen_KL_mean": -322.7941589355469, "KL/mean": -440.9080810546875, "KL/rejected_KL_mean": -559.02197265625, "KL/std": 260.0172424316406, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0019028335809707642, "fcm_dpo/delta": 0.00016645342111587524, "fcm_dpo/margin": 236.227783203125, "fcm_dpo/q_t": 0.39966845512390137, "grad_norm": 32.32646179199219, "learning_rate": 1.890215699729057e-07, "logits/chosen": -0.5459982752799988, "logits/rejected": -0.5231644511222839, "logps/chosen": -378.80609130859375, "logps/ref_chosen": -56.01192092895508, "logps/ref_rejected": -66.47896575927734, "logps/rejected": -625.5009765625, "loss": 1.0689, "margin_dpo/margin_mean": 236.227783203125, "margin_dpo/margin_std": 316.46826171875, "step": 424 }, { "KL/chosen_KL_mean": -347.72894287109375, "KL/mean": -442.76678466796875, "KL/rejected_KL_mean": -537.8046875, "KL/std": 238.0361785888672, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0019408970838412642, "fcm_dpo/delta": 0.08372058719396591, "fcm_dpo/margin": 190.07571411132812, "fcm_dpo/q_t": 0.4140620231628418, "grad_norm": 27.985706329345703, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -0.5209067463874817, "logits/rejected": -0.5194746255874634, "logps/chosen": -394.59796142578125, "logps/ref_chosen": -46.86899948120117, "logps/ref_rejected": -95.92545318603516, "logps/rejected": -633.7301025390625, "loss": 1.1256, "margin_dpo/margin_mean": 190.07571411132812, "margin_dpo/margin_std": 287.202392578125, "step": 425 }, { "KL/chosen_KL_mean": -310.969970703125, "KL/mean": -432.341552734375, "KL/rejected_KL_mean": -553.713134765625, "KL/std": 245.95452880859375, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0019467303063720465, "fcm_dpo/delta": -0.024154789745807648, "fcm_dpo/margin": 242.74319458007812, "fcm_dpo/q_t": 0.3915029764175415, "grad_norm": 32.165985107421875, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -0.5235848426818848, "logits/rejected": -0.4914889931678772, "logps/chosen": -387.55352783203125, "logps/ref_chosen": -76.58354187011719, "logps/ref_rejected": -81.26658630371094, "logps/rejected": -634.979736328125, "loss": 1.0396, "margin_dpo/margin_mean": 242.74319458007812, "margin_dpo/margin_std": 277.2171630859375, "step": 426 }, { "KL/chosen_KL_mean": -299.4535827636719, "KL/mean": -382.96673583984375, "KL/rejected_KL_mean": -466.47991943359375, "KL/std": 207.34808349609375, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.00196866225451231, "fcm_dpo/delta": 0.12538892030715942, "fcm_dpo/margin": 167.02633666992188, "fcm_dpo/q_t": 0.4241478443145752, "grad_norm": 25.871044158935547, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -0.544752299785614, "logits/rejected": -0.5285120010375977, "logps/chosen": -364.3074645996094, "logps/ref_chosen": -64.8538818359375, "logps/ref_rejected": -78.5660171508789, "logps/rejected": -545.0458984375, "loss": 1.14, "margin_dpo/margin_mean": 167.02633666992188, "margin_dpo/margin_std": 245.90792846679688, "step": 427 }, { "KL/chosen_KL_mean": -375.49188232421875, "KL/mean": -489.534423828125, "KL/rejected_KL_mean": -603.576904296875, "KL/std": 270.32257080078125, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0019773703534156084, "fcm_dpo/delta": -0.001193435862660408, "fcm_dpo/margin": 228.0850372314453, "fcm_dpo/q_t": 0.39808985590934753, "grad_norm": 33.139686584472656, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -0.5463074445724487, "logits/rejected": -0.5484579205513, "logps/chosen": -438.1285400390625, "logps/ref_chosen": -62.63666534423828, "logps/ref_rejected": -103.28181457519531, "logps/rejected": -706.8587646484375, "loss": 1.0805, "margin_dpo/margin_mean": 228.08505249023438, "margin_dpo/margin_std": 324.0859069824219, "step": 428 }, { "KL/chosen_KL_mean": -365.306640625, "KL/mean": -460.9208984375, "KL/rejected_KL_mean": -556.53515625, "KL/std": 241.172607421875, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.002017046557739377, "fcm_dpo/delta": 0.06516958028078079, "fcm_dpo/margin": 191.228515625, "fcm_dpo/q_t": 0.4123389720916748, "grad_norm": 34.08958435058594, "learning_rate": 1.828194884925749e-07, "logits/chosen": -0.5600682497024536, "logits/rejected": -0.5419148206710815, "logps/chosen": -446.5406494140625, "logps/ref_chosen": -81.23401641845703, "logps/ref_rejected": -91.79493713378906, "logps/rejected": -648.330078125, "loss": 1.1252, "margin_dpo/margin_mean": 191.228515625, "margin_dpo/margin_std": 289.6811828613281, "step": 429 }, { "KL/chosen_KL_mean": -306.03424072265625, "KL/mean": -404.863037109375, "KL/rejected_KL_mean": -503.69183349609375, "KL/std": 227.00172424316406, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0020285071805119514, "fcm_dpo/delta": 0.05085544288158417, "fcm_dpo/margin": 197.65756225585938, "fcm_dpo/q_t": 0.4082902669906616, "grad_norm": 29.763206481933594, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -0.5655794143676758, "logits/rejected": -0.5617367625236511, "logps/chosen": -366.95458984375, "logps/ref_chosen": -60.920326232910156, "logps/ref_rejected": -104.42280578613281, "logps/rejected": -608.1146240234375, "loss": 1.0833, "margin_dpo/margin_mean": 197.65757751464844, "margin_dpo/margin_std": 247.31101989746094, "step": 430 }, { "KL/chosen_KL_mean": -292.8988952636719, "KL/mean": -434.15472412109375, "KL/rejected_KL_mean": -575.41064453125, "KL/std": 256.44805908203125, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0019896533340215683, "fcm_dpo/delta": -0.11906355619430542, "fcm_dpo/margin": 282.5116882324219, "fcm_dpo/q_t": 0.37088543176651, "grad_norm": 27.917299270629883, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -0.4833042621612549, "logits/rejected": -0.48560959100723267, "logps/chosen": -350.24761962890625, "logps/ref_chosen": -57.34874725341797, "logps/ref_rejected": -92.84022521972656, "logps/rejected": -668.2508544921875, "loss": 0.9744, "margin_dpo/margin_mean": 282.5116882324219, "margin_dpo/margin_std": 277.9963684082031, "step": 431 }, { "KL/chosen_KL_mean": -300.67681884765625, "KL/mean": -429.38427734375, "KL/rejected_KL_mean": -558.0917358398438, "KL/std": 243.09274291992188, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0019767279736697674, "fcm_dpo/delta": -0.0625704973936081, "fcm_dpo/margin": 257.4149169921875, "fcm_dpo/q_t": 0.38336220383644104, "grad_norm": 39.286922454833984, "learning_rate": 1.791192214186223e-07, "logits/chosen": -0.4827229380607605, "logits/rejected": -0.4681396484375, "logps/chosen": -371.7516174316406, "logps/ref_chosen": -71.07479095458984, "logps/ref_rejected": -98.57952880859375, "logps/rejected": -656.6712646484375, "loss": 1.0035, "margin_dpo/margin_mean": 257.4149169921875, "margin_dpo/margin_std": 251.6300048828125, "step": 432 }, { "KL/chosen_KL_mean": -386.28424072265625, "KL/mean": -480.7707824707031, "KL/rejected_KL_mean": -575.25732421875, "KL/std": 247.94790649414062, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0019827033393085003, "fcm_dpo/delta": 0.07806547731161118, "fcm_dpo/margin": 188.97311401367188, "fcm_dpo/q_t": 0.41317322850227356, "grad_norm": 32.629791259765625, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -0.5527180433273315, "logits/rejected": -0.53752601146698, "logps/chosen": -444.55743408203125, "logps/ref_chosen": -58.273193359375, "logps/ref_rejected": -95.95089721679688, "logps/rejected": -671.208251953125, "loss": 1.1252, "margin_dpo/margin_mean": 188.97311401367188, "margin_dpo/margin_std": 289.7763671875, "step": 433 }, { "KL/chosen_KL_mean": -345.12957763671875, "KL/mean": -454.7913818359375, "KL/rejected_KL_mean": -564.453125, "KL/std": 267.315673828125, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0019846938084810972, "fcm_dpo/delta": 0.015174292027950287, "fcm_dpo/margin": 219.32354736328125, "fcm_dpo/q_t": 0.407636821269989, "grad_norm": 24.881118774414062, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -0.549630880355835, "logits/rejected": -0.5502759218215942, "logps/chosen": -407.1033020019531, "logps/ref_chosen": -61.97370147705078, "logps/ref_rejected": -78.49861145019531, "logps/rejected": -642.9517822265625, "loss": 1.0916, "margin_dpo/margin_mean": 219.32354736328125, "margin_dpo/margin_std": 339.36029052734375, "step": 434 }, { "KL/chosen_KL_mean": -320.1326904296875, "KL/mean": -441.7854919433594, "KL/rejected_KL_mean": -563.4382934570312, "KL/std": 257.5054931640625, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0019807624630630016, "fcm_dpo/delta": -0.03362615779042244, "fcm_dpo/margin": 243.30557250976562, "fcm_dpo/q_t": 0.390411376953125, "grad_norm": 30.694725036621094, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -0.5910142660140991, "logits/rejected": -0.5799299478530884, "logps/chosen": -371.634765625, "logps/ref_chosen": -51.502052307128906, "logps/ref_rejected": -87.56689453125, "logps/rejected": -651.0051879882812, "loss": 1.0414, "margin_dpo/margin_mean": 243.30557250976562, "margin_dpo/margin_std": 290.63226318359375, "step": 435 }, { "KL/chosen_KL_mean": -340.8654479980469, "KL/mean": -449.2633056640625, "KL/rejected_KL_mean": -557.6612548828125, "KL/std": 235.69265747070312, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0019800812005996704, "fcm_dpo/delta": 0.02152244932949543, "fcm_dpo/margin": 216.79571533203125, "fcm_dpo/q_t": 0.4021452069282532, "grad_norm": 45.93106460571289, "learning_rate": 1.742118314717391e-07, "logits/chosen": -0.5817707180976868, "logits/rejected": -0.5527193546295166, "logps/chosen": -412.2691650390625, "logps/ref_chosen": -71.40371704101562, "logps/ref_rejected": -82.72775268554688, "logps/rejected": -640.3889770507812, "loss": 1.0708, "margin_dpo/margin_mean": 216.79571533203125, "margin_dpo/margin_std": 274.1136779785156, "step": 436 }, { "KL/chosen_KL_mean": -344.4331359863281, "KL/mean": -455.29022216796875, "KL/rejected_KL_mean": -566.1473388671875, "KL/std": 226.3272705078125, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0019896202720701694, "fcm_dpo/delta": 0.009276720695197582, "fcm_dpo/margin": 221.71414184570312, "fcm_dpo/q_t": 0.39847952127456665, "grad_norm": 27.636913299560547, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -0.5687664747238159, "logits/rejected": -0.5476101636886597, "logps/chosen": -409.1773986816406, "logps/ref_chosen": -64.7442626953125, "logps/ref_rejected": -82.04356384277344, "logps/rejected": -648.1908569335938, "loss": 1.0609, "margin_dpo/margin_mean": 221.714111328125, "margin_dpo/margin_std": 271.2551574707031, "step": 437 }, { "KL/chosen_KL_mean": -358.9455261230469, "KL/mean": -482.2747802734375, "KL/rejected_KL_mean": -605.60400390625, "KL/std": 256.87603759765625, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0019732650835067034, "fcm_dpo/delta": -0.039032407104969025, "fcm_dpo/margin": 246.65850830078125, "fcm_dpo/q_t": 0.3885589838027954, "grad_norm": 39.22119140625, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -0.5658591985702515, "logits/rejected": -0.5471130609512329, "logps/chosen": -417.9642028808594, "logps/ref_chosen": -59.0186653137207, "logps/ref_rejected": -83.07682800292969, "logps/rejected": -688.6808471679688, "loss": 1.0287, "margin_dpo/margin_mean": 246.65850830078125, "margin_dpo/margin_std": 279.75701904296875, "step": 438 }, { "KL/chosen_KL_mean": -371.96002197265625, "KL/mean": -471.221923828125, "KL/rejected_KL_mean": -570.48388671875, "KL/std": 252.96397399902344, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.0019477336900308728, "fcm_dpo/delta": -0.07375746965408325, "fcm_dpo/margin": 198.52389526367188, "fcm_dpo/q_t": 0.4123408794403076, "grad_norm": 32.960567474365234, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -0.5992504358291626, "logits/rejected": -0.6015431880950928, "logps/chosen": -425.74407958984375, "logps/ref_chosen": -53.78407669067383, "logps/ref_rejected": -83.98545837402344, "logps/rejected": -654.4693603515625, "loss": 1.1156, "margin_dpo/margin_mean": 198.52391052246094, "margin_dpo/margin_std": 282.75445556640625, "step": 439 }, { "KL/chosen_KL_mean": -395.271240234375, "KL/mean": -509.26043701171875, "KL/rejected_KL_mean": -623.2496337890625, "KL/std": 295.75421142578125, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0019553429447114468, "fcm_dpo/delta": 0.004188535735011101, "fcm_dpo/margin": 227.97836303710938, "fcm_dpo/q_t": 0.40090298652648926, "grad_norm": 33.02730941772461, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -0.610379695892334, "logits/rejected": -0.6039736270904541, "logps/chosen": -473.8379821777344, "logps/ref_chosen": -78.56671905517578, "logps/ref_rejected": -96.49775695800781, "logps/rejected": -719.7474365234375, "loss": 1.07, "margin_dpo/margin_mean": 227.97836303710938, "margin_dpo/margin_std": 305.73785400390625, "step": 440 }, { "KL/chosen_KL_mean": -428.8697509765625, "KL/mean": -545.2886352539062, "KL/rejected_KL_mean": -661.70751953125, "KL/std": 302.4857482910156, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.001954545732587576, "fcm_dpo/delta": -0.0055627599358558655, "fcm_dpo/margin": 232.83779907226562, "fcm_dpo/q_t": 0.40071725845336914, "grad_norm": 33.655216217041016, "learning_rate": 1.681227682404166e-07, "logits/chosen": -0.6163830757141113, "logits/rejected": -0.6027711033821106, "logps/chosen": -489.6941833496094, "logps/ref_chosen": -60.824440002441406, "logps/ref_rejected": -96.47080993652344, "logps/rejected": -758.1783447265625, "loss": 1.0982, "margin_dpo/margin_mean": 232.83778381347656, "margin_dpo/margin_std": 361.769775390625, "step": 441 }, { "KL/chosen_KL_mean": -376.24456787109375, "KL/mean": -508.6070251464844, "KL/rejected_KL_mean": -640.969482421875, "KL/std": 298.88250732421875, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0019426376093178988, "fcm_dpo/delta": -0.06845666468143463, "fcm_dpo/margin": 264.7249755859375, "fcm_dpo/q_t": 0.3868436813354492, "grad_norm": 34.53351974487305, "learning_rate": 1.669113001300851e-07, "logits/chosen": -0.5752331018447876, "logits/rejected": -0.5666894912719727, "logps/chosen": -423.2557678222656, "logps/ref_chosen": -47.01121520996094, "logps/ref_rejected": -76.53926086425781, "logps/rejected": -717.5087890625, "loss": 1.0305, "margin_dpo/margin_mean": 264.7249755859375, "margin_dpo/margin_std": 324.138427734375, "step": 442 }, { "KL/chosen_KL_mean": -424.26495361328125, "KL/mean": -506.6785583496094, "KL/rejected_KL_mean": -589.0921020507812, "KL/std": 271.61700439453125, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0019203823758289218, "fcm_dpo/delta": -0.0008800772484391928, "fcm_dpo/margin": 164.82717895507812, "fcm_dpo/q_t": 0.42879998683929443, "grad_norm": 39.51858901977539, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -0.5896974802017212, "logits/rejected": -0.5681110620498657, "logps/chosen": -495.5379638671875, "logps/ref_chosen": -71.27301788330078, "logps/ref_rejected": -86.679931640625, "logps/rejected": -675.7720947265625, "loss": 1.1907, "margin_dpo/margin_mean": 164.82717895507812, "margin_dpo/margin_std": 329.18048095703125, "step": 443 }, { "KL/chosen_KL_mean": -414.8670654296875, "KL/mean": -552.189208984375, "KL/rejected_KL_mean": -689.5113525390625, "KL/std": 302.2316589355469, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.001895219786092639, "fcm_dpo/delta": -0.07462961971759796, "fcm_dpo/margin": 274.64422607421875, "fcm_dpo/q_t": 0.3836691081523895, "grad_norm": 32.75905990600586, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -0.5605142116546631, "logits/rejected": -0.5698819756507874, "logps/chosen": -472.0807800292969, "logps/ref_chosen": -57.213706970214844, "logps/ref_rejected": -97.25489807128906, "logps/rejected": -786.7662353515625, "loss": 1.0206, "margin_dpo/margin_mean": 274.6442565917969, "margin_dpo/margin_std": 333.3004150390625, "step": 444 }, { "KL/chosen_KL_mean": -366.47662353515625, "KL/mean": -495.1947326660156, "KL/rejected_KL_mean": -623.912841796875, "KL/std": 253.20156860351562, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0018830732442438602, "fcm_dpo/delta": -0.036526892334222794, "fcm_dpo/margin": 257.4361877441406, "fcm_dpo/q_t": 0.39043277502059937, "grad_norm": 34.652286529541016, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -0.5867752432823181, "logits/rejected": -0.5800052285194397, "logps/chosen": -433.77642822265625, "logps/ref_chosen": -67.29979705810547, "logps/ref_rejected": -92.68267059326172, "logps/rejected": -716.595458984375, "loss": 1.0375, "margin_dpo/margin_mean": 257.4361572265625, "margin_dpo/margin_std": 305.46954345703125, "step": 445 }, { "KL/chosen_KL_mean": -326.2323913574219, "KL/mean": -462.51434326171875, "KL/rejected_KL_mean": -598.79638671875, "KL/std": 260.0819396972656, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0018716863123700023, "fcm_dpo/delta": -0.06374150514602661, "fcm_dpo/margin": 272.56396484375, "fcm_dpo/q_t": 0.3816561698913574, "grad_norm": 33.55470275878906, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -0.5853652954101562, "logits/rejected": -0.5997810363769531, "logps/chosen": -385.33087158203125, "logps/ref_chosen": -59.098487854003906, "logps/ref_rejected": -101.26419067382812, "logps/rejected": -700.060546875, "loss": 0.9998, "margin_dpo/margin_mean": 272.56396484375, "margin_dpo/margin_std": 263.48199462890625, "step": 446 }, { "KL/chosen_KL_mean": -344.34954833984375, "KL/mean": -484.68585205078125, "KL/rejected_KL_mean": -625.0221557617188, "KL/std": 302.6340637207031, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0018296856433153152, "fcm_dpo/delta": -0.06744357943534851, "fcm_dpo/margin": 280.672607421875, "fcm_dpo/q_t": 0.38556453585624695, "grad_norm": 44.91852569580078, "learning_rate": 1.608874379754465e-07, "logits/chosen": -0.6326247453689575, "logits/rejected": -0.6444536447525024, "logps/chosen": -400.42486572265625, "logps/ref_chosen": -56.07533264160156, "logps/ref_rejected": -98.69475555419922, "logps/rejected": -723.7169189453125, "loss": 1.0182, "margin_dpo/margin_mean": 280.672607421875, "margin_dpo/margin_std": 336.85003662109375, "step": 447 }, { "KL/chosen_KL_mean": -371.67254638671875, "KL/mean": -509.5328369140625, "KL/rejected_KL_mean": -647.3931884765625, "KL/std": 269.0086975097656, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.001821416663005948, "fcm_dpo/delta": -0.05504203215241432, "fcm_dpo/margin": 275.7206726074219, "fcm_dpo/q_t": 0.3859594464302063, "grad_norm": 46.23940658569336, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -0.5190054178237915, "logits/rejected": -0.526978075504303, "logps/chosen": -431.6763916015625, "logps/ref_chosen": -60.00384521484375, "logps/ref_rejected": -102.26465606689453, "logps/rejected": -749.6578369140625, "loss": 1.02, "margin_dpo/margin_mean": 275.720703125, "margin_dpo/margin_std": 309.9884033203125, "step": 448 }, { "KL/chosen_KL_mean": -379.67486572265625, "KL/mean": -511.91595458984375, "KL/rejected_KL_mean": -644.1571044921875, "KL/std": 313.6270751953125, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0017995501402765512, "fcm_dpo/delta": -0.02732807956635952, "fcm_dpo/margin": 264.48223876953125, "fcm_dpo/q_t": 0.3959568738937378, "grad_norm": 28.76017951965332, "learning_rate": 1.584941086944423e-07, "logits/chosen": -0.5502740144729614, "logits/rejected": -0.5482885837554932, "logps/chosen": -447.20147705078125, "logps/ref_chosen": -67.52661895751953, "logps/ref_rejected": -88.59690856933594, "logps/rejected": -732.7540283203125, "loss": 1.0654, "margin_dpo/margin_mean": 264.48223876953125, "margin_dpo/margin_std": 377.0546875, "step": 449 }, { "KL/chosen_KL_mean": -308.00689697265625, "KL/mean": -457.80841064453125, "KL/rejected_KL_mean": -607.6099853515625, "KL/std": 286.56097412109375, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0017785783857107162, "fcm_dpo/delta": -0.08745589852333069, "fcm_dpo/margin": 299.60308837890625, "fcm_dpo/q_t": 0.376192569732666, "grad_norm": 43.1098747253418, "learning_rate": 1.573010452010098e-07, "logits/chosen": -0.569983959197998, "logits/rejected": -0.5806500315666199, "logps/chosen": -365.114990234375, "logps/ref_chosen": -57.10811996459961, "logps/ref_rejected": -102.75494384765625, "logps/rejected": -710.3648681640625, "loss": 0.9844, "margin_dpo/margin_mean": 299.6030578613281, "margin_dpo/margin_std": 291.56182861328125, "step": 450 }, { "KL/chosen_KL_mean": -399.1632080078125, "KL/mean": -514.521484375, "KL/rejected_KL_mean": -629.8797607421875, "KL/std": 308.2806396484375, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0017831064760684967, "fcm_dpo/delta": 0.040064115077257156, "fcm_dpo/margin": 230.71658325195312, "fcm_dpo/q_t": 0.4046586751937866, "grad_norm": 30.686725616455078, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -0.6119334697723389, "logits/rejected": -0.5944463014602661, "logps/chosen": -457.63201904296875, "logps/ref_chosen": -58.46883010864258, "logps/ref_rejected": -72.92941284179688, "logps/rejected": -702.8092041015625, "loss": 1.1071, "margin_dpo/margin_mean": 230.71658325195312, "margin_dpo/margin_std": 350.39141845703125, "step": 451 }, { "KL/chosen_KL_mean": -281.5623779296875, "KL/mean": -419.28631591796875, "KL/rejected_KL_mean": -557.01025390625, "KL/std": 256.2310791015625, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0017701241886243224, "fcm_dpo/delta": -0.039577219635248184, "fcm_dpo/margin": 275.44793701171875, "fcm_dpo/q_t": 0.3859231472015381, "grad_norm": 22.7626895904541, "learning_rate": 1.549222776991186e-07, "logits/chosen": -0.5375150442123413, "logits/rejected": -0.5545802116394043, "logps/chosen": -331.95294189453125, "logps/ref_chosen": -50.39055252075195, "logps/ref_rejected": -97.77142333984375, "logps/rejected": -654.7816772460938, "loss": 1.0059, "margin_dpo/margin_mean": 275.4479064941406, "margin_dpo/margin_std": 258.84332275390625, "step": 452 }, { "KL/chosen_KL_mean": -345.3682556152344, "KL/mean": -464.7120361328125, "KL/rejected_KL_mean": -584.0557861328125, "KL/std": 255.49658203125, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0017713573761284351, "fcm_dpo/delta": 0.028329838067293167, "fcm_dpo/margin": 238.6875, "fcm_dpo/q_t": 0.40426090359687805, "grad_norm": 28.043825149536133, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -0.5721216201782227, "logits/rejected": -0.553996205329895, "logps/chosen": -403.0831298828125, "logps/ref_chosen": -57.71485137939453, "logps/ref_rejected": -82.20741271972656, "logps/rejected": -666.26318359375, "loss": 1.0727, "margin_dpo/margin_mean": 238.68748474121094, "margin_dpo/margin_std": 301.5249328613281, "step": 453 }, { "KL/chosen_KL_mean": -398.77349853515625, "KL/mean": -537.2942504882812, "KL/rejected_KL_mean": -675.8150024414062, "KL/std": 296.11334228515625, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0017631438095122576, "fcm_dpo/delta": -0.0406990572810173, "fcm_dpo/margin": 277.04150390625, "fcm_dpo/q_t": 0.38935303688049316, "grad_norm": 35.47222900390625, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -0.6266130208969116, "logits/rejected": -0.6151348352432251, "logps/chosen": -459.7191467285156, "logps/ref_chosen": -60.945648193359375, "logps/ref_rejected": -84.95079040527344, "logps/rejected": -760.7657470703125, "loss": 1.0307, "margin_dpo/margin_mean": 277.04150390625, "margin_dpo/margin_std": 327.983154296875, "step": 454 }, { "KL/chosen_KL_mean": -345.7542724609375, "KL/mean": -505.41656494140625, "KL/rejected_KL_mean": -665.078857421875, "KL/std": 305.7221374511719, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0017463660333305597, "fcm_dpo/delta": -0.11417567729949951, "fcm_dpo/margin": 319.32452392578125, "fcm_dpo/q_t": 0.3756142258644104, "grad_norm": 33.242618560791016, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -0.5865689516067505, "logits/rejected": -0.6037914156913757, "logps/chosen": -390.6409912109375, "logps/ref_chosen": -44.88671112060547, "logps/ref_rejected": -115.30147552490234, "logps/rejected": -780.3803100585938, "loss": 0.9926, "margin_dpo/margin_mean": 319.32452392578125, "margin_dpo/margin_std": 348.2029724121094, "step": 455 }, { "KL/chosen_KL_mean": -358.7083740234375, "KL/mean": -513.3322143554688, "KL/rejected_KL_mean": -667.9560546875, "KL/std": 299.8691711425781, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.001704810420051217, "fcm_dpo/delta": -0.08146022260189056, "fcm_dpo/margin": 309.2476806640625, "fcm_dpo/q_t": 0.3785049617290497, "grad_norm": 22.669376373291016, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -0.587792158126831, "logits/rejected": -0.6062419414520264, "logps/chosen": -415.74517822265625, "logps/ref_chosen": -57.036781311035156, "logps/ref_rejected": -105.21784210205078, "logps/rejected": -773.1738891601562, "loss": 0.9816, "margin_dpo/margin_mean": 309.2476806640625, "margin_dpo/margin_std": 289.26556396484375, "step": 456 }, { "KL/chosen_KL_mean": -343.7048645019531, "KL/mean": -496.25714111328125, "KL/rejected_KL_mean": -648.8094482421875, "KL/std": 288.51763916015625, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0016754315001890063, "fcm_dpo/delta": -0.0648340955376625, "fcm_dpo/margin": 305.10455322265625, "fcm_dpo/q_t": 0.3803033232688904, "grad_norm": 34.253662109375, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -0.5840927362442017, "logits/rejected": -0.5754066705703735, "logps/chosen": -397.9473876953125, "logps/ref_chosen": -54.24253845214844, "logps/ref_rejected": -85.10956573486328, "logps/rejected": -733.9189453125, "loss": 0.9921, "margin_dpo/margin_mean": 305.1045227050781, "margin_dpo/margin_std": 281.67279052734375, "step": 457 }, { "KL/chosen_KL_mean": -343.4403076171875, "KL/mean": -487.31805419921875, "KL/rejected_KL_mean": -631.19580078125, "KL/std": 269.67620849609375, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.001662727678194642, "fcm_dpo/delta": -0.030117183923721313, "fcm_dpo/margin": 287.7554931640625, "fcm_dpo/q_t": 0.3899010121822357, "grad_norm": 26.614864349365234, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -0.5804847478866577, "logits/rejected": -0.5738258361816406, "logps/chosen": -398.84918212890625, "logps/ref_chosen": -55.40888214111328, "logps/ref_rejected": -97.68325805664062, "logps/rejected": -728.8790283203125, "loss": 1.025, "margin_dpo/margin_mean": 287.7554931640625, "margin_dpo/margin_std": 308.74249267578125, "step": 458 }, { "KL/chosen_KL_mean": -373.6165466308594, "KL/mean": -522.7772827148438, "KL/rejected_KL_mean": -671.93798828125, "KL/std": 304.5191955566406, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0016487675020471215, "fcm_dpo/delta": -0.04434160515666008, "fcm_dpo/margin": 298.32147216796875, "fcm_dpo/q_t": 0.38747796416282654, "grad_norm": 26.16275405883789, "learning_rate": 1.466771464027316e-07, "logits/chosen": -0.6179863214492798, "logits/rejected": -0.6297715902328491, "logps/chosen": -420.17401123046875, "logps/ref_chosen": -46.55748748779297, "logps/ref_rejected": -86.16854095458984, "logps/rejected": -758.1065673828125, "loss": 1.0262, "margin_dpo/margin_mean": 298.32147216796875, "margin_dpo/margin_std": 336.20526123046875, "step": 459 }, { "KL/chosen_KL_mean": -432.34326171875, "KL/mean": -589.347412109375, "KL/rejected_KL_mean": -746.3515625, "KL/std": 301.2634582519531, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.001635140273720026, "fcm_dpo/delta": -0.06683729588985443, "fcm_dpo/margin": 314.00830078125, "fcm_dpo/q_t": 0.38239821791648865, "grad_norm": 37.637847900390625, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -0.6397815942764282, "logits/rejected": -0.6632001399993896, "logps/chosen": -483.9781799316406, "logps/ref_chosen": -51.63489532470703, "logps/ref_rejected": -104.11935424804688, "logps/rejected": -850.470947265625, "loss": 1.0048, "margin_dpo/margin_mean": 314.00830078125, "margin_dpo/margin_std": 332.333984375, "step": 460 }, { "KL/chosen_KL_mean": -436.39190673828125, "KL/mean": -561.193115234375, "KL/rejected_KL_mean": -685.9942626953125, "KL/std": 296.3171691894531, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0016416897997260094, "fcm_dpo/delta": 0.041765857487916946, "fcm_dpo/margin": 249.60235595703125, "fcm_dpo/q_t": 0.40686777234077454, "grad_norm": 29.86077880859375, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -0.6551775932312012, "logits/rejected": -0.6613701581954956, "logps/chosen": -491.5738525390625, "logps/ref_chosen": -55.18195724487305, "logps/ref_rejected": -86.47689819335938, "logps/rejected": -772.47119140625, "loss": 1.0875, "margin_dpo/margin_mean": 249.60235595703125, "margin_dpo/margin_std": 334.3131408691406, "step": 461 }, { "KL/chosen_KL_mean": -455.2949523925781, "KL/mean": -557.1524658203125, "KL/rejected_KL_mean": -659.0100708007812, "KL/std": 303.5262451171875, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.00166890409309417, "fcm_dpo/delta": 0.11394853889942169, "fcm_dpo/margin": 203.7151336669922, "fcm_dpo/q_t": 0.42322608828544617, "grad_norm": 42.896141052246094, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -0.7339116334915161, "logits/rejected": -0.7233344912528992, "logps/chosen": -525.2229614257812, "logps/ref_chosen": -69.92803192138672, "logps/ref_rejected": -78.84111022949219, "logps/rejected": -737.8511962890625, "loss": 1.147, "margin_dpo/margin_mean": 203.7151336669922, "margin_dpo/margin_std": 331.66717529296875, "step": 462 }, { "KL/chosen_KL_mean": -450.4989013671875, "KL/mean": -588.5634765625, "KL/rejected_KL_mean": -726.6280517578125, "KL/std": 329.5040283203125, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0016832472756505013, "fcm_dpo/delta": -0.015869732946157455, "fcm_dpo/margin": 276.129150390625, "fcm_dpo/q_t": 0.396880567073822, "grad_norm": 38.95415115356445, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -0.7291879057884216, "logits/rejected": -0.7324599623680115, "logps/chosen": -505.7732849121094, "logps/ref_chosen": -55.27437210083008, "logps/ref_rejected": -89.02497863769531, "logps/rejected": -815.6530151367188, "loss": 1.0696, "margin_dpo/margin_mean": 276.129150390625, "margin_dpo/margin_std": 379.4897155761719, "step": 463 }, { "KL/chosen_KL_mean": -442.44622802734375, "KL/mean": -652.0371704101562, "KL/rejected_KL_mean": -861.628173828125, "KL/std": 379.09393310546875, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0016052606515586376, "fcm_dpo/delta": -0.24131107330322266, "fcm_dpo/margin": 419.1819152832031, "fcm_dpo/q_t": 0.3485097885131836, "grad_norm": 27.98969841003418, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -0.6850185394287109, "logits/rejected": -0.7362366914749146, "logps/chosen": -493.3585205078125, "logps/ref_chosen": -50.91230010986328, "logps/ref_rejected": -102.4893798828125, "logps/rejected": -964.1175537109375, "loss": 0.9105, "margin_dpo/margin_mean": 419.18194580078125, "margin_dpo/margin_std": 395.11285400390625, "step": 464 }, { "KL/chosen_KL_mean": -431.66802978515625, "KL/mean": -607.3240966796875, "KL/rejected_KL_mean": -782.980224609375, "KL/std": 374.92095947265625, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0015757776563987136, "fcm_dpo/delta": -0.109546959400177, "fcm_dpo/margin": 351.31219482421875, "fcm_dpo/q_t": 0.373515784740448, "grad_norm": 42.90937423706055, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -0.6763527393341064, "logits/rejected": -0.6878402233123779, "logps/chosen": -491.78485107421875, "logps/ref_chosen": -60.116851806640625, "logps/ref_rejected": -113.94602966308594, "logps/rejected": -896.92626953125, "loss": 0.9949, "margin_dpo/margin_mean": 351.31219482421875, "margin_dpo/margin_std": 390.9532775878906, "step": 465 }, { "KL/chosen_KL_mean": -476.77154541015625, "KL/mean": -619.39599609375, "KL/rejected_KL_mean": -762.0203857421875, "KL/std": 340.4292297363281, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0015551161486655474, "fcm_dpo/delta": 0.00630560889840126, "fcm_dpo/margin": 285.2489013671875, "fcm_dpo/q_t": 0.3987009823322296, "grad_norm": 34.12393569946289, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -0.7104131579399109, "logits/rejected": -0.7117249965667725, "logps/chosen": -529.6925048828125, "logps/ref_chosen": -52.920921325683594, "logps/ref_rejected": -90.3154296875, "logps/rejected": -852.3358764648438, "loss": 1.0748, "margin_dpo/margin_mean": 285.2489013671875, "margin_dpo/margin_std": 380.52740478515625, "step": 466 }, { "KL/chosen_KL_mean": -609.2482299804688, "KL/mean": -759.8226928710938, "KL/rejected_KL_mean": -910.3971557617188, "KL/std": 434.1422119140625, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.001543498132377863, "fcm_dpo/delta": -0.01767650991678238, "fcm_dpo/margin": 301.14898681640625, "fcm_dpo/q_t": 0.40035611391067505, "grad_norm": 57.92776870727539, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -0.7824667692184448, "logits/rejected": -0.7698106169700623, "logps/chosen": -687.9640502929688, "logps/ref_chosen": -78.7158203125, "logps/ref_rejected": -102.86019897460938, "logps/rejected": -1013.25732421875, "loss": 1.131, "margin_dpo/margin_mean": 301.14898681640625, "margin_dpo/margin_std": 524.8055419921875, "step": 467 }, { "KL/chosen_KL_mean": -472.4308166503906, "KL/mean": -655.0126953125, "KL/rejected_KL_mean": -837.5946044921875, "KL/std": 395.85833740234375, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0015256652841344476, "fcm_dpo/delta": -0.11409050226211548, "fcm_dpo/margin": 365.1637878417969, "fcm_dpo/q_t": 0.3762136697769165, "grad_norm": 44.05681228637695, "learning_rate": 1.362737437810114e-07, "logits/chosen": -0.774760365486145, "logits/rejected": -0.7754192352294922, "logps/chosen": -542.3662109375, "logps/ref_chosen": -69.93536376953125, "logps/ref_rejected": -101.02880859375, "logps/rejected": -938.6234130859375, "loss": 1.003, "margin_dpo/margin_mean": 365.16375732421875, "margin_dpo/margin_std": 439.28802490234375, "step": 468 }, { "KL/chosen_KL_mean": -497.517578125, "KL/mean": -682.90234375, "KL/rejected_KL_mean": -868.287109375, "KL/std": 357.67254638671875, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.001489096088334918, "fcm_dpo/delta": -0.10935281962156296, "fcm_dpo/margin": 370.7695007324219, "fcm_dpo/q_t": 0.3750303089618683, "grad_norm": 42.75261688232422, "learning_rate": 1.351323902551631e-07, "logits/chosen": -0.7901207208633423, "logits/rejected": -0.7956848740577698, "logps/chosen": -565.6422729492188, "logps/ref_chosen": -68.12469482421875, "logps/ref_rejected": -104.78640747070312, "logps/rejected": -973.073486328125, "loss": 0.9883, "margin_dpo/margin_mean": 370.76947021484375, "margin_dpo/margin_std": 395.64892578125, "step": 469 }, { "KL/chosen_KL_mean": -409.7502746582031, "KL/mean": -582.3035888671875, "KL/rejected_KL_mean": -754.8568115234375, "KL/std": 381.45758056640625, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0014777039177715778, "fcm_dpo/delta": -0.06313945353031158, "fcm_dpo/margin": 345.1065673828125, "fcm_dpo/q_t": 0.38330626487731934, "grad_norm": 28.172273635864258, "learning_rate": 1.339940635976592e-07, "logits/chosen": -0.7635469436645508, "logits/rejected": -0.77159583568573, "logps/chosen": -453.54217529296875, "logps/ref_chosen": -43.791927337646484, "logps/ref_rejected": -82.70285034179688, "logps/rejected": -837.5596923828125, "loss": 1.0253, "margin_dpo/margin_mean": 345.1065673828125, "margin_dpo/margin_std": 412.84716796875, "step": 470 }, { "KL/chosen_KL_mean": -509.9143981933594, "KL/mean": -653.645751953125, "KL/rejected_KL_mean": -797.3770751953125, "KL/std": 380.884033203125, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0014690503012388945, "fcm_dpo/delta": 0.028516050428152084, "fcm_dpo/margin": 287.4627380371094, "fcm_dpo/q_t": 0.4048519730567932, "grad_norm": 34.24898147583008, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -0.8426990509033203, "logits/rejected": -0.8456913232803345, "logps/chosen": -573.25390625, "logps/ref_chosen": -63.33952331542969, "logps/ref_rejected": -83.61048126220703, "logps/rejected": -880.987548828125, "loss": 1.0843, "margin_dpo/margin_mean": 287.46270751953125, "margin_dpo/margin_std": 388.9373779296875, "step": 471 }, { "KL/chosen_KL_mean": -516.6842041015625, "KL/mean": -687.0798950195312, "KL/rejected_KL_mean": -857.4755859375, "KL/std": 442.78399658203125, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0014645196497440338, "fcm_dpo/delta": -0.051864590495824814, "fcm_dpo/margin": 340.7913513183594, "fcm_dpo/q_t": 0.392802894115448, "grad_norm": 33.31867599487305, "learning_rate": 1.317266107909975e-07, "logits/chosen": -0.8658884763717651, "logits/rejected": -0.8394947648048401, "logps/chosen": -600.350341796875, "logps/ref_chosen": -83.66610717773438, "logps/ref_rejected": -117.20919799804688, "logps/rejected": -974.684814453125, "loss": 1.0571, "margin_dpo/margin_mean": 340.79132080078125, "margin_dpo/margin_std": 477.96142578125, "step": 472 }, { "KL/chosen_KL_mean": -618.8231201171875, "KL/mean": -719.730712890625, "KL/rejected_KL_mean": -820.6383056640625, "KL/std": 455.2254943847656, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0014934407081454992, "fcm_dpo/delta": 0.15348844230175018, "fcm_dpo/margin": 201.815185546875, "fcm_dpo/q_t": 0.43637731671333313, "grad_norm": 90.4559555053711, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -0.8223817348480225, "logits/rejected": -0.790869951248169, "logps/chosen": -682.320068359375, "logps/ref_chosen": -63.49696731567383, "logps/ref_rejected": -81.14657592773438, "logps/rejected": -901.784912109375, "loss": 1.2702, "margin_dpo/margin_mean": 201.815185546875, "margin_dpo/margin_std": 588.429443359375, "step": 473 }, { "KL/chosen_KL_mean": -513.7476196289062, "KL/mean": -648.7586669921875, "KL/rejected_KL_mean": -783.769775390625, "KL/std": 405.5135498046875, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0015309953596442938, "fcm_dpo/delta": 0.036234356462955475, "fcm_dpo/margin": 270.0220947265625, "fcm_dpo/q_t": 0.4074624478816986, "grad_norm": 41.096923828125, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -0.8282029628753662, "logits/rejected": -0.8300504684448242, "logps/chosen": -566.359619140625, "logps/ref_chosen": -52.6119384765625, "logps/ref_rejected": -90.08041381835938, "logps/rejected": -873.8501586914062, "loss": 1.1394, "margin_dpo/margin_mean": 270.0220947265625, "margin_dpo/margin_std": 470.0948791503906, "step": 474 }, { "KL/chosen_KL_mean": -387.83551025390625, "KL/mean": -572.85498046875, "KL/rejected_KL_mean": -757.87451171875, "KL/std": 353.4816589355469, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.001494031399488449, "fcm_dpo/delta": -0.10902509838342667, "fcm_dpo/margin": 370.0389404296875, "fcm_dpo/q_t": 0.37435996532440186, "grad_norm": 37.775211334228516, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -0.8475956916809082, "logits/rejected": -0.8703779578208923, "logps/chosen": -430.3307189941406, "logps/ref_chosen": -42.49519348144531, "logps/ref_rejected": -90.06294250488281, "logps/rejected": -847.9374389648438, "loss": 0.9778, "margin_dpo/margin_mean": 370.0389404296875, "margin_dpo/margin_std": 375.77508544921875, "step": 475 }, { "KL/chosen_KL_mean": -462.84210205078125, "KL/mean": -608.64599609375, "KL/rejected_KL_mean": -754.4498291015625, "KL/std": 369.49456787109375, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.001488366862758994, "fcm_dpo/delta": 0.016667019575834274, "fcm_dpo/margin": 291.60772705078125, "fcm_dpo/q_t": 0.4024167060852051, "grad_norm": 39.59403610229492, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -0.8512526154518127, "logits/rejected": -0.8595808148384094, "logps/chosen": -505.79150390625, "logps/ref_chosen": -42.94938278198242, "logps/ref_rejected": -73.71023559570312, "logps/rejected": -828.1600341796875, "loss": 1.0732, "margin_dpo/margin_mean": 291.60772705078125, "margin_dpo/margin_std": 385.09527587890625, "step": 476 }, { "KL/chosen_KL_mean": -478.7229919433594, "KL/mean": -620.529296875, "KL/rejected_KL_mean": -762.3355712890625, "KL/std": 378.3432312011719, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0015028079506009817, "fcm_dpo/delta": 0.024538554251194, "fcm_dpo/margin": 283.6126403808594, "fcm_dpo/q_t": 0.40365082025527954, "grad_norm": 31.907373428344727, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -0.8746368288993835, "logits/rejected": -0.8382138013839722, "logps/chosen": -549.49560546875, "logps/ref_chosen": -70.77261352539062, "logps/ref_rejected": -76.13737487792969, "logps/rejected": -838.4730224609375, "loss": 1.1097, "margin_dpo/margin_mean": 283.6126403808594, "margin_dpo/margin_std": 449.53643798828125, "step": 477 }, { "KL/chosen_KL_mean": -380.40606689453125, "KL/mean": -544.5728149414062, "KL/rejected_KL_mean": -708.739501953125, "KL/std": 327.50653076171875, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.001494677970185876, "fcm_dpo/delta": -0.04291198402643204, "fcm_dpo/margin": 328.3334655761719, "fcm_dpo/q_t": 0.38815736770629883, "grad_norm": 32.1937141418457, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.7859635353088379, "logits/rejected": -0.8034379482269287, "logps/chosen": -421.84661865234375, "logps/ref_chosen": -41.440513610839844, "logps/ref_rejected": -85.36196899414062, "logps/rejected": -794.1015014648438, "loss": 1.0287, "margin_dpo/margin_mean": 328.3334655761719, "margin_dpo/margin_std": 377.14154052734375, "step": 478 }, { "KL/chosen_KL_mean": -488.9976501464844, "KL/mean": -642.759033203125, "KL/rejected_KL_mean": -796.5204467773438, "KL/std": 384.7127380371094, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.001495036412961781, "fcm_dpo/delta": -0.011177890002727509, "fcm_dpo/margin": 307.5227966308594, "fcm_dpo/q_t": 0.39697474241256714, "grad_norm": 30.52669334411621, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -0.8063873052597046, "logits/rejected": -0.8286073207855225, "logps/chosen": -542.9055786132812, "logps/ref_chosen": -53.907920837402344, "logps/ref_rejected": -95.1163330078125, "logps/rejected": -891.6367797851562, "loss": 1.0735, "margin_dpo/margin_mean": 307.5227966308594, "margin_dpo/margin_std": 426.5589904785156, "step": 479 }, { "KL/chosen_KL_mean": -568.2318725585938, "KL/mean": -696.13330078125, "KL/rejected_KL_mean": -824.03466796875, "KL/std": 368.392333984375, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0014745690859854221, "fcm_dpo/delta": -0.06113547831773758, "fcm_dpo/margin": 255.80279541015625, "fcm_dpo/q_t": 0.4146702289581299, "grad_norm": 50.518821716308594, "learning_rate": 1.227838333989088e-07, "logits/chosen": -0.842763364315033, "logits/rejected": -0.8381805419921875, "logps/chosen": -626.91455078125, "logps/ref_chosen": -58.682701110839844, "logps/ref_rejected": -82.93248748779297, "logps/rejected": -906.9671630859375, "loss": 1.1417, "margin_dpo/margin_mean": 255.8028106689453, "margin_dpo/margin_std": 410.9605712890625, "step": 480 }, { "KL/chosen_KL_mean": -473.3878173828125, "KL/mean": -666.9186401367188, "KL/rejected_KL_mean": -860.449462890625, "KL/std": 400.59881591796875, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.0014397315680980682, "fcm_dpo/delta": -0.11371327191591263, "fcm_dpo/margin": 387.06158447265625, "fcm_dpo/q_t": 0.3753812313079834, "grad_norm": 39.827056884765625, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -0.8515434265136719, "logits/rejected": -0.8767064809799194, "logps/chosen": -528.3521118164062, "logps/ref_chosen": -54.964271545410156, "logps/ref_rejected": -92.42044067382812, "logps/rejected": -952.869873046875, "loss": 0.9919, "margin_dpo/margin_mean": 387.06158447265625, "margin_dpo/margin_std": 437.6363830566406, "step": 481 }, { "KL/chosen_KL_mean": -537.002197265625, "KL/mean": -625.9591064453125, "KL/rejected_KL_mean": -714.9160766601562, "KL/std": 389.1396484375, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0014753930736333132, "fcm_dpo/delta": 0.19242650270462036, "fcm_dpo/margin": 177.91390991210938, "fcm_dpo/q_t": 0.4377623200416565, "grad_norm": 50.80120849609375, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -0.7530587315559387, "logits/rejected": -0.7404736280441284, "logps/chosen": -604.5556640625, "logps/ref_chosen": -67.553466796875, "logps/ref_rejected": -87.58953857421875, "logps/rejected": -802.505615234375, "loss": 1.2491, "margin_dpo/margin_mean": 177.91390991210938, "margin_dpo/margin_std": 463.9508361816406, "step": 482 }, { "KL/chosen_KL_mean": -468.40838623046875, "KL/mean": -670.406494140625, "KL/rejected_KL_mean": -872.4046630859375, "KL/std": 407.040283203125, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0014546338934451342, "fcm_dpo/delta": -0.1465245485305786, "fcm_dpo/margin": 403.99627685546875, "fcm_dpo/q_t": 0.37036433815956116, "grad_norm": 31.81561851501465, "learning_rate": 1.194847979251979e-07, "logits/chosen": -0.8379670977592468, "logits/rejected": -0.8484210968017578, "logps/chosen": -531.7381591796875, "logps/ref_chosen": -63.32981872558594, "logps/ref_rejected": -95.78697204589844, "logps/rejected": -968.191650390625, "loss": 0.9783, "margin_dpo/margin_mean": 403.99627685546875, "margin_dpo/margin_std": 463.239013671875, "step": 483 }, { "KL/chosen_KL_mean": -399.396484375, "KL/mean": -573.8797607421875, "KL/rejected_KL_mean": -748.3630981445312, "KL/std": 383.2296142578125, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0014413539320230484, "fcm_dpo/delta": -0.05605652928352356, "fcm_dpo/margin": 348.9666748046875, "fcm_dpo/q_t": 0.3857998847961426, "grad_norm": 47.86911392211914, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -0.825036883354187, "logits/rejected": -0.8444268703460693, "logps/chosen": -458.53460693359375, "logps/ref_chosen": -59.13812255859375, "logps/ref_rejected": -84.37144470214844, "logps/rejected": -832.7345581054688, "loss": 1.0165, "margin_dpo/margin_mean": 348.9666442871094, "margin_dpo/margin_std": 382.70794677734375, "step": 484 }, { "KL/chosen_KL_mean": -446.0076904296875, "KL/mean": -606.5994873046875, "KL/rejected_KL_mean": -767.1912841796875, "KL/std": 386.23712158203125, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0014315908774733543, "fcm_dpo/delta": -0.01041480153799057, "fcm_dpo/margin": 321.18359375, "fcm_dpo/q_t": 0.39765143394470215, "grad_norm": 30.10974884033203, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -0.7779227495193481, "logits/rejected": -0.7912708520889282, "logps/chosen": -504.85723876953125, "logps/ref_chosen": -58.849571228027344, "logps/ref_rejected": -103.36408233642578, "logps/rejected": -870.555419921875, "loss": 1.0713, "margin_dpo/margin_mean": 321.1835632324219, "margin_dpo/margin_std": 446.01361083984375, "step": 485 }, { "KL/chosen_KL_mean": -487.64141845703125, "KL/mean": -678.8277587890625, "KL/rejected_KL_mean": -870.01416015625, "KL/std": 434.8795471191406, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0014062260743230581, "fcm_dpo/delta": -0.09276005625724792, "fcm_dpo/margin": 382.3727722167969, "fcm_dpo/q_t": 0.38320037722587585, "grad_norm": 37.55424118041992, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -0.8121265172958374, "logits/rejected": -0.8253448009490967, "logps/chosen": -542.901123046875, "logps/ref_chosen": -55.25966262817383, "logps/ref_rejected": -92.13936614990234, "logps/rejected": -962.153564453125, "loss": 1.0407, "margin_dpo/margin_mean": 382.3727722167969, "margin_dpo/margin_std": 533.3743286132812, "step": 486 }, { "KL/chosen_KL_mean": -539.01416015625, "KL/mean": -692.552001953125, "KL/rejected_KL_mean": -846.08984375, "KL/std": 414.926025390625, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0014086742885410786, "fcm_dpo/delta": 0.017959173768758774, "fcm_dpo/margin": 307.0757751464844, "fcm_dpo/q_t": 0.40038296580314636, "grad_norm": 47.89488220214844, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -0.8488699197769165, "logits/rejected": -0.8536651730537415, "logps/chosen": -592.077392578125, "logps/ref_chosen": -53.06330871582031, "logps/ref_rejected": -92.41883087158203, "logps/rejected": -938.5086669921875, "loss": 1.0884, "margin_dpo/margin_mean": 307.0757751464844, "margin_dpo/margin_std": 442.1873779296875, "step": 487 }, { "KL/chosen_KL_mean": -430.9812927246094, "KL/mean": -593.21337890625, "KL/rejected_KL_mean": -755.4454345703125, "KL/std": 377.4382629394531, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0014093462377786636, "fcm_dpo/delta": -0.00790850818157196, "fcm_dpo/margin": 324.46417236328125, "fcm_dpo/q_t": 0.3960450291633606, "grad_norm": 43.94591522216797, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -0.7722993493080139, "logits/rejected": -0.7786195278167725, "logps/chosen": -483.2094421386719, "logps/ref_chosen": -52.22815704345703, "logps/ref_rejected": -84.00656127929688, "logps/rejected": -839.4520263671875, "loss": 1.0493, "margin_dpo/margin_mean": 324.4642028808594, "margin_dpo/margin_std": 387.154541015625, "step": 488 }, { "KL/chosen_KL_mean": -415.774658203125, "KL/mean": -566.9913940429688, "KL/rejected_KL_mean": -718.2081298828125, "KL/std": 378.1129455566406, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.0014117740793153644, "fcm_dpo/delta": 0.023877691477537155, "fcm_dpo/margin": 302.43341064453125, "fcm_dpo/q_t": 0.40576857328414917, "grad_norm": 32.45310974121094, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -0.8120991587638855, "logits/rejected": -0.8098138570785522, "logps/chosen": -471.7642822265625, "logps/ref_chosen": -55.989627838134766, "logps/ref_rejected": -79.39812469482422, "logps/rejected": -797.606201171875, "loss": 1.0815, "margin_dpo/margin_mean": 302.43341064453125, "margin_dpo/margin_std": 418.1960754394531, "step": 489 }, { "KL/chosen_KL_mean": -514.631591796875, "KL/mean": -681.1277465820312, "KL/rejected_KL_mean": -847.6239013671875, "KL/std": 470.99456787109375, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0014093939680606127, "fcm_dpo/delta": -0.0203517135232687, "fcm_dpo/margin": 332.9923095703125, "fcm_dpo/q_t": 0.39851754903793335, "grad_norm": 56.83975601196289, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -0.8639624118804932, "logits/rejected": -0.9009851217269897, "logps/chosen": -566.998046875, "logps/ref_chosen": -52.36639404296875, "logps/ref_rejected": -110.4090576171875, "logps/rejected": -958.032958984375, "loss": 1.1059, "margin_dpo/margin_mean": 332.9923095703125, "margin_dpo/margin_std": 549.37158203125, "step": 490 }, { "KL/chosen_KL_mean": -476.43017578125, "KL/mean": -588.9735107421875, "KL/rejected_KL_mean": -701.5167236328125, "KL/std": 392.32257080078125, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.001434988807886839, "fcm_dpo/delta": 0.13088186085224152, "fcm_dpo/margin": 225.08653259277344, "fcm_dpo/q_t": 0.42855405807495117, "grad_norm": 35.58346176147461, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -0.8441819548606873, "logits/rejected": -0.8365911841392517, "logps/chosen": -536.5464477539062, "logps/ref_chosen": -60.11626434326172, "logps/ref_rejected": -73.27278900146484, "logps/rejected": -774.78955078125, "loss": 1.1687, "margin_dpo/margin_mean": 225.08653259277344, "margin_dpo/margin_std": 409.4179382324219, "step": 491 }, { "KL/chosen_KL_mean": -505.2855224609375, "KL/mean": -620.657958984375, "KL/rejected_KL_mean": -736.030517578125, "KL/std": 418.7660827636719, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0014617950655519962, "fcm_dpo/delta": 0.11679119616746902, "fcm_dpo/margin": 230.74496459960938, "fcm_dpo/q_t": 0.42761561274528503, "grad_norm": 37.419185638427734, "learning_rate": 1.097764975115576e-07, "logits/chosen": -0.9038804769515991, "logits/rejected": -0.8865909576416016, "logps/chosen": -559.2796630859375, "logps/ref_chosen": -53.994178771972656, "logps/ref_rejected": -72.65962219238281, "logps/rejected": -808.6901245117188, "loss": 1.1954, "margin_dpo/margin_mean": 230.74497985839844, "margin_dpo/margin_std": 487.385498046875, "step": 492 }, { "KL/chosen_KL_mean": -544.4580688476562, "KL/mean": -659.949951171875, "KL/rejected_KL_mean": -775.44189453125, "KL/std": 443.6405944824219, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0014702860498800874, "fcm_dpo/delta": -0.01809088885784149, "fcm_dpo/margin": 230.98385620117188, "fcm_dpo/q_t": 0.42237934470176697, "grad_norm": 41.59807586669922, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -0.9416635036468506, "logits/rejected": -0.9134702682495117, "logps/chosen": -619.955322265625, "logps/ref_chosen": -75.49723815917969, "logps/ref_rejected": -87.32301330566406, "logps/rejected": -862.764892578125, "loss": 1.1716, "margin_dpo/margin_mean": 230.98385620117188, "margin_dpo/margin_std": 431.32763671875, "step": 493 }, { "KL/chosen_KL_mean": -424.497802734375, "KL/mean": -618.718017578125, "KL/rejected_KL_mean": -812.938232421875, "KL/std": 400.7784423828125, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.0014452817849814892, "fcm_dpo/delta": -0.1179928109049797, "fcm_dpo/margin": 388.44036865234375, "fcm_dpo/q_t": 0.3718945384025574, "grad_norm": 39.06217575073242, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -0.7399489879608154, "logits/rejected": -0.7753201127052307, "logps/chosen": -465.8570556640625, "logps/ref_chosen": -41.35926818847656, "logps/ref_rejected": -86.09136962890625, "logps/rejected": -899.029541015625, "loss": 0.9768, "margin_dpo/margin_mean": 388.44036865234375, "margin_dpo/margin_std": 396.9564208984375, "step": 494 }, { "KL/chosen_KL_mean": -481.8299865722656, "KL/mean": -655.73876953125, "KL/rejected_KL_mean": -829.6475830078125, "KL/std": 413.8068542480469, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.00142820761539042, "fcm_dpo/delta": -0.049179498106241226, "fcm_dpo/margin": 347.817626953125, "fcm_dpo/q_t": 0.39178192615509033, "grad_norm": 40.44029235839844, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -0.91786789894104, "logits/rejected": -0.9246433973312378, "logps/chosen": -545.3650512695312, "logps/ref_chosen": -63.53507995605469, "logps/ref_rejected": -91.42443084716797, "logps/rejected": -921.072021484375, "loss": 1.0539, "margin_dpo/margin_mean": 347.817626953125, "margin_dpo/margin_std": 487.64837646484375, "step": 495 }, { "KL/chosen_KL_mean": -596.630859375, "KL/mean": -693.388916015625, "KL/rejected_KL_mean": -790.1470336914062, "KL/std": 331.32080078125, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.001459787250496447, "fcm_dpo/delta": 0.17228274047374725, "fcm_dpo/margin": 193.51617431640625, "fcm_dpo/q_t": 0.4351438879966736, "grad_norm": 62.60634994506836, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -0.8866853713989258, "logits/rejected": -0.8603767156600952, "logps/chosen": -669.2227783203125, "logps/ref_chosen": -72.5919189453125, "logps/ref_rejected": -84.32933807373047, "logps/rejected": -874.4763793945312, "loss": 1.1956, "margin_dpo/margin_mean": 193.5161895751953, "margin_dpo/margin_std": 373.43487548828125, "step": 496 }, { "KL/chosen_KL_mean": -574.8228759765625, "KL/mean": -666.93798828125, "KL/rejected_KL_mean": -759.0531005859375, "KL/std": 406.2542724609375, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.001477421261370182, "fcm_dpo/delta": 0.019095903262495995, "fcm_dpo/margin": 184.23023986816406, "fcm_dpo/q_t": 0.4375380277633667, "grad_norm": 46.24803924560547, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -0.8670381307601929, "logits/rejected": -0.8514336943626404, "logps/chosen": -633.4168701171875, "logps/ref_chosen": -58.59397506713867, "logps/ref_rejected": -76.28836822509766, "logps/rejected": -835.3414916992188, "loss": 1.2152, "margin_dpo/margin_mean": 184.23023986816406, "margin_dpo/margin_std": 388.11932373046875, "step": 497 }, { "KL/chosen_KL_mean": -540.2715454101562, "KL/mean": -685.60107421875, "KL/rejected_KL_mean": -830.9306640625, "KL/std": 424.14251708984375, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0014816848561167717, "fcm_dpo/delta": 0.020207226276397705, "fcm_dpo/margin": 290.6590881347656, "fcm_dpo/q_t": 0.40341994166374207, "grad_norm": 44.21152877807617, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -0.8600029945373535, "logits/rejected": -0.8464577198028564, "logps/chosen": -611.4771728515625, "logps/ref_chosen": -71.20565795898438, "logps/ref_rejected": -83.95803833007812, "logps/rejected": -914.888671875, "loss": 1.1199, "margin_dpo/margin_mean": 290.6590576171875, "margin_dpo/margin_std": 487.44805908203125, "step": 498 }, { "KL/chosen_KL_mean": -472.27783203125, "KL/mean": -655.0006103515625, "KL/rejected_KL_mean": -837.723388671875, "KL/std": 415.3236999511719, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0014626685297116637, "fcm_dpo/delta": -0.08938349783420563, "fcm_dpo/margin": 365.4455261230469, "fcm_dpo/q_t": 0.3801768124103546, "grad_norm": 38.70993423461914, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -0.8755702972412109, "logits/rejected": -0.9045356512069702, "logps/chosen": -523.5330200195312, "logps/ref_chosen": -51.25519561767578, "logps/ref_rejected": -101.07870483398438, "logps/rejected": -938.8020629882812, "loss": 1.0234, "margin_dpo/margin_mean": 365.4455261230469, "margin_dpo/margin_std": 463.28094482421875, "step": 499 }, { "KL/chosen_KL_mean": -572.775634765625, "KL/mean": -727.460205078125, "KL/rejected_KL_mean": -882.1448974609375, "KL/std": 375.6351318359375, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0014550636988133192, "fcm_dpo/delta": -0.00019872747361660004, "fcm_dpo/margin": 309.3692932128906, "fcm_dpo/q_t": 0.39942067861557007, "grad_norm": 42.67442321777344, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -0.8324778079986572, "logits/rejected": -0.8552345037460327, "logps/chosen": -629.8030395507812, "logps/ref_chosen": -57.027442932128906, "logps/ref_rejected": -93.93421173095703, "logps/rejected": -976.0791015625, "loss": 1.091, "margin_dpo/margin_mean": 309.36932373046875, "margin_dpo/margin_std": 463.14117431640625, "step": 500 }, { "KL/chosen_KL_mean": -506.867431640625, "KL/mean": -670.6456909179688, "KL/rejected_KL_mean": -834.4239501953125, "KL/std": 402.8489990234375, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.00145415507722646, "fcm_dpo/delta": -0.027667174115777016, "fcm_dpo/margin": 327.5564880371094, "fcm_dpo/q_t": 0.39601457118988037, "grad_norm": 43.718650817871094, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -0.8393124341964722, "logits/rejected": -0.8210893273353577, "logps/chosen": -561.2269287109375, "logps/ref_chosen": -54.359527587890625, "logps/ref_rejected": -80.15670013427734, "logps/rejected": -914.5806274414062, "loss": 1.0852, "margin_dpo/margin_mean": 327.55645751953125, "margin_dpo/margin_std": 501.5951232910156, "step": 501 }, { "KL/chosen_KL_mean": -455.6566162109375, "KL/mean": -623.0474243164062, "KL/rejected_KL_mean": -790.4382934570312, "KL/std": 357.35693359375, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0014448589645326138, "fcm_dpo/delta": -0.035458121448755264, "fcm_dpo/margin": 334.7816467285156, "fcm_dpo/q_t": 0.3888877034187317, "grad_norm": 36.984375, "learning_rate": 9.934134090518592e-08, "logits/chosen": -0.7750924825668335, "logits/rejected": -0.7572276592254639, "logps/chosen": -523.2571411132812, "logps/ref_chosen": -67.60050964355469, "logps/ref_rejected": -82.94876098632812, "logps/rejected": -873.3870849609375, "loss": 1.021, "margin_dpo/margin_mean": 334.7816467285156, "margin_dpo/margin_std": 354.950927734375, "step": 502 }, { "KL/chosen_KL_mean": -465.2955322265625, "KL/mean": -621.4876098632812, "KL/rejected_KL_mean": -777.6796875, "KL/std": 354.7469177246094, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.0014357357285916805, "fcm_dpo/delta": 0.0014605261385440826, "fcm_dpo/margin": 312.3841552734375, "fcm_dpo/q_t": 0.39940670132637024, "grad_norm": 31.054977416992188, "learning_rate": 9.831921068732571e-08, "logits/chosen": -0.7540430426597595, "logits/rejected": -0.7392014861106873, "logps/chosen": -520.3739624023438, "logps/ref_chosen": -55.078407287597656, "logps/ref_rejected": -82.50544738769531, "logps/rejected": -860.1851196289062, "loss": 1.0571, "margin_dpo/margin_mean": 312.3841552734375, "margin_dpo/margin_std": 388.606201171875, "step": 503 }, { "KL/chosen_KL_mean": -503.01611328125, "KL/mean": -684.0391845703125, "KL/rejected_KL_mean": -865.0621948242188, "KL/std": 420.90887451171875, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0014258314622566104, "fcm_dpo/delta": -0.06977090984582901, "fcm_dpo/margin": 362.0460510253906, "fcm_dpo/q_t": 0.38591358065605164, "grad_norm": 38.77161407470703, "learning_rate": 9.730107739932805e-08, "logits/chosen": -0.8095864057540894, "logits/rejected": -0.8270820379257202, "logps/chosen": -562.9818115234375, "logps/ref_chosen": -59.96575164794922, "logps/ref_rejected": -103.76212310791016, "logps/rejected": -968.8243408203125, "loss": 1.036, "margin_dpo/margin_mean": 362.0460205078125, "margin_dpo/margin_std": 463.34405517578125, "step": 504 }, { "KL/chosen_KL_mean": -570.5095825195312, "KL/mean": -671.4177856445312, "KL/rejected_KL_mean": -772.3260498046875, "KL/std": 422.823486328125, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0014556320384144783, "fcm_dpo/delta": 0.16071611642837524, "fcm_dpo/margin": 201.81646728515625, "fcm_dpo/q_t": 0.43397390842437744, "grad_norm": 46.75086975097656, "learning_rate": 9.628696786995188e-08, "logits/chosen": -0.825531005859375, "logits/rejected": -0.7959909439086914, "logps/chosen": -646.6644897460938, "logps/ref_chosen": -76.1549072265625, "logps/ref_rejected": -88.58537292480469, "logps/rejected": -860.911376953125, "loss": 1.1987, "margin_dpo/margin_mean": 201.81646728515625, "margin_dpo/margin_std": 405.55169677734375, "step": 505 }, { "KL/chosen_KL_mean": -451.24114990234375, "KL/mean": -619.97607421875, "KL/rejected_KL_mean": -788.7110595703125, "KL/std": 395.23736572265625, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0014494328061118722, "fcm_dpo/delta": -0.0418471023440361, "fcm_dpo/margin": 337.46990966796875, "fcm_dpo/q_t": 0.3904111683368683, "grad_norm": 32.250953674316406, "learning_rate": 9.527690882192635e-08, "logits/chosen": -0.8289726376533508, "logits/rejected": -0.8355446457862854, "logps/chosen": -500.20166015625, "logps/ref_chosen": -48.96050262451172, "logps/ref_rejected": -78.41505432128906, "logps/rejected": -867.1260986328125, "loss": 1.044, "margin_dpo/margin_mean": 337.46990966796875, "margin_dpo/margin_std": 427.08843994140625, "step": 506 }, { "KL/chosen_KL_mean": -512.9838256835938, "KL/mean": -654.876953125, "KL/rejected_KL_mean": -796.7701416015625, "KL/std": 450.25811767578125, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.001459820894524455, "fcm_dpo/delta": 0.037282489240169525, "fcm_dpo/margin": 283.78631591796875, "fcm_dpo/q_t": 0.41154006123542786, "grad_norm": 34.5527229309082, "learning_rate": 9.427092687124691e-08, "logits/chosen": -0.8142247200012207, "logits/rejected": -0.8100612163543701, "logps/chosen": -579.7853393554688, "logps/ref_chosen": -66.80149841308594, "logps/ref_rejected": -95.37289428710938, "logps/rejected": -892.14306640625, "loss": 1.1213, "margin_dpo/margin_mean": 283.7863464355469, "margin_dpo/margin_std": 480.62255859375, "step": 507 }, { "KL/chosen_KL_mean": -530.6473999023438, "KL/mean": -646.10302734375, "KL/rejected_KL_mean": -761.5587158203125, "KL/std": 424.214111328125, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0014920630492269993, "fcm_dpo/delta": 0.10881674289703369, "fcm_dpo/margin": 230.91127014160156, "fcm_dpo/q_t": 0.4257569909095764, "grad_norm": 52.76668930053711, "learning_rate": 9.326904852647344e-08, "logits/chosen": -0.816558837890625, "logits/rejected": -0.8101228475570679, "logps/chosen": -601.9508666992188, "logps/ref_chosen": -71.303466796875, "logps/ref_rejected": -95.6275405883789, "logps/rejected": -857.186279296875, "loss": 1.1953, "margin_dpo/margin_mean": 230.91127014160156, "margin_dpo/margin_std": 495.2095947265625, "step": 508 }, { "KL/chosen_KL_mean": -414.64959716796875, "KL/mean": -541.17529296875, "KL/rejected_KL_mean": -667.700927734375, "KL/std": 303.1500244140625, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0015182525385171175, "fcm_dpo/delta": 0.06775674968957901, "fcm_dpo/margin": 253.0513458251953, "fcm_dpo/q_t": 0.4133894145488739, "grad_norm": 35.25603103637695, "learning_rate": 9.227130018803195e-08, "logits/chosen": -0.7462732791900635, "logits/rejected": -0.7376848459243774, "logps/chosen": -478.46856689453125, "logps/ref_chosen": -63.81895065307617, "logps/ref_rejected": -83.25643920898438, "logps/rejected": -750.9573974609375, "loss": 1.1125, "margin_dpo/margin_mean": 253.05133056640625, "margin_dpo/margin_std": 366.0683288574219, "step": 509 }, { "KL/chosen_KL_mean": -472.19097900390625, "KL/mean": -642.7222900390625, "KL/rejected_KL_mean": -813.2535400390625, "KL/std": 352.47137451171875, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0015035050455480814, "fcm_dpo/delta": -0.06613890826702118, "fcm_dpo/margin": 341.0625305175781, "fcm_dpo/q_t": 0.3812809884548187, "grad_norm": 32.88304138183594, "learning_rate": 9.127770814751932e-08, "logits/chosen": -0.724981963634491, "logits/rejected": -0.7406862378120422, "logps/chosen": -524.0694580078125, "logps/ref_chosen": -51.878448486328125, "logps/ref_rejected": -102.7651596069336, "logps/rejected": -916.0186767578125, "loss": 1.0011, "margin_dpo/margin_mean": 341.0625305175781, "margin_dpo/margin_std": 347.5075378417969, "step": 510 }, { "KL/chosen_KL_mean": -436.04022216796875, "KL/mean": -575.18603515625, "KL/rejected_KL_mean": -714.331787109375, "KL/std": 371.4091491699219, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.001506120664998889, "fcm_dpo/delta": 0.032133154571056366, "fcm_dpo/margin": 278.29156494140625, "fcm_dpo/q_t": 0.4063592255115509, "grad_norm": 38.73003387451172, "learning_rate": 9.028829858700973e-08, "logits/chosen": -0.7694292068481445, "logits/rejected": -0.7684683799743652, "logps/chosen": -496.2783203125, "logps/ref_chosen": -60.23811721801758, "logps/ref_rejected": -92.85676574707031, "logps/rejected": -807.1885986328125, "loss": 1.1117, "margin_dpo/margin_mean": 278.29156494140625, "margin_dpo/margin_std": 445.8023681640625, "step": 511 }, { "KL/chosen_KL_mean": -349.73565673828125, "KL/mean": -525.7312622070312, "KL/rejected_KL_mean": -701.726806640625, "KL/std": 337.34075927734375, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0014879079535603523, "fcm_dpo/delta": -0.07789607346057892, "fcm_dpo/margin": 351.9912109375, "fcm_dpo/q_t": 0.37877798080444336, "grad_norm": 44.94166946411133, "learning_rate": 8.930309757836516e-08, "logits/chosen": -0.7359640002250671, "logits/rejected": -0.7423201203346252, "logps/chosen": -404.64117431640625, "logps/ref_chosen": -54.905494689941406, "logps/ref_rejected": -81.87586975097656, "logps/rejected": -783.6027221679688, "loss": 0.9899, "margin_dpo/margin_mean": 351.9912109375, "margin_dpo/margin_std": 344.17669677734375, "step": 512 }, { "KL/chosen_KL_mean": -427.519287109375, "KL/mean": -548.4598388671875, "KL/rejected_KL_mean": -669.4005126953125, "KL/std": 306.87774658203125, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0014939235989004374, "fcm_dpo/delta": 0.09168398380279541, "fcm_dpo/margin": 241.88121032714844, "fcm_dpo/q_t": 0.4173031449317932, "grad_norm": 37.183837890625, "learning_rate": 8.832213108254863e-08, "logits/chosen": -0.7703028917312622, "logits/rejected": -0.7496137619018555, "logps/chosen": -492.43572998046875, "logps/ref_chosen": -64.91644287109375, "logps/ref_rejected": -76.06245422363281, "logps/rejected": -745.4629516601562, "loss": 1.1291, "margin_dpo/margin_mean": 241.88119506835938, "margin_dpo/margin_std": 356.2818908691406, "step": 513 }, { "KL/chosen_KL_mean": -438.71282958984375, "KL/mean": -566.1483154296875, "KL/rejected_KL_mean": -693.583740234375, "KL/std": 342.07403564453125, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.0015267003327608109, "fcm_dpo/delta": 0.06326548755168915, "fcm_dpo/margin": 254.87098693847656, "fcm_dpo/q_t": 0.41354596614837646, "grad_norm": 29.647640228271484, "learning_rate": 8.734542494893954e-08, "logits/chosen": -0.7364473342895508, "logits/rejected": -0.7220122814178467, "logps/chosen": -512.9423828125, "logps/ref_chosen": -74.22957611083984, "logps/ref_rejected": -78.945556640625, "logps/rejected": -772.529296875, "loss": 1.1138, "margin_dpo/margin_mean": 254.87098693847656, "margin_dpo/margin_std": 387.6467590332031, "step": 514 }, { "KL/chosen_KL_mean": -369.8049011230469, "KL/mean": -478.7362060546875, "KL/rejected_KL_mean": -587.6676025390625, "KL/std": 298.692138671875, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0015549686504527926, "fcm_dpo/delta": 0.11522873491048813, "fcm_dpo/margin": 217.8626708984375, "fcm_dpo/q_t": 0.4216243028640747, "grad_norm": 44.44442367553711, "learning_rate": 8.637300491465272e-08, "logits/chosen": -0.716699481010437, "logits/rejected": -0.7241976261138916, "logps/chosen": -420.20648193359375, "logps/ref_chosen": -50.40156555175781, "logps/ref_rejected": -87.09774780273438, "logps/rejected": -674.7653198242188, "loss": 1.1426, "margin_dpo/margin_mean": 217.8626708984375, "margin_dpo/margin_std": 344.031494140625, "step": 515 }, { "KL/chosen_KL_mean": -394.34375, "KL/mean": -553.999267578125, "KL/rejected_KL_mean": -713.6547241210938, "KL/std": 348.014892578125, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.001558106392621994, "fcm_dpo/delta": -0.050014954060316086, "fcm_dpo/margin": 319.31097412109375, "fcm_dpo/q_t": 0.3847929537296295, "grad_norm": 32.48160934448242, "learning_rate": 8.540489660386064e-08, "logits/chosen": -0.7692265510559082, "logits/rejected": -0.7910833358764648, "logps/chosen": -458.9933166503906, "logps/ref_chosen": -64.64956665039062, "logps/ref_rejected": -111.72237396240234, "logps/rejected": -825.3770751953125, "loss": 1.0142, "margin_dpo/margin_mean": 319.31097412109375, "margin_dpo/margin_std": 336.8834228515625, "step": 516 }, { "KL/chosen_KL_mean": -414.7637939453125, "KL/mean": -587.2954711914062, "KL/rejected_KL_mean": -759.8271484375, "KL/std": 365.62969970703125, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0015280582010746002, "fcm_dpo/delta": -0.08185283839702606, "fcm_dpo/margin": 345.0633850097656, "fcm_dpo/q_t": 0.3820088505744934, "grad_norm": 34.728092193603516, "learning_rate": 8.444112552711752e-08, "logits/chosen": -0.7479615211486816, "logits/rejected": -0.7408775091171265, "logps/chosen": -475.6773681640625, "logps/ref_chosen": -60.913551330566406, "logps/ref_rejected": -89.08308410644531, "logps/rejected": -848.9102783203125, "loss": 1.0071, "margin_dpo/margin_mean": 345.0633544921875, "margin_dpo/margin_std": 395.7159423828125, "step": 517 }, { "KL/chosen_KL_mean": -405.2890625, "KL/mean": -543.8110961914062, "KL/rejected_KL_mean": -682.3330688476562, "KL/std": 322.93963623046875, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.001526032341644168, "fcm_dpo/delta": 0.02827056683599949, "fcm_dpo/margin": 277.0439453125, "fcm_dpo/q_t": 0.40260082483291626, "grad_norm": 40.5467529296875, "learning_rate": 8.348171708068747e-08, "logits/chosen": -0.7425769567489624, "logits/rejected": -0.752970814704895, "logps/chosen": -462.7449951171875, "logps/ref_chosen": -57.45589065551758, "logps/ref_rejected": -85.31269836425781, "logps/rejected": -767.645751953125, "loss": 1.0789, "margin_dpo/margin_mean": 277.0439453125, "margin_dpo/margin_std": 359.08953857421875, "step": 518 }, { "KL/chosen_KL_mean": -415.0616760253906, "KL/mean": -519.9083251953125, "KL/rejected_KL_mean": -624.7550048828125, "KL/std": 290.7900695800781, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0015673746820539236, "fcm_dpo/delta": 0.1248587965965271, "fcm_dpo/margin": 209.69326782226562, "fcm_dpo/q_t": 0.42542368173599243, "grad_norm": 40.930904388427734, "learning_rate": 8.25266965458755e-08, "logits/chosen": -0.7818655967712402, "logits/rejected": -0.7639808058738708, "logps/chosen": -489.125, "logps/ref_chosen": -74.06331634521484, "logps/ref_rejected": -104.44416809082031, "logps/rejected": -729.1990966796875, "loss": 1.1625, "margin_dpo/margin_mean": 209.69326782226562, "margin_dpo/margin_std": 365.93060302734375, "step": 519 }, { "KL/chosen_KL_mean": -439.8741455078125, "KL/mean": -570.5068969726562, "KL/rejected_KL_mean": -701.1396484375, "KL/std": 334.30096435546875, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0015773207414895296, "fcm_dpo/delta": 0.03948332369327545, "fcm_dpo/margin": 261.2655029296875, "fcm_dpo/q_t": 0.4074580669403076, "grad_norm": 35.82596206665039, "learning_rate": 8.15760890883607e-08, "logits/chosen": -0.6950489282608032, "logits/rejected": -0.695152759552002, "logps/chosen": -510.1739501953125, "logps/ref_chosen": -70.2998275756836, "logps/ref_rejected": -99.98133850097656, "logps/rejected": -801.1209716796875, "loss": 1.0935, "margin_dpo/margin_mean": 261.2655029296875, "margin_dpo/margin_std": 358.3149108886719, "step": 520 }, { "KL/chosen_KL_mean": -390.12164306640625, "KL/mean": -535.8663330078125, "KL/rejected_KL_mean": -681.611083984375, "KL/std": 346.71759033203125, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.001592535525560379, "fcm_dpo/delta": -0.015837617218494415, "fcm_dpo/margin": 291.48944091796875, "fcm_dpo/q_t": 0.3943830728530884, "grad_norm": 36.59835433959961, "learning_rate": 8.062991975753378e-08, "logits/chosen": -0.7629398107528687, "logits/rejected": -0.7609637379646301, "logps/chosen": -448.2645568847656, "logps/ref_chosen": -58.14292526245117, "logps/ref_rejected": -83.28060913085938, "logps/rejected": -764.8916625976562, "loss": 1.0487, "margin_dpo/margin_mean": 291.48944091796875, "margin_dpo/margin_std": 345.49517822265625, "step": 521 }, { "KL/chosen_KL_mean": -454.544677734375, "KL/mean": -588.1632080078125, "KL/rejected_KL_mean": -721.7817993164062, "KL/std": 362.4481201171875, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0015862009022384882, "fcm_dpo/delta": 0.02726486325263977, "fcm_dpo/margin": 267.2371520996094, "fcm_dpo/q_t": 0.4044070243835449, "grad_norm": 43.29872131347656, "learning_rate": 7.968821348583643e-08, "logits/chosen": -0.805392324924469, "logits/rejected": -0.808029055595398, "logps/chosen": -501.09234619140625, "logps/ref_chosen": -46.54766845703125, "logps/ref_rejected": -66.01388549804688, "logps/rejected": -787.795654296875, "loss": 1.0949, "margin_dpo/margin_mean": 267.2371520996094, "margin_dpo/margin_std": 387.8713684082031, "step": 522 }, { "KL/chosen_KL_mean": -484.6162109375, "KL/mean": -624.419677734375, "KL/rejected_KL_mean": -764.2230834960938, "KL/std": 422.138671875, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0015884996391832829, "fcm_dpo/delta": 0.006105409003794193, "fcm_dpo/margin": 279.60687255859375, "fcm_dpo/q_t": 0.4030250906944275, "grad_norm": 40.565879821777344, "learning_rate": 7.875099508810484e-08, "logits/chosen": -0.8060259819030762, "logits/rejected": -0.8016799688339233, "logps/chosen": -546.3858032226562, "logps/ref_chosen": -61.76960372924805, "logps/ref_rejected": -83.76141357421875, "logps/rejected": -847.9844970703125, "loss": 1.1083, "margin_dpo/margin_mean": 279.6069030761719, "margin_dpo/margin_std": 456.46990966796875, "step": 523 }, { "KL/chosen_KL_mean": -479.27288818359375, "KL/mean": -626.4615478515625, "KL/rejected_KL_mean": -773.6500854492188, "KL/std": 402.63641357421875, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0015760126989334822, "fcm_dpo/delta": -0.01566839963197708, "fcm_dpo/margin": 294.37713623046875, "fcm_dpo/q_t": 0.3938806653022766, "grad_norm": 39.6019172668457, "learning_rate": 7.781828926091535e-08, "logits/chosen": -0.8418277502059937, "logits/rejected": -0.8301990628242493, "logps/chosen": -557.344970703125, "logps/ref_chosen": -78.0720443725586, "logps/ref_rejected": -81.30198669433594, "logps/rejected": -854.9520874023438, "loss": 1.0749, "margin_dpo/margin_mean": 294.37713623046875, "margin_dpo/margin_std": 403.175048828125, "step": 524 }, { "KL/chosen_KL_mean": -466.339111328125, "KL/mean": -654.3277587890625, "KL/rejected_KL_mean": -842.3164672851562, "KL/std": 413.3827209472656, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.001546194078400731, "fcm_dpo/delta": -0.14039839804172516, "fcm_dpo/margin": 375.97735595703125, "fcm_dpo/q_t": 0.371675968170166, "grad_norm": 39.6491584777832, "learning_rate": 7.689012058193384e-08, "logits/chosen": -0.7878221273422241, "logits/rejected": -0.8154387474060059, "logps/chosen": -517.1669921875, "logps/ref_chosen": -50.827857971191406, "logps/ref_rejected": -100.05294036865234, "logps/rejected": -942.369384765625, "loss": 0.9844, "margin_dpo/margin_mean": 375.97735595703125, "margin_dpo/margin_std": 432.4591064453125, "step": 525 }, { "KL/chosen_KL_mean": -510.3265075683594, "KL/mean": -695.115966796875, "KL/rejected_KL_mean": -879.9052734375, "KL/std": 410.5618896484375, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.001521292608231306, "fcm_dpo/delta": -0.118798166513443, "fcm_dpo/margin": 369.578857421875, "fcm_dpo/q_t": 0.3733553886413574, "grad_norm": 34.82008361816406, "learning_rate": 7.596651350926836e-08, "logits/chosen": -0.8415229320526123, "logits/rejected": -0.8387963175773621, "logps/chosen": -573.4937744140625, "logps/ref_chosen": -63.167236328125, "logps/ref_rejected": -86.30934143066406, "logps/rejected": -966.2146606445312, "loss": 1.0076, "margin_dpo/margin_mean": 369.5788269042969, "margin_dpo/margin_std": 444.198486328125, "step": 526 }, { "KL/chosen_KL_mean": -529.979248046875, "KL/mean": -658.5853271484375, "KL/rejected_KL_mean": -787.1915283203125, "KL/std": 419.4071044921875, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.00151540944352746, "fcm_dpo/delta": 0.06268136203289032, "fcm_dpo/margin": 257.2122802734375, "fcm_dpo/q_t": 0.41041019558906555, "grad_norm": 34.486732482910156, "learning_rate": 7.504749238082414e-08, "logits/chosen": -0.9882876873016357, "logits/rejected": -0.9622901678085327, "logps/chosen": -601.10791015625, "logps/ref_chosen": -71.12867736816406, "logps/ref_rejected": -78.3425521850586, "logps/rejected": -865.5340576171875, "loss": 1.1056, "margin_dpo/margin_mean": 257.2122802734375, "margin_dpo/margin_std": 361.535888671875, "step": 527 }, { "KL/chosen_KL_mean": -559.691162109375, "KL/mean": -719.7154541015625, "KL/rejected_KL_mean": -879.73974609375, "KL/std": 414.49090576171875, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0015139649622142315, "fcm_dpo/delta": -0.0363934151828289, "fcm_dpo/margin": 320.04864501953125, "fcm_dpo/q_t": 0.3962225615978241, "grad_norm": 43.06204605102539, "learning_rate": 7.413308141366254e-08, "logits/chosen": -0.9302852153778076, "logits/rejected": -0.9159493446350098, "logps/chosen": -627.7805786132812, "logps/ref_chosen": -68.0894546508789, "logps/ref_rejected": -93.91006469726562, "logps/rejected": -973.6497802734375, "loss": 1.0788, "margin_dpo/margin_mean": 320.04864501953125, "margin_dpo/margin_std": 486.42547607421875, "step": 528 }, { "KL/chosen_KL_mean": -652.5494995117188, "KL/mean": -759.435546875, "KL/rejected_KL_mean": -866.321533203125, "KL/std": 369.5655517578125, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0015353154158219695, "fcm_dpo/delta": 0.12608586251735687, "fcm_dpo/margin": 213.7720947265625, "fcm_dpo/q_t": 0.4263901710510254, "grad_norm": 58.701969146728516, "learning_rate": 7.322330470336313e-08, "logits/chosen": -0.9356893301010132, "logits/rejected": -0.9435217380523682, "logps/chosen": -708.1244506835938, "logps/ref_chosen": -55.57495880126953, "logps/ref_rejected": -89.20909118652344, "logps/rejected": -955.5306396484375, "loss": 1.2075, "margin_dpo/margin_mean": 213.7720947265625, "margin_dpo/margin_std": 462.04559326171875, "step": 529 }, { "KL/chosen_KL_mean": -566.465576171875, "KL/mean": -738.3973999023438, "KL/rejected_KL_mean": -910.3292236328125, "KL/std": 460.990478515625, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0015337592922151089, "fcm_dpo/delta": -0.08168676495552063, "fcm_dpo/margin": 343.86358642578125, "fcm_dpo/q_t": 0.3879829943180084, "grad_norm": 61.309452056884766, "learning_rate": 7.231818622338822e-08, "logits/chosen": -0.8594100475311279, "logits/rejected": -0.853233814239502, "logps/chosen": -614.0670166015625, "logps/ref_chosen": -47.601417541503906, "logps/ref_rejected": -87.2845230102539, "logps/rejected": -997.6137084960938, "loss": 1.0842, "margin_dpo/margin_mean": 343.86358642578125, "margin_dpo/margin_std": 559.4108276367188, "step": 530 }, { "KL/chosen_KL_mean": -629.9609375, "KL/mean": -774.3426513671875, "KL/rejected_KL_mean": -918.724365234375, "KL/std": 479.8919677734375, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.001531377900391817, "fcm_dpo/delta": 0.008073193021118641, "fcm_dpo/margin": 288.76336669921875, "fcm_dpo/q_t": 0.40305599570274353, "grad_norm": 46.5568733215332, "learning_rate": 7.141774982445147e-08, "logits/chosen": -0.9584941864013672, "logits/rejected": -0.9381182193756104, "logps/chosen": -685.20703125, "logps/ref_chosen": -55.246063232421875, "logps/ref_rejected": -70.60598754882812, "logps/rejected": -989.330322265625, "loss": 1.1007, "margin_dpo/margin_mean": 288.7633972167969, "margin_dpo/margin_std": 451.8643798828125, "step": 531 }, { "KL/chosen_KL_mean": -607.1270751953125, "KL/mean": -774.4474487304688, "KL/rejected_KL_mean": -941.767822265625, "KL/std": 463.4101867675781, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0015030648792162538, "fcm_dpo/delta": -0.05784127488732338, "fcm_dpo/margin": 334.64068603515625, "fcm_dpo/q_t": 0.39187368750572205, "grad_norm": 56.40995788574219, "learning_rate": 7.052201923388953e-08, "logits/chosen": -0.9158481359481812, "logits/rejected": -0.8925724029541016, "logps/chosen": -677.4130859375, "logps/ref_chosen": -70.28601837158203, "logps/ref_rejected": -86.5913314819336, "logps/rejected": -1028.359130859375, "loss": 1.0945, "margin_dpo/margin_mean": 334.64068603515625, "margin_dpo/margin_std": 533.584716796875, "step": 532 }, { "KL/chosen_KL_mean": -560.2667236328125, "KL/mean": -678.7752685546875, "KL/rejected_KL_mean": -797.2838134765625, "KL/std": 397.15875244140625, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0015424289740622044, "fcm_dpo/delta": 0.08588970452547073, "fcm_dpo/margin": 237.0171661376953, "fcm_dpo/q_t": 0.419960618019104, "grad_norm": 48.91042709350586, "learning_rate": 6.963101805503646e-08, "logits/chosen": -0.9363778829574585, "logits/rejected": -0.9149080514907837, "logps/chosen": -625.121826171875, "logps/ref_chosen": -64.8551025390625, "logps/ref_rejected": -76.58805847167969, "logps/rejected": -873.8719482421875, "loss": 1.1801, "margin_dpo/margin_mean": 237.01715087890625, "margin_dpo/margin_std": 479.1162109375, "step": 533 }, { "KL/chosen_KL_mean": -579.787841796875, "KL/mean": -736.6604614257812, "KL/rejected_KL_mean": -893.5330200195312, "KL/std": 427.12310791015625, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0015210118144750595, "fcm_dpo/delta": -0.0296638160943985, "fcm_dpo/margin": 313.74517822265625, "fcm_dpo/q_t": 0.3943442106246948, "grad_norm": 33.77534484863281, "learning_rate": 6.874476976660184e-08, "logits/chosen": -0.9673094749450684, "logits/rejected": -0.9635472297668457, "logps/chosen": -639.9072265625, "logps/ref_chosen": -60.119388580322266, "logps/ref_rejected": -78.54347229003906, "logps/rejected": -972.0764770507812, "loss": 1.0675, "margin_dpo/margin_mean": 313.74517822265625, "margin_dpo/margin_std": 436.4530334472656, "step": 534 }, { "KL/chosen_KL_mean": -489.6236267089844, "KL/mean": -671.65185546875, "KL/rejected_KL_mean": -853.6800537109375, "KL/std": 420.79559326171875, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0015158955939114094, "fcm_dpo/delta": -0.10817711055278778, "fcm_dpo/margin": 364.056396484375, "fcm_dpo/q_t": 0.37864556908607483, "grad_norm": 30.668663024902344, "learning_rate": 6.786329772205246e-08, "logits/chosen": -0.8600409030914307, "logits/rejected": -0.855995774269104, "logps/chosen": -543.953857421875, "logps/ref_chosen": -54.330238342285156, "logps/ref_rejected": -96.30763244628906, "logps/rejected": -949.9876708984375, "loss": 1.0123, "margin_dpo/margin_mean": 364.056396484375, "margin_dpo/margin_std": 439.95440673828125, "step": 535 }, { "KL/chosen_KL_mean": -434.6279296875, "KL/mean": -647.1360473632812, "KL/rejected_KL_mean": -859.6441650390625, "KL/std": 475.992431640625, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.001455649035051465, "fcm_dpo/delta": -0.17993277311325073, "fcm_dpo/margin": 425.0162048339844, "fcm_dpo/q_t": 0.3700369596481323, "grad_norm": 36.75252914428711, "learning_rate": 6.698662514899638e-08, "logits/chosen": -0.8524526357650757, "logits/rejected": -0.8777526021003723, "logps/chosen": -481.7084655761719, "logps/ref_chosen": -47.08053207397461, "logps/ref_rejected": -89.09783935546875, "logps/rejected": -948.7420043945312, "loss": 0.9893, "margin_dpo/margin_mean": 425.0162048339844, "margin_dpo/margin_std": 565.5301513671875, "step": 536 }, { "KL/chosen_KL_mean": -446.46673583984375, "KL/mean": -590.6380615234375, "KL/rejected_KL_mean": -734.809326171875, "KL/std": 377.60614013671875, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.001456779194995761, "fcm_dpo/delta": 0.03063591942191124, "fcm_dpo/margin": 288.34259033203125, "fcm_dpo/q_t": 0.40409615635871887, "grad_norm": 41.455623626708984, "learning_rate": 6.611477514857114e-08, "logits/chosen": -0.8700658679008484, "logits/rejected": -0.845777153968811, "logps/chosen": -504.2142028808594, "logps/ref_chosen": -57.747467041015625, "logps/ref_rejected": -70.43838500976562, "logps/rejected": -805.2477416992188, "loss": 1.1113, "margin_dpo/margin_mean": 288.34259033203125, "margin_dpo/margin_std": 448.64703369140625, "step": 537 }, { "KL/chosen_KL_mean": -551.7672119140625, "KL/mean": -713.7427978515625, "KL/rejected_KL_mean": -875.7183837890625, "KL/std": 390.9005126953125, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0014447167050093412, "fcm_dpo/delta": -0.01904422789812088, "fcm_dpo/margin": 323.95123291015625, "fcm_dpo/q_t": 0.3952096700668335, "grad_norm": 37.458518981933594, "learning_rate": 6.524777069483525e-08, "logits/chosen": -0.859600305557251, "logits/rejected": -0.8362915515899658, "logps/chosen": -618.18310546875, "logps/ref_chosen": -66.41594696044922, "logps/ref_rejected": -84.22808837890625, "logps/rejected": -959.9464721679688, "loss": 1.0501, "margin_dpo/margin_mean": 323.95123291015625, "margin_dpo/margin_std": 409.5311279296875, "step": 538 }, { "KL/chosen_KL_mean": -473.65972900390625, "KL/mean": -622.8203735351562, "KL/rejected_KL_mean": -771.98095703125, "KL/std": 333.13140869140625, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.0014522131532430649, "fcm_dpo/delta": 0.017473286017775536, "fcm_dpo/margin": 298.3211975097656, "fcm_dpo/q_t": 0.401156485080719, "grad_norm": 32.28611373901367, "learning_rate": 6.438563463416221e-08, "logits/chosen": -0.8876945376396179, "logits/rejected": -0.8707320690155029, "logps/chosen": -532.152587890625, "logps/ref_chosen": -58.492855072021484, "logps/ref_rejected": -91.85395050048828, "logps/rejected": -863.8348999023438, "loss": 1.0662, "margin_dpo/margin_mean": 298.3211975097656, "margin_dpo/margin_std": 367.8041076660156, "step": 539 }, { "KL/chosen_KL_mean": -474.45867919921875, "KL/mean": -679.921142578125, "KL/rejected_KL_mean": -885.3836669921875, "KL/std": 437.00653076171875, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0014261383330449462, "fcm_dpo/delta": -0.14425253868103027, "fcm_dpo/margin": 410.92498779296875, "fcm_dpo/q_t": 0.37306225299835205, "grad_norm": 47.68659210205078, "learning_rate": 6.352838968463919e-08, "logits/chosen": -0.8324606418609619, "logits/rejected": -0.8481217622756958, "logps/chosen": -537.9412231445312, "logps/ref_chosen": -63.482513427734375, "logps/ref_rejected": -116.42999267578125, "logps/rejected": -1001.8136596679688, "loss": 0.998, "margin_dpo/margin_mean": 410.92498779296875, "margin_dpo/margin_std": 496.6783447265625, "step": 540 }, { "KL/chosen_KL_mean": -559.8260498046875, "KL/mean": -674.4676513671875, "KL/rejected_KL_mean": -789.1092529296875, "KL/std": 374.70294189453125, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0014067405136302114, "fcm_dpo/delta": -0.012565853074193, "fcm_dpo/margin": 229.2831573486328, "fcm_dpo/q_t": 0.427273690700531, "grad_norm": 42.75546646118164, "learning_rate": 6.267605843546767e-08, "logits/chosen": -0.937226414680481, "logits/rejected": -0.9252729415893555, "logps/chosen": -638.1063842773438, "logps/ref_chosen": -78.28036499023438, "logps/ref_rejected": -103.273681640625, "logps/rejected": -892.3829345703125, "loss": 1.1859, "margin_dpo/margin_mean": 229.28314208984375, "margin_dpo/margin_std": 446.5557556152344, "step": 541 }, { "KL/chosen_KL_mean": -491.94915771484375, "KL/mean": -692.276611328125, "KL/rejected_KL_mean": -892.6041259765625, "KL/std": 434.2323913574219, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.001373377162963152, "fcm_dpo/delta": -0.10823685675859451, "fcm_dpo/margin": 400.65496826171875, "fcm_dpo/q_t": 0.37876492738723755, "grad_norm": 37.0699462890625, "learning_rate": 6.182866334636888e-08, "logits/chosen": -0.9341820478439331, "logits/rejected": -0.9597252011299133, "logps/chosen": -549.4341430664062, "logps/ref_chosen": -57.48497009277344, "logps/ref_rejected": -96.47506713867188, "logps/rejected": -989.0792236328125, "loss": 1.0183, "margin_dpo/margin_mean": 400.65496826171875, "margin_dpo/margin_std": 501.817138671875, "step": 542 }, { "KL/chosen_KL_mean": -553.80517578125, "KL/mean": -692.8021240234375, "KL/rejected_KL_mean": -831.799072265625, "KL/std": 508.77581787109375, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0013881053309887648, "fcm_dpo/delta": 0.06674302369356155, "fcm_dpo/margin": 277.9939880371094, "fcm_dpo/q_t": 0.4248654842376709, "grad_norm": 33.460391998291016, "learning_rate": 6.098622674699147e-08, "logits/chosen": -0.8752850890159607, "logits/rejected": -0.8996802568435669, "logps/chosen": -614.4226684570312, "logps/ref_chosen": -60.61750793457031, "logps/ref_rejected": -105.59896850585938, "logps/rejected": -937.3980712890625, "loss": 1.1719, "margin_dpo/margin_mean": 277.9939880371094, "margin_dpo/margin_std": 586.6079711914062, "step": 543 }, { "KL/chosen_KL_mean": -558.5087890625, "KL/mean": -727.3214111328125, "KL/rejected_KL_mean": -896.134033203125, "KL/std": 419.15606689453125, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0013904159422963858, "fcm_dpo/delta": -0.0204063281416893, "fcm_dpo/margin": 337.62530517578125, "fcm_dpo/q_t": 0.3948058485984802, "grad_norm": 34.55818557739258, "learning_rate": 6.01487708363232e-08, "logits/chosen": -0.8828062415122986, "logits/rejected": -0.8915610313415527, "logps/chosen": -618.1510620117188, "logps/ref_chosen": -59.642303466796875, "logps/ref_rejected": -100.95469665527344, "logps/rejected": -997.0887451171875, "loss": 1.0625, "margin_dpo/margin_mean": 337.62530517578125, "margin_dpo/margin_std": 459.617431640625, "step": 544 }, { "KL/chosen_KL_mean": -547.4271850585938, "KL/mean": -733.0721435546875, "KL/rejected_KL_mean": -918.7172241210938, "KL/std": 412.1142883300781, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0013767255004495382, "fcm_dpo/delta": -0.0644376277923584, "fcm_dpo/margin": 371.29010009765625, "fcm_dpo/q_t": 0.38572588562965393, "grad_norm": 41.80028533935547, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -0.8214889764785767, "logits/rejected": -0.8422670960426331, "logps/chosen": -615.0758056640625, "logps/ref_chosen": -67.64859771728516, "logps/ref_rejected": -95.90800476074219, "logps/rejected": -1014.625244140625, "loss": 1.0307, "margin_dpo/margin_mean": 371.29010009765625, "margin_dpo/margin_std": 462.669189453125, "step": 545 }, { "KL/chosen_KL_mean": -489.4063720703125, "KL/mean": -630.6460571289062, "KL/rejected_KL_mean": -771.8857421875, "KL/std": 367.344970703125, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0013837218284606934, "fcm_dpo/delta": 0.06155392527580261, "fcm_dpo/margin": 282.47943115234375, "fcm_dpo/q_t": 0.40864139795303345, "grad_norm": 30.46135711669922, "learning_rate": 5.848888922025552e-08, "logits/chosen": -0.8541392087936401, "logits/rejected": -0.8365869522094727, "logps/chosen": -540.150634765625, "logps/ref_chosen": -50.744232177734375, "logps/ref_rejected": -81.86622619628906, "logps/rejected": -853.751953125, "loss": 1.0977, "margin_dpo/margin_mean": 282.4794006347656, "margin_dpo/margin_std": 375.0701904296875, "step": 546 }, { "KL/chosen_KL_mean": -507.4529724121094, "KL/mean": -669.9639282226562, "KL/rejected_KL_mean": -832.4749145507812, "KL/std": 411.0699462890625, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.001391751691699028, "fcm_dpo/delta": -0.002595599740743637, "fcm_dpo/margin": 325.02191162109375, "fcm_dpo/q_t": 0.39831429719924927, "grad_norm": 56.123722076416016, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -0.8213566541671753, "logits/rejected": -0.8242645263671875, "logps/chosen": -581.1406860351562, "logps/ref_chosen": -73.6877212524414, "logps/ref_rejected": -90.76136779785156, "logps/rejected": -923.2362670898438, "loss": 1.072, "margin_dpo/margin_mean": 325.0219421386719, "margin_dpo/margin_std": 445.09332275390625, "step": 547 }, { "KL/chosen_KL_mean": -518.0079345703125, "KL/mean": -678.672119140625, "KL/rejected_KL_mean": -839.3363037109375, "KL/std": 424.4984130859375, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0013872598065063357, "fcm_dpo/delta": 0.0044105639681220055, "fcm_dpo/margin": 321.3283996582031, "fcm_dpo/q_t": 0.4029526114463806, "grad_norm": 34.89374542236328, "learning_rate": 5.684919345471029e-08, "logits/chosen": -0.8904018402099609, "logits/rejected": -0.8851450681686401, "logps/chosen": -583.2542724609375, "logps/ref_chosen": -65.24634552001953, "logps/ref_rejected": -94.11807250976562, "logps/rejected": -933.4544067382812, "loss": 1.076, "margin_dpo/margin_mean": 321.3283996582031, "margin_dpo/margin_std": 461.08441162109375, "step": 548 }, { "KL/chosen_KL_mean": -546.9786376953125, "KL/mean": -668.66064453125, "KL/rejected_KL_mean": -790.3426513671875, "KL/std": 357.53765869140625, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0014170631766319275, "fcm_dpo/delta": 0.10819605737924576, "fcm_dpo/margin": 243.364013671875, "fcm_dpo/q_t": 0.4238278865814209, "grad_norm": 43.048282623291016, "learning_rate": 5.603696935852426e-08, "logits/chosen": -0.8407612442970276, "logits/rejected": -0.8215111494064331, "logps/chosen": -596.1910400390625, "logps/ref_chosen": -49.21235656738281, "logps/ref_rejected": -73.91031646728516, "logps/rejected": -864.2529907226562, "loss": 1.1562, "margin_dpo/margin_mean": 243.364013671875, "margin_dpo/margin_std": 427.2247619628906, "step": 549 }, { "KL/chosen_KL_mean": -551.3169555664062, "KL/mean": -692.5491943359375, "KL/rejected_KL_mean": -833.7813720703125, "KL/std": 400.9772033691406, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0014286930672824383, "fcm_dpo/delta": 0.04842350259423256, "fcm_dpo/margin": 282.4644470214844, "fcm_dpo/q_t": 0.4092787206172943, "grad_norm": 44.95556640625, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -0.8258721828460693, "logits/rejected": -0.8465061187744141, "logps/chosen": -608.1239013671875, "logps/ref_chosen": -56.80695343017578, "logps/ref_rejected": -95.12580871582031, "logps/rejected": -928.9072265625, "loss": 1.1081, "margin_dpo/margin_mean": 282.4644470214844, "margin_dpo/margin_std": 430.794921875, "step": 550 }, { "KL/chosen_KL_mean": -480.2443542480469, "KL/mean": -703.80859375, "KL/rejected_KL_mean": -927.3729248046875, "KL/std": 418.818115234375, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.001391774625517428, "fcm_dpo/delta": -0.18421801924705505, "fcm_dpo/margin": 447.1285095214844, "fcm_dpo/q_t": 0.3581318259239197, "grad_norm": 61.52654266357422, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -0.8109757304191589, "logits/rejected": -0.8539941310882568, "logps/chosen": -539.3507080078125, "logps/ref_chosen": -59.10633087158203, "logps/ref_rejected": -111.67280578613281, "logps/rejected": -1039.045654296875, "loss": 0.9306, "margin_dpo/margin_mean": 447.1285095214844, "margin_dpo/margin_std": 412.7304382324219, "step": 551 }, { "KL/chosen_KL_mean": -472.6966857910156, "KL/mean": -719.04345703125, "KL/rejected_KL_mean": -965.3901977539062, "KL/std": 499.03668212890625, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0013249278999865055, "fcm_dpo/delta": -0.2206534445285797, "fcm_dpo/margin": 492.69354248046875, "fcm_dpo/q_t": 0.35978394746780396, "grad_norm": 48.82020568847656, "learning_rate": 5.363104864490034e-08, "logits/chosen": -0.8634598255157471, "logits/rejected": -0.885054886341095, "logps/chosen": -535.05126953125, "logps/ref_chosen": -62.35459899902344, "logps/ref_rejected": -104.56210327148438, "logps/rejected": -1069.952392578125, "loss": 0.9524, "margin_dpo/margin_mean": 492.69354248046875, "margin_dpo/margin_std": 561.2657470703125, "step": 552 }, { "KL/chosen_KL_mean": -526.5339965820312, "KL/mean": -661.3082275390625, "KL/rejected_KL_mean": -796.0825805664062, "KL/std": 403.27972412109375, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0013364567421376705, "fcm_dpo/delta": 0.09321236610412598, "fcm_dpo/margin": 269.548583984375, "fcm_dpo/q_t": 0.42117840051651, "grad_norm": 31.599390029907227, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -0.8489539623260498, "logits/rejected": -0.8407363891601562, "logps/chosen": -594.7928466796875, "logps/ref_chosen": -68.25881958007812, "logps/ref_rejected": -98.0971450805664, "logps/rejected": -894.1797485351562, "loss": 1.1387, "margin_dpo/margin_mean": 269.548583984375, "margin_dpo/margin_std": 446.8441162109375, "step": 553 }, { "KL/chosen_KL_mean": -540.2756958007812, "KL/mean": -729.8994750976562, "KL/rejected_KL_mean": -919.523193359375, "KL/std": 443.01739501953125, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0013425007928162813, "fcm_dpo/delta": -0.0627017393708229, "fcm_dpo/margin": 379.24749755859375, "fcm_dpo/q_t": 0.3905681073665619, "grad_norm": 32.90158462524414, "learning_rate": 5.205293880283551e-08, "logits/chosen": -0.8220857381820679, "logits/rejected": -0.7953811883926392, "logps/chosen": -608.223388671875, "logps/ref_chosen": -67.94767761230469, "logps/ref_rejected": -89.78272247314453, "logps/rejected": -1009.305908203125, "loss": 1.065, "margin_dpo/margin_mean": 379.24749755859375, "margin_dpo/margin_std": 553.5936889648438, "step": 554 }, { "KL/chosen_KL_mean": -558.227783203125, "KL/mean": -759.487548828125, "KL/rejected_KL_mean": -960.7473754882812, "KL/std": 469.44622802734375, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.001314850407652557, "fcm_dpo/delta": -0.0836186334490776, "fcm_dpo/margin": 402.51959228515625, "fcm_dpo/q_t": 0.38830822706222534, "grad_norm": 29.105838775634766, "learning_rate": 5.127169765359515e-08, "logits/chosen": -0.8994369506835938, "logits/rejected": -0.9377848505973816, "logps/chosen": -611.5582885742188, "logps/ref_chosen": -53.33049011230469, "logps/ref_rejected": -108.47937774658203, "logps/rejected": -1069.226806640625, "loss": 1.0599, "margin_dpo/margin_mean": 402.5195617675781, "margin_dpo/margin_std": 603.406494140625, "step": 555 }, { "KL/chosen_KL_mean": -527.7468872070312, "KL/mean": -654.3409423828125, "KL/rejected_KL_mean": -780.9349365234375, "KL/std": 365.3113708496094, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.0013305556494742632, "fcm_dpo/delta": 0.11709566414356232, "fcm_dpo/margin": 253.18809509277344, "fcm_dpo/q_t": 0.42365801334381104, "grad_norm": 41.89741134643555, "learning_rate": 5.049569317994012e-08, "logits/chosen": -0.8416695594787598, "logits/rejected": -0.8288569450378418, "logps/chosen": -586.391357421875, "logps/ref_chosen": -58.64447021484375, "logps/ref_rejected": -101.34040832519531, "logps/rejected": -882.2753295898438, "loss": 1.1364, "margin_dpo/margin_mean": 253.18807983398438, "margin_dpo/margin_std": 379.06951904296875, "step": 556 }, { "KL/chosen_KL_mean": -560.56640625, "KL/mean": -744.7011108398438, "KL/rejected_KL_mean": -928.8358154296875, "KL/std": 482.0714111328125, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.0013270850758999586, "fcm_dpo/delta": -0.04098203778266907, "fcm_dpo/margin": 368.2693176269531, "fcm_dpo/q_t": 0.394328236579895, "grad_norm": 52.759002685546875, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -0.8616045117378235, "logits/rejected": -0.8892296552658081, "logps/chosen": -628.4071044921875, "logps/ref_chosen": -67.84066009521484, "logps/ref_rejected": -109.93965911865234, "logps/rejected": -1038.775390625, "loss": 1.078, "margin_dpo/margin_mean": 368.26934814453125, "margin_dpo/margin_std": 555.1060791015625, "step": 557 }, { "KL/chosen_KL_mean": -500.85626220703125, "KL/mean": -749.5455932617188, "KL/rejected_KL_mean": -998.2349853515625, "KL/std": 460.37445068359375, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0012872398365288973, "fcm_dpo/delta": -0.20348095893859863, "fcm_dpo/margin": 497.3787536621094, "fcm_dpo/q_t": 0.3545846939086914, "grad_norm": 32.1386833190918, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -0.934282660484314, "logits/rejected": -0.9463480114936829, "logps/chosen": -563.2244873046875, "logps/ref_chosen": -62.36824035644531, "logps/ref_rejected": -102.16102600097656, "logps/rejected": -1100.39599609375, "loss": 0.9448, "margin_dpo/margin_mean": 497.3787536621094, "margin_dpo/margin_std": 516.7579956054688, "step": 558 }, { "KL/chosen_KL_mean": -570.8775634765625, "KL/mean": -772.5048828125, "KL/rejected_KL_mean": -974.1322631835938, "KL/std": 453.21221923828125, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0012640644563362002, "fcm_dpo/delta": -0.06291316449642181, "fcm_dpo/margin": 403.254638671875, "fcm_dpo/q_t": 0.38447457551956177, "grad_norm": 36.51702880859375, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -0.9993324875831604, "logits/rejected": -0.9968659281730652, "logps/chosen": -631.6298828125, "logps/ref_chosen": -60.752323150634766, "logps/ref_rejected": -93.44229125976562, "logps/rejected": -1067.574462890625, "loss": 1.0214, "margin_dpo/margin_mean": 403.254638671875, "margin_dpo/margin_std": 473.37640380859375, "step": 559 }, { "KL/chosen_KL_mean": -540.346923828125, "KL/mean": -687.1452026367188, "KL/rejected_KL_mean": -833.9434814453125, "KL/std": 408.98394775390625, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0012668755371123552, "fcm_dpo/delta": 0.08083228766918182, "fcm_dpo/margin": 293.5965881347656, "fcm_dpo/q_t": 0.415485143661499, "grad_norm": 34.44981002807617, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -0.8077666759490967, "logits/rejected": -0.783920168876648, "logps/chosen": -598.45068359375, "logps/ref_chosen": -58.10382080078125, "logps/ref_rejected": -79.99122619628906, "logps/rejected": -913.9346923828125, "loss": 1.1252, "margin_dpo/margin_mean": 293.5965881347656, "margin_dpo/margin_std": 440.9327392578125, "step": 560 }, { "KL/chosen_KL_mean": -604.890869140625, "KL/mean": -737.5593872070312, "KL/rejected_KL_mean": -870.2279052734375, "KL/std": 406.80145263671875, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0013053957372903824, "fcm_dpo/delta": 0.10657086968421936, "fcm_dpo/margin": 265.33697509765625, "fcm_dpo/q_t": 0.41968947649002075, "grad_norm": 41.92377471923828, "learning_rate": 4.669493178106432e-08, "logits/chosen": -0.9422174692153931, "logits/rejected": -0.9543702602386475, "logps/chosen": -655.8037719726562, "logps/ref_chosen": -50.912879943847656, "logps/ref_rejected": -99.06856536865234, "logps/rejected": -969.2964477539062, "loss": 1.1646, "margin_dpo/margin_mean": 265.33697509765625, "margin_dpo/margin_std": 489.222412109375, "step": 561 }, { "KL/chosen_KL_mean": -577.7340698242188, "KL/mean": -753.0718994140625, "KL/rejected_KL_mean": -928.40966796875, "KL/std": 466.2320556640625, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0012981008039787412, "fcm_dpo/delta": -0.006240431219339371, "fcm_dpo/margin": 350.67572021484375, "fcm_dpo/q_t": 0.39807677268981934, "grad_norm": 43.02619552612305, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -0.8793528079986572, "logits/rejected": -0.8840149641036987, "logps/chosen": -637.198486328125, "logps/ref_chosen": -59.46440124511719, "logps/ref_rejected": -96.54266357421875, "logps/rejected": -1024.952392578125, "loss": 1.0778, "margin_dpo/margin_mean": 350.6756896972656, "margin_dpo/margin_std": 499.5946350097656, "step": 562 }, { "KL/chosen_KL_mean": -621.6233520507812, "KL/mean": -756.9385986328125, "KL/rejected_KL_mean": -892.2537841796875, "KL/std": 477.40576171875, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.001299937255680561, "fcm_dpo/delta": -0.04279060661792755, "fcm_dpo/margin": 270.6304931640625, "fcm_dpo/q_t": 0.41562989354133606, "grad_norm": 61.1856575012207, "learning_rate": 4.521198892775202e-08, "logits/chosen": -0.8713696599006653, "logits/rejected": -0.8709218502044678, "logps/chosen": -682.2315673828125, "logps/ref_chosen": -60.60819625854492, "logps/ref_rejected": -94.56770324707031, "logps/rejected": -986.821533203125, "loss": 1.1986, "margin_dpo/margin_mean": 270.6304931640625, "margin_dpo/margin_std": 564.2909545898438, "step": 563 }, { "KL/chosen_KL_mean": -546.1058959960938, "KL/mean": -717.114501953125, "KL/rejected_KL_mean": -888.1231079101562, "KL/std": 416.9074401855469, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0012988729868084192, "fcm_dpo/delta": 0.0058783069252967834, "fcm_dpo/margin": 342.0172119140625, "fcm_dpo/q_t": 0.3986850678920746, "grad_norm": 41.15483474731445, "learning_rate": 4.447860229910544e-08, "logits/chosen": -0.9513506889343262, "logits/rejected": -0.9380670785903931, "logps/chosen": -620.374267578125, "logps/ref_chosen": -74.26837921142578, "logps/ref_rejected": -93.23818969726562, "logps/rejected": -981.3612670898438, "loss": 1.0557, "margin_dpo/margin_mean": 342.0171813964844, "margin_dpo/margin_std": 402.26812744140625, "step": 564 }, { "KL/chosen_KL_mean": -569.7815551757812, "KL/mean": -757.853271484375, "KL/rejected_KL_mean": -945.9249267578125, "KL/std": 480.99176025390625, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0012876607943326235, "fcm_dpo/delta": -0.03614979237318039, "fcm_dpo/margin": 376.14337158203125, "fcm_dpo/q_t": 0.3959125280380249, "grad_norm": 36.2006721496582, "learning_rate": 4.375063135042445e-08, "logits/chosen": -0.8705281615257263, "logits/rejected": -0.8715401887893677, "logps/chosen": -638.801513671875, "logps/ref_chosen": -69.0199203491211, "logps/ref_rejected": -85.7789306640625, "logps/rejected": -1031.703857421875, "loss": 1.0783, "margin_dpo/margin_mean": 376.14337158203125, "margin_dpo/margin_std": 571.8416748046875, "step": 565 }, { "KL/chosen_KL_mean": -550.79443359375, "KL/mean": -754.5787963867188, "KL/rejected_KL_mean": -958.3630981445312, "KL/std": 515.3678588867188, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0012800227850675583, "fcm_dpo/delta": -0.07595265656709671, "fcm_dpo/margin": 407.5686950683594, "fcm_dpo/q_t": 0.3866634964942932, "grad_norm": 34.75884246826172, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -0.8881973028182983, "logits/rejected": -0.908134400844574, "logps/chosen": -617.3397827148438, "logps/ref_chosen": -66.5453109741211, "logps/ref_rejected": -103.86932373046875, "logps/rejected": -1062.232421875, "loss": 1.067, "margin_dpo/margin_mean": 407.56866455078125, "margin_dpo/margin_std": 599.103271484375, "step": 566 }, { "KL/chosen_KL_mean": -554.0453491210938, "KL/mean": -714.4046630859375, "KL/rejected_KL_mean": -874.7639770507812, "KL/std": 369.9399108886719, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.001275490503758192, "fcm_dpo/delta": 0.042663056403398514, "fcm_dpo/margin": 320.71868896484375, "fcm_dpo/q_t": 0.4055943787097931, "grad_norm": 32.36269760131836, "learning_rate": 4.231101308059165e-08, "logits/chosen": -0.9792404174804688, "logits/rejected": -0.988882303237915, "logps/chosen": -606.9036254882812, "logps/ref_chosen": -52.85829544067383, "logps/ref_rejected": -85.37095642089844, "logps/rejected": -960.1349487304688, "loss": 1.0835, "margin_dpo/margin_mean": 320.71868896484375, "margin_dpo/margin_std": 416.3003845214844, "step": 567 }, { "KL/chosen_KL_mean": -536.5615234375, "KL/mean": -743.4915771484375, "KL/rejected_KL_mean": -950.421630859375, "KL/std": 419.36846923828125, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.001262638601474464, "fcm_dpo/delta": -0.07662282139062881, "fcm_dpo/margin": 413.8601989746094, "fcm_dpo/q_t": 0.3797609806060791, "grad_norm": 38.35757064819336, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -0.8925222158432007, "logits/rejected": -0.9186618328094482, "logps/chosen": -581.75390625, "logps/ref_chosen": -45.1923828125, "logps/ref_rejected": -89.09236907958984, "logps/rejected": -1039.5140380859375, "loss": 1.0, "margin_dpo/margin_mean": 413.86016845703125, "margin_dpo/margin_std": 437.47808837890625, "step": 568 }, { "KL/chosen_KL_mean": -592.8074951171875, "KL/mean": -766.2435913085938, "KL/rejected_KL_mean": -939.6796875, "KL/std": 510.0214538574219, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.001268592313863337, "fcm_dpo/delta": 0.009497091174125671, "fcm_dpo/margin": 346.8720703125, "fcm_dpo/q_t": 0.40101489424705505, "grad_norm": 50.98748016357422, "learning_rate": 4.089328585837512e-08, "logits/chosen": -0.9049087166786194, "logits/rejected": -0.907079815864563, "logps/chosen": -656.528076171875, "logps/ref_chosen": -63.72056198120117, "logps/ref_rejected": -79.10325622558594, "logps/rejected": -1018.782958984375, "loss": 1.1008, "margin_dpo/margin_mean": 346.8721008300781, "margin_dpo/margin_std": 524.1836547851562, "step": 569 }, { "KL/chosen_KL_mean": -560.402099609375, "KL/mean": -718.6865234375, "KL/rejected_KL_mean": -876.970947265625, "KL/std": 420.0425720214844, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0012691834708675742, "fcm_dpo/delta": 0.05026433989405632, "fcm_dpo/margin": 316.56884765625, "fcm_dpo/q_t": 0.40913310647010803, "grad_norm": 30.43377685546875, "learning_rate": 4.019267817841834e-08, "logits/chosen": -0.9766222238540649, "logits/rejected": -0.968289852142334, "logps/chosen": -622.0166625976562, "logps/ref_chosen": -61.61454391479492, "logps/ref_rejected": -82.14186096191406, "logps/rejected": -959.11279296875, "loss": 1.098, "margin_dpo/margin_mean": 316.56884765625, "margin_dpo/margin_std": 439.4469909667969, "step": 570 }, { "KL/chosen_KL_mean": -550.743896484375, "KL/mean": -744.7813720703125, "KL/rejected_KL_mean": -938.81884765625, "KL/std": 436.6773986816406, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.0012616091407835484, "fcm_dpo/delta": -0.042000800371170044, "fcm_dpo/margin": 388.07489013671875, "fcm_dpo/q_t": 0.3914453387260437, "grad_norm": 35.876590728759766, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -0.8716817498207092, "logits/rejected": -0.8817984461784363, "logps/chosen": -603.7979736328125, "logps/ref_chosen": -53.05406188964844, "logps/ref_rejected": -91.33682250976562, "logps/rejected": -1030.1556396484375, "loss": 1.0597, "margin_dpo/margin_mean": 388.0748596191406, "margin_dpo/margin_std": 536.5525512695312, "step": 571 }, { "KL/chosen_KL_mean": -602.5777587890625, "KL/mean": -799.7861938476562, "KL/rejected_KL_mean": -996.99462890625, "KL/std": 500.5901184082031, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0012506109196692705, "fcm_dpo/delta": -0.045844513922929764, "fcm_dpo/margin": 394.4167785644531, "fcm_dpo/q_t": 0.3941086232662201, "grad_norm": 31.384313583374023, "learning_rate": 3.880806698864086e-08, "logits/chosen": -0.9135201573371887, "logits/rejected": -0.9341009259223938, "logps/chosen": -651.037109375, "logps/ref_chosen": -48.45928955078125, "logps/ref_rejected": -83.55703735351562, "logps/rejected": -1080.5516357421875, "loss": 1.074, "margin_dpo/margin_mean": 394.416748046875, "margin_dpo/margin_std": 598.5731811523438, "step": 572 }, { "KL/chosen_KL_mean": -587.642578125, "KL/mean": -758.319580078125, "KL/rejected_KL_mean": -928.9966430664062, "KL/std": 452.03662109375, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0012581718619912863, "fcm_dpo/delta": 0.02130233310163021, "fcm_dpo/margin": 341.35406494140625, "fcm_dpo/q_t": 0.4040926694869995, "grad_norm": 29.551095962524414, "learning_rate": 3.812409996461275e-08, "logits/chosen": -0.9765233993530273, "logits/rejected": -0.9776440858840942, "logps/chosen": -639.26513671875, "logps/ref_chosen": -51.62262725830078, "logps/ref_rejected": -85.32499694824219, "logps/rejected": -1014.3216552734375, "loss": 1.0751, "margin_dpo/margin_mean": 341.35406494140625, "margin_dpo/margin_std": 455.6972961425781, "step": 573 }, { "KL/chosen_KL_mean": -532.7286376953125, "KL/mean": -718.6502685546875, "KL/rejected_KL_mean": -904.57177734375, "KL/std": 414.60821533203125, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0012568001402541995, "fcm_dpo/delta": -0.018277421593666077, "fcm_dpo/margin": 371.84307861328125, "fcm_dpo/q_t": 0.3949261009693146, "grad_norm": 34.49797058105469, "learning_rate": 3.74457160675965e-08, "logits/chosen": -0.9791123867034912, "logits/rejected": -1.0038515329360962, "logps/chosen": -583.7731323242188, "logps/ref_chosen": -51.04446029663086, "logps/ref_rejected": -92.80640411376953, "logps/rejected": -997.378173828125, "loss": 1.0554, "margin_dpo/margin_mean": 371.84307861328125, "margin_dpo/margin_std": 477.19677734375, "step": 574 }, { "KL/chosen_KL_mean": -602.52099609375, "KL/mean": -776.2808837890625, "KL/rejected_KL_mean": -950.040771484375, "KL/std": 425.5224304199219, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.001243308070115745, "fcm_dpo/delta": 0.01694381982088089, "fcm_dpo/margin": 347.519775390625, "fcm_dpo/q_t": 0.4027356505393982, "grad_norm": 36.27347183227539, "learning_rate": 3.677293317363864e-08, "logits/chosen": -0.8385648727416992, "logits/rejected": -0.8408699631690979, "logps/chosen": -674.3111572265625, "logps/ref_chosen": -71.7901382446289, "logps/ref_rejected": -95.38619995117188, "logps/rejected": -1045.427001953125, "loss": 1.1118, "margin_dpo/margin_mean": 347.519775390625, "margin_dpo/margin_std": 533.6824951171875, "step": 575 }, { "KL/chosen_KL_mean": -582.2991943359375, "KL/mean": -714.9830322265625, "KL/rejected_KL_mean": -847.6668090820312, "KL/std": 401.23394775390625, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.001272881869226694, "fcm_dpo/delta": 0.1162538081407547, "fcm_dpo/margin": 265.3677062988281, "fcm_dpo/q_t": 0.4249209761619568, "grad_norm": 36.19147491455078, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -0.9529926776885986, "logits/rejected": -0.9747903347015381, "logps/chosen": -636.5621337890625, "logps/ref_chosen": -54.262962341308594, "logps/ref_rejected": -100.75428009033203, "logps/rejected": -948.4210815429688, "loss": 1.1681, "margin_dpo/margin_mean": 265.3677062988281, "margin_dpo/margin_std": 483.37457275390625, "step": 576 }, { "KL/chosen_KL_mean": -515.85498046875, "KL/mean": -682.475830078125, "KL/rejected_KL_mean": -849.0966186523438, "KL/std": 433.720947265625, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.001288012252189219, "fcm_dpo/delta": 0.021687885746359825, "fcm_dpo/margin": 333.24163818359375, "fcm_dpo/q_t": 0.40454742312431335, "grad_norm": 30.929237365722656, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -0.9174989461898804, "logits/rejected": -0.9000959396362305, "logps/chosen": -577.7646484375, "logps/ref_chosen": -61.909706115722656, "logps/ref_rejected": -84.07069396972656, "logps/rejected": -933.1673583984375, "loss": 1.0953, "margin_dpo/margin_mean": 333.24163818359375, "margin_dpo/margin_std": 466.4169006347656, "step": 577 }, { "KL/chosen_KL_mean": -510.0753173828125, "KL/mean": -700.24609375, "KL/rejected_KL_mean": -890.4168701171875, "KL/std": 436.525146484375, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.0012763895792886615, "fcm_dpo/delta": -0.038308389484882355, "fcm_dpo/margin": 380.341552734375, "fcm_dpo/q_t": 0.39006108045578003, "grad_norm": 34.35224914550781, "learning_rate": 3.478836705390808e-08, "logits/chosen": -0.8245141506195068, "logits/rejected": -0.8499029874801636, "logps/chosen": -559.3389892578125, "logps/ref_chosen": -49.26368713378906, "logps/ref_rejected": -83.4362564086914, "logps/rejected": -973.8531494140625, "loss": 1.0352, "margin_dpo/margin_mean": 380.341552734375, "margin_dpo/margin_std": 445.07977294921875, "step": 578 }, { "KL/chosen_KL_mean": -582.0475463867188, "KL/mean": -706.8804321289062, "KL/rejected_KL_mean": -831.7132568359375, "KL/std": 444.97283935546875, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.001303818542510271, "fcm_dpo/delta": 0.12885679304599762, "fcm_dpo/margin": 249.66574096679688, "fcm_dpo/q_t": 0.4256833791732788, "grad_norm": 54.624481201171875, "learning_rate": 3.41381639738331e-08, "logits/chosen": -0.8717827796936035, "logits/rejected": -0.8690820932388306, "logps/chosen": -640.933349609375, "logps/ref_chosen": -58.88581848144531, "logps/ref_rejected": -94.78762817382812, "logps/rejected": -926.5009155273438, "loss": 1.178, "margin_dpo/margin_mean": 249.66574096679688, "margin_dpo/margin_std": 475.2186279296875, "step": 579 }, { "KL/chosen_KL_mean": -424.3729248046875, "KL/mean": -635.8189697265625, "KL/rejected_KL_mean": -847.26513671875, "KL/std": 469.45257568359375, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0012895048130303621, "fcm_dpo/delta": -0.10174834728240967, "fcm_dpo/margin": 422.89215087890625, "fcm_dpo/q_t": 0.3806644380092621, "grad_norm": 30.684194564819336, "learning_rate": 3.349364905389032e-08, "logits/chosen": -0.8051278591156006, "logits/rejected": -0.8295614123344421, "logps/chosen": -473.07977294921875, "logps/ref_chosen": -48.70683670043945, "logps/ref_rejected": -81.7583999633789, "logps/rejected": -929.0235595703125, "loss": 1.0126, "margin_dpo/margin_mean": 422.89215087890625, "margin_dpo/margin_std": 529.8594360351562, "step": 580 }, { "KL/chosen_KL_mean": -568.9508056640625, "KL/mean": -720.18798828125, "KL/rejected_KL_mean": -871.4251708984375, "KL/std": 439.51373291015625, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0013018709141761065, "fcm_dpo/delta": 0.058401986956596375, "fcm_dpo/margin": 302.474365234375, "fcm_dpo/q_t": 0.41300997138023376, "grad_norm": 31.58887481689453, "learning_rate": 3.285483927764726e-08, "logits/chosen": -0.9429810047149658, "logits/rejected": -0.9472246170043945, "logps/chosen": -631.173095703125, "logps/ref_chosen": -62.22235107421875, "logps/ref_rejected": -91.73568725585938, "logps/rejected": -963.1608276367188, "loss": 1.1176, "margin_dpo/margin_mean": 302.474365234375, "margin_dpo/margin_std": 481.599853515625, "step": 581 }, { "KL/chosen_KL_mean": -494.30291748046875, "KL/mean": -672.12841796875, "KL/rejected_KL_mean": -849.9539184570312, "KL/std": 394.76776123046875, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.001313531189225614, "fcm_dpo/delta": -0.0199383944272995, "fcm_dpo/margin": 355.65106201171875, "fcm_dpo/q_t": 0.39279705286026, "grad_norm": 40.20054244995117, "learning_rate": 3.222175147833556e-08, "logits/chosen": -0.8778091669082642, "logits/rejected": -0.8947253227233887, "logps/chosen": -552.5315551757812, "logps/ref_chosen": -58.228660583496094, "logps/ref_rejected": -110.06959533691406, "logps/rejected": -960.0234985351562, "loss": 1.0566, "margin_dpo/margin_mean": 355.65106201171875, "margin_dpo/margin_std": 427.9756164550781, "step": 582 }, { "KL/chosen_KL_mean": -576.75927734375, "KL/mean": -685.605224609375, "KL/rejected_KL_mean": -794.4511108398438, "KL/std": 418.72412109375, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0012986322399228811, "fcm_dpo/delta": -0.0004855971201322973, "fcm_dpo/margin": 217.69180297851562, "fcm_dpo/q_t": 0.43750786781311035, "grad_norm": 56.99325180053711, "learning_rate": 3.159440233840763e-08, "logits/chosen": -0.8520580530166626, "logits/rejected": -0.8441060781478882, "logps/chosen": -633.6221923828125, "logps/ref_chosen": -56.86286163330078, "logps/ref_rejected": -88.4039306640625, "logps/rejected": -882.8550415039062, "loss": 1.2263, "margin_dpo/margin_mean": 217.69180297851562, "margin_dpo/margin_std": 497.66668701171875, "step": 583 }, { "KL/chosen_KL_mean": -477.69525146484375, "KL/mean": -696.723876953125, "KL/rejected_KL_mean": -915.7525634765625, "KL/std": 448.5584716796875, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0012766908621415496, "fcm_dpo/delta": -0.11572134494781494, "fcm_dpo/margin": 438.05731201171875, "fcm_dpo/q_t": 0.3751903176307678, "grad_norm": 29.102798461914062, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -0.8786138296127319, "logits/rejected": -0.8900209665298462, "logps/chosen": -534.595947265625, "logps/ref_chosen": -56.90068054199219, "logps/ref_rejected": -97.63606262207031, "logps/rejected": -1013.3885498046875, "loss": 0.9859, "margin_dpo/margin_mean": 438.05731201171875, "margin_dpo/margin_std": 475.702880859375, "step": 584 }, { "KL/chosen_KL_mean": -591.5244140625, "KL/mean": -777.2154541015625, "KL/rejected_KL_mean": -962.9066162109375, "KL/std": 488.06884765625, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0012623495422303677, "fcm_dpo/delta": -0.019886359572410583, "fcm_dpo/margin": 371.38226318359375, "fcm_dpo/q_t": 0.39668798446655273, "grad_norm": 32.74428176879883, "learning_rate": 3.035698600998121e-08, "logits/chosen": -0.936137318611145, "logits/rejected": -0.9553546905517578, "logps/chosen": -652.4983520507812, "logps/ref_chosen": -60.973968505859375, "logps/ref_rejected": -84.16952514648438, "logps/rejected": -1047.076171875, "loss": 1.0889, "margin_dpo/margin_mean": 371.38226318359375, "margin_dpo/margin_std": 573.6552734375, "step": 585 }, { "KL/chosen_KL_mean": -594.9310302734375, "KL/mean": -728.787841796875, "KL/rejected_KL_mean": -862.6447143554688, "KL/std": 421.7008056640625, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0012830430641770363, "fcm_dpo/delta": 0.11041760444641113, "fcm_dpo/margin": 267.71368408203125, "fcm_dpo/q_t": 0.4225107431411743, "grad_norm": 33.24190902709961, "learning_rate": 2.974695142855388e-08, "logits/chosen": -0.9146158695220947, "logits/rejected": -0.9288753271102905, "logps/chosen": -651.78662109375, "logps/ref_chosen": -56.85559844970703, "logps/ref_rejected": -91.80261993408203, "logps/rejected": -954.4473266601562, "loss": 1.1626, "margin_dpo/margin_mean": 267.71368408203125, "margin_dpo/margin_std": 484.7624816894531, "step": 586 }, { "KL/chosen_KL_mean": -430.05645751953125, "KL/mean": -608.1459350585938, "KL/rejected_KL_mean": -786.2353515625, "KL/std": 451.8103332519531, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0012925666524097323, "fcm_dpo/delta": -0.010944999754428864, "fcm_dpo/margin": 356.178955078125, "fcm_dpo/q_t": 0.3954969048500061, "grad_norm": 31.56755256652832, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -0.9272117614746094, "logits/rejected": -0.9497717022895813, "logps/chosen": -474.748046875, "logps/ref_chosen": -44.69159698486328, "logps/ref_rejected": -82.62385559082031, "logps/rejected": -868.8592529296875, "loss": 1.0577, "margin_dpo/margin_mean": 356.178955078125, "margin_dpo/margin_std": 455.32611083984375, "step": 587 }, { "KL/chosen_KL_mean": -548.4373779296875, "KL/mean": -706.7396240234375, "KL/rejected_KL_mean": -865.0418701171875, "KL/std": 395.88555908203125, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0012917927233502269, "fcm_dpo/delta": 0.04255141317844391, "fcm_dpo/margin": 316.6044616699219, "fcm_dpo/q_t": 0.40650418400764465, "grad_norm": 28.8896427154541, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -0.9216235876083374, "logits/rejected": -0.9430979490280151, "logps/chosen": -598.7322998046875, "logps/ref_chosen": -50.29494857788086, "logps/ref_rejected": -107.36988067626953, "logps/rejected": -972.4117431640625, "loss": 1.0905, "margin_dpo/margin_mean": 316.6044921875, "margin_dpo/margin_std": 424.40228271484375, "step": 588 }, { "KL/chosen_KL_mean": -545.638671875, "KL/mean": -745.33154296875, "KL/rejected_KL_mean": -945.0244140625, "KL/std": 435.88623046875, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0012890798971056938, "fcm_dpo/delta": -0.06830345839262009, "fcm_dpo/margin": 399.3857116699219, "fcm_dpo/q_t": 0.3833511471748352, "grad_norm": 31.347322463989258, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -0.8932241201400757, "logits/rejected": -0.8985201120376587, "logps/chosen": -605.568603515625, "logps/ref_chosen": -59.929908752441406, "logps/ref_rejected": -111.65534973144531, "logps/rejected": -1056.6796875, "loss": 1.0145, "margin_dpo/margin_mean": 399.3857421875, "margin_dpo/margin_std": 455.48895263671875, "step": 589 }, { "KL/chosen_KL_mean": -489.78375244140625, "KL/mean": -678.56396484375, "KL/rejected_KL_mean": -867.3441772460938, "KL/std": 415.4945983886719, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0012754870112985373, "fcm_dpo/delta": -0.03320199251174927, "fcm_dpo/margin": 377.56036376953125, "fcm_dpo/q_t": 0.3908342719078064, "grad_norm": 33.08230972290039, "learning_rate": 2.736501028272095e-08, "logits/chosen": -0.8838923573493958, "logits/rejected": -0.8997941017150879, "logps/chosen": -545.5935668945312, "logps/ref_chosen": -55.80979537963867, "logps/ref_rejected": -106.06282043457031, "logps/rejected": -973.406982421875, "loss": 1.0352, "margin_dpo/margin_mean": 377.5603332519531, "margin_dpo/margin_std": 445.62188720703125, "step": 590 }, { "KL/chosen_KL_mean": -525.9390869140625, "KL/mean": -702.4129638671875, "KL/rejected_KL_mean": -878.88671875, "KL/std": 408.47216796875, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.001272842986509204, "fcm_dpo/delta": 0.0007620565593242645, "fcm_dpo/margin": 352.9476623535156, "fcm_dpo/q_t": 0.39712613821029663, "grad_norm": 35.617069244384766, "learning_rate": 2.678415274939408e-08, "logits/chosen": -0.9034570455551147, "logits/rejected": -0.8874039053916931, "logps/chosen": -582.1796875, "logps/ref_chosen": -56.24061965942383, "logps/ref_rejected": -83.78629302978516, "logps/rejected": -962.673095703125, "loss": 1.0647, "margin_dpo/margin_mean": 352.9476623535156, "margin_dpo/margin_std": 454.56365966796875, "step": 591 }, { "KL/chosen_KL_mean": -567.338623046875, "KL/mean": -723.809814453125, "KL/rejected_KL_mean": -880.2809448242188, "KL/std": 425.2896423339844, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0012821652926504612, "fcm_dpo/delta": 0.050824809819459915, "fcm_dpo/margin": 312.9423522949219, "fcm_dpo/q_t": 0.41050198674201965, "grad_norm": 32.38739776611328, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -0.9035841226577759, "logits/rejected": -0.8997513055801392, "logps/chosen": -615.2788696289062, "logps/ref_chosen": -47.94025421142578, "logps/ref_rejected": -75.73287963867188, "logps/rejected": -956.0137939453125, "loss": 1.1585, "margin_dpo/margin_mean": 312.94232177734375, "margin_dpo/margin_std": 582.2163696289062, "step": 592 }, { "KL/chosen_KL_mean": -525.773681640625, "KL/mean": -653.7761840820312, "KL/rejected_KL_mean": -781.7787475585938, "KL/std": 448.0130310058594, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.0013066052924841642, "fcm_dpo/delta": 0.1196751743555069, "fcm_dpo/margin": 256.0050964355469, "fcm_dpo/q_t": 0.4265802502632141, "grad_norm": 46.937931060791016, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.8258799314498901, "logits/rejected": -0.8155261278152466, "logps/chosen": -574.4644165039062, "logps/ref_chosen": -48.690757751464844, "logps/ref_rejected": -60.90800094604492, "logps/rejected": -842.686767578125, "loss": 1.1792, "margin_dpo/margin_mean": 256.0050964355469, "margin_dpo/margin_std": 498.3818359375, "step": 593 }, { "KL/chosen_KL_mean": -498.7892761230469, "KL/mean": -662.3046264648438, "KL/rejected_KL_mean": -825.8199462890625, "KL/std": 448.3605651855469, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0013319381978362799, "fcm_dpo/delta": 0.013642996549606323, "fcm_dpo/margin": 327.03070068359375, "fcm_dpo/q_t": 0.4028928875923157, "grad_norm": 40.40144729614258, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -0.8832203149795532, "logits/rejected": -0.8690311312675476, "logps/chosen": -553.72412109375, "logps/ref_chosen": -54.93488693237305, "logps/ref_rejected": -86.09967803955078, "logps/rejected": -911.919677734375, "loss": 1.0955, "margin_dpo/margin_mean": 327.03070068359375, "margin_dpo/margin_std": 473.59136962890625, "step": 594 }, { "KL/chosen_KL_mean": -497.3454895019531, "KL/mean": -668.4541015625, "KL/rejected_KL_mean": -839.5626831054688, "KL/std": 416.5923156738281, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0013188087614253163, "fcm_dpo/delta": -0.001495562493801117, "fcm_dpo/margin": 342.2171630859375, "fcm_dpo/q_t": 0.40133988857269287, "grad_norm": 41.03437042236328, "learning_rate": 2.451969280180849e-08, "logits/chosen": -0.8576099872589111, "logits/rejected": -0.8629014492034912, "logps/chosen": -546.765869140625, "logps/ref_chosen": -49.4204216003418, "logps/ref_rejected": -80.62731170654297, "logps/rejected": -920.1900024414062, "loss": 1.0676, "margin_dpo/margin_mean": 342.2171936035156, "margin_dpo/margin_std": 466.0132751464844, "step": 595 }, { "KL/chosen_KL_mean": -563.5756225585938, "KL/mean": -703.8986206054688, "KL/rejected_KL_mean": -844.2215576171875, "KL/std": 430.84686279296875, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.001339010545052588, "fcm_dpo/delta": 0.07695081830024719, "fcm_dpo/margin": 280.64593505859375, "fcm_dpo/q_t": 0.4185601472854614, "grad_norm": 49.19902801513672, "learning_rate": 2.396839494982103e-08, "logits/chosen": -0.8751660585403442, "logits/rejected": -0.8460420370101929, "logps/chosen": -623.3673095703125, "logps/ref_chosen": -59.791683197021484, "logps/ref_rejected": -80.09111785888672, "logps/rejected": -924.3126831054688, "loss": 1.1472, "margin_dpo/margin_mean": 280.6459045410156, "margin_dpo/margin_std": 503.51751708984375, "step": 596 }, { "KL/chosen_KL_mean": -540.3389282226562, "KL/mean": -752.1220703125, "KL/rejected_KL_mean": -963.9052124023438, "KL/std": 479.4707336425781, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0013066258979961276, "fcm_dpo/delta": -0.11277859658002853, "fcm_dpo/margin": 423.5662536621094, "fcm_dpo/q_t": 0.3791978359222412, "grad_norm": 32.8747673034668, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -0.8611253499984741, "logits/rejected": -0.8989757299423218, "logps/chosen": -597.5997314453125, "logps/ref_chosen": -57.26078796386719, "logps/ref_rejected": -100.6937255859375, "logps/rejected": -1064.598876953125, "loss": 1.0194, "margin_dpo/margin_mean": 423.56622314453125, "margin_dpo/margin_std": 527.2634887695312, "step": 597 }, { "KL/chosen_KL_mean": -550.748046875, "KL/mean": -720.5945434570312, "KL/rejected_KL_mean": -890.441162109375, "KL/std": 418.1409912109375, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0013012320268899202, "fcm_dpo/delta": 0.0065561458468437195, "fcm_dpo/margin": 339.69317626953125, "fcm_dpo/q_t": 0.39901459217071533, "grad_norm": 52.765960693359375, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -0.9263842701911926, "logits/rejected": -0.9303134679794312, "logps/chosen": -603.2664794921875, "logps/ref_chosen": -52.51850509643555, "logps/ref_rejected": -89.44385528564453, "logps/rejected": -979.885009765625, "loss": 1.0824, "margin_dpo/margin_mean": 339.69317626953125, "margin_dpo/margin_std": 458.0146179199219, "step": 598 }, { "KL/chosen_KL_mean": -541.9983520507812, "KL/mean": -689.47216796875, "KL/rejected_KL_mean": -836.946044921875, "KL/std": 402.40704345703125, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0013150456361472607, "fcm_dpo/delta": 0.06361641734838486, "fcm_dpo/margin": 294.94769287109375, "fcm_dpo/q_t": 0.4082716703414917, "grad_norm": 37.7735595703125, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -0.8490918874740601, "logits/rejected": -0.8530697822570801, "logps/chosen": -591.801025390625, "logps/ref_chosen": -49.802677154541016, "logps/ref_rejected": -82.978515625, "logps/rejected": -919.924560546875, "loss": 1.1011, "margin_dpo/margin_mean": 294.94769287109375, "margin_dpo/margin_std": 372.10333251953125, "step": 599 }, { "KL/chosen_KL_mean": -582.5357666015625, "KL/mean": -719.8656616210938, "KL/rejected_KL_mean": -857.1956787109375, "KL/std": 403.26904296875, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.0013454308500513434, "fcm_dpo/delta": 0.08363349735736847, "fcm_dpo/margin": 274.6598815917969, "fcm_dpo/q_t": 0.417849063873291, "grad_norm": 35.837806701660156, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -0.9528838396072388, "logits/rejected": -0.9467679262161255, "logps/chosen": -648.9705810546875, "logps/ref_chosen": -66.43487548828125, "logps/ref_rejected": -85.45649719238281, "logps/rejected": -942.652099609375, "loss": 1.1447, "margin_dpo/margin_mean": 274.6598815917969, "margin_dpo/margin_std": 474.7832946777344, "step": 600 }, { "KL/chosen_KL_mean": -604.0224609375, "KL/mean": -778.8104858398438, "KL/rejected_KL_mean": -953.5985717773438, "KL/std": 454.6282958984375, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.001345046446658671, "fcm_dpo/delta": -0.021434810012578964, "fcm_dpo/margin": 349.5760803222656, "fcm_dpo/q_t": 0.39219212532043457, "grad_norm": 34.02439880371094, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -0.9269833564758301, "logits/rejected": -0.9384532570838928, "logps/chosen": -663.1560668945312, "logps/ref_chosen": -59.13361358642578, "logps/ref_rejected": -94.69093322753906, "logps/rejected": -1048.28955078125, "loss": 1.0524, "margin_dpo/margin_mean": 349.57611083984375, "margin_dpo/margin_std": 442.3649597167969, "step": 601 }, { "KL/chosen_KL_mean": -419.7701416015625, "KL/mean": -627.7533569335938, "KL/rejected_KL_mean": -835.736572265625, "KL/std": 411.73822021484375, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0013339366996660829, "fcm_dpo/delta": -0.11114930361509323, "fcm_dpo/margin": 415.96649169921875, "fcm_dpo/q_t": 0.3747555613517761, "grad_norm": 50.596588134765625, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -0.9215075969696045, "logits/rejected": -0.944821834564209, "logps/chosen": -468.3636474609375, "logps/ref_chosen": -48.59352111816406, "logps/ref_rejected": -87.6685562133789, "logps/rejected": -923.4051513671875, "loss": 0.9849, "margin_dpo/margin_mean": 415.96649169921875, "margin_dpo/margin_std": 435.55535888671875, "step": 602 }, { "KL/chosen_KL_mean": -496.24456787109375, "KL/mean": -692.5504150390625, "KL/rejected_KL_mean": -888.8563232421875, "KL/std": 452.21258544921875, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0013055261224508286, "fcm_dpo/delta": -0.0659245103597641, "fcm_dpo/margin": 392.6116943359375, "fcm_dpo/q_t": 0.3891829252243042, "grad_norm": 44.00050354003906, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -0.9159995317459106, "logits/rejected": -0.9068554639816284, "logps/chosen": -566.6591796875, "logps/ref_chosen": -70.41461944580078, "logps/ref_rejected": -100.32559967041016, "logps/rejected": -989.181884765625, "loss": 1.0493, "margin_dpo/margin_mean": 392.6116943359375, "margin_dpo/margin_std": 547.1536865234375, "step": 603 }, { "KL/chosen_KL_mean": -512.2185668945312, "KL/mean": -721.4580078125, "KL/rejected_KL_mean": -930.697509765625, "KL/std": 442.514404296875, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0012818844988942146, "fcm_dpo/delta": -0.09144748747348785, "fcm_dpo/margin": 418.4788818359375, "fcm_dpo/q_t": 0.38152605295181274, "grad_norm": 31.77489471435547, "learning_rate": 1.977362051376158e-08, "logits/chosen": -0.8875434398651123, "logits/rejected": -0.9096465110778809, "logps/chosen": -558.6766357421875, "logps/ref_chosen": -46.45808029174805, "logps/ref_rejected": -91.8544921875, "logps/rejected": -1022.552001953125, "loss": 1.021, "margin_dpo/margin_mean": 418.4788818359375, "margin_dpo/margin_std": 527.2413940429688, "step": 604 }, { "KL/chosen_KL_mean": -536.88916015625, "KL/mean": -706.5093994140625, "KL/rejected_KL_mean": -876.129638671875, "KL/std": 427.982666015625, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0012768646702170372, "fcm_dpo/delta": 0.017488297075033188, "fcm_dpo/margin": 339.24053955078125, "fcm_dpo/q_t": 0.404773086309433, "grad_norm": 37.503013610839844, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -0.8663319945335388, "logits/rejected": -0.8798903226852417, "logps/chosen": -603.1384887695312, "logps/ref_chosen": -66.24933624267578, "logps/ref_rejected": -102.30496978759766, "logps/rejected": -978.4346313476562, "loss": 1.0879, "margin_dpo/margin_mean": 339.24053955078125, "margin_dpo/margin_std": 500.1103515625, "step": 605 }, { "KL/chosen_KL_mean": -531.7333984375, "KL/mean": -710.1092529296875, "KL/rejected_KL_mean": -888.4852294921875, "KL/std": 433.4575500488281, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0012862420408055186, "fcm_dpo/delta": -0.009775873273611069, "fcm_dpo/margin": 356.7518005371094, "fcm_dpo/q_t": 0.39728018641471863, "grad_norm": 26.138517379760742, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -0.8830243349075317, "logits/rejected": -0.8961066603660583, "logps/chosen": -586.552490234375, "logps/ref_chosen": -54.819122314453125, "logps/ref_rejected": -98.37146759033203, "logps/rejected": -986.856689453125, "loss": 1.0611, "margin_dpo/margin_mean": 356.7518005371094, "margin_dpo/margin_std": 462.89691162109375, "step": 606 }, { "KL/chosen_KL_mean": -550.5501708984375, "KL/mean": -696.69384765625, "KL/rejected_KL_mean": -842.837646484375, "KL/std": 417.550537109375, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0012980522587895393, "fcm_dpo/delta": 0.07278678566217422, "fcm_dpo/margin": 292.28741455078125, "fcm_dpo/q_t": 0.4148247241973877, "grad_norm": 28.818361282348633, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -0.8506742715835571, "logits/rejected": -0.8361988067626953, "logps/chosen": -608.6342163085938, "logps/ref_chosen": -58.08403778076172, "logps/ref_rejected": -79.777099609375, "logps/rejected": -922.61474609375, "loss": 1.1233, "margin_dpo/margin_mean": 292.2874450683594, "margin_dpo/margin_std": 453.874755859375, "step": 607 }, { "KL/chosen_KL_mean": -506.8470153808594, "KL/mean": -678.7813720703125, "KL/rejected_KL_mean": -850.7156982421875, "KL/std": 392.28564453125, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0013015069998800755, "fcm_dpo/delta": 0.0024357233196496964, "fcm_dpo/margin": 343.86871337890625, "fcm_dpo/q_t": 0.39717209339141846, "grad_norm": 34.781368255615234, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -0.9160112142562866, "logits/rejected": -0.9214470386505127, "logps/chosen": -564.2978515625, "logps/ref_chosen": -57.450836181640625, "logps/ref_rejected": -94.77339172363281, "logps/rejected": -945.4890747070312, "loss": 1.0498, "margin_dpo/margin_mean": 343.8686828613281, "margin_dpo/margin_std": 397.05908203125, "step": 608 }, { "KL/chosen_KL_mean": -532.5485229492188, "KL/mean": -741.901611328125, "KL/rejected_KL_mean": -951.254638671875, "KL/std": 516.5100708007812, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0012833920773118734, "fcm_dpo/delta": -0.09222295880317688, "fcm_dpo/margin": 418.7061462402344, "fcm_dpo/q_t": 0.3859896659851074, "grad_norm": 31.00870704650879, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -0.9779306650161743, "logits/rejected": -0.9924079179763794, "logps/chosen": -591.3538818359375, "logps/ref_chosen": -58.805355072021484, "logps/ref_rejected": -88.81600952148438, "logps/rejected": -1040.0706787109375, "loss": 1.0368, "margin_dpo/margin_mean": 418.70611572265625, "margin_dpo/margin_std": 598.0675048828125, "step": 609 }, { "KL/chosen_KL_mean": -518.366455078125, "KL/mean": -659.4246826171875, "KL/rejected_KL_mean": -800.4828491210938, "KL/std": 397.7653503417969, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0012927600182592869, "fcm_dpo/delta": 0.08842732757329941, "fcm_dpo/margin": 282.1164245605469, "fcm_dpo/q_t": 0.4162673056125641, "grad_norm": 38.39983367919922, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -0.8724183440208435, "logits/rejected": -0.8451335430145264, "logps/chosen": -584.0615234375, "logps/ref_chosen": -65.69503784179688, "logps/ref_rejected": -83.40538787841797, "logps/rejected": -883.8882446289062, "loss": 1.1323, "margin_dpo/margin_mean": 282.11639404296875, "margin_dpo/margin_std": 448.6925048828125, "step": 610 }, { "KL/chosen_KL_mean": -544.4277954101562, "KL/mean": -774.05908203125, "KL/rejected_KL_mean": -1003.6903076171875, "KL/std": 526.8722534179688, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0012711257440969348, "fcm_dpo/delta": -0.1420023888349533, "fcm_dpo/margin": 459.2625732421875, "fcm_dpo/q_t": 0.3745759129524231, "grad_norm": 33.196842193603516, "learning_rate": 1.6421423736208e-08, "logits/chosen": -0.934251070022583, "logits/rejected": -0.9714481830596924, "logps/chosen": -597.0272216796875, "logps/ref_chosen": -52.59946823120117, "logps/ref_rejected": -86.33099365234375, "logps/rejected": -1090.021240234375, "loss": 1.0071, "margin_dpo/margin_mean": 459.2625732421875, "margin_dpo/margin_std": 592.6350708007812, "step": 611 }, { "KL/chosen_KL_mean": -576.2154541015625, "KL/mean": -766.717529296875, "KL/rejected_KL_mean": -957.2196655273438, "KL/std": 433.15924072265625, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0012588687241077423, "fcm_dpo/delta": -0.03120381385087967, "fcm_dpo/margin": 381.0041809082031, "fcm_dpo/q_t": 0.3909718096256256, "grad_norm": 33.79661178588867, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -0.9247915744781494, "logits/rejected": -0.9296138286590576, "logps/chosen": -635.5391845703125, "logps/ref_chosen": -59.32372283935547, "logps/ref_rejected": -88.31239318847656, "logps/rejected": -1045.531982421875, "loss": 1.042, "margin_dpo/margin_mean": 381.00421142578125, "margin_dpo/margin_std": 465.9805908203125, "step": 612 }, { "KL/chosen_KL_mean": -513.846923828125, "KL/mean": -734.3082275390625, "KL/rejected_KL_mean": -954.7694091796875, "KL/std": 482.4210205078125, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0012329479213804007, "fcm_dpo/delta": -0.09923183172941208, "fcm_dpo/margin": 440.9225158691406, "fcm_dpo/q_t": 0.37773290276527405, "grad_norm": 37.04204177856445, "learning_rate": 1.551886292185553e-08, "logits/chosen": -0.9263854026794434, "logits/rejected": -0.9770371317863464, "logps/chosen": -573.576904296875, "logps/ref_chosen": -59.72996520996094, "logps/ref_rejected": -105.10752868652344, "logps/rejected": -1059.876953125, "loss": 0.9984, "margin_dpo/margin_mean": 440.9224853515625, "margin_dpo/margin_std": 497.67266845703125, "step": 613 }, { "KL/chosen_KL_mean": -550.5446166992188, "KL/mean": -752.9097290039062, "KL/rejected_KL_mean": -955.2749633789062, "KL/std": 451.09417724609375, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0012219803174957633, "fcm_dpo/delta": -0.04686766862869263, "fcm_dpo/margin": 404.73028564453125, "fcm_dpo/q_t": 0.3896007239818573, "grad_norm": 47.749725341796875, "learning_rate": 1.507684480352292e-08, "logits/chosen": -0.8647533655166626, "logits/rejected": -0.9272447824478149, "logps/chosen": -603.483642578125, "logps/ref_chosen": -52.93898010253906, "logps/ref_rejected": -104.67938232421875, "logps/rejected": -1059.954345703125, "loss": 1.0469, "margin_dpo/margin_mean": 404.73028564453125, "margin_dpo/margin_std": 529.9136962890625, "step": 614 }, { "KL/chosen_KL_mean": -545.2410888671875, "KL/mean": -722.5862426757812, "KL/rejected_KL_mean": -899.931396484375, "KL/std": 499.9029541015625, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.001225535525009036, "fcm_dpo/delta": 0.015523616224527359, "fcm_dpo/margin": 354.69036865234375, "fcm_dpo/q_t": 0.40307652950286865, "grad_norm": 34.70551681518555, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -0.8659655451774597, "logits/rejected": -0.8805777430534363, "logps/chosen": -611.058349609375, "logps/ref_chosen": -65.81727600097656, "logps/ref_rejected": -95.17749786376953, "logps/rejected": -995.1089477539062, "loss": 1.0965, "margin_dpo/margin_mean": 354.69036865234375, "margin_dpo/margin_std": 541.21923828125, "step": 615 }, { "KL/chosen_KL_mean": -641.4904174804688, "KL/mean": -785.576416015625, "KL/rejected_KL_mean": -929.6624145507812, "KL/std": 414.92694091796875, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0012396399397403002, "fcm_dpo/delta": 0.09611323475837708, "fcm_dpo/margin": 288.1719970703125, "fcm_dpo/q_t": 0.4201861619949341, "grad_norm": 30.241304397583008, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -0.9229423999786377, "logits/rejected": -0.9040850400924683, "logps/chosen": -706.623291015625, "logps/ref_chosen": -65.13285827636719, "logps/ref_rejected": -74.70050048828125, "logps/rejected": -1004.3629150390625, "loss": 1.1428, "margin_dpo/margin_mean": 288.1719970703125, "margin_dpo/margin_std": 478.29656982421875, "step": 616 }, { "KL/chosen_KL_mean": -588.5724487304688, "KL/mean": -692.660400390625, "KL/rejected_KL_mean": -796.7484130859375, "KL/std": 374.8060302734375, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.001253496389836073, "fcm_dpo/delta": 0.07038042694330215, "fcm_dpo/margin": 208.1759490966797, "fcm_dpo/q_t": 0.4403090476989746, "grad_norm": 39.535221099853516, "learning_rate": 1.378797888467345e-08, "logits/chosen": -0.8760578632354736, "logits/rejected": -0.8447480201721191, "logps/chosen": -651.5780029296875, "logps/ref_chosen": -63.005550384521484, "logps/ref_rejected": -64.234130859375, "logps/rejected": -860.9824829101562, "loss": 1.2101, "margin_dpo/margin_mean": 208.17596435546875, "margin_dpo/margin_std": 424.15655517578125, "step": 617 }, { "KL/chosen_KL_mean": -610.806640625, "KL/mean": -831.383544921875, "KL/rejected_KL_mean": -1051.9605712890625, "KL/std": 508.52911376953125, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0012494558468461037, "fcm_dpo/delta": -0.10715337097644806, "fcm_dpo/margin": 441.1539611816406, "fcm_dpo/q_t": 0.38160985708236694, "grad_norm": 42.04853439331055, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -0.9283947348594666, "logits/rejected": -0.9563186168670654, "logps/chosen": -677.907958984375, "logps/ref_chosen": -67.10134887695312, "logps/ref_rejected": -92.15340423583984, "logps/rejected": -1144.114013671875, "loss": 1.0438, "margin_dpo/margin_mean": 441.15399169921875, "margin_dpo/margin_std": 628.577880859375, "step": 618 }, { "KL/chosen_KL_mean": -599.4898071289062, "KL/mean": -771.0576171875, "KL/rejected_KL_mean": -942.6253662109375, "KL/std": 476.91485595703125, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0012401568237692118, "fcm_dpo/delta": 0.02554713748395443, "fcm_dpo/margin": 343.1356201171875, "fcm_dpo/q_t": 0.411257266998291, "grad_norm": 43.26385498046875, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -0.911241352558136, "logits/rejected": -0.922271728515625, "logps/chosen": -655.468017578125, "logps/ref_chosen": -55.978233337402344, "logps/ref_rejected": -93.1854019165039, "logps/rejected": -1035.810791015625, "loss": 1.1354, "margin_dpo/margin_mean": 343.1356201171875, "margin_dpo/margin_std": 618.1133422851562, "step": 619 }, { "KL/chosen_KL_mean": -556.5227661132812, "KL/mean": -716.720703125, "KL/rejected_KL_mean": -876.918701171875, "KL/std": 417.01019287109375, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0012462441809475422, "fcm_dpo/delta": 0.05265815928578377, "fcm_dpo/margin": 320.3960266113281, "fcm_dpo/q_t": 0.4107811450958252, "grad_norm": 32.7905158996582, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -0.9634062051773071, "logits/rejected": -0.9648847579956055, "logps/chosen": -616.3202514648438, "logps/ref_chosen": -59.79750061035156, "logps/ref_rejected": -78.41075134277344, "logps/rejected": -955.3294677734375, "loss": 1.1067, "margin_dpo/margin_mean": 320.39599609375, "margin_dpo/margin_std": 471.8426513671875, "step": 620 }, { "KL/chosen_KL_mean": -537.1493530273438, "KL/mean": -776.6170654296875, "KL/rejected_KL_mean": -1016.0848388671875, "KL/std": 520.2274169921875, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0012356394436210394, "fcm_dpo/delta": -0.15062512457370758, "fcm_dpo/margin": 478.9354248046875, "fcm_dpo/q_t": 0.36890918016433716, "grad_norm": 55.19932174682617, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -0.9478855729103088, "logits/rejected": -0.9874153137207031, "logps/chosen": -591.0831298828125, "logps/ref_chosen": -53.93375778198242, "logps/ref_rejected": -88.36951446533203, "logps/rejected": -1104.454345703125, "loss": 0.9921, "margin_dpo/margin_mean": 478.93548583984375, "margin_dpo/margin_std": 571.7491455078125, "step": 621 }, { "KL/chosen_KL_mean": -530.1005249023438, "KL/mean": -701.41552734375, "KL/rejected_KL_mean": -872.7305908203125, "KL/std": 392.04901123046875, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0012181454803794622, "fcm_dpo/delta": 0.03374656289815903, "fcm_dpo/margin": 342.63006591796875, "fcm_dpo/q_t": 0.40700438618659973, "grad_norm": 32.37852096557617, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -0.8444244861602783, "logits/rejected": -0.8345458507537842, "logps/chosen": -590.3863525390625, "logps/ref_chosen": -60.28582000732422, "logps/ref_rejected": -85.51873779296875, "logps/rejected": -958.2493896484375, "loss": 1.0933, "margin_dpo/margin_mean": 342.6300354003906, "margin_dpo/margin_std": 495.0185546875, "step": 622 }, { "KL/chosen_KL_mean": -592.4083251953125, "KL/mean": -726.1845092773438, "KL/rejected_KL_mean": -859.960693359375, "KL/std": 408.55181884765625, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.001248663291335106, "fcm_dpo/delta": 0.11987321078777313, "fcm_dpo/margin": 267.5523681640625, "fcm_dpo/q_t": 0.4267767071723938, "grad_norm": 42.03673553466797, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -0.9228973388671875, "logits/rejected": -0.9195783138275146, "logps/chosen": -656.5652465820312, "logps/ref_chosen": -64.1569595336914, "logps/ref_rejected": -85.08304595947266, "logps/rejected": -945.043701171875, "loss": 1.1644, "margin_dpo/margin_mean": 267.5523681640625, "margin_dpo/margin_std": 492.16070556640625, "step": 623 }, { "KL/chosen_KL_mean": -580.66943359375, "KL/mean": -783.0391235351562, "KL/rejected_KL_mean": -985.4088134765625, "KL/std": 437.61358642578125, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.001246345229446888, "fcm_dpo/delta": -0.05730106681585312, "fcm_dpo/margin": 404.7393798828125, "fcm_dpo/q_t": 0.3833310604095459, "grad_norm": 38.60550308227539, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -0.9752143621444702, "logits/rejected": -0.9746447801589966, "logps/chosen": -652.588134765625, "logps/ref_chosen": -71.91862487792969, "logps/ref_rejected": -97.13203430175781, "logps/rejected": -1082.540771484375, "loss": 1.0216, "margin_dpo/margin_mean": 404.7393493652344, "margin_dpo/margin_std": 462.248779296875, "step": 624 }, { "KL/chosen_KL_mean": -558.2151489257812, "KL/mean": -771.4315185546875, "KL/rejected_KL_mean": -984.64794921875, "KL/std": 474.3367919921875, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0012284404365345836, "fcm_dpo/delta": -0.0778719037771225, "fcm_dpo/margin": 426.43280029296875, "fcm_dpo/q_t": 0.381242036819458, "grad_norm": 58.66389846801758, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -0.879206657409668, "logits/rejected": -0.8958455324172974, "logps/chosen": -616.5572509765625, "logps/ref_chosen": -58.342071533203125, "logps/ref_rejected": -86.09038543701172, "logps/rejected": -1070.73828125, "loss": 0.9944, "margin_dpo/margin_mean": 426.43280029296875, "margin_dpo/margin_std": 444.8097229003906, "step": 625 }, { "KL/chosen_KL_mean": -676.9705810546875, "KL/mean": -801.7283935546875, "KL/rejected_KL_mean": -926.4862060546875, "KL/std": 520.6180419921875, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.001250275643542409, "fcm_dpo/delta": 0.14224281907081604, "fcm_dpo/margin": 249.51565551757812, "fcm_dpo/q_t": 0.4286021590232849, "grad_norm": 42.33663558959961, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -0.8796349167823792, "logits/rejected": -0.8687018156051636, "logps/chosen": -752.0831298828125, "logps/ref_chosen": -75.11260986328125, "logps/ref_rejected": -99.188720703125, "logps/rejected": -1025.6749267578125, "loss": 1.1997, "margin_dpo/margin_mean": 249.51565551757812, "margin_dpo/margin_std": 512.447998046875, "step": 626 }, { "KL/chosen_KL_mean": -472.2640686035156, "KL/mean": -725.2449340820312, "KL/rejected_KL_mean": -978.2257080078125, "KL/std": 566.2703247070312, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0012319653760641813, "fcm_dpo/delta": -0.1845388561487198, "fcm_dpo/margin": 505.96173095703125, "fcm_dpo/q_t": 0.3714005947113037, "grad_norm": 25.161611557006836, "learning_rate": 9.897955805412e-09, "logits/chosen": -0.8306639194488525, "logits/rejected": -0.8944810032844543, "logps/chosen": -520.0072021484375, "logps/ref_chosen": -47.74314880371094, "logps/ref_rejected": -106.75448608398438, "logps/rejected": -1084.980224609375, "loss": 0.9902, "margin_dpo/margin_mean": 505.96173095703125, "margin_dpo/margin_std": 648.461181640625, "step": 627 }, { "KL/chosen_KL_mean": -601.7943115234375, "KL/mean": -791.7022705078125, "KL/rejected_KL_mean": -981.6102294921875, "KL/std": 455.6260986328125, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0012090300442650914, "fcm_dpo/delta": -0.009682614356279373, "fcm_dpo/margin": 379.8159484863281, "fcm_dpo/q_t": 0.3963756263256073, "grad_norm": 35.05025863647461, "learning_rate": 9.543589206795238e-09, "logits/chosen": -0.9712698459625244, "logits/rejected": -0.9801019430160522, "logps/chosen": -661.9771728515625, "logps/ref_chosen": -60.182945251464844, "logps/ref_rejected": -101.55467224121094, "logps/rejected": -1083.1649169921875, "loss": 1.0631, "margin_dpo/margin_mean": 379.81591796875, "margin_dpo/margin_std": 499.5301513671875, "step": 628 }, { "KL/chosen_KL_mean": -594.9044189453125, "KL/mean": -773.1024169921875, "KL/rejected_KL_mean": -951.3004150390625, "KL/std": 454.1546936035156, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.001212790608406067, "fcm_dpo/delta": 0.01857149973511696, "fcm_dpo/margin": 356.3959655761719, "fcm_dpo/q_t": 0.4004923105239868, "grad_norm": 38.04690170288086, "learning_rate": 9.19555885822887e-09, "logits/chosen": -0.9257842302322388, "logits/rejected": -0.9372028708457947, "logps/chosen": -659.117919921875, "logps/ref_chosen": -64.21354675292969, "logps/ref_rejected": -91.65367126464844, "logps/rejected": -1042.9541015625, "loss": 1.0681, "margin_dpo/margin_mean": 356.3959655761719, "margin_dpo/margin_std": 443.0477600097656, "step": 629 }, { "KL/chosen_KL_mean": -544.481689453125, "KL/mean": -651.665283203125, "KL/rejected_KL_mean": -758.8487548828125, "KL/std": 456.8427429199219, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0012220152420923114, "fcm_dpo/delta": 0.0643744021654129, "fcm_dpo/margin": 214.36705017089844, "fcm_dpo/q_t": 0.44493043422698975, "grad_norm": 52.316890716552734, "learning_rate": 8.85387393063622e-09, "logits/chosen": -0.9155005216598511, "logits/rejected": -0.889801561832428, "logps/chosen": -603.772705078125, "logps/ref_chosen": -59.29100036621094, "logps/ref_rejected": -83.59829711914062, "logps/rejected": -842.4470825195312, "loss": 1.2379, "margin_dpo/margin_mean": 214.36705017089844, "margin_dpo/margin_std": 532.4544677734375, "step": 630 }, { "KL/chosen_KL_mean": -656.3375244140625, "KL/mean": -813.904541015625, "KL/rejected_KL_mean": -971.4715576171875, "KL/std": 471.61029052734375, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0012422431027516723, "fcm_dpo/delta": 0.06083694100379944, "fcm_dpo/margin": 315.13397216796875, "fcm_dpo/q_t": 0.41228383779525757, "grad_norm": 37.45056915283203, "learning_rate": 8.518543427732949e-09, "logits/chosen": -0.9725683927536011, "logits/rejected": -0.9720879197120667, "logps/chosen": -715.7911376953125, "logps/ref_chosen": -59.45360565185547, "logps/ref_rejected": -80.95156860351562, "logps/rejected": -1052.423095703125, "loss": 1.1406, "margin_dpo/margin_mean": 315.1339416503906, "margin_dpo/margin_std": 551.551513671875, "step": 631 }, { "KL/chosen_KL_mean": -576.4053344726562, "KL/mean": -749.2000732421875, "KL/rejected_KL_mean": -921.9948120117188, "KL/std": 414.60711669921875, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0012430009664967656, "fcm_dpo/delta": 0.020996015518903732, "fcm_dpo/margin": 345.5894775390625, "fcm_dpo/q_t": 0.40269041061401367, "grad_norm": 38.403629302978516, "learning_rate": 8.189576185789637e-09, "logits/chosen": -0.9375253915786743, "logits/rejected": -0.9352479577064514, "logps/chosen": -637.7568969726562, "logps/ref_chosen": -61.35155487060547, "logps/ref_rejected": -86.16017150878906, "logps/rejected": -1008.1549682617188, "loss": 1.0947, "margin_dpo/margin_mean": 345.5894775390625, "margin_dpo/margin_std": 501.6351318359375, "step": 632 }, { "KL/chosen_KL_mean": -614.3057861328125, "KL/mean": -746.3451538085938, "KL/rejected_KL_mean": -878.384521484375, "KL/std": 419.60113525390625, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.001277608796954155, "fcm_dpo/delta": 0.11562201380729675, "fcm_dpo/margin": 264.0787353515625, "fcm_dpo/q_t": 0.4232080578804016, "grad_norm": 41.69476318359375, "learning_rate": 7.866980873399015e-09, "logits/chosen": -0.9602603912353516, "logits/rejected": -0.9682069420814514, "logps/chosen": -671.5839233398438, "logps/ref_chosen": -57.27816390991211, "logps/ref_rejected": -91.58395385742188, "logps/rejected": -969.968505859375, "loss": 1.1685, "margin_dpo/margin_mean": 264.0787353515625, "margin_dpo/margin_std": 484.4879150390625, "step": 633 }, { "KL/chosen_KL_mean": -703.184326171875, "KL/mean": -820.115478515625, "KL/rejected_KL_mean": -937.046630859375, "KL/std": 514.7780151367188, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0013053391594439745, "fcm_dpo/delta": 0.14952440559864044, "fcm_dpo/margin": 233.86233520507812, "fcm_dpo/q_t": 0.4344010651111603, "grad_norm": 45.4162712097168, "learning_rate": 7.550765991247654e-09, "logits/chosen": -0.852074146270752, "logits/rejected": -0.8421380519866943, "logps/chosen": -769.80322265625, "logps/ref_chosen": -66.61896514892578, "logps/ref_rejected": -107.12564849853516, "logps/rejected": -1044.1722412109375, "loss": 1.2035, "margin_dpo/margin_mean": 233.86233520507812, "margin_dpo/margin_std": 505.3544921875, "step": 634 }, { "KL/chosen_KL_mean": -600.64111328125, "KL/mean": -754.6363525390625, "KL/rejected_KL_mean": -908.6317138671875, "KL/std": 523.8653564453125, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0013240852858871222, "fcm_dpo/delta": 0.043988071382045746, "fcm_dpo/margin": 307.99053955078125, "fcm_dpo/q_t": 0.41226810216903687, "grad_norm": 45.625946044921875, "learning_rate": 7.240939871891699e-09, "logits/chosen": -0.9453951120376587, "logits/rejected": -0.9212517738342285, "logps/chosen": -674.5966796875, "logps/ref_chosen": -73.95551300048828, "logps/ref_rejected": -82.50045776367188, "logps/rejected": -991.1321411132812, "loss": 1.122, "margin_dpo/margin_mean": 307.99053955078125, "margin_dpo/margin_std": 517.03857421875, "step": 635 }, { "KL/chosen_KL_mean": -575.39306640625, "KL/mean": -755.2526245117188, "KL/rejected_KL_mean": -935.1121826171875, "KL/std": 504.8092346191406, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0013328897766768932, "fcm_dpo/delta": -0.03171185031533241, "fcm_dpo/margin": 359.71905517578125, "fcm_dpo/q_t": 0.3958790898323059, "grad_norm": 34.01151657104492, "learning_rate": 6.937510679537628e-09, "logits/chosen": -0.8697236776351929, "logits/rejected": -0.8689707517623901, "logps/chosen": -635.02197265625, "logps/ref_chosen": -59.628910064697266, "logps/ref_rejected": -81.97883605957031, "logps/rejected": -1017.0910034179688, "loss": 1.0553, "margin_dpo/margin_mean": 359.71905517578125, "margin_dpo/margin_std": 498.491455078125, "step": 636 }, { "KL/chosen_KL_mean": -581.6488037109375, "KL/mean": -774.9420166015625, "KL/rejected_KL_mean": -968.2352294921875, "KL/std": 499.24310302734375, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0013042533537372947, "fcm_dpo/delta": -0.05777234956622124, "fcm_dpo/margin": 386.58642578125, "fcm_dpo/q_t": 0.38885462284088135, "grad_norm": 30.27159881591797, "learning_rate": 6.640486409826785e-09, "logits/chosen": -0.9543923139572144, "logits/rejected": -0.989661455154419, "logps/chosen": -631.3014526367188, "logps/ref_chosen": -49.652687072753906, "logps/ref_rejected": -98.40513610839844, "logps/rejected": -1066.640380859375, "loss": 1.0433, "margin_dpo/margin_mean": 386.5863952636719, "margin_dpo/margin_std": 510.89727783203125, "step": 637 }, { "KL/chosen_KL_mean": -564.4012451171875, "KL/mean": -726.737548828125, "KL/rejected_KL_mean": -889.0739135742188, "KL/std": 475.2950439453125, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0013248416362330317, "fcm_dpo/delta": 0.017911911010742188, "fcm_dpo/margin": 324.6727294921875, "fcm_dpo/q_t": 0.39857015013694763, "grad_norm": 40.99110412597656, "learning_rate": 6.349874889624962e-09, "logits/chosen": -0.8504060506820679, "logits/rejected": -0.8326265811920166, "logps/chosen": -622.557861328125, "logps/ref_chosen": -58.156639099121094, "logps/ref_rejected": -79.3014907836914, "logps/rejected": -968.3754272460938, "loss": 1.1229, "margin_dpo/margin_mean": 324.6727294921875, "margin_dpo/margin_std": 536.3638916015625, "step": 638 }, { "KL/chosen_KL_mean": -756.171875, "KL/mean": -841.8418579101562, "KL/rejected_KL_mean": -927.5118408203125, "KL/std": 439.4996337890625, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0013258617836982012, "fcm_dpo/delta": 0.11202803254127502, "fcm_dpo/margin": 171.33995056152344, "fcm_dpo/q_t": 0.4497663080692291, "grad_norm": 92.11475372314453, "learning_rate": 6.065683776815933e-09, "logits/chosen": -0.8558847904205322, "logits/rejected": -0.8006620407104492, "logps/chosen": -828.4951171875, "logps/ref_chosen": -72.32319641113281, "logps/ref_rejected": -74.2749252319336, "logps/rejected": -1001.7867431640625, "loss": 1.2864, "margin_dpo/margin_mean": 171.3399658203125, "margin_dpo/margin_std": 525.029541015625, "step": 639 }, { "KL/chosen_KL_mean": -601.2142333984375, "KL/mean": -849.1571044921875, "KL/rejected_KL_mean": -1097.10009765625, "KL/std": 536.4822998046875, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0013078118208795786, "fcm_dpo/delta": -0.21178483963012695, "fcm_dpo/margin": 495.8858337402344, "fcm_dpo/q_t": 0.3637746274471283, "grad_norm": 36.71278381347656, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -0.8685541749000549, "logits/rejected": -0.8943710923194885, "logps/chosen": -657.3485717773438, "logps/ref_chosen": -56.13436508178711, "logps/ref_rejected": -108.60014343261719, "logps/rejected": -1205.7001953125, "loss": 0.9841, "margin_dpo/margin_mean": 495.8858337402344, "margin_dpo/margin_std": 631.3609008789062, "step": 640 }, { "KL/chosen_KL_mean": -672.4144287109375, "KL/mean": -824.10107421875, "KL/rejected_KL_mean": -975.7877197265625, "KL/std": 447.71893310546875, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0012949431547895074, "fcm_dpo/delta": 0.05952838063240051, "fcm_dpo/margin": 303.37322998046875, "fcm_dpo/q_t": 0.41488122940063477, "grad_norm": 38.896240234375, "learning_rate": 5.516592558795746e-09, "logits/chosen": -0.936628520488739, "logits/rejected": -0.9431087970733643, "logps/chosen": -737.411376953125, "logps/ref_chosen": -64.99689483642578, "logps/ref_rejected": -86.99232482910156, "logps/rejected": -1062.780029296875, "loss": 1.1532, "margin_dpo/margin_mean": 303.37322998046875, "margin_dpo/margin_std": 572.5907592773438, "step": 641 }, { "KL/chosen_KL_mean": -648.189453125, "KL/mean": -826.8723754882812, "KL/rejected_KL_mean": -1005.5552368164062, "KL/std": 566.3262939453125, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0013033505529165268, "fcm_dpo/delta": -0.01691451296210289, "fcm_dpo/margin": 357.3657531738281, "fcm_dpo/q_t": 0.40872862935066223, "grad_norm": 40.749515533447266, "learning_rate": 5.251706922648868e-09, "logits/chosen": -0.8487492203712463, "logits/rejected": -0.8746271133422852, "logps/chosen": -713.8787231445312, "logps/ref_chosen": -65.68924713134766, "logps/ref_rejected": -110.24205017089844, "logps/rejected": -1115.79736328125, "loss": 1.1299, "margin_dpo/margin_mean": 357.36578369140625, "margin_dpo/margin_std": 689.962646484375, "step": 642 }, { "KL/chosen_KL_mean": -570.0316162109375, "KL/mean": -730.2611694335938, "KL/rejected_KL_mean": -890.4908447265625, "KL/std": 434.6820068359375, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0012935586273670197, "fcm_dpo/delta": 0.036149993538856506, "fcm_dpo/margin": 320.4592590332031, "fcm_dpo/q_t": 0.4059324860572815, "grad_norm": 48.46284484863281, "learning_rate": 4.993270631642038e-09, "logits/chosen": -0.9710277318954468, "logits/rejected": -0.9704718589782715, "logps/chosen": -621.9815673828125, "logps/ref_chosen": -51.94999694824219, "logps/ref_rejected": -87.46833801269531, "logps/rejected": -977.9591674804688, "loss": 1.0894, "margin_dpo/margin_mean": 320.459228515625, "margin_dpo/margin_std": 425.56072998046875, "step": 643 }, { "KL/chosen_KL_mean": -573.2258911132812, "KL/mean": -717.8189697265625, "KL/rejected_KL_mean": -862.4119873046875, "KL/std": 502.07171630859375, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0013184216804802418, "fcm_dpo/delta": 0.07150032371282578, "fcm_dpo/margin": 289.1861572265625, "fcm_dpo/q_t": 0.4177606701850891, "grad_norm": 49.20467758178711, "learning_rate": 4.741290495811873e-09, "logits/chosen": -0.8860732316970825, "logits/rejected": -0.8877465128898621, "logps/chosen": -632.2435302734375, "logps/ref_chosen": -59.017662048339844, "logps/ref_rejected": -87.13668823242188, "logps/rejected": -949.5487060546875, "loss": 1.1697, "margin_dpo/margin_mean": 289.1861267089844, "margin_dpo/margin_std": 570.093505859375, "step": 644 }, { "KL/chosen_KL_mean": -601.1654052734375, "KL/mean": -690.848876953125, "KL/rejected_KL_mean": -780.5322265625, "KL/std": 411.4150390625, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.001336642075330019, "fcm_dpo/delta": 0.04631023108959198, "fcm_dpo/margin": 179.36688232421875, "fcm_dpo/q_t": 0.44901636242866516, "grad_norm": 76.54983520507812, "learning_rate": 4.495773155069299e-09, "logits/chosen": -0.8880874514579773, "logits/rejected": -0.8746832609176636, "logps/chosen": -657.0413818359375, "logps/ref_chosen": -55.87602233886719, "logps/ref_rejected": -97.78080749511719, "logps/rejected": -878.3131103515625, "loss": 1.2902, "margin_dpo/margin_mean": 179.36685180664062, "margin_dpo/margin_std": 539.417236328125, "step": 645 }, { "KL/chosen_KL_mean": -561.1621704101562, "KL/mean": -693.0960693359375, "KL/rejected_KL_mean": -825.0301513671875, "KL/std": 372.6351013183594, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0013508081901818514, "fcm_dpo/delta": 0.09708744287490845, "fcm_dpo/margin": 263.867919921875, "fcm_dpo/q_t": 0.41975581645965576, "grad_norm": 37.58830261230469, "learning_rate": 4.256725079024553e-09, "logits/chosen": -0.9159163236618042, "logits/rejected": -0.8985805511474609, "logps/chosen": -622.43798828125, "logps/ref_chosen": -61.275787353515625, "logps/ref_rejected": -77.50580596923828, "logps/rejected": -902.535888671875, "loss": 1.1401, "margin_dpo/margin_mean": 263.867919921875, "margin_dpo/margin_std": 423.1612548828125, "step": 646 }, { "KL/chosen_KL_mean": -518.5579833984375, "KL/mean": -685.169677734375, "KL/rejected_KL_mean": -851.7813720703125, "KL/std": 454.782470703125, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0013628093292936683, "fcm_dpo/delta": -0.004321117885410786, "fcm_dpo/margin": 333.2234802246094, "fcm_dpo/q_t": 0.3974537253379822, "grad_norm": 30.513471603393555, "learning_rate": 4.024152566816791e-09, "logits/chosen": -0.841652512550354, "logits/rejected": -0.865833044052124, "logps/chosen": -573.410400390625, "logps/ref_chosen": -54.8524169921875, "logps/ref_rejected": -93.5194091796875, "logps/rejected": -945.30078125, "loss": 1.0628, "margin_dpo/margin_mean": 333.2234802246094, "margin_dpo/margin_std": 430.2264404296875, "step": 647 }, { "KL/chosen_KL_mean": -529.4920654296875, "KL/mean": -776.0848388671875, "KL/rejected_KL_mean": -1022.6776733398438, "KL/std": 523.10302734375, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0013180954847484827, "fcm_dpo/delta": -0.21416297554969788, "fcm_dpo/margin": 493.1856689453125, "fcm_dpo/q_t": 0.3626644015312195, "grad_norm": 38.04413604736328, "learning_rate": 3.798061746947995e-09, "logits/chosen": -0.9507880210876465, "logits/rejected": -0.9994143843650818, "logps/chosen": -583.6634521484375, "logps/ref_chosen": -54.17146682739258, "logps/ref_rejected": -98.7127914428711, "logps/rejected": -1121.3905029296875, "loss": 0.9686, "margin_dpo/margin_mean": 493.1856689453125, "margin_dpo/margin_std": 615.1771240234375, "step": 648 }, { "KL/chosen_KL_mean": -556.886962890625, "KL/mean": -677.7119750976562, "KL/rejected_KL_mean": -798.5369873046875, "KL/std": 411.56622314453125, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0013347992207854986, "fcm_dpo/delta": 0.13102683424949646, "fcm_dpo/margin": 241.6501007080078, "fcm_dpo/q_t": 0.43245118856430054, "grad_norm": 34.881595611572266, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -0.9540548920631409, "logits/rejected": -0.9418928027153015, "logps/chosen": -619.3673095703125, "logps/ref_chosen": -62.480350494384766, "logps/ref_rejected": -80.07717895507812, "logps/rejected": -878.6141967773438, "loss": 1.1995, "margin_dpo/margin_mean": 241.65011596679688, "margin_dpo/margin_std": 513.494873046875, "step": 649 }, { "KL/chosen_KL_mean": -591.3101196289062, "KL/mean": -781.4625244140625, "KL/rejected_KL_mean": -971.6149291992188, "KL/std": 496.02606201171875, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.001323535107076168, "fcm_dpo/delta": -0.05659695714712143, "fcm_dpo/margin": 380.3048095703125, "fcm_dpo/q_t": 0.3918585777282715, "grad_norm": 32.53947830200195, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -0.9155662655830383, "logits/rejected": -0.9315616488456726, "logps/chosen": -647.4029541015625, "logps/ref_chosen": -56.09281921386719, "logps/ref_rejected": -98.26483917236328, "logps/rejected": -1069.8797607421875, "loss": 1.074, "margin_dpo/margin_mean": 380.3048095703125, "margin_dpo/margin_std": 578.767578125, "step": 650 }, { "KL/chosen_KL_mean": -417.0550842285156, "KL/mean": -646.5875244140625, "KL/rejected_KL_mean": -876.1199951171875, "KL/std": 477.76422119140625, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0012921846937388182, "fcm_dpo/delta": -0.1525677740573883, "fcm_dpo/margin": 459.06494140625, "fcm_dpo/q_t": 0.3671586513519287, "grad_norm": 39.78868865966797, "learning_rate": 3.158738163478475e-09, "logits/chosen": -0.957332968711853, "logits/rejected": -1.0047751665115356, "logps/chosen": -460.48052978515625, "logps/ref_chosen": -43.42544937133789, "logps/ref_rejected": -99.95791625976562, "logps/rejected": -976.077880859375, "loss": 0.9626, "margin_dpo/margin_mean": 459.06494140625, "margin_dpo/margin_std": 480.53533935546875, "step": 651 }, { "KL/chosen_KL_mean": -518.1369018554688, "KL/mean": -699.426513671875, "KL/rejected_KL_mean": -880.716064453125, "KL/std": 489.3262023925781, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0012830387568101287, "fcm_dpo/delta": -0.01596066914498806, "fcm_dpo/margin": 362.57916259765625, "fcm_dpo/q_t": 0.39959916472435, "grad_norm": 35.97466278076172, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -0.9103597402572632, "logits/rejected": -0.9288771152496338, "logps/chosen": -580.7137451171875, "logps/ref_chosen": -62.57680892944336, "logps/ref_rejected": -111.76779174804688, "logps/rejected": -992.48388671875, "loss": 1.0846, "margin_dpo/margin_mean": 362.57916259765625, "margin_dpo/margin_std": 556.9105834960938, "step": 652 }, { "KL/chosen_KL_mean": -623.66650390625, "KL/mean": -787.0184936523438, "KL/rejected_KL_mean": -950.3704833984375, "KL/std": 511.86151123046875, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0012876938562840223, "fcm_dpo/delta": 0.030522624030709267, "fcm_dpo/margin": 326.70391845703125, "fcm_dpo/q_t": 0.4086763262748718, "grad_norm": 36.10502243041992, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -0.9751501083374023, "logits/rejected": -0.9935754537582397, "logps/chosen": -684.779541015625, "logps/ref_chosen": -61.11295700073242, "logps/ref_rejected": -103.24960327148438, "logps/rejected": -1053.6201171875, "loss": 1.116, "margin_dpo/margin_mean": 326.7038879394531, "margin_dpo/margin_std": 535.5750122070312, "step": 653 }, { "KL/chosen_KL_mean": -577.2074584960938, "KL/mean": -730.6589965820312, "KL/rejected_KL_mean": -884.1104736328125, "KL/std": 413.8174133300781, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0012976478319615126, "fcm_dpo/delta": 0.05392075330018997, "fcm_dpo/margin": 306.90301513671875, "fcm_dpo/q_t": 0.4124031662940979, "grad_norm": 32.3806266784668, "learning_rate": 2.577954022936174e-09, "logits/chosen": -0.9655961394309998, "logits/rejected": -0.9741114377975464, "logps/chosen": -638.9356079101562, "logps/ref_chosen": -61.7281379699707, "logps/ref_rejected": -98.7738037109375, "logps/rejected": -982.88427734375, "loss": 1.1149, "margin_dpo/margin_mean": 306.90301513671875, "margin_dpo/margin_std": 477.42364501953125, "step": 654 }, { "KL/chosen_KL_mean": -528.3974609375, "KL/mean": -698.2972412109375, "KL/rejected_KL_mean": -868.197021484375, "KL/std": 432.225830078125, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.001303337630815804, "fcm_dpo/delta": 0.007457331754267216, "fcm_dpo/margin": 339.799560546875, "fcm_dpo/q_t": 0.4031388759613037, "grad_norm": 38.2427864074707, "learning_rate": 2.397392281198729e-09, "logits/chosen": -0.9318946599960327, "logits/rejected": -0.9627610445022583, "logps/chosen": -577.9742431640625, "logps/ref_chosen": -49.576812744140625, "logps/ref_rejected": -98.29183197021484, "logps/rejected": -966.4888305664062, "loss": 1.0911, "margin_dpo/margin_mean": 339.799560546875, "margin_dpo/margin_std": 511.25555419921875, "step": 655 }, { "KL/chosen_KL_mean": -561.493408203125, "KL/mean": -830.2171630859375, "KL/rejected_KL_mean": -1098.9407958984375, "KL/std": 532.9222412109375, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0012617453467100859, "fcm_dpo/delta": -0.24415498971939087, "fcm_dpo/margin": 537.447509765625, "fcm_dpo/q_t": 0.3515738248825073, "grad_norm": 71.94105529785156, "learning_rate": 2.223355098446622e-09, "logits/chosen": -0.852911651134491, "logits/rejected": -0.9044617414474487, "logps/chosen": -614.0428466796875, "logps/ref_chosen": -52.54943084716797, "logps/ref_rejected": -113.67464447021484, "logps/rejected": -1212.615478515625, "loss": 0.9165, "margin_dpo/margin_mean": 537.4474487304688, "margin_dpo/margin_std": 541.6453857421875, "step": 656 }, { "KL/chosen_KL_mean": -531.96435546875, "KL/mean": -757.275146484375, "KL/rejected_KL_mean": -982.5860595703125, "KL/std": 519.469482421875, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0012197259347885847, "fcm_dpo/delta": -0.10577631741762161, "fcm_dpo/margin": 450.6217041015625, "fcm_dpo/q_t": 0.3783363699913025, "grad_norm": 34.344337463378906, "learning_rate": 2.055847060721566e-09, "logits/chosen": -0.9864233136177063, "logits/rejected": -1.019301414489746, "logps/chosen": -578.6649169921875, "logps/ref_chosen": -46.700538635253906, "logps/ref_rejected": -97.91487121582031, "logps/rejected": -1080.5009765625, "loss": 1.0025, "margin_dpo/margin_mean": 450.6216735839844, "margin_dpo/margin_std": 536.0845947265625, "step": 657 }, { "KL/chosen_KL_mean": -574.1098022460938, "KL/mean": -751.432373046875, "KL/rejected_KL_mean": -928.7550048828125, "KL/std": 408.3322448730469, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0012142022605985403, "fcm_dpo/delta": 0.019960545003414154, "fcm_dpo/margin": 354.645263671875, "fcm_dpo/q_t": 0.4018761217594147, "grad_norm": 34.58537292480469, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -0.9124845266342163, "logits/rejected": -0.9352097511291504, "logps/chosen": -635.0679931640625, "logps/ref_chosen": -60.95820999145508, "logps/ref_rejected": -95.93949127197266, "logps/rejected": -1024.694580078125, "loss": 1.076, "margin_dpo/margin_mean": 354.645263671875, "margin_dpo/margin_std": 462.6511535644531, "step": 658 }, { "KL/chosen_KL_mean": -514.0313110351562, "KL/mean": -695.9007568359375, "KL/rejected_KL_mean": -877.7700805664062, "KL/std": 428.44195556640625, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0012178844772279263, "fcm_dpo/delta": 0.0071207694709300995, "fcm_dpo/margin": 363.7388000488281, "fcm_dpo/q_t": 0.4005226492881775, "grad_norm": 28.91624641418457, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -0.8975902795791626, "logits/rejected": -0.8781511783599854, "logps/chosen": -590.7742919921875, "logps/ref_chosen": -76.74298095703125, "logps/ref_rejected": -87.4709701538086, "logps/rejected": -965.2410278320312, "loss": 1.0658, "margin_dpo/margin_mean": 363.73876953125, "margin_dpo/margin_std": 468.4670104980469, "step": 659 }, { "KL/chosen_KL_mean": -553.8775024414062, "KL/mean": -770.415283203125, "KL/rejected_KL_mean": -986.953125, "KL/std": 480.31988525390625, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0012099661398679018, "fcm_dpo/delta": -0.07803238183259964, "fcm_dpo/margin": 433.0755920410156, "fcm_dpo/q_t": 0.3820180296897888, "grad_norm": 49.984649658203125, "learning_rate": 1.592541096695571e-09, "logits/chosen": -0.9479080438613892, "logits/rejected": -0.9651952981948853, "logps/chosen": -612.9254150390625, "logps/ref_chosen": -59.04788589477539, "logps/ref_rejected": -75.96005249023438, "logps/rejected": -1062.9130859375, "loss": 1.0167, "margin_dpo/margin_mean": 433.0755615234375, "margin_dpo/margin_std": 514.4667358398438, "step": 660 }, { "KL/chosen_KL_mean": -509.1634216308594, "KL/mean": -698.8631591796875, "KL/rejected_KL_mean": -888.5628662109375, "KL/std": 509.0642395019531, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0012084790505468845, "fcm_dpo/delta": -0.009469591081142426, "fcm_dpo/margin": 379.3994140625, "fcm_dpo/q_t": 0.3975834846496582, "grad_norm": 42.885379791259766, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -0.9597800970077515, "logits/rejected": -0.9635380506515503, "logps/chosen": -559.83740234375, "logps/ref_chosen": -50.673973083496094, "logps/ref_rejected": -86.00569152832031, "logps/rejected": -974.5685424804688, "loss": 1.0607, "margin_dpo/margin_mean": 379.3994140625, "margin_dpo/margin_std": 500.22235107421875, "step": 661 }, { "KL/chosen_KL_mean": -577.4758911132812, "KL/mean": -740.566650390625, "KL/rejected_KL_mean": -903.6572875976562, "KL/std": 462.82403564453125, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.001211101422086358, "fcm_dpo/delta": 0.05724027380347252, "fcm_dpo/margin": 326.181396484375, "fcm_dpo/q_t": 0.41278716921806335, "grad_norm": 32.96103286743164, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -0.8911588191986084, "logits/rejected": -0.887296736240387, "logps/chosen": -646.7369384765625, "logps/ref_chosen": -69.26106262207031, "logps/ref_rejected": -89.05593872070312, "logps/rejected": -992.7132568359375, "loss": 1.1349, "margin_dpo/margin_mean": 326.181396484375, "margin_dpo/margin_std": 560.8839721679688, "step": 662 }, { "KL/chosen_KL_mean": -556.5789794921875, "KL/mean": -736.2896118164062, "KL/rejected_KL_mean": -916.000244140625, "KL/std": 514.8147583007812, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.001216050237417221, "fcm_dpo/delta": 0.013502318412065506, "fcm_dpo/margin": 359.4212951660156, "fcm_dpo/q_t": 0.4045909643173218, "grad_norm": 31.383352279663086, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -0.9621092677116394, "logits/rejected": -0.9762744903564453, "logps/chosen": -621.4578857421875, "logps/ref_chosen": -64.87890625, "logps/ref_rejected": -113.92536926269531, "logps/rejected": -1029.9256591796875, "loss": 1.1097, "margin_dpo/margin_mean": 359.4212951660156, "margin_dpo/margin_std": 600.01806640625, "step": 663 }, { "KL/chosen_KL_mean": -570.5477294921875, "KL/mean": -781.1873779296875, "KL/rejected_KL_mean": -991.8270263671875, "KL/std": 502.1649169921875, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0012026217300444841, "fcm_dpo/delta": -0.060244910418987274, "fcm_dpo/margin": 421.27935791015625, "fcm_dpo/q_t": 0.38703733682632446, "grad_norm": 32.28097915649414, "learning_rate": 1.066455926241383e-09, "logits/chosen": -0.9074277877807617, "logits/rejected": -0.931912899017334, "logps/chosen": -631.4361572265625, "logps/ref_chosen": -60.88847351074219, "logps/ref_rejected": -105.521728515625, "logps/rejected": -1097.3487548828125, "loss": 1.035, "margin_dpo/margin_mean": 421.27935791015625, "margin_dpo/margin_std": 532.16455078125, "step": 664 }, { "KL/chosen_KL_mean": -507.58367919921875, "KL/mean": -684.880126953125, "KL/rejected_KL_mean": -862.1764526367188, "KL/std": 417.1563720703125, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0012078677536919713, "fcm_dpo/delta": 0.02266955003142357, "fcm_dpo/margin": 354.5927734375, "fcm_dpo/q_t": 0.4022970199584961, "grad_norm": 43.988059997558594, "learning_rate": 9.513254770636137e-10, "logits/chosen": -1.0135409832000732, "logits/rejected": -1.0313185453414917, "logps/chosen": -568.1478271484375, "logps/ref_chosen": -60.56413269042969, "logps/ref_rejected": -84.80882263183594, "logps/rejected": -946.9852905273438, "loss": 1.0642, "margin_dpo/margin_mean": 354.5927734375, "margin_dpo/margin_std": 423.1929626464844, "step": 665 }, { "KL/chosen_KL_mean": -559.8313598632812, "KL/mean": -742.6654052734375, "KL/rejected_KL_mean": -925.49951171875, "KL/std": 443.823486328125, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0012086930219084024, "fcm_dpo/delta": 0.008344907313585281, "fcm_dpo/margin": 365.66802978515625, "fcm_dpo/q_t": 0.4003828167915344, "grad_norm": 30.839902877807617, "learning_rate": 8.427576920763956e-10, "logits/chosen": -0.8351026773452759, "logits/rejected": -0.8394874334335327, "logps/chosen": -624.2513427734375, "logps/ref_chosen": -64.41996002197266, "logps/ref_rejected": -95.8916244506836, "logps/rejected": -1021.39111328125, "loss": 1.0731, "margin_dpo/margin_mean": 365.66802978515625, "margin_dpo/margin_std": 484.2020568847656, "step": 666 }, { "KL/chosen_KL_mean": -613.4276123046875, "KL/mean": -823.474365234375, "KL/rejected_KL_mean": -1033.5211181640625, "KL/std": 469.56622314453125, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.001200200291350484, "fcm_dpo/delta": -0.05712345987558365, "fcm_dpo/margin": 420.093505859375, "fcm_dpo/q_t": 0.38684695959091187, "grad_norm": 43.96973419189453, "learning_rate": 7.407554321417764e-10, "logits/chosen": -0.8881455659866333, "logits/rejected": -0.8855566382408142, "logps/chosen": -682.7046508789062, "logps/ref_chosen": -69.27702331542969, "logps/ref_rejected": -87.83549499511719, "logps/rejected": -1121.3565673828125, "loss": 1.0312, "margin_dpo/margin_mean": 420.093505859375, "margin_dpo/margin_std": 518.9374389648438, "step": 667 }, { "KL/chosen_KL_mean": -647.3582763671875, "KL/mean": -796.2598876953125, "KL/rejected_KL_mean": -945.1614990234375, "KL/std": 497.18328857421875, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.001220060046762228, "fcm_dpo/delta": 0.08885399252176285, "fcm_dpo/margin": 297.80328369140625, "fcm_dpo/q_t": 0.420962393283844, "grad_norm": 41.373592376708984, "learning_rate": 6.453213851142225e-10, "logits/chosen": -0.9342516660690308, "logits/rejected": -0.938195526599884, "logps/chosen": -719.9622802734375, "logps/ref_chosen": -72.60400390625, "logps/ref_rejected": -103.73905944824219, "logps/rejected": -1048.9005126953125, "loss": 1.1663, "margin_dpo/margin_mean": 297.80328369140625, "margin_dpo/margin_std": 567.736083984375, "step": 668 }, { "KL/chosen_KL_mean": -506.7996826171875, "KL/mean": -717.8054809570312, "KL/rejected_KL_mean": -928.811279296875, "KL/std": 482.9616394042969, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0012088122311979532, "fcm_dpo/delta": -0.06332193315029144, "fcm_dpo/margin": 422.0116271972656, "fcm_dpo/q_t": 0.3849877417087555, "grad_norm": 31.68146514892578, "learning_rate": 5.564580657695939e-10, "logits/chosen": -0.8937242031097412, "logits/rejected": -0.8956891894340515, "logps/chosen": -552.9160766601562, "logps/ref_chosen": -46.116416931152344, "logps/ref_rejected": -77.92434692382812, "logps/rejected": -1006.735595703125, "loss": 1.0352, "margin_dpo/margin_mean": 422.0115966796875, "margin_dpo/margin_std": 537.2428588867188, "step": 669 }, { "KL/chosen_KL_mean": -483.7650146484375, "KL/mean": -699.0269775390625, "KL/rejected_KL_mean": -914.2888793945312, "KL/std": 446.49737548828125, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0011966878082603216, "fcm_dpo/delta": -0.06883969902992249, "fcm_dpo/margin": 430.52386474609375, "fcm_dpo/q_t": 0.38446512818336487, "grad_norm": 30.194900512695312, "learning_rate": 4.741678157389739e-10, "logits/chosen": -0.8538155555725098, "logits/rejected": -0.8683494329452515, "logps/chosen": -546.1107788085938, "logps/ref_chosen": -62.34575271606445, "logps/ref_rejected": -96.9405517578125, "logps/rejected": -1011.2294311523438, "loss": 1.0326, "margin_dpo/margin_mean": 430.52386474609375, "margin_dpo/margin_std": 539.012451171875, "step": 670 }, { "KL/chosen_KL_mean": -600.821533203125, "KL/mean": -776.72021484375, "KL/rejected_KL_mean": -952.618896484375, "KL/std": 440.49432373046875, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0011965722078457475, "fcm_dpo/delta": 0.02998637594282627, "fcm_dpo/margin": 351.7973327636719, "fcm_dpo/q_t": 0.4032197594642639, "grad_norm": 28.47294807434082, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -0.939659059047699, "logits/rejected": -0.9617972373962402, "logps/chosen": -648.8216552734375, "logps/ref_chosen": -48.00010681152344, "logps/ref_rejected": -83.81932067871094, "logps/rejected": -1036.438232421875, "loss": 1.0956, "margin_dpo/margin_mean": 351.7973327636719, "margin_dpo/margin_std": 507.2611083984375, "step": 671 }, { "KL/chosen_KL_mean": -638.2493896484375, "KL/mean": -821.84765625, "KL/rejected_KL_mean": -1005.44580078125, "KL/std": 517.1207275390625, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0011946873273700476, "fcm_dpo/delta": 0.011807839386165142, "fcm_dpo/margin": 367.1964111328125, "fcm_dpo/q_t": 0.40388986468315125, "grad_norm": 41.209190368652344, "learning_rate": 3.293150240547549e-10, "logits/chosen": -0.9921166896820068, "logits/rejected": -0.9929635524749756, "logps/chosen": -696.8327026367188, "logps/ref_chosen": -58.58328628540039, "logps/ref_rejected": -93.14015197753906, "logps/rejected": -1098.5859375, "loss": 1.1038, "margin_dpo/margin_mean": 367.1964111328125, "margin_dpo/margin_std": 573.3291015625, "step": 672 }, { "KL/chosen_KL_mean": -594.8367919921875, "KL/mean": -771.89111328125, "KL/rejected_KL_mean": -948.9454345703125, "KL/std": 466.8351745605469, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0012018701527267694, "fcm_dpo/delta": 0.02548855170607567, "fcm_dpo/margin": 354.10858154296875, "fcm_dpo/q_t": 0.4053458571434021, "grad_norm": 41.65024948120117, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -0.9323441982269287, "logits/rejected": -0.9339570999145508, "logps/chosen": -641.5599365234375, "logps/ref_chosen": -46.72320556640625, "logps/ref_rejected": -85.29623413085938, "logps/rejected": -1034.24169921875, "loss": 1.0899, "margin_dpo/margin_mean": 354.10858154296875, "margin_dpo/margin_std": 522.732421875, "step": 673 }, { "KL/chosen_KL_mean": -497.871337890625, "KL/mean": -712.6817626953125, "KL/rejected_KL_mean": -927.4921875, "KL/std": 467.4238586425781, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0011941473931074142, "fcm_dpo/delta": -0.06639358401298523, "fcm_dpo/margin": 429.620849609375, "fcm_dpo/q_t": 0.3882029950618744, "grad_norm": 29.358726501464844, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -0.8644385933876038, "logits/rejected": -0.879269003868103, "logps/chosen": -543.31689453125, "logps/ref_chosen": -45.445526123046875, "logps/ref_rejected": -70.04593658447266, "logps/rejected": -997.5380859375, "loss": 1.0309, "margin_dpo/margin_mean": 429.620849609375, "margin_dpo/margin_std": 553.901611328125, "step": 674 }, { "KL/chosen_KL_mean": -564.3474731445312, "KL/mean": -789.6226196289062, "KL/rejected_KL_mean": -1014.897705078125, "KL/std": 509.0472412109375, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0011646319180727005, "fcm_dpo/delta": -0.08057983219623566, "fcm_dpo/margin": 450.55023193359375, "fcm_dpo/q_t": 0.3845682144165039, "grad_norm": 28.507211685180664, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -0.9137386083602905, "logits/rejected": -0.92195725440979, "logps/chosen": -608.5238037109375, "logps/ref_chosen": -44.17628479003906, "logps/ref_rejected": -74.09197998046875, "logps/rejected": -1088.98974609375, "loss": 1.0194, "margin_dpo/margin_mean": 450.55023193359375, "margin_dpo/margin_std": 533.7015380859375, "step": 675 }, { "KL/chosen_KL_mean": -601.19189453125, "KL/mean": -811.8079833984375, "KL/rejected_KL_mean": -1022.424072265625, "KL/std": 475.8211364746094, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0011619501747190952, "fcm_dpo/delta": -0.041463352739810944, "fcm_dpo/margin": 421.23211669921875, "fcm_dpo/q_t": 0.3885056972503662, "grad_norm": 27.30488395690918, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -0.8971314430236816, "logits/rejected": -0.9086474180221558, "logps/chosen": -672.5903930664062, "logps/ref_chosen": -71.39852905273438, "logps/ref_rejected": -88.3587646484375, "logps/rejected": -1110.7828369140625, "loss": 1.0226, "margin_dpo/margin_mean": 421.23211669921875, "margin_dpo/margin_std": 460.779296875, "step": 676 }, { "KL/chosen_KL_mean": -617.59375, "KL/mean": -808.9059448242188, "KL/rejected_KL_mean": -1000.2181396484375, "KL/std": 447.6944580078125, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0011601599399000406, "fcm_dpo/delta": 0.006374634802341461, "fcm_dpo/margin": 382.6244201660156, "fcm_dpo/q_t": 0.40171724557876587, "grad_norm": 29.162525177001953, "learning_rate": 8.23423165278725e-11, "logits/chosen": -0.965959906578064, "logits/rejected": -0.9512898325920105, "logps/chosen": -674.1211547851562, "logps/ref_chosen": -56.527435302734375, "logps/ref_rejected": -78.22654724121094, "logps/rejected": -1078.4447021484375, "loss": 1.0718, "margin_dpo/margin_mean": 382.6243896484375, "margin_dpo/margin_std": 521.0079345703125, "step": 677 }, { "KL/chosen_KL_mean": -509.26019287109375, "KL/mean": -732.9322509765625, "KL/rejected_KL_mean": -956.6041259765625, "KL/std": 503.4005126953125, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0011527533642947674, "fcm_dpo/delta": -0.06928122043609619, "fcm_dpo/margin": 447.3440246582031, "fcm_dpo/q_t": 0.3845905065536499, "grad_norm": 39.06260299682617, "learning_rate": 5.270012410216185e-11, "logits/chosen": -0.9112190008163452, "logits/rejected": -0.9373574256896973, "logps/chosen": -555.3946533203125, "logps/ref_chosen": -46.13447570800781, "logps/ref_rejected": -80.60462951660156, "logps/rejected": -1037.208740234375, "loss": 1.0295, "margin_dpo/margin_mean": 447.343994140625, "margin_dpo/margin_std": 562.4586181640625, "step": 678 }, { "KL/chosen_KL_mean": -587.34375, "KL/mean": -751.5213012695312, "KL/rejected_KL_mean": -915.6988525390625, "KL/std": 422.2051086425781, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0011560373241081834, "fcm_dpo/delta": 0.0732286274433136, "fcm_dpo/margin": 328.3551025390625, "fcm_dpo/q_t": 0.4141232371330261, "grad_norm": 31.085031509399414, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -0.9332907199859619, "logits/rejected": -0.9196882843971252, "logps/chosen": -637.638671875, "logps/ref_chosen": -50.294921875, "logps/ref_rejected": -76.59813690185547, "logps/rejected": -992.2969970703125, "loss": 1.1126, "margin_dpo/margin_mean": 328.3551025390625, "margin_dpo/margin_std": 476.9893798828125, "step": 679 }, { "KL/chosen_KL_mean": -586.6705322265625, "KL/mean": -796.8468017578125, "KL/rejected_KL_mean": -1007.023193359375, "KL/std": 541.9744873046875, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.001149947289377451, "fcm_dpo/delta": -0.035634323954582214, "fcm_dpo/margin": 420.35260009765625, "fcm_dpo/q_t": 0.3903007507324219, "grad_norm": 38.45374298095703, "learning_rate": 1.31753782067201e-11, "logits/chosen": -0.9182882905006409, "logits/rejected": -0.9428098201751709, "logps/chosen": -663.5863037109375, "logps/ref_chosen": -76.91569519042969, "logps/ref_rejected": -112.384765625, "logps/rejected": -1119.407958984375, "loss": 1.0688, "margin_dpo/margin_mean": 420.35260009765625, "margin_dpo/margin_std": 602.3566284179688, "step": 680 }, { "KL/chosen_KL_mean": -585.1146240234375, "KL/mean": -759.0559692382812, "KL/rejected_KL_mean": -932.997314453125, "KL/std": 455.07745361328125, "epoch": 1.0, "fcm_dpo/beta": 0.0011691637337207794, "fcm_dpo/delta": 0.043633054941892624, "fcm_dpo/margin": 347.8826904296875, "fcm_dpo/q_t": 0.4084250032901764, "grad_norm": 33.54962158203125, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -0.9781302213668823, "logits/rejected": -0.9905676245689392, "logps/chosen": -646.0718994140625, "logps/ref_chosen": -60.957279205322266, "logps/ref_rejected": -88.55797576904297, "logps/rejected": -1021.5552978515625, "loss": 1.1007, "margin_dpo/margin_mean": 347.8826904296875, "margin_dpo/margin_std": 481.5751953125, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0758972272998961, "train_runtime": 1729.0144, "train_samples_per_second": 25.216, "train_steps_per_second": 0.394 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }