{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.00527191162109375, "KL/mean": 0.016706019639968872, "KL/rejected_KL_mean": 0.028141021728515625, "KL/std": 0.272699236869812, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02287006378173828, "fcm_dpo/q_t": 0.5005706548690796, "grad_norm": 83.51212310791016, "learning_rate": 0.0, "logits/chosen": -0.4974287748336792, "logits/rejected": -0.43299180269241333, "logps/chosen": -50.1435661315918, "logps/ref_chosen": -50.14883804321289, "logps/ref_rejected": -74.1280517578125, "logps/rejected": -74.09991455078125, "loss": 1.389, "margin_dpo/margin_mean": -0.02287048101425171, "margin_dpo/margin_std": 0.41920793056488037, "step": 1 }, { "KL/chosen_KL_mean": -0.03498649597167969, "KL/mean": -0.00212840735912323, "KL/rejected_KL_mean": 0.030735015869140625, "KL/std": 0.24797174334526062, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06572261452674866, "fcm_dpo/q_t": 0.5016425848007202, "grad_norm": 72.20186614990234, "learning_rate": 7.246376811594203e-09, "logits/chosen": -0.49536412954330444, "logits/rejected": -0.4594460427761078, "logps/chosen": -52.65568923950195, "logps/ref_chosen": -52.620704650878906, "logps/ref_rejected": -75.30413818359375, "logps/rejected": -75.27340698242188, "loss": 1.3932, "margin_dpo/margin_mean": -0.06572240591049194, "margin_dpo/margin_std": 0.35048407316207886, "step": 2 }, { "KL/chosen_KL_mean": -0.00826263427734375, "KL/mean": 0.004124656319618225, "KL/rejected_KL_mean": 0.016513824462890625, "KL/std": 0.2530524432659149, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02477547526359558, "fcm_dpo/q_t": 0.5006192922592163, "grad_norm": 70.89527130126953, "learning_rate": 1.4492753623188406e-08, "logits/chosen": -0.4815559387207031, "logits/rejected": -0.4421369135379791, "logps/chosen": -60.98986053466797, "logps/ref_chosen": -60.981597900390625, "logps/ref_rejected": -68.67259216308594, "logps/rejected": -68.65608215332031, "loss": 1.3891, "margin_dpo/margin_mean": -0.02477523684501648, "margin_dpo/margin_std": 0.3734402656555176, "step": 3 }, { "KL/chosen_KL_mean": -0.02429962158203125, "KL/mean": 0.015571162104606628, "KL/rejected_KL_mean": 0.05544281005859375, "KL/std": 0.252570241689682, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07974502444267273, "fcm_dpo/q_t": 0.50199294090271, "grad_norm": 72.09413146972656, "learning_rate": 2.1739130434782606e-08, "logits/chosen": -0.4686494469642639, "logits/rejected": -0.44108062982559204, "logps/chosen": -56.79201126098633, "logps/ref_chosen": -56.7677116394043, "logps/ref_rejected": -86.64710998535156, "logps/rejected": -86.59166717529297, "loss": 1.3946, "margin_dpo/margin_mean": -0.079745352268219, "margin_dpo/margin_std": 0.35485363006591797, "step": 4 }, { "KL/chosen_KL_mean": 0.0070819854736328125, "KL/mean": 0.02031552791595459, "KL/rejected_KL_mean": 0.0335540771484375, "KL/std": 0.24011048674583435, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0264662504196167, "fcm_dpo/q_t": 0.5006614923477173, "grad_norm": 89.8433609008789, "learning_rate": 2.898550724637681e-08, "logits/chosen": -0.5143770575523376, "logits/rejected": -0.47061315178871155, "logps/chosen": -53.852294921875, "logps/ref_chosen": -53.859375, "logps/ref_rejected": -84.14918518066406, "logps/rejected": -84.11563110351562, "loss": 1.3892, "margin_dpo/margin_mean": -0.026465564966201782, "margin_dpo/margin_std": 0.3306843340396881, "step": 5 }, { "KL/chosen_KL_mean": -0.03646278381347656, "KL/mean": -0.02894607186317444, "KL/rejected_KL_mean": -0.021427154541015625, "KL/std": 0.2451232373714447, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.015035033226013184, "fcm_dpo/q_t": 0.5003755688667297, "grad_norm": 92.09574127197266, "learning_rate": 3.6231884057971014e-08, "logits/chosen": -0.522139310836792, "logits/rejected": -0.4815322160720825, "logps/chosen": -63.0439453125, "logps/ref_chosen": -63.007484436035156, "logps/ref_rejected": -92.64534759521484, "logps/rejected": -92.66677856445312, "loss": 1.3881, "margin_dpo/margin_mean": -0.015034288167953491, "margin_dpo/margin_std": 0.3625510036945343, "step": 6 }, { "KL/chosen_KL_mean": 0.0068035125732421875, "KL/mean": 0.020816445350646973, "KL/rejected_KL_mean": 0.03482818603515625, "KL/std": 0.2857266068458557, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.028023719787597656, "fcm_dpo/q_t": 0.5007004141807556, "grad_norm": 82.25199890136719, "learning_rate": 4.347826086956521e-08, "logits/chosen": -0.5038982629776001, "logits/rejected": -0.4701980650424957, "logps/chosen": -57.76801681518555, "logps/ref_chosen": -57.774818420410156, "logps/ref_rejected": -103.92059326171875, "logps/rejected": -103.8857650756836, "loss": 1.3896, "margin_dpo/margin_mean": -0.02802315354347229, "margin_dpo/margin_std": 0.42090797424316406, "step": 7 }, { "KL/chosen_KL_mean": 0.03360557556152344, "KL/mean": 0.027956083416938782, "KL/rejected_KL_mean": 0.022308349609375, "KL/std": 0.2712041437625885, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011296361684799194, "fcm_dpo/q_t": 0.4997172951698303, "grad_norm": 78.32466888427734, "learning_rate": 5.0724637681159424e-08, "logits/chosen": -0.5075632333755493, "logits/rejected": -0.4815378189086914, "logps/chosen": -58.682430267333984, "logps/ref_chosen": -58.716033935546875, "logps/ref_rejected": -79.3114242553711, "logps/rejected": -79.28911590576172, "loss": 1.3855, "margin_dpo/margin_mean": 0.01129680871963501, "margin_dpo/margin_std": 0.3612467646598816, "step": 8 }, { "KL/chosen_KL_mean": -0.0057525634765625, "KL/mean": -0.018807604908943176, "KL/rejected_KL_mean": -0.031864166259765625, "KL/std": 0.26702290773391724, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.026109427213668823, "fcm_dpo/q_t": 0.499347448348999, "grad_norm": 84.89402770996094, "learning_rate": 5.797101449275362e-08, "logits/chosen": -0.5175442099571228, "logits/rejected": -0.4723869264125824, "logps/chosen": -69.87259674072266, "logps/ref_chosen": -69.8668441772461, "logps/ref_rejected": -99.6026611328125, "logps/rejected": -99.63452911376953, "loss": 1.384, "margin_dpo/margin_mean": 0.02610918879508972, "margin_dpo/margin_std": 0.359569787979126, "step": 9 }, { "KL/chosen_KL_mean": 0.0016269683837890625, "KL/mean": 0.005393713712692261, "KL/rejected_KL_mean": 0.00916290283203125, "KL/std": 0.2506978213787079, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.007536411285400391, "fcm_dpo/q_t": 0.5001886487007141, "grad_norm": 70.9793930053711, "learning_rate": 6.521739130434782e-08, "logits/chosen": -0.4903263449668884, "logits/rejected": -0.446901798248291, "logps/chosen": -48.356056213378906, "logps/ref_chosen": -48.35768508911133, "logps/ref_rejected": -80.37206268310547, "logps/rejected": -80.36289978027344, "loss": 1.3874, "margin_dpo/margin_mean": -0.007535874843597412, "margin_dpo/margin_std": 0.35438239574432373, "step": 10 }, { "KL/chosen_KL_mean": 0.045078277587890625, "KL/mean": 0.014331206679344177, "KL/rejected_KL_mean": -0.016414642333984375, "KL/std": 0.236942321062088, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06148645281791687, "fcm_dpo/q_t": 0.49846333265304565, "grad_norm": 68.05949401855469, "learning_rate": 7.246376811594203e-08, "logits/chosen": -0.45714545249938965, "logits/rejected": -0.43192940950393677, "logps/chosen": -52.971778869628906, "logps/ref_chosen": -53.01685333251953, "logps/ref_rejected": -87.78038024902344, "logps/rejected": -87.79679107666016, "loss": 1.3804, "margin_dpo/margin_mean": 0.06148698925971985, "margin_dpo/margin_std": 0.33037588000297546, "step": 11 }, { "KL/chosen_KL_mean": -0.022512435913085938, "KL/mean": -0.03226548433303833, "KL/rejected_KL_mean": -0.04201507568359375, "KL/std": 0.2767062187194824, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.019503533840179443, "fcm_dpo/q_t": 0.4995127022266388, "grad_norm": 96.18541717529297, "learning_rate": 7.971014492753623e-08, "logits/chosen": -0.5195867419242859, "logits/rejected": -0.4828483462333679, "logps/chosen": -61.82794952392578, "logps/ref_chosen": -61.80543518066406, "logps/ref_rejected": -104.8582763671875, "logps/rejected": -104.90028381347656, "loss": 1.3846, "margin_dpo/margin_mean": 0.019502699375152588, "margin_dpo/margin_std": 0.3449036180973053, "step": 12 }, { "KL/chosen_KL_mean": -0.00475311279296875, "KL/mean": -0.012506544589996338, "KL/rejected_KL_mean": -0.020259857177734375, "KL/std": 0.26460498571395874, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.015512943267822266, "fcm_dpo/q_t": 0.49961256980895996, "grad_norm": 79.2775650024414, "learning_rate": 8.695652173913042e-08, "logits/chosen": -0.48537132143974304, "logits/rejected": -0.45758724212646484, "logps/chosen": -64.26510620117188, "logps/ref_chosen": -64.2603530883789, "logps/ref_rejected": -87.20307922363281, "logps/rejected": -87.22334289550781, "loss": 1.3851, "margin_dpo/margin_mean": 0.015512913465499878, "margin_dpo/margin_std": 0.339786171913147, "step": 13 }, { "KL/chosen_KL_mean": -0.0053424835205078125, "KL/mean": -0.026964426040649414, "KL/rejected_KL_mean": -0.04859161376953125, "KL/std": 0.2796820104122162, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04325270652770996, "fcm_dpo/q_t": 0.49891969561576843, "grad_norm": 85.09868621826172, "learning_rate": 9.420289855072464e-08, "logits/chosen": -0.49180495738983154, "logits/rejected": -0.45296943187713623, "logps/chosen": -58.11555480957031, "logps/ref_chosen": -58.11021041870117, "logps/ref_rejected": -104.04708099365234, "logps/rejected": -104.09567260742188, "loss": 1.3825, "margin_dpo/margin_mean": 0.043252259492874146, "margin_dpo/margin_std": 0.44487419724464417, "step": 14 }, { "KL/chosen_KL_mean": -0.014326095581054688, "KL/mean": -0.034481316804885864, "KL/rejected_KL_mean": -0.054637908935546875, "KL/std": 0.24156004190444946, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04031383991241455, "fcm_dpo/q_t": 0.4989916980266571, "grad_norm": 64.1568374633789, "learning_rate": 1.0144927536231885e-07, "logits/chosen": -0.5022023916244507, "logits/rejected": -0.4839800298213959, "logps/chosen": -56.98123550415039, "logps/ref_chosen": -56.96691131591797, "logps/ref_rejected": -80.80863952636719, "logps/rejected": -80.86328125, "loss": 1.3826, "margin_dpo/margin_mean": 0.04031360149383545, "margin_dpo/margin_std": 0.3656350076198578, "step": 15 }, { "KL/chosen_KL_mean": 0.029695510864257812, "KL/mean": -0.008761048316955566, "KL/rejected_KL_mean": -0.047214508056640625, "KL/std": 0.2751133441925049, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07690972089767456, "fcm_dpo/q_t": 0.49807754158973694, "grad_norm": 83.51917266845703, "learning_rate": 1.0869565217391303e-07, "logits/chosen": -0.5214604139328003, "logits/rejected": -0.47995007038116455, "logps/chosen": -61.71019744873047, "logps/ref_chosen": -61.739891052246094, "logps/ref_rejected": -84.36947631835938, "logps/rejected": -84.41668701171875, "loss": 1.3791, "margin_dpo/margin_mean": 0.07690989971160889, "margin_dpo/margin_std": 0.42422112822532654, "step": 16 }, { "KL/chosen_KL_mean": 0.06725311279296875, "KL/mean": 0.007791772484779358, "KL/rejected_KL_mean": -0.051666259765625, "KL/std": 0.29778245091438293, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1189166009426117, "fcm_dpo/q_t": 0.4970288872718811, "grad_norm": 78.13703155517578, "learning_rate": 1.1594202898550725e-07, "logits/chosen": -0.5090387463569641, "logits/rejected": -0.4729403853416443, "logps/chosen": -67.64308166503906, "logps/ref_chosen": -67.71033477783203, "logps/ref_rejected": -85.37865447998047, "logps/rejected": -85.4303207397461, "loss": 1.3749, "margin_dpo/margin_mean": 0.11891645193099976, "margin_dpo/margin_std": 0.415335476398468, "step": 17 }, { "KL/chosen_KL_mean": -0.0026035308837890625, "KL/mean": -0.012814626097679138, "KL/rejected_KL_mean": -0.0230255126953125, "KL/std": 0.2600899636745453, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.020421653985977173, "fcm_dpo/q_t": 0.4994891285896301, "grad_norm": 82.34599304199219, "learning_rate": 1.2318840579710146e-07, "logits/chosen": -0.48533254861831665, "logits/rejected": -0.4290851950645447, "logps/chosen": -47.742095947265625, "logps/ref_chosen": -47.7394905090332, "logps/ref_rejected": -75.4722900390625, "logps/rejected": -75.49531555175781, "loss": 1.3846, "margin_dpo/margin_mean": 0.020421594381332397, "margin_dpo/margin_std": 0.32879841327667236, "step": 18 }, { "KL/chosen_KL_mean": 0.024383544921875, "KL/mean": 0.0039770156145095825, "KL/rejected_KL_mean": -0.01642608642578125, "KL/std": 0.23556864261627197, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.040806323289871216, "fcm_dpo/q_t": 0.49897968769073486, "grad_norm": 74.32623291015625, "learning_rate": 1.3043478260869563e-07, "logits/chosen": -0.4715813398361206, "logits/rejected": -0.4200263023376465, "logps/chosen": -70.18097686767578, "logps/ref_chosen": -70.20536041259766, "logps/ref_rejected": -89.7575912475586, "logps/rejected": -89.77401733398438, "loss": 1.3825, "margin_dpo/margin_mean": 0.0408073365688324, "margin_dpo/margin_std": 0.30957865715026855, "step": 19 }, { "KL/chosen_KL_mean": -0.03499412536621094, "KL/mean": -0.04548324644565582, "KL/rejected_KL_mean": -0.05596923828125, "KL/std": 0.2744811177253723, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.020983070135116577, "fcm_dpo/q_t": 0.4994752109050751, "grad_norm": 74.43611145019531, "learning_rate": 1.3768115942028986e-07, "logits/chosen": -0.5649498105049133, "logits/rejected": -0.5099019408226013, "logps/chosen": -50.838233947753906, "logps/ref_chosen": -50.80324172973633, "logps/ref_rejected": -78.82334899902344, "logps/rejected": -78.87931823730469, "loss": 1.3846, "margin_dpo/margin_mean": 0.020983099937438965, "margin_dpo/margin_std": 0.3876810669898987, "step": 20 }, { "KL/chosen_KL_mean": 0.050243377685546875, "KL/mean": -0.018013283610343933, "KL/rejected_KL_mean": -0.08626937866210938, "KL/std": 0.3195314407348633, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1365136206150055, "fcm_dpo/q_t": 0.49658873677253723, "grad_norm": 77.88671875, "learning_rate": 1.4492753623188405e-07, "logits/chosen": -0.5110548138618469, "logits/rejected": -0.48853152990341187, "logps/chosen": -50.01277160644531, "logps/ref_chosen": -50.063018798828125, "logps/ref_rejected": -77.86878967285156, "logps/rejected": -77.95506286621094, "loss": 1.3731, "margin_dpo/margin_mean": 0.13651368021965027, "margin_dpo/margin_std": 0.4034786820411682, "step": 21 }, { "KL/chosen_KL_mean": -0.012243270874023438, "KL/mean": -0.08835446834564209, "KL/rejected_KL_mean": -0.16446685791015625, "KL/std": 0.28480368852615356, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1522234082221985, "fcm_dpo/q_t": 0.49619609117507935, "grad_norm": 84.3863525390625, "learning_rate": 1.5217391304347825e-07, "logits/chosen": -0.505510151386261, "logits/rejected": -0.462830126285553, "logps/chosen": -59.06987762451172, "logps/ref_chosen": -59.05763626098633, "logps/ref_rejected": -97.50466918945312, "logps/rejected": -97.66914367675781, "loss": 1.3716, "margin_dpo/margin_mean": 0.1522233486175537, "margin_dpo/margin_std": 0.4398460388183594, "step": 22 }, { "KL/chosen_KL_mean": 0.016481399536132812, "KL/mean": -0.06710737943649292, "KL/rejected_KL_mean": -0.1506977081298828, "KL/std": 0.340972900390625, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16717880964279175, "fcm_dpo/q_t": 0.49582529067993164, "grad_norm": 80.40861511230469, "learning_rate": 1.5942028985507245e-07, "logits/chosen": -0.5021919012069702, "logits/rejected": -0.48034122586250305, "logps/chosen": -60.061214447021484, "logps/ref_chosen": -60.07769775390625, "logps/ref_rejected": -81.13955688476562, "logps/rejected": -81.29025268554688, "loss": 1.3703, "margin_dpo/margin_mean": 0.16717904806137085, "margin_dpo/margin_std": 0.5109284520149231, "step": 23 }, { "KL/chosen_KL_mean": 0.0286712646484375, "KL/mean": -0.07983805984258652, "KL/rejected_KL_mean": -0.18835067749023438, "KL/std": 0.29715216159820557, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2170248180627823, "fcm_dpo/q_t": 0.49457818269729614, "grad_norm": 83.2780990600586, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5041535496711731, "logits/rejected": -0.48770707845687866, "logps/chosen": -44.26236343383789, "logps/ref_chosen": -44.29103469848633, "logps/ref_rejected": -99.12521362304688, "logps/rejected": -99.31356811523438, "loss": 1.3652, "margin_dpo/margin_mean": 0.21702483296394348, "margin_dpo/margin_std": 0.4385562539100647, "step": 24 }, { "KL/chosen_KL_mean": 0.02964019775390625, "KL/mean": -0.08741536736488342, "KL/rejected_KL_mean": -0.2044677734375, "KL/std": 0.34008604288101196, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23411154747009277, "fcm_dpo/q_t": 0.4941510558128357, "grad_norm": 74.14248657226562, "learning_rate": 1.7391304347826085e-07, "logits/chosen": -0.4980379641056061, "logits/rejected": -0.4681099057197571, "logps/chosen": -52.50741195678711, "logps/ref_chosen": -52.537052154541016, "logps/ref_rejected": -89.34219360351562, "logps/rejected": -89.54666137695312, "loss": 1.3635, "margin_dpo/margin_mean": 0.234111487865448, "margin_dpo/margin_std": 0.41896557807922363, "step": 25 }, { "KL/chosen_KL_mean": 0.05152320861816406, "KL/mean": -0.12674298882484436, "KL/rejected_KL_mean": -0.3050079345703125, "KL/std": 0.4253169298171997, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3565349578857422, "fcm_dpo/q_t": 0.49109458923339844, "grad_norm": 87.5750732421875, "learning_rate": 1.8115942028985507e-07, "logits/chosen": -0.5453453063964844, "logits/rejected": -0.513872504234314, "logps/chosen": -53.87128448486328, "logps/ref_chosen": -53.92280578613281, "logps/ref_rejected": -103.35971069335938, "logps/rejected": -103.66471862792969, "loss": 1.3516, "margin_dpo/margin_mean": 0.3565359115600586, "margin_dpo/margin_std": 0.4943789839744568, "step": 26 }, { "KL/chosen_KL_mean": 0.11599159240722656, "KL/mean": -0.13826510310173035, "KL/rejected_KL_mean": -0.39252471923828125, "KL/std": 0.476246178150177, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5085150599479675, "fcm_dpo/q_t": 0.48730653524398804, "grad_norm": 93.57472229003906, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.5514802932739258, "logits/rejected": -0.5156171321868896, "logps/chosen": -42.78253936767578, "logps/ref_chosen": -42.898529052734375, "logps/ref_rejected": -98.72419738769531, "logps/rejected": -99.11672973632812, "loss": 1.3369, "margin_dpo/margin_mean": 0.5085144639015198, "margin_dpo/margin_std": 0.5824718475341797, "step": 27 }, { "KL/chosen_KL_mean": 0.011579513549804688, "KL/mean": -0.13822337985038757, "KL/rejected_KL_mean": -0.2880287170410156, "KL/std": 0.42481285333633423, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2996072769165039, "fcm_dpo/q_t": 0.49251770973205566, "grad_norm": 75.85013580322266, "learning_rate": 1.9565217391304347e-07, "logits/chosen": -0.497283399105072, "logits/rejected": -0.4410708546638489, "logps/chosen": -60.544921875, "logps/ref_chosen": -60.55650329589844, "logps/ref_rejected": -91.40111541748047, "logps/rejected": -91.68914794921875, "loss": 1.3574, "margin_dpo/margin_mean": 0.2996074855327606, "margin_dpo/margin_std": 0.562934160232544, "step": 28 }, { "KL/chosen_KL_mean": 0.12349700927734375, "KL/mean": -0.16272687911987305, "KL/rejected_KL_mean": -0.4489555358886719, "KL/std": 0.4961986243724823, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5724496841430664, "fcm_dpo/q_t": 0.4857064485549927, "grad_norm": 90.49112701416016, "learning_rate": 2.028985507246377e-07, "logits/chosen": -0.5566602945327759, "logits/rejected": -0.5104827880859375, "logps/chosen": -57.684288024902344, "logps/ref_chosen": -57.80778503417969, "logps/ref_rejected": -97.39434814453125, "logps/rejected": -97.84330749511719, "loss": 1.3307, "margin_dpo/margin_mean": 0.5724506378173828, "margin_dpo/margin_std": 0.5618862509727478, "step": 29 }, { "KL/chosen_KL_mean": 0.11771583557128906, "KL/mean": -0.2122490406036377, "KL/rejected_KL_mean": -0.5422172546386719, "KL/std": 0.5805710554122925, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.659930944442749, "fcm_dpo/q_t": 0.4835302233695984, "grad_norm": 87.53767395019531, "learning_rate": 2.1014492753623187e-07, "logits/chosen": -0.4962325990200043, "logits/rejected": -0.4656328558921814, "logps/chosen": -52.45965576171875, "logps/ref_chosen": -52.577369689941406, "logps/ref_rejected": -98.48920440673828, "logps/rejected": -99.03141784667969, "loss": 1.3225, "margin_dpo/margin_mean": 0.659930944442749, "margin_dpo/margin_std": 0.6480692625045776, "step": 30 }, { "KL/chosen_KL_mean": 0.10221672058105469, "KL/mean": -0.15314733982086182, "KL/rejected_KL_mean": -0.4085121154785156, "KL/std": 0.5654332637786865, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.510732889175415, "fcm_dpo/q_t": 0.48725682497024536, "grad_norm": 67.43456268310547, "learning_rate": 2.1739130434782607e-07, "logits/chosen": -0.49099212884902954, "logits/rejected": -0.44460946321487427, "logps/chosen": -63.70470428466797, "logps/ref_chosen": -63.806922912597656, "logps/ref_rejected": -72.89400482177734, "logps/rejected": -73.30252075195312, "loss": 1.3371, "margin_dpo/margin_mean": 0.5107325315475464, "margin_dpo/margin_std": 0.6857659220695496, "step": 31 }, { "KL/chosen_KL_mean": 0.2014923095703125, "KL/mean": -0.19277532398700714, "KL/rejected_KL_mean": -0.5870437622070312, "KL/std": 0.6862952709197998, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7885334491729736, "fcm_dpo/q_t": 0.4803431034088135, "grad_norm": 80.95833587646484, "learning_rate": 2.2463768115942027e-07, "logits/chosen": -0.5271800756454468, "logits/rejected": -0.4867057204246521, "logps/chosen": -62.53803253173828, "logps/ref_chosen": -62.739524841308594, "logps/ref_rejected": -89.3175048828125, "logps/rejected": -89.904541015625, "loss": 1.3106, "margin_dpo/margin_mean": 0.7885338664054871, "margin_dpo/margin_std": 0.8146342635154724, "step": 32 }, { "KL/chosen_KL_mean": 0.13482093811035156, "KL/mean": -0.1859203577041626, "KL/rejected_KL_mean": -0.5066604614257812, "KL/std": 0.5469956994056702, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6414769291877747, "fcm_dpo/q_t": 0.4839860796928406, "grad_norm": 72.35101318359375, "learning_rate": 2.318840579710145e-07, "logits/chosen": -0.5072885751724243, "logits/rejected": -0.481620728969574, "logps/chosen": -53.12615203857422, "logps/ref_chosen": -53.26097106933594, "logps/ref_rejected": -87.8851318359375, "logps/rejected": -88.39179992675781, "loss": 1.3242, "margin_dpo/margin_mean": 0.6414770483970642, "margin_dpo/margin_std": 0.6191001534461975, "step": 33 }, { "KL/chosen_KL_mean": 0.08164787292480469, "KL/mean": -0.32943296432495117, "KL/rejected_KL_mean": -0.7405166625976562, "KL/std": 0.7699615359306335, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8221615552902222, "fcm_dpo/q_t": 0.47951143980026245, "grad_norm": 77.51274871826172, "learning_rate": 2.391304347826087e-07, "logits/chosen": -0.48901137709617615, "logits/rejected": -0.4716091752052307, "logps/chosen": -50.735679626464844, "logps/ref_chosen": -50.81732940673828, "logps/ref_rejected": -101.92184448242188, "logps/rejected": -102.66236114501953, "loss": 1.3079, "margin_dpo/margin_mean": 0.8221613764762878, "margin_dpo/margin_std": 0.9267702102661133, "step": 34 }, { "KL/chosen_KL_mean": 0.11441421508789062, "KL/mean": -0.5115079879760742, "KL/rejected_KL_mean": -1.1374282836914062, "KL/std": 1.0209949016571045, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2518417835235596, "fcm_dpo/q_t": 0.46891528367996216, "grad_norm": 82.27989196777344, "learning_rate": 2.463768115942029e-07, "logits/chosen": -0.5277581214904785, "logits/rejected": -0.49088025093078613, "logps/chosen": -50.91007614135742, "logps/ref_chosen": -51.02449035644531, "logps/ref_rejected": -106.82443237304688, "logps/rejected": -107.96185302734375, "loss": 1.2684, "margin_dpo/margin_mean": 1.2518424987792969, "margin_dpo/margin_std": 1.1583997011184692, "step": 35 }, { "KL/chosen_KL_mean": 0.054615020751953125, "KL/mean": -0.5534474849700928, "KL/rejected_KL_mean": -1.1615104675292969, "KL/std": 1.1053463220596313, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2161287069320679, "fcm_dpo/q_t": 0.469775915145874, "grad_norm": 72.23912048339844, "learning_rate": 2.536231884057971e-07, "logits/chosen": -0.5657041072845459, "logits/rejected": -0.5297294855117798, "logps/chosen": -51.93687438964844, "logps/ref_chosen": -51.991493225097656, "logps/ref_rejected": -86.0406265258789, "logps/rejected": -87.20213317871094, "loss": 1.2726, "margin_dpo/margin_mean": 1.2161275148391724, "margin_dpo/margin_std": 1.2665972709655762, "step": 36 }, { "KL/chosen_KL_mean": 0.014173507690429688, "KL/mean": -0.5154328942298889, "KL/rejected_KL_mean": -1.0450401306152344, "KL/std": 1.0978405475616455, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.059217929840088, "fcm_dpo/q_t": 0.4737340807914734, "grad_norm": 61.976402282714844, "learning_rate": 2.6086956521739126e-07, "logits/chosen": -0.5021863579750061, "logits/rejected": -0.4578307271003723, "logps/chosen": -62.79293441772461, "logps/ref_chosen": -62.807106018066406, "logps/ref_rejected": -77.89507293701172, "logps/rejected": -78.94010925292969, "loss": 1.288, "margin_dpo/margin_mean": 1.0592180490493774, "margin_dpo/margin_std": 1.3723002672195435, "step": 37 }, { "KL/chosen_KL_mean": 0.17836570739746094, "KL/mean": -0.5307787656784058, "KL/rejected_KL_mean": -1.2399253845214844, "KL/std": 1.3880457878112793, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.418290615081787, "fcm_dpo/q_t": 0.4649973213672638, "grad_norm": 69.40721893310547, "learning_rate": 2.681159420289855e-07, "logits/chosen": -0.5123181343078613, "logits/rejected": -0.47957277297973633, "logps/chosen": -48.212154388427734, "logps/ref_chosen": -48.39051818847656, "logps/ref_rejected": -97.91244506835938, "logps/rejected": -99.1523666381836, "loss": 1.2564, "margin_dpo/margin_mean": 1.4182908535003662, "margin_dpo/margin_std": 1.6710336208343506, "step": 38 }, { "KL/chosen_KL_mean": 0.11879920959472656, "KL/mean": -0.7346929311752319, "KL/rejected_KL_mean": -1.5881805419921875, "KL/std": 1.3292229175567627, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7069776058197021, "fcm_dpo/q_t": 0.4576621651649475, "grad_norm": 74.3014907836914, "learning_rate": 2.753623188405797e-07, "logits/chosen": -0.5710403919219971, "logits/rejected": -0.5320708155632019, "logps/chosen": -50.631675720214844, "logps/ref_chosen": -50.75047302246094, "logps/ref_rejected": -78.56951141357422, "logps/rejected": -80.1576919555664, "loss": 1.228, "margin_dpo/margin_mean": 1.7069778442382812, "margin_dpo/margin_std": 1.4111273288726807, "step": 39 }, { "KL/chosen_KL_mean": 0.2220172882080078, "KL/mean": -0.5618829131126404, "KL/rejected_KL_mean": -1.3457870483398438, "KL/std": 1.41432523727417, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5678019523620605, "fcm_dpo/q_t": 0.46130138635635376, "grad_norm": 60.054290771484375, "learning_rate": 2.8260869565217386e-07, "logits/chosen": -0.507063627243042, "logits/rejected": -0.4761931002140045, "logps/chosen": -57.7630500793457, "logps/ref_chosen": -57.985069274902344, "logps/ref_rejected": -74.3000717163086, "logps/rejected": -75.64585876464844, "loss": 1.2432, "margin_dpo/margin_mean": 1.5678024291992188, "margin_dpo/margin_std": 1.709043264389038, "step": 40 }, { "KL/chosen_KL_mean": 0.0051975250244140625, "KL/mean": -0.9326056838035583, "KL/rejected_KL_mean": -1.8704071044921875, "KL/std": 1.854329228401184, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8756054639816284, "fcm_dpo/q_t": 0.4538491666316986, "grad_norm": 68.05853271484375, "learning_rate": 2.898550724637681e-07, "logits/chosen": -0.540554940700531, "logits/rejected": -0.5041079521179199, "logps/chosen": -62.69062042236328, "logps/ref_chosen": -62.69581604003906, "logps/ref_rejected": -97.02352905273438, "logps/rejected": -98.89393615722656, "loss": 1.2179, "margin_dpo/margin_mean": 1.8756061792373657, "margin_dpo/margin_std": 2.0067355632781982, "step": 41 }, { "KL/chosen_KL_mean": 0.21923828125, "KL/mean": -1.0403838157653809, "KL/rejected_KL_mean": -2.300006866455078, "KL/std": 2.130916118621826, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.1539303958415985, "fcm_dpo/delta": -0.01988767832517624, "fcm_dpo/margin": 2.5192437171936035, "fcm_dpo/q_t": 0.4016793966293335, "grad_norm": 98.06798553466797, "learning_rate": 2.971014492753623e-07, "logits/chosen": -0.5261090993881226, "logits/rejected": -0.4792720377445221, "logps/chosen": -58.747188568115234, "logps/ref_chosen": -58.966426849365234, "logps/ref_rejected": -109.90837097167969, "logps/rejected": -112.20838165283203, "loss": 1.0939, "margin_dpo/margin_mean": 2.5192432403564453, "margin_dpo/margin_std": 2.3735432624816895, "step": 42 }, { "KL/chosen_KL_mean": 0.5401420593261719, "KL/mean": -0.6934725046157837, "KL/rejected_KL_mean": -1.927093505859375, "KL/std": 1.8467273712158203, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.18102575838565826, "fcm_dpo/delta": 0.06920016556978226, "fcm_dpo/margin": 2.4672341346740723, "fcm_dpo/q_t": 0.4202408194541931, "grad_norm": 109.3116455078125, "learning_rate": 3.043478260869565e-07, "logits/chosen": -0.540740966796875, "logits/rejected": -0.5158405303955078, "logps/chosen": -53.61585235595703, "logps/ref_chosen": -54.15599822998047, "logps/ref_rejected": -96.48019409179688, "logps/rejected": -98.40728759765625, "loss": 1.0181, "margin_dpo/margin_mean": 2.4672341346740723, "margin_dpo/margin_std": 1.9857051372528076, "step": 43 }, { "KL/chosen_KL_mean": 0.1969776153564453, "KL/mean": -1.1819499731063843, "KL/rejected_KL_mean": -2.5608787536621094, "KL/std": 2.232034206390381, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.14869627356529236, "fcm_dpo/delta": 0.008696533739566803, "fcm_dpo/margin": 2.7578556537628174, "fcm_dpo/q_t": 0.40634626150131226, "grad_norm": 103.40689849853516, "learning_rate": 3.115942028985507e-07, "logits/chosen": -0.49857497215270996, "logits/rejected": -0.47800567746162415, "logps/chosen": -49.88152313232422, "logps/ref_chosen": -50.07849884033203, "logps/ref_rejected": -108.78376007080078, "logps/rejected": -111.34463500976562, "loss": 1.0466, "margin_dpo/margin_mean": 2.7578554153442383, "margin_dpo/margin_std": 2.197627067565918, "step": 44 }, { "KL/chosen_KL_mean": 0.1326732635498047, "KL/mean": -1.014523983001709, "KL/rejected_KL_mean": -2.1617279052734375, "KL/std": 2.0959479808807373, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.15769031643867493, "fcm_dpo/delta": -0.06119377911090851, "fcm_dpo/margin": 2.2944016456604004, "fcm_dpo/q_t": 0.39686933159828186, "grad_norm": 85.24789428710938, "learning_rate": 3.188405797101449e-07, "logits/chosen": -0.4741653800010681, "logits/rejected": -0.46142202615737915, "logps/chosen": -48.282257080078125, "logps/ref_chosen": -48.4149284362793, "logps/ref_rejected": -77.93643188476562, "logps/rejected": -80.09815979003906, "loss": 1.1099, "margin_dpo/margin_mean": 2.2944014072418213, "margin_dpo/margin_std": 2.6267547607421875, "step": 45 }, { "KL/chosen_KL_mean": 0.2033252716064453, "KL/mean": -1.2960678339004517, "KL/rejected_KL_mean": -2.795459747314453, "KL/std": 2.6904854774475098, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.1267249435186386, "fcm_dpo/delta": -0.04247567802667618, "fcm_dpo/margin": 2.9987845420837402, "fcm_dpo/q_t": 0.39998096227645874, "grad_norm": 75.10869598388672, "learning_rate": 3.260869565217391e-07, "logits/chosen": -0.5231410264968872, "logits/rejected": -0.4719325304031372, "logps/chosen": -55.79610061645508, "logps/ref_chosen": -55.999427795410156, "logps/ref_rejected": -95.652587890625, "logps/rejected": -98.44804382324219, "loss": 1.1114, "margin_dpo/margin_mean": 2.9987847805023193, "margin_dpo/margin_std": 3.3123412132263184, "step": 46 }, { "KL/chosen_KL_mean": 0.4382801055908203, "KL/mean": -1.083209753036499, "KL/rejected_KL_mean": -2.6046981811523438, "KL/std": 2.6264405250549316, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10618115961551666, "fcm_dpo/delta": 0.0855509415268898, "fcm_dpo/margin": 3.042980670928955, "fcm_dpo/q_t": 0.452394038438797, "grad_norm": 67.91606140136719, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.5615835189819336, "logits/rejected": -0.5081955194473267, "logps/chosen": -57.48780059814453, "logps/ref_chosen": -57.92607879638672, "logps/ref_rejected": -94.67920684814453, "logps/rejected": -97.28390502929688, "loss": 1.0978, "margin_dpo/margin_mean": 3.0429811477661133, "margin_dpo/margin_std": 2.6995935440063477, "step": 47 }, { "KL/chosen_KL_mean": 0.11672401428222656, "KL/mean": -1.4517210721969604, "KL/rejected_KL_mean": -3.0201644897460938, "KL/std": 2.6644673347473145, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.07081333547830582, "fcm_dpo/delta": -0.07440280169248581, "fcm_dpo/margin": 3.1368894577026367, "fcm_dpo/q_t": 0.41475388407707214, "grad_norm": 55.53757095336914, "learning_rate": 3.4057971014492755e-07, "logits/chosen": -0.5801494121551514, "logits/rejected": -0.5212501287460327, "logps/chosen": -57.07135009765625, "logps/ref_chosen": -57.188072204589844, "logps/ref_rejected": -88.0166015625, "logps/rejected": -91.0367660522461, "loss": 1.1888, "margin_dpo/margin_mean": 3.1368894577026367, "margin_dpo/margin_std": 2.9127197265625, "step": 48 }, { "KL/chosen_KL_mean": 0.3498554229736328, "KL/mean": -1.5747132301330566, "KL/rejected_KL_mean": -3.499279022216797, "KL/std": 3.408785581588745, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.09813052415847778, "fcm_dpo/delta": -0.0769207701086998, "fcm_dpo/margin": 3.8491296768188477, "fcm_dpo/q_t": 0.3957645893096924, "grad_norm": 59.666378021240234, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.5373194813728333, "logits/rejected": -0.47734540700912476, "logps/chosen": -61.33541488647461, "logps/ref_chosen": -61.685272216796875, "logps/ref_rejected": -83.76747131347656, "logps/rejected": -87.26675415039062, "loss": 1.1176, "margin_dpo/margin_mean": 3.8491291999816895, "margin_dpo/margin_std": 3.9004533290863037, "step": 49 }, { "KL/chosen_KL_mean": -0.15655517578125, "KL/mean": -2.1865463256835938, "KL/rejected_KL_mean": -4.216529846191406, "KL/std": 3.433018445968628, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.03281348943710327, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.059974193572998, "fcm_dpo/q_t": 0.4669080376625061, "grad_norm": 26.208717346191406, "learning_rate": 3.5507246376811595e-07, "logits/chosen": -0.5582753419876099, "logits/rejected": -0.522718608379364, "logps/chosen": -58.88069152832031, "logps/ref_chosen": -58.72413635253906, "logps/ref_rejected": -96.35814666748047, "logps/rejected": -100.57467651367188, "loss": 1.2617, "margin_dpo/margin_mean": 4.059974193572998, "margin_dpo/margin_std": 3.9510397911071777, "step": 50 }, { "KL/chosen_KL_mean": -0.26395416259765625, "KL/mean": -2.266573429107666, "KL/rejected_KL_mean": -4.269195556640625, "KL/std": 4.127612113952637, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.03281348943710327, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.005241870880127, "fcm_dpo/q_t": 0.46752673387527466, "grad_norm": 23.76943588256836, "learning_rate": 3.6231884057971015e-07, "logits/chosen": -0.5171994566917419, "logits/rejected": -0.4839722514152527, "logps/chosen": -61.63762283325195, "logps/ref_chosen": -61.3736686706543, "logps/ref_rejected": -76.00199890136719, "logps/rejected": -80.27119445800781, "loss": 1.2662, "margin_dpo/margin_mean": 4.005241394042969, "margin_dpo/margin_std": 5.0708160400390625, "step": 51 }, { "KL/chosen_KL_mean": 0.34140777587890625, "KL/mean": -2.696107864379883, "KL/rejected_KL_mean": -5.7336273193359375, "KL/std": 4.7038187980651855, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.03281348943710327, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.075035572052002, "fcm_dpo/q_t": 0.4507228136062622, "grad_norm": 28.817222595214844, "learning_rate": 3.695652173913043e-07, "logits/chosen": -0.5797228217124939, "logits/rejected": -0.525948166847229, "logps/chosen": -51.995948791503906, "logps/ref_chosen": -52.33735656738281, "logps/ref_rejected": -79.97391510009766, "logps/rejected": -85.7075424194336, "loss": 1.2035, "margin_dpo/margin_mean": 6.07503604888916, "margin_dpo/margin_std": 4.994811058044434, "step": 52 }, { "KL/chosen_KL_mean": -0.22272109985351562, "KL/mean": -3.3271031379699707, "KL/rejected_KL_mean": -6.431484222412109, "KL/std": 5.386918544769287, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.05016179382801056, "fcm_dpo/delta": 0.07214375585317612, "fcm_dpo/margin": 6.208770275115967, "fcm_dpo/q_t": 0.4501815438270569, "grad_norm": 38.668609619140625, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.6235509514808655, "logits/rejected": -0.6023184061050415, "logps/chosen": -53.53737258911133, "logps/ref_chosen": -53.31465148925781, "logps/ref_rejected": -91.78359985351562, "logps/rejected": -98.21508026123047, "loss": 1.1228, "margin_dpo/margin_mean": 6.208770275115967, "margin_dpo/margin_std": 6.1717681884765625, "step": 53 }, { "KL/chosen_KL_mean": -0.36040306091308594, "KL/mean": -3.0783324241638184, "KL/rejected_KL_mean": -5.796260833740234, "KL/std": 5.118019104003906, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.06915527582168579, "fcm_dpo/delta": -0.035718172788619995, "fcm_dpo/margin": 5.435856819152832, "fcm_dpo/q_t": 0.3983124792575836, "grad_norm": 43.57950210571289, "learning_rate": 3.8405797101449274e-07, "logits/chosen": -0.6051186323165894, "logits/rejected": -0.5521863102912903, "logps/chosen": -51.04906463623047, "logps/ref_chosen": -50.68865966796875, "logps/ref_rejected": -91.71539306640625, "logps/rejected": -97.51165771484375, "loss": 1.0854, "margin_dpo/margin_mean": 5.435855865478516, "margin_dpo/margin_std": 5.120929718017578, "step": 54 }, { "KL/chosen_KL_mean": -0.9818630218505859, "KL/mean": -4.163016319274902, "KL/rejected_KL_mean": -7.344165802001953, "KL/std": 6.130636215209961, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.05893651396036148, "fcm_dpo/delta": -0.0688391923904419, "fcm_dpo/margin": 6.362303733825684, "fcm_dpo/q_t": 0.3950488865375519, "grad_norm": 41.72751998901367, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.6603301167488098, "logits/rejected": -0.5991765260696411, "logps/chosen": -63.59709930419922, "logps/ref_chosen": -62.615234375, "logps/ref_rejected": -88.99349975585938, "logps/rejected": -96.3376693725586, "loss": 1.1394, "margin_dpo/margin_mean": 6.362303733825684, "margin_dpo/margin_std": 7.427134037017822, "step": 55 }, { "KL/chosen_KL_mean": -0.7029590606689453, "KL/mean": -3.7865757942199707, "KL/rejected_KL_mean": -6.870197296142578, "KL/std": 6.062340259552002, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.02372913435101509, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.167237758636475, "fcm_dpo/q_t": 0.46382689476013184, "grad_norm": 18.926555633544922, "learning_rate": 3.9855072463768114e-07, "logits/chosen": -0.6277616024017334, "logits/rejected": -0.5860629677772522, "logps/chosen": -58.63568878173828, "logps/ref_chosen": -57.9327278137207, "logps/ref_rejected": -94.1744384765625, "logps/rejected": -101.04463195800781, "loss": 1.2529, "margin_dpo/margin_mean": 6.167238235473633, "margin_dpo/margin_std": 7.341178894042969, "step": 56 }, { "KL/chosen_KL_mean": -0.7325782775878906, "KL/mean": -4.221752166748047, "KL/rejected_KL_mean": -7.710926055908203, "KL/std": 6.029708385467529, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.02372913435101509, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.978342056274414, "fcm_dpo/q_t": 0.458979070186615, "grad_norm": 21.024066925048828, "learning_rate": 4.057971014492754e-07, "logits/chosen": -0.6242918968200684, "logits/rejected": -0.5983684062957764, "logps/chosen": -71.22785949707031, "logps/ref_chosen": -70.49528503417969, "logps/ref_rejected": -95.56546020507812, "logps/rejected": -103.27638244628906, "loss": 1.2339, "margin_dpo/margin_mean": 6.978342533111572, "margin_dpo/margin_std": 6.642031669616699, "step": 57 }, { "KL/chosen_KL_mean": -1.0038623809814453, "KL/mean": -4.862548351287842, "KL/rejected_KL_mean": -8.721233367919922, "KL/std": 6.818406105041504, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.04264631122350693, "fcm_dpo/delta": 0.09533653408288956, "fcm_dpo/margin": 7.717365264892578, "fcm_dpo/q_t": 0.4548690915107727, "grad_norm": 33.23288345336914, "learning_rate": 4.1304347826086954e-07, "logits/chosen": -0.6018198132514954, "logits/rejected": -0.5264607071876526, "logps/chosen": -63.136802673339844, "logps/ref_chosen": -62.13294219970703, "logps/ref_rejected": -84.61729431152344, "logps/rejected": -93.33853149414062, "loss": 1.108, "margin_dpo/margin_mean": 7.7173662185668945, "margin_dpo/margin_std": 8.156654357910156, "step": 58 }, { "KL/chosen_KL_mean": -1.4942893981933594, "KL/mean": -5.754292011260986, "KL/rejected_KL_mean": -10.014289855957031, "KL/std": 7.55443000793457, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.01908048428595066, "fcm_dpo/delta": -0.1171402782201767, "fcm_dpo/margin": 8.52000617980957, "fcm_dpo/q_t": 0.41208070516586304, "grad_norm": 19.520845413208008, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.6208063364028931, "logits/rejected": -0.580247163772583, "logps/chosen": -53.426815032958984, "logps/ref_chosen": -51.932525634765625, "logps/ref_rejected": -88.88520050048828, "logps/rejected": -98.89949035644531, "loss": 1.2374, "margin_dpo/margin_mean": 8.520007133483887, "margin_dpo/margin_std": 8.690988540649414, "step": 59 }, { "KL/chosen_KL_mean": -2.7874374389648438, "KL/mean": -6.149212837219238, "KL/rejected_KL_mean": -9.510990142822266, "KL/std": 6.844639778137207, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.01908048428595066, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.723544597625732, "fcm_dpo/q_t": 0.4681331217288971, "grad_norm": 17.15147590637207, "learning_rate": 4.2753623188405794e-07, "logits/chosen": -0.6261638402938843, "logits/rejected": -0.5673079490661621, "logps/chosen": -63.7296257019043, "logps/ref_chosen": -60.94218826293945, "logps/ref_rejected": -85.39340209960938, "logps/rejected": -94.90438842773438, "loss": 1.2677, "margin_dpo/margin_mean": 6.723544597625732, "margin_dpo/margin_std": 7.779609680175781, "step": 60 }, { "KL/chosen_KL_mean": -1.5964698791503906, "KL/mean": -5.815464973449707, "KL/rejected_KL_mean": -10.034461975097656, "KL/std": 9.321011543273926, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.01908048428595066, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.437992095947266, "fcm_dpo/q_t": 0.4607029855251312, "grad_norm": 17.691282272338867, "learning_rate": 4.3478260869565214e-07, "logits/chosen": -0.6190700531005859, "logits/rejected": -0.585500955581665, "logps/chosen": -62.22999572753906, "logps/ref_chosen": -60.633522033691406, "logps/ref_rejected": -89.85249328613281, "logps/rejected": -99.886962890625, "loss": 1.244, "margin_dpo/margin_mean": 8.437992095947266, "margin_dpo/margin_std": 11.62002182006836, "step": 61 }, { "KL/chosen_KL_mean": -1.81982421875, "KL/mean": -5.002796173095703, "KL/rejected_KL_mean": -8.185768127441406, "KL/std": 6.878681182861328, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.01908048428595066, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.365945816040039, "fcm_dpo/q_t": 0.4699386656284332, "grad_norm": 14.713135719299316, "learning_rate": 4.420289855072464e-07, "logits/chosen": -0.609377384185791, "logits/rejected": -0.5751521587371826, "logps/chosen": -57.97059631347656, "logps/ref_chosen": -56.15077209472656, "logps/ref_rejected": -75.56619262695312, "logps/rejected": -83.75196838378906, "loss": 1.2752, "margin_dpo/margin_mean": 6.365945816040039, "margin_dpo/margin_std": 8.420263290405273, "step": 62 }, { "KL/chosen_KL_mean": -3.0295848846435547, "KL/mean": -7.3545379638671875, "KL/rejected_KL_mean": -11.679492950439453, "KL/std": 8.834741592407227, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.01908048428595066, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.649913787841797, "fcm_dpo/q_t": 0.45918020606040955, "grad_norm": 17.89116668701172, "learning_rate": 4.4927536231884053e-07, "logits/chosen": -0.624936044216156, "logits/rejected": -0.5820035338401794, "logps/chosen": -76.1769790649414, "logps/ref_chosen": -73.14739227294922, "logps/ref_rejected": -97.61006164550781, "logps/rejected": -109.28955078125, "loss": 1.236, "margin_dpo/margin_mean": 8.64991283416748, "margin_dpo/margin_std": 9.18691635131836, "step": 63 }, { "KL/chosen_KL_mean": -1.2771644592285156, "KL/mean": -6.465435981750488, "KL/rejected_KL_mean": -11.653709411621094, "KL/std": 9.661531448364258, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.05195356905460358, "fcm_dpo/delta": 0.12018927931785583, "fcm_dpo/margin": 10.376541137695312, "fcm_dpo/q_t": 0.43298691511154175, "grad_norm": 37.171104431152344, "learning_rate": 4.5652173913043473e-07, "logits/chosen": -0.5985631942749023, "logits/rejected": -0.5670138597488403, "logps/chosen": -55.27576446533203, "logps/ref_chosen": -53.998600006103516, "logps/ref_rejected": -93.53019714355469, "logps/rejected": -105.18391418457031, "loss": 1.0256, "margin_dpo/margin_mean": 10.376540184020996, "margin_dpo/margin_std": 11.894201278686523, "step": 64 }, { "KL/chosen_KL_mean": -3.6957435607910156, "KL/mean": -8.960607528686523, "KL/rejected_KL_mean": -14.225475311279297, "KL/std": 10.946447372436523, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": -0.18592241406440735, "fcm_dpo/margin": 10.529730796813965, "fcm_dpo/q_t": 0.4064898192882538, "grad_norm": 11.372929573059082, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.6771352887153625, "logits/rejected": -0.6651996374130249, "logps/chosen": -68.53174591064453, "logps/ref_chosen": -64.83599853515625, "logps/ref_rejected": -109.94645690917969, "logps/rejected": -124.17193603515625, "loss": 1.2885, "margin_dpo/margin_mean": 10.529730796813965, "margin_dpo/margin_std": 12.0299072265625, "step": 65 }, { "KL/chosen_KL_mean": -3.5717506408691406, "KL/mean": -8.375314712524414, "KL/rejected_KL_mean": -13.178871154785156, "KL/std": 9.874279975891113, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.607120513916016, "fcm_dpo/q_t": 0.4763941168785095, "grad_norm": 9.938729286193848, "learning_rate": 4.7101449275362313e-07, "logits/chosen": -0.6698400974273682, "logits/rejected": -0.6391260623931885, "logps/chosen": -55.01527404785156, "logps/ref_chosen": -51.44352722167969, "logps/ref_rejected": -75.63629913330078, "logps/rejected": -88.81517028808594, "loss": 1.2972, "margin_dpo/margin_mean": 9.607121467590332, "margin_dpo/margin_std": 11.98110580444336, "step": 66 }, { "KL/chosen_KL_mean": -3.3190994262695312, "KL/mean": -8.360406875610352, "KL/rejected_KL_mean": -13.401721954345703, "KL/std": 10.303428649902344, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.082622528076172, "fcm_dpo/q_t": 0.47523707151412964, "grad_norm": 10.002849578857422, "learning_rate": 4.782608695652174e-07, "logits/chosen": -0.6043215394020081, "logits/rejected": -0.5609872341156006, "logps/chosen": -62.65990447998047, "logps/ref_chosen": -59.34080505371094, "logps/ref_rejected": -72.78728485107422, "logps/rejected": -86.18901062011719, "loss": 1.293, "margin_dpo/margin_mean": 10.082620620727539, "margin_dpo/margin_std": 12.531867027282715, "step": 67 }, { "KL/chosen_KL_mean": -4.522434234619141, "KL/mean": -9.367167472839355, "KL/rejected_KL_mean": -14.211898803710938, "KL/std": 9.465312957763672, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.689462661743164, "fcm_dpo/q_t": 0.47615760564804077, "grad_norm": 9.184582710266113, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.6481267213821411, "logits/rejected": -0.5857020020484924, "logps/chosen": -69.728271484375, "logps/ref_chosen": -65.2058334350586, "logps/ref_rejected": -77.20724487304688, "logps/rejected": -91.41914367675781, "loss": 1.2957, "margin_dpo/margin_mean": 9.689462661743164, "margin_dpo/margin_std": 10.790483474731445, "step": 68 }, { "KL/chosen_KL_mean": -5.772527694702148, "KL/mean": -12.4307222366333, "KL/rejected_KL_mean": -19.088916778564453, "KL/std": 12.551191329956055, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.31639289855957, "fcm_dpo/q_t": 0.4673622250556946, "grad_norm": 11.88070297241211, "learning_rate": 4.927536231884058e-07, "logits/chosen": -0.633673906326294, "logits/rejected": -0.6116066575050354, "logps/chosen": -65.59176635742188, "logps/ref_chosen": -59.81924057006836, "logps/ref_rejected": -103.38886260986328, "logps/rejected": -122.477783203125, "loss": 1.264, "margin_dpo/margin_mean": 13.31639289855957, "margin_dpo/margin_std": 14.175312042236328, "step": 69 }, { "KL/chosen_KL_mean": -8.516521453857422, "KL/mean": -15.706306457519531, "KL/rejected_KL_mean": -22.89609146118164, "KL/std": 16.06771469116211, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.37956428527832, "fcm_dpo/q_t": 0.46483179926872253, "grad_norm": 11.995932579040527, "learning_rate": 5e-07, "logits/chosen": -0.6291612386703491, "logits/rejected": -0.5962782502174377, "logps/chosen": -70.44715881347656, "logps/ref_chosen": -61.930641174316406, "logps/ref_rejected": -91.06078338623047, "logps/rejected": -113.95687866210938, "loss": 1.2564, "margin_dpo/margin_mean": 14.37956428527832, "margin_dpo/margin_std": 17.146224975585938, "step": 70 }, { "KL/chosen_KL_mean": -9.071426391601562, "KL/mean": -17.838401794433594, "KL/rejected_KL_mean": -26.605384826660156, "KL/std": 17.67050552368164, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.53395652770996, "fcm_dpo/q_t": 0.4574003219604492, "grad_norm": 13.351109504699707, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.6778059601783752, "logits/rejected": -0.6437849998474121, "logps/chosen": -70.82176208496094, "logps/ref_chosen": -61.750335693359375, "logps/ref_rejected": -97.33662414550781, "logps/rejected": -123.94200134277344, "loss": 1.2301, "margin_dpo/margin_mean": 17.533954620361328, "margin_dpo/margin_std": 19.450382232666016, "step": 71 }, { "KL/chosen_KL_mean": -10.510194778442383, "KL/mean": -19.48761749267578, "KL/rejected_KL_mean": -28.465042114257812, "KL/std": 18.958513259887695, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.954853057861328, "fcm_dpo/q_t": 0.456373393535614, "grad_norm": 12.942930221557617, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.6576621532440186, "logits/rejected": -0.6210486888885498, "logps/chosen": -76.56360626220703, "logps/ref_chosen": -66.05341339111328, "logps/ref_rejected": -95.2869873046875, "logps/rejected": -123.75202941894531, "loss": 1.2293, "margin_dpo/margin_mean": 17.954853057861328, "margin_dpo/margin_std": 23.021011352539062, "step": 72 }, { "KL/chosen_KL_mean": -12.810209274291992, "KL/mean": -21.97817611694336, "KL/rejected_KL_mean": -31.14615249633789, "KL/std": 24.339290618896484, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.009888127446174622, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.335941314697266, "fcm_dpo/q_t": 0.45616060495376587, "grad_norm": 13.09295654296875, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.6870651245117188, "logits/rejected": -0.6481956243515015, "logps/chosen": -79.06648254394531, "logps/ref_chosen": -66.25627136230469, "logps/ref_rejected": -90.45613098144531, "logps/rejected": -121.60228729248047, "loss": 1.2364, "margin_dpo/margin_mean": 18.335941314697266, "margin_dpo/margin_std": 31.158466339111328, "step": 73 }, { "KL/chosen_KL_mean": -13.415903091430664, "KL/mean": -23.937971115112305, "KL/rejected_KL_mean": -34.46004104614258, "KL/std": 27.361499786376953, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.019809015095233917, "fcm_dpo/delta": 0.04002254828810692, "fcm_dpo/margin": 21.04413414001465, "fcm_dpo/q_t": 0.42780494689941406, "grad_norm": 22.95852279663086, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.6472772359848022, "logits/rejected": -0.6437745690345764, "logps/chosen": -66.84078979492188, "logps/ref_chosen": -53.42488098144531, "logps/ref_rejected": -95.94693756103516, "logps/rejected": -130.406982421875, "loss": 1.1006, "margin_dpo/margin_mean": 21.04413604736328, "margin_dpo/margin_std": 34.61186218261719, "step": 74 }, { "KL/chosen_KL_mean": -13.621763229370117, "KL/mean": -28.440505981445312, "KL/rejected_KL_mean": -43.259239196777344, "KL/std": 28.2391357421875, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.011893379501998425, "fcm_dpo/delta": -0.11567549407482147, "fcm_dpo/margin": 29.637487411499023, "fcm_dpo/q_t": 0.3860425055027008, "grad_norm": 16.972978591918945, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.6723306179046631, "logits/rejected": -0.6689744591712952, "logps/chosen": -65.48342895507812, "logps/ref_chosen": -51.861663818359375, "logps/ref_rejected": -111.25398254394531, "logps/rejected": -154.51321411132812, "loss": 1.1449, "margin_dpo/margin_mean": 29.63748550415039, "margin_dpo/margin_std": 32.814430236816406, "step": 75 }, { "KL/chosen_KL_mean": -16.395811080932617, "KL/mean": -25.452667236328125, "KL/rejected_KL_mean": -34.509517669677734, "KL/std": 23.012611389160156, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.113712310791016, "fcm_dpo/q_t": 0.4790630340576172, "grad_norm": 7.068286895751953, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.7196544408798218, "logits/rejected": -0.6837272644042969, "logps/chosen": -69.66184997558594, "logps/ref_chosen": -53.26603698730469, "logps/ref_rejected": -78.21662902832031, "logps/rejected": -112.72615051269531, "loss": 1.3076, "margin_dpo/margin_mean": 18.113712310791016, "margin_dpo/margin_std": 25.670862197875977, "step": 76 }, { "KL/chosen_KL_mean": -15.274223327636719, "KL/mean": -30.40639877319336, "KL/rejected_KL_mean": -45.53857421875, "KL/std": 31.36872100830078, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.26435089111328, "fcm_dpo/q_t": 0.4651873707771301, "grad_norm": 9.03381633758545, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.6737496256828308, "logits/rejected": -0.639689564704895, "logps/chosen": -73.37090301513672, "logps/ref_chosen": -58.0966796875, "logps/ref_rejected": -93.77361297607422, "logps/rejected": -139.31219482421875, "loss": 1.2578, "margin_dpo/margin_mean": 30.26434898376465, "margin_dpo/margin_std": 35.96830749511719, "step": 77 }, { "KL/chosen_KL_mean": -15.677688598632812, "KL/mean": -26.735885620117188, "KL/rejected_KL_mean": -37.79408264160156, "KL/std": 26.46023941040039, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.116390228271484, "fcm_dpo/q_t": 0.4745638370513916, "grad_norm": 7.281389236450195, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.644652247428894, "logits/rejected": -0.6242761015892029, "logps/chosen": -71.29147338867188, "logps/ref_chosen": -55.61378479003906, "logps/ref_rejected": -84.93436431884766, "logps/rejected": -122.72844696044922, "loss": 1.2917, "margin_dpo/margin_mean": 22.116392135620117, "margin_dpo/margin_std": 31.19441795349121, "step": 78 }, { "KL/chosen_KL_mean": -16.822900772094727, "KL/mean": -28.855121612548828, "KL/rejected_KL_mean": -40.8873405456543, "KL/std": 29.233537673950195, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.064435958862305, "fcm_dpo/q_t": 0.4723590612411499, "grad_norm": 7.421467304229736, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6622911095619202, "logits/rejected": -0.6371140480041504, "logps/chosen": -72.27339172363281, "logps/ref_chosen": -55.45048522949219, "logps/ref_rejected": -87.64756774902344, "logps/rejected": -128.534912109375, "loss": 1.2846, "margin_dpo/margin_mean": 24.064434051513672, "margin_dpo/margin_std": 35.614994049072266, "step": 79 }, { "KL/chosen_KL_mean": -21.039852142333984, "KL/mean": -33.44929504394531, "KL/rejected_KL_mean": -45.858741760253906, "KL/std": 33.63664245605469, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.818889617919922, "fcm_dpo/q_t": 0.4716973304748535, "grad_norm": 8.430150032043457, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7373731732368469, "logits/rejected": -0.7053723335266113, "logps/chosen": -79.55914306640625, "logps/ref_chosen": -58.519290924072266, "logps/ref_rejected": -87.54750061035156, "logps/rejected": -133.40625, "loss": 1.2836, "margin_dpo/margin_mean": 24.818889617919922, "margin_dpo/margin_std": 39.66154098510742, "step": 80 }, { "KL/chosen_KL_mean": -21.12224578857422, "KL/mean": -38.02497863769531, "KL/rejected_KL_mean": -54.927711486816406, "KL/std": 39.90003204345703, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.80546188354492, "fcm_dpo/q_t": 0.4615294337272644, "grad_norm": 9.538393020629883, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6799561977386475, "logits/rejected": -0.6571604013442993, "logps/chosen": -87.57111358642578, "logps/ref_chosen": -66.44886779785156, "logps/ref_rejected": -129.66270446777344, "logps/rejected": -184.59042358398438, "loss": 1.2478, "margin_dpo/margin_mean": 33.80546188354492, "margin_dpo/margin_std": 47.89176940917969, "step": 81 }, { "KL/chosen_KL_mean": -24.305784225463867, "KL/mean": -37.62807846069336, "KL/rejected_KL_mean": -50.950374603271484, "KL/std": 36.823760986328125, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.64459228515625, "fcm_dpo/q_t": 0.4697510600090027, "grad_norm": 8.230402946472168, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.6198301315307617, "logits/rejected": -0.6155990958213806, "logps/chosen": -76.53816986083984, "logps/ref_chosen": -52.232383728027344, "logps/ref_rejected": -90.74325561523438, "logps/rejected": -141.69363403320312, "loss": 1.2812, "margin_dpo/margin_mean": 26.644596099853516, "margin_dpo/margin_std": 51.493133544921875, "step": 82 }, { "KL/chosen_KL_mean": -25.275197982788086, "KL/mean": -42.2493782043457, "KL/rejected_KL_mean": -59.22356033325195, "KL/std": 38.86623001098633, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.9483642578125, "fcm_dpo/q_t": 0.4612438678741455, "grad_norm": 9.907917022705078, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6601589322090149, "logits/rejected": -0.6394397616386414, "logps/chosen": -81.10258483886719, "logps/ref_chosen": -55.82738494873047, "logps/ref_rejected": -103.71589660644531, "logps/rejected": -162.939453125, "loss": 1.2466, "margin_dpo/margin_mean": 33.948368072509766, "margin_dpo/margin_std": 46.71687698364258, "step": 83 }, { "KL/chosen_KL_mean": -26.676267623901367, "KL/mean": -39.81716537475586, "KL/rejected_KL_mean": -52.958065032958984, "KL/std": 34.16858673095703, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.28179359436035, "fcm_dpo/q_t": 0.46975940465927124, "grad_norm": 8.419811248779297, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.6571387052536011, "logits/rejected": -0.6230216026306152, "logps/chosen": -93.85243225097656, "logps/ref_chosen": -67.1761703491211, "logps/ref_rejected": -87.29859924316406, "logps/rejected": -140.2566680908203, "loss": 1.2748, "margin_dpo/margin_mean": 26.281795501708984, "margin_dpo/margin_std": 33.972564697265625, "step": 84 }, { "KL/chosen_KL_mean": -26.707151412963867, "KL/mean": -40.551788330078125, "KL/rejected_KL_mean": -54.396427154541016, "KL/std": 34.13534927368164, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.689273834228516, "fcm_dpo/q_t": 0.4682667851448059, "grad_norm": 8.280872344970703, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.6289738416671753, "logits/rejected": -0.5984049439430237, "logps/chosen": -85.11376953125, "logps/ref_chosen": -58.4066162109375, "logps/ref_rejected": -78.63880157470703, "logps/rejected": -133.0352325439453, "loss": 1.2717, "margin_dpo/margin_mean": 27.689273834228516, "margin_dpo/margin_std": 41.85002136230469, "step": 85 }, { "KL/chosen_KL_mean": -36.066551208496094, "KL/mean": -50.90142822265625, "KL/rejected_KL_mean": -65.7363052368164, "KL/std": 51.3672981262207, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.004640421364456415, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.669748306274414, "fcm_dpo/q_t": 0.46719542145729065, "grad_norm": 9.010835647583008, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6406770348548889, "logits/rejected": -0.6358238458633423, "logps/chosen": -92.20401000976562, "logps/ref_chosen": -56.13746643066406, "logps/ref_rejected": -88.12165069580078, "logps/rejected": -153.8579559326172, "loss": 1.2809, "margin_dpo/margin_mean": 29.669748306274414, "margin_dpo/margin_std": 72.3070068359375, "step": 86 }, { "KL/chosen_KL_mean": -35.95440673828125, "KL/mean": -55.21514129638672, "KL/rejected_KL_mean": -74.47587585449219, "KL/std": 52.28978729248047, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.012798674404621124, "fcm_dpo/delta": 0.10860809683799744, "fcm_dpo/margin": 38.52145767211914, "fcm_dpo/q_t": 0.433593213558197, "grad_norm": 24.902225494384766, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.6695064306259155, "logits/rejected": -0.6587918996810913, "logps/chosen": -91.59049987792969, "logps/ref_chosen": -55.63609313964844, "logps/ref_rejected": -95.46757507324219, "logps/rejected": -169.94345092773438, "loss": 1.1243, "margin_dpo/margin_mean": 38.52145767211914, "margin_dpo/margin_std": 65.65547943115234, "step": 87 }, { "KL/chosen_KL_mean": -46.075992584228516, "KL/mean": -60.8734245300293, "KL/rejected_KL_mean": -75.67085266113281, "KL/std": 57.557559967041016, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.007393650244921446, "fcm_dpo/delta": -0.062026459723711014, "fcm_dpo/margin": 29.5948543548584, "fcm_dpo/q_t": 0.4339485168457031, "grad_norm": 14.445233345031738, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.6514161825180054, "logits/rejected": -0.6543829441070557, "logps/chosen": -119.74714660644531, "logps/ref_chosen": -73.67115020751953, "logps/ref_rejected": -106.70849609375, "logps/rejected": -182.3793487548828, "loss": 1.2408, "margin_dpo/margin_mean": 29.594852447509766, "margin_dpo/margin_std": 64.25787353515625, "step": 88 }, { "KL/chosen_KL_mean": -32.12773132324219, "KL/mean": -48.858558654785156, "KL/rejected_KL_mean": -65.5893783569336, "KL/std": 51.16180419921875, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.009811213240027428, "fcm_dpo/delta": 0.05031702667474747, "fcm_dpo/margin": 33.46165466308594, "fcm_dpo/q_t": 0.44110527634620667, "grad_norm": 17.186080932617188, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.6749956607818604, "logits/rejected": -0.6535379886627197, "logps/chosen": -92.75264739990234, "logps/ref_chosen": -60.624916076660156, "logps/ref_rejected": -82.08354949951172, "logps/rejected": -147.6729278564453, "loss": 1.1533, "margin_dpo/margin_mean": 33.46165466308594, "margin_dpo/margin_std": 60.91192626953125, "step": 89 }, { "KL/chosen_KL_mean": -36.43550109863281, "KL/mean": -62.53802490234375, "KL/rejected_KL_mean": -88.64056396484375, "KL/std": 68.16754913330078, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.0063932533375918865, "fcm_dpo/delta": -0.19979318976402283, "fcm_dpo/margin": 52.205047607421875, "fcm_dpo/q_t": 0.38456064462661743, "grad_norm": 15.716353416442871, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6015010476112366, "logits/rejected": -0.6259626150131226, "logps/chosen": -89.72080993652344, "logps/ref_chosen": -53.285308837890625, "logps/ref_rejected": -111.54470825195312, "logps/rejected": -200.18527221679688, "loss": 1.2609, "margin_dpo/margin_mean": 52.20505142211914, "margin_dpo/margin_std": 86.0552978515625, "step": 90 }, { "KL/chosen_KL_mean": -40.36619567871094, "KL/mean": -57.59088897705078, "KL/rejected_KL_mean": -74.81558227539062, "KL/std": 51.144317626953125, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.44939041137695, "fcm_dpo/q_t": 0.4857920706272125, "grad_norm": 3.791431188583374, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6380863189697266, "logits/rejected": -0.6169267892837524, "logps/chosen": -102.16915130615234, "logps/ref_chosen": -61.802955627441406, "logps/ref_rejected": -87.87395477294922, "logps/rejected": -162.68954467773438, "loss": 1.3334, "margin_dpo/margin_mean": 34.44939041137695, "margin_dpo/margin_std": 70.18351745605469, "step": 91 }, { "KL/chosen_KL_mean": -35.530731201171875, "KL/mean": -54.40737533569336, "KL/rejected_KL_mean": -73.28401184082031, "KL/std": 54.11292266845703, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 37.7532844543457, "fcm_dpo/q_t": 0.48441988229751587, "grad_norm": 4.081094264984131, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.6290682554244995, "logits/rejected": -0.6120110750198364, "logps/chosen": -87.17150115966797, "logps/ref_chosen": -51.640769958496094, "logps/ref_rejected": -77.88117980957031, "logps/rejected": -151.16519165039062, "loss": 1.3277, "margin_dpo/margin_mean": 37.7532844543457, "margin_dpo/margin_std": 65.40171813964844, "step": 92 }, { "KL/chosen_KL_mean": -39.78628158569336, "KL/mean": -58.02899169921875, "KL/rejected_KL_mean": -76.27169799804688, "KL/std": 48.12828063964844, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.48542404174805, "fcm_dpo/q_t": 0.48493102192878723, "grad_norm": 4.032928466796875, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.6539796590805054, "logits/rejected": -0.6243857145309448, "logps/chosen": -92.31552124023438, "logps/ref_chosen": -52.529239654541016, "logps/ref_rejected": -77.16075134277344, "logps/rejected": -153.4324493408203, "loss": 1.329, "margin_dpo/margin_mean": 36.48542404174805, "margin_dpo/margin_std": 57.570045471191406, "step": 93 }, { "KL/chosen_KL_mean": -43.512413024902344, "KL/mean": -66.17471313476562, "KL/rejected_KL_mean": -88.83700561523438, "KL/std": 59.194549560546875, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.32459259033203, "fcm_dpo/q_t": 0.4812876582145691, "grad_norm": 4.497649192810059, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.5847138166427612, "logits/rejected": -0.5839489698410034, "logps/chosen": -104.73502349853516, "logps/ref_chosen": -61.22261047363281, "logps/ref_rejected": -99.59902954101562, "logps/rejected": -188.43603515625, "loss": 1.3155, "margin_dpo/margin_mean": 45.32460021972656, "margin_dpo/margin_std": 64.99853515625, "step": 94 }, { "KL/chosen_KL_mean": -43.288063049316406, "KL/mean": -61.76789855957031, "KL/rejected_KL_mean": -80.24773406982422, "KL/std": 63.553306579589844, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.95966720581055, "fcm_dpo/q_t": 0.48472627997398376, "grad_norm": 4.214771270751953, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.6216604113578796, "logits/rejected": -0.6044141054153442, "logps/chosen": -95.81170654296875, "logps/ref_chosen": -52.523643493652344, "logps/ref_rejected": -75.8803482055664, "logps/rejected": -156.12808227539062, "loss": 1.3292, "margin_dpo/margin_mean": 36.95966339111328, "margin_dpo/margin_std": 67.57984161376953, "step": 95 }, { "KL/chosen_KL_mean": -46.38825225830078, "KL/mean": -72.26142883300781, "KL/rejected_KL_mean": -98.13460540771484, "KL/std": 66.60676574707031, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.74634552001953, "fcm_dpo/q_t": 0.4787074327468872, "grad_norm": 4.915516376495361, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.5953140258789062, "logits/rejected": -0.5802059173583984, "logps/chosen": -108.54522705078125, "logps/ref_chosen": -62.15697479248047, "logps/ref_rejected": -96.59601593017578, "logps/rejected": -194.73062133789062, "loss": 1.3066, "margin_dpo/margin_mean": 51.74634552001953, "margin_dpo/margin_std": 77.54536437988281, "step": 96 }, { "KL/chosen_KL_mean": -47.02070999145508, "KL/mean": -65.5418701171875, "KL/rejected_KL_mean": -84.06302642822266, "KL/std": 60.283546447753906, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 37.042320251464844, "fcm_dpo/q_t": 0.4846979081630707, "grad_norm": 4.432089328765869, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.4999476671218872, "logits/rejected": -0.4703494906425476, "logps/chosen": -101.66707611083984, "logps/ref_chosen": -54.646366119384766, "logps/ref_rejected": -76.96475219726562, "logps/rejected": -161.02777099609375, "loss": 1.3285, "margin_dpo/margin_mean": 37.042320251464844, "margin_dpo/margin_std": 60.82837677001953, "step": 97 }, { "KL/chosen_KL_mean": -63.49620056152344, "KL/mean": -85.60096740722656, "KL/rejected_KL_mean": -107.70573425292969, "KL/std": 73.20948028564453, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.20954132080078, "fcm_dpo/q_t": 0.4817187786102295, "grad_norm": 5.483908653259277, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.5312040448188782, "logits/rejected": -0.5291551947593689, "logps/chosen": -128.75482177734375, "logps/ref_chosen": -65.25862884521484, "logps/ref_rejected": -96.5274887084961, "logps/rejected": -204.23321533203125, "loss": 1.3201, "margin_dpo/margin_mean": 44.20954132080078, "margin_dpo/margin_std": 89.74884033203125, "step": 98 }, { "KL/chosen_KL_mean": -47.63540267944336, "KL/mean": -77.33531188964844, "KL/rejected_KL_mean": -107.03522491455078, "KL/std": 73.14799499511719, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.39982604980469, "fcm_dpo/q_t": 0.47561943531036377, "grad_norm": 5.031144618988037, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.5268571376800537, "logits/rejected": -0.5339376926422119, "logps/chosen": -93.27388763427734, "logps/ref_chosen": -45.638484954833984, "logps/ref_rejected": -86.43793487548828, "logps/rejected": -193.47315979003906, "loss": 1.2962, "margin_dpo/margin_mean": 59.39982604980469, "margin_dpo/margin_std": 93.68307495117188, "step": 99 }, { "KL/chosen_KL_mean": -58.90423583984375, "KL/mean": -70.80164337158203, "KL/rejected_KL_mean": -82.69905090332031, "KL/std": 66.31571960449219, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.79481315612793, "fcm_dpo/q_t": 0.49016737937927246, "grad_norm": 4.631414413452148, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.5172683596611023, "logits/rejected": -0.4957023859024048, "logps/chosen": -116.49821472167969, "logps/ref_chosen": -57.59397888183594, "logps/ref_rejected": -74.06021118164062, "logps/rejected": -156.75926208496094, "loss": 1.3523, "margin_dpo/margin_mean": 23.794811248779297, "margin_dpo/margin_std": 82.77680206298828, "step": 100 }, { "KL/chosen_KL_mean": -70.8927230834961, "KL/mean": -88.97613525390625, "KL/rejected_KL_mean": -107.05953979492188, "KL/std": 81.61466979980469, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.166805267333984, "fcm_dpo/q_t": 0.48512861132621765, "grad_norm": 4.663822650909424, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.5123007297515869, "logits/rejected": -0.4988871216773987, "logps/chosen": -132.5415802001953, "logps/ref_chosen": -61.64885330200195, "logps/ref_rejected": -83.18968200683594, "logps/rejected": -190.24923706054688, "loss": 1.3335, "margin_dpo/margin_mean": 36.16680908203125, "margin_dpo/margin_std": 94.42303466796875, "step": 101 }, { "KL/chosen_KL_mean": -74.67913818359375, "KL/mean": -87.16828918457031, "KL/rejected_KL_mean": -99.65742492675781, "KL/std": 79.29058837890625, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.978286743164062, "fcm_dpo/q_t": 0.489687979221344, "grad_norm": 5.5214524269104, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.5297523140907288, "logits/rejected": -0.5086390972137451, "logps/chosen": -138.7580108642578, "logps/ref_chosen": -64.0788803100586, "logps/ref_rejected": -68.18707275390625, "logps/rejected": -167.84451293945312, "loss": 1.352, "margin_dpo/margin_mean": 24.978286743164062, "margin_dpo/margin_std": 96.62683868408203, "step": 102 }, { "KL/chosen_KL_mean": -69.34735870361328, "KL/mean": -98.80657958984375, "KL/rejected_KL_mean": -128.26580810546875, "KL/std": 92.38734436035156, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.918426513671875, "fcm_dpo/q_t": 0.47593387961387634, "grad_norm": 5.950850963592529, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.46111953258514404, "logits/rejected": -0.441642165184021, "logps/chosen": -130.64663696289062, "logps/ref_chosen": -61.299278259277344, "logps/ref_rejected": -93.57270812988281, "logps/rejected": -221.8385009765625, "loss": 1.3007, "margin_dpo/margin_mean": 58.918426513671875, "margin_dpo/margin_std": 119.41735076904297, "step": 103 }, { "KL/chosen_KL_mean": -67.7513198852539, "KL/mean": -96.42362976074219, "KL/rejected_KL_mean": -125.09593200683594, "KL/std": 93.0230712890625, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.34462356567383, "fcm_dpo/q_t": 0.476635605096817, "grad_norm": 8.443946838378906, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.4595398008823395, "logits/rejected": -0.4493861794471741, "logps/chosen": -122.12409210205078, "logps/ref_chosen": -54.372772216796875, "logps/ref_rejected": -89.5647201538086, "logps/rejected": -214.66064453125, "loss": 1.3031, "margin_dpo/margin_mean": 57.344627380371094, "margin_dpo/margin_std": 118.23411560058594, "step": 104 }, { "KL/chosen_KL_mean": -68.69507598876953, "KL/mean": -111.2301254272461, "KL/rejected_KL_mean": -153.76516723632812, "KL/std": 86.79656219482422, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.07008361816406, "fcm_dpo/q_t": 0.46504706144332886, "grad_norm": 6.229670524597168, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.4372522830963135, "logits/rejected": -0.4284954369068146, "logps/chosen": -123.33402252197266, "logps/ref_chosen": -54.638946533203125, "logps/ref_rejected": -97.97351837158203, "logps/rejected": -251.7386932373047, "loss": 1.2576, "margin_dpo/margin_mean": 85.07008361816406, "margin_dpo/margin_std": 102.4740219116211, "step": 105 }, { "KL/chosen_KL_mean": -80.03727722167969, "KL/mean": -107.10738372802734, "KL/rejected_KL_mean": -134.177490234375, "KL/std": 90.39563751220703, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.140220642089844, "fcm_dpo/q_t": 0.47773557901382446, "grad_norm": 5.816026210784912, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.38607245683670044, "logits/rejected": -0.3913835883140564, "logps/chosen": -134.87017822265625, "logps/ref_chosen": -54.83289337158203, "logps/ref_rejected": -85.22461700439453, "logps/rejected": -219.40211486816406, "loss": 1.3052, "margin_dpo/margin_mean": 54.14021682739258, "margin_dpo/margin_std": 96.50797271728516, "step": 106 }, { "KL/chosen_KL_mean": -96.32496643066406, "KL/mean": -126.12309265136719, "KL/rejected_KL_mean": -155.92123413085938, "KL/std": 109.03052520751953, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.59624481201172, "fcm_dpo/q_t": 0.47559940814971924, "grad_norm": 6.986702919006348, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.42685896158218384, "logits/rejected": -0.4117262363433838, "logps/chosen": -166.03277587890625, "logps/ref_chosen": -69.70780944824219, "logps/ref_rejected": -94.73950958251953, "logps/rejected": -250.66073608398438, "loss": 1.2995, "margin_dpo/margin_mean": 59.596248626708984, "margin_dpo/margin_std": 115.8310775756836, "step": 107 }, { "KL/chosen_KL_mean": -79.45794677734375, "KL/mean": -121.89816284179688, "KL/rejected_KL_mean": -164.33836364746094, "KL/std": 121.62930297851562, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.88043212890625, "fcm_dpo/q_t": 0.4657972455024719, "grad_norm": 8.013676643371582, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.36991098523139954, "logits/rejected": -0.36751043796539307, "logps/chosen": -135.46783447265625, "logps/ref_chosen": -56.0098876953125, "logps/ref_rejected": -95.79601287841797, "logps/rejected": -260.1343994140625, "loss": 1.2671, "margin_dpo/margin_mean": 84.88043212890625, "margin_dpo/margin_std": 156.81539916992188, "step": 108 }, { "KL/chosen_KL_mean": -85.31330871582031, "KL/mean": -126.01399230957031, "KL/rejected_KL_mean": -166.71469116210938, "KL/std": 120.20221710205078, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.40138244628906, "fcm_dpo/q_t": 0.4666248559951782, "grad_norm": 6.836396217346191, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.36495447158813477, "logits/rejected": -0.3560246229171753, "logps/chosen": -148.19879150390625, "logps/ref_chosen": -62.88549041748047, "logps/ref_rejected": -98.68573760986328, "logps/rejected": -265.4004211425781, "loss": 1.2734, "margin_dpo/margin_mean": 81.40138244628906, "margin_dpo/margin_std": 159.874755859375, "step": 109 }, { "KL/chosen_KL_mean": -91.46940612792969, "KL/mean": -125.52250671386719, "KL/rejected_KL_mean": -159.5756072998047, "KL/std": 117.68600463867188, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.10619354248047, "fcm_dpo/q_t": 0.4723784029483795, "grad_norm": 8.115654945373535, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.32437682151794434, "logits/rejected": -0.3229568600654602, "logps/chosen": -150.22308349609375, "logps/ref_chosen": -58.753684997558594, "logps/ref_rejected": -79.75001525878906, "logps/rejected": -239.32562255859375, "loss": 1.2933, "margin_dpo/margin_mean": 68.10619354248047, "margin_dpo/margin_std": 149.91207885742188, "step": 110 }, { "KL/chosen_KL_mean": -101.22291564941406, "KL/mean": -143.97872924804688, "KL/rejected_KL_mean": -186.7345733642578, "KL/std": 137.13291931152344, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.51166534423828, "fcm_dpo/q_t": 0.46532922983169556, "grad_norm": 7.223021507263184, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.33777564764022827, "logits/rejected": -0.33077672123908997, "logps/chosen": -169.84701538085938, "logps/ref_chosen": -68.62410736083984, "logps/ref_rejected": -98.42886352539062, "logps/rejected": -285.1634521484375, "loss": 1.2733, "margin_dpo/margin_mean": 85.51167297363281, "margin_dpo/margin_std": 186.15402221679688, "step": 111 }, { "KL/chosen_KL_mean": -90.35836029052734, "KL/mean": -112.38624572753906, "KL/rejected_KL_mean": -134.4141387939453, "KL/std": 86.84256744384766, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.05577087402344, "fcm_dpo/q_t": 0.4818727374076843, "grad_norm": 7.099332809448242, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.2616153955459595, "logits/rejected": -0.2658056318759918, "logps/chosen": -140.60800170898438, "logps/ref_chosen": -50.24964141845703, "logps/ref_rejected": -64.77442932128906, "logps/rejected": -199.18856811523438, "loss": 1.3241, "margin_dpo/margin_mean": 44.05577087402344, "margin_dpo/margin_std": 115.68550872802734, "step": 112 }, { "KL/chosen_KL_mean": -115.47476196289062, "KL/mean": -136.68405151367188, "KL/rejected_KL_mean": -157.8933563232422, "KL/std": 115.38925170898438, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.41858673095703, "fcm_dpo/q_t": 0.4827752709388733, "grad_norm": 6.795632839202881, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.28555989265441895, "logits/rejected": -0.27538594603538513, "logps/chosen": -182.18771362304688, "logps/ref_chosen": -66.71295166015625, "logps/ref_rejected": -77.96870422363281, "logps/rejected": -235.862060546875, "loss": 1.327, "margin_dpo/margin_mean": 42.41858673095703, "margin_dpo/margin_std": 120.11697387695312, "step": 113 }, { "KL/chosen_KL_mean": -96.08192443847656, "KL/mean": -136.51783752441406, "KL/rejected_KL_mean": -176.95375061035156, "KL/std": 128.7473602294922, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.87183380126953, "fcm_dpo/q_t": 0.4677523374557495, "grad_norm": 7.049834251403809, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.2732029855251312, "logits/rejected": -0.2635844945907593, "logps/chosen": -153.86700439453125, "logps/ref_chosen": -57.78507995605469, "logps/ref_rejected": -87.10966491699219, "logps/rejected": -264.06341552734375, "loss": 1.2745, "margin_dpo/margin_mean": 80.87183380126953, "margin_dpo/margin_std": 159.99867248535156, "step": 114 }, { "KL/chosen_KL_mean": -116.97596740722656, "KL/mean": -145.7581787109375, "KL/rejected_KL_mean": -174.54037475585938, "KL/std": 134.91941833496094, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.56439971923828, "fcm_dpo/q_t": 0.4768410921096802, "grad_norm": 6.489831924438477, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.31962960958480835, "logits/rejected": -0.32673972845077515, "logps/chosen": -182.55862426757812, "logps/ref_chosen": -65.5826416015625, "logps/ref_rejected": -98.56552124023438, "logps/rejected": -273.10589599609375, "loss": 1.3161, "margin_dpo/margin_mean": 57.56439971923828, "margin_dpo/margin_std": 182.9209442138672, "step": 115 }, { "KL/chosen_KL_mean": -81.60614013671875, "KL/mean": -113.20507049560547, "KL/rejected_KL_mean": -144.80401611328125, "KL/std": 99.23644256591797, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.19786834716797, "fcm_dpo/q_t": 0.47415468096733093, "grad_norm": 7.067198276519775, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.31514549255371094, "logits/rejected": -0.32795295119285583, "logps/chosen": -133.00645446777344, "logps/ref_chosen": -51.40031433105469, "logps/ref_rejected": -80.5218505859375, "logps/rejected": -225.32586669921875, "loss": 1.2933, "margin_dpo/margin_mean": 63.19786071777344, "margin_dpo/margin_std": 112.28499603271484, "step": 116 }, { "KL/chosen_KL_mean": -113.90751647949219, "KL/mean": -129.44996643066406, "KL/rejected_KL_mean": -144.99240112304688, "KL/std": 108.0283203125, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.08489227294922, "fcm_dpo/q_t": 0.4872492551803589, "grad_norm": 7.091765880584717, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.3172186017036438, "logits/rejected": -0.3100709021091461, "logps/chosen": -183.2059326171875, "logps/ref_chosen": -69.29840850830078, "logps/ref_rejected": -66.583984375, "logps/rejected": -211.57638549804688, "loss": 1.3476, "margin_dpo/margin_mean": 31.084890365600586, "margin_dpo/margin_std": 132.21383666992188, "step": 117 }, { "KL/chosen_KL_mean": -81.68537902832031, "KL/mean": -112.67160034179688, "KL/rejected_KL_mean": -143.6578369140625, "KL/std": 100.91627502441406, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.97246551513672, "fcm_dpo/q_t": 0.4746900796890259, "grad_norm": 6.520020008087158, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.3794967532157898, "logits/rejected": -0.37046414613723755, "logps/chosen": -137.32635498046875, "logps/ref_chosen": -55.6409797668457, "logps/ref_rejected": -75.66905975341797, "logps/rejected": -219.32688903808594, "loss": 1.2958, "margin_dpo/margin_mean": 61.97245788574219, "margin_dpo/margin_std": 119.08011627197266, "step": 118 }, { "KL/chosen_KL_mean": -119.08765411376953, "KL/mean": -146.5293426513672, "KL/rejected_KL_mean": -173.97103881835938, "KL/std": 110.69290924072266, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.88337326049805, "fcm_dpo/q_t": 0.4776170551776886, "grad_norm": 6.680598735809326, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.3600790500640869, "logits/rejected": -0.36749690771102905, "logps/chosen": -192.59783935546875, "logps/ref_chosen": -73.51019287109375, "logps/ref_rejected": -102.977294921875, "logps/rejected": -276.9483337402344, "loss": 1.3114, "margin_dpo/margin_mean": 54.88337326049805, "margin_dpo/margin_std": 140.7128448486328, "step": 119 }, { "KL/chosen_KL_mean": -117.65304565429688, "KL/mean": -159.9397735595703, "KL/rejected_KL_mean": -202.22650146484375, "KL/std": 132.9193115234375, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.0016584109980612993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.57344055175781, "fcm_dpo/q_t": 0.46578913927078247, "grad_norm": 8.834454536437988, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.3894355893135071, "logits/rejected": -0.38755276799201965, "logps/chosen": -194.4338836669922, "logps/ref_chosen": -76.78083801269531, "logps/ref_rejected": -108.02374267578125, "logps/rejected": -310.250244140625, "loss": 1.2665, "margin_dpo/margin_mean": 84.57344055175781, "margin_dpo/margin_std": 151.76153564453125, "step": 120 }, { "KL/chosen_KL_mean": -108.66337585449219, "KL/mean": -158.3170928955078, "KL/rejected_KL_mean": -207.9708251953125, "KL/std": 133.39573669433594, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.005431335419416428, "fcm_dpo/delta": 0.13696809113025665, "fcm_dpo/margin": 99.30744934082031, "fcm_dpo/q_t": 0.43542176485061646, "grad_norm": 21.185283660888672, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.3457489609718323, "logits/rejected": -0.348524272441864, "logps/chosen": -170.45327758789062, "logps/ref_chosen": -61.789894104003906, "logps/ref_rejected": -109.99456787109375, "logps/rejected": -317.96539306640625, "loss": 1.096, "margin_dpo/margin_mean": 99.30744934082031, "margin_dpo/margin_std": 157.79302978515625, "step": 121 }, { "KL/chosen_KL_mean": -86.52001953125, "KL/mean": -148.9921112060547, "KL/rejected_KL_mean": -211.46420288085938, "KL/std": 132.5182342529297, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": -0.16569742560386658, "fcm_dpo/margin": 124.94419860839844, "fcm_dpo/q_t": 0.40711045265197754, "grad_norm": 7.291841983795166, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.30201756954193115, "logits/rejected": -0.31801754236221313, "logps/chosen": -133.42222595214844, "logps/ref_chosen": -46.9022102355957, "logps/ref_rejected": -106.71418762207031, "logps/rejected": -318.17840576171875, "loss": 1.2453, "margin_dpo/margin_mean": 124.9441909790039, "margin_dpo/margin_std": 147.9832305908203, "step": 122 }, { "KL/chosen_KL_mean": -110.82725524902344, "KL/mean": -149.17156982421875, "KL/rejected_KL_mean": -187.515869140625, "KL/std": 125.73814392089844, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.68860626220703, "fcm_dpo/q_t": 0.4764896631240845, "grad_norm": 5.018327236175537, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.2894030809402466, "logits/rejected": -0.29247063398361206, "logps/chosen": -172.1658935546875, "logps/ref_chosen": -61.33863830566406, "logps/ref_rejected": -87.775390625, "logps/rejected": -275.291259765625, "loss": 1.3046, "margin_dpo/margin_mean": 76.68860626220703, "margin_dpo/margin_std": 171.46743774414062, "step": 123 }, { "KL/chosen_KL_mean": -133.59381103515625, "KL/mean": -191.1050262451172, "KL/rejected_KL_mean": -248.61622619628906, "KL/std": 179.1632080078125, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.02241516113281, "fcm_dpo/q_t": 0.46520543098449707, "grad_norm": 5.619283676147461, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.25618571043014526, "logits/rejected": -0.2648266553878784, "logps/chosen": -205.04214477539062, "logps/ref_chosen": -71.44833374023438, "logps/ref_rejected": -117.58056640625, "logps/rejected": -366.19677734375, "loss": 1.269, "margin_dpo/margin_mean": 115.02241516113281, "margin_dpo/margin_std": 230.38388061523438, "step": 124 }, { "KL/chosen_KL_mean": -101.74983215332031, "KL/mean": -149.4581298828125, "KL/rejected_KL_mean": -197.16644287109375, "KL/std": 146.47589111328125, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.41659545898438, "fcm_dpo/q_t": 0.4708002507686615, "grad_norm": 5.07564640045166, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.18893632292747498, "logits/rejected": -0.20313698053359985, "logps/chosen": -151.88677978515625, "logps/ref_chosen": -50.136940002441406, "logps/ref_rejected": -83.98861694335938, "logps/rejected": -281.1550598144531, "loss": 1.2842, "margin_dpo/margin_mean": 95.41659545898438, "margin_dpo/margin_std": 181.84890747070312, "step": 125 }, { "KL/chosen_KL_mean": -110.52033996582031, "KL/mean": -162.48785400390625, "KL/rejected_KL_mean": -214.45535278320312, "KL/std": 144.8026123046875, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.93502807617188, "fcm_dpo/q_t": 0.46845173835754395, "grad_norm": 5.675121307373047, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.2591179609298706, "logits/rejected": -0.27047020196914673, "logps/chosen": -166.18740844726562, "logps/ref_chosen": -55.66706848144531, "logps/ref_rejected": -98.1297607421875, "logps/rejected": -312.5851135253906, "loss": 1.2746, "margin_dpo/margin_mean": 103.93502807617188, "margin_dpo/margin_std": 185.01034545898438, "step": 126 }, { "KL/chosen_KL_mean": -114.12653350830078, "KL/mean": -142.77590942382812, "KL/rejected_KL_mean": -171.42526245117188, "KL/std": 126.23076629638672, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.298728942871094, "fcm_dpo/q_t": 0.4821869730949402, "grad_norm": 5.324838638305664, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.2520139515399933, "logits/rejected": -0.25741541385650635, "logps/chosen": -170.68121337890625, "logps/ref_chosen": -56.55467987060547, "logps/ref_rejected": -76.7957763671875, "logps/rejected": -248.22103881835938, "loss": 1.3258, "margin_dpo/margin_mean": 57.298728942871094, "margin_dpo/margin_std": 157.20663452148438, "step": 127 }, { "KL/chosen_KL_mean": -117.71072387695312, "KL/mean": -154.01956176757812, "KL/rejected_KL_mean": -190.32843017578125, "KL/std": 155.84530639648438, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.61770629882812, "fcm_dpo/q_t": 0.4778425097465515, "grad_norm": 6.066002368927002, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.23010198771953583, "logits/rejected": -0.24306002259254456, "logps/chosen": -175.83168029785156, "logps/ref_chosen": -58.12095642089844, "logps/ref_rejected": -76.43896484375, "logps/rejected": -266.76739501953125, "loss": 1.3139, "margin_dpo/margin_mean": 72.61770629882812, "margin_dpo/margin_std": 201.6669158935547, "step": 128 }, { "KL/chosen_KL_mean": -156.19540405273438, "KL/mean": -194.95504760742188, "KL/rejected_KL_mean": -233.71466064453125, "KL/std": 197.4521484375, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.51925659179688, "fcm_dpo/q_t": 0.47747713327407837, "grad_norm": 6.777254581451416, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.2056041806936264, "logits/rejected": -0.2097330093383789, "logps/chosen": -223.11178588867188, "logps/ref_chosen": -66.91637420654297, "logps/ref_rejected": -96.6422119140625, "logps/rejected": -330.35687255859375, "loss": 1.3193, "margin_dpo/margin_mean": 77.51925659179688, "margin_dpo/margin_std": 269.54656982421875, "step": 129 }, { "KL/chosen_KL_mean": -108.04925537109375, "KL/mean": -162.80902099609375, "KL/rejected_KL_mean": -217.56878662109375, "KL/std": 142.10467529296875, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.51951599121094, "fcm_dpo/q_t": 0.4666077494621277, "grad_norm": 7.042036533355713, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.15569573640823364, "logits/rejected": -0.15476679801940918, "logps/chosen": -152.7161102294922, "logps/ref_chosen": -44.66685104370117, "logps/ref_rejected": -82.78165435791016, "logps/rejected": -300.3504333496094, "loss": 1.2689, "margin_dpo/margin_mean": 109.51950073242188, "margin_dpo/margin_std": 188.17529296875, "step": 130 }, { "KL/chosen_KL_mean": -112.39668273925781, "KL/mean": -174.34365844726562, "KL/rejected_KL_mean": -236.29067993164062, "KL/std": 177.99996948242188, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.89396667480469, "fcm_dpo/q_t": 0.46196436882019043, "grad_norm": 7.167151927947998, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.15293475985527039, "logits/rejected": -0.15630626678466797, "logps/chosen": -157.32127380371094, "logps/ref_chosen": -44.924591064453125, "logps/ref_rejected": -88.44401550292969, "logps/rejected": -324.73468017578125, "loss": 1.2519, "margin_dpo/margin_mean": 123.89398193359375, "margin_dpo/margin_std": 189.91864013671875, "step": 131 }, { "KL/chosen_KL_mean": -142.0668487548828, "KL/mean": -188.7615966796875, "KL/rejected_KL_mean": -235.45635986328125, "KL/std": 182.57955932617188, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.3895034790039, "fcm_dpo/q_t": 0.47222959995269775, "grad_norm": 6.9561357498168945, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.1588452309370041, "logits/rejected": -0.1891343593597412, "logps/chosen": -201.06793212890625, "logps/ref_chosen": -59.00108337402344, "logps/ref_rejected": -87.89215087890625, "logps/rejected": -323.3485107421875, "loss": 1.295, "margin_dpo/margin_mean": 93.3895034790039, "margin_dpo/margin_std": 238.1549530029297, "step": 132 }, { "KL/chosen_KL_mean": -163.40499877929688, "KL/mean": -203.98348999023438, "KL/rejected_KL_mean": -244.56201171875, "KL/std": 153.83062744140625, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.15701293945312, "fcm_dpo/q_t": 0.47538745403289795, "grad_norm": 7.251523494720459, "learning_rate": 4.870401618977415e-07, "logits/chosen": -0.15137168765068054, "logits/rejected": -0.1648947298526764, "logps/chosen": -230.00949096679688, "logps/ref_chosen": -66.60449981689453, "logps/ref_rejected": -96.33355712890625, "logps/rejected": -340.89556884765625, "loss": 1.3037, "margin_dpo/margin_mean": 81.15701293945312, "margin_dpo/margin_std": 203.13357543945312, "step": 133 }, { "KL/chosen_KL_mean": -126.16668701171875, "KL/mean": -169.11993408203125, "KL/rejected_KL_mean": -212.0731964111328, "KL/std": 131.4796600341797, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.0012442920124158263, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.90650939941406, "fcm_dpo/q_t": 0.47373300790786743, "grad_norm": 7.264800071716309, "learning_rate": 4.866292092063986e-07, "logits/chosen": -0.1166238859295845, "logits/rejected": -0.13339810073375702, "logps/chosen": -178.23594665527344, "logps/ref_chosen": -52.06925582885742, "logps/ref_rejected": -87.6545181274414, "logps/rejected": -299.72772216796875, "loss": 1.2925, "margin_dpo/margin_mean": 85.9065170288086, "margin_dpo/margin_std": 161.20672607421875, "step": 134 }, { "KL/chosen_KL_mean": -130.97235107421875, "KL/mean": -212.07064819335938, "KL/rejected_KL_mean": -293.1689453125, "KL/std": 209.9551544189453, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.0025665950961411, "fcm_dpo/delta": 0.04211168363690376, "fcm_dpo/margin": 162.19659423828125, "fcm_dpo/q_t": 0.42933958768844604, "grad_norm": 16.52530288696289, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.09394662082195282, "logits/rejected": -0.10758954286575317, "logps/chosen": -181.32620239257812, "logps/ref_chosen": -50.353858947753906, "logps/ref_rejected": -115.97975158691406, "logps/rejected": -409.148681640625, "loss": 1.1038, "margin_dpo/margin_mean": 162.1966094970703, "margin_dpo/margin_std": 266.60357666015625, "step": 135 }, { "KL/chosen_KL_mean": -160.09901428222656, "KL/mean": -211.46200561523438, "KL/rejected_KL_mean": -262.8249816894531, "KL/std": 210.24295043945312, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.0018958807922899723, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.72596740722656, "fcm_dpo/q_t": 0.45904627442359924, "grad_norm": 9.262153625488281, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.1536802351474762, "logits/rejected": -0.16803759336471558, "logps/chosen": -225.17152404785156, "logps/ref_chosen": -65.072509765625, "logps/ref_rejected": -96.32122802734375, "logps/rejected": -359.1462097167969, "loss": 1.2667, "margin_dpo/margin_mean": 102.72596740722656, "margin_dpo/margin_std": 286.5235595703125, "step": 136 }, { "KL/chosen_KL_mean": -135.36795043945312, "KL/mean": -214.21568298339844, "KL/rejected_KL_mean": -293.06341552734375, "KL/std": 230.4893798828125, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.002469564788043499, "fcm_dpo/delta": -0.016328319907188416, "fcm_dpo/margin": 157.69546508789062, "fcm_dpo/q_t": 0.4195204973220825, "grad_norm": 12.261474609375, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.10306895524263382, "logits/rejected": -0.11605556309223175, "logps/chosen": -184.1270751953125, "logps/ref_chosen": -48.759117126464844, "logps/ref_rejected": -113.86376953125, "logps/rejected": -406.92718505859375, "loss": 1.141, "margin_dpo/margin_mean": 157.69546508789062, "margin_dpo/margin_std": 293.899658203125, "step": 137 }, { "KL/chosen_KL_mean": -148.42919921875, "KL/mean": -201.90768432617188, "KL/rejected_KL_mean": -255.3861846923828, "KL/std": 171.12557983398438, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.0016102675581350923, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.95697021484375, "fcm_dpo/q_t": 0.4581053853034973, "grad_norm": 9.790092468261719, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.11189040541648865, "logits/rejected": -0.11567579954862595, "logps/chosen": -208.9488525390625, "logps/ref_chosen": -60.519649505615234, "logps/ref_rejected": -93.19694519042969, "logps/rejected": -348.5831298828125, "loss": 1.2412, "margin_dpo/margin_mean": 106.95697021484375, "margin_dpo/margin_std": 175.25364685058594, "step": 138 }, { "KL/chosen_KL_mean": -127.69628143310547, "KL/mean": -184.2217559814453, "KL/rejected_KL_mean": -240.7472381591797, "KL/std": 150.19712829589844, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.0016102675581350923, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.05096435546875, "fcm_dpo/q_t": 0.4562827944755554, "grad_norm": 9.610363006591797, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.11106328666210175, "logits/rejected": -0.11769875884056091, "logps/chosen": -174.58766174316406, "logps/ref_chosen": -46.89138412475586, "logps/ref_rejected": -79.72798156738281, "logps/rejected": -320.4752197265625, "loss": 1.2391, "margin_dpo/margin_mean": 113.05096435546875, "margin_dpo/margin_std": 206.3931427001953, "step": 139 }, { "KL/chosen_KL_mean": -158.94659423828125, "KL/mean": -203.65492248535156, "KL/rejected_KL_mean": -248.36328125, "KL/std": 166.0042724609375, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.0016102675581350923, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.41668701171875, "fcm_dpo/q_t": 0.46475258469581604, "grad_norm": 9.001358032226562, "learning_rate": 4.840329401637809e-07, "logits/chosen": -0.11058294773101807, "logits/rejected": -0.12042254209518433, "logps/chosen": -217.92129516601562, "logps/ref_chosen": -58.97471618652344, "logps/ref_rejected": -83.28410339355469, "logps/rejected": -331.64739990234375, "loss": 1.2763, "margin_dpo/margin_mean": 89.41668701171875, "margin_dpo/margin_std": 212.3289794921875, "step": 140 }, { "KL/chosen_KL_mean": -191.18380737304688, "KL/mean": -233.52825927734375, "KL/rejected_KL_mean": -275.8727111816406, "KL/std": 238.5574188232422, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.0016102675581350923, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.68890380859375, "fcm_dpo/q_t": 0.4663596749305725, "grad_norm": 15.454120635986328, "learning_rate": 4.83578576263792e-07, "logits/chosen": -0.10689105093479156, "logits/rejected": -0.12749376893043518, "logps/chosen": -266.25946044921875, "logps/ref_chosen": -75.07566833496094, "logps/ref_rejected": -98.1922607421875, "logps/rejected": -374.0649719238281, "loss": 1.3095, "margin_dpo/margin_mean": 84.68890380859375, "margin_dpo/margin_std": 295.63360595703125, "step": 141 }, { "KL/chosen_KL_mean": -161.26834106445312, "KL/mean": -219.98394775390625, "KL/rejected_KL_mean": -278.6995849609375, "KL/std": 205.99090576171875, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.002624105429276824, "fcm_dpo/delta": 0.08150181919336319, "fcm_dpo/margin": 117.43124389648438, "fcm_dpo/q_t": 0.4544960558414459, "grad_norm": 13.912692070007324, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.09879723191261292, "logits/rejected": -0.11309005320072174, "logps/chosen": -219.2962646484375, "logps/ref_chosen": -58.027931213378906, "logps/ref_rejected": -94.58222961425781, "logps/rejected": -373.28179931640625, "loss": 1.1618, "margin_dpo/margin_mean": 117.43125915527344, "margin_dpo/margin_std": 238.91827392578125, "step": 142 }, { "KL/chosen_KL_mean": -157.7050323486328, "KL/mean": -193.36447143554688, "KL/rejected_KL_mean": -229.02392578125, "KL/std": 160.72268676757812, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.004260506015270948, "fcm_dpo/delta": 0.015796946361660957, "fcm_dpo/margin": 71.31889343261719, "fcm_dpo/q_t": 0.4452533721923828, "grad_norm": 27.70894432067871, "learning_rate": 4.826513955607734e-07, "logits/chosen": -0.09957142174243927, "logits/rejected": -0.11266869306564331, "logps/chosen": -215.30148315429688, "logps/ref_chosen": -57.59645080566406, "logps/ref_rejected": -78.99957275390625, "logps/rejected": -308.02349853515625, "loss": 1.2612, "margin_dpo/margin_mean": 71.31889343261719, "margin_dpo/margin_std": 196.57472229003906, "step": 143 }, { "KL/chosen_KL_mean": -155.84344482421875, "KL/mean": -195.27426147460938, "KL/rejected_KL_mean": -234.705078125, "KL/std": 159.63229370117188, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.004793780855834484, "fcm_dpo/delta": -0.03707936033606529, "fcm_dpo/margin": 78.86161804199219, "fcm_dpo/q_t": 0.4113781452178955, "grad_norm": 32.351070404052734, "learning_rate": 4.821786031898176e-07, "logits/chosen": -0.12348122149705887, "logits/rejected": -0.15042133629322052, "logps/chosen": -215.74981689453125, "logps/ref_chosen": -59.90636444091797, "logps/ref_rejected": -82.00025939941406, "logps/rejected": -316.705322265625, "loss": 1.2729, "margin_dpo/margin_mean": 78.86161804199219, "margin_dpo/margin_std": 176.07415771484375, "step": 144 }, { "KL/chosen_KL_mean": -133.75552368164062, "KL/mean": -175.75448608398438, "KL/rejected_KL_mean": -217.75347900390625, "KL/std": 131.30337524414062, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.004883944522589445, "fcm_dpo/delta": 0.020339418202638626, "fcm_dpo/margin": 83.99797058105469, "fcm_dpo/q_t": 0.41977953910827637, "grad_norm": 38.0810661315918, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.21709679067134857, "logits/rejected": -0.23458248376846313, "logps/chosen": -190.35618591308594, "logps/ref_chosen": -56.60066604614258, "logps/ref_rejected": -77.86631774902344, "logps/rejected": -295.61981201171875, "loss": 1.1677, "margin_dpo/margin_mean": 83.99796295166016, "margin_dpo/margin_std": 166.22833251953125, "step": 145 }, { "KL/chosen_KL_mean": -174.54449462890625, "KL/mean": -203.33346557617188, "KL/rejected_KL_mean": -232.12242126464844, "KL/std": 144.7928009033203, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.009946160949766636, "fcm_dpo/delta": 0.1288416087627411, "fcm_dpo/margin": 57.57792663574219, "fcm_dpo/q_t": 0.4425297677516937, "grad_norm": 173.9798583984375, "learning_rate": 4.812146767012779e-07, "logits/chosen": -0.14733245968818665, "logits/rejected": -0.1549091786146164, "logps/chosen": -240.54495239257812, "logps/ref_chosen": -66.00045013427734, "logps/ref_rejected": -81.70278930664062, "logps/rejected": -313.8251953125, "loss": 1.6518, "margin_dpo/margin_mean": 57.57793426513672, "margin_dpo/margin_std": 177.08001708984375, "step": 146 }, { "KL/chosen_KL_mean": -131.95578002929688, "KL/mean": -173.01547241210938, "KL/rejected_KL_mean": -214.07516479492188, "KL/std": 144.98574829101562, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": -0.4109819829463959, "fcm_dpo/margin": 82.11940002441406, "fcm_dpo/q_t": 0.42657575011253357, "grad_norm": 1.725198745727539, "learning_rate": 4.807235679840536e-07, "logits/chosen": -0.20732438564300537, "logits/rejected": -0.22212302684783936, "logps/chosen": -185.36126708984375, "logps/ref_chosen": -53.405487060546875, "logps/ref_rejected": -71.39060974121094, "logps/rejected": -285.4657897949219, "loss": 1.3691, "margin_dpo/margin_mean": 82.11940002441406, "margin_dpo/margin_std": 182.35836791992188, "step": 147 }, { "KL/chosen_KL_mean": -140.5421905517578, "KL/mean": -184.202392578125, "KL/rejected_KL_mean": -227.8626251220703, "KL/std": 168.81472778320312, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.32041931152344, "fcm_dpo/q_t": 0.4953227639198303, "grad_norm": 2.0062618255615234, "learning_rate": 4.802263794862384e-07, "logits/chosen": -0.23836994171142578, "logits/rejected": -0.24250321090221405, "logps/chosen": -205.47927856445312, "logps/ref_chosen": -64.93708038330078, "logps/ref_rejected": -103.09384155273438, "logps/rejected": -330.95648193359375, "loss": 1.3681, "margin_dpo/margin_mean": 87.32041931152344, "margin_dpo/margin_std": 185.8931884765625, "step": 148 }, { "KL/chosen_KL_mean": -124.4274673461914, "KL/mean": -176.48355102539062, "KL/rejected_KL_mean": -228.5396270751953, "KL/std": 142.7633514404297, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.11214447021484, "fcm_dpo/q_t": 0.4944196343421936, "grad_norm": 1.6070046424865723, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.20803114771842957, "logits/rejected": -0.21203464269638062, "logps/chosen": -182.90122985839844, "logps/ref_chosen": -58.47376251220703, "logps/ref_rejected": -99.31474304199219, "logps/rejected": -327.8543701171875, "loss": 1.3645, "margin_dpo/margin_mean": 104.11215209960938, "margin_dpo/margin_std": 176.35842895507812, "step": 149 }, { "KL/chosen_KL_mean": -107.64643096923828, "KL/mean": -164.7757568359375, "KL/rejected_KL_mean": -221.9050750732422, "KL/std": 159.32635498046875, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.25865173339844, "fcm_dpo/q_t": 0.49387815594673157, "grad_norm": 1.4557676315307617, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.2157842516899109, "logits/rejected": -0.2243994176387787, "logps/chosen": -153.35223388671875, "logps/ref_chosen": -45.705810546875, "logps/ref_rejected": -83.34759521484375, "logps/rejected": -305.252685546875, "loss": 1.3624, "margin_dpo/margin_mean": 114.2586441040039, "margin_dpo/margin_std": 193.01649475097656, "step": 150 }, { "KL/chosen_KL_mean": -149.71823120117188, "KL/mean": -199.18092346191406, "KL/rejected_KL_mean": -248.64361572265625, "KL/std": 174.56417846679688, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.92537689208984, "fcm_dpo/q_t": 0.49469929933547974, "grad_norm": 1.555479884147644, "learning_rate": 4.786984671220053e-07, "logits/chosen": -0.2548075318336487, "logits/rejected": -0.2567106783390045, "logps/chosen": -220.2890625, "logps/ref_chosen": -70.57083129882812, "logps/ref_rejected": -100.46382141113281, "logps/rejected": -349.107421875, "loss": 1.3657, "margin_dpo/margin_mean": 98.92536926269531, "margin_dpo/margin_std": 200.89501953125, "step": 151 }, { "KL/chosen_KL_mean": -130.66839599609375, "KL/mean": -196.2487335205078, "KL/rejected_KL_mean": -261.8291015625, "KL/std": 162.598876953125, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.16065979003906, "fcm_dpo/q_t": 0.4929710030555725, "grad_norm": 1.7390542030334473, "learning_rate": 4.78177092112495e-07, "logits/chosen": -0.22812870144844055, "logits/rejected": -0.23715990781784058, "logps/chosen": -190.83279418945312, "logps/ref_chosen": -60.16438674926758, "logps/ref_rejected": -106.14045715332031, "logps/rejected": -367.96954345703125, "loss": 1.3588, "margin_dpo/margin_mean": 131.16067504882812, "margin_dpo/margin_std": 200.10365295410156, "step": 152 }, { "KL/chosen_KL_mean": -126.16310119628906, "KL/mean": -180.94070434570312, "KL/rejected_KL_mean": -235.71829223632812, "KL/std": 180.3782501220703, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.55519104003906, "fcm_dpo/q_t": 0.49413180351257324, "grad_norm": 1.530712366104126, "learning_rate": 4.776497044244016e-07, "logits/chosen": -0.21236395835876465, "logits/rejected": -0.22474971413612366, "logps/chosen": -182.47837829589844, "logps/ref_chosen": -56.315277099609375, "logps/ref_rejected": -85.65583801269531, "logps/rejected": -321.3741149902344, "loss": 1.3636, "margin_dpo/margin_mean": 109.55519104003906, "margin_dpo/margin_std": 232.18374633789062, "step": 153 }, { "KL/chosen_KL_mean": -158.0811767578125, "KL/mean": -203.92166137695312, "KL/rejected_KL_mean": -249.7621307373047, "KL/std": 173.6383514404297, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.68096923828125, "fcm_dpo/q_t": 0.4950861930847168, "grad_norm": 1.4504088163375854, "learning_rate": 4.771163179548808e-07, "logits/chosen": -0.21836550533771515, "logits/rejected": -0.23400922119617462, "logps/chosen": -220.82374572753906, "logps/ref_chosen": -62.74256896972656, "logps/ref_rejected": -104.24420166015625, "logps/rejected": -354.00634765625, "loss": 1.3674, "margin_dpo/margin_mean": 91.68096923828125, "margin_dpo/margin_std": 240.62171936035156, "step": 154 }, { "KL/chosen_KL_mean": -142.25665283203125, "KL/mean": -183.7818145751953, "KL/rejected_KL_mean": -225.3070068359375, "KL/std": 151.2476348876953, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.05033874511719, "fcm_dpo/q_t": 0.49554821848869324, "grad_norm": 1.3787976503372192, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.23564405739307404, "logits/rejected": -0.23982316255569458, "logps/chosen": -202.9098358154297, "logps/ref_chosen": -60.65318298339844, "logps/ref_rejected": -77.49220275878906, "logps/rejected": -302.7991943359375, "loss": 1.369, "margin_dpo/margin_mean": 83.05035400390625, "margin_dpo/margin_std": 198.95108032226562, "step": 155 }, { "KL/chosen_KL_mean": -190.4660186767578, "KL/mean": -208.18328857421875, "KL/rejected_KL_mean": -225.90054321289062, "KL/std": 172.94052124023438, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.43452453613281, "fcm_dpo/q_t": 0.49810007214546204, "grad_norm": 1.2596914768218994, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -0.19964438676834106, "logits/rejected": -0.22770291566848755, "logps/chosen": -259.9578857421875, "logps/ref_chosen": -69.49188232421875, "logps/ref_rejected": -77.16929626464844, "logps/rejected": -303.06982421875, "loss": 1.3793, "margin_dpo/margin_mean": 35.43452453613281, "margin_dpo/margin_std": 223.39340209960938, "step": 156 }, { "KL/chosen_KL_mean": -160.28289794921875, "KL/mean": -230.226806640625, "KL/rejected_KL_mean": -300.17071533203125, "KL/std": 185.30099487304688, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.88784790039062, "fcm_dpo/q_t": 0.49250328540802, "grad_norm": 1.719684362411499, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -0.09867404401302338, "logits/rejected": -0.11982227861881256, "logps/chosen": -221.65133666992188, "logps/ref_chosen": -61.368438720703125, "logps/ref_rejected": -107.64636993408203, "logps/rejected": -407.8171081542969, "loss": 1.3572, "margin_dpo/margin_mean": 139.88784790039062, "margin_dpo/margin_std": 224.976318359375, "step": 157 }, { "KL/chosen_KL_mean": -170.56492614746094, "KL/mean": -253.34765625, "KL/rejected_KL_mean": -336.1304016113281, "KL/std": 241.12974548339844, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 165.5655059814453, "fcm_dpo/q_t": 0.491136372089386, "grad_norm": 2.193638324737549, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -0.13828858733177185, "logits/rejected": -0.16126753389835358, "logps/chosen": -228.17784118652344, "logps/ref_chosen": -57.612918853759766, "logps/ref_rejected": -113.6946792602539, "logps/rejected": -449.8250732421875, "loss": 1.3521, "margin_dpo/margin_mean": 165.5655059814453, "margin_dpo/margin_std": 297.131591796875, "step": 158 }, { "KL/chosen_KL_mean": -226.7890167236328, "KL/mean": -262.0523681640625, "KL/rejected_KL_mean": -297.31573486328125, "KL/std": 231.03848266601562, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.52671813964844, "fcm_dpo/q_t": 0.49621930718421936, "grad_norm": 1.778730869293213, "learning_rate": 4.743599013306165e-07, "logits/chosen": -0.11859369277954102, "logits/rejected": -0.12011769413948059, "logps/chosen": -308.349365234375, "logps/ref_chosen": -81.56034851074219, "logps/ref_rejected": -88.89871215820312, "logps/rejected": -386.21441650390625, "loss": 1.372, "margin_dpo/margin_mean": 70.52671813964844, "margin_dpo/margin_std": 254.50335693359375, "step": 159 }, { "KL/chosen_KL_mean": -208.1883544921875, "KL/mean": -273.8866271972656, "KL/rejected_KL_mean": -339.5848693847656, "KL/std": 218.83474731445312, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.39654541015625, "fcm_dpo/q_t": 0.4929630160331726, "grad_norm": 1.8856486082077026, "learning_rate": 4.737908228387656e-07, "logits/chosen": -0.12426799535751343, "logits/rejected": -0.14046964049339294, "logps/chosen": -273.91925048828125, "logps/ref_chosen": -65.73088073730469, "logps/ref_rejected": -97.21781921386719, "logps/rejected": -436.80267333984375, "loss": 1.3592, "margin_dpo/margin_mean": 131.3965301513672, "margin_dpo/margin_std": 274.5325927734375, "step": 160 }, { "KL/chosen_KL_mean": -171.75802612304688, "KL/mean": -228.2024688720703, "KL/rejected_KL_mean": -284.64691162109375, "KL/std": 182.69235229492188, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.88887786865234, "fcm_dpo/q_t": 0.4939511716365814, "grad_norm": 1.585429310798645, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -0.12636083364486694, "logits/rejected": -0.1504145711660385, "logps/chosen": -224.1945037841797, "logps/ref_chosen": -52.43647003173828, "logps/ref_rejected": -83.43095397949219, "logps/rejected": -368.0778503417969, "loss": 1.3628, "margin_dpo/margin_mean": 112.88887786865234, "margin_dpo/margin_std": 216.60546875, "step": 161 }, { "KL/chosen_KL_mean": -180.044921875, "KL/mean": -228.87269592285156, "KL/rejected_KL_mean": -277.700439453125, "KL/std": 198.00787353515625, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.65554809570312, "fcm_dpo/q_t": 0.49477049708366394, "grad_norm": 1.6253687143325806, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.20148800313472748, "logits/rejected": -0.2152554988861084, "logps/chosen": -242.655517578125, "logps/ref_chosen": -62.6105842590332, "logps/ref_rejected": -89.39057922363281, "logps/rejected": -367.091064453125, "loss": 1.3662, "margin_dpo/margin_mean": 97.65554809570312, "margin_dpo/margin_std": 248.9880828857422, "step": 162 }, { "KL/chosen_KL_mean": -198.3851776123047, "KL/mean": -241.20689392089844, "KL/rejected_KL_mean": -284.0285949707031, "KL/std": 196.6535186767578, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.64341735839844, "fcm_dpo/q_t": 0.4954114258289337, "grad_norm": 1.5751463174819946, "learning_rate": 4.720482655449212e-07, "logits/chosen": -0.11102645099163055, "logits/rejected": -0.11732280254364014, "logps/chosen": -253.4068145751953, "logps/ref_chosen": -55.021629333496094, "logps/ref_rejected": -75.418212890625, "logps/rejected": -359.44683837890625, "loss": 1.3687, "margin_dpo/margin_mean": 85.64341735839844, "margin_dpo/margin_std": 241.90493774414062, "step": 163 }, { "KL/chosen_KL_mean": -186.4625244140625, "KL/mean": -250.75245666503906, "KL/rejected_KL_mean": -315.0423583984375, "KL/std": 190.93157958984375, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.579833984375, "fcm_dpo/q_t": 0.49310967326164246, "grad_norm": 1.862160325050354, "learning_rate": 4.714556901942599e-07, "logits/chosen": -0.07840365171432495, "logits/rejected": -0.09606952965259552, "logps/chosen": -242.10321044921875, "logps/ref_chosen": -55.64066696166992, "logps/ref_rejected": -79.66463470458984, "logps/rejected": -394.7070007324219, "loss": 1.3595, "margin_dpo/margin_mean": 128.579833984375, "margin_dpo/margin_std": 218.310302734375, "step": 164 }, { "KL/chosen_KL_mean": -206.51309204101562, "KL/mean": -245.36529541015625, "KL/rejected_KL_mean": -284.2174987792969, "KL/std": 193.79730224609375, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.70442199707031, "fcm_dpo/q_t": 0.49583813548088074, "grad_norm": 1.5955755710601807, "learning_rate": 4.708572792802069e-07, "logits/chosen": -0.10220310091972351, "logits/rejected": -0.1038433313369751, "logps/chosen": -267.82379150390625, "logps/ref_chosen": -61.310691833496094, "logps/ref_rejected": -73.67060852050781, "logps/rejected": -357.88812255859375, "loss": 1.3706, "margin_dpo/margin_mean": 77.70442199707031, "margin_dpo/margin_std": 272.9258117675781, "step": 165 }, { "KL/chosen_KL_mean": -183.43780517578125, "KL/mean": -275.9177551269531, "KL/rejected_KL_mean": -368.397705078125, "KL/std": 286.80517578125, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 184.95986938476562, "fcm_dpo/q_t": 0.4901168942451477, "grad_norm": 2.0007615089416504, "learning_rate": 4.702530485714461e-07, "logits/chosen": -0.11677154153585434, "logits/rejected": -0.1256922483444214, "logps/chosen": -234.42141723632812, "logps/ref_chosen": -50.98360061645508, "logps/ref_rejected": -98.09512329101562, "logps/rejected": -466.4927978515625, "loss": 1.3487, "margin_dpo/margin_mean": 184.95986938476562, "margin_dpo/margin_std": 381.46697998046875, "step": 166 }, { "KL/chosen_KL_mean": -194.1734619140625, "KL/mean": -280.3365173339844, "KL/rejected_KL_mean": -366.4996032714844, "KL/std": 239.65505981445312, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 172.326171875, "fcm_dpo/q_t": 0.49077457189559937, "grad_norm": 2.102647542953491, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.1024477556347847, "logits/rejected": -0.11726316809654236, "logps/chosen": -244.59754943847656, "logps/ref_chosen": -50.424095153808594, "logps/ref_rejected": -96.03042602539062, "logps/rejected": -462.530029296875, "loss": 1.3508, "margin_dpo/margin_mean": 172.32615661621094, "margin_dpo/margin_std": 293.95623779296875, "step": 167 }, { "KL/chosen_KL_mean": -213.13294982910156, "KL/mean": -268.35931396484375, "KL/rejected_KL_mean": -323.58563232421875, "KL/std": 203.27450561523438, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.45269775390625, "fcm_dpo/q_t": 0.49408331513404846, "grad_norm": 1.7750434875488281, "learning_rate": 4.690271916109034e-07, "logits/chosen": -0.03718934208154678, "logits/rejected": -0.04697761684656143, "logps/chosen": -262.59576416015625, "logps/ref_chosen": -49.462825775146484, "logps/ref_rejected": -75.30855560302734, "logps/rejected": -398.8941650390625, "loss": 1.3635, "margin_dpo/margin_mean": 110.45269775390625, "margin_dpo/margin_std": 252.35952758789062, "step": 168 }, { "KL/chosen_KL_mean": -244.06619262695312, "KL/mean": -290.67816162109375, "KL/rejected_KL_mean": -337.2901306152344, "KL/std": 225.60400390625, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.22395324707031, "fcm_dpo/q_t": 0.4950127601623535, "grad_norm": 2.7092039585113525, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -0.061293989419937134, "logits/rejected": -0.06470146030187607, "logps/chosen": -303.86962890625, "logps/ref_chosen": -59.803443908691406, "logps/ref_rejected": -83.34574890136719, "logps/rejected": -420.6358947753906, "loss": 1.3676, "margin_dpo/margin_mean": 93.22394561767578, "margin_dpo/margin_std": 315.66082763671875, "step": 169 }, { "KL/chosen_KL_mean": -223.83151245117188, "KL/mean": -274.68341064453125, "KL/rejected_KL_mean": -325.53533935546875, "KL/std": 236.1355743408203, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.70378112792969, "fcm_dpo/q_t": 0.49454981088638306, "grad_norm": 2.4471933841705322, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -0.06102032959461212, "logits/rejected": -0.07576321065425873, "logps/chosen": -273.30328369140625, "logps/ref_chosen": -49.471771240234375, "logps/ref_rejected": -75.91734313964844, "logps/rejected": -401.45269775390625, "loss": 1.3656, "margin_dpo/margin_mean": 101.70378112792969, "margin_dpo/margin_std": 283.95513916015625, "step": 170 }, { "KL/chosen_KL_mean": -346.0281066894531, "KL/mean": -386.1025390625, "KL/rejected_KL_mean": -426.177001953125, "KL/std": 327.9516296386719, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.14892578125, "fcm_dpo/q_t": 0.4957178831100464, "grad_norm": 4.012329578399658, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -0.04151609167456627, "logits/rejected": -0.0675613209605217, "logps/chosen": -430.52740478515625, "logps/ref_chosen": -84.49931335449219, "logps/ref_rejected": -109.38209533691406, "logps/rejected": -535.55908203125, "loss": 1.3712, "margin_dpo/margin_mean": 80.14892578125, "margin_dpo/margin_std": 414.32763671875, "step": 171 }, { "KL/chosen_KL_mean": -278.4682312011719, "KL/mean": -318.6019287109375, "KL/rejected_KL_mean": -358.73565673828125, "KL/std": 296.55731201171875, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.26742553710938, "fcm_dpo/q_t": 0.49569272994995117, "grad_norm": 2.9481098651885986, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -0.057665470987558365, "logits/rejected": -0.08217429369688034, "logps/chosen": -347.12213134765625, "logps/ref_chosen": -68.65391540527344, "logps/ref_rejected": -85.43667602539062, "logps/rejected": -444.1723327636719, "loss": 1.3703, "margin_dpo/margin_mean": 80.26742553710938, "margin_dpo/margin_std": 307.19091796875, "step": 172 }, { "KL/chosen_KL_mean": -234.19972229003906, "KL/mean": -281.9803771972656, "KL/rejected_KL_mean": -329.7610168457031, "KL/std": 252.82623291015625, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.56130981445312, "fcm_dpo/q_t": 0.4948866665363312, "grad_norm": 1.8302026987075806, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -0.0890713781118393, "logits/rejected": -0.1072927713394165, "logps/chosen": -297.2506103515625, "logps/ref_chosen": -63.050880432128906, "logps/ref_rejected": -78.68392181396484, "logps/rejected": -408.4449462890625, "loss": 1.3668, "margin_dpo/margin_mean": 95.5613021850586, "margin_dpo/margin_std": 271.940673828125, "step": 173 }, { "KL/chosen_KL_mean": -216.39028930664062, "KL/mean": -295.0453796386719, "KL/rejected_KL_mean": -373.700439453125, "KL/std": 275.541748046875, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 157.31015014648438, "fcm_dpo/q_t": 0.49158352613449097, "grad_norm": 3.099987268447876, "learning_rate": 4.652116329460919e-07, "logits/chosen": -0.02464154362678528, "logits/rejected": -0.03820271044969559, "logps/chosen": -269.7532653808594, "logps/ref_chosen": -53.36296844482422, "logps/ref_rejected": -101.91120910644531, "logps/rejected": -475.61163330078125, "loss": 1.3543, "margin_dpo/margin_mean": 157.31015014648438, "margin_dpo/margin_std": 358.8941650390625, "step": 174 }, { "KL/chosen_KL_mean": -168.59716796875, "KL/mean": -259.63287353515625, "KL/rejected_KL_mean": -350.6685485839844, "KL/std": 224.63267517089844, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 182.07138061523438, "fcm_dpo/q_t": 0.4902472496032715, "grad_norm": 3.9441075325012207, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.058357544243335724, "logits/rejected": -0.0670979917049408, "logps/chosen": -214.01492309570312, "logps/ref_chosen": -45.417762756347656, "logps/ref_rejected": -89.50579833984375, "logps/rejected": -440.1743469238281, "loss": 1.3484, "margin_dpo/margin_mean": 182.07138061523438, "margin_dpo/margin_std": 263.89202880859375, "step": 175 }, { "KL/chosen_KL_mean": -193.49896240234375, "KL/mean": -283.0251159667969, "KL/rejected_KL_mean": -372.55126953125, "KL/std": 272.05682373046875, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 179.05230712890625, "fcm_dpo/q_t": 0.49042147397994995, "grad_norm": 3.722703456878662, "learning_rate": 4.638942309888058e-07, "logits/chosen": -0.07522699236869812, "logits/rejected": -0.08893322944641113, "logps/chosen": -243.95181274414062, "logps/ref_chosen": -50.452842712402344, "logps/ref_rejected": -95.5589599609375, "logps/rejected": -468.1102294921875, "loss": 1.3495, "margin_dpo/margin_mean": 179.05230712890625, "margin_dpo/margin_std": 324.3958435058594, "step": 176 }, { "KL/chosen_KL_mean": -248.76412963867188, "KL/mean": -327.17547607421875, "KL/rejected_KL_mean": -405.5867919921875, "KL/std": 315.8409423828125, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.8226318359375, "fcm_dpo/q_t": 0.4916095733642578, "grad_norm": 2.4874935150146484, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -0.09781675040721893, "logits/rejected": -0.12420766055583954, "logps/chosen": -309.9805908203125, "logps/ref_chosen": -61.216468811035156, "logps/ref_rejected": -95.89378356933594, "logps/rejected": -501.4805908203125, "loss": 1.3544, "margin_dpo/margin_mean": 156.82264709472656, "margin_dpo/margin_std": 351.21734619140625, "step": 177 }, { "KL/chosen_KL_mean": -260.517578125, "KL/mean": -355.90155029296875, "KL/rejected_KL_mean": -451.2855224609375, "KL/std": 332.3189697265625, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 190.7679443359375, "fcm_dpo/q_t": 0.48978862166404724, "grad_norm": 3.6528143882751465, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -0.004009534604847431, "logits/rejected": -0.022813618183135986, "logps/chosen": -318.7823486328125, "logps/ref_chosen": -58.26478958129883, "logps/ref_rejected": -105.3653335571289, "logps/rejected": -556.65087890625, "loss": 1.3472, "margin_dpo/margin_mean": 190.7679443359375, "margin_dpo/margin_std": 349.21771240234375, "step": 178 }, { "KL/chosen_KL_mean": -270.78173828125, "KL/mean": -333.99163818359375, "KL/rejected_KL_mean": -397.20159912109375, "KL/std": 320.001708984375, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.41985321044922, "fcm_dpo/q_t": 0.4932493567466736, "grad_norm": 2.433837652206421, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -0.06824183464050293, "logits/rejected": -0.09712938964366913, "logps/chosen": -331.84002685546875, "logps/ref_chosen": -61.05832290649414, "logps/ref_rejected": -90.52782440185547, "logps/rejected": -487.72943115234375, "loss": 1.3614, "margin_dpo/margin_mean": 126.41983795166016, "margin_dpo/margin_std": 422.2001037597656, "step": 179 }, { "KL/chosen_KL_mean": -251.91395568847656, "KL/mean": -345.0408935546875, "KL/rejected_KL_mean": -438.16778564453125, "KL/std": 260.55181884765625, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 186.25384521484375, "fcm_dpo/q_t": 0.4900313913822174, "grad_norm": 2.2225091457366943, "learning_rate": 4.611919330113591e-07, "logits/chosen": -0.046570487320423126, "logits/rejected": -0.06946319341659546, "logps/chosen": -306.2566833496094, "logps/ref_chosen": -54.34272003173828, "logps/ref_rejected": -98.21183776855469, "logps/rejected": -536.379638671875, "loss": 1.3482, "margin_dpo/margin_mean": 186.25384521484375, "margin_dpo/margin_std": 353.3677978515625, "step": 180 }, { "KL/chosen_KL_mean": -218.62420654296875, "KL/mean": -267.3025207519531, "KL/rejected_KL_mean": -315.9808654785156, "KL/std": 229.04489135742188, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.35667419433594, "fcm_dpo/q_t": 0.49478650093078613, "grad_norm": 2.206993341445923, "learning_rate": 4.605024008834863e-07, "logits/chosen": -0.05855263024568558, "logits/rejected": -0.06639339029788971, "logps/chosen": -273.6246643066406, "logps/ref_chosen": -55.000457763671875, "logps/ref_rejected": -61.656166076660156, "logps/rejected": -377.63702392578125, "loss": 1.3666, "margin_dpo/margin_mean": 97.35667419433594, "margin_dpo/margin_std": 312.8726806640625, "step": 181 }, { "KL/chosen_KL_mean": -229.5802001953125, "KL/mean": -339.115966796875, "KL/rejected_KL_mean": -448.6517333984375, "KL/std": 309.89312744140625, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 219.07156372070312, "fcm_dpo/q_t": 0.48828235268592834, "grad_norm": 2.526320695877075, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.07598879933357239, "logits/rejected": -0.09682799130678177, "logps/chosen": -270.68804931640625, "logps/ref_chosen": -41.107852935791016, "logps/ref_rejected": -89.5215835571289, "logps/rejected": -538.17333984375, "loss": 1.3417, "margin_dpo/margin_mean": 219.07154846191406, "margin_dpo/margin_std": 401.08624267578125, "step": 182 }, { "KL/chosen_KL_mean": -323.77984619140625, "KL/mean": -366.191162109375, "KL/rejected_KL_mean": -408.6024475097656, "KL/std": 279.89532470703125, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.82255554199219, "fcm_dpo/q_t": 0.49546507000923157, "grad_norm": 3.5669703483581543, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -0.05871109664440155, "logits/rejected": -0.08482161164283752, "logps/chosen": -381.304443359375, "logps/ref_chosen": -57.52456283569336, "logps/ref_rejected": -75.97572326660156, "logps/rejected": -484.57818603515625, "loss": 1.3695, "margin_dpo/margin_mean": 84.82254791259766, "margin_dpo/margin_std": 320.90704345703125, "step": 183 }, { "KL/chosen_KL_mean": -315.3003845214844, "KL/mean": -351.2978515625, "KL/rejected_KL_mean": -387.2952575683594, "KL/std": 260.4176025390625, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.99484252929688, "fcm_dpo/q_t": 0.4961445927619934, "grad_norm": 2.970120668411255, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -0.059889525175094604, "logits/rejected": -0.0684148371219635, "logps/chosen": -373.8453369140625, "logps/ref_chosen": -58.544952392578125, "logps/ref_rejected": -76.63406372070312, "logps/rejected": -463.9293212890625, "loss": 1.3723, "margin_dpo/margin_mean": 71.9948501586914, "margin_dpo/margin_std": 333.66949462890625, "step": 184 }, { "KL/chosen_KL_mean": -327.66729736328125, "KL/mean": -346.7569580078125, "KL/rejected_KL_mean": -365.84661865234375, "KL/std": 300.96337890625, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.17930221557617, "fcm_dpo/q_t": 0.49796029925346375, "grad_norm": 3.1760153770446777, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -0.02878168597817421, "logits/rejected": -0.03691637143492699, "logps/chosen": -389.69317626953125, "logps/ref_chosen": -62.025848388671875, "logps/ref_rejected": -73.7625961303711, "logps/rejected": -439.6092224121094, "loss": 1.3798, "margin_dpo/margin_mean": 38.17930221557617, "margin_dpo/margin_std": 370.1015625, "step": 185 }, { "KL/chosen_KL_mean": -320.67669677734375, "KL/mean": -373.763427734375, "KL/rejected_KL_mean": -426.8501892089844, "KL/std": 282.6938171386719, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.17350769042969, "fcm_dpo/q_t": 0.4943085312843323, "grad_norm": 2.898653745651245, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -0.1143377274274826, "logits/rejected": -0.15146398544311523, "logps/chosen": -390.0301513671875, "logps/ref_chosen": -69.35346984863281, "logps/ref_rejected": -88.07244873046875, "logps/rejected": -514.9226684570312, "loss": 1.3653, "margin_dpo/margin_mean": 106.17350769042969, "margin_dpo/margin_std": 373.8057556152344, "step": 186 }, { "KL/chosen_KL_mean": -235.82952880859375, "KL/mean": -303.52093505859375, "KL/rejected_KL_mean": -371.21234130859375, "KL/std": 257.0897521972656, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.3828125, "fcm_dpo/q_t": 0.4927484393119812, "grad_norm": 2.4242630004882812, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -0.0784909650683403, "logits/rejected": -0.09007595479488373, "logps/chosen": -288.58599853515625, "logps/ref_chosen": -52.7564582824707, "logps/ref_rejected": -81.96910095214844, "logps/rejected": -453.18145751953125, "loss": 1.3583, "margin_dpo/margin_mean": 135.38282775878906, "margin_dpo/margin_std": 272.33575439453125, "step": 187 }, { "KL/chosen_KL_mean": -211.16748046875, "KL/mean": -291.66912841796875, "KL/rejected_KL_mean": -372.1707763671875, "KL/std": 238.9845733642578, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.00326538085938, "fcm_dpo/q_t": 0.4913794994354248, "grad_norm": 2.7519357204437256, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -0.10255219042301178, "logits/rejected": -0.13276180624961853, "logps/chosen": -260.58294677734375, "logps/ref_chosen": -49.415489196777344, "logps/ref_rejected": -89.54043579101562, "logps/rejected": -461.711181640625, "loss": 1.353, "margin_dpo/margin_mean": 161.00326538085938, "margin_dpo/margin_std": 281.9280090332031, "step": 188 }, { "KL/chosen_KL_mean": -245.26499938964844, "KL/mean": -297.78826904296875, "KL/rejected_KL_mean": -350.3115234375, "KL/std": 250.982666015625, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.0465087890625, "fcm_dpo/q_t": 0.49437761306762695, "grad_norm": 3.0305230617523193, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -0.13245689868927002, "logits/rejected": -0.15994009375572205, "logps/chosen": -297.6639709472656, "logps/ref_chosen": -52.39896011352539, "logps/ref_rejected": -72.16735076904297, "logps/rejected": -422.4788513183594, "loss": 1.3651, "margin_dpo/margin_mean": 105.0465087890625, "margin_dpo/margin_std": 324.1693115234375, "step": 189 }, { "KL/chosen_KL_mean": -259.7969970703125, "KL/mean": -342.51513671875, "KL/rejected_KL_mean": -425.2333068847656, "KL/std": 296.94696044921875, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 165.436279296875, "fcm_dpo/q_t": 0.4911511242389679, "grad_norm": 2.5856246948242188, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -0.1271246075630188, "logits/rejected": -0.15945151448249817, "logps/chosen": -324.4801025390625, "logps/ref_chosen": -64.68305969238281, "logps/ref_rejected": -102.55052185058594, "logps/rejected": -527.7838134765625, "loss": 1.3527, "margin_dpo/margin_mean": 165.436279296875, "margin_dpo/margin_std": 360.4592590332031, "step": 190 }, { "KL/chosen_KL_mean": -253.1461181640625, "KL/mean": -364.49249267578125, "KL/rejected_KL_mean": -475.8388977050781, "KL/std": 322.4438171386719, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 222.6928253173828, "fcm_dpo/q_t": 0.4880927801132202, "grad_norm": 3.680899143218994, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -0.12386652827262878, "logits/rejected": -0.11776319891214371, "logps/chosen": -321.80499267578125, "logps/ref_chosen": -68.65887451171875, "logps/ref_rejected": -110.1396713256836, "logps/rejected": -585.9785766601562, "loss": 1.3408, "margin_dpo/margin_mean": 222.6928253173828, "margin_dpo/margin_std": 386.87347412109375, "step": 191 }, { "KL/chosen_KL_mean": -300.694091796875, "KL/mean": -368.8135986328125, "KL/rejected_KL_mean": -436.93310546875, "KL/std": 301.52886962890625, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 136.23902893066406, "fcm_dpo/q_t": 0.49271297454833984, "grad_norm": 2.103487253189087, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -0.09601673483848572, "logits/rejected": -0.10303403437137604, "logps/chosen": -370.4210205078125, "logps/ref_chosen": -69.72691345214844, "logps/ref_rejected": -103.32135009765625, "logps/rejected": -540.2544555664062, "loss": 1.359, "margin_dpo/margin_mean": 136.23902893066406, "margin_dpo/margin_std": 378.644287109375, "step": 192 }, { "KL/chosen_KL_mean": -283.2509460449219, "KL/mean": -325.3600769042969, "KL/rejected_KL_mean": -367.46923828125, "KL/std": 314.67529296875, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.21826171875, "fcm_dpo/q_t": 0.49550098180770874, "grad_norm": 2.3068125247955322, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -0.07964509725570679, "logits/rejected": -0.1134941577911377, "logps/chosen": -343.44146728515625, "logps/ref_chosen": -60.19049835205078, "logps/ref_rejected": -76.40755462646484, "logps/rejected": -443.87677001953125, "loss": 1.3702, "margin_dpo/margin_mean": 84.21826934814453, "margin_dpo/margin_std": 401.8240966796875, "step": 193 }, { "KL/chosen_KL_mean": -178.2296905517578, "KL/mean": -237.00930786132812, "KL/rejected_KL_mean": -295.7889099121094, "KL/std": 197.4080810546875, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.55923461914062, "fcm_dpo/q_t": 0.49369940161705017, "grad_norm": 1.7572847604751587, "learning_rate": 4.510405240853854e-07, "logits/chosen": 0.0028687757439911366, "logits/rejected": -0.012111449614167213, "logps/chosen": -216.070068359375, "logps/ref_chosen": -37.84037399291992, "logps/ref_rejected": -60.684783935546875, "logps/rejected": -356.47369384765625, "loss": 1.3618, "margin_dpo/margin_mean": 117.55921936035156, "margin_dpo/margin_std": 219.46182250976562, "step": 194 }, { "KL/chosen_KL_mean": -269.02020263671875, "KL/mean": -363.59765625, "KL/rejected_KL_mean": -458.17510986328125, "KL/std": 306.4468994140625, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 189.15487670898438, "fcm_dpo/q_t": 0.4898764491081238, "grad_norm": 3.175015687942505, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -0.10875138640403748, "logits/rejected": -0.11520682275295258, "logps/chosen": -323.9117736816406, "logps/ref_chosen": -54.891571044921875, "logps/ref_rejected": -96.77095794677734, "logps/rejected": -554.946044921875, "loss": 1.3473, "margin_dpo/margin_mean": 189.15487670898438, "margin_dpo/margin_std": 321.9908752441406, "step": 195 }, { "KL/chosen_KL_mean": -243.0733642578125, "KL/mean": -316.908447265625, "KL/rejected_KL_mean": -390.7435302734375, "KL/std": 265.1428527832031, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 147.670166015625, "fcm_dpo/q_t": 0.4920935332775116, "grad_norm": 2.1525228023529053, "learning_rate": 4.495043068200599e-07, "logits/chosen": -0.10881956666707993, "logits/rejected": -0.12770894169807434, "logps/chosen": -296.318603515625, "logps/ref_chosen": -53.245243072509766, "logps/ref_rejected": -76.05294799804688, "logps/rejected": -466.7964782714844, "loss": 1.3564, "margin_dpo/margin_mean": 147.670166015625, "margin_dpo/margin_std": 344.3466491699219, "step": 196 }, { "KL/chosen_KL_mean": -274.3104248046875, "KL/mean": -325.2745361328125, "KL/rejected_KL_mean": -376.23858642578125, "KL/std": 251.0189971923828, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.92816925048828, "fcm_dpo/q_t": 0.4945385158061981, "grad_norm": 2.172877073287964, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -0.10666961967945099, "logits/rejected": -0.11786352097988129, "logps/chosen": -334.73077392578125, "logps/ref_chosen": -60.42033386230469, "logps/ref_rejected": -77.20890808105469, "logps/rejected": -453.447509765625, "loss": 1.3655, "margin_dpo/margin_mean": 101.92816162109375, "margin_dpo/margin_std": 281.21978759765625, "step": 197 }, { "KL/chosen_KL_mean": -272.42474365234375, "KL/mean": -371.7640380859375, "KL/rejected_KL_mean": -471.1033020019531, "KL/std": 313.407470703125, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 198.67855834960938, "fcm_dpo/q_t": 0.4893642067909241, "grad_norm": 2.680528402328491, "learning_rate": 4.479470611971645e-07, "logits/chosen": -0.13605228066444397, "logits/rejected": -0.1457493156194687, "logps/chosen": -327.4609375, "logps/ref_chosen": -55.03618621826172, "logps/ref_rejected": -97.24325561523438, "logps/rejected": -568.3465576171875, "loss": 1.3453, "margin_dpo/margin_mean": 198.67855834960938, "margin_dpo/margin_std": 319.24285888671875, "step": 198 }, { "KL/chosen_KL_mean": -274.304443359375, "KL/mean": -358.85919189453125, "KL/rejected_KL_mean": -443.41387939453125, "KL/std": 275.95819091796875, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 169.10943603515625, "fcm_dpo/q_t": 0.4909524619579315, "grad_norm": 3.292473077774048, "learning_rate": 4.471606039587695e-07, "logits/chosen": -0.14171649515628815, "logits/rejected": -0.13244497776031494, "logps/chosen": -331.13330078125, "logps/ref_chosen": -56.828826904296875, "logps/ref_rejected": -84.64820861816406, "logps/rejected": -528.0621337890625, "loss": 1.352, "margin_dpo/margin_mean": 169.10943603515625, "margin_dpo/margin_std": 374.6290283203125, "step": 199 }, { "KL/chosen_KL_mean": -292.3767395019531, "KL/mean": -373.58990478515625, "KL/rejected_KL_mean": -454.8030090332031, "KL/std": 326.7904052734375, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 162.42630004882812, "fcm_dpo/q_t": 0.49131280183792114, "grad_norm": 3.315680980682373, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -0.07947467267513275, "logits/rejected": -0.10722815245389938, "logps/chosen": -345.44378662109375, "logps/ref_chosen": -53.06706237792969, "logps/ref_rejected": -80.60843658447266, "logps/rejected": -535.4114379882812, "loss": 1.3542, "margin_dpo/margin_mean": 162.42628479003906, "margin_dpo/margin_std": 453.01910400390625, "step": 200 }, { "KL/chosen_KL_mean": -343.0718688964844, "KL/mean": -435.6768493652344, "KL/rejected_KL_mean": -528.2818603515625, "KL/std": 393.9222717285156, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 185.20999145507812, "fcm_dpo/q_t": 0.49011439085006714, "grad_norm": 3.324786424636841, "learning_rate": 4.455721242469372e-07, "logits/chosen": -0.23055091500282288, "logits/rejected": -0.24232345819473267, "logps/chosen": -418.4740905761719, "logps/ref_chosen": -75.4022216796875, "logps/ref_rejected": -114.80821990966797, "logps/rejected": -643.090087890625, "loss": 1.3498, "margin_dpo/margin_mean": 185.20999145507812, "margin_dpo/margin_std": 481.95977783203125, "step": 201 }, { "KL/chosen_KL_mean": -312.4244689941406, "KL/mean": -377.79998779296875, "KL/rejected_KL_mean": -443.1754455566406, "KL/std": 326.81707763671875, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.75100708007812, "fcm_dpo/q_t": 0.4930059015750885, "grad_norm": 3.468919515609741, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -0.1302487850189209, "logits/rejected": -0.14801092445850372, "logps/chosen": -362.5257873535156, "logps/ref_chosen": -50.101318359375, "logps/ref_rejected": -86.98503112792969, "logps/rejected": -530.1604614257812, "loss": 1.3601, "margin_dpo/margin_mean": 130.75100708007812, "margin_dpo/margin_std": 375.7841796875, "step": 202 }, { "KL/chosen_KL_mean": -355.20648193359375, "KL/mean": -438.63665771484375, "KL/rejected_KL_mean": -522.0668334960938, "KL/std": 384.030029296875, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 166.86033630371094, "fcm_dpo/q_t": 0.491105854511261, "grad_norm": 3.1627941131591797, "learning_rate": 4.439630306414758e-07, "logits/chosen": -0.12041978538036346, "logits/rejected": -0.12802743911743164, "logps/chosen": -415.816162109375, "logps/ref_chosen": -60.60969543457031, "logps/ref_rejected": -85.89596557617188, "logps/rejected": -607.9627685546875, "loss": 1.3532, "margin_dpo/margin_mean": 166.86032104492188, "margin_dpo/margin_std": 459.39349365234375, "step": 203 }, { "KL/chosen_KL_mean": -420.5950622558594, "KL/mean": -458.009033203125, "KL/rejected_KL_mean": -495.42303466796875, "KL/std": 421.4627990722656, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.8279800415039, "fcm_dpo/q_t": 0.49598783254623413, "grad_norm": 4.85085391998291, "learning_rate": 4.431508065452897e-07, "logits/chosen": -0.22811344265937805, "logits/rejected": -0.23437625169754028, "logps/chosen": -500.760009765625, "logps/ref_chosen": -80.16496276855469, "logps/ref_rejected": -87.69590759277344, "logps/rejected": -583.1189575195312, "loss": 1.3732, "margin_dpo/margin_mean": 74.8279800415039, "margin_dpo/margin_std": 487.329833984375, "step": 204 }, { "KL/chosen_KL_mean": -364.5575256347656, "KL/mean": -444.526611328125, "KL/rejected_KL_mean": -524.4957275390625, "KL/std": 428.46484375, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 159.93820190429688, "fcm_dpo/q_t": 0.49142158031463623, "grad_norm": 4.245858192443848, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -0.1302354335784912, "logits/rejected": -0.14217376708984375, "logps/chosen": -423.9422607421875, "logps/ref_chosen": -59.384735107421875, "logps/ref_rejected": -85.12505340576172, "logps/rejected": -609.6207885742188, "loss": 1.3552, "margin_dpo/margin_mean": 159.93820190429688, "margin_dpo/margin_std": 488.643310546875, "step": 205 }, { "KL/chosen_KL_mean": -275.8797607421875, "KL/mean": -371.9986572265625, "KL/rejected_KL_mean": -468.1175842285156, "KL/std": 284.67950439453125, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 192.2378387451172, "fcm_dpo/q_t": 0.489697128534317, "grad_norm": 3.0303680896759033, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.15259331464767456, "logits/rejected": -0.15699225664138794, "logps/chosen": -322.84423828125, "logps/ref_chosen": -46.964500427246094, "logps/ref_rejected": -98.9534912109375, "logps/rejected": -567.071044921875, "loss": 1.3468, "margin_dpo/margin_mean": 192.2378387451172, "margin_dpo/margin_std": 329.2159423828125, "step": 206 }, { "KL/chosen_KL_mean": -274.165283203125, "KL/mean": -373.14715576171875, "KL/rejected_KL_mean": -472.1290283203125, "KL/std": 336.2535095214844, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 197.9637451171875, "fcm_dpo/q_t": 0.4894152283668518, "grad_norm": 4.557210445404053, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -0.20802870392799377, "logits/rejected": -0.2097472846508026, "logps/chosen": -330.2215576171875, "logps/ref_chosen": -56.05625915527344, "logps/ref_rejected": -84.44779968261719, "logps/rejected": -556.5768432617188, "loss": 1.3464, "margin_dpo/margin_mean": 197.9637451171875, "margin_dpo/margin_std": 424.3826904296875, "step": 207 }, { "KL/chosen_KL_mean": -368.54052734375, "KL/mean": -451.637451171875, "KL/rejected_KL_mean": -534.734375, "KL/std": 326.3774108886719, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 166.19386291503906, "fcm_dpo/q_t": 0.49110543727874756, "grad_norm": 2.7596354484558105, "learning_rate": 4.398512291636768e-07, "logits/chosen": -0.20875459909439087, "logits/rejected": -0.21936002373695374, "logps/chosen": -435.6081237792969, "logps/ref_chosen": -67.06761169433594, "logps/ref_rejected": -94.28689575195312, "logps/rejected": -629.0213012695312, "loss": 1.353, "margin_dpo/margin_mean": 166.19387817382812, "margin_dpo/margin_std": 422.05780029296875, "step": 208 }, { "KL/chosen_KL_mean": -301.03485107421875, "KL/mean": -364.5048828125, "KL/rejected_KL_mean": -427.9748840332031, "KL/std": 285.0235900878906, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.94000244140625, "fcm_dpo/q_t": 0.4932047724723816, "grad_norm": 2.5512523651123047, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -0.1063951924443245, "logits/rejected": -0.124468132853508, "logps/chosen": -357.216552734375, "logps/ref_chosen": -56.18169403076172, "logps/ref_rejected": -80.94152069091797, "logps/rejected": -508.9163818359375, "loss": 1.3607, "margin_dpo/margin_mean": 126.94000244140625, "margin_dpo/margin_std": 349.48199462890625, "step": 209 }, { "KL/chosen_KL_mean": -243.17129516601562, "KL/mean": -323.7019958496094, "KL/rejected_KL_mean": -404.2326965332031, "KL/std": 286.4044189453125, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.06141662597656, "fcm_dpo/q_t": 0.49137187004089355, "grad_norm": 3.259232997894287, "learning_rate": 4.381713366536311e-07, "logits/chosen": -0.1291487067937851, "logits/rejected": -0.14884674549102783, "logps/chosen": -289.5430908203125, "logps/ref_chosen": -46.371822357177734, "logps/ref_rejected": -76.68162536621094, "logps/rejected": -480.914306640625, "loss": 1.3535, "margin_dpo/margin_mean": 161.06143188476562, "margin_dpo/margin_std": 353.78619384765625, "step": 210 }, { "KL/chosen_KL_mean": -375.2530517578125, "KL/mean": -432.86187744140625, "KL/rejected_KL_mean": -490.470703125, "KL/std": 338.1985778808594, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.21764373779297, "fcm_dpo/q_t": 0.4938422441482544, "grad_norm": 2.629580497741699, "learning_rate": 4.373239415645323e-07, "logits/chosen": -0.13434600830078125, "logits/rejected": -0.1492466777563095, "logps/chosen": -454.1854248046875, "logps/ref_chosen": -78.93235778808594, "logps/ref_rejected": -86.82098388671875, "logps/rejected": -577.2916870117188, "loss": 1.3638, "margin_dpo/margin_mean": 115.21763610839844, "margin_dpo/margin_std": 410.43426513671875, "step": 211 }, { "KL/chosen_KL_mean": -300.54364013671875, "KL/mean": -434.3825988769531, "KL/rejected_KL_mean": -568.2215576171875, "KL/std": 372.66168212890625, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 267.6778869628906, "fcm_dpo/q_t": 0.4856899082660675, "grad_norm": 5.687210559844971, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -0.11962936818599701, "logits/rejected": -0.1340862214565277, "logps/chosen": -358.74066162109375, "logps/ref_chosen": -58.19701385498047, "logps/ref_rejected": -103.05785369873047, "logps/rejected": -671.2794189453125, "loss": 1.332, "margin_dpo/margin_mean": 267.6778869628906, "margin_dpo/margin_std": 430.5390625, "step": 212 }, { "KL/chosen_KL_mean": -320.9078063964844, "KL/mean": -405.0869140625, "KL/rejected_KL_mean": -489.2659912109375, "KL/std": 353.5803527832031, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 168.35821533203125, "fcm_dpo/q_t": 0.4909900426864624, "grad_norm": 3.844770908355713, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -0.09740199148654938, "logits/rejected": -0.12570348381996155, "logps/chosen": -388.4205322265625, "logps/ref_chosen": -67.51271057128906, "logps/ref_rejected": -93.91471862792969, "logps/rejected": -583.1807250976562, "loss": 1.3531, "margin_dpo/margin_mean": 168.35821533203125, "margin_dpo/margin_std": 470.94073486328125, "step": 213 }, { "KL/chosen_KL_mean": -238.504638671875, "KL/mean": -338.5084228515625, "KL/rejected_KL_mean": -438.51214599609375, "KL/std": 269.5102233886719, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 200.00753784179688, "fcm_dpo/q_t": 0.4892929792404175, "grad_norm": 3.0564281940460205, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -0.06811121106147766, "logits/rejected": -0.07607707381248474, "logps/chosen": -280.1095275878906, "logps/ref_chosen": -41.604888916015625, "logps/ref_rejected": -77.51741027832031, "logps/rejected": -516.029541015625, "loss": 1.3451, "margin_dpo/margin_mean": 200.0075225830078, "margin_dpo/margin_std": 330.0015869140625, "step": 214 }, { "KL/chosen_KL_mean": -308.419677734375, "KL/mean": -413.72235107421875, "KL/rejected_KL_mean": -519.0250854492188, "KL/std": 338.5547790527344, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 210.60540771484375, "fcm_dpo/q_t": 0.4887344241142273, "grad_norm": 4.060998916625977, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -0.1189073771238327, "logits/rejected": -0.13333210349082947, "logps/chosen": -361.69891357421875, "logps/ref_chosen": -53.279266357421875, "logps/ref_rejected": -89.96464538574219, "logps/rejected": -608.98974609375, "loss": 1.3432, "margin_dpo/margin_mean": 210.60540771484375, "margin_dpo/margin_std": 367.251708984375, "step": 215 }, { "KL/chosen_KL_mean": -300.3120422363281, "KL/mean": -386.34326171875, "KL/rejected_KL_mean": -472.37445068359375, "KL/std": 297.8331298828125, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 172.0624237060547, "fcm_dpo/q_t": 0.49078911542892456, "grad_norm": 3.7745511531829834, "learning_rate": 4.330133748510036e-07, "logits/chosen": -0.12054896354675293, "logits/rejected": -0.12499400973320007, "logps/chosen": -349.1998291015625, "logps/ref_chosen": -48.887794494628906, "logps/ref_rejected": -77.19892883300781, "logps/rejected": -549.5733642578125, "loss": 1.3514, "margin_dpo/margin_mean": 172.0624237060547, "margin_dpo/margin_std": 384.0893249511719, "step": 216 }, { "KL/chosen_KL_mean": -347.5061340332031, "KL/mean": -462.15380859375, "KL/rejected_KL_mean": -576.801513671875, "KL/std": 366.660888671875, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 229.29534912109375, "fcm_dpo/q_t": 0.48774290084838867, "grad_norm": 3.4250800609588623, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -0.09066063165664673, "logits/rejected": -0.10337048768997192, "logps/chosen": -397.3514404296875, "logps/ref_chosen": -49.845306396484375, "logps/ref_rejected": -100.07832336425781, "logps/rejected": -676.8798217773438, "loss": 1.3402, "margin_dpo/margin_mean": 229.29534912109375, "margin_dpo/margin_std": 456.92340087890625, "step": 217 }, { "KL/chosen_KL_mean": -365.1001892089844, "KL/mean": -444.54656982421875, "KL/rejected_KL_mean": -523.9929809570312, "KL/std": 365.8412170410156, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 158.892822265625, "fcm_dpo/q_t": 0.4915262460708618, "grad_norm": 3.7529687881469727, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -0.14683587849140167, "logits/rejected": -0.16048407554626465, "logps/chosen": -423.6768798828125, "logps/ref_chosen": -58.576683044433594, "logps/ref_rejected": -87.84639739990234, "logps/rejected": -611.83935546875, "loss": 1.3551, "margin_dpo/margin_mean": 158.892822265625, "margin_dpo/margin_std": 475.37939453125, "step": 218 }, { "KL/chosen_KL_mean": -386.04595947265625, "KL/mean": -442.3204650878906, "KL/rejected_KL_mean": -498.594970703125, "KL/std": 393.7286376953125, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.54901123046875, "fcm_dpo/q_t": 0.4940031170845032, "grad_norm": 4.430355548858643, "learning_rate": 4.303689819449636e-07, "logits/chosen": -0.19574600458145142, "logits/rejected": -0.21589338779449463, "logps/chosen": -447.12982177734375, "logps/ref_chosen": -61.083858489990234, "logps/ref_rejected": -85.83042907714844, "logps/rejected": -584.4254150390625, "loss": 1.3655, "margin_dpo/margin_mean": 112.54901885986328, "margin_dpo/margin_std": 510.53802490234375, "step": 219 }, { "KL/chosen_KL_mean": -407.1309814453125, "KL/mean": -457.27777099609375, "KL/rejected_KL_mean": -507.4245910644531, "KL/std": 324.55487060546875, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.29361724853516, "fcm_dpo/q_t": 0.49463126063346863, "grad_norm": 3.9129955768585205, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -0.22126300632953644, "logits/rejected": -0.21606583893299103, "logps/chosen": -477.16229248046875, "logps/ref_chosen": -70.03128051757812, "logps/ref_rejected": -87.68551635742188, "logps/rejected": -595.110107421875, "loss": 1.3665, "margin_dpo/margin_mean": 100.29361724853516, "margin_dpo/margin_std": 378.6761169433594, "step": 220 }, { "KL/chosen_KL_mean": -290.21429443359375, "KL/mean": -445.29681396484375, "KL/rejected_KL_mean": -600.3792724609375, "KL/std": 374.867919921875, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 310.1649475097656, "fcm_dpo/q_t": 0.4834199547767639, "grad_norm": 7.014521598815918, "learning_rate": 4.285822501755485e-07, "logits/chosen": -0.23672357201576233, "logits/rejected": -0.24645334482192993, "logps/chosen": -342.3690185546875, "logps/ref_chosen": -52.15470886230469, "logps/ref_rejected": -106.46768188476562, "logps/rejected": -706.846923828125, "loss": 1.323, "margin_dpo/margin_mean": 310.1649475097656, "margin_dpo/margin_std": 433.8343811035156, "step": 221 }, { "KL/chosen_KL_mean": -317.06378173828125, "KL/mean": -416.68621826171875, "KL/rejected_KL_mean": -516.3086547851562, "KL/std": 329.75885009765625, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 199.24484252929688, "fcm_dpo/q_t": 0.48933765292167664, "grad_norm": 3.5604019165039062, "learning_rate": 4.276818137766118e-07, "logits/chosen": -0.24718379974365234, "logits/rejected": -0.27352237701416016, "logps/chosen": -378.0348815917969, "logps/ref_chosen": -60.971099853515625, "logps/ref_rejected": -100.00115203857422, "logps/rejected": -616.309814453125, "loss": 1.3459, "margin_dpo/margin_mean": 199.24484252929688, "margin_dpo/margin_std": 405.140869140625, "step": 222 }, { "KL/chosen_KL_mean": -326.71197509765625, "KL/mean": -409.00048828125, "KL/rejected_KL_mean": -491.2890319824219, "KL/std": 325.3358154296875, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.57704162597656, "fcm_dpo/q_t": 0.49121803045272827, "grad_norm": 3.8082237243652344, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -0.13181817531585693, "logits/rejected": -0.14403775334358215, "logps/chosen": -379.3525390625, "logps/ref_chosen": -52.64057540893555, "logps/ref_rejected": -82.82502746582031, "logps/rejected": -574.1140747070312, "loss": 1.3537, "margin_dpo/margin_mean": 164.57704162597656, "margin_dpo/margin_std": 439.234619140625, "step": 223 }, { "KL/chosen_KL_mean": -307.269287109375, "KL/mean": -398.96954345703125, "KL/rejected_KL_mean": -490.6697998046875, "KL/std": 385.791748046875, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 183.40045166015625, "fcm_dpo/q_t": 0.49022960662841797, "grad_norm": 3.7661983966827393, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -0.1584322154521942, "logits/rejected": -0.1633099913597107, "logps/chosen": -355.8647155761719, "logps/ref_chosen": -48.59541320800781, "logps/ref_rejected": -77.11648559570312, "logps/rejected": -567.7862548828125, "loss": 1.3501, "margin_dpo/margin_mean": 183.40045166015625, "margin_dpo/margin_std": 472.4094543457031, "step": 224 }, { "KL/chosen_KL_mean": -362.7328186035156, "KL/mean": -477.8741760253906, "KL/rejected_KL_mean": -593.0155639648438, "KL/std": 375.5802001953125, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 230.28273010253906, "fcm_dpo/q_t": 0.4877273440361023, "grad_norm": 3.3898706436157227, "learning_rate": 4.249525076191759e-07, "logits/chosen": -0.14341585338115692, "logits/rejected": -0.15620696544647217, "logps/chosen": -420.7332763671875, "logps/ref_chosen": -58.000465393066406, "logps/ref_rejected": -99.90291595458984, "logps/rejected": -692.91845703125, "loss": 1.3406, "margin_dpo/margin_mean": 230.28274536132812, "margin_dpo/margin_std": 511.2721862792969, "step": 225 }, { "KL/chosen_KL_mean": -322.0030517578125, "KL/mean": -404.18328857421875, "KL/rejected_KL_mean": -486.36346435546875, "KL/std": 324.75518798828125, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.36044311523438, "fcm_dpo/q_t": 0.49121221899986267, "grad_norm": 3.274152994155884, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -0.19513732194900513, "logits/rejected": -0.19808658957481384, "logps/chosen": -380.90185546875, "logps/ref_chosen": -58.898799896240234, "logps/ref_rejected": -78.68775939941406, "logps/rejected": -565.05126953125, "loss": 1.3535, "margin_dpo/margin_mean": 164.36044311523438, "margin_dpo/margin_std": 427.1483154296875, "step": 226 }, { "KL/chosen_KL_mean": -357.85113525390625, "KL/mean": -482.2805480957031, "KL/rejected_KL_mean": -606.7099609375, "KL/std": 433.9095458984375, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 248.85882568359375, "fcm_dpo/q_t": 0.48674476146698, "grad_norm": 3.5155231952667236, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -0.20451249182224274, "logits/rejected": -0.20552946627140045, "logps/chosen": -416.92333984375, "logps/ref_chosen": -59.072181701660156, "logps/ref_rejected": -99.41236877441406, "logps/rejected": -706.122314453125, "loss": 1.3364, "margin_dpo/margin_mean": 248.85882568359375, "margin_dpo/margin_std": 477.48724365234375, "step": 227 }, { "KL/chosen_KL_mean": -384.4470520019531, "KL/mean": -456.94488525390625, "KL/rejected_KL_mean": -529.4427490234375, "KL/std": 377.869140625, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 144.99569702148438, "fcm_dpo/q_t": 0.4922485947608948, "grad_norm": 3.4961659908294678, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -0.15710148215293884, "logits/rejected": -0.1570408046245575, "logps/chosen": -450.33831787109375, "logps/ref_chosen": -65.89128875732422, "logps/ref_rejected": -91.04875183105469, "logps/rejected": -620.4915161132812, "loss": 1.3584, "margin_dpo/margin_mean": 144.99569702148438, "margin_dpo/margin_std": 505.9360656738281, "step": 228 }, { "KL/chosen_KL_mean": -427.20635986328125, "KL/mean": -472.91815185546875, "KL/rejected_KL_mean": -518.6299438476562, "KL/std": 411.049560546875, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.4235610961914, "fcm_dpo/q_t": 0.49511969089508057, "grad_norm": 4.707752227783203, "learning_rate": 4.212490049118951e-07, "logits/chosen": -0.23124007880687714, "logits/rejected": -0.23222056031227112, "logps/chosen": -497.9127502441406, "logps/ref_chosen": -70.70637512207031, "logps/ref_rejected": -84.52741241455078, "logps/rejected": -603.1573486328125, "loss": 1.3692, "margin_dpo/margin_mean": 91.4235610961914, "margin_dpo/margin_std": 452.021240234375, "step": 229 }, { "KL/chosen_KL_mean": -248.62451171875, "KL/mean": -390.31243896484375, "KL/rejected_KL_mean": -532.0003662109375, "KL/std": 303.40869140625, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 283.3758544921875, "fcm_dpo/q_t": 0.48483169078826904, "grad_norm": 3.5236966609954834, "learning_rate": 4.203117865141635e-07, "logits/chosen": -0.10768608748912811, "logits/rejected": -0.11796237528324127, "logps/chosen": -287.9065246582031, "logps/ref_chosen": -39.282005310058594, "logps/ref_rejected": -85.62191009521484, "logps/rejected": -617.622314453125, "loss": 1.3278, "margin_dpo/margin_mean": 283.3758544921875, "margin_dpo/margin_std": 349.93524169921875, "step": 230 }, { "KL/chosen_KL_mean": -329.960693359375, "KL/mean": -397.3656921386719, "KL/rejected_KL_mean": -464.77069091796875, "KL/std": 273.9499816894531, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.8100128173828, "fcm_dpo/q_t": 0.49278855323791504, "grad_norm": 3.0640761852264404, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -0.19377461075782776, "logits/rejected": -0.1955992579460144, "logps/chosen": -393.23712158203125, "logps/ref_chosen": -63.27644348144531, "logps/ref_rejected": -74.1239013671875, "logps/rejected": -538.8945922851562, "loss": 1.359, "margin_dpo/margin_mean": 134.81002807617188, "margin_dpo/margin_std": 346.1912536621094, "step": 231 }, { "KL/chosen_KL_mean": -400.56787109375, "KL/mean": -449.58966064453125, "KL/rejected_KL_mean": -498.61151123046875, "KL/std": 334.50140380859375, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.04367065429688, "fcm_dpo/q_t": 0.4947741627693176, "grad_norm": 5.453951358795166, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -0.19259104132652283, "logits/rejected": -0.20236793160438538, "logps/chosen": -471.3166198730469, "logps/ref_chosen": -70.74876403808594, "logps/ref_rejected": -83.97706604003906, "logps/rejected": -582.5885620117188, "loss": 1.3671, "margin_dpo/margin_mean": 98.04367065429688, "margin_dpo/margin_std": 388.8453369140625, "step": 232 }, { "KL/chosen_KL_mean": -313.58184814453125, "KL/mean": -445.1939697265625, "KL/rejected_KL_mean": -576.8060302734375, "KL/std": 369.1995849609375, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 263.2242431640625, "fcm_dpo/q_t": 0.48596352338790894, "grad_norm": 4.564657211303711, "learning_rate": 4.174733034541245e-07, "logits/chosen": -0.19956615567207336, "logits/rejected": -0.21853141486644745, "logps/chosen": -368.46478271484375, "logps/ref_chosen": -54.8829345703125, "logps/ref_rejected": -107.4800796508789, "logps/rejected": -684.2861328125, "loss": 1.334, "margin_dpo/margin_mean": 263.2242431640625, "margin_dpo/margin_std": 516.1271362304688, "step": 233 }, { "KL/chosen_KL_mean": -285.1370849609375, "KL/mean": -427.95343017578125, "KL/rejected_KL_mean": -570.7697143554688, "KL/std": 362.5662841796875, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 285.6326904296875, "fcm_dpo/q_t": 0.4847436547279358, "grad_norm": 3.6827473640441895, "learning_rate": 4.165182829193126e-07, "logits/chosen": -0.20543476939201355, "logits/rejected": -0.2246108204126358, "logps/chosen": -329.2315673828125, "logps/ref_chosen": -44.094520568847656, "logps/ref_rejected": -100.00663757324219, "logps/rejected": -670.7763671875, "loss": 1.3283, "margin_dpo/margin_mean": 285.6326599121094, "margin_dpo/margin_std": 430.40399169921875, "step": 234 }, { "KL/chosen_KL_mean": -379.80419921875, "KL/mean": -452.5826416015625, "KL/rejected_KL_mean": -525.361083984375, "KL/std": 368.2503662109375, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.556884765625, "fcm_dpo/q_t": 0.4922545552253723, "grad_norm": 4.7766242027282715, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -0.1708293855190277, "logits/rejected": -0.18167471885681152, "logps/chosen": -442.0421142578125, "logps/ref_chosen": -62.237911224365234, "logps/ref_rejected": -90.39506530761719, "logps/rejected": -615.756103515625, "loss": 1.3582, "margin_dpo/margin_mean": 145.556884765625, "margin_dpo/margin_std": 496.21075439453125, "step": 235 }, { "KL/chosen_KL_mean": -263.3704833984375, "KL/mean": -413.715087890625, "KL/rejected_KL_mean": -564.0596923828125, "KL/std": 374.974609375, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 300.689208984375, "fcm_dpo/q_t": 0.48393750190734863, "grad_norm": 10.610316276550293, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -0.1809891164302826, "logits/rejected": -0.1887068748474121, "logps/chosen": -312.71185302734375, "logps/ref_chosen": -49.34136199951172, "logps/ref_rejected": -103.51162719726562, "logps/rejected": -667.5713500976562, "loss": 1.325, "margin_dpo/margin_mean": 300.689208984375, "margin_dpo/margin_std": 420.67352294921875, "step": 236 }, { "KL/chosen_KL_mean": -378.86138916015625, "KL/mean": -480.03680419921875, "KL/rejected_KL_mean": -581.2122802734375, "KL/std": 396.18927001953125, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 202.35086059570312, "fcm_dpo/q_t": 0.4892232120037079, "grad_norm": 4.470569610595703, "learning_rate": 4.136269950853473e-07, "logits/chosen": -0.180775448679924, "logits/rejected": -0.19322538375854492, "logps/chosen": -433.0295104980469, "logps/ref_chosen": -54.168121337890625, "logps/ref_rejected": -94.78036499023438, "logps/rejected": -675.9926147460938, "loss": 1.3464, "margin_dpo/margin_mean": 202.35086059570312, "margin_dpo/margin_std": 515.0560913085938, "step": 237 }, { "KL/chosen_KL_mean": -354.31298828125, "KL/mean": -458.6822509765625, "KL/rejected_KL_mean": -563.051513671875, "KL/std": 386.575439453125, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 208.73849487304688, "fcm_dpo/q_t": 0.48885422945022583, "grad_norm": 4.142996311187744, "learning_rate": 4.126545750510605e-07, "logits/chosen": -0.18518191576004028, "logits/rejected": -0.19046491384506226, "logps/chosen": -408.2861328125, "logps/ref_chosen": -53.973121643066406, "logps/ref_rejected": -89.41795349121094, "logps/rejected": -652.469482421875, "loss": 1.3443, "margin_dpo/margin_mean": 208.7384796142578, "margin_dpo/margin_std": 444.2572021484375, "step": 238 }, { "KL/chosen_KL_mean": -395.993408203125, "KL/mean": -492.276123046875, "KL/rejected_KL_mean": -588.558837890625, "KL/std": 340.7003173828125, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 192.56541442871094, "fcm_dpo/q_t": 0.4896959066390991, "grad_norm": 3.825457811355591, "learning_rate": 4.116778689174514e-07, "logits/chosen": -0.16619744896888733, "logits/rejected": -0.18394985795021057, "logps/chosen": -454.09124755859375, "logps/ref_chosen": -58.09782409667969, "logps/ref_rejected": -93.59294128417969, "logps/rejected": -682.1517333984375, "loss": 1.3476, "margin_dpo/margin_mean": 192.5654296875, "margin_dpo/margin_std": 428.50225830078125, "step": 239 }, { "KL/chosen_KL_mean": -454.6864013671875, "KL/mean": -506.56591796875, "KL/rejected_KL_mean": -558.4453735351562, "KL/std": 395.06903076171875, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.75895690917969, "fcm_dpo/q_t": 0.49443739652633667, "grad_norm": 3.9248123168945312, "learning_rate": 4.106969024216348e-07, "logits/chosen": -0.19379109144210815, "logits/rejected": -0.19150272011756897, "logps/chosen": -515.3009033203125, "logps/ref_chosen": -60.6144905090332, "logps/ref_rejected": -74.1185302734375, "logps/rejected": -632.5639038085938, "loss": 1.367, "margin_dpo/margin_mean": 103.75895690917969, "margin_dpo/margin_std": 491.4389953613281, "step": 240 }, { "KL/chosen_KL_mean": -433.6764221191406, "KL/mean": -542.6292724609375, "KL/rejected_KL_mean": -651.5821533203125, "KL/std": 436.55352783203125, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 217.90567016601562, "fcm_dpo/q_t": 0.4883774518966675, "grad_norm": 4.787429332733154, "learning_rate": 4.097117014129903e-07, "logits/chosen": -0.22374919056892395, "logits/rejected": -0.22779551148414612, "logps/chosen": -499.76751708984375, "logps/ref_chosen": -66.091064453125, "logps/ref_rejected": -88.06088256835938, "logps/rejected": -739.6430053710938, "loss": 1.3435, "margin_dpo/margin_mean": 217.9056854248047, "margin_dpo/margin_std": 529.632568359375, "step": 241 }, { "KL/chosen_KL_mean": -466.04278564453125, "KL/mean": -527.4720458984375, "KL/rejected_KL_mean": -588.9013671875, "KL/std": 415.62457275390625, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.85851287841797, "fcm_dpo/q_t": 0.4934060573577881, "grad_norm": 4.762578010559082, "learning_rate": 4.087222918524807e-07, "logits/chosen": -0.2294858694076538, "logits/rejected": -0.23094022274017334, "logps/chosen": -533.90673828125, "logps/ref_chosen": -67.86392974853516, "logps/ref_rejected": -83.36033630371094, "logps/rejected": -672.2616577148438, "loss": 1.364, "margin_dpo/margin_mean": 122.85851287841797, "margin_dpo/margin_std": 575.6241455078125, "step": 242 }, { "KL/chosen_KL_mean": -402.603759765625, "KL/mean": -486.40032958984375, "KL/rejected_KL_mean": -570.1968994140625, "KL/std": 361.3595886230469, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 167.5931396484375, "fcm_dpo/q_t": 0.4910268187522888, "grad_norm": 4.299717903137207, "learning_rate": 4.07728699811968e-07, "logits/chosen": -0.22352221608161926, "logits/rejected": -0.23877659440040588, "logps/chosen": -465.6880187988281, "logps/ref_chosen": -63.0842399597168, "logps/ref_rejected": -76.33563232421875, "logps/rejected": -646.5325927734375, "loss": 1.3528, "margin_dpo/margin_mean": 167.59312438964844, "margin_dpo/margin_std": 422.52716064453125, "step": 243 }, { "KL/chosen_KL_mean": -334.23394775390625, "KL/mean": -451.40582275390625, "KL/rejected_KL_mean": -568.57763671875, "KL/std": 372.09405517578125, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 234.34368896484375, "fcm_dpo/q_t": 0.48746874928474426, "grad_norm": 4.2224202156066895, "learning_rate": 4.067309514735267e-07, "logits/chosen": -0.2551266849040985, "logits/rejected": -0.27021169662475586, "logps/chosen": -395.3746643066406, "logps/ref_chosen": -61.140689849853516, "logps/ref_rejected": -94.89193725585938, "logps/rejected": -663.4696044921875, "loss": 1.3385, "margin_dpo/margin_mean": 234.34368896484375, "margin_dpo/margin_std": 386.5777893066406, "step": 244 }, { "KL/chosen_KL_mean": -376.46820068359375, "KL/mean": -438.8470153808594, "KL/rejected_KL_mean": -501.2258605957031, "KL/std": 329.3731689453125, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.75768280029297, "fcm_dpo/q_t": 0.4933217763900757, "grad_norm": 3.401407480239868, "learning_rate": 4.057290731287531e-07, "logits/chosen": -0.2517125606536865, "logits/rejected": -0.24654418230056763, "logps/chosen": -443.73046875, "logps/ref_chosen": -67.26228332519531, "logps/ref_rejected": -87.64010620117188, "logps/rejected": -588.865966796875, "loss": 1.3615, "margin_dpo/margin_mean": 124.75768280029297, "margin_dpo/margin_std": 384.70745849609375, "step": 245 }, { "KL/chosen_KL_mean": -330.4073486328125, "KL/mean": -413.43792724609375, "KL/rejected_KL_mean": -496.468505859375, "KL/std": 368.08837890625, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 166.0610809326172, "fcm_dpo/q_t": 0.4911186397075653, "grad_norm": 4.492984294891357, "learning_rate": 4.047230911780736e-07, "logits/chosen": -0.25868698954582214, "logits/rejected": -0.2564406394958496, "logps/chosen": -397.10430908203125, "logps/ref_chosen": -66.69696807861328, "logps/ref_rejected": -84.34634399414062, "logps/rejected": -580.8148193359375, "loss": 1.353, "margin_dpo/margin_mean": 166.0610809326172, "margin_dpo/margin_std": 415.17022705078125, "step": 246 }, { "KL/chosen_KL_mean": -346.08819580078125, "KL/mean": -501.11810302734375, "KL/rejected_KL_mean": -656.1480712890625, "KL/std": 380.09771728515625, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 310.059814453125, "fcm_dpo/q_t": 0.48344653844833374, "grad_norm": 6.050650596618652, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -0.23237216472625732, "logits/rejected": -0.24301677942276, "logps/chosen": -402.69354248046875, "logps/ref_chosen": -56.6053466796875, "logps/ref_rejected": -106.29326629638672, "logps/rejected": -762.4412841796875, "loss": 1.3237, "margin_dpo/margin_mean": 310.0598449707031, "margin_dpo/margin_std": 467.6119689941406, "step": 247 }, { "KL/chosen_KL_mean": -282.9842529296875, "KL/mean": -411.3094482421875, "KL/rejected_KL_mean": -539.6346435546875, "KL/std": 325.2694396972656, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 256.65032958984375, "fcm_dpo/q_t": 0.4862789809703827, "grad_norm": 4.713438034057617, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -0.18797361850738525, "logits/rejected": -0.21102368831634521, "logps/chosen": -327.0274963378906, "logps/ref_chosen": -44.043216705322266, "logps/ref_rejected": -91.85687255859375, "logps/rejected": -631.4915161132812, "loss": 1.3337, "margin_dpo/margin_mean": 256.65032958984375, "margin_dpo/margin_std": 363.392822265625, "step": 248 }, { "KL/chosen_KL_mean": -423.87060546875, "KL/mean": -470.3067626953125, "KL/rejected_KL_mean": -516.742919921875, "KL/std": 390.5693054199219, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.87237548828125, "fcm_dpo/q_t": 0.49507373571395874, "grad_norm": 3.928159236907959, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -0.16814130544662476, "logits/rejected": -0.17811042070388794, "logps/chosen": -486.31292724609375, "logps/ref_chosen": -62.442352294921875, "logps/ref_rejected": -80.46806335449219, "logps/rejected": -597.2110595703125, "loss": 1.3692, "margin_dpo/margin_mean": 92.87238311767578, "margin_dpo/margin_std": 484.80364990234375, "step": 249 }, { "KL/chosen_KL_mean": -410.2246398925781, "KL/mean": -485.90521240234375, "KL/rejected_KL_mean": -561.5857543945312, "KL/std": 377.61981201171875, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 151.361083984375, "fcm_dpo/q_t": 0.49189114570617676, "grad_norm": 4.632651329040527, "learning_rate": 4.006586590948141e-07, "logits/chosen": -0.1685868501663208, "logits/rejected": -0.17339754104614258, "logps/chosen": -475.861328125, "logps/ref_chosen": -65.63668823242188, "logps/ref_rejected": -73.87184143066406, "logps/rejected": -635.4575805664062, "loss": 1.3563, "margin_dpo/margin_mean": 151.361083984375, "margin_dpo/margin_std": 436.9773254394531, "step": 250 }, { "KL/chosen_KL_mean": -401.18408203125, "KL/mean": -462.01068115234375, "KL/rejected_KL_mean": -522.8372192382812, "KL/std": 380.92681884765625, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.65315246582031, "fcm_dpo/q_t": 0.49349498748779297, "grad_norm": 4.47353458404541, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -0.1774073839187622, "logits/rejected": -0.18519741296768188, "logps/chosen": -458.3667907714844, "logps/ref_chosen": -57.182716369628906, "logps/ref_rejected": -77.66343688964844, "logps/rejected": -600.5006713867188, "loss": 1.3627, "margin_dpo/margin_mean": 121.65316772460938, "margin_dpo/margin_std": 439.44793701171875, "step": 251 }, { "KL/chosen_KL_mean": -429.9615173339844, "KL/mean": -518.6616821289062, "KL/rejected_KL_mean": -607.36181640625, "KL/std": 336.4705505371094, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 177.4003143310547, "fcm_dpo/q_t": 0.4905024766921997, "grad_norm": 4.228276252746582, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -0.20860856771469116, "logits/rejected": -0.19886748492717743, "logps/chosen": -501.64715576171875, "logps/ref_chosen": -71.68563842773438, "logps/ref_rejected": -84.75799560546875, "logps/rejected": -692.1198120117188, "loss": 1.3501, "margin_dpo/margin_mean": 177.4003143310547, "margin_dpo/margin_std": 356.32275390625, "step": 252 }, { "KL/chosen_KL_mean": -392.266357421875, "KL/mean": -508.9656066894531, "KL/rejected_KL_mean": -625.664794921875, "KL/std": 355.4100341796875, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.3984832763672, "fcm_dpo/q_t": 0.4875214695930481, "grad_norm": 3.7770397663116455, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -0.19992950558662415, "logits/rejected": -0.19830244779586792, "logps/chosen": -461.4002990722656, "logps/ref_chosen": -69.1339340209961, "logps/ref_rejected": -98.70252990722656, "logps/rejected": -724.3673706054688, "loss": 1.3391, "margin_dpo/margin_mean": 233.39849853515625, "margin_dpo/margin_std": 419.17742919921875, "step": 253 }, { "KL/chosen_KL_mean": -392.25958251953125, "KL/mean": -469.58319091796875, "KL/rejected_KL_mean": -546.90673828125, "KL/std": 388.672607421875, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.64715576171875, "fcm_dpo/q_t": 0.4917369484901428, "grad_norm": 4.755723476409912, "learning_rate": 3.965307091713037e-07, "logits/chosen": -0.14065057039260864, "logits/rejected": -0.1489948332309723, "logps/chosen": -446.41461181640625, "logps/ref_chosen": -54.154998779296875, "logps/ref_rejected": -90.30764770507812, "logps/rejected": -637.21435546875, "loss": 1.3558, "margin_dpo/margin_mean": 154.64715576171875, "margin_dpo/margin_std": 455.6612854003906, "step": 254 }, { "KL/chosen_KL_mean": -382.1084899902344, "KL/mean": -481.79656982421875, "KL/rejected_KL_mean": -581.484619140625, "KL/std": 380.31219482421875, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 199.37615966796875, "fcm_dpo/q_t": 0.48935091495513916, "grad_norm": 3.1735172271728516, "learning_rate": 3.954890003969163e-07, "logits/chosen": -0.12860265374183655, "logits/rejected": -0.1277311146259308, "logps/chosen": -439.2501525878906, "logps/ref_chosen": -57.14167022705078, "logps/ref_rejected": -90.2085952758789, "logps/rejected": -671.6932373046875, "loss": 1.3468, "margin_dpo/margin_mean": 199.37615966796875, "margin_dpo/margin_std": 478.7210388183594, "step": 255 }, { "KL/chosen_KL_mean": -343.8599548339844, "KL/mean": -461.6353454589844, "KL/rejected_KL_mean": -579.4107666015625, "KL/std": 413.0328674316406, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 235.55078125, "fcm_dpo/q_t": 0.4874057471752167, "grad_norm": 4.488049030303955, "learning_rate": 3.944434578520628e-07, "logits/chosen": -0.10436695069074631, "logits/rejected": -0.12606412172317505, "logps/chosen": -399.0234375, "logps/ref_chosen": -55.163490295410156, "logps/ref_rejected": -92.56291961669922, "logps/rejected": -671.9736328125, "loss": 1.3388, "margin_dpo/margin_mean": 235.55078125, "margin_dpo/margin_std": 458.3751220703125, "step": 256 }, { "KL/chosen_KL_mean": -310.64581298828125, "KL/mean": -431.5376281738281, "KL/rejected_KL_mean": -552.429443359375, "KL/std": 358.80474853515625, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 241.78359985351562, "fcm_dpo/q_t": 0.4870920777320862, "grad_norm": 4.349320888519287, "learning_rate": 3.933941090877615e-07, "logits/chosen": -0.13144733011722565, "logits/rejected": -0.13504740595817566, "logps/chosen": -360.06951904296875, "logps/ref_chosen": -49.42369842529297, "logps/ref_rejected": -79.53791809082031, "logps/rejected": -631.9673461914062, "loss": 1.3377, "margin_dpo/margin_mean": 241.78358459472656, "margin_dpo/margin_std": 457.3230895996094, "step": 257 }, { "KL/chosen_KL_mean": -410.16473388671875, "KL/mean": -527.0640869140625, "KL/rejected_KL_mean": -643.96337890625, "KL/std": 395.2886047363281, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.79864501953125, "fcm_dpo/q_t": 0.4875029921531677, "grad_norm": 3.8991451263427734, "learning_rate": 3.923409817553284e-07, "logits/chosen": -0.10106329619884491, "logits/rejected": -0.10983704030513763, "logps/chosen": -469.54888916015625, "logps/ref_chosen": -59.384124755859375, "logps/ref_rejected": -95.99010467529297, "logps/rejected": -739.9534912109375, "loss": 1.3397, "margin_dpo/margin_mean": 233.79864501953125, "margin_dpo/margin_std": 500.43988037109375, "step": 258 }, { "KL/chosen_KL_mean": -380.03558349609375, "KL/mean": -472.2444152832031, "KL/rejected_KL_mean": -564.4532470703125, "KL/std": 359.38470458984375, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 184.4176788330078, "fcm_dpo/q_t": 0.49013984203338623, "grad_norm": 3.6584014892578125, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -0.19331489503383636, "logits/rejected": -0.2091323733329773, "logps/chosen": -432.8639221191406, "logps/ref_chosen": -52.828346252441406, "logps/ref_rejected": -89.191650390625, "logps/rejected": -653.6448974609375, "loss": 1.3494, "margin_dpo/margin_mean": 184.41766357421875, "margin_dpo/margin_std": 437.661865234375, "step": 259 }, { "KL/chosen_KL_mean": -365.8872375488281, "KL/mean": -518.8782958984375, "KL/rejected_KL_mean": -671.8694458007812, "KL/std": 458.7395324707031, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 305.98223876953125, "fcm_dpo/q_t": 0.48385488986968994, "grad_norm": 3.9910993576049805, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -0.15303289890289307, "logits/rejected": -0.16400694847106934, "logps/chosen": -413.3049011230469, "logps/ref_chosen": -47.41767501831055, "logps/ref_rejected": -95.08978271484375, "logps/rejected": -766.959228515625, "loss": 1.3258, "margin_dpo/margin_mean": 305.98223876953125, "margin_dpo/margin_std": 550.6990966796875, "step": 260 }, { "KL/chosen_KL_mean": -419.76605224609375, "KL/mean": -529.2583618164062, "KL/rejected_KL_mean": -638.7506713867188, "KL/std": 460.34503173828125, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 218.98464965820312, "fcm_dpo/q_t": 0.4883573055267334, "grad_norm": 5.955752372741699, "learning_rate": 3.891592063515376e-07, "logits/chosen": -0.08432209491729736, "logits/rejected": -0.09958672523498535, "logps/chosen": -472.79742431640625, "logps/ref_chosen": -53.03137969970703, "logps/ref_rejected": -88.51494598388672, "logps/rejected": -727.265625, "loss": 1.3439, "margin_dpo/margin_mean": 218.98464965820312, "margin_dpo/margin_std": 580.8803100585938, "step": 261 }, { "KL/chosen_KL_mean": -492.68560791015625, "KL/mean": -573.60986328125, "KL/rejected_KL_mean": -654.5340576171875, "KL/std": 429.9180603027344, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.84848022460938, "fcm_dpo/q_t": 0.49136343598365784, "grad_norm": 5.33870792388916, "learning_rate": 3.880912432401264e-07, "logits/chosen": -0.10938498377799988, "logits/rejected": -0.10725777596235275, "logps/chosen": -552.3057861328125, "logps/ref_chosen": -59.620140075683594, "logps/ref_rejected": -86.41853332519531, "logps/rejected": -740.95263671875, "loss": 1.3542, "margin_dpo/margin_mean": 161.84848022460938, "margin_dpo/margin_std": 447.91815185546875, "step": 262 }, { "KL/chosen_KL_mean": -442.79217529296875, "KL/mean": -572.5969848632812, "KL/rejected_KL_mean": -702.40185546875, "KL/std": 522.2442016601562, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 259.609619140625, "fcm_dpo/q_t": 0.4861585795879364, "grad_norm": 3.8708550930023193, "learning_rate": 3.870196412960302e-07, "logits/chosen": -0.12460441887378693, "logits/rejected": -0.13606280088424683, "logps/chosen": -502.2131042480469, "logps/ref_chosen": -59.42094421386719, "logps/ref_rejected": -96.85720825195312, "logps/rejected": -799.259033203125, "loss": 1.3362, "margin_dpo/margin_mean": 259.609619140625, "margin_dpo/margin_std": 626.2482299804688, "step": 263 }, { "KL/chosen_KL_mean": -469.1533508300781, "KL/mean": -582.8370361328125, "KL/rejected_KL_mean": -696.5206909179688, "KL/std": 439.26593017578125, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 227.36734008789062, "fcm_dpo/q_t": 0.4878886938095093, "grad_norm": 3.9945685863494873, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -0.15970063209533691, "logits/rejected": -0.17009925842285156, "logps/chosen": -531.8754272460938, "logps/ref_chosen": -62.722084045410156, "logps/ref_rejected": -93.85620880126953, "logps/rejected": -790.376953125, "loss": 1.342, "margin_dpo/margin_mean": 227.36734008789062, "margin_dpo/margin_std": 566.97900390625, "step": 264 }, { "KL/chosen_KL_mean": -494.053466796875, "KL/mean": -583.5369873046875, "KL/rejected_KL_mean": -673.0205078125, "KL/std": 433.59552001953125, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 178.96697998046875, "fcm_dpo/q_t": 0.49045416712760925, "grad_norm": 4.128632068634033, "learning_rate": 3.848656339557562e-07, "logits/chosen": -0.09494436532258987, "logits/rejected": -0.10667555034160614, "logps/chosen": -556.02490234375, "logps/ref_chosen": -61.971466064453125, "logps/ref_rejected": -88.02059936523438, "logps/rejected": -761.0410766601562, "loss": 1.3517, "margin_dpo/margin_mean": 178.96697998046875, "margin_dpo/margin_std": 544.5645751953125, "step": 265 }, { "KL/chosen_KL_mean": -539.0609741210938, "KL/mean": -561.758544921875, "KL/rejected_KL_mean": -584.4561767578125, "KL/std": 398.85784912109375, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.395179748535156, "fcm_dpo/q_t": 0.49756669998168945, "grad_norm": 6.132102966308594, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -0.11285087466239929, "logits/rejected": -0.12594473361968994, "logps/chosen": -606.16064453125, "logps/ref_chosen": -67.09967041015625, "logps/ref_rejected": -67.97122192382812, "logps/rejected": -652.4273681640625, "loss": 1.3796, "margin_dpo/margin_mean": 45.39517593383789, "margin_dpo/margin_std": 510.51422119140625, "step": 266 }, { "KL/chosen_KL_mean": -456.233642578125, "KL/mean": -543.4124755859375, "KL/rejected_KL_mean": -630.59130859375, "KL/std": 447.64996337890625, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 174.35768127441406, "fcm_dpo/q_t": 0.49071285128593445, "grad_norm": 4.379497528076172, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -0.13587993383407593, "logits/rejected": -0.14027976989746094, "logps/chosen": -525.204345703125, "logps/ref_chosen": -68.97075653076172, "logps/ref_rejected": -90.16844940185547, "logps/rejected": -720.759765625, "loss": 1.3531, "margin_dpo/margin_mean": 174.357666015625, "margin_dpo/margin_std": 578.6551513671875, "step": 267 }, { "KL/chosen_KL_mean": -435.64410400390625, "KL/mean": -557.5379638671875, "KL/rejected_KL_mean": -679.4317626953125, "KL/std": 507.63592529296875, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 243.78765869140625, "fcm_dpo/q_t": 0.48697763681411743, "grad_norm": 4.641546249389648, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -0.17142120003700256, "logits/rejected": -0.17281493544578552, "logps/chosen": -491.54443359375, "logps/ref_chosen": -55.90031051635742, "logps/ref_rejected": -101.64763641357422, "logps/rejected": -781.0794067382812, "loss": 1.3387, "margin_dpo/margin_mean": 243.78765869140625, "margin_dpo/margin_std": 590.0145263671875, "step": 268 }, { "KL/chosen_KL_mean": -501.7329406738281, "KL/mean": -626.0015869140625, "KL/rejected_KL_mean": -750.270263671875, "KL/std": 443.25628662109375, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 248.5373077392578, "fcm_dpo/q_t": 0.4867471754550934, "grad_norm": 4.408675193786621, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -0.17563828825950623, "logits/rejected": -0.192486971616745, "logps/chosen": -571.7724609375, "logps/ref_chosen": -70.03955841064453, "logps/ref_rejected": -107.34937286376953, "logps/rejected": -857.61962890625, "loss": 1.3379, "margin_dpo/margin_mean": 248.53729248046875, "margin_dpo/margin_std": 601.0906982421875, "step": 269 }, { "KL/chosen_KL_mean": -464.98211669921875, "KL/mean": -577.4337158203125, "KL/rejected_KL_mean": -689.8851928710938, "KL/std": 445.669189453125, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 224.90313720703125, "fcm_dpo/q_t": 0.48800837993621826, "grad_norm": 4.137222766876221, "learning_rate": 3.794189242333106e-07, "logits/chosen": -0.21441274881362915, "logits/rejected": -0.22988632321357727, "logps/chosen": -534.515625, "logps/ref_chosen": -69.53347778320312, "logps/ref_rejected": -109.92864990234375, "logps/rejected": -799.8138427734375, "loss": 1.3429, "margin_dpo/margin_mean": 224.9031219482422, "margin_dpo/margin_std": 603.002685546875, "step": 270 }, { "KL/chosen_KL_mean": -383.33953857421875, "KL/mean": -498.1918640136719, "KL/rejected_KL_mean": -613.044189453125, "KL/std": 389.174560546875, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 229.7046661376953, "fcm_dpo/q_t": 0.48772376775741577, "grad_norm": 3.387704849243164, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -0.15242159366607666, "logits/rejected": -0.17212322354316711, "logps/chosen": -440.1041259765625, "logps/ref_chosen": -56.76456832885742, "logps/ref_rejected": -92.51383972167969, "logps/rejected": -705.5580444335938, "loss": 1.3403, "margin_dpo/margin_mean": 229.70465087890625, "margin_dpo/margin_std": 480.60894775390625, "step": 271 }, { "KL/chosen_KL_mean": -352.65478515625, "KL/mean": -527.4397583007812, "KL/rejected_KL_mean": -702.2247314453125, "KL/std": 438.55596923828125, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 349.5699768066406, "fcm_dpo/q_t": 0.48131972551345825, "grad_norm": 8.041435241699219, "learning_rate": 3.772161666010912e-07, "logits/chosen": -0.15468257665634155, "logits/rejected": -0.16906066238880157, "logps/chosen": -402.15191650390625, "logps/ref_chosen": -49.497154235839844, "logps/ref_rejected": -105.54279327392578, "logps/rejected": -807.767578125, "loss": 1.3154, "margin_dpo/margin_mean": 349.5699462890625, "margin_dpo/margin_std": 480.5194091796875, "step": 272 }, { "KL/chosen_KL_mean": -454.3601989746094, "KL/mean": -596.029541015625, "KL/rejected_KL_mean": -737.698974609375, "KL/std": 466.94976806640625, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 283.33868408203125, "fcm_dpo/q_t": 0.48487192392349243, "grad_norm": 4.082789421081543, "learning_rate": 3.761097448550755e-07, "logits/chosen": -0.16157250106334686, "logits/rejected": -0.16911356151103973, "logps/chosen": -517.3355712890625, "logps/ref_chosen": -62.97539520263672, "logps/ref_rejected": -92.49858093261719, "logps/rejected": -830.197509765625, "loss": 1.3297, "margin_dpo/margin_mean": 283.33868408203125, "margin_dpo/margin_std": 531.16064453125, "step": 273 }, { "KL/chosen_KL_mean": -471.19451904296875, "KL/mean": -561.68115234375, "KL/rejected_KL_mean": -652.167724609375, "KL/std": 369.1112365722656, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 180.9732208251953, "fcm_dpo/q_t": 0.49033892154693604, "grad_norm": 4.708371639251709, "learning_rate": 3.75e-07, "logits/chosen": -0.10799916088581085, "logits/rejected": -0.10551673173904419, "logps/chosen": -526.8622436523438, "logps/ref_chosen": -55.66770935058594, "logps/ref_rejected": -77.33308410644531, "logps/rejected": -729.5008544921875, "loss": 1.3505, "margin_dpo/margin_mean": 180.97323608398438, "margin_dpo/margin_std": 471.4014587402344, "step": 274 }, { "KL/chosen_KL_mean": -417.6435546875, "KL/mean": -547.9447631835938, "KL/rejected_KL_mean": -678.2459716796875, "KL/std": 425.20245361328125, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 260.6024169921875, "fcm_dpo/q_t": 0.48607248067855835, "grad_norm": 5.847025394439697, "learning_rate": 3.738869612786737e-07, "logits/chosen": -0.15583306550979614, "logits/rejected": -0.16265209019184113, "logps/chosen": -466.2382507324219, "logps/ref_chosen": -48.594703674316406, "logps/ref_rejected": -93.30369567871094, "logps/rejected": -771.5496826171875, "loss": 1.3342, "margin_dpo/margin_mean": 260.6024169921875, "margin_dpo/margin_std": 507.73193359375, "step": 275 }, { "KL/chosen_KL_mean": -472.4503173828125, "KL/mean": -557.5875244140625, "KL/rejected_KL_mean": -642.7247314453125, "KL/std": 441.7587890625, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 170.2743377685547, "fcm_dpo/q_t": 0.4909123182296753, "grad_norm": 5.609163284301758, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -0.14788061380386353, "logits/rejected": -0.1665324866771698, "logps/chosen": -529.0277099609375, "logps/ref_chosen": -56.57740783691406, "logps/ref_rejected": -70.36566925048828, "logps/rejected": -713.09033203125, "loss": 1.3544, "margin_dpo/margin_mean": 170.2743377685547, "margin_dpo/margin_std": 601.065185546875, "step": 276 }, { "KL/chosen_KL_mean": -494.017578125, "KL/mean": -627.9515380859375, "KL/rejected_KL_mean": -761.8856201171875, "KL/std": 439.302001953125, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 267.8680725097656, "fcm_dpo/q_t": 0.4856921434402466, "grad_norm": 4.117493629455566, "learning_rate": 3.71651119641714e-07, "logits/chosen": -0.14363861083984375, "logits/rejected": -0.13916437327861786, "logps/chosen": -550.2891235351562, "logps/ref_chosen": -56.27156066894531, "logps/ref_rejected": -92.88127136230469, "logps/rejected": -854.766845703125, "loss": 1.333, "margin_dpo/margin_mean": 267.8680419921875, "margin_dpo/margin_std": 535.217529296875, "step": 277 }, { "KL/chosen_KL_mean": -475.8216552734375, "KL/mean": -630.0145263671875, "KL/rejected_KL_mean": -784.2073974609375, "KL/std": 520.323486328125, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 308.3857116699219, "fcm_dpo/q_t": 0.4835778772830963, "grad_norm": 4.502967357635498, "learning_rate": 3.705283756425872e-07, "logits/chosen": -0.10104444622993469, "logits/rejected": -0.11467007547616959, "logps/chosen": -528.7635498046875, "logps/ref_chosen": -52.94194030761719, "logps/ref_rejected": -91.25357818603516, "logps/rejected": -875.4609375, "loss": 1.3266, "margin_dpo/margin_mean": 308.3857116699219, "margin_dpo/margin_std": 675.7715454101562, "step": 278 }, { "KL/chosen_KL_mean": -527.8968505859375, "KL/mean": -687.3350830078125, "KL/rejected_KL_mean": -846.7733154296875, "KL/std": 570.1357421875, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 318.8764953613281, "fcm_dpo/q_t": 0.4832102060317993, "grad_norm": 6.868563175201416, "learning_rate": 3.6940245560867e-07, "logits/chosen": -0.11306406557559967, "logits/rejected": -0.1245322972536087, "logps/chosen": -576.5381469726562, "logps/ref_chosen": -48.641319274902344, "logps/ref_rejected": -87.8514404296875, "logps/rejected": -934.624755859375, "loss": 1.3266, "margin_dpo/margin_mean": 318.8764953613281, "margin_dpo/margin_std": 792.3270874023438, "step": 279 }, { "KL/chosen_KL_mean": -510.629150390625, "KL/mean": -667.2784423828125, "KL/rejected_KL_mean": -823.9278564453125, "KL/std": 481.0506591796875, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 313.29864501953125, "fcm_dpo/q_t": 0.48329728841781616, "grad_norm": 4.903005123138428, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -0.13975031673908234, "logits/rejected": -0.1519356071949005, "logps/chosen": -569.42626953125, "logps/ref_chosen": -58.797122955322266, "logps/ref_rejected": -98.61885070800781, "logps/rejected": -922.546630859375, "loss": 1.3249, "margin_dpo/margin_mean": 313.29864501953125, "margin_dpo/margin_std": 634.85693359375, "step": 280 }, { "KL/chosen_KL_mean": -512.4186401367188, "KL/mean": -614.4647216796875, "KL/rejected_KL_mean": -716.5108032226562, "KL/std": 448.61798095703125, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 204.09217834472656, "fcm_dpo/q_t": 0.48906901478767395, "grad_norm": 6.864736557006836, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -0.14065885543823242, "logits/rejected": -0.15409055352210999, "logps/chosen": -567.9071655273438, "logps/ref_chosen": -55.488521575927734, "logps/ref_rejected": -80.88258361816406, "logps/rejected": -797.3933715820312, "loss": 1.3468, "margin_dpo/margin_mean": 204.09219360351562, "margin_dpo/margin_std": 557.4766845703125, "step": 281 }, { "KL/chosen_KL_mean": -550.32373046875, "KL/mean": -651.2105712890625, "KL/rejected_KL_mean": -752.0973510742188, "KL/std": 611.913818359375, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 201.77359008789062, "fcm_dpo/q_t": 0.48951423168182373, "grad_norm": 6.504598140716553, "learning_rate": 3.660059364023408e-07, "logits/chosen": -0.18802456557750702, "logits/rejected": -0.1919810175895691, "logps/chosen": -623.3939208984375, "logps/ref_chosen": -73.07014465332031, "logps/ref_rejected": -95.35098266601562, "logps/rejected": -847.4483642578125, "loss": 1.3497, "margin_dpo/margin_mean": 201.77359008789062, "margin_dpo/margin_std": 712.3954467773438, "step": 282 }, { "KL/chosen_KL_mean": -534.77490234375, "KL/mean": -697.13916015625, "KL/rejected_KL_mean": -859.5034790039062, "KL/std": 511.788818359375, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 324.72857666015625, "fcm_dpo/q_t": 0.4826442003250122, "grad_norm": 6.365213871002197, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -0.13428352773189545, "logits/rejected": -0.14583396911621094, "logps/chosen": -596.67333984375, "logps/ref_chosen": -61.89844512939453, "logps/ref_rejected": -96.98655700683594, "logps/rejected": -956.489990234375, "loss": 1.3214, "margin_dpo/margin_mean": 324.72857666015625, "margin_dpo/margin_std": 536.8545532226562, "step": 283 }, { "KL/chosen_KL_mean": -439.5884704589844, "KL/mean": -578.688232421875, "KL/rejected_KL_mean": -717.7880249023438, "KL/std": 436.02374267578125, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 278.19952392578125, "fcm_dpo/q_t": 0.48516762256622314, "grad_norm": 5.291563987731934, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -0.18925726413726807, "logits/rejected": -0.18296435475349426, "logps/chosen": -498.0240173339844, "logps/ref_chosen": -58.4355354309082, "logps/ref_rejected": -93.46926879882812, "logps/rejected": -811.25732421875, "loss": 1.3307, "margin_dpo/margin_mean": 278.19952392578125, "margin_dpo/margin_std": 519.5226440429688, "step": 284 }, { "KL/chosen_KL_mean": -519.1543579101562, "KL/mean": -606.4476928710938, "KL/rejected_KL_mean": -693.740966796875, "KL/std": 376.2451171875, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 174.586669921875, "fcm_dpo/q_t": 0.4906350374221802, "grad_norm": 4.204360485076904, "learning_rate": 3.625819059005228e-07, "logits/chosen": -0.14472293853759766, "logits/rejected": -0.1445767879486084, "logps/chosen": -585.3865966796875, "logps/ref_chosen": -66.23219299316406, "logps/ref_rejected": -99.1268310546875, "logps/rejected": -792.8677978515625, "loss": 1.3521, "margin_dpo/margin_mean": 174.586669921875, "margin_dpo/margin_std": 497.9844970703125, "step": 285 }, { "KL/chosen_KL_mean": -518.2225341796875, "KL/mean": -618.0490112304688, "KL/rejected_KL_mean": -717.87548828125, "KL/std": 502.3042297363281, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 199.6529541015625, "fcm_dpo/q_t": 0.48932021856307983, "grad_norm": 4.919650077819824, "learning_rate": 3.614345889441346e-07, "logits/chosen": -0.14968830347061157, "logits/rejected": -0.1579841524362564, "logps/chosen": -591.173583984375, "logps/ref_chosen": -72.95100402832031, "logps/ref_rejected": -88.58845520019531, "logps/rejected": -806.4639282226562, "loss": 1.3487, "margin_dpo/margin_mean": 199.6529541015625, "margin_dpo/margin_std": 646.603515625, "step": 286 }, { "KL/chosen_KL_mean": -417.3997497558594, "KL/mean": -486.1492919921875, "KL/rejected_KL_mean": -554.8988647460938, "KL/std": 410.86279296875, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.49908447265625, "fcm_dpo/q_t": 0.49265867471694946, "grad_norm": 3.5626771450042725, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -0.14080211520195007, "logits/rejected": -0.1525568813085556, "logps/chosen": -478.94091796875, "logps/ref_chosen": -61.54115295410156, "logps/ref_rejected": -77.69607543945312, "logps/rejected": -632.594970703125, "loss": 1.36, "margin_dpo/margin_mean": 137.49908447265625, "margin_dpo/margin_std": 493.0873718261719, "step": 287 }, { "KL/chosen_KL_mean": -323.7350769042969, "KL/mean": -437.349365234375, "KL/rejected_KL_mean": -550.963623046875, "KL/std": 331.2504577636719, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 227.22857666015625, "fcm_dpo/q_t": 0.4878368079662323, "grad_norm": 3.0430126190185547, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -0.1575808823108673, "logits/rejected": -0.16179470717906952, "logps/chosen": -380.39630126953125, "logps/ref_chosen": -56.661224365234375, "logps/ref_rejected": -87.33570098876953, "logps/rejected": -638.29931640625, "loss": 1.3402, "margin_dpo/margin_mean": 227.22857666015625, "margin_dpo/margin_std": 409.6346435546875, "step": 288 }, { "KL/chosen_KL_mean": -281.5063171386719, "KL/mean": -421.5150451660156, "KL/rejected_KL_mean": -561.5237426757812, "KL/std": 394.57177734375, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 280.01739501953125, "fcm_dpo/q_t": 0.48507311940193176, "grad_norm": 6.204428195953369, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -0.11954732984304428, "logits/rejected": -0.1284411996603012, "logps/chosen": -326.7367248535156, "logps/ref_chosen": -45.23039245605469, "logps/ref_rejected": -87.64266967773438, "logps/rejected": -649.1663818359375, "loss": 1.3298, "margin_dpo/margin_mean": 280.01739501953125, "margin_dpo/margin_std": 468.9491882324219, "step": 289 }, { "KL/chosen_KL_mean": -315.4320983886719, "KL/mean": -489.9077453613281, "KL/rejected_KL_mean": -664.3833618164062, "KL/std": 450.1668701171875, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 348.9513244628906, "fcm_dpo/q_t": 0.48141106963157654, "grad_norm": 7.722722053527832, "learning_rate": 3.568162605525952e-07, "logits/chosen": -0.12268570065498352, "logits/rejected": -0.14039406180381775, "logps/chosen": -370.9035949707031, "logps/ref_chosen": -55.47149658203125, "logps/ref_rejected": -116.70857238769531, "logps/rejected": -781.0919189453125, "loss": 1.3162, "margin_dpo/margin_mean": 348.9512939453125, "margin_dpo/margin_std": 530.4649658203125, "step": 290 }, { "KL/chosen_KL_mean": -322.58355712890625, "KL/mean": -421.381103515625, "KL/rejected_KL_mean": -520.1787109375, "KL/std": 350.70855712890625, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 197.59518432617188, "fcm_dpo/q_t": 0.4894236922264099, "grad_norm": 4.782266139984131, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -0.14178414642810822, "logits/rejected": -0.1538257598876953, "logps/chosen": -385.8439025878906, "logps/ref_chosen": -63.26036834716797, "logps/ref_rejected": -89.29708862304688, "logps/rejected": -609.475830078125, "loss": 1.3469, "margin_dpo/margin_mean": 197.5951690673828, "margin_dpo/margin_std": 465.94696044921875, "step": 291 }, { "KL/chosen_KL_mean": -300.87261962890625, "KL/mean": -426.175537109375, "KL/rejected_KL_mean": -551.4783935546875, "KL/std": 354.70635986328125, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 250.60580444335938, "fcm_dpo/q_t": 0.48658984899520874, "grad_norm": 4.581497669219971, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -0.1559632122516632, "logits/rejected": -0.16916170716285706, "logps/chosen": -354.7911376953125, "logps/ref_chosen": -53.91852951049805, "logps/ref_rejected": -89.96138000488281, "logps/rejected": -641.4398193359375, "loss": 1.3355, "margin_dpo/margin_mean": 250.60580444335938, "margin_dpo/margin_std": 433.79608154296875, "step": 292 }, { "KL/chosen_KL_mean": -365.57171630859375, "KL/mean": -445.22955322265625, "KL/rejected_KL_mean": -524.887451171875, "KL/std": 410.1599426269531, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 159.31570434570312, "fcm_dpo/q_t": 0.4914965033531189, "grad_norm": 3.991530656814575, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -0.15570124983787537, "logits/rejected": -0.1721823364496231, "logps/chosen": -425.94775390625, "logps/ref_chosen": -60.376033782958984, "logps/ref_rejected": -77.85244750976562, "logps/rejected": -602.7398681640625, "loss": 1.3547, "margin_dpo/margin_mean": 159.315673828125, "margin_dpo/margin_std": 447.989501953125, "step": 293 }, { "KL/chosen_KL_mean": -289.04150390625, "KL/mean": -404.94580078125, "KL/rejected_KL_mean": -520.8502197265625, "KL/std": 356.51409912109375, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 231.8087158203125, "fcm_dpo/q_t": 0.4876137375831604, "grad_norm": 5.801051616668701, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -0.11081454157829285, "logits/rejected": -0.12332391738891602, "logps/chosen": -337.1290283203125, "logps/ref_chosen": -48.0875358581543, "logps/ref_rejected": -81.89698791503906, "logps/rejected": -602.7471923828125, "loss": 1.3401, "margin_dpo/margin_mean": 231.80870056152344, "margin_dpo/margin_std": 499.6016540527344, "step": 294 }, { "KL/chosen_KL_mean": -377.87554931640625, "KL/mean": -518.6566162109375, "KL/rejected_KL_mean": -659.437744140625, "KL/std": 447.6157531738281, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 281.56219482421875, "fcm_dpo/q_t": 0.48502007126808167, "grad_norm": 3.4757144451141357, "learning_rate": 3.509802512179737e-07, "logits/chosen": -0.13640262186527252, "logits/rejected": -0.13862435519695282, "logps/chosen": -427.80023193359375, "logps/ref_chosen": -49.92467498779297, "logps/ref_rejected": -87.45632934570312, "logps/rejected": -746.89404296875, "loss": 1.3308, "margin_dpo/margin_mean": 281.56219482421875, "margin_dpo/margin_std": 583.3878173828125, "step": 295 }, { "KL/chosen_KL_mean": -489.06207275390625, "KL/mean": -576.8760986328125, "KL/rejected_KL_mean": -664.6901245117188, "KL/std": 395.9642333984375, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 175.62806701660156, "fcm_dpo/q_t": 0.49063578248023987, "grad_norm": 3.92090106010437, "learning_rate": 3.498049431928577e-07, "logits/chosen": -0.12152425199747086, "logits/rejected": -0.12600529193878174, "logps/chosen": -554.5533447265625, "logps/ref_chosen": -65.49124145507812, "logps/ref_rejected": -93.08908081054688, "logps/rejected": -757.7791748046875, "loss": 1.3523, "margin_dpo/margin_mean": 175.62808227539062, "margin_dpo/margin_std": 526.7503662109375, "step": 296 }, { "KL/chosen_KL_mean": -468.5126037597656, "KL/mean": -587.1865234375, "KL/rejected_KL_mean": -705.8604736328125, "KL/std": 483.5227966308594, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 237.34788513183594, "fcm_dpo/q_t": 0.48738476634025574, "grad_norm": 5.903518199920654, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.19832845032215118, "logits/rejected": -0.213718980550766, "logps/chosen": -524.9895629882812, "logps/ref_chosen": -56.476951599121094, "logps/ref_rejected": -95.1385498046875, "logps/rejected": -800.9990234375, "loss": 1.3395, "margin_dpo/margin_mean": 237.34786987304688, "margin_dpo/margin_std": 542.2998046875, "step": 297 }, { "KL/chosen_KL_mean": -583.92919921875, "KL/mean": -760.1380004882812, "KL/rejected_KL_mean": -936.3467407226562, "KL/std": 648.7681884765625, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 352.41748046875, "fcm_dpo/q_t": 0.4814052879810333, "grad_norm": 5.72074556350708, "learning_rate": 3.474464683231698e-07, "logits/chosen": -0.15087062120437622, "logits/rejected": -0.1606747806072235, "logps/chosen": -651.25439453125, "logps/ref_chosen": -67.32516479492188, "logps/ref_rejected": -116.66217041015625, "logps/rejected": -1053.0089111328125, "loss": 1.3195, "margin_dpo/margin_mean": 352.41748046875, "margin_dpo/margin_std": 788.0234375, "step": 298 }, { "KL/chosen_KL_mean": -473.8212890625, "KL/mean": -579.8349609375, "KL/rejected_KL_mean": -685.8486328125, "KL/std": 470.7117919921875, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 212.02735900878906, "fcm_dpo/q_t": 0.4887089133262634, "grad_norm": 7.583713054656982, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.23694965243339539, "logits/rejected": -0.2537599205970764, "logps/chosen": -522.7833862304688, "logps/ref_chosen": -48.96209716796875, "logps/ref_rejected": -84.32823944091797, "logps/rejected": -770.1768798828125, "loss": 1.3463, "margin_dpo/margin_mean": 212.02735900878906, "margin_dpo/margin_std": 633.6422119140625, "step": 299 }, { "KL/chosen_KL_mean": -611.6695556640625, "KL/mean": -765.8885498046875, "KL/rejected_KL_mean": -920.107666015625, "KL/std": 540.96484375, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 308.4380798339844, "fcm_dpo/q_t": 0.48355600237846375, "grad_norm": 7.862407207489014, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -0.19781388342380524, "logits/rejected": -0.21643571555614471, "logps/chosen": -670.7432861328125, "logps/ref_chosen": -59.07371139526367, "logps/ref_rejected": -95.9664535522461, "logps/rejected": -1016.0740966796875, "loss": 1.327, "margin_dpo/margin_mean": 308.4381103515625, "margin_dpo/margin_std": 709.8028564453125, "step": 300 }, { "KL/chosen_KL_mean": -573.643310546875, "KL/mean": -723.9158935546875, "KL/rejected_KL_mean": -874.1885375976562, "KL/std": 545.066162109375, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 300.5451965332031, "fcm_dpo/q_t": 0.48413360118865967, "grad_norm": 6.303064346313477, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -0.19899940490722656, "logits/rejected": -0.21283400058746338, "logps/chosen": -630.8927001953125, "logps/ref_chosen": -57.249366760253906, "logps/ref_rejected": -92.35354614257812, "logps/rejected": -966.5421142578125, "loss": 1.3293, "margin_dpo/margin_mean": 300.5451965332031, "margin_dpo/margin_std": 747.690185546875, "step": 301 }, { "KL/chosen_KL_mean": -525.3902587890625, "KL/mean": -676.2835693359375, "KL/rejected_KL_mean": -827.1768798828125, "KL/std": 550.9168701171875, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 301.78662109375, "fcm_dpo/q_t": 0.4839833378791809, "grad_norm": 6.866415500640869, "learning_rate": 3.426989547989902e-07, "logits/chosen": -0.2429056167602539, "logits/rejected": -0.23922690749168396, "logps/chosen": -576.5882568359375, "logps/ref_chosen": -51.197994232177734, "logps/ref_rejected": -97.22636413574219, "logps/rejected": -924.4032592773438, "loss": 1.3291, "margin_dpo/margin_mean": 301.78662109375, "margin_dpo/margin_std": 743.7776489257812, "step": 302 }, { "KL/chosen_KL_mean": -630.103515625, "KL/mean": -715.7218017578125, "KL/rejected_KL_mean": -801.3401489257812, "KL/std": 567.647705078125, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 171.23655700683594, "fcm_dpo/q_t": 0.4908214807510376, "grad_norm": 6.0209126472473145, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -0.23265428841114044, "logits/rejected": -0.24132990837097168, "logps/chosen": -696.8175048828125, "logps/ref_chosen": -66.71394348144531, "logps/ref_rejected": -86.94542694091797, "logps/rejected": -888.2855834960938, "loss": 1.3558, "margin_dpo/margin_mean": 171.23655700683594, "margin_dpo/margin_std": 712.87548828125, "step": 303 }, { "KL/chosen_KL_mean": -616.35107421875, "KL/mean": -709.6608276367188, "KL/rejected_KL_mean": -802.9705810546875, "KL/std": 510.59539794921875, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 186.61947631835938, "fcm_dpo/q_t": 0.49002233147621155, "grad_norm": 5.741556167602539, "learning_rate": 3.403104165467883e-07, "logits/chosen": -0.2312694489955902, "logits/rejected": -0.2368827909231186, "logps/chosen": -688.3017578125, "logps/ref_chosen": -71.95069885253906, "logps/ref_rejected": -90.47203063964844, "logps/rejected": -893.442626953125, "loss": 1.3511, "margin_dpo/margin_mean": 186.61947631835938, "margin_dpo/margin_std": 618.4774169921875, "step": 304 }, { "KL/chosen_KL_mean": -570.4090576171875, "KL/mean": -677.1749267578125, "KL/rejected_KL_mean": -783.9407958984375, "KL/std": 503.7247009277344, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 213.53167724609375, "fcm_dpo/q_t": 0.4886142611503601, "grad_norm": 5.134280681610107, "learning_rate": 3.391125620245535e-07, "logits/chosen": -0.27188488841056824, "logits/rejected": -0.2840760350227356, "logps/chosen": -637.204345703125, "logps/ref_chosen": -66.79523468017578, "logps/ref_rejected": -92.75459289550781, "logps/rejected": -876.6953735351562, "loss": 1.3456, "margin_dpo/margin_mean": 213.53167724609375, "margin_dpo/margin_std": 633.0828857421875, "step": 305 }, { "KL/chosen_KL_mean": -537.7818603515625, "KL/mean": -625.0843505859375, "KL/rejected_KL_mean": -712.386962890625, "KL/std": 452.66357421875, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 174.60516357421875, "fcm_dpo/q_t": 0.4906894564628601, "grad_norm": 5.521829128265381, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -0.19213925302028656, "logits/rejected": -0.19932372868061066, "logps/chosen": -607.4656982421875, "logps/ref_chosen": -69.68389892578125, "logps/ref_rejected": -85.15919494628906, "logps/rejected": -797.546142578125, "loss": 1.3524, "margin_dpo/margin_mean": 174.6051483154297, "margin_dpo/margin_std": 521.4979858398438, "step": 306 }, { "KL/chosen_KL_mean": -450.71343994140625, "KL/mean": -559.7984619140625, "KL/rejected_KL_mean": -668.8834228515625, "KL/std": 426.73651123046875, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 218.17001342773438, "fcm_dpo/q_t": 0.4883463978767395, "grad_norm": 4.214844226837158, "learning_rate": 3.367098400098881e-07, "logits/chosen": -0.22033822536468506, "logits/rejected": -0.224267840385437, "logps/chosen": -520.8788452148438, "logps/ref_chosen": -70.16542053222656, "logps/ref_rejected": -86.97230529785156, "logps/rejected": -755.855712890625, "loss": 1.3434, "margin_dpo/margin_mean": 218.17001342773438, "margin_dpo/margin_std": 541.6478271484375, "step": 307 }, { "KL/chosen_KL_mean": -401.5634765625, "KL/mean": -537.181396484375, "KL/rejected_KL_mean": -672.79931640625, "KL/std": 440.7513122558594, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 271.23590087890625, "fcm_dpo/q_t": 0.48551321029663086, "grad_norm": 4.456334590911865, "learning_rate": 3.355050358314172e-07, "logits/chosen": -0.19973334670066833, "logits/rejected": -0.2119927853345871, "logps/chosen": -456.8084716796875, "logps/ref_chosen": -55.2449951171875, "logps/ref_rejected": -79.37226104736328, "logps/rejected": -752.171630859375, "loss": 1.3319, "margin_dpo/margin_mean": 271.2358703613281, "margin_dpo/margin_std": 503.54925537109375, "step": 308 }, { "KL/chosen_KL_mean": -369.8251953125, "KL/mean": -482.3707275390625, "KL/rejected_KL_mean": -594.9161987304688, "KL/std": 423.8313903808594, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 225.0910186767578, "fcm_dpo/q_t": 0.48792463541030884, "grad_norm": 4.421578407287598, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -0.16784876585006714, "logits/rejected": -0.18317045271396637, "logps/chosen": -418.7843017578125, "logps/ref_chosen": -48.959083557128906, "logps/ref_rejected": -82.34072875976562, "logps/rejected": -677.2569580078125, "loss": 1.3417, "margin_dpo/margin_mean": 225.0910186767578, "margin_dpo/margin_std": 493.40032958984375, "step": 309 }, { "KL/chosen_KL_mean": -440.13433837890625, "KL/mean": -548.5415649414062, "KL/rejected_KL_mean": -656.9487915039062, "KL/std": 404.796875, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 216.81442260742188, "fcm_dpo/q_t": 0.48840105533599854, "grad_norm": 3.7958574295043945, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -0.15935003757476807, "logits/rejected": -0.17317405343055725, "logps/chosen": -502.8761291503906, "logps/ref_chosen": -62.74177932739258, "logps/ref_rejected": -79.9302978515625, "logps/rejected": -736.8790893554688, "loss": 1.3427, "margin_dpo/margin_mean": 216.81442260742188, "margin_dpo/margin_std": 443.70294189453125, "step": 310 }, { "KL/chosen_KL_mean": -415.8055419921875, "KL/mean": -523.12060546875, "KL/rejected_KL_mean": -630.4356689453125, "KL/std": 363.2186279296875, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 214.63011169433594, "fcm_dpo/q_t": 0.4885249733924866, "grad_norm": 4.3125104904174805, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -0.21918794512748718, "logits/rejected": -0.22478339076042175, "logps/chosen": -468.83349609375, "logps/ref_chosen": -53.02798080444336, "logps/ref_rejected": -77.43820190429688, "logps/rejected": -707.8738403320312, "loss": 1.3428, "margin_dpo/margin_mean": 214.63011169433594, "margin_dpo/margin_std": 417.1317443847656, "step": 311 }, { "KL/chosen_KL_mean": -427.406005859375, "KL/mean": -540.7687377929688, "KL/rejected_KL_mean": -654.1314697265625, "KL/std": 440.4998779296875, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 226.72544860839844, "fcm_dpo/q_t": 0.4879283308982849, "grad_norm": 4.058272361755371, "learning_rate": 3.306636061080487e-07, "logits/chosen": -0.17278644442558289, "logits/rejected": -0.18266448378562927, "logps/chosen": -476.7982177734375, "logps/ref_chosen": -49.39221954345703, "logps/ref_rejected": -75.79280853271484, "logps/rejected": -729.92431640625, "loss": 1.3417, "margin_dpo/margin_mean": 226.7254638671875, "margin_dpo/margin_std": 535.8837280273438, "step": 312 }, { "KL/chosen_KL_mean": -382.3973388671875, "KL/mean": -525.8050537109375, "KL/rejected_KL_mean": -669.2127685546875, "KL/std": 460.6397399902344, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 286.8153991699219, "fcm_dpo/q_t": 0.48471421003341675, "grad_norm": 4.11210298538208, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -0.18957430124282837, "logits/rejected": -0.2057763636112213, "logps/chosen": -432.55010986328125, "logps/ref_chosen": -50.152740478515625, "logps/ref_rejected": -86.40620422363281, "logps/rejected": -755.6190185546875, "loss": 1.3291, "margin_dpo/margin_mean": 286.8153991699219, "margin_dpo/margin_std": 513.8978271484375, "step": 313 }, { "KL/chosen_KL_mean": -473.88775634765625, "KL/mean": -590.21435546875, "KL/rejected_KL_mean": -706.541015625, "KL/std": 460.99267578125, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 232.65328979492188, "fcm_dpo/q_t": 0.4876299202442169, "grad_norm": 3.9764833450317383, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -0.23278236389160156, "logits/rejected": -0.2444659173488617, "logps/chosen": -531.1253051757812, "logps/ref_chosen": -57.237579345703125, "logps/ref_rejected": -97.5965347290039, "logps/rejected": -804.1375732421875, "loss": 1.3414, "margin_dpo/margin_mean": 232.65330505371094, "margin_dpo/margin_std": 601.2679443359375, "step": 314 }, { "KL/chosen_KL_mean": -396.1183166503906, "KL/mean": -510.1334228515625, "KL/rejected_KL_mean": -624.1485595703125, "KL/std": 454.22100830078125, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 228.03021240234375, "fcm_dpo/q_t": 0.4878466725349426, "grad_norm": 4.002105236053467, "learning_rate": 3.270101039870797e-07, "logits/chosen": -0.1942490041255951, "logits/rejected": -0.20377588272094727, "logps/chosen": -445.18792724609375, "logps/ref_chosen": -49.06958770751953, "logps/ref_rejected": -85.68087768554688, "logps/rejected": -709.8294067382812, "loss": 1.3415, "margin_dpo/margin_mean": 228.03021240234375, "margin_dpo/margin_std": 553.0503540039062, "step": 315 }, { "KL/chosen_KL_mean": -419.86102294921875, "KL/mean": -615.5865478515625, "KL/rejected_KL_mean": -811.3121948242188, "KL/std": 490.57891845703125, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 391.451171875, "fcm_dpo/q_t": 0.4791603088378906, "grad_norm": 5.238584995269775, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -0.2003282904624939, "logits/rejected": -0.20365044474601746, "logps/chosen": -474.12176513671875, "logps/ref_chosen": -54.26074981689453, "logps/ref_rejected": -101.2814712524414, "logps/rejected": -912.5936279296875, "loss": 1.3085, "margin_dpo/margin_mean": 391.451171875, "margin_dpo/margin_std": 604.1680908203125, "step": 316 }, { "KL/chosen_KL_mean": -472.4468994140625, "KL/mean": -679.812744140625, "KL/rejected_KL_mean": -887.178466796875, "KL/std": 529.8192138671875, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 414.73162841796875, "fcm_dpo/q_t": 0.47804775834083557, "grad_norm": 4.577226638793945, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -0.2022346556186676, "logits/rejected": -0.2135677933692932, "logps/chosen": -528.5411376953125, "logps/ref_chosen": -56.094207763671875, "logps/ref_rejected": -100.69905090332031, "logps/rejected": -987.8775634765625, "loss": 1.3053, "margin_dpo/margin_mean": 414.73162841796875, "margin_dpo/margin_std": 667.3003540039062, "step": 317 }, { "KL/chosen_KL_mean": -563.6049194335938, "KL/mean": -657.4984130859375, "KL/rejected_KL_mean": -751.3919067382812, "KL/std": 476.6451721191406, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 187.7869873046875, "fcm_dpo/q_t": 0.48997563123703003, "grad_norm": 5.925684452056885, "learning_rate": 3.233383385962115e-07, "logits/chosen": -0.2424195110797882, "logits/rejected": -0.2464376538991928, "logps/chosen": -628.2506103515625, "logps/ref_chosen": -64.64569854736328, "logps/ref_rejected": -82.76425170898438, "logps/rejected": -834.1561279296875, "loss": 1.3496, "margin_dpo/margin_mean": 187.78700256347656, "margin_dpo/margin_std": 521.0235595703125, "step": 318 }, { "KL/chosen_KL_mean": -436.5062255859375, "KL/mean": -647.1476440429688, "KL/rejected_KL_mean": -857.7891235351562, "KL/std": 617.1673583984375, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 421.28289794921875, "fcm_dpo/q_t": 0.47768670320510864, "grad_norm": 7.061251163482666, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -0.19704844057559967, "logits/rejected": -0.20887039601802826, "logps/chosen": -485.88995361328125, "logps/ref_chosen": -49.383758544921875, "logps/ref_rejected": -113.90650939941406, "logps/rejected": -971.6956176757812, "loss": 1.3037, "margin_dpo/margin_mean": 421.28289794921875, "margin_dpo/margin_std": 688.0316162109375, "step": 319 }, { "KL/chosen_KL_mean": -505.7894592285156, "KL/mean": -670.9154052734375, "KL/rejected_KL_mean": -836.041259765625, "KL/std": 497.88201904296875, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 330.25177001953125, "fcm_dpo/q_t": 0.4823932647705078, "grad_norm": 4.2582244873046875, "learning_rate": 3.208807785813777e-07, "logits/chosen": -0.15992069244384766, "logits/rejected": -0.17337793111801147, "logps/chosen": -565.2943725585938, "logps/ref_chosen": -59.50489044189453, "logps/ref_rejected": -97.66717529296875, "logps/rejected": -933.70849609375, "loss": 1.3212, "margin_dpo/margin_mean": 330.25177001953125, "margin_dpo/margin_std": 617.7380981445312, "step": 320 }, { "KL/chosen_KL_mean": -604.0084228515625, "KL/mean": -743.7211303710938, "KL/rejected_KL_mean": -883.433837890625, "KL/std": 562.9217529296875, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 279.4254455566406, "fcm_dpo/q_t": 0.4851454198360443, "grad_norm": 5.155450344085693, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -0.14682038128376007, "logits/rejected": -0.15771764516830444, "logps/chosen": -665.5570678710938, "logps/ref_chosen": -61.548683166503906, "logps/ref_rejected": -91.64103698730469, "logps/rejected": -975.074951171875, "loss": 1.3322, "margin_dpo/margin_mean": 279.42547607421875, "margin_dpo/margin_std": 654.2274169921875, "step": 321 }, { "KL/chosen_KL_mean": -511.4791259765625, "KL/mean": -655.0626831054688, "KL/rejected_KL_mean": -798.646240234375, "KL/std": 467.83734130859375, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 287.16717529296875, "fcm_dpo/q_t": 0.4846848249435425, "grad_norm": 5.708774089813232, "learning_rate": 3.184157475180207e-07, "logits/chosen": -0.17217618227005005, "logits/rejected": -0.18240973353385925, "logps/chosen": -568.7691650390625, "logps/ref_chosen": -57.29003143310547, "logps/ref_rejected": -95.74992370605469, "logps/rejected": -894.396240234375, "loss": 1.3294, "margin_dpo/margin_mean": 287.16717529296875, "margin_dpo/margin_std": 554.1672973632812, "step": 322 }, { "KL/chosen_KL_mean": -529.456787109375, "KL/mean": -639.5374755859375, "KL/rejected_KL_mean": -749.6182861328125, "KL/std": 478.3594970703125, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 220.1614990234375, "fcm_dpo/q_t": 0.4882357716560364, "grad_norm": 5.594461441040039, "learning_rate": 3.171805115074251e-07, "logits/chosen": -0.1582409143447876, "logits/rejected": -0.17803357541561127, "logps/chosen": -580.690673828125, "logps/ref_chosen": -51.23395919799805, "logps/ref_rejected": -75.06192016601562, "logps/rejected": -824.68017578125, "loss": 1.3433, "margin_dpo/margin_mean": 220.1614990234375, "margin_dpo/margin_std": 536.7333984375, "step": 323 }, { "KL/chosen_KL_mean": -634.5806884765625, "KL/mean": -732.8265380859375, "KL/rejected_KL_mean": -831.072509765625, "KL/std": 546.5975952148438, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 196.49183654785156, "fcm_dpo/q_t": 0.48958471417427063, "grad_norm": 6.335326194763184, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -0.18952664732933044, "logits/rejected": -0.20132169127464294, "logps/chosen": -699.7158203125, "logps/ref_chosen": -65.13516998291016, "logps/ref_rejected": -86.47750854492188, "logps/rejected": -917.5499877929688, "loss": 1.3508, "margin_dpo/margin_mean": 196.49185180664062, "margin_dpo/margin_std": 704.073974609375, "step": 324 }, { "KL/chosen_KL_mean": -488.4103698730469, "KL/mean": -557.0770263671875, "KL/rejected_KL_mean": -625.7437133789062, "KL/std": 416.86346435546875, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.33335876464844, "fcm_dpo/q_t": 0.4926797151565552, "grad_norm": 5.764030456542969, "learning_rate": 3.147047612756302e-07, "logits/chosen": -0.23148123919963837, "logits/rejected": -0.2455836832523346, "logps/chosen": -544.6259765625, "logps/ref_chosen": -56.215599060058594, "logps/ref_rejected": -70.08592987060547, "logps/rejected": -695.8296508789062, "loss": 1.3603, "margin_dpo/margin_mean": 137.3333740234375, "margin_dpo/margin_std": 519.9088745117188, "step": 325 }, { "KL/chosen_KL_mean": -546.6886596679688, "KL/mean": -624.8822021484375, "KL/rejected_KL_mean": -703.0758666992188, "KL/std": 415.1629333496094, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.38719177246094, "fcm_dpo/q_t": 0.4916384220123291, "grad_norm": 5.459661960601807, "learning_rate": 3.134643122927519e-07, "logits/chosen": -0.23426343500614166, "logits/rejected": -0.24521896243095398, "logps/chosen": -619.4136352539062, "logps/ref_chosen": -72.72496032714844, "logps/ref_rejected": -79.8467788696289, "logps/rejected": -782.922607421875, "loss": 1.3553, "margin_dpo/margin_mean": 156.38717651367188, "margin_dpo/margin_std": 438.69140625, "step": 326 }, { "KL/chosen_KL_mean": -510.6233825683594, "KL/mean": -662.1951904296875, "KL/rejected_KL_mean": -813.7669677734375, "KL/std": 460.990478515625, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 303.1435852050781, "fcm_dpo/q_t": 0.48379653692245483, "grad_norm": 6.2359209060668945, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -0.19739487767219543, "logits/rejected": -0.2052072286605835, "logps/chosen": -579.7578125, "logps/ref_chosen": -69.13441467285156, "logps/ref_rejected": -111.93377685546875, "logps/rejected": -925.7008056640625, "loss": 1.3262, "margin_dpo/margin_mean": 303.14361572265625, "margin_dpo/margin_std": 575.9744873046875, "step": 327 }, { "KL/chosen_KL_mean": -442.1131591796875, "KL/mean": -588.37548828125, "KL/rejected_KL_mean": -734.6378173828125, "KL/std": 493.975830078125, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 292.524658203125, "fcm_dpo/q_t": 0.4843920171260834, "grad_norm": 5.148623943328857, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -0.2125837206840515, "logits/rejected": -0.2268255650997162, "logps/chosen": -501.80035400390625, "logps/ref_chosen": -59.68719482421875, "logps/ref_rejected": -90.85499572753906, "logps/rejected": -825.4927978515625, "loss": 1.3284, "margin_dpo/margin_mean": 292.524658203125, "margin_dpo/margin_std": 573.4638671875, "step": 328 }, { "KL/chosen_KL_mean": -491.8397216796875, "KL/mean": -647.6664428710938, "KL/rejected_KL_mean": -803.4930419921875, "KL/std": 506.99774169921875, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 311.6534118652344, "fcm_dpo/q_t": 0.48337361216545105, "grad_norm": 4.15192985534668, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -0.1763824224472046, "logits/rejected": -0.19332997500896454, "logps/chosen": -557.0859375, "logps/ref_chosen": -65.2461929321289, "logps/ref_rejected": -100.69770812988281, "logps/rejected": -904.1907958984375, "loss": 1.325, "margin_dpo/margin_mean": 311.65338134765625, "margin_dpo/margin_std": 609.5528564453125, "step": 329 }, { "KL/chosen_KL_mean": -342.6683349609375, "KL/mean": -501.531982421875, "KL/rejected_KL_mean": -660.3956298828125, "KL/std": 415.15380859375, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 317.727294921875, "fcm_dpo/q_t": 0.483031302690506, "grad_norm": 7.011267185211182, "learning_rate": 3.084861204504122e-07, "logits/chosen": -0.14472857117652893, "logits/rejected": -0.1459788680076599, "logps/chosen": -389.66668701171875, "logps/ref_chosen": -46.998348236083984, "logps/ref_rejected": -86.87684631347656, "logps/rejected": -747.2724609375, "loss": 1.3222, "margin_dpo/margin_mean": 317.7272644042969, "margin_dpo/margin_std": 499.0316162109375, "step": 330 }, { "KL/chosen_KL_mean": -391.0220642089844, "KL/mean": -550.6251220703125, "KL/rejected_KL_mean": -710.228271484375, "KL/std": 350.403076171875, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 319.2061462402344, "fcm_dpo/q_t": 0.4829198718070984, "grad_norm": 5.114991664886475, "learning_rate": 3.072376374875335e-07, "logits/chosen": -0.16779181361198425, "logits/rejected": -0.1757754683494568, "logps/chosen": -441.54632568359375, "logps/ref_chosen": -50.52424621582031, "logps/ref_rejected": -89.01544189453125, "logps/rejected": -799.2437133789062, "loss": 1.3205, "margin_dpo/margin_mean": 319.2061462402344, "margin_dpo/margin_std": 355.31463623046875, "step": 331 }, { "KL/chosen_KL_mean": -429.01025390625, "KL/mean": -513.4847412109375, "KL/rejected_KL_mean": -597.9593505859375, "KL/std": 391.617919921875, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 168.9490966796875, "fcm_dpo/q_t": 0.4909921884536743, "grad_norm": 4.994297981262207, "learning_rate": 3.059876462596758e-07, "logits/chosen": -0.14846037328243256, "logits/rejected": -0.15179188549518585, "logps/chosen": -478.1905212402344, "logps/ref_chosen": -49.18028259277344, "logps/ref_rejected": -76.48515319824219, "logps/rejected": -674.4444580078125, "loss": 1.3528, "margin_dpo/margin_mean": 168.9490966796875, "margin_dpo/margin_std": 455.59197998046875, "step": 332 }, { "KL/chosen_KL_mean": -516.8207397460938, "KL/mean": -647.05615234375, "KL/rejected_KL_mean": -777.2916259765625, "KL/std": 472.027099609375, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 260.4708251953125, "fcm_dpo/q_t": 0.48608171939849854, "grad_norm": 5.020219326019287, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -0.22815987467765808, "logits/rejected": -0.24586617946624756, "logps/chosen": -580.5764770507812, "logps/ref_chosen": -63.75574493408203, "logps/ref_rejected": -95.04411315917969, "logps/rejected": -872.335693359375, "loss": 1.3361, "margin_dpo/margin_mean": 260.4708251953125, "margin_dpo/margin_std": 628.2245483398438, "step": 333 }, { "KL/chosen_KL_mean": -535.3387451171875, "KL/mean": -647.19287109375, "KL/rejected_KL_mean": -759.047119140625, "KL/std": 525.2220458984375, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 223.70834350585938, "fcm_dpo/q_t": 0.48809105157852173, "grad_norm": 5.767882823944092, "learning_rate": 3.034832708016243e-07, "logits/chosen": -0.2623814344406128, "logits/rejected": -0.27332639694213867, "logps/chosen": -602.3184814453125, "logps/ref_chosen": -66.97975158691406, "logps/ref_rejected": -95.31692504882812, "logps/rejected": -854.364013671875, "loss": 1.3445, "margin_dpo/margin_mean": 223.70834350585938, "margin_dpo/margin_std": 687.914794921875, "step": 334 }, { "KL/chosen_KL_mean": -553.5545654296875, "KL/mean": -645.1278076171875, "KL/rejected_KL_mean": -736.701171875, "KL/std": 531.672119140625, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 183.14663696289062, "fcm_dpo/q_t": 0.4902423322200775, "grad_norm": 5.44934606552124, "learning_rate": 3.022289525640531e-07, "logits/chosen": -0.2666965126991272, "logits/rejected": -0.27081555128097534, "logps/chosen": -616.0970458984375, "logps/ref_chosen": -62.54248046875, "logps/ref_rejected": -87.61770629882812, "logps/rejected": -824.31884765625, "loss": 1.351, "margin_dpo/margin_mean": 183.14662170410156, "margin_dpo/margin_std": 550.9996948242188, "step": 335 }, { "KL/chosen_KL_mean": -554.318115234375, "KL/mean": -750.877685546875, "KL/rejected_KL_mean": -947.4371948242188, "KL/std": 650.4267578125, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 393.1190490722656, "fcm_dpo/q_t": 0.47911518812179565, "grad_norm": 7.3909687995910645, "learning_rate": 3.009732580450086e-07, "logits/chosen": -0.21140223741531372, "logits/rejected": -0.21927091479301453, "logps/chosen": -608.8492431640625, "logps/ref_chosen": -54.53115463256836, "logps/ref_rejected": -104.40424346923828, "logps/rejected": -1051.8414306640625, "loss": 1.3116, "margin_dpo/margin_mean": 393.11907958984375, "margin_dpo/margin_std": 823.8309326171875, "step": 336 }, { "KL/chosen_KL_mean": -571.74658203125, "KL/mean": -740.125732421875, "KL/rejected_KL_mean": -908.5048828125, "KL/std": 476.75103759765625, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 336.75836181640625, "fcm_dpo/q_t": 0.4820534288883209, "grad_norm": 6.862320899963379, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -0.25504040718078613, "logits/rejected": -0.2622433006763458, "logps/chosen": -636.875244140625, "logps/ref_chosen": -65.12869262695312, "logps/ref_rejected": -101.72701263427734, "logps/rejected": -1010.23193359375, "loss": 1.3197, "margin_dpo/margin_mean": 336.75836181640625, "margin_dpo/margin_std": 612.5994873046875, "step": 337 }, { "KL/chosen_KL_mean": -500.8605651855469, "KL/mean": -660.3416748046875, "KL/rejected_KL_mean": -819.82275390625, "KL/std": 447.0543212890625, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 318.9621887207031, "fcm_dpo/q_t": 0.48294728994369507, "grad_norm": 5.339162349700928, "learning_rate": 2.984578725527675e-07, "logits/chosen": -0.25964879989624023, "logits/rejected": -0.27327626943588257, "logps/chosen": -559.2832641601562, "logps/ref_chosen": -58.422706604003906, "logps/ref_rejected": -89.06854248046875, "logps/rejected": -908.8912963867188, "loss": 1.3215, "margin_dpo/margin_mean": 318.9621887207031, "margin_dpo/margin_std": 463.43841552734375, "step": 338 }, { "KL/chosen_KL_mean": -594.2242431640625, "KL/mean": -748.5675048828125, "KL/rejected_KL_mean": -902.9107666015625, "KL/std": 501.53179931640625, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 308.68658447265625, "fcm_dpo/q_t": 0.48354700207710266, "grad_norm": 6.215461730957031, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -0.2669673264026642, "logits/rejected": -0.27526944875717163, "logps/chosen": -654.219482421875, "logps/ref_chosen": -59.99531555175781, "logps/ref_rejected": -103.9109115600586, "logps/rejected": -1006.8216552734375, "loss": 1.3258, "margin_dpo/margin_mean": 308.6865539550781, "margin_dpo/margin_std": 577.9491577148438, "step": 339 }, { "KL/chosen_KL_mean": -631.038330078125, "KL/mean": -757.1285400390625, "KL/rejected_KL_mean": -883.21875, "KL/std": 515.5421752929688, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 252.18031311035156, "fcm_dpo/q_t": 0.48659059405326843, "grad_norm": 7.41240930557251, "learning_rate": 2.959373794541426e-07, "logits/chosen": -0.2925180494785309, "logits/rejected": -0.2974250912666321, "logps/chosen": -683.8685913085938, "logps/ref_chosen": -52.83022689819336, "logps/ref_rejected": -73.10723114013672, "logps/rejected": -956.325927734375, "loss": 1.3383, "margin_dpo/margin_mean": 252.18031311035156, "margin_dpo/margin_std": 664.0989990234375, "step": 340 }, { "KL/chosen_KL_mean": -659.0855712890625, "KL/mean": -854.7269287109375, "KL/rejected_KL_mean": -1050.3682861328125, "KL/std": 603.756591796875, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 391.2827453613281, "fcm_dpo/q_t": 0.47924739122390747, "grad_norm": 8.219803810119629, "learning_rate": 2.946753005532965e-07, "logits/chosen": -0.30277353525161743, "logits/rejected": -0.31048113107681274, "logps/chosen": -706.9853515625, "logps/ref_chosen": -47.899803161621094, "logps/ref_rejected": -101.80987548828125, "logps/rejected": -1152.17822265625, "loss": 1.3112, "margin_dpo/margin_mean": 391.28277587890625, "margin_dpo/margin_std": 779.0953369140625, "step": 341 }, { "KL/chosen_KL_mean": -684.173828125, "KL/mean": -808.3017578125, "KL/rejected_KL_mean": -932.4296875, "KL/std": 543.6679077148438, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 248.255859375, "fcm_dpo/q_t": 0.4867771863937378, "grad_norm": 7.662797451019287, "learning_rate": 2.934120444167326e-07, "logits/chosen": -0.3431963324546814, "logits/rejected": -0.34687745571136475, "logps/chosen": -756.1705322265625, "logps/ref_chosen": -71.99664306640625, "logps/ref_rejected": -92.58959197998047, "logps/rejected": -1025.019287109375, "loss": 1.3395, "margin_dpo/margin_mean": 248.255859375, "margin_dpo/margin_std": 699.264892578125, "step": 342 }, { "KL/chosen_KL_mean": -627.5256958007812, "KL/mean": -823.489013671875, "KL/rejected_KL_mean": -1019.4523315429688, "KL/std": 555.4841918945312, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 391.92657470703125, "fcm_dpo/q_t": 0.4790971279144287, "grad_norm": 7.085535526275635, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -0.31986576318740845, "logits/rejected": -0.33300432562828064, "logps/chosen": -681.9312744140625, "logps/ref_chosen": -54.405616760253906, "logps/ref_rejected": -111.04142761230469, "logps/rejected": -1130.4937744140625, "loss": 1.3076, "margin_dpo/margin_mean": 391.92657470703125, "margin_dpo/margin_std": 557.3361206054688, "step": 343 }, { "KL/chosen_KL_mean": -689.4730224609375, "KL/mean": -871.2525634765625, "KL/rejected_KL_mean": -1053.0322265625, "KL/std": 726.9136962890625, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 363.5591735839844, "fcm_dpo/q_t": 0.48091691732406616, "grad_norm": 9.887747764587402, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -0.27034977078437805, "logits/rejected": -0.2850009799003601, "logps/chosen": -743.4376220703125, "logps/ref_chosen": -53.96466827392578, "logps/ref_rejected": -90.62336730957031, "logps/rejected": -1143.655517578125, "loss": 1.3198, "margin_dpo/margin_mean": 363.55914306640625, "margin_dpo/margin_std": 895.619140625, "step": 344 }, { "KL/chosen_KL_mean": -792.3978881835938, "KL/mean": -966.5239868164062, "KL/rejected_KL_mean": -1140.650146484375, "KL/std": 544.9487915039062, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 348.252197265625, "fcm_dpo/q_t": 0.48146194219589233, "grad_norm": 7.0328803062438965, "learning_rate": 2.896155456223163e-07, "logits/chosen": -0.279766321182251, "logits/rejected": -0.2895045280456543, "logps/chosen": -854.0836181640625, "logps/ref_chosen": -61.685699462890625, "logps/ref_rejected": -99.49041748046875, "logps/rejected": -1240.1405029296875, "loss": 1.3185, "margin_dpo/margin_mean": 348.2522277832031, "margin_dpo/margin_std": 693.2333984375, "step": 345 }, { "KL/chosen_KL_mean": -717.4666748046875, "KL/mean": -873.6412353515625, "KL/rejected_KL_mean": -1029.8157958984375, "KL/std": 560.0009155273438, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 312.34906005859375, "fcm_dpo/q_t": 0.4833087921142578, "grad_norm": 7.525673866271973, "learning_rate": 2.883479137196714e-07, "logits/chosen": -0.24226948618888855, "logits/rejected": -0.2475300431251526, "logps/chosen": -772.7229614257812, "logps/ref_chosen": -55.256263732910156, "logps/ref_rejected": -77.41532135009766, "logps/rejected": -1107.2310791015625, "loss": 1.3241, "margin_dpo/margin_mean": 312.34906005859375, "margin_dpo/margin_std": 564.9100952148438, "step": 346 }, { "KL/chosen_KL_mean": -728.3167114257812, "KL/mean": -851.8155517578125, "KL/rejected_KL_mean": -975.3143310546875, "KL/std": 575.36181640625, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 246.99755859375, "fcm_dpo/q_t": 0.4868515729904175, "grad_norm": 8.351125717163086, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -0.20478571951389313, "logits/rejected": -0.21141314506530762, "logps/chosen": -785.8829345703125, "logps/ref_chosen": -57.56623840332031, "logps/ref_rejected": -92.35509490966797, "logps/rejected": -1067.66943359375, "loss": 1.3399, "margin_dpo/margin_mean": 246.99752807617188, "margin_dpo/margin_std": 715.6409912109375, "step": 347 }, { "KL/chosen_KL_mean": -581.2276611328125, "KL/mean": -719.3048095703125, "KL/rejected_KL_mean": -857.382080078125, "KL/std": 477.3961181640625, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 276.1544494628906, "fcm_dpo/q_t": 0.485293447971344, "grad_norm": 5.781659126281738, "learning_rate": 2.858096518347179e-07, "logits/chosen": -0.2297358363866806, "logits/rejected": -0.2374425232410431, "logps/chosen": -637.5453491210938, "logps/ref_chosen": -56.31770324707031, "logps/ref_rejected": -89.13836669921875, "logps/rejected": -946.5204467773438, "loss": 1.3329, "margin_dpo/margin_mean": 276.1544494628906, "margin_dpo/margin_std": 637.7725219726562, "step": 348 }, { "KL/chosen_KL_mean": -541.278564453125, "KL/mean": -737.878662109375, "KL/rejected_KL_mean": -934.478759765625, "KL/std": 600.9308471679688, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 393.2002258300781, "fcm_dpo/q_t": 0.47914326190948486, "grad_norm": 16.43769645690918, "learning_rate": 2.845390887379706e-07, "logits/chosen": -0.16476523876190186, "logits/rejected": -0.1687617003917694, "logps/chosen": -599.3040771484375, "logps/ref_chosen": -58.025516510009766, "logps/ref_rejected": -97.50515747070312, "logps/rejected": -1031.98388671875, "loss": 1.3102, "margin_dpo/margin_mean": 393.2002258300781, "margin_dpo/margin_std": 747.94921875, "step": 349 }, { "KL/chosen_KL_mean": -617.7612915039062, "KL/mean": -776.6915283203125, "KL/rejected_KL_mean": -935.6217041015625, "KL/std": 595.83447265625, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 317.8603820800781, "fcm_dpo/q_t": 0.4831061363220215, "grad_norm": 10.051895141601562, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -0.21435394883155823, "logits/rejected": -0.23840287327766418, "logps/chosen": -682.091796875, "logps/ref_chosen": -64.33049011230469, "logps/ref_rejected": -89.87164306640625, "logps/rejected": -1025.4932861328125, "loss": 1.3262, "margin_dpo/margin_mean": 317.8603820800781, "margin_dpo/margin_std": 751.39208984375, "step": 350 }, { "KL/chosen_KL_mean": -567.7418823242188, "KL/mean": -760.047119140625, "KL/rejected_KL_mean": -952.3523559570312, "KL/std": 605.5358276367188, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 384.61053466796875, "fcm_dpo/q_t": 0.4796157479286194, "grad_norm": 6.818278789520264, "learning_rate": 2.819952656376487e-07, "logits/chosen": -0.23846173286437988, "logits/rejected": -0.25154590606689453, "logps/chosen": -628.4140014648438, "logps/ref_chosen": -60.6721305847168, "logps/ref_rejected": -101.5654296875, "logps/rejected": -1053.9178466796875, "loss": 1.3132, "margin_dpo/margin_mean": 384.61053466796875, "margin_dpo/margin_std": 796.4315795898438, "step": 351 }, { "KL/chosen_KL_mean": -745.8148193359375, "KL/mean": -805.4356689453125, "KL/rejected_KL_mean": -865.0565185546875, "KL/std": 641.70947265625, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.24172973632812, "fcm_dpo/q_t": 0.49365556240081787, "grad_norm": 10.332298278808594, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -0.2152498960494995, "logits/rejected": -0.20715099573135376, "logps/chosen": -816.75830078125, "logps/ref_chosen": -70.9434585571289, "logps/ref_rejected": -76.6419677734375, "logps/rejected": -941.698486328125, "loss": 1.3666, "margin_dpo/margin_mean": 119.2417221069336, "margin_dpo/margin_std": 703.8898315429688, "step": 352 }, { "KL/chosen_KL_mean": -665.606689453125, "KL/mean": -830.3818359375, "KL/rejected_KL_mean": -995.156982421875, "KL/std": 639.630859375, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 329.55035400390625, "fcm_dpo/q_t": 0.4826132357120514, "grad_norm": 10.239563941955566, "learning_rate": 2.794480701395219e-07, "logits/chosen": -0.24213480949401855, "logits/rejected": -0.2454732209444046, "logps/chosen": -724.001953125, "logps/ref_chosen": -58.39533996582031, "logps/ref_rejected": -80.33553314208984, "logps/rejected": -1075.4925537109375, "loss": 1.3252, "margin_dpo/margin_mean": 329.55035400390625, "margin_dpo/margin_std": 818.1527099609375, "step": 353 }, { "KL/chosen_KL_mean": -645.8775634765625, "KL/mean": -835.401123046875, "KL/rejected_KL_mean": -1024.924560546875, "KL/std": 729.5835571289062, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 379.0469665527344, "fcm_dpo/q_t": 0.48002883791923523, "grad_norm": 9.386272430419922, "learning_rate": 2.781732916288303e-07, "logits/chosen": -0.27470749616622925, "logits/rejected": -0.28340351581573486, "logps/chosen": -705.6806030273438, "logps/ref_chosen": -59.80299377441406, "logps/ref_rejected": -88.75750732421875, "logps/rejected": -1113.68212890625, "loss": 1.3138, "margin_dpo/margin_mean": 379.0469970703125, "margin_dpo/margin_std": 765.1248779296875, "step": 354 }, { "KL/chosen_KL_mean": -676.1866455078125, "KL/mean": -879.7360229492188, "KL/rejected_KL_mean": -1083.285400390625, "KL/std": 590.9983520507812, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 407.0987243652344, "fcm_dpo/q_t": 0.47829318046569824, "grad_norm": 7.246441841125488, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -0.3153689503669739, "logits/rejected": -0.32817888259887695, "logps/chosen": -730.315185546875, "logps/ref_chosen": -54.12849807739258, "logps/ref_rejected": -82.40606689453125, "logps/rejected": -1165.69140625, "loss": 1.3055, "margin_dpo/margin_mean": 407.0987243652344, "margin_dpo/margin_std": 628.2813720703125, "step": 355 }, { "KL/chosen_KL_mean": -843.4148559570312, "KL/mean": -896.946533203125, "KL/rejected_KL_mean": -950.4781494140625, "KL/std": 694.555908203125, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.06331634521484, "fcm_dpo/q_t": 0.4944502115249634, "grad_norm": 13.43985366821289, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -0.3528267443180084, "logits/rejected": -0.346576452255249, "logps/chosen": -908.088623046875, "logps/ref_chosen": -64.6738052368164, "logps/ref_rejected": -75.89926147460938, "logps/rejected": -1026.37744140625, "loss": 1.3731, "margin_dpo/margin_mean": 107.06331634521484, "margin_dpo/margin_std": 916.18310546875, "step": 356 }, { "KL/chosen_KL_mean": -742.629638671875, "KL/mean": -851.8154907226562, "KL/rejected_KL_mean": -961.0013427734375, "KL/std": 635.5723876953125, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 218.3716583251953, "fcm_dpo/q_t": 0.48831281065940857, "grad_norm": 11.403450965881348, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -0.34209227561950684, "logits/rejected": -0.3437783718109131, "logps/chosen": -795.35546875, "logps/ref_chosen": -52.725799560546875, "logps/ref_rejected": -86.84115600585938, "logps/rejected": -1047.842529296875, "loss": 1.3475, "margin_dpo/margin_mean": 218.3716583251953, "margin_dpo/margin_std": 797.1085205078125, "step": 357 }, { "KL/chosen_KL_mean": -685.59619140625, "KL/mean": -857.2205200195312, "KL/rejected_KL_mean": -1028.844970703125, "KL/std": 697.257568359375, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 343.24871826171875, "fcm_dpo/q_t": 0.4819628894329071, "grad_norm": 6.759915828704834, "learning_rate": 2.730670898658255e-07, "logits/chosen": -0.38823795318603516, "logits/rejected": -0.39373278617858887, "logps/chosen": -748.8016357421875, "logps/ref_chosen": -63.20543670654297, "logps/ref_rejected": -88.373291015625, "logps/rejected": -1117.21826171875, "loss": 1.3229, "margin_dpo/margin_mean": 343.24871826171875, "margin_dpo/margin_std": 876.3508911132812, "step": 358 }, { "KL/chosen_KL_mean": -853.0894775390625, "KL/mean": -1017.8627319335938, "KL/rejected_KL_mean": -1182.635986328125, "KL/std": 709.51904296875, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 329.54638671875, "fcm_dpo/q_t": 0.4825381338596344, "grad_norm": 10.495577812194824, "learning_rate": 2.717889356869146e-07, "logits/chosen": -0.3527218997478485, "logits/rejected": -0.3608393669128418, "logps/chosen": -909.459716796875, "logps/ref_chosen": -56.370216369628906, "logps/ref_rejected": -82.17375183105469, "logps/rejected": -1264.8096923828125, "loss": 1.3279, "margin_dpo/margin_mean": 329.54638671875, "margin_dpo/margin_std": 976.3280639648438, "step": 359 }, { "KL/chosen_KL_mean": -797.648193359375, "KL/mean": -908.0338745117188, "KL/rejected_KL_mean": -1018.4195556640625, "KL/std": 799.6057739257812, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 220.7712860107422, "fcm_dpo/q_t": 0.4882170855998993, "grad_norm": 9.871207237243652, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -0.30490997433662415, "logits/rejected": -0.2990133762359619, "logps/chosen": -849.108642578125, "logps/ref_chosen": -51.460384368896484, "logps/ref_rejected": -69.83892059326172, "logps/rejected": -1088.2584228515625, "loss": 1.3449, "margin_dpo/margin_mean": 220.77130126953125, "margin_dpo/margin_std": 689.1402587890625, "step": 360 }, { "KL/chosen_KL_mean": -762.4368896484375, "KL/mean": -887.9981689453125, "KL/rejected_KL_mean": -1013.559326171875, "KL/std": 713.6858520507812, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 251.12237548828125, "fcm_dpo/q_t": 0.4866461157798767, "grad_norm": 6.593760967254639, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -0.2922874689102173, "logits/rejected": -0.3055328130722046, "logps/chosen": -816.3064575195312, "logps/ref_chosen": -53.86951446533203, "logps/ref_rejected": -90.7692642211914, "logps/rejected": -1104.32861328125, "loss": 1.3401, "margin_dpo/margin_mean": 251.12237548828125, "margin_dpo/margin_std": 772.4447631835938, "step": 361 }, { "KL/chosen_KL_mean": -624.3651123046875, "KL/mean": -883.3717041015625, "KL/rejected_KL_mean": -1142.37841796875, "KL/std": 716.97802734375, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 518.0132446289062, "fcm_dpo/q_t": 0.47263094782829285, "grad_norm": 13.076441764831543, "learning_rate": 2.679511629948319e-07, "logits/chosen": -0.30570244789123535, "logits/rejected": -0.3104197382926941, "logps/chosen": -683.004150390625, "logps/ref_chosen": -58.639060974121094, "logps/ref_rejected": -105.58195495605469, "logps/rejected": -1247.9603271484375, "loss": 1.2863, "margin_dpo/margin_mean": 518.01318359375, "margin_dpo/margin_std": 827.43408203125, "step": 362 }, { "KL/chosen_KL_mean": -607.5775146484375, "KL/mean": -829.8275146484375, "KL/rejected_KL_mean": -1052.0775146484375, "KL/std": 665.65380859375, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 444.5, "fcm_dpo/q_t": 0.4764130711555481, "grad_norm": 6.851832389831543, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -0.29445457458496094, "logits/rejected": -0.31152263283729553, "logps/chosen": -652.1359252929688, "logps/ref_chosen": -44.558380126953125, "logps/ref_rejected": -74.69496154785156, "logps/rejected": -1126.7724609375, "loss": 1.2996, "margin_dpo/margin_mean": 444.5, "margin_dpo/margin_std": 740.0186767578125, "step": 363 }, { "KL/chosen_KL_mean": -626.3096923828125, "KL/mean": -794.18212890625, "KL/rejected_KL_mean": -962.0546264648438, "KL/std": 631.0501708984375, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 335.74493408203125, "fcm_dpo/q_t": 0.48214179277420044, "grad_norm": 7.443741321563721, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -0.30771857500076294, "logits/rejected": -0.32001155614852905, "logps/chosen": -675.204345703125, "logps/ref_chosen": -48.894622802734375, "logps/ref_rejected": -91.395751953125, "logps/rejected": -1053.4503173828125, "loss": 1.3219, "margin_dpo/margin_mean": 335.7449035644531, "margin_dpo/margin_std": 725.6460571289062, "step": 364 }, { "KL/chosen_KL_mean": -686.364013671875, "KL/mean": -917.268798828125, "KL/rejected_KL_mean": -1148.173583984375, "KL/std": 698.85107421875, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 461.8096923828125, "fcm_dpo/q_t": 0.4755679666996002, "grad_norm": 10.261777877807617, "learning_rate": 2.641091331089811e-07, "logits/chosen": -0.31375348567962646, "logits/rejected": -0.3348388671875, "logps/chosen": -737.856689453125, "logps/ref_chosen": -51.49274444580078, "logps/ref_rejected": -92.70166778564453, "logps/rejected": -1240.875244140625, "loss": 1.2973, "margin_dpo/margin_mean": 461.8096923828125, "margin_dpo/margin_std": 814.092041015625, "step": 365 }, { "KL/chosen_KL_mean": -645.4697265625, "KL/mean": -870.0846557617188, "KL/rejected_KL_mean": -1094.69970703125, "KL/std": 782.6004028320312, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 449.22991943359375, "fcm_dpo/q_t": 0.4762273132801056, "grad_norm": 7.029806613922119, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -0.32882604002952576, "logits/rejected": -0.3432251214981079, "logps/chosen": -690.1902465820312, "logps/ref_chosen": -44.7205696105957, "logps/ref_rejected": -83.31040954589844, "logps/rejected": -1178.010009765625, "loss": 1.3052, "margin_dpo/margin_mean": 449.2299499511719, "margin_dpo/margin_std": 1045.4915771484375, "step": 366 }, { "KL/chosen_KL_mean": -684.59326171875, "KL/mean": -842.3504638671875, "KL/rejected_KL_mean": -1000.1076049804688, "KL/std": 633.3516845703125, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 315.514404296875, "fcm_dpo/q_t": 0.4832141101360321, "grad_norm": 6.7098283767700195, "learning_rate": 2.615458646614349e-07, "logits/chosen": -0.3017085790634155, "logits/rejected": -0.3029378056526184, "logps/chosen": -742.9986572265625, "logps/ref_chosen": -58.405418395996094, "logps/ref_rejected": -76.75132751464844, "logps/rejected": -1076.85888671875, "loss": 1.326, "margin_dpo/margin_mean": 315.514404296875, "margin_dpo/margin_std": 734.252685546875, "step": 367 }, { "KL/chosen_KL_mean": -611.4556884765625, "KL/mean": -943.615966796875, "KL/rejected_KL_mean": -1275.7762451171875, "KL/std": 775.81884765625, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 664.3204345703125, "fcm_dpo/q_t": 0.46484458446502686, "grad_norm": 25.336750030517578, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -0.2958596348762512, "logits/rejected": -0.3062882423400879, "logps/chosen": -655.908203125, "logps/ref_chosen": -44.452518463134766, "logps/ref_rejected": -98.55526733398438, "logps/rejected": -1374.33154296875, "loss": 1.2562, "margin_dpo/margin_mean": 664.3204345703125, "margin_dpo/margin_std": 804.1547241210938, "step": 368 }, { "KL/chosen_KL_mean": -814.8972778320312, "KL/mean": -954.0897216796875, "KL/rejected_KL_mean": -1093.2823486328125, "KL/std": 686.5308227539062, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 278.38507080078125, "fcm_dpo/q_t": 0.48527395725250244, "grad_norm": 9.702912330627441, "learning_rate": 2.589813792448196e-07, "logits/chosen": -0.3168843388557434, "logits/rejected": -0.32195502519607544, "logps/chosen": -886.27880859375, "logps/ref_chosen": -71.38150024414062, "logps/ref_rejected": -91.29582214355469, "logps/rejected": -1184.5782470703125, "loss": 1.337, "margin_dpo/margin_mean": 278.38507080078125, "margin_dpo/margin_std": 897.73291015625, "step": 369 }, { "KL/chosen_KL_mean": -883.7716064453125, "KL/mean": -960.92041015625, "KL/rejected_KL_mean": -1038.0693359375, "KL/std": 785.7099609375, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.29771423339844, "fcm_dpo/q_t": 0.491788387298584, "grad_norm": 19.420272827148438, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -0.3010944128036499, "logits/rejected": -0.301439106464386, "logps/chosen": -955.3790283203125, "logps/ref_chosen": -71.60749816894531, "logps/ref_rejected": -97.25978088378906, "logps/rejected": -1135.3291015625, "loss": 1.3638, "margin_dpo/margin_mean": 154.2977294921875, "margin_dpo/margin_std": 940.9684448242188, "step": 370 }, { "KL/chosen_KL_mean": -807.43408203125, "KL/mean": -988.1475219726562, "KL/rejected_KL_mean": -1168.861083984375, "KL/std": 789.1993408203125, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 361.4268798828125, "fcm_dpo/q_t": 0.48083311319351196, "grad_norm": 7.047267913818359, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -0.3067021667957306, "logits/rejected": -0.3089839816093445, "logps/chosen": -876.8486328125, "logps/ref_chosen": -69.41448974609375, "logps/ref_rejected": -99.17217254638672, "logps/rejected": -1268.033203125, "loss": 1.3188, "margin_dpo/margin_mean": 361.4268798828125, "margin_dpo/margin_std": 860.834228515625, "step": 371 }, { "KL/chosen_KL_mean": -753.4928588867188, "KL/mean": -979.7862548828125, "KL/rejected_KL_mean": -1206.079833984375, "KL/std": 863.1746215820312, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 452.5868835449219, "fcm_dpo/q_t": 0.47651100158691406, "grad_norm": 18.109085083007812, "learning_rate": 2.551329606220976e-07, "logits/chosen": -0.28337526321411133, "logits/rejected": -0.30206161737442017, "logps/chosen": -815.3108520507812, "logps/ref_chosen": -61.8179931640625, "logps/ref_rejected": -78.53948974609375, "logps/rejected": -1284.6192626953125, "loss": 1.3037, "margin_dpo/margin_mean": 452.5869140625, "margin_dpo/margin_std": 1032.625732421875, "step": 372 }, { "KL/chosen_KL_mean": -822.2640380859375, "KL/mean": -992.8091430664062, "KL/rejected_KL_mean": -1163.354248046875, "KL/std": 807.1524047851562, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 341.0901794433594, "fcm_dpo/q_t": 0.4819341003894806, "grad_norm": 9.496335983276367, "learning_rate": 2.538498388222517e-07, "logits/chosen": -0.25716885924339294, "logits/rejected": -0.26763084530830383, "logps/chosen": -886.481201171875, "logps/ref_chosen": -64.21713256835938, "logps/ref_rejected": -85.95960998535156, "logps/rejected": -1249.31396484375, "loss": 1.324, "margin_dpo/margin_mean": 341.0901794433594, "margin_dpo/margin_std": 889.7625122070312, "step": 373 }, { "KL/chosen_KL_mean": -697.943115234375, "KL/mean": -933.2283325195312, "KL/rejected_KL_mean": -1168.5135498046875, "KL/std": 870.4952392578125, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 470.57049560546875, "fcm_dpo/q_t": 0.47532325983047485, "grad_norm": 11.988911628723145, "learning_rate": 2.525666155755725e-07, "logits/chosen": -0.3318430483341217, "logits/rejected": -0.33818480372428894, "logps/chosen": -768.59326171875, "logps/ref_chosen": -70.65018463134766, "logps/ref_rejected": -93.64016723632812, "logps/rejected": -1262.1536865234375, "loss": 1.3009, "margin_dpo/margin_mean": 470.57049560546875, "margin_dpo/margin_std": 1058.4764404296875, "step": 374 }, { "KL/chosen_KL_mean": -752.4849243164062, "KL/mean": -971.480712890625, "KL/rejected_KL_mean": -1190.4764404296875, "KL/std": 719.1031494140625, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 437.9915771484375, "fcm_dpo/q_t": 0.47678330540657043, "grad_norm": 14.679875373840332, "learning_rate": 2.512833246961859e-07, "logits/chosen": -0.33781394362449646, "logits/rejected": -0.35372868180274963, "logps/chosen": -812.565185546875, "logps/ref_chosen": -60.080223083496094, "logps/ref_rejected": -88.93830871582031, "logps/rejected": -1279.414794921875, "loss": 1.3037, "margin_dpo/margin_mean": 437.9915771484375, "margin_dpo/margin_std": 881.63671875, "step": 375 }, { "KL/chosen_KL_mean": -957.6834716796875, "KL/mean": -1178.202880859375, "KL/rejected_KL_mean": -1398.72216796875, "KL/std": 931.96435546875, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 441.0386962890625, "fcm_dpo/q_t": 0.4765964150428772, "grad_norm": 15.095070838928223, "learning_rate": 2.5e-07, "logits/chosen": -0.3589029312133789, "logits/rejected": -0.3605373501777649, "logps/chosen": -1020.34375, "logps/ref_chosen": -62.660308837890625, "logps/ref_rejected": -105.52660369873047, "logps/rejected": -1504.248779296875, "loss": 1.3074, "margin_dpo/margin_mean": 441.0386962890625, "margin_dpo/margin_std": 1077.4156494140625, "step": 376 }, { "KL/chosen_KL_mean": -1065.7764892578125, "KL/mean": -1337.8623046875, "KL/rejected_KL_mean": -1609.947998046875, "KL/std": 1192.957763671875, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 544.1714477539062, "fcm_dpo/q_t": 0.47193536162376404, "grad_norm": 10.05336856842041, "learning_rate": 2.487166753038141e-07, "logits/chosen": -0.3872339129447937, "logits/rejected": -0.40310975909233093, "logps/chosen": -1120.2552490234375, "logps/ref_chosen": -54.478736877441406, "logps/ref_rejected": -98.70335388183594, "logps/rejected": -1708.6513671875, "loss": 1.2974, "margin_dpo/margin_mean": 544.1714477539062, "margin_dpo/margin_std": 1465.950439453125, "step": 377 }, { "KL/chosen_KL_mean": -899.20458984375, "KL/mean": -1161.714111328125, "KL/rejected_KL_mean": -1424.2236328125, "KL/std": 916.3400268554688, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 525.0191650390625, "fcm_dpo/q_t": 0.4725556969642639, "grad_norm": 14.26543140411377, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -0.42005640268325806, "logits/rejected": -0.44007664918899536, "logps/chosen": -944.2251586914062, "logps/ref_chosen": -45.02053451538086, "logps/ref_rejected": -88.0469741821289, "logps/rejected": -1512.270751953125, "loss": 1.2936, "margin_dpo/margin_mean": 525.0191650390625, "margin_dpo/margin_std": 1212.896728515625, "step": 378 }, { "KL/chosen_KL_mean": -1210.5340576171875, "KL/mean": -1521.940673828125, "KL/rejected_KL_mean": -1833.34716796875, "KL/std": 1398.197509765625, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 622.81298828125, "fcm_dpo/q_t": 0.4689289629459381, "grad_norm": 16.45003318786621, "learning_rate": 2.461501611777483e-07, "logits/chosen": -0.41934919357299805, "logits/rejected": -0.44911909103393555, "logps/chosen": -1263.7161865234375, "logps/ref_chosen": -53.182098388671875, "logps/ref_rejected": -114.3001708984375, "logps/rejected": -1947.6473388671875, "loss": 1.2913, "margin_dpo/margin_mean": 622.81298828125, "margin_dpo/margin_std": 1755.5107421875, "step": 379 }, { "KL/chosen_KL_mean": -1069.46337890625, "KL/mean": -1406.3856201171875, "KL/rejected_KL_mean": -1743.3077392578125, "KL/std": 1310.0059814453125, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 673.8443603515625, "fcm_dpo/q_t": 0.4651104509830475, "grad_norm": 16.879432678222656, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -0.43862009048461914, "logits/rejected": -0.4662778377532959, "logps/chosen": -1120.8165283203125, "logps/ref_chosen": -51.3530387878418, "logps/ref_rejected": -104.19169616699219, "logps/rejected": -1847.49951171875, "loss": 1.2767, "margin_dpo/margin_mean": 673.8442993164062, "margin_dpo/margin_std": 1641.3406982421875, "step": 380 }, { "KL/chosen_KL_mean": -1037.765380859375, "KL/mean": -1180.1309814453125, "KL/rejected_KL_mean": -1322.49658203125, "KL/std": 1107.69970703125, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 284.7312316894531, "fcm_dpo/q_t": 0.48559147119522095, "grad_norm": 25.2132625579834, "learning_rate": 2.435840528363426e-07, "logits/chosen": -0.4325522482395172, "logits/rejected": -0.4366803765296936, "logps/chosen": -1095.568359375, "logps/ref_chosen": -57.80306625366211, "logps/ref_rejected": -79.21940612792969, "logps/rejected": -1401.716064453125, "loss": 1.3524, "margin_dpo/margin_mean": 284.73126220703125, "margin_dpo/margin_std": 1517.94580078125, "step": 381 }, { "KL/chosen_KL_mean": -853.733642578125, "KL/mean": -1091.995361328125, "KL/rejected_KL_mean": -1330.25732421875, "KL/std": 791.60595703125, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 476.52362060546875, "fcm_dpo/q_t": 0.47479626536369324, "grad_norm": 9.29355525970459, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -0.4174577593803406, "logits/rejected": -0.42780351638793945, "logps/chosen": -919.75390625, "logps/ref_chosen": -66.02030181884766, "logps/ref_rejected": -110.71016693115234, "logps/rejected": -1440.9674072265625, "loss": 1.2955, "margin_dpo/margin_mean": 476.52362060546875, "margin_dpo/margin_std": 863.3676147460938, "step": 382 }, { "KL/chosen_KL_mean": -779.2030029296875, "KL/mean": -1037.3272705078125, "KL/rejected_KL_mean": -1295.451416015625, "KL/std": 926.0235595703125, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 516.2486572265625, "fcm_dpo/q_t": 0.47278502583503723, "grad_norm": 11.367318153381348, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -0.39227527379989624, "logits/rejected": -0.41163206100463867, "logps/chosen": -829.594482421875, "logps/ref_chosen": -50.39148712158203, "logps/ref_rejected": -93.71589660644531, "logps/rejected": -1389.16748046875, "loss": 1.2977, "margin_dpo/margin_mean": 516.2485961914062, "margin_dpo/margin_std": 1285.308837890625, "step": 383 }, { "KL/chosen_KL_mean": -682.0059814453125, "KL/mean": -844.9951171875, "KL/rejected_KL_mean": -1007.984375, "KL/std": 720.017578125, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 325.9783935546875, "fcm_dpo/q_t": 0.48265010118484497, "grad_norm": 9.724451065063477, "learning_rate": 2.397362428170992e-07, "logits/chosen": -0.4297202527523041, "logits/rejected": -0.4280009865760803, "logps/chosen": -734.0520629882812, "logps/ref_chosen": -52.046104431152344, "logps/ref_rejected": -85.76089477539062, "logps/rejected": -1093.7452392578125, "loss": 1.3234, "margin_dpo/margin_mean": 325.9783630371094, "margin_dpo/margin_std": 708.7644653320312, "step": 384 }, { "KL/chosen_KL_mean": -567.719970703125, "KL/mean": -734.8634033203125, "KL/rejected_KL_mean": -902.0068359375, "KL/std": 536.7576904296875, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 334.28692626953125, "fcm_dpo/q_t": 0.4821450114250183, "grad_norm": 15.683626174926758, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -0.36829620599746704, "logits/rejected": -0.37683045864105225, "logps/chosen": -633.2720947265625, "logps/ref_chosen": -65.55215454101562, "logps/ref_rejected": -77.82792663574219, "logps/rejected": -979.834716796875, "loss": 1.3211, "margin_dpo/margin_mean": 334.28692626953125, "margin_dpo/margin_std": 671.5125732421875, "step": 385 }, { "KL/chosen_KL_mean": -539.3319091796875, "KL/mean": -767.2230224609375, "KL/rejected_KL_mean": -995.1141357421875, "KL/std": 658.7130126953125, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 455.7822265625, "fcm_dpo/q_t": 0.4757852554321289, "grad_norm": 15.568609237670898, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -0.35743653774261475, "logits/rejected": -0.36365118622779846, "logps/chosen": -597.5537109375, "logps/ref_chosen": -58.22185516357422, "logps/ref_rejected": -92.32742309570312, "logps/rejected": -1087.441650390625, "loss": 1.2979, "margin_dpo/margin_mean": 455.7822570800781, "margin_dpo/margin_std": 782.3114013671875, "step": 386 }, { "KL/chosen_KL_mean": -720.632080078125, "KL/mean": -928.3619384765625, "KL/rejected_KL_mean": -1136.091796875, "KL/std": 814.0172119140625, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 415.4595947265625, "fcm_dpo/q_t": 0.4782324433326721, "grad_norm": 9.539994239807129, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -0.4057348966598511, "logits/rejected": -0.41669708490371704, "logps/chosen": -787.051513671875, "logps/ref_chosen": -66.41944885253906, "logps/ref_rejected": -92.16915893554688, "logps/rejected": -1228.2608642578125, "loss": 1.3127, "margin_dpo/margin_mean": 415.45965576171875, "margin_dpo/margin_std": 1032.929443359375, "step": 387 }, { "KL/chosen_KL_mean": -648.9365234375, "KL/mean": -877.4620971679688, "KL/rejected_KL_mean": -1105.9876708984375, "KL/std": 719.4478759765625, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 457.05108642578125, "fcm_dpo/q_t": 0.47609227895736694, "grad_norm": 7.468353271484375, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -0.3469131588935852, "logits/rejected": -0.3659330904483795, "logps/chosen": -699.0660400390625, "logps/ref_chosen": -50.129459381103516, "logps/ref_rejected": -104.43305969238281, "logps/rejected": -1210.420654296875, "loss": 1.3012, "margin_dpo/margin_mean": 457.05108642578125, "margin_dpo/margin_std": 926.29736328125, "step": 388 }, { "KL/chosen_KL_mean": -743.910888671875, "KL/mean": -964.4112548828125, "KL/rejected_KL_mean": -1184.91162109375, "KL/std": 831.6654052734375, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 441.00067138671875, "fcm_dpo/q_t": 0.4765745997428894, "grad_norm": 15.217109680175781, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -0.394728422164917, "logits/rejected": -0.4151400327682495, "logps/chosen": -801.8175048828125, "logps/ref_chosen": -57.906593322753906, "logps/ref_rejected": -77.91454315185547, "logps/rejected": -1262.826171875, "loss": 1.3065, "margin_dpo/margin_mean": 441.00067138671875, "margin_dpo/margin_std": 1042.0787353515625, "step": 389 }, { "KL/chosen_KL_mean": -730.2432861328125, "KL/mean": -956.1160888671875, "KL/rejected_KL_mean": -1181.9888916015625, "KL/std": 893.8538208007812, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 451.74566650390625, "fcm_dpo/q_t": 0.4772656559944153, "grad_norm": 14.349326133728027, "learning_rate": 2.320488370051681e-07, "logits/chosen": -0.36197686195373535, "logits/rejected": -0.3790530562400818, "logps/chosen": -779.4691772460938, "logps/ref_chosen": -49.22591781616211, "logps/ref_rejected": -85.5281982421875, "logps/rejected": -1267.51708984375, "loss": 1.3101, "margin_dpo/margin_mean": 451.7456970214844, "margin_dpo/margin_std": 1213.085693359375, "step": 390 }, { "KL/chosen_KL_mean": -934.3198852539062, "KL/mean": -995.6199951171875, "KL/rejected_KL_mean": -1056.920166015625, "KL/std": 943.6309814453125, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.60029602050781, "fcm_dpo/q_t": 0.49323803186416626, "grad_norm": 19.008642196655273, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -0.35622894763946533, "logits/rejected": -0.3512190580368042, "logps/chosen": -998.6495361328125, "logps/ref_chosen": -64.32965087890625, "logps/ref_rejected": -86.73820495605469, "logps/rejected": -1143.6583251953125, "loss": 1.3729, "margin_dpo/margin_mean": 122.60028076171875, "margin_dpo/margin_std": 1043.1915283203125, "step": 391 }, { "KL/chosen_KL_mean": -610.44775390625, "KL/mean": -939.5457763671875, "KL/rejected_KL_mean": -1268.643798828125, "KL/std": 832.3636474609375, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 658.1959838867188, "fcm_dpo/q_t": 0.4652509093284607, "grad_norm": 24.722393035888672, "learning_rate": 2.294897926507156e-07, "logits/chosen": -0.356148362159729, "logits/rejected": -0.3655499219894409, "logps/chosen": -663.9517822265625, "logps/ref_chosen": -53.50397872924805, "logps/ref_rejected": -102.34584045410156, "logps/rejected": -1370.9896240234375, "loss": 1.2592, "margin_dpo/margin_mean": 658.1959228515625, "margin_dpo/margin_std": 873.6790771484375, "step": 392 }, { "KL/chosen_KL_mean": -772.148681640625, "KL/mean": -1021.87158203125, "KL/rejected_KL_mean": -1271.5946044921875, "KL/std": 1009.1136474609375, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 499.4459228515625, "fcm_dpo/q_t": 0.4741513729095459, "grad_norm": 10.550104141235352, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -0.36827749013900757, "logits/rejected": -0.38175201416015625, "logps/chosen": -818.62255859375, "logps/ref_chosen": -46.473915100097656, "logps/ref_rejected": -71.96885681152344, "logps/rejected": -1343.5634765625, "loss": 1.3002, "margin_dpo/margin_mean": 499.4459228515625, "margin_dpo/margin_std": 1268.5267333984375, "step": 393 }, { "KL/chosen_KL_mean": -866.2552490234375, "KL/mean": -1105.1195068359375, "KL/rejected_KL_mean": -1343.98388671875, "KL/std": 1064.1612548828125, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 477.7286376953125, "fcm_dpo/q_t": 0.4750671088695526, "grad_norm": 10.232756614685059, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -0.39185428619384766, "logits/rejected": -0.4062337279319763, "logps/chosen": -919.1668090820312, "logps/ref_chosen": -52.91154861450195, "logps/ref_rejected": -90.8226318359375, "logps/rejected": -1434.8065185546875, "loss": 1.2995, "margin_dpo/margin_mean": 477.7286376953125, "margin_dpo/margin_std": 1065.57275390625, "step": 394 }, { "KL/chosen_KL_mean": -986.03759765625, "KL/mean": -1240.0714111328125, "KL/rejected_KL_mean": -1494.10498046875, "KL/std": 1024.42822265625, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 508.0675048828125, "fcm_dpo/q_t": 0.473642498254776, "grad_norm": 12.610128402709961, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -0.4311944246292114, "logits/rejected": -0.44871261715888977, "logps/chosen": -1048.583740234375, "logps/ref_chosen": -62.546112060546875, "logps/ref_rejected": -83.78262329101562, "logps/rejected": -1577.8876953125, "loss": 1.3004, "margin_dpo/margin_mean": 508.0675048828125, "margin_dpo/margin_std": 1298.3829345703125, "step": 395 }, { "KL/chosen_KL_mean": -1033.08447265625, "KL/mean": -1220.91015625, "KL/rejected_KL_mean": -1408.7357177734375, "KL/std": 1084.0780029296875, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 375.6512451171875, "fcm_dpo/q_t": 0.4801104664802551, "grad_norm": 10.486356735229492, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -0.43420612812042236, "logits/rejected": -0.4390130639076233, "logps/chosen": -1102.0804443359375, "logps/ref_chosen": -68.99594116210938, "logps/ref_rejected": -88.64665985107422, "logps/rejected": -1497.38232421875, "loss": 1.3169, "margin_dpo/margin_mean": 375.6512451171875, "margin_dpo/margin_std": 911.8556518554688, "step": 396 }, { "KL/chosen_KL_mean": -1034.108154296875, "KL/mean": -1459.783447265625, "KL/rejected_KL_mean": -1885.45849609375, "KL/std": 1350.3555908203125, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 851.3504028320312, "fcm_dpo/q_t": 0.45677071809768677, "grad_norm": 15.112154006958008, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -0.4225345849990845, "logits/rejected": -0.46220195293426514, "logps/chosen": -1095.3853759765625, "logps/ref_chosen": -61.27716827392578, "logps/ref_rejected": -103.11612701416016, "logps/rejected": -1988.57470703125, "loss": 1.2402, "margin_dpo/margin_mean": 851.3504028320312, "margin_dpo/margin_std": 1585.188720703125, "step": 397 }, { "KL/chosen_KL_mean": -1233.375244140625, "KL/mean": -1574.85107421875, "KL/rejected_KL_mean": -1916.3270263671875, "KL/std": 1434.3673095703125, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 682.951904296875, "fcm_dpo/q_t": 0.46584078669548035, "grad_norm": 23.73081398010254, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -0.4878053069114685, "logits/rejected": -0.5119198560714722, "logps/chosen": -1301.5267333984375, "logps/ref_chosen": -68.15155029296875, "logps/ref_rejected": -108.52360534667969, "logps/rejected": -2024.8505859375, "loss": 1.2838, "margin_dpo/margin_mean": 682.951904296875, "margin_dpo/margin_std": 1882.985595703125, "step": 398 }, { "KL/chosen_KL_mean": -991.1121826171875, "KL/mean": -1204.9122314453125, "KL/rejected_KL_mean": -1418.7122802734375, "KL/std": 1063.78857421875, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 427.60003662109375, "fcm_dpo/q_t": 0.4776034951210022, "grad_norm": 33.797943115234375, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -0.4256382882595062, "logits/rejected": -0.4321112334728241, "logps/chosen": -1052.001953125, "logps/ref_chosen": -60.889801025390625, "logps/ref_rejected": -77.965576171875, "logps/rejected": -1496.6778564453125, "loss": 1.3194, "margin_dpo/margin_mean": 427.60003662109375, "margin_dpo/margin_std": 1410.41748046875, "step": 399 }, { "KL/chosen_KL_mean": -877.5531005859375, "KL/mean": -1235.20166015625, "KL/rejected_KL_mean": -1592.8502197265625, "KL/std": 992.3692016601562, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 715.2971801757812, "fcm_dpo/q_t": 0.46237778663635254, "grad_norm": 10.352943420410156, "learning_rate": 2.192779273338215e-07, "logits/chosen": -0.426511287689209, "logits/rejected": -0.44806522130966187, "logps/chosen": -941.1966552734375, "logps/ref_chosen": -63.64359664916992, "logps/ref_rejected": -105.252685546875, "logps/rejected": -1698.1029052734375, "loss": 1.2569, "margin_dpo/margin_mean": 715.2971801757812, "margin_dpo/margin_std": 1255.512451171875, "step": 400 }, { "KL/chosen_KL_mean": -964.7339477539062, "KL/mean": -1108.6376953125, "KL/rejected_KL_mean": -1252.54150390625, "KL/std": 1027.1666259765625, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 287.8074951171875, "fcm_dpo/q_t": 0.48464304208755493, "grad_norm": 16.782976150512695, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -0.3969235122203827, "logits/rejected": -0.39872997999191284, "logps/chosen": -1021.89697265625, "logps/ref_chosen": -57.16303253173828, "logps/ref_rejected": -83.79249572753906, "logps/rejected": -1336.333984375, "loss": 1.3476, "margin_dpo/margin_mean": 287.8074951171875, "margin_dpo/margin_std": 1398.988037109375, "step": 401 }, { "KL/chosen_KL_mean": -710.2174072265625, "KL/mean": -1041.47021484375, "KL/rejected_KL_mean": -1372.72314453125, "KL/std": 974.0499877929688, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 662.5057983398438, "fcm_dpo/q_t": 0.4654858708381653, "grad_norm": 8.760465621948242, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -0.4055900573730469, "logits/rejected": -0.4254668951034546, "logps/chosen": -760.957763671875, "logps/ref_chosen": -50.74037170410156, "logps/ref_rejected": -81.0460433959961, "logps/rejected": -1453.7691650390625, "loss": 1.264, "margin_dpo/margin_mean": 662.5057373046875, "margin_dpo/margin_std": 1141.46630859375, "step": 402 }, { "KL/chosen_KL_mean": -607.0709228515625, "KL/mean": -857.822021484375, "KL/rejected_KL_mean": -1108.5732421875, "KL/std": 957.948486328125, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 501.5023498535156, "fcm_dpo/q_t": 0.47387418150901794, "grad_norm": 21.134559631347656, "learning_rate": 2.154609112620295e-07, "logits/chosen": -0.3894062042236328, "logits/rejected": -0.41035395860671997, "logps/chosen": -654.2182006835938, "logps/ref_chosen": -47.14731216430664, "logps/ref_rejected": -77.2666015625, "logps/rejected": -1185.83984375, "loss": 1.2943, "margin_dpo/margin_mean": 501.5023193359375, "margin_dpo/margin_std": 1025.334228515625, "step": 403 }, { "KL/chosen_KL_mean": -573.6932373046875, "KL/mean": -833.4141845703125, "KL/rejected_KL_mean": -1093.13525390625, "KL/std": 773.28173828125, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 519.4420776367188, "fcm_dpo/q_t": 0.47276201844215393, "grad_norm": 26.00446128845215, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -0.3456383943557739, "logits/rejected": -0.35928741097450256, "logps/chosen": -621.5684814453125, "logps/ref_chosen": -47.875274658203125, "logps/ref_rejected": -77.15499877929688, "logps/rejected": -1170.290283203125, "loss": 1.2903, "margin_dpo/margin_mean": 519.4420166015625, "margin_dpo/margin_std": 1035.61083984375, "step": 404 }, { "KL/chosen_KL_mean": -729.4765625, "KL/mean": -907.5888671875, "KL/rejected_KL_mean": -1085.7010498046875, "KL/std": 880.6702880859375, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 356.22454833984375, "fcm_dpo/q_t": 0.48145124316215515, "grad_norm": 15.456494331359863, "learning_rate": 2.129207286861638e-07, "logits/chosen": -0.37179362773895264, "logits/rejected": -0.37930575013160706, "logps/chosen": -794.6394653320312, "logps/ref_chosen": -65.16290283203125, "logps/ref_rejected": -87.18678283691406, "logps/rejected": -1172.887939453125, "loss": 1.3278, "margin_dpo/margin_mean": 356.22454833984375, "margin_dpo/margin_std": 1170.162841796875, "step": 405 }, { "KL/chosen_KL_mean": -589.8623046875, "KL/mean": -890.648681640625, "KL/rejected_KL_mean": -1191.43505859375, "KL/std": 858.3298950195312, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 601.5728149414062, "fcm_dpo/q_t": 0.4688172936439514, "grad_norm": 22.13282585144043, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -0.38036268949508667, "logits/rejected": -0.4102054834365845, "logps/chosen": -639.6031494140625, "logps/ref_chosen": -49.740814208984375, "logps/ref_rejected": -92.07862854003906, "logps/rejected": -1283.513671875, "loss": 1.2762, "margin_dpo/margin_mean": 601.5728759765625, "margin_dpo/margin_std": 1076.5947265625, "step": 406 }, { "KL/chosen_KL_mean": -646.1621704101562, "KL/mean": -756.427978515625, "KL/rejected_KL_mean": -866.6937255859375, "KL/std": 696.524169921875, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 220.5315399169922, "fcm_dpo/q_t": 0.48832640051841736, "grad_norm": 9.917305946350098, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -0.3698161244392395, "logits/rejected": -0.3660295903682709, "logps/chosen": -702.4928588867188, "logps/ref_chosen": -56.33069610595703, "logps/ref_rejected": -77.51209259033203, "logps/rejected": -944.205810546875, "loss": 1.3508, "margin_dpo/margin_mean": 220.5315399169922, "margin_dpo/margin_std": 968.973876953125, "step": 407 }, { "KL/chosen_KL_mean": -636.0607299804688, "KL/mean": -787.4437866210938, "KL/rejected_KL_mean": -938.826904296875, "KL/std": 689.7813720703125, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 302.76611328125, "fcm_dpo/q_t": 0.4842342138290405, "grad_norm": 18.907068252563477, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -0.36819028854370117, "logits/rejected": -0.37358710169792175, "logps/chosen": -705.8500366210938, "logps/ref_chosen": -69.789306640625, "logps/ref_rejected": -90.09693908691406, "logps/rejected": -1028.923828125, "loss": 1.3326, "margin_dpo/margin_mean": 302.7661437988281, "margin_dpo/margin_std": 921.2728881835938, "step": 408 }, { "KL/chosen_KL_mean": -648.5272216796875, "KL/mean": -830.0584716796875, "KL/rejected_KL_mean": -1011.5897216796875, "KL/std": 854.2369384765625, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 363.0625305175781, "fcm_dpo/q_t": 0.4812033176422119, "grad_norm": 13.033660888671875, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -0.34932005405426025, "logits/rejected": -0.3593984544277191, "logps/chosen": -715.8446044921875, "logps/ref_chosen": -67.31744384765625, "logps/ref_rejected": -84.904296875, "logps/rejected": -1096.4940185546875, "loss": 1.3264, "margin_dpo/margin_mean": 363.0625305175781, "margin_dpo/margin_std": 1165.644775390625, "step": 409 }, { "KL/chosen_KL_mean": -634.2596435546875, "KL/mean": -884.68310546875, "KL/rejected_KL_mean": -1135.1065673828125, "KL/std": 723.4794311523438, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 500.8468322753906, "fcm_dpo/q_t": 0.4734492003917694, "grad_norm": 18.643348693847656, "learning_rate": 2.065879555832674e-07, "logits/chosen": -0.406690776348114, "logits/rejected": -0.41948169469833374, "logps/chosen": -685.7250366210938, "logps/ref_chosen": -51.465354919433594, "logps/ref_rejected": -83.198974609375, "logps/rejected": -1218.3055419921875, "loss": 1.2898, "margin_dpo/margin_mean": 500.84686279296875, "margin_dpo/margin_std": 840.258544921875, "step": 410 }, { "KL/chosen_KL_mean": -756.8233642578125, "KL/mean": -1003.681640625, "KL/rejected_KL_mean": -1250.539794921875, "KL/std": 1031.4246826171875, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 493.71636962890625, "fcm_dpo/q_t": 0.4749707281589508, "grad_norm": 13.850335121154785, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -0.41555359959602356, "logits/rejected": -0.43157142400741577, "logps/chosen": -809.1306762695312, "logps/ref_chosen": -52.30727005004883, "logps/ref_rejected": -80.69495391845703, "logps/rejected": -1331.2347412109375, "loss": 1.3047, "margin_dpo/margin_mean": 493.71636962890625, "margin_dpo/margin_std": 1287.54248046875, "step": 411 }, { "KL/chosen_KL_mean": -895.0568237304688, "KL/mean": -1222.3822021484375, "KL/rejected_KL_mean": -1549.7076416015625, "KL/std": 1135.393310546875, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 654.65087890625, "fcm_dpo/q_t": 0.466141939163208, "grad_norm": 13.520402908325195, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -0.4506192207336426, "logits/rejected": -0.4855607748031616, "logps/chosen": -948.200927734375, "logps/ref_chosen": -53.144126892089844, "logps/ref_rejected": -100.0608139038086, "logps/rejected": -1649.7684326171875, "loss": 1.2784, "margin_dpo/margin_mean": 654.65087890625, "margin_dpo/margin_std": 1556.40234375, "step": 412 }, { "KL/chosen_KL_mean": -1110.75390625, "KL/mean": -1391.29638671875, "KL/rejected_KL_mean": -1671.839111328125, "KL/std": 1254.1436767578125, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 561.085205078125, "fcm_dpo/q_t": 0.47032126784324646, "grad_norm": 16.13272476196289, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -0.4394839406013489, "logits/rejected": -0.455962598323822, "logps/chosen": -1172.3358154296875, "logps/ref_chosen": -61.58196258544922, "logps/ref_rejected": -99.47340393066406, "logps/rejected": -1771.3125, "loss": 1.2944, "margin_dpo/margin_mean": 561.0852661132812, "margin_dpo/margin_std": 1475.930908203125, "step": 413 }, { "KL/chosen_KL_mean": -960.60693359375, "KL/mean": -1207.114013671875, "KL/rejected_KL_mean": -1453.6209716796875, "KL/std": 1134.6884765625, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 493.0139465332031, "fcm_dpo/q_t": 0.47513312101364136, "grad_norm": 23.284347534179688, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -0.43949374556541443, "logits/rejected": -0.45314937829971313, "logps/chosen": -1007.2384643554688, "logps/ref_chosen": -46.63148498535156, "logps/ref_rejected": -87.64653015136719, "logps/rejected": -1541.2674560546875, "loss": 1.3087, "margin_dpo/margin_mean": 493.013916015625, "margin_dpo/margin_std": 1478.425048828125, "step": 414 }, { "KL/chosen_KL_mean": -1096.2464599609375, "KL/mean": -1322.900634765625, "KL/rejected_KL_mean": -1549.5546875, "KL/std": 1193.9111328125, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 453.30828857421875, "fcm_dpo/q_t": 0.476369708776474, "grad_norm": 14.205986022949219, "learning_rate": 2.002837796667909e-07, "logits/chosen": -0.5072834491729736, "logits/rejected": -0.5145879983901978, "logps/chosen": -1174.86474609375, "logps/ref_chosen": -78.6182861328125, "logps/ref_rejected": -100.47752380371094, "logps/rejected": -1650.0322265625, "loss": 1.3179, "margin_dpo/margin_mean": 453.3082580566406, "margin_dpo/margin_std": 1521.6240234375, "step": 415 }, { "KL/chosen_KL_mean": -895.443115234375, "KL/mean": -1211.381103515625, "KL/rejected_KL_mean": -1527.31884765625, "KL/std": 1104.52685546875, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 631.8758544921875, "fcm_dpo/q_t": 0.4671364426612854, "grad_norm": 18.065832138061523, "learning_rate": 1.990267419549914e-07, "logits/chosen": -0.42411717772483826, "logits/rejected": -0.4482443332672119, "logps/chosen": -953.7222900390625, "logps/ref_chosen": -58.27912521362305, "logps/ref_rejected": -90.56871795654297, "logps/rejected": -1617.8875732421875, "loss": 1.2725, "margin_dpo/margin_mean": 631.8757934570312, "margin_dpo/margin_std": 1231.4012451171875, "step": 416 }, { "KL/chosen_KL_mean": -706.0615234375, "KL/mean": -973.61181640625, "KL/rejected_KL_mean": -1241.162109375, "KL/std": 916.9801025390625, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.00021450650820042938, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 535.1005859375, "fcm_dpo/q_t": 0.4720717668533325, "grad_norm": 10.751476287841797, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -0.4009503126144409, "logits/rejected": -0.4133146107196808, "logps/chosen": -756.26025390625, "logps/ref_chosen": -50.1987190246582, "logps/ref_rejected": -68.15184020996094, "logps/rejected": -1309.31396484375, "loss": 1.2872, "margin_dpo/margin_mean": 535.1005859375, "margin_dpo/margin_std": 1025.5020751953125, "step": 417 }, { "KL/chosen_KL_mean": -857.3790893554688, "KL/mean": -1115.459716796875, "KL/rejected_KL_mean": -1373.5404052734375, "KL/std": 974.062255859375, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.00038203209987841547, "fcm_dpo/delta": 0.09407737106084824, "fcm_dpo/margin": 516.1611938476562, "fcm_dpo/q_t": 0.47300809621810913, "grad_norm": 26.765777587890625, "learning_rate": 1.965167291983757e-07, "logits/chosen": -0.45634764432907104, "logits/rejected": -0.4652099609375, "logps/chosen": -939.3576049804688, "logps/ref_chosen": -81.97846984863281, "logps/ref_rejected": -104.69148254394531, "logps/rejected": -1478.23193359375, "loss": 1.2179, "margin_dpo/margin_mean": 516.1611938476562, "margin_dpo/margin_std": 1353.4468994140625, "step": 418 }, { "KL/chosen_KL_mean": -676.5275268554688, "KL/mean": -1033.2412109375, "KL/rejected_KL_mean": -1389.954833984375, "KL/std": 921.36865234375, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0005777660990133882, "fcm_dpo/delta": 0.011424558237195015, "fcm_dpo/margin": 713.4273071289062, "fcm_dpo/q_t": 0.4135735332965851, "grad_norm": 23.824268341064453, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -0.406105101108551, "logits/rejected": -0.43193238973617554, "logps/chosen": -729.4761962890625, "logps/ref_chosen": -52.948646545410156, "logps/ref_rejected": -91.58309936523438, "logps/rejected": -1481.5379638671875, "loss": 1.0996, "margin_dpo/margin_mean": 713.4273071289062, "margin_dpo/margin_std": 1061.15673828125, "step": 419 }, { "KL/chosen_KL_mean": -1244.012939453125, "KL/mean": -1279.085693359375, "KL/rejected_KL_mean": -1314.1585693359375, "KL/std": 1466.0330810546875, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.000616069242823869, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.14578247070312, "fcm_dpo/q_t": 0.47235944867134094, "grad_norm": 252.84213256835938, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -0.4455627501010895, "logits/rejected": -0.4392239451408386, "logps/chosen": -1321.78271484375, "logps/ref_chosen": -77.7699203491211, "logps/ref_rejected": -69.31985473632812, "logps/rejected": -1383.478515625, "loss": 1.5994, "margin_dpo/margin_mean": 70.14576721191406, "margin_dpo/margin_std": 2004.8504638671875, "step": 420 }, { "KL/chosen_KL_mean": -741.7689208984375, "KL/mean": -928.7274780273438, "KL/rejected_KL_mean": -1115.68603515625, "KL/std": 884.5521240234375, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.000668178778141737, "fcm_dpo/delta": 0.015629228204488754, "fcm_dpo/margin": 373.91705322265625, "fcm_dpo/q_t": 0.4435460865497589, "grad_norm": 44.24656677246094, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -0.45810630917549133, "logits/rejected": -0.4658089280128479, "logps/chosen": -795.5347900390625, "logps/ref_chosen": -53.765865325927734, "logps/ref_rejected": -89.28144836425781, "logps/rejected": -1204.967529296875, "loss": 1.2457, "margin_dpo/margin_mean": 373.91705322265625, "margin_dpo/margin_std": 1011.509765625, "step": 421 }, { "KL/chosen_KL_mean": -949.32373046875, "KL/mean": -1192.359130859375, "KL/rejected_KL_mean": -1435.394775390625, "KL/std": 1171.419677734375, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0010301879374310374, "fcm_dpo/delta": 0.06974077969789505, "fcm_dpo/margin": 486.0709533691406, "fcm_dpo/q_t": 0.4236348569393158, "grad_norm": 89.19212341308594, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -0.4476343095302582, "logits/rejected": -0.464691162109375, "logps/chosen": -1017.95751953125, "logps/ref_chosen": -68.6337661743164, "logps/ref_rejected": -87.86351013183594, "logps/rejected": -1523.2581787109375, "loss": 1.3495, "margin_dpo/margin_mean": 486.0709228515625, "margin_dpo/margin_std": 1334.6302490234375, "step": 422 }, { "KL/chosen_KL_mean": -751.098876953125, "KL/mean": -995.6851806640625, "KL/rejected_KL_mean": -1240.2713623046875, "KL/std": 885.8568115234375, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": -0.1725754588842392, "fcm_dpo/margin": 489.1725158691406, "fcm_dpo/q_t": 0.4245114326477051, "grad_norm": 22.50880241394043, "learning_rate": 1.902669377503756e-07, "logits/chosen": -0.45179301500320435, "logits/rejected": -0.46642887592315674, "logps/chosen": -806.0891723632812, "logps/ref_chosen": -54.99030303955078, "logps/ref_rejected": -86.30654907226562, "logps/rejected": -1326.577880859375, "loss": 1.2827, "margin_dpo/margin_mean": 489.1724853515625, "margin_dpo/margin_std": 1063.1331787109375, "step": 423 }, { "KL/chosen_KL_mean": -590.9034423828125, "KL/mean": -812.5848388671875, "KL/rejected_KL_mean": -1034.266357421875, "KL/std": 783.5313720703125, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 443.36285400390625, "fcm_dpo/q_t": 0.47206780314445496, "grad_norm": 10.314723014831543, "learning_rate": 1.890215699729057e-07, "logits/chosen": -0.38947632908821106, "logits/rejected": -0.3932424783706665, "logps/chosen": -646.9153442382812, "logps/ref_chosen": -56.01192092895508, "logps/ref_rejected": -66.47896575927734, "logps/rejected": -1100.745361328125, "loss": 1.2904, "margin_dpo/margin_mean": 443.36285400390625, "margin_dpo/margin_std": 945.1494750976562, "step": 424 }, { "KL/chosen_KL_mean": -649.426513671875, "KL/mean": -852.2593994140625, "KL/rejected_KL_mean": -1055.0921630859375, "KL/std": 737.5302124023438, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 405.6656188964844, "fcm_dpo/q_t": 0.4741717576980591, "grad_norm": 16.524267196655273, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -0.36731043457984924, "logits/rejected": -0.38339000940322876, "logps/chosen": -696.2955322265625, "logps/ref_chosen": -46.86899948120117, "logps/ref_rejected": -95.92545318603516, "logps/rejected": -1151.017578125, "loss": 1.3006, "margin_dpo/margin_mean": 405.6656188964844, "margin_dpo/margin_std": 980.6445922851562, "step": 425 }, { "KL/chosen_KL_mean": -496.1842041015625, "KL/mean": -661.1705322265625, "KL/rejected_KL_mean": -826.15673828125, "KL/std": 525.2192993164062, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 329.9725341796875, "fcm_dpo/q_t": 0.4789263606071472, "grad_norm": 17.888946533203125, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -0.3716368079185486, "logits/rejected": -0.363339364528656, "logps/chosen": -572.7677612304688, "logps/ref_chosen": -76.58354187011719, "logps/ref_rejected": -81.26658630371094, "logps/rejected": -907.42333984375, "loss": 1.3107, "margin_dpo/margin_mean": 329.9725646972656, "margin_dpo/margin_std": 637.1539916992188, "step": 426 }, { "KL/chosen_KL_mean": -430.772216796875, "KL/mean": -575.3130493164062, "KL/rejected_KL_mean": -719.8538818359375, "KL/std": 486.26800537109375, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 289.08160400390625, "fcm_dpo/q_t": 0.481527715921402, "grad_norm": 23.913623809814453, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -0.3612695634365082, "logits/rejected": -0.35891193151474, "logps/chosen": -495.6260986328125, "logps/ref_chosen": -64.8538818359375, "logps/ref_rejected": -78.5660171508789, "logps/rejected": -798.4198608398438, "loss": 1.3182, "margin_dpo/margin_mean": 289.08160400390625, "margin_dpo/margin_std": 542.7352294921875, "step": 427 }, { "KL/chosen_KL_mean": -599.3250732421875, "KL/mean": -861.975341796875, "KL/rejected_KL_mean": -1124.62548828125, "KL/std": 748.1722412109375, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 525.3004760742188, "fcm_dpo/q_t": 0.46697381138801575, "grad_norm": 43.41276168823242, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -0.41682755947113037, "logits/rejected": -0.4279855489730835, "logps/chosen": -661.9617919921875, "logps/ref_chosen": -62.63666534423828, "logps/ref_rejected": -103.28181457519531, "logps/rejected": -1227.9073486328125, "loss": 1.2697, "margin_dpo/margin_mean": 525.3004760742188, "margin_dpo/margin_std": 932.6864013671875, "step": 428 }, { "KL/chosen_KL_mean": -824.616943359375, "KL/mean": -973.2047119140625, "KL/rejected_KL_mean": -1121.79248046875, "KL/std": 866.7525024414062, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 297.17559814453125, "fcm_dpo/q_t": 0.48108816146850586, "grad_norm": 22.956989288330078, "learning_rate": 1.828194884925749e-07, "logits/chosen": -0.42566531896591187, "logits/rejected": -0.4174090027809143, "logps/chosen": -905.8509521484375, "logps/ref_chosen": -81.23401641845703, "logps/ref_rejected": -91.79493713378906, "logps/rejected": -1213.58740234375, "loss": 1.3307, "margin_dpo/margin_mean": 297.1755676269531, "margin_dpo/margin_std": 1031.1573486328125, "step": 429 }, { "KL/chosen_KL_mean": -615.6317138671875, "KL/mean": -830.307861328125, "KL/rejected_KL_mean": -1044.98388671875, "KL/std": 806.1045532226562, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 429.3522033691406, "fcm_dpo/q_t": 0.473649799823761, "grad_norm": 22.414127349853516, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -0.42373788356781006, "logits/rejected": -0.4414384365081787, "logps/chosen": -676.5520629882812, "logps/ref_chosen": -60.920326232910156, "logps/ref_rejected": -104.42280578613281, "logps/rejected": -1149.40673828125, "loss": 1.2974, "margin_dpo/margin_mean": 429.35223388671875, "margin_dpo/margin_std": 1028.1934814453125, "step": 430 }, { "KL/chosen_KL_mean": -606.7068481445312, "KL/mean": -882.7003173828125, "KL/rejected_KL_mean": -1158.69384765625, "KL/std": 827.6256103515625, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.00025757343973964453, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 551.987060546875, "fcm_dpo/q_t": 0.46554499864578247, "grad_norm": 22.636369705200195, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -0.41880902647972107, "logits/rejected": -0.43652844429016113, "logps/chosen": -664.0556030273438, "logps/ref_chosen": -57.34874725341797, "logps/ref_rejected": -92.84022521972656, "logps/rejected": -1251.5340576171875, "loss": 1.269, "margin_dpo/margin_mean": 551.987060546875, "margin_dpo/margin_std": 1050.994384765625, "step": 431 }, { "KL/chosen_KL_mean": -738.18017578125, "KL/mean": -1104.6925048828125, "KL/rejected_KL_mean": -1471.204833984375, "KL/std": 1035.17333984375, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.00038527188007719815, "fcm_dpo/delta": 0.06889130920171738, "fcm_dpo/margin": 733.0245971679688, "fcm_dpo/q_t": 0.4547889232635498, "grad_norm": 28.267343521118164, "learning_rate": 1.791192214186223e-07, "logits/chosen": -0.43129995465278625, "logits/rejected": -0.4455578923225403, "logps/chosen": -809.2550048828125, "logps/ref_chosen": -71.07479095458984, "logps/ref_rejected": -98.57952880859375, "logps/rejected": -1569.7843017578125, "loss": 1.1671, "margin_dpo/margin_mean": 733.0245971679688, "margin_dpo/margin_std": 1155.4307861328125, "step": 432 }, { "KL/chosen_KL_mean": -1069.6123046875, "KL/mean": -1359.947265625, "KL/rejected_KL_mean": -1650.2822265625, "KL/std": 1036.75927734375, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0006475095869973302, "fcm_dpo/delta": -0.06112983077764511, "fcm_dpo/margin": 580.669921875, "fcm_dpo/q_t": 0.4136975407600403, "grad_norm": 42.71298599243164, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -0.5129560232162476, "logits/rejected": -0.5314432382583618, "logps/chosen": -1127.885498046875, "logps/ref_chosen": -58.273193359375, "logps/ref_rejected": -95.95089721679688, "logps/rejected": -1746.233154296875, "loss": 1.2069, "margin_dpo/margin_mean": 580.669921875, "margin_dpo/margin_std": 1201.7972412109375, "step": 433 }, { "KL/chosen_KL_mean": -1070.666748046875, "KL/mean": -1364.0224609375, "KL/rejected_KL_mean": -1657.3782958984375, "KL/std": 1538.32275390625, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0002783612289931625, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 586.7115478515625, "fcm_dpo/q_t": 0.4694068133831024, "grad_norm": 28.23455810546875, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -0.5182013511657715, "logits/rejected": -0.5297871232032776, "logps/chosen": -1132.640380859375, "logps/ref_chosen": -61.97370147705078, "logps/ref_rejected": -78.49861145019531, "logps/rejected": -1735.876953125, "loss": 1.2991, "margin_dpo/margin_mean": 586.7114868164062, "margin_dpo/margin_std": 2062.91650390625, "step": 434 }, { "KL/chosen_KL_mean": -1081.22021484375, "KL/mean": -1402.046875, "KL/rejected_KL_mean": -1722.873779296875, "KL/std": 1234.45849609375, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0002783612289931625, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 641.6536865234375, "fcm_dpo/q_t": 0.45899978280067444, "grad_norm": 16.346473693847656, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -0.5801471471786499, "logits/rejected": -0.5891480445861816, "logps/chosen": -1132.72216796875, "logps/ref_chosen": -51.502052307128906, "logps/ref_rejected": -87.56689453125, "logps/rejected": -1810.440673828125, "loss": 1.2605, "margin_dpo/margin_mean": 641.6536865234375, "margin_dpo/margin_std": 1534.156494140625, "step": 435 }, { "KL/chosen_KL_mean": -1159.755126953125, "KL/mean": -1270.0465087890625, "KL/rejected_KL_mean": -1380.3380126953125, "KL/std": 917.111328125, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0002783612289931625, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 220.5829315185547, "fcm_dpo/q_t": 0.4855788052082062, "grad_norm": 49.0058708190918, "learning_rate": 1.742118314717391e-07, "logits/chosen": -0.5959898233413696, "logits/rejected": -0.5875068306922913, "logps/chosen": -1231.15869140625, "logps/ref_chosen": -71.40371704101562, "logps/ref_rejected": -82.72775268554688, "logps/rejected": -1463.065673828125, "loss": 1.3546, "margin_dpo/margin_mean": 220.58294677734375, "margin_dpo/margin_std": 1219.72509765625, "step": 436 }, { "KL/chosen_KL_mean": -1182.003173828125, "KL/mean": -1367.249755859375, "KL/rejected_KL_mean": -1552.496337890625, "KL/std": 1103.9197998046875, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0002783612289931625, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 370.4931640625, "fcm_dpo/q_t": 0.4748278260231018, "grad_norm": 45.79405212402344, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -0.5938007235527039, "logits/rejected": -0.5952749252319336, "logps/chosen": -1246.747314453125, "logps/ref_chosen": -64.7442626953125, "logps/ref_rejected": -82.04356384277344, "logps/rejected": -1634.539794921875, "loss": 1.3213, "margin_dpo/margin_mean": 370.4931640625, "margin_dpo/margin_std": 1342.3785400390625, "step": 437 }, { "KL/chosen_KL_mean": -1072.06591796875, "KL/mean": -1324.57470703125, "KL/rejected_KL_mean": -1577.08349609375, "KL/std": 1039.245849609375, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0002783612289931625, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 505.0177001953125, "fcm_dpo/q_t": 0.46574634313583374, "grad_norm": 17.366430282592773, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -0.5239421129226685, "logits/rejected": -0.5232424736022949, "logps/chosen": -1131.08447265625, "logps/ref_chosen": -59.0186653137207, "logps/ref_rejected": -83.07682800292969, "logps/rejected": -1660.160400390625, "loss": 1.2731, "margin_dpo/margin_mean": 505.0177307128906, "margin_dpo/margin_std": 1082.669677734375, "step": 438 }, { "KL/chosen_KL_mean": -940.6976318359375, "KL/mean": -1246.2998046875, "KL/rejected_KL_mean": -1551.90185546875, "KL/std": 1239.5167236328125, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.00039391464088112116, "fcm_dpo/delta": 0.034721456468105316, "fcm_dpo/margin": 611.2044067382812, "fcm_dpo/q_t": 0.4621936082839966, "grad_norm": 33.72039794921875, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -0.5249619483947754, "logits/rejected": -0.5584954023361206, "logps/chosen": -994.481689453125, "logps/ref_chosen": -53.78407669067383, "logps/ref_rejected": -83.98545837402344, "logps/rejected": -1635.887451171875, "loss": 1.2525, "margin_dpo/margin_mean": 611.2044067382812, "margin_dpo/margin_std": 1597.325439453125, "step": 439 }, { "KL/chosen_KL_mean": -993.7286376953125, "KL/mean": -1251.96337890625, "KL/rejected_KL_mean": -1510.1981201171875, "KL/std": 1111.6761474609375, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0006941439351066947, "fcm_dpo/delta": 0.0925978422164917, "fcm_dpo/margin": 516.469482421875, "fcm_dpo/q_t": 0.4560081958770752, "grad_norm": 44.43145751953125, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -0.5849356651306152, "logits/rejected": -0.5931211709976196, "logps/chosen": -1072.29541015625, "logps/ref_chosen": -78.56671905517578, "logps/ref_rejected": -96.49775695800781, "logps/rejected": -1606.6959228515625, "loss": 1.2332, "margin_dpo/margin_mean": 516.469482421875, "margin_dpo/margin_std": 1347.8345947265625, "step": 440 }, { "KL/chosen_KL_mean": -1073.631103515625, "KL/mean": -1258.8494873046875, "KL/rejected_KL_mean": -1444.06787109375, "KL/std": 1060.948486328125, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0010014603612944484, "fcm_dpo/delta": -0.04774302989244461, "fcm_dpo/margin": 370.4366455078125, "fcm_dpo/q_t": 0.40342986583709717, "grad_norm": 167.0828399658203, "learning_rate": 1.681227682404166e-07, "logits/chosen": -0.5610836744308472, "logits/rejected": -0.566421389579773, "logps/chosen": -1134.45556640625, "logps/ref_chosen": -60.824440002441406, "logps/ref_rejected": -96.47080993652344, "logps/rejected": -1540.5386962890625, "loss": 1.3984, "margin_dpo/margin_mean": 370.4366455078125, "margin_dpo/margin_std": 1346.23828125, "step": 441 }, { "KL/chosen_KL_mean": -677.8002319335938, "KL/mean": -963.8506469726562, "KL/rejected_KL_mean": -1249.901123046875, "KL/std": 910.0428466796875, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0006988396635279059, "fcm_dpo/delta": -0.21735386550426483, "fcm_dpo/margin": 572.1007080078125, "fcm_dpo/q_t": 0.39990121126174927, "grad_norm": 58.9281120300293, "learning_rate": 1.669113001300851e-07, "logits/chosen": -0.5104144215583801, "logits/rejected": -0.5223550796508789, "logps/chosen": -724.8114624023438, "logps/ref_chosen": -47.01121520996094, "logps/ref_rejected": -76.53926086425781, "logps/rejected": -1326.4403076171875, "loss": 1.3103, "margin_dpo/margin_mean": 572.1007080078125, "margin_dpo/margin_std": 1180.846923828125, "step": 442 }, { "KL/chosen_KL_mean": -682.045654296875, "KL/mean": -812.193359375, "KL/rejected_KL_mean": -942.3411254882812, "KL/std": 811.3766479492188, "epoch": 0.6505139500734214, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 260.2955627441406, "fcm_dpo/q_t": 0.4954499900341034, "grad_norm": 5.396309852600098, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -0.477186918258667, "logits/rejected": -0.4765312373638153, "logps/chosen": -753.318603515625, "logps/ref_chosen": -71.27301788330078, "logps/ref_rejected": -86.679931640625, "logps/rejected": -1029.0211181640625, "loss": 1.3694, "margin_dpo/margin_mean": 260.2955322265625, "margin_dpo/margin_std": 990.7225341796875, "step": 443 }, { "KL/chosen_KL_mean": -610.6414184570312, "KL/mean": -837.4876708984375, "KL/rejected_KL_mean": -1064.3338623046875, "KL/std": 821.0623779296875, "epoch": 0.6519823788546255, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 453.6924743652344, "fcm_dpo/q_t": 0.4920549988746643, "grad_norm": 5.598912239074707, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -0.42270809412002563, "logits/rejected": -0.42965829372406006, "logps/chosen": -667.8551025390625, "logps/ref_chosen": -57.213706970214844, "logps/ref_rejected": -97.25489807128906, "logps/rejected": -1161.5887451171875, "loss": 1.3558, "margin_dpo/margin_mean": 453.69244384765625, "margin_dpo/margin_std": 947.6956787109375, "step": 444 }, { "KL/chosen_KL_mean": -393.3489685058594, "KL/mean": -593.373291015625, "KL/rejected_KL_mean": -793.3975830078125, "KL/std": 562.3740234375, "epoch": 0.6534508076358296, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 400.0485534667969, "fcm_dpo/q_t": 0.4929931163787842, "grad_norm": 7.666253089904785, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -0.38615715503692627, "logits/rejected": -0.38465434312820435, "logps/chosen": -460.6487731933594, "logps/ref_chosen": -67.29979705810547, "logps/ref_rejected": -92.68267059326172, "logps/rejected": -886.0802001953125, "loss": 1.3592, "margin_dpo/margin_mean": 400.048583984375, "margin_dpo/margin_std": 728.7763061523438, "step": 445 }, { "KL/chosen_KL_mean": -392.439208984375, "KL/mean": -607.7997436523438, "KL/rejected_KL_mean": -823.1602783203125, "KL/std": 526.53857421875, "epoch": 0.6549192364170338, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 430.7210693359375, "fcm_dpo/q_t": 0.4924466013908386, "grad_norm": 7.6744914054870605, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -0.3997993469238281, "logits/rejected": -0.41034257411956787, "logps/chosen": -451.5377197265625, "logps/ref_chosen": -59.098487854003906, "logps/ref_rejected": -101.26419067382812, "logps/rejected": -924.4244384765625, "loss": 1.3568, "margin_dpo/margin_mean": 430.7210693359375, "margin_dpo/margin_std": 591.9511108398438, "step": 446 }, { "KL/chosen_KL_mean": -391.9793701171875, "KL/mean": -633.8006591796875, "KL/rejected_KL_mean": -875.6220703125, "KL/std": 740.5856323242188, "epoch": 0.6563876651982379, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 483.64263916015625, "fcm_dpo/q_t": 0.49153846502304077, "grad_norm": 7.994978427886963, "learning_rate": 1.608874379754465e-07, "logits/chosen": -0.434841513633728, "logits/rejected": -0.4521018862724304, "logps/chosen": -448.0547180175781, "logps/ref_chosen": -56.07533264160156, "logps/ref_rejected": -98.69475555419922, "logps/rejected": -974.3168334960938, "loss": 1.3536, "margin_dpo/margin_mean": 483.64263916015625, "margin_dpo/margin_std": 850.4795532226562, "step": 447 }, { "KL/chosen_KL_mean": -371.1406555175781, "KL/mean": -603.8892822265625, "KL/rejected_KL_mean": -836.6378784179688, "KL/std": 571.3126220703125, "epoch": 0.657856093979442, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 465.4972229003906, "fcm_dpo/q_t": 0.49184244871139526, "grad_norm": 7.214312553405762, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -0.4187026917934418, "logits/rejected": -0.4283428192138672, "logps/chosen": -431.1445007324219, "logps/ref_chosen": -60.00384521484375, "logps/ref_rejected": -102.26465606689453, "logps/rejected": -938.9025268554688, "loss": 1.3545, "margin_dpo/margin_mean": 465.4972229003906, "margin_dpo/margin_std": 671.6864013671875, "step": 448 }, { "KL/chosen_KL_mean": -494.8532409667969, "KL/mean": -674.86083984375, "KL/rejected_KL_mean": -854.8682861328125, "KL/std": 626.908203125, "epoch": 0.6593245227606461, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 360.01507568359375, "fcm_dpo/q_t": 0.4936937391757965, "grad_norm": 8.744072914123535, "learning_rate": 1.584941086944423e-07, "logits/chosen": -0.43162280321121216, "logits/rejected": -0.4255381226539612, "logps/chosen": -562.3798828125, "logps/ref_chosen": -67.52661895751953, "logps/ref_rejected": -88.59690856933594, "logps/rejected": -943.4652099609375, "loss": 1.3619, "margin_dpo/margin_mean": 360.01507568359375, "margin_dpo/margin_std": 755.4654541015625, "step": 449 }, { "KL/chosen_KL_mean": -329.07965087890625, "KL/mean": -557.316650390625, "KL/rejected_KL_mean": -785.5537109375, "KL/std": 504.0699462890625, "epoch": 0.6607929515418502, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 456.47406005859375, "fcm_dpo/q_t": 0.49199458956718445, "grad_norm": 10.857216835021973, "learning_rate": 1.573010452010098e-07, "logits/chosen": -0.38689732551574707, "logits/rejected": -0.40724536776542664, "logps/chosen": -386.1877746582031, "logps/ref_chosen": -57.10811996459961, "logps/ref_rejected": -102.75494384765625, "logps/rejected": -888.3086547851562, "loss": 1.3549, "margin_dpo/margin_mean": 456.47406005859375, "margin_dpo/margin_std": 548.2633666992188, "step": 450 }, { "KL/chosen_KL_mean": -549.2373046875, "KL/mean": -656.2725830078125, "KL/rejected_KL_mean": -763.307861328125, "KL/std": 700.226318359375, "epoch": 0.6622613803230544, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 214.07054138183594, "fcm_dpo/q_t": 0.4962322413921356, "grad_norm": 6.819939613342285, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -0.44771769642829895, "logits/rejected": -0.43964436650276184, "logps/chosen": -607.7061767578125, "logps/ref_chosen": -58.46883010864258, "logps/ref_rejected": -72.92941284179688, "logps/rejected": -836.2373046875, "loss": 1.3725, "margin_dpo/margin_mean": 214.07054138183594, "margin_dpo/margin_std": 986.6764526367188, "step": 451 }, { "KL/chosen_KL_mean": -501.6789855957031, "KL/mean": -689.10693359375, "KL/rejected_KL_mean": -876.5348510742188, "KL/std": 670.5494384765625, "epoch": 0.6637298091042585, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 374.8558654785156, "fcm_dpo/q_t": 0.4934268891811371, "grad_norm": 3.6413915157318115, "learning_rate": 1.549222776991186e-07, "logits/chosen": -0.4632149636745453, "logits/rejected": -0.47757190465927124, "logps/chosen": -552.069580078125, "logps/ref_chosen": -50.39055252075195, "logps/ref_rejected": -97.77142333984375, "logps/rejected": -974.3062744140625, "loss": 1.3608, "margin_dpo/margin_mean": 374.85589599609375, "margin_dpo/margin_std": 725.793701171875, "step": 452 }, { "KL/chosen_KL_mean": -527.853515625, "KL/mean": -772.795654296875, "KL/rejected_KL_mean": -1017.73779296875, "KL/std": 713.8123779296875, "epoch": 0.6651982378854625, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 489.8843078613281, "fcm_dpo/q_t": 0.4914173185825348, "grad_norm": 6.951712608337402, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -0.4904746413230896, "logits/rejected": -0.4950370788574219, "logps/chosen": -585.568359375, "logps/ref_chosen": -57.71485137939453, "logps/ref_rejected": -82.20741271972656, "logps/rejected": -1099.9451904296875, "loss": 1.3532, "margin_dpo/margin_mean": 489.88427734375, "margin_dpo/margin_std": 879.7352294921875, "step": 453 }, { "KL/chosen_KL_mean": -756.3292236328125, "KL/mean": -986.6734619140625, "KL/rejected_KL_mean": -1217.017578125, "KL/std": 858.3633422851562, "epoch": 0.6666666666666666, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 460.6884765625, "fcm_dpo/q_t": 0.4919373393058777, "grad_norm": 4.276784896850586, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -0.48667120933532715, "logits/rejected": -0.4960416555404663, "logps/chosen": -817.27490234375, "logps/ref_chosen": -60.945648193359375, "logps/ref_rejected": -84.95079040527344, "logps/rejected": -1301.968505859375, "loss": 1.3556, "margin_dpo/margin_mean": 460.6884765625, "margin_dpo/margin_std": 1064.3143310546875, "step": 454 }, { "KL/chosen_KL_mean": -746.62548828125, "KL/mean": -1129.1025390625, "KL/rejected_KL_mean": -1511.579345703125, "KL/std": 1107.7918701171875, "epoch": 0.6681350954478708, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 764.9539794921875, "fcm_dpo/q_t": 0.4866509437561035, "grad_norm": 7.09210729598999, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -0.5352227687835693, "logits/rejected": -0.5812809467315674, "logps/chosen": -791.51220703125, "logps/ref_chosen": -44.88671112060547, "logps/ref_rejected": -115.30147552490234, "logps/rejected": -1626.880859375, "loss": 1.3358, "margin_dpo/margin_mean": 764.953857421875, "margin_dpo/margin_std": 1381.763671875, "step": 455 }, { "KL/chosen_KL_mean": -1079.772216796875, "KL/mean": -1386.8994140625, "KL/rejected_KL_mean": -1694.026611328125, "KL/std": 1328.23046875, "epoch": 0.6696035242290749, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 614.2544555664062, "fcm_dpo/q_t": 0.48927462100982666, "grad_norm": 4.1105146408081055, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -0.5510225296020508, "logits/rejected": -0.5779255628585815, "logps/chosen": -1136.8089599609375, "logps/ref_chosen": -57.036781311035156, "logps/ref_rejected": -105.21784210205078, "logps/rejected": -1799.244384765625, "loss": 1.3461, "margin_dpo/margin_mean": 614.25439453125, "margin_dpo/margin_std": 1356.8857421875, "step": 456 }, { "KL/chosen_KL_mean": -928.1408081054688, "KL/mean": -1333.076416015625, "KL/rejected_KL_mean": -1738.0120849609375, "KL/std": 1123.67529296875, "epoch": 0.671071953010279, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 809.871337890625, "fcm_dpo/q_t": 0.4858279526233673, "grad_norm": 6.433835506439209, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -0.6030972003936768, "logits/rejected": -0.6290802359580994, "logps/chosen": -982.38330078125, "logps/ref_chosen": -54.24253845214844, "logps/ref_rejected": -85.10956573486328, "logps/rejected": -1823.12158203125, "loss": 1.3325, "margin_dpo/margin_mean": 809.871337890625, "margin_dpo/margin_std": 1329.330810546875, "step": 457 }, { "KL/chosen_KL_mean": -1218.3466796875, "KL/mean": -1525.91845703125, "KL/rejected_KL_mean": -1833.4901123046875, "KL/std": 1092.089599609375, "epoch": 0.6725403817914831, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 615.1434326171875, "fcm_dpo/q_t": 0.4892372488975525, "grad_norm": 5.473250865936279, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -0.6052824854850769, "logits/rejected": -0.6187411546707153, "logps/chosen": -1273.755615234375, "logps/ref_chosen": -55.40888214111328, "logps/ref_rejected": -97.68325805664062, "logps/rejected": -1931.17333984375, "loss": 1.3456, "margin_dpo/margin_mean": 615.1434326171875, "margin_dpo/margin_std": 1242.5, "step": 458 }, { "KL/chosen_KL_mean": -1342.32373046875, "KL/mean": -1710.5882568359375, "KL/rejected_KL_mean": -2078.8525390625, "KL/std": 1349.3956298828125, "epoch": 0.6740088105726872, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 736.52880859375, "fcm_dpo/q_t": 0.48712393641471863, "grad_norm": 6.021612644195557, "learning_rate": 1.466771464027316e-07, "logits/chosen": -0.6226590871810913, "logits/rejected": -0.6531392335891724, "logps/chosen": -1388.88134765625, "logps/ref_chosen": -46.55748748779297, "logps/ref_rejected": -86.16854095458984, "logps/rejected": -2165.02099609375, "loss": 1.339, "margin_dpo/margin_mean": 736.52880859375, "margin_dpo/margin_std": 1749.3363037109375, "step": 459 }, { "KL/chosen_KL_mean": -1793.916259765625, "KL/mean": -2120.57568359375, "KL/rejected_KL_mean": -2447.23486328125, "KL/std": 1669.0478515625, "epoch": 0.6754772393538914, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 653.318603515625, "fcm_dpo/q_t": 0.4886649549007416, "grad_norm": 8.884930610656738, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -0.6757256984710693, "logits/rejected": -0.7148680686950684, "logps/chosen": -1845.55126953125, "logps/ref_chosen": -51.63489532470703, "logps/ref_rejected": -104.11935424804688, "logps/rejected": -2551.3544921875, "loss": 1.3459, "margin_dpo/margin_mean": 653.3186645507812, "margin_dpo/margin_std": 1995.814697265625, "step": 460 }, { "KL/chosen_KL_mean": -1595.17822265625, "KL/mean": -1813.5162353515625, "KL/rejected_KL_mean": -2031.854248046875, "KL/std": 1255.39697265625, "epoch": 0.6769456681350955, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 436.6760559082031, "fcm_dpo/q_t": 0.49233686923980713, "grad_norm": 9.014615058898926, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -0.7007662057876587, "logits/rejected": -0.7139623761177063, "logps/chosen": -1650.360107421875, "logps/ref_chosen": -55.18195724487305, "logps/ref_rejected": -86.47689819335938, "logps/rejected": -2118.3310546875, "loss": 1.359, "margin_dpo/margin_mean": 436.67608642578125, "margin_dpo/margin_std": 1588.939453125, "step": 461 }, { "KL/chosen_KL_mean": -1918.244384765625, "KL/mean": -2012.81298828125, "KL/rejected_KL_mean": -2107.3818359375, "KL/std": 1555.3056640625, "epoch": 0.6784140969162996, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 189.1370086669922, "fcm_dpo/q_t": 0.4966925382614136, "grad_norm": 25.653871536254883, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -0.6912171244621277, "logits/rejected": -0.6754558086395264, "logps/chosen": -1988.172607421875, "logps/ref_chosen": -69.92803192138672, "logps/ref_rejected": -78.84111022949219, "logps/rejected": -2186.22265625, "loss": 1.3773, "margin_dpo/margin_mean": 189.13702392578125, "margin_dpo/margin_std": 1842.75927734375, "step": 462 }, { "KL/chosen_KL_mean": -1629.407958984375, "KL/mean": -1926.924072265625, "KL/rejected_KL_mean": -2224.440185546875, "KL/std": 1563.47412109375, "epoch": 0.6798825256975036, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 595.0322265625, "fcm_dpo/q_t": 0.48969152569770813, "grad_norm": 10.694584846496582, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -0.6257902383804321, "logits/rejected": -0.6413998603820801, "logps/chosen": -1684.682373046875, "logps/ref_chosen": -55.27437210083008, "logps/ref_rejected": -89.02497863769531, "logps/rejected": -2313.46533203125, "loss": 1.3502, "margin_dpo/margin_mean": 595.0322875976562, "margin_dpo/margin_std": 2058.953125, "step": 463 }, { "KL/chosen_KL_mean": -1547.507080078125, "KL/mean": -2104.552001953125, "KL/rejected_KL_mean": -2661.59716796875, "KL/std": 1696.195068359375, "epoch": 0.6813509544787077, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1114.0902099609375, "fcm_dpo/q_t": 0.48058074712753296, "grad_norm": 6.717080593109131, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -0.6039457321166992, "logits/rejected": -0.6586399078369141, "logps/chosen": -1598.4193115234375, "logps/ref_chosen": -50.91230010986328, "logps/ref_rejected": -102.4893798828125, "logps/rejected": -2764.08642578125, "loss": 1.3144, "margin_dpo/margin_mean": 1114.0902099609375, "margin_dpo/margin_std": 1911.744140625, "step": 464 }, { "KL/chosen_KL_mean": -1389.27978515625, "KL/mean": -1775.259765625, "KL/rejected_KL_mean": -2161.23974609375, "KL/std": 1636.043212890625, "epoch": 0.6828193832599119, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 771.9598388671875, "fcm_dpo/q_t": 0.4866272807121277, "grad_norm": 5.775012016296387, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -0.5333804488182068, "logits/rejected": -0.556584358215332, "logps/chosen": -1449.396728515625, "logps/ref_chosen": -60.116851806640625, "logps/ref_rejected": -113.94602966308594, "logps/rejected": -2275.185791015625, "loss": 1.3381, "margin_dpo/margin_mean": 771.9598388671875, "margin_dpo/margin_std": 1992.7213134765625, "step": 465 }, { "KL/chosen_KL_mean": -1304.01220703125, "KL/mean": -1642.657470703125, "KL/rejected_KL_mean": -1981.302734375, "KL/std": 1344.09765625, "epoch": 0.684287812041116, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 677.29052734375, "fcm_dpo/q_t": 0.4881631135940552, "grad_norm": 4.701225757598877, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -0.5309747457504272, "logits/rejected": -0.5521206855773926, "logps/chosen": -1356.9332275390625, "logps/ref_chosen": -52.920921325683594, "logps/ref_rejected": -90.3154296875, "logps/rejected": -2071.6181640625, "loss": 1.343, "margin_dpo/margin_mean": 677.29052734375, "margin_dpo/margin_std": 1733.6058349609375, "step": 466 }, { "KL/chosen_KL_mean": -2066.13720703125, "KL/mean": -2241.6923828125, "KL/rejected_KL_mean": -2417.24755859375, "KL/std": 1884.09375, "epoch": 0.6857562408223201, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 351.11004638671875, "fcm_dpo/q_t": 0.4938202500343323, "grad_norm": 13.84892463684082, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -0.5912429690361023, "logits/rejected": -0.5880587100982666, "logps/chosen": -2144.85302734375, "logps/ref_chosen": -78.7158203125, "logps/ref_rejected": -102.86019897460938, "logps/rejected": -2520.10791015625, "loss": 1.3708, "margin_dpo/margin_mean": 351.1100158691406, "margin_dpo/margin_std": 2587.14794921875, "step": 467 }, { "KL/chosen_KL_mean": -1504.03369140625, "KL/mean": -1898.8018798828125, "KL/rejected_KL_mean": -2293.570068359375, "KL/std": 1620.427001953125, "epoch": 0.6872246696035242, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 789.5361328125, "fcm_dpo/q_t": 0.48619183897972107, "grad_norm": 10.929081916809082, "learning_rate": 1.362737437810114e-07, "logits/chosen": -0.5442489385604858, "logits/rejected": -0.5691794753074646, "logps/chosen": -1573.9691162109375, "logps/ref_chosen": -69.93536376953125, "logps/ref_rejected": -101.02880859375, "logps/rejected": -2394.598876953125, "loss": 1.3377, "margin_dpo/margin_mean": 789.5361938476562, "margin_dpo/margin_std": 2186.982666015625, "step": 468 }, { "KL/chosen_KL_mean": -1296.493896484375, "KL/mean": -1764.9542236328125, "KL/rejected_KL_mean": -2233.41455078125, "KL/std": 1456.62353515625, "epoch": 0.6886930983847284, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 936.920654296875, "fcm_dpo/q_t": 0.4836467504501343, "grad_norm": 10.054937362670898, "learning_rate": 1.351323902551631e-07, "logits/chosen": -0.5154561996459961, "logits/rejected": -0.545080840587616, "logps/chosen": -1364.61865234375, "logps/ref_chosen": -68.12469482421875, "logps/ref_rejected": -104.78640747070312, "logps/rejected": -2338.201171875, "loss": 1.3254, "margin_dpo/margin_mean": 936.920654296875, "margin_dpo/margin_std": 1700.6353759765625, "step": 469 }, { "KL/chosen_KL_mean": -1085.716552734375, "KL/mean": -1395.3118896484375, "KL/rejected_KL_mean": -1704.9072265625, "KL/std": 1203.486328125, "epoch": 0.6901615271659325, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 619.190673828125, "fcm_dpo/q_t": 0.4891727566719055, "grad_norm": 4.784751892089844, "learning_rate": 1.339940635976592e-07, "logits/chosen": -0.4637961983680725, "logits/rejected": -0.4888746738433838, "logps/chosen": -1129.508544921875, "logps/ref_chosen": -43.791927337646484, "logps/ref_rejected": -82.70285034179688, "logps/rejected": -1787.610107421875, "loss": 1.3458, "margin_dpo/margin_mean": 619.190673828125, "margin_dpo/margin_std": 1384.6240234375, "step": 470 }, { "KL/chosen_KL_mean": -1458.683837890625, "KL/mean": -1615.782958984375, "KL/rejected_KL_mean": -1772.8822021484375, "KL/std": 1343.892578125, "epoch": 0.6916299559471366, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 314.1982727050781, "fcm_dpo/q_t": 0.4945109486579895, "grad_norm": 14.473358154296875, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -0.5264901518821716, "logits/rejected": -0.5227891206741333, "logps/chosen": -1522.0234375, "logps/ref_chosen": -63.33952331542969, "logps/ref_rejected": -83.61048126220703, "logps/rejected": -1856.49267578125, "loss": 1.3675, "margin_dpo/margin_mean": 314.19830322265625, "margin_dpo/margin_std": 1578.263671875, "step": 471 }, { "KL/chosen_KL_mean": -1396.530029296875, "KL/mean": -1693.50439453125, "KL/rejected_KL_mean": -1990.4786376953125, "KL/std": 1501.013671875, "epoch": 0.6930983847283406, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 593.9485473632812, "fcm_dpo/q_t": 0.4896436929702759, "grad_norm": 5.583034992218018, "learning_rate": 1.317266107909975e-07, "logits/chosen": -0.5575324296951294, "logits/rejected": -0.5637389421463013, "logps/chosen": -1480.1962890625, "logps/ref_chosen": -83.66610717773438, "logps/ref_rejected": -117.20919799804688, "logps/rejected": -2107.68798828125, "loss": 1.3496, "margin_dpo/margin_mean": 593.9485473632812, "margin_dpo/margin_std": 1845.80224609375, "step": 472 }, { "KL/chosen_KL_mean": -1678.685791015625, "KL/mean": -1702.64501953125, "KL/rejected_KL_mean": -1726.6043701171875, "KL/std": 1605.0902099609375, "epoch": 0.6945668135095447, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 47.91863250732422, "fcm_dpo/q_t": 0.49903565645217896, "grad_norm": 10.380570411682129, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -0.5147888660430908, "logits/rejected": -0.5062965154647827, "logps/chosen": -1742.1827392578125, "logps/ref_chosen": -63.49696731567383, "logps/ref_rejected": -81.14657592773438, "logps/rejected": -1807.7509765625, "loss": 1.389, "margin_dpo/margin_mean": 47.918617248535156, "margin_dpo/margin_std": 2152.85302734375, "step": 473 }, { "KL/chosen_KL_mean": -1045.2838134765625, "KL/mean": -1396.827880859375, "KL/rejected_KL_mean": -1748.3720703125, "KL/std": 1296.06787109375, "epoch": 0.6960352422907489, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 703.088134765625, "fcm_dpo/q_t": 0.4877893924713135, "grad_norm": 12.54921817779541, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -0.44941064715385437, "logits/rejected": -0.465454638004303, "logps/chosen": -1097.895751953125, "logps/ref_chosen": -52.6119384765625, "logps/ref_rejected": -90.08041381835938, "logps/rejected": -1838.452392578125, "loss": 1.3413, "margin_dpo/margin_mean": 703.088134765625, "margin_dpo/margin_std": 1566.402099609375, "step": 474 }, { "KL/chosen_KL_mean": -696.1770629882812, "KL/mean": -1096.912109375, "KL/rejected_KL_mean": -1497.64697265625, "KL/std": 1150.5213623046875, "epoch": 0.697503671071953, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 801.469970703125, "fcm_dpo/q_t": 0.48600390553474426, "grad_norm": 16.383455276489258, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -0.43204575777053833, "logits/rejected": -0.46623319387435913, "logps/chosen": -738.6722412109375, "logps/ref_chosen": -42.49519348144531, "logps/ref_rejected": -90.06294250488281, "logps/rejected": -1587.7099609375, "loss": 1.3334, "margin_dpo/margin_mean": 801.469970703125, "margin_dpo/margin_std": 1452.920654296875, "step": 475 }, { "KL/chosen_KL_mean": -922.133056640625, "KL/mean": -1211.9774169921875, "KL/rejected_KL_mean": -1501.82177734375, "KL/std": 1047.379150390625, "epoch": 0.6989720998531571, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 579.6885986328125, "fcm_dpo/q_t": 0.4898582696914673, "grad_norm": 7.450405597686768, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -0.4594970941543579, "logits/rejected": -0.4757981300354004, "logps/chosen": -965.0824584960938, "logps/ref_chosen": -42.94938278198242, "logps/ref_rejected": -73.71023559570312, "logps/rejected": -1575.531982421875, "loss": 1.3477, "margin_dpo/margin_mean": 579.6885986328125, "margin_dpo/margin_std": 1162.8359375, "step": 476 }, { "KL/chosen_KL_mean": -1014.1900634765625, "KL/mean": -1202.794189453125, "KL/rejected_KL_mean": -1391.398193359375, "KL/std": 987.1451416015625, "epoch": 0.7004405286343612, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 377.20819091796875, "fcm_dpo/q_t": 0.4933883547782898, "grad_norm": 4.9775261878967285, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -0.48688119649887085, "logits/rejected": -0.47269320487976074, "logps/chosen": -1084.962646484375, "logps/ref_chosen": -70.77261352539062, "logps/ref_rejected": -76.13737487792969, "logps/rejected": -1467.53564453125, "loss": 1.3615, "margin_dpo/margin_mean": 377.20819091796875, "margin_dpo/margin_std": 1102.049560546875, "step": 477 }, { "KL/chosen_KL_mean": -741.496337890625, "KL/mean": -1104.615966796875, "KL/rejected_KL_mean": -1467.7357177734375, "KL/std": 1056.3428955078125, "epoch": 0.7019089574155654, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 726.2393798828125, "fcm_dpo/q_t": 0.48729372024536133, "grad_norm": 13.8693208694458, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.4414135813713074, "logits/rejected": -0.4705617129802704, "logps/chosen": -782.9368286132812, "logps/ref_chosen": -41.440513610839844, "logps/ref_rejected": -85.36196899414062, "logps/rejected": -1553.09765625, "loss": 1.3379, "margin_dpo/margin_mean": 726.2393798828125, "margin_dpo/margin_std": 1235.259033203125, "step": 478 }, { "KL/chosen_KL_mean": -1152.8778076171875, "KL/mean": -1485.199951171875, "KL/rejected_KL_mean": -1817.52197265625, "KL/std": 1490.449951171875, "epoch": 0.7033773861967695, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 664.6441650390625, "fcm_dpo/q_t": 0.4884074628353119, "grad_norm": 4.815835952758789, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -0.4350011944770813, "logits/rejected": -0.4591215252876282, "logps/chosen": -1206.7857666015625, "logps/ref_chosen": -53.907920837402344, "logps/ref_rejected": -95.1163330078125, "logps/rejected": -1912.638427734375, "loss": 1.343, "margin_dpo/margin_mean": 664.6441650390625, "margin_dpo/margin_std": 1489.7821044921875, "step": 479 }, { "KL/chosen_KL_mean": -1410.0196533203125, "KL/mean": -1525.3935546875, "KL/rejected_KL_mean": -1640.767578125, "KL/std": 1323.5804443359375, "epoch": 0.7048458149779736, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 230.74789428710938, "fcm_dpo/q_t": 0.4959403872489929, "grad_norm": 8.32140064239502, "learning_rate": 1.227838333989088e-07, "logits/chosen": -0.5214420557022095, "logits/rejected": -0.5171930193901062, "logps/chosen": -1468.702392578125, "logps/ref_chosen": -58.682701110839844, "logps/ref_rejected": -82.93248748779297, "logps/rejected": -1723.7000732421875, "loss": 1.3738, "margin_dpo/margin_mean": 230.7479248046875, "margin_dpo/margin_std": 1702.75048828125, "step": 480 }, { "KL/chosen_KL_mean": -1059.987548828125, "KL/mean": -1491.16796875, "KL/rejected_KL_mean": -1922.3486328125, "KL/std": 1393.0323486328125, "epoch": 0.7063142437591777, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 862.3609619140625, "fcm_dpo/q_t": 0.484965980052948, "grad_norm": 4.985651969909668, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -0.5206190347671509, "logits/rejected": -0.562711238861084, "logps/chosen": -1114.9517822265625, "logps/ref_chosen": -54.964271545410156, "logps/ref_rejected": -92.42044067382812, "logps/rejected": -2014.76904296875, "loss": 1.33, "margin_dpo/margin_mean": 862.3610229492188, "margin_dpo/margin_std": 1640.7061767578125, "step": 481 }, { "KL/chosen_KL_mean": -1543.61083984375, "KL/mean": -1666.1243896484375, "KL/rejected_KL_mean": -1788.637939453125, "KL/std": 1566.256591796875, "epoch": 0.7077826725403817, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 245.027099609375, "fcm_dpo/q_t": 0.49571603536605835, "grad_norm": 9.695770263671875, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -0.5083905458450317, "logits/rejected": -0.5077981948852539, "logps/chosen": -1611.164306640625, "logps/ref_chosen": -67.553466796875, "logps/ref_rejected": -87.58953857421875, "logps/rejected": -1876.2275390625, "loss": 1.374, "margin_dpo/margin_mean": 245.02706909179688, "margin_dpo/margin_std": 1968.4217529296875, "step": 482 }, { "KL/chosen_KL_mean": -1194.04248046875, "KL/mean": -1479.513671875, "KL/rejected_KL_mean": -1764.9847412109375, "KL/std": 1215.059326171875, "epoch": 0.7092511013215859, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 570.9423217773438, "fcm_dpo/q_t": 0.4899991750717163, "grad_norm": 7.4929304122924805, "learning_rate": 1.194847979251979e-07, "logits/chosen": -0.5671132206916809, "logits/rejected": -0.5771450996398926, "logps/chosen": -1257.372314453125, "logps/ref_chosen": -63.32981872558594, "logps/ref_rejected": -95.78697204589844, "logps/rejected": -1860.771728515625, "loss": 1.3497, "margin_dpo/margin_mean": 570.9423217773438, "margin_dpo/margin_std": 1546.977783203125, "step": 483 }, { "KL/chosen_KL_mean": -1078.635498046875, "KL/mean": -1429.60009765625, "KL/rejected_KL_mean": -1780.564697265625, "KL/std": 1515.4169921875, "epoch": 0.71071953010279, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 701.9291381835938, "fcm_dpo/q_t": 0.48787161707878113, "grad_norm": 6.547210216522217, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -0.4985977113246918, "logits/rejected": -0.5153236389160156, "logps/chosen": -1137.773681640625, "logps/ref_chosen": -59.13812255859375, "logps/ref_rejected": -84.37144470214844, "logps/rejected": -1864.93603515625, "loss": 1.3425, "margin_dpo/margin_mean": 701.9291381835938, "margin_dpo/margin_std": 1918.406494140625, "step": 484 }, { "KL/chosen_KL_mean": -998.8741455078125, "KL/mean": -1385.538818359375, "KL/rejected_KL_mean": -1772.2037353515625, "KL/std": 1393.5875244140625, "epoch": 0.7121879588839941, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 773.32958984375, "fcm_dpo/q_t": 0.4865483045578003, "grad_norm": 6.182162761688232, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -0.5134660005569458, "logits/rejected": -0.5451745390892029, "logps/chosen": -1057.7236328125, "logps/ref_chosen": -58.849571228027344, "logps/ref_rejected": -103.36408233642578, "logps/rejected": -1875.56787109375, "loss": 1.3371, "margin_dpo/margin_mean": 773.32958984375, "margin_dpo/margin_std": 1857.324462890625, "step": 485 }, { "KL/chosen_KL_mean": -1066.927734375, "KL/mean": -1443.706787109375, "KL/rejected_KL_mean": -1820.4857177734375, "KL/std": 1382.1572265625, "epoch": 0.7136563876651982, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 753.5579833984375, "fcm_dpo/q_t": 0.48684683442115784, "grad_norm": 4.5740437507629395, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -0.46131497621536255, "logits/rejected": -0.49075794219970703, "logps/chosen": -1122.1875, "logps/ref_chosen": -55.25966262817383, "logps/ref_rejected": -92.13936614990234, "logps/rejected": -1912.625, "loss": 1.3381, "margin_dpo/margin_mean": 753.5579833984375, "margin_dpo/margin_std": 1785.576171875, "step": 486 }, { "KL/chosen_KL_mean": -1072.845458984375, "KL/mean": -1368.38525390625, "KL/rejected_KL_mean": -1663.9249267578125, "KL/std": 1040.571533203125, "epoch": 0.7151248164464024, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 591.0794677734375, "fcm_dpo/q_t": 0.4896491765975952, "grad_norm": 9.592281341552734, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -0.47907352447509766, "logits/rejected": -0.4934772253036499, "logps/chosen": -1125.90869140625, "logps/ref_chosen": -53.06330871582031, "logps/ref_rejected": -92.41883087158203, "logps/rejected": -1756.34375, "loss": 1.3469, "margin_dpo/margin_mean": 591.0795288085938, "margin_dpo/margin_std": 1158.6181640625, "step": 487 }, { "KL/chosen_KL_mean": -761.5409545898438, "KL/mean": -1126.8492431640625, "KL/rejected_KL_mean": -1492.157470703125, "KL/std": 1100.26611328125, "epoch": 0.7165932452276065, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 730.6165161132812, "fcm_dpo/q_t": 0.487249493598938, "grad_norm": 11.395562171936035, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -0.4758809208869934, "logits/rejected": -0.49622082710266113, "logps/chosen": -813.7691040039062, "logps/ref_chosen": -52.22815704345703, "logps/ref_rejected": -84.00656127929688, "logps/rejected": -1576.1640625, "loss": 1.3382, "margin_dpo/margin_mean": 730.6165771484375, "margin_dpo/margin_std": 1362.75, "step": 488 }, { "KL/chosen_KL_mean": -914.7315673828125, "KL/mean": -1159.464599609375, "KL/rejected_KL_mean": -1404.197509765625, "KL/std": 1258.12744140625, "epoch": 0.7180616740088106, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 489.4659423828125, "fcm_dpo/q_t": 0.4914560914039612, "grad_norm": 5.10060977935791, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -0.553819477558136, "logits/rejected": -0.5615823268890381, "logps/chosen": -970.72119140625, "logps/ref_chosen": -55.989627838134766, "logps/ref_rejected": -79.39812469482422, "logps/rejected": -1483.595703125, "loss": 1.3549, "margin_dpo/margin_mean": 489.4659118652344, "margin_dpo/margin_std": 1443.1240234375, "step": 489 }, { "KL/chosen_KL_mean": -992.151123046875, "KL/mean": -1396.121337890625, "KL/rejected_KL_mean": -1800.091552734375, "KL/std": 1577.330078125, "epoch": 0.7195301027900147, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 807.9404296875, "fcm_dpo/q_t": 0.4860500693321228, "grad_norm": 6.243213653564453, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -0.5025284290313721, "logits/rejected": -0.5427592992782593, "logps/chosen": -1044.517578125, "logps/ref_chosen": -52.36639404296875, "logps/ref_rejected": -110.4090576171875, "logps/rejected": -1910.5006103515625, "loss": 1.3358, "margin_dpo/margin_mean": 807.9404296875, "margin_dpo/margin_std": 1997.775146484375, "step": 490 }, { "KL/chosen_KL_mean": -961.5281982421875, "KL/mean": -1136.3218994140625, "KL/rejected_KL_mean": -1311.1156005859375, "KL/std": 1014.553955078125, "epoch": 0.7209985315712188, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 349.58746337890625, "fcm_dpo/q_t": 0.4938778281211853, "grad_norm": 3.7116713523864746, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -0.5231382846832275, "logits/rejected": -0.5342763662338257, "logps/chosen": -1021.6444702148438, "logps/ref_chosen": -60.11626434326172, "logps/ref_rejected": -73.27278900146484, "logps/rejected": -1384.388427734375, "loss": 1.3634, "margin_dpo/margin_mean": 349.58746337890625, "margin_dpo/margin_std": 1089.209228515625, "step": 491 }, { "KL/chosen_KL_mean": -1040.841796875, "KL/mean": -1144.0247802734375, "KL/rejected_KL_mean": -1247.20751953125, "KL/std": 1073.8876953125, "epoch": 0.7224669603524229, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 206.36590576171875, "fcm_dpo/q_t": 0.4963826835155487, "grad_norm": 5.653712272644043, "learning_rate": 1.097764975115576e-07, "logits/chosen": -0.5174509286880493, "logits/rejected": -0.5069364905357361, "logps/chosen": -1094.8359375, "logps/ref_chosen": -53.994178771972656, "logps/ref_rejected": -72.65962219238281, "logps/rejected": -1319.8671875, "loss": 1.3731, "margin_dpo/margin_mean": 206.36590576171875, "margin_dpo/margin_std": 1005.8743896484375, "step": 492 }, { "KL/chosen_KL_mean": -1245.180908203125, "KL/mean": -1378.21533203125, "KL/rejected_KL_mean": -1511.2498779296875, "KL/std": 1198.19384765625, "epoch": 0.723935389133627, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 266.0688781738281, "fcm_dpo/q_t": 0.49536770582199097, "grad_norm": 9.439188003540039, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -0.5633972883224487, "logits/rejected": -0.5487905740737915, "logps/chosen": -1320.67822265625, "logps/ref_chosen": -75.49723815917969, "logps/ref_rejected": -87.32301330566406, "logps/rejected": -1598.5728759765625, "loss": 1.3702, "margin_dpo/margin_mean": 266.06890869140625, "margin_dpo/margin_std": 1407.188232421875, "step": 493 }, { "KL/chosen_KL_mean": -743.1343994140625, "KL/mean": -1110.82080078125, "KL/rejected_KL_mean": -1478.5072021484375, "KL/std": 1011.7152709960938, "epoch": 0.7254038179148311, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 735.372802734375, "fcm_dpo/q_t": 0.48713040351867676, "grad_norm": 10.48877239227295, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -0.4758908748626709, "logits/rejected": -0.5146316289901733, "logps/chosen": -784.49365234375, "logps/ref_chosen": -41.35926818847656, "logps/ref_rejected": -86.09136962890625, "logps/rejected": -1564.5986328125, "loss": 1.3374, "margin_dpo/margin_mean": 735.372802734375, "margin_dpo/margin_std": 1294.782470703125, "step": 494 }, { "KL/chosen_KL_mean": -1080.8409423828125, "KL/mean": -1387.46923828125, "KL/rejected_KL_mean": -1694.097412109375, "KL/std": 1105.668212890625, "epoch": 0.7268722466960352, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 613.2564086914062, "fcm_dpo/q_t": 0.4892851710319519, "grad_norm": 5.530251502990723, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -0.6073266267776489, "logits/rejected": -0.6195170879364014, "logps/chosen": -1144.3759765625, "logps/ref_chosen": -63.53507995605469, "logps/ref_rejected": -91.42443084716797, "logps/rejected": -1785.52197265625, "loss": 1.347, "margin_dpo/margin_mean": 613.25634765625, "margin_dpo/margin_std": 1621.5955810546875, "step": 495 }, { "KL/chosen_KL_mean": -1282.886962890625, "KL/mean": -1321.6571044921875, "KL/rejected_KL_mean": -1360.427001953125, "KL/std": 1002.799560546875, "epoch": 0.7283406754772394, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.54009246826172, "fcm_dpo/q_t": 0.49863314628601074, "grad_norm": 8.960994720458984, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -0.5475114583969116, "logits/rejected": -0.5427240133285522, "logps/chosen": -1355.4788818359375, "logps/ref_chosen": -72.5919189453125, "logps/ref_rejected": -84.32933807373047, "logps/rejected": -1444.75634765625, "loss": 1.3831, "margin_dpo/margin_mean": 77.54010009765625, "margin_dpo/margin_std": 1289.9385986328125, "step": 496 }, { "KL/chosen_KL_mean": -949.1077270507812, "KL/mean": -1054.76904296875, "KL/rejected_KL_mean": -1160.430419921875, "KL/std": 835.3856201171875, "epoch": 0.7298091042584435, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 211.32281494140625, "fcm_dpo/q_t": 0.4962995648384094, "grad_norm": 3.6305837631225586, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -0.5167309045791626, "logits/rejected": -0.5133849382400513, "logps/chosen": -1007.70166015625, "logps/ref_chosen": -58.59397506713867, "logps/ref_rejected": -76.28836822509766, "logps/rejected": -1236.71875, "loss": 1.3724, "margin_dpo/margin_mean": 211.32281494140625, "margin_dpo/margin_std": 839.1533813476562, "step": 497 }, { "KL/chosen_KL_mean": -1179.71044921875, "KL/mean": -1404.1376953125, "KL/rejected_KL_mean": -1628.5648193359375, "KL/std": 1239.938232421875, "epoch": 0.7312775330396476, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 448.8544006347656, "fcm_dpo/q_t": 0.4921446144580841, "grad_norm": 4.796504974365234, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -0.5920599102973938, "logits/rejected": -0.5912704467773438, "logps/chosen": -1250.916015625, "logps/ref_chosen": -71.20565795898438, "logps/ref_rejected": -83.95803833007812, "logps/rejected": -1712.52294921875, "loss": 1.3579, "margin_dpo/margin_mean": 448.8544006347656, "margin_dpo/margin_std": 1510.208740234375, "step": 498 }, { "KL/chosen_KL_mean": -847.0122680664062, "KL/mean": -1311.192138671875, "KL/rejected_KL_mean": -1775.3720703125, "KL/std": 1260.3809814453125, "epoch": 0.7327459618208517, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 928.3598022460938, "fcm_dpo/q_t": 0.4837894141674042, "grad_norm": 6.413288593292236, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -0.5122471451759338, "logits/rejected": -0.5531229972839355, "logps/chosen": -898.2674560546875, "logps/ref_chosen": -51.25519561767578, "logps/ref_rejected": -101.07870483398438, "logps/rejected": -1876.45068359375, "loss": 1.3257, "margin_dpo/margin_mean": 928.3598022460938, "margin_dpo/margin_std": 1673.77978515625, "step": 499 }, { "KL/chosen_KL_mean": -1049.207763671875, "KL/mean": -1407.1966552734375, "KL/rejected_KL_mean": -1765.185546875, "KL/std": 1214.736328125, "epoch": 0.7342143906020558, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 715.9777221679688, "fcm_dpo/q_t": 0.4875142276287079, "grad_norm": 4.70263671875, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -0.5440601706504822, "logits/rejected": -0.5752770900726318, "logps/chosen": -1106.2352294921875, "logps/ref_chosen": -57.027442932128906, "logps/ref_rejected": -93.93421173095703, "logps/rejected": -1859.1197509765625, "loss": 1.34, "margin_dpo/margin_mean": 715.977783203125, "margin_dpo/margin_std": 1645.666259765625, "step": 500 }, { "KL/chosen_KL_mean": -943.0932006835938, "KL/mean": -1164.473876953125, "KL/rejected_KL_mean": -1385.8544921875, "KL/std": 1028.2890625, "epoch": 0.73568281938326, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 442.7614440917969, "fcm_dpo/q_t": 0.4922564625740051, "grad_norm": 5.623405933380127, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -0.5240232944488525, "logits/rejected": -0.5279651880264282, "logps/chosen": -997.4527587890625, "logps/ref_chosen": -54.359527587890625, "logps/ref_rejected": -80.15670013427734, "logps/rejected": -1466.01123046875, "loss": 1.3582, "margin_dpo/margin_mean": 442.76141357421875, "margin_dpo/margin_std": 1491.604248046875, "step": 501 }, { "KL/chosen_KL_mean": -986.6317749023438, "KL/mean": -1168.455078125, "KL/rejected_KL_mean": -1350.2783203125, "KL/std": 1039.948486328125, "epoch": 0.737151248164464, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 363.64642333984375, "fcm_dpo/q_t": 0.4936283826828003, "grad_norm": 8.514199256896973, "learning_rate": 9.934134090518592e-08, "logits/chosen": -0.5406728982925415, "logits/rejected": -0.5402701497077942, "logps/chosen": -1054.2322998046875, "logps/ref_chosen": -67.60050964355469, "logps/ref_rejected": -82.94876098632812, "logps/rejected": -1433.22705078125, "loss": 1.3624, "margin_dpo/margin_mean": 363.64642333984375, "margin_dpo/margin_std": 1048.6695556640625, "step": 502 }, { "KL/chosen_KL_mean": -824.9329833984375, "KL/mean": -1109.140869140625, "KL/rejected_KL_mean": -1393.3486328125, "KL/std": 1051.9873046875, "epoch": 0.7386196769456681, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 568.41552734375, "fcm_dpo/q_t": 0.4900738596916199, "grad_norm": 6.4620280265808105, "learning_rate": 9.831921068732571e-08, "logits/chosen": -0.516451358795166, "logits/rejected": -0.528926432132721, "logps/chosen": -880.0114135742188, "logps/ref_chosen": -55.078407287597656, "logps/ref_rejected": -82.50544738769531, "logps/rejected": -1475.85400390625, "loss": 1.3488, "margin_dpo/margin_mean": 568.41552734375, "margin_dpo/margin_std": 1233.030029296875, "step": 503 }, { "KL/chosen_KL_mean": -967.4698486328125, "KL/mean": -1384.71337890625, "KL/rejected_KL_mean": -1801.9569091796875, "KL/std": 1324.2451171875, "epoch": 0.7400881057268722, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 834.4871826171875, "fcm_dpo/q_t": 0.485454797744751, "grad_norm": 8.587855339050293, "learning_rate": 9.730107739932805e-08, "logits/chosen": -0.5549697279930115, "logits/rejected": -0.5928115844726562, "logps/chosen": -1027.435546875, "logps/ref_chosen": -59.96575164794922, "logps/ref_rejected": -103.76212310791016, "logps/rejected": -1905.718994140625, "loss": 1.332, "margin_dpo/margin_mean": 834.4871826171875, "margin_dpo/margin_std": 1630.097900390625, "step": 504 }, { "KL/chosen_KL_mean": -1171.11376953125, "KL/mean": -1224.62890625, "KL/rejected_KL_mean": -1278.1439208984375, "KL/std": 1174.8758544921875, "epoch": 0.7415565345080763, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.03019714355469, "fcm_dpo/q_t": 0.4981127381324768, "grad_norm": 7.053234577178955, "learning_rate": 9.628696786995188e-08, "logits/chosen": -0.5928906202316284, "logits/rejected": -0.5843832492828369, "logps/chosen": -1247.2685546875, "logps/ref_chosen": -76.1549072265625, "logps/ref_rejected": -88.58537292480469, "logps/rejected": -1366.729248046875, "loss": 1.3813, "margin_dpo/margin_mean": 107.03020477294922, "margin_dpo/margin_std": 1291.50732421875, "step": 505 }, { "KL/chosen_KL_mean": -677.691650390625, "KL/mean": -1025.074462890625, "KL/rejected_KL_mean": -1372.457275390625, "KL/std": 931.436767578125, "epoch": 0.7430249632892805, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 694.765625, "fcm_dpo/q_t": 0.48783206939697266, "grad_norm": 6.811020851135254, "learning_rate": 9.527690882192635e-08, "logits/chosen": -0.561811089515686, "logits/rejected": -0.5848267078399658, "logps/chosen": -726.6522216796875, "logps/ref_chosen": -48.96050262451172, "logps/ref_rejected": -78.41505432128906, "logps/rejected": -1450.872314453125, "loss": 1.3395, "margin_dpo/margin_mean": 694.765625, "margin_dpo/margin_std": 1041.9476318359375, "step": 506 }, { "KL/chosen_KL_mean": -971.807373046875, "KL/mean": -1220.935302734375, "KL/rejected_KL_mean": -1470.0631103515625, "KL/std": 1240.0423583984375, "epoch": 0.7444933920704846, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 498.2557067871094, "fcm_dpo/q_t": 0.49129223823547363, "grad_norm": 5.5018086433410645, "learning_rate": 9.427092687124691e-08, "logits/chosen": -0.5913007259368896, "logits/rejected": -0.6078677177429199, "logps/chosen": -1038.60888671875, "logps/ref_chosen": -66.80149841308594, "logps/ref_rejected": -95.37289428710938, "logps/rejected": -1565.43603515625, "loss": 1.354, "margin_dpo/margin_mean": 498.2557373046875, "margin_dpo/margin_std": 1392.36083984375, "step": 507 }, { "KL/chosen_KL_mean": -939.2349853515625, "KL/mean": -1200.439697265625, "KL/rejected_KL_mean": -1461.644287109375, "KL/std": 1131.05126953125, "epoch": 0.7459618208516887, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 522.409423828125, "fcm_dpo/q_t": 0.4908856451511383, "grad_norm": 3.9309823513031006, "learning_rate": 9.326904852647344e-08, "logits/chosen": -0.5682249665260315, "logits/rejected": -0.5859044194221497, "logps/chosen": -1010.5384521484375, "logps/ref_chosen": -71.303466796875, "logps/ref_rejected": -95.6275405883789, "logps/rejected": -1557.2718505859375, "loss": 1.3528, "margin_dpo/margin_mean": 522.409423828125, "margin_dpo/margin_std": 1477.3380126953125, "step": 508 }, { "KL/chosen_KL_mean": -770.8653564453125, "KL/mean": -939.4214477539062, "KL/rejected_KL_mean": -1107.9775390625, "KL/std": 911.1104736328125, "epoch": 0.7474302496328928, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 337.11224365234375, "fcm_dpo/q_t": 0.49410343170166016, "grad_norm": 5.916812419891357, "learning_rate": 9.227130018803195e-08, "logits/chosen": -0.5424858927726746, "logits/rejected": -0.5421060919761658, "logps/chosen": -834.684326171875, "logps/ref_chosen": -63.81895065307617, "logps/ref_rejected": -83.25643920898438, "logps/rejected": -1191.2340087890625, "loss": 1.3649, "margin_dpo/margin_mean": 337.1122131347656, "margin_dpo/margin_std": 1224.0833740234375, "step": 509 }, { "KL/chosen_KL_mean": -931.734130859375, "KL/mean": -1244.052978515625, "KL/rejected_KL_mean": -1556.371826171875, "KL/std": 978.7109375, "epoch": 0.748898678414097, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 624.6377563476562, "fcm_dpo/q_t": 0.4890550374984741, "grad_norm": 4.738297462463379, "learning_rate": 9.127770814751932e-08, "logits/chosen": -0.5328891277313232, "logits/rejected": -0.5605103969573975, "logps/chosen": -983.612548828125, "logps/ref_chosen": -51.878448486328125, "logps/ref_rejected": -102.7651596069336, "logps/rejected": -1659.1370849609375, "loss": 1.344, "margin_dpo/margin_mean": 624.6378173828125, "margin_dpo/margin_std": 936.5046997070312, "step": 510 }, { "KL/chosen_KL_mean": -913.4978637695312, "KL/mean": -1169.09619140625, "KL/rejected_KL_mean": -1424.6943359375, "KL/std": 1147.4625244140625, "epoch": 0.750367107195301, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 511.196533203125, "fcm_dpo/q_t": 0.4910575747489929, "grad_norm": 6.8262619972229, "learning_rate": 9.028829858700973e-08, "logits/chosen": -0.5740972757339478, "logits/rejected": -0.5858516693115234, "logps/chosen": -973.7359619140625, "logps/ref_chosen": -60.23811721801758, "logps/ref_rejected": -92.85676574707031, "logps/rejected": -1517.5511474609375, "loss": 1.3535, "margin_dpo/margin_mean": 511.1965026855469, "margin_dpo/margin_std": 1480.135009765625, "step": 511 }, { "KL/chosen_KL_mean": -768.2742919921875, "KL/mean": -1137.4937744140625, "KL/rejected_KL_mean": -1506.713134765625, "KL/std": 1220.0850830078125, "epoch": 0.7518355359765051, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 738.43896484375, "fcm_dpo/q_t": 0.4870929718017578, "grad_norm": 6.630341529846191, "learning_rate": 8.930309757836516e-08, "logits/chosen": -0.5527737140655518, "logits/rejected": -0.5937384366989136, "logps/chosen": -823.1798095703125, "logps/ref_chosen": -54.905494689941406, "logps/ref_rejected": -81.87586975097656, "logps/rejected": -1588.589111328125, "loss": 1.3376, "margin_dpo/margin_mean": 738.43896484375, "margin_dpo/margin_std": 1411.79248046875, "step": 512 }, { "KL/chosen_KL_mean": -1048.791015625, "KL/mean": -1195.49755859375, "KL/rejected_KL_mean": -1342.2042236328125, "KL/std": 1093.606689453125, "epoch": 0.7533039647577092, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 293.4132080078125, "fcm_dpo/q_t": 0.49488991498947144, "grad_norm": 9.842702865600586, "learning_rate": 8.832213108254863e-08, "logits/chosen": -0.567176342010498, "logits/rejected": -0.552276611328125, "logps/chosen": -1113.70751953125, "logps/ref_chosen": -64.91644287109375, "logps/ref_rejected": -76.06245422363281, "logps/rejected": -1418.2666015625, "loss": 1.3686, "margin_dpo/margin_mean": 293.4131774902344, "margin_dpo/margin_std": 1416.5491943359375, "step": 513 }, { "KL/chosen_KL_mean": -1270.703369140625, "KL/mean": -1452.780517578125, "KL/rejected_KL_mean": -1634.857666015625, "KL/std": 1571.225341796875, "epoch": 0.7547723935389133, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 364.1544189453125, "fcm_dpo/q_t": 0.4936813712120056, "grad_norm": 8.145087242126465, "learning_rate": 8.734542494893954e-08, "logits/chosen": -0.6005032658576965, "logits/rejected": -0.6026064157485962, "logps/chosen": -1344.932861328125, "logps/ref_chosen": -74.22957611083984, "logps/ref_rejected": -78.945556640625, "logps/rejected": -1713.80322265625, "loss": 1.3656, "margin_dpo/margin_mean": 364.1544494628906, "margin_dpo/margin_std": 1899.677001953125, "step": 514 }, { "KL/chosen_KL_mean": -822.61181640625, "KL/mean": -1065.546142578125, "KL/rejected_KL_mean": -1308.4805908203125, "KL/std": 936.01123046875, "epoch": 0.7562408223201175, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 485.86871337890625, "fcm_dpo/q_t": 0.4914933145046234, "grad_norm": 14.613423347473145, "learning_rate": 8.637300491465272e-08, "logits/chosen": -0.5557729601860046, "logits/rejected": -0.5711244940757751, "logps/chosen": -873.013427734375, "logps/ref_chosen": -50.40156555175781, "logps/ref_rejected": -87.09774780273438, "logps/rejected": -1395.578369140625, "loss": 1.3542, "margin_dpo/margin_mean": 485.8686828613281, "margin_dpo/margin_std": 1161.409423828125, "step": 515 }, { "KL/chosen_KL_mean": -1072.080810546875, "KL/mean": -1407.988037109375, "KL/rejected_KL_mean": -1743.8955078125, "KL/std": 1599.38037109375, "epoch": 0.7577092511013216, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 671.8146362304688, "fcm_dpo/q_t": 0.4883235692977905, "grad_norm": 7.461482048034668, "learning_rate": 8.540489660386064e-08, "logits/chosen": -0.5883182287216187, "logits/rejected": -0.621609091758728, "logps/chosen": -1136.7303466796875, "logps/ref_chosen": -64.64956665039062, "logps/ref_rejected": -111.72237396240234, "logps/rejected": -1855.6177978515625, "loss": 1.3438, "margin_dpo/margin_mean": 671.8145751953125, "margin_dpo/margin_std": 1814.15576171875, "step": 516 }, { "KL/chosen_KL_mean": -1061.081787109375, "KL/mean": -1346.005859375, "KL/rejected_KL_mean": -1630.9300537109375, "KL/std": 1233.93896484375, "epoch": 0.7591776798825257, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 569.8483276367188, "fcm_dpo/q_t": 0.4900466501712799, "grad_norm": 7.388690948486328, "learning_rate": 8.444112552711752e-08, "logits/chosen": -0.5805338621139526, "logits/rejected": -0.5963911414146423, "logps/chosen": -1121.995361328125, "logps/ref_chosen": -60.913551330566406, "logps/ref_rejected": -89.08308410644531, "logps/rejected": -1720.01318359375, "loss": 1.3499, "margin_dpo/margin_mean": 569.8483276367188, "margin_dpo/margin_std": 1582.470947265625, "step": 517 }, { "KL/chosen_KL_mean": -1056.78125, "KL/mean": -1327.392578125, "KL/rejected_KL_mean": -1598.0040283203125, "KL/std": 1047.9013671875, "epoch": 0.7606461086637298, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 541.22265625, "fcm_dpo/q_t": 0.49052339792251587, "grad_norm": 4.970123291015625, "learning_rate": 8.348171708068747e-08, "logits/chosen": -0.6027194261550903, "logits/rejected": -0.6282007694244385, "logps/chosen": -1114.2371826171875, "logps/ref_chosen": -57.45589065551758, "logps/ref_rejected": -85.31269836425781, "logps/rejected": -1683.316650390625, "loss": 1.3508, "margin_dpo/margin_mean": 541.22265625, "margin_dpo/margin_std": 1296.53173828125, "step": 518 }, { "KL/chosen_KL_mean": -1016.2708740234375, "KL/mean": -1329.120849609375, "KL/rejected_KL_mean": -1641.970947265625, "KL/std": 1342.2520751953125, "epoch": 0.762114537444934, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 625.7000732421875, "fcm_dpo/q_t": 0.4891040325164795, "grad_norm": 7.316686630249023, "learning_rate": 8.25266965458755e-08, "logits/chosen": -0.5834439992904663, "logits/rejected": -0.5853796005249023, "logps/chosen": -1090.334228515625, "logps/ref_chosen": -74.06331634521484, "logps/ref_rejected": -104.44416809082031, "logps/rejected": -1746.415283203125, "loss": 1.347, "margin_dpo/margin_mean": 625.7000732421875, "margin_dpo/margin_std": 1800.982421875, "step": 519 }, { "KL/chosen_KL_mean": -1117.6549072265625, "KL/mean": -1392.603759765625, "KL/rejected_KL_mean": -1667.552490234375, "KL/std": 1104.5682373046875, "epoch": 0.7635829662261381, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 549.8974609375, "fcm_dpo/q_t": 0.49038708209991455, "grad_norm": 6.708014965057373, "learning_rate": 8.15760890883607e-08, "logits/chosen": -0.5891440510749817, "logits/rejected": -0.6028249859809875, "logps/chosen": -1187.9547119140625, "logps/ref_chosen": -70.2998275756836, "logps/ref_rejected": -99.98133850097656, "logps/rejected": -1767.533935546875, "loss": 1.3504, "margin_dpo/margin_mean": 549.8974609375, "margin_dpo/margin_std": 1362.424560546875, "step": 520 }, { "KL/chosen_KL_mean": -902.4603271484375, "KL/mean": -1152.8935546875, "KL/rejected_KL_mean": -1403.32666015625, "KL/std": 1028.9302978515625, "epoch": 0.7650513950073421, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 500.8663330078125, "fcm_dpo/q_t": 0.4912322163581848, "grad_norm": 4.412991523742676, "learning_rate": 8.062991975753378e-08, "logits/chosen": -0.5892634391784668, "logits/rejected": -0.5929391980171204, "logps/chosen": -960.603271484375, "logps/ref_chosen": -58.14292526245117, "logps/ref_rejected": -83.28060913085938, "logps/rejected": -1486.6072998046875, "loss": 1.3533, "margin_dpo/margin_mean": 500.8663330078125, "margin_dpo/margin_std": 1217.1341552734375, "step": 521 }, { "KL/chosen_KL_mean": -1055.103271484375, "KL/mean": -1362.3404541015625, "KL/rejected_KL_mean": -1669.57763671875, "KL/std": 1621.5078125, "epoch": 0.7665198237885462, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 614.4742431640625, "fcm_dpo/q_t": 0.4893651604652405, "grad_norm": 4.380655288696289, "learning_rate": 7.968821348583643e-08, "logits/chosen": -0.5628671646118164, "logits/rejected": -0.5865719318389893, "logps/chosen": -1101.6510009765625, "logps/ref_chosen": -46.54766845703125, "logps/ref_rejected": -66.01388549804688, "logps/rejected": -1735.591552734375, "loss": 1.3489, "margin_dpo/margin_mean": 614.4742431640625, "margin_dpo/margin_std": 2025.41748046875, "step": 522 }, { "KL/chosen_KL_mean": -1028.6043701171875, "KL/mean": -1341.892578125, "KL/rejected_KL_mean": -1655.180908203125, "KL/std": 1353.0596923828125, "epoch": 0.7679882525697503, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 626.5765380859375, "fcm_dpo/q_t": 0.48908570408821106, "grad_norm": 7.474643707275391, "learning_rate": 7.875099508810484e-08, "logits/chosen": -0.6015632152557373, "logits/rejected": -0.6170424222946167, "logps/chosen": -1090.3740234375, "logps/ref_chosen": -61.76960372924805, "logps/ref_rejected": -83.76141357421875, "logps/rejected": -1738.9423828125, "loss": 1.346, "margin_dpo/margin_mean": 626.5764770507812, "margin_dpo/margin_std": 1597.18505859375, "step": 523 }, { "KL/chosen_KL_mean": -1175.5543212890625, "KL/mean": -1509.3448486328125, "KL/rejected_KL_mean": -1843.1353759765625, "KL/std": 1454.0643310546875, "epoch": 0.7694566813509545, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 667.5810546875, "fcm_dpo/q_t": 0.48826104402542114, "grad_norm": 5.199390411376953, "learning_rate": 7.781828926091535e-08, "logits/chosen": -0.6230287551879883, "logits/rejected": -0.6203910112380981, "logps/chosen": -1253.6263427734375, "logps/ref_chosen": -78.0720443725586, "logps/ref_rejected": -81.30198669433594, "logps/rejected": -1924.4373779296875, "loss": 1.3434, "margin_dpo/margin_mean": 667.5810546875, "margin_dpo/margin_std": 1561.3837890625, "step": 524 }, { "KL/chosen_KL_mean": -979.4048461914062, "KL/mean": -1361.4544677734375, "KL/rejected_KL_mean": -1743.50390625, "KL/std": 1232.875732421875, "epoch": 0.7709251101321586, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 764.09912109375, "fcm_dpo/q_t": 0.4866584539413452, "grad_norm": 7.510035037994385, "learning_rate": 7.689012058193384e-08, "logits/chosen": -0.5819898843765259, "logits/rejected": -0.6207159757614136, "logps/chosen": -1030.232666015625, "logps/ref_chosen": -50.827857971191406, "logps/ref_rejected": -100.05294036865234, "logps/rejected": -1843.556884765625, "loss": 1.3363, "margin_dpo/margin_mean": 764.09912109375, "margin_dpo/margin_std": 1525.293212890625, "step": 525 }, { "KL/chosen_KL_mean": -1006.8646240234375, "KL/mean": -1335.72021484375, "KL/rejected_KL_mean": -1664.5760498046875, "KL/std": 1145.358642578125, "epoch": 0.7723935389133627, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 657.71142578125, "fcm_dpo/q_t": 0.4884929656982422, "grad_norm": 4.733247756958008, "learning_rate": 7.596651350926836e-08, "logits/chosen": -0.6167298555374146, "logits/rejected": -0.6341676712036133, "logps/chosen": -1070.0318603515625, "logps/ref_chosen": -63.167236328125, "logps/ref_rejected": -86.30934143066406, "logps/rejected": -1750.8853759765625, "loss": 1.3427, "margin_dpo/margin_mean": 657.71142578125, "margin_dpo/margin_std": 1287.364990234375, "step": 526 }, { "KL/chosen_KL_mean": -1146.7838134765625, "KL/mean": -1294.6519775390625, "KL/rejected_KL_mean": -1442.520263671875, "KL/std": 1127.4002685546875, "epoch": 0.7738619676945668, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 295.73638916015625, "fcm_dpo/q_t": 0.49481672048568726, "grad_norm": 8.143030166625977, "learning_rate": 7.504749238082414e-08, "logits/chosen": -0.6452538967132568, "logits/rejected": -0.6307415962219238, "logps/chosen": -1217.91259765625, "logps/ref_chosen": -71.12867736816406, "logps/ref_rejected": -78.3425521850586, "logps/rejected": -1520.86279296875, "loss": 1.3671, "margin_dpo/margin_mean": 295.73638916015625, "margin_dpo/margin_std": 1076.11376953125, "step": 527 }, { "KL/chosen_KL_mean": -924.047607421875, "KL/mean": -1256.195556640625, "KL/rejected_KL_mean": -1588.343505859375, "KL/std": 1196.41650390625, "epoch": 0.775330396475771, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 664.2958374023438, "fcm_dpo/q_t": 0.4884028434753418, "grad_norm": 4.170558929443359, "learning_rate": 7.413308141366254e-08, "logits/chosen": -0.6017875671386719, "logits/rejected": -0.6084692478179932, "logps/chosen": -992.1370849609375, "logps/ref_chosen": -68.0894546508789, "logps/ref_rejected": -93.91006469726562, "logps/rejected": -1682.2535400390625, "loss": 1.3432, "margin_dpo/margin_mean": 664.2958374023438, "margin_dpo/margin_std": 1553.2027587890625, "step": 528 }, { "KL/chosen_KL_mean": -1159.14599609375, "KL/mean": -1451.514404296875, "KL/rejected_KL_mean": -1743.8826904296875, "KL/std": 1228.73193359375, "epoch": 0.7767988252569751, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 584.7366333007812, "fcm_dpo/q_t": 0.48979973793029785, "grad_norm": 4.807751655578613, "learning_rate": 7.322330470336313e-08, "logits/chosen": -0.5786510109901428, "logits/rejected": -0.5997955799102783, "logps/chosen": -1214.720947265625, "logps/ref_chosen": -55.57495880126953, "logps/ref_rejected": -89.20909118652344, "logps/rejected": -1833.091796875, "loss": 1.3491, "margin_dpo/margin_mean": 584.7366333007812, "margin_dpo/margin_std": 1670.16748046875, "step": 529 }, { "KL/chosen_KL_mean": -882.88427734375, "KL/mean": -1237.404296875, "KL/rejected_KL_mean": -1591.9244384765625, "KL/std": 1404.204345703125, "epoch": 0.7782672540381792, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 709.0401000976562, "fcm_dpo/q_t": 0.4876710772514343, "grad_norm": 8.733548164367676, "learning_rate": 7.231818622338822e-08, "logits/chosen": -0.5622015595436096, "logits/rejected": -0.5846884250640869, "logps/chosen": -930.4857177734375, "logps/ref_chosen": -47.601417541503906, "logps/ref_rejected": -87.2845230102539, "logps/rejected": -1679.208984375, "loss": 1.3413, "margin_dpo/margin_mean": 709.0401611328125, "margin_dpo/margin_std": 1824.845458984375, "step": 530 }, { "KL/chosen_KL_mean": -859.3704833984375, "KL/mean": -1154.43994140625, "KL/rejected_KL_mean": -1449.509521484375, "KL/std": 1112.5634765625, "epoch": 0.7797356828193832, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 590.1390380859375, "fcm_dpo/q_t": 0.48967117071151733, "grad_norm": 5.344662189483643, "learning_rate": 7.141774982445147e-08, "logits/chosen": -0.5867226123809814, "logits/rejected": -0.6062394380569458, "logps/chosen": -914.6165161132812, "logps/ref_chosen": -55.246063232421875, "logps/ref_rejected": -70.60598754882812, "logps/rejected": -1520.115478515625, "loss": 1.3468, "margin_dpo/margin_mean": 590.1390380859375, "margin_dpo/margin_std": 1108.8994140625, "step": 531 }, { "KL/chosen_KL_mean": -1094.8494873046875, "KL/mean": -1391.7200927734375, "KL/rejected_KL_mean": -1688.590576171875, "KL/std": 1260.90576171875, "epoch": 0.7812041116005873, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 593.7410278320312, "fcm_dpo/q_t": 0.4896160364151001, "grad_norm": 7.872839450836182, "learning_rate": 7.052201923388953e-08, "logits/chosen": -0.5879281759262085, "logits/rejected": -0.5886451005935669, "logps/chosen": -1165.135498046875, "logps/ref_chosen": -70.28601837158203, "logps/ref_rejected": -86.5913314819336, "logps/rejected": -1775.181884765625, "loss": 1.3489, "margin_dpo/margin_mean": 593.740966796875, "margin_dpo/margin_std": 1750.036376953125, "step": 532 }, { "KL/chosen_KL_mean": -933.3600463867188, "KL/mean": -1131.0, "KL/rejected_KL_mean": -1328.639892578125, "KL/std": 966.4365844726562, "epoch": 0.7826725403817915, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 395.27978515625, "fcm_dpo/q_t": 0.4930812120437622, "grad_norm": 7.988112449645996, "learning_rate": 6.963101805503646e-08, "logits/chosen": -0.5576300024986267, "logits/rejected": -0.5628528594970703, "logps/chosen": -998.2151489257812, "logps/ref_chosen": -64.8551025390625, "logps/ref_rejected": -76.58805847167969, "logps/rejected": -1405.2279052734375, "loss": 1.3604, "margin_dpo/margin_mean": 395.27978515625, "margin_dpo/margin_std": 1123.464599609375, "step": 533 }, { "KL/chosen_KL_mean": -992.2314453125, "KL/mean": -1203.13671875, "KL/rejected_KL_mean": -1414.042236328125, "KL/std": 950.9990234375, "epoch": 0.7841409691629956, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 421.810791015625, "fcm_dpo/q_t": 0.49261125922203064, "grad_norm": 3.841683864593506, "learning_rate": 6.874476976660184e-08, "logits/chosen": -0.5702451467514038, "logits/rejected": -0.5771702527999878, "logps/chosen": -1052.350830078125, "logps/ref_chosen": -60.119388580322266, "logps/ref_rejected": -78.54347229003906, "logps/rejected": -1492.585693359375, "loss": 1.3584, "margin_dpo/margin_mean": 421.810791015625, "margin_dpo/margin_std": 1089.174560546875, "step": 534 }, { "KL/chosen_KL_mean": -888.7866821289062, "KL/mean": -1202.163330078125, "KL/rejected_KL_mean": -1515.5400390625, "KL/std": 1040.59130859375, "epoch": 0.7856093979441997, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 626.75341796875, "fcm_dpo/q_t": 0.48904871940612793, "grad_norm": 8.149191856384277, "learning_rate": 6.786329772205246e-08, "logits/chosen": -0.552294135093689, "logits/rejected": -0.5709767937660217, "logps/chosen": -943.1168823242188, "logps/ref_chosen": -54.330238342285156, "logps/ref_rejected": -96.30763244628906, "logps/rejected": -1611.84765625, "loss": 1.345, "margin_dpo/margin_mean": 626.75341796875, "margin_dpo/margin_std": 1275.3328857421875, "step": 535 }, { "KL/chosen_KL_mean": -841.0042724609375, "KL/mean": -1279.9410400390625, "KL/rejected_KL_mean": -1718.8778076171875, "KL/std": 1255.70849609375, "epoch": 0.7870778267254038, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 877.8734130859375, "fcm_dpo/q_t": 0.4847047030925751, "grad_norm": 5.253050327301025, "learning_rate": 6.698662514899638e-08, "logits/chosen": -0.5551550388336182, "logits/rejected": -0.5932704210281372, "logps/chosen": -888.0848388671875, "logps/ref_chosen": -47.08053207397461, "logps/ref_rejected": -89.09783935546875, "logps/rejected": -1807.9755859375, "loss": 1.3289, "margin_dpo/margin_mean": 877.8734741210938, "margin_dpo/margin_std": 1561.20703125, "step": 536 }, { "KL/chosen_KL_mean": -840.8751831054688, "KL/mean": -1096.5126953125, "KL/rejected_KL_mean": -1352.150146484375, "KL/std": 1151.702880859375, "epoch": 0.788546255506608, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 511.2747802734375, "fcm_dpo/q_t": 0.49105796217918396, "grad_norm": 7.912595748901367, "learning_rate": 6.611477514857114e-08, "logits/chosen": -0.5522580742835999, "logits/rejected": -0.5589759945869446, "logps/chosen": -898.6226806640625, "logps/ref_chosen": -57.747467041015625, "logps/ref_rejected": -70.43838500976562, "logps/rejected": -1422.58837890625, "loss": 1.3537, "margin_dpo/margin_mean": 511.2747802734375, "margin_dpo/margin_std": 1513.6673583984375, "step": 537 }, { "KL/chosen_KL_mean": -1396.2117919921875, "KL/mean": -1632.014404296875, "KL/rejected_KL_mean": -1867.817138671875, "KL/std": 1259.4403076171875, "epoch": 0.7900146842878121, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 471.6053466796875, "fcm_dpo/q_t": 0.49175089597702026, "grad_norm": 9.711124420166016, "learning_rate": 6.524777069483525e-08, "logits/chosen": -0.5915812253952026, "logits/rejected": -0.5872704982757568, "logps/chosen": -1462.627685546875, "logps/ref_chosen": -66.41594696044922, "logps/ref_rejected": -84.22808837890625, "logps/rejected": -1952.045166015625, "loss": 1.3565, "margin_dpo/margin_mean": 471.6053466796875, "margin_dpo/margin_std": 1497.396484375, "step": 538 }, { "KL/chosen_KL_mean": -1097.3935546875, "KL/mean": -1476.907958984375, "KL/rejected_KL_mean": -1856.42236328125, "KL/std": 1343.3634033203125, "epoch": 0.7914831130690162, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 759.0288696289062, "fcm_dpo/q_t": 0.48674440383911133, "grad_norm": 5.35037899017334, "learning_rate": 6.438563463416221e-08, "logits/chosen": -0.6155024766921997, "logits/rejected": -0.6297394633293152, "logps/chosen": -1155.886474609375, "logps/ref_chosen": -58.492855072021484, "logps/ref_rejected": -91.85395050048828, "logps/rejected": -1948.2763671875, "loss": 1.3363, "margin_dpo/margin_mean": 759.0289306640625, "margin_dpo/margin_std": 1419.5430908203125, "step": 539 }, { "KL/chosen_KL_mean": -1084.130615234375, "KL/mean": -1591.833740234375, "KL/rejected_KL_mean": -2099.53662109375, "KL/std": 1494.089111328125, "epoch": 0.7929515418502202, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1015.4059448242188, "fcm_dpo/q_t": 0.48234492540359497, "grad_norm": 17.94775390625, "learning_rate": 6.352838968463919e-08, "logits/chosen": -0.5842588543891907, "logits/rejected": -0.6213306188583374, "logps/chosen": -1147.61328125, "logps/ref_chosen": -63.482513427734375, "logps/ref_rejected": -116.42999267578125, "logps/rejected": -2215.966796875, "loss": 1.3212, "margin_dpo/margin_mean": 1015.4058837890625, "margin_dpo/margin_std": 1876.0716552734375, "step": 540 }, { "KL/chosen_KL_mean": -1515.512939453125, "KL/mean": -1609.35546875, "KL/rejected_KL_mean": -1703.19775390625, "KL/std": 1322.71044921875, "epoch": 0.7944199706314243, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 187.684814453125, "fcm_dpo/q_t": 0.49672752618789673, "grad_norm": 9.675416946411133, "learning_rate": 6.267605843546767e-08, "logits/chosen": -0.6445415019989014, "logits/rejected": -0.6379104852676392, "logps/chosen": -1593.7933349609375, "logps/ref_chosen": -78.28036499023438, "logps/ref_rejected": -103.273681640625, "logps/rejected": -1806.471435546875, "loss": 1.3765, "margin_dpo/margin_mean": 187.68482971191406, "margin_dpo/margin_std": 1610.0963134765625, "step": 541 }, { "KL/chosen_KL_mean": -1154.2373046875, "KL/mean": -1602.501220703125, "KL/rejected_KL_mean": -2050.76513671875, "KL/std": 1615.48046875, "epoch": 0.7958883994126285, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 896.5280151367188, "fcm_dpo/q_t": 0.48452144861221313, "grad_norm": 4.6800994873046875, "learning_rate": 6.182866334636888e-08, "logits/chosen": -0.6557344198226929, "logits/rejected": -0.6881058216094971, "logps/chosen": -1211.72216796875, "logps/ref_chosen": -57.48497009277344, "logps/ref_rejected": -96.47506713867188, "logps/rejected": -2147.240234375, "loss": 1.33, "margin_dpo/margin_mean": 896.5280151367188, "margin_dpo/margin_std": 2071.083740234375, "step": 542 }, { "KL/chosen_KL_mean": -1310.2457275390625, "KL/mean": -1533.172119140625, "KL/rejected_KL_mean": -1756.0986328125, "KL/std": 1376.15087890625, "epoch": 0.7973568281938326, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 445.85296630859375, "fcm_dpo/q_t": 0.4922771453857422, "grad_norm": 4.305510520935059, "learning_rate": 6.098622674699147e-08, "logits/chosen": -0.664671778678894, "logits/rejected": -0.6874226331710815, "logps/chosen": -1370.86328125, "logps/ref_chosen": -60.61750793457031, "logps/ref_rejected": -105.59896850585938, "logps/rejected": -1861.6976318359375, "loss": 1.36, "margin_dpo/margin_mean": 445.8529357910156, "margin_dpo/margin_std": 1953.89453125, "step": 543 }, { "KL/chosen_KL_mean": -1286.40478515625, "KL/mean": -1731.963134765625, "KL/rejected_KL_mean": -2177.521484375, "KL/std": 1765.681396484375, "epoch": 0.7988252569750367, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 891.1168212890625, "fcm_dpo/q_t": 0.48461323976516724, "grad_norm": 8.807916641235352, "learning_rate": 6.01487708363232e-08, "logits/chosen": -0.6092058420181274, "logits/rejected": -0.6435132026672363, "logps/chosen": -1346.047119140625, "logps/ref_chosen": -59.642303466796875, "logps/ref_rejected": -100.95469665527344, "logps/rejected": -2278.476318359375, "loss": 1.3311, "margin_dpo/margin_mean": 891.1168212890625, "margin_dpo/margin_std": 2284.972900390625, "step": 544 }, { "KL/chosen_KL_mean": -1260.89306640625, "KL/mean": -1646.8466796875, "KL/rejected_KL_mean": -2032.800537109375, "KL/std": 1534.07568359375, "epoch": 0.8002936857562408, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 771.9075927734375, "fcm_dpo/q_t": 0.48656225204467773, "grad_norm": 6.247243881225586, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -0.5921393632888794, "logits/rejected": -0.6117866635322571, "logps/chosen": -1328.54150390625, "logps/ref_chosen": -67.64859771728516, "logps/ref_rejected": -95.90800476074219, "logps/rejected": -2128.70849609375, "loss": 1.337, "margin_dpo/margin_mean": 771.9075317382812, "margin_dpo/margin_std": 1806.5426025390625, "step": 545 }, { "KL/chosen_KL_mean": -1186.041748046875, "KL/mean": -1404.183837890625, "KL/rejected_KL_mean": -1622.325927734375, "KL/std": 1201.185546875, "epoch": 0.801762114537445, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 436.28411865234375, "fcm_dpo/q_t": 0.4923797845840454, "grad_norm": 12.886591911315918, "learning_rate": 5.848888922025552e-08, "logits/chosen": -0.6253443360328674, "logits/rejected": -0.6243206262588501, "logps/chosen": -1236.785888671875, "logps/ref_chosen": -50.744232177734375, "logps/ref_rejected": -81.86622619628906, "logps/rejected": -1704.192138671875, "loss": 1.3586, "margin_dpo/margin_mean": 436.28411865234375, "margin_dpo/margin_std": 1439.834716796875, "step": 546 }, { "KL/chosen_KL_mean": -1329.843505859375, "KL/mean": -1714.03662109375, "KL/rejected_KL_mean": -2098.2294921875, "KL/std": 1930.3846435546875, "epoch": 0.8032305433186491, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 768.38623046875, "fcm_dpo/q_t": 0.4867956340312958, "grad_norm": 9.838577270507812, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -0.5816187858581543, "logits/rejected": -0.6018866300582886, "logps/chosen": -1403.53125, "logps/ref_chosen": -73.6877212524414, "logps/ref_rejected": -90.76136779785156, "logps/rejected": -2188.990966796875, "loss": 1.3413, "margin_dpo/margin_mean": 768.38623046875, "margin_dpo/margin_std": 2552.30029296875, "step": 547 }, { "KL/chosen_KL_mean": -1230.2508544921875, "KL/mean": -1500.3779296875, "KL/rejected_KL_mean": -1770.5048828125, "KL/std": 1448.751708984375, "epoch": 0.8046989720998532, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 540.2540283203125, "fcm_dpo/q_t": 0.49062401056289673, "grad_norm": 4.971753120422363, "learning_rate": 5.684919345471029e-08, "logits/chosen": -0.6555124521255493, "logits/rejected": -0.6675156950950623, "logps/chosen": -1295.4971923828125, "logps/ref_chosen": -65.24634552001953, "logps/ref_rejected": -94.11807250976562, "logps/rejected": -1864.6229248046875, "loss": 1.3527, "margin_dpo/margin_mean": 540.2540283203125, "margin_dpo/margin_std": 1749.078125, "step": 548 }, { "KL/chosen_KL_mean": -1231.84814453125, "KL/mean": -1339.867431640625, "KL/rejected_KL_mean": -1447.8865966796875, "KL/std": 1256.0994873046875, "epoch": 0.8061674008810573, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 216.03839111328125, "fcm_dpo/q_t": 0.4962120056152344, "grad_norm": 6.037284851074219, "learning_rate": 5.603696935852426e-08, "logits/chosen": -0.5892428159713745, "logits/rejected": -0.5814424753189087, "logps/chosen": -1281.060546875, "logps/ref_chosen": -49.21235656738281, "logps/ref_rejected": -73.91031646728516, "logps/rejected": -1521.796875, "loss": 1.3742, "margin_dpo/margin_mean": 216.0384063720703, "margin_dpo/margin_std": 1532.143310546875, "step": 549 }, { "KL/chosen_KL_mean": -1138.1700439453125, "KL/mean": -1449.984375, "KL/rejected_KL_mean": -1761.798828125, "KL/std": 1296.947998046875, "epoch": 0.8076358296622613, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 623.6287841796875, "fcm_dpo/q_t": 0.489163339138031, "grad_norm": 4.7602033615112305, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -0.6239445209503174, "logits/rejected": -0.6451071500778198, "logps/chosen": -1194.97705078125, "logps/ref_chosen": -56.80695343017578, "logps/ref_rejected": -95.12580871582031, "logps/rejected": -1856.924560546875, "loss": 1.3463, "margin_dpo/margin_mean": 623.6287841796875, "margin_dpo/margin_std": 1583.240478515625, "step": 550 }, { "KL/chosen_KL_mean": -903.1275634765625, "KL/mean": -1493.087158203125, "KL/rejected_KL_mean": -2083.04638671875, "KL/std": 1501.976806640625, "epoch": 0.8091042584434655, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1179.9189453125, "fcm_dpo/q_t": 0.4793858528137207, "grad_norm": 11.45347785949707, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -0.5236402750015259, "logits/rejected": -0.5718011260032654, "logps/chosen": -962.23388671875, "logps/ref_chosen": -59.10633087158203, "logps/ref_rejected": -111.67280578613281, "logps/rejected": -2194.71923828125, "loss": 1.3087, "margin_dpo/margin_mean": 1179.9189453125, "margin_dpo/margin_std": 1684.580322265625, "step": 551 }, { "KL/chosen_KL_mean": -930.559326171875, "KL/mean": -1511.77001953125, "KL/rejected_KL_mean": -2092.98046875, "KL/std": 1509.347412109375, "epoch": 0.8105726872246696, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1162.4212646484375, "fcm_dpo/q_t": 0.4797826111316681, "grad_norm": 5.244424343109131, "learning_rate": 5.363104864490034e-08, "logits/chosen": -0.5785896182060242, "logits/rejected": -0.6230120658874512, "logps/chosen": -992.9139404296875, "logps/ref_chosen": -62.35459899902344, "logps/ref_rejected": -104.56210327148438, "logps/rejected": -2197.54248046875, "loss": 1.3114, "margin_dpo/margin_mean": 1162.421142578125, "margin_dpo/margin_std": 1976.2950439453125, "step": 552 }, { "KL/chosen_KL_mean": -1078.3341064453125, "KL/mean": -1394.103515625, "KL/rejected_KL_mean": -1709.872802734375, "KL/std": 1449.83984375, "epoch": 0.8120411160058737, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 631.5387573242188, "fcm_dpo/q_t": 0.4890480637550354, "grad_norm": 3.988219976425171, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -0.62043297290802, "logits/rejected": -0.6310804486274719, "logps/chosen": -1146.5928955078125, "logps/ref_chosen": -68.25881958007812, "logps/ref_rejected": -98.0971450805664, "logps/rejected": -1807.969970703125, "loss": 1.3467, "margin_dpo/margin_mean": 631.5386962890625, "margin_dpo/margin_std": 1802.6300048828125, "step": 553 }, { "KL/chosen_KL_mean": -1069.6845703125, "KL/mean": -1364.756103515625, "KL/rejected_KL_mean": -1659.8275146484375, "KL/std": 1380.947998046875, "epoch": 0.8135095447870778, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 590.1428833007812, "fcm_dpo/q_t": 0.4896678328514099, "grad_norm": 8.758122444152832, "learning_rate": 5.205293880283551e-08, "logits/chosen": -0.5634171962738037, "logits/rejected": -0.5632882714271545, "logps/chosen": -1137.63232421875, "logps/ref_chosen": -67.94767761230469, "logps/ref_rejected": -89.78272247314453, "logps/rejected": -1749.6102294921875, "loss": 1.35, "margin_dpo/margin_mean": 590.1428833007812, "margin_dpo/margin_std": 1906.44287109375, "step": 554 }, { "KL/chosen_KL_mean": -1021.588623046875, "KL/mean": -1599.221923828125, "KL/rejected_KL_mean": -2176.855224609375, "KL/std": 1767.3104248046875, "epoch": 0.8149779735682819, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1155.2666015625, "fcm_dpo/q_t": 0.48013854026794434, "grad_norm": 10.302849769592285, "learning_rate": 5.127169765359515e-08, "logits/chosen": -0.6012399792671204, "logits/rejected": -0.6592134237289429, "logps/chosen": -1074.919189453125, "logps/ref_chosen": -53.33049011230469, "logps/ref_rejected": -108.47937774658203, "logps/rejected": -2285.33447265625, "loss": 1.3141, "margin_dpo/margin_mean": 1155.2666015625, "margin_dpo/margin_std": 2393.09033203125, "step": 555 }, { "KL/chosen_KL_mean": -982.6744384765625, "KL/mean": -1277.244384765625, "KL/rejected_KL_mean": -1571.814208984375, "KL/std": 1051.68212890625, "epoch": 0.8164464023494861, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 589.1397705078125, "fcm_dpo/q_t": 0.4896823763847351, "grad_norm": 13.509299278259277, "learning_rate": 5.049569317994012e-08, "logits/chosen": -0.615922212600708, "logits/rejected": -0.6318573355674744, "logps/chosen": -1041.31884765625, "logps/ref_chosen": -58.64447021484375, "logps/ref_rejected": -101.34040832519531, "logps/rejected": -1673.1546630859375, "loss": 1.3468, "margin_dpo/margin_mean": 589.1397705078125, "margin_dpo/margin_std": 1059.956787109375, "step": 556 }, { "KL/chosen_KL_mean": -1247.9884033203125, "KL/mean": -1629.21337890625, "KL/rejected_KL_mean": -2010.438232421875, "KL/std": 1569.9039306640625, "epoch": 0.8179148311306902, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 762.44970703125, "fcm_dpo/q_t": 0.4867032766342163, "grad_norm": 9.238645553588867, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -0.6624920964241028, "logits/rejected": -0.6939408779144287, "logps/chosen": -1315.8291015625, "logps/ref_chosen": -67.84066009521484, "logps/ref_rejected": -109.93965911865234, "logps/rejected": -2120.3779296875, "loss": 1.3377, "margin_dpo/margin_mean": 762.44970703125, "margin_dpo/margin_std": 1803.6630859375, "step": 557 }, { "KL/chosen_KL_mean": -1106.59033203125, "KL/mean": -1585.306396484375, "KL/rejected_KL_mean": -2064.022216796875, "KL/std": 1658.90380859375, "epoch": 0.8193832599118943, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 957.43212890625, "fcm_dpo/q_t": 0.48335278034210205, "grad_norm": 7.449563980102539, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -0.6731094717979431, "logits/rejected": -0.7122669219970703, "logps/chosen": -1168.9586181640625, "logps/ref_chosen": -62.36824035644531, "logps/ref_rejected": -102.16102600097656, "logps/rejected": -2166.183349609375, "loss": 1.3256, "margin_dpo/margin_mean": 957.4320678710938, "margin_dpo/margin_std": 2066.8369140625, "step": 558 }, { "KL/chosen_KL_mean": -1256.0625, "KL/mean": -1648.341064453125, "KL/rejected_KL_mean": -2040.619873046875, "KL/std": 1527.5263671875, "epoch": 0.8208516886930984, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 784.557373046875, "fcm_dpo/q_t": 0.4863905608654022, "grad_norm": 9.646005630493164, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -0.70490962266922, "logits/rejected": -0.7145035266876221, "logps/chosen": -1316.814697265625, "logps/ref_chosen": -60.752323150634766, "logps/ref_rejected": -93.44229125976562, "logps/rejected": -2134.06201171875, "loss": 1.3373, "margin_dpo/margin_mean": 784.557373046875, "margin_dpo/margin_std": 2029.985595703125, "step": 559 }, { "KL/chosen_KL_mean": -1072.104736328125, "KL/mean": -1330.7318115234375, "KL/rejected_KL_mean": -1589.35888671875, "KL/std": 1190.2357177734375, "epoch": 0.8223201174743024, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 517.25390625, "fcm_dpo/q_t": 0.4909706115722656, "grad_norm": 8.734450340270996, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -0.5731639862060547, "logits/rejected": -0.5752190351486206, "logps/chosen": -1130.2086181640625, "logps/ref_chosen": -58.10382080078125, "logps/ref_rejected": -79.99122619628906, "logps/rejected": -1669.35009765625, "loss": 1.3529, "margin_dpo/margin_mean": 517.2539672851562, "margin_dpo/margin_std": 1448.2685546875, "step": 560 }, { "KL/chosen_KL_mean": -1208.382568359375, "KL/mean": -1487.0732421875, "KL/rejected_KL_mean": -1765.763916015625, "KL/std": 1353.732421875, "epoch": 0.8237885462555066, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 557.3814697265625, "fcm_dpo/q_t": 0.49026691913604736, "grad_norm": 5.121155738830566, "learning_rate": 4.669493178106432e-08, "logits/chosen": -0.6740288734436035, "logits/rejected": -0.6940807104110718, "logps/chosen": -1259.29541015625, "logps/ref_chosen": -50.912879943847656, "logps/ref_rejected": -99.06856536865234, "logps/rejected": -1864.83251953125, "loss": 1.3516, "margin_dpo/margin_mean": 557.3814697265625, "margin_dpo/margin_std": 1794.720458984375, "step": 561 }, { "KL/chosen_KL_mean": -977.68701171875, "KL/mean": -1403.8623046875, "KL/rejected_KL_mean": -1830.037841796875, "KL/std": 1357.09423828125, "epoch": 0.8252569750367107, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 852.3507080078125, "fcm_dpo/q_t": 0.48517775535583496, "grad_norm": 6.560747146606445, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -0.6164703965187073, "logits/rejected": -0.6432167291641235, "logps/chosen": -1037.1513671875, "logps/ref_chosen": -59.46440124511719, "logps/ref_rejected": -96.54266357421875, "logps/rejected": -1926.580322265625, "loss": 1.331, "margin_dpo/margin_mean": 852.3507690429688, "margin_dpo/margin_std": 1607.605712890625, "step": 562 }, { "KL/chosen_KL_mean": -1360.0205078125, "KL/mean": -1654.7742919921875, "KL/rejected_KL_mean": -1949.5281982421875, "KL/std": 1659.429931640625, "epoch": 0.8267254038179148, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 589.5076904296875, "fcm_dpo/q_t": 0.4897218942642212, "grad_norm": 8.3922700881958, "learning_rate": 4.521198892775202e-08, "logits/chosen": -0.6155869960784912, "logits/rejected": -0.6347240209579468, "logps/chosen": -1420.628662109375, "logps/ref_chosen": -60.60819625854492, "logps/ref_rejected": -94.56770324707031, "logps/rejected": -2044.095947265625, "loss": 1.3502, "margin_dpo/margin_mean": 589.5076904296875, "margin_dpo/margin_std": 1928.743408203125, "step": 563 }, { "KL/chosen_KL_mean": -1113.524658203125, "KL/mean": -1359.48486328125, "KL/rejected_KL_mean": -1605.445068359375, "KL/std": 1332.8668212890625, "epoch": 0.8281938325991189, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 491.92047119140625, "fcm_dpo/q_t": 0.49136167764663696, "grad_norm": 7.608941555023193, "learning_rate": 4.447860229910544e-08, "logits/chosen": -0.6324988603591919, "logits/rejected": -0.6346863508224487, "logps/chosen": -1187.79296875, "logps/ref_chosen": -74.26837921142578, "logps/ref_rejected": -93.23818969726562, "logps/rejected": -1698.6832275390625, "loss": 1.3556, "margin_dpo/margin_mean": 491.9204406738281, "margin_dpo/margin_std": 1603.78125, "step": 564 }, { "KL/chosen_KL_mean": -1407.1187744140625, "KL/mean": -1665.321044921875, "KL/rejected_KL_mean": -1923.5233154296875, "KL/std": 1837.506103515625, "epoch": 0.8296622613803231, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 516.404541015625, "fcm_dpo/q_t": 0.4908890128135681, "grad_norm": 13.449864387512207, "learning_rate": 4.375063135042445e-08, "logits/chosen": -0.663166880607605, "logits/rejected": -0.6663168668746948, "logps/chosen": -1476.138671875, "logps/ref_chosen": -69.0199203491211, "logps/ref_rejected": -85.7789306640625, "logps/rejected": -2009.30224609375, "loss": 1.3573, "margin_dpo/margin_mean": 516.404541015625, "margin_dpo/margin_std": 2329.1201171875, "step": 565 }, { "KL/chosen_KL_mean": -1349.890869140625, "KL/mean": -1728.78564453125, "KL/rejected_KL_mean": -2107.68017578125, "KL/std": 1578.119873046875, "epoch": 0.8311306901615272, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 757.7894287109375, "fcm_dpo/q_t": 0.48675039410591125, "grad_norm": 5.23419713973999, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -0.6609238386154175, "logits/rejected": -0.6840362548828125, "logps/chosen": -1416.436279296875, "logps/ref_chosen": -66.5453109741211, "logps/ref_rejected": -103.86932373046875, "logps/rejected": -2211.5498046875, "loss": 1.3386, "margin_dpo/margin_mean": 757.7894287109375, "margin_dpo/margin_std": 1948.74609375, "step": 566 }, { "KL/chosen_KL_mean": -1095.9215087890625, "KL/mean": -1309.902099609375, "KL/rejected_KL_mean": -1523.882568359375, "KL/std": 1136.513427734375, "epoch": 0.8325991189427313, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 427.9610595703125, "fcm_dpo/q_t": 0.4924955368041992, "grad_norm": 6.573452472686768, "learning_rate": 4.231101308059165e-08, "logits/chosen": -0.6667437553405762, "logits/rejected": -0.676659107208252, "logps/chosen": -1148.77978515625, "logps/ref_chosen": -52.85829544067383, "logps/ref_rejected": -85.37095642089844, "logps/rejected": -1609.25341796875, "loss": 1.3586, "margin_dpo/margin_mean": 427.9610595703125, "margin_dpo/margin_std": 1303.414306640625, "step": 567 }, { "KL/chosen_KL_mean": -958.6199951171875, "KL/mean": -1419.5540771484375, "KL/rejected_KL_mean": -1880.4881591796875, "KL/std": 1383.7803955078125, "epoch": 0.8340675477239354, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 921.8681030273438, "fcm_dpo/q_t": 0.4839458167552948, "grad_norm": 5.986705780029297, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -0.6187951564788818, "logits/rejected": -0.6642179489135742, "logps/chosen": -1003.8123779296875, "logps/ref_chosen": -45.1923828125, "logps/ref_rejected": -89.09236907958984, "logps/rejected": -1969.58056640625, "loss": 1.3265, "margin_dpo/margin_mean": 921.8680419921875, "margin_dpo/margin_std": 1768.663818359375, "step": 568 }, { "KL/chosen_KL_mean": -1272.758056640625, "KL/mean": -1551.21484375, "KL/rejected_KL_mean": -1829.671630859375, "KL/std": 1590.348876953125, "epoch": 0.8355359765051396, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 556.9136962890625, "fcm_dpo/q_t": 0.4902852773666382, "grad_norm": 6.188879013061523, "learning_rate": 4.089328585837512e-08, "logits/chosen": -0.6311490535736084, "logits/rejected": -0.6400599479675293, "logps/chosen": -1336.478515625, "logps/ref_chosen": -63.72056198120117, "logps/ref_rejected": -79.10325622558594, "logps/rejected": -1908.77490234375, "loss": 1.3522, "margin_dpo/margin_mean": 556.9136962890625, "margin_dpo/margin_std": 1879.6124267578125, "step": 569 }, { "KL/chosen_KL_mean": -1346.568603515625, "KL/mean": -1499.462890625, "KL/rejected_KL_mean": -1652.357177734375, "KL/std": 1435.96875, "epoch": 0.8370044052863436, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 305.7886962890625, "fcm_dpo/q_t": 0.4945737421512604, "grad_norm": 8.600349426269531, "learning_rate": 4.019267817841834e-08, "logits/chosen": -0.6977522373199463, "logits/rejected": -0.6893934011459351, "logps/chosen": -1408.18310546875, "logps/ref_chosen": -61.61454391479492, "logps/ref_rejected": -82.14186096191406, "logps/rejected": -1734.4991455078125, "loss": 1.3696, "margin_dpo/margin_mean": 305.7887268066406, "margin_dpo/margin_std": 1899.91796875, "step": 570 }, { "KL/chosen_KL_mean": -1334.66357421875, "KL/mean": -1602.7347412109375, "KL/rejected_KL_mean": -1870.8057861328125, "KL/std": 1609.7418212890625, "epoch": 0.8384728340675477, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 536.1421508789062, "fcm_dpo/q_t": 0.49062836170196533, "grad_norm": 9.382889747619629, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -0.6136207580566406, "logits/rejected": -0.6315397024154663, "logps/chosen": -1387.7177734375, "logps/ref_chosen": -53.05406188964844, "logps/ref_rejected": -91.33682250976562, "logps/rejected": -1962.142578125, "loss": 1.3535, "margin_dpo/margin_mean": 536.1421508789062, "margin_dpo/margin_std": 1897.035400390625, "step": 571 }, { "KL/chosen_KL_mean": -1041.9559326171875, "KL/mean": -1387.749267578125, "KL/rejected_KL_mean": -1733.54248046875, "KL/std": 1231.0006103515625, "epoch": 0.8399412628487518, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 691.5866088867188, "fcm_dpo/q_t": 0.48791706562042236, "grad_norm": 5.244479179382324, "learning_rate": 3.880806698864086e-08, "logits/chosen": -0.6141128540039062, "logits/rejected": -0.6465499401092529, "logps/chosen": -1090.415283203125, "logps/ref_chosen": -48.45928955078125, "logps/ref_rejected": -83.55703735351562, "logps/rejected": -1817.099609375, "loss": 1.3413, "margin_dpo/margin_mean": 691.5865478515625, "margin_dpo/margin_std": 1555.69189453125, "step": 572 }, { "KL/chosen_KL_mean": -1082.886474609375, "KL/mean": -1360.33203125, "KL/rejected_KL_mean": -1637.777587890625, "KL/std": 1212.4599609375, "epoch": 0.8414096916299559, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 554.8909912109375, "fcm_dpo/q_t": 0.49030232429504395, "grad_norm": 9.327940940856934, "learning_rate": 3.812409996461275e-08, "logits/chosen": -0.606751561164856, "logits/rejected": -0.6213085651397705, "logps/chosen": -1134.5091552734375, "logps/ref_chosen": -51.62262725830078, "logps/ref_rejected": -85.32499694824219, "logps/rejected": -1723.1025390625, "loss": 1.3498, "margin_dpo/margin_mean": 554.8909912109375, "margin_dpo/margin_std": 1286.421142578125, "step": 573 }, { "KL/chosen_KL_mean": -1081.611328125, "KL/mean": -1515.553466796875, "KL/rejected_KL_mean": -1949.4954833984375, "KL/std": 1357.4376220703125, "epoch": 0.8428781204111601, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 867.8842163085938, "fcm_dpo/q_t": 0.4848443865776062, "grad_norm": 5.942359447479248, "learning_rate": 3.74457160675965e-08, "logits/chosen": -0.6815818548202515, "logits/rejected": -0.7281221151351929, "logps/chosen": -1132.65576171875, "logps/ref_chosen": -51.04446029663086, "logps/ref_rejected": -92.80640411376953, "logps/rejected": -2042.302001953125, "loss": 1.3294, "margin_dpo/margin_mean": 867.8841552734375, "margin_dpo/margin_std": 1571.0694580078125, "step": 574 }, { "KL/chosen_KL_mean": -1305.66552734375, "KL/mean": -1599.9190673828125, "KL/rejected_KL_mean": -1894.1724853515625, "KL/std": 1470.6298828125, "epoch": 0.8443465491923642, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 588.5068359375, "fcm_dpo/q_t": 0.48974353075027466, "grad_norm": 7.068485736846924, "learning_rate": 3.677293317363864e-08, "logits/chosen": -0.6297034025192261, "logits/rejected": -0.6442345380783081, "logps/chosen": -1377.455810546875, "logps/ref_chosen": -71.7901382446289, "logps/ref_rejected": -95.38619995117188, "logps/rejected": -1989.55859375, "loss": 1.3495, "margin_dpo/margin_mean": 588.5068359375, "margin_dpo/margin_std": 1797.72509765625, "step": 575 }, { "KL/chosen_KL_mean": -1039.58740234375, "KL/mean": -1412.387451171875, "KL/rejected_KL_mean": -1785.1873779296875, "KL/std": 1391.4742431640625, "epoch": 0.8458149779735683, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 745.5999755859375, "fcm_dpo/q_t": 0.48703137040138245, "grad_norm": 12.236254692077637, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -0.6339189410209656, "logits/rejected": -0.6675552129745483, "logps/chosen": -1093.850341796875, "logps/ref_chosen": -54.262962341308594, "logps/ref_rejected": -100.75428009033203, "logps/rejected": -1885.941650390625, "loss": 1.3381, "margin_dpo/margin_mean": 745.5999755859375, "margin_dpo/margin_std": 1672.704345703125, "step": 576 }, { "KL/chosen_KL_mean": -1099.926513671875, "KL/mean": -1301.64794921875, "KL/rejected_KL_mean": -1503.369384765625, "KL/std": 1184.22705078125, "epoch": 0.8472834067547724, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 403.4429626464844, "fcm_dpo/q_t": 0.49293655157089233, "grad_norm": 6.366522312164307, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -0.6079691648483276, "logits/rejected": -0.6053318381309509, "logps/chosen": -1161.836181640625, "logps/ref_chosen": -61.909706115722656, "logps/ref_rejected": -84.07069396972656, "logps/rejected": -1587.440185546875, "loss": 1.3608, "margin_dpo/margin_mean": 403.4429626464844, "margin_dpo/margin_std": 1407.972412109375, "step": 577 }, { "KL/chosen_KL_mean": -874.846923828125, "KL/mean": -1265.8436279296875, "KL/rejected_KL_mean": -1656.84033203125, "KL/std": 1145.15771484375, "epoch": 0.8487518355359766, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 781.993408203125, "fcm_dpo/q_t": 0.48631131649017334, "grad_norm": 4.333878517150879, "learning_rate": 3.478836705390808e-08, "logits/chosen": -0.5466019511222839, "logits/rejected": -0.5783486366271973, "logps/chosen": -924.110595703125, "logps/ref_chosen": -49.26368713378906, "logps/ref_rejected": -83.4362564086914, "logps/rejected": -1740.276611328125, "loss": 1.3339, "margin_dpo/margin_mean": 781.993408203125, "margin_dpo/margin_std": 1137.68408203125, "step": 578 }, { "KL/chosen_KL_mean": -1144.806396484375, "KL/mean": -1350.88134765625, "KL/rejected_KL_mean": -1556.956298828125, "KL/std": 1468.2509765625, "epoch": 0.8502202643171806, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 412.1499938964844, "fcm_dpo/q_t": 0.49288052320480347, "grad_norm": 6.493415355682373, "learning_rate": 3.41381639738331e-08, "logits/chosen": -0.609187662601471, "logits/rejected": -0.6248157024383545, "logps/chosen": -1203.6922607421875, "logps/ref_chosen": -58.88581848144531, "logps/ref_rejected": -94.78762817382812, "logps/rejected": -1651.743896484375, "loss": 1.362, "margin_dpo/margin_mean": 412.14996337890625, "margin_dpo/margin_std": 1850.548095703125, "step": 579 }, { "KL/chosen_KL_mean": -775.547119140625, "KL/mean": -1131.787109375, "KL/rejected_KL_mean": -1488.02685546875, "KL/std": 1250.650390625, "epoch": 0.8516886930983847, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 712.479736328125, "fcm_dpo/q_t": 0.48757392168045044, "grad_norm": 5.055511951446533, "learning_rate": 3.349364905389032e-08, "logits/chosen": -0.5566902160644531, "logits/rejected": -0.5791162848472595, "logps/chosen": -824.2539672851562, "logps/ref_chosen": -48.70683670043945, "logps/ref_rejected": -81.7583999633789, "logps/rejected": -1569.7852783203125, "loss": 1.3396, "margin_dpo/margin_mean": 712.479736328125, "margin_dpo/margin_std": 1466.5977783203125, "step": 580 }, { "KL/chosen_KL_mean": -1139.18310546875, "KL/mean": -1376.47412109375, "KL/rejected_KL_mean": -1613.764892578125, "KL/std": 1217.5745849609375, "epoch": 0.8531571218795888, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 474.5818786621094, "fcm_dpo/q_t": 0.4917064607143402, "grad_norm": 6.296034812927246, "learning_rate": 3.285483927764726e-08, "logits/chosen": -0.6595457196235657, "logits/rejected": -0.6783765554428101, "logps/chosen": -1201.4053955078125, "logps/ref_chosen": -62.22235107421875, "logps/ref_rejected": -91.73568725585938, "logps/rejected": -1705.500732421875, "loss": 1.3559, "margin_dpo/margin_mean": 474.5818786621094, "margin_dpo/margin_std": 1438.438232421875, "step": 581 }, { "KL/chosen_KL_mean": -1091.2376708984375, "KL/mean": -1487.45654296875, "KL/rejected_KL_mean": -1883.6754150390625, "KL/std": 1237.882080078125, "epoch": 0.8546255506607929, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 792.4378051757812, "fcm_dpo/q_t": 0.4861498475074768, "grad_norm": 11.559684753417969, "learning_rate": 3.222175147833556e-08, "logits/chosen": -0.6552462577819824, "logits/rejected": -0.6873466968536377, "logps/chosen": -1149.46630859375, "logps/ref_chosen": -58.228660583496094, "logps/ref_rejected": -110.06959533691406, "logps/rejected": -1993.7449951171875, "loss": 1.3342, "margin_dpo/margin_mean": 792.437744140625, "margin_dpo/margin_std": 1396.3656005859375, "step": 582 }, { "KL/chosen_KL_mean": -1086.141357421875, "KL/mean": -1331.1463623046875, "KL/rejected_KL_mean": -1576.1513671875, "KL/std": 1458.034423828125, "epoch": 0.856093979441997, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 490.01007080078125, "fcm_dpo/q_t": 0.4915131628513336, "grad_norm": 12.745737075805664, "learning_rate": 3.159440233840763e-08, "logits/chosen": -0.6324259042739868, "logits/rejected": -0.6466102600097656, "logps/chosen": -1143.004150390625, "logps/ref_chosen": -56.86286163330078, "logps/ref_rejected": -88.4039306640625, "logps/rejected": -1664.5552978515625, "loss": 1.3569, "margin_dpo/margin_mean": 490.0100402832031, "margin_dpo/margin_std": 1838.203125, "step": 583 }, { "KL/chosen_KL_mean": -1118.363037109375, "KL/mean": -1506.4857177734375, "KL/rejected_KL_mean": -1894.6083984375, "KL/std": 1478.693359375, "epoch": 0.8575624082232012, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 776.2452392578125, "fcm_dpo/q_t": 0.4864642024040222, "grad_norm": 16.514495849609375, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -0.6495020389556885, "logits/rejected": -0.6715532541275024, "logps/chosen": -1175.2637939453125, "logps/ref_chosen": -56.90068054199219, "logps/ref_rejected": -97.63606262207031, "logps/rejected": -1992.244384765625, "loss": 1.3358, "margin_dpo/margin_mean": 776.2452392578125, "margin_dpo/margin_std": 1611.7119140625, "step": 584 }, { "KL/chosen_KL_mean": -1159.185546875, "KL/mean": -1560.73681640625, "KL/rejected_KL_mean": -1962.2880859375, "KL/std": 1544.73486328125, "epoch": 0.8590308370044053, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 803.1026611328125, "fcm_dpo/q_t": 0.4860289692878723, "grad_norm": 4.359808444976807, "learning_rate": 3.035698600998121e-08, "logits/chosen": -0.6542025804519653, "logits/rejected": -0.6867420077323914, "logps/chosen": -1220.159423828125, "logps/ref_chosen": -60.973968505859375, "logps/ref_rejected": -84.16952514648438, "logps/rejected": -2046.4576416015625, "loss": 1.3362, "margin_dpo/margin_mean": 803.1026611328125, "margin_dpo/margin_std": 2070.01416015625, "step": 585 }, { "KL/chosen_KL_mean": -1311.316162109375, "KL/mean": -1604.206787109375, "KL/rejected_KL_mean": -1897.097412109375, "KL/std": 1670.98046875, "epoch": 0.8604992657856094, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 585.7813720703125, "fcm_dpo/q_t": 0.4897732436656952, "grad_norm": 6.482140064239502, "learning_rate": 2.974695142855388e-08, "logits/chosen": -0.6228867173194885, "logits/rejected": -0.6541084051132202, "logps/chosen": -1368.171630859375, "logps/ref_chosen": -56.85559844970703, "logps/ref_rejected": -91.80261993408203, "logps/rejected": -1988.9000244140625, "loss": 1.3486, "margin_dpo/margin_mean": 585.7813720703125, "margin_dpo/margin_std": 1524.811767578125, "step": 586 }, { "KL/chosen_KL_mean": -844.4599609375, "KL/mean": -1170.924560546875, "KL/rejected_KL_mean": -1497.38916015625, "KL/std": 1240.437255859375, "epoch": 0.8619676945668135, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 652.92919921875, "fcm_dpo/q_t": 0.48856592178344727, "grad_norm": 9.642579078674316, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -0.6443250775337219, "logits/rejected": -0.6658428907394409, "logps/chosen": -889.151611328125, "logps/ref_chosen": -44.69159698486328, "logps/ref_rejected": -82.62385559082031, "logps/rejected": -1580.0130615234375, "loss": 1.3437, "margin_dpo/margin_mean": 652.92919921875, "margin_dpo/margin_std": 1477.093994140625, "step": 587 }, { "KL/chosen_KL_mean": -1141.727294921875, "KL/mean": -1513.7247314453125, "KL/rejected_KL_mean": -1885.722412109375, "KL/std": 1343.573486328125, "epoch": 0.8634361233480177, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 743.9951171875, "fcm_dpo/q_t": 0.48701196908950806, "grad_norm": 7.112242221832275, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -0.6173974275588989, "logits/rejected": -0.6490628719329834, "logps/chosen": -1192.022216796875, "logps/ref_chosen": -50.29494857788086, "logps/ref_rejected": -107.36988067626953, "logps/rejected": -1993.0921630859375, "loss": 1.3384, "margin_dpo/margin_mean": 743.9951171875, "margin_dpo/margin_std": 1684.069091796875, "step": 588 }, { "KL/chosen_KL_mean": -1334.051513671875, "KL/mean": -1767.8013916015625, "KL/rejected_KL_mean": -2201.55126953125, "KL/std": 1667.269287109375, "epoch": 0.8649045521292217, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 867.4995727539062, "fcm_dpo/q_t": 0.48490506410598755, "grad_norm": 7.0310516357421875, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -0.6013877391815186, "logits/rejected": -0.6371986269950867, "logps/chosen": -1393.9814453125, "logps/ref_chosen": -59.929908752441406, "logps/ref_rejected": -111.65534973144531, "logps/rejected": -2313.20654296875, "loss": 1.3308, "margin_dpo/margin_mean": 867.4996337890625, "margin_dpo/margin_std": 1884.604736328125, "step": 589 }, { "KL/chosen_KL_mean": -1027.572998046875, "KL/mean": -1449.776611328125, "KL/rejected_KL_mean": -1871.980224609375, "KL/std": 1585.0914306640625, "epoch": 0.8663729809104258, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 844.4072875976562, "fcm_dpo/q_t": 0.4853343367576599, "grad_norm": 6.130049228668213, "learning_rate": 2.736501028272095e-08, "logits/chosen": -0.6168742775917053, "logits/rejected": -0.6527448892593384, "logps/chosen": -1083.3828125, "logps/ref_chosen": -55.80979537963867, "logps/ref_rejected": -106.06282043457031, "logps/rejected": -1978.0430908203125, "loss": 1.3328, "margin_dpo/margin_mean": 844.4072265625, "margin_dpo/margin_std": 1984.2763671875, "step": 590 }, { "KL/chosen_KL_mean": -1019.3551025390625, "KL/mean": -1367.507568359375, "KL/rejected_KL_mean": -1715.6600341796875, "KL/std": 1360.0946044921875, "epoch": 0.8678414096916299, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 696.304931640625, "fcm_dpo/q_t": 0.4878562390804291, "grad_norm": 6.305633068084717, "learning_rate": 2.678415274939408e-08, "logits/chosen": -0.6568745970726013, "logits/rejected": -0.6684634685516357, "logps/chosen": -1075.595703125, "logps/ref_chosen": -56.24061965942383, "logps/ref_rejected": -83.78629302978516, "logps/rejected": -1799.4462890625, "loss": 1.342, "margin_dpo/margin_mean": 696.304931640625, "margin_dpo/margin_std": 1778.0966796875, "step": 591 }, { "KL/chosen_KL_mean": -1184.225830078125, "KL/mean": -1476.70849609375, "KL/rejected_KL_mean": -1769.1912841796875, "KL/std": 1288.901123046875, "epoch": 0.869309838472834, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 584.965576171875, "fcm_dpo/q_t": 0.4897754192352295, "grad_norm": 8.049051284790039, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -0.6074360609054565, "logits/rejected": -0.6240702271461487, "logps/chosen": -1232.166015625, "logps/ref_chosen": -47.94025421142578, "logps/ref_rejected": -75.73287963867188, "logps/rejected": -1844.9241943359375, "loss": 1.3498, "margin_dpo/margin_mean": 584.965576171875, "margin_dpo/margin_std": 1812.1029052734375, "step": 592 }, { "KL/chosen_KL_mean": -1095.839599609375, "KL/mean": -1293.16259765625, "KL/rejected_KL_mean": -1490.4853515625, "KL/std": 1231.2266845703125, "epoch": 0.8707782672540382, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 394.645751953125, "fcm_dpo/q_t": 0.493096262216568, "grad_norm": 4.078222751617432, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.6004587411880493, "logits/rejected": -0.6096137166023254, "logps/chosen": -1144.5303955078125, "logps/ref_chosen": -48.690757751464844, "logps/ref_rejected": -60.90800094604492, "logps/rejected": -1551.3934326171875, "loss": 1.3614, "margin_dpo/margin_mean": 394.645751953125, "margin_dpo/margin_std": 1439.93701171875, "step": 593 }, { "KL/chosen_KL_mean": -1057.113037109375, "KL/mean": -1357.0494384765625, "KL/rejected_KL_mean": -1656.9857177734375, "KL/std": 1238.75146484375, "epoch": 0.8722466960352423, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 599.8726806640625, "fcm_dpo/q_t": 0.4895210862159729, "grad_norm": 7.381805419921875, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -0.6215956211090088, "logits/rejected": -0.6254661083221436, "logps/chosen": -1112.0478515625, "logps/ref_chosen": -54.93488693237305, "logps/ref_rejected": -86.09967803955078, "logps/rejected": -1743.08544921875, "loss": 1.3476, "margin_dpo/margin_mean": 599.8727416992188, "margin_dpo/margin_std": 1491.129150390625, "step": 594 }, { "KL/chosen_KL_mean": -986.426513671875, "KL/mean": -1438.9229736328125, "KL/rejected_KL_mean": -1891.41943359375, "KL/std": 1537.6827392578125, "epoch": 0.8737151248164464, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 904.9927978515625, "fcm_dpo/q_t": 0.4843044877052307, "grad_norm": 8.513747215270996, "learning_rate": 2.451969280180849e-08, "logits/chosen": -0.6116786003112793, "logits/rejected": -0.6459616422653198, "logps/chosen": -1035.846923828125, "logps/ref_chosen": -49.4204216003418, "logps/ref_rejected": -80.62731170654297, "logps/rejected": -1972.046630859375, "loss": 1.3286, "margin_dpo/margin_mean": 904.992919921875, "margin_dpo/margin_std": 1967.465087890625, "step": 595 }, { "KL/chosen_KL_mean": -1317.722900390625, "KL/mean": -1460.748779296875, "KL/rejected_KL_mean": -1603.77490234375, "KL/std": 1665.4937744140625, "epoch": 0.8751835535976505, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 286.0518798828125, "fcm_dpo/q_t": 0.49503082036972046, "grad_norm": 5.618963241577148, "learning_rate": 2.396839494982103e-08, "logits/chosen": -0.6746952533721924, "logits/rejected": -0.6740427017211914, "logps/chosen": -1377.5145263671875, "logps/ref_chosen": -59.791683197021484, "logps/ref_rejected": -80.09111785888672, "logps/rejected": -1683.865966796875, "loss": 1.3718, "margin_dpo/margin_mean": 286.0519104003906, "margin_dpo/margin_std": 2089.884521484375, "step": 596 }, { "KL/chosen_KL_mean": -1239.4058837890625, "KL/mean": -1711.820556640625, "KL/rejected_KL_mean": -2184.235107421875, "KL/std": 1657.8519287109375, "epoch": 0.8766519823788547, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 944.8291625976562, "fcm_dpo/q_t": 0.4835811257362366, "grad_norm": 6.500996112823486, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -0.6523314118385315, "logits/rejected": -0.6965519785881042, "logps/chosen": -1296.666748046875, "logps/ref_chosen": -57.26078796386719, "logps/ref_rejected": -100.6937255859375, "logps/rejected": -2284.9287109375, "loss": 1.3265, "margin_dpo/margin_mean": 944.8292236328125, "margin_dpo/margin_std": 1901.935791015625, "step": 597 }, { "KL/chosen_KL_mean": -1068.144287109375, "KL/mean": -1389.80712890625, "KL/rejected_KL_mean": -1711.4698486328125, "KL/std": 1323.55322265625, "epoch": 0.8781204111600588, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 643.3255615234375, "fcm_dpo/q_t": 0.4887608289718628, "grad_norm": 4.988048076629639, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -0.602400541305542, "logits/rejected": -0.6199566721916199, "logps/chosen": -1120.662841796875, "logps/ref_chosen": -52.51850509643555, "logps/ref_rejected": -89.44385528564453, "logps/rejected": -1800.913818359375, "loss": 1.3452, "margin_dpo/margin_mean": 643.3255615234375, "margin_dpo/margin_std": 1611.5604248046875, "step": 598 }, { "KL/chosen_KL_mean": -1212.7427978515625, "KL/mean": -1437.69384765625, "KL/rejected_KL_mean": -1662.644775390625, "KL/std": 1404.40869140625, "epoch": 0.8795888399412628, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 449.90191650390625, "fcm_dpo/q_t": 0.4921151101589203, "grad_norm": 5.333652973175049, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -0.6382207274436951, "logits/rejected": -0.6507253050804138, "logps/chosen": -1262.5455322265625, "logps/ref_chosen": -49.802677154541016, "logps/ref_rejected": -82.978515625, "logps/rejected": -1745.623291015625, "loss": 1.3583, "margin_dpo/margin_mean": 449.90191650390625, "margin_dpo/margin_std": 1570.1396484375, "step": 599 }, { "KL/chosen_KL_mean": -1502.628173828125, "KL/mean": -1721.712646484375, "KL/rejected_KL_mean": -1940.797119140625, "KL/std": 1682.96484375, "epoch": 0.8810572687224669, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 438.1690673828125, "fcm_dpo/q_t": 0.49244120717048645, "grad_norm": 6.994654655456543, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -0.6980600357055664, "logits/rejected": -0.7049777507781982, "logps/chosen": -1569.06298828125, "logps/ref_chosen": -66.43487548828125, "logps/ref_rejected": -85.45649719238281, "logps/rejected": -2026.253662109375, "loss": 1.3628, "margin_dpo/margin_mean": 438.1690368652344, "margin_dpo/margin_std": 2316.57080078125, "step": 600 }, { "KL/chosen_KL_mean": -1398.1495361328125, "KL/mean": -1735.676513671875, "KL/rejected_KL_mean": -2073.203369140625, "KL/std": 1529.931640625, "epoch": 0.882525697503671, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 675.0538330078125, "fcm_dpo/q_t": 0.4882049858570099, "grad_norm": 6.039072513580322, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -0.6777108907699585, "logits/rejected": -0.6996618509292603, "logps/chosen": -1457.283203125, "logps/ref_chosen": -59.13361358642578, "logps/ref_rejected": -94.69093322753906, "logps/rejected": -2167.894287109375, "loss": 1.344, "margin_dpo/margin_mean": 675.0538330078125, "margin_dpo/margin_std": 1895.3690185546875, "step": 601 }, { "KL/chosen_KL_mean": -859.4080810546875, "KL/mean": -1380.189697265625, "KL/rejected_KL_mean": -1900.97119140625, "KL/std": 1388.6561279296875, "epoch": 0.8839941262848752, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1041.5631103515625, "fcm_dpo/q_t": 0.4818532466888428, "grad_norm": 14.612542152404785, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -0.6874485015869141, "logits/rejected": -0.7312799692153931, "logps/chosen": -908.0015869140625, "logps/ref_chosen": -48.59352111816406, "logps/ref_rejected": -87.6685562133789, "logps/rejected": -1988.6396484375, "loss": 1.3185, "margin_dpo/margin_mean": 1041.5631103515625, "margin_dpo/margin_std": 1663.5296630859375, "step": 602 }, { "KL/chosen_KL_mean": -1425.88330078125, "KL/mean": -1779.10498046875, "KL/rejected_KL_mean": -2132.32666015625, "KL/std": 1935.75244140625, "epoch": 0.8854625550660793, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 706.4432983398438, "fcm_dpo/q_t": 0.48793095350265503, "grad_norm": 14.034811019897461, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -0.7051316499710083, "logits/rejected": -0.7167890071868896, "logps/chosen": -1496.2979736328125, "logps/ref_chosen": -70.41461944580078, "logps/ref_rejected": -100.32559967041016, "logps/rejected": -2232.65234375, "loss": 1.3446, "margin_dpo/margin_mean": 706.4432983398438, "margin_dpo/margin_std": 2401.55517578125, "step": 603 }, { "KL/chosen_KL_mean": -1154.302001953125, "KL/mean": -1677.122802734375, "KL/rejected_KL_mean": -2199.943603515625, "KL/std": 1617.4171142578125, "epoch": 0.8869309838472834, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1045.6416015625, "fcm_dpo/q_t": 0.4818729758262634, "grad_norm": 6.103328704833984, "learning_rate": 1.977362051376158e-08, "logits/chosen": -0.6412931680679321, "logits/rejected": -0.6906075477600098, "logps/chosen": -1200.760009765625, "logps/ref_chosen": -46.45808029174805, "logps/ref_rejected": -91.8544921875, "logps/rejected": -2291.798095703125, "loss": 1.3206, "margin_dpo/margin_mean": 1045.6414794921875, "margin_dpo/margin_std": 2262.8173828125, "step": 604 }, { "KL/chosen_KL_mean": -1212.1982421875, "KL/mean": -1576.411865234375, "KL/rejected_KL_mean": -1940.62548828125, "KL/std": 1592.7261962890625, "epoch": 0.8883994126284875, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 728.4273681640625, "fcm_dpo/q_t": 0.48738065361976624, "grad_norm": 7.565073013305664, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -0.6622729301452637, "logits/rejected": -0.6903505325317383, "logps/chosen": -1278.447509765625, "logps/ref_chosen": -66.24933624267578, "logps/ref_rejected": -102.30496978759766, "logps/rejected": -2042.930419921875, "loss": 1.3417, "margin_dpo/margin_mean": 728.4273681640625, "margin_dpo/margin_std": 2157.499267578125, "step": 605 }, { "KL/chosen_KL_mean": -1284.903564453125, "KL/mean": -1668.8912353515625, "KL/rejected_KL_mean": -2052.87890625, "KL/std": 2009.388916015625, "epoch": 0.8898678414096917, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 767.9752807617188, "fcm_dpo/q_t": 0.48712000250816345, "grad_norm": 10.69570255279541, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -0.6595591306686401, "logits/rejected": -0.695212721824646, "logps/chosen": -1339.72265625, "logps/ref_chosen": -54.819122314453125, "logps/ref_rejected": -98.37146759033203, "logps/rejected": -2151.250244140625, "loss": 1.3417, "margin_dpo/margin_mean": 767.9752807617188, "margin_dpo/margin_std": 2621.770263671875, "step": 606 }, { "KL/chosen_KL_mean": -1284.68359375, "KL/mean": -1566.37744140625, "KL/rejected_KL_mean": -1848.0712890625, "KL/std": 1655.87255859375, "epoch": 0.8913362701908958, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 563.3876342773438, "fcm_dpo/q_t": 0.4901534914970398, "grad_norm": 10.138040542602539, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -0.6720389127731323, "logits/rejected": -0.6777141690254211, "logps/chosen": -1342.767578125, "logps/ref_chosen": -58.08403778076172, "logps/ref_rejected": -79.777099609375, "logps/rejected": -1927.848388671875, "loss": 1.3511, "margin_dpo/margin_mean": 563.3876953125, "margin_dpo/margin_std": 1770.9501953125, "step": 607 }, { "KL/chosen_KL_mean": -1165.683837890625, "KL/mean": -1508.021240234375, "KL/rejected_KL_mean": -1850.35888671875, "KL/std": 1325.984130859375, "epoch": 0.8928046989720999, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 684.6749877929688, "fcm_dpo/q_t": 0.48802483081817627, "grad_norm": 5.801738262176514, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -0.6812508702278137, "logits/rejected": -0.7010186910629272, "logps/chosen": -1223.1346435546875, "logps/ref_chosen": -57.450836181640625, "logps/ref_rejected": -94.77339172363281, "logps/rejected": -1945.1322021484375, "loss": 1.3419, "margin_dpo/margin_mean": 684.675048828125, "margin_dpo/margin_std": 1577.7777099609375, "step": 608 }, { "KL/chosen_KL_mean": -1125.0963134765625, "KL/mean": -1520.38818359375, "KL/rejected_KL_mean": -1915.679931640625, "KL/std": 1332.13232421875, "epoch": 0.8942731277533039, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 790.583740234375, "fcm_dpo/q_t": 0.48619094491004944, "grad_norm": 10.305415153503418, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -0.7046061754226685, "logits/rejected": -0.7308906316757202, "logps/chosen": -1183.901611328125, "logps/ref_chosen": -58.805355072021484, "logps/ref_rejected": -88.81600952148438, "logps/rejected": -2004.4959716796875, "loss": 1.3355, "margin_dpo/margin_mean": 790.583740234375, "margin_dpo/margin_std": 1792.01611328125, "step": 609 }, { "KL/chosen_KL_mean": -1282.007568359375, "KL/mean": -1432.61962890625, "KL/rejected_KL_mean": -1583.231689453125, "KL/std": 1448.6640625, "epoch": 0.895741556534508, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 301.22406005859375, "fcm_dpo/q_t": 0.49473780393600464, "grad_norm": 11.932384490966797, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -0.6814596652984619, "logits/rejected": -0.6760205030441284, "logps/chosen": -1347.7025146484375, "logps/ref_chosen": -65.69503784179688, "logps/ref_rejected": -83.40538787841797, "logps/rejected": -1666.636962890625, "loss": 1.3701, "margin_dpo/margin_mean": 301.22406005859375, "margin_dpo/margin_std": 1961.03515625, "step": 610 }, { "KL/chosen_KL_mean": -1148.4498291015625, "KL/mean": -1598.172607421875, "KL/rejected_KL_mean": -2047.8955078125, "KL/std": 1594.612060546875, "epoch": 0.8972099853157122, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 899.4454956054688, "fcm_dpo/q_t": 0.48433127999305725, "grad_norm": 6.665002822875977, "learning_rate": 1.6421423736208e-08, "logits/chosen": -0.6560553312301636, "logits/rejected": -0.6957667469978333, "logps/chosen": -1201.04931640625, "logps/ref_chosen": -52.59946823120117, "logps/ref_rejected": -86.33099365234375, "logps/rejected": -2134.226318359375, "loss": 1.3283, "margin_dpo/margin_mean": 899.4454345703125, "margin_dpo/margin_std": 1819.7413330078125, "step": 611 }, { "KL/chosen_KL_mean": -1225.57861328125, "KL/mean": -1524.23291015625, "KL/rejected_KL_mean": -1822.88720703125, "KL/std": 1422.014404296875, "epoch": 0.8986784140969163, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 597.30859375, "fcm_dpo/q_t": 0.4895847737789154, "grad_norm": 8.166829109191895, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -0.6424583792686462, "logits/rejected": -0.671108067035675, "logps/chosen": -1284.90234375, "logps/ref_chosen": -59.32372283935547, "logps/ref_rejected": -88.31239318847656, "logps/rejected": -1911.19970703125, "loss": 1.3491, "margin_dpo/margin_mean": 597.30859375, "margin_dpo/margin_std": 1860.5745849609375, "step": 612 }, { "KL/chosen_KL_mean": -1104.078857421875, "KL/mean": -1501.256591796875, "KL/rejected_KL_mean": -1898.4345703125, "KL/std": 1444.134521484375, "epoch": 0.9001468428781204, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 794.355712890625, "fcm_dpo/q_t": 0.4861546754837036, "grad_norm": 6.351512432098389, "learning_rate": 1.551886292185553e-08, "logits/chosen": -0.6626429557800293, "logits/rejected": -0.7097995281219482, "logps/chosen": -1163.808837890625, "logps/ref_chosen": -59.72996520996094, "logps/ref_rejected": -105.10752868652344, "logps/rejected": -2003.5419921875, "loss": 1.3355, "margin_dpo/margin_mean": 794.3555908203125, "margin_dpo/margin_std": 1837.294189453125, "step": 613 }, { "KL/chosen_KL_mean": -1072.1090087890625, "KL/mean": -1519.0906982421875, "KL/rejected_KL_mean": -1966.072509765625, "KL/std": 1425.783447265625, "epoch": 0.9016152716593245, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 893.96337890625, "fcm_dpo/q_t": 0.4844154119491577, "grad_norm": 8.47018051147461, "learning_rate": 1.507684480352292e-08, "logits/chosen": -0.6103712320327759, "logits/rejected": -0.6593648195266724, "logps/chosen": -1125.048095703125, "logps/ref_chosen": -52.93898010253906, "logps/ref_rejected": -104.67938232421875, "logps/rejected": -2070.751953125, "loss": 1.3294, "margin_dpo/margin_mean": 893.96337890625, "margin_dpo/margin_std": 1988.4140625, "step": 614 }, { "KL/chosen_KL_mean": -1103.2506103515625, "KL/mean": -1468.0848388671875, "KL/rejected_KL_mean": -1832.919189453125, "KL/std": 1459.701171875, "epoch": 0.9030837004405287, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 729.66845703125, "fcm_dpo/q_t": 0.4873241186141968, "grad_norm": 5.5929274559021, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -0.6631003022193909, "logits/rejected": -0.687023401260376, "logps/chosen": -1169.06787109375, "logps/ref_chosen": -65.81727600097656, "logps/ref_rejected": -95.17749786376953, "logps/rejected": -1928.0966796875, "loss": 1.3401, "margin_dpo/margin_mean": 729.66845703125, "margin_dpo/margin_std": 1842.6663818359375, "step": 615 }, { "KL/chosen_KL_mean": -1339.16796875, "KL/mean": -1633.54150390625, "KL/rejected_KL_mean": -1927.914794921875, "KL/std": 1515.873779296875, "epoch": 0.9045521292217328, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 588.7468872070312, "fcm_dpo/q_t": 0.489709734916687, "grad_norm": 6.771057605743408, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -0.6890289783477783, "logits/rejected": -0.6939765214920044, "logps/chosen": -1404.3009033203125, "logps/ref_chosen": -65.13285827636719, "logps/ref_rejected": -74.70050048828125, "logps/rejected": -2002.615234375, "loss": 1.3497, "margin_dpo/margin_mean": 588.746826171875, "margin_dpo/margin_std": 1862.400390625, "step": 616 }, { "KL/chosen_KL_mean": -1498.676513671875, "KL/mean": -1554.958251953125, "KL/rejected_KL_mean": -1611.240234375, "KL/std": 1531.433837890625, "epoch": 0.9060205580029369, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.56379699707031, "fcm_dpo/q_t": 0.4979705214500427, "grad_norm": 20.132686614990234, "learning_rate": 1.378797888467345e-08, "logits/chosen": -0.6434112787246704, "logits/rejected": -0.6232435703277588, "logps/chosen": -1561.68212890625, "logps/ref_chosen": -63.005550384521484, "logps/ref_rejected": -64.234130859375, "logps/rejected": -1675.474365234375, "loss": 1.3831, "margin_dpo/margin_mean": 112.56379699707031, "margin_dpo/margin_std": 1948.6531982421875, "step": 617 }, { "KL/chosen_KL_mean": -1434.4833984375, "KL/mean": -1825.2366943359375, "KL/rejected_KL_mean": -2215.989990234375, "KL/std": 1607.75, "epoch": 0.9074889867841409, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 781.506591796875, "fcm_dpo/q_t": 0.4863564670085907, "grad_norm": 9.600302696228027, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -0.6751698851585388, "logits/rejected": -0.7124015092849731, "logps/chosen": -1501.584716796875, "logps/ref_chosen": -67.10134887695312, "logps/ref_rejected": -92.15340423583984, "logps/rejected": -2308.1435546875, "loss": 1.3373, "margin_dpo/margin_mean": 781.506591796875, "margin_dpo/margin_std": 1978.0693359375, "step": 618 }, { "KL/chosen_KL_mean": -1300.69775390625, "KL/mean": -1552.975830078125, "KL/rejected_KL_mean": -1805.2540283203125, "KL/std": 1608.731689453125, "epoch": 0.908957415565345, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 504.5563659667969, "fcm_dpo/q_t": 0.4911313056945801, "grad_norm": 7.555780410766602, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -0.6084803342819214, "logits/rejected": -0.6177768707275391, "logps/chosen": -1356.6759033203125, "logps/ref_chosen": -55.978233337402344, "logps/ref_rejected": -93.1854019165039, "logps/rejected": -1898.439453125, "loss": 1.3564, "margin_dpo/margin_mean": 504.55633544921875, "margin_dpo/margin_std": 2043.724609375, "step": 619 }, { "KL/chosen_KL_mean": -1157.4849853515625, "KL/mean": -1404.56494140625, "KL/rejected_KL_mean": -1651.644775390625, "KL/std": 1310.7586669921875, "epoch": 0.9104258443465492, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 494.15985107421875, "fcm_dpo/q_t": 0.49135249853134155, "grad_norm": 6.826314449310303, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -0.6301568746566772, "logits/rejected": -0.6424782276153564, "logps/chosen": -1217.282470703125, "logps/ref_chosen": -59.79750061035156, "logps/ref_rejected": -78.41075134277344, "logps/rejected": -1730.0556640625, "loss": 1.3558, "margin_dpo/margin_mean": 494.15985107421875, "margin_dpo/margin_std": 1771.6112060546875, "step": 620 }, { "KL/chosen_KL_mean": -1000.7633056640625, "KL/mean": -1483.132568359375, "KL/rejected_KL_mean": -1965.501708984375, "KL/std": 1498.1484375, "epoch": 0.9118942731277533, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 964.7385864257812, "fcm_dpo/q_t": 0.48318374156951904, "grad_norm": 6.330345630645752, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -0.6472010612487793, "logits/rejected": -0.6926671266555786, "logps/chosen": -1054.697021484375, "logps/ref_chosen": -53.93375778198242, "logps/ref_rejected": -88.36951446533203, "logps/rejected": -2053.871337890625, "loss": 1.3245, "margin_dpo/margin_mean": 964.7385864257812, "margin_dpo/margin_std": 1884.843017578125, "step": 621 }, { "KL/chosen_KL_mean": -1014.5686645507812, "KL/mean": -1319.3843994140625, "KL/rejected_KL_mean": -1624.2001953125, "KL/std": 1281.564697265625, "epoch": 0.9133627019089574, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 609.6315307617188, "fcm_dpo/q_t": 0.48939478397369385, "grad_norm": 6.59990930557251, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -0.6089707612991333, "logits/rejected": -0.6166027784347534, "logps/chosen": -1074.8544921875, "logps/ref_chosen": -60.28582000732422, "logps/ref_rejected": -85.51873779296875, "logps/rejected": -1709.718994140625, "loss": 1.3473, "margin_dpo/margin_mean": 609.6315307617188, "margin_dpo/margin_std": 1645.434814453125, "step": 622 }, { "KL/chosen_KL_mean": -1287.5997314453125, "KL/mean": -1454.433349609375, "KL/rejected_KL_mean": -1621.266845703125, "KL/std": 1537.554443359375, "epoch": 0.9148311306901615, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 333.66717529296875, "fcm_dpo/q_t": 0.4941401183605194, "grad_norm": 15.589156150817871, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -0.6384043097496033, "logits/rejected": -0.6354462504386902, "logps/chosen": -1351.756591796875, "logps/ref_chosen": -64.1569595336914, "logps/ref_rejected": -85.08304595947266, "logps/rejected": -1706.349853515625, "loss": 1.3675, "margin_dpo/margin_mean": 333.6671142578125, "margin_dpo/margin_std": 1856.07421875, "step": 623 }, { "KL/chosen_KL_mean": -1173.687744140625, "KL/mean": -1618.4342041015625, "KL/rejected_KL_mean": -2063.1806640625, "KL/std": 1536.404296875, "epoch": 0.9162995594713657, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 889.4931640625, "fcm_dpo/q_t": 0.4844810962677002, "grad_norm": 5.250894546508789, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -0.6779031753540039, "logits/rejected": -0.6975363492965698, "logps/chosen": -1245.6063232421875, "logps/ref_chosen": -71.91862487792969, "logps/ref_rejected": -97.13203430175781, "logps/rejected": -2160.312744140625, "loss": 1.3297, "margin_dpo/margin_mean": 889.4930419921875, "margin_dpo/margin_std": 1989.96533203125, "step": 624 }, { "KL/chosen_KL_mean": -1133.847412109375, "KL/mean": -1514.25537109375, "KL/rejected_KL_mean": -1894.663330078125, "KL/std": 1352.28173828125, "epoch": 0.9177679882525698, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 760.81591796875, "fcm_dpo/q_t": 0.4867083430290222, "grad_norm": 6.905755996704102, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -0.5888922214508057, "logits/rejected": -0.605700671672821, "logps/chosen": -1192.189453125, "logps/ref_chosen": -58.342071533203125, "logps/ref_rejected": -86.09038543701172, "logps/rejected": -1980.753662109375, "loss": 1.3361, "margin_dpo/margin_mean": 760.81591796875, "margin_dpo/margin_std": 1390.97021484375, "step": 625 }, { "KL/chosen_KL_mean": -1193.1416015625, "KL/mean": -1446.146728515625, "KL/rejected_KL_mean": -1699.152099609375, "KL/std": 1531.291259765625, "epoch": 0.9192364170337739, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 506.01043701171875, "fcm_dpo/q_t": 0.491168349981308, "grad_norm": 6.740545749664307, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -0.6106196045875549, "logits/rejected": -0.6149002909660339, "logps/chosen": -1268.254150390625, "logps/ref_chosen": -75.11260986328125, "logps/ref_rejected": -99.188720703125, "logps/rejected": -1798.3408203125, "loss": 1.3553, "margin_dpo/margin_mean": 506.01043701171875, "margin_dpo/margin_std": 1811.4095458984375, "step": 626 }, { "KL/chosen_KL_mean": -773.64111328125, "KL/mean": -1223.371337890625, "KL/rejected_KL_mean": -1673.101318359375, "KL/std": 1513.056884765625, "epoch": 0.920704845814978, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 899.4602661132812, "fcm_dpo/q_t": 0.4843396246433258, "grad_norm": 10.459403991699219, "learning_rate": 9.897955805412e-09, "logits/chosen": -0.6010231971740723, "logits/rejected": -0.6437910795211792, "logps/chosen": -821.38427734375, "logps/ref_chosen": -47.74314880371094, "logps/ref_rejected": -106.75448608398438, "logps/rejected": -1779.85595703125, "loss": 1.3293, "margin_dpo/margin_mean": 899.460205078125, "margin_dpo/margin_std": 2023.463623046875, "step": 627 }, { "KL/chosen_KL_mean": -1010.72412109375, "KL/mean": -1478.8046875, "KL/rejected_KL_mean": -1946.8851318359375, "KL/std": 1570.787109375, "epoch": 0.922173274596182, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 936.1610107421875, "fcm_dpo/q_t": 0.48373740911483765, "grad_norm": 8.426224708557129, "learning_rate": 9.543589206795238e-09, "logits/chosen": -0.6274340152740479, "logits/rejected": -0.6652277708053589, "logps/chosen": -1070.906982421875, "logps/ref_chosen": -60.182945251464844, "logps/ref_rejected": -101.55467224121094, "logps/rejected": -2048.439697265625, "loss": 1.3262, "margin_dpo/margin_mean": 936.1610717773438, "margin_dpo/margin_std": 1925.196044921875, "step": 628 }, { "KL/chosen_KL_mean": -1270.9169921875, "KL/mean": -1610.190673828125, "KL/rejected_KL_mean": -1949.46435546875, "KL/std": 1889.252197265625, "epoch": 0.9236417033773862, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 678.5474243164062, "fcm_dpo/q_t": 0.48802652955055237, "grad_norm": 6.176880359649658, "learning_rate": 9.19555885822887e-09, "logits/chosen": -0.6713994741439819, "logits/rejected": -0.6894298791885376, "logps/chosen": -1335.13037109375, "logps/ref_chosen": -64.21354675292969, "logps/ref_rejected": -91.65367126464844, "logps/rejected": -2041.1180419921875, "loss": 1.3461, "margin_dpo/margin_mean": 678.5474853515625, "margin_dpo/margin_std": 2344.418701171875, "step": 629 }, { "KL/chosen_KL_mean": -1025.02001953125, "KL/mean": -1086.37158203125, "KL/rejected_KL_mean": -1147.72314453125, "KL/std": 1131.6832275390625, "epoch": 0.9251101321585903, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.7031021118164, "fcm_dpo/q_t": 0.4978800415992737, "grad_norm": 10.462772369384766, "learning_rate": 8.85387393063622e-09, "logits/chosen": -0.6387168169021606, "logits/rejected": -0.6161863803863525, "logps/chosen": -1084.31103515625, "logps/ref_chosen": -59.29100036621094, "logps/ref_rejected": -83.59829711914062, "logps/rejected": -1231.3214111328125, "loss": 1.3809, "margin_dpo/margin_mean": 122.70311737060547, "margin_dpo/margin_std": 1551.34130859375, "step": 630 }, { "KL/chosen_KL_mean": -1031.8355712890625, "KL/mean": -1342.90185546875, "KL/rejected_KL_mean": -1653.9681396484375, "KL/std": 1206.7095947265625, "epoch": 0.9265785609397944, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 622.132568359375, "fcm_dpo/q_t": 0.48913437128067017, "grad_norm": 6.256035327911377, "learning_rate": 8.518543427732949e-09, "logits/chosen": -0.643370509147644, "logits/rejected": -0.6664153337478638, "logps/chosen": -1091.2891845703125, "logps/ref_chosen": -59.45360565185547, "logps/ref_rejected": -80.95156860351562, "logps/rejected": -1734.919677734375, "loss": 1.3464, "margin_dpo/margin_mean": 622.1326904296875, "margin_dpo/margin_std": 1622.26513671875, "step": 631 }, { "KL/chosen_KL_mean": -1001.5509643554688, "KL/mean": -1220.533935546875, "KL/rejected_KL_mean": -1439.5169677734375, "KL/std": 1116.1923828125, "epoch": 0.9280469897209985, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 437.966064453125, "fcm_dpo/q_t": 0.4923382103443146, "grad_norm": 4.83076810836792, "learning_rate": 8.189576185789637e-09, "logits/chosen": -0.6454243659973145, "logits/rejected": -0.6561754941940308, "logps/chosen": -1062.902587890625, "logps/ref_chosen": -61.35155487060547, "logps/ref_rejected": -86.16017150878906, "logps/rejected": -1525.6771240234375, "loss": 1.359, "margin_dpo/margin_mean": 437.966064453125, "margin_dpo/margin_std": 1603.9329833984375, "step": 632 }, { "KL/chosen_KL_mean": -1161.1319580078125, "KL/mean": -1383.982666015625, "KL/rejected_KL_mean": -1606.83349609375, "KL/std": 1076.3099365234375, "epoch": 0.9295154185022027, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 445.701416015625, "fcm_dpo/q_t": 0.4922114312648773, "grad_norm": 5.985357284545898, "learning_rate": 7.866980873399015e-09, "logits/chosen": -0.6461182832717896, "logits/rejected": -0.6646542549133301, "logps/chosen": -1218.41015625, "logps/ref_chosen": -57.27816390991211, "logps/ref_rejected": -91.58395385742188, "logps/rejected": -1698.4173583984375, "loss": 1.3575, "margin_dpo/margin_mean": 445.701416015625, "margin_dpo/margin_std": 1343.09716796875, "step": 633 }, { "KL/chosen_KL_mean": -1291.4398193359375, "KL/mean": -1477.098876953125, "KL/rejected_KL_mean": -1662.758056640625, "KL/std": 1391.55859375, "epoch": 0.9309838472834068, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 371.3182373046875, "fcm_dpo/q_t": 0.493512898683548, "grad_norm": 6.06099796295166, "learning_rate": 7.550765991247654e-09, "logits/chosen": -0.6100882291793823, "logits/rejected": -0.6161783337593079, "logps/chosen": -1358.058837890625, "logps/ref_chosen": -66.61896514892578, "logps/ref_rejected": -107.12564849853516, "logps/rejected": -1769.8836669921875, "loss": 1.3639, "margin_dpo/margin_mean": 371.3182373046875, "margin_dpo/margin_std": 1650.0980224609375, "step": 634 }, { "KL/chosen_KL_mean": -1296.58935546875, "KL/mean": -1485.6541748046875, "KL/rejected_KL_mean": -1674.71875, "KL/std": 1795.396728515625, "epoch": 0.9324522760646109, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 378.1294860839844, "fcm_dpo/q_t": 0.4934682250022888, "grad_norm": 10.481958389282227, "learning_rate": 7.240939871891699e-09, "logits/chosen": -0.6858574151992798, "logits/rejected": -0.6715748310089111, "logps/chosen": -1370.544921875, "logps/ref_chosen": -73.95551300048828, "logps/ref_rejected": -82.50045776367188, "logps/rejected": -1757.21923828125, "loss": 1.3643, "margin_dpo/margin_mean": 378.1295166015625, "margin_dpo/margin_std": 1888.038818359375, "step": 635 }, { "KL/chosen_KL_mean": -1168.95947265625, "KL/mean": -1345.31640625, "KL/rejected_KL_mean": -1521.6732177734375, "KL/std": 1588.704345703125, "epoch": 0.933920704845815, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 352.7136535644531, "fcm_dpo/q_t": 0.49370384216308594, "grad_norm": 7.528316020965576, "learning_rate": 6.937510679537628e-09, "logits/chosen": -0.6131812334060669, "logits/rejected": -0.6081274747848511, "logps/chosen": -1228.58837890625, "logps/ref_chosen": -59.628910064697266, "logps/ref_rejected": -81.97883605957031, "logps/rejected": -1603.652099609375, "loss": 1.3669, "margin_dpo/margin_mean": 352.713623046875, "margin_dpo/margin_std": 2045.9951171875, "step": 636 }, { "KL/chosen_KL_mean": -982.5504150390625, "KL/mean": -1417.7578125, "KL/rejected_KL_mean": -1852.9652099609375, "KL/std": 1381.57568359375, "epoch": 0.9353891336270191, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 870.4149169921875, "fcm_dpo/q_t": 0.48480871319770813, "grad_norm": 6.873878479003906, "learning_rate": 6.640486409826785e-09, "logits/chosen": -0.642035961151123, "logits/rejected": -0.6937611103057861, "logps/chosen": -1032.203125, "logps/ref_chosen": -49.652687072753906, "logps/ref_rejected": -98.40513610839844, "logps/rejected": -1951.370361328125, "loss": 1.3292, "margin_dpo/margin_mean": 870.4149169921875, "margin_dpo/margin_std": 1557.965087890625, "step": 637 }, { "KL/chosen_KL_mean": -980.171142578125, "KL/mean": -1293.06494140625, "KL/rejected_KL_mean": -1605.9588623046875, "KL/std": 1329.313720703125, "epoch": 0.9368575624082232, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 625.7876586914062, "fcm_dpo/q_t": 0.4890254735946655, "grad_norm": 4.65677547454834, "learning_rate": 6.349874889624962e-09, "logits/chosen": -0.6139806509017944, "logits/rejected": -0.6274735331535339, "logps/chosen": -1038.327880859375, "logps/ref_chosen": -58.156639099121094, "logps/ref_rejected": -79.3014907836914, "logps/rejected": -1685.26025390625, "loss": 1.3471, "margin_dpo/margin_mean": 625.78759765625, "margin_dpo/margin_std": 1827.26708984375, "step": 638 }, { "KL/chosen_KL_mean": -1459.7200927734375, "KL/mean": -1391.7880859375, "KL/rejected_KL_mean": -1323.856201171875, "KL/std": 1219.503662109375, "epoch": 0.9383259911894273, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -135.86386108398438, "fcm_dpo/q_t": 0.5023590326309204, "grad_norm": 26.913433074951172, "learning_rate": 6.065683776815933e-09, "logits/chosen": -0.6304311752319336, "logits/rejected": -0.5896961092948914, "logps/chosen": -1532.0433349609375, "logps/ref_chosen": -72.32319641113281, "logps/ref_rejected": -74.2749252319336, "logps/rejected": -1398.131103515625, "loss": 1.3987, "margin_dpo/margin_mean": -135.86386108398438, "margin_dpo/margin_std": 1491.50634765625, "step": 639 }, { "KL/chosen_KL_mean": -954.1146850585938, "KL/mean": -1476.706298828125, "KL/rejected_KL_mean": -1999.2978515625, "KL/std": 1782.7164306640625, "epoch": 0.9397944199706314, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1045.18310546875, "fcm_dpo/q_t": 0.4820125997066498, "grad_norm": 6.035217761993408, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -0.591851532459259, "logits/rejected": -0.6358239650726318, "logps/chosen": -1010.2490234375, "logps/ref_chosen": -56.13436508178711, "logps/ref_rejected": -108.60014343261719, "logps/rejected": -2107.89794921875, "loss": 1.3206, "margin_dpo/margin_mean": 1045.18310546875, "margin_dpo/margin_std": 2260.031005859375, "step": 640 }, { "KL/chosen_KL_mean": -1025.90771484375, "KL/mean": -1481.028076171875, "KL/rejected_KL_mean": -1936.1484375, "KL/std": 1427.94873046875, "epoch": 0.9412628487518355, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 910.2408447265625, "fcm_dpo/q_t": 0.4841909408569336, "grad_norm": 10.455007553100586, "learning_rate": 5.516592558795746e-09, "logits/chosen": -0.5904731750488281, "logits/rejected": -0.6200425624847412, "logps/chosen": -1090.904541015625, "logps/ref_chosen": -64.99689483642578, "logps/ref_rejected": -86.99232482910156, "logps/rejected": -2023.140869140625, "loss": 1.3278, "margin_dpo/margin_mean": 910.2408447265625, "margin_dpo/margin_std": 1894.365966796875, "step": 641 }, { "KL/chosen_KL_mean": -1221.719970703125, "KL/mean": -1628.12744140625, "KL/rejected_KL_mean": -2034.5347900390625, "KL/std": 1844.57666015625, "epoch": 0.9427312775330396, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 812.81494140625, "fcm_dpo/q_t": 0.4858717918395996, "grad_norm": 5.243577003479004, "learning_rate": 5.251706922648868e-09, "logits/chosen": -0.5993083715438843, "logits/rejected": -0.6353590488433838, "logps/chosen": -1287.4091796875, "logps/ref_chosen": -65.68924713134766, "logps/ref_rejected": -110.24205017089844, "logps/rejected": -2144.77685546875, "loss": 1.3379, "margin_dpo/margin_mean": 812.8150024414062, "margin_dpo/margin_std": 2420.636474609375, "step": 642 }, { "KL/chosen_KL_mean": -1205.4664306640625, "KL/mean": -1394.8968505859375, "KL/rejected_KL_mean": -1584.3272705078125, "KL/std": 1325.056640625, "epoch": 0.9441997063142438, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 378.8608703613281, "fcm_dpo/q_t": 0.4933786988258362, "grad_norm": 5.956773281097412, "learning_rate": 4.993270631642038e-09, "logits/chosen": -0.6313992738723755, "logits/rejected": -0.6435078382492065, "logps/chosen": -1257.41650390625, "logps/ref_chosen": -51.94999694824219, "logps/ref_rejected": -87.46833801269531, "logps/rejected": -1671.795654296875, "loss": 1.3637, "margin_dpo/margin_mean": 378.86090087890625, "margin_dpo/margin_std": 1685.135986328125, "step": 643 }, { "KL/chosen_KL_mean": -1009.8892822265625, "KL/mean": -1339.213623046875, "KL/rejected_KL_mean": -1668.5380859375, "KL/std": 1681.36669921875, "epoch": 0.9456681350954479, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 658.6486206054688, "fcm_dpo/q_t": 0.48869824409484863, "grad_norm": 5.554941654205322, "learning_rate": 4.741290495811873e-09, "logits/chosen": -0.6091523170471191, "logits/rejected": -0.6272720098495483, "logps/chosen": -1068.906982421875, "logps/ref_chosen": -59.017662048339844, "logps/ref_rejected": -87.13668823242188, "logps/rejected": -1755.6746826171875, "loss": 1.3465, "margin_dpo/margin_mean": 658.6486206054688, "margin_dpo/margin_std": 1983.645751953125, "step": 644 }, { "KL/chosen_KL_mean": -1136.2744140625, "KL/mean": -1262.766845703125, "KL/rejected_KL_mean": -1389.25927734375, "KL/std": 1186.4443359375, "epoch": 0.947136563876652, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 252.98484802246094, "fcm_dpo/q_t": 0.49558210372924805, "grad_norm": 8.011646270751953, "learning_rate": 4.495773155069299e-09, "logits/chosen": -0.6204506158828735, "logits/rejected": -0.6305866241455078, "logps/chosen": -1192.150390625, "logps/ref_chosen": -55.87602233886719, "logps/ref_rejected": -97.78080749511719, "logps/rejected": -1487.0400390625, "loss": 1.3723, "margin_dpo/margin_mean": 252.98483276367188, "margin_dpo/margin_std": 1717.4600830078125, "step": 645 }, { "KL/chosen_KL_mean": -990.78173828125, "KL/mean": -1174.6834716796875, "KL/rejected_KL_mean": -1358.585205078125, "KL/std": 1137.93994140625, "epoch": 0.9486049926578561, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 367.80340576171875, "fcm_dpo/q_t": 0.49355489015579224, "grad_norm": 5.103117942810059, "learning_rate": 4.256725079024553e-09, "logits/chosen": -0.6123974919319153, "logits/rejected": -0.6034915447235107, "logps/chosen": -1052.0574951171875, "logps/ref_chosen": -61.275787353515625, "logps/ref_rejected": -77.50580596923828, "logps/rejected": -1436.091064453125, "loss": 1.3629, "margin_dpo/margin_mean": 367.80340576171875, "margin_dpo/margin_std": 1322.474365234375, "step": 646 }, { "KL/chosen_KL_mean": -822.7291259765625, "KL/mean": -1107.7655029296875, "KL/rejected_KL_mean": -1392.802001953125, "KL/std": 1028.25341796875, "epoch": 0.9500734214390602, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 570.0728759765625, "fcm_dpo/q_t": 0.49000051617622375, "grad_norm": 10.267186164855957, "learning_rate": 4.024152566816791e-09, "logits/chosen": -0.5852940678596497, "logits/rejected": -0.6047427654266357, "logps/chosen": -877.58154296875, "logps/ref_chosen": -54.8524169921875, "logps/ref_rejected": -93.5194091796875, "logps/rejected": -1486.3214111328125, "loss": 1.3484, "margin_dpo/margin_mean": 570.0728759765625, "margin_dpo/margin_std": 1177.3486328125, "step": 647 }, { "KL/chosen_KL_mean": -1093.44384765625, "KL/mean": -1523.8251953125, "KL/rejected_KL_mean": -1954.206298828125, "KL/std": 1495.839111328125, "epoch": 0.9515418502202643, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 860.7623291015625, "fcm_dpo/q_t": 0.4849778413772583, "grad_norm": 10.467401504516602, "learning_rate": 3.798061746947995e-09, "logits/chosen": -0.6163946390151978, "logits/rejected": -0.6602287292480469, "logps/chosen": -1147.6153564453125, "logps/ref_chosen": -54.17146682739258, "logps/ref_rejected": -98.7127914428711, "logps/rejected": -2052.9189453125, "loss": 1.3305, "margin_dpo/margin_mean": 860.7623901367188, "margin_dpo/margin_std": 1694.208984375, "step": 648 }, { "KL/chosen_KL_mean": -1002.08935546875, "KL/mean": -1291.5301513671875, "KL/rejected_KL_mean": -1580.9708251953125, "KL/std": 1385.653076171875, "epoch": 0.9530102790014684, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 578.8814697265625, "fcm_dpo/q_t": 0.4899559020996094, "grad_norm": 4.749213218688965, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -0.6620218753814697, "logits/rejected": -0.6723358631134033, "logps/chosen": -1064.5697021484375, "logps/ref_chosen": -62.480350494384766, "logps/ref_rejected": -80.07717895507812, "logps/rejected": -1661.048095703125, "loss": 1.3503, "margin_dpo/margin_mean": 578.8814697265625, "margin_dpo/margin_std": 1804.99658203125, "step": 649 }, { "KL/chosen_KL_mean": -1217.0234375, "KL/mean": -1500.784423828125, "KL/rejected_KL_mean": -1784.54541015625, "KL/std": 1650.326416015625, "epoch": 0.9544787077826725, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 567.52197265625, "fcm_dpo/q_t": 0.4899527430534363, "grad_norm": 4.821526527404785, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -0.6028755903244019, "logits/rejected": -0.6270517706871033, "logps/chosen": -1273.1163330078125, "logps/ref_chosen": -56.09281921386719, "logps/ref_rejected": -98.26483917236328, "logps/rejected": -1882.810302734375, "loss": 1.353, "margin_dpo/margin_mean": 567.52197265625, "margin_dpo/margin_std": 2182.583984375, "step": 650 }, { "KL/chosen_KL_mean": -809.0330200195312, "KL/mean": -1301.23486328125, "KL/rejected_KL_mean": -1793.436767578125, "KL/std": 1311.9166259765625, "epoch": 0.9559471365638766, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 984.4036865234375, "fcm_dpo/q_t": 0.48278218507766724, "grad_norm": 13.06442928314209, "learning_rate": 3.158738163478475e-09, "logits/chosen": -0.647139847278595, "logits/rejected": -0.6998910903930664, "logps/chosen": -852.45849609375, "logps/ref_chosen": -43.42544937133789, "logps/ref_rejected": -99.95791625976562, "logps/rejected": -1893.3946533203125, "loss": 1.3208, "margin_dpo/margin_mean": 984.4036865234375, "margin_dpo/margin_std": 1392.693359375, "step": 651 }, { "KL/chosen_KL_mean": -990.5150756835938, "KL/mean": -1336.54736328125, "KL/rejected_KL_mean": -1682.579833984375, "KL/std": 1426.03564453125, "epoch": 0.9574155653450808, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 692.0648193359375, "fcm_dpo/q_t": 0.4879456162452698, "grad_norm": 13.026046752929688, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -0.6543087959289551, "logits/rejected": -0.6866675019264221, "logps/chosen": -1053.091796875, "logps/ref_chosen": -62.57680892944336, "logps/ref_rejected": -111.76779174804688, "logps/rejected": -1794.34765625, "loss": 1.3424, "margin_dpo/margin_mean": 692.0648193359375, "margin_dpo/margin_std": 1824.699951171875, "step": 652 }, { "KL/chosen_KL_mean": -1127.3262939453125, "KL/mean": -1536.869384765625, "KL/rejected_KL_mean": -1946.41259765625, "KL/std": 1666.6322021484375, "epoch": 0.9588839941262849, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 819.0863647460938, "fcm_dpo/q_t": 0.48590990900993347, "grad_norm": 12.233861923217773, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -0.6801096200942993, "logits/rejected": -0.7179126143455505, "logps/chosen": -1188.439208984375, "logps/ref_chosen": -61.11295700073242, "logps/ref_rejected": -103.24960327148438, "logps/rejected": -2049.662109375, "loss": 1.3351, "margin_dpo/margin_mean": 819.0863647460938, "margin_dpo/margin_std": 2013.414794921875, "step": 653 }, { "KL/chosen_KL_mean": -1065.35546875, "KL/mean": -1342.395263671875, "KL/rejected_KL_mean": -1619.43505859375, "KL/std": 1395.85400390625, "epoch": 0.960352422907489, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 554.0797119140625, "fcm_dpo/q_t": 0.49033522605895996, "grad_norm": 5.433686256408691, "learning_rate": 2.577954022936174e-09, "logits/chosen": -0.6489402651786804, "logits/rejected": -0.6714223623275757, "logps/chosen": -1127.08349609375, "logps/ref_chosen": -61.7281379699707, "logps/ref_rejected": -98.7738037109375, "logps/rejected": -1718.208984375, "loss": 1.3513, "margin_dpo/margin_mean": 554.0797119140625, "margin_dpo/margin_std": 1696.0687255859375, "step": 654 }, { "KL/chosen_KL_mean": -971.6968994140625, "KL/mean": -1341.12060546875, "KL/rejected_KL_mean": -1710.54443359375, "KL/std": 1493.365478515625, "epoch": 0.9618208516886931, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 738.8475341796875, "fcm_dpo/q_t": 0.4871307611465454, "grad_norm": 12.034753799438477, "learning_rate": 2.397392281198729e-09, "logits/chosen": -0.5706413984298706, "logits/rejected": -0.6150772571563721, "logps/chosen": -1021.273681640625, "logps/ref_chosen": -49.576812744140625, "logps/ref_rejected": -98.29183197021484, "logps/rejected": -1808.8363037109375, "loss": 1.339, "margin_dpo/margin_mean": 738.8475341796875, "margin_dpo/margin_std": 1762.6845703125, "step": 655 }, { "KL/chosen_KL_mean": -1052.604736328125, "KL/mean": -1667.0531005859375, "KL/rejected_KL_mean": -2281.50146484375, "KL/std": 1618.218505859375, "epoch": 0.9632892804698973, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1228.896484375, "fcm_dpo/q_t": 0.478542685508728, "grad_norm": 15.938714027404785, "learning_rate": 2.223355098446622e-09, "logits/chosen": -0.5950414538383484, "logits/rejected": -0.6505473852157593, "logps/chosen": -1105.154296875, "logps/ref_chosen": -52.54943084716797, "logps/ref_rejected": -113.67464447021484, "logps/rejected": -2395.176025390625, "loss": 1.3054, "margin_dpo/margin_mean": 1228.896484375, "margin_dpo/margin_std": 1683.515625, "step": 656 }, { "KL/chosen_KL_mean": -1101.297607421875, "KL/mean": -1607.488037109375, "KL/rejected_KL_mean": -2113.678466796875, "KL/std": 1809.890380859375, "epoch": 0.9647577092511013, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1012.3809814453125, "fcm_dpo/q_t": 0.4823700189590454, "grad_norm": 7.817019939422607, "learning_rate": 2.055847060721566e-09, "logits/chosen": -0.6291069984436035, "logits/rejected": -0.6917046308517456, "logps/chosen": -1147.998046875, "logps/ref_chosen": -46.700538635253906, "logps/ref_rejected": -97.91487121582031, "logps/rejected": -2211.59326171875, "loss": 1.3215, "margin_dpo/margin_mean": 1012.3810424804688, "margin_dpo/margin_std": 2023.4796142578125, "step": 657 }, { "KL/chosen_KL_mean": -1130.634033203125, "KL/mean": -1422.20703125, "KL/rejected_KL_mean": -1713.7802734375, "KL/std": 1217.5029296875, "epoch": 0.9662261380323054, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 583.1461791992188, "fcm_dpo/q_t": 0.4897571802139282, "grad_norm": 7.5315752029418945, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -0.5903108716011047, "logits/rejected": -0.6097769737243652, "logps/chosen": -1191.59228515625, "logps/ref_chosen": -60.95820999145508, "logps/ref_rejected": -95.93949127197266, "logps/rejected": -1809.7197265625, "loss": 1.3492, "margin_dpo/margin_mean": 583.1461181640625, "margin_dpo/margin_std": 1597.01708984375, "step": 658 }, { "KL/chosen_KL_mean": -1060.837158203125, "KL/mean": -1350.650146484375, "KL/rejected_KL_mean": -1640.4630126953125, "KL/std": 1334.296142578125, "epoch": 0.9676945668135095, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 579.6257934570312, "fcm_dpo/q_t": 0.48988625407218933, "grad_norm": 6.343628883361816, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -0.6019806861877441, "logits/rejected": -0.5888450145721436, "logps/chosen": -1137.580078125, "logps/ref_chosen": -76.74298095703125, "logps/ref_rejected": -87.4709701538086, "logps/rejected": -1727.9339599609375, "loss": 1.3493, "margin_dpo/margin_mean": 579.6257934570312, "margin_dpo/margin_std": 1626.3175048828125, "step": 659 }, { "KL/chosen_KL_mean": -1123.677490234375, "KL/mean": -1415.98095703125, "KL/rejected_KL_mean": -1708.28466796875, "KL/std": 1241.296630859375, "epoch": 0.9691629955947136, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 584.6071166992188, "fcm_dpo/q_t": 0.4897761940956116, "grad_norm": 15.733861923217773, "learning_rate": 1.592541096695571e-09, "logits/chosen": -0.6595280170440674, "logits/rejected": -0.6711503267288208, "logps/chosen": -1182.725341796875, "logps/ref_chosen": -59.04788589477539, "logps/ref_rejected": -75.96005249023438, "logps/rejected": -1784.24462890625, "loss": 1.3485, "margin_dpo/margin_mean": 584.6071166992188, "margin_dpo/margin_std": 1485.17626953125, "step": 660 }, { "KL/chosen_KL_mean": -902.3294067382812, "KL/mean": -1418.966796875, "KL/rejected_KL_mean": -1935.60400390625, "KL/std": 2257.097412109375, "epoch": 0.9706314243759178, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1033.27490234375, "fcm_dpo/q_t": 0.48256513476371765, "grad_norm": 8.566376686096191, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -0.6667207479476929, "logits/rejected": -0.7094805240631104, "logps/chosen": -953.00341796875, "logps/ref_chosen": -50.673973083496094, "logps/ref_rejected": -86.00569152832031, "logps/rejected": -2021.60986328125, "loss": 1.3242, "margin_dpo/margin_mean": 1033.27490234375, "margin_dpo/margin_std": 2576.364501953125, "step": 661 }, { "KL/chosen_KL_mean": -1044.2470703125, "KL/mean": -1371.6861572265625, "KL/rejected_KL_mean": -1699.1253662109375, "KL/std": 1256.9990234375, "epoch": 0.9720998531571219, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 654.878173828125, "fcm_dpo/q_t": 0.48855116963386536, "grad_norm": 4.699316501617432, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -0.6130234003067017, "logits/rejected": -0.6202554106712341, "logps/chosen": -1113.5081787109375, "logps/ref_chosen": -69.26106262207031, "logps/ref_rejected": -89.05593872070312, "logps/rejected": -1788.1812744140625, "loss": 1.3441, "margin_dpo/margin_mean": 654.8782348632812, "margin_dpo/margin_std": 1609.4761962890625, "step": 662 }, { "KL/chosen_KL_mean": -954.5285034179688, "KL/mean": -1455.802001953125, "KL/rejected_KL_mean": -1957.0755615234375, "KL/std": 1663.22412109375, "epoch": 0.973568281938326, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1002.547119140625, "fcm_dpo/q_t": 0.48265382647514343, "grad_norm": 7.0070013999938965, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -0.6588038206100464, "logits/rejected": -0.7001410126686096, "logps/chosen": -1019.4073486328125, "logps/ref_chosen": -64.87890625, "logps/ref_rejected": -113.92536926269531, "logps/rejected": -2071.0009765625, "loss": 1.3231, "margin_dpo/margin_mean": 1002.5470581054688, "margin_dpo/margin_std": 2189.273193359375, "step": 663 }, { "KL/chosen_KL_mean": -1293.694091796875, "KL/mean": -1853.238525390625, "KL/rejected_KL_mean": -2412.78271484375, "KL/std": 2207.93994140625, "epoch": 0.9750367107195301, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1119.088623046875, "fcm_dpo/q_t": 0.4808417856693268, "grad_norm": 5.824970722198486, "learning_rate": 1.066455926241383e-09, "logits/chosen": -0.6086447238922119, "logits/rejected": -0.6509624123573303, "logps/chosen": -1354.5826416015625, "logps/ref_chosen": -60.88847351074219, "logps/ref_rejected": -105.521728515625, "logps/rejected": -2518.304443359375, "loss": 1.3188, "margin_dpo/margin_mean": 1119.088623046875, "margin_dpo/margin_std": 2782.05810546875, "step": 664 }, { "KL/chosen_KL_mean": -998.0809326171875, "KL/mean": -1378.9786376953125, "KL/rejected_KL_mean": -1759.8763427734375, "KL/std": 1507.407470703125, "epoch": 0.9765051395007343, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 761.79541015625, "fcm_dpo/q_t": 0.4867151081562042, "grad_norm": 8.16478443145752, "learning_rate": 9.513254770636137e-10, "logits/chosen": -0.6453874111175537, "logits/rejected": -0.682604193687439, "logps/chosen": -1058.6451416015625, "logps/ref_chosen": -60.56413269042969, "logps/ref_rejected": -84.80882263183594, "logps/rejected": -1844.68505859375, "loss": 1.3374, "margin_dpo/margin_mean": 761.79541015625, "margin_dpo/margin_std": 1707.666748046875, "step": 665 }, { "KL/chosen_KL_mean": -1018.7044067382812, "KL/mean": -1426.4305419921875, "KL/rejected_KL_mean": -1834.1566162109375, "KL/std": 1435.4952392578125, "epoch": 0.9779735682819384, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 815.4522705078125, "fcm_dpo/q_t": 0.48577576875686646, "grad_norm": 5.858867645263672, "learning_rate": 8.427576920763956e-10, "logits/chosen": -0.6097148060798645, "logits/rejected": -0.6266220808029175, "logps/chosen": -1083.124267578125, "logps/ref_chosen": -64.41996002197266, "logps/ref_rejected": -95.8916244506836, "logps/rejected": -1930.04833984375, "loss": 1.3337, "margin_dpo/margin_mean": 815.4522705078125, "margin_dpo/margin_std": 1725.0411376953125, "step": 666 }, { "KL/chosen_KL_mean": -1350.70068359375, "KL/mean": -1672.00048828125, "KL/rejected_KL_mean": -1993.3004150390625, "KL/std": 1688.798583984375, "epoch": 0.9794419970631424, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 642.599853515625, "fcm_dpo/q_t": 0.4888229966163635, "grad_norm": 10.685932159423828, "learning_rate": 7.407554321417764e-10, "logits/chosen": -0.5980401039123535, "logits/rejected": -0.6029761433601379, "logps/chosen": -1419.9775390625, "logps/ref_chosen": -69.27702331542969, "logps/ref_rejected": -87.83549499511719, "logps/rejected": -2081.1357421875, "loss": 1.3471, "margin_dpo/margin_mean": 642.599853515625, "margin_dpo/margin_std": 2042.0830078125, "step": 667 }, { "KL/chosen_KL_mean": -1268.402099609375, "KL/mean": -1488.158935546875, "KL/rejected_KL_mean": -1707.915771484375, "KL/std": 1302.4078369140625, "epoch": 0.9809104258443465, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 439.513671875, "fcm_dpo/q_t": 0.4923250377178192, "grad_norm": 15.58211898803711, "learning_rate": 6.453213851142225e-10, "logits/chosen": -0.6111631989479065, "logits/rejected": -0.6139740943908691, "logps/chosen": -1341.006103515625, "logps/ref_chosen": -72.60400390625, "logps/ref_rejected": -103.73905944824219, "logps/rejected": -1811.65478515625, "loss": 1.3589, "margin_dpo/margin_mean": 439.5137023925781, "margin_dpo/margin_std": 1599.6151123046875, "step": 668 }, { "KL/chosen_KL_mean": -872.9949951171875, "KL/mean": -1229.5106201171875, "KL/rejected_KL_mean": -1586.026123046875, "KL/std": 1136.226318359375, "epoch": 0.9823788546255506, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 713.03125, "fcm_dpo/q_t": 0.48755186796188354, "grad_norm": 4.998262882232666, "learning_rate": 5.564580657695939e-10, "logits/chosen": -0.6233928203582764, "logits/rejected": -0.6421560049057007, "logps/chosen": -919.1114501953125, "logps/ref_chosen": -46.116416931152344, "logps/ref_rejected": -77.92434692382812, "logps/rejected": -1663.9505615234375, "loss": 1.3394, "margin_dpo/margin_mean": 713.0311889648438, "margin_dpo/margin_std": 1437.951904296875, "step": 669 }, { "KL/chosen_KL_mean": -1085.6097412109375, "KL/mean": -1610.455810546875, "KL/rejected_KL_mean": -2135.3017578125, "KL/std": 1911.545166015625, "epoch": 0.9838472834067548, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1049.692138671875, "fcm_dpo/q_t": 0.48183974623680115, "grad_norm": 7.568104267120361, "learning_rate": 4.741678157389739e-10, "logits/chosen": -0.6080718040466309, "logits/rejected": -0.6510467529296875, "logps/chosen": -1147.9554443359375, "logps/ref_chosen": -62.34575271606445, "logps/ref_rejected": -96.9405517578125, "logps/rejected": -2232.242431640625, "loss": 1.3207, "margin_dpo/margin_mean": 1049.692138671875, "margin_dpo/margin_std": 2298.290283203125, "step": 670 }, { "KL/chosen_KL_mean": -1065.5369873046875, "KL/mean": -1437.20361328125, "KL/rejected_KL_mean": -1808.870361328125, "KL/std": 1385.409423828125, "epoch": 0.9853157121879589, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 743.3333129882812, "fcm_dpo/q_t": 0.4870268702507019, "grad_norm": 8.674588203430176, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -0.6010315418243408, "logits/rejected": -0.6403799057006836, "logps/chosen": -1113.537109375, "logps/ref_chosen": -48.00010681152344, "logps/ref_rejected": -83.81932067871094, "logps/rejected": -1892.689697265625, "loss": 1.3395, "margin_dpo/margin_mean": 743.3333740234375, "margin_dpo/margin_std": 1940.243896484375, "step": 671 }, { "KL/chosen_KL_mean": -1323.4586181640625, "KL/mean": -1571.2021484375, "KL/rejected_KL_mean": -1818.945556640625, "KL/std": 1406.629150390625, "epoch": 0.986784140969163, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 495.4870910644531, "fcm_dpo/q_t": 0.4913468062877655, "grad_norm": 15.005202293395996, "learning_rate": 3.293150240547549e-10, "logits/chosen": -0.6775461435317993, "logits/rejected": -0.6908239126205444, "logps/chosen": -1382.0418701171875, "logps/ref_chosen": -58.58328628540039, "logps/ref_rejected": -93.14015197753906, "logps/rejected": -1912.085693359375, "loss": 1.3548, "margin_dpo/margin_mean": 495.4870300292969, "margin_dpo/margin_std": 1540.307373046875, "step": 672 }, { "KL/chosen_KL_mean": -1022.36083984375, "KL/mean": -1316.231689453125, "KL/rejected_KL_mean": -1610.1025390625, "KL/std": 1145.38427734375, "epoch": 0.9882525697503671, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 587.74169921875, "fcm_dpo/q_t": 0.4897216856479645, "grad_norm": 6.545974254608154, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -0.6184306740760803, "logits/rejected": -0.6311053037643433, "logps/chosen": -1069.083984375, "logps/ref_chosen": -46.72320556640625, "logps/ref_rejected": -85.29623413085938, "logps/rejected": -1695.3988037109375, "loss": 1.3476, "margin_dpo/margin_mean": 587.74169921875, "margin_dpo/margin_std": 1306.4228515625, "step": 673 }, { "KL/chosen_KL_mean": -847.9764404296875, "KL/mean": -1195.31689453125, "KL/rejected_KL_mean": -1542.6575927734375, "KL/std": 1038.550048828125, "epoch": 0.9897209985315712, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 694.6810302734375, "fcm_dpo/q_t": 0.4878394603729248, "grad_norm": 13.266809463500977, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -0.6065397262573242, "logits/rejected": -0.6238285303115845, "logps/chosen": -893.4219970703125, "logps/ref_chosen": -45.445526123046875, "logps/ref_rejected": -70.04593658447266, "logps/rejected": -1612.7034912109375, "loss": 1.3403, "margin_dpo/margin_mean": 694.6810302734375, "margin_dpo/margin_std": 1318.276611328125, "step": 674 }, { "KL/chosen_KL_mean": -1095.146484375, "KL/mean": -1370.689697265625, "KL/rejected_KL_mean": -1646.23291015625, "KL/std": 1590.983154296875, "epoch": 0.9911894273127754, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 551.0866088867188, "fcm_dpo/q_t": 0.4904383718967438, "grad_norm": 9.25955867767334, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -0.6318482756614685, "logits/rejected": -0.6467227935791016, "logps/chosen": -1139.32275390625, "logps/ref_chosen": -44.17628479003906, "logps/ref_rejected": -74.09197998046875, "logps/rejected": -1720.324951171875, "loss": 1.3534, "margin_dpo/margin_mean": 551.0866088867188, "margin_dpo/margin_std": 2041.986083984375, "step": 675 }, { "KL/chosen_KL_mean": -1177.5093994140625, "KL/mean": -1528.5595703125, "KL/rejected_KL_mean": -1879.60986328125, "KL/std": 1375.15185546875, "epoch": 0.9926578560939795, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 702.1004638671875, "fcm_dpo/q_t": 0.4877719581127167, "grad_norm": 5.452136516571045, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -0.6294345259666443, "logits/rejected": -0.6576676964759827, "logps/chosen": -1248.907958984375, "logps/ref_chosen": -71.39852905273438, "logps/ref_rejected": -88.3587646484375, "logps/rejected": -1967.968505859375, "loss": 1.3415, "margin_dpo/margin_mean": 702.1004028320312, "margin_dpo/margin_std": 1760.55029296875, "step": 676 }, { "KL/chosen_KL_mean": -1092.815185546875, "KL/mean": -1425.9197998046875, "KL/rejected_KL_mean": -1759.0244140625, "KL/std": 1277.3245849609375, "epoch": 0.9941262848751835, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 666.209228515625, "fcm_dpo/q_t": 0.4884171485900879, "grad_norm": 10.614191055297852, "learning_rate": 8.23423165278725e-11, "logits/chosen": -0.6302313804626465, "logits/rejected": -0.6473731994628906, "logps/chosen": -1149.342529296875, "logps/ref_chosen": -56.527435302734375, "logps/ref_rejected": -78.22654724121094, "logps/rejected": -1837.2509765625, "loss": 1.3443, "margin_dpo/margin_mean": 666.209228515625, "margin_dpo/margin_std": 1806.208251953125, "step": 677 }, { "KL/chosen_KL_mean": -861.671630859375, "KL/mean": -1364.0057373046875, "KL/rejected_KL_mean": -1866.33984375, "KL/std": 1516.4210205078125, "epoch": 0.9955947136563876, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1004.6680908203125, "fcm_dpo/q_t": 0.48245441913604736, "grad_norm": 14.39037036895752, "learning_rate": 5.270012410216185e-11, "logits/chosen": -0.6077277064323425, "logits/rejected": -0.6513392925262451, "logps/chosen": -907.80615234375, "logps/ref_chosen": -46.13447570800781, "logps/ref_rejected": -80.60462951660156, "logps/rejected": -1946.9443359375, "loss": 1.3211, "margin_dpo/margin_mean": 1004.6680908203125, "margin_dpo/margin_std": 1805.224609375, "step": 678 }, { "KL/chosen_KL_mean": -1052.888671875, "KL/mean": -1298.56982421875, "KL/rejected_KL_mean": -1544.251220703125, "KL/std": 1319.9560546875, "epoch": 0.9970631424375918, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 491.36260986328125, "fcm_dpo/q_t": 0.4915056824684143, "grad_norm": 5.045799732208252, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -0.6154893636703491, "logits/rejected": -0.624566912651062, "logps/chosen": -1103.18359375, "logps/ref_chosen": -50.294921875, "logps/ref_rejected": -76.59813690185547, "logps/rejected": -1620.849365234375, "loss": 1.356, "margin_dpo/margin_mean": 491.36260986328125, "margin_dpo/margin_std": 1768.9346923828125, "step": 679 }, { "KL/chosen_KL_mean": -1185.60595703125, "KL/mean": -1536.31689453125, "KL/rejected_KL_mean": -1887.0277099609375, "KL/std": 1552.267822265625, "epoch": 0.9985315712187959, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 701.421630859375, "fcm_dpo/q_t": 0.4878092408180237, "grad_norm": 5.114809989929199, "learning_rate": 1.31753782067201e-11, "logits/chosen": -0.6257789731025696, "logits/rejected": -0.6503081321716309, "logps/chosen": -1262.521728515625, "logps/ref_chosen": -76.91569519042969, "logps/ref_rejected": -112.384765625, "logps/rejected": -1999.4124755859375, "loss": 1.3422, "margin_dpo/margin_mean": 701.421630859375, "margin_dpo/margin_std": 1905.763427734375, "step": 680 }, { "KL/chosen_KL_mean": -1252.8399658203125, "KL/mean": -1509.438720703125, "KL/rejected_KL_mean": -1766.037353515625, "KL/std": 1613.639404296875, "epoch": 1.0, "fcm_dpo/beta": 7.018564065219834e-05, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 513.1973876953125, "fcm_dpo/q_t": 0.49122941493988037, "grad_norm": 13.061714172363281, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -0.6694597005844116, "logits/rejected": -0.6893618702888489, "logps/chosen": -1313.7972412109375, "logps/ref_chosen": -60.957279205322266, "logps/ref_rejected": -88.55797576904297, "logps/rejected": -1854.5953369140625, "loss": 1.3573, "margin_dpo/margin_mean": 513.1973876953125, "margin_dpo/margin_std": 2175.52685546875, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.3263754091949014, "train_runtime": 1741.0925, "train_samples_per_second": 25.041, "train_steps_per_second": 0.391 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }